This patch is being sent for use in the various Rust GPU drivers that are under development. It provides the additional feature of work items that are executed after a delay. The design of the existing workqueue is rather extensible, as most of the logic is reused for delayed work items even though a different work item type is required. The new logic consists of: * A new DelayedWork struct that wraps struct delayed_work. * A new impl_has_delayed_work! macro that provides adjusted versions of the container_of logic, that is suitable with delayed work items. * A `enqueue_delayed` method that can enqueue a delayed work item. This patch does *not* rely on the fact that `struct delayed_work` contains `struct work_struct` at offset zero. It will continue to work even if the layout is changed to hold the `work` field at a different offset. Please see the example introduced at the top of the file for example usage of delayed work items.
Unified · Split
| @@ -131,10 +131,69 @@ · 130 unmodified lines | |||
| 131 | //! # print_2_later(MyStruct::new(41, 42).unwrap()); | 131 | //! # print_2_later(MyStruct::new(41, 42).unwrap()); |
| 132 | //! ``` | 132 | //! ``` |
| 133 | //! | 133 | //! |
| 134 | //! This example shows how you can schedule delayed work items: | ||
| 135 | //! | ||
| 136 | //! ``` | ||
| 137 | //! use kernel::sync::Arc; | ||
| 138 | //! use kernel::workqueue::{self, impl_has_delayed_work, new_delayed_work, DelayedWork, WorkItem}; | ||
| 139 | //! | ||
| 140 | //! #[pin_data] | ||
| 141 | //! struct MyStruct { | ||
| 142 | //! value: i32, | ||
| 143 | //! #[pin] | ||
| 144 | //! work: DelayedWork<MyStruct>, | ||
| 145 | //! } | ||
| 146 | //! | ||
| 147 | //! impl_has_delayed_work! { | ||
| 148 | //! impl HasDelayedWork<Self> for MyStruct { self.work } | ||
| 149 | //! } | ||
| 150 | //! | ||
| 151 | //! impl MyStruct { | ||
| 152 | //! fn new(value: i32) -> Result<Arc<Self>> { | ||
| 153 | //! Arc::pin_init( | ||
| 154 | //! pin_init!(MyStruct { | ||
| 155 | //! value, | ||
| 156 | //! work <- new_delayed_work!("MyStruct::work"), | ||
| 157 | //! }), | ||
| 158 | //! GFP_KERNEL, | ||
| 159 | //! ) | ||
| 160 | //! } | ||
| 161 | //! } | ||
| 162 | //! | ||
| 163 | //! impl WorkItem for MyStruct { | ||
| 164 | //! type Pointer = Arc<MyStruct>; | ||
| 165 | //! | ||
| 166 | //! fn run(this: Arc<MyStruct>) { | ||
| 167 | //! pr_info!("The value is: {}\n", this.value); | ||
| 168 | //! } | ||
| 169 | //! } | ||
| 170 | //! | ||
| 171 | //! /// This method will enqueue the struct for execution on the system workqueue, where its value | ||
| 172 | //! /// will be printed 12 jiffies later. | ||
| 173 | //! fn print_later(val: Arc<MyStruct>) { | ||
| 174 | //! let _ = workqueue::system().enqueue_delayed(val, 12); | ||
| 175 | //! } | ||
| 176 | //! | ||
| 177 | //! /// It is also possible to use the ordinary `enqueue` method together with `DelayedWork`. This | ||
| 178 | //! /// is equivalent to calling `enqueue_delayed` with a delay of zero. | ||
| 179 | //! fn print_now(val: Arc<MyStruct>) { | ||
| 180 | //! let _ = workqueue::system().enqueue(val); | ||
| 181 | //! } | ||
| 182 | //! # print_later(MyStruct::new(42).unwrap()); | ||
| 183 | //! # print_now(MyStruct::new(42).unwrap()); | ||
| 184 | //! ``` | ||
| 185 | //! | ||
| 134 | //! C header: [`include/linux/workqueue.h`](srctree/include/linux/workqueue.h) | 186 | //! C header: [`include/linux/workqueue.h`](srctree/include/linux/workqueue.h) |
| 135 | 187 | ||
| 136 | use crate::alloc::{AllocError, Flags}; | 188 | use crate::{ |
| 137 | use crate::{prelude::*, sync::Arc, sync::LockClassKey, types::Opaque}; | 189 | alloc::{AllocError, Flags}, |
| 190 | container_of, | ||
| 191 | prelude::*, | ||
| 192 | sync::Arc, | ||
| 193 | sync::LockClassKey, | ||
| 194 | time::Jiffies, | ||
| 195 | types::Opaque, | ||
| 196 | }; | ||
| 138 | use core::marker::PhantomData; | 197 | use core::marker::PhantomData; |
| 139 | 198 | ||
| 140 | /// Creates a [`Work`] initialiser with the given name and a newly-created lock class. | 199 | /// Creates a [`Work`] initialiser with the given name and a newly-created lock class. |
| @@ -146,6 +205,33 @@ macro_rules! new_work { · 5 unmodified lines | |||
| 146 | } | 205 | } |
| 147 | pub use new_work; | 206 | pub use new_work; |
| 148 | 207 | ||
| 208 | /// Creates a [`DelayedWork`] initialiser with the given name and a newly-created lock class. | ||
| 209 | #[macro_export] | ||
| 210 | macro_rules! new_delayed_work { | ||
| 211 | () => { | ||
| 212 | $crate::workqueue::DelayedWork::new( | ||
| 213 | $crate::optional_name!(), | ||
| 214 | $crate::static_lock_class!(), | ||
| 215 | $crate::c_str!(::core::concat!( | ||
| 216 | ::core::file!(), | ||
| 217 | ":", | ||
| 218 | ::core::line!(), | ||
| 219 | "_timer" | ||
| 220 | )), | ||
| 221 | $crate::static_lock_class!(), | ||
| 222 | ) | ||
| 223 | }; | ||
| 224 | ($name:literal) => { | ||
| 225 | $crate::workqueue::DelayedWork::new( | ||
| 226 | $crate::c_str!($name), | ||
| 227 | $crate::static_lock_class!(), | ||
| 228 | $crate::c_str!(::core::concat!($name, "_timer")), | ||
| 229 | $crate::static_lock_class!(), | ||
| 230 | ) | ||
| 231 | }; | ||
| 232 | } | ||
| 233 | pub use new_delayed_work; | ||
| 234 | |||
| 149 | /// A kernel work queue. | 235 | /// A kernel work queue. |
| 150 | /// | 236 | /// |
| 151 | /// Wraps the kernel's C `struct workqueue_struct`. | 237 | /// Wraps the kernel's C `struct workqueue_struct`. |
| @@ -206,6 +292,42 @@ impl Queue { · 54 unmodified lines | |||
| 206 | } | 292 | } |
| 207 | } | 293 | } |
| 208 | 294 | ||
| 295 | /// Enqueues a delayed work item. | ||
| 296 | /// | ||
| 297 | /// This may fail if the work item is already enqueued in a workqueue. | ||
| 298 | /// | ||
| 299 | /// The work item will be submitted using `WORK_CPU_UNBOUND`. | ||
| 300 | pub fn enqueue_delayed<W, const ID: u64>(&self, w: W, delay: Jiffies) -> W::EnqueueOutput | ||
| 301 | where | ||
| 302 | W: RawDelayedWorkItem<ID> + Send + 'static, | ||
| 303 | { | ||
| 304 | let queue_ptr = self.0.get(); | ||
| 305 | |||
| 306 | // SAFETY: We only return `false` if the `work_struct` is already in a workqueue. The other | ||
| 307 | // `__enqueue` requirements are not relevant since `W` is `Send` and static. | ||
| 308 | // | ||
| 309 | // The call to `bindings::queue_delayed_work_on` will dereference the provided raw pointer, | ||
| 310 | // which is ok because `__enqueue` guarantees that the pointer is valid for the duration of | ||
| 311 | // this closure, and the safety requirements of `RawDelayedWorkItem` expands this | ||
| 312 | // requirement to apply to the entire `delayed_work`. | ||
| 313 | // | ||
| 314 | // Furthermore, if the C workqueue code accesses the pointer after this call to | ||
| 315 | // `__enqueue`, then the work item was successfully enqueued, and | ||
| 316 | // `bindings::queue_delayed_work_on` will have returned true. In this case, `__enqueue` | ||
| 317 | // promises that the raw pointer will stay valid until we call the function pointer in the | ||
| 318 | // `work_struct`, so the access is ok. | ||
| 319 | unsafe { | ||
| 320 | w.__enqueue(move |work_ptr| { | ||
| 321 | bindings::queue_delayed_work_on( | ||
| 322 | bindings::wq_misc_consts_WORK_CPU_UNBOUND as ffi::c_int, | ||
| 323 | queue_ptr, | ||
| 324 | container_of!(work_ptr, bindings::delayed_work, work), | ||
| 325 | delay, | ||
| 326 | ) | ||
| 327 | }) | ||
| 328 | } | ||
| 329 | } | ||
| 330 | |||
| 209 | /// Tries to spawn the given function or closure as a work item. | 331 | /// Tries to spawn the given function or closure as a work item. |
| 210 | /// | 332 | /// |
| 211 | /// This method can fail because it allocates memory to store the work item. | 333 | /// This method can fail because it allocates memory to store the work item. |
| @@ -298,6 +420,16 @@ pub unsafe trait RawWorkItem<const ID: u64> { · 86 unmodified lines | |||
| 298 | F: FnOnce(*mut bindings::work_struct) -> bool; | 420 | F: FnOnce(*mut bindings::work_struct) -> bool; |
| 299 | } | 421 | } |
| 300 | 422 | ||
| 423 | /// A raw delayed work item. | ||
| 424 | /// | ||
| 425 | /// # Safety | ||
| 426 | /// | ||
| 427 | /// If the `__enqueue` method in the `RawWorkItem` implementation calls the closure, then the | ||
| 428 | /// provided pointer must point at the `work` field of a valid `delayed_work`, and the guarantees | ||
| 429 | /// that `__enqueue` provides about accessing the `work_struct` must also apply to the rest of the | ||
| 430 | /// `delayed_work` struct. | ||
| 431 | pub unsafe trait RawDelayedWorkItem<const ID: u64>: RawWorkItem<ID> {} | ||
| 432 | |||
| 301 | /// Defines the method that should be called directly when a work item is executed. | 433 | /// Defines the method that should be called directly when a work item is executed. |
| 302 | /// | 434 | /// |
| 303 | /// This trait is implemented by `Pin<KBox<T>>` and [`Arc<T>`], and is mainly intended to be | 435 | /// This trait is implemented by `Pin<KBox<T>>` and [`Arc<T>`], and is mainly intended to be |
| @@ -407,7 +539,7 @@ impl<T: ?Sized, const ID: u64> Work<T, ID> { · 103 unmodified lines | |||
| 407 | } | 539 | } |
| 408 | } | 540 | } |
| 409 | 541 | ||
| 410 | /// Declares that a type has a [`Work<T, ID>`] field. | 542 | /// Declares that a type contains a [`Work<T, ID>`]. |
| 411 | /// | 543 | /// |
| 412 | /// The intended way of using this trait is via the [`impl_has_work!`] macro. You can use the macro | 544 | /// The intended way of using this trait is via the [`impl_has_work!`] macro. You can use the macro |
| 413 | /// like this: | 545 | /// like this: |
| @@ -506,6 +638,178 @@ impl_has_work! { · 92 unmodified lines | |||
| 506 | impl{T} HasWork<Self> for ClosureWork<T> { self.work } | 638 | impl{T} HasWork<Self> for ClosureWork<T> { self.work } |
| 507 | } | 639 | } |
| 508 | 640 | ||
| 641 | /// Links for a delayed work item. | ||
| 642 | /// | ||
| 643 | /// This struct contains a function pointer to the [`run`] function from the [`WorkItemPointer`] | ||
| 644 | /// trait, and defines the linked list pointers necessary to enqueue a work item in a workqueue in | ||
| 645 | /// a delayed manner. | ||
| 646 | /// | ||
| 647 | /// Wraps the kernel's C `struct delayed_work`. | ||
| 648 | /// | ||
| 649 | /// This is a helper type used to associate a `delayed_work` with the [`WorkItem`] that uses it. | ||
| 650 | /// | ||
| 651 | /// [`run`]: WorkItemPointer::run | ||
| 652 | #[pin_data] | ||
| 653 | #[repr(transparent)] | ||
| 654 | pub struct DelayedWork<T: ?Sized, const ID: u64 = 0> { | ||
| 655 | #[pin] | ||
| 656 | dwork: Opaque<bindings::delayed_work>, | ||
| 657 | _inner: PhantomData<T>, | ||
| 658 | } | ||
| 659 | |||
| 660 | // SAFETY: Kernel work items are usable from any thread. | ||
| 661 | // | ||
| 662 | // We do not need to constrain `T` since the work item does not actually contain a `T`. | ||
| 663 | unsafe impl<T: ?Sized, const ID: u64> Send for DelayedWork<T, ID> {} | ||
| 664 | // SAFETY: Kernel work items are usable from any thread. | ||
| 665 | // | ||
| 666 | // We do not need to constrain `T` since the work item does not actually contain a `T`. | ||
| 667 | unsafe impl<T: ?Sized, const ID: u64> Sync for DelayedWork<T, ID> {} | ||
| 668 | |||
| 669 | impl<T: ?Sized, const ID: u64> DelayedWork<T, ID> { | ||
| 670 | /// Creates a new instance of [`DelayedWork`]. | ||
| 671 | #[inline] | ||
| 672 | pub fn new( | ||
| 673 | work_name: &'static CStr, | ||
| 674 | work_key: Pin<&'static LockClassKey>, | ||
| 675 | timer_name: &'static CStr, | ||
| 676 | timer_key: Pin<&'static LockClassKey>, | ||
| 677 | ) -> impl PinInit<Self> | ||
| 678 | where | ||
| 679 | T: WorkItem<ID>, | ||
| 680 | { | ||
| 681 | pin_init!(Self { | ||
| 682 | dwork <- Opaque::ffi_init(|slot: *mut bindings::delayed_work| { | ||
| 683 | // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as | ||
| 684 | // the work item function. | ||
| 685 | unsafe { | ||
| 686 | bindings::init_work_with_key( | ||
| 687 | core::ptr::addr_of_mut!((*slot).work), | ||
| 688 | Some(T::Pointer::run), | ||
| 689 | false, | ||
| 690 | work_name.as_char_ptr(), | ||
| 691 | work_key.as_ptr(), | ||
| 692 | ) | ||
| 693 | } | ||
| 694 | |||
| 695 | // SAFETY: The `delayed_work_timer_fn` function pointer can be used here because | ||
| 696 | // the timer is embedded in a `struct delayed_work`, and only ever scheduled via | ||
| 697 | // the core workqueue code, and configured to run in irqsafe context. | ||
| 698 | unsafe { | ||
| 699 | bindings::timer_init_key( | ||
| 700 | core::ptr::addr_of_mut!((*slot).timer), | ||
| 701 | Some(bindings::delayed_work_timer_fn), | ||
| 702 | bindings::TIMER_IRQSAFE, | ||
| 703 | timer_name.as_char_ptr(), | ||
| 704 | timer_key.as_ptr(), | ||
| 705 | ) | ||
| 706 | } | ||
| 707 | }), | ||
| 708 | _inner: PhantomData, | ||
| 709 | }) | ||
| 710 | } | ||
| 711 | |||
| 712 | /// Get a pointer to the inner `delayed_work`. | ||
| 713 | /// | ||
| 714 | /// # Safety | ||
| 715 | /// | ||
| 716 | /// The provided pointer must not be dangling and must be properly aligned. (But the memory | ||
| 717 | /// need not be initialized.) | ||
| 718 | #[inline] | ||
| 719 | pub unsafe fn raw_as_work(ptr: *const Self) -> *mut Work<T, ID> { | ||
| 720 | // SAFETY: The caller promises that the pointer is aligned and not dangling. | ||
| 721 | let dw: *mut bindings::delayed_work = | ||
| 722 | unsafe { Opaque::cast_into(core::ptr::addr_of!((*ptr).dwork)) }; | ||
| 723 | // SAFETY: The caller promises that the pointer is aligned and not dangling. | ||
| 724 | let wrk: *mut bindings::work_struct = unsafe { core::ptr::addr_of_mut!((*dw).work) }; | ||
| 725 | // CAST: Work and work_struct have compatible layouts. | ||
| 726 | wrk.cast() | ||
| 727 | } | ||
| 728 | } | ||
| 729 | |||
| 730 | /// Declares that a type contains a [`DelayedWork<T, ID>`]. | ||
| 731 | /// | ||
| 732 | /// # Safety | ||
| 733 | /// | ||
| 734 | /// The `HasWork<T, ID>` implementation must return a `work_struct` that is stored in the `work` | ||
| 735 | /// field of a `delayed_work` with the same access rules as the `work_struct`. | ||
| 736 | pub unsafe trait HasDelayedWork<T, const ID: u64 = 0>: HasWork<T, ID> {} | ||
| 737 | |||
| 738 | /// Used to safely implement the [`HasDelayedWork<T, ID>`] trait. | ||
| 739 | /// | ||
| 740 | /// This macro also implements the [`HasWork`] trait, so you do not need to use [`impl_has_work!`] | ||
| 741 | /// when using this macro. | ||
| 742 | /// | ||
| 743 | /// # Examples | ||
| 744 | /// | ||
| 745 | /// ``` | ||
| 746 | /// use kernel::sync::Arc; | ||
| 747 | /// use kernel::workqueue::{self, impl_has_delayed_work, DelayedWork}; | ||
| 748 | /// | ||
| 749 | /// struct MyStruct<'a, T, const N: usize> { | ||
| 750 | /// work_field: DelayedWork<MyStruct<'a, T, N>, 17>, | ||
| 751 | /// f: fn(&'a [T; N]), | ||
| 752 | /// } | ||
| 753 | /// | ||
| 754 | /// impl_has_delayed_work! { | ||
| 755 | /// impl{'a, T, const N: usize} HasDelayedWork<MyStruct<'a, T, N>, 17> | ||
| 756 | /// for MyStruct<'a, T, N> { self.work_field } | ||
| 757 | /// } | ||
| 758 | /// ``` | ||
| 759 | #[macro_export] | ||
| 760 | macro_rules! impl_has_delayed_work { | ||
| 761 | ($(impl$({$($generics:tt)*})? | ||
| 762 | HasDelayedWork<$work_type:ty $(, $id:tt)?> | ||
| 763 | for $self:ty | ||
| 764 | { self.$field:ident } | ||
| 765 | )*) => {$( | ||
| 766 | // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right | ||
| 767 | // type. | ||
| 768 | unsafe impl$(<$($generics)+>)? | ||
| 769 | $crate::workqueue::HasDelayedWork<$work_type $(, $id)?> for $self {} | ||
| 770 | |||
| 771 | // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right | ||
| 772 | // type. | ||
| 773 | unsafe impl$(<$($generics)+>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self { | ||
| 774 | #[inline] | ||
| 775 | unsafe fn raw_get_work( | ||
| 776 | ptr: *mut Self | ||
| 777 | ) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> { | ||
| 778 | // SAFETY: The caller promises that the pointer is not dangling. | ||
| 779 | let ptr: *mut $crate::workqueue::DelayedWork<$work_type $(, $id)?> = unsafe { | ||
| 780 | ::core::ptr::addr_of_mut!((*ptr).$field) | ||
| 781 | }; | ||
| 782 | |||
| 783 | // SAFETY: The caller promises that the pointer is not dangling. | ||
| 784 | unsafe { $crate::workqueue::DelayedWork::raw_as_work(ptr) } | ||
| 785 | } | ||
| 786 | |||
| 787 | #[inline] | ||
| 788 | unsafe fn work_container_of( | ||
| 789 | ptr: *mut $crate::workqueue::Work<$work_type $(, $id)?>, | ||
| 790 | ) -> *mut Self { | ||
| 791 | // SAFETY: The caller promises that the pointer points at a field of the right type | ||
| 792 | // in the right kind of struct. | ||
| 793 | let ptr = unsafe { $crate::workqueue::Work::raw_get(ptr) }; | ||
| 794 | |||
| 795 | // SAFETY: The caller promises that the pointer points at a field of the right type | ||
| 796 | // in the right kind of struct. | ||
| 797 | let delayed_work = unsafe { | ||
| 798 | $crate::container_of!(ptr, $crate::bindings::delayed_work, work) | ||
| 799 | }; | ||
| 800 | |||
| 801 | let delayed_work: *mut $crate::workqueue::DelayedWork<$work_type $(, $id)?> = | ||
| 802 | delayed_work.cast(); | ||
| 803 | |||
| 804 | // SAFETY: The caller promises that the pointer points at a field of the right type | ||
| 805 | // in the right kind of struct. | ||
| 806 | unsafe { $crate::container_of!(delayed_work, Self, $field) } | ||
| 807 | } | ||
| 808 | } | ||
| 809 | )*}; | ||
| 810 | } | ||
| 811 | pub use impl_has_delayed_work; | ||
| 812 | |||
| 509 | // SAFETY: The `__enqueue` implementation in RawWorkItem uses a `work_struct` initialized with the | 813 | // SAFETY: The `__enqueue` implementation in RawWorkItem uses a `work_struct` initialized with the |
| 510 | // `run` method of this trait as the function pointer because: | 814 | // `run` method of this trait as the function pointer because: |
| 511 | // - `__enqueue` gets the `work_struct` from the `Work` field, using `T::raw_get_work`. | 815 | // - `__enqueue` gets the `work_struct` from the `Work` field, using `T::raw_get_work`. |
| @@ -567,6 +871,16 @@ where · 55 unmodified lines | |||
| 567 | } | 871 | } |
| 568 | } | 872 | } |
| 569 | 873 | ||
| 874 | // SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in | ||
| 875 | // `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of | ||
| 876 | // the `delayed_work` has the same access rules as its `work` field. | ||
| 877 | unsafe impl<T, const ID: u64> RawDelayedWorkItem<ID> for Arc<T> | ||
| 878 | where | ||
| 879 | T: WorkItem<ID, Pointer = Self>, | ||
| 880 | T: HasDelayedWork<T, ID>, | ||
| 881 | { | ||
| 882 | } | ||
| 883 | |||
| 570 | // SAFETY: TODO. | 884 | // SAFETY: TODO. |
| 571 | unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Pin<KBox<T>> | 885 | unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Pin<KBox<T>> |
| 572 | where | 886 | where |
| @@ -617,6 +931,16 @@ where · 44 unmodified lines | |||
| 617 | } | 931 | } |
| 618 | } | 932 | } |
| 619 | 933 | ||
| 934 | // SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in | ||
| 935 | // `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of | ||
| 936 | // the `delayed_work` has the same access rules as its `work` field. | ||
| 937 | unsafe impl<T, const ID: u64> RawDelayedWorkItem<ID> for Pin<KBox<T>> | ||
| 938 | where | ||
| 939 | T: WorkItem<ID, Pointer = Self>, | ||
| 940 | T: HasDelayedWork<T, ID>, | ||
| 941 | { | ||
| 942 | } | ||
| 943 | |||
| 620 | /// Returns the system work queue (`system_wq`). | 944 | /// Returns the system work queue (`system_wq`). |
| 621 | /// | 945 | /// |
| 622 | /// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are | 946 | /// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are |