This patch is being sent for use in the various Rust GPU drivers that are under development. It provides the additional feature of work items that are executed after a delay. The design of the existing workqueue is rather extensible, as most of the logic is reused for delayed work items even though a different work item type is required. The new logic consists of: * A new DelayedWork struct that wraps struct delayed_work. * A new impl_has_delayed_work! macro that provides adjusted versions of the container_of logic, that is suitable with delayed work items. * A `enqueue_delayed` method that can enqueue a delayed work item. This patch does *not* rely on the fact that `struct delayed_work` contains `struct work_struct` at offset zero. It will continue to work even if the layout is changed to hold the `work` field at a different offset. Please see the example introduced at the top of the file for example usage of delayed work items.
Showing the changes to rust/kernel/workqueue.rs (1 of 1 file). Show all
Unified · Split
| @@ -131,10 +131,69 @@ · 130 unmodified lines | ||
| 131 | 131 | //! # print_2_later(MyStruct::new(41, 42).unwrap()); |
| 132 | 132 | //! ``` |
| 133 | 133 | //! |
| 134 | +//! This example shows how you can schedule delayed work items: | |
| 135 | +//! | |
| 136 | +//! ``` | |
| 137 | +//! use kernel::sync::Arc; | |
| 138 | +//! use kernel::workqueue::{self, impl_has_delayed_work, new_delayed_work, DelayedWork, WorkItem}; | |
| 139 | +//! | |
| 140 | +//! #[pin_data] | |
| 141 | +//! struct MyStruct { | |
| 142 | +//! value: i32, | |
| 143 | +//! #[pin] | |
| 144 | +//! work: DelayedWork<MyStruct>, | |
| 145 | +//! } | |
| 146 | +//! | |
| 147 | +//! impl_has_delayed_work! { | |
| 148 | +//! impl HasDelayedWork<Self> for MyStruct { self.work } | |
| 149 | +//! } | |
| 150 | +//! | |
| 151 | +//! impl MyStruct { | |
| 152 | +//! fn new(value: i32) -> Result<Arc<Self>> { | |
| 153 | +//! Arc::pin_init( | |
| 154 | +//! pin_init!(MyStruct { | |
| 155 | +//! value, | |
| 156 | +//! work <- new_delayed_work!("MyStruct::work"), | |
| 157 | +//! }), | |
| 158 | +//! GFP_KERNEL, | |
| 159 | +//! ) | |
| 160 | +//! } | |
| 161 | +//! } | |
| 162 | +//! | |
| 163 | +//! impl WorkItem for MyStruct { | |
| 164 | +//! type Pointer = Arc<MyStruct>; | |
| 165 | +//! | |
| 166 | +//! fn run(this: Arc<MyStruct>) { | |
| 167 | +//! pr_info!("The value is: {}\n", this.value); | |
| 168 | +//! } | |
| 169 | +//! } | |
| 170 | +//! | |
| 171 | +//! /// This method will enqueue the struct for execution on the system workqueue, where its value | |
| 172 | +//! /// will be printed 12 jiffies later. | |
| 173 | +//! fn print_later(val: Arc<MyStruct>) { | |
| 174 | +//! let _ = workqueue::system().enqueue_delayed(val, 12); | |
| 175 | +//! } | |
| 176 | +//! | |
| 177 | +//! /// It is also possible to use the ordinary `enqueue` method together with `DelayedWork`. This | |
| 178 | +//! /// is equivalent to calling `enqueue_delayed` with a delay of zero. | |
| 179 | +//! fn print_now(val: Arc<MyStruct>) { | |
| 180 | +//! let _ = workqueue::system().enqueue(val); | |
| 181 | +//! } | |
| 182 | +//! # print_later(MyStruct::new(42).unwrap()); | |
| 183 | +//! # print_now(MyStruct::new(42).unwrap()); | |
| 184 | +//! ``` | |
| 185 | +//! | |
| 134 | 186 | //! C header: [`include/linux/workqueue.h`](srctree/include/linux/workqueue.h) |
| 135 | 187 | |
| 136 | −use crate::alloc::{AllocError, Flags}; | |
| 137 | −use crate::{prelude::*, sync::Arc, sync::LockClassKey, types::Opaque}; | |
| 188 | +use crate::{ | |
| 189 | + alloc::{AllocError, Flags}, | |
| 190 | + container_of, | |
| 191 | + prelude::*, | |
| 192 | + sync::Arc, | |
| 193 | + sync::LockClassKey, | |
| 194 | + time::Jiffies, | |
| 195 | + types::Opaque, | |
| 196 | +}; | |
| 138 | 197 | use core::marker::PhantomData; |
| 139 | 198 | |
| 140 | 199 | /// Creates a [`Work`] initialiser with the given name and a newly-created lock class. |
| @@ -146,6 +205,33 @@ macro_rules! new_work { · 5 unmodified lines | ||
| 146 | 205 | } |
| 147 | 206 | pub use new_work; |
| 148 | 207 | |
| 208 | +/// Creates a [`DelayedWork`] initialiser with the given name and a newly-created lock class. | |
| 209 | +#[macro_export] | |
| 210 | +macro_rules! new_delayed_work { | |
| 211 | + () => { | |
| 212 | + $crate::workqueue::DelayedWork::new( | |
| 213 | + $crate::optional_name!(), | |
| 214 | + $crate::static_lock_class!(), | |
| 215 | + $crate::c_str!(::core::concat!( | |
| 216 | + ::core::file!(), | |
| 217 | + ":", | |
| 218 | + ::core::line!(), | |
| 219 | + "_timer" | |
| 220 | + )), | |
| 221 | + $crate::static_lock_class!(), | |
| 222 | + ) | |
| 223 | + }; | |
| 224 | + ($name:literal) => { | |
| 225 | + $crate::workqueue::DelayedWork::new( | |
| 226 | + $crate::c_str!($name), | |
| 227 | + $crate::static_lock_class!(), | |
| 228 | + $crate::c_str!(::core::concat!($name, "_timer")), | |
| 229 | + $crate::static_lock_class!(), | |
| 230 | + ) | |
| 231 | + }; | |
| 232 | +} | |
| 233 | +pub use new_delayed_work; | |
| 234 | + | |
| 149 | 235 | /// A kernel work queue. |
| 150 | 236 | /// |
| 151 | 237 | /// Wraps the kernel's C `struct workqueue_struct`. |
| @@ -206,6 +292,42 @@ impl Queue { · 54 unmodified lines | ||
| 206 | 292 | } |
| 207 | 293 | } |
| 208 | 294 | |
| 295 | + /// Enqueues a delayed work item. | |
| 296 | + /// | |
| 297 | + /// This may fail if the work item is already enqueued in a workqueue. | |
| 298 | + /// | |
| 299 | + /// The work item will be submitted using `WORK_CPU_UNBOUND`. | |
| 300 | + pub fn enqueue_delayed<W, const ID: u64>(&self, w: W, delay: Jiffies) -> W::EnqueueOutput | |
| 301 | + where | |
| 302 | + W: RawDelayedWorkItem<ID> + Send + 'static, | |
| 303 | + { | |
| 304 | + let queue_ptr = self.0.get(); | |
| 305 | + | |
| 306 | + // SAFETY: We only return `false` if the `work_struct` is already in a workqueue. The other | |
| 307 | + // `__enqueue` requirements are not relevant since `W` is `Send` and static. | |
| 308 | + // | |
| 309 | + // The call to `bindings::queue_delayed_work_on` will dereference the provided raw pointer, | |
| 310 | + // which is ok because `__enqueue` guarantees that the pointer is valid for the duration of | |
| 311 | + // this closure, and the safety requirements of `RawDelayedWorkItem` expands this | |
| 312 | + // requirement to apply to the entire `delayed_work`. | |
| 313 | + // | |
| 314 | + // Furthermore, if the C workqueue code accesses the pointer after this call to | |
| 315 | + // `__enqueue`, then the work item was successfully enqueued, and | |
| 316 | + // `bindings::queue_delayed_work_on` will have returned true. In this case, `__enqueue` | |
| 317 | + // promises that the raw pointer will stay valid until we call the function pointer in the | |
| 318 | + // `work_struct`, so the access is ok. | |
| 319 | + unsafe { | |
| 320 | + w.__enqueue(move |work_ptr| { | |
| 321 | + bindings::queue_delayed_work_on( | |
| 322 | + bindings::wq_misc_consts_WORK_CPU_UNBOUND as ffi::c_int, | |
| 323 | + queue_ptr, | |
| 324 | + container_of!(work_ptr, bindings::delayed_work, work), | |
| 325 | + delay, | |
| 326 | + ) | |
| 327 | + }) | |
| 328 | + } | |
| 329 | + } | |
| 330 | + | |
| 209 | 331 | /// Tries to spawn the given function or closure as a work item. |
| 210 | 332 | /// |
| 211 | 333 | /// This method can fail because it allocates memory to store the work item. |
| @@ -298,6 +420,16 @@ pub unsafe trait RawWorkItem<const ID: u64> { · 86 unmodified lines | ||
| 298 | 420 | F: FnOnce(*mut bindings::work_struct) -> bool; |
| 299 | 421 | } |
| 300 | 422 | |
| 423 | +/// A raw delayed work item. | |
| 424 | +/// | |
| 425 | +/// # Safety | |
| 426 | +/// | |
| 427 | +/// If the `__enqueue` method in the `RawWorkItem` implementation calls the closure, then the | |
| 428 | +/// provided pointer must point at the `work` field of a valid `delayed_work`, and the guarantees | |
| 429 | +/// that `__enqueue` provides about accessing the `work_struct` must also apply to the rest of the | |
| 430 | +/// `delayed_work` struct. | |
| 431 | +pub unsafe trait RawDelayedWorkItem<const ID: u64>: RawWorkItem<ID> {} | |
| 432 | + | |
| 301 | 433 | /// Defines the method that should be called directly when a work item is executed. |
| 302 | 434 | /// |
| 303 | 435 | /// This trait is implemented by `Pin<KBox<T>>` and [`Arc<T>`], and is mainly intended to be |
| @@ -407,7 +539,7 @@ impl<T: ?Sized, const ID: u64> Work<T, ID> { · 103 unmodified lines | ||
| 407 | 539 | } |
| 408 | 540 | } |
| 409 | 541 | |
| 410 | −/// Declares that a type has a [`Work<T, ID>`] field. | |
| 542 | +/// Declares that a type contains a [`Work<T, ID>`]. | |
| 411 | 543 | /// |
| 412 | 544 | /// The intended way of using this trait is via the [`impl_has_work!`] macro. You can use the macro |
| 413 | 545 | /// like this: |
| @@ -506,6 +638,178 @@ impl_has_work! { · 92 unmodified lines | ||
| 506 | 638 | impl{T} HasWork<Self> for ClosureWork<T> { self.work } |
| 507 | 639 | } |
| 508 | 640 | |
| 641 | +/// Links for a delayed work item. | |
| 642 | +/// | |
| 643 | +/// This struct contains a function pointer to the [`run`] function from the [`WorkItemPointer`] | |
| 644 | +/// trait, and defines the linked list pointers necessary to enqueue a work item in a workqueue in | |
| 645 | +/// a delayed manner. | |
| 646 | +/// | |
| 647 | +/// Wraps the kernel's C `struct delayed_work`. | |
| 648 | +/// | |
| 649 | +/// This is a helper type used to associate a `delayed_work` with the [`WorkItem`] that uses it. | |
| 650 | +/// | |
| 651 | +/// [`run`]: WorkItemPointer::run | |
| 652 | +#[pin_data] | |
| 653 | +#[repr(transparent)] | |
| 654 | +pub struct DelayedWork<T: ?Sized, const ID: u64 = 0> { | |
| 655 | + #[pin] | |
| 656 | + dwork: Opaque<bindings::delayed_work>, | |
| 657 | + _inner: PhantomData<T>, | |
| 658 | +} | |
| 659 | + | |
| 660 | +// SAFETY: Kernel work items are usable from any thread. | |
| 661 | +// | |
| 662 | +// We do not need to constrain `T` since the work item does not actually contain a `T`. | |
| 663 | +unsafe impl<T: ?Sized, const ID: u64> Send for DelayedWork<T, ID> {} | |
| 664 | +// SAFETY: Kernel work items are usable from any thread. | |
| 665 | +// | |
| 666 | +// We do not need to constrain `T` since the work item does not actually contain a `T`. | |
| 667 | +unsafe impl<T: ?Sized, const ID: u64> Sync for DelayedWork<T, ID> {} | |
| 668 | + | |
| 669 | +impl<T: ?Sized, const ID: u64> DelayedWork<T, ID> { | |
| 670 | + /// Creates a new instance of [`DelayedWork`]. | |
| 671 | + #[inline] | |
| 672 | + pub fn new( | |
| 673 | + work_name: &'static CStr, | |
| 674 | + work_key: Pin<&'static LockClassKey>, | |
| 675 | + timer_name: &'static CStr, | |
| 676 | + timer_key: Pin<&'static LockClassKey>, | |
| 677 | + ) -> impl PinInit<Self> | |
| 678 | + where | |
| 679 | + T: WorkItem<ID>, | |
| 680 | + { | |
| 681 | + pin_init!(Self { | |
| 682 | + dwork <- Opaque::ffi_init(|slot: *mut bindings::delayed_work| { | |
| 683 | + // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as | |
| 684 | + // the work item function. | |
| 685 | + unsafe { | |
| 686 | + bindings::init_work_with_key( | |
| 687 | + core::ptr::addr_of_mut!((*slot).work), | |
| 688 | + Some(T::Pointer::run), | |
| 689 | + false, | |
| 690 | + work_name.as_char_ptr(), | |
| 691 | + work_key.as_ptr(), | |
| 692 | + ) | |
| 693 | + } | |
| 694 | + | |
| 695 | + // SAFETY: The `delayed_work_timer_fn` function pointer can be used here because | |
| 696 | + // the timer is embedded in a `struct delayed_work`, and only ever scheduled via | |
| 697 | + // the core workqueue code, and configured to run in irqsafe context. | |
| 698 | + unsafe { | |
| 699 | + bindings::timer_init_key( | |
| 700 | + core::ptr::addr_of_mut!((*slot).timer), | |
| 701 | + Some(bindings::delayed_work_timer_fn), | |
| 702 | + bindings::TIMER_IRQSAFE, | |
| 703 | + timer_name.as_char_ptr(), | |
| 704 | + timer_key.as_ptr(), | |
| 705 | + ) | |
| 706 | + } | |
| 707 | + }), | |
| 708 | + _inner: PhantomData, | |
| 709 | + }) | |
| 710 | + } | |
| 711 | + | |
| 712 | + /// Get a pointer to the inner `delayed_work`. | |
| 713 | + /// | |
| 714 | + /// # Safety | |
| 715 | + /// | |
| 716 | + /// The provided pointer must not be dangling and must be properly aligned. (But the memory | |
| 717 | + /// need not be initialized.) | |
| 718 | + #[inline] | |
| 719 | + pub unsafe fn raw_as_work(ptr: *const Self) -> *mut Work<T, ID> { | |
| 720 | + // SAFETY: The caller promises that the pointer is aligned and not dangling. | |
| 721 | + let dw: *mut bindings::delayed_work = | |
| 722 | + unsafe { Opaque::cast_into(core::ptr::addr_of!((*ptr).dwork)) }; | |
| 723 | + // SAFETY: The caller promises that the pointer is aligned and not dangling. | |
| 724 | + let wrk: *mut bindings::work_struct = unsafe { core::ptr::addr_of_mut!((*dw).work) }; | |
| 725 | + // CAST: Work and work_struct have compatible layouts. | |
| 726 | + wrk.cast() | |
| 727 | + } | |
| 728 | +} | |
| 729 | + | |
| 730 | +/// Declares that a type contains a [`DelayedWork<T, ID>`]. | |
| 731 | +/// | |
| 732 | +/// # Safety | |
| 733 | +/// | |
| 734 | +/// The `HasWork<T, ID>` implementation must return a `work_struct` that is stored in the `work` | |
| 735 | +/// field of a `delayed_work` with the same access rules as the `work_struct`. | |
| 736 | +pub unsafe trait HasDelayedWork<T, const ID: u64 = 0>: HasWork<T, ID> {} | |
| 737 | + | |
| 738 | +/// Used to safely implement the [`HasDelayedWork<T, ID>`] trait. | |
| 739 | +/// | |
| 740 | +/// This macro also implements the [`HasWork`] trait, so you do not need to use [`impl_has_work!`] | |
| 741 | +/// when using this macro. | |
| 742 | +/// | |
| 743 | +/// # Examples | |
| 744 | +/// | |
| 745 | +/// ``` | |
| 746 | +/// use kernel::sync::Arc; | |
| 747 | +/// use kernel::workqueue::{self, impl_has_delayed_work, DelayedWork}; | |
| 748 | +/// | |
| 749 | +/// struct MyStruct<'a, T, const N: usize> { | |
| 750 | +/// work_field: DelayedWork<MyStruct<'a, T, N>, 17>, | |
| 751 | +/// f: fn(&'a [T; N]), | |
| 752 | +/// } | |
| 753 | +/// | |
| 754 | +/// impl_has_delayed_work! { | |
| 755 | +/// impl{'a, T, const N: usize} HasDelayedWork<MyStruct<'a, T, N>, 17> | |
| 756 | +/// for MyStruct<'a, T, N> { self.work_field } | |
| 757 | +/// } | |
| 758 | +/// ``` | |
| 759 | +#[macro_export] | |
| 760 | +macro_rules! impl_has_delayed_work { | |
| 761 | + ($(impl$({$($generics:tt)*})? | |
| 762 | + HasDelayedWork<$work_type:ty $(, $id:tt)?> | |
| 763 | + for $self:ty | |
| 764 | + { self.$field:ident } | |
| 765 | + )*) => {$( | |
| 766 | + // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right | |
| 767 | + // type. | |
| 768 | + unsafe impl$(<$($generics)+>)? | |
| 769 | + $crate::workqueue::HasDelayedWork<$work_type $(, $id)?> for $self {} | |
| 770 | + | |
| 771 | + // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right | |
| 772 | + // type. | |
| 773 | + unsafe impl$(<$($generics)+>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self { | |
| 774 | + #[inline] | |
| 775 | + unsafe fn raw_get_work( | |
| 776 | + ptr: *mut Self | |
| 777 | + ) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> { | |
| 778 | + // SAFETY: The caller promises that the pointer is not dangling. | |
| 779 | + let ptr: *mut $crate::workqueue::DelayedWork<$work_type $(, $id)?> = unsafe { | |
| 780 | + ::core::ptr::addr_of_mut!((*ptr).$field) | |
| 781 | + }; | |
| 782 | + | |
| 783 | + // SAFETY: The caller promises that the pointer is not dangling. | |
| 784 | + unsafe { $crate::workqueue::DelayedWork::raw_as_work(ptr) } | |
| 785 | + } | |
| 786 | + | |
| 787 | + #[inline] | |
| 788 | + unsafe fn work_container_of( | |
| 789 | + ptr: *mut $crate::workqueue::Work<$work_type $(, $id)?>, | |
| 790 | + ) -> *mut Self { | |
| 791 | + // SAFETY: The caller promises that the pointer points at a field of the right type | |
| 792 | + // in the right kind of struct. | |
| 793 | + let ptr = unsafe { $crate::workqueue::Work::raw_get(ptr) }; | |
| 794 | + | |
| 795 | + // SAFETY: The caller promises that the pointer points at a field of the right type | |
| 796 | + // in the right kind of struct. | |
| 797 | + let delayed_work = unsafe { | |
| 798 | + $crate::container_of!(ptr, $crate::bindings::delayed_work, work) | |
| 799 | + }; | |
| 800 | + | |
| 801 | + let delayed_work: *mut $crate::workqueue::DelayedWork<$work_type $(, $id)?> = | |
| 802 | + delayed_work.cast(); | |
| 803 | + | |
| 804 | + // SAFETY: The caller promises that the pointer points at a field of the right type | |
| 805 | + // in the right kind of struct. | |
| 806 | + unsafe { $crate::container_of!(delayed_work, Self, $field) } | |
| 807 | + } | |
| 808 | + } | |
| 809 | + )*}; | |
| 810 | +} | |
| 811 | +pub use impl_has_delayed_work; | |
| 812 | + | |
| 509 | 813 | // SAFETY: The `__enqueue` implementation in RawWorkItem uses a `work_struct` initialized with the |
| 510 | 814 | // `run` method of this trait as the function pointer because: |
| 511 | 815 | // - `__enqueue` gets the `work_struct` from the `Work` field, using `T::raw_get_work`. |
| @@ -567,6 +871,16 @@ where · 55 unmodified lines | ||
| 567 | 871 | } |
| 568 | 872 | } |
| 569 | 873 | |
| 874 | +// SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in | |
| 875 | +// `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of | |
| 876 | +// the `delayed_work` has the same access rules as its `work` field. | |
| 877 | +unsafe impl<T, const ID: u64> RawDelayedWorkItem<ID> for Arc<T> | |
| 878 | +where | |
| 879 | + T: WorkItem<ID, Pointer = Self>, | |
| 880 | + T: HasDelayedWork<T, ID>, | |
| 881 | +{ | |
| 882 | +} | |
| 883 | + | |
| 570 | 884 | // SAFETY: TODO. |
| 571 | 885 | unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Pin<KBox<T>> |
| 572 | 886 | where |
| @@ -617,6 +931,16 @@ where · 44 unmodified lines | ||
| 617 | 931 | } |
| 618 | 932 | } |
| 619 | 933 | |
| 934 | +// SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in | |
| 935 | +// `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of | |
| 936 | +// the `delayed_work` has the same access rules as its `work` field. | |
| 937 | +unsafe impl<T, const ID: u64> RawDelayedWorkItem<ID> for Pin<KBox<T>> | |
| 938 | +where | |
| 939 | + T: WorkItem<ID, Pointer = Self>, | |
| 940 | + T: HasDelayedWork<T, ID>, | |
| 941 | +{ | |
| 942 | +} | |
| 943 | + | |
| 620 | 944 | /// Returns the system work queue (`system_wq`). |
| 621 | 945 | /// |
| 622 | 946 | /// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are |