impl<T: ?Sized, const ID: u64> Work<T, ID> { /// Creates a new instance of [`Work`]. #[inline] pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self> where T: WorkItem<ID>, { pin_init!(Self { work <- Opaque::ffi_init(|slot| { // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as // the work item function. unsafe { bindings::init_work_with_key( slot, Some(T::Pointer::run), false, name.as_char_ptr(), key.as_ptr(), ) } }), _inner: PhantomData, }) } /// Get a pointer to the inner `work_struct`. /// /// # Safety /// /// The provided pointer must not be dangling and must be properly aligned. (But the memory /// need not be initialized.) #[inline] pub unsafe fn raw_get(ptr: *const Self) -> *mut bindings::work_struct { // SAFETY: The caller promises that the pointer is aligned and not dangling. // // A pointer cast would also be ok due to `#[repr(transparent)]`. We use `addr_of!` so that // the compiler does not complain that the `work` field is unused. unsafe { Opaque::cast_into(core::ptr::addr_of!((*ptr).work)) } }} /// Declares that a type contains a [`Work<T, ID>`].////// The intended way of using this trait is via the [`impl_has_work!`] macro. You can use the macro/// like this:////// ```no_run/// use kernel::workqueue::{impl_has_work, Work};////// struct MyWorkItem {/// work_field: Work<MyWorkItem, 1>,/// }////// impl_has_work! {/// impl HasWork<MyWorkItem, 1> for MyWorkItem { self.work_field }/// }/// ```////// Note that since the [`Work`] type is annotated with an id, you can have several `work_struct`/// fields by using a different id for each one.////// # Safety////// The methods [`raw_get_work`] and [`work_container_of`] must return valid pointers and must be/// true inverses of each other; that is, they must satisfy the following invariants:/// - `work_container_of(raw_get_work(ptr)) == ptr` for any `ptr: *mut Self`./// - `raw_get_work(work_container_of(ptr)) == ptr` for any `ptr: *mut Work<T, ID>`.////// [`impl_has_work!`]: crate::impl_has_work/// [`raw_get_work`]: HasWork::raw_get_work/// [`work_container_of`]: HasWork::work_container_ofpub unsafe trait HasWork<T, const ID: u64 = 0> { /// Returns a pointer to the [`Work<T, ID>`] field. /// /// # Safety /// /// The provided pointer must point at a valid struct of type `Self`. unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work<T, ID>; /// Returns a pointer to the struct containing the [`Work<T, ID>`] field. /// /// # Safety /// /// The pointer must point at a [`Work<T, ID>`] field in a struct of type `Self`. unsafe fn work_container_of(ptr: *mut Work<T, ID>) -> *mut Self;} /// Used to safely implement the [`HasWork<T, ID>`] trait.////// # Examples////// ```/// use kernel::sync::Arc;/// use kernel::workqueue::{self, impl_has_work, Work};////// struct MyStruct<'a, T, const N: usize> {/// work_field: Work<MyStruct<'a, T, N>, 17>,/// f: fn(&'a [T; N]),/// }////// impl_has_work! {/// impl{'a, T, const N: usize} HasWork<MyStruct<'a, T, N>, 17>/// for MyStruct<'a, T, N> { self.work_field }/// }/// ```#[macro_export]macro_rules! impl_has_work { ($(impl$({$($generics:tt)*})? HasWork<$work_type:ty $(, $id:tt)?> for $self:ty { self.$field:ident } )*) => {$( // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right // type. unsafe impl$(<$($generics)+>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self { #[inline] unsafe fn raw_get_work(ptr: *mut Self) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> { // SAFETY: The caller promises that the pointer is not dangling. unsafe { ::core::ptr::addr_of_mut!((*ptr).$field) } } #[inline] unsafe fn work_container_of( ptr: *mut $crate::workqueue::Work<$work_type $(, $id)?>, ) -> *mut Self { // SAFETY: The caller promises that the pointer points at a field of the right type // in the right kind of struct. unsafe { $crate::container_of!(ptr, Self, $field) } } } )*};}pub use impl_has_work; impl_has_work! { impl{T} HasWork<Self> for ClosureWork<T> { self.work }} /// Links for a delayed work item.////// This struct contains a function pointer to the [`run`] function from the [`WorkItemPointer`]/// trait, and defines the linked list pointers necessary to enqueue a work item in a workqueue in/// a delayed manner.////// Wraps the kernel's C `struct delayed_work`.////// This is a helper type used to associate a `delayed_work` with the [`WorkItem`] that uses it.////// [`run`]: WorkItemPointer::run#[pin_data]#[repr(transparent)]pub struct DelayedWork<T: ?Sized, const ID: u64 = 0> { #[pin] dwork: Opaque<bindings::delayed_work>, _inner: PhantomData<T>,} // SAFETY: Kernel work items are usable from any thread.//// We do not need to constrain `T` since the work item does not actually contain a `T`.unsafe impl<T: ?Sized, const ID: u64> Send for DelayedWork<T, ID> {}// SAFETY: Kernel work items are usable from any thread.//// We do not need to constrain `T` since the work item does not actually contain a `T`.unsafe impl<T: ?Sized, const ID: u64> Sync for DelayedWork<T, ID> {} impl<T: ?Sized, const ID: u64> DelayedWork<T, ID> { /// Creates a new instance of [`DelayedWork`]. #[inline] pub fn new( work_name: &'static CStr, work_key: Pin<&'static LockClassKey>, timer_name: &'static CStr, timer_key: Pin<&'static LockClassKey>, ) -> impl PinInit<Self> where T: WorkItem<ID>, { pin_init!(Self { dwork <- Opaque::ffi_init(|slot: *mut bindings::delayed_work| { // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as // the work item function. unsafe { bindings::init_work_with_key( core::ptr::addr_of_mut!((*slot).work), Some(T::Pointer::run), false, work_name.as_char_ptr(), work_key.as_ptr(), ) } // SAFETY: The `delayed_work_timer_fn` function pointer can be used here because // the timer is embedded in a `struct delayed_work`, and only ever scheduled via // the core workqueue code, and configured to run in irqsafe context. unsafe { bindings::timer_init_key( core::ptr::addr_of_mut!((*slot).timer), Some(bindings::delayed_work_timer_fn), bindings::TIMER_IRQSAFE, timer_name.as_char_ptr(), timer_key.as_ptr(), ) } }), _inner: PhantomData, }) } /// Get a pointer to the inner `delayed_work`. /// /// # Safety /// /// The provided pointer must not be dangling and must be properly aligned. (But the memory /// need not be initialized.) #[inline] pub unsafe fn raw_as_work(ptr: *const Self) -> *mut Work<T, ID> { // SAFETY: The caller promises that the pointer is aligned and not dangling. let dw: *mut bindings::delayed_work = unsafe { Opaque::cast_into(core::ptr::addr_of!((*ptr).dwork)) }; // SAFETY: The caller promises that the pointer is aligned and not dangling. let wrk: *mut bindings::work_struct = unsafe { core::ptr::addr_of_mut!((*dw).work) }; // CAST: Work and work_struct have compatible layouts. wrk.cast() }} /// Declares that a type contains a [`DelayedWork<T, ID>`].////// # Safety////// The `HasWork<T, ID>` implementation must return a `work_struct` that is stored in the `work`/// field of a `delayed_work` with the same access rules as the `work_struct`.pub unsafe trait HasDelayedWork<T, const ID: u64 = 0>: HasWork<T, ID> {} /// Used to safely implement the [`HasDelayedWork<T, ID>`] trait.////// This macro also implements the [`HasWork`] trait, so you do not need to use [`impl_has_work!`]/// when using this macro.////// # Examples////// ```/// use kernel::sync::Arc;/// use kernel::workqueue::{self, impl_has_delayed_work, DelayedWork};////// struct MyStruct<'a, T, const N: usize> {/// work_field: DelayedWork<MyStruct<'a, T, N>, 17>,/// f: fn(&'a [T; N]),/// }////// impl_has_delayed_work! {/// impl{'a, T, const N: usize} HasDelayedWork<MyStruct<'a, T, N>, 17>/// for MyStruct<'a, T, N> { self.work_field }/// }/// ```#[macro_export]macro_rules! impl_has_delayed_work { ($(impl$({$($generics:tt)*})? HasDelayedWork<$work_type:ty $(, $id:tt)?> for $self:ty { self.$field:ident } )*) => {$( // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right // type. unsafe impl$(<$($generics)+>)? $crate::workqueue::HasDelayedWork<$work_type $(, $id)?> for $self {} // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right // type. unsafe impl$(<$($generics)+>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self { #[inline] unsafe fn raw_get_work( ptr: *mut Self ) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> { // SAFETY: The caller promises that the pointer is not dangling. let ptr: *mut $crate::workqueue::DelayedWork<$work_type $(, $id)?> = unsafe { ::core::ptr::addr_of_mut!((*ptr).$field) }; // SAFETY: The caller promises that the pointer is not dangling. unsafe { $crate::workqueue::DelayedWork::raw_as_work(ptr) } } #[inline] unsafe fn work_container_of( ptr: *mut $crate::workqueue::Work<$work_type $(, $id)?>, ) -> *mut Self { // SAFETY: The caller promises that the pointer points at a field of the right type // in the right kind of struct. let ptr = unsafe { $crate::workqueue::Work::raw_get(ptr) }; // SAFETY: The caller promises that the pointer points at a field of the right type // in the right kind of struct. let delayed_work = unsafe { $crate::container_of!(ptr, $crate::bindings::delayed_work, work) }; let delayed_work: *mut $crate::workqueue::DelayedWork<$work_type $(, $id)?> = delayed_work.cast(); // SAFETY: The caller promises that the pointer points at a field of the right type // in the right kind of struct. unsafe { $crate::container_of!(delayed_work, Self, $field) } } } )*};}pub use impl_has_delayed_work; // SAFETY: The `__enqueue` implementation in RawWorkItem uses a `work_struct` initialized with the// `run` method of this trait as the function pointer because:// - `__enqueue` gets the `work_struct` from the `Work` field, using `T::raw_get_work`.// - The only safe way to create a `Work` object is through `Work::new`.// - `Work::new` makes sure that `T::Pointer::run` is passed to `init_work_with_key`.// - Finally `Work` and `RawWorkItem` guarantee that the correct `Work` field// will be used because of the ID const generic bound. This makes sure that `T::raw_get_work`// uses the correct offset for the `Work` field, and `Work::new` picks the correct// implementation of `WorkItemPointer` for `Arc<T>`.unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Arc<T>where T: WorkItem<ID, Pointer = Self>, T: HasWork<T, ID>,{ unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { // The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. let ptr = ptr.cast::<Work<T, ID>>(); // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. let ptr = unsafe { T::work_container_of(ptr) }; // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. let arc = unsafe { Arc::from_raw(ptr) }; T::run(arc) }} // SAFETY: The `work_struct` raw pointer is guaranteed to be valid for the duration of the call to// the closure because we get it from an `Arc`, which means that the ref count will be at least 1,// and we don't drop the `Arc` ourselves. If `queue_work_on` returns true, it is further guaranteed// to be valid until a call to the function pointer in `work_struct` because we leak the memory it// points to, and only reclaim it if the closure returns false, or in `WorkItemPointer::run`, which// is what the function pointer in the `work_struct` must be pointing to, according to the safety// requirements of `WorkItemPointer`.unsafe impl<T, const ID: u64> RawWorkItem<ID> for Arc<T>where T: WorkItem<ID, Pointer = Self>, T: HasWork<T, ID>,{ type EnqueueOutput = Result<(), Self>; unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput where F: FnOnce(*mut bindings::work_struct) -> bool, { // Casting between const and mut is not a problem as long as the pointer is a raw pointer. let ptr = Arc::into_raw(self).cast_mut(); // SAFETY: Pointers into an `Arc` point at a valid value. let work_ptr = unsafe { T::raw_get_work(ptr) }; // SAFETY: `raw_get_work` returns a pointer to a valid value. let work_ptr = unsafe { Work::raw_get(work_ptr) }; if queue_work_on(work_ptr) { Ok(()) } else { // SAFETY: The work queue has not taken ownership of the pointer. Err(unsafe { Arc::from_raw(ptr) }) } }} // SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in// `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of// the `delayed_work` has the same access rules as its `work` field.unsafe impl<T, const ID: u64> RawDelayedWorkItem<ID> for Arc<T>where T: WorkItem<ID, Pointer = Self>, T: HasDelayedWork<T, ID>,{} // SAFETY: TODO.unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Pin<KBox<T>>where T: WorkItem<ID, Pointer = Self>, T: HasWork<T, ID>,{ unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { // The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. let ptr = ptr.cast::<Work<T, ID>>(); // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. let ptr = unsafe { T::work_container_of(ptr) }; // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. let boxed = unsafe { KBox::from_raw(ptr) }; // SAFETY: The box was already pinned when it was enqueued. let pinned = unsafe { Pin::new_unchecked(boxed) }; T::run(pinned) }} // SAFETY: TODO.unsafe impl<T, const ID: u64> RawWorkItem<ID> for Pin<KBox<T>>where T: WorkItem<ID, Pointer = Self>, T: HasWork<T, ID>,{ type EnqueueOutput = (); unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput where F: FnOnce(*mut bindings::work_struct) -> bool, { // SAFETY: We're not going to move `self` or any of its fields, so its okay to temporarily // remove the `Pin` wrapper. let boxed = unsafe { Pin::into_inner_unchecked(self) }; let ptr = KBox::into_raw(boxed); // SAFETY: Pointers into a `KBox` point at a valid value. let work_ptr = unsafe { T::raw_get_work(ptr) }; // SAFETY: `raw_get_work` returns a pointer to a valid value. let work_ptr = unsafe { Work::raw_get(work_ptr) }; if !queue_work_on(work_ptr) { // SAFETY: This method requires exclusive ownership of the box, so it cannot be in a // workqueue. unsafe { ::core::hint::unreachable_unchecked() } } }} // SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in// `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of// the `delayed_work` has the same access rules as its `work` field.unsafe impl<T, const ID: u64> RawDelayedWorkItem<ID> for Pin<KBox<T>>where T: WorkItem<ID, Pointer = Self>, T: HasDelayedWork<T, ID>,{} // SAFETY: Like the `Arc<T>` implementation, the `__enqueue` implementation for// `ARef<T>` obtains a `work_struct` from the `Work` field using// `T::raw_get_work`, so the same safety reasoning applies://// - `__enqueue` gets the `work_struct` from the `Work` field, using `T::raw_get_work`.// - The only safe way to create a `Work` object is through `Work::new`.// - `Work::new` makes sure that `T::Pointer::run` is passed to `init_work_with_key`.// - Finally `Work` and `RawWorkItem` guarantee that the correct `Work` field// will be used because of the ID const generic bound. This makes sure that `T::raw_get_work`// uses the correct offset for the `Work` field, and `Work::new` picks the correct// implementation of `WorkItemPointer` for `ARef<T>`.unsafe impl<T, const ID: u64> WorkItemPointer<ID> for ARef<T>where T: AlwaysRefCounted, T: WorkItem<ID, Pointer = Self>, T: HasWork<T, ID>,{ unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { // The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. let ptr = ptr.cast::<Work<T, ID>>(); // SAFETY: This computes the pointer that `__enqueue` got from // `ARef::into_raw`. let ptr = unsafe { T::work_container_of(ptr) }; // SAFETY: The safety contract of `work_container_of` ensures that it // returns a valid non-null pointer. let ptr = unsafe { NonNull::new_unchecked(ptr) }; // SAFETY: This pointer comes from `ARef::into_raw` and we've been given // back ownership. let aref = unsafe { ARef::from_raw(ptr) }; T::run(aref) }} // SAFETY: The `work_struct` raw pointer is guaranteed to be valid for the duration of the call to// the closure because we get it from an `ARef`, which means that the ref count will be at least 1,// and we don't drop the `ARef` ourselves. If `queue_work_on` returns true, it is further guaranteed// to be valid until a call to the function pointer in `work_struct` because we leak the memory it// points to, and only reclaim it if the closure returns false, or in `WorkItemPointer::run`, which// is what the function pointer in the `work_struct` must be pointing to, according to the safety// requirements of `WorkItemPointer`.unsafe impl<T, const ID: u64> RawWorkItem<ID> for ARef<T>where T: AlwaysRefCounted, T: WorkItem<ID, Pointer = Self>, T: HasWork<T, ID>,{ type EnqueueOutput = Result<(), Self>; unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput where F: FnOnce(*mut bindings::work_struct) -> bool, { let ptr = ARef::into_raw(self);Lines 501–1000 of 1114. Previous lines Load more
impl<T: ?Sized, const ID: u64> Work<T, ID> { /// Creates a new instance of [`Work`]. #[inline] pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self> where T: WorkItem<ID>, { pin_init!(Self { work <- Opaque::ffi_init(|slot| { // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as // the work item function. unsafe { bindings::init_work_with_key( slot, Some(T::Pointer::run), false, name.as_char_ptr(), key.as_ptr(), ) } }), _inner: PhantomData, }) } /// Get a pointer to the inner `work_struct`. /// /// # Safety /// /// The provided pointer must not be dangling and must be properly aligned. (But the memory /// need not be initialized.) #[inline] pub unsafe fn raw_get(ptr: *const Self) -> *mut bindings::work_struct { // SAFETY: The caller promises that the pointer is aligned and not dangling. // // A pointer cast would also be ok due to `#[repr(transparent)]`. We use `addr_of!` so that // the compiler does not complain that the `work` field is unused. unsafe { Opaque::cast_into(core::ptr::addr_of!((*ptr).work)) } } } /// Declares that a type contains a [`Work<T, ID>`]. /// /// The intended way of using this trait is via the [`impl_has_work!`] macro. You can use the macro /// like this: /// /// ```no_run /// use kernel::workqueue::{impl_has_work, Work}; /// /// struct MyWorkItem { /// work_field: Work<MyWorkItem, 1>, /// } /// /// impl_has_work! { /// impl HasWork<MyWorkItem, 1> for MyWorkItem { self.work_field } /// } /// ``` /// /// Note that since the [`Work`] type is annotated with an id, you can have several `work_struct` /// fields by using a different id for each one. /// /// # Safety /// /// The methods [`raw_get_work`] and [`work_container_of`] must return valid pointers and must be /// true inverses of each other; that is, they must satisfy the following invariants: /// - `work_container_of(raw_get_work(ptr)) == ptr` for any `ptr: *mut Self`. /// - `raw_get_work(work_container_of(ptr)) == ptr` for any `ptr: *mut Work<T, ID>`. /// /// [`impl_has_work!`]: crate::impl_has_work /// [`raw_get_work`]: HasWork::raw_get_work /// [`work_container_of`]: HasWork::work_container_of pub unsafe trait HasWork<T, const ID: u64 = 0> { /// Returns a pointer to the [`Work<T, ID>`] field. /// /// # Safety /// /// The provided pointer must point at a valid struct of type `Self`. unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work<T, ID>; /// Returns a pointer to the struct containing the [`Work<T, ID>`] field. /// /// # Safety /// /// The pointer must point at a [`Work<T, ID>`] field in a struct of type `Self`. unsafe fn work_container_of(ptr: *mut Work<T, ID>) -> *mut Self; } /// Used to safely implement the [`HasWork<T, ID>`] trait. /// /// # Examples /// /// ``` /// use kernel::sync::Arc; /// use kernel::workqueue::{self, impl_has_work, Work}; /// /// struct MyStruct<'a, T, const N: usize> { /// work_field: Work<MyStruct<'a, T, N>, 17>, /// f: fn(&'a [T; N]), /// } /// /// impl_has_work! { /// impl{'a, T, const N: usize} HasWork<MyStruct<'a, T, N>, 17> /// for MyStruct<'a, T, N> { self.work_field } /// } /// ``` #[macro_export] macro_rules! impl_has_work { ($(impl$({$($generics:tt)*})? HasWork<$work_type:ty $(, $id:tt)?> for $self:ty { self.$field:ident } )*) => {$( // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right // type. unsafe impl$(<$($generics)+>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self { #[inline] unsafe fn raw_get_work(ptr: *mut Self) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> { // SAFETY: The caller promises that the pointer is not dangling. unsafe { ::core::ptr::addr_of_mut!((*ptr).$field) } } #[inline] unsafe fn work_container_of( ptr: *mut $crate::workqueue::Work<$work_type $(, $id)?>, ) -> *mut Self { // SAFETY: The caller promises that the pointer points at a field of the right type // in the right kind of struct. unsafe { $crate::container_of!(ptr, Self, $field) } } } )*}; } pub use impl_has_work; impl_has_work! { impl{T} HasWork<Self> for ClosureWork<T> { self.work } } /// Links for a delayed work item. /// /// This struct contains a function pointer to the [`run`] function from the [`WorkItemPointer`] /// trait, and defines the linked list pointers necessary to enqueue a work item in a workqueue in /// a delayed manner. /// /// Wraps the kernel's C `struct delayed_work`. /// /// This is a helper type used to associate a `delayed_work` with the [`WorkItem`] that uses it. /// /// [`run`]: WorkItemPointer::run #[pin_data] #[repr(transparent)] pub struct DelayedWork<T: ?Sized, const ID: u64 = 0> { #[pin] dwork: Opaque<bindings::delayed_work>, _inner: PhantomData<T>, } // SAFETY: Kernel work items are usable from any thread. // // We do not need to constrain `T` since the work item does not actually contain a `T`. unsafe impl<T: ?Sized, const ID: u64> Send for DelayedWork<T, ID> {} // SAFETY: Kernel work items are usable from any thread. // // We do not need to constrain `T` since the work item does not actually contain a `T`. unsafe impl<T: ?Sized, const ID: u64> Sync for DelayedWork<T, ID> {} impl<T: ?Sized, const ID: u64> DelayedWork<T, ID> { /// Creates a new instance of [`DelayedWork`]. #[inline] pub fn new( work_name: &'static CStr, work_key: Pin<&'static LockClassKey>, timer_name: &'static CStr, timer_key: Pin<&'static LockClassKey>, ) -> impl PinInit<Self> where T: WorkItem<ID>, { pin_init!(Self { dwork <- Opaque::ffi_init(|slot: *mut bindings::delayed_work| { // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as // the work item function. unsafe { bindings::init_work_with_key( core::ptr::addr_of_mut!((*slot).work), Some(T::Pointer::run), false, work_name.as_char_ptr(), work_key.as_ptr(), ) } // SAFETY: The `delayed_work_timer_fn` function pointer can be used here because // the timer is embedded in a `struct delayed_work`, and only ever scheduled via // the core workqueue code, and configured to run in irqsafe context. unsafe { bindings::timer_init_key( core::ptr::addr_of_mut!((*slot).timer), Some(bindings::delayed_work_timer_fn), bindings::TIMER_IRQSAFE, timer_name.as_char_ptr(), timer_key.as_ptr(), ) } }), _inner: PhantomData, }) } /// Get a pointer to the inner `delayed_work`. /// /// # Safety /// /// The provided pointer must not be dangling and must be properly aligned. (But the memory /// need not be initialized.) #[inline] pub unsafe fn raw_as_work(ptr: *const Self) -> *mut Work<T, ID> { // SAFETY: The caller promises that the pointer is aligned and not dangling. let dw: *mut bindings::delayed_work = unsafe { Opaque::cast_into(core::ptr::addr_of!((*ptr).dwork)) }; // SAFETY: The caller promises that the pointer is aligned and not dangling. let wrk: *mut bindings::work_struct = unsafe { core::ptr::addr_of_mut!((*dw).work) }; // CAST: Work and work_struct have compatible layouts. wrk.cast() } } /// Declares that a type contains a [`DelayedWork<T, ID>`]. /// /// # Safety /// /// The `HasWork<T, ID>` implementation must return a `work_struct` that is stored in the `work` /// field of a `delayed_work` with the same access rules as the `work_struct`. pub unsafe trait HasDelayedWork<T, const ID: u64 = 0>: HasWork<T, ID> {} /// Used to safely implement the [`HasDelayedWork<T, ID>`] trait. /// /// This macro also implements the [`HasWork`] trait, so you do not need to use [`impl_has_work!`] /// when using this macro. /// /// # Examples /// /// ``` /// use kernel::sync::Arc; /// use kernel::workqueue::{self, impl_has_delayed_work, DelayedWork}; /// /// struct MyStruct<'a, T, const N: usize> { /// work_field: DelayedWork<MyStruct<'a, T, N>, 17>, /// f: fn(&'a [T; N]), /// } /// /// impl_has_delayed_work! { /// impl{'a, T, const N: usize} HasDelayedWork<MyStruct<'a, T, N>, 17> /// for MyStruct<'a, T, N> { self.work_field } /// } /// ``` #[macro_export] macro_rules! impl_has_delayed_work { ($(impl$({$($generics:tt)*})? HasDelayedWork<$work_type:ty $(, $id:tt)?> for $self:ty { self.$field:ident } )*) => {$( // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right // type. unsafe impl$(<$($generics)+>)? $crate::workqueue::HasDelayedWork<$work_type $(, $id)?> for $self {} // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right // type. unsafe impl$(<$($generics)+>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self { #[inline] unsafe fn raw_get_work( ptr: *mut Self ) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> { // SAFETY: The caller promises that the pointer is not dangling. let ptr: *mut $crate::workqueue::DelayedWork<$work_type $(, $id)?> = unsafe { ::core::ptr::addr_of_mut!((*ptr).$field) }; // SAFETY: The caller promises that the pointer is not dangling. unsafe { $crate::workqueue::DelayedWork::raw_as_work(ptr) } } #[inline] unsafe fn work_container_of( ptr: *mut $crate::workqueue::Work<$work_type $(, $id)?>, ) -> *mut Self { // SAFETY: The caller promises that the pointer points at a field of the right type // in the right kind of struct. let ptr = unsafe { $crate::workqueue::Work::raw_get(ptr) }; // SAFETY: The caller promises that the pointer points at a field of the right type // in the right kind of struct. let delayed_work = unsafe { $crate::container_of!(ptr, $crate::bindings::delayed_work, work) }; let delayed_work: *mut $crate::workqueue::DelayedWork<$work_type $(, $id)?> = delayed_work.cast(); // SAFETY: The caller promises that the pointer points at a field of the right type // in the right kind of struct. unsafe { $crate::container_of!(delayed_work, Self, $field) } } } )*}; } pub use impl_has_delayed_work; // SAFETY: The `__enqueue` implementation in RawWorkItem uses a `work_struct` initialized with the // `run` method of this trait as the function pointer because: // - `__enqueue` gets the `work_struct` from the `Work` field, using `T::raw_get_work`. // - The only safe way to create a `Work` object is through `Work::new`. // - `Work::new` makes sure that `T::Pointer::run` is passed to `init_work_with_key`. // - Finally `Work` and `RawWorkItem` guarantee that the correct `Work` field // will be used because of the ID const generic bound. This makes sure that `T::raw_get_work` // uses the correct offset for the `Work` field, and `Work::new` picks the correct // implementation of `WorkItemPointer` for `Arc<T>`. unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Arc<T> where T: WorkItem<ID, Pointer = Self>, T: HasWork<T, ID>, { unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { // The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. let ptr = ptr.cast::<Work<T, ID>>(); // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. let ptr = unsafe { T::work_container_of(ptr) }; // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. let arc = unsafe { Arc::from_raw(ptr) }; T::run(arc) } } // SAFETY: The `work_struct` raw pointer is guaranteed to be valid for the duration of the call to // the closure because we get it from an `Arc`, which means that the ref count will be at least 1, // and we don't drop the `Arc` ourselves. If `queue_work_on` returns true, it is further guaranteed // to be valid until a call to the function pointer in `work_struct` because we leak the memory it // points to, and only reclaim it if the closure returns false, or in `WorkItemPointer::run`, which // is what the function pointer in the `work_struct` must be pointing to, according to the safety // requirements of `WorkItemPointer`. unsafe impl<T, const ID: u64> RawWorkItem<ID> for Arc<T> where T: WorkItem<ID, Pointer = Self>, T: HasWork<T, ID>, { type EnqueueOutput = Result<(), Self>; unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput where F: FnOnce(*mut bindings::work_struct) -> bool, { // Casting between const and mut is not a problem as long as the pointer is a raw pointer. let ptr = Arc::into_raw(self).cast_mut(); // SAFETY: Pointers into an `Arc` point at a valid value. let work_ptr = unsafe { T::raw_get_work(ptr) }; // SAFETY: `raw_get_work` returns a pointer to a valid value. let work_ptr = unsafe { Work::raw_get(work_ptr) }; if queue_work_on(work_ptr) { Ok(()) } else { // SAFETY: The work queue has not taken ownership of the pointer. Err(unsafe { Arc::from_raw(ptr) }) } } } // SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in // `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of // the `delayed_work` has the same access rules as its `work` field. unsafe impl<T, const ID: u64> RawDelayedWorkItem<ID> for Arc<T> where T: WorkItem<ID, Pointer = Self>, T: HasDelayedWork<T, ID>, { } // SAFETY: TODO. unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Pin<KBox<T>> where T: WorkItem<ID, Pointer = Self>, T: HasWork<T, ID>, { unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { // The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. let ptr = ptr.cast::<Work<T, ID>>(); // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. let ptr = unsafe { T::work_container_of(ptr) }; // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. let boxed = unsafe { KBox::from_raw(ptr) }; // SAFETY: The box was already pinned when it was enqueued. let pinned = unsafe { Pin::new_unchecked(boxed) }; T::run(pinned) } } // SAFETY: TODO. unsafe impl<T, const ID: u64> RawWorkItem<ID> for Pin<KBox<T>> where T: WorkItem<ID, Pointer = Self>, T: HasWork<T, ID>, { type EnqueueOutput = (); unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput where F: FnOnce(*mut bindings::work_struct) -> bool, { // SAFETY: We're not going to move `self` or any of its fields, so its okay to temporarily // remove the `Pin` wrapper. let boxed = unsafe { Pin::into_inner_unchecked(self) }; let ptr = KBox::into_raw(boxed); // SAFETY: Pointers into a `KBox` point at a valid value. let work_ptr = unsafe { T::raw_get_work(ptr) }; // SAFETY: `raw_get_work` returns a pointer to a valid value. let work_ptr = unsafe { Work::raw_get(work_ptr) }; if !queue_work_on(work_ptr) { // SAFETY: This method requires exclusive ownership of the box, so it cannot be in a // workqueue. unsafe { ::core::hint::unreachable_unchecked() } } } } // SAFETY: By the safety requirements of `HasDelayedWork`, the `work_struct` returned by methods in // `HasWork` provides a `work_struct` that is the `work` field of a `delayed_work`, and the rest of // the `delayed_work` has the same access rules as its `work` field. unsafe impl<T, const ID: u64> RawDelayedWorkItem<ID> for Pin<KBox<T>> where T: WorkItem<ID, Pointer = Self>, T: HasDelayedWork<T, ID>, { } // SAFETY: Like the `Arc<T>` implementation, the `__enqueue` implementation for // `ARef<T>` obtains a `work_struct` from the `Work` field using // `T::raw_get_work`, so the same safety reasoning applies: // // - `__enqueue` gets the `work_struct` from the `Work` field, using `T::raw_get_work`. // - The only safe way to create a `Work` object is through `Work::new`. // - `Work::new` makes sure that `T::Pointer::run` is passed to `init_work_with_key`. // - Finally `Work` and `RawWorkItem` guarantee that the correct `Work` field // will be used because of the ID const generic bound. This makes sure that `T::raw_get_work` // uses the correct offset for the `Work` field, and `Work::new` picks the correct // implementation of `WorkItemPointer` for `ARef<T>`. unsafe impl<T, const ID: u64> WorkItemPointer<ID> for ARef<T> where T: AlwaysRefCounted, T: WorkItem<ID, Pointer = Self>, T: HasWork<T, ID>, { unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { // The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. let ptr = ptr.cast::<Work<T, ID>>(); // SAFETY: This computes the pointer that `__enqueue` got from // `ARef::into_raw`. let ptr = unsafe { T::work_container_of(ptr) }; // SAFETY: The safety contract of `work_container_of` ensures that it // returns a valid non-null pointer. let ptr = unsafe { NonNull::new_unchecked(ptr) }; // SAFETY: This pointer comes from `ARef::into_raw` and we've been given // back ownership. let aref = unsafe { ARef::from_raw(ptr) }; T::run(aref) } } // SAFETY: The `work_struct` raw pointer is guaranteed to be valid for the duration of the call to // the closure because we get it from an `ARef`, which means that the ref count will be at least 1, // and we don't drop the `ARef` ourselves. If `queue_work_on` returns true, it is further guaranteed // to be valid until a call to the function pointer in `work_struct` because we leak the memory it // points to, and only reclaim it if the closure returns false, or in `WorkItemPointer::run`, which // is what the function pointer in the `work_struct` must be pointing to, according to the safety // requirements of `WorkItemPointer`. unsafe impl<T, const ID: u64> RawWorkItem<ID> for ARef<T> where T: AlwaysRefCounted, T: WorkItem<ID, Pointer = Self>, T: HasWork<T, ID>, { type EnqueueOutput = Result<(), Self>; unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput where F: FnOnce(*mut bindings::work_struct) -> bool, { let ptr = ARef::into_raw(self);