Add a Rust abstraction for sleepable RCU (SRCU), backed by C srcu_struct. Provide FFI helpers and a safe wrapper with a guard-based API for read-side critical sections. Cleanup is handled via `PinnedDrop`. It first checks for active read-side sections and emits a warning if any guards were leaked. In that case, it waits in `synchronize_srcu()` rather than risking a UAF by freeing the `srcu_struct` that is still reachable from the C side. It then uses `srcu_barrier()` to drain pending callbacks before finally calling `cleanup_srcu_struct()`.
Showing the changes to rust/kernel (2 of 2 files). Show all
Unified · Split
| @@ -21,6 +21,7 @@ pub mod poll; · 20 unmodified lines | ||
| 21 | 21 | pub mod rcu; |
| 22 | 22 | mod refcount; |
| 23 | 23 | mod set_once; |
| 24 | +pub mod srcu; | |
| 24 | 25 | |
| 25 | 26 | pub use arc::{Arc, ArcBorrow, UniqueArc}; |
| 26 | 27 | pub use completion::Completion; |
| @@ -31,6 +32,7 @@ pub use lock::spinlock::{new_spinlock, SpinLock, SpinLockGuard}; · 4 unmodified lines | ||
| 31 | 32 | pub use locked_by::LockedBy; |
| 32 | 33 | pub use refcount::Refcount; |
| 33 | 34 | pub use set_once::SetOnce; |
| 35 | +pub use srcu::Srcu; | |
| 34 | 36 | |
| 35 | 37 | /// Represents a lockdep class. |
| 36 | 38 | /// |
| @@ -0,0 +1,171 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0 | |
| 2 | + | |
| 3 | +//! Sleepable read-copy update (SRCU) support. | |
| 4 | +//! | |
| 5 | +//! C header: [`include/linux/srcu.h`](srctree/include/linux/srcu.h) | |
| 6 | + | |
| 7 | +use crate::{ | |
| 8 | + bindings, | |
| 9 | + error::to_result, | |
| 10 | + prelude::*, | |
| 11 | + sync::LockClassKey, | |
| 12 | + types::{ | |
| 13 | + NotThreadSafe, | |
| 14 | + Opaque, // | |
| 15 | + }, | |
| 16 | +}; | |
| 17 | + | |
| 18 | +use pin_init::pin_data; | |
| 19 | + | |
| 20 | +/// Creates an [`Srcu`] initialiser with the given name and a newly-created lock class. | |
| 21 | +#[doc(hidden)] | |
| 22 | +#[macro_export] | |
| 23 | +macro_rules! new_srcu { | |
| 24 | + ($($name:literal)?) => { | |
| 25 | + $crate::sync::Srcu::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) | |
| 26 | + }; | |
| 27 | +} | |
| 28 | +pub use new_srcu; | |
| 29 | + | |
| 30 | +/// Sleepable read-copy update primitive. | |
| 31 | +/// | |
| 32 | +/// SRCU readers may sleep while holding the read-side guard. | |
| 33 | +/// | |
| 34 | +/// The destructor waits for active readers and callbacks, so it may sleep. | |
| 35 | +/// If a read-side guard has been leaked, dropping an [`Srcu`] may never return. | |
| 36 | +/// | |
| 37 | +/// # Invariants | |
| 38 | +/// | |
| 39 | +/// This represents a valid `struct srcu_struct` initialized by the C SRCU API | |
| 40 | +/// and it remains pinned and valid until the pinned destructor runs. | |
| 41 | +#[repr(transparent)] | |
| 42 | +#[pin_data(PinnedDrop)] | |
| 43 | +pub struct Srcu { | |
| 44 | + #[pin] | |
| 45 | + inner: Opaque<bindings::srcu_struct>, | |
| 46 | +} | |
| 47 | + | |
| 48 | +impl Srcu { | |
| 49 | + /// Creates a new SRCU instance. | |
| 50 | + #[inline] | |
| 51 | + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self, Error> { | |
| 52 | + try_pin_init!(Self { | |
| 53 | + // INVARIANT: On success, the C initializer creates a valid `srcu_struct` and | |
| 54 | + // it remains pinned until `PinnedDrop` runs. | |
| 55 | + inner <- Opaque::try_ffi_init(|ptr: *mut bindings::srcu_struct| { | |
| 56 | + // SAFETY: `ptr` points to valid uninitialised memory for a `srcu_struct`. | |
| 57 | + to_result(unsafe { | |
| 58 | + bindings::init_srcu_struct_with_key(ptr, name.as_char_ptr(), key.as_ptr()) | |
| 59 | + }) | |
| 60 | + }), | |
| 61 | + }) | |
| 62 | + } | |
| 63 | + | |
| 64 | + /// Enters an SRCU read-side critical section. | |
| 65 | + /// | |
| 66 | + /// Leaking the returned [`Guard`] leaves the SRCU read-side critical | |
| 67 | + /// section active and makes `drop` sleep forever. | |
| 68 | + #[inline] | |
| 69 | + pub fn read_lock(&self) -> Guard<'_> { | |
| 70 | + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. | |
| 71 | + let idx = unsafe { bindings::srcu_read_lock(self.inner.get()) }; | |
| 72 | + | |
| 73 | + // INVARIANT: `idx` was returned by `srcu_read_lock()` for this `Srcu`. | |
| 74 | + Guard { | |
| 75 | + srcu: self, | |
| 76 | + idx, | |
| 77 | + _not_send: NotThreadSafe, | |
| 78 | + } | |
| 79 | + } | |
| 80 | + | |
| 81 | + /// Waits until all pre-existing SRCU readers have completed. | |
| 82 | + #[inline] | |
| 83 | + pub fn synchronize(&self) { | |
| 84 | + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. | |
| 85 | + unsafe { bindings::synchronize_srcu(self.inner.get()) }; | |
| 86 | + } | |
| 87 | + | |
| 88 | + /// Waits until all pre-existing SRCU readers have completed, expedited. | |
| 89 | + /// | |
| 90 | + /// This requests a lower-latency grace period than [`Srcu::synchronize`] typically | |
| 91 | + /// at the cost of higher system-wide overhead. Prefer [`Srcu::synchronize`] by default | |
| 92 | + /// and use this variant only when reducing reset or teardown latency is more important | |
| 93 | + /// than the extra cost. | |
| 94 | + #[inline] | |
| 95 | + pub fn synchronize_expedited(&self) { | |
| 96 | + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. | |
| 97 | + unsafe { bindings::synchronize_srcu_expedited(self.inner.get()) }; | |
| 98 | + } | |
| 99 | +} | |
| 100 | + | |
| 101 | +#[pinned_drop] | |
| 102 | +impl PinnedDrop for Srcu { | |
| 103 | + fn drop(self: Pin<&mut Self>) { | |
| 104 | + let ptr = self.inner.get(); | |
| 105 | + | |
| 106 | + if crate::warn_on!( | |
| 107 | + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct` | |
| 108 | + // and `srcu_readers_active()` only checks the active reader count. | |
| 109 | + unsafe { bindings::srcu_readers_active(ptr) } | |
| 110 | + ) { | |
| 111 | + // `cleanup_srcu_struct()` may return early if there are still active readers. | |
| 112 | + // This should only happen if a guard was leaked with `mem::forget`, which is | |
| 113 | + // "WRONG" code and may cause a UAF because Rust will free the `srcu_struct` | |
| 114 | + // while it is still referenced from the C side (e.g. by `call_srcu()` callbacks). | |
| 115 | + // | |
| 116 | + // Another consequence of leaking guards is that `call_srcu()` callbacks will | |
| 117 | + // never run because the grace period can never complete due to permanently | |
| 118 | + // active readers (i.e. leaked guards). | |
| 119 | + // | |
| 120 | + // If this ever happens, that means the guard was leaked by mistake and the | |
| 121 | + // caller must fix the bug. Sleeping here is intentional and less harmful | |
| 122 | + // than risking a UAF. | |
| 123 | + // | |
| 124 | + // SAFETY: By the type invariants, `self` contains a valid and pinned | |
| 125 | + // `struct srcu_struct`. | |
| 126 | + unsafe { bindings::synchronize_srcu(ptr) }; | |
| 127 | + } | |
| 128 | + | |
| 129 | + // Ensure all SRCU callbacks have been finished before freeing. | |
| 130 | + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. | |
| 131 | + unsafe { bindings::srcu_barrier(ptr) }; | |
| 132 | + | |
| 133 | + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. | |
| 134 | + unsafe { bindings::cleanup_srcu_struct(ptr) }; | |
| 135 | + } | |
| 136 | +} | |
| 137 | + | |
| 138 | +// SAFETY: `srcu_struct` may be shared and used across threads. | |
| 139 | +unsafe impl Send for Srcu {} | |
| 140 | +// SAFETY: `srcu_struct` may be shared and used concurrently. | |
| 141 | +unsafe impl Sync for Srcu {} | |
| 142 | + | |
| 143 | +/// Guard for an active SRCU read-side critical section on a particular [`Srcu`]. | |
| 144 | +/// | |
| 145 | +/// Leaking this guard with [`core::mem::forget`] leaves the SRCU read-side | |
| 146 | +/// critical section active and makes dropping the associated [`Srcu`] sleep forever. | |
| 147 | +/// | |
| 148 | +/// # Invariants | |
| 149 | +/// | |
| 150 | +/// `idx` is the index returned by `srcu_read_lock()` for `srcu`. | |
| 151 | +#[must_use = "if unused, the lock will be immediately unlocked"] | |
| 152 | +pub struct Guard<'a> { | |
| 153 | + srcu: &'a Srcu, | |
| 154 | + idx: i32, | |
| 155 | + _not_send: NotThreadSafe, | |
| 156 | +} | |
| 157 | + | |
| 158 | +impl Guard<'_> { | |
| 159 | + /// Explicitly releases the SRCU read-side critical section. | |
| 160 | + #[inline] | |
| 161 | + pub fn unlock(self) {} | |
| 162 | +} | |
| 163 | + | |
| 164 | +impl Drop for Guard<'_> { | |
| 165 | + #[inline] | |
| 166 | + fn drop(&mut self) { | |
| 167 | + // SAFETY: `Guard` is only constructible through `Srcu::read_lock()`, | |
| 168 | + // which returns a valid index for the SRCU instance. | |
| 169 | + unsafe { bindings::srcu_read_unlock(self.srcu.inner.get(), self.idx) }; | |
| 170 | + } | |
| 171 | +} | |