History

Merge tag 'driver-core-6.19-rc7' of git://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core

Pull driver core fixes from Danilo Krummrich: - Always inline I/O and IRQ methods using build_assert!() to avoid false positive build errors - Do not free the driver's device private data in I2C shutdown() avoiding race conditions that can lead to UAF bugs - Drop the driver's device private data after the driver has been fully unbound from its device to avoid UAF bugs from &Device<Bound> scopes, such as IRQ callbacks * tag 'driver-core-6.19-rc7' of git://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core: rust: driver: drop device private data post unbind rust: driver: add DriverData type to the DriverLayout trait rust: driver: add DEVICE_DRIVER_OFFSET to the DriverLayout trait rust: driver: introduce a DriverLayout trait rust: auxiliary: add Driver::unbind() callback rust: i2c: do not drop device private data on shutdown() rust: irq: always inline functions using build_assert with arguments rust: io: always inline functions using build_assert with arguments

Linus Torvalds committed 9 months ago

Commit 5dbeeb22 parents 12a0094 a995fe112 files · +209 −74Browse files

Unified · Split

12 files
  1. drivers/base/dd.c +2 −0
  2. include/linux/device/driver.h +9 −0
  3. rust/kernel/auxiliary.rs +33 −8
  4. rust/kernel/device.rs +11 −9
  5. rust/kernel/driver.rs +70 −16
  6. rust/kernel/i2c.rs +20 −11
  7. rust/kernel/io.rs +6 −3
  8. rust/kernel/io/resource.rs +2 −0
  9. rust/kernel/irq/flags.rs +2 −0
  10. rust/kernel/pci.rs +18 −9
  11. rust/kernel/platform.rs +18 −9
  12. rust/kernel/usb.rs +18 −9
drivers/base/dd.c+2−0
@@ -548,6 +548,8 @@ static DEVICE_ATTR_RW(state_synced); · 547 unmodified lines
548static void device_unbind_cleanup(struct device *dev)548static void device_unbind_cleanup(struct device *dev)
549{549{
550 devres_release_all(dev);550 devres_release_all(dev);
551 if (dev->driver->p_cb.post_unbind_rust)
552 dev->driver->p_cb.post_unbind_rust(dev);
551 arch_teardown_dma_ops(dev);553 arch_teardown_dma_ops(dev);
552 kfree(dev->dma_range_map);554 kfree(dev->dma_range_map);
553 dev->dma_range_map = NULL;555 dev->dma_range_map = NULL;
include/linux/device/driver.h+9−0
@@ -85,6 +85,8 @@ enum probe_type { · 84 unmodified lines
85 * uevent.85 * uevent.
86 * @p: Driver core's private data, no one other than the driver86 * @p: Driver core's private data, no one other than the driver
87 * core can touch this.87 * core can touch this.
88 * @p_cb: Callbacks private to the driver core; no one other than the
89 * driver core is allowed to touch this.
88 *90 *
89 * The device driver-model tracks all of the drivers known to the system.91 * The device driver-model tracks all of the drivers known to the system.
90 * The main reason for this tracking is to enable the driver core to match92 * The main reason for this tracking is to enable the driver core to match
@@ -119,6 +121,13 @@ struct device_driver { · 28 unmodified lines
119 void (*coredump) (struct device *dev);121 void (*coredump) (struct device *dev);
120 122
121 struct driver_private *p;123 struct driver_private *p;
124 struct {
125 /*
126 * Called after remove() and after all devres entries have been
127 * processed. This is a Rust only callback.
128 */
129 void (*post_unbind_rust)(struct device *dev);
130 } p_cb;
122};131};
123 132
124 133
rust/kernel/auxiliary.rs+33−8
@@ -23,13 +23,22 @@ use core::{ · 22 unmodified lines
23/// An adapter for the registration of auxiliary drivers.23/// An adapter for the registration of auxiliary drivers.
24pub struct Adapter<T: Driver>(T);24pub struct Adapter<T: Driver>(T);
25 25
26// SAFETY: A call to `unregister` for a given instance of `RegType` is guaranteed to be valid if26// SAFETY:
27// - `bindings::auxiliary_driver` is a C type declared as `repr(C)`.
28// - `T` is the type of the driver's device private data.
29// - `struct auxiliary_driver` embeds a `struct device_driver`.
30// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`.
31unsafe impl<T: Driver + 'static> driver::DriverLayout for Adapter<T> {
32 type DriverType = bindings::auxiliary_driver;
33 type DriverData = T;
34 const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver);
35}
36
37// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if
27// a preceding call to `register` has been successful.38// a preceding call to `register` has been successful.
28unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> {39unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> {
29 type RegType = bindings::auxiliary_driver;
30
31 unsafe fn register(40 unsafe fn register(
32 adrv: &Opaque<Self::RegType>,41 adrv: &Opaque<Self::DriverType>,
33 name: &'static CStr,42 name: &'static CStr,
34 module: &'static ThisModule,43 module: &'static ThisModule,
35 ) -> Result {44 ) -> Result {
@@ -41,14 +50,14 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { · 5 unmodified lines
41 (*adrv.get()).id_table = T::ID_TABLE.as_ptr();50 (*adrv.get()).id_table = T::ID_TABLE.as_ptr();
42 }51 }
43 52
44 // SAFETY: `adrv` is guaranteed to be a valid `RegType`.53 // SAFETY: `adrv` is guaranteed to be a valid `DriverType`.
45 to_result(unsafe {54 to_result(unsafe {
46 bindings::__auxiliary_driver_register(adrv.get(), module.0, name.as_char_ptr())55 bindings::__auxiliary_driver_register(adrv.get(), module.0, name.as_char_ptr())
47 })56 })
48 }57 }
49 58
50 unsafe fn unregister(adrv: &Opaque<Self::RegType>) {59 unsafe fn unregister(adrv: &Opaque<Self::DriverType>) {
51 // SAFETY: `adrv` is guaranteed to be a valid `RegType`.60 // SAFETY: `adrv` is guaranteed to be a valid `DriverType`.
52 unsafe { bindings::auxiliary_driver_unregister(adrv.get()) }61 unsafe { bindings::auxiliary_driver_unregister(adrv.get()) }
53 }62 }
54}63}
@@ -87,7 +96,9 @@ impl<T: Driver + 'static> Adapter<T> { · 32 unmodified lines
87 // SAFETY: `remove_callback` is only ever called after a successful call to96 // SAFETY: `remove_callback` is only ever called after a successful call to
88 // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called97 // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called
89 // and stored a `Pin<KBox<T>>`.98 // and stored a `Pin<KBox<T>>`.
90 drop(unsafe { adev.as_ref().drvdata_obtain::<T>() });99 let data = unsafe { adev.as_ref().drvdata_borrow::<T>() };
100
101 T::unbind(adev, data);
91 }102 }
92}103}
93 104
@@ -187,6 +198,20 @@ pub trait Driver { · 93 unmodified lines
187 ///198 ///
188 /// Called when an auxiliary device is matches a corresponding driver.199 /// Called when an auxiliary device is matches a corresponding driver.
189 fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> impl PinInit<Self, Error>;200 fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> impl PinInit<Self, Error>;
201
202 /// Auxiliary driver unbind.
203 ///
204 /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback
205 /// is optional.
206 ///
207 /// This callback serves as a place for drivers to perform teardown operations that require a
208 /// `&Device<Core>` or `&Device<Bound>` reference. For instance, drivers may try to perform I/O
209 /// operations to gracefully tear down the device.
210 ///
211 /// Otherwise, release operations for driver resources should be performed in `Self::drop`.
212 fn unbind(dev: &Device<device::Core>, this: Pin<&Self>) {
213 let _ = (dev, this);
214 }
190}215}
191 216
192/// The auxiliary device representation.217/// The auxiliary device representation.
rust/kernel/device.rs+11−9
@@ -232,30 +232,32 @@ impl Device<CoreInternal> { · 231 unmodified lines
232 ///232 ///
233 /// # Safety233 /// # Safety
234 ///234 ///
235 /// - Must only be called once after a preceding call to [`Device::set_drvdata`].
236 /// - The type `T` must match the type of the `ForeignOwnable` previously stored by235 /// - The type `T` must match the type of the `ForeignOwnable` previously stored by
237 /// [`Device::set_drvdata`].236 /// [`Device::set_drvdata`].
238 pub unsafe fn drvdata_obtain<T: 'static>(&self) -> Pin<KBox<T>> {237 pub(crate) unsafe fn drvdata_obtain<T: 'static>(&self) -> Option<Pin<KBox<T>>> {
239 // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`.238 // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`.
240 let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) };239 let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) };
241 240
242 // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`.241 // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`.
243 unsafe { bindings::dev_set_drvdata(self.as_raw(), core::ptr::null_mut()) };242 unsafe { bindings::dev_set_drvdata(self.as_raw(), core::ptr::null_mut()) };
244 243
244 if ptr.is_null() {
245 return None;
246 }
247
245 // SAFETY:248 // SAFETY:
246 // - By the safety requirements of this function, `ptr` comes from a previous call to249 // - If `ptr` is not NULL, it comes from a previous call to `into_foreign()`.
247 // `into_foreign()`.
248 // - `dev_get_drvdata()` guarantees to return the same pointer given to `dev_set_drvdata()`250 // - `dev_get_drvdata()` guarantees to return the same pointer given to `dev_set_drvdata()`
249 // in `into_foreign()`.251 // in `into_foreign()`.
250 unsafe { Pin::<KBox<T>>::from_foreign(ptr.cast()) }252 Some(unsafe { Pin::<KBox<T>>::from_foreign(ptr.cast()) })
251 }253 }
252 254
253 /// Borrow the driver's private data bound to this [`Device`].255 /// Borrow the driver's private data bound to this [`Device`].
254 ///256 ///
255 /// # Safety257 /// # Safety
256 ///258 ///
257 /// - Must only be called after a preceding call to [`Device::set_drvdata`] and before259 /// - Must only be called after a preceding call to [`Device::set_drvdata`] and before the
258 /// [`Device::drvdata_obtain`].260 /// device is fully unbound.
259 /// - The type `T` must match the type of the `ForeignOwnable` previously stored by261 /// - The type `T` must match the type of the `ForeignOwnable` previously stored by
260 /// [`Device::set_drvdata`].262 /// [`Device::set_drvdata`].
261 pub unsafe fn drvdata_borrow<T: 'static>(&self) -> Pin<&T> {263 pub unsafe fn drvdata_borrow<T: 'static>(&self) -> Pin<&T> {
@@ -271,7 +273,7 @@ impl Device<Bound> { · 9 unmodified lines
271 /// # Safety273 /// # Safety
272 ///274 ///
273 /// - Must only be called after a preceding call to [`Device::set_drvdata`] and before275 /// - Must only be called after a preceding call to [`Device::set_drvdata`] and before
274 /// [`Device::drvdata_obtain`].276 /// the device is fully unbound.
275 /// - The type `T` must match the type of the `ForeignOwnable` previously stored by277 /// - The type `T` must match the type of the `ForeignOwnable` previously stored by
276 /// [`Device::set_drvdata`].278 /// [`Device::set_drvdata`].
277 unsafe fn drvdata_unchecked<T: 'static>(&self) -> Pin<&T> {279 unsafe fn drvdata_unchecked<T: 'static>(&self) -> Pin<&T> {
@@ -320,7 +322,7 @@ impl Device<Bound> { · 42 unmodified lines
320 322
321 // SAFETY:323 // SAFETY:
322 // - The above check of `dev_get_drvdata()` guarantees that we are called after324 // - The above check of `dev_get_drvdata()` guarantees that we are called after
323 // `set_drvdata()` and before `drvdata_obtain()`.325 // `set_drvdata()`.
324 // - We've just checked that the type of the driver's private data is in fact `T`.326 // - We've just checked that the type of the driver's private data is in fact `T`.
325 Ok(unsafe { self.drvdata_unchecked() })327 Ok(unsafe { self.drvdata_unchecked() })
326 }328 }
rust/kernel/driver.rs+70−16
@@ -99,23 +99,43 @@ use crate::{acpi, device, of, str::CStr, try_pin_init, types::Opaque, ThisModule · 98 unmodified lines
99use core::pin::Pin;99use core::pin::Pin;
100use pin_init::{pin_data, pinned_drop, PinInit};100use pin_init::{pin_data, pinned_drop, PinInit};
101 101
102/// Trait describing the layout of a specific device driver.
103///
104/// This trait describes the layout of a specific driver structure, such as `struct pci_driver` or
105/// `struct platform_driver`.
106///
107/// # Safety
108///
109/// Implementors must guarantee that:
110/// - `DriverType` is `repr(C)`,
111/// - `DriverData` is the type of the driver's device private data.
112/// - `DriverType` embeds a valid `struct device_driver` at byte offset `DEVICE_DRIVER_OFFSET`.
113pub unsafe trait DriverLayout {
114 /// The specific driver type embedding a `struct device_driver`.
115 type DriverType: Default;
116
117 /// The type of the driver's device private data.
118 type DriverData;
119
120 /// Byte offset of the embedded `struct device_driver` within `DriverType`.
121 ///
122 /// This must correspond exactly to the location of the embedded `struct device_driver` field.
123 const DEVICE_DRIVER_OFFSET: usize;
124}
125
102/// The [`RegistrationOps`] trait serves as generic interface for subsystems (e.g., PCI, Platform,126/// The [`RegistrationOps`] trait serves as generic interface for subsystems (e.g., PCI, Platform,
103/// Amba, etc.) to provide the corresponding subsystem specific implementation to register /127/// Amba, etc.) to provide the corresponding subsystem specific implementation to register /
104/// unregister a driver of the particular type (`RegType`).128/// unregister a driver of the particular type (`DriverType`).
105///129///
106/// For instance, the PCI subsystem would set `RegType` to `bindings::pci_driver` and call130/// For instance, the PCI subsystem would set `DriverType` to `bindings::pci_driver` and call
107/// `bindings::__pci_register_driver` from `RegistrationOps::register` and131/// `bindings::__pci_register_driver` from `RegistrationOps::register` and
108/// `bindings::pci_unregister_driver` from `RegistrationOps::unregister`.132/// `bindings::pci_unregister_driver` from `RegistrationOps::unregister`.
109///133///
110/// # Safety134/// # Safety
111///135///
112/// A call to [`RegistrationOps::unregister`] for a given instance of `RegType` is only valid if a136/// A call to [`RegistrationOps::unregister`] for a given instance of `DriverType` is only valid if
113/// preceding call to [`RegistrationOps::register`] has been successful.137/// a preceding call to [`RegistrationOps::register`] has been successful.
114pub unsafe trait RegistrationOps {138pub unsafe trait RegistrationOps: DriverLayout {
115 /// The type that holds information about the registration. This is typically a struct defined
116 /// by the C portion of the kernel.
117 type RegType: Default;
118
119 /// Registers a driver.139 /// Registers a driver.
120 ///140 ///
121 /// # Safety141 /// # Safety
@@ -123,7 +143,7 @@ pub unsafe trait RegistrationOps { · 1 unmodified line
123 /// On success, `reg` must remain pinned and valid until the matching call to143 /// On success, `reg` must remain pinned and valid until the matching call to
124 /// [`RegistrationOps::unregister`].144 /// [`RegistrationOps::unregister`].
125 unsafe fn register(145 unsafe fn register(
126 reg: &Opaque<Self::RegType>,146 reg: &Opaque<Self::DriverType>,
127 name: &'static CStr,147 name: &'static CStr,
128 module: &'static ThisModule,148 module: &'static ThisModule,
129 ) -> Result;149 ) -> Result;
@@ -134,7 +154,7 @@ pub unsafe trait RegistrationOps { · 4 unmodified lines
134 ///154 ///
135 /// Must only be called after a preceding successful call to [`RegistrationOps::register`] for155 /// Must only be called after a preceding successful call to [`RegistrationOps::register`] for
136 /// the same `reg`.156 /// the same `reg`.
137 unsafe fn unregister(reg: &Opaque<Self::RegType>);157 unsafe fn unregister(reg: &Opaque<Self::DriverType>);
138}158}
139 159
140/// A [`Registration`] is a generic type that represents the registration of some driver type (e.g.160/// A [`Registration`] is a generic type that represents the registration of some driver type (e.g.
@@ -146,7 +166,7 @@ pub unsafe trait RegistrationOps { · 5 unmodified lines
146#[pin_data(PinnedDrop)]166#[pin_data(PinnedDrop)]
147pub struct Registration<T: RegistrationOps> {167pub struct Registration<T: RegistrationOps> {
148 #[pin]168 #[pin]
149 reg: Opaque<T::RegType>,169 reg: Opaque<T::DriverType>,
150}170}
151 171
152// SAFETY: `Registration` has no fields or methods accessible via `&Registration`, so it is safe to172// SAFETY: `Registration` has no fields or methods accessible via `&Registration`, so it is safe to
@@ -157,17 +177,51 @@ unsafe impl<T: RegistrationOps> Sync for Registration<T> {} · 4 unmodified lines
157// any thread, so `Registration` is `Send`.177// any thread, so `Registration` is `Send`.
158unsafe impl<T: RegistrationOps> Send for Registration<T> {}178unsafe impl<T: RegistrationOps> Send for Registration<T> {}
159 179
160impl<T: RegistrationOps> Registration<T> {180impl<T: RegistrationOps + 'static> Registration<T> {
181 extern "C" fn post_unbind_callback(dev: *mut bindings::device) {
182 // SAFETY: The driver core only ever calls the post unbind callback with a valid pointer to
183 // a `struct device`.
184 //
185 // INVARIANT: `dev` is valid for the duration of the `post_unbind_callback()`.
186 let dev = unsafe { &*dev.cast::<device::Device<device::CoreInternal>>() };
187
188 // `remove()` and all devres callbacks have been completed at this point, hence drop the
189 // driver's device private data.
190 //
191 // SAFETY: By the safety requirements of the `Driver` trait, `T::DriverData` is the
192 // driver's device private data type.
193 drop(unsafe { dev.drvdata_obtain::<T::DriverData>() });
194 }
195
196 /// Attach generic `struct device_driver` callbacks.
197 fn callbacks_attach(drv: &Opaque<T::DriverType>) {
198 let ptr = drv.get().cast::<u8>();
199
200 // SAFETY:
201 // - `drv.get()` yields a valid pointer to `Self::DriverType`.
202 // - Adding `DEVICE_DRIVER_OFFSET` yields the address of the embedded `struct device_driver`
203 // as guaranteed by the safety requirements of the `Driver` trait.
204 let base = unsafe { ptr.add(T::DEVICE_DRIVER_OFFSET) };
205
206 // CAST: `base` points to the offset of the embedded `struct device_driver`.
207 let base = base.cast::<bindings::device_driver>();
208
209 // SAFETY: It is safe to set the fields of `struct device_driver` on initialization.
210 unsafe { (*base).p_cb.post_unbind_rust = Some(Self::post_unbind_callback) };
211 }
212
161 /// Creates a new instance of the registration object.213 /// Creates a new instance of the registration object.
162 pub fn new(name: &'static CStr, module: &'static ThisModule) -> impl PinInit<Self, Error> {214 pub fn new(name: &'static CStr, module: &'static ThisModule) -> impl PinInit<Self, Error> {
163 try_pin_init!(Self {215 try_pin_init!(Self {
164 reg <- Opaque::try_ffi_init(|ptr: *mut T::RegType| {216 reg <- Opaque::try_ffi_init(|ptr: *mut T::DriverType| {
165 // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write.217 // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write.
166 unsafe { ptr.write(T::RegType::default()) };218 unsafe { ptr.write(T::DriverType::default()) };
167 219
168 // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write, and it has220 // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write, and it has
169 // just been initialised above, so it's also valid for read.221 // just been initialised above, so it's also valid for read.
170 let drv = unsafe { &*(ptr as *const Opaque<T::RegType>) };222 let drv = unsafe { &*(ptr as *const Opaque<T::DriverType>) };
223
224 Self::callbacks_attach(drv);
171 225
172 // SAFETY: `drv` is guaranteed to be pinned until `T::unregister`.226 // SAFETY: `drv` is guaranteed to be pinned until `T::unregister`.
173 unsafe { T::register(drv, name, module) }227 unsafe { T::register(drv, name, module) }
rust/kernel/i2c.rs+20−11
@@ -92,13 +92,22 @@ macro_rules! i2c_device_table { · 91 unmodified lines
92/// An adapter for the registration of I2C drivers.92/// An adapter for the registration of I2C drivers.
93pub struct Adapter<T: Driver>(T);93pub struct Adapter<T: Driver>(T);
94 94
95// SAFETY: A call to `unregister` for a given instance of `RegType` is guaranteed to be valid if95// SAFETY:
96// - `bindings::i2c_driver` is a C type declared as `repr(C)`.
97// - `T` is the type of the driver's device private data.
98// - `struct i2c_driver` embeds a `struct device_driver`.
99// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`.
100unsafe impl<T: Driver + 'static> driver::DriverLayout for Adapter<T> {
101 type DriverType = bindings::i2c_driver;
102 type DriverData = T;
103 const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver);
104}
105
106// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if
96// a preceding call to `register` has been successful.107// a preceding call to `register` has been successful.
97unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> {108unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> {
98 type RegType = bindings::i2c_driver;
99
100 unsafe fn register(109 unsafe fn register(
101 idrv: &Opaque<Self::RegType>,110 idrv: &Opaque<Self::DriverType>,
102 name: &'static CStr,111 name: &'static CStr,
103 module: &'static ThisModule,112 module: &'static ThisModule,
104 ) -> Result {113 ) -> Result {
@@ -133,12 +142,12 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { · 28 unmodified lines
133 (*idrv.get()).driver.acpi_match_table = acpi_table;142 (*idrv.get()).driver.acpi_match_table = acpi_table;
134 }143 }
135 144
136 // SAFETY: `idrv` is guaranteed to be a valid `RegType`.145 // SAFETY: `idrv` is guaranteed to be a valid `DriverType`.
137 to_result(unsafe { bindings::i2c_register_driver(module.0, idrv.get()) })146 to_result(unsafe { bindings::i2c_register_driver(module.0, idrv.get()) })
138 }147 }
139 148
140 unsafe fn unregister(idrv: &Opaque<Self::RegType>) {149 unsafe fn unregister(idrv: &Opaque<Self::DriverType>) {
141 // SAFETY: `idrv` is guaranteed to be a valid `RegType`.150 // SAFETY: `idrv` is guaranteed to be a valid `DriverType`.
142 unsafe { bindings::i2c_del_driver(idrv.get()) }151 unsafe { bindings::i2c_del_driver(idrv.get()) }
143 }152 }
144}153}
@@ -169,9 +178,9 @@ impl<T: Driver + 'static> Adapter<T> { · 24 unmodified lines
169 // SAFETY: `remove_callback` is only ever called after a successful call to178 // SAFETY: `remove_callback` is only ever called after a successful call to
170 // `probe_callback`, hence it's guaranteed that `I2cClient::set_drvdata()` has been called179 // `probe_callback`, hence it's guaranteed that `I2cClient::set_drvdata()` has been called
171 // and stored a `Pin<KBox<T>>`.180 // and stored a `Pin<KBox<T>>`.
172 let data = unsafe { idev.as_ref().drvdata_obtain::<T>() };181 let data = unsafe { idev.as_ref().drvdata_borrow::<T>() };
173 182
174 T::unbind(idev, data.as_ref());183 T::unbind(idev, data);
175 }184 }
176 185
177 extern "C" fn shutdown_callback(idev: *mut bindings::i2c_client) {186 extern "C" fn shutdown_callback(idev: *mut bindings::i2c_client) {
@@ -181,9 +190,9 @@ impl<T: Driver + 'static> Adapter<T> { · 3 unmodified lines
181 // SAFETY: `shutdown_callback` is only ever called after a successful call to190 // SAFETY: `shutdown_callback` is only ever called after a successful call to
182 // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called191 // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called
183 // and stored a `Pin<KBox<T>>`.192 // and stored a `Pin<KBox<T>>`.
184 let data = unsafe { idev.as_ref().drvdata_obtain::<T>() };193 let data = unsafe { idev.as_ref().drvdata_borrow::<T>() };
185 194
186 T::shutdown(idev, data.as_ref());195 T::shutdown(idev, data);
187 }196 }
188 197
189 /// The [`i2c::IdTable`] of the corresponding driver.198 /// The [`i2c::IdTable`] of the corresponding driver.
rust/kernel/io.rs+6−3
@@ -142,7 +142,8 @@ macro_rules! define_read { · 141 unmodified lines
142 /// Bound checks are performed on compile time, hence if the offset is not known at compile142 /// Bound checks are performed on compile time, hence if the offset is not known at compile
143 /// time, the build will fail.143 /// time, the build will fail.
144 $(#[$attr])*144 $(#[$attr])*
145 #[inline]145 // Always inline to optimize out error path of `io_addr_assert`.
146 #[inline(always)]
146 pub fn $name(&self, offset: usize) -> $type_name {147 pub fn $name(&self, offset: usize) -> $type_name {
147 let addr = self.io_addr_assert::<$type_name>(offset);148 let addr = self.io_addr_assert::<$type_name>(offset);
148 149
@@ -171,7 +172,8 @@ macro_rules! define_write { · 22 unmodified lines
171 /// Bound checks are performed on compile time, hence if the offset is not known at compile172 /// Bound checks are performed on compile time, hence if the offset is not known at compile
172 /// time, the build will fail.173 /// time, the build will fail.
173 $(#[$attr])*174 $(#[$attr])*
174 #[inline]175 // Always inline to optimize out error path of `io_addr_assert`.
176 #[inline(always)]
175 pub fn $name(&self, value: $type_name, offset: usize) {177 pub fn $name(&self, value: $type_name, offset: usize) {
176 let addr = self.io_addr_assert::<$type_name>(offset);178 let addr = self.io_addr_assert::<$type_name>(offset);
177 179
@@ -239,7 +241,8 @@ impl<const SIZE: usize> Io<SIZE> { · 61 unmodified lines
239 self.addr().checked_add(offset).ok_or(EINVAL)241 self.addr().checked_add(offset).ok_or(EINVAL)
240 }242 }
241 243
242 #[inline]244 // Always inline to optimize out error path of `build_assert`.
245 #[inline(always)]
243 fn io_addr_assert<U>(&self, offset: usize) -> usize {246 fn io_addr_assert<U>(&self, offset: usize) -> usize {
244 build_assert!(Self::offset_valid::<U>(offset, SIZE));247 build_assert!(Self::offset_valid::<U>(offset, SIZE));
245 248
rust/kernel/io/resource.rs+2−0
@@ -226,6 +226,8 @@ impl Flags { · 225 unmodified lines
226 /// Resource represents a memory region that must be ioremaped using `ioremap_np`.226 /// Resource represents a memory region that must be ioremaped using `ioremap_np`.
227 pub const IORESOURCE_MEM_NONPOSTED: Flags = Flags::new(bindings::IORESOURCE_MEM_NONPOSTED);227 pub const IORESOURCE_MEM_NONPOSTED: Flags = Flags::new(bindings::IORESOURCE_MEM_NONPOSTED);
228 228
229 // Always inline to optimize out error path of `build_assert`.
230 #[inline(always)]
229 const fn new(value: u32) -> Self {231 const fn new(value: u32) -> Self {
230 crate::build_assert!(value as u64 <= c_ulong::MAX as u64);232 crate::build_assert!(value as u64 <= c_ulong::MAX as u64);
231 Flags(value as c_ulong)233 Flags(value as c_ulong)
rust/kernel/irq/flags.rs+2−0
@@ -96,6 +96,8 @@ impl Flags { · 95 unmodified lines
96 self.096 self.0
97 }97 }
98 98
99 // Always inline to optimize out error path of `build_assert`.
100 #[inline(always)]
99 const fn new(value: u32) -> Self {101 const fn new(value: u32) -> Self {
100 build_assert!(value as u64 <= c_ulong::MAX as u64);102 build_assert!(value as u64 <= c_ulong::MAX as u64);
101 Self(value as c_ulong)103 Self(value as c_ulong)
rust/kernel/pci.rs+18−9
@@ -50,13 +50,22 @@ pub use self::irq::{ · 49 unmodified lines
50/// An adapter for the registration of PCI drivers.50/// An adapter for the registration of PCI drivers.
51pub struct Adapter<T: Driver>(T);51pub struct Adapter<T: Driver>(T);
52 52
53// SAFETY: A call to `unregister` for a given instance of `RegType` is guaranteed to be valid if53// SAFETY:
54// - `bindings::pci_driver` is a C type declared as `repr(C)`.
55// - `T` is the type of the driver's device private data.
56// - `struct pci_driver` embeds a `struct device_driver`.
57// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`.
58unsafe impl<T: Driver + 'static> driver::DriverLayout for Adapter<T> {
59 type DriverType = bindings::pci_driver;
60 type DriverData = T;
61 const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver);
62}
63
64// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if
54// a preceding call to `register` has been successful.65// a preceding call to `register` has been successful.
55unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> {66unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> {
56 type RegType = bindings::pci_driver;
57
58 unsafe fn register(67 unsafe fn register(
59 pdrv: &Opaque<Self::RegType>,68 pdrv: &Opaque<Self::DriverType>,
60 name: &'static CStr,69 name: &'static CStr,
61 module: &'static ThisModule,70 module: &'static ThisModule,
62 ) -> Result {71 ) -> Result {
@@ -68,14 +77,14 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { · 5 unmodified lines
68 (*pdrv.get()).id_table = T::ID_TABLE.as_ptr();77 (*pdrv.get()).id_table = T::ID_TABLE.as_ptr();
69 }78 }
70 79
71 // SAFETY: `pdrv` is guaranteed to be a valid `RegType`.80 // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`.
72 to_result(unsafe {81 to_result(unsafe {
73 bindings::__pci_register_driver(pdrv.get(), module.0, name.as_char_ptr())82 bindings::__pci_register_driver(pdrv.get(), module.0, name.as_char_ptr())
74 })83 })
75 }84 }
76 85
77 unsafe fn unregister(pdrv: &Opaque<Self::RegType>) {86 unsafe fn unregister(pdrv: &Opaque<Self::DriverType>) {
78 // SAFETY: `pdrv` is guaranteed to be a valid `RegType`.87 // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`.
79 unsafe { bindings::pci_unregister_driver(pdrv.get()) }88 unsafe { bindings::pci_unregister_driver(pdrv.get()) }
80 }89 }
81}90}
@@ -114,9 +123,9 @@ impl<T: Driver + 'static> Adapter<T> { · 32 unmodified lines
114 // SAFETY: `remove_callback` is only ever called after a successful call to123 // SAFETY: `remove_callback` is only ever called after a successful call to
115 // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called124 // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called
116 // and stored a `Pin<KBox<T>>`.125 // and stored a `Pin<KBox<T>>`.
117 let data = unsafe { pdev.as_ref().drvdata_obtain::<T>() };126 let data = unsafe { pdev.as_ref().drvdata_borrow::<T>() };
118 127
119 T::unbind(pdev, data.as_ref());128 T::unbind(pdev, data);
120 }129 }
121}130}
122 131
rust/kernel/platform.rs+18−9
@@ -26,13 +26,22 @@ use core::{ · 25 unmodified lines
26/// An adapter for the registration of platform drivers.26/// An adapter for the registration of platform drivers.
27pub struct Adapter<T: Driver>(T);27pub struct Adapter<T: Driver>(T);
28 28
29// SAFETY: A call to `unregister` for a given instance of `RegType` is guaranteed to be valid if29// SAFETY:
30// - `bindings::platform_driver` is a C type declared as `repr(C)`.
31// - `T` is the type of the driver's device private data.
32// - `struct platform_driver` embeds a `struct device_driver`.
33// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`.
34unsafe impl<T: Driver + 'static> driver::DriverLayout for Adapter<T> {
35 type DriverType = bindings::platform_driver;
36 type DriverData = T;
37 const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver);
38}
39
40// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if
30// a preceding call to `register` has been successful.41// a preceding call to `register` has been successful.
31unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> {42unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> {
32 type RegType = bindings::platform_driver;
33
34 unsafe fn register(43 unsafe fn register(
35 pdrv: &Opaque<Self::RegType>,44 pdrv: &Opaque<Self::DriverType>,
36 name: &'static CStr,45 name: &'static CStr,
37 module: &'static ThisModule,46 module: &'static ThisModule,
38 ) -> Result {47 ) -> Result {
@@ -55,12 +64,12 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { · 16 unmodified lines
55 (*pdrv.get()).driver.acpi_match_table = acpi_table;64 (*pdrv.get()).driver.acpi_match_table = acpi_table;
56 }65 }
57 66
58 // SAFETY: `pdrv` is guaranteed to be a valid `RegType`.67 // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`.
59 to_result(unsafe { bindings::__platform_driver_register(pdrv.get(), module.0) })68 to_result(unsafe { bindings::__platform_driver_register(pdrv.get(), module.0) })
60 }69 }
61 70
62 unsafe fn unregister(pdrv: &Opaque<Self::RegType>) {71 unsafe fn unregister(pdrv: &Opaque<Self::DriverType>) {
63 // SAFETY: `pdrv` is guaranteed to be a valid `RegType`.72 // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`.
64 unsafe { bindings::platform_driver_unregister(pdrv.get()) };73 unsafe { bindings::platform_driver_unregister(pdrv.get()) };
65 }74 }
66}75}
@@ -92,9 +101,9 @@ impl<T: Driver + 'static> Adapter<T> { · 25 unmodified lines
92 // SAFETY: `remove_callback` is only ever called after a successful call to101 // SAFETY: `remove_callback` is only ever called after a successful call to
93 // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called102 // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called
94 // and stored a `Pin<KBox<T>>`.103 // and stored a `Pin<KBox<T>>`.
95 let data = unsafe { pdev.as_ref().drvdata_obtain::<T>() };104 let data = unsafe { pdev.as_ref().drvdata_borrow::<T>() };
96 105
97 T::unbind(pdev, data.as_ref());106 T::unbind(pdev, data);
98 }107 }
99}108}
100 109
rust/kernel/usb.rs+18−9
@@ -27,13 +27,22 @@ use core::{ · 26 unmodified lines
27/// An adapter for the registration of USB drivers.27/// An adapter for the registration of USB drivers.
28pub struct Adapter<T: Driver>(T);28pub struct Adapter<T: Driver>(T);
29 29
30// SAFETY: A call to `unregister` for a given instance of `RegType` is guaranteed to be valid if30// SAFETY:
31// - `bindings::usb_driver` is a C type declared as `repr(C)`.
32// - `T` is the type of the driver's device private data.
33// - `struct usb_driver` embeds a `struct device_driver`.
34// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`.
35unsafe impl<T: Driver + 'static> driver::DriverLayout for Adapter<T> {
36 type DriverType = bindings::usb_driver;
37 type DriverData = T;
38 const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver);
39}
40
41// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if
31// a preceding call to `register` has been successful.42// a preceding call to `register` has been successful.
32unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> {43unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> {
33 type RegType = bindings::usb_driver;
34
35 unsafe fn register(44 unsafe fn register(
36 udrv: &Opaque<Self::RegType>,45 udrv: &Opaque<Self::DriverType>,
37 name: &'static CStr,46 name: &'static CStr,
38 module: &'static ThisModule,47 module: &'static ThisModule,
39 ) -> Result {48 ) -> Result {
@@ -45,14 +54,14 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { · 5 unmodified lines
45 (*udrv.get()).id_table = T::ID_TABLE.as_ptr();54 (*udrv.get()).id_table = T::ID_TABLE.as_ptr();
46 }55 }
47 56
48 // SAFETY: `udrv` is guaranteed to be a valid `RegType`.57 // SAFETY: `udrv` is guaranteed to be a valid `DriverType`.
49 to_result(unsafe {58 to_result(unsafe {
50 bindings::usb_register_driver(udrv.get(), module.0, name.as_char_ptr())59 bindings::usb_register_driver(udrv.get(), module.0, name.as_char_ptr())
51 })60 })
52 }61 }
53 62
54 unsafe fn unregister(udrv: &Opaque<Self::RegType>) {63 unsafe fn unregister(udrv: &Opaque<Self::DriverType>) {
55 // SAFETY: `udrv` is guaranteed to be a valid `RegType`.64 // SAFETY: `udrv` is guaranteed to be a valid `DriverType`.
56 unsafe { bindings::usb_deregister(udrv.get()) };65 unsafe { bindings::usb_deregister(udrv.get()) };
57 }66 }
58}67}
@@ -94,9 +103,9 @@ impl<T: Driver + 'static> Adapter<T> { · 35 unmodified lines
94 // SAFETY: `disconnect_callback` is only ever called after a successful call to103 // SAFETY: `disconnect_callback` is only ever called after a successful call to
95 // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called104 // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called
96 // and stored a `Pin<KBox<T>>`.105 // and stored a `Pin<KBox<T>>`.
97 let data = unsafe { dev.drvdata_obtain::<T>() };106 let data = unsafe { dev.drvdata_borrow::<T>() };
98 107
99 T::disconnect(intf, data.as_ref());108 T::disconnect(intf, data);
100 }109 }
101}110}
102 111