Pull driver core fixes from Danilo Krummrich: - Always inline I/O and IRQ methods using build_assert!() to avoid false positive build errors - Do not free the driver's device private data in I2C shutdown() avoiding race conditions that can lead to UAF bugs - Drop the driver's device private data after the driver has been fully unbound from its device to avoid UAF bugs from &Device<Bound> scopes, such as IRQ callbacks * tag 'driver-core-6.19-rc7' of git://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core: rust: driver: drop device private data post unbind rust: driver: add DriverData type to the DriverLayout trait rust: driver: add DEVICE_DRIVER_OFFSET to the DriverLayout trait rust: driver: introduce a DriverLayout trait rust: auxiliary: add Driver::unbind() callback rust: i2c: do not drop device private data on shutdown() rust: irq: always inline functions using build_assert with arguments rust: io: always inline functions using build_assert with arguments
Unified · Split
| @@ -548,6 +548,8 @@ static DEVICE_ATTR_RW(state_synced); · 547 unmodified lines | |||
| 548 | static void device_unbind_cleanup(struct device *dev) | 548 | static void device_unbind_cleanup(struct device *dev) |
| 549 | { | 549 | { |
| 550 | devres_release_all(dev); | 550 | devres_release_all(dev); |
| 551 | if (dev->driver->p_cb.post_unbind_rust) | ||
| 552 | dev->driver->p_cb.post_unbind_rust(dev); | ||
| 551 | arch_teardown_dma_ops(dev); | 553 | arch_teardown_dma_ops(dev); |
| 552 | kfree(dev->dma_range_map); | 554 | kfree(dev->dma_range_map); |
| 553 | dev->dma_range_map = NULL; | 555 | dev->dma_range_map = NULL; |
| @@ -85,6 +85,8 @@ enum probe_type { · 84 unmodified lines | |||
| 85 | * uevent. | 85 | * uevent. |
| 86 | * @p: Driver core's private data, no one other than the driver | 86 | * @p: Driver core's private data, no one other than the driver |
| 87 | * core can touch this. | 87 | * core can touch this. |
| 88 | * @p_cb: Callbacks private to the driver core; no one other than the | ||
| 89 | * driver core is allowed to touch this. | ||
| 88 | * | 90 | * |
| 89 | * The device driver-model tracks all of the drivers known to the system. | 91 | * The device driver-model tracks all of the drivers known to the system. |
| 90 | * The main reason for this tracking is to enable the driver core to match | 92 | * The main reason for this tracking is to enable the driver core to match |
| @@ -119,6 +121,13 @@ struct device_driver { · 28 unmodified lines | |||
| 119 | void (*coredump) (struct device *dev); | 121 | void (*coredump) (struct device *dev); |
| 120 | 122 | ||
| 121 | struct driver_private *p; | 123 | struct driver_private *p; |
| 124 | struct { | ||
| 125 | /* | ||
| 126 | * Called after remove() and after all devres entries have been | ||
| 127 | * processed. This is a Rust only callback. | ||
| 128 | */ | ||
| 129 | void (*post_unbind_rust)(struct device *dev); | ||
| 130 | } p_cb; | ||
| 122 | }; | 131 | }; |
| 123 | 132 | ||
| 124 | 133 | ||
| @@ -23,13 +23,22 @@ use core::{ · 22 unmodified lines | |||
| 23 | /// An adapter for the registration of auxiliary drivers. | 23 | /// An adapter for the registration of auxiliary drivers. |
| 24 | pub struct Adapter<T: Driver>(T); | 24 | pub struct Adapter<T: Driver>(T); |
| 25 | 25 | ||
| 26 | // SAFETY: A call to `unregister` for a given instance of `RegType` is guaranteed to be valid if | 26 | // SAFETY: |
| 27 | // - `bindings::auxiliary_driver` is a C type declared as `repr(C)`. | ||
| 28 | // - `T` is the type of the driver's device private data. | ||
| 29 | // - `struct auxiliary_driver` embeds a `struct device_driver`. | ||
| 30 | // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. | ||
| 31 | unsafe impl<T: Driver + 'static> driver::DriverLayout for Adapter<T> { | ||
| 32 | type DriverType = bindings::auxiliary_driver; | ||
| 33 | type DriverData = T; | ||
| 34 | const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); | ||
| 35 | } | ||
| 36 | |||
| 37 | // SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if | ||
| 27 | // a preceding call to `register` has been successful. | 38 | // a preceding call to `register` has been successful. |
| 28 | unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { | 39 | unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { |
| 29 | type RegType = bindings::auxiliary_driver; | ||
| 30 | |||
| 31 | unsafe fn register( | 40 | unsafe fn register( |
| 32 | adrv: &Opaque<Self::RegType>, | 41 | adrv: &Opaque<Self::DriverType>, |
| 33 | name: &'static CStr, | 42 | name: &'static CStr, |
| 34 | module: &'static ThisModule, | 43 | module: &'static ThisModule, |
| 35 | ) -> Result { | 44 | ) -> Result { |
| @@ -41,14 +50,14 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { · 5 unmodified lines | |||
| 41 | (*adrv.get()).id_table = T::ID_TABLE.as_ptr(); | 50 | (*adrv.get()).id_table = T::ID_TABLE.as_ptr(); |
| 42 | } | 51 | } |
| 43 | 52 | ||
| 44 | // SAFETY: `adrv` is guaranteed to be a valid `RegType`. | 53 | // SAFETY: `adrv` is guaranteed to be a valid `DriverType`. |
| 45 | to_result(unsafe { | 54 | to_result(unsafe { |
| 46 | bindings::__auxiliary_driver_register(adrv.get(), module.0, name.as_char_ptr()) | 55 | bindings::__auxiliary_driver_register(adrv.get(), module.0, name.as_char_ptr()) |
| 47 | }) | 56 | }) |
| 48 | } | 57 | } |
| 49 | 58 | ||
| 50 | unsafe fn unregister(adrv: &Opaque<Self::RegType>) { | 59 | unsafe fn unregister(adrv: &Opaque<Self::DriverType>) { |
| 51 | // SAFETY: `adrv` is guaranteed to be a valid `RegType`. | 60 | // SAFETY: `adrv` is guaranteed to be a valid `DriverType`. |
| 52 | unsafe { bindings::auxiliary_driver_unregister(adrv.get()) } | 61 | unsafe { bindings::auxiliary_driver_unregister(adrv.get()) } |
| 53 | } | 62 | } |
| 54 | } | 63 | } |
| @@ -87,7 +96,9 @@ impl<T: Driver + 'static> Adapter<T> { · 32 unmodified lines | |||
| 87 | // SAFETY: `remove_callback` is only ever called after a successful call to | 96 | // SAFETY: `remove_callback` is only ever called after a successful call to |
| 88 | // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called | 97 | // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called |
| 89 | // and stored a `Pin<KBox<T>>`. | 98 | // and stored a `Pin<KBox<T>>`. |
| 90 | drop(unsafe { adev.as_ref().drvdata_obtain::<T>() }); | 99 | let data = unsafe { adev.as_ref().drvdata_borrow::<T>() }; |
| 100 | |||
| 101 | T::unbind(adev, data); | ||
| 91 | } | 102 | } |
| 92 | } | 103 | } |
| 93 | 104 | ||
| @@ -187,6 +198,20 @@ pub trait Driver { · 93 unmodified lines | |||
| 187 | /// | 198 | /// |
| 188 | /// Called when an auxiliary device is matches a corresponding driver. | 199 | /// Called when an auxiliary device is matches a corresponding driver. |
| 189 | fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> impl PinInit<Self, Error>; | 200 | fn probe(dev: &Device<device::Core>, id_info: &Self::IdInfo) -> impl PinInit<Self, Error>; |
| 201 | |||
| 202 | /// Auxiliary driver unbind. | ||
| 203 | /// | ||
| 204 | /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback | ||
| 205 | /// is optional. | ||
| 206 | /// | ||
| 207 | /// This callback serves as a place for drivers to perform teardown operations that require a | ||
| 208 | /// `&Device<Core>` or `&Device<Bound>` reference. For instance, drivers may try to perform I/O | ||
| 209 | /// operations to gracefully tear down the device. | ||
| 210 | /// | ||
| 211 | /// Otherwise, release operations for driver resources should be performed in `Self::drop`. | ||
| 212 | fn unbind(dev: &Device<device::Core>, this: Pin<&Self>) { | ||
| 213 | let _ = (dev, this); | ||
| 214 | } | ||
| 190 | } | 215 | } |
| 191 | 216 | ||
| 192 | /// The auxiliary device representation. | 217 | /// The auxiliary device representation. |
| @@ -232,30 +232,32 @@ impl Device<CoreInternal> { · 231 unmodified lines | |||
| 232 | /// | 232 | /// |
| 233 | /// # Safety | 233 | /// # Safety |
| 234 | /// | 234 | /// |
| 235 | /// - Must only be called once after a preceding call to [`Device::set_drvdata`]. | ||
| 236 | /// - The type `T` must match the type of the `ForeignOwnable` previously stored by | 235 | /// - The type `T` must match the type of the `ForeignOwnable` previously stored by |
| 237 | /// [`Device::set_drvdata`]. | 236 | /// [`Device::set_drvdata`]. |
| 238 | pub unsafe fn drvdata_obtain<T: 'static>(&self) -> Pin<KBox<T>> { | 237 | pub(crate) unsafe fn drvdata_obtain<T: 'static>(&self) -> Option<Pin<KBox<T>>> { |
| 239 | // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. | 238 | // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. |
| 240 | let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; | 239 | let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; |
| 241 | 240 | ||
| 242 | // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. | 241 | // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. |
| 243 | unsafe { bindings::dev_set_drvdata(self.as_raw(), core::ptr::null_mut()) }; | 242 | unsafe { bindings::dev_set_drvdata(self.as_raw(), core::ptr::null_mut()) }; |
| 244 | 243 | ||
| 244 | if ptr.is_null() { | ||
| 245 | return None; | ||
| 246 | } | ||
| 247 | |||
| 245 | // SAFETY: | 248 | // SAFETY: |
| 246 | // - By the safety requirements of this function, `ptr` comes from a previous call to | 249 | // - If `ptr` is not NULL, it comes from a previous call to `into_foreign()`. |
| 247 | // `into_foreign()`. | ||
| 248 | // - `dev_get_drvdata()` guarantees to return the same pointer given to `dev_set_drvdata()` | 250 | // - `dev_get_drvdata()` guarantees to return the same pointer given to `dev_set_drvdata()` |
| 249 | // in `into_foreign()`. | 251 | // in `into_foreign()`. |
| 250 | unsafe { Pin::<KBox<T>>::from_foreign(ptr.cast()) } | 252 | Some(unsafe { Pin::<KBox<T>>::from_foreign(ptr.cast()) }) |
| 251 | } | 253 | } |
| 252 | 254 | ||
| 253 | /// Borrow the driver's private data bound to this [`Device`]. | 255 | /// Borrow the driver's private data bound to this [`Device`]. |
| 254 | /// | 256 | /// |
| 255 | /// # Safety | 257 | /// # Safety |
| 256 | /// | 258 | /// |
| 257 | /// - Must only be called after a preceding call to [`Device::set_drvdata`] and before | 259 | /// - Must only be called after a preceding call to [`Device::set_drvdata`] and before the |
| 258 | /// [`Device::drvdata_obtain`]. | 260 | /// device is fully unbound. |
| 259 | /// - The type `T` must match the type of the `ForeignOwnable` previously stored by | 261 | /// - The type `T` must match the type of the `ForeignOwnable` previously stored by |
| 260 | /// [`Device::set_drvdata`]. | 262 | /// [`Device::set_drvdata`]. |
| 261 | pub unsafe fn drvdata_borrow<T: 'static>(&self) -> Pin<&T> { | 263 | pub unsafe fn drvdata_borrow<T: 'static>(&self) -> Pin<&T> { |
| @@ -271,7 +273,7 @@ impl Device<Bound> { · 9 unmodified lines | |||
| 271 | /// # Safety | 273 | /// # Safety |
| 272 | /// | 274 | /// |
| 273 | /// - Must only be called after a preceding call to [`Device::set_drvdata`] and before | 275 | /// - Must only be called after a preceding call to [`Device::set_drvdata`] and before |
| 274 | /// [`Device::drvdata_obtain`]. | 276 | /// the device is fully unbound. |
| 275 | /// - The type `T` must match the type of the `ForeignOwnable` previously stored by | 277 | /// - The type `T` must match the type of the `ForeignOwnable` previously stored by |
| 276 | /// [`Device::set_drvdata`]. | 278 | /// [`Device::set_drvdata`]. |
| 277 | unsafe fn drvdata_unchecked<T: 'static>(&self) -> Pin<&T> { | 279 | unsafe fn drvdata_unchecked<T: 'static>(&self) -> Pin<&T> { |
| @@ -320,7 +322,7 @@ impl Device<Bound> { · 42 unmodified lines | |||
| 320 | 322 | ||
| 321 | // SAFETY: | 323 | // SAFETY: |
| 322 | // - The above check of `dev_get_drvdata()` guarantees that we are called after | 324 | // - The above check of `dev_get_drvdata()` guarantees that we are called after |
| 323 | // `set_drvdata()` and before `drvdata_obtain()`. | 325 | // `set_drvdata()`. |
| 324 | // - We've just checked that the type of the driver's private data is in fact `T`. | 326 | // - We've just checked that the type of the driver's private data is in fact `T`. |
| 325 | Ok(unsafe { self.drvdata_unchecked() }) | 327 | Ok(unsafe { self.drvdata_unchecked() }) |
| 326 | } | 328 | } |
| @@ -99,23 +99,43 @@ use crate::{acpi, device, of, str::CStr, try_pin_init, types::Opaque, ThisModule · 98 unmodified lines | |||
| 99 | use core::pin::Pin; | 99 | use core::pin::Pin; |
| 100 | use pin_init::{pin_data, pinned_drop, PinInit}; | 100 | use pin_init::{pin_data, pinned_drop, PinInit}; |
| 101 | 101 | ||
| 102 | /// Trait describing the layout of a specific device driver. | ||
| 103 | /// | ||
| 104 | /// This trait describes the layout of a specific driver structure, such as `struct pci_driver` or | ||
| 105 | /// `struct platform_driver`. | ||
| 106 | /// | ||
| 107 | /// # Safety | ||
| 108 | /// | ||
| 109 | /// Implementors must guarantee that: | ||
| 110 | /// - `DriverType` is `repr(C)`, | ||
| 111 | /// - `DriverData` is the type of the driver's device private data. | ||
| 112 | /// - `DriverType` embeds a valid `struct device_driver` at byte offset `DEVICE_DRIVER_OFFSET`. | ||
| 113 | pub unsafe trait DriverLayout { | ||
| 114 | /// The specific driver type embedding a `struct device_driver`. | ||
| 115 | type DriverType: Default; | ||
| 116 | |||
| 117 | /// The type of the driver's device private data. | ||
| 118 | type DriverData; | ||
| 119 | |||
| 120 | /// Byte offset of the embedded `struct device_driver` within `DriverType`. | ||
| 121 | /// | ||
| 122 | /// This must correspond exactly to the location of the embedded `struct device_driver` field. | ||
| 123 | const DEVICE_DRIVER_OFFSET: usize; | ||
| 124 | } | ||
| 125 | |||
| 102 | /// The [`RegistrationOps`] trait serves as generic interface for subsystems (e.g., PCI, Platform, | 126 | /// The [`RegistrationOps`] trait serves as generic interface for subsystems (e.g., PCI, Platform, |
| 103 | /// Amba, etc.) to provide the corresponding subsystem specific implementation to register / | 127 | /// Amba, etc.) to provide the corresponding subsystem specific implementation to register / |
| 104 | /// unregister a driver of the particular type (`RegType`). | 128 | /// unregister a driver of the particular type (`DriverType`). |
| 105 | /// | 129 | /// |
| 106 | /// For instance, the PCI subsystem would set `RegType` to `bindings::pci_driver` and call | 130 | /// For instance, the PCI subsystem would set `DriverType` to `bindings::pci_driver` and call |
| 107 | /// `bindings::__pci_register_driver` from `RegistrationOps::register` and | 131 | /// `bindings::__pci_register_driver` from `RegistrationOps::register` and |
| 108 | /// `bindings::pci_unregister_driver` from `RegistrationOps::unregister`. | 132 | /// `bindings::pci_unregister_driver` from `RegistrationOps::unregister`. |
| 109 | /// | 133 | /// |
| 110 | /// # Safety | 134 | /// # Safety |
| 111 | /// | 135 | /// |
| 112 | /// A call to [`RegistrationOps::unregister`] for a given instance of `RegType` is only valid if a | 136 | /// A call to [`RegistrationOps::unregister`] for a given instance of `DriverType` is only valid if |
| 113 | /// preceding call to [`RegistrationOps::register`] has been successful. | 137 | /// a preceding call to [`RegistrationOps::register`] has been successful. |
| 114 | pub unsafe trait RegistrationOps { | 138 | pub unsafe trait RegistrationOps: DriverLayout { |
| 115 | /// The type that holds information about the registration. This is typically a struct defined | ||
| 116 | /// by the C portion of the kernel. | ||
| 117 | type RegType: Default; | ||
| 118 | |||
| 119 | /// Registers a driver. | 139 | /// Registers a driver. |
| 120 | /// | 140 | /// |
| 121 | /// # Safety | 141 | /// # Safety |
| @@ -123,7 +143,7 @@ pub unsafe trait RegistrationOps { · 1 unmodified line | |||
| 123 | /// On success, `reg` must remain pinned and valid until the matching call to | 143 | /// On success, `reg` must remain pinned and valid until the matching call to |
| 124 | /// [`RegistrationOps::unregister`]. | 144 | /// [`RegistrationOps::unregister`]. |
| 125 | unsafe fn register( | 145 | unsafe fn register( |
| 126 | reg: &Opaque<Self::RegType>, | 146 | reg: &Opaque<Self::DriverType>, |
| 127 | name: &'static CStr, | 147 | name: &'static CStr, |
| 128 | module: &'static ThisModule, | 148 | module: &'static ThisModule, |
| 129 | ) -> Result; | 149 | ) -> Result; |
| @@ -134,7 +154,7 @@ pub unsafe trait RegistrationOps { · 4 unmodified lines | |||
| 134 | /// | 154 | /// |
| 135 | /// Must only be called after a preceding successful call to [`RegistrationOps::register`] for | 155 | /// Must only be called after a preceding successful call to [`RegistrationOps::register`] for |
| 136 | /// the same `reg`. | 156 | /// the same `reg`. |
| 137 | unsafe fn unregister(reg: &Opaque<Self::RegType>); | 157 | unsafe fn unregister(reg: &Opaque<Self::DriverType>); |
| 138 | } | 158 | } |
| 139 | 159 | ||
| 140 | /// A [`Registration`] is a generic type that represents the registration of some driver type (e.g. | 160 | /// A [`Registration`] is a generic type that represents the registration of some driver type (e.g. |
| @@ -146,7 +166,7 @@ pub unsafe trait RegistrationOps { · 5 unmodified lines | |||
| 146 | #[pin_data(PinnedDrop)] | 166 | #[pin_data(PinnedDrop)] |
| 147 | pub struct Registration<T: RegistrationOps> { | 167 | pub struct Registration<T: RegistrationOps> { |
| 148 | #[pin] | 168 | #[pin] |
| 149 | reg: Opaque<T::RegType>, | 169 | reg: Opaque<T::DriverType>, |
| 150 | } | 170 | } |
| 151 | 171 | ||
| 152 | // SAFETY: `Registration` has no fields or methods accessible via `&Registration`, so it is safe to | 172 | // SAFETY: `Registration` has no fields or methods accessible via `&Registration`, so it is safe to |
| @@ -157,17 +177,51 @@ unsafe impl<T: RegistrationOps> Sync for Registration<T> {} · 4 unmodified lines | |||
| 157 | // any thread, so `Registration` is `Send`. | 177 | // any thread, so `Registration` is `Send`. |
| 158 | unsafe impl<T: RegistrationOps> Send for Registration<T> {} | 178 | unsafe impl<T: RegistrationOps> Send for Registration<T> {} |
| 159 | 179 | ||
| 160 | impl<T: RegistrationOps> Registration<T> { | 180 | impl<T: RegistrationOps + 'static> Registration<T> { |
| 181 | extern "C" fn post_unbind_callback(dev: *mut bindings::device) { | ||
| 182 | // SAFETY: The driver core only ever calls the post unbind callback with a valid pointer to | ||
| 183 | // a `struct device`. | ||
| 184 | // | ||
| 185 | // INVARIANT: `dev` is valid for the duration of the `post_unbind_callback()`. | ||
| 186 | let dev = unsafe { &*dev.cast::<device::Device<device::CoreInternal>>() }; | ||
| 187 | |||
| 188 | // `remove()` and all devres callbacks have been completed at this point, hence drop the | ||
| 189 | // driver's device private data. | ||
| 190 | // | ||
| 191 | // SAFETY: By the safety requirements of the `Driver` trait, `T::DriverData` is the | ||
| 192 | // driver's device private data type. | ||
| 193 | drop(unsafe { dev.drvdata_obtain::<T::DriverData>() }); | ||
| 194 | } | ||
| 195 | |||
| 196 | /// Attach generic `struct device_driver` callbacks. | ||
| 197 | fn callbacks_attach(drv: &Opaque<T::DriverType>) { | ||
| 198 | let ptr = drv.get().cast::<u8>(); | ||
| 199 | |||
| 200 | // SAFETY: | ||
| 201 | // - `drv.get()` yields a valid pointer to `Self::DriverType`. | ||
| 202 | // - Adding `DEVICE_DRIVER_OFFSET` yields the address of the embedded `struct device_driver` | ||
| 203 | // as guaranteed by the safety requirements of the `Driver` trait. | ||
| 204 | let base = unsafe { ptr.add(T::DEVICE_DRIVER_OFFSET) }; | ||
| 205 | |||
| 206 | // CAST: `base` points to the offset of the embedded `struct device_driver`. | ||
| 207 | let base = base.cast::<bindings::device_driver>(); | ||
| 208 | |||
| 209 | // SAFETY: It is safe to set the fields of `struct device_driver` on initialization. | ||
| 210 | unsafe { (*base).p_cb.post_unbind_rust = Some(Self::post_unbind_callback) }; | ||
| 211 | } | ||
| 212 | |||
| 161 | /// Creates a new instance of the registration object. | 213 | /// Creates a new instance of the registration object. |
| 162 | pub fn new(name: &'static CStr, module: &'static ThisModule) -> impl PinInit<Self, Error> { | 214 | pub fn new(name: &'static CStr, module: &'static ThisModule) -> impl PinInit<Self, Error> { |
| 163 | try_pin_init!(Self { | 215 | try_pin_init!(Self { |
| 164 | reg <- Opaque::try_ffi_init(|ptr: *mut T::RegType| { | 216 | reg <- Opaque::try_ffi_init(|ptr: *mut T::DriverType| { |
| 165 | // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write. | 217 | // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write. |
| 166 | unsafe { ptr.write(T::RegType::default()) }; | 218 | unsafe { ptr.write(T::DriverType::default()) }; |
| 167 | 219 | ||
| 168 | // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write, and it has | 220 | // SAFETY: `try_ffi_init` guarantees that `ptr` is valid for write, and it has |
| 169 | // just been initialised above, so it's also valid for read. | 221 | // just been initialised above, so it's also valid for read. |
| 170 | let drv = unsafe { &*(ptr as *const Opaque<T::RegType>) }; | 222 | let drv = unsafe { &*(ptr as *const Opaque<T::DriverType>) }; |
| 223 | |||
| 224 | Self::callbacks_attach(drv); | ||
| 171 | 225 | ||
| 172 | // SAFETY: `drv` is guaranteed to be pinned until `T::unregister`. | 226 | // SAFETY: `drv` is guaranteed to be pinned until `T::unregister`. |
| 173 | unsafe { T::register(drv, name, module) } | 227 | unsafe { T::register(drv, name, module) } |
| @@ -92,13 +92,22 @@ macro_rules! i2c_device_table { · 91 unmodified lines | |||
| 92 | /// An adapter for the registration of I2C drivers. | 92 | /// An adapter for the registration of I2C drivers. |
| 93 | pub struct Adapter<T: Driver>(T); | 93 | pub struct Adapter<T: Driver>(T); |
| 94 | 94 | ||
| 95 | // SAFETY: A call to `unregister` for a given instance of `RegType` is guaranteed to be valid if | 95 | // SAFETY: |
| 96 | // - `bindings::i2c_driver` is a C type declared as `repr(C)`. | ||
| 97 | // - `T` is the type of the driver's device private data. | ||
| 98 | // - `struct i2c_driver` embeds a `struct device_driver`. | ||
| 99 | // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. | ||
| 100 | unsafe impl<T: Driver + 'static> driver::DriverLayout for Adapter<T> { | ||
| 101 | type DriverType = bindings::i2c_driver; | ||
| 102 | type DriverData = T; | ||
| 103 | const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); | ||
| 104 | } | ||
| 105 | |||
| 106 | // SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if | ||
| 96 | // a preceding call to `register` has been successful. | 107 | // a preceding call to `register` has been successful. |
| 97 | unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { | 108 | unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { |
| 98 | type RegType = bindings::i2c_driver; | ||
| 99 | |||
| 100 | unsafe fn register( | 109 | unsafe fn register( |
| 101 | idrv: &Opaque<Self::RegType>, | 110 | idrv: &Opaque<Self::DriverType>, |
| 102 | name: &'static CStr, | 111 | name: &'static CStr, |
| 103 | module: &'static ThisModule, | 112 | module: &'static ThisModule, |
| 104 | ) -> Result { | 113 | ) -> Result { |
| @@ -133,12 +142,12 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { · 28 unmodified lines | |||
| 133 | (*idrv.get()).driver.acpi_match_table = acpi_table; | 142 | (*idrv.get()).driver.acpi_match_table = acpi_table; |
| 134 | } | 143 | } |
| 135 | 144 | ||
| 136 | // SAFETY: `idrv` is guaranteed to be a valid `RegType`. | 145 | // SAFETY: `idrv` is guaranteed to be a valid `DriverType`. |
| 137 | to_result(unsafe { bindings::i2c_register_driver(module.0, idrv.get()) }) | 146 | to_result(unsafe { bindings::i2c_register_driver(module.0, idrv.get()) }) |
| 138 | } | 147 | } |
| 139 | 148 | ||
| 140 | unsafe fn unregister(idrv: &Opaque<Self::RegType>) { | 149 | unsafe fn unregister(idrv: &Opaque<Self::DriverType>) { |
| 141 | // SAFETY: `idrv` is guaranteed to be a valid `RegType`. | 150 | // SAFETY: `idrv` is guaranteed to be a valid `DriverType`. |
| 142 | unsafe { bindings::i2c_del_driver(idrv.get()) } | 151 | unsafe { bindings::i2c_del_driver(idrv.get()) } |
| 143 | } | 152 | } |
| 144 | } | 153 | } |
| @@ -169,9 +178,9 @@ impl<T: Driver + 'static> Adapter<T> { · 24 unmodified lines | |||
| 169 | // SAFETY: `remove_callback` is only ever called after a successful call to | 178 | // SAFETY: `remove_callback` is only ever called after a successful call to |
| 170 | // `probe_callback`, hence it's guaranteed that `I2cClient::set_drvdata()` has been called | 179 | // `probe_callback`, hence it's guaranteed that `I2cClient::set_drvdata()` has been called |
| 171 | // and stored a `Pin<KBox<T>>`. | 180 | // and stored a `Pin<KBox<T>>`. |
| 172 | let data = unsafe { idev.as_ref().drvdata_obtain::<T>() }; | 181 | let data = unsafe { idev.as_ref().drvdata_borrow::<T>() }; |
| 173 | 182 | ||
| 174 | T::unbind(idev, data.as_ref()); | 183 | T::unbind(idev, data); |
| 175 | } | 184 | } |
| 176 | 185 | ||
| 177 | extern "C" fn shutdown_callback(idev: *mut bindings::i2c_client) { | 186 | extern "C" fn shutdown_callback(idev: *mut bindings::i2c_client) { |
| @@ -181,9 +190,9 @@ impl<T: Driver + 'static> Adapter<T> { · 3 unmodified lines | |||
| 181 | // SAFETY: `shutdown_callback` is only ever called after a successful call to | 190 | // SAFETY: `shutdown_callback` is only ever called after a successful call to |
| 182 | // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called | 191 | // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called |
| 183 | // and stored a `Pin<KBox<T>>`. | 192 | // and stored a `Pin<KBox<T>>`. |
| 184 | let data = unsafe { idev.as_ref().drvdata_obtain::<T>() }; | 193 | let data = unsafe { idev.as_ref().drvdata_borrow::<T>() }; |
| 185 | 194 | ||
| 186 | T::shutdown(idev, data.as_ref()); | 195 | T::shutdown(idev, data); |
| 187 | } | 196 | } |
| 188 | 197 | ||
| 189 | /// The [`i2c::IdTable`] of the corresponding driver. | 198 | /// The [`i2c::IdTable`] of the corresponding driver. |
| @@ -142,7 +142,8 @@ macro_rules! define_read { · 141 unmodified lines | |||
| 142 | /// Bound checks are performed on compile time, hence if the offset is not known at compile | 142 | /// Bound checks are performed on compile time, hence if the offset is not known at compile |
| 143 | /// time, the build will fail. | 143 | /// time, the build will fail. |
| 144 | $(#[$attr])* | 144 | $(#[$attr])* |
| 145 | #[inline] | 145 | // Always inline to optimize out error path of `io_addr_assert`. |
| 146 | #[inline(always)] | ||
| 146 | pub fn $name(&self, offset: usize) -> $type_name { | 147 | pub fn $name(&self, offset: usize) -> $type_name { |
| 147 | let addr = self.io_addr_assert::<$type_name>(offset); | 148 | let addr = self.io_addr_assert::<$type_name>(offset); |
| 148 | 149 | ||
| @@ -171,7 +172,8 @@ macro_rules! define_write { · 22 unmodified lines | |||
| 171 | /// Bound checks are performed on compile time, hence if the offset is not known at compile | 172 | /// Bound checks are performed on compile time, hence if the offset is not known at compile |
| 172 | /// time, the build will fail. | 173 | /// time, the build will fail. |
| 173 | $(#[$attr])* | 174 | $(#[$attr])* |
| 174 | #[inline] | 175 | // Always inline to optimize out error path of `io_addr_assert`. |
| 176 | #[inline(always)] | ||
| 175 | pub fn $name(&self, value: $type_name, offset: usize) { | 177 | pub fn $name(&self, value: $type_name, offset: usize) { |
| 176 | let addr = self.io_addr_assert::<$type_name>(offset); | 178 | let addr = self.io_addr_assert::<$type_name>(offset); |
| 177 | 179 | ||
| @@ -239,7 +241,8 @@ impl<const SIZE: usize> Io<SIZE> { · 61 unmodified lines | |||
| 239 | self.addr().checked_add(offset).ok_or(EINVAL) | 241 | self.addr().checked_add(offset).ok_or(EINVAL) |
| 240 | } | 242 | } |
| 241 | 243 | ||
| 242 | #[inline] | 244 | // Always inline to optimize out error path of `build_assert`. |
| 245 | #[inline(always)] | ||
| 243 | fn io_addr_assert<U>(&self, offset: usize) -> usize { | 246 | fn io_addr_assert<U>(&self, offset: usize) -> usize { |
| 244 | build_assert!(Self::offset_valid::<U>(offset, SIZE)); | 247 | build_assert!(Self::offset_valid::<U>(offset, SIZE)); |
| 245 | 248 | ||
| @@ -226,6 +226,8 @@ impl Flags { · 225 unmodified lines | |||
| 226 | /// Resource represents a memory region that must be ioremaped using `ioremap_np`. | 226 | /// Resource represents a memory region that must be ioremaped using `ioremap_np`. |
| 227 | pub const IORESOURCE_MEM_NONPOSTED: Flags = Flags::new(bindings::IORESOURCE_MEM_NONPOSTED); | 227 | pub const IORESOURCE_MEM_NONPOSTED: Flags = Flags::new(bindings::IORESOURCE_MEM_NONPOSTED); |
| 228 | 228 | ||
| 229 | // Always inline to optimize out error path of `build_assert`. | ||
| 230 | #[inline(always)] | ||
| 229 | const fn new(value: u32) -> Self { | 231 | const fn new(value: u32) -> Self { |
| 230 | crate::build_assert!(value as u64 <= c_ulong::MAX as u64); | 232 | crate::build_assert!(value as u64 <= c_ulong::MAX as u64); |
| 231 | Flags(value as c_ulong) | 233 | Flags(value as c_ulong) |
| @@ -96,6 +96,8 @@ impl Flags { · 95 unmodified lines | |||
| 96 | self.0 | 96 | self.0 |
| 97 | } | 97 | } |
| 98 | 98 | ||
| 99 | // Always inline to optimize out error path of `build_assert`. | ||
| 100 | #[inline(always)] | ||
| 99 | const fn new(value: u32) -> Self { | 101 | const fn new(value: u32) -> Self { |
| 100 | build_assert!(value as u64 <= c_ulong::MAX as u64); | 102 | build_assert!(value as u64 <= c_ulong::MAX as u64); |
| 101 | Self(value as c_ulong) | 103 | Self(value as c_ulong) |
| @@ -50,13 +50,22 @@ pub use self::irq::{ · 49 unmodified lines | |||
| 50 | /// An adapter for the registration of PCI drivers. | 50 | /// An adapter for the registration of PCI drivers. |
| 51 | pub struct Adapter<T: Driver>(T); | 51 | pub struct Adapter<T: Driver>(T); |
| 52 | 52 | ||
| 53 | // SAFETY: A call to `unregister` for a given instance of `RegType` is guaranteed to be valid if | 53 | // SAFETY: |
| 54 | // - `bindings::pci_driver` is a C type declared as `repr(C)`. | ||
| 55 | // - `T` is the type of the driver's device private data. | ||
| 56 | // - `struct pci_driver` embeds a `struct device_driver`. | ||
| 57 | // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. | ||
| 58 | unsafe impl<T: Driver + 'static> driver::DriverLayout for Adapter<T> { | ||
| 59 | type DriverType = bindings::pci_driver; | ||
| 60 | type DriverData = T; | ||
| 61 | const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); | ||
| 62 | } | ||
| 63 | |||
| 64 | // SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if | ||
| 54 | // a preceding call to `register` has been successful. | 65 | // a preceding call to `register` has been successful. |
| 55 | unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { | 66 | unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { |
| 56 | type RegType = bindings::pci_driver; | ||
| 57 | |||
| 58 | unsafe fn register( | 67 | unsafe fn register( |
| 59 | pdrv: &Opaque<Self::RegType>, | 68 | pdrv: &Opaque<Self::DriverType>, |
| 60 | name: &'static CStr, | 69 | name: &'static CStr, |
| 61 | module: &'static ThisModule, | 70 | module: &'static ThisModule, |
| 62 | ) -> Result { | 71 | ) -> Result { |
| @@ -68,14 +77,14 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { · 5 unmodified lines | |||
| 68 | (*pdrv.get()).id_table = T::ID_TABLE.as_ptr(); | 77 | (*pdrv.get()).id_table = T::ID_TABLE.as_ptr(); |
| 69 | } | 78 | } |
| 70 | 79 | ||
| 71 | // SAFETY: `pdrv` is guaranteed to be a valid `RegType`. | 80 | // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. |
| 72 | to_result(unsafe { | 81 | to_result(unsafe { |
| 73 | bindings::__pci_register_driver(pdrv.get(), module.0, name.as_char_ptr()) | 82 | bindings::__pci_register_driver(pdrv.get(), module.0, name.as_char_ptr()) |
| 74 | }) | 83 | }) |
| 75 | } | 84 | } |
| 76 | 85 | ||
| 77 | unsafe fn unregister(pdrv: &Opaque<Self::RegType>) { | 86 | unsafe fn unregister(pdrv: &Opaque<Self::DriverType>) { |
| 78 | // SAFETY: `pdrv` is guaranteed to be a valid `RegType`. | 87 | // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. |
| 79 | unsafe { bindings::pci_unregister_driver(pdrv.get()) } | 88 | unsafe { bindings::pci_unregister_driver(pdrv.get()) } |
| 80 | } | 89 | } |
| 81 | } | 90 | } |
| @@ -114,9 +123,9 @@ impl<T: Driver + 'static> Adapter<T> { · 32 unmodified lines | |||
| 114 | // SAFETY: `remove_callback` is only ever called after a successful call to | 123 | // SAFETY: `remove_callback` is only ever called after a successful call to |
| 115 | // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called | 124 | // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called |
| 116 | // and stored a `Pin<KBox<T>>`. | 125 | // and stored a `Pin<KBox<T>>`. |
| 117 | let data = unsafe { pdev.as_ref().drvdata_obtain::<T>() }; | 126 | let data = unsafe { pdev.as_ref().drvdata_borrow::<T>() }; |
| 118 | 127 | ||
| 119 | T::unbind(pdev, data.as_ref()); | 128 | T::unbind(pdev, data); |
| 120 | } | 129 | } |
| 121 | } | 130 | } |
| 122 | 131 | ||
| @@ -26,13 +26,22 @@ use core::{ · 25 unmodified lines | |||
| 26 | /// An adapter for the registration of platform drivers. | 26 | /// An adapter for the registration of platform drivers. |
| 27 | pub struct Adapter<T: Driver>(T); | 27 | pub struct Adapter<T: Driver>(T); |
| 28 | 28 | ||
| 29 | // SAFETY: A call to `unregister` for a given instance of `RegType` is guaranteed to be valid if | 29 | // SAFETY: |
| 30 | // - `bindings::platform_driver` is a C type declared as `repr(C)`. | ||
| 31 | // - `T` is the type of the driver's device private data. | ||
| 32 | // - `struct platform_driver` embeds a `struct device_driver`. | ||
| 33 | // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. | ||
| 34 | unsafe impl<T: Driver + 'static> driver::DriverLayout for Adapter<T> { | ||
| 35 | type DriverType = bindings::platform_driver; | ||
| 36 | type DriverData = T; | ||
| 37 | const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); | ||
| 38 | } | ||
| 39 | |||
| 40 | // SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if | ||
| 30 | // a preceding call to `register` has been successful. | 41 | // a preceding call to `register` has been successful. |
| 31 | unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { | 42 | unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { |
| 32 | type RegType = bindings::platform_driver; | ||
| 33 | |||
| 34 | unsafe fn register( | 43 | unsafe fn register( |
| 35 | pdrv: &Opaque<Self::RegType>, | 44 | pdrv: &Opaque<Self::DriverType>, |
| 36 | name: &'static CStr, | 45 | name: &'static CStr, |
| 37 | module: &'static ThisModule, | 46 | module: &'static ThisModule, |
| 38 | ) -> Result { | 47 | ) -> Result { |
| @@ -55,12 +64,12 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { · 16 unmodified lines | |||
| 55 | (*pdrv.get()).driver.acpi_match_table = acpi_table; | 64 | (*pdrv.get()).driver.acpi_match_table = acpi_table; |
| 56 | } | 65 | } |
| 57 | 66 | ||
| 58 | // SAFETY: `pdrv` is guaranteed to be a valid `RegType`. | 67 | // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. |
| 59 | to_result(unsafe { bindings::__platform_driver_register(pdrv.get(), module.0) }) | 68 | to_result(unsafe { bindings::__platform_driver_register(pdrv.get(), module.0) }) |
| 60 | } | 69 | } |
| 61 | 70 | ||
| 62 | unsafe fn unregister(pdrv: &Opaque<Self::RegType>) { | 71 | unsafe fn unregister(pdrv: &Opaque<Self::DriverType>) { |
| 63 | // SAFETY: `pdrv` is guaranteed to be a valid `RegType`. | 72 | // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. |
| 64 | unsafe { bindings::platform_driver_unregister(pdrv.get()) }; | 73 | unsafe { bindings::platform_driver_unregister(pdrv.get()) }; |
| 65 | } | 74 | } |
| 66 | } | 75 | } |
| @@ -92,9 +101,9 @@ impl<T: Driver + 'static> Adapter<T> { · 25 unmodified lines | |||
| 92 | // SAFETY: `remove_callback` is only ever called after a successful call to | 101 | // SAFETY: `remove_callback` is only ever called after a successful call to |
| 93 | // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called | 102 | // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called |
| 94 | // and stored a `Pin<KBox<T>>`. | 103 | // and stored a `Pin<KBox<T>>`. |
| 95 | let data = unsafe { pdev.as_ref().drvdata_obtain::<T>() }; | 104 | let data = unsafe { pdev.as_ref().drvdata_borrow::<T>() }; |
| 96 | 105 | ||
| 97 | T::unbind(pdev, data.as_ref()); | 106 | T::unbind(pdev, data); |
| 98 | } | 107 | } |
| 99 | } | 108 | } |
| 100 | 109 | ||
| @@ -27,13 +27,22 @@ use core::{ · 26 unmodified lines | |||
| 27 | /// An adapter for the registration of USB drivers. | 27 | /// An adapter for the registration of USB drivers. |
| 28 | pub struct Adapter<T: Driver>(T); | 28 | pub struct Adapter<T: Driver>(T); |
| 29 | 29 | ||
| 30 | // SAFETY: A call to `unregister` for a given instance of `RegType` is guaranteed to be valid if | 30 | // SAFETY: |
| 31 | // - `bindings::usb_driver` is a C type declared as `repr(C)`. | ||
| 32 | // - `T` is the type of the driver's device private data. | ||
| 33 | // - `struct usb_driver` embeds a `struct device_driver`. | ||
| 34 | // - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. | ||
| 35 | unsafe impl<T: Driver + 'static> driver::DriverLayout for Adapter<T> { | ||
| 36 | type DriverType = bindings::usb_driver; | ||
| 37 | type DriverData = T; | ||
| 38 | const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); | ||
| 39 | } | ||
| 40 | |||
| 41 | // SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if | ||
| 31 | // a preceding call to `register` has been successful. | 42 | // a preceding call to `register` has been successful. |
| 32 | unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { | 43 | unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { |
| 33 | type RegType = bindings::usb_driver; | ||
| 34 | |||
| 35 | unsafe fn register( | 44 | unsafe fn register( |
| 36 | udrv: &Opaque<Self::RegType>, | 45 | udrv: &Opaque<Self::DriverType>, |
| 37 | name: &'static CStr, | 46 | name: &'static CStr, |
| 38 | module: &'static ThisModule, | 47 | module: &'static ThisModule, |
| 39 | ) -> Result { | 48 | ) -> Result { |
| @@ -45,14 +54,14 @@ unsafe impl<T: Driver + 'static> driver::RegistrationOps for Adapter<T> { · 5 unmodified lines | |||
| 45 | (*udrv.get()).id_table = T::ID_TABLE.as_ptr(); | 54 | (*udrv.get()).id_table = T::ID_TABLE.as_ptr(); |
| 46 | } | 55 | } |
| 47 | 56 | ||
| 48 | // SAFETY: `udrv` is guaranteed to be a valid `RegType`. | 57 | // SAFETY: `udrv` is guaranteed to be a valid `DriverType`. |
| 49 | to_result(unsafe { | 58 | to_result(unsafe { |
| 50 | bindings::usb_register_driver(udrv.get(), module.0, name.as_char_ptr()) | 59 | bindings::usb_register_driver(udrv.get(), module.0, name.as_char_ptr()) |
| 51 | }) | 60 | }) |
| 52 | } | 61 | } |
| 53 | 62 | ||
| 54 | unsafe fn unregister(udrv: &Opaque<Self::RegType>) { | 63 | unsafe fn unregister(udrv: &Opaque<Self::DriverType>) { |
| 55 | // SAFETY: `udrv` is guaranteed to be a valid `RegType`. | 64 | // SAFETY: `udrv` is guaranteed to be a valid `DriverType`. |
| 56 | unsafe { bindings::usb_deregister(udrv.get()) }; | 65 | unsafe { bindings::usb_deregister(udrv.get()) }; |
| 57 | } | 66 | } |
| 58 | } | 67 | } |
| @@ -94,9 +103,9 @@ impl<T: Driver + 'static> Adapter<T> { · 35 unmodified lines | |||
| 94 | // SAFETY: `disconnect_callback` is only ever called after a successful call to | 103 | // SAFETY: `disconnect_callback` is only ever called after a successful call to |
| 95 | // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called | 104 | // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called |
| 96 | // and stored a `Pin<KBox<T>>`. | 105 | // and stored a `Pin<KBox<T>>`. |
| 97 | let data = unsafe { dev.drvdata_obtain::<T>() }; | 106 | let data = unsafe { dev.drvdata_borrow::<T>() }; |
| 98 | 107 | ||
| 99 | T::disconnect(intf, data.as_ref()); | 108 | T::disconnect(intf, data); |
| 100 | } | 109 | } |
| 101 | } | 110 | } |
| 102 | 111 | ||