// SPDX-License-Identifier: GPL-2.0 //! Direct memory access (DMA).//!//! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h) use crate::{    bindings,    debugfs,    device::{        self,        Bound,        Core, //    },    error::to_result,    fs::file,    prelude::*,    ptr::KnownSize,    sync::aref::ARef,    transmute::{        AsBytes,        FromBytes, //    }, //    uaccess::UserSliceWriter,};use core::{    ops::{        Deref,        DerefMut, //    },    ptr::NonNull, //}; /// DMA address type.////// Represents a bus address used for Direct Memory Access (DMA) operations.////// This is an alias of the kernel's `dma_addr_t`, which may be `u32` or `u64` depending on/// `CONFIG_ARCH_DMA_ADDR_T_64BIT`.////// Note that this may be `u64` even on 32-bit architectures.pub type DmaAddress = bindings::dma_addr_t; /// Trait to be implemented by DMA capable bus devices.////// The [`dma::Device`](Device) trait should be implemented by bus specific device representations,/// where the underlying bus is DMA capable, such as:#[cfg_attr(CONFIG_PCI, doc = "* [`pci::Device`](kernel::pci::Device)")]/// * [`platform::Device`](::kernel::platform::Device)pub trait Device<'a>: AsRef<device::Device<Core<'a>>> {    /// Set up the device's DMA streaming addressing capabilities.    ///    /// This method is usually called once from `probe()` as soon as the device capabilities are    /// known.    ///    /// # Safety    ///    /// This method must not be called concurrently with any DMA allocation or mapping primitives,    /// such as [`Coherent::zeroed`].    unsafe fn dma_set_mask(&self, mask: DmaMask) -> Result {        // SAFETY:        // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.        // - The safety requirement of this function guarantees that there are no concurrent calls        //   to DMA allocation and mapping primitives using this mask.        to_result(unsafe { bindings::dma_set_mask(self.as_ref().as_raw(), mask.value()) })    }     /// Set up the device's DMA coherent addressing capabilities.    ///    /// This method is usually called once from `probe()` as soon as the device capabilities are    /// known.    ///    /// # Safety    ///    /// This method must not be called concurrently with any DMA allocation or mapping primitives,    /// such as [`Coherent::zeroed`].    unsafe fn dma_set_coherent_mask(&self, mask: DmaMask) -> Result {        // SAFETY:        // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.        // - The safety requirement of this function guarantees that there are no concurrent calls        //   to DMA allocation and mapping primitives using this mask.        to_result(unsafe { bindings::dma_set_coherent_mask(self.as_ref().as_raw(), mask.value()) })    }     /// Set up the device's DMA addressing capabilities.    ///    /// This is a combination of [`Device::dma_set_mask`] and [`Device::dma_set_coherent_mask`].    ///    /// This method is usually called once from `probe()` as soon as the device capabilities are    /// known.    ///    /// # Safety    ///    /// This method must not be called concurrently with any DMA allocation or mapping primitives,    /// such as [`Coherent::zeroed`].    unsafe fn dma_set_mask_and_coherent(&self, mask: DmaMask) -> Result {        // SAFETY:        // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.        // - The safety requirement of this function guarantees that there are no concurrent calls        //   to DMA allocation and mapping primitives using this mask.        to_result(unsafe {            bindings::dma_set_mask_and_coherent(self.as_ref().as_raw(), mask.value())        })    }     /// Set the maximum size of a single DMA segment the device may request.    ///    /// This method is usually called once from `probe()` as soon as the device capabilities are    /// known.    ///    /// # Safety    ///    /// This method must not be called concurrently with any DMA allocation or mapping primitives,    /// such as [`Coherent::zeroed`].    unsafe fn dma_set_max_seg_size(&self, size: u32) {        // SAFETY:        // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.        // - The safety requirement of this function guarantees that there are no concurrent calls        //   to DMA allocation and mapping primitives using this parameter.        unsafe { bindings::dma_set_max_seg_size(self.as_ref().as_raw(), size) }    }} /// A DMA mask that holds a bitmask with the lowest `n` bits set.////// Use [`DmaMask::new`] or [`DmaMask::try_new`] to construct a value. Values/// are guaranteed to never exceed the bit width of `u64`.////// This is the Rust equivalent of the C macro `DMA_BIT_MASK()`.#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub struct DmaMask(u64); impl DmaMask {    /// Constructs a `DmaMask` with the lowest `n` bits set to `1`.    ///    /// For `n <= 64`, sets exactly the lowest `n` bits.    /// For `n > 64`, results in a build error.    ///    /// # Examples    ///    /// ```    /// use kernel::dma::DmaMask;    ///    /// let mask0 = DmaMask::new::<0>();    /// assert_eq!(mask0.value(), 0);    ///    /// let mask1 = DmaMask::new::<1>();    /// assert_eq!(mask1.value(), 0b1);    ///    /// let mask64 = DmaMask::new::<64>();    /// assert_eq!(mask64.value(), u64::MAX);    ///    /// // Build failure.    /// // let mask_overflow = DmaMask::new::<100>();    /// ```    #[inline]    pub const fn new<const N: u32>() -> Self {        let Ok(mask) = Self::try_new(N) else {            build_error!("Invalid DMA Mask.");        };         mask    }     /// Constructs a `DmaMask` with the lowest `n` bits set to `1`.    ///    /// For `n <= 64`, sets exactly the lowest `n` bits.    /// For `n > 64`, returns [`EINVAL`].    ///    /// # Examples    ///    /// ```    /// use kernel::dma::DmaMask;    ///    /// let mask0 = DmaMask::try_new(0)?;    /// assert_eq!(mask0.value(), 0);    ///    /// let mask1 = DmaMask::try_new(1)?;    /// assert_eq!(mask1.value(), 0b1);    ///    /// let mask64 = DmaMask::try_new(64)?;    /// assert_eq!(mask64.value(), u64::MAX);    ///    /// let mask_overflow = DmaMask::try_new(100);    /// assert!(mask_overflow.is_err());    /// # Ok::<(), Error>(())    /// ```    #[inline]    pub const fn try_new(n: u32) -> Result<Self> {        Ok(Self(match n {            0 => 0,            1..=64 => u64::MAX >> (64 - n),            _ => return Err(EINVAL),        }))    }     /// Returns the underlying `u64` bitmask value.    #[inline]    pub const fn value(&self) -> u64 {        self.0    }} /// Possible attributes associated with a DMA mapping.////// They can be combined with the operators `|`, `&`, and `!`.////// Values can be used from the [`attrs`] module.////// # Examples////// ```/// # use kernel::device::{Bound, Device};/// use kernel::dma::{attrs::*, Coherent};////// # fn test(dev: &Device<Bound>) -> Result {/// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN;/// let c: Coherent<[u64]> =///     Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, attribs)?;/// # Ok::<(), Error>(()) }/// ```#[derive(Clone, Copy, PartialEq)]#[repr(transparent)]pub struct Attrs(u32); impl Attrs {    /// Get the raw representation of this attribute.    pub(crate) fn as_raw(self) -> crate::ffi::c_ulong {        self.0 as crate::ffi::c_ulong    }     /// Check whether `flags` is contained in `self`.    pub fn contains(self, flags: Attrs) -> bool {        (self & flags) == flags    }} impl core::ops::BitOr for Attrs {    type Output = Self;    fn bitor(self, rhs: Self) -> Self::Output {        Self(self.0 | rhs.0)    }} impl core::ops::BitAnd for Attrs {    type Output = Self;    fn bitand(self, rhs: Self) -> Self::Output {        Self(self.0 & rhs.0)    }} impl core::ops::Not for Attrs {    type Output = Self;    fn not(self) -> Self::Output {        Self(!self.0)    }} /// DMA mapping attributes.pub mod attrs {    use super::Attrs;     /// Specifies that reads and writes to the mapping may be weakly ordered, that is that reads    /// and writes may pass each other.    pub const DMA_ATTR_WEAK_ORDERING: Attrs = Attrs(bindings::DMA_ATTR_WEAK_ORDERING);     /// Specifies that writes to the mapping may be buffered to improve performance.    pub const DMA_ATTR_WRITE_COMBINE: Attrs = Attrs(bindings::DMA_ATTR_WRITE_COMBINE);     /// Allows platform code to skip synchronization of the CPU cache for the given buffer assuming    /// that it has been already transferred to 'device' domain.    pub const DMA_ATTR_SKIP_CPU_SYNC: Attrs = Attrs(bindings::DMA_ATTR_SKIP_CPU_SYNC);     /// Forces contiguous allocation of the buffer in physical memory.    pub const DMA_ATTR_FORCE_CONTIGUOUS: Attrs = Attrs(bindings::DMA_ATTR_FORCE_CONTIGUOUS);     /// Hints DMA-mapping subsystem that it's probably not worth the time to try    /// to allocate memory to in a way that gives better TLB efficiency.    pub const DMA_ATTR_ALLOC_SINGLE_PAGES: Attrs = Attrs(bindings::DMA_ATTR_ALLOC_SINGLE_PAGES);     /// This tells the DMA-mapping subsystem to suppress allocation failure reports (similarly to    /// `__GFP_NOWARN`).    pub const DMA_ATTR_NO_WARN: Attrs = Attrs(bindings::DMA_ATTR_NO_WARN);     /// Indicates that the buffer is fully accessible at an elevated privilege level (and    /// ideally inaccessible or at least read-only at lesser-privileged levels).    pub const DMA_ATTR_PRIVILEGED: Attrs = Attrs(bindings::DMA_ATTR_PRIVILEGED);     /// Indicates that the buffer is MMIO memory.    pub const DMA_ATTR_MMIO: Attrs = Attrs(bindings::DMA_ATTR_MMIO);} /// DMA data direction.////// Corresponds to the C [`enum dma_data_direction`].////// [`enum dma_data_direction`]: srctree/include/linux/dma-direction.h#[derive(Copy, Clone, PartialEq, Eq, Debug)]#[repr(u32)]pub enum DataDirection {    /// The DMA mapping is for bidirectional data transfer.    ///    /// This is used when the buffer can be both read from and written to by the device.    /// The cache for the corresponding memory region is both flushed and invalidated.    Bidirectional = Self::const_cast(bindings::dma_data_direction_DMA_BIDIRECTIONAL),     /// The DMA mapping is for data transfer from memory to the device (write).    ///    /// The CPU has prepared data in the buffer, and the device will read it.    /// The cache for the corresponding memory region is flushed before device access.    ToDevice = Self::const_cast(bindings::dma_data_direction_DMA_TO_DEVICE),     /// The DMA mapping is for data transfer from the device to memory (read).    ///    /// The device will write data into the buffer for the CPU to read.    /// The cache for the corresponding memory region is invalidated before CPU access.    FromDevice = Self::const_cast(bindings::dma_data_direction_DMA_FROM_DEVICE),     /// The DMA mapping is not for data transfer.    ///    /// This is primarily for debugging purposes. With this direction, the DMA mapping API    /// will not perform any cache coherency operations.    None = Self::const_cast(bindings::dma_data_direction_DMA_NONE),} impl DataDirection {    /// Casts the bindgen-generated enum type to a `u32` at compile time.    ///    /// This function will cause a compile-time error if the underlying value of the    /// C enum is out of bounds for `u32`.    const fn const_cast(val: bindings::dma_data_direction) -> u32 {        // CAST: The C standard allows compilers to choose different integer types for enums.        // To safely check the value, we cast it to a wide signed integer type (`i128`)        // which can hold any standard C integer enum type without truncation.        let wide_val = val as i128;         // Check if the value is outside the valid range for the target type `u32`.        // CAST: `u32::MAX` is cast to `i128` to match the type of `wide_val` for the comparison.        if wide_val < 0 || wide_val > u32::MAX as i128 {            // Trigger a compile-time error in a const context.            build_error!("C enum value is out of bounds for the target type `u32`.");        }         // CAST: This cast is valid because the check above guarantees that `wide_val`        // is within the representable range of `u32`.        wide_val as u32    }} impl From<DataDirection> for bindings::dma_data_direction {    /// Returns the raw representation of [`enum dma_data_direction`].    fn from(direction: DataDirection) -> Self {        // CAST: `direction as u32` gets the underlying representation of our `#[repr(u32)]` enum.        // The subsequent cast to `Self` (the bindgen type) assumes the C enum is compatible        // with the enum variants of `DataDirection`, which is a valid assumption given our        // compile-time checks.        direction as u32 as Self    }} /// CPU-owned DMA allocation that can be converted into a device-shared [`Coherent`] object.////// Unlike [`Coherent`], a [`CoherentBox`] is guaranteed to be fully owned by the CPU -- its DMA/// address is not exposed and it cannot be accessed by a device. This means it can safely be used/// like a normal boxed allocation (e.g. direct reads, writes, and mutable slices are all safe).////// A typical use is to allocate a [`CoherentBox`], populate it with normal CPU access, and then/// convert it into a [`Coherent`] object to share it with the device.////// # Examples////// `CoherentBox<T>`:////// ```/// # use kernel::device::{/// #     Bound,/// #     Device,/// # };/// use kernel::dma::{attrs::*,///     Coherent,///     CoherentBox,/// };////// # fn test(dev: &Device<Bound>) -> Result {/// let mut dmem: CoherentBox<u64> = CoherentBox::zeroed(dev, GFP_KERNEL)?;/// *dmem = 42;/// let dmem: Coherent<u64> = dmem.into();/// # Ok::<(), Error>(()) }/// ```////// `CoherentBox<[T]>`:///////// ```/// # use kernel::device::{/// #     Bound,/// #     Device,/// # };/// use kernel::dma::{attrs::*,///     Coherent,///     CoherentBox,/// };////// # fn test(dev: &Device<Bound>) -> Result {/// let mut dmem: CoherentBox<[u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?;/// dmem.fill(42);/// let dmem: Coherent<[u64]> = dmem.into();/// # Ok::<(), Error>(()) }/// ```pub struct CoherentBox<T: KnownSize + ?Sized>(Coherent<T>); impl<T: AsBytes + FromBytes> CoherentBox<[T]> {    /// [`CoherentBox`] variant of [`Coherent::zeroed_slice_with_attrs`].    #[inline]    pub fn zeroed_slice_with_attrs(        dev: &device::Device<Bound>,        count: usize,        gfp_flags: kernel::alloc::Flags,        dma_attrs: Attrs,    ) -> Result<Self> {        Coherent::zeroed_slice_with_attrs(dev, count, gfp_flags, dma_attrs).map(Self)    }     /// Same as [CoherentBox::zeroed_slice_with_attrs], but with `dma::Attrs(0)`.    #[inline]    pub fn zeroed_slice(        dev: &device::Device<Bound>,        count: usize,        gfp_flags: kernel::alloc::Flags,    ) -> Result<Self> {        Self::zeroed_slice_with_attrs(dev, count, gfp_flags, Attrs(0))    }     /// Initializes the element at `i` using the given initializer.    ///    /// Returns `EINVAL` if `i` is out of bounds.    pub fn init_at<E>(&mut self, i: usize, init: impl Init<T, E>) -> Result    where        Error: From<E>,    {        if i >= self.0.len() {            return Err(EINVAL);        }         let ptr = &raw mut self[i];         // SAFETY:        // - `ptr` is valid, properly aligned, and within this allocation.        // - `T: AsBytes + FromBytes` guarantees all bit patterns are valid, so partial writes on        //   error cannot leave the element in an invalid state.        // - The DMA address has not been exposed yet, so there is no concurrent device access.        unsafe { init.__init(ptr)? };         Ok(())    }     /// Allocates a region of coherent memory of the same size as `data` and initializes it with a    /// copy of its contents.    ///    /// This is the [`CoherentBox`] variant of [`Coherent::from_slice_with_attrs`].    ///    /// # Examples    ///    /// ```    /// use core::ops::Deref;    ///    /// # use kernel::device::{Bound, Device};    /// use kernel::dma::{    ///     attrs::*,    ///     CoherentBox    /// };    ///    /// # fn test(dev: &Device<Bound>) -> Result {    /// let data = [0u8, 1u8, 2u8, 3u8];    /// let c: CoherentBox<[u8]> =    ///     CoherentBox::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?;    ///    /// assert_eq!(c.deref(), &data);    /// # Ok::<(), Error>(()) }    /// ```    pub fn from_slice_with_attrs(        dev: &device::Device<Bound>,        data: &[T],        gfp_flags: kernel::alloc::Flags,        dma_attrs: Attrs,    ) -> Result<Self>    where        T: Copy,    {        let mut slice = Self(Coherent::<T>::alloc_slice_with_attrs(            dev,            data.len(),            gfp_flags,            dma_attrs,        )?);         // PANIC: `slice` was created with length `data.len()`.        slice.copy_from_slice(data);         Ok(slice)

Lines 1–500 of 1216. Load more