// SPDX-License-Identifier: GPL-2.0 //! Abstractions for scatter-gather lists.//!//! C header: [`include/linux/scatterlist.h`](srctree/include/linux/scatterlist.h)//!//! Scatter-gather (SG) I/O is a memory access technique that allows devices to perform DMA//! operations on data buffers that are not physically contiguous in memory. It works by creating a//! "scatter-gather list", an array where each entry specifies the address and length of a//! physically contiguous memory segment.//!//! The device's DMA controller can then read this list and process the segments sequentially as//! part of one logical I/O request. This avoids the need for a single, large, physically contiguous//! memory buffer, which can be difficult or impossible to allocate.//!//! This module provides safe Rust abstractions over the kernel's `struct scatterlist` and//! `struct sg_table` types.//!//! The main entry point is the [`SGTable`] type, which represents a complete scatter-gather table.//! It can be either://!//! - An owned table ([`SGTable<Owned<P>>`]), created from a Rust memory buffer (e.g., [`VVec`]).//! This type manages the allocation of the `struct sg_table`, the DMA mapping of the buffer, and//! the automatic cleanup of all resources.//! - A borrowed reference (&[`SGTable`]), which provides safe, read-only access to a table that was//! allocated by other (e.g., C) code.//!//! Individual entries in the table are represented by [`SGEntry`], which can be accessed by//! iterating over an [`SGTable`]. use crate::{ alloc, alloc::allocator::VmallocPageIter, bindings, device::{Bound, Device}, devres::Devres, dma, error, io::ResourceSize, page, prelude::*, sync::aref::ARef, types::Opaque,};use core::{ops::Deref, ptr::NonNull}; /// A single entry in a scatter-gather list.////// An `SGEntry` represents a single, physically contiguous segment of memory that has been mapped/// for DMA.////// Instances of this struct are obtained by iterating over an [`SGTable`]. Drivers do not create/// or own [`SGEntry`] objects directly.#[repr(transparent)]pub struct SGEntry(Opaque<bindings::scatterlist>); // SAFETY: `SGEntry` can be sent to any task.unsafe impl Send for SGEntry {} // SAFETY: `SGEntry` has no interior mutability and can be accessed concurrently.unsafe impl Sync for SGEntry {} impl SGEntry { /// Convert a raw `struct scatterlist *` to a `&'a SGEntry`. /// /// # Safety /// /// Callers must ensure that the `struct scatterlist` pointed to by `ptr` is valid for the /// lifetime `'a`. #[inline] unsafe fn from_raw<'a>(ptr: *mut bindings::scatterlist) -> &'a Self { // SAFETY: The safety requirements of this function guarantee that `ptr` is a valid pointer // to a `struct scatterlist` for the duration of `'a`. unsafe { &*ptr.cast() } } /// Obtain the raw `struct scatterlist *`. #[inline] fn as_raw(&self) -> *mut bindings::scatterlist { self.0.get() } /// Returns the DMA address of this SG entry. /// /// This is the address that the device should use to access the memory segment. #[inline] pub fn dma_address(&self) -> dma::DmaAddress { // SAFETY: `self.as_raw()` is a valid pointer to a `struct scatterlist`. unsafe { bindings::sg_dma_address(self.as_raw()) } } /// Returns the length of this SG entry in bytes. #[inline] pub fn dma_len(&self) -> ResourceSize { #[allow(clippy::useless_conversion)] // SAFETY: `self.as_raw()` is a valid pointer to a `struct scatterlist`. unsafe { bindings::sg_dma_len(self.as_raw()) }.into() }} /// The borrowed generic type of an [`SGTable`], representing a borrowed or externally managed/// table.#[repr(transparent)]pub struct Borrowed(Opaque<bindings::sg_table>); // SAFETY: `Borrowed` can be sent to any task.unsafe impl Send for Borrowed {} // SAFETY: `Borrowed` has no interior mutability and can be accessed concurrently.unsafe impl Sync for Borrowed {} /// A scatter-gather table.////// This struct is a wrapper around the kernel's `struct sg_table`. It manages a list of DMA-mapped/// memory segments that can be passed to a device for I/O operations.////// The generic parameter `T` is used as a generic type to distinguish between owned and borrowed/// tables.////// - [`SGTable<Owned>`]: An owned table created and managed entirely by Rust code. It handles/// allocation, DMA mapping, and cleanup of all associated resources. See [`SGTable::new`]./// - [`SGTable<Borrowed>`} (or simply [`SGTable`]): Represents a table whose lifetime is managed/// externally. It can be used safely via a borrowed reference `&'a SGTable`, where `'a` is the/// external lifetime.////// All [`SGTable`] variants can be iterated over the individual [`SGEntry`]s.#[repr(transparent)]#[pin_data]pub struct SGTable<T: private::Sealed = Borrowed> { #[pin] inner: T,} impl SGTable { /// Creates a borrowed `&'a SGTable` from a raw `struct sg_table` pointer. /// /// This allows safe access to an `sg_table` that is managed elsewhere (for example, in C code). /// /// # Safety /// /// Callers must ensure that: /// /// - the `struct sg_table` pointed to by `ptr` is valid for the entire lifetime of `'a`, /// - the data behind `ptr` is not modified concurrently for the duration of `'a`. #[inline] pub unsafe fn from_raw<'a>(ptr: *mut bindings::sg_table) -> &'a Self { // SAFETY: The safety requirements of this function guarantee that `ptr` is a valid pointer // to a `struct sg_table` for the duration of `'a`. unsafe { &*ptr.cast() } } #[inline] fn as_raw(&self) -> *mut bindings::sg_table { self.inner.0.get() } /// Returns an [`SGTableIter`] bound to the lifetime of `self`. pub fn iter(&self) -> SGTableIter<'_> { // SAFETY: `self.as_raw()` is a valid pointer to a `struct sg_table`. let nents = unsafe { (*self.as_raw()).nents }; let pos = if nents > 0 { // SAFETY: `self.as_raw()` is a valid pointer to a `struct sg_table`. let ptr = unsafe { (*self.as_raw()).sgl }; // SAFETY: `ptr` is guaranteed to be a valid pointer to a `struct scatterlist`. Some(unsafe { SGEntry::from_raw(ptr) }) } else { None }; SGTableIter { pos, nents } }} /// Represents the DMA mapping state of a `struct sg_table`.////// This is used as an inner type of [`Owned`] to manage the DMA mapping lifecycle.////// # Invariants////// - `sgt` is a valid pointer to a `struct sg_table` for the entire lifetime of the/// [`DmaMappedSgt`]./// - `sgt` is always DMA mapped.struct DmaMappedSgt { sgt: NonNull<bindings::sg_table>, dev: ARef<Device>, dir: dma::DataDirection,} // SAFETY: `DmaMappedSgt` can be sent to any task.unsafe impl Send for DmaMappedSgt {} // SAFETY: `DmaMappedSgt` has no interior mutability and can be accessed concurrently.unsafe impl Sync for DmaMappedSgt {} impl DmaMappedSgt { /// # Safety /// /// - `sgt` must be a valid pointer to a `struct sg_table` for the entire lifetime of the /// returned [`DmaMappedSgt`]. /// - The caller must guarantee that `sgt` remains DMA mapped for the entire lifetime of /// [`DmaMappedSgt`]. unsafe fn new( sgt: NonNull<bindings::sg_table>, dev: &Device<Bound>, dir: dma::DataDirection, ) -> Result<Self> { // SAFETY: // - `dev.as_raw()` is a valid pointer to a `struct device`, which is guaranteed to be // bound to a driver for the duration of this call. // - `sgt` is a valid pointer to a `struct sg_table`. error::to_result(unsafe { bindings::dma_map_sgtable(dev.as_raw(), sgt.as_ptr(), dir.into(), 0) })?; // INVARIANT: By the safety requirements of this function it is guaranteed that `sgt` is // valid for the entire lifetime of this object instance. Ok(Self { sgt, dev: dev.into(), dir, }) }} impl Drop for DmaMappedSgt { #[inline] fn drop(&mut self) { // SAFETY: // - `self.dev.as_raw()` is a pointer to a valid `struct device`. // - `self.dev` is the same device the mapping has been created for in `Self::new()`. // - `self.sgt.as_ptr()` is a valid pointer to a `struct sg_table` by the type invariants // of `Self`. // - `self.dir` is the same `dma::DataDirection` the mapping has been created with in // `Self::new()`. unsafe { bindings::dma_unmap_sgtable(self.dev.as_raw(), self.sgt.as_ptr(), self.dir.into(), 0) }; }} /// A transparent wrapper around a `struct sg_table`.////// While we could also create the `struct sg_table` in the constructor of [`Owned`], we can't tear/// down the `struct sg_table` in [`Owned::drop`]; the drop order in [`Owned`] matters.#[repr(transparent)]struct RawSGTable(Opaque<bindings::sg_table>); // SAFETY: `RawSGTable` can be sent to any task.unsafe impl Send for RawSGTable {} // SAFETY: `RawSGTable` has no interior mutability and can be accessed concurrently.unsafe impl Sync for RawSGTable {} impl RawSGTable { /// # Safety /// /// - `pages` must be a slice of valid `struct page *`. /// - The pages pointed to by `pages` must remain valid for the entire lifetime of the returned /// [`RawSGTable`]. unsafe fn new( pages: &mut [*mut bindings::page], size: usize, max_segment: u32, flags: alloc::Flags, ) -> Result<Self> { // `sg_alloc_table_from_pages_segment()` expects at least one page, otherwise it // produces a NPE. if pages.is_empty() { return Err(EINVAL); } let sgt = Opaque::zeroed(); // SAFETY: // - `sgt.get()` is a valid pointer to uninitialized memory. // - As by the check above, `pages` is not empty. error::to_result(unsafe { bindings::sg_alloc_table_from_pages_segment( sgt.get(), pages.as_mut_ptr(), pages.len().try_into()?, 0, size, max_segment, flags.as_raw(), ) })?; Ok(Self(sgt)) } #[inline] fn as_raw(&self) -> *mut bindings::sg_table { self.0.get() }} impl Drop for RawSGTable { #[inline] fn drop(&mut self) { // SAFETY: `sgt` is a valid and initialized `struct sg_table`. unsafe { bindings::sg_free_table(self.0.get()) }; }} /// The [`Owned`] generic type of an [`SGTable`].////// A [`SGTable<Owned>`] signifies that the [`SGTable`] owns all associated resources:////// - The backing memory pages./// - The `struct sg_table` allocation (`sgt`)./// - The DMA mapping, managed through a [`Devres`]-managed `DmaMappedSgt`.////// Users interact with this type through the [`SGTable`] handle and do not need to manage/// [`Owned`] directly.#[pin_data]pub struct Owned<P> { // Note: The drop order is relevant; we first have to unmap the `struct sg_table`, then free the // `struct sg_table` and finally free the backing pages. #[pin] dma: Devres<DmaMappedSgt>, sgt: RawSGTable, _pages: P,} // SAFETY: `Owned` can be sent to any task if `P` can be send to any task.unsafe impl<P: Send> Send for Owned<P> {} // SAFETY: `Owned` has no interior mutability and can be accessed concurrently if `P` can be// accessed concurrently.unsafe impl<P: Sync> Sync for Owned<P> {} impl<P> Owned<P>where for<'a> P: page::AsPageIter<Iter<'a> = VmallocPageIter<'a>> + 'static,{ fn new( dev: &Device<Bound>, mut pages: P, dir: dma::DataDirection, flags: alloc::Flags, ) -> Result<impl PinInit<Self, Error> + '_> { let page_iter = pages.page_iter(); let size = page_iter.size(); let mut page_vec: KVec<*mut bindings::page> = KVec::with_capacity(page_iter.page_count(), flags)?; for page in page_iter { page_vec.push(page.as_ptr(), flags)?; } // `dma_max_mapping_size` returns `size_t`, but `sg_alloc_table_from_pages_segment()` takes // an `unsigned int`. // // SAFETY: `dev.as_raw()` is a valid pointer to a `struct device`. let max_segment = match unsafe { bindings::dma_max_mapping_size(dev.as_raw()) } { 0 => u32::MAX, max_segment => u32::try_from(max_segment).unwrap_or(u32::MAX), }; Ok(try_pin_init!(&this in Self { // SAFETY: // - `page_vec` is a `KVec` of valid `struct page *` obtained from `pages`. // - The pages contained in `pages` remain valid for the entire lifetime of the // `RawSGTable`. sgt: unsafe { RawSGTable::new(&mut page_vec, size, max_segment, flags) }?, dma <- { // SAFETY: `this` is a valid pointer to uninitialized memory. let sgt = unsafe { &raw mut (*this.as_ptr()).sgt }.cast(); // SAFETY: `sgt` is guaranteed to be non-null. let sgt = unsafe { NonNull::new_unchecked(sgt) }; // SAFETY: // - It is guaranteed that the object returned by `DmaMappedSgt::new` won't out-live // `sgt`. // - `sgt` is never DMA unmapped manually. Devres::new(dev, unsafe { DmaMappedSgt::new(sgt, dev, dir) }) }, _pages: pages, })) }} impl<P> SGTable<Owned<P>>where for<'a> P: page::AsPageIter<Iter<'a> = VmallocPageIter<'a>> + 'static,{ /// Allocates a new scatter-gather table from the given pages and maps it for DMA. /// /// This constructor creates a new [`SGTable<Owned>`] that takes ownership of `P`. /// It allocates a `struct sg_table`, populates it with entries corresponding to the physical /// pages of `P`, and maps the table for DMA with the specified [`Device`] and /// [`dma::DataDirection`]. /// /// The DMA mapping is managed through [`Devres`], ensuring that the DMA mapping is unmapped /// once the associated [`Device`] is unbound, or when the [`SGTable<Owned>`] is dropped. /// /// # Parameters /// /// * `dev`: The [`Device`] that will be performing the DMA. /// * `pages`: The entity providing the backing pages. It must implement [`page::AsPageIter`]. /// The ownership of this entity is moved into the new [`SGTable<Owned>`]. /// * `dir`: The [`dma::DataDirection`] of the DMA transfer. /// * `flags`: Allocation flags for internal allocations (e.g., [`GFP_KERNEL`]). /// /// # Examples /// /// ``` /// use kernel::{ /// device::{Bound, Device}, /// dma, page, /// prelude::*, /// scatterlist::{SGTable, Owned}, /// }; /// /// fn test(dev: &Device<Bound>) -> Result { /// let size = 4 * page::PAGE_SIZE; /// let pages = VVec::<u8>::with_capacity(size, GFP_KERNEL)?; /// /// let sgt = KBox::pin_init(SGTable::new( /// dev, /// pages, /// dma::DataDirection::ToDevice, /// GFP_KERNEL, /// ), GFP_KERNEL)?; /// /// Ok(()) /// } /// ``` pub fn new( dev: &Device<Bound>, pages: P, dir: dma::DataDirection, flags: alloc::Flags, ) -> impl PinInit<Self, Error> + '_ { try_pin_init!(Self { inner <- Owned::new(dev, pages, dir, flags)? }) }} impl<P> Deref for SGTable<Owned<P>> { type Target = SGTable; #[inline] fn deref(&self) -> &Self::Target { // SAFETY: // - `self.inner.sgt.as_raw()` is a valid pointer to a `struct sg_table` for the entire // lifetime of `self`. // - The backing `struct sg_table` is not modified for the entire lifetime of `self`. unsafe { SGTable::from_raw(self.inner.sgt.as_raw()) } }} mod private { pub trait Sealed {} impl Sealed for super::Borrowed {} impl<P> Sealed for super::Owned<P> {}} /// An [`Iterator`] over the DMA mapped [`SGEntry`] items of an [`SGTable`].////// Note that the existence of an [`SGTableIter`] does not guarantee that the [`SGEntry`] items/// actually remain DMA mapped; they are prone to be unmapped on device unbind.pub struct SGTableIter<'a> { pos: Option<&'a SGEntry>, /// The number of DMA mapped entries in a `struct sg_table`. nents: c_uint,} impl<'a> Iterator for SGTableIter<'a> { type Item = &'a SGEntry; fn next(&mut self) -> Option<Self::Item> { let entry = self.pos?; self.nents = self.nents.saturating_sub(1); // SAFETY: `entry.as_raw()` is a valid pointer to a `struct scatterlist`. let next = unsafe { bindings::sg_next(entry.as_raw()) }; self.pos = (!next.is_null() && self.nents > 0).then(|| { // SAFETY: If `next` is not NULL, `sg_next()` guarantees to return a valid pointer to // the next `struct scatterlist`. unsafe { SGEntry::from_raw(next) } }); Some(entry) }}
// SPDX-License-Identifier: GPL-2.0
//! Abstractions for scatter-gather lists.
//!
//! C header: [`include/linux/scatterlist.h`](srctree/include/linux/scatterlist.h)
//!
//! Scatter-gather (SG) I/O is a memory access technique that allows devices to perform DMA
//! operations on data buffers that are not physically contiguous in memory. It works by creating a
//! "scatter-gather list", an array where each entry specifies the address and length of a
//! physically contiguous memory segment.
//!
//! The device's DMA controller can then read this list and process the segments sequentially as
//! part of one logical I/O request. This avoids the need for a single, large, physically contiguous
//! memory buffer, which can be difficult or impossible to allocate.
//!
//! This module provides safe Rust abstractions over the kernel's `struct scatterlist` and
//! `struct sg_table` types.
//!
//! The main entry point is the [`SGTable`] type, which represents a complete scatter-gather table.
//! It can be either:
//!
//! - An owned table ([`SGTable<Owned<P>>`]), created from a Rust memory buffer (e.g., [`VVec`]).
//! This type manages the allocation of the `struct sg_table`, the DMA mapping of the buffer, and
//! the automatic cleanup of all resources.
//! - A borrowed reference (&[`SGTable`]), which provides safe, read-only access to a table that was
//! allocated by other (e.g., C) code.
//!
//! Individual entries in the table are represented by [`SGEntry`], which can be accessed by
//! iterating over an [`SGTable`].
use crate::{
alloc,
alloc::allocator::VmallocPageIter,
bindings,
device::{Bound, Device},
devres::Devres,
dma, error,
io::ResourceSize,
page,
prelude::*,
sync::aref::ARef,
types::Opaque,
};
use core::{ops::Deref, ptr::NonNull};
/// A single entry in a scatter-gather list.
///
/// An `SGEntry` represents a single, physically contiguous segment of memory that has been mapped
/// for DMA.
///
/// Instances of this struct are obtained by iterating over an [`SGTable`]. Drivers do not create
/// or own [`SGEntry`] objects directly.
#[repr(transparent)]
pub struct SGEntry(Opaque<bindings::scatterlist>);
// SAFETY: `SGEntry` can be sent to any task.
unsafe impl Send for SGEntry {}
// SAFETY: `SGEntry` has no interior mutability and can be accessed concurrently.
unsafe impl Sync for SGEntry {}
impl SGEntry {
/// Convert a raw `struct scatterlist *` to a `&'a SGEntry`.
///
/// # Safety
///
/// Callers must ensure that the `struct scatterlist` pointed to by `ptr` is valid for the
/// lifetime `'a`.
#[inline]
unsafe fn from_raw<'a>(ptr: *mut bindings::scatterlist) -> &'a Self {
// SAFETY: The safety requirements of this function guarantee that `ptr` is a valid pointer
// to a `struct scatterlist` for the duration of `'a`.
unsafe { &*ptr.cast() }
}
/// Obtain the raw `struct scatterlist *`.
#[inline]
fn as_raw(&self) -> *mut bindings::scatterlist {
self.0.get()
}
/// Returns the DMA address of this SG entry.
///
/// This is the address that the device should use to access the memory segment.
#[inline]
pub fn dma_address(&self) -> dma::DmaAddress {
// SAFETY: `self.as_raw()` is a valid pointer to a `struct scatterlist`.
unsafe { bindings::sg_dma_address(self.as_raw()) }
}
/// Returns the length of this SG entry in bytes.
#[inline]
pub fn dma_len(&self) -> ResourceSize {
#[allow(clippy::useless_conversion)]
// SAFETY: `self.as_raw()` is a valid pointer to a `struct scatterlist`.
unsafe { bindings::sg_dma_len(self.as_raw()) }.into()
}
}
/// The borrowed generic type of an [`SGTable`], representing a borrowed or externally managed
/// table.
#[repr(transparent)]
pub struct Borrowed(Opaque<bindings::sg_table>);
// SAFETY: `Borrowed` can be sent to any task.
unsafe impl Send for Borrowed {}
// SAFETY: `Borrowed` has no interior mutability and can be accessed concurrently.
unsafe impl Sync for Borrowed {}
/// A scatter-gather table.
///
/// This struct is a wrapper around the kernel's `struct sg_table`. It manages a list of DMA-mapped
/// memory segments that can be passed to a device for I/O operations.
///
/// The generic parameter `T` is used as a generic type to distinguish between owned and borrowed
/// tables.
///
/// - [`SGTable<Owned>`]: An owned table created and managed entirely by Rust code. It handles
/// allocation, DMA mapping, and cleanup of all associated resources. See [`SGTable::new`].
/// - [`SGTable<Borrowed>`} (or simply [`SGTable`]): Represents a table whose lifetime is managed
/// externally. It can be used safely via a borrowed reference `&'a SGTable`, where `'a` is the
/// external lifetime.
///
/// All [`SGTable`] variants can be iterated over the individual [`SGEntry`]s.
#[repr(transparent)]
#[pin_data]
pub struct SGTable<T: private::Sealed = Borrowed> {
#[pin]
inner: T,
}
impl SGTable {
/// Creates a borrowed `&'a SGTable` from a raw `struct sg_table` pointer.
///
/// This allows safe access to an `sg_table` that is managed elsewhere (for example, in C code).
///
/// # Safety
///
/// Callers must ensure that:
///
/// - the `struct sg_table` pointed to by `ptr` is valid for the entire lifetime of `'a`,
/// - the data behind `ptr` is not modified concurrently for the duration of `'a`.
#[inline]
pub unsafe fn from_raw<'a>(ptr: *mut bindings::sg_table) -> &'a Self {
// SAFETY: The safety requirements of this function guarantee that `ptr` is a valid pointer
// to a `struct sg_table` for the duration of `'a`.
unsafe { &*ptr.cast() }
}
#[inline]
fn as_raw(&self) -> *mut bindings::sg_table {
self.inner.0.get()
}
/// Returns an [`SGTableIter`] bound to the lifetime of `self`.
pub fn iter(&self) -> SGTableIter<'_> {
// SAFETY: `self.as_raw()` is a valid pointer to a `struct sg_table`.
let nents = unsafe { (*self.as_raw()).nents };
let pos = if nents > 0 {
// SAFETY: `self.as_raw()` is a valid pointer to a `struct sg_table`.
let ptr = unsafe { (*self.as_raw()).sgl };
// SAFETY: `ptr` is guaranteed to be a valid pointer to a `struct scatterlist`.
Some(unsafe { SGEntry::from_raw(ptr) })
} else {
None
};
SGTableIter { pos, nents }
}
}
/// Represents the DMA mapping state of a `struct sg_table`.
///
/// This is used as an inner type of [`Owned`] to manage the DMA mapping lifecycle.
///
/// # Invariants
///
/// - `sgt` is a valid pointer to a `struct sg_table` for the entire lifetime of the
/// [`DmaMappedSgt`].
/// - `sgt` is always DMA mapped.
struct DmaMappedSgt {
sgt: NonNull<bindings::sg_table>,
dev: ARef<Device>,
dir: dma::DataDirection,
}
// SAFETY: `DmaMappedSgt` can be sent to any task.
unsafe impl Send for DmaMappedSgt {}
// SAFETY: `DmaMappedSgt` has no interior mutability and can be accessed concurrently.
unsafe impl Sync for DmaMappedSgt {}
impl DmaMappedSgt {
/// # Safety
///
/// - `sgt` must be a valid pointer to a `struct sg_table` for the entire lifetime of the
/// returned [`DmaMappedSgt`].
/// - The caller must guarantee that `sgt` remains DMA mapped for the entire lifetime of
/// [`DmaMappedSgt`].
unsafe fn new(
sgt: NonNull<bindings::sg_table>,
dev: &Device<Bound>,
dir: dma::DataDirection,
) -> Result<Self> {
// SAFETY:
// - `dev.as_raw()` is a valid pointer to a `struct device`, which is guaranteed to be
// bound to a driver for the duration of this call.
// - `sgt` is a valid pointer to a `struct sg_table`.
error::to_result(unsafe {
bindings::dma_map_sgtable(dev.as_raw(), sgt.as_ptr(), dir.into(), 0)
})?;
// INVARIANT: By the safety requirements of this function it is guaranteed that `sgt` is
// valid for the entire lifetime of this object instance.
Ok(Self {
sgt,
dev: dev.into(),
dir,
})
}
}
impl Drop for DmaMappedSgt {
#[inline]
fn drop(&mut self) {
// SAFETY:
// - `self.dev.as_raw()` is a pointer to a valid `struct device`.
// - `self.dev` is the same device the mapping has been created for in `Self::new()`.
// - `self.sgt.as_ptr()` is a valid pointer to a `struct sg_table` by the type invariants
// of `Self`.
// - `self.dir` is the same `dma::DataDirection` the mapping has been created with in
// `Self::new()`.
unsafe {
bindings::dma_unmap_sgtable(self.dev.as_raw(), self.sgt.as_ptr(), self.dir.into(), 0)
};
}
}
/// A transparent wrapper around a `struct sg_table`.
///
/// While we could also create the `struct sg_table` in the constructor of [`Owned`], we can't tear
/// down the `struct sg_table` in [`Owned::drop`]; the drop order in [`Owned`] matters.
#[repr(transparent)]
struct RawSGTable(Opaque<bindings::sg_table>);
// SAFETY: `RawSGTable` can be sent to any task.
unsafe impl Send for RawSGTable {}
// SAFETY: `RawSGTable` has no interior mutability and can be accessed concurrently.
unsafe impl Sync for RawSGTable {}
impl RawSGTable {
/// # Safety
///
/// - `pages` must be a slice of valid `struct page *`.
/// - The pages pointed to by `pages` must remain valid for the entire lifetime of the returned
/// [`RawSGTable`].
unsafe fn new(
pages: &mut [*mut bindings::page],
size: usize,
max_segment: u32,
flags: alloc::Flags,
) -> Result<Self> {
// `sg_alloc_table_from_pages_segment()` expects at least one page, otherwise it
// produces a NPE.
if pages.is_empty() {
return Err(EINVAL);
}
let sgt = Opaque::zeroed();
// SAFETY:
// - `sgt.get()` is a valid pointer to uninitialized memory.
// - As by the check above, `pages` is not empty.
error::to_result(unsafe {
bindings::sg_alloc_table_from_pages_segment(
sgt.get(),
pages.as_mut_ptr(),
pages.len().try_into()?,
0,
size,
max_segment,
flags.as_raw(),
)
})?;
Ok(Self(sgt))
}
#[inline]
fn as_raw(&self) -> *mut bindings::sg_table {
self.0.get()
}
}
impl Drop for RawSGTable {
#[inline]
fn drop(&mut self) {
// SAFETY: `sgt` is a valid and initialized `struct sg_table`.
unsafe { bindings::sg_free_table(self.0.get()) };
}
}
/// The [`Owned`] generic type of an [`SGTable`].
///
/// A [`SGTable<Owned>`] signifies that the [`SGTable`] owns all associated resources:
///
/// - The backing memory pages.
/// - The `struct sg_table` allocation (`sgt`).
/// - The DMA mapping, managed through a [`Devres`]-managed `DmaMappedSgt`.
///
/// Users interact with this type through the [`SGTable`] handle and do not need to manage
/// [`Owned`] directly.
#[pin_data]
pub struct Owned<P> {
// Note: The drop order is relevant; we first have to unmap the `struct sg_table`, then free the
// `struct sg_table` and finally free the backing pages.
#[pin]
dma: Devres<DmaMappedSgt>,
sgt: RawSGTable,
_pages: P,
}
// SAFETY: `Owned` can be sent to any task if `P` can be send to any task.
unsafe impl<P: Send> Send for Owned<P> {}
// SAFETY: `Owned` has no interior mutability and can be accessed concurrently if `P` can be
// accessed concurrently.
unsafe impl<P: Sync> Sync for Owned<P> {}
impl<P> Owned<P>
where
for<'a> P: page::AsPageIter<Iter<'a> = VmallocPageIter<'a>> + 'static,
{
fn new(
dev: &Device<Bound>,
mut pages: P,
dir: dma::DataDirection,
flags: alloc::Flags,
) -> Result<impl PinInit<Self, Error> + '_> {
let page_iter = pages.page_iter();
let size = page_iter.size();
let mut page_vec: KVec<*mut bindings::page> =
KVec::with_capacity(page_iter.page_count(), flags)?;
for page in page_iter {
page_vec.push(page.as_ptr(), flags)?;
}
// `dma_max_mapping_size` returns `size_t`, but `sg_alloc_table_from_pages_segment()` takes
// an `unsigned int`.
//
// SAFETY: `dev.as_raw()` is a valid pointer to a `struct device`.
let max_segment = match unsafe { bindings::dma_max_mapping_size(dev.as_raw()) } {
0 => u32::MAX,
max_segment => u32::try_from(max_segment).unwrap_or(u32::MAX),
};
Ok(try_pin_init!(&this in Self {
// SAFETY:
// - `page_vec` is a `KVec` of valid `struct page *` obtained from `pages`.
// - The pages contained in `pages` remain valid for the entire lifetime of the
// `RawSGTable`.
sgt: unsafe { RawSGTable::new(&mut page_vec, size, max_segment, flags) }?,
dma <- {
// SAFETY: `this` is a valid pointer to uninitialized memory.
let sgt = unsafe { &raw mut (*this.as_ptr()).sgt }.cast();
// SAFETY: `sgt` is guaranteed to be non-null.
let sgt = unsafe { NonNull::new_unchecked(sgt) };
// SAFETY:
// - It is guaranteed that the object returned by `DmaMappedSgt::new` won't out-live
// `sgt`.
// - `sgt` is never DMA unmapped manually.
Devres::new(dev, unsafe { DmaMappedSgt::new(sgt, dev, dir) })
},
_pages: pages,
}))
}
}
impl<P> SGTable<Owned<P>>
where
for<'a> P: page::AsPageIter<Iter<'a> = VmallocPageIter<'a>> + 'static,
{
/// Allocates a new scatter-gather table from the given pages and maps it for DMA.
///
/// This constructor creates a new [`SGTable<Owned>`] that takes ownership of `P`.
/// It allocates a `struct sg_table`, populates it with entries corresponding to the physical
/// pages of `P`, and maps the table for DMA with the specified [`Device`] and
/// [`dma::DataDirection`].
///
/// The DMA mapping is managed through [`Devres`], ensuring that the DMA mapping is unmapped
/// once the associated [`Device`] is unbound, or when the [`SGTable<Owned>`] is dropped.
///
/// # Parameters
///
/// * `dev`: The [`Device`] that will be performing the DMA.
/// * `pages`: The entity providing the backing pages. It must implement [`page::AsPageIter`].
/// The ownership of this entity is moved into the new [`SGTable<Owned>`].
/// * `dir`: The [`dma::DataDirection`] of the DMA transfer.
/// * `flags`: Allocation flags for internal allocations (e.g., [`GFP_KERNEL`]).
///
/// # Examples
///
/// ```
/// use kernel::{
/// device::{Bound, Device},
/// dma, page,
/// prelude::*,
/// scatterlist::{SGTable, Owned},
/// };
///
/// fn test(dev: &Device<Bound>) -> Result {
/// let size = 4 * page::PAGE_SIZE;
/// let pages = VVec::<u8>::with_capacity(size, GFP_KERNEL)?;
///
/// let sgt = KBox::pin_init(SGTable::new(
/// dev,
/// pages,
/// dma::DataDirection::ToDevice,
/// GFP_KERNEL,
/// ), GFP_KERNEL)?;
///
/// Ok(())
/// }
/// ```
pub fn new(
dev: &Device<Bound>,
pages: P,
dir: dma::DataDirection,
flags: alloc::Flags,
) -> impl PinInit<Self, Error> + '_ {
try_pin_init!(Self {
inner <- Owned::new(dev, pages, dir, flags)?
})
}
}
impl<P> Deref for SGTable<Owned<P>> {
type Target = SGTable;
#[inline]
fn deref(&self) -> &Self::Target {
// SAFETY:
// - `self.inner.sgt.as_raw()` is a valid pointer to a `struct sg_table` for the entire
// lifetime of `self`.
// - The backing `struct sg_table` is not modified for the entire lifetime of `self`.
unsafe { SGTable::from_raw(self.inner.sgt.as_raw()) }
}
}
mod private {
pub trait Sealed {}
impl Sealed for super::Borrowed {}
impl<P> Sealed for super::Owned<P> {}
}
/// An [`Iterator`] over the DMA mapped [`SGEntry`] items of an [`SGTable`].
///
/// Note that the existence of an [`SGTableIter`] does not guarantee that the [`SGEntry`] items
/// actually remain DMA mapped; they are prone to be unmapped on device unbind.
pub struct SGTableIter<'a> {
pos: Option<&'a SGEntry>,
/// The number of DMA mapped entries in a `struct sg_table`.
nents: c_uint,
}
impl<'a> Iterator for SGTableIter<'a> {
type Item = &'a SGEntry;
fn next(&mut self) -> Option<Self::Item> {
let entry = self.pos?;
self.nents = self.nents.saturating_sub(1);
// SAFETY: `entry.as_raw()` is a valid pointer to a `struct scatterlist`.
let next = unsafe { bindings::sg_next(entry.as_raw()) };
self.pos = (!next.is_null() && self.nents > 0).then(|| {
// SAFETY: If `next` is not NULL, `sg_next()` guarantees to return a valid pointer to
// the next `struct scatterlist`.
unsafe { SGEntry::from_raw(next) }
});
Some(entry)
}
}