History

Merge tag 'drm-rust-fixes-2026-03-12' of https://gitlab.freedesktop.org/drm/rust/kernel into drm-fixes

Core Changes: - Fix safety issue in dma_read! and dma_write!. Driver Changes (Nova Core): - Fix UB in DmaGspMem pointer accessors. - Fix stack overflow in GSP memory allocation. Signed-off-by: Dave Airlie <airlied@redhat.com>

Dave Airlie committed 8 months ago

Commit b28913e2 parents dd03650 0073a1710 files · +534 −195Browse files

Unified · Split

10 files
  1. drivers/gpu/nova-core/gsp.rs +24 −22
  2. drivers/gpu/nova-core/gsp/boot.rs +1 −1
  3. drivers/gpu/nova-core/gsp/cmdq.rs +33 −60
  4. drivers/gpu/nova-core/gsp/fw.rs +69 −32
  5. rust/kernel/dma.rs +50 −64
  6. rust/kernel/lib.rs +4 −0
  7. rust/kernel/ptr.rs +29 −1
  8. rust/kernel/ptr/projection.rs +305 −0
  9. samples/rust/rust_dma.rs +16 −14
  10. scripts/Makefile.build +3 −1
drivers/gpu/nova-core/gsp.rs+24−22
@@ -47,16 +47,12 @@ struct PteArray<const NUM_ENTRIES: usize>([u64; NUM_ENTRIES]); · 46 unmodified lines
4747 unsafe impl<const NUM_ENTRIES: usize> AsBytes for PteArray<NUM_ENTRIES> {}
4848
4949 impl<const NUM_PAGES: usize> PteArray<NUM_PAGES> {
50− /// Creates a new page table array mapping `NUM_PAGES` GSP pages starting at address `start`.
51− fn new(start: DmaAddress) -> Result<Self> {
52− let mut ptes = [0u64; NUM_PAGES];
53− for (i, pte) in ptes.iter_mut().enumerate() {
54− *pte = start
55− .checked_add(num::usize_as_u64(i) << GSP_PAGE_SHIFT)
56− .ok_or(EOVERFLOW)?;
57− }
58−
59− Ok(Self(ptes))
50+ /// Returns the page table entry for `index`, for a mapping starting at `start`.
51+ // TODO: Replace with `IoView` projection once available.
52+ fn entry(start: DmaAddress, index: usize) -> Result<u64> {
53+ start
54+ .checked_add(num::usize_as_u64(index) << GSP_PAGE_SHIFT)
55+ .ok_or(EOVERFLOW)
6056 }
6157 }
6258
@@ -86,16 +82,22 @@ impl LogBuffer { · 23 unmodified lines
8682 NUM_PAGES * GSP_PAGE_SIZE,
8783 GFP_KERNEL | __GFP_ZERO,
8884 )?);
89− let ptes = PteArray::<NUM_PAGES>::new(obj.0.dma_handle())?;
85+
86+ let start_addr = obj.0.dma_handle();
9087
9188 // SAFETY: `obj` has just been created and we are its sole user.
92− unsafe {
93− // Copy the self-mapping PTE at the expected location.
89+ let pte_region = unsafe {
9490 obj.0
95− .as_slice_mut(size_of::<u64>(), size_of_val(&ptes))?
96− .copy_from_slice(ptes.as_bytes())
91+ .as_slice_mut(size_of::<u64>(), NUM_PAGES * size_of::<u64>())?
9792 };
9893
94+ // Write values one by one to avoid an on-stack instance of `PteArray`.
95+ for (i, chunk) in pte_region.chunks_exact_mut(size_of::<u64>()).enumerate() {
96+ let pte_value = PteArray::<0>::entry(start_addr, i)?;
97+
98+ chunk.copy_from_slice(&pte_value.to_ne_bytes());
99+ }
100+
99101 Ok(obj)
100102 }
101103 }
@@ -143,14 +145,14 @@ impl Gsp { · 41 unmodified lines
143145 // _kgspInitLibosLoggingStructures (allocates memory for buffers)
144146 // kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array)
145147 dma_write!(
146− libos[0] = LibosMemoryRegionInitArgument::new("LOGINIT", &loginit.0)
147− )?;
148+ libos, [0]?, LibosMemoryRegionInitArgument::new("LOGINIT", &loginit.0)
149+ );
148150 dma_write!(
149− libos[1] = LibosMemoryRegionInitArgument::new("LOGINTR", &logintr.0)
150− )?;
151− dma_write!(libos[2] = LibosMemoryRegionInitArgument::new("LOGRM", &logrm.0))?;
152− dma_write!(rmargs[0].inner = fw::GspArgumentsCached::new(cmdq))?;
153− dma_write!(libos[3] = LibosMemoryRegionInitArgument::new("RMARGS", rmargs))?;
151+ libos, [1]?, LibosMemoryRegionInitArgument::new("LOGINTR", &logintr.0)
152+ );
153+ dma_write!(libos, [2]?, LibosMemoryRegionInitArgument::new("LOGRM", &logrm.0));
154+ dma_write!(rmargs, [0]?.inner, fw::GspArgumentsCached::new(cmdq));
155+ dma_write!(libos, [3]?, LibosMemoryRegionInitArgument::new("RMARGS", rmargs));
154156 },
155157 }))
156158 })
drivers/gpu/nova-core/gsp/boot.rs+1−1
@@ -157,7 +157,7 @@ impl super::Gsp { · 156 unmodified lines
157157
158158 let wpr_meta =
159159 CoherentAllocation::<GspFwWprMeta>::alloc_coherent(dev, 1, GFP_KERNEL | __GFP_ZERO)?;
160− dma_write!(wpr_meta[0] = GspFwWprMeta::new(&gsp_fw, &fb_layout))?;
160+ dma_write!(wpr_meta, [0]?, GspFwWprMeta::new(&gsp_fw, &fb_layout));
161161
162162 self.cmdq
163163 .send_command(bar, commands::SetSystemInfo::new(pdev))?;
drivers/gpu/nova-core/gsp/cmdq.rs+33−60
@@ -2,11 +2,7 @@ · 1 unmodified line
22
33 use core::{
44 cmp,
5− mem,
6− sync::atomic::{
7− fence,
8− Ordering, //
9− }, //
5+ mem, //
106 };
117
128 use kernel::{
@@ -146,30 +142,36 @@ static_assert!(align_of::<MsgqData>() == GSP_PAGE_SIZE); · 133 unmodified lines
146142 #[repr(C)]
147143 // There is no struct defined for this in the open-gpu-kernel-source headers.
148144 // Instead it is defined by code in `GspMsgQueuesInit()`.
149−struct Msgq {
145+// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module.
146+pub(super) struct Msgq {
150147 /// Header for sending messages, including the write pointer.
151− tx: MsgqTxHeader,
148+ pub(super) tx: MsgqTxHeader,
152149 /// Header for receiving messages, including the read pointer.
153− rx: MsgqRxHeader,
150+ pub(super) rx: MsgqRxHeader,
154151 /// The message queue proper.
155152 msgq: MsgqData,
156153 }
157154
158155 /// Structure shared between the driver and the GSP and containing the command and message queues.
159156 #[repr(C)]
160−struct GspMem {
157+// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module.
158+pub(super) struct GspMem {
161159 /// Self-mapping page table entries.
162− ptes: PteArray<{ GSP_PAGE_SIZE / size_of::<u64>() }>,
160+ ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>,
163161 /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the
164162 /// write and read pointers that the CPU updates.
165163 ///
166164 /// This member is read-only for the GSP.
167− cpuq: Msgq,
165+ pub(super) cpuq: Msgq,
168166 /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the
169167 /// write and read pointers that the GSP updates.
170168 ///
171169 /// This member is read-only for the driver.
172− gspq: Msgq,
170+ pub(super) gspq: Msgq,
171+}
172+
173+impl GspMem {
174+ const PTE_ARRAY_SIZE: usize = GSP_PAGE_SIZE / size_of::<u64>();
173175 }
174176
175177 // SAFETY: These structs don't meet the no-padding requirements of AsBytes but
@@ -201,9 +203,19 @@ impl DmaGspMem { · 25 unmodified lines
201203
202204 let gsp_mem =
203205 CoherentAllocation::<GspMem>::alloc_coherent(dev, 1, GFP_KERNEL | __GFP_ZERO)?;
204− dma_write!(gsp_mem[0].ptes = PteArray::new(gsp_mem.dma_handle())?)?;
205− dma_write!(gsp_mem[0].cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES))?;
206− dma_write!(gsp_mem[0].cpuq.rx = MsgqRxHeader::new())?;
206+
207+ let start = gsp_mem.dma_handle();
208+ // Write values one by one to avoid an on-stack instance of `PteArray`.
209+ for i in 0..GspMem::PTE_ARRAY_SIZE {
210+ dma_write!(gsp_mem, [0]?.ptes.0[i], PteArray::<0>::entry(start, i)?);
211+ }
212+
213+ dma_write!(
214+ gsp_mem,
215+ [0]?.cpuq.tx,
216+ MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES)
217+ );
218+ dma_write!(gsp_mem, [0]?.cpuq.rx, MsgqRxHeader::new());
207219
208220 Ok(Self(gsp_mem))
209221 }
@@ -317,12 +329,7 @@ impl DmaGspMem { · 107 unmodified lines
317329 //
318330 // - The returned value is between `0` and `MSGQ_NUM_PAGES`.
319331 fn gsp_write_ptr(&self) -> u32 {
320− let gsp_mem = self.0.start_ptr();
321−
322− // SAFETY:
323− // - The 'CoherentAllocation' contains at least one object.
324− // - By the invariants of `CoherentAllocation` the pointer is valid.
325− (unsafe { (*gsp_mem).gspq.tx.write_ptr() } % MSGQ_NUM_PAGES)
332+ super::fw::gsp_mem::gsp_write_ptr(&self.0)
326333 }
327334
328335 // Returns the index of the memory page the GSP will read the next command from.
@@ -331,12 +338,7 @@ impl DmaGspMem { · 2 unmodified lines
331338 //
332339 // - The returned value is between `0` and `MSGQ_NUM_PAGES`.
333340 fn gsp_read_ptr(&self) -> u32 {
334− let gsp_mem = self.0.start_ptr();
335−
336− // SAFETY:
337− // - The 'CoherentAllocation' contains at least one object.
338− // - By the invariants of `CoherentAllocation` the pointer is valid.
339− (unsafe { (*gsp_mem).gspq.rx.read_ptr() } % MSGQ_NUM_PAGES)
341+ super::fw::gsp_mem::gsp_read_ptr(&self.0)
340342 }
341343
342344 // Returns the index of the memory page the CPU can read the next message from.
@@ -345,27 +347,12 @@ impl DmaGspMem { · 2 unmodified lines
345347 //
346348 // - The returned value is between `0` and `MSGQ_NUM_PAGES`.
347349 fn cpu_read_ptr(&self) -> u32 {
348− let gsp_mem = self.0.start_ptr();
349−
350− // SAFETY:
351− // - The ['CoherentAllocation'] contains at least one object.
352− // - By the invariants of CoherentAllocation the pointer is valid.
353− (unsafe { (*gsp_mem).cpuq.rx.read_ptr() } % MSGQ_NUM_PAGES)
350+ super::fw::gsp_mem::cpu_read_ptr(&self.0)
354351 }
355352
356353 // Informs the GSP that it can send `elem_count` new pages into the message queue.
357354 fn advance_cpu_read_ptr(&mut self, elem_count: u32) {
358− let rptr = self.cpu_read_ptr().wrapping_add(elem_count) % MSGQ_NUM_PAGES;
359−
360− // Ensure read pointer is properly ordered.
361− fence(Ordering::SeqCst);
362−
363− let gsp_mem = self.0.start_ptr_mut();
364−
365− // SAFETY:
366− // - The 'CoherentAllocation' contains at least one object.
367− // - By the invariants of `CoherentAllocation` the pointer is valid.
368− unsafe { (*gsp_mem).cpuq.rx.set_read_ptr(rptr) };
355+ super::fw::gsp_mem::advance_cpu_read_ptr(&self.0, elem_count)
369356 }
370357
371358 // Returns the index of the memory page the CPU can write the next command to.
@@ -374,26 +361,12 @@ impl DmaGspMem { · 2 unmodified lines
374361 //
375362 // - The returned value is between `0` and `MSGQ_NUM_PAGES`.
376363 fn cpu_write_ptr(&self) -> u32 {
377− let gsp_mem = self.0.start_ptr();
378−
379− // SAFETY:
380− // - The 'CoherentAllocation' contains at least one object.
381− // - By the invariants of `CoherentAllocation` the pointer is valid.
382− (unsafe { (*gsp_mem).cpuq.tx.write_ptr() } % MSGQ_NUM_PAGES)
364+ super::fw::gsp_mem::cpu_write_ptr(&self.0)
383365 }
384366
385367 // Informs the GSP that it can process `elem_count` new pages from the command queue.
386368 fn advance_cpu_write_ptr(&mut self, elem_count: u32) {
387− let wptr = self.cpu_write_ptr().wrapping_add(elem_count) & MSGQ_NUM_PAGES;
388− let gsp_mem = self.0.start_ptr_mut();
389−
390− // SAFETY:
391− // - The 'CoherentAllocation' contains at least one object.
392− // - By the invariants of `CoherentAllocation` the pointer is valid.
393− unsafe { (*gsp_mem).cpuq.tx.set_write_ptr(wptr) };
394−
395− // Ensure all command data is visible before triggering the GSP read.
396− fence(Ordering::SeqCst);
369+ super::fw::gsp_mem::advance_cpu_write_ptr(&self.0, elem_count)
397370 }
398371 }
399372
drivers/gpu/nova-core/gsp/fw.rs+69−32
@@ -40,6 +40,75 @@ use crate::{ · 39 unmodified lines
4040 },
4141 };
4242
43+// TODO: Replace with `IoView` projections once available; the `unwrap()` calls go away once we
44+// switch to the new `dma::Coherent` API.
45+pub(super) mod gsp_mem {
46+ use core::sync::atomic::{
47+ fence,
48+ Ordering, //
49+ };
50+
51+ use kernel::{
52+ dma::CoherentAllocation,
53+ dma_read,
54+ dma_write,
55+ prelude::*, //
56+ };
57+
58+ use crate::gsp::cmdq::{
59+ GspMem,
60+ MSGQ_NUM_PAGES, //
61+ };
62+
63+ pub(in crate::gsp) fn gsp_write_ptr(qs: &CoherentAllocation<GspMem>) -> u32 {
64+ // PANIC: A `dma::CoherentAllocation` always contains at least one element.
65+ || -> Result<u32> { Ok(dma_read!(qs, [0]?.gspq.tx.0.writePtr) % MSGQ_NUM_PAGES) }().unwrap()
66+ }
67+
68+ pub(in crate::gsp) fn gsp_read_ptr(qs: &CoherentAllocation<GspMem>) -> u32 {
69+ // PANIC: A `dma::CoherentAllocation` always contains at least one element.
70+ || -> Result<u32> { Ok(dma_read!(qs, [0]?.gspq.rx.0.readPtr) % MSGQ_NUM_PAGES) }().unwrap()
71+ }
72+
73+ pub(in crate::gsp) fn cpu_read_ptr(qs: &CoherentAllocation<GspMem>) -> u32 {
74+ // PANIC: A `dma::CoherentAllocation` always contains at least one element.
75+ || -> Result<u32> { Ok(dma_read!(qs, [0]?.cpuq.rx.0.readPtr) % MSGQ_NUM_PAGES) }().unwrap()
76+ }
77+
78+ pub(in crate::gsp) fn advance_cpu_read_ptr(qs: &CoherentAllocation<GspMem>, count: u32) {
79+ let rptr = cpu_read_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES;
80+
81+ // Ensure read pointer is properly ordered.
82+ fence(Ordering::SeqCst);
83+
84+ // PANIC: A `dma::CoherentAllocation` always contains at least one element.
85+ || -> Result {
86+ dma_write!(qs, [0]?.cpuq.rx.0.readPtr, rptr);
87+ Ok(())
88+ }()
89+ .unwrap()
90+ }
91+
92+ pub(in crate::gsp) fn cpu_write_ptr(qs: &CoherentAllocation<GspMem>) -> u32 {
93+ // PANIC: A `dma::CoherentAllocation` always contains at least one element.
94+ || -> Result<u32> { Ok(dma_read!(qs, [0]?.cpuq.tx.0.writePtr) % MSGQ_NUM_PAGES) }().unwrap()
95+ }
96+
97+ pub(in crate::gsp) fn advance_cpu_write_ptr(qs: &CoherentAllocation<GspMem>, count: u32) {
98+ let wptr = cpu_write_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES;
99+
100+ // PANIC: A `dma::CoherentAllocation` always contains at least one element.
101+ || -> Result {
102+ dma_write!(qs, [0]?.cpuq.tx.0.writePtr, wptr);
103+ Ok(())
104+ }()
105+ .unwrap();
106+
107+ // Ensure all command data is visible before triggering the GSP read.
108+ fence(Ordering::SeqCst);
109+ }
110+}
111+
43112 /// Empty type to group methods related to heap parameters for running the GSP firmware.
44113 enum GspFwHeapParams {}
45114
@@ -708,22 +777,6 @@ impl MsgqTxHeader { · 662 unmodified lines
708777 entryOff: num::usize_into_u32::<GSP_PAGE_SIZE>(),
709778 })
710779 }
711−
712− /// Returns the value of the write pointer for this queue.
713− pub(crate) fn write_ptr(&self) -> u32 {
714− let ptr = core::ptr::from_ref(&self.0.writePtr);
715−
716− // SAFETY: `ptr` is a valid pointer to a `u32`.
717− unsafe { ptr.read_volatile() }
718− }
719−
720− /// Sets the value of the write pointer for this queue.
721− pub(crate) fn set_write_ptr(&mut self, val: u32) {
722− let ptr = core::ptr::from_mut(&mut self.0.writePtr);
723−
724− // SAFETY: `ptr` is a valid pointer to a `u32`.
725− unsafe { ptr.write_volatile(val) }
726− }
727780 }
728781
729782 // SAFETY: Padding is explicit and does not contain uninitialized data.
@@ -739,22 +792,6 @@ impl MsgqRxHeader { · 9 unmodified lines
739792 pub(crate) fn new() -> Self {
740793 Self(Default::default())
741794 }
742−
743− /// Returns the value of the read pointer for this queue.
744− pub(crate) fn read_ptr(&self) -> u32 {
745− let ptr = core::ptr::from_ref(&self.0.readPtr);
746−
747− // SAFETY: `ptr` is a valid pointer to a `u32`.
748− unsafe { ptr.read_volatile() }
749− }
750−
751− /// Sets the value of the read pointer for this queue.
752− pub(crate) fn set_read_ptr(&mut self, val: u32) {
753− let ptr = core::ptr::from_mut(&mut self.0.readPtr);
754−
755− // SAFETY: `ptr` is a valid pointer to a `u32`.
756− unsafe { ptr.write_volatile(val) }
757− }
758795 }
759796
760797 // SAFETY: Padding is explicit and does not contain uninitialized data.
rust/kernel/dma.rs+50−64
@@ -461,6 +461,19 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { · 460 unmodified lines
461461 self.count * core::mem::size_of::<T>()
462462 }
463463
464+ /// Returns the raw pointer to the allocated region in the CPU's virtual address space.
465+ #[inline]
466+ pub fn as_ptr(&self) -> *const [T] {
467+ core::ptr::slice_from_raw_parts(self.cpu_addr.as_ptr(), self.count)
468+ }
469+
470+ /// Returns the raw pointer to the allocated region in the CPU's virtual address space as
471+ /// a mutable pointer.
472+ #[inline]
473+ pub fn as_mut_ptr(&self) -> *mut [T] {
474+ core::ptr::slice_from_raw_parts_mut(self.cpu_addr.as_ptr(), self.count)
475+ }
476+
464477 /// Returns the base address to the allocated region in the CPU's virtual address space.
465478 pub fn start_ptr(&self) -> *const T {
466479 self.cpu_addr.as_ptr()
@@ -581,23 +594,6 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { · 114 unmodified lines
581594 Ok(())
582595 }
583596
584− /// Returns a pointer to an element from the region with bounds checking. `offset` is in
585− /// units of `T`, not the number of bytes.
586− ///
587− /// Public but hidden since it should only be used from [`dma_read`] and [`dma_write`] macros.
588− #[doc(hidden)]
589− pub fn item_from_index(&self, offset: usize) -> Result<*mut T> {
590− if offset >= self.count {
591− return Err(EINVAL);
592− }
593− // SAFETY:
594− // - The pointer is valid due to type invariant on `CoherentAllocation`
595− // and we've just checked that the range and index is within bounds.
596− // - `offset` can't overflow since it is smaller than `self.count` and we've checked
597− // that `self.count` won't overflow early in the constructor.
598− Ok(unsafe { self.cpu_addr.as_ptr().add(offset) })
599− }
600−
601597 /// Reads the value of `field` and ensures that its type is [`FromBytes`].
602598 ///
603599 /// # Safety
@@ -670,6 +666,9 @@ unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {} · 66 unmodified lines
670666
671667 /// Reads a field of an item from an allocated region of structs.
672668 ///
669+/// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating
670+/// to a [`CoherentAllocation`] and `proj` is a [projection specification](kernel::ptr::project!).
671+///
673672 /// # Examples
674673 ///
675674 /// ```
@@ -684,36 +683,29 @@ unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {} · 8 unmodified lines
684683 /// unsafe impl kernel::transmute::AsBytes for MyStruct{};
685684 ///
686685 /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result {
687−/// let whole = kernel::dma_read!(alloc[2]);
688−/// let field = kernel::dma_read!(alloc[1].field);
686+/// let whole = kernel::dma_read!(alloc, [2]?);
687+/// let field = kernel::dma_read!(alloc, [1]?.field);
689688 /// # Ok::<(), Error>(()) }
690689 /// ```
691690 #[macro_export]
692691 macro_rules! dma_read {
693− ($dma:expr, $idx: expr, $($field:tt)*) => {{
694− (|| -> ::core::result::Result<_, $crate::error::Error> {
695− let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
696− // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be
697− // dereferenced. The compiler also further validates the expression on whether `field`
698− // is a member of `item` when expanded by the macro.
699− unsafe {
700− let ptr_field = ::core::ptr::addr_of!((*item) $($field)*);
701− ::core::result::Result::Ok(
702− $crate::dma::CoherentAllocation::field_read(&$dma, ptr_field)
703− )
704− }
705− })()
692+ ($dma:expr, $($proj:tt)*) => {{
693+ let dma = &$dma;
694+ let ptr = $crate::ptr::project!(
695+ $crate::dma::CoherentAllocation::as_ptr(dma), $($proj)*
696+ );
697+ // SAFETY: The pointer created by the projection is within the DMA region.
698+ unsafe { $crate::dma::CoherentAllocation::field_read(dma, ptr) }
706699 }};
707− ($dma:ident [ $idx:expr ] $($field:tt)* ) => {
708− $crate::dma_read!($dma, $idx, $($field)*)
709− };
710− ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {
711− $crate::dma_read!($($dma).*, $idx, $($field)*)
712− };
713700 }
714701
715702 /// Writes to a field of an item from an allocated region of structs.
716703 ///
704+/// The syntax is of the form `kernel::dma_write!(dma, proj, val)` where `dma` is an expression
705+/// evaluating to a [`CoherentAllocation`], `proj` is a
706+/// [projection specification](kernel::ptr::project!), and `val` is the value to be written to the
707+/// projected location.
708+///
717709 /// # Examples
718710 ///
719711 /// ```
@@ -728,37 +720,31 @@ macro_rules! dma_read { · 8 unmodified lines
728720 /// unsafe impl kernel::transmute::AsBytes for MyStruct{};
729721 ///
730722 /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result {
731−/// kernel::dma_write!(alloc[2].member = 0xf);
732−/// kernel::dma_write!(alloc[1] = MyStruct { member: 0xf });
723+/// kernel::dma_write!(alloc, [2]?.member, 0xf);
724+/// kernel::dma_write!(alloc, [1]?, MyStruct { member: 0xf });
733725 /// # Ok::<(), Error>(()) }
734726 /// ```
735727 #[macro_export]
736728 macro_rules! dma_write {
737− ($dma:ident [ $idx:expr ] $($field:tt)*) => {{
738− $crate::dma_write!($dma, $idx, $($field)*)
739− }};
740− ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {{
741− $crate::dma_write!($($dma).*, $idx, $($field)*)
729+ (@parse [$dma:expr] [$($proj:tt)*] [, $val:expr]) => {{
730+ let dma = &$dma;
731+ let ptr = $crate::ptr::project!(
732+ mut $crate::dma::CoherentAllocation::as_mut_ptr(dma), $($proj)*
733+ );
734+ let val = $val;
735+ // SAFETY: The pointer created by the projection is within the DMA region.
736+ unsafe { $crate::dma::CoherentAllocation::field_write(dma, ptr, val) }
742737 }};
743− ($dma:expr, $idx: expr, = $val:expr) => {
744− (|| -> ::core::result::Result<_, $crate::error::Error> {
745− let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
746− // SAFETY: `item_from_index` ensures that `item` is always a valid item.
747− unsafe { $crate::dma::CoherentAllocation::field_write(&$dma, item, $val) }
748− ::core::result::Result::Ok(())
749− })()
738+ (@parse [$dma:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => {
739+ $crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*])
740+ };
741+ (@parse [$dma:expr] [$($proj:tt)*] [[$index:expr]? $($rest:tt)*]) => {
742+ $crate::dma_write!(@parse [$dma] [$($proj)* [$index]?] [$($rest)*])
743+ };
744+ (@parse [$dma:expr] [$($proj:tt)*] [[$index:expr] $($rest:tt)*]) => {
745+ $crate::dma_write!(@parse [$dma] [$($proj)* [$index]] [$($rest)*])
750746 };
751− ($dma:expr, $idx: expr, $(.$field:ident)* = $val:expr) => {
752− (|| -> ::core::result::Result<_, $crate::error::Error> {
753− let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
754− // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be
755− // dereferenced. The compiler also further validates the expression on whether `field`
756− // is a member of `item` when expanded by the macro.
757− unsafe {
758− let ptr_field = ::core::ptr::addr_of_mut!((*item) $(.$field)*);
759− $crate::dma::CoherentAllocation::field_write(&$dma, ptr_field, $val)
760− }
761− ::core::result::Result::Ok(())
762− })()
747+ ($dma:expr, $($rest:tt)*) => {
748+ $crate::dma_write!(@parse [$dma] [] [$($rest)*])
763749 };
764750 }
rust/kernel/lib.rs+4−0
@@ -20,6 +20,7 @@ · 19 unmodified lines
2020 #![feature(generic_nonzero)]
2121 #![feature(inline_const)]
2222 #![feature(pointer_is_aligned)]
23+#![feature(slice_ptr_len)]
2324 //
2425 // Stable since Rust 1.80.0.
2526 #![feature(slice_flatten)]
@@ -37,6 +38,9 @@ · 11 unmodified lines
3738 #![feature(const_ptr_write)]
3839 #![feature(const_refs_to_cell)]
3940 //
41+// Stable since Rust 1.84.0.
42+#![feature(strict_provenance)]
43+//
4044 // Expected to become stable.
4145 #![feature(arbitrary_self_types)]
4246 //
rust/kernel/ptr.rs+29−1
@@ -2,7 +2,13 @@ · 1 unmodified line
22
33 //! Types and functions to work with pointers and addresses.
44
5−use core::mem::align_of;
5+pub mod projection;
6+pub use crate::project_pointer as project;
7+
8+use core::mem::{
9+ align_of,
10+ size_of, //
11+};
612 use core::num::NonZero;
713
814 /// Type representing an alignment, which is always a power of two.
@@ -225,3 +231,25 @@ macro_rules! impl_alignable_uint { · 216 unmodified lines
225231 }
226232
227233 impl_alignable_uint!(u8, u16, u32, u64, usize);
234+
235+/// Trait to represent compile-time known size information.
236+///
237+/// This is a generalization of [`size_of`] that works for dynamically sized types.
238+pub trait KnownSize {
239+ /// Get the size of an object of this type in bytes, with the metadata of the given pointer.
240+ fn size(p: *const Self) -> usize;
241+}
242+
243+impl<T> KnownSize for T {
244+ #[inline(always)]
245+ fn size(_: *const Self) -> usize {
246+ size_of::<T>()
247+ }
248+}
249+
250+impl<T> KnownSize for [T] {
251+ #[inline(always)]
252+ fn size(p: *const Self) -> usize {
253+ p.len() * size_of::<T>()
254+ }
255+}
rust/kernel/ptr/projection.rs+305−0
@@ -0,0 +1,305 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+//! Infrastructure for handling projections.
4+
5+use core::{
6+ mem::MaybeUninit,
7+ ops::Deref, //
8+};
9+
10+use crate::prelude::*;
11+
12+/// Error raised when a projection is attempted on an array or slice out of bounds.
13+pub struct OutOfBound;
14+
15+impl From<OutOfBound> for Error {
16+ #[inline(always)]
17+ fn from(_: OutOfBound) -> Self {
18+ ERANGE
19+ }
20+}
21+
22+/// A helper trait to perform index projection.
23+///
24+/// This is similar to [`core::slice::SliceIndex`], but operates on raw pointers safely and
25+/// fallibly.
26+///
27+/// # Safety
28+///
29+/// The implementation of `index` and `get` (if [`Some`] is returned) must ensure that, if provided
30+/// input pointer `slice` and returned pointer `output`, then:
31+/// - `output` has the same provenance as `slice`;
32+/// - `output.byte_offset_from(slice)` is between 0 to
33+/// `KnownSize::size(slice) - KnownSize::size(output)`.
34+///
35+/// This means that if the input pointer is valid, then pointer returned by `get` or `index` is
36+/// also valid.
37+#[diagnostic::on_unimplemented(message = "`{Self}` cannot be used to index `{T}`")]
38+#[doc(hidden)]
39+pub unsafe trait ProjectIndex<T: ?Sized>: Sized {
40+ type Output: ?Sized;
41+
42+ /// Returns an index-projected pointer, if in bounds.
43+ fn get(self, slice: *mut T) -> Option<*mut Self::Output>;
44+
45+ /// Returns an index-projected pointer; fail the build if it cannot be proved to be in bounds.
46+ #[inline(always)]
47+ fn index(self, slice: *mut T) -> *mut Self::Output {
48+ Self::get(self, slice).unwrap_or_else(|| build_error!())
49+ }
50+}
51+
52+// Forward array impl to slice impl.
53+//
54+// SAFETY: Safety requirement guaranteed by the forwarded impl.
55+unsafe impl<T, I, const N: usize> ProjectIndex<[T; N]> for I
56+where
57+ I: ProjectIndex<[T]>,
58+{
59+ type Output = <I as ProjectIndex<[T]>>::Output;
60+
61+ #[inline(always)]
62+ fn get(self, slice: *mut [T; N]) -> Option<*mut Self::Output> {
63+ <I as ProjectIndex<[T]>>::get(self, slice)
64+ }
65+
66+ #[inline(always)]
67+ fn index(self, slice: *mut [T; N]) -> *mut Self::Output {
68+ <I as ProjectIndex<[T]>>::index(self, slice)
69+ }
70+}
71+
72+// SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to
73+// not exceed the required bound.
74+unsafe impl<T> ProjectIndex<[T]> for usize {
75+ type Output = T;
76+
77+ #[inline(always)]
78+ fn get(self, slice: *mut [T]) -> Option<*mut T> {
79+ if self >= slice.len() {
80+ None
81+ } else {
82+ Some(slice.cast::<T>().wrapping_add(self))
83+ }
84+ }
85+}
86+
87+// SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to
88+// not exceed the required bound.
89+unsafe impl<T> ProjectIndex<[T]> for core::ops::Range<usize> {
90+ type Output = [T];
91+
92+ #[inline(always)]
93+ fn get(self, slice: *mut [T]) -> Option<*mut [T]> {
94+ let new_len = self.end.checked_sub(self.start)?;
95+ if self.end > slice.len() {
96+ return None;
97+ }
98+ Some(core::ptr::slice_from_raw_parts_mut(
99+ slice.cast::<T>().wrapping_add(self.start),
100+ new_len,
101+ ))
102+ }
103+}
104+
105+// SAFETY: Safety requirement guaranteed by the forwarded impl.
106+unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeTo<usize> {
107+ type Output = [T];
108+
109+ #[inline(always)]
110+ fn get(self, slice: *mut [T]) -> Option<*mut [T]> {
111+ (0..self.end).get(slice)
112+ }
113+}
114+
115+// SAFETY: Safety requirement guaranteed by the forwarded impl.
116+unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeFrom<usize> {
117+ type Output = [T];
118+
119+ #[inline(always)]
120+ fn get(self, slice: *mut [T]) -> Option<*mut [T]> {
121+ (self.start..slice.len()).get(slice)
122+ }
123+}
124+
125+// SAFETY: `get` returned the pointer as is, so it always has the same provenance and offset of 0.
126+unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeFull {
127+ type Output = [T];
128+
129+ #[inline(always)]
130+ fn get(self, slice: *mut [T]) -> Option<*mut [T]> {
131+ Some(slice)
132+ }
133+}
134+
135+/// A helper trait to perform field projection.
136+///
137+/// This trait has a `DEREF` generic parameter so it can be implemented twice for types that
138+/// implement [`Deref`]. This will cause an ambiguity error and thus block [`Deref`] types being
139+/// used as base of projection, as they can inject unsoundness. Users therefore must not specify
140+/// `DEREF` and should always leave it to be inferred.
141+///
142+/// # Safety
143+///
144+/// `proj` may only invoke `f` with a valid allocation, as the documentation of [`Self::proj`]
145+/// describes.
146+#[doc(hidden)]
147+pub unsafe trait ProjectField<const DEREF: bool> {
148+ /// Project a pointer to a type to a pointer of a field.
149+ ///
150+ /// `f` may only be invoked with a valid allocation so it can safely obtain raw pointers to
151+ /// fields using `&raw mut`.
152+ ///
153+ /// This is needed because `base` might not point to a valid allocation, while `&raw mut`
154+ /// requires pointers to be in bounds of a valid allocation.
155+ ///
156+ /// # Safety
157+ ///
158+ /// `f` must return a pointer in bounds of the provided pointer.
159+ unsafe fn proj<F>(base: *mut Self, f: impl FnOnce(*mut Self) -> *mut F) -> *mut F;
160+}
161+
162+// NOTE: in theory, this API should work for `T: ?Sized` and `F: ?Sized`, too. However, we cannot
163+// currently support that as we need to obtain a valid allocation that `&raw const` can operate on.
164+//
165+// SAFETY: `proj` invokes `f` with valid allocation.
166+unsafe impl<T> ProjectField<false> for T {
167+ #[inline(always)]
168+ unsafe fn proj<F>(base: *mut Self, f: impl FnOnce(*mut Self) -> *mut F) -> *mut F {
169+ // Create a valid allocation to start projection, as `base` is not necessarily so. The
170+ // memory is never actually used so it will be optimized out, so it should work even for
171+ // very large `T` (`memoffset` crate also relies on this). To be extra certain, we also
172+ // annotate `f` closure with `#[inline(always)]` in the macro.
173+ let mut place = MaybeUninit::uninit();
174+ let place_base = place.as_mut_ptr();
175+ let field = f(place_base);
176+ // SAFETY: `field` is in bounds from `base` per safety requirement.
177+ let offset = unsafe { field.byte_offset_from(place_base) };
178+ // Use `wrapping_byte_offset` as `base` does not need to be of valid allocation.
179+ base.wrapping_byte_offset(offset).cast()
180+ }
181+}
182+
183+// SAFETY: Vacuously satisfied.
184+unsafe impl<T: Deref> ProjectField<true> for T {
185+ #[inline(always)]
186+ unsafe fn proj<F>(_: *mut Self, _: impl FnOnce(*mut Self) -> *mut F) -> *mut F {
187+ build_error!("this function is a guard against `Deref` impl and is never invoked");
188+ }
189+}
190+
191+/// Create a projection from a raw pointer.
192+///
193+/// The projected pointer is within the memory region marked by the input pointer. There is no
194+/// requirement that the input raw pointer needs to be valid, so this macro may be used for
195+/// projecting pointers outside normal address space, e.g. I/O pointers. However, if the input
196+/// pointer is valid, the projected pointer is also valid.
197+///
198+/// Supported projections include field projections and index projections.
199+/// It is not allowed to project into types that implement custom [`Deref`] or
200+/// [`Index`](core::ops::Index).
201+///
202+/// The macro has basic syntax of `kernel::ptr::project!(ptr, projection)`, where `ptr` is an
203+/// expression that evaluates to a raw pointer which serves as the base of projection. `projection`
204+/// can be a projection expression of form `.field` (normally identifier, or numeral in case of
205+/// tuple structs) or of form `[index]`.
206+///
207+/// If a mutable pointer is needed, the macro input can be prefixed with the `mut` keyword, i.e.
208+/// `kernel::ptr::project!(mut ptr, projection)`. By default, a const pointer is created.
209+///
210+/// `ptr::project!` macro can perform both fallible indexing and build-time checked indexing.
211+/// `[index]` form performs build-time bounds checking; if compiler fails to prove `[index]` is in
212+/// bounds, compilation will fail. `[index]?` can be used to perform runtime bounds checking;
213+/// `OutOfBound` error is raised via `?` if the index is out of bounds.
214+///
215+/// # Examples
216+///
217+/// Field projections are performed with `.field_name`:
218+///
219+/// ```
220+/// struct MyStruct { field: u32, }
221+/// let ptr: *const MyStruct = core::ptr::dangling();
222+/// let field_ptr: *const u32 = kernel::ptr::project!(ptr, .field);
223+///
224+/// struct MyTupleStruct(u32, u32);
225+///
226+/// fn proj(ptr: *const MyTupleStruct) {
227+/// let field_ptr: *const u32 = kernel::ptr::project!(ptr, .1);
228+/// }
229+/// ```
230+///
231+/// Index projections are performed with `[index]`:
232+///
233+/// ```
234+/// fn proj(ptr: *const [u8; 32]) -> Result {
235+/// let field_ptr: *const u8 = kernel::ptr::project!(ptr, [1]);
236+/// // The following invocation, if uncommented, would fail the build.
237+/// //
238+/// // kernel::ptr::project!(ptr, [128]);
239+///
240+/// // This will raise an `OutOfBound` error (which is convertible to `ERANGE`).
241+/// kernel::ptr::project!(ptr, [128]?);
242+/// Ok(())
243+/// }
244+/// ```
245+///
246+/// If you need to match on the error instead of propagate, put the invocation inside a closure:
247+///
248+/// ```
249+/// let ptr: *const [u8; 32] = core::ptr::dangling();
250+/// let field_ptr: Result<*const u8> = (|| -> Result<_> {
251+/// Ok(kernel::ptr::project!(ptr, [128]?))
252+/// })();
253+/// assert!(field_ptr.is_err());
254+/// ```
255+///
256+/// For mutable pointers, put `mut` as the first token in macro invocation.
257+///
258+/// ```
259+/// let ptr: *mut [(u8, u16); 32] = core::ptr::dangling_mut();
260+/// let field_ptr: *mut u16 = kernel::ptr::project!(mut ptr, [1].1);
261+/// ```
262+#[macro_export]
263+macro_rules! project_pointer {
264+ (@gen $ptr:ident, ) => {};
265+ // Field projection. `$field` needs to be `tt` to support tuple index like `.0`.
266+ (@gen $ptr:ident, .$field:tt $($rest:tt)*) => {
267+ // SAFETY: The provided closure always returns an in-bounds pointer.
268+ let $ptr = unsafe {
269+ $crate::ptr::projection::ProjectField::proj($ptr, #[inline(always)] |ptr| {
270+ // Check unaligned field. Not all users (e.g. DMA) can handle unaligned
271+ // projections.
272+ if false {
273+ let _ = &(*ptr).$field;
274+ }
275+ // SAFETY: `$field` is in bounds, and no implicit `Deref` is possible (if the
276+ // type implements `Deref`, Rust cannot infer the generic parameter `DEREF`).
277+ &raw mut (*ptr).$field
278+ })
279+ };
280+ $crate::ptr::project!(@gen $ptr, $($rest)*)
281+ };
282+ // Fallible index projection.
283+ (@gen $ptr:ident, [$index:expr]? $($rest:tt)*) => {
284+ let $ptr = $crate::ptr::projection::ProjectIndex::get($index, $ptr)
285+ .ok_or($crate::ptr::projection::OutOfBound)?;
286+ $crate::ptr::project!(@gen $ptr, $($rest)*)
287+ };
288+ // Build-time checked index projection.
289+ (@gen $ptr:ident, [$index:expr] $($rest:tt)*) => {
290+ let $ptr = $crate::ptr::projection::ProjectIndex::index($index, $ptr);
291+ $crate::ptr::project!(@gen $ptr, $($rest)*)
292+ };
293+ (mut $ptr:expr, $($proj:tt)*) => {{
294+ let ptr: *mut _ = $ptr;
295+ $crate::ptr::project!(@gen ptr, $($proj)*);
296+ ptr
297+ }};
298+ ($ptr:expr, $($proj:tt)*) => {{
299+ let ptr = <*const _>::cast_mut($ptr);
300+ // We currently always project using mutable pointer, as it is not decided whether `&raw
301+ // const` allows the resulting pointer to be mutated (see documentation of `addr_of!`).
302+ $crate::ptr::project!(@gen ptr, $($proj)*);
303+ ptr.cast_const()
304+ }};
305+}
samples/rust/rust_dma.rs+16−14
@@ -68,7 +68,7 @@ impl pci::Driver for DmaSampleDriver { · 67 unmodified lines
6868 CoherentAllocation::alloc_coherent(pdev.as_ref(), TEST_VALUES.len(), GFP_KERNEL)?;
6969
7070 for (i, value) in TEST_VALUES.into_iter().enumerate() {
71− kernel::dma_write!(ca[i] = MyStruct::new(value.0, value.1))?;
71+ kernel::dma_write!(ca, [i]?, MyStruct::new(value.0, value.1));
7272 }
7373
7474 let size = 4 * page::PAGE_SIZE;
@@ -85,24 +85,26 @@ impl pci::Driver for DmaSampleDriver { · 10 unmodified lines
8585 }
8686 }
8787
88+impl DmaSampleDriver {
89+ fn check_dma(&self) -> Result {
90+ for (i, value) in TEST_VALUES.into_iter().enumerate() {
91+ let val0 = kernel::dma_read!(self.ca, [i]?.h);
92+ let val1 = kernel::dma_read!(self.ca, [i]?.b);
93+
94+ assert_eq!(val0, value.0);
95+ assert_eq!(val1, value.1);
96+ }
97+
98+ Ok(())
99+ }
100+}
101+
88102 #[pinned_drop]
89103 impl PinnedDrop for DmaSampleDriver {
90104 fn drop(self: Pin<&mut Self>) {
91105 dev_info!(self.pdev, "Unload DMA test driver.\n");
92106
93− for (i, value) in TEST_VALUES.into_iter().enumerate() {
94− let val0 = kernel::dma_read!(self.ca[i].h);
95− let val1 = kernel::dma_read!(self.ca[i].b);
96− assert!(val0.is_ok());
97− assert!(val1.is_ok());
98−
99− if let Ok(val0) = val0 {
100− assert_eq!(val0, value.0);
101− }
102− if let Ok(val1) = val1 {
103− assert_eq!(val1, value.1);
104− }
105− }
107+ assert!(self.check_dma().is_ok());
106108
107109 for (i, entry) in self.sgt.iter().enumerate() {
108110 dev_info!(
scripts/Makefile.build+3−1
@@ -310,16 +310,18 @@ $(obj)/%.lst: $(obj)/%.c FORCE · 309 unmodified lines
310310
311311 # The features in this list are the ones allowed for non-`rust/` code.
312312 #
313+# - Stable since Rust 1.79.0: `feature(slice_ptr_len)`.
313314 # - Stable since Rust 1.81.0: `feature(lint_reasons)`.
314315 # - Stable since Rust 1.82.0: `feature(asm_const)`,
315316 # `feature(offset_of_nested)`, `feature(raw_ref_op)`.
317+# - Stable since Rust 1.84.0: `feature(strict_provenance)`.
316318 # - Stable since Rust 1.87.0: `feature(asm_goto)`.
317319 # - Expected to become stable: `feature(arbitrary_self_types)`.
318320 # - To be determined: `feature(used_with_arg)`.
319321 #
320322 # Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on
321323 # the unstable features in use.
322−rust_allowed_features := asm_const,asm_goto,arbitrary_self_types,lint_reasons,offset_of_nested,raw_ref_op,used_with_arg
324+rust_allowed_features := asm_const,asm_goto,arbitrary_self_types,lint_reasons,offset_of_nested,raw_ref_op,slice_ptr_len,strict_provenance,used_with_arg
323325
324326 # `--out-dir` is required to avoid temporaries being created by `rustc` in the
325327 # current working directory, which may be not accessible in the out-of-tree