lael-labs/linux

drivers/gpu/nova-core/gsp.rs+24−22

@@ -47,16 +47,12 @@ struct PteArray<const NUM_ENTRIES: usize>([u64; NUM_ENTRIES]); · 46 unmodified lines
47	unsafe impl<const NUM_ENTRIES: usize> AsBytes for PteArray<NUM_ENTRIES> {}	47	unsafe impl<const NUM_ENTRIES: usize> AsBytes for PteArray<NUM_ENTRIES> {}
48		48
49	impl<const NUM_PAGES: usize> PteArray<NUM_PAGES> {	49	impl<const NUM_PAGES: usize> PteArray<NUM_PAGES> {
50	/// Creates a new page table array mapping `NUM_PAGES` GSP pages starting at address `start`.	50	/// Returns the page table entry for `index`, for a mapping starting at `start`.
51	fn new(start: DmaAddress) -> Result<Self> {	51	// TODO: Replace with `IoView` projection once available.
52	let mut ptes = [0u64; NUM_PAGES];	52	fn entry(start: DmaAddress, index: usize) -> Result<u64> {
53	for (i, pte) in ptes.iter_mut().enumerate() {	53	start
54	*pte = start	54	.checked_add(num::usize_as_u64(index) << GSP_PAGE_SHIFT)
55	.checked_add(num::usize_as_u64(i) << GSP_PAGE_SHIFT)	55	.ok_or(EOVERFLOW)
56	.ok_or(EOVERFLOW)?;
57	}
58
59	Ok(Self(ptes))
60	}	56	}
61	}	57	}
62		58
@@ -86,16 +82,22 @@ impl LogBuffer { · 23 unmodified lines
86	NUM_PAGES * GSP_PAGE_SIZE,	82	NUM_PAGES * GSP_PAGE_SIZE,
87	GFP_KERNEL \| __GFP_ZERO,	83	GFP_KERNEL \| __GFP_ZERO,
88	)?);	84	)?);
89	let ptes = PteArray::<NUM_PAGES>::new(obj.0.dma_handle())?;	85
		86	let start_addr = obj.0.dma_handle();
90		87
91	// SAFETY: `obj` has just been created and we are its sole user.	88	// SAFETY: `obj` has just been created and we are its sole user.
92	unsafe {	89	let pte_region = unsafe {
93	// Copy the self-mapping PTE at the expected location.
94	obj.0	90	obj.0
95	.as_slice_mut(size_of::<u64>(), size_of_val(&ptes))?	91	.as_slice_mut(size_of::<u64>(), NUM_PAGES * size_of::<u64>())?
96	.copy_from_slice(ptes.as_bytes())
97	};	92	};
98		93
		94	// Write values one by one to avoid an on-stack instance of `PteArray`.
		95	for (i, chunk) in pte_region.chunks_exact_mut(size_of::<u64>()).enumerate() {
		96	let pte_value = PteArray::<0>::entry(start_addr, i)?;
		97
		98	chunk.copy_from_slice(&pte_value.to_ne_bytes());
		99	}
		100
99	Ok(obj)	101	Ok(obj)
100	}	102	}
101	}	103	}
@@ -143,14 +145,14 @@ impl Gsp { · 41 unmodified lines
143	// _kgspInitLibosLoggingStructures (allocates memory for buffers)	145	// _kgspInitLibosLoggingStructures (allocates memory for buffers)
144	// kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array)	146	// kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array)
145	dma_write!(	147	dma_write!(
146	libos[0] = LibosMemoryRegionInitArgument::new("LOGINIT", &loginit.0)	148	libos, [0]?, LibosMemoryRegionInitArgument::new("LOGINIT", &loginit.0)
147	)?;	149	);
148	dma_write!(	150	dma_write!(
149	libos[1] = LibosMemoryRegionInitArgument::new("LOGINTR", &logintr.0)	151	libos, [1]?, LibosMemoryRegionInitArgument::new("LOGINTR", &logintr.0)
150	)?;	152	);
151	dma_write!(libos[2] = LibosMemoryRegionInitArgument::new("LOGRM", &logrm.0))?;	153	dma_write!(libos, [2]?, LibosMemoryRegionInitArgument::new("LOGRM", &logrm.0));
152	dma_write!(rmargs[0].inner = fw::GspArgumentsCached::new(cmdq))?;	154	dma_write!(rmargs, [0]?.inner, fw::GspArgumentsCached::new(cmdq));
153	dma_write!(libos[3] = LibosMemoryRegionInitArgument::new("RMARGS", rmargs))?;	155	dma_write!(libos, [3]?, LibosMemoryRegionInitArgument::new("RMARGS", rmargs));
154	},	156	},
155	}))	157	}))
156	})	158	})

drivers/gpu/nova-core/gsp/boot.rs+1−1

@@ -157,7 +157,7 @@ impl super::Gsp { · 156 unmodified lines
157		157
158	let wpr_meta =	158	let wpr_meta =
159	CoherentAllocation::<GspFwWprMeta>::alloc_coherent(dev, 1, GFP_KERNEL \| __GFP_ZERO)?;	159	CoherentAllocation::<GspFwWprMeta>::alloc_coherent(dev, 1, GFP_KERNEL \| __GFP_ZERO)?;
160	dma_write!(wpr_meta[0] = GspFwWprMeta::new(&gsp_fw, &fb_layout))?;	160	dma_write!(wpr_meta, [0]?, GspFwWprMeta::new(&gsp_fw, &fb_layout));
161		161
162	self.cmdq	162	self.cmdq
163	.send_command(bar, commands::SetSystemInfo::new(pdev))?;	163	.send_command(bar, commands::SetSystemInfo::new(pdev))?;

drivers/gpu/nova-core/gsp/cmdq.rs+33−60

@@ -2,11 +2,7 @@ · 1 unmodified line
2		2
3	use core::{	3	use core::{
4	cmp,	4	cmp,
5	mem,	5	mem, //
6	sync::atomic::{
7	fence,
8	Ordering, //
9	}, //
10	};	6	};
11		7
12	use kernel::{	8	use kernel::{
@@ -146,30 +142,36 @@ static_assert!(align_of::<MsgqData>() == GSP_PAGE_SIZE); · 133 unmodified lines
146	#[repr(C)]	142	#[repr(C)]
147	// There is no struct defined for this in the open-gpu-kernel-source headers.	143	// There is no struct defined for this in the open-gpu-kernel-source headers.
148	// Instead it is defined by code in `GspMsgQueuesInit()`.	144	// Instead it is defined by code in `GspMsgQueuesInit()`.
149	struct Msgq {	145	// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module.
		146	pub(super) struct Msgq {
150	/// Header for sending messages, including the write pointer.	147	/// Header for sending messages, including the write pointer.
151	tx: MsgqTxHeader,	148	pub(super) tx: MsgqTxHeader,
152	/// Header for receiving messages, including the read pointer.	149	/// Header for receiving messages, including the read pointer.
153	rx: MsgqRxHeader,	150	pub(super) rx: MsgqRxHeader,
154	/// The message queue proper.	151	/// The message queue proper.
155	msgq: MsgqData,	152	msgq: MsgqData,
156	}	153	}
157		154
158	/// Structure shared between the driver and the GSP and containing the command and message queues.	155	/// Structure shared between the driver and the GSP and containing the command and message queues.
159	#[repr(C)]	156	#[repr(C)]
160	struct GspMem {	157	// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module.
		158	pub(super) struct GspMem {
161	/// Self-mapping page table entries.	159	/// Self-mapping page table entries.
162	ptes: PteArray<{ GSP_PAGE_SIZE / size_of::<u64>() }>,	160	ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>,
163	/// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the	161	/// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the
164	/// write and read pointers that the CPU updates.	162	/// write and read pointers that the CPU updates.
165	///	163	///
166	/// This member is read-only for the GSP.	164	/// This member is read-only for the GSP.
167	cpuq: Msgq,	165	pub(super) cpuq: Msgq,
168	/// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the	166	/// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the
169	/// write and read pointers that the GSP updates.	167	/// write and read pointers that the GSP updates.
170	///	168	///
171	/// This member is read-only for the driver.	169	/// This member is read-only for the driver.
172	gspq: Msgq,	170	pub(super) gspq: Msgq,
		171	}
		172
		173	impl GspMem {
		174	const PTE_ARRAY_SIZE: usize = GSP_PAGE_SIZE / size_of::<u64>();
173	}	175	}
174		176
175	// SAFETY: These structs don't meet the no-padding requirements of AsBytes but	177	// SAFETY: These structs don't meet the no-padding requirements of AsBytes but
@@ -201,9 +203,19 @@ impl DmaGspMem { · 25 unmodified lines
201		203
202	let gsp_mem =	204	let gsp_mem =
203	CoherentAllocation::<GspMem>::alloc_coherent(dev, 1, GFP_KERNEL \| __GFP_ZERO)?;	205	CoherentAllocation::<GspMem>::alloc_coherent(dev, 1, GFP_KERNEL \| __GFP_ZERO)?;
204	dma_write!(gsp_mem[0].ptes = PteArray::new(gsp_mem.dma_handle())?)?;	206
205	dma_write!(gsp_mem[0].cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES))?;	207	let start = gsp_mem.dma_handle();
206	dma_write!(gsp_mem[0].cpuq.rx = MsgqRxHeader::new())?;	208	// Write values one by one to avoid an on-stack instance of `PteArray`.
		209	for i in 0..GspMem::PTE_ARRAY_SIZE {
		210	dma_write!(gsp_mem, [0]?.ptes.0[i], PteArray::<0>::entry(start, i)?);
		211	}
		212
		213	dma_write!(
		214	gsp_mem,
		215	[0]?.cpuq.tx,
		216	MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES)
		217	);
		218	dma_write!(gsp_mem, [0]?.cpuq.rx, MsgqRxHeader::new());
207		219
208	Ok(Self(gsp_mem))	220	Ok(Self(gsp_mem))
209	}	221	}
@@ -317,12 +329,7 @@ impl DmaGspMem { · 107 unmodified lines
317	//	329	//
318	// - The returned value is between `0` and `MSGQ_NUM_PAGES`.	330	// - The returned value is between `0` and `MSGQ_NUM_PAGES`.
319	fn gsp_write_ptr(&self) -> u32 {	331	fn gsp_write_ptr(&self) -> u32 {
320	let gsp_mem = self.0.start_ptr();	332	super::fw::gsp_mem::gsp_write_ptr(&self.0)
321
322	// SAFETY:
323	// - The 'CoherentAllocation' contains at least one object.
324	// - By the invariants of `CoherentAllocation` the pointer is valid.
325	(unsafe { (*gsp_mem).gspq.tx.write_ptr() } % MSGQ_NUM_PAGES)
326	}	333	}
327		334
328	// Returns the index of the memory page the GSP will read the next command from.	335	// Returns the index of the memory page the GSP will read the next command from.
@@ -331,12 +338,7 @@ impl DmaGspMem { · 2 unmodified lines
331	//	338	//
332	// - The returned value is between `0` and `MSGQ_NUM_PAGES`.	339	// - The returned value is between `0` and `MSGQ_NUM_PAGES`.
333	fn gsp_read_ptr(&self) -> u32 {	340	fn gsp_read_ptr(&self) -> u32 {
334	let gsp_mem = self.0.start_ptr();	341	super::fw::gsp_mem::gsp_read_ptr(&self.0)
335
336	// SAFETY:
337	// - The 'CoherentAllocation' contains at least one object.
338	// - By the invariants of `CoherentAllocation` the pointer is valid.
339	(unsafe { (*gsp_mem).gspq.rx.read_ptr() } % MSGQ_NUM_PAGES)
340	}	342	}
341		343
342	// Returns the index of the memory page the CPU can read the next message from.	344	// Returns the index of the memory page the CPU can read the next message from.
@@ -345,27 +347,12 @@ impl DmaGspMem { · 2 unmodified lines
345	//	347	//
346	// - The returned value is between `0` and `MSGQ_NUM_PAGES`.	348	// - The returned value is between `0` and `MSGQ_NUM_PAGES`.
347	fn cpu_read_ptr(&self) -> u32 {	349	fn cpu_read_ptr(&self) -> u32 {
348	let gsp_mem = self.0.start_ptr();	350	super::fw::gsp_mem::cpu_read_ptr(&self.0)
349
350	// SAFETY:
351	// - The ['CoherentAllocation'] contains at least one object.
352	// - By the invariants of CoherentAllocation the pointer is valid.
353	(unsafe { (*gsp_mem).cpuq.rx.read_ptr() } % MSGQ_NUM_PAGES)
354	}	351	}
355		352
356	// Informs the GSP that it can send `elem_count` new pages into the message queue.	353	// Informs the GSP that it can send `elem_count` new pages into the message queue.
357	fn advance_cpu_read_ptr(&mut self, elem_count: u32) {	354	fn advance_cpu_read_ptr(&mut self, elem_count: u32) {
358	let rptr = self.cpu_read_ptr().wrapping_add(elem_count) % MSGQ_NUM_PAGES;	355	super::fw::gsp_mem::advance_cpu_read_ptr(&self.0, elem_count)
359
360	// Ensure read pointer is properly ordered.
361	fence(Ordering::SeqCst);
362
363	let gsp_mem = self.0.start_ptr_mut();
364
365	// SAFETY:
366	// - The 'CoherentAllocation' contains at least one object.
367	// - By the invariants of `CoherentAllocation` the pointer is valid.
368	unsafe { (*gsp_mem).cpuq.rx.set_read_ptr(rptr) };
369	}	356	}
370		357
371	// Returns the index of the memory page the CPU can write the next command to.	358	// Returns the index of the memory page the CPU can write the next command to.
@@ -374,26 +361,12 @@ impl DmaGspMem { · 2 unmodified lines
374	//	361	//
375	// - The returned value is between `0` and `MSGQ_NUM_PAGES`.	362	// - The returned value is between `0` and `MSGQ_NUM_PAGES`.
376	fn cpu_write_ptr(&self) -> u32 {	363	fn cpu_write_ptr(&self) -> u32 {
377	let gsp_mem = self.0.start_ptr();	364	super::fw::gsp_mem::cpu_write_ptr(&self.0)
378
379	// SAFETY:
380	// - The 'CoherentAllocation' contains at least one object.
381	// - By the invariants of `CoherentAllocation` the pointer is valid.
382	(unsafe { (*gsp_mem).cpuq.tx.write_ptr() } % MSGQ_NUM_PAGES)
383	}	365	}
384		366
385	// Informs the GSP that it can process `elem_count` new pages from the command queue.	367	// Informs the GSP that it can process `elem_count` new pages from the command queue.
386	fn advance_cpu_write_ptr(&mut self, elem_count: u32) {	368	fn advance_cpu_write_ptr(&mut self, elem_count: u32) {
387	let wptr = self.cpu_write_ptr().wrapping_add(elem_count) & MSGQ_NUM_PAGES;	369	super::fw::gsp_mem::advance_cpu_write_ptr(&self.0, elem_count)
388	let gsp_mem = self.0.start_ptr_mut();
389
390	// SAFETY:
391	// - The 'CoherentAllocation' contains at least one object.
392	// - By the invariants of `CoherentAllocation` the pointer is valid.
393	unsafe { (*gsp_mem).cpuq.tx.set_write_ptr(wptr) };
394
395	// Ensure all command data is visible before triggering the GSP read.
396	fence(Ordering::SeqCst);
397	}	370	}
398	}	371	}
399		372

drivers/gpu/nova-core/gsp/fw.rs+69−32

@@ -40,6 +40,75 @@ use crate::{ · 39 unmodified lines
40	},	40	},
41	};	41	};
42		42
		43	// TODO: Replace with `IoView` projections once available; the `unwrap()` calls go away once we
		44	// switch to the new `dma::Coherent` API.
		45	pub(super) mod gsp_mem {
		46	use core::sync::atomic::{
		47	fence,
		48	Ordering, //
		49	};
		50
		51	use kernel::{
		52	dma::CoherentAllocation,
		53	dma_read,
		54	dma_write,
		55	prelude::*, //
		56	};
		57
		58	use crate::gsp::cmdq::{
		59	GspMem,
		60	MSGQ_NUM_PAGES, //
		61	};
		62
		63	pub(in crate::gsp) fn gsp_write_ptr(qs: &CoherentAllocation<GspMem>) -> u32 {
		64	// PANIC: A `dma::CoherentAllocation` always contains at least one element.
		65	\|\| -> Result<u32> { Ok(dma_read!(qs, [0]?.gspq.tx.0.writePtr) % MSGQ_NUM_PAGES) }().unwrap()
		66	}
		67
		68	pub(in crate::gsp) fn gsp_read_ptr(qs: &CoherentAllocation<GspMem>) -> u32 {
		69	// PANIC: A `dma::CoherentAllocation` always contains at least one element.
		70	\|\| -> Result<u32> { Ok(dma_read!(qs, [0]?.gspq.rx.0.readPtr) % MSGQ_NUM_PAGES) }().unwrap()
		71	}
		72
		73	pub(in crate::gsp) fn cpu_read_ptr(qs: &CoherentAllocation<GspMem>) -> u32 {
		74	// PANIC: A `dma::CoherentAllocation` always contains at least one element.
		75	\|\| -> Result<u32> { Ok(dma_read!(qs, [0]?.cpuq.rx.0.readPtr) % MSGQ_NUM_PAGES) }().unwrap()
		76	}
		77
		78	pub(in crate::gsp) fn advance_cpu_read_ptr(qs: &CoherentAllocation<GspMem>, count: u32) {
		79	let rptr = cpu_read_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES;
		80
		81	// Ensure read pointer is properly ordered.
		82	fence(Ordering::SeqCst);
		83
		84	// PANIC: A `dma::CoherentAllocation` always contains at least one element.
		85	\|\| -> Result {
		86	dma_write!(qs, [0]?.cpuq.rx.0.readPtr, rptr);
		87	Ok(())
		88	}()
		89	.unwrap()
		90	}
		91
		92	pub(in crate::gsp) fn cpu_write_ptr(qs: &CoherentAllocation<GspMem>) -> u32 {
		93	// PANIC: A `dma::CoherentAllocation` always contains at least one element.
		94	\|\| -> Result<u32> { Ok(dma_read!(qs, [0]?.cpuq.tx.0.writePtr) % MSGQ_NUM_PAGES) }().unwrap()
		95	}
		96
		97	pub(in crate::gsp) fn advance_cpu_write_ptr(qs: &CoherentAllocation<GspMem>, count: u32) {
		98	let wptr = cpu_write_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES;
		99
		100	// PANIC: A `dma::CoherentAllocation` always contains at least one element.
		101	\|\| -> Result {
		102	dma_write!(qs, [0]?.cpuq.tx.0.writePtr, wptr);
		103	Ok(())
		104	}()
		105	.unwrap();
		106
		107	// Ensure all command data is visible before triggering the GSP read.
		108	fence(Ordering::SeqCst);
		109	}
		110	}
		111
43	/// Empty type to group methods related to heap parameters for running the GSP firmware.	112	/// Empty type to group methods related to heap parameters for running the GSP firmware.
44	enum GspFwHeapParams {}	113	enum GspFwHeapParams {}
45		114
@@ -708,22 +777,6 @@ impl MsgqTxHeader { · 662 unmodified lines
708	entryOff: num::usize_into_u32::<GSP_PAGE_SIZE>(),	777	entryOff: num::usize_into_u32::<GSP_PAGE_SIZE>(),
709	})	778	})
710	}	779	}
711
712	/// Returns the value of the write pointer for this queue.
713	pub(crate) fn write_ptr(&self) -> u32 {
714	let ptr = core::ptr::from_ref(&self.0.writePtr);
715
716	// SAFETY: `ptr` is a valid pointer to a `u32`.
717	unsafe { ptr.read_volatile() }
718	}
719
720	/// Sets the value of the write pointer for this queue.
721	pub(crate) fn set_write_ptr(&mut self, val: u32) {
722	let ptr = core::ptr::from_mut(&mut self.0.writePtr);
723
724	// SAFETY: `ptr` is a valid pointer to a `u32`.
725	unsafe { ptr.write_volatile(val) }
726	}
727	}	780	}
728		781
729	// SAFETY: Padding is explicit and does not contain uninitialized data.	782	// SAFETY: Padding is explicit and does not contain uninitialized data.
@@ -739,22 +792,6 @@ impl MsgqRxHeader { · 9 unmodified lines
739	pub(crate) fn new() -> Self {	792	pub(crate) fn new() -> Self {
740	Self(Default::default())	793	Self(Default::default())
741	}	794	}
742
743	/// Returns the value of the read pointer for this queue.
744	pub(crate) fn read_ptr(&self) -> u32 {
745	let ptr = core::ptr::from_ref(&self.0.readPtr);
746
747	// SAFETY: `ptr` is a valid pointer to a `u32`.
748	unsafe { ptr.read_volatile() }
749	}
750
751	/// Sets the value of the read pointer for this queue.
752	pub(crate) fn set_read_ptr(&mut self, val: u32) {
753	let ptr = core::ptr::from_mut(&mut self.0.readPtr);
754
755	// SAFETY: `ptr` is a valid pointer to a `u32`.
756	unsafe { ptr.write_volatile(val) }
757	}
758	}	795	}
759		796
760	// SAFETY: Padding is explicit and does not contain uninitialized data.	797	// SAFETY: Padding is explicit and does not contain uninitialized data.

rust/kernel/dma.rs+50−64

         self.count * core::mem::size_of::<T>()
     }
+    /// Returns the raw pointer to the allocated region in the CPU's virtual address space.
+    #[inline]
+    pub fn as_ptr(&self) -> *const [T] {
+        core::ptr::slice_from_raw_parts(self.cpu_addr.as_ptr(), self.count)
+    }
+    /// Returns the raw pointer to the allocated region in the CPU's virtual address space as
+    /// a mutable pointer.
+    #[inline]
+    pub fn as_mut_ptr(&self) -> *mut [T] {
+        core::ptr::slice_from_raw_parts_mut(self.cpu_addr.as_ptr(), self.count)
+    }
     /// Returns the base address to the allocated region in the CPU's virtual address space.
     pub fn start_ptr(&self) -> *const T {
         self.cpu_addr.as_ptr()
         Ok(())
     }
-    /// Returns a pointer to an element from the region with bounds checking. `offset` is in
-    /// units of `T`, not the number of bytes.
-    ///
-    /// Public but hidden since it should only be used from [`dma_read`] and [`dma_write`] macros.
-    #[doc(hidden)]
-    pub fn item_from_index(&self, offset: usize) -> Result<*mut T> {
-        if offset >= self.count {
-            return Err(EINVAL);
-        }
-        // SAFETY:
-        // - The pointer is valid due to type invariant on `CoherentAllocation`
-        // and we've just checked that the range and index is within bounds.
-        // - `offset` can't overflow since it is smaller than `self.count` and we've checked
-        // that `self.count` won't overflow early in the constructor.
-        Ok(unsafe { self.cpu_addr.as_ptr().add(offset) })
-    }
     /// Reads the value of `field` and ensures that its type is [`FromBytes`].
     ///
     /// # Safety
 /// Reads a field of an item from an allocated region of structs.
 ///
+/// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating
+/// to a [`CoherentAllocation`] and `proj` is a [projection specification](kernel::ptr::project!).
+///
 /// # Examples
 ///
 /// ```
 /// unsafe impl kernel::transmute::AsBytes for MyStruct{};
 ///
 /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result {
-/// let whole = kernel::dma_read!(alloc[2]);
+/// let whole = kernel::dma_read!(alloc, [2]?);
-/// let field = kernel::dma_read!(alloc[1].field);
+/// let field = kernel::dma_read!(alloc, [1]?.field);
 /// # Ok::<(), Error>(()) }
 /// ```
 #[macro_export]
 macro_rules! dma_read {
-    ($dma:expr, $idx: expr, $($field:tt)*) => {{
+    ($dma:expr, $($proj:tt)*) => {{
-        (|| -> ::core::result::Result<_, $crate::error::Error> {
+        let dma = &$dma;
-            let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
+        let ptr = $crate::ptr::project!(
-            // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be
+            $crate::dma::CoherentAllocation::as_ptr(dma), $($proj)*
-            // dereferenced. The compiler also further validates the expression on whether `field`
+        );
-            // is a member of `item` when expanded by the macro.
+        // SAFETY: The pointer created by the projection is within the DMA region.
-            unsafe {
+        unsafe { $crate::dma::CoherentAllocation::field_read(dma, ptr) }
-                let ptr_field = ::core::ptr::addr_of!((*item) $($field)*);
-                ::core::result::Result::Ok(
-                    $crate::dma::CoherentAllocation::field_read(&$dma, ptr_field)
-                )
-            }
-        })()
     }};
-    ($dma:ident [ $idx:expr ] $($field:tt)* ) => {
-        $crate::dma_read!($dma, $idx, $($field)*)
-    };
-    ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {
-        $crate::dma_read!($($dma).*, $idx, $($field)*)
-    };
 }
 /// Writes to a field of an item from an allocated region of structs.
 ///
+/// The syntax is of the form `kernel::dma_write!(dma, proj, val)` where `dma` is an expression
+/// evaluating to a [`CoherentAllocation`], `proj` is a
+/// [projection specification](kernel::ptr::project!), and `val` is the value to be written to the
+/// projected location.
+///
 /// # Examples
 ///
 /// ```
 /// unsafe impl kernel::transmute::AsBytes for MyStruct{};
 ///
 /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result {
-/// kernel::dma_write!(alloc[2].member = 0xf);
+/// kernel::dma_write!(alloc, [2]?.member, 0xf);
-/// kernel::dma_write!(alloc[1] = MyStruct { member: 0xf });
+/// kernel::dma_write!(alloc, [1]?, MyStruct { member: 0xf });
 /// # Ok::<(), Error>(()) }
 /// ```
 #[macro_export]
 macro_rules! dma_write {
-    ($dma:ident [ $idx:expr ] $($field:tt)*) => {{
+    (@parse [$dma:expr] [$($proj:tt)*] [, $val:expr]) => {{
-        $crate::dma_write!($dma, $idx, $($field)*)
+        let dma = &$dma;
-    }};
+        let ptr = $crate::ptr::project!(
-    ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {{
+            mut $crate::dma::CoherentAllocation::as_mut_ptr(dma), $($proj)*
-        $crate::dma_write!($($dma).*, $idx, $($field)*)
+        );
+        let val = $val;
+        // SAFETY: The pointer created by the projection is within the DMA region.
+        unsafe { $crate::dma::CoherentAllocation::field_write(dma, ptr, val) }
     }};
-    ($dma:expr, $idx: expr, = $val:expr) => {
+    (@parse [$dma:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => {
-        (|| -> ::core::result::Result<_, $crate::error::Error> {
+        $crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*])
-            let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
+    };
-            // SAFETY: `item_from_index` ensures that `item` is always a valid item.
+    (@parse [$dma:expr] [$($proj:tt)*] [[$index:expr]? $($rest:tt)*]) => {
-            unsafe { $crate::dma::CoherentAllocation::field_write(&$dma, item, $val) }
+        $crate::dma_write!(@parse [$dma] [$($proj)* [$index]?] [$($rest)*])
-            ::core::result::Result::Ok(())
+    };
-        })()
+    (@parse [$dma:expr] [$($proj:tt)*] [[$index:expr] $($rest:tt)*]) => {
+        $crate::dma_write!(@parse [$dma] [$($proj)* [$index]] [$($rest)*])
     };
-    ($dma:expr, $idx: expr, $(.$field:ident)* = $val:expr) => {
+    ($dma:expr, $($rest:tt)*) => {
-        (|| -> ::core::result::Result<_, $crate::error::Error> {
+        $crate::dma_write!(@parse [$dma] [] [$($rest)*])
-            let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
-            // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be
-            // dereferenced. The compiler also further validates the expression on whether `field`
-            // is a member of `item` when expanded by the macro.
-            unsafe {
-                let ptr_field = ::core::ptr::addr_of_mut!((*item) $(.$field)*);
-                $crate::dma::CoherentAllocation::field_write(&$dma, ptr_field, $val)
-            }
-            ::core::result::Result::Ok(())
-        })()
     };
 }

rust/kernel/lib.rs+4−0

 #![feature(generic_nonzero)]
 #![feature(inline_const)]
 #![feature(pointer_is_aligned)]
+#![feature(slice_ptr_len)]
 //
 // Stable since Rust 1.80.0.
 #![feature(slice_flatten)]
 #![feature(const_ptr_write)]
 #![feature(const_refs_to_cell)]
 //
+// Stable since Rust 1.84.0.
+#![feature(strict_provenance)]
+//
 // Expected to become stable.
 #![feature(arbitrary_self_types)]
 //

rust/kernel/ptr.rs+29−1

@@ -2,7 +2,13 @@ · 1 unmodified line
2		2
3	//! Types and functions to work with pointers and addresses.	3	//! Types and functions to work with pointers and addresses.
4		4
5	use core::mem::align_of;	5	pub mod projection;
		6	pub use crate::project_pointer as project;
		7
		8	use core::mem::{
		9	align_of,
		10	size_of, //
		11	};
6	use core::num::NonZero;	12	use core::num::NonZero;
7		13
8	/// Type representing an alignment, which is always a power of two.	14	/// Type representing an alignment, which is always a power of two.
@@ -225,3 +231,25 @@ macro_rules! impl_alignable_uint { · 216 unmodified lines
225	}	231	}
226		232
227	impl_alignable_uint!(u8, u16, u32, u64, usize);	233	impl_alignable_uint!(u8, u16, u32, u64, usize);
		234
		235	/// Trait to represent compile-time known size information.
		236	///
		237	/// This is a generalization of [`size_of`] that works for dynamically sized types.
		238	pub trait KnownSize {
		239	/// Get the size of an object of this type in bytes, with the metadata of the given pointer.
		240	fn size(p: *const Self) -> usize;
		241	}
		242
		243	impl<T> KnownSize for T {
		244	#[inline(always)]
		245	fn size(_: *const Self) -> usize {
		246	size_of::<T>()
		247	}
		248	}
		249
		250	impl<T> KnownSize for [T] {
		251	#[inline(always)]
		252	fn size(p: *const Self) -> usize {
		253	p.len() * size_of::<T>()
		254	}
		255	}

rust/kernel/ptr/projection.rs+305−0

@@ -0,0 +1,305 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	//! Infrastructure for handling projections.
	4
	5	use core::{
	6	mem::MaybeUninit,
	7	ops::Deref, //
	8	};
	9
	10	use crate::prelude::*;
	11
	12	/// Error raised when a projection is attempted on an array or slice out of bounds.
	13	pub struct OutOfBound;
	14
	15	impl From<OutOfBound> for Error {
	16	#[inline(always)]
	17	fn from(_: OutOfBound) -> Self {
	18	ERANGE
	19	}
	20	}
	21
	22	/// A helper trait to perform index projection.
	23	///
	24	/// This is similar to [`core::slice::SliceIndex`], but operates on raw pointers safely and
	25	/// fallibly.
	26	///
	27	/// # Safety
	28	///
	29	/// The implementation of `index` and `get` (if [`Some`] is returned) must ensure that, if provided
	30	/// input pointer `slice` and returned pointer `output`, then:
	31	/// - `output` has the same provenance as `slice`;
	32	/// - `output.byte_offset_from(slice)` is between 0 to
	33	/// `KnownSize::size(slice) - KnownSize::size(output)`.
	34	///
	35	/// This means that if the input pointer is valid, then pointer returned by `get` or `index` is
	36	/// also valid.
	37	#[diagnostic::on_unimplemented(message = "`{Self}` cannot be used to index `{T}`")]
	38	#[doc(hidden)]
	39	pub unsafe trait ProjectIndex<T: ?Sized>: Sized {
	40	type Output: ?Sized;
	41
	42	/// Returns an index-projected pointer, if in bounds.
	43	fn get(self, slice: mut T) -> Option<mut Self::Output>;
	44
	45	/// Returns an index-projected pointer; fail the build if it cannot be proved to be in bounds.
	46	#[inline(always)]
	47	fn index(self, slice: mut T) -> mut Self::Output {
	48	Self::get(self, slice).unwrap_or_else(\|\| build_error!())
	49	}
	50	}
	51
	52	// Forward array impl to slice impl.
	53	//
	54	// SAFETY: Safety requirement guaranteed by the forwarded impl.
	55	unsafe impl<T, I, const N: usize> ProjectIndex<[T; N]> for I
	56	where
	57	I: ProjectIndex<[T]>,
	58	{
	59	type Output = <I as ProjectIndex<[T]>>::Output;
	60
	61	#[inline(always)]
	62	fn get(self, slice: mut [T; N]) -> Option<mut Self::Output> {
	63	<I as ProjectIndex<[T]>>::get(self, slice)
	64	}
	65
	66	#[inline(always)]
	67	fn index(self, slice: mut [T; N]) -> mut Self::Output {
	68	<I as ProjectIndex<[T]>>::index(self, slice)
	69	}
	70	}
	71
	72	// SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to
	73	// not exceed the required bound.
	74	unsafe impl<T> ProjectIndex<[T]> for usize {
	75	type Output = T;
	76
	77	#[inline(always)]
	78	fn get(self, slice: mut [T]) -> Option<mut T> {
	79	if self >= slice.len() {
	80	None
	81	} else {
	82	Some(slice.cast::<T>().wrapping_add(self))
	83	}
	84	}
	85	}
	86
	87	// SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to
	88	// not exceed the required bound.
	89	unsafe impl<T> ProjectIndex<[T]> for core::ops::Range<usize> {
	90	type Output = [T];
	91
	92	#[inline(always)]
	93	fn get(self, slice: mut [T]) -> Option<mut [T]> {
	94	let new_len = self.end.checked_sub(self.start)?;
	95	if self.end > slice.len() {
	96	return None;
	97	}
	98	Some(core::ptr::slice_from_raw_parts_mut(
	99	slice.cast::<T>().wrapping_add(self.start),
	100	new_len,
	101	))
	102	}
	103	}
	104
	105	// SAFETY: Safety requirement guaranteed by the forwarded impl.
	106	unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeTo<usize> {
	107	type Output = [T];
	108
	109	#[inline(always)]
	110	fn get(self, slice: mut [T]) -> Option<mut [T]> {
	111	(0..self.end).get(slice)
	112	}
	113	}
	114
	115	// SAFETY: Safety requirement guaranteed by the forwarded impl.
	116	unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeFrom<usize> {
	117	type Output = [T];
	118
	119	#[inline(always)]
	120	fn get(self, slice: mut [T]) -> Option<mut [T]> {
	121	(self.start..slice.len()).get(slice)
	122	}
	123	}
	124
	125	// SAFETY: `get` returned the pointer as is, so it always has the same provenance and offset of 0.
	126	unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeFull {
	127	type Output = [T];
	128
	129	#[inline(always)]
	130	fn get(self, slice: mut [T]) -> Option<mut [T]> {
	131	Some(slice)
	132	}
	133	}
	134
	135	/// A helper trait to perform field projection.
	136	///
	137	/// This trait has a `DEREF` generic parameter so it can be implemented twice for types that
	138	/// implement [`Deref`]. This will cause an ambiguity error and thus block [`Deref`] types being
	139	/// used as base of projection, as they can inject unsoundness. Users therefore must not specify
	140	/// `DEREF` and should always leave it to be inferred.
	141	///
	142	/// # Safety
	143	///
	144	/// `proj` may only invoke `f` with a valid allocation, as the documentation of [`Self::proj`]
	145	/// describes.
	146	#[doc(hidden)]
	147	pub unsafe trait ProjectField<const DEREF: bool> {
	148	/// Project a pointer to a type to a pointer of a field.
	149	///
	150	/// `f` may only be invoked with a valid allocation so it can safely obtain raw pointers to
	151	/// fields using `&raw mut`.
	152	///
	153	/// This is needed because `base` might not point to a valid allocation, while `&raw mut`
	154	/// requires pointers to be in bounds of a valid allocation.
	155	///
	156	/// # Safety
	157	///
	158	/// `f` must return a pointer in bounds of the provided pointer.
	159	unsafe fn proj<F>(base: mut Self, f: impl FnOnce(mut Self) -> mut F) -> mut F;
	160	}
	161
	162	// NOTE: in theory, this API should work for `T: ?Sized` and `F: ?Sized`, too. However, we cannot
	163	// currently support that as we need to obtain a valid allocation that `&raw const` can operate on.
	164	//
	165	// SAFETY: `proj` invokes `f` with valid allocation.
	166	unsafe impl<T> ProjectField<false> for T {
	167	#[inline(always)]
	168	unsafe fn proj<F>(base: mut Self, f: impl FnOnce(mut Self) -> mut F) -> mut F {
	169	// Create a valid allocation to start projection, as `base` is not necessarily so. The
	170	// memory is never actually used so it will be optimized out, so it should work even for
	171	// very large `T` (`memoffset` crate also relies on this). To be extra certain, we also
	172	// annotate `f` closure with `#[inline(always)]` in the macro.
	173	let mut place = MaybeUninit::uninit();
	174	let place_base = place.as_mut_ptr();
	175	let field = f(place_base);
	176	// SAFETY: `field` is in bounds from `base` per safety requirement.
	177	let offset = unsafe { field.byte_offset_from(place_base) };
	178	// Use `wrapping_byte_offset` as `base` does not need to be of valid allocation.
	179	base.wrapping_byte_offset(offset).cast()
	180	}
	181	}
	182
	183	// SAFETY: Vacuously satisfied.
	184	unsafe impl<T: Deref> ProjectField<true> for T {
	185	#[inline(always)]
	186	unsafe fn proj<F>(_: mut Self, _: impl FnOnce(mut Self) -> mut F) -> mut F {
	187	build_error!("this function is a guard against `Deref` impl and is never invoked");
	188	}
	189	}
	190
	191	/// Create a projection from a raw pointer.
	192	///
	193	/// The projected pointer is within the memory region marked by the input pointer. There is no
	194	/// requirement that the input raw pointer needs to be valid, so this macro may be used for
	195	/// projecting pointers outside normal address space, e.g. I/O pointers. However, if the input
	196	/// pointer is valid, the projected pointer is also valid.
	197	///
	198	/// Supported projections include field projections and index projections.
	199	/// It is not allowed to project into types that implement custom [`Deref`] or
	200	/// [`Index`](core::ops::Index).
	201	///
	202	/// The macro has basic syntax of `kernel::ptr::project!(ptr, projection)`, where `ptr` is an
	203	/// expression that evaluates to a raw pointer which serves as the base of projection. `projection`
	204	/// can be a projection expression of form `.field` (normally identifier, or numeral in case of
	205	/// tuple structs) or of form `[index]`.
	206	///
	207	/// If a mutable pointer is needed, the macro input can be prefixed with the `mut` keyword, i.e.
	208	/// `kernel::ptr::project!(mut ptr, projection)`. By default, a const pointer is created.
	209	///
	210	/// `ptr::project!` macro can perform both fallible indexing and build-time checked indexing.
	211	/// `[index]` form performs build-time bounds checking; if compiler fails to prove `[index]` is in
	212	/// bounds, compilation will fail. `[index]?` can be used to perform runtime bounds checking;
	213	/// `OutOfBound` error is raised via `?` if the index is out of bounds.
	214	///
	215	/// # Examples
	216	///
	217	/// Field projections are performed with `.field_name`:
	218	///
	219	/// ```
	220	/// struct MyStruct { field: u32, }
	221	/// let ptr: *const MyStruct = core::ptr::dangling();
	222	/// let field_ptr: *const u32 = kernel::ptr::project!(ptr, .field);
	223	///
	224	/// struct MyTupleStruct(u32, u32);
	225	///
	226	/// fn proj(ptr: *const MyTupleStruct) {
	227	/// let field_ptr: *const u32 = kernel::ptr::project!(ptr, .1);
	228	/// }
	229	/// ```
	230	///
	231	/// Index projections are performed with `[index]`:
	232	///
	233	/// ```
	234	/// fn proj(ptr: *const [u8; 32]) -> Result {
	235	/// let field_ptr: *const u8 = kernel::ptr::project!(ptr, [1]);
	236	/// // The following invocation, if uncommented, would fail the build.
	237	/// //
	238	/// // kernel::ptr::project!(ptr, [128]);
	239	///
	240	/// // This will raise an `OutOfBound` error (which is convertible to `ERANGE`).
	241	/// kernel::ptr::project!(ptr, [128]?);
	242	/// Ok(())
	243	/// }
	244	/// ```
	245	///
	246	/// If you need to match on the error instead of propagate, put the invocation inside a closure:
	247	///
	248	/// ```
	249	/// let ptr: *const [u8; 32] = core::ptr::dangling();
	250	/// let field_ptr: Result<*const u8> = (\|\| -> Result<_> {
	251	/// Ok(kernel::ptr::project!(ptr, [128]?))
	252	/// })();
	253	/// assert!(field_ptr.is_err());
	254	/// ```
	255	///
	256	/// For mutable pointers, put `mut` as the first token in macro invocation.
	257	///
	258	/// ```
	259	/// let ptr: *mut [(u8, u16); 32] = core::ptr::dangling_mut();
	260	/// let field_ptr: *mut u16 = kernel::ptr::project!(mut ptr, [1].1);
	261	/// ```
	262	#[macro_export]
	263	macro_rules! project_pointer {
	264	(@gen $ptr:ident, ) => {};
	265	// Field projection. `$field` needs to be `tt` to support tuple index like `.0`.
	266	(@gen $ptr:ident, .$field:tt $($rest:tt)*) => {
	267	// SAFETY: The provided closure always returns an in-bounds pointer.
	268	let $ptr = unsafe {
	269	$crate::ptr::projection::ProjectField::proj($ptr, #[inline(always)] \|ptr\| {
	270	// Check unaligned field. Not all users (e.g. DMA) can handle unaligned
	271	// projections.
	272	if false {
	273	let _ = &(*ptr).$field;
	274	}
	275	// SAFETY: `$field` is in bounds, and no implicit `Deref` is possible (if the
	276	// type implements `Deref`, Rust cannot infer the generic parameter `DEREF`).
	277	&raw mut (*ptr).$field
	278	})
	279	};
	280	$crate::ptr::project!(@gen $ptr, $($rest)*)
	281	};
	282	// Fallible index projection.
	283	(@gen $ptr:ident, [$index:expr]? $($rest:tt)*) => {
	284	let $ptr = $crate::ptr::projection::ProjectIndex::get($index, $ptr)
	285	.ok_or($crate::ptr::projection::OutOfBound)?;
	286	$crate::ptr::project!(@gen $ptr, $($rest)*)
	287	};
	288	// Build-time checked index projection.
	289	(@gen $ptr:ident, [$index:expr] $($rest:tt)*) => {
	290	let $ptr = $crate::ptr::projection::ProjectIndex::index($index, $ptr);
	291	$crate::ptr::project!(@gen $ptr, $($rest)*)
	292	};
	293	(mut $ptr:expr, $($proj:tt)*) => {{
	294	let ptr: *mut _ = $ptr;
	295	$crate::ptr::project!(@gen ptr, $($proj)*);
	296	ptr
	297	}};
	298	($ptr:expr, $($proj:tt)*) => {{
	299	let ptr = <*const _>::cast_mut($ptr);
	300	// We currently always project using mutable pointer, as it is not decided whether `&raw
	301	// const` allows the resulting pointer to be mutated (see documentation of `addr_of!`).
	302	$crate::ptr::project!(@gen ptr, $($proj)*);
	303	ptr.cast_const()
	304	}};
	305	}

samples/rust/rust_dma.rs+16−14

                 CoherentAllocation::alloc_coherent(pdev.as_ref(), TEST_VALUES.len(), GFP_KERNEL)?;
             for (i, value) in TEST_VALUES.into_iter().enumerate() {
-                kernel::dma_write!(ca[i] = MyStruct::new(value.0, value.1))?;
+                kernel::dma_write!(ca, [i]?, MyStruct::new(value.0, value.1));
             }
             let size = 4 * page::PAGE_SIZE;
     }
 }
+impl DmaSampleDriver {
+    fn check_dma(&self) -> Result {
+        for (i, value) in TEST_VALUES.into_iter().enumerate() {
+            let val0 = kernel::dma_read!(self.ca, [i]?.h);
+            let val1 = kernel::dma_read!(self.ca, [i]?.b);
+            assert_eq!(val0, value.0);
+            assert_eq!(val1, value.1);
+        }
+        Ok(())
+    }
+}
 #[pinned_drop]
 impl PinnedDrop for DmaSampleDriver {
     fn drop(self: Pin<&mut Self>) {
         dev_info!(self.pdev, "Unload DMA test driver.\n");
-        for (i, value) in TEST_VALUES.into_iter().enumerate() {
+        assert!(self.check_dma().is_ok());
-            let val0 = kernel::dma_read!(self.ca[i].h);
-            let val1 = kernel::dma_read!(self.ca[i].b);
-            assert!(val0.is_ok());
-            assert!(val1.is_ok());
-            if let Ok(val0) = val0 {
-                assert_eq!(val0, value.0);
-            }
-            if let Ok(val1) = val1 {
-                assert_eq!(val1, value.1);
-            }
-        }
         for (i, entry) in self.sgt.iter().enumerate() {
             dev_info!(

scripts/Makefile.build+3−1

 # The features in this list are the ones allowed for non-`rust/` code.
 #
+#   - Stable since Rust 1.79.0: `feature(slice_ptr_len)`.
 #   - Stable since Rust 1.81.0: `feature(lint_reasons)`.
 #   - Stable since Rust 1.82.0: `feature(asm_const)`,
 #     `feature(offset_of_nested)`, `feature(raw_ref_op)`.
+#   - Stable since Rust 1.84.0: `feature(strict_provenance)`.
 #   - Stable since Rust 1.87.0: `feature(asm_goto)`.
 #   - Expected to become stable: `feature(arbitrary_self_types)`.
 #   - To be determined: `feature(used_with_arg)`.
 #
 # Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on
 # the unstable features in use.
-rust_allowed_features := asm_const,asm_goto,arbitrary_self_types,lint_reasons,offset_of_nested,raw_ref_op,used_with_arg
+rust_allowed_features := asm_const,asm_goto,arbitrary_self_types,lint_reasons,offset_of_nested,raw_ref_op,slice_ptr_len,strict_provenance,used_with_arg
 # `--out-dir` is required to avoid temporaries being created by `rustc` in the
 # current working directory, which may be not accessible in the out-of-tree