Core Changes: - Fix safety issue in dma_read! and dma_write!. Driver Changes (Nova Core): - Fix UB in DmaGspMem pointer accessors. - Fix stack overflow in GSP memory allocation. Signed-off-by: Dave Airlie <airlied@redhat.com>
Unified · Split
| @@ -47,16 +47,12 @@ struct PteArray<const NUM_ENTRIES: usize>([u64; NUM_ENTRIES]); · 46 unmodified lines | ||
| 47 | 47 | unsafe impl<const NUM_ENTRIES: usize> AsBytes for PteArray<NUM_ENTRIES> {} |
| 48 | 48 | |
| 49 | 49 | impl<const NUM_PAGES: usize> PteArray<NUM_PAGES> { |
| 50 | − /// Creates a new page table array mapping `NUM_PAGES` GSP pages starting at address `start`. | |
| 51 | − fn new(start: DmaAddress) -> Result<Self> { | |
| 52 | − let mut ptes = [0u64; NUM_PAGES]; | |
| 53 | − for (i, pte) in ptes.iter_mut().enumerate() { | |
| 54 | − *pte = start | |
| 55 | − .checked_add(num::usize_as_u64(i) << GSP_PAGE_SHIFT) | |
| 56 | − .ok_or(EOVERFLOW)?; | |
| 57 | − } | |
| 58 | − | |
| 59 | − Ok(Self(ptes)) | |
| 50 | + /// Returns the page table entry for `index`, for a mapping starting at `start`. | |
| 51 | + // TODO: Replace with `IoView` projection once available. | |
| 52 | + fn entry(start: DmaAddress, index: usize) -> Result<u64> { | |
| 53 | + start | |
| 54 | + .checked_add(num::usize_as_u64(index) << GSP_PAGE_SHIFT) | |
| 55 | + .ok_or(EOVERFLOW) | |
| 60 | 56 | } |
| 61 | 57 | } |
| 62 | 58 | |
| @@ -86,16 +82,22 @@ impl LogBuffer { · 23 unmodified lines | ||
| 86 | 82 | NUM_PAGES * GSP_PAGE_SIZE, |
| 87 | 83 | GFP_KERNEL | __GFP_ZERO, |
| 88 | 84 | )?); |
| 89 | − let ptes = PteArray::<NUM_PAGES>::new(obj.0.dma_handle())?; | |
| 85 | + | |
| 86 | + let start_addr = obj.0.dma_handle(); | |
| 90 | 87 | |
| 91 | 88 | // SAFETY: `obj` has just been created and we are its sole user. |
| 92 | − unsafe { | |
| 93 | − // Copy the self-mapping PTE at the expected location. | |
| 89 | + let pte_region = unsafe { | |
| 94 | 90 | obj.0 |
| 95 | − .as_slice_mut(size_of::<u64>(), size_of_val(&ptes))? | |
| 96 | − .copy_from_slice(ptes.as_bytes()) | |
| 91 | + .as_slice_mut(size_of::<u64>(), NUM_PAGES * size_of::<u64>())? | |
| 97 | 92 | }; |
| 98 | 93 | |
| 94 | + // Write values one by one to avoid an on-stack instance of `PteArray`. | |
| 95 | + for (i, chunk) in pte_region.chunks_exact_mut(size_of::<u64>()).enumerate() { | |
| 96 | + let pte_value = PteArray::<0>::entry(start_addr, i)?; | |
| 97 | + | |
| 98 | + chunk.copy_from_slice(&pte_value.to_ne_bytes()); | |
| 99 | + } | |
| 100 | + | |
| 99 | 101 | Ok(obj) |
| 100 | 102 | } |
| 101 | 103 | } |
| @@ -143,14 +145,14 @@ impl Gsp { · 41 unmodified lines | ||
| 143 | 145 | // _kgspInitLibosLoggingStructures (allocates memory for buffers) |
| 144 | 146 | // kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array) |
| 145 | 147 | dma_write!( |
| 146 | − libos[0] = LibosMemoryRegionInitArgument::new("LOGINIT", &loginit.0) | |
| 147 | − )?; | |
| 148 | + libos, [0]?, LibosMemoryRegionInitArgument::new("LOGINIT", &loginit.0) | |
| 149 | + ); | |
| 148 | 150 | dma_write!( |
| 149 | − libos[1] = LibosMemoryRegionInitArgument::new("LOGINTR", &logintr.0) | |
| 150 | − )?; | |
| 151 | − dma_write!(libos[2] = LibosMemoryRegionInitArgument::new("LOGRM", &logrm.0))?; | |
| 152 | − dma_write!(rmargs[0].inner = fw::GspArgumentsCached::new(cmdq))?; | |
| 153 | − dma_write!(libos[3] = LibosMemoryRegionInitArgument::new("RMARGS", rmargs))?; | |
| 151 | + libos, [1]?, LibosMemoryRegionInitArgument::new("LOGINTR", &logintr.0) | |
| 152 | + ); | |
| 153 | + dma_write!(libos, [2]?, LibosMemoryRegionInitArgument::new("LOGRM", &logrm.0)); | |
| 154 | + dma_write!(rmargs, [0]?.inner, fw::GspArgumentsCached::new(cmdq)); | |
| 155 | + dma_write!(libos, [3]?, LibosMemoryRegionInitArgument::new("RMARGS", rmargs)); | |
| 154 | 156 | }, |
| 155 | 157 | })) |
| 156 | 158 | }) |
| @@ -157,7 +157,7 @@ impl super::Gsp { · 156 unmodified lines | ||
| 157 | 157 | |
| 158 | 158 | let wpr_meta = |
| 159 | 159 | CoherentAllocation::<GspFwWprMeta>::alloc_coherent(dev, 1, GFP_KERNEL | __GFP_ZERO)?; |
| 160 | − dma_write!(wpr_meta[0] = GspFwWprMeta::new(&gsp_fw, &fb_layout))?; | |
| 160 | + dma_write!(wpr_meta, [0]?, GspFwWprMeta::new(&gsp_fw, &fb_layout)); | |
| 161 | 161 | |
| 162 | 162 | self.cmdq |
| 163 | 163 | .send_command(bar, commands::SetSystemInfo::new(pdev))?; |
| @@ -2,11 +2,7 @@ · 1 unmodified line | ||
| 2 | 2 | |
| 3 | 3 | use core::{ |
| 4 | 4 | cmp, |
| 5 | − mem, | |
| 6 | − sync::atomic::{ | |
| 7 | − fence, | |
| 8 | − Ordering, // | |
| 9 | − }, // | |
| 5 | + mem, // | |
| 10 | 6 | }; |
| 11 | 7 | |
| 12 | 8 | use kernel::{ |
| @@ -146,30 +142,36 @@ static_assert!(align_of::<MsgqData>() == GSP_PAGE_SIZE); · 133 unmodified lines | ||
| 146 | 142 | #[repr(C)] |
| 147 | 143 | // There is no struct defined for this in the open-gpu-kernel-source headers. |
| 148 | 144 | // Instead it is defined by code in `GspMsgQueuesInit()`. |
| 149 | −struct Msgq { | |
| 145 | +// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. | |
| 146 | +pub(super) struct Msgq { | |
| 150 | 147 | /// Header for sending messages, including the write pointer. |
| 151 | − tx: MsgqTxHeader, | |
| 148 | + pub(super) tx: MsgqTxHeader, | |
| 152 | 149 | /// Header for receiving messages, including the read pointer. |
| 153 | − rx: MsgqRxHeader, | |
| 150 | + pub(super) rx: MsgqRxHeader, | |
| 154 | 151 | /// The message queue proper. |
| 155 | 152 | msgq: MsgqData, |
| 156 | 153 | } |
| 157 | 154 | |
| 158 | 155 | /// Structure shared between the driver and the GSP and containing the command and message queues. |
| 159 | 156 | #[repr(C)] |
| 160 | −struct GspMem { | |
| 157 | +// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. | |
| 158 | +pub(super) struct GspMem { | |
| 161 | 159 | /// Self-mapping page table entries. |
| 162 | − ptes: PteArray<{ GSP_PAGE_SIZE / size_of::<u64>() }>, | |
| 160 | + ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>, | |
| 163 | 161 | /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the |
| 164 | 162 | /// write and read pointers that the CPU updates. |
| 165 | 163 | /// |
| 166 | 164 | /// This member is read-only for the GSP. |
| 167 | − cpuq: Msgq, | |
| 165 | + pub(super) cpuq: Msgq, | |
| 168 | 166 | /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the |
| 169 | 167 | /// write and read pointers that the GSP updates. |
| 170 | 168 | /// |
| 171 | 169 | /// This member is read-only for the driver. |
| 172 | − gspq: Msgq, | |
| 170 | + pub(super) gspq: Msgq, | |
| 171 | +} | |
| 172 | + | |
| 173 | +impl GspMem { | |
| 174 | + const PTE_ARRAY_SIZE: usize = GSP_PAGE_SIZE / size_of::<u64>(); | |
| 173 | 175 | } |
| 174 | 176 | |
| 175 | 177 | // SAFETY: These structs don't meet the no-padding requirements of AsBytes but |
| @@ -201,9 +203,19 @@ impl DmaGspMem { · 25 unmodified lines | ||
| 201 | 203 | |
| 202 | 204 | let gsp_mem = |
| 203 | 205 | CoherentAllocation::<GspMem>::alloc_coherent(dev, 1, GFP_KERNEL | __GFP_ZERO)?; |
| 204 | − dma_write!(gsp_mem[0].ptes = PteArray::new(gsp_mem.dma_handle())?)?; | |
| 205 | − dma_write!(gsp_mem[0].cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES))?; | |
| 206 | − dma_write!(gsp_mem[0].cpuq.rx = MsgqRxHeader::new())?; | |
| 206 | + | |
| 207 | + let start = gsp_mem.dma_handle(); | |
| 208 | + // Write values one by one to avoid an on-stack instance of `PteArray`. | |
| 209 | + for i in 0..GspMem::PTE_ARRAY_SIZE { | |
| 210 | + dma_write!(gsp_mem, [0]?.ptes.0[i], PteArray::<0>::entry(start, i)?); | |
| 211 | + } | |
| 212 | + | |
| 213 | + dma_write!( | |
| 214 | + gsp_mem, | |
| 215 | + [0]?.cpuq.tx, | |
| 216 | + MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES) | |
| 217 | + ); | |
| 218 | + dma_write!(gsp_mem, [0]?.cpuq.rx, MsgqRxHeader::new()); | |
| 207 | 219 | |
| 208 | 220 | Ok(Self(gsp_mem)) |
| 209 | 221 | } |
| @@ -317,12 +329,7 @@ impl DmaGspMem { · 107 unmodified lines | ||
| 317 | 329 | // |
| 318 | 330 | // - The returned value is between `0` and `MSGQ_NUM_PAGES`. |
| 319 | 331 | fn gsp_write_ptr(&self) -> u32 { |
| 320 | − let gsp_mem = self.0.start_ptr(); | |
| 321 | − | |
| 322 | − // SAFETY: | |
| 323 | − // - The 'CoherentAllocation' contains at least one object. | |
| 324 | − // - By the invariants of `CoherentAllocation` the pointer is valid. | |
| 325 | − (unsafe { (*gsp_mem).gspq.tx.write_ptr() } % MSGQ_NUM_PAGES) | |
| 332 | + super::fw::gsp_mem::gsp_write_ptr(&self.0) | |
| 326 | 333 | } |
| 327 | 334 | |
| 328 | 335 | // Returns the index of the memory page the GSP will read the next command from. |
| @@ -331,12 +338,7 @@ impl DmaGspMem { · 2 unmodified lines | ||
| 331 | 338 | // |
| 332 | 339 | // - The returned value is between `0` and `MSGQ_NUM_PAGES`. |
| 333 | 340 | fn gsp_read_ptr(&self) -> u32 { |
| 334 | − let gsp_mem = self.0.start_ptr(); | |
| 335 | − | |
| 336 | − // SAFETY: | |
| 337 | − // - The 'CoherentAllocation' contains at least one object. | |
| 338 | − // - By the invariants of `CoherentAllocation` the pointer is valid. | |
| 339 | − (unsafe { (*gsp_mem).gspq.rx.read_ptr() } % MSGQ_NUM_PAGES) | |
| 341 | + super::fw::gsp_mem::gsp_read_ptr(&self.0) | |
| 340 | 342 | } |
| 341 | 343 | |
| 342 | 344 | // Returns the index of the memory page the CPU can read the next message from. |
| @@ -345,27 +347,12 @@ impl DmaGspMem { · 2 unmodified lines | ||
| 345 | 347 | // |
| 346 | 348 | // - The returned value is between `0` and `MSGQ_NUM_PAGES`. |
| 347 | 349 | fn cpu_read_ptr(&self) -> u32 { |
| 348 | − let gsp_mem = self.0.start_ptr(); | |
| 349 | − | |
| 350 | − // SAFETY: | |
| 351 | − // - The ['CoherentAllocation'] contains at least one object. | |
| 352 | − // - By the invariants of CoherentAllocation the pointer is valid. | |
| 353 | − (unsafe { (*gsp_mem).cpuq.rx.read_ptr() } % MSGQ_NUM_PAGES) | |
| 350 | + super::fw::gsp_mem::cpu_read_ptr(&self.0) | |
| 354 | 351 | } |
| 355 | 352 | |
| 356 | 353 | // Informs the GSP that it can send `elem_count` new pages into the message queue. |
| 357 | 354 | fn advance_cpu_read_ptr(&mut self, elem_count: u32) { |
| 358 | − let rptr = self.cpu_read_ptr().wrapping_add(elem_count) % MSGQ_NUM_PAGES; | |
| 359 | − | |
| 360 | − // Ensure read pointer is properly ordered. | |
| 361 | − fence(Ordering::SeqCst); | |
| 362 | − | |
| 363 | − let gsp_mem = self.0.start_ptr_mut(); | |
| 364 | − | |
| 365 | − // SAFETY: | |
| 366 | − // - The 'CoherentAllocation' contains at least one object. | |
| 367 | − // - By the invariants of `CoherentAllocation` the pointer is valid. | |
| 368 | − unsafe { (*gsp_mem).cpuq.rx.set_read_ptr(rptr) }; | |
| 355 | + super::fw::gsp_mem::advance_cpu_read_ptr(&self.0, elem_count) | |
| 369 | 356 | } |
| 370 | 357 | |
| 371 | 358 | // Returns the index of the memory page the CPU can write the next command to. |
| @@ -374,26 +361,12 @@ impl DmaGspMem { · 2 unmodified lines | ||
| 374 | 361 | // |
| 375 | 362 | // - The returned value is between `0` and `MSGQ_NUM_PAGES`. |
| 376 | 363 | fn cpu_write_ptr(&self) -> u32 { |
| 377 | − let gsp_mem = self.0.start_ptr(); | |
| 378 | − | |
| 379 | − // SAFETY: | |
| 380 | − // - The 'CoherentAllocation' contains at least one object. | |
| 381 | − // - By the invariants of `CoherentAllocation` the pointer is valid. | |
| 382 | − (unsafe { (*gsp_mem).cpuq.tx.write_ptr() } % MSGQ_NUM_PAGES) | |
| 364 | + super::fw::gsp_mem::cpu_write_ptr(&self.0) | |
| 383 | 365 | } |
| 384 | 366 | |
| 385 | 367 | // Informs the GSP that it can process `elem_count` new pages from the command queue. |
| 386 | 368 | fn advance_cpu_write_ptr(&mut self, elem_count: u32) { |
| 387 | − let wptr = self.cpu_write_ptr().wrapping_add(elem_count) & MSGQ_NUM_PAGES; | |
| 388 | − let gsp_mem = self.0.start_ptr_mut(); | |
| 389 | − | |
| 390 | − // SAFETY: | |
| 391 | − // - The 'CoherentAllocation' contains at least one object. | |
| 392 | − // - By the invariants of `CoherentAllocation` the pointer is valid. | |
| 393 | − unsafe { (*gsp_mem).cpuq.tx.set_write_ptr(wptr) }; | |
| 394 | − | |
| 395 | − // Ensure all command data is visible before triggering the GSP read. | |
| 396 | − fence(Ordering::SeqCst); | |
| 369 | + super::fw::gsp_mem::advance_cpu_write_ptr(&self.0, elem_count) | |
| 397 | 370 | } |
| 398 | 371 | } |
| 399 | 372 | |
| @@ -40,6 +40,75 @@ use crate::{ · 39 unmodified lines | ||
| 40 | 40 | }, |
| 41 | 41 | }; |
| 42 | 42 | |
| 43 | +// TODO: Replace with `IoView` projections once available; the `unwrap()` calls go away once we | |
| 44 | +// switch to the new `dma::Coherent` API. | |
| 45 | +pub(super) mod gsp_mem { | |
| 46 | + use core::sync::atomic::{ | |
| 47 | + fence, | |
| 48 | + Ordering, // | |
| 49 | + }; | |
| 50 | + | |
| 51 | + use kernel::{ | |
| 52 | + dma::CoherentAllocation, | |
| 53 | + dma_read, | |
| 54 | + dma_write, | |
| 55 | + prelude::*, // | |
| 56 | + }; | |
| 57 | + | |
| 58 | + use crate::gsp::cmdq::{ | |
| 59 | + GspMem, | |
| 60 | + MSGQ_NUM_PAGES, // | |
| 61 | + }; | |
| 62 | + | |
| 63 | + pub(in crate::gsp) fn gsp_write_ptr(qs: &CoherentAllocation<GspMem>) -> u32 { | |
| 64 | + // PANIC: A `dma::CoherentAllocation` always contains at least one element. | |
| 65 | + || -> Result<u32> { Ok(dma_read!(qs, [0]?.gspq.tx.0.writePtr) % MSGQ_NUM_PAGES) }().unwrap() | |
| 66 | + } | |
| 67 | + | |
| 68 | + pub(in crate::gsp) fn gsp_read_ptr(qs: &CoherentAllocation<GspMem>) -> u32 { | |
| 69 | + // PANIC: A `dma::CoherentAllocation` always contains at least one element. | |
| 70 | + || -> Result<u32> { Ok(dma_read!(qs, [0]?.gspq.rx.0.readPtr) % MSGQ_NUM_PAGES) }().unwrap() | |
| 71 | + } | |
| 72 | + | |
| 73 | + pub(in crate::gsp) fn cpu_read_ptr(qs: &CoherentAllocation<GspMem>) -> u32 { | |
| 74 | + // PANIC: A `dma::CoherentAllocation` always contains at least one element. | |
| 75 | + || -> Result<u32> { Ok(dma_read!(qs, [0]?.cpuq.rx.0.readPtr) % MSGQ_NUM_PAGES) }().unwrap() | |
| 76 | + } | |
| 77 | + | |
| 78 | + pub(in crate::gsp) fn advance_cpu_read_ptr(qs: &CoherentAllocation<GspMem>, count: u32) { | |
| 79 | + let rptr = cpu_read_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; | |
| 80 | + | |
| 81 | + // Ensure read pointer is properly ordered. | |
| 82 | + fence(Ordering::SeqCst); | |
| 83 | + | |
| 84 | + // PANIC: A `dma::CoherentAllocation` always contains at least one element. | |
| 85 | + || -> Result { | |
| 86 | + dma_write!(qs, [0]?.cpuq.rx.0.readPtr, rptr); | |
| 87 | + Ok(()) | |
| 88 | + }() | |
| 89 | + .unwrap() | |
| 90 | + } | |
| 91 | + | |
| 92 | + pub(in crate::gsp) fn cpu_write_ptr(qs: &CoherentAllocation<GspMem>) -> u32 { | |
| 93 | + // PANIC: A `dma::CoherentAllocation` always contains at least one element. | |
| 94 | + || -> Result<u32> { Ok(dma_read!(qs, [0]?.cpuq.tx.0.writePtr) % MSGQ_NUM_PAGES) }().unwrap() | |
| 95 | + } | |
| 96 | + | |
| 97 | + pub(in crate::gsp) fn advance_cpu_write_ptr(qs: &CoherentAllocation<GspMem>, count: u32) { | |
| 98 | + let wptr = cpu_write_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; | |
| 99 | + | |
| 100 | + // PANIC: A `dma::CoherentAllocation` always contains at least one element. | |
| 101 | + || -> Result { | |
| 102 | + dma_write!(qs, [0]?.cpuq.tx.0.writePtr, wptr); | |
| 103 | + Ok(()) | |
| 104 | + }() | |
| 105 | + .unwrap(); | |
| 106 | + | |
| 107 | + // Ensure all command data is visible before triggering the GSP read. | |
| 108 | + fence(Ordering::SeqCst); | |
| 109 | + } | |
| 110 | +} | |
| 111 | + | |
| 43 | 112 | /// Empty type to group methods related to heap parameters for running the GSP firmware. |
| 44 | 113 | enum GspFwHeapParams {} |
| 45 | 114 | |
| @@ -708,22 +777,6 @@ impl MsgqTxHeader { · 662 unmodified lines | ||
| 708 | 777 | entryOff: num::usize_into_u32::<GSP_PAGE_SIZE>(), |
| 709 | 778 | }) |
| 710 | 779 | } |
| 711 | − | |
| 712 | − /// Returns the value of the write pointer for this queue. | |
| 713 | − pub(crate) fn write_ptr(&self) -> u32 { | |
| 714 | − let ptr = core::ptr::from_ref(&self.0.writePtr); | |
| 715 | − | |
| 716 | − // SAFETY: `ptr` is a valid pointer to a `u32`. | |
| 717 | − unsafe { ptr.read_volatile() } | |
| 718 | − } | |
| 719 | − | |
| 720 | − /// Sets the value of the write pointer for this queue. | |
| 721 | − pub(crate) fn set_write_ptr(&mut self, val: u32) { | |
| 722 | − let ptr = core::ptr::from_mut(&mut self.0.writePtr); | |
| 723 | − | |
| 724 | − // SAFETY: `ptr` is a valid pointer to a `u32`. | |
| 725 | − unsafe { ptr.write_volatile(val) } | |
| 726 | − } | |
| 727 | 780 | } |
| 728 | 781 | |
| 729 | 782 | // SAFETY: Padding is explicit and does not contain uninitialized data. |
| @@ -739,22 +792,6 @@ impl MsgqRxHeader { · 9 unmodified lines | ||
| 739 | 792 | pub(crate) fn new() -> Self { |
| 740 | 793 | Self(Default::default()) |
| 741 | 794 | } |
| 742 | − | |
| 743 | − /// Returns the value of the read pointer for this queue. | |
| 744 | − pub(crate) fn read_ptr(&self) -> u32 { | |
| 745 | − let ptr = core::ptr::from_ref(&self.0.readPtr); | |
| 746 | − | |
| 747 | − // SAFETY: `ptr` is a valid pointer to a `u32`. | |
| 748 | − unsafe { ptr.read_volatile() } | |
| 749 | − } | |
| 750 | − | |
| 751 | − /// Sets the value of the read pointer for this queue. | |
| 752 | − pub(crate) fn set_read_ptr(&mut self, val: u32) { | |
| 753 | − let ptr = core::ptr::from_mut(&mut self.0.readPtr); | |
| 754 | − | |
| 755 | − // SAFETY: `ptr` is a valid pointer to a `u32`. | |
| 756 | − unsafe { ptr.write_volatile(val) } | |
| 757 | − } | |
| 758 | 795 | } |
| 759 | 796 | |
| 760 | 797 | // SAFETY: Padding is explicit and does not contain uninitialized data. |
| @@ -461,6 +461,19 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { · 460 unmodified lines | ||
| 461 | 461 | self.count * core::mem::size_of::<T>() |
| 462 | 462 | } |
| 463 | 463 | |
| 464 | + /// Returns the raw pointer to the allocated region in the CPU's virtual address space. | |
| 465 | + #[inline] | |
| 466 | + pub fn as_ptr(&self) -> *const [T] { | |
| 467 | + core::ptr::slice_from_raw_parts(self.cpu_addr.as_ptr(), self.count) | |
| 468 | + } | |
| 469 | + | |
| 470 | + /// Returns the raw pointer to the allocated region in the CPU's virtual address space as | |
| 471 | + /// a mutable pointer. | |
| 472 | + #[inline] | |
| 473 | + pub fn as_mut_ptr(&self) -> *mut [T] { | |
| 474 | + core::ptr::slice_from_raw_parts_mut(self.cpu_addr.as_ptr(), self.count) | |
| 475 | + } | |
| 476 | + | |
| 464 | 477 | /// Returns the base address to the allocated region in the CPU's virtual address space. |
| 465 | 478 | pub fn start_ptr(&self) -> *const T { |
| 466 | 479 | self.cpu_addr.as_ptr() |
| @@ -581,23 +594,6 @@ impl<T: AsBytes + FromBytes> CoherentAllocation<T> { · 114 unmodified lines | ||
| 581 | 594 | Ok(()) |
| 582 | 595 | } |
| 583 | 596 | |
| 584 | − /// Returns a pointer to an element from the region with bounds checking. `offset` is in | |
| 585 | − /// units of `T`, not the number of bytes. | |
| 586 | − /// | |
| 587 | − /// Public but hidden since it should only be used from [`dma_read`] and [`dma_write`] macros. | |
| 588 | − #[doc(hidden)] | |
| 589 | − pub fn item_from_index(&self, offset: usize) -> Result<*mut T> { | |
| 590 | − if offset >= self.count { | |
| 591 | − return Err(EINVAL); | |
| 592 | − } | |
| 593 | − // SAFETY: | |
| 594 | − // - The pointer is valid due to type invariant on `CoherentAllocation` | |
| 595 | − // and we've just checked that the range and index is within bounds. | |
| 596 | − // - `offset` can't overflow since it is smaller than `self.count` and we've checked | |
| 597 | − // that `self.count` won't overflow early in the constructor. | |
| 598 | − Ok(unsafe { self.cpu_addr.as_ptr().add(offset) }) | |
| 599 | − } | |
| 600 | − | |
| 601 | 597 | /// Reads the value of `field` and ensures that its type is [`FromBytes`]. |
| 602 | 598 | /// |
| 603 | 599 | /// # Safety |
| @@ -670,6 +666,9 @@ unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {} · 66 unmodified lines | ||
| 670 | 666 | |
| 671 | 667 | /// Reads a field of an item from an allocated region of structs. |
| 672 | 668 | /// |
| 669 | +/// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating | |
| 670 | +/// to a [`CoherentAllocation`] and `proj` is a [projection specification](kernel::ptr::project!). | |
| 671 | +/// | |
| 673 | 672 | /// # Examples |
| 674 | 673 | /// |
| 675 | 674 | /// ``` |
| @@ -684,36 +683,29 @@ unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {} · 8 unmodified lines | ||
| 684 | 683 | /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; |
| 685 | 684 | /// |
| 686 | 685 | /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result { |
| 687 | −/// let whole = kernel::dma_read!(alloc[2]); | |
| 688 | −/// let field = kernel::dma_read!(alloc[1].field); | |
| 686 | +/// let whole = kernel::dma_read!(alloc, [2]?); | |
| 687 | +/// let field = kernel::dma_read!(alloc, [1]?.field); | |
| 689 | 688 | /// # Ok::<(), Error>(()) } |
| 690 | 689 | /// ``` |
| 691 | 690 | #[macro_export] |
| 692 | 691 | macro_rules! dma_read { |
| 693 | − ($dma:expr, $idx: expr, $($field:tt)*) => {{ | |
| 694 | − (|| -> ::core::result::Result<_, $crate::error::Error> { | |
| 695 | − let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; | |
| 696 | − // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be | |
| 697 | − // dereferenced. The compiler also further validates the expression on whether `field` | |
| 698 | − // is a member of `item` when expanded by the macro. | |
| 699 | − unsafe { | |
| 700 | − let ptr_field = ::core::ptr::addr_of!((*item) $($field)*); | |
| 701 | − ::core::result::Result::Ok( | |
| 702 | − $crate::dma::CoherentAllocation::field_read(&$dma, ptr_field) | |
| 703 | − ) | |
| 704 | − } | |
| 705 | − })() | |
| 692 | + ($dma:expr, $($proj:tt)*) => {{ | |
| 693 | + let dma = &$dma; | |
| 694 | + let ptr = $crate::ptr::project!( | |
| 695 | + $crate::dma::CoherentAllocation::as_ptr(dma), $($proj)* | |
| 696 | + ); | |
| 697 | + // SAFETY: The pointer created by the projection is within the DMA region. | |
| 698 | + unsafe { $crate::dma::CoherentAllocation::field_read(dma, ptr) } | |
| 706 | 699 | }}; |
| 707 | − ($dma:ident [ $idx:expr ] $($field:tt)* ) => { | |
| 708 | − $crate::dma_read!($dma, $idx, $($field)*) | |
| 709 | − }; | |
| 710 | − ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => { | |
| 711 | − $crate::dma_read!($($dma).*, $idx, $($field)*) | |
| 712 | − }; | |
| 713 | 700 | } |
| 714 | 701 | |
| 715 | 702 | /// Writes to a field of an item from an allocated region of structs. |
| 716 | 703 | /// |
| 704 | +/// The syntax is of the form `kernel::dma_write!(dma, proj, val)` where `dma` is an expression | |
| 705 | +/// evaluating to a [`CoherentAllocation`], `proj` is a | |
| 706 | +/// [projection specification](kernel::ptr::project!), and `val` is the value to be written to the | |
| 707 | +/// projected location. | |
| 708 | +/// | |
| 717 | 709 | /// # Examples |
| 718 | 710 | /// |
| 719 | 711 | /// ``` |
| @@ -728,37 +720,31 @@ macro_rules! dma_read { · 8 unmodified lines | ||
| 728 | 720 | /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; |
| 729 | 721 | /// |
| 730 | 722 | /// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result { |
| 731 | −/// kernel::dma_write!(alloc[2].member = 0xf); | |
| 732 | −/// kernel::dma_write!(alloc[1] = MyStruct { member: 0xf }); | |
| 723 | +/// kernel::dma_write!(alloc, [2]?.member, 0xf); | |
| 724 | +/// kernel::dma_write!(alloc, [1]?, MyStruct { member: 0xf }); | |
| 733 | 725 | /// # Ok::<(), Error>(()) } |
| 734 | 726 | /// ``` |
| 735 | 727 | #[macro_export] |
| 736 | 728 | macro_rules! dma_write { |
| 737 | − ($dma:ident [ $idx:expr ] $($field:tt)*) => {{ | |
| 738 | − $crate::dma_write!($dma, $idx, $($field)*) | |
| 739 | − }}; | |
| 740 | − ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {{ | |
| 741 | − $crate::dma_write!($($dma).*, $idx, $($field)*) | |
| 729 | + (@parse [$dma:expr] [$($proj:tt)*] [, $val:expr]) => {{ | |
| 730 | + let dma = &$dma; | |
| 731 | + let ptr = $crate::ptr::project!( | |
| 732 | + mut $crate::dma::CoherentAllocation::as_mut_ptr(dma), $($proj)* | |
| 733 | + ); | |
| 734 | + let val = $val; | |
| 735 | + // SAFETY: The pointer created by the projection is within the DMA region. | |
| 736 | + unsafe { $crate::dma::CoherentAllocation::field_write(dma, ptr, val) } | |
| 742 | 737 | }}; |
| 743 | − ($dma:expr, $idx: expr, = $val:expr) => { | |
| 744 | − (|| -> ::core::result::Result<_, $crate::error::Error> { | |
| 745 | − let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; | |
| 746 | − // SAFETY: `item_from_index` ensures that `item` is always a valid item. | |
| 747 | − unsafe { $crate::dma::CoherentAllocation::field_write(&$dma, item, $val) } | |
| 748 | − ::core::result::Result::Ok(()) | |
| 749 | − })() | |
| 738 | + (@parse [$dma:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { | |
| 739 | + $crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*]) | |
| 740 | + }; | |
| 741 | + (@parse [$dma:expr] [$($proj:tt)*] [[$index:expr]? $($rest:tt)*]) => { | |
| 742 | + $crate::dma_write!(@parse [$dma] [$($proj)* [$index]?] [$($rest)*]) | |
| 743 | + }; | |
| 744 | + (@parse [$dma:expr] [$($proj:tt)*] [[$index:expr] $($rest:tt)*]) => { | |
| 745 | + $crate::dma_write!(@parse [$dma] [$($proj)* [$index]] [$($rest)*]) | |
| 750 | 746 | }; |
| 751 | − ($dma:expr, $idx: expr, $(.$field:ident)* = $val:expr) => { | |
| 752 | − (|| -> ::core::result::Result<_, $crate::error::Error> { | |
| 753 | − let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?; | |
| 754 | − // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be | |
| 755 | − // dereferenced. The compiler also further validates the expression on whether `field` | |
| 756 | − // is a member of `item` when expanded by the macro. | |
| 757 | − unsafe { | |
| 758 | − let ptr_field = ::core::ptr::addr_of_mut!((*item) $(.$field)*); | |
| 759 | − $crate::dma::CoherentAllocation::field_write(&$dma, ptr_field, $val) | |
| 760 | − } | |
| 761 | − ::core::result::Result::Ok(()) | |
| 762 | − })() | |
| 747 | + ($dma:expr, $($rest:tt)*) => { | |
| 748 | + $crate::dma_write!(@parse [$dma] [] [$($rest)*]) | |
| 763 | 749 | }; |
| 764 | 750 | } |
| @@ -20,6 +20,7 @@ · 19 unmodified lines | ||
| 20 | 20 | #![feature(generic_nonzero)] |
| 21 | 21 | #![feature(inline_const)] |
| 22 | 22 | #![feature(pointer_is_aligned)] |
| 23 | +#![feature(slice_ptr_len)] | |
| 23 | 24 | // |
| 24 | 25 | // Stable since Rust 1.80.0. |
| 25 | 26 | #![feature(slice_flatten)] |
| @@ -37,6 +38,9 @@ · 11 unmodified lines | ||
| 37 | 38 | #![feature(const_ptr_write)] |
| 38 | 39 | #![feature(const_refs_to_cell)] |
| 39 | 40 | // |
| 41 | +// Stable since Rust 1.84.0. | |
| 42 | +#![feature(strict_provenance)] | |
| 43 | +// | |
| 40 | 44 | // Expected to become stable. |
| 41 | 45 | #![feature(arbitrary_self_types)] |
| 42 | 46 | // |
| @@ -2,7 +2,13 @@ · 1 unmodified line | ||
| 2 | 2 | |
| 3 | 3 | //! Types and functions to work with pointers and addresses. |
| 4 | 4 | |
| 5 | −use core::mem::align_of; | |
| 5 | +pub mod projection; | |
| 6 | +pub use crate::project_pointer as project; | |
| 7 | + | |
| 8 | +use core::mem::{ | |
| 9 | + align_of, | |
| 10 | + size_of, // | |
| 11 | +}; | |
| 6 | 12 | use core::num::NonZero; |
| 7 | 13 | |
| 8 | 14 | /// Type representing an alignment, which is always a power of two. |
| @@ -225,3 +231,25 @@ macro_rules! impl_alignable_uint { · 216 unmodified lines | ||
| 225 | 231 | } |
| 226 | 232 | |
| 227 | 233 | impl_alignable_uint!(u8, u16, u32, u64, usize); |
| 234 | + | |
| 235 | +/// Trait to represent compile-time known size information. | |
| 236 | +/// | |
| 237 | +/// This is a generalization of [`size_of`] that works for dynamically sized types. | |
| 238 | +pub trait KnownSize { | |
| 239 | + /// Get the size of an object of this type in bytes, with the metadata of the given pointer. | |
| 240 | + fn size(p: *const Self) -> usize; | |
| 241 | +} | |
| 242 | + | |
| 243 | +impl<T> KnownSize for T { | |
| 244 | + #[inline(always)] | |
| 245 | + fn size(_: *const Self) -> usize { | |
| 246 | + size_of::<T>() | |
| 247 | + } | |
| 248 | +} | |
| 249 | + | |
| 250 | +impl<T> KnownSize for [T] { | |
| 251 | + #[inline(always)] | |
| 252 | + fn size(p: *const Self) -> usize { | |
| 253 | + p.len() * size_of::<T>() | |
| 254 | + } | |
| 255 | +} | |
| @@ -0,0 +1,305 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0 | |
| 2 | + | |
| 3 | +//! Infrastructure for handling projections. | |
| 4 | + | |
| 5 | +use core::{ | |
| 6 | + mem::MaybeUninit, | |
| 7 | + ops::Deref, // | |
| 8 | +}; | |
| 9 | + | |
| 10 | +use crate::prelude::*; | |
| 11 | + | |
| 12 | +/// Error raised when a projection is attempted on an array or slice out of bounds. | |
| 13 | +pub struct OutOfBound; | |
| 14 | + | |
| 15 | +impl From<OutOfBound> for Error { | |
| 16 | + #[inline(always)] | |
| 17 | + fn from(_: OutOfBound) -> Self { | |
| 18 | + ERANGE | |
| 19 | + } | |
| 20 | +} | |
| 21 | + | |
| 22 | +/// A helper trait to perform index projection. | |
| 23 | +/// | |
| 24 | +/// This is similar to [`core::slice::SliceIndex`], but operates on raw pointers safely and | |
| 25 | +/// fallibly. | |
| 26 | +/// | |
| 27 | +/// # Safety | |
| 28 | +/// | |
| 29 | +/// The implementation of `index` and `get` (if [`Some`] is returned) must ensure that, if provided | |
| 30 | +/// input pointer `slice` and returned pointer `output`, then: | |
| 31 | +/// - `output` has the same provenance as `slice`; | |
| 32 | +/// - `output.byte_offset_from(slice)` is between 0 to | |
| 33 | +/// `KnownSize::size(slice) - KnownSize::size(output)`. | |
| 34 | +/// | |
| 35 | +/// This means that if the input pointer is valid, then pointer returned by `get` or `index` is | |
| 36 | +/// also valid. | |
| 37 | +#[diagnostic::on_unimplemented(message = "`{Self}` cannot be used to index `{T}`")] | |
| 38 | +#[doc(hidden)] | |
| 39 | +pub unsafe trait ProjectIndex<T: ?Sized>: Sized { | |
| 40 | + type Output: ?Sized; | |
| 41 | + | |
| 42 | + /// Returns an index-projected pointer, if in bounds. | |
| 43 | + fn get(self, slice: *mut T) -> Option<*mut Self::Output>; | |
| 44 | + | |
| 45 | + /// Returns an index-projected pointer; fail the build if it cannot be proved to be in bounds. | |
| 46 | + #[inline(always)] | |
| 47 | + fn index(self, slice: *mut T) -> *mut Self::Output { | |
| 48 | + Self::get(self, slice).unwrap_or_else(|| build_error!()) | |
| 49 | + } | |
| 50 | +} | |
| 51 | + | |
| 52 | +// Forward array impl to slice impl. | |
| 53 | +// | |
| 54 | +// SAFETY: Safety requirement guaranteed by the forwarded impl. | |
| 55 | +unsafe impl<T, I, const N: usize> ProjectIndex<[T; N]> for I | |
| 56 | +where | |
| 57 | + I: ProjectIndex<[T]>, | |
| 58 | +{ | |
| 59 | + type Output = <I as ProjectIndex<[T]>>::Output; | |
| 60 | + | |
| 61 | + #[inline(always)] | |
| 62 | + fn get(self, slice: *mut [T; N]) -> Option<*mut Self::Output> { | |
| 63 | + <I as ProjectIndex<[T]>>::get(self, slice) | |
| 64 | + } | |
| 65 | + | |
| 66 | + #[inline(always)] | |
| 67 | + fn index(self, slice: *mut [T; N]) -> *mut Self::Output { | |
| 68 | + <I as ProjectIndex<[T]>>::index(self, slice) | |
| 69 | + } | |
| 70 | +} | |
| 71 | + | |
| 72 | +// SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to | |
| 73 | +// not exceed the required bound. | |
| 74 | +unsafe impl<T> ProjectIndex<[T]> for usize { | |
| 75 | + type Output = T; | |
| 76 | + | |
| 77 | + #[inline(always)] | |
| 78 | + fn get(self, slice: *mut [T]) -> Option<*mut T> { | |
| 79 | + if self >= slice.len() { | |
| 80 | + None | |
| 81 | + } else { | |
| 82 | + Some(slice.cast::<T>().wrapping_add(self)) | |
| 83 | + } | |
| 84 | + } | |
| 85 | +} | |
| 86 | + | |
| 87 | +// SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to | |
| 88 | +// not exceed the required bound. | |
| 89 | +unsafe impl<T> ProjectIndex<[T]> for core::ops::Range<usize> { | |
| 90 | + type Output = [T]; | |
| 91 | + | |
| 92 | + #[inline(always)] | |
| 93 | + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { | |
| 94 | + let new_len = self.end.checked_sub(self.start)?; | |
| 95 | + if self.end > slice.len() { | |
| 96 | + return None; | |
| 97 | + } | |
| 98 | + Some(core::ptr::slice_from_raw_parts_mut( | |
| 99 | + slice.cast::<T>().wrapping_add(self.start), | |
| 100 | + new_len, | |
| 101 | + )) | |
| 102 | + } | |
| 103 | +} | |
| 104 | + | |
| 105 | +// SAFETY: Safety requirement guaranteed by the forwarded impl. | |
| 106 | +unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeTo<usize> { | |
| 107 | + type Output = [T]; | |
| 108 | + | |
| 109 | + #[inline(always)] | |
| 110 | + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { | |
| 111 | + (0..self.end).get(slice) | |
| 112 | + } | |
| 113 | +} | |
| 114 | + | |
| 115 | +// SAFETY: Safety requirement guaranteed by the forwarded impl. | |
| 116 | +unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeFrom<usize> { | |
| 117 | + type Output = [T]; | |
| 118 | + | |
| 119 | + #[inline(always)] | |
| 120 | + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { | |
| 121 | + (self.start..slice.len()).get(slice) | |
| 122 | + } | |
| 123 | +} | |
| 124 | + | |
| 125 | +// SAFETY: `get` returned the pointer as is, so it always has the same provenance and offset of 0. | |
| 126 | +unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeFull { | |
| 127 | + type Output = [T]; | |
| 128 | + | |
| 129 | + #[inline(always)] | |
| 130 | + fn get(self, slice: *mut [T]) -> Option<*mut [T]> { | |
| 131 | + Some(slice) | |
| 132 | + } | |
| 133 | +} | |
| 134 | + | |
| 135 | +/// A helper trait to perform field projection. | |
| 136 | +/// | |
| 137 | +/// This trait has a `DEREF` generic parameter so it can be implemented twice for types that | |
| 138 | +/// implement [`Deref`]. This will cause an ambiguity error and thus block [`Deref`] types being | |
| 139 | +/// used as base of projection, as they can inject unsoundness. Users therefore must not specify | |
| 140 | +/// `DEREF` and should always leave it to be inferred. | |
| 141 | +/// | |
| 142 | +/// # Safety | |
| 143 | +/// | |
| 144 | +/// `proj` may only invoke `f` with a valid allocation, as the documentation of [`Self::proj`] | |
| 145 | +/// describes. | |
| 146 | +#[doc(hidden)] | |
| 147 | +pub unsafe trait ProjectField<const DEREF: bool> { | |
| 148 | + /// Project a pointer to a type to a pointer of a field. | |
| 149 | + /// | |
| 150 | + /// `f` may only be invoked with a valid allocation so it can safely obtain raw pointers to | |
| 151 | + /// fields using `&raw mut`. | |
| 152 | + /// | |
| 153 | + /// This is needed because `base` might not point to a valid allocation, while `&raw mut` | |
| 154 | + /// requires pointers to be in bounds of a valid allocation. | |
| 155 | + /// | |
| 156 | + /// # Safety | |
| 157 | + /// | |
| 158 | + /// `f` must return a pointer in bounds of the provided pointer. | |
| 159 | + unsafe fn proj<F>(base: *mut Self, f: impl FnOnce(*mut Self) -> *mut F) -> *mut F; | |
| 160 | +} | |
| 161 | + | |
| 162 | +// NOTE: in theory, this API should work for `T: ?Sized` and `F: ?Sized`, too. However, we cannot | |
| 163 | +// currently support that as we need to obtain a valid allocation that `&raw const` can operate on. | |
| 164 | +// | |
| 165 | +// SAFETY: `proj` invokes `f` with valid allocation. | |
| 166 | +unsafe impl<T> ProjectField<false> for T { | |
| 167 | + #[inline(always)] | |
| 168 | + unsafe fn proj<F>(base: *mut Self, f: impl FnOnce(*mut Self) -> *mut F) -> *mut F { | |
| 169 | + // Create a valid allocation to start projection, as `base` is not necessarily so. The | |
| 170 | + // memory is never actually used so it will be optimized out, so it should work even for | |
| 171 | + // very large `T` (`memoffset` crate also relies on this). To be extra certain, we also | |
| 172 | + // annotate `f` closure with `#[inline(always)]` in the macro. | |
| 173 | + let mut place = MaybeUninit::uninit(); | |
| 174 | + let place_base = place.as_mut_ptr(); | |
| 175 | + let field = f(place_base); | |
| 176 | + // SAFETY: `field` is in bounds from `base` per safety requirement. | |
| 177 | + let offset = unsafe { field.byte_offset_from(place_base) }; | |
| 178 | + // Use `wrapping_byte_offset` as `base` does not need to be of valid allocation. | |
| 179 | + base.wrapping_byte_offset(offset).cast() | |
| 180 | + } | |
| 181 | +} | |
| 182 | + | |
| 183 | +// SAFETY: Vacuously satisfied. | |
| 184 | +unsafe impl<T: Deref> ProjectField<true> for T { | |
| 185 | + #[inline(always)] | |
| 186 | + unsafe fn proj<F>(_: *mut Self, _: impl FnOnce(*mut Self) -> *mut F) -> *mut F { | |
| 187 | + build_error!("this function is a guard against `Deref` impl and is never invoked"); | |
| 188 | + } | |
| 189 | +} | |
| 190 | + | |
| 191 | +/// Create a projection from a raw pointer. | |
| 192 | +/// | |
| 193 | +/// The projected pointer is within the memory region marked by the input pointer. There is no | |
| 194 | +/// requirement that the input raw pointer needs to be valid, so this macro may be used for | |
| 195 | +/// projecting pointers outside normal address space, e.g. I/O pointers. However, if the input | |
| 196 | +/// pointer is valid, the projected pointer is also valid. | |
| 197 | +/// | |
| 198 | +/// Supported projections include field projections and index projections. | |
| 199 | +/// It is not allowed to project into types that implement custom [`Deref`] or | |
| 200 | +/// [`Index`](core::ops::Index). | |
| 201 | +/// | |
| 202 | +/// The macro has basic syntax of `kernel::ptr::project!(ptr, projection)`, where `ptr` is an | |
| 203 | +/// expression that evaluates to a raw pointer which serves as the base of projection. `projection` | |
| 204 | +/// can be a projection expression of form `.field` (normally identifier, or numeral in case of | |
| 205 | +/// tuple structs) or of form `[index]`. | |
| 206 | +/// | |
| 207 | +/// If a mutable pointer is needed, the macro input can be prefixed with the `mut` keyword, i.e. | |
| 208 | +/// `kernel::ptr::project!(mut ptr, projection)`. By default, a const pointer is created. | |
| 209 | +/// | |
| 210 | +/// `ptr::project!` macro can perform both fallible indexing and build-time checked indexing. | |
| 211 | +/// `[index]` form performs build-time bounds checking; if compiler fails to prove `[index]` is in | |
| 212 | +/// bounds, compilation will fail. `[index]?` can be used to perform runtime bounds checking; | |
| 213 | +/// `OutOfBound` error is raised via `?` if the index is out of bounds. | |
| 214 | +/// | |
| 215 | +/// # Examples | |
| 216 | +/// | |
| 217 | +/// Field projections are performed with `.field_name`: | |
| 218 | +/// | |
| 219 | +/// ``` | |
| 220 | +/// struct MyStruct { field: u32, } | |
| 221 | +/// let ptr: *const MyStruct = core::ptr::dangling(); | |
| 222 | +/// let field_ptr: *const u32 = kernel::ptr::project!(ptr, .field); | |
| 223 | +/// | |
| 224 | +/// struct MyTupleStruct(u32, u32); | |
| 225 | +/// | |
| 226 | +/// fn proj(ptr: *const MyTupleStruct) { | |
| 227 | +/// let field_ptr: *const u32 = kernel::ptr::project!(ptr, .1); | |
| 228 | +/// } | |
| 229 | +/// ``` | |
| 230 | +/// | |
| 231 | +/// Index projections are performed with `[index]`: | |
| 232 | +/// | |
| 233 | +/// ``` | |
| 234 | +/// fn proj(ptr: *const [u8; 32]) -> Result { | |
| 235 | +/// let field_ptr: *const u8 = kernel::ptr::project!(ptr, [1]); | |
| 236 | +/// // The following invocation, if uncommented, would fail the build. | |
| 237 | +/// // | |
| 238 | +/// // kernel::ptr::project!(ptr, [128]); | |
| 239 | +/// | |
| 240 | +/// // This will raise an `OutOfBound` error (which is convertible to `ERANGE`). | |
| 241 | +/// kernel::ptr::project!(ptr, [128]?); | |
| 242 | +/// Ok(()) | |
| 243 | +/// } | |
| 244 | +/// ``` | |
| 245 | +/// | |
| 246 | +/// If you need to match on the error instead of propagate, put the invocation inside a closure: | |
| 247 | +/// | |
| 248 | +/// ``` | |
| 249 | +/// let ptr: *const [u8; 32] = core::ptr::dangling(); | |
| 250 | +/// let field_ptr: Result<*const u8> = (|| -> Result<_> { | |
| 251 | +/// Ok(kernel::ptr::project!(ptr, [128]?)) | |
| 252 | +/// })(); | |
| 253 | +/// assert!(field_ptr.is_err()); | |
| 254 | +/// ``` | |
| 255 | +/// | |
| 256 | +/// For mutable pointers, put `mut` as the first token in macro invocation. | |
| 257 | +/// | |
| 258 | +/// ``` | |
| 259 | +/// let ptr: *mut [(u8, u16); 32] = core::ptr::dangling_mut(); | |
| 260 | +/// let field_ptr: *mut u16 = kernel::ptr::project!(mut ptr, [1].1); | |
| 261 | +/// ``` | |
| 262 | +#[macro_export] | |
| 263 | +macro_rules! project_pointer { | |
| 264 | + (@gen $ptr:ident, ) => {}; | |
| 265 | + // Field projection. `$field` needs to be `tt` to support tuple index like `.0`. | |
| 266 | + (@gen $ptr:ident, .$field:tt $($rest:tt)*) => { | |
| 267 | + // SAFETY: The provided closure always returns an in-bounds pointer. | |
| 268 | + let $ptr = unsafe { | |
| 269 | + $crate::ptr::projection::ProjectField::proj($ptr, #[inline(always)] |ptr| { | |
| 270 | + // Check unaligned field. Not all users (e.g. DMA) can handle unaligned | |
| 271 | + // projections. | |
| 272 | + if false { | |
| 273 | + let _ = &(*ptr).$field; | |
| 274 | + } | |
| 275 | + // SAFETY: `$field` is in bounds, and no implicit `Deref` is possible (if the | |
| 276 | + // type implements `Deref`, Rust cannot infer the generic parameter `DEREF`). | |
| 277 | + &raw mut (*ptr).$field | |
| 278 | + }) | |
| 279 | + }; | |
| 280 | + $crate::ptr::project!(@gen $ptr, $($rest)*) | |
| 281 | + }; | |
| 282 | + // Fallible index projection. | |
| 283 | + (@gen $ptr:ident, [$index:expr]? $($rest:tt)*) => { | |
| 284 | + let $ptr = $crate::ptr::projection::ProjectIndex::get($index, $ptr) | |
| 285 | + .ok_or($crate::ptr::projection::OutOfBound)?; | |
| 286 | + $crate::ptr::project!(@gen $ptr, $($rest)*) | |
| 287 | + }; | |
| 288 | + // Build-time checked index projection. | |
| 289 | + (@gen $ptr:ident, [$index:expr] $($rest:tt)*) => { | |
| 290 | + let $ptr = $crate::ptr::projection::ProjectIndex::index($index, $ptr); | |
| 291 | + $crate::ptr::project!(@gen $ptr, $($rest)*) | |
| 292 | + }; | |
| 293 | + (mut $ptr:expr, $($proj:tt)*) => {{ | |
| 294 | + let ptr: *mut _ = $ptr; | |
| 295 | + $crate::ptr::project!(@gen ptr, $($proj)*); | |
| 296 | + ptr | |
| 297 | + }}; | |
| 298 | + ($ptr:expr, $($proj:tt)*) => {{ | |
| 299 | + let ptr = <*const _>::cast_mut($ptr); | |
| 300 | + // We currently always project using mutable pointer, as it is not decided whether `&raw | |
| 301 | + // const` allows the resulting pointer to be mutated (see documentation of `addr_of!`). | |
| 302 | + $crate::ptr::project!(@gen ptr, $($proj)*); | |
| 303 | + ptr.cast_const() | |
| 304 | + }}; | |
| 305 | +} | |
| @@ -68,7 +68,7 @@ impl pci::Driver for DmaSampleDriver { · 67 unmodified lines | ||
| 68 | 68 | CoherentAllocation::alloc_coherent(pdev.as_ref(), TEST_VALUES.len(), GFP_KERNEL)?; |
| 69 | 69 | |
| 70 | 70 | for (i, value) in TEST_VALUES.into_iter().enumerate() { |
| 71 | − kernel::dma_write!(ca[i] = MyStruct::new(value.0, value.1))?; | |
| 71 | + kernel::dma_write!(ca, [i]?, MyStruct::new(value.0, value.1)); | |
| 72 | 72 | } |
| 73 | 73 | |
| 74 | 74 | let size = 4 * page::PAGE_SIZE; |
| @@ -85,24 +85,26 @@ impl pci::Driver for DmaSampleDriver { · 10 unmodified lines | ||
| 85 | 85 | } |
| 86 | 86 | } |
| 87 | 87 | |
| 88 | +impl DmaSampleDriver { | |
| 89 | + fn check_dma(&self) -> Result { | |
| 90 | + for (i, value) in TEST_VALUES.into_iter().enumerate() { | |
| 91 | + let val0 = kernel::dma_read!(self.ca, [i]?.h); | |
| 92 | + let val1 = kernel::dma_read!(self.ca, [i]?.b); | |
| 93 | + | |
| 94 | + assert_eq!(val0, value.0); | |
| 95 | + assert_eq!(val1, value.1); | |
| 96 | + } | |
| 97 | + | |
| 98 | + Ok(()) | |
| 99 | + } | |
| 100 | +} | |
| 101 | + | |
| 88 | 102 | #[pinned_drop] |
| 89 | 103 | impl PinnedDrop for DmaSampleDriver { |
| 90 | 104 | fn drop(self: Pin<&mut Self>) { |
| 91 | 105 | dev_info!(self.pdev, "Unload DMA test driver.\n"); |
| 92 | 106 | |
| 93 | − for (i, value) in TEST_VALUES.into_iter().enumerate() { | |
| 94 | − let val0 = kernel::dma_read!(self.ca[i].h); | |
| 95 | − let val1 = kernel::dma_read!(self.ca[i].b); | |
| 96 | − assert!(val0.is_ok()); | |
| 97 | − assert!(val1.is_ok()); | |
| 98 | − | |
| 99 | − if let Ok(val0) = val0 { | |
| 100 | − assert_eq!(val0, value.0); | |
| 101 | − } | |
| 102 | − if let Ok(val1) = val1 { | |
| 103 | − assert_eq!(val1, value.1); | |
| 104 | − } | |
| 105 | − } | |
| 107 | + assert!(self.check_dma().is_ok()); | |
| 106 | 108 | |
| 107 | 109 | for (i, entry) in self.sgt.iter().enumerate() { |
| 108 | 110 | dev_info!( |
| @@ -310,16 +310,18 @@ $(obj)/%.lst: $(obj)/%.c FORCE · 309 unmodified lines | ||
| 310 | 310 | |
| 311 | 311 | # The features in this list are the ones allowed for non-`rust/` code. |
| 312 | 312 | # |
| 313 | +# - Stable since Rust 1.79.0: `feature(slice_ptr_len)`. | |
| 313 | 314 | # - Stable since Rust 1.81.0: `feature(lint_reasons)`. |
| 314 | 315 | # - Stable since Rust 1.82.0: `feature(asm_const)`, |
| 315 | 316 | # `feature(offset_of_nested)`, `feature(raw_ref_op)`. |
| 317 | +# - Stable since Rust 1.84.0: `feature(strict_provenance)`. | |
| 316 | 318 | # - Stable since Rust 1.87.0: `feature(asm_goto)`. |
| 317 | 319 | # - Expected to become stable: `feature(arbitrary_self_types)`. |
| 318 | 320 | # - To be determined: `feature(used_with_arg)`. |
| 319 | 321 | # |
| 320 | 322 | # Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on |
| 321 | 323 | # the unstable features in use. |
| 322 | −rust_allowed_features := asm_const,asm_goto,arbitrary_self_types,lint_reasons,offset_of_nested,raw_ref_op,used_with_arg | |
| 324 | +rust_allowed_features := asm_const,asm_goto,arbitrary_self_types,lint_reasons,offset_of_nested,raw_ref_op,slice_ptr_len,strict_provenance,used_with_arg | |
| 323 | 325 | |
| 324 | 326 | # `--out-dir` is required to avoid temporaries being created by `rustc` in the |
| 325 | 327 | # current working directory, which may be not accessible in the out-of-tree |