| @@ -0,0 +1,314 @@ |
| | 1 | // SPDX-License-Identifier: GPL-2.0 |
| | 2 | |
| | 3 | // Copyright (C) 2025 Google LLC. |
| | 4 | |
| | 5 | //! IO vectors. |
| | 6 | //! |
| | 7 | //! C headers: [`include/linux/iov_iter.h`](srctree/include/linux/iov_iter.h), |
| | 8 | //! [`include/linux/uio.h`](srctree/include/linux/uio.h) |
| | 9 | |
| | 10 | use crate::{ |
| | 11 | alloc::{Allocator, Flags}, |
| | 12 | bindings, |
| | 13 | prelude::*, |
| | 14 | types::Opaque, |
| | 15 | }; |
| | 16 | use core::{marker::PhantomData, mem::MaybeUninit, ptr, slice}; |
| | 17 | |
| | 18 | const ITER_SOURCE: bool = bindings::ITER_SOURCE != 0; |
| | 19 | const ITER_DEST: bool = bindings::ITER_DEST != 0; |
| | 20 | |
| | 21 | // Compile-time assertion for the above constants. |
| | 22 | const _: () = { |
| | 23 | build_assert!( |
| | 24 | ITER_SOURCE != ITER_DEST, |
| | 25 | "ITER_DEST and ITER_SOURCE should be different." |
| | 26 | ); |
| | 27 | }; |
| | 28 | |
| | 29 | /// An IO vector that acts as a source of data. |
| | 30 | /// |
| | 31 | /// The data may come from many different sources. This includes both things in kernel-space and |
| | 32 | /// reading from userspace. It's not necessarily the case that the data source is immutable, so |
| | 33 | /// rewinding the IO vector to read the same data twice is not guaranteed to result in the same |
| | 34 | /// bytes. It's also possible that the data source is mapped in a thread-local manner using e.g. |
| | 35 | /// `kmap_local_page()`, so this type is not `Send` to ensure that the mapping is read from the |
| | 36 | /// right context in that scenario. |
| | 37 | /// |
| | 38 | /// # Invariants |
| | 39 | /// |
| | 40 | /// Must hold a valid `struct iov_iter` with `data_source` set to `ITER_SOURCE`. For the duration |
| | 41 | /// of `'data`, it must be safe to read from this IO vector using the standard C methods for this |
| | 42 | /// purpose. |
| | 43 | #[repr(transparent)] |
| | 44 | pub struct IovIterSource<'data> { |
| | 45 | iov: Opaque<bindings::iov_iter>, |
| | 46 | /// Represent to the type system that this value contains a pointer to readable data it does |
| | 47 | /// not own. |
| | 48 | _source: PhantomData<&'data [u8]>, |
| | 49 | } |
| | 50 | |
| | 51 | impl<'data> IovIterSource<'data> { |
| | 52 | /// Obtain an `IovIterSource` from a raw pointer. |
| | 53 | /// |
| | 54 | /// # Safety |
| | 55 | /// |
| | 56 | /// * The referenced `struct iov_iter` must be valid and must only be accessed through the |
| | 57 | /// returned reference for the duration of `'iov`. |
| | 58 | /// * The referenced `struct iov_iter` must have `data_source` set to `ITER_SOURCE`. |
| | 59 | /// * For the duration of `'data`, it must be safe to read from this IO vector using the |
| | 60 | /// standard C methods for this purpose. |
| | 61 | #[track_caller] |
| | 62 | #[inline] |
| | 63 | pub unsafe fn from_raw<'iov>(ptr: *mut bindings::iov_iter) -> &'iov mut IovIterSource<'data> { |
| | 64 | // SAFETY: The caller ensures that `ptr` is valid. |
| | 65 | let data_source = unsafe { (*ptr).data_source }; |
| | 66 | assert_eq!(data_source, ITER_SOURCE); |
| | 67 | |
| | 68 | // SAFETY: The caller ensures the type invariants for the right durations, and |
| | 69 | // `IovIterSource` is layout compatible with `struct iov_iter`. |
| | 70 | unsafe { &mut *ptr.cast::<IovIterSource<'data>>() } |
| | 71 | } |
| | 72 | |
| | 73 | /// Access this as a raw `struct iov_iter`. |
| | 74 | #[inline] |
| | 75 | pub fn as_raw(&mut self) -> *mut bindings::iov_iter { |
| | 76 | self.iov.get() |
| | 77 | } |
| | 78 | |
| | 79 | /// Returns the number of bytes available in this IO vector. |
| | 80 | /// |
| | 81 | /// Note that this may overestimate the number of bytes. For example, reading from userspace |
| | 82 | /// memory could fail with `EFAULT`, which will be treated as the end of the IO vector. |
| | 83 | #[inline] |
| | 84 | pub fn len(&self) -> usize { |
| | 85 | // SAFETY: We have shared access to this IO vector, so we can read its `count` field. |
| | 86 | unsafe { |
| | 87 | (*self.iov.get()) |
| | 88 | .__bindgen_anon_1 |
| | 89 | .__bindgen_anon_1 |
| | 90 | .as_ref() |
| | 91 | .count |
| | 92 | } |
| | 93 | } |
| | 94 | |
| | 95 | /// Returns whether there are any bytes left in this IO vector. |
| | 96 | /// |
| | 97 | /// This may return `true` even if there are no more bytes available. For example, reading from |
| | 98 | /// userspace memory could fail with `EFAULT`, which will be treated as the end of the IO vector. |
| | 99 | #[inline] |
| | 100 | pub fn is_empty(&self) -> bool { |
| | 101 | self.len() == 0 |
| | 102 | } |
| | 103 | |
| | 104 | /// Advance this IO vector by `bytes` bytes. |
| | 105 | /// |
| | 106 | /// If `bytes` is larger than the size of this IO vector, it is advanced to the end. |
| | 107 | #[inline] |
| | 108 | pub fn advance(&mut self, bytes: usize) { |
| | 109 | // SAFETY: By the type invariants, `self.iov` is a valid IO vector. |
| | 110 | unsafe { bindings::iov_iter_advance(self.as_raw(), bytes) }; |
| | 111 | } |
| | 112 | |
| | 113 | /// Advance this IO vector backwards by `bytes` bytes. |
| | 114 | /// |
| | 115 | /// # Safety |
| | 116 | /// |
| | 117 | /// The IO vector must not be reverted to before its beginning. |
| | 118 | #[inline] |
| | 119 | pub unsafe fn revert(&mut self, bytes: usize) { |
| | 120 | // SAFETY: By the type invariants, `self.iov` is a valid IO vector, and the caller |
| | 121 | // ensures that `bytes` is in bounds. |
| | 122 | unsafe { bindings::iov_iter_revert(self.as_raw(), bytes) }; |
| | 123 | } |
| | 124 | |
| | 125 | /// Read data from this IO vector. |
| | 126 | /// |
| | 127 | /// Returns the number of bytes that have been copied. |
| | 128 | #[inline] |
| | 129 | pub fn copy_from_iter(&mut self, out: &mut [u8]) -> usize { |
| | 130 | // SAFETY: `Self::copy_from_iter_raw` guarantees that it will not write any uninitialized |
| | 131 | // bytes in the provided buffer, so `out` is still a valid `u8` slice after this call. |
| | 132 | let out = unsafe { &mut *(ptr::from_mut(out) as *mut [MaybeUninit<u8>]) }; |
| | 133 | |
| | 134 | self.copy_from_iter_raw(out).len() |
| | 135 | } |
| | 136 | |
| | 137 | /// Read data from this IO vector and append it to a vector. |
| | 138 | /// |
| | 139 | /// Returns the number of bytes that have been copied. |
| | 140 | #[inline] |
| | 141 | pub fn copy_from_iter_vec<A: Allocator>( |
| | 142 | &mut self, |
| | 143 | out: &mut Vec<u8, A>, |
| | 144 | flags: Flags, |
| | 145 | ) -> Result<usize> { |
| | 146 | out.reserve(self.len(), flags)?; |
| | 147 | let len = self.copy_from_iter_raw(out.spare_capacity_mut()).len(); |
| | 148 | // SAFETY: |
| | 149 | // - `len` is the length of a subslice of the spare capacity, so `len` is at most the |
| | 150 | // length of the spare capacity. |
| | 151 | // - `Self::copy_from_iter_raw` guarantees that the first `len` bytes of the spare capacity |
| | 152 | // have been initialized. |
| | 153 | unsafe { out.inc_len(len) }; |
| | 154 | Ok(len) |
| | 155 | } |
| | 156 | |
| | 157 | /// Read data from this IO vector into potentially uninitialized memory. |
| | 158 | /// |
| | 159 | /// Returns the sub-slice of the output that has been initialized. If the returned slice is |
| | 160 | /// shorter than the input buffer, then the entire IO vector has been read. |
| | 161 | /// |
| | 162 | /// This will never write uninitialized bytes to the provided buffer. |
| | 163 | #[inline] |
| | 164 | pub fn copy_from_iter_raw(&mut self, out: &mut [MaybeUninit<u8>]) -> &mut [u8] { |
| | 165 | let capacity = out.len(); |
| | 166 | let out = out.as_mut_ptr().cast::<u8>(); |
| | 167 | |
| | 168 | // GUARANTEES: The C API guarantees that it does not write uninitialized bytes to the |
| | 169 | // provided buffer. |
| | 170 | // SAFETY: |
| | 171 | // * By the type invariants, it is still valid to read from this IO vector. |
| | 172 | // * `out` is valid for writing for `capacity` bytes because it comes from a slice of |
| | 173 | // that length. |
| | 174 | let len = unsafe { bindings::_copy_from_iter(out.cast(), capacity, self.as_raw()) }; |
| | 175 | |
| | 176 | // SAFETY: The underlying C api guarantees that initialized bytes have been written to the |
| | 177 | // first `len` bytes of the spare capacity. |
| | 178 | unsafe { slice::from_raw_parts_mut(out, len) } |
| | 179 | } |
| | 180 | } |
| | 181 | |
| | 182 | /// An IO vector that acts as a destination for data. |
| | 183 | /// |
| | 184 | /// IO vectors support many different types of destinations. This includes both buffers in |
| | 185 | /// kernel-space and writing to userspace. It's possible that the destination buffer is mapped in a |
| | 186 | /// thread-local manner using e.g. `kmap_local_page()`, so this type is not `Send` to ensure that |
| | 187 | /// the mapping is written to the right context in that scenario. |
| | 188 | /// |
| | 189 | /// # Invariants |
| | 190 | /// |
| | 191 | /// Must hold a valid `struct iov_iter` with `data_source` set to `ITER_DEST`. For the duration of |
| | 192 | /// `'data`, it must be safe to write to this IO vector using the standard C methods for this |
| | 193 | /// purpose. |
| | 194 | #[repr(transparent)] |
| | 195 | pub struct IovIterDest<'data> { |
| | 196 | iov: Opaque<bindings::iov_iter>, |
| | 197 | /// Represent to the type system that this value contains a pointer to writable data it does |
| | 198 | /// not own. |
| | 199 | _source: PhantomData<&'data mut [u8]>, |
| | 200 | } |
| | 201 | |
| | 202 | impl<'data> IovIterDest<'data> { |
| | 203 | /// Obtain an `IovIterDest` from a raw pointer. |
| | 204 | /// |
| | 205 | /// # Safety |
| | 206 | /// |
| | 207 | /// * The referenced `struct iov_iter` must be valid and must only be accessed through the |
| | 208 | /// returned reference for the duration of `'iov`. |
| | 209 | /// * The referenced `struct iov_iter` must have `data_source` set to `ITER_DEST`. |
| | 210 | /// * For the duration of `'data`, it must be safe to write to this IO vector using the |
| | 211 | /// standard C methods for this purpose. |
| | 212 | #[track_caller] |
| | 213 | #[inline] |
| | 214 | pub unsafe fn from_raw<'iov>(ptr: *mut bindings::iov_iter) -> &'iov mut IovIterDest<'data> { |
| | 215 | // SAFETY: The caller ensures that `ptr` is valid. |
| | 216 | let data_source = unsafe { (*ptr).data_source }; |
| | 217 | assert_eq!(data_source, ITER_DEST); |
| | 218 | |
| | 219 | // SAFETY: The caller ensures the type invariants for the right durations, and |
| | 220 | // `IovIterSource` is layout compatible with `struct iov_iter`. |
| | 221 | unsafe { &mut *ptr.cast::<IovIterDest<'data>>() } |
| | 222 | } |
| | 223 | |
| | 224 | /// Access this as a raw `struct iov_iter`. |
| | 225 | #[inline] |
| | 226 | pub fn as_raw(&mut self) -> *mut bindings::iov_iter { |
| | 227 | self.iov.get() |
| | 228 | } |
| | 229 | |
| | 230 | /// Returns the number of bytes available in this IO vector. |
| | 231 | /// |
| | 232 | /// Note that this may overestimate the number of bytes. For example, reading from userspace |
| | 233 | /// memory could fail with EFAULT, which will be treated as the end of the IO vector. |
| | 234 | #[inline] |
| | 235 | pub fn len(&self) -> usize { |
| | 236 | // SAFETY: We have shared access to this IO vector, so we can read its `count` field. |
| | 237 | unsafe { |
| | 238 | (*self.iov.get()) |
| | 239 | .__bindgen_anon_1 |
| | 240 | .__bindgen_anon_1 |
| | 241 | .as_ref() |
| | 242 | .count |
| | 243 | } |
| | 244 | } |
| | 245 | |
| | 246 | /// Returns whether there are any bytes left in this IO vector. |
| | 247 | /// |
| | 248 | /// This may return `true` even if there are no more bytes available. For example, reading from |
| | 249 | /// userspace memory could fail with EFAULT, which will be treated as the end of the IO vector. |
| | 250 | #[inline] |
| | 251 | pub fn is_empty(&self) -> bool { |
| | 252 | self.len() == 0 |
| | 253 | } |
| | 254 | |
| | 255 | /// Advance this IO vector by `bytes` bytes. |
| | 256 | /// |
| | 257 | /// If `bytes` is larger than the size of this IO vector, it is advanced to the end. |
| | 258 | #[inline] |
| | 259 | pub fn advance(&mut self, bytes: usize) { |
| | 260 | // SAFETY: By the type invariants, `self.iov` is a valid IO vector. |
| | 261 | unsafe { bindings::iov_iter_advance(self.as_raw(), bytes) }; |
| | 262 | } |
| | 263 | |
| | 264 | /// Advance this IO vector backwards by `bytes` bytes. |
| | 265 | /// |
| | 266 | /// # Safety |
| | 267 | /// |
| | 268 | /// The IO vector must not be reverted to before its beginning. |
| | 269 | #[inline] |
| | 270 | pub unsafe fn revert(&mut self, bytes: usize) { |
| | 271 | // SAFETY: By the type invariants, `self.iov` is a valid IO vector, and the caller |
| | 272 | // ensures that `bytes` is in bounds. |
| | 273 | unsafe { bindings::iov_iter_revert(self.as_raw(), bytes) }; |
| | 274 | } |
| | 275 | |
| | 276 | /// Write data to this IO vector. |
| | 277 | /// |
| | 278 | /// Returns the number of bytes that were written. If this is shorter than the provided slice, |
| | 279 | /// then no more bytes can be written. |
| | 280 | #[inline] |
| | 281 | pub fn copy_to_iter(&mut self, input: &[u8]) -> usize { |
| | 282 | // SAFETY: |
| | 283 | // * By the type invariants, it is still valid to write to this IO vector. |
| | 284 | // * `input` is valid for `input.len()` bytes. |
| | 285 | unsafe { bindings::_copy_to_iter(input.as_ptr().cast(), input.len(), self.as_raw()) } |
| | 286 | } |
| | 287 | |
| | 288 | /// Utility for implementing `read_iter` given the full contents of the file. |
| | 289 | /// |
| | 290 | /// The full contents of the file being read from is represented by `contents`. This call will |
| | 291 | /// write the appropriate sub-slice of `contents` and update the file position in `ppos` so |
| | 292 | /// that the file will appear to contain `contents` even if takes multiple reads to read the |
| | 293 | /// entire file. |
| | 294 | #[inline] |
| | 295 | pub fn simple_read_from_buffer(&mut self, ppos: &mut i64, contents: &[u8]) -> Result<usize> { |
| | 296 | if *ppos < 0 { |
| | 297 | return Err(EINVAL); |
| | 298 | } |
| | 299 | let Ok(pos) = usize::try_from(*ppos) else { |
| | 300 | return Ok(0); |
| | 301 | }; |
| | 302 | if pos >= contents.len() { |
| | 303 | return Ok(0); |
| | 304 | } |
| | 305 | |
| | 306 | // BOUNDS: We just checked that `pos < contents.len()` above. |
| | 307 | let num_written = self.copy_to_iter(&contents[pos..]); |
| | 308 | |
| | 309 | // OVERFLOW: `pos+num_written <= contents.len() <= isize::MAX <= i64::MAX`. |
| | 310 | *ppos = (pos + num_written) as i64; |
| | 311 | |
| | 312 | Ok(num_written) |
| | 313 | } |
| | 314 | } |