| @@ -0,0 +1,314 @@ |
| 1 | +// SPDX-License-Identifier: GPL-2.0 |
| 2 | + |
| 3 | +// Copyright (C) 2025 Google LLC. |
| 4 | + |
| 5 | +//! IO vectors. |
| 6 | +//! |
| 7 | +//! C headers: [`include/linux/iov_iter.h`](srctree/include/linux/iov_iter.h), |
| 8 | +//! [`include/linux/uio.h`](srctree/include/linux/uio.h) |
| 9 | + |
| 10 | +use crate::{ |
| 11 | + alloc::{Allocator, Flags}, |
| 12 | + bindings, |
| 13 | + prelude::*, |
| 14 | + types::Opaque, |
| 15 | +}; |
| 16 | +use core::{marker::PhantomData, mem::MaybeUninit, ptr, slice}; |
| 17 | + |
| 18 | +const ITER_SOURCE: bool = bindings::ITER_SOURCE != 0; |
| 19 | +const ITER_DEST: bool = bindings::ITER_DEST != 0; |
| 20 | + |
| 21 | +// Compile-time assertion for the above constants. |
| 22 | +const _: () = { |
| 23 | + build_assert!( |
| 24 | + ITER_SOURCE != ITER_DEST, |
| 25 | + "ITER_DEST and ITER_SOURCE should be different." |
| 26 | + ); |
| 27 | +}; |
| 28 | + |
| 29 | +/// An IO vector that acts as a source of data. |
| 30 | +/// |
| 31 | +/// The data may come from many different sources. This includes both things in kernel-space and |
| 32 | +/// reading from userspace. It's not necessarily the case that the data source is immutable, so |
| 33 | +/// rewinding the IO vector to read the same data twice is not guaranteed to result in the same |
| 34 | +/// bytes. It's also possible that the data source is mapped in a thread-local manner using e.g. |
| 35 | +/// `kmap_local_page()`, so this type is not `Send` to ensure that the mapping is read from the |
| 36 | +/// right context in that scenario. |
| 37 | +/// |
| 38 | +/// # Invariants |
| 39 | +/// |
| 40 | +/// Must hold a valid `struct iov_iter` with `data_source` set to `ITER_SOURCE`. For the duration |
| 41 | +/// of `'data`, it must be safe to read from this IO vector using the standard C methods for this |
| 42 | +/// purpose. |
| 43 | +#[repr(transparent)] |
| 44 | +pub struct IovIterSource<'data> { |
| 45 | + iov: Opaque<bindings::iov_iter>, |
| 46 | + /// Represent to the type system that this value contains a pointer to readable data it does |
| 47 | + /// not own. |
| 48 | + _source: PhantomData<&'data [u8]>, |
| 49 | +} |
| 50 | + |
| 51 | +impl<'data> IovIterSource<'data> { |
| 52 | + /// Obtain an `IovIterSource` from a raw pointer. |
| 53 | + /// |
| 54 | + /// # Safety |
| 55 | + /// |
| 56 | + /// * The referenced `struct iov_iter` must be valid and must only be accessed through the |
| 57 | + /// returned reference for the duration of `'iov`. |
| 58 | + /// * The referenced `struct iov_iter` must have `data_source` set to `ITER_SOURCE`. |
| 59 | + /// * For the duration of `'data`, it must be safe to read from this IO vector using the |
| 60 | + /// standard C methods for this purpose. |
| 61 | + #[track_caller] |
| 62 | + #[inline] |
| 63 | + pub unsafe fn from_raw<'iov>(ptr: *mut bindings::iov_iter) -> &'iov mut IovIterSource<'data> { |
| 64 | + // SAFETY: The caller ensures that `ptr` is valid. |
| 65 | + let data_source = unsafe { (*ptr).data_source }; |
| 66 | + assert_eq!(data_source, ITER_SOURCE); |
| 67 | + |
| 68 | + // SAFETY: The caller ensures the type invariants for the right durations, and |
| 69 | + // `IovIterSource` is layout compatible with `struct iov_iter`. |
| 70 | + unsafe { &mut *ptr.cast::<IovIterSource<'data>>() } |
| 71 | + } |
| 72 | + |
| 73 | + /// Access this as a raw `struct iov_iter`. |
| 74 | + #[inline] |
| 75 | + pub fn as_raw(&mut self) -> *mut bindings::iov_iter { |
| 76 | + self.iov.get() |
| 77 | + } |
| 78 | + |
| 79 | + /// Returns the number of bytes available in this IO vector. |
| 80 | + /// |
| 81 | + /// Note that this may overestimate the number of bytes. For example, reading from userspace |
| 82 | + /// memory could fail with `EFAULT`, which will be treated as the end of the IO vector. |
| 83 | + #[inline] |
| 84 | + pub fn len(&self) -> usize { |
| 85 | + // SAFETY: We have shared access to this IO vector, so we can read its `count` field. |
| 86 | + unsafe { |
| 87 | + (*self.iov.get()) |
| 88 | + .__bindgen_anon_1 |
| 89 | + .__bindgen_anon_1 |
| 90 | + .as_ref() |
| 91 | + .count |
| 92 | + } |
| 93 | + } |
| 94 | + |
| 95 | + /// Returns whether there are any bytes left in this IO vector. |
| 96 | + /// |
| 97 | + /// This may return `true` even if there are no more bytes available. For example, reading from |
| 98 | + /// userspace memory could fail with `EFAULT`, which will be treated as the end of the IO vector. |
| 99 | + #[inline] |
| 100 | + pub fn is_empty(&self) -> bool { |
| 101 | + self.len() == 0 |
| 102 | + } |
| 103 | + |
| 104 | + /// Advance this IO vector by `bytes` bytes. |
| 105 | + /// |
| 106 | + /// If `bytes` is larger than the size of this IO vector, it is advanced to the end. |
| 107 | + #[inline] |
| 108 | + pub fn advance(&mut self, bytes: usize) { |
| 109 | + // SAFETY: By the type invariants, `self.iov` is a valid IO vector. |
| 110 | + unsafe { bindings::iov_iter_advance(self.as_raw(), bytes) }; |
| 111 | + } |
| 112 | + |
| 113 | + /// Advance this IO vector backwards by `bytes` bytes. |
| 114 | + /// |
| 115 | + /// # Safety |
| 116 | + /// |
| 117 | + /// The IO vector must not be reverted to before its beginning. |
| 118 | + #[inline] |
| 119 | + pub unsafe fn revert(&mut self, bytes: usize) { |
| 120 | + // SAFETY: By the type invariants, `self.iov` is a valid IO vector, and the caller |
| 121 | + // ensures that `bytes` is in bounds. |
| 122 | + unsafe { bindings::iov_iter_revert(self.as_raw(), bytes) }; |
| 123 | + } |
| 124 | + |
| 125 | + /// Read data from this IO vector. |
| 126 | + /// |
| 127 | + /// Returns the number of bytes that have been copied. |
| 128 | + #[inline] |
| 129 | + pub fn copy_from_iter(&mut self, out: &mut [u8]) -> usize { |
| 130 | + // SAFETY: `Self::copy_from_iter_raw` guarantees that it will not write any uninitialized |
| 131 | + // bytes in the provided buffer, so `out` is still a valid `u8` slice after this call. |
| 132 | + let out = unsafe { &mut *(ptr::from_mut(out) as *mut [MaybeUninit<u8>]) }; |
| 133 | + |
| 134 | + self.copy_from_iter_raw(out).len() |
| 135 | + } |
| 136 | + |
| 137 | + /// Read data from this IO vector and append it to a vector. |
| 138 | + /// |
| 139 | + /// Returns the number of bytes that have been copied. |
| 140 | + #[inline] |
| 141 | + pub fn copy_from_iter_vec<A: Allocator>( |
| 142 | + &mut self, |
| 143 | + out: &mut Vec<u8, A>, |
| 144 | + flags: Flags, |
| 145 | + ) -> Result<usize> { |
| 146 | + out.reserve(self.len(), flags)?; |
| 147 | + let len = self.copy_from_iter_raw(out.spare_capacity_mut()).len(); |
| 148 | + // SAFETY: |
| 149 | + // - `len` is the length of a subslice of the spare capacity, so `len` is at most the |
| 150 | + // length of the spare capacity. |
| 151 | + // - `Self::copy_from_iter_raw` guarantees that the first `len` bytes of the spare capacity |
| 152 | + // have been initialized. |
| 153 | + unsafe { out.inc_len(len) }; |
| 154 | + Ok(len) |
| 155 | + } |
| 156 | + |
| 157 | + /// Read data from this IO vector into potentially uninitialized memory. |
| 158 | + /// |
| 159 | + /// Returns the sub-slice of the output that has been initialized. If the returned slice is |
| 160 | + /// shorter than the input buffer, then the entire IO vector has been read. |
| 161 | + /// |
| 162 | + /// This will never write uninitialized bytes to the provided buffer. |
| 163 | + #[inline] |
| 164 | + pub fn copy_from_iter_raw(&mut self, out: &mut [MaybeUninit<u8>]) -> &mut [u8] { |
| 165 | + let capacity = out.len(); |
| 166 | + let out = out.as_mut_ptr().cast::<u8>(); |
| 167 | + |
| 168 | + // GUARANTEES: The C API guarantees that it does not write uninitialized bytes to the |
| 169 | + // provided buffer. |
| 170 | + // SAFETY: |
| 171 | + // * By the type invariants, it is still valid to read from this IO vector. |
| 172 | + // * `out` is valid for writing for `capacity` bytes because it comes from a slice of |
| 173 | + // that length. |
| 174 | + let len = unsafe { bindings::_copy_from_iter(out.cast(), capacity, self.as_raw()) }; |
| 175 | + |
| 176 | + // SAFETY: The underlying C api guarantees that initialized bytes have been written to the |
| 177 | + // first `len` bytes of the spare capacity. |
| 178 | + unsafe { slice::from_raw_parts_mut(out, len) } |
| 179 | + } |
| 180 | +} |
| 181 | + |
| 182 | +/// An IO vector that acts as a destination for data. |
| 183 | +/// |
| 184 | +/// IO vectors support many different types of destinations. This includes both buffers in |
| 185 | +/// kernel-space and writing to userspace. It's possible that the destination buffer is mapped in a |
| 186 | +/// thread-local manner using e.g. `kmap_local_page()`, so this type is not `Send` to ensure that |
| 187 | +/// the mapping is written to the right context in that scenario. |
| 188 | +/// |
| 189 | +/// # Invariants |
| 190 | +/// |
| 191 | +/// Must hold a valid `struct iov_iter` with `data_source` set to `ITER_DEST`. For the duration of |
| 192 | +/// `'data`, it must be safe to write to this IO vector using the standard C methods for this |
| 193 | +/// purpose. |
| 194 | +#[repr(transparent)] |
| 195 | +pub struct IovIterDest<'data> { |
| 196 | + iov: Opaque<bindings::iov_iter>, |
| 197 | + /// Represent to the type system that this value contains a pointer to writable data it does |
| 198 | + /// not own. |
| 199 | + _source: PhantomData<&'data mut [u8]>, |
| 200 | +} |
| 201 | + |
| 202 | +impl<'data> IovIterDest<'data> { |
| 203 | + /// Obtain an `IovIterDest` from a raw pointer. |
| 204 | + /// |
| 205 | + /// # Safety |
| 206 | + /// |
| 207 | + /// * The referenced `struct iov_iter` must be valid and must only be accessed through the |
| 208 | + /// returned reference for the duration of `'iov`. |
| 209 | + /// * The referenced `struct iov_iter` must have `data_source` set to `ITER_DEST`. |
| 210 | + /// * For the duration of `'data`, it must be safe to write to this IO vector using the |
| 211 | + /// standard C methods for this purpose. |
| 212 | + #[track_caller] |
| 213 | + #[inline] |
| 214 | + pub unsafe fn from_raw<'iov>(ptr: *mut bindings::iov_iter) -> &'iov mut IovIterDest<'data> { |
| 215 | + // SAFETY: The caller ensures that `ptr` is valid. |
| 216 | + let data_source = unsafe { (*ptr).data_source }; |
| 217 | + assert_eq!(data_source, ITER_DEST); |
| 218 | + |
| 219 | + // SAFETY: The caller ensures the type invariants for the right durations, and |
| 220 | + // `IovIterSource` is layout compatible with `struct iov_iter`. |
| 221 | + unsafe { &mut *ptr.cast::<IovIterDest<'data>>() } |
| 222 | + } |
| 223 | + |
| 224 | + /// Access this as a raw `struct iov_iter`. |
| 225 | + #[inline] |
| 226 | + pub fn as_raw(&mut self) -> *mut bindings::iov_iter { |
| 227 | + self.iov.get() |
| 228 | + } |
| 229 | + |
| 230 | + /// Returns the number of bytes available in this IO vector. |
| 231 | + /// |
| 232 | + /// Note that this may overestimate the number of bytes. For example, reading from userspace |
| 233 | + /// memory could fail with EFAULT, which will be treated as the end of the IO vector. |
| 234 | + #[inline] |
| 235 | + pub fn len(&self) -> usize { |
| 236 | + // SAFETY: We have shared access to this IO vector, so we can read its `count` field. |
| 237 | + unsafe { |
| 238 | + (*self.iov.get()) |
| 239 | + .__bindgen_anon_1 |
| 240 | + .__bindgen_anon_1 |
| 241 | + .as_ref() |
| 242 | + .count |
| 243 | + } |
| 244 | + } |
| 245 | + |
| 246 | + /// Returns whether there are any bytes left in this IO vector. |
| 247 | + /// |
| 248 | + /// This may return `true` even if there are no more bytes available. For example, reading from |
| 249 | + /// userspace memory could fail with EFAULT, which will be treated as the end of the IO vector. |
| 250 | + #[inline] |
| 251 | + pub fn is_empty(&self) -> bool { |
| 252 | + self.len() == 0 |
| 253 | + } |
| 254 | + |
| 255 | + /// Advance this IO vector by `bytes` bytes. |
| 256 | + /// |
| 257 | + /// If `bytes` is larger than the size of this IO vector, it is advanced to the end. |
| 258 | + #[inline] |
| 259 | + pub fn advance(&mut self, bytes: usize) { |
| 260 | + // SAFETY: By the type invariants, `self.iov` is a valid IO vector. |
| 261 | + unsafe { bindings::iov_iter_advance(self.as_raw(), bytes) }; |
| 262 | + } |
| 263 | + |
| 264 | + /// Advance this IO vector backwards by `bytes` bytes. |
| 265 | + /// |
| 266 | + /// # Safety |
| 267 | + /// |
| 268 | + /// The IO vector must not be reverted to before its beginning. |
| 269 | + #[inline] |
| 270 | + pub unsafe fn revert(&mut self, bytes: usize) { |
| 271 | + // SAFETY: By the type invariants, `self.iov` is a valid IO vector, and the caller |
| 272 | + // ensures that `bytes` is in bounds. |
| 273 | + unsafe { bindings::iov_iter_revert(self.as_raw(), bytes) }; |
| 274 | + } |
| 275 | + |
| 276 | + /// Write data to this IO vector. |
| 277 | + /// |
| 278 | + /// Returns the number of bytes that were written. If this is shorter than the provided slice, |
| 279 | + /// then no more bytes can be written. |
| 280 | + #[inline] |
| 281 | + pub fn copy_to_iter(&mut self, input: &[u8]) -> usize { |
| 282 | + // SAFETY: |
| 283 | + // * By the type invariants, it is still valid to write to this IO vector. |
| 284 | + // * `input` is valid for `input.len()` bytes. |
| 285 | + unsafe { bindings::_copy_to_iter(input.as_ptr().cast(), input.len(), self.as_raw()) } |
| 286 | + } |
| 287 | + |
| 288 | + /// Utility for implementing `read_iter` given the full contents of the file. |
| 289 | + /// |
| 290 | + /// The full contents of the file being read from is represented by `contents`. This call will |
| 291 | + /// write the appropriate sub-slice of `contents` and update the file position in `ppos` so |
| 292 | + /// that the file will appear to contain `contents` even if takes multiple reads to read the |
| 293 | + /// entire file. |
| 294 | + #[inline] |
| 295 | + pub fn simple_read_from_buffer(&mut self, ppos: &mut i64, contents: &[u8]) -> Result<usize> { |
| 296 | + if *ppos < 0 { |
| 297 | + return Err(EINVAL); |
| 298 | + } |
| 299 | + let Ok(pos) = usize::try_from(*ppos) else { |
| 300 | + return Ok(0); |
| 301 | + }; |
| 302 | + if pos >= contents.len() { |
| 303 | + return Ok(0); |
| 304 | + } |
| 305 | + |
| 306 | + // BOUNDS: We just checked that `pos < contents.len()` above. |
| 307 | + let num_written = self.copy_to_iter(&contents[pos..]); |
| 308 | + |
| 309 | + // OVERFLOW: `pos+num_written <= contents.len() <= isize::MAX <= i64::MAX`. |
| 310 | + *ppos = (pos + num_written) as i64; |
| 311 | + |
| 312 | + Ok(num_written) |
| 313 | + } |
| 314 | +} |