Add control commands for registering and unregistering shared memory buffers for zero-copy I/O: - UBLK_U_CMD_REG_BUF (0x18): pins pages from userspace, inserts PFN ranges into a per-device maple tree for O(log n) lookup during I/O. Buffer pointers are tracked in a per-device xarray. Returns the assigned buffer index. - UBLK_U_CMD_UNREG_BUF (0x19): removes PFN entries and unpins pages. Queue freeze/unfreeze is handled internally so userspace need not quiesce the device during registration. Also adds: - UBLK_IO_F_SHMEM_ZC flag and addr encoding helpers in UAPI header (16-bit buffer index supporting up to 65536 buffers) - Data structures (ublk_buf, ublk_buf_range) and xarray/maple tree - __ublk_ctrl_reg_buf() helper for PFN insertion with error unwinding - __ublk_ctrl_unreg_buf() helper for cleanup reuse - ublk_support_shmem_zc() / ublk_dev_support_shmem_zc() stubs (returning false — feature not enabled yet)
Showing the changes to include/uapi (1 of 2 files). Show all
Unified · Split
| @@ -57,6 +57,44 @@ · 56 unmodified lines | ||
| 57 | 57 | _IOWR('u', 0x16, struct ublksrv_ctrl_cmd) |
| 58 | 58 | #define UBLK_U_CMD_TRY_STOP_DEV \ |
| 59 | 59 | _IOWR('u', 0x17, struct ublksrv_ctrl_cmd) |
| 60 | +/* | |
| 61 | + * Register a shared memory buffer for zero-copy I/O. | |
| 62 | + * Input: ctrl_cmd.addr points to struct ublk_shmem_buf_reg (buffer VA + size) | |
| 63 | + * ctrl_cmd.len = sizeof(struct ublk_shmem_buf_reg) | |
| 64 | + * Result: >= 0 is the assigned buffer index, < 0 is error | |
| 65 | + * | |
| 66 | + * The kernel pins pages from the calling process's address space | |
| 67 | + * and inserts PFN ranges into a per-device maple tree. When a block | |
| 68 | + * request's pages match registered pages, the driver sets | |
| 69 | + * UBLK_IO_F_SHMEM_ZC and encodes the buffer index + offset in addr, | |
| 70 | + * allowing the server to access the data via its own mapping of the | |
| 71 | + * same shared memory — true zero copy. | |
| 72 | + * | |
| 73 | + * The memory can be backed by memfd, hugetlbfs, or any GUP-compatible | |
| 74 | + * shared mapping. Queue freeze is handled internally. | |
| 75 | + * | |
| 76 | + * The buffer VA and size are passed via a user buffer (not inline in | |
| 77 | + * ctrl_cmd) so that unprivileged devices can prepend the device path | |
| 78 | + * to ctrl_cmd.addr without corrupting the VA. | |
| 79 | + */ | |
| 80 | +#define UBLK_U_CMD_REG_BUF \ | |
| 81 | + _IOWR('u', 0x18, struct ublksrv_ctrl_cmd) | |
| 82 | +/* | |
| 83 | + * Unregister a shared memory buffer. | |
| 84 | + * Input: ctrl_cmd.data[0] = buffer index | |
| 85 | + */ | |
| 86 | +#define UBLK_U_CMD_UNREG_BUF \ | |
| 87 | + _IOWR('u', 0x19, struct ublksrv_ctrl_cmd) | |
| 88 | + | |
| 89 | +/* Parameter buffer for UBLK_U_CMD_REG_BUF, pointed to by ctrl_cmd.addr */ | |
| 90 | +struct ublk_shmem_buf_reg { | |
| 91 | + __u64 addr; /* userspace virtual address of shared memory */ | |
| 92 | + __u32 len; /* buffer size in bytes (page-aligned, max 4GB) */ | |
| 93 | + __u32 flags; | |
| 94 | +}; | |
| 95 | + | |
| 96 | +/* Pin pages without FOLL_WRITE; usable with write-sealed memfd */ | |
| 97 | +#define UBLK_SHMEM_BUF_READ_ONLY (1U << 0) | |
| 60 | 98 | /* |
| 61 | 99 | * 64bits are enough now, and it should be easy to extend in case of |
| 62 | 100 | * running out of feature flags |
| @@ -370,6 +408,7 @@ · 307 unmodified lines | ||
| 370 | 408 | /* Disable automatic partition scanning when device is started */ |
| 371 | 409 | #define UBLK_F_NO_AUTO_PART_SCAN (1ULL << 18) |
| 372 | 410 | |
| 411 | + | |
| 373 | 412 | /* device state */ |
| 374 | 413 | #define UBLK_S_DEV_DEAD 0 |
| 375 | 414 | #define UBLK_S_DEV_LIVE 1 |
| @@ -469,6 +508,12 @@ struct ublksrv_ctrl_dev_info { · 93 unmodified lines | ||
| 469 | 508 | #define UBLK_IO_F_NEED_REG_BUF (1U << 17) |
| 470 | 509 | /* Request has an integrity data buffer */ |
| 471 | 510 | #define UBLK_IO_F_INTEGRITY (1UL << 18) |
| 511 | +/* | |
| 512 | + * I/O buffer is in a registered shared memory buffer. When set, the addr | |
| 513 | + * field in ublksrv_io_desc encodes buffer index and byte offset instead | |
| 514 | + * of a userspace virtual address. | |
| 515 | + */ | |
| 516 | +#define UBLK_IO_F_SHMEM_ZC (1U << 19) | |
| 472 | 517 | |
| 473 | 518 | /* |
| 474 | 519 | * io cmd is described by this structure, and stored in share memory, indexed |
| @@ -743,4 +788,31 @@ struct ublk_params { · 268 unmodified lines | ||
| 743 | 788 | struct ublk_param_integrity integrity; |
| 744 | 789 | }; |
| 745 | 790 | |
| 791 | +/* | |
| 792 | + * Shared memory zero-copy addr encoding for UBLK_IO_F_SHMEM_ZC. | |
| 793 | + * | |
| 794 | + * When UBLK_IO_F_SHMEM_ZC is set, ublksrv_io_desc.addr is encoded as: | |
| 795 | + * bits [0:31] = byte offset within the buffer (up to 4GB) | |
| 796 | + * bits [32:47] = buffer index (up to 65536) | |
| 797 | + * bits [48:63] = reserved (must be zero) | |
| 798 | + */ | |
| 799 | +#define UBLK_SHMEM_ZC_OFF_MASK 0xffffffffULL | |
| 800 | +#define UBLK_SHMEM_ZC_IDX_OFF 32 | |
| 801 | +#define UBLK_SHMEM_ZC_IDX_MASK 0xffffULL | |
| 802 | + | |
| 803 | +static inline __u64 ublk_shmem_zc_addr(__u16 index, __u32 offset) | |
| 804 | +{ | |
| 805 | + return ((__u64)index << UBLK_SHMEM_ZC_IDX_OFF) | offset; | |
| 806 | +} | |
| 807 | + | |
| 808 | +static inline __u16 ublk_shmem_zc_index(__u64 addr) | |
| 809 | +{ | |
| 810 | + return (addr >> UBLK_SHMEM_ZC_IDX_OFF) & UBLK_SHMEM_ZC_IDX_MASK; | |
| 811 | +} | |
| 812 | + | |
| 813 | +static inline __u32 ublk_shmem_zc_offset(__u64 addr) | |
| 814 | +{ | |
| 815 | + return (__u32)(addr & UBLK_SHMEM_ZC_OFF_MASK); | |
| 816 | +} | |
| 817 | + | |
| 746 | 818 | #endif |