Add control commands for registering and unregistering shared memory buffers for zero-copy I/O: - UBLK_U_CMD_REG_BUF (0x18): pins pages from userspace, inserts PFN ranges into a per-device maple tree for O(log n) lookup during I/O. Buffer pointers are tracked in a per-device xarray. Returns the assigned buffer index. - UBLK_U_CMD_UNREG_BUF (0x19): removes PFN entries and unpins pages. Queue freeze/unfreeze is handled internally so userspace need not quiesce the device during registration. Also adds: - UBLK_IO_F_SHMEM_ZC flag and addr encoding helpers in UAPI header (16-bit buffer index supporting up to 65536 buffers) - Data structures (ublk_buf, ublk_buf_range) and xarray/maple tree - __ublk_ctrl_reg_buf() helper for PFN insertion with error unwinding - __ublk_ctrl_unreg_buf() helper for cleanup reuse - ublk_support_shmem_zc() / ublk_dev_support_shmem_zc() stubs (returning false — feature not enabled yet)
Showing the changes to include/uapi (1 of 2 files). Show all
Unified · Split
| @@ -57,6 +57,44 @@ · 56 unmodified lines | |||
| 57 | _IOWR('u', 0x16, struct ublksrv_ctrl_cmd) | 57 | _IOWR('u', 0x16, struct ublksrv_ctrl_cmd) |
| 58 | #define UBLK_U_CMD_TRY_STOP_DEV \ | 58 | #define UBLK_U_CMD_TRY_STOP_DEV \ |
| 59 | _IOWR('u', 0x17, struct ublksrv_ctrl_cmd) | 59 | _IOWR('u', 0x17, struct ublksrv_ctrl_cmd) |
| 60 | /* | ||
| 61 | * Register a shared memory buffer for zero-copy I/O. | ||
| 62 | * Input: ctrl_cmd.addr points to struct ublk_shmem_buf_reg (buffer VA + size) | ||
| 63 | * ctrl_cmd.len = sizeof(struct ublk_shmem_buf_reg) | ||
| 64 | * Result: >= 0 is the assigned buffer index, < 0 is error | ||
| 65 | * | ||
| 66 | * The kernel pins pages from the calling process's address space | ||
| 67 | * and inserts PFN ranges into a per-device maple tree. When a block | ||
| 68 | * request's pages match registered pages, the driver sets | ||
| 69 | * UBLK_IO_F_SHMEM_ZC and encodes the buffer index + offset in addr, | ||
| 70 | * allowing the server to access the data via its own mapping of the | ||
| 71 | * same shared memory — true zero copy. | ||
| 72 | * | ||
| 73 | * The memory can be backed by memfd, hugetlbfs, or any GUP-compatible | ||
| 74 | * shared mapping. Queue freeze is handled internally. | ||
| 75 | * | ||
| 76 | * The buffer VA and size are passed via a user buffer (not inline in | ||
| 77 | * ctrl_cmd) so that unprivileged devices can prepend the device path | ||
| 78 | * to ctrl_cmd.addr without corrupting the VA. | ||
| 79 | */ | ||
| 80 | #define UBLK_U_CMD_REG_BUF \ | ||
| 81 | _IOWR('u', 0x18, struct ublksrv_ctrl_cmd) | ||
| 82 | /* | ||
| 83 | * Unregister a shared memory buffer. | ||
| 84 | * Input: ctrl_cmd.data[0] = buffer index | ||
| 85 | */ | ||
| 86 | #define UBLK_U_CMD_UNREG_BUF \ | ||
| 87 | _IOWR('u', 0x19, struct ublksrv_ctrl_cmd) | ||
| 88 | |||
| 89 | /* Parameter buffer for UBLK_U_CMD_REG_BUF, pointed to by ctrl_cmd.addr */ | ||
| 90 | struct ublk_shmem_buf_reg { | ||
| 91 | __u64 addr; /* userspace virtual address of shared memory */ | ||
| 92 | __u32 len; /* buffer size in bytes (page-aligned, max 4GB) */ | ||
| 93 | __u32 flags; | ||
| 94 | }; | ||
| 95 | |||
| 96 | /* Pin pages without FOLL_WRITE; usable with write-sealed memfd */ | ||
| 97 | #define UBLK_SHMEM_BUF_READ_ONLY (1U << 0) | ||
| 60 | /* | 98 | /* |
| 61 | * 64bits are enough now, and it should be easy to extend in case of | 99 | * 64bits are enough now, and it should be easy to extend in case of |
| 62 | * running out of feature flags | 100 | * running out of feature flags |
| @@ -370,6 +408,7 @@ · 307 unmodified lines | |||
| 370 | /* Disable automatic partition scanning when device is started */ | 408 | /* Disable automatic partition scanning when device is started */ |
| 371 | #define UBLK_F_NO_AUTO_PART_SCAN (1ULL << 18) | 409 | #define UBLK_F_NO_AUTO_PART_SCAN (1ULL << 18) |
| 372 | 410 | ||
| 411 | |||
| 373 | /* device state */ | 412 | /* device state */ |
| 374 | #define UBLK_S_DEV_DEAD 0 | 413 | #define UBLK_S_DEV_DEAD 0 |
| 375 | #define UBLK_S_DEV_LIVE 1 | 414 | #define UBLK_S_DEV_LIVE 1 |
| @@ -469,6 +508,12 @@ struct ublksrv_ctrl_dev_info { · 93 unmodified lines | |||
| 469 | #define UBLK_IO_F_NEED_REG_BUF (1U << 17) | 508 | #define UBLK_IO_F_NEED_REG_BUF (1U << 17) |
| 470 | /* Request has an integrity data buffer */ | 509 | /* Request has an integrity data buffer */ |
| 471 | #define UBLK_IO_F_INTEGRITY (1UL << 18) | 510 | #define UBLK_IO_F_INTEGRITY (1UL << 18) |
| 511 | /* | ||
| 512 | * I/O buffer is in a registered shared memory buffer. When set, the addr | ||
| 513 | * field in ublksrv_io_desc encodes buffer index and byte offset instead | ||
| 514 | * of a userspace virtual address. | ||
| 515 | */ | ||
| 516 | #define UBLK_IO_F_SHMEM_ZC (1U << 19) | ||
| 472 | 517 | ||
| 473 | /* | 518 | /* |
| 474 | * io cmd is described by this structure, and stored in share memory, indexed | 519 | * io cmd is described by this structure, and stored in share memory, indexed |
| @@ -743,4 +788,31 @@ struct ublk_params { · 268 unmodified lines | |||
| 743 | struct ublk_param_integrity integrity; | 788 | struct ublk_param_integrity integrity; |
| 744 | }; | 789 | }; |
| 745 | 790 | ||
| 791 | /* | ||
| 792 | * Shared memory zero-copy addr encoding for UBLK_IO_F_SHMEM_ZC. | ||
| 793 | * | ||
| 794 | * When UBLK_IO_F_SHMEM_ZC is set, ublksrv_io_desc.addr is encoded as: | ||
| 795 | * bits [0:31] = byte offset within the buffer (up to 4GB) | ||
| 796 | * bits [32:47] = buffer index (up to 65536) | ||
| 797 | * bits [48:63] = reserved (must be zero) | ||
| 798 | */ | ||
| 799 | #define UBLK_SHMEM_ZC_OFF_MASK 0xffffffffULL | ||
| 800 | #define UBLK_SHMEM_ZC_IDX_OFF 32 | ||
| 801 | #define UBLK_SHMEM_ZC_IDX_MASK 0xffffULL | ||
| 802 | |||
| 803 | static inline __u64 ublk_shmem_zc_addr(__u16 index, __u32 offset) | ||
| 804 | { | ||
| 805 | return ((__u64)index << UBLK_SHMEM_ZC_IDX_OFF) | offset; | ||
| 806 | } | ||
| 807 | |||
| 808 | static inline __u16 ublk_shmem_zc_index(__u64 addr) | ||
| 809 | { | ||
| 810 | return (addr >> UBLK_SHMEM_ZC_IDX_OFF) & UBLK_SHMEM_ZC_IDX_MASK; | ||
| 811 | } | ||
| 812 | |||
| 813 | static inline __u32 ublk_shmem_zc_offset(__u64 addr) | ||
| 814 | { | ||
| 815 | return (__u32)(addr & UBLK_SHMEM_ZC_OFF_MASK); | ||
| 816 | } | ||
| 817 | |||
| 746 | #endif | 818 | #endif |