Add control commands for registering and unregistering shared memory buffers for zero-copy I/O: - UBLK_U_CMD_REG_BUF (0x18): pins pages from userspace, inserts PFN ranges into a per-device maple tree for O(log n) lookup during I/O. Buffer pointers are tracked in a per-device xarray. Returns the assigned buffer index. - UBLK_U_CMD_UNREG_BUF (0x19): removes PFN entries and unpins pages. Queue freeze/unfreeze is handled internally so userspace need not quiesce the device during registration. Also adds: - UBLK_IO_F_SHMEM_ZC flag and addr encoding helpers in UAPI header (16-bit buffer index supporting up to 65536 buffers) - Data structures (ublk_buf, ublk_buf_range) and xarray/maple tree - __ublk_ctrl_reg_buf() helper for PFN insertion with error unwinding - __ublk_ctrl_unreg_buf() helper for cleanup reuse - ublk_support_shmem_zc() / ublk_dev_support_shmem_zc() stubs (returning false — feature not enabled yet)
Unified · Split
| @@ -46,6 +46,8 @@ · 45 unmodified lines | ||
| 46 | 46 | #include <linux/kref.h> |
| 47 | 47 | #include <linux/kfifo.h> |
| 48 | 48 | #include <linux/blk-integrity.h> |
| 49 | +#include <linux/maple_tree.h> | |
| 50 | +#include <linux/xarray.h> | |
| 49 | 51 | #include <uapi/linux/fs.h> |
| 50 | 52 | #include <uapi/linux/ublk_cmd.h> |
| 51 | 53 | |
| @@ -58,6 +60,8 @@ · 6 unmodified lines | ||
| 58 | 60 | #define UBLK_CMD_UPDATE_SIZE _IOC_NR(UBLK_U_CMD_UPDATE_SIZE) |
| 59 | 61 | #define UBLK_CMD_QUIESCE_DEV _IOC_NR(UBLK_U_CMD_QUIESCE_DEV) |
| 60 | 62 | #define UBLK_CMD_TRY_STOP_DEV _IOC_NR(UBLK_U_CMD_TRY_STOP_DEV) |
| 63 | +#define UBLK_CMD_REG_BUF _IOC_NR(UBLK_U_CMD_REG_BUF) | |
| 64 | +#define UBLK_CMD_UNREG_BUF _IOC_NR(UBLK_U_CMD_UNREG_BUF) | |
| 61 | 65 | |
| 62 | 66 | #define UBLK_IO_REGISTER_IO_BUF _IOC_NR(UBLK_U_IO_REGISTER_IO_BUF) |
| 63 | 67 | #define UBLK_IO_UNREGISTER_IO_BUF _IOC_NR(UBLK_U_IO_UNREGISTER_IO_BUF) |
| @@ -289,6 +293,20 @@ struct ublk_queue { · 225 unmodified lines | ||
| 289 | 293 | struct ublk_io ios[] __counted_by(q_depth); |
| 290 | 294 | }; |
| 291 | 295 | |
| 296 | +/* Per-registered shared memory buffer */ | |
| 297 | +struct ublk_buf { | |
| 298 | + struct page **pages; | |
| 299 | + unsigned int nr_pages; | |
| 300 | +}; | |
| 301 | + | |
| 302 | +/* Maple tree value: maps a PFN range to buffer location */ | |
| 303 | +struct ublk_buf_range { | |
| 304 | + unsigned long base_pfn; | |
| 305 | + unsigned short buf_index; | |
| 306 | + unsigned short flags; | |
| 307 | + unsigned int base_offset; /* byte offset within buffer */ | |
| 308 | +}; | |
| 309 | + | |
| 292 | 310 | struct ublk_device { |
| 293 | 311 | struct gendisk *ub_disk; |
| 294 | 312 | |
| @@ -323,6 +341,10 @@ struct ublk_device { · 28 unmodified lines | ||
| 323 | 341 | |
| 324 | 342 | bool block_open; /* protected by open_mutex */ |
| 325 | 343 | |
| 344 | + /* shared memory zero copy */ | |
| 345 | + struct maple_tree buf_tree; | |
| 346 | + struct xarray bufs_xa; | |
| 347 | + | |
| 326 | 348 | struct ublk_queue *queues[]; |
| 327 | 349 | }; |
| 328 | 350 | |
| @@ -334,6 +356,7 @@ struct ublk_params_header { · 5 unmodified lines | ||
| 334 | 356 | |
| 335 | 357 | static void ublk_io_release(void *priv); |
| 336 | 358 | static void ublk_stop_dev_unlocked(struct ublk_device *ub); |
| 359 | +static void ublk_buf_cleanup(struct ublk_device *ub); | |
| 337 | 360 | static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq); |
| 338 | 361 | static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub, |
| 339 | 362 | u16 q_id, u16 tag, struct ublk_io *io); |
| @@ -398,6 +421,16 @@ static inline bool ublk_dev_support_zero_copy(const struct ublk_device *ub) · 58 unmodified lines | ||
| 398 | 421 | return ub->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY; |
| 399 | 422 | } |
| 400 | 423 | |
| 424 | +static inline bool ublk_support_shmem_zc(const struct ublk_queue *ubq) | |
| 425 | +{ | |
| 426 | + return false; | |
| 427 | +} | |
| 428 | + | |
| 429 | +static inline bool ublk_dev_support_shmem_zc(const struct ublk_device *ub) | |
| 430 | +{ | |
| 431 | + return false; | |
| 432 | +} | |
| 433 | + | |
| 401 | 434 | static inline bool ublk_support_auto_buf_reg(const struct ublk_queue *ubq) |
| 402 | 435 | { |
| 403 | 436 | return ubq->flags & UBLK_F_AUTO_BUF_REG; |
| @@ -1460,6 +1493,7 @@ static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req) · 1056 unmodified lines | ||
| 1460 | 1493 | iod->op_flags = ublk_op | ublk_req_build_flags(req); |
| 1461 | 1494 | iod->nr_sectors = blk_rq_sectors(req); |
| 1462 | 1495 | iod->start_sector = blk_rq_pos(req); |
| 1496 | + | |
| 1463 | 1497 | iod->addr = io->buf.addr; |
| 1464 | 1498 | |
| 1465 | 1499 | return BLK_STS_OK; |
| @@ -1665,6 +1699,7 @@ static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req, · 199 unmodified lines | ||
| 1665 | 1699 | { |
| 1666 | 1700 | unsigned mapped_bytes = ublk_map_io(ubq, req, io); |
| 1667 | 1701 | |
| 1702 | + | |
| 1668 | 1703 | /* partially mapped, update io descriptor */ |
| 1669 | 1704 | if (unlikely(mapped_bytes != blk_rq_bytes(req))) { |
| 1670 | 1705 | /* |
| @@ -4211,6 +4246,7 @@ static void ublk_cdev_rel(struct device *dev) · 2540 unmodified lines | ||
| 4211 | 4246 | { |
| 4212 | 4247 | struct ublk_device *ub = container_of(dev, struct ublk_device, cdev_dev); |
| 4213 | 4248 | |
| 4249 | + ublk_buf_cleanup(ub); | |
| 4214 | 4250 | blk_mq_free_tag_set(&ub->tag_set); |
| 4215 | 4251 | ublk_deinit_queues(ub); |
| 4216 | 4252 | ublk_free_dev_number(ub); |
| @@ -4630,6 +4666,8 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) · 413 unmodified lines | ||
| 4630 | 4666 | mutex_init(&ub->mutex); |
| 4631 | 4667 | spin_lock_init(&ub->lock); |
| 4632 | 4668 | mutex_init(&ub->cancel_mutex); |
| 4669 | + mt_init(&ub->buf_tree); | |
| 4670 | + xa_init_flags(&ub->bufs_xa, XA_FLAGS_ALLOC); | |
| 4633 | 4671 | INIT_WORK(&ub->partition_scan_work, ublk_partition_scan_work); |
| 4634 | 4672 | |
| 4635 | 4673 | ret = ublk_alloc_dev_number(ub, header->dev_id); |
| @@ -5173,6 +5211,255 @@ exit: · 537 unmodified lines | ||
| 5173 | 5211 | return err; |
| 5174 | 5212 | } |
| 5175 | 5213 | |
| 5214 | +/* | |
| 5215 | + * Drain inflight I/O and quiesce the queue. Freeze drains all inflight | |
| 5216 | + * requests, quiesce_nowait marks the queue so no new requests dispatch, | |
| 5217 | + * then unfreeze allows new submissions (which won't dispatch due to | |
| 5218 | + * quiesce). This keeps freeze and ub->mutex non-nested. | |
| 5219 | + */ | |
| 5220 | +static void ublk_quiesce_and_release(struct gendisk *disk) | |
| 5221 | +{ | |
| 5222 | + unsigned int memflags; | |
| 5223 | + | |
| 5224 | + memflags = blk_mq_freeze_queue(disk->queue); | |
| 5225 | + blk_mq_quiesce_queue_nowait(disk->queue); | |
| 5226 | + blk_mq_unfreeze_queue(disk->queue, memflags); | |
| 5227 | +} | |
| 5228 | + | |
| 5229 | +static void ublk_unquiesce_and_resume(struct gendisk *disk) | |
| 5230 | +{ | |
| 5231 | + blk_mq_unquiesce_queue(disk->queue); | |
| 5232 | +} | |
| 5233 | + | |
| 5234 | +/* Erase coalesced PFN ranges from the maple tree for pages [0, nr_pages) */ | |
| 5235 | +static void ublk_buf_erase_ranges(struct ublk_device *ub, | |
| 5236 | + struct ublk_buf *ubuf, | |
| 5237 | + unsigned long nr_pages) | |
| 5238 | +{ | |
| 5239 | + unsigned long i; | |
| 5240 | + | |
| 5241 | + for (i = 0; i < nr_pages; ) { | |
| 5242 | + unsigned long pfn = page_to_pfn(ubuf->pages[i]); | |
| 5243 | + unsigned long start = i; | |
| 5244 | + | |
| 5245 | + while (i + 1 < nr_pages && | |
| 5246 | + page_to_pfn(ubuf->pages[i + 1]) == pfn + (i - start) + 1) | |
| 5247 | + i++; | |
| 5248 | + i++; | |
| 5249 | + kfree(mtree_erase(&ub->buf_tree, pfn)); | |
| 5250 | + } | |
| 5251 | +} | |
| 5252 | + | |
| 5253 | +static int __ublk_ctrl_reg_buf(struct ublk_device *ub, | |
| 5254 | + struct ublk_buf *ubuf, int index, | |
| 5255 | + unsigned short flags) | |
| 5256 | +{ | |
| 5257 | + unsigned long nr_pages = ubuf->nr_pages; | |
| 5258 | + unsigned long i; | |
| 5259 | + int ret; | |
| 5260 | + | |
| 5261 | + for (i = 0; i < nr_pages; ) { | |
| 5262 | + unsigned long pfn = page_to_pfn(ubuf->pages[i]); | |
| 5263 | + unsigned long start = i; | |
| 5264 | + struct ublk_buf_range *range; | |
| 5265 | + | |
| 5266 | + /* Find run of consecutive PFNs */ | |
| 5267 | + while (i + 1 < nr_pages && | |
| 5268 | + page_to_pfn(ubuf->pages[i + 1]) == pfn + (i - start) + 1) | |
| 5269 | + i++; | |
| 5270 | + i++; /* past the last page in this run */ | |
| 5271 | + | |
| 5272 | + range = kzalloc(sizeof(*range), GFP_KERNEL); | |
| 5273 | + if (!range) { | |
| 5274 | + ret = -ENOMEM; | |
| 5275 | + goto unwind; | |
| 5276 | + } | |
| 5277 | + range->buf_index = index; | |
| 5278 | + range->flags = flags; | |
| 5279 | + range->base_pfn = pfn; | |
| 5280 | + range->base_offset = start << PAGE_SHIFT; | |
| 5281 | + | |
| 5282 | + ret = mtree_insert_range(&ub->buf_tree, pfn, | |
| 5283 | + pfn + (i - start) - 1, | |
| 5284 | + range, GFP_KERNEL); | |
| 5285 | + if (ret) { | |
| 5286 | + kfree(range); | |
| 5287 | + goto unwind; | |
| 5288 | + } | |
| 5289 | + } | |
| 5290 | + return 0; | |
| 5291 | + | |
| 5292 | +unwind: | |
| 5293 | + ublk_buf_erase_ranges(ub, ubuf, i); | |
| 5294 | + return ret; | |
| 5295 | +} | |
| 5296 | + | |
| 5297 | +/* | |
| 5298 | + * Register a shared memory buffer for zero-copy I/O. | |
| 5299 | + * Pins pages, builds PFN maple tree, freezes/unfreezes the queue | |
| 5300 | + * internally. Returns buffer index (>= 0) on success. | |
| 5301 | + */ | |
| 5302 | +static int ublk_ctrl_reg_buf(struct ublk_device *ub, | |
| 5303 | + struct ublksrv_ctrl_cmd *header) | |
| 5304 | +{ | |
| 5305 | + void __user *argp = (void __user *)(unsigned long)header->addr; | |
| 5306 | + struct ublk_shmem_buf_reg buf_reg; | |
| 5307 | + unsigned long addr, size, nr_pages; | |
| 5308 | + unsigned int gup_flags; | |
| 5309 | + struct gendisk *disk; | |
| 5310 | + struct ublk_buf *ubuf; | |
| 5311 | + long pinned; | |
| 5312 | + u32 index; | |
| 5313 | + int ret; | |
| 5314 | + | |
| 5315 | + if (!ublk_dev_support_shmem_zc(ub)) | |
| 5316 | + return -EOPNOTSUPP; | |
| 5317 | + | |
| 5318 | + memset(&buf_reg, 0, sizeof(buf_reg)); | |
| 5319 | + if (copy_from_user(&buf_reg, argp, | |
| 5320 | + min_t(size_t, header->len, sizeof(buf_reg)))) | |
| 5321 | + return -EFAULT; | |
| 5322 | + | |
| 5323 | + if (buf_reg.flags & ~UBLK_SHMEM_BUF_READ_ONLY) | |
| 5324 | + return -EINVAL; | |
| 5325 | + | |
| 5326 | + addr = buf_reg.addr; | |
| 5327 | + size = buf_reg.len; | |
| 5328 | + nr_pages = size >> PAGE_SHIFT; | |
| 5329 | + | |
| 5330 | + if (!size || !PAGE_ALIGNED(size) || !PAGE_ALIGNED(addr)) | |
| 5331 | + return -EINVAL; | |
| 5332 | + | |
| 5333 | + disk = ublk_get_disk(ub); | |
| 5334 | + if (!disk) | |
| 5335 | + return -ENODEV; | |
| 5336 | + | |
| 5337 | + /* Pin pages before quiescing (may sleep) */ | |
| 5338 | + ubuf = kzalloc(sizeof(*ubuf), GFP_KERNEL); | |
| 5339 | + if (!ubuf) { | |
| 5340 | + ret = -ENOMEM; | |
| 5341 | + goto put_disk; | |
| 5342 | + } | |
| 5343 | + | |
| 5344 | + ubuf->pages = kvmalloc_array(nr_pages, sizeof(*ubuf->pages), | |
| 5345 | + GFP_KERNEL); | |
| 5346 | + if (!ubuf->pages) { | |
| 5347 | + ret = -ENOMEM; | |
| 5348 | + goto err_free; | |
| 5349 | + } | |
| 5350 | + | |
| 5351 | + gup_flags = FOLL_LONGTERM; | |
| 5352 | + if (!(buf_reg.flags & UBLK_SHMEM_BUF_READ_ONLY)) | |
| 5353 | + gup_flags |= FOLL_WRITE; | |
| 5354 | + | |
| 5355 | + pinned = pin_user_pages_fast(addr, nr_pages, gup_flags, ubuf->pages); | |
| 5356 | + if (pinned < 0) { | |
| 5357 | + ret = pinned; | |
| 5358 | + goto err_free_pages; | |
| 5359 | + } | |
| 5360 | + if (pinned != nr_pages) { | |
| 5361 | + ret = -EFAULT; | |
| 5362 | + goto err_unpin; | |
| 5363 | + } | |
| 5364 | + ubuf->nr_pages = nr_pages; | |
| 5365 | + | |
| 5366 | + /* | |
| 5367 | + * Drain inflight I/O and quiesce the queue so no new requests | |
| 5368 | + * are dispatched while we modify the maple tree. Keep freeze | |
| 5369 | + * and mutex non-nested to avoid lock dependency. | |
| 5370 | + */ | |
| 5371 | + ublk_quiesce_and_release(disk); | |
| 5372 | + | |
| 5373 | + mutex_lock(&ub->mutex); | |
| 5374 | + | |
| 5375 | + ret = xa_alloc(&ub->bufs_xa, &index, ubuf, xa_limit_16b, GFP_KERNEL); | |
| 5376 | + if (ret) | |
| 5377 | + goto err_unlock; | |
| 5378 | + | |
| 5379 | + ret = __ublk_ctrl_reg_buf(ub, ubuf, index, buf_reg.flags); | |
| 5380 | + if (ret) { | |
| 5381 | + xa_erase(&ub->bufs_xa, index); | |
| 5382 | + goto err_unlock; | |
| 5383 | + } | |
| 5384 | + | |
| 5385 | + mutex_unlock(&ub->mutex); | |
| 5386 | + | |
| 5387 | + ublk_unquiesce_and_resume(disk); | |
| 5388 | + ublk_put_disk(disk); | |
| 5389 | + return index; | |
| 5390 | + | |
| 5391 | +err_unlock: | |
| 5392 | + mutex_unlock(&ub->mutex); | |
| 5393 | + ublk_unquiesce_and_resume(disk); | |
| 5394 | +err_unpin: | |
| 5395 | + unpin_user_pages(ubuf->pages, pinned); | |
| 5396 | +err_free_pages: | |
| 5397 | + kvfree(ubuf->pages); | |
| 5398 | +err_free: | |
| 5399 | + kfree(ubuf); | |
| 5400 | +put_disk: | |
| 5401 | + ublk_put_disk(disk); | |
| 5402 | + return ret; | |
| 5403 | +} | |
| 5404 | + | |
| 5405 | +static void __ublk_ctrl_unreg_buf(struct ublk_device *ub, | |
| 5406 | + struct ublk_buf *ubuf) | |
| 5407 | +{ | |
| 5408 | + ublk_buf_erase_ranges(ub, ubuf, ubuf->nr_pages); | |
| 5409 | + unpin_user_pages(ubuf->pages, ubuf->nr_pages); | |
| 5410 | + kvfree(ubuf->pages); | |
| 5411 | + kfree(ubuf); | |
| 5412 | +} | |
| 5413 | + | |
| 5414 | +static int ublk_ctrl_unreg_buf(struct ublk_device *ub, | |
| 5415 | + struct ublksrv_ctrl_cmd *header) | |
| 5416 | +{ | |
| 5417 | + int index = (int)header->data[0]; | |
| 5418 | + struct gendisk *disk; | |
| 5419 | + struct ublk_buf *ubuf; | |
| 5420 | + | |
| 5421 | + if (!ublk_dev_support_shmem_zc(ub)) | |
| 5422 | + return -EOPNOTSUPP; | |
| 5423 | + | |
| 5424 | + disk = ublk_get_disk(ub); | |
| 5425 | + if (!disk) | |
| 5426 | + return -ENODEV; | |
| 5427 | + | |
| 5428 | + /* Drain inflight I/O before modifying the maple tree */ | |
| 5429 | + ublk_quiesce_and_release(disk); | |
| 5430 | + | |
| 5431 | + mutex_lock(&ub->mutex); | |
| 5432 | + | |
| 5433 | + ubuf = xa_erase(&ub->bufs_xa, index); | |
| 5434 | + if (!ubuf) { | |
| 5435 | + mutex_unlock(&ub->mutex); | |
| 5436 | + ublk_unquiesce_and_resume(disk); | |
| 5437 | + ublk_put_disk(disk); | |
| 5438 | + return -ENOENT; | |
| 5439 | + } | |
| 5440 | + | |
| 5441 | + __ublk_ctrl_unreg_buf(ub, ubuf); | |
| 5442 | + | |
| 5443 | + mutex_unlock(&ub->mutex); | |
| 5444 | + | |
| 5445 | + ublk_unquiesce_and_resume(disk); | |
| 5446 | + ublk_put_disk(disk); | |
| 5447 | + return 0; | |
| 5448 | +} | |
| 5449 | + | |
| 5450 | +static void ublk_buf_cleanup(struct ublk_device *ub) | |
| 5451 | +{ | |
| 5452 | + struct ublk_buf *ubuf; | |
| 5453 | + unsigned long index; | |
| 5454 | + | |
| 5455 | + xa_for_each(&ub->bufs_xa, index, ubuf) | |
| 5456 | + __ublk_ctrl_unreg_buf(ub, ubuf); | |
| 5457 | + xa_destroy(&ub->bufs_xa); | |
| 5458 | + mtree_destroy(&ub->buf_tree); | |
| 5459 | +} | |
| 5460 | + | |
| 5461 | + | |
| 5462 | + | |
| 5176 | 5463 | static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, |
| 5177 | 5464 | u32 cmd_op, struct ublksrv_ctrl_cmd *header) |
| 5178 | 5465 | { |
| @@ -5230,6 +5517,8 @@ static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, · 51 unmodified lines | ||
| 5230 | 5517 | case UBLK_CMD_UPDATE_SIZE: |
| 5231 | 5518 | case UBLK_CMD_QUIESCE_DEV: |
| 5232 | 5519 | case UBLK_CMD_TRY_STOP_DEV: |
| 5520 | + case UBLK_CMD_REG_BUF: | |
| 5521 | + case UBLK_CMD_UNREG_BUF: | |
| 5233 | 5522 | mask = MAY_READ | MAY_WRITE; |
| 5234 | 5523 | break; |
| 5235 | 5524 | default: |
| @@ -5355,6 +5644,12 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, · 119 unmodified lines | ||
| 5355 | 5644 | case UBLK_CMD_TRY_STOP_DEV: |
| 5356 | 5645 | ret = ublk_ctrl_try_stop_dev(ub); |
| 5357 | 5646 | break; |
| 5647 | + case UBLK_CMD_REG_BUF: | |
| 5648 | + ret = ublk_ctrl_reg_buf(ub, &header); | |
| 5649 | + break; | |
| 5650 | + case UBLK_CMD_UNREG_BUF: | |
| 5651 | + ret = ublk_ctrl_unreg_buf(ub, &header); | |
| 5652 | + break; | |
| 5358 | 5653 | default: |
| 5359 | 5654 | ret = -EOPNOTSUPP; |
| 5360 | 5655 | break; |
| @@ -57,6 +57,44 @@ · 56 unmodified lines | ||
| 57 | 57 | _IOWR('u', 0x16, struct ublksrv_ctrl_cmd) |
| 58 | 58 | #define UBLK_U_CMD_TRY_STOP_DEV \ |
| 59 | 59 | _IOWR('u', 0x17, struct ublksrv_ctrl_cmd) |
| 60 | +/* | |
| 61 | + * Register a shared memory buffer for zero-copy I/O. | |
| 62 | + * Input: ctrl_cmd.addr points to struct ublk_shmem_buf_reg (buffer VA + size) | |
| 63 | + * ctrl_cmd.len = sizeof(struct ublk_shmem_buf_reg) | |
| 64 | + * Result: >= 0 is the assigned buffer index, < 0 is error | |
| 65 | + * | |
| 66 | + * The kernel pins pages from the calling process's address space | |
| 67 | + * and inserts PFN ranges into a per-device maple tree. When a block | |
| 68 | + * request's pages match registered pages, the driver sets | |
| 69 | + * UBLK_IO_F_SHMEM_ZC and encodes the buffer index + offset in addr, | |
| 70 | + * allowing the server to access the data via its own mapping of the | |
| 71 | + * same shared memory — true zero copy. | |
| 72 | + * | |
| 73 | + * The memory can be backed by memfd, hugetlbfs, or any GUP-compatible | |
| 74 | + * shared mapping. Queue freeze is handled internally. | |
| 75 | + * | |
| 76 | + * The buffer VA and size are passed via a user buffer (not inline in | |
| 77 | + * ctrl_cmd) so that unprivileged devices can prepend the device path | |
| 78 | + * to ctrl_cmd.addr without corrupting the VA. | |
| 79 | + */ | |
| 80 | +#define UBLK_U_CMD_REG_BUF \ | |
| 81 | + _IOWR('u', 0x18, struct ublksrv_ctrl_cmd) | |
| 82 | +/* | |
| 83 | + * Unregister a shared memory buffer. | |
| 84 | + * Input: ctrl_cmd.data[0] = buffer index | |
| 85 | + */ | |
| 86 | +#define UBLK_U_CMD_UNREG_BUF \ | |
| 87 | + _IOWR('u', 0x19, struct ublksrv_ctrl_cmd) | |
| 88 | + | |
| 89 | +/* Parameter buffer for UBLK_U_CMD_REG_BUF, pointed to by ctrl_cmd.addr */ | |
| 90 | +struct ublk_shmem_buf_reg { | |
| 91 | + __u64 addr; /* userspace virtual address of shared memory */ | |
| 92 | + __u32 len; /* buffer size in bytes (page-aligned, max 4GB) */ | |
| 93 | + __u32 flags; | |
| 94 | +}; | |
| 95 | + | |
| 96 | +/* Pin pages without FOLL_WRITE; usable with write-sealed memfd */ | |
| 97 | +#define UBLK_SHMEM_BUF_READ_ONLY (1U << 0) | |
| 60 | 98 | /* |
| 61 | 99 | * 64bits are enough now, and it should be easy to extend in case of |
| 62 | 100 | * running out of feature flags |
| @@ -370,6 +408,7 @@ · 307 unmodified lines | ||
| 370 | 408 | /* Disable automatic partition scanning when device is started */ |
| 371 | 409 | #define UBLK_F_NO_AUTO_PART_SCAN (1ULL << 18) |
| 372 | 410 | |
| 411 | + | |
| 373 | 412 | /* device state */ |
| 374 | 413 | #define UBLK_S_DEV_DEAD 0 |
| 375 | 414 | #define UBLK_S_DEV_LIVE 1 |
| @@ -469,6 +508,12 @@ struct ublksrv_ctrl_dev_info { · 93 unmodified lines | ||
| 469 | 508 | #define UBLK_IO_F_NEED_REG_BUF (1U << 17) |
| 470 | 509 | /* Request has an integrity data buffer */ |
| 471 | 510 | #define UBLK_IO_F_INTEGRITY (1UL << 18) |
| 511 | +/* | |
| 512 | + * I/O buffer is in a registered shared memory buffer. When set, the addr | |
| 513 | + * field in ublksrv_io_desc encodes buffer index and byte offset instead | |
| 514 | + * of a userspace virtual address. | |
| 515 | + */ | |
| 516 | +#define UBLK_IO_F_SHMEM_ZC (1U << 19) | |
| 472 | 517 | |
| 473 | 518 | /* |
| 474 | 519 | * io cmd is described by this structure, and stored in share memory, indexed |
| @@ -743,4 +788,31 @@ struct ublk_params { · 268 unmodified lines | ||
| 743 | 788 | struct ublk_param_integrity integrity; |
| 744 | 789 | }; |
| 745 | 790 | |
| 791 | +/* | |
| 792 | + * Shared memory zero-copy addr encoding for UBLK_IO_F_SHMEM_ZC. | |
| 793 | + * | |
| 794 | + * When UBLK_IO_F_SHMEM_ZC is set, ublksrv_io_desc.addr is encoded as: | |
| 795 | + * bits [0:31] = byte offset within the buffer (up to 4GB) | |
| 796 | + * bits [32:47] = buffer index (up to 65536) | |
| 797 | + * bits [48:63] = reserved (must be zero) | |
| 798 | + */ | |
| 799 | +#define UBLK_SHMEM_ZC_OFF_MASK 0xffffffffULL | |
| 800 | +#define UBLK_SHMEM_ZC_IDX_OFF 32 | |
| 801 | +#define UBLK_SHMEM_ZC_IDX_MASK 0xffffULL | |
| 802 | + | |
| 803 | +static inline __u64 ublk_shmem_zc_addr(__u16 index, __u32 offset) | |
| 804 | +{ | |
| 805 | + return ((__u64)index << UBLK_SHMEM_ZC_IDX_OFF) | offset; | |
| 806 | +} | |
| 807 | + | |
| 808 | +static inline __u16 ublk_shmem_zc_index(__u64 addr) | |
| 809 | +{ | |
| 810 | + return (addr >> UBLK_SHMEM_ZC_IDX_OFF) & UBLK_SHMEM_ZC_IDX_MASK; | |
| 811 | +} | |
| 812 | + | |
| 813 | +static inline __u32 ublk_shmem_zc_offset(__u64 addr) | |
| 814 | +{ | |
| 815 | + return (__u32)(addr & UBLK_SHMEM_ZC_OFF_MASK); | |
| 816 | +} | |
| 817 | + | |
| 746 | 818 | #endif |