Add control commands for registering and unregistering shared memory buffers for zero-copy I/O: - UBLK_U_CMD_REG_BUF (0x18): pins pages from userspace, inserts PFN ranges into a per-device maple tree for O(log n) lookup during I/O. Buffer pointers are tracked in a per-device xarray. Returns the assigned buffer index. - UBLK_U_CMD_UNREG_BUF (0x19): removes PFN entries and unpins pages. Queue freeze/unfreeze is handled internally so userspace need not quiesce the device during registration. Also adds: - UBLK_IO_F_SHMEM_ZC flag and addr encoding helpers in UAPI header (16-bit buffer index supporting up to 65536 buffers) - Data structures (ublk_buf, ublk_buf_range) and xarray/maple tree - __ublk_ctrl_reg_buf() helper for PFN insertion with error unwinding - __ublk_ctrl_unreg_buf() helper for cleanup reuse - ublk_support_shmem_zc() / ublk_dev_support_shmem_zc() stubs (returning false — feature not enabled yet)
Unified · Split
| @@ -46,6 +46,8 @@ · 45 unmodified lines | |||
| 46 | #include <linux/kref.h> | 46 | #include <linux/kref.h> |
| 47 | #include <linux/kfifo.h> | 47 | #include <linux/kfifo.h> |
| 48 | #include <linux/blk-integrity.h> | 48 | #include <linux/blk-integrity.h> |
| 49 | #include <linux/maple_tree.h> | ||
| 50 | #include <linux/xarray.h> | ||
| 49 | #include <uapi/linux/fs.h> | 51 | #include <uapi/linux/fs.h> |
| 50 | #include <uapi/linux/ublk_cmd.h> | 52 | #include <uapi/linux/ublk_cmd.h> |
| 51 | 53 | ||
| @@ -58,6 +60,8 @@ · 6 unmodified lines | |||
| 58 | #define UBLK_CMD_UPDATE_SIZE _IOC_NR(UBLK_U_CMD_UPDATE_SIZE) | 60 | #define UBLK_CMD_UPDATE_SIZE _IOC_NR(UBLK_U_CMD_UPDATE_SIZE) |
| 59 | #define UBLK_CMD_QUIESCE_DEV _IOC_NR(UBLK_U_CMD_QUIESCE_DEV) | 61 | #define UBLK_CMD_QUIESCE_DEV _IOC_NR(UBLK_U_CMD_QUIESCE_DEV) |
| 60 | #define UBLK_CMD_TRY_STOP_DEV _IOC_NR(UBLK_U_CMD_TRY_STOP_DEV) | 62 | #define UBLK_CMD_TRY_STOP_DEV _IOC_NR(UBLK_U_CMD_TRY_STOP_DEV) |
| 63 | #define UBLK_CMD_REG_BUF _IOC_NR(UBLK_U_CMD_REG_BUF) | ||
| 64 | #define UBLK_CMD_UNREG_BUF _IOC_NR(UBLK_U_CMD_UNREG_BUF) | ||
| 61 | 65 | ||
| 62 | #define UBLK_IO_REGISTER_IO_BUF _IOC_NR(UBLK_U_IO_REGISTER_IO_BUF) | 66 | #define UBLK_IO_REGISTER_IO_BUF _IOC_NR(UBLK_U_IO_REGISTER_IO_BUF) |
| 63 | #define UBLK_IO_UNREGISTER_IO_BUF _IOC_NR(UBLK_U_IO_UNREGISTER_IO_BUF) | 67 | #define UBLK_IO_UNREGISTER_IO_BUF _IOC_NR(UBLK_U_IO_UNREGISTER_IO_BUF) |
| @@ -289,6 +293,20 @@ struct ublk_queue { · 225 unmodified lines | |||
| 289 | struct ublk_io ios[] __counted_by(q_depth); | 293 | struct ublk_io ios[] __counted_by(q_depth); |
| 290 | }; | 294 | }; |
| 291 | 295 | ||
| 296 | /* Per-registered shared memory buffer */ | ||
| 297 | struct ublk_buf { | ||
| 298 | struct page **pages; | ||
| 299 | unsigned int nr_pages; | ||
| 300 | }; | ||
| 301 | |||
| 302 | /* Maple tree value: maps a PFN range to buffer location */ | ||
| 303 | struct ublk_buf_range { | ||
| 304 | unsigned long base_pfn; | ||
| 305 | unsigned short buf_index; | ||
| 306 | unsigned short flags; | ||
| 307 | unsigned int base_offset; /* byte offset within buffer */ | ||
| 308 | }; | ||
| 309 | |||
| 292 | struct ublk_device { | 310 | struct ublk_device { |
| 293 | struct gendisk *ub_disk; | 311 | struct gendisk *ub_disk; |
| 294 | 312 | ||
| @@ -323,6 +341,10 @@ struct ublk_device { · 28 unmodified lines | |||
| 323 | 341 | ||
| 324 | bool block_open; /* protected by open_mutex */ | 342 | bool block_open; /* protected by open_mutex */ |
| 325 | 343 | ||
| 344 | /* shared memory zero copy */ | ||
| 345 | struct maple_tree buf_tree; | ||
| 346 | struct xarray bufs_xa; | ||
| 347 | |||
| 326 | struct ublk_queue *queues[]; | 348 | struct ublk_queue *queues[]; |
| 327 | }; | 349 | }; |
| 328 | 350 | ||
| @@ -334,6 +356,7 @@ struct ublk_params_header { · 5 unmodified lines | |||
| 334 | 356 | ||
| 335 | static void ublk_io_release(void *priv); | 357 | static void ublk_io_release(void *priv); |
| 336 | static void ublk_stop_dev_unlocked(struct ublk_device *ub); | 358 | static void ublk_stop_dev_unlocked(struct ublk_device *ub); |
| 359 | static void ublk_buf_cleanup(struct ublk_device *ub); | ||
| 337 | static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq); | 360 | static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq); |
| 338 | static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub, | 361 | static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub, |
| 339 | u16 q_id, u16 tag, struct ublk_io *io); | 362 | u16 q_id, u16 tag, struct ublk_io *io); |
| @@ -398,6 +421,16 @@ static inline bool ublk_dev_support_zero_copy(const struct ublk_device *ub) · 58 unmodified lines | |||
| 398 | return ub->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY; | 421 | return ub->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY; |
| 399 | } | 422 | } |
| 400 | 423 | ||
| 424 | static inline bool ublk_support_shmem_zc(const struct ublk_queue *ubq) | ||
| 425 | { | ||
| 426 | return false; | ||
| 427 | } | ||
| 428 | |||
| 429 | static inline bool ublk_dev_support_shmem_zc(const struct ublk_device *ub) | ||
| 430 | { | ||
| 431 | return false; | ||
| 432 | } | ||
| 433 | |||
| 401 | static inline bool ublk_support_auto_buf_reg(const struct ublk_queue *ubq) | 434 | static inline bool ublk_support_auto_buf_reg(const struct ublk_queue *ubq) |
| 402 | { | 435 | { |
| 403 | return ubq->flags & UBLK_F_AUTO_BUF_REG; | 436 | return ubq->flags & UBLK_F_AUTO_BUF_REG; |
| @@ -1460,6 +1493,7 @@ static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req) · 1056 unmodified lines | |||
| 1460 | iod->op_flags = ublk_op | ublk_req_build_flags(req); | 1493 | iod->op_flags = ublk_op | ublk_req_build_flags(req); |
| 1461 | iod->nr_sectors = blk_rq_sectors(req); | 1494 | iod->nr_sectors = blk_rq_sectors(req); |
| 1462 | iod->start_sector = blk_rq_pos(req); | 1495 | iod->start_sector = blk_rq_pos(req); |
| 1496 | |||
| 1463 | iod->addr = io->buf.addr; | 1497 | iod->addr = io->buf.addr; |
| 1464 | 1498 | ||
| 1465 | return BLK_STS_OK; | 1499 | return BLK_STS_OK; |
| @@ -1665,6 +1699,7 @@ static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req, · 199 unmodified lines | |||
| 1665 | { | 1699 | { |
| 1666 | unsigned mapped_bytes = ublk_map_io(ubq, req, io); | 1700 | unsigned mapped_bytes = ublk_map_io(ubq, req, io); |
| 1667 | 1701 | ||
| 1702 | |||
| 1668 | /* partially mapped, update io descriptor */ | 1703 | /* partially mapped, update io descriptor */ |
| 1669 | if (unlikely(mapped_bytes != blk_rq_bytes(req))) { | 1704 | if (unlikely(mapped_bytes != blk_rq_bytes(req))) { |
| 1670 | /* | 1705 | /* |
| @@ -4211,6 +4246,7 @@ static void ublk_cdev_rel(struct device *dev) · 2540 unmodified lines | |||
| 4211 | { | 4246 | { |
| 4212 | struct ublk_device *ub = container_of(dev, struct ublk_device, cdev_dev); | 4247 | struct ublk_device *ub = container_of(dev, struct ublk_device, cdev_dev); |
| 4213 | 4248 | ||
| 4249 | ublk_buf_cleanup(ub); | ||
| 4214 | blk_mq_free_tag_set(&ub->tag_set); | 4250 | blk_mq_free_tag_set(&ub->tag_set); |
| 4215 | ublk_deinit_queues(ub); | 4251 | ublk_deinit_queues(ub); |
| 4216 | ublk_free_dev_number(ub); | 4252 | ublk_free_dev_number(ub); |
| @@ -4630,6 +4666,8 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) · 413 unmodified lines | |||
| 4630 | mutex_init(&ub->mutex); | 4666 | mutex_init(&ub->mutex); |
| 4631 | spin_lock_init(&ub->lock); | 4667 | spin_lock_init(&ub->lock); |
| 4632 | mutex_init(&ub->cancel_mutex); | 4668 | mutex_init(&ub->cancel_mutex); |
| 4669 | mt_init(&ub->buf_tree); | ||
| 4670 | xa_init_flags(&ub->bufs_xa, XA_FLAGS_ALLOC); | ||
| 4633 | INIT_WORK(&ub->partition_scan_work, ublk_partition_scan_work); | 4671 | INIT_WORK(&ub->partition_scan_work, ublk_partition_scan_work); |
| 4634 | 4672 | ||
| 4635 | ret = ublk_alloc_dev_number(ub, header->dev_id); | 4673 | ret = ublk_alloc_dev_number(ub, header->dev_id); |
| @@ -5173,6 +5211,255 @@ exit: · 537 unmodified lines | |||
| 5173 | return err; | 5211 | return err; |
| 5174 | } | 5212 | } |
| 5175 | 5213 | ||
| 5214 | /* | ||
| 5215 | * Drain inflight I/O and quiesce the queue. Freeze drains all inflight | ||
| 5216 | * requests, quiesce_nowait marks the queue so no new requests dispatch, | ||
| 5217 | * then unfreeze allows new submissions (which won't dispatch due to | ||
| 5218 | * quiesce). This keeps freeze and ub->mutex non-nested. | ||
| 5219 | */ | ||
| 5220 | static void ublk_quiesce_and_release(struct gendisk *disk) | ||
| 5221 | { | ||
| 5222 | unsigned int memflags; | ||
| 5223 | |||
| 5224 | memflags = blk_mq_freeze_queue(disk->queue); | ||
| 5225 | blk_mq_quiesce_queue_nowait(disk->queue); | ||
| 5226 | blk_mq_unfreeze_queue(disk->queue, memflags); | ||
| 5227 | } | ||
| 5228 | |||
| 5229 | static void ublk_unquiesce_and_resume(struct gendisk *disk) | ||
| 5230 | { | ||
| 5231 | blk_mq_unquiesce_queue(disk->queue); | ||
| 5232 | } | ||
| 5233 | |||
| 5234 | /* Erase coalesced PFN ranges from the maple tree for pages [0, nr_pages) */ | ||
| 5235 | static void ublk_buf_erase_ranges(struct ublk_device *ub, | ||
| 5236 | struct ublk_buf *ubuf, | ||
| 5237 | unsigned long nr_pages) | ||
| 5238 | { | ||
| 5239 | unsigned long i; | ||
| 5240 | |||
| 5241 | for (i = 0; i < nr_pages; ) { | ||
| 5242 | unsigned long pfn = page_to_pfn(ubuf->pages[i]); | ||
| 5243 | unsigned long start = i; | ||
| 5244 | |||
| 5245 | while (i + 1 < nr_pages && | ||
| 5246 | page_to_pfn(ubuf->pages[i + 1]) == pfn + (i - start) + 1) | ||
| 5247 | i++; | ||
| 5248 | i++; | ||
| 5249 | kfree(mtree_erase(&ub->buf_tree, pfn)); | ||
| 5250 | } | ||
| 5251 | } | ||
| 5252 | |||
| 5253 | static int __ublk_ctrl_reg_buf(struct ublk_device *ub, | ||
| 5254 | struct ublk_buf *ubuf, int index, | ||
| 5255 | unsigned short flags) | ||
| 5256 | { | ||
| 5257 | unsigned long nr_pages = ubuf->nr_pages; | ||
| 5258 | unsigned long i; | ||
| 5259 | int ret; | ||
| 5260 | |||
| 5261 | for (i = 0; i < nr_pages; ) { | ||
| 5262 | unsigned long pfn = page_to_pfn(ubuf->pages[i]); | ||
| 5263 | unsigned long start = i; | ||
| 5264 | struct ublk_buf_range *range; | ||
| 5265 | |||
| 5266 | /* Find run of consecutive PFNs */ | ||
| 5267 | while (i + 1 < nr_pages && | ||
| 5268 | page_to_pfn(ubuf->pages[i + 1]) == pfn + (i - start) + 1) | ||
| 5269 | i++; | ||
| 5270 | i++; /* past the last page in this run */ | ||
| 5271 | |||
| 5272 | range = kzalloc(sizeof(*range), GFP_KERNEL); | ||
| 5273 | if (!range) { | ||
| 5274 | ret = -ENOMEM; | ||
| 5275 | goto unwind; | ||
| 5276 | } | ||
| 5277 | range->buf_index = index; | ||
| 5278 | range->flags = flags; | ||
| 5279 | range->base_pfn = pfn; | ||
| 5280 | range->base_offset = start << PAGE_SHIFT; | ||
| 5281 | |||
| 5282 | ret = mtree_insert_range(&ub->buf_tree, pfn, | ||
| 5283 | pfn + (i - start) - 1, | ||
| 5284 | range, GFP_KERNEL); | ||
| 5285 | if (ret) { | ||
| 5286 | kfree(range); | ||
| 5287 | goto unwind; | ||
| 5288 | } | ||
| 5289 | } | ||
| 5290 | return 0; | ||
| 5291 | |||
| 5292 | unwind: | ||
| 5293 | ublk_buf_erase_ranges(ub, ubuf, i); | ||
| 5294 | return ret; | ||
| 5295 | } | ||
| 5296 | |||
| 5297 | /* | ||
| 5298 | * Register a shared memory buffer for zero-copy I/O. | ||
| 5299 | * Pins pages, builds PFN maple tree, freezes/unfreezes the queue | ||
| 5300 | * internally. Returns buffer index (>= 0) on success. | ||
| 5301 | */ | ||
| 5302 | static int ublk_ctrl_reg_buf(struct ublk_device *ub, | ||
| 5303 | struct ublksrv_ctrl_cmd *header) | ||
| 5304 | { | ||
| 5305 | void __user *argp = (void __user *)(unsigned long)header->addr; | ||
| 5306 | struct ublk_shmem_buf_reg buf_reg; | ||
| 5307 | unsigned long addr, size, nr_pages; | ||
| 5308 | unsigned int gup_flags; | ||
| 5309 | struct gendisk *disk; | ||
| 5310 | struct ublk_buf *ubuf; | ||
| 5311 | long pinned; | ||
| 5312 | u32 index; | ||
| 5313 | int ret; | ||
| 5314 | |||
| 5315 | if (!ublk_dev_support_shmem_zc(ub)) | ||
| 5316 | return -EOPNOTSUPP; | ||
| 5317 | |||
| 5318 | memset(&buf_reg, 0, sizeof(buf_reg)); | ||
| 5319 | if (copy_from_user(&buf_reg, argp, | ||
| 5320 | min_t(size_t, header->len, sizeof(buf_reg)))) | ||
| 5321 | return -EFAULT; | ||
| 5322 | |||
| 5323 | if (buf_reg.flags & ~UBLK_SHMEM_BUF_READ_ONLY) | ||
| 5324 | return -EINVAL; | ||
| 5325 | |||
| 5326 | addr = buf_reg.addr; | ||
| 5327 | size = buf_reg.len; | ||
| 5328 | nr_pages = size >> PAGE_SHIFT; | ||
| 5329 | |||
| 5330 | if (!size || !PAGE_ALIGNED(size) || !PAGE_ALIGNED(addr)) | ||
| 5331 | return -EINVAL; | ||
| 5332 | |||
| 5333 | disk = ublk_get_disk(ub); | ||
| 5334 | if (!disk) | ||
| 5335 | return -ENODEV; | ||
| 5336 | |||
| 5337 | /* Pin pages before quiescing (may sleep) */ | ||
| 5338 | ubuf = kzalloc(sizeof(*ubuf), GFP_KERNEL); | ||
| 5339 | if (!ubuf) { | ||
| 5340 | ret = -ENOMEM; | ||
| 5341 | goto put_disk; | ||
| 5342 | } | ||
| 5343 | |||
| 5344 | ubuf->pages = kvmalloc_array(nr_pages, sizeof(*ubuf->pages), | ||
| 5345 | GFP_KERNEL); | ||
| 5346 | if (!ubuf->pages) { | ||
| 5347 | ret = -ENOMEM; | ||
| 5348 | goto err_free; | ||
| 5349 | } | ||
| 5350 | |||
| 5351 | gup_flags = FOLL_LONGTERM; | ||
| 5352 | if (!(buf_reg.flags & UBLK_SHMEM_BUF_READ_ONLY)) | ||
| 5353 | gup_flags |= FOLL_WRITE; | ||
| 5354 | |||
| 5355 | pinned = pin_user_pages_fast(addr, nr_pages, gup_flags, ubuf->pages); | ||
| 5356 | if (pinned < 0) { | ||
| 5357 | ret = pinned; | ||
| 5358 | goto err_free_pages; | ||
| 5359 | } | ||
| 5360 | if (pinned != nr_pages) { | ||
| 5361 | ret = -EFAULT; | ||
| 5362 | goto err_unpin; | ||
| 5363 | } | ||
| 5364 | ubuf->nr_pages = nr_pages; | ||
| 5365 | |||
| 5366 | /* | ||
| 5367 | * Drain inflight I/O and quiesce the queue so no new requests | ||
| 5368 | * are dispatched while we modify the maple tree. Keep freeze | ||
| 5369 | * and mutex non-nested to avoid lock dependency. | ||
| 5370 | */ | ||
| 5371 | ublk_quiesce_and_release(disk); | ||
| 5372 | |||
| 5373 | mutex_lock(&ub->mutex); | ||
| 5374 | |||
| 5375 | ret = xa_alloc(&ub->bufs_xa, &index, ubuf, xa_limit_16b, GFP_KERNEL); | ||
| 5376 | if (ret) | ||
| 5377 | goto err_unlock; | ||
| 5378 | |||
| 5379 | ret = __ublk_ctrl_reg_buf(ub, ubuf, index, buf_reg.flags); | ||
| 5380 | if (ret) { | ||
| 5381 | xa_erase(&ub->bufs_xa, index); | ||
| 5382 | goto err_unlock; | ||
| 5383 | } | ||
| 5384 | |||
| 5385 | mutex_unlock(&ub->mutex); | ||
| 5386 | |||
| 5387 | ublk_unquiesce_and_resume(disk); | ||
| 5388 | ublk_put_disk(disk); | ||
| 5389 | return index; | ||
| 5390 | |||
| 5391 | err_unlock: | ||
| 5392 | mutex_unlock(&ub->mutex); | ||
| 5393 | ublk_unquiesce_and_resume(disk); | ||
| 5394 | err_unpin: | ||
| 5395 | unpin_user_pages(ubuf->pages, pinned); | ||
| 5396 | err_free_pages: | ||
| 5397 | kvfree(ubuf->pages); | ||
| 5398 | err_free: | ||
| 5399 | kfree(ubuf); | ||
| 5400 | put_disk: | ||
| 5401 | ublk_put_disk(disk); | ||
| 5402 | return ret; | ||
| 5403 | } | ||
| 5404 | |||
| 5405 | static void __ublk_ctrl_unreg_buf(struct ublk_device *ub, | ||
| 5406 | struct ublk_buf *ubuf) | ||
| 5407 | { | ||
| 5408 | ublk_buf_erase_ranges(ub, ubuf, ubuf->nr_pages); | ||
| 5409 | unpin_user_pages(ubuf->pages, ubuf->nr_pages); | ||
| 5410 | kvfree(ubuf->pages); | ||
| 5411 | kfree(ubuf); | ||
| 5412 | } | ||
| 5413 | |||
| 5414 | static int ublk_ctrl_unreg_buf(struct ublk_device *ub, | ||
| 5415 | struct ublksrv_ctrl_cmd *header) | ||
| 5416 | { | ||
| 5417 | int index = (int)header->data[0]; | ||
| 5418 | struct gendisk *disk; | ||
| 5419 | struct ublk_buf *ubuf; | ||
| 5420 | |||
| 5421 | if (!ublk_dev_support_shmem_zc(ub)) | ||
| 5422 | return -EOPNOTSUPP; | ||
| 5423 | |||
| 5424 | disk = ublk_get_disk(ub); | ||
| 5425 | if (!disk) | ||
| 5426 | return -ENODEV; | ||
| 5427 | |||
| 5428 | /* Drain inflight I/O before modifying the maple tree */ | ||
| 5429 | ublk_quiesce_and_release(disk); | ||
| 5430 | |||
| 5431 | mutex_lock(&ub->mutex); | ||
| 5432 | |||
| 5433 | ubuf = xa_erase(&ub->bufs_xa, index); | ||
| 5434 | if (!ubuf) { | ||
| 5435 | mutex_unlock(&ub->mutex); | ||
| 5436 | ublk_unquiesce_and_resume(disk); | ||
| 5437 | ublk_put_disk(disk); | ||
| 5438 | return -ENOENT; | ||
| 5439 | } | ||
| 5440 | |||
| 5441 | __ublk_ctrl_unreg_buf(ub, ubuf); | ||
| 5442 | |||
| 5443 | mutex_unlock(&ub->mutex); | ||
| 5444 | |||
| 5445 | ublk_unquiesce_and_resume(disk); | ||
| 5446 | ublk_put_disk(disk); | ||
| 5447 | return 0; | ||
| 5448 | } | ||
| 5449 | |||
| 5450 | static void ublk_buf_cleanup(struct ublk_device *ub) | ||
| 5451 | { | ||
| 5452 | struct ublk_buf *ubuf; | ||
| 5453 | unsigned long index; | ||
| 5454 | |||
| 5455 | xa_for_each(&ub->bufs_xa, index, ubuf) | ||
| 5456 | __ublk_ctrl_unreg_buf(ub, ubuf); | ||
| 5457 | xa_destroy(&ub->bufs_xa); | ||
| 5458 | mtree_destroy(&ub->buf_tree); | ||
| 5459 | } | ||
| 5460 | |||
| 5461 | |||
| 5462 | |||
| 5176 | static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, | 5463 | static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, |
| 5177 | u32 cmd_op, struct ublksrv_ctrl_cmd *header) | 5464 | u32 cmd_op, struct ublksrv_ctrl_cmd *header) |
| 5178 | { | 5465 | { |
| @@ -5230,6 +5517,8 @@ static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, · 51 unmodified lines | |||
| 5230 | case UBLK_CMD_UPDATE_SIZE: | 5517 | case UBLK_CMD_UPDATE_SIZE: |
| 5231 | case UBLK_CMD_QUIESCE_DEV: | 5518 | case UBLK_CMD_QUIESCE_DEV: |
| 5232 | case UBLK_CMD_TRY_STOP_DEV: | 5519 | case UBLK_CMD_TRY_STOP_DEV: |
| 5520 | case UBLK_CMD_REG_BUF: | ||
| 5521 | case UBLK_CMD_UNREG_BUF: | ||
| 5233 | mask = MAY_READ | MAY_WRITE; | 5522 | mask = MAY_READ | MAY_WRITE; |
| 5234 | break; | 5523 | break; |
| 5235 | default: | 5524 | default: |
| @@ -5355,6 +5644,12 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, · 119 unmodified lines | |||
| 5355 | case UBLK_CMD_TRY_STOP_DEV: | 5644 | case UBLK_CMD_TRY_STOP_DEV: |
| 5356 | ret = ublk_ctrl_try_stop_dev(ub); | 5645 | ret = ublk_ctrl_try_stop_dev(ub); |
| 5357 | break; | 5646 | break; |
| 5647 | case UBLK_CMD_REG_BUF: | ||
| 5648 | ret = ublk_ctrl_reg_buf(ub, &header); | ||
| 5649 | break; | ||
| 5650 | case UBLK_CMD_UNREG_BUF: | ||
| 5651 | ret = ublk_ctrl_unreg_buf(ub, &header); | ||
| 5652 | break; | ||
| 5358 | default: | 5653 | default: |
| 5359 | ret = -EOPNOTSUPP; | 5654 | ret = -EOPNOTSUPP; |
| 5360 | break; | 5655 | break; |
| @@ -57,6 +57,44 @@ · 56 unmodified lines | |||
| 57 | _IOWR('u', 0x16, struct ublksrv_ctrl_cmd) | 57 | _IOWR('u', 0x16, struct ublksrv_ctrl_cmd) |
| 58 | #define UBLK_U_CMD_TRY_STOP_DEV \ | 58 | #define UBLK_U_CMD_TRY_STOP_DEV \ |
| 59 | _IOWR('u', 0x17, struct ublksrv_ctrl_cmd) | 59 | _IOWR('u', 0x17, struct ublksrv_ctrl_cmd) |
| 60 | /* | ||
| 61 | * Register a shared memory buffer for zero-copy I/O. | ||
| 62 | * Input: ctrl_cmd.addr points to struct ublk_shmem_buf_reg (buffer VA + size) | ||
| 63 | * ctrl_cmd.len = sizeof(struct ublk_shmem_buf_reg) | ||
| 64 | * Result: >= 0 is the assigned buffer index, < 0 is error | ||
| 65 | * | ||
| 66 | * The kernel pins pages from the calling process's address space | ||
| 67 | * and inserts PFN ranges into a per-device maple tree. When a block | ||
| 68 | * request's pages match registered pages, the driver sets | ||
| 69 | * UBLK_IO_F_SHMEM_ZC and encodes the buffer index + offset in addr, | ||
| 70 | * allowing the server to access the data via its own mapping of the | ||
| 71 | * same shared memory — true zero copy. | ||
| 72 | * | ||
| 73 | * The memory can be backed by memfd, hugetlbfs, or any GUP-compatible | ||
| 74 | * shared mapping. Queue freeze is handled internally. | ||
| 75 | * | ||
| 76 | * The buffer VA and size are passed via a user buffer (not inline in | ||
| 77 | * ctrl_cmd) so that unprivileged devices can prepend the device path | ||
| 78 | * to ctrl_cmd.addr without corrupting the VA. | ||
| 79 | */ | ||
| 80 | #define UBLK_U_CMD_REG_BUF \ | ||
| 81 | _IOWR('u', 0x18, struct ublksrv_ctrl_cmd) | ||
| 82 | /* | ||
| 83 | * Unregister a shared memory buffer. | ||
| 84 | * Input: ctrl_cmd.data[0] = buffer index | ||
| 85 | */ | ||
| 86 | #define UBLK_U_CMD_UNREG_BUF \ | ||
| 87 | _IOWR('u', 0x19, struct ublksrv_ctrl_cmd) | ||
| 88 | |||
| 89 | /* Parameter buffer for UBLK_U_CMD_REG_BUF, pointed to by ctrl_cmd.addr */ | ||
| 90 | struct ublk_shmem_buf_reg { | ||
| 91 | __u64 addr; /* userspace virtual address of shared memory */ | ||
| 92 | __u32 len; /* buffer size in bytes (page-aligned, max 4GB) */ | ||
| 93 | __u32 flags; | ||
| 94 | }; | ||
| 95 | |||
| 96 | /* Pin pages without FOLL_WRITE; usable with write-sealed memfd */ | ||
| 97 | #define UBLK_SHMEM_BUF_READ_ONLY (1U << 0) | ||
| 60 | /* | 98 | /* |
| 61 | * 64bits are enough now, and it should be easy to extend in case of | 99 | * 64bits are enough now, and it should be easy to extend in case of |
| 62 | * running out of feature flags | 100 | * running out of feature flags |
| @@ -370,6 +408,7 @@ · 307 unmodified lines | |||
| 370 | /* Disable automatic partition scanning when device is started */ | 408 | /* Disable automatic partition scanning when device is started */ |
| 371 | #define UBLK_F_NO_AUTO_PART_SCAN (1ULL << 18) | 409 | #define UBLK_F_NO_AUTO_PART_SCAN (1ULL << 18) |
| 372 | 410 | ||
| 411 | |||
| 373 | /* device state */ | 412 | /* device state */ |
| 374 | #define UBLK_S_DEV_DEAD 0 | 413 | #define UBLK_S_DEV_DEAD 0 |
| 375 | #define UBLK_S_DEV_LIVE 1 | 414 | #define UBLK_S_DEV_LIVE 1 |
| @@ -469,6 +508,12 @@ struct ublksrv_ctrl_dev_info { · 93 unmodified lines | |||
| 469 | #define UBLK_IO_F_NEED_REG_BUF (1U << 17) | 508 | #define UBLK_IO_F_NEED_REG_BUF (1U << 17) |
| 470 | /* Request has an integrity data buffer */ | 509 | /* Request has an integrity data buffer */ |
| 471 | #define UBLK_IO_F_INTEGRITY (1UL << 18) | 510 | #define UBLK_IO_F_INTEGRITY (1UL << 18) |
| 511 | /* | ||
| 512 | * I/O buffer is in a registered shared memory buffer. When set, the addr | ||
| 513 | * field in ublksrv_io_desc encodes buffer index and byte offset instead | ||
| 514 | * of a userspace virtual address. | ||
| 515 | */ | ||
| 516 | #define UBLK_IO_F_SHMEM_ZC (1U << 19) | ||
| 472 | 517 | ||
| 473 | /* | 518 | /* |
| 474 | * io cmd is described by this structure, and stored in share memory, indexed | 519 | * io cmd is described by this structure, and stored in share memory, indexed |
| @@ -743,4 +788,31 @@ struct ublk_params { · 268 unmodified lines | |||
| 743 | struct ublk_param_integrity integrity; | 788 | struct ublk_param_integrity integrity; |
| 744 | }; | 789 | }; |
| 745 | 790 | ||
| 791 | /* | ||
| 792 | * Shared memory zero-copy addr encoding for UBLK_IO_F_SHMEM_ZC. | ||
| 793 | * | ||
| 794 | * When UBLK_IO_F_SHMEM_ZC is set, ublksrv_io_desc.addr is encoded as: | ||
| 795 | * bits [0:31] = byte offset within the buffer (up to 4GB) | ||
| 796 | * bits [32:47] = buffer index (up to 65536) | ||
| 797 | * bits [48:63] = reserved (must be zero) | ||
| 798 | */ | ||
| 799 | #define UBLK_SHMEM_ZC_OFF_MASK 0xffffffffULL | ||
| 800 | #define UBLK_SHMEM_ZC_IDX_OFF 32 | ||
| 801 | #define UBLK_SHMEM_ZC_IDX_MASK 0xffffULL | ||
| 802 | |||
| 803 | static inline __u64 ublk_shmem_zc_addr(__u16 index, __u32 offset) | ||
| 804 | { | ||
| 805 | return ((__u64)index << UBLK_SHMEM_ZC_IDX_OFF) | offset; | ||
| 806 | } | ||
| 807 | |||
| 808 | static inline __u16 ublk_shmem_zc_index(__u64 addr) | ||
| 809 | { | ||
| 810 | return (addr >> UBLK_SHMEM_ZC_IDX_OFF) & UBLK_SHMEM_ZC_IDX_MASK; | ||
| 811 | } | ||
| 812 | |||
| 813 | static inline __u32 ublk_shmem_zc_offset(__u64 addr) | ||
| 814 | { | ||
| 815 | return (__u32)(addr & UBLK_SHMEM_ZC_OFF_MASK); | ||
| 816 | } | ||
| 817 | |||
| 746 | #endif | 818 | #endif |