History

ublk: add UBLK_U_CMD_REG_BUF/UNREG_BUF control commands

Add control commands for registering and unregistering shared memory buffers for zero-copy I/O: - UBLK_U_CMD_REG_BUF (0x18): pins pages from userspace, inserts PFN ranges into a per-device maple tree for O(log n) lookup during I/O. Buffer pointers are tracked in a per-device xarray. Returns the assigned buffer index. - UBLK_U_CMD_UNREG_BUF (0x19): removes PFN entries and unpins pages. Queue freeze/unfreeze is handled internally so userspace need not quiesce the device during registration. Also adds: - UBLK_IO_F_SHMEM_ZC flag and addr encoding helpers in UAPI header (16-bit buffer index supporting up to 65536 buffers) - Data structures (ublk_buf, ublk_buf_range) and xarray/maple tree - __ublk_ctrl_reg_buf() helper for PFN insertion with error unwinding - __ublk_ctrl_unreg_buf() helper for cleanup reuse - ublk_support_shmem_zc() / ublk_dev_support_shmem_zc() stubs (returning false — feature not enabled yet)

Ming Lei committed 7 months ago

Commit 2fb0ded1 parent fa0cac92 files · +367 −0Browse files

Unified · Split

2 files
  1. drivers/block/ublk_drv.c +295 −0
  2. include/uapi/linux/ublk_cmd.h +72 −0
drivers/block/ublk_drv.c+295−0
@@ -46,6 +46,8 @@ · 45 unmodified lines
46#include <linux/kref.h>46#include <linux/kref.h>
47#include <linux/kfifo.h>47#include <linux/kfifo.h>
48#include <linux/blk-integrity.h>48#include <linux/blk-integrity.h>
49#include <linux/maple_tree.h>
50#include <linux/xarray.h>
49#include <uapi/linux/fs.h>51#include <uapi/linux/fs.h>
50#include <uapi/linux/ublk_cmd.h>52#include <uapi/linux/ublk_cmd.h>
51 53
@@ -58,6 +60,8 @@ · 6 unmodified lines
58#define UBLK_CMD_UPDATE_SIZE _IOC_NR(UBLK_U_CMD_UPDATE_SIZE)60#define UBLK_CMD_UPDATE_SIZE _IOC_NR(UBLK_U_CMD_UPDATE_SIZE)
59#define UBLK_CMD_QUIESCE_DEV _IOC_NR(UBLK_U_CMD_QUIESCE_DEV)61#define UBLK_CMD_QUIESCE_DEV _IOC_NR(UBLK_U_CMD_QUIESCE_DEV)
60#define UBLK_CMD_TRY_STOP_DEV _IOC_NR(UBLK_U_CMD_TRY_STOP_DEV)62#define UBLK_CMD_TRY_STOP_DEV _IOC_NR(UBLK_U_CMD_TRY_STOP_DEV)
63#define UBLK_CMD_REG_BUF _IOC_NR(UBLK_U_CMD_REG_BUF)
64#define UBLK_CMD_UNREG_BUF _IOC_NR(UBLK_U_CMD_UNREG_BUF)
61 65
62#define UBLK_IO_REGISTER_IO_BUF _IOC_NR(UBLK_U_IO_REGISTER_IO_BUF)66#define UBLK_IO_REGISTER_IO_BUF _IOC_NR(UBLK_U_IO_REGISTER_IO_BUF)
63#define UBLK_IO_UNREGISTER_IO_BUF _IOC_NR(UBLK_U_IO_UNREGISTER_IO_BUF)67#define UBLK_IO_UNREGISTER_IO_BUF _IOC_NR(UBLK_U_IO_UNREGISTER_IO_BUF)
@@ -289,6 +293,20 @@ struct ublk_queue { · 225 unmodified lines
289 struct ublk_io ios[] __counted_by(q_depth);293 struct ublk_io ios[] __counted_by(q_depth);
290};294};
291 295
296/* Per-registered shared memory buffer */
297struct ublk_buf {
298 struct page **pages;
299 unsigned int nr_pages;
300};
301
302/* Maple tree value: maps a PFN range to buffer location */
303struct ublk_buf_range {
304 unsigned long base_pfn;
305 unsigned short buf_index;
306 unsigned short flags;
307 unsigned int base_offset; /* byte offset within buffer */
308};
309
292struct ublk_device {310struct ublk_device {
293 struct gendisk *ub_disk;311 struct gendisk *ub_disk;
294 312
@@ -323,6 +341,10 @@ struct ublk_device { · 28 unmodified lines
323 341
324 bool block_open; /* protected by open_mutex */342 bool block_open; /* protected by open_mutex */
325 343
344 /* shared memory zero copy */
345 struct maple_tree buf_tree;
346 struct xarray bufs_xa;
347
326 struct ublk_queue *queues[];348 struct ublk_queue *queues[];
327};349};
328 350
@@ -334,6 +356,7 @@ struct ublk_params_header { · 5 unmodified lines
334 356
335static void ublk_io_release(void *priv);357static void ublk_io_release(void *priv);
336static void ublk_stop_dev_unlocked(struct ublk_device *ub);358static void ublk_stop_dev_unlocked(struct ublk_device *ub);
359static void ublk_buf_cleanup(struct ublk_device *ub);
337static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq);360static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq);
338static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub,361static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub,
339 u16 q_id, u16 tag, struct ublk_io *io);362 u16 q_id, u16 tag, struct ublk_io *io);
@@ -398,6 +421,16 @@ static inline bool ublk_dev_support_zero_copy(const struct ublk_device *ub) · 58 unmodified lines
398 return ub->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY;421 return ub->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY;
399}422}
400 423
424static inline bool ublk_support_shmem_zc(const struct ublk_queue *ubq)
425{
426 return false;
427}
428
429static inline bool ublk_dev_support_shmem_zc(const struct ublk_device *ub)
430{
431 return false;
432}
433
401static inline bool ublk_support_auto_buf_reg(const struct ublk_queue *ubq)434static inline bool ublk_support_auto_buf_reg(const struct ublk_queue *ubq)
402{435{
403 return ubq->flags & UBLK_F_AUTO_BUF_REG;436 return ubq->flags & UBLK_F_AUTO_BUF_REG;
@@ -1460,6 +1493,7 @@ static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req) · 1056 unmodified lines
1460 iod->op_flags = ublk_op | ublk_req_build_flags(req);1493 iod->op_flags = ublk_op | ublk_req_build_flags(req);
1461 iod->nr_sectors = blk_rq_sectors(req);1494 iod->nr_sectors = blk_rq_sectors(req);
1462 iod->start_sector = blk_rq_pos(req);1495 iod->start_sector = blk_rq_pos(req);
1496
1463 iod->addr = io->buf.addr;1497 iod->addr = io->buf.addr;
1464 1498
1465 return BLK_STS_OK;1499 return BLK_STS_OK;
@@ -1665,6 +1699,7 @@ static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req, · 199 unmodified lines
1665{1699{
1666 unsigned mapped_bytes = ublk_map_io(ubq, req, io);1700 unsigned mapped_bytes = ublk_map_io(ubq, req, io);
1667 1701
1702
1668 /* partially mapped, update io descriptor */1703 /* partially mapped, update io descriptor */
1669 if (unlikely(mapped_bytes != blk_rq_bytes(req))) {1704 if (unlikely(mapped_bytes != blk_rq_bytes(req))) {
1670 /*1705 /*
@@ -4211,6 +4246,7 @@ static void ublk_cdev_rel(struct device *dev) · 2540 unmodified lines
4211{4246{
4212 struct ublk_device *ub = container_of(dev, struct ublk_device, cdev_dev);4247 struct ublk_device *ub = container_of(dev, struct ublk_device, cdev_dev);
4213 4248
4249 ublk_buf_cleanup(ub);
4214 blk_mq_free_tag_set(&ub->tag_set);4250 blk_mq_free_tag_set(&ub->tag_set);
4215 ublk_deinit_queues(ub);4251 ublk_deinit_queues(ub);
4216 ublk_free_dev_number(ub);4252 ublk_free_dev_number(ub);
@@ -4630,6 +4666,8 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) · 413 unmodified lines
4630 mutex_init(&ub->mutex);4666 mutex_init(&ub->mutex);
4631 spin_lock_init(&ub->lock);4667 spin_lock_init(&ub->lock);
4632 mutex_init(&ub->cancel_mutex);4668 mutex_init(&ub->cancel_mutex);
4669 mt_init(&ub->buf_tree);
4670 xa_init_flags(&ub->bufs_xa, XA_FLAGS_ALLOC);
4633 INIT_WORK(&ub->partition_scan_work, ublk_partition_scan_work);4671 INIT_WORK(&ub->partition_scan_work, ublk_partition_scan_work);
4634 4672
4635 ret = ublk_alloc_dev_number(ub, header->dev_id);4673 ret = ublk_alloc_dev_number(ub, header->dev_id);
@@ -5173,6 +5211,255 @@ exit: · 537 unmodified lines
5173 return err;5211 return err;
5174}5212}
5175 5213
5214/*
5215 * Drain inflight I/O and quiesce the queue. Freeze drains all inflight
5216 * requests, quiesce_nowait marks the queue so no new requests dispatch,
5217 * then unfreeze allows new submissions (which won't dispatch due to
5218 * quiesce). This keeps freeze and ub->mutex non-nested.
5219 */
5220static void ublk_quiesce_and_release(struct gendisk *disk)
5221{
5222 unsigned int memflags;
5223
5224 memflags = blk_mq_freeze_queue(disk->queue);
5225 blk_mq_quiesce_queue_nowait(disk->queue);
5226 blk_mq_unfreeze_queue(disk->queue, memflags);
5227}
5228
5229static void ublk_unquiesce_and_resume(struct gendisk *disk)
5230{
5231 blk_mq_unquiesce_queue(disk->queue);
5232}
5233
5234/* Erase coalesced PFN ranges from the maple tree for pages [0, nr_pages) */
5235static void ublk_buf_erase_ranges(struct ublk_device *ub,
5236 struct ublk_buf *ubuf,
5237 unsigned long nr_pages)
5238{
5239 unsigned long i;
5240
5241 for (i = 0; i < nr_pages; ) {
5242 unsigned long pfn = page_to_pfn(ubuf->pages[i]);
5243 unsigned long start = i;
5244
5245 while (i + 1 < nr_pages &&
5246 page_to_pfn(ubuf->pages[i + 1]) == pfn + (i - start) + 1)
5247 i++;
5248 i++;
5249 kfree(mtree_erase(&ub->buf_tree, pfn));
5250 }
5251}
5252
5253static int __ublk_ctrl_reg_buf(struct ublk_device *ub,
5254 struct ublk_buf *ubuf, int index,
5255 unsigned short flags)
5256{
5257 unsigned long nr_pages = ubuf->nr_pages;
5258 unsigned long i;
5259 int ret;
5260
5261 for (i = 0; i < nr_pages; ) {
5262 unsigned long pfn = page_to_pfn(ubuf->pages[i]);
5263 unsigned long start = i;
5264 struct ublk_buf_range *range;
5265
5266 /* Find run of consecutive PFNs */
5267 while (i + 1 < nr_pages &&
5268 page_to_pfn(ubuf->pages[i + 1]) == pfn + (i - start) + 1)
5269 i++;
5270 i++; /* past the last page in this run */
5271
5272 range = kzalloc(sizeof(*range), GFP_KERNEL);
5273 if (!range) {
5274 ret = -ENOMEM;
5275 goto unwind;
5276 }
5277 range->buf_index = index;
5278 range->flags = flags;
5279 range->base_pfn = pfn;
5280 range->base_offset = start << PAGE_SHIFT;
5281
5282 ret = mtree_insert_range(&ub->buf_tree, pfn,
5283 pfn + (i - start) - 1,
5284 range, GFP_KERNEL);
5285 if (ret) {
5286 kfree(range);
5287 goto unwind;
5288 }
5289 }
5290 return 0;
5291
5292unwind:
5293 ublk_buf_erase_ranges(ub, ubuf, i);
5294 return ret;
5295}
5296
5297/*
5298 * Register a shared memory buffer for zero-copy I/O.
5299 * Pins pages, builds PFN maple tree, freezes/unfreezes the queue
5300 * internally. Returns buffer index (>= 0) on success.
5301 */
5302static int ublk_ctrl_reg_buf(struct ublk_device *ub,
5303 struct ublksrv_ctrl_cmd *header)
5304{
5305 void __user *argp = (void __user *)(unsigned long)header->addr;
5306 struct ublk_shmem_buf_reg buf_reg;
5307 unsigned long addr, size, nr_pages;
5308 unsigned int gup_flags;
5309 struct gendisk *disk;
5310 struct ublk_buf *ubuf;
5311 long pinned;
5312 u32 index;
5313 int ret;
5314
5315 if (!ublk_dev_support_shmem_zc(ub))
5316 return -EOPNOTSUPP;
5317
5318 memset(&buf_reg, 0, sizeof(buf_reg));
5319 if (copy_from_user(&buf_reg, argp,
5320 min_t(size_t, header->len, sizeof(buf_reg))))
5321 return -EFAULT;
5322
5323 if (buf_reg.flags & ~UBLK_SHMEM_BUF_READ_ONLY)
5324 return -EINVAL;
5325
5326 addr = buf_reg.addr;
5327 size = buf_reg.len;
5328 nr_pages = size >> PAGE_SHIFT;
5329
5330 if (!size || !PAGE_ALIGNED(size) || !PAGE_ALIGNED(addr))
5331 return -EINVAL;
5332
5333 disk = ublk_get_disk(ub);
5334 if (!disk)
5335 return -ENODEV;
5336
5337 /* Pin pages before quiescing (may sleep) */
5338 ubuf = kzalloc(sizeof(*ubuf), GFP_KERNEL);
5339 if (!ubuf) {
5340 ret = -ENOMEM;
5341 goto put_disk;
5342 }
5343
5344 ubuf->pages = kvmalloc_array(nr_pages, sizeof(*ubuf->pages),
5345 GFP_KERNEL);
5346 if (!ubuf->pages) {
5347 ret = -ENOMEM;
5348 goto err_free;
5349 }
5350
5351 gup_flags = FOLL_LONGTERM;
5352 if (!(buf_reg.flags & UBLK_SHMEM_BUF_READ_ONLY))
5353 gup_flags |= FOLL_WRITE;
5354
5355 pinned = pin_user_pages_fast(addr, nr_pages, gup_flags, ubuf->pages);
5356 if (pinned < 0) {
5357 ret = pinned;
5358 goto err_free_pages;
5359 }
5360 if (pinned != nr_pages) {
5361 ret = -EFAULT;
5362 goto err_unpin;
5363 }
5364 ubuf->nr_pages = nr_pages;
5365
5366 /*
5367 * Drain inflight I/O and quiesce the queue so no new requests
5368 * are dispatched while we modify the maple tree. Keep freeze
5369 * and mutex non-nested to avoid lock dependency.
5370 */
5371 ublk_quiesce_and_release(disk);
5372
5373 mutex_lock(&ub->mutex);
5374
5375 ret = xa_alloc(&ub->bufs_xa, &index, ubuf, xa_limit_16b, GFP_KERNEL);
5376 if (ret)
5377 goto err_unlock;
5378
5379 ret = __ublk_ctrl_reg_buf(ub, ubuf, index, buf_reg.flags);
5380 if (ret) {
5381 xa_erase(&ub->bufs_xa, index);
5382 goto err_unlock;
5383 }
5384
5385 mutex_unlock(&ub->mutex);
5386
5387 ublk_unquiesce_and_resume(disk);
5388 ublk_put_disk(disk);
5389 return index;
5390
5391err_unlock:
5392 mutex_unlock(&ub->mutex);
5393 ublk_unquiesce_and_resume(disk);
5394err_unpin:
5395 unpin_user_pages(ubuf->pages, pinned);
5396err_free_pages:
5397 kvfree(ubuf->pages);
5398err_free:
5399 kfree(ubuf);
5400put_disk:
5401 ublk_put_disk(disk);
5402 return ret;
5403}
5404
5405static void __ublk_ctrl_unreg_buf(struct ublk_device *ub,
5406 struct ublk_buf *ubuf)
5407{
5408 ublk_buf_erase_ranges(ub, ubuf, ubuf->nr_pages);
5409 unpin_user_pages(ubuf->pages, ubuf->nr_pages);
5410 kvfree(ubuf->pages);
5411 kfree(ubuf);
5412}
5413
5414static int ublk_ctrl_unreg_buf(struct ublk_device *ub,
5415 struct ublksrv_ctrl_cmd *header)
5416{
5417 int index = (int)header->data[0];
5418 struct gendisk *disk;
5419 struct ublk_buf *ubuf;
5420
5421 if (!ublk_dev_support_shmem_zc(ub))
5422 return -EOPNOTSUPP;
5423
5424 disk = ublk_get_disk(ub);
5425 if (!disk)
5426 return -ENODEV;
5427
5428 /* Drain inflight I/O before modifying the maple tree */
5429 ublk_quiesce_and_release(disk);
5430
5431 mutex_lock(&ub->mutex);
5432
5433 ubuf = xa_erase(&ub->bufs_xa, index);
5434 if (!ubuf) {
5435 mutex_unlock(&ub->mutex);
5436 ublk_unquiesce_and_resume(disk);
5437 ublk_put_disk(disk);
5438 return -ENOENT;
5439 }
5440
5441 __ublk_ctrl_unreg_buf(ub, ubuf);
5442
5443 mutex_unlock(&ub->mutex);
5444
5445 ublk_unquiesce_and_resume(disk);
5446 ublk_put_disk(disk);
5447 return 0;
5448}
5449
5450static void ublk_buf_cleanup(struct ublk_device *ub)
5451{
5452 struct ublk_buf *ubuf;
5453 unsigned long index;
5454
5455 xa_for_each(&ub->bufs_xa, index, ubuf)
5456 __ublk_ctrl_unreg_buf(ub, ubuf);
5457 xa_destroy(&ub->bufs_xa);
5458 mtree_destroy(&ub->buf_tree);
5459}
5460
5461
5462
5176static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub,5463static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub,
5177 u32 cmd_op, struct ublksrv_ctrl_cmd *header)5464 u32 cmd_op, struct ublksrv_ctrl_cmd *header)
5178{5465{
@@ -5230,6 +5517,8 @@ static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, · 51 unmodified lines
5230 case UBLK_CMD_UPDATE_SIZE:5517 case UBLK_CMD_UPDATE_SIZE:
5231 case UBLK_CMD_QUIESCE_DEV:5518 case UBLK_CMD_QUIESCE_DEV:
5232 case UBLK_CMD_TRY_STOP_DEV:5519 case UBLK_CMD_TRY_STOP_DEV:
5520 case UBLK_CMD_REG_BUF:
5521 case UBLK_CMD_UNREG_BUF:
5233 mask = MAY_READ | MAY_WRITE;5522 mask = MAY_READ | MAY_WRITE;
5234 break;5523 break;
5235 default:5524 default:
@@ -5355,6 +5644,12 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, · 119 unmodified lines
5355 case UBLK_CMD_TRY_STOP_DEV:5644 case UBLK_CMD_TRY_STOP_DEV:
5356 ret = ublk_ctrl_try_stop_dev(ub);5645 ret = ublk_ctrl_try_stop_dev(ub);
5357 break;5646 break;
5647 case UBLK_CMD_REG_BUF:
5648 ret = ublk_ctrl_reg_buf(ub, &header);
5649 break;
5650 case UBLK_CMD_UNREG_BUF:
5651 ret = ublk_ctrl_unreg_buf(ub, &header);
5652 break;
5358 default:5653 default:
5359 ret = -EOPNOTSUPP;5654 ret = -EOPNOTSUPP;
5360 break;5655 break;
include/uapi/linux/ublk_cmd.h+72−0
@@ -57,6 +57,44 @@ · 56 unmodified lines
57 _IOWR('u', 0x16, struct ublksrv_ctrl_cmd)57 _IOWR('u', 0x16, struct ublksrv_ctrl_cmd)
58#define UBLK_U_CMD_TRY_STOP_DEV \58#define UBLK_U_CMD_TRY_STOP_DEV \
59 _IOWR('u', 0x17, struct ublksrv_ctrl_cmd)59 _IOWR('u', 0x17, struct ublksrv_ctrl_cmd)
60/*
61 * Register a shared memory buffer for zero-copy I/O.
62 * Input: ctrl_cmd.addr points to struct ublk_shmem_buf_reg (buffer VA + size)
63 * ctrl_cmd.len = sizeof(struct ublk_shmem_buf_reg)
64 * Result: >= 0 is the assigned buffer index, < 0 is error
65 *
66 * The kernel pins pages from the calling process's address space
67 * and inserts PFN ranges into a per-device maple tree. When a block
68 * request's pages match registered pages, the driver sets
69 * UBLK_IO_F_SHMEM_ZC and encodes the buffer index + offset in addr,
70 * allowing the server to access the data via its own mapping of the
71 * same shared memory — true zero copy.
72 *
73 * The memory can be backed by memfd, hugetlbfs, or any GUP-compatible
74 * shared mapping. Queue freeze is handled internally.
75 *
76 * The buffer VA and size are passed via a user buffer (not inline in
77 * ctrl_cmd) so that unprivileged devices can prepend the device path
78 * to ctrl_cmd.addr without corrupting the VA.
79 */
80#define UBLK_U_CMD_REG_BUF \
81 _IOWR('u', 0x18, struct ublksrv_ctrl_cmd)
82/*
83 * Unregister a shared memory buffer.
84 * Input: ctrl_cmd.data[0] = buffer index
85 */
86#define UBLK_U_CMD_UNREG_BUF \
87 _IOWR('u', 0x19, struct ublksrv_ctrl_cmd)
88
89/* Parameter buffer for UBLK_U_CMD_REG_BUF, pointed to by ctrl_cmd.addr */
90struct ublk_shmem_buf_reg {
91 __u64 addr; /* userspace virtual address of shared memory */
92 __u32 len; /* buffer size in bytes (page-aligned, max 4GB) */
93 __u32 flags;
94};
95
96/* Pin pages without FOLL_WRITE; usable with write-sealed memfd */
97#define UBLK_SHMEM_BUF_READ_ONLY (1U << 0)
60/*98/*
61 * 64bits are enough now, and it should be easy to extend in case of99 * 64bits are enough now, and it should be easy to extend in case of
62 * running out of feature flags100 * running out of feature flags
@@ -370,6 +408,7 @@ · 307 unmodified lines
370/* Disable automatic partition scanning when device is started */408/* Disable automatic partition scanning when device is started */
371#define UBLK_F_NO_AUTO_PART_SCAN (1ULL << 18)409#define UBLK_F_NO_AUTO_PART_SCAN (1ULL << 18)
372 410
411
373/* device state */412/* device state */
374#define UBLK_S_DEV_DEAD 0413#define UBLK_S_DEV_DEAD 0
375#define UBLK_S_DEV_LIVE 1414#define UBLK_S_DEV_LIVE 1
@@ -469,6 +508,12 @@ struct ublksrv_ctrl_dev_info { · 93 unmodified lines
469#define UBLK_IO_F_NEED_REG_BUF (1U << 17)508#define UBLK_IO_F_NEED_REG_BUF (1U << 17)
470/* Request has an integrity data buffer */509/* Request has an integrity data buffer */
471#define UBLK_IO_F_INTEGRITY (1UL << 18)510#define UBLK_IO_F_INTEGRITY (1UL << 18)
511/*
512 * I/O buffer is in a registered shared memory buffer. When set, the addr
513 * field in ublksrv_io_desc encodes buffer index and byte offset instead
514 * of a userspace virtual address.
515 */
516#define UBLK_IO_F_SHMEM_ZC (1U << 19)
472 517
473/*518/*
474 * io cmd is described by this structure, and stored in share memory, indexed519 * io cmd is described by this structure, and stored in share memory, indexed
@@ -743,4 +788,31 @@ struct ublk_params { · 268 unmodified lines
743 struct ublk_param_integrity integrity;788 struct ublk_param_integrity integrity;
744};789};
745 790
791/*
792 * Shared memory zero-copy addr encoding for UBLK_IO_F_SHMEM_ZC.
793 *
794 * When UBLK_IO_F_SHMEM_ZC is set, ublksrv_io_desc.addr is encoded as:
795 * bits [0:31] = byte offset within the buffer (up to 4GB)
796 * bits [32:47] = buffer index (up to 65536)
797 * bits [48:63] = reserved (must be zero)
798 */
799#define UBLK_SHMEM_ZC_OFF_MASK 0xffffffffULL
800#define UBLK_SHMEM_ZC_IDX_OFF 32
801#define UBLK_SHMEM_ZC_IDX_MASK 0xffffULL
802
803static inline __u64 ublk_shmem_zc_addr(__u16 index, __u32 offset)
804{
805 return ((__u64)index << UBLK_SHMEM_ZC_IDX_OFF) | offset;
806}
807
808static inline __u16 ublk_shmem_zc_index(__u64 addr)
809{
810 return (addr >> UBLK_SHMEM_ZC_IDX_OFF) & UBLK_SHMEM_ZC_IDX_MASK;
811}
812
813static inline __u32 ublk_shmem_zc_offset(__u64 addr)
814{
815 return (__u32)(addr & UBLK_SHMEM_ZC_OFF_MASK);
816}
817
746#endif818#endif

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t