History

io_uring: add support for 128-byte SQEs

Normal SQEs are 64-bytes in length, which is fine for all the commands we support. However, in preparation for supporting passthrough IO, provide an option for setting up a ring with 128-byte SQEs. We continue to use the same type for io_uring_sqe, it's marked and commented with a zero sized array pad at the end. This provides up to 80 bytes of data for a passthrough command - 64 bytes for the extra added data, and 16 bytes available at the end of the existing SQE.

Jens Axboe committed 4 years ago

Commit ebdeb7c1 parent b5ba65d2 files · +19 −3Browse files

Unified · Split

2 files
  1. fs/io_uring.c +11 −3
  2. include/uapi/linux/io_uring.h +8 −0
fs/io_uring.c+11−3
@@ -8519,8 +8519,12 @@ static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) · 8518 unmodified lines
8519 * though the application is the one updating it.8519 * though the application is the one updating it.
8520 */8520 */
8521 head = READ_ONCE(ctx->sq_array[sq_idx]);8521 head = READ_ONCE(ctx->sq_array[sq_idx]);
8522 if (likely(head < ctx->sq_entries))8522 if (likely(head < ctx->sq_entries)) {
8523 /* double index for 128-byte SQEs, twice as long */
8524 if (ctx->flags & IORING_SETUP_SQE128)
8525 head <<= 1;
8523 return &ctx->sq_sqes[head];8526 return &ctx->sq_sqes[head];
8527 }
8524 8528
8525 /* drop invalid entries */8529 /* drop invalid entries */
8526 ctx->cq_extra--;8530 ctx->cq_extra--;
@@ -11689,7 +11693,10 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, · 3162 unmodified lines
11689 rings->sq_ring_entries = p->sq_entries;11693 rings->sq_ring_entries = p->sq_entries;
11690 rings->cq_ring_entries = p->cq_entries;11694 rings->cq_ring_entries = p->cq_entries;
11691 11695
11692 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);11696 if (p->flags & IORING_SETUP_SQE128)
11697 size = array_size(2 * sizeof(struct io_uring_sqe), p->sq_entries);
11698 else
11699 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
11693 if (size == SIZE_MAX) {11700 if (size == SIZE_MAX) {
11694 io_mem_free(ctx->rings);11701 io_mem_free(ctx->rings);
11695 ctx->rings = NULL;11702 ctx->rings = NULL;
@@ -11933,7 +11940,8 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) · 237 unmodified lines
11933 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |11940 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
11934 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |11941 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
11935 IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL |11942 IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL |
11936 IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG))11943 IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG |
11944 IORING_SETUP_SQE128))
11937 return -EINVAL;11945 return -EINVAL;
11938 11946
11939 return io_uring_create(entries, &p, params);11947 return io_uring_create(entries, &p, params);
include/uapi/linux/io_uring.h+8−0
@@ -63,6 +63,12 @@ struct io_uring_sqe { · 62 unmodified lines
63 };63 };
64 __u64 addr3;64 __u64 addr3;
65 __u64 __pad2[1];65 __u64 __pad2[1];
66
67 /*
68 * If the ring is initialized with IORING_SETUP_SQE128, then this field
69 * contains 64-bytes of padding, doubling the size of the SQE.
70 */
71 __u64 __big_sqe_pad[0];
66};72};
67 73
68enum {74enum {
@@ -119,6 +125,8 @@ enum { · 50 unmodified lines
119 */125 */
120#define IORING_SETUP_TASKRUN_FLAG (1U << 9)126#define IORING_SETUP_TASKRUN_FLAG (1U << 9)
121 127
128#define IORING_SETUP_SQE128 (1U << 10) /* SQEs are 128 byte */
129
122enum io_uring_op {130enum io_uring_op {
123 IORING_OP_NOP,131 IORING_OP_NOP,
124 IORING_OP_READV,132 IORING_OP_READV,