History

io_uring: support buffer selection for OP_READ and OP_RECV

If a server process has tons of pending socket connections, generally it uses epoll to wait for activity. When the socket is ready for reading (or writing), the task can select a buffer and issue a recv/send on the given fd. Now that we have fast (non-async thread) support, a task can have tons of pending reads or writes pending. But that means they need buffers to back that data, and if the number of connections is high enough, having them preallocated for all possible connections is unfeasible. With IORING_OP_PROVIDE_BUFFERS, an application can register buffers to use for any request. The request then sets IOSQE_BUFFER_SELECT in the sqe, and a given group ID in sqe->buf_group. When the fd becomes ready, a free buffer from the specified group is selected. If none are available, the request is terminated with -ENOBUFS. If successful, the CQE on completion will contain the buffer ID chosen in the cqe->flags member, encoded as: (buffer_id << IORING_CQE_BUFFER_SHIFT) | IORING_CQE_F_BUFFER; Once a buffer has been consumed by a request, it is no longer available and must be registered again with IORING_OP_PROVIDE_BUFFERS. Requests need to support this feature. For now, IORING_OP_READ and IORING_OP_RECV support it. This is checked on SQE submission, a CQE with res == -EOPNOTSUPP will be posted if attempted on unsupported requests.

Jens Axboe committed 6 years ago

Commit bcda7ba1 parent ddf03222 files · +199 −39Browse files

Unified · Split

2 files
  1. fs/io_uring.c +185 −39
  2. include/uapi/linux/io_uring.h +14 −0
fs/io_uring.c+185−39
@@ -395,7 +395,9 @@ struct io_sr_msg { · 394 unmodified lines
395395 void __user *buf;
396396 };
397397 int msg_flags;
398+ int bgid;
398399 size_t len;
400+ struct io_buffer *kbuf;
399401 };
400402
401403 struct io_open {
@@ -490,6 +492,7 @@ enum { · 88 unmodified lines
490492 REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
491493 REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
492494 REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
495+ REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,
493496
494497 REQ_F_LINK_NEXT_BIT,
495498 REQ_F_FAIL_LINK_BIT,
@@ -506,6 +509,7 @@ enum { · 10 unmodified lines
506509 REQ_F_NEED_CLEANUP_BIT,
507510 REQ_F_OVERFLOW_BIT,
508511 REQ_F_POLLED_BIT,
512+ REQ_F_BUFFER_SELECTED_BIT,
509513 };
510514
511515 enum {
@@ -519,6 +523,8 @@ enum { · 7 unmodified lines
519523 REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
520524 /* IOSQE_ASYNC */
521525 REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
526+ /* IOSQE_BUFFER_SELECT */
527+ REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT),
522528
523529 /* already grabbed next link */
524530 REQ_F_LINK_NEXT = BIT(REQ_F_LINK_NEXT_BIT),
@@ -550,6 +556,8 @@ enum { · 25 unmodified lines
550556 REQ_F_OVERFLOW = BIT(REQ_F_OVERFLOW_BIT),
551557 /* already went through poll handler */
552558 REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),
559+ /* buffer already selected */
560+ REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
553561 };
554562
555563 struct async_poll {
@@ -612,6 +620,7 @@ struct io_kiocb { · 56 unmodified lines
612620 struct callback_head task_work;
613621 struct hlist_node hash_node;
614622 struct async_poll *apoll;
623+ int cflags;
615624 };
616625 struct io_wq_work work;
617626 };
@@ -661,6 +670,8 @@ struct io_op_def { · 43 unmodified lines
661670 /* set if opcode supports polled "wait" */
662671 unsigned pollin : 1;
663672 unsigned pollout : 1;
673+ /* op supports buffer selection */
674+ unsigned buffer_select : 1;
664675 };
665676
666677 static const struct io_op_def io_op_defs[] = {
@@ -770,6 +781,7 @@ static const struct io_op_def io_op_defs[] = { · 103 unmodified lines
770781 .needs_file = 1,
771782 .unbound_nonreg_file = 1,
772783 .pollin = 1,
784+ .buffer_select = 1,
773785 },
774786 [IORING_OP_WRITE] = {
775787 .needs_mm = 1,
@@ -794,6 +806,7 @@ static const struct io_op_def io_op_defs[] = { · 18 unmodified lines
794806 .needs_file = 1,
795807 .unbound_nonreg_file = 1,
796808 .pollin = 1,
809+ .buffer_select = 1,
797810 },
798811 [IORING_OP_OPENAT2] = {
799812 .needs_file = 1,
@@ -1170,7 +1183,7 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) · 370 unmodified lines
11701183 if (cqe) {
11711184 WRITE_ONCE(cqe->user_data, req->user_data);
11721185 WRITE_ONCE(cqe->res, req->result);
1173− WRITE_ONCE(cqe->flags, 0);
1186+ WRITE_ONCE(cqe->flags, req->cflags);
11741187 } else {
11751188 WRITE_ONCE(ctx->rings->cq_overflow,
11761189 atomic_inc_return(&ctx->cached_cq_overflow));
@@ -1194,7 +1207,7 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) · 17 unmodified lines
11941207 return cqe != NULL;
11951208 }
11961209
1197−static void io_cqring_fill_event(struct io_kiocb *req, long res)
1210+static void __io_cqring_fill_event(struct io_kiocb *req, long res, long cflags)
11981211 {
11991212 struct io_ring_ctx *ctx = req->ctx;
12001213 struct io_uring_cqe *cqe;
@@ -1210,7 +1223,7 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res) · 9 unmodified lines
12101223 if (likely(cqe)) {
12111224 WRITE_ONCE(cqe->user_data, req->user_data);
12121225 WRITE_ONCE(cqe->res, res);
1213− WRITE_ONCE(cqe->flags, 0);
1226+ WRITE_ONCE(cqe->flags, cflags);
12141227 } else if (ctx->cq_overflow_flushed) {
12151228 WRITE_ONCE(ctx->rings->cq_overflow,
12161229 atomic_inc_return(&ctx->cached_cq_overflow));
@@ -1222,23 +1235,34 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res) · 5 unmodified lines
12221235 req->flags |= REQ_F_OVERFLOW;
12231236 refcount_inc(&req->refs);
12241237 req->result = res;
1238+ req->cflags = cflags;
12251239 list_add_tail(&req->list, &ctx->cq_overflow_list);
12261240 }
12271241 }
12281242
1229−static void io_cqring_add_event(struct io_kiocb *req, long res)
1243+static void io_cqring_fill_event(struct io_kiocb *req, long res)
1244+{
1245+ __io_cqring_fill_event(req, res, 0);
1246+}
1247+
1248+static void __io_cqring_add_event(struct io_kiocb *req, long res, long cflags)
12301249 {
12311250 struct io_ring_ctx *ctx = req->ctx;
12321251 unsigned long flags;
12331252
12341253 spin_lock_irqsave(&ctx->completion_lock, flags);
1235− io_cqring_fill_event(req, res);
1254+ __io_cqring_fill_event(req, res, cflags);
12361255 io_commit_cqring(ctx);
12371256 spin_unlock_irqrestore(&ctx->completion_lock, flags);
12381257
12391258 io_cqring_ev_posted(ctx);
12401259 }
12411260
1261+static void io_cqring_add_event(struct io_kiocb *req, long res)
1262+{
1263+ __io_cqring_add_event(req, res, 0);
1264+}
1265+
12421266 static inline bool io_is_fallback_req(struct io_kiocb *req)
12431267 {
12441268 return req == (struct io_kiocb *)
@@ -1660,6 +1684,18 @@ static inline bool io_req_multi_free(struct req_batch *rb, struct io_kiocb *req) · 415 unmodified lines
16601684 return true;
16611685 }
16621686
1687+static int io_put_kbuf(struct io_kiocb *req)
1688+{
1689+ struct io_buffer *kbuf = (struct io_buffer *) req->rw.addr;
1690+ int cflags;
1691+
1692+ cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
1693+ cflags |= IORING_CQE_F_BUFFER;
1694+ req->rw.addr = 0;
1695+ kfree(kbuf);
1696+ return cflags;
1697+}
1698+
16631699 /*
16641700 * Find and free completed poll iocbs
16651701 */
@@ -1671,10 +1707,15 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, · 5 unmodified lines
16711707
16721708 rb.to_free = rb.need_iter = 0;
16731709 while (!list_empty(done)) {
1710+ int cflags = 0;
1711+
16741712 req = list_first_entry(done, struct io_kiocb, list);
16751713 list_del(&req->list);
16761714
1677− io_cqring_fill_event(req, req->result);
1715+ if (req->flags & REQ_F_BUFFER_SELECTED)
1716+ cflags = io_put_kbuf(req);
1717+
1718+ __io_cqring_fill_event(req, req->result, cflags);
16781719 (*nr_events)++;
16791720
16801721 if (refcount_dec_and_test(&req->refs) &&
@@ -1849,13 +1890,16 @@ static inline void req_set_fail_links(struct io_kiocb *req) · 168 unmodified lines
18491890 static void io_complete_rw_common(struct kiocb *kiocb, long res)
18501891 {
18511892 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1893+ int cflags = 0;
18521894
18531895 if (kiocb->ki_flags & IOCB_WRITE)
18541896 kiocb_end_write(req);
18551897
18561898 if (res != req->result)
18571899 req_set_fail_links(req);
1858− io_cqring_add_event(req, res);
1900+ if (req->flags & REQ_F_BUFFER_SELECTED)
1901+ cflags = io_put_kbuf(req);
1902+ __io_cqring_add_event(req, res, cflags);
18591903 }
18601904
18611905 static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
@@ -2033,7 +2077,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 171 unmodified lines
20332077
20342078 req->rw.addr = READ_ONCE(sqe->addr);
20352079 req->rw.len = READ_ONCE(sqe->len);
2036− /* we own ->private, reuse it for the buffer index */
2080+ /* we own ->private, reuse it for the buffer index / buffer ID */
20372081 req->rw.kiocb.private = (void *) (unsigned long)
20382082 READ_ONCE(sqe->buf_index);
20392083 return 0;
@@ -2146,8 +2190,61 @@ static ssize_t io_import_fixed(struct io_kiocb *req, int rw, · 106 unmodified lines
21462190 return len;
21472191 }
21482192
2193+static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
2194+{
2195+ if (needs_lock)
2196+ mutex_unlock(&ctx->uring_lock);
2197+}
2198+
2199+static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
2200+{
2201+ /*
2202+ * "Normal" inline submissions always hold the uring_lock, since we
2203+ * grab it from the system call. Same is true for the SQPOLL offload.
2204+ * The only exception is when we've detached the request and issue it
2205+ * from an async worker thread, grab the lock for that case.
2206+ */
2207+ if (needs_lock)
2208+ mutex_lock(&ctx->uring_lock);
2209+}
2210+
2211+static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,
2212+ int bgid, struct io_buffer *kbuf,
2213+ bool needs_lock)
2214+{
2215+ struct io_buffer *head;
2216+
2217+ if (req->flags & REQ_F_BUFFER_SELECTED)
2218+ return kbuf;
2219+
2220+ io_ring_submit_lock(req->ctx, needs_lock);
2221+
2222+ lockdep_assert_held(&req->ctx->uring_lock);
2223+
2224+ head = idr_find(&req->ctx->io_buffer_idr, bgid);
2225+ if (head) {
2226+ if (!list_empty(&head->list)) {
2227+ kbuf = list_last_entry(&head->list, struct io_buffer,
2228+ list);
2229+ list_del(&kbuf->list);
2230+ } else {
2231+ kbuf = head;
2232+ idr_remove(&req->ctx->io_buffer_idr, bgid);
2233+ }
2234+ if (*len > kbuf->len)
2235+ *len = kbuf->len;
2236+ } else {
2237+ kbuf = ERR_PTR(-ENOBUFS);
2238+ }
2239+
2240+ io_ring_submit_unlock(req->ctx, needs_lock);
2241+
2242+ return kbuf;
2243+}
2244+
21492245 static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
2150− struct iovec **iovec, struct iov_iter *iter)
2246+ struct iovec **iovec, struct iov_iter *iter,
2247+ bool needs_lock)
21512248 {
21522249 void __user *buf = u64_to_user_ptr(req->rw.addr);
21532250 size_t sqe_len = req->rw.len;
@@ -2159,12 +2256,29 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req, · 5 unmodified lines
21592256 return io_import_fixed(req, rw, iter);
21602257 }
21612258
2162− /* buffer index only valid with fixed read/write */
2163− if (req->rw.kiocb.private)
2259+ /* buffer index only valid with fixed read/write, or buffer select */
2260+ if (req->rw.kiocb.private && !(req->flags & REQ_F_BUFFER_SELECT))
21642261 return -EINVAL;
21652262
21662263 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
21672264 ssize_t ret;
2265+
2266+ if (req->flags & REQ_F_BUFFER_SELECT) {
2267+ struct io_buffer *kbuf = (struct io_buffer *) req->rw.addr;
2268+ int bgid;
2269+
2270+ bgid = (int) (unsigned long) req->rw.kiocb.private;
2271+ kbuf = io_buffer_select(req, &sqe_len, bgid, kbuf,
2272+ needs_lock);
2273+ if (IS_ERR(kbuf)) {
2274+ *iovec = NULL;
2275+ return PTR_ERR(kbuf);
2276+ }
2277+ req->rw.addr = (u64) kbuf;
2278+ req->flags |= REQ_F_BUFFER_SELECTED;
2279+ buf = u64_to_user_ptr(kbuf->addr);
2280+ }
2281+
21682282 ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
21692283 *iovec = NULL;
21702284 return ret < 0 ? ret : sqe_len;
@@ -2307,7 +2421,7 @@ static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 136 unmodified lines
23072421 io = req->io;
23082422 io->rw.iov = io->rw.fast_iov;
23092423 req->io = NULL;
2310− ret = io_import_iovec(READ, req, &io->rw.iov, &iter);
2424+ ret = io_import_iovec(READ, req, &io->rw.iov, &iter, !force_nonblock);
23112425 req->io = io;
23122426 if (ret < 0)
23132427 return ret;
@@ -2324,7 +2438,7 @@ static int io_read(struct io_kiocb *req, bool force_nonblock) · 10 unmodified lines
23242438 size_t iov_count;
23252439 ssize_t io_size, ret;
23262440
2327− ret = io_import_iovec(READ, req, &iovec, &iter);
2441+ ret = io_import_iovec(READ, req, &iovec, &iter, !force_nonblock);
23282442 if (ret < 0)
23292443 return ret;
23302444
@@ -2396,7 +2510,7 @@ static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 65 unmodified lines
23962510 io = req->io;
23972511 io->rw.iov = io->rw.fast_iov;
23982512 req->io = NULL;
2399− ret = io_import_iovec(WRITE, req, &io->rw.iov, &iter);
2513+ ret = io_import_iovec(WRITE, req, &io->rw.iov, &iter, !force_nonblock);
24002514 req->io = io;
24012515 if (ret < 0)
24022516 return ret;
@@ -2413,7 +2527,7 @@ static int io_write(struct io_kiocb *req, bool force_nonblock) · 10 unmodified lines
24132527 size_t iov_count;
24142528 ssize_t ret, io_size;
24152529
2416− ret = io_import_iovec(WRITE, req, &iovec, &iter);
2530+ ret = io_import_iovec(WRITE, req, &iovec, &iter, !force_nonblock);
24172531 if (ret < 0)
24182532 return ret;
24192533
@@ -2850,24 +2964,6 @@ static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head) · 430 unmodified lines
28502964 return i ? i : -ENOMEM;
28512965 }
28522966
2853−static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
2854−{
2855− if (needs_lock)
2856− mutex_unlock(&ctx->uring_lock);
2857−}
2858−
2859−static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
2860−{
2861− /*
2862− * "Normal" inline submissions always hold the uring_lock, since we
2863− * grab it from the system call. Same is true for the SQPOLL offload.
2864− * The only exception is when we've detached the request and issue it
2865− * from an async worker thread, grab the lock for that case.
2866− */
2867− if (needs_lock)
2868− mutex_lock(&ctx->uring_lock);
2869−}
2870−
28712967 static int io_provide_buffers(struct io_kiocb *req, bool force_nonblock)
28722968 {
28732969 struct io_provide_buf *p = &req->pbuf;
@@ -3390,6 +3486,27 @@ static int io_send(struct io_kiocb *req, bool force_nonblock) · 516 unmodified lines
33903486 #endif
33913487 }
33923488
3489+static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,
3490+ int *cflags, bool needs_lock)
3491+{
3492+ struct io_sr_msg *sr = &req->sr_msg;
3493+ struct io_buffer *kbuf;
3494+
3495+ if (!(req->flags & REQ_F_BUFFER_SELECT))
3496+ return NULL;
3497+
3498+ kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
3499+ if (IS_ERR(kbuf))
3500+ return kbuf;
3501+
3502+ sr->kbuf = kbuf;
3503+ req->flags |= REQ_F_BUFFER_SELECTED;
3504+
3505+ *cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
3506+ *cflags |= IORING_CQE_F_BUFFER;
3507+ return kbuf;
3508+}
3509+
33933510 static int io_recvmsg_prep(struct io_kiocb *req,
33943511 const struct io_uring_sqe *sqe)
33953512 {
@@ -3401,6 +3518,7 @@ static int io_recvmsg_prep(struct io_kiocb *req, · 5 unmodified lines
34013518 sr->msg_flags = READ_ONCE(sqe->msg_flags);
34023519 sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
34033520 sr->len = READ_ONCE(sqe->len);
3521+ sr->bgid = READ_ONCE(sqe->buf_group);
34043522
34053523 #ifdef CONFIG_COMPAT
34063524 if (req->ctx->compat)
@@ -3490,8 +3608,9 @@ static int io_recvmsg(struct io_kiocb *req, bool force_nonblock) · 83 unmodified lines
34903608 static int io_recv(struct io_kiocb *req, bool force_nonblock)
34913609 {
34923610 #if defined(CONFIG_NET)
3611+ struct io_buffer *kbuf = NULL;
34933612 struct socket *sock;
3494− int ret;
3613+ int ret, cflags = 0;
34953614
34963615 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
34973616 return -EINVAL;
@@ -3499,15 +3618,25 @@ static int io_recv(struct io_kiocb *req, bool force_nonblock) · 1 unmodified line
34993618 sock = sock_from_file(req->file, &ret);
35003619 if (sock) {
35013620 struct io_sr_msg *sr = &req->sr_msg;
3621+ void __user *buf = sr->buf;
35023622 struct msghdr msg;
35033623 struct iovec iov;
35043624 unsigned flags;
35053625
3506− ret = import_single_range(READ, sr->buf, sr->len, &iov,
3626+ kbuf = io_recv_buffer_select(req, &cflags, !force_nonblock);
3627+ if (IS_ERR(kbuf))
3628+ return PTR_ERR(kbuf);
3629+ else if (kbuf)
3630+ buf = u64_to_user_ptr(kbuf->addr);
3631+
3632+ ret = import_single_range(READ, buf, sr->len, &iov,
35073633 &msg.msg_iter);
3508− if (ret)
3634+ if (ret) {
3635+ kfree(kbuf);
35093636 return ret;
3637+ }
35103638
3639+ req->flags |= REQ_F_NEED_CLEANUP;
35113640 msg.msg_name = NULL;
35123641 msg.msg_control = NULL;
35133642 msg.msg_controllen = 0;
@@ -3528,7 +3657,9 @@ static int io_recv(struct io_kiocb *req, bool force_nonblock) · 14 unmodified lines
35283657 ret = -EINTR;
35293658 }
35303659
3531− io_cqring_add_event(req, ret);
3660+ kfree(kbuf);
3661+ req->flags &= ~REQ_F_NEED_CLEANUP;
3662+ __io_cqring_add_event(req, ret, cflags);
35323663 if (ret < 0)
35333664 req_set_fail_links(req);
35343665 io_put_req(req);
@@ -4566,6 +4697,9 @@ static void io_cleanup_req(struct io_kiocb *req) · 1031 unmodified lines
45664697 case IORING_OP_READV:
45674698 case IORING_OP_READ_FIXED:
45684699 case IORING_OP_READ:
4700+ if (req->flags & REQ_F_BUFFER_SELECTED)
4701+ kfree((void *)(unsigned long)req->rw.addr);
4702+ /* fallthrough */
45694703 case IORING_OP_WRITEV:
45704704 case IORING_OP_WRITE_FIXED:
45714705 case IORING_OP_WRITE:
@@ -4577,6 +4711,10 @@ static void io_cleanup_req(struct io_kiocb *req) · 5 unmodified lines
45774711 if (io->msg.iov != io->msg.fast_iov)
45784712 kfree(io->msg.iov);
45794713 break;
4714+ case IORING_OP_RECV:
4715+ if (req->flags & REQ_F_BUFFER_SELECTED)
4716+ kfree(req->sr_msg.kbuf);
4717+ break;
45804718 case IORING_OP_OPENAT:
45814719 case IORING_OP_OPENAT2:
45824720 case IORING_OP_STATX:
@@ -5154,7 +5292,8 @@ static inline void io_queue_link_head(struct io_kiocb *req) · 571 unmodified lines
51545292 }
51555293
51565294 #define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
5157− IOSQE_IO_HARDLINK | IOSQE_ASYNC)
5295+ IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
5296+ IOSQE_BUFFER_SELECT)
51585297
51595298 static bool io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
51605299 struct io_submit_state *state, struct io_kiocb **link)
@@ -5171,6 +5310,12 @@ static bool io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 10 unmodified lines
51715310 goto err_req;
51725311 }
51735312
5313+ if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
5314+ !io_op_defs[req->opcode].buffer_select) {
5315+ ret = -EOPNOTSUPP;
5316+ goto err_req;
5317+ }
5318+
51745319 id = READ_ONCE(sqe->personality);
51755320 if (id) {
51765321 req->work.creds = idr_find(&ctx->personality_idr, id);
@@ -5183,7 +5328,8 @@ static bool io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 6 unmodified lines
51835328
51845329 /* same numerical values with corresponding REQ_F_*, safe to copy */
51855330 req->flags |= sqe_flags & (IOSQE_IO_DRAIN | IOSQE_IO_HARDLINK |
5186− IOSQE_ASYNC | IOSQE_FIXED_FILE);
5331+ IOSQE_ASYNC | IOSQE_FIXED_FILE |
5332+ IOSQE_BUFFER_SELECT);
51875333
51885334 ret = io_req_set_file(state, req, sqe);
51895335 if (unlikely(ret)) {
include/uapi/linux/io_uring.h+14−0
@@ -66,6 +66,7 @@ enum { · 65 unmodified lines
6666 IOSQE_IO_LINK_BIT,
6767 IOSQE_IO_HARDLINK_BIT,
6868 IOSQE_ASYNC_BIT,
69+ IOSQE_BUFFER_SELECT_BIT,
6970 };
7071
7172 /*
@@ -81,6 +82,8 @@ enum { · 9 unmodified lines
8182 #define IOSQE_IO_HARDLINK (1U << IOSQE_IO_HARDLINK_BIT)
8283 /* always go async */
8384 #define IOSQE_ASYNC (1U << IOSQE_ASYNC_BIT)
85+/* select buffer from sqe->buf_group */
86+#define IOSQE_BUFFER_SELECT (1U << IOSQE_BUFFER_SELECT_BIT)
8487
8588 /*
8689 * io_uring_setup() flags
@@ -155,6 +158,17 @@ struct io_uring_cqe { · 68 unmodified lines
155158 __u32 flags;
156159 };
157160
161+/*
162+ * cqe->flags
163+ *
164+ * IORING_CQE_F_BUFFER If set, the upper 16 bits are the buffer ID
165+ */
166+#define IORING_CQE_F_BUFFER (1U << 0)
167+
168+enum {
169+ IORING_CQE_BUFFER_SHIFT = 16,
170+};
171+
158172 /*
159173 * Magic offsets for the application to mmap the data it needs
160174 */