History

Merge tag 'for-6.1/io_uring-2022-10-03' of git://git.kernel.dk/linux

Pull io_uring updates from Jens Axboe: - Add supported for more directly managed task_work running. This is beneficial for real world applications that end up issuing lots of system calls as part of handling work. Normal task_work will always execute as we transition in and out of the kernel, even for "unrelated" system calls. It's more efficient to defer the handling of io_uring's deferred work until the application wants it to be run, generally in batches. As part of ongoing work to write an io_uring network backend for Thrift, this has been shown to greatly improve performance. (Dylan) - Add IOPOLL support for passthrough (Kanchan) - Improvements and fixes to the send zero-copy support (Pavel) - Partial IO handling fixes (Pavel) - CQE ordering fixes around CQ ring overflow (Pavel) - Support sendto() for non-zc as well (Pavel) - Support sendmsg for zerocopy (Pavel) - Networking iov_iter fix (Stefan) - Misc fixes and cleanups (Pavel, me) * tag 'for-6.1/io_uring-2022-10-03' of git://git.kernel.dk/linux: (56 commits) io_uring/net: fix notif cqe reordering io_uring/net: don't update msg_name if not provided io_uring: don't gate task_work run on TIF_NOTIFY_SIGNAL io_uring/rw: defer fsnotify calls to task context io_uring/net: fix fast_iov assignment in io_setup_async_msg() io_uring/net: fix non-zc send with address io_uring/net: don't skip notifs for failed requests io_uring/rw: don't lose short results on io_setup_async_rw() io_uring/rw: fix unexpected link breakage io_uring/net: fix cleanup double free free_iov init io_uring: fix CQE reordering io_uring/net: fix UAF in io_sendrecv_fail() selftest/net: adjust io_uring sendzc notif handling io_uring: ensure local task_work marks task as running io_uring/net: zerocopy sendmsg io_uring/net: combine fail handlers io_uring/net: rename io_sendzc() io_uring/net: support non-zerocopy sendto io_uring/net: refactor io_setup_async_addr io_uring/net: don't lose partial send_zc on fail ...

Linus Torvalds committed 4 years ago

Commit 0a78a372 parents 188943a 108893d30 files · +859 −326Browse files

Unified · Split

30 files
  1. block/blk-mq.c +2 −1
  2. drivers/nvme/host/core.c +1 −0
  3. drivers/nvme/host/ioctl.c +72 −5
  4. drivers/nvme/host/multipath.c +1 −0
  5. drivers/nvme/host/nvme.h +4 −0
  6. fs/eventfd.c +7 −3
  7. include/linux/blk-mq.h +1 −0
  8. include/linux/eventfd.h +1 −1
  9. include/linux/fs.h +2 −0
  10. include/linux/io_uring.h +6 −2
  11. include/linux/io_uring_types.h +4 −0
  12. include/linux/sched.h +1 −1
  13. include/trace/events/io_uring.h +29 −0
  14. include/uapi/linux/io_uring.h +8 −0
  15. io_uring/cancel.c +1 −1
  16. io_uring/fdinfo.c +32 −16
  17. io_uring/io_uring.c +244 −60
  18. io_uring/io_uring.h +51 −11
  19. io_uring/kbuf.h +0 −12
  20. io_uring/net.c +222 −86
  21. io_uring/net.h +8 −4
  22. io_uring/opdef.c +40 −4
  23. io_uring/opdef.h +1 −0
  24. io_uring/rsrc.c +1 −1
  25. io_uring/rw.c +93 −96
  26. io_uring/rw.h +1 −0
  27. io_uring/timeout.c +3 −10
  28. io_uring/timeout.h +1 −1
  29. io_uring/uring_cmd.c +9 −2
  30. tools/testing/selftests/net/io_uring_zerocopy_tx.c +13 −9
block/blk-mq.c+2−1
@@ -1233,7 +1233,7 @@ static void blk_end_sync_rq(struct request *rq, blk_status_t ret) · 1232 unmodified lines
12331233 complete(&wait->done);
12341234 }
12351235
1236−static bool blk_rq_is_poll(struct request *rq)
1236+bool blk_rq_is_poll(struct request *rq)
12371237 {
12381238 if (!rq->mq_hctx)
12391239 return false;
@@ -1243,6 +1243,7 @@ static bool blk_rq_is_poll(struct request *rq) · 3 unmodified lines
12431243 return false;
12441244 return true;
12451245 }
1246+EXPORT_SYMBOL_GPL(blk_rq_is_poll);
12461247
12471248 static void blk_rq_poll_completion(struct request *rq, struct completion *wait)
12481249 {
drivers/nvme/host/core.c+1−0
@@ -3976,6 +3976,7 @@ static const struct file_operations nvme_ns_chr_fops = { · 3975 unmodified lines
39763976 .unlocked_ioctl = nvme_ns_chr_ioctl,
39773977 .compat_ioctl = compat_ptr_ioctl,
39783978 .uring_cmd = nvme_ns_chr_uring_cmd,
3979+ .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll,
39793980 };
39803981
39813982 static int nvme_add_ns_cdev(struct nvme_ns *ns)
drivers/nvme/host/ioctl.c+72−5
@@ -391,11 +391,19 @@ static void nvme_uring_cmd_end_io(struct request *req, blk_status_t err) · 390 unmodified lines
391391 struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
392392 /* extract bio before reusing the same field for request */
393393 struct bio *bio = pdu->bio;
394+ void *cookie = READ_ONCE(ioucmd->cookie);
394395
395396 pdu->req = req;
396397 req->bio = bio;
397− /* this takes care of moving rest of completion-work to task context */
398− io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb);
398+
399+ /*
400+ * For iopoll, complete it directly.
401+ * Otherwise, move the completion to task work.
402+ */
403+ if (cookie != NULL && blk_rq_is_poll(req))
404+ nvme_uring_task_cb(ioucmd);
405+ else
406+ io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb);
399407 }
400408
401409 static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
@@ -445,7 +453,10 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, · 43 unmodified lines
445453 rq_flags = REQ_NOWAIT;
446454 blk_flags = BLK_MQ_REQ_NOWAIT;
447455 }
456+ if (issue_flags & IO_URING_F_IOPOLL)
457+ rq_flags |= REQ_POLLED;
448458
459+retry:
449460 req = nvme_alloc_user_request(q, &c, nvme_to_user_ptr(d.addr),
450461 d.data_len, nvme_to_user_ptr(d.metadata),
451462 d.metadata_len, 0, &meta, d.timeout_ms ?
@@ -456,6 +467,17 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, · 4 unmodified lines
456467 req->end_io = nvme_uring_cmd_end_io;
457468 req->end_io_data = ioucmd;
458469
470+ if (issue_flags & IO_URING_F_IOPOLL && rq_flags & REQ_POLLED) {
471+ if (unlikely(!req->bio)) {
472+ /* we can't poll this, so alloc regular req instead */
473+ blk_mq_free_request(req);
474+ rq_flags &= ~REQ_POLLED;
475+ goto retry;
476+ } else {
477+ WRITE_ONCE(ioucmd->cookie, req->bio);
478+ req->bio->bi_opf |= REQ_POLLED;
479+ }
480+ }
459481 /* to free bio on completion, as req->bio will be null at that time */
460482 pdu->bio = req->bio;
461483 pdu->meta = meta;
@@ -559,9 +581,6 @@ long nvme_ns_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg) · 97 unmodified lines
559581
560582 static int nvme_uring_cmd_checks(unsigned int issue_flags)
561583 {
562− /* IOPOLL not supported yet */
563− if (issue_flags & IO_URING_F_IOPOLL)
564− return -EOPNOTSUPP;
565584
566585 /* NVMe passthrough requires big SQE/CQE support */
567586 if ((issue_flags & (IO_URING_F_SQE128|IO_URING_F_CQE32)) !=
@@ -604,6 +623,25 @@ int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags) · 36 unmodified lines
604623 return nvme_ns_uring_cmd(ns, ioucmd, issue_flags);
605624 }
606625
626+int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
627+ struct io_comp_batch *iob,
628+ unsigned int poll_flags)
629+{
630+ struct bio *bio;
631+ int ret = 0;
632+ struct nvme_ns *ns;
633+ struct request_queue *q;
634+
635+ rcu_read_lock();
636+ bio = READ_ONCE(ioucmd->cookie);
637+ ns = container_of(file_inode(ioucmd->file)->i_cdev,
638+ struct nvme_ns, cdev);
639+ q = ns->queue;
640+ if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio && bio->bi_bdev)
641+ ret = bio_poll(bio, iob, poll_flags);
642+ rcu_read_unlock();
643+ return ret;
644+}
607645 #ifdef CONFIG_NVME_MULTIPATH
608646 static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
609647 void __user *argp, struct nvme_ns_head *head, int srcu_idx)
@@ -685,6 +723,31 @@ int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd, · 75 unmodified lines
685723 srcu_read_unlock(&head->srcu, srcu_idx);
686724 return ret;
687725 }
726+
727+int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
728+ struct io_comp_batch *iob,
729+ unsigned int poll_flags)
730+{
731+ struct cdev *cdev = file_inode(ioucmd->file)->i_cdev;
732+ struct nvme_ns_head *head = container_of(cdev, struct nvme_ns_head, cdev);
733+ int srcu_idx = srcu_read_lock(&head->srcu);
734+ struct nvme_ns *ns = nvme_find_path(head);
735+ struct bio *bio;
736+ int ret = 0;
737+ struct request_queue *q;
738+
739+ if (ns) {
740+ rcu_read_lock();
741+ bio = READ_ONCE(ioucmd->cookie);
742+ q = ns->queue;
743+ if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio
744+ && bio->bi_bdev)
745+ ret = bio_poll(bio, iob, poll_flags);
746+ rcu_read_unlock();
747+ }
748+ srcu_read_unlock(&head->srcu, srcu_idx);
749+ return ret;
750+}
688751 #endif /* CONFIG_NVME_MULTIPATH */
689752
690753 int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags)
@@ -692,6 +755,10 @@ int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags) · 1 unmodified line
692755 struct nvme_ctrl *ctrl = ioucmd->file->private_data;
693756 int ret;
694757
758+ /* IOPOLL not supported yet */
759+ if (issue_flags & IO_URING_F_IOPOLL)
760+ return -EOPNOTSUPP;
761+
695762 ret = nvme_uring_cmd_checks(issue_flags);
696763 if (ret)
697764 return ret;
drivers/nvme/host/multipath.c+1−0
@@ -439,6 +439,7 @@ static const struct file_operations nvme_ns_head_chr_fops = { · 438 unmodified lines
439439 .unlocked_ioctl = nvme_ns_head_chr_ioctl,
440440 .compat_ioctl = compat_ptr_ioctl,
441441 .uring_cmd = nvme_ns_head_chr_uring_cmd,
442+ .uring_cmd_iopoll = nvme_ns_head_chr_uring_cmd_iopoll,
442443 };
443444
444445 static int nvme_add_ns_head_cdev(struct nvme_ns_head *head)
drivers/nvme/host/nvme.h+4−0
@@ -821,6 +821,10 @@ long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd, · 820 unmodified lines
821821 unsigned long arg);
822822 long nvme_dev_ioctl(struct file *file, unsigned int cmd,
823823 unsigned long arg);
824+int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
825+ struct io_comp_batch *iob, unsigned int poll_flags);
826+int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
827+ struct io_comp_batch *iob, unsigned int poll_flags);
824828 int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd,
825829 unsigned int issue_flags);
826830 int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd,
fs/eventfd.c+7−3
@@ -69,17 +69,17 @@ __u64 eventfd_signal(struct eventfd_ctx *ctx, __u64 n) · 68 unmodified lines
6969 * it returns false, the eventfd_signal() call should be deferred to a
7070 * safe context.
7171 */
72− if (WARN_ON_ONCE(current->in_eventfd_signal))
72+ if (WARN_ON_ONCE(current->in_eventfd))
7373 return 0;
7474
7575 spin_lock_irqsave(&ctx->wqh.lock, flags);
76− current->in_eventfd_signal = 1;
76+ current->in_eventfd = 1;
7777 if (ULLONG_MAX - ctx->count < n)
7878 n = ULLONG_MAX - ctx->count;
7979 ctx->count += n;
8080 if (waitqueue_active(&ctx->wqh))
8181 wake_up_locked_poll(&ctx->wqh, EPOLLIN);
82− current->in_eventfd_signal = 0;
82+ current->in_eventfd = 0;
8383 spin_unlock_irqrestore(&ctx->wqh.lock, flags);
8484
8585 return n;
@@ -253,8 +253,10 @@ static ssize_t eventfd_read(struct kiocb *iocb, struct iov_iter *to) · 167 unmodified lines
253253 __set_current_state(TASK_RUNNING);
254254 }
255255 eventfd_ctx_do_read(ctx, &ucnt);
256+ current->in_eventfd = 1;
256257 if (waitqueue_active(&ctx->wqh))
257258 wake_up_locked_poll(&ctx->wqh, EPOLLOUT);
259+ current->in_eventfd = 0;
258260 spin_unlock_irq(&ctx->wqh.lock);
259261 if (unlikely(copy_to_iter(&ucnt, sizeof(ucnt), to) != sizeof(ucnt)))
260262 return -EFAULT;
@@ -301,8 +303,10 @@ static ssize_t eventfd_write(struct file *file, const char __user *buf, size_t c · 40 unmodified lines
301303 }
302304 if (likely(res > 0)) {
303305 ctx->count += ucnt;
306+ current->in_eventfd = 1;
304307 if (waitqueue_active(&ctx->wqh))
305308 wake_up_locked_poll(&ctx->wqh, EPOLLIN);
309+ current->in_eventfd = 0;
306310 }
307311 spin_unlock_irq(&ctx->wqh.lock);
308312
include/linux/blk-mq.h+1−0
@@ -980,6 +980,7 @@ int blk_rq_map_kern(struct request_queue *, struct request *, void *, · 979 unmodified lines
980980 int blk_rq_append_bio(struct request *rq, struct bio *bio);
981981 void blk_execute_rq_nowait(struct request *rq, bool at_head);
982982 blk_status_t blk_execute_rq(struct request *rq, bool at_head);
983+bool blk_rq_is_poll(struct request *rq);
983984
984985 struct req_iterator {
985986 struct bvec_iter iter;
include/linux/eventfd.h+1−1
@@ -46,7 +46,7 @@ void eventfd_ctx_do_read(struct eventfd_ctx *ctx, __u64 *cnt); · 45 unmodified lines
4646
4747 static inline bool eventfd_signal_allowed(void)
4848 {
49− return !current->in_eventfd_signal;
49+ return !current->in_eventfd;
5050 }
5151
5252 #else /* CONFIG_EVENTFD */
include/linux/fs.h+2−0
@@ -2133,6 +2133,8 @@ struct file_operations { · 2132 unmodified lines
21332133 loff_t len, unsigned int remap_flags);
21342134 int (*fadvise)(struct file *, loff_t, loff_t, int);
21352135 int (*uring_cmd)(struct io_uring_cmd *ioucmd, unsigned int issue_flags);
2136+ int (*uring_cmd_iopoll)(struct io_uring_cmd *, struct io_comp_batch *,
2137+ unsigned int poll_flags);
21362138 } __randomize_layout;
21372139
21382140 struct inode_operations {
include/linux/io_uring.h+6−2
@@ -20,8 +20,12 @@ enum io_uring_cmd_flags { · 19 unmodified lines
2020 struct io_uring_cmd {
2121 struct file *file;
2222 const void *cmd;
23− /* callback to defer completions to task context */
24− void (*task_work_cb)(struct io_uring_cmd *cmd);
23+ union {
24+ /* callback to defer completions to task context */
25+ void (*task_work_cb)(struct io_uring_cmd *cmd);
26+ /* used for polled completion */
27+ void *cookie;
28+ };
2529 u32 cmd_op;
2630 u32 pad;
2731 u8 pdu[32]; /* available inline for free use */
include/linux/io_uring_types.h+4−0
@@ -184,6 +184,8 @@ struct io_ev_fd { · 183 unmodified lines
184184 struct eventfd_ctx *cq_ev_fd;
185185 unsigned int eventfd_async: 1;
186186 struct rcu_head rcu;
187+ atomic_t refs;
188+ atomic_t ops;
187189 };
188190
189191 struct io_alloc_cache {
@@ -301,6 +303,8 @@ struct io_ring_ctx { · 111 unmodified lines
301303 struct io_hash_table cancel_table;
302304 bool poll_multi_queue;
303305
306+ struct llist_head work_llist;
307+
304308 struct list_head io_buffers_comp;
305309 } ____cacheline_aligned_in_smp;
306310
include/linux/sched.h+1−1
@@ -936,7 +936,7 @@ struct task_struct { · 935 unmodified lines
936936 #endif
937937 #ifdef CONFIG_EVENTFD
938938 /* Recursion prevention for eventfd_signal() */
939− unsigned in_eventfd_signal:1;
939+ unsigned in_eventfd:1;
940940 #endif
941941 #ifdef CONFIG_IOMMU_SVA
942942 unsigned pasid_activated:1;
include/trace/events/io_uring.h+29−0
@@ -655,6 +655,35 @@ TRACE_EVENT(io_uring_short_write, · 654 unmodified lines
655655 __entry->wanted, __entry->got)
656656 );
657657
658+/*
659+ * io_uring_local_work_run - ran ring local task work
660+ *
661+ * @tctx: pointer to a io_uring_ctx
662+ * @count: how many functions it ran
663+ * @loops: how many loops it ran
664+ *
665+ */
666+TRACE_EVENT(io_uring_local_work_run,
667+
668+ TP_PROTO(void *ctx, int count, unsigned int loops),
669+
670+ TP_ARGS(ctx, count, loops),
671+
672+ TP_STRUCT__entry (
673+ __field(void *, ctx )
674+ __field(int, count )
675+ __field(unsigned int, loops )
676+ ),
677+
678+ TP_fast_assign(
679+ __entry->ctx = ctx;
680+ __entry->count = count;
681+ __entry->loops = loops;
682+ ),
683+
684+ TP_printk("ring %p, count %d, loops %u", __entry->ctx, __entry->count, __entry->loops)
685+);
686+
658687 #endif /* _TRACE_IO_URING_H */
659688
660689 /* This part must be outside protection */
include/uapi/linux/io_uring.h+8−0
@@ -157,6 +157,13 @@ enum { · 156 unmodified lines
157157 */
158158 #define IORING_SETUP_SINGLE_ISSUER (1U << 12)
159159
160+/*
161+ * Defer running task work to get events.
162+ * Rather than running bits of task work whenever the task transitions
163+ * try to do it just before it is needed.
164+ */
165+#define IORING_SETUP_DEFER_TASKRUN (1U << 13)
166+
160167 enum io_uring_op {
161168 IORING_OP_NOP,
162169 IORING_OP_READV,
@@ -206,6 +213,7 @@ enum io_uring_op { · 43 unmodified lines
206213 IORING_OP_SOCKET,
207214 IORING_OP_URING_CMD,
208215 IORING_OP_SEND_ZC,
216+ IORING_OP_SENDMSG_ZC,
209217
210218 /* this goes last, obviously */
211219 IORING_OP_LAST,
io_uring/cancel.c+1−1
@@ -292,7 +292,7 @@ int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg) · 291 unmodified lines
292292 break;
293293
294294 mutex_unlock(&ctx->uring_lock);
295− ret = io_run_task_work_sig();
295+ ret = io_run_task_work_sig(ctx);
296296 if (ret < 0) {
297297 mutex_lock(&ctx->uring_lock);
298298 break;
io_uring/fdinfo.c+32−16
@@ -60,13 +60,15 @@ static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, · 59 unmodified lines
6060 unsigned int cq_head = READ_ONCE(r->cq.head);
6161 unsigned int cq_tail = READ_ONCE(r->cq.tail);
6262 unsigned int cq_shift = 0;
63+ unsigned int sq_shift = 0;
6364 unsigned int sq_entries, cq_entries;
6465 bool has_lock;
65− bool is_cqe32 = (ctx->flags & IORING_SETUP_CQE32);
6666 unsigned int i;
6767
68− if (is_cqe32)
68+ if (ctx->flags & IORING_SETUP_CQE32)
6969 cq_shift = 1;
70+ if (ctx->flags & IORING_SETUP_SQE128)
71+ sq_shift = 1;
7072
7173 /*
7274 * we may get imprecise sqe and cqe info if uring is actively running
@@ -82,19 +84,36 @@ static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, · 9 unmodified lines
8284 seq_printf(m, "CqHead:\t%u\n", cq_head);
8385 seq_printf(m, "CqTail:\t%u\n", cq_tail);
8486 seq_printf(m, "CachedCqTail:\t%u\n", ctx->cached_cq_tail);
85− seq_printf(m, "SQEs:\t%u\n", sq_tail - ctx->cached_sq_head);
87+ seq_printf(m, "SQEs:\t%u\n", sq_tail - sq_head);
8688 sq_entries = min(sq_tail - sq_head, ctx->sq_entries);
8789 for (i = 0; i < sq_entries; i++) {
8890 unsigned int entry = i + sq_head;
89− unsigned int sq_idx = READ_ONCE(ctx->sq_array[entry & sq_mask]);
9091 struct io_uring_sqe *sqe;
92+ unsigned int sq_idx;
9193
94+ sq_idx = READ_ONCE(ctx->sq_array[entry & sq_mask]);
9295 if (sq_idx > sq_mask)
9396 continue;
94− sqe = &ctx->sq_sqes[sq_idx];
95− seq_printf(m, "%5u: opcode:%d, fd:%d, flags:%x, user_data:%llu\n",
96− sq_idx, sqe->opcode, sqe->fd, sqe->flags,
97− sqe->user_data);
97+ sqe = &ctx->sq_sqes[sq_idx << 1];
98+ seq_printf(m, "%5u: opcode:%s, fd:%d, flags:%x, off:%llu, "
99+ "addr:0x%llx, rw_flags:0x%x, buf_index:%d "
100+ "user_data:%llu",
101+ sq_idx, io_uring_get_opcode(sqe->opcode), sqe->fd,
102+ sqe->flags, (unsigned long long) sqe->off,
103+ (unsigned long long) sqe->addr, sqe->rw_flags,
104+ sqe->buf_index, sqe->user_data);
105+ if (sq_shift) {
106+ u64 *sqeb = (void *) (sqe + 1);
107+ int size = sizeof(struct io_uring_sqe) / sizeof(u64);
108+ int j;
109+
110+ for (j = 0; j < size; j++) {
111+ seq_printf(m, ", e%d:0x%llx", j,
112+ (unsigned long long) *sqeb);
113+ sqeb++;
114+ }
115+ }
116+ seq_printf(m, "\n");
98117 }
99118 seq_printf(m, "CQEs:\t%u\n", cq_tail - cq_head);
100119 cq_entries = min(cq_tail - cq_head, ctx->cq_entries);
@@ -102,16 +121,13 @@ static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, · 1 unmodified line
102121 unsigned int entry = i + cq_head;
103122 struct io_uring_cqe *cqe = &r->cqes[(entry & cq_mask) << cq_shift];
104123
105− if (!is_cqe32) {
106− seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x\n",
124+ seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x",
107125 entry & cq_mask, cqe->user_data, cqe->res,
108126 cqe->flags);
109− } else {
110− seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x, "
111− "extra1:%llu, extra2:%llu\n",
112− entry & cq_mask, cqe->user_data, cqe->res,
113− cqe->flags, cqe->big_cqe[0], cqe->big_cqe[1]);
114− }
127+ if (cq_shift)
128+ seq_printf(m, ", extra1:%llu, extra2:%llu\n",
129+ cqe->big_cqe[0], cqe->big_cqe[1]);
130+ seq_printf(m, "\n");
115131 }
116132
117133 /*
io_uring/io_uring.c+244−60
@@ -125,6 +125,11 @@ enum { · 124 unmodified lines
125125 IO_CHECK_CQ_DROPPED_BIT,
126126 };
127127
128+enum {
129+ IO_EVENTFD_OP_SIGNAL_BIT,
130+ IO_EVENTFD_OP_FREE_BIT,
131+};
132+
128133 struct io_defer_entry {
129134 struct list_head list;
130135 struct io_kiocb *req;
@@ -142,7 +147,7 @@ static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, · 11 unmodified lines
142147 static void io_dismantle_req(struct io_kiocb *req);
143148 static void io_clean_op(struct io_kiocb *req);
144149 static void io_queue_sqe(struct io_kiocb *req);
145−
150+static void io_move_task_work_from_local(struct io_ring_ctx *ctx);
146151 static void __io_submit_flush_completions(struct io_ring_ctx *ctx);
147152
148153 static struct kmem_cache *req_cachep;
@@ -316,6 +321,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) · 167 unmodified lines
316321 INIT_LIST_HEAD(&ctx->rsrc_ref_list);
317322 INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work);
318323 init_llist_head(&ctx->rsrc_put_llist);
324+ init_llist_head(&ctx->work_llist);
319325 INIT_LIST_HEAD(&ctx->tctx_list);
320326 ctx->submit_state.free_list.next = NULL;
321327 INIT_WQ_LIST(&ctx->locked_free_list);
@@ -477,25 +483,28 @@ static __cold void io_queue_deferred(struct io_ring_ctx *ctx) · 155 unmodified lines
477483 }
478484 }
479485
480−static void io_eventfd_signal(struct io_ring_ctx *ctx)
486+
487+static void io_eventfd_ops(struct rcu_head *rcu)
481488 {
482− struct io_ev_fd *ev_fd;
483− bool skip;
489+ struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);
490+ int ops = atomic_xchg(&ev_fd->ops, 0);
484491
485− spin_lock(&ctx->completion_lock);
486− /*
487− * Eventfd should only get triggered when at least one event has been
488− * posted. Some applications rely on the eventfd notification count only
489− * changing IFF a new CQE has been added to the CQ ring. There's no
490− * depedency on 1:1 relationship between how many times this function is
491− * called (and hence the eventfd count) and number of CQEs posted to the
492− * CQ ring.
492+ if (ops & BIT(IO_EVENTFD_OP_SIGNAL_BIT))
493+ eventfd_signal(ev_fd->cq_ev_fd, 1);
494+
495+ /* IO_EVENTFD_OP_FREE_BIT may not be set here depending on callback
496+ * ordering in a race but if references are 0 we know we have to free
497+ * it regardless.
493498 */
494− skip = ctx->cached_cq_tail == ctx->evfd_last_cq_tail;
495− ctx->evfd_last_cq_tail = ctx->cached_cq_tail;
496− spin_unlock(&ctx->completion_lock);
497− if (skip)
498− return;
499+ if (atomic_dec_and_test(&ev_fd->refs)) {
500+ eventfd_ctx_put(ev_fd->cq_ev_fd);
501+ kfree(ev_fd);
502+ }
503+}
504+
505+static void io_eventfd_signal(struct io_ring_ctx *ctx)
506+{
507+ struct io_ev_fd *ev_fd = NULL;
499508
500509 rcu_read_lock();
501510 /*
@@ -513,13 +522,46 @@ static void io_eventfd_signal(struct io_ring_ctx *ctx) · 11 unmodified lines
513522 goto out;
514523 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
515524 goto out;
525+ if (ev_fd->eventfd_async && !io_wq_current_is_worker())
526+ goto out;
516527
517− if (!ev_fd->eventfd_async || io_wq_current_is_worker())
528+ if (likely(eventfd_signal_allowed())) {
518529 eventfd_signal(ev_fd->cq_ev_fd, 1);
530+ } else {
531+ atomic_inc(&ev_fd->refs);
532+ if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_SIGNAL_BIT), &ev_fd->ops))
533+ call_rcu(&ev_fd->rcu, io_eventfd_ops);
534+ else
535+ atomic_dec(&ev_fd->refs);
536+ }
537+
519538 out:
520539 rcu_read_unlock();
521540 }
522541
542+static void io_eventfd_flush_signal(struct io_ring_ctx *ctx)
543+{
544+ bool skip;
545+
546+ spin_lock(&ctx->completion_lock);
547+
548+ /*
549+ * Eventfd should only get triggered when at least one event has been
550+ * posted. Some applications rely on the eventfd notification count
551+ * only changing IFF a new CQE has been added to the CQ ring. There's
552+ * no depedency on 1:1 relationship between how many times this
553+ * function is called (and hence the eventfd count) and number of CQEs
554+ * posted to the CQ ring.
555+ */
556+ skip = ctx->cached_cq_tail == ctx->evfd_last_cq_tail;
557+ ctx->evfd_last_cq_tail = ctx->cached_cq_tail;
558+ spin_unlock(&ctx->completion_lock);
559+ if (skip)
560+ return;
561+
562+ io_eventfd_signal(ctx);
563+}
564+
523565 void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
524566 {
525567 if (ctx->off_timeout_used || ctx->drain_active) {
@@ -531,7 +573,7 @@ void __io_commit_cqring_flush(struct io_ring_ctx *ctx) · 5 unmodified lines
531573 spin_unlock(&ctx->completion_lock);
532574 }
533575 if (ctx->has_evfd)
534− io_eventfd_signal(ctx);
576+ io_eventfd_flush_signal(ctx);
535577 }
536578
537579 static inline void io_cqring_ev_posted(struct io_ring_ctx *ctx)
@@ -567,7 +609,7 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) · 29 unmodified lines
567609
568610 io_cq_lock(ctx);
569611 while (!list_empty(&ctx->cq_overflow_list)) {
570− struct io_uring_cqe *cqe = io_get_cqe(ctx);
612+ struct io_uring_cqe *cqe = io_get_cqe_overflow(ctx, true);
571613 struct io_overflow_cqe *ocqe;
572614
573615 if (!cqe && !force)
@@ -694,12 +736,19 @@ bool io_req_cqe_overflow(struct io_kiocb *req) · 120 unmodified lines
694736 * control dependency is enough as we're using WRITE_ONCE to
695737 * fill the cq entry
696738 */
697−struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx)
739+struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow)
698740 {
699741 struct io_rings *rings = ctx->rings;
700742 unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1);
701743 unsigned int free, queued, len;
702744
745+ /*
746+ * Posting into the CQ when there are pending overflowed CQEs may break
747+ * ordering guarantees, which will affect links, F_MORE users and more.
748+ * Force overflow the completion.
749+ */
750+ if (!overflow && (ctx->check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)))
751+ return NULL;
703752
704753 /* userspace may cheat modifying the tail, be safe and do min */
705754 queued = min(__io_cqring_events(ctx), ctx->cq_entries);
@@ -823,8 +872,12 @@ inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags) · 117 unmodified lines
823872
824873 void io_req_complete_failed(struct io_kiocb *req, s32 res)
825874 {
875+ const struct io_op_def *def = &io_op_defs[req->opcode];
876+
826877 req_set_fail(req);
827878 io_req_set_res(req, res, io_put_kbuf(req, IO_URING_F_UNLOCKED));
879+ if (def->fail)
880+ def->fail(req);
828881 io_req_complete_post(req);
829882 }
830883
@@ -1047,17 +1100,40 @@ void tctx_task_work(struct callback_head *cb) · 216 unmodified lines
10471100 trace_io_uring_task_work_run(tctx, count, loops);
10481101 }
10491102
1050−void io_req_task_work_add(struct io_kiocb *req)
1103+static void io_req_local_work_add(struct io_kiocb *req)
1104+{
1105+ struct io_ring_ctx *ctx = req->ctx;
1106+
1107+ if (!llist_add(&req->io_task_work.node, &ctx->work_llist))
1108+ return;
1109+
1110+ if (unlikely(atomic_read(&req->task->io_uring->in_idle))) {
1111+ io_move_task_work_from_local(ctx);
1112+ return;
1113+ }
1114+
1115+ if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
1116+ atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
1117+
1118+ if (ctx->has_evfd)
1119+ io_eventfd_signal(ctx);
1120+ io_cqring_wake(ctx);
1121+
1122+}
1123+
1124+static inline void __io_req_task_work_add(struct io_kiocb *req, bool allow_local)
10511125 {
10521126 struct io_uring_task *tctx = req->task->io_uring;
10531127 struct io_ring_ctx *ctx = req->ctx;
10541128 struct llist_node *node;
1055− bool running;
10561129
1057− running = !llist_add(&req->io_task_work.node, &tctx->task_list);
1130+ if (allow_local && ctx->flags & IORING_SETUP_DEFER_TASKRUN) {
1131+ io_req_local_work_add(req);
1132+ return;
1133+ }
10581134
10591135 /* task_work already pending, we're done */
1060− if (running)
1136+ if (!llist_add(&req->io_task_work.node, &tctx->task_list))
10611137 return;
10621138
10631139 if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
@@ -1077,6 +1153,84 @@ void io_req_task_work_add(struct io_kiocb *req) · 13 unmodified lines
10771153 }
10781154 }
10791155
1156+void io_req_task_work_add(struct io_kiocb *req)
1157+{
1158+ __io_req_task_work_add(req, true);
1159+}
1160+
1161+static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx)
1162+{
1163+ struct llist_node *node;
1164+
1165+ node = llist_del_all(&ctx->work_llist);
1166+ while (node) {
1167+ struct io_kiocb *req = container_of(node, struct io_kiocb,
1168+ io_task_work.node);
1169+
1170+ node = node->next;
1171+ __io_req_task_work_add(req, false);
1172+ }
1173+}
1174+
1175+int __io_run_local_work(struct io_ring_ctx *ctx, bool locked)
1176+{
1177+ struct llist_node *node;
1178+ struct llist_node fake;
1179+ struct llist_node *current_final = NULL;
1180+ int ret;
1181+ unsigned int loops = 1;
1182+
1183+ if (unlikely(ctx->submitter_task != current))
1184+ return -EEXIST;
1185+
1186+ node = io_llist_xchg(&ctx->work_llist, &fake);
1187+ ret = 0;
1188+again:
1189+ while (node != current_final) {
1190+ struct llist_node *next = node->next;
1191+ struct io_kiocb *req = container_of(node, struct io_kiocb,
1192+ io_task_work.node);
1193+ prefetch(container_of(next, struct io_kiocb, io_task_work.node));
1194+ req->io_task_work.func(req, &locked);
1195+ ret++;
1196+ node = next;
1197+ }
1198+
1199+ if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
1200+ atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
1201+
1202+ node = io_llist_cmpxchg(&ctx->work_llist, &fake, NULL);
1203+ if (node != &fake) {
1204+ loops++;
1205+ current_final = &fake;
1206+ node = io_llist_xchg(&ctx->work_llist, &fake);
1207+ goto again;
1208+ }
1209+
1210+ if (locked)
1211+ io_submit_flush_completions(ctx);
1212+ trace_io_uring_local_work_run(ctx, ret, loops);
1213+ return ret;
1214+
1215+}
1216+
1217+int io_run_local_work(struct io_ring_ctx *ctx)
1218+{
1219+ bool locked;
1220+ int ret;
1221+
1222+ if (llist_empty(&ctx->work_llist))
1223+ return 0;
1224+
1225+ __set_current_state(TASK_RUNNING);
1226+ locked = mutex_trylock(&ctx->uring_lock);
1227+ ret = __io_run_local_work(ctx, locked);
1228+ if (locked)
1229+ mutex_unlock(&ctx->uring_lock);
1230+
1231+ return ret;
1232+}
1233+
10801234 static void io_req_tw_post(struct io_kiocb *req, bool *locked)
10811235 {
10821236 io_req_complete_post(req);
@@ -1183,7 +1337,7 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx) · 100 unmodified lines
11831337 struct io_wq_work_node *node, *prev;
11841338 struct io_submit_state *state = &ctx->submit_state;
11851339
1186− spin_lock(&ctx->completion_lock);
1340+ io_cq_lock(ctx);
11871341 wq_list_for_each(node, prev, &state->compl_reqs) {
11881342 struct io_kiocb *req = container_of(node, struct io_kiocb,
11891343 comp_list);
@@ -1254,6 +1408,9 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) · 64 unmodified lines
12541408 int ret = 0;
12551409 unsigned long check_cq;
12561410
1411+ if (!io_allowed_run_tw(ctx))
1412+ return -EEXIST;
1413+
12571414 check_cq = READ_ONCE(ctx->check_cq);
12581415 if (unlikely(check_cq)) {
12591416 if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
@@ -1284,13 +1441,19 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) · 24 unmodified lines
12841441 * forever, while the workqueue is stuck trying to acquire the
12851442 * very same mutex.
12861443 */
1287− if (wq_list_empty(&ctx->iopoll_list)) {
1444+ if (wq_list_empty(&ctx->iopoll_list) ||
1445+ io_task_work_pending(ctx)) {
12881446 u32 tail = ctx->cached_cq_tail;
12891447
1290− mutex_unlock(&ctx->uring_lock);
1291− io_run_task_work();
1292− mutex_lock(&ctx->uring_lock);
1448+ if (!llist_empty(&ctx->work_llist))
1449+ __io_run_local_work(ctx, true);
12931450
1451+ if (task_work_pending(current) ||
1452+ wq_list_empty(&ctx->iopoll_list)) {
1453+ mutex_unlock(&ctx->uring_lock);
1454+ io_run_task_work();
1455+ mutex_lock(&ctx->uring_lock);
1456+ }
12941457 /* some requests don't go through iopoll_list */
12951458 if (tail != ctx->cached_cq_tail ||
12961459 wq_list_empty(&ctx->iopoll_list))
@@ -1732,10 +1895,6 @@ static void io_queue_async(struct io_kiocb *req, int ret) · 435 unmodified lines
17321895 io_req_task_queue(req);
17331896 break;
17341897 case IO_APOLL_ABORTED:
1735− /*
1736− * Queued up for async execution, worker will release
1737− * submit reference when the iocb is actually submitted.
1738− */
17391898 io_kbuf_recycle(req, 0);
17401899 io_queue_iowq(req, NULL);
17411900 break;
@@ -2149,6 +2308,13 @@ struct io_wait_queue { · 407 unmodified lines
21492308 unsigned nr_timeouts;
21502309 };
21512310
2311+static inline bool io_has_work(struct io_ring_ctx *ctx)
2312+{
2313+ return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) ||
2314+ ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&
2315+ !llist_empty(&ctx->work_llist));
2316+}
2317+
21522318 static inline bool io_should_wake(struct io_wait_queue *iowq)
21532319 {
21542320 struct io_ring_ctx *ctx = iowq->ctx;
@@ -2167,20 +2333,20 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, · 12 unmodified lines
21672333 {
21682334 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
21692335 wq);
2336+ struct io_ring_ctx *ctx = iowq->ctx;
21702337
21712338 /*
21722339 * Cannot safely flush overflowed CQEs from here, ensure we wake up
21732340 * the task, and the next invocation will do it.
21742341 */
2175− if (io_should_wake(iowq) ||
2176− test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &iowq->ctx->check_cq))
2342+ if (io_should_wake(iowq) || io_has_work(ctx))
21772343 return autoremove_wake_function(curr, mode, wake_flags, key);
21782344 return -1;
21792345 }
21802346
2181−int io_run_task_work_sig(void)
2347+int io_run_task_work_sig(struct io_ring_ctx *ctx)
21822348 {
2183− if (io_run_task_work())
2349+ if (io_run_task_work_ctx(ctx) > 0)
21842350 return 1;
21852351 if (task_sigpending(current))
21862352 return -EINTR;
@@ -2196,7 +2362,7 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, · 9 unmodified lines
21962362 unsigned long check_cq;
21972363
21982364 /* make sure we run task_work before checking for signals */
2199− ret = io_run_task_work_sig();
2365+ ret = io_run_task_work_sig(ctx);
22002366 if (ret || io_should_wake(iowq))
22012367 return ret;
22022368
@@ -2226,13 +2392,19 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 23 unmodified lines
22262392 ktime_t timeout = KTIME_MAX;
22272393 int ret;
22282394
2395+ if (!io_allowed_run_tw(ctx))
2396+ return -EEXIST;
2397+
22292398 do {
2399+ /* always run at least 1 task work to process local work */
2400+ ret = io_run_task_work_ctx(ctx);
2401+ if (ret < 0)
2402+ return ret;
22302403 io_cqring_overflow_flush(ctx);
2404+
22312405 if (io_cqring_events(ctx) >= min_events)
22322406 return 0;
2233− if (!io_run_task_work())
2234− break;
2235− } while (1);
2407+ } while (ret > 0);
22362408
22372409 if (sig) {
22382410 #ifdef CONFIG_COMPAT
@@ -2366,17 +2538,11 @@ static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, · 127 unmodified lines
23662538 ev_fd->eventfd_async = eventfd_async;
23672539 ctx->has_evfd = true;
23682540 rcu_assign_pointer(ctx->io_ev_fd, ev_fd);
2541+ atomic_set(&ev_fd->refs, 1);
2542+ atomic_set(&ev_fd->ops, 0);
23692543 return 0;
23702544 }
23712545
2372−static void io_eventfd_put(struct rcu_head *rcu)
2373−{
2374− struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);
2375−
2376− eventfd_ctx_put(ev_fd->cq_ev_fd);
2377− kfree(ev_fd);
2378−}
2379−
23802546 static int io_eventfd_unregister(struct io_ring_ctx *ctx)
23812547 {
23822548 struct io_ev_fd *ev_fd;
@@ -2386,7 +2552,8 @@ static int io_eventfd_unregister(struct io_ring_ctx *ctx) · 3 unmodified lines
23862552 if (ev_fd) {
23872553 ctx->has_evfd = false;
23882554 rcu_assign_pointer(ctx->io_ev_fd, NULL);
2389− call_rcu(&ev_fd->rcu, io_eventfd_put);
2555+ if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_FREE_BIT), &ev_fd->ops))
2556+ call_rcu(&ev_fd->rcu, io_eventfd_ops);
23902557 return 0;
23912558 }
23922559
@@ -2509,8 +2676,8 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) · 116 unmodified lines
25092676 * Users may get EPOLLIN meanwhile seeing nothing in cqring, this
25102677 * pushs them to do the flush.
25112678 */
2512− if (io_cqring_events(ctx) ||
2513− test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq))
2679+
2680+ if (io_cqring_events(ctx) || io_has_work(ctx))
25142681 mask |= EPOLLIN | EPOLLRDNORM;
25152682
25162683 return mask;
@@ -2573,6 +2740,9 @@ static __cold void io_ring_exit_work(struct work_struct *work) · 56 unmodified lines
25732740 * as nobody else will be looking for them.
25742741 */
25752742 do {
2743+ if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
2744+ io_move_task_work_from_local(ctx);
2745+
25762746 while (io_uring_try_cancel_requests(ctx, NULL, true))
25772747 cond_resched();
25782748
@@ -2770,13 +2940,15 @@ static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, · 191 unmodified lines
27702940 }
27712941 }
27722942
2943+ if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
2944+ ret |= io_run_local_work(ctx) > 0;
27732945 ret |= io_cancel_defer_files(ctx, task, cancel_all);
27742946 mutex_lock(&ctx->uring_lock);
27752947 ret |= io_poll_remove_all(ctx, task, cancel_all);
27762948 mutex_unlock(&ctx->uring_lock);
27772949 ret |= io_kill_timeouts(ctx, task, cancel_all);
27782950 if (task)
2779− ret |= io_run_task_work();
2951+ ret |= io_run_task_work() > 0;
27802952 return ret;
27812953 }
27822954
@@ -2992,8 +3164,6 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 209 unmodified lines
29923164 struct fd f;
29933165 long ret;
29943166
2995− io_run_task_work();
2996−
29973167 if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
29983168 IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG |
29993169 IORING_ENTER_REGISTERED_RING)))
@@ -3063,8 +3233,10 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 63 unmodified lines
30633233 goto iopoll_locked;
30643234 mutex_unlock(&ctx->uring_lock);
30653235 }
3236+
30663237 if (flags & IORING_ENTER_GETEVENTS) {
30673238 int ret2;
3239+
30683240 if (ctx->syscall_iopoll) {
30693241 /*
30703242 * We disallow the app entering submit/complete with
@@ -3293,17 +3465,29 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, · 222 unmodified lines
32933465 if (ctx->flags & IORING_SETUP_SQPOLL) {
32943466 /* IPI related flags don't make sense with SQPOLL */
32953467 if (ctx->flags & (IORING_SETUP_COOP_TASKRUN |
3296− IORING_SETUP_TASKRUN_FLAG))
3468+ IORING_SETUP_TASKRUN_FLAG |
3469+ IORING_SETUP_DEFER_TASKRUN))
32973470 goto err;
32983471 ctx->notify_method = TWA_SIGNAL_NO_IPI;
32993472 } else if (ctx->flags & IORING_SETUP_COOP_TASKRUN) {
33003473 ctx->notify_method = TWA_SIGNAL_NO_IPI;
33013474 } else {
3302− if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
3475+ if (ctx->flags & IORING_SETUP_TASKRUN_FLAG &&
3476+ !(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
33033477 goto err;
33043478 ctx->notify_method = TWA_SIGNAL;
33053479 }
33063480
3481+ /*
3482+ * For DEFER_TASKRUN we require the completion task to be the same as the
3483+ * submission task. This implies that there is only one submitter, so enforce
3484+ * that.
3485+ */
3486+ if (ctx->flags & IORING_SETUP_DEFER_TASKRUN &&
3487+ !(ctx->flags & IORING_SETUP_SINGLE_ISSUER)) {
3488+ goto err;
3489+ }
3490+
33073491 /*
33083492 * This is just grabbed for accounting purposes. When a process exits,
33093493 * the mm is exited and dropped before the files, hence we need to hang
@@ -3408,7 +3592,7 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) · 98 unmodified lines
34083592 IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL |
34093593 IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG |
34103594 IORING_SETUP_SQE128 | IORING_SETUP_CQE32 |
3411− IORING_SETUP_SINGLE_ISSUER))
3595+ IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN))
34123596 return -EINVAL;
34133597
34143598 return io_uring_create(entries, &p, params);
@@ -3874,7 +4058,7 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, · 459 unmodified lines
38744058
38754059 ctx = f.file->private_data;
38764060
3877− io_run_task_work();
4061+ io_run_task_work_ctx(ctx);
38784062
38794063 mutex_lock(&ctx->uring_lock);
38804064 ret = __io_uring_register(ctx, opcode, arg, nr_args);
io_uring/io_uring.h+51−11
@@ -24,9 +24,11 @@ enum { · 23 unmodified lines
2424 IOU_STOP_MULTISHOT = -ECANCELED,
2525 };
2626
27−struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx);
27+struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow);
2828 bool io_req_cqe_overflow(struct io_kiocb *req);
29−int io_run_task_work_sig(void);
29+int io_run_task_work_sig(struct io_ring_ctx *ctx);
30+int __io_run_local_work(struct io_ring_ctx *ctx, bool locked);
31+int io_run_local_work(struct io_ring_ctx *ctx);
3032 void io_req_complete_failed(struct io_kiocb *req, s32 res);
3133 void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);
3234 void io_req_complete_post(struct io_kiocb *req);
@@ -91,7 +93,8 @@ static inline void io_cq_lock(struct io_ring_ctx *ctx) · 58 unmodified lines
9193
9294 void io_cq_unlock_post(struct io_ring_ctx *ctx);
9395
94−static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
96+static inline struct io_uring_cqe *io_get_cqe_overflow(struct io_ring_ctx *ctx,
97+ bool overflow)
9598 {
9699 if (likely(ctx->cqe_cached < ctx->cqe_sentinel)) {
97100 struct io_uring_cqe *cqe = ctx->cqe_cached;
@@ -103,7 +106,12 @@ static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx) · 5 unmodified lines
103106 return cqe;
104107 }
105108
106− return __io_get_cqe(ctx);
109+ return __io_get_cqe(ctx, overflow);
110+}
111+
112+static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
113+{
114+ return io_get_cqe_overflow(ctx, false);
107115 }
108116
109117 static inline bool __io_fill_cqe_req(struct io_ring_ctx *ctx,
@@ -221,17 +229,43 @@ static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) · 111 unmodified lines
221229 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
222230 }
223231
224−static inline bool io_run_task_work(void)
232+static inline int io_run_task_work(void)
225233 {
226− if (test_thread_flag(TIF_NOTIFY_SIGNAL)) {
234+ if (task_work_pending(current)) {
235+ if (test_thread_flag(TIF_NOTIFY_SIGNAL))
236+ clear_notify_signal();
227237 __set_current_state(TASK_RUNNING);
228− clear_notify_signal();
229− if (task_work_pending(current))
230− task_work_run();
231− return true;
238+ task_work_run();
239+ return 1;
232240 }
233241
234− return false;
242+ return 0;
243+}
244+
245+static inline bool io_task_work_pending(struct io_ring_ctx *ctx)
246+{
247+ return test_thread_flag(TIF_NOTIFY_SIGNAL) ||
248+ !wq_list_empty(&ctx->work_llist);
249+}
250+
251+static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx)
252+{
253+ int ret = 0;
254+ int ret2;
255+
256+ if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
257+ ret = io_run_local_work(ctx);
258+
259+ /* want to run this after in case more is added */
260+ ret2 = io_run_task_work();
261+
262+ /* Try propagate error in favour of if tasks were run,
263+ * but still make sure to run them if requested
264+ */
265+ if (ret >= 0)
266+ ret += ret2;
267+
268+ return ret;
235269 }
236270
237271 static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)
@@ -301,4 +335,10 @@ static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) · 63 unmodified lines
301335 return container_of(node, struct io_kiocb, comp_list);
302336 }
303337
338+static inline bool io_allowed_run_tw(struct io_ring_ctx *ctx)
339+{
340+ return likely(!(ctx->flags & IORING_SETUP_DEFER_TASKRUN) ||
341+ ctx->submitter_task == current);
342+}
343+
304344 #endif
io_uring/kbuf.h+0−12
@@ -86,18 +86,6 @@ static inline bool io_do_buffer_select(struct io_kiocb *req) · 85 unmodified lines
8686
8787 static inline void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)
8888 {
89− /*
90− * READV uses fields in `struct io_rw` (len/addr) to stash the selected
91− * buffer data. However if that buffer is recycled the original request
92− * data stored in addr is lost. Therefore forbid recycling for now.
93− */
94− if (req->opcode == IORING_OP_READV) {
95− if ((req->flags & REQ_F_BUFFER_RING) && req->buf_list) {
96− req->buf_list->head++;
97− req->buf_list = NULL;
98− }
99− return;
100− }
10189 if (req->flags & REQ_F_BUFFER_SELECTED)
10290 io_kbuf_recycle_legacy(req, issue_flags);
10391 if (req->flags & REQ_F_BUFFER_RING)
io_uring/net.c+222−86
@@ -55,21 +55,14 @@ struct io_sr_msg { · 54 unmodified lines
5555 struct user_msghdr __user *umsg;
5656 void __user *buf;
5757 };
58+ unsigned len;
59+ unsigned done_io;
5860 unsigned msg_flags;
59− unsigned flags;
60− size_t len;
61− size_t done_io;
62−};
63−
64−struct io_sendzc {
65− struct file *file;
66− void __user *buf;
67− size_t len;
68− unsigned msg_flags;
69− unsigned flags;
70− unsigned addr_len;
61+ u16 flags;
62+ /* initialised and used only by !msg send variants */
63+ u16 addr_len;
7164 void __user *addr;
72− size_t done_io;
65+ /* used only for send zerocopy */
7366 struct io_kiocb *notif;
7467 };
7568
@@ -126,28 +119,36 @@ static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags) · 50 unmodified lines
126119 }
127120 }
128121
129−static struct io_async_msghdr *io_recvmsg_alloc_async(struct io_kiocb *req,
130− unsigned int issue_flags)
122+static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req,
123+ unsigned int issue_flags)
131124 {
132125 struct io_ring_ctx *ctx = req->ctx;
133126 struct io_cache_entry *entry;
127+ struct io_async_msghdr *hdr;
134128
135129 if (!(issue_flags & IO_URING_F_UNLOCKED) &&
136130 (entry = io_alloc_cache_get(&ctx->netmsg_cache)) != NULL) {
137− struct io_async_msghdr *hdr;
138−
139131 hdr = container_of(entry, struct io_async_msghdr, cache);
132+ hdr->free_iov = NULL;
140133 req->flags |= REQ_F_ASYNC_DATA;
141134 req->async_data = hdr;
142135 return hdr;
143136 }
144137
145− if (!io_alloc_async_data(req))
146− return req->async_data;
147−
138+ if (!io_alloc_async_data(req)) {
139+ hdr = req->async_data;
140+ hdr->free_iov = NULL;
141+ return hdr;
142+ }
148143 return NULL;
149144 }
150145
146+static inline struct io_async_msghdr *io_msg_alloc_async_prep(struct io_kiocb *req)
147+{
148+ /* ->prep_async is always called from the submission context */
149+ return io_msg_alloc_async(req, 0);
150+}
151+
151152 static int io_setup_async_msg(struct io_kiocb *req,
152153 struct io_async_msghdr *kmsg,
153154 unsigned int issue_flags)
@@ -156,17 +157,20 @@ static int io_setup_async_msg(struct io_kiocb *req, · 2 unmodified lines
156157
157158 if (req_has_async_data(req))
158159 return -EAGAIN;
159− async_msg = io_recvmsg_alloc_async(req, issue_flags);
160+ async_msg = io_msg_alloc_async(req, issue_flags);
160161 if (!async_msg) {
161162 kfree(kmsg->free_iov);
162163 return -ENOMEM;
163164 }
164165 req->flags |= REQ_F_NEED_CLEANUP;
165166 memcpy(async_msg, kmsg, sizeof(*kmsg));
166− async_msg->msg.msg_name = &async_msg->addr;
167+ if (async_msg->msg.msg_name)
168+ async_msg->msg.msg_name = &async_msg->addr;
167169 /* if were using fast_iov, set it to the new one */
168− if (!async_msg->free_iov)
169− async_msg->msg.msg_iter.iov = async_msg->fast_iov;
170+ if (!kmsg->free_iov) {
171+ size_t fast_idx = kmsg->msg.msg_iter.iov - kmsg->fast_iov;
172+ async_msg->msg.msg_iter.iov = &async_msg->fast_iov[fast_idx];
173+ }
170174
171175 return -EAGAIN;
172176 }
@@ -182,34 +186,34 @@ static int io_sendmsg_copy_hdr(struct io_kiocb *req, · 9 unmodified lines
182186 &iomsg->free_iov);
183187 }
184188
185−int io_sendzc_prep_async(struct io_kiocb *req)
189+int io_send_prep_async(struct io_kiocb *req)
186190 {
187− struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc);
191+ struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
188192 struct io_async_msghdr *io;
189193 int ret;
190194
191195 if (!zc->addr || req_has_async_data(req))
192196 return 0;
193− if (io_alloc_async_data(req))
197+ io = io_msg_alloc_async_prep(req);
198+ if (!io)
194199 return -ENOMEM;
195−
196− io = req->async_data;
197200 ret = move_addr_to_kernel(zc->addr, zc->addr_len, &io->addr);
198201 return ret;
199202 }
200203
201204 static int io_setup_async_addr(struct io_kiocb *req,
202− struct sockaddr_storage *addr,
205+ struct sockaddr_storage *addr_storage,
203206 unsigned int issue_flags)
204207 {
208+ struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
205209 struct io_async_msghdr *io;
206210
207− if (!addr || req_has_async_data(req))
211+ if (!sr->addr || req_has_async_data(req))
208212 return -EAGAIN;
209− if (io_alloc_async_data(req))
213+ io = io_msg_alloc_async(req, issue_flags);
214+ if (!io)
210215 return -ENOMEM;
211− io = req->async_data;
212− memcpy(&io->addr, addr, sizeof(io->addr));
216+ memcpy(&io->addr, addr_storage, sizeof(io->addr));
213217 return -EAGAIN;
214218 }
215219
@@ -217,6 +221,8 @@ int io_sendmsg_prep_async(struct io_kiocb *req) · 1 unmodified line
217221 {
218222 int ret;
219223
224+ if (!io_msg_alloc_async_prep(req))
225+ return -ENOMEM;
220226 ret = io_sendmsg_copy_hdr(req, req->async_data);
221227 if (!ret)
222228 req->flags |= REQ_F_NEED_CLEANUP;
@@ -234,8 +240,14 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 11 unmodified lines
234240 {
235241 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
236242
237− if (unlikely(sqe->file_index || sqe->addr2))
243+ if (req->opcode == IORING_OP_SEND) {
244+ if (READ_ONCE(sqe->__pad3[0]))
245+ return -EINVAL;
246+ sr->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2));
247+ sr->addr_len = READ_ONCE(sqe->addr_len);
248+ } else if (sqe->addr2 || sqe->file_index) {
238249 return -EINVAL;
250+ }
239251
240252 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
241253 sr->len = READ_ONCE(sqe->len);
@@ -291,13 +303,13 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) · 49 unmodified lines
291303 if (ret < min_ret) {
292304 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
293305 return io_setup_async_msg(req, kmsg, issue_flags);
294− if (ret == -ERESTARTSYS)
295− ret = -EINTR;
296306 if (ret > 0 && io_net_retry(sock, flags)) {
297307 sr->done_io += ret;
298308 req->flags |= REQ_F_PARTIAL_IO;
299309 return io_setup_async_msg(req, kmsg, issue_flags);
300310 }
311+ if (ret == -ERESTARTSYS)
312+ ret = -EINTR;
301313 req_set_fail(req);
302314 }
303315 /* fast path, check for non-NULL to avoid function call */
@@ -315,6 +327,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) · 11 unmodified lines
315327
316328 int io_send(struct io_kiocb *req, unsigned int issue_flags)
317329 {
330+ struct sockaddr_storage __address;
318331 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
319332 struct msghdr msg;
320333 struct iovec iov;
@@ -323,9 +336,29 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) · 2 unmodified lines
323336 int min_ret = 0;
324337 int ret;
325338
339+ msg.msg_name = NULL;
340+ msg.msg_control = NULL;
341+ msg.msg_controllen = 0;
342+ msg.msg_namelen = 0;
343+ msg.msg_ubuf = NULL;
344+
345+ if (sr->addr) {
346+ if (req_has_async_data(req)) {
347+ struct io_async_msghdr *io = req->async_data;
348+
349+ msg.msg_name = &io->addr;
350+ } else {
351+ ret = move_addr_to_kernel(sr->addr, sr->addr_len, &__address);
352+ if (unlikely(ret < 0))
353+ return ret;
354+ msg.msg_name = (struct sockaddr *)&__address;
355+ }
356+ msg.msg_namelen = sr->addr_len;
357+ }
358+
326359 if (!(req->flags & REQ_F_POLLED) &&
327360 (sr->flags & IORING_RECVSEND_POLL_FIRST))
328− return -EAGAIN;
361+ return io_setup_async_addr(req, &__address, issue_flags);
329362
330363 sock = sock_from_file(req->file);
331364 if (unlikely(!sock))
@@ -335,12 +368,6 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) · 3 unmodified lines
335368 if (unlikely(ret))
336369 return ret;
337370
338− msg.msg_name = NULL;
339− msg.msg_control = NULL;
340− msg.msg_controllen = 0;
341− msg.msg_namelen = 0;
342− msg.msg_ubuf = NULL;
343−
344371 flags = sr->msg_flags;
345372 if (issue_flags & IO_URING_F_NONBLOCK)
346373 flags |= MSG_DONTWAIT;
@@ -351,16 +378,17 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) · 4 unmodified lines
351378 ret = sock_sendmsg(sock, &msg);
352379 if (ret < min_ret) {
353380 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
354− return -EAGAIN;
355− if (ret == -ERESTARTSYS)
356− ret = -EINTR;
381+ return io_setup_async_addr(req, &__address, issue_flags);
382+
357383 if (ret > 0 && io_net_retry(sock, flags)) {
358384 sr->len -= ret;
359385 sr->buf += ret;
360386 sr->done_io += ret;
361387 req->flags |= REQ_F_PARTIAL_IO;
362− return -EAGAIN;
388+ return io_setup_async_addr(req, &__address, issue_flags);
363389 }
390+ if (ret == -ERESTARTSYS)
391+ ret = -EINTR;
364392 req_set_fail(req);
365393 }
366394 if (ret >= 0)
@@ -454,7 +482,6 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, · 87 unmodified lines
454482
455483 if (msg.msg_iovlen == 0) {
456484 sr->len = 0;
457− iomsg->free_iov = NULL;
458485 } else if (msg.msg_iovlen > 1) {
459486 return -EINVAL;
460487 } else {
@@ -465,7 +492,6 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, · 4 unmodified lines
465492 if (clen < 0)
466493 return -EINVAL;
467494 sr->len = clen;
468− iomsg->free_iov = NULL;
469495 }
470496
471497 if (req->flags & REQ_F_APOLL_MULTISHOT) {
@@ -504,6 +530,8 @@ int io_recvmsg_prep_async(struct io_kiocb *req) · 32 unmodified lines
504530 {
505531 int ret;
506532
533+ if (!io_msg_alloc_async_prep(req))
534+ return -ENOMEM;
507535 ret = io_recvmsg_copy_hdr(req, req->async_data);
508536 if (!ret)
509537 req->flags |= REQ_F_NEED_CLEANUP;
@@ -751,13 +779,13 @@ retry_multishot: · 241 unmodified lines
751779 }
752780 return ret;
753781 }
754− if (ret == -ERESTARTSYS)
755− ret = -EINTR;
756782 if (ret > 0 && io_net_retry(sock, flags)) {
757783 sr->done_io += ret;
758784 req->flags |= REQ_F_PARTIAL_IO;
759785 return io_setup_async_msg(req, kmsg, issue_flags);
760786 }
787+ if (ret == -ERESTARTSYS)
788+ ret = -EINTR;
761789 req_set_fail(req);
762790 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
763791 req_set_fail(req);
@@ -847,8 +875,6 @@ retry_multishot: · 83 unmodified lines
847875
848876 return -EAGAIN;
849877 }
850− if (ret == -ERESTARTSYS)
851− ret = -EINTR;
852878 if (ret > 0 && io_net_retry(sock, flags)) {
853879 sr->len -= ret;
854880 sr->buf += ret;
@@ -856,6 +882,8 @@ retry_multishot: · 1 unmodified line
856882 req->flags |= REQ_F_PARTIAL_IO;
857883 return -EAGAIN;
858884 }
885+ if (ret == -ERESTARTSYS)
886+ ret = -EINTR;
859887 req_set_fail(req);
860888 } else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
861889 out_free:
@@ -879,23 +907,30 @@ out_free: · 17 unmodified lines
879907 return ret;
880908 }
881909
882−void io_sendzc_cleanup(struct io_kiocb *req)
910+void io_send_zc_cleanup(struct io_kiocb *req)
883911 {
884− struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc);
912+ struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
913+ struct io_async_msghdr *io;
885914
886− zc->notif->flags |= REQ_F_CQE_SKIP;
887− io_notif_flush(zc->notif);
888− zc->notif = NULL;
915+ if (req_has_async_data(req)) {
916+ io = req->async_data;
917+ /* might be ->fast_iov if *msg_copy_hdr failed */
918+ if (io->free_iov != io->fast_iov)
919+ kfree(io->free_iov);
920+ }
921+ if (zc->notif) {
922+ io_notif_flush(zc->notif);
923+ zc->notif = NULL;
924+ }
889925 }
890926
891−int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
927+int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
892928 {
893− struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc);
929+ struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
894930 struct io_ring_ctx *ctx = req->ctx;
895931 struct io_kiocb *notif;
896932
897− if (READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3) ||
898− READ_ONCE(sqe->__pad3[0]))
933+ if (unlikely(READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3)))
899934 return -EINVAL;
900935 /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */
901936 if (req->flags & REQ_F_CQE_SKIP)
@@ -922,14 +957,24 @@ int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 20 unmodified lines
922957 io_req_set_rsrc_node(notif, ctx, 0);
923958 }
924959
960+ if (req->opcode == IORING_OP_SEND_ZC) {
961+ if (READ_ONCE(sqe->__pad3[0]))
962+ return -EINVAL;
963+ zc->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2));
964+ zc->addr_len = READ_ONCE(sqe->addr_len);
965+ } else {
966+ if (unlikely(sqe->addr2 || sqe->file_index))
967+ return -EINVAL;
968+ if (unlikely(zc->flags & IORING_RECVSEND_FIXED_BUF))
969+ return -EINVAL;
970+ }
971+
925972 zc->buf = u64_to_user_ptr(READ_ONCE(sqe->addr));
926973 zc->len = READ_ONCE(sqe->len);
927974 zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
928975 if (zc->msg_flags & MSG_DONTWAIT)
929976 req->flags |= REQ_F_NOWAIT;
930977
931− zc->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2));
932− zc->addr_len = READ_ONCE(sqe->addr_len);
933978 zc->done_io = 0;
934979
935980 #ifdef CONFIG_COMPAT
@@ -939,6 +984,13 @@ int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 3 unmodified lines
939984 return 0;
940985 }
941986
987+static int io_sg_from_iter_iovec(struct sock *sk, struct sk_buff *skb,
988+ struct iov_iter *from, size_t length)
989+{
990+ skb_zcopy_downgrade_managed(skb);
991+ return __zerocopy_sg_from_iter(NULL, sk, skb, from, length);
992+}
993+
942994 static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb,
943995 struct iov_iter *from, size_t length)
944996 {
@@ -949,13 +1001,10 @@ static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb, · 4 unmodified lines
9491001 ssize_t copied = 0;
9501002 unsigned long truesize = 0;
9511003
952− if (!shinfo->nr_frags)
1004+ if (!frag)
9531005 shinfo->flags |= SKBFL_MANAGED_FRAG_REFS;
954−
955− if (!skb_zcopy_managed(skb) || !iov_iter_is_bvec(from)) {
956− skb_zcopy_downgrade_managed(skb);
1006+ else if (unlikely(!skb_zcopy_managed(skb)))
9571007 return __zerocopy_sg_from_iter(NULL, sk, skb, from, length);
958− }
9591008
9601009 bi.bi_size = min(from->count, length);
9611010 bi.bi_bvec_done = from->iov_offset;
@@ -993,14 +1042,14 @@ static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb, · 31 unmodified lines
9931042 return ret;
9941043 }
9951044
996−int io_sendzc(struct io_kiocb *req, unsigned int issue_flags)
1045+int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)
9971046 {
998− struct sockaddr_storage __address, *addr = NULL;
999− struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc);
1047+ struct sockaddr_storage __address;
1048+ struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
10001049 struct msghdr msg;
10011050 struct iovec iov;
10021051 struct socket *sock;
1003− unsigned msg_flags, cflags;
1052+ unsigned msg_flags;
10041053 int ret, min_ret = 0;
10051054
10061055 sock = sock_from_file(req->file);
@@ -1016,26 +1065,26 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) · 9 unmodified lines
10161065 if (req_has_async_data(req)) {
10171066 struct io_async_msghdr *io = req->async_data;
10181067
1019− msg.msg_name = addr = &io->addr;
1068+ msg.msg_name = &io->addr;
10201069 } else {
10211070 ret = move_addr_to_kernel(zc->addr, zc->addr_len, &__address);
10221071 if (unlikely(ret < 0))
10231072 return ret;
10241073 msg.msg_name = (struct sockaddr *)&__address;
1025− addr = &__address;
10261074 }
10271075 msg.msg_namelen = zc->addr_len;
10281076 }
10291077
10301078 if (!(req->flags & REQ_F_POLLED) &&
10311079 (zc->flags & IORING_RECVSEND_POLL_FIRST))
1032− return io_setup_async_addr(req, addr, issue_flags);
1080+ return io_setup_async_addr(req, &__address, issue_flags);
10331081
10341082 if (zc->flags & IORING_RECVSEND_FIXED_BUF) {
10351083 ret = io_import_fixed(WRITE, &msg.msg_iter, req->imu,
10361084 (u64)(uintptr_t)zc->buf, zc->len);
10371085 if (unlikely(ret))
10381086 return ret;
1087+ msg.sg_from_iter = io_sg_from_iter;
10391088 } else {
10401089 ret = import_single_range(WRITE, zc->buf, zc->len, &iov,
10411090 &msg.msg_iter);
@@ -1044,6 +1093,7 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) · 2 unmodified lines
10441093 ret = io_notif_account_mem(zc->notif, zc->len);
10451094 if (unlikely(ret))
10461095 return ret;
1096+ msg.sg_from_iter = io_sg_from_iter_iovec;
10471097 }
10481098
10491099 msg_flags = zc->msg_flags | MSG_ZEROCOPY;
@@ -1054,22 +1104,19 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) · 4 unmodified lines
10541104
10551105 msg.msg_flags = msg_flags;
10561106 msg.msg_ubuf = &io_notif_to_data(zc->notif)->uarg;
1057− msg.sg_from_iter = io_sg_from_iter;
10581107 ret = sock_sendmsg(sock, &msg);
10591108
10601109 if (unlikely(ret < min_ret)) {
10611110 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1062− return io_setup_async_addr(req, addr, issue_flags);
1111+ return io_setup_async_addr(req, &__address, issue_flags);
10631112
10641113 if (ret > 0 && io_net_retry(sock, msg.msg_flags)) {
10651114 zc->len -= ret;
10661115 zc->buf += ret;
10671116 zc->done_io += ret;
10681117 req->flags |= REQ_F_PARTIAL_IO;
1069− return io_setup_async_addr(req, addr, issue_flags);
1118+ return io_setup_async_addr(req, &__address, issue_flags);
10701119 }
1071− if (ret < 0 && !zc->done_io)
1072− zc->notif->flags |= REQ_F_CQE_SKIP;
10731120 if (ret == -ERESTARTSYS)
10741121 ret = -EINTR;
10751122 req_set_fail(req);
@@ -1080,13 +1127,102 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) · 4 unmodified lines
10801127 else if (zc->done_io)
10811128 ret = zc->done_io;
10821129
1083− io_notif_flush(zc->notif);
1084− req->flags &= ~REQ_F_NEED_CLEANUP;
1085− cflags = ret >= 0 ? IORING_CQE_F_MORE : 0;
1086− io_req_set_res(req, ret, cflags);
1130+ /*
1131+ * If we're in io-wq we can't rely on tw ordering guarantees, defer
1132+ * flushing notif to io_send_zc_cleanup()
1133+ */
1134+ if (!(issue_flags & IO_URING_F_UNLOCKED)) {
1135+ io_notif_flush(zc->notif);
1136+ req->flags &= ~REQ_F_NEED_CLEANUP;
1137+ }
1138+ io_req_set_res(req, ret, IORING_CQE_F_MORE);
10871139 return IOU_OK;
10881140 }
10891141
1142+int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)
1143+{
1144+ struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1145+ struct io_async_msghdr iomsg, *kmsg;
1146+ struct socket *sock;
1147+ unsigned flags;
1148+ int ret, min_ret = 0;
1149+
1150+ sock = sock_from_file(req->file);
1151+ if (unlikely(!sock))
1152+ return -ENOTSOCK;
1153+
1154+ if (req_has_async_data(req)) {
1155+ kmsg = req->async_data;
1156+ } else {
1157+ ret = io_sendmsg_copy_hdr(req, &iomsg);
1158+ if (ret)
1159+ return ret;
1160+ kmsg = &iomsg;
1161+ }
1162+
1163+ if (!(req->flags & REQ_F_POLLED) &&
1164+ (sr->flags & IORING_RECVSEND_POLL_FIRST))
1165+ return io_setup_async_msg(req, kmsg, issue_flags);
1166+
1167+ flags = sr->msg_flags | MSG_ZEROCOPY;
1168+ if (issue_flags & IO_URING_F_NONBLOCK)
1169+ flags |= MSG_DONTWAIT;
1170+ if (flags & MSG_WAITALL)
1171+ min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1172+
1173+ kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg;
1174+ kmsg->msg.sg_from_iter = io_sg_from_iter_iovec;
1175+ ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
1176+
1177+ if (unlikely(ret < min_ret)) {
1178+ if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1179+ return io_setup_async_msg(req, kmsg, issue_flags);
1180+
1181+ if (ret > 0 && io_net_retry(sock, flags)) {
1182+ sr->done_io += ret;
1183+ req->flags |= REQ_F_PARTIAL_IO;
1184+ return io_setup_async_msg(req, kmsg, issue_flags);
1185+ }
1186+ if (ret == -ERESTARTSYS)
1187+ ret = -EINTR;
1188+ req_set_fail(req);
1189+ }
1190+ /* fast path, check for non-NULL to avoid function call */
1191+ if (kmsg->free_iov) {
1192+ kfree(kmsg->free_iov);
1193+ kmsg->free_iov = NULL;
1194+ }
1195+
1196+ io_netmsg_recycle(req, issue_flags);
1197+ if (ret >= 0)
1198+ ret += sr->done_io;
1199+ else if (sr->done_io)
1200+ ret = sr->done_io;
1201+
1202+ /*
1203+ * If we're in io-wq we can't rely on tw ordering guarantees, defer
1204+ * flushing notif to io_send_zc_cleanup()
1205+ */
1206+ if (!(issue_flags & IO_URING_F_UNLOCKED)) {
1207+ io_notif_flush(sr->notif);
1208+ req->flags &= ~REQ_F_NEED_CLEANUP;
1209+ }
1210+ io_req_set_res(req, ret, IORING_CQE_F_MORE);
1211+ return IOU_OK;
1212+}
1213+
1214+void io_sendrecv_fail(struct io_kiocb *req)
1215+{
1216+ struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1217+
1218+ if (req->flags & REQ_F_PARTIAL_IO)
1219+ req->cqe.res = sr->done_io;
1220+
1221+ if ((req->flags & REQ_F_NEED_CLEANUP) &&
1222+ (req->opcode == IORING_OP_SEND_ZC || req->opcode == IORING_OP_SENDMSG_ZC))
1223+ req->cqe.flags |= IORING_CQE_F_MORE;
1224+}
1225+
10901226 int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
10911227 {
10921228 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept);
io_uring/net.h+8−4
@@ -31,18 +31,21 @@ struct io_async_connect { · 30 unmodified lines
3131 int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
3232 int io_shutdown(struct io_kiocb *req, unsigned int issue_flags);
3333
34−int io_sendzc_prep_async(struct io_kiocb *req);
3534 int io_sendmsg_prep_async(struct io_kiocb *req);
3635 void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req);
3736 int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
3837 int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags);
38+
3939 int io_send(struct io_kiocb *req, unsigned int issue_flags);
40+int io_send_prep_async(struct io_kiocb *req);
4041
4142 int io_recvmsg_prep_async(struct io_kiocb *req);
4243 int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
4344 int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags);
4445 int io_recv(struct io_kiocb *req, unsigned int issue_flags);
4546
47+void io_sendrecv_fail(struct io_kiocb *req);
48+
4649 int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
4750 int io_accept(struct io_kiocb *req, unsigned int issue_flags);
4851
@@ -53,9 +56,10 @@ int io_connect_prep_async(struct io_kiocb *req); · 4 unmodified lines
5356 int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
5457 int io_connect(struct io_kiocb *req, unsigned int issue_flags);
5558
56−int io_sendzc(struct io_kiocb *req, unsigned int issue_flags);
57−int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
58−void io_sendzc_cleanup(struct io_kiocb *req);
59+int io_send_zc(struct io_kiocb *req, unsigned int issue_flags);
60+int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags);
61+int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
62+void io_send_zc_cleanup(struct io_kiocb *req);
5963
6064 void io_netmsg_cache_free(struct io_cache_entry *entry);
6165 #else
io_uring/opdef.c+40−4
@@ -69,6 +69,7 @@ const struct io_op_def io_op_defs[] = { · 68 unmodified lines
6969 .issue = io_read,
7070 .prep_async = io_readv_prep_async,
7171 .cleanup = io_readv_writev_cleanup,
72+ .fail = io_rw_fail,
7273 },
7374 [IORING_OP_WRITEV] = {
7475 .needs_file = 1,
@@ -85,6 +86,7 @@ const struct io_op_def io_op_defs[] = { · 10 unmodified lines
8586 .issue = io_write,
8687 .prep_async = io_writev_prep_async,
8788 .cleanup = io_readv_writev_cleanup,
89+ .fail = io_rw_fail,
8890 },
8991 [IORING_OP_FSYNC] = {
9092 .needs_file = 1,
@@ -105,6 +107,7 @@ const struct io_op_def io_op_defs[] = { · 14 unmodified lines
105107 .name = "READ_FIXED",
106108 .prep = io_prep_rw,
107109 .issue = io_read,
110+ .fail = io_rw_fail,
108111 },
109112 [IORING_OP_WRITE_FIXED] = {
110113 .needs_file = 1,
@@ -119,6 +122,7 @@ const struct io_op_def io_op_defs[] = { · 8 unmodified lines
119122 .name = "WRITE_FIXED",
120123 .prep = io_prep_rw,
121124 .issue = io_write,
125+ .fail = io_rw_fail,
122126 },
123127 [IORING_OP_POLL_ADD] = {
124128 .needs_file = 1,
@@ -146,6 +150,7 @@ const struct io_op_def io_op_defs[] = { · 21 unmodified lines
146150 .unbound_nonreg_file = 1,
147151 .pollout = 1,
148152 .ioprio = 1,
153+ .manual_alloc = 1,
149154 .name = "SENDMSG",
150155 #if defined(CONFIG_NET)
151156 .async_size = sizeof(struct io_async_msghdr),
@@ -153,6 +158,7 @@ const struct io_op_def io_op_defs[] = { · 1 unmodified line
153158 .issue = io_sendmsg,
154159 .prep_async = io_sendmsg_prep_async,
155160 .cleanup = io_sendmsg_recvmsg_cleanup,
161+ .fail = io_sendrecv_fail,
156162 #else
157163 .prep = io_eopnotsupp_prep,
158164 #endif
@@ -163,6 +169,7 @@ const struct io_op_def io_op_defs[] = { · 4 unmodified lines
163169 .pollin = 1,
164170 .buffer_select = 1,
165171 .ioprio = 1,
172+ .manual_alloc = 1,
166173 .name = "RECVMSG",
167174 #if defined(CONFIG_NET)
168175 .async_size = sizeof(struct io_async_msghdr),
@@ -170,6 +177,7 @@ const struct io_op_def io_op_defs[] = { · 1 unmodified line
170177 .issue = io_recvmsg,
171178 .prep_async = io_recvmsg_prep_async,
172179 .cleanup = io_sendmsg_recvmsg_cleanup,
180+ .fail = io_sendrecv_fail,
173181 #else
174182 .prep = io_eopnotsupp_prep,
175183 #endif
@@ -273,6 +281,7 @@ const struct io_op_def io_op_defs[] = { · 97 unmodified lines
273281 .name = "READ",
274282 .prep = io_prep_rw,
275283 .issue = io_read,
284+ .fail = io_rw_fail,
276285 },
277286 [IORING_OP_WRITE] = {
278287 .needs_file = 1,
@@ -287,6 +296,7 @@ const struct io_op_def io_op_defs[] = { · 8 unmodified lines
287296 .name = "WRITE",
288297 .prep = io_prep_rw,
289298 .issue = io_write,
299+ .fail = io_rw_fail,
290300 },
291301 [IORING_OP_FADVISE] = {
292302 .needs_file = 1,
@@ -306,10 +316,14 @@ const struct io_op_def io_op_defs[] = { · 13 unmodified lines
306316 .pollout = 1,
307317 .audit_skip = 1,
308318 .ioprio = 1,
319+ .manual_alloc = 1,
309320 .name = "SEND",
310321 #if defined(CONFIG_NET)
322+ .async_size = sizeof(struct io_async_msghdr),
311323 .prep = io_sendmsg_prep,
312324 .issue = io_send,
325+ .fail = io_sendrecv_fail,
326+ .prep_async = io_send_prep_async,
313327 #else
314328 .prep = io_eopnotsupp_prep,
315329 #endif
@@ -325,6 +339,7 @@ const struct io_op_def io_op_defs[] = { · 9 unmodified lines
325339 #if defined(CONFIG_NET)
326340 .prep = io_recvmsg_prep,
327341 .issue = io_recv,
342+ .fail = io_sendrecv_fail,
328343 #else
329344 .prep = io_eopnotsupp_prep,
330345 #endif
@@ -465,6 +480,7 @@ const struct io_op_def io_op_defs[] = { · 134 unmodified lines
465480 .needs_file = 1,
466481 .plug = 1,
467482 .name = "URING_CMD",
483+ .iopoll = 1,
468484 .async_size = uring_cmd_pdu_size(1),
469485 .prep = io_uring_cmd_prep,
470486 .issue = io_uring_cmd,
@@ -480,10 +496,30 @@ const struct io_op_def io_op_defs[] = { · 9 unmodified lines
480496 .manual_alloc = 1,
481497 #if defined(CONFIG_NET)
482498 .async_size = sizeof(struct io_async_msghdr),
483− .prep = io_sendzc_prep,
484− .issue = io_sendzc,
485− .prep_async = io_sendzc_prep_async,
486− .cleanup = io_sendzc_cleanup,
499+ .prep = io_send_zc_prep,
500+ .issue = io_send_zc,
501+ .prep_async = io_send_prep_async,
502+ .cleanup = io_send_zc_cleanup,
503+ .fail = io_sendrecv_fail,
504+#else
505+ .prep = io_eopnotsupp_prep,
506+#endif
507+ },
508+ [IORING_OP_SENDMSG_ZC] = {
509+ .name = "SENDMSG_ZC",
510+ .needs_file = 1,
511+ .unbound_nonreg_file = 1,
512+ .pollout = 1,
513+ .audit_skip = 1,
514+ .ioprio = 1,
515+ .manual_alloc = 1,
516+#if defined(CONFIG_NET)
517+ .async_size = sizeof(struct io_async_msghdr),
518+ .prep = io_send_zc_prep,
519+ .issue = io_sendmsg_zc,
520+ .prep_async = io_sendmsg_prep_async,
521+ .cleanup = io_send_zc_cleanup,
522+ .fail = io_sendrecv_fail,
487523 #else
488524 .prep = io_eopnotsupp_prep,
489525 #endif
io_uring/opdef.h+1−0
@@ -36,6 +36,7 @@ struct io_op_def { · 35 unmodified lines
3636 int (*issue)(struct io_kiocb *, unsigned int);
3737 int (*prep_async)(struct io_kiocb *);
3838 void (*cleanup)(struct io_kiocb *);
39+ void (*fail)(struct io_kiocb *);
3940 };
4041
4142 extern const struct io_op_def io_op_defs[];
io_uring/rsrc.c+1−1
@@ -341,7 +341,7 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, · 340 unmodified lines
341341 flush_delayed_work(&ctx->rsrc_put_work);
342342 reinit_completion(&data->done);
343343
344− ret = io_run_task_work_sig();
344+ ret = io_run_task_work_sig(ctx);
345345 mutex_lock(&ctx->uring_lock);
346346 } while (ret >= 0);
347347 data->quiesce = false;
io_uring/rw.c+93−96
@@ -33,6 +33,46 @@ static inline bool io_file_supports_nowait(struct io_kiocb *req) · 32 unmodified lines
3333 return req->flags & REQ_F_SUPPORT_NOWAIT;
3434 }
3535
36+#ifdef CONFIG_COMPAT
37+static int io_iov_compat_buffer_select_prep(struct io_rw *rw)
38+{
39+ struct compat_iovec __user *uiov;
40+ compat_ssize_t clen;
41+
42+ uiov = u64_to_user_ptr(rw->addr);
43+ if (!access_ok(uiov, sizeof(*uiov)))
44+ return -EFAULT;
45+ if (__get_user(clen, &uiov->iov_len))
46+ return -EFAULT;
47+ if (clen < 0)
48+ return -EINVAL;
49+
50+ rw->len = clen;
51+ return 0;
52+}
53+#endif
54+
55+static int io_iov_buffer_select_prep(struct io_kiocb *req)
56+{
57+ struct iovec __user *uiov;
58+ struct iovec iov;
59+ struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
60+
61+ if (rw->len != 1)
62+ return -EINVAL;
63+
64+#ifdef CONFIG_COMPAT
65+ if (req->ctx->compat)
66+ return io_iov_compat_buffer_select_prep(rw);
67+#endif
68+
69+ uiov = u64_to_user_ptr(rw->addr);
70+ if (copy_from_user(&iov, uiov, sizeof(*uiov)))
71+ return -EFAULT;
72+ rw->len = iov.iov_len;
73+ return 0;
74+}
75+
3676 int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3777 {
3878 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
@@ -69,6 +109,16 @@ int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 30 unmodified lines
69109 rw->addr = READ_ONCE(sqe->addr);
70110 rw->len = READ_ONCE(sqe->len);
71111 rw->flags = READ_ONCE(sqe->rw_flags);
112+
113+ /* Have to do this validation here, as this is in io_read() rw->len might
114+ * have chanaged due to buffer selection
115+ */
116+ if (req->opcode == IORING_OP_READV && req->flags & REQ_F_BUFFER_SELECT) {
117+ ret = io_iov_buffer_select_prep(req);
118+ if (ret)
119+ return ret;
120+ }
121+
72122 return 0;
73123 }
74124
@@ -186,14 +236,6 @@ static void kiocb_end_write(struct io_kiocb *req) · 111 unmodified lines
186236
187237 static bool __io_complete_rw_common(struct io_kiocb *req, long res)
188238 {
189− struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
190−
191− if (rw->kiocb.ki_flags & IOCB_WRITE) {
192− kiocb_end_write(req);
193− fsnotify_modify(req->file);
194− } else {
195− fsnotify_access(req->file);
196− }
197239 if (unlikely(res != req->cqe.res)) {
198240 if ((res == -EAGAIN || res == -EOPNOTSUPP) &&
199241 io_rw_should_reissue(req)) {
@@ -220,6 +262,20 @@ static inline int io_fixup_rw_res(struct io_kiocb *req, long res) · 20 unmodified lines
220262 return res;
221263 }
222264
265+static void io_req_rw_complete(struct io_kiocb *req, bool *locked)
266+{
267+ struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
268+
269+ if (rw->kiocb.ki_flags & IOCB_WRITE) {
270+ kiocb_end_write(req);
271+ fsnotify_modify(req->file);
272+ } else {
273+ fsnotify_access(req->file);
274+ }
275+
276+ io_req_task_complete(req, locked);
277+}
278+
223279 static void io_complete_rw(struct kiocb *kiocb, long res)
224280 {
225281 struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);
@@ -228,7 +284,7 @@ static void io_complete_rw(struct kiocb *kiocb, long res) · 2 unmodified lines
228284 if (__io_complete_rw_common(req, res))
229285 return;
230286 io_req_set_res(req, io_fixup_rw_res(req, res), 0);
231− req->io_task_work.func = io_req_task_complete;
287+ req->io_task_work.func = io_req_rw_complete;
232288 io_req_task_work_add(req);
233289 }
234290
@@ -279,79 +335,6 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret, · 44 unmodified lines
279335 return IOU_ISSUE_SKIP_COMPLETE;
280336 }
281337
282−#ifdef CONFIG_COMPAT
283−static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,
284− unsigned int issue_flags)
285−{
286− struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
287− struct compat_iovec __user *uiov;
288− compat_ssize_t clen;
289− void __user *buf;
290− size_t len;
291−
292− uiov = u64_to_user_ptr(rw->addr);
293− if (!access_ok(uiov, sizeof(*uiov)))
294− return -EFAULT;
295− if (__get_user(clen, &uiov->iov_len))
296− return -EFAULT;
297− if (clen < 0)
298− return -EINVAL;
299−
300− len = clen;
301− buf = io_buffer_select(req, &len, issue_flags);
302− if (!buf)
303− return -ENOBUFS;
304− rw->addr = (unsigned long) buf;
305− iov[0].iov_base = buf;
306− rw->len = iov[0].iov_len = (compat_size_t) len;
307− return 0;
308−}
309−#endif
310−
311−static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
312− unsigned int issue_flags)
313−{
314− struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
315− struct iovec __user *uiov = u64_to_user_ptr(rw->addr);
316− void __user *buf;
317− ssize_t len;
318−
319− if (copy_from_user(iov, uiov, sizeof(*uiov)))
320− return -EFAULT;
321−
322− len = iov[0].iov_len;
323− if (len < 0)
324− return -EINVAL;
325− buf = io_buffer_select(req, &len, issue_flags);
326− if (!buf)
327− return -ENOBUFS;
328− rw->addr = (unsigned long) buf;
329− iov[0].iov_base = buf;
330− rw->len = iov[0].iov_len = len;
331− return 0;
332−}
333−
334−static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
335− unsigned int issue_flags)
336−{
337− struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
338−
339− if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) {
340− iov[0].iov_base = u64_to_user_ptr(rw->addr);
341− iov[0].iov_len = rw->len;
342− return 0;
343− }
344− if (rw->len != 1)
345− return -EINVAL;
346−
347−#ifdef CONFIG_COMPAT
348− if (req->ctx->compat)
349− return io_compat_import(req, iov, issue_flags);
350−#endif
351−
352− return __io_iov_buffer_select(req, iov, issue_flags);
353−}
354−
355338 static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req,
356339 struct io_rw_state *s,
357340 unsigned int issue_flags)
@@ -374,7 +357,8 @@ static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, · 16 unmodified lines
374357 buf = u64_to_user_ptr(rw->addr);
375358 sqe_len = rw->len;
376359
377− if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
360+ if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE ||
361+ (req->flags & REQ_F_BUFFER_SELECT)) {
378362 if (io_do_buffer_select(req)) {
379363 buf = io_buffer_select(req, &sqe_len, issue_flags);
380364 if (!buf)
@@ -390,14 +374,6 @@ static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, · 9 unmodified lines
390374 }
391375
392376 iovec = s->fast_iov;
393− if (req->flags & REQ_F_BUFFER_SELECT) {
394− ret = io_iov_buffer_select(req, iovec, issue_flags);
395− if (ret)
396− return ERR_PTR(ret);
397− iov_iter_init(iter, ddir, iovec, 1, iovec->iov_len);
398− return NULL;
399− }
400−
401377 ret = __import_iovec(ddir, buf, sqe_len, UIO_FASTIOV, &iovec, iter,
402378 req->ctx->compat);
403379 if (unlikely(ret < 0))
@@ -794,10 +770,12 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) · 390 unmodified lines
794770 iov_iter_restore(&s->iter, &s->iter_state);
795771
796772 ret2 = io_setup_async_rw(req, iovec, s, true);
797− if (ret2)
798− return ret2;
799−
800773 iovec = NULL;
774+ if (ret2) {
775+ ret = ret > 0 ? ret : ret2;
776+ goto done;
777+ }
778+
801779 io = req->async_data;
802780 s = &io->s;
803781 /*
@@ -823,6 +801,7 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) · 19 unmodified lines
823801 return -EAGAIN;
824802 }
825803
804+ req->cqe.res = iov_iter_count(&s->iter);
826805 /*
827806 * Now retry read with the IOCB_WAITQ parts set in the iocb. If
828807 * we get -EIOCBQUEUED, then we'll get a notification when the
@@ -984,6 +963,14 @@ static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx) · 155 unmodified lines
984963 io_cqring_wake(ctx);
985964 }
986965
966+void io_rw_fail(struct io_kiocb *req)
967+{
968+ int res;
969+
970+ res = io_fixup_rw_res(req, req->cqe.res);
971+ io_req_set_res(req, res, req->cqe.flags);
972+}
973+
987974 int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)
988975 {
989976 struct io_wq_work_node *pos, *start, *prev;
@@ -1000,7 +987,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) · 10 unmodified lines
1000987
1001988 wq_list_for_each(pos, start, &ctx->iopoll_list) {
1002989 struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list);
1003− struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
990+ struct file *file = req->file;
1004991 int ret;
1005992
1006993 /*
@@ -1011,7 +998,17 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) · 4 unmodified lines
1011998 if (READ_ONCE(req->iopoll_completed))
1012999 break;
10131000
1014− ret = rw->kiocb.ki_filp->f_op->iopoll(&rw->kiocb, &iob, poll_flags);
1001+ if (req->opcode == IORING_OP_URING_CMD) {
1002+ struct io_uring_cmd *ioucmd;
1003+
1004+ ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
1005+ ret = file->f_op->uring_cmd_iopoll(ioucmd, &iob,
1006+ poll_flags);
1007+ } else {
1008+ struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
1009+
1010+ ret = file->f_op->iopoll(&rw->kiocb, &iob, poll_flags);
1011+ }
10151012 if (unlikely(ret < 0))
10161013 return ret;
10171014 else if (ret)
io_uring/rw.h+1−0
@@ -21,3 +21,4 @@ int io_readv_prep_async(struct io_kiocb *req); · 20 unmodified lines
2121 int io_write(struct io_kiocb *req, unsigned int issue_flags);
2222 int io_writev_prep_async(struct io_kiocb *req);
2323 void io_readv_writev_cleanup(struct io_kiocb *req);
24+void io_rw_fail(struct io_kiocb *req);
io_uring/timeout.c+3−10
@@ -149,11 +149,10 @@ static inline void io_remove_next_linked(struct io_kiocb *req) · 148 unmodified lines
149149 nxt->link = NULL;
150150 }
151151
152−bool io_disarm_next(struct io_kiocb *req)
152+void io_disarm_next(struct io_kiocb *req)
153153 __must_hold(&req->ctx->completion_lock)
154154 {
155155 struct io_kiocb *link = NULL;
156− bool posted = false;
157156
158157 if (req->flags & REQ_F_ARM_LTIMEOUT) {
159158 link = req->link;
@@ -161,7 +160,6 @@ bool io_disarm_next(struct io_kiocb *req) · 1 unmodified line
161160 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
162161 io_remove_next_linked(req);
163162 io_req_tw_post_queue(link, -ECANCELED, 0);
164− posted = true;
165163 }
166164 } else if (req->flags & REQ_F_LINK_TIMEOUT) {
167165 struct io_ring_ctx *ctx = req->ctx;
@@ -169,17 +167,12 @@ bool io_disarm_next(struct io_kiocb *req) · 1 unmodified line
169167 spin_lock_irq(&ctx->timeout_lock);
170168 link = io_disarm_linked_timeout(req);
171169 spin_unlock_irq(&ctx->timeout_lock);
172− if (link) {
173− posted = true;
170+ if (link)
174171 io_req_tw_post_queue(link, -ECANCELED, 0);
175− }
176172 }
177173 if (unlikely((req->flags & REQ_F_FAIL) &&
178− !(req->flags & REQ_F_HARDLINK))) {
179− posted |= (req->link != NULL);
174+ !(req->flags & REQ_F_HARDLINK)))
180175 io_fail_links(req);
181− }
182− return posted;
183176 }
184177
185178 struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req,
io_uring/timeout.h+1−1
@@ -27,7 +27,7 @@ int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd); · 26 unmodified lines
2727 __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
2828 bool cancel_all);
2929 void io_queue_linked_timeout(struct io_kiocb *req);
30−bool io_disarm_next(struct io_kiocb *req);
30+void io_disarm_next(struct io_kiocb *req);
3131
3232 int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
3333 int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
io_uring/uring_cmd.c+9−2
@@ -50,7 +50,11 @@ void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2) · 49 unmodified lines
5050 io_req_set_res(req, ret, 0);
5151 if (req->ctx->flags & IORING_SETUP_CQE32)
5252 io_req_set_cqe32_extra(req, res2, 0);
53− __io_req_complete(req, 0);
53+ if (req->ctx->flags & IORING_SETUP_IOPOLL)
54+ /* order with io_iopoll_req_issued() checking ->iopoll_complete */
55+ smp_store_release(&req->iopoll_completed, 1);
56+ else
57+ __io_req_complete(req, 0);
5458 }
5559 EXPORT_SYMBOL_GPL(io_uring_cmd_done);
5660
@@ -97,8 +101,11 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) · 40 unmodified lines
97101 issue_flags |= IO_URING_F_SQE128;
98102 if (ctx->flags & IORING_SETUP_CQE32)
99103 issue_flags |= IO_URING_F_CQE32;
100− if (ctx->flags & IORING_SETUP_IOPOLL)
104+ if (ctx->flags & IORING_SETUP_IOPOLL) {
101105 issue_flags |= IO_URING_F_IOPOLL;
106+ req->iopoll_completed = 0;
107+ WRITE_ONCE(ioucmd->cookie, NULL);
108+ }
102109
103110 if (req_has_async_data(req))
104111 ioucmd->cmd = req->async_data;
tools/testing/selftests/net/io_uring_zerocopy_tx.c+13−9
@@ -400,7 +400,6 @@ static void do_tx(int domain, int type, int protocol) · 399 unmodified lines
400400 cfg_payload_len, msg_flags);
401401 sqe->user_data = NONZC_TAG;
402402 } else {
403− compl_cqes++;
404403 io_uring_prep_sendzc(sqe, fd, payload,
405404 cfg_payload_len,
406405 msg_flags, zc_flags);
@@ -430,18 +429,23 @@ static void do_tx(int domain, int type, int protocol) · 23 unmodified lines
430429 if (cqe->flags & IORING_CQE_F_NOTIF) {
431430 if (cqe->flags & IORING_CQE_F_MORE)
432431 error(1, -EINVAL, "invalid notif flags");
432+ if (compl_cqes <= 0)
433+ error(1, -EINVAL, "notification mismatch");
433434 compl_cqes--;
434435 i--;
435− } else if (cqe->res <= 0) {
436− if (cqe->flags & IORING_CQE_F_MORE)
437− error(1, cqe->res, "more with a failed send");
438− error(1, cqe->res, "send failed");
439− } else {
440− if (cqe->user_data == ZC_TAG &&
441− !(cqe->flags & IORING_CQE_F_MORE))
442− error(1, cqe->res, "missing more flag");
436+ io_uring_cqe_seen(&ring);
437+ continue;
438+ }
439+ if (cqe->flags & IORING_CQE_F_MORE) {
440+ if (cqe->user_data != ZC_TAG)
441+ error(1, cqe->res, "unexpected F_MORE");
442+ compl_cqes++;
443+ }
444+ if (cqe->res >= 0) {
443445 packets++;
444446 bytes += cqe->res;
447+ } else if (cqe->res != -EAGAIN) {
448+ error(1, cqe->res, "send failed");
445449 }
446450 io_uring_cqe_seen(&ring);
447451 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t