History

Merge tag 'for-6.1/io_uring-2022-10-03' of git://git.kernel.dk/linux

Pull io_uring updates from Jens Axboe: - Add supported for more directly managed task_work running. This is beneficial for real world applications that end up issuing lots of system calls as part of handling work. Normal task_work will always execute as we transition in and out of the kernel, even for "unrelated" system calls. It's more efficient to defer the handling of io_uring's deferred work until the application wants it to be run, generally in batches. As part of ongoing work to write an io_uring network backend for Thrift, this has been shown to greatly improve performance. (Dylan) - Add IOPOLL support for passthrough (Kanchan) - Improvements and fixes to the send zero-copy support (Pavel) - Partial IO handling fixes (Pavel) - CQE ordering fixes around CQ ring overflow (Pavel) - Support sendto() for non-zc as well (Pavel) - Support sendmsg for zerocopy (Pavel) - Networking iov_iter fix (Stefan) - Misc fixes and cleanups (Pavel, me) * tag 'for-6.1/io_uring-2022-10-03' of git://git.kernel.dk/linux: (56 commits) io_uring/net: fix notif cqe reordering io_uring/net: don't update msg_name if not provided io_uring: don't gate task_work run on TIF_NOTIFY_SIGNAL io_uring/rw: defer fsnotify calls to task context io_uring/net: fix fast_iov assignment in io_setup_async_msg() io_uring/net: fix non-zc send with address io_uring/net: don't skip notifs for failed requests io_uring/rw: don't lose short results on io_setup_async_rw() io_uring/rw: fix unexpected link breakage io_uring/net: fix cleanup double free free_iov init io_uring: fix CQE reordering io_uring/net: fix UAF in io_sendrecv_fail() selftest/net: adjust io_uring sendzc notif handling io_uring: ensure local task_work marks task as running io_uring/net: zerocopy sendmsg io_uring/net: combine fail handlers io_uring/net: rename io_sendzc() io_uring/net: support non-zerocopy sendto io_uring/net: refactor io_setup_async_addr io_uring/net: don't lose partial send_zc on fail ...

Linus Torvalds committed 4 years ago

Commit 0a78a372 parents 188943a 108893d30 files · +859 −326Browse files

Unified · Split

30 files
  1. block/blk-mq.c +2 −1
  2. drivers/nvme/host/core.c +1 −0
  3. drivers/nvme/host/ioctl.c +72 −5
  4. drivers/nvme/host/multipath.c +1 −0
  5. drivers/nvme/host/nvme.h +4 −0
  6. fs/eventfd.c +7 −3
  7. include/linux/blk-mq.h +1 −0
  8. include/linux/eventfd.h +1 −1
  9. include/linux/fs.h +2 −0
  10. include/linux/io_uring.h +6 −2
  11. include/linux/io_uring_types.h +4 −0
  12. include/linux/sched.h +1 −1
  13. include/trace/events/io_uring.h +29 −0
  14. include/uapi/linux/io_uring.h +8 −0
  15. io_uring/cancel.c +1 −1
  16. io_uring/fdinfo.c +32 −16
  17. io_uring/io_uring.c +244 −60
  18. io_uring/io_uring.h +51 −11
  19. io_uring/kbuf.h +0 −12
  20. io_uring/net.c +222 −86
  21. io_uring/net.h +8 −4
  22. io_uring/opdef.c +40 −4
  23. io_uring/opdef.h +1 −0
  24. io_uring/rsrc.c +1 −1
  25. io_uring/rw.c +93 −96
  26. io_uring/rw.h +1 −0
  27. io_uring/timeout.c +3 −10
  28. io_uring/timeout.h +1 −1
  29. io_uring/uring_cmd.c +9 −2
  30. tools/testing/selftests/net/io_uring_zerocopy_tx.c +13 −9
block/blk-mq.c+2−1
@@ -1233,7 +1233,7 @@ static void blk_end_sync_rq(struct request *rq, blk_status_t ret) · 1232 unmodified lines
1233 complete(&wait->done);1233 complete(&wait->done);
1234}1234}
1235 1235
1236static bool blk_rq_is_poll(struct request *rq)1236bool blk_rq_is_poll(struct request *rq)
1237{1237{
1238 if (!rq->mq_hctx)1238 if (!rq->mq_hctx)
1239 return false;1239 return false;
@@ -1243,6 +1243,7 @@ static bool blk_rq_is_poll(struct request *rq) · 3 unmodified lines
1243 return false;1243 return false;
1244 return true;1244 return true;
1245}1245}
1246EXPORT_SYMBOL_GPL(blk_rq_is_poll);
1246 1247
1247static void blk_rq_poll_completion(struct request *rq, struct completion *wait)1248static void blk_rq_poll_completion(struct request *rq, struct completion *wait)
1248{1249{
drivers/nvme/host/core.c+1−0
@@ -3976,6 +3976,7 @@ static const struct file_operations nvme_ns_chr_fops = { · 3975 unmodified lines
3976 .unlocked_ioctl = nvme_ns_chr_ioctl,3976 .unlocked_ioctl = nvme_ns_chr_ioctl,
3977 .compat_ioctl = compat_ptr_ioctl,3977 .compat_ioctl = compat_ptr_ioctl,
3978 .uring_cmd = nvme_ns_chr_uring_cmd,3978 .uring_cmd = nvme_ns_chr_uring_cmd,
3979 .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll,
3979};3980};
3980 3981
3981static int nvme_add_ns_cdev(struct nvme_ns *ns)3982static int nvme_add_ns_cdev(struct nvme_ns *ns)
drivers/nvme/host/ioctl.c+72−5
@@ -391,11 +391,19 @@ static void nvme_uring_cmd_end_io(struct request *req, blk_status_t err) · 390 unmodified lines
391 struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);391 struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
392 /* extract bio before reusing the same field for request */392 /* extract bio before reusing the same field for request */
393 struct bio *bio = pdu->bio;393 struct bio *bio = pdu->bio;
394 void *cookie = READ_ONCE(ioucmd->cookie);
394 395
395 pdu->req = req;396 pdu->req = req;
396 req->bio = bio;397 req->bio = bio;
397 /* this takes care of moving rest of completion-work to task context */398
398 io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb);399 /*
400 * For iopoll, complete it directly.
401 * Otherwise, move the completion to task work.
402 */
403 if (cookie != NULL && blk_rq_is_poll(req))
404 nvme_uring_task_cb(ioucmd);
405 else
406 io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb);
399}407}
400 408
401static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,409static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
@@ -445,7 +453,10 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, · 43 unmodified lines
445 rq_flags = REQ_NOWAIT;453 rq_flags = REQ_NOWAIT;
446 blk_flags = BLK_MQ_REQ_NOWAIT;454 blk_flags = BLK_MQ_REQ_NOWAIT;
447 }455 }
456 if (issue_flags & IO_URING_F_IOPOLL)
457 rq_flags |= REQ_POLLED;
448 458
459retry:
449 req = nvme_alloc_user_request(q, &c, nvme_to_user_ptr(d.addr),460 req = nvme_alloc_user_request(q, &c, nvme_to_user_ptr(d.addr),
450 d.data_len, nvme_to_user_ptr(d.metadata),461 d.data_len, nvme_to_user_ptr(d.metadata),
451 d.metadata_len, 0, &meta, d.timeout_ms ?462 d.metadata_len, 0, &meta, d.timeout_ms ?
@@ -456,6 +467,17 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, · 4 unmodified lines
456 req->end_io = nvme_uring_cmd_end_io;467 req->end_io = nvme_uring_cmd_end_io;
457 req->end_io_data = ioucmd;468 req->end_io_data = ioucmd;
458 469
470 if (issue_flags & IO_URING_F_IOPOLL && rq_flags & REQ_POLLED) {
471 if (unlikely(!req->bio)) {
472 /* we can't poll this, so alloc regular req instead */
473 blk_mq_free_request(req);
474 rq_flags &= ~REQ_POLLED;
475 goto retry;
476 } else {
477 WRITE_ONCE(ioucmd->cookie, req->bio);
478 req->bio->bi_opf |= REQ_POLLED;
479 }
480 }
459 /* to free bio on completion, as req->bio will be null at that time */481 /* to free bio on completion, as req->bio will be null at that time */
460 pdu->bio = req->bio;482 pdu->bio = req->bio;
461 pdu->meta = meta;483 pdu->meta = meta;
@@ -559,9 +581,6 @@ long nvme_ns_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg) · 97 unmodified lines
559 581
560static int nvme_uring_cmd_checks(unsigned int issue_flags)582static int nvme_uring_cmd_checks(unsigned int issue_flags)
561{583{
562 /* IOPOLL not supported yet */
563 if (issue_flags & IO_URING_F_IOPOLL)
564 return -EOPNOTSUPP;
565 584
566 /* NVMe passthrough requires big SQE/CQE support */585 /* NVMe passthrough requires big SQE/CQE support */
567 if ((issue_flags & (IO_URING_F_SQE128|IO_URING_F_CQE32)) !=586 if ((issue_flags & (IO_URING_F_SQE128|IO_URING_F_CQE32)) !=
@@ -604,6 +623,25 @@ int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags) · 36 unmodified lines
604 return nvme_ns_uring_cmd(ns, ioucmd, issue_flags);623 return nvme_ns_uring_cmd(ns, ioucmd, issue_flags);
605}624}
606 625
626int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
627 struct io_comp_batch *iob,
628 unsigned int poll_flags)
629{
630 struct bio *bio;
631 int ret = 0;
632 struct nvme_ns *ns;
633 struct request_queue *q;
634
635 rcu_read_lock();
636 bio = READ_ONCE(ioucmd->cookie);
637 ns = container_of(file_inode(ioucmd->file)->i_cdev,
638 struct nvme_ns, cdev);
639 q = ns->queue;
640 if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio && bio->bi_bdev)
641 ret = bio_poll(bio, iob, poll_flags);
642 rcu_read_unlock();
643 return ret;
644}
607#ifdef CONFIG_NVME_MULTIPATH645#ifdef CONFIG_NVME_MULTIPATH
608static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,646static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
609 void __user *argp, struct nvme_ns_head *head, int srcu_idx)647 void __user *argp, struct nvme_ns_head *head, int srcu_idx)
@@ -685,6 +723,31 @@ int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd, · 75 unmodified lines
685 srcu_read_unlock(&head->srcu, srcu_idx);723 srcu_read_unlock(&head->srcu, srcu_idx);
686 return ret;724 return ret;
687}725}
726
727int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
728 struct io_comp_batch *iob,
729 unsigned int poll_flags)
730{
731 struct cdev *cdev = file_inode(ioucmd->file)->i_cdev;
732 struct nvme_ns_head *head = container_of(cdev, struct nvme_ns_head, cdev);
733 int srcu_idx = srcu_read_lock(&head->srcu);
734 struct nvme_ns *ns = nvme_find_path(head);
735 struct bio *bio;
736 int ret = 0;
737 struct request_queue *q;
738
739 if (ns) {
740 rcu_read_lock();
741 bio = READ_ONCE(ioucmd->cookie);
742 q = ns->queue;
743 if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio
744 && bio->bi_bdev)
745 ret = bio_poll(bio, iob, poll_flags);
746 rcu_read_unlock();
747 }
748 srcu_read_unlock(&head->srcu, srcu_idx);
749 return ret;
750}
688#endif /* CONFIG_NVME_MULTIPATH */751#endif /* CONFIG_NVME_MULTIPATH */
689 752
690int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags)753int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags)
@@ -692,6 +755,10 @@ int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags) · 1 unmodified line
692 struct nvme_ctrl *ctrl = ioucmd->file->private_data;755 struct nvme_ctrl *ctrl = ioucmd->file->private_data;
693 int ret;756 int ret;
694 757
758 /* IOPOLL not supported yet */
759 if (issue_flags & IO_URING_F_IOPOLL)
760 return -EOPNOTSUPP;
761
695 ret = nvme_uring_cmd_checks(issue_flags);762 ret = nvme_uring_cmd_checks(issue_flags);
696 if (ret)763 if (ret)
697 return ret;764 return ret;
drivers/nvme/host/multipath.c+1−0
@@ -439,6 +439,7 @@ static const struct file_operations nvme_ns_head_chr_fops = { · 438 unmodified lines
439 .unlocked_ioctl = nvme_ns_head_chr_ioctl,439 .unlocked_ioctl = nvme_ns_head_chr_ioctl,
440 .compat_ioctl = compat_ptr_ioctl,440 .compat_ioctl = compat_ptr_ioctl,
441 .uring_cmd = nvme_ns_head_chr_uring_cmd,441 .uring_cmd = nvme_ns_head_chr_uring_cmd,
442 .uring_cmd_iopoll = nvme_ns_head_chr_uring_cmd_iopoll,
442};443};
443 444
444static int nvme_add_ns_head_cdev(struct nvme_ns_head *head)445static int nvme_add_ns_head_cdev(struct nvme_ns_head *head)
drivers/nvme/host/nvme.h+4−0
@@ -821,6 +821,10 @@ long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd, · 820 unmodified lines
821 unsigned long arg);821 unsigned long arg);
822long nvme_dev_ioctl(struct file *file, unsigned int cmd,822long nvme_dev_ioctl(struct file *file, unsigned int cmd,
823 unsigned long arg);823 unsigned long arg);
824int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
825 struct io_comp_batch *iob, unsigned int poll_flags);
826int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
827 struct io_comp_batch *iob, unsigned int poll_flags);
824int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd,828int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd,
825 unsigned int issue_flags);829 unsigned int issue_flags);
826int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd,830int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd,
fs/eventfd.c+7−3
@@ -69,17 +69,17 @@ __u64 eventfd_signal(struct eventfd_ctx *ctx, __u64 n) · 68 unmodified lines
69 * it returns false, the eventfd_signal() call should be deferred to a69 * it returns false, the eventfd_signal() call should be deferred to a
70 * safe context.70 * safe context.
71 */71 */
72 if (WARN_ON_ONCE(current->in_eventfd_signal))72 if (WARN_ON_ONCE(current->in_eventfd))
73 return 0;73 return 0;
74 74
75 spin_lock_irqsave(&ctx->wqh.lock, flags);75 spin_lock_irqsave(&ctx->wqh.lock, flags);
76 current->in_eventfd_signal = 1;76 current->in_eventfd = 1;
77 if (ULLONG_MAX - ctx->count < n)77 if (ULLONG_MAX - ctx->count < n)
78 n = ULLONG_MAX - ctx->count;78 n = ULLONG_MAX - ctx->count;
79 ctx->count += n;79 ctx->count += n;
80 if (waitqueue_active(&ctx->wqh))80 if (waitqueue_active(&ctx->wqh))
81 wake_up_locked_poll(&ctx->wqh, EPOLLIN);81 wake_up_locked_poll(&ctx->wqh, EPOLLIN);
82 current->in_eventfd_signal = 0;82 current->in_eventfd = 0;
83 spin_unlock_irqrestore(&ctx->wqh.lock, flags);83 spin_unlock_irqrestore(&ctx->wqh.lock, flags);
84 84
85 return n;85 return n;
@@ -253,8 +253,10 @@ static ssize_t eventfd_read(struct kiocb *iocb, struct iov_iter *to) · 167 unmodified lines
253 __set_current_state(TASK_RUNNING);253 __set_current_state(TASK_RUNNING);
254 }254 }
255 eventfd_ctx_do_read(ctx, &ucnt);255 eventfd_ctx_do_read(ctx, &ucnt);
256 current->in_eventfd = 1;
256 if (waitqueue_active(&ctx->wqh))257 if (waitqueue_active(&ctx->wqh))
257 wake_up_locked_poll(&ctx->wqh, EPOLLOUT);258 wake_up_locked_poll(&ctx->wqh, EPOLLOUT);
259 current->in_eventfd = 0;
258 spin_unlock_irq(&ctx->wqh.lock);260 spin_unlock_irq(&ctx->wqh.lock);
259 if (unlikely(copy_to_iter(&ucnt, sizeof(ucnt), to) != sizeof(ucnt)))261 if (unlikely(copy_to_iter(&ucnt, sizeof(ucnt), to) != sizeof(ucnt)))
260 return -EFAULT;262 return -EFAULT;
@@ -301,8 +303,10 @@ static ssize_t eventfd_write(struct file *file, const char __user *buf, size_t c · 40 unmodified lines
301 }303 }
302 if (likely(res > 0)) {304 if (likely(res > 0)) {
303 ctx->count += ucnt;305 ctx->count += ucnt;
306 current->in_eventfd = 1;
304 if (waitqueue_active(&ctx->wqh))307 if (waitqueue_active(&ctx->wqh))
305 wake_up_locked_poll(&ctx->wqh, EPOLLIN);308 wake_up_locked_poll(&ctx->wqh, EPOLLIN);
309 current->in_eventfd = 0;
306 }310 }
307 spin_unlock_irq(&ctx->wqh.lock);311 spin_unlock_irq(&ctx->wqh.lock);
308 312
include/linux/blk-mq.h+1−0
@@ -980,6 +980,7 @@ int blk_rq_map_kern(struct request_queue *, struct request *, void *, · 979 unmodified lines
980int blk_rq_append_bio(struct request *rq, struct bio *bio);980int blk_rq_append_bio(struct request *rq, struct bio *bio);
981void blk_execute_rq_nowait(struct request *rq, bool at_head);981void blk_execute_rq_nowait(struct request *rq, bool at_head);
982blk_status_t blk_execute_rq(struct request *rq, bool at_head);982blk_status_t blk_execute_rq(struct request *rq, bool at_head);
983bool blk_rq_is_poll(struct request *rq);
983 984
984struct req_iterator {985struct req_iterator {
985 struct bvec_iter iter;986 struct bvec_iter iter;
include/linux/eventfd.h+1−1
@@ -46,7 +46,7 @@ void eventfd_ctx_do_read(struct eventfd_ctx *ctx, __u64 *cnt); · 45 unmodified lines
46 46
47static inline bool eventfd_signal_allowed(void)47static inline bool eventfd_signal_allowed(void)
48{48{
49 return !current->in_eventfd_signal;49 return !current->in_eventfd;
50}50}
51 51
52#else /* CONFIG_EVENTFD */52#else /* CONFIG_EVENTFD */
include/linux/fs.h+2−0
@@ -2133,6 +2133,8 @@ struct file_operations { · 2132 unmodified lines
2133 loff_t len, unsigned int remap_flags);2133 loff_t len, unsigned int remap_flags);
2134 int (*fadvise)(struct file *, loff_t, loff_t, int);2134 int (*fadvise)(struct file *, loff_t, loff_t, int);
2135 int (*uring_cmd)(struct io_uring_cmd *ioucmd, unsigned int issue_flags);2135 int (*uring_cmd)(struct io_uring_cmd *ioucmd, unsigned int issue_flags);
2136 int (*uring_cmd_iopoll)(struct io_uring_cmd *, struct io_comp_batch *,
2137 unsigned int poll_flags);
2136} __randomize_layout;2138} __randomize_layout;
2137 2139
2138struct inode_operations {2140struct inode_operations {
include/linux/io_uring.h+6−2
@@ -20,8 +20,12 @@ enum io_uring_cmd_flags { · 19 unmodified lines
20struct io_uring_cmd {20struct io_uring_cmd {
21 struct file *file;21 struct file *file;
22 const void *cmd;22 const void *cmd;
23 /* callback to defer completions to task context */23 union {
24 void (*task_work_cb)(struct io_uring_cmd *cmd);24 /* callback to defer completions to task context */
25 void (*task_work_cb)(struct io_uring_cmd *cmd);
26 /* used for polled completion */
27 void *cookie;
28 };
25 u32 cmd_op;29 u32 cmd_op;
26 u32 pad;30 u32 pad;
27 u8 pdu[32]; /* available inline for free use */31 u8 pdu[32]; /* available inline for free use */
include/linux/io_uring_types.h+4−0
@@ -184,6 +184,8 @@ struct io_ev_fd { · 183 unmodified lines
184 struct eventfd_ctx *cq_ev_fd;184 struct eventfd_ctx *cq_ev_fd;
185 unsigned int eventfd_async: 1;185 unsigned int eventfd_async: 1;
186 struct rcu_head rcu;186 struct rcu_head rcu;
187 atomic_t refs;
188 atomic_t ops;
187};189};
188 190
189struct io_alloc_cache {191struct io_alloc_cache {
@@ -301,6 +303,8 @@ struct io_ring_ctx { · 111 unmodified lines
301 struct io_hash_table cancel_table;303 struct io_hash_table cancel_table;
302 bool poll_multi_queue;304 bool poll_multi_queue;
303 305
306 struct llist_head work_llist;
307
304 struct list_head io_buffers_comp;308 struct list_head io_buffers_comp;
305 } ____cacheline_aligned_in_smp;309 } ____cacheline_aligned_in_smp;
306 310
include/linux/sched.h+1−1
@@ -936,7 +936,7 @@ struct task_struct { · 935 unmodified lines
936#endif936#endif
937#ifdef CONFIG_EVENTFD937#ifdef CONFIG_EVENTFD
938 /* Recursion prevention for eventfd_signal() */938 /* Recursion prevention for eventfd_signal() */
939 unsigned in_eventfd_signal:1;939 unsigned in_eventfd:1;
940#endif940#endif
941#ifdef CONFIG_IOMMU_SVA941#ifdef CONFIG_IOMMU_SVA
942 unsigned pasid_activated:1;942 unsigned pasid_activated:1;
include/trace/events/io_uring.h+29−0
@@ -655,6 +655,35 @@ TRACE_EVENT(io_uring_short_write, · 654 unmodified lines
655 __entry->wanted, __entry->got)655 __entry->wanted, __entry->got)
656);656);
657 657
658/*
659 * io_uring_local_work_run - ran ring local task work
660 *
661 * @tctx: pointer to a io_uring_ctx
662 * @count: how many functions it ran
663 * @loops: how many loops it ran
664 *
665 */
666TRACE_EVENT(io_uring_local_work_run,
667
668 TP_PROTO(void *ctx, int count, unsigned int loops),
669
670 TP_ARGS(ctx, count, loops),
671
672 TP_STRUCT__entry (
673 __field(void *, ctx )
674 __field(int, count )
675 __field(unsigned int, loops )
676 ),
677
678 TP_fast_assign(
679 __entry->ctx = ctx;
680 __entry->count = count;
681 __entry->loops = loops;
682 ),
683
684 TP_printk("ring %p, count %d, loops %u", __entry->ctx, __entry->count, __entry->loops)
685);
686
658#endif /* _TRACE_IO_URING_H */687#endif /* _TRACE_IO_URING_H */
659 688
660/* This part must be outside protection */689/* This part must be outside protection */
include/uapi/linux/io_uring.h+8−0
@@ -157,6 +157,13 @@ enum { · 156 unmodified lines
157 */157 */
158#define IORING_SETUP_SINGLE_ISSUER (1U << 12)158#define IORING_SETUP_SINGLE_ISSUER (1U << 12)
159 159
160/*
161 * Defer running task work to get events.
162 * Rather than running bits of task work whenever the task transitions
163 * try to do it just before it is needed.
164 */
165#define IORING_SETUP_DEFER_TASKRUN (1U << 13)
166
160enum io_uring_op {167enum io_uring_op {
161 IORING_OP_NOP,168 IORING_OP_NOP,
162 IORING_OP_READV,169 IORING_OP_READV,
@@ -206,6 +213,7 @@ enum io_uring_op { · 43 unmodified lines
206 IORING_OP_SOCKET,213 IORING_OP_SOCKET,
207 IORING_OP_URING_CMD,214 IORING_OP_URING_CMD,
208 IORING_OP_SEND_ZC,215 IORING_OP_SEND_ZC,
216 IORING_OP_SENDMSG_ZC,
209 217
210 /* this goes last, obviously */218 /* this goes last, obviously */
211 IORING_OP_LAST,219 IORING_OP_LAST,
io_uring/cancel.c+1−1
@@ -292,7 +292,7 @@ int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg) · 291 unmodified lines
292 break;292 break;
293 293
294 mutex_unlock(&ctx->uring_lock);294 mutex_unlock(&ctx->uring_lock);
295 ret = io_run_task_work_sig();295 ret = io_run_task_work_sig(ctx);
296 if (ret < 0) {296 if (ret < 0) {
297 mutex_lock(&ctx->uring_lock);297 mutex_lock(&ctx->uring_lock);
298 break;298 break;
io_uring/fdinfo.c+32−16
@@ -60,13 +60,15 @@ static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, · 59 unmodified lines
60 unsigned int cq_head = READ_ONCE(r->cq.head);60 unsigned int cq_head = READ_ONCE(r->cq.head);
61 unsigned int cq_tail = READ_ONCE(r->cq.tail);61 unsigned int cq_tail = READ_ONCE(r->cq.tail);
62 unsigned int cq_shift = 0;62 unsigned int cq_shift = 0;
63 unsigned int sq_shift = 0;
63 unsigned int sq_entries, cq_entries;64 unsigned int sq_entries, cq_entries;
64 bool has_lock;65 bool has_lock;
65 bool is_cqe32 = (ctx->flags & IORING_SETUP_CQE32);
66 unsigned int i;66 unsigned int i;
67 67
68 if (is_cqe32)68 if (ctx->flags & IORING_SETUP_CQE32)
69 cq_shift = 1;69 cq_shift = 1;
70 if (ctx->flags & IORING_SETUP_SQE128)
71 sq_shift = 1;
70 72
71 /*73 /*
72 * we may get imprecise sqe and cqe info if uring is actively running74 * we may get imprecise sqe and cqe info if uring is actively running
@@ -82,19 +84,36 @@ static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, · 9 unmodified lines
82 seq_printf(m, "CqHead:\t%u\n", cq_head);84 seq_printf(m, "CqHead:\t%u\n", cq_head);
83 seq_printf(m, "CqTail:\t%u\n", cq_tail);85 seq_printf(m, "CqTail:\t%u\n", cq_tail);
84 seq_printf(m, "CachedCqTail:\t%u\n", ctx->cached_cq_tail);86 seq_printf(m, "CachedCqTail:\t%u\n", ctx->cached_cq_tail);
85 seq_printf(m, "SQEs:\t%u\n", sq_tail - ctx->cached_sq_head);87 seq_printf(m, "SQEs:\t%u\n", sq_tail - sq_head);
86 sq_entries = min(sq_tail - sq_head, ctx->sq_entries);88 sq_entries = min(sq_tail - sq_head, ctx->sq_entries);
87 for (i = 0; i < sq_entries; i++) {89 for (i = 0; i < sq_entries; i++) {
88 unsigned int entry = i + sq_head;90 unsigned int entry = i + sq_head;
89 unsigned int sq_idx = READ_ONCE(ctx->sq_array[entry & sq_mask]);
90 struct io_uring_sqe *sqe;91 struct io_uring_sqe *sqe;
92 unsigned int sq_idx;
91 93
94 sq_idx = READ_ONCE(ctx->sq_array[entry & sq_mask]);
92 if (sq_idx > sq_mask)95 if (sq_idx > sq_mask)
93 continue;96 continue;
94 sqe = &ctx->sq_sqes[sq_idx];97 sqe = &ctx->sq_sqes[sq_idx << 1];
95 seq_printf(m, "%5u: opcode:%d, fd:%d, flags:%x, user_data:%llu\n",98 seq_printf(m, "%5u: opcode:%s, fd:%d, flags:%x, off:%llu, "
96 sq_idx, sqe->opcode, sqe->fd, sqe->flags,99 "addr:0x%llx, rw_flags:0x%x, buf_index:%d "
97 sqe->user_data);100 "user_data:%llu",
101 sq_idx, io_uring_get_opcode(sqe->opcode), sqe->fd,
102 sqe->flags, (unsigned long long) sqe->off,
103 (unsigned long long) sqe->addr, sqe->rw_flags,
104 sqe->buf_index, sqe->user_data);
105 if (sq_shift) {
106 u64 *sqeb = (void *) (sqe + 1);
107 int size = sizeof(struct io_uring_sqe) / sizeof(u64);
108 int j;
109
110 for (j = 0; j < size; j++) {
111 seq_printf(m, ", e%d:0x%llx", j,
112 (unsigned long long) *sqeb);
113 sqeb++;
114 }
115 }
116 seq_printf(m, "\n");
98 }117 }
99 seq_printf(m, "CQEs:\t%u\n", cq_tail - cq_head);118 seq_printf(m, "CQEs:\t%u\n", cq_tail - cq_head);
100 cq_entries = min(cq_tail - cq_head, ctx->cq_entries);119 cq_entries = min(cq_tail - cq_head, ctx->cq_entries);
@@ -102,16 +121,13 @@ static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, · 1 unmodified line
102 unsigned int entry = i + cq_head;121 unsigned int entry = i + cq_head;
103 struct io_uring_cqe *cqe = &r->cqes[(entry & cq_mask) << cq_shift];122 struct io_uring_cqe *cqe = &r->cqes[(entry & cq_mask) << cq_shift];
104 123
105 if (!is_cqe32) {124 seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x",
106 seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x\n",
107 entry & cq_mask, cqe->user_data, cqe->res,125 entry & cq_mask, cqe->user_data, cqe->res,
108 cqe->flags);126 cqe->flags);
109 } else {127 if (cq_shift)
110 seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x, "128 seq_printf(m, ", extra1:%llu, extra2:%llu\n",
111 "extra1:%llu, extra2:%llu\n",129 cqe->big_cqe[0], cqe->big_cqe[1]);
112 entry & cq_mask, cqe->user_data, cqe->res,130 seq_printf(m, "\n");
113 cqe->flags, cqe->big_cqe[0], cqe->big_cqe[1]);
114 }
115 }131 }
116 132
117 /*133 /*
io_uring/io_uring.c+244−60
@@ -125,6 +125,11 @@ enum { · 124 unmodified lines
125 IO_CHECK_CQ_DROPPED_BIT,125 IO_CHECK_CQ_DROPPED_BIT,
126};126};
127 127
128enum {
129 IO_EVENTFD_OP_SIGNAL_BIT,
130 IO_EVENTFD_OP_FREE_BIT,
131};
132
128struct io_defer_entry {133struct io_defer_entry {
129 struct list_head list;134 struct list_head list;
130 struct io_kiocb *req;135 struct io_kiocb *req;
@@ -142,7 +147,7 @@ static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, · 11 unmodified lines
142static void io_dismantle_req(struct io_kiocb *req);147static void io_dismantle_req(struct io_kiocb *req);
143static void io_clean_op(struct io_kiocb *req);148static void io_clean_op(struct io_kiocb *req);
144static void io_queue_sqe(struct io_kiocb *req);149static void io_queue_sqe(struct io_kiocb *req);
145 150static void io_move_task_work_from_local(struct io_ring_ctx *ctx);
146static void __io_submit_flush_completions(struct io_ring_ctx *ctx);151static void __io_submit_flush_completions(struct io_ring_ctx *ctx);
147 152
148static struct kmem_cache *req_cachep;153static struct kmem_cache *req_cachep;
@@ -316,6 +321,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) · 167 unmodified lines
316 INIT_LIST_HEAD(&ctx->rsrc_ref_list);321 INIT_LIST_HEAD(&ctx->rsrc_ref_list);
317 INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work);322 INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work);
318 init_llist_head(&ctx->rsrc_put_llist);323 init_llist_head(&ctx->rsrc_put_llist);
324 init_llist_head(&ctx->work_llist);
319 INIT_LIST_HEAD(&ctx->tctx_list);325 INIT_LIST_HEAD(&ctx->tctx_list);
320 ctx->submit_state.free_list.next = NULL;326 ctx->submit_state.free_list.next = NULL;
321 INIT_WQ_LIST(&ctx->locked_free_list);327 INIT_WQ_LIST(&ctx->locked_free_list);
@@ -477,25 +483,28 @@ static __cold void io_queue_deferred(struct io_ring_ctx *ctx) · 155 unmodified lines
477 }483 }
478}484}
479 485
480static void io_eventfd_signal(struct io_ring_ctx *ctx)486
487static void io_eventfd_ops(struct rcu_head *rcu)
481{488{
482 struct io_ev_fd *ev_fd;489 struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);
483 bool skip;490 int ops = atomic_xchg(&ev_fd->ops, 0);
484 491
485 spin_lock(&ctx->completion_lock);492 if (ops & BIT(IO_EVENTFD_OP_SIGNAL_BIT))
486 /*493 eventfd_signal(ev_fd->cq_ev_fd, 1);
487 * Eventfd should only get triggered when at least one event has been494
488 * posted. Some applications rely on the eventfd notification count only495 /* IO_EVENTFD_OP_FREE_BIT may not be set here depending on callback
489 * changing IFF a new CQE has been added to the CQ ring. There's no496 * ordering in a race but if references are 0 we know we have to free
490 * depedency on 1:1 relationship between how many times this function is497 * it regardless.
491 * called (and hence the eventfd count) and number of CQEs posted to the
492 * CQ ring.
493 */498 */
494 skip = ctx->cached_cq_tail == ctx->evfd_last_cq_tail;499 if (atomic_dec_and_test(&ev_fd->refs)) {
495 ctx->evfd_last_cq_tail = ctx->cached_cq_tail;500 eventfd_ctx_put(ev_fd->cq_ev_fd);
496 spin_unlock(&ctx->completion_lock);501 kfree(ev_fd);
497 if (skip)502 }
498 return;503}
504
505static void io_eventfd_signal(struct io_ring_ctx *ctx)
506{
507 struct io_ev_fd *ev_fd = NULL;
499 508
500 rcu_read_lock();509 rcu_read_lock();
501 /*510 /*
@@ -513,13 +522,46 @@ static void io_eventfd_signal(struct io_ring_ctx *ctx) · 11 unmodified lines
513 goto out;522 goto out;
514 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)523 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
515 goto out;524 goto out;
525 if (ev_fd->eventfd_async && !io_wq_current_is_worker())
526 goto out;
516 527
517 if (!ev_fd->eventfd_async || io_wq_current_is_worker())528 if (likely(eventfd_signal_allowed())) {
518 eventfd_signal(ev_fd->cq_ev_fd, 1);529 eventfd_signal(ev_fd->cq_ev_fd, 1);
530 } else {
531 atomic_inc(&ev_fd->refs);
532 if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_SIGNAL_BIT), &ev_fd->ops))
533 call_rcu(&ev_fd->rcu, io_eventfd_ops);
534 else
535 atomic_dec(&ev_fd->refs);
536 }
537
519out:538out:
520 rcu_read_unlock();539 rcu_read_unlock();
521}540}
522 541
542static void io_eventfd_flush_signal(struct io_ring_ctx *ctx)
543{
544 bool skip;
545
546 spin_lock(&ctx->completion_lock);
547
548 /*
549 * Eventfd should only get triggered when at least one event has been
550 * posted. Some applications rely on the eventfd notification count
551 * only changing IFF a new CQE has been added to the CQ ring. There's
552 * no depedency on 1:1 relationship between how many times this
553 * function is called (and hence the eventfd count) and number of CQEs
554 * posted to the CQ ring.
555 */
556 skip = ctx->cached_cq_tail == ctx->evfd_last_cq_tail;
557 ctx->evfd_last_cq_tail = ctx->cached_cq_tail;
558 spin_unlock(&ctx->completion_lock);
559 if (skip)
560 return;
561
562 io_eventfd_signal(ctx);
563}
564
523void __io_commit_cqring_flush(struct io_ring_ctx *ctx)565void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
524{566{
525 if (ctx->off_timeout_used || ctx->drain_active) {567 if (ctx->off_timeout_used || ctx->drain_active) {
@@ -531,7 +573,7 @@ void __io_commit_cqring_flush(struct io_ring_ctx *ctx) · 5 unmodified lines
531 spin_unlock(&ctx->completion_lock);573 spin_unlock(&ctx->completion_lock);
532 }574 }
533 if (ctx->has_evfd)575 if (ctx->has_evfd)
534 io_eventfd_signal(ctx);576 io_eventfd_flush_signal(ctx);
535}577}
536 578
537static inline void io_cqring_ev_posted(struct io_ring_ctx *ctx)579static inline void io_cqring_ev_posted(struct io_ring_ctx *ctx)
@@ -567,7 +609,7 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) · 29 unmodified lines
567 609
568 io_cq_lock(ctx);610 io_cq_lock(ctx);
569 while (!list_empty(&ctx->cq_overflow_list)) {611 while (!list_empty(&ctx->cq_overflow_list)) {
570 struct io_uring_cqe *cqe = io_get_cqe(ctx);612 struct io_uring_cqe *cqe = io_get_cqe_overflow(ctx, true);
571 struct io_overflow_cqe *ocqe;613 struct io_overflow_cqe *ocqe;
572 614
573 if (!cqe && !force)615 if (!cqe && !force)
@@ -694,12 +736,19 @@ bool io_req_cqe_overflow(struct io_kiocb *req) · 120 unmodified lines
694 * control dependency is enough as we're using WRITE_ONCE to736 * control dependency is enough as we're using WRITE_ONCE to
695 * fill the cq entry737 * fill the cq entry
696 */738 */
697struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx)739struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow)
698{740{
699 struct io_rings *rings = ctx->rings;741 struct io_rings *rings = ctx->rings;
700 unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1);742 unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1);
701 unsigned int free, queued, len;743 unsigned int free, queued, len;
702 744
745 /*
746 * Posting into the CQ when there are pending overflowed CQEs may break
747 * ordering guarantees, which will affect links, F_MORE users and more.
748 * Force overflow the completion.
749 */
750 if (!overflow && (ctx->check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)))
751 return NULL;
703 752
704 /* userspace may cheat modifying the tail, be safe and do min */753 /* userspace may cheat modifying the tail, be safe and do min */
705 queued = min(__io_cqring_events(ctx), ctx->cq_entries);754 queued = min(__io_cqring_events(ctx), ctx->cq_entries);
@@ -823,8 +872,12 @@ inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags) · 117 unmodified lines
823 872
824void io_req_complete_failed(struct io_kiocb *req, s32 res)873void io_req_complete_failed(struct io_kiocb *req, s32 res)
825{874{
875 const struct io_op_def *def = &io_op_defs[req->opcode];
876
826 req_set_fail(req);877 req_set_fail(req);
827 io_req_set_res(req, res, io_put_kbuf(req, IO_URING_F_UNLOCKED));878 io_req_set_res(req, res, io_put_kbuf(req, IO_URING_F_UNLOCKED));
879 if (def->fail)
880 def->fail(req);
828 io_req_complete_post(req);881 io_req_complete_post(req);
829}882}
830 883
@@ -1047,17 +1100,40 @@ void tctx_task_work(struct callback_head *cb) · 216 unmodified lines
1047 trace_io_uring_task_work_run(tctx, count, loops);1100 trace_io_uring_task_work_run(tctx, count, loops);
1048}1101}
1049 1102
1050void io_req_task_work_add(struct io_kiocb *req)1103static void io_req_local_work_add(struct io_kiocb *req)
1104{
1105 struct io_ring_ctx *ctx = req->ctx;
1106
1107 if (!llist_add(&req->io_task_work.node, &ctx->work_llist))
1108 return;
1109
1110 if (unlikely(atomic_read(&req->task->io_uring->in_idle))) {
1111 io_move_task_work_from_local(ctx);
1112 return;
1113 }
1114
1115 if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
1116 atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
1117
1118 if (ctx->has_evfd)
1119 io_eventfd_signal(ctx);
1120 io_cqring_wake(ctx);
1121
1122}
1123
1124static inline void __io_req_task_work_add(struct io_kiocb *req, bool allow_local)
1051{1125{
1052 struct io_uring_task *tctx = req->task->io_uring;1126 struct io_uring_task *tctx = req->task->io_uring;
1053 struct io_ring_ctx *ctx = req->ctx;1127 struct io_ring_ctx *ctx = req->ctx;
1054 struct llist_node *node;1128 struct llist_node *node;
1055 bool running;
1056 1129
1057 running = !llist_add(&req->io_task_work.node, &tctx->task_list);1130 if (allow_local && ctx->flags & IORING_SETUP_DEFER_TASKRUN) {
1131 io_req_local_work_add(req);
1132 return;
1133 }
1058 1134
1059 /* task_work already pending, we're done */1135 /* task_work already pending, we're done */
1060 if (running)1136 if (!llist_add(&req->io_task_work.node, &tctx->task_list))
1061 return;1137 return;
1062 1138
1063 if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)1139 if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
@@ -1077,6 +1153,84 @@ void io_req_task_work_add(struct io_kiocb *req) · 13 unmodified lines
1077 }1153 }
1078}1154}
1079 1155
1156void io_req_task_work_add(struct io_kiocb *req)
1157{
1158 __io_req_task_work_add(req, true);
1159}
1160
1161static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx)
1162{
1163 struct llist_node *node;
1164
1165 node = llist_del_all(&ctx->work_llist);
1166 while (node) {
1167 struct io_kiocb *req = container_of(node, struct io_kiocb,
1168 io_task_work.node);
1169
1170 node = node->next;
1171 __io_req_task_work_add(req, false);
1172 }
1173}
1174
1175int __io_run_local_work(struct io_ring_ctx *ctx, bool locked)
1176{
1177 struct llist_node *node;
1178 struct llist_node fake;
1179 struct llist_node *current_final = NULL;
1180 int ret;
1181 unsigned int loops = 1;
1182
1183 if (unlikely(ctx->submitter_task != current))
1184 return -EEXIST;
1185
1186 node = io_llist_xchg(&ctx->work_llist, &fake);
1187 ret = 0;
1188again:
1189 while (node != current_final) {
1190 struct llist_node *next = node->next;
1191 struct io_kiocb *req = container_of(node, struct io_kiocb,
1192 io_task_work.node);
1193 prefetch(container_of(next, struct io_kiocb, io_task_work.node));
1194 req->io_task_work.func(req, &locked);
1195 ret++;
1196 node = next;
1197 }
1198
1199 if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
1200 atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
1201
1202 node = io_llist_cmpxchg(&ctx->work_llist, &fake, NULL);
1203 if (node != &fake) {
1204 loops++;
1205 current_final = &fake;
1206 node = io_llist_xchg(&ctx->work_llist, &fake);
1207 goto again;
1208 }
1209
1210 if (locked)
1211 io_submit_flush_completions(ctx);
1212 trace_io_uring_local_work_run(ctx, ret, loops);
1213 return ret;
1214
1215}
1216
1217int io_run_local_work(struct io_ring_ctx *ctx)
1218{
1219 bool locked;
1220 int ret;
1221
1222 if (llist_empty(&ctx->work_llist))
1223 return 0;
1224
1225 __set_current_state(TASK_RUNNING);
1226 locked = mutex_trylock(&ctx->uring_lock);
1227 ret = __io_run_local_work(ctx, locked);
1228 if (locked)
1229 mutex_unlock(&ctx->uring_lock);
1230
1231 return ret;
1232}
1233
1080static void io_req_tw_post(struct io_kiocb *req, bool *locked)1234static void io_req_tw_post(struct io_kiocb *req, bool *locked)
1081{1235{
1082 io_req_complete_post(req);1236 io_req_complete_post(req);
@@ -1183,7 +1337,7 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx) · 100 unmodified lines
1183 struct io_wq_work_node *node, *prev;1337 struct io_wq_work_node *node, *prev;
1184 struct io_submit_state *state = &ctx->submit_state;1338 struct io_submit_state *state = &ctx->submit_state;
1185 1339
1186 spin_lock(&ctx->completion_lock);1340 io_cq_lock(ctx);
1187 wq_list_for_each(node, prev, &state->compl_reqs) {1341 wq_list_for_each(node, prev, &state->compl_reqs) {
1188 struct io_kiocb *req = container_of(node, struct io_kiocb,1342 struct io_kiocb *req = container_of(node, struct io_kiocb,
1189 comp_list);1343 comp_list);
@@ -1254,6 +1408,9 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) · 64 unmodified lines
1254 int ret = 0;1408 int ret = 0;
1255 unsigned long check_cq;1409 unsigned long check_cq;
1256 1410
1411 if (!io_allowed_run_tw(ctx))
1412 return -EEXIST;
1413
1257 check_cq = READ_ONCE(ctx->check_cq);1414 check_cq = READ_ONCE(ctx->check_cq);
1258 if (unlikely(check_cq)) {1415 if (unlikely(check_cq)) {
1259 if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))1416 if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
@@ -1284,13 +1441,19 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) · 24 unmodified lines
1284 * forever, while the workqueue is stuck trying to acquire the1441 * forever, while the workqueue is stuck trying to acquire the
1285 * very same mutex.1442 * very same mutex.
1286 */1443 */
1287 if (wq_list_empty(&ctx->iopoll_list)) {1444 if (wq_list_empty(&ctx->iopoll_list) ||
1445 io_task_work_pending(ctx)) {
1288 u32 tail = ctx->cached_cq_tail;1446 u32 tail = ctx->cached_cq_tail;
1289 1447
1290 mutex_unlock(&ctx->uring_lock);1448 if (!llist_empty(&ctx->work_llist))
1291 io_run_task_work();1449 __io_run_local_work(ctx, true);
1292 mutex_lock(&ctx->uring_lock);
1293 1450
1451 if (task_work_pending(current) ||
1452 wq_list_empty(&ctx->iopoll_list)) {
1453 mutex_unlock(&ctx->uring_lock);
1454 io_run_task_work();
1455 mutex_lock(&ctx->uring_lock);
1456 }
1294 /* some requests don't go through iopoll_list */1457 /* some requests don't go through iopoll_list */
1295 if (tail != ctx->cached_cq_tail ||1458 if (tail != ctx->cached_cq_tail ||
1296 wq_list_empty(&ctx->iopoll_list))1459 wq_list_empty(&ctx->iopoll_list))
@@ -1732,10 +1895,6 @@ static void io_queue_async(struct io_kiocb *req, int ret) · 435 unmodified lines
1732 io_req_task_queue(req);1895 io_req_task_queue(req);
1733 break;1896 break;
1734 case IO_APOLL_ABORTED:1897 case IO_APOLL_ABORTED:
1735 /*
1736 * Queued up for async execution, worker will release
1737 * submit reference when the iocb is actually submitted.
1738 */
1739 io_kbuf_recycle(req, 0);1898 io_kbuf_recycle(req, 0);
1740 io_queue_iowq(req, NULL);1899 io_queue_iowq(req, NULL);
1741 break;1900 break;
@@ -2149,6 +2308,13 @@ struct io_wait_queue { · 407 unmodified lines
2149 unsigned nr_timeouts;2308 unsigned nr_timeouts;
2150};2309};
2151 2310
2311static inline bool io_has_work(struct io_ring_ctx *ctx)
2312{
2313 return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) ||
2314 ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&
2315 !llist_empty(&ctx->work_llist));
2316}
2317
2152static inline bool io_should_wake(struct io_wait_queue *iowq)2318static inline bool io_should_wake(struct io_wait_queue *iowq)
2153{2319{
2154 struct io_ring_ctx *ctx = iowq->ctx;2320 struct io_ring_ctx *ctx = iowq->ctx;
@@ -2167,20 +2333,20 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, · 12 unmodified lines
2167{2333{
2168 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,2334 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
2169 wq);2335 wq);
2336 struct io_ring_ctx *ctx = iowq->ctx;
2170 2337
2171 /*2338 /*
2172 * Cannot safely flush overflowed CQEs from here, ensure we wake up2339 * Cannot safely flush overflowed CQEs from here, ensure we wake up
2173 * the task, and the next invocation will do it.2340 * the task, and the next invocation will do it.
2174 */2341 */
2175 if (io_should_wake(iowq) ||2342 if (io_should_wake(iowq) || io_has_work(ctx))
2176 test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &iowq->ctx->check_cq))
2177 return autoremove_wake_function(curr, mode, wake_flags, key);2343 return autoremove_wake_function(curr, mode, wake_flags, key);
2178 return -1;2344 return -1;
2179}2345}
2180 2346
2181int io_run_task_work_sig(void)2347int io_run_task_work_sig(struct io_ring_ctx *ctx)
2182{2348{
2183 if (io_run_task_work())2349 if (io_run_task_work_ctx(ctx) > 0)
2184 return 1;2350 return 1;
2185 if (task_sigpending(current))2351 if (task_sigpending(current))
2186 return -EINTR;2352 return -EINTR;
@@ -2196,7 +2362,7 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, · 9 unmodified lines
2196 unsigned long check_cq;2362 unsigned long check_cq;
2197 2363
2198 /* make sure we run task_work before checking for signals */2364 /* make sure we run task_work before checking for signals */
2199 ret = io_run_task_work_sig();2365 ret = io_run_task_work_sig(ctx);
2200 if (ret || io_should_wake(iowq))2366 if (ret || io_should_wake(iowq))
2201 return ret;2367 return ret;
2202 2368
@@ -2226,13 +2392,19 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 23 unmodified lines
2226 ktime_t timeout = KTIME_MAX;2392 ktime_t timeout = KTIME_MAX;
2227 int ret;2393 int ret;
2228 2394
2395 if (!io_allowed_run_tw(ctx))
2396 return -EEXIST;
2397
2229 do {2398 do {
2399 /* always run at least 1 task work to process local work */
2400 ret = io_run_task_work_ctx(ctx);
2401 if (ret < 0)
2402 return ret;
2230 io_cqring_overflow_flush(ctx);2403 io_cqring_overflow_flush(ctx);
2404
2231 if (io_cqring_events(ctx) >= min_events)2405 if (io_cqring_events(ctx) >= min_events)
2232 return 0;2406 return 0;
2233 if (!io_run_task_work())2407 } while (ret > 0);
2234 break;
2235 } while (1);
2236 2408
2237 if (sig) {2409 if (sig) {
2238#ifdef CONFIG_COMPAT2410#ifdef CONFIG_COMPAT
@@ -2366,17 +2538,11 @@ static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, · 127 unmodified lines
2366 ev_fd->eventfd_async = eventfd_async;2538 ev_fd->eventfd_async = eventfd_async;
2367 ctx->has_evfd = true;2539 ctx->has_evfd = true;
2368 rcu_assign_pointer(ctx->io_ev_fd, ev_fd);2540 rcu_assign_pointer(ctx->io_ev_fd, ev_fd);
2541 atomic_set(&ev_fd->refs, 1);
2542 atomic_set(&ev_fd->ops, 0);
2369 return 0;2543 return 0;
2370}2544}
2371 2545
2372static void io_eventfd_put(struct rcu_head *rcu)
2373{
2374 struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);
2375
2376 eventfd_ctx_put(ev_fd->cq_ev_fd);
2377 kfree(ev_fd);
2378}
2379
2380static int io_eventfd_unregister(struct io_ring_ctx *ctx)2546static int io_eventfd_unregister(struct io_ring_ctx *ctx)
2381{2547{
2382 struct io_ev_fd *ev_fd;2548 struct io_ev_fd *ev_fd;
@@ -2386,7 +2552,8 @@ static int io_eventfd_unregister(struct io_ring_ctx *ctx) · 3 unmodified lines
2386 if (ev_fd) {2552 if (ev_fd) {
2387 ctx->has_evfd = false;2553 ctx->has_evfd = false;
2388 rcu_assign_pointer(ctx->io_ev_fd, NULL);2554 rcu_assign_pointer(ctx->io_ev_fd, NULL);
2389 call_rcu(&ev_fd->rcu, io_eventfd_put);2555 if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_FREE_BIT), &ev_fd->ops))
2556 call_rcu(&ev_fd->rcu, io_eventfd_ops);
2390 return 0;2557 return 0;
2391 }2558 }
2392 2559
@@ -2509,8 +2676,8 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) · 116 unmodified lines
2509 * Users may get EPOLLIN meanwhile seeing nothing in cqring, this2676 * Users may get EPOLLIN meanwhile seeing nothing in cqring, this
2510 * pushs them to do the flush.2677 * pushs them to do the flush.
2511 */2678 */
2512 if (io_cqring_events(ctx) ||2679
2513 test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq))2680 if (io_cqring_events(ctx) || io_has_work(ctx))
2514 mask |= EPOLLIN | EPOLLRDNORM;2681 mask |= EPOLLIN | EPOLLRDNORM;
2515 2682
2516 return mask;2683 return mask;
@@ -2573,6 +2740,9 @@ static __cold void io_ring_exit_work(struct work_struct *work) · 56 unmodified lines
2573 * as nobody else will be looking for them.2740 * as nobody else will be looking for them.
2574 */2741 */
2575 do {2742 do {
2743 if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
2744 io_move_task_work_from_local(ctx);
2745
2576 while (io_uring_try_cancel_requests(ctx, NULL, true))2746 while (io_uring_try_cancel_requests(ctx, NULL, true))
2577 cond_resched();2747 cond_resched();
2578 2748
@@ -2770,13 +2940,15 @@ static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, · 191 unmodified lines
2770 }2940 }
2771 }2941 }
2772 2942
2943 if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
2944 ret |= io_run_local_work(ctx) > 0;
2773 ret |= io_cancel_defer_files(ctx, task, cancel_all);2945 ret |= io_cancel_defer_files(ctx, task, cancel_all);
2774 mutex_lock(&ctx->uring_lock);2946 mutex_lock(&ctx->uring_lock);
2775 ret |= io_poll_remove_all(ctx, task, cancel_all);2947 ret |= io_poll_remove_all(ctx, task, cancel_all);
2776 mutex_unlock(&ctx->uring_lock);2948 mutex_unlock(&ctx->uring_lock);
2777 ret |= io_kill_timeouts(ctx, task, cancel_all);2949 ret |= io_kill_timeouts(ctx, task, cancel_all);
2778 if (task)2950 if (task)
2779 ret |= io_run_task_work();2951 ret |= io_run_task_work() > 0;
2780 return ret;2952 return ret;
2781}2953}
2782 2954
@@ -2992,8 +3164,6 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 209 unmodified lines
2992 struct fd f;3164 struct fd f;
2993 long ret;3165 long ret;
2994 3166
2995 io_run_task_work();
2996
2997 if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |3167 if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
2998 IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG |3168 IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG |
2999 IORING_ENTER_REGISTERED_RING)))3169 IORING_ENTER_REGISTERED_RING)))
@@ -3063,8 +3233,10 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 63 unmodified lines
3063 goto iopoll_locked;3233 goto iopoll_locked;
3064 mutex_unlock(&ctx->uring_lock);3234 mutex_unlock(&ctx->uring_lock);
3065 }3235 }
3236
3066 if (flags & IORING_ENTER_GETEVENTS) {3237 if (flags & IORING_ENTER_GETEVENTS) {
3067 int ret2;3238 int ret2;
3239
3068 if (ctx->syscall_iopoll) {3240 if (ctx->syscall_iopoll) {
3069 /*3241 /*
3070 * We disallow the app entering submit/complete with3242 * We disallow the app entering submit/complete with
@@ -3293,17 +3465,29 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, · 222 unmodified lines
3293 if (ctx->flags & IORING_SETUP_SQPOLL) {3465 if (ctx->flags & IORING_SETUP_SQPOLL) {
3294 /* IPI related flags don't make sense with SQPOLL */3466 /* IPI related flags don't make sense with SQPOLL */
3295 if (ctx->flags & (IORING_SETUP_COOP_TASKRUN |3467 if (ctx->flags & (IORING_SETUP_COOP_TASKRUN |
3296 IORING_SETUP_TASKRUN_FLAG))3468 IORING_SETUP_TASKRUN_FLAG |
3469 IORING_SETUP_DEFER_TASKRUN))
3297 goto err;3470 goto err;
3298 ctx->notify_method = TWA_SIGNAL_NO_IPI;3471 ctx->notify_method = TWA_SIGNAL_NO_IPI;
3299 } else if (ctx->flags & IORING_SETUP_COOP_TASKRUN) {3472 } else if (ctx->flags & IORING_SETUP_COOP_TASKRUN) {
3300 ctx->notify_method = TWA_SIGNAL_NO_IPI;3473 ctx->notify_method = TWA_SIGNAL_NO_IPI;
3301 } else {3474 } else {
3302 if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)3475 if (ctx->flags & IORING_SETUP_TASKRUN_FLAG &&
3476 !(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
3303 goto err;3477 goto err;
3304 ctx->notify_method = TWA_SIGNAL;3478 ctx->notify_method = TWA_SIGNAL;
3305 }3479 }
3306 3480
3481 /*
3482 * For DEFER_TASKRUN we require the completion task to be the same as the
3483 * submission task. This implies that there is only one submitter, so enforce
3484 * that.
3485 */
3486 if (ctx->flags & IORING_SETUP_DEFER_TASKRUN &&
3487 !(ctx->flags & IORING_SETUP_SINGLE_ISSUER)) {
3488 goto err;
3489 }
3490
3307 /*3491 /*
3308 * This is just grabbed for accounting purposes. When a process exits,3492 * This is just grabbed for accounting purposes. When a process exits,
3309 * the mm is exited and dropped before the files, hence we need to hang3493 * the mm is exited and dropped before the files, hence we need to hang
@@ -3408,7 +3592,7 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) · 98 unmodified lines
3408 IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL |3592 IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL |
3409 IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG |3593 IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG |
3410 IORING_SETUP_SQE128 | IORING_SETUP_CQE32 |3594 IORING_SETUP_SQE128 | IORING_SETUP_CQE32 |
3411 IORING_SETUP_SINGLE_ISSUER))3595 IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN))
3412 return -EINVAL;3596 return -EINVAL;
3413 3597
3414 return io_uring_create(entries, &p, params);3598 return io_uring_create(entries, &p, params);
@@ -3874,7 +4058,7 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, · 459 unmodified lines
3874 4058
3875 ctx = f.file->private_data;4059 ctx = f.file->private_data;
3876 4060
3877 io_run_task_work();4061 io_run_task_work_ctx(ctx);
3878 4062
3879 mutex_lock(&ctx->uring_lock);4063 mutex_lock(&ctx->uring_lock);
3880 ret = __io_uring_register(ctx, opcode, arg, nr_args);4064 ret = __io_uring_register(ctx, opcode, arg, nr_args);
io_uring/io_uring.h+51−11
@@ -24,9 +24,11 @@ enum { · 23 unmodified lines
24 IOU_STOP_MULTISHOT = -ECANCELED,24 IOU_STOP_MULTISHOT = -ECANCELED,
25};25};
26 26
27struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx);27struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow);
28bool io_req_cqe_overflow(struct io_kiocb *req);28bool io_req_cqe_overflow(struct io_kiocb *req);
29int io_run_task_work_sig(void);29int io_run_task_work_sig(struct io_ring_ctx *ctx);
30int __io_run_local_work(struct io_ring_ctx *ctx, bool locked);
31int io_run_local_work(struct io_ring_ctx *ctx);
30void io_req_complete_failed(struct io_kiocb *req, s32 res);32void io_req_complete_failed(struct io_kiocb *req, s32 res);
31void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);33void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);
32void io_req_complete_post(struct io_kiocb *req);34void io_req_complete_post(struct io_kiocb *req);
@@ -91,7 +93,8 @@ static inline void io_cq_lock(struct io_ring_ctx *ctx) · 58 unmodified lines
91 93
92void io_cq_unlock_post(struct io_ring_ctx *ctx);94void io_cq_unlock_post(struct io_ring_ctx *ctx);
93 95
94static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)96static inline struct io_uring_cqe *io_get_cqe_overflow(struct io_ring_ctx *ctx,
97 bool overflow)
95{98{
96 if (likely(ctx->cqe_cached < ctx->cqe_sentinel)) {99 if (likely(ctx->cqe_cached < ctx->cqe_sentinel)) {
97 struct io_uring_cqe *cqe = ctx->cqe_cached;100 struct io_uring_cqe *cqe = ctx->cqe_cached;
@@ -103,7 +106,12 @@ static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx) · 5 unmodified lines
103 return cqe;106 return cqe;
104 }107 }
105 108
106 return __io_get_cqe(ctx);109 return __io_get_cqe(ctx, overflow);
110}
111
112static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
113{
114 return io_get_cqe_overflow(ctx, false);
107}115}
108 116
109static inline bool __io_fill_cqe_req(struct io_ring_ctx *ctx,117static inline bool __io_fill_cqe_req(struct io_ring_ctx *ctx,
@@ -221,17 +229,43 @@ static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) · 111 unmodified lines
221 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;229 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
222}230}
223 231
224static inline bool io_run_task_work(void)232static inline int io_run_task_work(void)
225{233{
226 if (test_thread_flag(TIF_NOTIFY_SIGNAL)) {234 if (task_work_pending(current)) {
235 if (test_thread_flag(TIF_NOTIFY_SIGNAL))
236 clear_notify_signal();
227 __set_current_state(TASK_RUNNING);237 __set_current_state(TASK_RUNNING);
228 clear_notify_signal();238 task_work_run();
229 if (task_work_pending(current))239 return 1;
230 task_work_run();
231 return true;
232 }240 }
233 241
234 return false;242 return 0;
243}
244
245static inline bool io_task_work_pending(struct io_ring_ctx *ctx)
246{
247 return test_thread_flag(TIF_NOTIFY_SIGNAL) ||
248 !wq_list_empty(&ctx->work_llist);
249}
250
251static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx)
252{
253 int ret = 0;
254 int ret2;
255
256 if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
257 ret = io_run_local_work(ctx);
258
259 /* want to run this after in case more is added */
260 ret2 = io_run_task_work();
261
262 /* Try propagate error in favour of if tasks were run,
263 * but still make sure to run them if requested
264 */
265 if (ret >= 0)
266 ret += ret2;
267
268 return ret;
235}269}
236 270
237static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)271static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)
@@ -301,4 +335,10 @@ static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) · 63 unmodified lines
301 return container_of(node, struct io_kiocb, comp_list);335 return container_of(node, struct io_kiocb, comp_list);
302}336}
303 337
338static inline bool io_allowed_run_tw(struct io_ring_ctx *ctx)
339{
340 return likely(!(ctx->flags & IORING_SETUP_DEFER_TASKRUN) ||
341 ctx->submitter_task == current);
342}
343
304#endif344#endif
io_uring/kbuf.h+0−12
@@ -86,18 +86,6 @@ static inline bool io_do_buffer_select(struct io_kiocb *req) · 85 unmodified lines
86 86
87static inline void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)87static inline void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)
88{88{
89 /*
90 * READV uses fields in `struct io_rw` (len/addr) to stash the selected
91 * buffer data. However if that buffer is recycled the original request
92 * data stored in addr is lost. Therefore forbid recycling for now.
93 */
94 if (req->opcode == IORING_OP_READV) {
95 if ((req->flags & REQ_F_BUFFER_RING) && req->buf_list) {
96 req->buf_list->head++;
97 req->buf_list = NULL;
98 }
99 return;
100 }
101 if (req->flags & REQ_F_BUFFER_SELECTED)89 if (req->flags & REQ_F_BUFFER_SELECTED)
102 io_kbuf_recycle_legacy(req, issue_flags);90 io_kbuf_recycle_legacy(req, issue_flags);
103 if (req->flags & REQ_F_BUFFER_RING)91 if (req->flags & REQ_F_BUFFER_RING)
io_uring/net.c+222−86
@@ -55,21 +55,14 @@ struct io_sr_msg { · 54 unmodified lines
55 struct user_msghdr __user *umsg;55 struct user_msghdr __user *umsg;
56 void __user *buf;56 void __user *buf;
57 };57 };
58 unsigned len;
59 unsigned done_io;
58 unsigned msg_flags;60 unsigned msg_flags;
59 unsigned flags;61 u16 flags;
60 size_t len;62 /* initialised and used only by !msg send variants */
61 size_t done_io;63 u16 addr_len;
62};
63
64struct io_sendzc {
65 struct file *file;
66 void __user *buf;
67 size_t len;
68 unsigned msg_flags;
69 unsigned flags;
70 unsigned addr_len;
71 void __user *addr;64 void __user *addr;
72 size_t done_io;65 /* used only for send zerocopy */
73 struct io_kiocb *notif;66 struct io_kiocb *notif;
74};67};
75 68
@@ -126,28 +119,36 @@ static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags) · 50 unmodified lines
126 }119 }
127}120}
128 121
129static struct io_async_msghdr *io_recvmsg_alloc_async(struct io_kiocb *req,122static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req,
130 unsigned int issue_flags)123 unsigned int issue_flags)
131{124{
132 struct io_ring_ctx *ctx = req->ctx;125 struct io_ring_ctx *ctx = req->ctx;
133 struct io_cache_entry *entry;126 struct io_cache_entry *entry;
127 struct io_async_msghdr *hdr;
134 128
135 if (!(issue_flags & IO_URING_F_UNLOCKED) &&129 if (!(issue_flags & IO_URING_F_UNLOCKED) &&
136 (entry = io_alloc_cache_get(&ctx->netmsg_cache)) != NULL) {130 (entry = io_alloc_cache_get(&ctx->netmsg_cache)) != NULL) {
137 struct io_async_msghdr *hdr;
138
139 hdr = container_of(entry, struct io_async_msghdr, cache);131 hdr = container_of(entry, struct io_async_msghdr, cache);
132 hdr->free_iov = NULL;
140 req->flags |= REQ_F_ASYNC_DATA;133 req->flags |= REQ_F_ASYNC_DATA;
141 req->async_data = hdr;134 req->async_data = hdr;
142 return hdr;135 return hdr;
143 }136 }
144 137
145 if (!io_alloc_async_data(req))138 if (!io_alloc_async_data(req)) {
146 return req->async_data;139 hdr = req->async_data;
147 140 hdr->free_iov = NULL;
141 return hdr;
142 }
148 return NULL;143 return NULL;
149}144}
150 145
146static inline struct io_async_msghdr *io_msg_alloc_async_prep(struct io_kiocb *req)
147{
148 /* ->prep_async is always called from the submission context */
149 return io_msg_alloc_async(req, 0);
150}
151
151static int io_setup_async_msg(struct io_kiocb *req,152static int io_setup_async_msg(struct io_kiocb *req,
152 struct io_async_msghdr *kmsg,153 struct io_async_msghdr *kmsg,
153 unsigned int issue_flags)154 unsigned int issue_flags)
@@ -156,17 +157,20 @@ static int io_setup_async_msg(struct io_kiocb *req, · 2 unmodified lines
156 157
157 if (req_has_async_data(req))158 if (req_has_async_data(req))
158 return -EAGAIN;159 return -EAGAIN;
159 async_msg = io_recvmsg_alloc_async(req, issue_flags);160 async_msg = io_msg_alloc_async(req, issue_flags);
160 if (!async_msg) {161 if (!async_msg) {
161 kfree(kmsg->free_iov);162 kfree(kmsg->free_iov);
162 return -ENOMEM;163 return -ENOMEM;
163 }164 }
164 req->flags |= REQ_F_NEED_CLEANUP;165 req->flags |= REQ_F_NEED_CLEANUP;
165 memcpy(async_msg, kmsg, sizeof(*kmsg));166 memcpy(async_msg, kmsg, sizeof(*kmsg));
166 async_msg->msg.msg_name = &async_msg->addr;167 if (async_msg->msg.msg_name)
168 async_msg->msg.msg_name = &async_msg->addr;
167 /* if were using fast_iov, set it to the new one */169 /* if were using fast_iov, set it to the new one */
168 if (!async_msg->free_iov)170 if (!kmsg->free_iov) {
169 async_msg->msg.msg_iter.iov = async_msg->fast_iov;171 size_t fast_idx = kmsg->msg.msg_iter.iov - kmsg->fast_iov;
172 async_msg->msg.msg_iter.iov = &async_msg->fast_iov[fast_idx];
173 }
170 174
171 return -EAGAIN;175 return -EAGAIN;
172}176}
@@ -182,34 +186,34 @@ static int io_sendmsg_copy_hdr(struct io_kiocb *req, · 9 unmodified lines
182 &iomsg->free_iov);186 &iomsg->free_iov);
183}187}
184 188
185int io_sendzc_prep_async(struct io_kiocb *req)189int io_send_prep_async(struct io_kiocb *req)
186{190{
187 struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc);191 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
188 struct io_async_msghdr *io;192 struct io_async_msghdr *io;
189 int ret;193 int ret;
190 194
191 if (!zc->addr || req_has_async_data(req))195 if (!zc->addr || req_has_async_data(req))
192 return 0;196 return 0;
193 if (io_alloc_async_data(req))197 io = io_msg_alloc_async_prep(req);
198 if (!io)
194 return -ENOMEM;199 return -ENOMEM;
195
196 io = req->async_data;
197 ret = move_addr_to_kernel(zc->addr, zc->addr_len, &io->addr);200 ret = move_addr_to_kernel(zc->addr, zc->addr_len, &io->addr);
198 return ret;201 return ret;
199}202}
200 203
201static int io_setup_async_addr(struct io_kiocb *req,204static int io_setup_async_addr(struct io_kiocb *req,
202 struct sockaddr_storage *addr,205 struct sockaddr_storage *addr_storage,
203 unsigned int issue_flags)206 unsigned int issue_flags)
204{207{
208 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
205 struct io_async_msghdr *io;209 struct io_async_msghdr *io;
206 210
207 if (!addr || req_has_async_data(req))211 if (!sr->addr || req_has_async_data(req))
208 return -EAGAIN;212 return -EAGAIN;
209 if (io_alloc_async_data(req))213 io = io_msg_alloc_async(req, issue_flags);
214 if (!io)
210 return -ENOMEM;215 return -ENOMEM;
211 io = req->async_data;216 memcpy(&io->addr, addr_storage, sizeof(io->addr));
212 memcpy(&io->addr, addr, sizeof(io->addr));
213 return -EAGAIN;217 return -EAGAIN;
214}218}
215 219
@@ -217,6 +221,8 @@ int io_sendmsg_prep_async(struct io_kiocb *req) · 1 unmodified line
217{221{
218 int ret;222 int ret;
219 223
224 if (!io_msg_alloc_async_prep(req))
225 return -ENOMEM;
220 ret = io_sendmsg_copy_hdr(req, req->async_data);226 ret = io_sendmsg_copy_hdr(req, req->async_data);
221 if (!ret)227 if (!ret)
222 req->flags |= REQ_F_NEED_CLEANUP;228 req->flags |= REQ_F_NEED_CLEANUP;
@@ -234,8 +240,14 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 11 unmodified lines
234{240{
235 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);241 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
236 242
237 if (unlikely(sqe->file_index || sqe->addr2))243 if (req->opcode == IORING_OP_SEND) {
244 if (READ_ONCE(sqe->__pad3[0]))
245 return -EINVAL;
246 sr->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2));
247 sr->addr_len = READ_ONCE(sqe->addr_len);
248 } else if (sqe->addr2 || sqe->file_index) {
238 return -EINVAL;249 return -EINVAL;
250 }
239 251
240 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));252 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
241 sr->len = READ_ONCE(sqe->len);253 sr->len = READ_ONCE(sqe->len);
@@ -291,13 +303,13 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) · 49 unmodified lines
291 if (ret < min_ret) {303 if (ret < min_ret) {
292 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))304 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
293 return io_setup_async_msg(req, kmsg, issue_flags);305 return io_setup_async_msg(req, kmsg, issue_flags);
294 if (ret == -ERESTARTSYS)
295 ret = -EINTR;
296 if (ret > 0 && io_net_retry(sock, flags)) {306 if (ret > 0 && io_net_retry(sock, flags)) {
297 sr->done_io += ret;307 sr->done_io += ret;
298 req->flags |= REQ_F_PARTIAL_IO;308 req->flags |= REQ_F_PARTIAL_IO;
299 return io_setup_async_msg(req, kmsg, issue_flags);309 return io_setup_async_msg(req, kmsg, issue_flags);
300 }310 }
311 if (ret == -ERESTARTSYS)
312 ret = -EINTR;
301 req_set_fail(req);313 req_set_fail(req);
302 }314 }
303 /* fast path, check for non-NULL to avoid function call */315 /* fast path, check for non-NULL to avoid function call */
@@ -315,6 +327,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) · 11 unmodified lines
315 327
316int io_send(struct io_kiocb *req, unsigned int issue_flags)328int io_send(struct io_kiocb *req, unsigned int issue_flags)
317{329{
330 struct sockaddr_storage __address;
318 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);331 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
319 struct msghdr msg;332 struct msghdr msg;
320 struct iovec iov;333 struct iovec iov;
@@ -323,9 +336,29 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) · 2 unmodified lines
323 int min_ret = 0;336 int min_ret = 0;
324 int ret;337 int ret;
325 338
339 msg.msg_name = NULL;
340 msg.msg_control = NULL;
341 msg.msg_controllen = 0;
342 msg.msg_namelen = 0;
343 msg.msg_ubuf = NULL;
344
345 if (sr->addr) {
346 if (req_has_async_data(req)) {
347 struct io_async_msghdr *io = req->async_data;
348
349 msg.msg_name = &io->addr;
350 } else {
351 ret = move_addr_to_kernel(sr->addr, sr->addr_len, &__address);
352 if (unlikely(ret < 0))
353 return ret;
354 msg.msg_name = (struct sockaddr *)&__address;
355 }
356 msg.msg_namelen = sr->addr_len;
357 }
358
326 if (!(req->flags & REQ_F_POLLED) &&359 if (!(req->flags & REQ_F_POLLED) &&
327 (sr->flags & IORING_RECVSEND_POLL_FIRST))360 (sr->flags & IORING_RECVSEND_POLL_FIRST))
328 return -EAGAIN;361 return io_setup_async_addr(req, &__address, issue_flags);
329 362
330 sock = sock_from_file(req->file);363 sock = sock_from_file(req->file);
331 if (unlikely(!sock))364 if (unlikely(!sock))
@@ -335,12 +368,6 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) · 3 unmodified lines
335 if (unlikely(ret))368 if (unlikely(ret))
336 return ret;369 return ret;
337 370
338 msg.msg_name = NULL;
339 msg.msg_control = NULL;
340 msg.msg_controllen = 0;
341 msg.msg_namelen = 0;
342 msg.msg_ubuf = NULL;
343
344 flags = sr->msg_flags;371 flags = sr->msg_flags;
345 if (issue_flags & IO_URING_F_NONBLOCK)372 if (issue_flags & IO_URING_F_NONBLOCK)
346 flags |= MSG_DONTWAIT;373 flags |= MSG_DONTWAIT;
@@ -351,16 +378,17 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) · 4 unmodified lines
351 ret = sock_sendmsg(sock, &msg);378 ret = sock_sendmsg(sock, &msg);
352 if (ret < min_ret) {379 if (ret < min_ret) {
353 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))380 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
354 return -EAGAIN;381 return io_setup_async_addr(req, &__address, issue_flags);
355 if (ret == -ERESTARTSYS)382
356 ret = -EINTR;
357 if (ret > 0 && io_net_retry(sock, flags)) {383 if (ret > 0 && io_net_retry(sock, flags)) {
358 sr->len -= ret;384 sr->len -= ret;
359 sr->buf += ret;385 sr->buf += ret;
360 sr->done_io += ret;386 sr->done_io += ret;
361 req->flags |= REQ_F_PARTIAL_IO;387 req->flags |= REQ_F_PARTIAL_IO;
362 return -EAGAIN;388 return io_setup_async_addr(req, &__address, issue_flags);
363 }389 }
390 if (ret == -ERESTARTSYS)
391 ret = -EINTR;
364 req_set_fail(req);392 req_set_fail(req);
365 }393 }
366 if (ret >= 0)394 if (ret >= 0)
@@ -454,7 +482,6 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, · 87 unmodified lines
454 482
455 if (msg.msg_iovlen == 0) {483 if (msg.msg_iovlen == 0) {
456 sr->len = 0;484 sr->len = 0;
457 iomsg->free_iov = NULL;
458 } else if (msg.msg_iovlen > 1) {485 } else if (msg.msg_iovlen > 1) {
459 return -EINVAL;486 return -EINVAL;
460 } else {487 } else {
@@ -465,7 +492,6 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, · 4 unmodified lines
465 if (clen < 0)492 if (clen < 0)
466 return -EINVAL;493 return -EINVAL;
467 sr->len = clen;494 sr->len = clen;
468 iomsg->free_iov = NULL;
469 }495 }
470 496
471 if (req->flags & REQ_F_APOLL_MULTISHOT) {497 if (req->flags & REQ_F_APOLL_MULTISHOT) {
@@ -504,6 +530,8 @@ int io_recvmsg_prep_async(struct io_kiocb *req) · 32 unmodified lines
504{530{
505 int ret;531 int ret;
506 532
533 if (!io_msg_alloc_async_prep(req))
534 return -ENOMEM;
507 ret = io_recvmsg_copy_hdr(req, req->async_data);535 ret = io_recvmsg_copy_hdr(req, req->async_data);
508 if (!ret)536 if (!ret)
509 req->flags |= REQ_F_NEED_CLEANUP;537 req->flags |= REQ_F_NEED_CLEANUP;
@@ -751,13 +779,13 @@ retry_multishot: · 241 unmodified lines
751 }779 }
752 return ret;780 return ret;
753 }781 }
754 if (ret == -ERESTARTSYS)
755 ret = -EINTR;
756 if (ret > 0 && io_net_retry(sock, flags)) {782 if (ret > 0 && io_net_retry(sock, flags)) {
757 sr->done_io += ret;783 sr->done_io += ret;
758 req->flags |= REQ_F_PARTIAL_IO;784 req->flags |= REQ_F_PARTIAL_IO;
759 return io_setup_async_msg(req, kmsg, issue_flags);785 return io_setup_async_msg(req, kmsg, issue_flags);
760 }786 }
787 if (ret == -ERESTARTSYS)
788 ret = -EINTR;
761 req_set_fail(req);789 req_set_fail(req);
762 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {790 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
763 req_set_fail(req);791 req_set_fail(req);
@@ -847,8 +875,6 @@ retry_multishot: · 83 unmodified lines
847 875
848 return -EAGAIN;876 return -EAGAIN;
849 }877 }
850 if (ret == -ERESTARTSYS)
851 ret = -EINTR;
852 if (ret > 0 && io_net_retry(sock, flags)) {878 if (ret > 0 && io_net_retry(sock, flags)) {
853 sr->len -= ret;879 sr->len -= ret;
854 sr->buf += ret;880 sr->buf += ret;
@@ -856,6 +882,8 @@ retry_multishot: · 1 unmodified line
856 req->flags |= REQ_F_PARTIAL_IO;882 req->flags |= REQ_F_PARTIAL_IO;
857 return -EAGAIN;883 return -EAGAIN;
858 }884 }
885 if (ret == -ERESTARTSYS)
886 ret = -EINTR;
859 req_set_fail(req);887 req_set_fail(req);
860 } else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {888 } else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
861out_free:889out_free:
@@ -879,23 +907,30 @@ out_free: · 17 unmodified lines
879 return ret;907 return ret;
880}908}
881 909
882void io_sendzc_cleanup(struct io_kiocb *req)910void io_send_zc_cleanup(struct io_kiocb *req)
883{911{
884 struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc);912 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
913 struct io_async_msghdr *io;
885 914
886 zc->notif->flags |= REQ_F_CQE_SKIP;915 if (req_has_async_data(req)) {
887 io_notif_flush(zc->notif);916 io = req->async_data;
888 zc->notif = NULL;917 /* might be ->fast_iov if *msg_copy_hdr failed */
918 if (io->free_iov != io->fast_iov)
919 kfree(io->free_iov);
920 }
921 if (zc->notif) {
922 io_notif_flush(zc->notif);
923 zc->notif = NULL;
924 }
889}925}
890 926
891int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)927int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
892{928{
893 struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc);929 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
894 struct io_ring_ctx *ctx = req->ctx;930 struct io_ring_ctx *ctx = req->ctx;
895 struct io_kiocb *notif;931 struct io_kiocb *notif;
896 932
897 if (READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3) ||933 if (unlikely(READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3)))
898 READ_ONCE(sqe->__pad3[0]))
899 return -EINVAL;934 return -EINVAL;
900 /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */935 /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */
901 if (req->flags & REQ_F_CQE_SKIP)936 if (req->flags & REQ_F_CQE_SKIP)
@@ -922,14 +957,24 @@ int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 20 unmodified lines
922 io_req_set_rsrc_node(notif, ctx, 0);957 io_req_set_rsrc_node(notif, ctx, 0);
923 }958 }
924 959
960 if (req->opcode == IORING_OP_SEND_ZC) {
961 if (READ_ONCE(sqe->__pad3[0]))
962 return -EINVAL;
963 zc->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2));
964 zc->addr_len = READ_ONCE(sqe->addr_len);
965 } else {
966 if (unlikely(sqe->addr2 || sqe->file_index))
967 return -EINVAL;
968 if (unlikely(zc->flags & IORING_RECVSEND_FIXED_BUF))
969 return -EINVAL;
970 }
971
925 zc->buf = u64_to_user_ptr(READ_ONCE(sqe->addr));972 zc->buf = u64_to_user_ptr(READ_ONCE(sqe->addr));
926 zc->len = READ_ONCE(sqe->len);973 zc->len = READ_ONCE(sqe->len);
927 zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;974 zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
928 if (zc->msg_flags & MSG_DONTWAIT)975 if (zc->msg_flags & MSG_DONTWAIT)
929 req->flags |= REQ_F_NOWAIT;976 req->flags |= REQ_F_NOWAIT;
930 977
931 zc->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2));
932 zc->addr_len = READ_ONCE(sqe->addr_len);
933 zc->done_io = 0;978 zc->done_io = 0;
934 979
935#ifdef CONFIG_COMPAT980#ifdef CONFIG_COMPAT
@@ -939,6 +984,13 @@ int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 3 unmodified lines
939 return 0;984 return 0;
940}985}
941 986
987static int io_sg_from_iter_iovec(struct sock *sk, struct sk_buff *skb,
988 struct iov_iter *from, size_t length)
989{
990 skb_zcopy_downgrade_managed(skb);
991 return __zerocopy_sg_from_iter(NULL, sk, skb, from, length);
992}
993
942static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb,994static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb,
943 struct iov_iter *from, size_t length)995 struct iov_iter *from, size_t length)
944{996{
@@ -949,13 +1001,10 @@ static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb, · 4 unmodified lines
949 ssize_t copied = 0;1001 ssize_t copied = 0;
950 unsigned long truesize = 0;1002 unsigned long truesize = 0;
951 1003
952 if (!shinfo->nr_frags)1004 if (!frag)
953 shinfo->flags |= SKBFL_MANAGED_FRAG_REFS;1005 shinfo->flags |= SKBFL_MANAGED_FRAG_REFS;
954 1006 else if (unlikely(!skb_zcopy_managed(skb)))
955 if (!skb_zcopy_managed(skb) || !iov_iter_is_bvec(from)) {
956 skb_zcopy_downgrade_managed(skb);
957 return __zerocopy_sg_from_iter(NULL, sk, skb, from, length);1007 return __zerocopy_sg_from_iter(NULL, sk, skb, from, length);
958 }
959 1008
960 bi.bi_size = min(from->count, length);1009 bi.bi_size = min(from->count, length);
961 bi.bi_bvec_done = from->iov_offset;1010 bi.bi_bvec_done = from->iov_offset;
@@ -993,14 +1042,14 @@ static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb, · 31 unmodified lines
993 return ret;1042 return ret;
994}1043}
995 1044
996int io_sendzc(struct io_kiocb *req, unsigned int issue_flags)1045int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)
997{1046{
998 struct sockaddr_storage __address, *addr = NULL;1047 struct sockaddr_storage __address;
999 struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc);1048 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
1000 struct msghdr msg;1049 struct msghdr msg;
1001 struct iovec iov;1050 struct iovec iov;
1002 struct socket *sock;1051 struct socket *sock;
1003 unsigned msg_flags, cflags;1052 unsigned msg_flags;
1004 int ret, min_ret = 0;1053 int ret, min_ret = 0;
1005 1054
1006 sock = sock_from_file(req->file);1055 sock = sock_from_file(req->file);
@@ -1016,26 +1065,26 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) · 9 unmodified lines
1016 if (req_has_async_data(req)) {1065 if (req_has_async_data(req)) {
1017 struct io_async_msghdr *io = req->async_data;1066 struct io_async_msghdr *io = req->async_data;
1018 1067
1019 msg.msg_name = addr = &io->addr;1068 msg.msg_name = &io->addr;
1020 } else {1069 } else {
1021 ret = move_addr_to_kernel(zc->addr, zc->addr_len, &__address);1070 ret = move_addr_to_kernel(zc->addr, zc->addr_len, &__address);
1022 if (unlikely(ret < 0))1071 if (unlikely(ret < 0))
1023 return ret;1072 return ret;
1024 msg.msg_name = (struct sockaddr *)&__address;1073 msg.msg_name = (struct sockaddr *)&__address;
1025 addr = &__address;
1026 }1074 }
1027 msg.msg_namelen = zc->addr_len;1075 msg.msg_namelen = zc->addr_len;
1028 }1076 }
1029 1077
1030 if (!(req->flags & REQ_F_POLLED) &&1078 if (!(req->flags & REQ_F_POLLED) &&
1031 (zc->flags & IORING_RECVSEND_POLL_FIRST))1079 (zc->flags & IORING_RECVSEND_POLL_FIRST))
1032 return io_setup_async_addr(req, addr, issue_flags);1080 return io_setup_async_addr(req, &__address, issue_flags);
1033 1081
1034 if (zc->flags & IORING_RECVSEND_FIXED_BUF) {1082 if (zc->flags & IORING_RECVSEND_FIXED_BUF) {
1035 ret = io_import_fixed(WRITE, &msg.msg_iter, req->imu,1083 ret = io_import_fixed(WRITE, &msg.msg_iter, req->imu,
1036 (u64)(uintptr_t)zc->buf, zc->len);1084 (u64)(uintptr_t)zc->buf, zc->len);
1037 if (unlikely(ret))1085 if (unlikely(ret))
1038 return ret;1086 return ret;
1087 msg.sg_from_iter = io_sg_from_iter;
1039 } else {1088 } else {
1040 ret = import_single_range(WRITE, zc->buf, zc->len, &iov,1089 ret = import_single_range(WRITE, zc->buf, zc->len, &iov,
1041 &msg.msg_iter);1090 &msg.msg_iter);
@@ -1044,6 +1093,7 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) · 2 unmodified lines
1044 ret = io_notif_account_mem(zc->notif, zc->len);1093 ret = io_notif_account_mem(zc->notif, zc->len);
1045 if (unlikely(ret))1094 if (unlikely(ret))
1046 return ret;1095 return ret;
1096 msg.sg_from_iter = io_sg_from_iter_iovec;
1047 }1097 }
1048 1098
1049 msg_flags = zc->msg_flags | MSG_ZEROCOPY;1099 msg_flags = zc->msg_flags | MSG_ZEROCOPY;
@@ -1054,22 +1104,19 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) · 4 unmodified lines
1054 1104
1055 msg.msg_flags = msg_flags;1105 msg.msg_flags = msg_flags;
1056 msg.msg_ubuf = &io_notif_to_data(zc->notif)->uarg;1106 msg.msg_ubuf = &io_notif_to_data(zc->notif)->uarg;
1057 msg.sg_from_iter = io_sg_from_iter;
1058 ret = sock_sendmsg(sock, &msg);1107 ret = sock_sendmsg(sock, &msg);
1059 1108
1060 if (unlikely(ret < min_ret)) {1109 if (unlikely(ret < min_ret)) {
1061 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))1110 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1062 return io_setup_async_addr(req, addr, issue_flags);1111 return io_setup_async_addr(req, &__address, issue_flags);
1063 1112
1064 if (ret > 0 && io_net_retry(sock, msg.msg_flags)) {1113 if (ret > 0 && io_net_retry(sock, msg.msg_flags)) {
1065 zc->len -= ret;1114 zc->len -= ret;
1066 zc->buf += ret;1115 zc->buf += ret;
1067 zc->done_io += ret;1116 zc->done_io += ret;
1068 req->flags |= REQ_F_PARTIAL_IO;1117 req->flags |= REQ_F_PARTIAL_IO;
1069 return io_setup_async_addr(req, addr, issue_flags);1118 return io_setup_async_addr(req, &__address, issue_flags);
1070 }1119 }
1071 if (ret < 0 && !zc->done_io)
1072 zc->notif->flags |= REQ_F_CQE_SKIP;
1073 if (ret == -ERESTARTSYS)1120 if (ret == -ERESTARTSYS)
1074 ret = -EINTR;1121 ret = -EINTR;
1075 req_set_fail(req);1122 req_set_fail(req);
@@ -1080,13 +1127,102 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) · 4 unmodified lines
1080 else if (zc->done_io)1127 else if (zc->done_io)
1081 ret = zc->done_io;1128 ret = zc->done_io;
1082 1129
1083 io_notif_flush(zc->notif);1130 /*
1084 req->flags &= ~REQ_F_NEED_CLEANUP;1131 * If we're in io-wq we can't rely on tw ordering guarantees, defer
1085 cflags = ret >= 0 ? IORING_CQE_F_MORE : 0;1132 * flushing notif to io_send_zc_cleanup()
1086 io_req_set_res(req, ret, cflags);1133 */
1134 if (!(issue_flags & IO_URING_F_UNLOCKED)) {
1135 io_notif_flush(zc->notif);
1136 req->flags &= ~REQ_F_NEED_CLEANUP;
1137 }
1138 io_req_set_res(req, ret, IORING_CQE_F_MORE);
1087 return IOU_OK;1139 return IOU_OK;
1088}1140}
1089 1141
1142int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)
1143{
1144 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1145 struct io_async_msghdr iomsg, *kmsg;
1146 struct socket *sock;
1147 unsigned flags;
1148 int ret, min_ret = 0;
1149
1150 sock = sock_from_file(req->file);
1151 if (unlikely(!sock))
1152 return -ENOTSOCK;
1153
1154 if (req_has_async_data(req)) {
1155 kmsg = req->async_data;
1156 } else {
1157 ret = io_sendmsg_copy_hdr(req, &iomsg);
1158 if (ret)
1159 return ret;
1160 kmsg = &iomsg;
1161 }
1162
1163 if (!(req->flags & REQ_F_POLLED) &&
1164 (sr->flags & IORING_RECVSEND_POLL_FIRST))
1165 return io_setup_async_msg(req, kmsg, issue_flags);
1166
1167 flags = sr->msg_flags | MSG_ZEROCOPY;
1168 if (issue_flags & IO_URING_F_NONBLOCK)
1169 flags |= MSG_DONTWAIT;
1170 if (flags & MSG_WAITALL)
1171 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1172
1173 kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg;
1174 kmsg->msg.sg_from_iter = io_sg_from_iter_iovec;
1175 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
1176
1177 if (unlikely(ret < min_ret)) {
1178 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1179 return io_setup_async_msg(req, kmsg, issue_flags);
1180
1181 if (ret > 0 && io_net_retry(sock, flags)) {
1182 sr->done_io += ret;
1183 req->flags |= REQ_F_PARTIAL_IO;
1184 return io_setup_async_msg(req, kmsg, issue_flags);
1185 }
1186 if (ret == -ERESTARTSYS)
1187 ret = -EINTR;
1188 req_set_fail(req);
1189 }
1190 /* fast path, check for non-NULL to avoid function call */
1191 if (kmsg->free_iov) {
1192 kfree(kmsg->free_iov);
1193 kmsg->free_iov = NULL;
1194 }
1195
1196 io_netmsg_recycle(req, issue_flags);
1197 if (ret >= 0)
1198 ret += sr->done_io;
1199 else if (sr->done_io)
1200 ret = sr->done_io;
1201
1202 /*
1203 * If we're in io-wq we can't rely on tw ordering guarantees, defer
1204 * flushing notif to io_send_zc_cleanup()
1205 */
1206 if (!(issue_flags & IO_URING_F_UNLOCKED)) {
1207 io_notif_flush(sr->notif);
1208 req->flags &= ~REQ_F_NEED_CLEANUP;
1209 }
1210 io_req_set_res(req, ret, IORING_CQE_F_MORE);
1211 return IOU_OK;
1212}
1213
1214void io_sendrecv_fail(struct io_kiocb *req)
1215{
1216 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1217
1218 if (req->flags & REQ_F_PARTIAL_IO)
1219 req->cqe.res = sr->done_io;
1220
1221 if ((req->flags & REQ_F_NEED_CLEANUP) &&
1222 (req->opcode == IORING_OP_SEND_ZC || req->opcode == IORING_OP_SENDMSG_ZC))
1223 req->cqe.flags |= IORING_CQE_F_MORE;
1224}
1225
1090int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)1226int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1091{1227{
1092 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept);1228 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept);
io_uring/net.h+8−4
@@ -31,18 +31,21 @@ struct io_async_connect { · 30 unmodified lines
31int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);31int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
32int io_shutdown(struct io_kiocb *req, unsigned int issue_flags);32int io_shutdown(struct io_kiocb *req, unsigned int issue_flags);
33 33
34int io_sendzc_prep_async(struct io_kiocb *req);
35int io_sendmsg_prep_async(struct io_kiocb *req);34int io_sendmsg_prep_async(struct io_kiocb *req);
36void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req);35void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req);
37int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);36int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
38int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags);37int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags);
38
39int io_send(struct io_kiocb *req, unsigned int issue_flags);39int io_send(struct io_kiocb *req, unsigned int issue_flags);
40int io_send_prep_async(struct io_kiocb *req);
40 41
41int io_recvmsg_prep_async(struct io_kiocb *req);42int io_recvmsg_prep_async(struct io_kiocb *req);
42int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);43int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
43int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags);44int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags);
44int io_recv(struct io_kiocb *req, unsigned int issue_flags);45int io_recv(struct io_kiocb *req, unsigned int issue_flags);
45 46
47void io_sendrecv_fail(struct io_kiocb *req);
48
46int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);49int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
47int io_accept(struct io_kiocb *req, unsigned int issue_flags);50int io_accept(struct io_kiocb *req, unsigned int issue_flags);
48 51
@@ -53,9 +56,10 @@ int io_connect_prep_async(struct io_kiocb *req); · 4 unmodified lines
53int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);56int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
54int io_connect(struct io_kiocb *req, unsigned int issue_flags);57int io_connect(struct io_kiocb *req, unsigned int issue_flags);
55 58
56int io_sendzc(struct io_kiocb *req, unsigned int issue_flags);59int io_send_zc(struct io_kiocb *req, unsigned int issue_flags);
57int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);60int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags);
58void io_sendzc_cleanup(struct io_kiocb *req);61int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
62void io_send_zc_cleanup(struct io_kiocb *req);
59 63
60void io_netmsg_cache_free(struct io_cache_entry *entry);64void io_netmsg_cache_free(struct io_cache_entry *entry);
61#else65#else
io_uring/opdef.c+40−4
@@ -69,6 +69,7 @@ const struct io_op_def io_op_defs[] = { · 68 unmodified lines
69 .issue = io_read,69 .issue = io_read,
70 .prep_async = io_readv_prep_async,70 .prep_async = io_readv_prep_async,
71 .cleanup = io_readv_writev_cleanup,71 .cleanup = io_readv_writev_cleanup,
72 .fail = io_rw_fail,
72 },73 },
73 [IORING_OP_WRITEV] = {74 [IORING_OP_WRITEV] = {
74 .needs_file = 1,75 .needs_file = 1,
@@ -85,6 +86,7 @@ const struct io_op_def io_op_defs[] = { · 10 unmodified lines
85 .issue = io_write,86 .issue = io_write,
86 .prep_async = io_writev_prep_async,87 .prep_async = io_writev_prep_async,
87 .cleanup = io_readv_writev_cleanup,88 .cleanup = io_readv_writev_cleanup,
89 .fail = io_rw_fail,
88 },90 },
89 [IORING_OP_FSYNC] = {91 [IORING_OP_FSYNC] = {
90 .needs_file = 1,92 .needs_file = 1,
@@ -105,6 +107,7 @@ const struct io_op_def io_op_defs[] = { · 14 unmodified lines
105 .name = "READ_FIXED",107 .name = "READ_FIXED",
106 .prep = io_prep_rw,108 .prep = io_prep_rw,
107 .issue = io_read,109 .issue = io_read,
110 .fail = io_rw_fail,
108 },111 },
109 [IORING_OP_WRITE_FIXED] = {112 [IORING_OP_WRITE_FIXED] = {
110 .needs_file = 1,113 .needs_file = 1,
@@ -119,6 +122,7 @@ const struct io_op_def io_op_defs[] = { · 8 unmodified lines
119 .name = "WRITE_FIXED",122 .name = "WRITE_FIXED",
120 .prep = io_prep_rw,123 .prep = io_prep_rw,
121 .issue = io_write,124 .issue = io_write,
125 .fail = io_rw_fail,
122 },126 },
123 [IORING_OP_POLL_ADD] = {127 [IORING_OP_POLL_ADD] = {
124 .needs_file = 1,128 .needs_file = 1,
@@ -146,6 +150,7 @@ const struct io_op_def io_op_defs[] = { · 21 unmodified lines
146 .unbound_nonreg_file = 1,150 .unbound_nonreg_file = 1,
147 .pollout = 1,151 .pollout = 1,
148 .ioprio = 1,152 .ioprio = 1,
153 .manual_alloc = 1,
149 .name = "SENDMSG",154 .name = "SENDMSG",
150#if defined(CONFIG_NET)155#if defined(CONFIG_NET)
151 .async_size = sizeof(struct io_async_msghdr),156 .async_size = sizeof(struct io_async_msghdr),
@@ -153,6 +158,7 @@ const struct io_op_def io_op_defs[] = { · 1 unmodified line
153 .issue = io_sendmsg,158 .issue = io_sendmsg,
154 .prep_async = io_sendmsg_prep_async,159 .prep_async = io_sendmsg_prep_async,
155 .cleanup = io_sendmsg_recvmsg_cleanup,160 .cleanup = io_sendmsg_recvmsg_cleanup,
161 .fail = io_sendrecv_fail,
156#else162#else
157 .prep = io_eopnotsupp_prep,163 .prep = io_eopnotsupp_prep,
158#endif164#endif
@@ -163,6 +169,7 @@ const struct io_op_def io_op_defs[] = { · 4 unmodified lines
163 .pollin = 1,169 .pollin = 1,
164 .buffer_select = 1,170 .buffer_select = 1,
165 .ioprio = 1,171 .ioprio = 1,
172 .manual_alloc = 1,
166 .name = "RECVMSG",173 .name = "RECVMSG",
167#if defined(CONFIG_NET)174#if defined(CONFIG_NET)
168 .async_size = sizeof(struct io_async_msghdr),175 .async_size = sizeof(struct io_async_msghdr),
@@ -170,6 +177,7 @@ const struct io_op_def io_op_defs[] = { · 1 unmodified line
170 .issue = io_recvmsg,177 .issue = io_recvmsg,
171 .prep_async = io_recvmsg_prep_async,178 .prep_async = io_recvmsg_prep_async,
172 .cleanup = io_sendmsg_recvmsg_cleanup,179 .cleanup = io_sendmsg_recvmsg_cleanup,
180 .fail = io_sendrecv_fail,
173#else181#else
174 .prep = io_eopnotsupp_prep,182 .prep = io_eopnotsupp_prep,
175#endif183#endif
@@ -273,6 +281,7 @@ const struct io_op_def io_op_defs[] = { · 97 unmodified lines
273 .name = "READ",281 .name = "READ",
274 .prep = io_prep_rw,282 .prep = io_prep_rw,
275 .issue = io_read,283 .issue = io_read,
284 .fail = io_rw_fail,
276 },285 },
277 [IORING_OP_WRITE] = {286 [IORING_OP_WRITE] = {
278 .needs_file = 1,287 .needs_file = 1,
@@ -287,6 +296,7 @@ const struct io_op_def io_op_defs[] = { · 8 unmodified lines
287 .name = "WRITE",296 .name = "WRITE",
288 .prep = io_prep_rw,297 .prep = io_prep_rw,
289 .issue = io_write,298 .issue = io_write,
299 .fail = io_rw_fail,
290 },300 },
291 [IORING_OP_FADVISE] = {301 [IORING_OP_FADVISE] = {
292 .needs_file = 1,302 .needs_file = 1,
@@ -306,10 +316,14 @@ const struct io_op_def io_op_defs[] = { · 13 unmodified lines
306 .pollout = 1,316 .pollout = 1,
307 .audit_skip = 1,317 .audit_skip = 1,
308 .ioprio = 1,318 .ioprio = 1,
319 .manual_alloc = 1,
309 .name = "SEND",320 .name = "SEND",
310#if defined(CONFIG_NET)321#if defined(CONFIG_NET)
322 .async_size = sizeof(struct io_async_msghdr),
311 .prep = io_sendmsg_prep,323 .prep = io_sendmsg_prep,
312 .issue = io_send,324 .issue = io_send,
325 .fail = io_sendrecv_fail,
326 .prep_async = io_send_prep_async,
313#else327#else
314 .prep = io_eopnotsupp_prep,328 .prep = io_eopnotsupp_prep,
315#endif329#endif
@@ -325,6 +339,7 @@ const struct io_op_def io_op_defs[] = { · 9 unmodified lines
325#if defined(CONFIG_NET)339#if defined(CONFIG_NET)
326 .prep = io_recvmsg_prep,340 .prep = io_recvmsg_prep,
327 .issue = io_recv,341 .issue = io_recv,
342 .fail = io_sendrecv_fail,
328#else343#else
329 .prep = io_eopnotsupp_prep,344 .prep = io_eopnotsupp_prep,
330#endif345#endif
@@ -465,6 +480,7 @@ const struct io_op_def io_op_defs[] = { · 134 unmodified lines
465 .needs_file = 1,480 .needs_file = 1,
466 .plug = 1,481 .plug = 1,
467 .name = "URING_CMD",482 .name = "URING_CMD",
483 .iopoll = 1,
468 .async_size = uring_cmd_pdu_size(1),484 .async_size = uring_cmd_pdu_size(1),
469 .prep = io_uring_cmd_prep,485 .prep = io_uring_cmd_prep,
470 .issue = io_uring_cmd,486 .issue = io_uring_cmd,
@@ -480,10 +496,30 @@ const struct io_op_def io_op_defs[] = { · 9 unmodified lines
480 .manual_alloc = 1,496 .manual_alloc = 1,
481#if defined(CONFIG_NET)497#if defined(CONFIG_NET)
482 .async_size = sizeof(struct io_async_msghdr),498 .async_size = sizeof(struct io_async_msghdr),
483 .prep = io_sendzc_prep,499 .prep = io_send_zc_prep,
484 .issue = io_sendzc,500 .issue = io_send_zc,
485 .prep_async = io_sendzc_prep_async,501 .prep_async = io_send_prep_async,
486 .cleanup = io_sendzc_cleanup,502 .cleanup = io_send_zc_cleanup,
503 .fail = io_sendrecv_fail,
504#else
505 .prep = io_eopnotsupp_prep,
506#endif
507 },
508 [IORING_OP_SENDMSG_ZC] = {
509 .name = "SENDMSG_ZC",
510 .needs_file = 1,
511 .unbound_nonreg_file = 1,
512 .pollout = 1,
513 .audit_skip = 1,
514 .ioprio = 1,
515 .manual_alloc = 1,
516#if defined(CONFIG_NET)
517 .async_size = sizeof(struct io_async_msghdr),
518 .prep = io_send_zc_prep,
519 .issue = io_sendmsg_zc,
520 .prep_async = io_sendmsg_prep_async,
521 .cleanup = io_send_zc_cleanup,
522 .fail = io_sendrecv_fail,
487#else523#else
488 .prep = io_eopnotsupp_prep,524 .prep = io_eopnotsupp_prep,
489#endif525#endif
io_uring/opdef.h+1−0
@@ -36,6 +36,7 @@ struct io_op_def { · 35 unmodified lines
36 int (*issue)(struct io_kiocb *, unsigned int);36 int (*issue)(struct io_kiocb *, unsigned int);
37 int (*prep_async)(struct io_kiocb *);37 int (*prep_async)(struct io_kiocb *);
38 void (*cleanup)(struct io_kiocb *);38 void (*cleanup)(struct io_kiocb *);
39 void (*fail)(struct io_kiocb *);
39};40};
40 41
41extern const struct io_op_def io_op_defs[];42extern const struct io_op_def io_op_defs[];
io_uring/rsrc.c+1−1
@@ -341,7 +341,7 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, · 340 unmodified lines
341 flush_delayed_work(&ctx->rsrc_put_work);341 flush_delayed_work(&ctx->rsrc_put_work);
342 reinit_completion(&data->done);342 reinit_completion(&data->done);
343 343
344 ret = io_run_task_work_sig();344 ret = io_run_task_work_sig(ctx);
345 mutex_lock(&ctx->uring_lock);345 mutex_lock(&ctx->uring_lock);
346 } while (ret >= 0);346 } while (ret >= 0);
347 data->quiesce = false;347 data->quiesce = false;
io_uring/rw.c+93−96
@@ -33,6 +33,46 @@ static inline bool io_file_supports_nowait(struct io_kiocb *req) · 32 unmodified lines
33 return req->flags & REQ_F_SUPPORT_NOWAIT;33 return req->flags & REQ_F_SUPPORT_NOWAIT;
34}34}
35 35
36#ifdef CONFIG_COMPAT
37static int io_iov_compat_buffer_select_prep(struct io_rw *rw)
38{
39 struct compat_iovec __user *uiov;
40 compat_ssize_t clen;
41
42 uiov = u64_to_user_ptr(rw->addr);
43 if (!access_ok(uiov, sizeof(*uiov)))
44 return -EFAULT;
45 if (__get_user(clen, &uiov->iov_len))
46 return -EFAULT;
47 if (clen < 0)
48 return -EINVAL;
49
50 rw->len = clen;
51 return 0;
52}
53#endif
54
55static int io_iov_buffer_select_prep(struct io_kiocb *req)
56{
57 struct iovec __user *uiov;
58 struct iovec iov;
59 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
60
61 if (rw->len != 1)
62 return -EINVAL;
63
64#ifdef CONFIG_COMPAT
65 if (req->ctx->compat)
66 return io_iov_compat_buffer_select_prep(rw);
67#endif
68
69 uiov = u64_to_user_ptr(rw->addr);
70 if (copy_from_user(&iov, uiov, sizeof(*uiov)))
71 return -EFAULT;
72 rw->len = iov.iov_len;
73 return 0;
74}
75
36int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)76int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)
37{77{
38 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);78 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
@@ -69,6 +109,16 @@ int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 30 unmodified lines
69 rw->addr = READ_ONCE(sqe->addr);109 rw->addr = READ_ONCE(sqe->addr);
70 rw->len = READ_ONCE(sqe->len);110 rw->len = READ_ONCE(sqe->len);
71 rw->flags = READ_ONCE(sqe->rw_flags);111 rw->flags = READ_ONCE(sqe->rw_flags);
112
113 /* Have to do this validation here, as this is in io_read() rw->len might
114 * have chanaged due to buffer selection
115 */
116 if (req->opcode == IORING_OP_READV && req->flags & REQ_F_BUFFER_SELECT) {
117 ret = io_iov_buffer_select_prep(req);
118 if (ret)
119 return ret;
120 }
121
72 return 0;122 return 0;
73}123}
74 124
@@ -186,14 +236,6 @@ static void kiocb_end_write(struct io_kiocb *req) · 111 unmodified lines
186 236
187static bool __io_complete_rw_common(struct io_kiocb *req, long res)237static bool __io_complete_rw_common(struct io_kiocb *req, long res)
188{238{
189 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
190
191 if (rw->kiocb.ki_flags & IOCB_WRITE) {
192 kiocb_end_write(req);
193 fsnotify_modify(req->file);
194 } else {
195 fsnotify_access(req->file);
196 }
197 if (unlikely(res != req->cqe.res)) {239 if (unlikely(res != req->cqe.res)) {
198 if ((res == -EAGAIN || res == -EOPNOTSUPP) &&240 if ((res == -EAGAIN || res == -EOPNOTSUPP) &&
199 io_rw_should_reissue(req)) {241 io_rw_should_reissue(req)) {
@@ -220,6 +262,20 @@ static inline int io_fixup_rw_res(struct io_kiocb *req, long res) · 20 unmodified lines
220 return res;262 return res;
221}263}
222 264
265static void io_req_rw_complete(struct io_kiocb *req, bool *locked)
266{
267 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
268
269 if (rw->kiocb.ki_flags & IOCB_WRITE) {
270 kiocb_end_write(req);
271 fsnotify_modify(req->file);
272 } else {
273 fsnotify_access(req->file);
274 }
275
276 io_req_task_complete(req, locked);
277}
278
223static void io_complete_rw(struct kiocb *kiocb, long res)279static void io_complete_rw(struct kiocb *kiocb, long res)
224{280{
225 struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);281 struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);
@@ -228,7 +284,7 @@ static void io_complete_rw(struct kiocb *kiocb, long res) · 2 unmodified lines
228 if (__io_complete_rw_common(req, res))284 if (__io_complete_rw_common(req, res))
229 return;285 return;
230 io_req_set_res(req, io_fixup_rw_res(req, res), 0);286 io_req_set_res(req, io_fixup_rw_res(req, res), 0);
231 req->io_task_work.func = io_req_task_complete;287 req->io_task_work.func = io_req_rw_complete;
232 io_req_task_work_add(req);288 io_req_task_work_add(req);
233}289}
234 290
@@ -279,79 +335,6 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret, · 44 unmodified lines
279 return IOU_ISSUE_SKIP_COMPLETE;335 return IOU_ISSUE_SKIP_COMPLETE;
280}336}
281 337
282#ifdef CONFIG_COMPAT
283static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,
284 unsigned int issue_flags)
285{
286 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
287 struct compat_iovec __user *uiov;
288 compat_ssize_t clen;
289 void __user *buf;
290 size_t len;
291
292 uiov = u64_to_user_ptr(rw->addr);
293 if (!access_ok(uiov, sizeof(*uiov)))
294 return -EFAULT;
295 if (__get_user(clen, &uiov->iov_len))
296 return -EFAULT;
297 if (clen < 0)
298 return -EINVAL;
299
300 len = clen;
301 buf = io_buffer_select(req, &len, issue_flags);
302 if (!buf)
303 return -ENOBUFS;
304 rw->addr = (unsigned long) buf;
305 iov[0].iov_base = buf;
306 rw->len = iov[0].iov_len = (compat_size_t) len;
307 return 0;
308}
309#endif
310
311static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
312 unsigned int issue_flags)
313{
314 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
315 struct iovec __user *uiov = u64_to_user_ptr(rw->addr);
316 void __user *buf;
317 ssize_t len;
318
319 if (copy_from_user(iov, uiov, sizeof(*uiov)))
320 return -EFAULT;
321
322 len = iov[0].iov_len;
323 if (len < 0)
324 return -EINVAL;
325 buf = io_buffer_select(req, &len, issue_flags);
326 if (!buf)
327 return -ENOBUFS;
328 rw->addr = (unsigned long) buf;
329 iov[0].iov_base = buf;
330 rw->len = iov[0].iov_len = len;
331 return 0;
332}
333
334static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
335 unsigned int issue_flags)
336{
337 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
338
339 if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) {
340 iov[0].iov_base = u64_to_user_ptr(rw->addr);
341 iov[0].iov_len = rw->len;
342 return 0;
343 }
344 if (rw->len != 1)
345 return -EINVAL;
346
347#ifdef CONFIG_COMPAT
348 if (req->ctx->compat)
349 return io_compat_import(req, iov, issue_flags);
350#endif
351
352 return __io_iov_buffer_select(req, iov, issue_flags);
353}
354
355static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req,338static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req,
356 struct io_rw_state *s,339 struct io_rw_state *s,
357 unsigned int issue_flags)340 unsigned int issue_flags)
@@ -374,7 +357,8 @@ static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, · 16 unmodified lines
374 buf = u64_to_user_ptr(rw->addr);357 buf = u64_to_user_ptr(rw->addr);
375 sqe_len = rw->len;358 sqe_len = rw->len;
376 359
377 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {360 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE ||
361 (req->flags & REQ_F_BUFFER_SELECT)) {
378 if (io_do_buffer_select(req)) {362 if (io_do_buffer_select(req)) {
379 buf = io_buffer_select(req, &sqe_len, issue_flags);363 buf = io_buffer_select(req, &sqe_len, issue_flags);
380 if (!buf)364 if (!buf)
@@ -390,14 +374,6 @@ static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, · 9 unmodified lines
390 }374 }
391 375
392 iovec = s->fast_iov;376 iovec = s->fast_iov;
393 if (req->flags & REQ_F_BUFFER_SELECT) {
394 ret = io_iov_buffer_select(req, iovec, issue_flags);
395 if (ret)
396 return ERR_PTR(ret);
397 iov_iter_init(iter, ddir, iovec, 1, iovec->iov_len);
398 return NULL;
399 }
400
401 ret = __import_iovec(ddir, buf, sqe_len, UIO_FASTIOV, &iovec, iter,377 ret = __import_iovec(ddir, buf, sqe_len, UIO_FASTIOV, &iovec, iter,
402 req->ctx->compat);378 req->ctx->compat);
403 if (unlikely(ret < 0))379 if (unlikely(ret < 0))
@@ -794,10 +770,12 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) · 390 unmodified lines
794 iov_iter_restore(&s->iter, &s->iter_state);770 iov_iter_restore(&s->iter, &s->iter_state);
795 771
796 ret2 = io_setup_async_rw(req, iovec, s, true);772 ret2 = io_setup_async_rw(req, iovec, s, true);
797 if (ret2)
798 return ret2;
799
800 iovec = NULL;773 iovec = NULL;
774 if (ret2) {
775 ret = ret > 0 ? ret : ret2;
776 goto done;
777 }
778
801 io = req->async_data;779 io = req->async_data;
802 s = &io->s;780 s = &io->s;
803 /*781 /*
@@ -823,6 +801,7 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) · 19 unmodified lines
823 return -EAGAIN;801 return -EAGAIN;
824 }802 }
825 803
804 req->cqe.res = iov_iter_count(&s->iter);
826 /*805 /*
827 * Now retry read with the IOCB_WAITQ parts set in the iocb. If806 * Now retry read with the IOCB_WAITQ parts set in the iocb. If
828 * we get -EIOCBQUEUED, then we'll get a notification when the807 * we get -EIOCBQUEUED, then we'll get a notification when the
@@ -984,6 +963,14 @@ static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx) · 155 unmodified lines
984 io_cqring_wake(ctx);963 io_cqring_wake(ctx);
985}964}
986 965
966void io_rw_fail(struct io_kiocb *req)
967{
968 int res;
969
970 res = io_fixup_rw_res(req, req->cqe.res);
971 io_req_set_res(req, res, req->cqe.flags);
972}
973
987int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)974int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)
988{975{
989 struct io_wq_work_node *pos, *start, *prev;976 struct io_wq_work_node *pos, *start, *prev;
@@ -1000,7 +987,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) · 10 unmodified lines
1000 987
1001 wq_list_for_each(pos, start, &ctx->iopoll_list) {988 wq_list_for_each(pos, start, &ctx->iopoll_list) {
1002 struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list);989 struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list);
1003 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);990 struct file *file = req->file;
1004 int ret;991 int ret;
1005 992
1006 /*993 /*
@@ -1011,7 +998,17 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) · 4 unmodified lines
1011 if (READ_ONCE(req->iopoll_completed))998 if (READ_ONCE(req->iopoll_completed))
1012 break;999 break;
1013 1000
1014 ret = rw->kiocb.ki_filp->f_op->iopoll(&rw->kiocb, &iob, poll_flags);1001 if (req->opcode == IORING_OP_URING_CMD) {
1002 struct io_uring_cmd *ioucmd;
1003
1004 ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
1005 ret = file->f_op->uring_cmd_iopoll(ioucmd, &iob,
1006 poll_flags);
1007 } else {
1008 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
1009
1010 ret = file->f_op->iopoll(&rw->kiocb, &iob, poll_flags);
1011 }
1015 if (unlikely(ret < 0))1012 if (unlikely(ret < 0))
1016 return ret;1013 return ret;
1017 else if (ret)1014 else if (ret)
io_uring/rw.h+1−0
@@ -21,3 +21,4 @@ int io_readv_prep_async(struct io_kiocb *req); · 20 unmodified lines
21int io_write(struct io_kiocb *req, unsigned int issue_flags);21int io_write(struct io_kiocb *req, unsigned int issue_flags);
22int io_writev_prep_async(struct io_kiocb *req);22int io_writev_prep_async(struct io_kiocb *req);
23void io_readv_writev_cleanup(struct io_kiocb *req);23void io_readv_writev_cleanup(struct io_kiocb *req);
24void io_rw_fail(struct io_kiocb *req);
io_uring/timeout.c+3−10
@@ -149,11 +149,10 @@ static inline void io_remove_next_linked(struct io_kiocb *req) · 148 unmodified lines
149 nxt->link = NULL;149 nxt->link = NULL;
150}150}
151 151
152bool io_disarm_next(struct io_kiocb *req)152void io_disarm_next(struct io_kiocb *req)
153 __must_hold(&req->ctx->completion_lock)153 __must_hold(&req->ctx->completion_lock)
154{154{
155 struct io_kiocb *link = NULL;155 struct io_kiocb *link = NULL;
156 bool posted = false;
157 156
158 if (req->flags & REQ_F_ARM_LTIMEOUT) {157 if (req->flags & REQ_F_ARM_LTIMEOUT) {
159 link = req->link;158 link = req->link;
@@ -161,7 +160,6 @@ bool io_disarm_next(struct io_kiocb *req) · 1 unmodified line
161 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {160 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
162 io_remove_next_linked(req);161 io_remove_next_linked(req);
163 io_req_tw_post_queue(link, -ECANCELED, 0);162 io_req_tw_post_queue(link, -ECANCELED, 0);
164 posted = true;
165 }163 }
166 } else if (req->flags & REQ_F_LINK_TIMEOUT) {164 } else if (req->flags & REQ_F_LINK_TIMEOUT) {
167 struct io_ring_ctx *ctx = req->ctx;165 struct io_ring_ctx *ctx = req->ctx;
@@ -169,17 +167,12 @@ bool io_disarm_next(struct io_kiocb *req) · 1 unmodified line
169 spin_lock_irq(&ctx->timeout_lock);167 spin_lock_irq(&ctx->timeout_lock);
170 link = io_disarm_linked_timeout(req);168 link = io_disarm_linked_timeout(req);
171 spin_unlock_irq(&ctx->timeout_lock);169 spin_unlock_irq(&ctx->timeout_lock);
172 if (link) {170 if (link)
173 posted = true;
174 io_req_tw_post_queue(link, -ECANCELED, 0);171 io_req_tw_post_queue(link, -ECANCELED, 0);
175 }
176 }172 }
177 if (unlikely((req->flags & REQ_F_FAIL) &&173 if (unlikely((req->flags & REQ_F_FAIL) &&
178 !(req->flags & REQ_F_HARDLINK))) {174 !(req->flags & REQ_F_HARDLINK)))
179 posted |= (req->link != NULL);
180 io_fail_links(req);175 io_fail_links(req);
181 }
182 return posted;
183}176}
184 177
185struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req,178struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req,
io_uring/timeout.h+1−1
@@ -27,7 +27,7 @@ int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd); · 26 unmodified lines
27__cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,27__cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
28 bool cancel_all);28 bool cancel_all);
29void io_queue_linked_timeout(struct io_kiocb *req);29void io_queue_linked_timeout(struct io_kiocb *req);
30bool io_disarm_next(struct io_kiocb *req);30void io_disarm_next(struct io_kiocb *req);
31 31
32int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);32int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
33int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);33int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
io_uring/uring_cmd.c+9−2
@@ -50,7 +50,11 @@ void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2) · 49 unmodified lines
50 io_req_set_res(req, ret, 0);50 io_req_set_res(req, ret, 0);
51 if (req->ctx->flags & IORING_SETUP_CQE32)51 if (req->ctx->flags & IORING_SETUP_CQE32)
52 io_req_set_cqe32_extra(req, res2, 0);52 io_req_set_cqe32_extra(req, res2, 0);
53 __io_req_complete(req, 0);53 if (req->ctx->flags & IORING_SETUP_IOPOLL)
54 /* order with io_iopoll_req_issued() checking ->iopoll_complete */
55 smp_store_release(&req->iopoll_completed, 1);
56 else
57 __io_req_complete(req, 0);
54}58}
55EXPORT_SYMBOL_GPL(io_uring_cmd_done);59EXPORT_SYMBOL_GPL(io_uring_cmd_done);
56 60
@@ -97,8 +101,11 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) · 40 unmodified lines
97 issue_flags |= IO_URING_F_SQE128;101 issue_flags |= IO_URING_F_SQE128;
98 if (ctx->flags & IORING_SETUP_CQE32)102 if (ctx->flags & IORING_SETUP_CQE32)
99 issue_flags |= IO_URING_F_CQE32;103 issue_flags |= IO_URING_F_CQE32;
100 if (ctx->flags & IORING_SETUP_IOPOLL)104 if (ctx->flags & IORING_SETUP_IOPOLL) {
101 issue_flags |= IO_URING_F_IOPOLL;105 issue_flags |= IO_URING_F_IOPOLL;
106 req->iopoll_completed = 0;
107 WRITE_ONCE(ioucmd->cookie, NULL);
108 }
102 109
103 if (req_has_async_data(req))110 if (req_has_async_data(req))
104 ioucmd->cmd = req->async_data;111 ioucmd->cmd = req->async_data;
tools/testing/selftests/net/io_uring_zerocopy_tx.c+13−9
@@ -400,7 +400,6 @@ static void do_tx(int domain, int type, int protocol) · 399 unmodified lines
400 cfg_payload_len, msg_flags);400 cfg_payload_len, msg_flags);
401 sqe->user_data = NONZC_TAG;401 sqe->user_data = NONZC_TAG;
402 } else {402 } else {
403 compl_cqes++;
404 io_uring_prep_sendzc(sqe, fd, payload,403 io_uring_prep_sendzc(sqe, fd, payload,
405 cfg_payload_len,404 cfg_payload_len,
406 msg_flags, zc_flags);405 msg_flags, zc_flags);
@@ -430,18 +429,23 @@ static void do_tx(int domain, int type, int protocol) · 23 unmodified lines
430 if (cqe->flags & IORING_CQE_F_NOTIF) {429 if (cqe->flags & IORING_CQE_F_NOTIF) {
431 if (cqe->flags & IORING_CQE_F_MORE)430 if (cqe->flags & IORING_CQE_F_MORE)
432 error(1, -EINVAL, "invalid notif flags");431 error(1, -EINVAL, "invalid notif flags");
432 if (compl_cqes <= 0)
433 error(1, -EINVAL, "notification mismatch");
433 compl_cqes--;434 compl_cqes--;
434 i--;435 i--;
435 } else if (cqe->res <= 0) {436 io_uring_cqe_seen(&ring);
436 if (cqe->flags & IORING_CQE_F_MORE)437 continue;
437 error(1, cqe->res, "more with a failed send");438 }
438 error(1, cqe->res, "send failed");439 if (cqe->flags & IORING_CQE_F_MORE) {
439 } else {440 if (cqe->user_data != ZC_TAG)
440 if (cqe->user_data == ZC_TAG &&441 error(1, cqe->res, "unexpected F_MORE");
441 !(cqe->flags & IORING_CQE_F_MORE))442 compl_cqes++;
442 error(1, cqe->res, "missing more flag");443 }
444 if (cqe->res >= 0) {
443 packets++;445 packets++;
444 bytes += cqe->res;446 bytes += cqe->res;
447 } else if (cqe->res != -EAGAIN) {
448 error(1, cqe->res, "send failed");
445 }449 }
446 io_uring_cqe_seen(&ring);450 io_uring_cqe_seen(&ring);
447 }451 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t