Pull io_uring updates from Jens Axboe: - Add supported for more directly managed task_work running. This is beneficial for real world applications that end up issuing lots of system calls as part of handling work. Normal task_work will always execute as we transition in and out of the kernel, even for "unrelated" system calls. It's more efficient to defer the handling of io_uring's deferred work until the application wants it to be run, generally in batches. As part of ongoing work to write an io_uring network backend for Thrift, this has been shown to greatly improve performance. (Dylan) - Add IOPOLL support for passthrough (Kanchan) - Improvements and fixes to the send zero-copy support (Pavel) - Partial IO handling fixes (Pavel) - CQE ordering fixes around CQ ring overflow (Pavel) - Support sendto() for non-zc as well (Pavel) - Support sendmsg for zerocopy (Pavel) - Networking iov_iter fix (Stefan) - Misc fixes and cleanups (Pavel, me) * tag 'for-6.1/io_uring-2022-10-03' of git://git.kernel.dk/linux: (56 commits) io_uring/net: fix notif cqe reordering io_uring/net: don't update msg_name if not provided io_uring: don't gate task_work run on TIF_NOTIFY_SIGNAL io_uring/rw: defer fsnotify calls to task context io_uring/net: fix fast_iov assignment in io_setup_async_msg() io_uring/net: fix non-zc send with address io_uring/net: don't skip notifs for failed requests io_uring/rw: don't lose short results on io_setup_async_rw() io_uring/rw: fix unexpected link breakage io_uring/net: fix cleanup double free free_iov init io_uring: fix CQE reordering io_uring/net: fix UAF in io_sendrecv_fail() selftest/net: adjust io_uring sendzc notif handling io_uring: ensure local task_work marks task as running io_uring/net: zerocopy sendmsg io_uring/net: combine fail handlers io_uring/net: rename io_sendzc() io_uring/net: support non-zerocopy sendto io_uring/net: refactor io_setup_async_addr io_uring/net: don't lose partial send_zc on fail ...
Unified · Split
| @@ -1233,7 +1233,7 @@ static void blk_end_sync_rq(struct request *rq, blk_status_t ret) · 1232 unmodified lines | |||
| 1233 | complete(&wait->done); | 1233 | complete(&wait->done); |
| 1234 | } | 1234 | } |
| 1235 | 1235 | ||
| 1236 | static bool blk_rq_is_poll(struct request *rq) | 1236 | bool blk_rq_is_poll(struct request *rq) |
| 1237 | { | 1237 | { |
| 1238 | if (!rq->mq_hctx) | 1238 | if (!rq->mq_hctx) |
| 1239 | return false; | 1239 | return false; |
| @@ -1243,6 +1243,7 @@ static bool blk_rq_is_poll(struct request *rq) · 3 unmodified lines | |||
| 1243 | return false; | 1243 | return false; |
| 1244 | return true; | 1244 | return true; |
| 1245 | } | 1245 | } |
| 1246 | EXPORT_SYMBOL_GPL(blk_rq_is_poll); | ||
| 1246 | 1247 | ||
| 1247 | static void blk_rq_poll_completion(struct request *rq, struct completion *wait) | 1248 | static void blk_rq_poll_completion(struct request *rq, struct completion *wait) |
| 1248 | { | 1249 | { |
| @@ -3976,6 +3976,7 @@ static const struct file_operations nvme_ns_chr_fops = { · 3975 unmodified lines | |||
| 3976 | .unlocked_ioctl = nvme_ns_chr_ioctl, | 3976 | .unlocked_ioctl = nvme_ns_chr_ioctl, |
| 3977 | .compat_ioctl = compat_ptr_ioctl, | 3977 | .compat_ioctl = compat_ptr_ioctl, |
| 3978 | .uring_cmd = nvme_ns_chr_uring_cmd, | 3978 | .uring_cmd = nvme_ns_chr_uring_cmd, |
| 3979 | .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll, | ||
| 3979 | }; | 3980 | }; |
| 3980 | 3981 | ||
| 3981 | static int nvme_add_ns_cdev(struct nvme_ns *ns) | 3982 | static int nvme_add_ns_cdev(struct nvme_ns *ns) |
| @@ -391,11 +391,19 @@ static void nvme_uring_cmd_end_io(struct request *req, blk_status_t err) · 390 unmodified lines | |||
| 391 | struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); | 391 | struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); |
| 392 | /* extract bio before reusing the same field for request */ | 392 | /* extract bio before reusing the same field for request */ |
| 393 | struct bio *bio = pdu->bio; | 393 | struct bio *bio = pdu->bio; |
| 394 | void *cookie = READ_ONCE(ioucmd->cookie); | ||
| 394 | 395 | ||
| 395 | pdu->req = req; | 396 | pdu->req = req; |
| 396 | req->bio = bio; | 397 | req->bio = bio; |
| 397 | /* this takes care of moving rest of completion-work to task context */ | 398 | |
| 398 | io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb); | 399 | /* |
| 400 | * For iopoll, complete it directly. | ||
| 401 | * Otherwise, move the completion to task work. | ||
| 402 | */ | ||
| 403 | if (cookie != NULL && blk_rq_is_poll(req)) | ||
| 404 | nvme_uring_task_cb(ioucmd); | ||
| 405 | else | ||
| 406 | io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb); | ||
| 399 | } | 407 | } |
| 400 | 408 | ||
| 401 | static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, | 409 | static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, |
| @@ -445,7 +453,10 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, · 43 unmodified lines | |||
| 445 | rq_flags = REQ_NOWAIT; | 453 | rq_flags = REQ_NOWAIT; |
| 446 | blk_flags = BLK_MQ_REQ_NOWAIT; | 454 | blk_flags = BLK_MQ_REQ_NOWAIT; |
| 447 | } | 455 | } |
| 456 | if (issue_flags & IO_URING_F_IOPOLL) | ||
| 457 | rq_flags |= REQ_POLLED; | ||
| 448 | 458 | ||
| 459 | retry: | ||
| 449 | req = nvme_alloc_user_request(q, &c, nvme_to_user_ptr(d.addr), | 460 | req = nvme_alloc_user_request(q, &c, nvme_to_user_ptr(d.addr), |
| 450 | d.data_len, nvme_to_user_ptr(d.metadata), | 461 | d.data_len, nvme_to_user_ptr(d.metadata), |
| 451 | d.metadata_len, 0, &meta, d.timeout_ms ? | 462 | d.metadata_len, 0, &meta, d.timeout_ms ? |
| @@ -456,6 +467,17 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, · 4 unmodified lines | |||
| 456 | req->end_io = nvme_uring_cmd_end_io; | 467 | req->end_io = nvme_uring_cmd_end_io; |
| 457 | req->end_io_data = ioucmd; | 468 | req->end_io_data = ioucmd; |
| 458 | 469 | ||
| 470 | if (issue_flags & IO_URING_F_IOPOLL && rq_flags & REQ_POLLED) { | ||
| 471 | if (unlikely(!req->bio)) { | ||
| 472 | /* we can't poll this, so alloc regular req instead */ | ||
| 473 | blk_mq_free_request(req); | ||
| 474 | rq_flags &= ~REQ_POLLED; | ||
| 475 | goto retry; | ||
| 476 | } else { | ||
| 477 | WRITE_ONCE(ioucmd->cookie, req->bio); | ||
| 478 | req->bio->bi_opf |= REQ_POLLED; | ||
| 479 | } | ||
| 480 | } | ||
| 459 | /* to free bio on completion, as req->bio will be null at that time */ | 481 | /* to free bio on completion, as req->bio will be null at that time */ |
| 460 | pdu->bio = req->bio; | 482 | pdu->bio = req->bio; |
| 461 | pdu->meta = meta; | 483 | pdu->meta = meta; |
| @@ -559,9 +581,6 @@ long nvme_ns_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg) · 97 unmodified lines | |||
| 559 | 581 | ||
| 560 | static int nvme_uring_cmd_checks(unsigned int issue_flags) | 582 | static int nvme_uring_cmd_checks(unsigned int issue_flags) |
| 561 | { | 583 | { |
| 562 | /* IOPOLL not supported yet */ | ||
| 563 | if (issue_flags & IO_URING_F_IOPOLL) | ||
| 564 | return -EOPNOTSUPP; | ||
| 565 | 584 | ||
| 566 | /* NVMe passthrough requires big SQE/CQE support */ | 585 | /* NVMe passthrough requires big SQE/CQE support */ |
| 567 | if ((issue_flags & (IO_URING_F_SQE128|IO_URING_F_CQE32)) != | 586 | if ((issue_flags & (IO_URING_F_SQE128|IO_URING_F_CQE32)) != |
| @@ -604,6 +623,25 @@ int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags) · 36 unmodified lines | |||
| 604 | return nvme_ns_uring_cmd(ns, ioucmd, issue_flags); | 623 | return nvme_ns_uring_cmd(ns, ioucmd, issue_flags); |
| 605 | } | 624 | } |
| 606 | 625 | ||
| 626 | int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd, | ||
| 627 | struct io_comp_batch *iob, | ||
| 628 | unsigned int poll_flags) | ||
| 629 | { | ||
| 630 | struct bio *bio; | ||
| 631 | int ret = 0; | ||
| 632 | struct nvme_ns *ns; | ||
| 633 | struct request_queue *q; | ||
| 634 | |||
| 635 | rcu_read_lock(); | ||
| 636 | bio = READ_ONCE(ioucmd->cookie); | ||
| 637 | ns = container_of(file_inode(ioucmd->file)->i_cdev, | ||
| 638 | struct nvme_ns, cdev); | ||
| 639 | q = ns->queue; | ||
| 640 | if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio && bio->bi_bdev) | ||
| 641 | ret = bio_poll(bio, iob, poll_flags); | ||
| 642 | rcu_read_unlock(); | ||
| 643 | return ret; | ||
| 644 | } | ||
| 607 | #ifdef CONFIG_NVME_MULTIPATH | 645 | #ifdef CONFIG_NVME_MULTIPATH |
| 608 | static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd, | 646 | static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd, |
| 609 | void __user *argp, struct nvme_ns_head *head, int srcu_idx) | 647 | void __user *argp, struct nvme_ns_head *head, int srcu_idx) |
| @@ -685,6 +723,31 @@ int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd, · 75 unmodified lines | |||
| 685 | srcu_read_unlock(&head->srcu, srcu_idx); | 723 | srcu_read_unlock(&head->srcu, srcu_idx); |
| 686 | return ret; | 724 | return ret; |
| 687 | } | 725 | } |
| 726 | |||
| 727 | int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd, | ||
| 728 | struct io_comp_batch *iob, | ||
| 729 | unsigned int poll_flags) | ||
| 730 | { | ||
| 731 | struct cdev *cdev = file_inode(ioucmd->file)->i_cdev; | ||
| 732 | struct nvme_ns_head *head = container_of(cdev, struct nvme_ns_head, cdev); | ||
| 733 | int srcu_idx = srcu_read_lock(&head->srcu); | ||
| 734 | struct nvme_ns *ns = nvme_find_path(head); | ||
| 735 | struct bio *bio; | ||
| 736 | int ret = 0; | ||
| 737 | struct request_queue *q; | ||
| 738 | |||
| 739 | if (ns) { | ||
| 740 | rcu_read_lock(); | ||
| 741 | bio = READ_ONCE(ioucmd->cookie); | ||
| 742 | q = ns->queue; | ||
| 743 | if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio | ||
| 744 | && bio->bi_bdev) | ||
| 745 | ret = bio_poll(bio, iob, poll_flags); | ||
| 746 | rcu_read_unlock(); | ||
| 747 | } | ||
| 748 | srcu_read_unlock(&head->srcu, srcu_idx); | ||
| 749 | return ret; | ||
| 750 | } | ||
| 688 | #endif /* CONFIG_NVME_MULTIPATH */ | 751 | #endif /* CONFIG_NVME_MULTIPATH */ |
| 689 | 752 | ||
| 690 | int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags) | 753 | int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags) |
| @@ -692,6 +755,10 @@ int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags) · 1 unmodified line | |||
| 692 | struct nvme_ctrl *ctrl = ioucmd->file->private_data; | 755 | struct nvme_ctrl *ctrl = ioucmd->file->private_data; |
| 693 | int ret; | 756 | int ret; |
| 694 | 757 | ||
| 758 | /* IOPOLL not supported yet */ | ||
| 759 | if (issue_flags & IO_URING_F_IOPOLL) | ||
| 760 | return -EOPNOTSUPP; | ||
| 761 | |||
| 695 | ret = nvme_uring_cmd_checks(issue_flags); | 762 | ret = nvme_uring_cmd_checks(issue_flags); |
| 696 | if (ret) | 763 | if (ret) |
| 697 | return ret; | 764 | return ret; |
| @@ -439,6 +439,7 @@ static const struct file_operations nvme_ns_head_chr_fops = { · 438 unmodified lines | |||
| 439 | .unlocked_ioctl = nvme_ns_head_chr_ioctl, | 439 | .unlocked_ioctl = nvme_ns_head_chr_ioctl, |
| 440 | .compat_ioctl = compat_ptr_ioctl, | 440 | .compat_ioctl = compat_ptr_ioctl, |
| 441 | .uring_cmd = nvme_ns_head_chr_uring_cmd, | 441 | .uring_cmd = nvme_ns_head_chr_uring_cmd, |
| 442 | .uring_cmd_iopoll = nvme_ns_head_chr_uring_cmd_iopoll, | ||
| 442 | }; | 443 | }; |
| 443 | 444 | ||
| 444 | static int nvme_add_ns_head_cdev(struct nvme_ns_head *head) | 445 | static int nvme_add_ns_head_cdev(struct nvme_ns_head *head) |
| @@ -821,6 +821,10 @@ long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd, · 820 unmodified lines | |||
| 821 | unsigned long arg); | 821 | unsigned long arg); |
| 822 | long nvme_dev_ioctl(struct file *file, unsigned int cmd, | 822 | long nvme_dev_ioctl(struct file *file, unsigned int cmd, |
| 823 | unsigned long arg); | 823 | unsigned long arg); |
| 824 | int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd, | ||
| 825 | struct io_comp_batch *iob, unsigned int poll_flags); | ||
| 826 | int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd, | ||
| 827 | struct io_comp_batch *iob, unsigned int poll_flags); | ||
| 824 | int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, | 828 | int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, |
| 825 | unsigned int issue_flags); | 829 | unsigned int issue_flags); |
| 826 | int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd, | 830 | int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd, |
| @@ -69,17 +69,17 @@ __u64 eventfd_signal(struct eventfd_ctx *ctx, __u64 n) · 68 unmodified lines | |||
| 69 | * it returns false, the eventfd_signal() call should be deferred to a | 69 | * it returns false, the eventfd_signal() call should be deferred to a |
| 70 | * safe context. | 70 | * safe context. |
| 71 | */ | 71 | */ |
| 72 | if (WARN_ON_ONCE(current->in_eventfd_signal)) | 72 | if (WARN_ON_ONCE(current->in_eventfd)) |
| 73 | return 0; | 73 | return 0; |
| 74 | 74 | ||
| 75 | spin_lock_irqsave(&ctx->wqh.lock, flags); | 75 | spin_lock_irqsave(&ctx->wqh.lock, flags); |
| 76 | current->in_eventfd_signal = 1; | 76 | current->in_eventfd = 1; |
| 77 | if (ULLONG_MAX - ctx->count < n) | 77 | if (ULLONG_MAX - ctx->count < n) |
| 78 | n = ULLONG_MAX - ctx->count; | 78 | n = ULLONG_MAX - ctx->count; |
| 79 | ctx->count += n; | 79 | ctx->count += n; |
| 80 | if (waitqueue_active(&ctx->wqh)) | 80 | if (waitqueue_active(&ctx->wqh)) |
| 81 | wake_up_locked_poll(&ctx->wqh, EPOLLIN); | 81 | wake_up_locked_poll(&ctx->wqh, EPOLLIN); |
| 82 | current->in_eventfd_signal = 0; | 82 | current->in_eventfd = 0; |
| 83 | spin_unlock_irqrestore(&ctx->wqh.lock, flags); | 83 | spin_unlock_irqrestore(&ctx->wqh.lock, flags); |
| 84 | 84 | ||
| 85 | return n; | 85 | return n; |
| @@ -253,8 +253,10 @@ static ssize_t eventfd_read(struct kiocb *iocb, struct iov_iter *to) · 167 unmodified lines | |||
| 253 | __set_current_state(TASK_RUNNING); | 253 | __set_current_state(TASK_RUNNING); |
| 254 | } | 254 | } |
| 255 | eventfd_ctx_do_read(ctx, &ucnt); | 255 | eventfd_ctx_do_read(ctx, &ucnt); |
| 256 | current->in_eventfd = 1; | ||
| 256 | if (waitqueue_active(&ctx->wqh)) | 257 | if (waitqueue_active(&ctx->wqh)) |
| 257 | wake_up_locked_poll(&ctx->wqh, EPOLLOUT); | 258 | wake_up_locked_poll(&ctx->wqh, EPOLLOUT); |
| 259 | current->in_eventfd = 0; | ||
| 258 | spin_unlock_irq(&ctx->wqh.lock); | 260 | spin_unlock_irq(&ctx->wqh.lock); |
| 259 | if (unlikely(copy_to_iter(&ucnt, sizeof(ucnt), to) != sizeof(ucnt))) | 261 | if (unlikely(copy_to_iter(&ucnt, sizeof(ucnt), to) != sizeof(ucnt))) |
| 260 | return -EFAULT; | 262 | return -EFAULT; |
| @@ -301,8 +303,10 @@ static ssize_t eventfd_write(struct file *file, const char __user *buf, size_t c · 40 unmodified lines | |||
| 301 | } | 303 | } |
| 302 | if (likely(res > 0)) { | 304 | if (likely(res > 0)) { |
| 303 | ctx->count += ucnt; | 305 | ctx->count += ucnt; |
| 306 | current->in_eventfd = 1; | ||
| 304 | if (waitqueue_active(&ctx->wqh)) | 307 | if (waitqueue_active(&ctx->wqh)) |
| 305 | wake_up_locked_poll(&ctx->wqh, EPOLLIN); | 308 | wake_up_locked_poll(&ctx->wqh, EPOLLIN); |
| 309 | current->in_eventfd = 0; | ||
| 306 | } | 310 | } |
| 307 | spin_unlock_irq(&ctx->wqh.lock); | 311 | spin_unlock_irq(&ctx->wqh.lock); |
| 308 | 312 | ||
| @@ -980,6 +980,7 @@ int blk_rq_map_kern(struct request_queue *, struct request *, void *, · 979 unmodified lines | |||
| 980 | int blk_rq_append_bio(struct request *rq, struct bio *bio); | 980 | int blk_rq_append_bio(struct request *rq, struct bio *bio); |
| 981 | void blk_execute_rq_nowait(struct request *rq, bool at_head); | 981 | void blk_execute_rq_nowait(struct request *rq, bool at_head); |
| 982 | blk_status_t blk_execute_rq(struct request *rq, bool at_head); | 982 | blk_status_t blk_execute_rq(struct request *rq, bool at_head); |
| 983 | bool blk_rq_is_poll(struct request *rq); | ||
| 983 | 984 | ||
| 984 | struct req_iterator { | 985 | struct req_iterator { |
| 985 | struct bvec_iter iter; | 986 | struct bvec_iter iter; |
| @@ -46,7 +46,7 @@ void eventfd_ctx_do_read(struct eventfd_ctx *ctx, __u64 *cnt); · 45 unmodified lines | |||
| 46 | 46 | ||
| 47 | static inline bool eventfd_signal_allowed(void) | 47 | static inline bool eventfd_signal_allowed(void) |
| 48 | { | 48 | { |
| 49 | return !current->in_eventfd_signal; | 49 | return !current->in_eventfd; |
| 50 | } | 50 | } |
| 51 | 51 | ||
| 52 | #else /* CONFIG_EVENTFD */ | 52 | #else /* CONFIG_EVENTFD */ |
| @@ -2133,6 +2133,8 @@ struct file_operations { · 2132 unmodified lines | |||
| 2133 | loff_t len, unsigned int remap_flags); | 2133 | loff_t len, unsigned int remap_flags); |
| 2134 | int (*fadvise)(struct file *, loff_t, loff_t, int); | 2134 | int (*fadvise)(struct file *, loff_t, loff_t, int); |
| 2135 | int (*uring_cmd)(struct io_uring_cmd *ioucmd, unsigned int issue_flags); | 2135 | int (*uring_cmd)(struct io_uring_cmd *ioucmd, unsigned int issue_flags); |
| 2136 | int (*uring_cmd_iopoll)(struct io_uring_cmd *, struct io_comp_batch *, | ||
| 2137 | unsigned int poll_flags); | ||
| 2136 | } __randomize_layout; | 2138 | } __randomize_layout; |
| 2137 | 2139 | ||
| 2138 | struct inode_operations { | 2140 | struct inode_operations { |
| @@ -20,8 +20,12 @@ enum io_uring_cmd_flags { · 19 unmodified lines | |||
| 20 | struct io_uring_cmd { | 20 | struct io_uring_cmd { |
| 21 | struct file *file; | 21 | struct file *file; |
| 22 | const void *cmd; | 22 | const void *cmd; |
| 23 | /* callback to defer completions to task context */ | 23 | union { |
| 24 | void (*task_work_cb)(struct io_uring_cmd *cmd); | 24 | /* callback to defer completions to task context */ |
| 25 | void (*task_work_cb)(struct io_uring_cmd *cmd); | ||
| 26 | /* used for polled completion */ | ||
| 27 | void *cookie; | ||
| 28 | }; | ||
| 25 | u32 cmd_op; | 29 | u32 cmd_op; |
| 26 | u32 pad; | 30 | u32 pad; |
| 27 | u8 pdu[32]; /* available inline for free use */ | 31 | u8 pdu[32]; /* available inline for free use */ |
| @@ -184,6 +184,8 @@ struct io_ev_fd { · 183 unmodified lines | |||
| 184 | struct eventfd_ctx *cq_ev_fd; | 184 | struct eventfd_ctx *cq_ev_fd; |
| 185 | unsigned int eventfd_async: 1; | 185 | unsigned int eventfd_async: 1; |
| 186 | struct rcu_head rcu; | 186 | struct rcu_head rcu; |
| 187 | atomic_t refs; | ||
| 188 | atomic_t ops; | ||
| 187 | }; | 189 | }; |
| 188 | 190 | ||
| 189 | struct io_alloc_cache { | 191 | struct io_alloc_cache { |
| @@ -301,6 +303,8 @@ struct io_ring_ctx { · 111 unmodified lines | |||
| 301 | struct io_hash_table cancel_table; | 303 | struct io_hash_table cancel_table; |
| 302 | bool poll_multi_queue; | 304 | bool poll_multi_queue; |
| 303 | 305 | ||
| 306 | struct llist_head work_llist; | ||
| 307 | |||
| 304 | struct list_head io_buffers_comp; | 308 | struct list_head io_buffers_comp; |
| 305 | } ____cacheline_aligned_in_smp; | 309 | } ____cacheline_aligned_in_smp; |
| 306 | 310 | ||
| @@ -936,7 +936,7 @@ struct task_struct { · 935 unmodified lines | |||
| 936 | #endif | 936 | #endif |
| 937 | #ifdef CONFIG_EVENTFD | 937 | #ifdef CONFIG_EVENTFD |
| 938 | /* Recursion prevention for eventfd_signal() */ | 938 | /* Recursion prevention for eventfd_signal() */ |
| 939 | unsigned in_eventfd_signal:1; | 939 | unsigned in_eventfd:1; |
| 940 | #endif | 940 | #endif |
| 941 | #ifdef CONFIG_IOMMU_SVA | 941 | #ifdef CONFIG_IOMMU_SVA |
| 942 | unsigned pasid_activated:1; | 942 | unsigned pasid_activated:1; |
| @@ -655,6 +655,35 @@ TRACE_EVENT(io_uring_short_write, · 654 unmodified lines | |||
| 655 | __entry->wanted, __entry->got) | 655 | __entry->wanted, __entry->got) |
| 656 | ); | 656 | ); |
| 657 | 657 | ||
| 658 | /* | ||
| 659 | * io_uring_local_work_run - ran ring local task work | ||
| 660 | * | ||
| 661 | * @tctx: pointer to a io_uring_ctx | ||
| 662 | * @count: how many functions it ran | ||
| 663 | * @loops: how many loops it ran | ||
| 664 | * | ||
| 665 | */ | ||
| 666 | TRACE_EVENT(io_uring_local_work_run, | ||
| 667 | |||
| 668 | TP_PROTO(void *ctx, int count, unsigned int loops), | ||
| 669 | |||
| 670 | TP_ARGS(ctx, count, loops), | ||
| 671 | |||
| 672 | TP_STRUCT__entry ( | ||
| 673 | __field(void *, ctx ) | ||
| 674 | __field(int, count ) | ||
| 675 | __field(unsigned int, loops ) | ||
| 676 | ), | ||
| 677 | |||
| 678 | TP_fast_assign( | ||
| 679 | __entry->ctx = ctx; | ||
| 680 | __entry->count = count; | ||
| 681 | __entry->loops = loops; | ||
| 682 | ), | ||
| 683 | |||
| 684 | TP_printk("ring %p, count %d, loops %u", __entry->ctx, __entry->count, __entry->loops) | ||
| 685 | ); | ||
| 686 | |||
| 658 | #endif /* _TRACE_IO_URING_H */ | 687 | #endif /* _TRACE_IO_URING_H */ |
| 659 | 688 | ||
| 660 | /* This part must be outside protection */ | 689 | /* This part must be outside protection */ |
| @@ -157,6 +157,13 @@ enum { · 156 unmodified lines | |||
| 157 | */ | 157 | */ |
| 158 | #define IORING_SETUP_SINGLE_ISSUER (1U << 12) | 158 | #define IORING_SETUP_SINGLE_ISSUER (1U << 12) |
| 159 | 159 | ||
| 160 | /* | ||
| 161 | * Defer running task work to get events. | ||
| 162 | * Rather than running bits of task work whenever the task transitions | ||
| 163 | * try to do it just before it is needed. | ||
| 164 | */ | ||
| 165 | #define IORING_SETUP_DEFER_TASKRUN (1U << 13) | ||
| 166 | |||
| 160 | enum io_uring_op { | 167 | enum io_uring_op { |
| 161 | IORING_OP_NOP, | 168 | IORING_OP_NOP, |
| 162 | IORING_OP_READV, | 169 | IORING_OP_READV, |
| @@ -206,6 +213,7 @@ enum io_uring_op { · 43 unmodified lines | |||
| 206 | IORING_OP_SOCKET, | 213 | IORING_OP_SOCKET, |
| 207 | IORING_OP_URING_CMD, | 214 | IORING_OP_URING_CMD, |
| 208 | IORING_OP_SEND_ZC, | 215 | IORING_OP_SEND_ZC, |
| 216 | IORING_OP_SENDMSG_ZC, | ||
| 209 | 217 | ||
| 210 | /* this goes last, obviously */ | 218 | /* this goes last, obviously */ |
| 211 | IORING_OP_LAST, | 219 | IORING_OP_LAST, |
| @@ -292,7 +292,7 @@ int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg) · 291 unmodified lines | |||
| 292 | break; | 292 | break; |
| 293 | 293 | ||
| 294 | mutex_unlock(&ctx->uring_lock); | 294 | mutex_unlock(&ctx->uring_lock); |
| 295 | ret = io_run_task_work_sig(); | 295 | ret = io_run_task_work_sig(ctx); |
| 296 | if (ret < 0) { | 296 | if (ret < 0) { |
| 297 | mutex_lock(&ctx->uring_lock); | 297 | mutex_lock(&ctx->uring_lock); |
| 298 | break; | 298 | break; |
| @@ -60,13 +60,15 @@ static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, · 59 unmodified lines | |||
| 60 | unsigned int cq_head = READ_ONCE(r->cq.head); | 60 | unsigned int cq_head = READ_ONCE(r->cq.head); |
| 61 | unsigned int cq_tail = READ_ONCE(r->cq.tail); | 61 | unsigned int cq_tail = READ_ONCE(r->cq.tail); |
| 62 | unsigned int cq_shift = 0; | 62 | unsigned int cq_shift = 0; |
| 63 | unsigned int sq_shift = 0; | ||
| 63 | unsigned int sq_entries, cq_entries; | 64 | unsigned int sq_entries, cq_entries; |
| 64 | bool has_lock; | 65 | bool has_lock; |
| 65 | bool is_cqe32 = (ctx->flags & IORING_SETUP_CQE32); | ||
| 66 | unsigned int i; | 66 | unsigned int i; |
| 67 | 67 | ||
| 68 | if (is_cqe32) | 68 | if (ctx->flags & IORING_SETUP_CQE32) |
| 69 | cq_shift = 1; | 69 | cq_shift = 1; |
| 70 | if (ctx->flags & IORING_SETUP_SQE128) | ||
| 71 | sq_shift = 1; | ||
| 70 | 72 | ||
| 71 | /* | 73 | /* |
| 72 | * we may get imprecise sqe and cqe info if uring is actively running | 74 | * we may get imprecise sqe and cqe info if uring is actively running |
| @@ -82,19 +84,36 @@ static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, · 9 unmodified lines | |||
| 82 | seq_printf(m, "CqHead:\t%u\n", cq_head); | 84 | seq_printf(m, "CqHead:\t%u\n", cq_head); |
| 83 | seq_printf(m, "CqTail:\t%u\n", cq_tail); | 85 | seq_printf(m, "CqTail:\t%u\n", cq_tail); |
| 84 | seq_printf(m, "CachedCqTail:\t%u\n", ctx->cached_cq_tail); | 86 | seq_printf(m, "CachedCqTail:\t%u\n", ctx->cached_cq_tail); |
| 85 | seq_printf(m, "SQEs:\t%u\n", sq_tail - ctx->cached_sq_head); | 87 | seq_printf(m, "SQEs:\t%u\n", sq_tail - sq_head); |
| 86 | sq_entries = min(sq_tail - sq_head, ctx->sq_entries); | 88 | sq_entries = min(sq_tail - sq_head, ctx->sq_entries); |
| 87 | for (i = 0; i < sq_entries; i++) { | 89 | for (i = 0; i < sq_entries; i++) { |
| 88 | unsigned int entry = i + sq_head; | 90 | unsigned int entry = i + sq_head; |
| 89 | unsigned int sq_idx = READ_ONCE(ctx->sq_array[entry & sq_mask]); | ||
| 90 | struct io_uring_sqe *sqe; | 91 | struct io_uring_sqe *sqe; |
| 92 | unsigned int sq_idx; | ||
| 91 | 93 | ||
| 94 | sq_idx = READ_ONCE(ctx->sq_array[entry & sq_mask]); | ||
| 92 | if (sq_idx > sq_mask) | 95 | if (sq_idx > sq_mask) |
| 93 | continue; | 96 | continue; |
| 94 | sqe = &ctx->sq_sqes[sq_idx]; | 97 | sqe = &ctx->sq_sqes[sq_idx << 1]; |
| 95 | seq_printf(m, "%5u: opcode:%d, fd:%d, flags:%x, user_data:%llu\n", | 98 | seq_printf(m, "%5u: opcode:%s, fd:%d, flags:%x, off:%llu, " |
| 96 | sq_idx, sqe->opcode, sqe->fd, sqe->flags, | 99 | "addr:0x%llx, rw_flags:0x%x, buf_index:%d " |
| 97 | sqe->user_data); | 100 | "user_data:%llu", |
| 101 | sq_idx, io_uring_get_opcode(sqe->opcode), sqe->fd, | ||
| 102 | sqe->flags, (unsigned long long) sqe->off, | ||
| 103 | (unsigned long long) sqe->addr, sqe->rw_flags, | ||
| 104 | sqe->buf_index, sqe->user_data); | ||
| 105 | if (sq_shift) { | ||
| 106 | u64 *sqeb = (void *) (sqe + 1); | ||
| 107 | int size = sizeof(struct io_uring_sqe) / sizeof(u64); | ||
| 108 | int j; | ||
| 109 | |||
| 110 | for (j = 0; j < size; j++) { | ||
| 111 | seq_printf(m, ", e%d:0x%llx", j, | ||
| 112 | (unsigned long long) *sqeb); | ||
| 113 | sqeb++; | ||
| 114 | } | ||
| 115 | } | ||
| 116 | seq_printf(m, "\n"); | ||
| 98 | } | 117 | } |
| 99 | seq_printf(m, "CQEs:\t%u\n", cq_tail - cq_head); | 118 | seq_printf(m, "CQEs:\t%u\n", cq_tail - cq_head); |
| 100 | cq_entries = min(cq_tail - cq_head, ctx->cq_entries); | 119 | cq_entries = min(cq_tail - cq_head, ctx->cq_entries); |
| @@ -102,16 +121,13 @@ static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, · 1 unmodified line | |||
| 102 | unsigned int entry = i + cq_head; | 121 | unsigned int entry = i + cq_head; |
| 103 | struct io_uring_cqe *cqe = &r->cqes[(entry & cq_mask) << cq_shift]; | 122 | struct io_uring_cqe *cqe = &r->cqes[(entry & cq_mask) << cq_shift]; |
| 104 | 123 | ||
| 105 | if (!is_cqe32) { | 124 | seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x", |
| 106 | seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x\n", | ||
| 107 | entry & cq_mask, cqe->user_data, cqe->res, | 125 | entry & cq_mask, cqe->user_data, cqe->res, |
| 108 | cqe->flags); | 126 | cqe->flags); |
| 109 | } else { | 127 | if (cq_shift) |
| 110 | seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x, " | 128 | seq_printf(m, ", extra1:%llu, extra2:%llu\n", |
| 111 | "extra1:%llu, extra2:%llu\n", | 129 | cqe->big_cqe[0], cqe->big_cqe[1]); |
| 112 | entry & cq_mask, cqe->user_data, cqe->res, | 130 | seq_printf(m, "\n"); |
| 113 | cqe->flags, cqe->big_cqe[0], cqe->big_cqe[1]); | ||
| 114 | } | ||
| 115 | } | 131 | } |
| 116 | 132 | ||
| 117 | /* | 133 | /* |
| @@ -125,6 +125,11 @@ enum { · 124 unmodified lines | |||
| 125 | IO_CHECK_CQ_DROPPED_BIT, | 125 | IO_CHECK_CQ_DROPPED_BIT, |
| 126 | }; | 126 | }; |
| 127 | 127 | ||
| 128 | enum { | ||
| 129 | IO_EVENTFD_OP_SIGNAL_BIT, | ||
| 130 | IO_EVENTFD_OP_FREE_BIT, | ||
| 131 | }; | ||
| 132 | |||
| 128 | struct io_defer_entry { | 133 | struct io_defer_entry { |
| 129 | struct list_head list; | 134 | struct list_head list; |
| 130 | struct io_kiocb *req; | 135 | struct io_kiocb *req; |
| @@ -142,7 +147,7 @@ static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, · 11 unmodified lines | |||
| 142 | static void io_dismantle_req(struct io_kiocb *req); | 147 | static void io_dismantle_req(struct io_kiocb *req); |
| 143 | static void io_clean_op(struct io_kiocb *req); | 148 | static void io_clean_op(struct io_kiocb *req); |
| 144 | static void io_queue_sqe(struct io_kiocb *req); | 149 | static void io_queue_sqe(struct io_kiocb *req); |
| 145 | 150 | static void io_move_task_work_from_local(struct io_ring_ctx *ctx); | |
| 146 | static void __io_submit_flush_completions(struct io_ring_ctx *ctx); | 151 | static void __io_submit_flush_completions(struct io_ring_ctx *ctx); |
| 147 | 152 | ||
| 148 | static struct kmem_cache *req_cachep; | 153 | static struct kmem_cache *req_cachep; |
| @@ -316,6 +321,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) · 167 unmodified lines | |||
| 316 | INIT_LIST_HEAD(&ctx->rsrc_ref_list); | 321 | INIT_LIST_HEAD(&ctx->rsrc_ref_list); |
| 317 | INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); | 322 | INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); |
| 318 | init_llist_head(&ctx->rsrc_put_llist); | 323 | init_llist_head(&ctx->rsrc_put_llist); |
| 324 | init_llist_head(&ctx->work_llist); | ||
| 319 | INIT_LIST_HEAD(&ctx->tctx_list); | 325 | INIT_LIST_HEAD(&ctx->tctx_list); |
| 320 | ctx->submit_state.free_list.next = NULL; | 326 | ctx->submit_state.free_list.next = NULL; |
| 321 | INIT_WQ_LIST(&ctx->locked_free_list); | 327 | INIT_WQ_LIST(&ctx->locked_free_list); |
| @@ -477,25 +483,28 @@ static __cold void io_queue_deferred(struct io_ring_ctx *ctx) · 155 unmodified lines | |||
| 477 | } | 483 | } |
| 478 | } | 484 | } |
| 479 | 485 | ||
| 480 | static void io_eventfd_signal(struct io_ring_ctx *ctx) | 486 | |
| 487 | static void io_eventfd_ops(struct rcu_head *rcu) | ||
| 481 | { | 488 | { |
| 482 | struct io_ev_fd *ev_fd; | 489 | struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu); |
| 483 | bool skip; | 490 | int ops = atomic_xchg(&ev_fd->ops, 0); |
| 484 | 491 | ||
| 485 | spin_lock(&ctx->completion_lock); | 492 | if (ops & BIT(IO_EVENTFD_OP_SIGNAL_BIT)) |
| 486 | /* | 493 | eventfd_signal(ev_fd->cq_ev_fd, 1); |
| 487 | * Eventfd should only get triggered when at least one event has been | 494 | |
| 488 | * posted. Some applications rely on the eventfd notification count only | 495 | /* IO_EVENTFD_OP_FREE_BIT may not be set here depending on callback |
| 489 | * changing IFF a new CQE has been added to the CQ ring. There's no | 496 | * ordering in a race but if references are 0 we know we have to free |
| 490 | * depedency on 1:1 relationship between how many times this function is | 497 | * it regardless. |
| 491 | * called (and hence the eventfd count) and number of CQEs posted to the | ||
| 492 | * CQ ring. | ||
| 493 | */ | 498 | */ |
| 494 | skip = ctx->cached_cq_tail == ctx->evfd_last_cq_tail; | 499 | if (atomic_dec_and_test(&ev_fd->refs)) { |
| 495 | ctx->evfd_last_cq_tail = ctx->cached_cq_tail; | 500 | eventfd_ctx_put(ev_fd->cq_ev_fd); |
| 496 | spin_unlock(&ctx->completion_lock); | 501 | kfree(ev_fd); |
| 497 | if (skip) | 502 | } |
| 498 | return; | 503 | } |
| 504 | |||
| 505 | static void io_eventfd_signal(struct io_ring_ctx *ctx) | ||
| 506 | { | ||
| 507 | struct io_ev_fd *ev_fd = NULL; | ||
| 499 | 508 | ||
| 500 | rcu_read_lock(); | 509 | rcu_read_lock(); |
| 501 | /* | 510 | /* |
| @@ -513,13 +522,46 @@ static void io_eventfd_signal(struct io_ring_ctx *ctx) · 11 unmodified lines | |||
| 513 | goto out; | 522 | goto out; |
| 514 | if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED) | 523 | if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED) |
| 515 | goto out; | 524 | goto out; |
| 525 | if (ev_fd->eventfd_async && !io_wq_current_is_worker()) | ||
| 526 | goto out; | ||
| 516 | 527 | ||
| 517 | if (!ev_fd->eventfd_async || io_wq_current_is_worker()) | 528 | if (likely(eventfd_signal_allowed())) { |
| 518 | eventfd_signal(ev_fd->cq_ev_fd, 1); | 529 | eventfd_signal(ev_fd->cq_ev_fd, 1); |
| 530 | } else { | ||
| 531 | atomic_inc(&ev_fd->refs); | ||
| 532 | if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_SIGNAL_BIT), &ev_fd->ops)) | ||
| 533 | call_rcu(&ev_fd->rcu, io_eventfd_ops); | ||
| 534 | else | ||
| 535 | atomic_dec(&ev_fd->refs); | ||
| 536 | } | ||
| 537 | |||
| 519 | out: | 538 | out: |
| 520 | rcu_read_unlock(); | 539 | rcu_read_unlock(); |
| 521 | } | 540 | } |
| 522 | 541 | ||
| 542 | static void io_eventfd_flush_signal(struct io_ring_ctx *ctx) | ||
| 543 | { | ||
| 544 | bool skip; | ||
| 545 | |||
| 546 | spin_lock(&ctx->completion_lock); | ||
| 547 | |||
| 548 | /* | ||
| 549 | * Eventfd should only get triggered when at least one event has been | ||
| 550 | * posted. Some applications rely on the eventfd notification count | ||
| 551 | * only changing IFF a new CQE has been added to the CQ ring. There's | ||
| 552 | * no depedency on 1:1 relationship between how many times this | ||
| 553 | * function is called (and hence the eventfd count) and number of CQEs | ||
| 554 | * posted to the CQ ring. | ||
| 555 | */ | ||
| 556 | skip = ctx->cached_cq_tail == ctx->evfd_last_cq_tail; | ||
| 557 | ctx->evfd_last_cq_tail = ctx->cached_cq_tail; | ||
| 558 | spin_unlock(&ctx->completion_lock); | ||
| 559 | if (skip) | ||
| 560 | return; | ||
| 561 | |||
| 562 | io_eventfd_signal(ctx); | ||
| 563 | } | ||
| 564 | |||
| 523 | void __io_commit_cqring_flush(struct io_ring_ctx *ctx) | 565 | void __io_commit_cqring_flush(struct io_ring_ctx *ctx) |
| 524 | { | 566 | { |
| 525 | if (ctx->off_timeout_used || ctx->drain_active) { | 567 | if (ctx->off_timeout_used || ctx->drain_active) { |
| @@ -531,7 +573,7 @@ void __io_commit_cqring_flush(struct io_ring_ctx *ctx) · 5 unmodified lines | |||
| 531 | spin_unlock(&ctx->completion_lock); | 573 | spin_unlock(&ctx->completion_lock); |
| 532 | } | 574 | } |
| 533 | if (ctx->has_evfd) | 575 | if (ctx->has_evfd) |
| 534 | io_eventfd_signal(ctx); | 576 | io_eventfd_flush_signal(ctx); |
| 535 | } | 577 | } |
| 536 | 578 | ||
| 537 | static inline void io_cqring_ev_posted(struct io_ring_ctx *ctx) | 579 | static inline void io_cqring_ev_posted(struct io_ring_ctx *ctx) |
| @@ -567,7 +609,7 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) · 29 unmodified lines | |||
| 567 | 609 | ||
| 568 | io_cq_lock(ctx); | 610 | io_cq_lock(ctx); |
| 569 | while (!list_empty(&ctx->cq_overflow_list)) { | 611 | while (!list_empty(&ctx->cq_overflow_list)) { |
| 570 | struct io_uring_cqe *cqe = io_get_cqe(ctx); | 612 | struct io_uring_cqe *cqe = io_get_cqe_overflow(ctx, true); |
| 571 | struct io_overflow_cqe *ocqe; | 613 | struct io_overflow_cqe *ocqe; |
| 572 | 614 | ||
| 573 | if (!cqe && !force) | 615 | if (!cqe && !force) |
| @@ -694,12 +736,19 @@ bool io_req_cqe_overflow(struct io_kiocb *req) · 120 unmodified lines | |||
| 694 | * control dependency is enough as we're using WRITE_ONCE to | 736 | * control dependency is enough as we're using WRITE_ONCE to |
| 695 | * fill the cq entry | 737 | * fill the cq entry |
| 696 | */ | 738 | */ |
| 697 | struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx) | 739 | struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow) |
| 698 | { | 740 | { |
| 699 | struct io_rings *rings = ctx->rings; | 741 | struct io_rings *rings = ctx->rings; |
| 700 | unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1); | 742 | unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1); |
| 701 | unsigned int free, queued, len; | 743 | unsigned int free, queued, len; |
| 702 | 744 | ||
| 745 | /* | ||
| 746 | * Posting into the CQ when there are pending overflowed CQEs may break | ||
| 747 | * ordering guarantees, which will affect links, F_MORE users and more. | ||
| 748 | * Force overflow the completion. | ||
| 749 | */ | ||
| 750 | if (!overflow && (ctx->check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))) | ||
| 751 | return NULL; | ||
| 703 | 752 | ||
| 704 | /* userspace may cheat modifying the tail, be safe and do min */ | 753 | /* userspace may cheat modifying the tail, be safe and do min */ |
| 705 | queued = min(__io_cqring_events(ctx), ctx->cq_entries); | 754 | queued = min(__io_cqring_events(ctx), ctx->cq_entries); |
| @@ -823,8 +872,12 @@ inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags) · 117 unmodified lines | |||
| 823 | 872 | ||
| 824 | void io_req_complete_failed(struct io_kiocb *req, s32 res) | 873 | void io_req_complete_failed(struct io_kiocb *req, s32 res) |
| 825 | { | 874 | { |
| 875 | const struct io_op_def *def = &io_op_defs[req->opcode]; | ||
| 876 | |||
| 826 | req_set_fail(req); | 877 | req_set_fail(req); |
| 827 | io_req_set_res(req, res, io_put_kbuf(req, IO_URING_F_UNLOCKED)); | 878 | io_req_set_res(req, res, io_put_kbuf(req, IO_URING_F_UNLOCKED)); |
| 879 | if (def->fail) | ||
| 880 | def->fail(req); | ||
| 828 | io_req_complete_post(req); | 881 | io_req_complete_post(req); |
| 829 | } | 882 | } |
| 830 | 883 | ||
| @@ -1047,17 +1100,40 @@ void tctx_task_work(struct callback_head *cb) · 216 unmodified lines | |||
| 1047 | trace_io_uring_task_work_run(tctx, count, loops); | 1100 | trace_io_uring_task_work_run(tctx, count, loops); |
| 1048 | } | 1101 | } |
| 1049 | 1102 | ||
| 1050 | void io_req_task_work_add(struct io_kiocb *req) | 1103 | static void io_req_local_work_add(struct io_kiocb *req) |
| 1104 | { | ||
| 1105 | struct io_ring_ctx *ctx = req->ctx; | ||
| 1106 | |||
| 1107 | if (!llist_add(&req->io_task_work.node, &ctx->work_llist)) | ||
| 1108 | return; | ||
| 1109 | |||
| 1110 | if (unlikely(atomic_read(&req->task->io_uring->in_idle))) { | ||
| 1111 | io_move_task_work_from_local(ctx); | ||
| 1112 | return; | ||
| 1113 | } | ||
| 1114 | |||
| 1115 | if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) | ||
| 1116 | atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); | ||
| 1117 | |||
| 1118 | if (ctx->has_evfd) | ||
| 1119 | io_eventfd_signal(ctx); | ||
| 1120 | io_cqring_wake(ctx); | ||
| 1121 | |||
| 1122 | } | ||
| 1123 | |||
| 1124 | static inline void __io_req_task_work_add(struct io_kiocb *req, bool allow_local) | ||
| 1051 | { | 1125 | { |
| 1052 | struct io_uring_task *tctx = req->task->io_uring; | 1126 | struct io_uring_task *tctx = req->task->io_uring; |
| 1053 | struct io_ring_ctx *ctx = req->ctx; | 1127 | struct io_ring_ctx *ctx = req->ctx; |
| 1054 | struct llist_node *node; | 1128 | struct llist_node *node; |
| 1055 | bool running; | ||
| 1056 | 1129 | ||
| 1057 | running = !llist_add(&req->io_task_work.node, &tctx->task_list); | 1130 | if (allow_local && ctx->flags & IORING_SETUP_DEFER_TASKRUN) { |
| 1131 | io_req_local_work_add(req); | ||
| 1132 | return; | ||
| 1133 | } | ||
| 1058 | 1134 | ||
| 1059 | /* task_work already pending, we're done */ | 1135 | /* task_work already pending, we're done */ |
| 1060 | if (running) | 1136 | if (!llist_add(&req->io_task_work.node, &tctx->task_list)) |
| 1061 | return; | 1137 | return; |
| 1062 | 1138 | ||
| 1063 | if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) | 1139 | if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) |
| @@ -1077,6 +1153,84 @@ void io_req_task_work_add(struct io_kiocb *req) · 13 unmodified lines | |||
| 1077 | } | 1153 | } |
| 1078 | } | 1154 | } |
| 1079 | 1155 | ||
| 1156 | void io_req_task_work_add(struct io_kiocb *req) | ||
| 1157 | { | ||
| 1158 | __io_req_task_work_add(req, true); | ||
| 1159 | } | ||
| 1160 | |||
| 1161 | static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx) | ||
| 1162 | { | ||
| 1163 | struct llist_node *node; | ||
| 1164 | |||
| 1165 | node = llist_del_all(&ctx->work_llist); | ||
| 1166 | while (node) { | ||
| 1167 | struct io_kiocb *req = container_of(node, struct io_kiocb, | ||
| 1168 | io_task_work.node); | ||
| 1169 | |||
| 1170 | node = node->next; | ||
| 1171 | __io_req_task_work_add(req, false); | ||
| 1172 | } | ||
| 1173 | } | ||
| 1174 | |||
| 1175 | int __io_run_local_work(struct io_ring_ctx *ctx, bool locked) | ||
| 1176 | { | ||
| 1177 | struct llist_node *node; | ||
| 1178 | struct llist_node fake; | ||
| 1179 | struct llist_node *current_final = NULL; | ||
| 1180 | int ret; | ||
| 1181 | unsigned int loops = 1; | ||
| 1182 | |||
| 1183 | if (unlikely(ctx->submitter_task != current)) | ||
| 1184 | return -EEXIST; | ||
| 1185 | |||
| 1186 | node = io_llist_xchg(&ctx->work_llist, &fake); | ||
| 1187 | ret = 0; | ||
| 1188 | again: | ||
| 1189 | while (node != current_final) { | ||
| 1190 | struct llist_node *next = node->next; | ||
| 1191 | struct io_kiocb *req = container_of(node, struct io_kiocb, | ||
| 1192 | io_task_work.node); | ||
| 1193 | prefetch(container_of(next, struct io_kiocb, io_task_work.node)); | ||
| 1194 | req->io_task_work.func(req, &locked); | ||
| 1195 | ret++; | ||
| 1196 | node = next; | ||
| 1197 | } | ||
| 1198 | |||
| 1199 | if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) | ||
| 1200 | atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); | ||
| 1201 | |||
| 1202 | node = io_llist_cmpxchg(&ctx->work_llist, &fake, NULL); | ||
| 1203 | if (node != &fake) { | ||
| 1204 | loops++; | ||
| 1205 | current_final = &fake; | ||
| 1206 | node = io_llist_xchg(&ctx->work_llist, &fake); | ||
| 1207 | goto again; | ||
| 1208 | } | ||
| 1209 | |||
| 1210 | if (locked) | ||
| 1211 | io_submit_flush_completions(ctx); | ||
| 1212 | trace_io_uring_local_work_run(ctx, ret, loops); | ||
| 1213 | return ret; | ||
| 1214 | |||
| 1215 | } | ||
| 1216 | |||
| 1217 | int io_run_local_work(struct io_ring_ctx *ctx) | ||
| 1218 | { | ||
| 1219 | bool locked; | ||
| 1220 | int ret; | ||
| 1221 | |||
| 1222 | if (llist_empty(&ctx->work_llist)) | ||
| 1223 | return 0; | ||
| 1224 | |||
| 1225 | __set_current_state(TASK_RUNNING); | ||
| 1226 | locked = mutex_trylock(&ctx->uring_lock); | ||
| 1227 | ret = __io_run_local_work(ctx, locked); | ||
| 1228 | if (locked) | ||
| 1229 | mutex_unlock(&ctx->uring_lock); | ||
| 1230 | |||
| 1231 | return ret; | ||
| 1232 | } | ||
| 1233 | |||
| 1080 | static void io_req_tw_post(struct io_kiocb *req, bool *locked) | 1234 | static void io_req_tw_post(struct io_kiocb *req, bool *locked) |
| 1081 | { | 1235 | { |
| 1082 | io_req_complete_post(req); | 1236 | io_req_complete_post(req); |
| @@ -1183,7 +1337,7 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx) · 100 unmodified lines | |||
| 1183 | struct io_wq_work_node *node, *prev; | 1337 | struct io_wq_work_node *node, *prev; |
| 1184 | struct io_submit_state *state = &ctx->submit_state; | 1338 | struct io_submit_state *state = &ctx->submit_state; |
| 1185 | 1339 | ||
| 1186 | spin_lock(&ctx->completion_lock); | 1340 | io_cq_lock(ctx); |
| 1187 | wq_list_for_each(node, prev, &state->compl_reqs) { | 1341 | wq_list_for_each(node, prev, &state->compl_reqs) { |
| 1188 | struct io_kiocb *req = container_of(node, struct io_kiocb, | 1342 | struct io_kiocb *req = container_of(node, struct io_kiocb, |
| 1189 | comp_list); | 1343 | comp_list); |
| @@ -1254,6 +1408,9 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) · 64 unmodified lines | |||
| 1254 | int ret = 0; | 1408 | int ret = 0; |
| 1255 | unsigned long check_cq; | 1409 | unsigned long check_cq; |
| 1256 | 1410 | ||
| 1411 | if (!io_allowed_run_tw(ctx)) | ||
| 1412 | return -EEXIST; | ||
| 1413 | |||
| 1257 | check_cq = READ_ONCE(ctx->check_cq); | 1414 | check_cq = READ_ONCE(ctx->check_cq); |
| 1258 | if (unlikely(check_cq)) { | 1415 | if (unlikely(check_cq)) { |
| 1259 | if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)) | 1416 | if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)) |
| @@ -1284,13 +1441,19 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) · 24 unmodified lines | |||
| 1284 | * forever, while the workqueue is stuck trying to acquire the | 1441 | * forever, while the workqueue is stuck trying to acquire the |
| 1285 | * very same mutex. | 1442 | * very same mutex. |
| 1286 | */ | 1443 | */ |
| 1287 | if (wq_list_empty(&ctx->iopoll_list)) { | 1444 | if (wq_list_empty(&ctx->iopoll_list) || |
| 1445 | io_task_work_pending(ctx)) { | ||
| 1288 | u32 tail = ctx->cached_cq_tail; | 1446 | u32 tail = ctx->cached_cq_tail; |
| 1289 | 1447 | ||
| 1290 | mutex_unlock(&ctx->uring_lock); | 1448 | if (!llist_empty(&ctx->work_llist)) |
| 1291 | io_run_task_work(); | 1449 | __io_run_local_work(ctx, true); |
| 1292 | mutex_lock(&ctx->uring_lock); | ||
| 1293 | 1450 | ||
| 1451 | if (task_work_pending(current) || | ||
| 1452 | wq_list_empty(&ctx->iopoll_list)) { | ||
| 1453 | mutex_unlock(&ctx->uring_lock); | ||
| 1454 | io_run_task_work(); | ||
| 1455 | mutex_lock(&ctx->uring_lock); | ||
| 1456 | } | ||
| 1294 | /* some requests don't go through iopoll_list */ | 1457 | /* some requests don't go through iopoll_list */ |
| 1295 | if (tail != ctx->cached_cq_tail || | 1458 | if (tail != ctx->cached_cq_tail || |
| 1296 | wq_list_empty(&ctx->iopoll_list)) | 1459 | wq_list_empty(&ctx->iopoll_list)) |
| @@ -1732,10 +1895,6 @@ static void io_queue_async(struct io_kiocb *req, int ret) · 435 unmodified lines | |||
| 1732 | io_req_task_queue(req); | 1895 | io_req_task_queue(req); |
| 1733 | break; | 1896 | break; |
| 1734 | case IO_APOLL_ABORTED: | 1897 | case IO_APOLL_ABORTED: |
| 1735 | /* | ||
| 1736 | * Queued up for async execution, worker will release | ||
| 1737 | * submit reference when the iocb is actually submitted. | ||
| 1738 | */ | ||
| 1739 | io_kbuf_recycle(req, 0); | 1898 | io_kbuf_recycle(req, 0); |
| 1740 | io_queue_iowq(req, NULL); | 1899 | io_queue_iowq(req, NULL); |
| 1741 | break; | 1900 | break; |
| @@ -2149,6 +2308,13 @@ struct io_wait_queue { · 407 unmodified lines | |||
| 2149 | unsigned nr_timeouts; | 2308 | unsigned nr_timeouts; |
| 2150 | }; | 2309 | }; |
| 2151 | 2310 | ||
| 2311 | static inline bool io_has_work(struct io_ring_ctx *ctx) | ||
| 2312 | { | ||
| 2313 | return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) || | ||
| 2314 | ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) && | ||
| 2315 | !llist_empty(&ctx->work_llist)); | ||
| 2316 | } | ||
| 2317 | |||
| 2152 | static inline bool io_should_wake(struct io_wait_queue *iowq) | 2318 | static inline bool io_should_wake(struct io_wait_queue *iowq) |
| 2153 | { | 2319 | { |
| 2154 | struct io_ring_ctx *ctx = iowq->ctx; | 2320 | struct io_ring_ctx *ctx = iowq->ctx; |
| @@ -2167,20 +2333,20 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, · 12 unmodified lines | |||
| 2167 | { | 2333 | { |
| 2168 | struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue, | 2334 | struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue, |
| 2169 | wq); | 2335 | wq); |
| 2336 | struct io_ring_ctx *ctx = iowq->ctx; | ||
| 2170 | 2337 | ||
| 2171 | /* | 2338 | /* |
| 2172 | * Cannot safely flush overflowed CQEs from here, ensure we wake up | 2339 | * Cannot safely flush overflowed CQEs from here, ensure we wake up |
| 2173 | * the task, and the next invocation will do it. | 2340 | * the task, and the next invocation will do it. |
| 2174 | */ | 2341 | */ |
| 2175 | if (io_should_wake(iowq) || | 2342 | if (io_should_wake(iowq) || io_has_work(ctx)) |
| 2176 | test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &iowq->ctx->check_cq)) | ||
| 2177 | return autoremove_wake_function(curr, mode, wake_flags, key); | 2343 | return autoremove_wake_function(curr, mode, wake_flags, key); |
| 2178 | return -1; | 2344 | return -1; |
| 2179 | } | 2345 | } |
| 2180 | 2346 | ||
| 2181 | int io_run_task_work_sig(void) | 2347 | int io_run_task_work_sig(struct io_ring_ctx *ctx) |
| 2182 | { | 2348 | { |
| 2183 | if (io_run_task_work()) | 2349 | if (io_run_task_work_ctx(ctx) > 0) |
| 2184 | return 1; | 2350 | return 1; |
| 2185 | if (task_sigpending(current)) | 2351 | if (task_sigpending(current)) |
| 2186 | return -EINTR; | 2352 | return -EINTR; |
| @@ -2196,7 +2362,7 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, · 9 unmodified lines | |||
| 2196 | unsigned long check_cq; | 2362 | unsigned long check_cq; |
| 2197 | 2363 | ||
| 2198 | /* make sure we run task_work before checking for signals */ | 2364 | /* make sure we run task_work before checking for signals */ |
| 2199 | ret = io_run_task_work_sig(); | 2365 | ret = io_run_task_work_sig(ctx); |
| 2200 | if (ret || io_should_wake(iowq)) | 2366 | if (ret || io_should_wake(iowq)) |
| 2201 | return ret; | 2367 | return ret; |
| 2202 | 2368 | ||
| @@ -2226,13 +2392,19 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 23 unmodified lines | |||
| 2226 | ktime_t timeout = KTIME_MAX; | 2392 | ktime_t timeout = KTIME_MAX; |
| 2227 | int ret; | 2393 | int ret; |
| 2228 | 2394 | ||
| 2395 | if (!io_allowed_run_tw(ctx)) | ||
| 2396 | return -EEXIST; | ||
| 2397 | |||
| 2229 | do { | 2398 | do { |
| 2399 | /* always run at least 1 task work to process local work */ | ||
| 2400 | ret = io_run_task_work_ctx(ctx); | ||
| 2401 | if (ret < 0) | ||
| 2402 | return ret; | ||
| 2230 | io_cqring_overflow_flush(ctx); | 2403 | io_cqring_overflow_flush(ctx); |
| 2404 | |||
| 2231 | if (io_cqring_events(ctx) >= min_events) | 2405 | if (io_cqring_events(ctx) >= min_events) |
| 2232 | return 0; | 2406 | return 0; |
| 2233 | if (!io_run_task_work()) | 2407 | } while (ret > 0); |
| 2234 | break; | ||
| 2235 | } while (1); | ||
| 2236 | 2408 | ||
| 2237 | if (sig) { | 2409 | if (sig) { |
| 2238 | #ifdef CONFIG_COMPAT | 2410 | #ifdef CONFIG_COMPAT |
| @@ -2366,17 +2538,11 @@ static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, · 127 unmodified lines | |||
| 2366 | ev_fd->eventfd_async = eventfd_async; | 2538 | ev_fd->eventfd_async = eventfd_async; |
| 2367 | ctx->has_evfd = true; | 2539 | ctx->has_evfd = true; |
| 2368 | rcu_assign_pointer(ctx->io_ev_fd, ev_fd); | 2540 | rcu_assign_pointer(ctx->io_ev_fd, ev_fd); |
| 2541 | atomic_set(&ev_fd->refs, 1); | ||
| 2542 | atomic_set(&ev_fd->ops, 0); | ||
| 2369 | return 0; | 2543 | return 0; |
| 2370 | } | 2544 | } |
| 2371 | 2545 | ||
| 2372 | static void io_eventfd_put(struct rcu_head *rcu) | ||
| 2373 | { | ||
| 2374 | struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu); | ||
| 2375 | |||
| 2376 | eventfd_ctx_put(ev_fd->cq_ev_fd); | ||
| 2377 | kfree(ev_fd); | ||
| 2378 | } | ||
| 2379 | |||
| 2380 | static int io_eventfd_unregister(struct io_ring_ctx *ctx) | 2546 | static int io_eventfd_unregister(struct io_ring_ctx *ctx) |
| 2381 | { | 2547 | { |
| 2382 | struct io_ev_fd *ev_fd; | 2548 | struct io_ev_fd *ev_fd; |
| @@ -2386,7 +2552,8 @@ static int io_eventfd_unregister(struct io_ring_ctx *ctx) · 3 unmodified lines | |||
| 2386 | if (ev_fd) { | 2552 | if (ev_fd) { |
| 2387 | ctx->has_evfd = false; | 2553 | ctx->has_evfd = false; |
| 2388 | rcu_assign_pointer(ctx->io_ev_fd, NULL); | 2554 | rcu_assign_pointer(ctx->io_ev_fd, NULL); |
| 2389 | call_rcu(&ev_fd->rcu, io_eventfd_put); | 2555 | if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_FREE_BIT), &ev_fd->ops)) |
| 2556 | call_rcu(&ev_fd->rcu, io_eventfd_ops); | ||
| 2390 | return 0; | 2557 | return 0; |
| 2391 | } | 2558 | } |
| 2392 | 2559 | ||
| @@ -2509,8 +2676,8 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) · 116 unmodified lines | |||
| 2509 | * Users may get EPOLLIN meanwhile seeing nothing in cqring, this | 2676 | * Users may get EPOLLIN meanwhile seeing nothing in cqring, this |
| 2510 | * pushs them to do the flush. | 2677 | * pushs them to do the flush. |
| 2511 | */ | 2678 | */ |
| 2512 | if (io_cqring_events(ctx) || | 2679 | |
| 2513 | test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq)) | 2680 | if (io_cqring_events(ctx) || io_has_work(ctx)) |
| 2514 | mask |= EPOLLIN | EPOLLRDNORM; | 2681 | mask |= EPOLLIN | EPOLLRDNORM; |
| 2515 | 2682 | ||
| 2516 | return mask; | 2683 | return mask; |
| @@ -2573,6 +2740,9 @@ static __cold void io_ring_exit_work(struct work_struct *work) · 56 unmodified lines | |||
| 2573 | * as nobody else will be looking for them. | 2740 | * as nobody else will be looking for them. |
| 2574 | */ | 2741 | */ |
| 2575 | do { | 2742 | do { |
| 2743 | if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) | ||
| 2744 | io_move_task_work_from_local(ctx); | ||
| 2745 | |||
| 2576 | while (io_uring_try_cancel_requests(ctx, NULL, true)) | 2746 | while (io_uring_try_cancel_requests(ctx, NULL, true)) |
| 2577 | cond_resched(); | 2747 | cond_resched(); |
| 2578 | 2748 | ||
| @@ -2770,13 +2940,15 @@ static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, · 191 unmodified lines | |||
| 2770 | } | 2940 | } |
| 2771 | } | 2941 | } |
| 2772 | 2942 | ||
| 2943 | if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) | ||
| 2944 | ret |= io_run_local_work(ctx) > 0; | ||
| 2773 | ret |= io_cancel_defer_files(ctx, task, cancel_all); | 2945 | ret |= io_cancel_defer_files(ctx, task, cancel_all); |
| 2774 | mutex_lock(&ctx->uring_lock); | 2946 | mutex_lock(&ctx->uring_lock); |
| 2775 | ret |= io_poll_remove_all(ctx, task, cancel_all); | 2947 | ret |= io_poll_remove_all(ctx, task, cancel_all); |
| 2776 | mutex_unlock(&ctx->uring_lock); | 2948 | mutex_unlock(&ctx->uring_lock); |
| 2777 | ret |= io_kill_timeouts(ctx, task, cancel_all); | 2949 | ret |= io_kill_timeouts(ctx, task, cancel_all); |
| 2778 | if (task) | 2950 | if (task) |
| 2779 | ret |= io_run_task_work(); | 2951 | ret |= io_run_task_work() > 0; |
| 2780 | return ret; | 2952 | return ret; |
| 2781 | } | 2953 | } |
| 2782 | 2954 | ||
| @@ -2992,8 +3164,6 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 209 unmodified lines | |||
| 2992 | struct fd f; | 3164 | struct fd f; |
| 2993 | long ret; | 3165 | long ret; |
| 2994 | 3166 | ||
| 2995 | io_run_task_work(); | ||
| 2996 | |||
| 2997 | if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP | | 3167 | if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP | |
| 2998 | IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG | | 3168 | IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG | |
| 2999 | IORING_ENTER_REGISTERED_RING))) | 3169 | IORING_ENTER_REGISTERED_RING))) |
| @@ -3063,8 +3233,10 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 63 unmodified lines | |||
| 3063 | goto iopoll_locked; | 3233 | goto iopoll_locked; |
| 3064 | mutex_unlock(&ctx->uring_lock); | 3234 | mutex_unlock(&ctx->uring_lock); |
| 3065 | } | 3235 | } |
| 3236 | |||
| 3066 | if (flags & IORING_ENTER_GETEVENTS) { | 3237 | if (flags & IORING_ENTER_GETEVENTS) { |
| 3067 | int ret2; | 3238 | int ret2; |
| 3239 | |||
| 3068 | if (ctx->syscall_iopoll) { | 3240 | if (ctx->syscall_iopoll) { |
| 3069 | /* | 3241 | /* |
| 3070 | * We disallow the app entering submit/complete with | 3242 | * We disallow the app entering submit/complete with |
| @@ -3293,17 +3465,29 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, · 222 unmodified lines | |||
| 3293 | if (ctx->flags & IORING_SETUP_SQPOLL) { | 3465 | if (ctx->flags & IORING_SETUP_SQPOLL) { |
| 3294 | /* IPI related flags don't make sense with SQPOLL */ | 3466 | /* IPI related flags don't make sense with SQPOLL */ |
| 3295 | if (ctx->flags & (IORING_SETUP_COOP_TASKRUN | | 3467 | if (ctx->flags & (IORING_SETUP_COOP_TASKRUN | |
| 3296 | IORING_SETUP_TASKRUN_FLAG)) | 3468 | IORING_SETUP_TASKRUN_FLAG | |
| 3469 | IORING_SETUP_DEFER_TASKRUN)) | ||
| 3297 | goto err; | 3470 | goto err; |
| 3298 | ctx->notify_method = TWA_SIGNAL_NO_IPI; | 3471 | ctx->notify_method = TWA_SIGNAL_NO_IPI; |
| 3299 | } else if (ctx->flags & IORING_SETUP_COOP_TASKRUN) { | 3472 | } else if (ctx->flags & IORING_SETUP_COOP_TASKRUN) { |
| 3300 | ctx->notify_method = TWA_SIGNAL_NO_IPI; | 3473 | ctx->notify_method = TWA_SIGNAL_NO_IPI; |
| 3301 | } else { | 3474 | } else { |
| 3302 | if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) | 3475 | if (ctx->flags & IORING_SETUP_TASKRUN_FLAG && |
| 3476 | !(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) | ||
| 3303 | goto err; | 3477 | goto err; |
| 3304 | ctx->notify_method = TWA_SIGNAL; | 3478 | ctx->notify_method = TWA_SIGNAL; |
| 3305 | } | 3479 | } |
| 3306 | 3480 | ||
| 3481 | /* | ||
| 3482 | * For DEFER_TASKRUN we require the completion task to be the same as the | ||
| 3483 | * submission task. This implies that there is only one submitter, so enforce | ||
| 3484 | * that. | ||
| 3485 | */ | ||
| 3486 | if (ctx->flags & IORING_SETUP_DEFER_TASKRUN && | ||
| 3487 | !(ctx->flags & IORING_SETUP_SINGLE_ISSUER)) { | ||
| 3488 | goto err; | ||
| 3489 | } | ||
| 3490 | |||
| 3307 | /* | 3491 | /* |
| 3308 | * This is just grabbed for accounting purposes. When a process exits, | 3492 | * This is just grabbed for accounting purposes. When a process exits, |
| 3309 | * the mm is exited and dropped before the files, hence we need to hang | 3493 | * the mm is exited and dropped before the files, hence we need to hang |
| @@ -3408,7 +3592,7 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) · 98 unmodified lines | |||
| 3408 | IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL | | 3592 | IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL | |
| 3409 | IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG | | 3593 | IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG | |
| 3410 | IORING_SETUP_SQE128 | IORING_SETUP_CQE32 | | 3594 | IORING_SETUP_SQE128 | IORING_SETUP_CQE32 | |
| 3411 | IORING_SETUP_SINGLE_ISSUER)) | 3595 | IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN)) |
| 3412 | return -EINVAL; | 3596 | return -EINVAL; |
| 3413 | 3597 | ||
| 3414 | return io_uring_create(entries, &p, params); | 3598 | return io_uring_create(entries, &p, params); |
| @@ -3874,7 +4058,7 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, · 459 unmodified lines | |||
| 3874 | 4058 | ||
| 3875 | ctx = f.file->private_data; | 4059 | ctx = f.file->private_data; |
| 3876 | 4060 | ||
| 3877 | io_run_task_work(); | 4061 | io_run_task_work_ctx(ctx); |
| 3878 | 4062 | ||
| 3879 | mutex_lock(&ctx->uring_lock); | 4063 | mutex_lock(&ctx->uring_lock); |
| 3880 | ret = __io_uring_register(ctx, opcode, arg, nr_args); | 4064 | ret = __io_uring_register(ctx, opcode, arg, nr_args); |
| @@ -24,9 +24,11 @@ enum { · 23 unmodified lines | |||
| 24 | IOU_STOP_MULTISHOT = -ECANCELED, | 24 | IOU_STOP_MULTISHOT = -ECANCELED, |
| 25 | }; | 25 | }; |
| 26 | 26 | ||
| 27 | struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx); | 27 | struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow); |
| 28 | bool io_req_cqe_overflow(struct io_kiocb *req); | 28 | bool io_req_cqe_overflow(struct io_kiocb *req); |
| 29 | int io_run_task_work_sig(void); | 29 | int io_run_task_work_sig(struct io_ring_ctx *ctx); |
| 30 | int __io_run_local_work(struct io_ring_ctx *ctx, bool locked); | ||
| 31 | int io_run_local_work(struct io_ring_ctx *ctx); | ||
| 30 | void io_req_complete_failed(struct io_kiocb *req, s32 res); | 32 | void io_req_complete_failed(struct io_kiocb *req, s32 res); |
| 31 | void __io_req_complete(struct io_kiocb *req, unsigned issue_flags); | 33 | void __io_req_complete(struct io_kiocb *req, unsigned issue_flags); |
| 32 | void io_req_complete_post(struct io_kiocb *req); | 34 | void io_req_complete_post(struct io_kiocb *req); |
| @@ -91,7 +93,8 @@ static inline void io_cq_lock(struct io_ring_ctx *ctx) · 58 unmodified lines | |||
| 91 | 93 | ||
| 92 | void io_cq_unlock_post(struct io_ring_ctx *ctx); | 94 | void io_cq_unlock_post(struct io_ring_ctx *ctx); |
| 93 | 95 | ||
| 94 | static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx) | 96 | static inline struct io_uring_cqe *io_get_cqe_overflow(struct io_ring_ctx *ctx, |
| 97 | bool overflow) | ||
| 95 | { | 98 | { |
| 96 | if (likely(ctx->cqe_cached < ctx->cqe_sentinel)) { | 99 | if (likely(ctx->cqe_cached < ctx->cqe_sentinel)) { |
| 97 | struct io_uring_cqe *cqe = ctx->cqe_cached; | 100 | struct io_uring_cqe *cqe = ctx->cqe_cached; |
| @@ -103,7 +106,12 @@ static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx) · 5 unmodified lines | |||
| 103 | return cqe; | 106 | return cqe; |
| 104 | } | 107 | } |
| 105 | 108 | ||
| 106 | return __io_get_cqe(ctx); | 109 | return __io_get_cqe(ctx, overflow); |
| 110 | } | ||
| 111 | |||
| 112 | static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx) | ||
| 113 | { | ||
| 114 | return io_get_cqe_overflow(ctx, false); | ||
| 107 | } | 115 | } |
| 108 | 116 | ||
| 109 | static inline bool __io_fill_cqe_req(struct io_ring_ctx *ctx, | 117 | static inline bool __io_fill_cqe_req(struct io_ring_ctx *ctx, |
| @@ -221,17 +229,43 @@ static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) · 111 unmodified lines | |||
| 221 | return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head; | 229 | return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head; |
| 222 | } | 230 | } |
| 223 | 231 | ||
| 224 | static inline bool io_run_task_work(void) | 232 | static inline int io_run_task_work(void) |
| 225 | { | 233 | { |
| 226 | if (test_thread_flag(TIF_NOTIFY_SIGNAL)) { | 234 | if (task_work_pending(current)) { |
| 235 | if (test_thread_flag(TIF_NOTIFY_SIGNAL)) | ||
| 236 | clear_notify_signal(); | ||
| 227 | __set_current_state(TASK_RUNNING); | 237 | __set_current_state(TASK_RUNNING); |
| 228 | clear_notify_signal(); | 238 | task_work_run(); |
| 229 | if (task_work_pending(current)) | 239 | return 1; |
| 230 | task_work_run(); | ||
| 231 | return true; | ||
| 232 | } | 240 | } |
| 233 | 241 | ||
| 234 | return false; | 242 | return 0; |
| 243 | } | ||
| 244 | |||
| 245 | static inline bool io_task_work_pending(struct io_ring_ctx *ctx) | ||
| 246 | { | ||
| 247 | return test_thread_flag(TIF_NOTIFY_SIGNAL) || | ||
| 248 | !wq_list_empty(&ctx->work_llist); | ||
| 249 | } | ||
| 250 | |||
| 251 | static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx) | ||
| 252 | { | ||
| 253 | int ret = 0; | ||
| 254 | int ret2; | ||
| 255 | |||
| 256 | if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) | ||
| 257 | ret = io_run_local_work(ctx); | ||
| 258 | |||
| 259 | /* want to run this after in case more is added */ | ||
| 260 | ret2 = io_run_task_work(); | ||
| 261 | |||
| 262 | /* Try propagate error in favour of if tasks were run, | ||
| 263 | * but still make sure to run them if requested | ||
| 264 | */ | ||
| 265 | if (ret >= 0) | ||
| 266 | ret += ret2; | ||
| 267 | |||
| 268 | return ret; | ||
| 235 | } | 269 | } |
| 236 | 270 | ||
| 237 | static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked) | 271 | static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked) |
| @@ -301,4 +335,10 @@ static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) · 63 unmodified lines | |||
| 301 | return container_of(node, struct io_kiocb, comp_list); | 335 | return container_of(node, struct io_kiocb, comp_list); |
| 302 | } | 336 | } |
| 303 | 337 | ||
| 338 | static inline bool io_allowed_run_tw(struct io_ring_ctx *ctx) | ||
| 339 | { | ||
| 340 | return likely(!(ctx->flags & IORING_SETUP_DEFER_TASKRUN) || | ||
| 341 | ctx->submitter_task == current); | ||
| 342 | } | ||
| 343 | |||
| 304 | #endif | 344 | #endif |
| @@ -86,18 +86,6 @@ static inline bool io_do_buffer_select(struct io_kiocb *req) · 85 unmodified lines | |||
| 86 | 86 | ||
| 87 | static inline void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) | 87 | static inline void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) |
| 88 | { | 88 | { |
| 89 | /* | ||
| 90 | * READV uses fields in `struct io_rw` (len/addr) to stash the selected | ||
| 91 | * buffer data. However if that buffer is recycled the original request | ||
| 92 | * data stored in addr is lost. Therefore forbid recycling for now. | ||
| 93 | */ | ||
| 94 | if (req->opcode == IORING_OP_READV) { | ||
| 95 | if ((req->flags & REQ_F_BUFFER_RING) && req->buf_list) { | ||
| 96 | req->buf_list->head++; | ||
| 97 | req->buf_list = NULL; | ||
| 98 | } | ||
| 99 | return; | ||
| 100 | } | ||
| 101 | if (req->flags & REQ_F_BUFFER_SELECTED) | 89 | if (req->flags & REQ_F_BUFFER_SELECTED) |
| 102 | io_kbuf_recycle_legacy(req, issue_flags); | 90 | io_kbuf_recycle_legacy(req, issue_flags); |
| 103 | if (req->flags & REQ_F_BUFFER_RING) | 91 | if (req->flags & REQ_F_BUFFER_RING) |
| @@ -55,21 +55,14 @@ struct io_sr_msg { · 54 unmodified lines | |||
| 55 | struct user_msghdr __user *umsg; | 55 | struct user_msghdr __user *umsg; |
| 56 | void __user *buf; | 56 | void __user *buf; |
| 57 | }; | 57 | }; |
| 58 | unsigned len; | ||
| 59 | unsigned done_io; | ||
| 58 | unsigned msg_flags; | 60 | unsigned msg_flags; |
| 59 | unsigned flags; | 61 | u16 flags; |
| 60 | size_t len; | 62 | /* initialised and used only by !msg send variants */ |
| 61 | size_t done_io; | 63 | u16 addr_len; |
| 62 | }; | ||
| 63 | |||
| 64 | struct io_sendzc { | ||
| 65 | struct file *file; | ||
| 66 | void __user *buf; | ||
| 67 | size_t len; | ||
| 68 | unsigned msg_flags; | ||
| 69 | unsigned flags; | ||
| 70 | unsigned addr_len; | ||
| 71 | void __user *addr; | 64 | void __user *addr; |
| 72 | size_t done_io; | 65 | /* used only for send zerocopy */ |
| 73 | struct io_kiocb *notif; | 66 | struct io_kiocb *notif; |
| 74 | }; | 67 | }; |
| 75 | 68 | ||
| @@ -126,28 +119,36 @@ static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags) · 50 unmodified lines | |||
| 126 | } | 119 | } |
| 127 | } | 120 | } |
| 128 | 121 | ||
| 129 | static struct io_async_msghdr *io_recvmsg_alloc_async(struct io_kiocb *req, | 122 | static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req, |
| 130 | unsigned int issue_flags) | 123 | unsigned int issue_flags) |
| 131 | { | 124 | { |
| 132 | struct io_ring_ctx *ctx = req->ctx; | 125 | struct io_ring_ctx *ctx = req->ctx; |
| 133 | struct io_cache_entry *entry; | 126 | struct io_cache_entry *entry; |
| 127 | struct io_async_msghdr *hdr; | ||
| 134 | 128 | ||
| 135 | if (!(issue_flags & IO_URING_F_UNLOCKED) && | 129 | if (!(issue_flags & IO_URING_F_UNLOCKED) && |
| 136 | (entry = io_alloc_cache_get(&ctx->netmsg_cache)) != NULL) { | 130 | (entry = io_alloc_cache_get(&ctx->netmsg_cache)) != NULL) { |
| 137 | struct io_async_msghdr *hdr; | ||
| 138 | |||
| 139 | hdr = container_of(entry, struct io_async_msghdr, cache); | 131 | hdr = container_of(entry, struct io_async_msghdr, cache); |
| 132 | hdr->free_iov = NULL; | ||
| 140 | req->flags |= REQ_F_ASYNC_DATA; | 133 | req->flags |= REQ_F_ASYNC_DATA; |
| 141 | req->async_data = hdr; | 134 | req->async_data = hdr; |
| 142 | return hdr; | 135 | return hdr; |
| 143 | } | 136 | } |
| 144 | 137 | ||
| 145 | if (!io_alloc_async_data(req)) | 138 | if (!io_alloc_async_data(req)) { |
| 146 | return req->async_data; | 139 | hdr = req->async_data; |
| 147 | 140 | hdr->free_iov = NULL; | |
| 141 | return hdr; | ||
| 142 | } | ||
| 148 | return NULL; | 143 | return NULL; |
| 149 | } | 144 | } |
| 150 | 145 | ||
| 146 | static inline struct io_async_msghdr *io_msg_alloc_async_prep(struct io_kiocb *req) | ||
| 147 | { | ||
| 148 | /* ->prep_async is always called from the submission context */ | ||
| 149 | return io_msg_alloc_async(req, 0); | ||
| 150 | } | ||
| 151 | |||
| 151 | static int io_setup_async_msg(struct io_kiocb *req, | 152 | static int io_setup_async_msg(struct io_kiocb *req, |
| 152 | struct io_async_msghdr *kmsg, | 153 | struct io_async_msghdr *kmsg, |
| 153 | unsigned int issue_flags) | 154 | unsigned int issue_flags) |
| @@ -156,17 +157,20 @@ static int io_setup_async_msg(struct io_kiocb *req, · 2 unmodified lines | |||
| 156 | 157 | ||
| 157 | if (req_has_async_data(req)) | 158 | if (req_has_async_data(req)) |
| 158 | return -EAGAIN; | 159 | return -EAGAIN; |
| 159 | async_msg = io_recvmsg_alloc_async(req, issue_flags); | 160 | async_msg = io_msg_alloc_async(req, issue_flags); |
| 160 | if (!async_msg) { | 161 | if (!async_msg) { |
| 161 | kfree(kmsg->free_iov); | 162 | kfree(kmsg->free_iov); |
| 162 | return -ENOMEM; | 163 | return -ENOMEM; |
| 163 | } | 164 | } |
| 164 | req->flags |= REQ_F_NEED_CLEANUP; | 165 | req->flags |= REQ_F_NEED_CLEANUP; |
| 165 | memcpy(async_msg, kmsg, sizeof(*kmsg)); | 166 | memcpy(async_msg, kmsg, sizeof(*kmsg)); |
| 166 | async_msg->msg.msg_name = &async_msg->addr; | 167 | if (async_msg->msg.msg_name) |
| 168 | async_msg->msg.msg_name = &async_msg->addr; | ||
| 167 | /* if were using fast_iov, set it to the new one */ | 169 | /* if were using fast_iov, set it to the new one */ |
| 168 | if (!async_msg->free_iov) | 170 | if (!kmsg->free_iov) { |
| 169 | async_msg->msg.msg_iter.iov = async_msg->fast_iov; | 171 | size_t fast_idx = kmsg->msg.msg_iter.iov - kmsg->fast_iov; |
| 172 | async_msg->msg.msg_iter.iov = &async_msg->fast_iov[fast_idx]; | ||
| 173 | } | ||
| 170 | 174 | ||
| 171 | return -EAGAIN; | 175 | return -EAGAIN; |
| 172 | } | 176 | } |
| @@ -182,34 +186,34 @@ static int io_sendmsg_copy_hdr(struct io_kiocb *req, · 9 unmodified lines | |||
| 182 | &iomsg->free_iov); | 186 | &iomsg->free_iov); |
| 183 | } | 187 | } |
| 184 | 188 | ||
| 185 | int io_sendzc_prep_async(struct io_kiocb *req) | 189 | int io_send_prep_async(struct io_kiocb *req) |
| 186 | { | 190 | { |
| 187 | struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc); | 191 | struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); |
| 188 | struct io_async_msghdr *io; | 192 | struct io_async_msghdr *io; |
| 189 | int ret; | 193 | int ret; |
| 190 | 194 | ||
| 191 | if (!zc->addr || req_has_async_data(req)) | 195 | if (!zc->addr || req_has_async_data(req)) |
| 192 | return 0; | 196 | return 0; |
| 193 | if (io_alloc_async_data(req)) | 197 | io = io_msg_alloc_async_prep(req); |
| 198 | if (!io) | ||
| 194 | return -ENOMEM; | 199 | return -ENOMEM; |
| 195 | |||
| 196 | io = req->async_data; | ||
| 197 | ret = move_addr_to_kernel(zc->addr, zc->addr_len, &io->addr); | 200 | ret = move_addr_to_kernel(zc->addr, zc->addr_len, &io->addr); |
| 198 | return ret; | 201 | return ret; |
| 199 | } | 202 | } |
| 200 | 203 | ||
| 201 | static int io_setup_async_addr(struct io_kiocb *req, | 204 | static int io_setup_async_addr(struct io_kiocb *req, |
| 202 | struct sockaddr_storage *addr, | 205 | struct sockaddr_storage *addr_storage, |
| 203 | unsigned int issue_flags) | 206 | unsigned int issue_flags) |
| 204 | { | 207 | { |
| 208 | struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); | ||
| 205 | struct io_async_msghdr *io; | 209 | struct io_async_msghdr *io; |
| 206 | 210 | ||
| 207 | if (!addr || req_has_async_data(req)) | 211 | if (!sr->addr || req_has_async_data(req)) |
| 208 | return -EAGAIN; | 212 | return -EAGAIN; |
| 209 | if (io_alloc_async_data(req)) | 213 | io = io_msg_alloc_async(req, issue_flags); |
| 214 | if (!io) | ||
| 210 | return -ENOMEM; | 215 | return -ENOMEM; |
| 211 | io = req->async_data; | 216 | memcpy(&io->addr, addr_storage, sizeof(io->addr)); |
| 212 | memcpy(&io->addr, addr, sizeof(io->addr)); | ||
| 213 | return -EAGAIN; | 217 | return -EAGAIN; |
| 214 | } | 218 | } |
| 215 | 219 | ||
| @@ -217,6 +221,8 @@ int io_sendmsg_prep_async(struct io_kiocb *req) · 1 unmodified line | |||
| 217 | { | 221 | { |
| 218 | int ret; | 222 | int ret; |
| 219 | 223 | ||
| 224 | if (!io_msg_alloc_async_prep(req)) | ||
| 225 | return -ENOMEM; | ||
| 220 | ret = io_sendmsg_copy_hdr(req, req->async_data); | 226 | ret = io_sendmsg_copy_hdr(req, req->async_data); |
| 221 | if (!ret) | 227 | if (!ret) |
| 222 | req->flags |= REQ_F_NEED_CLEANUP; | 228 | req->flags |= REQ_F_NEED_CLEANUP; |
| @@ -234,8 +240,14 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 11 unmodified lines | |||
| 234 | { | 240 | { |
| 235 | struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); | 241 | struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); |
| 236 | 242 | ||
| 237 | if (unlikely(sqe->file_index || sqe->addr2)) | 243 | if (req->opcode == IORING_OP_SEND) { |
| 244 | if (READ_ONCE(sqe->__pad3[0])) | ||
| 245 | return -EINVAL; | ||
| 246 | sr->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); | ||
| 247 | sr->addr_len = READ_ONCE(sqe->addr_len); | ||
| 248 | } else if (sqe->addr2 || sqe->file_index) { | ||
| 238 | return -EINVAL; | 249 | return -EINVAL; |
| 250 | } | ||
| 239 | 251 | ||
| 240 | sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); | 252 | sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); |
| 241 | sr->len = READ_ONCE(sqe->len); | 253 | sr->len = READ_ONCE(sqe->len); |
| @@ -291,13 +303,13 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) · 49 unmodified lines | |||
| 291 | if (ret < min_ret) { | 303 | if (ret < min_ret) { |
| 292 | if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) | 304 | if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) |
| 293 | return io_setup_async_msg(req, kmsg, issue_flags); | 305 | return io_setup_async_msg(req, kmsg, issue_flags); |
| 294 | if (ret == -ERESTARTSYS) | ||
| 295 | ret = -EINTR; | ||
| 296 | if (ret > 0 && io_net_retry(sock, flags)) { | 306 | if (ret > 0 && io_net_retry(sock, flags)) { |
| 297 | sr->done_io += ret; | 307 | sr->done_io += ret; |
| 298 | req->flags |= REQ_F_PARTIAL_IO; | 308 | req->flags |= REQ_F_PARTIAL_IO; |
| 299 | return io_setup_async_msg(req, kmsg, issue_flags); | 309 | return io_setup_async_msg(req, kmsg, issue_flags); |
| 300 | } | 310 | } |
| 311 | if (ret == -ERESTARTSYS) | ||
| 312 | ret = -EINTR; | ||
| 301 | req_set_fail(req); | 313 | req_set_fail(req); |
| 302 | } | 314 | } |
| 303 | /* fast path, check for non-NULL to avoid function call */ | 315 | /* fast path, check for non-NULL to avoid function call */ |
| @@ -315,6 +327,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) · 11 unmodified lines | |||
| 315 | 327 | ||
| 316 | int io_send(struct io_kiocb *req, unsigned int issue_flags) | 328 | int io_send(struct io_kiocb *req, unsigned int issue_flags) |
| 317 | { | 329 | { |
| 330 | struct sockaddr_storage __address; | ||
| 318 | struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); | 331 | struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); |
| 319 | struct msghdr msg; | 332 | struct msghdr msg; |
| 320 | struct iovec iov; | 333 | struct iovec iov; |
| @@ -323,9 +336,29 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) · 2 unmodified lines | |||
| 323 | int min_ret = 0; | 336 | int min_ret = 0; |
| 324 | int ret; | 337 | int ret; |
| 325 | 338 | ||
| 339 | msg.msg_name = NULL; | ||
| 340 | msg.msg_control = NULL; | ||
| 341 | msg.msg_controllen = 0; | ||
| 342 | msg.msg_namelen = 0; | ||
| 343 | msg.msg_ubuf = NULL; | ||
| 344 | |||
| 345 | if (sr->addr) { | ||
| 346 | if (req_has_async_data(req)) { | ||
| 347 | struct io_async_msghdr *io = req->async_data; | ||
| 348 | |||
| 349 | msg.msg_name = &io->addr; | ||
| 350 | } else { | ||
| 351 | ret = move_addr_to_kernel(sr->addr, sr->addr_len, &__address); | ||
| 352 | if (unlikely(ret < 0)) | ||
| 353 | return ret; | ||
| 354 | msg.msg_name = (struct sockaddr *)&__address; | ||
| 355 | } | ||
| 356 | msg.msg_namelen = sr->addr_len; | ||
| 357 | } | ||
| 358 | |||
| 326 | if (!(req->flags & REQ_F_POLLED) && | 359 | if (!(req->flags & REQ_F_POLLED) && |
| 327 | (sr->flags & IORING_RECVSEND_POLL_FIRST)) | 360 | (sr->flags & IORING_RECVSEND_POLL_FIRST)) |
| 328 | return -EAGAIN; | 361 | return io_setup_async_addr(req, &__address, issue_flags); |
| 329 | 362 | ||
| 330 | sock = sock_from_file(req->file); | 363 | sock = sock_from_file(req->file); |
| 331 | if (unlikely(!sock)) | 364 | if (unlikely(!sock)) |
| @@ -335,12 +368,6 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) · 3 unmodified lines | |||
| 335 | if (unlikely(ret)) | 368 | if (unlikely(ret)) |
| 336 | return ret; | 369 | return ret; |
| 337 | 370 | ||
| 338 | msg.msg_name = NULL; | ||
| 339 | msg.msg_control = NULL; | ||
| 340 | msg.msg_controllen = 0; | ||
| 341 | msg.msg_namelen = 0; | ||
| 342 | msg.msg_ubuf = NULL; | ||
| 343 | |||
| 344 | flags = sr->msg_flags; | 371 | flags = sr->msg_flags; |
| 345 | if (issue_flags & IO_URING_F_NONBLOCK) | 372 | if (issue_flags & IO_URING_F_NONBLOCK) |
| 346 | flags |= MSG_DONTWAIT; | 373 | flags |= MSG_DONTWAIT; |
| @@ -351,16 +378,17 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) · 4 unmodified lines | |||
| 351 | ret = sock_sendmsg(sock, &msg); | 378 | ret = sock_sendmsg(sock, &msg); |
| 352 | if (ret < min_ret) { | 379 | if (ret < min_ret) { |
| 353 | if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) | 380 | if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) |
| 354 | return -EAGAIN; | 381 | return io_setup_async_addr(req, &__address, issue_flags); |
| 355 | if (ret == -ERESTARTSYS) | 382 | |
| 356 | ret = -EINTR; | ||
| 357 | if (ret > 0 && io_net_retry(sock, flags)) { | 383 | if (ret > 0 && io_net_retry(sock, flags)) { |
| 358 | sr->len -= ret; | 384 | sr->len -= ret; |
| 359 | sr->buf += ret; | 385 | sr->buf += ret; |
| 360 | sr->done_io += ret; | 386 | sr->done_io += ret; |
| 361 | req->flags |= REQ_F_PARTIAL_IO; | 387 | req->flags |= REQ_F_PARTIAL_IO; |
| 362 | return -EAGAIN; | 388 | return io_setup_async_addr(req, &__address, issue_flags); |
| 363 | } | 389 | } |
| 390 | if (ret == -ERESTARTSYS) | ||
| 391 | ret = -EINTR; | ||
| 364 | req_set_fail(req); | 392 | req_set_fail(req); |
| 365 | } | 393 | } |
| 366 | if (ret >= 0) | 394 | if (ret >= 0) |
| @@ -454,7 +482,6 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, · 87 unmodified lines | |||
| 454 | 482 | ||
| 455 | if (msg.msg_iovlen == 0) { | 483 | if (msg.msg_iovlen == 0) { |
| 456 | sr->len = 0; | 484 | sr->len = 0; |
| 457 | iomsg->free_iov = NULL; | ||
| 458 | } else if (msg.msg_iovlen > 1) { | 485 | } else if (msg.msg_iovlen > 1) { |
| 459 | return -EINVAL; | 486 | return -EINVAL; |
| 460 | } else { | 487 | } else { |
| @@ -465,7 +492,6 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, · 4 unmodified lines | |||
| 465 | if (clen < 0) | 492 | if (clen < 0) |
| 466 | return -EINVAL; | 493 | return -EINVAL; |
| 467 | sr->len = clen; | 494 | sr->len = clen; |
| 468 | iomsg->free_iov = NULL; | ||
| 469 | } | 495 | } |
| 470 | 496 | ||
| 471 | if (req->flags & REQ_F_APOLL_MULTISHOT) { | 497 | if (req->flags & REQ_F_APOLL_MULTISHOT) { |
| @@ -504,6 +530,8 @@ int io_recvmsg_prep_async(struct io_kiocb *req) · 32 unmodified lines | |||
| 504 | { | 530 | { |
| 505 | int ret; | 531 | int ret; |
| 506 | 532 | ||
| 533 | if (!io_msg_alloc_async_prep(req)) | ||
| 534 | return -ENOMEM; | ||
| 507 | ret = io_recvmsg_copy_hdr(req, req->async_data); | 535 | ret = io_recvmsg_copy_hdr(req, req->async_data); |
| 508 | if (!ret) | 536 | if (!ret) |
| 509 | req->flags |= REQ_F_NEED_CLEANUP; | 537 | req->flags |= REQ_F_NEED_CLEANUP; |
| @@ -751,13 +779,13 @@ retry_multishot: · 241 unmodified lines | |||
| 751 | } | 779 | } |
| 752 | return ret; | 780 | return ret; |
| 753 | } | 781 | } |
| 754 | if (ret == -ERESTARTSYS) | ||
| 755 | ret = -EINTR; | ||
| 756 | if (ret > 0 && io_net_retry(sock, flags)) { | 782 | if (ret > 0 && io_net_retry(sock, flags)) { |
| 757 | sr->done_io += ret; | 783 | sr->done_io += ret; |
| 758 | req->flags |= REQ_F_PARTIAL_IO; | 784 | req->flags |= REQ_F_PARTIAL_IO; |
| 759 | return io_setup_async_msg(req, kmsg, issue_flags); | 785 | return io_setup_async_msg(req, kmsg, issue_flags); |
| 760 | } | 786 | } |
| 787 | if (ret == -ERESTARTSYS) | ||
| 788 | ret = -EINTR; | ||
| 761 | req_set_fail(req); | 789 | req_set_fail(req); |
| 762 | } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { | 790 | } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { |
| 763 | req_set_fail(req); | 791 | req_set_fail(req); |
| @@ -847,8 +875,6 @@ retry_multishot: · 83 unmodified lines | |||
| 847 | 875 | ||
| 848 | return -EAGAIN; | 876 | return -EAGAIN; |
| 849 | } | 877 | } |
| 850 | if (ret == -ERESTARTSYS) | ||
| 851 | ret = -EINTR; | ||
| 852 | if (ret > 0 && io_net_retry(sock, flags)) { | 878 | if (ret > 0 && io_net_retry(sock, flags)) { |
| 853 | sr->len -= ret; | 879 | sr->len -= ret; |
| 854 | sr->buf += ret; | 880 | sr->buf += ret; |
| @@ -856,6 +882,8 @@ retry_multishot: · 1 unmodified line | |||
| 856 | req->flags |= REQ_F_PARTIAL_IO; | 882 | req->flags |= REQ_F_PARTIAL_IO; |
| 857 | return -EAGAIN; | 883 | return -EAGAIN; |
| 858 | } | 884 | } |
| 885 | if (ret == -ERESTARTSYS) | ||
| 886 | ret = -EINTR; | ||
| 859 | req_set_fail(req); | 887 | req_set_fail(req); |
| 860 | } else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { | 888 | } else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { |
| 861 | out_free: | 889 | out_free: |
| @@ -879,23 +907,30 @@ out_free: · 17 unmodified lines | |||
| 879 | return ret; | 907 | return ret; |
| 880 | } | 908 | } |
| 881 | 909 | ||
| 882 | void io_sendzc_cleanup(struct io_kiocb *req) | 910 | void io_send_zc_cleanup(struct io_kiocb *req) |
| 883 | { | 911 | { |
| 884 | struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc); | 912 | struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); |
| 913 | struct io_async_msghdr *io; | ||
| 885 | 914 | ||
| 886 | zc->notif->flags |= REQ_F_CQE_SKIP; | 915 | if (req_has_async_data(req)) { |
| 887 | io_notif_flush(zc->notif); | 916 | io = req->async_data; |
| 888 | zc->notif = NULL; | 917 | /* might be ->fast_iov if *msg_copy_hdr failed */ |
| 918 | if (io->free_iov != io->fast_iov) | ||
| 919 | kfree(io->free_iov); | ||
| 920 | } | ||
| 921 | if (zc->notif) { | ||
| 922 | io_notif_flush(zc->notif); | ||
| 923 | zc->notif = NULL; | ||
| 924 | } | ||
| 889 | } | 925 | } |
| 890 | 926 | ||
| 891 | int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) | 927 | int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) |
| 892 | { | 928 | { |
| 893 | struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc); | 929 | struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); |
| 894 | struct io_ring_ctx *ctx = req->ctx; | 930 | struct io_ring_ctx *ctx = req->ctx; |
| 895 | struct io_kiocb *notif; | 931 | struct io_kiocb *notif; |
| 896 | 932 | ||
| 897 | if (READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3) || | 933 | if (unlikely(READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3))) |
| 898 | READ_ONCE(sqe->__pad3[0])) | ||
| 899 | return -EINVAL; | 934 | return -EINVAL; |
| 900 | /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */ | 935 | /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */ |
| 901 | if (req->flags & REQ_F_CQE_SKIP) | 936 | if (req->flags & REQ_F_CQE_SKIP) |
| @@ -922,14 +957,24 @@ int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 20 unmodified lines | |||
| 922 | io_req_set_rsrc_node(notif, ctx, 0); | 957 | io_req_set_rsrc_node(notif, ctx, 0); |
| 923 | } | 958 | } |
| 924 | 959 | ||
| 960 | if (req->opcode == IORING_OP_SEND_ZC) { | ||
| 961 | if (READ_ONCE(sqe->__pad3[0])) | ||
| 962 | return -EINVAL; | ||
| 963 | zc->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); | ||
| 964 | zc->addr_len = READ_ONCE(sqe->addr_len); | ||
| 965 | } else { | ||
| 966 | if (unlikely(sqe->addr2 || sqe->file_index)) | ||
| 967 | return -EINVAL; | ||
| 968 | if (unlikely(zc->flags & IORING_RECVSEND_FIXED_BUF)) | ||
| 969 | return -EINVAL; | ||
| 970 | } | ||
| 971 | |||
| 925 | zc->buf = u64_to_user_ptr(READ_ONCE(sqe->addr)); | 972 | zc->buf = u64_to_user_ptr(READ_ONCE(sqe->addr)); |
| 926 | zc->len = READ_ONCE(sqe->len); | 973 | zc->len = READ_ONCE(sqe->len); |
| 927 | zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; | 974 | zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; |
| 928 | if (zc->msg_flags & MSG_DONTWAIT) | 975 | if (zc->msg_flags & MSG_DONTWAIT) |
| 929 | req->flags |= REQ_F_NOWAIT; | 976 | req->flags |= REQ_F_NOWAIT; |
| 930 | 977 | ||
| 931 | zc->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); | ||
| 932 | zc->addr_len = READ_ONCE(sqe->addr_len); | ||
| 933 | zc->done_io = 0; | 978 | zc->done_io = 0; |
| 934 | 979 | ||
| 935 | #ifdef CONFIG_COMPAT | 980 | #ifdef CONFIG_COMPAT |
| @@ -939,6 +984,13 @@ int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 3 unmodified lines | |||
| 939 | return 0; | 984 | return 0; |
| 940 | } | 985 | } |
| 941 | 986 | ||
| 987 | static int io_sg_from_iter_iovec(struct sock *sk, struct sk_buff *skb, | ||
| 988 | struct iov_iter *from, size_t length) | ||
| 989 | { | ||
| 990 | skb_zcopy_downgrade_managed(skb); | ||
| 991 | return __zerocopy_sg_from_iter(NULL, sk, skb, from, length); | ||
| 992 | } | ||
| 993 | |||
| 942 | static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb, | 994 | static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb, |
| 943 | struct iov_iter *from, size_t length) | 995 | struct iov_iter *from, size_t length) |
| 944 | { | 996 | { |
| @@ -949,13 +1001,10 @@ static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb, · 4 unmodified lines | |||
| 949 | ssize_t copied = 0; | 1001 | ssize_t copied = 0; |
| 950 | unsigned long truesize = 0; | 1002 | unsigned long truesize = 0; |
| 951 | 1003 | ||
| 952 | if (!shinfo->nr_frags) | 1004 | if (!frag) |
| 953 | shinfo->flags |= SKBFL_MANAGED_FRAG_REFS; | 1005 | shinfo->flags |= SKBFL_MANAGED_FRAG_REFS; |
| 954 | 1006 | else if (unlikely(!skb_zcopy_managed(skb))) | |
| 955 | if (!skb_zcopy_managed(skb) || !iov_iter_is_bvec(from)) { | ||
| 956 | skb_zcopy_downgrade_managed(skb); | ||
| 957 | return __zerocopy_sg_from_iter(NULL, sk, skb, from, length); | 1007 | return __zerocopy_sg_from_iter(NULL, sk, skb, from, length); |
| 958 | } | ||
| 959 | 1008 | ||
| 960 | bi.bi_size = min(from->count, length); | 1009 | bi.bi_size = min(from->count, length); |
| 961 | bi.bi_bvec_done = from->iov_offset; | 1010 | bi.bi_bvec_done = from->iov_offset; |
| @@ -993,14 +1042,14 @@ static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb, · 31 unmodified lines | |||
| 993 | return ret; | 1042 | return ret; |
| 994 | } | 1043 | } |
| 995 | 1044 | ||
| 996 | int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) | 1045 | int io_send_zc(struct io_kiocb *req, unsigned int issue_flags) |
| 997 | { | 1046 | { |
| 998 | struct sockaddr_storage __address, *addr = NULL; | 1047 | struct sockaddr_storage __address; |
| 999 | struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc); | 1048 | struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); |
| 1000 | struct msghdr msg; | 1049 | struct msghdr msg; |
| 1001 | struct iovec iov; | 1050 | struct iovec iov; |
| 1002 | struct socket *sock; | 1051 | struct socket *sock; |
| 1003 | unsigned msg_flags, cflags; | 1052 | unsigned msg_flags; |
| 1004 | int ret, min_ret = 0; | 1053 | int ret, min_ret = 0; |
| 1005 | 1054 | ||
| 1006 | sock = sock_from_file(req->file); | 1055 | sock = sock_from_file(req->file); |
| @@ -1016,26 +1065,26 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) · 9 unmodified lines | |||
| 1016 | if (req_has_async_data(req)) { | 1065 | if (req_has_async_data(req)) { |
| 1017 | struct io_async_msghdr *io = req->async_data; | 1066 | struct io_async_msghdr *io = req->async_data; |
| 1018 | 1067 | ||
| 1019 | msg.msg_name = addr = &io->addr; | 1068 | msg.msg_name = &io->addr; |
| 1020 | } else { | 1069 | } else { |
| 1021 | ret = move_addr_to_kernel(zc->addr, zc->addr_len, &__address); | 1070 | ret = move_addr_to_kernel(zc->addr, zc->addr_len, &__address); |
| 1022 | if (unlikely(ret < 0)) | 1071 | if (unlikely(ret < 0)) |
| 1023 | return ret; | 1072 | return ret; |
| 1024 | msg.msg_name = (struct sockaddr *)&__address; | 1073 | msg.msg_name = (struct sockaddr *)&__address; |
| 1025 | addr = &__address; | ||
| 1026 | } | 1074 | } |
| 1027 | msg.msg_namelen = zc->addr_len; | 1075 | msg.msg_namelen = zc->addr_len; |
| 1028 | } | 1076 | } |
| 1029 | 1077 | ||
| 1030 | if (!(req->flags & REQ_F_POLLED) && | 1078 | if (!(req->flags & REQ_F_POLLED) && |
| 1031 | (zc->flags & IORING_RECVSEND_POLL_FIRST)) | 1079 | (zc->flags & IORING_RECVSEND_POLL_FIRST)) |
| 1032 | return io_setup_async_addr(req, addr, issue_flags); | 1080 | return io_setup_async_addr(req, &__address, issue_flags); |
| 1033 | 1081 | ||
| 1034 | if (zc->flags & IORING_RECVSEND_FIXED_BUF) { | 1082 | if (zc->flags & IORING_RECVSEND_FIXED_BUF) { |
| 1035 | ret = io_import_fixed(WRITE, &msg.msg_iter, req->imu, | 1083 | ret = io_import_fixed(WRITE, &msg.msg_iter, req->imu, |
| 1036 | (u64)(uintptr_t)zc->buf, zc->len); | 1084 | (u64)(uintptr_t)zc->buf, zc->len); |
| 1037 | if (unlikely(ret)) | 1085 | if (unlikely(ret)) |
| 1038 | return ret; | 1086 | return ret; |
| 1087 | msg.sg_from_iter = io_sg_from_iter; | ||
| 1039 | } else { | 1088 | } else { |
| 1040 | ret = import_single_range(WRITE, zc->buf, zc->len, &iov, | 1089 | ret = import_single_range(WRITE, zc->buf, zc->len, &iov, |
| 1041 | &msg.msg_iter); | 1090 | &msg.msg_iter); |
| @@ -1044,6 +1093,7 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) · 2 unmodified lines | |||
| 1044 | ret = io_notif_account_mem(zc->notif, zc->len); | 1093 | ret = io_notif_account_mem(zc->notif, zc->len); |
| 1045 | if (unlikely(ret)) | 1094 | if (unlikely(ret)) |
| 1046 | return ret; | 1095 | return ret; |
| 1096 | msg.sg_from_iter = io_sg_from_iter_iovec; | ||
| 1047 | } | 1097 | } |
| 1048 | 1098 | ||
| 1049 | msg_flags = zc->msg_flags | MSG_ZEROCOPY; | 1099 | msg_flags = zc->msg_flags | MSG_ZEROCOPY; |
| @@ -1054,22 +1104,19 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) · 4 unmodified lines | |||
| 1054 | 1104 | ||
| 1055 | msg.msg_flags = msg_flags; | 1105 | msg.msg_flags = msg_flags; |
| 1056 | msg.msg_ubuf = &io_notif_to_data(zc->notif)->uarg; | 1106 | msg.msg_ubuf = &io_notif_to_data(zc->notif)->uarg; |
| 1057 | msg.sg_from_iter = io_sg_from_iter; | ||
| 1058 | ret = sock_sendmsg(sock, &msg); | 1107 | ret = sock_sendmsg(sock, &msg); |
| 1059 | 1108 | ||
| 1060 | if (unlikely(ret < min_ret)) { | 1109 | if (unlikely(ret < min_ret)) { |
| 1061 | if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) | 1110 | if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) |
| 1062 | return io_setup_async_addr(req, addr, issue_flags); | 1111 | return io_setup_async_addr(req, &__address, issue_flags); |
| 1063 | 1112 | ||
| 1064 | if (ret > 0 && io_net_retry(sock, msg.msg_flags)) { | 1113 | if (ret > 0 && io_net_retry(sock, msg.msg_flags)) { |
| 1065 | zc->len -= ret; | 1114 | zc->len -= ret; |
| 1066 | zc->buf += ret; | 1115 | zc->buf += ret; |
| 1067 | zc->done_io += ret; | 1116 | zc->done_io += ret; |
| 1068 | req->flags |= REQ_F_PARTIAL_IO; | 1117 | req->flags |= REQ_F_PARTIAL_IO; |
| 1069 | return io_setup_async_addr(req, addr, issue_flags); | 1118 | return io_setup_async_addr(req, &__address, issue_flags); |
| 1070 | } | 1119 | } |
| 1071 | if (ret < 0 && !zc->done_io) | ||
| 1072 | zc->notif->flags |= REQ_F_CQE_SKIP; | ||
| 1073 | if (ret == -ERESTARTSYS) | 1120 | if (ret == -ERESTARTSYS) |
| 1074 | ret = -EINTR; | 1121 | ret = -EINTR; |
| 1075 | req_set_fail(req); | 1122 | req_set_fail(req); |
| @@ -1080,13 +1127,102 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) · 4 unmodified lines | |||
| 1080 | else if (zc->done_io) | 1127 | else if (zc->done_io) |
| 1081 | ret = zc->done_io; | 1128 | ret = zc->done_io; |
| 1082 | 1129 | ||
| 1083 | io_notif_flush(zc->notif); | 1130 | /* |
| 1084 | req->flags &= ~REQ_F_NEED_CLEANUP; | 1131 | * If we're in io-wq we can't rely on tw ordering guarantees, defer |
| 1085 | cflags = ret >= 0 ? IORING_CQE_F_MORE : 0; | 1132 | * flushing notif to io_send_zc_cleanup() |
| 1086 | io_req_set_res(req, ret, cflags); | 1133 | */ |
| 1134 | if (!(issue_flags & IO_URING_F_UNLOCKED)) { | ||
| 1135 | io_notif_flush(zc->notif); | ||
| 1136 | req->flags &= ~REQ_F_NEED_CLEANUP; | ||
| 1137 | } | ||
| 1138 | io_req_set_res(req, ret, IORING_CQE_F_MORE); | ||
| 1087 | return IOU_OK; | 1139 | return IOU_OK; |
| 1088 | } | 1140 | } |
| 1089 | 1141 | ||
| 1142 | int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags) | ||
| 1143 | { | ||
| 1144 | struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); | ||
| 1145 | struct io_async_msghdr iomsg, *kmsg; | ||
| 1146 | struct socket *sock; | ||
| 1147 | unsigned flags; | ||
| 1148 | int ret, min_ret = 0; | ||
| 1149 | |||
| 1150 | sock = sock_from_file(req->file); | ||
| 1151 | if (unlikely(!sock)) | ||
| 1152 | return -ENOTSOCK; | ||
| 1153 | |||
| 1154 | if (req_has_async_data(req)) { | ||
| 1155 | kmsg = req->async_data; | ||
| 1156 | } else { | ||
| 1157 | ret = io_sendmsg_copy_hdr(req, &iomsg); | ||
| 1158 | if (ret) | ||
| 1159 | return ret; | ||
| 1160 | kmsg = &iomsg; | ||
| 1161 | } | ||
| 1162 | |||
| 1163 | if (!(req->flags & REQ_F_POLLED) && | ||
| 1164 | (sr->flags & IORING_RECVSEND_POLL_FIRST)) | ||
| 1165 | return io_setup_async_msg(req, kmsg, issue_flags); | ||
| 1166 | |||
| 1167 | flags = sr->msg_flags | MSG_ZEROCOPY; | ||
| 1168 | if (issue_flags & IO_URING_F_NONBLOCK) | ||
| 1169 | flags |= MSG_DONTWAIT; | ||
| 1170 | if (flags & MSG_WAITALL) | ||
| 1171 | min_ret = iov_iter_count(&kmsg->msg.msg_iter); | ||
| 1172 | |||
| 1173 | kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg; | ||
| 1174 | kmsg->msg.sg_from_iter = io_sg_from_iter_iovec; | ||
| 1175 | ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); | ||
| 1176 | |||
| 1177 | if (unlikely(ret < min_ret)) { | ||
| 1178 | if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) | ||
| 1179 | return io_setup_async_msg(req, kmsg, issue_flags); | ||
| 1180 | |||
| 1181 | if (ret > 0 && io_net_retry(sock, flags)) { | ||
| 1182 | sr->done_io += ret; | ||
| 1183 | req->flags |= REQ_F_PARTIAL_IO; | ||
| 1184 | return io_setup_async_msg(req, kmsg, issue_flags); | ||
| 1185 | } | ||
| 1186 | if (ret == -ERESTARTSYS) | ||
| 1187 | ret = -EINTR; | ||
| 1188 | req_set_fail(req); | ||
| 1189 | } | ||
| 1190 | /* fast path, check for non-NULL to avoid function call */ | ||
| 1191 | if (kmsg->free_iov) { | ||
| 1192 | kfree(kmsg->free_iov); | ||
| 1193 | kmsg->free_iov = NULL; | ||
| 1194 | } | ||
| 1195 | |||
| 1196 | io_netmsg_recycle(req, issue_flags); | ||
| 1197 | if (ret >= 0) | ||
| 1198 | ret += sr->done_io; | ||
| 1199 | else if (sr->done_io) | ||
| 1200 | ret = sr->done_io; | ||
| 1201 | |||
| 1202 | /* | ||
| 1203 | * If we're in io-wq we can't rely on tw ordering guarantees, defer | ||
| 1204 | * flushing notif to io_send_zc_cleanup() | ||
| 1205 | */ | ||
| 1206 | if (!(issue_flags & IO_URING_F_UNLOCKED)) { | ||
| 1207 | io_notif_flush(sr->notif); | ||
| 1208 | req->flags &= ~REQ_F_NEED_CLEANUP; | ||
| 1209 | } | ||
| 1210 | io_req_set_res(req, ret, IORING_CQE_F_MORE); | ||
| 1211 | return IOU_OK; | ||
| 1212 | } | ||
| 1213 | |||
| 1214 | void io_sendrecv_fail(struct io_kiocb *req) | ||
| 1215 | { | ||
| 1216 | struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); | ||
| 1217 | |||
| 1218 | if (req->flags & REQ_F_PARTIAL_IO) | ||
| 1219 | req->cqe.res = sr->done_io; | ||
| 1220 | |||
| 1221 | if ((req->flags & REQ_F_NEED_CLEANUP) && | ||
| 1222 | (req->opcode == IORING_OP_SEND_ZC || req->opcode == IORING_OP_SENDMSG_ZC)) | ||
| 1223 | req->cqe.flags |= IORING_CQE_F_MORE; | ||
| 1224 | } | ||
| 1225 | |||
| 1090 | int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) | 1226 | int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) |
| 1091 | { | 1227 | { |
| 1092 | struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); | 1228 | struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); |
| @@ -31,18 +31,21 @@ struct io_async_connect { · 30 unmodified lines | |||
| 31 | int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); | 31 | int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); |
| 32 | int io_shutdown(struct io_kiocb *req, unsigned int issue_flags); | 32 | int io_shutdown(struct io_kiocb *req, unsigned int issue_flags); |
| 33 | 33 | ||
| 34 | int io_sendzc_prep_async(struct io_kiocb *req); | ||
| 35 | int io_sendmsg_prep_async(struct io_kiocb *req); | 34 | int io_sendmsg_prep_async(struct io_kiocb *req); |
| 36 | void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req); | 35 | void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req); |
| 37 | int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); | 36 | int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); |
| 38 | int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags); | 37 | int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags); |
| 38 | |||
| 39 | int io_send(struct io_kiocb *req, unsigned int issue_flags); | 39 | int io_send(struct io_kiocb *req, unsigned int issue_flags); |
| 40 | int io_send_prep_async(struct io_kiocb *req); | ||
| 40 | 41 | ||
| 41 | int io_recvmsg_prep_async(struct io_kiocb *req); | 42 | int io_recvmsg_prep_async(struct io_kiocb *req); |
| 42 | int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); | 43 | int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); |
| 43 | int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags); | 44 | int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags); |
| 44 | int io_recv(struct io_kiocb *req, unsigned int issue_flags); | 45 | int io_recv(struct io_kiocb *req, unsigned int issue_flags); |
| 45 | 46 | ||
| 47 | void io_sendrecv_fail(struct io_kiocb *req); | ||
| 48 | |||
| 46 | int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); | 49 | int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); |
| 47 | int io_accept(struct io_kiocb *req, unsigned int issue_flags); | 50 | int io_accept(struct io_kiocb *req, unsigned int issue_flags); |
| 48 | 51 | ||
| @@ -53,9 +56,10 @@ int io_connect_prep_async(struct io_kiocb *req); · 4 unmodified lines | |||
| 53 | int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); | 56 | int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); |
| 54 | int io_connect(struct io_kiocb *req, unsigned int issue_flags); | 57 | int io_connect(struct io_kiocb *req, unsigned int issue_flags); |
| 55 | 58 | ||
| 56 | int io_sendzc(struct io_kiocb *req, unsigned int issue_flags); | 59 | int io_send_zc(struct io_kiocb *req, unsigned int issue_flags); |
| 57 | int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); | 60 | int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags); |
| 58 | void io_sendzc_cleanup(struct io_kiocb *req); | 61 | int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); |
| 62 | void io_send_zc_cleanup(struct io_kiocb *req); | ||
| 59 | 63 | ||
| 60 | void io_netmsg_cache_free(struct io_cache_entry *entry); | 64 | void io_netmsg_cache_free(struct io_cache_entry *entry); |
| 61 | #else | 65 | #else |
| @@ -69,6 +69,7 @@ const struct io_op_def io_op_defs[] = { · 68 unmodified lines | |||
| 69 | .issue = io_read, | 69 | .issue = io_read, |
| 70 | .prep_async = io_readv_prep_async, | 70 | .prep_async = io_readv_prep_async, |
| 71 | .cleanup = io_readv_writev_cleanup, | 71 | .cleanup = io_readv_writev_cleanup, |
| 72 | .fail = io_rw_fail, | ||
| 72 | }, | 73 | }, |
| 73 | [IORING_OP_WRITEV] = { | 74 | [IORING_OP_WRITEV] = { |
| 74 | .needs_file = 1, | 75 | .needs_file = 1, |
| @@ -85,6 +86,7 @@ const struct io_op_def io_op_defs[] = { · 10 unmodified lines | |||
| 85 | .issue = io_write, | 86 | .issue = io_write, |
| 86 | .prep_async = io_writev_prep_async, | 87 | .prep_async = io_writev_prep_async, |
| 87 | .cleanup = io_readv_writev_cleanup, | 88 | .cleanup = io_readv_writev_cleanup, |
| 89 | .fail = io_rw_fail, | ||
| 88 | }, | 90 | }, |
| 89 | [IORING_OP_FSYNC] = { | 91 | [IORING_OP_FSYNC] = { |
| 90 | .needs_file = 1, | 92 | .needs_file = 1, |
| @@ -105,6 +107,7 @@ const struct io_op_def io_op_defs[] = { · 14 unmodified lines | |||
| 105 | .name = "READ_FIXED", | 107 | .name = "READ_FIXED", |
| 106 | .prep = io_prep_rw, | 108 | .prep = io_prep_rw, |
| 107 | .issue = io_read, | 109 | .issue = io_read, |
| 110 | .fail = io_rw_fail, | ||
| 108 | }, | 111 | }, |
| 109 | [IORING_OP_WRITE_FIXED] = { | 112 | [IORING_OP_WRITE_FIXED] = { |
| 110 | .needs_file = 1, | 113 | .needs_file = 1, |
| @@ -119,6 +122,7 @@ const struct io_op_def io_op_defs[] = { · 8 unmodified lines | |||
| 119 | .name = "WRITE_FIXED", | 122 | .name = "WRITE_FIXED", |
| 120 | .prep = io_prep_rw, | 123 | .prep = io_prep_rw, |
| 121 | .issue = io_write, | 124 | .issue = io_write, |
| 125 | .fail = io_rw_fail, | ||
| 122 | }, | 126 | }, |
| 123 | [IORING_OP_POLL_ADD] = { | 127 | [IORING_OP_POLL_ADD] = { |
| 124 | .needs_file = 1, | 128 | .needs_file = 1, |
| @@ -146,6 +150,7 @@ const struct io_op_def io_op_defs[] = { · 21 unmodified lines | |||
| 146 | .unbound_nonreg_file = 1, | 150 | .unbound_nonreg_file = 1, |
| 147 | .pollout = 1, | 151 | .pollout = 1, |
| 148 | .ioprio = 1, | 152 | .ioprio = 1, |
| 153 | .manual_alloc = 1, | ||
| 149 | .name = "SENDMSG", | 154 | .name = "SENDMSG", |
| 150 | #if defined(CONFIG_NET) | 155 | #if defined(CONFIG_NET) |
| 151 | .async_size = sizeof(struct io_async_msghdr), | 156 | .async_size = sizeof(struct io_async_msghdr), |
| @@ -153,6 +158,7 @@ const struct io_op_def io_op_defs[] = { · 1 unmodified line | |||
| 153 | .issue = io_sendmsg, | 158 | .issue = io_sendmsg, |
| 154 | .prep_async = io_sendmsg_prep_async, | 159 | .prep_async = io_sendmsg_prep_async, |
| 155 | .cleanup = io_sendmsg_recvmsg_cleanup, | 160 | .cleanup = io_sendmsg_recvmsg_cleanup, |
| 161 | .fail = io_sendrecv_fail, | ||
| 156 | #else | 162 | #else |
| 157 | .prep = io_eopnotsupp_prep, | 163 | .prep = io_eopnotsupp_prep, |
| 158 | #endif | 164 | #endif |
| @@ -163,6 +169,7 @@ const struct io_op_def io_op_defs[] = { · 4 unmodified lines | |||
| 163 | .pollin = 1, | 169 | .pollin = 1, |
| 164 | .buffer_select = 1, | 170 | .buffer_select = 1, |
| 165 | .ioprio = 1, | 171 | .ioprio = 1, |
| 172 | .manual_alloc = 1, | ||
| 166 | .name = "RECVMSG", | 173 | .name = "RECVMSG", |
| 167 | #if defined(CONFIG_NET) | 174 | #if defined(CONFIG_NET) |
| 168 | .async_size = sizeof(struct io_async_msghdr), | 175 | .async_size = sizeof(struct io_async_msghdr), |
| @@ -170,6 +177,7 @@ const struct io_op_def io_op_defs[] = { · 1 unmodified line | |||
| 170 | .issue = io_recvmsg, | 177 | .issue = io_recvmsg, |
| 171 | .prep_async = io_recvmsg_prep_async, | 178 | .prep_async = io_recvmsg_prep_async, |
| 172 | .cleanup = io_sendmsg_recvmsg_cleanup, | 179 | .cleanup = io_sendmsg_recvmsg_cleanup, |
| 180 | .fail = io_sendrecv_fail, | ||
| 173 | #else | 181 | #else |
| 174 | .prep = io_eopnotsupp_prep, | 182 | .prep = io_eopnotsupp_prep, |
| 175 | #endif | 183 | #endif |
| @@ -273,6 +281,7 @@ const struct io_op_def io_op_defs[] = { · 97 unmodified lines | |||
| 273 | .name = "READ", | 281 | .name = "READ", |
| 274 | .prep = io_prep_rw, | 282 | .prep = io_prep_rw, |
| 275 | .issue = io_read, | 283 | .issue = io_read, |
| 284 | .fail = io_rw_fail, | ||
| 276 | }, | 285 | }, |
| 277 | [IORING_OP_WRITE] = { | 286 | [IORING_OP_WRITE] = { |
| 278 | .needs_file = 1, | 287 | .needs_file = 1, |
| @@ -287,6 +296,7 @@ const struct io_op_def io_op_defs[] = { · 8 unmodified lines | |||
| 287 | .name = "WRITE", | 296 | .name = "WRITE", |
| 288 | .prep = io_prep_rw, | 297 | .prep = io_prep_rw, |
| 289 | .issue = io_write, | 298 | .issue = io_write, |
| 299 | .fail = io_rw_fail, | ||
| 290 | }, | 300 | }, |
| 291 | [IORING_OP_FADVISE] = { | 301 | [IORING_OP_FADVISE] = { |
| 292 | .needs_file = 1, | 302 | .needs_file = 1, |
| @@ -306,10 +316,14 @@ const struct io_op_def io_op_defs[] = { · 13 unmodified lines | |||
| 306 | .pollout = 1, | 316 | .pollout = 1, |
| 307 | .audit_skip = 1, | 317 | .audit_skip = 1, |
| 308 | .ioprio = 1, | 318 | .ioprio = 1, |
| 319 | .manual_alloc = 1, | ||
| 309 | .name = "SEND", | 320 | .name = "SEND", |
| 310 | #if defined(CONFIG_NET) | 321 | #if defined(CONFIG_NET) |
| 322 | .async_size = sizeof(struct io_async_msghdr), | ||
| 311 | .prep = io_sendmsg_prep, | 323 | .prep = io_sendmsg_prep, |
| 312 | .issue = io_send, | 324 | .issue = io_send, |
| 325 | .fail = io_sendrecv_fail, | ||
| 326 | .prep_async = io_send_prep_async, | ||
| 313 | #else | 327 | #else |
| 314 | .prep = io_eopnotsupp_prep, | 328 | .prep = io_eopnotsupp_prep, |
| 315 | #endif | 329 | #endif |
| @@ -325,6 +339,7 @@ const struct io_op_def io_op_defs[] = { · 9 unmodified lines | |||
| 325 | #if defined(CONFIG_NET) | 339 | #if defined(CONFIG_NET) |
| 326 | .prep = io_recvmsg_prep, | 340 | .prep = io_recvmsg_prep, |
| 327 | .issue = io_recv, | 341 | .issue = io_recv, |
| 342 | .fail = io_sendrecv_fail, | ||
| 328 | #else | 343 | #else |
| 329 | .prep = io_eopnotsupp_prep, | 344 | .prep = io_eopnotsupp_prep, |
| 330 | #endif | 345 | #endif |
| @@ -465,6 +480,7 @@ const struct io_op_def io_op_defs[] = { · 134 unmodified lines | |||
| 465 | .needs_file = 1, | 480 | .needs_file = 1, |
| 466 | .plug = 1, | 481 | .plug = 1, |
| 467 | .name = "URING_CMD", | 482 | .name = "URING_CMD", |
| 483 | .iopoll = 1, | ||
| 468 | .async_size = uring_cmd_pdu_size(1), | 484 | .async_size = uring_cmd_pdu_size(1), |
| 469 | .prep = io_uring_cmd_prep, | 485 | .prep = io_uring_cmd_prep, |
| 470 | .issue = io_uring_cmd, | 486 | .issue = io_uring_cmd, |
| @@ -480,10 +496,30 @@ const struct io_op_def io_op_defs[] = { · 9 unmodified lines | |||
| 480 | .manual_alloc = 1, | 496 | .manual_alloc = 1, |
| 481 | #if defined(CONFIG_NET) | 497 | #if defined(CONFIG_NET) |
| 482 | .async_size = sizeof(struct io_async_msghdr), | 498 | .async_size = sizeof(struct io_async_msghdr), |
| 483 | .prep = io_sendzc_prep, | 499 | .prep = io_send_zc_prep, |
| 484 | .issue = io_sendzc, | 500 | .issue = io_send_zc, |
| 485 | .prep_async = io_sendzc_prep_async, | 501 | .prep_async = io_send_prep_async, |
| 486 | .cleanup = io_sendzc_cleanup, | 502 | .cleanup = io_send_zc_cleanup, |
| 503 | .fail = io_sendrecv_fail, | ||
| 504 | #else | ||
| 505 | .prep = io_eopnotsupp_prep, | ||
| 506 | #endif | ||
| 507 | }, | ||
| 508 | [IORING_OP_SENDMSG_ZC] = { | ||
| 509 | .name = "SENDMSG_ZC", | ||
| 510 | .needs_file = 1, | ||
| 511 | .unbound_nonreg_file = 1, | ||
| 512 | .pollout = 1, | ||
| 513 | .audit_skip = 1, | ||
| 514 | .ioprio = 1, | ||
| 515 | .manual_alloc = 1, | ||
| 516 | #if defined(CONFIG_NET) | ||
| 517 | .async_size = sizeof(struct io_async_msghdr), | ||
| 518 | .prep = io_send_zc_prep, | ||
| 519 | .issue = io_sendmsg_zc, | ||
| 520 | .prep_async = io_sendmsg_prep_async, | ||
| 521 | .cleanup = io_send_zc_cleanup, | ||
| 522 | .fail = io_sendrecv_fail, | ||
| 487 | #else | 523 | #else |
| 488 | .prep = io_eopnotsupp_prep, | 524 | .prep = io_eopnotsupp_prep, |
| 489 | #endif | 525 | #endif |
| @@ -36,6 +36,7 @@ struct io_op_def { · 35 unmodified lines | |||
| 36 | int (*issue)(struct io_kiocb *, unsigned int); | 36 | int (*issue)(struct io_kiocb *, unsigned int); |
| 37 | int (*prep_async)(struct io_kiocb *); | 37 | int (*prep_async)(struct io_kiocb *); |
| 38 | void (*cleanup)(struct io_kiocb *); | 38 | void (*cleanup)(struct io_kiocb *); |
| 39 | void (*fail)(struct io_kiocb *); | ||
| 39 | }; | 40 | }; |
| 40 | 41 | ||
| 41 | extern const struct io_op_def io_op_defs[]; | 42 | extern const struct io_op_def io_op_defs[]; |
| @@ -341,7 +341,7 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, · 340 unmodified lines | |||
| 341 | flush_delayed_work(&ctx->rsrc_put_work); | 341 | flush_delayed_work(&ctx->rsrc_put_work); |
| 342 | reinit_completion(&data->done); | 342 | reinit_completion(&data->done); |
| 343 | 343 | ||
| 344 | ret = io_run_task_work_sig(); | 344 | ret = io_run_task_work_sig(ctx); |
| 345 | mutex_lock(&ctx->uring_lock); | 345 | mutex_lock(&ctx->uring_lock); |
| 346 | } while (ret >= 0); | 346 | } while (ret >= 0); |
| 347 | data->quiesce = false; | 347 | data->quiesce = false; |
| @@ -33,6 +33,46 @@ static inline bool io_file_supports_nowait(struct io_kiocb *req) · 32 unmodified lines | |||
| 33 | return req->flags & REQ_F_SUPPORT_NOWAIT; | 33 | return req->flags & REQ_F_SUPPORT_NOWAIT; |
| 34 | } | 34 | } |
| 35 | 35 | ||
| 36 | #ifdef CONFIG_COMPAT | ||
| 37 | static int io_iov_compat_buffer_select_prep(struct io_rw *rw) | ||
| 38 | { | ||
| 39 | struct compat_iovec __user *uiov; | ||
| 40 | compat_ssize_t clen; | ||
| 41 | |||
| 42 | uiov = u64_to_user_ptr(rw->addr); | ||
| 43 | if (!access_ok(uiov, sizeof(*uiov))) | ||
| 44 | return -EFAULT; | ||
| 45 | if (__get_user(clen, &uiov->iov_len)) | ||
| 46 | return -EFAULT; | ||
| 47 | if (clen < 0) | ||
| 48 | return -EINVAL; | ||
| 49 | |||
| 50 | rw->len = clen; | ||
| 51 | return 0; | ||
| 52 | } | ||
| 53 | #endif | ||
| 54 | |||
| 55 | static int io_iov_buffer_select_prep(struct io_kiocb *req) | ||
| 56 | { | ||
| 57 | struct iovec __user *uiov; | ||
| 58 | struct iovec iov; | ||
| 59 | struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); | ||
| 60 | |||
| 61 | if (rw->len != 1) | ||
| 62 | return -EINVAL; | ||
| 63 | |||
| 64 | #ifdef CONFIG_COMPAT | ||
| 65 | if (req->ctx->compat) | ||
| 66 | return io_iov_compat_buffer_select_prep(rw); | ||
| 67 | #endif | ||
| 68 | |||
| 69 | uiov = u64_to_user_ptr(rw->addr); | ||
| 70 | if (copy_from_user(&iov, uiov, sizeof(*uiov))) | ||
| 71 | return -EFAULT; | ||
| 72 | rw->len = iov.iov_len; | ||
| 73 | return 0; | ||
| 74 | } | ||
| 75 | |||
| 36 | int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) | 76 | int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) |
| 37 | { | 77 | { |
| 38 | struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); | 78 | struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); |
| @@ -69,6 +109,16 @@ int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 30 unmodified lines | |||
| 69 | rw->addr = READ_ONCE(sqe->addr); | 109 | rw->addr = READ_ONCE(sqe->addr); |
| 70 | rw->len = READ_ONCE(sqe->len); | 110 | rw->len = READ_ONCE(sqe->len); |
| 71 | rw->flags = READ_ONCE(sqe->rw_flags); | 111 | rw->flags = READ_ONCE(sqe->rw_flags); |
| 112 | |||
| 113 | /* Have to do this validation here, as this is in io_read() rw->len might | ||
| 114 | * have chanaged due to buffer selection | ||
| 115 | */ | ||
| 116 | if (req->opcode == IORING_OP_READV && req->flags & REQ_F_BUFFER_SELECT) { | ||
| 117 | ret = io_iov_buffer_select_prep(req); | ||
| 118 | if (ret) | ||
| 119 | return ret; | ||
| 120 | } | ||
| 121 | |||
| 72 | return 0; | 122 | return 0; |
| 73 | } | 123 | } |
| 74 | 124 | ||
| @@ -186,14 +236,6 @@ static void kiocb_end_write(struct io_kiocb *req) · 111 unmodified lines | |||
| 186 | 236 | ||
| 187 | static bool __io_complete_rw_common(struct io_kiocb *req, long res) | 237 | static bool __io_complete_rw_common(struct io_kiocb *req, long res) |
| 188 | { | 238 | { |
| 189 | struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); | ||
| 190 | |||
| 191 | if (rw->kiocb.ki_flags & IOCB_WRITE) { | ||
| 192 | kiocb_end_write(req); | ||
| 193 | fsnotify_modify(req->file); | ||
| 194 | } else { | ||
| 195 | fsnotify_access(req->file); | ||
| 196 | } | ||
| 197 | if (unlikely(res != req->cqe.res)) { | 239 | if (unlikely(res != req->cqe.res)) { |
| 198 | if ((res == -EAGAIN || res == -EOPNOTSUPP) && | 240 | if ((res == -EAGAIN || res == -EOPNOTSUPP) && |
| 199 | io_rw_should_reissue(req)) { | 241 | io_rw_should_reissue(req)) { |
| @@ -220,6 +262,20 @@ static inline int io_fixup_rw_res(struct io_kiocb *req, long res) · 20 unmodified lines | |||
| 220 | return res; | 262 | return res; |
| 221 | } | 263 | } |
| 222 | 264 | ||
| 265 | static void io_req_rw_complete(struct io_kiocb *req, bool *locked) | ||
| 266 | { | ||
| 267 | struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); | ||
| 268 | |||
| 269 | if (rw->kiocb.ki_flags & IOCB_WRITE) { | ||
| 270 | kiocb_end_write(req); | ||
| 271 | fsnotify_modify(req->file); | ||
| 272 | } else { | ||
| 273 | fsnotify_access(req->file); | ||
| 274 | } | ||
| 275 | |||
| 276 | io_req_task_complete(req, locked); | ||
| 277 | } | ||
| 278 | |||
| 223 | static void io_complete_rw(struct kiocb *kiocb, long res) | 279 | static void io_complete_rw(struct kiocb *kiocb, long res) |
| 224 | { | 280 | { |
| 225 | struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb); | 281 | struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb); |
| @@ -228,7 +284,7 @@ static void io_complete_rw(struct kiocb *kiocb, long res) · 2 unmodified lines | |||
| 228 | if (__io_complete_rw_common(req, res)) | 284 | if (__io_complete_rw_common(req, res)) |
| 229 | return; | 285 | return; |
| 230 | io_req_set_res(req, io_fixup_rw_res(req, res), 0); | 286 | io_req_set_res(req, io_fixup_rw_res(req, res), 0); |
| 231 | req->io_task_work.func = io_req_task_complete; | 287 | req->io_task_work.func = io_req_rw_complete; |
| 232 | io_req_task_work_add(req); | 288 | io_req_task_work_add(req); |
| 233 | } | 289 | } |
| 234 | 290 | ||
| @@ -279,79 +335,6 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret, · 44 unmodified lines | |||
| 279 | return IOU_ISSUE_SKIP_COMPLETE; | 335 | return IOU_ISSUE_SKIP_COMPLETE; |
| 280 | } | 336 | } |
| 281 | 337 | ||
| 282 | #ifdef CONFIG_COMPAT | ||
| 283 | static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov, | ||
| 284 | unsigned int issue_flags) | ||
| 285 | { | ||
| 286 | struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); | ||
| 287 | struct compat_iovec __user *uiov; | ||
| 288 | compat_ssize_t clen; | ||
| 289 | void __user *buf; | ||
| 290 | size_t len; | ||
| 291 | |||
| 292 | uiov = u64_to_user_ptr(rw->addr); | ||
| 293 | if (!access_ok(uiov, sizeof(*uiov))) | ||
| 294 | return -EFAULT; | ||
| 295 | if (__get_user(clen, &uiov->iov_len)) | ||
| 296 | return -EFAULT; | ||
| 297 | if (clen < 0) | ||
| 298 | return -EINVAL; | ||
| 299 | |||
| 300 | len = clen; | ||
| 301 | buf = io_buffer_select(req, &len, issue_flags); | ||
| 302 | if (!buf) | ||
| 303 | return -ENOBUFS; | ||
| 304 | rw->addr = (unsigned long) buf; | ||
| 305 | iov[0].iov_base = buf; | ||
| 306 | rw->len = iov[0].iov_len = (compat_size_t) len; | ||
| 307 | return 0; | ||
| 308 | } | ||
| 309 | #endif | ||
| 310 | |||
| 311 | static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, | ||
| 312 | unsigned int issue_flags) | ||
| 313 | { | ||
| 314 | struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); | ||
| 315 | struct iovec __user *uiov = u64_to_user_ptr(rw->addr); | ||
| 316 | void __user *buf; | ||
| 317 | ssize_t len; | ||
| 318 | |||
| 319 | if (copy_from_user(iov, uiov, sizeof(*uiov))) | ||
| 320 | return -EFAULT; | ||
| 321 | |||
| 322 | len = iov[0].iov_len; | ||
| 323 | if (len < 0) | ||
| 324 | return -EINVAL; | ||
| 325 | buf = io_buffer_select(req, &len, issue_flags); | ||
| 326 | if (!buf) | ||
| 327 | return -ENOBUFS; | ||
| 328 | rw->addr = (unsigned long) buf; | ||
| 329 | iov[0].iov_base = buf; | ||
| 330 | rw->len = iov[0].iov_len = len; | ||
| 331 | return 0; | ||
| 332 | } | ||
| 333 | |||
| 334 | static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, | ||
| 335 | unsigned int issue_flags) | ||
| 336 | { | ||
| 337 | struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); | ||
| 338 | |||
| 339 | if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) { | ||
| 340 | iov[0].iov_base = u64_to_user_ptr(rw->addr); | ||
| 341 | iov[0].iov_len = rw->len; | ||
| 342 | return 0; | ||
| 343 | } | ||
| 344 | if (rw->len != 1) | ||
| 345 | return -EINVAL; | ||
| 346 | |||
| 347 | #ifdef CONFIG_COMPAT | ||
| 348 | if (req->ctx->compat) | ||
| 349 | return io_compat_import(req, iov, issue_flags); | ||
| 350 | #endif | ||
| 351 | |||
| 352 | return __io_iov_buffer_select(req, iov, issue_flags); | ||
| 353 | } | ||
| 354 | |||
| 355 | static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, | 338 | static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, |
| 356 | struct io_rw_state *s, | 339 | struct io_rw_state *s, |
| 357 | unsigned int issue_flags) | 340 | unsigned int issue_flags) |
| @@ -374,7 +357,8 @@ static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, · 16 unmodified lines | |||
| 374 | buf = u64_to_user_ptr(rw->addr); | 357 | buf = u64_to_user_ptr(rw->addr); |
| 375 | sqe_len = rw->len; | 358 | sqe_len = rw->len; |
| 376 | 359 | ||
| 377 | if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) { | 360 | if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE || |
| 361 | (req->flags & REQ_F_BUFFER_SELECT)) { | ||
| 378 | if (io_do_buffer_select(req)) { | 362 | if (io_do_buffer_select(req)) { |
| 379 | buf = io_buffer_select(req, &sqe_len, issue_flags); | 363 | buf = io_buffer_select(req, &sqe_len, issue_flags); |
| 380 | if (!buf) | 364 | if (!buf) |
| @@ -390,14 +374,6 @@ static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, · 9 unmodified lines | |||
| 390 | } | 374 | } |
| 391 | 375 | ||
| 392 | iovec = s->fast_iov; | 376 | iovec = s->fast_iov; |
| 393 | if (req->flags & REQ_F_BUFFER_SELECT) { | ||
| 394 | ret = io_iov_buffer_select(req, iovec, issue_flags); | ||
| 395 | if (ret) | ||
| 396 | return ERR_PTR(ret); | ||
| 397 | iov_iter_init(iter, ddir, iovec, 1, iovec->iov_len); | ||
| 398 | return NULL; | ||
| 399 | } | ||
| 400 | |||
| 401 | ret = __import_iovec(ddir, buf, sqe_len, UIO_FASTIOV, &iovec, iter, | 377 | ret = __import_iovec(ddir, buf, sqe_len, UIO_FASTIOV, &iovec, iter, |
| 402 | req->ctx->compat); | 378 | req->ctx->compat); |
| 403 | if (unlikely(ret < 0)) | 379 | if (unlikely(ret < 0)) |
| @@ -794,10 +770,12 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) · 390 unmodified lines | |||
| 794 | iov_iter_restore(&s->iter, &s->iter_state); | 770 | iov_iter_restore(&s->iter, &s->iter_state); |
| 795 | 771 | ||
| 796 | ret2 = io_setup_async_rw(req, iovec, s, true); | 772 | ret2 = io_setup_async_rw(req, iovec, s, true); |
| 797 | if (ret2) | ||
| 798 | return ret2; | ||
| 799 | |||
| 800 | iovec = NULL; | 773 | iovec = NULL; |
| 774 | if (ret2) { | ||
| 775 | ret = ret > 0 ? ret : ret2; | ||
| 776 | goto done; | ||
| 777 | } | ||
| 778 | |||
| 801 | io = req->async_data; | 779 | io = req->async_data; |
| 802 | s = &io->s; | 780 | s = &io->s; |
| 803 | /* | 781 | /* |
| @@ -823,6 +801,7 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) · 19 unmodified lines | |||
| 823 | return -EAGAIN; | 801 | return -EAGAIN; |
| 824 | } | 802 | } |
| 825 | 803 | ||
| 804 | req->cqe.res = iov_iter_count(&s->iter); | ||
| 826 | /* | 805 | /* |
| 827 | * Now retry read with the IOCB_WAITQ parts set in the iocb. If | 806 | * Now retry read with the IOCB_WAITQ parts set in the iocb. If |
| 828 | * we get -EIOCBQUEUED, then we'll get a notification when the | 807 | * we get -EIOCBQUEUED, then we'll get a notification when the |
| @@ -984,6 +963,14 @@ static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx) · 155 unmodified lines | |||
| 984 | io_cqring_wake(ctx); | 963 | io_cqring_wake(ctx); |
| 985 | } | 964 | } |
| 986 | 965 | ||
| 966 | void io_rw_fail(struct io_kiocb *req) | ||
| 967 | { | ||
| 968 | int res; | ||
| 969 | |||
| 970 | res = io_fixup_rw_res(req, req->cqe.res); | ||
| 971 | io_req_set_res(req, res, req->cqe.flags); | ||
| 972 | } | ||
| 973 | |||
| 987 | int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) | 974 | int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) |
| 988 | { | 975 | { |
| 989 | struct io_wq_work_node *pos, *start, *prev; | 976 | struct io_wq_work_node *pos, *start, *prev; |
| @@ -1000,7 +987,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) · 10 unmodified lines | |||
| 1000 | 987 | ||
| 1001 | wq_list_for_each(pos, start, &ctx->iopoll_list) { | 988 | wq_list_for_each(pos, start, &ctx->iopoll_list) { |
| 1002 | struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list); | 989 | struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list); |
| 1003 | struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); | 990 | struct file *file = req->file; |
| 1004 | int ret; | 991 | int ret; |
| 1005 | 992 | ||
| 1006 | /* | 993 | /* |
| @@ -1011,7 +998,17 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) · 4 unmodified lines | |||
| 1011 | if (READ_ONCE(req->iopoll_completed)) | 998 | if (READ_ONCE(req->iopoll_completed)) |
| 1012 | break; | 999 | break; |
| 1013 | 1000 | ||
| 1014 | ret = rw->kiocb.ki_filp->f_op->iopoll(&rw->kiocb, &iob, poll_flags); | 1001 | if (req->opcode == IORING_OP_URING_CMD) { |
| 1002 | struct io_uring_cmd *ioucmd; | ||
| 1003 | |||
| 1004 | ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); | ||
| 1005 | ret = file->f_op->uring_cmd_iopoll(ioucmd, &iob, | ||
| 1006 | poll_flags); | ||
| 1007 | } else { | ||
| 1008 | struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); | ||
| 1009 | |||
| 1010 | ret = file->f_op->iopoll(&rw->kiocb, &iob, poll_flags); | ||
| 1011 | } | ||
| 1015 | if (unlikely(ret < 0)) | 1012 | if (unlikely(ret < 0)) |
| 1016 | return ret; | 1013 | return ret; |
| 1017 | else if (ret) | 1014 | else if (ret) |
| @@ -21,3 +21,4 @@ int io_readv_prep_async(struct io_kiocb *req); · 20 unmodified lines | |||
| 21 | int io_write(struct io_kiocb *req, unsigned int issue_flags); | 21 | int io_write(struct io_kiocb *req, unsigned int issue_flags); |
| 22 | int io_writev_prep_async(struct io_kiocb *req); | 22 | int io_writev_prep_async(struct io_kiocb *req); |
| 23 | void io_readv_writev_cleanup(struct io_kiocb *req); | 23 | void io_readv_writev_cleanup(struct io_kiocb *req); |
| 24 | void io_rw_fail(struct io_kiocb *req); | ||
| @@ -149,11 +149,10 @@ static inline void io_remove_next_linked(struct io_kiocb *req) · 148 unmodified lines | |||
| 149 | nxt->link = NULL; | 149 | nxt->link = NULL; |
| 150 | } | 150 | } |
| 151 | 151 | ||
| 152 | bool io_disarm_next(struct io_kiocb *req) | 152 | void io_disarm_next(struct io_kiocb *req) |
| 153 | __must_hold(&req->ctx->completion_lock) | 153 | __must_hold(&req->ctx->completion_lock) |
| 154 | { | 154 | { |
| 155 | struct io_kiocb *link = NULL; | 155 | struct io_kiocb *link = NULL; |
| 156 | bool posted = false; | ||
| 157 | 156 | ||
| 158 | if (req->flags & REQ_F_ARM_LTIMEOUT) { | 157 | if (req->flags & REQ_F_ARM_LTIMEOUT) { |
| 159 | link = req->link; | 158 | link = req->link; |
| @@ -161,7 +160,6 @@ bool io_disarm_next(struct io_kiocb *req) · 1 unmodified line | |||
| 161 | if (link && link->opcode == IORING_OP_LINK_TIMEOUT) { | 160 | if (link && link->opcode == IORING_OP_LINK_TIMEOUT) { |
| 162 | io_remove_next_linked(req); | 161 | io_remove_next_linked(req); |
| 163 | io_req_tw_post_queue(link, -ECANCELED, 0); | 162 | io_req_tw_post_queue(link, -ECANCELED, 0); |
| 164 | posted = true; | ||
| 165 | } | 163 | } |
| 166 | } else if (req->flags & REQ_F_LINK_TIMEOUT) { | 164 | } else if (req->flags & REQ_F_LINK_TIMEOUT) { |
| 167 | struct io_ring_ctx *ctx = req->ctx; | 165 | struct io_ring_ctx *ctx = req->ctx; |
| @@ -169,17 +167,12 @@ bool io_disarm_next(struct io_kiocb *req) · 1 unmodified line | |||
| 169 | spin_lock_irq(&ctx->timeout_lock); | 167 | spin_lock_irq(&ctx->timeout_lock); |
| 170 | link = io_disarm_linked_timeout(req); | 168 | link = io_disarm_linked_timeout(req); |
| 171 | spin_unlock_irq(&ctx->timeout_lock); | 169 | spin_unlock_irq(&ctx->timeout_lock); |
| 172 | if (link) { | 170 | if (link) |
| 173 | posted = true; | ||
| 174 | io_req_tw_post_queue(link, -ECANCELED, 0); | 171 | io_req_tw_post_queue(link, -ECANCELED, 0); |
| 175 | } | ||
| 176 | } | 172 | } |
| 177 | if (unlikely((req->flags & REQ_F_FAIL) && | 173 | if (unlikely((req->flags & REQ_F_FAIL) && |
| 178 | !(req->flags & REQ_F_HARDLINK))) { | 174 | !(req->flags & REQ_F_HARDLINK))) |
| 179 | posted |= (req->link != NULL); | ||
| 180 | io_fail_links(req); | 175 | io_fail_links(req); |
| 181 | } | ||
| 182 | return posted; | ||
| 183 | } | 176 | } |
| 184 | 177 | ||
| 185 | struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req, | 178 | struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req, |
| @@ -27,7 +27,7 @@ int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd); · 26 unmodified lines | |||
| 27 | __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, | 27 | __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, |
| 28 | bool cancel_all); | 28 | bool cancel_all); |
| 29 | void io_queue_linked_timeout(struct io_kiocb *req); | 29 | void io_queue_linked_timeout(struct io_kiocb *req); |
| 30 | bool io_disarm_next(struct io_kiocb *req); | 30 | void io_disarm_next(struct io_kiocb *req); |
| 31 | 31 | ||
| 32 | int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); | 32 | int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); |
| 33 | int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); | 33 | int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); |
| @@ -50,7 +50,11 @@ void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2) · 49 unmodified lines | |||
| 50 | io_req_set_res(req, ret, 0); | 50 | io_req_set_res(req, ret, 0); |
| 51 | if (req->ctx->flags & IORING_SETUP_CQE32) | 51 | if (req->ctx->flags & IORING_SETUP_CQE32) |
| 52 | io_req_set_cqe32_extra(req, res2, 0); | 52 | io_req_set_cqe32_extra(req, res2, 0); |
| 53 | __io_req_complete(req, 0); | 53 | if (req->ctx->flags & IORING_SETUP_IOPOLL) |
| 54 | /* order with io_iopoll_req_issued() checking ->iopoll_complete */ | ||
| 55 | smp_store_release(&req->iopoll_completed, 1); | ||
| 56 | else | ||
| 57 | __io_req_complete(req, 0); | ||
| 54 | } | 58 | } |
| 55 | EXPORT_SYMBOL_GPL(io_uring_cmd_done); | 59 | EXPORT_SYMBOL_GPL(io_uring_cmd_done); |
| 56 | 60 | ||
| @@ -97,8 +101,11 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) · 40 unmodified lines | |||
| 97 | issue_flags |= IO_URING_F_SQE128; | 101 | issue_flags |= IO_URING_F_SQE128; |
| 98 | if (ctx->flags & IORING_SETUP_CQE32) | 102 | if (ctx->flags & IORING_SETUP_CQE32) |
| 99 | issue_flags |= IO_URING_F_CQE32; | 103 | issue_flags |= IO_URING_F_CQE32; |
| 100 | if (ctx->flags & IORING_SETUP_IOPOLL) | 104 | if (ctx->flags & IORING_SETUP_IOPOLL) { |
| 101 | issue_flags |= IO_URING_F_IOPOLL; | 105 | issue_flags |= IO_URING_F_IOPOLL; |
| 106 | req->iopoll_completed = 0; | ||
| 107 | WRITE_ONCE(ioucmd->cookie, NULL); | ||
| 108 | } | ||
| 102 | 109 | ||
| 103 | if (req_has_async_data(req)) | 110 | if (req_has_async_data(req)) |
| 104 | ioucmd->cmd = req->async_data; | 111 | ioucmd->cmd = req->async_data; |
| @@ -400,7 +400,6 @@ static void do_tx(int domain, int type, int protocol) · 399 unmodified lines | |||
| 400 | cfg_payload_len, msg_flags); | 400 | cfg_payload_len, msg_flags); |
| 401 | sqe->user_data = NONZC_TAG; | 401 | sqe->user_data = NONZC_TAG; |
| 402 | } else { | 402 | } else { |
| 403 | compl_cqes++; | ||
| 404 | io_uring_prep_sendzc(sqe, fd, payload, | 403 | io_uring_prep_sendzc(sqe, fd, payload, |
| 405 | cfg_payload_len, | 404 | cfg_payload_len, |
| 406 | msg_flags, zc_flags); | 405 | msg_flags, zc_flags); |
| @@ -430,18 +429,23 @@ static void do_tx(int domain, int type, int protocol) · 23 unmodified lines | |||
| 430 | if (cqe->flags & IORING_CQE_F_NOTIF) { | 429 | if (cqe->flags & IORING_CQE_F_NOTIF) { |
| 431 | if (cqe->flags & IORING_CQE_F_MORE) | 430 | if (cqe->flags & IORING_CQE_F_MORE) |
| 432 | error(1, -EINVAL, "invalid notif flags"); | 431 | error(1, -EINVAL, "invalid notif flags"); |
| 432 | if (compl_cqes <= 0) | ||
| 433 | error(1, -EINVAL, "notification mismatch"); | ||
| 433 | compl_cqes--; | 434 | compl_cqes--; |
| 434 | i--; | 435 | i--; |
| 435 | } else if (cqe->res <= 0) { | 436 | io_uring_cqe_seen(&ring); |
| 436 | if (cqe->flags & IORING_CQE_F_MORE) | 437 | continue; |
| 437 | error(1, cqe->res, "more with a failed send"); | 438 | } |
| 438 | error(1, cqe->res, "send failed"); | 439 | if (cqe->flags & IORING_CQE_F_MORE) { |
| 439 | } else { | 440 | if (cqe->user_data != ZC_TAG) |
| 440 | if (cqe->user_data == ZC_TAG && | 441 | error(1, cqe->res, "unexpected F_MORE"); |
| 441 | !(cqe->flags & IORING_CQE_F_MORE)) | 442 | compl_cqes++; |
| 442 | error(1, cqe->res, "missing more flag"); | 443 | } |
| 444 | if (cqe->res >= 0) { | ||
| 443 | packets++; | 445 | packets++; |
| 444 | bytes += cqe->res; | 446 | bytes += cqe->res; |
| 447 | } else if (cqe->res != -EAGAIN) { | ||
| 448 | error(1, cqe->res, "send failed"); | ||
| 445 | } | 449 | } |
| 446 | io_uring_cqe_seen(&ring); | 450 | io_uring_cqe_seen(&ring); |
| 447 | } | 451 | } |