| @@ -679,6 +679,13 @@ static void io_put_req(struct io_kiocb *req) · 678 unmodified lines |
| 679 | io_free_req(req); | 679 | io_free_req(req); |
| 680 | } | 680 | } |
| 681 | | 681 | |
| | 682 | static unsigned io_cqring_events(struct io_cq_ring *ring) |
| | 683 | { |
| | 684 | /* See comment at the top of this file */ |
| | 685 | smp_rmb(); |
| | 686 | return READ_ONCE(ring->r.tail) - READ_ONCE(ring->r.head); |
| | 687 | } |
| | 688 | |
| 682 | /* | 689 | /* |
| 683 | * Find and free completed poll iocbs | 690 | * Find and free completed poll iocbs |
| 684 | */ | 691 | */ |
| @@ -771,7 +778,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, · 86 unmodified lines |
| 771 | static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events, | 778 | static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events, |
| 772 | long min) | 779 | long min) |
| 773 | { | 780 | { |
| 774 | while (!list_empty(&ctx->poll_list)) { | 781 | while (!list_empty(&ctx->poll_list) && !need_resched()) { |
| 775 | int ret; | 782 | int ret; |
| 776 | | 783 | |
| 777 | ret = io_do_iopoll(ctx, nr_events, min); | 784 | ret = io_do_iopoll(ctx, nr_events, min); |
| @@ -798,6 +805,12 @@ static void io_iopoll_reap_events(struct io_ring_ctx *ctx) · 20 unmodified lines |
| 798 | unsigned int nr_events = 0; | 805 | unsigned int nr_events = 0; |
| 799 | | 806 | |
| 800 | io_iopoll_getevents(ctx, &nr_events, 1); | 807 | io_iopoll_getevents(ctx, &nr_events, 1); |
| | 808 | |
| | 809 | /* |
| | 810 | * Ensure we allow local-to-the-cpu processing to take place, |
| | 811 | * in this case we need to ensure that we reap all events. |
| | 812 | */ |
| | 813 | cond_resched(); |
| 801 | } | 814 | } |
| 802 | mutex_unlock(&ctx->uring_lock); | 815 | mutex_unlock(&ctx->uring_lock); |
| 803 | } | 816 | } |
| @@ -805,11 +818,42 @@ static void io_iopoll_reap_events(struct io_ring_ctx *ctx) · 1 unmodified line |
| 805 | static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events, | 818 | static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events, |
| 806 | long min) | 819 | long min) |
| 807 | { | 820 | { |
| 808 | int ret = 0; | 821 | int iters, ret = 0; |
| 809 | | 822 | |
| | 823 | /* |
| | 824 | * We disallow the app entering submit/complete with polling, but we |
| | 825 | * still need to lock the ring to prevent racing with polled issue |
| | 826 | * that got punted to a workqueue. |
| | 827 | */ |
| | 828 | mutex_lock(&ctx->uring_lock); |
| | 829 | |
| | 830 | iters = 0; |
| 810 | do { | 831 | do { |
| 811 | int tmin = 0; | 832 | int tmin = 0; |
| 812 | | 833 | |
| | 834 | /* |
| | 835 | * Don't enter poll loop if we already have events pending. |
| | 836 | * If we do, we can potentially be spinning for commands that |
| | 837 | * already triggered a CQE (eg in error). |
| | 838 | */ |
| | 839 | if (io_cqring_events(ctx->cq_ring)) |
| | 840 | break; |
| | 841 | |
| | 842 | /* |
| | 843 | * If a submit got punted to a workqueue, we can have the |
| | 844 | * application entering polling for a command before it gets |
| | 845 | * issued. That app will hold the uring_lock for the duration |
| | 846 | * of the poll right here, so we need to take a breather every |
| | 847 | * now and then to ensure that the issue has a chance to add |
| | 848 | * the poll to the issued list. Otherwise we can spin here |
| | 849 | * forever, while the workqueue is stuck trying to acquire the |
| | 850 | * very same mutex. |
| | 851 | */ |
| | 852 | if (!(++iters & 7)) { |
| | 853 | mutex_unlock(&ctx->uring_lock); |
| | 854 | mutex_lock(&ctx->uring_lock); |
| | 855 | } |
| | 856 | |
| 813 | if (*nr_events < min) | 857 | if (*nr_events < min) |
| 814 | tmin = min - *nr_events; | 858 | tmin = min - *nr_events; |
| 815 | | 859 | |
| @@ -819,6 +863,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events, · 3 unmodified lines |
| 819 | ret = 0; | 863 | ret = 0; |
| 820 | } while (min && !*nr_events && !need_resched()); | 864 | } while (min && !*nr_events && !need_resched()); |
| 821 | | 865 | |
| | 866 | mutex_unlock(&ctx->uring_lock); |
| 822 | return ret; | 867 | return ret; |
| 823 | } | 868 | } |
| 824 | | 869 | |
| @@ -2280,15 +2325,7 @@ static int io_sq_thread(void *data) · 1455 unmodified lines |
| 2280 | unsigned nr_events = 0; | 2325 | unsigned nr_events = 0; |
| 2281 | | 2326 | |
| 2282 | if (ctx->flags & IORING_SETUP_IOPOLL) { | 2327 | if (ctx->flags & IORING_SETUP_IOPOLL) { |
| 2283 | /* | | |
| 2284 | * We disallow the app entering submit/complete | | |
| 2285 | * with polling, but we still need to lock the | | |
| 2286 | * ring to prevent racing with polled issue | | |
| 2287 | * that got punted to a workqueue. | | |
| 2288 | */ | | |
| 2289 | mutex_lock(&ctx->uring_lock); | | |
| 2290 | io_iopoll_check(ctx, &nr_events, 0); | 2328 | io_iopoll_check(ctx, &nr_events, 0); |
| 2291 | mutex_unlock(&ctx->uring_lock); | | |
| 2292 | } else { | 2329 | } else { |
| 2293 | /* | 2330 | /* |
| 2294 | * Normal IO, just pretend everything completed. | 2331 | * Normal IO, just pretend everything completed. |
| @@ -2433,13 +2470,6 @@ static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit) · 138 unmodified lines |
| 2433 | return submit; | 2470 | return submit; |
| 2434 | } | 2471 | } |
| 2435 | | 2472 | |
| 2436 | static unsigned io_cqring_events(struct io_cq_ring *ring) | | |
| 2437 | { | | |
| 2438 | /* See comment at the top of this file */ | | |
| 2439 | smp_rmb(); | | |
| 2440 | return READ_ONCE(ring->r.tail) - READ_ONCE(ring->r.head); | | |
| 2441 | } | | |
| 2442 | | | |
| 2443 | /* | 2473 | /* |
| 2444 | * Wait until events become available, if we don't already have some. The | 2474 | * Wait until events become available, if we don't already have some. The |
| 2445 | * application must reap them itself, as they reside on the shared cq ring. | 2475 | * application must reap them itself, as they reside on the shared cq ring. |
| @@ -3190,9 +3220,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 744 unmodified lines |
| 3190 | min_complete = min(min_complete, ctx->cq_entries); | 3220 | min_complete = min(min_complete, ctx->cq_entries); |
| 3191 | | 3221 | |
| 3192 | if (ctx->flags & IORING_SETUP_IOPOLL) { | 3222 | if (ctx->flags & IORING_SETUP_IOPOLL) { |
| 3193 | mutex_lock(&ctx->uring_lock); | | |
| 3194 | ret = io_iopoll_check(ctx, &nr_events, min_complete); | 3223 | ret = io_iopoll_check(ctx, &nr_events, min_complete); |
| 3195 | mutex_unlock(&ctx->uring_lock); | | |
| 3196 | } else { | 3224 | } else { |
| 3197 | ret = io_cqring_wait(ctx, min_complete, sig, sigsz); | 3225 | ret = io_cqring_wait(ctx, min_complete, sig, sigsz); |
| 3198 | } | 3226 | } |