| @@ -679,6 +679,13 @@ static void io_put_req(struct io_kiocb *req) · 678 unmodified lines |
| 679 | 679 | io_free_req(req); |
| 680 | 680 | } |
| 681 | 681 | |
| 682 | +static unsigned io_cqring_events(struct io_cq_ring *ring) |
| 683 | +{ |
| 684 | + /* See comment at the top of this file */ |
| 685 | + smp_rmb(); |
| 686 | + return READ_ONCE(ring->r.tail) - READ_ONCE(ring->r.head); |
| 687 | +} |
| 688 | + |
| 682 | 689 | /* |
| 683 | 690 | * Find and free completed poll iocbs |
| 684 | 691 | */ |
| @@ -771,7 +778,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, · 86 unmodified lines |
| 771 | 778 | static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events, |
| 772 | 779 | long min) |
| 773 | 780 | { |
| 774 | | − while (!list_empty(&ctx->poll_list)) { |
| 781 | + while (!list_empty(&ctx->poll_list) && !need_resched()) { |
| 775 | 782 | int ret; |
| 776 | 783 | |
| 777 | 784 | ret = io_do_iopoll(ctx, nr_events, min); |
| @@ -798,6 +805,12 @@ static void io_iopoll_reap_events(struct io_ring_ctx *ctx) · 20 unmodified lines |
| 798 | 805 | unsigned int nr_events = 0; |
| 799 | 806 | |
| 800 | 807 | io_iopoll_getevents(ctx, &nr_events, 1); |
| 808 | + |
| 809 | + /* |
| 810 | + * Ensure we allow local-to-the-cpu processing to take place, |
| 811 | + * in this case we need to ensure that we reap all events. |
| 812 | + */ |
| 813 | + cond_resched(); |
| 801 | 814 | } |
| 802 | 815 | mutex_unlock(&ctx->uring_lock); |
| 803 | 816 | } |
| @@ -805,11 +818,42 @@ static void io_iopoll_reap_events(struct io_ring_ctx *ctx) · 1 unmodified line |
| 805 | 818 | static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events, |
| 806 | 819 | long min) |
| 807 | 820 | { |
| 808 | | − int ret = 0; |
| 821 | + int iters, ret = 0; |
| 809 | 822 | |
| 823 | + /* |
| 824 | + * We disallow the app entering submit/complete with polling, but we |
| 825 | + * still need to lock the ring to prevent racing with polled issue |
| 826 | + * that got punted to a workqueue. |
| 827 | + */ |
| 828 | + mutex_lock(&ctx->uring_lock); |
| 829 | + |
| 830 | + iters = 0; |
| 810 | 831 | do { |
| 811 | 832 | int tmin = 0; |
| 812 | 833 | |
| 834 | + /* |
| 835 | + * Don't enter poll loop if we already have events pending. |
| 836 | + * If we do, we can potentially be spinning for commands that |
| 837 | + * already triggered a CQE (eg in error). |
| 838 | + */ |
| 839 | + if (io_cqring_events(ctx->cq_ring)) |
| 840 | + break; |
| 841 | + |
| 842 | + /* |
| 843 | + * If a submit got punted to a workqueue, we can have the |
| 844 | + * application entering polling for a command before it gets |
| 845 | + * issued. That app will hold the uring_lock for the duration |
| 846 | + * of the poll right here, so we need to take a breather every |
| 847 | + * now and then to ensure that the issue has a chance to add |
| 848 | + * the poll to the issued list. Otherwise we can spin here |
| 849 | + * forever, while the workqueue is stuck trying to acquire the |
| 850 | + * very same mutex. |
| 851 | + */ |
| 852 | + if (!(++iters & 7)) { |
| 853 | + mutex_unlock(&ctx->uring_lock); |
| 854 | + mutex_lock(&ctx->uring_lock); |
| 855 | + } |
| 856 | + |
| 813 | 857 | if (*nr_events < min) |
| 814 | 858 | tmin = min - *nr_events; |
| 815 | 859 | |
| @@ -819,6 +863,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events, · 3 unmodified lines |
| 819 | 863 | ret = 0; |
| 820 | 864 | } while (min && !*nr_events && !need_resched()); |
| 821 | 865 | |
| 866 | + mutex_unlock(&ctx->uring_lock); |
| 822 | 867 | return ret; |
| 823 | 868 | } |
| 824 | 869 | |
| @@ -2280,15 +2325,7 @@ static int io_sq_thread(void *data) · 1455 unmodified lines |
| 2280 | 2325 | unsigned nr_events = 0; |
| 2281 | 2326 | |
| 2282 | 2327 | if (ctx->flags & IORING_SETUP_IOPOLL) { |
| 2283 | | − /* |
| 2284 | | − * We disallow the app entering submit/complete |
| 2285 | | − * with polling, but we still need to lock the |
| 2286 | | − * ring to prevent racing with polled issue |
| 2287 | | − * that got punted to a workqueue. |
| 2288 | | − */ |
| 2289 | | − mutex_lock(&ctx->uring_lock); |
| 2290 | 2328 | io_iopoll_check(ctx, &nr_events, 0); |
| 2291 | | − mutex_unlock(&ctx->uring_lock); |
| 2292 | 2329 | } else { |
| 2293 | 2330 | /* |
| 2294 | 2331 | * Normal IO, just pretend everything completed. |
| @@ -2433,13 +2470,6 @@ static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit) · 138 unmodified lines |
| 2433 | 2470 | return submit; |
| 2434 | 2471 | } |
| 2435 | 2472 | |
| 2436 | | −static unsigned io_cqring_events(struct io_cq_ring *ring) |
| 2437 | | −{ |
| 2438 | | − /* See comment at the top of this file */ |
| 2439 | | − smp_rmb(); |
| 2440 | | − return READ_ONCE(ring->r.tail) - READ_ONCE(ring->r.head); |
| 2441 | | −} |
| 2442 | | − |
| 2443 | 2473 | /* |
| 2444 | 2474 | * Wait until events become available, if we don't already have some. The |
| 2445 | 2475 | * application must reap them itself, as they reside on the shared cq ring. |
| @@ -3190,9 +3220,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 744 unmodified lines |
| 3190 | 3220 | min_complete = min(min_complete, ctx->cq_entries); |
| 3191 | 3221 | |
| 3192 | 3222 | if (ctx->flags & IORING_SETUP_IOPOLL) { |
| 3193 | | − mutex_lock(&ctx->uring_lock); |
| 3194 | 3223 | ret = io_iopoll_check(ctx, &nr_events, min_complete); |
| 3195 | | − mutex_unlock(&ctx->uring_lock); |
| 3196 | 3224 | } else { |
| 3197 | 3225 | ret = io_cqring_wait(ctx, min_complete, sig, sigsz); |
| 3198 | 3226 | } |