Commit fb5ccc98782f ("io_uring: Fix broken links with offloading") introduced a potential performance regression with unconditionally taking mm even for READ/WRITE_FIXED operations. Return the logic handling it back. mm-faulted requests will go through the generic submission path, so honoring links and drains, but will fail further on req->has_user check.
Unified · Split
| @@ -2726,13 +2726,14 @@ static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s) · 2725 unmodified lines | |||
| 2726 | } | 2726 | } |
| 2727 | 2727 | ||
| 2728 | static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr, | 2728 | static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr, |
| 2729 | bool has_user, bool mm_fault) | 2729 | struct mm_struct **mm) |
| 2730 | { | 2730 | { |
| 2731 | struct io_submit_state state, *statep = NULL; | 2731 | struct io_submit_state state, *statep = NULL; |
| 2732 | struct io_kiocb *link = NULL; | 2732 | struct io_kiocb *link = NULL; |
| 2733 | struct io_kiocb *shadow_req = NULL; | 2733 | struct io_kiocb *shadow_req = NULL; |
| 2734 | bool prev_was_link = false; | 2734 | bool prev_was_link = false; |
| 2735 | int i, submitted = 0; | 2735 | int i, submitted = 0; |
| 2736 | bool mm_fault = false; | ||
| 2736 | 2737 | ||
| 2737 | if (nr > IO_PLUG_THRESHOLD) { | 2738 | if (nr > IO_PLUG_THRESHOLD) { |
| 2738 | io_submit_state_start(&state, ctx, nr); | 2739 | io_submit_state_start(&state, ctx, nr); |
| @@ -2745,6 +2746,14 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr, · 6 unmodified lines | |||
| 2745 | if (!io_get_sqring(ctx, &s)) | 2746 | if (!io_get_sqring(ctx, &s)) |
| 2746 | break; | 2747 | break; |
| 2747 | 2748 | ||
| 2749 | if (io_sqe_needs_user(s.sqe) && !*mm) { | ||
| 2750 | mm_fault = mm_fault || !mmget_not_zero(ctx->sqo_mm); | ||
| 2751 | if (!mm_fault) { | ||
| 2752 | use_mm(ctx->sqo_mm); | ||
| 2753 | *mm = ctx->sqo_mm; | ||
| 2754 | } | ||
| 2755 | } | ||
| 2756 | |||
| 2748 | /* | 2757 | /* |
| 2749 | * If previous wasn't linked and we have a linked command, | 2758 | * If previous wasn't linked and we have a linked command, |
| 2750 | * that's the end of the chain. Submit the previous link. | 2759 | * that's the end of the chain. Submit the previous link. |
| @@ -2768,17 +2777,12 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr, · 17 unmodified lines | |||
| 2768 | } | 2777 | } |
| 2769 | 2778 | ||
| 2770 | out: | 2779 | out: |
| 2771 | if (unlikely(mm_fault)) { | 2780 | s.has_user = *mm != NULL; |
| 2772 | io_cqring_add_event(ctx, s.sqe->user_data, | 2781 | s.in_async = true; |
| 2773 | -EFAULT); | 2782 | s.needs_fixed_file = true; |
| 2774 | } else { | 2783 | trace_io_uring_submit_sqe(ctx, true, true); |
| 2775 | s.has_user = has_user; | 2784 | io_submit_sqe(ctx, &s, statep, &link); |
| 2776 | s.in_async = true; | 2785 | submitted++; |
| 2777 | s.needs_fixed_file = true; | ||
| 2778 | trace_io_uring_submit_sqe(ctx, true, true); | ||
| 2779 | io_submit_sqe(ctx, &s, statep, &link); | ||
| 2780 | submitted++; | ||
| 2781 | } | ||
| 2782 | } | 2786 | } |
| 2783 | 2787 | ||
| 2784 | if (link) | 2788 | if (link) |
| @@ -2805,7 +2809,6 @@ static int io_sq_thread(void *data) · 20 unmodified lines | |||
| 2805 | 2809 | ||
| 2806 | timeout = inflight = 0; | 2810 | timeout = inflight = 0; |
| 2807 | while (!kthread_should_park()) { | 2811 | while (!kthread_should_park()) { |
| 2808 | bool mm_fault = false; | ||
| 2809 | unsigned int to_submit; | 2812 | unsigned int to_submit; |
| 2810 | 2813 | ||
| 2811 | if (inflight) { | 2814 | if (inflight) { |
| @@ -2890,18 +2893,8 @@ static int io_sq_thread(void *data) · 78 unmodified lines | |||
| 2890 | ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP; | 2893 | ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP; |
| 2891 | } | 2894 | } |
| 2892 | 2895 | ||
| 2893 | /* Unless all new commands are FIXED regions, grab mm */ | ||
| 2894 | if (!cur_mm) { | ||
| 2895 | mm_fault = !mmget_not_zero(ctx->sqo_mm); | ||
| 2896 | if (!mm_fault) { | ||
| 2897 | use_mm(ctx->sqo_mm); | ||
| 2898 | cur_mm = ctx->sqo_mm; | ||
| 2899 | } | ||
| 2900 | } | ||
| 2901 | |||
| 2902 | to_submit = min(to_submit, ctx->sq_entries); | 2896 | to_submit = min(to_submit, ctx->sq_entries); |
| 2903 | inflight += io_submit_sqes(ctx, to_submit, cur_mm != NULL, | 2897 | inflight += io_submit_sqes(ctx, to_submit, &cur_mm); |
| 2904 | mm_fault); | ||
| 2905 | 2898 | ||
| 2906 | /* Commit SQ ring head once we've consumed all SQEs */ | 2899 | /* Commit SQ ring head once we've consumed all SQEs */ |
| 2907 | io_commit_sqring(ctx); | 2900 | io_commit_sqring(ctx); |