The only user of io_req_complete32()-like functions is cmd requests. Instead of keeping the whole complete32 family, remove them and provide the extras in already added for inline completions req->extra{1,2}. When fill_cqe_res() finds CQE32 option enabled it'll use those fields to fill a 32B cqe.
Unified · Split
| @@ -2513,33 +2513,6 @@ static inline bool __io_fill_cqe_req(struct io_ring_ctx *ctx, · 2512 unmodified lines | |||
| 2513 | } | 2513 | } |
| 2514 | } | 2514 | } |
| 2515 | 2515 | ||
| 2516 | static inline bool __io_fill_cqe32_req(struct io_ring_ctx *ctx, | ||
| 2517 | struct io_kiocb *req) | ||
| 2518 | { | ||
| 2519 | struct io_uring_cqe *cqe; | ||
| 2520 | u64 extra1 = req->extra1; | ||
| 2521 | u64 extra2 = req->extra2; | ||
| 2522 | |||
| 2523 | trace_io_uring_complete(req->ctx, req, req->cqe.user_data, | ||
| 2524 | req->cqe.res, req->cqe.flags, extra1, extra2); | ||
| 2525 | |||
| 2526 | /* | ||
| 2527 | * If we can't get a cq entry, userspace overflowed the | ||
| 2528 | * submission (by quite a lot). Increment the overflow count in | ||
| 2529 | * the ring. | ||
| 2530 | */ | ||
| 2531 | cqe = io_get_cqe(ctx); | ||
| 2532 | if (likely(cqe)) { | ||
| 2533 | memcpy(cqe, &req->cqe, sizeof(struct io_uring_cqe)); | ||
| 2534 | cqe->big_cqe[0] = extra1; | ||
| 2535 | cqe->big_cqe[1] = extra2; | ||
| 2536 | return true; | ||
| 2537 | } | ||
| 2538 | |||
| 2539 | return io_cqring_event_overflow(ctx, req->cqe.user_data, req->cqe.res, | ||
| 2540 | req->cqe.flags, extra1, extra2); | ||
| 2541 | } | ||
| 2542 | |||
| 2543 | static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, | 2516 | static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, |
| 2544 | s32 res, u32 cflags) | 2517 | s32 res, u32 cflags) |
| 2545 | { | 2518 | { |
| @@ -2590,19 +2563,6 @@ static void __io_req_complete_post(struct io_kiocb *req, s32 res, · 44 unmodified lines | |||
| 2590 | __io_req_complete_put(req); | 2563 | __io_req_complete_put(req); |
| 2591 | } | 2564 | } |
| 2592 | 2565 | ||
| 2593 | static void __io_req_complete_post32(struct io_kiocb *req, s32 res, | ||
| 2594 | u32 cflags, u64 extra1, u64 extra2) | ||
| 2595 | { | ||
| 2596 | if (!(req->flags & REQ_F_CQE_SKIP)) { | ||
| 2597 | req->cqe.res = res; | ||
| 2598 | req->cqe.flags = cflags; | ||
| 2599 | req->extra1 = extra1; | ||
| 2600 | req->extra2 = extra2; | ||
| 2601 | __io_fill_cqe32_req(req->ctx, req); | ||
| 2602 | } | ||
| 2603 | __io_req_complete_put(req); | ||
| 2604 | } | ||
| 2605 | |||
| 2606 | static void io_req_complete_post(struct io_kiocb *req, s32 res, u32 cflags) | 2566 | static void io_req_complete_post(struct io_kiocb *req, s32 res, u32 cflags) |
| 2607 | { | 2567 | { |
| 2608 | struct io_ring_ctx *ctx = req->ctx; | 2568 | struct io_ring_ctx *ctx = req->ctx; |
| @@ -2614,18 +2574,6 @@ static void io_req_complete_post(struct io_kiocb *req, s32 res, u32 cflags) · 5 unmodified lines | |||
| 2614 | io_cqring_ev_posted(ctx); | 2574 | io_cqring_ev_posted(ctx); |
| 2615 | } | 2575 | } |
| 2616 | 2576 | ||
| 2617 | static void io_req_complete_post32(struct io_kiocb *req, s32 res, | ||
| 2618 | u32 cflags, u64 extra1, u64 extra2) | ||
| 2619 | { | ||
| 2620 | struct io_ring_ctx *ctx = req->ctx; | ||
| 2621 | |||
| 2622 | spin_lock(&ctx->completion_lock); | ||
| 2623 | __io_req_complete_post32(req, res, cflags, extra1, extra2); | ||
| 2624 | io_commit_cqring(ctx); | ||
| 2625 | spin_unlock(&ctx->completion_lock); | ||
| 2626 | io_cqring_ev_posted(ctx); | ||
| 2627 | } | ||
| 2628 | |||
| 2629 | static inline void io_req_complete_state(struct io_kiocb *req, s32 res, | 2577 | static inline void io_req_complete_state(struct io_kiocb *req, s32 res, |
| 2630 | u32 cflags) | 2578 | u32 cflags) |
| 2631 | { | 2579 | { |
| @@ -2643,19 +2591,6 @@ static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags, · 11 unmodified lines | |||
| 2643 | io_req_complete_post(req, res, cflags); | 2591 | io_req_complete_post(req, res, cflags); |
| 2644 | } | 2592 | } |
| 2645 | 2593 | ||
| 2646 | static inline void __io_req_complete32(struct io_kiocb *req, | ||
| 2647 | unsigned int issue_flags, s32 res, | ||
| 2648 | u32 cflags, u64 extra1, u64 extra2) | ||
| 2649 | { | ||
| 2650 | if (issue_flags & IO_URING_F_COMPLETE_DEFER) { | ||
| 2651 | io_req_complete_state(req, res, cflags); | ||
| 2652 | req->extra1 = extra1; | ||
| 2653 | req->extra2 = extra2; | ||
| 2654 | } else { | ||
| 2655 | io_req_complete_post32(req, res, cflags, extra1, extra2); | ||
| 2656 | } | ||
| 2657 | } | ||
| 2658 | |||
| 2659 | static inline void io_req_complete(struct io_kiocb *req, s32 res) | 2594 | static inline void io_req_complete(struct io_kiocb *req, s32 res) |
| 2660 | { | 2595 | { |
| 2661 | if (res < 0) | 2596 | if (res < 0) |
| @@ -5079,6 +5014,13 @@ void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd, · 2417 unmodified lines | |||
| 5079 | } | 5014 | } |
| 5080 | EXPORT_SYMBOL_GPL(io_uring_cmd_complete_in_task); | 5015 | EXPORT_SYMBOL_GPL(io_uring_cmd_complete_in_task); |
| 5081 | 5016 | ||
| 5017 | static inline void io_req_set_cqe32_extra(struct io_kiocb *req, | ||
| 5018 | u64 extra1, u64 extra2) | ||
| 5019 | { | ||
| 5020 | req->extra1 = extra1; | ||
| 5021 | req->extra2 = extra2; | ||
| 5022 | } | ||
| 5023 | |||
| 5082 | /* | 5024 | /* |
| 5083 | * Called by consumers of io_uring_cmd, if they originally returned | 5025 | * Called by consumers of io_uring_cmd, if they originally returned |
| 5084 | * -EIOCBQUEUED upon receiving the command. | 5026 | * -EIOCBQUEUED upon receiving the command. |
| @@ -5089,10 +5031,10 @@ void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2) · 4 unmodified lines | |||
| 5089 | 5031 | ||
| 5090 | if (ret < 0) | 5032 | if (ret < 0) |
| 5091 | req_set_fail(req); | 5033 | req_set_fail(req); |
| 5034 | |||
| 5092 | if (req->ctx->flags & IORING_SETUP_CQE32) | 5035 | if (req->ctx->flags & IORING_SETUP_CQE32) |
| 5093 | __io_req_complete32(req, 0, ret, 0, res2, 0); | 5036 | io_req_set_cqe32_extra(req, res2, 0); |
| 5094 | else | 5037 | io_req_complete(req, ret); |
| 5095 | io_req_complete(req, ret); | ||
| 5096 | } | 5038 | } |
| 5097 | EXPORT_SYMBOL_GPL(io_uring_cmd_done); | 5039 | EXPORT_SYMBOL_GPL(io_uring_cmd_done); |
| 5098 | 5040 | ||