History

io_uring: pass in io_kiocb to fill/add CQ handlers

This is in preparation for handling CQ ring overflow a bit smarter. We should not have any functional changes in this patch. Most of the changes are fairly straight forward, the only ones that stick out a bit are the ones that change __io_free_req() to take the reference count into account. If the request hasn't been submitted yet, we know it's safe to simply ignore references and free it. But let's clean these up too, as later patches will depend on the caller doing the right thing if the completion logging grabs a reference to the request.

Jens Axboe committed 7 years ago

Commit 78e19bb1 parent 84f97dc1 file · +50 −46Browse files

Unified · Split

fs/io_uring.c+50−46
@@ -369,10 +369,10 @@ struct io_submit_state { · 368 unmodified lines
369369 };
370370
371371 static void io_wq_submit_work(struct io_wq_work **workptr);
372−static void io_cqring_fill_event(struct io_ring_ctx *ctx, u64 ki_user_data,
373− long res);
372+static void io_cqring_fill_event(struct io_kiocb *req, long res);
374373 static void __io_free_req(struct io_kiocb *req);
375374 static void io_put_req(struct io_kiocb *req, struct io_kiocb **nxtptr);
375+static void io_double_put_req(struct io_kiocb *req);
376376
377377 static struct kmem_cache *req_cachep;
378378
@@ -535,8 +535,8 @@ static void io_kill_timeout(struct io_kiocb *req) · 156 unmodified lines
535535 if (ret != -1) {
536536 atomic_inc(&req->ctx->cq_timeouts);
537537 list_del_init(&req->list);
538− io_cqring_fill_event(req->ctx, req->user_data, 0);
539− __io_free_req(req);
538+ io_cqring_fill_event(req, 0);
539+ io_put_req(req, NULL);
540540 }
541541 }
542542
@@ -588,12 +588,12 @@ static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx) · 45 unmodified lines
588588 return &rings->cqes[tail & ctx->cq_mask];
589589 }
590590
591−static void io_cqring_fill_event(struct io_ring_ctx *ctx, u64 ki_user_data,
592− long res)
591+static void io_cqring_fill_event(struct io_kiocb *req, long res)
593592 {
593+ struct io_ring_ctx *ctx = req->ctx;
594594 struct io_uring_cqe *cqe;
595595
596− trace_io_uring_complete(ctx, ki_user_data, res);
596+ trace_io_uring_complete(ctx, req->user_data, res);
597597
598598 /*
599599 * If we can't get a cq entry, userspace overflowed the
@@ -602,7 +602,7 @@ static void io_cqring_fill_event(struct io_ring_ctx *ctx, u64 ki_user_data, · 2 unmodified lines
602602 */
603603 cqe = io_get_cqring(ctx);
604604 if (cqe) {
605− WRITE_ONCE(cqe->user_data, ki_user_data);
605+ WRITE_ONCE(cqe->user_data, req->user_data);
606606 WRITE_ONCE(cqe->res, res);
607607 WRITE_ONCE(cqe->flags, 0);
608608 } else {
@@ -621,13 +621,13 @@ static void io_cqring_ev_posted(struct io_ring_ctx *ctx) · 12 unmodified lines
621621 eventfd_signal(ctx->cq_ev_fd, 1);
622622 }
623623
624−static void io_cqring_add_event(struct io_ring_ctx *ctx, u64 user_data,
625− long res)
624+static void io_cqring_add_event(struct io_kiocb *req, long res)
626625 {
626+ struct io_ring_ctx *ctx = req->ctx;
627627 unsigned long flags;
628628
629629 spin_lock_irqsave(&ctx->completion_lock, flags);
630− io_cqring_fill_event(ctx, user_data, res);
630+ io_cqring_fill_event(req, res);
631631 io_commit_cqring(ctx);
632632 spin_unlock_irqrestore(&ctx->completion_lock, flags);
633633
@@ -721,10 +721,10 @@ static bool io_link_cancel_timeout(struct io_ring_ctx *ctx, · 87 unmodified lines
721721
722722 ret = hrtimer_try_to_cancel(&req->timeout.timer);
723723 if (ret != -1) {
724− io_cqring_fill_event(ctx, req->user_data, -ECANCELED);
724+ io_cqring_fill_event(req, -ECANCELED);
725725 io_commit_cqring(ctx);
726726 req->flags &= ~REQ_F_LINK;
727− __io_free_req(req);
727+ io_put_req(req, NULL);
728728 return true;
729729 }
730730
@@ -795,8 +795,8 @@ static void io_fail_links(struct io_kiocb *req) · 64 unmodified lines
795795 link->submit.sqe->opcode == IORING_OP_LINK_TIMEOUT) {
796796 io_link_cancel_timeout(ctx, link);
797797 } else {
798− io_cqring_fill_event(ctx, link->user_data, -ECANCELED);
799− __io_free_req(link);
798+ io_cqring_fill_event(link, -ECANCELED);
799+ io_double_put_req(link);
800800 }
801801 }
802802
@@ -866,6 +866,13 @@ static void io_put_req(struct io_kiocb *req, struct io_kiocb **nxtptr) · 63 unmodified lines
866866 }
867867 }
868868
869+static void io_double_put_req(struct io_kiocb *req)
870+{
871+ /* drop both submit and complete references */
872+ if (refcount_sub_and_test(2, &req->refs))
873+ __io_free_req(req);
874+}
875+
869876 static unsigned io_cqring_events(struct io_ring_ctx *ctx)
870877 {
871878 struct io_rings *rings = ctx->rings;
@@ -898,7 +905,7 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, · 26 unmodified lines
898905 req = list_first_entry(done, struct io_kiocb, list);
899906 list_del(&req->list);
900907
901− io_cqring_fill_event(ctx, req->user_data, req->result);
908+ io_cqring_fill_event(req, req->result);
902909 (*nr_events)++;
903910
904911 if (refcount_dec_and_test(&req->refs)) {
@@ -1094,7 +1101,7 @@ static void io_complete_rw_common(struct kiocb *kiocb, long res) · 189 unmodified lines
10941101
10951102 if ((req->flags & REQ_F_LINK) && res != req->result)
10961103 req->flags |= REQ_F_FAIL_LINK;
1097− io_cqring_add_event(req->ctx, req->user_data, res);
1104+ io_cqring_add_event(req, res);
10981105 }
10991106
11001107 static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
@@ -1595,15 +1602,14 @@ out_free: · 494 unmodified lines
15951602 /*
15961603 * IORING_OP_NOP just posts a completion event, nothing else.
15971604 */
1598−static int io_nop(struct io_kiocb *req, u64 user_data)
1605+static int io_nop(struct io_kiocb *req)
15991606 {
16001607 struct io_ring_ctx *ctx = req->ctx;
1601− long err = 0;
16021608
16031609 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
16041610 return -EINVAL;
16051611
1606− io_cqring_add_event(ctx, user_data, err);
1612+ io_cqring_add_event(req, 0);
16071613 io_put_req(req, NULL);
16081614 return 0;
16091615 }
@@ -1650,7 +1656,7 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 40 unmodified lines
16501656
16511657 if (ret < 0 && (req->flags & REQ_F_LINK))
16521658 req->flags |= REQ_F_FAIL_LINK;
1653− io_cqring_add_event(req->ctx, sqe->user_data, ret);
1659+ io_cqring_add_event(req, ret);
16541660 io_put_req(req, nxt);
16551661 return 0;
16561662 }
@@ -1697,7 +1703,7 @@ static int io_sync_file_range(struct io_kiocb *req, · 40 unmodified lines
16971703
16981704 if (ret < 0 && (req->flags & REQ_F_LINK))
16991705 req->flags |= REQ_F_FAIL_LINK;
1700− io_cqring_add_event(req->ctx, sqe->user_data, ret);
1706+ io_cqring_add_event(req, ret);
17011707 io_put_req(req, nxt);
17021708 return 0;
17031709 }
@@ -1733,7 +1739,7 @@ static int io_send_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 29 unmodified lines
17331739 return ret;
17341740 }
17351741
1736− io_cqring_add_event(req->ctx, sqe->user_data, ret);
1742+ io_cqring_add_event(req, ret);
17371743 if (ret < 0 && (req->flags & REQ_F_LINK))
17381744 req->flags |= REQ_F_FAIL_LINK;
17391745 io_put_req(req, nxt);
@@ -1789,7 +1795,7 @@ static int io_accept(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 49 unmodified lines
17891795 }
17901796 if (ret < 0 && (req->flags & REQ_F_LINK))
17911797 req->flags |= REQ_F_FAIL_LINK;
1792− io_cqring_add_event(req->ctx, sqe->user_data, ret);
1798+ io_cqring_add_event(req, ret);
17931799 io_put_req(req, nxt);
17941800 return 0;
17951801 #else
@@ -1850,7 +1856,7 @@ static int io_poll_remove(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 54 unmodified lines
18501856 }
18511857 spin_unlock_irq(&ctx->completion_lock);
18521858
1853− io_cqring_add_event(req->ctx, sqe->user_data, ret);
1859+ io_cqring_add_event(req, ret);
18541860 if (ret < 0 && (req->flags & REQ_F_LINK))
18551861 req->flags |= REQ_F_FAIL_LINK;
18561862 io_put_req(req, NULL);
@@ -1861,7 +1867,7 @@ static void io_poll_complete(struct io_ring_ctx *ctx, struct io_kiocb *req, · 4 unmodified lines
18611867 __poll_t mask)
18621868 {
18631869 req->poll.done = true;
1864− io_cqring_fill_event(ctx, req->user_data, mangle_poll(mask));
1870+ io_cqring_fill_event(req, mangle_poll(mask));
18651871 io_commit_cqring(ctx);
18661872 }
18671873
@@ -2055,7 +2061,7 @@ static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) · 187 unmodified lines
20552061 list_del_init(&req->list);
20562062 }
20572063
2058− io_cqring_fill_event(ctx, req->user_data, -ETIME);
2064+ io_cqring_fill_event(req, -ETIME);
20592065 io_commit_cqring(ctx);
20602066 spin_unlock_irqrestore(&ctx->completion_lock, flags);
20612067
@@ -2099,7 +2105,7 @@ static int io_timeout_remove(struct io_kiocb *req, · 37 unmodified lines
20992105 /* didn't find timeout */
21002106 if (ret) {
21012107 fill_ev:
2102− io_cqring_fill_event(ctx, req->user_data, ret);
2108+ io_cqring_fill_event(req, ret);
21032109 io_commit_cqring(ctx);
21042110 spin_unlock_irq(&ctx->completion_lock);
21052111 io_cqring_ev_posted(ctx);
@@ -2115,8 +2121,8 @@ fill_ev: · 9 unmodified lines
21152121 goto fill_ev;
21162122 }
21172123
2118− io_cqring_fill_event(ctx, req->user_data, 0);
2119− io_cqring_fill_event(ctx, treq->user_data, -ECANCELED);
2124+ io_cqring_fill_event(req, 0);
2125+ io_cqring_fill_event(treq, -ECANCELED);
21202126 io_commit_cqring(ctx);
21212127 spin_unlock_irq(&ctx->completion_lock);
21222128 io_cqring_ev_posted(ctx);
@@ -2256,7 +2262,7 @@ static int io_async_cancel(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 133 unmodified lines
22562262
22572263 if (ret < 0 && (req->flags & REQ_F_LINK))
22582264 req->flags |= REQ_F_FAIL_LINK;
2259− io_cqring_add_event(req->ctx, sqe->user_data, ret);
2265+ io_cqring_add_event(req, ret);
22602266 io_put_req(req, nxt);
22612267 return 0;
22622268 }
@@ -2295,12 +2301,10 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, · 32 unmodified lines
22952301 int ret, opcode;
22962302 struct sqe_submit *s = &req->submit;
22972303
2298− req->user_data = READ_ONCE(s->sqe->user_data);
2299−
23002304 opcode = READ_ONCE(s->sqe->opcode);
23012305 switch (opcode) {
23022306 case IORING_OP_NOP:
2303− ret = io_nop(req, req->user_data);
2307+ ret = io_nop(req);
23042308 break;
23052309 case IORING_OP_READV:
23062310 if (unlikely(s->sqe->buf_index))
@@ -2409,7 +2413,7 @@ static void io_wq_submit_work(struct io_wq_work **workptr) · 102 unmodified lines
24092413 if (ret) {
24102414 if (req->flags & REQ_F_LINK)
24112415 req->flags |= REQ_F_FAIL_LINK;
2412− io_cqring_add_event(ctx, sqe->user_data, ret);
2416+ io_cqring_add_event(req, ret);
24132417 io_put_req(req, NULL);
24142418 }
24152419
@@ -2539,7 +2543,7 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) · 123 unmodified lines
25392543 ret = io_async_cancel_one(ctx, user_data);
25402544 }
25412545
2542− io_cqring_add_event(ctx, req->user_data, ret);
2546+ io_cqring_add_event(req, ret);
25432547 io_put_req(req, NULL);
25442548 return HRTIMER_NORESTART;
25452549 }
@@ -2582,7 +2586,7 @@ err: · 36 unmodified lines
25822586 * failed by the regular submission path.
25832587 */
25842588 list_del(&nxt->list);
2585− io_cqring_fill_event(ctx, nxt->user_data, ret);
2589+ io_cqring_fill_event(nxt, ret);
25862590 trace_io_uring_fail_link(req, nxt);
25872591 io_commit_cqring(ctx);
25882592 io_put_req(nxt, NULL);
@@ -2655,7 +2659,7 @@ err: · 66 unmodified lines
26552659
26562660 /* and drop final reference, if we failed */
26572661 if (ret) {
2658− io_cqring_add_event(ctx, req->user_data, ret);
2662+ io_cqring_add_event(req, ret);
26592663 if (req->flags & REQ_F_LINK)
26602664 req->flags |= REQ_F_FAIL_LINK;
26612665 io_put_req(req, NULL);
@@ -2671,8 +2675,8 @@ static int io_queue_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req) · 9 unmodified lines
26712675 ret = io_req_defer(ctx, req);
26722676 if (ret) {
26732677 if (ret != -EIOCBQUEUED) {
2674− io_cqring_add_event(ctx, req->submit.sqe->user_data, ret);
2675− io_free_req(req, NULL);
2678+ io_cqring_add_event(req, ret);
2679+ io_double_put_req(req);
26762680 }
26772681 return 0;
26782682 }
@@ -2698,8 +2702,8 @@ static int io_queue_link_head(struct io_ring_ctx *ctx, struct io_kiocb *req, · 19 unmodified lines
26982702 ret = io_req_defer(ctx, req);
26992703 if (ret) {
27002704 if (ret != -EIOCBQUEUED) {
2701− io_cqring_add_event(ctx, req->submit.sqe->user_data, ret);
2702− io_free_req(req, NULL);
2705+ io_cqring_add_event(req, ret);
2706+ io_double_put_req(req);
27032707 __io_free_req(shadow);
27042708 return 0;
27052709 }
@@ -2732,6 +2736,8 @@ static void io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, · 26 unmodified lines
27322736 struct sqe_submit *s = &req->submit;
27332737 int ret;
27342738
2739+ req->user_data = s->sqe->user_data;
2740+
27352741 /* enforce forwards compatibility on users */
27362742 if (unlikely(s->sqe->flags & ~SQE_VALID_FLAGS)) {
27372743 ret = -EINVAL;
@@ -2741,13 +2747,11 @@ static void io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, · 3 unmodified lines
27412747 ret = io_req_set_file(ctx, state, req);
27422748 if (unlikely(ret)) {
27432749 err_req:
2744− io_cqring_add_event(ctx, s->sqe->user_data, ret);
2745− io_free_req(req, NULL);
2750+ io_cqring_add_event(req, ret);
2751+ io_double_put_req(req);
27462752 return;
27472753 }
27482754
2749− req->user_data = s->sqe->user_data;
2750−
27512755 /*
27522756 * If we already have a head request, queue this one for async
27532757 * submittal once the head completes. If we don't have a head but