History

io_uring: add support for sqe links

With SQE links, we can create chains of dependent SQEs. One example would be queueing an SQE that's a read from one file descriptor, with the linked SQE being a write to another with the same set of buffers. An SQE link will not stall the pipeline, it'll just ensure that dependent SQEs aren't issued before the previous link has completed. Any error at submission or completion time will break the chain of SQEs. For completions, this also includes short reads or writes, as the next SQE could depend on the previous one being fully completed. Any SQE in a chain that gets canceled due to any of the above errors, will get an CQE fill with -ECANCELED as the error value.

Jens Axboe committed 7 years ago

Commit 9e645e11 parent 9d93a3f2 files · +194 −48Browse files

Unified · Split

2 files
  1. fs/io_uring.c +193 −48
  2. include/uapi/linux/io_uring.h +1 −0
fs/io_uring.c+193−48
@@ -322,6 +322,7 @@ struct io_kiocb { · 321 unmodified lines
322322
323323 struct io_ring_ctx *ctx;
324324 struct list_head list;
325+ struct list_head link_list;
325326 unsigned int flags;
326327 refcount_t refs;
327328 #define REQ_F_NOWAIT 1 /* must not punt to workers */
@@ -330,8 +331,10 @@ struct io_kiocb { · 2 unmodified lines
330331 #define REQ_F_SEQ_PREV 8 /* sequential with previous */
331332 #define REQ_F_IO_DRAIN 16 /* drain existing IO first */
332333 #define REQ_F_IO_DRAINED 32 /* drain done */
334+#define REQ_F_LINK 64 /* linked sqes */
335+#define REQ_F_FAIL_LINK 128 /* fail rest of links */
333336 u64 user_data;
334− u32 error; /* iopoll result from callback */
337+ u32 result;
335338 u32 sequence;
336339
337340 struct work_struct work;
@@ -583,6 +586,7 @@ static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx, · 245 unmodified lines
583586 req->flags = 0;
584587 /* one is dropped after submission, the other at completion */
585588 refcount_set(&req->refs, 2);
589+ req->result = 0;
586590 return req;
587591 out:
588592 io_ring_drop_ctx_refs(ctx, 1);
@@ -598,7 +602,7 @@ static void io_free_req_many(struct io_ring_ctx *ctx, void **reqs, int *nr) · 9 unmodified lines
598602 }
599603 }
600604
601−static void io_free_req(struct io_kiocb *req)
605+static void __io_free_req(struct io_kiocb *req)
602606 {
603607 if (req->file && !(req->flags & REQ_F_FIXED_FILE))
604608 fput(req->file);
@@ -606,6 +610,63 @@ static void io_free_req(struct io_kiocb *req) · 1 unmodified line
606610 kmem_cache_free(req_cachep, req);
607611 }
608612
613+static void io_req_link_next(struct io_kiocb *req)
614+{
615+ struct io_kiocb *nxt;
616+
617+ /*
618+ * The list should never be empty when we are called here. But could
619+ * potentially happen if the chain is messed up, check to be on the
620+ * safe side.
621+ */
622+ nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb, list);
623+ if (nxt) {
624+ list_del(&nxt->list);
625+ if (!list_empty(&req->link_list)) {
626+ INIT_LIST_HEAD(&nxt->link_list);
627+ list_splice(&req->link_list, &nxt->link_list);
628+ nxt->flags |= REQ_F_LINK;
629+ }
630+
631+ INIT_WORK(&nxt->work, io_sq_wq_submit_work);
632+ queue_work(req->ctx->sqo_wq, &nxt->work);
633+ }
634+}
635+
636+/*
637+ * Called if REQ_F_LINK is set, and we fail the head request
638+ */
639+static void io_fail_links(struct io_kiocb *req)
640+{
641+ struct io_kiocb *link;
642+
643+ while (!list_empty(&req->link_list)) {
644+ link = list_first_entry(&req->link_list, struct io_kiocb, list);
645+ list_del(&link->list);
646+
647+ io_cqring_add_event(req->ctx, link->user_data, -ECANCELED);
648+ __io_free_req(link);
649+ }
650+}
651+
652+static void io_free_req(struct io_kiocb *req)
653+{
654+ /*
655+ * If LINK is set, we have dependent requests in this chain. If we
656+ * didn't fail this request, queue the first one up, moving any other
657+ * dependencies to the next request. In case of failure, fail the rest
658+ * of the chain.
659+ */
660+ if (req->flags & REQ_F_LINK) {
661+ if (req->flags & REQ_F_FAIL_LINK)
662+ io_fail_links(req);
663+ else
664+ io_req_link_next(req);
665+ }
666+
667+ __io_free_req(req);
668+}
669+
609670 static void io_put_req(struct io_kiocb *req)
610671 {
611672 if (refcount_dec_and_test(&req->refs))
@@ -627,16 +688,17 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, · 15 unmodified lines
627688 req = list_first_entry(done, struct io_kiocb, list);
628689 list_del(&req->list);
629690
630− io_cqring_fill_event(ctx, req->user_data, req->error);
691+ io_cqring_fill_event(ctx, req->user_data, req->result);
631692 (*nr_events)++;
632693
633694 if (refcount_dec_and_test(&req->refs)) {
634695 /* If we're not using fixed files, we have to pair the
635696 * completion part with the file put. Use regular
636697 * completions for those, only batch free for fixed
637− * file.
698+ * file and non-linked commands.
638699 */
639− if (req->flags & REQ_F_FIXED_FILE) {
700+ if ((req->flags & (REQ_F_FIXED_FILE|REQ_F_LINK)) ==
701+ REQ_F_FIXED_FILE) {
640702 reqs[to_free++] = req;
641703 if (to_free == ARRAY_SIZE(reqs))
642704 io_free_req_many(ctx, reqs, &to_free);
@@ -775,6 +837,8 @@ static void io_complete_rw(struct kiocb *kiocb, long res, long res2) · 132 unmodified lines
775837
776838 kiocb_end_write(kiocb);
777839
840+ if ((req->flags & REQ_F_LINK) && res != req->result)
841+ req->flags |= REQ_F_FAIL_LINK;
778842 io_cqring_add_event(req->ctx, req->user_data, res);
779843 io_put_req(req);
780844 }
@@ -785,7 +849,9 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) · 4 unmodified lines
785849
786850 kiocb_end_write(kiocb);
787851
788− req->error = res;
852+ if ((req->flags & REQ_F_LINK) && res != req->result)
853+ req->flags |= REQ_F_FAIL_LINK;
854+ req->result = res;
789855 if (res != -EAGAIN)
790856 req->flags |= REQ_F_IOPOLL_COMPLETED;
791857 }
@@ -928,7 +994,6 @@ static int io_prep_rw(struct io_kiocb *req, const struct sqe_submit *s, · 136 unmodified lines
928994 !kiocb->ki_filp->f_op->iopoll)
929995 return -EOPNOTSUPP;
930996
931− req->error = 0;
932997 kiocb->ki_flags |= IOCB_HIPRI;
933998 kiocb->ki_complete = io_complete_rw_iopoll;
934999 } else {
@@ -1106,6 +1171,9 @@ static int io_read(struct io_kiocb *req, const struct sqe_submit *s, · 171 unmodified lines
11061171 return ret;
11071172
11081173 read_size = ret;
1174+ if (req->flags & REQ_F_LINK)
1175+ req->result = read_size;
1176+
11091177 iov_count = iov_iter_count(&iter);
11101178 ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_count);
11111179 if (!ret) {
@@ -1163,6 +1231,9 @@ static int io_write(struct io_kiocb *req, const struct sqe_submit *s, · 51 unmodified lines
11631231 if (ret < 0)
11641232 return ret;
11651233
1234+ if (req->flags & REQ_F_LINK)
1235+ req->result = ret;
1236+
11661237 iov_count = iov_iter_count(&iter);
11671238
11681239 ret = -EAGAIN;
@@ -1266,6 +1337,8 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 97 unmodified lines
12661337 end > 0 ? end : LLONG_MAX,
12671338 fsync_flags & IORING_FSYNC_DATASYNC);
12681339
1340+ if (ret < 0 && (req->flags & REQ_F_LINK))
1341+ req->flags |= REQ_F_FAIL_LINK;
12691342 io_cqring_add_event(req->ctx, sqe->user_data, ret);
12701343 io_put_req(req);
12711344 return 0;
@@ -1310,6 +1383,8 @@ static int io_sync_file_range(struct io_kiocb *req, · 38 unmodified lines
13101383
13111384 ret = sync_file_range(req->rw.ki_filp, sqe_off, sqe_len, flags);
13121385
1386+ if (ret < 0 && (req->flags & REQ_F_LINK))
1387+ req->flags |= REQ_F_FAIL_LINK;
13131388 io_cqring_add_event(req->ctx, sqe->user_data, ret);
13141389 io_put_req(req);
13151390 return 0;
@@ -1562,9 +1637,10 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, · 246 unmodified lines
15621637 {
15631638 int ret, opcode;
15641639
1640+ req->user_data = READ_ONCE(s->sqe->user_data);
1641+
15651642 if (unlikely(s->index >= ctx->sq_entries))
15661643 return -EINVAL;
1567− req->user_data = READ_ONCE(s->sqe->user_data);
15681644
15691645 opcode = READ_ONCE(s->sqe->opcode);
15701646 switch (opcode) {
@@ -1608,7 +1684,7 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, · 37 unmodified lines
16081684 return ret;
16091685
16101686 if (ctx->flags & IORING_SETUP_IOPOLL) {
1611− if (req->error == -EAGAIN)
1687+ if (req->result == -EAGAIN)
16121688 return -EAGAIN;
16131689
16141690 /* workqueue context doesn't hold uring_lock, grab it now */
@@ -1834,31 +1910,11 @@ static int io_req_set_file(struct io_ring_ctx *ctx, const struct sqe_submit *s, · 219 unmodified lines
18341910 return 0;
18351911 }
18361912
1837−static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
1838− struct io_submit_state *state)
1913+static int io_queue_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
1914+ struct sqe_submit *s)
18391915 {
1840− struct io_kiocb *req;
18411916 int ret;
18421917
1843− /* enforce forwards compatibility on users */
1844− if (unlikely(s->sqe->flags & ~(IOSQE_FIXED_FILE | IOSQE_IO_DRAIN)))
1845− return -EINVAL;
1846−
1847− req = io_get_req(ctx, state);
1848− if (unlikely(!req))
1849− return -EAGAIN;
1850−
1851− ret = io_req_set_file(ctx, s, state, req);
1852− if (unlikely(ret))
1853− goto out;
1854−
1855− ret = io_req_defer(ctx, req, s->sqe);
1856− if (ret) {
1857− if (ret == -EIOCBQUEUED)
1858− ret = 0;
1859− return ret;
1860− }
1861−
18621918 ret = __io_submit_sqe(ctx, req, s, true);
18631919 if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
18641920 struct io_uring_sqe *sqe_copy;
@@ -1881,24 +1937,93 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s, · 16 unmodified lines
18811937
18821938 /*
18831939 * Queued up for async execution, worker will release
1884− * submit reference when the iocb is actually
1885− * submitted.
1940+ * submit reference when the iocb is actually submitted.
18861941 */
18871942 return 0;
18881943 }
18891944 }
18901945
1891−out:
18921946 /* drop submission reference */
18931947 io_put_req(req);
18941948
18951949 /* and drop final reference, if we failed */
1896− if (ret)
1950+ if (ret) {
1951+ io_cqring_add_event(ctx, req->user_data, ret);
1952+ if (req->flags & REQ_F_LINK)
1953+ req->flags |= REQ_F_FAIL_LINK;
18971954 io_put_req(req);
1955+ }
18981956
18991957 return ret;
19001958 }
19011959
1960+#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK)
1961+
1962+static void io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
1963+ struct io_submit_state *state, struct io_kiocb **link)
1964+{
1965+ struct io_uring_sqe *sqe_copy;
1966+ struct io_kiocb *req;
1967+ int ret;
1968+
1969+ /* enforce forwards compatibility on users */
1970+ if (unlikely(s->sqe->flags & ~SQE_VALID_FLAGS)) {
1971+ ret = -EINVAL;
1972+ goto err;
1973+ }
1974+
1975+ req = io_get_req(ctx, state);
1976+ if (unlikely(!req)) {
1977+ ret = -EAGAIN;
1978+ goto err;
1979+ }
1980+
1981+ ret = io_req_set_file(ctx, s, state, req);
1982+ if (unlikely(ret)) {
1983+err_req:
1984+ io_free_req(req);
1985+err:
1986+ io_cqring_add_event(ctx, s->sqe->user_data, ret);
1987+ return;
1988+ }
1989+
1990+ ret = io_req_defer(ctx, req, s->sqe);
1991+ if (ret) {
1992+ if (ret != -EIOCBQUEUED)
1993+ goto err_req;
1994+ return;
1995+ }
1996+
1997+ /*
1998+ * If we already have a head request, queue this one for async
1999+ * submittal once the head completes. If we don't have a head but
2000+ * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
2001+ * submitted sync once the chain is complete. If none of those
2002+ * conditions are true (normal request), then just queue it.
2003+ */
2004+ if (*link) {
2005+ struct io_kiocb *prev = *link;
2006+
2007+ sqe_copy = kmemdup(s->sqe, sizeof(*sqe_copy), GFP_KERNEL);
2008+ if (!sqe_copy) {
2009+ ret = -EAGAIN;
2010+ goto err_req;
2011+ }
2012+
2013+ s->sqe = sqe_copy;
2014+ memcpy(&req->submit, s, sizeof(*s));
2015+ list_add_tail(&req->list, &prev->link_list);
2016+ } else if (s->sqe->flags & IOSQE_IO_LINK) {
2017+ req->flags |= REQ_F_LINK;
2018+
2019+ memcpy(&req->submit, s, sizeof(*s));
2020+ INIT_LIST_HEAD(&req->link_list);
2021+ *link = req;
2022+ } else {
2023+ io_queue_sqe(ctx, req, s);
2024+ }
2025+}
2026+
19022027 /*
19032028 * Batched submission is done, ensure local IO is flushed out.
19042029 */
@@ -1981,7 +2106,9 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, struct sqe_submit *sqes, · 76 unmodified lines
19812106 unsigned int nr, bool has_user, bool mm_fault)
19822107 {
19832108 struct io_submit_state state, *statep = NULL;
1984− int ret, i, submitted = 0;
2109+ struct io_kiocb *link = NULL;
2110+ bool prev_was_link = false;
2111+ int i, submitted = 0;
19852112
19862113 if (nr > IO_PLUG_THRESHOLD) {
19872114 io_submit_state_start(&state, ctx, nr);
@@ -1989,22 +2116,30 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, struct sqe_submit *sqes, · 1 unmodified line
19892116 }
19902117
19912118 for (i = 0; i < nr; i++) {
2119+ /*
2120+ * If previous wasn't linked and we have a linked command,
2121+ * that's the end of the chain. Submit the previous link.
2122+ */
2123+ if (!prev_was_link && link) {
2124+ io_queue_sqe(ctx, link, &link->submit);
2125+ link = NULL;
2126+ }
2127+ prev_was_link = (sqes[i].sqe->flags & IOSQE_IO_LINK) != 0;
2128+
19922129 if (unlikely(mm_fault)) {
1993− ret = -EFAULT;
2130+ io_cqring_add_event(ctx, sqes[i].sqe->user_data,
2131+ -EFAULT);
19942132 } else {
19952133 sqes[i].has_user = has_user;
19962134 sqes[i].needs_lock = true;
19972135 sqes[i].needs_fixed_file = true;
1998− ret = io_submit_sqe(ctx, &sqes[i], statep);
1999− }
2000− if (!ret) {
2136+ io_submit_sqe(ctx, &sqes[i], statep, &link);
20012137 submitted++;
2002− continue;
20032138 }
2004−
2005− io_cqring_add_event(ctx, sqes[i].sqe->user_data, ret);
20062139 }
20072140
2141+ if (link)
2142+ io_queue_sqe(ctx, link, &link->submit);
20082143 if (statep)
20092144 io_submit_state_end(&state);
20102145
@@ -2145,6 +2280,8 @@ static int io_sq_thread(void *data) · 134 unmodified lines
21452280 static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)
21462281 {
21472282 struct io_submit_state state, *statep = NULL;
2283+ struct io_kiocb *link = NULL;
2284+ bool prev_was_link = false;
21482285 int i, submit = 0;
21492286
21502287 if (to_submit > IO_PLUG_THRESHOLD) {
@@ -2154,22 +2291,30 @@ static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit) · 3 unmodified lines
21542291
21552292 for (i = 0; i < to_submit; i++) {
21562293 struct sqe_submit s;
2157− int ret;
21582294
21592295 if (!io_get_sqring(ctx, &s))
21602296 break;
21612297
2298+ /*
2299+ * If previous wasn't linked and we have a linked command,
2300+ * that's the end of the chain. Submit the previous link.
2301+ */
2302+ if (!prev_was_link && link) {
2303+ io_queue_sqe(ctx, link, &link->submit);
2304+ link = NULL;
2305+ }
2306+ prev_was_link = (s.sqe->flags & IOSQE_IO_LINK) != 0;
2307+
21622308 s.has_user = true;
21632309 s.needs_lock = false;
21642310 s.needs_fixed_file = false;
21652311 submit++;
2166−
2167− ret = io_submit_sqe(ctx, &s, statep);
2168− if (ret)
2169− io_cqring_add_event(ctx, s.sqe->user_data, ret);
2312+ io_submit_sqe(ctx, &s, statep, &link);
21702313 }
21712314 io_commit_sqring(ctx);
21722315
2316+ if (link)
2317+ io_queue_sqe(ctx, link, &link->submit);
21732318 if (statep)
21742319 io_submit_state_end(statep);
21752320
include/uapi/linux/io_uring.h+1−0
@@ -40,6 +40,7 @@ struct io_uring_sqe { · 39 unmodified lines
4040 */
4141 #define IOSQE_FIXED_FILE (1U << 0) /* use fixed fileset */
4242 #define IOSQE_IO_DRAIN (1U << 1) /* issue after inflight IO */
43+#define IOSQE_IO_LINK (1U << 2) /* links next sqe */
4344
4445 /*
4546 * io_uring_setup() flags