History

io_uring: add support for sqe links

With SQE links, we can create chains of dependent SQEs. One example would be queueing an SQE that's a read from one file descriptor, with the linked SQE being a write to another with the same set of buffers. An SQE link will not stall the pipeline, it'll just ensure that dependent SQEs aren't issued before the previous link has completed. Any error at submission or completion time will break the chain of SQEs. For completions, this also includes short reads or writes, as the next SQE could depend on the previous one being fully completed. Any SQE in a chain that gets canceled due to any of the above errors, will get an CQE fill with -ECANCELED as the error value.

Jens Axboe committed 7 years ago

Commit 9e645e11 parent 9d93a3f2 files · +194 −48Browse files

Unified · Split

2 files
  1. fs/io_uring.c +193 −48
  2. include/uapi/linux/io_uring.h +1 −0
fs/io_uring.c+193−48
@@ -322,6 +322,7 @@ struct io_kiocb { · 321 unmodified lines
322 322
323 struct io_ring_ctx *ctx;323 struct io_ring_ctx *ctx;
324 struct list_head list;324 struct list_head list;
325 struct list_head link_list;
325 unsigned int flags;326 unsigned int flags;
326 refcount_t refs;327 refcount_t refs;
327#define REQ_F_NOWAIT 1 /* must not punt to workers */328#define REQ_F_NOWAIT 1 /* must not punt to workers */
@@ -330,8 +331,10 @@ struct io_kiocb { · 2 unmodified lines
330#define REQ_F_SEQ_PREV 8 /* sequential with previous */331#define REQ_F_SEQ_PREV 8 /* sequential with previous */
331#define REQ_F_IO_DRAIN 16 /* drain existing IO first */332#define REQ_F_IO_DRAIN 16 /* drain existing IO first */
332#define REQ_F_IO_DRAINED 32 /* drain done */333#define REQ_F_IO_DRAINED 32 /* drain done */
334#define REQ_F_LINK 64 /* linked sqes */
335#define REQ_F_FAIL_LINK 128 /* fail rest of links */
333 u64 user_data;336 u64 user_data;
334 u32 error; /* iopoll result from callback */337 u32 result;
335 u32 sequence;338 u32 sequence;
336 339
337 struct work_struct work;340 struct work_struct work;
@@ -583,6 +586,7 @@ static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx, · 245 unmodified lines
583 req->flags = 0;586 req->flags = 0;
584 /* one is dropped after submission, the other at completion */587 /* one is dropped after submission, the other at completion */
585 refcount_set(&req->refs, 2);588 refcount_set(&req->refs, 2);
589 req->result = 0;
586 return req;590 return req;
587out:591out:
588 io_ring_drop_ctx_refs(ctx, 1);592 io_ring_drop_ctx_refs(ctx, 1);
@@ -598,7 +602,7 @@ static void io_free_req_many(struct io_ring_ctx *ctx, void **reqs, int *nr) · 9 unmodified lines
598 }602 }
599}603}
600 604
601static void io_free_req(struct io_kiocb *req)605static void __io_free_req(struct io_kiocb *req)
602{606{
603 if (req->file && !(req->flags & REQ_F_FIXED_FILE))607 if (req->file && !(req->flags & REQ_F_FIXED_FILE))
604 fput(req->file);608 fput(req->file);
@@ -606,6 +610,63 @@ static void io_free_req(struct io_kiocb *req) · 1 unmodified line
606 kmem_cache_free(req_cachep, req);610 kmem_cache_free(req_cachep, req);
607}611}
608 612
613static void io_req_link_next(struct io_kiocb *req)
614{
615 struct io_kiocb *nxt;
616
617 /*
618 * The list should never be empty when we are called here. But could
619 * potentially happen if the chain is messed up, check to be on the
620 * safe side.
621 */
622 nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb, list);
623 if (nxt) {
624 list_del(&nxt->list);
625 if (!list_empty(&req->link_list)) {
626 INIT_LIST_HEAD(&nxt->link_list);
627 list_splice(&req->link_list, &nxt->link_list);
628 nxt->flags |= REQ_F_LINK;
629 }
630
631 INIT_WORK(&nxt->work, io_sq_wq_submit_work);
632 queue_work(req->ctx->sqo_wq, &nxt->work);
633 }
634}
635
636/*
637 * Called if REQ_F_LINK is set, and we fail the head request
638 */
639static void io_fail_links(struct io_kiocb *req)
640{
641 struct io_kiocb *link;
642
643 while (!list_empty(&req->link_list)) {
644 link = list_first_entry(&req->link_list, struct io_kiocb, list);
645 list_del(&link->list);
646
647 io_cqring_add_event(req->ctx, link->user_data, -ECANCELED);
648 __io_free_req(link);
649 }
650}
651
652static void io_free_req(struct io_kiocb *req)
653{
654 /*
655 * If LINK is set, we have dependent requests in this chain. If we
656 * didn't fail this request, queue the first one up, moving any other
657 * dependencies to the next request. In case of failure, fail the rest
658 * of the chain.
659 */
660 if (req->flags & REQ_F_LINK) {
661 if (req->flags & REQ_F_FAIL_LINK)
662 io_fail_links(req);
663 else
664 io_req_link_next(req);
665 }
666
667 __io_free_req(req);
668}
669
609static void io_put_req(struct io_kiocb *req)670static void io_put_req(struct io_kiocb *req)
610{671{
611 if (refcount_dec_and_test(&req->refs))672 if (refcount_dec_and_test(&req->refs))
@@ -627,16 +688,17 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, · 15 unmodified lines
627 req = list_first_entry(done, struct io_kiocb, list);688 req = list_first_entry(done, struct io_kiocb, list);
628 list_del(&req->list);689 list_del(&req->list);
629 690
630 io_cqring_fill_event(ctx, req->user_data, req->error);691 io_cqring_fill_event(ctx, req->user_data, req->result);
631 (*nr_events)++;692 (*nr_events)++;
632 693
633 if (refcount_dec_and_test(&req->refs)) {694 if (refcount_dec_and_test(&req->refs)) {
634 /* If we're not using fixed files, we have to pair the695 /* If we're not using fixed files, we have to pair the
635 * completion part with the file put. Use regular696 * completion part with the file put. Use regular
636 * completions for those, only batch free for fixed697 * completions for those, only batch free for fixed
637 * file.698 * file and non-linked commands.
638 */699 */
639 if (req->flags & REQ_F_FIXED_FILE) {700 if ((req->flags & (REQ_F_FIXED_FILE|REQ_F_LINK)) ==
701 REQ_F_FIXED_FILE) {
640 reqs[to_free++] = req;702 reqs[to_free++] = req;
641 if (to_free == ARRAY_SIZE(reqs))703 if (to_free == ARRAY_SIZE(reqs))
642 io_free_req_many(ctx, reqs, &to_free);704 io_free_req_many(ctx, reqs, &to_free);
@@ -775,6 +837,8 @@ static void io_complete_rw(struct kiocb *kiocb, long res, long res2) · 132 unmodified lines
775 837
776 kiocb_end_write(kiocb);838 kiocb_end_write(kiocb);
777 839
840 if ((req->flags & REQ_F_LINK) && res != req->result)
841 req->flags |= REQ_F_FAIL_LINK;
778 io_cqring_add_event(req->ctx, req->user_data, res);842 io_cqring_add_event(req->ctx, req->user_data, res);
779 io_put_req(req);843 io_put_req(req);
780}844}
@@ -785,7 +849,9 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) · 4 unmodified lines
785 849
786 kiocb_end_write(kiocb);850 kiocb_end_write(kiocb);
787 851
788 req->error = res;852 if ((req->flags & REQ_F_LINK) && res != req->result)
853 req->flags |= REQ_F_FAIL_LINK;
854 req->result = res;
789 if (res != -EAGAIN)855 if (res != -EAGAIN)
790 req->flags |= REQ_F_IOPOLL_COMPLETED;856 req->flags |= REQ_F_IOPOLL_COMPLETED;
791}857}
@@ -928,7 +994,6 @@ static int io_prep_rw(struct io_kiocb *req, const struct sqe_submit *s, · 136 unmodified lines
928 !kiocb->ki_filp->f_op->iopoll)994 !kiocb->ki_filp->f_op->iopoll)
929 return -EOPNOTSUPP;995 return -EOPNOTSUPP;
930 996
931 req->error = 0;
932 kiocb->ki_flags |= IOCB_HIPRI;997 kiocb->ki_flags |= IOCB_HIPRI;
933 kiocb->ki_complete = io_complete_rw_iopoll;998 kiocb->ki_complete = io_complete_rw_iopoll;
934 } else {999 } else {
@@ -1106,6 +1171,9 @@ static int io_read(struct io_kiocb *req, const struct sqe_submit *s, · 171 unmodified lines
1106 return ret;1171 return ret;
1107 1172
1108 read_size = ret;1173 read_size = ret;
1174 if (req->flags & REQ_F_LINK)
1175 req->result = read_size;
1176
1109 iov_count = iov_iter_count(&iter);1177 iov_count = iov_iter_count(&iter);
1110 ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_count);1178 ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_count);
1111 if (!ret) {1179 if (!ret) {
@@ -1163,6 +1231,9 @@ static int io_write(struct io_kiocb *req, const struct sqe_submit *s, · 51 unmodified lines
1163 if (ret < 0)1231 if (ret < 0)
1164 return ret;1232 return ret;
1165 1233
1234 if (req->flags & REQ_F_LINK)
1235 req->result = ret;
1236
1166 iov_count = iov_iter_count(&iter);1237 iov_count = iov_iter_count(&iter);
1167 1238
1168 ret = -EAGAIN;1239 ret = -EAGAIN;
@@ -1266,6 +1337,8 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 97 unmodified lines
1266 end > 0 ? end : LLONG_MAX,1337 end > 0 ? end : LLONG_MAX,
1267 fsync_flags & IORING_FSYNC_DATASYNC);1338 fsync_flags & IORING_FSYNC_DATASYNC);
1268 1339
1340 if (ret < 0 && (req->flags & REQ_F_LINK))
1341 req->flags |= REQ_F_FAIL_LINK;
1269 io_cqring_add_event(req->ctx, sqe->user_data, ret);1342 io_cqring_add_event(req->ctx, sqe->user_data, ret);
1270 io_put_req(req);1343 io_put_req(req);
1271 return 0;1344 return 0;
@@ -1310,6 +1383,8 @@ static int io_sync_file_range(struct io_kiocb *req, · 38 unmodified lines
1310 1383
1311 ret = sync_file_range(req->rw.ki_filp, sqe_off, sqe_len, flags);1384 ret = sync_file_range(req->rw.ki_filp, sqe_off, sqe_len, flags);
1312 1385
1386 if (ret < 0 && (req->flags & REQ_F_LINK))
1387 req->flags |= REQ_F_FAIL_LINK;
1313 io_cqring_add_event(req->ctx, sqe->user_data, ret);1388 io_cqring_add_event(req->ctx, sqe->user_data, ret);
1314 io_put_req(req);1389 io_put_req(req);
1315 return 0;1390 return 0;
@@ -1562,9 +1637,10 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, · 246 unmodified lines
1562{1637{
1563 int ret, opcode;1638 int ret, opcode;
1564 1639
1640 req->user_data = READ_ONCE(s->sqe->user_data);
1641
1565 if (unlikely(s->index >= ctx->sq_entries))1642 if (unlikely(s->index >= ctx->sq_entries))
1566 return -EINVAL;1643 return -EINVAL;
1567 req->user_data = READ_ONCE(s->sqe->user_data);
1568 1644
1569 opcode = READ_ONCE(s->sqe->opcode);1645 opcode = READ_ONCE(s->sqe->opcode);
1570 switch (opcode) {1646 switch (opcode) {
@@ -1608,7 +1684,7 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, · 37 unmodified lines
1608 return ret;1684 return ret;
1609 1685
1610 if (ctx->flags & IORING_SETUP_IOPOLL) {1686 if (ctx->flags & IORING_SETUP_IOPOLL) {
1611 if (req->error == -EAGAIN)1687 if (req->result == -EAGAIN)
1612 return -EAGAIN;1688 return -EAGAIN;
1613 1689
1614 /* workqueue context doesn't hold uring_lock, grab it now */1690 /* workqueue context doesn't hold uring_lock, grab it now */
@@ -1834,31 +1910,11 @@ static int io_req_set_file(struct io_ring_ctx *ctx, const struct sqe_submit *s, · 219 unmodified lines
1834 return 0;1910 return 0;
1835}1911}
1836 1912
1837static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,1913static int io_queue_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
1838 struct io_submit_state *state)1914 struct sqe_submit *s)
1839{1915{
1840 struct io_kiocb *req;
1841 int ret;1916 int ret;
1842 1917
1843 /* enforce forwards compatibility on users */
1844 if (unlikely(s->sqe->flags & ~(IOSQE_FIXED_FILE | IOSQE_IO_DRAIN)))
1845 return -EINVAL;
1846
1847 req = io_get_req(ctx, state);
1848 if (unlikely(!req))
1849 return -EAGAIN;
1850
1851 ret = io_req_set_file(ctx, s, state, req);
1852 if (unlikely(ret))
1853 goto out;
1854
1855 ret = io_req_defer(ctx, req, s->sqe);
1856 if (ret) {
1857 if (ret == -EIOCBQUEUED)
1858 ret = 0;
1859 return ret;
1860 }
1861
1862 ret = __io_submit_sqe(ctx, req, s, true);1918 ret = __io_submit_sqe(ctx, req, s, true);
1863 if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {1919 if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
1864 struct io_uring_sqe *sqe_copy;1920 struct io_uring_sqe *sqe_copy;
@@ -1881,24 +1937,93 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s, · 16 unmodified lines
1881 1937
1882 /*1938 /*
1883 * Queued up for async execution, worker will release1939 * Queued up for async execution, worker will release
1884 * submit reference when the iocb is actually1940 * submit reference when the iocb is actually submitted.
1885 * submitted.
1886 */1941 */
1887 return 0;1942 return 0;
1888 }1943 }
1889 }1944 }
1890 1945
1891out:
1892 /* drop submission reference */1946 /* drop submission reference */
1893 io_put_req(req);1947 io_put_req(req);
1894 1948
1895 /* and drop final reference, if we failed */1949 /* and drop final reference, if we failed */
1896 if (ret)1950 if (ret) {
1951 io_cqring_add_event(ctx, req->user_data, ret);
1952 if (req->flags & REQ_F_LINK)
1953 req->flags |= REQ_F_FAIL_LINK;
1897 io_put_req(req);1954 io_put_req(req);
1955 }
1898 1956
1899 return ret;1957 return ret;
1900}1958}
1901 1959
1960#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK)
1961
1962static void io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
1963 struct io_submit_state *state, struct io_kiocb **link)
1964{
1965 struct io_uring_sqe *sqe_copy;
1966 struct io_kiocb *req;
1967 int ret;
1968
1969 /* enforce forwards compatibility on users */
1970 if (unlikely(s->sqe->flags & ~SQE_VALID_FLAGS)) {
1971 ret = -EINVAL;
1972 goto err;
1973 }
1974
1975 req = io_get_req(ctx, state);
1976 if (unlikely(!req)) {
1977 ret = -EAGAIN;
1978 goto err;
1979 }
1980
1981 ret = io_req_set_file(ctx, s, state, req);
1982 if (unlikely(ret)) {
1983err_req:
1984 io_free_req(req);
1985err:
1986 io_cqring_add_event(ctx, s->sqe->user_data, ret);
1987 return;
1988 }
1989
1990 ret = io_req_defer(ctx, req, s->sqe);
1991 if (ret) {
1992 if (ret != -EIOCBQUEUED)
1993 goto err_req;
1994 return;
1995 }
1996
1997 /*
1998 * If we already have a head request, queue this one for async
1999 * submittal once the head completes. If we don't have a head but
2000 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
2001 * submitted sync once the chain is complete. If none of those
2002 * conditions are true (normal request), then just queue it.
2003 */
2004 if (*link) {
2005 struct io_kiocb *prev = *link;
2006
2007 sqe_copy = kmemdup(s->sqe, sizeof(*sqe_copy), GFP_KERNEL);
2008 if (!sqe_copy) {
2009 ret = -EAGAIN;
2010 goto err_req;
2011 }
2012
2013 s->sqe = sqe_copy;
2014 memcpy(&req->submit, s, sizeof(*s));
2015 list_add_tail(&req->list, &prev->link_list);
2016 } else if (s->sqe->flags & IOSQE_IO_LINK) {
2017 req->flags |= REQ_F_LINK;
2018
2019 memcpy(&req->submit, s, sizeof(*s));
2020 INIT_LIST_HEAD(&req->link_list);
2021 *link = req;
2022 } else {
2023 io_queue_sqe(ctx, req, s);
2024 }
2025}
2026
1902/*2027/*
1903 * Batched submission is done, ensure local IO is flushed out.2028 * Batched submission is done, ensure local IO is flushed out.
1904 */2029 */
@@ -1981,7 +2106,9 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, struct sqe_submit *sqes, · 76 unmodified lines
1981 unsigned int nr, bool has_user, bool mm_fault)2106 unsigned int nr, bool has_user, bool mm_fault)
1982{2107{
1983 struct io_submit_state state, *statep = NULL;2108 struct io_submit_state state, *statep = NULL;
1984 int ret, i, submitted = 0;2109 struct io_kiocb *link = NULL;
2110 bool prev_was_link = false;
2111 int i, submitted = 0;
1985 2112
1986 if (nr > IO_PLUG_THRESHOLD) {2113 if (nr > IO_PLUG_THRESHOLD) {
1987 io_submit_state_start(&state, ctx, nr);2114 io_submit_state_start(&state, ctx, nr);
@@ -1989,22 +2116,30 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, struct sqe_submit *sqes, · 1 unmodified line
1989 }2116 }
1990 2117
1991 for (i = 0; i < nr; i++) {2118 for (i = 0; i < nr; i++) {
2119 /*
2120 * If previous wasn't linked and we have a linked command,
2121 * that's the end of the chain. Submit the previous link.
2122 */
2123 if (!prev_was_link && link) {
2124 io_queue_sqe(ctx, link, &link->submit);
2125 link = NULL;
2126 }
2127 prev_was_link = (sqes[i].sqe->flags & IOSQE_IO_LINK) != 0;
2128
1992 if (unlikely(mm_fault)) {2129 if (unlikely(mm_fault)) {
1993 ret = -EFAULT;2130 io_cqring_add_event(ctx, sqes[i].sqe->user_data,
2131 -EFAULT);
1994 } else {2132 } else {
1995 sqes[i].has_user = has_user;2133 sqes[i].has_user = has_user;
1996 sqes[i].needs_lock = true;2134 sqes[i].needs_lock = true;
1997 sqes[i].needs_fixed_file = true;2135 sqes[i].needs_fixed_file = true;
1998 ret = io_submit_sqe(ctx, &sqes[i], statep);2136 io_submit_sqe(ctx, &sqes[i], statep, &link);
1999 }
2000 if (!ret) {
2001 submitted++;2137 submitted++;
2002 continue;
2003 }2138 }
2004
2005 io_cqring_add_event(ctx, sqes[i].sqe->user_data, ret);
2006 }2139 }
2007 2140
2141 if (link)
2142 io_queue_sqe(ctx, link, &link->submit);
2008 if (statep)2143 if (statep)
2009 io_submit_state_end(&state);2144 io_submit_state_end(&state);
2010 2145
@@ -2145,6 +2280,8 @@ static int io_sq_thread(void *data) · 134 unmodified lines
2145static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)2280static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)
2146{2281{
2147 struct io_submit_state state, *statep = NULL;2282 struct io_submit_state state, *statep = NULL;
2283 struct io_kiocb *link = NULL;
2284 bool prev_was_link = false;
2148 int i, submit = 0;2285 int i, submit = 0;
2149 2286
2150 if (to_submit > IO_PLUG_THRESHOLD) {2287 if (to_submit > IO_PLUG_THRESHOLD) {
@@ -2154,22 +2291,30 @@ static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit) · 3 unmodified lines
2154 2291
2155 for (i = 0; i < to_submit; i++) {2292 for (i = 0; i < to_submit; i++) {
2156 struct sqe_submit s;2293 struct sqe_submit s;
2157 int ret;
2158 2294
2159 if (!io_get_sqring(ctx, &s))2295 if (!io_get_sqring(ctx, &s))
2160 break;2296 break;
2161 2297
2298 /*
2299 * If previous wasn't linked and we have a linked command,
2300 * that's the end of the chain. Submit the previous link.
2301 */
2302 if (!prev_was_link && link) {
2303 io_queue_sqe(ctx, link, &link->submit);
2304 link = NULL;
2305 }
2306 prev_was_link = (s.sqe->flags & IOSQE_IO_LINK) != 0;
2307
2162 s.has_user = true;2308 s.has_user = true;
2163 s.needs_lock = false;2309 s.needs_lock = false;
2164 s.needs_fixed_file = false;2310 s.needs_fixed_file = false;
2165 submit++;2311 submit++;
2166 2312 io_submit_sqe(ctx, &s, statep, &link);
2167 ret = io_submit_sqe(ctx, &s, statep);
2168 if (ret)
2169 io_cqring_add_event(ctx, s.sqe->user_data, ret);
2170 }2313 }
2171 io_commit_sqring(ctx);2314 io_commit_sqring(ctx);
2172 2315
2316 if (link)
2317 io_queue_sqe(ctx, link, &link->submit);
2173 if (statep)2318 if (statep)
2174 io_submit_state_end(statep);2319 io_submit_state_end(statep);
2175 2320
include/uapi/linux/io_uring.h+1−0
@@ -40,6 +40,7 @@ struct io_uring_sqe { · 39 unmodified lines
40 */40 */
41#define IOSQE_FIXED_FILE (1U << 0) /* use fixed fileset */41#define IOSQE_FIXED_FILE (1U << 0) /* use fixed fileset */
42#define IOSQE_IO_DRAIN (1U << 1) /* issue after inflight IO */42#define IOSQE_IO_DRAIN (1U << 1) /* issue after inflight IO */
43#define IOSQE_IO_LINK (1U << 2) /* links next sqe */
43 44
44/*45/*
45 * io_uring_setup() flags46 * io_uring_setup() flags