History

Merge tag 'io_uring-futex-2023-10-30' of git://git.kernel.dk/linux

Pull io_uring futex support from Jens Axboe: "This adds support for using futexes through io_uring - first futex wake and wait, and then the vectored variant of waiting, futex waitv. For both wait/wake/waitv, we support the bitset variant, as the 'normal' variants can be easily implemented on top of that. PI and requeue are not supported through io_uring, just the above mentioned parts. This may change in the future, but in the spirit of keeping this small (and based on what people have been asking for), this is what we currently have. Wake support is pretty straight forward, most of the thought has gone into the wait side to avoid needing to offload wait operations to a blocking context. Instead, we rely on the usual callbacks to retry and post a completion event, when appropriate. As far as I can recall, the first request for futex support with io_uring came from Andres Freund, working on postgres. His aio rework of postgres was one of the early adopters of io_uring, and futex support was a natural extension for that. This is relevant from both a usability point of view, as well as for effiency and performance. In Andres's words, for the former: Futex wait support in io_uring makes it a lot easier to avoid deadlocks in concurrent programs that have their own buffer pool: Obviously pages in the application buffer pool have to be locked during IO. If the initiator of IO A needs to wait for a held lock B, the holder of lock B might wait for the IO A to complete. The ability to wait for a lock and IO completions at the same time provides an efficient way to avoid such deadlocks and in terms of effiency, even without unlocking the full potential yet, Andres says: Futex wake support in io_uring is useful because it allows for more efficient directed wakeups. For some "locks" postgres has queues implemented in userspace, with wakeup logic that cannot easily be implemented with FUTEX_WAKE_BITSET on a single "futex word" (imagine waiting for journal flushes to have completed up to a certain point). Thus a "lock release" sometimes need to wake up many processes in a row. A quick-and-dirty conversion to doing these wakeups via io_uring lead to a 3% throughput increase, with 12% fewer context switches, albeit in a fairly extreme workload" * tag 'io_uring-futex-2023-10-30' of git://git.kernel.dk/linux: io_uring: add support for vectored futex waits futex: make the vectored futex operations available futex: make futex_parse_waitv() available as a helper futex: add wake_data to struct futex_q io_uring: add support for futex wake and wait futex: abstract out a __futex_wake_mark() helper futex: factor out the futex wake handling futex: move FUTEX2_VALID_MASK to futex.h

Linus Torvalds committed 3 years ago

Commit 4de520f2 parents f5277ad 8f3501913 files · +545 −27Browse files

Unified · Split

13 files
  1. include/linux/io_uring_types.h +5 −0
  2. include/uapi/linux/io_uring.h +4 −0
  3. io_uring/Makefile +1 −0
  4. io_uring/cancel.c +5 −0
  5. io_uring/cancel.h +4 −0
  6. io_uring/futex.c +386 −0
  7. io_uring/futex.h +36 −0
  8. io_uring/io_uring.c +7 −0
  9. io_uring/opdef.c +34 −0
  10. kernel/futex/futex.h +20 −0
  11. kernel/futex/requeue.c +2 −1
  12. kernel/futex/syscalls.c +11 −7
  13. kernel/futex/waitwake.c +30 −19
include/linux/io_uring_types.h+5−0
@@ -321,6 +321,11 @@ struct io_ring_ctx { · 320 unmodified lines
321 321
322 struct hlist_head waitid_list;322 struct hlist_head waitid_list;
323 323
324#ifdef CONFIG_FUTEX
325 struct hlist_head futex_list;
326 struct io_alloc_cache futex_cache;
327#endif
328
324 const struct cred *sq_creds; /* cred used for __io_sq_thread() */329 const struct cred *sq_creds; /* cred used for __io_sq_thread() */
325 struct io_sq_data *sq_data; /* if using sq thread polling */330 struct io_sq_data *sq_data; /* if using sq thread polling */
326 331
include/uapi/linux/io_uring.h+4−0
@@ -70,6 +70,7 @@ struct io_uring_sqe { · 69 unmodified lines
70 __u32 msg_ring_flags;70 __u32 msg_ring_flags;
71 __u32 uring_cmd_flags;71 __u32 uring_cmd_flags;
72 __u32 waitid_flags;72 __u32 waitid_flags;
73 __u32 futex_flags;
73 };74 };
74 __u64 user_data; /* data to be passed back at completion time */75 __u64 user_data; /* data to be passed back at completion time */
75 /* pack this to avoid bogus arm OABI complaints */76 /* pack this to avoid bogus arm OABI complaints */
@@ -249,6 +250,9 @@ enum io_uring_op { · 173 unmodified lines
249 IORING_OP_SENDMSG_ZC,250 IORING_OP_SENDMSG_ZC,
250 IORING_OP_READ_MULTISHOT,251 IORING_OP_READ_MULTISHOT,
251 IORING_OP_WAITID,252 IORING_OP_WAITID,
253 IORING_OP_FUTEX_WAIT,
254 IORING_OP_FUTEX_WAKE,
255 IORING_OP_FUTEX_WAITV,
252 256
253 /* this goes last, obviously */257 /* this goes last, obviously */
254 IORING_OP_LAST,258 IORING_OP_LAST,
io_uring/Makefile+1−0
@@ -10,3 +10,4 @@ obj-$(CONFIG_IO_URING) += io_uring.o xattr.o nop.o fs.o splice.o \ · 9 unmodified lines
10 cancel.o kbuf.o rsrc.o rw.o opdef.o \10 cancel.o kbuf.o rsrc.o rw.o opdef.o \
11 notif.o waitid.o11 notif.o waitid.o
12obj-$(CONFIG_IO_WQ) += io-wq.o12obj-$(CONFIG_IO_WQ) += io-wq.o
13obj-$(CONFIG_FUTEX) += futex.o
io_uring/cancel.c+5−0
@@ -16,6 +16,7 @@ · 15 unmodified lines
16#include "poll.h"16#include "poll.h"
17#include "timeout.h"17#include "timeout.h"
18#include "waitid.h"18#include "waitid.h"
19#include "futex.h"
19#include "cancel.h"20#include "cancel.h"
20 21
21struct io_cancel {22struct io_cancel {
@@ -124,6 +125,10 @@ int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd, · 102 unmodified lines
124 if (ret != -ENOENT)125 if (ret != -ENOENT)
125 return ret;126 return ret;
126 127
128 ret = io_futex_cancel(ctx, cd, issue_flags);
129 if (ret != -ENOENT)
130 return ret;
131
127 spin_lock(&ctx->completion_lock);132 spin_lock(&ctx->completion_lock);
128 if (!(cd->flags & IORING_ASYNC_CANCEL_FD))133 if (!(cd->flags & IORING_ASYNC_CANCEL_FD))
129 ret = io_timeout_cancel(ctx, cd);134 ret = io_timeout_cancel(ctx, cd);
io_uring/cancel.h+4−0
@@ -1,4 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2#ifndef IORING_CANCEL_H
3#define IORING_CANCEL_H
2 4
3#include <linux/io_uring_types.h>5#include <linux/io_uring_types.h>
4 6
@@ -22,3 +24,5 @@ void init_hash_table(struct io_hash_table *table, unsigned size); · 17 unmodified lines
22 24
23int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg);25int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg);
24bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd);26bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd);
27
28#endif
io_uring/futex.c+386−0
@@ -0,0 +1,386 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/fs.h>
5#include <linux/file.h>
6#include <linux/io_uring.h>
7
8#include <uapi/linux/io_uring.h>
9
10#include "../kernel/futex/futex.h"
11#include "io_uring.h"
12#include "rsrc.h"
13#include "futex.h"
14
15struct io_futex {
16 struct file *file;
17 union {
18 u32 __user *uaddr;
19 struct futex_waitv __user *uwaitv;
20 };
21 unsigned long futex_val;
22 unsigned long futex_mask;
23 unsigned long futexv_owned;
24 u32 futex_flags;
25 unsigned int futex_nr;
26 bool futexv_unqueued;
27};
28
29struct io_futex_data {
30 union {
31 struct futex_q q;
32 struct io_cache_entry cache;
33 };
34 struct io_kiocb *req;
35};
36
37void io_futex_cache_init(struct io_ring_ctx *ctx)
38{
39 io_alloc_cache_init(&ctx->futex_cache, IO_NODE_ALLOC_CACHE_MAX,
40 sizeof(struct io_futex_data));
41}
42
43static void io_futex_cache_entry_free(struct io_cache_entry *entry)
44{
45 kfree(container_of(entry, struct io_futex_data, cache));
46}
47
48void io_futex_cache_free(struct io_ring_ctx *ctx)
49{
50 io_alloc_cache_free(&ctx->futex_cache, io_futex_cache_entry_free);
51}
52
53static void __io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts)
54{
55 req->async_data = NULL;
56 hlist_del_init(&req->hash_node);
57 io_req_task_complete(req, ts);
58}
59
60static void io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts)
61{
62 struct io_futex_data *ifd = req->async_data;
63 struct io_ring_ctx *ctx = req->ctx;
64
65 io_tw_lock(ctx, ts);
66 if (!io_alloc_cache_put(&ctx->futex_cache, &ifd->cache))
67 kfree(ifd);
68 __io_futex_complete(req, ts);
69}
70
71static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts)
72{
73 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
74 struct futex_vector *futexv = req->async_data;
75
76 io_tw_lock(req->ctx, ts);
77
78 if (!iof->futexv_unqueued) {
79 int res;
80
81 res = futex_unqueue_multiple(futexv, iof->futex_nr);
82 if (res != -1)
83 io_req_set_res(req, res, 0);
84 }
85
86 kfree(req->async_data);
87 req->flags &= ~REQ_F_ASYNC_DATA;
88 __io_futex_complete(req, ts);
89}
90
91static bool io_futexv_claim(struct io_futex *iof)
92{
93 if (test_bit(0, &iof->futexv_owned) ||
94 test_and_set_bit_lock(0, &iof->futexv_owned))
95 return false;
96 return true;
97}
98
99static bool __io_futex_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req)
100{
101 /* futex wake already done or in progress */
102 if (req->opcode == IORING_OP_FUTEX_WAIT) {
103 struct io_futex_data *ifd = req->async_data;
104
105 if (!futex_unqueue(&ifd->q))
106 return false;
107 req->io_task_work.func = io_futex_complete;
108 } else {
109 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
110
111 if (!io_futexv_claim(iof))
112 return false;
113 req->io_task_work.func = io_futexv_complete;
114 }
115
116 hlist_del_init(&req->hash_node);
117 io_req_set_res(req, -ECANCELED, 0);
118 io_req_task_work_add(req);
119 return true;
120}
121
122int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
123 unsigned int issue_flags)
124{
125 struct hlist_node *tmp;
126 struct io_kiocb *req;
127 int nr = 0;
128
129 if (cd->flags & (IORING_ASYNC_CANCEL_FD|IORING_ASYNC_CANCEL_FD_FIXED))
130 return -ENOENT;
131
132 io_ring_submit_lock(ctx, issue_flags);
133 hlist_for_each_entry_safe(req, tmp, &ctx->futex_list, hash_node) {
134 if (req->cqe.user_data != cd->data &&
135 !(cd->flags & IORING_ASYNC_CANCEL_ANY))
136 continue;
137 if (__io_futex_cancel(ctx, req))
138 nr++;
139 if (!(cd->flags & IORING_ASYNC_CANCEL_ALL))
140 break;
141 }
142 io_ring_submit_unlock(ctx, issue_flags);
143
144 if (nr)
145 return nr;
146
147 return -ENOENT;
148}
149
150bool io_futex_remove_all(struct io_ring_ctx *ctx, struct task_struct *task,
151 bool cancel_all)
152{
153 struct hlist_node *tmp;
154 struct io_kiocb *req;
155 bool found = false;
156
157 lockdep_assert_held(&ctx->uring_lock);
158
159 hlist_for_each_entry_safe(req, tmp, &ctx->futex_list, hash_node) {
160 if (!io_match_task_safe(req, task, cancel_all))
161 continue;
162 __io_futex_cancel(ctx, req);
163 found = true;
164 }
165
166 return found;
167}
168
169int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
170{
171 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
172 u32 flags;
173
174 if (unlikely(sqe->len || sqe->futex_flags || sqe->buf_index ||
175 sqe->file_index))
176 return -EINVAL;
177
178 iof->uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr));
179 iof->futex_val = READ_ONCE(sqe->addr2);
180 iof->futex_mask = READ_ONCE(sqe->addr3);
181 flags = READ_ONCE(sqe->fd);
182
183 if (flags & ~FUTEX2_VALID_MASK)
184 return -EINVAL;
185
186 iof->futex_flags = futex2_to_flags(flags);
187 if (!futex_flags_valid(iof->futex_flags))
188 return -EINVAL;
189
190 if (!futex_validate_input(iof->futex_flags, iof->futex_val) ||
191 !futex_validate_input(iof->futex_flags, iof->futex_mask))
192 return -EINVAL;
193
194 return 0;
195}
196
197static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q)
198{
199 struct io_kiocb *req = q->wake_data;
200 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
201
202 if (!io_futexv_claim(iof))
203 return;
204 if (unlikely(!__futex_wake_mark(q)))
205 return;
206
207 io_req_set_res(req, 0, 0);
208 req->io_task_work.func = io_futexv_complete;
209 io_req_task_work_add(req);
210}
211
212int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
213{
214 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
215 struct futex_vector *futexv;
216 int ret;
217
218 /* No flags or mask supported for waitv */
219 if (unlikely(sqe->fd || sqe->buf_index || sqe->file_index ||
220 sqe->addr2 || sqe->futex_flags || sqe->addr3))
221 return -EINVAL;
222
223 iof->uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr));
224 iof->futex_nr = READ_ONCE(sqe->len);
225 if (!iof->futex_nr || iof->futex_nr > FUTEX_WAITV_MAX)
226 return -EINVAL;
227
228 futexv = kcalloc(iof->futex_nr, sizeof(*futexv), GFP_KERNEL);
229 if (!futexv)
230 return -ENOMEM;
231
232 ret = futex_parse_waitv(futexv, iof->uwaitv, iof->futex_nr,
233 io_futex_wakev_fn, req);
234 if (ret) {
235 kfree(futexv);
236 return ret;
237 }
238
239 iof->futexv_owned = 0;
240 iof->futexv_unqueued = 0;
241 req->flags |= REQ_F_ASYNC_DATA;
242 req->async_data = futexv;
243 return 0;
244}
245
246static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q)
247{
248 struct io_futex_data *ifd = container_of(q, struct io_futex_data, q);
249 struct io_kiocb *req = ifd->req;
250
251 if (unlikely(!__futex_wake_mark(q)))
252 return;
253
254 io_req_set_res(req, 0, 0);
255 req->io_task_work.func = io_futex_complete;
256 io_req_task_work_add(req);
257}
258
259static struct io_futex_data *io_alloc_ifd(struct io_ring_ctx *ctx)
260{
261 struct io_cache_entry *entry;
262
263 entry = io_alloc_cache_get(&ctx->futex_cache);
264 if (entry)
265 return container_of(entry, struct io_futex_data, cache);
266
267 return kmalloc(sizeof(struct io_futex_data), GFP_NOWAIT);
268}
269
270int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags)
271{
272 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
273 struct futex_vector *futexv = req->async_data;
274 struct io_ring_ctx *ctx = req->ctx;
275 int ret, woken = -1;
276
277 io_ring_submit_lock(ctx, issue_flags);
278
279 ret = futex_wait_multiple_setup(futexv, iof->futex_nr, &woken);
280
281 /*
282 * Error case, ret is < 0. Mark the request as failed.
283 */
284 if (unlikely(ret < 0)) {
285 io_ring_submit_unlock(ctx, issue_flags);
286 req_set_fail(req);
287 io_req_set_res(req, ret, 0);
288 kfree(futexv);
289 req->async_data = NULL;
290 req->flags &= ~REQ_F_ASYNC_DATA;
291 return IOU_OK;
292 }
293
294 /*
295 * 0 return means that we successfully setup the waiters, and that
296 * nobody triggered a wakeup while we were doing so. If the wakeup
297 * happened post setup, the task_work will be run post this issue and
298 * under the submission lock. 1 means We got woken while setting up,
299 * let that side do the completion. Note that
300 * futex_wait_multiple_setup() will have unqueued all the futexes in
301 * this case. Mark us as having done that already, since this is
302 * different from normal wakeup.
303 */
304 if (!ret) {
305 /*
306 * If futex_wait_multiple_setup() returns 0 for a
307 * successful setup, then the task state will not be
308 * runnable. This is fine for the sync syscall, as
309 * it'll be blocking unless we already got one of the
310 * futexes woken, but it obviously won't work for an
311 * async invocation. Mark us runnable again.
312 */
313 __set_current_state(TASK_RUNNING);
314 hlist_add_head(&req->hash_node, &ctx->futex_list);
315 } else {
316 iof->futexv_unqueued = 1;
317 if (woken != -1)
318 io_req_set_res(req, woken, 0);
319 }
320
321 io_ring_submit_unlock(ctx, issue_flags);
322 return IOU_ISSUE_SKIP_COMPLETE;
323}
324
325int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags)
326{
327 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
328 struct io_ring_ctx *ctx = req->ctx;
329 struct io_futex_data *ifd = NULL;
330 struct futex_hash_bucket *hb;
331 int ret;
332
333 if (!iof->futex_mask) {
334 ret = -EINVAL;
335 goto done;
336 }
337
338 io_ring_submit_lock(ctx, issue_flags);
339 ifd = io_alloc_ifd(ctx);
340 if (!ifd) {
341 ret = -ENOMEM;
342 goto done_unlock;
343 }
344
345 req->async_data = ifd;
346 ifd->q = futex_q_init;
347 ifd->q.bitset = iof->futex_mask;
348 ifd->q.wake = io_futex_wake_fn;
349 ifd->req = req;
350
351 ret = futex_wait_setup(iof->uaddr, iof->futex_val, iof->futex_flags,
352 &ifd->q, &hb);
353 if (!ret) {
354 hlist_add_head(&req->hash_node, &ctx->futex_list);
355 io_ring_submit_unlock(ctx, issue_flags);
356
357 futex_queue(&ifd->q, hb);
358 return IOU_ISSUE_SKIP_COMPLETE;
359 }
360
361done_unlock:
362 io_ring_submit_unlock(ctx, issue_flags);
363done:
364 if (ret < 0)
365 req_set_fail(req);
366 io_req_set_res(req, ret, 0);
367 kfree(ifd);
368 return IOU_OK;
369}
370
371int io_futex_wake(struct io_kiocb *req, unsigned int issue_flags)
372{
373 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
374 int ret;
375
376 /*
377 * Strict flags - ensure that waking 0 futexes yields a 0 result.
378 * See commit 43adf8449510 ("futex: FLAGS_STRICT") for details.
379 */
380 ret = futex_wake(iof->uaddr, FLAGS_STRICT | iof->futex_flags,
381 iof->futex_val, iof->futex_mask);
382 if (ret < 0)
383 req_set_fail(req);
384 io_req_set_res(req, ret, 0);
385 return IOU_OK;
386}
io_uring/futex.h+36−0
@@ -0,0 +1,36 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include "cancel.h"
4
5int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
6int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
7int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags);
8int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags);
9int io_futex_wake(struct io_kiocb *req, unsigned int issue_flags);
10
11#if defined(CONFIG_FUTEX)
12int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
13 unsigned int issue_flags);
14bool io_futex_remove_all(struct io_ring_ctx *ctx, struct task_struct *task,
15 bool cancel_all);
16void io_futex_cache_init(struct io_ring_ctx *ctx);
17void io_futex_cache_free(struct io_ring_ctx *ctx);
18#else
19static inline int io_futex_cancel(struct io_ring_ctx *ctx,
20 struct io_cancel_data *cd,
21 unsigned int issue_flags)
22{
23 return 0;
24}
25static inline bool io_futex_remove_all(struct io_ring_ctx *ctx,
26 struct task_struct *task, bool cancel_all)
27{
28 return false;
29}
30static inline void io_futex_cache_init(struct io_ring_ctx *ctx)
31{
32}
33static inline void io_futex_cache_free(struct io_ring_ctx *ctx)
34{
35}
36#endif
io_uring/io_uring.c+7−0
@@ -93,6 +93,7 @@ · 92 unmodified lines
93#include "net.h"93#include "net.h"
94#include "notif.h"94#include "notif.h"
95#include "waitid.h"95#include "waitid.h"
96#include "futex.h"
96 97
97#include "timeout.h"98#include "timeout.h"
98#include "poll.h"99#include "poll.h"
@@ -330,6 +331,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) · 231 unmodified lines
330 sizeof(struct async_poll));331 sizeof(struct async_poll));
331 io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,332 io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,
332 sizeof(struct io_async_msghdr));333 sizeof(struct io_async_msghdr));
334 io_futex_cache_init(ctx);
333 init_completion(&ctx->ref_comp);335 init_completion(&ctx->ref_comp);
334 xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);336 xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
335 mutex_init(&ctx->uring_lock);337 mutex_init(&ctx->uring_lock);
@@ -349,6 +351,9 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) · 13 unmodified lines
349 ctx->submit_state.free_list.next = NULL;351 ctx->submit_state.free_list.next = NULL;
350 INIT_WQ_LIST(&ctx->locked_free_list);352 INIT_WQ_LIST(&ctx->locked_free_list);
351 INIT_HLIST_HEAD(&ctx->waitid_list);353 INIT_HLIST_HEAD(&ctx->waitid_list);
354#ifdef CONFIG_FUTEX
355 INIT_HLIST_HEAD(&ctx->futex_list);
356#endif
352 INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);357 INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);
353 INIT_WQ_LIST(&ctx->submit_state.compl_reqs);358 INIT_WQ_LIST(&ctx->submit_state.compl_reqs);
354 INIT_HLIST_HEAD(&ctx->cancelable_uring_cmd);359 INIT_HLIST_HEAD(&ctx->cancelable_uring_cmd);
@@ -2914,6 +2919,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) · 2559 unmodified lines
2914 io_eventfd_unregister(ctx);2919 io_eventfd_unregister(ctx);
2915 io_alloc_cache_free(&ctx->apoll_cache, io_apoll_cache_free);2920 io_alloc_cache_free(&ctx->apoll_cache, io_apoll_cache_free);
2916 io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free);2921 io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free);
2922 io_futex_cache_free(ctx);
2917 io_destroy_buffers(ctx);2923 io_destroy_buffers(ctx);
2918 mutex_unlock(&ctx->uring_lock);2924 mutex_unlock(&ctx->uring_lock);
2919 if (ctx->sq_creds)2925 if (ctx->sq_creds)
@@ -3357,6 +3363,7 @@ static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, · 437 unmodified lines
3357 mutex_lock(&ctx->uring_lock);3363 mutex_lock(&ctx->uring_lock);
3358 ret |= io_poll_remove_all(ctx, task, cancel_all);3364 ret |= io_poll_remove_all(ctx, task, cancel_all);
3359 ret |= io_waitid_remove_all(ctx, task, cancel_all);3365 ret |= io_waitid_remove_all(ctx, task, cancel_all);
3366 ret |= io_futex_remove_all(ctx, task, cancel_all);
3360 ret |= io_uring_try_cancel_uring_cmd(ctx, task, cancel_all);3367 ret |= io_uring_try_cancel_uring_cmd(ctx, task, cancel_all);
3361 mutex_unlock(&ctx->uring_lock);3368 mutex_unlock(&ctx->uring_lock);
3362 ret |= io_kill_timeouts(ctx, task, cancel_all);3369 ret |= io_kill_timeouts(ctx, task, cancel_all);
io_uring/opdef.c+34−0
@@ -34,6 +34,7 @@ · 33 unmodified lines
34#include "cancel.h"34#include "cancel.h"
35#include "rw.h"35#include "rw.h"
36#include "waitid.h"36#include "waitid.h"
37#include "futex.h"
37 38
38static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)39static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
39{40{
@@ -444,6 +445,30 @@ const struct io_issue_def io_issue_defs[] = { · 404 unmodified lines
444 .prep = io_waitid_prep,445 .prep = io_waitid_prep,
445 .issue = io_waitid,446 .issue = io_waitid,
446 },447 },
448 [IORING_OP_FUTEX_WAIT] = {
449#if defined(CONFIG_FUTEX)
450 .prep = io_futex_prep,
451 .issue = io_futex_wait,
452#else
453 .prep = io_eopnotsupp_prep,
454#endif
455 },
456 [IORING_OP_FUTEX_WAKE] = {
457#if defined(CONFIG_FUTEX)
458 .prep = io_futex_prep,
459 .issue = io_futex_wake,
460#else
461 .prep = io_eopnotsupp_prep,
462#endif
463 },
464 [IORING_OP_FUTEX_WAITV] = {
465#if defined(CONFIG_FUTEX)
466 .prep = io_futexv_prep,
467 .issue = io_futexv_wait,
468#else
469 .prep = io_eopnotsupp_prep,
470#endif
471 },
447};472};
448 473
449const struct io_cold_def io_cold_defs[] = {474const struct io_cold_def io_cold_defs[] = {
@@ -670,6 +695,15 @@ const struct io_cold_def io_cold_defs[] = { · 220 unmodified lines
670 .name = "WAITID",695 .name = "WAITID",
671 .async_size = sizeof(struct io_waitid_async),696 .async_size = sizeof(struct io_waitid_async),
672 },697 },
698 [IORING_OP_FUTEX_WAIT] = {
699 .name = "FUTEX_WAIT",
700 },
701 [IORING_OP_FUTEX_WAKE] = {
702 .name = "FUTEX_WAKE",
703 },
704 [IORING_OP_FUTEX_WAITV] = {
705 .name = "FUTEX_WAITV",
706 },
673};707};
674 708
675const char *io_uring_get_opcode(u8 opcode)709const char *io_uring_get_opcode(u8 opcode)
kernel/futex/futex.h+20−0
@@ -52,6 +52,8 @@ static inline unsigned int futex_to_flags(unsigned int op) · 51 unmodified lines
52 return flags;52 return flags;
53}53}
54 54
55#define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK | FUTEX2_PRIVATE)
56
55/* FUTEX2_ to FLAGS_ */57/* FUTEX2_ to FLAGS_ */
56static inline unsigned int futex2_to_flags(unsigned int flags2)58static inline unsigned int futex2_to_flags(unsigned int flags2)
57{59{
@@ -137,11 +139,16 @@ struct futex_pi_state { · 79 unmodified lines
137 union futex_key key;139 union futex_key key;
138} __randomize_layout;140} __randomize_layout;
139 141
142struct futex_q;
143typedef void (futex_wake_fn)(struct wake_q_head *wake_q, struct futex_q *q);
144
140/**145/**
141 * struct futex_q - The hashed futex queue entry, one per waiting task146 * struct futex_q - The hashed futex queue entry, one per waiting task
142 * @list: priority-sorted list of tasks waiting on this futex147 * @list: priority-sorted list of tasks waiting on this futex
143 * @task: the task waiting on the futex148 * @task: the task waiting on the futex
144 * @lock_ptr: the hash bucket lock149 * @lock_ptr: the hash bucket lock
150 * @wake: the wake handler for this queue
151 * @wake_data: data associated with the wake handler
145 * @key: the key the futex is hashed on152 * @key: the key the futex is hashed on
146 * @pi_state: optional priority inheritance state153 * @pi_state: optional priority inheritance state
147 * @rt_waiter: rt_waiter storage for use with requeue_pi154 * @rt_waiter: rt_waiter storage for use with requeue_pi
@@ -166,6 +173,8 @@ struct futex_q { · 18 unmodified lines
166 173
167 struct task_struct *task;174 struct task_struct *task;
168 spinlock_t *lock_ptr;175 spinlock_t *lock_ptr;
176 futex_wake_fn *wake;
177 void *wake_data;
169 union futex_key key;178 union futex_key key;
170 struct futex_pi_state *pi_state;179 struct futex_pi_state *pi_state;
171 struct rt_mutex_waiter *rt_waiter;180 struct rt_mutex_waiter *rt_waiter;
@@ -212,6 +221,7 @@ extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, · 40 unmodified lines
212 struct futex_q *q, struct futex_hash_bucket **hb);221 struct futex_q *q, struct futex_hash_bucket **hb);
213extern void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q,222extern void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q,
214 struct hrtimer_sleeper *timeout);223 struct hrtimer_sleeper *timeout);
224extern bool __futex_wake_mark(struct futex_q *q);
215extern void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q);225extern void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q);
216 226
217extern int fault_in_user_writeable(u32 __user *uaddr);227extern int fault_in_user_writeable(u32 __user *uaddr);
@@ -351,6 +361,16 @@ struct futex_vector { · 133 unmodified lines
351 struct futex_q q;361 struct futex_q q;
352};362};
353 363
364extern int futex_parse_waitv(struct futex_vector *futexv,
365 struct futex_waitv __user *uwaitv,
366 unsigned int nr_futexes, futex_wake_fn *wake,
367 void *wake_data);
368
369extern int futex_wait_multiple_setup(struct futex_vector *vs, int count,
370 int *woken);
371
372extern int futex_unqueue_multiple(struct futex_vector *v, int count);
373
354extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count,374extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count,
355 struct hrtimer_sleeper *to);375 struct hrtimer_sleeper *to);
356 376
kernel/futex/requeue.c+2−1
@@ -58,6 +58,7 @@ enum { · 57 unmodified lines
58 58
59const struct futex_q futex_q_init = {59const struct futex_q futex_q_init = {
60 /* list gets initialized in futex_queue()*/60 /* list gets initialized in futex_queue()*/
61 .wake = futex_wake_mark,
61 .key = FUTEX_KEY_INIT,62 .key = FUTEX_KEY_INIT,
62 .bitset = FUTEX_BITSET_MATCH_ANY,63 .bitset = FUTEX_BITSET_MATCH_ANY,
63 .requeue_state = ATOMIC_INIT(Q_REQUEUE_PI_NONE),64 .requeue_state = ATOMIC_INIT(Q_REQUEUE_PI_NONE),
@@ -593,7 +594,7 @@ retry_private: · 529 unmodified lines
593 /* Plain futexes just wake or requeue and are done */594 /* Plain futexes just wake or requeue and are done */
594 if (!requeue_pi) {595 if (!requeue_pi) {
595 if (++task_count <= nr_wake)596 if (++task_count <= nr_wake)
596 futex_wake_mark(&wake_q, this);597 this->wake(&wake_q, this);
597 else598 else
598 requeue_futex(this, hb1, hb2, &key2);599 requeue_futex(this, hb1, hb2, &key2);
599 continue;600 continue;
kernel/futex/syscalls.c+11−7
@@ -179,19 +179,20 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val, · 178 unmodified lines
179 return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3);179 return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3);
180}180}
181 181
182#define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK | FUTEX2_PRIVATE)
183
184/**182/**
185 * futex_parse_waitv - Parse a waitv array from userspace183 * futex_parse_waitv - Parse a waitv array from userspace
186 * @futexv: Kernel side list of waiters to be filled184 * @futexv: Kernel side list of waiters to be filled
187 * @uwaitv: Userspace list to be parsed185 * @uwaitv: Userspace list to be parsed
188 * @nr_futexes: Length of futexv186 * @nr_futexes: Length of futexv
187 * @wake: Wake to call when futex is woken
188 * @wake_data: Data for the wake handler
189 *189 *
190 * Return: Error code on failure, 0 on success190 * Return: Error code on failure, 0 on success
191 */191 */
192static int futex_parse_waitv(struct futex_vector *futexv,192int futex_parse_waitv(struct futex_vector *futexv,
193 struct futex_waitv __user *uwaitv,193 struct futex_waitv __user *uwaitv,
194 unsigned int nr_futexes)194 unsigned int nr_futexes, futex_wake_fn *wake,
195 void *wake_data)
195{196{
196 struct futex_waitv aux;197 struct futex_waitv aux;
197 unsigned int i;198 unsigned int i;
@@ -216,6 +217,8 @@ static int futex_parse_waitv(struct futex_vector *futexv, · 18 unmodified lines
216 futexv[i].w.val = aux.val;217 futexv[i].w.val = aux.val;
217 futexv[i].w.uaddr = aux.uaddr;218 futexv[i].w.uaddr = aux.uaddr;
218 futexv[i].q = futex_q_init;219 futexv[i].q = futex_q_init;
220 futexv[i].q.wake = wake;
221 futexv[i].q.wake_data = wake_data;
219 }222 }
220 223
221 return 0;224 return 0;
@@ -308,7 +311,8 @@ SYSCALL_DEFINE5(futex_waitv, struct futex_waitv __user *, waiters, · 86 unmodified lines
308 goto destroy_timer;311 goto destroy_timer;
309 }312 }
310 313
311 ret = futex_parse_waitv(futexv, waiters, nr_futexes);314 ret = futex_parse_waitv(futexv, waiters, nr_futexes, futex_wake_mark,
315 NULL);
312 if (!ret)316 if (!ret)
313 ret = futex_wait_multiple(futexv, nr_futexes, timeout ? &to : NULL);317 ret = futex_wait_multiple(futexv, nr_futexes, timeout ? &to : NULL);
314 318
@@ -423,7 +427,7 @@ SYSCALL_DEFINE4(futex_requeue, · 108 unmodified lines
423 if (!waiters)427 if (!waiters)
424 return -EINVAL;428 return -EINVAL;
425 429
426 ret = futex_parse_waitv(futexes, waiters, 2);430 ret = futex_parse_waitv(futexes, waiters, 2, futex_wake_mark, NULL);
427 if (ret)431 if (ret)
428 return ret;432 return ret;
429 433
kernel/futex/waitwake.c+30−19
@@ -106,20 +106,11 @@ · 105 unmodified lines
106 * double_lock_hb() and double_unlock_hb(), respectively.106 * double_lock_hb() and double_unlock_hb(), respectively.
107 */107 */
108 108
109/*109bool __futex_wake_mark(struct futex_q *q)
110 * The hash bucket lock must be held when this is called.
111 * Afterwards, the futex_q must not be accessed. Callers
112 * must ensure to later call wake_up_q() for the actual
113 * wakeups to occur.
114 */
115void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q)
116{110{
117 struct task_struct *p = q->task;
118
119 if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))111 if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))
120 return;112 return false;
121 113
122 get_task_struct(p);
123 __futex_unqueue(q);114 __futex_unqueue(q);
124 /*115 /*
125 * The waiting task can free the futex_q as soon as q->lock_ptr = NULL116 * The waiting task can free the futex_q as soon as q->lock_ptr = NULL
@@ -130,6 +121,26 @@ void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q) · 4 unmodified lines
130 */121 */
131 smp_store_release(&q->lock_ptr, NULL);122 smp_store_release(&q->lock_ptr, NULL);
132 123
124 return true;
125}
126
127/*
128 * The hash bucket lock must be held when this is called.
129 * Afterwards, the futex_q must not be accessed. Callers
130 * must ensure to later call wake_up_q() for the actual
131 * wakeups to occur.
132 */
133void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q)
134{
135 struct task_struct *p = q->task;
136
137 get_task_struct(p);
138
139 if (!__futex_wake_mark(q)) {
140 put_task_struct(p);
141 return;
142 }
143
133 /*144 /*
134 * Queue the task for later wakeup for after we've released145 * Queue the task for later wakeup for after we've released
135 * the hb->lock.146 * the hb->lock.
@@ -177,7 +188,7 @@ int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) · 41 unmodified lines
177 if (!(this->bitset & bitset))188 if (!(this->bitset & bitset))
178 continue;189 continue;
179 190
180 futex_wake_mark(&wake_q, this);191 this->wake(&wake_q, this);
181 if (++ret >= nr_wake)192 if (++ret >= nr_wake)
182 break;193 break;
183 }194 }
@@ -292,7 +303,7 @@ retry_private: · 108 unmodified lines
292 ret = -EINVAL;303 ret = -EINVAL;
293 goto out_unlock;304 goto out_unlock;
294 }305 }
295 futex_wake_mark(&wake_q, this);306 this->wake(&wake_q, this);
296 if (++ret >= nr_wake)307 if (++ret >= nr_wake)
297 break;308 break;
298 }309 }
@@ -306,7 +317,7 @@ retry_private: · 7 unmodified lines
306 ret = -EINVAL;317 ret = -EINVAL;
307 goto out_unlock;318 goto out_unlock;
308 }319 }
309 futex_wake_mark(&wake_q, this);320 this->wake(&wake_q, this);
310 if (++op_ret >= nr_wake2)321 if (++op_ret >= nr_wake2)
311 break;322 break;
312 }323 }
@@ -361,7 +372,7 @@ void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q, · 48 unmodified lines
361}372}
362 373
363/**374/**
364 * unqueue_multiple - Remove various futexes from their hash bucket375 * futex_unqueue_multiple - Remove various futexes from their hash bucket
365 * @v: The list of futexes to unqueue376 * @v: The list of futexes to unqueue
366 * @count: Number of futexes in the list377 * @count: Number of futexes in the list
367 *378 *
@@ -371,7 +382,7 @@ void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q, · 3 unmodified lines
371 * - >=0 - Index of the last futex that was awoken;382 * - >=0 - Index of the last futex that was awoken;
372 * - -1 - No futex was awoken383 * - -1 - No futex was awoken
373 */384 */
374static int unqueue_multiple(struct futex_vector *v, int count)385int futex_unqueue_multiple(struct futex_vector *v, int count)
375{386{
376 int ret = -1, i;387 int ret = -1, i;
377 388
@@ -399,7 +410,7 @@ static int unqueue_multiple(struct futex_vector *v, int count) · 21 unmodified lines
399 * - 0 - Success410 * - 0 - Success
400 * - <0 - -EFAULT, -EWOULDBLOCK or -EINVAL411 * - <0 - -EFAULT, -EWOULDBLOCK or -EINVAL
401 */412 */
402static int futex_wait_multiple_setup(struct futex_vector *vs, int count, int *woken)413int futex_wait_multiple_setup(struct futex_vector *vs, int count, int *woken)
403{414{
404 struct futex_hash_bucket *hb;415 struct futex_hash_bucket *hb;
405 bool retry = false;416 bool retry = false;
@@ -461,7 +472,7 @@ retry: · 55 unmodified lines
461 * was woken, we don't return error and return this index to472 * was woken, we don't return error and return this index to
462 * userspace473 * userspace
463 */474 */
464 *woken = unqueue_multiple(vs, i);475 *woken = futex_unqueue_multiple(vs, i);
465 if (*woken >= 0)476 if (*woken >= 0)
466 return 1;477 return 1;
467 478
@@ -546,7 +557,7 @@ int futex_wait_multiple(struct futex_vector *vs, unsigned int count, · 78 unmodified lines
546 557
547 __set_current_state(TASK_RUNNING);558 __set_current_state(TASK_RUNNING);
548 559
549 ret = unqueue_multiple(vs, count);560 ret = futex_unqueue_multiple(vs, count);
550 if (ret >= 0)561 if (ret >= 0)
551 return ret;562 return ret;
552 563