Merge tag 'io_uring-futex-2023-10-30' of git://git.kernel.dk/linux

Pull io_uring futex support from Jens Axboe: "This adds support for using futexes through io_uring - first futex wake and wait, and then the vectored variant of waiting, futex waitv. For both wait/wake/waitv, we support the bitset variant, as the 'normal' variants can be easily implemented on top of that. PI and requeue are not supported through io_uring, just the above mentioned parts. This may change in the future, but in the spirit of keeping this small (and based on what people have been asking for), this is what we currently have. Wake support is pretty straight forward, most of the thought has gone into the wait side to avoid needing to offload wait operations to a blocking context. Instead, we rely on the usual callbacks to retry and post a completion event, when appropriate. As far as I can recall, the first request for futex support with io_uring came from Andres Freund, working on postgres. His aio rework of postgres was one of the early adopters of io_uring, and futex support was a natural extension for that. This is relevant from both a usability point of view, as well as for effiency and performance. In Andres's words, for the former: Futex wait support in io_uring makes it a lot easier to avoid deadlocks in concurrent programs that have their own buffer pool: Obviously pages in the application buffer pool have to be locked during IO. If the initiator of IO A needs to wait for a held lock B, the holder of lock B might wait for the IO A to complete. The ability to wait for a lock and IO completions at the same time provides an efficient way to avoid such deadlocks and in terms of effiency, even without unlocking the full potential yet, Andres says: Futex wake support in io_uring is useful because it allows for more efficient directed wakeups. For some "locks" postgres has queues implemented in userspace, with wakeup logic that cannot easily be implemented with FUTEX_WAKE_BITSET on a single "futex word" (imagine waiting for journal flushes to have completed up to a certain point). Thus a "lock release" sometimes need to wake up many processes in a row. A quick-and-dirty conversion to doing these wakeups via io_uring lead to a 3% throughput increase, with 12% fewer context switches, albeit in a fairly extreme workload" * tag 'io_uring-futex-2023-10-30' of git://git.kernel.dk/linux: io_uring: add support for vectored futex waits futex: make the vectored futex operations available futex: make futex_parse_waitv() available as a helper futex: add wake_data to struct futex_q io_uring: add support for futex wake and wait futex: abstract out a __futex_wake_mark() helper futex: factor out the futex wake handling futex: move FUTEX2_VALID_MASK to futex.h

Linus Torvalds committed 3 years ago

Commit 4de520f2 parents f5277ad 8f3501913 files · +545 −27Browse files

Unified · Split

13 files

include/linux/io_uring_types.h +5 −0
include/uapi/linux/io_uring.h +4 −0
io_uring/Makefile +1 −0
io_uring/cancel.c +5 −0
io_uring/cancel.h +4 −0
io_uring/futex.c +386 −0
io_uring/futex.h +36 −0
io_uring/io_uring.c +7 −0
io_uring/opdef.c +34 −0
kernel/futex/futex.h +20 −0
kernel/futex/requeue.c +2 −1
kernel/futex/syscalls.c +11 −7
kernel/futex/waitwake.c +30 −19

include/linux/io_uring_types.h+5−0

@@ -321,6 +321,11 @@ struct io_ring_ctx { · 320 unmodified lines
321	321
322	322	struct hlist_head waitid_list;
323	323
	324	+#ifdef CONFIG_FUTEX
	325	+ struct hlist_head futex_list;
	326	+ struct io_alloc_cache futex_cache;
	327	+#endif
	328	+
324	329	const struct cred sq_creds; / cred used for __io_sq_thread() */
325	330	struct io_sq_data sq_data; / if using sq thread polling */
326	331

include/uapi/linux/io_uring.h+4−0

@@ -70,6 +70,7 @@ struct io_uring_sqe { · 69 unmodified lines
70	70	__u32 msg_ring_flags;
71	71	__u32 uring_cmd_flags;
72	72	__u32 waitid_flags;
	73	+ __u32 futex_flags;
73	74	};
74	75	__u64 user_data; /* data to be passed back at completion time */
75	76	/* pack this to avoid bogus arm OABI complaints */
@@ -249,6 +250,9 @@ enum io_uring_op { · 173 unmodified lines
249	250	IORING_OP_SENDMSG_ZC,
250	251	IORING_OP_READ_MULTISHOT,
251	252	IORING_OP_WAITID,
	253	+ IORING_OP_FUTEX_WAIT,
	254	+ IORING_OP_FUTEX_WAKE,
	255	+ IORING_OP_FUTEX_WAITV,
252	256
253	257	/* this goes last, obviously */
254	258	IORING_OP_LAST,

io_uring/Makefile+1−0

@@ -10,3 +10,4 @@ obj-$(CONFIG_IO_URING) += io_uring.o xattr.o nop.o fs.o splice.o \ · 9 unmodified lines
10	10	cancel.o kbuf.o rsrc.o rw.o opdef.o \
11	11	notif.o waitid.o
12	12	obj-$(CONFIG_IO_WQ) += io-wq.o
	13	+obj-$(CONFIG_FUTEX) += futex.o

io_uring/cancel.c+5−0

@@ -16,6 +16,7 @@ · 15 unmodified lines
16	16	#include "poll.h"
17	17	#include "timeout.h"
18	18	#include "waitid.h"
	19	+#include "futex.h"
19	20	#include "cancel.h"
20	21
21	22	struct io_cancel {
@@ -124,6 +125,10 @@ int io_try_cancel(struct io_uring_task tctx, struct io_cancel_data cd, · 102 unmodified lines
124	125	if (ret != -ENOENT)
125	126	return ret;
126	127
	128	+ ret = io_futex_cancel(ctx, cd, issue_flags);
	129	+ if (ret != -ENOENT)
	130	+ return ret;
	131	+
127	132	spin_lock(&ctx->completion_lock);
128	133	if (!(cd->flags & IORING_ASYNC_CANCEL_FD))
129	134	ret = io_timeout_cancel(ctx, cd);

io_uring/cancel.h+4−0

@@ -1,4 +1,6 @@
1	1	// SPDX-License-Identifier: GPL-2.0
	2	+#ifndef IORING_CANCEL_H
	3	+#define IORING_CANCEL_H
2	4
3	5	#include <linux/io_uring_types.h>
4	6
@@ -22,3 +24,5 @@ void init_hash_table(struct io_hash_table *table, unsigned size); · 17 unmodified lines
22	24
23	25	int io_sync_cancel(struct io_ring_ctx ctx, void __user arg);
24	26	bool io_cancel_req_match(struct io_kiocb req, struct io_cancel_data cd);
	27	+
	28	+#endif

io_uring/futex.c+386−0

@@ -0,0 +1,386 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+#include <linux/kernel.h>
	3	+#include <linux/errno.h>
	4	+#include <linux/fs.h>
	5	+#include <linux/file.h>
	6	+#include <linux/io_uring.h>
	7	+
	8	+#include <uapi/linux/io_uring.h>
	9	+
	10	+#include "../kernel/futex/futex.h"
	11	+#include "io_uring.h"
	12	+#include "rsrc.h"
	13	+#include "futex.h"
	14	+
	15	+struct io_futex {
	16	+ struct file *file;
	17	+ union {
	18	+ u32 __user *uaddr;
	19	+ struct futex_waitv __user *uwaitv;
	20	+ };
	21	+ unsigned long futex_val;
	22	+ unsigned long futex_mask;
	23	+ unsigned long futexv_owned;
	24	+ u32 futex_flags;
	25	+ unsigned int futex_nr;
	26	+ bool futexv_unqueued;
	27	+};
	28	+
	29	+struct io_futex_data {
	30	+ union {
	31	+ struct futex_q q;
	32	+ struct io_cache_entry cache;
	33	+ };
	34	+ struct io_kiocb *req;
	35	+};
	36	+
	37	+void io_futex_cache_init(struct io_ring_ctx *ctx)
	38	+{
	39	+ io_alloc_cache_init(&ctx->futex_cache, IO_NODE_ALLOC_CACHE_MAX,
	40	+ sizeof(struct io_futex_data));
	41	+}
	42	+
	43	+static void io_futex_cache_entry_free(struct io_cache_entry *entry)
	44	+{
	45	+ kfree(container_of(entry, struct io_futex_data, cache));
	46	+}
	47	+
	48	+void io_futex_cache_free(struct io_ring_ctx *ctx)
	49	+{
	50	+ io_alloc_cache_free(&ctx->futex_cache, io_futex_cache_entry_free);
	51	+}
	52	+
	53	+static void __io_futex_complete(struct io_kiocb req, struct io_tw_state ts)
	54	+{
	55	+ req->async_data = NULL;
	56	+ hlist_del_init(&req->hash_node);
	57	+ io_req_task_complete(req, ts);
	58	+}
	59	+
	60	+static void io_futex_complete(struct io_kiocb req, struct io_tw_state ts)
	61	+{
	62	+ struct io_futex_data *ifd = req->async_data;
	63	+ struct io_ring_ctx *ctx = req->ctx;
	64	+
	65	+ io_tw_lock(ctx, ts);
	66	+ if (!io_alloc_cache_put(&ctx->futex_cache, &ifd->cache))
	67	+ kfree(ifd);
	68	+ __io_futex_complete(req, ts);
	69	+}
	70	+
	71	+static void io_futexv_complete(struct io_kiocb req, struct io_tw_state ts)
	72	+{
	73	+ struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
	74	+ struct futex_vector *futexv = req->async_data;
	75	+
	76	+ io_tw_lock(req->ctx, ts);
	77	+
	78	+ if (!iof->futexv_unqueued) {
	79	+ int res;
	80	+
	81	+ res = futex_unqueue_multiple(futexv, iof->futex_nr);
	82	+ if (res != -1)
	83	+ io_req_set_res(req, res, 0);
	84	+ }
	85	+
	86	+ kfree(req->async_data);
	87	+ req->flags &= ~REQ_F_ASYNC_DATA;
	88	+ __io_futex_complete(req, ts);
	89	+}
	90	+
	91	+static bool io_futexv_claim(struct io_futex *iof)
	92	+{
	93	+ if (test_bit(0, &iof->futexv_owned) \|\|
	94	+ test_and_set_bit_lock(0, &iof->futexv_owned))
	95	+ return false;
	96	+ return true;
	97	+}
	98	+
	99	+static bool __io_futex_cancel(struct io_ring_ctx ctx, struct io_kiocb req)
	100	+{
	101	+ /* futex wake already done or in progress */
	102	+ if (req->opcode == IORING_OP_FUTEX_WAIT) {
	103	+ struct io_futex_data *ifd = req->async_data;
	104	+
	105	+ if (!futex_unqueue(&ifd->q))
	106	+ return false;
	107	+ req->io_task_work.func = io_futex_complete;
	108	+ } else {
	109	+ struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
	110	+
	111	+ if (!io_futexv_claim(iof))
	112	+ return false;
	113	+ req->io_task_work.func = io_futexv_complete;
	114	+ }
	115	+
	116	+ hlist_del_init(&req->hash_node);
	117	+ io_req_set_res(req, -ECANCELED, 0);
	118	+ io_req_task_work_add(req);
	119	+ return true;
	120	+}
	121	+
	122	+int io_futex_cancel(struct io_ring_ctx ctx, struct io_cancel_data cd,
	123	+ unsigned int issue_flags)
	124	+{
	125	+ struct hlist_node *tmp;
	126	+ struct io_kiocb *req;
	127	+ int nr = 0;
	128	+
	129	+ if (cd->flags & (IORING_ASYNC_CANCEL_FD\|IORING_ASYNC_CANCEL_FD_FIXED))
	130	+ return -ENOENT;
	131	+
	132	+ io_ring_submit_lock(ctx, issue_flags);
	133	+ hlist_for_each_entry_safe(req, tmp, &ctx->futex_list, hash_node) {
	134	+ if (req->cqe.user_data != cd->data &&
	135	+ !(cd->flags & IORING_ASYNC_CANCEL_ANY))
	136	+ continue;
	137	+ if (__io_futex_cancel(ctx, req))
	138	+ nr++;
	139	+ if (!(cd->flags & IORING_ASYNC_CANCEL_ALL))
	140	+ break;
	141	+ }
	142	+ io_ring_submit_unlock(ctx, issue_flags);
	143	+
	144	+ if (nr)
	145	+ return nr;
	146	+
	147	+ return -ENOENT;
	148	+}
	149	+
	150	+bool io_futex_remove_all(struct io_ring_ctx ctx, struct task_struct task,
	151	+ bool cancel_all)
	152	+{
	153	+ struct hlist_node *tmp;
	154	+ struct io_kiocb *req;
	155	+ bool found = false;
	156	+
	157	+ lockdep_assert_held(&ctx->uring_lock);
	158	+
	159	+ hlist_for_each_entry_safe(req, tmp, &ctx->futex_list, hash_node) {
	160	+ if (!io_match_task_safe(req, task, cancel_all))
	161	+ continue;
	162	+ __io_futex_cancel(ctx, req);
	163	+ found = true;
	164	+ }
	165	+
	166	+ return found;
	167	+}
	168	+
	169	+int io_futex_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
	170	+{
	171	+ struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
	172	+ u32 flags;
	173	+
	174	+ if (unlikely(sqe->len \|\| sqe->futex_flags \|\| sqe->buf_index \|\|
	175	+ sqe->file_index))
	176	+ return -EINVAL;
	177	+
	178	+ iof->uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr));
	179	+ iof->futex_val = READ_ONCE(sqe->addr2);
	180	+ iof->futex_mask = READ_ONCE(sqe->addr3);
	181	+ flags = READ_ONCE(sqe->fd);
	182	+
	183	+ if (flags & ~FUTEX2_VALID_MASK)
	184	+ return -EINVAL;
	185	+
	186	+ iof->futex_flags = futex2_to_flags(flags);
	187	+ if (!futex_flags_valid(iof->futex_flags))
	188	+ return -EINVAL;
	189	+
	190	+ if (!futex_validate_input(iof->futex_flags, iof->futex_val) \|\|
	191	+ !futex_validate_input(iof->futex_flags, iof->futex_mask))
	192	+ return -EINVAL;
	193	+
	194	+ return 0;
	195	+}
	196	+
	197	+static void io_futex_wakev_fn(struct wake_q_head wake_q, struct futex_q q)
	198	+{
	199	+ struct io_kiocb *req = q->wake_data;
	200	+ struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
	201	+
	202	+ if (!io_futexv_claim(iof))
	203	+ return;
	204	+ if (unlikely(!__futex_wake_mark(q)))
	205	+ return;
	206	+
	207	+ io_req_set_res(req, 0, 0);
	208	+ req->io_task_work.func = io_futexv_complete;
	209	+ io_req_task_work_add(req);
	210	+}
	211	+
	212	+int io_futexv_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
	213	+{
	214	+ struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
	215	+ struct futex_vector *futexv;
	216	+ int ret;
	217	+
	218	+ /* No flags or mask supported for waitv */
	219	+ if (unlikely(sqe->fd \|\| sqe->buf_index \|\| sqe->file_index \|\|
	220	+ sqe->addr2 \|\| sqe->futex_flags \|\| sqe->addr3))
	221	+ return -EINVAL;
	222	+
	223	+ iof->uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr));
	224	+ iof->futex_nr = READ_ONCE(sqe->len);
	225	+ if (!iof->futex_nr \|\| iof->futex_nr > FUTEX_WAITV_MAX)
	226	+ return -EINVAL;
	227	+
	228	+ futexv = kcalloc(iof->futex_nr, sizeof(*futexv), GFP_KERNEL);
	229	+ if (!futexv)
	230	+ return -ENOMEM;
	231	+
	232	+ ret = futex_parse_waitv(futexv, iof->uwaitv, iof->futex_nr,
	233	+ io_futex_wakev_fn, req);
	234	+ if (ret) {
	235	+ kfree(futexv);
	236	+ return ret;
	237	+ }
	238	+
	239	+ iof->futexv_owned = 0;
	240	+ iof->futexv_unqueued = 0;
	241	+ req->flags \|= REQ_F_ASYNC_DATA;
	242	+ req->async_data = futexv;
	243	+ return 0;
	244	+}
	245	+
	246	+static void io_futex_wake_fn(struct wake_q_head wake_q, struct futex_q q)
	247	+{
	248	+ struct io_futex_data *ifd = container_of(q, struct io_futex_data, q);
	249	+ struct io_kiocb *req = ifd->req;
	250	+
	251	+ if (unlikely(!__futex_wake_mark(q)))
	252	+ return;
	253	+
	254	+ io_req_set_res(req, 0, 0);
	255	+ req->io_task_work.func = io_futex_complete;
	256	+ io_req_task_work_add(req);
	257	+}
	258	+
	259	+static struct io_futex_data io_alloc_ifd(struct io_ring_ctx ctx)
	260	+{
	261	+ struct io_cache_entry *entry;
	262	+
	263	+ entry = io_alloc_cache_get(&ctx->futex_cache);
	264	+ if (entry)
	265	+ return container_of(entry, struct io_futex_data, cache);
	266	+
	267	+ return kmalloc(sizeof(struct io_futex_data), GFP_NOWAIT);
	268	+}
	269	+
	270	+int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags)
	271	+{
	272	+ struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
	273	+ struct futex_vector *futexv = req->async_data;
	274	+ struct io_ring_ctx *ctx = req->ctx;
	275	+ int ret, woken = -1;
	276	+
	277	+ io_ring_submit_lock(ctx, issue_flags);
	278	+
	279	+ ret = futex_wait_multiple_setup(futexv, iof->futex_nr, &woken);
	280	+
	281	+ /*
	282	+ * Error case, ret is < 0. Mark the request as failed.
	283	+ */
	284	+ if (unlikely(ret < 0)) {
	285	+ io_ring_submit_unlock(ctx, issue_flags);
	286	+ req_set_fail(req);
	287	+ io_req_set_res(req, ret, 0);
	288	+ kfree(futexv);
	289	+ req->async_data = NULL;
	290	+ req->flags &= ~REQ_F_ASYNC_DATA;
	291	+ return IOU_OK;
	292	+ }
	293	+
	294	+ /*
	295	+ * 0 return means that we successfully setup the waiters, and that
	296	+ * nobody triggered a wakeup while we were doing so. If the wakeup
	297	+ * happened post setup, the task_work will be run post this issue and
	298	+ * under the submission lock. 1 means We got woken while setting up,
	299	+ * let that side do the completion. Note that
	300	+ * futex_wait_multiple_setup() will have unqueued all the futexes in
	301	+ * this case. Mark us as having done that already, since this is
	302	+ * different from normal wakeup.
	303	+ */
	304	+ if (!ret) {
	305	+ /*
	306	+ * If futex_wait_multiple_setup() returns 0 for a
	307	+ * successful setup, then the task state will not be
	308	+ * runnable. This is fine for the sync syscall, as
	309	+ * it'll be blocking unless we already got one of the
	310	+ * futexes woken, but it obviously won't work for an
	311	+ * async invocation. Mark us runnable again.
	312	+ */
	313	+ __set_current_state(TASK_RUNNING);
	314	+ hlist_add_head(&req->hash_node, &ctx->futex_list);
	315	+ } else {
	316	+ iof->futexv_unqueued = 1;
	317	+ if (woken != -1)
	318	+ io_req_set_res(req, woken, 0);
	319	+ }
	320	+
	321	+ io_ring_submit_unlock(ctx, issue_flags);
	322	+ return IOU_ISSUE_SKIP_COMPLETE;
	323	+}
	324	+
	325	+int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags)
	326	+{
	327	+ struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
	328	+ struct io_ring_ctx *ctx = req->ctx;
	329	+ struct io_futex_data *ifd = NULL;
	330	+ struct futex_hash_bucket *hb;
	331	+ int ret;
	332	+
	333	+ if (!iof->futex_mask) {
	334	+ ret = -EINVAL;
	335	+ goto done;
	336	+ }
	337	+
	338	+ io_ring_submit_lock(ctx, issue_flags);
	339	+ ifd = io_alloc_ifd(ctx);
	340	+ if (!ifd) {
	341	+ ret = -ENOMEM;
	342	+ goto done_unlock;
	343	+ }
	344	+
	345	+ req->async_data = ifd;
	346	+ ifd->q = futex_q_init;
	347	+ ifd->q.bitset = iof->futex_mask;
	348	+ ifd->q.wake = io_futex_wake_fn;
	349	+ ifd->req = req;
	350	+
	351	+ ret = futex_wait_setup(iof->uaddr, iof->futex_val, iof->futex_flags,
	352	+ &ifd->q, &hb);
	353	+ if (!ret) {
	354	+ hlist_add_head(&req->hash_node, &ctx->futex_list);
	355	+ io_ring_submit_unlock(ctx, issue_flags);
	356	+
	357	+ futex_queue(&ifd->q, hb);
	358	+ return IOU_ISSUE_SKIP_COMPLETE;
	359	+ }
	360	+
	361	+done_unlock:
	362	+ io_ring_submit_unlock(ctx, issue_flags);
	363	+done:
	364	+ if (ret < 0)
	365	+ req_set_fail(req);
	366	+ io_req_set_res(req, ret, 0);
	367	+ kfree(ifd);
	368	+ return IOU_OK;
	369	+}
	370	+
	371	+int io_futex_wake(struct io_kiocb *req, unsigned int issue_flags)
	372	+{
	373	+ struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
	374	+ int ret;
	375	+
	376	+ /*
	377	+ * Strict flags - ensure that waking 0 futexes yields a 0 result.
	378	+ * See commit 43adf8449510 ("futex: FLAGS_STRICT") for details.
	379	+ */
	380	+ ret = futex_wake(iof->uaddr, FLAGS_STRICT \| iof->futex_flags,
	381	+ iof->futex_val, iof->futex_mask);
	382	+ if (ret < 0)
	383	+ req_set_fail(req);
	384	+ io_req_set_res(req, ret, 0);
	385	+ return IOU_OK;
	386	+}

io_uring/futex.h+36−0

@@ -0,0 +1,36 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+#include "cancel.h"
	4	+
	5	+int io_futex_prep(struct io_kiocb req, const struct io_uring_sqe sqe);
	6	+int io_futexv_prep(struct io_kiocb req, const struct io_uring_sqe sqe);
	7	+int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags);
	8	+int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags);
	9	+int io_futex_wake(struct io_kiocb *req, unsigned int issue_flags);
	10	+
	11	+#if defined(CONFIG_FUTEX)
	12	+int io_futex_cancel(struct io_ring_ctx ctx, struct io_cancel_data cd,
	13	+ unsigned int issue_flags);
	14	+bool io_futex_remove_all(struct io_ring_ctx ctx, struct task_struct task,
	15	+ bool cancel_all);
	16	+void io_futex_cache_init(struct io_ring_ctx *ctx);
	17	+void io_futex_cache_free(struct io_ring_ctx *ctx);
	18	+#else
	19	+static inline int io_futex_cancel(struct io_ring_ctx *ctx,
	20	+ struct io_cancel_data *cd,
	21	+ unsigned int issue_flags)
	22	+{
	23	+ return 0;
	24	+}
	25	+static inline bool io_futex_remove_all(struct io_ring_ctx *ctx,
	26	+ struct task_struct *task, bool cancel_all)
	27	+{
	28	+ return false;
	29	+}
	30	+static inline void io_futex_cache_init(struct io_ring_ctx *ctx)
	31	+{
	32	+}
	33	+static inline void io_futex_cache_free(struct io_ring_ctx *ctx)
	34	+{
	35	+}
	36	+#endif

io_uring/io_uring.c+7−0

@@ -93,6 +93,7 @@ · 92 unmodified lines
93	93	#include "net.h"
94	94	#include "notif.h"
95	95	#include "waitid.h"
	96	+#include "futex.h"
96	97
97	98	#include "timeout.h"
98	99	#include "poll.h"
@@ -330,6 +331,7 @@ static __cold struct io_ring_ctx io_ring_ctx_alloc(struct io_uring_params p) · 231 unmodified lines
330	331	sizeof(struct async_poll));
331	332	io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,
332	333	sizeof(struct io_async_msghdr));
	334	+ io_futex_cache_init(ctx);
333	335	init_completion(&ctx->ref_comp);
334	336	xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
335	337	mutex_init(&ctx->uring_lock);
@@ -349,6 +351,9 @@ static __cold struct io_ring_ctx io_ring_ctx_alloc(struct io_uring_params p) · 13 unmodified lines
349	351	ctx->submit_state.free_list.next = NULL;
350	352	INIT_WQ_LIST(&ctx->locked_free_list);
351	353	INIT_HLIST_HEAD(&ctx->waitid_list);
	354	+#ifdef CONFIG_FUTEX
	355	+ INIT_HLIST_HEAD(&ctx->futex_list);
	356	+#endif
352	357	INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);
353	358	INIT_WQ_LIST(&ctx->submit_state.compl_reqs);
354	359	INIT_HLIST_HEAD(&ctx->cancelable_uring_cmd);
@@ -2914,6 +2919,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) · 2559 unmodified lines
2914	2919	io_eventfd_unregister(ctx);
2915	2920	io_alloc_cache_free(&ctx->apoll_cache, io_apoll_cache_free);
2916	2921	io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free);
	2922	+ io_futex_cache_free(ctx);
2917	2923	io_destroy_buffers(ctx);
2918	2924	mutex_unlock(&ctx->uring_lock);
2919	2925	if (ctx->sq_creds)
@@ -3357,6 +3363,7 @@ static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, · 437 unmodified lines
3357	3363	mutex_lock(&ctx->uring_lock);
3358	3364	ret \|= io_poll_remove_all(ctx, task, cancel_all);
3359	3365	ret \|= io_waitid_remove_all(ctx, task, cancel_all);
	3366	+ ret \|= io_futex_remove_all(ctx, task, cancel_all);
3360	3367	ret \|= io_uring_try_cancel_uring_cmd(ctx, task, cancel_all);
3361	3368	mutex_unlock(&ctx->uring_lock);
3362	3369	ret \|= io_kill_timeouts(ctx, task, cancel_all);

io_uring/opdef.c+34−0

@@ -34,6 +34,7 @@ · 33 unmodified lines
34	34	#include "cancel.h"
35	35	#include "rw.h"
36	36	#include "waitid.h"
	37	+#include "futex.h"
37	38
38	39	static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
39	40	{
@@ -444,6 +445,30 @@ const struct io_issue_def io_issue_defs[] = { · 404 unmodified lines
444	445	.prep = io_waitid_prep,
445	446	.issue = io_waitid,
446	447	},
	448	+ [IORING_OP_FUTEX_WAIT] = {
	449	+#if defined(CONFIG_FUTEX)
	450	+ .prep = io_futex_prep,
	451	+ .issue = io_futex_wait,
	452	+#else
	453	+ .prep = io_eopnotsupp_prep,
	454	+#endif
	455	+ },
	456	+ [IORING_OP_FUTEX_WAKE] = {
	457	+#if defined(CONFIG_FUTEX)
	458	+ .prep = io_futex_prep,
	459	+ .issue = io_futex_wake,
	460	+#else
	461	+ .prep = io_eopnotsupp_prep,
	462	+#endif
	463	+ },
	464	+ [IORING_OP_FUTEX_WAITV] = {
	465	+#if defined(CONFIG_FUTEX)
	466	+ .prep = io_futexv_prep,
	467	+ .issue = io_futexv_wait,
	468	+#else
	469	+ .prep = io_eopnotsupp_prep,
	470	+#endif
	471	+ },
447	472	};
448	473
449	474	const struct io_cold_def io_cold_defs[] = {
@@ -670,6 +695,15 @@ const struct io_cold_def io_cold_defs[] = { · 220 unmodified lines
670	695	.name = "WAITID",
671	696	.async_size = sizeof(struct io_waitid_async),
672	697	},
	698	+ [IORING_OP_FUTEX_WAIT] = {
	699	+ .name = "FUTEX_WAIT",
	700	+ },
	701	+ [IORING_OP_FUTEX_WAKE] = {
	702	+ .name = "FUTEX_WAKE",
	703	+ },
	704	+ [IORING_OP_FUTEX_WAITV] = {
	705	+ .name = "FUTEX_WAITV",
	706	+ },
673	707	};
674	708
675	709	const char *io_uring_get_opcode(u8 opcode)

kernel/futex/futex.h+20−0

@@ -52,6 +52,8 @@ static inline unsigned int futex_to_flags(unsigned int op) · 51 unmodified lines
52	52	return flags;
53	53	}
54	54
	55	+#define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK \| FUTEX2_PRIVATE)
	56	+
55	57	/* FUTEX2_ to FLAGS_ */
56	58	static inline unsigned int futex2_to_flags(unsigned int flags2)
57	59	{
@@ -137,11 +139,16 @@ struct futex_pi_state { · 79 unmodified lines
137	139	union futex_key key;
138	140	} __randomize_layout;
139	141
	142	+struct futex_q;
	143	+typedef void (futex_wake_fn)(struct wake_q_head wake_q, struct futex_q q);
	144	+
140	145	/**
141	146	* struct futex_q - The hashed futex queue entry, one per waiting task
142	147	* @list: priority-sorted list of tasks waiting on this futex
143	148	* @task: the task waiting on the futex
144	149	* @lock_ptr: the hash bucket lock
	150	+ * @wake: the wake handler for this queue
	151	+ * @wake_data: data associated with the wake handler
145	152	* @key: the key the futex is hashed on
146	153	* @pi_state: optional priority inheritance state
147	154	* @rt_waiter: rt_waiter storage for use with requeue_pi
@@ -166,6 +173,8 @@ struct futex_q { · 18 unmodified lines
166	173
167	174	struct task_struct *task;
168	175	spinlock_t *lock_ptr;
	176	+ futex_wake_fn *wake;
	177	+ void *wake_data;
169	178	union futex_key key;
170	179	struct futex_pi_state *pi_state;
171	180	struct rt_mutex_waiter *rt_waiter;
@@ -212,6 +221,7 @@ extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, · 40 unmodified lines
212	221	struct futex_q q, struct futex_hash_bucket *hb);
213	222	extern void futex_wait_queue(struct futex_hash_bucket hb, struct futex_q q,
214	223	struct hrtimer_sleeper *timeout);
	224	+extern bool __futex_wake_mark(struct futex_q *q);
215	225	extern void futex_wake_mark(struct wake_q_head wake_q, struct futex_q q);
216	226
217	227	extern int fault_in_user_writeable(u32 __user *uaddr);
@@ -351,6 +361,16 @@ struct futex_vector { · 133 unmodified lines
351	361	struct futex_q q;
352	362	};
353	363
	364	+extern int futex_parse_waitv(struct futex_vector *futexv,
	365	+ struct futex_waitv __user *uwaitv,
	366	+ unsigned int nr_futexes, futex_wake_fn *wake,
	367	+ void *wake_data);
	368	+
	369	+extern int futex_wait_multiple_setup(struct futex_vector *vs, int count,
	370	+ int *woken);
	371	+
	372	+extern int futex_unqueue_multiple(struct futex_vector *v, int count);
	373	+
354	374	extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count,
355	375	struct hrtimer_sleeper *to);
356	376

kernel/futex/requeue.c+2−1

@@ -58,6 +58,7 @@ enum { · 57 unmodified lines
58	58
59	59	const struct futex_q futex_q_init = {
60	60	/* list gets initialized in futex_queue()*/
	61	+ .wake = futex_wake_mark,
61	62	.key = FUTEX_KEY_INIT,
62	63	.bitset = FUTEX_BITSET_MATCH_ANY,
63	64	.requeue_state = ATOMIC_INIT(Q_REQUEUE_PI_NONE),
@@ -593,7 +594,7 @@ retry_private: · 529 unmodified lines
593	594	/* Plain futexes just wake or requeue and are done */
594	595	if (!requeue_pi) {
595	596	if (++task_count <= nr_wake)
596		− futex_wake_mark(&wake_q, this);
	597	+ this->wake(&wake_q, this);
597	598	else
598	599	requeue_futex(this, hb1, hb2, &key2);
599	600	continue;

kernel/futex/syscalls.c+11−7

@@ -179,19 +179,20 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val, · 178 unmodified lines
179	179	return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3);
180	180	}
181	181
182		−#define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK \| FUTEX2_PRIVATE)
183		−
184	182	/**
185	183	* futex_parse_waitv - Parse a waitv array from userspace
186	184	* @futexv: Kernel side list of waiters to be filled
187	185	* @uwaitv: Userspace list to be parsed
188	186	* @nr_futexes: Length of futexv
	187	+ * @wake: Wake to call when futex is woken
	188	+ * @wake_data: Data for the wake handler
189	189	*
190	190	* Return: Error code on failure, 0 on success
191	191	*/
192		−static int futex_parse_waitv(struct futex_vector *futexv,
193		− struct futex_waitv __user *uwaitv,
194		− unsigned int nr_futexes)
	192	+int futex_parse_waitv(struct futex_vector *futexv,
	193	+ struct futex_waitv __user *uwaitv,
	194	+ unsigned int nr_futexes, futex_wake_fn *wake,
	195	+ void *wake_data)
195	196	{
196	197	struct futex_waitv aux;
197	198	unsigned int i;
@@ -216,6 +217,8 @@ static int futex_parse_waitv(struct futex_vector *futexv, · 18 unmodified lines
216	217	futexv[i].w.val = aux.val;
217	218	futexv[i].w.uaddr = aux.uaddr;
218	219	futexv[i].q = futex_q_init;
	220	+ futexv[i].q.wake = wake;
	221	+ futexv[i].q.wake_data = wake_data;
219	222	}
220	223
221	224	return 0;
@@ -308,7 +311,8 @@ SYSCALL_DEFINE5(futex_waitv, struct futex_waitv __user *, waiters, · 86 unmodified lines
308	311	goto destroy_timer;
309	312	}
310	313
311		− ret = futex_parse_waitv(futexv, waiters, nr_futexes);
	314	+ ret = futex_parse_waitv(futexv, waiters, nr_futexes, futex_wake_mark,
	315	+ NULL);
312	316	if (!ret)
313	317	ret = futex_wait_multiple(futexv, nr_futexes, timeout ? &to : NULL);
314	318
@@ -423,7 +427,7 @@ SYSCALL_DEFINE4(futex_requeue, · 108 unmodified lines
423	427	if (!waiters)
424	428	return -EINVAL;
425	429
426		− ret = futex_parse_waitv(futexes, waiters, 2);
	430	+ ret = futex_parse_waitv(futexes, waiters, 2, futex_wake_mark, NULL);
427	431	if (ret)
428	432	return ret;
429	433

kernel/futex/waitwake.c+30−19

@@ -106,20 +106,11 @@ · 105 unmodified lines
106	106	* double_lock_hb() and double_unlock_hb(), respectively.
107	107	*/
108	108
109		−/*
110		− * The hash bucket lock must be held when this is called.
111		− * Afterwards, the futex_q must not be accessed. Callers
112		− * must ensure to later call wake_up_q() for the actual
113		− * wakeups to occur.
114		− */
115		−void futex_wake_mark(struct wake_q_head wake_q, struct futex_q q)
	109	+bool __futex_wake_mark(struct futex_q *q)
116	110	{
117		− struct task_struct *p = q->task;
118		−
119	111	if (WARN(q->pi_state \|\| q->rt_waiter, "refusing to wake PI futex\n"))
120		− return;
	112	+ return false;
121	113
122		− get_task_struct(p);
123	114	__futex_unqueue(q);
124	115	/*
125	116	* The waiting task can free the futex_q as soon as q->lock_ptr = NULL
@@ -130,6 +121,26 @@ void futex_wake_mark(struct wake_q_head wake_q, struct futex_q q) · 4 unmodified lines
130	121	*/
131	122	smp_store_release(&q->lock_ptr, NULL);
132	123
	124	+ return true;
	125	+}
	126	+
	127	+/*
	128	+ * The hash bucket lock must be held when this is called.
	129	+ * Afterwards, the futex_q must not be accessed. Callers
	130	+ * must ensure to later call wake_up_q() for the actual
	131	+ * wakeups to occur.
	132	+ */
	133	+void futex_wake_mark(struct wake_q_head wake_q, struct futex_q q)
	134	+{
	135	+ struct task_struct *p = q->task;
	136	+
	137	+ get_task_struct(p);
	138	+
	139	+ if (!__futex_wake_mark(q)) {
	140	+ put_task_struct(p);
	141	+ return;
	142	+ }
	143	+
133	144	/*
134	145	* Queue the task for later wakeup for after we've released
135	146	* the hb->lock.
@@ -177,7 +188,7 @@ int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) · 41 unmodified lines
177	188	if (!(this->bitset & bitset))
178	189	continue;
179	190
180		− futex_wake_mark(&wake_q, this);
	191	+ this->wake(&wake_q, this);
181	192	if (++ret >= nr_wake)
182	193	break;
183	194	}
@@ -292,7 +303,7 @@ retry_private: · 108 unmodified lines
292	303	ret = -EINVAL;
293	304	goto out_unlock;
294	305	}
295		− futex_wake_mark(&wake_q, this);
	306	+ this->wake(&wake_q, this);
296	307	if (++ret >= nr_wake)
297	308	break;
298	309	}
@@ -306,7 +317,7 @@ retry_private: · 7 unmodified lines
306	317	ret = -EINVAL;
307	318	goto out_unlock;
308	319	}
309		− futex_wake_mark(&wake_q, this);
	320	+ this->wake(&wake_q, this);
310	321	if (++op_ret >= nr_wake2)
311	322	break;
312	323	}
@@ -361,7 +372,7 @@ void futex_wait_queue(struct futex_hash_bucket hb, struct futex_q q, · 48 unmodified lines
361	372	}
362	373
363	374	/**
364		− * unqueue_multiple - Remove various futexes from their hash bucket
	375	+ * futex_unqueue_multiple - Remove various futexes from their hash bucket
365	376	* @v: The list of futexes to unqueue
366	377	* @count: Number of futexes in the list
367	378	*
@@ -371,7 +382,7 @@ void futex_wait_queue(struct futex_hash_bucket hb, struct futex_q q, · 3 unmodified lines
371	382	* - >=0 - Index of the last futex that was awoken;
372	383	* - -1 - No futex was awoken
373	384	*/
374		−static int unqueue_multiple(struct futex_vector *v, int count)
	385	+int futex_unqueue_multiple(struct futex_vector *v, int count)
375	386	{
376	387	int ret = -1, i;
377	388
@@ -399,7 +410,7 @@ static int unqueue_multiple(struct futex_vector *v, int count) · 21 unmodified lines
399	410	* - 0 - Success
400	411	* - <0 - -EFAULT, -EWOULDBLOCK or -EINVAL
401	412	*/
402		−static int futex_wait_multiple_setup(struct futex_vector vs, int count, int woken)
	413	+int futex_wait_multiple_setup(struct futex_vector vs, int count, int woken)
403	414	{
404	415	struct futex_hash_bucket *hb;
405	416	bool retry = false;
@@ -461,7 +472,7 @@ retry: · 55 unmodified lines
461	472	* was woken, we don't return error and return this index to
462	473	* userspace
463	474	*/
464		− *woken = unqueue_multiple(vs, i);
	475	+ *woken = futex_unqueue_multiple(vs, i);
465	476	if (*woken >= 0)
466	477	return 1;
467	478
@@ -546,7 +557,7 @@ int futex_wait_multiple(struct futex_vector *vs, unsigned int count, · 78 unmodified lines
546	557
547	558	__set_current_state(TASK_RUNNING);
548	559
549		− ret = unqueue_multiple(vs, count);
	560	+ ret = futex_unqueue_multiple(vs, count);
550	561	if (ret >= 0)
551	562	return ret;
552	563