Merge tag 'for-6.7/io_uring-2023-10-30' of git://git.kernel.dk/linux

Pull io_uring updates from Jens Axboe: "This contains the core io_uring updates, of which there are not many, and adds support for using WAITID through io_uring and hence not needing to block on these kinds of events. Outside of that, tweaks to the legacy provided buffer handling and some cleanups related to cancelations for uring_cmd support" * tag 'for-6.7/io_uring-2023-10-30' of git://git.kernel.dk/linux: io_uring/poll: use IOU_F_TWQ_LAZY_WAKE for wakeups io_uring/kbuf: Use slab for struct io_buffer objects io_uring/kbuf: Allow the full buffer id space for provided buffers io_uring/kbuf: Fix check of BID wrapping in provided buffers io_uring/rsrc: cleanup io_pin_pages() io_uring: cancelable uring_cmd io_uring: retain top 8bits of uring_cmd flags for kernel internal use io_uring: add IORING_OP_WAITID support exit: add internal include file with helpers exit: add kernel_waitid_prepare() helper exit: move core of do_wait() into helper exit: abstract out should_wake helper for child_wait_callback() io_uring/rw: add support for IORING_OP_READ_MULTISHOT io_uring/rw: mark readv/writev as vectored in the opcode definition io_uring/rw: split io_read() into a helper

Linus Torvalds committed 3 years ago

Commit ffa059b2 parents ca995ce 6ce4a9319 files · +782 −120Browse files

Unified · Split

19 files

include/linux/io_uring.h +18 −0
include/linux/io_uring_types.h +8 −2
include/uapi/linux/io_uring.h +5 −3
io_uring/Makefile +2 −1
io_uring/cancel.c +5 −0
io_uring/io_uring.c +42 −1
io_uring/io_uring.h +1 −0
io_uring/kbuf.c +33 −25
io_uring/opdef.c +23 −1
io_uring/opdef.h +2 −0
io_uring/poll.c +1 −1
io_uring/rsrc.c +17 −20
io_uring/rw.c +86 −6
io_uring/rw.h +2 −0
io_uring/uring_cmd.c +48 −1
io_uring/waitid.c +372 −0
io_uring/waitid.h +15 −0
kernel/exit.c +72 −59
kernel/exit.h +30 −0

include/linux/io_uring.h+18−0

@@ -20,8 +20,15 @@ enum io_uring_cmd_flags { · 19 unmodified lines
20	20	IO_URING_F_SQE128 = (1 << 8),
21	21	IO_URING_F_CQE32 = (1 << 9),
22	22	IO_URING_F_IOPOLL = (1 << 10),
	23	+
	24	+ /* set when uring wants to cancel a previously issued command */
	25	+ IO_URING_F_CANCEL = (1 << 11),
23	26	};
24	27
	28	+/* only top 8 bits of sqe->uring_cmd_flags for kernel internal use */
	29	+#define IORING_URING_CMD_CANCELABLE (1U << 30)
	30	+#define IORING_URING_CMD_POLLED (1U << 31)
	31	+
25	32	struct io_uring_cmd {
26	33	struct file *file;
27	34	const struct io_uring_sqe *sqe;
@@ -82,6 +89,9 @@ static inline void io_uring_free(struct task_struct *tsk) · 54 unmodified lines
82	89	__io_uring_free(tsk);
83	90	}
84	91	int io_uring_cmd_sock(struct io_uring_cmd *cmd, unsigned int issue_flags);
	92	+void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
	93	+ unsigned int issue_flags);
	94	+struct task_struct io_uring_cmd_get_task(struct io_uring_cmd cmd);
85	95	#else
86	96	static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
87	97	struct iov_iter iter, void ioucmd)
@@ -122,6 +132,14 @@ static inline int io_uring_cmd_sock(struct io_uring_cmd *cmd, · 34 unmodified lines
122	132	{
123	133	return -EOPNOTSUPP;
124	134	}
	135	+static inline void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
	136	+ unsigned int issue_flags)
	137	+{
	138	+}
	139	+static inline struct task_struct io_uring_cmd_get_task(struct io_uring_cmd cmd)
	140	+{
	141	+ return NULL;
	142	+}
125	143	#endif
126	144
127	145	#endif

include/linux/io_uring_types.h+8−2

@@ -265,6 +265,12 @@ struct io_ring_ctx { · 264 unmodified lines
265	265	*/
266	266	struct io_wq_work_list iopoll_list;
267	267	bool poll_multi_queue;
	268	+
	269	+ /*
	270	+ * Any cancelable uring_cmd is added to this list in
	271	+ * ->uring_cmd() by io_uring_cmd_insert_cancelable()
	272	+ */
	273	+ struct hlist_head cancelable_uring_cmd;
268	274	} ____cacheline_aligned_in_smp;
269	275
270	276	struct {
@@ -313,6 +319,8 @@ struct io_ring_ctx { · 42 unmodified lines
313	319	struct list_head cq_overflow_list;
314	320	struct io_hash_table cancel_table;
315	321
	322	+ struct hlist_head waitid_list;
	323	+
316	324	const struct cred sq_creds; / cred used for __io_sq_thread() */
317	325	struct io_sq_data sq_data; / if using sq thread polling */
318	326
@@ -342,8 +350,6 @@ struct io_ring_ctx { · 23 unmodified lines
342	350	struct wait_queue_head rsrc_quiesce_wq;
343	351	unsigned rsrc_quiesce;
344	352
345		− struct list_head io_buffers_pages;
346		−
347	353	#if defined(CONFIG_UNIX)
348	354	struct socket *ring_sock;
349	355	#endif

include/uapi/linux/io_uring.h+5−3

@@ -65,6 +65,7 @@ struct io_uring_sqe { · 64 unmodified lines
65	65	__u32 xattr_flags;
66	66	__u32 msg_ring_flags;
67	67	__u32 uring_cmd_flags;
	68	+ __u32 waitid_flags;
68	69	};
69	70	__u64 user_data; /* data to be passed back at completion time */
70	71	/* pack this to avoid bogus arm OABI complaints */
@@ -240,19 +241,20 @@ enum io_uring_op { · 169 unmodified lines
240	241	IORING_OP_URING_CMD,
241	242	IORING_OP_SEND_ZC,
242	243	IORING_OP_SENDMSG_ZC,
	244	+ IORING_OP_READ_MULTISHOT,
	245	+ IORING_OP_WAITID,
243	246
244	247	/* this goes last, obviously */
245	248	IORING_OP_LAST,
246	249	};
247	250
248	251	/*
249		− * sqe->uring_cmd_flags
	252	+ * sqe->uring_cmd_flags top 8bits aren't available for userspace
250	253	* IORING_URING_CMD_FIXED use registered buffer; pass this flag
251	254	* along with setting sqe->buf_index.
252		− * IORING_URING_CMD_POLLED driver use only
253	255	*/
254	256	#define IORING_URING_CMD_FIXED (1U << 0)
255		−#define IORING_URING_CMD_POLLED (1U << 31)
	257	+#define IORING_URING_CMD_MASK IORING_URING_CMD_FIXED
256	258
257	259
258	260	/*

io_uring/Makefile+2−1

@@ -7,5 +7,6 @@ obj-$(CONFIG_IO_URING) += io_uring.o xattr.o nop.o fs.o splice.o \ · 6 unmodified lines
7	7	openclose.o uring_cmd.o epoll.o \
8	8	statx.o net.o msg_ring.o timeout.o \
9	9	sqpoll.o fdinfo.o tctx.o poll.o \
10		− cancel.o kbuf.o rsrc.o rw.o opdef.o notif.o
	10	+ cancel.o kbuf.o rsrc.o rw.o opdef.o \
	11	+ notif.o waitid.o
11	12	obj-$(CONFIG_IO_WQ) += io-wq.o

io_uring/cancel.c+5−0

@@ -15,6 +15,7 @@ · 14 unmodified lines
15	15	#include "tctx.h"
16	16	#include "poll.h"
17	17	#include "timeout.h"
	18	+#include "waitid.h"
18	19	#include "cancel.h"
19	20
20	21	struct io_cancel {
@@ -119,6 +120,10 @@ int io_try_cancel(struct io_uring_task tctx, struct io_cancel_data cd, · 98 unmodified lines
119	120	if (ret != -ENOENT)
120	121	return ret;
121	122
	123	+ ret = io_waitid_cancel(ctx, cd, issue_flags);
	124	+ if (ret != -ENOENT)
	125	+ return ret;
	126	+
122	127	spin_lock(&ctx->completion_lock);
123	128	if (!(cd->flags & IORING_ASYNC_CANCEL_FD))
124	129	ret = io_timeout_cancel(ctx, cd);

io_uring/io_uring.c+42−1

@@ -92,6 +92,7 @@ · 91 unmodified lines
92	92	#include "cancel.h"
93	93	#include "net.h"
94	94	#include "notif.h"
	95	+#include "waitid.h"
95	96
96	97	#include "timeout.h"
97	98	#include "poll.h"
@@ -338,7 +339,6 @@ static __cold struct io_ring_ctx io_ring_ctx_alloc(struct io_uring_params p) · 240 unmodified lines
338	339	spin_lock_init(&ctx->completion_lock);
339	340	spin_lock_init(&ctx->timeout_lock);
340	341	INIT_WQ_LIST(&ctx->iopoll_list);
341		− INIT_LIST_HEAD(&ctx->io_buffers_pages);
342	342	INIT_LIST_HEAD(&ctx->io_buffers_comp);
343	343	INIT_LIST_HEAD(&ctx->defer_list);
344	344	INIT_LIST_HEAD(&ctx->timeout_list);
@@ -348,8 +348,10 @@ static __cold struct io_ring_ctx io_ring_ctx_alloc(struct io_uring_params p) · 3 unmodified lines
348	348	INIT_LIST_HEAD(&ctx->tctx_list);
349	349	ctx->submit_state.free_list.next = NULL;
350	350	INIT_WQ_LIST(&ctx->locked_free_list);
	351	+ INIT_HLIST_HEAD(&ctx->waitid_list);
351	352	INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);
352	353	INIT_WQ_LIST(&ctx->submit_state.compl_reqs);
	354	+ INIT_HLIST_HEAD(&ctx->cancelable_uring_cmd);
353	355	return ctx;
354	356	err:
355	357	kfree(ctx->cancel_table.hbs);
@@ -3276,6 +3278,37 @@ static __cold bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx) · 2920 unmodified lines
3276	3278	return ret;
3277	3279	}
3278	3280
	3281	+static bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx,
	3282	+ struct task_struct *task, bool cancel_all)
	3283	+{
	3284	+ struct hlist_node *tmp;
	3285	+ struct io_kiocb *req;
	3286	+ bool ret = false;
	3287	+
	3288	+ lockdep_assert_held(&ctx->uring_lock);
	3289	+
	3290	+ hlist_for_each_entry_safe(req, tmp, &ctx->cancelable_uring_cmd,
	3291	+ hash_node) {
	3292	+ struct io_uring_cmd *cmd = io_kiocb_to_cmd(req,
	3293	+ struct io_uring_cmd);
	3294	+ struct file *file = req->file;
	3295	+
	3296	+ if (!cancel_all && req->task != task)
	3297	+ continue;
	3298	+
	3299	+ if (cmd->flags & IORING_URING_CMD_CANCELABLE) {
	3300	+ /* ->sqe isn't available if no async data */
	3301	+ if (!req_has_async_data(req))
	3302	+ cmd->sqe = NULL;
	3303	+ file->f_op->uring_cmd(cmd, IO_URING_F_CANCEL);
	3304	+ ret = true;
	3305	+ }
	3306	+ }
	3307	+ io_submit_flush_completions(ctx);
	3308	+
	3309	+ return ret;
	3310	+}
	3311	+
3279	3312	static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
3280	3313	struct task_struct *task,
3281	3314	bool cancel_all)
@@ -3323,6 +3356,8 @@ static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, · 41 unmodified lines
3323	3356	ret \|= io_cancel_defer_files(ctx, task, cancel_all);
3324	3357	mutex_lock(&ctx->uring_lock);
3325	3358	ret \|= io_poll_remove_all(ctx, task, cancel_all);
	3359	+ ret \|= io_waitid_remove_all(ctx, task, cancel_all);
	3360	+ ret \|= io_uring_try_cancel_uring_cmd(ctx, task, cancel_all);
3326	3361	mutex_unlock(&ctx->uring_lock);
3327	3362	ret \|= io_kill_timeouts(ctx, task, cancel_all);
3328	3363	if (task)
@@ -4686,6 +4721,9 @@ static int __init io_uring_init(void) · 1357 unmodified lines
4686	4721
4687	4722	BUILD_BUG_ON(sizeof(atomic_t) != sizeof(u32));
4688	4723
	4724	+ /* top 8bits are for internal use */
	4725	+ BUILD_BUG_ON((IORING_URING_CMD_MASK & 0xff000000) != 0);
	4726	+
4689	4727	io_uring_optable_init();
4690	4728
4691	4729	/*
@@ -4701,6 +4739,9 @@ static int __init io_uring_init(void) · 9 unmodified lines
4701	4739	SLAB_ACCOUNT \| SLAB_TYPESAFE_BY_RCU,
4702	4740	offsetof(struct io_kiocb, cmd.data),
4703	4741	sizeof_field(struct io_kiocb, cmd.data), NULL);
	4742	+ io_buf_cachep = kmem_cache_create("io_buffer", sizeof(struct io_buffer), 0,
	4743	+ SLAB_HWCACHE_ALIGN \| SLAB_PANIC \| SLAB_ACCOUNT,
	4744	+ NULL);
4704	4745
4705	4746	#ifdef CONFIG_SYSCTL
4706	4747	register_sysctl_init("kernel", kernel_io_uring_disabled_table);

io_uring/io_uring.h+1−0

@@ -343,6 +343,7 @@ static inline bool io_req_cache_empty(struct io_ring_ctx *ctx) · 342 unmodified lines
343	343	}
344	344
345	345	extern struct kmem_cache *req_cachep;
	346	+extern struct kmem_cache *io_buf_cachep;
346	347
347	348	static inline struct io_kiocb io_extract_req(struct io_ring_ctx ctx)
348	349	{

io_uring/kbuf.c+33−25

@@ -19,12 +19,17 @@ · 18 unmodified lines
19	19
20	20	#define BGID_ARRAY 64
21	21
	22	+/* BIDs are addressed by a 16-bit field in a CQE */
	23	+#define MAX_BIDS_PER_BGID (1 << 16)
	24	+
	25	+struct kmem_cache *io_buf_cachep;
	26	+
22	27	struct io_provide_buf {
23	28	struct file *file;
24	29	__u64 addr;
25	30	__u32 len;
26	31	__u32 bgid;
27		− __u16 nbufs;
	32	+ __u32 nbufs;
28	33	__u16 bid;
29	34	};
30	35
@@ -255,6 +260,8 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx, · 224 unmodified lines
255	260	void io_destroy_buffers(struct io_ring_ctx *ctx)
256	261	{
257	262	struct io_buffer_list *bl;
	263	+ struct list_head item, tmp;
	264	+ struct io_buffer *buf;
258	265	unsigned long index;
259	266	int i;
260	267
@@ -270,12 +277,9 @@ void io_destroy_buffers(struct io_ring_ctx *ctx) · 9 unmodified lines
270	277	kfree(bl);
271	278	}
272	279
273		− while (!list_empty(&ctx->io_buffers_pages)) {
274		− struct page *page;
275		−
276		− page = list_first_entry(&ctx->io_buffers_pages, struct page, lru);
277		− list_del_init(&page->lru);
278		− __free_page(page);
	280	+ list_for_each_safe(item, tmp, &ctx->io_buffers_cache) {
	281	+ buf = list_entry(item, struct io_buffer, list);
	282	+ kmem_cache_free(io_buf_cachep, buf);
279	283	}
280	284	}
281	285
@@ -289,7 +293,7 @@ int io_remove_buffers_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 7 unmodified lines
289	293	return -EINVAL;
290	294
291	295	tmp = READ_ONCE(sqe->fd);
292		− if (!tmp \|\| tmp > USHRT_MAX)
	296	+ if (!tmp \|\| tmp > MAX_BIDS_PER_BGID)
293	297	return -EINVAL;
294	298
295	299	memset(p, 0, sizeof(*p));
@@ -332,7 +336,7 @@ int io_provide_buffers_prep(struct io_kiocb req, const struct io_uring_sqe sqe · 36 unmodified lines
332	336	return -EINVAL;
333	337
334	338	tmp = READ_ONCE(sqe->fd);
335		− if (!tmp \|\| tmp > USHRT_MAX)
	339	+ if (!tmp \|\| tmp > MAX_BIDS_PER_BGID)
336	340	return -E2BIG;
337	341	p->nbufs = tmp;
338	342	p->addr = READ_ONCE(sqe->addr);
@@ -352,17 +356,18 @@ int io_provide_buffers_prep(struct io_kiocb req, const struct io_uring_sqe sqe · 13 unmodified lines
352	356	tmp = READ_ONCE(sqe->off);
353	357	if (tmp > USHRT_MAX)
354	358	return -E2BIG;
355		− if (tmp + p->nbufs >= USHRT_MAX)
	359	+ if (tmp + p->nbufs > MAX_BIDS_PER_BGID)
356	360	return -EINVAL;
357	361	p->bid = tmp;
358	362	return 0;
359	363	}
360	364
	365	+#define IO_BUFFER_ALLOC_BATCH 64
	366	+
361	367	static int io_refill_buffer_cache(struct io_ring_ctx *ctx)
362	368	{
363		− struct io_buffer *buf;
364		− struct page *page;
365		− int bufs_in_page;
	369	+ struct io_buffer *bufs[IO_BUFFER_ALLOC_BATCH];
	370	+ int allocated;
366	371
367	372	/*
368	373	* Completions that don't happen inline (eg not under uring_lock) will
@@ -382,22 +387,25 @@ static int io_refill_buffer_cache(struct io_ring_ctx *ctx) · 13 unmodified lines
382	387
383	388	/*
384	389	* No free buffers and no completion entries either. Allocate a new
385		− * page worth of buffer entries and add those to our freelist.
	390	+ * batch of buffer entries and add those to our freelist.
386	391	*/
387		− page = alloc_page(GFP_KERNEL_ACCOUNT);
388		− if (!page)
389		− return -ENOMEM;
390		−
391		− list_add(&page->lru, &ctx->io_buffers_pages);
392	392
393		− buf = page_address(page);
394		− bufs_in_page = PAGE_SIZE / sizeof(*buf);
395		− while (bufs_in_page) {
396		− list_add_tail(&buf->list, &ctx->io_buffers_cache);
397		− buf++;
398		− bufs_in_page--;
	393	+ allocated = kmem_cache_alloc_bulk(io_buf_cachep, GFP_KERNEL_ACCOUNT,
	394	+ ARRAY_SIZE(bufs), (void **) bufs);
	395	+ if (unlikely(!allocated)) {
	396	+ /*
	397	+ * Bulk alloc is all-or-nothing. If we fail to get a batch,
	398	+ * retry single alloc to be on the safe side.
	399	+ */
	400	+ bufs[0] = kmem_cache_alloc(io_buf_cachep, GFP_KERNEL);
	401	+ if (!bufs[0])
	402	+ return -ENOMEM;
	403	+ allocated = 1;
399	404	}
400	405
	406	+ while (allocated)
	407	+ list_add_tail(&bufs[--allocated]->list, &ctx->io_buffers_cache);
	408	+
401	409	return 0;
402	410	}
403	411

io_uring/opdef.c+23−1

@@ -33,6 +33,7 @@ · 32 unmodified lines
33	33	#include "poll.h"
34	34	#include "cancel.h"
35	35	#include "rw.h"
	36	+#include "waitid.h"
36	37
37	38	static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
38	39	{
@@ -63,6 +64,7 @@ const struct io_issue_def io_issue_defs[] = { · 24 unmodified lines
63	64	.ioprio = 1,
64	65	.iopoll = 1,
65	66	.iopoll_queue = 1,
	67	+ .vectored = 1,
66	68	.prep = io_prep_rw,
67	69	.issue = io_read,
68	70	},
@@ -76,6 +78,7 @@ const struct io_issue_def io_issue_defs[] = { · 7 unmodified lines
76	78	.ioprio = 1,
77	79	.iopoll = 1,
78	80	.iopoll_queue = 1,
	81	+ .vectored = 1,
79	82	.prep = io_prep_rw,
80	83	.issue = io_write,
81	84	},
@@ -428,9 +431,21 @@ const struct io_issue_def io_issue_defs[] = { · 346 unmodified lines
428	431	.prep = io_eopnotsupp_prep,
429	432	#endif
430	433	},
	434	+ [IORING_OP_READ_MULTISHOT] = {
	435	+ .needs_file = 1,
	436	+ .unbound_nonreg_file = 1,
	437	+ .pollin = 1,
	438	+ .buffer_select = 1,
	439	+ .audit_skip = 1,
	440	+ .prep = io_read_mshot_prep,
	441	+ .issue = io_read_mshot,
	442	+ },
	443	+ [IORING_OP_WAITID] = {
	444	+ .prep = io_waitid_prep,
	445	+ .issue = io_waitid,
	446	+ },
431	447	};
432	448
433		−
434	449	const struct io_cold_def io_cold_defs[] = {
435	450	[IORING_OP_NOP] = {
436	451	.name = "NOP",
@@ -648,6 +663,13 @@ const struct io_cold_def io_cold_defs[] = { · 211 unmodified lines
648	663	.fail = io_sendrecv_fail,
649	664	#endif
650	665	},
	666	+ [IORING_OP_READ_MULTISHOT] = {
	667	+ .name = "READ_MULTISHOT",
	668	+ },
	669	+ [IORING_OP_WAITID] = {
	670	+ .name = "WAITID",
	671	+ .async_size = sizeof(struct io_waitid_async),
	672	+ },
651	673	};
652	674
653	675	const char *io_uring_get_opcode(u8 opcode)

io_uring/opdef.h+2−0

@@ -29,6 +29,8 @@ struct io_issue_def { · 28 unmodified lines
29	29	unsigned iopoll_queue : 1;
30	30	/* opcode specific path will handle ->async_data allocation if needed */
31	31	unsigned manual_alloc : 1;
	32	+ /* vectored opcode, set if 1) vectored, and 2) handler needs to know */
	33	+ unsigned vectored : 1;
32	34
33	35	int (issue)(struct io_kiocb , unsigned int);
34	36	int (prep)(struct io_kiocb , const struct io_uring_sqe *);

io_uring/poll.c+1−1

@@ -370,7 +370,7 @@ static void __io_poll_execute(struct io_kiocb *req, int mask) · 369 unmodified lines
370	370	req->io_task_work.func = io_poll_task_func;
371	371
372	372	trace_io_uring_task_add(req, mask);
373		− io_req_task_work_add(req);
	373	+ __io_req_task_work_add(req, IOU_F_TWQ_LAZY_WAKE);
374	374	}
375	375
376	376	static inline void io_poll_execute(struct io_kiocb *req, int res)

io_uring/rsrc.c+17−20

@@ -1037,39 +1037,36 @@ struct page *io_pin_pages(unsigned long ubuf, unsigned long len, int npages) · 1036 unmodified lines
1037	1037	{
1038	1038	unsigned long start, end, nr_pages;
1039	1039	struct page **pages = NULL;
1040		− int pret, ret = -ENOMEM;
	1040	+ int ret;
1041	1041
1042	1042	end = (ubuf + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1043	1043	start = ubuf >> PAGE_SHIFT;
1044	1044	nr_pages = end - start;
	1045	+ WARN_ON(!nr_pages);
1045	1046
1046	1047	pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
1047	1048	if (!pages)
1048		− goto done;
	1049	+ return ERR_PTR(-ENOMEM);
1049	1050
1050		− ret = 0;
1051	1051	mmap_read_lock(current->mm);
1052		− pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE \| FOLL_LONGTERM,
1053		− pages);
1054		− if (pret == nr_pages)
	1052	+ ret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE \| FOLL_LONGTERM, pages);
	1053	+ mmap_read_unlock(current->mm);
	1054	+
	1055	+ /* success, mapped all pages */
	1056	+ if (ret == nr_pages) {
1055	1057	*npages = nr_pages;
1056		− else
1057		− ret = pret < 0 ? pret : -EFAULT;
	1058	+ return pages;
	1059	+ }
1058	1060
1059		− mmap_read_unlock(current->mm);
1060		− if (ret) {
	1061	+ /* partial map, or didn't map anything */
	1062	+ if (ret >= 0) {
1061	1063	/* if we did partial map, release any pages we did get */
1062		− if (pret > 0)
1063		− unpin_user_pages(pages, pret);
1064		− goto done;
1065		− }
1066		− ret = 0;
1067		−done:
1068		− if (ret < 0) {
1069		− kvfree(pages);
1070		− pages = ERR_PTR(ret);
	1064	+ if (ret)
	1065	+ unpin_user_pages(pages, ret);
	1066	+ ret = -EFAULT;
1071	1067	}
1072		− return pages;
	1068	+ kvfree(pages);
	1069	+ return ERR_PTR(ret);
1073	1070	}
1074	1071
1075	1072	static int io_sqe_buffer_register(struct io_ring_ctx ctx, struct iovec iov,

io_uring/rw.c+86−6

@@ -123,6 +123,22 @@ int io_prep_rw(struct io_kiocb req, const struct io_uring_sqe sqe) · 122 unmodified lines
123	123	return 0;
124	124	}
125	125
	126	+/*
	127	+ * Multishot read is prepared just like a normal read/write request, only
	128	+ * difference is that we set the MULTISHOT flag.
	129	+ */
	130	+int io_read_mshot_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
	131	+{
	132	+ int ret;
	133	+
	134	+ ret = io_prep_rw(req, sqe);
	135	+ if (unlikely(ret))
	136	+ return ret;
	137	+
	138	+ req->flags \|= REQ_F_APOLL_MULTISHOT;
	139	+ return 0;
	140	+}
	141	+
126	142	void io_readv_writev_cleanup(struct io_kiocb *req)
127	143	{
128	144	struct io_async_rw *io = req->async_data;
@@ -388,8 +404,7 @@ static struct iovec __io_import_iovec(int ddir, struct io_kiocb req, · 259 unmodified lines
388	404	buf = u64_to_user_ptr(rw->addr);
389	405	sqe_len = rw->len;
390	406
391		− if (opcode == IORING_OP_READ \|\| opcode == IORING_OP_WRITE \|\|
392		− (req->flags & REQ_F_BUFFER_SELECT)) {
	407	+ if (!io_issue_defs[opcode].vectored \|\| req->flags & REQ_F_BUFFER_SELECT) {
393	408	if (io_do_buffer_select(req)) {
394	409	buf = io_buffer_select(req, &sqe_len, issue_flags);
395	410	if (!buf)
@@ -708,7 +723,7 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode) · 312 unmodified lines
708	723	return 0;
709	724	}
710	725
711		−int io_read(struct io_kiocb *req, unsigned int issue_flags)
	726	+static int __io_read(struct io_kiocb *req, unsigned int issue_flags)
712	727	{
713	728	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
714	729	struct io_rw_state __s, *s = &__s;
@@ -776,8 +791,11 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) · 61 unmodified lines
776	791
777	792	if (ret == -EAGAIN \|\| (req->flags & REQ_F_REISSUE)) {
778	793	req->flags &= ~REQ_F_REISSUE;
779		− /* if we can poll, just do that */
780		− if (req->opcode == IORING_OP_READ && file_can_poll(req->file))
	794	+ /*
	795	+ * If we can poll, just do that. For a vectored read, we'll
	796	+ * need to copy state first.
	797	+ */
	798	+ if (file_can_poll(req->file) && !io_issue_defs[req->opcode].vectored)
781	799	return -EAGAIN;
782	800	/* IOPOLL retry should happen for io-wq threads */
783	801	if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
@@ -853,7 +871,69 @@ done: · 69 unmodified lines
853	871	/* it's faster to check here then delegate to kfree */
854	872	if (iovec)
855	873	kfree(iovec);
856		− return kiocb_done(req, ret, issue_flags);
	874	+ return ret;
	875	+}
	876	+
	877	+int io_read(struct io_kiocb *req, unsigned int issue_flags)
	878	+{
	879	+ int ret;
	880	+
	881	+ ret = __io_read(req, issue_flags);
	882	+ if (ret >= 0)
	883	+ return kiocb_done(req, ret, issue_flags);
	884	+
	885	+ return ret;
	886	+}
	887	+
	888	+int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)
	889	+{
	890	+ unsigned int cflags = 0;
	891	+ int ret;
	892	+
	893	+ /*
	894	+ * Multishot MUST be used on a pollable file
	895	+ */
	896	+ if (!file_can_poll(req->file))
	897	+ return -EBADFD;
	898	+
	899	+ ret = __io_read(req, issue_flags);
	900	+
	901	+ /*
	902	+ * If we get -EAGAIN, recycle our buffer and just let normal poll
	903	+ * handling arm it.
	904	+ */
	905	+ if (ret == -EAGAIN) {
	906	+ io_kbuf_recycle(req, issue_flags);
	907	+ return -EAGAIN;
	908	+ }
	909	+
	910	+ /*
	911	+ * Any successful return value will keep the multishot read armed.
	912	+ */
	913	+ if (ret > 0) {
	914	+ /*
	915	+ * Put our buffer and post a CQE. If we fail to post a CQE, then
	916	+ * jump to the termination path. This request is then done.
	917	+ */
	918	+ cflags = io_put_kbuf(req, issue_flags);
	919	+
	920	+ if (io_fill_cqe_req_aux(req,
	921	+ issue_flags & IO_URING_F_COMPLETE_DEFER,
	922	+ ret, cflags \| IORING_CQE_F_MORE)) {
	923	+ if (issue_flags & IO_URING_F_MULTISHOT)
	924	+ return IOU_ISSUE_SKIP_COMPLETE;
	925	+ return -EAGAIN;
	926	+ }
	927	+ }
	928	+
	929	+ /*
	930	+ * Either an error, or we've hit overflow posting the CQE. For any
	931	+ * multishot request, hitting overflow will terminate it.
	932	+ */
	933	+ io_req_set_res(req, ret, cflags);
	934	+ if (issue_flags & IO_URING_F_MULTISHOT)
	935	+ return IOU_STOP_MULTISHOT;
	936	+ return IOU_OK;
857	937	}
858	938
859	939	int io_write(struct io_kiocb *req, unsigned int issue_flags)

io_uring/rw.h+2−0

@@ -23,3 +23,5 @@ int io_writev_prep_async(struct io_kiocb *req); · 22 unmodified lines
23	23	void io_readv_writev_cleanup(struct io_kiocb *req);
24	24	void io_rw_fail(struct io_kiocb *req);
25	25	void io_req_rw_complete(struct io_kiocb req, struct io_tw_state ts);
	26	+int io_read_mshot_prep(struct io_kiocb req, const struct io_uring_sqe sqe);
	27	+int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags);

io_uring/uring_cmd.c+48−1

@@ -13,6 +13,51 @@ · 12 unmodified lines
13	13	#include "rsrc.h"
14	14	#include "uring_cmd.h"
15	15
	16	+static void io_uring_cmd_del_cancelable(struct io_uring_cmd *cmd,
	17	+ unsigned int issue_flags)
	18	+{
	19	+ struct io_kiocb *req = cmd_to_io_kiocb(cmd);
	20	+ struct io_ring_ctx *ctx = req->ctx;
	21	+
	22	+ if (!(cmd->flags & IORING_URING_CMD_CANCELABLE))
	23	+ return;
	24	+
	25	+ cmd->flags &= ~IORING_URING_CMD_CANCELABLE;
	26	+ io_ring_submit_lock(ctx, issue_flags);
	27	+ hlist_del(&req->hash_node);
	28	+ io_ring_submit_unlock(ctx, issue_flags);
	29	+}
	30	+
	31	+/*
	32	+ * Mark this command as concelable, then io_uring_try_cancel_uring_cmd()
	33	+ * will try to cancel this issued command by sending ->uring_cmd() with
	34	+ * issue_flags of IO_URING_F_CANCEL.
	35	+ *
	36	+ * The command is guaranteed to not be done when calling ->uring_cmd()
	37	+ * with IO_URING_F_CANCEL, but it is driver's responsibility to deal
	38	+ * with race between io_uring canceling and normal completion.
	39	+ */
	40	+void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
	41	+ unsigned int issue_flags)
	42	+{
	43	+ struct io_kiocb *req = cmd_to_io_kiocb(cmd);
	44	+ struct io_ring_ctx *ctx = req->ctx;
	45	+
	46	+ if (!(cmd->flags & IORING_URING_CMD_CANCELABLE)) {
	47	+ cmd->flags \|= IORING_URING_CMD_CANCELABLE;
	48	+ io_ring_submit_lock(ctx, issue_flags);
	49	+ hlist_add_head(&req->hash_node, &ctx->cancelable_uring_cmd);
	50	+ io_ring_submit_unlock(ctx, issue_flags);
	51	+ }
	52	+}
	53	+EXPORT_SYMBOL_GPL(io_uring_cmd_mark_cancelable);
	54	+
	55	+struct task_struct io_uring_cmd_get_task(struct io_uring_cmd cmd)
	56	+{
	57	+ return cmd_to_io_kiocb(cmd)->task;
	58	+}
	59	+EXPORT_SYMBOL_GPL(io_uring_cmd_get_task);
	60	+
16	61	static void io_uring_cmd_work(struct io_kiocb req, struct io_tw_state ts)
17	62	{
18	63	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
@@ -56,6 +101,8 @@ void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2, · 37 unmodified lines
56	101	{
57	102	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
58	103
	104	+ io_uring_cmd_del_cancelable(ioucmd, issue_flags);
	105	+
59	106	if (ret < 0)
60	107	req_set_fail(req);
61	108
@@ -91,7 +138,7 @@ int io_uring_cmd_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 29 unmodified lines
91	138	return -EINVAL;
92	139
93	140	ioucmd->flags = READ_ONCE(sqe->uring_cmd_flags);
94		− if (ioucmd->flags & ~IORING_URING_CMD_FIXED)
	141	+ if (ioucmd->flags & ~IORING_URING_CMD_MASK)
95	142	return -EINVAL;
96	143
97	144	if (ioucmd->flags & IORING_URING_CMD_FIXED) {

io_uring/waitid.c+372−0

@@ -0,0 +1,372 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * Support for async notification of waitid
	4	+ */
	5	+#include <linux/kernel.h>
	6	+#include <linux/errno.h>
	7	+#include <linux/fs.h>
	8	+#include <linux/file.h>
	9	+#include <linux/compat.h>
	10	+#include <linux/io_uring.h>
	11	+
	12	+#include <uapi/linux/io_uring.h>
	13	+
	14	+#include "io_uring.h"
	15	+#include "cancel.h"
	16	+#include "waitid.h"
	17	+#include "../kernel/exit.h"
	18	+
	19	+static void io_waitid_cb(struct io_kiocb req, struct io_tw_state ts);
	20	+
	21	+#define IO_WAITID_CANCEL_FLAG BIT(31)
	22	+#define IO_WAITID_REF_MASK GENMASK(30, 0)
	23	+
	24	+struct io_waitid {
	25	+ struct file *file;
	26	+ int which;
	27	+ pid_t upid;
	28	+ int options;
	29	+ atomic_t refs;
	30	+ struct wait_queue_head *head;
	31	+ struct siginfo __user *infop;
	32	+ struct waitid_info info;
	33	+};
	34	+
	35	+static void io_waitid_free(struct io_kiocb *req)
	36	+{
	37	+ struct io_waitid_async *iwa = req->async_data;
	38	+
	39	+ put_pid(iwa->wo.wo_pid);
	40	+ kfree(req->async_data);
	41	+ req->async_data = NULL;
	42	+ req->flags &= ~REQ_F_ASYNC_DATA;
	43	+}
	44	+
	45	+#ifdef CONFIG_COMPAT
	46	+static bool io_waitid_compat_copy_si(struct io_waitid *iw, int signo)
	47	+{
	48	+ struct compat_siginfo __user *infop;
	49	+ bool ret;
	50	+
	51	+ infop = (struct compat_siginfo __user *) iw->infop;
	52	+
	53	+ if (!user_write_access_begin(infop, sizeof(*infop)))
	54	+ return false;
	55	+
	56	+ unsafe_put_user(signo, &infop->si_signo, Efault);
	57	+ unsafe_put_user(0, &infop->si_errno, Efault);
	58	+ unsafe_put_user(iw->info.cause, &infop->si_code, Efault);
	59	+ unsafe_put_user(iw->info.pid, &infop->si_pid, Efault);
	60	+ unsafe_put_user(iw->info.uid, &infop->si_uid, Efault);
	61	+ unsafe_put_user(iw->info.status, &infop->si_status, Efault);
	62	+ ret = true;
	63	+done:
	64	+ user_write_access_end();
	65	+ return ret;
	66	+Efault:
	67	+ ret = false;
	68	+ goto done;
	69	+}
	70	+#endif
	71	+
	72	+static bool io_waitid_copy_si(struct io_kiocb *req, int signo)
	73	+{
	74	+ struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
	75	+ bool ret;
	76	+
	77	+ if (!iw->infop)
	78	+ return true;
	79	+
	80	+#ifdef CONFIG_COMPAT
	81	+ if (req->ctx->compat)
	82	+ return io_waitid_compat_copy_si(iw, signo);
	83	+#endif
	84	+
	85	+ if (!user_write_access_begin(iw->infop, sizeof(*iw->infop)))
	86	+ return false;
	87	+
	88	+ unsafe_put_user(signo, &iw->infop->si_signo, Efault);
	89	+ unsafe_put_user(0, &iw->infop->si_errno, Efault);
	90	+ unsafe_put_user(iw->info.cause, &iw->infop->si_code, Efault);
	91	+ unsafe_put_user(iw->info.pid, &iw->infop->si_pid, Efault);
	92	+ unsafe_put_user(iw->info.uid, &iw->infop->si_uid, Efault);
	93	+ unsafe_put_user(iw->info.status, &iw->infop->si_status, Efault);
	94	+ ret = true;
	95	+done:
	96	+ user_write_access_end();
	97	+ return ret;
	98	+Efault:
	99	+ ret = false;
	100	+ goto done;
	101	+}
	102	+
	103	+static int io_waitid_finish(struct io_kiocb *req, int ret)
	104	+{
	105	+ int signo = 0;
	106	+
	107	+ if (ret > 0) {
	108	+ signo = SIGCHLD;
	109	+ ret = 0;
	110	+ }
	111	+
	112	+ if (!io_waitid_copy_si(req, signo))
	113	+ ret = -EFAULT;
	114	+ io_waitid_free(req);
	115	+ return ret;
	116	+}
	117	+
	118	+static void io_waitid_complete(struct io_kiocb *req, int ret)
	119	+{
	120	+ struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
	121	+ struct io_tw_state ts = { .locked = true };
	122	+
	123	+ /* anyone completing better be holding a reference */
	124	+ WARN_ON_ONCE(!(atomic_read(&iw->refs) & IO_WAITID_REF_MASK));
	125	+
	126	+ lockdep_assert_held(&req->ctx->uring_lock);
	127	+
	128	+ /*
	129	+ * Did cancel find it meanwhile?
	130	+ */
	131	+ if (hlist_unhashed(&req->hash_node))
	132	+ return;
	133	+
	134	+ hlist_del_init(&req->hash_node);
	135	+
	136	+ ret = io_waitid_finish(req, ret);
	137	+ if (ret < 0)
	138	+ req_set_fail(req);
	139	+ io_req_set_res(req, ret, 0);
	140	+ io_req_task_complete(req, &ts);
	141	+}
	142	+
	143	+static bool __io_waitid_cancel(struct io_ring_ctx ctx, struct io_kiocb req)
	144	+{
	145	+ struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
	146	+ struct io_waitid_async *iwa = req->async_data;
	147	+
	148	+ /*
	149	+ * Mark us canceled regardless of ownership. This will prevent a
	150	+ * potential retry from a spurious wakeup.
	151	+ */
	152	+ atomic_or(IO_WAITID_CANCEL_FLAG, &iw->refs);
	153	+
	154	+ /* claim ownership */
	155	+ if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK)
	156	+ return false;
	157	+
	158	+ spin_lock_irq(&iw->head->lock);
	159	+ list_del_init(&iwa->wo.child_wait.entry);
	160	+ spin_unlock_irq(&iw->head->lock);
	161	+ io_waitid_complete(req, -ECANCELED);
	162	+ return true;
	163	+}
	164	+
	165	+int io_waitid_cancel(struct io_ring_ctx ctx, struct io_cancel_data cd,
	166	+ unsigned int issue_flags)
	167	+{
	168	+ struct hlist_node *tmp;
	169	+ struct io_kiocb *req;
	170	+ int nr = 0;
	171	+
	172	+ if (cd->flags & (IORING_ASYNC_CANCEL_FD\|IORING_ASYNC_CANCEL_FD_FIXED))
	173	+ return -ENOENT;
	174	+
	175	+ io_ring_submit_lock(ctx, issue_flags);
	176	+ hlist_for_each_entry_safe(req, tmp, &ctx->waitid_list, hash_node) {
	177	+ if (req->cqe.user_data != cd->data &&
	178	+ !(cd->flags & IORING_ASYNC_CANCEL_ANY))
	179	+ continue;
	180	+ if (__io_waitid_cancel(ctx, req))
	181	+ nr++;
	182	+ if (!(cd->flags & IORING_ASYNC_CANCEL_ALL))
	183	+ break;
	184	+ }
	185	+ io_ring_submit_unlock(ctx, issue_flags);
	186	+
	187	+ if (nr)
	188	+ return nr;
	189	+
	190	+ return -ENOENT;
	191	+}
	192	+
	193	+bool io_waitid_remove_all(struct io_ring_ctx ctx, struct task_struct task,
	194	+ bool cancel_all)
	195	+{
	196	+ struct hlist_node *tmp;
	197	+ struct io_kiocb *req;
	198	+ bool found = false;
	199	+
	200	+ lockdep_assert_held(&ctx->uring_lock);
	201	+
	202	+ hlist_for_each_entry_safe(req, tmp, &ctx->waitid_list, hash_node) {
	203	+ if (!io_match_task_safe(req, task, cancel_all))
	204	+ continue;
	205	+ __io_waitid_cancel(ctx, req);
	206	+ found = true;
	207	+ }
	208	+
	209	+ return found;
	210	+}
	211	+
	212	+static inline bool io_waitid_drop_issue_ref(struct io_kiocb *req)
	213	+{
	214	+ struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
	215	+ struct io_waitid_async *iwa = req->async_data;
	216	+
	217	+ if (!atomic_sub_return(1, &iw->refs))
	218	+ return false;
	219	+
	220	+ /*
	221	+ * Wakeup triggered, racing with us. It was prevented from
	222	+ * completing because of that, queue up the tw to do that.
	223	+ */
	224	+ req->io_task_work.func = io_waitid_cb;
	225	+ io_req_task_work_add(req);
	226	+ remove_wait_queue(iw->head, &iwa->wo.child_wait);
	227	+ return true;
	228	+}
	229	+
	230	+static void io_waitid_cb(struct io_kiocb req, struct io_tw_state ts)
	231	+{
	232	+ struct io_waitid_async *iwa = req->async_data;
	233	+ struct io_ring_ctx *ctx = req->ctx;
	234	+ int ret;
	235	+
	236	+ io_tw_lock(ctx, ts);
	237	+
	238	+ ret = __do_wait(&iwa->wo);
	239	+
	240	+ /*
	241	+ * If we get -ERESTARTSYS here, we need to re-arm and check again
	242	+ * to ensure we get another callback. If the retry works, then we can
	243	+ * just remove ourselves from the waitqueue again and finish the
	244	+ * request.
	245	+ */
	246	+ if (unlikely(ret == -ERESTARTSYS)) {
	247	+ struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
	248	+
	249	+ /* Don't retry if cancel found it meanwhile */
	250	+ ret = -ECANCELED;
	251	+ if (!(atomic_read(&iw->refs) & IO_WAITID_CANCEL_FLAG)) {
	252	+ iw->head = &current->signal->wait_chldexit;
	253	+ add_wait_queue(iw->head, &iwa->wo.child_wait);
	254	+ ret = __do_wait(&iwa->wo);
	255	+ if (ret == -ERESTARTSYS) {
	256	+ /* retry armed, drop our ref */
	257	+ io_waitid_drop_issue_ref(req);
	258	+ return;
	259	+ }
	260	+
	261	+ remove_wait_queue(iw->head, &iwa->wo.child_wait);
	262	+ }
	263	+ }
	264	+
	265	+ io_waitid_complete(req, ret);
	266	+}
	267	+
	268	+static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode,
	269	+ int sync, void *key)
	270	+{
	271	+ struct wait_opts *wo = container_of(wait, struct wait_opts, child_wait);
	272	+ struct io_waitid_async *iwa = container_of(wo, struct io_waitid_async, wo);
	273	+ struct io_kiocb *req = iwa->req;
	274	+ struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
	275	+ struct task_struct *p = key;
	276	+
	277	+ if (!pid_child_should_wake(wo, p))
	278	+ return 0;
	279	+
	280	+ /* cancel is in progress */
	281	+ if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK)
	282	+ return 1;
	283	+
	284	+ req->io_task_work.func = io_waitid_cb;
	285	+ io_req_task_work_add(req);
	286	+ list_del_init(&wait->entry);
	287	+ return 1;
	288	+}
	289	+
	290	+int io_waitid_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
	291	+{
	292	+ struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
	293	+
	294	+ if (sqe->addr \|\| sqe->buf_index \|\| sqe->addr3 \|\| sqe->waitid_flags)
	295	+ return -EINVAL;
	296	+
	297	+ iw->which = READ_ONCE(sqe->len);
	298	+ iw->upid = READ_ONCE(sqe->fd);
	299	+ iw->options = READ_ONCE(sqe->file_index);
	300	+ iw->infop = u64_to_user_ptr(READ_ONCE(sqe->addr2));
	301	+ return 0;
	302	+}
	303	+
	304	+int io_waitid(struct io_kiocb *req, unsigned int issue_flags)
	305	+{
	306	+ struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
	307	+ struct io_ring_ctx *ctx = req->ctx;
	308	+ struct io_waitid_async *iwa;
	309	+ int ret;
	310	+
	311	+ if (io_alloc_async_data(req))
	312	+ return -ENOMEM;
	313	+
	314	+ iwa = req->async_data;
	315	+ iwa->req = req;
	316	+
	317	+ ret = kernel_waitid_prepare(&iwa->wo, iw->which, iw->upid, &iw->info,
	318	+ iw->options, NULL);
	319	+ if (ret)
	320	+ goto done;
	321	+
	322	+ /*
	323	+ * Mark the request as busy upfront, in case we're racing with the
	324	+ * wakeup. If we are, then we'll notice when we drop this initial
	325	+ * reference again after arming.
	326	+ */
	327	+ atomic_set(&iw->refs, 1);
	328	+
	329	+ /*
	330	+ * Cancel must hold the ctx lock, so there's no risk of cancelation
	331	+ * finding us until a) we remain on the list, and b) the lock is
	332	+ * dropped. We only need to worry about racing with the wakeup
	333	+ * callback.
	334	+ */
	335	+ io_ring_submit_lock(ctx, issue_flags);
	336	+ hlist_add_head(&req->hash_node, &ctx->waitid_list);
	337	+
	338	+ init_waitqueue_func_entry(&iwa->wo.child_wait, io_waitid_wait);
	339	+ iwa->wo.child_wait.private = req->task;
	340	+ iw->head = &current->signal->wait_chldexit;
	341	+ add_wait_queue(iw->head, &iwa->wo.child_wait);
	342	+
	343	+ ret = __do_wait(&iwa->wo);
	344	+ if (ret == -ERESTARTSYS) {
	345	+ /*
	346	+ * Nobody else grabbed a reference, it'll complete when we get
	347	+ * a waitqueue callback, or if someone cancels it.
	348	+ */
	349	+ if (!io_waitid_drop_issue_ref(req)) {
	350	+ io_ring_submit_unlock(ctx, issue_flags);
	351	+ return IOU_ISSUE_SKIP_COMPLETE;
	352	+ }
	353	+
	354	+ /*
	355	+ * Wakeup triggered, racing with us. It was prevented from
	356	+ * completing because of that, queue up the tw to do that.
	357	+ */
	358	+ io_ring_submit_unlock(ctx, issue_flags);
	359	+ return IOU_ISSUE_SKIP_COMPLETE;
	360	+ }
	361	+
	362	+ hlist_del_init(&req->hash_node);
	363	+ remove_wait_queue(iw->head, &iwa->wo.child_wait);
	364	+ ret = io_waitid_finish(req, ret);
	365	+
	366	+ io_ring_submit_unlock(ctx, issue_flags);
	367	+done:
	368	+ if (ret < 0)
	369	+ req_set_fail(req);
	370	+ io_req_set_res(req, ret, 0);
	371	+ return IOU_OK;
	372	+}

io_uring/waitid.h+15−0

@@ -0,0 +1,15 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+#include "../kernel/exit.h"
	4	+
	5	+struct io_waitid_async {
	6	+ struct io_kiocb *req;
	7	+ struct wait_opts wo;
	8	+};
	9	+
	10	+int io_waitid_prep(struct io_kiocb req, const struct io_uring_sqe sqe);
	11	+int io_waitid(struct io_kiocb *req, unsigned int issue_flags);
	12	+int io_waitid_cancel(struct io_ring_ctx ctx, struct io_cancel_data cd,
	13	+ unsigned int issue_flags);
	14	+bool io_waitid_remove_all(struct io_ring_ctx ctx, struct task_struct task,
	15	+ bool cancel_all);

kernel/exit.c+72−59

@@ -74,6 +74,8 @@ · 73 unmodified lines
74	74	#include <asm/unistd.h>
75	75	#include <asm/mmu_context.h>
76	76
	77	+#include "exit.h"
	78	+
77	79	/*
78	80	* The default value should be high enough to not crash a system that randomly
79	81	* crashes its kernel from time to time, but low enough to at least not permit
@@ -1037,26 +1039,6 @@ SYSCALL_DEFINE1(exit_group, int, error_code) · 957 unmodified lines
1037	1039	return 0;
1038	1040	}
1039	1041
1040		−struct waitid_info {
1041		− pid_t pid;
1042		− uid_t uid;
1043		− int status;
1044		− int cause;
1045		−};
1046		−
1047		−struct wait_opts {
1048		− enum pid_type wo_type;
1049		− int wo_flags;
1050		− struct pid *wo_pid;
1051		−
1052		− struct waitid_info *wo_info;
1053		− int wo_stat;
1054		− struct rusage *wo_rusage;
1055		−
1056		− wait_queue_entry_t child_wait;
1057		− int notask_error;
1058		−};
1059		−
1060	1042	static int eligible_pid(struct wait_opts wo, struct task_struct p)
1061	1043	{
1062	1044	return wo->wo_type == PIDTYPE_MAX \|\|
@@ -1520,6 +1502,17 @@ static int ptrace_do_wait(struct wait_opts wo, struct task_struct tsk) · 457 unmodified lines
1520	1502	return 0;
1521	1503	}
1522	1504
	1505	+bool pid_child_should_wake(struct wait_opts wo, struct task_struct p)
	1506	+{
	1507	+ if (!eligible_pid(wo, p))
	1508	+ return false;
	1509	+
	1510	+ if ((wo->wo_flags & __WNOTHREAD) && wo->child_wait.private != p->parent)
	1511	+ return false;
	1512	+
	1513	+ return true;
	1514	+}
	1515	+
1523	1516	static int child_wait_callback(wait_queue_entry_t *wait, unsigned mode,
1524	1517	int sync, void *key)
1525	1518	{
@@ -1527,13 +1520,10 @@ static int child_wait_callback(wait_queue_entry_t *wait, unsigned mode, · 1 unmodified line
1527	1520	child_wait);
1528	1521	struct task_struct *p = key;
1529	1522
1530		− if (!eligible_pid(wo, p))
1531		− return 0;
	1523	+ if (pid_child_should_wake(wo, p))
	1524	+ return default_wake_function(wait, mode, sync, key);
1532	1525
1533		− if ((wo->wo_flags & __WNOTHREAD) && wait->private != p->parent)
1534		− return 0;
1535		−
1536		− return default_wake_function(wait, mode, sync, key);
	1526	+ return 0;
1537	1527	}
1538	1528
1539	1529	void __wake_up_parent(struct task_struct p, struct task_struct parent)
@@ -1582,16 +1572,10 @@ static int do_wait_pid(struct wait_opts *wo) · 42 unmodified lines
1582	1572	return 0;
1583	1573	}
1584	1574
1585		−static long do_wait(struct wait_opts *wo)
	1575	+long __do_wait(struct wait_opts *wo)
1586	1576	{
1587		− int retval;
	1577	+ long retval;
1588	1578
1589		− trace_sched_process_wait(wo->wo_pid);
1590		−
1591		− init_waitqueue_func_entry(&wo->child_wait, child_wait_callback);
1592		− wo->child_wait.private = current;
1593		− add_wait_queue(&current->signal->wait_chldexit, &wo->child_wait);
1594		−repeat:
1595	1579	/*
1596	1580	* If there is nothing that can match our criteria, just get out.
1597	1581	* We will clear ->notask_error to zero if we see any child that
@@ -1603,24 +1587,23 @@ repeat: · 5 unmodified lines
1603	1587	(!wo->wo_pid \|\| !pid_has_task(wo->wo_pid, wo->wo_type)))
1604	1588	goto notask;
1605	1589
1606		− set_current_state(TASK_INTERRUPTIBLE);
1607	1590	read_lock(&tasklist_lock);
1608	1591
1609	1592	if (wo->wo_type == PIDTYPE_PID) {
1610	1593	retval = do_wait_pid(wo);
1611	1594	if (retval)
1612		− goto end;
	1595	+ return retval;
1613	1596	} else {
1614	1597	struct task_struct *tsk = current;
1615	1598
1616	1599	do {
1617	1600	retval = do_wait_thread(wo, tsk);
1618	1601	if (retval)
1619		− goto end;
	1602	+ return retval;
1620	1603
1621	1604	retval = ptrace_do_wait(wo, tsk);
1622	1605	if (retval)
1623		− goto end;
	1606	+ return retval;
1624	1607
1625	1608	if (wo->wo_flags & __WNOTHREAD)
1626	1609	break;
@@ -1630,27 +1613,44 @@ repeat: · 3 unmodified lines
1630	1613
1631	1614	notask:
1632	1615	retval = wo->notask_error;
1633		− if (!retval && !(wo->wo_flags & WNOHANG)) {
1634		− retval = -ERESTARTSYS;
1635		− if (!signal_pending(current)) {
1636		− schedule();
1637		− goto repeat;
1638		− }
1639		− }
1640		−end:
	1616	+ if (!retval && !(wo->wo_flags & WNOHANG))
	1617	+ return -ERESTARTSYS;
	1618	+
	1619	+ return retval;
	1620	+}
	1621	+
	1622	+static long do_wait(struct wait_opts *wo)
	1623	+{
	1624	+ int retval;
	1625	+
	1626	+ trace_sched_process_wait(wo->wo_pid);
	1627	+
	1628	+ init_waitqueue_func_entry(&wo->child_wait, child_wait_callback);
	1629	+ wo->child_wait.private = current;
	1630	+ add_wait_queue(&current->signal->wait_chldexit, &wo->child_wait);
	1631	+
	1632	+ do {
	1633	+ set_current_state(TASK_INTERRUPTIBLE);
	1634	+ retval = __do_wait(wo);
	1635	+ if (retval != -ERESTARTSYS)
	1636	+ break;
	1637	+ if (signal_pending(current))
	1638	+ break;
	1639	+ schedule();
	1640	+ } while (1);
	1641	+
1641	1642	__set_current_state(TASK_RUNNING);
1642	1643	remove_wait_queue(&current->signal->wait_chldexit, &wo->child_wait);
1643	1644	return retval;
1644	1645	}
1645	1646
1646		−static long kernel_waitid(int which, pid_t upid, struct waitid_info *infop,
1647		− int options, struct rusage *ru)
	1647	+int kernel_waitid_prepare(struct wait_opts *wo, int which, pid_t upid,
	1648	+ struct waitid_info *infop, int options,
	1649	+ struct rusage *ru)
1648	1650	{
1649		− struct wait_opts wo;
	1651	+ unsigned int f_flags = 0;
1650	1652	struct pid *pid = NULL;
1651	1653	enum pid_type type;
1652		− long ret;
1653		− unsigned int f_flags = 0;
1654	1654
1655	1655	if (options & ~(WNOHANG\|WNOWAIT\|WEXITED\|WSTOPPED\|WCONTINUED\|
1656	1656	__WNOTHREAD\|__WCLONE\|__WALL))
@@ -1693,19 +1693,32 @@ static long kernel_waitid(int which, pid_t upid, struct waitid_info *infop, · 36 unmodified lines
1693	1693	return -EINVAL;
1694	1694	}
1695	1695
1696		− wo.wo_type = type;
1697		− wo.wo_pid = pid;
1698		− wo.wo_flags = options;
1699		− wo.wo_info = infop;
1700		− wo.wo_rusage = ru;
	1696	+ wo->wo_type = type;
	1697	+ wo->wo_pid = pid;
	1698	+ wo->wo_flags = options;
	1699	+ wo->wo_info = infop;
	1700	+ wo->wo_rusage = ru;
1701	1701	if (f_flags & O_NONBLOCK)
1702		− wo.wo_flags \|= WNOHANG;
	1702	+ wo->wo_flags \|= WNOHANG;
	1703	+
	1704	+ return 0;
	1705	+}
	1706	+
	1707	+static long kernel_waitid(int which, pid_t upid, struct waitid_info *infop,
	1708	+ int options, struct rusage *ru)
	1709	+{
	1710	+ struct wait_opts wo;
	1711	+ long ret;
	1712	+
	1713	+ ret = kernel_waitid_prepare(&wo, which, upid, infop, options, ru);
	1714	+ if (ret)
	1715	+ return ret;
1703	1716
1704	1717	ret = do_wait(&wo);
1705		− if (!ret && !(options & WNOHANG) && (f_flags & O_NONBLOCK))
	1718	+ if (!ret && !(options & WNOHANG) && (wo.wo_flags & WNOHANG))
1706	1719	ret = -EAGAIN;
1707	1720
1708		− put_pid(pid);
	1721	+ put_pid(wo.wo_pid);
1709	1722	return ret;
1710	1723	}
1711	1724

kernel/exit.h+30−0

@@ -0,0 +1,30 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+#ifndef LINUX_WAITID_H
	3	+#define LINUX_WAITID_H
	4	+
	5	+struct waitid_info {
	6	+ pid_t pid;
	7	+ uid_t uid;
	8	+ int status;
	9	+ int cause;
	10	+};
	11	+
	12	+struct wait_opts {
	13	+ enum pid_type wo_type;
	14	+ int wo_flags;
	15	+ struct pid *wo_pid;
	16	+
	17	+ struct waitid_info *wo_info;
	18	+ int wo_stat;
	19	+ struct rusage *wo_rusage;
	20	+
	21	+ wait_queue_entry_t child_wait;
	22	+ int notask_error;
	23	+};
	24	+
	25	+bool pid_child_should_wake(struct wait_opts wo, struct task_struct p);
	26	+long __do_wait(struct wait_opts *wo);
	27	+int kernel_waitid_prepare(struct wait_opts *wo, int which, pid_t upid,
	28	+ struct waitid_info *infop, int options,
	29	+ struct rusage *ru);
	30	+#endif