History

Merge tag 'for-6.7/io_uring-2023-10-30' of git://git.kernel.dk/linux

Pull io_uring updates from Jens Axboe: "This contains the core io_uring updates, of which there are not many, and adds support for using WAITID through io_uring and hence not needing to block on these kinds of events. Outside of that, tweaks to the legacy provided buffer handling and some cleanups related to cancelations for uring_cmd support" * tag 'for-6.7/io_uring-2023-10-30' of git://git.kernel.dk/linux: io_uring/poll: use IOU_F_TWQ_LAZY_WAKE for wakeups io_uring/kbuf: Use slab for struct io_buffer objects io_uring/kbuf: Allow the full buffer id space for provided buffers io_uring/kbuf: Fix check of BID wrapping in provided buffers io_uring/rsrc: cleanup io_pin_pages() io_uring: cancelable uring_cmd io_uring: retain top 8bits of uring_cmd flags for kernel internal use io_uring: add IORING_OP_WAITID support exit: add internal include file with helpers exit: add kernel_waitid_prepare() helper exit: move core of do_wait() into helper exit: abstract out should_wake helper for child_wait_callback() io_uring/rw: add support for IORING_OP_READ_MULTISHOT io_uring/rw: mark readv/writev as vectored in the opcode definition io_uring/rw: split io_read() into a helper

Linus Torvalds committed 3 years ago

Commit ffa059b2 parents ca995ce 6ce4a9319 files · +782 −120Browse files

Unified · Split

19 files
  1. include/linux/io_uring.h +18 −0
  2. include/linux/io_uring_types.h +8 −2
  3. include/uapi/linux/io_uring.h +5 −3
  4. io_uring/Makefile +2 −1
  5. io_uring/cancel.c +5 −0
  6. io_uring/io_uring.c +42 −1
  7. io_uring/io_uring.h +1 −0
  8. io_uring/kbuf.c +33 −25
  9. io_uring/opdef.c +23 −1
  10. io_uring/opdef.h +2 −0
  11. io_uring/poll.c +1 −1
  12. io_uring/rsrc.c +17 −20
  13. io_uring/rw.c +86 −6
  14. io_uring/rw.h +2 −0
  15. io_uring/uring_cmd.c +48 −1
  16. io_uring/waitid.c +372 −0
  17. io_uring/waitid.h +15 −0
  18. kernel/exit.c +72 −59
  19. kernel/exit.h +30 −0
include/linux/io_uring.h+18−0
@@ -20,8 +20,15 @@ enum io_uring_cmd_flags { · 19 unmodified lines
20 IO_URING_F_SQE128 = (1 << 8),20 IO_URING_F_SQE128 = (1 << 8),
21 IO_URING_F_CQE32 = (1 << 9),21 IO_URING_F_CQE32 = (1 << 9),
22 IO_URING_F_IOPOLL = (1 << 10),22 IO_URING_F_IOPOLL = (1 << 10),
23
24 /* set when uring wants to cancel a previously issued command */
25 IO_URING_F_CANCEL = (1 << 11),
23};26};
24 27
28/* only top 8 bits of sqe->uring_cmd_flags for kernel internal use */
29#define IORING_URING_CMD_CANCELABLE (1U << 30)
30#define IORING_URING_CMD_POLLED (1U << 31)
31
25struct io_uring_cmd {32struct io_uring_cmd {
26 struct file *file;33 struct file *file;
27 const struct io_uring_sqe *sqe;34 const struct io_uring_sqe *sqe;
@@ -82,6 +89,9 @@ static inline void io_uring_free(struct task_struct *tsk) · 54 unmodified lines
82 __io_uring_free(tsk);89 __io_uring_free(tsk);
83}90}
84int io_uring_cmd_sock(struct io_uring_cmd *cmd, unsigned int issue_flags);91int io_uring_cmd_sock(struct io_uring_cmd *cmd, unsigned int issue_flags);
92void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
93 unsigned int issue_flags);
94struct task_struct *io_uring_cmd_get_task(struct io_uring_cmd *cmd);
85#else95#else
86static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,96static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
87 struct iov_iter *iter, void *ioucmd)97 struct iov_iter *iter, void *ioucmd)
@@ -122,6 +132,14 @@ static inline int io_uring_cmd_sock(struct io_uring_cmd *cmd, · 34 unmodified lines
122{132{
123 return -EOPNOTSUPP;133 return -EOPNOTSUPP;
124}134}
135static inline void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
136 unsigned int issue_flags)
137{
138}
139static inline struct task_struct *io_uring_cmd_get_task(struct io_uring_cmd *cmd)
140{
141 return NULL;
142}
125#endif143#endif
126 144
127#endif145#endif
include/linux/io_uring_types.h+8−2
@@ -265,6 +265,12 @@ struct io_ring_ctx { · 264 unmodified lines
265 */265 */
266 struct io_wq_work_list iopoll_list;266 struct io_wq_work_list iopoll_list;
267 bool poll_multi_queue;267 bool poll_multi_queue;
268
269 /*
270 * Any cancelable uring_cmd is added to this list in
271 * ->uring_cmd() by io_uring_cmd_insert_cancelable()
272 */
273 struct hlist_head cancelable_uring_cmd;
268 } ____cacheline_aligned_in_smp;274 } ____cacheline_aligned_in_smp;
269 275
270 struct {276 struct {
@@ -313,6 +319,8 @@ struct io_ring_ctx { · 42 unmodified lines
313 struct list_head cq_overflow_list;319 struct list_head cq_overflow_list;
314 struct io_hash_table cancel_table;320 struct io_hash_table cancel_table;
315 321
322 struct hlist_head waitid_list;
323
316 const struct cred *sq_creds; /* cred used for __io_sq_thread() */324 const struct cred *sq_creds; /* cred used for __io_sq_thread() */
317 struct io_sq_data *sq_data; /* if using sq thread polling */325 struct io_sq_data *sq_data; /* if using sq thread polling */
318 326
@@ -342,8 +350,6 @@ struct io_ring_ctx { · 23 unmodified lines
342 struct wait_queue_head rsrc_quiesce_wq;350 struct wait_queue_head rsrc_quiesce_wq;
343 unsigned rsrc_quiesce;351 unsigned rsrc_quiesce;
344 352
345 struct list_head io_buffers_pages;
346
347 #if defined(CONFIG_UNIX)353 #if defined(CONFIG_UNIX)
348 struct socket *ring_sock;354 struct socket *ring_sock;
349 #endif355 #endif
include/uapi/linux/io_uring.h+5−3
@@ -65,6 +65,7 @@ struct io_uring_sqe { · 64 unmodified lines
65 __u32 xattr_flags;65 __u32 xattr_flags;
66 __u32 msg_ring_flags;66 __u32 msg_ring_flags;
67 __u32 uring_cmd_flags;67 __u32 uring_cmd_flags;
68 __u32 waitid_flags;
68 };69 };
69 __u64 user_data; /* data to be passed back at completion time */70 __u64 user_data; /* data to be passed back at completion time */
70 /* pack this to avoid bogus arm OABI complaints */71 /* pack this to avoid bogus arm OABI complaints */
@@ -240,19 +241,20 @@ enum io_uring_op { · 169 unmodified lines
240 IORING_OP_URING_CMD,241 IORING_OP_URING_CMD,
241 IORING_OP_SEND_ZC,242 IORING_OP_SEND_ZC,
242 IORING_OP_SENDMSG_ZC,243 IORING_OP_SENDMSG_ZC,
244 IORING_OP_READ_MULTISHOT,
245 IORING_OP_WAITID,
243 246
244 /* this goes last, obviously */247 /* this goes last, obviously */
245 IORING_OP_LAST,248 IORING_OP_LAST,
246};249};
247 250
248/*251/*
249 * sqe->uring_cmd_flags252 * sqe->uring_cmd_flags top 8bits aren't available for userspace
250 * IORING_URING_CMD_FIXED use registered buffer; pass this flag253 * IORING_URING_CMD_FIXED use registered buffer; pass this flag
251 * along with setting sqe->buf_index.254 * along with setting sqe->buf_index.
252 * IORING_URING_CMD_POLLED driver use only
253 */255 */
254#define IORING_URING_CMD_FIXED (1U << 0)256#define IORING_URING_CMD_FIXED (1U << 0)
255#define IORING_URING_CMD_POLLED (1U << 31)257#define IORING_URING_CMD_MASK IORING_URING_CMD_FIXED
256 258
257 259
258/*260/*
io_uring/Makefile+2−1
@@ -7,5 +7,6 @@ obj-$(CONFIG_IO_URING) += io_uring.o xattr.o nop.o fs.o splice.o \ · 6 unmodified lines
7 openclose.o uring_cmd.o epoll.o \7 openclose.o uring_cmd.o epoll.o \
8 statx.o net.o msg_ring.o timeout.o \8 statx.o net.o msg_ring.o timeout.o \
9 sqpoll.o fdinfo.o tctx.o poll.o \9 sqpoll.o fdinfo.o tctx.o poll.o \
10 cancel.o kbuf.o rsrc.o rw.o opdef.o notif.o10 cancel.o kbuf.o rsrc.o rw.o opdef.o \
11 notif.o waitid.o
11obj-$(CONFIG_IO_WQ) += io-wq.o12obj-$(CONFIG_IO_WQ) += io-wq.o
io_uring/cancel.c+5−0
@@ -15,6 +15,7 @@ · 14 unmodified lines
15#include "tctx.h"15#include "tctx.h"
16#include "poll.h"16#include "poll.h"
17#include "timeout.h"17#include "timeout.h"
18#include "waitid.h"
18#include "cancel.h"19#include "cancel.h"
19 20
20struct io_cancel {21struct io_cancel {
@@ -119,6 +120,10 @@ int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd, · 98 unmodified lines
119 if (ret != -ENOENT)120 if (ret != -ENOENT)
120 return ret;121 return ret;
121 122
123 ret = io_waitid_cancel(ctx, cd, issue_flags);
124 if (ret != -ENOENT)
125 return ret;
126
122 spin_lock(&ctx->completion_lock);127 spin_lock(&ctx->completion_lock);
123 if (!(cd->flags & IORING_ASYNC_CANCEL_FD))128 if (!(cd->flags & IORING_ASYNC_CANCEL_FD))
124 ret = io_timeout_cancel(ctx, cd);129 ret = io_timeout_cancel(ctx, cd);
io_uring/io_uring.c+42−1
@@ -92,6 +92,7 @@ · 91 unmodified lines
92#include "cancel.h"92#include "cancel.h"
93#include "net.h"93#include "net.h"
94#include "notif.h"94#include "notif.h"
95#include "waitid.h"
95 96
96#include "timeout.h"97#include "timeout.h"
97#include "poll.h"98#include "poll.h"
@@ -338,7 +339,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) · 240 unmodified lines
338 spin_lock_init(&ctx->completion_lock);339 spin_lock_init(&ctx->completion_lock);
339 spin_lock_init(&ctx->timeout_lock);340 spin_lock_init(&ctx->timeout_lock);
340 INIT_WQ_LIST(&ctx->iopoll_list);341 INIT_WQ_LIST(&ctx->iopoll_list);
341 INIT_LIST_HEAD(&ctx->io_buffers_pages);
342 INIT_LIST_HEAD(&ctx->io_buffers_comp);342 INIT_LIST_HEAD(&ctx->io_buffers_comp);
343 INIT_LIST_HEAD(&ctx->defer_list);343 INIT_LIST_HEAD(&ctx->defer_list);
344 INIT_LIST_HEAD(&ctx->timeout_list);344 INIT_LIST_HEAD(&ctx->timeout_list);
@@ -348,8 +348,10 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) · 3 unmodified lines
348 INIT_LIST_HEAD(&ctx->tctx_list);348 INIT_LIST_HEAD(&ctx->tctx_list);
349 ctx->submit_state.free_list.next = NULL;349 ctx->submit_state.free_list.next = NULL;
350 INIT_WQ_LIST(&ctx->locked_free_list);350 INIT_WQ_LIST(&ctx->locked_free_list);
351 INIT_HLIST_HEAD(&ctx->waitid_list);
351 INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);352 INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);
352 INIT_WQ_LIST(&ctx->submit_state.compl_reqs);353 INIT_WQ_LIST(&ctx->submit_state.compl_reqs);
354 INIT_HLIST_HEAD(&ctx->cancelable_uring_cmd);
353 return ctx;355 return ctx;
354err:356err:
355 kfree(ctx->cancel_table.hbs);357 kfree(ctx->cancel_table.hbs);
@@ -3276,6 +3278,37 @@ static __cold bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx) · 2920 unmodified lines
3276 return ret;3278 return ret;
3277}3279}
3278 3280
3281static bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx,
3282 struct task_struct *task, bool cancel_all)
3283{
3284 struct hlist_node *tmp;
3285 struct io_kiocb *req;
3286 bool ret = false;
3287
3288 lockdep_assert_held(&ctx->uring_lock);
3289
3290 hlist_for_each_entry_safe(req, tmp, &ctx->cancelable_uring_cmd,
3291 hash_node) {
3292 struct io_uring_cmd *cmd = io_kiocb_to_cmd(req,
3293 struct io_uring_cmd);
3294 struct file *file = req->file;
3295
3296 if (!cancel_all && req->task != task)
3297 continue;
3298
3299 if (cmd->flags & IORING_URING_CMD_CANCELABLE) {
3300 /* ->sqe isn't available if no async data */
3301 if (!req_has_async_data(req))
3302 cmd->sqe = NULL;
3303 file->f_op->uring_cmd(cmd, IO_URING_F_CANCEL);
3304 ret = true;
3305 }
3306 }
3307 io_submit_flush_completions(ctx);
3308
3309 return ret;
3310}
3311
3279static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,3312static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
3280 struct task_struct *task,3313 struct task_struct *task,
3281 bool cancel_all)3314 bool cancel_all)
@@ -3323,6 +3356,8 @@ static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, · 41 unmodified lines
3323 ret |= io_cancel_defer_files(ctx, task, cancel_all);3356 ret |= io_cancel_defer_files(ctx, task, cancel_all);
3324 mutex_lock(&ctx->uring_lock);3357 mutex_lock(&ctx->uring_lock);
3325 ret |= io_poll_remove_all(ctx, task, cancel_all);3358 ret |= io_poll_remove_all(ctx, task, cancel_all);
3359 ret |= io_waitid_remove_all(ctx, task, cancel_all);
3360 ret |= io_uring_try_cancel_uring_cmd(ctx, task, cancel_all);
3326 mutex_unlock(&ctx->uring_lock);3361 mutex_unlock(&ctx->uring_lock);
3327 ret |= io_kill_timeouts(ctx, task, cancel_all);3362 ret |= io_kill_timeouts(ctx, task, cancel_all);
3328 if (task)3363 if (task)
@@ -4686,6 +4721,9 @@ static int __init io_uring_init(void) · 1357 unmodified lines
4686 4721
4687 BUILD_BUG_ON(sizeof(atomic_t) != sizeof(u32));4722 BUILD_BUG_ON(sizeof(atomic_t) != sizeof(u32));
4688 4723
4724 /* top 8bits are for internal use */
4725 BUILD_BUG_ON((IORING_URING_CMD_MASK & 0xff000000) != 0);
4726
4689 io_uring_optable_init();4727 io_uring_optable_init();
4690 4728
4691 /*4729 /*
@@ -4701,6 +4739,9 @@ static int __init io_uring_init(void) · 9 unmodified lines
4701 SLAB_ACCOUNT | SLAB_TYPESAFE_BY_RCU,4739 SLAB_ACCOUNT | SLAB_TYPESAFE_BY_RCU,
4702 offsetof(struct io_kiocb, cmd.data),4740 offsetof(struct io_kiocb, cmd.data),
4703 sizeof_field(struct io_kiocb, cmd.data), NULL);4741 sizeof_field(struct io_kiocb, cmd.data), NULL);
4742 io_buf_cachep = kmem_cache_create("io_buffer", sizeof(struct io_buffer), 0,
4743 SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT,
4744 NULL);
4704 4745
4705#ifdef CONFIG_SYSCTL4746#ifdef CONFIG_SYSCTL
4706 register_sysctl_init("kernel", kernel_io_uring_disabled_table);4747 register_sysctl_init("kernel", kernel_io_uring_disabled_table);
io_uring/io_uring.h+1−0
@@ -343,6 +343,7 @@ static inline bool io_req_cache_empty(struct io_ring_ctx *ctx) · 342 unmodified lines
343}343}
344 344
345extern struct kmem_cache *req_cachep;345extern struct kmem_cache *req_cachep;
346extern struct kmem_cache *io_buf_cachep;
346 347
347static inline struct io_kiocb *io_extract_req(struct io_ring_ctx *ctx)348static inline struct io_kiocb *io_extract_req(struct io_ring_ctx *ctx)
348{349{
io_uring/kbuf.c+33−25
@@ -19,12 +19,17 @@ · 18 unmodified lines
19 19
20#define BGID_ARRAY 6420#define BGID_ARRAY 64
21 21
22/* BIDs are addressed by a 16-bit field in a CQE */
23#define MAX_BIDS_PER_BGID (1 << 16)
24
25struct kmem_cache *io_buf_cachep;
26
22struct io_provide_buf {27struct io_provide_buf {
23 struct file *file;28 struct file *file;
24 __u64 addr;29 __u64 addr;
25 __u32 len;30 __u32 len;
26 __u32 bgid;31 __u32 bgid;
27 __u16 nbufs;32 __u32 nbufs;
28 __u16 bid;33 __u16 bid;
29};34};
30 35
@@ -255,6 +260,8 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx, · 224 unmodified lines
255void io_destroy_buffers(struct io_ring_ctx *ctx)260void io_destroy_buffers(struct io_ring_ctx *ctx)
256{261{
257 struct io_buffer_list *bl;262 struct io_buffer_list *bl;
263 struct list_head *item, *tmp;
264 struct io_buffer *buf;
258 unsigned long index;265 unsigned long index;
259 int i;266 int i;
260 267
@@ -270,12 +277,9 @@ void io_destroy_buffers(struct io_ring_ctx *ctx) · 9 unmodified lines
270 kfree(bl);277 kfree(bl);
271 }278 }
272 279
273 while (!list_empty(&ctx->io_buffers_pages)) {280 list_for_each_safe(item, tmp, &ctx->io_buffers_cache) {
274 struct page *page;281 buf = list_entry(item, struct io_buffer, list);
275 282 kmem_cache_free(io_buf_cachep, buf);
276 page = list_first_entry(&ctx->io_buffers_pages, struct page, lru);
277 list_del_init(&page->lru);
278 __free_page(page);
279 }283 }
280}284}
281 285
@@ -289,7 +293,7 @@ int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 7 unmodified lines
289 return -EINVAL;293 return -EINVAL;
290 294
291 tmp = READ_ONCE(sqe->fd);295 tmp = READ_ONCE(sqe->fd);
292 if (!tmp || tmp > USHRT_MAX)296 if (!tmp || tmp > MAX_BIDS_PER_BGID)
293 return -EINVAL;297 return -EINVAL;
294 298
295 memset(p, 0, sizeof(*p));299 memset(p, 0, sizeof(*p));
@@ -332,7 +336,7 @@ int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe · 36 unmodified lines
332 return -EINVAL;336 return -EINVAL;
333 337
334 tmp = READ_ONCE(sqe->fd);338 tmp = READ_ONCE(sqe->fd);
335 if (!tmp || tmp > USHRT_MAX)339 if (!tmp || tmp > MAX_BIDS_PER_BGID)
336 return -E2BIG;340 return -E2BIG;
337 p->nbufs = tmp;341 p->nbufs = tmp;
338 p->addr = READ_ONCE(sqe->addr);342 p->addr = READ_ONCE(sqe->addr);
@@ -352,17 +356,18 @@ int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe · 13 unmodified lines
352 tmp = READ_ONCE(sqe->off);356 tmp = READ_ONCE(sqe->off);
353 if (tmp > USHRT_MAX)357 if (tmp > USHRT_MAX)
354 return -E2BIG;358 return -E2BIG;
355 if (tmp + p->nbufs >= USHRT_MAX)359 if (tmp + p->nbufs > MAX_BIDS_PER_BGID)
356 return -EINVAL;360 return -EINVAL;
357 p->bid = tmp;361 p->bid = tmp;
358 return 0;362 return 0;
359}363}
360 364
365#define IO_BUFFER_ALLOC_BATCH 64
366
361static int io_refill_buffer_cache(struct io_ring_ctx *ctx)367static int io_refill_buffer_cache(struct io_ring_ctx *ctx)
362{368{
363 struct io_buffer *buf;369 struct io_buffer *bufs[IO_BUFFER_ALLOC_BATCH];
364 struct page *page;370 int allocated;
365 int bufs_in_page;
366 371
367 /*372 /*
368 * Completions that don't happen inline (eg not under uring_lock) will373 * Completions that don't happen inline (eg not under uring_lock) will
@@ -382,22 +387,25 @@ static int io_refill_buffer_cache(struct io_ring_ctx *ctx) · 13 unmodified lines
382 387
383 /*388 /*
384 * No free buffers and no completion entries either. Allocate a new389 * No free buffers and no completion entries either. Allocate a new
385 * page worth of buffer entries and add those to our freelist.390 * batch of buffer entries and add those to our freelist.
386 */391 */
387 page = alloc_page(GFP_KERNEL_ACCOUNT);
388 if (!page)
389 return -ENOMEM;
390
391 list_add(&page->lru, &ctx->io_buffers_pages);
392 392
393 buf = page_address(page);393 allocated = kmem_cache_alloc_bulk(io_buf_cachep, GFP_KERNEL_ACCOUNT,
394 bufs_in_page = PAGE_SIZE / sizeof(*buf);394 ARRAY_SIZE(bufs), (void **) bufs);
395 while (bufs_in_page) {395 if (unlikely(!allocated)) {
396 list_add_tail(&buf->list, &ctx->io_buffers_cache);396 /*
397 buf++;397 * Bulk alloc is all-or-nothing. If we fail to get a batch,
398 bufs_in_page--;398 * retry single alloc to be on the safe side.
399 */
400 bufs[0] = kmem_cache_alloc(io_buf_cachep, GFP_KERNEL);
401 if (!bufs[0])
402 return -ENOMEM;
403 allocated = 1;
399 }404 }
400 405
406 while (allocated)
407 list_add_tail(&bufs[--allocated]->list, &ctx->io_buffers_cache);
408
401 return 0;409 return 0;
402}410}
403 411
io_uring/opdef.c+23−1
@@ -33,6 +33,7 @@ · 32 unmodified lines
33#include "poll.h"33#include "poll.h"
34#include "cancel.h"34#include "cancel.h"
35#include "rw.h"35#include "rw.h"
36#include "waitid.h"
36 37
37static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)38static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
38{39{
@@ -63,6 +64,7 @@ const struct io_issue_def io_issue_defs[] = { · 24 unmodified lines
63 .ioprio = 1,64 .ioprio = 1,
64 .iopoll = 1,65 .iopoll = 1,
65 .iopoll_queue = 1,66 .iopoll_queue = 1,
67 .vectored = 1,
66 .prep = io_prep_rw,68 .prep = io_prep_rw,
67 .issue = io_read,69 .issue = io_read,
68 },70 },
@@ -76,6 +78,7 @@ const struct io_issue_def io_issue_defs[] = { · 7 unmodified lines
76 .ioprio = 1,78 .ioprio = 1,
77 .iopoll = 1,79 .iopoll = 1,
78 .iopoll_queue = 1,80 .iopoll_queue = 1,
81 .vectored = 1,
79 .prep = io_prep_rw,82 .prep = io_prep_rw,
80 .issue = io_write,83 .issue = io_write,
81 },84 },
@@ -428,9 +431,21 @@ const struct io_issue_def io_issue_defs[] = { · 346 unmodified lines
428 .prep = io_eopnotsupp_prep,431 .prep = io_eopnotsupp_prep,
429#endif432#endif
430 },433 },
434 [IORING_OP_READ_MULTISHOT] = {
435 .needs_file = 1,
436 .unbound_nonreg_file = 1,
437 .pollin = 1,
438 .buffer_select = 1,
439 .audit_skip = 1,
440 .prep = io_read_mshot_prep,
441 .issue = io_read_mshot,
442 },
443 [IORING_OP_WAITID] = {
444 .prep = io_waitid_prep,
445 .issue = io_waitid,
446 },
431};447};
432 448
433
434const struct io_cold_def io_cold_defs[] = {449const struct io_cold_def io_cold_defs[] = {
435 [IORING_OP_NOP] = {450 [IORING_OP_NOP] = {
436 .name = "NOP",451 .name = "NOP",
@@ -648,6 +663,13 @@ const struct io_cold_def io_cold_defs[] = { · 211 unmodified lines
648 .fail = io_sendrecv_fail,663 .fail = io_sendrecv_fail,
649#endif664#endif
650 },665 },
666 [IORING_OP_READ_MULTISHOT] = {
667 .name = "READ_MULTISHOT",
668 },
669 [IORING_OP_WAITID] = {
670 .name = "WAITID",
671 .async_size = sizeof(struct io_waitid_async),
672 },
651};673};
652 674
653const char *io_uring_get_opcode(u8 opcode)675const char *io_uring_get_opcode(u8 opcode)
io_uring/opdef.h+2−0
@@ -29,6 +29,8 @@ struct io_issue_def { · 28 unmodified lines
29 unsigned iopoll_queue : 1;29 unsigned iopoll_queue : 1;
30 /* opcode specific path will handle ->async_data allocation if needed */30 /* opcode specific path will handle ->async_data allocation if needed */
31 unsigned manual_alloc : 1;31 unsigned manual_alloc : 1;
32 /* vectored opcode, set if 1) vectored, and 2) handler needs to know */
33 unsigned vectored : 1;
32 34
33 int (*issue)(struct io_kiocb *, unsigned int);35 int (*issue)(struct io_kiocb *, unsigned int);
34 int (*prep)(struct io_kiocb *, const struct io_uring_sqe *);36 int (*prep)(struct io_kiocb *, const struct io_uring_sqe *);
io_uring/poll.c+1−1
@@ -370,7 +370,7 @@ static void __io_poll_execute(struct io_kiocb *req, int mask) · 369 unmodified lines
370 req->io_task_work.func = io_poll_task_func;370 req->io_task_work.func = io_poll_task_func;
371 371
372 trace_io_uring_task_add(req, mask);372 trace_io_uring_task_add(req, mask);
373 io_req_task_work_add(req);373 __io_req_task_work_add(req, IOU_F_TWQ_LAZY_WAKE);
374}374}
375 375
376static inline void io_poll_execute(struct io_kiocb *req, int res)376static inline void io_poll_execute(struct io_kiocb *req, int res)
io_uring/rsrc.c+17−20
@@ -1037,39 +1037,36 @@ struct page **io_pin_pages(unsigned long ubuf, unsigned long len, int *npages) · 1036 unmodified lines
1037{1037{
1038 unsigned long start, end, nr_pages;1038 unsigned long start, end, nr_pages;
1039 struct page **pages = NULL;1039 struct page **pages = NULL;
1040 int pret, ret = -ENOMEM;1040 int ret;
1041 1041
1042 end = (ubuf + len + PAGE_SIZE - 1) >> PAGE_SHIFT;1042 end = (ubuf + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1043 start = ubuf >> PAGE_SHIFT;1043 start = ubuf >> PAGE_SHIFT;
1044 nr_pages = end - start;1044 nr_pages = end - start;
1045 WARN_ON(!nr_pages);
1045 1046
1046 pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);1047 pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
1047 if (!pages)1048 if (!pages)
1048 goto done;1049 return ERR_PTR(-ENOMEM);
1049 1050
1050 ret = 0;
1051 mmap_read_lock(current->mm);1051 mmap_read_lock(current->mm);
1052 pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM,1052 ret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM, pages);
1053 pages);1053 mmap_read_unlock(current->mm);
1054 if (pret == nr_pages)1054
1055 /* success, mapped all pages */
1056 if (ret == nr_pages) {
1055 *npages = nr_pages;1057 *npages = nr_pages;
1056 else1058 return pages;
1057 ret = pret < 0 ? pret : -EFAULT;1059 }
1058 1060
1059 mmap_read_unlock(current->mm);1061 /* partial map, or didn't map anything */
1060 if (ret) {1062 if (ret >= 0) {
1061 /* if we did partial map, release any pages we did get */1063 /* if we did partial map, release any pages we did get */
1062 if (pret > 0)1064 if (ret)
1063 unpin_user_pages(pages, pret);1065 unpin_user_pages(pages, ret);
1064 goto done;1066 ret = -EFAULT;
1065 }
1066 ret = 0;
1067done:
1068 if (ret < 0) {
1069 kvfree(pages);
1070 pages = ERR_PTR(ret);
1071 }1067 }
1072 return pages;1068 kvfree(pages);
1069 return ERR_PTR(ret);
1073}1070}
1074 1071
1075static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,1072static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
io_uring/rw.c+86−6
@@ -123,6 +123,22 @@ int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 122 unmodified lines
123 return 0;123 return 0;
124}124}
125 125
126/*
127 * Multishot read is prepared just like a normal read/write request, only
128 * difference is that we set the MULTISHOT flag.
129 */
130int io_read_mshot_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
131{
132 int ret;
133
134 ret = io_prep_rw(req, sqe);
135 if (unlikely(ret))
136 return ret;
137
138 req->flags |= REQ_F_APOLL_MULTISHOT;
139 return 0;
140}
141
126void io_readv_writev_cleanup(struct io_kiocb *req)142void io_readv_writev_cleanup(struct io_kiocb *req)
127{143{
128 struct io_async_rw *io = req->async_data;144 struct io_async_rw *io = req->async_data;
@@ -388,8 +404,7 @@ static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, · 259 unmodified lines
388 buf = u64_to_user_ptr(rw->addr);404 buf = u64_to_user_ptr(rw->addr);
389 sqe_len = rw->len;405 sqe_len = rw->len;
390 406
391 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE ||407 if (!io_issue_defs[opcode].vectored || req->flags & REQ_F_BUFFER_SELECT) {
392 (req->flags & REQ_F_BUFFER_SELECT)) {
393 if (io_do_buffer_select(req)) {408 if (io_do_buffer_select(req)) {
394 buf = io_buffer_select(req, &sqe_len, issue_flags);409 buf = io_buffer_select(req, &sqe_len, issue_flags);
395 if (!buf)410 if (!buf)
@@ -708,7 +723,7 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode) · 312 unmodified lines
708 return 0;723 return 0;
709}724}
710 725
711int io_read(struct io_kiocb *req, unsigned int issue_flags)726static int __io_read(struct io_kiocb *req, unsigned int issue_flags)
712{727{
713 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);728 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
714 struct io_rw_state __s, *s = &__s;729 struct io_rw_state __s, *s = &__s;
@@ -776,8 +791,11 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) · 61 unmodified lines
776 791
777 if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) {792 if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) {
778 req->flags &= ~REQ_F_REISSUE;793 req->flags &= ~REQ_F_REISSUE;
779 /* if we can poll, just do that */794 /*
780 if (req->opcode == IORING_OP_READ && file_can_poll(req->file))795 * If we can poll, just do that. For a vectored read, we'll
796 * need to copy state first.
797 */
798 if (file_can_poll(req->file) && !io_issue_defs[req->opcode].vectored)
781 return -EAGAIN;799 return -EAGAIN;
782 /* IOPOLL retry should happen for io-wq threads */800 /* IOPOLL retry should happen for io-wq threads */
783 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))801 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
@@ -853,7 +871,69 @@ done: · 69 unmodified lines
853 /* it's faster to check here then delegate to kfree */871 /* it's faster to check here then delegate to kfree */
854 if (iovec)872 if (iovec)
855 kfree(iovec);873 kfree(iovec);
856 return kiocb_done(req, ret, issue_flags);874 return ret;
875}
876
877int io_read(struct io_kiocb *req, unsigned int issue_flags)
878{
879 int ret;
880
881 ret = __io_read(req, issue_flags);
882 if (ret >= 0)
883 return kiocb_done(req, ret, issue_flags);
884
885 return ret;
886}
887
888int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)
889{
890 unsigned int cflags = 0;
891 int ret;
892
893 /*
894 * Multishot MUST be used on a pollable file
895 */
896 if (!file_can_poll(req->file))
897 return -EBADFD;
898
899 ret = __io_read(req, issue_flags);
900
901 /*
902 * If we get -EAGAIN, recycle our buffer and just let normal poll
903 * handling arm it.
904 */
905 if (ret == -EAGAIN) {
906 io_kbuf_recycle(req, issue_flags);
907 return -EAGAIN;
908 }
909
910 /*
911 * Any successful return value will keep the multishot read armed.
912 */
913 if (ret > 0) {
914 /*
915 * Put our buffer and post a CQE. If we fail to post a CQE, then
916 * jump to the termination path. This request is then done.
917 */
918 cflags = io_put_kbuf(req, issue_flags);
919
920 if (io_fill_cqe_req_aux(req,
921 issue_flags & IO_URING_F_COMPLETE_DEFER,
922 ret, cflags | IORING_CQE_F_MORE)) {
923 if (issue_flags & IO_URING_F_MULTISHOT)
924 return IOU_ISSUE_SKIP_COMPLETE;
925 return -EAGAIN;
926 }
927 }
928
929 /*
930 * Either an error, or we've hit overflow posting the CQE. For any
931 * multishot request, hitting overflow will terminate it.
932 */
933 io_req_set_res(req, ret, cflags);
934 if (issue_flags & IO_URING_F_MULTISHOT)
935 return IOU_STOP_MULTISHOT;
936 return IOU_OK;
857}937}
858 938
859int io_write(struct io_kiocb *req, unsigned int issue_flags)939int io_write(struct io_kiocb *req, unsigned int issue_flags)
io_uring/rw.h+2−0
@@ -23,3 +23,5 @@ int io_writev_prep_async(struct io_kiocb *req); · 22 unmodified lines
23void io_readv_writev_cleanup(struct io_kiocb *req);23void io_readv_writev_cleanup(struct io_kiocb *req);
24void io_rw_fail(struct io_kiocb *req);24void io_rw_fail(struct io_kiocb *req);
25void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts);25void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts);
26int io_read_mshot_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
27int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags);
io_uring/uring_cmd.c+48−1
@@ -13,6 +13,51 @@ · 12 unmodified lines
13#include "rsrc.h"13#include "rsrc.h"
14#include "uring_cmd.h"14#include "uring_cmd.h"
15 15
16static void io_uring_cmd_del_cancelable(struct io_uring_cmd *cmd,
17 unsigned int issue_flags)
18{
19 struct io_kiocb *req = cmd_to_io_kiocb(cmd);
20 struct io_ring_ctx *ctx = req->ctx;
21
22 if (!(cmd->flags & IORING_URING_CMD_CANCELABLE))
23 return;
24
25 cmd->flags &= ~IORING_URING_CMD_CANCELABLE;
26 io_ring_submit_lock(ctx, issue_flags);
27 hlist_del(&req->hash_node);
28 io_ring_submit_unlock(ctx, issue_flags);
29}
30
31/*
32 * Mark this command as concelable, then io_uring_try_cancel_uring_cmd()
33 * will try to cancel this issued command by sending ->uring_cmd() with
34 * issue_flags of IO_URING_F_CANCEL.
35 *
36 * The command is guaranteed to not be done when calling ->uring_cmd()
37 * with IO_URING_F_CANCEL, but it is driver's responsibility to deal
38 * with race between io_uring canceling and normal completion.
39 */
40void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
41 unsigned int issue_flags)
42{
43 struct io_kiocb *req = cmd_to_io_kiocb(cmd);
44 struct io_ring_ctx *ctx = req->ctx;
45
46 if (!(cmd->flags & IORING_URING_CMD_CANCELABLE)) {
47 cmd->flags |= IORING_URING_CMD_CANCELABLE;
48 io_ring_submit_lock(ctx, issue_flags);
49 hlist_add_head(&req->hash_node, &ctx->cancelable_uring_cmd);
50 io_ring_submit_unlock(ctx, issue_flags);
51 }
52}
53EXPORT_SYMBOL_GPL(io_uring_cmd_mark_cancelable);
54
55struct task_struct *io_uring_cmd_get_task(struct io_uring_cmd *cmd)
56{
57 return cmd_to_io_kiocb(cmd)->task;
58}
59EXPORT_SYMBOL_GPL(io_uring_cmd_get_task);
60
16static void io_uring_cmd_work(struct io_kiocb *req, struct io_tw_state *ts)61static void io_uring_cmd_work(struct io_kiocb *req, struct io_tw_state *ts)
17{62{
18 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);63 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
@@ -56,6 +101,8 @@ void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2, · 37 unmodified lines
56{101{
57 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);102 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
58 103
104 io_uring_cmd_del_cancelable(ioucmd, issue_flags);
105
59 if (ret < 0)106 if (ret < 0)
60 req_set_fail(req);107 req_set_fail(req);
61 108
@@ -91,7 +138,7 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 29 unmodified lines
91 return -EINVAL;138 return -EINVAL;
92 139
93 ioucmd->flags = READ_ONCE(sqe->uring_cmd_flags);140 ioucmd->flags = READ_ONCE(sqe->uring_cmd_flags);
94 if (ioucmd->flags & ~IORING_URING_CMD_FIXED)141 if (ioucmd->flags & ~IORING_URING_CMD_MASK)
95 return -EINVAL;142 return -EINVAL;
96 143
97 if (ioucmd->flags & IORING_URING_CMD_FIXED) {144 if (ioucmd->flags & IORING_URING_CMD_FIXED) {
io_uring/waitid.c+372−0
@@ -0,0 +1,372 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Support for async notification of waitid
4 */
5#include <linux/kernel.h>
6#include <linux/errno.h>
7#include <linux/fs.h>
8#include <linux/file.h>
9#include <linux/compat.h>
10#include <linux/io_uring.h>
11
12#include <uapi/linux/io_uring.h>
13
14#include "io_uring.h"
15#include "cancel.h"
16#include "waitid.h"
17#include "../kernel/exit.h"
18
19static void io_waitid_cb(struct io_kiocb *req, struct io_tw_state *ts);
20
21#define IO_WAITID_CANCEL_FLAG BIT(31)
22#define IO_WAITID_REF_MASK GENMASK(30, 0)
23
24struct io_waitid {
25 struct file *file;
26 int which;
27 pid_t upid;
28 int options;
29 atomic_t refs;
30 struct wait_queue_head *head;
31 struct siginfo __user *infop;
32 struct waitid_info info;
33};
34
35static void io_waitid_free(struct io_kiocb *req)
36{
37 struct io_waitid_async *iwa = req->async_data;
38
39 put_pid(iwa->wo.wo_pid);
40 kfree(req->async_data);
41 req->async_data = NULL;
42 req->flags &= ~REQ_F_ASYNC_DATA;
43}
44
45#ifdef CONFIG_COMPAT
46static bool io_waitid_compat_copy_si(struct io_waitid *iw, int signo)
47{
48 struct compat_siginfo __user *infop;
49 bool ret;
50
51 infop = (struct compat_siginfo __user *) iw->infop;
52
53 if (!user_write_access_begin(infop, sizeof(*infop)))
54 return false;
55
56 unsafe_put_user(signo, &infop->si_signo, Efault);
57 unsafe_put_user(0, &infop->si_errno, Efault);
58 unsafe_put_user(iw->info.cause, &infop->si_code, Efault);
59 unsafe_put_user(iw->info.pid, &infop->si_pid, Efault);
60 unsafe_put_user(iw->info.uid, &infop->si_uid, Efault);
61 unsafe_put_user(iw->info.status, &infop->si_status, Efault);
62 ret = true;
63done:
64 user_write_access_end();
65 return ret;
66Efault:
67 ret = false;
68 goto done;
69}
70#endif
71
72static bool io_waitid_copy_si(struct io_kiocb *req, int signo)
73{
74 struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
75 bool ret;
76
77 if (!iw->infop)
78 return true;
79
80#ifdef CONFIG_COMPAT
81 if (req->ctx->compat)
82 return io_waitid_compat_copy_si(iw, signo);
83#endif
84
85 if (!user_write_access_begin(iw->infop, sizeof(*iw->infop)))
86 return false;
87
88 unsafe_put_user(signo, &iw->infop->si_signo, Efault);
89 unsafe_put_user(0, &iw->infop->si_errno, Efault);
90 unsafe_put_user(iw->info.cause, &iw->infop->si_code, Efault);
91 unsafe_put_user(iw->info.pid, &iw->infop->si_pid, Efault);
92 unsafe_put_user(iw->info.uid, &iw->infop->si_uid, Efault);
93 unsafe_put_user(iw->info.status, &iw->infop->si_status, Efault);
94 ret = true;
95done:
96 user_write_access_end();
97 return ret;
98Efault:
99 ret = false;
100 goto done;
101}
102
103static int io_waitid_finish(struct io_kiocb *req, int ret)
104{
105 int signo = 0;
106
107 if (ret > 0) {
108 signo = SIGCHLD;
109 ret = 0;
110 }
111
112 if (!io_waitid_copy_si(req, signo))
113 ret = -EFAULT;
114 io_waitid_free(req);
115 return ret;
116}
117
118static void io_waitid_complete(struct io_kiocb *req, int ret)
119{
120 struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
121 struct io_tw_state ts = { .locked = true };
122
123 /* anyone completing better be holding a reference */
124 WARN_ON_ONCE(!(atomic_read(&iw->refs) & IO_WAITID_REF_MASK));
125
126 lockdep_assert_held(&req->ctx->uring_lock);
127
128 /*
129 * Did cancel find it meanwhile?
130 */
131 if (hlist_unhashed(&req->hash_node))
132 return;
133
134 hlist_del_init(&req->hash_node);
135
136 ret = io_waitid_finish(req, ret);
137 if (ret < 0)
138 req_set_fail(req);
139 io_req_set_res(req, ret, 0);
140 io_req_task_complete(req, &ts);
141}
142
143static bool __io_waitid_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req)
144{
145 struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
146 struct io_waitid_async *iwa = req->async_data;
147
148 /*
149 * Mark us canceled regardless of ownership. This will prevent a
150 * potential retry from a spurious wakeup.
151 */
152 atomic_or(IO_WAITID_CANCEL_FLAG, &iw->refs);
153
154 /* claim ownership */
155 if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK)
156 return false;
157
158 spin_lock_irq(&iw->head->lock);
159 list_del_init(&iwa->wo.child_wait.entry);
160 spin_unlock_irq(&iw->head->lock);
161 io_waitid_complete(req, -ECANCELED);
162 return true;
163}
164
165int io_waitid_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
166 unsigned int issue_flags)
167{
168 struct hlist_node *tmp;
169 struct io_kiocb *req;
170 int nr = 0;
171
172 if (cd->flags & (IORING_ASYNC_CANCEL_FD|IORING_ASYNC_CANCEL_FD_FIXED))
173 return -ENOENT;
174
175 io_ring_submit_lock(ctx, issue_flags);
176 hlist_for_each_entry_safe(req, tmp, &ctx->waitid_list, hash_node) {
177 if (req->cqe.user_data != cd->data &&
178 !(cd->flags & IORING_ASYNC_CANCEL_ANY))
179 continue;
180 if (__io_waitid_cancel(ctx, req))
181 nr++;
182 if (!(cd->flags & IORING_ASYNC_CANCEL_ALL))
183 break;
184 }
185 io_ring_submit_unlock(ctx, issue_flags);
186
187 if (nr)
188 return nr;
189
190 return -ENOENT;
191}
192
193bool io_waitid_remove_all(struct io_ring_ctx *ctx, struct task_struct *task,
194 bool cancel_all)
195{
196 struct hlist_node *tmp;
197 struct io_kiocb *req;
198 bool found = false;
199
200 lockdep_assert_held(&ctx->uring_lock);
201
202 hlist_for_each_entry_safe(req, tmp, &ctx->waitid_list, hash_node) {
203 if (!io_match_task_safe(req, task, cancel_all))
204 continue;
205 __io_waitid_cancel(ctx, req);
206 found = true;
207 }
208
209 return found;
210}
211
212static inline bool io_waitid_drop_issue_ref(struct io_kiocb *req)
213{
214 struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
215 struct io_waitid_async *iwa = req->async_data;
216
217 if (!atomic_sub_return(1, &iw->refs))
218 return false;
219
220 /*
221 * Wakeup triggered, racing with us. It was prevented from
222 * completing because of that, queue up the tw to do that.
223 */
224 req->io_task_work.func = io_waitid_cb;
225 io_req_task_work_add(req);
226 remove_wait_queue(iw->head, &iwa->wo.child_wait);
227 return true;
228}
229
230static void io_waitid_cb(struct io_kiocb *req, struct io_tw_state *ts)
231{
232 struct io_waitid_async *iwa = req->async_data;
233 struct io_ring_ctx *ctx = req->ctx;
234 int ret;
235
236 io_tw_lock(ctx, ts);
237
238 ret = __do_wait(&iwa->wo);
239
240 /*
241 * If we get -ERESTARTSYS here, we need to re-arm and check again
242 * to ensure we get another callback. If the retry works, then we can
243 * just remove ourselves from the waitqueue again and finish the
244 * request.
245 */
246 if (unlikely(ret == -ERESTARTSYS)) {
247 struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
248
249 /* Don't retry if cancel found it meanwhile */
250 ret = -ECANCELED;
251 if (!(atomic_read(&iw->refs) & IO_WAITID_CANCEL_FLAG)) {
252 iw->head = &current->signal->wait_chldexit;
253 add_wait_queue(iw->head, &iwa->wo.child_wait);
254 ret = __do_wait(&iwa->wo);
255 if (ret == -ERESTARTSYS) {
256 /* retry armed, drop our ref */
257 io_waitid_drop_issue_ref(req);
258 return;
259 }
260
261 remove_wait_queue(iw->head, &iwa->wo.child_wait);
262 }
263 }
264
265 io_waitid_complete(req, ret);
266}
267
268static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode,
269 int sync, void *key)
270{
271 struct wait_opts *wo = container_of(wait, struct wait_opts, child_wait);
272 struct io_waitid_async *iwa = container_of(wo, struct io_waitid_async, wo);
273 struct io_kiocb *req = iwa->req;
274 struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
275 struct task_struct *p = key;
276
277 if (!pid_child_should_wake(wo, p))
278 return 0;
279
280 /* cancel is in progress */
281 if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK)
282 return 1;
283
284 req->io_task_work.func = io_waitid_cb;
285 io_req_task_work_add(req);
286 list_del_init(&wait->entry);
287 return 1;
288}
289
290int io_waitid_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
291{
292 struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
293
294 if (sqe->addr || sqe->buf_index || sqe->addr3 || sqe->waitid_flags)
295 return -EINVAL;
296
297 iw->which = READ_ONCE(sqe->len);
298 iw->upid = READ_ONCE(sqe->fd);
299 iw->options = READ_ONCE(sqe->file_index);
300 iw->infop = u64_to_user_ptr(READ_ONCE(sqe->addr2));
301 return 0;
302}
303
304int io_waitid(struct io_kiocb *req, unsigned int issue_flags)
305{
306 struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
307 struct io_ring_ctx *ctx = req->ctx;
308 struct io_waitid_async *iwa;
309 int ret;
310
311 if (io_alloc_async_data(req))
312 return -ENOMEM;
313
314 iwa = req->async_data;
315 iwa->req = req;
316
317 ret = kernel_waitid_prepare(&iwa->wo, iw->which, iw->upid, &iw->info,
318 iw->options, NULL);
319 if (ret)
320 goto done;
321
322 /*
323 * Mark the request as busy upfront, in case we're racing with the
324 * wakeup. If we are, then we'll notice when we drop this initial
325 * reference again after arming.
326 */
327 atomic_set(&iw->refs, 1);
328
329 /*
330 * Cancel must hold the ctx lock, so there's no risk of cancelation
331 * finding us until a) we remain on the list, and b) the lock is
332 * dropped. We only need to worry about racing with the wakeup
333 * callback.
334 */
335 io_ring_submit_lock(ctx, issue_flags);
336 hlist_add_head(&req->hash_node, &ctx->waitid_list);
337
338 init_waitqueue_func_entry(&iwa->wo.child_wait, io_waitid_wait);
339 iwa->wo.child_wait.private = req->task;
340 iw->head = &current->signal->wait_chldexit;
341 add_wait_queue(iw->head, &iwa->wo.child_wait);
342
343 ret = __do_wait(&iwa->wo);
344 if (ret == -ERESTARTSYS) {
345 /*
346 * Nobody else grabbed a reference, it'll complete when we get
347 * a waitqueue callback, or if someone cancels it.
348 */
349 if (!io_waitid_drop_issue_ref(req)) {
350 io_ring_submit_unlock(ctx, issue_flags);
351 return IOU_ISSUE_SKIP_COMPLETE;
352 }
353
354 /*
355 * Wakeup triggered, racing with us. It was prevented from
356 * completing because of that, queue up the tw to do that.
357 */
358 io_ring_submit_unlock(ctx, issue_flags);
359 return IOU_ISSUE_SKIP_COMPLETE;
360 }
361
362 hlist_del_init(&req->hash_node);
363 remove_wait_queue(iw->head, &iwa->wo.child_wait);
364 ret = io_waitid_finish(req, ret);
365
366 io_ring_submit_unlock(ctx, issue_flags);
367done:
368 if (ret < 0)
369 req_set_fail(req);
370 io_req_set_res(req, ret, 0);
371 return IOU_OK;
372}
io_uring/waitid.h+15−0
@@ -0,0 +1,15 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include "../kernel/exit.h"
4
5struct io_waitid_async {
6 struct io_kiocb *req;
7 struct wait_opts wo;
8};
9
10int io_waitid_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
11int io_waitid(struct io_kiocb *req, unsigned int issue_flags);
12int io_waitid_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
13 unsigned int issue_flags);
14bool io_waitid_remove_all(struct io_ring_ctx *ctx, struct task_struct *task,
15 bool cancel_all);
kernel/exit.c+72−59
@@ -74,6 +74,8 @@ · 73 unmodified lines
74#include <asm/unistd.h>74#include <asm/unistd.h>
75#include <asm/mmu_context.h>75#include <asm/mmu_context.h>
76 76
77#include "exit.h"
78
77/*79/*
78 * The default value should be high enough to not crash a system that randomly80 * The default value should be high enough to not crash a system that randomly
79 * crashes its kernel from time to time, but low enough to at least not permit81 * crashes its kernel from time to time, but low enough to at least not permit
@@ -1037,26 +1039,6 @@ SYSCALL_DEFINE1(exit_group, int, error_code) · 957 unmodified lines
1037 return 0;1039 return 0;
1038}1040}
1039 1041
1040struct waitid_info {
1041 pid_t pid;
1042 uid_t uid;
1043 int status;
1044 int cause;
1045};
1046
1047struct wait_opts {
1048 enum pid_type wo_type;
1049 int wo_flags;
1050 struct pid *wo_pid;
1051
1052 struct waitid_info *wo_info;
1053 int wo_stat;
1054 struct rusage *wo_rusage;
1055
1056 wait_queue_entry_t child_wait;
1057 int notask_error;
1058};
1059
1060static int eligible_pid(struct wait_opts *wo, struct task_struct *p)1042static int eligible_pid(struct wait_opts *wo, struct task_struct *p)
1061{1043{
1062 return wo->wo_type == PIDTYPE_MAX ||1044 return wo->wo_type == PIDTYPE_MAX ||
@@ -1520,6 +1502,17 @@ static int ptrace_do_wait(struct wait_opts *wo, struct task_struct *tsk) · 457 unmodified lines
1520 return 0;1502 return 0;
1521}1503}
1522 1504
1505bool pid_child_should_wake(struct wait_opts *wo, struct task_struct *p)
1506{
1507 if (!eligible_pid(wo, p))
1508 return false;
1509
1510 if ((wo->wo_flags & __WNOTHREAD) && wo->child_wait.private != p->parent)
1511 return false;
1512
1513 return true;
1514}
1515
1523static int child_wait_callback(wait_queue_entry_t *wait, unsigned mode,1516static int child_wait_callback(wait_queue_entry_t *wait, unsigned mode,
1524 int sync, void *key)1517 int sync, void *key)
1525{1518{
@@ -1527,13 +1520,10 @@ static int child_wait_callback(wait_queue_entry_t *wait, unsigned mode, · 1 unmodified line
1527 child_wait);1520 child_wait);
1528 struct task_struct *p = key;1521 struct task_struct *p = key;
1529 1522
1530 if (!eligible_pid(wo, p))1523 if (pid_child_should_wake(wo, p))
1531 return 0;1524 return default_wake_function(wait, mode, sync, key);
1532 1525
1533 if ((wo->wo_flags & __WNOTHREAD) && wait->private != p->parent)1526 return 0;
1534 return 0;
1535
1536 return default_wake_function(wait, mode, sync, key);
1537}1527}
1538 1528
1539void __wake_up_parent(struct task_struct *p, struct task_struct *parent)1529void __wake_up_parent(struct task_struct *p, struct task_struct *parent)
@@ -1582,16 +1572,10 @@ static int do_wait_pid(struct wait_opts *wo) · 42 unmodified lines
1582 return 0;1572 return 0;
1583}1573}
1584 1574
1585static long do_wait(struct wait_opts *wo)1575long __do_wait(struct wait_opts *wo)
1586{1576{
1587 int retval;1577 long retval;
1588 1578
1589 trace_sched_process_wait(wo->wo_pid);
1590
1591 init_waitqueue_func_entry(&wo->child_wait, child_wait_callback);
1592 wo->child_wait.private = current;
1593 add_wait_queue(&current->signal->wait_chldexit, &wo->child_wait);
1594repeat:
1595 /*1579 /*
1596 * If there is nothing that can match our criteria, just get out.1580 * If there is nothing that can match our criteria, just get out.
1597 * We will clear ->notask_error to zero if we see any child that1581 * We will clear ->notask_error to zero if we see any child that
@@ -1603,24 +1587,23 @@ repeat: · 5 unmodified lines
1603 (!wo->wo_pid || !pid_has_task(wo->wo_pid, wo->wo_type)))1587 (!wo->wo_pid || !pid_has_task(wo->wo_pid, wo->wo_type)))
1604 goto notask;1588 goto notask;
1605 1589
1606 set_current_state(TASK_INTERRUPTIBLE);
1607 read_lock(&tasklist_lock);1590 read_lock(&tasklist_lock);
1608 1591
1609 if (wo->wo_type == PIDTYPE_PID) {1592 if (wo->wo_type == PIDTYPE_PID) {
1610 retval = do_wait_pid(wo);1593 retval = do_wait_pid(wo);
1611 if (retval)1594 if (retval)
1612 goto end;1595 return retval;
1613 } else {1596 } else {
1614 struct task_struct *tsk = current;1597 struct task_struct *tsk = current;
1615 1598
1616 do {1599 do {
1617 retval = do_wait_thread(wo, tsk);1600 retval = do_wait_thread(wo, tsk);
1618 if (retval)1601 if (retval)
1619 goto end;1602 return retval;
1620 1603
1621 retval = ptrace_do_wait(wo, tsk);1604 retval = ptrace_do_wait(wo, tsk);
1622 if (retval)1605 if (retval)
1623 goto end;1606 return retval;
1624 1607
1625 if (wo->wo_flags & __WNOTHREAD)1608 if (wo->wo_flags & __WNOTHREAD)
1626 break;1609 break;
@@ -1630,27 +1613,44 @@ repeat: · 3 unmodified lines
1630 1613
1631notask:1614notask:
1632 retval = wo->notask_error;1615 retval = wo->notask_error;
1633 if (!retval && !(wo->wo_flags & WNOHANG)) {1616 if (!retval && !(wo->wo_flags & WNOHANG))
1634 retval = -ERESTARTSYS;1617 return -ERESTARTSYS;
1635 if (!signal_pending(current)) {1618
1636 schedule();1619 return retval;
1637 goto repeat;1620}
1638 }1621
1639 }1622static long do_wait(struct wait_opts *wo)
1640end:1623{
1624 int retval;
1625
1626 trace_sched_process_wait(wo->wo_pid);
1627
1628 init_waitqueue_func_entry(&wo->child_wait, child_wait_callback);
1629 wo->child_wait.private = current;
1630 add_wait_queue(&current->signal->wait_chldexit, &wo->child_wait);
1631
1632 do {
1633 set_current_state(TASK_INTERRUPTIBLE);
1634 retval = __do_wait(wo);
1635 if (retval != -ERESTARTSYS)
1636 break;
1637 if (signal_pending(current))
1638 break;
1639 schedule();
1640 } while (1);
1641
1641 __set_current_state(TASK_RUNNING);1642 __set_current_state(TASK_RUNNING);
1642 remove_wait_queue(&current->signal->wait_chldexit, &wo->child_wait);1643 remove_wait_queue(&current->signal->wait_chldexit, &wo->child_wait);
1643 return retval;1644 return retval;
1644}1645}
1645 1646
1646static long kernel_waitid(int which, pid_t upid, struct waitid_info *infop,1647int kernel_waitid_prepare(struct wait_opts *wo, int which, pid_t upid,
1647 int options, struct rusage *ru)1648 struct waitid_info *infop, int options,
1649 struct rusage *ru)
1648{1650{
1649 struct wait_opts wo;1651 unsigned int f_flags = 0;
1650 struct pid *pid = NULL;1652 struct pid *pid = NULL;
1651 enum pid_type type;1653 enum pid_type type;
1652 long ret;
1653 unsigned int f_flags = 0;
1654 1654
1655 if (options & ~(WNOHANG|WNOWAIT|WEXITED|WSTOPPED|WCONTINUED|1655 if (options & ~(WNOHANG|WNOWAIT|WEXITED|WSTOPPED|WCONTINUED|
1656 __WNOTHREAD|__WCLONE|__WALL))1656 __WNOTHREAD|__WCLONE|__WALL))
@@ -1693,19 +1693,32 @@ static long kernel_waitid(int which, pid_t upid, struct waitid_info *infop, · 36 unmodified lines
1693 return -EINVAL;1693 return -EINVAL;
1694 }1694 }
1695 1695
1696 wo.wo_type = type;1696 wo->wo_type = type;
1697 wo.wo_pid = pid;1697 wo->wo_pid = pid;
1698 wo.wo_flags = options;1698 wo->wo_flags = options;
1699 wo.wo_info = infop;1699 wo->wo_info = infop;
1700 wo.wo_rusage = ru;1700 wo->wo_rusage = ru;
1701 if (f_flags & O_NONBLOCK)1701 if (f_flags & O_NONBLOCK)
1702 wo.wo_flags |= WNOHANG;1702 wo->wo_flags |= WNOHANG;
1703
1704 return 0;
1705}
1706
1707static long kernel_waitid(int which, pid_t upid, struct waitid_info *infop,
1708 int options, struct rusage *ru)
1709{
1710 struct wait_opts wo;
1711 long ret;
1712
1713 ret = kernel_waitid_prepare(&wo, which, upid, infop, options, ru);
1714 if (ret)
1715 return ret;
1703 1716
1704 ret = do_wait(&wo);1717 ret = do_wait(&wo);
1705 if (!ret && !(options & WNOHANG) && (f_flags & O_NONBLOCK))1718 if (!ret && !(options & WNOHANG) && (wo.wo_flags & WNOHANG))
1706 ret = -EAGAIN;1719 ret = -EAGAIN;
1707 1720
1708 put_pid(pid);1721 put_pid(wo.wo_pid);
1709 return ret;1722 return ret;
1710}1723}
1711 1724
kernel/exit.h+30−0
@@ -0,0 +1,30 @@
1// SPDX-License-Identifier: GPL-2.0-only
2#ifndef LINUX_WAITID_H
3#define LINUX_WAITID_H
4
5struct waitid_info {
6 pid_t pid;
7 uid_t uid;
8 int status;
9 int cause;
10};
11
12struct wait_opts {
13 enum pid_type wo_type;
14 int wo_flags;
15 struct pid *wo_pid;
16
17 struct waitid_info *wo_info;
18 int wo_stat;
19 struct rusage *wo_rusage;
20
21 wait_queue_entry_t child_wait;
22 int notask_error;
23};
24
25bool pid_child_should_wake(struct wait_opts *wo, struct task_struct *p);
26long __do_wait(struct wait_opts *wo);
27int kernel_waitid_prepare(struct wait_opts *wo, int which, pid_t upid,
28 struct waitid_info *infop, int options,
29 struct rusage *ru);
30#endif