History

Merge tag 'for-5.19/io_uring-2022-05-22' of git://git.kernel.dk/linux-block

Pull io_uring updates from Jens Axboe: "Here are the main io_uring changes for 5.19. This contains: - Fixes for sparse type warnings (Christoph, Vasily) - Support for multi-shot accept (Hao) - Support for io_uring managed fixed files, rather than always needing the applicationt o manage the indices (me) - Fix for a spurious poll wakeup (Dylan) - CQE overflow fixes (Dylan) - Support more types of cancelations (me) - Support for co-operative task_work signaling, rather than always forcing an IPI (me) - Support for doing poll first when appropriate, rather than always attempting a transfer first (me) - Provided buffer cleanups and support for mapped buffers (me) - Improve how io_uring handles inflight SCM files (Pavel) - Speedups for registered files (Pavel, me) - Organize the completion data in a struct in io_kiocb rather than keep it in separate spots (Pavel) - task_work improvements (Pavel) - Cleanup and optimize the submission path, in general and for handling links (Pavel) - Speedups for registered resource handling (Pavel) - Support sparse buffers and file maps (Pavel, me) - Various fixes and cleanups (Almog, Pavel, me)" * tag 'for-5.19/io_uring-2022-05-22' of git://git.kernel.dk/linux-block: (111 commits) io_uring: fix incorrect __kernel_rwf_t cast io_uring: disallow mixed provided buffer group registrations io_uring: initialize io_buffer_list head when shared ring is unregistered io_uring: add fully sparse buffer registration io_uring: use rcu_dereference in io_close io_uring: consistently use the EPOLL* defines io_uring: make apoll_events a __poll_t io_uring: drop a spurious inline on a forward declaration io_uring: don't use ERR_PTR for user pointers io_uring: use a rwf_t for io_rw.flags io_uring: add support for ring mapped supplied buffers io_uring: add io_pin_pages() helper io_uring: add buffer selection support to IORING_OP_NOP io_uring: fix locking state for empty buffer group io_uring: implement multishot mode for accept io_uring: let fast poll support multishot io_uring: add REQ_F_APOLL_MULTISHOT for requests io_uring: add IORING_ACCEPT_MULTISHOT for accept io_uring: only wake when the correct events are set io_uring: avoid io-wq -EAGAIN looping for !IOPOLL ...

Linus Torvalds committed 4 years ago

Commit 3a166bd2 parents 1e57930 0e7579c8 files · +1813 −1031Browse files

Unified · Split

8 files
  1. fs/io-wq.c +2 −2
  2. fs/io-wq.h +1 −0
  3. fs/io_uring.c +1643 −1018
  4. include/linux/sched/signal.h +11 −2
  5. include/linux/task_work.h +1 −0
  6. include/trace/events/io_uring.h +42 −2
  7. include/uapi/linux/io_uring.h +94 −1
  8. kernel/task_work.c +19 −6
fs/io-wq.c+2−2
@@ -871,7 +871,7 @@ static bool io_wq_for_each_worker(struct io_wqe *wqe, · 870 unmodified lines
871 871
872static bool io_wq_worker_wake(struct io_worker *worker, void *data)872static bool io_wq_worker_wake(struct io_worker *worker, void *data)
873{873{
874 set_notify_signal(worker->task);874 __set_notify_signal(worker->task);
875 wake_up_process(worker->task);875 wake_up_process(worker->task);
876 return false;876 return false;
877}877}
@@ -991,7 +991,7 @@ static bool __io_wq_worker_cancel(struct io_worker *worker, · 113 unmodified lines
991{991{
992 if (work && match->fn(work, match->data)) {992 if (work && match->fn(work, match->data)) {
993 work->flags |= IO_WQ_WORK_CANCEL;993 work->flags |= IO_WQ_WORK_CANCEL;
994 set_notify_signal(worker->task);994 __set_notify_signal(worker->task);
995 return true;995 return true;
996 }996 }
997 997
fs/io-wq.h+1−0
@@ -155,6 +155,7 @@ struct io_wq_work_node *wq_stack_extract(struct io_wq_work_node *stack) · 154 unmodified lines
155struct io_wq_work {155struct io_wq_work {
156 struct io_wq_work_node list;156 struct io_wq_work_node list;
157 unsigned flags;157 unsigned flags;
158 int cancel_seq;
158};159};
159 160
160static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)161static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)
fs/io_uring.c+1643−1018
@@ -94,7 +94,7 @@ · 93 unmodified lines
94#define IORING_SQPOLL_CAP_ENTRIES_VALUE 894#define IORING_SQPOLL_CAP_ENTRIES_VALUE 8
95 95
96/* only define max */96/* only define max */
97#define IORING_MAX_FIXED_FILES (1U << 15)97#define IORING_MAX_FIXED_FILES (1U << 20)
98#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \98#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
99 IORING_REGISTER_LAST + IORING_OP_LAST)99 IORING_REGISTER_LAST + IORING_OP_LAST)
100 100
@@ -113,6 +113,11 @@ · 12 unmodified lines
113#define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \113#define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \
114 REQ_F_POLLED | REQ_F_CREDS | REQ_F_ASYNC_DATA)114 REQ_F_POLLED | REQ_F_CREDS | REQ_F_ASYNC_DATA)
115 115
116#define IO_REQ_CLEAN_SLOW_FLAGS (REQ_F_REFCOUNT | REQ_F_LINK | REQ_F_HARDLINK |\
117 IO_REQ_CLEAN_FLAGS)
118
119#define IO_APOLL_MULTI_POLLED (REQ_F_APOLL_MULTISHOT | REQ_F_POLLED)
120
116#define IO_TCTX_REFS_CACHE_NR (1U << 10)121#define IO_TCTX_REFS_CACHE_NR (1U << 10)
117 122
118struct io_uring {123struct io_uring {
@@ -166,7 +171,7 @@ struct io_rings { · 47 unmodified lines
166 * The application needs a full memory barrier before checking171 * The application needs a full memory barrier before checking
167 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.172 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
168 */173 */
169 u32 sq_flags;174 atomic_t sq_flags;
170 /*175 /*
171 * Runtime CQ flags176 * Runtime CQ flags
172 *177 *
@@ -220,6 +225,23 @@ struct io_overflow_cqe { · 47 unmodified lines
220 struct list_head list;225 struct list_head list;
221};226};
222 227
228/*
229 * FFS_SCM is only available on 64-bit archs, for 32-bit we just define it as 0
230 * and define IO_URING_SCM_ALL. For this case, we use SCM for all files as we
231 * can't safely always dereference the file when the task has exited and ring
232 * cleanup is done. If a file is tracked and part of SCM, then unix gc on
233 * process exit may reap it before __io_sqe_files_unregister() is run.
234 */
235#define FFS_NOWAIT 0x1UL
236#define FFS_ISREG 0x2UL
237#if defined(CONFIG_64BIT)
238#define FFS_SCM 0x4UL
239#else
240#define IO_URING_SCM_ALL
241#define FFS_SCM 0x0UL
242#endif
243#define FFS_MASK ~(FFS_NOWAIT|FFS_ISREG|FFS_SCM)
244
223struct io_fixed_file {245struct io_fixed_file {
224 /* file * with additional FFS_* flags */246 /* file * with additional FFS_* flags */
225 unsigned long file_ptr;247 unsigned long file_ptr;
@@ -237,6 +259,8 @@ struct io_rsrc_put { · 11 unmodified lines
237 259
238struct io_file_table {260struct io_file_table {
239 struct io_fixed_file *files;261 struct io_fixed_file *files;
262 unsigned long *bitmap;
263 unsigned int alloc_hint;
240};264};
241 265
242struct io_rsrc_node {266struct io_rsrc_node {
@@ -261,10 +285,26 @@ struct io_rsrc_data { · 18 unmodified lines
261 bool quiesce;285 bool quiesce;
262};286};
263 287
288#define IO_BUFFER_LIST_BUF_PER_PAGE (PAGE_SIZE / sizeof(struct io_uring_buf))
264struct io_buffer_list {289struct io_buffer_list {
265 struct list_head list;290 /*
266 struct list_head buf_list;291 * If ->buf_nr_pages is set, then buf_pages/buf_ring are used. If not,
292 * then these are classic provided buffers and ->buf_list is used.
293 */
294 union {
295 struct list_head buf_list;
296 struct {
297 struct page **buf_pages;
298 struct io_uring_buf_ring *buf_ring;
299 };
300 };
267 __u16 bgid;301 __u16 bgid;
302
303 /* below is for ring provided buffers */
304 __u16 buf_nr_pages;
305 __u16 nr_entries;
306 __u32 head;
307 __u32 mask;
268};308};
269 309
270struct io_buffer {310struct io_buffer {
@@ -337,7 +377,7 @@ struct io_ev_fd { · 66 unmodified lines
337 struct rcu_head rcu;377 struct rcu_head rcu;
338};378};
339 379
340#define IO_BUFFERS_HASH_BITS 5380#define BGID_ARRAY 64
341 381
342struct io_ring_ctx {382struct io_ring_ctx {
343 /* const or read-mostly hot data */383 /* const or read-mostly hot data */
@@ -346,6 +386,7 @@ struct io_ring_ctx { · 2 unmodified lines
346 386
347 struct io_rings *rings;387 struct io_rings *rings;
348 unsigned int flags;388 unsigned int flags;
389 enum task_work_notify_mode notify_method;
349 unsigned int compat: 1;390 unsigned int compat: 1;
350 unsigned int drain_next: 1;391 unsigned int drain_next: 1;
351 unsigned int restricted: 1;392 unsigned int restricted: 1;
@@ -353,6 +394,7 @@ struct io_ring_ctx { · 1 unmodified line
353 unsigned int drain_active: 1;394 unsigned int drain_active: 1;
354 unsigned int drain_disabled: 1;395 unsigned int drain_disabled: 1;
355 unsigned int has_evfd: 1;396 unsigned int has_evfd: 1;
397 unsigned int syscall_iopoll: 1;
356 } ____cacheline_aligned_in_smp;398 } ____cacheline_aligned_in_smp;
357 399
358 /* submission data */400 /* submission data */
@@ -382,17 +424,21 @@ struct io_ring_ctx { · 23 unmodified lines
382 */424 */
383 struct io_rsrc_node *rsrc_node;425 struct io_rsrc_node *rsrc_node;
384 int rsrc_cached_refs;426 int rsrc_cached_refs;
427 atomic_t cancel_seq;
385 struct io_file_table file_table;428 struct io_file_table file_table;
386 unsigned nr_user_files;429 unsigned nr_user_files;
387 unsigned nr_user_bufs;430 unsigned nr_user_bufs;
388 struct io_mapped_ubuf **user_bufs;431 struct io_mapped_ubuf **user_bufs;
389 432
390 struct io_submit_state submit_state;433 struct io_submit_state submit_state;
434
435 struct io_buffer_list *io_bl;
436 struct xarray io_bl_xa;
437 struct list_head io_buffers_cache;
438
391 struct list_head timeout_list;439 struct list_head timeout_list;
392 struct list_head ltimeout_list;440 struct list_head ltimeout_list;
393 struct list_head cq_overflow_list;441 struct list_head cq_overflow_list;
394 struct list_head *io_buffers;
395 struct list_head io_buffers_cache;
396 struct list_head apoll_cache;442 struct list_head apoll_cache;
397 struct xarray personalities;443 struct xarray personalities;
398 u32 pers_next;444 u32 pers_next;
@@ -409,9 +455,16 @@ struct io_ring_ctx { · 10 unmodified lines
409 struct wait_queue_head sqo_sq_wait;455 struct wait_queue_head sqo_sq_wait;
410 struct list_head sqd_list;456 struct list_head sqd_list;
411 457
412 unsigned long check_cq_overflow;458 unsigned long check_cq;
413 459
414 struct {460 struct {
461 /*
462 * We cache a range of free CQEs we can use, once exhausted it
463 * should go through a slower range setup, see __io_get_cqe()
464 */
465 struct io_uring_cqe *cqe_cached;
466 struct io_uring_cqe *cqe_sentinel;
467
415 unsigned cached_cq_tail;468 unsigned cached_cq_tail;
416 unsigned cq_entries;469 unsigned cq_entries;
417 struct io_ev_fd __rcu *io_ev_fd;470 struct io_ev_fd __rcu *io_ev_fd;
@@ -557,6 +610,8 @@ struct io_sync { · 139 unmodified lines
557struct io_cancel {610struct io_cancel {
558 struct file *file;611 struct file *file;
559 u64 addr;612 u64 addr;
613 u32 flags;
614 s32 fd;
560};615};
561 616
562struct io_timeout {617struct io_timeout {
@@ -585,7 +640,7 @@ struct io_rw { · 22 unmodified lines
585 struct kiocb kiocb;640 struct kiocb kiocb;
586 u64 addr;641 u64 addr;
587 u32 len;642 u32 len;
588 u32 flags;643 rwf_t flags;
589};644};
590 645
591struct io_connect {646struct io_connect {
@@ -602,9 +657,9 @@ struct io_sr_msg { · 10 unmodified lines
602 void __user *buf;657 void __user *buf;
603 };658 };
604 int msg_flags;659 int msg_flags;
605 int bgid;
606 size_t len;660 size_t len;
607 size_t done_io;661 size_t done_io;
662 unsigned int flags;
608};663};
609 664
610struct io_open {665struct io_open {
@@ -766,6 +821,7 @@ enum { · 155 unmodified lines
766 REQ_F_NEED_CLEANUP_BIT,821 REQ_F_NEED_CLEANUP_BIT,
767 REQ_F_POLLED_BIT,822 REQ_F_POLLED_BIT,
768 REQ_F_BUFFER_SELECTED_BIT,823 REQ_F_BUFFER_SELECTED_BIT,
824 REQ_F_BUFFER_RING_BIT,
769 REQ_F_COMPLETE_INLINE_BIT,825 REQ_F_COMPLETE_INLINE_BIT,
770 REQ_F_REISSUE_BIT,826 REQ_F_REISSUE_BIT,
771 REQ_F_CREDS_BIT,827 REQ_F_CREDS_BIT,
@@ -776,6 +832,7 @@ enum { · 4 unmodified lines
776 REQ_F_SINGLE_POLL_BIT,832 REQ_F_SINGLE_POLL_BIT,
777 REQ_F_DOUBLE_POLL_BIT,833 REQ_F_DOUBLE_POLL_BIT,
778 REQ_F_PARTIAL_IO_BIT,834 REQ_F_PARTIAL_IO_BIT,
835 REQ_F_APOLL_MULTISHOT_BIT,
779 /* keep async read/write and isreg together and in order */836 /* keep async read/write and isreg together and in order */
780 REQ_F_SUPPORT_NOWAIT_BIT,837 REQ_F_SUPPORT_NOWAIT_BIT,
781 REQ_F_ISREG_BIT,838 REQ_F_ISREG_BIT,
@@ -816,6 +873,8 @@ enum { · 34 unmodified lines
816 REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),873 REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),
817 /* buffer already selected */874 /* buffer already selected */
818 REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),875 REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
876 /* buffer selected from ring, needs commit */
877 REQ_F_BUFFER_RING = BIT(REQ_F_BUFFER_RING_BIT),
819 /* completion is deferred through io_comp_state */878 /* completion is deferred through io_comp_state */
820 REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT),879 REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT),
821 /* caller should reissue async */880 /* caller should reissue async */
@@ -840,6 +899,8 @@ enum { · 18 unmodified lines
840 REQ_F_DOUBLE_POLL = BIT(REQ_F_DOUBLE_POLL_BIT),899 REQ_F_DOUBLE_POLL = BIT(REQ_F_DOUBLE_POLL_BIT),
841 /* request has already done partial IO */900 /* request has already done partial IO */
842 REQ_F_PARTIAL_IO = BIT(REQ_F_PARTIAL_IO_BIT),901 REQ_F_PARTIAL_IO = BIT(REQ_F_PARTIAL_IO_BIT),
902 /* fast poll multishot mode */
903 REQ_F_APOLL_MULTISHOT = BIT(REQ_F_APOLL_MULTISHOT_BIT),
843};904};
844 905
845struct async_poll {906struct async_poll {
@@ -862,6 +923,21 @@ enum { · 16 unmodified lines
862 IORING_RSRC_BUFFER = 1,923 IORING_RSRC_BUFFER = 1,
863};924};
864 925
926struct io_cqe {
927 __u64 user_data;
928 __s32 res;
929 /* fd initially, then cflags for completion */
930 union {
931 __u32 flags;
932 int fd;
933 };
934};
935
936enum {
937 IO_CHECK_CQ_OVERFLOW_BIT,
938 IO_CHECK_CQ_DROPPED_BIT,
939};
940
865/*941/*
866 * NOTE! Each of the iocb union members has the file pointer942 * NOTE! Each of the iocb union members has the file pointer
867 * as the first entry in their struct definition. So you can943 * as the first entry in their struct definition. So you can
@@ -902,29 +978,40 @@ struct io_kiocb { · 34 unmodified lines
902 u8 opcode;978 u8 opcode;
903 /* polled IO has completed */979 /* polled IO has completed */
904 u8 iopoll_completed;980 u8 iopoll_completed;
981 /*
982 * Can be either a fixed buffer index, or used with provided buffers.
983 * For the latter, before issue it points to the buffer group ID,
984 * and after selection it points to the buffer ID itself.
985 */
905 u16 buf_index;986 u16 buf_index;
906 unsigned int flags;987 unsigned int flags;
907 988
908 u64 user_data;989 struct io_cqe cqe;
909 u32 result;
910 /* fd initially, then cflags for completion */
911 union {
912 u32 cflags;
913 int fd;
914 };
915 990
916 struct io_ring_ctx *ctx;991 struct io_ring_ctx *ctx;
917 struct task_struct *task;992 struct task_struct *task;
918 993
919 struct percpu_ref *fixed_rsrc_refs;994 struct io_rsrc_node *rsrc_node;
920 /* store used ubuf, so we can prevent reloading */995
921 struct io_mapped_ubuf *imu;996 union {
997 /* store used ubuf, so we can prevent reloading */
998 struct io_mapped_ubuf *imu;
999
1000 /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */
1001 struct io_buffer *kbuf;
1002
1003 /*
1004 * stores buffer ID for ring provided buffers, valid IFF
1005 * REQ_F_BUFFER_RING is set.
1006 */
1007 struct io_buffer_list *buf_list;
1008 };
922 1009
923 union {1010 union {
924 /* used by request caches, completion batching and iopoll */1011 /* used by request caches, completion batching and iopoll */
925 struct io_wq_work_node comp_list;1012 struct io_wq_work_node comp_list;
926 /* cache ->apoll->events */1013 /* cache ->apoll->events */
927 int apoll_events;1014 __poll_t apoll_events;
928 };1015 };
929 atomic_t refs;1016 atomic_t refs;
930 atomic_t poll_refs;1017 atomic_t poll_refs;
@@ -935,8 +1022,6 @@ struct io_kiocb { · 4 unmodified lines
935 struct async_poll *apoll;1022 struct async_poll *apoll;
936 /* opcode allocated if it needs to store data for async defer */1023 /* opcode allocated if it needs to store data for async defer */
937 void *async_data;1024 void *async_data;
938 /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */
939 struct io_buffer *kbuf;
940 /* linked requests, IFF REQ_F_HARDLINK or REQ_F_LINK are set */1025 /* linked requests, IFF REQ_F_HARDLINK or REQ_F_LINK are set */
941 struct io_kiocb *link;1026 struct io_kiocb *link;
942 /* custom credentials, valid IFF REQ_F_CREDS is set */1027 /* custom credentials, valid IFF REQ_F_CREDS is set */
@@ -956,6 +1041,16 @@ struct io_defer_entry { · 13 unmodified lines
956 u32 seq;1041 u32 seq;
957};1042};
958 1043
1044struct io_cancel_data {
1045 struct io_ring_ctx *ctx;
1046 union {
1047 u64 data;
1048 struct file *file;
1049 };
1050 u32 flags;
1051 int seq;
1052};
1053
959struct io_op_def {1054struct io_op_def {
960 /* needs req->file assigned */1055 /* needs req->file assigned */
961 unsigned needs_file : 1;1056 unsigned needs_file : 1;
@@ -977,12 +1072,20 @@ struct io_op_def { · 15 unmodified lines
977 unsigned not_supported : 1;1072 unsigned not_supported : 1;
978 /* skip auditing */1073 /* skip auditing */
979 unsigned audit_skip : 1;1074 unsigned audit_skip : 1;
1075 /* supports ioprio */
1076 unsigned ioprio : 1;
1077 /* supports iopoll */
1078 unsigned iopoll : 1;
980 /* size of async data needed, if any */1079 /* size of async data needed, if any */
981 unsigned short async_size;1080 unsigned short async_size;
982};1081};
983 1082
984static const struct io_op_def io_op_defs[] = {1083static const struct io_op_def io_op_defs[] = {
985 [IORING_OP_NOP] = {},1084 [IORING_OP_NOP] = {
1085 .audit_skip = 1,
1086 .iopoll = 1,
1087 .buffer_select = 1,
1088 },
986 [IORING_OP_READV] = {1089 [IORING_OP_READV] = {
987 .needs_file = 1,1090 .needs_file = 1,
988 .unbound_nonreg_file = 1,1091 .unbound_nonreg_file = 1,
@@ -991,6 +1094,8 @@ static const struct io_op_def io_op_defs[] = { · 2 unmodified lines
991 .needs_async_setup = 1,1094 .needs_async_setup = 1,
992 .plug = 1,1095 .plug = 1,
993 .audit_skip = 1,1096 .audit_skip = 1,
1097 .ioprio = 1,
1098 .iopoll = 1,
994 .async_size = sizeof(struct io_async_rw),1099 .async_size = sizeof(struct io_async_rw),
995 },1100 },
996 [IORING_OP_WRITEV] = {1101 [IORING_OP_WRITEV] = {
@@ -1001,6 +1106,8 @@ static const struct io_op_def io_op_defs[] = { · 4 unmodified lines
1001 .needs_async_setup = 1,1106 .needs_async_setup = 1,
1002 .plug = 1,1107 .plug = 1,
1003 .audit_skip = 1,1108 .audit_skip = 1,
1109 .ioprio = 1,
1110 .iopoll = 1,
1004 .async_size = sizeof(struct io_async_rw),1111 .async_size = sizeof(struct io_async_rw),
1005 },1112 },
1006 [IORING_OP_FSYNC] = {1113 [IORING_OP_FSYNC] = {
@@ -1013,6 +1120,8 @@ static const struct io_op_def io_op_defs[] = { · 6 unmodified lines
1013 .pollin = 1,1120 .pollin = 1,
1014 .plug = 1,1121 .plug = 1,
1015 .audit_skip = 1,1122 .audit_skip = 1,
1123 .ioprio = 1,
1124 .iopoll = 1,
1016 .async_size = sizeof(struct io_async_rw),1125 .async_size = sizeof(struct io_async_rw),
1017 },1126 },
1018 [IORING_OP_WRITE_FIXED] = {1127 [IORING_OP_WRITE_FIXED] = {
@@ -1022,6 +1131,8 @@ static const struct io_op_def io_op_defs[] = { · 3 unmodified lines
1022 .pollout = 1,1131 .pollout = 1,
1023 .plug = 1,1132 .plug = 1,
1024 .audit_skip = 1,1133 .audit_skip = 1,
1134 .ioprio = 1,
1135 .iopoll = 1,
1025 .async_size = sizeof(struct io_async_rw),1136 .async_size = sizeof(struct io_async_rw),
1026 },1137 },
1027 [IORING_OP_POLL_ADD] = {1138 [IORING_OP_POLL_ADD] = {
@@ -1064,6 +1175,7 @@ static const struct io_op_def io_op_defs[] = { · 36 unmodified lines
1064 .unbound_nonreg_file = 1,1175 .unbound_nonreg_file = 1,
1065 .pollin = 1,1176 .pollin = 1,
1066 .poll_exclusive = 1,1177 .poll_exclusive = 1,
1178 .ioprio = 1, /* used for flags */
1067 },1179 },
1068 [IORING_OP_ASYNC_CANCEL] = {1180 [IORING_OP_ASYNC_CANCEL] = {
1069 .audit_skip = 1,1181 .audit_skip = 1,
@@ -1086,6 +1198,7 @@ static const struct io_op_def io_op_defs[] = { · 16 unmodified lines
1086 [IORING_OP_CLOSE] = {},1198 [IORING_OP_CLOSE] = {},
1087 [IORING_OP_FILES_UPDATE] = {1199 [IORING_OP_FILES_UPDATE] = {
1088 .audit_skip = 1,1200 .audit_skip = 1,
1201 .iopoll = 1,
1089 },1202 },
1090 [IORING_OP_STATX] = {1203 [IORING_OP_STATX] = {
1091 .audit_skip = 1,1204 .audit_skip = 1,
@@ -1097,6 +1210,8 @@ static const struct io_op_def io_op_defs[] = { · 5 unmodified lines
1097 .buffer_select = 1,1210 .buffer_select = 1,
1098 .plug = 1,1211 .plug = 1,
1099 .audit_skip = 1,1212 .audit_skip = 1,
1213 .ioprio = 1,
1214 .iopoll = 1,
1100 .async_size = sizeof(struct io_async_rw),1215 .async_size = sizeof(struct io_async_rw),
1101 },1216 },
1102 [IORING_OP_WRITE] = {1217 [IORING_OP_WRITE] = {
@@ -1106,6 +1221,8 @@ static const struct io_op_def io_op_defs[] = { · 3 unmodified lines
1106 .pollout = 1,1221 .pollout = 1,
1107 .plug = 1,1222 .plug = 1,
1108 .audit_skip = 1,1223 .audit_skip = 1,
1224 .ioprio = 1,
1225 .iopoll = 1,
1109 .async_size = sizeof(struct io_async_rw),1226 .async_size = sizeof(struct io_async_rw),
1110 },1227 },
1111 [IORING_OP_FADVISE] = {1228 [IORING_OP_FADVISE] = {
@@ -1140,9 +1257,11 @@ static const struct io_op_def io_op_defs[] = { · 28 unmodified lines
1140 },1257 },
1141 [IORING_OP_PROVIDE_BUFFERS] = {1258 [IORING_OP_PROVIDE_BUFFERS] = {
1142 .audit_skip = 1,1259 .audit_skip = 1,
1260 .iopoll = 1,
1143 },1261 },
1144 [IORING_OP_REMOVE_BUFFERS] = {1262 [IORING_OP_REMOVE_BUFFERS] = {
1145 .audit_skip = 1,1263 .audit_skip = 1,
1264 .iopoll = 1,
1146 },1265 },
1147 [IORING_OP_TEE] = {1266 [IORING_OP_TEE] = {
1148 .needs_file = 1,1267 .needs_file = 1,
@@ -1160,11 +1279,13 @@ static const struct io_op_def io_op_defs[] = { · 11 unmodified lines
1160 [IORING_OP_LINKAT] = {},1279 [IORING_OP_LINKAT] = {},
1161 [IORING_OP_MSG_RING] = {1280 [IORING_OP_MSG_RING] = {
1162 .needs_file = 1,1281 .needs_file = 1,
1282 .iopoll = 1,
1163 },1283 },
1164};1284};
1165 1285
1166/* requests with any of those set should undergo io_disarm_next() */1286/* requests with any of those set should undergo io_disarm_next() */
1167#define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL)1287#define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL)
1288#define IO_REQ_LINK_FLAGS (REQ_F_LINK | REQ_F_HARDLINK)
1168 1289
1169static bool io_disarm_next(struct io_kiocb *req);1290static bool io_disarm_next(struct io_kiocb *req);
1170static void io_uring_del_tctx_node(unsigned long index);1291static void io_uring_del_tctx_node(unsigned long index);
@@ -1173,10 +1294,7 @@ static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, · 2 unmodified lines
1173 bool cancel_all);1294 bool cancel_all);
1174static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd);1295static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd);
1175 1296
1176static void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags);1297static void __io_req_complete_post(struct io_kiocb *req, s32 res, u32 cflags);
1177
1178static void io_put_req(struct io_kiocb *req);
1179static void io_put_req_deferred(struct io_kiocb *req);
1180static void io_dismantle_req(struct io_kiocb *req);1298static void io_dismantle_req(struct io_kiocb *req);
1181static void io_queue_linked_timeout(struct io_kiocb *req);1299static void io_queue_linked_timeout(struct io_kiocb *req);
1182static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,1300static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
@@ -1185,10 +1303,10 @@ static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, · 2 unmodified lines
1185static void io_clean_op(struct io_kiocb *req);1303static void io_clean_op(struct io_kiocb *req);
1186static inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,1304static inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
1187 unsigned issue_flags);1305 unsigned issue_flags);
1188static inline struct file *io_file_get_normal(struct io_kiocb *req, int fd);1306static struct file *io_file_get_normal(struct io_kiocb *req, int fd);
1189static void io_drop_inflight_file(struct io_kiocb *req);1307static void io_drop_inflight_file(struct io_kiocb *req);
1190static bool io_assign_file(struct io_kiocb *req, unsigned int issue_flags);1308static bool io_assign_file(struct io_kiocb *req, unsigned int issue_flags);
1191static void __io_queue_sqe(struct io_kiocb *req);1309static void io_queue_sqe(struct io_kiocb *req);
1192static void io_rsrc_put_work(struct work_struct *work);1310static void io_rsrc_put_work(struct work_struct *work);
1193 1311
1194static void io_req_task_queue(struct io_kiocb *req);1312static void io_req_task_queue(struct io_kiocb *req);
@@ -1201,6 +1319,7 @@ static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags); · 6 unmodified lines
1201 1319
1202static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer);1320static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer);
1203static void io_eventfd_signal(struct io_ring_ctx *ctx);1321static void io_eventfd_signal(struct io_ring_ctx *ctx);
1322static void io_req_tw_post_queue(struct io_kiocb *req, s32 res, u32 cflags);
1204 1323
1205static struct kmem_cache *req_cachep;1324static struct kmem_cache *req_cachep;
1206 1325
@@ -1219,6 +1338,42 @@ struct sock *io_uring_get_socket(struct file *file) · 12 unmodified lines
1219}1338}
1220EXPORT_SYMBOL(io_uring_get_socket);1339EXPORT_SYMBOL(io_uring_get_socket);
1221 1340
1341#if defined(CONFIG_UNIX)
1342static inline bool io_file_need_scm(struct file *filp)
1343{
1344#if defined(IO_URING_SCM_ALL)
1345 return true;
1346#else
1347 return !!unix_get_socket(filp);
1348#endif
1349}
1350#else
1351static inline bool io_file_need_scm(struct file *filp)
1352{
1353 return false;
1354}
1355#endif
1356
1357static void io_ring_submit_unlock(struct io_ring_ctx *ctx, unsigned issue_flags)
1358{
1359 lockdep_assert_held(&ctx->uring_lock);
1360 if (issue_flags & IO_URING_F_UNLOCKED)
1361 mutex_unlock(&ctx->uring_lock);
1362}
1363
1364static void io_ring_submit_lock(struct io_ring_ctx *ctx, unsigned issue_flags)
1365{
1366 /*
1367 * "Normal" inline submissions always hold the uring_lock, since we
1368 * grab it from the system call. Same is true for the SQPOLL offload.
1369 * The only exception is when we've detached the request and issue it
1370 * from an async worker thread, grab the lock for that case.
1371 */
1372 if (issue_flags & IO_URING_F_UNLOCKED)
1373 mutex_lock(&ctx->uring_lock);
1374 lockdep_assert_held(&ctx->uring_lock);
1375}
1376
1222static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)1377static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)
1223{1378{
1224 if (!*locked) {1379 if (!*locked) {
@@ -1280,31 +1435,36 @@ static inline void io_req_set_refcount(struct io_kiocb *req) · 55 unmodified lines
1280 1435
1281#define IO_RSRC_REF_BATCH 1001436#define IO_RSRC_REF_BATCH 100
1282 1437
1438static void io_rsrc_put_node(struct io_rsrc_node *node, int nr)
1439{
1440 percpu_ref_put_many(&node->refs, nr);
1441}
1442
1283static inline void io_req_put_rsrc_locked(struct io_kiocb *req,1443static inline void io_req_put_rsrc_locked(struct io_kiocb *req,
1284 struct io_ring_ctx *ctx)1444 struct io_ring_ctx *ctx)
1285 __must_hold(&ctx->uring_lock)1445 __must_hold(&ctx->uring_lock)
1286{1446{
1287 struct percpu_ref *ref = req->fixed_rsrc_refs;1447 struct io_rsrc_node *node = req->rsrc_node;
1288 1448
1289 if (ref) {1449 if (node) {
1290 if (ref == &ctx->rsrc_node->refs)1450 if (node == ctx->rsrc_node)
1291 ctx->rsrc_cached_refs++;1451 ctx->rsrc_cached_refs++;
1292 else1452 else
1293 percpu_ref_put(ref);1453 io_rsrc_put_node(node, 1);
1294 }1454 }
1295}1455}
1296 1456
1297static inline void io_req_put_rsrc(struct io_kiocb *req, struct io_ring_ctx *ctx)1457static inline void io_req_put_rsrc(struct io_kiocb *req)
1298{1458{
1299 if (req->fixed_rsrc_refs)1459 if (req->rsrc_node)
1300 percpu_ref_put(req->fixed_rsrc_refs);1460 io_rsrc_put_node(req->rsrc_node, 1);
1301}1461}
1302 1462
1303static __cold void io_rsrc_refs_drop(struct io_ring_ctx *ctx)1463static __cold void io_rsrc_refs_drop(struct io_ring_ctx *ctx)
1304 __must_hold(&ctx->uring_lock)1464 __must_hold(&ctx->uring_lock)
1305{1465{
1306 if (ctx->rsrc_cached_refs) {1466 if (ctx->rsrc_cached_refs) {
1307 percpu_ref_put_many(&ctx->rsrc_node->refs, ctx->rsrc_cached_refs);1467 io_rsrc_put_node(ctx->rsrc_node, ctx->rsrc_cached_refs);
1308 ctx->rsrc_cached_refs = 0;1468 ctx->rsrc_cached_refs = 0;
1309 }1469 }
1310}1470}
@@ -1320,8 +1480,8 @@ static inline void io_req_set_rsrc_node(struct io_kiocb *req, · 9 unmodified lines
1320 struct io_ring_ctx *ctx,1480 struct io_ring_ctx *ctx,
1321 unsigned int issue_flags)1481 unsigned int issue_flags)
1322{1482{
1323 if (!req->fixed_rsrc_refs) {1483 if (!req->rsrc_node) {
1324 req->fixed_rsrc_refs = &ctx->rsrc_node->refs;1484 req->rsrc_node = ctx->rsrc_node;
1325 1485
1326 if (!(issue_flags & IO_URING_F_UNLOCKED)) {1486 if (!(issue_flags & IO_URING_F_UNLOCKED)) {
1327 lockdep_assert_held(&ctx->uring_lock);1487 lockdep_assert_held(&ctx->uring_lock);
@@ -1329,28 +1489,30 @@ static inline void io_req_set_rsrc_node(struct io_kiocb *req, · 1 unmodified line
1329 if (unlikely(ctx->rsrc_cached_refs < 0))1489 if (unlikely(ctx->rsrc_cached_refs < 0))
1330 io_rsrc_refs_refill(ctx);1490 io_rsrc_refs_refill(ctx);
1331 } else {1491 } else {
1332 percpu_ref_get(req->fixed_rsrc_refs);1492 percpu_ref_get(&req->rsrc_node->refs);
1333 }1493 }
1334 }1494 }
1335}1495}
1336 1496
1337static unsigned int __io_put_kbuf(struct io_kiocb *req, struct list_head *list)1497static unsigned int __io_put_kbuf(struct io_kiocb *req, struct list_head *list)
1338{1498{
1339 struct io_buffer *kbuf = req->kbuf;1499 if (req->flags & REQ_F_BUFFER_RING) {
1340 unsigned int cflags;1500 if (req->buf_list)
1501 req->buf_list->head++;
1502 req->flags &= ~REQ_F_BUFFER_RING;
1503 } else {
1504 list_add(&req->kbuf->list, list);
1505 req->flags &= ~REQ_F_BUFFER_SELECTED;
1506 }
1341 1507
1342 cflags = IORING_CQE_F_BUFFER | (kbuf->bid << IORING_CQE_BUFFER_SHIFT);1508 return IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT);
1343 req->flags &= ~REQ_F_BUFFER_SELECTED;
1344 list_add(&kbuf->list, list);
1345 req->kbuf = NULL;
1346 return cflags;
1347}1509}
1348 1510
1349static inline unsigned int io_put_kbuf_comp(struct io_kiocb *req)1511static inline unsigned int io_put_kbuf_comp(struct io_kiocb *req)
1350{1512{
1351 lockdep_assert_held(&req->ctx->completion_lock);1513 lockdep_assert_held(&req->ctx->completion_lock);
1352 1514
1353 if (likely(!(req->flags & REQ_F_BUFFER_SELECTED)))1515 if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))
1354 return 0;1516 return 0;
1355 return __io_put_kbuf(req, &req->ctx->io_buffers_comp);1517 return __io_put_kbuf(req, &req->ctx->io_buffers_comp);
1356}1518}
@@ -1360,7 +1522,7 @@ static inline unsigned int io_put_kbuf(struct io_kiocb *req, · 3 unmodified lines
1360{1522{
1361 unsigned int cflags;1523 unsigned int cflags;
1362 1524
1363 if (likely(!(req->flags & REQ_F_BUFFER_SELECTED)))1525 if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))
1364 return 0;1526 return 0;
1365 1527
1366 /*1528 /*
@@ -1375,7 +1537,10 @@ static inline unsigned int io_put_kbuf(struct io_kiocb *req, · 8 unmodified lines
1375 * We migrate buffers from the comp_list to the issue cache list1537 * We migrate buffers from the comp_list to the issue cache list
1376 * when we need one.1538 * when we need one.
1377 */1539 */
1378 if (issue_flags & IO_URING_F_UNLOCKED) {1540 if (req->flags & REQ_F_BUFFER_RING) {
1541 /* no buffers to recycle for this case */
1542 cflags = __io_put_kbuf(req, NULL);
1543 } else if (issue_flags & IO_URING_F_UNLOCKED) {
1379 struct io_ring_ctx *ctx = req->ctx;1544 struct io_ring_ctx *ctx = req->ctx;
1380 1545
1381 spin_lock(&ctx->completion_lock);1546 spin_lock(&ctx->completion_lock);
@@ -1393,15 +1558,10 @@ static inline unsigned int io_put_kbuf(struct io_kiocb *req, · 11 unmodified lines
1393static struct io_buffer_list *io_buffer_get_list(struct io_ring_ctx *ctx,1558static struct io_buffer_list *io_buffer_get_list(struct io_ring_ctx *ctx,
1394 unsigned int bgid)1559 unsigned int bgid)
1395{1560{
1396 struct list_head *hash_list;1561 if (ctx->io_bl && bgid < BGID_ARRAY)
1397 struct io_buffer_list *bl;1562 return &ctx->io_bl[bgid];
1398
1399 hash_list = &ctx->io_buffers[hash_32(bgid, IO_BUFFERS_HASH_BITS)];
1400 list_for_each_entry(bl, hash_list, list)
1401 if (bl->bgid == bgid || bgid == -1U)
1402 return bl;
1403 1563
1404 return NULL;1564 return xa_load(&ctx->io_bl_xa, bgid);
1405}1565}
1406 1566
1407static void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)1567static void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)
@@ -1410,25 +1570,33 @@ static void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) · 2 unmodified lines
1410 struct io_buffer_list *bl;1570 struct io_buffer_list *bl;
1411 struct io_buffer *buf;1571 struct io_buffer *buf;
1412 1572
1413 if (likely(!(req->flags & REQ_F_BUFFER_SELECTED)))1573 if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))
1414 return;1574 return;
1415 /* don't recycle if we already did IO to this buffer */1575 /* don't recycle if we already did IO to this buffer */
1416 if (req->flags & REQ_F_PARTIAL_IO)1576 if (req->flags & REQ_F_PARTIAL_IO)
1417 return;1577 return;
1578 /*
1579 * We don't need to recycle for REQ_F_BUFFER_RING, we can just clear
1580 * the flag and hence ensure that bl->head doesn't get incremented.
1581 * If the tail has already been incremented, hang on to it.
1582 */
1583 if (req->flags & REQ_F_BUFFER_RING) {
1584 if (req->buf_list) {
1585 req->buf_index = req->buf_list->bgid;
1586 req->flags &= ~REQ_F_BUFFER_RING;
1587 }
1588 return;
1589 }
1418 1590
1419 if (issue_flags & IO_URING_F_UNLOCKED)1591 io_ring_submit_lock(ctx, issue_flags);
1420 mutex_lock(&ctx->uring_lock);
1421
1422 lockdep_assert_held(&ctx->uring_lock);
1423 1592
1424 buf = req->kbuf;1593 buf = req->kbuf;
1425 bl = io_buffer_get_list(ctx, buf->bgid);1594 bl = io_buffer_get_list(ctx, buf->bgid);
1426 list_add(&buf->list, &bl->buf_list);1595 list_add(&buf->list, &bl->buf_list);
1427 req->flags &= ~REQ_F_BUFFER_SELECTED;1596 req->flags &= ~REQ_F_BUFFER_SELECTED;
1428 req->kbuf = NULL;1597 req->buf_index = buf->bgid;
1429 1598
1430 if (issue_flags & IO_URING_F_UNLOCKED)1599 io_ring_submit_unlock(ctx, issue_flags);
1431 mutex_unlock(&ctx->uring_lock);
1432}1600}
1433 1601
1434static bool io_match_task(struct io_kiocb *head, struct task_struct *task,1602static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
@@ -1469,7 +1637,12 @@ static inline void req_set_fail(struct io_kiocb *req) · 34 unmodified lines
1469static inline void req_fail_link_node(struct io_kiocb *req, int res)1637static inline void req_fail_link_node(struct io_kiocb *req, int res)
1470{1638{
1471 req_set_fail(req);1639 req_set_fail(req);
1472 req->result = res;1640 req->cqe.res = res;
1641}
1642
1643static inline void io_req_add_to_cache(struct io_kiocb *req, struct io_ring_ctx *ctx)
1644{
1645 wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list);
1473}1646}
1474 1647
1475static __cold void io_ring_ctx_ref_free(struct percpu_ref *ref)1648static __cold void io_ring_ctx_ref_free(struct percpu_ref *ref)
@@ -1506,12 +1679,14 @@ static __cold void io_fallback_req_func(struct work_struct *work) · 30 unmodified lines
1506static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)1679static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
1507{1680{
1508 struct io_ring_ctx *ctx;1681 struct io_ring_ctx *ctx;
1509 int i, hash_bits;1682 int hash_bits;
1510 1683
1511 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);1684 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1512 if (!ctx)1685 if (!ctx)
1513 return NULL;1686 return NULL;
1514 1687
1688 xa_init(&ctx->io_bl_xa);
1689
1515 /*1690 /*
1516 * Use 5 bits less than the max cq entries, that should give us around1691 * Use 5 bits less than the max cq entries, that should give us around
1517 * 32 entries per hash list if totally full and uniformly spread.1692 * 32 entries per hash list if totally full and uniformly spread.
@@ -1533,13 +1708,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) · 15 unmodified lines
1533 /* set invalid range, so io_import_fixed() fails meeting it */1708 /* set invalid range, so io_import_fixed() fails meeting it */
1534 ctx->dummy_ubuf->ubuf = -1UL;1709 ctx->dummy_ubuf->ubuf = -1UL;
1535 1710
1536 ctx->io_buffers = kcalloc(1U << IO_BUFFERS_HASH_BITS,
1537 sizeof(struct list_head), GFP_KERNEL);
1538 if (!ctx->io_buffers)
1539 goto err;
1540 for (i = 0; i < (1U << IO_BUFFERS_HASH_BITS); i++)
1541 INIT_LIST_HEAD(&ctx->io_buffers[i]);
1542
1543 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,1711 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
1544 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))1712 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
1545 goto err;1713 goto err;
@@ -1575,7 +1743,8 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) · 29 unmodified lines
1575err:1743err:
1576 kfree(ctx->dummy_ubuf);1744 kfree(ctx->dummy_ubuf);
1577 kfree(ctx->cancel_hash);1745 kfree(ctx->cancel_hash);
1578 kfree(ctx->io_buffers);1746 kfree(ctx->io_bl);
1747 xa_destroy(&ctx->io_bl_xa);
1579 kfree(ctx);1748 kfree(ctx);
1580 return NULL;1749 return NULL;
1581}1750}
@@ -1599,10 +1768,6 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq) · 17 unmodified lines
1599 return false;1768 return false;
1600}1769}
1601 1770
1602#define FFS_NOWAIT 0x1UL
1603#define FFS_ISREG 0x2UL
1604#define FFS_MASK ~(FFS_NOWAIT|FFS_ISREG)
1605
1606static inline bool io_req_ffs_set(struct io_kiocb *req)1771static inline bool io_req_ffs_set(struct io_kiocb *req)
1607{1772{
1608 return req->flags & REQ_F_FIXED_FILE;1773 return req->flags & REQ_F_FIXED_FILE;
@@ -1629,6 +1794,17 @@ static inline struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) · 20 unmodified lines
1629 return __io_prep_linked_timeout(req);1794 return __io_prep_linked_timeout(req);
1630}1795}
1631 1796
1797static noinline void __io_arm_ltimeout(struct io_kiocb *req)
1798{
1799 io_queue_linked_timeout(__io_prep_linked_timeout(req));
1800}
1801
1802static inline void io_arm_ltimeout(struct io_kiocb *req)
1803{
1804 if (unlikely(req->flags & REQ_F_ARM_LTIMEOUT))
1805 __io_arm_ltimeout(req);
1806}
1807
1632static void io_prep_async_work(struct io_kiocb *req)1808static void io_prep_async_work(struct io_kiocb *req)
1633{1809{
1634 const struct io_op_def *def = &io_op_defs[req->opcode];1810 const struct io_op_def *def = &io_op_defs[req->opcode];
@@ -1641,6 +1817,7 @@ static void io_prep_async_work(struct io_kiocb *req) · 6 unmodified lines
1641 1817
1642 req->work.list.next = NULL;1818 req->work.list.next = NULL;
1643 req->work.flags = 0;1819 req->work.flags = 0;
1820 req->work.cancel_seq = atomic_read(&ctx->cancel_seq);
1644 if (req->flags & REQ_F_FORCE_ASYNC)1821 if (req->flags & REQ_F_FORCE_ASYNC)
1645 req->work.flags |= IO_WQ_WORK_CONCURRENT;1822 req->work.flags |= IO_WQ_WORK_CONCURRENT;
1646 1823
@@ -1672,17 +1849,15 @@ static void io_prep_async_link(struct io_kiocb *req) · 25 unmodified lines
1672 1849
1673static inline void io_req_add_compl_list(struct io_kiocb *req)1850static inline void io_req_add_compl_list(struct io_kiocb *req)
1674{1851{
1675 struct io_ring_ctx *ctx = req->ctx;1852 struct io_submit_state *state = &req->ctx->submit_state;
1676 struct io_submit_state *state = &ctx->submit_state;
1677 1853
1678 if (!(req->flags & REQ_F_CQE_SKIP))1854 if (!(req->flags & REQ_F_CQE_SKIP))
1679 ctx->submit_state.flush_cqes = true;1855 state->flush_cqes = true;
1680 wq_list_add_tail(&req->comp_list, &state->compl_reqs);1856 wq_list_add_tail(&req->comp_list, &state->compl_reqs);
1681}1857}
1682 1858
1683static void io_queue_async_work(struct io_kiocb *req, bool *dont_use)1859static void io_queue_iowq(struct io_kiocb *req, bool *dont_use)
1684{1860{
1685 struct io_ring_ctx *ctx = req->ctx;
1686 struct io_kiocb *link = io_prep_linked_timeout(req);1861 struct io_kiocb *link = io_prep_linked_timeout(req);
1687 struct io_uring_task *tctx = req->task->io_uring;1862 struct io_uring_task *tctx = req->task->io_uring;
1688 1863
@@ -1702,8 +1877,9 @@ static void io_queue_async_work(struct io_kiocb *req, bool *dont_use) · 13 unmodified lines
1702 if (WARN_ON_ONCE(!same_thread_group(req->task, current)))1877 if (WARN_ON_ONCE(!same_thread_group(req->task, current)))
1703 req->work.flags |= IO_WQ_WORK_CANCEL;1878 req->work.flags |= IO_WQ_WORK_CANCEL;
1704 1879
1705 trace_io_uring_queue_async_work(ctx, req, req->user_data, req->opcode, req->flags,1880 trace_io_uring_queue_async_work(req->ctx, req, req->cqe.user_data,
1706 &req->work, io_wq_is_hashed(&req->work));1881 req->opcode, req->flags, &req->work,
1882 io_wq_is_hashed(&req->work));
1707 io_wq_enqueue(tctx->io_wq, &req->work);1883 io_wq_enqueue(tctx->io_wq, &req->work);
1708 if (link)1884 if (link)
1709 io_queue_linked_timeout(link);1885 io_queue_linked_timeout(link);
@@ -1721,8 +1897,7 @@ static void io_kill_timeout(struct io_kiocb *req, int status) · 11 unmodified lines
1721 atomic_set(&req->ctx->cq_timeouts,1897 atomic_set(&req->ctx->cq_timeouts,
1722 atomic_read(&req->ctx->cq_timeouts) + 1);1898 atomic_read(&req->ctx->cq_timeouts) + 1);
1723 list_del_init(&req->timeout.list);1899 list_del_init(&req->timeout.list);
1724 io_fill_cqe_req(req, status, 0);1900 io_req_tw_post_queue(req, status, 0);
1725 io_put_req_deferred(req);
1726 }1901 }
1727}1902}
1728 1903
@@ -1804,21 +1979,38 @@ static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) · 75 unmodified lines
1804 return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);1979 return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
1805}1980}
1806 1981
1807static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)1982/*
1983 * writes to the cq entry need to come after reading head; the
1984 * control dependency is enough as we're using WRITE_ONCE to
1985 * fill the cq entry
1986 */
1987static noinline struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx)
1808{1988{
1809 struct io_rings *rings = ctx->rings;1989 struct io_rings *rings = ctx->rings;
1810 unsigned tail, mask = ctx->cq_entries - 1;1990 unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1);
1811 1991 unsigned int free, queued, len;
1812 /*1992
1813 * writes to the cq entry need to come after reading head; the1993 /* userspace may cheat modifying the tail, be safe and do min */
1814 * control dependency is enough as we're using WRITE_ONCE to1994 queued = min(__io_cqring_events(ctx), ctx->cq_entries);
1815 * fill the cq entry1995 free = ctx->cq_entries - queued;
1816 */1996 /* we need a contiguous range, limit based on the current array offset */
1817 if (__io_cqring_events(ctx) == ctx->cq_entries)1997 len = min(free, ctx->cq_entries - off);
1998 if (!len)
1818 return NULL;1999 return NULL;
1819 2000
1820 tail = ctx->cached_cq_tail++;2001 ctx->cached_cq_tail++;
1821 return &rings->cqes[tail & mask];2002 ctx->cqe_cached = &rings->cqes[off];
2003 ctx->cqe_sentinel = ctx->cqe_cached + len;
2004 return ctx->cqe_cached++;
2005}
2006
2007static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
2008{
2009 if (likely(ctx->cqe_cached < ctx->cqe_sentinel)) {
2010 ctx->cached_cq_tail++;
2011 return ctx->cqe_cached++;
2012 }
2013 return __io_get_cqe(ctx);
1822}2014}
1823 2015
1824static void io_eventfd_signal(struct io_ring_ctx *ctx)2016static void io_eventfd_signal(struct io_ring_ctx *ctx)
@@ -1915,13 +2107,11 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) · 90 unmodified lines
1915 2107
1916 all_flushed = list_empty(&ctx->cq_overflow_list);2108 all_flushed = list_empty(&ctx->cq_overflow_list);
1917 if (all_flushed) {2109 if (all_flushed) {
1918 clear_bit(0, &ctx->check_cq_overflow);2110 clear_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq);
1919 WRITE_ONCE(ctx->rings->sq_flags,2111 atomic_andnot(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
1920 ctx->rings->sq_flags & ~IORING_SQ_CQ_OVERFLOW);
1921 }2112 }
1922 2113
1923 if (posted)2114 io_commit_cqring(ctx);
1924 io_commit_cqring(ctx);
1925 spin_unlock(&ctx->completion_lock);2115 spin_unlock(&ctx->completion_lock);
1926 if (posted)2116 if (posted)
1927 io_cqring_ev_posted(ctx);2117 io_cqring_ev_posted(ctx);
@@ -1932,7 +2122,7 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx) · 4 unmodified lines
1932{2122{
1933 bool ret = true;2123 bool ret = true;
1934 2124
1935 if (test_bit(0, &ctx->check_cq_overflow)) {2125 if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq)) {
1936 /* iopoll syncs against uring_lock, not completion_lock */2126 /* iopoll syncs against uring_lock, not completion_lock */
1937 if (ctx->flags & IORING_SETUP_IOPOLL)2127 if (ctx->flags & IORING_SETUP_IOPOLL)
1938 mutex_lock(&ctx->uring_lock);2128 mutex_lock(&ctx->uring_lock);
@@ -1944,19 +2134,23 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx) · 5 unmodified lines
1944 return ret;2134 return ret;
1945}2135}
1946 2136
1947/* must to be called somewhat shortly after putting a request */2137static void __io_put_task(struct task_struct *task, int nr)
1948static inline void io_put_task(struct task_struct *task, int nr)
1949{2138{
1950 struct io_uring_task *tctx = task->io_uring;2139 struct io_uring_task *tctx = task->io_uring;
1951 2140
1952 if (likely(task == current)) {2141 percpu_counter_sub(&tctx->inflight, nr);
1953 tctx->cached_refs += nr;2142 if (unlikely(atomic_read(&tctx->in_idle)))
1954 } else {2143 wake_up(&tctx->wait);
1955 percpu_counter_sub(&tctx->inflight, nr);2144 put_task_struct_many(task, nr);
1956 if (unlikely(atomic_read(&tctx->in_idle)))2145}
1957 wake_up(&tctx->wait);2146
1958 put_task_struct_many(task, nr);2147/* must to be called somewhat shortly after putting a request */
1959 }2148static inline void io_put_task(struct task_struct *task, int nr)
2149{
2150 if (likely(task == current))
2151 task->io_uring->cached_refs += nr;
2152 else
2153 __io_put_task(task, nr);
1960}2154}
1961 2155
1962static void io_task_refs_refill(struct io_uring_task *tctx)2156static void io_task_refs_refill(struct io_uring_task *tctx)
@@ -1995,6 +2189,7 @@ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, · 32 unmodified lines
1995 struct io_overflow_cqe *ocqe;2189 struct io_overflow_cqe *ocqe;
1996 2190
1997 ocqe = kmalloc(sizeof(*ocqe), GFP_ATOMIC | __GFP_ACCOUNT);2191 ocqe = kmalloc(sizeof(*ocqe), GFP_ATOMIC | __GFP_ACCOUNT);
2192 trace_io_uring_cqe_overflow(ctx, user_data, res, cflags, ocqe);
1998 if (!ocqe) {2193 if (!ocqe) {
1999 /*2194 /*
2000 * If we're in ring overflow flush mode, or in task cancel mode,2195 * If we're in ring overflow flush mode, or in task cancel mode,
@@ -2002,12 +2197,12 @@ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, · 1 unmodified line
2002 * on the floor.2197 * on the floor.
2003 */2198 */
2004 io_account_cq_overflow(ctx);2199 io_account_cq_overflow(ctx);
2200 set_bit(IO_CHECK_CQ_DROPPED_BIT, &ctx->check_cq);
2005 return false;2201 return false;
2006 }2202 }
2007 if (list_empty(&ctx->cq_overflow_list)) {2203 if (list_empty(&ctx->cq_overflow_list)) {
2008 set_bit(0, &ctx->check_cq_overflow);2204 set_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq);
2009 WRITE_ONCE(ctx->rings->sq_flags,2205 atomic_or(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
2010 ctx->rings->sq_flags | IORING_SQ_CQ_OVERFLOW);
2011 2206
2012 }2207 }
2013 ocqe->cqe.user_data = user_data;2208 ocqe->cqe.user_data = user_data;
@@ -2037,16 +2232,32 @@ static inline bool __io_fill_cqe(struct io_ring_ctx *ctx, u64 user_data, · 23 unmodified lines
2037 return io_cqring_event_overflow(ctx, user_data, res, cflags);2232 return io_cqring_event_overflow(ctx, user_data, res, cflags);
2038}2233}
2039 2234
2040static inline bool __io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)2235static inline bool __io_fill_cqe_req_filled(struct io_ring_ctx *ctx,
2236 struct io_kiocb *req)
2041{2237{
2042 trace_io_uring_complete(req->ctx, req, req->user_data, res, cflags);2238 struct io_uring_cqe *cqe;
2043 return __io_fill_cqe(req->ctx, req->user_data, res, cflags);2239
2240 trace_io_uring_complete(req->ctx, req, req->cqe.user_data,
2241 req->cqe.res, req->cqe.flags);
2242
2243 /*
2244 * If we can't get a cq entry, userspace overflowed the
2245 * submission (by quite a lot). Increment the overflow count in
2246 * the ring.
2247 */
2248 cqe = io_get_cqe(ctx);
2249 if (likely(cqe)) {
2250 memcpy(cqe, &req->cqe, sizeof(*cqe));
2251 return true;
2252 }
2253 return io_cqring_event_overflow(ctx, req->cqe.user_data,
2254 req->cqe.res, req->cqe.flags);
2044}2255}
2045 2256
2046static noinline void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)2257static inline bool __io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)
2047{2258{
2048 if (!(req->flags & REQ_F_CQE_SKIP))2259 trace_io_uring_complete(req->ctx, req, req->cqe.user_data, res, cflags);
2049 __io_fill_cqe_req(req, res, cflags);2260 return __io_fill_cqe(req->ctx, req->cqe.user_data, res, cflags);
2050}2261}
2051 2262
2052static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data,2263static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data,
@@ -2069,7 +2280,7 @@ static void __io_req_complete_post(struct io_kiocb *req, s32 res, · 16 unmodified lines
2069 * free_list cache.2280 * free_list cache.
2070 */2281 */
2071 if (req_ref_put_and_test(req)) {2282 if (req_ref_put_and_test(req)) {
2072 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {2283 if (req->flags & IO_REQ_LINK_FLAGS) {
2073 if (req->flags & IO_DISARM_MASK)2284 if (req->flags & IO_DISARM_MASK)
2074 io_disarm_next(req);2285 io_disarm_next(req);
2075 if (req->link) {2286 if (req->link) {
@@ -2077,7 +2288,7 @@ static void __io_req_complete_post(struct io_kiocb *req, s32 res, · 1 unmodified line
2077 req->link = NULL;2288 req->link = NULL;
2078 }2289 }
2079 }2290 }
2080 io_req_put_rsrc(req, ctx);2291 io_req_put_rsrc(req);
2081 /*2292 /*
2082 * Selected buffer deallocation in io_clean_op() assumes that2293 * Selected buffer deallocation in io_clean_op() assumes that
2083 * we don't hold ->completion_lock. Clean them here to avoid2294 * we don't hold ->completion_lock. Clean them here to avoid
@@ -2106,8 +2317,8 @@ static void io_req_complete_post(struct io_kiocb *req, s32 res, · 22 unmodified lines
2106static inline void io_req_complete_state(struct io_kiocb *req, s32 res,2317static inline void io_req_complete_state(struct io_kiocb *req, s32 res,
2107 u32 cflags)2318 u32 cflags)
2108{2319{
2109 req->result = res;2320 req->cqe.res = res;
2110 req->cflags = cflags;2321 req->cqe.flags = cflags;
2111 req->flags |= REQ_F_COMPLETE_INLINE;2322 req->flags |= REQ_F_COMPLETE_INLINE;
2112}2323}
2113 2324
@@ -2131,17 +2342,6 @@ static void io_req_complete_failed(struct io_kiocb *req, s32 res) · 17 unmodified lines
2131 io_req_complete_post(req, res, io_put_kbuf(req, IO_URING_F_UNLOCKED));2342 io_req_complete_post(req, res, io_put_kbuf(req, IO_URING_F_UNLOCKED));
2132}2343}
2133 2344
2134static void io_req_complete_fail_submit(struct io_kiocb *req)
2135{
2136 /*
2137 * We don't submit, fail them all, for that replace hardlinks with
2138 * normal links. Extra REQ_F_LINK is tolerated.
2139 */
2140 req->flags &= ~REQ_F_HARDLINK;
2141 req->flags |= REQ_F_LINK;
2142 io_req_complete_failed(req, req->result);
2143}
2144
2145/*2345/*
2146 * Don't initialise the fields below on every allocation, but do that in2346 * Don't initialise the fields below on every allocation, but do that in
2147 * advance and keep them valid across allocations.2347 * advance and keep them valid across allocations.
@@ -2152,7 +2352,7 @@ static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx) · 4 unmodified lines
2152 req->link = NULL;2352 req->link = NULL;
2153 req->async_data = NULL;2353 req->async_data = NULL;
2154 /* not necessary, but safer to zero */2354 /* not necessary, but safer to zero */
2155 req->result = 0;2355 req->cqe.res = 0;
2156}2356}
2157 2357
2158static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,2358static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,
@@ -2164,19 +2364,9 @@ static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx, · 5 unmodified lines
2164 spin_unlock(&ctx->completion_lock);2364 spin_unlock(&ctx->completion_lock);
2165}2365}
2166 2366
2167/* Returns true IFF there are requests in the cache */2367static inline bool io_req_cache_empty(struct io_ring_ctx *ctx)
2168static bool io_flush_cached_reqs(struct io_ring_ctx *ctx)
2169{2368{
2170 struct io_submit_state *state = &ctx->submit_state;2369 return !ctx->submit_state.free_list.next;
2171
2172 /*
2173 * If we have more than a batch's worth of requests in our IRQ side
2174 * locked cache, grab the lock and move them over to our submission
2175 * side cache.
2176 */
2177 if (READ_ONCE(ctx->locked_free_nr) > IO_COMPL_BATCH)
2178 io_flush_cached_locked_reqs(ctx, state);
2179 return !!state->free_list.next;
2180}2370}
2181 2371
2182/*2372/*
@@ -2188,14 +2378,20 @@ static bool io_flush_cached_reqs(struct io_ring_ctx *ctx) · 5 unmodified lines
2188static __cold bool __io_alloc_req_refill(struct io_ring_ctx *ctx)2378static __cold bool __io_alloc_req_refill(struct io_ring_ctx *ctx)
2189 __must_hold(&ctx->uring_lock)2379 __must_hold(&ctx->uring_lock)
2190{2380{
2191 struct io_submit_state *state = &ctx->submit_state;
2192 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;2381 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
2193 void *reqs[IO_REQ_ALLOC_BATCH];2382 void *reqs[IO_REQ_ALLOC_BATCH];
2194 struct io_kiocb *req;
2195 int ret, i;2383 int ret, i;
2196 2384
2197 if (likely(state->free_list.next || io_flush_cached_reqs(ctx)))2385 /*
2198 return true;2386 * If we have more than a batch's worth of requests in our IRQ side
2387 * locked cache, grab the lock and move them over to our submission
2388 * side cache.
2389 */
2390 if (data_race(ctx->locked_free_nr) > IO_COMPL_BATCH) {
2391 io_flush_cached_locked_reqs(ctx, &ctx->submit_state);
2392 if (!io_req_cache_empty(ctx))
2393 return true;
2394 }
2199 2395
2200 ret = kmem_cache_alloc_bulk(req_cachep, gfp, ARRAY_SIZE(reqs), reqs);2396 ret = kmem_cache_alloc_bulk(req_cachep, gfp, ARRAY_SIZE(reqs), reqs);
2201 2397
@@ -2212,17 +2408,17 @@ static __cold bool __io_alloc_req_refill(struct io_ring_ctx *ctx) · 10 unmodified lines
2212 2408
2213 percpu_ref_get_many(&ctx->refs, ret);2409 percpu_ref_get_many(&ctx->refs, ret);
2214 for (i = 0; i < ret; i++) {2410 for (i = 0; i < ret; i++) {
2215 req = reqs[i];2411 struct io_kiocb *req = reqs[i];
2216 2412
2217 io_preinit_req(req, ctx);2413 io_preinit_req(req, ctx);
2218 wq_stack_add_head(&req->comp_list, &state->free_list);2414 io_req_add_to_cache(req, ctx);
2219 }2415 }
2220 return true;2416 return true;
2221}2417}
2222 2418
2223static inline bool io_alloc_req_refill(struct io_ring_ctx *ctx)2419static inline bool io_alloc_req_refill(struct io_ring_ctx *ctx)
2224{2420{
2225 if (unlikely(!ctx->submit_state.free_list.next))2421 if (unlikely(io_req_cache_empty(ctx)))
2226 return __io_alloc_req_refill(ctx);2422 return __io_alloc_req_refill(ctx);
2227 return true;2423 return true;
2228}2424}
@@ -2251,11 +2447,11 @@ static inline void io_dismantle_req(struct io_kiocb *req) · 22 unmodified lines
2251 io_put_file(req->file);2447 io_put_file(req->file);
2252}2448}
2253 2449
2254static __cold void __io_free_req(struct io_kiocb *req)2450static __cold void io_free_req(struct io_kiocb *req)
2255{2451{
2256 struct io_ring_ctx *ctx = req->ctx;2452 struct io_ring_ctx *ctx = req->ctx;
2257 2453
2258 io_req_put_rsrc(req, ctx);2454 io_req_put_rsrc(req);
2259 io_dismantle_req(req);2455 io_dismantle_req(req);
2260 io_put_task(req->task, 1);2456 io_put_task(req->task, 1);
2261 2457
@@ -2273,7 +2469,7 @@ static inline void io_remove_next_linked(struct io_kiocb *req) · 11 unmodified lines
2273 nxt->link = NULL;2469 nxt->link = NULL;
2274}2470}
2275 2471
2276static bool io_kill_linked_timeout(struct io_kiocb *req)2472static struct io_kiocb *io_disarm_linked_timeout(struct io_kiocb *req)
2277 __must_hold(&req->ctx->completion_lock)2473 __must_hold(&req->ctx->completion_lock)
2278 __must_hold(&req->ctx->timeout_lock)2474 __must_hold(&req->ctx->timeout_lock)
2279{2475{
@@ -2286,13 +2482,10 @@ static bool io_kill_linked_timeout(struct io_kiocb *req) · 6 unmodified lines
2286 link->timeout.head = NULL;2482 link->timeout.head = NULL;
2287 if (hrtimer_try_to_cancel(&io->timer) != -1) {2483 if (hrtimer_try_to_cancel(&io->timer) != -1) {
2288 list_del(&link->timeout.list);2484 list_del(&link->timeout.list);
2289 /* leave REQ_F_CQE_SKIP to io_fill_cqe_req */2485 return link;
2290 io_fill_cqe_req(link, -ECANCELED, 0);
2291 io_put_req_deferred(link);
2292 return true;
2293 }2486 }
2294 }2487 }
2295 return false;2488 return NULL;
2296}2489}
2297 2490
2298static void io_fail_links(struct io_kiocb *req)2491static void io_fail_links(struct io_kiocb *req)
@@ -2306,19 +2499,19 @@ static void io_fail_links(struct io_kiocb *req) · 7 unmodified lines
2306 long res = -ECANCELED;2499 long res = -ECANCELED;
2307 2500
2308 if (link->flags & REQ_F_FAIL)2501 if (link->flags & REQ_F_FAIL)
2309 res = link->result;2502 res = link->cqe.res;
2310 2503
2311 nxt = link->link;2504 nxt = link->link;
2312 link->link = NULL;2505 link->link = NULL;
2313 2506
2314 trace_io_uring_fail_link(req->ctx, req, req->user_data,2507 trace_io_uring_fail_link(req->ctx, req, req->cqe.user_data,
2315 req->opcode, link);2508 req->opcode, link);
2316 2509
2317 if (!ignore_cqes) {2510 if (ignore_cqes)
2511 link->flags |= REQ_F_CQE_SKIP;
2512 else
2318 link->flags &= ~REQ_F_CQE_SKIP;2513 link->flags &= ~REQ_F_CQE_SKIP;
2319 io_fill_cqe_req(link, res, 0);2514 __io_req_complete_post(link, res, 0);
2320 }
2321 io_put_req_deferred(link);
2322 link = nxt;2515 link = nxt;
2323 }2516 }
2324}2517}
@@ -2326,25 +2519,27 @@ static void io_fail_links(struct io_kiocb *req) · 1 unmodified line
2326static bool io_disarm_next(struct io_kiocb *req)2519static bool io_disarm_next(struct io_kiocb *req)
2327 __must_hold(&req->ctx->completion_lock)2520 __must_hold(&req->ctx->completion_lock)
2328{2521{
2522 struct io_kiocb *link = NULL;
2329 bool posted = false;2523 bool posted = false;
2330 2524
2331 if (req->flags & REQ_F_ARM_LTIMEOUT) {2525 if (req->flags & REQ_F_ARM_LTIMEOUT) {
2332 struct io_kiocb *link = req->link;2526 link = req->link;
2333
2334 req->flags &= ~REQ_F_ARM_LTIMEOUT;2527 req->flags &= ~REQ_F_ARM_LTIMEOUT;
2335 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {2528 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
2336 io_remove_next_linked(req);2529 io_remove_next_linked(req);
2337 /* leave REQ_F_CQE_SKIP to io_fill_cqe_req */2530 io_req_tw_post_queue(link, -ECANCELED, 0);
2338 io_fill_cqe_req(link, -ECANCELED, 0);
2339 io_put_req_deferred(link);
2340 posted = true;2531 posted = true;
2341 }2532 }
2342 } else if (req->flags & REQ_F_LINK_TIMEOUT) {2533 } else if (req->flags & REQ_F_LINK_TIMEOUT) {
2343 struct io_ring_ctx *ctx = req->ctx;2534 struct io_ring_ctx *ctx = req->ctx;
2344 2535
2345 spin_lock_irq(&ctx->timeout_lock);2536 spin_lock_irq(&ctx->timeout_lock);
2346 posted = io_kill_linked_timeout(req);2537 link = io_disarm_linked_timeout(req);
2347 spin_unlock_irq(&ctx->timeout_lock);2538 spin_unlock_irq(&ctx->timeout_lock);
2539 if (link) {
2540 posted = true;
2541 io_req_tw_post_queue(link, -ECANCELED, 0);
2542 }
2348 }2543 }
2349 if (unlikely((req->flags & REQ_F_FAIL) &&2544 if (unlikely((req->flags & REQ_F_FAIL) &&
2350 !(req->flags & REQ_F_HARDLINK))) {2545 !(req->flags & REQ_F_HARDLINK))) {
@@ -2361,8 +2556,7 @@ static void __io_req_find_next_prep(struct io_kiocb *req) · 10 unmodified lines
2361 2556
2362 spin_lock(&ctx->completion_lock);2557 spin_lock(&ctx->completion_lock);
2363 posted = io_disarm_next(req);2558 posted = io_disarm_next(req);
2364 if (posted)2559 io_commit_cqring(ctx);
2365 io_commit_cqring(ctx);
2366 spin_unlock(&ctx->completion_lock);2560 spin_unlock(&ctx->completion_lock);
2367 if (posted)2561 if (posted)
2368 io_cqring_ev_posted(ctx);2562 io_cqring_ev_posted(ctx);
@@ -2372,8 +2566,6 @@ static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req) · 3 unmodified lines
2372{2566{
2373 struct io_kiocb *nxt;2567 struct io_kiocb *nxt;
2374 2568
2375 if (likely(!(req->flags & (REQ_F_LINK|REQ_F_HARDLINK))))
2376 return NULL;
2377 /*2569 /*
2378 * If LINK is set, we have dependent requests in this chain. If we2570 * If LINK is set, we have dependent requests in this chain. If we
2379 * didn't fail this request, queue the first one up, moving any other2571 * didn't fail this request, queue the first one up, moving any other
@@ -2391,6 +2583,8 @@ static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked) · 11 unmodified lines
2391{2583{
2392 if (!ctx)2584 if (!ctx)
2393 return;2585 return;
2586 if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
2587 atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
2394 if (*locked) {2588 if (*locked) {
2395 io_submit_flush_completions(ctx);2589 io_submit_flush_completions(ctx);
2396 mutex_unlock(&ctx->uring_lock);2590 mutex_unlock(&ctx->uring_lock);
@@ -2434,7 +2628,7 @@ static void handle_prev_tw_list(struct io_wq_work_node *node, · 37 unmodified lines
2434 if (likely(*uring_locked))2628 if (likely(*uring_locked))
2435 req->io_task_work.func(req, uring_locked);2629 req->io_task_work.func(req, uring_locked);
2436 else2630 else
2437 __io_req_complete_post(req, req->result,2631 __io_req_complete_post(req, req->cqe.res,
2438 io_put_kbuf_comp(req));2632 io_put_kbuf_comp(req));
2439 node = next;2633 node = next;
2440 } while (node);2634 } while (node);
@@ -2475,10 +2669,6 @@ static void tctx_task_work(struct callback_head *cb) · 34 unmodified lines
2475 while (1) {2669 while (1) {
2476 struct io_wq_work_node *node1, *node2;2670 struct io_wq_work_node *node1, *node2;
2477 2671
2478 if (!tctx->task_list.first &&
2479 !tctx->prior_task_list.first && uring_locked)
2480 io_submit_flush_completions(ctx);
2481
2482 spin_lock_irq(&tctx->task_lock);2672 spin_lock_irq(&tctx->task_lock);
2483 node1 = tctx->prior_task_list.first;2673 node1 = tctx->prior_task_list.first;
2484 node2 = tctx->task_list.first;2674 node2 = tctx->task_list.first;
@@ -2492,10 +2682,13 @@ static void tctx_task_work(struct callback_head *cb) · 7 unmodified lines
2492 2682
2493 if (node1)2683 if (node1)
2494 handle_prev_tw_list(node1, &ctx, &uring_locked);2684 handle_prev_tw_list(node1, &ctx, &uring_locked);
2495
2496 if (node2)2685 if (node2)
2497 handle_tw_list(node2, &ctx, &uring_locked);2686 handle_tw_list(node2, &ctx, &uring_locked);
2498 cond_resched();2687 cond_resched();
2688
2689 if (data_race(!tctx->task_list.first) &&
2690 data_race(!tctx->prior_task_list.first) && uring_locked)
2691 io_submit_flush_completions(ctx);
2499 }2692 }
2500 2693
2501 ctx_flush_and_put(ctx, &uring_locked);2694 ctx_flush_and_put(ctx, &uring_locked);
@@ -2508,8 +2701,8 @@ static void tctx_task_work(struct callback_head *cb) · 6 unmodified lines
2508static void io_req_task_work_add(struct io_kiocb *req, bool priority)2701static void io_req_task_work_add(struct io_kiocb *req, bool priority)
2509{2702{
2510 struct task_struct *tsk = req->task;2703 struct task_struct *tsk = req->task;
2704 struct io_ring_ctx *ctx = req->ctx;
2511 struct io_uring_task *tctx = tsk->io_uring;2705 struct io_uring_task *tctx = tsk->io_uring;
2512 enum task_work_notify_mode notify;
2513 struct io_wq_work_node *node;2706 struct io_wq_work_node *node;
2514 unsigned long flags;2707 unsigned long flags;
2515 bool running;2708 bool running;
@@ -2532,18 +2725,11 @@ static void io_req_task_work_add(struct io_kiocb *req, bool priority) · 16 unmodified lines
2532 if (running)2725 if (running)
2533 return;2726 return;
2534 2727
2535 /*2728 if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
2536 * SQPOLL kernel thread doesn't need notification, just a wakeup. For2729 atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
2537 * all other cases, use TWA_SIGNAL unconditionally to ensure we're2730
2538 * processing task_work. There's no reliable way to tell if TWA_RESUME2731 if (likely(!task_work_add(tsk, &tctx->task_work, ctx->notify_method)))
2539 * will do the job.
2540 */
2541 notify = (req->ctx->flags & IORING_SETUP_SQPOLL) ? TWA_NONE : TWA_SIGNAL;
2542 if (likely(!task_work_add(tsk, &tctx->task_work, notify))) {
2543 if (notify == TWA_NONE)
2544 wake_up_process(tsk);
2545 return;2732 return;
2546 }
2547 2733
2548 spin_lock_irqsave(&tctx->task_lock, flags);2734 spin_lock_irqsave(&tctx->task_lock, flags);
2549 tctx->task_running = false;2735 tctx->task_running = false;
@@ -2559,30 +2745,39 @@ static void io_req_task_work_add(struct io_kiocb *req, bool priority) · 9 unmodified lines
2559 }2745 }
2560}2746}
2561 2747
2562static void io_req_task_cancel(struct io_kiocb *req, bool *locked)2748static void io_req_tw_post(struct io_kiocb *req, bool *locked)
2563{2749{
2564 struct io_ring_ctx *ctx = req->ctx;2750 io_req_complete_post(req, req->cqe.res, req->cqe.flags);
2751}
2565 2752
2753static void io_req_tw_post_queue(struct io_kiocb *req, s32 res, u32 cflags)
2754{
2755 req->cqe.res = res;
2756 req->cqe.flags = cflags;
2757 req->io_task_work.func = io_req_tw_post;
2758 io_req_task_work_add(req, false);
2759}
2760
2761static void io_req_task_cancel(struct io_kiocb *req, bool *locked)
2762{
2566 /* not needed for normal modes, but SQPOLL depends on it */2763 /* not needed for normal modes, but SQPOLL depends on it */
2567 io_tw_lock(ctx, locked);2764 io_tw_lock(req->ctx, locked);
2568 io_req_complete_failed(req, req->result);2765 io_req_complete_failed(req, req->cqe.res);
2569}2766}
2570 2767
2571static void io_req_task_submit(struct io_kiocb *req, bool *locked)2768static void io_req_task_submit(struct io_kiocb *req, bool *locked)
2572{2769{
2573 struct io_ring_ctx *ctx = req->ctx;2770 io_tw_lock(req->ctx, locked);
2574
2575 io_tw_lock(ctx, locked);
2576 /* req->task == current here, checking PF_EXITING is safe */2771 /* req->task == current here, checking PF_EXITING is safe */
2577 if (likely(!(req->task->flags & PF_EXITING)))2772 if (likely(!(req->task->flags & PF_EXITING)))
2578 __io_queue_sqe(req);2773 io_queue_sqe(req);
2579 else2774 else
2580 io_req_complete_failed(req, -EFAULT);2775 io_req_complete_failed(req, -EFAULT);
2581}2776}
2582 2777
2583static void io_req_task_queue_fail(struct io_kiocb *req, int ret)2778static void io_req_task_queue_fail(struct io_kiocb *req, int ret)
2584{2779{
2585 req->result = ret;2780 req->cqe.res = ret;
2586 req->io_task_work.func = io_req_task_cancel;2781 req->io_task_work.func = io_req_task_cancel;
2587 io_req_task_work_add(req, false);2782 io_req_task_work_add(req, false);
2588}2783}
@@ -2595,11 +2790,11 @@ static void io_req_task_queue(struct io_kiocb *req) · 6 unmodified lines
2595 2790
2596static void io_req_task_queue_reissue(struct io_kiocb *req)2791static void io_req_task_queue_reissue(struct io_kiocb *req)
2597{2792{
2598 req->io_task_work.func = io_queue_async_work;2793 req->io_task_work.func = io_queue_iowq;
2599 io_req_task_work_add(req, false);2794 io_req_task_work_add(req, false);
2600}2795}
2601 2796
2602static inline void io_queue_next(struct io_kiocb *req)2797static void io_queue_next(struct io_kiocb *req)
2603{2798{
2604 struct io_kiocb *nxt = io_req_find_next(req);2799 struct io_kiocb *nxt = io_req_find_next(req);
2605 2800
@@ -2607,17 +2802,6 @@ static inline void io_queue_next(struct io_kiocb *req) · 1 unmodified line
2607 io_req_task_queue(nxt);2802 io_req_task_queue(nxt);
2608}2803}
2609 2804
2610static void io_free_req(struct io_kiocb *req)
2611{
2612 io_queue_next(req);
2613 __io_free_req(req);
2614}
2615
2616static void io_free_req_work(struct io_kiocb *req, bool *locked)
2617{
2618 io_free_req(req);
2619}
2620
2621static void io_free_batch_list(struct io_ring_ctx *ctx,2805static void io_free_batch_list(struct io_ring_ctx *ctx,
2622 struct io_wq_work_node *node)2806 struct io_wq_work_node *node)
2623 __must_hold(&ctx->uring_lock)2807 __must_hold(&ctx->uring_lock)
@@ -2629,15 +2813,30 @@ static void io_free_batch_list(struct io_ring_ctx *ctx, · 5 unmodified lines
2629 struct io_kiocb *req = container_of(node, struct io_kiocb,2813 struct io_kiocb *req = container_of(node, struct io_kiocb,
2630 comp_list);2814 comp_list);
2631 2815
2632 if (unlikely(req->flags & REQ_F_REFCOUNT)) {2816 if (unlikely(req->flags & IO_REQ_CLEAN_SLOW_FLAGS)) {
2633 node = req->comp_list.next;2817 if (req->flags & REQ_F_REFCOUNT) {
2634 if (!req_ref_put_and_test(req))2818 node = req->comp_list.next;
2635 continue;2819 if (!req_ref_put_and_test(req))
2820 continue;
2821 }
2822 if ((req->flags & REQ_F_POLLED) && req->apoll) {
2823 struct async_poll *apoll = req->apoll;
2824
2825 if (apoll->double_poll)
2826 kfree(apoll->double_poll);
2827 list_add(&apoll->poll.wait.entry,
2828 &ctx->apoll_cache);
2829 req->flags &= ~REQ_F_POLLED;
2830 }
2831 if (req->flags & IO_REQ_LINK_FLAGS)
2832 io_queue_next(req);
2833 if (unlikely(req->flags & IO_REQ_CLEAN_FLAGS))
2834 io_clean_op(req);
2636 }2835 }
2836 if (!(req->flags & REQ_F_FIXED_FILE))
2837 io_put_file(req->file);
2637 2838
2638 io_req_put_rsrc_locked(req, ctx);2839 io_req_put_rsrc_locked(req, ctx);
2639 io_queue_next(req);
2640 io_dismantle_req(req);
2641 2840
2642 if (req->task != task) {2841 if (req->task != task) {
2643 if (task)2842 if (task)
@@ -2647,7 +2846,7 @@ static void io_free_batch_list(struct io_ring_ctx *ctx, · 3 unmodified lines
2647 }2846 }
2648 task_refs++;2847 task_refs++;
2649 node = req->comp_list.next;2848 node = req->comp_list.next;
2650 wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list);2849 io_req_add_to_cache(req, ctx);
2651 } while (node);2850 } while (node);
2652 2851
2653 if (task)2852 if (task)
@@ -2667,16 +2866,7 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx) · 13 unmodified lines
2667 comp_list);2866 comp_list);
2668 2867
2669 if (!(req->flags & REQ_F_CQE_SKIP))2868 if (!(req->flags & REQ_F_CQE_SKIP))
2670 __io_fill_cqe_req(req, req->result, req->cflags);2869 __io_fill_cqe_req_filled(ctx, req);
2671 if ((req->flags & REQ_F_POLLED) && req->apoll) {
2672 struct async_poll *apoll = req->apoll;
2673
2674 if (apoll->double_poll)
2675 kfree(apoll->double_poll);
2676 list_add(&apoll->poll.wait.entry,
2677 &ctx->apoll_cache);
2678 req->flags &= ~REQ_F_POLLED;
2679 }
2680 }2870 }
2681 2871
2682 io_commit_cqring(ctx);2872 io_commit_cqring(ctx);
@@ -2698,23 +2888,18 @@ static inline struct io_kiocb *io_put_req_find_next(struct io_kiocb *req) · 15 unmodified lines
2698 struct io_kiocb *nxt = NULL;2888 struct io_kiocb *nxt = NULL;
2699 2889
2700 if (req_ref_put_and_test(req)) {2890 if (req_ref_put_and_test(req)) {
2701 nxt = io_req_find_next(req);2891 if (unlikely(req->flags & IO_REQ_LINK_FLAGS))
2702 __io_free_req(req);2892 nxt = io_req_find_next(req);
2893 io_free_req(req);
2703 }2894 }
2704 return nxt;2895 return nxt;
2705}2896}
2706 2897
2707static inline void io_put_req(struct io_kiocb *req)2898static inline void io_put_req(struct io_kiocb *req)
2708{2899{
2709 if (req_ref_put_and_test(req))2900 if (req_ref_put_and_test(req)) {
2901 io_queue_next(req);
2710 io_free_req(req);2902 io_free_req(req);
2711}
2712
2713static inline void io_put_req_deferred(struct io_kiocb *req)
2714{
2715 if (req_ref_put_and_test(req)) {
2716 req->io_task_work.func = io_free_req_work;
2717 io_req_task_work_add(req, false);
2718 }2903 }
2719}2904}
2720 2905
@@ -2800,7 +2985,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) · 79 unmodified lines
2800 nr_events++;2985 nr_events++;
2801 if (unlikely(req->flags & REQ_F_CQE_SKIP))2986 if (unlikely(req->flags & REQ_F_CQE_SKIP))
2802 continue;2987 continue;
2803 __io_fill_cqe_req(req, req->result, io_put_kbuf(req, 0));2988 __io_fill_cqe_req(req, req->cqe.res, io_put_kbuf(req, 0));
2804 }2989 }
2805 2990
2806 if (unlikely(!nr_events))2991 if (unlikely(!nr_events))
@@ -2846,22 +3031,26 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) · 39 unmodified lines
2846{3031{
2847 unsigned int nr_events = 0;3032 unsigned int nr_events = 0;
2848 int ret = 0;3033 int ret = 0;
3034 unsigned long check_cq;
2849 3035
2850 /*
2851 * We disallow the app entering submit/complete with polling, but we
2852 * still need to lock the ring to prevent racing with polled issue
2853 * that got punted to a workqueue.
2854 */
2855 mutex_lock(&ctx->uring_lock);
2856 /*3036 /*
2857 * Don't enter poll loop if we already have events pending.3037 * Don't enter poll loop if we already have events pending.
2858 * If we do, we can potentially be spinning for commands that3038 * If we do, we can potentially be spinning for commands that
2859 * already triggered a CQE (eg in error).3039 * already triggered a CQE (eg in error).
2860 */3040 */
2861 if (test_bit(0, &ctx->check_cq_overflow))3041 check_cq = READ_ONCE(ctx->check_cq);
3042 if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
2862 __io_cqring_overflow_flush(ctx, false);3043 __io_cqring_overflow_flush(ctx, false);
2863 if (io_cqring_events(ctx))3044 if (io_cqring_events(ctx))
2864 goto out;3045 return 0;
3046
3047 /*
3048 * Similarly do not spin if we have not informed the user of any
3049 * dropped CQE.
3050 */
3051 if (unlikely(check_cq & BIT(IO_CHECK_CQ_DROPPED_BIT)))
3052 return -EBADR;
3053
2865 do {3054 do {
2866 /*3055 /*
2867 * If a submit got punted to a workqueue, we can have the3056 * If a submit got punted to a workqueue, we can have the
@@ -2891,8 +3080,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) · 23 unmodified lines
2891 nr_events += ret;3080 nr_events += ret;
2892 ret = 0;3081 ret = 0;
2893 } while (nr_events < min && !need_resched());3082 } while (nr_events < min && !need_resched());
2894out:3083
2895 mutex_unlock(&ctx->uring_lock);
2896 return ret;3084 return ret;
2897}3085}
2898 3086
@@ -2965,21 +3153,21 @@ static bool __io_complete_rw_common(struct io_kiocb *req, long res) · 66 unmodified lines
2965 } else {3153 } else {
2966 fsnotify_access(req->file);3154 fsnotify_access(req->file);
2967 }3155 }
2968 if (unlikely(res != req->result)) {3156 if (unlikely(res != req->cqe.res)) {
2969 if ((res == -EAGAIN || res == -EOPNOTSUPP) &&3157 if ((res == -EAGAIN || res == -EOPNOTSUPP) &&
2970 io_rw_should_reissue(req)) {3158 io_rw_should_reissue(req)) {
2971 req->flags |= REQ_F_REISSUE;3159 req->flags |= REQ_F_REISSUE;
2972 return true;3160 return true;
2973 }3161 }
2974 req_set_fail(req);3162 req_set_fail(req);
2975 req->result = res;3163 req->cqe.res = res;
2976 }3164 }
2977 return false;3165 return false;
2978}3166}
2979 3167
2980static inline void io_req_task_complete(struct io_kiocb *req, bool *locked)3168static inline void io_req_task_complete(struct io_kiocb *req, bool *locked)
2981{3169{
2982 int res = req->result;3170 int res = req->cqe.res;
2983 3171
2984 if (*locked) {3172 if (*locked) {
2985 io_req_complete_state(req, res, io_put_kbuf(req, 0));3173 io_req_complete_state(req, res, io_put_kbuf(req, 0));
@@ -2995,7 +3183,7 @@ static void __io_complete_rw(struct io_kiocb *req, long res, · 9 unmodified lines
2995{3183{
2996 if (__io_complete_rw_common(req, res))3184 if (__io_complete_rw_common(req, res))
2997 return;3185 return;
2998 __io_req_complete(req, issue_flags, req->result,3186 __io_req_complete(req, issue_flags, req->cqe.res,
2999 io_put_kbuf(req, issue_flags));3187 io_put_kbuf(req, issue_flags));
3000}3188}
3001 3189
@@ -3005,7 +3193,7 @@ static void io_complete_rw(struct kiocb *kiocb, long res) · 3 unmodified lines
3005 3193
3006 if (__io_complete_rw_common(req, res))3194 if (__io_complete_rw_common(req, res))
3007 return;3195 return;
3008 req->result = res;3196 req->cqe.res = res;
3009 req->io_task_work.func = io_req_task_complete;3197 req->io_task_work.func = io_req_task_complete;
3010 io_req_task_work_add(req, !!(req->ctx->flags & IORING_SETUP_SQPOLL));3198 io_req_task_work_add(req, !!(req->ctx->flags & IORING_SETUP_SQPOLL));
3011}3199}
@@ -3016,12 +3204,12 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res) · 4 unmodified lines
3016 3204
3017 if (kiocb->ki_flags & IOCB_WRITE)3205 if (kiocb->ki_flags & IOCB_WRITE)
3018 kiocb_end_write(req);3206 kiocb_end_write(req);
3019 if (unlikely(res != req->result)) {3207 if (unlikely(res != req->cqe.res)) {
3020 if (res == -EAGAIN && io_rw_should_reissue(req)) {3208 if (res == -EAGAIN && io_rw_should_reissue(req)) {
3021 req->flags |= REQ_F_REISSUE;3209 req->flags |= REQ_F_REISSUE;
3022 return;3210 return;
3023 }3211 }
3024 req->result = res;3212 req->cqe.res = res;
3025 }3213 }
3026 3214
3027 /* order with io_iopoll_complete() checking ->iopoll_completed */3215 /* order with io_iopoll_complete() checking ->iopoll_completed */
@@ -3131,6 +3319,8 @@ static unsigned int io_file_get_flags(struct file *file) · 103 unmodified lines
3131 res |= FFS_ISREG;3319 res |= FFS_ISREG;
3132 if (__io_file_supports_nowait(file, mode))3320 if (__io_file_supports_nowait(file, mode))
3133 res |= FFS_NOWAIT;3321 res |= FFS_NOWAIT;
3322 if (io_file_need_scm(file))
3323 res |= FFS_SCM;
3134 return res;3324 return res;
3135}3325}
3136 3326
@@ -3162,6 +3352,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 25 unmodified lines
3162 req->rw.addr = READ_ONCE(sqe->addr);3352 req->rw.addr = READ_ONCE(sqe->addr);
3163 req->rw.len = READ_ONCE(sqe->len);3353 req->rw.len = READ_ONCE(sqe->len);
3164 req->rw.flags = READ_ONCE(sqe->rw_flags);3354 req->rw.flags = READ_ONCE(sqe->rw_flags);
3355 /* used for fixed read/write too - just read unconditionally */
3165 req->buf_index = READ_ONCE(sqe->buf_index);3356 req->buf_index = READ_ONCE(sqe->buf_index);
3166 return 0;3357 return 0;
3167}3358}
@@ -3310,77 +3501,96 @@ static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter, · 142 unmodified lines
3310 return __io_import_fixed(req, rw, iter, imu);3501 return __io_import_fixed(req, rw, iter, imu);
3311}3502}
3312 3503
3313static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)3504static int io_buffer_add_list(struct io_ring_ctx *ctx,
3505 struct io_buffer_list *bl, unsigned int bgid)
3314{3506{
3315 if (needs_lock)
3316 mutex_unlock(&ctx->uring_lock);
3317}
3318
3319static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
3320{
3321 /*
3322 * "Normal" inline submissions always hold the uring_lock, since we
3323 * grab it from the system call. Same is true for the SQPOLL offload.
3324 * The only exception is when we've detached the request and issue it
3325 * from an async worker thread, grab the lock for that case.
3326 */
3327 if (needs_lock)
3328 mutex_lock(&ctx->uring_lock);
3329}
3330
3331static void io_buffer_add_list(struct io_ring_ctx *ctx,
3332 struct io_buffer_list *bl, unsigned int bgid)
3333{
3334 struct list_head *list;
3335
3336 list = &ctx->io_buffers[hash_32(bgid, IO_BUFFERS_HASH_BITS)];
3337 INIT_LIST_HEAD(&bl->buf_list);
3338 bl->bgid = bgid;3507 bl->bgid = bgid;
3339 list_add(&bl->list, list);3508 if (bgid < BGID_ARRAY)
3509 return 0;
3510
3511 return xa_err(xa_store(&ctx->io_bl_xa, bgid, bl, GFP_KERNEL));
3340}3512}
3341 3513
3342static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,3514static void __user *io_provided_buffer_select(struct io_kiocb *req, size_t *len,
3343 int bgid, unsigned int issue_flags)3515 struct io_buffer_list *bl)
3344{3516{
3345 struct io_buffer *kbuf = req->kbuf;3517 if (!list_empty(&bl->buf_list)) {
3346 bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;3518 struct io_buffer *kbuf;
3347 struct io_ring_ctx *ctx = req->ctx;
3348 struct io_buffer_list *bl;
3349
3350 if (req->flags & REQ_F_BUFFER_SELECTED)
3351 return kbuf;
3352 3519
3353 io_ring_submit_lock(ctx, needs_lock);
3354
3355 lockdep_assert_held(&ctx->uring_lock);
3356
3357 bl = io_buffer_get_list(ctx, bgid);
3358 if (bl && !list_empty(&bl->buf_list)) {
3359 kbuf = list_first_entry(&bl->buf_list, struct io_buffer, list);3520 kbuf = list_first_entry(&bl->buf_list, struct io_buffer, list);
3360 list_del(&kbuf->list);3521 list_del(&kbuf->list);
3361 if (*len > kbuf->len)3522 if (*len > kbuf->len)
3362 *len = kbuf->len;3523 *len = kbuf->len;
3363 req->flags |= REQ_F_BUFFER_SELECTED;3524 req->flags |= REQ_F_BUFFER_SELECTED;
3364 req->kbuf = kbuf;3525 req->kbuf = kbuf;
3526 req->buf_index = kbuf->bid;
3527 return u64_to_user_ptr(kbuf->addr);
3528 }
3529 return NULL;
3530}
3531
3532static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len,
3533 struct io_buffer_list *bl,
3534 unsigned int issue_flags)
3535{
3536 struct io_uring_buf_ring *br = bl->buf_ring;
3537 struct io_uring_buf *buf;
3538 __u32 head = bl->head;
3539
3540 if (unlikely(smp_load_acquire(&br->tail) == head)) {
3541 io_ring_submit_unlock(req->ctx, issue_flags);
3542 return NULL;
3543 }
3544
3545 head &= bl->mask;
3546 if (head < IO_BUFFER_LIST_BUF_PER_PAGE) {
3547 buf = &br->bufs[head];
3365 } else {3548 } else {
3366 kbuf = ERR_PTR(-ENOBUFS);3549 int off = head & (IO_BUFFER_LIST_BUF_PER_PAGE - 1);
3550 int index = head / IO_BUFFER_LIST_BUF_PER_PAGE - 1;
3551 buf = page_address(bl->buf_pages[index]);
3552 buf += off;
3367 }3553 }
3554 if (*len > buf->len)
3555 *len = buf->len;
3556 req->flags |= REQ_F_BUFFER_RING;
3557 req->buf_list = bl;
3558 req->buf_index = buf->bid;
3368 3559
3369 io_ring_submit_unlock(req->ctx, needs_lock);3560 if (issue_flags & IO_URING_F_UNLOCKED) {
3370 return kbuf;3561 /*
3562 * If we came in unlocked, we have no choice but to consume the
3563 * buffer here. This does mean it'll be pinned until the IO
3564 * completes. But coming in unlocked means we're in io-wq
3565 * context, hence there should be no further retry. For the
3566 * locked case, the caller must ensure to call the commit when
3567 * the transfer completes (or if we get -EAGAIN and must poll
3568 * or retry).
3569 */
3570 req->buf_list = NULL;
3571 bl->head++;
3572 }
3573 return u64_to_user_ptr(buf->addr);
3371}3574}
3372 3575
3373static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,3576static void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
3374 unsigned int issue_flags)3577 unsigned int issue_flags)
3375{3578{
3376 struct io_buffer *kbuf;3579 struct io_ring_ctx *ctx = req->ctx;
3377 u16 bgid;3580 struct io_buffer_list *bl;
3581 void __user *ret = NULL;
3582
3583 io_ring_submit_lock(req->ctx, issue_flags);
3378 3584
3379 bgid = req->buf_index;3585 bl = io_buffer_get_list(ctx, req->buf_index);
3380 kbuf = io_buffer_select(req, len, bgid, issue_flags);3586 if (likely(bl)) {
3381 if (IS_ERR(kbuf))3587 if (bl->buf_nr_pages)
3382 return kbuf;3588 ret = io_ring_buffer_select(req, len, bl, issue_flags);
3383 return u64_to_user_ptr(kbuf->addr);3589 else
3590 ret = io_provided_buffer_select(req, len, bl);
3591 }
3592 io_ring_submit_unlock(req->ctx, issue_flags);
3593 return ret;
3384}3594}
3385 3595
3386#ifdef CONFIG_COMPAT3596#ifdef CONFIG_COMPAT
@@ -3390,7 +3600,7 @@ static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov, · 3 unmodified lines
3390 struct compat_iovec __user *uiov;3600 struct compat_iovec __user *uiov;
3391 compat_ssize_t clen;3601 compat_ssize_t clen;
3392 void __user *buf;3602 void __user *buf;
3393 ssize_t len;3603 size_t len;
3394 3604
3395 uiov = u64_to_user_ptr(req->rw.addr);3605 uiov = u64_to_user_ptr(req->rw.addr);
3396 if (!access_ok(uiov, sizeof(*uiov)))3606 if (!access_ok(uiov, sizeof(*uiov)))
@@ -3401,11 +3611,12 @@ static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov, · 4 unmodified lines
3401 return -EINVAL;3611 return -EINVAL;
3402 3612
3403 len = clen;3613 len = clen;
3404 buf = io_rw_buffer_select(req, &len, issue_flags);3614 buf = io_buffer_select(req, &len, issue_flags);
3405 if (IS_ERR(buf))3615 if (!buf)
3406 return PTR_ERR(buf);3616 return -ENOBUFS;
3617 req->rw.addr = (unsigned long) buf;
3407 iov[0].iov_base = buf;3618 iov[0].iov_base = buf;
3408 iov[0].iov_len = (compat_size_t) len;3619 req->rw.len = iov[0].iov_len = (compat_size_t) len;
3409 return 0;3620 return 0;
3410}3621}
3411#endif3622#endif
@@ -3423,22 +3634,21 @@ static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, · 11 unmodified lines
3423 len = iov[0].iov_len;3634 len = iov[0].iov_len;
3424 if (len < 0)3635 if (len < 0)
3425 return -EINVAL;3636 return -EINVAL;
3426 buf = io_rw_buffer_select(req, &len, issue_flags);3637 buf = io_buffer_select(req, &len, issue_flags);
3427 if (IS_ERR(buf))3638 if (!buf)
3428 return PTR_ERR(buf);3639 return -ENOBUFS;
3640 req->rw.addr = (unsigned long) buf;
3429 iov[0].iov_base = buf;3641 iov[0].iov_base = buf;
3430 iov[0].iov_len = len;3642 req->rw.len = iov[0].iov_len = len;
3431 return 0;3643 return 0;
3432}3644}
3433 3645
3434static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,3646static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
3435 unsigned int issue_flags)3647 unsigned int issue_flags)
3436{3648{
3437 if (req->flags & REQ_F_BUFFER_SELECTED) {3649 if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) {
3438 struct io_buffer *kbuf = req->kbuf;3650 iov[0].iov_base = u64_to_user_ptr(req->rw.addr);
3439 3651 iov[0].iov_len = req->rw.len;
3440 iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
3441 iov[0].iov_len = kbuf->len;
3442 return 0;3652 return 0;
3443 }3653 }
3444 if (req->rw.len != 1)3654 if (req->rw.len != 1)
@@ -3452,6 +3662,13 @@ static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, · 7 unmodified lines
3452 return __io_iov_buffer_select(req, iov, issue_flags);3662 return __io_iov_buffer_select(req, iov, issue_flags);
3453}3663}
3454 3664
3665static inline bool io_do_buffer_select(struct io_kiocb *req)
3666{
3667 if (!(req->flags & REQ_F_BUFFER_SELECT))
3668 return false;
3669 return !(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING));
3670}
3671
3455static struct iovec *__io_import_iovec(int rw, struct io_kiocb *req,3672static struct iovec *__io_import_iovec(int rw, struct io_kiocb *req,
3456 struct io_rw_state *s,3673 struct io_rw_state *s,
3457 unsigned int issue_flags)3674 unsigned int issue_flags)
@@ -3470,18 +3687,15 @@ static struct iovec *__io_import_iovec(int rw, struct io_kiocb *req, · 12 unmodified lines
3470 return NULL;3687 return NULL;
3471 }3688 }
3472 3689
3473 /* buffer index only valid with fixed read/write, or buffer select */
3474 if (unlikely(req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT)))
3475 return ERR_PTR(-EINVAL);
3476
3477 buf = u64_to_user_ptr(req->rw.addr);3690 buf = u64_to_user_ptr(req->rw.addr);
3478 sqe_len = req->rw.len;3691 sqe_len = req->rw.len;
3479 3692
3480 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {3693 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
3481 if (req->flags & REQ_F_BUFFER_SELECT) {3694 if (io_do_buffer_select(req)) {
3482 buf = io_rw_buffer_select(req, &sqe_len, issue_flags);3695 buf = io_buffer_select(req, &sqe_len, issue_flags);
3483 if (IS_ERR(buf))3696 if (!buf)
3484 return ERR_CAST(buf);3697 return ERR_PTR(-ENOBUFS);
3698 req->rw.addr = (unsigned long) buf;
3485 req->rw.len = sqe_len;3699 req->rw.len = sqe_len;
3486 }3700 }
3487 3701
@@ -3836,7 +4050,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) · 348 unmodified lines
3836 kfree(iovec);4050 kfree(iovec);
3837 return ret;4051 return ret;
3838 }4052 }
3839 req->result = iov_iter_count(&s->iter);4053 req->cqe.res = iov_iter_count(&s->iter);
3840 4054
3841 if (force_nonblock) {4055 if (force_nonblock) {
3842 /* If the file doesn't support async, just async punt */4056 /* If the file doesn't support async, just async punt */
@@ -3852,7 +4066,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) · 9 unmodified lines
3852 4066
3853 ppos = io_kiocb_update_pos(req);4067 ppos = io_kiocb_update_pos(req);
3854 4068
3855 ret = rw_verify_area(READ, req->file, ppos, req->result);4069 ret = rw_verify_area(READ, req->file, ppos, req->cqe.res);
3856 if (unlikely(ret)) {4070 if (unlikely(ret)) {
3857 kfree(iovec);4071 kfree(iovec);
3858 return ret;4072 return ret;
@@ -3874,7 +4088,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) · 15 unmodified lines
3874 ret = 0;4088 ret = 0;
3875 } else if (ret == -EIOCBQUEUED) {4089 } else if (ret == -EIOCBQUEUED) {
3876 goto out_free;4090 goto out_free;
3877 } else if (ret == req->result || ret <= 0 || !force_nonblock ||4091 } else if (ret == req->cqe.res || ret <= 0 || !force_nonblock ||
3878 (req->flags & REQ_F_NOWAIT) || !need_read_all(req)) {4092 (req->flags & REQ_F_NOWAIT) || !need_read_all(req)) {
3879 /* read all, failed, already did sync or don't want to retry */4093 /* read all, failed, already did sync or don't want to retry */
3880 goto done;4094 goto done;
@@ -3964,7 +4178,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) · 83 unmodified lines
3964 kfree(iovec);4178 kfree(iovec);
3965 return ret;4179 return ret;
3966 }4180 }
3967 req->result = iov_iter_count(&s->iter);4181 req->cqe.res = iov_iter_count(&s->iter);
3968 4182
3969 if (force_nonblock) {4183 if (force_nonblock) {
3970 /* If the file doesn't support async, just async punt */4184 /* If the file doesn't support async, just async punt */
@@ -3984,7 +4198,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) · 13 unmodified lines
3984 4198
3985 ppos = io_kiocb_update_pos(req);4199 ppos = io_kiocb_update_pos(req);
3986 4200
3987 ret = rw_verify_area(WRITE, req->file, ppos, req->result);4201 ret = rw_verify_area(WRITE, req->file, ppos, req->cqe.res);
3988 if (unlikely(ret))4202 if (unlikely(ret))
3989 goto out_free;4203 goto out_free;
3990 4204
@@ -4048,9 +4262,7 @@ static int io_renameat_prep(struct io_kiocb *req, · 57 unmodified lines
4048 struct io_rename *ren = &req->rename;4262 struct io_rename *ren = &req->rename;
4049 const char __user *oldf, *newf;4263 const char __user *oldf, *newf;
4050 4264
4051 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))4265 if (sqe->buf_index || sqe->splice_fd_in)
4052 return -EINVAL;
4053 if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)
4054 return -EINVAL;4266 return -EINVAL;
4055 if (unlikely(req->flags & REQ_F_FIXED_FILE))4267 if (unlikely(req->flags & REQ_F_FIXED_FILE))
4056 return -EBADF;4268 return -EBADF;
@@ -4099,10 +4311,7 @@ static int io_unlinkat_prep(struct io_kiocb *req, · 42 unmodified lines
4099 struct io_unlink *un = &req->unlink;4311 struct io_unlink *un = &req->unlink;
4100 const char __user *fname;4312 const char __user *fname;
4101 4313
4102 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))4314 if (sqe->off || sqe->len || sqe->buf_index || sqe->splice_fd_in)
4103 return -EINVAL;
4104 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
4105 sqe->splice_fd_in)
4106 return -EINVAL;4315 return -EINVAL;
4107 if (unlikely(req->flags & REQ_F_FIXED_FILE))4316 if (unlikely(req->flags & REQ_F_FIXED_FILE))
4108 return -EBADF;4317 return -EBADF;
@@ -4148,10 +4357,7 @@ static int io_mkdirat_prep(struct io_kiocb *req, · 39 unmodified lines
4148 struct io_mkdir *mkd = &req->mkdir;4357 struct io_mkdir *mkd = &req->mkdir;
4149 const char __user *fname;4358 const char __user *fname;
4150 4359
4151 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))4360 if (sqe->off || sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in)
4152 return -EINVAL;
4153 if (sqe->ioprio || sqe->off || sqe->rw_flags || sqe->buf_index ||
4154 sqe->splice_fd_in)
4155 return -EINVAL;4361 return -EINVAL;
4156 if (unlikely(req->flags & REQ_F_FIXED_FILE))4362 if (unlikely(req->flags & REQ_F_FIXED_FILE))
4157 return -EBADF;4363 return -EBADF;
@@ -4191,10 +4397,7 @@ static int io_symlinkat_prep(struct io_kiocb *req, · 33 unmodified lines
4191 struct io_symlink *sl = &req->symlink;4397 struct io_symlink *sl = &req->symlink;
4192 const char __user *oldpath, *newpath;4398 const char __user *oldpath, *newpath;
4193 4399
4194 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))4400 if (sqe->len || sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in)
4195 return -EINVAL;
4196 if (sqe->ioprio || sqe->len || sqe->rw_flags || sqe->buf_index ||
4197 sqe->splice_fd_in)
4198 return -EINVAL;4401 return -EINVAL;
4199 if (unlikely(req->flags & REQ_F_FIXED_FILE))4402 if (unlikely(req->flags & REQ_F_FIXED_FILE))
4200 return -EBADF;4403 return -EBADF;
@@ -4240,9 +4443,7 @@ static int io_linkat_prep(struct io_kiocb *req, · 39 unmodified lines
4240 struct io_hardlink *lnk = &req->hardlink;4443 struct io_hardlink *lnk = &req->hardlink;
4241 const char __user *oldf, *newf;4444 const char __user *oldf, *newf;
4242 4445
4243 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))4446 if (sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in)
4244 return -EINVAL;
4245 if (sqe->ioprio || sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in)
4246 return -EINVAL;4447 return -EINVAL;
4247 if (unlikely(req->flags & REQ_F_FIXED_FILE))4448 if (unlikely(req->flags & REQ_F_FIXED_FILE))
4248 return -EBADF;4449 return -EBADF;
@@ -4289,9 +4490,7 @@ static int io_shutdown_prep(struct io_kiocb *req, · 40 unmodified lines
4289 const struct io_uring_sqe *sqe)4490 const struct io_uring_sqe *sqe)
4290{4491{
4291#if defined(CONFIG_NET)4492#if defined(CONFIG_NET)
4292 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))4493 if (unlikely(sqe->off || sqe->addr || sqe->rw_flags ||
4293 return -EINVAL;
4294 if (unlikely(sqe->ioprio || sqe->off || sqe->addr || sqe->rw_flags ||
4295 sqe->buf_index || sqe->splice_fd_in))4494 sqe->buf_index || sqe->splice_fd_in))
4296 return -EINVAL;4495 return -EINVAL;
4297 4496
@@ -4331,9 +4530,6 @@ static int __io_splice_prep(struct io_kiocb *req, · 33 unmodified lines
4331 struct io_splice *sp = &req->splice;4530 struct io_splice *sp = &req->splice;
4332 unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;4531 unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
4333 4532
4334 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4335 return -EINVAL;
4336
4337 sp->len = READ_ONCE(sqe->len);4533 sp->len = READ_ONCE(sqe->len);
4338 sp->flags = READ_ONCE(sqe->splice_flags);4534 sp->flags = READ_ONCE(sqe->splice_flags);
4339 if (unlikely(sp->flags & ~valid_flags))4535 if (unlikely(sp->flags & ~valid_flags))
@@ -4432,20 +4628,25 @@ done: · 92 unmodified lines
4432 */4628 */
4433static int io_nop(struct io_kiocb *req, unsigned int issue_flags)4629static int io_nop(struct io_kiocb *req, unsigned int issue_flags)
4434{4630{
4435 struct io_ring_ctx *ctx = req->ctx;4631 void __user *buf;
4436 4632
4437 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))4633 if (req->flags & REQ_F_BUFFER_SELECT) {
4438 return -EINVAL;4634 size_t len = 1;
4635
4636 buf = io_buffer_select(req, &len, issue_flags);
4637 if (!buf)
4638 return -ENOBUFS;
4639 }
4439 4640
4440 __io_req_complete(req, issue_flags, 0, 0);4641 __io_req_complete(req, issue_flags, 0, io_put_kbuf(req, issue_flags));
4441 return 0;4642 return 0;
4442}4643}
4443 4644
4444static int io_msg_ring_prep(struct io_kiocb *req,4645static int io_msg_ring_prep(struct io_kiocb *req,
4445 const struct io_uring_sqe *sqe)4646 const struct io_uring_sqe *sqe)
4446{4647{
4447 if (unlikely(sqe->addr || sqe->ioprio || sqe->rw_flags ||4648 if (unlikely(sqe->addr || sqe->rw_flags || sqe->splice_fd_in ||
4448 sqe->splice_fd_in || sqe->buf_index || sqe->personality))4649 sqe->buf_index || sqe->personality))
4449 return -EINVAL;4650 return -EINVAL;
4450 4651
4451 req->msg.user_data = READ_ONCE(sqe->off);4652 req->msg.user_data = READ_ONCE(sqe->off);
@@ -4489,12 +4690,7 @@ done: · 37 unmodified lines
4489 4690
4490static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)4691static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4491{4692{
4492 struct io_ring_ctx *ctx = req->ctx;4693 if (unlikely(sqe->addr || sqe->buf_index || sqe->splice_fd_in))
4493
4494 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
4495 return -EINVAL;
4496 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index ||
4497 sqe->splice_fd_in))
4498 return -EINVAL;4694 return -EINVAL;
4499 4695
4500 req->sync.flags = READ_ONCE(sqe->fsync_flags);4696 req->sync.flags = READ_ONCE(sqe->fsync_flags);
@@ -4527,10 +4723,7 @@ static int io_fsync(struct io_kiocb *req, unsigned int issue_flags) · 26 unmodified lines
4527static int io_fallocate_prep(struct io_kiocb *req,4723static int io_fallocate_prep(struct io_kiocb *req,
4528 const struct io_uring_sqe *sqe)4724 const struct io_uring_sqe *sqe)
4529{4725{
4530 if (sqe->ioprio || sqe->buf_index || sqe->rw_flags ||4726 if (sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
4531 sqe->splice_fd_in)
4532 return -EINVAL;
4533 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4534 return -EINVAL;4727 return -EINVAL;
4535 4728
4536 req->sync.off = READ_ONCE(sqe->off);4729 req->sync.off = READ_ONCE(sqe->off);
@@ -4561,9 +4754,7 @@ static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe · 24 unmodified lines
4561 const char __user *fname;4754 const char __user *fname;
4562 int ret;4755 int ret;
4563 4756
4564 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))4757 if (unlikely(sqe->buf_index))
4565 return -EINVAL;
4566 if (unlikely(sqe->ioprio || sqe->buf_index))
4567 return -EINVAL;4758 return -EINVAL;
4568 if (unlikely(req->flags & REQ_F_FIXED_FILE))4759 if (unlikely(req->flags & REQ_F_FIXED_FILE))
4569 return -EBADF;4760 return -EBADF;
@@ -4618,6 +4809,61 @@ static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 48 unmodified lines
4618 return __io_openat_prep(req, sqe);4809 return __io_openat_prep(req, sqe);
4619}4810}
4620 4811
4812static int io_file_bitmap_get(struct io_ring_ctx *ctx)
4813{
4814 struct io_file_table *table = &ctx->file_table;
4815 unsigned long nr = ctx->nr_user_files;
4816 int ret;
4817
4818 if (table->alloc_hint >= nr)
4819 table->alloc_hint = 0;
4820
4821 do {
4822 ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint);
4823 if (ret != nr) {
4824 table->alloc_hint = ret + 1;
4825 return ret;
4826 }
4827 if (!table->alloc_hint)
4828 break;
4829
4830 nr = table->alloc_hint;
4831 table->alloc_hint = 0;
4832 } while (1);
4833
4834 return -ENFILE;
4835}
4836
4837static int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags,
4838 struct file *file, unsigned int file_slot)
4839{
4840 bool alloc_slot = file_slot == IORING_FILE_INDEX_ALLOC;
4841 struct io_ring_ctx *ctx = req->ctx;
4842 int ret;
4843
4844 if (alloc_slot) {
4845 io_ring_submit_lock(ctx, issue_flags);
4846 ret = io_file_bitmap_get(ctx);
4847 if (unlikely(ret < 0)) {
4848 io_ring_submit_unlock(ctx, issue_flags);
4849 return ret;
4850 }
4851
4852 file_slot = ret;
4853 } else {
4854 file_slot--;
4855 }
4856
4857 ret = io_install_fixed_file(req, file, issue_flags, file_slot);
4858 if (alloc_slot) {
4859 io_ring_submit_unlock(ctx, issue_flags);
4860 if (!ret)
4861 return file_slot;
4862 }
4863
4864 return ret;
4865}
4866
4621static int io_openat2(struct io_kiocb *req, unsigned int issue_flags)4867static int io_openat2(struct io_kiocb *req, unsigned int issue_flags)
4622{4868{
4623 struct open_flags op;4869 struct open_flags op;
@@ -4673,8 +4919,8 @@ static int io_openat2(struct io_kiocb *req, unsigned int issue_flags) · 49 unmodified lines
4673 if (!fixed)4919 if (!fixed)
4674 fd_install(ret, file);4920 fd_install(ret, file);
4675 else4921 else
4676 ret = io_install_fixed_file(req, file, issue_flags,4922 ret = io_fixed_fd_install(req, issue_flags, file,
4677 req->open.file_slot - 1);4923 req->open.file_slot);
4678err:4924err:
4679 putname(req->open.filename);4925 putname(req->open.filename);
4680 req->flags &= ~REQ_F_NEED_CLEANUP;4926 req->flags &= ~REQ_F_NEED_CLEANUP;
@@ -4695,7 +4941,7 @@ static int io_remove_buffers_prep(struct io_kiocb *req, · 14 unmodified lines
4695 struct io_provide_buf *p = &req->pbuf;4941 struct io_provide_buf *p = &req->pbuf;
4696 u64 tmp;4942 u64 tmp;
4697 4943
4698 if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off ||4944 if (sqe->rw_flags || sqe->addr || sqe->len || sqe->off ||
4699 sqe->splice_fd_in)4945 sqe->splice_fd_in)
4700 return -EINVAL;4946 return -EINVAL;
4701 4947
@@ -4718,6 +4964,20 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx, · 16 unmodified lines
4718 if (!nbufs)4964 if (!nbufs)
4719 return 0;4965 return 0;
4720 4966
4967 if (bl->buf_nr_pages) {
4968 int j;
4969
4970 i = bl->buf_ring->tail - bl->head;
4971 for (j = 0; j < bl->buf_nr_pages; j++)
4972 unpin_user_page(bl->buf_pages[j]);
4973 kvfree(bl->buf_pages);
4974 bl->buf_pages = NULL;
4975 bl->buf_nr_pages = 0;
4976 /* make sure it's seen as empty */
4977 INIT_LIST_HEAD(&bl->buf_list);
4978 return i;
4979 }
4980
4721 /* the head kbuf is the list itself */4981 /* the head kbuf is the list itself */
4722 while (!list_empty(&bl->buf_list)) {4982 while (!list_empty(&bl->buf_list)) {
4723 struct io_buffer *nxt;4983 struct io_buffer *nxt;
@@ -4739,22 +4999,23 @@ static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) · 15 unmodified lines
4739 struct io_ring_ctx *ctx = req->ctx;4999 struct io_ring_ctx *ctx = req->ctx;
4740 struct io_buffer_list *bl;5000 struct io_buffer_list *bl;
4741 int ret = 0;5001 int ret = 0;
4742 bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
4743
4744 io_ring_submit_lock(ctx, needs_lock);
4745 5002
4746 lockdep_assert_held(&ctx->uring_lock);5003 io_ring_submit_lock(ctx, issue_flags);
4747 5004
4748 ret = -ENOENT;5005 ret = -ENOENT;
4749 bl = io_buffer_get_list(ctx, p->bgid);5006 bl = io_buffer_get_list(ctx, p->bgid);
4750 if (bl)5007 if (bl) {
4751 ret = __io_remove_buffers(ctx, bl, p->nbufs);5008 ret = -EINVAL;
5009 /* can't use provide/remove buffers command on mapped buffers */
5010 if (!bl->buf_nr_pages)
5011 ret = __io_remove_buffers(ctx, bl, p->nbufs);
5012 }
4752 if (ret < 0)5013 if (ret < 0)
4753 req_set_fail(req);5014 req_set_fail(req);
4754 5015
4755 /* complete before unlock, IOPOLL may need the lock */5016 /* complete before unlock, IOPOLL may need the lock */
4756 __io_req_complete(req, issue_flags, ret, 0);5017 __io_req_complete(req, issue_flags, ret, 0);
4757 io_ring_submit_unlock(ctx, needs_lock);5018 io_ring_submit_unlock(ctx, issue_flags);
4758 return 0;5019 return 0;
4759}5020}
4760 5021
@@ -4765,7 +5026,7 @@ static int io_provide_buffers_prep(struct io_kiocb *req, · 4 unmodified lines
4765 struct io_provide_buf *p = &req->pbuf;5026 struct io_provide_buf *p = &req->pbuf;
4766 u64 tmp;5027 u64 tmp;
4767 5028
4768 if (sqe->ioprio || sqe->rw_flags || sqe->splice_fd_in)5029 if (sqe->rw_flags || sqe->splice_fd_in)
4769 return -EINVAL;5030 return -EINVAL;
4770 5031
4771 tmp = READ_ONCE(sqe->fd);5032 tmp = READ_ONCE(sqe->fd);
@@ -4862,26 +5123,56 @@ static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf, · 90 unmodified lines
4862 return i ? 0 : -ENOMEM;5123 return i ? 0 : -ENOMEM;
4863}5124}
4864 5125
5126static __cold int io_init_bl_list(struct io_ring_ctx *ctx)
5127{
5128 int i;
5129
5130 ctx->io_bl = kcalloc(BGID_ARRAY, sizeof(struct io_buffer_list),
5131 GFP_KERNEL);
5132 if (!ctx->io_bl)
5133 return -ENOMEM;
5134
5135 for (i = 0; i < BGID_ARRAY; i++) {
5136 INIT_LIST_HEAD(&ctx->io_bl[i].buf_list);
5137 ctx->io_bl[i].bgid = i;
5138 }
5139
5140 return 0;
5141}
5142
4865static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)5143static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
4866{5144{
4867 struct io_provide_buf *p = &req->pbuf;5145 struct io_provide_buf *p = &req->pbuf;
4868 struct io_ring_ctx *ctx = req->ctx;5146 struct io_ring_ctx *ctx = req->ctx;
4869 struct io_buffer_list *bl;5147 struct io_buffer_list *bl;
4870 int ret = 0;5148 int ret = 0;
4871 bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
4872 5149
4873 io_ring_submit_lock(ctx, needs_lock);5150 io_ring_submit_lock(ctx, issue_flags);
4874 5151
4875 lockdep_assert_held(&ctx->uring_lock);5152 if (unlikely(p->bgid < BGID_ARRAY && !ctx->io_bl)) {
5153 ret = io_init_bl_list(ctx);
5154 if (ret)
5155 goto err;
5156 }
4876 5157
4877 bl = io_buffer_get_list(ctx, p->bgid);5158 bl = io_buffer_get_list(ctx, p->bgid);
4878 if (unlikely(!bl)) {5159 if (unlikely(!bl)) {
4879 bl = kmalloc(sizeof(*bl), GFP_KERNEL);5160 bl = kzalloc(sizeof(*bl), GFP_KERNEL);
4880 if (!bl) {5161 if (!bl) {
4881 ret = -ENOMEM;5162 ret = -ENOMEM;
4882 goto err;5163 goto err;
4883 }5164 }
4884 io_buffer_add_list(ctx, bl, p->bgid);5165 INIT_LIST_HEAD(&bl->buf_list);
5166 ret = io_buffer_add_list(ctx, bl, p->bgid);
5167 if (ret) {
5168 kfree(bl);
5169 goto err;
5170 }
5171 }
5172 /* can't add buffers via this command for a mapped buffer ring */
5173 if (bl->buf_nr_pages) {
5174 ret = -EINVAL;
5175 goto err;
4885 }5176 }
4886 5177
4887 ret = io_add_buffers(ctx, p, bl);5178 ret = io_add_buffers(ctx, p, bl);
@@ -4890,7 +5181,7 @@ err: · 2 unmodified lines
4890 req_set_fail(req);5181 req_set_fail(req);
4891 /* complete before unlock, IOPOLL may need the lock */5182 /* complete before unlock, IOPOLL may need the lock */
4892 __io_req_complete(req, issue_flags, ret, 0);5183 __io_req_complete(req, issue_flags, ret, 0);
4893 io_ring_submit_unlock(ctx, needs_lock);5184 io_ring_submit_unlock(ctx, issue_flags);
4894 return 0;5185 return 0;
4895}5186}
4896 5187
@@ -4898,9 +5189,7 @@ static int io_epoll_ctl_prep(struct io_kiocb *req, · 1 unmodified line
4898 const struct io_uring_sqe *sqe)5189 const struct io_uring_sqe *sqe)
4899{5190{
4900#if defined(CONFIG_EPOLL)5191#if defined(CONFIG_EPOLL)
4901 if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)5192 if (sqe->buf_index || sqe->splice_fd_in)
4902 return -EINVAL;
4903 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4904 return -EINVAL;5193 return -EINVAL;
4905 5194
4906 req->epoll.epfd = READ_ONCE(sqe->fd);5195 req->epoll.epfd = READ_ONCE(sqe->fd);
@@ -4944,9 +5233,7 @@ static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags) · 37 unmodified lines
4944static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)5233static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4945{5234{
4946#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)5235#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
4947 if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->splice_fd_in)5236 if (sqe->buf_index || sqe->off || sqe->splice_fd_in)
4948 return -EINVAL;
4949 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4950 return -EINVAL;5237 return -EINVAL;
4951 5238
4952 req->madvise.addr = READ_ONCE(sqe->addr);5239 req->madvise.addr = READ_ONCE(sqe->addr);
@@ -4979,9 +5266,7 @@ static int io_madvise(struct io_kiocb *req, unsigned int issue_flags) · 26 unmodified lines
4979 5266
4980static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)5267static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4981{5268{
4982 if (sqe->ioprio || sqe->buf_index || sqe->addr || sqe->splice_fd_in)5269 if (sqe->buf_index || sqe->addr || sqe->splice_fd_in)
4983 return -EINVAL;
4984 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4985 return -EINVAL;5270 return -EINVAL;
4986 5271
4987 req->fadvise.offset = READ_ONCE(sqe->off);5272 req->fadvise.offset = READ_ONCE(sqe->off);
@@ -5017,9 +5302,7 @@ static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 29 unmodified lines
5017{5302{
5018 const char __user *path;5303 const char __user *path;
5019 5304
5020 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))5305 if (sqe->buf_index || sqe->splice_fd_in)
5021 return -EINVAL;
5022 if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)
5023 return -EINVAL;5306 return -EINVAL;
5024 if (req->flags & REQ_F_FIXED_FILE)5307 if (req->flags & REQ_F_FIXED_FILE)
5025 return -EBADF;5308 return -EBADF;
@@ -5064,10 +5347,7 @@ static int io_statx(struct io_kiocb *req, unsigned int issue_flags) · 38 unmodified lines
5064 5347
5065static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)5348static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5066{5349{
5067 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))5350 if (sqe->off || sqe->addr || sqe->len || sqe->rw_flags || sqe->buf_index)
5068 return -EINVAL;
5069 if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
5070 sqe->rw_flags || sqe->buf_index)
5071 return -EINVAL;5351 return -EINVAL;
5072 if (req->flags & REQ_F_FIXED_FILE)5352 if (req->flags & REQ_F_FIXED_FILE)
5073 return -EBADF;5353 return -EBADF;
@@ -5099,7 +5379,8 @@ static int io_close(struct io_kiocb *req, unsigned int issue_flags) · 25 unmodified lines
5099 spin_unlock(&files->file_lock);5379 spin_unlock(&files->file_lock);
5100 goto err;5380 goto err;
5101 }5381 }
5102 file = fdt->fd[close->fd];5382 file = rcu_dereference_protected(fdt->fd[close->fd],
5383 lockdep_is_held(&files->file_lock));
5103 if (!file || file->f_op == &io_uring_fops) {5384 if (!file || file->f_op == &io_uring_fops) {
5104 spin_unlock(&files->file_lock);5385 spin_unlock(&files->file_lock);
5105 file = NULL;5386 file = NULL;
@@ -5133,12 +5414,7 @@ err: · 27 unmodified lines
5133 5414
5134static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)5415static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5135{5416{
5136 struct io_ring_ctx *ctx = req->ctx;5417 if (unlikely(sqe->addr || sqe->buf_index || sqe->splice_fd_in))
5137
5138 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
5139 return -EINVAL;
5140 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index ||
5141 sqe->splice_fd_in))
5142 return -EINVAL;5418 return -EINVAL;
5143 5419
5144 req->sync.off = READ_ONCE(sqe->off);5420 req->sync.off = READ_ONCE(sqe->off);
@@ -5164,6 +5440,13 @@ static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags) · 19 unmodified lines
5164}5440}
5165 5441
5166#if defined(CONFIG_NET)5442#if defined(CONFIG_NET)
5443static bool io_net_retry(struct socket *sock, int flags)
5444{
5445 if (!(flags & MSG_WAITALL))
5446 return false;
5447 return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET;
5448}
5449
5167static int io_setup_async_msg(struct io_kiocb *req,5450static int io_setup_async_msg(struct io_kiocb *req,
5168 struct io_async_msghdr *kmsg)5451 struct io_async_msghdr *kmsg)
5169{5452{
@@ -5209,13 +5492,16 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 39 unmodified lines
5209{5492{
5210 struct io_sr_msg *sr = &req->sr_msg;5493 struct io_sr_msg *sr = &req->sr_msg;
5211 5494
5212 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))5495 if (unlikely(sqe->file_index))
5213 return -EINVAL;5496 return -EINVAL;
5214 if (unlikely(sqe->addr2 || sqe->file_index))5497 if (unlikely(sqe->addr2 || sqe->file_index))
5215 return -EINVAL;5498 return -EINVAL;
5216 5499
5217 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));5500 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
5218 sr->len = READ_ONCE(sqe->len);5501 sr->len = READ_ONCE(sqe->len);
5502 sr->flags = READ_ONCE(sqe->addr2);
5503 if (sr->flags & ~IORING_RECVSEND_POLL_FIRST)
5504 return -EINVAL;
5219 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;5505 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
5220 if (sr->msg_flags & MSG_DONTWAIT)5506 if (sr->msg_flags & MSG_DONTWAIT)
5221 req->flags |= REQ_F_NOWAIT;5507 req->flags |= REQ_F_NOWAIT;
@@ -5224,12 +5510,14 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 2 unmodified lines
5224 if (req->ctx->compat)5510 if (req->ctx->compat)
5225 sr->msg_flags |= MSG_CMSG_COMPAT;5511 sr->msg_flags |= MSG_CMSG_COMPAT;
5226#endif5512#endif
5513 sr->done_io = 0;
5227 return 0;5514 return 0;
5228}5515}
5229 5516
5230static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)5517static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
5231{5518{
5232 struct io_async_msghdr iomsg, *kmsg;5519 struct io_async_msghdr iomsg, *kmsg;
5520 struct io_sr_msg *sr = &req->sr_msg;
5233 struct socket *sock;5521 struct socket *sock;
5234 unsigned flags;5522 unsigned flags;
5235 int min_ret = 0;5523 int min_ret = 0;
@@ -5248,7 +5536,11 @@ static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) · 12 unmodified lines
5248 kmsg = &iomsg;5536 kmsg = &iomsg;
5249 }5537 }
5250 5538
5251 flags = req->sr_msg.msg_flags;5539 if (!(req->flags & REQ_F_POLLED) &&
5540 (sr->flags & IORING_RECVSEND_POLL_FIRST))
5541 return io_setup_async_msg(req, kmsg);
5542
5543 flags = sr->msg_flags;
5252 if (issue_flags & IO_URING_F_NONBLOCK)5544 if (issue_flags & IO_URING_F_NONBLOCK)
5253 flags |= MSG_DONTWAIT;5545 flags |= MSG_DONTWAIT;
5254 if (flags & MSG_WAITALL)5546 if (flags & MSG_WAITALL)
@@ -5261,12 +5553,21 @@ static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) · 6 unmodified lines
5261 return io_setup_async_msg(req, kmsg);5553 return io_setup_async_msg(req, kmsg);
5262 if (ret == -ERESTARTSYS)5554 if (ret == -ERESTARTSYS)
5263 ret = -EINTR;5555 ret = -EINTR;
5556 if (ret > 0 && io_net_retry(sock, flags)) {
5557 sr->done_io += ret;
5558 req->flags |= REQ_F_PARTIAL_IO;
5559 return io_setup_async_msg(req, kmsg);
5560 }
5264 req_set_fail(req);5561 req_set_fail(req);
5265 }5562 }
5266 /* fast path, check for non-NULL to avoid function call */5563 /* fast path, check for non-NULL to avoid function call */
5267 if (kmsg->free_iov)5564 if (kmsg->free_iov)
5268 kfree(kmsg->free_iov);5565 kfree(kmsg->free_iov);
5269 req->flags &= ~REQ_F_NEED_CLEANUP;5566 req->flags &= ~REQ_F_NEED_CLEANUP;
5567 if (ret >= 0)
5568 ret += sr->done_io;
5569 else if (sr->done_io)
5570 ret = sr->done_io;
5270 __io_req_complete(req, issue_flags, ret, 0);5571 __io_req_complete(req, issue_flags, ret, 0);
5271 return 0;5572 return 0;
5272}5573}
@@ -5281,6 +5582,10 @@ static int io_send(struct io_kiocb *req, unsigned int issue_flags) · 8 unmodified lines
5281 int min_ret = 0;5582 int min_ret = 0;
5282 int ret;5583 int ret;
5283 5584
5585 if (!(req->flags & REQ_F_POLLED) &&
5586 (sr->flags & IORING_RECVSEND_POLL_FIRST))
5587 return -EAGAIN;
5588
5284 sock = sock_from_file(req->file);5589 sock = sock_from_file(req->file);
5285 if (unlikely(!sock))5590 if (unlikely(!sock))
5286 return -ENOTSOCK;5591 return -ENOTSOCK;
@@ -5294,7 +5599,7 @@ static int io_send(struct io_kiocb *req, unsigned int issue_flags) · 7 unmodified lines
5294 msg.msg_controllen = 0;5599 msg.msg_controllen = 0;
5295 msg.msg_namelen = 0;5600 msg.msg_namelen = 0;
5296 5601
5297 flags = req->sr_msg.msg_flags;5602 flags = sr->msg_flags;
5298 if (issue_flags & IO_URING_F_NONBLOCK)5603 if (issue_flags & IO_URING_F_NONBLOCK)
5299 flags |= MSG_DONTWAIT;5604 flags |= MSG_DONTWAIT;
5300 if (flags & MSG_WAITALL)5605 if (flags & MSG_WAITALL)
@@ -5307,8 +5612,19 @@ static int io_send(struct io_kiocb *req, unsigned int issue_flags) · 6 unmodified lines
5307 return -EAGAIN;5612 return -EAGAIN;
5308 if (ret == -ERESTARTSYS)5613 if (ret == -ERESTARTSYS)
5309 ret = -EINTR;5614 ret = -EINTR;
5615 if (ret > 0 && io_net_retry(sock, flags)) {
5616 sr->len -= ret;
5617 sr->buf += ret;
5618 sr->done_io += ret;
5619 req->flags |= REQ_F_PARTIAL_IO;
5620 return -EAGAIN;
5621 }
5310 req_set_fail(req);5622 req_set_fail(req);
5311 }5623 }
5624 if (ret >= 0)
5625 ret += sr->done_io;
5626 else if (sr->done_io)
5627 ret = sr->done_io;
5312 __io_req_complete(req, issue_flags, ret, 0);5628 __io_req_complete(req, issue_flags, ret, 0);
5313 return 0;5629 return 0;
5314}5630}
@@ -5400,14 +5716,6 @@ static int io_recvmsg_copy_hdr(struct io_kiocb *req, · 85 unmodified lines
5400 return __io_recvmsg_copy_hdr(req, iomsg);5716 return __io_recvmsg_copy_hdr(req, iomsg);
5401}5717}
5402 5718
5403static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,
5404 unsigned int issue_flags)
5405{
5406 struct io_sr_msg *sr = &req->sr_msg;
5407
5408 return io_buffer_select(req, &sr->len, sr->bgid, issue_flags);
5409}
5410
5411static int io_recvmsg_prep_async(struct io_kiocb *req)5719static int io_recvmsg_prep_async(struct io_kiocb *req)
5412{5720{
5413 int ret;5721 int ret;
@@ -5422,14 +5730,16 @@ static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 8 unmodified lines
5422{5730{
5423 struct io_sr_msg *sr = &req->sr_msg;5731 struct io_sr_msg *sr = &req->sr_msg;
5424 5732
5425 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))5733 if (unlikely(sqe->file_index))
5426 return -EINVAL;5734 return -EINVAL;
5427 if (unlikely(sqe->addr2 || sqe->file_index))5735 if (unlikely(sqe->addr2 || sqe->file_index))
5428 return -EINVAL;5736 return -EINVAL;
5429 5737
5430 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));5738 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
5431 sr->len = READ_ONCE(sqe->len);5739 sr->len = READ_ONCE(sqe->len);
5432 sr->bgid = READ_ONCE(sqe->buf_group);5740 sr->flags = READ_ONCE(sqe->addr2);
5741 if (sr->flags & ~IORING_RECVSEND_POLL_FIRST)
5742 return -EINVAL;
5433 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;5743 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
5434 if (sr->msg_flags & MSG_DONTWAIT)5744 if (sr->msg_flags & MSG_DONTWAIT)
5435 req->flags |= REQ_F_NOWAIT;5745 req->flags |= REQ_F_NOWAIT;
@@ -5442,19 +5752,11 @@ static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 6 unmodified lines
5442 return 0;5752 return 0;
5443}5753}
5444 5754
5445static bool io_net_retry(struct socket *sock, int flags)
5446{
5447 if (!(flags & MSG_WAITALL))
5448 return false;
5449 return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET;
5450}
5451
5452static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)5755static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
5453{5756{
5454 struct io_async_msghdr iomsg, *kmsg;5757 struct io_async_msghdr iomsg, *kmsg;
5455 struct io_sr_msg *sr = &req->sr_msg;5758 struct io_sr_msg *sr = &req->sr_msg;
5456 struct socket *sock;5759 struct socket *sock;
5457 struct io_buffer *kbuf;
5458 unsigned flags;5760 unsigned flags;
5459 int ret, min_ret = 0;5761 int ret, min_ret = 0;
5460 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;5762 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
@@ -5472,24 +5774,29 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) · 11 unmodified lines
5472 kmsg = &iomsg;5774 kmsg = &iomsg;
5473 }5775 }
5474 5776
5475 if (req->flags & REQ_F_BUFFER_SELECT) {5777 if (!(req->flags & REQ_F_POLLED) &&
5476 kbuf = io_recv_buffer_select(req, issue_flags);5778 (sr->flags & IORING_RECVSEND_POLL_FIRST))
5477 if (IS_ERR(kbuf))5779 return io_setup_async_msg(req, kmsg);
5478 return PTR_ERR(kbuf);5780
5479 kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);5781 if (io_do_buffer_select(req)) {
5480 kmsg->fast_iov[0].iov_len = req->sr_msg.len;5782 void __user *buf;
5481 iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov,5783
5482 1, req->sr_msg.len);5784 buf = io_buffer_select(req, &sr->len, issue_flags);
5785 if (!buf)
5786 return -ENOBUFS;
5787 kmsg->fast_iov[0].iov_base = buf;
5788 kmsg->fast_iov[0].iov_len = sr->len;
5789 iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov, 1,
5790 sr->len);
5483 }5791 }
5484 5792
5485 flags = req->sr_msg.msg_flags;5793 flags = sr->msg_flags;
5486 if (force_nonblock)5794 if (force_nonblock)
5487 flags |= MSG_DONTWAIT;5795 flags |= MSG_DONTWAIT;
5488 if (flags & MSG_WAITALL)5796 if (flags & MSG_WAITALL)
5489 min_ret = iov_iter_count(&kmsg->msg.msg_iter);5797 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
5490 5798
5491 ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg,5799 ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, kmsg->uaddr, flags);
5492 kmsg->uaddr, flags);
5493 if (ret < min_ret) {5800 if (ret < min_ret) {
5494 if (ret == -EAGAIN && force_nonblock)5801 if (ret == -EAGAIN && force_nonblock)
5495 return io_setup_async_msg(req, kmsg);5802 return io_setup_async_msg(req, kmsg);
@@ -5519,28 +5826,32 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) · 23 unmodified lines
5519 5826
5520static int io_recv(struct io_kiocb *req, unsigned int issue_flags)5827static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
5521{5828{
5522 struct io_buffer *kbuf;
5523 struct io_sr_msg *sr = &req->sr_msg;5829 struct io_sr_msg *sr = &req->sr_msg;
5524 struct msghdr msg;5830 struct msghdr msg;
5525 void __user *buf = sr->buf;
5526 struct socket *sock;5831 struct socket *sock;
5527 struct iovec iov;5832 struct iovec iov;
5528 unsigned flags;5833 unsigned flags;
5529 int ret, min_ret = 0;5834 int ret, min_ret = 0;
5530 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;5835 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
5531 5836
5837 if (!(req->flags & REQ_F_POLLED) &&
5838 (sr->flags & IORING_RECVSEND_POLL_FIRST))
5839 return -EAGAIN;
5840
5532 sock = sock_from_file(req->file);5841 sock = sock_from_file(req->file);
5533 if (unlikely(!sock))5842 if (unlikely(!sock))
5534 return -ENOTSOCK;5843 return -ENOTSOCK;
5535 5844
5536 if (req->flags & REQ_F_BUFFER_SELECT) {5845 if (io_do_buffer_select(req)) {
5537 kbuf = io_recv_buffer_select(req, issue_flags);5846 void __user *buf;
5538 if (IS_ERR(kbuf))5847
5539 return PTR_ERR(kbuf);5848 buf = io_buffer_select(req, &sr->len, issue_flags);
5540 buf = u64_to_user_ptr(kbuf->addr);5849 if (!buf)
5850 return -ENOBUFS;
5851 sr->buf = buf;
5541 }5852 }
5542 5853
5543 ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter);5854 ret = import_single_range(READ, sr->buf, sr->len, &iov, &msg.msg_iter);
5544 if (unlikely(ret))5855 if (unlikely(ret))
5545 goto out_free;5856 goto out_free;
5546 5857
@@ -5551,7 +5862,7 @@ static int io_recv(struct io_kiocb *req, unsigned int issue_flags) · 4 unmodified lines
5551 msg.msg_iocb = NULL;5862 msg.msg_iocb = NULL;
5552 msg.msg_flags = 0;5863 msg.msg_flags = 0;
5553 5864
5554 flags = req->sr_msg.msg_flags;5865 flags = sr->msg_flags;
5555 if (force_nonblock)5866 if (force_nonblock)
5556 flags |= MSG_DONTWAIT;5867 flags |= MSG_DONTWAIT;
5557 if (flags & MSG_WAITALL)5868 if (flags & MSG_WAITALL)
@@ -5587,29 +5898,39 @@ out_free: · 29 unmodified lines
5587static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)5898static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5588{5899{
5589 struct io_accept *accept = &req->accept;5900 struct io_accept *accept = &req->accept;
5901 unsigned flags;
5590 5902
5591 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))5903 if (sqe->len || sqe->buf_index)
5592 return -EINVAL;
5593 if (sqe->ioprio || sqe->len || sqe->buf_index)
5594 return -EINVAL;5904 return -EINVAL;
5595 5905
5596 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));5906 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
5597 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));5907 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
5598 accept->flags = READ_ONCE(sqe->accept_flags);5908 accept->flags = READ_ONCE(sqe->accept_flags);
5599 accept->nofile = rlimit(RLIMIT_NOFILE);5909 accept->nofile = rlimit(RLIMIT_NOFILE);
5910 flags = READ_ONCE(sqe->ioprio);
5911 if (flags & ~IORING_ACCEPT_MULTISHOT)
5912 return -EINVAL;
5600 5913
5601 accept->file_slot = READ_ONCE(sqe->file_index);5914 accept->file_slot = READ_ONCE(sqe->file_index);
5602 if (accept->file_slot && (accept->flags & SOCK_CLOEXEC))5915 if (accept->file_slot) {
5603 return -EINVAL;5916 if (accept->flags & SOCK_CLOEXEC)
5917 return -EINVAL;
5918 if (flags & IORING_ACCEPT_MULTISHOT &&
5919 accept->file_slot != IORING_FILE_INDEX_ALLOC)
5920 return -EINVAL;
5921 }
5604 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))5922 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
5605 return -EINVAL;5923 return -EINVAL;
5606 if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK))5924 if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK))
5607 accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK;5925 accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK;
5926 if (flags & IORING_ACCEPT_MULTISHOT)
5927 req->flags |= REQ_F_APOLL_MULTISHOT;
5608 return 0;5928 return 0;
5609}5929}
5610 5930
5611static int io_accept(struct io_kiocb *req, unsigned int issue_flags)5931static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
5612{5932{
5933 struct io_ring_ctx *ctx = req->ctx;
5613 struct io_accept *accept = &req->accept;5934 struct io_accept *accept = &req->accept;
5614 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;5935 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
5615 unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;5936 unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
@@ -5617,6 +5938,7 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags) · 1 unmodified line
5617 struct file *file;5938 struct file *file;
5618 int ret, fd;5939 int ret, fd;
5619 5940
5941retry:
5620 if (!fixed) {5942 if (!fixed) {
5621 fd = __get_unused_fd_flags(accept->flags, accept->nofile);5943 fd = __get_unused_fd_flags(accept->flags, accept->nofile);
5622 if (unlikely(fd < 0))5944 if (unlikely(fd < 0))
@@ -5628,8 +5950,17 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags) · 5 unmodified lines
5628 if (!fixed)5950 if (!fixed)
5629 put_unused_fd(fd);5951 put_unused_fd(fd);
5630 ret = PTR_ERR(file);5952 ret = PTR_ERR(file);
5631 if (ret == -EAGAIN && force_nonblock)5953 if (ret == -EAGAIN && force_nonblock) {
5632 return -EAGAIN;5954 /*
5955 * if it's multishot and polled, we don't need to
5956 * return EAGAIN to arm the poll infra since it
5957 * has already been done
5958 */
5959 if ((req->flags & IO_APOLL_MULTI_POLLED) ==
5960 IO_APOLL_MULTI_POLLED)
5961 ret = 0;
5962 return ret;
5963 }
5633 if (ret == -ERESTARTSYS)5964 if (ret == -ERESTARTSYS)
5634 ret = -EINTR;5965 ret = -EINTR;
5635 req_set_fail(req);5966 req_set_fail(req);
@@ -5637,11 +5968,30 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags) · 1 unmodified line
5637 fd_install(fd, file);5968 fd_install(fd, file);
5638 ret = fd;5969 ret = fd;
5639 } else {5970 } else {
5640 ret = io_install_fixed_file(req, file, issue_flags,5971 ret = io_fixed_fd_install(req, issue_flags, file,
5641 accept->file_slot - 1);5972 accept->file_slot);
5642 }5973 }
5643 __io_req_complete(req, issue_flags, ret, 0);5974
5644 return 0;5975 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
5976 __io_req_complete(req, issue_flags, ret, 0);
5977 return 0;
5978 }
5979 if (ret >= 0) {
5980 bool filled;
5981
5982 spin_lock(&ctx->completion_lock);
5983 filled = io_fill_cqe_aux(ctx, req->cqe.user_data, ret,
5984 IORING_CQE_F_MORE);
5985 io_commit_cqring(ctx);
5986 spin_unlock(&ctx->completion_lock);
5987 if (filled) {
5988 io_cqring_ev_posted(ctx);
5989 goto retry;
5990 }
5991 ret = -ECANCELED;
5992 }
5993
5994 return ret;
5645}5995}
5646 5996
5647static int io_connect_prep_async(struct io_kiocb *req)5997static int io_connect_prep_async(struct io_kiocb *req)
@@ -5656,10 +6006,7 @@ static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 8 unmodified lines
5656{6006{
5657 struct io_connect *conn = &req->connect;6007 struct io_connect *conn = &req->connect;
5658 6008
5659 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))6009 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
5660 return -EINVAL;
5661 if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags ||
5662 sqe->splice_fd_in)
5663 return -EINVAL;6010 return -EINVAL;
5664 6011
5665 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));6012 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
@@ -5782,7 +6129,7 @@ static void io_poll_req_insert(struct io_kiocb *req) · 116 unmodified lines
5782 struct io_ring_ctx *ctx = req->ctx;6129 struct io_ring_ctx *ctx = req->ctx;
5783 struct hlist_head *list;6130 struct hlist_head *list;
5784 6131
5785 list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];6132 list = &ctx->cancel_hash[hash_long(req->cqe.user_data, ctx->cancel_hash_bits)];
5786 hlist_add_head(&req->hash_node, list);6133 hlist_add_head(&req->hash_node, list);
5787}6134}
5788 6135
@@ -5841,22 +6188,23 @@ static void io_poll_remove_entries(struct io_kiocb *req) · 52 unmodified lines
5841 rcu_read_unlock();6188 rcu_read_unlock();
5842}6189}
5843 6190
6191static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags);
5844/*6192/*
5845 * All poll tw should go through this. Checks for poll events, manages6193 * All poll tw should go through this. Checks for poll events, manages
5846 * references, does rewait, etc.6194 * references, does rewait, etc.
5847 *6195 *
5848 * Returns a negative error on failure. >0 when no action require, which is6196 * Returns a negative error on failure. >0 when no action require, which is
5849 * either spurious wakeup or multishot CQE is served. 0 when it's done with6197 * either spurious wakeup or multishot CQE is served. 0 when it's done with
5850 * the request, then the mask is stored in req->result.6198 * the request, then the mask is stored in req->cqe.res.
5851 */6199 */
5852static int io_poll_check_events(struct io_kiocb *req, bool locked)6200static int io_poll_check_events(struct io_kiocb *req, bool *locked)
5853{6201{
5854 struct io_ring_ctx *ctx = req->ctx;6202 struct io_ring_ctx *ctx = req->ctx;
5855 int v;6203 int v, ret;
5856 6204
5857 /* req->task == current here, checking PF_EXITING is safe */6205 /* req->task == current here, checking PF_EXITING is safe */
5858 if (unlikely(req->task->flags & PF_EXITING))6206 if (unlikely(req->task->flags & PF_EXITING))
5859 io_poll_mark_cancelled(req);6207 return -ECANCELED;
5860 6208
5861 do {6209 do {
5862 v = atomic_read(&req->poll_refs);6210 v = atomic_read(&req->poll_refs);
@@ -5867,32 +6215,46 @@ static int io_poll_check_events(struct io_kiocb *req, bool locked) · 4 unmodified lines
5867 if (v & IO_POLL_CANCEL_FLAG)6215 if (v & IO_POLL_CANCEL_FLAG)
5868 return -ECANCELED;6216 return -ECANCELED;
5869 6217
5870 if (!req->result) {6218 if (!req->cqe.res) {
5871 struct poll_table_struct pt = { ._key = req->apoll_events };6219 struct poll_table_struct pt = { ._key = req->apoll_events };
5872 unsigned flags = locked ? 0 : IO_URING_F_UNLOCKED;6220 unsigned flags = locked ? 0 : IO_URING_F_UNLOCKED;
5873 6221
5874 if (unlikely(!io_assign_file(req, flags)))6222 if (unlikely(!io_assign_file(req, flags)))
5875 return -EBADF;6223 return -EBADF;
5876 req->result = vfs_poll(req->file, &pt) & req->apoll_events;6224 req->cqe.res = vfs_poll(req->file, &pt) & req->apoll_events;
5877 }6225 }
5878 6226
5879 /* multishot, just fill an CQE and proceed */6227 if ((unlikely(!req->cqe.res)))
5880 if (req->result && !(req->apoll_events & EPOLLONESHOT)) {6228 continue;
5881 __poll_t mask = mangle_poll(req->result & req->apoll_events);6229 if (req->apoll_events & EPOLLONESHOT)
6230 return 0;
6231
6232 /* multishot, just fill a CQE and proceed */
6233 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
6234 __poll_t mask = mangle_poll(req->cqe.res &
6235 req->apoll_events);
5882 bool filled;6236 bool filled;
5883 6237
5884 spin_lock(&ctx->completion_lock);6238 spin_lock(&ctx->completion_lock);
5885 filled = io_fill_cqe_aux(ctx, req->user_data, mask,6239 filled = io_fill_cqe_aux(ctx, req->cqe.user_data,
5886 IORING_CQE_F_MORE);6240 mask, IORING_CQE_F_MORE);
5887 io_commit_cqring(ctx);6241 io_commit_cqring(ctx);
5888 spin_unlock(&ctx->completion_lock);6242 spin_unlock(&ctx->completion_lock);
5889 if (unlikely(!filled))6243 if (filled) {
5890 return -ECANCELED;6244 io_cqring_ev_posted(ctx);
5891 io_cqring_ev_posted(ctx);6245 continue;
5892 } else if (req->result) {6246 }
5893 return 0;6247 return -ECANCELED;
5894 }6248 }
5895 6249
6250 io_tw_lock(req->ctx, locked);
6251 if (unlikely(req->task->flags & PF_EXITING))
6252 return -EFAULT;
6253 ret = io_issue_sqe(req,
6254 IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);
6255 if (ret)
6256 return ret;
6257
5896 /*6258 /*
5897 * Release all references, retry if someone tried to restart6259 * Release all references, retry if someone tried to restart
5898 * task_work while we were executing it.6260 * task_work while we were executing it.
@@ -5907,21 +6269,21 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked) · 8 unmodified lines
5907 struct io_ring_ctx *ctx = req->ctx;6269 struct io_ring_ctx *ctx = req->ctx;
5908 int ret;6270 int ret;
5909 6271
5910 ret = io_poll_check_events(req, *locked);6272 ret = io_poll_check_events(req, locked);
5911 if (ret > 0)6273 if (ret > 0)
5912 return;6274 return;
5913 6275
5914 if (!ret) {6276 if (!ret) {
5915 req->result = mangle_poll(req->result & req->poll.events);6277 req->cqe.res = mangle_poll(req->cqe.res & req->poll.events);
5916 } else {6278 } else {
5917 req->result = ret;6279 req->cqe.res = ret;
5918 req_set_fail(req);6280 req_set_fail(req);
5919 }6281 }
5920 6282
5921 io_poll_remove_entries(req);6283 io_poll_remove_entries(req);
5922 spin_lock(&ctx->completion_lock);6284 spin_lock(&ctx->completion_lock);
5923 hash_del(&req->hash_node);6285 hash_del(&req->hash_node);
5924 __io_req_complete_post(req, req->result, 0);6286 __io_req_complete_post(req, req->cqe.res, 0);
5925 io_commit_cqring(ctx);6287 io_commit_cqring(ctx);
5926 spin_unlock(&ctx->completion_lock);6288 spin_unlock(&ctx->completion_lock);
5927 io_cqring_ev_posted(ctx);6289 io_cqring_ev_posted(ctx);
@@ -5932,7 +6294,7 @@ static void io_apoll_task_func(struct io_kiocb *req, bool *locked) · 4 unmodified lines
5932 struct io_ring_ctx *ctx = req->ctx;6294 struct io_ring_ctx *ctx = req->ctx;
5933 int ret;6295 int ret;
5934 6296
5935 ret = io_poll_check_events(req, *locked);6297 ret = io_poll_check_events(req, locked);
5936 if (ret > 0)6298 if (ret > 0)
5937 return;6299 return;
5938 6300
@@ -5947,9 +6309,9 @@ static void io_apoll_task_func(struct io_kiocb *req, bool *locked) · 8 unmodified lines
5947 io_req_complete_failed(req, ret);6309 io_req_complete_failed(req, ret);
5948}6310}
5949 6311
5950static void __io_poll_execute(struct io_kiocb *req, int mask, int events)6312static void __io_poll_execute(struct io_kiocb *req, int mask, __poll_t events)
5951{6313{
5952 req->result = mask;6314 req->cqe.res = mask;
5953 /*6315 /*
5954 * This is useful for poll that is armed on behalf of another6316 * This is useful for poll that is armed on behalf of another
5955 * request, and where the wakeup path could be on a different6317 * request, and where the wakeup path could be on a different
@@ -5962,11 +6324,12 @@ static void __io_poll_execute(struct io_kiocb *req, int mask, int events) · 6 unmodified lines
5962 else6324 else
5963 req->io_task_work.func = io_apoll_task_func;6325 req->io_task_work.func = io_apoll_task_func;
5964 6326
5965 trace_io_uring_task_add(req->ctx, req, req->user_data, req->opcode, mask);6327 trace_io_uring_task_add(req->ctx, req, req->cqe.user_data, req->opcode, mask);
5966 io_req_task_work_add(req, false);6328 io_req_task_work_add(req, false);
5967}6329}
5968 6330
5969static inline void io_poll_execute(struct io_kiocb *req, int res, int events)6331static inline void io_poll_execute(struct io_kiocb *req, int res,
6332 __poll_t events)
5970{6333{
5971 if (io_poll_get_ownership(req))6334 if (io_poll_get_ownership(req))
5972 __io_poll_execute(req, res, events);6335 __io_poll_execute(req, res, events);
@@ -5981,6 +6344,7 @@ static void io_poll_cancel_req(struct io_kiocb *req) · 8 unmodified lines
5981 6344
5982#define wqe_to_req(wait) ((void *)((unsigned long) (wait)->private & ~1))6345#define wqe_to_req(wait) ((void *)((unsigned long) (wait)->private & ~1))
5983#define wqe_is_double(wait) ((unsigned long) (wait)->private & 1)6346#define wqe_is_double(wait) ((unsigned long) (wait)->private & 1)
6347#define IO_ASYNC_POLL_COMMON (EPOLLONESHOT | EPOLLPRI)
5984 6348
5985static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,6349static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
5986 void *key)6350 void *key)
@@ -6015,7 +6379,7 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, · 28 unmodified lines
6015 }6379 }
6016 6380
6017 /* for instances that support it check for an event match first */6381 /* for instances that support it check for an event match first */
6018 if (mask && !(mask & poll->events))6382 if (mask && !(mask & (poll->events & ~IO_ASYNC_POLL_COMMON)))
6019 return 0;6383 return 0;
6020 6384
6021 if (io_poll_get_ownership(req)) {6385 if (io_poll_get_ownership(req)) {
@@ -6101,6 +6465,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, · 79 unmodified lines
6101 int v;6465 int v;
6102 6466
6103 INIT_HLIST_NODE(&req->hash_node);6467 INIT_HLIST_NODE(&req->hash_node);
6468 req->work.cancel_seq = atomic_read(&ctx->cancel_seq);
6104 io_init_poll_iocb(poll, mask, io_poll_wake);6469 io_init_poll_iocb(poll, mask, io_poll_wake);
6105 poll->file = req->file;6470 poll->file = req->file;
6106 6471
@@ -6171,28 +6536,34 @@ static int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) · 64 unmodified lines
6171 struct io_ring_ctx *ctx = req->ctx;6536 struct io_ring_ctx *ctx = req->ctx;
6172 struct async_poll *apoll;6537 struct async_poll *apoll;
6173 struct io_poll_table ipt;6538 struct io_poll_table ipt;
6174 __poll_t mask = EPOLLONESHOT | POLLERR | POLLPRI;6539 __poll_t mask = POLLPRI | POLLERR;
6175 int ret;6540 int ret;
6176 6541
6177 if (!def->pollin && !def->pollout)6542 if (!def->pollin && !def->pollout)
6178 return IO_APOLL_ABORTED;6543 return IO_APOLL_ABORTED;
6179 if (!file_can_poll(req->file) || (req->flags & REQ_F_POLLED))6544 if (!file_can_poll(req->file))
6180 return IO_APOLL_ABORTED;6545 return IO_APOLL_ABORTED;
6546 if ((req->flags & (REQ_F_POLLED|REQ_F_PARTIAL_IO)) == REQ_F_POLLED)
6547 return IO_APOLL_ABORTED;
6548 if (!(req->flags & REQ_F_APOLL_MULTISHOT))
6549 mask |= EPOLLONESHOT;
6181 6550
6182 if (def->pollin) {6551 if (def->pollin) {
6183 mask |= POLLIN | POLLRDNORM;6552 mask |= EPOLLIN | EPOLLRDNORM;
6184 6553
6185 /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */6554 /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
6186 if ((req->opcode == IORING_OP_RECVMSG) &&6555 if ((req->opcode == IORING_OP_RECVMSG) &&
6187 (req->sr_msg.msg_flags & MSG_ERRQUEUE))6556 (req->sr_msg.msg_flags & MSG_ERRQUEUE))
6188 mask &= ~POLLIN;6557 mask &= ~EPOLLIN;
6189 } else {6558 } else {
6190 mask |= POLLOUT | POLLWRNORM;6559 mask |= EPOLLOUT | EPOLLWRNORM;
6191 }6560 }
6192 if (def->poll_exclusive)6561 if (def->poll_exclusive)
6193 mask |= EPOLLEXCLUSIVE;6562 mask |= EPOLLEXCLUSIVE;
6194 if (!(issue_flags & IO_URING_F_UNLOCKED) &&6563 if (req->flags & REQ_F_POLLED) {
6195 !list_empty(&ctx->apoll_cache)) {6564 apoll = req->apoll;
6565 } else if (!(issue_flags & IO_URING_F_UNLOCKED) &&
6566 !list_empty(&ctx->apoll_cache)) {
6196 apoll = list_first_entry(&ctx->apoll_cache, struct async_poll,6567 apoll = list_first_entry(&ctx->apoll_cache, struct async_poll,
6197 poll.wait.entry);6568 poll.wait.entry);
6198 list_del_init(&apoll->poll.wait.entry);6569 list_del_init(&apoll->poll.wait.entry);
@@ -6212,7 +6583,7 @@ static int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) · 13 unmodified lines
6212 if (ret || ipt.error)6583 if (ret || ipt.error)
6213 return ret ? IO_APOLL_READY : IO_APOLL_ABORTED;6584 return ret ? IO_APOLL_READY : IO_APOLL_ABORTED;
6214 6585
6215 trace_io_uring_poll_arm(ctx, req, req->user_data, req->opcode,6586 trace_io_uring_poll_arm(ctx, req, req->cqe.user_data, req->opcode,
6216 mask, apoll->poll.events);6587 mask, apoll->poll.events);
6217 return IO_APOLL_OK;6588 return IO_APOLL_OK;
6218}6589}
@@ -6245,24 +6616,53 @@ static __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, · 26 unmodified lines
6245 return found;6616 return found;
6246}6617}
6247 6618
6248static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, __u64 sqe_addr,6619static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only,
6249 bool poll_only)6620 struct io_cancel_data *cd)
6250 __must_hold(&ctx->completion_lock)6621 __must_hold(&ctx->completion_lock)
6251{6622{
6252 struct hlist_head *list;6623 struct hlist_head *list;
6253 struct io_kiocb *req;6624 struct io_kiocb *req;
6254 6625
6255 list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];6626 list = &ctx->cancel_hash[hash_long(cd->data, ctx->cancel_hash_bits)];
6256 hlist_for_each_entry(req, list, hash_node) {6627 hlist_for_each_entry(req, list, hash_node) {
6257 if (sqe_addr != req->user_data)6628 if (cd->data != req->cqe.user_data)
6258 continue;6629 continue;
6259 if (poll_only && req->opcode != IORING_OP_POLL_ADD)6630 if (poll_only && req->opcode != IORING_OP_POLL_ADD)
6260 continue;6631 continue;
6632 if (cd->flags & IORING_ASYNC_CANCEL_ALL) {
6633 if (cd->seq == req->work.cancel_seq)
6634 continue;
6635 req->work.cancel_seq = cd->seq;
6636 }
6261 return req;6637 return req;
6262 }6638 }
6263 return NULL;6639 return NULL;
6264}6640}
6265 6641
6642static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx,
6643 struct io_cancel_data *cd)
6644 __must_hold(&ctx->completion_lock)
6645{
6646 struct io_kiocb *req;
6647 int i;
6648
6649 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
6650 struct hlist_head *list;
6651
6652 list = &ctx->cancel_hash[i];
6653 hlist_for_each_entry(req, list, hash_node) {
6654 if (!(cd->flags & IORING_ASYNC_CANCEL_ANY) &&
6655 req->file != cd->file)
6656 continue;
6657 if (cd->seq == req->work.cancel_seq)
6658 continue;
6659 req->work.cancel_seq = cd->seq;
6660 return req;
6661 }
6662 }
6663 return NULL;
6664}
6665
6266static bool io_poll_disarm(struct io_kiocb *req)6666static bool io_poll_disarm(struct io_kiocb *req)
6267 __must_hold(&ctx->completion_lock)6667 __must_hold(&ctx->completion_lock)
6268{6668{
@@ -6273,12 +6673,15 @@ static bool io_poll_disarm(struct io_kiocb *req) · 4 unmodified lines
6273 return true;6673 return true;
6274}6674}
6275 6675
6276static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr,6676static int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
6277 bool poll_only)
6278 __must_hold(&ctx->completion_lock)6677 __must_hold(&ctx->completion_lock)
6279{6678{
6280 struct io_kiocb *req = io_poll_find(ctx, sqe_addr, poll_only);6679 struct io_kiocb *req;
6281 6680
6681 if (cd->flags & (IORING_ASYNC_CANCEL_FD|IORING_ASYNC_CANCEL_ANY))
6682 req = io_poll_file_find(ctx, cd);
6683 else
6684 req = io_poll_find(ctx, false, cd);
6282 if (!req)6685 if (!req)
6283 return -ENOENT;6686 return -ENOENT;
6284 io_poll_cancel_req(req);6687 io_poll_cancel_req(req);
@@ -6305,9 +6708,7 @@ static int io_poll_update_prep(struct io_kiocb *req, · 20 unmodified lines
6305 struct io_poll_update *upd = &req->poll_update;6708 struct io_poll_update *upd = &req->poll_update;
6306 u32 flags;6709 u32 flags;
6307 6710
6308 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))6711 if (sqe->buf_index || sqe->splice_fd_in)
6309 return -EINVAL;
6310 if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)
6311 return -EINVAL;6712 return -EINVAL;
6312 flags = READ_ONCE(sqe->len);6713 flags = READ_ONCE(sqe->len);
6313 if (flags & ~(IORING_POLL_UPDATE_EVENTS | IORING_POLL_UPDATE_USER_DATA |6714 if (flags & ~(IORING_POLL_UPDATE_EVENTS | IORING_POLL_UPDATE_USER_DATA |
@@ -6337,9 +6738,7 @@ static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe · 23 unmodified lines
6337 struct io_poll_iocb *poll = &req->poll;6738 struct io_poll_iocb *poll = &req->poll;
6338 u32 flags;6739 u32 flags;
6339 6740
6340 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))6741 if (sqe->buf_index || sqe->off || sqe->addr)
6341 return -EINVAL;
6342 if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->addr)
6343 return -EINVAL;6742 return -EINVAL;
6344 flags = READ_ONCE(sqe->len);6743 flags = READ_ONCE(sqe->len);
6345 if (flags & ~IORING_POLL_ADD_MULTI)6744 if (flags & ~IORING_POLL_ADD_MULTI)
@@ -6369,13 +6768,14 @@ static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) · 23 unmodified lines
6369 6768
6370static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags)6769static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags)
6371{6770{
6771 struct io_cancel_data cd = { .data = req->poll_update.old_user_data, };
6372 struct io_ring_ctx *ctx = req->ctx;6772 struct io_ring_ctx *ctx = req->ctx;
6373 struct io_kiocb *preq;6773 struct io_kiocb *preq;
6374 int ret2, ret = 0;6774 int ret2, ret = 0;
6375 bool locked;6775 bool locked;
6376 6776
6377 spin_lock(&ctx->completion_lock);6777 spin_lock(&ctx->completion_lock);
6378 preq = io_poll_find(ctx, req->poll_update.old_user_data, true);6778 preq = io_poll_find(ctx, true, &cd);
6379 if (!preq || !io_poll_disarm(preq)) {6779 if (!preq || !io_poll_disarm(preq)) {
6380 spin_unlock(&ctx->completion_lock);6780 spin_unlock(&ctx->completion_lock);
6381 ret = preq ? -EALREADY : -ENOENT;6781 ret = preq ? -EALREADY : -ENOENT;
@@ -6391,7 +6791,7 @@ static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags) · 9 unmodified lines
6391 preq->poll.events |= IO_POLL_UNMASK;6791 preq->poll.events |= IO_POLL_UNMASK;
6392 }6792 }
6393 if (req->poll_update.update_user_data)6793 if (req->poll_update.update_user_data)
6394 preq->user_data = req->poll_update.new_user_data;6794 preq->cqe.user_data = req->poll_update.new_user_data;
6395 6795
6396 ret2 = io_poll_add(preq, issue_flags);6796 ret2 = io_poll_add(preq, issue_flags);
6397 /* successfully updated, don't complete poll request */6797 /* successfully updated, don't complete poll request */
@@ -6400,7 +6800,7 @@ static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags) · 2 unmodified lines
6400 }6800 }
6401 6801
6402 req_set_fail(preq);6802 req_set_fail(preq);
6403 preq->result = -ECANCELED;6803 preq->cqe.res = -ECANCELED;
6404 locked = !(issue_flags & IO_URING_F_UNLOCKED);6804 locked = !(issue_flags & IO_URING_F_UNLOCKED);
6405 io_req_task_complete(preq, &locked);6805 io_req_task_complete(preq, &locked);
6406out:6806out:
@@ -6428,14 +6828,14 @@ static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) · 21 unmodified lines
6428 if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS))6828 if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS))
6429 req_set_fail(req);6829 req_set_fail(req);
6430 6830
6431 req->result = -ETIME;6831 req->cqe.res = -ETIME;
6432 req->io_task_work.func = io_req_task_complete;6832 req->io_task_work.func = io_req_task_complete;
6433 io_req_task_work_add(req, false);6833 io_req_task_work_add(req, false);
6434 return HRTIMER_NORESTART;6834 return HRTIMER_NORESTART;
6435}6835}
6436 6836
6437static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,6837static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
6438 __u64 user_data)6838 struct io_cancel_data *cd)
6439 __must_hold(&ctx->timeout_lock)6839 __must_hold(&ctx->timeout_lock)
6440{6840{
6441 struct io_timeout_data *io;6841 struct io_timeout_data *io;
@@ -6443,9 +6843,16 @@ static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx, · 1 unmodified line
6443 bool found = false;6843 bool found = false;
6444 6844
6445 list_for_each_entry(req, &ctx->timeout_list, timeout.list) {6845 list_for_each_entry(req, &ctx->timeout_list, timeout.list) {
6446 found = user_data == req->user_data;6846 if (!(cd->flags & IORING_ASYNC_CANCEL_ANY) &&
6447 if (found)6847 cd->data != req->cqe.user_data)
6448 break;6848 continue;
6849 if (cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY)) {
6850 if (cd->seq == req->work.cancel_seq)
6851 continue;
6852 req->work.cancel_seq = cd->seq;
6853 }
6854 found = true;
6855 break;
6449 }6856 }
6450 if (!found)6857 if (!found)
6451 return ERR_PTR(-ENOENT);6858 return ERR_PTR(-ENOENT);
@@ -6457,11 +6864,14 @@ static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx, · 5 unmodified lines
6457 return req;6864 return req;
6458}6865}
6459 6866
6460static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)6867static int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
6461 __must_hold(&ctx->completion_lock)6868 __must_hold(&ctx->completion_lock)
6462 __must_hold(&ctx->timeout_lock)
6463{6869{
6464 struct io_kiocb *req = io_timeout_extract(ctx, user_data);6870 struct io_kiocb *req;
6871
6872 spin_lock_irq(&ctx->timeout_lock);
6873 req = io_timeout_extract(ctx, cd);
6874 spin_unlock_irq(&ctx->timeout_lock);
6465 6875
6466 if (IS_ERR(req))6876 if (IS_ERR(req))
6467 return PTR_ERR(req);6877 return PTR_ERR(req);
@@ -6494,7 +6904,7 @@ static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, · 26 unmodified lines
6494 bool found = false;6904 bool found = false;
6495 6905
6496 list_for_each_entry(req, &ctx->ltimeout_list, timeout.list) {6906 list_for_each_entry(req, &ctx->ltimeout_list, timeout.list) {
6497 found = user_data == req->user_data;6907 found = user_data == req->cqe.user_data;
6498 if (found)6908 if (found)
6499 break;6909 break;
6500 }6910 }
@@ -6514,7 +6924,8 @@ static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, · 13 unmodified lines
6514 struct timespec64 *ts, enum hrtimer_mode mode)6924 struct timespec64 *ts, enum hrtimer_mode mode)
6515 __must_hold(&ctx->timeout_lock)6925 __must_hold(&ctx->timeout_lock)
6516{6926{
6517 struct io_kiocb *req = io_timeout_extract(ctx, user_data);6927 struct io_cancel_data cd = { .data = user_data, };
6928 struct io_kiocb *req = io_timeout_extract(ctx, &cd);
6518 struct io_timeout_data *data;6929 struct io_timeout_data *data;
6519 6930
6520 if (IS_ERR(req))6931 if (IS_ERR(req))
@@ -6534,11 +6945,9 @@ static int io_timeout_remove_prep(struct io_kiocb *req, · 13 unmodified lines
6534{6945{
6535 struct io_timeout_rem *tr = &req->timeout_rem;6946 struct io_timeout_rem *tr = &req->timeout_rem;
6536 6947
6537 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
6538 return -EINVAL;
6539 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))6948 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
6540 return -EINVAL;6949 return -EINVAL;
6541 if (sqe->ioprio || sqe->buf_index || sqe->len || sqe->splice_fd_in)6950 if (sqe->buf_index || sqe->len || sqe->splice_fd_in)
6542 return -EINVAL;6951 return -EINVAL;
6543 6952
6544 tr->ltimeout = false;6953 tr->ltimeout = false;
@@ -6579,10 +6988,10 @@ static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags) · 34 unmodified lines
6579 int ret;6988 int ret;
6580 6989
6581 if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE)) {6990 if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE)) {
6991 struct io_cancel_data cd = { .data = tr->addr, };
6992
6582 spin_lock(&ctx->completion_lock);6993 spin_lock(&ctx->completion_lock);
6583 spin_lock_irq(&ctx->timeout_lock);6994 ret = io_timeout_cancel(ctx, &cd);
6584 ret = io_timeout_cancel(ctx, tr->addr);
6585 spin_unlock_irq(&ctx->timeout_lock);
6586 spin_unlock(&ctx->completion_lock);6995 spin_unlock(&ctx->completion_lock);
6587 } else {6996 } else {
6588 enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags);6997 enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags);
@@ -6608,10 +7017,7 @@ static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 19 unmodified lines
6608 unsigned flags;7017 unsigned flags;
6609 u32 off = READ_ONCE(sqe->off);7018 u32 off = READ_ONCE(sqe->off);
6610 7019
6611 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))7020 if (sqe->buf_index || sqe->len != 1 || sqe->splice_fd_in)
6612 return -EINVAL;
6613 if (sqe->ioprio || sqe->buf_index || sqe->len != 1 ||
6614 sqe->splice_fd_in)
6615 return -EINVAL;7021 return -EINVAL;
6616 if (off && is_timeout_link)7022 if (off && is_timeout_link)
6617 return -EINVAL;7023 return -EINVAL;
@@ -6710,30 +7116,42 @@ add: · 92 unmodified lines
6710 return 0;7116 return 0;
6711}7117}
6712 7118
6713struct io_cancel_data {
6714 struct io_ring_ctx *ctx;
6715 u64 user_data;
6716};
6717
6718static bool io_cancel_cb(struct io_wq_work *work, void *data)7119static bool io_cancel_cb(struct io_wq_work *work, void *data)
6719{7120{
6720 struct io_kiocb *req = container_of(work, struct io_kiocb, work);7121 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
6721 struct io_cancel_data *cd = data;7122 struct io_cancel_data *cd = data;
6722 7123
6723 return req->ctx == cd->ctx && req->user_data == cd->user_data;7124 if (req->ctx != cd->ctx)
7125 return false;
7126 if (cd->flags & IORING_ASYNC_CANCEL_ANY) {
7127 ;
7128 } else if (cd->flags & IORING_ASYNC_CANCEL_FD) {
7129 if (req->file != cd->file)
7130 return false;
7131 } else {
7132 if (req->cqe.user_data != cd->data)
7133 return false;
7134 }
7135 if (cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY)) {
7136 if (cd->seq == req->work.cancel_seq)
7137 return false;
7138 req->work.cancel_seq = cd->seq;
7139 }
7140 return true;
6724}7141}
6725 7142
6726static int io_async_cancel_one(struct io_uring_task *tctx, u64 user_data,7143static int io_async_cancel_one(struct io_uring_task *tctx,
6727 struct io_ring_ctx *ctx)7144 struct io_cancel_data *cd)
6728{7145{
6729 struct io_cancel_data data = { .ctx = ctx, .user_data = user_data, };
6730 enum io_wq_cancel cancel_ret;7146 enum io_wq_cancel cancel_ret;
6731 int ret = 0;7147 int ret = 0;
7148 bool all;
6732 7149
6733 if (!tctx || !tctx->io_wq)7150 if (!tctx || !tctx->io_wq)
6734 return -ENOENT;7151 return -ENOENT;
6735 7152
6736 cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, &data, false);7153 all = cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY);
7154 cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, cd, all);
6737 switch (cancel_ret) {7155 switch (cancel_ret) {
6738 case IO_WQ_CANCEL_OK:7156 case IO_WQ_CANCEL_OK:
6739 ret = 0;7157 ret = 0;
@@ -6749,14 +7167,14 @@ static int io_async_cancel_one(struct io_uring_task *tctx, u64 user_data, · 9 unmodified lines
6749 return ret;7167 return ret;
6750}7168}
6751 7169
6752static int io_try_cancel_userdata(struct io_kiocb *req, u64 sqe_addr)7170static int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd)
6753{7171{
6754 struct io_ring_ctx *ctx = req->ctx;7172 struct io_ring_ctx *ctx = req->ctx;
6755 int ret;7173 int ret;
6756 7174
6757 WARN_ON_ONCE(!io_wq_current_is_worker() && req->task != current);7175 WARN_ON_ONCE(!io_wq_current_is_worker() && req->task != current);
6758 7176
6759 ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx);7177 ret = io_async_cancel_one(req->task->io_uring, cd);
6760 /*7178 /*
6761 * Fall-through even for -EALREADY, as we may have poll armed7179 * Fall-through even for -EALREADY, as we may have poll armed
6762 * that need unarming.7180 * that need unarming.
@@ -6765,56 +7183,98 @@ static int io_try_cancel_userdata(struct io_kiocb *req, u64 sqe_addr) · 2 unmodified lines
6765 return 0;7183 return 0;
6766 7184
6767 spin_lock(&ctx->completion_lock);7185 spin_lock(&ctx->completion_lock);
6768 ret = io_poll_cancel(ctx, sqe_addr, false);7186 ret = io_poll_cancel(ctx, cd);
6769 if (ret != -ENOENT)7187 if (ret != -ENOENT)
6770 goto out;7188 goto out;
6771 7189 if (!(cd->flags & IORING_ASYNC_CANCEL_FD))
6772 spin_lock_irq(&ctx->timeout_lock);7190 ret = io_timeout_cancel(ctx, cd);
6773 ret = io_timeout_cancel(ctx, sqe_addr);
6774 spin_unlock_irq(&ctx->timeout_lock);
6775out:7191out:
6776 spin_unlock(&ctx->completion_lock);7192 spin_unlock(&ctx->completion_lock);
6777 return ret;7193 return ret;
6778}7194}
6779 7195
7196#define CANCEL_FLAGS (IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_FD | \
7197 IORING_ASYNC_CANCEL_ANY)
7198
6780static int io_async_cancel_prep(struct io_kiocb *req,7199static int io_async_cancel_prep(struct io_kiocb *req,
6781 const struct io_uring_sqe *sqe)7200 const struct io_uring_sqe *sqe)
6782{7201{
6783 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))7202 if (unlikely(req->flags & REQ_F_BUFFER_SELECT))
6784 return -EINVAL;
6785 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
6786 return -EINVAL;7203 return -EINVAL;
6787 if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags ||7204 if (sqe->off || sqe->len || sqe->splice_fd_in)
6788 sqe->splice_fd_in)
6789 return -EINVAL;7205 return -EINVAL;
6790 7206
6791 req->cancel.addr = READ_ONCE(sqe->addr);7207 req->cancel.addr = READ_ONCE(sqe->addr);
7208 req->cancel.flags = READ_ONCE(sqe->cancel_flags);
7209 if (req->cancel.flags & ~CANCEL_FLAGS)
7210 return -EINVAL;
7211 if (req->cancel.flags & IORING_ASYNC_CANCEL_FD) {
7212 if (req->cancel.flags & IORING_ASYNC_CANCEL_ANY)
7213 return -EINVAL;
7214 req->cancel.fd = READ_ONCE(sqe->fd);
7215 }
7216
6792 return 0;7217 return 0;
6793}7218}
6794 7219
6795static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)7220static int __io_async_cancel(struct io_cancel_data *cd, struct io_kiocb *req,
7221 unsigned int issue_flags)
6796{7222{
6797 struct io_ring_ctx *ctx = req->ctx;7223 bool all = cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY);
6798 u64 sqe_addr = req->cancel.addr;7224 struct io_ring_ctx *ctx = cd->ctx;
6799 bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
6800 struct io_tctx_node *node;7225 struct io_tctx_node *node;
6801 int ret;7226 int ret, nr = 0;
6802 7227
6803 ret = io_try_cancel_userdata(req, sqe_addr);7228 do {
6804 if (ret != -ENOENT)7229 ret = io_try_cancel(req, cd);
6805 goto done;7230 if (ret == -ENOENT)
7231 break;
7232 if (!all)
7233 return ret;
7234 nr++;
7235 } while (1);
6806 7236
6807 /* slow path, try all io-wq's */7237 /* slow path, try all io-wq's */
6808 io_ring_submit_lock(ctx, needs_lock);7238 io_ring_submit_lock(ctx, issue_flags);
6809 ret = -ENOENT;7239 ret = -ENOENT;
6810 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {7240 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
6811 struct io_uring_task *tctx = node->task->io_uring;7241 struct io_uring_task *tctx = node->task->io_uring;
6812 7242
6813 ret = io_async_cancel_one(tctx, req->cancel.addr, ctx);7243 ret = io_async_cancel_one(tctx, cd);
6814 if (ret != -ENOENT)7244 if (ret != -ENOENT) {
6815 break;7245 if (!all)
7246 break;
7247 nr++;
7248 }
7249 }
7250 io_ring_submit_unlock(ctx, issue_flags);
7251 return all ? nr : ret;
7252}
7253
7254static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
7255{
7256 struct io_cancel_data cd = {
7257 .ctx = req->ctx,
7258 .data = req->cancel.addr,
7259 .flags = req->cancel.flags,
7260 .seq = atomic_inc_return(&req->ctx->cancel_seq),
7261 };
7262 int ret;
7263
7264 if (cd.flags & IORING_ASYNC_CANCEL_FD) {
7265 if (req->flags & REQ_F_FIXED_FILE)
7266 req->file = io_file_get_fixed(req, req->cancel.fd,
7267 issue_flags);
7268 else
7269 req->file = io_file_get_normal(req, req->cancel.fd);
7270 if (!req->file) {
7271 ret = -EBADF;
7272 goto done;
7273 }
7274 cd.file = req->file;
6816 }7275 }
6817 io_ring_submit_unlock(ctx, needs_lock);7276
7277 ret = __io_async_cancel(&cd, req, issue_flags);
6818done:7278done:
6819 if (ret < 0)7279 if (ret < 0)
6820 req_set_fail(req);7280 req_set_fail(req);
@@ -6827,7 +7287,7 @@ static int io_rsrc_update_prep(struct io_kiocb *req, · 6 unmodified lines
6827{7287{
6828 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))7288 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
6829 return -EINVAL;7289 return -EINVAL;
6830 if (sqe->ioprio || sqe->rw_flags || sqe->splice_fd_in)7290 if (sqe->rw_flags || sqe->splice_fd_in)
6831 return -EINVAL;7291 return -EINVAL;
6832 7292
6833 req->rsrc_update.offset = READ_ONCE(sqe->off);7293 req->rsrc_update.offset = READ_ONCE(sqe->off);
@@ -6841,7 +7301,6 @@ static int io_rsrc_update_prep(struct io_kiocb *req, · 7 unmodified lines
6841static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)7301static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
6842{7302{
6843 struct io_ring_ctx *ctx = req->ctx;7303 struct io_ring_ctx *ctx = req->ctx;
6844 bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
6845 struct io_uring_rsrc_update2 up;7304 struct io_uring_rsrc_update2 up;
6846 int ret;7305 int ret;
6847 7306
@@ -6852,10 +7311,10 @@ static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) · 4 unmodified lines
6852 up.resv = 0;7311 up.resv = 0;
6853 up.resv2 = 0;7312 up.resv2 = 0;
6854 7313
6855 io_ring_submit_lock(ctx, needs_lock);7314 io_ring_submit_lock(ctx, issue_flags);
6856 ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE,7315 ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE,
6857 &up, req->rsrc_update.nr_args);7316 &up, req->rsrc_update.nr_args);
6858 io_ring_submit_unlock(ctx, needs_lock);7317 io_ring_submit_unlock(ctx, issue_flags);
6859 7318
6860 if (ret < 0)7319 if (ret < 0)
6861 req_set_fail(req);7320 req_set_fail(req);
@@ -6954,7 +7413,7 @@ static int io_req_prep_async(struct io_kiocb *req) · 92 unmodified lines
6954 7413
6955 /* assign early for deferred execution for non-fixed file */7414 /* assign early for deferred execution for non-fixed file */
6956 if (def->needs_file && !(req->flags & REQ_F_FIXED_FILE))7415 if (def->needs_file && !(req->flags & REQ_F_FIXED_FILE))
6957 req->file = io_file_get_normal(req, req->fd);7416 req->file = io_file_get_normal(req, req->cqe.fd);
6958 if (!def->needs_async_setup)7417 if (!def->needs_async_setup)
6959 return 0;7418 return 0;
6960 if (WARN_ON_ONCE(req_has_async_data(req)))7419 if (WARN_ON_ONCE(req_has_async_data(req)))
@@ -6982,9 +7441,10 @@ static int io_req_prep_async(struct io_kiocb *req) · 21 unmodified lines
6982static u32 io_get_sequence(struct io_kiocb *req)7441static u32 io_get_sequence(struct io_kiocb *req)
6983{7442{
6984 u32 seq = req->ctx->cached_sq_head;7443 u32 seq = req->ctx->cached_sq_head;
7444 struct io_kiocb *cur;
6985 7445
6986 /* need original cached_sq_head, but it was increased for each req */7446 /* need original cached_sq_head, but it was increased for each req */
6987 io_for_each_link(req, req)7447 io_for_each_link(cur, req)
6988 seq--;7448 seq--;
6989 return seq;7449 return seq;
6990}7450}
@@ -7027,7 +7487,7 @@ fail: · 36 unmodified lines
7027 goto queue;7487 goto queue;
7028 }7488 }
7029 7489
7030 trace_io_uring_defer(ctx, req, req->user_data, req->opcode);7490 trace_io_uring_defer(ctx, req, req->cqe.user_data, req->opcode);
7031 de->req = req;7491 de->req = req;
7032 de->seq = seq;7492 de->seq = seq;
7033 list_add_tail(&de->list, &ctx->defer_list);7493 list_add_tail(&de->list, &ctx->defer_list);
@@ -7111,15 +7571,11 @@ static bool io_assign_file(struct io_kiocb *req, unsigned int issue_flags) · 77 unmodified lines
7111 return true;7571 return true;
7112 7572
7113 if (req->flags & REQ_F_FIXED_FILE)7573 if (req->flags & REQ_F_FIXED_FILE)
7114 req->file = io_file_get_fixed(req, req->fd, issue_flags);7574 req->file = io_file_get_fixed(req, req->cqe.fd, issue_flags);
7115 else7575 else
7116 req->file = io_file_get_normal(req, req->fd);7576 req->file = io_file_get_normal(req, req->cqe.fd);
7117 if (req->file)
7118 return true;
7119 7577
7120 req_set_fail(req);7578 return !!req->file;
7121 req->result = -EBADF;
7122 return false;
7123}7579}
7124 7580
7125static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)7581static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
@@ -7282,7 +7738,6 @@ static void io_wq_submit_work(struct io_wq_work *work) · 156 unmodified lines
7282 const struct io_op_def *def = &io_op_defs[req->opcode];7738 const struct io_op_def *def = &io_op_defs[req->opcode];
7283 unsigned int issue_flags = IO_URING_F_UNLOCKED;7739 unsigned int issue_flags = IO_URING_F_UNLOCKED;
7284 bool needs_poll = false;7740 bool needs_poll = false;
7285 struct io_kiocb *timeout;
7286 int ret = 0, err = -ECANCELED;7741 int ret = 0, err = -ECANCELED;
7287 7742
7288 /* one will be dropped by ->io_free_work() after returning to io-wq */7743 /* one will be dropped by ->io_free_work() after returning to io-wq */
@@ -7291,10 +7746,7 @@ static void io_wq_submit_work(struct io_wq_work *work) · 2 unmodified lines
7291 else7746 else
7292 req_ref_get(req);7747 req_ref_get(req);
7293 7748
7294 timeout = io_prep_linked_timeout(req);7749 io_arm_ltimeout(req);
7295 if (timeout)
7296 io_queue_linked_timeout(timeout);
7297
7298 7750
7299 /* either cancelled or io-wq is dying, so don't touch tctx->iowq */7751 /* either cancelled or io-wq is dying, so don't touch tctx->iowq */
7300 if (work->flags & IO_WQ_WORK_CANCEL) {7752 if (work->flags & IO_WQ_WORK_CANCEL) {
@@ -7327,6 +7779,8 @@ fail: · 26 unmodified lines
7327 * wait for request slots on the block side.7779 * wait for request slots on the block side.
7328 */7780 */
7329 if (!needs_poll) {7781 if (!needs_poll) {
7782 if (!(req->ctx->flags & IORING_SETUP_IOPOLL))
7783 break;
7330 cond_resched();7784 cond_resched();
7331 continue;7785 continue;
7332 }7786 }
@@ -7372,8 +7826,7 @@ static inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd, · 39 unmodified lines
7372 struct file *file = NULL;7826 struct file *file = NULL;
7373 unsigned long file_ptr;7827 unsigned long file_ptr;
7374 7828
7375 if (issue_flags & IO_URING_F_UNLOCKED)7829 io_ring_submit_lock(ctx, issue_flags);
7376 mutex_lock(&ctx->uring_lock);
7377 7830
7378 if (unlikely((unsigned int)fd >= ctx->nr_user_files))7831 if (unlikely((unsigned int)fd >= ctx->nr_user_files))
7379 goto out;7832 goto out;
@@ -7384,9 +7837,9 @@ static inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd, · 4 unmodified lines
7384 /* mask in overlapping REQ_F and FFS bits */7837 /* mask in overlapping REQ_F and FFS bits */
7385 req->flags |= (file_ptr << REQ_F_SUPPORT_NOWAIT_BIT);7838 req->flags |= (file_ptr << REQ_F_SUPPORT_NOWAIT_BIT);
7386 io_req_set_rsrc_node(req, ctx, 0);7839 io_req_set_rsrc_node(req, ctx, 0);
7840 WARN_ON_ONCE(file && !test_bit(fd, ctx->file_table.bitmap));
7387out:7841out:
7388 if (issue_flags & IO_URING_F_UNLOCKED)7842 io_ring_submit_unlock(ctx, issue_flags);
7389 mutex_unlock(&ctx->uring_lock);
7390 return file;7843 return file;
7391}7844}
7392 7845
@@ -7407,7 +7860,7 @@ static struct file *io_file_get_normal(struct io_kiocb *req, int fd) · 14 unmodified lines
7407{7860{
7408 struct file *file = fget(fd);7861 struct file *file = fget(fd);
7409 7862
7410 trace_io_uring_file_get(req->ctx, req, req->user_data, fd);7863 trace_io_uring_file_get(req->ctx, req, req->cqe.user_data, fd);
7411 7864
7412 /* we don't allow fixed io_uring files */7865 /* we don't allow fixed io_uring files */
7413 if (file && file->f_op == &io_uring_fops)7866 if (file && file->f_op == &io_uring_fops)
@@ -7421,8 +7874,14 @@ static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked) · 7 unmodified lines
7421 int ret = -ENOENT;7874 int ret = -ENOENT;
7422 7875
7423 if (prev) {7876 if (prev) {
7424 if (!(req->task->flags & PF_EXITING))7877 if (!(req->task->flags & PF_EXITING)) {
7425 ret = io_try_cancel_userdata(req, prev->user_data);7878 struct io_cancel_data cd = {
7879 .ctx = req->ctx,
7880 .data = prev->cqe.user_data,
7881 };
7882
7883 ret = io_try_cancel(req, &cd);
7884 }
7426 io_req_complete_post(req, ret ?: -ETIME, 0);7885 io_req_complete_post(req, ret ?: -ETIME, 0);
7427 io_put_req(prev);7886 io_put_req(prev);
7428 } else {7887 } else {
@@ -7482,10 +7941,17 @@ static void io_queue_linked_timeout(struct io_kiocb *req) · 53 unmodified lines
7482 io_put_req(req);7941 io_put_req(req);
7483}7942}
7484 7943
7485static void io_queue_sqe_arm_apoll(struct io_kiocb *req)7944static void io_queue_async(struct io_kiocb *req, int ret)
7486 __must_hold(&req->ctx->uring_lock)7945 __must_hold(&req->ctx->uring_lock)
7487{7946{
7488 struct io_kiocb *linked_timeout = io_prep_linked_timeout(req);7947 struct io_kiocb *linked_timeout;
7948
7949 if (ret != -EAGAIN || (req->flags & REQ_F_NOWAIT)) {
7950 io_req_complete_failed(req, ret);
7951 return;
7952 }
7953
7954 linked_timeout = io_prep_linked_timeout(req);
7489 7955
7490 switch (io_arm_poll_handler(req, 0)) {7956 switch (io_arm_poll_handler(req, 0)) {
7491 case IO_APOLL_READY:7957 case IO_APOLL_READY:
@@ -7496,7 +7962,7 @@ static void io_queue_sqe_arm_apoll(struct io_kiocb *req) · 4 unmodified lines
7496 * Queued up for async execution, worker will release7962 * Queued up for async execution, worker will release
7497 * submit reference when the iocb is actually submitted.7963 * submit reference when the iocb is actually submitted.
7498 */7964 */
7499 io_queue_async_work(req, NULL);7965 io_queue_iowq(req, NULL);
7500 break;7966 break;
7501 case IO_APOLL_OK:7967 case IO_APOLL_OK:
7502 break;7968 break;
@@ -7506,10 +7972,9 @@ static void io_queue_sqe_arm_apoll(struct io_kiocb *req) · 3 unmodified lines
7506 io_queue_linked_timeout(linked_timeout);7972 io_queue_linked_timeout(linked_timeout);
7507}7973}
7508 7974
7509static inline void __io_queue_sqe(struct io_kiocb *req)7975static inline void io_queue_sqe(struct io_kiocb *req)
7510 __must_hold(&req->ctx->uring_lock)7976 __must_hold(&req->ctx->uring_lock)
7511{7977{
7512 struct io_kiocb *linked_timeout;
7513 int ret;7978 int ret;
7514 7979
7515 ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);7980 ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);
@@ -7522,22 +7987,23 @@ static inline void __io_queue_sqe(struct io_kiocb *req) · 6 unmodified lines
7522 * We async punt it if the file wasn't marked NOWAIT, or if the file7987 * We async punt it if the file wasn't marked NOWAIT, or if the file
7523 * doesn't support non-blocking read/write attempts7988 * doesn't support non-blocking read/write attempts
7524 */7989 */
7525 if (likely(!ret)) {7990 if (likely(!ret))
7526 linked_timeout = io_prep_linked_timeout(req);7991 io_arm_ltimeout(req);
7527 if (linked_timeout)7992 else
7528 io_queue_linked_timeout(linked_timeout);7993 io_queue_async(req, ret);
7529 } else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
7530 io_queue_sqe_arm_apoll(req);
7531 } else {
7532 io_req_complete_failed(req, ret);
7533 }
7534}7994}
7535 7995
7536static void io_queue_sqe_fallback(struct io_kiocb *req)7996static void io_queue_sqe_fallback(struct io_kiocb *req)
7537 __must_hold(&req->ctx->uring_lock)7997 __must_hold(&req->ctx->uring_lock)
7538{7998{
7539 if (req->flags & REQ_F_FAIL) {7999 if (unlikely(req->flags & REQ_F_FAIL)) {
7540 io_req_complete_fail_submit(req);8000 /*
8001 * We don't submit, fail them all, for that replace hardlinks
8002 * with normal links. Extra REQ_F_LINK is tolerated.
8003 */
8004 req->flags &= ~REQ_F_HARDLINK;
8005 req->flags |= REQ_F_LINK;
8006 io_req_complete_failed(req, req->cqe.res);
7541 } else if (unlikely(req->ctx->drain_active)) {8007 } else if (unlikely(req->ctx->drain_active)) {
7542 io_drain_req(req);8008 io_drain_req(req);
7543 } else {8009 } else {
@@ -7546,19 +8012,10 @@ static void io_queue_sqe_fallback(struct io_kiocb *req) · 2 unmodified lines
7546 if (unlikely(ret))8012 if (unlikely(ret))
7547 io_req_complete_failed(req, ret);8013 io_req_complete_failed(req, ret);
7548 else8014 else
7549 io_queue_async_work(req, NULL);8015 io_queue_iowq(req, NULL);
7550 }8016 }
7551}8017}
7552 8018
7553static inline void io_queue_sqe(struct io_kiocb *req)
7554 __must_hold(&req->ctx->uring_lock)
7555{
7556 if (likely(!(req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL))))
7557 __io_queue_sqe(req);
7558 else
7559 io_queue_sqe_fallback(req);
7560}
7561
7562/*8019/*
7563 * Check SQE restrictions (opcode and flags).8020 * Check SQE restrictions (opcode and flags).
7564 *8021 *
@@ -7613,9 +8070,9 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, · 48 unmodified lines
7613 req->opcode = opcode = READ_ONCE(sqe->opcode);8070 req->opcode = opcode = READ_ONCE(sqe->opcode);
7614 /* same numerical values with corresponding REQ_F_*, safe to copy */8071 /* same numerical values with corresponding REQ_F_*, safe to copy */
7615 req->flags = sqe_flags = READ_ONCE(sqe->flags);8072 req->flags = sqe_flags = READ_ONCE(sqe->flags);
7616 req->user_data = READ_ONCE(sqe->user_data);8073 req->cqe.user_data = READ_ONCE(sqe->user_data);
7617 req->file = NULL;8074 req->file = NULL;
7618 req->fixed_rsrc_refs = NULL;8075 req->rsrc_node = NULL;
7619 req->task = current;8076 req->task = current;
7620 8077
7621 if (unlikely(opcode >= IORING_OP_LAST)) {8078 if (unlikely(opcode >= IORING_OP_LAST)) {
@@ -7626,9 +8083,11 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, · 4 unmodified lines
7626 /* enforce forwards compatibility on users */8083 /* enforce forwards compatibility on users */
7627 if (sqe_flags & ~SQE_VALID_FLAGS)8084 if (sqe_flags & ~SQE_VALID_FLAGS)
7628 return -EINVAL;8085 return -EINVAL;
7629 if ((sqe_flags & IOSQE_BUFFER_SELECT) &&8086 if (sqe_flags & IOSQE_BUFFER_SELECT) {
7630 !io_op_defs[opcode].buffer_select)8087 if (!io_op_defs[opcode].buffer_select)
7631 return -EOPNOTSUPP;8088 return -EOPNOTSUPP;
8089 req->buf_index = READ_ONCE(sqe->buf_group);
8090 }
7632 if (sqe_flags & IOSQE_CQE_SKIP_SUCCESS)8091 if (sqe_flags & IOSQE_CQE_SKIP_SUCCESS)
7633 ctx->drain_disabled = true;8092 ctx->drain_disabled = true;
7634 if (sqe_flags & IOSQE_IO_DRAIN) {8093 if (sqe_flags & IOSQE_IO_DRAIN) {
@@ -7651,10 +8110,15 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, · 16 unmodified lines
7651 }8110 }
7652 }8111 }
7653 8112
8113 if (!io_op_defs[opcode].ioprio && sqe->ioprio)
8114 return -EINVAL;
8115 if (!io_op_defs[opcode].iopoll && (ctx->flags & IORING_SETUP_IOPOLL))
8116 return -EINVAL;
8117
7654 if (io_op_defs[opcode].needs_file) {8118 if (io_op_defs[opcode].needs_file) {
7655 struct io_submit_state *state = &ctx->submit_state;8119 struct io_submit_state *state = &ctx->submit_state;
7656 8120
7657 req->fd = READ_ONCE(sqe->fd);8121 req->cqe.fd = READ_ONCE(sqe->fd);
7658 8122
7659 /*8123 /*
7660 * Plug now if we have more than 2 IO left after this, and the8124 * Plug now if we have more than 2 IO left after this, and the
@@ -7686,7 +8150,44 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, · 25 unmodified lines
7686 return io_req_prep(req, sqe);8150 return io_req_prep(req, sqe);
7687}8151}
7688 8152
7689static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,8153static __cold int io_submit_fail_init(const struct io_uring_sqe *sqe,
8154 struct io_kiocb *req, int ret)
8155{
8156 struct io_ring_ctx *ctx = req->ctx;
8157 struct io_submit_link *link = &ctx->submit_state.link;
8158 struct io_kiocb *head = link->head;
8159
8160 trace_io_uring_req_failed(sqe, ctx, req, ret);
8161
8162 /*
8163 * Avoid breaking links in the middle as it renders links with SQPOLL
8164 * unusable. Instead of failing eagerly, continue assembling the link if
8165 * applicable and mark the head with REQ_F_FAIL. The link flushing code
8166 * should find the flag and handle the rest.
8167 */
8168 req_fail_link_node(req, ret);
8169 if (head && !(head->flags & REQ_F_FAIL))
8170 req_fail_link_node(head, -ECANCELED);
8171
8172 if (!(req->flags & IO_REQ_LINK_FLAGS)) {
8173 if (head) {
8174 link->last->link = req;
8175 link->head = NULL;
8176 req = head;
8177 }
8178 io_queue_sqe_fallback(req);
8179 return ret;
8180 }
8181
8182 if (head)
8183 link->last->link = req;
8184 else
8185 link->head = req;
8186 link->last = req;
8187 return 0;
8188}
8189
8190static inline int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
7690 const struct io_uring_sqe *sqe)8191 const struct io_uring_sqe *sqe)
7691 __must_hold(&ctx->uring_lock)8192 __must_hold(&ctx->uring_lock)
7692{8193{
@@ -7694,35 +8195,11 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, · 1 unmodified line
7694 int ret;8195 int ret;
7695 8196
7696 ret = io_init_req(ctx, req, sqe);8197 ret = io_init_req(ctx, req, sqe);
7697 if (unlikely(ret)) {8198 if (unlikely(ret))
7698 trace_io_uring_req_failed(sqe, ctx, req, ret);8199 return io_submit_fail_init(sqe, req, ret);
7699
7700 /* fail even hard links since we don't submit */
7701 if (link->head) {
7702 /*
7703 * we can judge a link req is failed or cancelled by if
7704 * REQ_F_FAIL is set, but the head is an exception since
7705 * it may be set REQ_F_FAIL because of other req's failure
7706 * so let's leverage req->result to distinguish if a head
7707 * is set REQ_F_FAIL because of its failure or other req's
7708 * failure so that we can set the correct ret code for it.
7709 * init result here to avoid affecting the normal path.
7710 */
7711 if (!(link->head->flags & REQ_F_FAIL))
7712 req_fail_link_node(link->head, -ECANCELED);
7713 } else if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
7714 /*
7715 * the current req is a normal req, we should return
7716 * error and thus break the submittion loop.
7717 */
7718 io_req_complete_failed(req, ret);
7719 return ret;
7720 }
7721 req_fail_link_node(req, ret);
7722 }
7723 8200
7724 /* don't need @sqe from now on */8201 /* don't need @sqe from now on */
7725 trace_io_uring_submit_sqe(ctx, req, req->user_data, req->opcode,8202 trace_io_uring_submit_sqe(ctx, req, req->cqe.user_data, req->opcode,
7726 req->flags, true,8203 req->flags, true,
7727 ctx->flags & IORING_SETUP_SQPOLL);8204 ctx->flags & IORING_SETUP_SQPOLL);
7728 8205
@@ -7733,29 +8210,32 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, · 4 unmodified lines
7733 * submitted sync once the chain is complete. If none of those8210 * submitted sync once the chain is complete. If none of those
7734 * conditions are true (normal request), then just queue it.8211 * conditions are true (normal request), then just queue it.
7735 */8212 */
7736 if (link->head) {8213 if (unlikely(link->head)) {
7737 struct io_kiocb *head = link->head;8214 ret = io_req_prep_async(req);
7738 8215 if (unlikely(ret))
7739 if (!(req->flags & REQ_F_FAIL)) {8216 return io_submit_fail_init(sqe, req, ret);
7740 ret = io_req_prep_async(req);8217
7741 if (unlikely(ret)) {8218 trace_io_uring_link(ctx, req, link->head);
7742 req_fail_link_node(req, ret);
7743 if (!(head->flags & REQ_F_FAIL))
7744 req_fail_link_node(head, -ECANCELED);
7745 }
7746 }
7747 trace_io_uring_link(ctx, req, head);
7748 link->last->link = req;8219 link->last->link = req;
7749 link->last = req;8220 link->last = req;
7750 8221
7751 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK))8222 if (req->flags & IO_REQ_LINK_FLAGS)
7752 return 0;8223 return 0;
7753 /* last request of a link, enqueue the link */8224 /* last request of the link, flush it */
8225 req = link->head;
7754 link->head = NULL;8226 link->head = NULL;
7755 req = head;8227 if (req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL))
7756 } else if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {8228 goto fallback;
7757 link->head = req;8229
7758 link->last = req;8230 } else if (unlikely(req->flags & (IO_REQ_LINK_FLAGS |
8231 REQ_F_FORCE_ASYNC | REQ_F_FAIL))) {
8232 if (req->flags & IO_REQ_LINK_FLAGS) {
8233 link->head = req;
8234 link->last = req;
8235 } else {
8236fallback:
8237 io_queue_sqe_fallback(req);
8238 }
7759 return 0;8239 return 0;
7760 }8240 }
7761 8241
@@ -7770,8 +8250,8 @@ static void io_submit_state_end(struct io_ring_ctx *ctx) · 8 unmodified lines
7770{8250{
7771 struct io_submit_state *state = &ctx->submit_state;8251 struct io_submit_state *state = &ctx->submit_state;
7772 8252
7773 if (state->link.head)8253 if (unlikely(state->link.head))
7774 io_queue_sqe(state->link.head);8254 io_queue_sqe_fallback(state->link.head);
7775 /* flush only after queuing links as they can generate completions */8255 /* flush only after queuing links as they can generate completions */
7776 io_submit_flush_completions(ctx);8256 io_submit_flush_completions(ctx);
7777 if (state->plug_started)8257 if (state->plug_started)
@@ -7839,54 +8319,52 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) · 61 unmodified lines
7839 __must_hold(&ctx->uring_lock)8319 __must_hold(&ctx->uring_lock)
7840{8320{
7841 unsigned int entries = io_sqring_entries(ctx);8321 unsigned int entries = io_sqring_entries(ctx);
7842 int submitted = 0;8322 unsigned int left;
8323 int ret;
7843 8324
7844 if (unlikely(!entries))8325 if (unlikely(!entries))
7845 return 0;8326 return 0;
7846 /* make sure SQ entry isn't read before tail */8327 /* make sure SQ entry isn't read before tail */
7847 nr = min3(nr, ctx->sq_entries, entries);8328 ret = left = min3(nr, ctx->sq_entries, entries);
7848 io_get_task_refs(nr);8329 io_get_task_refs(left);
8330 io_submit_state_start(&ctx->submit_state, left);
7849 8331
7850 io_submit_state_start(&ctx->submit_state, nr);
7851 do {8332 do {
7852 const struct io_uring_sqe *sqe;8333 const struct io_uring_sqe *sqe;
7853 struct io_kiocb *req;8334 struct io_kiocb *req;
7854 8335
7855 if (unlikely(!io_alloc_req_refill(ctx))) {8336 if (unlikely(!io_alloc_req_refill(ctx)))
7856 if (!submitted)
7857 submitted = -EAGAIN;
7858 break;8337 break;
7859 }
7860 req = io_alloc_req(ctx);8338 req = io_alloc_req(ctx);
7861 sqe = io_get_sqe(ctx);8339 sqe = io_get_sqe(ctx);
7862 if (unlikely(!sqe)) {8340 if (unlikely(!sqe)) {
7863 wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list);8341 io_req_add_to_cache(req, ctx);
7864 break;8342 break;
7865 }8343 }
7866 /* will complete beyond this point, count as submitted */
7867 submitted++;
7868 if (io_submit_sqe(ctx, req, sqe)) {
7869 /*
7870 * Continue submitting even for sqe failure if the
7871 * ring was setup with IORING_SETUP_SUBMIT_ALL
7872 */
7873 if (!(ctx->flags & IORING_SETUP_SUBMIT_ALL))
7874 break;
7875 }
7876 } while (submitted < nr);
7877 8344
7878 if (unlikely(submitted != nr)) {8345 /*
7879 int ref_used = (submitted == -EAGAIN) ? 0 : submitted;8346 * Continue submitting even for sqe failure if the
7880 int unused = nr - ref_used;8347 * ring was setup with IORING_SETUP_SUBMIT_ALL
8348 */
8349 if (unlikely(io_submit_sqe(ctx, req, sqe)) &&
8350 !(ctx->flags & IORING_SETUP_SUBMIT_ALL)) {
8351 left--;
8352 break;
8353 }
8354 } while (--left);
7881 8355
7882 current->io_uring->cached_refs += unused;8356 if (unlikely(left)) {
8357 ret -= left;
8358 /* try again if it submitted nothing and can't allocate a req */
8359 if (!ret && io_req_cache_empty(ctx))
8360 ret = -EAGAIN;
8361 current->io_uring->cached_refs += left;
7883 }8362 }
7884 8363
7885 io_submit_state_end(ctx);8364 io_submit_state_end(ctx);
7886 /* Commit SQ ring head once we've consumed and submitted all SQEs */8365 /* Commit SQ ring head once we've consumed and submitted all SQEs */
7887 io_commit_sqring(ctx);8366 io_commit_sqring(ctx);
7888 8367 return ret;
7889 return submitted;
7890}8368}
7891 8369
7892static inline bool io_sqd_events_pending(struct io_sq_data *sqd)8370static inline bool io_sqd_events_pending(struct io_sq_data *sqd)
@@ -7894,23 +8372,6 @@ static inline bool io_sqd_events_pending(struct io_sq_data *sqd) · 1 unmodified line
7894 return READ_ONCE(sqd->state);8372 return READ_ONCE(sqd->state);
7895}8373}
7896 8374
7897static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
7898{
7899 /* Tell userspace we may need a wakeup call */
7900 spin_lock(&ctx->completion_lock);
7901 WRITE_ONCE(ctx->rings->sq_flags,
7902 ctx->rings->sq_flags | IORING_SQ_NEED_WAKEUP);
7903 spin_unlock(&ctx->completion_lock);
7904}
7905
7906static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
7907{
7908 spin_lock(&ctx->completion_lock);
7909 WRITE_ONCE(ctx->rings->sq_flags,
7910 ctx->rings->sq_flags & ~IORING_SQ_NEED_WAKEUP);
7911 spin_unlock(&ctx->completion_lock);
7912}
7913
7914static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)8375static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)
7915{8376{
7916 unsigned int to_submit;8377 unsigned int to_submit;
@@ -8026,8 +8487,8 @@ static int io_sq_thread(void *data) · 109 unmodified lines
8026 bool needs_sched = true;8487 bool needs_sched = true;
8027 8488
8028 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {8489 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
8029 io_ring_set_wakeup_flag(ctx);8490 atomic_or(IORING_SQ_NEED_WAKEUP,
8030 8491 &ctx->rings->sq_flags);
8031 if ((ctx->flags & IORING_SETUP_IOPOLL) &&8492 if ((ctx->flags & IORING_SETUP_IOPOLL) &&
8032 !wq_list_empty(&ctx->iopoll_list)) {8493 !wq_list_empty(&ctx->iopoll_list)) {
8033 needs_sched = false;8494 needs_sched = false;
@@ -8038,7 +8499,7 @@ static int io_sq_thread(void *data) · 4 unmodified lines
8038 * Ensure the store of the wakeup flag is not8499 * Ensure the store of the wakeup flag is not
8039 * reordered with the load of the SQ tail8500 * reordered with the load of the SQ tail
8040 */8501 */
8041 smp_mb();8502 smp_mb__after_atomic();
8042 8503
8043 if (io_sqring_entries(ctx)) {8504 if (io_sqring_entries(ctx)) {
8044 needs_sched = false;8505 needs_sched = false;
@@ -8052,7 +8513,8 @@ static int io_sq_thread(void *data) · 7 unmodified lines
8052 mutex_lock(&sqd->lock);8513 mutex_lock(&sqd->lock);
8053 }8514 }
8054 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)8515 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
8055 io_ring_clear_wakeup_flag(ctx);8516 atomic_andnot(IORING_SQ_NEED_WAKEUP,
8517 &ctx->rings->sq_flags);
8056 }8518 }
8057 8519
8058 finish_wait(&sqd->wait, &wait);8520 finish_wait(&sqd->wait, &wait);
@@ -8062,7 +8524,7 @@ static int io_sq_thread(void *data) · 3 unmodified lines
8062 io_uring_cancel_generic(true, sqd);8524 io_uring_cancel_generic(true, sqd);
8063 sqd->thread = NULL;8525 sqd->thread = NULL;
8064 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)8526 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
8065 io_ring_set_wakeup_flag(ctx);8527 atomic_or(IORING_SQ_NEED_WAKEUP, &ctx->rings->sq_flags);
8066 io_run_task_work();8528 io_run_task_work();
8067 mutex_unlock(&sqd->lock);8529 mutex_unlock(&sqd->lock);
8068 8530
@@ -8102,7 +8564,8 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, · 33 unmodified lines
8102 * Cannot safely flush overflowed CQEs from here, ensure we wake up8564 * Cannot safely flush overflowed CQEs from here, ensure we wake up
8103 * the task, and the next invocation will do it.8565 * the task, and the next invocation will do it.
8104 */8566 */
8105 if (io_should_wake(iowq) || test_bit(0, &iowq->ctx->check_cq_overflow))8567 if (io_should_wake(iowq) ||
8568 test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &iowq->ctx->check_cq))
8106 return autoremove_wake_function(curr, mode, wake_flags, key);8569 return autoremove_wake_function(curr, mode, wake_flags, key);
8107 return -1;8570 return -1;
8108}8571}
@@ -8124,15 +8587,18 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, · 15 unmodified lines
8124 ktime_t timeout)8587 ktime_t timeout)
8125{8588{
8126 int ret;8589 int ret;
8590 unsigned long check_cq;
8127 8591
8128 /* make sure we run task_work before checking for signals */8592 /* make sure we run task_work before checking for signals */
8129 ret = io_run_task_work_sig();8593 ret = io_run_task_work_sig();
8130 if (ret || io_should_wake(iowq))8594 if (ret || io_should_wake(iowq))
8131 return ret;8595 return ret;
8596 check_cq = READ_ONCE(ctx->check_cq);
8132 /* let the caller flush overflows, retry */8597 /* let the caller flush overflows, retry */
8133 if (test_bit(0, &ctx->check_cq_overflow))8598 if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
8134 return 1;8599 return 1;
8135 8600 if (unlikely(check_cq & BIT(IO_CHECK_CQ_DROPPED_BIT)))
8601 return -EBADR;
8136 if (!schedule_hrtimeout(&timeout, HRTIMER_MODE_ABS))8602 if (!schedule_hrtimeout(&timeout, HRTIMER_MODE_ABS))
8137 return -ETIME;8603 return -ETIME;
8138 return 1;8604 return 1;
@@ -8197,10 +8663,10 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 58 unmodified lines
8197 prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq,8663 prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq,
8198 TASK_INTERRUPTIBLE);8664 TASK_INTERRUPTIBLE);
8199 ret = io_cqring_wait_schedule(ctx, &iowq, timeout);8665 ret = io_cqring_wait_schedule(ctx, &iowq, timeout);
8200 finish_wait(&ctx->cq_wait, &iowq.wq);
8201 cond_resched();8666 cond_resched();
8202 } while (ret > 0);8667 } while (ret > 0);
8203 8668
8669 finish_wait(&ctx->cq_wait, &iowq.wq);
8204 restore_saved_sigmask_unless(ret == -EINTR);8670 restore_saved_sigmask_unless(ret == -EINTR);
8205 8671
8206 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;8672 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
@@ -8438,17 +8904,57 @@ static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files) · 231 unmodified lines
8438{8904{
8439 table->files = kvcalloc(nr_files, sizeof(table->files[0]),8905 table->files = kvcalloc(nr_files, sizeof(table->files[0]),
8440 GFP_KERNEL_ACCOUNT);8906 GFP_KERNEL_ACCOUNT);
8441 return !!table->files;8907 if (unlikely(!table->files))
8908 return false;
8909
8910 table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT);
8911 if (unlikely(!table->bitmap)) {
8912 kvfree(table->files);
8913 return false;
8914 }
8915
8916 return true;
8442}8917}
8443 8918
8444static void io_free_file_tables(struct io_file_table *table)8919static void io_free_file_tables(struct io_file_table *table)
8445{8920{
8446 kvfree(table->files);8921 kvfree(table->files);
8922 bitmap_free(table->bitmap);
8447 table->files = NULL;8923 table->files = NULL;
8924 table->bitmap = NULL;
8925}
8926
8927static inline void io_file_bitmap_set(struct io_file_table *table, int bit)
8928{
8929 WARN_ON_ONCE(test_bit(bit, table->bitmap));
8930 __set_bit(bit, table->bitmap);
8931 if (bit == table->alloc_hint)
8932 table->alloc_hint++;
8933}
8934
8935static inline void io_file_bitmap_clear(struct io_file_table *table, int bit)
8936{
8937 __clear_bit(bit, table->bitmap);
8938 table->alloc_hint = bit;
8448}8939}
8449 8940
8450static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)8941static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
8451{8942{
8943#if !defined(IO_URING_SCM_ALL)
8944 int i;
8945
8946 for (i = 0; i < ctx->nr_user_files; i++) {
8947 struct file *file = io_file_from_index(ctx, i);
8948
8949 if (!file)
8950 continue;
8951 if (io_fixed_file_slot(&ctx->file_table, i)->file_ptr & FFS_SCM)
8952 continue;
8953 io_file_bitmap_clear(&ctx->file_table, i);
8954 fput(file);
8955 }
8956#endif
8957
8452#if defined(CONFIG_UNIX)8958#if defined(CONFIG_UNIX)
8453 if (ctx->ring_sock) {8959 if (ctx->ring_sock) {
8454 struct sock *sock = ctx->ring_sock->sk;8960 struct sock *sock = ctx->ring_sock->sk;
@@ -8457,16 +8963,6 @@ static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) · 2 unmodified lines
8457 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)8963 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
8458 kfree_skb(skb);8964 kfree_skb(skb);
8459 }8965 }
8460#else
8461 int i;
8462
8463 for (i = 0; i < ctx->nr_user_files; i++) {
8464 struct file *file;
8465
8466 file = io_file_from_index(ctx, i);
8467 if (file)
8468 fput(file);
8469 }
8470#endif8966#endif
8471 io_free_file_tables(&ctx->file_table);8967 io_free_file_tables(&ctx->file_table);
8472 io_rsrc_data_free(ctx->file_data);8968 io_rsrc_data_free(ctx->file_data);
@@ -8611,107 +9107,66 @@ static struct io_sq_data *io_get_sq_data(struct io_uring_params *p, · 138 unmodified lines
8611 return sqd;9107 return sqd;
8612}9108}
8613 9109
8614#if defined(CONFIG_UNIX)
8615/*9110/*
8616 * Ensure the UNIX gc is aware of our file set, so we are certain that9111 * Ensure the UNIX gc is aware of our file set, so we are certain that
8617 * the io_uring can be safely unregistered on process exit, even if we have9112 * the io_uring can be safely unregistered on process exit, even if we have
8618 * loops in the file referencing.9113 * loops in the file referencing. We account only files that can hold other
9114 * files because otherwise they can't form a loop and so are not interesting
9115 * for GC.
8619 */9116 */
8620static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)9117static int io_scm_file_account(struct io_ring_ctx *ctx, struct file *file)
8621{9118{
9119#if defined(CONFIG_UNIX)
8622 struct sock *sk = ctx->ring_sock->sk;9120 struct sock *sk = ctx->ring_sock->sk;
9121 struct sk_buff_head *head = &sk->sk_receive_queue;
8623 struct scm_fp_list *fpl;9122 struct scm_fp_list *fpl;
8624 struct sk_buff *skb;9123 struct sk_buff *skb;
8625 int i, nr_files;
8626
8627 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
8628 if (!fpl)
8629 return -ENOMEM;
8630
8631 skb = alloc_skb(0, GFP_KERNEL);
8632 if (!skb) {
8633 kfree(fpl);
8634 return -ENOMEM;
8635 }
8636
8637 skb->sk = sk;
8638 9124
8639 nr_files = 0;9125 if (likely(!io_file_need_scm(file)))
8640 fpl->user = get_uid(current_user());9126 return 0;
8641 for (i = 0; i < nr; i++) {
8642 struct file *file = io_file_from_index(ctx, i + offset);
8643
8644 if (!file)
8645 continue;
8646 fpl->fp[nr_files] = get_file(file);
8647 unix_inflight(fpl->user, fpl->fp[nr_files]);
8648 nr_files++;
8649 }
8650
8651 if (nr_files) {
8652 fpl->max = SCM_MAX_FD;
8653 fpl->count = nr_files;
8654 UNIXCB(skb).fp = fpl;
8655 skb->destructor = unix_destruct_scm;
8656 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
8657 skb_queue_head(&sk->sk_receive_queue, skb);
8658
8659 for (i = 0; i < nr; i++) {
8660 struct file *file = io_file_from_index(ctx, i + offset);
8661
8662 if (file)
8663 fput(file);
8664 }
8665 } else {
8666 kfree_skb(skb);
8667 free_uid(fpl->user);
8668 kfree(fpl);
8669 }
8670
8671 return 0;
8672}
8673
8674/*
8675 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
8676 * causes regular reference counting to break down. We rely on the UNIX
8677 * garbage collection to take care of this problem for us.
8678 */
8679static int io_sqe_files_scm(struct io_ring_ctx *ctx)
8680{
8681 unsigned left, total;
8682 int ret = 0;
8683 9127
8684 total = 0;9128 /*
8685 left = ctx->nr_user_files;9129 * See if we can merge this file into an existing skb SCM_RIGHTS
8686 while (left) {9130 * file set. If there's no room, fall back to allocating a new skb
8687 unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);9131 * and filling it in.
9132 */
9133 spin_lock_irq(&head->lock);
9134 skb = skb_peek(head);
9135 if (skb && UNIXCB(skb).fp->count < SCM_MAX_FD)
9136 __skb_unlink(skb, head);
9137 else
9138 skb = NULL;
9139 spin_unlock_irq(&head->lock);
8688 9140
8689 ret = __io_sqe_files_scm(ctx, this_files, total);9141 if (!skb) {
8690 if (ret)9142 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
8691 break;9143 if (!fpl)
8692 left -= this_files;9144 return -ENOMEM;
8693 total += this_files;
8694 }
8695 9145
8696 if (!ret)9146 skb = alloc_skb(0, GFP_KERNEL);
8697 return 0;9147 if (!skb) {
9148 kfree(fpl);
9149 return -ENOMEM;
9150 }
8698 9151
8699 while (total < ctx->nr_user_files) {9152 fpl->user = get_uid(current_user());
8700 struct file *file = io_file_from_index(ctx, total);9153 fpl->max = SCM_MAX_FD;
9154 fpl->count = 0;
8701 9155
8702 if (file)9156 UNIXCB(skb).fp = fpl;
8703 fput(file);9157 skb->sk = sk;
8704 total++;9158 skb->destructor = unix_destruct_scm;
9159 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
8705 }9160 }
8706 9161
8707 return ret;9162 fpl = UNIXCB(skb).fp;
8708}9163 fpl->fp[fpl->count++] = get_file(file);
8709#else9164 unix_inflight(fpl->user, file);
8710static int io_sqe_files_scm(struct io_ring_ctx *ctx)9165 skb_queue_head(head, skb);
8711{9166 fput(file);
9167#endif
8712 return 0;9168 return 0;
8713}9169}
8714#endif
8715 9170
8716static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)9171static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
8717{9172{
@@ -8722,6 +9177,11 @@ static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc) · 4 unmodified lines
8722 struct sk_buff *skb;9177 struct sk_buff *skb;
8723 int i;9178 int i;
8724 9179
9180 if (!io_file_need_scm(file)) {
9181 fput(file);
9182 return;
9183 }
9184
8725 __skb_queue_head_init(&list);9185 __skb_queue_head_init(&list);
8726 9186
8727 /*9187 /*
@@ -8786,15 +9246,17 @@ static void __io_rsrc_put_work(struct io_rsrc_node *ref_node) · 58 unmodified lines
8786 list_del(&prsrc->list);9246 list_del(&prsrc->list);
8787 9247
8788 if (prsrc->tag) {9248 if (prsrc->tag) {
8789 bool lock_ring = ctx->flags & IORING_SETUP_IOPOLL;9249 if (ctx->flags & IORING_SETUP_IOPOLL)
9250 mutex_lock(&ctx->uring_lock);
8790 9251
8791 io_ring_submit_lock(ctx, lock_ring);
8792 spin_lock(&ctx->completion_lock);9252 spin_lock(&ctx->completion_lock);
8793 io_fill_cqe_aux(ctx, prsrc->tag, 0, 0);9253 io_fill_cqe_aux(ctx, prsrc->tag, 0, 0);
8794 io_commit_cqring(ctx);9254 io_commit_cqring(ctx);
8795 spin_unlock(&ctx->completion_lock);9255 spin_unlock(&ctx->completion_lock);
8796 io_cqring_ev_posted(ctx);9256 io_cqring_ev_posted(ctx);
8797 io_ring_submit_unlock(ctx, lock_ring);9257
9258 if (ctx->flags & IORING_SETUP_IOPOLL)
9259 mutex_unlock(&ctx->uring_lock);
8798 }9260 }
8799 9261
8800 rsrc_data->do_put(ctx, prsrc);9262 rsrc_data->do_put(ctx, prsrc);
@@ -8848,27 +9310,31 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, · 47 unmodified lines
8848 if (ret)9310 if (ret)
8849 return ret;9311 return ret;
8850 9312
8851 ret = -ENOMEM;9313 if (!io_alloc_file_tables(&ctx->file_table, nr_args)) {
8852 if (!io_alloc_file_tables(&ctx->file_table, nr_args))9314 io_rsrc_data_free(ctx->file_data);
8853 goto out_free;9315 ctx->file_data = NULL;
9316 return -ENOMEM;
9317 }
8854 9318
8855 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {9319 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
8856 if (copy_from_user(&fd, &fds[i], sizeof(fd))) {9320 struct io_fixed_file *file_slot;
9321
9322 if (fds && copy_from_user(&fd, &fds[i], sizeof(fd))) {
8857 ret = -EFAULT;9323 ret = -EFAULT;
8858 goto out_fput;9324 goto fail;
8859 }9325 }
8860 /* allow sparse sets */9326 /* allow sparse sets */
8861 if (fd == -1) {9327 if (!fds || fd == -1) {
8862 ret = -EINVAL;9328 ret = -EINVAL;
8863 if (unlikely(*io_get_tag_slot(ctx->file_data, i)))9329 if (unlikely(*io_get_tag_slot(ctx->file_data, i)))
8864 goto out_fput;9330 goto fail;
8865 continue;9331 continue;
8866 }9332 }
8867 9333
8868 file = fget(fd);9334 file = fget(fd);
8869 ret = -EBADF;9335 ret = -EBADF;
8870 if (unlikely(!file))9336 if (unlikely(!file))
8871 goto out_fput;9337 goto fail;
8872 9338
8873 /*9339 /*
8874 * Don't allow io_uring instances to be registered. If UNIX9340 * Don't allow io_uring instances to be registered. If UNIX
@@ -8879,74 +9345,23 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, · 4 unmodified lines
8879 */9345 */
8880 if (file->f_op == &io_uring_fops) {9346 if (file->f_op == &io_uring_fops) {
8881 fput(file);9347 fput(file);
8882 goto out_fput;9348 goto fail;
8883 }9349 }
8884 io_fixed_file_set(io_fixed_file_slot(&ctx->file_table, i), file);9350 ret = io_scm_file_account(ctx, file);
8885 }9351 if (ret) {
8886
8887 ret = io_sqe_files_scm(ctx);
8888 if (ret) {
8889 __io_sqe_files_unregister(ctx);
8890 return ret;
8891 }
8892
8893 io_rsrc_node_switch(ctx, NULL);
8894 return ret;
8895out_fput:
8896 for (i = 0; i < ctx->nr_user_files; i++) {
8897 file = io_file_from_index(ctx, i);
8898 if (file)
8899 fput(file);9352 fput(file);
8900 }9353 goto fail;
8901 io_free_file_tables(&ctx->file_table);
8902 ctx->nr_user_files = 0;
8903out_free:
8904 io_rsrc_data_free(ctx->file_data);
8905 ctx->file_data = NULL;
8906 return ret;
8907}
8908
8909static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
8910 int index)
8911{
8912#if defined(CONFIG_UNIX)
8913 struct sock *sock = ctx->ring_sock->sk;
8914 struct sk_buff_head *head = &sock->sk_receive_queue;
8915 struct sk_buff *skb;
8916
8917 /*
8918 * See if we can merge this file into an existing skb SCM_RIGHTS
8919 * file set. If there's no room, fall back to allocating a new skb
8920 * and filling it in.
8921 */
8922 spin_lock_irq(&head->lock);
8923 skb = skb_peek(head);
8924 if (skb) {
8925 struct scm_fp_list *fpl = UNIXCB(skb).fp;
8926
8927 if (fpl->count < SCM_MAX_FD) {
8928 __skb_unlink(skb, head);
8929 spin_unlock_irq(&head->lock);
8930 fpl->fp[fpl->count] = get_file(file);
8931 unix_inflight(fpl->user, fpl->fp[fpl->count]);
8932 fpl->count++;
8933 spin_lock_irq(&head->lock);
8934 __skb_queue_head(head, skb);
8935 } else {
8936 skb = NULL;
8937 }9354 }
8938 }9355 file_slot = io_fixed_file_slot(&ctx->file_table, i);
8939 spin_unlock_irq(&head->lock);9356 io_fixed_file_set(file_slot, file);
8940 9357 io_file_bitmap_set(&ctx->file_table, i);
8941 if (skb) {
8942 fput(file);
8943 return 0;
8944 }9358 }
8945 9359
8946 return __io_sqe_files_scm(ctx, 1, index);9360 io_rsrc_node_switch(ctx, NULL);
8947#else
8948 return 0;9361 return 0;
8949#endif9362fail:
9363 __io_sqe_files_unregister(ctx);
9364 return ret;
8950}9365}
8951 9366
8952static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,9367static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
@@ -8970,12 +9385,11 @@ static int io_install_fixed_file(struct io_kiocb *req, struct file *file, · 17 unmodified lines
8970 unsigned int issue_flags, u32 slot_index)9385 unsigned int issue_flags, u32 slot_index)
8971{9386{
8972 struct io_ring_ctx *ctx = req->ctx;9387 struct io_ring_ctx *ctx = req->ctx;
8973 bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
8974 bool needs_switch = false;9388 bool needs_switch = false;
8975 struct io_fixed_file *file_slot;9389 struct io_fixed_file *file_slot;
8976 int ret = -EBADF;9390 int ret = -EBADF;
8977 9391
8978 io_ring_submit_lock(ctx, needs_lock);9392 io_ring_submit_lock(ctx, issue_flags);
8979 if (file->f_op == &io_uring_fops)9393 if (file->f_op == &io_uring_fops)
8980 goto err;9394 goto err;
8981 ret = -ENXIO;9395 ret = -ENXIO;
@@ -9001,22 +9415,20 @@ static int io_install_fixed_file(struct io_kiocb *req, struct file *file, · 19 unmodified lines
9001 if (ret)9415 if (ret)
9002 goto err;9416 goto err;
9003 file_slot->file_ptr = 0;9417 file_slot->file_ptr = 0;
9418 io_file_bitmap_clear(&ctx->file_table, slot_index);
9004 needs_switch = true;9419 needs_switch = true;
9005 }9420 }
9006 9421
9007 *io_get_tag_slot(ctx->file_data, slot_index) = 0;9422 ret = io_scm_file_account(ctx, file);
9008 io_fixed_file_set(file_slot, file);9423 if (!ret) {
9009 ret = io_sqe_file_register(ctx, file, slot_index);9424 *io_get_tag_slot(ctx->file_data, slot_index) = 0;
9010 if (ret) {9425 io_fixed_file_set(file_slot, file);
9011 file_slot->file_ptr = 0;9426 io_file_bitmap_set(&ctx->file_table, slot_index);
9012 goto err;
9013 }9427 }
9014
9015 ret = 0;
9016err:9428err:
9017 if (needs_switch)9429 if (needs_switch)
9018 io_rsrc_node_switch(ctx, ctx->file_data);9430 io_rsrc_node_switch(ctx, ctx->file_data);
9019 io_ring_submit_unlock(ctx, needs_lock);9431 io_ring_submit_unlock(ctx, issue_flags);
9020 if (ret)9432 if (ret)
9021 fput(file);9433 fput(file);
9022 return ret;9434 return ret;
@@ -9026,12 +9438,11 @@ static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags) · 3 unmodified lines
9026{9438{
9027 unsigned int offset = req->close.file_slot - 1;9439 unsigned int offset = req->close.file_slot - 1;
9028 struct io_ring_ctx *ctx = req->ctx;9440 struct io_ring_ctx *ctx = req->ctx;
9029 bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
9030 struct io_fixed_file *file_slot;9441 struct io_fixed_file *file_slot;
9031 struct file *file;9442 struct file *file;
9032 int ret;9443 int ret;
9033 9444
9034 io_ring_submit_lock(ctx, needs_lock);9445 io_ring_submit_lock(ctx, issue_flags);
9035 ret = -ENXIO;9446 ret = -ENXIO;
9036 if (unlikely(!ctx->file_data))9447 if (unlikely(!ctx->file_data))
9037 goto out;9448 goto out;
@@ -9054,10 +9465,11 @@ static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags) · 16 unmodified lines
9054 goto out;9465 goto out;
9055 9466
9056 file_slot->file_ptr = 0;9467 file_slot->file_ptr = 0;
9468 io_file_bitmap_clear(&ctx->file_table, offset);
9057 io_rsrc_node_switch(ctx, ctx->file_data);9469 io_rsrc_node_switch(ctx, ctx->file_data);
9058 ret = 0;9470 ret = 0;
9059out:9471out:
9060 io_ring_submit_unlock(ctx, needs_lock);9472 io_ring_submit_unlock(ctx, issue_flags);
9061 return ret;9473 return ret;
9062}9474}
9063 9475
@@ -9103,6 +9515,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, · 39 unmodified lines
9103 if (err)9515 if (err)
9104 break;9516 break;
9105 file_slot->file_ptr = 0;9517 file_slot->file_ptr = 0;
9518 io_file_bitmap_clear(&ctx->file_table, i);
9106 needs_switch = true;9519 needs_switch = true;
9107 }9520 }
9108 if (fd != -1) {9521 if (fd != -1) {
@@ -9124,14 +9537,14 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, · 15 unmodified lines
9124 err = -EBADF;9537 err = -EBADF;
9125 break;9538 break;
9126 }9539 }
9127 *io_get_tag_slot(data, i) = tag;9540 err = io_scm_file_account(ctx, file);
9128 io_fixed_file_set(file_slot, file);
9129 err = io_sqe_file_register(ctx, file, i);
9130 if (err) {9541 if (err) {
9131 file_slot->file_ptr = 0;
9132 fput(file);9542 fput(file);
9133 break;9543 break;
9134 }9544 }
9545 *io_get_tag_slot(data, i) = tag;
9546 io_fixed_file_set(file_slot, file);
9547 io_file_bitmap_set(&ctx->file_table, i);
9135 }9548 }
9136 }9549 }
9137 9550
@@ -9559,30 +9972,18 @@ static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages, · 421 unmodified lines
9559 return ret;9972 return ret;
9560}9973}
9561 9974
9562static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,9975static struct page **io_pin_pages(unsigned long ubuf, unsigned long len,
9563 struct io_mapped_ubuf **pimu,9976 int *npages)
9564 struct page **last_hpage)
9565{9977{
9566 struct io_mapped_ubuf *imu = NULL;9978 unsigned long start, end, nr_pages;
9567 struct vm_area_struct **vmas = NULL;9979 struct vm_area_struct **vmas = NULL;
9568 struct page **pages = NULL;9980 struct page **pages = NULL;
9569 unsigned long off, start, end, ubuf;9981 int i, pret, ret = -ENOMEM;
9570 size_t size;
9571 int ret, pret, nr_pages, i;
9572
9573 if (!iov->iov_base) {
9574 *pimu = ctx->dummy_ubuf;
9575 return 0;
9576 }
9577 9982
9578 ubuf = (unsigned long) iov->iov_base;9983 end = (ubuf + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
9579 end = (ubuf + iov->iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
9580 start = ubuf >> PAGE_SHIFT;9984 start = ubuf >> PAGE_SHIFT;
9581 nr_pages = end - start;9985 nr_pages = end - start;
9582 9986
9583 *pimu = NULL;
9584 ret = -ENOMEM;
9585
9586 pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);9987 pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
9587 if (!pages)9988 if (!pages)
9588 goto done;9989 goto done;
@@ -9592,10 +9993,6 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, · 3 unmodified lines
9592 if (!vmas)9993 if (!vmas)
9593 goto done;9994 goto done;
9594 9995
9595 imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);
9596 if (!imu)
9597 goto done;
9598
9599 ret = 0;9996 ret = 0;
9600 mmap_read_lock(current->mm);9997 mmap_read_lock(current->mm);
9601 pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM,9998 pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
@@ -9613,6 +10010,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, · 11 unmodified lines
9613 break;10010 break;
9614 }10011 }
9615 }10012 }
10013 *npages = nr_pages;
9616 } else {10014 } else {
9617 ret = pret < 0 ? pret : -EFAULT;10015 ret = pret < 0 ? pret : -EFAULT;
9618 }10016 }
@@ -9626,14 +10024,53 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, · 7 unmodified lines
9626 unpin_user_pages(pages, pret);10024 unpin_user_pages(pages, pret);
9627 goto done;10025 goto done;
9628 }10026 }
10027 ret = 0;
10028done:
10029 kvfree(vmas);
10030 if (ret < 0) {
10031 kvfree(pages);
10032 pages = ERR_PTR(ret);
10033 }
10034 return pages;
10035}
10036
10037static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
10038 struct io_mapped_ubuf **pimu,
10039 struct page **last_hpage)
10040{
10041 struct io_mapped_ubuf *imu = NULL;
10042 struct page **pages = NULL;
10043 unsigned long off;
10044 size_t size;
10045 int ret, nr_pages, i;
10046
10047 if (!iov->iov_base) {
10048 *pimu = ctx->dummy_ubuf;
10049 return 0;
10050 }
10051
10052 *pimu = NULL;
10053 ret = -ENOMEM;
10054
10055 pages = io_pin_pages((unsigned long) iov->iov_base, iov->iov_len,
10056 &nr_pages);
10057 if (IS_ERR(pages)) {
10058 ret = PTR_ERR(pages);
10059 pages = NULL;
10060 goto done;
10061 }
10062
10063 imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);
10064 if (!imu)
10065 goto done;
9629 10066
9630 ret = io_buffer_account_pin(ctx, pages, pret, imu, last_hpage);10067 ret = io_buffer_account_pin(ctx, pages, nr_pages, imu, last_hpage);
9631 if (ret) {10068 if (ret) {
9632 unpin_user_pages(pages, pret);10069 unpin_user_pages(pages, nr_pages);
9633 goto done;10070 goto done;
9634 }10071 }
9635 10072
9636 off = ubuf & ~PAGE_MASK;10073 off = (unsigned long) iov->iov_base & ~PAGE_MASK;
9637 size = iov->iov_len;10074 size = iov->iov_len;
9638 for (i = 0; i < nr_pages; i++) {10075 for (i = 0; i < nr_pages; i++) {
9639 size_t vec_len;10076 size_t vec_len;
@@ -9646,8 +10083,8 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, · 6 unmodified lines
9646 size -= vec_len;10083 size -= vec_len;
9647 }10084 }
9648 /* store original address for later verification */10085 /* store original address for later verification */
9649 imu->ubuf = ubuf;10086 imu->ubuf = (unsigned long) iov->iov_base;
9650 imu->ubuf_end = ubuf + iov->iov_len;10087 imu->ubuf_end = imu->ubuf + iov->iov_len;
9651 imu->nr_bvecs = nr_pages;10088 imu->nr_bvecs = nr_pages;
9652 *pimu = imu;10089 *pimu = imu;
9653 ret = 0;10090 ret = 0;
@@ -9655,7 +10092,6 @@ done: · 1 unmodified line
9655 if (ret)10092 if (ret)
9656 kvfree(imu);10093 kvfree(imu);
9657 kvfree(pages);10094 kvfree(pages);
9658 kvfree(vmas);
9659 return ret;10095 return ret;
9660}10096}
9661 10097
@@ -9714,12 +10150,17 @@ static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, · 52 unmodified lines
9714 }10150 }
9715 10151
9716 for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) {10152 for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) {
9717 ret = io_copy_iov(ctx, &iov, arg, i);10153 if (arg) {
9718 if (ret)10154 ret = io_copy_iov(ctx, &iov, arg, i);
9719 break;10155 if (ret)
9720 ret = io_buffer_validate(&iov);10156 break;
9721 if (ret)10157 ret = io_buffer_validate(&iov);
9722 break;10158 if (ret)
10159 break;
10160 } else {
10161 memset(&iov, 0, sizeof(iov));
10162 }
10163
9723 if (!iov.iov_base && *io_get_tag_slot(data, i)) {10164 if (!iov.iov_base && *io_get_tag_slot(data, i)) {
9724 ret = -EINVAL;10165 ret = -EINVAL;
9725 break;10166 break;
@@ -9858,19 +10299,19 @@ static int io_eventfd_unregister(struct io_ring_ctx *ctx) · 132 unmodified lines
9858 10299
9859static void io_destroy_buffers(struct io_ring_ctx *ctx)10300static void io_destroy_buffers(struct io_ring_ctx *ctx)
9860{10301{
10302 struct io_buffer_list *bl;
10303 unsigned long index;
9861 int i;10304 int i;
9862 10305
9863 for (i = 0; i < (1U << IO_BUFFERS_HASH_BITS); i++) {10306 for (i = 0; i < BGID_ARRAY; i++) {
9864 struct list_head *list = &ctx->io_buffers[i];10307 if (!ctx->io_bl)
9865 10308 break;
9866 while (!list_empty(list)) {10309 __io_remove_buffers(ctx, &ctx->io_bl[i], -1U);
9867 struct io_buffer_list *bl;10310 }
9868 10311
9869 bl = list_first_entry(list, struct io_buffer_list, list);10312 xa_for_each(&ctx->io_bl_xa, index, bl) {
9870 __io_remove_buffers(ctx, bl, -1U);10313 xa_erase(&ctx->io_bl_xa, bl->bgid);
9871 list_del(&bl->list);10314 __io_remove_buffers(ctx, bl, -1U);
9872 kfree(bl);
9873 }
9874 }10315 }
9875 10316
9876 while (!list_empty(&ctx->io_buffers_pages)) {10317 while (!list_empty(&ctx->io_buffers_pages)) {
@@ -9890,7 +10331,7 @@ static void io_req_caches_free(struct io_ring_ctx *ctx) · 13 unmodified lines
9890 mutex_lock(&ctx->uring_lock);10331 mutex_lock(&ctx->uring_lock);
9891 io_flush_cached_locked_reqs(ctx, state);10332 io_flush_cached_locked_reqs(ctx, state);
9892 10333
9893 while (state->free_list.next) {10334 while (!io_req_cache_empty(ctx)) {
9894 struct io_wq_work_node *node;10335 struct io_wq_work_node *node;
9895 struct io_kiocb *req;10336 struct io_kiocb *req;
9896 10337
@@ -9979,7 +10420,8 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) · 82 unmodified lines
9979 io_wq_put_hash(ctx->hash_map);10420 io_wq_put_hash(ctx->hash_map);
9980 kfree(ctx->cancel_hash);10421 kfree(ctx->cancel_hash);
9981 kfree(ctx->dummy_ubuf);10422 kfree(ctx->dummy_ubuf);
9982 kfree(ctx->io_buffers);10423 kfree(ctx->io_bl);
10424 xa_destroy(&ctx->io_bl_xa);
9983 kfree(ctx);10425 kfree(ctx);
9984}10426}
9985 10427
@@ -10010,7 +10452,8 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) · 24 unmodified lines
10010 * Users may get EPOLLIN meanwhile seeing nothing in cqring, this10452 * Users may get EPOLLIN meanwhile seeing nothing in cqring, this
10011 * pushs them to do the flush.10453 * pushs them to do the flush.
10012 */10454 */
10013 if (io_cqring_events(ctx) || test_bit(0, &ctx->check_cq_overflow))10455 if (io_cqring_events(ctx) ||
10456 test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq))
10014 mask |= EPOLLIN | EPOLLRDNORM;10457 mask |= EPOLLIN | EPOLLRDNORM;
10015 10458
10016 return mask;10459 return mask;
@@ -10142,8 +10585,7 @@ static __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, · 125 unmodified lines
10142 }10585 }
10143 }10586 }
10144 spin_unlock_irq(&ctx->timeout_lock);10587 spin_unlock_irq(&ctx->timeout_lock);
10145 if (canceled != 0)10588 io_commit_cqring(ctx);
10146 io_commit_cqring(ctx);
10147 spin_unlock(&ctx->completion_lock);10589 spin_unlock(&ctx->completion_lock);
10148 if (canceled != 0)10590 if (canceled != 0)
10149 io_cqring_ev_posted(ctx);10591 io_cqring_ev_posted(ctx);
@@ -10163,11 +10605,13 @@ static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) · 13 unmodified lines
10163 io_unregister_personality(ctx, index);10605 io_unregister_personality(ctx, index);
10164 mutex_unlock(&ctx->uring_lock);10606 mutex_unlock(&ctx->uring_lock);
10165 10607
10166 io_kill_timeouts(ctx, NULL, true);10608 /* failed during ring init, it couldn't have issued any requests */
10167 io_poll_remove_all(ctx, NULL, true);10609 if (ctx->rings) {
10168 10610 io_kill_timeouts(ctx, NULL, true);
10169 /* if we failed setting up the ctx, we might not have any rings */10611 io_poll_remove_all(ctx, NULL, true);
10170 io_iopoll_try_reap_events(ctx);10612 /* if we failed setting up the ctx, we might not have any rings */
10613 io_iopoll_try_reap_events(ctx);
10614 }
10171 10615
10172 INIT_WORK(&ctx->exit_work, io_ring_exit_work);10616 INIT_WORK(&ctx->exit_work, io_ring_exit_work);
10173 /*10617 /*
@@ -10259,6 +10703,10 @@ static __cold void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, · 85 unmodified lines
10259 struct io_task_cancel cancel = { .task = task, .all = cancel_all, };10703 struct io_task_cancel cancel = { .task = task, .all = cancel_all, };
10260 struct io_uring_task *tctx = task ? task->io_uring : NULL;10704 struct io_uring_task *tctx = task ? task->io_uring : NULL;
10261 10705
10706 /* failed during ring init, it couldn't have issued any requests */
10707 if (!ctx->rings)
10708 return;
10709
10262 while (1) {10710 while (1) {
10263 enum io_wq_cancel cret;10711 enum io_wq_cancel cret;
10264 bool ret = false;10712 bool ret = false;
@@ -10704,6 +11152,19 @@ static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx) · 439 unmodified lines
10704 return 0;11152 return 0;
10705}11153}
10706 11154
11155static int io_validate_ext_arg(unsigned flags, const void __user *argp, size_t argsz)
11156{
11157 if (flags & IORING_ENTER_EXT_ARG) {
11158 struct io_uring_getevents_arg arg;
11159
11160 if (argsz != sizeof(arg))
11161 return -EINVAL;
11162 if (copy_from_user(&arg, argp, sizeof(arg)))
11163 return -EFAULT;
11164 }
11165 return 0;
11166}
11167
10707static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz,11168static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz,
10708 struct __kernel_timespec __user **ts,11169 struct __kernel_timespec __user **ts,
10709 const sigset_t __user **sig)11170 const sigset_t __user **sig)
@@ -10741,7 +11202,6 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 31 unmodified lines
10741 size_t, argsz)11202 size_t, argsz)
10742{11203{
10743 struct io_ring_ctx *ctx;11204 struct io_ring_ctx *ctx;
10744 int submitted = 0;
10745 struct fd f;11205 struct fd f;
10746 long ret;11206 long ret;
10747 11207
@@ -10804,39 +11264,64 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 56 unmodified lines
10804 if (ret)11264 if (ret)
10805 goto out;11265 goto out;
10806 }11266 }
10807 submitted = to_submit;11267 ret = to_submit;
10808 } else if (to_submit) {11268 } else if (to_submit) {
10809 ret = io_uring_add_tctx_node(ctx);11269 ret = io_uring_add_tctx_node(ctx);
10810 if (unlikely(ret))11270 if (unlikely(ret))
10811 goto out;11271 goto out;
10812 mutex_lock(&ctx->uring_lock);
10813 submitted = io_submit_sqes(ctx, to_submit);
10814 mutex_unlock(&ctx->uring_lock);
10815 11272
10816 if (submitted != to_submit)11273 mutex_lock(&ctx->uring_lock);
11274 ret = io_submit_sqes(ctx, to_submit);
11275 if (ret != to_submit) {
11276 mutex_unlock(&ctx->uring_lock);
10817 goto out;11277 goto out;
11278 }
11279 if ((flags & IORING_ENTER_GETEVENTS) && ctx->syscall_iopoll)
11280 goto iopoll_locked;
11281 mutex_unlock(&ctx->uring_lock);
10818 }11282 }
10819 if (flags & IORING_ENTER_GETEVENTS) {11283 if (flags & IORING_ENTER_GETEVENTS) {
10820 const sigset_t __user *sig;11284 int ret2;
10821 struct __kernel_timespec __user *ts;11285 if (ctx->syscall_iopoll) {
10822 11286 /*
10823 ret = io_get_ext_arg(flags, argp, &argsz, &ts, &sig);11287 * We disallow the app entering submit/complete with
10824 if (unlikely(ret))11288 * polling, but we still need to lock the ring to
10825 goto out;11289 * prevent racing with polled issue that got punted to
11290 * a workqueue.
11291 */
11292 mutex_lock(&ctx->uring_lock);
11293iopoll_locked:
11294 ret2 = io_validate_ext_arg(flags, argp, argsz);
11295 if (likely(!ret2)) {
11296 min_complete = min(min_complete,
11297 ctx->cq_entries);
11298 ret2 = io_iopoll_check(ctx, min_complete);
11299 }
11300 mutex_unlock(&ctx->uring_lock);
11301 } else {
11302 const sigset_t __user *sig;
11303 struct __kernel_timespec __user *ts;
11304
11305 ret2 = io_get_ext_arg(flags, argp, &argsz, &ts, &sig);
11306 if (likely(!ret2)) {
11307 min_complete = min(min_complete,
11308 ctx->cq_entries);
11309 ret2 = io_cqring_wait(ctx, min_complete, sig,
11310 argsz, ts);
11311 }
11312 }
10826 11313
10827 min_complete = min(min_complete, ctx->cq_entries);11314 if (!ret) {
11315 ret = ret2;
10828 11316
10829 /*11317 /*
10830 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user11318 * EBADR indicates that one or more CQE were dropped.
10831 * space applications don't need to do io completion events11319 * Once the user has been informed we can clear the bit
10832 * polling again, they can rely on io_sq_thread to do polling11320 * as they are obviously ok with those drops.
10833 * work, which can reduce cpu usage and uring_lock contention.11321 */
10834 */11322 if (unlikely(ret2 == -EBADR))
10835 if (ctx->flags & IORING_SETUP_IOPOLL &&11323 clear_bit(IO_CHECK_CQ_DROPPED_BIT,
10836 !(ctx->flags & IORING_SETUP_SQPOLL)) {11324 &ctx->check_cq);
10837 ret = io_iopoll_check(ctx, min_complete);
10838 } else {
10839 ret = io_cqring_wait(ctx, min_complete, sig, argsz, ts);
10840 }11325 }
10841 }11326 }
10842 11327
@@ -10845,7 +11330,7 @@ out: · 2 unmodified lines
10845out_fput:11330out_fput:
10846 if (!(flags & IORING_ENTER_REGISTERED_RING))11331 if (!(flags & IORING_ENTER_REGISTERED_RING))
10847 fdput(f);11332 fdput(f);
10848 return submitted ? submitted : ret;11333 return ret;
10849}11334}
10850 11335
10851#ifdef CONFIG_PROC_FS11336#ifdef CONFIG_PROC_FS
@@ -11162,10 +11647,40 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, · 310 unmodified lines
11162 ctx = io_ring_ctx_alloc(p);11647 ctx = io_ring_ctx_alloc(p);
11163 if (!ctx)11648 if (!ctx)
11164 return -ENOMEM;11649 return -ENOMEM;
11650
11651 /*
11652 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
11653 * space applications don't need to do io completion events
11654 * polling again, they can rely on io_sq_thread to do polling
11655 * work, which can reduce cpu usage and uring_lock contention.
11656 */
11657 if (ctx->flags & IORING_SETUP_IOPOLL &&
11658 !(ctx->flags & IORING_SETUP_SQPOLL))
11659 ctx->syscall_iopoll = 1;
11660
11165 ctx->compat = in_compat_syscall();11661 ctx->compat = in_compat_syscall();
11166 if (!capable(CAP_IPC_LOCK))11662 if (!capable(CAP_IPC_LOCK))
11167 ctx->user = get_uid(current_user());11663 ctx->user = get_uid(current_user());
11168 11664
11665 /*
11666 * For SQPOLL, we just need a wakeup, always. For !SQPOLL, if
11667 * COOP_TASKRUN is set, then IPIs are never needed by the app.
11668 */
11669 ret = -EINVAL;
11670 if (ctx->flags & IORING_SETUP_SQPOLL) {
11671 /* IPI related flags don't make sense with SQPOLL */
11672 if (ctx->flags & (IORING_SETUP_COOP_TASKRUN |
11673 IORING_SETUP_TASKRUN_FLAG))
11674 goto err;
11675 ctx->notify_method = TWA_SIGNAL_NO_IPI;
11676 } else if (ctx->flags & IORING_SETUP_COOP_TASKRUN) {
11677 ctx->notify_method = TWA_SIGNAL_NO_IPI;
11678 } else {
11679 if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
11680 goto err;
11681 ctx->notify_method = TWA_SIGNAL;
11682 }
11683
11169 /*11684 /*
11170 * This is just grabbed for accounting purposes. When a process exits,11685 * This is just grabbed for accounting purposes. When a process exits,
11171 * the mm is exited and dropped before the files, hence we need to hang11686 * the mm is exited and dropped before the files, hence we need to hang
@@ -11263,10 +11778,11 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) · 91 unmodified lines
11263 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |11778 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
11264 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |11779 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
11265 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |11780 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
11266 IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL))11781 IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL |
11782 IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG))
11267 return -EINVAL;11783 return -EINVAL;
11268 11784
11269 return io_uring_create(entries, &p, params);11785 return io_uring_create(entries, &p, params);
11270}11786}
11271 11787
11272SYSCALL_DEFINE2(io_uring_setup, u32, entries,11788SYSCALL_DEFINE2(io_uring_setup, u32, entries,
@@ -11479,14 +11995,20 @@ static __cold int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, · 206 unmodified lines
11479 memset(&rr, 0, sizeof(rr));11995 memset(&rr, 0, sizeof(rr));
11480 if (copy_from_user(&rr, arg, size))11996 if (copy_from_user(&rr, arg, size))
11481 return -EFAULT;11997 return -EFAULT;
11482 if (!rr.nr || rr.resv || rr.resv2)11998 if (!rr.nr || rr.resv2)
11999 return -EINVAL;
12000 if (rr.flags & ~IORING_RSRC_REGISTER_SPARSE)
11483 return -EINVAL;12001 return -EINVAL;
11484 12002
11485 switch (type) {12003 switch (type) {
11486 case IORING_RSRC_FILE:12004 case IORING_RSRC_FILE:
12005 if (rr.flags & IORING_RSRC_REGISTER_SPARSE && rr.data)
12006 break;
11487 return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data),12007 return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data),
11488 rr.nr, u64_to_user_ptr(rr.tags));12008 rr.nr, u64_to_user_ptr(rr.tags));
11489 case IORING_RSRC_BUFFER:12009 case IORING_RSRC_BUFFER:
12010 if (rr.flags & IORING_RSRC_REGISTER_SPARSE && rr.data)
12011 break;
11490 return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data),12012 return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data),
11491 rr.nr, u64_to_user_ptr(rr.tags));12013 rr.nr, u64_to_user_ptr(rr.tags));
11492 }12014 }
@@ -11621,6 +12143,85 @@ err: · 128 unmodified lines
11621 return ret;12143 return ret;
11622}12144}
11623 12145
12146static int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg)
12147{
12148 struct io_uring_buf_ring *br;
12149 struct io_uring_buf_reg reg;
12150 struct io_buffer_list *bl;
12151 struct page **pages;
12152 int nr_pages;
12153
12154 if (copy_from_user(&reg, arg, sizeof(reg)))
12155 return -EFAULT;
12156
12157 if (reg.pad || reg.resv[0] || reg.resv[1] || reg.resv[2])
12158 return -EINVAL;
12159 if (!reg.ring_addr)
12160 return -EFAULT;

126 more lines aren’t shown here.

include/linux/sched/signal.h+11−2
@@ -355,14 +355,23 @@ static inline void clear_notify_signal(void) · 354 unmodified lines
355 smp_mb__after_atomic();355 smp_mb__after_atomic();
356}356}
357 357
358/*
359 * Returns 'true' if kick_process() is needed to force a transition from
360 * user -> kernel to guarantee expedient run of TWA_SIGNAL based task_work.
361 */
362static inline bool __set_notify_signal(struct task_struct *task)
363{
364 return !test_and_set_tsk_thread_flag(task, TIF_NOTIFY_SIGNAL) &&
365 !wake_up_state(task, TASK_INTERRUPTIBLE);
366}
367
358/*368/*
359 * Called to break out of interruptible wait loops, and enter the369 * Called to break out of interruptible wait loops, and enter the
360 * exit_to_user_mode_loop().370 * exit_to_user_mode_loop().
361 */371 */
362static inline void set_notify_signal(struct task_struct *task)372static inline void set_notify_signal(struct task_struct *task)
363{373{
364 if (!test_and_set_tsk_thread_flag(task, TIF_NOTIFY_SIGNAL) &&374 if (__set_notify_signal(task))
365 !wake_up_state(task, TASK_INTERRUPTIBLE))
366 kick_process(task);375 kick_process(task);
367}376}
368 377
include/linux/task_work.h+1−0
@@ -17,6 +17,7 @@ enum task_work_notify_mode { · 16 unmodified lines
17 TWA_NONE,17 TWA_NONE,
18 TWA_RESUME,18 TWA_RESUME,
19 TWA_SIGNAL,19 TWA_SIGNAL,
20 TWA_SIGNAL_NO_IPI,
20};21};
21 22
22static inline bool task_work_pending(struct task_struct *task)23static inline bool task_work_pending(struct task_struct *task)
include/trace/events/io_uring.h+42−2
@@ -520,7 +520,7 @@ TRACE_EVENT(io_uring_req_failed, · 519 unmodified lines
520 __entry->off = sqe->off;520 __entry->off = sqe->off;
521 __entry->addr = sqe->addr;521 __entry->addr = sqe->addr;
522 __entry->len = sqe->len;522 __entry->len = sqe->len;
523 __entry->op_flags = sqe->rw_flags;523 __entry->op_flags = sqe->poll32_events;
524 __entry->buf_index = sqe->buf_index;524 __entry->buf_index = sqe->buf_index;
525 __entry->personality = sqe->personality;525 __entry->personality = sqe->personality;
526 __entry->file_index = sqe->file_index;526 __entry->file_index = sqe->file_index;
@@ -530,7 +530,7 @@ TRACE_EVENT(io_uring_req_failed, · 3 unmodified lines
530 ),530 ),
531 531
532 TP_printk("ring %p, req %p, user_data 0x%llx, "532 TP_printk("ring %p, req %p, user_data 0x%llx, "
533 "op %d, flags 0x%x, prio=%d, off=%llu, addr=%llu, "533 "op %d, flags 0x%x, prio=%d, off=%llu, addr=%llu, "
534 "len=%u, rw_flags=0x%x, buf_index=%d, "534 "len=%u, rw_flags=0x%x, buf_index=%d, "
535 "personality=%d, file_index=%d, pad=0x%llx/%llx, error=%d",535 "personality=%d, file_index=%d, pad=0x%llx/%llx, error=%d",
536 __entry->ctx, __entry->req, __entry->user_data,536 __entry->ctx, __entry->req, __entry->user_data,
@@ -543,6 +543,46 @@ TRACE_EVENT(io_uring_req_failed, · 6 unmodified lines
543 (unsigned long long) __entry->pad2, __entry->error)543 (unsigned long long) __entry->pad2, __entry->error)
544);544);
545 545
546
547/*
548 * io_uring_cqe_overflow - a CQE overflowed
549 *
550 * @ctx: pointer to a ring context structure
551 * @user_data: user data associated with the request
552 * @res: CQE result
553 * @cflags: CQE flags
554 * @ocqe: pointer to the overflow cqe (if available)
555 *
556 */
557TRACE_EVENT(io_uring_cqe_overflow,
558
559 TP_PROTO(void *ctx, unsigned long long user_data, s32 res, u32 cflags,
560 void *ocqe),
561
562 TP_ARGS(ctx, user_data, res, cflags, ocqe),
563
564 TP_STRUCT__entry (
565 __field( void *, ctx )
566 __field( unsigned long long, user_data )
567 __field( s32, res )
568 __field( u32, cflags )
569 __field( void *, ocqe )
570 ),
571
572 TP_fast_assign(
573 __entry->ctx = ctx;
574 __entry->user_data = user_data;
575 __entry->res = res;
576 __entry->cflags = cflags;
577 __entry->ocqe = ocqe;
578 ),
579
580 TP_printk("ring %p, user_data 0x%llx, res %d, flags %x, "
581 "overflow_cqe %p",
582 __entry->ctx, __entry->user_data, __entry->res,
583 __entry->cflags, __entry->ocqe)
584);
585
546#endif /* _TRACE_IO_URING_H */586#endif /* _TRACE_IO_URING_H */
547 587
548/* This part must be outside protection */588/* This part must be outside protection */
include/uapi/linux/io_uring.h+94−1
@@ -63,6 +63,15 @@ struct io_uring_sqe { · 62 unmodified lines
63 __u64 __pad2[2];63 __u64 __pad2[2];
64};64};
65 65
66/*
67 * If sqe->file_index is set to this for opcodes that instantiate a new
68 * direct descriptor (like openat/openat2/accept), then io_uring will allocate
69 * an available direct descriptor instead of having the application pass one
70 * in. The picked direct descriptor will be returned in cqe->res, or -ENFILE
71 * if the space is full.
72 */
73#define IORING_FILE_INDEX_ALLOC (~0U)
74
66enum {75enum {
67 IOSQE_FIXED_FILE_BIT,76 IOSQE_FIXED_FILE_BIT,
68 IOSQE_IO_DRAIN_BIT,77 IOSQE_IO_DRAIN_BIT,
@@ -102,6 +111,20 @@ enum { · 33 unmodified lines
102#define IORING_SETUP_ATTACH_WQ (1U << 5) /* attach to existing wq */111#define IORING_SETUP_ATTACH_WQ (1U << 5) /* attach to existing wq */
103#define IORING_SETUP_R_DISABLED (1U << 6) /* start with ring disabled */112#define IORING_SETUP_R_DISABLED (1U << 6) /* start with ring disabled */
104#define IORING_SETUP_SUBMIT_ALL (1U << 7) /* continue submit on error */113#define IORING_SETUP_SUBMIT_ALL (1U << 7) /* continue submit on error */
114/*
115 * Cooperative task running. When requests complete, they often require
116 * forcing the submitter to transition to the kernel to complete. If this
117 * flag is set, work will be done when the task transitions anyway, rather
118 * than force an inter-processor interrupt reschedule. This avoids interrupting
119 * a task running in userspace, and saves an IPI.
120 */
121#define IORING_SETUP_COOP_TASKRUN (1U << 8)
122/*
123 * If COOP_TASKRUN is set, get notified if task work is available for
124 * running and a kernel transition would be needed to run it. This sets
125 * IORING_SQ_TASKRUN in the sq ring flags. Not valid with COOP_TASKRUN.
126 */
127#define IORING_SETUP_TASKRUN_FLAG (1U << 9)
105 128
106enum {129enum {
107 IORING_OP_NOP,130 IORING_OP_NOP,
@@ -187,6 +210,33 @@ enum { · 79 unmodified lines
187#define IORING_POLL_UPDATE_EVENTS (1U << 1)210#define IORING_POLL_UPDATE_EVENTS (1U << 1)
188#define IORING_POLL_UPDATE_USER_DATA (1U << 2)211#define IORING_POLL_UPDATE_USER_DATA (1U << 2)
189 212
213/*
214 * ASYNC_CANCEL flags.
215 *
216 * IORING_ASYNC_CANCEL_ALL Cancel all requests that match the given key
217 * IORING_ASYNC_CANCEL_FD Key off 'fd' for cancelation rather than the
218 * request 'user_data'
219 * IORING_ASYNC_CANCEL_ANY Match any request
220 */
221#define IORING_ASYNC_CANCEL_ALL (1U << 0)
222#define IORING_ASYNC_CANCEL_FD (1U << 1)
223#define IORING_ASYNC_CANCEL_ANY (1U << 2)
224
225/*
226 * send/sendmsg and recv/recvmsg flags (sqe->addr2)
227 *
228 * IORING_RECVSEND_POLL_FIRST If set, instead of first attempting to send
229 * or receive and arm poll if that yields an
230 * -EAGAIN result, arm poll upfront and skip
231 * the initial transfer attempt.
232 */
233#define IORING_RECVSEND_POLL_FIRST (1U << 0)
234
235/*
236 * accept flags stored in sqe->ioprio
237 */
238#define IORING_ACCEPT_MULTISHOT (1U << 0)
239
190/*240/*
191 * IO completion data structure (Completion Queue Entry)241 * IO completion data structure (Completion Queue Entry)
192 */242 */
@@ -236,6 +286,7 @@ struct io_sqring_offsets { · 43 unmodified lines
236 */286 */
237#define IORING_SQ_NEED_WAKEUP (1U << 0) /* needs io_uring_enter wakeup */287#define IORING_SQ_NEED_WAKEUP (1U << 0) /* needs io_uring_enter wakeup */
238#define IORING_SQ_CQ_OVERFLOW (1U << 1) /* CQ ring is overflown */288#define IORING_SQ_CQ_OVERFLOW (1U << 1) /* CQ ring is overflown */
289#define IORING_SQ_TASKRUN (1U << 2) /* task should enter the kernel */
239 290
240struct io_cqring_offsets {291struct io_cqring_offsets {
241 __u32 head;292 __u32 head;
@@ -333,6 +384,10 @@ enum { · 91 unmodified lines
333 IORING_REGISTER_RING_FDS = 20,384 IORING_REGISTER_RING_FDS = 20,
334 IORING_UNREGISTER_RING_FDS = 21,385 IORING_UNREGISTER_RING_FDS = 21,
335 386
387 /* register ring based provide buffer group */
388 IORING_REGISTER_PBUF_RING = 22,
389 IORING_UNREGISTER_PBUF_RING = 23,
390
336 /* this goes last */391 /* this goes last */
337 IORING_REGISTER_LAST392 IORING_REGISTER_LAST
338};393};
@@ -350,9 +405,15 @@ struct io_uring_files_update { · 11 unmodified lines
350 __aligned_u64 /* __s32 * */ fds;405 __aligned_u64 /* __s32 * */ fds;
351};406};
352 407
408/*
409 * Register a fully sparse file space, rather than pass in an array of all
410 * -1 file descriptors.
411 */
412#define IORING_RSRC_REGISTER_SPARSE (1U << 0)
413
353struct io_uring_rsrc_register {414struct io_uring_rsrc_register {
354 __u32 nr;415 __u32 nr;
355 __u32 resv;416 __u32 flags;
356 __u64 resv2;417 __u64 resv2;
357 __aligned_u64 data;418 __aligned_u64 data;
358 __aligned_u64 tags;419 __aligned_u64 tags;
@@ -404,6 +465,38 @@ struct io_uring_restriction { · 45 unmodified lines
404 __u32 resv2[3];465 __u32 resv2[3];
405};466};
406 467
468struct io_uring_buf {
469 __u64 addr;
470 __u32 len;
471 __u16 bid;
472 __u16 resv;
473};
474
475struct io_uring_buf_ring {
476 union {
477 /*
478 * To avoid spilling into more pages than we need to, the
479 * ring tail is overlaid with the io_uring_buf->resv field.
480 */
481 struct {
482 __u64 resv1;
483 __u32 resv2;
484 __u16 resv3;
485 __u16 tail;
486 };
487 struct io_uring_buf bufs[0];
488 };
489};
490
491/* argument for IORING_(UN)REGISTER_PBUF_RING */
492struct io_uring_buf_reg {
493 __u64 ring_addr;
494 __u32 ring_entries;
495 __u16 bgid;
496 __u16 pad;
497 __u64 resv[3];
498};
499
407/*500/*
408 * io_uring_restriction->opcode values501 * io_uring_restriction->opcode values
409 */502 */
kernel/task_work.c+19−6
@@ -12,12 +12,22 @@ static struct callback_head work_exited; /* all we need is ->next == NULL */ · 11 unmodified lines
12 * @notify: how to notify the targeted task12 * @notify: how to notify the targeted task
13 *13 *
14 * Queue @work for task_work_run() below and notify the @task if @notify14 * Queue @work for task_work_run() below and notify the @task if @notify
15 * is @TWA_RESUME or @TWA_SIGNAL. @TWA_SIGNAL works like signals, in that the15 * is @TWA_RESUME, @TWA_SIGNAL, or @TWA_SIGNAL_NO_IPI.
16 * it will interrupt the targeted task and run the task_work. @TWA_RESUME16 *
17 * work is run only when the task exits the kernel and returns to user mode,17 * @TWA_SIGNAL works like signals, in that the it will interrupt the targeted
18 * or before entering guest mode. Fails if the @task is exiting/exited and thus18 * task and run the task_work, regardless of whether the task is currently
19 * it can't process this @work. Otherwise @work->func() will be called when the19 * running in the kernel or userspace.
20 * @task goes through one of the aforementioned transitions, or exits.20 * @TWA_SIGNAL_NO_IPI works like @TWA_SIGNAL, except it doesn't send a
21 * reschedule IPI to force the targeted task to reschedule and run task_work.
22 * This can be advantageous if there's no strict requirement that the
23 * task_work be run as soon as possible, just whenever the task enters the
24 * kernel anyway.
25 * @TWA_RESUME work is run only when the task exits the kernel and returns to
26 * user mode, or before entering guest mode.
27 *
28 * Fails if the @task is exiting/exited and thus it can't process this @work.
29 * Otherwise @work->func() will be called when the @task goes through one of
30 * the aforementioned transitions, or exits.
21 *31 *
22 * If the targeted task is exiting, then an error is returned and the work item32 * If the targeted task is exiting, then an error is returned and the work item
23 * is not queued. It's up to the caller to arrange for an alternative mechanism33 * is not queued. It's up to the caller to arrange for an alternative mechanism
@@ -53,6 +63,9 @@ int task_work_add(struct task_struct *task, struct callback_head *work, · 29 unmodified lines
53 case TWA_SIGNAL:63 case TWA_SIGNAL:
54 set_notify_signal(task);64 set_notify_signal(task);
55 break;65 break;
66 case TWA_SIGNAL_NO_IPI:
67 __set_notify_signal(task);
68 break;
56 default:69 default:
57 WARN_ON_ONCE(1);70 WARN_ON_ONCE(1);
58 break;71 break;