Merge tag 'for-5.19/io_uring-2022-05-22' of git://git.kernel.dk/linux-block

Pull io_uring updates from Jens Axboe: "Here are the main io_uring changes for 5.19. This contains: - Fixes for sparse type warnings (Christoph, Vasily) - Support for multi-shot accept (Hao) - Support for io_uring managed fixed files, rather than always needing the applicationt o manage the indices (me) - Fix for a spurious poll wakeup (Dylan) - CQE overflow fixes (Dylan) - Support more types of cancelations (me) - Support for co-operative task_work signaling, rather than always forcing an IPI (me) - Support for doing poll first when appropriate, rather than always attempting a transfer first (me) - Provided buffer cleanups and support for mapped buffers (me) - Improve how io_uring handles inflight SCM files (Pavel) - Speedups for registered files (Pavel, me) - Organize the completion data in a struct in io_kiocb rather than keep it in separate spots (Pavel) - task_work improvements (Pavel) - Cleanup and optimize the submission path, in general and for handling links (Pavel) - Speedups for registered resource handling (Pavel) - Support sparse buffers and file maps (Pavel, me) - Various fixes and cleanups (Almog, Pavel, me)" * tag 'for-5.19/io_uring-2022-05-22' of git://git.kernel.dk/linux-block: (111 commits) io_uring: fix incorrect __kernel_rwf_t cast io_uring: disallow mixed provided buffer group registrations io_uring: initialize io_buffer_list head when shared ring is unregistered io_uring: add fully sparse buffer registration io_uring: use rcu_dereference in io_close io_uring: consistently use the EPOLL* defines io_uring: make apoll_events a __poll_t io_uring: drop a spurious inline on a forward declaration io_uring: don't use ERR_PTR for user pointers io_uring: use a rwf_t for io_rw.flags io_uring: add support for ring mapped supplied buffers io_uring: add io_pin_pages() helper io_uring: add buffer selection support to IORING_OP_NOP io_uring: fix locking state for empty buffer group io_uring: implement multishot mode for accept io_uring: let fast poll support multishot io_uring: add REQ_F_APOLL_MULTISHOT for requests io_uring: add IORING_ACCEPT_MULTISHOT for accept io_uring: only wake when the correct events are set io_uring: avoid io-wq -EAGAIN looping for !IOPOLL ...

Linus Torvalds committed 4 years ago

Commit 3a166bd2 parents 1e57930 0e7579c8 files · +1813 −1031Browse files

Unified · Split

8 files

fs/io-wq.c +2 −2
fs/io-wq.h +1 −0
fs/io_uring.c +1643 −1018
include/linux/sched/signal.h +11 −2
include/linux/task_work.h +1 −0
include/trace/events/io_uring.h +42 −2
include/uapi/linux/io_uring.h +94 −1
kernel/task_work.c +19 −6

fs/io-wq.c+2−2

@@ -871,7 +871,7 @@ static bool io_wq_for_each_worker(struct io_wqe *wqe, · 870 unmodified lines
871	871
872	872	static bool io_wq_worker_wake(struct io_worker worker, void data)
873	873	{
874		− set_notify_signal(worker->task);
	874	+ __set_notify_signal(worker->task);
875	875	wake_up_process(worker->task);
876	876	return false;
877	877	}
@@ -991,7 +991,7 @@ static bool __io_wq_worker_cancel(struct io_worker *worker, · 113 unmodified lines
991	991	{
992	992	if (work && match->fn(work, match->data)) {
993	993	work->flags \|= IO_WQ_WORK_CANCEL;
994		− set_notify_signal(worker->task);
	994	+ __set_notify_signal(worker->task);
995	995	return true;
996	996	}
997	997

fs/io-wq.h+1−0

@@ -155,6 +155,7 @@ struct io_wq_work_node wq_stack_extract(struct io_wq_work_node stack) · 154 unmodified lines
155	155	struct io_wq_work {
156	156	struct io_wq_work_node list;
157	157	unsigned flags;
	158	+ int cancel_seq;
158	159	};
159	160
160	161	static inline struct io_wq_work wq_next_work(struct io_wq_work work)

fs/io_uring.c+1643−1018

@@ -94,7 +94,7 @@ · 93 unmodified lines
94	94	#define IORING_SQPOLL_CAP_ENTRIES_VALUE 8
95	95
96	96	/* only define max */
97		−#define IORING_MAX_FIXED_FILES (1U << 15)
	97	+#define IORING_MAX_FIXED_FILES (1U << 20)
98	98	#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
99	99	IORING_REGISTER_LAST + IORING_OP_LAST)
100	100
@@ -113,6 +113,11 @@ · 12 unmodified lines
113	113	#define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED \| REQ_F_NEED_CLEANUP \| \
114	114	REQ_F_POLLED \| REQ_F_CREDS \| REQ_F_ASYNC_DATA)
115	115
	116	+#define IO_REQ_CLEAN_SLOW_FLAGS (REQ_F_REFCOUNT \| REQ_F_LINK \| REQ_F_HARDLINK \|\
	117	+ IO_REQ_CLEAN_FLAGS)
	118	+
	119	+#define IO_APOLL_MULTI_POLLED (REQ_F_APOLL_MULTISHOT \| REQ_F_POLLED)
	120	+
116	121	#define IO_TCTX_REFS_CACHE_NR (1U << 10)
117	122
118	123	struct io_uring {
@@ -166,7 +171,7 @@ struct io_rings { · 47 unmodified lines
166	171	* The application needs a full memory barrier before checking
167	172	* for IORING_SQ_NEED_WAKEUP after updating the sq tail.
168	173	*/
169		− u32 sq_flags;
	174	+ atomic_t sq_flags;
170	175	/*
171	176	* Runtime CQ flags
172	177	*
@@ -220,6 +225,23 @@ struct io_overflow_cqe { · 47 unmodified lines
220	225	struct list_head list;
221	226	};
222	227
	228	+/*
	229	+ * FFS_SCM is only available on 64-bit archs, for 32-bit we just define it as 0
	230	+ * and define IO_URING_SCM_ALL. For this case, we use SCM for all files as we
	231	+ * can't safely always dereference the file when the task has exited and ring
	232	+ * cleanup is done. If a file is tracked and part of SCM, then unix gc on
	233	+ * process exit may reap it before __io_sqe_files_unregister() is run.
	234	+ */
	235	+#define FFS_NOWAIT 0x1UL
	236	+#define FFS_ISREG 0x2UL
	237	+#if defined(CONFIG_64BIT)
	238	+#define FFS_SCM 0x4UL
	239	+#else
	240	+#define IO_URING_SCM_ALL
	241	+#define FFS_SCM 0x0UL
	242	+#endif
	243	+#define FFS_MASK ~(FFS_NOWAIT\|FFS_ISREG\|FFS_SCM)
	244	+
223	245	struct io_fixed_file {
224	246	/* file * with additional FFS_* flags */
225	247	unsigned long file_ptr;
@@ -237,6 +259,8 @@ struct io_rsrc_put { · 11 unmodified lines
237	259
238	260	struct io_file_table {
239	261	struct io_fixed_file *files;
	262	+ unsigned long *bitmap;
	263	+ unsigned int alloc_hint;
240	264	};
241	265
242	266	struct io_rsrc_node {
@@ -261,10 +285,26 @@ struct io_rsrc_data { · 18 unmodified lines
261	285	bool quiesce;
262	286	};
263	287
	288	+#define IO_BUFFER_LIST_BUF_PER_PAGE (PAGE_SIZE / sizeof(struct io_uring_buf))
264	289	struct io_buffer_list {
265		− struct list_head list;
266		− struct list_head buf_list;
	290	+ /*
	291	+ * If ->buf_nr_pages is set, then buf_pages/buf_ring are used. If not,
	292	+ * then these are classic provided buffers and ->buf_list is used.
	293	+ */
	294	+ union {
	295	+ struct list_head buf_list;
	296	+ struct {
	297	+ struct page **buf_pages;
	298	+ struct io_uring_buf_ring *buf_ring;
	299	+ };
	300	+ };
267	301	__u16 bgid;
	302	+
	303	+ /* below is for ring provided buffers */
	304	+ __u16 buf_nr_pages;
	305	+ __u16 nr_entries;
	306	+ __u32 head;
	307	+ __u32 mask;
268	308	};
269	309
270	310	struct io_buffer {
@@ -337,7 +377,7 @@ struct io_ev_fd { · 66 unmodified lines
337	377	struct rcu_head rcu;
338	378	};
339	379
340		−#define IO_BUFFERS_HASH_BITS 5
	380	+#define BGID_ARRAY 64
341	381
342	382	struct io_ring_ctx {
343	383	/* const or read-mostly hot data */
@@ -346,6 +386,7 @@ struct io_ring_ctx { · 2 unmodified lines
346	386
347	387	struct io_rings *rings;
348	388	unsigned int flags;
	389	+ enum task_work_notify_mode notify_method;
349	390	unsigned int compat: 1;
350	391	unsigned int drain_next: 1;
351	392	unsigned int restricted: 1;
@@ -353,6 +394,7 @@ struct io_ring_ctx { · 1 unmodified line
353	394	unsigned int drain_active: 1;
354	395	unsigned int drain_disabled: 1;
355	396	unsigned int has_evfd: 1;
	397	+ unsigned int syscall_iopoll: 1;
356	398	} ____cacheline_aligned_in_smp;
357	399
358	400	/* submission data */
@@ -382,17 +424,21 @@ struct io_ring_ctx { · 23 unmodified lines
382	424	*/
383	425	struct io_rsrc_node *rsrc_node;
384	426	int rsrc_cached_refs;
	427	+ atomic_t cancel_seq;
385	428	struct io_file_table file_table;
386	429	unsigned nr_user_files;
387	430	unsigned nr_user_bufs;
388	431	struct io_mapped_ubuf **user_bufs;
389	432
390	433	struct io_submit_state submit_state;
	434	+
	435	+ struct io_buffer_list *io_bl;
	436	+ struct xarray io_bl_xa;
	437	+ struct list_head io_buffers_cache;
	438	+
391	439	struct list_head timeout_list;
392	440	struct list_head ltimeout_list;
393	441	struct list_head cq_overflow_list;
394		− struct list_head *io_buffers;
395		− struct list_head io_buffers_cache;
396	442	struct list_head apoll_cache;
397	443	struct xarray personalities;
398	444	u32 pers_next;
@@ -409,9 +455,16 @@ struct io_ring_ctx { · 10 unmodified lines
409	455	struct wait_queue_head sqo_sq_wait;
410	456	struct list_head sqd_list;
411	457
412		− unsigned long check_cq_overflow;
	458	+ unsigned long check_cq;
413	459
414	460	struct {
	461	+ /*
	462	+ * We cache a range of free CQEs we can use, once exhausted it
	463	+ * should go through a slower range setup, see __io_get_cqe()
	464	+ */
	465	+ struct io_uring_cqe *cqe_cached;
	466	+ struct io_uring_cqe *cqe_sentinel;
	467	+
415	468	unsigned cached_cq_tail;
416	469	unsigned cq_entries;
417	470	struct io_ev_fd __rcu *io_ev_fd;
@@ -557,6 +610,8 @@ struct io_sync { · 139 unmodified lines
557	610	struct io_cancel {
558	611	struct file *file;
559	612	u64 addr;
	613	+ u32 flags;
	614	+ s32 fd;
560	615	};
561	616
562	617	struct io_timeout {
@@ -585,7 +640,7 @@ struct io_rw { · 22 unmodified lines
585	640	struct kiocb kiocb;
586	641	u64 addr;
587	642	u32 len;
588		− u32 flags;
	643	+ rwf_t flags;
589	644	};
590	645
591	646	struct io_connect {
@@ -602,9 +657,9 @@ struct io_sr_msg { · 10 unmodified lines
602	657	void __user *buf;
603	658	};
604	659	int msg_flags;
605		− int bgid;
606	660	size_t len;
607	661	size_t done_io;
	662	+ unsigned int flags;
608	663	};
609	664
610	665	struct io_open {
@@ -766,6 +821,7 @@ enum { · 155 unmodified lines
766	821	REQ_F_NEED_CLEANUP_BIT,
767	822	REQ_F_POLLED_BIT,
768	823	REQ_F_BUFFER_SELECTED_BIT,
	824	+ REQ_F_BUFFER_RING_BIT,
769	825	REQ_F_COMPLETE_INLINE_BIT,
770	826	REQ_F_REISSUE_BIT,
771	827	REQ_F_CREDS_BIT,
@@ -776,6 +832,7 @@ enum { · 4 unmodified lines
776	832	REQ_F_SINGLE_POLL_BIT,
777	833	REQ_F_DOUBLE_POLL_BIT,
778	834	REQ_F_PARTIAL_IO_BIT,
	835	+ REQ_F_APOLL_MULTISHOT_BIT,
779	836	/* keep async read/write and isreg together and in order */
780	837	REQ_F_SUPPORT_NOWAIT_BIT,
781	838	REQ_F_ISREG_BIT,
@@ -816,6 +873,8 @@ enum { · 34 unmodified lines
816	873	REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),
817	874	/* buffer already selected */
818	875	REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
	876	+ /* buffer selected from ring, needs commit */
	877	+ REQ_F_BUFFER_RING = BIT(REQ_F_BUFFER_RING_BIT),
819	878	/* completion is deferred through io_comp_state */
820	879	REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT),
821	880	/* caller should reissue async */
@@ -840,6 +899,8 @@ enum { · 18 unmodified lines
840	899	REQ_F_DOUBLE_POLL = BIT(REQ_F_DOUBLE_POLL_BIT),
841	900	/* request has already done partial IO */
842	901	REQ_F_PARTIAL_IO = BIT(REQ_F_PARTIAL_IO_BIT),
	902	+ /* fast poll multishot mode */
	903	+ REQ_F_APOLL_MULTISHOT = BIT(REQ_F_APOLL_MULTISHOT_BIT),
843	904	};
844	905
845	906	struct async_poll {
@@ -862,6 +923,21 @@ enum { · 16 unmodified lines
862	923	IORING_RSRC_BUFFER = 1,
863	924	};
864	925
	926	+struct io_cqe {
	927	+ __u64 user_data;
	928	+ __s32 res;
	929	+ /* fd initially, then cflags for completion */
	930	+ union {
	931	+ __u32 flags;
	932	+ int fd;
	933	+ };
	934	+};
	935	+
	936	+enum {
	937	+ IO_CHECK_CQ_OVERFLOW_BIT,
	938	+ IO_CHECK_CQ_DROPPED_BIT,
	939	+};
	940	+
865	941	/*
866	942	* NOTE! Each of the iocb union members has the file pointer
867	943	* as the first entry in their struct definition. So you can
@@ -902,29 +978,40 @@ struct io_kiocb { · 34 unmodified lines
902	978	u8 opcode;
903	979	/* polled IO has completed */
904	980	u8 iopoll_completed;
	981	+ /*
	982	+ * Can be either a fixed buffer index, or used with provided buffers.
	983	+ * For the latter, before issue it points to the buffer group ID,
	984	+ * and after selection it points to the buffer ID itself.
	985	+ */
905	986	u16 buf_index;
906	987	unsigned int flags;
907	988
908		− u64 user_data;
909		− u32 result;
910		− /* fd initially, then cflags for completion */
911		− union {
912		− u32 cflags;
913		− int fd;
914		− };
	989	+ struct io_cqe cqe;
915	990
916	991	struct io_ring_ctx *ctx;
917	992	struct task_struct *task;
918	993
919		− struct percpu_ref *fixed_rsrc_refs;
920		− /* store used ubuf, so we can prevent reloading */
921		− struct io_mapped_ubuf *imu;
	994	+ struct io_rsrc_node *rsrc_node;
	995	+
	996	+ union {
	997	+ /* store used ubuf, so we can prevent reloading */
	998	+ struct io_mapped_ubuf *imu;
	999	+
	1000	+ /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */
	1001	+ struct io_buffer *kbuf;
	1002	+
	1003	+ /*
	1004	+ * stores buffer ID for ring provided buffers, valid IFF
	1005	+ * REQ_F_BUFFER_RING is set.
	1006	+ */
	1007	+ struct io_buffer_list *buf_list;
	1008	+ };
922	1009
923	1010	union {
924	1011	/* used by request caches, completion batching and iopoll */
925	1012	struct io_wq_work_node comp_list;
926	1013	/* cache ->apoll->events */
927		− int apoll_events;
	1014	+ __poll_t apoll_events;
928	1015	};
929	1016	atomic_t refs;
930	1017	atomic_t poll_refs;
@@ -935,8 +1022,6 @@ struct io_kiocb { · 4 unmodified lines
935	1022	struct async_poll *apoll;
936	1023	/* opcode allocated if it needs to store data for async defer */
937	1024	void *async_data;
938		− /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */
939		− struct io_buffer *kbuf;
940	1025	/* linked requests, IFF REQ_F_HARDLINK or REQ_F_LINK are set */
941	1026	struct io_kiocb *link;
942	1027	/* custom credentials, valid IFF REQ_F_CREDS is set */
@@ -956,6 +1041,16 @@ struct io_defer_entry { · 13 unmodified lines
956	1041	u32 seq;
957	1042	};
958	1043
	1044	+struct io_cancel_data {
	1045	+ struct io_ring_ctx *ctx;
	1046	+ union {
	1047	+ u64 data;
	1048	+ struct file *file;
	1049	+ };
	1050	+ u32 flags;
	1051	+ int seq;
	1052	+};
	1053	+
959	1054	struct io_op_def {
960	1055	/* needs req->file assigned */
961	1056	unsigned needs_file : 1;
@@ -977,12 +1072,20 @@ struct io_op_def { · 15 unmodified lines
977	1072	unsigned not_supported : 1;
978	1073	/* skip auditing */
979	1074	unsigned audit_skip : 1;
	1075	+ /* supports ioprio */
	1076	+ unsigned ioprio : 1;
	1077	+ /* supports iopoll */
	1078	+ unsigned iopoll : 1;
980	1079	/* size of async data needed, if any */
981	1080	unsigned short async_size;
982	1081	};
983	1082
984	1083	static const struct io_op_def io_op_defs[] = {
985		− [IORING_OP_NOP] = {},
	1084	+ [IORING_OP_NOP] = {
	1085	+ .audit_skip = 1,
	1086	+ .iopoll = 1,
	1087	+ .buffer_select = 1,
	1088	+ },
986	1089	[IORING_OP_READV] = {
987	1090	.needs_file = 1,
988	1091	.unbound_nonreg_file = 1,
@@ -991,6 +1094,8 @@ static const struct io_op_def io_op_defs[] = { · 2 unmodified lines
991	1094	.needs_async_setup = 1,
992	1095	.plug = 1,
993	1096	.audit_skip = 1,
	1097	+ .ioprio = 1,
	1098	+ .iopoll = 1,
994	1099	.async_size = sizeof(struct io_async_rw),
995	1100	},
996	1101	[IORING_OP_WRITEV] = {
@@ -1001,6 +1106,8 @@ static const struct io_op_def io_op_defs[] = { · 4 unmodified lines
1001	1106	.needs_async_setup = 1,
1002	1107	.plug = 1,
1003	1108	.audit_skip = 1,
	1109	+ .ioprio = 1,
	1110	+ .iopoll = 1,
1004	1111	.async_size = sizeof(struct io_async_rw),
1005	1112	},
1006	1113	[IORING_OP_FSYNC] = {
@@ -1013,6 +1120,8 @@ static const struct io_op_def io_op_defs[] = { · 6 unmodified lines
1013	1120	.pollin = 1,
1014	1121	.plug = 1,
1015	1122	.audit_skip = 1,
	1123	+ .ioprio = 1,
	1124	+ .iopoll = 1,
1016	1125	.async_size = sizeof(struct io_async_rw),
1017	1126	},
1018	1127	[IORING_OP_WRITE_FIXED] = {
@@ -1022,6 +1131,8 @@ static const struct io_op_def io_op_defs[] = { · 3 unmodified lines
1022	1131	.pollout = 1,
1023	1132	.plug = 1,
1024	1133	.audit_skip = 1,
	1134	+ .ioprio = 1,
	1135	+ .iopoll = 1,
1025	1136	.async_size = sizeof(struct io_async_rw),
1026	1137	},
1027	1138	[IORING_OP_POLL_ADD] = {
@@ -1064,6 +1175,7 @@ static const struct io_op_def io_op_defs[] = { · 36 unmodified lines
1064	1175	.unbound_nonreg_file = 1,
1065	1176	.pollin = 1,
1066	1177	.poll_exclusive = 1,
	1178	+ .ioprio = 1, /* used for flags */
1067	1179	},
1068	1180	[IORING_OP_ASYNC_CANCEL] = {
1069	1181	.audit_skip = 1,
@@ -1086,6 +1198,7 @@ static const struct io_op_def io_op_defs[] = { · 16 unmodified lines
1086	1198	[IORING_OP_CLOSE] = {},
1087	1199	[IORING_OP_FILES_UPDATE] = {
1088	1200	.audit_skip = 1,
	1201	+ .iopoll = 1,
1089	1202	},
1090	1203	[IORING_OP_STATX] = {
1091	1204	.audit_skip = 1,
@@ -1097,6 +1210,8 @@ static const struct io_op_def io_op_defs[] = { · 5 unmodified lines
1097	1210	.buffer_select = 1,
1098	1211	.plug = 1,
1099	1212	.audit_skip = 1,
	1213	+ .ioprio = 1,
	1214	+ .iopoll = 1,
1100	1215	.async_size = sizeof(struct io_async_rw),
1101	1216	},
1102	1217	[IORING_OP_WRITE] = {
@@ -1106,6 +1221,8 @@ static const struct io_op_def io_op_defs[] = { · 3 unmodified lines
1106	1221	.pollout = 1,
1107	1222	.plug = 1,
1108	1223	.audit_skip = 1,
	1224	+ .ioprio = 1,
	1225	+ .iopoll = 1,
1109	1226	.async_size = sizeof(struct io_async_rw),
1110	1227	},
1111	1228	[IORING_OP_FADVISE] = {
@@ -1140,9 +1257,11 @@ static const struct io_op_def io_op_defs[] = { · 28 unmodified lines
1140	1257	},
1141	1258	[IORING_OP_PROVIDE_BUFFERS] = {
1142	1259	.audit_skip = 1,
	1260	+ .iopoll = 1,
1143	1261	},
1144	1262	[IORING_OP_REMOVE_BUFFERS] = {
1145	1263	.audit_skip = 1,
	1264	+ .iopoll = 1,
1146	1265	},
1147	1266	[IORING_OP_TEE] = {
1148	1267	.needs_file = 1,
@@ -1160,11 +1279,13 @@ static const struct io_op_def io_op_defs[] = { · 11 unmodified lines
1160	1279	[IORING_OP_LINKAT] = {},
1161	1280	[IORING_OP_MSG_RING] = {
1162	1281	.needs_file = 1,
	1282	+ .iopoll = 1,
1163	1283	},
1164	1284	};
1165	1285
1166	1286	/* requests with any of those set should undergo io_disarm_next() */
1167	1287	#define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT \| REQ_F_LINK_TIMEOUT \| REQ_F_FAIL)
	1288	+#define IO_REQ_LINK_FLAGS (REQ_F_LINK \| REQ_F_HARDLINK)
1168	1289
1169	1290	static bool io_disarm_next(struct io_kiocb *req);
1170	1291	static void io_uring_del_tctx_node(unsigned long index);
@@ -1173,10 +1294,7 @@ static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, · 2 unmodified lines
1173	1294	bool cancel_all);
1174	1295	static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd);
1175	1296
1176		−static void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags);
1177		−
1178		−static void io_put_req(struct io_kiocb *req);
1179		−static void io_put_req_deferred(struct io_kiocb *req);
	1297	+static void __io_req_complete_post(struct io_kiocb *req, s32 res, u32 cflags);
1180	1298	static void io_dismantle_req(struct io_kiocb *req);
1181	1299	static void io_queue_linked_timeout(struct io_kiocb *req);
1182	1300	static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
@@ -1185,10 +1303,10 @@ static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type, · 2 unmodified lines
1185	1303	static void io_clean_op(struct io_kiocb *req);
1186	1304	static inline struct file io_file_get_fixed(struct io_kiocb req, int fd,
1187	1305	unsigned issue_flags);
1188		−static inline struct file io_file_get_normal(struct io_kiocb req, int fd);
	1306	+static struct file io_file_get_normal(struct io_kiocb req, int fd);
1189	1307	static void io_drop_inflight_file(struct io_kiocb *req);
1190	1308	static bool io_assign_file(struct io_kiocb *req, unsigned int issue_flags);
1191		−static void __io_queue_sqe(struct io_kiocb *req);
	1309	+static void io_queue_sqe(struct io_kiocb *req);
1192	1310	static void io_rsrc_put_work(struct work_struct *work);
1193	1311
1194	1312	static void io_req_task_queue(struct io_kiocb *req);
@@ -1201,6 +1319,7 @@ static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags); · 6 unmodified lines
1201	1319
1202	1320	static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer);
1203	1321	static void io_eventfd_signal(struct io_ring_ctx *ctx);
	1322	+static void io_req_tw_post_queue(struct io_kiocb *req, s32 res, u32 cflags);
1204	1323
1205	1324	static struct kmem_cache *req_cachep;
1206	1325
@@ -1219,6 +1338,42 @@ struct sock io_uring_get_socket(struct file file) · 12 unmodified lines
1219	1338	}
1220	1339	EXPORT_SYMBOL(io_uring_get_socket);
1221	1340
	1341	+#if defined(CONFIG_UNIX)
	1342	+static inline bool io_file_need_scm(struct file *filp)
	1343	+{
	1344	+#if defined(IO_URING_SCM_ALL)
	1345	+ return true;
	1346	+#else
	1347	+ return !!unix_get_socket(filp);
	1348	+#endif
	1349	+}
	1350	+#else
	1351	+static inline bool io_file_need_scm(struct file *filp)
	1352	+{
	1353	+ return false;
	1354	+}
	1355	+#endif
	1356	+
	1357	+static void io_ring_submit_unlock(struct io_ring_ctx *ctx, unsigned issue_flags)
	1358	+{
	1359	+ lockdep_assert_held(&ctx->uring_lock);
	1360	+ if (issue_flags & IO_URING_F_UNLOCKED)
	1361	+ mutex_unlock(&ctx->uring_lock);
	1362	+}
	1363	+
	1364	+static void io_ring_submit_lock(struct io_ring_ctx *ctx, unsigned issue_flags)
	1365	+{
	1366	+ /*
	1367	+ * "Normal" inline submissions always hold the uring_lock, since we
	1368	+ * grab it from the system call. Same is true for the SQPOLL offload.
	1369	+ * The only exception is when we've detached the request and issue it
	1370	+ * from an async worker thread, grab the lock for that case.
	1371	+ */
	1372	+ if (issue_flags & IO_URING_F_UNLOCKED)
	1373	+ mutex_lock(&ctx->uring_lock);
	1374	+ lockdep_assert_held(&ctx->uring_lock);
	1375	+}
	1376	+
1222	1377	static inline void io_tw_lock(struct io_ring_ctx ctx, bool locked)
1223	1378	{
1224	1379	if (!*locked) {
@@ -1280,31 +1435,36 @@ static inline void io_req_set_refcount(struct io_kiocb *req) · 55 unmodified lines
1280	1435
1281	1436	#define IO_RSRC_REF_BATCH 100
1282	1437
	1438	+static void io_rsrc_put_node(struct io_rsrc_node *node, int nr)
	1439	+{
	1440	+ percpu_ref_put_many(&node->refs, nr);
	1441	+}
	1442	+
1283	1443	static inline void io_req_put_rsrc_locked(struct io_kiocb *req,
1284	1444	struct io_ring_ctx *ctx)
1285	1445	__must_hold(&ctx->uring_lock)
1286	1446	{
1287		− struct percpu_ref *ref = req->fixed_rsrc_refs;
	1447	+ struct io_rsrc_node *node = req->rsrc_node;
1288	1448
1289		− if (ref) {
1290		− if (ref == &ctx->rsrc_node->refs)
	1449	+ if (node) {
	1450	+ if (node == ctx->rsrc_node)
1291	1451	ctx->rsrc_cached_refs++;
1292	1452	else
1293		− percpu_ref_put(ref);
	1453	+ io_rsrc_put_node(node, 1);
1294	1454	}
1295	1455	}
1296	1456
1297		−static inline void io_req_put_rsrc(struct io_kiocb req, struct io_ring_ctx ctx)
	1457	+static inline void io_req_put_rsrc(struct io_kiocb *req)
1298	1458	{
1299		− if (req->fixed_rsrc_refs)
1300		− percpu_ref_put(req->fixed_rsrc_refs);
	1459	+ if (req->rsrc_node)
	1460	+ io_rsrc_put_node(req->rsrc_node, 1);
1301	1461	}
1302	1462
1303	1463	static __cold void io_rsrc_refs_drop(struct io_ring_ctx *ctx)
1304	1464	__must_hold(&ctx->uring_lock)
1305	1465	{
1306	1466	if (ctx->rsrc_cached_refs) {
1307		− percpu_ref_put_many(&ctx->rsrc_node->refs, ctx->rsrc_cached_refs);
	1467	+ io_rsrc_put_node(ctx->rsrc_node, ctx->rsrc_cached_refs);
1308	1468	ctx->rsrc_cached_refs = 0;
1309	1469	}
1310	1470	}
@@ -1320,8 +1480,8 @@ static inline void io_req_set_rsrc_node(struct io_kiocb *req, · 9 unmodified lines
1320	1480	struct io_ring_ctx *ctx,
1321	1481	unsigned int issue_flags)
1322	1482	{
1323		− if (!req->fixed_rsrc_refs) {
1324		− req->fixed_rsrc_refs = &ctx->rsrc_node->refs;
	1483	+ if (!req->rsrc_node) {
	1484	+ req->rsrc_node = ctx->rsrc_node;
1325	1485
1326	1486	if (!(issue_flags & IO_URING_F_UNLOCKED)) {
1327	1487	lockdep_assert_held(&ctx->uring_lock);
@@ -1329,28 +1489,30 @@ static inline void io_req_set_rsrc_node(struct io_kiocb *req, · 1 unmodified line
1329	1489	if (unlikely(ctx->rsrc_cached_refs < 0))
1330	1490	io_rsrc_refs_refill(ctx);
1331	1491	} else {
1332		− percpu_ref_get(req->fixed_rsrc_refs);
	1492	+ percpu_ref_get(&req->rsrc_node->refs);
1333	1493	}
1334	1494	}
1335	1495	}
1336	1496
1337	1497	static unsigned int __io_put_kbuf(struct io_kiocb req, struct list_head list)
1338	1498	{
1339		− struct io_buffer *kbuf = req->kbuf;
1340		− unsigned int cflags;
	1499	+ if (req->flags & REQ_F_BUFFER_RING) {
	1500	+ if (req->buf_list)
	1501	+ req->buf_list->head++;
	1502	+ req->flags &= ~REQ_F_BUFFER_RING;
	1503	+ } else {
	1504	+ list_add(&req->kbuf->list, list);
	1505	+ req->flags &= ~REQ_F_BUFFER_SELECTED;
	1506	+ }
1341	1507
1342		− cflags = IORING_CQE_F_BUFFER \| (kbuf->bid << IORING_CQE_BUFFER_SHIFT);
1343		− req->flags &= ~REQ_F_BUFFER_SELECTED;
1344		− list_add(&kbuf->list, list);
1345		− req->kbuf = NULL;
1346		− return cflags;
	1508	+ return IORING_CQE_F_BUFFER \| (req->buf_index << IORING_CQE_BUFFER_SHIFT);
1347	1509	}
1348	1510
1349	1511	static inline unsigned int io_put_kbuf_comp(struct io_kiocb *req)
1350	1512	{
1351	1513	lockdep_assert_held(&req->ctx->completion_lock);
1352	1514
1353		− if (likely(!(req->flags & REQ_F_BUFFER_SELECTED)))
	1515	+ if (!(req->flags & (REQ_F_BUFFER_SELECTED\|REQ_F_BUFFER_RING)))
1354	1516	return 0;
1355	1517	return __io_put_kbuf(req, &req->ctx->io_buffers_comp);
1356	1518	}
@@ -1360,7 +1522,7 @@ static inline unsigned int io_put_kbuf(struct io_kiocb *req, · 3 unmodified lines
1360	1522	{
1361	1523	unsigned int cflags;
1362	1524
1363		− if (likely(!(req->flags & REQ_F_BUFFER_SELECTED)))
	1525	+ if (!(req->flags & (REQ_F_BUFFER_SELECTED\|REQ_F_BUFFER_RING)))
1364	1526	return 0;
1365	1527
1366	1528	/*
@@ -1375,7 +1537,10 @@ static inline unsigned int io_put_kbuf(struct io_kiocb *req, · 8 unmodified lines
1375	1537	* We migrate buffers from the comp_list to the issue cache list
1376	1538	* when we need one.
1377	1539	*/
1378		− if (issue_flags & IO_URING_F_UNLOCKED) {
	1540	+ if (req->flags & REQ_F_BUFFER_RING) {
	1541	+ /* no buffers to recycle for this case */
	1542	+ cflags = __io_put_kbuf(req, NULL);
	1543	+ } else if (issue_flags & IO_URING_F_UNLOCKED) {
1379	1544	struct io_ring_ctx *ctx = req->ctx;
1380	1545
1381	1546	spin_lock(&ctx->completion_lock);
@@ -1393,15 +1558,10 @@ static inline unsigned int io_put_kbuf(struct io_kiocb *req, · 11 unmodified lines
1393	1558	static struct io_buffer_list io_buffer_get_list(struct io_ring_ctx ctx,
1394	1559	unsigned int bgid)
1395	1560	{
1396		− struct list_head *hash_list;
1397		− struct io_buffer_list *bl;
1398		−
1399		− hash_list = &ctx->io_buffers[hash_32(bgid, IO_BUFFERS_HASH_BITS)];
1400		− list_for_each_entry(bl, hash_list, list)
1401		− if (bl->bgid == bgid \|\| bgid == -1U)
1402		− return bl;
	1561	+ if (ctx->io_bl && bgid < BGID_ARRAY)
	1562	+ return &ctx->io_bl[bgid];
1403	1563
1404		− return NULL;
	1564	+ return xa_load(&ctx->io_bl_xa, bgid);
1405	1565	}
1406	1566
1407	1567	static void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)
@@ -1410,25 +1570,33 @@ static void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) · 2 unmodified lines
1410	1570	struct io_buffer_list *bl;
1411	1571	struct io_buffer *buf;
1412	1572
1413		− if (likely(!(req->flags & REQ_F_BUFFER_SELECTED)))
	1573	+ if (!(req->flags & (REQ_F_BUFFER_SELECTED\|REQ_F_BUFFER_RING)))
1414	1574	return;
1415	1575	/* don't recycle if we already did IO to this buffer */
1416	1576	if (req->flags & REQ_F_PARTIAL_IO)
1417	1577	return;
	1578	+ /*
	1579	+ * We don't need to recycle for REQ_F_BUFFER_RING, we can just clear
	1580	+ * the flag and hence ensure that bl->head doesn't get incremented.
	1581	+ * If the tail has already been incremented, hang on to it.
	1582	+ */
	1583	+ if (req->flags & REQ_F_BUFFER_RING) {
	1584	+ if (req->buf_list) {
	1585	+ req->buf_index = req->buf_list->bgid;
	1586	+ req->flags &= ~REQ_F_BUFFER_RING;
	1587	+ }
	1588	+ return;
	1589	+ }
1418	1590
1419		− if (issue_flags & IO_URING_F_UNLOCKED)
1420		− mutex_lock(&ctx->uring_lock);
1421		−
1422		− lockdep_assert_held(&ctx->uring_lock);
	1591	+ io_ring_submit_lock(ctx, issue_flags);
1423	1592
1424	1593	buf = req->kbuf;
1425	1594	bl = io_buffer_get_list(ctx, buf->bgid);
1426	1595	list_add(&buf->list, &bl->buf_list);
1427	1596	req->flags &= ~REQ_F_BUFFER_SELECTED;
1428		− req->kbuf = NULL;
	1597	+ req->buf_index = buf->bgid;
1429	1598
1430		− if (issue_flags & IO_URING_F_UNLOCKED)
1431		− mutex_unlock(&ctx->uring_lock);
	1599	+ io_ring_submit_unlock(ctx, issue_flags);
1432	1600	}
1433	1601
1434	1602	static bool io_match_task(struct io_kiocb head, struct task_struct task,
@@ -1469,7 +1637,12 @@ static inline void req_set_fail(struct io_kiocb *req) · 34 unmodified lines
1469	1637	static inline void req_fail_link_node(struct io_kiocb *req, int res)
1470	1638	{
1471	1639	req_set_fail(req);
1472		− req->result = res;
	1640	+ req->cqe.res = res;
	1641	+}
	1642	+
	1643	+static inline void io_req_add_to_cache(struct io_kiocb req, struct io_ring_ctx ctx)
	1644	+{
	1645	+ wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list);
1473	1646	}
1474	1647
1475	1648	static __cold void io_ring_ctx_ref_free(struct percpu_ref *ref)
@@ -1506,12 +1679,14 @@ static __cold void io_fallback_req_func(struct work_struct *work) · 30 unmodified lines
1506	1679	static __cold struct io_ring_ctx io_ring_ctx_alloc(struct io_uring_params p)
1507	1680	{
1508	1681	struct io_ring_ctx *ctx;
1509		− int i, hash_bits;
	1682	+ int hash_bits;
1510	1683
1511	1684	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1512	1685	if (!ctx)
1513	1686	return NULL;
1514	1687
	1688	+ xa_init(&ctx->io_bl_xa);
	1689	+
1515	1690	/*
1516	1691	* Use 5 bits less than the max cq entries, that should give us around
1517	1692	* 32 entries per hash list if totally full and uniformly spread.
@@ -1533,13 +1708,6 @@ static __cold struct io_ring_ctx io_ring_ctx_alloc(struct io_uring_params p) · 15 unmodified lines
1533	1708	/* set invalid range, so io_import_fixed() fails meeting it */
1534	1709	ctx->dummy_ubuf->ubuf = -1UL;
1535	1710
1536		− ctx->io_buffers = kcalloc(1U << IO_BUFFERS_HASH_BITS,
1537		− sizeof(struct list_head), GFP_KERNEL);
1538		− if (!ctx->io_buffers)
1539		− goto err;
1540		− for (i = 0; i < (1U << IO_BUFFERS_HASH_BITS); i++)
1541		− INIT_LIST_HEAD(&ctx->io_buffers[i]);
1542		−
1543	1711	if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
1544	1712	PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
1545	1713	goto err;
@@ -1575,7 +1743,8 @@ static __cold struct io_ring_ctx io_ring_ctx_alloc(struct io_uring_params p) · 29 unmodified lines
1575	1743	err:
1576	1744	kfree(ctx->dummy_ubuf);
1577	1745	kfree(ctx->cancel_hash);
1578		− kfree(ctx->io_buffers);
	1746	+ kfree(ctx->io_bl);
	1747	+ xa_destroy(&ctx->io_bl_xa);
1579	1748	kfree(ctx);
1580	1749	return NULL;
1581	1750	}
@@ -1599,10 +1768,6 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq) · 17 unmodified lines
1599	1768	return false;
1600	1769	}
1601	1770
1602		−#define FFS_NOWAIT 0x1UL
1603		−#define FFS_ISREG 0x2UL
1604		−#define FFS_MASK ~(FFS_NOWAIT\|FFS_ISREG)
1605		−
1606	1771	static inline bool io_req_ffs_set(struct io_kiocb *req)
1607	1772	{
1608	1773	return req->flags & REQ_F_FIXED_FILE;
@@ -1629,6 +1794,17 @@ static inline struct io_kiocb io_prep_linked_timeout(struct io_kiocb req) · 20 unmodified lines
1629	1794	return __io_prep_linked_timeout(req);
1630	1795	}
1631	1796
	1797	+static noinline void __io_arm_ltimeout(struct io_kiocb *req)
	1798	+{
	1799	+ io_queue_linked_timeout(__io_prep_linked_timeout(req));
	1800	+}
	1801	+
	1802	+static inline void io_arm_ltimeout(struct io_kiocb *req)
	1803	+{
	1804	+ if (unlikely(req->flags & REQ_F_ARM_LTIMEOUT))
	1805	+ __io_arm_ltimeout(req);
	1806	+}
	1807	+
1632	1808	static void io_prep_async_work(struct io_kiocb *req)
1633	1809	{
1634	1810	const struct io_op_def *def = &io_op_defs[req->opcode];
@@ -1641,6 +1817,7 @@ static void io_prep_async_work(struct io_kiocb *req) · 6 unmodified lines
1641	1817
1642	1818	req->work.list.next = NULL;
1643	1819	req->work.flags = 0;
	1820	+ req->work.cancel_seq = atomic_read(&ctx->cancel_seq);
1644	1821	if (req->flags & REQ_F_FORCE_ASYNC)
1645	1822	req->work.flags \|= IO_WQ_WORK_CONCURRENT;
1646	1823
@@ -1672,17 +1849,15 @@ static void io_prep_async_link(struct io_kiocb *req) · 25 unmodified lines
1672	1849
1673	1850	static inline void io_req_add_compl_list(struct io_kiocb *req)
1674	1851	{
1675		− struct io_ring_ctx *ctx = req->ctx;
1676		− struct io_submit_state *state = &ctx->submit_state;
	1852	+ struct io_submit_state *state = &req->ctx->submit_state;
1677	1853
1678	1854	if (!(req->flags & REQ_F_CQE_SKIP))
1679		− ctx->submit_state.flush_cqes = true;
	1855	+ state->flush_cqes = true;
1680	1856	wq_list_add_tail(&req->comp_list, &state->compl_reqs);
1681	1857	}
1682	1858
1683		−static void io_queue_async_work(struct io_kiocb req, bool dont_use)
	1859	+static void io_queue_iowq(struct io_kiocb req, bool dont_use)
1684	1860	{
1685		− struct io_ring_ctx *ctx = req->ctx;
1686	1861	struct io_kiocb *link = io_prep_linked_timeout(req);
1687	1862	struct io_uring_task *tctx = req->task->io_uring;
1688	1863
@@ -1702,8 +1877,9 @@ static void io_queue_async_work(struct io_kiocb req, bool dont_use) · 13 unmodified lines
1702	1877	if (WARN_ON_ONCE(!same_thread_group(req->task, current)))
1703	1878	req->work.flags \|= IO_WQ_WORK_CANCEL;
1704	1879
1705		− trace_io_uring_queue_async_work(ctx, req, req->user_data, req->opcode, req->flags,
1706		− &req->work, io_wq_is_hashed(&req->work));
	1880	+ trace_io_uring_queue_async_work(req->ctx, req, req->cqe.user_data,
	1881	+ req->opcode, req->flags, &req->work,
	1882	+ io_wq_is_hashed(&req->work));
1707	1883	io_wq_enqueue(tctx->io_wq, &req->work);
1708	1884	if (link)
1709	1885	io_queue_linked_timeout(link);
@@ -1721,8 +1897,7 @@ static void io_kill_timeout(struct io_kiocb *req, int status) · 11 unmodified lines
1721	1897	atomic_set(&req->ctx->cq_timeouts,
1722	1898	atomic_read(&req->ctx->cq_timeouts) + 1);
1723	1899	list_del_init(&req->timeout.list);
1724		− io_fill_cqe_req(req, status, 0);
1725		− io_put_req_deferred(req);
	1900	+ io_req_tw_post_queue(req, status, 0);
1726	1901	}
1727	1902	}
1728	1903
@@ -1804,21 +1979,38 @@ static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) · 75 unmodified lines
1804	1979	return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
1805	1980	}
1806	1981
1807		−static inline struct io_uring_cqe io_get_cqe(struct io_ring_ctx ctx)
	1982	+/*
	1983	+ * writes to the cq entry need to come after reading head; the
	1984	+ * control dependency is enough as we're using WRITE_ONCE to
	1985	+ * fill the cq entry
	1986	+ */
	1987	+static noinline struct io_uring_cqe __io_get_cqe(struct io_ring_ctx ctx)
1808	1988	{
1809	1989	struct io_rings *rings = ctx->rings;
1810		− unsigned tail, mask = ctx->cq_entries - 1;
1811		−
1812		− /*
1813		− * writes to the cq entry need to come after reading head; the
1814		− * control dependency is enough as we're using WRITE_ONCE to
1815		− * fill the cq entry
1816		− */
1817		− if (__io_cqring_events(ctx) == ctx->cq_entries)
	1990	+ unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1);
	1991	+ unsigned int free, queued, len;
	1992	+
	1993	+ /* userspace may cheat modifying the tail, be safe and do min */
	1994	+ queued = min(__io_cqring_events(ctx), ctx->cq_entries);
	1995	+ free = ctx->cq_entries - queued;
	1996	+ /* we need a contiguous range, limit based on the current array offset */
	1997	+ len = min(free, ctx->cq_entries - off);
	1998	+ if (!len)
1818	1999	return NULL;
1819	2000
1820		− tail = ctx->cached_cq_tail++;
1821		− return &rings->cqes[tail & mask];
	2001	+ ctx->cached_cq_tail++;
	2002	+ ctx->cqe_cached = &rings->cqes[off];
	2003	+ ctx->cqe_sentinel = ctx->cqe_cached + len;
	2004	+ return ctx->cqe_cached++;
	2005	+}
	2006	+
	2007	+static inline struct io_uring_cqe io_get_cqe(struct io_ring_ctx ctx)
	2008	+{
	2009	+ if (likely(ctx->cqe_cached < ctx->cqe_sentinel)) {
	2010	+ ctx->cached_cq_tail++;
	2011	+ return ctx->cqe_cached++;
	2012	+ }
	2013	+ return __io_get_cqe(ctx);
1822	2014	}
1823	2015
1824	2016	static void io_eventfd_signal(struct io_ring_ctx *ctx)
@@ -1915,13 +2107,11 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) · 90 unmodified lines
1915	2107
1916	2108	all_flushed = list_empty(&ctx->cq_overflow_list);
1917	2109	if (all_flushed) {
1918		− clear_bit(0, &ctx->check_cq_overflow);
1919		− WRITE_ONCE(ctx->rings->sq_flags,
1920		− ctx->rings->sq_flags & ~IORING_SQ_CQ_OVERFLOW);
	2110	+ clear_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq);
	2111	+ atomic_andnot(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
1921	2112	}
1922	2113
1923		− if (posted)
1924		− io_commit_cqring(ctx);
	2114	+ io_commit_cqring(ctx);
1925	2115	spin_unlock(&ctx->completion_lock);
1926	2116	if (posted)
1927	2117	io_cqring_ev_posted(ctx);
@@ -1932,7 +2122,7 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx) · 4 unmodified lines
1932	2122	{
1933	2123	bool ret = true;
1934	2124
1935		− if (test_bit(0, &ctx->check_cq_overflow)) {
	2125	+ if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq)) {
1936	2126	/* iopoll syncs against uring_lock, not completion_lock */
1937	2127	if (ctx->flags & IORING_SETUP_IOPOLL)
1938	2128	mutex_lock(&ctx->uring_lock);
@@ -1944,19 +2134,23 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx) · 5 unmodified lines
1944	2134	return ret;
1945	2135	}
1946	2136
1947		−/* must to be called somewhat shortly after putting a request */
1948		−static inline void io_put_task(struct task_struct *task, int nr)
	2137	+static void __io_put_task(struct task_struct *task, int nr)
1949	2138	{
1950	2139	struct io_uring_task *tctx = task->io_uring;
1951	2140
1952		− if (likely(task == current)) {
1953		− tctx->cached_refs += nr;
1954		− } else {
1955		− percpu_counter_sub(&tctx->inflight, nr);
1956		− if (unlikely(atomic_read(&tctx->in_idle)))
1957		− wake_up(&tctx->wait);
1958		− put_task_struct_many(task, nr);
1959		− }
	2141	+ percpu_counter_sub(&tctx->inflight, nr);
	2142	+ if (unlikely(atomic_read(&tctx->in_idle)))
	2143	+ wake_up(&tctx->wait);
	2144	+ put_task_struct_many(task, nr);
	2145	+}
	2146	+
	2147	+/* must to be called somewhat shortly after putting a request */
	2148	+static inline void io_put_task(struct task_struct *task, int nr)
	2149	+{
	2150	+ if (likely(task == current))
	2151	+ task->io_uring->cached_refs += nr;
	2152	+ else
	2153	+ __io_put_task(task, nr);
1960	2154	}
1961	2155
1962	2156	static void io_task_refs_refill(struct io_uring_task *tctx)
@@ -1995,6 +2189,7 @@ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, · 32 unmodified lines
1995	2189	struct io_overflow_cqe *ocqe;
1996	2190
1997	2191	ocqe = kmalloc(sizeof(*ocqe), GFP_ATOMIC \| __GFP_ACCOUNT);
	2192	+ trace_io_uring_cqe_overflow(ctx, user_data, res, cflags, ocqe);
1998	2193	if (!ocqe) {
1999	2194	/*
2000	2195	* If we're in ring overflow flush mode, or in task cancel mode,
@@ -2002,12 +2197,12 @@ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, · 1 unmodified line
2002	2197	* on the floor.
2003	2198	*/
2004	2199	io_account_cq_overflow(ctx);
	2200	+ set_bit(IO_CHECK_CQ_DROPPED_BIT, &ctx->check_cq);
2005	2201	return false;
2006	2202	}
2007	2203	if (list_empty(&ctx->cq_overflow_list)) {
2008		− set_bit(0, &ctx->check_cq_overflow);
2009		− WRITE_ONCE(ctx->rings->sq_flags,
2010		− ctx->rings->sq_flags \| IORING_SQ_CQ_OVERFLOW);
	2204	+ set_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq);
	2205	+ atomic_or(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
2011	2206
2012	2207	}
2013	2208	ocqe->cqe.user_data = user_data;
@@ -2037,16 +2232,32 @@ static inline bool __io_fill_cqe(struct io_ring_ctx *ctx, u64 user_data, · 23 unmodified lines
2037	2232	return io_cqring_event_overflow(ctx, user_data, res, cflags);
2038	2233	}
2039	2234
2040		−static inline bool __io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)
	2235	+static inline bool __io_fill_cqe_req_filled(struct io_ring_ctx *ctx,
	2236	+ struct io_kiocb *req)
2041	2237	{
2042		− trace_io_uring_complete(req->ctx, req, req->user_data, res, cflags);
2043		− return __io_fill_cqe(req->ctx, req->user_data, res, cflags);
	2238	+ struct io_uring_cqe *cqe;
	2239	+
	2240	+ trace_io_uring_complete(req->ctx, req, req->cqe.user_data,
	2241	+ req->cqe.res, req->cqe.flags);
	2242	+
	2243	+ /*
	2244	+ * If we can't get a cq entry, userspace overflowed the
	2245	+ * submission (by quite a lot). Increment the overflow count in
	2246	+ * the ring.
	2247	+ */
	2248	+ cqe = io_get_cqe(ctx);
	2249	+ if (likely(cqe)) {
	2250	+ memcpy(cqe, &req->cqe, sizeof(*cqe));
	2251	+ return true;
	2252	+ }
	2253	+ return io_cqring_event_overflow(ctx, req->cqe.user_data,
	2254	+ req->cqe.res, req->cqe.flags);
2044	2255	}
2045	2256
2046		−static noinline void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)
	2257	+static inline bool __io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)
2047	2258	{
2048		− if (!(req->flags & REQ_F_CQE_SKIP))
2049		− __io_fill_cqe_req(req, res, cflags);
	2259	+ trace_io_uring_complete(req->ctx, req, req->cqe.user_data, res, cflags);
	2260	+ return __io_fill_cqe(req->ctx, req->cqe.user_data, res, cflags);
2050	2261	}
2051	2262
2052	2263	static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data,
@@ -2069,7 +2280,7 @@ static void __io_req_complete_post(struct io_kiocb *req, s32 res, · 16 unmodified lines
2069	2280	* free_list cache.
2070	2281	*/
2071	2282	if (req_ref_put_and_test(req)) {
2072		− if (req->flags & (REQ_F_LINK \| REQ_F_HARDLINK)) {
	2283	+ if (req->flags & IO_REQ_LINK_FLAGS) {
2073	2284	if (req->flags & IO_DISARM_MASK)
2074	2285	io_disarm_next(req);
2075	2286	if (req->link) {
@@ -2077,7 +2288,7 @@ static void __io_req_complete_post(struct io_kiocb *req, s32 res, · 1 unmodified line
2077	2288	req->link = NULL;
2078	2289	}
2079	2290	}
2080		− io_req_put_rsrc(req, ctx);
	2291	+ io_req_put_rsrc(req);
2081	2292	/*
2082	2293	* Selected buffer deallocation in io_clean_op() assumes that
2083	2294	* we don't hold ->completion_lock. Clean them here to avoid
@@ -2106,8 +2317,8 @@ static void io_req_complete_post(struct io_kiocb *req, s32 res, · 22 unmodified lines
2106	2317	static inline void io_req_complete_state(struct io_kiocb *req, s32 res,
2107	2318	u32 cflags)
2108	2319	{
2109		− req->result = res;
2110		− req->cflags = cflags;
	2320	+ req->cqe.res = res;
	2321	+ req->cqe.flags = cflags;
2111	2322	req->flags \|= REQ_F_COMPLETE_INLINE;
2112	2323	}
2113	2324
@@ -2131,17 +2342,6 @@ static void io_req_complete_failed(struct io_kiocb *req, s32 res) · 17 unmodified lines
2131	2342	io_req_complete_post(req, res, io_put_kbuf(req, IO_URING_F_UNLOCKED));
2132	2343	}
2133	2344
2134		−static void io_req_complete_fail_submit(struct io_kiocb *req)
2135		−{
2136		− /*
2137		− * We don't submit, fail them all, for that replace hardlinks with
2138		− * normal links. Extra REQ_F_LINK is tolerated.
2139		− */
2140		− req->flags &= ~REQ_F_HARDLINK;
2141		− req->flags \|= REQ_F_LINK;
2142		− io_req_complete_failed(req, req->result);
2143		−}
2144		−
2145	2345	/*
2146	2346	* Don't initialise the fields below on every allocation, but do that in
2147	2347	* advance and keep them valid across allocations.
@@ -2152,7 +2352,7 @@ static void io_preinit_req(struct io_kiocb req, struct io_ring_ctx ctx) · 4 unmodified lines
2152	2352	req->link = NULL;
2153	2353	req->async_data = NULL;
2154	2354	/* not necessary, but safer to zero */
2155		− req->result = 0;
	2355	+ req->cqe.res = 0;
2156	2356	}
2157	2357
2158	2358	static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,
@@ -2164,19 +2364,9 @@ static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx, · 5 unmodified lines
2164	2364	spin_unlock(&ctx->completion_lock);
2165	2365	}
2166	2366
2167		−/* Returns true IFF there are requests in the cache */
2168		−static bool io_flush_cached_reqs(struct io_ring_ctx *ctx)
	2367	+static inline bool io_req_cache_empty(struct io_ring_ctx *ctx)
2169	2368	{
2170		− struct io_submit_state *state = &ctx->submit_state;
2171		−
2172		− /*
2173		− * If we have more than a batch's worth of requests in our IRQ side
2174		− * locked cache, grab the lock and move them over to our submission
2175		− * side cache.
2176		− */
2177		− if (READ_ONCE(ctx->locked_free_nr) > IO_COMPL_BATCH)
2178		− io_flush_cached_locked_reqs(ctx, state);
2179		− return !!state->free_list.next;
	2369	+ return !ctx->submit_state.free_list.next;
2180	2370	}
2181	2371
2182	2372	/*
@@ -2188,14 +2378,20 @@ static bool io_flush_cached_reqs(struct io_ring_ctx *ctx) · 5 unmodified lines
2188	2378	static __cold bool __io_alloc_req_refill(struct io_ring_ctx *ctx)
2189	2379	__must_hold(&ctx->uring_lock)
2190	2380	{
2191		− struct io_submit_state *state = &ctx->submit_state;
2192	2381	gfp_t gfp = GFP_KERNEL \| __GFP_NOWARN;
2193	2382	void *reqs[IO_REQ_ALLOC_BATCH];
2194		− struct io_kiocb *req;
2195	2383	int ret, i;
2196	2384
2197		− if (likely(state->free_list.next \|\| io_flush_cached_reqs(ctx)))
2198		− return true;
	2385	+ /*
	2386	+ * If we have more than a batch's worth of requests in our IRQ side
	2387	+ * locked cache, grab the lock and move them over to our submission
	2388	+ * side cache.
	2389	+ */
	2390	+ if (data_race(ctx->locked_free_nr) > IO_COMPL_BATCH) {
	2391	+ io_flush_cached_locked_reqs(ctx, &ctx->submit_state);
	2392	+ if (!io_req_cache_empty(ctx))
	2393	+ return true;
	2394	+ }
2199	2395
2200	2396	ret = kmem_cache_alloc_bulk(req_cachep, gfp, ARRAY_SIZE(reqs), reqs);
2201	2397
@@ -2212,17 +2408,17 @@ static __cold bool __io_alloc_req_refill(struct io_ring_ctx *ctx) · 10 unmodified lines
2212	2408
2213	2409	percpu_ref_get_many(&ctx->refs, ret);
2214	2410	for (i = 0; i < ret; i++) {
2215		− req = reqs[i];
	2411	+ struct io_kiocb *req = reqs[i];
2216	2412
2217	2413	io_preinit_req(req, ctx);
2218		− wq_stack_add_head(&req->comp_list, &state->free_list);
	2414	+ io_req_add_to_cache(req, ctx);
2219	2415	}
2220	2416	return true;
2221	2417	}
2222	2418
2223	2419	static inline bool io_alloc_req_refill(struct io_ring_ctx *ctx)
2224	2420	{
2225		− if (unlikely(!ctx->submit_state.free_list.next))
	2421	+ if (unlikely(io_req_cache_empty(ctx)))
2226	2422	return __io_alloc_req_refill(ctx);
2227	2423	return true;
2228	2424	}
@@ -2251,11 +2447,11 @@ static inline void io_dismantle_req(struct io_kiocb *req) · 22 unmodified lines
2251	2447	io_put_file(req->file);
2252	2448	}
2253	2449
2254		−static __cold void __io_free_req(struct io_kiocb *req)
	2450	+static __cold void io_free_req(struct io_kiocb *req)
2255	2451	{
2256	2452	struct io_ring_ctx *ctx = req->ctx;
2257	2453
2258		− io_req_put_rsrc(req, ctx);
	2454	+ io_req_put_rsrc(req);
2259	2455	io_dismantle_req(req);
2260	2456	io_put_task(req->task, 1);
2261	2457
@@ -2273,7 +2469,7 @@ static inline void io_remove_next_linked(struct io_kiocb *req) · 11 unmodified lines
2273	2469	nxt->link = NULL;
2274	2470	}
2275	2471
2276		−static bool io_kill_linked_timeout(struct io_kiocb *req)
	2472	+static struct io_kiocb io_disarm_linked_timeout(struct io_kiocb req)
2277	2473	__must_hold(&req->ctx->completion_lock)
2278	2474	__must_hold(&req->ctx->timeout_lock)
2279	2475	{
@@ -2286,13 +2482,10 @@ static bool io_kill_linked_timeout(struct io_kiocb *req) · 6 unmodified lines
2286	2482	link->timeout.head = NULL;
2287	2483	if (hrtimer_try_to_cancel(&io->timer) != -1) {
2288	2484	list_del(&link->timeout.list);
2289		− /* leave REQ_F_CQE_SKIP to io_fill_cqe_req */
2290		− io_fill_cqe_req(link, -ECANCELED, 0);
2291		− io_put_req_deferred(link);
2292		− return true;
	2485	+ return link;
2293	2486	}
2294	2487	}
2295		− return false;
	2488	+ return NULL;
2296	2489	}
2297	2490
2298	2491	static void io_fail_links(struct io_kiocb *req)
@@ -2306,19 +2499,19 @@ static void io_fail_links(struct io_kiocb *req) · 7 unmodified lines
2306	2499	long res = -ECANCELED;
2307	2500
2308	2501	if (link->flags & REQ_F_FAIL)
2309		− res = link->result;
	2502	+ res = link->cqe.res;
2310	2503
2311	2504	nxt = link->link;
2312	2505	link->link = NULL;
2313	2506
2314		− trace_io_uring_fail_link(req->ctx, req, req->user_data,
	2507	+ trace_io_uring_fail_link(req->ctx, req, req->cqe.user_data,
2315	2508	req->opcode, link);
2316	2509
2317		− if (!ignore_cqes) {
	2510	+ if (ignore_cqes)
	2511	+ link->flags \|= REQ_F_CQE_SKIP;
	2512	+ else
2318	2513	link->flags &= ~REQ_F_CQE_SKIP;
2319		− io_fill_cqe_req(link, res, 0);
2320		− }
2321		− io_put_req_deferred(link);
	2514	+ __io_req_complete_post(link, res, 0);
2322	2515	link = nxt;
2323	2516	}
2324	2517	}
@@ -2326,25 +2519,27 @@ static void io_fail_links(struct io_kiocb *req) · 1 unmodified line
2326	2519	static bool io_disarm_next(struct io_kiocb *req)
2327	2520	__must_hold(&req->ctx->completion_lock)
2328	2521	{
	2522	+ struct io_kiocb *link = NULL;
2329	2523	bool posted = false;
2330	2524
2331	2525	if (req->flags & REQ_F_ARM_LTIMEOUT) {
2332		− struct io_kiocb *link = req->link;
2333		−
	2526	+ link = req->link;
2334	2527	req->flags &= ~REQ_F_ARM_LTIMEOUT;
2335	2528	if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
2336	2529	io_remove_next_linked(req);
2337		− /* leave REQ_F_CQE_SKIP to io_fill_cqe_req */
2338		− io_fill_cqe_req(link, -ECANCELED, 0);
2339		− io_put_req_deferred(link);
	2530	+ io_req_tw_post_queue(link, -ECANCELED, 0);
2340	2531	posted = true;
2341	2532	}
2342	2533	} else if (req->flags & REQ_F_LINK_TIMEOUT) {
2343	2534	struct io_ring_ctx *ctx = req->ctx;
2344	2535
2345	2536	spin_lock_irq(&ctx->timeout_lock);
2346		− posted = io_kill_linked_timeout(req);
	2537	+ link = io_disarm_linked_timeout(req);
2347	2538	spin_unlock_irq(&ctx->timeout_lock);
	2539	+ if (link) {
	2540	+ posted = true;
	2541	+ io_req_tw_post_queue(link, -ECANCELED, 0);
	2542	+ }
2348	2543	}
2349	2544	if (unlikely((req->flags & REQ_F_FAIL) &&
2350	2545	!(req->flags & REQ_F_HARDLINK))) {
@@ -2361,8 +2556,7 @@ static void __io_req_find_next_prep(struct io_kiocb *req) · 10 unmodified lines
2361	2556
2362	2557	spin_lock(&ctx->completion_lock);
2363	2558	posted = io_disarm_next(req);
2364		− if (posted)
2365		− io_commit_cqring(ctx);
	2559	+ io_commit_cqring(ctx);
2366	2560	spin_unlock(&ctx->completion_lock);
2367	2561	if (posted)
2368	2562	io_cqring_ev_posted(ctx);
@@ -2372,8 +2566,6 @@ static inline struct io_kiocb io_req_find_next(struct io_kiocb req) · 3 unmodified lines
2372	2566	{
2373	2567	struct io_kiocb *nxt;
2374	2568
2375		− if (likely(!(req->flags & (REQ_F_LINK\|REQ_F_HARDLINK))))
2376		− return NULL;
2377	2569	/*
2378	2570	* If LINK is set, we have dependent requests in this chain. If we
2379	2571	* didn't fail this request, queue the first one up, moving any other
@@ -2391,6 +2583,8 @@ static void ctx_flush_and_put(struct io_ring_ctx ctx, bool locked) · 11 unmodified lines
2391	2583	{
2392	2584	if (!ctx)
2393	2585	return;
	2586	+ if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
	2587	+ atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
2394	2588	if (*locked) {
2395	2589	io_submit_flush_completions(ctx);
2396	2590	mutex_unlock(&ctx->uring_lock);
@@ -2434,7 +2628,7 @@ static void handle_prev_tw_list(struct io_wq_work_node *node, · 37 unmodified lines
2434	2628	if (likely(*uring_locked))
2435	2629	req->io_task_work.func(req, uring_locked);
2436	2630	else
2437		− __io_req_complete_post(req, req->result,
	2631	+ __io_req_complete_post(req, req->cqe.res,
2438	2632	io_put_kbuf_comp(req));
2439	2633	node = next;
2440	2634	} while (node);
@@ -2475,10 +2669,6 @@ static void tctx_task_work(struct callback_head *cb) · 34 unmodified lines
2475	2669	while (1) {
2476	2670	struct io_wq_work_node node1, node2;
2477	2671
2478		− if (!tctx->task_list.first &&
2479		− !tctx->prior_task_list.first && uring_locked)
2480		− io_submit_flush_completions(ctx);
2481		−
2482	2672	spin_lock_irq(&tctx->task_lock);
2483	2673	node1 = tctx->prior_task_list.first;
2484	2674	node2 = tctx->task_list.first;
@@ -2492,10 +2682,13 @@ static void tctx_task_work(struct callback_head *cb) · 7 unmodified lines
2492	2682
2493	2683	if (node1)
2494	2684	handle_prev_tw_list(node1, &ctx, &uring_locked);
2495		−
2496	2685	if (node2)
2497	2686	handle_tw_list(node2, &ctx, &uring_locked);
2498	2687	cond_resched();
	2688	+
	2689	+ if (data_race(!tctx->task_list.first) &&
	2690	+ data_race(!tctx->prior_task_list.first) && uring_locked)
	2691	+ io_submit_flush_completions(ctx);
2499	2692	}
2500	2693
2501	2694	ctx_flush_and_put(ctx, &uring_locked);
@@ -2508,8 +2701,8 @@ static void tctx_task_work(struct callback_head *cb) · 6 unmodified lines
2508	2701	static void io_req_task_work_add(struct io_kiocb *req, bool priority)
2509	2702	{
2510	2703	struct task_struct *tsk = req->task;
	2704	+ struct io_ring_ctx *ctx = req->ctx;
2511	2705	struct io_uring_task *tctx = tsk->io_uring;
2512		− enum task_work_notify_mode notify;
2513	2706	struct io_wq_work_node *node;
2514	2707	unsigned long flags;
2515	2708	bool running;
@@ -2532,18 +2725,11 @@ static void io_req_task_work_add(struct io_kiocb *req, bool priority) · 16 unmodified lines
2532	2725	if (running)
2533	2726	return;
2534	2727
2535		− /*
2536		− * SQPOLL kernel thread doesn't need notification, just a wakeup. For
2537		− * all other cases, use TWA_SIGNAL unconditionally to ensure we're
2538		− * processing task_work. There's no reliable way to tell if TWA_RESUME
2539		− * will do the job.
2540		− */
2541		− notify = (req->ctx->flags & IORING_SETUP_SQPOLL) ? TWA_NONE : TWA_SIGNAL;
2542		− if (likely(!task_work_add(tsk, &tctx->task_work, notify))) {
2543		− if (notify == TWA_NONE)
2544		− wake_up_process(tsk);
	2728	+ if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
	2729	+ atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
	2730	+
	2731	+ if (likely(!task_work_add(tsk, &tctx->task_work, ctx->notify_method)))
2545	2732	return;
2546		− }
2547	2733
2548	2734	spin_lock_irqsave(&tctx->task_lock, flags);
2549	2735	tctx->task_running = false;
@@ -2559,30 +2745,39 @@ static void io_req_task_work_add(struct io_kiocb *req, bool priority) · 9 unmodified lines
2559	2745	}
2560	2746	}
2561	2747
2562		−static void io_req_task_cancel(struct io_kiocb req, bool locked)
	2748	+static void io_req_tw_post(struct io_kiocb req, bool locked)
2563	2749	{
2564		− struct io_ring_ctx *ctx = req->ctx;
	2750	+ io_req_complete_post(req, req->cqe.res, req->cqe.flags);
	2751	+}
2565	2752
	2753	+static void io_req_tw_post_queue(struct io_kiocb *req, s32 res, u32 cflags)
	2754	+{
	2755	+ req->cqe.res = res;
	2756	+ req->cqe.flags = cflags;
	2757	+ req->io_task_work.func = io_req_tw_post;
	2758	+ io_req_task_work_add(req, false);
	2759	+}
	2760	+
	2761	+static void io_req_task_cancel(struct io_kiocb req, bool locked)
	2762	+{
2566	2763	/* not needed for normal modes, but SQPOLL depends on it */
2567		− io_tw_lock(ctx, locked);
2568		− io_req_complete_failed(req, req->result);
	2764	+ io_tw_lock(req->ctx, locked);
	2765	+ io_req_complete_failed(req, req->cqe.res);
2569	2766	}
2570	2767
2571	2768	static void io_req_task_submit(struct io_kiocb req, bool locked)
2572	2769	{
2573		− struct io_ring_ctx *ctx = req->ctx;
2574		−
2575		− io_tw_lock(ctx, locked);
	2770	+ io_tw_lock(req->ctx, locked);
2576	2771	/* req->task == current here, checking PF_EXITING is safe */
2577	2772	if (likely(!(req->task->flags & PF_EXITING)))
2578		− __io_queue_sqe(req);
	2773	+ io_queue_sqe(req);
2579	2774	else
2580	2775	io_req_complete_failed(req, -EFAULT);
2581	2776	}
2582	2777
2583	2778	static void io_req_task_queue_fail(struct io_kiocb *req, int ret)
2584	2779	{
2585		− req->result = ret;
	2780	+ req->cqe.res = ret;
2586	2781	req->io_task_work.func = io_req_task_cancel;
2587	2782	io_req_task_work_add(req, false);
2588	2783	}
@@ -2595,11 +2790,11 @@ static void io_req_task_queue(struct io_kiocb *req) · 6 unmodified lines
2595	2790
2596	2791	static void io_req_task_queue_reissue(struct io_kiocb *req)
2597	2792	{
2598		− req->io_task_work.func = io_queue_async_work;
	2793	+ req->io_task_work.func = io_queue_iowq;
2599	2794	io_req_task_work_add(req, false);
2600	2795	}
2601	2796
2602		−static inline void io_queue_next(struct io_kiocb *req)
	2797	+static void io_queue_next(struct io_kiocb *req)
2603	2798	{
2604	2799	struct io_kiocb *nxt = io_req_find_next(req);
2605	2800
@@ -2607,17 +2802,6 @@ static inline void io_queue_next(struct io_kiocb *req) · 1 unmodified line
2607	2802	io_req_task_queue(nxt);
2608	2803	}
2609	2804
2610		−static void io_free_req(struct io_kiocb *req)
2611		−{
2612		− io_queue_next(req);
2613		− __io_free_req(req);
2614		−}
2615		−
2616		−static void io_free_req_work(struct io_kiocb req, bool locked)
2617		−{
2618		− io_free_req(req);
2619		−}
2620		−
2621	2805	static void io_free_batch_list(struct io_ring_ctx *ctx,
2622	2806	struct io_wq_work_node *node)
2623	2807	__must_hold(&ctx->uring_lock)
@@ -2629,15 +2813,30 @@ static void io_free_batch_list(struct io_ring_ctx *ctx, · 5 unmodified lines
2629	2813	struct io_kiocb *req = container_of(node, struct io_kiocb,
2630	2814	comp_list);
2631	2815
2632		− if (unlikely(req->flags & REQ_F_REFCOUNT)) {
2633		− node = req->comp_list.next;
2634		− if (!req_ref_put_and_test(req))
2635		− continue;
	2816	+ if (unlikely(req->flags & IO_REQ_CLEAN_SLOW_FLAGS)) {
	2817	+ if (req->flags & REQ_F_REFCOUNT) {
	2818	+ node = req->comp_list.next;
	2819	+ if (!req_ref_put_and_test(req))
	2820	+ continue;
	2821	+ }
	2822	+ if ((req->flags & REQ_F_POLLED) && req->apoll) {
	2823	+ struct async_poll *apoll = req->apoll;
	2824	+
	2825	+ if (apoll->double_poll)
	2826	+ kfree(apoll->double_poll);
	2827	+ list_add(&apoll->poll.wait.entry,
	2828	+ &ctx->apoll_cache);
	2829	+ req->flags &= ~REQ_F_POLLED;
	2830	+ }
	2831	+ if (req->flags & IO_REQ_LINK_FLAGS)
	2832	+ io_queue_next(req);
	2833	+ if (unlikely(req->flags & IO_REQ_CLEAN_FLAGS))
	2834	+ io_clean_op(req);
2636	2835	}
	2836	+ if (!(req->flags & REQ_F_FIXED_FILE))
	2837	+ io_put_file(req->file);
2637	2838
2638	2839	io_req_put_rsrc_locked(req, ctx);
2639		− io_queue_next(req);
2640		− io_dismantle_req(req);
2641	2840
2642	2841	if (req->task != task) {
2643	2842	if (task)
@@ -2647,7 +2846,7 @@ static void io_free_batch_list(struct io_ring_ctx *ctx, · 3 unmodified lines
2647	2846	}
2648	2847	task_refs++;
2649	2848	node = req->comp_list.next;
2650		− wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list);
	2849	+ io_req_add_to_cache(req, ctx);
2651	2850	} while (node);
2652	2851
2653	2852	if (task)
@@ -2667,16 +2866,7 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx) · 13 unmodified lines
2667	2866	comp_list);
2668	2867
2669	2868	if (!(req->flags & REQ_F_CQE_SKIP))
2670		− __io_fill_cqe_req(req, req->result, req->cflags);
2671		− if ((req->flags & REQ_F_POLLED) && req->apoll) {
2672		− struct async_poll *apoll = req->apoll;
2673		−
2674		− if (apoll->double_poll)
2675		− kfree(apoll->double_poll);
2676		− list_add(&apoll->poll.wait.entry,
2677		− &ctx->apoll_cache);
2678		− req->flags &= ~REQ_F_POLLED;
2679		− }
	2869	+ __io_fill_cqe_req_filled(ctx, req);
2680	2870	}
2681	2871
2682	2872	io_commit_cqring(ctx);
@@ -2698,23 +2888,18 @@ static inline struct io_kiocb io_put_req_find_next(struct io_kiocb req) · 15 unmodified lines
2698	2888	struct io_kiocb *nxt = NULL;
2699	2889
2700	2890	if (req_ref_put_and_test(req)) {
2701		− nxt = io_req_find_next(req);
2702		− __io_free_req(req);
	2891	+ if (unlikely(req->flags & IO_REQ_LINK_FLAGS))
	2892	+ nxt = io_req_find_next(req);
	2893	+ io_free_req(req);
2703	2894	}
2704	2895	return nxt;
2705	2896	}
2706	2897
2707	2898	static inline void io_put_req(struct io_kiocb *req)
2708	2899	{
2709		− if (req_ref_put_and_test(req))
	2900	+ if (req_ref_put_and_test(req)) {
	2901	+ io_queue_next(req);
2710	2902	io_free_req(req);
2711		−}
2712		−
2713		−static inline void io_put_req_deferred(struct io_kiocb *req)
2714		−{
2715		− if (req_ref_put_and_test(req)) {
2716		− req->io_task_work.func = io_free_req_work;
2717		− io_req_task_work_add(req, false);
2718	2903	}
2719	2904	}
2720	2905
@@ -2800,7 +2985,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) · 79 unmodified lines
2800	2985	nr_events++;
2801	2986	if (unlikely(req->flags & REQ_F_CQE_SKIP))
2802	2987	continue;
2803		− __io_fill_cqe_req(req, req->result, io_put_kbuf(req, 0));
	2988	+ __io_fill_cqe_req(req, req->cqe.res, io_put_kbuf(req, 0));
2804	2989	}
2805	2990
2806	2991	if (unlikely(!nr_events))
@@ -2846,22 +3031,26 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) · 39 unmodified lines
2846	3031	{
2847	3032	unsigned int nr_events = 0;
2848	3033	int ret = 0;
	3034	+ unsigned long check_cq;
2849	3035
2850		− /*
2851		− * We disallow the app entering submit/complete with polling, but we
2852		− * still need to lock the ring to prevent racing with polled issue
2853		− * that got punted to a workqueue.
2854		− */
2855		− mutex_lock(&ctx->uring_lock);
2856	3036	/*
2857	3037	* Don't enter poll loop if we already have events pending.
2858	3038	* If we do, we can potentially be spinning for commands that
2859	3039	* already triggered a CQE (eg in error).
2860	3040	*/
2861		− if (test_bit(0, &ctx->check_cq_overflow))
	3041	+ check_cq = READ_ONCE(ctx->check_cq);
	3042	+ if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
2862	3043	__io_cqring_overflow_flush(ctx, false);
2863	3044	if (io_cqring_events(ctx))
2864		− goto out;
	3045	+ return 0;
	3046	+
	3047	+ /*
	3048	+ * Similarly do not spin if we have not informed the user of any
	3049	+ * dropped CQE.
	3050	+ */
	3051	+ if (unlikely(check_cq & BIT(IO_CHECK_CQ_DROPPED_BIT)))
	3052	+ return -EBADR;
	3053	+
2865	3054	do {
2866	3055	/*
2867	3056	* If a submit got punted to a workqueue, we can have the
@@ -2891,8 +3080,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) · 23 unmodified lines
2891	3080	nr_events += ret;
2892	3081	ret = 0;
2893	3082	} while (nr_events < min && !need_resched());
2894		−out:
2895		− mutex_unlock(&ctx->uring_lock);
	3083	+
2896	3084	return ret;
2897	3085	}
2898	3086
@@ -2965,21 +3153,21 @@ static bool __io_complete_rw_common(struct io_kiocb *req, long res) · 66 unmodified lines
2965	3153	} else {
2966	3154	fsnotify_access(req->file);
2967	3155	}
2968		− if (unlikely(res != req->result)) {
	3156	+ if (unlikely(res != req->cqe.res)) {
2969	3157	if ((res == -EAGAIN \|\| res == -EOPNOTSUPP) &&
2970	3158	io_rw_should_reissue(req)) {
2971	3159	req->flags \|= REQ_F_REISSUE;
2972	3160	return true;
2973	3161	}
2974	3162	req_set_fail(req);
2975		− req->result = res;
	3163	+ req->cqe.res = res;
2976	3164	}
2977	3165	return false;
2978	3166	}
2979	3167
2980	3168	static inline void io_req_task_complete(struct io_kiocb req, bool locked)
2981	3169	{
2982		− int res = req->result;
	3170	+ int res = req->cqe.res;
2983	3171
2984	3172	if (*locked) {
2985	3173	io_req_complete_state(req, res, io_put_kbuf(req, 0));
@@ -2995,7 +3183,7 @@ static void __io_complete_rw(struct io_kiocb *req, long res, · 9 unmodified lines
2995	3183	{
2996	3184	if (__io_complete_rw_common(req, res))
2997	3185	return;
2998		− __io_req_complete(req, issue_flags, req->result,
	3186	+ __io_req_complete(req, issue_flags, req->cqe.res,
2999	3187	io_put_kbuf(req, issue_flags));
3000	3188	}
3001	3189
@@ -3005,7 +3193,7 @@ static void io_complete_rw(struct kiocb *kiocb, long res) · 3 unmodified lines
3005	3193
3006	3194	if (__io_complete_rw_common(req, res))
3007	3195	return;
3008		− req->result = res;
	3196	+ req->cqe.res = res;
3009	3197	req->io_task_work.func = io_req_task_complete;
3010	3198	io_req_task_work_add(req, !!(req->ctx->flags & IORING_SETUP_SQPOLL));
3011	3199	}
@@ -3016,12 +3204,12 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res) · 4 unmodified lines
3016	3204
3017	3205	if (kiocb->ki_flags & IOCB_WRITE)
3018	3206	kiocb_end_write(req);
3019		− if (unlikely(res != req->result)) {
	3207	+ if (unlikely(res != req->cqe.res)) {
3020	3208	if (res == -EAGAIN && io_rw_should_reissue(req)) {
3021	3209	req->flags \|= REQ_F_REISSUE;
3022	3210	return;
3023	3211	}
3024		− req->result = res;
	3212	+ req->cqe.res = res;
3025	3213	}
3026	3214
3027	3215	/* order with io_iopoll_complete() checking ->iopoll_completed */
@@ -3131,6 +3319,8 @@ static unsigned int io_file_get_flags(struct file *file) · 103 unmodified lines
3131	3319	res \|= FFS_ISREG;
3132	3320	if (__io_file_supports_nowait(file, mode))
3133	3321	res \|= FFS_NOWAIT;
	3322	+ if (io_file_need_scm(file))
	3323	+ res \|= FFS_SCM;
3134	3324	return res;
3135	3325	}
3136	3326
@@ -3162,6 +3352,7 @@ static int io_prep_rw(struct io_kiocb req, const struct io_uring_sqe sqe) · 25 unmodified lines
3162	3352	req->rw.addr = READ_ONCE(sqe->addr);
3163	3353	req->rw.len = READ_ONCE(sqe->len);
3164	3354	req->rw.flags = READ_ONCE(sqe->rw_flags);
	3355	+ /* used for fixed read/write too - just read unconditionally */
3165	3356	req->buf_index = READ_ONCE(sqe->buf_index);
3166	3357	return 0;
3167	3358	}
@@ -3310,77 +3501,96 @@ static int io_import_fixed(struct io_kiocb req, int rw, struct iov_iter iter, · 142 unmodified lines
3310	3501	return __io_import_fixed(req, rw, iter, imu);
3311	3502	}
3312	3503
3313		−static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
	3504	+static int io_buffer_add_list(struct io_ring_ctx *ctx,
	3505	+ struct io_buffer_list *bl, unsigned int bgid)
3314	3506	{
3315		− if (needs_lock)
3316		− mutex_unlock(&ctx->uring_lock);
3317		−}
3318		−
3319		−static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
3320		−{
3321		− /*
3322		− * "Normal" inline submissions always hold the uring_lock, since we
3323		− * grab it from the system call. Same is true for the SQPOLL offload.
3324		− * The only exception is when we've detached the request and issue it
3325		− * from an async worker thread, grab the lock for that case.
3326		− */
3327		− if (needs_lock)
3328		− mutex_lock(&ctx->uring_lock);
3329		−}
3330		−
3331		−static void io_buffer_add_list(struct io_ring_ctx *ctx,
3332		− struct io_buffer_list *bl, unsigned int bgid)
3333		−{
3334		− struct list_head *list;
3335		−
3336		− list = &ctx->io_buffers[hash_32(bgid, IO_BUFFERS_HASH_BITS)];
3337		− INIT_LIST_HEAD(&bl->buf_list);
3338	3507	bl->bgid = bgid;
3339		− list_add(&bl->list, list);
	3508	+ if (bgid < BGID_ARRAY)
	3509	+ return 0;
	3510	+
	3511	+ return xa_err(xa_store(&ctx->io_bl_xa, bgid, bl, GFP_KERNEL));
3340	3512	}
3341	3513
3342		−static struct io_buffer io_buffer_select(struct io_kiocb req, size_t *len,
3343		− int bgid, unsigned int issue_flags)
	3514	+static void __user io_provided_buffer_select(struct io_kiocb req, size_t *len,
	3515	+ struct io_buffer_list *bl)
3344	3516	{
3345		− struct io_buffer *kbuf = req->kbuf;
3346		− bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
3347		− struct io_ring_ctx *ctx = req->ctx;
3348		− struct io_buffer_list *bl;
3349		−
3350		− if (req->flags & REQ_F_BUFFER_SELECTED)
3351		− return kbuf;
	3517	+ if (!list_empty(&bl->buf_list)) {
	3518	+ struct io_buffer *kbuf;
3352	3519
3353		− io_ring_submit_lock(ctx, needs_lock);
3354		−
3355		− lockdep_assert_held(&ctx->uring_lock);
3356		−
3357		− bl = io_buffer_get_list(ctx, bgid);
3358		− if (bl && !list_empty(&bl->buf_list)) {
3359	3520	kbuf = list_first_entry(&bl->buf_list, struct io_buffer, list);
3360	3521	list_del(&kbuf->list);
3361	3522	if (*len > kbuf->len)
3362	3523	*len = kbuf->len;
3363	3524	req->flags \|= REQ_F_BUFFER_SELECTED;
3364	3525	req->kbuf = kbuf;
	3526	+ req->buf_index = kbuf->bid;
	3527	+ return u64_to_user_ptr(kbuf->addr);
	3528	+ }
	3529	+ return NULL;
	3530	+}
	3531	+
	3532	+static void __user io_ring_buffer_select(struct io_kiocb req, size_t *len,
	3533	+ struct io_buffer_list *bl,
	3534	+ unsigned int issue_flags)
	3535	+{
	3536	+ struct io_uring_buf_ring *br = bl->buf_ring;
	3537	+ struct io_uring_buf *buf;
	3538	+ __u32 head = bl->head;
	3539	+
	3540	+ if (unlikely(smp_load_acquire(&br->tail) == head)) {
	3541	+ io_ring_submit_unlock(req->ctx, issue_flags);
	3542	+ return NULL;
	3543	+ }
	3544	+
	3545	+ head &= bl->mask;
	3546	+ if (head < IO_BUFFER_LIST_BUF_PER_PAGE) {
	3547	+ buf = &br->bufs[head];
3365	3548	} else {
3366		− kbuf = ERR_PTR(-ENOBUFS);
	3549	+ int off = head & (IO_BUFFER_LIST_BUF_PER_PAGE - 1);
	3550	+ int index = head / IO_BUFFER_LIST_BUF_PER_PAGE - 1;
	3551	+ buf = page_address(bl->buf_pages[index]);
	3552	+ buf += off;
3367	3553	}
	3554	+ if (*len > buf->len)
	3555	+ *len = buf->len;
	3556	+ req->flags \|= REQ_F_BUFFER_RING;
	3557	+ req->buf_list = bl;
	3558	+ req->buf_index = buf->bid;
3368	3559
3369		− io_ring_submit_unlock(req->ctx, needs_lock);
3370		− return kbuf;
	3560	+ if (issue_flags & IO_URING_F_UNLOCKED) {
	3561	+ /*
	3562	+ * If we came in unlocked, we have no choice but to consume the
	3563	+ * buffer here. This does mean it'll be pinned until the IO
	3564	+ * completes. But coming in unlocked means we're in io-wq
	3565	+ * context, hence there should be no further retry. For the
	3566	+ * locked case, the caller must ensure to call the commit when
	3567	+ * the transfer completes (or if we get -EAGAIN and must poll
	3568	+ * or retry).
	3569	+ */
	3570	+ req->buf_list = NULL;
	3571	+ bl->head++;
	3572	+ }
	3573	+ return u64_to_user_ptr(buf->addr);
3371	3574	}
3372	3575
3373		−static void __user io_rw_buffer_select(struct io_kiocb req, size_t *len,
3374		− unsigned int issue_flags)
	3576	+static void __user io_buffer_select(struct io_kiocb req, size_t *len,
	3577	+ unsigned int issue_flags)
3375	3578	{
3376		− struct io_buffer *kbuf;
3377		− u16 bgid;
	3579	+ struct io_ring_ctx *ctx = req->ctx;
	3580	+ struct io_buffer_list *bl;
	3581	+ void __user *ret = NULL;
	3582	+
	3583	+ io_ring_submit_lock(req->ctx, issue_flags);
3378	3584
3379		− bgid = req->buf_index;
3380		− kbuf = io_buffer_select(req, len, bgid, issue_flags);
3381		− if (IS_ERR(kbuf))
3382		− return kbuf;
3383		− return u64_to_user_ptr(kbuf->addr);
	3585	+ bl = io_buffer_get_list(ctx, req->buf_index);
	3586	+ if (likely(bl)) {
	3587	+ if (bl->buf_nr_pages)
	3588	+ ret = io_ring_buffer_select(req, len, bl, issue_flags);
	3589	+ else
	3590	+ ret = io_provided_buffer_select(req, len, bl);
	3591	+ }
	3592	+ io_ring_submit_unlock(req->ctx, issue_flags);
	3593	+ return ret;
3384	3594	}
3385	3595
3386	3596	#ifdef CONFIG_COMPAT
@@ -3390,7 +3600,7 @@ static ssize_t io_compat_import(struct io_kiocb req, struct iovec iov, · 3 unmodified lines
3390	3600	struct compat_iovec __user *uiov;
3391	3601	compat_ssize_t clen;
3392	3602	void __user *buf;
3393		− ssize_t len;
	3603	+ size_t len;
3394	3604
3395	3605	uiov = u64_to_user_ptr(req->rw.addr);
3396	3606	if (!access_ok(uiov, sizeof(*uiov)))
@@ -3401,11 +3611,12 @@ static ssize_t io_compat_import(struct io_kiocb req, struct iovec iov, · 4 unmodified lines
3401	3611	return -EINVAL;
3402	3612
3403	3613	len = clen;
3404		− buf = io_rw_buffer_select(req, &len, issue_flags);
3405		− if (IS_ERR(buf))
3406		− return PTR_ERR(buf);
	3614	+ buf = io_buffer_select(req, &len, issue_flags);
	3615	+ if (!buf)
	3616	+ return -ENOBUFS;
	3617	+ req->rw.addr = (unsigned long) buf;
3407	3618	iov[0].iov_base = buf;
3408		− iov[0].iov_len = (compat_size_t) len;
	3619	+ req->rw.len = iov[0].iov_len = (compat_size_t) len;
3409	3620	return 0;
3410	3621	}
3411	3622	#endif
@@ -3423,22 +3634,21 @@ static ssize_t __io_iov_buffer_select(struct io_kiocb req, struct iovec iov, · 11 unmodified lines
3423	3634	len = iov[0].iov_len;
3424	3635	if (len < 0)
3425	3636	return -EINVAL;
3426		− buf = io_rw_buffer_select(req, &len, issue_flags);
3427		− if (IS_ERR(buf))
3428		− return PTR_ERR(buf);
	3637	+ buf = io_buffer_select(req, &len, issue_flags);
	3638	+ if (!buf)
	3639	+ return -ENOBUFS;
	3640	+ req->rw.addr = (unsigned long) buf;
3429	3641	iov[0].iov_base = buf;
3430		− iov[0].iov_len = len;
	3642	+ req->rw.len = iov[0].iov_len = len;
3431	3643	return 0;
3432	3644	}
3433	3645
3434	3646	static ssize_t io_iov_buffer_select(struct io_kiocb req, struct iovec iov,
3435	3647	unsigned int issue_flags)
3436	3648	{
3437		− if (req->flags & REQ_F_BUFFER_SELECTED) {
3438		− struct io_buffer *kbuf = req->kbuf;
3439		−
3440		− iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
3441		− iov[0].iov_len = kbuf->len;
	3649	+ if (req->flags & (REQ_F_BUFFER_SELECTED\|REQ_F_BUFFER_RING)) {
	3650	+ iov[0].iov_base = u64_to_user_ptr(req->rw.addr);
	3651	+ iov[0].iov_len = req->rw.len;
3442	3652	return 0;
3443	3653	}
3444	3654	if (req->rw.len != 1)
@@ -3452,6 +3662,13 @@ static ssize_t io_iov_buffer_select(struct io_kiocb req, struct iovec iov, · 7 unmodified lines
3452	3662	return __io_iov_buffer_select(req, iov, issue_flags);
3453	3663	}
3454	3664
	3665	+static inline bool io_do_buffer_select(struct io_kiocb *req)
	3666	+{
	3667	+ if (!(req->flags & REQ_F_BUFFER_SELECT))
	3668	+ return false;
	3669	+ return !(req->flags & (REQ_F_BUFFER_SELECTED\|REQ_F_BUFFER_RING));
	3670	+}
	3671	+
3455	3672	static struct iovec __io_import_iovec(int rw, struct io_kiocb req,
3456	3673	struct io_rw_state *s,
3457	3674	unsigned int issue_flags)
@@ -3470,18 +3687,15 @@ static struct iovec __io_import_iovec(int rw, struct io_kiocb req, · 12 unmodified lines
3470	3687	return NULL;
3471	3688	}
3472	3689
3473		− /* buffer index only valid with fixed read/write, or buffer select */
3474		− if (unlikely(req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT)))
3475		− return ERR_PTR(-EINVAL);
3476		−
3477	3690	buf = u64_to_user_ptr(req->rw.addr);
3478	3691	sqe_len = req->rw.len;
3479	3692
3480	3693	if (opcode == IORING_OP_READ \|\| opcode == IORING_OP_WRITE) {
3481		− if (req->flags & REQ_F_BUFFER_SELECT) {
3482		− buf = io_rw_buffer_select(req, &sqe_len, issue_flags);
3483		− if (IS_ERR(buf))
3484		− return ERR_CAST(buf);
	3694	+ if (io_do_buffer_select(req)) {
	3695	+ buf = io_buffer_select(req, &sqe_len, issue_flags);
	3696	+ if (!buf)
	3697	+ return ERR_PTR(-ENOBUFS);
	3698	+ req->rw.addr = (unsigned long) buf;
3485	3699	req->rw.len = sqe_len;
3486	3700	}
3487	3701
@@ -3836,7 +4050,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) · 348 unmodified lines
3836	4050	kfree(iovec);
3837	4051	return ret;
3838	4052	}
3839		− req->result = iov_iter_count(&s->iter);
	4053	+ req->cqe.res = iov_iter_count(&s->iter);
3840	4054
3841	4055	if (force_nonblock) {
3842	4056	/* If the file doesn't support async, just async punt */
@@ -3852,7 +4066,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) · 9 unmodified lines
3852	4066
3853	4067	ppos = io_kiocb_update_pos(req);
3854	4068
3855		− ret = rw_verify_area(READ, req->file, ppos, req->result);
	4069	+ ret = rw_verify_area(READ, req->file, ppos, req->cqe.res);
3856	4070	if (unlikely(ret)) {
3857	4071	kfree(iovec);
3858	4072	return ret;
@@ -3874,7 +4088,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) · 15 unmodified lines
3874	4088	ret = 0;
3875	4089	} else if (ret == -EIOCBQUEUED) {
3876	4090	goto out_free;
3877		− } else if (ret == req->result \|\| ret <= 0 \|\| !force_nonblock \|\|
	4091	+ } else if (ret == req->cqe.res \|\| ret <= 0 \|\| !force_nonblock \|\|
3878	4092	(req->flags & REQ_F_NOWAIT) \|\| !need_read_all(req)) {
3879	4093	/* read all, failed, already did sync or don't want to retry */
3880	4094	goto done;
@@ -3964,7 +4178,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) · 83 unmodified lines
3964	4178	kfree(iovec);
3965	4179	return ret;
3966	4180	}
3967		− req->result = iov_iter_count(&s->iter);
	4181	+ req->cqe.res = iov_iter_count(&s->iter);
3968	4182
3969	4183	if (force_nonblock) {
3970	4184	/* If the file doesn't support async, just async punt */
@@ -3984,7 +4198,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) · 13 unmodified lines
3984	4198
3985	4199	ppos = io_kiocb_update_pos(req);
3986	4200
3987		− ret = rw_verify_area(WRITE, req->file, ppos, req->result);
	4201	+ ret = rw_verify_area(WRITE, req->file, ppos, req->cqe.res);
3988	4202	if (unlikely(ret))
3989	4203	goto out_free;
3990	4204
@@ -4048,9 +4262,7 @@ static int io_renameat_prep(struct io_kiocb *req, · 57 unmodified lines
4048	4262	struct io_rename *ren = &req->rename;
4049	4263	const char __user oldf, newf;
4050	4264
4051		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4052		− return -EINVAL;
4053		− if (sqe->ioprio \|\| sqe->buf_index \|\| sqe->splice_fd_in)
	4265	+ if (sqe->buf_index \|\| sqe->splice_fd_in)
4054	4266	return -EINVAL;
4055	4267	if (unlikely(req->flags & REQ_F_FIXED_FILE))
4056	4268	return -EBADF;
@@ -4099,10 +4311,7 @@ static int io_unlinkat_prep(struct io_kiocb *req, · 42 unmodified lines
4099	4311	struct io_unlink *un = &req->unlink;
4100	4312	const char __user *fname;
4101	4313
4102		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4103		− return -EINVAL;
4104		− if (sqe->ioprio \|\| sqe->off \|\| sqe->len \|\| sqe->buf_index \|\|
4105		− sqe->splice_fd_in)
	4314	+ if (sqe->off \|\| sqe->len \|\| sqe->buf_index \|\| sqe->splice_fd_in)
4106	4315	return -EINVAL;
4107	4316	if (unlikely(req->flags & REQ_F_FIXED_FILE))
4108	4317	return -EBADF;
@@ -4148,10 +4357,7 @@ static int io_mkdirat_prep(struct io_kiocb *req, · 39 unmodified lines
4148	4357	struct io_mkdir *mkd = &req->mkdir;
4149	4358	const char __user *fname;
4150	4359
4151		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4152		− return -EINVAL;
4153		− if (sqe->ioprio \|\| sqe->off \|\| sqe->rw_flags \|\| sqe->buf_index \|\|
4154		− sqe->splice_fd_in)
	4360	+ if (sqe->off \|\| sqe->rw_flags \|\| sqe->buf_index \|\| sqe->splice_fd_in)
4155	4361	return -EINVAL;
4156	4362	if (unlikely(req->flags & REQ_F_FIXED_FILE))
4157	4363	return -EBADF;
@@ -4191,10 +4397,7 @@ static int io_symlinkat_prep(struct io_kiocb *req, · 33 unmodified lines
4191	4397	struct io_symlink *sl = &req->symlink;
4192	4398	const char __user oldpath, newpath;
4193	4399
4194		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4195		− return -EINVAL;
4196		− if (sqe->ioprio \|\| sqe->len \|\| sqe->rw_flags \|\| sqe->buf_index \|\|
4197		− sqe->splice_fd_in)
	4400	+ if (sqe->len \|\| sqe->rw_flags \|\| sqe->buf_index \|\| sqe->splice_fd_in)
4198	4401	return -EINVAL;
4199	4402	if (unlikely(req->flags & REQ_F_FIXED_FILE))
4200	4403	return -EBADF;
@@ -4240,9 +4443,7 @@ static int io_linkat_prep(struct io_kiocb *req, · 39 unmodified lines
4240	4443	struct io_hardlink *lnk = &req->hardlink;
4241	4444	const char __user oldf, newf;
4242	4445
4243		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4244		− return -EINVAL;
4245		− if (sqe->ioprio \|\| sqe->rw_flags \|\| sqe->buf_index \|\| sqe->splice_fd_in)
	4446	+ if (sqe->rw_flags \|\| sqe->buf_index \|\| sqe->splice_fd_in)
4246	4447	return -EINVAL;
4247	4448	if (unlikely(req->flags & REQ_F_FIXED_FILE))
4248	4449	return -EBADF;
@@ -4289,9 +4490,7 @@ static int io_shutdown_prep(struct io_kiocb *req, · 40 unmodified lines
4289	4490	const struct io_uring_sqe *sqe)
4290	4491	{
4291	4492	#if defined(CONFIG_NET)
4292		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4293		− return -EINVAL;
4294		− if (unlikely(sqe->ioprio \|\| sqe->off \|\| sqe->addr \|\| sqe->rw_flags \|\|
	4493	+ if (unlikely(sqe->off \|\| sqe->addr \|\| sqe->rw_flags \|\|
4295	4494	sqe->buf_index \|\| sqe->splice_fd_in))
4296	4495	return -EINVAL;
4297	4496
@@ -4331,9 +4530,6 @@ static int __io_splice_prep(struct io_kiocb *req, · 33 unmodified lines
4331	4530	struct io_splice *sp = &req->splice;
4332	4531	unsigned int valid_flags = SPLICE_F_FD_IN_FIXED \| SPLICE_F_ALL;
4333	4532
4334		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4335		− return -EINVAL;
4336		−
4337	4533	sp->len = READ_ONCE(sqe->len);
4338	4534	sp->flags = READ_ONCE(sqe->splice_flags);
4339	4535	if (unlikely(sp->flags & ~valid_flags))
@@ -4432,20 +4628,25 @@ done: · 92 unmodified lines
4432	4628	*/
4433	4629	static int io_nop(struct io_kiocb *req, unsigned int issue_flags)
4434	4630	{
4435		− struct io_ring_ctx *ctx = req->ctx;
	4631	+ void __user *buf;
4436	4632
4437		− if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
4438		− return -EINVAL;
	4633	+ if (req->flags & REQ_F_BUFFER_SELECT) {
	4634	+ size_t len = 1;
	4635	+
	4636	+ buf = io_buffer_select(req, &len, issue_flags);
	4637	+ if (!buf)
	4638	+ return -ENOBUFS;
	4639	+ }
4439	4640
4440		− __io_req_complete(req, issue_flags, 0, 0);
	4641	+ __io_req_complete(req, issue_flags, 0, io_put_kbuf(req, issue_flags));
4441	4642	return 0;
4442	4643	}
4443	4644
4444	4645	static int io_msg_ring_prep(struct io_kiocb *req,
4445	4646	const struct io_uring_sqe *sqe)
4446	4647	{
4447		− if (unlikely(sqe->addr \|\| sqe->ioprio \|\| sqe->rw_flags \|\|
4448		− sqe->splice_fd_in \|\| sqe->buf_index \|\| sqe->personality))
	4648	+ if (unlikely(sqe->addr \|\| sqe->rw_flags \|\| sqe->splice_fd_in \|\|
	4649	+ sqe->buf_index \|\| sqe->personality))
4449	4650	return -EINVAL;
4450	4651
4451	4652	req->msg.user_data = READ_ONCE(sqe->off);
@@ -4489,12 +4690,7 @@ done: · 37 unmodified lines
4489	4690
4490	4691	static int io_fsync_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
4491	4692	{
4492		− struct io_ring_ctx *ctx = req->ctx;
4493		−
4494		− if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
4495		− return -EINVAL;
4496		− if (unlikely(sqe->addr \|\| sqe->ioprio \|\| sqe->buf_index \|\|
4497		− sqe->splice_fd_in))
	4693	+ if (unlikely(sqe->addr \|\| sqe->buf_index \|\| sqe->splice_fd_in))
4498	4694	return -EINVAL;
4499	4695
4500	4696	req->sync.flags = READ_ONCE(sqe->fsync_flags);
@@ -4527,10 +4723,7 @@ static int io_fsync(struct io_kiocb *req, unsigned int issue_flags) · 26 unmodified lines
4527	4723	static int io_fallocate_prep(struct io_kiocb *req,
4528	4724	const struct io_uring_sqe *sqe)
4529	4725	{
4530		− if (sqe->ioprio \|\| sqe->buf_index \|\| sqe->rw_flags \|\|
4531		− sqe->splice_fd_in)
4532		− return -EINVAL;
4533		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
	4726	+ if (sqe->buf_index \|\| sqe->rw_flags \|\| sqe->splice_fd_in)
4534	4727	return -EINVAL;
4535	4728
4536	4729	req->sync.off = READ_ONCE(sqe->off);
@@ -4561,9 +4754,7 @@ static int __io_openat_prep(struct io_kiocb req, const struct io_uring_sqe sqe · 24 unmodified lines
4561	4754	const char __user *fname;
4562	4755	int ret;
4563	4756
4564		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4565		− return -EINVAL;
4566		− if (unlikely(sqe->ioprio \|\| sqe->buf_index))
	4757	+ if (unlikely(sqe->buf_index))
4567	4758	return -EINVAL;
4568	4759	if (unlikely(req->flags & REQ_F_FIXED_FILE))
4569	4760	return -EBADF;
@@ -4618,6 +4809,61 @@ static int io_openat2_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 48 unmodified lines
4618	4809	return __io_openat_prep(req, sqe);
4619	4810	}
4620	4811
	4812	+static int io_file_bitmap_get(struct io_ring_ctx *ctx)
	4813	+{
	4814	+ struct io_file_table *table = &ctx->file_table;
	4815	+ unsigned long nr = ctx->nr_user_files;
	4816	+ int ret;
	4817	+
	4818	+ if (table->alloc_hint >= nr)
	4819	+ table->alloc_hint = 0;
	4820	+
	4821	+ do {
	4822	+ ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint);
	4823	+ if (ret != nr) {
	4824	+ table->alloc_hint = ret + 1;
	4825	+ return ret;
	4826	+ }
	4827	+ if (!table->alloc_hint)
	4828	+ break;
	4829	+
	4830	+ nr = table->alloc_hint;
	4831	+ table->alloc_hint = 0;
	4832	+ } while (1);
	4833	+
	4834	+ return -ENFILE;
	4835	+}
	4836	+
	4837	+static int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags,
	4838	+ struct file *file, unsigned int file_slot)
	4839	+{
	4840	+ bool alloc_slot = file_slot == IORING_FILE_INDEX_ALLOC;
	4841	+ struct io_ring_ctx *ctx = req->ctx;
	4842	+ int ret;
	4843	+
	4844	+ if (alloc_slot) {
	4845	+ io_ring_submit_lock(ctx, issue_flags);
	4846	+ ret = io_file_bitmap_get(ctx);
	4847	+ if (unlikely(ret < 0)) {
	4848	+ io_ring_submit_unlock(ctx, issue_flags);
	4849	+ return ret;
	4850	+ }
	4851	+
	4852	+ file_slot = ret;
	4853	+ } else {
	4854	+ file_slot--;
	4855	+ }
	4856	+
	4857	+ ret = io_install_fixed_file(req, file, issue_flags, file_slot);
	4858	+ if (alloc_slot) {
	4859	+ io_ring_submit_unlock(ctx, issue_flags);
	4860	+ if (!ret)
	4861	+ return file_slot;
	4862	+ }
	4863	+
	4864	+ return ret;
	4865	+}
	4866	+
4621	4867	static int io_openat2(struct io_kiocb *req, unsigned int issue_flags)
4622	4868	{
4623	4869	struct open_flags op;
@@ -4673,8 +4919,8 @@ static int io_openat2(struct io_kiocb *req, unsigned int issue_flags) · 49 unmodified lines
4673	4919	if (!fixed)
4674	4920	fd_install(ret, file);
4675	4921	else
4676		− ret = io_install_fixed_file(req, file, issue_flags,
4677		− req->open.file_slot - 1);
	4922	+ ret = io_fixed_fd_install(req, issue_flags, file,
	4923	+ req->open.file_slot);
4678	4924	err:
4679	4925	putname(req->open.filename);
4680	4926	req->flags &= ~REQ_F_NEED_CLEANUP;
@@ -4695,7 +4941,7 @@ static int io_remove_buffers_prep(struct io_kiocb *req, · 14 unmodified lines
4695	4941	struct io_provide_buf *p = &req->pbuf;
4696	4942	u64 tmp;
4697	4943
4698		− if (sqe->ioprio \|\| sqe->rw_flags \|\| sqe->addr \|\| sqe->len \|\| sqe->off \|\|
	4944	+ if (sqe->rw_flags \|\| sqe->addr \|\| sqe->len \|\| sqe->off \|\|
4699	4945	sqe->splice_fd_in)
4700	4946	return -EINVAL;
4701	4947
@@ -4718,6 +4964,20 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx, · 16 unmodified lines
4718	4964	if (!nbufs)
4719	4965	return 0;
4720	4966
	4967	+ if (bl->buf_nr_pages) {
	4968	+ int j;
	4969	+
	4970	+ i = bl->buf_ring->tail - bl->head;
	4971	+ for (j = 0; j < bl->buf_nr_pages; j++)
	4972	+ unpin_user_page(bl->buf_pages[j]);
	4973	+ kvfree(bl->buf_pages);
	4974	+ bl->buf_pages = NULL;
	4975	+ bl->buf_nr_pages = 0;
	4976	+ /* make sure it's seen as empty */
	4977	+ INIT_LIST_HEAD(&bl->buf_list);
	4978	+ return i;
	4979	+ }
	4980	+
4721	4981	/* the head kbuf is the list itself */
4722	4982	while (!list_empty(&bl->buf_list)) {
4723	4983	struct io_buffer *nxt;
@@ -4739,22 +4999,23 @@ static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) · 15 unmodified lines
4739	4999	struct io_ring_ctx *ctx = req->ctx;
4740	5000	struct io_buffer_list *bl;
4741	5001	int ret = 0;
4742		− bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
4743		−
4744		− io_ring_submit_lock(ctx, needs_lock);
4745	5002
4746		− lockdep_assert_held(&ctx->uring_lock);
	5003	+ io_ring_submit_lock(ctx, issue_flags);
4747	5004
4748	5005	ret = -ENOENT;
4749	5006	bl = io_buffer_get_list(ctx, p->bgid);
4750		− if (bl)
4751		− ret = __io_remove_buffers(ctx, bl, p->nbufs);
	5007	+ if (bl) {
	5008	+ ret = -EINVAL;
	5009	+ /* can't use provide/remove buffers command on mapped buffers */
	5010	+ if (!bl->buf_nr_pages)
	5011	+ ret = __io_remove_buffers(ctx, bl, p->nbufs);
	5012	+ }
4752	5013	if (ret < 0)
4753	5014	req_set_fail(req);
4754	5015
4755	5016	/* complete before unlock, IOPOLL may need the lock */
4756	5017	__io_req_complete(req, issue_flags, ret, 0);
4757		− io_ring_submit_unlock(ctx, needs_lock);
	5018	+ io_ring_submit_unlock(ctx, issue_flags);
4758	5019	return 0;
4759	5020	}
4760	5021
@@ -4765,7 +5026,7 @@ static int io_provide_buffers_prep(struct io_kiocb *req, · 4 unmodified lines
4765	5026	struct io_provide_buf *p = &req->pbuf;
4766	5027	u64 tmp;
4767	5028
4768		− if (sqe->ioprio \|\| sqe->rw_flags \|\| sqe->splice_fd_in)
	5029	+ if (sqe->rw_flags \|\| sqe->splice_fd_in)
4769	5030	return -EINVAL;
4770	5031
4771	5032	tmp = READ_ONCE(sqe->fd);
@@ -4862,26 +5123,56 @@ static int io_add_buffers(struct io_ring_ctx ctx, struct io_provide_buf pbuf, · 90 unmodified lines
4862	5123	return i ? 0 : -ENOMEM;
4863	5124	}
4864	5125
	5126	+static __cold int io_init_bl_list(struct io_ring_ctx *ctx)
	5127	+{
	5128	+ int i;
	5129	+
	5130	+ ctx->io_bl = kcalloc(BGID_ARRAY, sizeof(struct io_buffer_list),
	5131	+ GFP_KERNEL);
	5132	+ if (!ctx->io_bl)
	5133	+ return -ENOMEM;
	5134	+
	5135	+ for (i = 0; i < BGID_ARRAY; i++) {
	5136	+ INIT_LIST_HEAD(&ctx->io_bl[i].buf_list);
	5137	+ ctx->io_bl[i].bgid = i;
	5138	+ }
	5139	+
	5140	+ return 0;
	5141	+}
	5142	+
4865	5143	static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
4866	5144	{
4867	5145	struct io_provide_buf *p = &req->pbuf;
4868	5146	struct io_ring_ctx *ctx = req->ctx;
4869	5147	struct io_buffer_list *bl;
4870	5148	int ret = 0;
4871		− bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
4872	5149
4873		− io_ring_submit_lock(ctx, needs_lock);
	5150	+ io_ring_submit_lock(ctx, issue_flags);
4874	5151
4875		− lockdep_assert_held(&ctx->uring_lock);
	5152	+ if (unlikely(p->bgid < BGID_ARRAY && !ctx->io_bl)) {
	5153	+ ret = io_init_bl_list(ctx);
	5154	+ if (ret)
	5155	+ goto err;
	5156	+ }
4876	5157
4877	5158	bl = io_buffer_get_list(ctx, p->bgid);
4878	5159	if (unlikely(!bl)) {
4879		− bl = kmalloc(sizeof(*bl), GFP_KERNEL);
	5160	+ bl = kzalloc(sizeof(*bl), GFP_KERNEL);
4880	5161	if (!bl) {
4881	5162	ret = -ENOMEM;
4882	5163	goto err;
4883	5164	}
4884		− io_buffer_add_list(ctx, bl, p->bgid);
	5165	+ INIT_LIST_HEAD(&bl->buf_list);
	5166	+ ret = io_buffer_add_list(ctx, bl, p->bgid);
	5167	+ if (ret) {
	5168	+ kfree(bl);
	5169	+ goto err;
	5170	+ }
	5171	+ }
	5172	+ /* can't add buffers via this command for a mapped buffer ring */
	5173	+ if (bl->buf_nr_pages) {
	5174	+ ret = -EINVAL;
	5175	+ goto err;
4885	5176	}
4886	5177
4887	5178	ret = io_add_buffers(ctx, p, bl);
@@ -4890,7 +5181,7 @@ err: · 2 unmodified lines
4890	5181	req_set_fail(req);
4891	5182	/* complete before unlock, IOPOLL may need the lock */
4892	5183	__io_req_complete(req, issue_flags, ret, 0);
4893		− io_ring_submit_unlock(ctx, needs_lock);
	5184	+ io_ring_submit_unlock(ctx, issue_flags);
4894	5185	return 0;
4895	5186	}
4896	5187
@@ -4898,9 +5189,7 @@ static int io_epoll_ctl_prep(struct io_kiocb *req, · 1 unmodified line
4898	5189	const struct io_uring_sqe *sqe)
4899	5190	{
4900	5191	#if defined(CONFIG_EPOLL)
4901		− if (sqe->ioprio \|\| sqe->buf_index \|\| sqe->splice_fd_in)
4902		− return -EINVAL;
4903		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
	5192	+ if (sqe->buf_index \|\| sqe->splice_fd_in)
4904	5193	return -EINVAL;
4905	5194
4906	5195	req->epoll.epfd = READ_ONCE(sqe->fd);
@@ -4944,9 +5233,7 @@ static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags) · 37 unmodified lines
4944	5233	static int io_madvise_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
4945	5234	{
4946	5235	#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
4947		− if (sqe->ioprio \|\| sqe->buf_index \|\| sqe->off \|\| sqe->splice_fd_in)
4948		− return -EINVAL;
4949		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
	5236	+ if (sqe->buf_index \|\| sqe->off \|\| sqe->splice_fd_in)
4950	5237	return -EINVAL;
4951	5238
4952	5239	req->madvise.addr = READ_ONCE(sqe->addr);
@@ -4979,9 +5266,7 @@ static int io_madvise(struct io_kiocb *req, unsigned int issue_flags) · 26 unmodified lines
4979	5266
4980	5267	static int io_fadvise_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
4981	5268	{
4982		− if (sqe->ioprio \|\| sqe->buf_index \|\| sqe->addr \|\| sqe->splice_fd_in)
4983		− return -EINVAL;
4984		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
	5269	+ if (sqe->buf_index \|\| sqe->addr \|\| sqe->splice_fd_in)
4985	5270	return -EINVAL;
4986	5271
4987	5272	req->fadvise.offset = READ_ONCE(sqe->off);
@@ -5017,9 +5302,7 @@ static int io_statx_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 29 unmodified lines
5017	5302	{
5018	5303	const char __user *path;
5019	5304
5020		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5021		− return -EINVAL;
5022		− if (sqe->ioprio \|\| sqe->buf_index \|\| sqe->splice_fd_in)
	5305	+ if (sqe->buf_index \|\| sqe->splice_fd_in)
5023	5306	return -EINVAL;
5024	5307	if (req->flags & REQ_F_FIXED_FILE)
5025	5308	return -EBADF;
@@ -5064,10 +5347,7 @@ static int io_statx(struct io_kiocb *req, unsigned int issue_flags) · 38 unmodified lines
5064	5347
5065	5348	static int io_close_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
5066	5349	{
5067		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5068		− return -EINVAL;
5069		− if (sqe->ioprio \|\| sqe->off \|\| sqe->addr \|\| sqe->len \|\|
5070		− sqe->rw_flags \|\| sqe->buf_index)
	5350	+ if (sqe->off \|\| sqe->addr \|\| sqe->len \|\| sqe->rw_flags \|\| sqe->buf_index)
5071	5351	return -EINVAL;
5072	5352	if (req->flags & REQ_F_FIXED_FILE)
5073	5353	return -EBADF;
@@ -5099,7 +5379,8 @@ static int io_close(struct io_kiocb *req, unsigned int issue_flags) · 25 unmodified lines
5099	5379	spin_unlock(&files->file_lock);
5100	5380	goto err;
5101	5381	}
5102		− file = fdt->fd[close->fd];
	5382	+ file = rcu_dereference_protected(fdt->fd[close->fd],
	5383	+ lockdep_is_held(&files->file_lock));
5103	5384	if (!file \|\| file->f_op == &io_uring_fops) {
5104	5385	spin_unlock(&files->file_lock);
5105	5386	file = NULL;
@@ -5133,12 +5414,7 @@ err: · 27 unmodified lines
5133	5414
5134	5415	static int io_sfr_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
5135	5416	{
5136		− struct io_ring_ctx *ctx = req->ctx;
5137		−
5138		− if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
5139		− return -EINVAL;
5140		− if (unlikely(sqe->addr \|\| sqe->ioprio \|\| sqe->buf_index \|\|
5141		− sqe->splice_fd_in))
	5417	+ if (unlikely(sqe->addr \|\| sqe->buf_index \|\| sqe->splice_fd_in))
5142	5418	return -EINVAL;
5143	5419
5144	5420	req->sync.off = READ_ONCE(sqe->off);
@@ -5164,6 +5440,13 @@ static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags) · 19 unmodified lines
5164	5440	}
5165	5441
5166	5442	#if defined(CONFIG_NET)
	5443	+static bool io_net_retry(struct socket *sock, int flags)
	5444	+{
	5445	+ if (!(flags & MSG_WAITALL))
	5446	+ return false;
	5447	+ return sock->type == SOCK_STREAM \|\| sock->type == SOCK_SEQPACKET;
	5448	+}
	5449	+
5167	5450	static int io_setup_async_msg(struct io_kiocb *req,
5168	5451	struct io_async_msghdr *kmsg)
5169	5452	{
@@ -5209,13 +5492,16 @@ static int io_sendmsg_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 39 unmodified lines
5209	5492	{
5210	5493	struct io_sr_msg *sr = &req->sr_msg;
5211	5494
5212		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
	5495	+ if (unlikely(sqe->file_index))
5213	5496	return -EINVAL;
5214	5497	if (unlikely(sqe->addr2 \|\| sqe->file_index))
5215	5498	return -EINVAL;
5216	5499
5217	5500	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
5218	5501	sr->len = READ_ONCE(sqe->len);
	5502	+ sr->flags = READ_ONCE(sqe->addr2);
	5503	+ if (sr->flags & ~IORING_RECVSEND_POLL_FIRST)
	5504	+ return -EINVAL;
5219	5505	sr->msg_flags = READ_ONCE(sqe->msg_flags) \| MSG_NOSIGNAL;
5220	5506	if (sr->msg_flags & MSG_DONTWAIT)
5221	5507	req->flags \|= REQ_F_NOWAIT;
@@ -5224,12 +5510,14 @@ static int io_sendmsg_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 2 unmodified lines
5224	5510	if (req->ctx->compat)
5225	5511	sr->msg_flags \|= MSG_CMSG_COMPAT;
5226	5512	#endif
	5513	+ sr->done_io = 0;
5227	5514	return 0;
5228	5515	}
5229	5516
5230	5517	static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
5231	5518	{
5232	5519	struct io_async_msghdr iomsg, *kmsg;
	5520	+ struct io_sr_msg *sr = &req->sr_msg;
5233	5521	struct socket *sock;
5234	5522	unsigned flags;
5235	5523	int min_ret = 0;
@@ -5248,7 +5536,11 @@ static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) · 12 unmodified lines
5248	5536	kmsg = &iomsg;
5249	5537	}
5250	5538
5251		− flags = req->sr_msg.msg_flags;
	5539	+ if (!(req->flags & REQ_F_POLLED) &&
	5540	+ (sr->flags & IORING_RECVSEND_POLL_FIRST))
	5541	+ return io_setup_async_msg(req, kmsg);
	5542	+
	5543	+ flags = sr->msg_flags;
5252	5544	if (issue_flags & IO_URING_F_NONBLOCK)
5253	5545	flags \|= MSG_DONTWAIT;
5254	5546	if (flags & MSG_WAITALL)
@@ -5261,12 +5553,21 @@ static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) · 6 unmodified lines
5261	5553	return io_setup_async_msg(req, kmsg);
5262	5554	if (ret == -ERESTARTSYS)
5263	5555	ret = -EINTR;
	5556	+ if (ret > 0 && io_net_retry(sock, flags)) {
	5557	+ sr->done_io += ret;
	5558	+ req->flags \|= REQ_F_PARTIAL_IO;
	5559	+ return io_setup_async_msg(req, kmsg);
	5560	+ }
5264	5561	req_set_fail(req);
5265	5562	}
5266	5563	/* fast path, check for non-NULL to avoid function call */
5267	5564	if (kmsg->free_iov)
5268	5565	kfree(kmsg->free_iov);
5269	5566	req->flags &= ~REQ_F_NEED_CLEANUP;
	5567	+ if (ret >= 0)
	5568	+ ret += sr->done_io;
	5569	+ else if (sr->done_io)
	5570	+ ret = sr->done_io;
5270	5571	__io_req_complete(req, issue_flags, ret, 0);
5271	5572	return 0;
5272	5573	}
@@ -5281,6 +5582,10 @@ static int io_send(struct io_kiocb *req, unsigned int issue_flags) · 8 unmodified lines
5281	5582	int min_ret = 0;
5282	5583	int ret;
5283	5584
	5585	+ if (!(req->flags & REQ_F_POLLED) &&
	5586	+ (sr->flags & IORING_RECVSEND_POLL_FIRST))
	5587	+ return -EAGAIN;
	5588	+
5284	5589	sock = sock_from_file(req->file);
5285	5590	if (unlikely(!sock))
5286	5591	return -ENOTSOCK;
@@ -5294,7 +5599,7 @@ static int io_send(struct io_kiocb *req, unsigned int issue_flags) · 7 unmodified lines
5294	5599	msg.msg_controllen = 0;
5295	5600	msg.msg_namelen = 0;
5296	5601
5297		− flags = req->sr_msg.msg_flags;
	5602	+ flags = sr->msg_flags;
5298	5603	if (issue_flags & IO_URING_F_NONBLOCK)
5299	5604	flags \|= MSG_DONTWAIT;
5300	5605	if (flags & MSG_WAITALL)
@@ -5307,8 +5612,19 @@ static int io_send(struct io_kiocb *req, unsigned int issue_flags) · 6 unmodified lines
5307	5612	return -EAGAIN;
5308	5613	if (ret == -ERESTARTSYS)
5309	5614	ret = -EINTR;
	5615	+ if (ret > 0 && io_net_retry(sock, flags)) {
	5616	+ sr->len -= ret;
	5617	+ sr->buf += ret;
	5618	+ sr->done_io += ret;
	5619	+ req->flags \|= REQ_F_PARTIAL_IO;
	5620	+ return -EAGAIN;
	5621	+ }
5310	5622	req_set_fail(req);
5311	5623	}
	5624	+ if (ret >= 0)
	5625	+ ret += sr->done_io;
	5626	+ else if (sr->done_io)
	5627	+ ret = sr->done_io;
5312	5628	__io_req_complete(req, issue_flags, ret, 0);
5313	5629	return 0;
5314	5630	}
@@ -5400,14 +5716,6 @@ static int io_recvmsg_copy_hdr(struct io_kiocb *req, · 85 unmodified lines
5400	5716	return __io_recvmsg_copy_hdr(req, iomsg);
5401	5717	}
5402	5718
5403		−static struct io_buffer io_recv_buffer_select(struct io_kiocb req,
5404		− unsigned int issue_flags)
5405		−{
5406		− struct io_sr_msg *sr = &req->sr_msg;
5407		−
5408		− return io_buffer_select(req, &sr->len, sr->bgid, issue_flags);
5409		−}
5410		−
5411	5719	static int io_recvmsg_prep_async(struct io_kiocb *req)
5412	5720	{
5413	5721	int ret;
@@ -5422,14 +5730,16 @@ static int io_recvmsg_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 8 unmodified lines
5422	5730	{
5423	5731	struct io_sr_msg *sr = &req->sr_msg;
5424	5732
5425		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
	5733	+ if (unlikely(sqe->file_index))
5426	5734	return -EINVAL;
5427	5735	if (unlikely(sqe->addr2 \|\| sqe->file_index))
5428	5736	return -EINVAL;
5429	5737
5430	5738	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
5431	5739	sr->len = READ_ONCE(sqe->len);
5432		− sr->bgid = READ_ONCE(sqe->buf_group);
	5740	+ sr->flags = READ_ONCE(sqe->addr2);
	5741	+ if (sr->flags & ~IORING_RECVSEND_POLL_FIRST)
	5742	+ return -EINVAL;
5433	5743	sr->msg_flags = READ_ONCE(sqe->msg_flags) \| MSG_NOSIGNAL;
5434	5744	if (sr->msg_flags & MSG_DONTWAIT)
5435	5745	req->flags \|= REQ_F_NOWAIT;
@@ -5442,19 +5752,11 @@ static int io_recvmsg_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 6 unmodified lines
5442	5752	return 0;
5443	5753	}
5444	5754
5445		−static bool io_net_retry(struct socket *sock, int flags)
5446		−{
5447		− if (!(flags & MSG_WAITALL))
5448		− return false;
5449		− return sock->type == SOCK_STREAM \|\| sock->type == SOCK_SEQPACKET;
5450		−}
5451		−
5452	5755	static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
5453	5756	{
5454	5757	struct io_async_msghdr iomsg, *kmsg;
5455	5758	struct io_sr_msg *sr = &req->sr_msg;
5456	5759	struct socket *sock;
5457		− struct io_buffer *kbuf;
5458	5760	unsigned flags;
5459	5761	int ret, min_ret = 0;
5460	5762	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
@@ -5472,24 +5774,29 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) · 11 unmodified lines
5472	5774	kmsg = &iomsg;
5473	5775	}
5474	5776
5475		− if (req->flags & REQ_F_BUFFER_SELECT) {
5476		− kbuf = io_recv_buffer_select(req, issue_flags);
5477		− if (IS_ERR(kbuf))
5478		− return PTR_ERR(kbuf);
5479		− kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
5480		− kmsg->fast_iov[0].iov_len = req->sr_msg.len;
5481		− iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov,
5482		− 1, req->sr_msg.len);
	5777	+ if (!(req->flags & REQ_F_POLLED) &&
	5778	+ (sr->flags & IORING_RECVSEND_POLL_FIRST))
	5779	+ return io_setup_async_msg(req, kmsg);
	5780	+
	5781	+ if (io_do_buffer_select(req)) {
	5782	+ void __user *buf;
	5783	+
	5784	+ buf = io_buffer_select(req, &sr->len, issue_flags);
	5785	+ if (!buf)
	5786	+ return -ENOBUFS;
	5787	+ kmsg->fast_iov[0].iov_base = buf;
	5788	+ kmsg->fast_iov[0].iov_len = sr->len;
	5789	+ iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov, 1,
	5790	+ sr->len);
5483	5791	}
5484	5792
5485		− flags = req->sr_msg.msg_flags;
	5793	+ flags = sr->msg_flags;
5486	5794	if (force_nonblock)
5487	5795	flags \|= MSG_DONTWAIT;
5488	5796	if (flags & MSG_WAITALL)
5489	5797	min_ret = iov_iter_count(&kmsg->msg.msg_iter);
5490	5798
5491		− ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg,
5492		− kmsg->uaddr, flags);
	5799	+ ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, kmsg->uaddr, flags);
5493	5800	if (ret < min_ret) {
5494	5801	if (ret == -EAGAIN && force_nonblock)
5495	5802	return io_setup_async_msg(req, kmsg);
@@ -5519,28 +5826,32 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) · 23 unmodified lines
5519	5826
5520	5827	static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
5521	5828	{
5522		− struct io_buffer *kbuf;
5523	5829	struct io_sr_msg *sr = &req->sr_msg;
5524	5830	struct msghdr msg;
5525		− void __user *buf = sr->buf;
5526	5831	struct socket *sock;
5527	5832	struct iovec iov;
5528	5833	unsigned flags;
5529	5834	int ret, min_ret = 0;
5530	5835	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
5531	5836
	5837	+ if (!(req->flags & REQ_F_POLLED) &&
	5838	+ (sr->flags & IORING_RECVSEND_POLL_FIRST))
	5839	+ return -EAGAIN;
	5840	+
5532	5841	sock = sock_from_file(req->file);
5533	5842	if (unlikely(!sock))
5534	5843	return -ENOTSOCK;
5535	5844
5536		− if (req->flags & REQ_F_BUFFER_SELECT) {
5537		− kbuf = io_recv_buffer_select(req, issue_flags);
5538		− if (IS_ERR(kbuf))
5539		− return PTR_ERR(kbuf);
5540		− buf = u64_to_user_ptr(kbuf->addr);
	5845	+ if (io_do_buffer_select(req)) {
	5846	+ void __user *buf;
	5847	+
	5848	+ buf = io_buffer_select(req, &sr->len, issue_flags);
	5849	+ if (!buf)
	5850	+ return -ENOBUFS;
	5851	+ sr->buf = buf;
5541	5852	}
5542	5853
5543		− ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter);
	5854	+ ret = import_single_range(READ, sr->buf, sr->len, &iov, &msg.msg_iter);
5544	5855	if (unlikely(ret))
5545	5856	goto out_free;
5546	5857
@@ -5551,7 +5862,7 @@ static int io_recv(struct io_kiocb *req, unsigned int issue_flags) · 4 unmodified lines
5551	5862	msg.msg_iocb = NULL;
5552	5863	msg.msg_flags = 0;
5553	5864
5554		− flags = req->sr_msg.msg_flags;
	5865	+ flags = sr->msg_flags;
5555	5866	if (force_nonblock)
5556	5867	flags \|= MSG_DONTWAIT;
5557	5868	if (flags & MSG_WAITALL)
@@ -5587,29 +5898,39 @@ out_free: · 29 unmodified lines
5587	5898	static int io_accept_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
5588	5899	{
5589	5900	struct io_accept *accept = &req->accept;
	5901	+ unsigned flags;
5590	5902
5591		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5592		− return -EINVAL;
5593		− if (sqe->ioprio \|\| sqe->len \|\| sqe->buf_index)
	5903	+ if (sqe->len \|\| sqe->buf_index)
5594	5904	return -EINVAL;
5595	5905
5596	5906	accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
5597	5907	accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
5598	5908	accept->flags = READ_ONCE(sqe->accept_flags);
5599	5909	accept->nofile = rlimit(RLIMIT_NOFILE);
	5910	+ flags = READ_ONCE(sqe->ioprio);
	5911	+ if (flags & ~IORING_ACCEPT_MULTISHOT)
	5912	+ return -EINVAL;
5600	5913
5601	5914	accept->file_slot = READ_ONCE(sqe->file_index);
5602		− if (accept->file_slot && (accept->flags & SOCK_CLOEXEC))
5603		− return -EINVAL;
	5915	+ if (accept->file_slot) {
	5916	+ if (accept->flags & SOCK_CLOEXEC)
	5917	+ return -EINVAL;
	5918	+ if (flags & IORING_ACCEPT_MULTISHOT &&
	5919	+ accept->file_slot != IORING_FILE_INDEX_ALLOC)
	5920	+ return -EINVAL;
	5921	+ }
5604	5922	if (accept->flags & ~(SOCK_CLOEXEC \| SOCK_NONBLOCK))
5605	5923	return -EINVAL;
5606	5924	if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK))
5607	5925	accept->flags = (accept->flags & ~SOCK_NONBLOCK) \| O_NONBLOCK;
	5926	+ if (flags & IORING_ACCEPT_MULTISHOT)
	5927	+ req->flags \|= REQ_F_APOLL_MULTISHOT;
5608	5928	return 0;
5609	5929	}
5610	5930
5611	5931	static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
5612	5932	{
	5933	+ struct io_ring_ctx *ctx = req->ctx;
5613	5934	struct io_accept *accept = &req->accept;
5614	5935	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
5615	5936	unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
@@ -5617,6 +5938,7 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags) · 1 unmodified line
5617	5938	struct file *file;
5618	5939	int ret, fd;
5619	5940
	5941	+retry:
5620	5942	if (!fixed) {
5621	5943	fd = __get_unused_fd_flags(accept->flags, accept->nofile);
5622	5944	if (unlikely(fd < 0))
@@ -5628,8 +5950,17 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags) · 5 unmodified lines
5628	5950	if (!fixed)
5629	5951	put_unused_fd(fd);
5630	5952	ret = PTR_ERR(file);
5631		− if (ret == -EAGAIN && force_nonblock)
5632		− return -EAGAIN;
	5953	+ if (ret == -EAGAIN && force_nonblock) {
	5954	+ /*
	5955	+ * if it's multishot and polled, we don't need to
	5956	+ * return EAGAIN to arm the poll infra since it
	5957	+ * has already been done
	5958	+ */
	5959	+ if ((req->flags & IO_APOLL_MULTI_POLLED) ==
	5960	+ IO_APOLL_MULTI_POLLED)
	5961	+ ret = 0;
	5962	+ return ret;
	5963	+ }
5633	5964	if (ret == -ERESTARTSYS)
5634	5965	ret = -EINTR;
5635	5966	req_set_fail(req);
@@ -5637,11 +5968,30 @@ static int io_accept(struct io_kiocb *req, unsigned int issue_flags) · 1 unmodified line
5637	5968	fd_install(fd, file);
5638	5969	ret = fd;
5639	5970	} else {
5640		− ret = io_install_fixed_file(req, file, issue_flags,
5641		− accept->file_slot - 1);
	5971	+ ret = io_fixed_fd_install(req, issue_flags, file,
	5972	+ accept->file_slot);
5642	5973	}
5643		− __io_req_complete(req, issue_flags, ret, 0);
5644		− return 0;
	5974	+
	5975	+ if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
	5976	+ __io_req_complete(req, issue_flags, ret, 0);
	5977	+ return 0;
	5978	+ }
	5979	+ if (ret >= 0) {
	5980	+ bool filled;
	5981	+
	5982	+ spin_lock(&ctx->completion_lock);
	5983	+ filled = io_fill_cqe_aux(ctx, req->cqe.user_data, ret,
	5984	+ IORING_CQE_F_MORE);
	5985	+ io_commit_cqring(ctx);
	5986	+ spin_unlock(&ctx->completion_lock);
	5987	+ if (filled) {
	5988	+ io_cqring_ev_posted(ctx);
	5989	+ goto retry;
	5990	+ }
	5991	+ ret = -ECANCELED;
	5992	+ }
	5993	+
	5994	+ return ret;
5645	5995	}
5646	5996
5647	5997	static int io_connect_prep_async(struct io_kiocb *req)
@@ -5656,10 +6006,7 @@ static int io_connect_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 8 unmodified lines
5656	6006	{
5657	6007	struct io_connect *conn = &req->connect;
5658	6008
5659		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5660		− return -EINVAL;
5661		− if (sqe->ioprio \|\| sqe->len \|\| sqe->buf_index \|\| sqe->rw_flags \|\|
5662		− sqe->splice_fd_in)
	6009	+ if (sqe->len \|\| sqe->buf_index \|\| sqe->rw_flags \|\| sqe->splice_fd_in)
5663	6010	return -EINVAL;
5664	6011
5665	6012	conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
@@ -5782,7 +6129,7 @@ static void io_poll_req_insert(struct io_kiocb *req) · 116 unmodified lines
5782	6129	struct io_ring_ctx *ctx = req->ctx;
5783	6130	struct hlist_head *list;
5784	6131
5785		− list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
	6132	+ list = &ctx->cancel_hash[hash_long(req->cqe.user_data, ctx->cancel_hash_bits)];
5786	6133	hlist_add_head(&req->hash_node, list);
5787	6134	}
5788	6135
@@ -5841,22 +6188,23 @@ static void io_poll_remove_entries(struct io_kiocb *req) · 52 unmodified lines
5841	6188	rcu_read_unlock();
5842	6189	}
5843	6190
	6191	+static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags);
5844	6192	/*
5845	6193	* All poll tw should go through this. Checks for poll events, manages
5846	6194	* references, does rewait, etc.
5847	6195	*
5848	6196	* Returns a negative error on failure. >0 when no action require, which is
5849	6197	* either spurious wakeup or multishot CQE is served. 0 when it's done with
5850		− * the request, then the mask is stored in req->result.
	6198	+ * the request, then the mask is stored in req->cqe.res.
5851	6199	*/
5852		−static int io_poll_check_events(struct io_kiocb *req, bool locked)
	6200	+static int io_poll_check_events(struct io_kiocb req, bool locked)
5853	6201	{
5854	6202	struct io_ring_ctx *ctx = req->ctx;
5855		− int v;
	6203	+ int v, ret;
5856	6204
5857	6205	/* req->task == current here, checking PF_EXITING is safe */
5858	6206	if (unlikely(req->task->flags & PF_EXITING))
5859		− io_poll_mark_cancelled(req);
	6207	+ return -ECANCELED;
5860	6208
5861	6209	do {
5862	6210	v = atomic_read(&req->poll_refs);
@@ -5867,32 +6215,46 @@ static int io_poll_check_events(struct io_kiocb *req, bool locked) · 4 unmodified lines
5867	6215	if (v & IO_POLL_CANCEL_FLAG)
5868	6216	return -ECANCELED;
5869	6217
5870		− if (!req->result) {
	6218	+ if (!req->cqe.res) {
5871	6219	struct poll_table_struct pt = { ._key = req->apoll_events };
5872	6220	unsigned flags = locked ? 0 : IO_URING_F_UNLOCKED;
5873	6221
5874	6222	if (unlikely(!io_assign_file(req, flags)))
5875	6223	return -EBADF;
5876		− req->result = vfs_poll(req->file, &pt) & req->apoll_events;
	6224	+ req->cqe.res = vfs_poll(req->file, &pt) & req->apoll_events;
5877	6225	}
5878	6226
5879		− /* multishot, just fill an CQE and proceed */
5880		− if (req->result && !(req->apoll_events & EPOLLONESHOT)) {
5881		− __poll_t mask = mangle_poll(req->result & req->apoll_events);
	6227	+ if ((unlikely(!req->cqe.res)))
	6228	+ continue;
	6229	+ if (req->apoll_events & EPOLLONESHOT)
	6230	+ return 0;
	6231	+
	6232	+ /* multishot, just fill a CQE and proceed */
	6233	+ if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
	6234	+ __poll_t mask = mangle_poll(req->cqe.res &
	6235	+ req->apoll_events);
5882	6236	bool filled;
5883	6237
5884	6238	spin_lock(&ctx->completion_lock);
5885		− filled = io_fill_cqe_aux(ctx, req->user_data, mask,
5886		− IORING_CQE_F_MORE);
	6239	+ filled = io_fill_cqe_aux(ctx, req->cqe.user_data,
	6240	+ mask, IORING_CQE_F_MORE);
5887	6241	io_commit_cqring(ctx);
5888	6242	spin_unlock(&ctx->completion_lock);
5889		− if (unlikely(!filled))
5890		− return -ECANCELED;
5891		− io_cqring_ev_posted(ctx);
5892		− } else if (req->result) {
5893		− return 0;
	6243	+ if (filled) {
	6244	+ io_cqring_ev_posted(ctx);
	6245	+ continue;
	6246	+ }
	6247	+ return -ECANCELED;
5894	6248	}
5895	6249
	6250	+ io_tw_lock(req->ctx, locked);
	6251	+ if (unlikely(req->task->flags & PF_EXITING))
	6252	+ return -EFAULT;
	6253	+ ret = io_issue_sqe(req,
	6254	+ IO_URING_F_NONBLOCK\|IO_URING_F_COMPLETE_DEFER);
	6255	+ if (ret)
	6256	+ return ret;
	6257	+
5896	6258	/*
5897	6259	* Release all references, retry if someone tried to restart
5898	6260	* task_work while we were executing it.
@@ -5907,21 +6269,21 @@ static void io_poll_task_func(struct io_kiocb req, bool locked) · 8 unmodified lines
5907	6269	struct io_ring_ctx *ctx = req->ctx;
5908	6270	int ret;
5909	6271
5910		− ret = io_poll_check_events(req, *locked);
	6272	+ ret = io_poll_check_events(req, locked);
5911	6273	if (ret > 0)
5912	6274	return;
5913	6275
5914	6276	if (!ret) {
5915		− req->result = mangle_poll(req->result & req->poll.events);
	6277	+ req->cqe.res = mangle_poll(req->cqe.res & req->poll.events);
5916	6278	} else {
5917		− req->result = ret;
	6279	+ req->cqe.res = ret;
5918	6280	req_set_fail(req);
5919	6281	}
5920	6282
5921	6283	io_poll_remove_entries(req);
5922	6284	spin_lock(&ctx->completion_lock);
5923	6285	hash_del(&req->hash_node);
5924		− __io_req_complete_post(req, req->result, 0);
	6286	+ __io_req_complete_post(req, req->cqe.res, 0);
5925	6287	io_commit_cqring(ctx);
5926	6288	spin_unlock(&ctx->completion_lock);
5927	6289	io_cqring_ev_posted(ctx);
@@ -5932,7 +6294,7 @@ static void io_apoll_task_func(struct io_kiocb req, bool locked) · 4 unmodified lines
5932	6294	struct io_ring_ctx *ctx = req->ctx;
5933	6295	int ret;
5934	6296
5935		− ret = io_poll_check_events(req, *locked);
	6297	+ ret = io_poll_check_events(req, locked);
5936	6298	if (ret > 0)
5937	6299	return;
5938	6300
@@ -5947,9 +6309,9 @@ static void io_apoll_task_func(struct io_kiocb req, bool locked) · 8 unmodified lines
5947	6309	io_req_complete_failed(req, ret);
5948	6310	}
5949	6311
5950		−static void __io_poll_execute(struct io_kiocb *req, int mask, int events)
	6312	+static void __io_poll_execute(struct io_kiocb *req, int mask, __poll_t events)
5951	6313	{
5952		− req->result = mask;
	6314	+ req->cqe.res = mask;
5953	6315	/*
5954	6316	* This is useful for poll that is armed on behalf of another
5955	6317	* request, and where the wakeup path could be on a different
@@ -5962,11 +6324,12 @@ static void __io_poll_execute(struct io_kiocb *req, int mask, int events) · 6 unmodified lines
5962	6324	else
5963	6325	req->io_task_work.func = io_apoll_task_func;
5964	6326
5965		− trace_io_uring_task_add(req->ctx, req, req->user_data, req->opcode, mask);
	6327	+ trace_io_uring_task_add(req->ctx, req, req->cqe.user_data, req->opcode, mask);
5966	6328	io_req_task_work_add(req, false);
5967	6329	}
5968	6330
5969		−static inline void io_poll_execute(struct io_kiocb *req, int res, int events)
	6331	+static inline void io_poll_execute(struct io_kiocb *req, int res,
	6332	+ __poll_t events)
5970	6333	{
5971	6334	if (io_poll_get_ownership(req))
5972	6335	__io_poll_execute(req, res, events);
@@ -5981,6 +6344,7 @@ static void io_poll_cancel_req(struct io_kiocb *req) · 8 unmodified lines
5981	6344
5982	6345	#define wqe_to_req(wait) ((void *)((unsigned long) (wait)->private & ~1))
5983	6346	#define wqe_is_double(wait) ((unsigned long) (wait)->private & 1)
	6347	+#define IO_ASYNC_POLL_COMMON (EPOLLONESHOT \| EPOLLPRI)
5984	6348
5985	6349	static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
5986	6350	void *key)
@@ -6015,7 +6379,7 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, · 28 unmodified lines
6015	6379	}
6016	6380
6017	6381	/* for instances that support it check for an event match first */
6018		− if (mask && !(mask & poll->events))
	6382	+ if (mask && !(mask & (poll->events & ~IO_ASYNC_POLL_COMMON)))
6019	6383	return 0;
6020	6384
6021	6385	if (io_poll_get_ownership(req)) {
@@ -6101,6 +6465,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, · 79 unmodified lines
6101	6465	int v;
6102	6466
6103	6467	INIT_HLIST_NODE(&req->hash_node);
	6468	+ req->work.cancel_seq = atomic_read(&ctx->cancel_seq);
6104	6469	io_init_poll_iocb(poll, mask, io_poll_wake);
6105	6470	poll->file = req->file;
6106	6471
@@ -6171,28 +6536,34 @@ static int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) · 64 unmodified lines
6171	6536	struct io_ring_ctx *ctx = req->ctx;
6172	6537	struct async_poll *apoll;
6173	6538	struct io_poll_table ipt;
6174		− __poll_t mask = EPOLLONESHOT \| POLLERR \| POLLPRI;
	6539	+ __poll_t mask = POLLPRI \| POLLERR;
6175	6540	int ret;
6176	6541
6177	6542	if (!def->pollin && !def->pollout)
6178	6543	return IO_APOLL_ABORTED;
6179		− if (!file_can_poll(req->file) \|\| (req->flags & REQ_F_POLLED))
	6544	+ if (!file_can_poll(req->file))
6180	6545	return IO_APOLL_ABORTED;
	6546	+ if ((req->flags & (REQ_F_POLLED\|REQ_F_PARTIAL_IO)) == REQ_F_POLLED)
	6547	+ return IO_APOLL_ABORTED;
	6548	+ if (!(req->flags & REQ_F_APOLL_MULTISHOT))
	6549	+ mask \|= EPOLLONESHOT;
6181	6550
6182	6551	if (def->pollin) {
6183		− mask \|= POLLIN \| POLLRDNORM;
	6552	+ mask \|= EPOLLIN \| EPOLLRDNORM;
6184	6553
6185	6554	/* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
6186	6555	if ((req->opcode == IORING_OP_RECVMSG) &&
6187	6556	(req->sr_msg.msg_flags & MSG_ERRQUEUE))
6188		− mask &= ~POLLIN;
	6557	+ mask &= ~EPOLLIN;
6189	6558	} else {
6190		− mask \|= POLLOUT \| POLLWRNORM;
	6559	+ mask \|= EPOLLOUT \| EPOLLWRNORM;
6191	6560	}
6192	6561	if (def->poll_exclusive)
6193	6562	mask \|= EPOLLEXCLUSIVE;
6194		− if (!(issue_flags & IO_URING_F_UNLOCKED) &&
6195		− !list_empty(&ctx->apoll_cache)) {
	6563	+ if (req->flags & REQ_F_POLLED) {
	6564	+ apoll = req->apoll;
	6565	+ } else if (!(issue_flags & IO_URING_F_UNLOCKED) &&
	6566	+ !list_empty(&ctx->apoll_cache)) {
6196	6567	apoll = list_first_entry(&ctx->apoll_cache, struct async_poll,
6197	6568	poll.wait.entry);
6198	6569	list_del_init(&apoll->poll.wait.entry);
@@ -6212,7 +6583,7 @@ static int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) · 13 unmodified lines
6212	6583	if (ret \|\| ipt.error)
6213	6584	return ret ? IO_APOLL_READY : IO_APOLL_ABORTED;
6214	6585
6215		− trace_io_uring_poll_arm(ctx, req, req->user_data, req->opcode,
	6586	+ trace_io_uring_poll_arm(ctx, req, req->cqe.user_data, req->opcode,
6216	6587	mask, apoll->poll.events);
6217	6588	return IO_APOLL_OK;
6218	6589	}
@@ -6245,24 +6616,53 @@ static __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, · 26 unmodified lines
6245	6616	return found;
6246	6617	}
6247	6618
6248		−static struct io_kiocb io_poll_find(struct io_ring_ctx ctx, __u64 sqe_addr,
6249		− bool poll_only)
	6619	+static struct io_kiocb io_poll_find(struct io_ring_ctx ctx, bool poll_only,
	6620	+ struct io_cancel_data *cd)
6250	6621	__must_hold(&ctx->completion_lock)
6251	6622	{
6252	6623	struct hlist_head *list;
6253	6624	struct io_kiocb *req;
6254	6625
6255		− list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
	6626	+ list = &ctx->cancel_hash[hash_long(cd->data, ctx->cancel_hash_bits)];
6256	6627	hlist_for_each_entry(req, list, hash_node) {
6257		− if (sqe_addr != req->user_data)
	6628	+ if (cd->data != req->cqe.user_data)
6258	6629	continue;
6259	6630	if (poll_only && req->opcode != IORING_OP_POLL_ADD)
6260	6631	continue;
	6632	+ if (cd->flags & IORING_ASYNC_CANCEL_ALL) {
	6633	+ if (cd->seq == req->work.cancel_seq)
	6634	+ continue;
	6635	+ req->work.cancel_seq = cd->seq;
	6636	+ }
6261	6637	return req;
6262	6638	}
6263	6639	return NULL;
6264	6640	}
6265	6641
	6642	+static struct io_kiocb io_poll_file_find(struct io_ring_ctx ctx,
	6643	+ struct io_cancel_data *cd)
	6644	+ __must_hold(&ctx->completion_lock)
	6645	+{
	6646	+ struct io_kiocb *req;
	6647	+ int i;
	6648	+
	6649	+ for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
	6650	+ struct hlist_head *list;
	6651	+
	6652	+ list = &ctx->cancel_hash[i];
	6653	+ hlist_for_each_entry(req, list, hash_node) {
	6654	+ if (!(cd->flags & IORING_ASYNC_CANCEL_ANY) &&
	6655	+ req->file != cd->file)
	6656	+ continue;
	6657	+ if (cd->seq == req->work.cancel_seq)
	6658	+ continue;
	6659	+ req->work.cancel_seq = cd->seq;
	6660	+ return req;
	6661	+ }
	6662	+ }
	6663	+ return NULL;
	6664	+}
	6665	+
6266	6666	static bool io_poll_disarm(struct io_kiocb *req)
6267	6667	__must_hold(&ctx->completion_lock)
6268	6668	{
@@ -6273,12 +6673,15 @@ static bool io_poll_disarm(struct io_kiocb *req) · 4 unmodified lines
6273	6673	return true;
6274	6674	}
6275	6675
6276		−static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr,
6277		− bool poll_only)
	6676	+static int io_poll_cancel(struct io_ring_ctx ctx, struct io_cancel_data cd)
6278	6677	__must_hold(&ctx->completion_lock)
6279	6678	{
6280		− struct io_kiocb *req = io_poll_find(ctx, sqe_addr, poll_only);
	6679	+ struct io_kiocb *req;
6281	6680
	6681	+ if (cd->flags & (IORING_ASYNC_CANCEL_FD\|IORING_ASYNC_CANCEL_ANY))
	6682	+ req = io_poll_file_find(ctx, cd);
	6683	+ else
	6684	+ req = io_poll_find(ctx, false, cd);
6282	6685	if (!req)
6283	6686	return -ENOENT;
6284	6687	io_poll_cancel_req(req);
@@ -6305,9 +6708,7 @@ static int io_poll_update_prep(struct io_kiocb *req, · 20 unmodified lines
6305	6708	struct io_poll_update *upd = &req->poll_update;
6306	6709	u32 flags;
6307	6710
6308		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
6309		− return -EINVAL;
6310		− if (sqe->ioprio \|\| sqe->buf_index \|\| sqe->splice_fd_in)
	6711	+ if (sqe->buf_index \|\| sqe->splice_fd_in)
6311	6712	return -EINVAL;
6312	6713	flags = READ_ONCE(sqe->len);
6313	6714	if (flags & ~(IORING_POLL_UPDATE_EVENTS \| IORING_POLL_UPDATE_USER_DATA \|
@@ -6337,9 +6738,7 @@ static int io_poll_add_prep(struct io_kiocb req, const struct io_uring_sqe sqe · 23 unmodified lines
6337	6738	struct io_poll_iocb *poll = &req->poll;
6338	6739	u32 flags;
6339	6740
6340		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
6341		− return -EINVAL;
6342		− if (sqe->ioprio \|\| sqe->buf_index \|\| sqe->off \|\| sqe->addr)
	6741	+ if (sqe->buf_index \|\| sqe->off \|\| sqe->addr)
6343	6742	return -EINVAL;
6344	6743	flags = READ_ONCE(sqe->len);
6345	6744	if (flags & ~IORING_POLL_ADD_MULTI)
@@ -6369,13 +6768,14 @@ static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) · 23 unmodified lines
6369	6768
6370	6769	static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags)
6371	6770	{
	6771	+ struct io_cancel_data cd = { .data = req->poll_update.old_user_data, };
6372	6772	struct io_ring_ctx *ctx = req->ctx;
6373	6773	struct io_kiocb *preq;
6374	6774	int ret2, ret = 0;
6375	6775	bool locked;
6376	6776
6377	6777	spin_lock(&ctx->completion_lock);
6378		− preq = io_poll_find(ctx, req->poll_update.old_user_data, true);
	6778	+ preq = io_poll_find(ctx, true, &cd);
6379	6779	if (!preq \|\| !io_poll_disarm(preq)) {
6380	6780	spin_unlock(&ctx->completion_lock);
6381	6781	ret = preq ? -EALREADY : -ENOENT;
@@ -6391,7 +6791,7 @@ static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags) · 9 unmodified lines
6391	6791	preq->poll.events \|= IO_POLL_UNMASK;
6392	6792	}
6393	6793	if (req->poll_update.update_user_data)
6394		− preq->user_data = req->poll_update.new_user_data;
	6794	+ preq->cqe.user_data = req->poll_update.new_user_data;
6395	6795
6396	6796	ret2 = io_poll_add(preq, issue_flags);
6397	6797	/* successfully updated, don't complete poll request */
@@ -6400,7 +6800,7 @@ static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags) · 2 unmodified lines
6400	6800	}
6401	6801
6402	6802	req_set_fail(preq);
6403		− preq->result = -ECANCELED;
	6803	+ preq->cqe.res = -ECANCELED;
6404	6804	locked = !(issue_flags & IO_URING_F_UNLOCKED);
6405	6805	io_req_task_complete(preq, &locked);
6406	6806	out:
@@ -6428,14 +6828,14 @@ static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) · 21 unmodified lines
6428	6828	if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS))
6429	6829	req_set_fail(req);
6430	6830
6431		− req->result = -ETIME;
	6831	+ req->cqe.res = -ETIME;
6432	6832	req->io_task_work.func = io_req_task_complete;
6433	6833	io_req_task_work_add(req, false);
6434	6834	return HRTIMER_NORESTART;
6435	6835	}
6436	6836
6437	6837	static struct io_kiocb io_timeout_extract(struct io_ring_ctx ctx,
6438		− __u64 user_data)
	6838	+ struct io_cancel_data *cd)
6439	6839	__must_hold(&ctx->timeout_lock)
6440	6840	{
6441	6841	struct io_timeout_data *io;
@@ -6443,9 +6843,16 @@ static struct io_kiocb io_timeout_extract(struct io_ring_ctx ctx, · 1 unmodified line
6443	6843	bool found = false;
6444	6844
6445	6845	list_for_each_entry(req, &ctx->timeout_list, timeout.list) {
6446		− found = user_data == req->user_data;
6447		− if (found)
6448		− break;
	6846	+ if (!(cd->flags & IORING_ASYNC_CANCEL_ANY) &&
	6847	+ cd->data != req->cqe.user_data)
	6848	+ continue;
	6849	+ if (cd->flags & (IORING_ASYNC_CANCEL_ALL\|IORING_ASYNC_CANCEL_ANY)) {
	6850	+ if (cd->seq == req->work.cancel_seq)
	6851	+ continue;
	6852	+ req->work.cancel_seq = cd->seq;
	6853	+ }
	6854	+ found = true;
	6855	+ break;
6449	6856	}
6450	6857	if (!found)
6451	6858	return ERR_PTR(-ENOENT);
@@ -6457,11 +6864,14 @@ static struct io_kiocb io_timeout_extract(struct io_ring_ctx ctx, · 5 unmodified lines
6457	6864	return req;
6458	6865	}
6459	6866
6460		−static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
	6867	+static int io_timeout_cancel(struct io_ring_ctx ctx, struct io_cancel_data cd)
6461	6868	__must_hold(&ctx->completion_lock)
6462		− __must_hold(&ctx->timeout_lock)
6463	6869	{
6464		− struct io_kiocb *req = io_timeout_extract(ctx, user_data);
	6870	+ struct io_kiocb *req;
	6871	+
	6872	+ spin_lock_irq(&ctx->timeout_lock);
	6873	+ req = io_timeout_extract(ctx, cd);
	6874	+ spin_unlock_irq(&ctx->timeout_lock);
6465	6875
6466	6876	if (IS_ERR(req))
6467	6877	return PTR_ERR(req);
@@ -6494,7 +6904,7 @@ static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, · 26 unmodified lines
6494	6904	bool found = false;
6495	6905
6496	6906	list_for_each_entry(req, &ctx->ltimeout_list, timeout.list) {
6497		− found = user_data == req->user_data;
	6907	+ found = user_data == req->cqe.user_data;
6498	6908	if (found)
6499	6909	break;
6500	6910	}
@@ -6514,7 +6924,8 @@ static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, · 13 unmodified lines
6514	6924	struct timespec64 *ts, enum hrtimer_mode mode)
6515	6925	__must_hold(&ctx->timeout_lock)
6516	6926	{
6517		− struct io_kiocb *req = io_timeout_extract(ctx, user_data);
	6927	+ struct io_cancel_data cd = { .data = user_data, };
	6928	+ struct io_kiocb *req = io_timeout_extract(ctx, &cd);
6518	6929	struct io_timeout_data *data;
6519	6930
6520	6931	if (IS_ERR(req))
@@ -6534,11 +6945,9 @@ static int io_timeout_remove_prep(struct io_kiocb *req, · 13 unmodified lines
6534	6945	{
6535	6946	struct io_timeout_rem *tr = &req->timeout_rem;
6536	6947
6537		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
6538		− return -EINVAL;
6539	6948	if (unlikely(req->flags & (REQ_F_FIXED_FILE \| REQ_F_BUFFER_SELECT)))
6540	6949	return -EINVAL;
6541		− if (sqe->ioprio \|\| sqe->buf_index \|\| sqe->len \|\| sqe->splice_fd_in)
	6950	+ if (sqe->buf_index \|\| sqe->len \|\| sqe->splice_fd_in)
6542	6951	return -EINVAL;
6543	6952
6544	6953	tr->ltimeout = false;
@@ -6579,10 +6988,10 @@ static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags) · 34 unmodified lines
6579	6988	int ret;
6580	6989
6581	6990	if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE)) {
	6991	+ struct io_cancel_data cd = { .data = tr->addr, };
	6992	+
6582	6993	spin_lock(&ctx->completion_lock);
6583		− spin_lock_irq(&ctx->timeout_lock);
6584		− ret = io_timeout_cancel(ctx, tr->addr);
6585		− spin_unlock_irq(&ctx->timeout_lock);
	6994	+ ret = io_timeout_cancel(ctx, &cd);
6586	6995	spin_unlock(&ctx->completion_lock);
6587	6996	} else {
6588	6997	enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags);
@@ -6608,10 +7017,7 @@ static int io_timeout_prep(struct io_kiocb req, const struct io_uring_sqe sqe, · 19 unmodified lines
6608	7017	unsigned flags;
6609	7018	u32 off = READ_ONCE(sqe->off);
6610	7019
6611		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
6612		− return -EINVAL;
6613		− if (sqe->ioprio \|\| sqe->buf_index \|\| sqe->len != 1 \|\|
6614		− sqe->splice_fd_in)
	7020	+ if (sqe->buf_index \|\| sqe->len != 1 \|\| sqe->splice_fd_in)
6615	7021	return -EINVAL;
6616	7022	if (off && is_timeout_link)
6617	7023	return -EINVAL;
@@ -6710,30 +7116,42 @@ add: · 92 unmodified lines
6710	7116	return 0;
6711	7117	}
6712	7118
6713		−struct io_cancel_data {
6714		− struct io_ring_ctx *ctx;
6715		− u64 user_data;
6716		−};
6717		−
6718	7119	static bool io_cancel_cb(struct io_wq_work work, void data)
6719	7120	{
6720	7121	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
6721	7122	struct io_cancel_data *cd = data;
6722	7123
6723		− return req->ctx == cd->ctx && req->user_data == cd->user_data;
	7124	+ if (req->ctx != cd->ctx)
	7125	+ return false;
	7126	+ if (cd->flags & IORING_ASYNC_CANCEL_ANY) {
	7127	+ ;
	7128	+ } else if (cd->flags & IORING_ASYNC_CANCEL_FD) {
	7129	+ if (req->file != cd->file)
	7130	+ return false;
	7131	+ } else {
	7132	+ if (req->cqe.user_data != cd->data)
	7133	+ return false;
	7134	+ }
	7135	+ if (cd->flags & (IORING_ASYNC_CANCEL_ALL\|IORING_ASYNC_CANCEL_ANY)) {
	7136	+ if (cd->seq == req->work.cancel_seq)
	7137	+ return false;
	7138	+ req->work.cancel_seq = cd->seq;
	7139	+ }
	7140	+ return true;
6724	7141	}
6725	7142
6726		−static int io_async_cancel_one(struct io_uring_task *tctx, u64 user_data,
6727		− struct io_ring_ctx *ctx)
	7143	+static int io_async_cancel_one(struct io_uring_task *tctx,
	7144	+ struct io_cancel_data *cd)
6728	7145	{
6729		− struct io_cancel_data data = { .ctx = ctx, .user_data = user_data, };
6730	7146	enum io_wq_cancel cancel_ret;
6731	7147	int ret = 0;
	7148	+ bool all;
6732	7149
6733	7150	if (!tctx \|\| !tctx->io_wq)
6734	7151	return -ENOENT;
6735	7152
6736		− cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, &data, false);
	7153	+ all = cd->flags & (IORING_ASYNC_CANCEL_ALL\|IORING_ASYNC_CANCEL_ANY);
	7154	+ cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, cd, all);
6737	7155	switch (cancel_ret) {
6738	7156	case IO_WQ_CANCEL_OK:
6739	7157	ret = 0;
@@ -6749,14 +7167,14 @@ static int io_async_cancel_one(struct io_uring_task *tctx, u64 user_data, · 9 unmodified lines
6749	7167	return ret;
6750	7168	}
6751	7169
6752		−static int io_try_cancel_userdata(struct io_kiocb *req, u64 sqe_addr)
	7170	+static int io_try_cancel(struct io_kiocb req, struct io_cancel_data cd)
6753	7171	{
6754	7172	struct io_ring_ctx *ctx = req->ctx;
6755	7173	int ret;
6756	7174
6757	7175	WARN_ON_ONCE(!io_wq_current_is_worker() && req->task != current);
6758	7176
6759		− ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx);
	7177	+ ret = io_async_cancel_one(req->task->io_uring, cd);
6760	7178	/*
6761	7179	* Fall-through even for -EALREADY, as we may have poll armed
6762	7180	* that need unarming.
@@ -6765,56 +7183,98 @@ static int io_try_cancel_userdata(struct io_kiocb *req, u64 sqe_addr) · 2 unmodified lines
6765	7183	return 0;
6766	7184
6767	7185	spin_lock(&ctx->completion_lock);
6768		− ret = io_poll_cancel(ctx, sqe_addr, false);
	7186	+ ret = io_poll_cancel(ctx, cd);
6769	7187	if (ret != -ENOENT)
6770	7188	goto out;
6771		−
6772		− spin_lock_irq(&ctx->timeout_lock);
6773		− ret = io_timeout_cancel(ctx, sqe_addr);
6774		− spin_unlock_irq(&ctx->timeout_lock);
	7189	+ if (!(cd->flags & IORING_ASYNC_CANCEL_FD))
	7190	+ ret = io_timeout_cancel(ctx, cd);
6775	7191	out:
6776	7192	spin_unlock(&ctx->completion_lock);
6777	7193	return ret;
6778	7194	}
6779	7195
	7196	+#define CANCEL_FLAGS (IORING_ASYNC_CANCEL_ALL \| IORING_ASYNC_CANCEL_FD \| \
	7197	+ IORING_ASYNC_CANCEL_ANY)
	7198	+
6780	7199	static int io_async_cancel_prep(struct io_kiocb *req,
6781	7200	const struct io_uring_sqe *sqe)
6782	7201	{
6783		− if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
6784		− return -EINVAL;
6785		− if (unlikely(req->flags & (REQ_F_FIXED_FILE \| REQ_F_BUFFER_SELECT)))
	7202	+ if (unlikely(req->flags & REQ_F_BUFFER_SELECT))
6786	7203	return -EINVAL;
6787		− if (sqe->ioprio \|\| sqe->off \|\| sqe->len \|\| sqe->cancel_flags \|\|
6788		− sqe->splice_fd_in)
	7204	+ if (sqe->off \|\| sqe->len \|\| sqe->splice_fd_in)
6789	7205	return -EINVAL;
6790	7206
6791	7207	req->cancel.addr = READ_ONCE(sqe->addr);
	7208	+ req->cancel.flags = READ_ONCE(sqe->cancel_flags);
	7209	+ if (req->cancel.flags & ~CANCEL_FLAGS)
	7210	+ return -EINVAL;
	7211	+ if (req->cancel.flags & IORING_ASYNC_CANCEL_FD) {
	7212	+ if (req->cancel.flags & IORING_ASYNC_CANCEL_ANY)
	7213	+ return -EINVAL;
	7214	+ req->cancel.fd = READ_ONCE(sqe->fd);
	7215	+ }
	7216	+
6792	7217	return 0;
6793	7218	}
6794	7219
6795		−static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
	7220	+static int __io_async_cancel(struct io_cancel_data cd, struct io_kiocb req,
	7221	+ unsigned int issue_flags)
6796	7222	{
6797		− struct io_ring_ctx *ctx = req->ctx;
6798		− u64 sqe_addr = req->cancel.addr;
6799		− bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
	7223	+ bool all = cd->flags & (IORING_ASYNC_CANCEL_ALL\|IORING_ASYNC_CANCEL_ANY);
	7224	+ struct io_ring_ctx *ctx = cd->ctx;
6800	7225	struct io_tctx_node *node;
6801		− int ret;
	7226	+ int ret, nr = 0;
6802	7227
6803		− ret = io_try_cancel_userdata(req, sqe_addr);
6804		− if (ret != -ENOENT)
6805		− goto done;
	7228	+ do {
	7229	+ ret = io_try_cancel(req, cd);
	7230	+ if (ret == -ENOENT)
	7231	+ break;
	7232	+ if (!all)
	7233	+ return ret;
	7234	+ nr++;
	7235	+ } while (1);
6806	7236
6807	7237	/* slow path, try all io-wq's */
6808		− io_ring_submit_lock(ctx, needs_lock);
	7238	+ io_ring_submit_lock(ctx, issue_flags);
6809	7239	ret = -ENOENT;
6810	7240	list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
6811	7241	struct io_uring_task *tctx = node->task->io_uring;
6812	7242
6813		− ret = io_async_cancel_one(tctx, req->cancel.addr, ctx);
6814		− if (ret != -ENOENT)
6815		− break;
	7243	+ ret = io_async_cancel_one(tctx, cd);
	7244	+ if (ret != -ENOENT) {
	7245	+ if (!all)
	7246	+ break;
	7247	+ nr++;
	7248	+ }
	7249	+ }
	7250	+ io_ring_submit_unlock(ctx, issue_flags);
	7251	+ return all ? nr : ret;
	7252	+}
	7253	+
	7254	+static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
	7255	+{
	7256	+ struct io_cancel_data cd = {
	7257	+ .ctx = req->ctx,
	7258	+ .data = req->cancel.addr,
	7259	+ .flags = req->cancel.flags,
	7260	+ .seq = atomic_inc_return(&req->ctx->cancel_seq),
	7261	+ };
	7262	+ int ret;
	7263	+
	7264	+ if (cd.flags & IORING_ASYNC_CANCEL_FD) {
	7265	+ if (req->flags & REQ_F_FIXED_FILE)
	7266	+ req->file = io_file_get_fixed(req, req->cancel.fd,
	7267	+ issue_flags);
	7268	+ else
	7269	+ req->file = io_file_get_normal(req, req->cancel.fd);
	7270	+ if (!req->file) {
	7271	+ ret = -EBADF;
	7272	+ goto done;
	7273	+ }
	7274	+ cd.file = req->file;
6816	7275	}
6817		− io_ring_submit_unlock(ctx, needs_lock);
	7276	+
	7277	+ ret = __io_async_cancel(&cd, req, issue_flags);
6818	7278	done:
6819	7279	if (ret < 0)
6820	7280	req_set_fail(req);
@@ -6827,7 +7287,7 @@ static int io_rsrc_update_prep(struct io_kiocb *req, · 6 unmodified lines
6827	7287	{
6828	7288	if (unlikely(req->flags & (REQ_F_FIXED_FILE \| REQ_F_BUFFER_SELECT)))
6829	7289	return -EINVAL;
6830		− if (sqe->ioprio \|\| sqe->rw_flags \|\| sqe->splice_fd_in)
	7290	+ if (sqe->rw_flags \|\| sqe->splice_fd_in)
6831	7291	return -EINVAL;
6832	7292
6833	7293	req->rsrc_update.offset = READ_ONCE(sqe->off);
@@ -6841,7 +7301,6 @@ static int io_rsrc_update_prep(struct io_kiocb *req, · 7 unmodified lines
6841	7301	static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
6842	7302	{
6843	7303	struct io_ring_ctx *ctx = req->ctx;
6844		− bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
6845	7304	struct io_uring_rsrc_update2 up;
6846	7305	int ret;
6847	7306
@@ -6852,10 +7311,10 @@ static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) · 4 unmodified lines
6852	7311	up.resv = 0;
6853	7312	up.resv2 = 0;
6854	7313
6855		− io_ring_submit_lock(ctx, needs_lock);
	7314	+ io_ring_submit_lock(ctx, issue_flags);
6856	7315	ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE,
6857	7316	&up, req->rsrc_update.nr_args);
6858		− io_ring_submit_unlock(ctx, needs_lock);
	7317	+ io_ring_submit_unlock(ctx, issue_flags);
6859	7318
6860	7319	if (ret < 0)
6861	7320	req_set_fail(req);
@@ -6954,7 +7413,7 @@ static int io_req_prep_async(struct io_kiocb *req) · 92 unmodified lines
6954	7413
6955	7414	/* assign early for deferred execution for non-fixed file */
6956	7415	if (def->needs_file && !(req->flags & REQ_F_FIXED_FILE))
6957		− req->file = io_file_get_normal(req, req->fd);
	7416	+ req->file = io_file_get_normal(req, req->cqe.fd);
6958	7417	if (!def->needs_async_setup)
6959	7418	return 0;
6960	7419	if (WARN_ON_ONCE(req_has_async_data(req)))
@@ -6982,9 +7441,10 @@ static int io_req_prep_async(struct io_kiocb *req) · 21 unmodified lines
6982	7441	static u32 io_get_sequence(struct io_kiocb *req)
6983	7442	{
6984	7443	u32 seq = req->ctx->cached_sq_head;
	7444	+ struct io_kiocb *cur;
6985	7445
6986	7446	/* need original cached_sq_head, but it was increased for each req */
6987		− io_for_each_link(req, req)
	7447	+ io_for_each_link(cur, req)
6988	7448	seq--;
6989	7449	return seq;
6990	7450	}
@@ -7027,7 +7487,7 @@ fail: · 36 unmodified lines
7027	7487	goto queue;
7028	7488	}
7029	7489
7030		− trace_io_uring_defer(ctx, req, req->user_data, req->opcode);
	7490	+ trace_io_uring_defer(ctx, req, req->cqe.user_data, req->opcode);
7031	7491	de->req = req;
7032	7492	de->seq = seq;
7033	7493	list_add_tail(&de->list, &ctx->defer_list);
@@ -7111,15 +7571,11 @@ static bool io_assign_file(struct io_kiocb *req, unsigned int issue_flags) · 77 unmodified lines
7111	7571	return true;
7112	7572
7113	7573	if (req->flags & REQ_F_FIXED_FILE)
7114		− req->file = io_file_get_fixed(req, req->fd, issue_flags);
	7574	+ req->file = io_file_get_fixed(req, req->cqe.fd, issue_flags);
7115	7575	else
7116		− req->file = io_file_get_normal(req, req->fd);
7117		− if (req->file)
7118		− return true;
	7576	+ req->file = io_file_get_normal(req, req->cqe.fd);
7119	7577
7120		− req_set_fail(req);
7121		− req->result = -EBADF;
7122		− return false;
	7578	+ return !!req->file;
7123	7579	}
7124	7580
7125	7581	static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
@@ -7282,7 +7738,6 @@ static void io_wq_submit_work(struct io_wq_work *work) · 156 unmodified lines
7282	7738	const struct io_op_def *def = &io_op_defs[req->opcode];
7283	7739	unsigned int issue_flags = IO_URING_F_UNLOCKED;
7284	7740	bool needs_poll = false;
7285		− struct io_kiocb *timeout;
7286	7741	int ret = 0, err = -ECANCELED;
7287	7742
7288	7743	/* one will be dropped by ->io_free_work() after returning to io-wq */
@@ -7291,10 +7746,7 @@ static void io_wq_submit_work(struct io_wq_work *work) · 2 unmodified lines
7291	7746	else
7292	7747	req_ref_get(req);
7293	7748
7294		− timeout = io_prep_linked_timeout(req);
7295		− if (timeout)
7296		− io_queue_linked_timeout(timeout);
7297		−
	7749	+ io_arm_ltimeout(req);
7298	7750
7299	7751	/* either cancelled or io-wq is dying, so don't touch tctx->iowq */
7300	7752	if (work->flags & IO_WQ_WORK_CANCEL) {
@@ -7327,6 +7779,8 @@ fail: · 26 unmodified lines
7327	7779	* wait for request slots on the block side.
7328	7780	*/
7329	7781	if (!needs_poll) {
	7782	+ if (!(req->ctx->flags & IORING_SETUP_IOPOLL))
	7783	+ break;
7330	7784	cond_resched();
7331	7785	continue;
7332	7786	}
@@ -7372,8 +7826,7 @@ static inline struct file io_file_get_fixed(struct io_kiocb req, int fd, · 39 unmodified lines
7372	7826	struct file *file = NULL;
7373	7827	unsigned long file_ptr;
7374	7828
7375		− if (issue_flags & IO_URING_F_UNLOCKED)
7376		− mutex_lock(&ctx->uring_lock);
	7829	+ io_ring_submit_lock(ctx, issue_flags);
7377	7830
7378	7831	if (unlikely((unsigned int)fd >= ctx->nr_user_files))
7379	7832	goto out;
@@ -7384,9 +7837,9 @@ static inline struct file io_file_get_fixed(struct io_kiocb req, int fd, · 4 unmodified lines
7384	7837	/* mask in overlapping REQ_F and FFS bits */
7385	7838	req->flags \|= (file_ptr << REQ_F_SUPPORT_NOWAIT_BIT);
7386	7839	io_req_set_rsrc_node(req, ctx, 0);
	7840	+ WARN_ON_ONCE(file && !test_bit(fd, ctx->file_table.bitmap));
7387	7841	out:
7388		− if (issue_flags & IO_URING_F_UNLOCKED)
7389		− mutex_unlock(&ctx->uring_lock);
	7842	+ io_ring_submit_unlock(ctx, issue_flags);
7390	7843	return file;
7391	7844	}
7392	7845
@@ -7407,7 +7860,7 @@ static struct file io_file_get_normal(struct io_kiocb req, int fd) · 14 unmodified lines
7407	7860	{
7408	7861	struct file *file = fget(fd);
7409	7862
7410		− trace_io_uring_file_get(req->ctx, req, req->user_data, fd);
	7863	+ trace_io_uring_file_get(req->ctx, req, req->cqe.user_data, fd);
7411	7864
7412	7865	/* we don't allow fixed io_uring files */
7413	7866	if (file && file->f_op == &io_uring_fops)
@@ -7421,8 +7874,14 @@ static void io_req_task_link_timeout(struct io_kiocb req, bool locked) · 7 unmodified lines
7421	7874	int ret = -ENOENT;
7422	7875
7423	7876	if (prev) {
7424		− if (!(req->task->flags & PF_EXITING))
7425		− ret = io_try_cancel_userdata(req, prev->user_data);
	7877	+ if (!(req->task->flags & PF_EXITING)) {
	7878	+ struct io_cancel_data cd = {
	7879	+ .ctx = req->ctx,
	7880	+ .data = prev->cqe.user_data,
	7881	+ };
	7882	+
	7883	+ ret = io_try_cancel(req, &cd);
	7884	+ }
7426	7885	io_req_complete_post(req, ret ?: -ETIME, 0);
7427	7886	io_put_req(prev);
7428	7887	} else {
@@ -7482,10 +7941,17 @@ static void io_queue_linked_timeout(struct io_kiocb *req) · 53 unmodified lines
7482	7941	io_put_req(req);
7483	7942	}
7484	7943
7485		−static void io_queue_sqe_arm_apoll(struct io_kiocb *req)
	7944	+static void io_queue_async(struct io_kiocb *req, int ret)
7486	7945	__must_hold(&req->ctx->uring_lock)
7487	7946	{
7488		− struct io_kiocb *linked_timeout = io_prep_linked_timeout(req);
	7947	+ struct io_kiocb *linked_timeout;
	7948	+
	7949	+ if (ret != -EAGAIN \|\| (req->flags & REQ_F_NOWAIT)) {
	7950	+ io_req_complete_failed(req, ret);
	7951	+ return;
	7952	+ }
	7953	+
	7954	+ linked_timeout = io_prep_linked_timeout(req);
7489	7955
7490	7956	switch (io_arm_poll_handler(req, 0)) {
7491	7957	case IO_APOLL_READY:
@@ -7496,7 +7962,7 @@ static void io_queue_sqe_arm_apoll(struct io_kiocb *req) · 4 unmodified lines
7496	7962	* Queued up for async execution, worker will release
7497	7963	* submit reference when the iocb is actually submitted.
7498	7964	*/
7499		− io_queue_async_work(req, NULL);
	7965	+ io_queue_iowq(req, NULL);
7500	7966	break;
7501	7967	case IO_APOLL_OK:
7502	7968	break;
@@ -7506,10 +7972,9 @@ static void io_queue_sqe_arm_apoll(struct io_kiocb *req) · 3 unmodified lines
7506	7972	io_queue_linked_timeout(linked_timeout);
7507	7973	}
7508	7974
7509		−static inline void __io_queue_sqe(struct io_kiocb *req)
	7975	+static inline void io_queue_sqe(struct io_kiocb *req)
7510	7976	__must_hold(&req->ctx->uring_lock)
7511	7977	{
7512		− struct io_kiocb *linked_timeout;
7513	7978	int ret;
7514	7979
7515	7980	ret = io_issue_sqe(req, IO_URING_F_NONBLOCK\|IO_URING_F_COMPLETE_DEFER);
@@ -7522,22 +7987,23 @@ static inline void __io_queue_sqe(struct io_kiocb *req) · 6 unmodified lines
7522	7987	* We async punt it if the file wasn't marked NOWAIT, or if the file
7523	7988	* doesn't support non-blocking read/write attempts
7524	7989	*/
7525		− if (likely(!ret)) {
7526		− linked_timeout = io_prep_linked_timeout(req);
7527		− if (linked_timeout)
7528		− io_queue_linked_timeout(linked_timeout);
7529		− } else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
7530		− io_queue_sqe_arm_apoll(req);
7531		− } else {
7532		− io_req_complete_failed(req, ret);
7533		− }
	7990	+ if (likely(!ret))
	7991	+ io_arm_ltimeout(req);
	7992	+ else
	7993	+ io_queue_async(req, ret);
7534	7994	}
7535	7995
7536	7996	static void io_queue_sqe_fallback(struct io_kiocb *req)
7537	7997	__must_hold(&req->ctx->uring_lock)
7538	7998	{
7539		− if (req->flags & REQ_F_FAIL) {
7540		− io_req_complete_fail_submit(req);
	7999	+ if (unlikely(req->flags & REQ_F_FAIL)) {
	8000	+ /*
	8001	+ * We don't submit, fail them all, for that replace hardlinks
	8002	+ * with normal links. Extra REQ_F_LINK is tolerated.
	8003	+ */
	8004	+ req->flags &= ~REQ_F_HARDLINK;
	8005	+ req->flags \|= REQ_F_LINK;
	8006	+ io_req_complete_failed(req, req->cqe.res);
7541	8007	} else if (unlikely(req->ctx->drain_active)) {
7542	8008	io_drain_req(req);
7543	8009	} else {
@@ -7546,19 +8012,10 @@ static void io_queue_sqe_fallback(struct io_kiocb *req) · 2 unmodified lines
7546	8012	if (unlikely(ret))
7547	8013	io_req_complete_failed(req, ret);
7548	8014	else
7549		− io_queue_async_work(req, NULL);
	8015	+ io_queue_iowq(req, NULL);
7550	8016	}
7551	8017	}
7552	8018
7553		−static inline void io_queue_sqe(struct io_kiocb *req)
7554		− __must_hold(&req->ctx->uring_lock)
7555		−{
7556		− if (likely(!(req->flags & (REQ_F_FORCE_ASYNC \| REQ_F_FAIL))))
7557		− __io_queue_sqe(req);
7558		− else
7559		− io_queue_sqe_fallback(req);
7560		−}
7561		−
7562	8019	/*
7563	8020	* Check SQE restrictions (opcode and flags).
7564	8021	*
@@ -7613,9 +8070,9 @@ static int io_init_req(struct io_ring_ctx ctx, struct io_kiocb req, · 48 unmodified lines
7613	8070	req->opcode = opcode = READ_ONCE(sqe->opcode);
7614	8071	/* same numerical values with corresponding REQ_F_, safe to copy /
7615	8072	req->flags = sqe_flags = READ_ONCE(sqe->flags);
7616		− req->user_data = READ_ONCE(sqe->user_data);
	8073	+ req->cqe.user_data = READ_ONCE(sqe->user_data);
7617	8074	req->file = NULL;
7618		− req->fixed_rsrc_refs = NULL;
	8075	+ req->rsrc_node = NULL;
7619	8076	req->task = current;
7620	8077
7621	8078	if (unlikely(opcode >= IORING_OP_LAST)) {
@@ -7626,9 +8083,11 @@ static int io_init_req(struct io_ring_ctx ctx, struct io_kiocb req, · 4 unmodified lines
7626	8083	/* enforce forwards compatibility on users */
7627	8084	if (sqe_flags & ~SQE_VALID_FLAGS)
7628	8085	return -EINVAL;
7629		− if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
7630		− !io_op_defs[opcode].buffer_select)
7631		− return -EOPNOTSUPP;
	8086	+ if (sqe_flags & IOSQE_BUFFER_SELECT) {
	8087	+ if (!io_op_defs[opcode].buffer_select)
	8088	+ return -EOPNOTSUPP;
	8089	+ req->buf_index = READ_ONCE(sqe->buf_group);
	8090	+ }
7632	8091	if (sqe_flags & IOSQE_CQE_SKIP_SUCCESS)
7633	8092	ctx->drain_disabled = true;
7634	8093	if (sqe_flags & IOSQE_IO_DRAIN) {
@@ -7651,10 +8110,15 @@ static int io_init_req(struct io_ring_ctx ctx, struct io_kiocb req, · 16 unmodified lines
7651	8110	}
7652	8111	}
7653	8112
	8113	+ if (!io_op_defs[opcode].ioprio && sqe->ioprio)
	8114	+ return -EINVAL;
	8115	+ if (!io_op_defs[opcode].iopoll && (ctx->flags & IORING_SETUP_IOPOLL))
	8116	+ return -EINVAL;
	8117	+
7654	8118	if (io_op_defs[opcode].needs_file) {
7655	8119	struct io_submit_state *state = &ctx->submit_state;
7656	8120
7657		− req->fd = READ_ONCE(sqe->fd);
	8121	+ req->cqe.fd = READ_ONCE(sqe->fd);
7658	8122
7659	8123	/*
7660	8124	* Plug now if we have more than 2 IO left after this, and the
@@ -7686,7 +8150,44 @@ static int io_init_req(struct io_ring_ctx ctx, struct io_kiocb req, · 25 unmodified lines
7686	8150	return io_req_prep(req, sqe);
7687	8151	}
7688	8152
7689		−static int io_submit_sqe(struct io_ring_ctx ctx, struct io_kiocb req,
	8153	+static __cold int io_submit_fail_init(const struct io_uring_sqe *sqe,
	8154	+ struct io_kiocb *req, int ret)
	8155	+{
	8156	+ struct io_ring_ctx *ctx = req->ctx;
	8157	+ struct io_submit_link *link = &ctx->submit_state.link;
	8158	+ struct io_kiocb *head = link->head;
	8159	+
	8160	+ trace_io_uring_req_failed(sqe, ctx, req, ret);
	8161	+
	8162	+ /*
	8163	+ * Avoid breaking links in the middle as it renders links with SQPOLL
	8164	+ * unusable. Instead of failing eagerly, continue assembling the link if
	8165	+ * applicable and mark the head with REQ_F_FAIL. The link flushing code
	8166	+ * should find the flag and handle the rest.
	8167	+ */
	8168	+ req_fail_link_node(req, ret);
	8169	+ if (head && !(head->flags & REQ_F_FAIL))
	8170	+ req_fail_link_node(head, -ECANCELED);
	8171	+
	8172	+ if (!(req->flags & IO_REQ_LINK_FLAGS)) {
	8173	+ if (head) {
	8174	+ link->last->link = req;
	8175	+ link->head = NULL;
	8176	+ req = head;
	8177	+ }
	8178	+ io_queue_sqe_fallback(req);
	8179	+ return ret;
	8180	+ }
	8181	+
	8182	+ if (head)
	8183	+ link->last->link = req;
	8184	+ else
	8185	+ link->head = req;
	8186	+ link->last = req;
	8187	+ return 0;
	8188	+}
	8189	+
	8190	+static inline int io_submit_sqe(struct io_ring_ctx ctx, struct io_kiocb req,
7690	8191	const struct io_uring_sqe *sqe)
7691	8192	__must_hold(&ctx->uring_lock)
7692	8193	{
@@ -7694,35 +8195,11 @@ static int io_submit_sqe(struct io_ring_ctx ctx, struct io_kiocb req, · 1 unmodified line
7694	8195	int ret;
7695	8196
7696	8197	ret = io_init_req(ctx, req, sqe);
7697		− if (unlikely(ret)) {
7698		− trace_io_uring_req_failed(sqe, ctx, req, ret);
7699		−
7700		− /* fail even hard links since we don't submit */
7701		− if (link->head) {
7702		− /*
7703		− * we can judge a link req is failed or cancelled by if
7704		− * REQ_F_FAIL is set, but the head is an exception since
7705		− * it may be set REQ_F_FAIL because of other req's failure
7706		− * so let's leverage req->result to distinguish if a head
7707		− * is set REQ_F_FAIL because of its failure or other req's
7708		− * failure so that we can set the correct ret code for it.
7709		− * init result here to avoid affecting the normal path.
7710		− */
7711		− if (!(link->head->flags & REQ_F_FAIL))
7712		− req_fail_link_node(link->head, -ECANCELED);
7713		− } else if (!(req->flags & (REQ_F_LINK \| REQ_F_HARDLINK))) {
7714		− /*
7715		− * the current req is a normal req, we should return
7716		− * error and thus break the submittion loop.
7717		− */
7718		− io_req_complete_failed(req, ret);
7719		− return ret;
7720		− }
7721		− req_fail_link_node(req, ret);
7722		− }
	8198	+ if (unlikely(ret))
	8199	+ return io_submit_fail_init(sqe, req, ret);
7723	8200
7724	8201	/* don't need @sqe from now on */
7725		− trace_io_uring_submit_sqe(ctx, req, req->user_data, req->opcode,
	8202	+ trace_io_uring_submit_sqe(ctx, req, req->cqe.user_data, req->opcode,
7726	8203	req->flags, true,
7727	8204	ctx->flags & IORING_SETUP_SQPOLL);
7728	8205
@@ -7733,29 +8210,32 @@ static int io_submit_sqe(struct io_ring_ctx ctx, struct io_kiocb req, · 4 unmodified lines
7733	8210	* submitted sync once the chain is complete. If none of those
7734	8211	* conditions are true (normal request), then just queue it.
7735	8212	*/
7736		− if (link->head) {
7737		− struct io_kiocb *head = link->head;
7738		−
7739		− if (!(req->flags & REQ_F_FAIL)) {
7740		− ret = io_req_prep_async(req);
7741		− if (unlikely(ret)) {
7742		− req_fail_link_node(req, ret);
7743		− if (!(head->flags & REQ_F_FAIL))
7744		− req_fail_link_node(head, -ECANCELED);
7745		− }
7746		− }
7747		− trace_io_uring_link(ctx, req, head);
	8213	+ if (unlikely(link->head)) {
	8214	+ ret = io_req_prep_async(req);
	8215	+ if (unlikely(ret))
	8216	+ return io_submit_fail_init(sqe, req, ret);
	8217	+
	8218	+ trace_io_uring_link(ctx, req, link->head);
7748	8219	link->last->link = req;
7749	8220	link->last = req;
7750	8221
7751		− if (req->flags & (REQ_F_LINK \| REQ_F_HARDLINK))
	8222	+ if (req->flags & IO_REQ_LINK_FLAGS)
7752	8223	return 0;
7753		− /* last request of a link, enqueue the link */
	8224	+ /* last request of the link, flush it */
	8225	+ req = link->head;
7754	8226	link->head = NULL;
7755		− req = head;
7756		− } else if (req->flags & (REQ_F_LINK \| REQ_F_HARDLINK)) {
7757		− link->head = req;
7758		− link->last = req;
	8227	+ if (req->flags & (REQ_F_FORCE_ASYNC \| REQ_F_FAIL))
	8228	+ goto fallback;
	8229	+
	8230	+ } else if (unlikely(req->flags & (IO_REQ_LINK_FLAGS \|
	8231	+ REQ_F_FORCE_ASYNC \| REQ_F_FAIL))) {
	8232	+ if (req->flags & IO_REQ_LINK_FLAGS) {
	8233	+ link->head = req;
	8234	+ link->last = req;
	8235	+ } else {
	8236	+fallback:
	8237	+ io_queue_sqe_fallback(req);
	8238	+ }
7759	8239	return 0;
7760	8240	}
7761	8241
@@ -7770,8 +8250,8 @@ static void io_submit_state_end(struct io_ring_ctx *ctx) · 8 unmodified lines
7770	8250	{
7771	8251	struct io_submit_state *state = &ctx->submit_state;
7772	8252
7773		− if (state->link.head)
7774		− io_queue_sqe(state->link.head);
	8253	+ if (unlikely(state->link.head))
	8254	+ io_queue_sqe_fallback(state->link.head);
7775	8255	/* flush only after queuing links as they can generate completions */
7776	8256	io_submit_flush_completions(ctx);
7777	8257	if (state->plug_started)
@@ -7839,54 +8319,52 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) · 61 unmodified lines
7839	8319	__must_hold(&ctx->uring_lock)
7840	8320	{
7841	8321	unsigned int entries = io_sqring_entries(ctx);
7842		− int submitted = 0;
	8322	+ unsigned int left;
	8323	+ int ret;
7843	8324
7844	8325	if (unlikely(!entries))
7845	8326	return 0;
7846	8327	/* make sure SQ entry isn't read before tail */
7847		− nr = min3(nr, ctx->sq_entries, entries);
7848		− io_get_task_refs(nr);
	8328	+ ret = left = min3(nr, ctx->sq_entries, entries);
	8329	+ io_get_task_refs(left);
	8330	+ io_submit_state_start(&ctx->submit_state, left);
7849	8331
7850		− io_submit_state_start(&ctx->submit_state, nr);
7851	8332	do {
7852	8333	const struct io_uring_sqe *sqe;
7853	8334	struct io_kiocb *req;
7854	8335
7855		− if (unlikely(!io_alloc_req_refill(ctx))) {
7856		− if (!submitted)
7857		− submitted = -EAGAIN;
	8336	+ if (unlikely(!io_alloc_req_refill(ctx)))
7858	8337	break;
7859		− }
7860	8338	req = io_alloc_req(ctx);
7861	8339	sqe = io_get_sqe(ctx);
7862	8340	if (unlikely(!sqe)) {
7863		− wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list);
	8341	+ io_req_add_to_cache(req, ctx);
7864	8342	break;
7865	8343	}
7866		− /* will complete beyond this point, count as submitted */
7867		− submitted++;
7868		− if (io_submit_sqe(ctx, req, sqe)) {
7869		− /*
7870		− * Continue submitting even for sqe failure if the
7871		− * ring was setup with IORING_SETUP_SUBMIT_ALL
7872		− */
7873		− if (!(ctx->flags & IORING_SETUP_SUBMIT_ALL))
7874		− break;
7875		− }
7876		− } while (submitted < nr);
7877	8344
7878		− if (unlikely(submitted != nr)) {
7879		− int ref_used = (submitted == -EAGAIN) ? 0 : submitted;
7880		− int unused = nr - ref_used;
	8345	+ /*
	8346	+ * Continue submitting even for sqe failure if the
	8347	+ * ring was setup with IORING_SETUP_SUBMIT_ALL
	8348	+ */
	8349	+ if (unlikely(io_submit_sqe(ctx, req, sqe)) &&
	8350	+ !(ctx->flags & IORING_SETUP_SUBMIT_ALL)) {
	8351	+ left--;
	8352	+ break;
	8353	+ }
	8354	+ } while (--left);
7881	8355
7882		− current->io_uring->cached_refs += unused;
	8356	+ if (unlikely(left)) {
	8357	+ ret -= left;
	8358	+ /* try again if it submitted nothing and can't allocate a req */
	8359	+ if (!ret && io_req_cache_empty(ctx))
	8360	+ ret = -EAGAIN;
	8361	+ current->io_uring->cached_refs += left;
7883	8362	}
7884	8363
7885	8364	io_submit_state_end(ctx);
7886	8365	/* Commit SQ ring head once we've consumed and submitted all SQEs */
7887	8366	io_commit_sqring(ctx);
7888		−
7889		− return submitted;
	8367	+ return ret;
7890	8368	}
7891	8369
7892	8370	static inline bool io_sqd_events_pending(struct io_sq_data *sqd)
@@ -7894,23 +8372,6 @@ static inline bool io_sqd_events_pending(struct io_sq_data *sqd) · 1 unmodified line
7894	8372	return READ_ONCE(sqd->state);
7895	8373	}
7896	8374
7897		−static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
7898		−{
7899		− /* Tell userspace we may need a wakeup call */
7900		− spin_lock(&ctx->completion_lock);
7901		− WRITE_ONCE(ctx->rings->sq_flags,
7902		− ctx->rings->sq_flags \| IORING_SQ_NEED_WAKEUP);
7903		− spin_unlock(&ctx->completion_lock);
7904		−}
7905		−
7906		−static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
7907		−{
7908		− spin_lock(&ctx->completion_lock);
7909		− WRITE_ONCE(ctx->rings->sq_flags,
7910		− ctx->rings->sq_flags & ~IORING_SQ_NEED_WAKEUP);
7911		− spin_unlock(&ctx->completion_lock);
7912		−}
7913		−
7914	8375	static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)
7915	8376	{
7916	8377	unsigned int to_submit;
@@ -8026,8 +8487,8 @@ static int io_sq_thread(void *data) · 109 unmodified lines
8026	8487	bool needs_sched = true;
8027	8488
8028	8489	list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
8029		− io_ring_set_wakeup_flag(ctx);
8030		−
	8490	+ atomic_or(IORING_SQ_NEED_WAKEUP,
	8491	+ &ctx->rings->sq_flags);
8031	8492	if ((ctx->flags & IORING_SETUP_IOPOLL) &&
8032	8493	!wq_list_empty(&ctx->iopoll_list)) {
8033	8494	needs_sched = false;
@@ -8038,7 +8499,7 @@ static int io_sq_thread(void *data) · 4 unmodified lines
8038	8499	* Ensure the store of the wakeup flag is not
8039	8500	* reordered with the load of the SQ tail
8040	8501	*/
8041		− smp_mb();
	8502	+ smp_mb__after_atomic();
8042	8503
8043	8504	if (io_sqring_entries(ctx)) {
8044	8505	needs_sched = false;
@@ -8052,7 +8513,8 @@ static int io_sq_thread(void *data) · 7 unmodified lines
8052	8513	mutex_lock(&sqd->lock);
8053	8514	}
8054	8515	list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
8055		− io_ring_clear_wakeup_flag(ctx);
	8516	+ atomic_andnot(IORING_SQ_NEED_WAKEUP,
	8517	+ &ctx->rings->sq_flags);
8056	8518	}
8057	8519
8058	8520	finish_wait(&sqd->wait, &wait);
@@ -8062,7 +8524,7 @@ static int io_sq_thread(void *data) · 3 unmodified lines
8062	8524	io_uring_cancel_generic(true, sqd);
8063	8525	sqd->thread = NULL;
8064	8526	list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
8065		− io_ring_set_wakeup_flag(ctx);
	8527	+ atomic_or(IORING_SQ_NEED_WAKEUP, &ctx->rings->sq_flags);
8066	8528	io_run_task_work();
8067	8529	mutex_unlock(&sqd->lock);
8068	8530
@@ -8102,7 +8564,8 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, · 33 unmodified lines
8102	8564	* Cannot safely flush overflowed CQEs from here, ensure we wake up
8103	8565	* the task, and the next invocation will do it.
8104	8566	*/
8105		− if (io_should_wake(iowq) \|\| test_bit(0, &iowq->ctx->check_cq_overflow))
	8567	+ if (io_should_wake(iowq) \|\|
	8568	+ test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &iowq->ctx->check_cq))
8106	8569	return autoremove_wake_function(curr, mode, wake_flags, key);
8107	8570	return -1;
8108	8571	}
@@ -8124,15 +8587,18 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, · 15 unmodified lines
8124	8587	ktime_t timeout)
8125	8588	{
8126	8589	int ret;
	8590	+ unsigned long check_cq;
8127	8591
8128	8592	/* make sure we run task_work before checking for signals */
8129	8593	ret = io_run_task_work_sig();
8130	8594	if (ret \|\| io_should_wake(iowq))
8131	8595	return ret;
	8596	+ check_cq = READ_ONCE(ctx->check_cq);
8132	8597	/* let the caller flush overflows, retry */
8133		− if (test_bit(0, &ctx->check_cq_overflow))
	8598	+ if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
8134	8599	return 1;
8135		−
	8600	+ if (unlikely(check_cq & BIT(IO_CHECK_CQ_DROPPED_BIT)))
	8601	+ return -EBADR;
8136	8602	if (!schedule_hrtimeout(&timeout, HRTIMER_MODE_ABS))
8137	8603	return -ETIME;
8138	8604	return 1;
@@ -8197,10 +8663,10 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 58 unmodified lines
8197	8663	prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq,
8198	8664	TASK_INTERRUPTIBLE);
8199	8665	ret = io_cqring_wait_schedule(ctx, &iowq, timeout);
8200		− finish_wait(&ctx->cq_wait, &iowq.wq);
8201	8666	cond_resched();
8202	8667	} while (ret > 0);
8203	8668
	8669	+ finish_wait(&ctx->cq_wait, &iowq.wq);
8204	8670	restore_saved_sigmask_unless(ret == -EINTR);
8205	8671
8206	8672	return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
@@ -8438,17 +8904,57 @@ static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files) · 231 unmodified lines
8438	8904	{
8439	8905	table->files = kvcalloc(nr_files, sizeof(table->files[0]),
8440	8906	GFP_KERNEL_ACCOUNT);
8441		− return !!table->files;
	8907	+ if (unlikely(!table->files))
	8908	+ return false;
	8909	+
	8910	+ table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT);
	8911	+ if (unlikely(!table->bitmap)) {
	8912	+ kvfree(table->files);
	8913	+ return false;
	8914	+ }
	8915	+
	8916	+ return true;
8442	8917	}
8443	8918
8444	8919	static void io_free_file_tables(struct io_file_table *table)
8445	8920	{
8446	8921	kvfree(table->files);
	8922	+ bitmap_free(table->bitmap);
8447	8923	table->files = NULL;
	8924	+ table->bitmap = NULL;
	8925	+}
	8926	+
	8927	+static inline void io_file_bitmap_set(struct io_file_table *table, int bit)
	8928	+{
	8929	+ WARN_ON_ONCE(test_bit(bit, table->bitmap));
	8930	+ __set_bit(bit, table->bitmap);
	8931	+ if (bit == table->alloc_hint)
	8932	+ table->alloc_hint++;
	8933	+}
	8934	+
	8935	+static inline void io_file_bitmap_clear(struct io_file_table *table, int bit)
	8936	+{
	8937	+ __clear_bit(bit, table->bitmap);
	8938	+ table->alloc_hint = bit;
8448	8939	}
8449	8940
8450	8941	static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
8451	8942	{
	8943	+#if !defined(IO_URING_SCM_ALL)
	8944	+ int i;
	8945	+
	8946	+ for (i = 0; i < ctx->nr_user_files; i++) {
	8947	+ struct file *file = io_file_from_index(ctx, i);
	8948	+
	8949	+ if (!file)
	8950	+ continue;
	8951	+ if (io_fixed_file_slot(&ctx->file_table, i)->file_ptr & FFS_SCM)
	8952	+ continue;
	8953	+ io_file_bitmap_clear(&ctx->file_table, i);
	8954	+ fput(file);
	8955	+ }
	8956	+#endif
	8957	+
8452	8958	#if defined(CONFIG_UNIX)
8453	8959	if (ctx->ring_sock) {
8454	8960	struct sock *sock = ctx->ring_sock->sk;
@@ -8457,16 +8963,6 @@ static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) · 2 unmodified lines
8457	8963	while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
8458	8964	kfree_skb(skb);
8459	8965	}
8460		−#else
8461		− int i;
8462		−
8463		− for (i = 0; i < ctx->nr_user_files; i++) {
8464		− struct file *file;
8465		−
8466		− file = io_file_from_index(ctx, i);
8467		− if (file)
8468		− fput(file);
8469		− }
8470	8966	#endif
8471	8967	io_free_file_tables(&ctx->file_table);
8472	8968	io_rsrc_data_free(ctx->file_data);
@@ -8611,107 +9107,66 @@ static struct io_sq_data io_get_sq_data(struct io_uring_params p, · 138 unmodified lines
8611	9107	return sqd;
8612	9108	}
8613	9109
8614		−#if defined(CONFIG_UNIX)
8615	9110	/*
8616	9111	* Ensure the UNIX gc is aware of our file set, so we are certain that
8617	9112	* the io_uring can be safely unregistered on process exit, even if we have
8618		− * loops in the file referencing.
	9113	+ * loops in the file referencing. We account only files that can hold other
	9114	+ * files because otherwise they can't form a loop and so are not interesting
	9115	+ * for GC.
8619	9116	*/
8620		−static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
	9117	+static int io_scm_file_account(struct io_ring_ctx ctx, struct file file)
8621	9118	{
	9119	+#if defined(CONFIG_UNIX)
8622	9120	struct sock *sk = ctx->ring_sock->sk;
	9121	+ struct sk_buff_head *head = &sk->sk_receive_queue;
8623	9122	struct scm_fp_list *fpl;
8624	9123	struct sk_buff *skb;
8625		− int i, nr_files;
8626		−
8627		− fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
8628		− if (!fpl)
8629		− return -ENOMEM;
8630		−
8631		− skb = alloc_skb(0, GFP_KERNEL);
8632		− if (!skb) {
8633		− kfree(fpl);
8634		− return -ENOMEM;
8635		− }
8636		−
8637		− skb->sk = sk;
8638	9124
8639		− nr_files = 0;
8640		− fpl->user = get_uid(current_user());
8641		− for (i = 0; i < nr; i++) {
8642		− struct file *file = io_file_from_index(ctx, i + offset);
8643		−
8644		− if (!file)
8645		− continue;
8646		− fpl->fp[nr_files] = get_file(file);
8647		− unix_inflight(fpl->user, fpl->fp[nr_files]);
8648		− nr_files++;
8649		− }
8650		−
8651		− if (nr_files) {
8652		− fpl->max = SCM_MAX_FD;
8653		− fpl->count = nr_files;
8654		− UNIXCB(skb).fp = fpl;
8655		− skb->destructor = unix_destruct_scm;
8656		− refcount_add(skb->truesize, &sk->sk_wmem_alloc);
8657		− skb_queue_head(&sk->sk_receive_queue, skb);
8658		−
8659		− for (i = 0; i < nr; i++) {
8660		− struct file *file = io_file_from_index(ctx, i + offset);
8661		−
8662		− if (file)
8663		− fput(file);
8664		− }
8665		− } else {
8666		− kfree_skb(skb);
8667		− free_uid(fpl->user);
8668		− kfree(fpl);
8669		− }
8670		−
8671		− return 0;
8672		−}
8673		−
8674		−/*
8675		− * If UNIX sockets are enabled, fd passing can cause a reference cycle which
8676		− * causes regular reference counting to break down. We rely on the UNIX
8677		− * garbage collection to take care of this problem for us.
8678		− */
8679		−static int io_sqe_files_scm(struct io_ring_ctx *ctx)
8680		−{
8681		− unsigned left, total;
8682		− int ret = 0;
	9125	+ if (likely(!io_file_need_scm(file)))
	9126	+ return 0;
8683	9127
8684		− total = 0;
8685		− left = ctx->nr_user_files;
8686		− while (left) {
8687		− unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
	9128	+ /*
	9129	+ * See if we can merge this file into an existing skb SCM_RIGHTS
	9130	+ * file set. If there's no room, fall back to allocating a new skb
	9131	+ * and filling it in.
	9132	+ */
	9133	+ spin_lock_irq(&head->lock);
	9134	+ skb = skb_peek(head);
	9135	+ if (skb && UNIXCB(skb).fp->count < SCM_MAX_FD)
	9136	+ __skb_unlink(skb, head);
	9137	+ else
	9138	+ skb = NULL;
	9139	+ spin_unlock_irq(&head->lock);
8688	9140
8689		− ret = __io_sqe_files_scm(ctx, this_files, total);
8690		− if (ret)
8691		− break;
8692		− left -= this_files;
8693		− total += this_files;
8694		− }
	9141	+ if (!skb) {
	9142	+ fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
	9143	+ if (!fpl)
	9144	+ return -ENOMEM;
8695	9145
8696		− if (!ret)
8697		− return 0;
	9146	+ skb = alloc_skb(0, GFP_KERNEL);
	9147	+ if (!skb) {
	9148	+ kfree(fpl);
	9149	+ return -ENOMEM;
	9150	+ }
8698	9151
8699		− while (total < ctx->nr_user_files) {
8700		− struct file *file = io_file_from_index(ctx, total);
	9152	+ fpl->user = get_uid(current_user());
	9153	+ fpl->max = SCM_MAX_FD;
	9154	+ fpl->count = 0;
8701	9155
8702		− if (file)
8703		− fput(file);
8704		− total++;
	9156	+ UNIXCB(skb).fp = fpl;
	9157	+ skb->sk = sk;
	9158	+ skb->destructor = unix_destruct_scm;
	9159	+ refcount_add(skb->truesize, &sk->sk_wmem_alloc);
8705	9160	}
8706	9161
8707		− return ret;
8708		−}
8709		−#else
8710		−static int io_sqe_files_scm(struct io_ring_ctx *ctx)
8711		−{
	9162	+ fpl = UNIXCB(skb).fp;
	9163	+ fpl->fp[fpl->count++] = get_file(file);
	9164	+ unix_inflight(fpl->user, file);
	9165	+ skb_queue_head(head, skb);
	9166	+ fput(file);
	9167	+#endif
8712	9168	return 0;
8713	9169	}
8714		−#endif
8715	9170
8716	9171	static void io_rsrc_file_put(struct io_ring_ctx ctx, struct io_rsrc_put prsrc)
8717	9172	{
@@ -8722,6 +9177,11 @@ static void io_rsrc_file_put(struct io_ring_ctx ctx, struct io_rsrc_put prsrc) · 4 unmodified lines
8722	9177	struct sk_buff *skb;
8723	9178	int i;
8724	9179
	9180	+ if (!io_file_need_scm(file)) {
	9181	+ fput(file);
	9182	+ return;
	9183	+ }
	9184	+
8725	9185	__skb_queue_head_init(&list);
8726	9186
8727	9187	/*
@@ -8786,15 +9246,17 @@ static void __io_rsrc_put_work(struct io_rsrc_node *ref_node) · 58 unmodified lines
8786	9246	list_del(&prsrc->list);
8787	9247
8788	9248	if (prsrc->tag) {
8789		− bool lock_ring = ctx->flags & IORING_SETUP_IOPOLL;
	9249	+ if (ctx->flags & IORING_SETUP_IOPOLL)
	9250	+ mutex_lock(&ctx->uring_lock);
8790	9251
8791		− io_ring_submit_lock(ctx, lock_ring);
8792	9252	spin_lock(&ctx->completion_lock);
8793	9253	io_fill_cqe_aux(ctx, prsrc->tag, 0, 0);
8794	9254	io_commit_cqring(ctx);
8795	9255	spin_unlock(&ctx->completion_lock);
8796	9256	io_cqring_ev_posted(ctx);
8797		− io_ring_submit_unlock(ctx, lock_ring);
	9257	+
	9258	+ if (ctx->flags & IORING_SETUP_IOPOLL)
	9259	+ mutex_unlock(&ctx->uring_lock);
8798	9260	}
8799	9261
8800	9262	rsrc_data->do_put(ctx, prsrc);
@@ -8848,27 +9310,31 @@ static int io_sqe_files_register(struct io_ring_ctx ctx, void __user arg, · 47 unmodified lines
8848	9310	if (ret)
8849	9311	return ret;
8850	9312
8851		− ret = -ENOMEM;
8852		− if (!io_alloc_file_tables(&ctx->file_table, nr_args))
8853		− goto out_free;
	9313	+ if (!io_alloc_file_tables(&ctx->file_table, nr_args)) {
	9314	+ io_rsrc_data_free(ctx->file_data);
	9315	+ ctx->file_data = NULL;
	9316	+ return -ENOMEM;
	9317	+ }
8854	9318
8855	9319	for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
8856		− if (copy_from_user(&fd, &fds[i], sizeof(fd))) {
	9320	+ struct io_fixed_file *file_slot;
	9321	+
	9322	+ if (fds && copy_from_user(&fd, &fds[i], sizeof(fd))) {
8857	9323	ret = -EFAULT;
8858		− goto out_fput;
	9324	+ goto fail;
8859	9325	}
8860	9326	/* allow sparse sets */
8861		− if (fd == -1) {
	9327	+ if (!fds \|\| fd == -1) {
8862	9328	ret = -EINVAL;
8863	9329	if (unlikely(*io_get_tag_slot(ctx->file_data, i)))
8864		− goto out_fput;
	9330	+ goto fail;
8865	9331	continue;
8866	9332	}
8867	9333
8868	9334	file = fget(fd);
8869	9335	ret = -EBADF;
8870	9336	if (unlikely(!file))
8871		− goto out_fput;
	9337	+ goto fail;
8872	9338
8873	9339	/*
8874	9340	* Don't allow io_uring instances to be registered. If UNIX
@@ -8879,74 +9345,23 @@ static int io_sqe_files_register(struct io_ring_ctx ctx, void __user arg, · 4 unmodified lines
8879	9345	*/
8880	9346	if (file->f_op == &io_uring_fops) {
8881	9347	fput(file);
8882		− goto out_fput;
	9348	+ goto fail;
8883	9349	}
8884		− io_fixed_file_set(io_fixed_file_slot(&ctx->file_table, i), file);
8885		− }
8886		−
8887		− ret = io_sqe_files_scm(ctx);
8888		− if (ret) {
8889		− __io_sqe_files_unregister(ctx);
8890		− return ret;
8891		− }
8892		−
8893		− io_rsrc_node_switch(ctx, NULL);
8894		− return ret;
8895		−out_fput:
8896		− for (i = 0; i < ctx->nr_user_files; i++) {
8897		− file = io_file_from_index(ctx, i);
8898		− if (file)
	9350	+ ret = io_scm_file_account(ctx, file);
	9351	+ if (ret) {
8899	9352	fput(file);
8900		− }
8901		− io_free_file_tables(&ctx->file_table);
8902		− ctx->nr_user_files = 0;
8903		−out_free:
8904		− io_rsrc_data_free(ctx->file_data);
8905		− ctx->file_data = NULL;
8906		− return ret;
8907		−}
8908		−
8909		−static int io_sqe_file_register(struct io_ring_ctx ctx, struct file file,
8910		− int index)
8911		−{
8912		−#if defined(CONFIG_UNIX)
8913		− struct sock *sock = ctx->ring_sock->sk;
8914		− struct sk_buff_head *head = &sock->sk_receive_queue;
8915		− struct sk_buff *skb;
8916		−
8917		− /*
8918		− * See if we can merge this file into an existing skb SCM_RIGHTS
8919		− * file set. If there's no room, fall back to allocating a new skb
8920		− * and filling it in.
8921		− */
8922		− spin_lock_irq(&head->lock);
8923		− skb = skb_peek(head);
8924		− if (skb) {
8925		− struct scm_fp_list *fpl = UNIXCB(skb).fp;
8926		−
8927		− if (fpl->count < SCM_MAX_FD) {
8928		− __skb_unlink(skb, head);
8929		− spin_unlock_irq(&head->lock);
8930		− fpl->fp[fpl->count] = get_file(file);
8931		− unix_inflight(fpl->user, fpl->fp[fpl->count]);
8932		− fpl->count++;
8933		− spin_lock_irq(&head->lock);
8934		− __skb_queue_head(head, skb);
8935		− } else {
8936		− skb = NULL;
	9353	+ goto fail;
8937	9354	}
8938		− }
8939		− spin_unlock_irq(&head->lock);
8940		−
8941		− if (skb) {
8942		− fput(file);
8943		− return 0;
	9355	+ file_slot = io_fixed_file_slot(&ctx->file_table, i);
	9356	+ io_fixed_file_set(file_slot, file);
	9357	+ io_file_bitmap_set(&ctx->file_table, i);
8944	9358	}
8945	9359
8946		− return __io_sqe_files_scm(ctx, 1, index);
8947		−#else
	9360	+ io_rsrc_node_switch(ctx, NULL);
8948	9361	return 0;
8949		−#endif
	9362	+fail:
	9363	+ __io_sqe_files_unregister(ctx);
	9364	+ return ret;
8950	9365	}
8951	9366
8952	9367	static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
@@ -8970,12 +9385,11 @@ static int io_install_fixed_file(struct io_kiocb req, struct file file, · 17 unmodified lines
8970	9385	unsigned int issue_flags, u32 slot_index)
8971	9386	{
8972	9387	struct io_ring_ctx *ctx = req->ctx;
8973		− bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
8974	9388	bool needs_switch = false;
8975	9389	struct io_fixed_file *file_slot;
8976	9390	int ret = -EBADF;
8977	9391
8978		− io_ring_submit_lock(ctx, needs_lock);
	9392	+ io_ring_submit_lock(ctx, issue_flags);
8979	9393	if (file->f_op == &io_uring_fops)
8980	9394	goto err;
8981	9395	ret = -ENXIO;
@@ -9001,22 +9415,20 @@ static int io_install_fixed_file(struct io_kiocb req, struct file file, · 19 unmodified lines
9001	9415	if (ret)
9002	9416	goto err;
9003	9417	file_slot->file_ptr = 0;
	9418	+ io_file_bitmap_clear(&ctx->file_table, slot_index);
9004	9419	needs_switch = true;
9005	9420	}
9006	9421
9007		− *io_get_tag_slot(ctx->file_data, slot_index) = 0;
9008		− io_fixed_file_set(file_slot, file);
9009		− ret = io_sqe_file_register(ctx, file, slot_index);
9010		− if (ret) {
9011		− file_slot->file_ptr = 0;
9012		− goto err;
	9422	+ ret = io_scm_file_account(ctx, file);
	9423	+ if (!ret) {
	9424	+ *io_get_tag_slot(ctx->file_data, slot_index) = 0;
	9425	+ io_fixed_file_set(file_slot, file);
	9426	+ io_file_bitmap_set(&ctx->file_table, slot_index);
9013	9427	}
9014		−
9015		− ret = 0;
9016	9428	err:
9017	9429	if (needs_switch)
9018	9430	io_rsrc_node_switch(ctx, ctx->file_data);
9019		− io_ring_submit_unlock(ctx, needs_lock);
	9431	+ io_ring_submit_unlock(ctx, issue_flags);
9020	9432	if (ret)
9021	9433	fput(file);
9022	9434	return ret;
@@ -9026,12 +9438,11 @@ static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags) · 3 unmodified lines
9026	9438	{
9027	9439	unsigned int offset = req->close.file_slot - 1;
9028	9440	struct io_ring_ctx *ctx = req->ctx;
9029		− bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
9030	9441	struct io_fixed_file *file_slot;
9031	9442	struct file *file;
9032	9443	int ret;
9033	9444
9034		− io_ring_submit_lock(ctx, needs_lock);
	9445	+ io_ring_submit_lock(ctx, issue_flags);
9035	9446	ret = -ENXIO;
9036	9447	if (unlikely(!ctx->file_data))
9037	9448	goto out;
@@ -9054,10 +9465,11 @@ static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags) · 16 unmodified lines
9054	9465	goto out;
9055	9466
9056	9467	file_slot->file_ptr = 0;
	9468	+ io_file_bitmap_clear(&ctx->file_table, offset);
9057	9469	io_rsrc_node_switch(ctx, ctx->file_data);
9058	9470	ret = 0;
9059	9471	out:
9060		− io_ring_submit_unlock(ctx, needs_lock);
	9472	+ io_ring_submit_unlock(ctx, issue_flags);
9061	9473	return ret;
9062	9474	}
9063	9475
@@ -9103,6 +9515,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, · 39 unmodified lines
9103	9515	if (err)
9104	9516	break;
9105	9517	file_slot->file_ptr = 0;
	9518	+ io_file_bitmap_clear(&ctx->file_table, i);
9106	9519	needs_switch = true;
9107	9520	}
9108	9521	if (fd != -1) {
@@ -9124,14 +9537,14 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, · 15 unmodified lines
9124	9537	err = -EBADF;
9125	9538	break;
9126	9539	}
9127		− *io_get_tag_slot(data, i) = tag;
9128		− io_fixed_file_set(file_slot, file);
9129		− err = io_sqe_file_register(ctx, file, i);
	9540	+ err = io_scm_file_account(ctx, file);
9130	9541	if (err) {
9131		− file_slot->file_ptr = 0;
9132	9542	fput(file);
9133	9543	break;
9134	9544	}
	9545	+ *io_get_tag_slot(data, i) = tag;
	9546	+ io_fixed_file_set(file_slot, file);
	9547	+ io_file_bitmap_set(&ctx->file_table, i);
9135	9548	}
9136	9549	}
9137	9550
@@ -9559,30 +9972,18 @@ static int io_buffer_account_pin(struct io_ring_ctx ctx, struct page *pages, · 421 unmodified lines
9559	9972	return ret;
9560	9973	}
9561	9974
9562		−static int io_sqe_buffer_register(struct io_ring_ctx ctx, struct iovec iov,
9563		− struct io_mapped_ubuf **pimu,
9564		− struct page **last_hpage)
	9975	+static struct page **io_pin_pages(unsigned long ubuf, unsigned long len,
	9976	+ int *npages)
9565	9977	{
9566		− struct io_mapped_ubuf *imu = NULL;
	9978	+ unsigned long start, end, nr_pages;
9567	9979	struct vm_area_struct **vmas = NULL;
9568	9980	struct page **pages = NULL;
9569		− unsigned long off, start, end, ubuf;
9570		− size_t size;
9571		− int ret, pret, nr_pages, i;
9572		−
9573		− if (!iov->iov_base) {
9574		− *pimu = ctx->dummy_ubuf;
9575		− return 0;
9576		− }
	9981	+ int i, pret, ret = -ENOMEM;
9577	9982
9578		− ubuf = (unsigned long) iov->iov_base;
9579		− end = (ubuf + iov->iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
	9983	+ end = (ubuf + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
9580	9984	start = ubuf >> PAGE_SHIFT;
9581	9985	nr_pages = end - start;
9582	9986
9583		− *pimu = NULL;
9584		− ret = -ENOMEM;
9585		−
9586	9987	pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
9587	9988	if (!pages)
9588	9989	goto done;
@@ -9592,10 +9993,6 @@ static int io_sqe_buffer_register(struct io_ring_ctx ctx, struct iovec iov, · 3 unmodified lines
9592	9993	if (!vmas)
9593	9994	goto done;
9594	9995
9595		− imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);
9596		− if (!imu)
9597		− goto done;
9598		−
9599	9996	ret = 0;
9600	9997	mmap_read_lock(current->mm);
9601	9998	pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE \| FOLL_LONGTERM,
@@ -9613,6 +10010,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx ctx, struct iovec iov, · 11 unmodified lines
9613	10010	break;
9614	10011	}
9615	10012	}
	10013	+ *npages = nr_pages;
9616	10014	} else {
9617	10015	ret = pret < 0 ? pret : -EFAULT;
9618	10016	}
@@ -9626,14 +10024,53 @@ static int io_sqe_buffer_register(struct io_ring_ctx ctx, struct iovec iov, · 7 unmodified lines
9626	10024	unpin_user_pages(pages, pret);
9627	10025	goto done;
9628	10026	}
	10027	+ ret = 0;
	10028	+done:
	10029	+ kvfree(vmas);
	10030	+ if (ret < 0) {
	10031	+ kvfree(pages);
	10032	+ pages = ERR_PTR(ret);
	10033	+ }
	10034	+ return pages;
	10035	+}
	10036	+
	10037	+static int io_sqe_buffer_register(struct io_ring_ctx ctx, struct iovec iov,
	10038	+ struct io_mapped_ubuf **pimu,
	10039	+ struct page **last_hpage)
	10040	+{
	10041	+ struct io_mapped_ubuf *imu = NULL;
	10042	+ struct page **pages = NULL;
	10043	+ unsigned long off;
	10044	+ size_t size;
	10045	+ int ret, nr_pages, i;
	10046	+
	10047	+ if (!iov->iov_base) {
	10048	+ *pimu = ctx->dummy_ubuf;
	10049	+ return 0;
	10050	+ }
	10051	+
	10052	+ *pimu = NULL;
	10053	+ ret = -ENOMEM;
	10054	+
	10055	+ pages = io_pin_pages((unsigned long) iov->iov_base, iov->iov_len,
	10056	+ &nr_pages);
	10057	+ if (IS_ERR(pages)) {
	10058	+ ret = PTR_ERR(pages);
	10059	+ pages = NULL;
	10060	+ goto done;
	10061	+ }
	10062	+
	10063	+ imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);
	10064	+ if (!imu)
	10065	+ goto done;
9629	10066
9630		− ret = io_buffer_account_pin(ctx, pages, pret, imu, last_hpage);
	10067	+ ret = io_buffer_account_pin(ctx, pages, nr_pages, imu, last_hpage);
9631	10068	if (ret) {
9632		− unpin_user_pages(pages, pret);
	10069	+ unpin_user_pages(pages, nr_pages);
9633	10070	goto done;
9634	10071	}
9635	10072
9636		− off = ubuf & ~PAGE_MASK;
	10073	+ off = (unsigned long) iov->iov_base & ~PAGE_MASK;
9637	10074	size = iov->iov_len;
9638	10075	for (i = 0; i < nr_pages; i++) {
9639	10076	size_t vec_len;
@@ -9646,8 +10083,8 @@ static int io_sqe_buffer_register(struct io_ring_ctx ctx, struct iovec iov, · 6 unmodified lines
9646	10083	size -= vec_len;
9647	10084	}
9648	10085	/* store original address for later verification */
9649		− imu->ubuf = ubuf;
9650		− imu->ubuf_end = ubuf + iov->iov_len;
	10086	+ imu->ubuf = (unsigned long) iov->iov_base;
	10087	+ imu->ubuf_end = imu->ubuf + iov->iov_len;
9651	10088	imu->nr_bvecs = nr_pages;
9652	10089	*pimu = imu;
9653	10090	ret = 0;
@@ -9655,7 +10092,6 @@ done: · 1 unmodified line
9655	10092	if (ret)
9656	10093	kvfree(imu);
9657	10094	kvfree(pages);
9658		− kvfree(vmas);
9659	10095	return ret;
9660	10096	}
9661	10097
@@ -9714,12 +10150,17 @@ static int io_sqe_buffers_register(struct io_ring_ctx ctx, void __user arg, · 52 unmodified lines
9714	10150	}
9715	10151
9716	10152	for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) {
9717		− ret = io_copy_iov(ctx, &iov, arg, i);
9718		− if (ret)
9719		− break;
9720		− ret = io_buffer_validate(&iov);
9721		− if (ret)
9722		− break;
	10153	+ if (arg) {
	10154	+ ret = io_copy_iov(ctx, &iov, arg, i);
	10155	+ if (ret)
	10156	+ break;
	10157	+ ret = io_buffer_validate(&iov);
	10158	+ if (ret)
	10159	+ break;
	10160	+ } else {
	10161	+ memset(&iov, 0, sizeof(iov));
	10162	+ }
	10163	+
9723	10164	if (!iov.iov_base && *io_get_tag_slot(data, i)) {
9724	10165	ret = -EINVAL;
9725	10166	break;
@@ -9858,19 +10299,19 @@ static int io_eventfd_unregister(struct io_ring_ctx *ctx) · 132 unmodified lines
9858	10299
9859	10300	static void io_destroy_buffers(struct io_ring_ctx *ctx)
9860	10301	{
	10302	+ struct io_buffer_list *bl;
	10303	+ unsigned long index;
9861	10304	int i;
9862	10305
9863		− for (i = 0; i < (1U << IO_BUFFERS_HASH_BITS); i++) {
9864		− struct list_head *list = &ctx->io_buffers[i];
9865		−
9866		− while (!list_empty(list)) {
9867		− struct io_buffer_list *bl;
	10306	+ for (i = 0; i < BGID_ARRAY; i++) {
	10307	+ if (!ctx->io_bl)
	10308	+ break;
	10309	+ __io_remove_buffers(ctx, &ctx->io_bl[i], -1U);
	10310	+ }
9868	10311
9869		− bl = list_first_entry(list, struct io_buffer_list, list);
9870		− __io_remove_buffers(ctx, bl, -1U);
9871		− list_del(&bl->list);
9872		− kfree(bl);
9873		− }
	10312	+ xa_for_each(&ctx->io_bl_xa, index, bl) {
	10313	+ xa_erase(&ctx->io_bl_xa, bl->bgid);
	10314	+ __io_remove_buffers(ctx, bl, -1U);
9874	10315	}
9875	10316
9876	10317	while (!list_empty(&ctx->io_buffers_pages)) {
@@ -9890,7 +10331,7 @@ static void io_req_caches_free(struct io_ring_ctx *ctx) · 13 unmodified lines
9890	10331	mutex_lock(&ctx->uring_lock);
9891	10332	io_flush_cached_locked_reqs(ctx, state);
9892	10333
9893		− while (state->free_list.next) {
	10334	+ while (!io_req_cache_empty(ctx)) {
9894	10335	struct io_wq_work_node *node;
9895	10336	struct io_kiocb *req;
9896	10337
@@ -9979,7 +10420,8 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) · 82 unmodified lines
9979	10420	io_wq_put_hash(ctx->hash_map);
9980	10421	kfree(ctx->cancel_hash);
9981	10422	kfree(ctx->dummy_ubuf);
9982		− kfree(ctx->io_buffers);
	10423	+ kfree(ctx->io_bl);
	10424	+ xa_destroy(&ctx->io_bl_xa);
9983	10425	kfree(ctx);
9984	10426	}
9985	10427
@@ -10010,7 +10452,8 @@ static __poll_t io_uring_poll(struct file file, poll_table wait) · 24 unmodified lines
10010	10452	* Users may get EPOLLIN meanwhile seeing nothing in cqring, this
10011	10453	* pushs them to do the flush.
10012	10454	*/
10013		− if (io_cqring_events(ctx) \|\| test_bit(0, &ctx->check_cq_overflow))
	10455	+ if (io_cqring_events(ctx) \|\|
	10456	+ test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq))
10014	10457	mask \|= EPOLLIN \| EPOLLRDNORM;
10015	10458
10016	10459	return mask;
@@ -10142,8 +10585,7 @@ static __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, · 125 unmodified lines
10142	10585	}
10143	10586	}
10144	10587	spin_unlock_irq(&ctx->timeout_lock);
10145		− if (canceled != 0)
10146		− io_commit_cqring(ctx);
	10588	+ io_commit_cqring(ctx);
10147	10589	spin_unlock(&ctx->completion_lock);
10148	10590	if (canceled != 0)
10149	10591	io_cqring_ev_posted(ctx);
@@ -10163,11 +10605,13 @@ static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) · 13 unmodified lines
10163	10605	io_unregister_personality(ctx, index);
10164	10606	mutex_unlock(&ctx->uring_lock);
10165	10607
10166		− io_kill_timeouts(ctx, NULL, true);
10167		− io_poll_remove_all(ctx, NULL, true);
10168		−
10169		− /* if we failed setting up the ctx, we might not have any rings */
10170		− io_iopoll_try_reap_events(ctx);
	10608	+ /* failed during ring init, it couldn't have issued any requests */
	10609	+ if (ctx->rings) {
	10610	+ io_kill_timeouts(ctx, NULL, true);
	10611	+ io_poll_remove_all(ctx, NULL, true);
	10612	+ /* if we failed setting up the ctx, we might not have any rings */
	10613	+ io_iopoll_try_reap_events(ctx);
	10614	+ }
10171	10615
10172	10616	INIT_WORK(&ctx->exit_work, io_ring_exit_work);
10173	10617	/*
@@ -10259,6 +10703,10 @@ static __cold void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, · 85 unmodified lines
10259	10703	struct io_task_cancel cancel = { .task = task, .all = cancel_all, };
10260	10704	struct io_uring_task *tctx = task ? task->io_uring : NULL;
10261	10705
	10706	+ /* failed during ring init, it couldn't have issued any requests */
	10707	+ if (!ctx->rings)
	10708	+ return;
	10709	+
10262	10710	while (1) {
10263	10711	enum io_wq_cancel cret;
10264	10712	bool ret = false;
@@ -10704,6 +11152,19 @@ static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx) · 439 unmodified lines
10704	11152	return 0;
10705	11153	}
10706	11154
	11155	+static int io_validate_ext_arg(unsigned flags, const void __user *argp, size_t argsz)
	11156	+{
	11157	+ if (flags & IORING_ENTER_EXT_ARG) {
	11158	+ struct io_uring_getevents_arg arg;
	11159	+
	11160	+ if (argsz != sizeof(arg))
	11161	+ return -EINVAL;
	11162	+ if (copy_from_user(&arg, argp, sizeof(arg)))
	11163	+ return -EFAULT;
	11164	+ }
	11165	+ return 0;
	11166	+}
	11167	+
10707	11168	static int io_get_ext_arg(unsigned flags, const void __user argp, size_t argsz,
10708	11169	struct __kernel_timespec __user **ts,
10709	11170	const sigset_t __user **sig)
@@ -10741,7 +11202,6 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 31 unmodified lines
10741	11202	size_t, argsz)
10742	11203	{
10743	11204	struct io_ring_ctx *ctx;
10744		− int submitted = 0;
10745	11205	struct fd f;
10746	11206	long ret;
10747	11207
@@ -10804,39 +11264,64 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 56 unmodified lines
10804	11264	if (ret)
10805	11265	goto out;
10806	11266	}
10807		− submitted = to_submit;
	11267	+ ret = to_submit;
10808	11268	} else if (to_submit) {
10809	11269	ret = io_uring_add_tctx_node(ctx);
10810	11270	if (unlikely(ret))
10811	11271	goto out;
10812		− mutex_lock(&ctx->uring_lock);
10813		− submitted = io_submit_sqes(ctx, to_submit);
10814		− mutex_unlock(&ctx->uring_lock);
10815	11272
10816		− if (submitted != to_submit)
	11273	+ mutex_lock(&ctx->uring_lock);
	11274	+ ret = io_submit_sqes(ctx, to_submit);
	11275	+ if (ret != to_submit) {
	11276	+ mutex_unlock(&ctx->uring_lock);
10817	11277	goto out;
	11278	+ }
	11279	+ if ((flags & IORING_ENTER_GETEVENTS) && ctx->syscall_iopoll)
	11280	+ goto iopoll_locked;
	11281	+ mutex_unlock(&ctx->uring_lock);
10818	11282	}
10819	11283	if (flags & IORING_ENTER_GETEVENTS) {
10820		− const sigset_t __user *sig;
10821		− struct __kernel_timespec __user *ts;
10822		−
10823		− ret = io_get_ext_arg(flags, argp, &argsz, &ts, &sig);
10824		− if (unlikely(ret))
10825		− goto out;
	11284	+ int ret2;
	11285	+ if (ctx->syscall_iopoll) {
	11286	+ /*
	11287	+ * We disallow the app entering submit/complete with
	11288	+ * polling, but we still need to lock the ring to
	11289	+ * prevent racing with polled issue that got punted to
	11290	+ * a workqueue.
	11291	+ */
	11292	+ mutex_lock(&ctx->uring_lock);
	11293	+iopoll_locked:
	11294	+ ret2 = io_validate_ext_arg(flags, argp, argsz);
	11295	+ if (likely(!ret2)) {
	11296	+ min_complete = min(min_complete,
	11297	+ ctx->cq_entries);
	11298	+ ret2 = io_iopoll_check(ctx, min_complete);
	11299	+ }
	11300	+ mutex_unlock(&ctx->uring_lock);
	11301	+ } else {
	11302	+ const sigset_t __user *sig;
	11303	+ struct __kernel_timespec __user *ts;
	11304	+
	11305	+ ret2 = io_get_ext_arg(flags, argp, &argsz, &ts, &sig);
	11306	+ if (likely(!ret2)) {
	11307	+ min_complete = min(min_complete,
	11308	+ ctx->cq_entries);
	11309	+ ret2 = io_cqring_wait(ctx, min_complete, sig,
	11310	+ argsz, ts);
	11311	+ }
	11312	+ }
10826	11313
10827		− min_complete = min(min_complete, ctx->cq_entries);
	11314	+ if (!ret) {
	11315	+ ret = ret2;
10828	11316
10829		− /*
10830		− * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
10831		− * space applications don't need to do io completion events
10832		− * polling again, they can rely on io_sq_thread to do polling
10833		− * work, which can reduce cpu usage and uring_lock contention.
10834		− */
10835		− if (ctx->flags & IORING_SETUP_IOPOLL &&
10836		− !(ctx->flags & IORING_SETUP_SQPOLL)) {
10837		− ret = io_iopoll_check(ctx, min_complete);
10838		− } else {
10839		− ret = io_cqring_wait(ctx, min_complete, sig, argsz, ts);
	11317	+ /*
	11318	+ * EBADR indicates that one or more CQE were dropped.
	11319	+ * Once the user has been informed we can clear the bit
	11320	+ * as they are obviously ok with those drops.
	11321	+ */
	11322	+ if (unlikely(ret2 == -EBADR))
	11323	+ clear_bit(IO_CHECK_CQ_DROPPED_BIT,
	11324	+ &ctx->check_cq);
10840	11325	}
10841	11326	}
10842	11327
@@ -10845,7 +11330,7 @@ out: · 2 unmodified lines
10845	11330	out_fput:
10846	11331	if (!(flags & IORING_ENTER_REGISTERED_RING))
10847	11332	fdput(f);
10848		− return submitted ? submitted : ret;
	11333	+ return ret;
10849	11334	}
10850	11335
10851	11336	#ifdef CONFIG_PROC_FS
@@ -11162,10 +11647,40 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, · 310 unmodified lines
11162	11647	ctx = io_ring_ctx_alloc(p);
11163	11648	if (!ctx)
11164	11649	return -ENOMEM;
	11650	+
	11651	+ /*
	11652	+ * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
	11653	+ * space applications don't need to do io completion events
	11654	+ * polling again, they can rely on io_sq_thread to do polling
	11655	+ * work, which can reduce cpu usage and uring_lock contention.
	11656	+ */
	11657	+ if (ctx->flags & IORING_SETUP_IOPOLL &&
	11658	+ !(ctx->flags & IORING_SETUP_SQPOLL))
	11659	+ ctx->syscall_iopoll = 1;
	11660	+
11165	11661	ctx->compat = in_compat_syscall();
11166	11662	if (!capable(CAP_IPC_LOCK))
11167	11663	ctx->user = get_uid(current_user());
11168	11664
	11665	+ /*
	11666	+ * For SQPOLL, we just need a wakeup, always. For !SQPOLL, if
	11667	+ * COOP_TASKRUN is set, then IPIs are never needed by the app.
	11668	+ */
	11669	+ ret = -EINVAL;
	11670	+ if (ctx->flags & IORING_SETUP_SQPOLL) {
	11671	+ /* IPI related flags don't make sense with SQPOLL */
	11672	+ if (ctx->flags & (IORING_SETUP_COOP_TASKRUN \|
	11673	+ IORING_SETUP_TASKRUN_FLAG))
	11674	+ goto err;
	11675	+ ctx->notify_method = TWA_SIGNAL_NO_IPI;
	11676	+ } else if (ctx->flags & IORING_SETUP_COOP_TASKRUN) {
	11677	+ ctx->notify_method = TWA_SIGNAL_NO_IPI;
	11678	+ } else {
	11679	+ if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
	11680	+ goto err;
	11681	+ ctx->notify_method = TWA_SIGNAL;
	11682	+ }
	11683	+
11169	11684	/*
11170	11685	* This is just grabbed for accounting purposes. When a process exits,
11171	11686	* the mm is exited and dropped before the files, hence we need to hang
@@ -11263,10 +11778,11 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) · 91 unmodified lines
11263	11778	if (p.flags & ~(IORING_SETUP_IOPOLL \| IORING_SETUP_SQPOLL \|
11264	11779	IORING_SETUP_SQ_AFF \| IORING_SETUP_CQSIZE \|
11265	11780	IORING_SETUP_CLAMP \| IORING_SETUP_ATTACH_WQ \|
11266		− IORING_SETUP_R_DISABLED \| IORING_SETUP_SUBMIT_ALL))
	11781	+ IORING_SETUP_R_DISABLED \| IORING_SETUP_SUBMIT_ALL \|
	11782	+ IORING_SETUP_COOP_TASKRUN \| IORING_SETUP_TASKRUN_FLAG))
11267	11783	return -EINVAL;
11268	11784
11269		− return io_uring_create(entries, &p, params);
	11785	+ return io_uring_create(entries, &p, params);
11270	11786	}
11271	11787
11272	11788	SYSCALL_DEFINE2(io_uring_setup, u32, entries,
@@ -11479,14 +11995,20 @@ static __cold int io_register_rsrc(struct io_ring_ctx ctx, void __user arg, · 206 unmodified lines
11479	11995	memset(&rr, 0, sizeof(rr));
11480	11996	if (copy_from_user(&rr, arg, size))
11481	11997	return -EFAULT;
11482		− if (!rr.nr \|\| rr.resv \|\| rr.resv2)
	11998	+ if (!rr.nr \|\| rr.resv2)
	11999	+ return -EINVAL;
	12000	+ if (rr.flags & ~IORING_RSRC_REGISTER_SPARSE)
11483	12001	return -EINVAL;
11484	12002
11485	12003	switch (type) {
11486	12004	case IORING_RSRC_FILE:
	12005	+ if (rr.flags & IORING_RSRC_REGISTER_SPARSE && rr.data)
	12006	+ break;
11487	12007	return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data),
11488	12008	rr.nr, u64_to_user_ptr(rr.tags));
11489	12009	case IORING_RSRC_BUFFER:
	12010	+ if (rr.flags & IORING_RSRC_REGISTER_SPARSE && rr.data)
	12011	+ break;
11490	12012	return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data),
11491	12013	rr.nr, u64_to_user_ptr(rr.tags));
11492	12014	}
@@ -11621,6 +12143,85 @@ err: · 128 unmodified lines
11621	12143	return ret;
11622	12144	}
11623	12145
	12146	+static int io_register_pbuf_ring(struct io_ring_ctx ctx, void __user arg)
	12147	+{
	12148	+ struct io_uring_buf_ring *br;
	12149	+ struct io_uring_buf_reg reg;
	12150	+ struct io_buffer_list *bl;
	12151	+ struct page **pages;
	12152	+ int nr_pages;
	12153	+
	12154	+ if (copy_from_user(&reg, arg, sizeof(reg)))
	12155	+ return -EFAULT;
	12156	+
	12157	+ if (reg.pad \|\| reg.resv[0] \|\| reg.resv[1] \|\| reg.resv[2])
	12158	+ return -EINVAL;
	12159	+ if (!reg.ring_addr)
	12160	+ return -EFAULT;

126 more lines aren’t shown here.

include/linux/sched/signal.h+11−2

@@ -355,14 +355,23 @@ static inline void clear_notify_signal(void) · 354 unmodified lines
355	355	smp_mb__after_atomic();
356	356	}
357	357
	358	+/*
	359	+ * Returns 'true' if kick_process() is needed to force a transition from
	360	+ * user -> kernel to guarantee expedient run of TWA_SIGNAL based task_work.
	361	+ */
	362	+static inline bool __set_notify_signal(struct task_struct *task)
	363	+{
	364	+ return !test_and_set_tsk_thread_flag(task, TIF_NOTIFY_SIGNAL) &&
	365	+ !wake_up_state(task, TASK_INTERRUPTIBLE);
	366	+}
	367	+
358	368	/*
359	369	* Called to break out of interruptible wait loops, and enter the
360	370	* exit_to_user_mode_loop().
361	371	*/
362	372	static inline void set_notify_signal(struct task_struct *task)
363	373	{
364		− if (!test_and_set_tsk_thread_flag(task, TIF_NOTIFY_SIGNAL) &&
365		− !wake_up_state(task, TASK_INTERRUPTIBLE))
	374	+ if (__set_notify_signal(task))
366	375	kick_process(task);
367	376	}
368	377

include/linux/task_work.h+1−0

@@ -17,6 +17,7 @@ enum task_work_notify_mode { · 16 unmodified lines
17	17	TWA_NONE,
18	18	TWA_RESUME,
19	19	TWA_SIGNAL,
	20	+ TWA_SIGNAL_NO_IPI,
20	21	};
21	22
22	23	static inline bool task_work_pending(struct task_struct *task)

include/trace/events/io_uring.h+42−2

@@ -520,7 +520,7 @@ TRACE_EVENT(io_uring_req_failed, · 519 unmodified lines
520	520	__entry->off = sqe->off;
521	521	__entry->addr = sqe->addr;
522	522	__entry->len = sqe->len;
523		− __entry->op_flags = sqe->rw_flags;
	523	+ __entry->op_flags = sqe->poll32_events;
524	524	__entry->buf_index = sqe->buf_index;
525	525	__entry->personality = sqe->personality;
526	526	__entry->file_index = sqe->file_index;
@@ -530,7 +530,7 @@ TRACE_EVENT(io_uring_req_failed, · 3 unmodified lines
530	530	),
531	531
532	532	TP_printk("ring %p, req %p, user_data 0x%llx, "
533		− "op %d, flags 0x%x, prio=%d, off=%llu, addr=%llu, "
	533	+ "op %d, flags 0x%x, prio=%d, off=%llu, addr=%llu, "
534	534	"len=%u, rw_flags=0x%x, buf_index=%d, "
535	535	"personality=%d, file_index=%d, pad=0x%llx/%llx, error=%d",
536	536	__entry->ctx, __entry->req, __entry->user_data,
@@ -543,6 +543,46 @@ TRACE_EVENT(io_uring_req_failed, · 6 unmodified lines
543	543	(unsigned long long) __entry->pad2, __entry->error)
544	544	);
545	545
	546	+
	547	+/*
	548	+ * io_uring_cqe_overflow - a CQE overflowed
	549	+ *
	550	+ * @ctx: pointer to a ring context structure
	551	+ * @user_data: user data associated with the request
	552	+ * @res: CQE result
	553	+ * @cflags: CQE flags
	554	+ * @ocqe: pointer to the overflow cqe (if available)
	555	+ *
	556	+ */
	557	+TRACE_EVENT(io_uring_cqe_overflow,
	558	+
	559	+ TP_PROTO(void *ctx, unsigned long long user_data, s32 res, u32 cflags,
	560	+ void *ocqe),
	561	+
	562	+ TP_ARGS(ctx, user_data, res, cflags, ocqe),
	563	+
	564	+ TP_STRUCT__entry (
	565	+ __field( void *, ctx )
	566	+ __field( unsigned long long, user_data )
	567	+ __field( s32, res )
	568	+ __field( u32, cflags )
	569	+ __field( void *, ocqe )
	570	+ ),
	571	+
	572	+ TP_fast_assign(
	573	+ __entry->ctx = ctx;
	574	+ __entry->user_data = user_data;
	575	+ __entry->res = res;
	576	+ __entry->cflags = cflags;
	577	+ __entry->ocqe = ocqe;
	578	+ ),
	579	+
	580	+ TP_printk("ring %p, user_data 0x%llx, res %d, flags %x, "
	581	+ "overflow_cqe %p",
	582	+ __entry->ctx, __entry->user_data, __entry->res,
	583	+ __entry->cflags, __entry->ocqe)
	584	+);
	585	+
546	586	#endif /* _TRACE_IO_URING_H */
547	587
548	588	/* This part must be outside protection */

include/uapi/linux/io_uring.h+94−1

@@ -63,6 +63,15 @@ struct io_uring_sqe { · 62 unmodified lines
63	63	__u64 __pad2[2];
64	64	};
65	65
	66	+/*
	67	+ * If sqe->file_index is set to this for opcodes that instantiate a new
	68	+ * direct descriptor (like openat/openat2/accept), then io_uring will allocate
	69	+ * an available direct descriptor instead of having the application pass one
	70	+ * in. The picked direct descriptor will be returned in cqe->res, or -ENFILE
	71	+ * if the space is full.
	72	+ */
	73	+#define IORING_FILE_INDEX_ALLOC (~0U)
	74	+
66	75	enum {
67	76	IOSQE_FIXED_FILE_BIT,
68	77	IOSQE_IO_DRAIN_BIT,
@@ -102,6 +111,20 @@ enum { · 33 unmodified lines
102	111	#define IORING_SETUP_ATTACH_WQ (1U << 5) /* attach to existing wq */
103	112	#define IORING_SETUP_R_DISABLED (1U << 6) /* start with ring disabled */
104	113	#define IORING_SETUP_SUBMIT_ALL (1U << 7) /* continue submit on error */
	114	+/*
	115	+ * Cooperative task running. When requests complete, they often require
	116	+ * forcing the submitter to transition to the kernel to complete. If this
	117	+ * flag is set, work will be done when the task transitions anyway, rather
	118	+ * than force an inter-processor interrupt reschedule. This avoids interrupting
	119	+ * a task running in userspace, and saves an IPI.
	120	+ */
	121	+#define IORING_SETUP_COOP_TASKRUN (1U << 8)
	122	+/*
	123	+ * If COOP_TASKRUN is set, get notified if task work is available for
	124	+ * running and a kernel transition would be needed to run it. This sets
	125	+ * IORING_SQ_TASKRUN in the sq ring flags. Not valid with COOP_TASKRUN.
	126	+ */
	127	+#define IORING_SETUP_TASKRUN_FLAG (1U << 9)
105	128
106	129	enum {
107	130	IORING_OP_NOP,
@@ -187,6 +210,33 @@ enum { · 79 unmodified lines
187	210	#define IORING_POLL_UPDATE_EVENTS (1U << 1)
188	211	#define IORING_POLL_UPDATE_USER_DATA (1U << 2)
189	212
	213	+/*
	214	+ * ASYNC_CANCEL flags.
	215	+ *
	216	+ * IORING_ASYNC_CANCEL_ALL Cancel all requests that match the given key
	217	+ * IORING_ASYNC_CANCEL_FD Key off 'fd' for cancelation rather than the
	218	+ * request 'user_data'
	219	+ * IORING_ASYNC_CANCEL_ANY Match any request
	220	+ */
	221	+#define IORING_ASYNC_CANCEL_ALL (1U << 0)
	222	+#define IORING_ASYNC_CANCEL_FD (1U << 1)
	223	+#define IORING_ASYNC_CANCEL_ANY (1U << 2)
	224	+
	225	+/*
	226	+ * send/sendmsg and recv/recvmsg flags (sqe->addr2)
	227	+ *
	228	+ * IORING_RECVSEND_POLL_FIRST If set, instead of first attempting to send
	229	+ * or receive and arm poll if that yields an
	230	+ * -EAGAIN result, arm poll upfront and skip
	231	+ * the initial transfer attempt.
	232	+ */
	233	+#define IORING_RECVSEND_POLL_FIRST (1U << 0)
	234	+
	235	+/*
	236	+ * accept flags stored in sqe->ioprio
	237	+ */
	238	+#define IORING_ACCEPT_MULTISHOT (1U << 0)
	239	+
190	240	/*
191	241	* IO completion data structure (Completion Queue Entry)
192	242	*/
@@ -236,6 +286,7 @@ struct io_sqring_offsets { · 43 unmodified lines
236	286	*/
237	287	#define IORING_SQ_NEED_WAKEUP (1U << 0) /* needs io_uring_enter wakeup */
238	288	#define IORING_SQ_CQ_OVERFLOW (1U << 1) /* CQ ring is overflown */
	289	+#define IORING_SQ_TASKRUN (1U << 2) /* task should enter the kernel */
239	290
240	291	struct io_cqring_offsets {
241	292	__u32 head;
@@ -333,6 +384,10 @@ enum { · 91 unmodified lines
333	384	IORING_REGISTER_RING_FDS = 20,
334	385	IORING_UNREGISTER_RING_FDS = 21,
335	386
	387	+ /* register ring based provide buffer group */
	388	+ IORING_REGISTER_PBUF_RING = 22,
	389	+ IORING_UNREGISTER_PBUF_RING = 23,
	390	+
336	391	/* this goes last */
337	392	IORING_REGISTER_LAST
338	393	};
@@ -350,9 +405,15 @@ struct io_uring_files_update { · 11 unmodified lines
350	405	__aligned_u64 /* __s32 * */ fds;
351	406	};
352	407
	408	+/*
	409	+ * Register a fully sparse file space, rather than pass in an array of all
	410	+ * -1 file descriptors.
	411	+ */
	412	+#define IORING_RSRC_REGISTER_SPARSE (1U << 0)
	413	+
353	414	struct io_uring_rsrc_register {
354	415	__u32 nr;
355		− __u32 resv;
	416	+ __u32 flags;
356	417	__u64 resv2;
357	418	__aligned_u64 data;
358	419	__aligned_u64 tags;
@@ -404,6 +465,38 @@ struct io_uring_restriction { · 45 unmodified lines
404	465	__u32 resv2[3];
405	466	};
406	467
	468	+struct io_uring_buf {
	469	+ __u64 addr;
	470	+ __u32 len;
	471	+ __u16 bid;
	472	+ __u16 resv;
	473	+};
	474	+
	475	+struct io_uring_buf_ring {
	476	+ union {
	477	+ /*
	478	+ * To avoid spilling into more pages than we need to, the
	479	+ * ring tail is overlaid with the io_uring_buf->resv field.
	480	+ */
	481	+ struct {
	482	+ __u64 resv1;
	483	+ __u32 resv2;
	484	+ __u16 resv3;
	485	+ __u16 tail;
	486	+ };
	487	+ struct io_uring_buf bufs[0];
	488	+ };
	489	+};
	490	+
	491	+/* argument for IORING_(UN)REGISTER_PBUF_RING */
	492	+struct io_uring_buf_reg {
	493	+ __u64 ring_addr;
	494	+ __u32 ring_entries;
	495	+ __u16 bgid;
	496	+ __u16 pad;
	497	+ __u64 resv[3];
	498	+};
	499	+
407	500	/*
408	501	* io_uring_restriction->opcode values
409	502	*/

kernel/task_work.c+19−6

@@ -12,12 +12,22 @@ static struct callback_head work_exited; /* all we need is ->next == NULL */ · 11 unmodified lines
12	12	* @notify: how to notify the targeted task
13	13	*
14	14	* Queue @work for task_work_run() below and notify the @task if @notify
15		− * is @TWA_RESUME or @TWA_SIGNAL. @TWA_SIGNAL works like signals, in that the
16		− * it will interrupt the targeted task and run the task_work. @TWA_RESUME
17		− * work is run only when the task exits the kernel and returns to user mode,
18		− * or before entering guest mode. Fails if the @task is exiting/exited and thus
19		− * it can't process this @work. Otherwise @work->func() will be called when the
20		− * @task goes through one of the aforementioned transitions, or exits.
	15	+ * is @TWA_RESUME, @TWA_SIGNAL, or @TWA_SIGNAL_NO_IPI.
	16	+ *
	17	+ * @TWA_SIGNAL works like signals, in that the it will interrupt the targeted
	18	+ * task and run the task_work, regardless of whether the task is currently
	19	+ * running in the kernel or userspace.
	20	+ * @TWA_SIGNAL_NO_IPI works like @TWA_SIGNAL, except it doesn't send a
	21	+ * reschedule IPI to force the targeted task to reschedule and run task_work.
	22	+ * This can be advantageous if there's no strict requirement that the
	23	+ * task_work be run as soon as possible, just whenever the task enters the
	24	+ * kernel anyway.
	25	+ * @TWA_RESUME work is run only when the task exits the kernel and returns to
	26	+ * user mode, or before entering guest mode.
	27	+ *
	28	+ * Fails if the @task is exiting/exited and thus it can't process this @work.
	29	+ * Otherwise @work->func() will be called when the @task goes through one of
	30	+ * the aforementioned transitions, or exits.
21	31	*
22	32	* If the targeted task is exiting, then an error is returned and the work item
23	33	* is not queued. It's up to the caller to arrange for an alternative mechanism
@@ -53,6 +63,9 @@ int task_work_add(struct task_struct task, struct callback_head work, · 29 unmodified lines
53	63	case TWA_SIGNAL:
54	64	set_notify_signal(task);
55	65	break;
	66	+ case TWA_SIGNAL_NO_IPI:
	67	+ __set_notify_signal(task);
	68	+ break;
56	69	default:
57	70	WARN_ON_ONCE(1);
58	71	break;