Merge tag 'for-5.7/io_uring-2020-03-29' of git://git.kernel.dk/linux-block

Pull io_uring updates from Jens Axboe: "Here are the io_uring changes for this merge window. Light on new features this time around (just splice + buffer selection), lots of cleanups, fixes, and improvements to existing support. In particular, this contains: - Cleanup fixed file update handling for stack fallback (Hillf) - Re-work of how pollable async IO is handled, we no longer require thread offload to handle that. Instead we rely using poll to drive this, with task_work execution. - In conjunction with the above, allow expendable buffer selection, so that poll+recv (for example) no longer has to be a split operation. - Make sure we honor RLIMIT_FSIZE for buffered writes - Add support for splice (Pavel) - Linked work inheritance fixes and optimizations (Pavel) - Async work fixes and cleanups (Pavel) - Improve io-wq locking (Pavel) - Hashed link write improvements (Pavel) - SETUP_IOPOLL|SETUP_SQPOLL improvements (Xiaoguang)" * tag 'for-5.7/io_uring-2020-03-29' of git://git.kernel.dk/linux-block: (54 commits) io_uring: cleanup io_alloc_async_ctx() io_uring: fix missing 'return' in comment io-wq: handle hashed writes in chains io-uring: drop 'free_pfile' in struct io_file_put io-uring: drop completion when removing file io_uring: Fix ->data corruption on re-enqueue io-wq: close cancel gap for hashed linked work io_uring: make spdxcheck.py happy io_uring: honor original task RLIMIT_FSIZE io-wq: hash dependent work io-wq: split hashing and enqueueing io-wq: don't resched if there is no work io-wq: remove duplicated cancel code io_uring: fix truncated async read/readv and write/writev retry io_uring: dual license io_uring.h uapi header io_uring: io_uring_enter(2) don't poll while SETUP_IOPOLL|SETUP_SQPOLL enabled io_uring: Fix unused function warnings io_uring: add end-of-bits marker and build time verify it io_uring: provide means of removing buffers io_uring: add IOSQE_BUFFER_SELECT support for IORING_OP_RECVMSG ...

Linus Torvalds committed 6 years ago

Commit e59cd882 parents 1592614 3d9932a12 files · +1826 −856Browse files

Unified · Split

12 files

fs/io-wq.c +162 −206
fs/io-wq.h +46 −19
fs/io_uring.c +1406 −609
fs/splice.c +3 −3
include/linux/socket.h +4 −0
include/linux/splice.h +3 −0
include/net/compat.h +3 −0
include/trace/events/io_uring.h +103 −0
include/uapi/linux/io_uring.h +38 −4
kernel/task_work.c +14 −4
net/compat.c +23 −7
net/socket.c +21 −4

fs/io-wq.c+162−206

@@ -69,6 +69,8 @@ struct io_worker { · 68 unmodified lines
69	69	#define IO_WQ_HASH_ORDER 5
70	70	#endif
71	71
	72	+#define IO_WQ_NR_HASH_BUCKETS (1u << IO_WQ_HASH_ORDER)
	73	+
72	74	struct io_wqe_acct {
73	75	unsigned nr_workers;
74	76	unsigned max_workers;
@@ -98,6 +100,7 @@ struct io_wqe { · 23 unmodified lines
98	100	struct list_head all_list;
99	101
100	102	struct io_wq *wq;
	103	+ struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS];
101	104	};
102	105
103	106	/*
@@ -107,8 +110,7 @@ struct io_wq { · 3 unmodified lines
107	110	struct io_wqe **wqes;
108	111	unsigned long state;
109	112
110		− get_work_fn *get_work;
111		− put_work_fn *put_work;
	113	+ free_work_fn *free_work;
112	114
113	115	struct task_struct *manager;
114	116	struct user_struct *user;
@@ -376,26 +378,35 @@ static bool __io_worker_idle(struct io_wqe wqe, struct io_worker worker) · 261 unmodified lines
376	378	return __io_worker_unuse(wqe, worker);
377	379	}
378	380
379		−static struct io_wq_work io_get_next_work(struct io_wqe wqe, unsigned *hash)
	381	+static inline unsigned int io_get_work_hash(struct io_wq_work *work)
	382	+{
	383	+ return work->flags >> IO_WQ_HASH_SHIFT;
	384	+}
	385	+
	386	+static struct io_wq_work io_get_next_work(struct io_wqe wqe)
380	387	__must_hold(wqe->lock)
381	388	{
382	389	struct io_wq_work_node node, prev;
383		− struct io_wq_work *work;
	390	+ struct io_wq_work work, tail;
	391	+ unsigned int hash;
384	392
385	393	wq_list_for_each(node, prev, &wqe->work_list) {
386	394	work = container_of(node, struct io_wq_work, list);
387	395
388	396	/* not hashed, can run anytime */
389		− if (!(work->flags & IO_WQ_WORK_HASHED)) {
390		− wq_node_del(&wqe->work_list, node, prev);
	397	+ if (!io_wq_is_hashed(work)) {
	398	+ wq_list_del(&wqe->work_list, node, prev);
391	399	return work;
392	400	}
393	401
394	402	/* hashed, can run if not already running */
395		− *hash = work->flags >> IO_WQ_HASH_SHIFT;
396		− if (!(wqe->hash_map & BIT_ULL(*hash))) {
397		− wqe->hash_map \|= BIT_ULL(*hash);
398		− wq_node_del(&wqe->work_list, node, prev);
	403	+ hash = io_get_work_hash(work);
	404	+ if (!(wqe->hash_map & BIT(hash))) {
	405	+ wqe->hash_map \|= BIT(hash);
	406	+ /* all items with this hash lie in [work, tail] */
	407	+ tail = wqe->hash_tail[hash];
	408	+ wqe->hash_tail[hash] = NULL;
	409	+ wq_list_cut(&wqe->work_list, &tail->list, prev);
399	410	return work;
400	411	}
401	412	}
@@ -440,16 +451,49 @@ static void io_wq_switch_creds(struct io_worker *worker, · 38 unmodified lines
440	451	worker->saved_creds = old_creds;
441	452	}
442	453
	454	+static void io_impersonate_work(struct io_worker *worker,
	455	+ struct io_wq_work *work)
	456	+{
	457	+ if (work->files && current->files != work->files) {
	458	+ task_lock(current);
	459	+ current->files = work->files;
	460	+ task_unlock(current);
	461	+ }
	462	+ if (work->fs && current->fs != work->fs)
	463	+ current->fs = work->fs;
	464	+ if (work->mm != worker->mm)
	465	+ io_wq_switch_mm(worker, work);
	466	+ if (worker->cur_creds != work->creds)
	467	+ io_wq_switch_creds(worker, work);
	468	+}
	469	+
	470	+static void io_assign_current_work(struct io_worker *worker,
	471	+ struct io_wq_work *work)
	472	+{
	473	+ if (work) {
	474	+ /* flush pending signals before assigning new work */
	475	+ if (signal_pending(current))
	476	+ flush_signals(current);
	477	+ cond_resched();
	478	+ }
	479	+
	480	+ spin_lock_irq(&worker->lock);
	481	+ worker->cur_work = work;
	482	+ spin_unlock_irq(&worker->lock);
	483	+}
	484	+
	485	+static void io_wqe_enqueue(struct io_wqe wqe, struct io_wq_work work);
	486	+
443	487	static void io_worker_handle_work(struct io_worker *worker)
444	488	__releases(wqe->lock)
445	489	{
446		− struct io_wq_work work, old_work = NULL, *put_work = NULL;
447	490	struct io_wqe *wqe = worker->wqe;
448	491	struct io_wq *wq = wqe->wq;
449	492
450	493	do {
451		− unsigned hash = -1U;
452		−
	494	+ struct io_wq_work *work;
	495	+ unsigned int hash;
	496	+get_next:
453	497	/*
454	498	* If we got some work, mark us as busy. If we didn't, but
455	499	* the list isn't empty, it means we stalled on hashed work.
@@ -457,81 +501,60 @@ static void io_worker_handle_work(struct io_worker *worker) · 1 unmodified line
457	501	* can't make progress, any work completion or insertion will
458	502	* clear the stalled flag.
459	503	*/
460		− work = io_get_next_work(wqe, &hash);
	504	+ work = io_get_next_work(wqe);
461	505	if (work)
462	506	__io_worker_busy(wqe, worker, work);
463	507	else if (!wq_list_empty(&wqe->work_list))
464	508	wqe->flags \|= IO_WQE_FLAG_STALLED;
465	509
466	510	spin_unlock_irq(&wqe->lock);
467		− if (put_work && wq->put_work)
468		− wq->put_work(old_work);
469	511	if (!work)
470	512	break;
471		−next:
472		− /* flush any pending signals before assigning new work */
473		− if (signal_pending(current))
474		− flush_signals(current);
475		−
476		− cond_resched();
477		−
478		− spin_lock_irq(&worker->lock);
479		− worker->cur_work = work;
480		− spin_unlock_irq(&worker->lock);
481		−
482		− if (work->flags & IO_WQ_WORK_CB)
483		− work->func(&work);
484		−
485		− if (work->files && current->files != work->files) {
486		− task_lock(current);
487		− current->files = work->files;
488		− task_unlock(current);
489		− }
490		− if (work->fs && current->fs != work->fs)
491		− current->fs = work->fs;
492		− if (work->mm != worker->mm)
493		− io_wq_switch_mm(worker, work);
494		− if (worker->cur_creds != work->creds)
495		− io_wq_switch_creds(worker, work);
496		− /*
497		− * OK to set IO_WQ_WORK_CANCEL even for uncancellable work,
498		− * the worker function will do the right thing.
499		− */
500		− if (test_bit(IO_WQ_BIT_CANCEL, &wq->state))
501		− work->flags \|= IO_WQ_WORK_CANCEL;
502		− if (worker->mm)
503		− work->flags \|= IO_WQ_WORK_HAS_MM;
504		−
505		− if (wq->get_work) {
506		− put_work = work;
507		− wq->get_work(work);
508		− }
509		−
510		− old_work = work;
511		− work->func(&work);
512		−
513		− spin_lock_irq(&worker->lock);
514		− worker->cur_work = NULL;
515		− spin_unlock_irq(&worker->lock);
516		−
517		− spin_lock_irq(&wqe->lock);
518		−
519		− if (hash != -1U) {
520		− wqe->hash_map &= ~BIT_ULL(hash);
521		− wqe->flags &= ~IO_WQE_FLAG_STALLED;
522		− }
523		− if (work && work != old_work) {
524		− spin_unlock_irq(&wqe->lock);
525		−
526		− if (put_work && wq->put_work) {
527		− wq->put_work(put_work);
528		− put_work = NULL;
	513	+ io_assign_current_work(worker, work);
	514	+
	515	+ /* handle a whole dependent link */
	516	+ do {
	517	+ struct io_wq_work old_work, next_hashed, *linked;
	518	+
	519	+ next_hashed = wq_next_work(work);
	520	+ io_impersonate_work(worker, work);
	521	+ /*
	522	+ * OK to set IO_WQ_WORK_CANCEL even for uncancellable
	523	+ * work, the worker function will do the right thing.
	524	+ */
	525	+ if (test_bit(IO_WQ_BIT_CANCEL, &wq->state))
	526	+ work->flags \|= IO_WQ_WORK_CANCEL;
	527	+
	528	+ hash = io_get_work_hash(work);
	529	+ linked = old_work = work;
	530	+ linked->func(&linked);
	531	+ linked = (old_work == linked) ? NULL : linked;
	532	+
	533	+ work = next_hashed;
	534	+ if (!work && linked && !io_wq_is_hashed(linked)) {
	535	+ work = linked;
	536	+ linked = NULL;
529	537	}
	538	+ io_assign_current_work(worker, work);
	539	+ wq->free_work(old_work);
	540	+
	541	+ if (linked)
	542	+ io_wqe_enqueue(wqe, linked);
	543	+
	544	+ if (hash != -1U && !next_hashed) {
	545	+ spin_lock_irq(&wqe->lock);
	546	+ wqe->hash_map &= ~BIT_ULL(hash);
	547	+ wqe->flags &= ~IO_WQE_FLAG_STALLED;
	548	+ /* dependent work is not hashed */
	549	+ hash = -1U;
	550	+ /* skip unnecessary unlock-lock wqe->lock */
	551	+ if (!work)
	552	+ goto get_next;
	553	+ spin_unlock_irq(&wqe->lock);
	554	+ }
	555	+ } while (work);
530	556
531		− /* dependent work not hashed */
532		− hash = -1U;
533		− goto next;
534		− }
	557	+ spin_lock_irq(&wqe->lock);
535	558	} while (1);
536	559	}
537	560
@@ -747,17 +770,40 @@ static bool io_wq_can_queue(struct io_wqe wqe, struct io_wqe_acct acct, · 209 unmodified lines
747	770	return true;
748	771	}
749	772
750		−static void io_run_cancel(struct io_wq_work *work)
	773	+static void io_run_cancel(struct io_wq_work work, struct io_wqe wqe)
751	774	{
	775	+ struct io_wq *wq = wqe->wq;
	776	+
752	777	do {
753	778	struct io_wq_work *old_work = work;
754	779
755	780	work->flags \|= IO_WQ_WORK_CANCEL;
756	781	work->func(&work);
757	782	work = (work == old_work) ? NULL : work;
	783	+ wq->free_work(old_work);
758	784	} while (work);
759	785	}
760	786
	787	+static void io_wqe_insert_work(struct io_wqe wqe, struct io_wq_work work)
	788	+{
	789	+ unsigned int hash;
	790	+ struct io_wq_work *tail;
	791	+
	792	+ if (!io_wq_is_hashed(work)) {
	793	+append:
	794	+ wq_list_add_tail(&work->list, &wqe->work_list);
	795	+ return;
	796	+ }
	797	+
	798	+ hash = io_get_work_hash(work);
	799	+ tail = wqe->hash_tail[hash];
	800	+ wqe->hash_tail[hash] = work;
	801	+ if (!tail)
	802	+ goto append;
	803	+
	804	+ wq_list_add_after(&work->list, &tail->list, &wqe->work_list);
	805	+}
	806	+
761	807	static void io_wqe_enqueue(struct io_wqe wqe, struct io_wq_work work)
762	808	{
763	809	struct io_wqe_acct *acct = io_work_get_acct(wqe, work);
@@ -771,13 +817,13 @@ static void io_wqe_enqueue(struct io_wqe wqe, struct io_wq_work work) · 7 unmodified lines
771	817	* It's close enough to not be an issue, fork() has the same delay.
772	818	*/
773	819	if (unlikely(!io_wq_can_queue(wqe, acct, work))) {
774		− io_run_cancel(work);
	820	+ io_run_cancel(work, wqe);
775	821	return;
776	822	}
777	823
778	824	work_flags = work->flags;
779	825	spin_lock_irqsave(&wqe->lock, flags);
780		− wq_list_add_tail(&work->list, &wqe->work_list);
	826	+ io_wqe_insert_work(wqe, work);
781	827	wqe->flags &= ~IO_WQE_FLAG_STALLED;
782	828	spin_unlock_irqrestore(&wqe->lock, flags);
783	829
@@ -794,19 +840,15 @@ void io_wq_enqueue(struct io_wq wq, struct io_wq_work work) · 10 unmodified lines
794	840	}
795	841
796	842	/*
797		− * Enqueue work, hashed by some key. Work items that hash to the same value
798		− * will not be done in parallel. Used to limit concurrent writes, generally
799		− * hashed by inode.
	843	+ * Work items that hash to the same value will not be done in parallel.
	844	+ * Used to limit concurrent writes, generally hashed by inode.
800	845	*/
801		−void io_wq_enqueue_hashed(struct io_wq wq, struct io_wq_work work, void *val)
	846	+void io_wq_hash_work(struct io_wq_work work, void val)
802	847	{
803		− struct io_wqe *wqe = wq->wqes[numa_node_id()];
804		− unsigned bit;
805		−
	848	+ unsigned int bit;
806	849
807	850	bit = hash_ptr(val, IO_WQ_HASH_ORDER);
808	851	work->flags \|= (IO_WQ_WORK_HASHED \| (bit << IO_WQ_HASH_SHIFT));
809		− io_wqe_enqueue(wqe, work);
810	852	}
811	853
812	854	static bool io_wqe_worker_send_sig(struct io_worker worker, void data)
@@ -856,14 +898,13 @@ void io_wq_cancel_all(struct io_wq *wq) · 43 unmodified lines
856	898	}
857	899
858	900	struct io_cb_cancel_data {
859		− struct io_wqe *wqe;
860		− work_cancel_fn *cancel;
861		− void *caller_data;
	901	+ work_cancel_fn *fn;
	902	+ void *data;
862	903	};
863	904
864		−static bool io_work_cancel(struct io_worker worker, void cancel_data)
	905	+static bool io_wq_worker_cancel(struct io_worker worker, void data)
865	906	{
866		− struct io_cb_cancel_data *data = cancel_data;
	907	+ struct io_cb_cancel_data *match = data;
867	908	unsigned long flags;
868	909	bool ret = false;
869	910
@@ -874,83 +915,7 @@ static bool io_work_cancel(struct io_worker worker, void cancel_data) · 4 unmodified lines
874	915	spin_lock_irqsave(&worker->lock, flags);
875	916	if (worker->cur_work &&
876	917	!(worker->cur_work->flags & IO_WQ_WORK_NO_CANCEL) &&
877		− data->cancel(worker->cur_work, data->caller_data)) {
878		− send_sig(SIGINT, worker->task, 1);
879		− ret = true;
880		− }
881		− spin_unlock_irqrestore(&worker->lock, flags);
882		−
883		− return ret;
884		−}
885		−
886		−static enum io_wq_cancel io_wqe_cancel_cb_work(struct io_wqe *wqe,
887		− work_cancel_fn *cancel,
888		− void *cancel_data)
889		−{
890		− struct io_cb_cancel_data data = {
891		− .wqe = wqe,
892		− .cancel = cancel,
893		− .caller_data = cancel_data,
894		− };
895		− struct io_wq_work_node node, prev;
896		− struct io_wq_work *work;
897		− unsigned long flags;
898		− bool found = false;
899		−
900		− spin_lock_irqsave(&wqe->lock, flags);
901		− wq_list_for_each(node, prev, &wqe->work_list) {
902		− work = container_of(node, struct io_wq_work, list);
903		−
904		− if (cancel(work, cancel_data)) {
905		− wq_node_del(&wqe->work_list, node, prev);
906		− found = true;
907		− break;
908		− }
909		− }
910		− spin_unlock_irqrestore(&wqe->lock, flags);
911		−
912		− if (found) {
913		− io_run_cancel(work);
914		− return IO_WQ_CANCEL_OK;
915		− }
916		−
917		− rcu_read_lock();
918		− found = io_wq_for_each_worker(wqe, io_work_cancel, &data);
919		− rcu_read_unlock();
920		− return found ? IO_WQ_CANCEL_RUNNING : IO_WQ_CANCEL_NOTFOUND;
921		−}
922		−
923		−enum io_wq_cancel io_wq_cancel_cb(struct io_wq wq, work_cancel_fn cancel,
924		− void *data)
925		−{
926		− enum io_wq_cancel ret = IO_WQ_CANCEL_NOTFOUND;
927		− int node;
928		−
929		− for_each_node(node) {
930		− struct io_wqe *wqe = wq->wqes[node];
931		−
932		− ret = io_wqe_cancel_cb_work(wqe, cancel, data);
933		− if (ret != IO_WQ_CANCEL_NOTFOUND)
934		− break;
935		− }
936		−
937		− return ret;
938		−}
939		−
940		−struct work_match {
941		− bool (fn)(struct io_wq_work , void *data);
942		− void *data;
943		−};
944		−
945		−static bool io_wq_worker_cancel(struct io_worker worker, void data)
946		−{
947		− struct work_match *match = data;
948		− unsigned long flags;
949		− bool ret = false;
950		−
951		− spin_lock_irqsave(&worker->lock, flags);
952		− if (match->fn(worker->cur_work, match->data) &&
953		− !(worker->cur_work->flags & IO_WQ_WORK_NO_CANCEL)) {
	918	+ match->fn(worker->cur_work, match->data)) {
954	919	send_sig(SIGINT, worker->task, 1);
955	920	ret = true;
956	921	}
@@ -960,7 +925,7 @@ static bool io_wq_worker_cancel(struct io_worker worker, void data) · 3 unmodified lines
960	925	}
961	926
962	927	static enum io_wq_cancel io_wqe_cancel_work(struct io_wqe *wqe,
963		− struct work_match *match)
	928	+ struct io_cb_cancel_data *match)
964	929	{
965	930	struct io_wq_work_node node, prev;
966	931	struct io_wq_work *work;
@@ -977,7 +942,7 @@ static enum io_wq_cancel io_wqe_cancel_work(struct io_wqe *wqe, · 10 unmodified lines
977	942	work = container_of(node, struct io_wq_work, list);
978	943
979	944	if (match->fn(work, match->data)) {
980		− wq_node_del(&wqe->work_list, node, prev);
	945	+ wq_list_del(&wqe->work_list, node, prev);
981	946	found = true;
982	947	break;
983	948	}
@@ -985,7 +950,7 @@ static enum io_wq_cancel io_wqe_cancel_work(struct io_wqe *wqe, · 1 unmodified line
985	950	spin_unlock_irqrestore(&wqe->lock, flags);
986	951
987	952	if (found) {
988		− io_run_cancel(work);
	953	+ io_run_cancel(work, wqe);
989	954	return IO_WQ_CANCEL_OK;
990	955	}
991	956
@@ -1001,22 +966,16 @@ static enum io_wq_cancel io_wqe_cancel_work(struct io_wqe *wqe, · 9 unmodified lines
1001	966	return found ? IO_WQ_CANCEL_RUNNING : IO_WQ_CANCEL_NOTFOUND;
1002	967	}
1003	968
1004		−static bool io_wq_work_match(struct io_wq_work work, void data)
1005		−{
1006		− return work == data;
1007		−}
1008		−
1009		−enum io_wq_cancel io_wq_cancel_work(struct io_wq wq, struct io_wq_work cwork)
	969	+enum io_wq_cancel io_wq_cancel_cb(struct io_wq wq, work_cancel_fn cancel,
	970	+ void *data)
1010	971	{
1011		− struct work_match match = {
1012		− .fn = io_wq_work_match,
1013		− .data = cwork
	972	+ struct io_cb_cancel_data match = {
	973	+ .fn = cancel,
	974	+ .data = data,
1014	975	};
1015	976	enum io_wq_cancel ret = IO_WQ_CANCEL_NOTFOUND;
1016	977	int node;
1017	978
1018		− cwork->flags \|= IO_WQ_WORK_CANCEL;
1019		−
1020	979	for_each_node(node) {
1021	980	struct io_wqe *wqe = wq->wqes[node];
1022	981
@@ -1028,33 +987,28 @@ enum io_wq_cancel io_wq_cancel_work(struct io_wq wq, struct io_wq_work cwork) · 5 unmodified lines
1028	987	return ret;
1029	988	}
1030	989
	990	+static bool io_wq_io_cb_cancel_data(struct io_wq_work work, void data)
	991	+{
	992	+ return work == data;
	993	+}
	994	+
	995	+enum io_wq_cancel io_wq_cancel_work(struct io_wq wq, struct io_wq_work cwork)
	996	+{
	997	+ return io_wq_cancel_cb(wq, io_wq_io_cb_cancel_data, (void *)cwork);
	998	+}
	999	+
1031	1000	static bool io_wq_pid_match(struct io_wq_work work, void data)
1032	1001	{
1033	1002	pid_t pid = (pid_t) (unsigned long) data;
1034	1003
1035		− if (work)
1036		− return work->task_pid == pid;
1037		− return false;
	1004	+ return work->task_pid == pid;
1038	1005	}
1039	1006
1040	1007	enum io_wq_cancel io_wq_cancel_pid(struct io_wq *wq, pid_t pid)
1041	1008	{
1042		− struct work_match match = {
1043		− .fn = io_wq_pid_match,
1044		− .data = (void *) (unsigned long) pid
1045		− };
1046		− enum io_wq_cancel ret = IO_WQ_CANCEL_NOTFOUND;
1047		− int node;
1048		−
1049		− for_each_node(node) {
1050		− struct io_wqe *wqe = wq->wqes[node];
	1009	+ void data = (void ) (unsigned long) pid;
1051	1010
1052		− ret = io_wqe_cancel_work(wqe, &match);
1053		− if (ret != IO_WQ_CANCEL_NOTFOUND)
1054		− break;
1055		− }
1056		−
1057		− return ret;
	1011	+ return io_wq_cancel_cb(wq, io_wq_pid_match, data);
1058	1012	}
1059	1013
1060	1014	struct io_wq io_wq_create(unsigned bounded, struct io_wq_data data)
@@ -1062,6 +1016,9 @@ struct io_wq io_wq_create(unsigned bounded, struct io_wq_data data) · 1 unmodified line
1062	1016	int ret = -ENOMEM, node;
1063	1017	struct io_wq *wq;
1064	1018
	1019	+ if (WARN_ON_ONCE(!data->free_work))
	1020	+ return ERR_PTR(-EINVAL);
	1021	+
1065	1022	wq = kzalloc(sizeof(*wq), GFP_KERNEL);
1066	1023	if (!wq)
1067	1024	return ERR_PTR(-ENOMEM);
@@ -1072,8 +1029,7 @@ struct io_wq io_wq_create(unsigned bounded, struct io_wq_data data) · 4 unmodified lines
1072	1029	return ERR_PTR(-ENOMEM);
1073	1030	}
1074	1031
1075		− wq->get_work = data->get_work;
1076		− wq->put_work = data->put_work;
	1032	+ wq->free_work = data->free_work;
1077	1033
1078	1034	/* caller must already hold a reference to this */
1079	1035	wq->user = data->user;
@@ -1130,7 +1086,7 @@ err: · 50 unmodified lines
1130	1086
1131	1087	bool io_wq_get(struct io_wq wq, struct io_wq_data data)
1132	1088	{
1133		− if (data->get_work != wq->get_work \|\| data->put_work != wq->put_work)
	1089	+ if (data->free_work != wq->free_work)
1134	1090	return false;
1135	1091
1136	1092	return refcount_inc_not_zero(&wq->use_refs);

fs/io-wq.h+46−19

@@ -5,10 +5,8 @@ struct io_wq; · 4 unmodified lines
5	5
6	6	enum {
7	7	IO_WQ_WORK_CANCEL = 1,
8		− IO_WQ_WORK_HAS_MM = 2,
9	8	IO_WQ_WORK_HASHED = 4,
10	9	IO_WQ_WORK_UNBOUND = 32,
11		− IO_WQ_WORK_CB = 128,
12	10	IO_WQ_WORK_NO_CANCEL = 256,
13	11	IO_WQ_WORK_CONCURRENT = 512,
14	12
@@ -30,6 +28,18 @@ struct io_wq_work_list { · 15 unmodified lines
30	28	struct io_wq_work_node *last;
31	29	};
32	30
	31	+static inline void wq_list_add_after(struct io_wq_work_node *node,
	32	+ struct io_wq_work_node *pos,
	33	+ struct io_wq_work_list *list)
	34	+{
	35	+ struct io_wq_work_node *next = pos->next;
	36	+
	37	+ pos->next = node;
	38	+ node->next = next;
	39	+ if (!next)
	40	+ list->last = node;
	41	+}
	42	+
33	43	static inline void wq_list_add_tail(struct io_wq_work_node *node,
34	44	struct io_wq_work_list *list)
35	45	{
@@ -42,17 +52,26 @@ static inline void wq_list_add_tail(struct io_wq_work_node *node, · 6 unmodified lines
42	52	}
43	53	}
44	54
45		−static inline void wq_node_del(struct io_wq_work_list *list,
46		− struct io_wq_work_node *node,
	55	+static inline void wq_list_cut(struct io_wq_work_list *list,
	56	+ struct io_wq_work_node *last,
47	57	struct io_wq_work_node *prev)
48	58	{
49		− if (node == list->first)
50		− WRITE_ONCE(list->first, node->next);
51		− if (node == list->last)
	59	+ /* first in the list, if prev==NULL */
	60	+ if (!prev)
	61	+ WRITE_ONCE(list->first, last->next);
	62	+ else
	63	+ prev->next = last->next;
	64	+
	65	+ if (last == list->last)
52	66	list->last = prev;
53		− if (prev)
54		− prev->next = node->next;
55		− node->next = NULL;
	67	+ last->next = NULL;
	68	+}
	69	+
	70	+static inline void wq_list_del(struct io_wq_work_list *list,
	71	+ struct io_wq_work_node *node,
	72	+ struct io_wq_work_node *prev)
	73	+{
	74	+ wq_list_cut(list, node, prev);
56	75	}
57	76
58	77	#define wq_list_for_each(pos, prv, head) \
@@ -65,10 +84,7 @@ static inline void wq_node_del(struct io_wq_work_list *list, · 6 unmodified lines
65	84	} while (0)
66	85
67	86	struct io_wq_work {
68		− union {
69		− struct io_wq_work_node list;
70		− void *data;
71		− };
	87	+ struct io_wq_work_node list;
72	88	void (func)(struct io_wq_work *);
73	89	struct files_struct *files;
74	90	struct mm_struct *mm;
@@ -83,14 +99,20 @@ struct io_wq_work { · 8 unmodified lines
83	99	*(work) = (struct io_wq_work){ .func = _func }; \
84	100	} while (0) \
85	101
86		−typedef void (get_work_fn)(struct io_wq_work *);
87		−typedef void (put_work_fn)(struct io_wq_work *);
	102	+static inline struct io_wq_work wq_next_work(struct io_wq_work work)
	103	+{
	104	+ if (!work->list.next)
	105	+ return NULL;
	106	+
	107	+ return container_of(work->list.next, struct io_wq_work, list);
	108	+}
	109	+
	110	+typedef void (free_work_fn)(struct io_wq_work *);
88	111
89	112	struct io_wq_data {
90	113	struct user_struct *user;
91	114
92		− get_work_fn *get_work;
93		− put_work_fn *put_work;
	115	+ free_work_fn *free_work;
94	116	};
95	117
96	118	struct io_wq io_wq_create(unsigned bounded, struct io_wq_data data);
@@ -98,7 +120,12 @@ bool io_wq_get(struct io_wq wq, struct io_wq_data data); · 1 unmodified line
98	120	void io_wq_destroy(struct io_wq *wq);
99	121
100	122	void io_wq_enqueue(struct io_wq wq, struct io_wq_work work);
101		−void io_wq_enqueue_hashed(struct io_wq wq, struct io_wq_work work, void *val);
	123	+void io_wq_hash_work(struct io_wq_work work, void val);
	124	+
	125	+static inline bool io_wq_is_hashed(struct io_wq_work *work)
	126	+{
	127	+ return work->flags & IO_WQ_WORK_HASHED;
	128	+}
102	129
103	130	void io_wq_cancel_all(struct io_wq *wq);
104	131	enum io_wq_cancel io_wq_cancel_work(struct io_wq wq, struct io_wq_work cwork);

fs/io_uring.c+1406−609

@@ -44,6 +44,7 @@ · 43 unmodified lines
44	44	#include <linux/errno.h>
45	45	#include <linux/syscalls.h>
46	46	#include <linux/compat.h>
	47	+#include <net/compat.h>
47	48	#include <linux/refcount.h>
48	49	#include <linux/uio.h>
49	50	#include <linux/bits.h>
@@ -76,6 +77,8 @@ · 26 unmodified lines
76	77	#include <linux/fadvise.h>
77	78	#include <linux/eventpoll.h>
78	79	#include <linux/fs_struct.h>
	80	+#include <linux/splice.h>
	81	+#include <linux/task_work.h>
79	82
80	83	#define CREATE_TRACE_POINTS
81	84	#include <trace/events/io_uring.h>
@@ -193,6 +196,13 @@ struct fixed_file_data { · 111 unmodified lines
193	196	struct completion done;
194	197	};
195	198
	199	+struct io_buffer {
	200	+ struct list_head list;
	201	+ __u64 addr;
	202	+ __s32 len;
	203	+ __u16 bid;
	204	+};
	205	+
196	206	struct io_ring_ctx {
197	207	struct {
198	208	struct percpu_ref refs;
@@ -270,6 +280,8 @@ struct io_ring_ctx { · 71 unmodified lines
270	280	struct socket *ring_sock;
271	281	#endif
272	282
	283	+ struct idr io_buffer_idr;
	284	+
273	285	struct idr personality_idr;
274	286
275	287	struct {
@@ -290,7 +302,6 @@ struct io_ring_ctx { · 14 unmodified lines
290	302
291	303	struct {
292	304	spinlock_t completion_lock;
293		− struct llist_head poll_llist;
294	305
295	306	/*
296	307	* ->poll_list is protected by the ctx->uring_lock for
@@ -386,7 +397,9 @@ struct io_sr_msg { · 89 unmodified lines
386	397	void __user *buf;
387	398	};
388	399	int msg_flags;
	400	+ int bgid;
389	401	size_t len;
	402	+ struct io_buffer *kbuf;
390	403	};
391	404
392	405	struct io_open {
@@ -430,6 +443,24 @@ struct io_epoll { · 37 unmodified lines
430	443	struct epoll_event event;
431	444	};
432	445
	446	+struct io_splice {
	447	+ struct file *file_out;
	448	+ struct file *file_in;
	449	+ loff_t off_out;
	450	+ loff_t off_in;
	451	+ u64 len;
	452	+ unsigned int flags;
	453	+};
	454	+
	455	+struct io_provide_buf {
	456	+ struct file *file;
	457	+ __u64 addr;
	458	+ __s32 len;
	459	+ __u32 bgid;
	460	+ __u16 nbufs;
	461	+ __u16 bid;
	462	+};
	463	+
433	464	struct io_async_connect {
434	465	struct sockaddr_storage address;
435	466	};
@@ -464,6 +495,7 @@ enum { · 28 unmodified lines
464	495	REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
465	496	REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
466	497	REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
	498	+ REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,
467	499
468	500	REQ_F_LINK_NEXT_BIT,
469	501	REQ_F_FAIL_LINK_BIT,
@@ -479,6 +511,11 @@ enum { · 9 unmodified lines
479	511	REQ_F_COMP_LOCKED_BIT,
480	512	REQ_F_NEED_CLEANUP_BIT,
481	513	REQ_F_OVERFLOW_BIT,
	514	+ REQ_F_POLLED_BIT,
	515	+ REQ_F_BUFFER_SELECTED_BIT,
	516	+
	517	+ /* not a real bit, just to check we're not overflowing the space */
	518	+ __REQ_F_LAST_BIT,
482	519	};
483	520
484	521	enum {
@@ -492,6 +529,8 @@ enum { · 7 unmodified lines
492	529	REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
493	530	/* IOSQE_ASYNC */
494	531	REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
	532	+ /* IOSQE_BUFFER_SELECT */
	533	+ REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT),
495	534
496	535	/* already grabbed next link */
497	536	REQ_F_LINK_NEXT = BIT(REQ_F_LINK_NEXT_BIT),
@@ -521,6 +560,15 @@ enum { · 23 unmodified lines
521	560	REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT),
522	561	/* in overflow list */
523	562	REQ_F_OVERFLOW = BIT(REQ_F_OVERFLOW_BIT),
	563	+ /* already went through poll handler */
	564	+ REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),
	565	+ /* buffer already selected */
	566	+ REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
	567	+};
	568	+
	569	+struct async_poll {
	570	+ struct io_poll_iocb poll;
	571	+ struct io_wq_work work;
524	572	};
525	573
526	574	/*
@@ -546,32 +594,45 @@ struct io_kiocb { · 19 unmodified lines
546	594	struct io_fadvise fadvise;
547	595	struct io_madvise madvise;
548	596	struct io_epoll epoll;
	597	+ struct io_splice splice;
	598	+ struct io_provide_buf pbuf;
549	599	};
550	600
551	601	struct io_async_ctx *io;
552		− /*
553		− * llist_node is only used for poll deferred completions
554		− */
555		− struct llist_node llist_node;
556		− bool in_async;
557	602	bool needs_fixed_file;
558	603	u8 opcode;
559	604
560	605	struct io_ring_ctx *ctx;
561		− union {
562		− struct list_head list;
563		− struct hlist_node hash_node;
564		− };
565		− struct list_head link_list;
	606	+ struct list_head list;
566	607	unsigned int flags;
567	608	refcount_t refs;
	609	+ union {
	610	+ struct task_struct *task;
	611	+ unsigned long fsize;
	612	+ };
568	613	u64 user_data;
569	614	u32 result;
570	615	u32 sequence;
571	616
	617	+ struct list_head link_list;
	618	+
572	619	struct list_head inflight_entry;
573	620
574		− struct io_wq_work work;
	621	+ union {
	622	+ /*
	623	+ * Only commands that never go async can use the below fields,
	624	+ * obviously. Right now only IORING_OP_POLL_ADD uses them, and
	625	+ * async armed poll handlers for regular commands. The latter
	626	+ * restore the work, if needed.
	627	+ */
	628	+ struct {
	629	+ struct callback_head task_work;
	630	+ struct hlist_node hash_node;
	631	+ struct async_poll *apoll;
	632	+ int cflags;
	633	+ };
	634	+ struct io_wq_work work;
	635	+ };
575	636	};
576	637
577	638	#define IO_PLUG_THRESHOLD 2
@@ -615,6 +676,11 @@ struct io_op_def { · 37 unmodified lines
615	676	unsigned file_table : 1;
616	677	/* needs ->fs */
617	678	unsigned needs_fs : 1;
	679	+ /* set if opcode supports polled "wait" */
	680	+ unsigned pollin : 1;
	681	+ unsigned pollout : 1;
	682	+ /* op supports buffer selection */
	683	+ unsigned buffer_select : 1;
618	684	};
619	685
620	686	static const struct io_op_def io_op_defs[] = {
@@ -624,6 +690,8 @@ static const struct io_op_def io_op_defs[] = { · 3 unmodified lines
624	690	.needs_mm = 1,
625	691	.needs_file = 1,
626	692	.unbound_nonreg_file = 1,
	693	+ .pollin = 1,
	694	+ .buffer_select = 1,
627	695	},
628	696	[IORING_OP_WRITEV] = {
629	697	.async_ctx = 1,
@@ -631,6 +699,7 @@ static const struct io_op_def io_op_defs[] = { · 1 unmodified line
631	699	.needs_file = 1,
632	700	.hash_reg_file = 1,
633	701	.unbound_nonreg_file = 1,
	702	+ .pollout = 1,
634	703	},
635	704	[IORING_OP_FSYNC] = {
636	705	.needs_file = 1,
@@ -638,11 +707,13 @@ static const struct io_op_def io_op_defs[] = { · 1 unmodified line
638	707	[IORING_OP_READ_FIXED] = {
639	708	.needs_file = 1,
640	709	.unbound_nonreg_file = 1,
	710	+ .pollin = 1,
641	711	},
642	712	[IORING_OP_WRITE_FIXED] = {
643	713	.needs_file = 1,
644	714	.hash_reg_file = 1,
645	715	.unbound_nonreg_file = 1,
	716	+ .pollout = 1,
646	717	},
647	718	[IORING_OP_POLL_ADD] = {
648	719	.needs_file = 1,
@@ -658,6 +729,7 @@ static const struct io_op_def io_op_defs[] = { · 9 unmodified lines
658	729	.needs_file = 1,
659	730	.unbound_nonreg_file = 1,
660	731	.needs_fs = 1,
	732	+ .pollout = 1,
661	733	},
662	734	[IORING_OP_RECVMSG] = {
663	735	.async_ctx = 1,
@@ -665,6 +737,8 @@ static const struct io_op_def io_op_defs[] = { · 1 unmodified line
665	737	.needs_file = 1,
666	738	.unbound_nonreg_file = 1,
667	739	.needs_fs = 1,
	740	+ .pollin = 1,
	741	+ .buffer_select = 1,
668	742	},
669	743	[IORING_OP_TIMEOUT] = {
670	744	.async_ctx = 1,
@@ -676,6 +750,7 @@ static const struct io_op_def io_op_defs[] = { · 5 unmodified lines
676	750	.needs_file = 1,
677	751	.unbound_nonreg_file = 1,
678	752	.file_table = 1,
	753	+ .pollin = 1,
679	754	},
680	755	[IORING_OP_ASYNC_CANCEL] = {},
681	756	[IORING_OP_LINK_TIMEOUT] = {
@@ -687,6 +762,7 @@ static const struct io_op_def io_op_defs[] = { · 5 unmodified lines
687	762	.needs_mm = 1,
688	763	.needs_file = 1,
689	764	.unbound_nonreg_file = 1,
	765	+ .pollout = 1,
690	766	},
691	767	[IORING_OP_FALLOCATE] = {
692	768	.needs_file = 1,
@@ -715,11 +791,14 @@ static const struct io_op_def io_op_defs[] = { · 22 unmodified lines
715	791	.needs_mm = 1,
716	792	.needs_file = 1,
717	793	.unbound_nonreg_file = 1,
	794	+ .pollin = 1,
	795	+ .buffer_select = 1,
718	796	},
719	797	[IORING_OP_WRITE] = {
720	798	.needs_mm = 1,
721	799	.needs_file = 1,
722	800	.unbound_nonreg_file = 1,
	801	+ .pollout = 1,
723	802	},
724	803	[IORING_OP_FADVISE] = {
725	804	.needs_file = 1,
@@ -731,11 +810,14 @@ static const struct io_op_def io_op_defs[] = { · 5 unmodified lines
731	810	.needs_mm = 1,
732	811	.needs_file = 1,
733	812	.unbound_nonreg_file = 1,
	813	+ .pollout = 1,
734	814	},
735	815	[IORING_OP_RECV] = {
736	816	.needs_mm = 1,
737	817	.needs_file = 1,
738	818	.unbound_nonreg_file = 1,
	819	+ .pollin = 1,
	820	+ .buffer_select = 1,
739	821	},
740	822	[IORING_OP_OPENAT2] = {
741	823	.needs_file = 1,
@@ -747,6 +829,13 @@ static const struct io_op_def io_op_defs[] = { · 5 unmodified lines
747	829	.unbound_nonreg_file = 1,
748	830	.file_table = 1,
749	831	},
	832	+ [IORING_OP_SPLICE] = {
	833	+ .needs_file = 1,
	834	+ .hash_reg_file = 1,
	835	+ .unbound_nonreg_file = 1,
	836	+ },
	837	+ [IORING_OP_PROVIDE_BUFFERS] = {},
	838	+ [IORING_OP_REMOVE_BUFFERS] = {},
750	839	};
751	840
752	841	static void io_wq_submit_work(struct io_wq_work **workptr);
@@ -761,6 +850,10 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, · 8 unmodified lines
761	850	static int io_grab_files(struct io_kiocb *req);
762	851	static void io_ring_file_ref_flush(struct fixed_file_data *data);
763	852	static void io_cleanup_req(struct io_kiocb *req);
	853	+static int io_file_get(struct io_submit_state state, struct io_kiocb req,
	854	+ int fd, struct file **out_file, bool fixed);
	855	+static void __io_queue_sqe(struct io_kiocb *req,
	856	+ const struct io_uring_sqe *sqe);
764	857
765	858	static struct kmem_cache *req_cachep;
766	859
@@ -827,11 +920,11 @@ static struct io_ring_ctx io_ring_ctx_alloc(struct io_uring_params p) · 60 unmodified lines
827	920	INIT_LIST_HEAD(&ctx->cq_overflow_list);
828	921	init_completion(&ctx->completions[0]);
829	922	init_completion(&ctx->completions[1]);
	923	+ idr_init(&ctx->io_buffer_idr);
830	924	idr_init(&ctx->personality_idr);
831	925	mutex_init(&ctx->uring_lock);
832	926	init_waitqueue_head(&ctx->wait);
833	927	spin_lock_init(&ctx->completion_lock);
834		− init_llist_head(&ctx->poll_llist);
835	928	INIT_LIST_HEAD(&ctx->poll_list);
836	929	INIT_LIST_HEAD(&ctx->defer_list);
837	930	INIT_LIST_HEAD(&ctx->timeout_list);
@@ -952,15 +1045,14 @@ static inline void io_req_work_drop_env(struct io_kiocb *req) · 114 unmodified lines
952	1045	}
953	1046	}
954	1047
955		−static inline bool io_prep_async_work(struct io_kiocb *req,
	1048	+static inline void io_prep_async_work(struct io_kiocb *req,
956	1049	struct io_kiocb **link)
957	1050	{
958	1051	const struct io_op_def *def = &io_op_defs[req->opcode];
959		− bool do_hashed = false;
960	1052
961	1053	if (req->flags & REQ_F_ISREG) {
962	1054	if (def->hash_reg_file)
963		− do_hashed = true;
	1055	+ io_wq_hash_work(&req->work, file_inode(req->file));
964	1056	} else {
965	1057	if (def->unbound_nonreg_file)
966	1058	req->work.flags \|= IO_WQ_WORK_UNBOUND;
@@ -969,25 +1061,18 @@ static inline bool io_prep_async_work(struct io_kiocb *req, · 2 unmodified lines
969	1061	io_req_work_grab_env(req, def);
970	1062
971	1063	*link = io_prep_linked_timeout(req);
972		− return do_hashed;
973	1064	}
974	1065
975	1066	static inline void io_queue_async_work(struct io_kiocb *req)
976	1067	{
977	1068	struct io_ring_ctx *ctx = req->ctx;
978	1069	struct io_kiocb *link;
979		− bool do_hashed;
980	1070
981		− do_hashed = io_prep_async_work(req, &link);
	1071	+ io_prep_async_work(req, &link);
982	1072
983		− trace_io_uring_queue_async_work(ctx, do_hashed, req, &req->work,
984		− req->flags);
985		− if (!do_hashed) {
986		− io_wq_enqueue(ctx->io_wq, &req->work);
987		− } else {
988		− io_wq_enqueue_hashed(ctx->io_wq, &req->work,
989		− file_inode(req->file));
990		− }
	1073	+ trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
	1074	+ &req->work, req->flags);
	1075	+ io_wq_enqueue(ctx->io_wq, &req->work);
991	1076
992	1077	if (link)
993	1078	io_queue_linked_timeout(link);
@@ -1054,24 +1139,19 @@ static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx) · 60 unmodified lines
1054	1139	return false;
1055	1140	if (!ctx->eventfd_async)
1056	1141	return true;
1057		− return io_wq_current_is_worker() \|\| in_interrupt();
	1142	+ return io_wq_current_is_worker();
1058	1143	}
1059	1144
1060		−static void __io_cqring_ev_posted(struct io_ring_ctx *ctx, bool trigger_ev)
	1145	+static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1061	1146	{
1062	1147	if (waitqueue_active(&ctx->wait))
1063	1148	wake_up(&ctx->wait);
1064	1149	if (waitqueue_active(&ctx->sqo_wait))
1065	1150	wake_up(&ctx->sqo_wait);
1066		− if (trigger_ev)
	1151	+ if (io_should_trigger_evfd(ctx))
1067	1152	eventfd_signal(ctx->cq_ev_fd, 1);
1068	1153	}
1069	1154
1070		−static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1071		−{
1072		− __io_cqring_ev_posted(ctx, io_should_trigger_evfd(ctx));
1073		−}
1074		−
1075	1155	/* Returns true if there are no backlogged entries after the flush */
1076	1156	static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
1077	1157	{
@@ -1108,7 +1188,7 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) · 30 unmodified lines
1108	1188	if (cqe) {
1109	1189	WRITE_ONCE(cqe->user_data, req->user_data);
1110	1190	WRITE_ONCE(cqe->res, req->result);
1111		− WRITE_ONCE(cqe->flags, 0);
	1191	+ WRITE_ONCE(cqe->flags, req->cflags);
1112	1192	} else {
1113	1193	WRITE_ONCE(ctx->rings->cq_overflow,
1114	1194	atomic_inc_return(&ctx->cached_cq_overflow));
@@ -1132,7 +1212,7 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) · 17 unmodified lines
1132	1212	return cqe != NULL;
1133	1213	}
1134	1214
1135		−static void io_cqring_fill_event(struct io_kiocb *req, long res)
	1215	+static void __io_cqring_fill_event(struct io_kiocb *req, long res, long cflags)
1136	1216	{
1137	1217	struct io_ring_ctx *ctx = req->ctx;
1138	1218	struct io_uring_cqe *cqe;
@@ -1148,7 +1228,7 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res) · 9 unmodified lines
1148	1228	if (likely(cqe)) {
1149	1229	WRITE_ONCE(cqe->user_data, req->user_data);
1150	1230	WRITE_ONCE(cqe->res, res);
1151		− WRITE_ONCE(cqe->flags, 0);
	1231	+ WRITE_ONCE(cqe->flags, cflags);
1152	1232	} else if (ctx->cq_overflow_flushed) {
1153	1233	WRITE_ONCE(ctx->rings->cq_overflow,
1154	1234	atomic_inc_return(&ctx->cached_cq_overflow));
@@ -1160,23 +1240,34 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res) · 5 unmodified lines
1160	1240	req->flags \|= REQ_F_OVERFLOW;
1161	1241	refcount_inc(&req->refs);
1162	1242	req->result = res;
	1243	+ req->cflags = cflags;
1163	1244	list_add_tail(&req->list, &ctx->cq_overflow_list);
1164	1245	}
1165	1246	}
1166	1247
1167		−static void io_cqring_add_event(struct io_kiocb *req, long res)
	1248	+static void io_cqring_fill_event(struct io_kiocb *req, long res)
	1249	+{
	1250	+ __io_cqring_fill_event(req, res, 0);
	1251	+}
	1252	+
	1253	+static void __io_cqring_add_event(struct io_kiocb *req, long res, long cflags)
1168	1254	{
1169	1255	struct io_ring_ctx *ctx = req->ctx;
1170	1256	unsigned long flags;
1171	1257
1172	1258	spin_lock_irqsave(&ctx->completion_lock, flags);
1173		− io_cqring_fill_event(req, res);
	1259	+ __io_cqring_fill_event(req, res, cflags);
1174	1260	io_commit_cqring(ctx);
1175	1261	spin_unlock_irqrestore(&ctx->completion_lock, flags);
1176	1262
1177	1263	io_cqring_ev_posted(ctx);
1178	1264	}
1179	1265
	1266	+static void io_cqring_add_event(struct io_kiocb *req, long res)
	1267	+{
	1268	+ __io_cqring_add_event(req, res, 0);
	1269	+}
	1270	+
1180	1271	static inline bool io_is_fallback_req(struct io_kiocb *req)
1181	1272	{
1182	1273	return req == (struct io_kiocb *)
@@ -1246,6 +1337,15 @@ fallback: · 63 unmodified lines
1246	1337	return NULL;
1247	1338	}
1248	1339
	1340	+static inline void io_put_file(struct io_kiocb req, struct file file,
	1341	+ bool fixed)
	1342	+{
	1343	+ if (fixed)
	1344	+ percpu_ref_put(&req->ctx->file_data->refs);
	1345	+ else
	1346	+ fput(file);
	1347	+}
	1348	+
1249	1349	static void __io_req_do_free(struct io_kiocb *req)
1250	1350	{
1251	1351	if (likely(!io_is_fallback_req(req)))
@@ -1256,18 +1356,12 @@ static void __io_req_do_free(struct io_kiocb *req) · 4 unmodified lines
1256	1356
1257	1357	static void __io_req_aux_free(struct io_kiocb *req)
1258	1358	{
1259		− struct io_ring_ctx *ctx = req->ctx;
1260		−
1261	1359	if (req->flags & REQ_F_NEED_CLEANUP)
1262	1360	io_cleanup_req(req);
1263	1361
1264	1362	kfree(req->io);
1265		− if (req->file) {
1266		− if (req->flags & REQ_F_FIXED_FILE)
1267		− percpu_ref_put(&ctx->file_data->refs);
1268		− else
1269		− fput(req->file);
1270		− }
	1363	+ if (req->file)
	1364	+ io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE));
1271	1365
1272	1366	io_req_work_drop_env(req);
1273	1367	}
@@ -1474,6 +1568,30 @@ static void io_free_req(struct io_kiocb *req) · 200 unmodified lines
1474	1568	io_queue_async_work(nxt);
1475	1569	}
1476	1570
	1571	+static void io_link_work_cb(struct io_wq_work **workptr)
	1572	+{
	1573	+ struct io_kiocb req = container_of(workptr, struct io_kiocb, work);
	1574	+ struct io_kiocb *link;
	1575	+
	1576	+ link = list_first_entry(&req->link_list, struct io_kiocb, link_list);
	1577	+ io_queue_linked_timeout(link);
	1578	+ io_wq_submit_work(workptr);
	1579	+}
	1580	+
	1581	+static void io_wq_assign_next(struct io_wq_work *workptr, struct io_kiocb nxt)
	1582	+{
	1583	+ struct io_kiocb *link;
	1584	+ const struct io_op_def *def = &io_op_defs[nxt->opcode];
	1585	+
	1586	+ if ((nxt->flags & REQ_F_ISREG) && def->hash_reg_file)
	1587	+ io_wq_hash_work(&nxt->work, file_inode(nxt->file));
	1588	+
	1589	+ *workptr = &nxt->work;
	1590	+ link = io_prep_linked_timeout(nxt);
	1591	+ if (link)
	1592	+ nxt->work.func = io_link_work_cb;
	1593	+}
	1594	+
1477	1595	/*
1478	1596	* Drop reference to request, return next in chain (if there is one) if this
1479	1597	* was the last reference to this request.
@@ -1493,6 +1611,26 @@ static void io_put_req(struct io_kiocb *req) · 13 unmodified lines
1493	1611	io_free_req(req);
1494	1612	}
1495	1613
	1614	+static void io_steal_work(struct io_kiocb *req,
	1615	+ struct io_wq_work **workptr)
	1616	+{
	1617	+ /*
	1618	+ * It's in an io-wq worker, so there always should be at least
	1619	+ * one reference, which will be dropped in io_put_work() just
	1620	+ * after the current handler returns.
	1621	+ *
	1622	+ * It also means, that if the counter dropped to 1, then there is
	1623	+ * no asynchronous users left, so it's safe to steal the next work.
	1624	+ */
	1625	+ if (refcount_read(&req->refs) == 1) {
	1626	+ struct io_kiocb *nxt = NULL;
	1627	+
	1628	+ io_req_find_next(req, &nxt);
	1629	+ if (nxt)
	1630	+ io_wq_assign_next(workptr, nxt);
	1631	+ }
	1632	+}
	1633	+
1496	1634	/*
1497	1635	* Must only be used if we don't need to care about links, usually from
1498	1636	* within the completion handling itself.
@@ -1554,6 +1692,19 @@ static inline bool io_req_multi_free(struct req_batch rb, struct io_kiocb req) · 55 unmodified lines
1554	1692	return true;
1555	1693	}
1556	1694
	1695	+static int io_put_kbuf(struct io_kiocb *req)
	1696	+{
	1697	+ struct io_buffer *kbuf;
	1698	+ int cflags;
	1699	+
	1700	+ kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
	1701	+ cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
	1702	+ cflags \|= IORING_CQE_F_BUFFER;
	1703	+ req->rw.addr = 0;
	1704	+ kfree(kbuf);
	1705	+ return cflags;
	1706	+}
	1707	+
1557	1708	/*
1558	1709	* Find and free completed poll iocbs
1559	1710	*/
@@ -1565,10 +1716,15 @@ static void io_iopoll_complete(struct io_ring_ctx ctx, unsigned int nr_events, · 5 unmodified lines
1565	1716
1566	1717	rb.to_free = rb.need_iter = 0;
1567	1718	while (!list_empty(done)) {
	1719	+ int cflags = 0;
	1720	+
1568	1721	req = list_first_entry(done, struct io_kiocb, list);
1569	1722	list_del(&req->list);
1570	1723
1571		− io_cqring_fill_event(req, req->result);
	1724	+ if (req->flags & REQ_F_BUFFER_SELECTED)
	1725	+ cflags = io_put_kbuf(req);
	1726	+
	1727	+ __io_cqring_fill_event(req, req->result, cflags);
1572	1728	(*nr_events)++;
1573	1729
1574	1730	if (refcount_dec_and_test(&req->refs) &&
@@ -1577,6 +1733,8 @@ static void io_iopoll_complete(struct io_ring_ctx ctx, unsigned int nr_events, · 2 unmodified lines
1577	1733	}
1578	1734
1579	1735	io_commit_cqring(ctx);
	1736	+ if (ctx->flags & IORING_SETUP_SQPOLL)
	1737	+ io_cqring_ev_posted(ctx);
1580	1738	io_free_req_many(ctx, &rb);
1581	1739	}
1582	1740
@@ -1743,13 +1901,16 @@ static inline void req_set_fail_links(struct io_kiocb *req) · 160 unmodified lines
1743	1901	static void io_complete_rw_common(struct kiocb *kiocb, long res)
1744	1902	{
1745	1903	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
	1904	+ int cflags = 0;
1746	1905
1747	1906	if (kiocb->ki_flags & IOCB_WRITE)
1748	1907	kiocb_end_write(req);
1749	1908
1750	1909	if (res != req->result)
1751	1910	req_set_fail_links(req);
1752		− io_cqring_add_event(req, res);
	1911	+ if (req->flags & REQ_F_BUFFER_SELECTED)
	1912	+ cflags = io_put_kbuf(req);
	1913	+ __io_cqring_add_event(req, res, cflags);
1753	1914	}
1754	1915
1755	1916	static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
@@ -1760,17 +1921,6 @@ static void io_complete_rw(struct kiocb *kiocb, long res, long res2) · 4 unmodified lines
1760	1921	io_put_req(req);
1761	1922	}
1762	1923
1763		−static struct io_kiocb __io_complete_rw(struct kiocb kiocb, long res)
1764		−{
1765		− struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1766		− struct io_kiocb *nxt = NULL;
1767		−
1768		− io_complete_rw_common(kiocb, res);
1769		− io_put_req_find_next(req, &nxt);
1770		−
1771		− return nxt;
1772		−}
1773		−
1774	1924	static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
1775	1925	{
1776	1926	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
@@ -1841,7 +1991,7 @@ static void io_file_put(struct io_submit_state *state) · 64 unmodified lines
1841	1991	* assuming most submissions are for one file, or at least that each file
1842	1992	* has more than one submission.
1843	1993	*/
1844		−static struct file io_file_get(struct io_submit_state state, int fd)
	1994	+static struct file __io_file_get(struct io_submit_state state, int fd)
1845	1995	{
1846	1996	if (!state)
1847	1997	return fget(fd);
@@ -1938,7 +2088,7 @@ static int io_prep_rw(struct io_kiocb req, const struct io_uring_sqe sqe, · 90 unmodified lines
1938	2088
1939	2089	req->rw.addr = READ_ONCE(sqe->addr);
1940	2090	req->rw.len = READ_ONCE(sqe->len);
1941		− /* we own ->private, reuse it for the buffer index */
	2091	+ /* we own ->private, reuse it for the buffer index / buffer ID */
1942	2092	req->rw.kiocb.private = (void *) (unsigned long)
1943	2093	READ_ONCE(sqe->buf_index);
1944	2094	return 0;
@@ -1965,15 +2115,14 @@ static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret) · 20 unmodified lines
1965	2115	}
1966	2116	}
1967	2117
1968		−static void kiocb_done(struct kiocb kiocb, ssize_t ret, struct io_kiocb *nxt,
1969		− bool in_async)
	2118	+static void kiocb_done(struct kiocb *kiocb, ssize_t ret)
1970	2119	{
1971	2120	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1972	2121
1973	2122	if (req->flags & REQ_F_CUR_POS)
1974	2123	req->file->f_pos = kiocb->ki_pos;
1975		− if (in_async && ret >= 0 && kiocb->ki_complete == io_complete_rw)
1976		− *nxt = __io_complete_rw(kiocb, ret);
	2124	+ if (ret >= 0 && kiocb->ki_complete == io_complete_rw)
	2125	+ io_complete_rw(kiocb, ret, 0);
1977	2126	else
1978	2127	io_rw_done(kiocb, ret);
1979	2128	}
@@ -2052,11 +2201,147 @@ static ssize_t io_import_fixed(struct io_kiocb *req, int rw, · 72 unmodified lines
2052	2201	return len;
2053	2202	}
2054	2203
	2204	+static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
	2205	+{
	2206	+ if (needs_lock)
	2207	+ mutex_unlock(&ctx->uring_lock);
	2208	+}
	2209	+
	2210	+static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
	2211	+{
	2212	+ /*
	2213	+ * "Normal" inline submissions always hold the uring_lock, since we
	2214	+ * grab it from the system call. Same is true for the SQPOLL offload.
	2215	+ * The only exception is when we've detached the request and issue it
	2216	+ * from an async worker thread, grab the lock for that case.
	2217	+ */
	2218	+ if (needs_lock)
	2219	+ mutex_lock(&ctx->uring_lock);
	2220	+}
	2221	+
	2222	+static struct io_buffer io_buffer_select(struct io_kiocb req, size_t *len,
	2223	+ int bgid, struct io_buffer *kbuf,
	2224	+ bool needs_lock)
	2225	+{
	2226	+ struct io_buffer *head;
	2227	+
	2228	+ if (req->flags & REQ_F_BUFFER_SELECTED)
	2229	+ return kbuf;
	2230	+
	2231	+ io_ring_submit_lock(req->ctx, needs_lock);
	2232	+
	2233	+ lockdep_assert_held(&req->ctx->uring_lock);
	2234	+
	2235	+ head = idr_find(&req->ctx->io_buffer_idr, bgid);
	2236	+ if (head) {
	2237	+ if (!list_empty(&head->list)) {
	2238	+ kbuf = list_last_entry(&head->list, struct io_buffer,
	2239	+ list);
	2240	+ list_del(&kbuf->list);
	2241	+ } else {
	2242	+ kbuf = head;
	2243	+ idr_remove(&req->ctx->io_buffer_idr, bgid);
	2244	+ }
	2245	+ if (*len > kbuf->len)
	2246	+ *len = kbuf->len;
	2247	+ } else {
	2248	+ kbuf = ERR_PTR(-ENOBUFS);
	2249	+ }
	2250	+
	2251	+ io_ring_submit_unlock(req->ctx, needs_lock);
	2252	+
	2253	+ return kbuf;
	2254	+}
	2255	+
	2256	+static void __user io_rw_buffer_select(struct io_kiocb req, size_t *len,
	2257	+ bool needs_lock)
	2258	+{
	2259	+ struct io_buffer *kbuf;
	2260	+ int bgid;
	2261	+
	2262	+ kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
	2263	+ bgid = (int) (unsigned long) req->rw.kiocb.private;
	2264	+ kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock);
	2265	+ if (IS_ERR(kbuf))
	2266	+ return kbuf;
	2267	+ req->rw.addr = (u64) (unsigned long) kbuf;
	2268	+ req->flags \|= REQ_F_BUFFER_SELECTED;
	2269	+ return u64_to_user_ptr(kbuf->addr);
	2270	+}
	2271	+
	2272	+#ifdef CONFIG_COMPAT
	2273	+static ssize_t io_compat_import(struct io_kiocb req, struct iovec iov,
	2274	+ bool needs_lock)
	2275	+{
	2276	+ struct compat_iovec __user *uiov;
	2277	+ compat_ssize_t clen;
	2278	+ void __user *buf;
	2279	+ ssize_t len;
	2280	+
	2281	+ uiov = u64_to_user_ptr(req->rw.addr);
	2282	+ if (!access_ok(uiov, sizeof(*uiov)))
	2283	+ return -EFAULT;
	2284	+ if (__get_user(clen, &uiov->iov_len))
	2285	+ return -EFAULT;
	2286	+ if (clen < 0)
	2287	+ return -EINVAL;
	2288	+
	2289	+ len = clen;
	2290	+ buf = io_rw_buffer_select(req, &len, needs_lock);
	2291	+ if (IS_ERR(buf))
	2292	+ return PTR_ERR(buf);
	2293	+ iov[0].iov_base = buf;
	2294	+ iov[0].iov_len = (compat_size_t) len;
	2295	+ return 0;
	2296	+}
	2297	+#endif
	2298	+
	2299	+static ssize_t __io_iov_buffer_select(struct io_kiocb req, struct iovec iov,
	2300	+ bool needs_lock)
	2301	+{
	2302	+ struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr);
	2303	+ void __user *buf;
	2304	+ ssize_t len;
	2305	+
	2306	+ if (copy_from_user(iov, uiov, sizeof(*uiov)))
	2307	+ return -EFAULT;
	2308	+
	2309	+ len = iov[0].iov_len;
	2310	+ if (len < 0)
	2311	+ return -EINVAL;
	2312	+ buf = io_rw_buffer_select(req, &len, needs_lock);
	2313	+ if (IS_ERR(buf))
	2314	+ return PTR_ERR(buf);
	2315	+ iov[0].iov_base = buf;
	2316	+ iov[0].iov_len = len;
	2317	+ return 0;
	2318	+}
	2319	+
	2320	+static ssize_t io_iov_buffer_select(struct io_kiocb req, struct iovec iov,
	2321	+ bool needs_lock)
	2322	+{
	2323	+ if (req->flags & REQ_F_BUFFER_SELECTED)
	2324	+ return 0;
	2325	+ if (!req->rw.len)
	2326	+ return 0;
	2327	+ else if (req->rw.len > 1)
	2328	+ return -EINVAL;
	2329	+
	2330	+#ifdef CONFIG_COMPAT
	2331	+ if (req->ctx->compat)
	2332	+ return io_compat_import(req, iov, needs_lock);
	2333	+#endif
	2334	+
	2335	+ return __io_iov_buffer_select(req, iov, needs_lock);
	2336	+}
	2337	+
2055	2338	static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
2056		− struct iovec *iovec, struct iov_iter iter)
	2339	+ struct iovec *iovec, struct iov_iter iter,
	2340	+ bool needs_lock)
2057	2341	{
2058	2342	void __user *buf = u64_to_user_ptr(req->rw.addr);
2059	2343	size_t sqe_len = req->rw.len;
	2344	+ ssize_t ret;
2060	2345	u8 opcode;
2061	2346
2062	2347	opcode = req->opcode;
@@ -2065,12 +2350,20 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req, · 2 unmodified lines
2065	2350	return io_import_fixed(req, rw, iter);
2066	2351	}
2067	2352
2068		− /* buffer index only valid with fixed read/write */
2069		− if (req->rw.kiocb.private)
	2353	+ /* buffer index only valid with fixed read/write, or buffer select */
	2354	+ if (req->rw.kiocb.private && !(req->flags & REQ_F_BUFFER_SELECT))
2070	2355	return -EINVAL;
2071	2356
2072	2357	if (opcode == IORING_OP_READ \|\| opcode == IORING_OP_WRITE) {
2073		− ssize_t ret;
	2358	+ if (req->flags & REQ_F_BUFFER_SELECT) {
	2359	+ buf = io_rw_buffer_select(req, &sqe_len, needs_lock);
	2360	+ if (IS_ERR(buf)) {
	2361	+ *iovec = NULL;
	2362	+ return PTR_ERR(buf);
	2363	+ }
	2364	+ req->rw.len = sqe_len;
	2365	+ }
	2366	+
2074	2367	ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
2075	2368	*iovec = NULL;
2076	2369	return ret < 0 ? ret : sqe_len;
@@ -2086,6 +2379,16 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req, · 9 unmodified lines
2086	2379	return iorw->size;
2087	2380	}
2088	2381
	2382	+ if (req->flags & REQ_F_BUFFER_SELECT) {
	2383	+ ret = io_iov_buffer_select(req, *iovec, needs_lock);
	2384	+ if (!ret) {
	2385	+ ret = (*iovec)->iov_len;
	2386	+ iov_iter_init(iter, rw, *iovec, 1, ret);
	2387	+ }
	2388	+ *iovec = NULL;
	2389	+ return ret;
	2390	+ }
	2391	+
2089	2392	#ifdef CONFIG_COMPAT
2090	2393	if (req->ctx->compat)
2091	2394	return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
@@ -2169,12 +2472,18 @@ static void io_req_map_rw(struct io_kiocb *req, ssize_t io_size, · 77 unmodified lines
2169	2472	}
2170	2473	}
2171	2474
	2475	+static inline int __io_alloc_async_ctx(struct io_kiocb *req)
	2476	+{
	2477	+ req->io = kmalloc(sizeof(*req->io), GFP_KERNEL);
	2478	+ return req->io == NULL;
	2479	+}
	2480	+
2172	2481	static int io_alloc_async_ctx(struct io_kiocb *req)
2173	2482	{
2174	2483	if (!io_op_defs[req->opcode].async_ctx)
2175	2484	return 0;
2176		− req->io = kmalloc(sizeof(*req->io), GFP_KERNEL);
2177		− return req->io == NULL;
	2485	+
	2486	+ return __io_alloc_async_ctx(req);
2178	2487	}
2179	2488
2180	2489	static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size,
@@ -2184,7 +2493,7 @@ static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size, · 3 unmodified lines
2184	2493	if (!io_op_defs[req->opcode].async_ctx)
2185	2494	return 0;
2186	2495	if (!req->io) {
2187		− if (io_alloc_async_ctx(req))
	2496	+ if (__io_alloc_async_ctx(req))
2188	2497	return -ENOMEM;
2189	2498
2190	2499	io_req_map_rw(req, io_size, iovec, fast_iov, iter);
@@ -2213,7 +2522,7 @@ static int io_read_prep(struct io_kiocb req, const struct io_uring_sqe sqe, · 22 unmodified lines
2213	2522	io = req->io;
2214	2523	io->rw.iov = io->rw.fast_iov;
2215	2524	req->io = NULL;
2216		− ret = io_import_iovec(READ, req, &io->rw.iov, &iter);
	2525	+ ret = io_import_iovec(READ, req, &io->rw.iov, &iter, !force_nonblock);
2217	2526	req->io = io;
2218	2527	if (ret < 0)
2219	2528	return ret;
@@ -2222,8 +2531,7 @@ static int io_read_prep(struct io_kiocb req, const struct io_uring_sqe sqe, · 2 unmodified lines
2222	2531	return 0;
2223	2532	}
2224	2533
2225		−static int io_read(struct io_kiocb req, struct io_kiocb *nxt,
2226		− bool force_nonblock)
	2534	+static int io_read(struct io_kiocb *req, bool force_nonblock)
2227	2535	{
2228	2536	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
2229	2537	struct kiocb *kiocb = &req->rw.kiocb;
@@ -2231,13 +2539,13 @@ static int io_read(struct io_kiocb req, struct io_kiocb *nxt, · 1 unmodified line
2231	2539	size_t iov_count;
2232	2540	ssize_t io_size, ret;
2233	2541
2234		− ret = io_import_iovec(READ, req, &iovec, &iter);
	2542	+ ret = io_import_iovec(READ, req, &iovec, &iter, !force_nonblock);
2235	2543	if (ret < 0)
2236	2544	return ret;
2237	2545
2238	2546	/* Ensure we clear previously set non-block flag */
2239	2547	if (!force_nonblock)
2240		− req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
	2548	+ kiocb->ki_flags &= ~IOCB_NOWAIT;
2241	2549
2242	2550	req->result = 0;
2243	2551	io_size = ret;
@@ -2248,10 +2556,8 @@ static int io_read(struct io_kiocb req, struct io_kiocb *nxt, · 4 unmodified lines
2248	2556	* If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
2249	2557	* we know to async punt it even if it was opened O_NONBLOCK
2250	2558	*/
2251		− if (force_nonblock && !io_file_supports_async(req->file)) {
2252		− req->flags \|= REQ_F_MUST_PUNT;
	2559	+ if (force_nonblock && !io_file_supports_async(req->file))
2253	2560	goto copy_iov;
2254		− }
2255	2561
2256	2562	iov_count = iov_iter_count(&iter);
2257	2563	ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count);
@@ -2265,13 +2571,16 @@ static int io_read(struct io_kiocb req, struct io_kiocb *nxt, · 7 unmodified lines
2265	2571
2266	2572	/* Catch -EAGAIN return for forced non-blocking submission */
2267	2573	if (!force_nonblock \|\| ret2 != -EAGAIN) {
2268		− kiocb_done(kiocb, ret2, nxt, req->in_async);
	2574	+ kiocb_done(kiocb, ret2);
2269	2575	} else {
2270	2576	copy_iov:
2271	2577	ret = io_setup_async_rw(req, io_size, iovec,
2272	2578	inline_vecs, &iter);
2273	2579	if (ret)
2274	2580	goto out_free;
	2581	+ /* any defer here is final, must blocking retry */
	2582	+ if (!(req->flags & REQ_F_NOWAIT))
	2583	+ req->flags \|= REQ_F_MUST_PUNT;
2275	2584	return -EAGAIN;
2276	2585	}
2277	2586	}
@@ -2295,6 +2604,8 @@ static int io_write_prep(struct io_kiocb req, const struct io_uring_sqe sqe, · 17 unmodified lines
2295	2604	if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
2296	2605	return -EBADF;
2297	2606
	2607	+ req->fsize = rlimit(RLIMIT_FSIZE);
	2608	+
2298	2609	/* either don't need iovec imported or already have it */
2299	2610	if (!req->io \|\| req->flags & REQ_F_NEED_CLEANUP)
2300	2611	return 0;
@@ -2302,7 +2613,7 @@ static int io_write_prep(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
2302	2613	io = req->io;
2303	2614	io->rw.iov = io->rw.fast_iov;
2304	2615	req->io = NULL;
2305		− ret = io_import_iovec(WRITE, req, &io->rw.iov, &iter);
	2616	+ ret = io_import_iovec(WRITE, req, &io->rw.iov, &iter, !force_nonblock);
2306	2617	req->io = io;
2307	2618	if (ret < 0)
2308	2619	return ret;
@@ -2311,8 +2622,7 @@ static int io_write_prep(struct io_kiocb req, const struct io_uring_sqe sqe, · 2 unmodified lines
2311	2622	return 0;
2312	2623	}
2313	2624
2314		−static int io_write(struct io_kiocb req, struct io_kiocb *nxt,
2315		− bool force_nonblock)
	2625	+static int io_write(struct io_kiocb *req, bool force_nonblock)
2316	2626	{
2317	2627	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
2318	2628	struct kiocb *kiocb = &req->rw.kiocb;
@@ -2320,7 +2630,7 @@ static int io_write(struct io_kiocb req, struct io_kiocb *nxt, · 1 unmodified line
2320	2630	size_t iov_count;
2321	2631	ssize_t ret, io_size;
2322	2632
2323		− ret = io_import_iovec(WRITE, req, &iovec, &iter);
	2633	+ ret = io_import_iovec(WRITE, req, &iovec, &iter, !force_nonblock);
2324	2634	if (ret < 0)
2325	2635	return ret;
2326	2636
@@ -2337,10 +2647,8 @@ static int io_write(struct io_kiocb req, struct io_kiocb *nxt, · 10 unmodified lines
2337	2647	* If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
2338	2648	* we know to async punt it even if it was opened O_NONBLOCK
2339	2649	*/
2340		− if (force_nonblock && !io_file_supports_async(req->file)) {
2341		− req->flags \|= REQ_F_MUST_PUNT;
	2650	+ if (force_nonblock && !io_file_supports_async(req->file))
2342	2651	goto copy_iov;
2343		− }
2344	2652
2345	2653	/* file path doesn't support NOWAIT for non-direct_IO */
2346	2654	if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
@@ -2367,24 +2675,33 @@ static int io_write(struct io_kiocb req, struct io_kiocb *nxt, · 20 unmodified lines
2367	2675	}
2368	2676	kiocb->ki_flags \|= IOCB_WRITE;
2369	2677
	2678	+ if (!force_nonblock)
	2679	+ current->signal->rlim[RLIMIT_FSIZE].rlim_cur = req->fsize;
	2680	+
2370	2681	if (req->file->f_op->write_iter)
2371	2682	ret2 = call_write_iter(req->file, kiocb, &iter);
2372	2683	else
2373	2684	ret2 = loop_rw_iter(WRITE, req->file, kiocb, &iter);
	2685	+
	2686	+ if (!force_nonblock)
	2687	+ current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY;
	2688	+
2374	2689	/*
2375		− * Raw bdev writes will -EOPNOTSUPP for IOCB_NOWAIT. Just
	2690	+ * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
2376	2691	* retry them without IOCB_NOWAIT.
2377	2692	*/
2378	2693	if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
2379	2694	ret2 = -EAGAIN;
2380	2695	if (!force_nonblock \|\| ret2 != -EAGAIN) {
2381		− kiocb_done(kiocb, ret2, nxt, req->in_async);
	2696	+ kiocb_done(kiocb, ret2);
2382	2697	} else {
2383	2698	copy_iov:
2384	2699	ret = io_setup_async_rw(req, io_size, iovec,
2385	2700	inline_vecs, &iter);
2386	2701	if (ret)
2387	2702	goto out_free;
	2703	+ /* any defer here is final, must blocking retry */
	2704	+ req->flags \|= REQ_F_MUST_PUNT;
2388	2705	return -EAGAIN;
2389	2706	}
2390	2707	}
@@ -2394,6 +2711,76 @@ out_free: · 3 unmodified lines
2394	2711	return ret;
2395	2712	}
2396	2713
	2714	+static int io_splice_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
	2715	+{
	2716	+ struct io_splice* sp = &req->splice;
	2717	+ unsigned int valid_flags = SPLICE_F_FD_IN_FIXED \| SPLICE_F_ALL;
	2718	+ int ret;
	2719	+
	2720	+ if (req->flags & REQ_F_NEED_CLEANUP)
	2721	+ return 0;
	2722	+
	2723	+ sp->file_in = NULL;
	2724	+ sp->off_in = READ_ONCE(sqe->splice_off_in);
	2725	+ sp->off_out = READ_ONCE(sqe->off);
	2726	+ sp->len = READ_ONCE(sqe->len);
	2727	+ sp->flags = READ_ONCE(sqe->splice_flags);
	2728	+
	2729	+ if (unlikely(sp->flags & ~valid_flags))
	2730	+ return -EINVAL;
	2731	+
	2732	+ ret = io_file_get(NULL, req, READ_ONCE(sqe->splice_fd_in), &sp->file_in,
	2733	+ (sp->flags & SPLICE_F_FD_IN_FIXED));
	2734	+ if (ret)
	2735	+ return ret;
	2736	+ req->flags \|= REQ_F_NEED_CLEANUP;
	2737	+
	2738	+ if (!S_ISREG(file_inode(sp->file_in)->i_mode))
	2739	+ req->work.flags \|= IO_WQ_WORK_UNBOUND;
	2740	+
	2741	+ return 0;
	2742	+}
	2743	+
	2744	+static bool io_splice_punt(struct file *file)
	2745	+{
	2746	+ if (get_pipe_info(file))
	2747	+ return false;
	2748	+ if (!io_file_supports_async(file))
	2749	+ return true;
	2750	+ return !(file->f_mode & O_NONBLOCK);
	2751	+}
	2752	+
	2753	+static int io_splice(struct io_kiocb *req, bool force_nonblock)
	2754	+{
	2755	+ struct io_splice *sp = &req->splice;
	2756	+ struct file *in = sp->file_in;
	2757	+ struct file *out = sp->file_out;
	2758	+ unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
	2759	+ loff_t poff_in, poff_out;
	2760	+ long ret;
	2761	+
	2762	+ if (force_nonblock) {
	2763	+ if (io_splice_punt(in) \|\| io_splice_punt(out))
	2764	+ return -EAGAIN;
	2765	+ flags \|= SPLICE_F_NONBLOCK;
	2766	+ }
	2767	+
	2768	+ poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
	2769	+ poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
	2770	+ ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
	2771	+ if (force_nonblock && ret == -EAGAIN)
	2772	+ return -EAGAIN;
	2773	+
	2774	+ io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
	2775	+ req->flags &= ~REQ_F_NEED_CLEANUP;
	2776	+
	2777	+ io_cqring_add_event(req, ret);
	2778	+ if (ret != sp->len)
	2779	+ req_set_fail_links(req);
	2780	+ io_put_req(req);
	2781	+ return 0;
	2782	+}
	2783	+
2397	2784	/*
2398	2785	* IORING_OP_NOP just posts a completion event, nothing else.
2399	2786	*/
@@ -2442,85 +2829,63 @@ static bool io_req_cancelled(struct io_kiocb *req) · 42 unmodified lines
2442	2829	return false;
2443	2830	}
2444	2831
2445		−static void io_link_work_cb(struct io_wq_work **workptr)
2446		−{
2447		− struct io_wq_work work = workptr;
2448		− struct io_kiocb *link = work->data;
2449		−
2450		− io_queue_linked_timeout(link);
2451		− work->func = io_wq_submit_work;
2452		−}
2453		−
2454		−static void io_wq_assign_next(struct io_wq_work *workptr, struct io_kiocb nxt)
2455		−{
2456		− struct io_kiocb *link;
2457		−
2458		− io_prep_async_work(nxt, &link);
2459		− *workptr = &nxt->work;
2460		− if (link) {
2461		− nxt->work.flags \|= IO_WQ_WORK_CB;
2462		− nxt->work.func = io_link_work_cb;
2463		− nxt->work.data = link;
2464		− }
2465		−}
2466		−
2467		−static void io_fsync_finish(struct io_wq_work **workptr)
	2832	+static void __io_fsync(struct io_kiocb *req)
2468	2833	{
2469		− struct io_kiocb req = container_of(workptr, struct io_kiocb, work);
2470	2834	loff_t end = req->sync.off + req->sync.len;
2471		− struct io_kiocb *nxt = NULL;
2472	2835	int ret;
2473	2836
2474		− if (io_req_cancelled(req))
2475		− return;
2476		−
2477	2837	ret = vfs_fsync_range(req->file, req->sync.off,
2478	2838	end > 0 ? end : LLONG_MAX,
2479	2839	req->sync.flags & IORING_FSYNC_DATASYNC);
2480	2840	if (ret < 0)
2481	2841	req_set_fail_links(req);
2482	2842	io_cqring_add_event(req, ret);
2483		− io_put_req_find_next(req, &nxt);
2484		− if (nxt)
2485		− io_wq_assign_next(workptr, nxt);
	2843	+ io_put_req(req);
2486	2844	}
2487	2845
2488		−static int io_fsync(struct io_kiocb req, struct io_kiocb *nxt,
2489		− bool force_nonblock)
	2846	+static void io_fsync_finish(struct io_wq_work **workptr)
2490	2847	{
2491		− struct io_wq_work work, old_work;
	2848	+ struct io_kiocb req = container_of(workptr, struct io_kiocb, work);
	2849	+
	2850	+ if (io_req_cancelled(req))
	2851	+ return;
	2852	+ __io_fsync(req);
	2853	+ io_steal_work(req, workptr);
	2854	+}
2492	2855
	2856	+static int io_fsync(struct io_kiocb *req, bool force_nonblock)
	2857	+{
2493	2858	/* fsync always requires a blocking context */
2494	2859	if (force_nonblock) {
2495		− io_put_req(req);
2496	2860	req->work.func = io_fsync_finish;
2497	2861	return -EAGAIN;
2498	2862	}
2499		−
2500		− work = old_work = &req->work;
2501		− io_fsync_finish(&work);
2502		− if (work && work != old_work)
2503		− *nxt = container_of(work, struct io_kiocb, work);
	2863	+ __io_fsync(req);
2504	2864	return 0;
2505	2865	}
2506	2866
2507		−static void io_fallocate_finish(struct io_wq_work **workptr)
	2867	+static void __io_fallocate(struct io_kiocb *req)
2508	2868	{
2509		− struct io_kiocb req = container_of(workptr, struct io_kiocb, work);
2510		− struct io_kiocb *nxt = NULL;
2511	2869	int ret;
2512	2870
2513		− if (io_req_cancelled(req))
2514		− return;
2515		−
	2871	+ current->signal->rlim[RLIMIT_FSIZE].rlim_cur = req->fsize;
2516	2872	ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
2517	2873	req->sync.len);
	2874	+ current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY;
2518	2875	if (ret < 0)
2519	2876	req_set_fail_links(req);
2520	2877	io_cqring_add_event(req, ret);
2521		− io_put_req_find_next(req, &nxt);
2522		− if (nxt)
2523		− io_wq_assign_next(workptr, nxt);
	2878	+ io_put_req(req);
	2879	+}
	2880	+
	2881	+static void io_fallocate_finish(struct io_wq_work **workptr)
	2882	+{
	2883	+ struct io_kiocb req = container_of(workptr, struct io_kiocb, work);
	2884	+
	2885	+ if (io_req_cancelled(req))
	2886	+ return;
	2887	+ __io_fallocate(req);
	2888	+ io_steal_work(req, workptr);
2524	2889	}
2525	2890
2526	2891	static int io_fallocate_prep(struct io_kiocb *req,
@@ -2532,26 +2897,19 @@ static int io_fallocate_prep(struct io_kiocb *req, · 5 unmodified lines
2532	2897	req->sync.off = READ_ONCE(sqe->off);
2533	2898	req->sync.len = READ_ONCE(sqe->addr);
2534	2899	req->sync.mode = READ_ONCE(sqe->len);
	2900	+ req->fsize = rlimit(RLIMIT_FSIZE);
2535	2901	return 0;
2536	2902	}
2537	2903
2538		−static int io_fallocate(struct io_kiocb req, struct io_kiocb *nxt,
2539		− bool force_nonblock)
	2904	+static int io_fallocate(struct io_kiocb *req, bool force_nonblock)
2540	2905	{
2541		− struct io_wq_work work, old_work;
2542		−
2543	2906	/* fallocate always requiring blocking context */
2544	2907	if (force_nonblock) {
2545		− io_put_req(req);
2546	2908	req->work.func = io_fallocate_finish;
2547	2909	return -EAGAIN;
2548	2910	}
2549	2911
2550		− work = old_work = &req->work;
2551		− io_fallocate_finish(&work);
2552		− if (work && work != old_work)
2553		− *nxt = container_of(work, struct io_kiocb, work);
2554		−
	2912	+ __io_fallocate(req);
2555	2913	return 0;
2556	2914	}
2557	2915
@@ -2626,8 +2984,7 @@ static int io_openat2_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 68 unmodified lines
2626	2984	return 0;
2627	2985	}
2628	2986
2629		−static int io_openat2(struct io_kiocb req, struct io_kiocb *nxt,
2630		− bool force_nonblock)
	2987	+static int io_openat2(struct io_kiocb *req, bool force_nonblock)
2631	2988	{
2632	2989	struct open_flags op;
2633	2990	struct file *file;
@@ -2658,15 +3015,171 @@ err: · 24 unmodified lines
2658	3015	if (ret < 0)
2659	3016	req_set_fail_links(req);
2660	3017	io_cqring_add_event(req, ret);
2661		− io_put_req_find_next(req, nxt);
	3018	+ io_put_req(req);
2662	3019	return 0;
2663	3020	}
2664	3021
2665		−static int io_openat(struct io_kiocb req, struct io_kiocb *nxt,
2666		− bool force_nonblock)
	3022	+static int io_openat(struct io_kiocb *req, bool force_nonblock)
2667	3023	{
2668	3024	req->open.how = build_open_how(req->open.how.flags, req->open.how.mode);
2669		− return io_openat2(req, nxt, force_nonblock);
	3025	+ return io_openat2(req, force_nonblock);
	3026	+}
	3027	+
	3028	+static int io_remove_buffers_prep(struct io_kiocb *req,
	3029	+ const struct io_uring_sqe *sqe)
	3030	+{
	3031	+ struct io_provide_buf *p = &req->pbuf;
	3032	+ u64 tmp;
	3033	+
	3034	+ if (sqe->ioprio \|\| sqe->rw_flags \|\| sqe->addr \|\| sqe->len \|\| sqe->off)
	3035	+ return -EINVAL;
	3036	+
	3037	+ tmp = READ_ONCE(sqe->fd);
	3038	+ if (!tmp \|\| tmp > USHRT_MAX)
	3039	+ return -EINVAL;
	3040	+
	3041	+ memset(p, 0, sizeof(*p));
	3042	+ p->nbufs = tmp;
	3043	+ p->bgid = READ_ONCE(sqe->buf_group);
	3044	+ return 0;
	3045	+}
	3046	+
	3047	+static int __io_remove_buffers(struct io_ring_ctx ctx, struct io_buffer buf,
	3048	+ int bgid, unsigned nbufs)
	3049	+{
	3050	+ unsigned i = 0;
	3051	+
	3052	+ /* shouldn't happen */
	3053	+ if (!nbufs)
	3054	+ return 0;
	3055	+
	3056	+ /* the head kbuf is the list itself */
	3057	+ while (!list_empty(&buf->list)) {
	3058	+ struct io_buffer *nxt;
	3059	+
	3060	+ nxt = list_first_entry(&buf->list, struct io_buffer, list);
	3061	+ list_del(&nxt->list);
	3062	+ kfree(nxt);
	3063	+ if (++i == nbufs)
	3064	+ return i;
	3065	+ }
	3066	+ i++;
	3067	+ kfree(buf);
	3068	+ idr_remove(&ctx->io_buffer_idr, bgid);
	3069	+
	3070	+ return i;
	3071	+}
	3072	+
	3073	+static int io_remove_buffers(struct io_kiocb *req, bool force_nonblock)
	3074	+{
	3075	+ struct io_provide_buf *p = &req->pbuf;
	3076	+ struct io_ring_ctx *ctx = req->ctx;
	3077	+ struct io_buffer *head;
	3078	+ int ret = 0;
	3079	+
	3080	+ io_ring_submit_lock(ctx, !force_nonblock);
	3081	+
	3082	+ lockdep_assert_held(&ctx->uring_lock);
	3083	+
	3084	+ ret = -ENOENT;
	3085	+ head = idr_find(&ctx->io_buffer_idr, p->bgid);
	3086	+ if (head)
	3087	+ ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);
	3088	+
	3089	+ io_ring_submit_lock(ctx, !force_nonblock);
	3090	+ if (ret < 0)
	3091	+ req_set_fail_links(req);
	3092	+ io_cqring_add_event(req, ret);
	3093	+ io_put_req(req);
	3094	+ return 0;
	3095	+}
	3096	+
	3097	+static int io_provide_buffers_prep(struct io_kiocb *req,
	3098	+ const struct io_uring_sqe *sqe)
	3099	+{
	3100	+ struct io_provide_buf *p = &req->pbuf;
	3101	+ u64 tmp;
	3102	+
	3103	+ if (sqe->ioprio \|\| sqe->rw_flags)
	3104	+ return -EINVAL;
	3105	+
	3106	+ tmp = READ_ONCE(sqe->fd);
	3107	+ if (!tmp \|\| tmp > USHRT_MAX)
	3108	+ return -E2BIG;
	3109	+ p->nbufs = tmp;
	3110	+ p->addr = READ_ONCE(sqe->addr);
	3111	+ p->len = READ_ONCE(sqe->len);
	3112	+
	3113	+ if (!access_ok(u64_to_user_ptr(p->addr), p->len))
	3114	+ return -EFAULT;
	3115	+
	3116	+ p->bgid = READ_ONCE(sqe->buf_group);
	3117	+ tmp = READ_ONCE(sqe->off);
	3118	+ if (tmp > USHRT_MAX)
	3119	+ return -E2BIG;
	3120	+ p->bid = tmp;
	3121	+ return 0;
	3122	+}
	3123	+
	3124	+static int io_add_buffers(struct io_provide_buf pbuf, struct io_buffer *head)
	3125	+{
	3126	+ struct io_buffer *buf;
	3127	+ u64 addr = pbuf->addr;
	3128	+ int i, bid = pbuf->bid;
	3129	+
	3130	+ for (i = 0; i < pbuf->nbufs; i++) {
	3131	+ buf = kmalloc(sizeof(*buf), GFP_KERNEL);
	3132	+ if (!buf)
	3133	+ break;
	3134	+
	3135	+ buf->addr = addr;
	3136	+ buf->len = pbuf->len;
	3137	+ buf->bid = bid;
	3138	+ addr += pbuf->len;
	3139	+ bid++;
	3140	+ if (!*head) {
	3141	+ INIT_LIST_HEAD(&buf->list);
	3142	+ *head = buf;
	3143	+ } else {
	3144	+ list_add_tail(&buf->list, &(*head)->list);
	3145	+ }
	3146	+ }
	3147	+
	3148	+ return i ? i : -ENOMEM;
	3149	+}
	3150	+
	3151	+static int io_provide_buffers(struct io_kiocb *req, bool force_nonblock)
	3152	+{
	3153	+ struct io_provide_buf *p = &req->pbuf;
	3154	+ struct io_ring_ctx *ctx = req->ctx;
	3155	+ struct io_buffer head, list;
	3156	+ int ret = 0;
	3157	+
	3158	+ io_ring_submit_lock(ctx, !force_nonblock);
	3159	+
	3160	+ lockdep_assert_held(&ctx->uring_lock);
	3161	+
	3162	+ list = head = idr_find(&ctx->io_buffer_idr, p->bgid);
	3163	+
	3164	+ ret = io_add_buffers(p, &head);
	3165	+ if (ret < 0)
	3166	+ goto out;
	3167	+
	3168	+ if (!list) {
	3169	+ ret = idr_alloc(&ctx->io_buffer_idr, head, p->bgid, p->bgid + 1,
	3170	+ GFP_KERNEL);
	3171	+ if (ret < 0) {
	3172	+ __io_remove_buffers(ctx, head, p->bgid, -1U);
	3173	+ goto out;
	3174	+ }
	3175	+ }
	3176	+out:
	3177	+ io_ring_submit_unlock(ctx, !force_nonblock);
	3178	+ if (ret < 0)
	3179	+ req_set_fail_links(req);
	3180	+ io_cqring_add_event(req, ret);
	3181	+ io_put_req(req);
	3182	+ return 0;
2670	3183	}
2671	3184
2672	3185	static int io_epoll_ctl_prep(struct io_kiocb *req,
@@ -2694,8 +3207,7 @@ static int io_epoll_ctl_prep(struct io_kiocb *req, · 21 unmodified lines
2694	3207	#endif
2695	3208	}
2696	3209
2697		−static int io_epoll_ctl(struct io_kiocb req, struct io_kiocb *nxt,
2698		− bool force_nonblock)
	3210	+static int io_epoll_ctl(struct io_kiocb *req, bool force_nonblock)
2699	3211	{
2700	3212	#if defined(CONFIG_EPOLL)
2701	3213	struct io_epoll *ie = &req->epoll;
@@ -2708,7 +3220,7 @@ static int io_epoll_ctl(struct io_kiocb req, struct io_kiocb *nxt, · 6 unmodified lines
2708	3220	if (ret < 0)
2709	3221	req_set_fail_links(req);
2710	3222	io_cqring_add_event(req, ret);
2711		− io_put_req_find_next(req, nxt);
	3223	+ io_put_req(req);
2712	3224	return 0;
2713	3225	#else
2714	3226	return -EOPNOTSUPP;
@@ -2730,8 +3242,7 @@ static int io_madvise_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 15 unmodified lines
2730	3242	#endif
2731	3243	}
2732	3244
2733		−static int io_madvise(struct io_kiocb req, struct io_kiocb *nxt,
2734		− bool force_nonblock)
	3245	+static int io_madvise(struct io_kiocb *req, bool force_nonblock)
2735	3246	{
2736	3247	#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
2737	3248	struct io_madvise *ma = &req->madvise;
@@ -2744,7 +3255,7 @@ static int io_madvise(struct io_kiocb req, struct io_kiocb *nxt, · 6 unmodified lines
2744	3255	if (ret < 0)
2745	3256	req_set_fail_links(req);
2746	3257	io_cqring_add_event(req, ret);
2747		− io_put_req_find_next(req, nxt);
	3258	+ io_put_req(req);
2748	3259	return 0;
2749	3260	#else
2750	3261	return -EOPNOTSUPP;
@@ -2762,8 +3273,7 @@ static int io_fadvise_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 11 unmodified lines
2762	3273	return 0;
2763	3274	}
2764	3275
2765		−static int io_fadvise(struct io_kiocb req, struct io_kiocb *nxt,
2766		− bool force_nonblock)
	3276	+static int io_fadvise(struct io_kiocb *req, bool force_nonblock)
2767	3277	{
2768	3278	struct io_fadvise *fa = &req->fadvise;
2769	3279	int ret;
@@ -2783,7 +3293,7 @@ static int io_fadvise(struct io_kiocb req, struct io_kiocb *nxt, · 13 unmodified lines
2783	3293	if (ret < 0)
2784	3294	req_set_fail_links(req);
2785	3295	io_cqring_add_event(req, ret);
2786		− io_put_req_find_next(req, nxt);
	3296	+ io_put_req(req);
2787	3297	return 0;
2788	3298	}
2789	3299
@@ -2820,8 +3330,7 @@ static int io_statx_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 30 unmodified lines
2820	3330	return 0;
2821	3331	}
2822	3332
2823		−static int io_statx(struct io_kiocb req, struct io_kiocb *nxt,
2824		− bool force_nonblock)
	3333	+static int io_statx(struct io_kiocb *req, bool force_nonblock)
2825	3334	{
2826	3335	struct io_open *ctx = &req->open;
2827	3336	unsigned lookup_flags;
@@ -2858,7 +3367,7 @@ err: · 30 unmodified lines
2858	3367	if (ret < 0)
2859	3368	req_set_fail_links(req);
2860	3369	io_cqring_add_event(req, ret);
2861		− io_put_req_find_next(req, nxt);
	3370	+ io_put_req(req);
2862	3371	return 0;
2863	3372	}
2864	3373
@@ -2885,7 +3394,7 @@ static int io_close_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 20 unmodified lines
2885	3394	}
2886	3395
2887	3396	/* only called when __close_fd_get_file() is done */
2888		−static void __io_close_finish(struct io_kiocb req, struct io_kiocb *nxt)
	3397	+static void __io_close_finish(struct io_kiocb *req)
2889	3398	{
2890	3399	int ret;
2891	3400
@@ -2894,22 +3403,19 @@ static void __io_close_finish(struct io_kiocb req, struct io_kiocb *nxt) · 2 unmodified lines
2894	3403	req_set_fail_links(req);
2895	3404	io_cqring_add_event(req, ret);
2896	3405	fput(req->close.put_file);
2897		− io_put_req_find_next(req, nxt);
	3406	+ io_put_req(req);
2898	3407	}
2899	3408
2900	3409	static void io_close_finish(struct io_wq_work **workptr)
2901	3410	{
2902	3411	struct io_kiocb req = container_of(workptr, struct io_kiocb, work);
2903		− struct io_kiocb *nxt = NULL;
2904	3412
2905	3413	/* not cancellable, don't do io_req_cancelled() */
2906		− __io_close_finish(req, &nxt);
2907		− if (nxt)
2908		− io_wq_assign_next(workptr, nxt);
	3414	+ __io_close_finish(req);
	3415	+ io_steal_work(req, workptr);
2909	3416	}
2910	3417
2911		−static int io_close(struct io_kiocb req, struct io_kiocb *nxt,
2912		− bool force_nonblock)
	3418	+static int io_close(struct io_kiocb *req, bool force_nonblock)
2913	3419	{
2914	3420	int ret;
2915	3421
@@ -2919,23 +3425,25 @@ static int io_close(struct io_kiocb req, struct io_kiocb *nxt, · 3 unmodified lines
2919	3425	return ret;
2920	3426
2921	3427	/* if the file has a flush method, be safe and punt to async */
2922		− if (req->close.put_file->f_op->flush && !io_wq_current_is_worker())
2923		− goto eagain;
	3428	+ if (req->close.put_file->f_op->flush && force_nonblock) {
	3429	+ /* submission ref will be dropped, take it for async */
	3430	+ refcount_inc(&req->refs);
	3431	+
	3432	+ req->work.func = io_close_finish;
	3433	+ /*
	3434	+ * Do manual async queue here to avoid grabbing files - we don't
	3435	+ * need the files, and it'll cause io_close_finish() to close
	3436	+ * the file again and cause a double CQE entry for this request
	3437	+ */
	3438	+ io_queue_async_work(req);
	3439	+ return 0;
	3440	+ }
2924	3441
2925	3442	/*
2926	3443	* No ->flush(), safely close from here and just punt the
2927	3444	* fput() to async context.
2928	3445	*/
2929		− __io_close_finish(req, nxt);
2930		− return 0;
2931		−eagain:
2932		− req->work.func = io_close_finish;
2933		− /*
2934		− * Do manual async queue here to avoid grabbing files - we don't
2935		− * need the files, and it'll cause io_close_finish() to close
2936		− * the file again and cause a double CQE entry for this request
2937		− */
2938		− io_queue_async_work(req);
	3446	+ __io_close_finish(req);
2939	3447	return 0;
2940	3448	}
2941	3449
@@ -2957,47 +3465,62 @@ static int io_prep_sfr(struct io_kiocb req, const struct io_uring_sqe sqe) · 15 unmodified lines
2957	3465	return 0;
2958	3466	}
2959	3467
2960		−static void io_sync_file_range_finish(struct io_wq_work **workptr)
	3468	+static void __io_sync_file_range(struct io_kiocb *req)
2961	3469	{
2962		− struct io_kiocb req = container_of(workptr, struct io_kiocb, work);
2963		− struct io_kiocb *nxt = NULL;
2964	3470	int ret;
2965	3471
2966		− if (io_req_cancelled(req))
2967		− return;
2968		−
2969	3472	ret = sync_file_range(req->file, req->sync.off, req->sync.len,
2970	3473	req->sync.flags);
2971	3474	if (ret < 0)
2972	3475	req_set_fail_links(req);
2973	3476	io_cqring_add_event(req, ret);
2974		− io_put_req_find_next(req, &nxt);
	3477	+ io_put_req(req);
	3478	+}
	3479	+
	3480	+
	3481	+static void io_sync_file_range_finish(struct io_wq_work **workptr)
	3482	+{
	3483	+ struct io_kiocb req = container_of(workptr, struct io_kiocb, work);
	3484	+ struct io_kiocb *nxt = NULL;
	3485	+
	3486	+ if (io_req_cancelled(req))
	3487	+ return;
	3488	+ __io_sync_file_range(req);
	3489	+ io_put_req(req); /* put submission ref */
2975	3490	if (nxt)
2976	3491	io_wq_assign_next(workptr, nxt);
2977	3492	}
2978	3493
2979		−static int io_sync_file_range(struct io_kiocb req, struct io_kiocb *nxt,
2980		− bool force_nonblock)
	3494	+static int io_sync_file_range(struct io_kiocb *req, bool force_nonblock)
2981	3495	{
2982		− struct io_wq_work work, old_work;
2983		−
2984	3496	/* sync_file_range always requires a blocking context */
2985	3497	if (force_nonblock) {
2986		− io_put_req(req);
2987	3498	req->work.func = io_sync_file_range_finish;
2988	3499	return -EAGAIN;
2989	3500	}
2990	3501
2991		− work = old_work = &req->work;
2992		− io_sync_file_range_finish(&work);
2993		− if (work && work != old_work)
2994		− *nxt = container_of(work, struct io_kiocb, work);
	3502	+ __io_sync_file_range(req);
2995	3503	return 0;
2996	3504	}
2997	3505
	3506	+#if defined(CONFIG_NET)
	3507	+static int io_setup_async_msg(struct io_kiocb *req,
	3508	+ struct io_async_msghdr *kmsg)
	3509	+{
	3510	+ if (req->io)
	3511	+ return -EAGAIN;
	3512	+ if (io_alloc_async_ctx(req)) {
	3513	+ if (kmsg->iov != kmsg->fast_iov)
	3514	+ kfree(kmsg->iov);
	3515	+ return -ENOMEM;
	3516	+ }
	3517	+ req->flags \|= REQ_F_NEED_CLEANUP;
	3518	+ memcpy(&req->io->msg, kmsg, sizeof(*kmsg));
	3519	+ return -EAGAIN;
	3520	+}
	3521	+
2998	3522	static int io_sendmsg_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
2999	3523	{
3000		−#if defined(CONFIG_NET)
3001	3524	struct io_sr_msg *sr = &req->sr_msg;
3002	3525	struct io_async_ctx *io = req->io;
3003	3526	int ret;
@@ -3023,15 +3546,10 @@ static int io_sendmsg_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 19 unmodified lines
3023	3546	if (!ret)
3024	3547	req->flags \|= REQ_F_NEED_CLEANUP;
3025	3548	return ret;
3026		−#else
3027		− return -EOPNOTSUPP;
3028		−#endif
3029	3549	}
3030	3550
3031		−static int io_sendmsg(struct io_kiocb req, struct io_kiocb *nxt,
3032		− bool force_nonblock)
	3551	+static int io_sendmsg(struct io_kiocb *req, bool force_nonblock)
3033	3552	{
3034		−#if defined(CONFIG_NET)
3035	3553	struct io_async_msghdr *kmsg = NULL;
3036	3554	struct socket *sock;
3037	3555	int ret;
@@ -3071,18 +3589,8 @@ static int io_sendmsg(struct io_kiocb req, struct io_kiocb *nxt, · 33 unmodified lines
3071	3589	flags \|= MSG_DONTWAIT;
3072	3590
3073	3591	ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
3074		− if (force_nonblock && ret == -EAGAIN) {
3075		− if (req->io)
3076		− return -EAGAIN;
3077		− if (io_alloc_async_ctx(req)) {
3078		− if (kmsg->iov != kmsg->fast_iov)
3079		− kfree(kmsg->iov);
3080		− return -ENOMEM;
3081		− }
3082		− req->flags \|= REQ_F_NEED_CLEANUP;
3083		− memcpy(&req->io->msg, &io.msg, sizeof(io.msg));
3084		− return -EAGAIN;
3085		− }
	3592	+ if (force_nonblock && ret == -EAGAIN)
	3593	+ return io_setup_async_msg(req, kmsg);
3086	3594	if (ret == -ERESTARTSYS)
3087	3595	ret = -EINTR;
3088	3596	}
@@ -3093,17 +3601,12 @@ static int io_sendmsg(struct io_kiocb req, struct io_kiocb *nxt, · 4 unmodified lines
3093	3601	io_cqring_add_event(req, ret);
3094	3602	if (ret < 0)
3095	3603	req_set_fail_links(req);
3096		− io_put_req_find_next(req, nxt);
	3604	+ io_put_req(req);
3097	3605	return 0;
3098		−#else
3099		− return -EOPNOTSUPP;
3100		−#endif
3101	3606	}
3102	3607
3103		−static int io_send(struct io_kiocb req, struct io_kiocb *nxt,
3104		− bool force_nonblock)
	3608	+static int io_send(struct io_kiocb *req, bool force_nonblock)
3105	3609	{
3106		−#if defined(CONFIG_NET)
3107	3610	struct socket *sock;
3108	3611	int ret;
3109	3612
@@ -3144,17 +3647,120 @@ static int io_send(struct io_kiocb req, struct io_kiocb *nxt, · 34 unmodified lines
3144	3647	io_cqring_add_event(req, ret);
3145	3648	if (ret < 0)
3146	3649	req_set_fail_links(req);
3147		− io_put_req_find_next(req, nxt);
	3650	+ io_put_req(req);
3148	3651	return 0;
3149		−#else
3150		− return -EOPNOTSUPP;
	3652	+}
	3653	+
	3654	+static int __io_recvmsg_copy_hdr(struct io_kiocb req, struct io_async_ctx io)
	3655	+{
	3656	+ struct io_sr_msg *sr = &req->sr_msg;
	3657	+ struct iovec __user *uiov;
	3658	+ size_t iov_len;
	3659	+ int ret;
	3660	+
	3661	+ ret = __copy_msghdr_from_user(&io->msg.msg, sr->msg, &io->msg.uaddr,
	3662	+ &uiov, &iov_len);
	3663	+ if (ret)
	3664	+ return ret;
	3665	+
	3666	+ if (req->flags & REQ_F_BUFFER_SELECT) {
	3667	+ if (iov_len > 1)
	3668	+ return -EINVAL;
	3669	+ if (copy_from_user(io->msg.iov, uiov, sizeof(*uiov)))
	3670	+ return -EFAULT;
	3671	+ sr->len = io->msg.iov[0].iov_len;
	3672	+ iov_iter_init(&io->msg.msg.msg_iter, READ, io->msg.iov, 1,
	3673	+ sr->len);
	3674	+ io->msg.iov = NULL;
	3675	+ } else {
	3676	+ ret = import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
	3677	+ &io->msg.iov, &io->msg.msg.msg_iter);
	3678	+ if (ret > 0)
	3679	+ ret = 0;
	3680	+ }
	3681	+
	3682	+ return ret;
	3683	+}
	3684	+
	3685	+#ifdef CONFIG_COMPAT
	3686	+static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
	3687	+ struct io_async_ctx *io)
	3688	+{
	3689	+ struct compat_msghdr __user *msg_compat;
	3690	+ struct io_sr_msg *sr = &req->sr_msg;
	3691	+ struct compat_iovec __user *uiov;
	3692	+ compat_uptr_t ptr;
	3693	+ compat_size_t len;
	3694	+ int ret;
	3695	+
	3696	+ msg_compat = (struct compat_msghdr __user *) sr->msg;
	3697	+ ret = __get_compat_msghdr(&io->msg.msg, msg_compat, &io->msg.uaddr,
	3698	+ &ptr, &len);
	3699	+ if (ret)
	3700	+ return ret;
	3701	+
	3702	+ uiov = compat_ptr(ptr);
	3703	+ if (req->flags & REQ_F_BUFFER_SELECT) {
	3704	+ compat_ssize_t clen;
	3705	+
	3706	+ if (len > 1)
	3707	+ return -EINVAL;
	3708	+ if (!access_ok(uiov, sizeof(*uiov)))
	3709	+ return -EFAULT;
	3710	+ if (__get_user(clen, &uiov->iov_len))
	3711	+ return -EFAULT;
	3712	+ if (clen < 0)
	3713	+ return -EINVAL;
	3714	+ sr->len = io->msg.iov[0].iov_len;
	3715	+ io->msg.iov = NULL;
	3716	+ } else {
	3717	+ ret = compat_import_iovec(READ, uiov, len, UIO_FASTIOV,
	3718	+ &io->msg.iov,
	3719	+ &io->msg.msg.msg_iter);
	3720	+ if (ret < 0)
	3721	+ return ret;
	3722	+ }
	3723	+
	3724	+ return 0;
	3725	+}
	3726	+#endif
	3727	+
	3728	+static int io_recvmsg_copy_hdr(struct io_kiocb req, struct io_async_ctx io)
	3729	+{
	3730	+ io->msg.iov = io->msg.fast_iov;
	3731	+
	3732	+#ifdef CONFIG_COMPAT
	3733	+ if (req->ctx->compat)
	3734	+ return __io_compat_recvmsg_copy_hdr(req, io);
3151	3735	#endif
	3736	+
	3737	+ return __io_recvmsg_copy_hdr(req, io);
	3738	+}
	3739	+
	3740	+static struct io_buffer io_recv_buffer_select(struct io_kiocb req,
	3741	+ int *cflags, bool needs_lock)
	3742	+{
	3743	+ struct io_sr_msg *sr = &req->sr_msg;
	3744	+ struct io_buffer *kbuf;
	3745	+
	3746	+ if (!(req->flags & REQ_F_BUFFER_SELECT))
	3747	+ return NULL;
	3748	+
	3749	+ kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
	3750	+ if (IS_ERR(kbuf))
	3751	+ return kbuf;
	3752	+
	3753	+ sr->kbuf = kbuf;
	3754	+ req->flags \|= REQ_F_BUFFER_SELECTED;
	3755	+
	3756	+ *cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
	3757	+ *cflags \|= IORING_CQE_F_BUFFER;
	3758	+ return kbuf;
3152	3759	}
3153	3760
3154	3761	static int io_recvmsg_prep(struct io_kiocb *req,
3155	3762	const struct io_uring_sqe *sqe)
3156	3763	{
3157		−#if defined(CONFIG_NET)
3158	3764	struct io_sr_msg *sr = &req->sr_msg;
3159	3765	struct io_async_ctx *io = req->io;
3160	3766	int ret;
@@ -3162,6 +3768,7 @@ static int io_recvmsg_prep(struct io_kiocb *req, · 1 unmodified line
3162	3768	sr->msg_flags = READ_ONCE(sqe->msg_flags);
3163	3769	sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
3164	3770	sr->len = READ_ONCE(sqe->len);
	3771	+ sr->bgid = READ_ONCE(sqe->buf_group);
3165	3772
3166	3773	#ifdef CONFIG_COMPAT
3167	3774	if (req->ctx->compat)
@@ -3174,30 +3781,24 @@ static int io_recvmsg_prep(struct io_kiocb *req, · 6 unmodified lines
3174	3781	if (req->flags & REQ_F_NEED_CLEANUP)
3175	3782	return 0;
3176	3783
3177		− io->msg.iov = io->msg.fast_iov;
3178		− ret = recvmsg_copy_msghdr(&io->msg.msg, sr->msg, sr->msg_flags,
3179		− &io->msg.uaddr, &io->msg.iov);
	3784	+ ret = io_recvmsg_copy_hdr(req, io);
3180	3785	if (!ret)
3181	3786	req->flags \|= REQ_F_NEED_CLEANUP;
3182	3787	return ret;
3183		−#else
3184		− return -EOPNOTSUPP;
3185		−#endif
3186	3788	}
3187	3789
3188		−static int io_recvmsg(struct io_kiocb req, struct io_kiocb *nxt,
3189		− bool force_nonblock)
	3790	+static int io_recvmsg(struct io_kiocb *req, bool force_nonblock)
3190	3791	{
3191		−#if defined(CONFIG_NET)
3192	3792	struct io_async_msghdr *kmsg = NULL;
3193	3793	struct socket *sock;
3194		− int ret;
	3794	+ int ret, cflags = 0;
3195	3795
3196	3796	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3197	3797	return -EINVAL;
3198	3798
3199	3799	sock = sock_from_file(req->file, &ret);
3200	3800	if (sock) {
	3801	+ struct io_buffer *kbuf;
3201	3802	struct io_async_ctx io;
3202	3803	unsigned flags;
3203	3804
@@ -3209,19 +3810,23 @@ static int io_recvmsg(struct io_kiocb req, struct io_kiocb *nxt, · 5 unmodified lines
3209	3810	kmsg->iov = kmsg->fast_iov;
3210	3811	kmsg->msg.msg_iter.iov = kmsg->iov;
3211	3812	} else {
3212		− struct io_sr_msg *sr = &req->sr_msg;
3213		−
3214	3813	kmsg = &io.msg;
3215	3814	kmsg->msg.msg_name = &io.msg.addr;
3216	3815
3217		− io.msg.iov = io.msg.fast_iov;
3218		− ret = recvmsg_copy_msghdr(&io.msg.msg, sr->msg,
3219		− sr->msg_flags, &io.msg.uaddr,
3220		− &io.msg.iov);
	3816	+ ret = io_recvmsg_copy_hdr(req, &io);
3221	3817	if (ret)
3222	3818	return ret;
3223	3819	}
3224	3820
	3821	+ kbuf = io_recv_buffer_select(req, &cflags, !force_nonblock);
	3822	+ if (IS_ERR(kbuf)) {
	3823	+ return PTR_ERR(kbuf);
	3824	+ } else if (kbuf) {
	3825	+ kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
	3826	+ iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->iov,
	3827	+ 1, req->sr_msg.len);
	3828	+ }
	3829	+
3225	3830	flags = req->sr_msg.msg_flags;
3226	3831	if (flags & MSG_DONTWAIT)
3227	3832	req->flags \|= REQ_F_NOWAIT;
@@ -3230,18 +3835,8 @@ static int io_recvmsg(struct io_kiocb req, struct io_kiocb *nxt, · 2 unmodified lines
3230	3835
3231	3836	ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.msg,
3232	3837	kmsg->uaddr, flags);
3233		− if (force_nonblock && ret == -EAGAIN) {
3234		− if (req->io)
3235		− return -EAGAIN;
3236		− if (io_alloc_async_ctx(req)) {
3237		− if (kmsg->iov != kmsg->fast_iov)
3238		− kfree(kmsg->iov);
3239		− return -ENOMEM;
3240		− }
3241		− memcpy(&req->io->msg, &io.msg, sizeof(io.msg));
3242		− req->flags \|= REQ_F_NEED_CLEANUP;
3243		− return -EAGAIN;
3244		− }
	3838	+ if (force_nonblock && ret == -EAGAIN)
	3839	+ return io_setup_async_msg(req, kmsg);
3245	3840	if (ret == -ERESTARTSYS)
3246	3841	ret = -EINTR;
3247	3842	}
@@ -3249,22 +3844,18 @@ static int io_recvmsg(struct io_kiocb req, struct io_kiocb *nxt, · 1 unmodified line
3249	3844	if (kmsg && kmsg->iov != kmsg->fast_iov)
3250	3845	kfree(kmsg->iov);
3251	3846	req->flags &= ~REQ_F_NEED_CLEANUP;
3252		− io_cqring_add_event(req, ret);
	3847	+ __io_cqring_add_event(req, ret, cflags);
3253	3848	if (ret < 0)
3254	3849	req_set_fail_links(req);
3255		− io_put_req_find_next(req, nxt);
	3850	+ io_put_req(req);
3256	3851	return 0;
3257		−#else
3258		− return -EOPNOTSUPP;
3259		−#endif
3260	3852	}
3261	3853
3262		−static int io_recv(struct io_kiocb req, struct io_kiocb *nxt,
3263		− bool force_nonblock)
	3854	+static int io_recv(struct io_kiocb *req, bool force_nonblock)
3264	3855	{
3265		−#if defined(CONFIG_NET)
	3856	+ struct io_buffer *kbuf = NULL;
3266	3857	struct socket *sock;
3267		− int ret;
	3858	+ int ret, cflags = 0;
3268	3859
3269	3860	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3270	3861	return -EINVAL;
@@ -3272,15 +3863,25 @@ static int io_recv(struct io_kiocb req, struct io_kiocb *nxt, · 1 unmodified line
3272	3863	sock = sock_from_file(req->file, &ret);
3273	3864	if (sock) {
3274	3865	struct io_sr_msg *sr = &req->sr_msg;
	3866	+ void __user *buf = sr->buf;
3275	3867	struct msghdr msg;
3276	3868	struct iovec iov;
3277	3869	unsigned flags;
3278	3870
3279		− ret = import_single_range(READ, sr->buf, sr->len, &iov,
	3871	+ kbuf = io_recv_buffer_select(req, &cflags, !force_nonblock);
	3872	+ if (IS_ERR(kbuf))
	3873	+ return PTR_ERR(kbuf);
	3874	+ else if (kbuf)
	3875	+ buf = u64_to_user_ptr(kbuf->addr);
	3876	+
	3877	+ ret = import_single_range(READ, buf, sr->len, &iov,
3280	3878	&msg.msg_iter);
3281		− if (ret)
	3879	+ if (ret) {
	3880	+ kfree(kbuf);
3282	3881	return ret;
	3882	+ }
3283	3883
	3884	+ req->flags \|= REQ_F_NEED_CLEANUP;
3284	3885	msg.msg_name = NULL;
3285	3886	msg.msg_control = NULL;
3286	3887	msg.msg_controllen = 0;
@@ -3301,20 +3902,17 @@ static int io_recv(struct io_kiocb req, struct io_kiocb *nxt, · 14 unmodified lines
3301	3902	ret = -EINTR;
3302	3903	}
3303	3904
3304		− io_cqring_add_event(req, ret);
	3905	+ kfree(kbuf);
	3906	+ req->flags &= ~REQ_F_NEED_CLEANUP;
	3907	+ __io_cqring_add_event(req, ret, cflags);
3305	3908	if (ret < 0)
3306	3909	req_set_fail_links(req);
3307		− io_put_req_find_next(req, nxt);
	3910	+ io_put_req(req);
3308	3911	return 0;
3309		−#else
3310		− return -EOPNOTSUPP;
3311		−#endif
3312	3912	}
3313	3913
3314		−
3315	3914	static int io_accept_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
3316	3915	{
3317		−#if defined(CONFIG_NET)
3318	3916	struct io_accept *accept = &req->accept;
3319	3917
3320	3918	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL\|IORING_SETUP_SQPOLL)))
@@ -3327,14 +3925,9 @@ static int io_accept_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 6 unmodified lines
3327	3925	accept->flags = READ_ONCE(sqe->accept_flags);
3328	3926	accept->nofile = rlimit(RLIMIT_NOFILE);
3329	3927	return 0;
3330		−#else
3331		− return -EOPNOTSUPP;
3332		−#endif
3333	3928	}
3334	3929
3335		−#if defined(CONFIG_NET)
3336		−static int __io_accept(struct io_kiocb req, struct io_kiocb *nxt,
3337		− bool force_nonblock)
	3930	+static int __io_accept(struct io_kiocb *req, bool force_nonblock)
3338	3931	{
3339	3932	struct io_accept *accept = &req->accept;
3340	3933	unsigned file_flags;
@@ -3351,44 +3944,34 @@ static int __io_accept(struct io_kiocb req, struct io_kiocb *nxt, · 10 unmodified lines
3351	3944	if (ret < 0)
3352	3945	req_set_fail_links(req);
3353	3946	io_cqring_add_event(req, ret);
3354		− io_put_req_find_next(req, nxt);
	3947	+ io_put_req(req);
3355	3948	return 0;
3356	3949	}
3357	3950
3358	3951	static void io_accept_finish(struct io_wq_work **workptr)
3359	3952	{
3360	3953	struct io_kiocb req = container_of(workptr, struct io_kiocb, work);
3361		− struct io_kiocb *nxt = NULL;
3362	3954
3363	3955	if (io_req_cancelled(req))
3364	3956	return;
3365		− __io_accept(req, &nxt, false);
3366		− if (nxt)
3367		− io_wq_assign_next(workptr, nxt);
	3957	+ __io_accept(req, false);
	3958	+ io_steal_work(req, workptr);
3368	3959	}
3369		−#endif
3370	3960
3371		−static int io_accept(struct io_kiocb req, struct io_kiocb *nxt,
3372		− bool force_nonblock)
	3961	+static int io_accept(struct io_kiocb *req, bool force_nonblock)
3373	3962	{
3374		−#if defined(CONFIG_NET)
3375	3963	int ret;
3376	3964
3377		− ret = __io_accept(req, nxt, force_nonblock);
	3965	+ ret = __io_accept(req, force_nonblock);
3378	3966	if (ret == -EAGAIN && force_nonblock) {
3379	3967	req->work.func = io_accept_finish;
3380		− io_put_req(req);
3381	3968	return -EAGAIN;
3382	3969	}
3383	3970	return 0;
3384		−#else
3385		− return -EOPNOTSUPP;
3386		−#endif
3387	3971	}
3388	3972
3389	3973	static int io_connect_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
3390	3974	{
3391		−#if defined(CONFIG_NET)
3392	3975	struct io_connect *conn = &req->connect;
3393	3976	struct io_async_ctx *io = req->io;
3394	3977
@@ -3405,15 +3988,10 @@ static int io_connect_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 10 unmodified lines
3405	3988
3406	3989	return move_addr_to_kernel(conn->addr, conn->addr_len,
3407	3990	&io->connect.address);
3408		−#else
3409		− return -EOPNOTSUPP;
3410		−#endif
3411	3991	}
3412	3992
3413		−static int io_connect(struct io_kiocb req, struct io_kiocb *nxt,
3414		− bool force_nonblock)
	3993	+static int io_connect(struct io_kiocb *req, bool force_nonblock)
3415	3994	{
3416		−#if defined(CONFIG_NET)
3417	3995	struct io_async_ctx __io, *io;
3418	3996	unsigned file_flags;
3419	3997	int ret;
@@ -3449,25 +4027,301 @@ out: · 29 unmodified lines
3449	4027	if (ret < 0)
3450	4028	req_set_fail_links(req);
3451	4029	io_cqring_add_event(req, ret);
3452		− io_put_req_find_next(req, nxt);
	4030	+ io_put_req(req);
3453	4031	return 0;
3454		−#else
	4032	+}
	4033	+#else /* !CONFIG_NET */
	4034	+static int io_sendmsg_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
	4035	+{
3455	4036	return -EOPNOTSUPP;
3456		−#endif
3457	4037	}
3458	4038
3459		−static void io_poll_remove_one(struct io_kiocb *req)
	4039	+static int io_sendmsg(struct io_kiocb *req, bool force_nonblock)
	4040	+{
	4041	+ return -EOPNOTSUPP;
	4042	+}
	4043	+
	4044	+static int io_send(struct io_kiocb *req, bool force_nonblock)
	4045	+{
	4046	+ return -EOPNOTSUPP;
	4047	+}
	4048	+
	4049	+static int io_recvmsg_prep(struct io_kiocb *req,
	4050	+ const struct io_uring_sqe *sqe)
	4051	+{
	4052	+ return -EOPNOTSUPP;
	4053	+}
	4054	+
	4055	+static int io_recvmsg(struct io_kiocb *req, bool force_nonblock)
	4056	+{
	4057	+ return -EOPNOTSUPP;
	4058	+}
	4059	+
	4060	+static int io_recv(struct io_kiocb *req, bool force_nonblock)
	4061	+{
	4062	+ return -EOPNOTSUPP;
	4063	+}
	4064	+
	4065	+static int io_accept_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
	4066	+{
	4067	+ return -EOPNOTSUPP;
	4068	+}
	4069	+
	4070	+static int io_accept(struct io_kiocb *req, bool force_nonblock)
	4071	+{
	4072	+ return -EOPNOTSUPP;
	4073	+}
	4074	+
	4075	+static int io_connect_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
	4076	+{
	4077	+ return -EOPNOTSUPP;
	4078	+}
	4079	+
	4080	+static int io_connect(struct io_kiocb *req, bool force_nonblock)
	4081	+{
	4082	+ return -EOPNOTSUPP;
	4083	+}
	4084	+#endif /* CONFIG_NET */
	4085	+
	4086	+struct io_poll_table {
	4087	+ struct poll_table_struct pt;
	4088	+ struct io_kiocb *req;
	4089	+ int error;
	4090	+};
	4091	+
	4092	+static void __io_queue_proc(struct io_poll_iocb poll, struct io_poll_table pt,
	4093	+ struct wait_queue_head *head)
	4094	+{
	4095	+ if (unlikely(poll->head)) {
	4096	+ pt->error = -EINVAL;
	4097	+ return;
	4098	+ }
	4099	+
	4100	+ pt->error = 0;
	4101	+ poll->head = head;
	4102	+ add_wait_queue(head, &poll->wait);
	4103	+}
	4104	+
	4105	+static void io_async_queue_proc(struct file file, struct wait_queue_head head,
	4106	+ struct poll_table_struct *p)
	4107	+{
	4108	+ struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
	4109	+
	4110	+ __io_queue_proc(&pt->req->apoll->poll, pt, head);
	4111	+}
	4112	+
	4113	+static int __io_async_wake(struct io_kiocb req, struct io_poll_iocb poll,
	4114	+ __poll_t mask, task_work_func_t func)
	4115	+{
	4116	+ struct task_struct *tsk;
	4117	+
	4118	+ /* for instances that support it check for an event match first: */
	4119	+ if (mask && !(mask & poll->events))
	4120	+ return 0;
	4121	+
	4122	+ trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);
	4123	+
	4124	+ list_del_init(&poll->wait.entry);
	4125	+
	4126	+ tsk = req->task;
	4127	+ req->result = mask;
	4128	+ init_task_work(&req->task_work, func);
	4129	+ /*
	4130	+ * If this fails, then the task is exiting. If that is the case, then
	4131	+ * the exit check will ultimately cancel these work items. Hence we
	4132	+ * don't need to check here and handle it specifically.
	4133	+ */
	4134	+ task_work_add(tsk, &req->task_work, true);
	4135	+ wake_up_process(tsk);
	4136	+ return 1;
	4137	+}
	4138	+
	4139	+static void io_async_task_func(struct callback_head *cb)
	4140	+{
	4141	+ struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
	4142	+ struct async_poll *apoll = req->apoll;
	4143	+ struct io_ring_ctx *ctx = req->ctx;
	4144	+
	4145	+ trace_io_uring_task_run(req->ctx, req->opcode, req->user_data);
	4146	+
	4147	+ WARN_ON_ONCE(!list_empty(&req->apoll->poll.wait.entry));
	4148	+
	4149	+ if (hash_hashed(&req->hash_node)) {
	4150	+ spin_lock_irq(&ctx->completion_lock);
	4151	+ hash_del(&req->hash_node);
	4152	+ spin_unlock_irq(&ctx->completion_lock);
	4153	+ }
	4154	+
	4155	+ /* restore ->work in case we need to retry again */
	4156	+ memcpy(&req->work, &apoll->work, sizeof(req->work));
	4157	+
	4158	+ __set_current_state(TASK_RUNNING);
	4159	+ mutex_lock(&ctx->uring_lock);
	4160	+ __io_queue_sqe(req, NULL);
	4161	+ mutex_unlock(&ctx->uring_lock);
	4162	+
	4163	+ kfree(apoll);
	4164	+}
	4165	+
	4166	+static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
	4167	+ void *key)
	4168	+{
	4169	+ struct io_kiocb *req = wait->private;
	4170	+ struct io_poll_iocb *poll = &req->apoll->poll;
	4171	+
	4172	+ trace_io_uring_poll_wake(req->ctx, req->opcode, req->user_data,
	4173	+ key_to_poll(key));
	4174	+
	4175	+ return __io_async_wake(req, poll, key_to_poll(key), io_async_task_func);
	4176	+}
	4177	+
	4178	+static void io_poll_req_insert(struct io_kiocb *req)
	4179	+{
	4180	+ struct io_ring_ctx *ctx = req->ctx;
	4181	+ struct hlist_head *list;
	4182	+
	4183	+ list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
	4184	+ hlist_add_head(&req->hash_node, list);
	4185	+}
	4186	+
	4187	+static __poll_t __io_arm_poll_handler(struct io_kiocb *req,
	4188	+ struct io_poll_iocb *poll,
	4189	+ struct io_poll_table *ipt, __poll_t mask,
	4190	+ wait_queue_func_t wake_func)
	4191	+ __acquires(&ctx->completion_lock)
	4192	+{
	4193	+ struct io_ring_ctx *ctx = req->ctx;
	4194	+ bool cancel = false;
	4195	+
	4196	+ poll->file = req->file;
	4197	+ poll->head = NULL;
	4198	+ poll->done = poll->canceled = false;
	4199	+ poll->events = mask;
	4200	+
	4201	+ ipt->pt._key = mask;
	4202	+ ipt->req = req;
	4203	+ ipt->error = -EINVAL;
	4204	+
	4205	+ INIT_LIST_HEAD(&poll->wait.entry);
	4206	+ init_waitqueue_func_entry(&poll->wait, wake_func);
	4207	+ poll->wait.private = req;
	4208	+
	4209	+ mask = vfs_poll(req->file, &ipt->pt) & poll->events;
	4210	+
	4211	+ spin_lock_irq(&ctx->completion_lock);
	4212	+ if (likely(poll->head)) {
	4213	+ spin_lock(&poll->head->lock);
	4214	+ if (unlikely(list_empty(&poll->wait.entry))) {
	4215	+ if (ipt->error)
	4216	+ cancel = true;
	4217	+ ipt->error = 0;
	4218	+ mask = 0;
	4219	+ }
	4220	+ if (mask \|\| ipt->error)
	4221	+ list_del_init(&poll->wait.entry);
	4222	+ else if (cancel)
	4223	+ WRITE_ONCE(poll->canceled, true);
	4224	+ else if (!poll->done) /* actually waiting for an event */
	4225	+ io_poll_req_insert(req);
	4226	+ spin_unlock(&poll->head->lock);
	4227	+ }
	4228	+
	4229	+ return mask;
	4230	+}
	4231	+
	4232	+static bool io_arm_poll_handler(struct io_kiocb *req)
	4233	+{
	4234	+ const struct io_op_def *def = &io_op_defs[req->opcode];
	4235	+ struct io_ring_ctx *ctx = req->ctx;
	4236	+ struct async_poll *apoll;
	4237	+ struct io_poll_table ipt;
	4238	+ __poll_t mask, ret;
	4239	+
	4240	+ if (!req->file \|\| !file_can_poll(req->file))
	4241	+ return false;
	4242	+ if (req->flags & (REQ_F_MUST_PUNT \| REQ_F_POLLED))
	4243	+ return false;
	4244	+ if (!def->pollin && !def->pollout)
	4245	+ return false;
	4246	+
	4247	+ apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
	4248	+ if (unlikely(!apoll))
	4249	+ return false;
	4250	+
	4251	+ req->flags \|= REQ_F_POLLED;
	4252	+ memcpy(&apoll->work, &req->work, sizeof(req->work));
	4253	+
	4254	+ /*
	4255	+ * Don't need a reference here, as we're adding it to the task
	4256	+ * task_works list. If the task exits, the list is pruned.
	4257	+ */
	4258	+ req->task = current;
	4259	+ req->apoll = apoll;
	4260	+ INIT_HLIST_NODE(&req->hash_node);
	4261	+
	4262	+ mask = 0;
	4263	+ if (def->pollin)
	4264	+ mask \|= POLLIN \| POLLRDNORM;
	4265	+ if (def->pollout)
	4266	+ mask \|= POLLOUT \| POLLWRNORM;
	4267	+ mask \|= POLLERR \| POLLPRI;
	4268	+
	4269	+ ipt.pt._qproc = io_async_queue_proc;
	4270	+
	4271	+ ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask,
	4272	+ io_async_wake);
	4273	+ if (ret) {
	4274	+ ipt.error = 0;
	4275	+ apoll->poll.done = true;
	4276	+ spin_unlock_irq(&ctx->completion_lock);
	4277	+ memcpy(&req->work, &apoll->work, sizeof(req->work));
	4278	+ kfree(apoll);
	4279	+ return false;
	4280	+ }
	4281	+ spin_unlock_irq(&ctx->completion_lock);
	4282	+ trace_io_uring_poll_arm(ctx, req->opcode, req->user_data, mask,
	4283	+ apoll->poll.events);
	4284	+ return true;
	4285	+}
	4286	+
	4287	+static bool __io_poll_remove_one(struct io_kiocb *req,
	4288	+ struct io_poll_iocb *poll)
3460	4289	{
3461		− struct io_poll_iocb *poll = &req->poll;
	4290	+ bool do_complete = false;
3462	4291
3463	4292	spin_lock(&poll->head->lock);
3464	4293	WRITE_ONCE(poll->canceled, true);
3465	4294	if (!list_empty(&poll->wait.entry)) {
3466	4295	list_del_init(&poll->wait.entry);
3467		− io_queue_async_work(req);
	4296	+ do_complete = true;
3468	4297	}
3469	4298	spin_unlock(&poll->head->lock);
	4299	+ return do_complete;
	4300	+}
	4301	+
	4302	+static bool io_poll_remove_one(struct io_kiocb *req)
	4303	+{
	4304	+ bool do_complete;
	4305	+
	4306	+ if (req->opcode == IORING_OP_POLL_ADD) {
	4307	+ do_complete = __io_poll_remove_one(req, &req->poll);
	4308	+ } else {
	4309	+ /* non-poll requests have submit ref still */
	4310	+ do_complete = __io_poll_remove_one(req, &req->apoll->poll);
	4311	+ if (do_complete)
	4312	+ io_put_req(req);
	4313	+ }
	4314	+
3470	4315	hash_del(&req->hash_node);
	4316	+
	4317	+ if (do_complete) {
	4318	+ io_cqring_fill_event(req, -ECANCELED);
	4319	+ io_commit_cqring(req->ctx);
	4320	+ req->flags \|= REQ_F_COMP_LOCKED;
	4321	+ io_put_req(req);
	4322	+ }
	4323	+
	4324	+ return do_complete;
3471	4325	}
3472	4326
3473	4327	static void io_poll_remove_all(struct io_ring_ctx *ctx)
@@ -3485,6 +4339,8 @@ static void io_poll_remove_all(struct io_ring_ctx *ctx) · 11 unmodified lines
3485	4339	io_poll_remove_one(req);
3486	4340	}
3487	4341	spin_unlock_irq(&ctx->completion_lock);
	4342	+
	4343	+ io_cqring_ev_posted(ctx);
3488	4344	}
3489	4345
3490	4346	static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr)
@@ -3494,10 +4350,11 @@ static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr) · 3 unmodified lines
3494	4350
3495	4351	list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
3496	4352	hlist_for_each_entry(req, list, hash_node) {
3497		− if (sqe_addr == req->user_data) {
3498		− io_poll_remove_one(req);
	4353	+ if (sqe_addr != req->user_data)
	4354	+ continue;
	4355	+ if (io_poll_remove_one(req))
3499	4356	return 0;
3500		− }
	4357	+ return -EALREADY;
3501	4358	}
3502	4359
3503	4360	return -ENOENT;
@@ -3543,186 +4400,54 @@ static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error) · 39 unmodified lines
3543	4400	struct io_ring_ctx *ctx = req->ctx;
3544	4401
3545	4402	req->poll.done = true;
3546		− if (error)
3547		− io_cqring_fill_event(req, error);
3548		− else
3549		− io_cqring_fill_event(req, mangle_poll(mask));
	4403	+ io_cqring_fill_event(req, error ? error : mangle_poll(mask));
3550	4404	io_commit_cqring(ctx);
3551	4405	}
3552	4406
3553		−static void io_poll_complete_work(struct io_wq_work **workptr)
	4407	+static void io_poll_task_handler(struct io_kiocb req, struct io_kiocb *nxt)
3554	4408	{
3555		− struct io_wq_work work = workptr;
3556		− struct io_kiocb *req = container_of(work, struct io_kiocb, work);
3557		− struct io_poll_iocb *poll = &req->poll;
3558		− struct poll_table_struct pt = { ._key = poll->events };
3559	4409	struct io_ring_ctx *ctx = req->ctx;
3560		− struct io_kiocb *nxt = NULL;
3561		− __poll_t mask = 0;
3562		− int ret = 0;
3563		−
3564		− if (work->flags & IO_WQ_WORK_CANCEL) {
3565		− WRITE_ONCE(poll->canceled, true);
3566		− ret = -ECANCELED;
3567		− } else if (READ_ONCE(poll->canceled)) {
3568		− ret = -ECANCELED;
3569		− }
3570		−
3571		− if (ret != -ECANCELED)
3572		− mask = vfs_poll(poll->file, &pt) & poll->events;
3573	4410
3574		− /*
3575		− * Note that ->ki_cancel callers also delete iocb from active_reqs after
3576		− * calling ->ki_cancel. We need the ctx_lock roundtrip here to
3577		− * synchronize with them. In the cancellation case the list_del_init
3578		− * itself is not actually needed, but harmless so we keep it in to
3579		− * avoid further branches in the fast path.
3580		− */
3581	4411	spin_lock_irq(&ctx->completion_lock);
3582		− if (!mask && ret != -ECANCELED) {
3583		− add_wait_queue(poll->head, &poll->wait);
3584		− spin_unlock_irq(&ctx->completion_lock);
3585		− return;
3586		− }
3587	4412	hash_del(&req->hash_node);
3588		− io_poll_complete(req, mask, ret);
	4413	+ io_poll_complete(req, req->result, 0);
	4414	+ req->flags \|= REQ_F_COMP_LOCKED;
	4415	+ io_put_req_find_next(req, nxt);
3589	4416	spin_unlock_irq(&ctx->completion_lock);
3590	4417
3591	4418	io_cqring_ev_posted(ctx);
3592		−
3593		− if (ret < 0)
3594		− req_set_fail_links(req);
3595		− io_put_req_find_next(req, &nxt);
3596		− if (nxt)
3597		− io_wq_assign_next(workptr, nxt);
3598	4419	}
3599	4420
3600		−static void __io_poll_flush(struct io_ring_ctx ctx, struct llist_node nodes)
	4421	+static void io_poll_task_func(struct callback_head *cb)
3601	4422	{
3602		− struct io_kiocb req, tmp;
3603		− struct req_batch rb;
	4423	+ struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
	4424	+ struct io_kiocb *nxt = NULL;
3604	4425
3605		− rb.to_free = rb.need_iter = 0;
3606		− spin_lock_irq(&ctx->completion_lock);
3607		− llist_for_each_entry_safe(req, tmp, nodes, llist_node) {
3608		− hash_del(&req->hash_node);
3609		− io_poll_complete(req, req->result, 0);
	4426	+ io_poll_task_handler(req, &nxt);
	4427	+ if (nxt) {
	4428	+ struct io_ring_ctx *ctx = nxt->ctx;
3610	4429
3611		− if (refcount_dec_and_test(&req->refs) &&
3612		− !io_req_multi_free(&rb, req)) {
3613		− req->flags \|= REQ_F_COMP_LOCKED;
3614		− io_free_req(req);
3615		− }
	4430	+ mutex_lock(&ctx->uring_lock);
	4431	+ __io_queue_sqe(nxt, NULL);
	4432	+ mutex_unlock(&ctx->uring_lock);
3616	4433	}
3617		− spin_unlock_irq(&ctx->completion_lock);
3618		−
3619		− io_cqring_ev_posted(ctx);
3620		− io_free_req_many(ctx, &rb);
3621		−}
3622		−
3623		−static void io_poll_flush(struct io_wq_work **workptr)
3624		−{
3625		− struct io_kiocb req = container_of(workptr, struct io_kiocb, work);
3626		− struct llist_node *nodes;
3627		−
3628		− nodes = llist_del_all(&req->ctx->poll_llist);
3629		− if (nodes)
3630		− __io_poll_flush(req->ctx, nodes);
3631		−}
3632		−
3633		−static void io_poll_trigger_evfd(struct io_wq_work **workptr)
3634		−{
3635		− struct io_kiocb req = container_of(workptr, struct io_kiocb, work);
3636		−
3637		− eventfd_signal(req->ctx->cq_ev_fd, 1);
3638		− io_put_req(req);
3639	4434	}
3640	4435
3641	4436	static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
3642	4437	void *key)
3643	4438	{
3644		− struct io_poll_iocb *poll = wait->private;
3645		− struct io_kiocb *req = container_of(poll, struct io_kiocb, poll);
3646		− struct io_ring_ctx *ctx = req->ctx;
3647		− __poll_t mask = key_to_poll(key);
3648		−
3649		− /* for instances that support it check for an event match first: */
3650		− if (mask && !(mask & poll->events))
3651		− return 0;
3652		−
3653		− list_del_init(&poll->wait.entry);
3654		−
3655		− /*
3656		− * Run completion inline if we can. We're using trylock here because
3657		− * we are violating the completion_lock -> poll wq lock ordering.
3658		− * If we have a link timeout we're going to need the completion_lock
3659		− * for finalizing the request, mark us as having grabbed that already.
3660		− */
3661		− if (mask) {
3662		− unsigned long flags;
3663		−
3664		− if (llist_empty(&ctx->poll_llist) &&
3665		− spin_trylock_irqsave(&ctx->completion_lock, flags)) {
3666		− bool trigger_ev;
3667		−
3668		− hash_del(&req->hash_node);
3669		− io_poll_complete(req, mask, 0);
3670		−
3671		− trigger_ev = io_should_trigger_evfd(ctx);
3672		− if (trigger_ev && eventfd_signal_count()) {
3673		− trigger_ev = false;
3674		− req->work.func = io_poll_trigger_evfd;
3675		− } else {
3676		− req->flags \|= REQ_F_COMP_LOCKED;
3677		− io_put_req(req);
3678		− req = NULL;
3679		− }
3680		− spin_unlock_irqrestore(&ctx->completion_lock, flags);
3681		− __io_cqring_ev_posted(ctx, trigger_ev);
3682		− } else {
3683		− req->result = mask;
3684		− req->llist_node.next = NULL;
3685		− /* if the list wasn't empty, we're done */
3686		− if (!llist_add(&req->llist_node, &ctx->poll_llist))
3687		− req = NULL;
3688		− else
3689		− req->work.func = io_poll_flush;
3690		− }
3691		− }
3692		− if (req)
3693		− io_queue_async_work(req);
	4439	+ struct io_kiocb *req = wait->private;
	4440	+ struct io_poll_iocb *poll = &req->poll;
3694	4441
3695		− return 1;
	4442	+ return __io_async_wake(req, poll, key_to_poll(key), io_poll_task_func);
3696	4443	}
3697	4444
3698		−struct io_poll_table {
3699		− struct poll_table_struct pt;
3700		− struct io_kiocb *req;
3701		− int error;
3702		−};
3703		−
3704	4445	static void io_poll_queue_proc(struct file file, struct wait_queue_head head,
3705	4446	struct poll_table_struct *p)
3706	4447	{
3707	4448	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
3708	4449
3709		− if (unlikely(pt->req->poll.head)) {
3710		− pt->error = -EINVAL;
3711		− return;
3712		− }
3713		−
3714		− pt->error = 0;
3715		− pt->req->poll.head = head;
3716		− add_wait_queue(head, &pt->req->poll.wait);
3717		−}
3718		−
3719		−static void io_poll_req_insert(struct io_kiocb *req)
3720		−{
3721		− struct io_ring_ctx *ctx = req->ctx;
3722		− struct hlist_head *list;
3723		−
3724		− list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
3725		− hlist_add_head(&req->hash_node, list);
	4450	+ __io_queue_proc(&pt->req->poll, pt, head);
3726	4451	}
3727	4452
3728	4453	static int io_poll_add_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
@@ -3739,55 +4464,29 @@ static int io_poll_add_prep(struct io_kiocb req, const struct io_uring_sqe sqe · 10 unmodified lines
3739	4464
3740	4465	events = READ_ONCE(sqe->poll_events);
3741	4466	poll->events = demangle_poll(events) \| EPOLLERR \| EPOLLHUP;
	4467	+
	4468	+ /*
	4469	+ * Don't need a reference here, as we're adding it to the task
	4470	+ * task_works list. If the task exits, the list is pruned.
	4471	+ */
	4472	+ req->task = current;
3742	4473	return 0;
3743	4474	}
3744	4475
3745		−static int io_poll_add(struct io_kiocb req, struct io_kiocb *nxt)
	4476	+static int io_poll_add(struct io_kiocb *req)
3746	4477	{
3747	4478	struct io_poll_iocb *poll = &req->poll;
3748	4479	struct io_ring_ctx *ctx = req->ctx;
3749	4480	struct io_poll_table ipt;
3750		− bool cancel = false;
3751	4481	__poll_t mask;
3752	4482
3753		− INIT_IO_WORK(&req->work, io_poll_complete_work);
3754	4483	INIT_HLIST_NODE(&req->hash_node);
3755		−
3756		− poll->head = NULL;
3757		− poll->done = false;
3758		− poll->canceled = false;
3759		−
3760		− ipt.pt._qproc = io_poll_queue_proc;
3761		− ipt.pt._key = poll->events;
3762		− ipt.req = req;
3763		− ipt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */
3764		−
3765		− /* initialized the list so that we can do list_empty checks */
3766		− INIT_LIST_HEAD(&poll->wait.entry);
3767		− init_waitqueue_func_entry(&poll->wait, io_poll_wake);
3768		− poll->wait.private = poll;
3769		−
3770	4484	INIT_LIST_HEAD(&req->list);
	4485	+ ipt.pt._qproc = io_poll_queue_proc;
3771	4486
3772		− mask = vfs_poll(poll->file, &ipt.pt) & poll->events;
	4487	+ mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events,
	4488	+ io_poll_wake);
3773	4489
3774		− spin_lock_irq(&ctx->completion_lock);
3775		− if (likely(poll->head)) {
3776		− spin_lock(&poll->head->lock);
3777		− if (unlikely(list_empty(&poll->wait.entry))) {
3778		− if (ipt.error)
3779		− cancel = true;
3780		− ipt.error = 0;
3781		− mask = 0;
3782		− }
3783		− if (mask \|\| ipt.error)
3784		− list_del_init(&poll->wait.entry);
3785		− else if (cancel)
3786		− WRITE_ONCE(poll->canceled, true);
3787		− else if (!poll->done) /* actually waiting for an event */
3788		− io_poll_req_insert(req);
3789		− spin_unlock(&poll->head->lock);
3790		− }
3791	4490	if (mask) { /* no async, we'd stolen it */
3792	4491	ipt.error = 0;
3793	4492	io_poll_complete(req, mask, 0);
@@ -3796,7 +4495,7 @@ static int io_poll_add(struct io_kiocb req, struct io_kiocb *nxt) · 2 unmodified lines
3796	4495
3797	4496	if (mask) {
3798	4497	io_cqring_ev_posted(ctx);
3799		− io_put_req_find_next(req, nxt);
	4498	+ io_put_req(req);
3800	4499	}
3801	4500	return ipt.error;
3802	4501	}
@@ -4045,7 +4744,7 @@ static int io_async_cancel_one(struct io_ring_ctx ctx, void sqe_addr) · 242 unmodified lines
4045	4744
4046	4745	static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
4047	4746	struct io_kiocb *req, __u64 sqe_addr,
4048		− struct io_kiocb **nxt, int success_ret)
	4747	+ int success_ret)
4049	4748	{
4050	4749	unsigned long flags;
4051	4750	int ret;
@@ -4071,7 +4770,7 @@ done: · 19 unmodified lines
4071	4770
4072	4771	if (ret < 0)
4073	4772	req_set_fail_links(req);
4074		− io_put_req_find_next(req, nxt);
	4773	+ io_put_req(req);
4075	4774	}
4076	4775
4077	4776	static int io_async_cancel_prep(struct io_kiocb *req,
@@ -4087,11 +4786,11 @@ static int io_async_cancel_prep(struct io_kiocb *req, · 9 unmodified lines
4087	4786	return 0;
4088	4787	}
4089	4788
4090		−static int io_async_cancel(struct io_kiocb req, struct io_kiocb *nxt)
	4789	+static int io_async_cancel(struct io_kiocb *req)
4091	4790	{
4092	4791	struct io_ring_ctx *ctx = req->ctx;
4093	4792
4094		− io_async_find_and_cancel(ctx, req, req->cancel.addr, nxt, 0);
	4793	+ io_async_find_and_cancel(ctx, req, req->cancel.addr, 0);
4095	4794	return 0;
4096	4795	}
4097	4796
@@ -4226,6 +4925,15 @@ static int io_req_defer_prep(struct io_kiocb *req, · 128 unmodified lines
4226	4925	case IORING_OP_EPOLL_CTL:
4227	4926	ret = io_epoll_ctl_prep(req, sqe);
4228	4927	break;
	4928	+ case IORING_OP_SPLICE:
	4929	+ ret = io_splice_prep(req, sqe);
	4930	+ break;
	4931	+ case IORING_OP_PROVIDE_BUFFERS:
	4932	+ ret = io_provide_buffers_prep(req, sqe);
	4933	+ break;
	4934	+ case IORING_OP_REMOVE_BUFFERS:
	4935	+ ret = io_remove_buffers_prep(req, sqe);
	4936	+ break;
4229	4937	default:
4230	4938	printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
4231	4939	req->opcode);
@@ -4272,29 +4980,43 @@ static void io_cleanup_req(struct io_kiocb *req) · 40 unmodified lines
4272	4980	case IORING_OP_READV:
4273	4981	case IORING_OP_READ_FIXED:
4274	4982	case IORING_OP_READ:
	4983	+ if (req->flags & REQ_F_BUFFER_SELECTED)
	4984	+ kfree((void *)(unsigned long)req->rw.addr);
	4985	+ /* fallthrough */
4275	4986	case IORING_OP_WRITEV:
4276	4987	case IORING_OP_WRITE_FIXED:
4277	4988	case IORING_OP_WRITE:
4278	4989	if (io->rw.iov != io->rw.fast_iov)
4279	4990	kfree(io->rw.iov);
4280	4991	break;
4281		− case IORING_OP_SENDMSG:
4282	4992	case IORING_OP_RECVMSG:
	4993	+ if (req->flags & REQ_F_BUFFER_SELECTED)
	4994	+ kfree(req->sr_msg.kbuf);
	4995	+ /* fallthrough */
	4996	+ case IORING_OP_SENDMSG:
4283	4997	if (io->msg.iov != io->msg.fast_iov)
4284	4998	kfree(io->msg.iov);
4285	4999	break;
	5000	+ case IORING_OP_RECV:
	5001	+ if (req->flags & REQ_F_BUFFER_SELECTED)
	5002	+ kfree(req->sr_msg.kbuf);
	5003	+ break;
4286	5004	case IORING_OP_OPENAT:
4287	5005	case IORING_OP_OPENAT2:
4288	5006	case IORING_OP_STATX:
4289	5007	putname(req->open.filename);
4290	5008	break;
	5009	+ case IORING_OP_SPLICE:
	5010	+ io_put_file(req, req->splice.file_in,
	5011	+ (req->splice.flags & SPLICE_F_FD_IN_FIXED));
	5012	+ break;
4291	5013	}
4292	5014
4293	5015	req->flags &= ~REQ_F_NEED_CLEANUP;
4294	5016	}
4295	5017
4296	5018	static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe,
4297		− struct io_kiocb **nxt, bool force_nonblock)
	5019	+ bool force_nonblock)
4298	5020	{
4299	5021	struct io_ring_ctx *ctx = req->ctx;
4300	5022	int ret;
@@ -4311,7 +5033,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 10 unmodified lines
4311	5033	if (ret < 0)
4312	5034	break;
4313	5035	}
4314		− ret = io_read(req, nxt, force_nonblock);
	5036	+ ret = io_read(req, force_nonblock);
4315	5037	break;
4316	5038	case IORING_OP_WRITEV:
4317	5039	case IORING_OP_WRITE_FIXED:
@@ -4321,7 +5043,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 3 unmodified lines
4321	5043	if (ret < 0)
4322	5044	break;
4323	5045	}
4324		− ret = io_write(req, nxt, force_nonblock);
	5046	+ ret = io_write(req, force_nonblock);
4325	5047	break;
4326	5048	case IORING_OP_FSYNC:
4327	5049	if (sqe) {
@@ -4329,7 +5051,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
4329	5051	if (ret < 0)
4330	5052	break;
4331	5053	}
4332		− ret = io_fsync(req, nxt, force_nonblock);
	5054	+ ret = io_fsync(req, force_nonblock);
4333	5055	break;
4334	5056	case IORING_OP_POLL_ADD:
4335	5057	if (sqe) {
@@ -4337,7 +5059,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
4337	5059	if (ret)
4338	5060	break;
4339	5061	}
4340		− ret = io_poll_add(req, nxt);
	5062	+ ret = io_poll_add(req);
4341	5063	break;
4342	5064	case IORING_OP_POLL_REMOVE:
4343	5065	if (sqe) {
@@ -4353,7 +5075,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 9 unmodified lines
4353	5075	if (ret < 0)
4354	5076	break;
4355	5077	}
4356		− ret = io_sync_file_range(req, nxt, force_nonblock);
	5078	+ ret = io_sync_file_range(req, force_nonblock);
4357	5079	break;
4358	5080	case IORING_OP_SENDMSG:
4359	5081	case IORING_OP_SEND:
@@ -4363,9 +5085,9 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 3 unmodified lines
4363	5085	break;
4364	5086	}
4365	5087	if (req->opcode == IORING_OP_SENDMSG)
4366		− ret = io_sendmsg(req, nxt, force_nonblock);
	5088	+ ret = io_sendmsg(req, force_nonblock);
4367	5089	else
4368		− ret = io_send(req, nxt, force_nonblock);
	5090	+ ret = io_send(req, force_nonblock);
4369	5091	break;
4370	5092	case IORING_OP_RECVMSG:
4371	5093	case IORING_OP_RECV:
@@ -4375,9 +5097,9 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 3 unmodified lines
4375	5097	break;
4376	5098	}
4377	5099	if (req->opcode == IORING_OP_RECVMSG)
4378		− ret = io_recvmsg(req, nxt, force_nonblock);
	5100	+ ret = io_recvmsg(req, force_nonblock);
4379	5101	else
4380		− ret = io_recv(req, nxt, force_nonblock);
	5102	+ ret = io_recv(req, force_nonblock);
4381	5103	break;
4382	5104	case IORING_OP_TIMEOUT:
4383	5105	if (sqe) {
@@ -4401,7 +5123,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 17 unmodified lines
4401	5123	if (ret)
4402	5124	break;
4403	5125	}
4404		− ret = io_accept(req, nxt, force_nonblock);
	5126	+ ret = io_accept(req, force_nonblock);
4405	5127	break;
4406	5128	case IORING_OP_CONNECT:
4407	5129	if (sqe) {
@@ -4409,7 +5131,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
4409	5131	if (ret)
4410	5132	break;
4411	5133	}
4412		− ret = io_connect(req, nxt, force_nonblock);
	5134	+ ret = io_connect(req, force_nonblock);
4413	5135	break;
4414	5136	case IORING_OP_ASYNC_CANCEL:
4415	5137	if (sqe) {
@@ -4417,7 +5139,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
4417	5139	if (ret)
4418	5140	break;
4419	5141	}
4420		− ret = io_async_cancel(req, nxt);
	5142	+ ret = io_async_cancel(req);
4421	5143	break;
4422	5144	case IORING_OP_FALLOCATE:
4423	5145	if (sqe) {
@@ -4425,7 +5147,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
4425	5147	if (ret)
4426	5148	break;
4427	5149	}
4428		− ret = io_fallocate(req, nxt, force_nonblock);
	5150	+ ret = io_fallocate(req, force_nonblock);
4429	5151	break;
4430	5152	case IORING_OP_OPENAT:
4431	5153	if (sqe) {
@@ -4433,7 +5155,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
4433	5155	if (ret)
4434	5156	break;
4435	5157	}
4436		− ret = io_openat(req, nxt, force_nonblock);
	5158	+ ret = io_openat(req, force_nonblock);
4437	5159	break;
4438	5160	case IORING_OP_CLOSE:
4439	5161	if (sqe) {
@@ -4441,7 +5163,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
4441	5163	if (ret)
4442	5164	break;
4443	5165	}
4444		− ret = io_close(req, nxt, force_nonblock);
	5166	+ ret = io_close(req, force_nonblock);
4445	5167	break;
4446	5168	case IORING_OP_FILES_UPDATE:
4447	5169	if (sqe) {
@@ -4457,7 +5179,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 9 unmodified lines
4457	5179	if (ret)
4458	5180	break;
4459	5181	}
4460		− ret = io_statx(req, nxt, force_nonblock);
	5182	+ ret = io_statx(req, force_nonblock);
4461	5183	break;
4462	5184	case IORING_OP_FADVISE:
4463	5185	if (sqe) {
@@ -4465,7 +5187,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
4465	5187	if (ret)
4466	5188	break;
4467	5189	}
4468		− ret = io_fadvise(req, nxt, force_nonblock);
	5190	+ ret = io_fadvise(req, force_nonblock);
4469	5191	break;
4470	5192	case IORING_OP_MADVISE:
4471	5193	if (sqe) {
@@ -4473,7 +5195,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
4473	5195	if (ret)
4474	5196	break;
4475	5197	}
4476		− ret = io_madvise(req, nxt, force_nonblock);
	5198	+ ret = io_madvise(req, force_nonblock);
4477	5199	break;
4478	5200	case IORING_OP_OPENAT2:
4479	5201	if (sqe) {
@@ -4481,7 +5203,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
4481	5203	if (ret)
4482	5204	break;
4483	5205	}
4484		− ret = io_openat2(req, nxt, force_nonblock);
	5206	+ ret = io_openat2(req, force_nonblock);
4485	5207	break;
4486	5208	case IORING_OP_EPOLL_CTL:
4487	5209	if (sqe) {
@@ -4489,7 +5211,31 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
4489	5211	if (ret)
4490	5212	break;
4491	5213	}
4492		− ret = io_epoll_ctl(req, nxt, force_nonblock);
	5214	+ ret = io_epoll_ctl(req, force_nonblock);
	5215	+ break;
	5216	+ case IORING_OP_SPLICE:
	5217	+ if (sqe) {
	5218	+ ret = io_splice_prep(req, sqe);
	5219	+ if (ret < 0)
	5220	+ break;
	5221	+ }
	5222	+ ret = io_splice(req, force_nonblock);
	5223	+ break;
	5224	+ case IORING_OP_PROVIDE_BUFFERS:
	5225	+ if (sqe) {
	5226	+ ret = io_provide_buffers_prep(req, sqe);
	5227	+ if (ret)
	5228	+ break;
	5229	+ }
	5230	+ ret = io_provide_buffers(req, force_nonblock);
	5231	+ break;
	5232	+ case IORING_OP_REMOVE_BUFFERS:
	5233	+ if (sqe) {
	5234	+ ret = io_remove_buffers_prep(req, sqe);
	5235	+ if (ret)
	5236	+ break;
	5237	+ }
	5238	+ ret = io_remove_buffers(req, force_nonblock);
4493	5239	break;
4494	5240	default:
4495	5241	ret = -EINVAL;
@@ -4522,7 +5268,6 @@ static void io_wq_submit_work(struct io_wq_work **workptr) · 26 unmodified lines
4522	5268	{
4523	5269	struct io_wq_work work = workptr;
4524	5270	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
4525		− struct io_kiocb *nxt = NULL;
4526	5271	int ret = 0;
4527	5272
4528	5273	/* if NO_CANCEL is set, we must still run the work */
@@ -4532,9 +5277,8 @@ static void io_wq_submit_work(struct io_wq_work **workptr) · 3 unmodified lines
4532	5277	}
4533	5278
4534	5279	if (!ret) {
4535		− req->in_async = true;
4536	5280	do {
4537		− ret = io_issue_sqe(req, NULL, &nxt, false);
	5281	+ ret = io_issue_sqe(req, NULL, false);
4538	5282	/*
4539	5283	* We can get EAGAIN for polled IO even though we're
4540	5284	* forcing a sync submission from here, since we can't
@@ -4546,18 +5290,13 @@ static void io_wq_submit_work(struct io_wq_work **workptr) · 5 unmodified lines
4546	5290	} while (1);
4547	5291	}
4548	5292
4549		− /* drop submission reference */
4550		− io_put_req(req);
4551		−
4552	5293	if (ret) {
4553	5294	req_set_fail_links(req);
4554	5295	io_cqring_add_event(req, ret);
4555	5296	io_put_req(req);
4556	5297	}
4557	5298
4558		− /* if a dependent link is ready, pass it back */
4559		− if (!ret && nxt)
4560		− io_wq_assign_next(workptr, nxt);
	5299	+ io_steal_work(req, workptr);
4561	5300	}
4562	5301
4563	5302	static int io_req_needs_file(struct io_kiocb *req, int fd)
@@ -4578,41 +5317,52 @@ static inline struct file io_file_from_index(struct io_ring_ctx ctx, · 14 unmodified lines
4578	5317	return table->files[index & IORING_FILE_TABLE_MASK];;
4579	5318	}
4580	5319
4581		−static int io_req_set_file(struct io_submit_state state, struct io_kiocb req,
4582		− const struct io_uring_sqe *sqe)
	5320	+static int io_file_get(struct io_submit_state state, struct io_kiocb req,
	5321	+ int fd, struct file **out_file, bool fixed)
4583	5322	{
4584	5323	struct io_ring_ctx *ctx = req->ctx;
4585		− unsigned flags;
4586		− int fd;
4587		−
4588		− flags = READ_ONCE(sqe->flags);
4589		− fd = READ_ONCE(sqe->fd);
4590		−
4591		− if (!io_req_needs_file(req, fd))
4592		− return 0;
	5324	+ struct file *file;
4593	5325
4594		− if (flags & IOSQE_FIXED_FILE) {
	5326	+ if (fixed) {
4595	5327	if (unlikely(!ctx->file_data \|\|
4596	5328	(unsigned) fd >= ctx->nr_user_files))
4597	5329	return -EBADF;
4598	5330	fd = array_index_nospec(fd, ctx->nr_user_files);
4599		− req->file = io_file_from_index(ctx, fd);
4600		− if (!req->file)
	5331	+ file = io_file_from_index(ctx, fd);
	5332	+ if (!file)
4601	5333	return -EBADF;
4602		− req->flags \|= REQ_F_FIXED_FILE;
4603	5334	percpu_ref_get(&ctx->file_data->refs);
4604	5335	} else {
4605		− if (req->needs_fixed_file)
4606		− return -EBADF;
4607	5336	trace_io_uring_file_get(ctx, fd);
4608		− req->file = io_file_get(state, fd);
4609		− if (unlikely(!req->file))
	5337	+ file = __io_file_get(state, fd);
	5338	+ if (unlikely(!file))
4610	5339	return -EBADF;
4611	5340	}
4612	5341
	5342	+ *out_file = file;
4613	5343	return 0;
4614	5344	}
4615	5345
	5346	+static int io_req_set_file(struct io_submit_state state, struct io_kiocb req,
	5347	+ const struct io_uring_sqe *sqe)
	5348	+{
	5349	+ unsigned flags;
	5350	+ int fd;
	5351	+ bool fixed;
	5352	+
	5353	+ flags = READ_ONCE(sqe->flags);
	5354	+ fd = READ_ONCE(sqe->fd);
	5355	+
	5356	+ if (!io_req_needs_file(req, fd))
	5357	+ return 0;
	5358	+
	5359	+ fixed = (flags & IOSQE_FIXED_FILE);
	5360	+ if (unlikely(!fixed && req->needs_fixed_file))
	5361	+ return -EBADF;
	5362	+
	5363	+ return io_file_get(state, req, fd, &req->file, fixed);
	5364	+}
	5365	+
4616	5366	static int io_grab_files(struct io_kiocb *req)
4617	5367	{
4618	5368	int ret = -EBADF;
@@ -4672,8 +5422,7 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) · 53 unmodified lines
4672	5422
4673	5423	if (prev) {
4674	5424	req_set_fail_links(prev);
4675		− io_async_find_and_cancel(ctx, req, prev->user_data, NULL,
4676		− -ETIME);
	5425	+ io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME);
4677	5426	io_put_req(prev);
4678	5427	} else {
4679	5428	io_cqring_add_event(req, -ETIME);
@@ -4710,6 +5459,9 @@ static struct io_kiocb io_prep_linked_timeout(struct io_kiocb req) · 30 unmodified lines
4710	5459
4711	5460	if (!(req->flags & REQ_F_LINK))
4712	5461	return NULL;
	5462	+ /* for polled retry, if flag is set, we already went through here */
	5463	+ if (req->flags & REQ_F_POLLED)
	5464	+ return NULL;
4713	5465
4714	5466	nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,
4715	5467	link_list);
@@ -4723,7 +5475,7 @@ static struct io_kiocb io_prep_linked_timeout(struct io_kiocb req) · 7 unmodified lines
4723	5475	static void __io_queue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe)
4724	5476	{
4725	5477	struct io_kiocb *linked_timeout;
4726		− struct io_kiocb *nxt = NULL;
	5478	+ struct io_kiocb *nxt;
4727	5479	const struct cred *old_creds = NULL;
4728	5480	int ret;
4729	5481
@@ -4739,7 +5491,7 @@ again: · 9 unmodified lines
4739	5491	old_creds = override_creds(req->work.creds);
4740	5492	}
4741	5493
4742		− ret = io_issue_sqe(req, sqe, &nxt, true);
	5494	+ ret = io_issue_sqe(req, sqe, true);
4743	5495
4744	5496	/*
4745	5497	* We async punt it if the file wasn't marked NOWAIT, or if the file
@@ -4747,6 +5499,11 @@ again: · 1 unmodified line
4747	5499	*/
4748	5500	if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) \|\|
4749	5501	(req->flags & REQ_F_MUST_PUNT))) {
	5502	+ if (io_arm_poll_handler(req)) {
	5503	+ if (linked_timeout)
	5504	+ io_queue_linked_timeout(linked_timeout);
	5505	+ goto exit;
	5506	+ }
4750	5507	punt:
4751	5508	if (io_op_defs[req->opcode].file_table) {
4752	5509	ret = io_grab_files(req);
@@ -4759,10 +5516,11 @@ punt: · 6 unmodified lines
4759	5516	* submit reference when the iocb is actually submitted.
4760	5517	*/
4761	5518	io_queue_async_work(req);
4762		− goto done_req;
	5519	+ goto exit;
4763	5520	}
4764	5521
4765	5522	err:
	5523	+ nxt = NULL;
4766	5524	/* drop submission reference */
4767	5525	io_put_req_find_next(req, &nxt);
4768	5526
@@ -4779,15 +5537,14 @@ err: · 10 unmodified lines
4779	5537	req_set_fail_links(req);
4780	5538	io_put_req(req);
4781	5539	}
4782		−done_req:
4783	5540	if (nxt) {
4784	5541	req = nxt;
4785		− nxt = NULL;
4786	5542
4787	5543	if (req->flags & REQ_F_FORCE_ASYNC)
4788	5544	goto punt;
4789	5545	goto again;
4790	5546	}
	5547	+exit:
4791	5548	if (old_creds)
4792	5549	revert_creds(old_creds);
4793	5550	}
@@ -4829,7 +5586,8 @@ static inline void io_queue_link_head(struct io_kiocb *req) · 35 unmodified lines
4829	5586	}
4830	5587
4831	5588	#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE\|IOSQE_IO_DRAIN\|IOSQE_IO_LINK\| \
4832		− IOSQE_IO_HARDLINK \| IOSQE_ASYNC)
	5589	+ IOSQE_IO_HARDLINK \| IOSQE_ASYNC \| \
	5590	+ IOSQE_BUFFER_SELECT)
4833	5591
4834	5592	static bool io_submit_sqe(struct io_kiocb req, const struct io_uring_sqe sqe,
4835	5593	struct io_submit_state state, struct io_kiocb *link)
@@ -4846,6 +5604,12 @@ static bool io_submit_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 10 unmodified lines
4846	5604	goto err_req;
4847	5605	}
4848	5606
	5607	+ if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
	5608	+ !io_op_defs[req->opcode].buffer_select) {
	5609	+ ret = -EOPNOTSUPP;
	5610	+ goto err_req;
	5611	+ }
	5612	+
4849	5613	id = READ_ONCE(sqe->personality);
4850	5614	if (id) {
4851	5615	req->work.creds = idr_find(&ctx->personality_idr, id);
@@ -4857,8 +5621,9 @@ static bool io_submit_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 5 unmodified lines
4857	5621	}
4858	5622
4859	5623	/* same numerical values with corresponding REQ_F_, safe to copy /
4860		− req->flags \|= sqe_flags & (IOSQE_IO_DRAIN\|IOSQE_IO_HARDLINK\|
4861		− IOSQE_ASYNC);
	5624	+ req->flags \|= sqe_flags & (IOSQE_IO_DRAIN \| IOSQE_IO_HARDLINK \|
	5625	+ IOSQE_ASYNC \| IOSQE_FIXED_FILE \|
	5626	+ IOSQE_BUFFER_SELECT);
4862	5627
4863	5628	ret = io_req_set_file(state, req, sqe);
4864	5629	if (unlikely(ret)) {
@@ -5079,7 +5844,6 @@ fail_req: · 214 unmodified lines
5079	5844	*mm = ctx->sqo_mm;
5080	5845	}
5081	5846
5082		− req->in_async = async;
5083	5847	req->needs_fixed_file = async;
5084	5848	trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
5085	5849	true, async);
@@ -5163,6 +5927,8 @@ static int io_sq_thread(void *data) · 77 unmodified lines
5163	5927	if (!list_empty(&ctx->poll_list) \|\|
5164	5928	(!time_after(jiffies, timeout) && ret != -EBUSY &&
5165	5929	!percpu_ref_is_dying(&ctx->refs))) {
	5930	+ if (current->task_works)
	5931	+ task_work_run();
5166	5932	cond_resched();
5167	5933	continue;
5168	5934	}
@@ -5194,6 +5960,10 @@ static int io_sq_thread(void *data) · 25 unmodified lines
5194	5960	finish_wait(&ctx->sqo_wait, &wait);
5195	5961	break;
5196	5962	}
	5963	+ if (current->task_works) {
	5964	+ task_work_run();
	5965	+ continue;
	5966	+ }
5197	5967	if (signal_pending(current))
5198	5968	flush_signals(current);
5199	5969	schedule();
@@ -5213,6 +5983,9 @@ static int io_sq_thread(void *data) · 13 unmodified lines
5213	5983	timeout = jiffies + ctx->sq_thread_idle;
5214	5984	}
5215	5985
	5986	+ if (current->task_works)
	5987	+ task_work_run();
	5988	+
5216	5989	set_fs(old_fs);
5217	5990	if (cur_mm) {
5218	5991	unuse_mm(cur_mm);
@@ -5277,8 +6050,13 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 58 unmodified lines
5277	6050	struct io_rings *rings = ctx->rings;
5278	6051	int ret = 0;
5279	6052
5280		− if (io_cqring_events(ctx, false) >= min_events)
5281		− return 0;
	6053	+ do {
	6054	+ if (io_cqring_events(ctx, false) >= min_events)
	6055	+ return 0;
	6056	+ if (!current->task_works)
	6057	+ break;
	6058	+ task_work_run();
	6059	+ } while (1);
5282	6060
5283	6061	if (sig) {
5284	6062	#ifdef CONFIG_COMPAT
@@ -5298,6 +6076,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 13 unmodified lines
5298	6076	do {
5299	6077	prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
5300	6078	TASK_INTERRUPTIBLE);
	6079	+ if (current->task_works)
	6080	+ task_work_run();
5301	6081	if (io_should_wake(&iowq, false))
5302	6082	break;
5303	6083	schedule();
@@ -5607,7 +6387,6 @@ static void io_ring_file_put(struct io_ring_ctx ctx, struct file file) · 303 unmodified lines
5607	6387	struct io_file_put {
5608	6388	struct llist_node llist;
5609	6389	struct file *file;
5610		− struct completion *done;
5611	6390	};
5612	6391
5613	6392	static void io_ring_file_ref_flush(struct fixed_file_data *data)
@@ -5618,10 +6397,7 @@ static void io_ring_file_ref_flush(struct fixed_file_data *data) · 4 unmodified lines
5618	6397	while ((node = llist_del_all(&data->put_llist)) != NULL) {
5619	6398	llist_for_each_entry_safe(pfile, tmp, node, llist) {
5620	6399	io_ring_file_put(data->ctx, pfile->file);
5621		− if (pfile->done)
5622		− complete(pfile->done);
5623		− else
5624		− kfree(pfile);
	6400	+ kfree(pfile);
5625	6401	}
5626	6402	}
5627	6403	}
@@ -5816,33 +6592,18 @@ static void io_atomic_switch(struct percpu_ref *ref) · 188 unmodified lines
5816	6592	percpu_ref_get(&data->refs);
5817	6593	}
5818	6594
5819		−static bool io_queue_file_removal(struct fixed_file_data *data,
	6595	+static int io_queue_file_removal(struct fixed_file_data *data,
5820	6596	struct file *file)
5821	6597	{
5822		− struct io_file_put *pfile, pfile_stack;
5823		− DECLARE_COMPLETION_ONSTACK(done);
	6598	+ struct io_file_put *pfile;
5824	6599
5825		− /*
5826		− * If we fail allocating the struct we need for doing async reomval
5827		− * of this file, just punt to sync and wait for it.
5828		− */
5829	6600	pfile = kzalloc(sizeof(*pfile), GFP_KERNEL);
5830		− if (!pfile) {
5831		− pfile = &pfile_stack;
5832		− pfile->done = &done;
5833		− }
	6601	+ if (!pfile)
	6602	+ return -ENOMEM;
5834	6603
5835	6604	pfile->file = file;
5836	6605	llist_add(&pfile->llist, &data->put_llist);
5837		−
5838		− if (pfile == &pfile_stack) {
5839		− percpu_ref_switch_to_atomic(&data->refs, io_atomic_switch);
5840		− wait_for_completion(&done);
5841		− flush_work(&data->ref_work);
5842		− return false;
5843		− }
5844		−
5845		− return true;
	6606	+ return 0;
5846	6607	}
5847	6608
5848	6609	static int __io_sqe_files_update(struct io_ring_ctx *ctx,
@@ -5877,9 +6638,11 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, · 28 unmodified lines
5877	6638	index = i & IORING_FILE_TABLE_MASK;
5878	6639	if (table->files[index]) {
5879	6640	file = io_file_from_index(ctx, index);
	6641	+ err = io_queue_file_removal(data, file);
	6642	+ if (err)
	6643	+ break;
5880	6644	table->files[index] = NULL;
5881		− if (io_queue_file_removal(data, file))
5882		− ref_switch = true;
	6645	+ ref_switch = true;
5883	6646	}
5884	6647	if (fd != -1) {
5885	6648	file = fget(fd);
@@ -5932,20 +6695,14 @@ static int io_sqe_files_update(struct io_ring_ctx ctx, void __user arg, · 46 unmodified lines
5932	6695	return __io_sqe_files_update(ctx, &up, nr_args);
5933	6696	}
5934	6697
5935		−static void io_put_work(struct io_wq_work *work)
	6698	+static void io_free_work(struct io_wq_work *work)
5936	6699	{
5937	6700	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
5938	6701
	6702	+ /* Consider that io_steal_work() relies on this ref */
5939	6703	io_put_req(req);
5940	6704	}
5941	6705
5942		−static void io_get_work(struct io_wq_work *work)
5943		−{
5944		− struct io_kiocb *req = container_of(work, struct io_kiocb, work);
5945		−
5946		− refcount_inc(&req->refs);
5947		−}
5948		−
5949	6706	static int io_init_wq_offload(struct io_ring_ctx *ctx,
5950	6707	struct io_uring_params *p)
5951	6708	{
@@ -5956,8 +6713,7 @@ static int io_init_wq_offload(struct io_ring_ctx *ctx, · 4 unmodified lines
5956	6713	int ret = 0;
5957	6714
5958	6715	data.user = ctx->user;
5959		− data.get_work = io_get_work;
5960		− data.put_work = io_put_work;
	6716	+ data.free_work = io_free_work;
5961	6717
5962	6718	if (!(p->flags & IORING_SETUP_ATTACH_WQ)) {
5963	6719	/* Do QD, or 4 * CPUS, whatever is smallest */
@@ -6359,6 +7115,21 @@ static int io_eventfd_unregister(struct io_ring_ctx *ctx) · 395 unmodified lines
6359	7115	return -ENXIO;
6360	7116	}
6361	7117
	7118	+static int __io_destroy_buffers(int id, void p, void data)
	7119	+{
	7120	+ struct io_ring_ctx *ctx = data;
	7121	+ struct io_buffer *buf = p;
	7122	+
	7123	+ __io_remove_buffers(ctx, buf, id, -1U);
	7124	+ return 0;
	7125	+}
	7126	+
	7127	+static void io_destroy_buffers(struct io_ring_ctx *ctx)
	7128	+{
	7129	+ idr_for_each(&ctx->io_buffer_idr, __io_destroy_buffers, ctx);
	7130	+ idr_destroy(&ctx->io_buffer_idr);
	7131	+}
	7132	+
6362	7133	static void io_ring_ctx_free(struct io_ring_ctx *ctx)
6363	7134	{
6364	7135	io_finish_async(ctx);
@@ -6369,6 +7140,7 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) · 4 unmodified lines
6369	7140	io_sqe_buffer_unregister(ctx);
6370	7141	io_sqe_files_unregister(ctx);
6371	7142	io_eventfd_unregister(ctx);
	7143	+ io_destroy_buffers(ctx);
6372	7144	idr_destroy(&ctx->personality_idr);
6373	7145
6374	7146	#if defined(CONFIG_UNIX)
@@ -6623,6 +7395,9 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 248 unmodified lines
6623	7395	int submitted = 0;
6624	7396	struct fd f;
6625	7397
	7398	+ if (current->task_works)
	7399	+ task_work_run();
	7400	+
6626	7401	if (flags & ~(IORING_ENTER_GETEVENTS \| IORING_ENTER_SQ_WAKEUP))
6627	7402	return -EINVAL;
6628	7403
@@ -6669,7 +7444,14 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 40 unmodified lines
6669	7444
6670	7445	min_complete = min(min_complete, ctx->cq_entries);
6671	7446
6672		− if (ctx->flags & IORING_SETUP_IOPOLL) {
	7447	+ /*
	7448	+ * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
	7449	+ * space applications don't need to do io completion events
	7450	+ * polling again, they can rely on io_sq_thread to do polling
	7451	+ * work, which can reduce cpu usage and uring_lock contention.
	7452	+ */
	7453	+ if (ctx->flags & IORING_SETUP_IOPOLL &&
	7454	+ !(ctx->flags & IORING_SETUP_SQPOLL)) {
6673	7455	ret = io_iopoll_check(ctx, &nr_events, min_complete);
6674	7456	} else {
6675	7457	ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
@@ -6745,6 +7527,17 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx ctx, struct seq_file m) · 69 unmodified lines
6745	7527	seq_printf(m, "Personalities:\n");
6746	7528	idr_for_each(&ctx->personality_idr, io_uring_show_cred, m);
6747	7529	}
	7530	+ seq_printf(m, "PollList:\n");
	7531	+ spin_lock_irq(&ctx->completion_lock);
	7532	+ for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
	7533	+ struct hlist_head *list = &ctx->cancel_hash[i];
	7534	+ struct io_kiocb *req;
	7535	+
	7536	+ hlist_for_each_entry(req, list, hash_node)
	7537	+ seq_printf(m, " op=%d, task_works=%d\n", req->opcode,
	7538	+ req->task->task_works != NULL);
	7539	+ }
	7540	+ spin_unlock_irq(&ctx->completion_lock);
6748	7541	mutex_unlock(&ctx->uring_lock);
6749	7542	}
6750	7543
@@ -6961,7 +7754,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p) · 210 unmodified lines
6961	7754
6962	7755	p->features = IORING_FEAT_SINGLE_MMAP \| IORING_FEAT_NODROP \|
6963	7756	IORING_FEAT_SUBMIT_STABLE \| IORING_FEAT_RW_CUR_POS \|
6964		− IORING_FEAT_CUR_PERSONALITY;
	7757	+ IORING_FEAT_CUR_PERSONALITY \| IORING_FEAT_FAST_POLL;
6965	7758	trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
6966	7759	return ret;
6967	7760	err:
@@ -7239,6 +8032,7 @@ static int __init io_uring_init(void) · 271 unmodified lines
7239	8032	BUILD_BUG_SQE_ELEM(8, __u64, off);
7240	8033	BUILD_BUG_SQE_ELEM(8, __u64, addr2);
7241	8034	BUILD_BUG_SQE_ELEM(16, __u64, addr);
	8035	+ BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in);
7242	8036	BUILD_BUG_SQE_ELEM(24, __u32, len);
7243	8037	BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags);
7244	8038	BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);
@@ -7253,11 +8047,14 @@ static int __init io_uring_init(void) · 8 unmodified lines
7253	8047	BUILD_BUG_SQE_ELEM(28, __u32, open_flags);
7254	8048	BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);
7255	8049	BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);
	8050	+ BUILD_BUG_SQE_ELEM(28, __u32, splice_flags);
7256	8051	BUILD_BUG_SQE_ELEM(32, __u64, user_data);
7257	8052	BUILD_BUG_SQE_ELEM(40, __u16, buf_index);
7258	8053	BUILD_BUG_SQE_ELEM(42, __u16, personality);
	8054	+ BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in);
7259	8055
7260	8056	BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
	8057	+ BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int));
7261	8058	req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN \| SLAB_PANIC);
7262	8059	return 0;
7263	8060	};

fs/splice.c+3−3

@@ -1109,9 +1109,9 @@ static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, · 1108 unmodified lines
1109	1109	/*
1110	1110	* Determine where to splice to/from.
1111	1111	*/
1112		−static long do_splice(struct file in, loff_t __user off_in,
1113		− struct file out, loff_t __user off_out,
1114		− size_t len, unsigned int flags)
	1112	+long do_splice(struct file in, loff_t __user off_in,
	1113	+ struct file out, loff_t __user off_out,
	1114	+ size_t len, unsigned int flags)
1115	1115	{
1116	1116	struct pipe_inode_info *ipipe;
1117	1117	struct pipe_inode_info *opipe;

include/linux/socket.h+4−0

@@ -391,6 +391,10 @@ extern int recvmsg_copy_msghdr(struct msghdr *msg, · 390 unmodified lines
391	391	struct user_msghdr __user *umsg, unsigned flags,
392	392	struct sockaddr __user **uaddr,
393	393	struct iovec **iov);
	394	+extern int __copy_msghdr_from_user(struct msghdr *kmsg,
	395	+ struct user_msghdr __user *umsg,
	396	+ struct sockaddr __user **save_addr,
	397	+ struct iovec __user *uiov, size_t nsegs);
394	398
395	399	/* helpers which do the actual work for syscalls */
396	400	extern int __sys_recvfrom(int fd, void __user *ubuf, size_t size,

include/linux/splice.h+3−0

@@ -78,6 +78,9 @@ extern ssize_t add_to_pipe(struct pipe_inode_info *, · 77 unmodified lines
78	78	struct pipe_buffer *);
79	79	extern ssize_t splice_direct_to_actor(struct file , struct splice_desc ,
80	80	splice_direct_actor *);
	81	+extern long do_splice(struct file in, loff_t __user off_in,
	82	+ struct file out, loff_t __user off_out,
	83	+ size_t len, unsigned int flags);
81	84
82	85	/*
83	86	* for dynamic pipe sizing

include/net/compat.h+3−0

@@ -38,6 +38,9 @@ struct compat_cmsghdr { · 37 unmodified lines
38	38	#define compat_mmsghdr mmsghdr
39	39	#endif /* defined(CONFIG_COMPAT) */
40	40
	41	+int __get_compat_msghdr(struct msghdr kmsg, struct compat_msghdr __user umsg,
	42	+ struct sockaddr __user *save_addr, compat_uptr_t ptr,
	43	+ compat_size_t *len);
41	44	int get_compat_msghdr(struct msghdr , struct compat_msghdr __user ,
42	45	struct sockaddr __user , struct iovec );
43	46	struct sock_fprog __user get_compat_bpf_fprog(char __user optval);

include/trace/events/io_uring.h+103−0

@@ -357,6 +357,109 @@ TRACE_EVENT(io_uring_submit_sqe, · 356 unmodified lines
357	357	__entry->force_nonblock, __entry->sq_thread)
358	358	);
359	359
	360	+TRACE_EVENT(io_uring_poll_arm,
	361	+
	362	+ TP_PROTO(void *ctx, u8 opcode, u64 user_data, int mask, int events),
	363	+
	364	+ TP_ARGS(ctx, opcode, user_data, mask, events),
	365	+
	366	+ TP_STRUCT__entry (
	367	+ __field( void *, ctx )
	368	+ __field( u8, opcode )
	369	+ __field( u64, user_data )
	370	+ __field( int, mask )
	371	+ __field( int, events )
	372	+ ),
	373	+
	374	+ TP_fast_assign(
	375	+ __entry->ctx = ctx;
	376	+ __entry->opcode = opcode;
	377	+ __entry->user_data = user_data;
	378	+ __entry->mask = mask;
	379	+ __entry->events = events;
	380	+ ),
	381	+
	382	+ TP_printk("ring %p, op %d, data 0x%llx, mask 0x%x, events 0x%x",
	383	+ __entry->ctx, __entry->opcode,
	384	+ (unsigned long long) __entry->user_data,
	385	+ __entry->mask, __entry->events)
	386	+);
	387	+
	388	+TRACE_EVENT(io_uring_poll_wake,
	389	+
	390	+ TP_PROTO(void *ctx, u8 opcode, u64 user_data, int mask),
	391	+
	392	+ TP_ARGS(ctx, opcode, user_data, mask),
	393	+
	394	+ TP_STRUCT__entry (
	395	+ __field( void *, ctx )
	396	+ __field( u8, opcode )
	397	+ __field( u64, user_data )
	398	+ __field( int, mask )
	399	+ ),
	400	+
	401	+ TP_fast_assign(
	402	+ __entry->ctx = ctx;
	403	+ __entry->opcode = opcode;
	404	+ __entry->user_data = user_data;
	405	+ __entry->mask = mask;
	406	+ ),
	407	+
	408	+ TP_printk("ring %p, op %d, data 0x%llx, mask 0x%x",
	409	+ __entry->ctx, __entry->opcode,
	410	+ (unsigned long long) __entry->user_data,
	411	+ __entry->mask)
	412	+);
	413	+
	414	+TRACE_EVENT(io_uring_task_add,
	415	+
	416	+ TP_PROTO(void *ctx, u8 opcode, u64 user_data, int mask),
	417	+
	418	+ TP_ARGS(ctx, opcode, user_data, mask),
	419	+
	420	+ TP_STRUCT__entry (
	421	+ __field( void *, ctx )
	422	+ __field( u8, opcode )
	423	+ __field( u64, user_data )
	424	+ __field( int, mask )
	425	+ ),
	426	+
	427	+ TP_fast_assign(
	428	+ __entry->ctx = ctx;
	429	+ __entry->opcode = opcode;
	430	+ __entry->user_data = user_data;
	431	+ __entry->mask = mask;
	432	+ ),
	433	+
	434	+ TP_printk("ring %p, op %d, data 0x%llx, mask %x",
	435	+ __entry->ctx, __entry->opcode,
	436	+ (unsigned long long) __entry->user_data,
	437	+ __entry->mask)
	438	+);
	439	+
	440	+TRACE_EVENT(io_uring_task_run,
	441	+
	442	+ TP_PROTO(void *ctx, u8 opcode, u64 user_data),
	443	+
	444	+ TP_ARGS(ctx, opcode, user_data),
	445	+
	446	+ TP_STRUCT__entry (
	447	+ __field( void *, ctx )
	448	+ __field( u8, opcode )
	449	+ __field( u64, user_data )
	450	+ ),
	451	+
	452	+ TP_fast_assign(
	453	+ __entry->ctx = ctx;
	454	+ __entry->opcode = opcode;
	455	+ __entry->user_data = user_data;
	456	+ ),
	457	+
	458	+ TP_printk("ring %p, op %d, data 0x%llx",
	459	+ __entry->ctx, __entry->opcode,
	460	+ (unsigned long long) __entry->user_data)
	461	+);
	462	+
360	463	#endif /* _TRACE_IO_URING_H */
361	464
362	465	/* This part must be outside protection */

include/uapi/linux/io_uring.h+38−4

@@ -1,4 +1,4 @@
1		−/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
	1	+/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */
2	2	/*
3	3	* Header file for the io_uring interface.
4	4	*
@@ -23,7 +23,10 @@ struct io_uring_sqe { · 18 unmodified lines
23	23	__u64 off; /* offset into file */
24	24	__u64 addr2;
25	25	};
26		− __u64 addr; /* pointer to buffer or iovecs */
	26	+ union {
	27	+ __u64 addr; /* pointer to buffer or iovecs */
	28	+ __u64 splice_off_in;
	29	+ };
27	30	__u32 len; /* buffer size or number of iovecs */
28	31	union {
29	32	__kernel_rwf_t rw_flags;
@@ -37,14 +40,21 @@ struct io_uring_sqe { · 7 unmodified lines
37	40	__u32 open_flags;
38	41	__u32 statx_flags;
39	42	__u32 fadvise_advice;
	43	+ __u32 splice_flags;
40	44	};
41	45	__u64 user_data; /* data to be passed back at completion time */
42	46	union {
43	47	struct {
44		− /* index into fixed buffers, if used */
45		− __u16 buf_index;
	48	+ /* pack this to avoid bogus arm OABI complaints */
	49	+ union {
	50	+ /* index into fixed buffers, if used */
	51	+ __u16 buf_index;
	52	+ /* for grouped buffer selection */
	53	+ __u16 buf_group;
	54	+ } __attribute__((packed));
46	55	/* personality to use, if used */
47	56	__u16 personality;
	57	+ __s32 splice_fd_in;
48	58	};
49	59	__u64 __pad2[3];
50	60	};
@@ -56,6 +66,7 @@ enum { · 5 unmodified lines
56	66	IOSQE_IO_LINK_BIT,
57	67	IOSQE_IO_HARDLINK_BIT,
58	68	IOSQE_ASYNC_BIT,
	69	+ IOSQE_BUFFER_SELECT_BIT,
59	70	};
60	71
61	72	/*
@@ -71,6 +82,8 @@ enum { · 9 unmodified lines
71	82	#define IOSQE_IO_HARDLINK (1U << IOSQE_IO_HARDLINK_BIT)
72	83	/* always go async */
73	84	#define IOSQE_ASYNC (1U << IOSQE_ASYNC_BIT)
	85	+/* select buffer from sqe->buf_group */
	86	+#define IOSQE_BUFFER_SELECT (1U << IOSQE_BUFFER_SELECT_BIT)
74	87
75	88	/*
76	89	* io_uring_setup() flags
@@ -113,6 +126,9 @@ enum { · 36 unmodified lines
113	126	IORING_OP_RECV,
114	127	IORING_OP_OPENAT2,
115	128	IORING_OP_EPOLL_CTL,
	129	+ IORING_OP_SPLICE,
	130	+ IORING_OP_PROVIDE_BUFFERS,
	131	+ IORING_OP_REMOVE_BUFFERS,
116	132
117	133	/* this goes last, obviously */
118	134	IORING_OP_LAST,
@@ -128,6 +144,12 @@ enum { · 9 unmodified lines
128	144	*/
129	145	#define IORING_TIMEOUT_ABS (1U << 0)
130	146
	147	+/*
	148	+ * sqe->splice_flags
	149	+ * extends splice(2) flags
	150	+ */
	151	+#define SPLICE_F_FD_IN_FIXED (1U << 31) /* the last bit of __u32 */
	152	+
131	153	/*
132	154	* IO completion data structure (Completion Queue Entry)
133	155	*/
@@ -137,6 +159,17 @@ struct io_uring_cqe { · 3 unmodified lines
137	159	__u32 flags;
138	160	};
139	161
	162	+/*
	163	+ * cqe->flags
	164	+ *
	165	+ * IORING_CQE_F_BUFFER If set, the upper 16 bits are the buffer ID
	166	+ */
	167	+#define IORING_CQE_F_BUFFER (1U << 0)
	168	+
	169	+enum {
	170	+ IORING_CQE_BUFFER_SHIFT = 16,
	171	+};
	172	+
140	173	/*
141	174	* Magic offsets for the application to mmap the data it needs
142	175	*/
@@ -204,6 +237,7 @@ struct io_uring_params { · 61 unmodified lines
204	237	#define IORING_FEAT_SUBMIT_STABLE (1U << 2)
205	238	#define IORING_FEAT_RW_CUR_POS (1U << 3)
206	239	#define IORING_FEAT_CUR_PERSONALITY (1U << 4)
	240	+#define IORING_FEAT_FAST_POLL (1U << 5)
207	241
208	242	/*
209	243	* io_uring_register(2) opcodes and arguments

kernel/task_work.c+14−4

@@ -97,16 +97,26 @@ void task_work_run(void) · 96 unmodified lines
97	97	* work->func() can do task_work_add(), do not set
98	98	* work_exited unless the list is empty.
99	99	*/
100		− raw_spin_lock_irq(&task->pi_lock);
101	100	do {
	101	+ head = NULL;
102	102	work = READ_ONCE(task->task_works);
103		− head = !work && (task->flags & PF_EXITING) ?
104		− &work_exited : NULL;
	103	+ if (!work) {
	104	+ if (task->flags & PF_EXITING)
	105	+ head = &work_exited;
	106	+ else
	107	+ break;
	108	+ }
105	109	} while (cmpxchg(&task->task_works, work, head) != work);
106		− raw_spin_unlock_irq(&task->pi_lock);
107	110
108	111	if (!work)
109	112	break;
	113	+ /*
	114	+ * Synchronize with task_work_cancel(). It can not remove
	115	+ * the first entry == work, cmpxchg(task_works) must fail.
	116	+ * But it can remove another entry from the ->next list.
	117	+ */
	118	+ raw_spin_lock_irq(&task->pi_lock);
	119	+ raw_spin_unlock_irq(&task->pi_lock);
110	120
111	121	do {
112	122	next = work->next;

net/compat.c+23−7

@@ -33,10 +33,10 @@ · 32 unmodified lines
33	33	#include <linux/uaccess.h>
34	34	#include <net/compat.h>
35	35
36		−int get_compat_msghdr(struct msghdr *kmsg,
37		− struct compat_msghdr __user *umsg,
38		− struct sockaddr __user **save_addr,
39		− struct iovec **iov)
	36	+int __get_compat_msghdr(struct msghdr *kmsg,
	37	+ struct compat_msghdr __user *umsg,
	38	+ struct sockaddr __user **save_addr,
	39	+ compat_uptr_t ptr, compat_size_t len)
40	40	{
41	41	struct compat_msghdr msg;
42	42	ssize_t err;
@@ -79,10 +79,26 @@ int get_compat_msghdr(struct msghdr *kmsg, · 36 unmodified lines
79	79	return -EMSGSIZE;
80	80
81	81	kmsg->msg_iocb = NULL;
	82	+ *ptr = msg.msg_iov;
	83	+ *len = msg.msg_iovlen;
	84	+ return 0;
	85	+}
	86	+
	87	+int get_compat_msghdr(struct msghdr *kmsg,
	88	+ struct compat_msghdr __user *umsg,
	89	+ struct sockaddr __user **save_addr,
	90	+ struct iovec **iov)
	91	+{
	92	+ compat_uptr_t ptr;
	93	+ compat_size_t len;
	94	+ ssize_t err;
	95	+
	96	+ err = __get_compat_msghdr(kmsg, umsg, save_addr, &ptr, &len);
	97	+ if (err)
	98	+ return err;
82	99
83		− err = compat_import_iovec(save_addr ? READ : WRITE,
84		− compat_ptr(msg.msg_iov), msg.msg_iovlen,
85		− UIO_FASTIOV, iov, &kmsg->msg_iter);
	100	+ err = compat_import_iovec(save_addr ? READ : WRITE, compat_ptr(ptr),
	101	+ len, UIO_FASTIOV, iov, &kmsg->msg_iter);
86	102	return err < 0 ? err : 0;
87	103	}
88	104

net/socket.c+21−4

@@ -2228,10 +2228,10 @@ struct used_address { · 2227 unmodified lines
2228	2228	unsigned int name_len;
2229	2229	};
2230	2230
2231		−static int copy_msghdr_from_user(struct msghdr *kmsg,
2232		− struct user_msghdr __user *umsg,
2233		− struct sockaddr __user **save_addr,
2234		− struct iovec **iov)
	2231	+int __copy_msghdr_from_user(struct msghdr *kmsg,
	2232	+ struct user_msghdr __user *umsg,
	2233	+ struct sockaddr __user **save_addr,
	2234	+ struct iovec __user *uiov, size_t nsegs)
2235	2235	{
2236	2236	struct user_msghdr msg;
2237	2237	ssize_t err;
@@ -2273,6 +2273,23 @@ static int copy_msghdr_from_user(struct msghdr *kmsg, · 35 unmodified lines
2273	2273	return -EMSGSIZE;
2274	2274
2275	2275	kmsg->msg_iocb = NULL;
	2276	+ *uiov = msg.msg_iov;
	2277	+ *nsegs = msg.msg_iovlen;
	2278	+ return 0;
	2279	+}
	2280	+
	2281	+static int copy_msghdr_from_user(struct msghdr *kmsg,
	2282	+ struct user_msghdr __user *umsg,
	2283	+ struct sockaddr __user **save_addr,
	2284	+ struct iovec **iov)
	2285	+{
	2286	+ struct user_msghdr msg;
	2287	+ ssize_t err;
	2288	+
	2289	+ err = __copy_msghdr_from_user(kmsg, umsg, save_addr, &msg.msg_iov,
	2290	+ &msg.msg_iovlen);
	2291	+ if (err)
	2292	+ return err;
2276	2293
2277	2294	err = import_iovec(save_addr ? READ : WRITE,
2278	2295	msg.msg_iov, msg.msg_iovlen,