Merge tag 'for-5.9/io_uring-20200802' of git://git.kernel.dk/linux-block

Pull io_uring updates from Jens Axboe: "Lots of cleanups in here, hardening the code and/or making it easier to read and fixing bugs, but a core feature/change too adding support for real async buffered reads. With the latter in place, we just need buffered write async support and we're done relying on kthreads for the fast path. In detail: - Cleanup how memory accounting is done on ring setup/free (Bijan) - sq array offset calculation fixup (Dmitry) - Consistently handle blocking off O_DIRECT submission path (me) - Support proper async buffered reads, instead of relying on kthread offload for that. This uses the page waitqueue to drive retries from task_work, like we handle poll based retry. (me) - IO completion optimizations (me) - Fix race with accounting and ring fd install (me) - Support EPOLLEXCLUSIVE (Jiufei) - Get rid of the io_kiocb unionizing, made possible by shrinking other bits (Pavel) - Completion side cleanups (Pavel) - Cleanup REQ_F_ flags handling, and kill off many of them (Pavel) - Request environment grabbing cleanups (Pavel) - File and socket read/write cleanups (Pavel) - Improve kiocb_set_rw_flags() (Pavel) - Tons of fixes and cleanups (Pavel) - IORING_SQ_NEED_WAKEUP clear fix (Xiaoguang)" * tag 'for-5.9/io_uring-20200802' of git://git.kernel.dk/linux-block: (127 commits) io_uring: flip if handling after io_setup_async_rw fs: optimise kiocb_set_rw_flags() io_uring: don't touch 'ctx' after installing file descriptor io_uring: get rid of atomic FAA for cq_timeouts io_uring: consolidate *_check_overflow accounting io_uring: fix stalled deferred requests io_uring: fix racy overflow count reporting io_uring: deduplicate __io_complete_rw() io_uring: de-unionise io_kiocb io-wq: update hash bits io_uring: fix missing io_queue_linked_timeout() io_uring: mark ->work uninitialised after cleanup io_uring: deduplicate io_grab_files() calls io_uring: don't do opcode prep twice io_uring: clear IORING_SQ_NEED_WAKEUP after executing task works io_uring: batch put_task_struct() tasks: add put_task_struct_many() io_uring: return locked and pinned page accounting io_uring: don't miscount pinned memory io_uring: don't open-code recv kbuf managment ...

Linus Torvalds committed 6 years ago

Commit cdc8fcb2 parents 382625d fa15baf14 files · +1644 −1181Browse files

Unified · Split

14 files

block/blk-core.c +6 −0
fs/block_dev.c +1 −1
fs/btrfs/file.c +1 −1
fs/io-wq.c +5 −9
fs/io-wq.h +6 −5
fs/io_uring.c +1482 −1129
fs/xfs/xfs_file.c +1 −1
include/linux/blkdev.h +1 −0
include/linux/fs.h +20 −6
include/linux/pagemap.h +43 −0
include/linux/sched/task.h +6 −0
include/uapi/linux/io_uring.h +3 −1
mm/filemap.c +64 −27
tools/io_uring/liburing.h +5 −1

block/blk-core.c+6−0

@@ -960,9 +960,14 @@ static noinline_for_stack bool submit_bio_checks(struct bio *bio) · 959 unmodified lines
960	960	{
961	961	struct request_queue *q = bio->bi_disk->queue;
962	962	blk_status_t status = BLK_STS_IOERR;
	963	+ struct blk_plug *plug;
963	964
964	965	might_sleep();
965	966
	967	+ plug = blk_mq_plug(q, bio);
	968	+ if (plug && plug->nowait)
	969	+ bio->bi_opf \|= REQ_NOWAIT;
	970	+
966	971	/*
967	972	* For a REQ_NOWAIT based request, return -EOPNOTSUPP
968	973	* if queue is not a request based queue.
@@ -1802,6 +1807,7 @@ void blk_start_plug(struct blk_plug *plug) · 833 unmodified lines
1802	1807	INIT_LIST_HEAD(&plug->cb_list);
1803	1808	plug->rq_count = 0;
1804	1809	plug->multiple_queues = false;
	1810	+ plug->nowait = false;
1805	1811
1806	1812	/*
1807	1813	* Store ordering should not be needed here, since a potential

fs/block_dev.c+1−1

@@ -1734,7 +1734,7 @@ static int blkdev_open(struct inode * inode, struct file * filp) · 1733 unmodified lines
1734	1734	*/
1735	1735	filp->f_flags \|= O_LARGEFILE;
1736	1736
1737		− filp->f_mode \|= FMODE_NOWAIT;
	1737	+ filp->f_mode \|= FMODE_NOWAIT \| FMODE_BUF_RASYNC;
1738	1738
1739	1739	if (filp->f_flags & O_NDELAY)
1740	1740	filp->f_mode \|= FMODE_NDELAY;

fs/btrfs/file.c+1−1

@@ -3537,7 +3537,7 @@ static loff_t btrfs_file_llseek(struct file *file, loff_t offset, int whence) · 3536 unmodified lines
3537	3537
3538	3538	static int btrfs_file_open(struct inode inode, struct file filp)
3539	3539	{
3540		− filp->f_mode \|= FMODE_NOWAIT;
	3540	+ filp->f_mode \|= FMODE_NOWAIT \| FMODE_BUF_RASYNC;
3541	3541	return generic_file_open(inode, filp);
3542	3542	}
3543	3543

fs/io-wq.c+5−9

@@ -462,6 +462,7 @@ static void io_impersonate_work(struct io_worker *worker, · 461 unmodified lines
462	462	io_wq_switch_mm(worker, work);
463	463	if (worker->cur_creds != work->creds)
464	464	io_wq_switch_creds(worker, work);
	465	+ current->signal->rlim[RLIMIT_FSIZE].rlim_cur = work->fsize;
465	466	}
466	467
467	468	static void io_assign_current_work(struct io_worker *worker,
@@ -489,7 +490,6 @@ static void io_worker_handle_work(struct io_worker *worker) · 21 unmodified lines
489	490
490	491	do {
491	492	struct io_wq_work *work;
492		− unsigned int hash;
493	493	get_next:
494	494	/*
495	495	* If we got some work, mark us as busy. If we didn't, but
@@ -512,6 +512,7 @@ get_next: · 16 unmodified lines
512	512	/* handle a whole dependent link */
513	513	do {
514	514	struct io_wq_work old_work, next_hashed, *linked;
	515	+ unsigned int hash = io_get_work_hash(work);
515	516
516	517	next_hashed = wq_next_work(work);
517	518	io_impersonate_work(worker, work);
@@ -522,10 +523,8 @@ get_next: · 4 unmodified lines
522	523	if (test_bit(IO_WQ_BIT_CANCEL, &wq->state))
523	524	work->flags \|= IO_WQ_WORK_CANCEL;
524	525
525		− hash = io_get_work_hash(work);
526		− linked = old_work = work;
527		− wq->do_work(&linked);
528		− linked = (old_work == linked) ? NULL : linked;
	526	+ old_work = work;
	527	+ linked = wq->do_work(work);
529	528
530	529	work = next_hashed;
531	530	if (!work && linked && !io_wq_is_hashed(linked)) {
@@ -542,8 +541,6 @@ get_next: · 10 unmodified lines
542	541	spin_lock_irq(&wqe->lock);
543	542	wqe->hash_map &= ~BIT_ULL(hash);
544	543	wqe->flags &= ~IO_WQE_FLAG_STALLED;
545		− /* dependent work is not hashed */
546		− hash = -1U;
547	544	/* skip unnecessary unlock-lock wqe->lock */
548	545	if (!work)
549	546	goto get_next;
@@ -781,8 +778,7 @@ static void io_run_cancel(struct io_wq_work work, struct io_wqe wqe) · 231 unmodified lines
781	778	struct io_wq_work *old_work = work;
782	779
783	780	work->flags \|= IO_WQ_WORK_CANCEL;
784		− wq->do_work(&work);
785		− work = (work == old_work) ? NULL : work;
	781	+ work = wq->do_work(work);
786	782	wq->free_work(old_work);
787	783	} while (work);
788	784	}

fs/io-wq.h+6−5

@@ -5,10 +5,10 @@ struct io_wq; · 4 unmodified lines
5	5
6	6	enum {
7	7	IO_WQ_WORK_CANCEL = 1,
8		− IO_WQ_WORK_HASHED = 4,
9		− IO_WQ_WORK_UNBOUND = 32,
10		− IO_WQ_WORK_NO_CANCEL = 256,
11		− IO_WQ_WORK_CONCURRENT = 512,
	8	+ IO_WQ_WORK_HASHED = 2,
	9	+ IO_WQ_WORK_UNBOUND = 4,
	10	+ IO_WQ_WORK_NO_CANCEL = 8,
	11	+ IO_WQ_WORK_CONCURRENT = 16,
12	12
13	13	IO_WQ_HASH_SHIFT = 24, /* upper 8 bits are used for hash key */
14	14	};
@@ -89,6 +89,7 @@ struct io_wq_work { · 74 unmodified lines
89	89	struct mm_struct *mm;
90	90	const struct cred *creds;
91	91	struct fs_struct *fs;
	92	+ unsigned long fsize;
92	93	unsigned flags;
93	94	};
94	95
@@ -101,7 +102,7 @@ static inline struct io_wq_work wq_next_work(struct io_wq_work work) · 6 unmodified lines
101	102	}
102	103
103	104	typedef void (free_work_fn)(struct io_wq_work *);
104		−typedef void (io_wq_work_fn)(struct io_wq_work **);
	105	+typedef struct io_wq_work (io_wq_work_fn)(struct io_wq_work );
105	106
106	107	struct io_wq_data {
107	108	struct user_struct *user;

fs/io_uring.c+1482−1129

@@ -78,6 +78,7 @@ · 77 unmodified lines
78	78	#include <linux/fs_struct.h>
79	79	#include <linux/splice.h>
80	80	#include <linux/task_work.h>
	81	+#include <linux/pagemap.h>
81	82
82	83	#define CREATE_TRACE_POINTS
83	84	#include <trace/events/io_uring.h>
@@ -226,7 +227,7 @@ struct io_ring_ctx { · 142 unmodified lines
226	227	struct {
227	228	unsigned int flags;
228	229	unsigned int compat: 1;
229		− unsigned int account_mem: 1;
	230	+ unsigned int limit_mem: 1;
230	231	unsigned int cq_overflow_flushed: 1;
231	232	unsigned int drain_next: 1;
232	233	unsigned int eventfd_async: 1;
@@ -319,12 +320,12 @@ struct io_ring_ctx { · 86 unmodified lines
319	320	spinlock_t completion_lock;
320	321
321	322	/*
322		− * ->poll_list is protected by the ctx->uring_lock for
	323	+ * ->iopoll_list is protected by the ctx->uring_lock for
323	324	* io_uring instances that don't use IORING_SETUP_SQPOLL.
324	325	* For SQPOLL, only the single threaded io_sq_thread() will
325	326	* manipulate the list, hence no extra locking is needed there.
326	327	*/
327		− struct list_head poll_list;
	328	+ struct list_head iopoll_list;
328	329	struct hlist_head *cancel_hash;
329	330	unsigned cancel_hash_bits;
330	331	bool poll_multi_file;
@@ -395,6 +396,7 @@ struct io_timeout { · 64 unmodified lines
395	396	int flags;
396	397	u32 off;
397	398	u32 target_seq;
	399	+ struct list_head list;
398	400	};
399	401
400	402	struct io_rw {
@@ -413,7 +415,7 @@ struct io_connect { · 12 unmodified lines
413	415	struct io_sr_msg {
414	416	struct file *file;
415	417	union {
416		− struct user_msghdr __user *msg;
	418	+ struct user_msghdr __user *umsg;
417	419	void __user *buf;
418	420	};
419	421	int msg_flags;
@@ -486,6 +488,12 @@ struct io_statx { · 66 unmodified lines
486	488	struct statx __user *buffer;
487	489	};
488	490
	491	+struct io_completion {
	492	+ struct file *file;
	493	+ struct list_head list;
	494	+ int cflags;
	495	+};
	496	+
489	497	struct io_async_connect {
490	498	struct sockaddr_storage address;
491	499	};
@@ -503,6 +511,7 @@ struct io_async_rw { · 11 unmodified lines
503	511	struct iovec *iov;
504	512	ssize_t nr_segs;
505	513	ssize_t size;
	514	+ struct wait_page_queue wpq;
506	515	};
507	516
508	517	struct io_async_ctx {
@@ -523,23 +532,18 @@ enum { · 14 unmodified lines
523	532	REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,
524	533
525	534	REQ_F_LINK_HEAD_BIT,
526		− REQ_F_LINK_NEXT_BIT,
527	535	REQ_F_FAIL_LINK_BIT,
528	536	REQ_F_INFLIGHT_BIT,
529	537	REQ_F_CUR_POS_BIT,
530	538	REQ_F_NOWAIT_BIT,
531	539	REQ_F_LINK_TIMEOUT_BIT,
532		− REQ_F_TIMEOUT_BIT,
533	540	REQ_F_ISREG_BIT,
534		− REQ_F_MUST_PUNT_BIT,
535		− REQ_F_TIMEOUT_NOSEQ_BIT,
536	541	REQ_F_COMP_LOCKED_BIT,
537	542	REQ_F_NEED_CLEANUP_BIT,
538	543	REQ_F_OVERFLOW_BIT,
539	544	REQ_F_POLLED_BIT,
540	545	REQ_F_BUFFER_SELECTED_BIT,
541	546	REQ_F_NO_FILE_TABLE_BIT,
542		− REQ_F_QUEUE_TIMEOUT_BIT,
543	547	REQ_F_WORK_INITIALIZED_BIT,
544	548	REQ_F_TASK_PINNED_BIT,
545	549
@@ -563,8 +567,6 @@ enum { · 17 unmodified lines
563	567
564	568	/* head of a link */
565	569	REQ_F_LINK_HEAD = BIT(REQ_F_LINK_HEAD_BIT),
566		− /* already grabbed next link */
567		− REQ_F_LINK_NEXT = BIT(REQ_F_LINK_NEXT_BIT),
568	570	/* fail rest of links */
569	571	REQ_F_FAIL_LINK = BIT(REQ_F_FAIL_LINK_BIT),
570	572	/* on inflight list */
@@ -575,14 +577,8 @@ enum { · 4 unmodified lines
575	577	REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),
576	578	/* has linked timeout */
577	579	REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),
578		− /* timeout request */
579		− REQ_F_TIMEOUT = BIT(REQ_F_TIMEOUT_BIT),
580	580	/* regular file */
581	581	REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
582		− /* must be punted even for NONBLOCK */
583		− REQ_F_MUST_PUNT = BIT(REQ_F_MUST_PUNT_BIT),
584		− /* no timeout sequence */
585		− REQ_F_TIMEOUT_NOSEQ = BIT(REQ_F_TIMEOUT_NOSEQ_BIT),
586	582	/* completion under lock */
587	583	REQ_F_COMP_LOCKED = BIT(REQ_F_COMP_LOCKED_BIT),
588	584	/* needs cleanup */
@@ -595,8 +591,6 @@ enum { · 6 unmodified lines
595	591	REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
596	592	/* doesn't need file table for this request */
597	593	REQ_F_NO_FILE_TABLE = BIT(REQ_F_NO_FILE_TABLE_BIT),
598		− /* needs to queue linked timeout */
599		− REQ_F_QUEUE_TIMEOUT = BIT(REQ_F_QUEUE_TIMEOUT_BIT),
600	594	/* io_wq_work is initialized */
601	595	REQ_F_WORK_INITIALIZED = BIT(REQ_F_WORK_INITIALIZED_BIT),
602	596	/* req->task is refcounted */
@@ -606,7 +600,6 @@ enum { · 3 unmodified lines
606	600	struct async_poll {
607	601	struct io_poll_iocb poll;
608	602	struct io_poll_iocb *double_poll;
609		− struct io_wq_work work;
610	603	};
611	604
612	605	/*
@@ -635,51 +628,54 @@ struct io_kiocb { · 22 unmodified lines
635	628	struct io_splice splice;
636	629	struct io_provide_buf pbuf;
637	630	struct io_statx statx;
	631	+ /* use only after cleaning per-op data, see io_clean_op() */
	632	+ struct io_completion compl;
638	633	};
639	634
640	635	struct io_async_ctx *io;
641		− int cflags;
642	636	u8 opcode;
643	637	/* polled IO has completed */
644	638	u8 iopoll_completed;
645	639
646	640	u16 buf_index;
	641	+ u32 result;
647	642
648		− struct io_ring_ctx *ctx;
649		− struct list_head list;
650		− unsigned int flags;
651		− refcount_t refs;
652		− struct task_struct *task;
653		− unsigned long fsize;
654		− u64 user_data;
655		− u32 result;
656		− u32 sequence;
657		−
658		− struct list_head link_list;
	643	+ struct io_ring_ctx *ctx;
	644	+ unsigned int flags;
	645	+ refcount_t refs;
	646	+ struct task_struct *task;
	647	+ u64 user_data;
659	648
660		− struct list_head inflight_entry;
	649	+ struct list_head link_list;
661	650
662		− struct percpu_ref *fixed_file_refs;
	651	+ /*
	652	+ * 1. used with ctx->iopoll_list with reads/writes
	653	+ * 2. to track reqs with ->files (see io_op_def::file_table)
	654	+ */
	655	+ struct list_head inflight_entry;
	656	+
	657	+ struct percpu_ref *fixed_file_refs;
	658	+ struct callback_head task_work;
	659	+ /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
	660	+ struct hlist_node hash_node;
	661	+ struct async_poll *apoll;
	662	+ struct io_wq_work work;
	663	+};
663	664
664		− union {
665		− /*
666		− * Only commands that never go async can use the below fields,
667		− * obviously. Right now only IORING_OP_POLL_ADD uses them, and
668		− * async armed poll handlers for regular commands. The latter
669		− * restore the work, if needed.
670		− */
671		− struct {
672		− struct callback_head task_work;
673		− struct hlist_node hash_node;
674		− struct async_poll *apoll;
675		− };
676		− struct io_wq_work work;
677		− };
	665	+struct io_defer_entry {
	666	+ struct list_head list;
	667	+ struct io_kiocb *req;
	668	+ u32 seq;
678	669	};
679	670
680		−#define IO_PLUG_THRESHOLD 2
681	671	#define IO_IOPOLL_BATCH 8
682	672
	673	+struct io_comp_state {
	674	+ unsigned int nr;
	675	+ struct list_head list;
	676	+ struct io_ring_ctx *ctx;
	677	+};
	678	+
683	679	struct io_submit_state {
684	680	struct blk_plug plug;
685	681
@@ -689,13 +685,17 @@ struct io_submit_state { · 3 unmodified lines
689	685	void *reqs[IO_IOPOLL_BATCH];
690	686	unsigned int free_reqs;
691	687
	688	+ /*
	689	+ * Batch completion logic
	690	+ */
	691	+ struct io_comp_state comp;
	692	+
692	693	/*
693	694	* File reference cache
694	695	*/
695	696	struct file *file;
696	697	unsigned int fd;
697	698	unsigned int has_refs;
698		− unsigned int used_refs;
699	699	unsigned int ios_left;
700	700	};
701	701
@@ -723,6 +723,7 @@ struct io_op_def { · 21 unmodified lines
723	723	unsigned pollout : 1;
724	724	/* op supports buffer selection */
725	725	unsigned buffer_select : 1;
	726	+ unsigned needs_fsize : 1;
726	727	};
727	728
728	729	static const struct io_op_def io_op_defs[] = {
@@ -742,6 +743,7 @@ static const struct io_op_def io_op_defs[] = { · 13 unmodified lines
742	743	.hash_reg_file = 1,
743	744	.unbound_nonreg_file = 1,
744	745	.pollout = 1,
	746	+ .needs_fsize = 1,
745	747	},
746	748	[IORING_OP_FSYNC] = {
747	749	.needs_file = 1,
@@ -756,6 +758,7 @@ static const struct io_op_def io_op_defs[] = { · 8 unmodified lines
756	758	.hash_reg_file = 1,
757	759	.unbound_nonreg_file = 1,
758	760	.pollout = 1,
	761	+ .needs_fsize = 1,
759	762	},
760	763	[IORING_OP_POLL_ADD] = {
761	764	.needs_file = 1,
@@ -808,6 +811,7 @@ static const struct io_op_def io_op_defs[] = { · 46 unmodified lines
808	811	},
809	812	[IORING_OP_FALLOCATE] = {
810	813	.needs_file = 1,
	814	+ .needs_fsize = 1,
811	815	},
812	816	[IORING_OP_OPENAT] = {
813	817	.file_table = 1,
@@ -839,6 +843,7 @@ static const struct io_op_def io_op_defs[] = { · 25 unmodified lines
839	843	.needs_file = 1,
840	844	.unbound_nonreg_file = 1,
841	845	.pollout = 1,
	846	+ .needs_fsize = 1,
842	847	},
843	848	[IORING_OP_FADVISE] = {
844	849	.needs_file = 1,
@@ -881,22 +886,37 @@ static const struct io_op_def io_op_defs[] = { · 36 unmodified lines
881	886	},
882	887	};
883	888
884		−static void io_wq_submit_work(struct io_wq_work **workptr);
	889	+enum io_mem_account {
	890	+ ACCT_LOCKED,
	891	+ ACCT_PINNED,
	892	+};
	893	+
	894	+static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
	895	+ struct io_comp_state *cs);
885	896	static void io_cqring_fill_event(struct io_kiocb *req, long res);
886	897	static void io_put_req(struct io_kiocb *req);
	898	+static void io_double_put_req(struct io_kiocb *req);
887	899	static void __io_double_put_req(struct io_kiocb *req);
888	900	static struct io_kiocb io_prep_linked_timeout(struct io_kiocb req);
889	901	static void io_queue_linked_timeout(struct io_kiocb *req);
890	902	static int __io_sqe_files_update(struct io_ring_ctx *ctx,
891	903	struct io_uring_files_update *ip,
892	904	unsigned nr_args);
893		−static int io_grab_files(struct io_kiocb *req);
894		−static void io_complete_rw_common(struct kiocb *kiocb, long res);
895		−static void io_cleanup_req(struct io_kiocb *req);
	905	+static int io_prep_work_files(struct io_kiocb *req);
	906	+static void __io_clean_op(struct io_kiocb *req);
896	907	static int io_file_get(struct io_submit_state state, struct io_kiocb req,
897	908	int fd, struct file **out_file, bool fixed);
898	909	static void __io_queue_sqe(struct io_kiocb *req,
899		− const struct io_uring_sqe *sqe);
	910	+ const struct io_uring_sqe *sqe,
	911	+ struct io_comp_state *cs);
	912	+static void io_file_put_work(struct work_struct *work);
	913	+
	914	+static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
	915	+ struct iovec *iovec, struct iov_iter iter,
	916	+ bool needs_lock);
	917	+static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size,
	918	+ struct iovec iovec, struct iovec fast_iov,
	919	+ struct iov_iter *iter);
900	920
901	921	static struct kmem_cache *req_cachep;
902	922
@@ -923,6 +943,12 @@ static void io_get_req_task(struct io_kiocb *req) · 20 unmodified lines
923	943	req->flags \|= REQ_F_TASK_PINNED;
924	944	}
925	945
	946	+static inline void io_clean_op(struct io_kiocb *req)
	947	+{
	948	+ if (req->flags & (REQ_F_NEED_CLEANUP \| REQ_F_BUFFER_SELECTED))
	949	+ __io_clean_op(req);
	950	+}
	951	+
926	952	/* not idempotent -- it doesn't clear REQ_F_TASK_PINNED */
927	953	static void __io_put_req_task(struct io_kiocb *req)
928	954	{
@@ -930,7 +956,41 @@ static void __io_put_req_task(struct io_kiocb *req) · 1 unmodified line
930	956	put_task_struct(req->task);
931	957	}
932	958
933		−static void io_file_put_work(struct work_struct *work);
	959	+static void io_sq_thread_drop_mm(void)
	960	+{
	961	+ struct mm_struct *mm = current->mm;
	962	+
	963	+ if (mm) {
	964	+ kthread_unuse_mm(mm);
	965	+ mmput(mm);
	966	+ }
	967	+}
	968	+
	969	+static int __io_sq_thread_acquire_mm(struct io_ring_ctx *ctx)
	970	+{
	971	+ if (!current->mm) {
	972	+ if (unlikely(!(ctx->flags & IORING_SETUP_SQPOLL) \|\|
	973	+ !mmget_not_zero(ctx->sqo_mm)))
	974	+ return -EFAULT;
	975	+ kthread_use_mm(ctx->sqo_mm);
	976	+ }
	977	+
	978	+ return 0;
	979	+}
	980	+
	981	+static int io_sq_thread_acquire_mm(struct io_ring_ctx *ctx,
	982	+ struct io_kiocb *req)
	983	+{
	984	+ if (!io_op_defs[req->opcode].needs_mm)
	985	+ return 0;
	986	+ return __io_sq_thread_acquire_mm(ctx);
	987	+}
	988	+
	989	+static inline void req_set_fail_links(struct io_kiocb *req)
	990	+{
	991	+ if ((req->flags & (REQ_F_LINK \| REQ_F_HARDLINK)) == REQ_F_LINK)
	992	+ req->flags \|= REQ_F_FAIL_LINK;
	993	+}
934	994
935	995	/*
936	996	* Note: must call io_req_init_async() for the first time you
@@ -957,6 +1017,11 @@ static void io_ring_ctx_ref_free(struct percpu_ref *ref) · 20 unmodified lines
957	1017	complete(&ctx->ref_comp);
958	1018	}
959	1019
	1020	+static inline bool io_is_timeout_noseq(struct io_kiocb *req)
	1021	+{
	1022	+ return !req->timeout.off;
	1023	+}
	1024	+
960	1025	static struct io_ring_ctx io_ring_ctx_alloc(struct io_uring_params p)
961	1026	{
962	1027	struct io_ring_ctx *ctx;
@@ -1000,7 +1065,7 @@ static struct io_ring_ctx io_ring_ctx_alloc(struct io_uring_params p) · 37 unmodified lines
1000	1065	mutex_init(&ctx->uring_lock);
1001	1066	init_waitqueue_head(&ctx->wait);
1002	1067	spin_lock_init(&ctx->completion_lock);
1003		− INIT_LIST_HEAD(&ctx->poll_list);
	1068	+ INIT_LIST_HEAD(&ctx->iopoll_list);
1004	1069	INIT_LIST_HEAD(&ctx->defer_list);
1005	1070	INIT_LIST_HEAD(&ctx->timeout_list);
1006	1071	init_waitqueue_head(&ctx->inflight_wait);
@@ -1017,18 +1082,14 @@ err: · 10 unmodified lines
1017	1082	return NULL;
1018	1083	}
1019	1084
1020		−static inline bool __req_need_defer(struct io_kiocb *req)
	1085	+static bool req_need_defer(struct io_kiocb *req, u32 seq)
1021	1086	{
1022		− struct io_ring_ctx *ctx = req->ctx;
	1087	+ if (unlikely(req->flags & REQ_F_IO_DRAIN)) {
	1088	+ struct io_ring_ctx *ctx = req->ctx;
1023	1089
1024		− return req->sequence != ctx->cached_cq_tail
	1090	+ return seq != ctx->cached_cq_tail
1025	1091	+ atomic_read(&ctx->cached_cq_overflow);
1026		−}
1027		−
1028		−static inline bool req_need_defer(struct io_kiocb *req)
1029		−{
1030		− if (unlikely(req->flags & REQ_F_IO_DRAIN))
1031		− return __req_need_defer(req);
	1092	+ }
1032	1093
1033	1094	return false;
1034	1095	}
@@ -1046,28 +1107,7 @@ static void __io_commit_cqring(struct io_ring_ctx *ctx) · 11 unmodified lines
1046	1107	}
1047	1108	}
1048	1109
1049		−static inline void io_req_work_grab_env(struct io_kiocb *req,
1050		− const struct io_op_def *def)
1051		−{
1052		− if (!req->work.mm && def->needs_mm) {
1053		− mmgrab(current->mm);
1054		− req->work.mm = current->mm;
1055		− }
1056		− if (!req->work.creds)
1057		− req->work.creds = get_current_cred();
1058		− if (!req->work.fs && def->needs_fs) {
1059		− spin_lock(&current->fs->lock);
1060		− if (!current->fs->in_exec) {
1061		− req->work.fs = current->fs;
1062		− req->work.fs->users++;
1063		− } else {
1064		− req->work.flags \|= IO_WQ_WORK_CANCEL;
1065		− }
1066		− spin_unlock(&current->fs->lock);
1067		− }
1068		−}
1069		−
1070		−static inline void io_req_work_drop_env(struct io_kiocb *req)
	1110	+static void io_req_clean_work(struct io_kiocb *req)
1071	1111	{
1072	1112	if (!(req->flags & REQ_F_WORK_INITIALIZED))
1073	1113	return;
@@ -1089,11 +1129,12 @@ static inline void io_req_work_drop_env(struct io_kiocb *req) · 15 unmodified lines
1089	1129	spin_unlock(&req->work.fs->lock);
1090	1130	if (fs)
1091	1131	free_fs_struct(fs);
	1132	+ req->work.fs = NULL;
1092	1133	}
	1134	+ req->flags &= ~REQ_F_WORK_INITIALIZED;
1093	1135	}
1094	1136
1095		−static inline void io_prep_async_work(struct io_kiocb *req,
1096		− struct io_kiocb **link)
	1137	+static void io_prep_async_work(struct io_kiocb *req)
1097	1138	{
1098	1139	const struct io_op_def *def = &io_op_defs[req->opcode];
1099	1140
@@ -1106,18 +1147,42 @@ static inline void io_prep_async_work(struct io_kiocb *req, · 6 unmodified lines
1106	1147	if (def->unbound_nonreg_file)
1107	1148	req->work.flags \|= IO_WQ_WORK_UNBOUND;
1108	1149	}
	1150	+ if (!req->work.mm && def->needs_mm) {
	1151	+ mmgrab(current->mm);
	1152	+ req->work.mm = current->mm;
	1153	+ }
	1154	+ if (!req->work.creds)
	1155	+ req->work.creds = get_current_cred();
	1156	+ if (!req->work.fs && def->needs_fs) {
	1157	+ spin_lock(&current->fs->lock);
	1158	+ if (!current->fs->in_exec) {
	1159	+ req->work.fs = current->fs;
	1160	+ req->work.fs->users++;
	1161	+ } else {
	1162	+ req->work.flags \|= IO_WQ_WORK_CANCEL;
	1163	+ }
	1164	+ spin_unlock(&current->fs->lock);
	1165	+ }
	1166	+ if (def->needs_fsize)
	1167	+ req->work.fsize = rlimit(RLIMIT_FSIZE);
	1168	+ else
	1169	+ req->work.fsize = RLIM_INFINITY;
	1170	+}
1109	1171
1110		− io_req_work_grab_env(req, def);
	1172	+static void io_prep_async_link(struct io_kiocb *req)
	1173	+{
	1174	+ struct io_kiocb *cur;
1111	1175
1112		− *link = io_prep_linked_timeout(req);
	1176	+ io_prep_async_work(req);
	1177	+ if (req->flags & REQ_F_LINK_HEAD)
	1178	+ list_for_each_entry(cur, &req->link_list, link_list)
	1179	+ io_prep_async_work(cur);
1113	1180	}
1114	1181
1115		−static inline void io_queue_async_work(struct io_kiocb *req)
	1182	+static void __io_queue_async_work(struct io_kiocb *req)
1116	1183	{
1117	1184	struct io_ring_ctx *ctx = req->ctx;
1118		− struct io_kiocb *link;
1119		−
1120		− io_prep_async_work(req, &link);
	1185	+ struct io_kiocb *link = io_prep_linked_timeout(req);
1121	1186
1122	1187	trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
1123	1188	&req->work, req->flags);
@@ -1127,14 +1192,22 @@ static inline void io_queue_async_work(struct io_kiocb *req) · 3 unmodified lines
1127	1192	io_queue_linked_timeout(link);
1128	1193	}
1129	1194
	1195	+static void io_queue_async_work(struct io_kiocb *req)
	1196	+{
	1197	+ /* init ->work of the whole link before punting */
	1198	+ io_prep_async_link(req);
	1199	+ __io_queue_async_work(req);
	1200	+}
	1201	+
1130	1202	static void io_kill_timeout(struct io_kiocb *req)
1131	1203	{
1132	1204	int ret;
1133	1205
1134	1206	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
1135	1207	if (ret != -1) {
1136		− atomic_inc(&req->ctx->cq_timeouts);
1137		− list_del_init(&req->list);
	1208	+ atomic_set(&req->ctx->cq_timeouts,
	1209	+ atomic_read(&req->ctx->cq_timeouts) + 1);
	1210	+ list_del_init(&req->timeout.list);
1138	1211	req->flags \|= REQ_F_COMP_LOCKED;
1139	1212	io_cqring_fill_event(req, 0);
1140	1213	io_put_req(req);
@@ -1146,7 +1219,7 @@ static void io_kill_timeouts(struct io_ring_ctx *ctx) · 5 unmodified lines
1146	1219	struct io_kiocb req, tmp;
1147	1220
1148	1221	spin_lock_irq(&ctx->completion_lock);
1149		− list_for_each_entry_safe(req, tmp, &ctx->timeout_list, list)
	1222	+ list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list)
1150	1223	io_kill_timeout(req);
1151	1224	spin_unlock_irq(&ctx->completion_lock);
1152	1225	}
@@ -1154,13 +1227,15 @@ static void io_kill_timeouts(struct io_ring_ctx *ctx) · 1 unmodified line
1154	1227	static void __io_queue_deferred(struct io_ring_ctx *ctx)
1155	1228	{
1156	1229	do {
1157		− struct io_kiocb *req = list_first_entry(&ctx->defer_list,
1158		− struct io_kiocb, list);
	1230	+ struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
	1231	+ struct io_defer_entry, list);
1159	1232
1160		− if (req_need_defer(req))
	1233	+ if (req_need_defer(de->req, de->seq))
1161	1234	break;
1162		− list_del_init(&req->list);
1163		− io_queue_async_work(req);
	1235	+ list_del_init(&de->list);
	1236	+ /* punt-init is done before queueing for defer */
	1237	+ __io_queue_async_work(de->req);
	1238	+ kfree(de);
1164	1239	} while (!list_empty(&ctx->defer_list));
1165	1240	}
1166	1241
@@ -1168,15 +1243,15 @@ static void io_flush_timeouts(struct io_ring_ctx *ctx) · 1 unmodified line
1168	1243	{
1169	1244	while (!list_empty(&ctx->timeout_list)) {
1170	1245	struct io_kiocb *req = list_first_entry(&ctx->timeout_list,
1171		− struct io_kiocb, list);
	1246	+ struct io_kiocb, timeout.list);
1172	1247
1173		− if (req->flags & REQ_F_TIMEOUT_NOSEQ)
	1248	+ if (io_is_timeout_noseq(req))
1174	1249	break;
1175	1250	if (req->timeout.target_seq != ctx->cached_cq_tail
1176	1251	- atomic_read(&ctx->cq_timeouts))
1177	1252	break;
1178	1253
1179		− list_del_init(&req->list);
	1254	+ list_del_init(&req->timeout.list);
1180	1255	io_kill_timeout(req);
1181	1256	}
1182	1257	}
@@ -1229,6 +1304,15 @@ static void io_cqring_ev_posted(struct io_ring_ctx *ctx) · 46 unmodified lines
1229	1304	eventfd_signal(ctx->cq_ev_fd, 1);
1230	1305	}
1231	1306
	1307	+static void io_cqring_mark_overflow(struct io_ring_ctx *ctx)
	1308	+{
	1309	+ if (list_empty(&ctx->cq_overflow_list)) {
	1310	+ clear_bit(0, &ctx->sq_check_overflow);
	1311	+ clear_bit(0, &ctx->cq_check_overflow);
	1312	+ ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW;
	1313	+ }
	1314	+}
	1315	+
1232	1316	/* Returns true if there are no backlogged entries after the flush */
1233	1317	static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
1234	1318	{
@@ -1259,13 +1343,13 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) · 24 unmodified lines
1259	1343	break;
1260	1344
1261	1345	req = list_first_entry(&ctx->cq_overflow_list, struct io_kiocb,
1262		− list);
1263		− list_move(&req->list, &list);
	1346	+ compl.list);
	1347	+ list_move(&req->compl.list, &list);
1264	1348	req->flags &= ~REQ_F_OVERFLOW;
1265	1349	if (cqe) {
1266	1350	WRITE_ONCE(cqe->user_data, req->user_data);
1267	1351	WRITE_ONCE(cqe->res, req->result);
1268		− WRITE_ONCE(cqe->flags, req->cflags);
	1352	+ WRITE_ONCE(cqe->flags, req->compl.cflags);
1269	1353	} else {
1270	1354	WRITE_ONCE(ctx->rings->cq_overflow,
1271	1355	atomic_inc_return(&ctx->cached_cq_overflow));
@@ -1273,17 +1357,14 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) · 1 unmodified line
1273	1357	}
1274	1358
1275	1359	io_commit_cqring(ctx);
1276		− if (cqe) {
1277		− clear_bit(0, &ctx->sq_check_overflow);
1278		− clear_bit(0, &ctx->cq_check_overflow);
1279		− ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW;
1280		− }
	1360	+ io_cqring_mark_overflow(ctx);
	1361	+
1281	1362	spin_unlock_irqrestore(&ctx->completion_lock, flags);
1282	1363	io_cqring_ev_posted(ctx);
1283	1364
1284	1365	while (!list_empty(&list)) {
1285		− req = list_first_entry(&list, struct io_kiocb, list);
1286		− list_del(&req->list);
	1366	+ req = list_first_entry(&list, struct io_kiocb, compl.list);
	1367	+ list_del(&req->compl.list);
1287	1368	io_put_req(req);
1288	1369	}
1289	1370
@@ -1316,11 +1397,12 @@ static void __io_cqring_fill_event(struct io_kiocb *req, long res, long cflags) · 26 unmodified lines
1316	1397	set_bit(0, &ctx->cq_check_overflow);
1317	1398	ctx->rings->sq_flags \|= IORING_SQ_CQ_OVERFLOW;
1318	1399	}
	1400	+ io_clean_op(req);
1319	1401	req->flags \|= REQ_F_OVERFLOW;
1320		− refcount_inc(&req->refs);
1321	1402	req->result = res;
1322		− req->cflags = cflags;
1323		− list_add_tail(&req->list, &ctx->cq_overflow_list);
	1403	+ req->compl.cflags = cflags;
	1404	+ refcount_inc(&req->refs);
	1405	+ list_add_tail(&req->compl.list, &ctx->cq_overflow_list);
1324	1406	}
1325	1407	}
1326	1408
@@ -1329,7 +1411,7 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res) · 2 unmodified lines
1329	1411	__io_cqring_fill_event(req, res, 0);
1330	1412	}
1331	1413
1332		−static void __io_cqring_add_event(struct io_kiocb *req, long res, long cflags)
	1414	+static void io_cqring_add_event(struct io_kiocb *req, long res, long cflags)
1333	1415	{
1334	1416	struct io_ring_ctx *ctx = req->ctx;
1335	1417	unsigned long flags;
@@ -1342,9 +1424,52 @@ static void __io_cqring_add_event(struct io_kiocb *req, long res, long cflags) · 6 unmodified lines
1342	1424	io_cqring_ev_posted(ctx);
1343	1425	}
1344	1426
1345		−static void io_cqring_add_event(struct io_kiocb *req, long res)
	1427	+static void io_submit_flush_completions(struct io_comp_state *cs)
	1428	+{
	1429	+ struct io_ring_ctx *ctx = cs->ctx;
	1430	+
	1431	+ spin_lock_irq(&ctx->completion_lock);
	1432	+ while (!list_empty(&cs->list)) {
	1433	+ struct io_kiocb *req;
	1434	+
	1435	+ req = list_first_entry(&cs->list, struct io_kiocb, compl.list);
	1436	+ list_del(&req->compl.list);
	1437	+ __io_cqring_fill_event(req, req->result, req->compl.cflags);
	1438	+ if (!(req->flags & REQ_F_LINK_HEAD)) {
	1439	+ req->flags \|= REQ_F_COMP_LOCKED;
	1440	+ io_put_req(req);
	1441	+ } else {
	1442	+ spin_unlock_irq(&ctx->completion_lock);
	1443	+ io_put_req(req);
	1444	+ spin_lock_irq(&ctx->completion_lock);
	1445	+ }
	1446	+ }
	1447	+ io_commit_cqring(ctx);
	1448	+ spin_unlock_irq(&ctx->completion_lock);
	1449	+
	1450	+ io_cqring_ev_posted(ctx);
	1451	+ cs->nr = 0;
	1452	+}
	1453	+
	1454	+static void __io_req_complete(struct io_kiocb *req, long res, unsigned cflags,
	1455	+ struct io_comp_state *cs)
	1456	+{
	1457	+ if (!cs) {
	1458	+ io_cqring_add_event(req, res, cflags);
	1459	+ io_put_req(req);
	1460	+ } else {
	1461	+ io_clean_op(req);
	1462	+ req->result = res;
	1463	+ req->compl.cflags = cflags;
	1464	+ list_add_tail(&req->compl.list, &cs->list);
	1465	+ if (++cs->nr >= 32)
	1466	+ io_submit_flush_completions(cs);
	1467	+ }
	1468	+}
	1469	+
	1470	+static void io_req_complete(struct io_kiocb *req, long res)
1346	1471	{
1347		− __io_cqring_add_event(req, res, 0);
	1472	+ __io_req_complete(req, res, 0, NULL);
1348	1473	}
1349	1474
1350	1475	static inline bool io_is_fallback_req(struct io_kiocb *req)
@@ -1370,11 +1495,7 @@ static struct io_kiocb io_alloc_req(struct io_ring_ctx ctx, · 19 unmodified lines
1370	1495	gfp_t gfp = GFP_KERNEL \| __GFP_NOWARN;
1371	1496	struct io_kiocb *req;
1372	1497
1373		− if (!state) {
1374		− req = kmem_cache_alloc(req_cachep, gfp);
1375		− if (unlikely(!req))
1376		− goto fallback;
1377		− } else if (!state->free_reqs) {
	1498	+ if (!state->free_reqs) {
1378	1499	size_t sz;
1379	1500	int ret;
1380	1501
@@ -1412,21 +1533,15 @@ static inline void io_put_file(struct io_kiocb req, struct file file, · 31 unmodified lines
1412	1533	fput(file);
1413	1534	}
1414	1535
1415		−static void __io_req_aux_free(struct io_kiocb *req)
	1536	+static void io_dismantle_req(struct io_kiocb *req)
1416	1537	{
1417		− if (req->flags & REQ_F_NEED_CLEANUP)
1418		− io_cleanup_req(req);
	1538	+ io_clean_op(req);
1419	1539
1420		− kfree(req->io);
	1540	+ if (req->io)
	1541	+ kfree(req->io);
1421	1542	if (req->file)
1422	1543	io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE));
1423		− __io_put_req_task(req);
1424		− io_req_work_drop_env(req);
1425		−}
1426		−
1427		−static void __io_free_req(struct io_kiocb *req)
1428		−{
1429		− __io_req_aux_free(req);
	1544	+ io_req_clean_work(req);
1430	1545
1431	1546	if (req->flags & REQ_F_INFLIGHT) {
1432	1547	struct io_ring_ctx *ctx = req->ctx;
@@ -1438,57 +1553,20 @@ static void __io_free_req(struct io_kiocb *req) · 5 unmodified lines
1438	1553	wake_up(&ctx->inflight_wait);
1439	1554	spin_unlock_irqrestore(&ctx->inflight_lock, flags);
1440	1555	}
1441		−
1442		− percpu_ref_put(&req->ctx->refs);
1443		− if (likely(!io_is_fallback_req(req)))
1444		− kmem_cache_free(req_cachep, req);
1445		− else
1446		− clear_bit_unlock(0, (unsigned long *) &req->ctx->fallback_req);
1447	1556	}
1448	1557
1449		−struct req_batch {
1450		− void *reqs[IO_IOPOLL_BATCH];
1451		− int to_free;
1452		− int need_iter;
1453		−};
1454		−
1455		−static void io_free_req_many(struct io_ring_ctx ctx, struct req_batch rb)
	1558	+static void __io_free_req(struct io_kiocb *req)
1456	1559	{
1457		− if (!rb->to_free)
1458		− return;
1459		− if (rb->need_iter) {
1460		− int i, inflight = 0;
1461		− unsigned long flags;
1462		−
1463		− for (i = 0; i < rb->to_free; i++) {
1464		− struct io_kiocb *req = rb->reqs[i];
1465		−
1466		− if (req->flags & REQ_F_INFLIGHT)
1467		− inflight++;
1468		− __io_req_aux_free(req);
1469		− }
1470		− if (!inflight)
1471		− goto do_free;
1472		−
1473		− spin_lock_irqsave(&ctx->inflight_lock, flags);
1474		− for (i = 0; i < rb->to_free; i++) {
1475		− struct io_kiocb *req = rb->reqs[i];
1476		−
1477		− if (req->flags & REQ_F_INFLIGHT) {
1478		− list_del(&req->inflight_entry);
1479		− if (!--inflight)
1480		− break;
1481		− }
1482		− }
1483		− spin_unlock_irqrestore(&ctx->inflight_lock, flags);
	1560	+ struct io_ring_ctx *ctx;
1484	1561
1485		− if (waitqueue_active(&ctx->inflight_wait))
1486		− wake_up(&ctx->inflight_wait);
1487		− }
1488		−do_free:
1489		− kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs);
1490		− percpu_ref_put_many(&ctx->refs, rb->to_free);
1491		− rb->to_free = rb->need_iter = 0;
	1562	+ io_dismantle_req(req);
	1563	+ __io_put_req_task(req);
	1564	+ ctx = req->ctx;
	1565	+ if (likely(!io_is_fallback_req(req)))
	1566	+ kmem_cache_free(req_cachep, req);
	1567	+ else
	1568	+ clear_bit_unlock(0, (unsigned long *) &ctx->fallback_req);
	1569	+ percpu_ref_put(&ctx->refs);
1492	1570	}
1493	1571
1494	1572	static bool io_link_cancel_timeout(struct io_kiocb *req)
@@ -1508,53 +1586,67 @@ static bool io_link_cancel_timeout(struct io_kiocb *req) · 13 unmodified lines
1508	1586	return false;
1509	1587	}
1510	1588
1511		−static void io_req_link_next(struct io_kiocb req, struct io_kiocb *nxtptr)
	1589	+static bool __io_kill_linked_timeout(struct io_kiocb *req)
	1590	+{
	1591	+ struct io_kiocb *link;
	1592	+ bool wake_ev;
	1593	+
	1594	+ if (list_empty(&req->link_list))
	1595	+ return false;
	1596	+ link = list_first_entry(&req->link_list, struct io_kiocb, link_list);
	1597	+ if (link->opcode != IORING_OP_LINK_TIMEOUT)
	1598	+ return false;
	1599	+
	1600	+ list_del_init(&link->link_list);
	1601	+ wake_ev = io_link_cancel_timeout(link);
	1602	+ req->flags &= ~REQ_F_LINK_TIMEOUT;
	1603	+ return wake_ev;
	1604	+}
	1605	+
	1606	+static void io_kill_linked_timeout(struct io_kiocb *req)
1512	1607	{
1513	1608	struct io_ring_ctx *ctx = req->ctx;
1514		− bool wake_ev = false;
	1609	+ bool wake_ev;
1515	1610
1516		− /* Already got next link */
1517		− if (req->flags & REQ_F_LINK_NEXT)
1518		− return;
	1611	+ if (!(req->flags & REQ_F_COMP_LOCKED)) {
	1612	+ unsigned long flags;
	1613	+
	1614	+ spin_lock_irqsave(&ctx->completion_lock, flags);
	1615	+ wake_ev = __io_kill_linked_timeout(req);
	1616	+ spin_unlock_irqrestore(&ctx->completion_lock, flags);
	1617	+ } else {
	1618	+ wake_ev = __io_kill_linked_timeout(req);
	1619	+ }
	1620	+
	1621	+ if (wake_ev)
	1622	+ io_cqring_ev_posted(ctx);
	1623	+}
	1624	+
	1625	+static struct io_kiocb io_req_link_next(struct io_kiocb req)
	1626	+{
	1627	+ struct io_kiocb *nxt;
1519	1628
1520	1629	/*
1521	1630	* The list should never be empty when we are called here. But could
1522	1631	* potentially happen if the chain is messed up, check to be on the
1523	1632	* safe side.
1524	1633	*/
1525		− while (!list_empty(&req->link_list)) {
1526		− struct io_kiocb *nxt = list_first_entry(&req->link_list,
1527		− struct io_kiocb, link_list);
1528		−
1529		− if (unlikely((req->flags & REQ_F_LINK_TIMEOUT) &&
1530		− (nxt->flags & REQ_F_TIMEOUT))) {
1531		− list_del_init(&nxt->link_list);
1532		− wake_ev \|= io_link_cancel_timeout(nxt);
1533		− req->flags &= ~REQ_F_LINK_TIMEOUT;
1534		− continue;
1535		− }
1536		−
1537		− list_del_init(&req->link_list);
1538		− if (!list_empty(&nxt->link_list))
1539		− nxt->flags \|= REQ_F_LINK_HEAD;
1540		− *nxtptr = nxt;
1541		− break;
1542		− }
	1634	+ if (unlikely(list_empty(&req->link_list)))
	1635	+ return NULL;
1543	1636
1544		− req->flags \|= REQ_F_LINK_NEXT;
1545		− if (wake_ev)
1546		− io_cqring_ev_posted(ctx);
	1637	+ nxt = list_first_entry(&req->link_list, struct io_kiocb, link_list);
	1638	+ list_del_init(&req->link_list);
	1639	+ if (!list_empty(&nxt->link_list))
	1640	+ nxt->flags \|= REQ_F_LINK_HEAD;
	1641	+ return nxt;
1547	1642	}
1548	1643
1549	1644	/*
1550	1645	* Called if REQ_F_LINK_HEAD is set, and we fail the head request
1551	1646	*/
1552		−static void io_fail_links(struct io_kiocb *req)
	1647	+static void __io_fail_links(struct io_kiocb *req)
1553	1648	{
1554	1649	struct io_ring_ctx *ctx = req->ctx;
1555		− unsigned long flags;
1556		−
1557		− spin_lock_irqsave(&ctx->completion_lock, flags);
1558	1650
1559	1651	while (!list_empty(&req->link_list)) {
1560	1652	struct io_kiocb *link = list_first_entry(&req->link_list,
@@ -1563,88 +1655,225 @@ static void io_fail_links(struct io_kiocb *req) · 2 unmodified lines
1563	1655	list_del_init(&link->link_list);
1564	1656	trace_io_uring_fail_link(req, link);
1565	1657
1566		− if ((req->flags & REQ_F_LINK_TIMEOUT) &&
1567		− link->opcode == IORING_OP_LINK_TIMEOUT) {
1568		− io_link_cancel_timeout(link);
1569		− } else {
1570		− io_cqring_fill_event(link, -ECANCELED);
1571		− __io_double_put_req(link);
1572		− }
	1658	+ io_cqring_fill_event(link, -ECANCELED);
	1659	+ __io_double_put_req(link);
1573	1660	req->flags &= ~REQ_F_LINK_TIMEOUT;
1574	1661	}
1575	1662
1576	1663	io_commit_cqring(ctx);
1577		− spin_unlock_irqrestore(&ctx->completion_lock, flags);
1578	1664	io_cqring_ev_posted(ctx);
1579	1665	}
1580	1666
1581		−static void io_req_find_next(struct io_kiocb req, struct io_kiocb *nxt)
	1667	+static void io_fail_links(struct io_kiocb *req)
1582	1668	{
1583		− if (likely(!(req->flags & REQ_F_LINK_HEAD)))
1584		− return;
	1669	+ struct io_ring_ctx *ctx = req->ctx;
1585	1670
1586		− /*
1587		− * If LINK is set, we have dependent requests in this chain. If we
1588		− * didn't fail this request, queue the first one up, moving any other
1589		− * dependencies to the next request. In case of failure, fail the rest
1590		− * of the chain.
1591		− */
1592		− if (req->flags & REQ_F_FAIL_LINK) {
1593		− io_fail_links(req);
1594		− } else if ((req->flags & (REQ_F_LINK_TIMEOUT \| REQ_F_COMP_LOCKED)) ==
1595		− REQ_F_LINK_TIMEOUT) {
1596		− struct io_ring_ctx *ctx = req->ctx;
	1671	+ if (!(req->flags & REQ_F_COMP_LOCKED)) {
1597	1672	unsigned long flags;
1598	1673
1599		− /*
1600		− * If this is a timeout link, we could be racing with the
1601		− * timeout timer. Grab the completion lock for this case to
1602		− * protect against that.
1603		− */
1604	1674	spin_lock_irqsave(&ctx->completion_lock, flags);
1605		− io_req_link_next(req, nxt);
	1675	+ __io_fail_links(req);
1606	1676	spin_unlock_irqrestore(&ctx->completion_lock, flags);
1607	1677	} else {
1608		− io_req_link_next(req, nxt);
	1678	+ __io_fail_links(req);
1609	1679	}
1610		−}
1611	1680
1612		−static void io_free_req(struct io_kiocb *req)
	1681	+ io_cqring_ev_posted(ctx);
	1682	+}
	1683	+
	1684	+static struct io_kiocb __io_req_find_next(struct io_kiocb req)
1613	1685	{
1614		− struct io_kiocb *nxt = NULL;
	1686	+ req->flags &= ~REQ_F_LINK_HEAD;
	1687	+ if (req->flags & REQ_F_LINK_TIMEOUT)
	1688	+ io_kill_linked_timeout(req);
1615	1689
1616		− io_req_find_next(req, &nxt);
1617		− __io_free_req(req);
	1690	+ /*
	1691	+ * If LINK is set, we have dependent requests in this chain. If we
	1692	+ * didn't fail this request, queue the first one up, moving any other
	1693	+ * dependencies to the next request. In case of failure, fail the rest
	1694	+ * of the chain.
	1695	+ */
	1696	+ if (likely(!(req->flags & REQ_F_FAIL_LINK)))
	1697	+ return io_req_link_next(req);
	1698	+ io_fail_links(req);
	1699	+ return NULL;
	1700	+}
	1701	+
	1702	+static struct io_kiocb io_req_find_next(struct io_kiocb req)
	1703	+{
	1704	+ if (likely(!(req->flags & REQ_F_LINK_HEAD)))
	1705	+ return NULL;
	1706	+ return __io_req_find_next(req);
	1707	+}
	1708	+
	1709	+static int io_req_task_work_add(struct io_kiocb req, struct callback_head cb)
	1710	+{
	1711	+ struct task_struct *tsk = req->task;
	1712	+ struct io_ring_ctx *ctx = req->ctx;
	1713	+ int ret, notify = TWA_RESUME;
	1714	+
	1715	+ /*
	1716	+ * SQPOLL kernel thread doesn't need notification, just a wakeup.
	1717	+ * If we're not using an eventfd, then TWA_RESUME is always fine,
	1718	+ * as we won't have dependencies between request completions for
	1719	+ * other kernel wait conditions.
	1720	+ */
	1721	+ if (ctx->flags & IORING_SETUP_SQPOLL)
	1722	+ notify = 0;
	1723	+ else if (ctx->cq_ev_fd)
	1724	+ notify = TWA_SIGNAL;
	1725	+
	1726	+ ret = task_work_add(tsk, cb, notify);
	1727	+ if (!ret)
	1728	+ wake_up_process(tsk);
	1729	+ return ret;
	1730	+}
	1731	+
	1732	+static void __io_req_task_cancel(struct io_kiocb *req, int error)
	1733	+{
	1734	+ struct io_ring_ctx *ctx = req->ctx;
	1735	+
	1736	+ spin_lock_irq(&ctx->completion_lock);
	1737	+ io_cqring_fill_event(req, error);
	1738	+ io_commit_cqring(ctx);
	1739	+ spin_unlock_irq(&ctx->completion_lock);
	1740	+
	1741	+ io_cqring_ev_posted(ctx);
	1742	+ req_set_fail_links(req);
	1743	+ io_double_put_req(req);
	1744	+}
	1745	+
	1746	+static void io_req_task_cancel(struct callback_head *cb)
	1747	+{
	1748	+ struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
	1749	+
	1750	+ __io_req_task_cancel(req, -ECANCELED);
	1751	+}
	1752	+
	1753	+static void __io_req_task_submit(struct io_kiocb *req)
	1754	+{
	1755	+ struct io_ring_ctx *ctx = req->ctx;
	1756	+
	1757	+ if (!__io_sq_thread_acquire_mm(ctx)) {
	1758	+ mutex_lock(&ctx->uring_lock);
	1759	+ __io_queue_sqe(req, NULL, NULL);
	1760	+ mutex_unlock(&ctx->uring_lock);
	1761	+ } else {
	1762	+ __io_req_task_cancel(req, -EFAULT);
	1763	+ }
	1764	+}
	1765	+
	1766	+static void io_req_task_submit(struct callback_head *cb)
	1767	+{
	1768	+ struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
	1769	+
	1770	+ __io_req_task_submit(req);
	1771	+}
	1772	+
	1773	+static void io_req_task_queue(struct io_kiocb *req)
	1774	+{
	1775	+ int ret;
	1776	+
	1777	+ init_task_work(&req->task_work, io_req_task_submit);
	1778	+
	1779	+ ret = io_req_task_work_add(req, &req->task_work);
	1780	+ if (unlikely(ret)) {
	1781	+ struct task_struct *tsk;
	1782	+
	1783	+ init_task_work(&req->task_work, io_req_task_cancel);
	1784	+ tsk = io_wq_get_task(req->ctx->io_wq);
	1785	+ task_work_add(tsk, &req->task_work, 0);
	1786	+ wake_up_process(tsk);
	1787	+ }
	1788	+}
	1789	+
	1790	+static void io_queue_next(struct io_kiocb *req)
	1791	+{
	1792	+ struct io_kiocb *nxt = io_req_find_next(req);
1618	1793
1619	1794	if (nxt)
1620		− io_queue_async_work(nxt);
	1795	+ io_req_task_queue(nxt);
1621	1796	}
1622	1797
1623		−static void io_wq_assign_next(struct io_wq_work *workptr, struct io_kiocb nxt)
	1798	+static void io_free_req(struct io_kiocb *req)
1624	1799	{
1625		− struct io_kiocb *link;
1626		− const struct io_op_def *def = &io_op_defs[nxt->opcode];
	1800	+ io_queue_next(req);
	1801	+ __io_free_req(req);
	1802	+}
1627	1803
1628		− if ((nxt->flags & REQ_F_ISREG) && def->hash_reg_file)
1629		− io_wq_hash_work(&nxt->work, file_inode(nxt->file));
	1804	+struct req_batch {
	1805	+ void *reqs[IO_IOPOLL_BATCH];
	1806	+ int to_free;
1630	1807
1631		− *workptr = &nxt->work;
1632		− link = io_prep_linked_timeout(nxt);
1633		− if (link)
1634		− nxt->flags \|= REQ_F_QUEUE_TIMEOUT;
	1808	+ struct task_struct *task;
	1809	+ int task_refs;
	1810	+};
	1811	+
	1812	+static inline void io_init_req_batch(struct req_batch *rb)
	1813	+{
	1814	+ rb->to_free = 0;
	1815	+ rb->task_refs = 0;
	1816	+ rb->task = NULL;
	1817	+}
	1818	+
	1819	+static void __io_req_free_batch_flush(struct io_ring_ctx *ctx,
	1820	+ struct req_batch *rb)
	1821	+{
	1822	+ kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs);
	1823	+ percpu_ref_put_many(&ctx->refs, rb->to_free);
	1824	+ rb->to_free = 0;
	1825	+}
	1826	+
	1827	+static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
	1828	+ struct req_batch *rb)
	1829	+{
	1830	+ if (rb->to_free)
	1831	+ __io_req_free_batch_flush(ctx, rb);
	1832	+ if (rb->task) {
	1833	+ put_task_struct_many(rb->task, rb->task_refs);
	1834	+ rb->task = NULL;
	1835	+ }
	1836	+}
	1837	+
	1838	+static void io_req_free_batch(struct req_batch rb, struct io_kiocb req)
	1839	+{
	1840	+ if (unlikely(io_is_fallback_req(req))) {
	1841	+ io_free_req(req);
	1842	+ return;
	1843	+ }
	1844	+ if (req->flags & REQ_F_LINK_HEAD)
	1845	+ io_queue_next(req);
	1846	+
	1847	+ if (req->flags & REQ_F_TASK_PINNED) {
	1848	+ if (req->task != rb->task) {
	1849	+ if (rb->task)
	1850	+ put_task_struct_many(rb->task, rb->task_refs);
	1851	+ rb->task = req->task;
	1852	+ rb->task_refs = 0;
	1853	+ }
	1854	+ rb->task_refs++;
	1855	+ req->flags &= ~REQ_F_TASK_PINNED;
	1856	+ }
	1857	+
	1858	+ io_dismantle_req(req);
	1859	+ rb->reqs[rb->to_free++] = req;
	1860	+ if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs)))
	1861	+ __io_req_free_batch_flush(req->ctx, rb);
1635	1862	}
1636	1863
1637	1864	/*
1638	1865	* Drop reference to request, return next in chain (if there is one) if this
1639	1866	* was the last reference to this request.
1640	1867	*/
1641		−__attribute__((nonnull))
1642		−static void io_put_req_find_next(struct io_kiocb req, struct io_kiocb *nxtptr)
	1868	+static struct io_kiocb io_put_req_find_next(struct io_kiocb req)
1643	1869	{
	1870	+ struct io_kiocb *nxt = NULL;
	1871	+
1644	1872	if (refcount_dec_and_test(&req->refs)) {
1645		− io_req_find_next(req, nxtptr);
	1873	+ nxt = io_req_find_next(req);
1646	1874	__io_free_req(req);
1647	1875	}
	1876	+ return nxt;
1648	1877	}
1649	1878
1650	1879	static void io_put_req(struct io_kiocb *req)
@@ -1653,24 +1882,20 @@ static void io_put_req(struct io_kiocb *req) · 2 unmodified lines
1653	1882	io_free_req(req);
1654	1883	}
1655	1884
1656		−static void io_steal_work(struct io_kiocb *req,
1657		− struct io_wq_work **workptr)
	1885	+static struct io_wq_work io_steal_work(struct io_kiocb req)
1658	1886	{
	1887	+ struct io_kiocb *nxt;
	1888	+
1659	1889	/*
1660		− * It's in an io-wq worker, so there always should be at least
1661		− * one reference, which will be dropped in io_put_work() just
1662		− * after the current handler returns.
1663		− *
1664		− * It also means, that if the counter dropped to 1, then there is
1665		− * no asynchronous users left, so it's safe to steal the next work.
	1890	+ * A ref is owned by io-wq in which context we're. So, if that's the
	1891	+ * last one, it's safe to steal next work. False negatives are Ok,
	1892	+ * it just will be re-punted async in io_put_work()
1666	1893	*/
1667		− if (refcount_read(&req->refs) == 1) {
1668		− struct io_kiocb *nxt = NULL;
	1894	+ if (refcount_read(&req->refs) != 1)
	1895	+ return NULL;
1669	1896
1670		− io_req_find_next(req, &nxt);
1671		− if (nxt)
1672		− io_wq_assign_next(workptr, nxt);
1673		− }
	1897	+ nxt = io_req_find_next(req);
	1898	+ return nxt ? &nxt->work : NULL;
1674	1899	}
1675	1900
1676	1901	/*
@@ -1720,31 +1945,34 @@ static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) · 43 unmodified lines
1720	1945	return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
1721	1946	}
1722	1947
1723		−static inline bool io_req_multi_free(struct req_batch rb, struct io_kiocb req)
	1948	+static unsigned int io_put_kbuf(struct io_kiocb req, struct io_buffer kbuf)
1724	1949	{
1725		− if ((req->flags & REQ_F_LINK_HEAD) \|\| io_is_fallback_req(req))
1726		− return false;
	1950	+ unsigned int cflags;
1727	1951
1728		− if (req->file \|\| req->io)
1729		− rb->need_iter++;
1730		−
1731		− rb->reqs[rb->to_free++] = req;
1732		− if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs)))
1733		− io_free_req_many(req->ctx, rb);
1734		− return true;
	1952	+ cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
	1953	+ cflags \|= IORING_CQE_F_BUFFER;
	1954	+ req->flags &= ~REQ_F_BUFFER_SELECTED;
	1955	+ kfree(kbuf);
	1956	+ return cflags;
1735	1957	}
1736	1958
1737		−static int io_put_kbuf(struct io_kiocb *req)
	1959	+static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req)
1738	1960	{
1739	1961	struct io_buffer *kbuf;
1740		− int cflags;
1741	1962
1742	1963	kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
1743		− cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
1744		− cflags \|= IORING_CQE_F_BUFFER;
1745		− req->rw.addr = 0;
1746		− kfree(kbuf);
1747		− return cflags;
	1964	+ return io_put_kbuf(req, kbuf);
	1965	+}
	1966	+
	1967	+static inline bool io_run_task_work(void)
	1968	+{
	1969	+ if (current->task_works) {
	1970	+ __set_current_state(TASK_RUNNING);
	1971	+ task_work_run();
	1972	+ return true;
	1973	+ }
	1974	+
	1975	+ return false;
1748	1976	}
1749	1977
1750	1978	static void io_iopoll_queue(struct list_head *again)
@@ -1752,18 +1980,9 @@ static void io_iopoll_queue(struct list_head *again) · 1 unmodified line
1752	1980	struct io_kiocb *req;
1753	1981
1754	1982	do {
1755		− req = list_first_entry(again, struct io_kiocb, list);
1756		− list_del(&req->list);
1757		−
1758		− /* shouldn't happen unless io_uring is dying, cancel reqs */
1759		− if (unlikely(!current->mm)) {
1760		− io_complete_rw_common(&req->rw.kiocb, -EAGAIN);
1761		− io_put_req(req);
1762		− continue;
1763		− }
1764		−
1765		− refcount_inc(&req->refs);
1766		− io_queue_async_work(req);
	1983	+ req = list_first_entry(again, struct io_kiocb, inflight_entry);
	1984	+ list_del(&req->inflight_entry);
	1985	+ __io_complete_rw(req, -EAGAIN, 0, NULL);
1767	1986	} while (!list_empty(again));
1768	1987	}
1769	1988
@@ -1780,33 +1999,32 @@ static void io_iopoll_complete(struct io_ring_ctx ctx, unsigned int nr_events, · 10 unmodified lines
1780	1999	/* order with ->result store in io_complete_rw_iopoll() */
1781	2000	smp_rmb();
1782	2001
1783		− rb.to_free = rb.need_iter = 0;
	2002	+ io_init_req_batch(&rb);
1784	2003	while (!list_empty(done)) {
1785	2004	int cflags = 0;
1786	2005
1787		− req = list_first_entry(done, struct io_kiocb, list);
	2006	+ req = list_first_entry(done, struct io_kiocb, inflight_entry);
1788	2007	if (READ_ONCE(req->result) == -EAGAIN) {
1789	2008	req->iopoll_completed = 0;
1790		− list_move_tail(&req->list, &again);
	2009	+ list_move_tail(&req->inflight_entry, &again);
1791	2010	continue;
1792	2011	}
1793		− list_del(&req->list);
	2012	+ list_del(&req->inflight_entry);
1794	2013
1795	2014	if (req->flags & REQ_F_BUFFER_SELECTED)
1796		− cflags = io_put_kbuf(req);
	2015	+ cflags = io_put_rw_kbuf(req);
1797	2016
1798	2017	__io_cqring_fill_event(req, req->result, cflags);
1799	2018	(*nr_events)++;
1800	2019
1801		− if (refcount_dec_and_test(&req->refs) &&
1802		− !io_req_multi_free(&rb, req))
1803		− io_free_req(req);
	2020	+ if (refcount_dec_and_test(&req->refs))
	2021	+ io_req_free_batch(&rb, req);
1804	2022	}
1805	2023
1806	2024	io_commit_cqring(ctx);
1807	2025	if (ctx->flags & IORING_SETUP_SQPOLL)
1808	2026	io_cqring_ev_posted(ctx);
1809		− io_free_req_many(ctx, &rb);
	2027	+ io_req_free_batch_finish(ctx, &rb);
1810	2028
1811	2029	if (!list_empty(&again))
1812	2030	io_iopoll_queue(&again);
@@ -1827,7 +2045,7 @@ static int io_do_iopoll(struct io_ring_ctx ctx, unsigned int nr_events, · 14 unmodified lines
1827	2045	spin = !ctx->poll_multi_file && *nr_events < min;
1828	2046
1829	2047	ret = 0;
1830		− list_for_each_entry_safe(req, tmp, &ctx->poll_list, list) {
	2048	+ list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) {
1831	2049	struct kiocb *kiocb = &req->rw.kiocb;
1832	2050
1833	2051	/*
@@ -1836,7 +2054,7 @@ static int io_do_iopoll(struct io_ring_ctx ctx, unsigned int nr_events, · 2 unmodified lines
1836	2054	* and complete those lists first, if we have entries there.
1837	2055	*/
1838	2056	if (READ_ONCE(req->iopoll_completed)) {
1839		− list_move_tail(&req->list, &done);
	2057	+ list_move_tail(&req->inflight_entry, &done);
1840	2058	continue;
1841	2059	}
1842	2060	if (!list_empty(&done))
@@ -1846,6 +2064,10 @@ static int io_do_iopoll(struct io_ring_ctx ctx, unsigned int nr_events, · 3 unmodified lines
1846	2064	if (ret < 0)
1847	2065	break;
1848	2066
	2067	+ /* iopoll may have completed current req */
	2068	+ if (READ_ONCE(req->iopoll_completed))
	2069	+ list_move_tail(&req->inflight_entry, &done);
	2070	+
1849	2071	if (ret && spin)
1850	2072	spin = false;
1851	2073	ret = 0;
@@ -1865,13 +2087,13 @@ static int io_do_iopoll(struct io_ring_ctx ctx, unsigned int nr_events, · 13 unmodified lines
1865	2087	static int io_iopoll_getevents(struct io_ring_ctx ctx, unsigned int nr_events,
1866	2088	long min)
1867	2089	{
1868		− while (!list_empty(&ctx->poll_list) && !need_resched()) {
	2090	+ while (!list_empty(&ctx->iopoll_list) && !need_resched()) {
1869	2091	int ret;
1870	2092
1871	2093	ret = io_do_iopoll(ctx, nr_events, min);
1872	2094	if (ret < 0)
1873	2095	return ret;
1874		− if (!min \|\| *nr_events >= min)
	2096	+ if (*nr_events >= min)
1875	2097	return 0;
1876	2098	}
1877	2099
@@ -1882,29 +2104,37 @@ static int io_iopoll_getevents(struct io_ring_ctx ctx, unsigned int nr_events, · 4 unmodified lines
1882	2104	* We can't just wait for polled events to come to us, we have to actively
1883	2105	* find and complete them.
1884	2106	*/
1885		−static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
	2107	+static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
1886	2108	{
1887	2109	if (!(ctx->flags & IORING_SETUP_IOPOLL))
1888	2110	return;
1889	2111
1890	2112	mutex_lock(&ctx->uring_lock);
1891		− while (!list_empty(&ctx->poll_list)) {
	2113	+ while (!list_empty(&ctx->iopoll_list)) {
1892	2114	unsigned int nr_events = 0;
1893	2115
1894		− io_iopoll_getevents(ctx, &nr_events, 1);
	2116	+ io_do_iopoll(ctx, &nr_events, 0);
1895	2117
	2118	+ /* let it sleep and repeat later if can't complete a request */
	2119	+ if (nr_events == 0)
	2120	+ break;
1896	2121	/*
1897	2122	* Ensure we allow local-to-the-cpu processing to take place,
1898	2123	* in this case we need to ensure that we reap all events.
	2124	+ * Also let task_work, etc. to progress by releasing the mutex
1899	2125	*/
1900		− cond_resched();
	2126	+ if (need_resched()) {
	2127	+ mutex_unlock(&ctx->uring_lock);
	2128	+ cond_resched();
	2129	+ mutex_lock(&ctx->uring_lock);
	2130	+ }
1901	2131	}
1902	2132	mutex_unlock(&ctx->uring_lock);
1903	2133	}
1904	2134
1905		−static int io_iopoll_check(struct io_ring_ctx ctx, unsigned nr_events,
1906		− long min)
	2135	+static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
1907	2136	{
	2137	+ unsigned int nr_events = 0;
1908	2138	int iters = 0, ret = 0;
1909	2139
1910	2140	/*
@@ -1914,8 +2144,6 @@ static int io_iopoll_check(struct io_ring_ctx ctx, unsigned nr_events, · 3 unmodified lines
1914	2144	*/
1915	2145	mutex_lock(&ctx->uring_lock);
1916	2146	do {
1917		− int tmin = 0;
1918		−
1919	2147	/*
1920	2148	* Don't enter poll loop if we already have events pending.
1921	2149	* If we do, we can potentially be spinning for commands that
@@ -1936,17 +2164,15 @@ static int io_iopoll_check(struct io_ring_ctx ctx, unsigned nr_events, · 14 unmodified lines
1936	2164	*/
1937	2165	if (!(++iters & 7)) {
1938	2166	mutex_unlock(&ctx->uring_lock);
	2167	+ io_run_task_work();
1939	2168	mutex_lock(&ctx->uring_lock);
1940	2169	}
1941	2170
1942		− if (*nr_events < min)
1943		− tmin = min - *nr_events;
1944		−
1945		− ret = io_iopoll_getevents(ctx, nr_events, tmin);
	2171	+ ret = io_iopoll_getevents(ctx, &nr_events, min);
1946	2172	if (ret <= 0)
1947	2173	break;
1948	2174	ret = 0;
1949		− } while (min && !*nr_events && !need_resched());
	2175	+ } while (min && !nr_events && !need_resched());
1950	2176
1951	2177	mutex_unlock(&ctx->uring_lock);
1952	2178	return ret;
@@ -1966,13 +2192,8 @@ static void kiocb_end_write(struct io_kiocb *req) · 13 unmodified lines
1966	2192	file_end_write(req->file);
1967	2193	}
1968	2194
1969		−static inline void req_set_fail_links(struct io_kiocb *req)
1970		−{
1971		− if ((req->flags & (REQ_F_LINK \| REQ_F_HARDLINK)) == REQ_F_LINK)
1972		− req->flags \|= REQ_F_FAIL_LINK;
1973		−}
1974		−
1975		−static void io_complete_rw_common(struct kiocb *kiocb, long res)
	2195	+static void io_complete_rw_common(struct kiocb *kiocb, long res,
	2196	+ struct io_comp_state *cs)
1976	2197	{
1977	2198	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1978	2199	int cflags = 0;
@@ -1983,16 +2204,96 @@ static void io_complete_rw_common(struct kiocb *kiocb, long res) · 4 unmodified lines
1983	2204	if (res != req->result)
1984	2205	req_set_fail_links(req);
1985	2206	if (req->flags & REQ_F_BUFFER_SELECTED)
1986		− cflags = io_put_kbuf(req);
1987		− __io_cqring_add_event(req, res, cflags);
	2207	+ cflags = io_put_rw_kbuf(req);
	2208	+ __io_req_complete(req, res, cflags, cs);
	2209	+}
	2210	+
	2211	+#ifdef CONFIG_BLOCK
	2212	+static bool io_resubmit_prep(struct io_kiocb *req, int error)
	2213	+{
	2214	+ struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
	2215	+ ssize_t ret = -ECANCELED;
	2216	+ struct iov_iter iter;
	2217	+ int rw;
	2218	+
	2219	+ if (error) {
	2220	+ ret = error;
	2221	+ goto end_req;
	2222	+ }
	2223	+
	2224	+ switch (req->opcode) {
	2225	+ case IORING_OP_READV:
	2226	+ case IORING_OP_READ_FIXED:
	2227	+ case IORING_OP_READ:
	2228	+ rw = READ;
	2229	+ break;
	2230	+ case IORING_OP_WRITEV:
	2231	+ case IORING_OP_WRITE_FIXED:
	2232	+ case IORING_OP_WRITE:
	2233	+ rw = WRITE;
	2234	+ break;
	2235	+ default:
	2236	+ printk_once(KERN_WARNING "io_uring: bad opcode in resubmit %d\n",
	2237	+ req->opcode);
	2238	+ goto end_req;
	2239	+ }
	2240	+
	2241	+ ret = io_import_iovec(rw, req, &iovec, &iter, false);
	2242	+ if (ret < 0)
	2243	+ goto end_req;
	2244	+ ret = io_setup_async_rw(req, ret, iovec, inline_vecs, &iter);
	2245	+ if (!ret)
	2246	+ return true;
	2247	+ kfree(iovec);
	2248	+end_req:
	2249	+ req_set_fail_links(req);
	2250	+ io_req_complete(req, ret);
	2251	+ return false;
	2252	+}
	2253	+
	2254	+static void io_rw_resubmit(struct callback_head *cb)
	2255	+{
	2256	+ struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
	2257	+ struct io_ring_ctx *ctx = req->ctx;
	2258	+ int err;
	2259	+
	2260	+ err = io_sq_thread_acquire_mm(ctx, req);
	2261	+
	2262	+ if (io_resubmit_prep(req, err)) {
	2263	+ refcount_inc(&req->refs);
	2264	+ io_queue_async_work(req);
	2265	+ }
	2266	+}
	2267	+#endif
	2268	+
	2269	+static bool io_rw_reissue(struct io_kiocb *req, long res)
	2270	+{
	2271	+#ifdef CONFIG_BLOCK
	2272	+ int ret;
	2273	+
	2274	+ if ((res != -EAGAIN && res != -EOPNOTSUPP) \|\| io_wq_current_is_worker())
	2275	+ return false;
	2276	+
	2277	+ init_task_work(&req->task_work, io_rw_resubmit);
	2278	+ ret = io_req_task_work_add(req, &req->task_work);
	2279	+ if (!ret)
	2280	+ return true;
	2281	+#endif
	2282	+ return false;
	2283	+}
	2284	+
	2285	+static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
	2286	+ struct io_comp_state *cs)
	2287	+{
	2288	+ if (!io_rw_reissue(req, res))
	2289	+ io_complete_rw_common(&req->rw.kiocb, res, cs);
1988	2290	}
1989	2291
1990	2292	static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
1991	2293	{
1992	2294	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1993	2295
1994		− io_complete_rw_common(kiocb, res);
1995		− io_put_req(req);
	2296	+ __io_complete_rw(req, res, res2, NULL);
1996	2297	}
1997	2298
1998	2299	static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
@@ -2026,13 +2327,13 @@ static void io_iopoll_req_issued(struct io_kiocb *req) · 27 unmodified lines
2026	2327	* how we do polling eventually, not spinning if we're on potentially
2027	2328	* different devices.
2028	2329	*/
2029		− if (list_empty(&ctx->poll_list)) {
	2330	+ if (list_empty(&ctx->iopoll_list)) {
2030	2331	ctx->poll_multi_file = false;
2031	2332	} else if (!ctx->poll_multi_file) {
2032	2333	struct io_kiocb *list_req;
2033	2334
2034		− list_req = list_first_entry(&ctx->poll_list, struct io_kiocb,
2035		− list);
	2335	+ list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb,
	2336	+ inflight_entry);
2036	2337	if (list_req->file != req->file)
2037	2338	ctx->poll_multi_file = true;
2038	2339	}
@@ -2042,9 +2343,9 @@ static void io_iopoll_req_issued(struct io_kiocb *req) · 3 unmodified lines
2042	2343	* it to the front so we find it first.
2043	2344	*/
2044	2345	if (READ_ONCE(req->iopoll_completed))
2045		− list_add(&req->list, &ctx->poll_list);
	2346	+ list_add(&req->inflight_entry, &ctx->iopoll_list);
2046	2347	else
2047		− list_add_tail(&req->list, &ctx->poll_list);
	2348	+ list_add_tail(&req->inflight_entry, &ctx->iopoll_list);
2048	2349
2049	2350	if ((ctx->flags & IORING_SETUP_SQPOLL) &&
2050	2351	wq_has_sleeper(&ctx->sqo_wait))
@@ -2053,10 +2354,8 @@ static void io_iopoll_req_issued(struct io_kiocb *req) · 2 unmodified lines
2053	2354
2054	2355	static void __io_state_file_put(struct io_submit_state *state)
2055	2356	{
2056		− int diff = state->has_refs - state->used_refs;
2057		−
2058		− if (diff)
2059		− fput_many(state->file, diff);
	2357	+ if (state->has_refs)
	2358	+ fput_many(state->file, state->has_refs);
2060	2359	state->file = NULL;
2061	2360	}
2062	2361
@@ -2078,7 +2377,7 @@ static struct file __io_file_get(struct io_submit_state state, int fd) · 15 unmodified lines
2078	2377
2079	2378	if (state->file) {
2080	2379	if (state->fd == fd) {
2081		− state->used_refs++;
	2380	+ state->has_refs--;
2082	2381	state->ios_left--;
2083	2382	return state->file;
2084	2383	}
@@ -2089,12 +2388,20 @@ static struct file __io_file_get(struct io_submit_state state, int fd) · 4 unmodified lines
2089	2388	return NULL;
2090	2389
2091	2390	state->fd = fd;
2092		− state->has_refs = state->ios_left;
2093		− state->used_refs = 1;
2094	2391	state->ios_left--;
	2392	+ state->has_refs = state->ios_left;
2095	2393	return state->file;
2096	2394	}
2097	2395
	2396	+static bool io_bdev_nowait(struct block_device *bdev)
	2397	+{
	2398	+#ifdef CONFIG_BLOCK
	2399	+ return !bdev \|\| queue_is_mq(bdev_get_queue(bdev));
	2400	+#else
	2401	+ return true;
	2402	+#endif
	2403	+}
	2404	+
2098	2405	/*
2099	2406	* If we tracked the file through the SCM inflight mechanism, we could support
2100	2407	* any file. For now, just ensure that anything potentially problematic is done
@@ -2104,10 +2411,19 @@ static bool io_file_supports_async(struct file *file, int rw) · 3 unmodified lines
2104	2411	{
2105	2412	umode_t mode = file_inode(file)->i_mode;
2106	2413
2107		− if (S_ISBLK(mode) \|\| S_ISCHR(mode) \|\| S_ISSOCK(mode))
2108		− return true;
2109		− if (S_ISREG(mode) && file->f_op != &io_uring_fops)
	2414	+ if (S_ISBLK(mode)) {
	2415	+ if (io_bdev_nowait(file->f_inode->i_bdev))
	2416	+ return true;
	2417	+ return false;
	2418	+ }
	2419	+ if (S_ISCHR(mode) \|\| S_ISSOCK(mode))
2110	2420	return true;
	2421	+ if (S_ISREG(mode)) {
	2422	+ if (io_bdev_nowait(file->f_inode->i_sb->s_bdev) &&
	2423	+ file->f_op != &io_uring_fops)
	2424	+ return true;
	2425	+ return false;
	2426	+ }
2111	2427
2112	2428	/* any ->read/write should understand O_NONBLOCK */
2113	2429	if (file->f_flags & O_NONBLOCK)
@@ -2158,6 +2474,9 @@ static int io_prep_rw(struct io_kiocb req, const struct io_uring_sqe sqe, · 44 unmodified lines
2158	2474	if (kiocb->ki_flags & IOCB_NOWAIT)
2159	2475	req->flags \|= REQ_F_NOWAIT;
2160	2476
	2477	+ if (kiocb->ki_flags & IOCB_DIRECT)
	2478	+ io_get_req_task(req);
	2479	+
2161	2480	if (force_nonblock)
2162	2481	kiocb->ki_flags \|= IOCB_NOWAIT;
2163	2482
@@ -2168,8 +2487,8 @@ static int io_prep_rw(struct io_kiocb req, const struct io_uring_sqe sqe, · 4 unmodified lines
2168	2487
2169	2488	kiocb->ki_flags \|= IOCB_HIPRI;
2170	2489	kiocb->ki_complete = io_complete_rw_iopoll;
2171		− req->result = 0;
2172	2490	req->iopoll_completed = 0;
	2491	+ io_get_req_task(req);
2173	2492	} else {
2174	2493	if (kiocb->ki_flags & IOCB_HIPRI)
2175	2494	return -EINVAL;
@@ -2203,14 +2522,15 @@ static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret) · 27 unmodified lines
2203	2522	}
2204	2523	}
2205	2524
2206		−static void kiocb_done(struct kiocb *kiocb, ssize_t ret)
	2525	+static void kiocb_done(struct kiocb *kiocb, ssize_t ret,
	2526	+ struct io_comp_state *cs)
2207	2527	{
2208	2528	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
2209	2529
2210	2530	if (req->flags & REQ_F_CUR_POS)
2211	2531	req->file->f_pos = kiocb->ki_pos;
2212	2532	if (ret >= 0 && kiocb->ki_complete == io_complete_rw)
2213		− io_complete_rw(kiocb, ret, 0);
	2533	+ __io_complete_rw(req, ret, 0, cs);
2214	2534	else
2215	2535	io_rw_done(kiocb, ret);
2216	2536	}
@@ -2466,10 +2786,8 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req, · 249 unmodified lines
2466	2786	if (req->io) {
2467	2787	struct io_async_rw *iorw = &req->io->rw;
2468	2788
2469		− *iovec = iorw->iov;
2470		− iov_iter_init(iter, rw, *iovec, iorw->nr_segs, iorw->size);
2471		− if (iorw->iov == iorw->fast_iov)
2472		− *iovec = NULL;
	2789	+ iov_iter_init(iter, rw, iorw->iov, iorw->nr_segs, iorw->size);
	2790	+ *iovec = NULL;
2473	2791	return iorw->size;
2474	2792	}
2475	2793
@@ -2554,15 +2872,17 @@ static void io_req_map_rw(struct io_kiocb *req, ssize_t io_size, · 78 unmodified lines
2554	2872	struct iovec iovec, struct iovec fast_iov,
2555	2873	struct iov_iter *iter)
2556	2874	{
2557		− req->io->rw.nr_segs = iter->nr_segs;
2558		− req->io->rw.size = io_size;
2559		− req->io->rw.iov = iovec;
2560		− if (!req->io->rw.iov) {
2561		− req->io->rw.iov = req->io->rw.fast_iov;
2562		− if (req->io->rw.iov != fast_iov)
2563		− memcpy(req->io->rw.iov, fast_iov,
	2875	+ struct io_async_rw *rw = &req->io->rw;
	2876	+
	2877	+ rw->nr_segs = iter->nr_segs;
	2878	+ rw->size = io_size;
	2879	+ if (!iovec) {
	2880	+ rw->iov = rw->fast_iov;
	2881	+ if (rw->iov != fast_iov)
	2882	+ memcpy(rw->iov, fast_iov,
2564	2883	sizeof(struct iovec) * iter->nr_segs);
2565	2884	} else {
	2885	+ rw->iov = iovec;
2566	2886	req->flags \|= REQ_F_NEED_CLEANUP;
2567	2887	}
2568	2888	}
@@ -2593,95 +2913,205 @@ static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size, · 24 unmodified lines
2593	2913
2594	2914	io_req_map_rw(req, io_size, iovec, fast_iov, iter);
2595	2915	}
2596		− return 0;
	2916	+ return 0;
	2917	+}
	2918	+
	2919	+static inline int io_rw_prep_async(struct io_kiocb *req, int rw,
	2920	+ bool force_nonblock)
	2921	+{
	2922	+ struct io_async_ctx *io = req->io;
	2923	+ struct iov_iter iter;
	2924	+ ssize_t ret;
	2925	+
	2926	+ io->rw.iov = io->rw.fast_iov;
	2927	+ req->io = NULL;
	2928	+ ret = io_import_iovec(rw, req, &io->rw.iov, &iter, !force_nonblock);
	2929	+ req->io = io;
	2930	+ if (unlikely(ret < 0))
	2931	+ return ret;
	2932	+
	2933	+ io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
	2934	+ return 0;
	2935	+}
	2936	+
	2937	+static int io_read_prep(struct io_kiocb req, const struct io_uring_sqe sqe,
	2938	+ bool force_nonblock)
	2939	+{
	2940	+ ssize_t ret;
	2941	+
	2942	+ ret = io_prep_rw(req, sqe, force_nonblock);
	2943	+ if (ret)
	2944	+ return ret;
	2945	+
	2946	+ if (unlikely(!(req->file->f_mode & FMODE_READ)))
	2947	+ return -EBADF;
	2948	+
	2949	+ /* either don't need iovec imported or already have it */
	2950	+ if (!req->io \|\| req->flags & REQ_F_NEED_CLEANUP)
	2951	+ return 0;
	2952	+ return io_rw_prep_async(req, READ, force_nonblock);
	2953	+}
	2954	+
	2955	+static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
	2956	+ int sync, void *arg)
	2957	+{
	2958	+ struct wait_page_queue *wpq;
	2959	+ struct io_kiocb *req = wait->private;
	2960	+ struct wait_page_key *key = arg;
	2961	+ int ret;
	2962	+
	2963	+ wpq = container_of(wait, struct wait_page_queue, wait);
	2964	+
	2965	+ if (!wake_page_match(wpq, key))
	2966	+ return 0;
	2967	+
	2968	+ /* Stop waking things up if the page is locked again */
	2969	+ if (test_bit(key->bit_nr, &key->page->flags))
	2970	+ return -1;
	2971	+
	2972	+ list_del_init(&wait->entry);
	2973	+
	2974	+ init_task_work(&req->task_work, io_req_task_submit);
	2975	+ /* submit ref gets dropped, acquire a new one */
	2976	+ refcount_inc(&req->refs);
	2977	+ ret = io_req_task_work_add(req, &req->task_work);
	2978	+ if (unlikely(ret)) {
	2979	+ struct task_struct *tsk;
	2980	+
	2981	+ /* queue just for cancelation */
	2982	+ init_task_work(&req->task_work, io_req_task_cancel);
	2983	+ tsk = io_wq_get_task(req->ctx->io_wq);
	2984	+ task_work_add(tsk, &req->task_work, 0);
	2985	+ wake_up_process(tsk);
	2986	+ }
	2987	+ return 1;
	2988	+}
	2989	+
	2990	+static inline int kiocb_wait_page_queue_init(struct kiocb *kiocb,
	2991	+ struct wait_page_queue *wait,
	2992	+ wait_queue_func_t func,
	2993	+ void *data)
	2994	+{
	2995	+ /* Can't support async wakeup with polled IO */
	2996	+ if (kiocb->ki_flags & IOCB_HIPRI)
	2997	+ return -EINVAL;
	2998	+ if (kiocb->ki_filp->f_mode & FMODE_BUF_RASYNC) {
	2999	+ wait->wait.func = func;
	3000	+ wait->wait.private = data;
	3001	+ wait->wait.flags = 0;
	3002	+ INIT_LIST_HEAD(&wait->wait.entry);
	3003	+ kiocb->ki_flags \|= IOCB_WAITQ;
	3004	+ kiocb->ki_waitq = wait;
	3005	+ return 0;
	3006	+ }
	3007	+
	3008	+ return -EOPNOTSUPP;
2597	3009	}
2598	3010
2599		−static int io_read_prep(struct io_kiocb req, const struct io_uring_sqe sqe,
2600		− bool force_nonblock)
	3011	+
	3012	+static bool io_rw_should_retry(struct io_kiocb *req)
2601	3013	{
2602		− struct io_async_ctx *io;
2603		− struct iov_iter iter;
2604		− ssize_t ret;
	3014	+ struct kiocb *kiocb = &req->rw.kiocb;
	3015	+ int ret;
2605	3016
2606		− ret = io_prep_rw(req, sqe, force_nonblock);
2607		− if (ret)
2608		− return ret;
	3017	+ /* never retry for NOWAIT, we just complete with -EAGAIN */
	3018	+ if (req->flags & REQ_F_NOWAIT)
	3019	+ return false;
2609	3020
2610		− if (unlikely(!(req->file->f_mode & FMODE_READ)))
2611		− return -EBADF;
	3021	+ /* already tried, or we're doing O_DIRECT */
	3022	+ if (kiocb->ki_flags & (IOCB_DIRECT \| IOCB_WAITQ))
	3023	+ return false;
	3024	+ /*
	3025	+ * just use poll if we can, and don't attempt if the fs doesn't
	3026	+ * support callback based unlocks
	3027	+ */
	3028	+ if (file_can_poll(req->file) \|\| !(req->file->f_mode & FMODE_BUF_RASYNC))
	3029	+ return false;
2612	3030
2613		− /* either don't need iovec imported or already have it */
2614		− if (!req->io \|\| req->flags & REQ_F_NEED_CLEANUP)
2615		− return 0;
	3031	+ /*
	3032	+ * If request type doesn't require req->io to defer in general,
	3033	+ * we need to allocate it here
	3034	+ */
	3035	+ if (!req->io && __io_alloc_async_ctx(req))
	3036	+ return false;
2616	3037
2617		− io = req->io;
2618		− io->rw.iov = io->rw.fast_iov;
2619		− req->io = NULL;
2620		− ret = io_import_iovec(READ, req, &io->rw.iov, &iter, !force_nonblock);
2621		− req->io = io;
2622		− if (ret < 0)
2623		− return ret;
	3038	+ ret = kiocb_wait_page_queue_init(kiocb, &req->io->rw.wpq,
	3039	+ io_async_buf_func, req);
	3040	+ if (!ret) {
	3041	+ io_get_req_task(req);
	3042	+ return true;
	3043	+ }
2624	3044
2625		− io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
2626		− return 0;
	3045	+ return false;
	3046	+}
	3047	+
	3048	+static int io_iter_do_read(struct io_kiocb req, struct iov_iter iter)
	3049	+{
	3050	+ if (req->file->f_op->read_iter)
	3051	+ return call_read_iter(req->file, &req->rw.kiocb, iter);
	3052	+ return loop_rw_iter(READ, req->file, &req->rw.kiocb, iter);
2627	3053	}
2628	3054
2629		−static int io_read(struct io_kiocb *req, bool force_nonblock)
	3055	+static int io_read(struct io_kiocb *req, bool force_nonblock,
	3056	+ struct io_comp_state *cs)
2630	3057	{
2631	3058	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
2632	3059	struct kiocb *kiocb = &req->rw.kiocb;
2633	3060	struct iov_iter iter;
2634	3061	size_t iov_count;
2635		− ssize_t io_size, ret;
	3062	+ ssize_t io_size, ret, ret2;
	3063	+ unsigned long nr_segs;
2636	3064
2637	3065	ret = io_import_iovec(READ, req, &iovec, &iter, !force_nonblock);
2638	3066	if (ret < 0)
2639	3067	return ret;
	3068	+ io_size = ret;
	3069	+ req->result = io_size;
2640	3070
2641	3071	/* Ensure we clear previously set non-block flag */
2642	3072	if (!force_nonblock)
2643	3073	kiocb->ki_flags &= ~IOCB_NOWAIT;
2644	3074
2645		− req->result = 0;
2646		− io_size = ret;
2647		− if (req->flags & REQ_F_LINK_HEAD)
2648		− req->result = io_size;
2649		−
2650		− /*
2651		− * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
2652		− * we know to async punt it even if it was opened O_NONBLOCK
2653		− */
	3075	+ /* If the file doesn't support async, just async punt */
2654	3076	if (force_nonblock && !io_file_supports_async(req->file, READ))
2655	3077	goto copy_iov;
2656	3078
2657	3079	iov_count = iov_iter_count(&iter);
	3080	+ nr_segs = iter.nr_segs;
2658	3081	ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count);
2659		− if (!ret) {
2660		− ssize_t ret2;
	3082	+ if (unlikely(ret))
	3083	+ goto out_free;
2661	3084
2662		− if (req->file->f_op->read_iter)
2663		− ret2 = call_read_iter(req->file, kiocb, &iter);
2664		− else
2665		− ret2 = loop_rw_iter(READ, req->file, kiocb, &iter);
	3085	+ ret2 = io_iter_do_read(req, &iter);
2666	3086
2667		− /* Catch -EAGAIN return for forced non-blocking submission */
2668		− if (!force_nonblock \|\| ret2 != -EAGAIN) {
2669		− kiocb_done(kiocb, ret2);
2670		− } else {
	3087	+ /* Catch -EAGAIN return for forced non-blocking submission */
	3088	+ if (!force_nonblock \|\| (ret2 != -EAGAIN && ret2 != -EIO)) {
	3089	+ kiocb_done(kiocb, ret2, cs);
	3090	+ } else {
	3091	+ iter.count = iov_count;
	3092	+ iter.nr_segs = nr_segs;
2671	3093	copy_iov:
2672		− ret = io_setup_async_rw(req, io_size, iovec,
2673		− inline_vecs, &iter);
2674		− if (ret)
	3094	+ ret = io_setup_async_rw(req, io_size, iovec, inline_vecs,
	3095	+ &iter);
	3096	+ if (ret)
	3097	+ goto out_free;
	3098	+ /* it's copied and will be cleaned with ->io */
	3099	+ iovec = NULL;
	3100	+ /* if we can retry, do so with the callbacks armed */
	3101	+ if (io_rw_should_retry(req)) {
	3102	+ ret2 = io_iter_do_read(req, &iter);
	3103	+ if (ret2 == -EIOCBQUEUED) {
2675	3104	goto out_free;
2676		− /* any defer here is final, must blocking retry */
2677		− if (!(req->flags & REQ_F_NOWAIT) &&
2678		− !file_can_poll(req->file))
2679		− req->flags \|= REQ_F_MUST_PUNT;
2680		− return -EAGAIN;
	3105	+ } else if (ret2 != -EAGAIN) {
	3106	+ kiocb_done(kiocb, ret2, cs);
	3107	+ goto out_free;
	3108	+ }
2681	3109	}
	3110	+ kiocb->ki_flags &= ~IOCB_WAITQ;
	3111	+ return -EAGAIN;
2682	3112	}
2683	3113	out_free:
2684		− if (!(req->flags & REQ_F_NEED_CLEANUP))
	3114	+ if (iovec)
2685	3115	kfree(iovec);
2686	3116	return ret;
2687	3117	}
@@ -2689,8 +3119,6 @@ out_free: · 1 unmodified line
2689	3119	static int io_write_prep(struct io_kiocb req, const struct io_uring_sqe sqe,
2690	3120	bool force_nonblock)
2691	3121	{
2692		− struct io_async_ctx *io;
2693		− struct iov_iter iter;
2694	3122	ssize_t ret;
2695	3123
2696	3124	ret = io_prep_rw(req, sqe, force_nonblock);
@@ -2700,49 +3128,33 @@ static int io_write_prep(struct io_kiocb req, const struct io_uring_sqe sqe, · 3 unmodified lines
2700	3128	if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
2701	3129	return -EBADF;
2702	3130
2703		− req->fsize = rlimit(RLIMIT_FSIZE);
2704		−
2705	3131	/* either don't need iovec imported or already have it */
2706	3132	if (!req->io \|\| req->flags & REQ_F_NEED_CLEANUP)
2707	3133	return 0;
2708		−
2709		− io = req->io;
2710		− io->rw.iov = io->rw.fast_iov;
2711		− req->io = NULL;
2712		− ret = io_import_iovec(WRITE, req, &io->rw.iov, &iter, !force_nonblock);
2713		− req->io = io;
2714		− if (ret < 0)
2715		− return ret;
2716		−
2717		− io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
2718		− return 0;
	3134	+ return io_rw_prep_async(req, WRITE, force_nonblock);
2719	3135	}
2720	3136
2721		−static int io_write(struct io_kiocb *req, bool force_nonblock)
	3137	+static int io_write(struct io_kiocb *req, bool force_nonblock,
	3138	+ struct io_comp_state *cs)
2722	3139	{
2723	3140	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
2724	3141	struct kiocb *kiocb = &req->rw.kiocb;
2725	3142	struct iov_iter iter;
2726	3143	size_t iov_count;
2727		− ssize_t ret, io_size;
	3144	+ ssize_t ret, ret2, io_size;
	3145	+ unsigned long nr_segs;
2728	3146
2729	3147	ret = io_import_iovec(WRITE, req, &iovec, &iter, !force_nonblock);
2730	3148	if (ret < 0)
2731	3149	return ret;
	3150	+ io_size = ret;
	3151	+ req->result = io_size;
2732	3152
2733	3153	/* Ensure we clear previously set non-block flag */
2734	3154	if (!force_nonblock)
2735	3155	req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
2736	3156
2737		− req->result = 0;
2738		− io_size = ret;
2739		− if (req->flags & REQ_F_LINK_HEAD)
2740		− req->result = io_size;
2741		−
2742		− /*
2743		− * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
2744		− * we know to async punt it even if it was opened O_NONBLOCK
2745		− */
	3157	+ /* If the file doesn't support async, just async punt */
2746	3158	if (force_nonblock && !io_file_supports_async(req->file, WRITE))
2747	3159	goto copy_iov;
2748	3160
@@ -2752,59 +3164,53 @@ static int io_write(struct io_kiocb *req, bool force_nonblock) · 3 unmodified lines
2752	3164	goto copy_iov;
2753	3165
2754	3166	iov_count = iov_iter_count(&iter);
	3167	+ nr_segs = iter.nr_segs;
2755	3168	ret = rw_verify_area(WRITE, req->file, &kiocb->ki_pos, iov_count);
2756		− if (!ret) {
2757		− ssize_t ret2;
2758		−
2759		− /*
2760		− * Open-code file_start_write here to grab freeze protection,
2761		− * which will be released by another thread in
2762		− * io_complete_rw(). Fool lockdep by telling it the lock got
2763		− * released so that it doesn't complain about the held lock when
2764		− * we return to userspace.
2765		− */
2766		− if (req->flags & REQ_F_ISREG) {
2767		− __sb_start_write(file_inode(req->file)->i_sb,
2768		− SB_FREEZE_WRITE, true);
2769		− __sb_writers_release(file_inode(req->file)->i_sb,
2770		− SB_FREEZE_WRITE);
2771		− }
2772		− kiocb->ki_flags \|= IOCB_WRITE;
2773		−
2774		− if (!force_nonblock)
2775		− current->signal->rlim[RLIMIT_FSIZE].rlim_cur = req->fsize;
	3169	+ if (unlikely(ret))
	3170	+ goto out_free;
2776	3171
2777		− if (req->file->f_op->write_iter)
2778		− ret2 = call_write_iter(req->file, kiocb, &iter);
2779		− else
2780		− ret2 = loop_rw_iter(WRITE, req->file, kiocb, &iter);
	3172	+ /*
	3173	+ * Open-code file_start_write here to grab freeze protection,
	3174	+ * which will be released by another thread in
	3175	+ * io_complete_rw(). Fool lockdep by telling it the lock got
	3176	+ * released so that it doesn't complain about the held lock when
	3177	+ * we return to userspace.
	3178	+ */
	3179	+ if (req->flags & REQ_F_ISREG) {
	3180	+ __sb_start_write(file_inode(req->file)->i_sb,
	3181	+ SB_FREEZE_WRITE, true);
	3182	+ __sb_writers_release(file_inode(req->file)->i_sb,
	3183	+ SB_FREEZE_WRITE);
	3184	+ }
	3185	+ kiocb->ki_flags \|= IOCB_WRITE;
2781	3186
2782		− if (!force_nonblock)
2783		− current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY;
	3187	+ if (req->file->f_op->write_iter)
	3188	+ ret2 = call_write_iter(req->file, kiocb, &iter);
	3189	+ else
	3190	+ ret2 = loop_rw_iter(WRITE, req->file, kiocb, &iter);
2784	3191
2785		− /*
2786		− * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
2787		− * retry them without IOCB_NOWAIT.
2788		− */
2789		− if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
2790		− ret2 = -EAGAIN;
2791		− if (!force_nonblock \|\| ret2 != -EAGAIN) {
2792		− kiocb_done(kiocb, ret2);
2793		− } else {
	3192	+ /*
	3193	+ * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
	3194	+ * retry them without IOCB_NOWAIT.
	3195	+ */
	3196	+ if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
	3197	+ ret2 = -EAGAIN;
	3198	+ if (!force_nonblock \|\| ret2 != -EAGAIN) {
	3199	+ kiocb_done(kiocb, ret2, cs);
	3200	+ } else {
	3201	+ iter.count = iov_count;
	3202	+ iter.nr_segs = nr_segs;
2794	3203	copy_iov:
2795		− ret = io_setup_async_rw(req, io_size, iovec,
2796		− inline_vecs, &iter);
2797		− if (ret)
2798		− goto out_free;
2799		− /* any defer here is final, must blocking retry */
2800		− if (!(req->flags & REQ_F_NOWAIT) &&
2801		− !file_can_poll(req->file))
2802		− req->flags \|= REQ_F_MUST_PUNT;
2803		− return -EAGAIN;
2804		− }
	3204	+ ret = io_setup_async_rw(req, io_size, iovec, inline_vecs,
	3205	+ &iter);
	3206	+ if (ret)
	3207	+ goto out_free;
	3208	+ /* it's copied and will be cleaned with ->io */
	3209	+ iovec = NULL;
	3210	+ return -EAGAIN;
2805	3211	}
2806	3212	out_free:
2807		− if (!(req->flags & REQ_F_NEED_CLEANUP))
	3213	+ if (iovec)
2808	3214	kfree(iovec);
2809	3215	return ret;
2810	3216	}
@@ -2870,10 +3276,9 @@ static int io_tee(struct io_kiocb *req, bool force_nonblock) · 59 unmodified lines
2870	3276	io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
2871	3277	req->flags &= ~REQ_F_NEED_CLEANUP;
2872	3278
2873		− io_cqring_add_event(req, ret);
2874	3279	if (ret != sp->len)
2875	3280	req_set_fail_links(req);
2876		− io_put_req(req);
	3281	+ io_req_complete(req, ret);
2877	3282	return 0;
2878	3283	}
2879	3284
@@ -2907,25 +3312,23 @@ static int io_splice(struct io_kiocb *req, bool force_nonblock) · 27 unmodified lines
2907	3312	io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
2908	3313	req->flags &= ~REQ_F_NEED_CLEANUP;
2909	3314
2910		− io_cqring_add_event(req, ret);
2911	3315	if (ret != sp->len)
2912	3316	req_set_fail_links(req);
2913		− io_put_req(req);
	3317	+ io_req_complete(req, ret);
2914	3318	return 0;
2915	3319	}
2916	3320
2917	3321	/*
2918	3322	* IORING_OP_NOP just posts a completion event, nothing else.
2919	3323	*/
2920		−static int io_nop(struct io_kiocb *req)
	3324	+static int io_nop(struct io_kiocb req, struct io_comp_state cs)
2921	3325	{
2922	3326	struct io_ring_ctx *ctx = req->ctx;
2923	3327
2924	3328	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
2925	3329	return -EINVAL;
2926	3330
2927		− io_cqring_add_event(req, 0);
2928		− io_put_req(req);
	3331	+ __io_req_complete(req, 0, 0, cs);
2929	3332	return 0;
2930	3333	}
2931	3334
@@ -2964,8 +3367,7 @@ static int io_fsync(struct io_kiocb *req, bool force_nonblock) · 32 unmodified lines
2964	3367	req->sync.flags & IORING_FSYNC_DATASYNC);
2965	3368	if (ret < 0)
2966	3369	req_set_fail_links(req);
2967		− io_cqring_add_event(req, ret);
2968		− io_put_req(req);
	3370	+ io_req_complete(req, ret);
2969	3371	return 0;
2970	3372	}
2971	3373
@@ -2980,7 +3382,6 @@ static int io_fallocate_prep(struct io_kiocb *req, · 8 unmodified lines
2980	3382	req->sync.off = READ_ONCE(sqe->off);
2981	3383	req->sync.len = READ_ONCE(sqe->addr);
2982	3384	req->sync.mode = READ_ONCE(sqe->len);
2983		− req->fsize = rlimit(RLIMIT_FSIZE);
2984	3385	return 0;
2985	3386	}
2986	3387
@@ -2991,15 +3392,11 @@ static int io_fallocate(struct io_kiocb *req, bool force_nonblock) · 4 unmodified lines
2991	3392	/* fallocate always requiring blocking context */
2992	3393	if (force_nonblock)
2993	3394	return -EAGAIN;
2994		−
2995		− current->signal->rlim[RLIMIT_FSIZE].rlim_cur = req->fsize;
2996	3395	ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
2997	3396	req->sync.len);
2998		− current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY;
2999	3397	if (ret < 0)
3000	3398	req_set_fail_links(req);
3001		− io_cqring_add_event(req, ret);
3002		− io_put_req(req);
	3399	+ io_req_complete(req, ret);
3003	3400	return 0;
3004	3401	}
3005	3402
@@ -3095,8 +3492,7 @@ err: · 89 unmodified lines
3095	3492	req->flags &= ~REQ_F_NEED_CLEANUP;
3096	3493	if (ret < 0)
3097	3494	req_set_fail_links(req);
3098		− io_cqring_add_event(req, ret);
3099		− io_put_req(req);
	3495	+ io_req_complete(req, ret);
3100	3496	return 0;
3101	3497	}
3102	3498
@@ -3150,7 +3546,8 @@ static int __io_remove_buffers(struct io_ring_ctx ctx, struct io_buffer buf, · 47 unmodified lines
3150	3546	return i;
3151	3547	}
3152	3548
3153		−static int io_remove_buffers(struct io_kiocb *req, bool force_nonblock)
	3549	+static int io_remove_buffers(struct io_kiocb *req, bool force_nonblock,
	3550	+ struct io_comp_state *cs)
3154	3551	{
3155	3552	struct io_provide_buf *p = &req->pbuf;
3156	3553	struct io_ring_ctx *ctx = req->ctx;
@@ -3169,8 +3566,7 @@ static int io_remove_buffers(struct io_kiocb *req, bool force_nonblock) · 12 unmodified lines
3169	3566	io_ring_submit_lock(ctx, !force_nonblock);
3170	3567	if (ret < 0)
3171	3568	req_set_fail_links(req);
3172		− io_cqring_add_event(req, ret);
3173		− io_put_req(req);
	3569	+ __io_req_complete(req, ret, 0, cs);
3174	3570	return 0;
3175	3571	}
3176	3572
@@ -3228,7 +3624,8 @@ static int io_add_buffers(struct io_provide_buf pbuf, struct io_buffer *head) · 51 unmodified lines
3228	3624	return i ? i : -ENOMEM;
3229	3625	}
3230	3626
3231		−static int io_provide_buffers(struct io_kiocb *req, bool force_nonblock)
	3627	+static int io_provide_buffers(struct io_kiocb *req, bool force_nonblock,
	3628	+ struct io_comp_state *cs)
3232	3629	{
3233	3630	struct io_provide_buf *p = &req->pbuf;
3234	3631	struct io_ring_ctx *ctx = req->ctx;
@@ -3257,8 +3654,7 @@ out: · 22 unmodified lines
3257	3654	io_ring_submit_unlock(ctx, !force_nonblock);
3258	3655	if (ret < 0)
3259	3656	req_set_fail_links(req);
3260		− io_cqring_add_event(req, ret);
3261		− io_put_req(req);
	3657	+ __io_req_complete(req, ret, 0, cs);
3262	3658	return 0;
3263	3659	}
3264	3660
@@ -3289,7 +3685,8 @@ static int io_epoll_ctl_prep(struct io_kiocb *req, · 24 unmodified lines
3289	3685	#endif
3290	3686	}
3291	3687
3292		−static int io_epoll_ctl(struct io_kiocb *req, bool force_nonblock)
	3688	+static int io_epoll_ctl(struct io_kiocb *req, bool force_nonblock,
	3689	+ struct io_comp_state *cs)
3293	3690	{
3294	3691	#if defined(CONFIG_EPOLL)
3295	3692	struct io_epoll *ie = &req->epoll;
@@ -3301,8 +3698,7 @@ static int io_epoll_ctl(struct io_kiocb *req, bool force_nonblock) · 5 unmodified lines
3301	3698
3302	3699	if (ret < 0)
3303	3700	req_set_fail_links(req);
3304		− io_cqring_add_event(req, ret);
3305		− io_put_req(req);
	3701	+ __io_req_complete(req, ret, 0, cs);
3306	3702	return 0;
3307	3703	#else
3308	3704	return -EOPNOTSUPP;
@@ -3338,8 +3734,7 @@ static int io_madvise(struct io_kiocb *req, bool force_nonblock) · 29 unmodified lines
3338	3734	ret = do_madvise(ma->addr, ma->len, ma->advice);
3339	3735	if (ret < 0)
3340	3736	req_set_fail_links(req);
3341		− io_cqring_add_event(req, ret);
3342		− io_put_req(req);
	3737	+ io_req_complete(req, ret);
3343	3738	return 0;
3344	3739	#else
3345	3740	return -EOPNOTSUPP;
@@ -3378,8 +3773,7 @@ static int io_fadvise(struct io_kiocb *req, bool force_nonblock) · 32 unmodified lines
3378	3773	ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
3379	3774	if (ret < 0)
3380	3775	req_set_fail_links(req);
3381		− io_cqring_add_event(req, ret);
3382		− io_put_req(req);
	3776	+ io_req_complete(req, ret);
3383	3777	return 0;
3384	3778	}
3385	3779
@@ -3418,8 +3812,7 @@ static int io_statx(struct io_kiocb *req, bool force_nonblock) · 32 unmodified lines
3418	3812
3419	3813	if (ret < 0)
3420	3814	req_set_fail_links(req);
3421		− io_cqring_add_event(req, ret);
3422		− io_put_req(req);
	3815	+ io_req_complete(req, ret);
3423	3816	return 0;
3424	3817	}
3425	3818
@@ -3450,7 +3843,8 @@ static int io_close_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 24 unmodified lines
3450	3843	return 0;
3451	3844	}
3452	3845
3453		−static int io_close(struct io_kiocb *req, bool force_nonblock)
	3846	+static int io_close(struct io_kiocb *req, bool force_nonblock,
	3847	+ struct io_comp_state *cs)
3454	3848	{
3455	3849	struct io_close *close = &req->close;
3456	3850	int ret;
@@ -3464,8 +3858,10 @@ static int io_close(struct io_kiocb *req, bool force_nonblock) · 7 unmodified lines
3464	3858
3465	3859	/* if the file has a flush method, be safe and punt to async */
3466	3860	if (close->put_file->f_op->flush && force_nonblock) {
	3861	+ /* was never set, but play safe */
	3862	+ req->flags &= ~REQ_F_NOWAIT;
3467	3863	/* avoid grabbing files - we don't need the files */
3468		− req->flags \|= REQ_F_NO_FILE_TABLE \| REQ_F_MUST_PUNT;
	3864	+ req->flags \|= REQ_F_NO_FILE_TABLE;
3469	3865	return -EAGAIN;
3470	3866	}
3471	3867
@@ -3473,10 +3869,9 @@ static int io_close(struct io_kiocb *req, bool force_nonblock) · 1 unmodified line
3473	3869	ret = filp_close(close->put_file, req->work.files);
3474	3870	if (ret < 0)
3475	3871	req_set_fail_links(req);
3476		− io_cqring_add_event(req, ret);
3477	3872	fput(close->put_file);
3478	3873	close->put_file = NULL;
3479		− io_put_req(req);
	3874	+ __io_req_complete(req, ret, 0, cs);
3480	3875	return 0;
3481	3876	}
3482	3877
@@ -3510,8 +3905,7 @@ static int io_sync_file_range(struct io_kiocb *req, bool force_nonblock) · 27 unmodified lines
3510	3905	req->sync.flags);
3511	3906	if (ret < 0)
3512	3907	req_set_fail_links(req);
3513		− io_cqring_add_event(req, ret);
3514		− io_put_req(req);
	3908	+ io_req_complete(req, ret);
3515	3909	return 0;
3516	3910	}
3517	3911
@@ -3531,6 +3925,15 @@ static int io_setup_async_msg(struct io_kiocb *req, · 13 unmodified lines
3531	3925	return -EAGAIN;
3532	3926	}
3533	3927
	3928	+static int io_sendmsg_copy_hdr(struct io_kiocb *req,
	3929	+ struct io_async_msghdr *iomsg)
	3930	+{
	3931	+ iomsg->iov = iomsg->fast_iov;
	3932	+ iomsg->msg.msg_name = &iomsg->addr;
	3933	+ return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg,
	3934	+ req->sr_msg.msg_flags, &iomsg->iov);
	3935	+}
	3936	+
3534	3937	static int io_sendmsg_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
3535	3938	{
3536	3939	struct io_sr_msg *sr = &req->sr_msg;
@@ -3541,7 +3944,7 @@ static int io_sendmsg_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 4 unmodified lines
3541	3944	return -EINVAL;
3542	3945
3543	3946	sr->msg_flags = READ_ONCE(sqe->msg_flags);
3544		− sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
	3947	+ sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
3545	3948	sr->len = READ_ONCE(sqe->len);
3546	3949
3547	3950	#ifdef CONFIG_COMPAT
@@ -3555,136 +3958,126 @@ static int io_sendmsg_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 7 unmodified lines
3555	3958	if (req->flags & REQ_F_NEED_CLEANUP)
3556	3959	return 0;
3557	3960
3558		− io->msg.msg.msg_name = &io->msg.addr;
3559		− io->msg.iov = io->msg.fast_iov;
3560		− ret = sendmsg_copy_msghdr(&io->msg.msg, sr->msg, sr->msg_flags,
3561		− &io->msg.iov);
	3961	+ ret = io_sendmsg_copy_hdr(req, &io->msg);
3562	3962	if (!ret)
3563	3963	req->flags \|= REQ_F_NEED_CLEANUP;
3564	3964	return ret;
3565	3965	}
3566	3966
3567		−static int io_sendmsg(struct io_kiocb *req, bool force_nonblock)
	3967	+static int io_sendmsg(struct io_kiocb *req, bool force_nonblock,
	3968	+ struct io_comp_state *cs)
3568	3969	{
3569		− struct io_async_msghdr *kmsg = NULL;
	3970	+ struct io_async_msghdr iomsg, *kmsg;
3570	3971	struct socket *sock;
	3972	+ unsigned flags;
3571	3973	int ret;
3572	3974
3573	3975	sock = sock_from_file(req->file, &ret);
3574		− if (sock) {
3575		− struct io_async_ctx io;
3576		− unsigned flags;
3577		−
3578		− if (req->io) {
3579		− kmsg = &req->io->msg;
3580		− kmsg->msg.msg_name = &req->io->msg.addr;
3581		− /* if iov is set, it's allocated already */
3582		− if (!kmsg->iov)
3583		− kmsg->iov = kmsg->fast_iov;
3584		− kmsg->msg.msg_iter.iov = kmsg->iov;
3585		− } else {
3586		− struct io_sr_msg *sr = &req->sr_msg;
3587		−
3588		− kmsg = &io.msg;
3589		− kmsg->msg.msg_name = &io.msg.addr;
	3976	+ if (unlikely(!sock))
	3977	+ return ret;
3590	3978
3591		− io.msg.iov = io.msg.fast_iov;
3592		− ret = sendmsg_copy_msghdr(&io.msg.msg, sr->msg,
3593		− sr->msg_flags, &io.msg.iov);
3594		− if (ret)
3595		− return ret;
3596		− }
	3979	+ if (req->io) {
	3980	+ kmsg = &req->io->msg;
	3981	+ kmsg->msg.msg_name = &req->io->msg.addr;
	3982	+ /* if iov is set, it's allocated already */
	3983	+ if (!kmsg->iov)
	3984	+ kmsg->iov = kmsg->fast_iov;
	3985	+ kmsg->msg.msg_iter.iov = kmsg->iov;
	3986	+ } else {
	3987	+ ret = io_sendmsg_copy_hdr(req, &iomsg);
	3988	+ if (ret)
	3989	+ return ret;
	3990	+ kmsg = &iomsg;
	3991	+ }
3597	3992
3598		− flags = req->sr_msg.msg_flags;
3599		− if (flags & MSG_DONTWAIT)
3600		− req->flags \|= REQ_F_NOWAIT;
3601		− else if (force_nonblock)
3602		− flags \|= MSG_DONTWAIT;
	3993	+ flags = req->sr_msg.msg_flags;
	3994	+ if (flags & MSG_DONTWAIT)
	3995	+ req->flags \|= REQ_F_NOWAIT;
	3996	+ else if (force_nonblock)
	3997	+ flags \|= MSG_DONTWAIT;
3603	3998
3604		− ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
3605		− if (force_nonblock && ret == -EAGAIN)
3606		− return io_setup_async_msg(req, kmsg);
3607		− if (ret == -ERESTARTSYS)
3608		− ret = -EINTR;
3609		− }
	3999	+ ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
	4000	+ if (force_nonblock && ret == -EAGAIN)
	4001	+ return io_setup_async_msg(req, kmsg);
	4002	+ if (ret == -ERESTARTSYS)
	4003	+ ret = -EINTR;
3610	4004
3611		− if (kmsg && kmsg->iov != kmsg->fast_iov)
	4005	+ if (kmsg->iov != kmsg->fast_iov)
3612	4006	kfree(kmsg->iov);
3613	4007	req->flags &= ~REQ_F_NEED_CLEANUP;
3614		− io_cqring_add_event(req, ret);
3615	4008	if (ret < 0)
3616	4009	req_set_fail_links(req);
3617		− io_put_req(req);
	4010	+ __io_req_complete(req, ret, 0, cs);
3618	4011	return 0;
3619	4012	}
3620	4013
3621		−static int io_send(struct io_kiocb *req, bool force_nonblock)
	4014	+static int io_send(struct io_kiocb *req, bool force_nonblock,
	4015	+ struct io_comp_state *cs)
3622	4016	{
	4017	+ struct io_sr_msg *sr = &req->sr_msg;
	4018	+ struct msghdr msg;
	4019	+ struct iovec iov;
3623	4020	struct socket *sock;
	4021	+ unsigned flags;
3624	4022	int ret;
3625	4023
3626	4024	sock = sock_from_file(req->file, &ret);
3627		− if (sock) {
3628		− struct io_sr_msg *sr = &req->sr_msg;
3629		− struct msghdr msg;
3630		− struct iovec iov;
3631		− unsigned flags;
	4025	+ if (unlikely(!sock))
	4026	+ return ret;
3632	4027
3633		− ret = import_single_range(WRITE, sr->buf, sr->len, &iov,
3634		− &msg.msg_iter);
3635		− if (ret)
3636		− return ret;
	4028	+ ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
	4029	+ if (unlikely(ret))
	4030	+ return ret;;
3637	4031
3638		− msg.msg_name = NULL;
3639		− msg.msg_control = NULL;
3640		− msg.msg_controllen = 0;
3641		− msg.msg_namelen = 0;
	4032	+ msg.msg_name = NULL;
	4033	+ msg.msg_control = NULL;
	4034	+ msg.msg_controllen = 0;
	4035	+ msg.msg_namelen = 0;
3642	4036
3643		− flags = req->sr_msg.msg_flags;
3644		− if (flags & MSG_DONTWAIT)
3645		− req->flags \|= REQ_F_NOWAIT;
3646		− else if (force_nonblock)
3647		− flags \|= MSG_DONTWAIT;
	4037	+ flags = req->sr_msg.msg_flags;
	4038	+ if (flags & MSG_DONTWAIT)
	4039	+ req->flags \|= REQ_F_NOWAIT;
	4040	+ else if (force_nonblock)
	4041	+ flags \|= MSG_DONTWAIT;
3648	4042
3649		− msg.msg_flags = flags;
3650		− ret = sock_sendmsg(sock, &msg);
3651		− if (force_nonblock && ret == -EAGAIN)
3652		− return -EAGAIN;
3653		− if (ret == -ERESTARTSYS)
3654		− ret = -EINTR;
3655		− }
	4043	+ msg.msg_flags = flags;
	4044	+ ret = sock_sendmsg(sock, &msg);
	4045	+ if (force_nonblock && ret == -EAGAIN)
	4046	+ return -EAGAIN;
	4047	+ if (ret == -ERESTARTSYS)
	4048	+ ret = -EINTR;
3656	4049
3657		− io_cqring_add_event(req, ret);
3658	4050	if (ret < 0)
3659	4051	req_set_fail_links(req);
3660		− io_put_req(req);
	4052	+ __io_req_complete(req, ret, 0, cs);
3661	4053	return 0;
3662	4054	}
3663	4055
3664		−static int __io_recvmsg_copy_hdr(struct io_kiocb req, struct io_async_ctx io)
	4056	+static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
	4057	+ struct io_async_msghdr *iomsg)
3665	4058	{
3666	4059	struct io_sr_msg *sr = &req->sr_msg;
3667	4060	struct iovec __user *uiov;
3668	4061	size_t iov_len;
3669	4062	int ret;
3670	4063
3671		− ret = __copy_msghdr_from_user(&io->msg.msg, sr->msg, &io->msg.uaddr,
3672		− &uiov, &iov_len);
	4064	+ ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg,
	4065	+ &iomsg->uaddr, &uiov, &iov_len);
3673	4066	if (ret)
3674	4067	return ret;
3675	4068
3676	4069	if (req->flags & REQ_F_BUFFER_SELECT) {
3677	4070	if (iov_len > 1)
3678	4071	return -EINVAL;
3679		− if (copy_from_user(io->msg.iov, uiov, sizeof(*uiov)))
	4072	+ if (copy_from_user(iomsg->iov, uiov, sizeof(*uiov)))
3680	4073	return -EFAULT;
3681		− sr->len = io->msg.iov[0].iov_len;
3682		− iov_iter_init(&io->msg.msg.msg_iter, READ, io->msg.iov, 1,
	4074	+ sr->len = iomsg->iov[0].iov_len;
	4075	+ iov_iter_init(&iomsg->msg.msg_iter, READ, iomsg->iov, 1,
3683	4076	sr->len);
3684		− io->msg.iov = NULL;
	4077	+ iomsg->iov = NULL;
3685	4078	} else {
3686	4079	ret = import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
3687		− &io->msg.iov, &io->msg.msg.msg_iter);
	4080	+ &iomsg->iov, &iomsg->msg.msg_iter);
3688	4081	if (ret > 0)
3689	4082	ret = 0;
3690	4083	}
@@ -3694,7 +4087,7 @@ static int __io_recvmsg_copy_hdr(struct io_kiocb req, struct io_async_ctx io) · 3 unmodified lines
3694	4087
3695	4088	#ifdef CONFIG_COMPAT
3696	4089	static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
3697		− struct io_async_ctx *io)
	4090	+ struct io_async_msghdr *iomsg)
3698	4091	{
3699	4092	struct compat_msghdr __user *msg_compat;
3700	4093	struct io_sr_msg *sr = &req->sr_msg;
@@ -3703,8 +4096,8 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, · 2 unmodified lines
3703	4096	compat_size_t len;
3704	4097	int ret;
3705	4098
3706		− msg_compat = (struct compat_msghdr __user *) sr->msg;
3707		− ret = __get_compat_msghdr(&io->msg.msg, msg_compat, &io->msg.uaddr,
	4099	+ msg_compat = (struct compat_msghdr __user *) sr->umsg;
	4100	+ ret = __get_compat_msghdr(&iomsg->msg, msg_compat, &iomsg->uaddr,
3708	4101	&ptr, &len);
3709	4102	if (ret)
3710	4103	return ret;
@@ -3721,12 +4114,12 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, · 10 unmodified lines
3721	4114	return -EFAULT;
3722	4115	if (clen < 0)
3723	4116	return -EINVAL;
3724		− sr->len = io->msg.iov[0].iov_len;
3725		− io->msg.iov = NULL;
	4117	+ sr->len = iomsg->iov[0].iov_len;
	4118	+ iomsg->iov = NULL;
3726	4119	} else {
3727	4120	ret = compat_import_iovec(READ, uiov, len, UIO_FASTIOV,
3728		− &io->msg.iov,
3729		− &io->msg.msg.msg_iter);
	4121	+ &iomsg->iov,
	4122	+ &iomsg->msg.msg_iter);
3730	4123	if (ret < 0)
3731	4124	return ret;
3732	4125	}
@@ -3735,40 +4128,40 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, · 2 unmodified lines
3735	4128	}
3736	4129	#endif
3737	4130
3738		−static int io_recvmsg_copy_hdr(struct io_kiocb req, struct io_async_ctx io)
	4131	+static int io_recvmsg_copy_hdr(struct io_kiocb *req,
	4132	+ struct io_async_msghdr *iomsg)
3739	4133	{
3740		− io->msg.msg.msg_name = &io->msg.addr;
3741		− io->msg.iov = io->msg.fast_iov;
	4134	+ iomsg->msg.msg_name = &iomsg->addr;
	4135	+ iomsg->iov = iomsg->fast_iov;
3742	4136
3743	4137	#ifdef CONFIG_COMPAT
3744	4138	if (req->ctx->compat)
3745		− return __io_compat_recvmsg_copy_hdr(req, io);
	4139	+ return __io_compat_recvmsg_copy_hdr(req, iomsg);
3746	4140	#endif
3747	4141
3748		− return __io_recvmsg_copy_hdr(req, io);
	4142	+ return __io_recvmsg_copy_hdr(req, iomsg);
3749	4143	}
3750	4144
3751	4145	static struct io_buffer io_recv_buffer_select(struct io_kiocb req,
3752		− int *cflags, bool needs_lock)
	4146	+ bool needs_lock)
3753	4147	{
3754	4148	struct io_sr_msg *sr = &req->sr_msg;
3755	4149	struct io_buffer *kbuf;
3756	4150
3757		− if (!(req->flags & REQ_F_BUFFER_SELECT))
3758		− return NULL;
3759		−
3760	4151	kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
3761	4152	if (IS_ERR(kbuf))
3762	4153	return kbuf;
3763	4154
3764	4155	sr->kbuf = kbuf;
3765	4156	req->flags \|= REQ_F_BUFFER_SELECTED;
3766		−
3767		− *cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
3768		− *cflags \|= IORING_CQE_F_BUFFER;
3769	4157	return kbuf;
3770	4158	}
3771	4159
	4160	+static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req)
	4161	+{
	4162	+ return io_put_kbuf(req, req->sr_msg.kbuf);
	4163	+}
	4164	+
3772	4165	static int io_recvmsg_prep(struct io_kiocb *req,
3773	4166	const struct io_uring_sqe *sqe)
3774	4167	{
@@ -3780,7 +4173,7 @@ static int io_recvmsg_prep(struct io_kiocb *req, · 5 unmodified lines
3780	4173	return -EINVAL;
3781	4174
3782	4175	sr->msg_flags = READ_ONCE(sqe->msg_flags);
3783		− sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
	4176	+ sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
3784	4177	sr->len = READ_ONCE(sqe->len);
3785	4178	sr->bgid = READ_ONCE(sqe->buf_group);
3786	4179
@@ -3795,133 +4188,123 @@ static int io_recvmsg_prep(struct io_kiocb *req, · 8 unmodified lines
3795	4188	if (req->flags & REQ_F_NEED_CLEANUP)
3796	4189	return 0;
3797	4190
3798		− ret = io_recvmsg_copy_hdr(req, io);
	4191	+ ret = io_recvmsg_copy_hdr(req, &io->msg);
3799	4192	if (!ret)
3800	4193	req->flags \|= REQ_F_NEED_CLEANUP;
3801	4194	return ret;
3802	4195	}
3803	4196
3804		−static int io_recvmsg(struct io_kiocb *req, bool force_nonblock)
	4197	+static int io_recvmsg(struct io_kiocb *req, bool force_nonblock,
	4198	+ struct io_comp_state *cs)
3805	4199	{
3806		− struct io_async_msghdr *kmsg = NULL;
	4200	+ struct io_async_msghdr iomsg, *kmsg;
3807	4201	struct socket *sock;
	4202	+ struct io_buffer *kbuf;
	4203	+ unsigned flags;
3808	4204	int ret, cflags = 0;
3809	4205
3810	4206	sock = sock_from_file(req->file, &ret);
3811		− if (sock) {
3812		− struct io_buffer *kbuf;
3813		− struct io_async_ctx io;
3814		− unsigned flags;
3815		−
3816		− if (req->io) {
3817		− kmsg = &req->io->msg;
3818		− kmsg->msg.msg_name = &req->io->msg.addr;
3819		− /* if iov is set, it's allocated already */
3820		− if (!kmsg->iov)
3821		− kmsg->iov = kmsg->fast_iov;
3822		− kmsg->msg.msg_iter.iov = kmsg->iov;
3823		− } else {
3824		− kmsg = &io.msg;
3825		− kmsg->msg.msg_name = &io.msg.addr;
	4207	+ if (unlikely(!sock))
	4208	+ return ret;
3826	4209
3827		− ret = io_recvmsg_copy_hdr(req, &io);
3828		− if (ret)
3829		− return ret;
3830		− }
	4210	+ if (req->io) {
	4211	+ kmsg = &req->io->msg;
	4212	+ kmsg->msg.msg_name = &req->io->msg.addr;
	4213	+ /* if iov is set, it's allocated already */
	4214	+ if (!kmsg->iov)
	4215	+ kmsg->iov = kmsg->fast_iov;
	4216	+ kmsg->msg.msg_iter.iov = kmsg->iov;
	4217	+ } else {
	4218	+ ret = io_recvmsg_copy_hdr(req, &iomsg);
	4219	+ if (ret)
	4220	+ return ret;
	4221	+ kmsg = &iomsg;
	4222	+ }
3831	4223
3832		− kbuf = io_recv_buffer_select(req, &cflags, !force_nonblock);
3833		− if (IS_ERR(kbuf)) {
	4224	+ if (req->flags & REQ_F_BUFFER_SELECT) {
	4225	+ kbuf = io_recv_buffer_select(req, !force_nonblock);
	4226	+ if (IS_ERR(kbuf))
3834	4227	return PTR_ERR(kbuf);
3835		− } else if (kbuf) {
3836		− kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
3837		− iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->iov,
3838		− 1, req->sr_msg.len);
3839		− }
	4228	+ kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
	4229	+ iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->iov,
	4230	+ 1, req->sr_msg.len);
	4231	+ }
3840	4232
3841		− flags = req->sr_msg.msg_flags;
3842		− if (flags & MSG_DONTWAIT)
3843		− req->flags \|= REQ_F_NOWAIT;
3844		− else if (force_nonblock)
3845		− flags \|= MSG_DONTWAIT;
	4233	+ flags = req->sr_msg.msg_flags;
	4234	+ if (flags & MSG_DONTWAIT)
	4235	+ req->flags \|= REQ_F_NOWAIT;
	4236	+ else if (force_nonblock)
	4237	+ flags \|= MSG_DONTWAIT;
3846	4238
3847		− ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.msg,
3848		− kmsg->uaddr, flags);
3849		− if (force_nonblock && ret == -EAGAIN) {
3850		− ret = io_setup_async_msg(req, kmsg);
3851		− if (ret != -EAGAIN)
3852		− kfree(kbuf);
3853		− return ret;
3854		− }
3855		− if (ret == -ERESTARTSYS)
3856		− ret = -EINTR;
3857		− if (kbuf)
3858		− kfree(kbuf);
3859		− }
	4239	+ ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg,
	4240	+ kmsg->uaddr, flags);
	4241	+ if (force_nonblock && ret == -EAGAIN)
	4242	+ return io_setup_async_msg(req, kmsg);
	4243	+ if (ret == -ERESTARTSYS)
	4244	+ ret = -EINTR;
3860	4245
3861		− if (kmsg && kmsg->iov != kmsg->fast_iov)
	4246	+ if (req->flags & REQ_F_BUFFER_SELECTED)
	4247	+ cflags = io_put_recv_kbuf(req);
	4248	+ if (kmsg->iov != kmsg->fast_iov)
3862	4249	kfree(kmsg->iov);
3863	4250	req->flags &= ~REQ_F_NEED_CLEANUP;
3864		− __io_cqring_add_event(req, ret, cflags);
3865	4251	if (ret < 0)
3866	4252	req_set_fail_links(req);
3867		− io_put_req(req);
	4253	+ __io_req_complete(req, ret, cflags, cs);
3868	4254	return 0;
3869	4255	}
3870	4256
3871		−static int io_recv(struct io_kiocb *req, bool force_nonblock)
	4257	+static int io_recv(struct io_kiocb *req, bool force_nonblock,
	4258	+ struct io_comp_state *cs)
3872	4259	{
3873		− struct io_buffer *kbuf = NULL;
	4260	+ struct io_buffer *kbuf;
	4261	+ struct io_sr_msg *sr = &req->sr_msg;
	4262	+ struct msghdr msg;
	4263	+ void __user *buf = sr->buf;
3874	4264	struct socket *sock;
	4265	+ struct iovec iov;
	4266	+ unsigned flags;
3875	4267	int ret, cflags = 0;
3876	4268
3877	4269	sock = sock_from_file(req->file, &ret);
3878		− if (sock) {
3879		− struct io_sr_msg *sr = &req->sr_msg;
3880		− void __user *buf = sr->buf;
3881		− struct msghdr msg;
3882		− struct iovec iov;
3883		− unsigned flags;
	4270	+ if (unlikely(!sock))
	4271	+ return ret;
3884	4272
3885		− kbuf = io_recv_buffer_select(req, &cflags, !force_nonblock);
	4273	+ if (req->flags & REQ_F_BUFFER_SELECT) {
	4274	+ kbuf = io_recv_buffer_select(req, !force_nonblock);
3886	4275	if (IS_ERR(kbuf))
3887	4276	return PTR_ERR(kbuf);
3888		− else if (kbuf)
3889		− buf = u64_to_user_ptr(kbuf->addr);
	4277	+ buf = u64_to_user_ptr(kbuf->addr);
	4278	+ }
3890	4279
3891		− ret = import_single_range(READ, buf, sr->len, &iov,
3892		− &msg.msg_iter);
3893		− if (ret) {
3894		− kfree(kbuf);
3895		− return ret;
3896		− }
	4280	+ ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter);
	4281	+ if (unlikely(ret))
	4282	+ goto out_free;
3897	4283
3898		− req->flags \|= REQ_F_NEED_CLEANUP;
3899		− msg.msg_name = NULL;
3900		− msg.msg_control = NULL;
3901		− msg.msg_controllen = 0;
3902		− msg.msg_namelen = 0;
3903		− msg.msg_iocb = NULL;
3904		− msg.msg_flags = 0;
3905		−
3906		− flags = req->sr_msg.msg_flags;
3907		− if (flags & MSG_DONTWAIT)
3908		− req->flags \|= REQ_F_NOWAIT;
3909		− else if (force_nonblock)
3910		− flags \|= MSG_DONTWAIT;
3911		−
3912		− ret = sock_recvmsg(sock, &msg, flags);
3913		− if (force_nonblock && ret == -EAGAIN)
3914		− return -EAGAIN;
3915		− if (ret == -ERESTARTSYS)
3916		− ret = -EINTR;
3917		− }
	4284	+ msg.msg_name = NULL;
	4285	+ msg.msg_control = NULL;
	4286	+ msg.msg_controllen = 0;
	4287	+ msg.msg_namelen = 0;
	4288	+ msg.msg_iocb = NULL;
	4289	+ msg.msg_flags = 0;
3918	4290
3919		− kfree(kbuf);
3920		− req->flags &= ~REQ_F_NEED_CLEANUP;
3921		− __io_cqring_add_event(req, ret, cflags);
	4291	+ flags = req->sr_msg.msg_flags;
	4292	+ if (flags & MSG_DONTWAIT)
	4293	+ req->flags \|= REQ_F_NOWAIT;
	4294	+ else if (force_nonblock)
	4295	+ flags \|= MSG_DONTWAIT;
	4296	+
	4297	+ ret = sock_recvmsg(sock, &msg, flags);
	4298	+ if (force_nonblock && ret == -EAGAIN)
	4299	+ return -EAGAIN;
	4300	+ if (ret == -ERESTARTSYS)
	4301	+ ret = -EINTR;
	4302	+out_free:
	4303	+ if (req->flags & REQ_F_BUFFER_SELECTED)
	4304	+ cflags = io_put_recv_kbuf(req);
3922	4305	if (ret < 0)
3923	4306	req_set_fail_links(req);
3924		− io_put_req(req);
	4307	+ __io_req_complete(req, ret, cflags, cs);
3925	4308	return 0;
3926	4309	}
3927	4310
@@ -3941,7 +4324,8 @@ static int io_accept_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 13 unmodified lines
3941	4324	return 0;
3942	4325	}
3943	4326
3944		−static int io_accept(struct io_kiocb *req, bool force_nonblock)
	4327	+static int io_accept(struct io_kiocb *req, bool force_nonblock,
	4328	+ struct io_comp_state *cs)
3945	4329	{
3946	4330	struct io_accept *accept = &req->accept;
3947	4331	unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
@@ -3960,8 +4344,7 @@ static int io_accept(struct io_kiocb *req, bool force_nonblock) · 12 unmodified lines
3960	4344	ret = -EINTR;
3961	4345	req_set_fail_links(req);
3962	4346	}
3963		− io_cqring_add_event(req, ret);
3964		− io_put_req(req);
	4347	+ __io_req_complete(req, ret, 0, cs);
3965	4348	return 0;
3966	4349	}
3967	4350
@@ -3985,7 +4368,8 @@ static int io_connect_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 17 unmodified lines
3985	4368	&io->connect.address);
3986	4369	}
3987	4370
3988		−static int io_connect(struct io_kiocb *req, bool force_nonblock)
	4371	+static int io_connect(struct io_kiocb *req, bool force_nonblock,
	4372	+ struct io_comp_state *cs)
3989	4373	{
3990	4374	struct io_async_ctx __io, *io;
3991	4375	unsigned file_flags;
@@ -4021,8 +4405,7 @@ static int io_connect(struct io_kiocb *req, bool force_nonblock) · 29 unmodified lines
4021	4405	out:
4022	4406	if (ret < 0)
4023	4407	req_set_fail_links(req);
4024		− io_cqring_add_event(req, ret);
4025		− io_put_req(req);
	4408	+ __io_req_complete(req, ret, 0, cs);
4026	4409	return 0;
4027	4410	}
4028	4411	#else /* !CONFIG_NET */
@@ -4031,12 +4414,14 @@ static int io_sendmsg_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 2 unmodified lines
4031	4414	return -EOPNOTSUPP;
4032	4415	}
4033	4416
4034		−static int io_sendmsg(struct io_kiocb *req, bool force_nonblock)
	4417	+static int io_sendmsg(struct io_kiocb *req, bool force_nonblock,
	4418	+ struct io_comp_state *cs)
4035	4419	{
4036	4420	return -EOPNOTSUPP;
4037	4421	}
4038	4422
4039		−static int io_send(struct io_kiocb *req, bool force_nonblock)
	4423	+static int io_send(struct io_kiocb *req, bool force_nonblock,
	4424	+ struct io_comp_state *cs)
4040	4425	{
4041	4426	return -EOPNOTSUPP;
4042	4427	}
@@ -4047,12 +4432,14 @@ static int io_recvmsg_prep(struct io_kiocb *req, · 4 unmodified lines
4047	4432	return -EOPNOTSUPP;
4048	4433	}
4049	4434
4050		−static int io_recvmsg(struct io_kiocb *req, bool force_nonblock)
	4435	+static int io_recvmsg(struct io_kiocb *req, bool force_nonblock,
	4436	+ struct io_comp_state *cs)
4051	4437	{
4052	4438	return -EOPNOTSUPP;
4053	4439	}
4054	4440
4055		−static int io_recv(struct io_kiocb *req, bool force_nonblock)
	4441	+static int io_recv(struct io_kiocb *req, bool force_nonblock,
	4442	+ struct io_comp_state *cs)
4056	4443	{
4057	4444	return -EOPNOTSUPP;
4058	4445	}
@@ -4062,55 +4449,33 @@ static int io_accept_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 3 unmodified lines
4062	4449	return -EOPNOTSUPP;
4063	4450	}
4064	4451
4065		−static int io_accept(struct io_kiocb *req, bool force_nonblock)
4066		−{
4067		− return -EOPNOTSUPP;
4068		−}
4069		−
4070		−static int io_connect_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
4071		−{
4072		− return -EOPNOTSUPP;
4073		−}
4074		−
4075		−static int io_connect(struct io_kiocb *req, bool force_nonblock)
4076		−{
4077		− return -EOPNOTSUPP;
4078		−}
4079		−#endif /* CONFIG_NET */
4080		−
4081		−struct io_poll_table {
4082		− struct poll_table_struct pt;
4083		− struct io_kiocb *req;
4084		− int error;
4085		−};
4086		−
4087		−static int io_req_task_work_add(struct io_kiocb req, struct callback_head cb)
4088		−{
4089		− struct task_struct *tsk = req->task;
4090		− struct io_ring_ctx *ctx = req->ctx;
4091		− int ret, notify = TWA_RESUME;
4092		−
4093		− /*
4094		− * SQPOLL kernel thread doesn't need notification, just a wakeup.
4095		− * If we're not using an eventfd, then TWA_RESUME is always fine,
4096		− * as we won't have dependencies between request completions for
4097		− * other kernel wait conditions.
4098		− */
4099		− if (ctx->flags & IORING_SETUP_SQPOLL)
4100		− notify = 0;
4101		− else if (ctx->cq_ev_fd)
4102		− notify = TWA_SIGNAL;
	4452	+static int io_accept(struct io_kiocb *req, bool force_nonblock,
	4453	+ struct io_comp_state *cs)
	4454	+{
	4455	+ return -EOPNOTSUPP;
	4456	+}
4103	4457
4104		− ret = task_work_add(tsk, cb, notify);
4105		− if (!ret)
4106		− wake_up_process(tsk);
4107		− return ret;
	4458	+static int io_connect_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
	4459	+{
	4460	+ return -EOPNOTSUPP;
	4461	+}
	4462	+
	4463	+static int io_connect(struct io_kiocb *req, bool force_nonblock,
	4464	+ struct io_comp_state *cs)
	4465	+{
	4466	+ return -EOPNOTSUPP;
4108	4467	}
	4468	+#endif /* CONFIG_NET */
	4469	+
	4470	+struct io_poll_table {
	4471	+ struct poll_table_struct pt;
	4472	+ struct io_kiocb *req;
	4473	+ int error;
	4474	+};
4109	4475
4110	4476	static int __io_async_wake(struct io_kiocb req, struct io_poll_iocb poll,
4111	4477	__poll_t mask, task_work_func_t func)
4112	4478	{
4113		− struct task_struct *tsk;
4114	4479	int ret;
4115	4480
4116	4481	/* for instances that support it check for an event match first: */
@@ -4121,7 +4486,6 @@ static int __io_async_wake(struct io_kiocb req, struct io_poll_iocb poll, · 4 unmodified lines
4121	4486
4122	4487	list_del_init(&poll->wait.entry);
4123	4488
4124		− tsk = req->task;
4125	4489	req->result = mask;
4126	4490	init_task_work(&req->task_work, func);
4127	4491	/*
@@ -4132,6 +4496,8 @@ static int __io_async_wake(struct io_kiocb req, struct io_poll_iocb poll, · 4 unmodified lines
4132	4496	*/
4133	4497	ret = io_req_task_work_add(req, &req->task_work);
4134	4498	if (unlikely(ret)) {
	4499	+ struct task_struct *tsk;
	4500	+
4135	4501	WRITE_ONCE(poll->canceled, true);
4136	4502	tsk = io_wq_get_task(req->ctx->io_wq);
4137	4503	task_work_add(tsk, &req->task_work, 0);
@@ -4199,9 +4565,10 @@ static void io_poll_task_handler(struct io_kiocb req, struct io_kiocb *nxt) · 61 unmodified lines
4199	4565
4200	4566	hash_del(&req->hash_node);
4201	4567	io_poll_complete(req, req->result, 0);
	4568	+ req->flags \|= REQ_F_COMP_LOCKED;
	4569	+ *nxt = io_put_req_find_next(req);
4202	4570	spin_unlock_irq(&ctx->completion_lock);
4203	4571
4204		− io_put_req_find_next(req, nxt);
4205	4572	io_cqring_ev_posted(ctx);
4206	4573	}
4207	4574
@@ -4211,13 +4578,8 @@ static void io_poll_task_func(struct callback_head *cb) · 3 unmodified lines
4211	4578	struct io_kiocb *nxt = NULL;
4212	4579
4213	4580	io_poll_task_handler(req, &nxt);
4214		− if (nxt) {
4215		− struct io_ring_ctx *ctx = nxt->ctx;
4216		−
4217		− mutex_lock(&ctx->uring_lock);
4218		− __io_queue_sqe(nxt, NULL);
4219		− mutex_unlock(&ctx->uring_lock);
4220		− }
	4581	+ if (nxt)
	4582	+ __io_req_task_submit(nxt);
4221	4583	}
4222	4584
4223	4585	static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode,
@@ -4287,7 +4649,11 @@ static void __io_queue_proc(struct io_poll_iocb poll, struct io_poll_table pt, · 63 unmodified lines
4287	4649
4288	4650	pt->error = 0;
4289	4651	poll->head = head;
4290		− add_wait_queue(head, &poll->wait);
	4652	+
	4653	+ if (poll->events & EPOLLEXCLUSIVE)
	4654	+ add_wait_queue_exclusive(head, &poll->wait);
	4655	+ else
	4656	+ add_wait_queue(head, &poll->wait);
4291	4657	}
4292	4658
4293	4659	static void io_async_queue_proc(struct file file, struct wait_queue_head head,
@@ -4299,34 +4665,11 @@ static void io_async_queue_proc(struct file file, struct wait_queue_head head, · 5 unmodified lines
4299	4665	__io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);
4300	4666	}
4301	4667
4302		−static void io_sq_thread_drop_mm(struct io_ring_ctx *ctx)
4303		−{
4304		− struct mm_struct *mm = current->mm;
4305		−
4306		− if (mm) {
4307		− kthread_unuse_mm(mm);
4308		− mmput(mm);
4309		− }
4310		−}
4311		−
4312		−static int io_sq_thread_acquire_mm(struct io_ring_ctx *ctx,
4313		− struct io_kiocb *req)
4314		−{
4315		− if (io_op_defs[req->opcode].needs_mm && !current->mm) {
4316		− if (unlikely(!mmget_not_zero(ctx->sqo_mm)))
4317		− return -EFAULT;
4318		− kthread_use_mm(ctx->sqo_mm);
4319		− }
4320		−
4321		− return 0;
4322		−}
4323		−
4324	4668	static void io_async_task_func(struct callback_head *cb)
4325	4669	{
4326	4670	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
4327	4671	struct async_poll *apoll = req->apoll;
4328	4672	struct io_ring_ctx *ctx = req->ctx;
4329		− bool canceled = false;
4330	4673
4331	4674	trace_io_uring_task_run(req->ctx, req->opcode, req->user_data);
4332	4675
@@ -4336,40 +4679,19 @@ static void io_async_task_func(struct callback_head *cb) · 3 unmodified lines
4336	4679	}
4337	4680
4338	4681	/* If req is still hashed, it cannot have been canceled. Don't check. */
4339		− if (hash_hashed(&req->hash_node)) {
	4682	+ if (hash_hashed(&req->hash_node))
4340	4683	hash_del(&req->hash_node);
4341		− } else {
4342		− canceled = READ_ONCE(apoll->poll.canceled);
4343		− if (canceled) {
4344		− io_cqring_fill_event(req, -ECANCELED);
4345		− io_commit_cqring(ctx);
4346		− }
4347		− }
4348	4684
4349	4685	io_poll_remove_double(req, apoll->double_poll);
4350	4686	spin_unlock_irq(&ctx->completion_lock);
4351	4687
4352		− /* restore ->work in case we need to retry again */
4353		− if (req->flags & REQ_F_WORK_INITIALIZED)
4354		− memcpy(&req->work, &apoll->work, sizeof(req->work));
	4688	+ if (!READ_ONCE(apoll->poll.canceled))
	4689	+ __io_req_task_submit(req);
	4690	+ else
	4691	+ __io_req_task_cancel(req, -ECANCELED);
	4692	+
4355	4693	kfree(apoll->double_poll);
4356	4694	kfree(apoll);
4357		−
4358		− if (!canceled) {
4359		− __set_current_state(TASK_RUNNING);
4360		− if (io_sq_thread_acquire_mm(ctx, req)) {
4361		− io_cqring_add_event(req, -EFAULT);
4362		− goto end_req;
4363		− }
4364		− mutex_lock(&ctx->uring_lock);
4365		− __io_queue_sqe(req, NULL);
4366		− mutex_unlock(&ctx->uring_lock);
4367		− } else {
4368		− io_cqring_ev_posted(ctx);
4369		−end_req:
4370		− req_set_fail_links(req);
4371		− io_double_put_req(req);
4372		− }
4373	4695	}
4374	4696
4375	4697	static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
@@ -4402,8 +4724,8 @@ static __poll_t __io_arm_poll_handler(struct io_kiocb *req, · 26 unmodified lines
4402	4724	struct io_ring_ctx *ctx = req->ctx;
4403	4725	bool cancel = false;
4404	4726
4405		− poll->file = req->file;
4406	4727	io_init_poll_iocb(poll, mask, wake_func);
	4728	+ poll->file = req->file;
4407	4729	poll->wait.private = req;
4408	4730
4409	4731	ipt->pt._key = mask;
@@ -4443,7 +4765,7 @@ static bool io_arm_poll_handler(struct io_kiocb *req) · 33 unmodified lines
4443	4765
4444	4766	if (!req->file \|\| !file_can_poll(req->file))
4445	4767	return false;
4446		− if (req->flags & (REQ_F_MUST_PUNT \| REQ_F_POLLED))
	4768	+ if (req->flags & REQ_F_POLLED)
4447	4769	return false;
4448	4770	if (!def->pollin && !def->pollout)
4449	4771	return false;
@@ -4454,9 +4776,6 @@ static bool io_arm_poll_handler(struct io_kiocb *req) · 4 unmodified lines
4454	4776	apoll->double_poll = NULL;
4455	4777
4456	4778	req->flags \|= REQ_F_POLLED;
4457		− if (req->flags & REQ_F_WORK_INITIALIZED)
4458		− memcpy(&apoll->work, &req->work, sizeof(req->work));
4459		−
4460	4779	io_get_req_task(req);
4461	4780	req->apoll = apoll;
4462	4781	INIT_HLIST_NODE(&req->hash_node);
@@ -4475,8 +4794,6 @@ static bool io_arm_poll_handler(struct io_kiocb *req) · 12 unmodified lines
4475	4794	if (ret) {
4476	4795	io_poll_remove_double(req, apoll->double_poll);
4477	4796	spin_unlock_irq(&ctx->completion_lock);
4478		− if (req->flags & REQ_F_WORK_INITIALIZED)
4479		− memcpy(&req->work, &apoll->work, sizeof(req->work));
4480	4797	kfree(apoll->double_poll);
4481	4798	kfree(apoll);
4482	4799	return false;
@@ -4519,14 +4836,6 @@ static bool io_poll_remove_one(struct io_kiocb *req) · 36 unmodified lines
4519	4836	do_complete = __io_poll_remove_one(req, &apoll->poll);
4520	4837	if (do_complete) {
4521	4838	io_put_req(req);
4522		− /*
4523		− * restore ->work because we will call
4524		− * io_req_work_drop_env below when dropping the
4525		− * final reference.
4526		− */
4527		− if (req->flags & REQ_F_WORK_INITIALIZED)
4528		− memcpy(&req->work, &apoll->work,
4529		− sizeof(req->work));
4530	4839	kfree(apoll->double_poll);
4531	4840	kfree(apoll);
4532	4841	}
@@ -4607,10 +4916,9 @@ static int io_poll_remove(struct io_kiocb *req) · 74 unmodified lines
4607	4916	ret = io_poll_cancel(ctx, addr);
4608	4917	spin_unlock_irq(&ctx->completion_lock);
4609	4918
4610		− io_cqring_add_event(req, ret);
4611	4919	if (ret < 0)
4612	4920	req_set_fail_links(req);
4613		− io_put_req(req);
	4921	+ io_req_complete(req, ret);
4614	4922	return 0;
4615	4923	}
4616	4924
@@ -4634,7 +4942,7 @@ static void io_poll_queue_proc(struct file file, struct wait_queue_head head, · 17 unmodified lines
4634	4942	static int io_poll_add_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
4635	4943	{
4636	4944	struct io_poll_iocb *poll = &req->poll;
4637		− u16 events;
	4945	+ u32 events;
4638	4946
4639	4947	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4640	4948	return -EINVAL;
@@ -4643,8 +4951,12 @@ static int io_poll_add_prep(struct io_kiocb req, const struct io_uring_sqe sqe · 2 unmodified lines
4643	4951	if (!poll->file)
4644	4952	return -EBADF;
4645	4953
4646		− events = READ_ONCE(sqe->poll_events);
4647		− poll->events = demangle_poll(events) \| EPOLLERR \| EPOLLHUP;
	4954	+ events = READ_ONCE(sqe->poll32_events);
	4955	+#ifdef __BIG_ENDIAN
	4956	+ events = swahw32(events);
	4957	+#endif
	4958	+ poll->events = demangle_poll(events) \| EPOLLERR \| EPOLLHUP \|
	4959	+ (events & EPOLLEXCLUSIVE);
4648	4960
4649	4961	io_get_req_task(req);
4650	4962	return 0;
@@ -4657,12 +4969,7 @@ static int io_poll_add(struct io_kiocb *req) · 6 unmodified lines
4657	4969	struct io_poll_table ipt;
4658	4970	__poll_t mask;
4659	4971
4660		− /* ->work is in union with hash_node and others */
4661		− io_req_work_drop_env(req);
4662		− req->flags &= ~REQ_F_WORK_INITIALIZED;
4663		−
4664	4972	INIT_HLIST_NODE(&req->hash_node);
4665		− INIT_LIST_HEAD(&req->list);
4666	4973	ipt.pt._qproc = io_poll_queue_proc;
4667	4974
4668	4975	mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events,
@@ -4689,15 +4996,16 @@ static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) · 20 unmodified lines
4689	4996	struct io_ring_ctx *ctx = req->ctx;
4690	4997	unsigned long flags;
4691	4998
4692		− atomic_inc(&ctx->cq_timeouts);
4693		−
4694	4999	spin_lock_irqsave(&ctx->completion_lock, flags);
	5000	+ atomic_set(&req->ctx->cq_timeouts,
	5001	+ atomic_read(&req->ctx->cq_timeouts) + 1);
	5002	+
4695	5003	/*
4696	5004	* We could be racing with timeout deletion. If the list is empty,
4697	5005	* then timeout lookup already found it and will be handling it.
4698	5006	*/
4699		− if (!list_empty(&req->list))
4700		− list_del_init(&req->list);
	5007	+ if (!list_empty(&req->timeout.list))
	5008	+ list_del_init(&req->timeout.list);
4701	5009
4702	5010	io_cqring_fill_event(req, -ETIME);
4703	5011	io_commit_cqring(ctx);
@@ -4714,9 +5022,9 @@ static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data) · 10 unmodified lines
4714	5022	struct io_kiocb *req;
4715	5023	int ret = -ENOENT;
4716	5024
4717		− list_for_each_entry(req, &ctx->timeout_list, list) {
	5025	+ list_for_each_entry(req, &ctx->timeout_list, timeout.list) {
4718	5026	if (user_data == req->user_data) {
4719		− list_del_init(&req->list);
	5027	+ list_del_init(&req->timeout.list);
4720	5028	ret = 0;
4721	5029	break;
4722	5030	}
@@ -4798,7 +5106,6 @@ static int io_timeout_prep(struct io_kiocb req, const struct io_uring_sqe sqe, · 75 unmodified lines
4798	5106
4799	5107	data = &req->io->timeout;
4800	5108	data->req = req;
4801		− req->flags \|= REQ_F_TIMEOUT;
4802	5109
4803	5110	if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
4804	5111	return -EFAULT;
@@ -4826,8 +5133,7 @@ static int io_timeout(struct io_kiocb *req) · 21 unmodified lines
4826	5133	* timeout event to be satisfied. If it isn't set, then this is
4827	5134	* a pure timeout request, sequence isn't used.
4828	5135	*/
4829		− if (!off) {
4830		− req->flags \|= REQ_F_TIMEOUT_NOSEQ;
	5136	+ if (io_is_timeout_noseq(req)) {
4831	5137	entry = ctx->timeout_list.prev;
4832	5138	goto add;
4833	5139	}
@@ -4840,16 +5146,17 @@ static int io_timeout(struct io_kiocb *req) · 6 unmodified lines
4840	5146	* the one we need first.
4841	5147	*/
4842	5148	list_for_each_prev(entry, &ctx->timeout_list) {
4843		− struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, list);
	5149	+ struct io_kiocb *nxt = list_entry(entry, struct io_kiocb,
	5150	+ timeout.list);
4844	5151
4845		− if (nxt->flags & REQ_F_TIMEOUT_NOSEQ)
	5152	+ if (io_is_timeout_noseq(nxt))
4846	5153	continue;
4847	5154	/* nxt.seq is behind @tail, otherwise would've been completed */
4848	5155	if (off >= nxt->timeout.target_seq - tail)
4849	5156	break;
4850	5157	}
4851	5158	add:
4852		− list_add(&req->list, entry);
	5159	+ list_add(&req->timeout.list, entry);
4853	5160	data->timer.function = io_timeout_fn;
4854	5161	hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
4855	5162	spin_unlock_irq(&ctx->completion_lock);
@@ -4953,7 +5260,8 @@ static int io_files_update_prep(struct io_kiocb *req, · 97 unmodified lines
4953	5260	return 0;
4954	5261	}
4955	5262
4956		−static int io_files_update(struct io_kiocb *req, bool force_nonblock)
	5263	+static int io_files_update(struct io_kiocb *req, bool force_nonblock,
	5264	+ struct io_comp_state *cs)
4957	5265	{
4958	5266	struct io_ring_ctx *ctx = req->ctx;
4959	5267	struct io_uring_files_update up;
@@ -4971,8 +5279,7 @@ static int io_files_update(struct io_kiocb *req, bool force_nonblock) · 11 unmodified lines
4971	5279
4972	5280	if (ret < 0)
4973	5281	req_set_fail_links(req);
4974		− io_cqring_add_event(req, ret);
4975		− io_put_req(req);
	5282	+ __io_req_complete(req, ret, 0, cs);
4976	5283	return 0;
4977	5284	}
4978	5285
@@ -4984,15 +5291,11 @@ static int io_req_defer_prep(struct io_kiocb *req, · 5 unmodified lines
4984	5291	if (!sqe)
4985	5292	return 0;
4986	5293
4987		− io_req_init_async(req);
4988		−
4989		− if (io_op_defs[req->opcode].file_table) {
4990		− ret = io_grab_files(req);
4991		− if (unlikely(ret))
4992		− return ret;
4993		− }
4994		−
4995		− io_req_work_grab_env(req, &io_op_defs[req->opcode]);
	5294	+ if (io_alloc_async_ctx(req))
	5295	+ return -EAGAIN;
	5296	+ ret = io_prep_work_files(req);
	5297	+ if (unlikely(ret))
	5298	+ return ret;
4996	5299
4997	5300	switch (req->opcode) {
4998	5301	case IORING_OP_NOP:
@@ -5094,86 +5397,117 @@ static int io_req_defer_prep(struct io_kiocb *req, · 95 unmodified lines
5094	5397	return ret;
5095	5398	}
5096	5399
	5400	+static u32 io_get_sequence(struct io_kiocb *req)
	5401	+{
	5402	+ struct io_kiocb *pos;
	5403	+ struct io_ring_ctx *ctx = req->ctx;
	5404	+ u32 total_submitted, nr_reqs = 1;
	5405	+
	5406	+ if (req->flags & REQ_F_LINK_HEAD)
	5407	+ list_for_each_entry(pos, &req->link_list, link_list)
	5408	+ nr_reqs++;
	5409	+
	5410	+ total_submitted = ctx->cached_sq_head - ctx->cached_sq_dropped;
	5411	+ return total_submitted - nr_reqs;
	5412	+}
	5413	+
5097	5414	static int io_req_defer(struct io_kiocb req, const struct io_uring_sqe sqe)
5098	5415	{
5099	5416	struct io_ring_ctx *ctx = req->ctx;
	5417	+ struct io_defer_entry *de;
5100	5418	int ret;
	5419	+ u32 seq;
5101	5420
5102	5421	/* Still need defer if there is pending req in defer list. */
5103		− if (!req_need_defer(req) && list_empty_careful(&ctx->defer_list))
	5422	+ if (likely(list_empty_careful(&ctx->defer_list) &&
	5423	+ !(req->flags & REQ_F_IO_DRAIN)))
	5424	+ return 0;
	5425	+
	5426	+ seq = io_get_sequence(req);
	5427	+ /* Still a chance to pass the sequence check */
	5428	+ if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list))
5104	5429	return 0;
5105	5430
5106	5431	if (!req->io) {
5107		− if (io_alloc_async_ctx(req))
5108		− return -EAGAIN;
5109	5432	ret = io_req_defer_prep(req, sqe);
5110		− if (ret < 0)
	5433	+ if (ret)
5111	5434	return ret;
5112	5435	}
	5436	+ io_prep_async_link(req);
	5437	+ de = kmalloc(sizeof(*de), GFP_KERNEL);
	5438	+ if (!de)
	5439	+ return -ENOMEM;
5113	5440
5114	5441	spin_lock_irq(&ctx->completion_lock);
5115		− if (!req_need_defer(req) && list_empty(&ctx->defer_list)) {
	5442	+ if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
5116	5443	spin_unlock_irq(&ctx->completion_lock);
5117		− return 0;
	5444	+ kfree(de);
	5445	+ io_queue_async_work(req);
	5446	+ return -EIOCBQUEUED;
5118	5447	}
5119	5448
5120	5449	trace_io_uring_defer(ctx, req, req->user_data);
5121		− list_add_tail(&req->list, &ctx->defer_list);
	5450	+ de->req = req;
	5451	+ de->seq = seq;
	5452	+ list_add_tail(&de->list, &ctx->defer_list);
5122	5453	spin_unlock_irq(&ctx->completion_lock);
5123	5454	return -EIOCBQUEUED;
5124	5455	}
5125	5456
5126		−static void io_cleanup_req(struct io_kiocb *req)
	5457	+static void __io_clean_op(struct io_kiocb *req)
5127	5458	{
5128	5459	struct io_async_ctx *io = req->io;
5129	5460
5130		− switch (req->opcode) {
5131		− case IORING_OP_READV:
5132		− case IORING_OP_READ_FIXED:
5133		− case IORING_OP_READ:
5134		− if (req->flags & REQ_F_BUFFER_SELECTED)
	5461	+ if (req->flags & REQ_F_BUFFER_SELECTED) {
	5462	+ switch (req->opcode) {
	5463	+ case IORING_OP_READV:
	5464	+ case IORING_OP_READ_FIXED:
	5465	+ case IORING_OP_READ:
5135	5466	kfree((void *)(unsigned long)req->rw.addr);
5136		− /* fallthrough */
5137		− case IORING_OP_WRITEV:
5138		− case IORING_OP_WRITE_FIXED:
5139		− case IORING_OP_WRITE:
5140		− if (io->rw.iov != io->rw.fast_iov)
5141		− kfree(io->rw.iov);
5142		− break;
5143		− case IORING_OP_RECVMSG:
5144		− if (req->flags & REQ_F_BUFFER_SELECTED)
5145		− kfree(req->sr_msg.kbuf);
5146		− /* fallthrough */
5147		− case IORING_OP_SENDMSG:
5148		− if (io->msg.iov != io->msg.fast_iov)
5149		− kfree(io->msg.iov);
5150		− break;
5151		− case IORING_OP_RECV:
5152		− if (req->flags & REQ_F_BUFFER_SELECTED)
	5467	+ break;
	5468	+ case IORING_OP_RECVMSG:
	5469	+ case IORING_OP_RECV:
5153	5470	kfree(req->sr_msg.kbuf);
5154		− break;
5155		− case IORING_OP_OPENAT:
5156		− case IORING_OP_OPENAT2:
5157		− break;
5158		− case IORING_OP_SPLICE:
5159		− case IORING_OP_TEE:
5160		− io_put_file(req, req->splice.file_in,
5161		− (req->splice.flags & SPLICE_F_FD_IN_FIXED));
5162		− break;
	5471	+ break;
	5472	+ }
	5473	+ req->flags &= ~REQ_F_BUFFER_SELECTED;
	5474	+ }
	5475	+
	5476	+ if (req->flags & REQ_F_NEED_CLEANUP) {
	5477	+ switch (req->opcode) {
	5478	+ case IORING_OP_READV:
	5479	+ case IORING_OP_READ_FIXED:
	5480	+ case IORING_OP_READ:
	5481	+ case IORING_OP_WRITEV:
	5482	+ case IORING_OP_WRITE_FIXED:
	5483	+ case IORING_OP_WRITE:
	5484	+ if (io->rw.iov != io->rw.fast_iov)
	5485	+ kfree(io->rw.iov);
	5486	+ break;
	5487	+ case IORING_OP_RECVMSG:
	5488	+ case IORING_OP_SENDMSG:
	5489	+ if (io->msg.iov != io->msg.fast_iov)
	5490	+ kfree(io->msg.iov);
	5491	+ break;
	5492	+ case IORING_OP_SPLICE:
	5493	+ case IORING_OP_TEE:
	5494	+ io_put_file(req, req->splice.file_in,
	5495	+ (req->splice.flags & SPLICE_F_FD_IN_FIXED));
	5496	+ break;
	5497	+ }
	5498	+ req->flags &= ~REQ_F_NEED_CLEANUP;
5163	5499	}
5164		−
5165		− req->flags &= ~REQ_F_NEED_CLEANUP;
5166	5500	}
5167	5501
5168	5502	static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe,
5169		− bool force_nonblock)
	5503	+ bool force_nonblock, struct io_comp_state *cs)
5170	5504	{
5171	5505	struct io_ring_ctx *ctx = req->ctx;
5172	5506	int ret;
5173	5507
5174	5508	switch (req->opcode) {
5175	5509	case IORING_OP_NOP:
5176		− ret = io_nop(req);
	5510	+ ret = io_nop(req, cs);
5177	5511	break;
5178	5512	case IORING_OP_READV:
5179	5513	case IORING_OP_READ_FIXED:
@@ -5183,7 +5517,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 3 unmodified lines
5183	5517	if (ret < 0)
5184	5518	break;
5185	5519	}
5186		− ret = io_read(req, force_nonblock);
	5520	+ ret = io_read(req, force_nonblock, cs);
5187	5521	break;
5188	5522	case IORING_OP_WRITEV:
5189	5523	case IORING_OP_WRITE_FIXED:
@@ -5193,7 +5527,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 3 unmodified lines
5193	5527	if (ret < 0)
5194	5528	break;
5195	5529	}
5196		− ret = io_write(req, force_nonblock);
	5530	+ ret = io_write(req, force_nonblock, cs);
5197	5531	break;
5198	5532	case IORING_OP_FSYNC:
5199	5533	if (sqe) {
@@ -5235,9 +5569,9 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 35 unmodified lines
5235	5569	break;
5236	5570	}
5237	5571	if (req->opcode == IORING_OP_SENDMSG)
5238		− ret = io_sendmsg(req, force_nonblock);
	5572	+ ret = io_sendmsg(req, force_nonblock, cs);
5239	5573	else
5240		− ret = io_send(req, force_nonblock);
	5574	+ ret = io_send(req, force_nonblock, cs);
5241	5575	break;
5242	5576	case IORING_OP_RECVMSG:
5243	5577	case IORING_OP_RECV:
@@ -5247,9 +5581,9 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 3 unmodified lines
5247	5581	break;
5248	5582	}
5249	5583	if (req->opcode == IORING_OP_RECVMSG)
5250		− ret = io_recvmsg(req, force_nonblock);
	5584	+ ret = io_recvmsg(req, force_nonblock, cs);
5251	5585	else
5252		− ret = io_recv(req, force_nonblock);
	5586	+ ret = io_recv(req, force_nonblock, cs);
5253	5587	break;
5254	5588	case IORING_OP_TIMEOUT:
5255	5589	if (sqe) {
@@ -5273,7 +5607,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 17 unmodified lines
5273	5607	if (ret)
5274	5608	break;
5275	5609	}
5276		− ret = io_accept(req, force_nonblock);
	5610	+ ret = io_accept(req, force_nonblock, cs);
5277	5611	break;
5278	5612	case IORING_OP_CONNECT:
5279	5613	if (sqe) {
@@ -5281,7 +5615,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
5281	5615	if (ret)
5282	5616	break;
5283	5617	}
5284		− ret = io_connect(req, force_nonblock);
	5618	+ ret = io_connect(req, force_nonblock, cs);
5285	5619	break;
5286	5620	case IORING_OP_ASYNC_CANCEL:
5287	5621	if (sqe) {
@@ -5313,7 +5647,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 25 unmodified lines
5313	5647	if (ret)
5314	5648	break;
5315	5649	}
5316		− ret = io_close(req, force_nonblock);
	5650	+ ret = io_close(req, force_nonblock, cs);
5317	5651	break;
5318	5652	case IORING_OP_FILES_UPDATE:
5319	5653	if (sqe) {
@@ -5321,7 +5655,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
5321	5655	if (ret)
5322	5656	break;
5323	5657	}
5324		− ret = io_files_update(req, force_nonblock);
	5658	+ ret = io_files_update(req, force_nonblock, cs);
5325	5659	break;
5326	5660	case IORING_OP_STATX:
5327	5661	if (sqe) {
@@ -5361,7 +5695,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 33 unmodified lines
5361	5695	if (ret)
5362	5696	break;
5363	5697	}
5364		− ret = io_epoll_ctl(req, force_nonblock);
	5698	+ ret = io_epoll_ctl(req, force_nonblock, cs);
5365	5699	break;
5366	5700	case IORING_OP_SPLICE:
5367	5701	if (sqe) {
@@ -5377,7 +5711,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 9 unmodified lines
5377	5711	if (ret)
5378	5712	break;
5379	5713	}
5380		− ret = io_provide_buffers(req, force_nonblock);
	5714	+ ret = io_provide_buffers(req, force_nonblock, cs);
5381	5715	break;
5382	5716	case IORING_OP_REMOVE_BUFFERS:
5383	5717	if (sqe) {
@@ -5385,7 +5719,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
5385	5719	if (ret)
5386	5720	break;
5387	5721	}
5388		− ret = io_remove_buffers(req, force_nonblock);
	5722	+ ret = io_remove_buffers(req, force_nonblock, cs);
5389	5723	break;
5390	5724	case IORING_OP_TEE:
5391	5725	if (sqe) {
@@ -5420,25 +5754,15 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 28 unmodified lines
5420	5754	return 0;
5421	5755	}
5422	5756
5423		−static void io_arm_async_linked_timeout(struct io_kiocb *req)
5424		−{
5425		− struct io_kiocb *link;
5426		−
5427		− /* link head's timeout is queued in io_queue_async_work() */
5428		− if (!(req->flags & REQ_F_QUEUE_TIMEOUT))
5429		− return;
5430		−
5431		− link = list_first_entry(&req->link_list, struct io_kiocb, link_list);
5432		− io_queue_linked_timeout(link);
5433		−}
5434		−
5435		−static void io_wq_submit_work(struct io_wq_work **workptr)
	5757	+static struct io_wq_work io_wq_submit_work(struct io_wq_work work)
5436	5758	{
5437		− struct io_wq_work work = workptr;
5438	5759	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
	5760	+ struct io_kiocb *timeout;
5439	5761	int ret = 0;
5440	5762
5441		− io_arm_async_linked_timeout(req);
	5763	+ timeout = io_prep_linked_timeout(req);
	5764	+ if (timeout)
	5765	+ io_queue_linked_timeout(timeout);
5442	5766
5443	5767	/* if NO_CANCEL is set, we must still run the work */
5444	5768	if ((work->flags & (IO_WQ_WORK_CANCEL\|IO_WQ_WORK_NO_CANCEL)) ==
@@ -5448,7 +5772,7 @@ static void io_wq_submit_work(struct io_wq_work **workptr) · 3 unmodified lines
5448	5772
5449	5773	if (!ret) {
5450	5774	do {
5451		− ret = io_issue_sqe(req, NULL, false);
	5775	+ ret = io_issue_sqe(req, NULL, false, NULL);
5452	5776	/*
5453	5777	* We can get EAGAIN for polled IO even though we're
5454	5778	* forcing a sync submission from here, since we can't
@@ -5462,11 +5786,10 @@ static void io_wq_submit_work(struct io_wq_work **workptr) · 7 unmodified lines
5462	5786
5463	5787	if (ret) {
5464	5788	req_set_fail_links(req);
5465		− io_cqring_add_event(req, ret);
5466		− io_put_req(req);
	5789	+ io_req_complete(req, ret);
5467	5790	}
5468	5791
5469		− io_steal_work(req, workptr);
	5792	+ return io_steal_work(req);
5470	5793	}
5471	5794
5472	5795	static inline struct file io_file_from_index(struct io_ring_ctx ctx,
@@ -5523,6 +5846,8 @@ static int io_grab_files(struct io_kiocb *req) · 50 unmodified lines
5523	5846	int ret = -EBADF;
5524	5847	struct io_ring_ctx *ctx = req->ctx;
5525	5848
	5849	+ io_req_init_async(req);
	5850	+
5526	5851	if (req->work.files \|\| (req->flags & REQ_F_NO_FILE_TABLE))
5527	5852	return 0;
5528	5853	if (!ctx->ring_file)
@@ -5548,6 +5873,13 @@ static int io_grab_files(struct io_kiocb *req) · 19 unmodified lines
5548	5873	return ret;
5549	5874	}
5550	5875
	5876	+static inline int io_prep_work_files(struct io_kiocb *req)
	5877	+{
	5878	+ if (!io_op_defs[req->opcode].file_table)
	5879	+ return 0;
	5880	+ return io_grab_files(req);
	5881	+}
	5882	+
5551	5883	static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
5552	5884	{
5553	5885	struct io_timeout_data *data = container_of(timer,
@@ -5580,8 +5912,7 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) · 26 unmodified lines
5580	5912	io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME);
5581	5913	io_put_req(prev);
5582	5914	} else {
5583		− io_cqring_add_event(req, -ETIME);
5584		− io_put_req(req);
	5915	+ io_req_complete(req, -ETIME);
5585	5916	}
5586	5917	return HRTIMER_NORESTART;
5587	5918	}
@@ -5614,8 +5945,7 @@ static struct io_kiocb io_prep_linked_timeout(struct io_kiocb req) · 26 unmodified lines
5614	5945
5615	5946	if (!(req->flags & REQ_F_LINK_HEAD))
5616	5947	return NULL;
5617		− /* for polled retry, if flag is set, we already went through here */
5618		− if (req->flags & REQ_F_POLLED)
	5948	+ if (req->flags & REQ_F_LINK_TIMEOUT)
5619	5949	return NULL;
5620	5950
5621	5951	nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,
@@ -5627,7 +5957,8 @@ static struct io_kiocb io_prep_linked_timeout(struct io_kiocb req) · 5 unmodified lines
5627	5957	return nxt;
5628	5958	}
5629	5959
5630		−static void __io_queue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe)
	5960	+static void __io_queue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe,
	5961	+ struct io_comp_state *cs)
5631	5962	{
5632	5963	struct io_kiocb *linked_timeout;
5633	5964	struct io_kiocb *nxt;
@@ -5647,54 +5978,45 @@ again: · 13 unmodified lines
5647	5978	old_creds = override_creds(req->work.creds);
5648	5979	}
5649	5980
5650		− ret = io_issue_sqe(req, sqe, true);
	5981	+ ret = io_issue_sqe(req, sqe, true, cs);
5651	5982
5652	5983	/*
5653	5984	* We async punt it if the file wasn't marked NOWAIT, or if the file
5654	5985	* doesn't support non-blocking read/write attempts
5655	5986	*/
5656		− if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) \|\|
5657		− (req->flags & REQ_F_MUST_PUNT))) {
5658		− if (io_arm_poll_handler(req)) {
5659		− if (linked_timeout)
5660		− io_queue_linked_timeout(linked_timeout);
5661		− goto exit;
5662		− }
	5987	+ if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
	5988	+ if (!io_arm_poll_handler(req)) {
5663	5989	punt:
5664		− io_req_init_async(req);
5665		−
5666		− if (io_op_defs[req->opcode].file_table) {
5667		− ret = io_grab_files(req);
5668		− if (ret)
	5990	+ ret = io_prep_work_files(req);
	5991	+ if (unlikely(ret))
5669	5992	goto err;
	5993	+ /*
	5994	+ * Queued up for async execution, worker will release
	5995	+ * submit reference when the iocb is actually submitted.
	5996	+ */
	5997	+ io_queue_async_work(req);
5670	5998	}
5671	5999
5672		− /*
5673		− * Queued up for async execution, worker will release
5674		− * submit reference when the iocb is actually submitted.
5675		− */
5676		− io_queue_async_work(req);
	6000	+ if (linked_timeout)
	6001	+ io_queue_linked_timeout(linked_timeout);
5677	6002	goto exit;
5678	6003	}
5679	6004
	6005	+ if (unlikely(ret)) {
5680	6006	err:
5681		− nxt = NULL;
5682		− /* drop submission reference */
5683		− io_put_req_find_next(req, &nxt);
5684		−
5685		− if (linked_timeout) {
5686		− if (!ret)
5687		− io_queue_linked_timeout(linked_timeout);
5688		− else
5689		− io_put_req(linked_timeout);
5690		− }
5691		−
5692		− /* and drop final reference, if we failed */
5693		− if (ret) {
5694		− io_cqring_add_event(req, ret);
	6007	+ /* un-prep timeout, so it'll be killed as any other linked */
	6008	+ req->flags &= ~REQ_F_LINK_TIMEOUT;
5695	6009	req_set_fail_links(req);
5696	6010	io_put_req(req);
	6011	+ io_req_complete(req, ret);
	6012	+ goto exit;
5697	6013	}
	6014	+
	6015	+ /* drop submission reference */
	6016	+ nxt = io_put_req_find_next(req);
	6017	+ if (linked_timeout)
	6018	+ io_queue_linked_timeout(linked_timeout);
	6019	+
5698	6020	if (nxt) {
5699	6021	req = nxt;
5700	6022
@@ -5707,7 +6029,8 @@ exit: · 6 unmodified lines
5707	6029	revert_creds(old_creds);
5708	6030	}
5709	6031
5710		−static void io_queue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe)
	6032	+static void io_queue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe,
	6033	+ struct io_comp_state *cs)
5711	6034	{
5712	6035	int ret;
5713	6036
@@ -5715,17 +6038,14 @@ static void io_queue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe) · 1 unmodified line
5715	6038	if (ret) {
5716	6039	if (ret != -EIOCBQUEUED) {
5717	6040	fail_req:
5718		− io_cqring_add_event(req, ret);
5719	6041	req_set_fail_links(req);
5720		− io_double_put_req(req);
	6042	+ io_put_req(req);
	6043	+ io_req_complete(req, ret);
5721	6044	}
5722	6045	} else if (req->flags & REQ_F_FORCE_ASYNC) {
5723	6046	if (!req->io) {
5724		− ret = -EAGAIN;
5725		− if (io_alloc_async_ctx(req))
5726		− goto fail_req;
5727	6047	ret = io_req_defer_prep(req, sqe);
5728		− if (unlikely(ret < 0))
	6048	+ if (unlikely(ret))
5729	6049	goto fail_req;
5730	6050	}
5731	6051
@@ -5737,21 +6057,22 @@ fail_req: · 5 unmodified lines
5737	6057	req->work.flags \|= IO_WQ_WORK_CONCURRENT;
5738	6058	io_queue_async_work(req);
5739	6059	} else {
5740		− __io_queue_sqe(req, sqe);
	6060	+ __io_queue_sqe(req, sqe, cs);
5741	6061	}
5742	6062	}
5743	6063
5744		−static inline void io_queue_link_head(struct io_kiocb *req)
	6064	+static inline void io_queue_link_head(struct io_kiocb *req,
	6065	+ struct io_comp_state *cs)
5745	6066	{
5746	6067	if (unlikely(req->flags & REQ_F_FAIL_LINK)) {
5747		− io_cqring_add_event(req, -ECANCELED);
5748		− io_double_put_req(req);
	6068	+ io_put_req(req);
	6069	+ io_req_complete(req, -ECANCELED);
5749	6070	} else
5750		− io_queue_sqe(req, NULL);
	6071	+ io_queue_sqe(req, NULL, cs);
5751	6072	}
5752	6073
5753	6074	static int io_submit_sqe(struct io_kiocb req, const struct io_uring_sqe sqe,
5754		− struct io_kiocb **link)
	6075	+ struct io_kiocb *link, struct io_comp_state cs)
5755	6076	{
5756	6077	struct io_ring_ctx *ctx = req->ctx;
5757	6078	int ret;
@@ -5777,21 +6098,19 @@ static int io_submit_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 19 unmodified lines
5777	6098	head->flags \|= REQ_F_IO_DRAIN;
5778	6099	ctx->drain_next = 1;
5779	6100	}
5780		− if (io_alloc_async_ctx(req))
5781		− return -EAGAIN;
5782		−
5783	6101	ret = io_req_defer_prep(req, sqe);
5784		− if (ret) {
	6102	+ if (unlikely(ret)) {
5785	6103	/* fail even hard links since we don't submit */
5786	6104	head->flags \|= REQ_F_FAIL_LINK;
5787	6105	return ret;
5788	6106	}
5789	6107	trace_io_uring_link(ctx, req, head);
	6108	+ io_get_req_task(req);
5790	6109	list_add_tail(&req->link_list, &head->link_list);
5791	6110
5792	6111	/* last request of a link, enqueue the link */
5793	6112	if (!(req->flags & (REQ_F_LINK \| REQ_F_HARDLINK))) {
5794		− io_queue_link_head(head);
	6113	+ io_queue_link_head(head, cs);
5795	6114	*link = NULL;
5796	6115	}
5797	6116	} else {
@@ -5803,15 +6122,12 @@ static int io_submit_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 5 unmodified lines
5803	6122	req->flags \|= REQ_F_LINK_HEAD;
5804	6123	INIT_LIST_HEAD(&req->link_list);
5805	6124
5806		− if (io_alloc_async_ctx(req))
5807		− return -EAGAIN;
5808		−
5809	6125	ret = io_req_defer_prep(req, sqe);
5810		− if (ret)
	6126	+ if (unlikely(ret))
5811	6127	req->flags \|= REQ_F_FAIL_LINK;
5812	6128	*link = req;
5813	6129	} else {
5814		− io_queue_sqe(req, sqe);
	6130	+ io_queue_sqe(req, sqe, cs);
5815	6131	}
5816	6132	}
5817	6133
@@ -5823,6 +6139,8 @@ static int io_submit_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 5 unmodified lines
5823	6139	*/
5824	6140	static void io_submit_state_end(struct io_submit_state *state)
5825	6141	{
	6142	+ if (!list_empty(&state->comp.list))
	6143	+ io_submit_flush_completions(&state->comp);
5826	6144	blk_finish_plug(&state->plug);
5827	6145	io_state_file_put(state);
5828	6146	if (state->free_reqs)
@@ -5833,9 +6151,15 @@ static void io_submit_state_end(struct io_submit_state *state) · 4 unmodified lines
5833	6151	* Start submission side cache.
5834	6152	*/
5835	6153	static void io_submit_state_start(struct io_submit_state *state,
5836		− unsigned int max_ios)
	6154	+ struct io_ring_ctx *ctx, unsigned int max_ios)
5837	6155	{
5838	6156	blk_start_plug(&state->plug);
	6157	+#ifdef CONFIG_BLOCK
	6158	+ state->plug.nowait = true;
	6159	+#endif
	6160	+ state->comp.nr = 0;
	6161	+ INIT_LIST_HEAD(&state->comp.list);
	6162	+ state->comp.ctx = ctx;
5839	6163	state->free_reqs = 0;
5840	6164	state->file = NULL;
5841	6165	state->ios_left = max_ios;
@@ -5900,12 +6224,6 @@ static int io_init_req(struct io_ring_ctx ctx, struct io_kiocb req, · 58 unmodified lines
5900	6224	unsigned int sqe_flags;
5901	6225	int id;
5902	6226
5903		− /*
5904		− * All io need record the previous position, if LINK vs DARIN,
5905		− * it can be used to mark the position of the first IO in the
5906		− * link list.
5907		− */
5908		− req->sequence = ctx->cached_sq_head - ctx->cached_sq_dropped;
5909	6227	req->opcode = READ_ONCE(sqe->opcode);
5910	6228	req->user_data = READ_ONCE(sqe->user_data);
5911	6229	req->io = NULL;
@@ -5953,7 +6271,7 @@ static int io_init_req(struct io_ring_ctx ctx, struct io_kiocb req, · 41 unmodified lines
5953	6271	static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
5954	6272	struct file *ring_file, int ring_fd)
5955	6273	{
5956		− struct io_submit_state state, *statep = NULL;
	6274	+ struct io_submit_state state;
5957	6275	struct io_kiocb *link = NULL;
5958	6276	int i, submitted = 0;
5959	6277
@@ -5970,10 +6288,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr, · 10 unmodified lines
5970	6288	if (!percpu_ref_tryget_many(&ctx->refs, nr))
5971	6289	return -EAGAIN;
5972	6290
5973		− if (nr > IO_PLUG_THRESHOLD) {
5974		− io_submit_state_start(&state, nr);
5975		− statep = &state;
5976		− }
	6291	+ io_submit_state_start(&state, ctx, nr);
5977	6292
5978	6293	ctx->ring_fd = ring_fd;
5979	6294	ctx->ring_file = ring_file;
@@ -5988,28 +6303,28 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr, · 8 unmodified lines
5988	6303	io_consume_sqe(ctx);
5989	6304	break;
5990	6305	}
5991		− req = io_alloc_req(ctx, statep);
	6306	+ req = io_alloc_req(ctx, &state);
5992	6307	if (unlikely(!req)) {
5993	6308	if (!submitted)
5994	6309	submitted = -EAGAIN;
5995	6310	break;
5996	6311	}
5997	6312
5998		− err = io_init_req(ctx, req, sqe, statep);
	6313	+ err = io_init_req(ctx, req, sqe, &state);
5999	6314	io_consume_sqe(ctx);
6000	6315	/* will complete beyond this point, count as submitted */
6001	6316	submitted++;
6002	6317
6003	6318	if (unlikely(err)) {
6004	6319	fail_req:
6005		− io_cqring_add_event(req, err);
6006		− io_double_put_req(req);
	6320	+ io_put_req(req);
	6321	+ io_req_complete(req, err);
6007	6322	break;
6008	6323	}
6009	6324
6010	6325	trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
6011	6326	true, io_async_submit(ctx));
6012		− err = io_submit_sqe(req, sqe, &link);
	6327	+ err = io_submit_sqe(req, sqe, &link, &state.comp);
6013	6328	if (err)
6014	6329	goto fail_req;
6015	6330	}
@@ -6020,9 +6335,8 @@ fail_req: · 4 unmodified lines
6020	6335	percpu_ref_put_many(&ctx->refs, nr - ref_used);
6021	6336	}
6022	6337	if (link)
6023		− io_queue_link_head(link);
6024		− if (statep)
6025		− io_submit_state_end(&state);
	6338	+ io_queue_link_head(link, &state.comp);
	6339	+ io_submit_state_end(&state);
6026	6340
6027	6341	/* Commit SQ ring head once we've consumed and submitted all SQEs */
6028	6342	io_commit_sqring(ctx);
@@ -6030,6 +6344,21 @@ fail_req: · 1 unmodified line
6030	6344	return submitted;
6031	6345	}
6032	6346
	6347	+static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
	6348	+{
	6349	+ /* Tell userspace we may need a wakeup call */
	6350	+ spin_lock_irq(&ctx->completion_lock);
	6351	+ ctx->rings->sq_flags \|= IORING_SQ_NEED_WAKEUP;
	6352	+ spin_unlock_irq(&ctx->completion_lock);
	6353	+}
	6354	+
	6355	+static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
	6356	+{
	6357	+ spin_lock_irq(&ctx->completion_lock);
	6358	+ ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
	6359	+ spin_unlock_irq(&ctx->completion_lock);
	6360	+}
	6361	+
6033	6362	static int io_sq_thread(void *data)
6034	6363	{
6035	6364	struct io_ring_ctx *ctx = data;
@@ -6046,12 +6375,12 @@ static int io_sq_thread(void *data) · 10 unmodified lines
6046	6375	while (!kthread_should_park()) {
6047	6376	unsigned int to_submit;
6048	6377
6049		− if (!list_empty(&ctx->poll_list)) {
	6378	+ if (!list_empty(&ctx->iopoll_list)) {
6050	6379	unsigned nr_events = 0;
6051	6380
6052	6381	mutex_lock(&ctx->uring_lock);
6053		− if (!list_empty(&ctx->poll_list))
6054		− io_iopoll_getevents(ctx, &nr_events, 0);
	6382	+ if (!list_empty(&ctx->iopoll_list) && !need_resched())
	6383	+ io_do_iopoll(ctx, &nr_events, 0);
6055	6384	else
6056	6385	timeout = jiffies + ctx->sq_thread_idle;
6057	6386	mutex_unlock(&ctx->uring_lock);
@@ -6070,7 +6399,7 @@ static int io_sq_thread(void *data) · 12 unmodified lines
6070	6399	* adding ourselves to the waitqueue, as the unuse/drop
6071	6400	* may sleep.
6072	6401	*/
6073		− io_sq_thread_drop_mm(ctx);
	6402	+ io_sq_thread_drop_mm();
6074	6403
6075	6404	/*
6076	6405	* We're polling. If we're within the defined idle
@@ -6079,11 +6408,10 @@ static int io_sq_thread(void *data) · 2 unmodified lines
6079	6408	* more IO, we should wait for the application to
6080	6409	* reap events and wake us up.
6081	6410	*/
6082		− if (!list_empty(&ctx->poll_list) \|\| need_resched() \|\|
	6411	+ if (!list_empty(&ctx->iopoll_list) \|\| need_resched() \|\|
6083	6412	(!time_after(jiffies, timeout) && ret != -EBUSY &&
6084	6413	!percpu_ref_is_dying(&ctx->refs))) {
6085		− if (current->task_works)
6086		− task_work_run();
	6414	+ io_run_task_work();
6087	6415	cond_resched();
6088	6416	continue;
6089	6417	}
@@ -6093,21 +6421,18 @@ static int io_sq_thread(void *data) · 3 unmodified lines
6093	6421
6094	6422	/*
6095	6423	* While doing polled IO, before going to sleep, we need
6096		− * to check if there are new reqs added to poll_list, it
6097		− * is because reqs may have been punted to io worker and
6098		− * will be added to poll_list later, hence check the
6099		− * poll_list again.
	6424	+ * to check if there are new reqs added to iopoll_list,
	6425	+ * it is because reqs may have been punted to io worker
	6426	+ * and will be added to iopoll_list later, hence check
	6427	+ * the iopoll_list again.
6100	6428	*/
6101	6429	if ((ctx->flags & IORING_SETUP_IOPOLL) &&
6102		− !list_empty_careful(&ctx->poll_list)) {
	6430	+ !list_empty_careful(&ctx->iopoll_list)) {
6103	6431	finish_wait(&ctx->sqo_wait, &wait);
6104	6432	continue;
6105	6433	}
6106	6434
6107		− /* Tell userspace we may need a wakeup call */
6108		− spin_lock_irq(&ctx->completion_lock);
6109		− ctx->rings->sq_flags \|= IORING_SQ_NEED_WAKEUP;
6110		− spin_unlock_irq(&ctx->completion_lock);
	6435	+ io_ring_set_wakeup_flag(ctx);
6111	6436
6112	6437	to_submit = io_sqring_entries(ctx);
6113	6438	if (!to_submit \|\| ret == -EBUSY) {
@@ -6115,9 +6440,9 @@ static int io_sq_thread(void *data) · 1 unmodified line
6115	6440	finish_wait(&ctx->sqo_wait, &wait);
6116	6441	break;
6117	6442	}
6118		− if (current->task_works) {
6119		− task_work_run();
	6443	+ if (io_run_task_work()) {
6120	6444	finish_wait(&ctx->sqo_wait, &wait);
	6445	+ io_ring_clear_wakeup_flag(ctx);
6121	6446	continue;
6122	6447	}
6123	6448	if (signal_pending(current))
@@ -6125,17 +6450,13 @@ static int io_sq_thread(void *data) · 1 unmodified line
6125	6450	schedule();
6126	6451	finish_wait(&ctx->sqo_wait, &wait);
6127	6452
6128		− spin_lock_irq(&ctx->completion_lock);
6129		− ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
6130		− spin_unlock_irq(&ctx->completion_lock);
	6453	+ io_ring_clear_wakeup_flag(ctx);
6131	6454	ret = 0;
6132	6455	continue;
6133	6456	}
6134	6457	finish_wait(&ctx->sqo_wait, &wait);
6135	6458
6136		− spin_lock_irq(&ctx->completion_lock);
6137		− ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
6138		− spin_unlock_irq(&ctx->completion_lock);
	6459	+ io_ring_clear_wakeup_flag(ctx);
6139	6460	}
6140	6461
6141	6462	mutex_lock(&ctx->uring_lock);
@@ -6145,10 +6466,9 @@ static int io_sq_thread(void *data) · 3 unmodified lines
6145	6466	timeout = jiffies + ctx->sq_thread_idle;
6146	6467	}
6147	6468
6148		− if (current->task_works)
6149		− task_work_run();
	6469	+ io_run_task_work();
6150	6470
6151		− io_sq_thread_drop_mm(ctx);
	6471	+ io_sq_thread_drop_mm();
6152	6472	revert_creds(old_cred);
6153	6473
6154	6474	kthread_parkme();
@@ -6211,9 +6531,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 56 unmodified lines
6211	6531	do {
6212	6532	if (io_cqring_events(ctx, false) >= min_events)
6213	6533	return 0;
6214		− if (!current->task_works)
	6534	+ if (!io_run_task_work())
6215	6535	break;
6216		− task_work_run();
6217	6536	} while (1);
6218	6537
6219	6538	if (sig) {
@@ -6235,8 +6554,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 15 unmodified lines
6235	6554	prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
6236	6555	TASK_INTERRUPTIBLE);
6237	6556	/* make sure we run task_work before checking for signals */
6238		− if (current->task_works)
6239		− task_work_run();
	6557	+ if (io_run_task_work())
	6558	+ continue;
6240	6559	if (signal_pending(current)) {
6241	6560	if (current->jobctl & JOBCTL_TASK_WORK) {
6242	6561	spin_lock_irq(&current->sighand->siglock);
@@ -7022,17 +7341,21 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx, · 779 unmodified lines
7022	7341	return 0;
7023	7342	err:
7024	7343	io_finish_async(ctx);
7025		− mmdrop(ctx->sqo_mm);
7026		− ctx->sqo_mm = NULL;
	7344	+ if (ctx->sqo_mm) {
	7345	+ mmdrop(ctx->sqo_mm);
	7346	+ ctx->sqo_mm = NULL;
	7347	+ }
7027	7348	return ret;
7028	7349	}
7029	7350
7030		−static void io_unaccount_mem(struct user_struct *user, unsigned long nr_pages)
	7351	+static inline void __io_unaccount_mem(struct user_struct *user,
	7352	+ unsigned long nr_pages)
7031	7353	{
7032	7354	atomic_long_sub(nr_pages, &user->locked_vm);
7033	7355	}
7034	7356
7035		−static int io_account_mem(struct user_struct *user, unsigned long nr_pages)
	7357	+static inline int __io_account_mem(struct user_struct *user,
	7358	+ unsigned long nr_pages)
7036	7359	{
7037	7360	unsigned long page_limit, cur_pages, new_pages;
7038	7361
@@ -7050,6 +7373,41 @@ static int io_account_mem(struct user_struct *user, unsigned long nr_pages) · 11 unmodified lines
7050	7373	return 0;
7051	7374	}
7052	7375
	7376	+static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
	7377	+ enum io_mem_account acct)
	7378	+{
	7379	+ if (ctx->limit_mem)
	7380	+ __io_unaccount_mem(ctx->user, nr_pages);
	7381	+
	7382	+ if (ctx->sqo_mm) {
	7383	+ if (acct == ACCT_LOCKED)
	7384	+ ctx->sqo_mm->locked_vm -= nr_pages;
	7385	+ else if (acct == ACCT_PINNED)
	7386	+ atomic64_sub(nr_pages, &ctx->sqo_mm->pinned_vm);
	7387	+ }
	7388	+}
	7389	+
	7390	+static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
	7391	+ enum io_mem_account acct)
	7392	+{
	7393	+ int ret;
	7394	+
	7395	+ if (ctx->limit_mem) {
	7396	+ ret = __io_account_mem(ctx->user, nr_pages);
	7397	+ if (ret)
	7398	+ return ret;
	7399	+ }
	7400	+
	7401	+ if (ctx->sqo_mm) {
	7402	+ if (acct == ACCT_LOCKED)
	7403	+ ctx->sqo_mm->locked_vm += nr_pages;
	7404	+ else if (acct == ACCT_PINNED)
	7405	+ atomic64_add(nr_pages, &ctx->sqo_mm->pinned_vm);
	7406	+ }
	7407	+
	7408	+ return 0;
	7409	+}
	7410	+
7053	7411	static void io_mem_free(void *ptr)
7054	7412	{
7055	7413	struct page *page;
@@ -7086,6 +7444,9 @@ static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries, · 30 unmodified lines
7086	7444	return SIZE_MAX;
7087	7445	#endif
7088	7446
	7447	+ if (sq_offset)
	7448	+ *sq_offset = off;
	7449	+
7089	7450	sq_array_size = array_size(sizeof(u32), sq_entries);
7090	7451	if (sq_array_size == SIZE_MAX)
7091	7452	return SIZE_MAX;
@@ -7093,9 +7454,6 @@ static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries, · 1 unmodified line
7093	7454	if (check_add_overflow(off, sq_array_size, &off))
7094	7455	return SIZE_MAX;
7095	7456
7096		− if (sq_offset)
7097		− *sq_offset = off;
7098		−
7099	7457	return off;
7100	7458	}
7101	7459
@@ -7124,8 +7482,7 @@ static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx) · 22 unmodified lines
7124	7482	for (j = 0; j < imu->nr_bvecs; j++)
7125	7483	unpin_user_page(imu->bvec[j].bv_page);
7126	7484
7127		− if (ctx->account_mem)
7128		− io_unaccount_mem(ctx->user, imu->nr_bvecs);
	7485	+ io_unaccount_mem(ctx, imu->nr_bvecs, ACCT_PINNED);
7129	7486	kvfree(imu->bvec);
7130	7487	imu->nr_bvecs = 0;
7131	7488	}
@@ -7208,11 +7565,9 @@ static int io_sqe_buffer_register(struct io_ring_ctx ctx, void __user arg, · 76 unmodified lines
7208	7565	start = ubuf >> PAGE_SHIFT;
7209	7566	nr_pages = end - start;
7210	7567
7211		− if (ctx->account_mem) {
7212		− ret = io_account_mem(ctx->user, nr_pages);
7213		− if (ret)
7214		− goto err;
7215		− }
	7568	+ ret = io_account_mem(ctx, nr_pages, ACCT_PINNED);
	7569	+ if (ret)
	7570	+ goto err;
7216	7571
7217	7572	ret = 0;
7218	7573	if (!pages \|\| nr_pages > got_pages) {
@@ -7225,8 +7580,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx ctx, void __user arg, · 6 unmodified lines
7225	7580	GFP_KERNEL);
7226	7581	if (!pages \|\| !vmas) {
7227	7582	ret = -ENOMEM;
7228		− if (ctx->account_mem)
7229		− io_unaccount_mem(ctx->user, nr_pages);
	7583	+ io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
7230	7584	goto err;
7231	7585	}
7232	7586	got_pages = nr_pages;
@@ -7236,8 +7590,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx ctx, void __user arg, · 3 unmodified lines
7236	7590	GFP_KERNEL);
7237	7591	ret = -ENOMEM;
7238	7592	if (!imu->bvec) {
7239		− if (ctx->account_mem)
7240		− io_unaccount_mem(ctx->user, nr_pages);
	7593	+ io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
7241	7594	goto err;
7242	7595	}
7243	7596
@@ -7268,8 +7621,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx ctx, void __user arg, · 24 unmodified lines
7268	7621	*/
7269	7622	if (pret > 0)
7270	7623	unpin_user_pages(pages, pret);
7271		− if (ctx->account_mem)
7272		− io_unaccount_mem(ctx->user, nr_pages);
	7624	+ io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
7273	7625	kvfree(imu->bvec);
7274	7626	goto err;
7275	7627	}
@@ -7353,11 +7705,12 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx) · 77 unmodified lines
7353	7705	static void io_ring_ctx_free(struct io_ring_ctx *ctx)
7354	7706	{
7355	7707	io_finish_async(ctx);
7356		− if (ctx->sqo_mm)
	7708	+ io_sqe_buffer_unregister(ctx);
	7709	+ if (ctx->sqo_mm) {
7357	7710	mmdrop(ctx->sqo_mm);
	7711	+ ctx->sqo_mm = NULL;
	7712	+ }
7358	7713
7359		− io_iopoll_reap_events(ctx);
7360		− io_sqe_buffer_unregister(ctx);
7361	7714	io_sqe_files_unregister(ctx);
7362	7715	io_eventfd_unregister(ctx);
7363	7716	io_destroy_buffers(ctx);
@@ -7421,11 +7774,8 @@ static int io_remove_personalities(int id, void p, void data) · 57 unmodified lines
7421	7774
7422	7775	static void io_ring_exit_work(struct work_struct *work)
7423	7776	{
7424		− struct io_ring_ctx *ctx;
7425		−
7426		− ctx = container_of(work, struct io_ring_ctx, exit_work);
7427		− if (ctx->rings)
7428		− io_cqring_overflow_flush(ctx, true);
	7777	+ struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
	7778	+ exit_work);
7429	7779
7430	7780	/*
7431	7781	* If we're doing polled IO and end up having requests being
@@ -7433,11 +7783,11 @@ static void io_ring_exit_work(struct work_struct *work) · 1 unmodified line
7433	7783	* we're waiting for refs to drop. We need to reap these manually,
7434	7784	* as nobody else will be looking for them.
7435	7785	*/
7436		− while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20)) {
7437		− io_iopoll_reap_events(ctx);
	7786	+ do {
7438	7787	if (ctx->rings)
7439	7788	io_cqring_overflow_flush(ctx, true);
7440		− }
	7789	+ io_iopoll_try_reap_events(ctx);
	7790	+ } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20));
7441	7791	io_ring_ctx_free(ctx);
7442	7792	}
7443	7793
@@ -7453,10 +7803,10 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) · 9 unmodified lines
7453	7803	if (ctx->io_wq)
7454	7804	io_wq_cancel_all(ctx->io_wq);
7455	7805
7456		− io_iopoll_reap_events(ctx);
7457	7806	/* if we failed setting up the ctx, we might not have any rings */
7458	7807	if (ctx->rings)
7459	7808	io_cqring_overflow_flush(ctx, true);
	7809	+ io_iopoll_try_reap_events(ctx);
7460	7810	idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx);
7461	7811
7462	7812	/*
@@ -7464,9 +7814,8 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) · 1 unmodified line
7464	7814	* is closed but resources aren't reaped yet. This can cause
7465	7815	* spurious failure in setting up a new ring.
7466	7816	*/
7467		− if (ctx->account_mem)
7468		− io_unaccount_mem(ctx->user,
7469		− ring_pages(ctx->sq_entries, ctx->cq_entries));
	7817	+ io_unaccount_mem(ctx, ring_pages(ctx->sq_entries, ctx->cq_entries),
	7818	+ ACCT_LOCKED);
7470	7819
7471	7820	INIT_WORK(&ctx->exit_work, io_ring_exit_work);
7472	7821	queue_work(system_wq, &ctx->exit_work);
@@ -7522,17 +7871,14 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx, · 49 unmodified lines
7522	7871
7523	7872	if (cancel_req->flags & REQ_F_OVERFLOW) {
7524	7873	spin_lock_irq(&ctx->completion_lock);
7525		− list_del(&cancel_req->list);
	7874	+ list_del(&cancel_req->compl.list);
7526	7875	cancel_req->flags &= ~REQ_F_OVERFLOW;
7527		− if (list_empty(&ctx->cq_overflow_list)) {
7528		− clear_bit(0, &ctx->sq_check_overflow);
7529		− clear_bit(0, &ctx->cq_check_overflow);
7530		− ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW;
7531		− }
7532		− spin_unlock_irq(&ctx->completion_lock);
7533	7876
	7877	+ io_cqring_mark_overflow(ctx);
7534	7878	WRITE_ONCE(ctx->rings->cq_overflow,
7535	7879	atomic_inc_return(&ctx->cached_cq_overflow));
	7880	+ io_commit_cqring(ctx);
	7881	+ spin_unlock_irq(&ctx->completion_lock);
7536	7882
7537	7883	/*
7538	7884	* Put inflight ref and overflow ref. If that's
@@ -7655,8 +8001,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 116 unmodified lines
7655	8001	int submitted = 0;
7656	8002	struct fd f;
7657	8003
7658		− if (current->task_works)
7659		− task_work_run();
	8004	+ io_run_task_work();
7660	8005
7661	8006	if (flags & ~(IORING_ENTER_GETEVENTS \| IORING_ENTER_SQ_WAKEUP))
7662	8007	return -EINVAL;
@@ -7695,8 +8040,6 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 32 unmodified lines
7695	8040	goto out;
7696	8041	}
7697	8042	if (flags & IORING_ENTER_GETEVENTS) {
7698		− unsigned nr_events = 0;
7699		−
7700	8043	min_complete = min(min_complete, ctx->cq_entries);
7701	8044
7702	8045	/*
@@ -7707,7 +8050,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 4 unmodified lines
7707	8050	*/
7708	8051	if (ctx->flags & IORING_SETUP_IOPOLL &&
7709	8052	!(ctx->flags & IORING_SETUP_SQPOLL)) {
7710		− ret = io_iopoll_check(ctx, &nr_events, min_complete);
	8053	+ ret = io_iopoll_check(ctx, min_complete);
7711	8054	} else {
7712	8055	ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
7713	8056	}
@@ -7912,7 +8255,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, · 198 unmodified lines
7912	8255	{
7913	8256	struct user_struct *user = NULL;
7914	8257	struct io_ring_ctx *ctx;
7915		− bool account_mem;
	8258	+ bool limit_mem;
7916	8259	int ret;
7917	8260
7918	8261	if (!entries)
@@ -7951,10 +8294,10 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, · 32 unmodified lines
7951	8294	}
7952	8295
7953	8296	user = get_uid(current_user());
7954		− account_mem = !capable(CAP_IPC_LOCK);
	8297	+ limit_mem = !capable(CAP_IPC_LOCK);
7955	8298
7956		− if (account_mem) {
7957		− ret = io_account_mem(user,
	8299	+ if (limit_mem) {
	8300	+ ret = __io_account_mem(user,
7958	8301	ring_pages(p->sq_entries, p->cq_entries));
7959	8302	if (ret) {
7960	8303	free_uid(user);
@@ -7964,14 +8307,13 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, · 3 unmodified lines
7964	8307
7965	8308	ctx = io_ring_ctx_alloc(p);
7966	8309	if (!ctx) {
7967		− if (account_mem)
7968		− io_unaccount_mem(user, ring_pages(p->sq_entries,
	8310	+ if (limit_mem)
	8311	+ __io_unaccount_mem(user, ring_pages(p->sq_entries,
7969	8312	p->cq_entries));
7970	8313	free_uid(user);
7971	8314	return -ENOMEM;
7972	8315	}
7973	8316	ctx->compat = in_compat_syscall();
7974		− ctx->account_mem = account_mem;
7975	8317	ctx->user = user;
7976	8318	ctx->creds = get_current_cred();
7977	8319
@@ -8003,12 +8345,22 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, · 25 unmodified lines
8003	8345
8004	8346	p->features = IORING_FEAT_SINGLE_MMAP \| IORING_FEAT_NODROP \|
8005	8347	IORING_FEAT_SUBMIT_STABLE \| IORING_FEAT_RW_CUR_POS \|
8006		− IORING_FEAT_CUR_PERSONALITY \| IORING_FEAT_FAST_POLL;
	8348	+ IORING_FEAT_CUR_PERSONALITY \| IORING_FEAT_FAST_POLL \|
	8349	+ IORING_FEAT_POLL_32BITS;
8007	8350
8008	8351	if (copy_to_user(params, p, sizeof(*p))) {
8009	8352	ret = -EFAULT;
8010	8353	goto err;
8011	8354	}
	8355	+
	8356	+ /*
	8357	+ * Account memory _before_ installing the file descriptor. Once
	8358	+ * the descriptor is installed, it can get closed at any time.
	8359	+ */
	8360	+ io_account_mem(ctx, ring_pages(p->sq_entries, p->cq_entries),
	8361	+ ACCT_LOCKED);
	8362	+ ctx->limit_mem = limit_mem;
	8363	+
8012	8364	/*
8013	8365	* Install ring fd as the very last thing, so we don't risk someone
8014	8366	* having closed it before we finish setup
@@ -8292,7 +8644,8 @@ static int __init io_uring_init(void) · 277 unmodified lines
8292	8644	BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);
8293	8645	BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
8294	8646	BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags);
8295		− BUILD_BUG_SQE_ELEM(28, __u16, poll_events);
	8647	+ BUILD_BUG_SQE_ELEM(28, /* compat */ __u16, poll_events);
	8648	+ BUILD_BUG_SQE_ELEM(28, __u32, poll32_events);
8296	8649	BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags);
8297	8650	BUILD_BUG_SQE_ELEM(28, __u32, msg_flags);
8298	8651	BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags);

fs/xfs/xfs_file.c+1−1

@@ -1080,7 +1080,7 @@ xfs_file_open( · 1079 unmodified lines
1080	1080	return -EFBIG;
1081	1081	if (XFS_FORCED_SHUTDOWN(XFS_M(inode->i_sb)))
1082	1082	return -EIO;
1083		− file->f_mode \|= FMODE_NOWAIT;
	1083	+ file->f_mode \|= FMODE_NOWAIT \| FMODE_BUF_RASYNC;
1084	1084	return 0;
1085	1085	}
1086	1086

include/linux/blkdev.h+1−0

@@ -1180,6 +1180,7 @@ struct blk_plug { · 1179 unmodified lines
1180	1180	struct list_head cb_list; /* md requires an unplug callback */
1181	1181	unsigned short rq_count;
1182	1182	bool multiple_queues;
	1183	+ bool nowait;
1183	1184	};
1184	1185	#define BLK_MAX_REQUEST_COUNT 16
1185	1186	#define BLK_PLUG_FLUSH_SIZE (128 * 1024)

include/linux/fs.h+20−6

@@ -175,6 +175,9 @@ typedef int (dio_iodone_t)(struct kiocb *iocb, loff_t offset, · 174 unmodified lines
175	175	/* File does not contribute to nr_files count */
176	176	#define FMODE_NOACCOUNT ((__force fmode_t)0x20000000)
177	177
	178	+/* File supports async buffered reads */
	179	+#define FMODE_BUF_RASYNC ((__force fmode_t)0x40000000)
	180	+
178	181	/*
179	182	* Flag for rw_copy_check_uvector and compat_rw_copy_check_uvector
180	183	* that indicates that they should check the contents of the iovec are
@@ -315,6 +318,8 @@ enum rw_hint { · 134 unmodified lines
315	318	#define IOCB_SYNC (1 << 5)
316	319	#define IOCB_WRITE (1 << 6)
317	320	#define IOCB_NOWAIT (1 << 7)
	321	+/* iocb->ki_waitq is valid */
	322	+#define IOCB_WAITQ (1 << 8)
318	323	#define IOCB_NOIO (1 << 9)
319	324
320	325	struct kiocb {
@@ -329,7 +334,10 @@ struct kiocb { · 8 unmodified lines
329	334	int ki_flags;
330	335	u16 ki_hint;
331	336	u16 ki_ioprio; /* See linux/ioprio.h */
332		− unsigned int ki_cookie; /* for ->iopoll */
	337	+ union {
	338	+ unsigned int ki_cookie; /* for ->iopoll */
	339	+ struct wait_page_queue ki_waitq; / for async buffered IO */
	340	+ };
333	341
334	342	randomized_struct_fields_end
335	343	};
@@ -3275,22 +3283,28 @@ static inline int iocb_flags(struct file *file) · 2939 unmodified lines
3275	3283
3276	3284	static inline int kiocb_set_rw_flags(struct kiocb *ki, rwf_t flags)
3277	3285	{
	3286	+ int kiocb_flags = 0;
	3287	+
	3288	+ if (!flags)
	3289	+ return 0;
3278	3290	if (unlikely(flags & ~RWF_SUPPORTED))
3279	3291	return -EOPNOTSUPP;
3280	3292
3281	3293	if (flags & RWF_NOWAIT) {
3282	3294	if (!(ki->ki_filp->f_mode & FMODE_NOWAIT))
3283	3295	return -EOPNOTSUPP;
3284		− ki->ki_flags \|= IOCB_NOWAIT;
	3296	+ kiocb_flags \|= IOCB_NOWAIT;
3285	3297	}
3286	3298	if (flags & RWF_HIPRI)
3287		− ki->ki_flags \|= IOCB_HIPRI;
	3299	+ kiocb_flags \|= IOCB_HIPRI;
3288	3300	if (flags & RWF_DSYNC)
3289		− ki->ki_flags \|= IOCB_DSYNC;
	3301	+ kiocb_flags \|= IOCB_DSYNC;
3290	3302	if (flags & RWF_SYNC)
3291		− ki->ki_flags \|= (IOCB_DSYNC \| IOCB_SYNC);
	3303	+ kiocb_flags \|= (IOCB_DSYNC \| IOCB_SYNC);
3292	3304	if (flags & RWF_APPEND)
3293		− ki->ki_flags \|= IOCB_APPEND;
	3305	+ kiocb_flags \|= IOCB_APPEND;
	3306	+
	3307	+ ki->ki_flags \|= kiocb_flags;
3294	3308	return 0;
3295	3309	}
3296	3310

include/linux/pagemap.h+43−0

@@ -496,8 +496,35 @@ static inline pgoff_t linear_page_index(struct vm_area_struct *vma, · 495 unmodified lines
496	496	return pgoff;
497	497	}
498	498
	499	+/* This has the same layout as wait_bit_key - see fs/cachefiles/rdwr.c */
	500	+struct wait_page_key {
	501	+ struct page *page;
	502	+ int bit_nr;
	503	+ int page_match;
	504	+};
	505	+
	506	+struct wait_page_queue {
	507	+ struct page *page;
	508	+ int bit_nr;
	509	+ wait_queue_entry_t wait;
	510	+};
	511	+
	512	+static inline bool wake_page_match(struct wait_page_queue *wait_page,
	513	+ struct wait_page_key *key)
	514	+{
	515	+ if (wait_page->page != key->page)
	516	+ return false;
	517	+ key->page_match = 1;
	518	+
	519	+ if (wait_page->bit_nr != key->bit_nr)
	520	+ return false;
	521	+
	522	+ return true;
	523	+}
	524	+
499	525	extern void __lock_page(struct page *page);
500	526	extern int __lock_page_killable(struct page *page);
	527	+extern int __lock_page_async(struct page page, struct wait_page_queue wait);
501	528	extern int __lock_page_or_retry(struct page page, struct mm_struct mm,
502	529	unsigned int flags);
503	530	extern void unlock_page(struct page *page);
@@ -534,6 +561,22 @@ static inline int lock_page_killable(struct page *page) · 30 unmodified lines
534	561	return 0;
535	562	}
536	563
	564	+/*
	565	+ * lock_page_async - Lock the page, unless this would block. If the page
	566	+ * is already locked, then queue a callback when the page becomes unlocked.
	567	+ * This callback can then retry the operation.
	568	+ *
	569	+ * Returns 0 if the page is locked successfully, or -EIOCBQUEUED if the page
	570	+ * was already locked and the callback defined in 'wait' was queued.
	571	+ */
	572	+static inline int lock_page_async(struct page *page,
	573	+ struct wait_page_queue *wait)
	574	+{
	575	+ if (!trylock_page(page))
	576	+ return __lock_page_async(page, wait);
	577	+ return 0;
	578	+}
	579	+
537	580	/*
538	581	* lock_page_or_retry - Lock the page, unless this would block and the
539	582	* caller indicated that it can handle a retry.

include/linux/sched/task.h+6−0

@@ -126,6 +126,12 @@ static inline void put_task_struct(struct task_struct *t) · 125 unmodified lines
126	126	__put_task_struct(t);
127	127	}
128	128
	129	+static inline void put_task_struct_many(struct task_struct *t, int nr)
	130	+{
	131	+ if (refcount_sub_and_test(nr, &t->usage))
	132	+ __put_task_struct(t);
	133	+}
	134	+
129	135	void put_task_struct_rcu_user(struct task_struct *task);
130	136
131	137	#ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT

include/uapi/linux/io_uring.h+3−1

@@ -31,7 +31,8 @@ struct io_uring_sqe { · 30 unmodified lines
31	31	union {
32	32	__kernel_rwf_t rw_flags;
33	33	__u32 fsync_flags;
34		− __u16 poll_events;
	34	+ __u16 poll_events; /* compatibility */
	35	+ __u32 poll32_events; /* word-reversed for BE */
35	36	__u32 sync_range_flags;
36	37	__u32 msg_flags;
37	38	__u32 timeout_flags;
@@ -249,6 +250,7 @@ struct io_uring_params { · 211 unmodified lines
249	250	#define IORING_FEAT_RW_CUR_POS (1U << 3)
250	251	#define IORING_FEAT_CUR_PERSONALITY (1U << 4)
251	252	#define IORING_FEAT_FAST_POLL (1U << 5)
	253	+#define IORING_FEAT_POLL_32BITS (1U << 6)
252	254
253	255	/*
254	256	* io_uring_register(2) opcodes and arguments

mm/filemap.c+64−27

@@ -987,19 +987,6 @@ void __init pagecache_init(void) · 986 unmodified lines
987	987	page_writeback_init();
988	988	}
989	989
990		−/* This has the same layout as wait_bit_key - see fs/cachefiles/rdwr.c */
991		−struct wait_page_key {
992		− struct page *page;
993		− int bit_nr;
994		− int page_match;
995		−};
996		−
997		−struct wait_page_queue {
998		− struct page *page;
999		− int bit_nr;
1000		− wait_queue_entry_t wait;
1001		−};
1002		−
1003	990	static int wake_page_function(wait_queue_entry_t wait, unsigned mode, int sync, void arg)
1004	991	{
1005	992	int ret;
@@ -1007,11 +994,7 @@ static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync, · 1 unmodified line
1007	994	struct wait_page_queue *wait_page
1008	995	= container_of(wait, struct wait_page_queue, wait);
1009	996
1010		− if (wait_page->page != key->page)
1011		− return 0;
1012		− key->page_match = 1;
1013		−
1014		− if (wait_page->bit_nr != key->bit_nr)
	997	+ if (!wake_page_match(wait_page, key))
1015	998	return 0;
1016	999
1017	1000	/*
@@ -1240,6 +1223,44 @@ int wait_on_page_bit_killable(struct page *page, int bit_nr) · 222 unmodified lines
1240	1223	}
1241	1224	EXPORT_SYMBOL(wait_on_page_bit_killable);
1242	1225
	1226	+static int __wait_on_page_locked_async(struct page *page,
	1227	+ struct wait_page_queue *wait, bool set)
	1228	+{
	1229	+ struct wait_queue_head *q = page_waitqueue(page);
	1230	+ int ret = 0;
	1231	+
	1232	+ wait->page = page;
	1233	+ wait->bit_nr = PG_locked;
	1234	+
	1235	+ spin_lock_irq(&q->lock);
	1236	+ __add_wait_queue_entry_tail(q, &wait->wait);
	1237	+ SetPageWaiters(page);
	1238	+ if (set)
	1239	+ ret = !trylock_page(page);
	1240	+ else
	1241	+ ret = PageLocked(page);
	1242	+ /*
	1243	+ * If we were succesful now, we know we're still on the
	1244	+ * waitqueue as we're still under the lock. This means it's
	1245	+ * safe to remove and return success, we know the callback
	1246	+ * isn't going to trigger.
	1247	+ */
	1248	+ if (!ret)
	1249	+ __remove_wait_queue(q, &wait->wait);
	1250	+ else
	1251	+ ret = -EIOCBQUEUED;
	1252	+ spin_unlock_irq(&q->lock);
	1253	+ return ret;
	1254	+}
	1255	+
	1256	+static int wait_on_page_locked_async(struct page *page,
	1257	+ struct wait_page_queue *wait)
	1258	+{
	1259	+ if (!PageLocked(page))
	1260	+ return 0;
	1261	+ return __wait_on_page_locked_async(compound_head(page), wait, false);
	1262	+}
	1263	+
1243	1264	/**
1244	1265	* put_and_wait_on_page_locked - Drop a reference and wait for it to be unlocked
1245	1266	* @page: The page to wait for.
@@ -1402,6 +1423,11 @@ int __lock_page_killable(struct page *__page) · 156 unmodified lines
1402	1423	}
1403	1424	EXPORT_SYMBOL_GPL(__lock_page_killable);
1404	1425
	1426	+int __lock_page_async(struct page page, struct wait_page_queue wait)
	1427	+{
	1428	+ return __wait_on_page_locked_async(page, wait, true);
	1429	+}
	1430	+
1405	1431	/*
1406	1432	* Return values:
1407	1433	* 1 - page is locked; mmap_lock is still held.
@@ -2061,7 +2087,7 @@ find_page: · 653 unmodified lines
2061	2087
2062	2088	page = find_get_page(mapping, index);
2063	2089	if (!page) {
2064		− if (iocb->ki_flags & (IOCB_NOWAIT \| IOCB_NOIO))
	2090	+ if (iocb->ki_flags & IOCB_NOIO)
2065	2091	goto would_block;
2066	2092	page_cache_sync_readahead(mapping,
2067	2093	ra, filp,
@@ -2080,17 +2106,25 @@ find_page: · 12 unmodified lines
2080	2106	index, last_index - index);
2081	2107	}
2082	2108	if (!PageUptodate(page)) {
2083		− if (iocb->ki_flags & IOCB_NOWAIT) {
2084		− put_page(page);
2085		− goto would_block;
2086		− }
2087		−
2088	2109	/*
2089	2110	* See comment in do_read_cache_page on why
2090	2111	* wait_on_page_locked is used to avoid unnecessarily
2091	2112	* serialisations and why it's safe.
2092	2113	*/
2093		− error = wait_on_page_locked_killable(page);
	2114	+ if (iocb->ki_flags & IOCB_WAITQ) {
	2115	+ if (written) {
	2116	+ put_page(page);
	2117	+ goto out;
	2118	+ }
	2119	+ error = wait_on_page_locked_async(page,
	2120	+ iocb->ki_waitq);
	2121	+ } else {
	2122	+ if (iocb->ki_flags & IOCB_NOWAIT) {
	2123	+ put_page(page);
	2124	+ goto would_block;
	2125	+ }
	2126	+ error = wait_on_page_locked_killable(page);
	2127	+ }
2094	2128	if (unlikely(error))
2095	2129	goto readpage_error;
2096	2130	if (PageUptodate(page))
@@ -2178,7 +2212,10 @@ page_ok: · 81 unmodified lines
2178	2212
2179	2213	page_not_up_to_date:
2180	2214	/* Get exclusive access to the page ... */
2181		− error = lock_page_killable(page);
	2215	+ if (iocb->ki_flags & IOCB_WAITQ)
	2216	+ error = lock_page_async(page, iocb->ki_waitq);
	2217	+ else
	2218	+ error = lock_page_killable(page);
2182	2219	if (unlikely(error))
2183	2220	goto readpage_error;
2184	2221
@@ -2197,7 +2234,7 @@ page_not_up_to_date_locked: · 12 unmodified lines
2197	2234	}
2198	2235
2199	2236	readpage:
2200		− if (iocb->ki_flags & IOCB_NOIO) {
	2237	+ if (iocb->ki_flags & (IOCB_NOIO \| IOCB_NOWAIT)) {
2201	2238	unlock_page(page);
2202	2239	put_page(page);
2203	2240	goto would_block;

tools/io_uring/liburing.h+5−1

@@ -10,6 +10,7 @@ extern "C" { · 9 unmodified lines
10	10	#include <string.h>
11	11	#include "../../include/uapi/linux/io_uring.h"
12	12	#include <inttypes.h>
	13	+#include <linux/swab.h>
13	14	#include "barrier.h"
14	15
15	16	/*
@@ -145,11 +146,14 @@ static inline void io_uring_prep_write_fixed(struct io_uring_sqe *sqe, int fd, · 129 unmodified lines
145	146	}
146	147
147	148	static inline void io_uring_prep_poll_add(struct io_uring_sqe *sqe, int fd,
148		− short poll_mask)
	149	+ unsigned poll_mask)
149	150	{
150	151	memset(sqe, 0, sizeof(*sqe));
151	152	sqe->opcode = IORING_OP_POLL_ADD;
152	153	sqe->fd = fd;
	154	+#if __BYTE_ORDER == __BIG_ENDIAN
	155	+ poll_mask = __swahw32(poll_mask);
	156	+#endif
153	157	sqe->poll_events = poll_mask;
154	158	}
155	159