Merge tag 'for-5.9/io_uring-20200802' of git://git.kernel.dk/linux-block

Pull io_uring updates from Jens Axboe: "Lots of cleanups in here, hardening the code and/or making it easier to read and fixing bugs, but a core feature/change too adding support for real async buffered reads. With the latter in place, we just need buffered write async support and we're done relying on kthreads for the fast path. In detail: - Cleanup how memory accounting is done on ring setup/free (Bijan) - sq array offset calculation fixup (Dmitry) - Consistently handle blocking off O_DIRECT submission path (me) - Support proper async buffered reads, instead of relying on kthread offload for that. This uses the page waitqueue to drive retries from task_work, like we handle poll based retry. (me) - IO completion optimizations (me) - Fix race with accounting and ring fd install (me) - Support EPOLLEXCLUSIVE (Jiufei) - Get rid of the io_kiocb unionizing, made possible by shrinking other bits (Pavel) - Completion side cleanups (Pavel) - Cleanup REQ_F_ flags handling, and kill off many of them (Pavel) - Request environment grabbing cleanups (Pavel) - File and socket read/write cleanups (Pavel) - Improve kiocb_set_rw_flags() (Pavel) - Tons of fixes and cleanups (Pavel) - IORING_SQ_NEED_WAKEUP clear fix (Xiaoguang)" * tag 'for-5.9/io_uring-20200802' of git://git.kernel.dk/linux-block: (127 commits) io_uring: flip if handling after io_setup_async_rw fs: optimise kiocb_set_rw_flags() io_uring: don't touch 'ctx' after installing file descriptor io_uring: get rid of atomic FAA for cq_timeouts io_uring: consolidate *_check_overflow accounting io_uring: fix stalled deferred requests io_uring: fix racy overflow count reporting io_uring: deduplicate __io_complete_rw() io_uring: de-unionise io_kiocb io-wq: update hash bits io_uring: fix missing io_queue_linked_timeout() io_uring: mark ->work uninitialised after cleanup io_uring: deduplicate io_grab_files() calls io_uring: don't do opcode prep twice io_uring: clear IORING_SQ_NEED_WAKEUP after executing task works io_uring: batch put_task_struct() tasks: add put_task_struct_many() io_uring: return locked and pinned page accounting io_uring: don't miscount pinned memory io_uring: don't open-code recv kbuf managment ...

Linus Torvalds committed 6 years ago

Commit cdc8fcb2 parents 382625d fa15baf14 files · +1644 −1181Browse files

Unified · Split

14 files

block/blk-core.c +6 −0
fs/block_dev.c +1 −1
fs/btrfs/file.c +1 −1
fs/io-wq.c +5 −9
fs/io-wq.h +6 −5
fs/io_uring.c +1482 −1129
fs/xfs/xfs_file.c +1 −1
include/linux/blkdev.h +1 −0
include/linux/fs.h +20 −6
include/linux/pagemap.h +43 −0
include/linux/sched/task.h +6 −0
include/uapi/linux/io_uring.h +3 −1
mm/filemap.c +64 −27
tools/io_uring/liburing.h +5 −1

block/blk-core.c+6−0

 {
 	struct request_queue *q = bio->bi_disk->queue;
 	blk_status_t status = BLK_STS_IOERR;
+	struct blk_plug *plug;
 	might_sleep();
+	plug = blk_mq_plug(q, bio);
+	if (plug && plug->nowait)
+		bio->bi_opf |= REQ_NOWAIT;
 	/*
 	 * For a REQ_NOWAIT based request, return -EOPNOTSUPP
 	 * if queue is not a request based queue.
 	INIT_LIST_HEAD(&plug->cb_list);
 	plug->rq_count = 0;
 	plug->multiple_queues = false;
+	plug->nowait = false;
 	/*
 	 * Store ordering should not be needed here, since a potential

fs/block_dev.c+1−1

@@ -1734,7 +1734,7 @@ static int blkdev_open(struct inode * inode, struct file * filp) · 1733 unmodified lines
1734	*/	1734	*/
1735	filp->f_flags \|= O_LARGEFILE;	1735	filp->f_flags \|= O_LARGEFILE;
1736		1736
1737	filp->f_mode \|= FMODE_NOWAIT;	1737	filp->f_mode \|= FMODE_NOWAIT \| FMODE_BUF_RASYNC;
1738		1738
1739	if (filp->f_flags & O_NDELAY)	1739	if (filp->f_flags & O_NDELAY)
1740	filp->f_mode \|= FMODE_NDELAY;	1740	filp->f_mode \|= FMODE_NDELAY;

fs/btrfs/file.c+1−1

@@ -3537,7 +3537,7 @@ static loff_t btrfs_file_llseek(struct file *file, loff_t offset, int whence) · 3536 unmodified lines
3537		3537
3538	static int btrfs_file_open(struct inode inode, struct file filp)	3538	static int btrfs_file_open(struct inode inode, struct file filp)
3539	{	3539	{
3540	filp->f_mode \|= FMODE_NOWAIT;	3540	filp->f_mode \|= FMODE_NOWAIT \| FMODE_BUF_RASYNC;
3541	return generic_file_open(inode, filp);	3541	return generic_file_open(inode, filp);
3542	}	3542	}
3543		3543

fs/io-wq.c+5−9

 		io_wq_switch_mm(worker, work);
 	if (worker->cur_creds != work->creds)
 		io_wq_switch_creds(worker, work);
+	current->signal->rlim[RLIMIT_FSIZE].rlim_cur = work->fsize;
 }
 static void io_assign_current_work(struct io_worker *worker,
 	do {
 		struct io_wq_work *work;
-		unsigned int hash;
 get_next:
 		/*
 		 * If we got some work, mark us as busy. If we didn't, but
 		/* handle a whole dependent link */
 		do {
 			struct io_wq_work *old_work, *next_hashed, *linked;
+			unsigned int hash = io_get_work_hash(work);
 			next_hashed = wq_next_work(work);
 			io_impersonate_work(worker, work);
 			if (test_bit(IO_WQ_BIT_CANCEL, &wq->state))
 				work->flags |= IO_WQ_WORK_CANCEL;
-			hash = io_get_work_hash(work);
+			old_work = work;
-			linked = old_work = work;
+			linked = wq->do_work(work);
-			wq->do_work(&linked);
-			linked = (old_work == linked) ? NULL : linked;
 			work = next_hashed;
 			if (!work && linked && !io_wq_is_hashed(linked)) {
 				spin_lock_irq(&wqe->lock);
 				wqe->hash_map &= ~BIT_ULL(hash);
 				wqe->flags &= ~IO_WQE_FLAG_STALLED;
-				/* dependent work is not hashed */
-				hash = -1U;
 				/* skip unnecessary unlock-lock wqe->lock */
 				if (!work)
 					goto get_next;
 		struct io_wq_work *old_work = work;
 		work->flags |= IO_WQ_WORK_CANCEL;
-		wq->do_work(&work);
+		work = wq->do_work(work);
-		work = (work == old_work) ? NULL : work;
 		wq->free_work(old_work);
 	} while (work);
 }

fs/io-wq.h+6−5

 enum {
 	IO_WQ_WORK_CANCEL	= 1,
-	IO_WQ_WORK_HASHED	= 4,
+	IO_WQ_WORK_HASHED	= 2,
-	IO_WQ_WORK_UNBOUND	= 32,
+	IO_WQ_WORK_UNBOUND	= 4,
-	IO_WQ_WORK_NO_CANCEL	= 256,
+	IO_WQ_WORK_NO_CANCEL	= 8,
-	IO_WQ_WORK_CONCURRENT	= 512,
+	IO_WQ_WORK_CONCURRENT	= 16,
 	IO_WQ_HASH_SHIFT	= 24,	/* upper 8 bits are used for hash key */
 };
 	struct mm_struct *mm;
 	const struct cred *creds;
 	struct fs_struct *fs;
+	unsigned long fsize;
 	unsigned flags;
 };
 }
 typedef void (free_work_fn)(struct io_wq_work *);
-typedef void (io_wq_work_fn)(struct io_wq_work **);
+typedef struct io_wq_work *(io_wq_work_fn)(struct io_wq_work *);
 struct io_wq_data {
 	struct user_struct *user;

fs/io_uring.c+1482−1129

@@ -78,6 +78,7 @@ · 77 unmodified lines
78	#include <linux/fs_struct.h>	78	#include <linux/fs_struct.h>
79	#include <linux/splice.h>	79	#include <linux/splice.h>
80	#include <linux/task_work.h>	80	#include <linux/task_work.h>
		81	#include <linux/pagemap.h>
81		82
82	#define CREATE_TRACE_POINTS	83	#define CREATE_TRACE_POINTS
83	#include <trace/events/io_uring.h>	84	#include <trace/events/io_uring.h>
@@ -226,7 +227,7 @@ struct io_ring_ctx { · 142 unmodified lines
226	struct {	227	struct {
227	unsigned int flags;	228	unsigned int flags;
228	unsigned int compat: 1;	229	unsigned int compat: 1;
229	unsigned int account_mem: 1;	230	unsigned int limit_mem: 1;
230	unsigned int cq_overflow_flushed: 1;	231	unsigned int cq_overflow_flushed: 1;
231	unsigned int drain_next: 1;	232	unsigned int drain_next: 1;
232	unsigned int eventfd_async: 1;	233	unsigned int eventfd_async: 1;
@@ -319,12 +320,12 @@ struct io_ring_ctx { · 86 unmodified lines
319	spinlock_t completion_lock;	320	spinlock_t completion_lock;
320		321
321	/*	322	/*
322	* ->poll_list is protected by the ctx->uring_lock for	323	* ->iopoll_list is protected by the ctx->uring_lock for
323	* io_uring instances that don't use IORING_SETUP_SQPOLL.	324	* io_uring instances that don't use IORING_SETUP_SQPOLL.
324	* For SQPOLL, only the single threaded io_sq_thread() will	325	* For SQPOLL, only the single threaded io_sq_thread() will
325	* manipulate the list, hence no extra locking is needed there.	326	* manipulate the list, hence no extra locking is needed there.
326	*/	327	*/
327	struct list_head poll_list;	328	struct list_head iopoll_list;
328	struct hlist_head *cancel_hash;	329	struct hlist_head *cancel_hash;
329	unsigned cancel_hash_bits;	330	unsigned cancel_hash_bits;
330	bool poll_multi_file;	331	bool poll_multi_file;
@@ -395,6 +396,7 @@ struct io_timeout { · 64 unmodified lines
395	int flags;	396	int flags;
396	u32 off;	397	u32 off;
397	u32 target_seq;	398	u32 target_seq;
		399	struct list_head list;
398	};	400	};
399		401
400	struct io_rw {	402	struct io_rw {
@@ -413,7 +415,7 @@ struct io_connect { · 12 unmodified lines
413	struct io_sr_msg {	415	struct io_sr_msg {
414	struct file *file;	416	struct file *file;
415	union {	417	union {
416	struct user_msghdr __user *msg;	418	struct user_msghdr __user *umsg;
417	void __user *buf;	419	void __user *buf;
418	};	420	};
419	int msg_flags;	421	int msg_flags;
@@ -486,6 +488,12 @@ struct io_statx { · 66 unmodified lines
486	struct statx __user *buffer;	488	struct statx __user *buffer;
487	};	489	};
488		490
		491	struct io_completion {
		492	struct file *file;
		493	struct list_head list;
		494	int cflags;
		495	};
		496
489	struct io_async_connect {	497	struct io_async_connect {
490	struct sockaddr_storage address;	498	struct sockaddr_storage address;
491	};	499	};
@@ -503,6 +511,7 @@ struct io_async_rw { · 11 unmodified lines
503	struct iovec *iov;	511	struct iovec *iov;
504	ssize_t nr_segs;	512	ssize_t nr_segs;
505	ssize_t size;	513	ssize_t size;
		514	struct wait_page_queue wpq;
506	};	515	};
507		516
508	struct io_async_ctx {	517	struct io_async_ctx {
@@ -523,23 +532,18 @@ enum { · 14 unmodified lines
523	REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,	532	REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,
524		533
525	REQ_F_LINK_HEAD_BIT,	534	REQ_F_LINK_HEAD_BIT,
526	REQ_F_LINK_NEXT_BIT,
527	REQ_F_FAIL_LINK_BIT,	535	REQ_F_FAIL_LINK_BIT,
528	REQ_F_INFLIGHT_BIT,	536	REQ_F_INFLIGHT_BIT,
529	REQ_F_CUR_POS_BIT,	537	REQ_F_CUR_POS_BIT,
530	REQ_F_NOWAIT_BIT,	538	REQ_F_NOWAIT_BIT,
531	REQ_F_LINK_TIMEOUT_BIT,	539	REQ_F_LINK_TIMEOUT_BIT,
532	REQ_F_TIMEOUT_BIT,
533	REQ_F_ISREG_BIT,	540	REQ_F_ISREG_BIT,
534	REQ_F_MUST_PUNT_BIT,
535	REQ_F_TIMEOUT_NOSEQ_BIT,
536	REQ_F_COMP_LOCKED_BIT,	541	REQ_F_COMP_LOCKED_BIT,
537	REQ_F_NEED_CLEANUP_BIT,	542	REQ_F_NEED_CLEANUP_BIT,
538	REQ_F_OVERFLOW_BIT,	543	REQ_F_OVERFLOW_BIT,
539	REQ_F_POLLED_BIT,	544	REQ_F_POLLED_BIT,
540	REQ_F_BUFFER_SELECTED_BIT,	545	REQ_F_BUFFER_SELECTED_BIT,
541	REQ_F_NO_FILE_TABLE_BIT,	546	REQ_F_NO_FILE_TABLE_BIT,
542	REQ_F_QUEUE_TIMEOUT_BIT,
543	REQ_F_WORK_INITIALIZED_BIT,	547	REQ_F_WORK_INITIALIZED_BIT,
544	REQ_F_TASK_PINNED_BIT,	548	REQ_F_TASK_PINNED_BIT,
545		549
@@ -563,8 +567,6 @@ enum { · 17 unmodified lines
563		567
564	/* head of a link */	568	/* head of a link */
565	REQ_F_LINK_HEAD = BIT(REQ_F_LINK_HEAD_BIT),	569	REQ_F_LINK_HEAD = BIT(REQ_F_LINK_HEAD_BIT),
566	/* already grabbed next link */
567	REQ_F_LINK_NEXT = BIT(REQ_F_LINK_NEXT_BIT),
568	/* fail rest of links */	570	/* fail rest of links */
569	REQ_F_FAIL_LINK = BIT(REQ_F_FAIL_LINK_BIT),	571	REQ_F_FAIL_LINK = BIT(REQ_F_FAIL_LINK_BIT),
570	/* on inflight list */	572	/* on inflight list */
@@ -575,14 +577,8 @@ enum { · 4 unmodified lines
575	REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),	577	REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),
576	/* has linked timeout */	578	/* has linked timeout */
577	REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),	579	REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),
578	/* timeout request */
579	REQ_F_TIMEOUT = BIT(REQ_F_TIMEOUT_BIT),
580	/* regular file */	580	/* regular file */
581	REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),	581	REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
582	/* must be punted even for NONBLOCK */
583	REQ_F_MUST_PUNT = BIT(REQ_F_MUST_PUNT_BIT),
584	/* no timeout sequence */
585	REQ_F_TIMEOUT_NOSEQ = BIT(REQ_F_TIMEOUT_NOSEQ_BIT),
586	/* completion under lock */	582	/* completion under lock */
587	REQ_F_COMP_LOCKED = BIT(REQ_F_COMP_LOCKED_BIT),	583	REQ_F_COMP_LOCKED = BIT(REQ_F_COMP_LOCKED_BIT),
588	/* needs cleanup */	584	/* needs cleanup */
@@ -595,8 +591,6 @@ enum { · 6 unmodified lines
595	REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),	591	REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
596	/* doesn't need file table for this request */	592	/* doesn't need file table for this request */
597	REQ_F_NO_FILE_TABLE = BIT(REQ_F_NO_FILE_TABLE_BIT),	593	REQ_F_NO_FILE_TABLE = BIT(REQ_F_NO_FILE_TABLE_BIT),
598	/* needs to queue linked timeout */
599	REQ_F_QUEUE_TIMEOUT = BIT(REQ_F_QUEUE_TIMEOUT_BIT),
600	/* io_wq_work is initialized */	594	/* io_wq_work is initialized */
601	REQ_F_WORK_INITIALIZED = BIT(REQ_F_WORK_INITIALIZED_BIT),	595	REQ_F_WORK_INITIALIZED = BIT(REQ_F_WORK_INITIALIZED_BIT),
602	/* req->task is refcounted */	596	/* req->task is refcounted */
@@ -606,7 +600,6 @@ enum { · 3 unmodified lines
606	struct async_poll {	600	struct async_poll {
607	struct io_poll_iocb poll;	601	struct io_poll_iocb poll;
608	struct io_poll_iocb *double_poll;	602	struct io_poll_iocb *double_poll;
609	struct io_wq_work work;
610	};	603	};
611		604
612	/*	605	/*
@@ -635,51 +628,54 @@ struct io_kiocb { · 22 unmodified lines
635	struct io_splice splice;	628	struct io_splice splice;
636	struct io_provide_buf pbuf;	629	struct io_provide_buf pbuf;
637	struct io_statx statx;	630	struct io_statx statx;
		631	/* use only after cleaning per-op data, see io_clean_op() */
		632	struct io_completion compl;
638	};	633	};
639		634
640	struct io_async_ctx *io;	635	struct io_async_ctx *io;
641	int cflags;
642	u8 opcode;	636	u8 opcode;
643	/* polled IO has completed */	637	/* polled IO has completed */
644	u8 iopoll_completed;	638	u8 iopoll_completed;
645		639
646	u16 buf_index;	640	u16 buf_index;
		641	u32 result;
647		642
648	struct io_ring_ctx *ctx;	643	struct io_ring_ctx *ctx;
649	struct list_head list;	644	unsigned int flags;
650	unsigned int flags;	645	refcount_t refs;
651	refcount_t refs;	646	struct task_struct *task;
652	struct task_struct *task;	647	u64 user_data;
653	unsigned long fsize;
654	u64 user_data;
655	u32 result;
656	u32 sequence;
657
658	struct list_head link_list;
659		648
660	struct list_head inflight_entry;	649	struct list_head link_list;
661		650
662	struct percpu_ref *fixed_file_refs;	651	/*
		652	* 1. used with ctx->iopoll_list with reads/writes
		653	* 2. to track reqs with ->files (see io_op_def::file_table)
		654	*/
		655	struct list_head inflight_entry;
		656
		657	struct percpu_ref *fixed_file_refs;
		658	struct callback_head task_work;
		659	/* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
		660	struct hlist_node hash_node;
		661	struct async_poll *apoll;
		662	struct io_wq_work work;
		663	};
663		664
664	union {	665	struct io_defer_entry {
665	/*	666	struct list_head list;
666	* Only commands that never go async can use the below fields,	667	struct io_kiocb *req;
667	* obviously. Right now only IORING_OP_POLL_ADD uses them, and	668	u32 seq;
668	* async armed poll handlers for regular commands. The latter
669	* restore the work, if needed.
670	*/
671	struct {
672	struct callback_head task_work;
673	struct hlist_node hash_node;
674	struct async_poll *apoll;
675	};
676	struct io_wq_work work;
677	};
678	};	669	};
679		670
680	#define IO_PLUG_THRESHOLD 2
681	#define IO_IOPOLL_BATCH 8	671	#define IO_IOPOLL_BATCH 8
682		672
		673	struct io_comp_state {
		674	unsigned int nr;
		675	struct list_head list;
		676	struct io_ring_ctx *ctx;
		677	};
		678
683	struct io_submit_state {	679	struct io_submit_state {
684	struct blk_plug plug;	680	struct blk_plug plug;
685		681
@@ -689,13 +685,17 @@ struct io_submit_state { · 3 unmodified lines
689	void *reqs[IO_IOPOLL_BATCH];	685	void *reqs[IO_IOPOLL_BATCH];
690	unsigned int free_reqs;	686	unsigned int free_reqs;
691		687
		688	/*
		689	* Batch completion logic
		690	*/
		691	struct io_comp_state comp;
		692
692	/*	693	/*
693	* File reference cache	694	* File reference cache
694	*/	695	*/
695	struct file *file;	696	struct file *file;
696	unsigned int fd;	697	unsigned int fd;
697	unsigned int has_refs;	698	unsigned int has_refs;
698	unsigned int used_refs;
699	unsigned int ios_left;	699	unsigned int ios_left;
700	};	700	};
701		701
@@ -723,6 +723,7 @@ struct io_op_def { · 21 unmodified lines
723	unsigned pollout : 1;	723	unsigned pollout : 1;
724	/* op supports buffer selection */	724	/* op supports buffer selection */
725	unsigned buffer_select : 1;	725	unsigned buffer_select : 1;
		726	unsigned needs_fsize : 1;
726	};	727	};
727		728
728	static const struct io_op_def io_op_defs[] = {	729	static const struct io_op_def io_op_defs[] = {
@@ -742,6 +743,7 @@ static const struct io_op_def io_op_defs[] = { · 13 unmodified lines
742	.hash_reg_file = 1,	743	.hash_reg_file = 1,
743	.unbound_nonreg_file = 1,	744	.unbound_nonreg_file = 1,
744	.pollout = 1,	745	.pollout = 1,
		746	.needs_fsize = 1,
745	},	747	},
746	[IORING_OP_FSYNC] = {	748	[IORING_OP_FSYNC] = {
747	.needs_file = 1,	749	.needs_file = 1,
@@ -756,6 +758,7 @@ static const struct io_op_def io_op_defs[] = { · 8 unmodified lines
756	.hash_reg_file = 1,	758	.hash_reg_file = 1,
757	.unbound_nonreg_file = 1,	759	.unbound_nonreg_file = 1,
758	.pollout = 1,	760	.pollout = 1,
		761	.needs_fsize = 1,
759	},	762	},
760	[IORING_OP_POLL_ADD] = {	763	[IORING_OP_POLL_ADD] = {
761	.needs_file = 1,	764	.needs_file = 1,
@@ -808,6 +811,7 @@ static const struct io_op_def io_op_defs[] = { · 46 unmodified lines
808	},	811	},
809	[IORING_OP_FALLOCATE] = {	812	[IORING_OP_FALLOCATE] = {
810	.needs_file = 1,	813	.needs_file = 1,
		814	.needs_fsize = 1,
811	},	815	},
812	[IORING_OP_OPENAT] = {	816	[IORING_OP_OPENAT] = {
813	.file_table = 1,	817	.file_table = 1,
@@ -839,6 +843,7 @@ static const struct io_op_def io_op_defs[] = { · 25 unmodified lines
839	.needs_file = 1,	843	.needs_file = 1,
840	.unbound_nonreg_file = 1,	844	.unbound_nonreg_file = 1,
841	.pollout = 1,	845	.pollout = 1,
		846	.needs_fsize = 1,
842	},	847	},
843	[IORING_OP_FADVISE] = {	848	[IORING_OP_FADVISE] = {
844	.needs_file = 1,	849	.needs_file = 1,
@@ -881,22 +886,37 @@ static const struct io_op_def io_op_defs[] = { · 36 unmodified lines
881	},	886	},
882	};	887	};
883		888
884	static void io_wq_submit_work(struct io_wq_work **workptr);	889	enum io_mem_account {
		890	ACCT_LOCKED,
		891	ACCT_PINNED,
		892	};
		893
		894	static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
		895	struct io_comp_state *cs);
885	static void io_cqring_fill_event(struct io_kiocb *req, long res);	896	static void io_cqring_fill_event(struct io_kiocb *req, long res);
886	static void io_put_req(struct io_kiocb *req);	897	static void io_put_req(struct io_kiocb *req);
		898	static void io_double_put_req(struct io_kiocb *req);
887	static void __io_double_put_req(struct io_kiocb *req);	899	static void __io_double_put_req(struct io_kiocb *req);
888	static struct io_kiocb io_prep_linked_timeout(struct io_kiocb req);	900	static struct io_kiocb io_prep_linked_timeout(struct io_kiocb req);
889	static void io_queue_linked_timeout(struct io_kiocb *req);	901	static void io_queue_linked_timeout(struct io_kiocb *req);
890	static int __io_sqe_files_update(struct io_ring_ctx *ctx,	902	static int __io_sqe_files_update(struct io_ring_ctx *ctx,
891	struct io_uring_files_update *ip,	903	struct io_uring_files_update *ip,
892	unsigned nr_args);	904	unsigned nr_args);
893	static int io_grab_files(struct io_kiocb *req);	905	static int io_prep_work_files(struct io_kiocb *req);
894	static void io_complete_rw_common(struct kiocb *kiocb, long res);	906	static void __io_clean_op(struct io_kiocb *req);
895	static void io_cleanup_req(struct io_kiocb *req);
896	static int io_file_get(struct io_submit_state state, struct io_kiocb req,	907	static int io_file_get(struct io_submit_state state, struct io_kiocb req,
897	int fd, struct file **out_file, bool fixed);	908	int fd, struct file **out_file, bool fixed);
898	static void __io_queue_sqe(struct io_kiocb *req,	909	static void __io_queue_sqe(struct io_kiocb *req,
899	const struct io_uring_sqe *sqe);	910	const struct io_uring_sqe *sqe,
		911	struct io_comp_state *cs);
		912	static void io_file_put_work(struct work_struct *work);
		913
		914	static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
		915	struct iovec *iovec, struct iov_iter iter,
		916	bool needs_lock);
		917	static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size,
		918	struct iovec iovec, struct iovec fast_iov,
		919	struct iov_iter *iter);
900		920
901	static struct kmem_cache *req_cachep;	921	static struct kmem_cache *req_cachep;
902		922
@@ -923,6 +943,12 @@ static void io_get_req_task(struct io_kiocb *req) · 20 unmodified lines
923	req->flags \|= REQ_F_TASK_PINNED;	943	req->flags \|= REQ_F_TASK_PINNED;
924	}	944	}
925		945
		946	static inline void io_clean_op(struct io_kiocb *req)
		947	{
		948	if (req->flags & (REQ_F_NEED_CLEANUP \| REQ_F_BUFFER_SELECTED))
		949	__io_clean_op(req);
		950	}
		951
926	/* not idempotent -- it doesn't clear REQ_F_TASK_PINNED */	952	/* not idempotent -- it doesn't clear REQ_F_TASK_PINNED */
927	static void __io_put_req_task(struct io_kiocb *req)	953	static void __io_put_req_task(struct io_kiocb *req)
928	{	954	{
@@ -930,7 +956,41 @@ static void __io_put_req_task(struct io_kiocb *req) · 1 unmodified line
930	put_task_struct(req->task);	956	put_task_struct(req->task);
931	}	957	}
932		958
933	static void io_file_put_work(struct work_struct *work);	959	static void io_sq_thread_drop_mm(void)
		960	{
		961	struct mm_struct *mm = current->mm;
		962
		963	if (mm) {
		964	kthread_unuse_mm(mm);
		965	mmput(mm);
		966	}
		967	}
		968
		969	static int __io_sq_thread_acquire_mm(struct io_ring_ctx *ctx)
		970	{
		971	if (!current->mm) {
		972	if (unlikely(!(ctx->flags & IORING_SETUP_SQPOLL) \|\|
		973	!mmget_not_zero(ctx->sqo_mm)))
		974	return -EFAULT;
		975	kthread_use_mm(ctx->sqo_mm);
		976	}
		977
		978	return 0;
		979	}
		980
		981	static int io_sq_thread_acquire_mm(struct io_ring_ctx *ctx,
		982	struct io_kiocb *req)
		983	{
		984	if (!io_op_defs[req->opcode].needs_mm)
		985	return 0;
		986	return __io_sq_thread_acquire_mm(ctx);
		987	}
		988
		989	static inline void req_set_fail_links(struct io_kiocb *req)
		990	{
		991	if ((req->flags & (REQ_F_LINK \| REQ_F_HARDLINK)) == REQ_F_LINK)
		992	req->flags \|= REQ_F_FAIL_LINK;
		993	}
934		994
935	/*	995	/*
936	* Note: must call io_req_init_async() for the first time you	996	* Note: must call io_req_init_async() for the first time you
@@ -957,6 +1017,11 @@ static void io_ring_ctx_ref_free(struct percpu_ref *ref) · 20 unmodified lines
957	complete(&ctx->ref_comp);	1017	complete(&ctx->ref_comp);
958	}	1018	}
959		1019
		1020	static inline bool io_is_timeout_noseq(struct io_kiocb *req)
		1021	{
		1022	return !req->timeout.off;
		1023	}
		1024
960	static struct io_ring_ctx io_ring_ctx_alloc(struct io_uring_params p)	1025	static struct io_ring_ctx io_ring_ctx_alloc(struct io_uring_params p)
961	{	1026	{
962	struct io_ring_ctx *ctx;	1027	struct io_ring_ctx *ctx;
@@ -1000,7 +1065,7 @@ static struct io_ring_ctx io_ring_ctx_alloc(struct io_uring_params p) · 37 unmodified lines
1000	mutex_init(&ctx->uring_lock);	1065	mutex_init(&ctx->uring_lock);
1001	init_waitqueue_head(&ctx->wait);	1066	init_waitqueue_head(&ctx->wait);
1002	spin_lock_init(&ctx->completion_lock);	1067	spin_lock_init(&ctx->completion_lock);
1003	INIT_LIST_HEAD(&ctx->poll_list);	1068	INIT_LIST_HEAD(&ctx->iopoll_list);
1004	INIT_LIST_HEAD(&ctx->defer_list);	1069	INIT_LIST_HEAD(&ctx->defer_list);
1005	INIT_LIST_HEAD(&ctx->timeout_list);	1070	INIT_LIST_HEAD(&ctx->timeout_list);
1006	init_waitqueue_head(&ctx->inflight_wait);	1071	init_waitqueue_head(&ctx->inflight_wait);
@@ -1017,18 +1082,14 @@ err: · 10 unmodified lines
1017	return NULL;	1082	return NULL;
1018	}	1083	}
1019		1084
1020	static inline bool __req_need_defer(struct io_kiocb *req)	1085	static bool req_need_defer(struct io_kiocb *req, u32 seq)
1021	{	1086	{
1022	struct io_ring_ctx *ctx = req->ctx;	1087	if (unlikely(req->flags & REQ_F_IO_DRAIN)) {
		1088	struct io_ring_ctx *ctx = req->ctx;
1023		1089
1024	return req->sequence != ctx->cached_cq_tail	1090	return seq != ctx->cached_cq_tail
1025	+ atomic_read(&ctx->cached_cq_overflow);	1091	+ atomic_read(&ctx->cached_cq_overflow);
1026	}	1092	}
1027
1028	static inline bool req_need_defer(struct io_kiocb *req)
1029	{
1030	if (unlikely(req->flags & REQ_F_IO_DRAIN))
1031	return __req_need_defer(req);
1032		1093
1033	return false;	1094	return false;
1034	}	1095	}
@@ -1046,28 +1107,7 @@ static void __io_commit_cqring(struct io_ring_ctx *ctx) · 11 unmodified lines
1046	}	1107	}
1047	}	1108	}
1048		1109
1049	static inline void io_req_work_grab_env(struct io_kiocb *req,	1110	static void io_req_clean_work(struct io_kiocb *req)
1050	const struct io_op_def *def)
1051	{
1052	if (!req->work.mm && def->needs_mm) {
1053	mmgrab(current->mm);
1054	req->work.mm = current->mm;
1055	}
1056	if (!req->work.creds)
1057	req->work.creds = get_current_cred();
1058	if (!req->work.fs && def->needs_fs) {
1059	spin_lock(&current->fs->lock);
1060	if (!current->fs->in_exec) {
1061	req->work.fs = current->fs;
1062	req->work.fs->users++;
1063	} else {
1064	req->work.flags \|= IO_WQ_WORK_CANCEL;
1065	}
1066	spin_unlock(&current->fs->lock);
1067	}
1068	}
1069
1070	static inline void io_req_work_drop_env(struct io_kiocb *req)
1071	{	1111	{
1072	if (!(req->flags & REQ_F_WORK_INITIALIZED))	1112	if (!(req->flags & REQ_F_WORK_INITIALIZED))
1073	return;	1113	return;
@@ -1089,11 +1129,12 @@ static inline void io_req_work_drop_env(struct io_kiocb *req) · 15 unmodified lines
1089	spin_unlock(&req->work.fs->lock);	1129	spin_unlock(&req->work.fs->lock);
1090	if (fs)	1130	if (fs)
1091	free_fs_struct(fs);	1131	free_fs_struct(fs);
		1132	req->work.fs = NULL;
1092	}	1133	}
		1134	req->flags &= ~REQ_F_WORK_INITIALIZED;
1093	}	1135	}
1094		1136
1095	static inline void io_prep_async_work(struct io_kiocb *req,	1137	static void io_prep_async_work(struct io_kiocb *req)
1096	struct io_kiocb **link)
1097	{	1138	{
1098	const struct io_op_def *def = &io_op_defs[req->opcode];	1139	const struct io_op_def *def = &io_op_defs[req->opcode];
1099		1140
@@ -1106,18 +1147,42 @@ static inline void io_prep_async_work(struct io_kiocb *req, · 6 unmodified lines
1106	if (def->unbound_nonreg_file)	1147	if (def->unbound_nonreg_file)
1107	req->work.flags \|= IO_WQ_WORK_UNBOUND;	1148	req->work.flags \|= IO_WQ_WORK_UNBOUND;
1108	}	1149	}
		1150	if (!req->work.mm && def->needs_mm) {
		1151	mmgrab(current->mm);
		1152	req->work.mm = current->mm;
		1153	}
		1154	if (!req->work.creds)
		1155	req->work.creds = get_current_cred();
		1156	if (!req->work.fs && def->needs_fs) {
		1157	spin_lock(&current->fs->lock);
		1158	if (!current->fs->in_exec) {
		1159	req->work.fs = current->fs;
		1160	req->work.fs->users++;
		1161	} else {
		1162	req->work.flags \|= IO_WQ_WORK_CANCEL;
		1163	}
		1164	spin_unlock(&current->fs->lock);
		1165	}
		1166	if (def->needs_fsize)
		1167	req->work.fsize = rlimit(RLIMIT_FSIZE);
		1168	else
		1169	req->work.fsize = RLIM_INFINITY;
		1170	}
1109		1171
1110	io_req_work_grab_env(req, def);	1172	static void io_prep_async_link(struct io_kiocb *req)
		1173	{
		1174	struct io_kiocb *cur;
1111		1175
1112	*link = io_prep_linked_timeout(req);	1176	io_prep_async_work(req);
		1177	if (req->flags & REQ_F_LINK_HEAD)
		1178	list_for_each_entry(cur, &req->link_list, link_list)
		1179	io_prep_async_work(cur);
1113	}	1180	}
1114		1181
1115	static inline void io_queue_async_work(struct io_kiocb *req)	1182	static void __io_queue_async_work(struct io_kiocb *req)
1116	{	1183	{
1117	struct io_ring_ctx *ctx = req->ctx;	1184	struct io_ring_ctx *ctx = req->ctx;
1118	struct io_kiocb *link;	1185	struct io_kiocb *link = io_prep_linked_timeout(req);
1119
1120	io_prep_async_work(req, &link);
1121		1186
1122	trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,	1187	trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
1123	&req->work, req->flags);	1188	&req->work, req->flags);
@@ -1127,14 +1192,22 @@ static inline void io_queue_async_work(struct io_kiocb *req) · 3 unmodified lines
1127	io_queue_linked_timeout(link);	1192	io_queue_linked_timeout(link);
1128	}	1193	}
1129		1194
		1195	static void io_queue_async_work(struct io_kiocb *req)
		1196	{
		1197	/* init ->work of the whole link before punting */
		1198	io_prep_async_link(req);
		1199	__io_queue_async_work(req);
		1200	}
		1201
1130	static void io_kill_timeout(struct io_kiocb *req)	1202	static void io_kill_timeout(struct io_kiocb *req)
1131	{	1203	{
1132	int ret;	1204	int ret;
1133		1205
1134	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);	1206	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
1135	if (ret != -1) {	1207	if (ret != -1) {
1136	atomic_inc(&req->ctx->cq_timeouts);	1208	atomic_set(&req->ctx->cq_timeouts,
1137	list_del_init(&req->list);	1209	atomic_read(&req->ctx->cq_timeouts) + 1);
		1210	list_del_init(&req->timeout.list);
1138	req->flags \|= REQ_F_COMP_LOCKED;	1211	req->flags \|= REQ_F_COMP_LOCKED;
1139	io_cqring_fill_event(req, 0);	1212	io_cqring_fill_event(req, 0);
1140	io_put_req(req);	1213	io_put_req(req);
@@ -1146,7 +1219,7 @@ static void io_kill_timeouts(struct io_ring_ctx *ctx) · 5 unmodified lines
1146	struct io_kiocb req, tmp;	1219	struct io_kiocb req, tmp;
1147		1220
1148	spin_lock_irq(&ctx->completion_lock);	1221	spin_lock_irq(&ctx->completion_lock);
1149	list_for_each_entry_safe(req, tmp, &ctx->timeout_list, list)	1222	list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list)
1150	io_kill_timeout(req);	1223	io_kill_timeout(req);
1151	spin_unlock_irq(&ctx->completion_lock);	1224	spin_unlock_irq(&ctx->completion_lock);
1152	}	1225	}
@@ -1154,13 +1227,15 @@ static void io_kill_timeouts(struct io_ring_ctx *ctx) · 1 unmodified line
1154	static void __io_queue_deferred(struct io_ring_ctx *ctx)	1227	static void __io_queue_deferred(struct io_ring_ctx *ctx)
1155	{	1228	{
1156	do {	1229	do {
1157	struct io_kiocb *req = list_first_entry(&ctx->defer_list,	1230	struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
1158	struct io_kiocb, list);	1231	struct io_defer_entry, list);
1159		1232
1160	if (req_need_defer(req))	1233	if (req_need_defer(de->req, de->seq))
1161	break;	1234	break;
1162	list_del_init(&req->list);	1235	list_del_init(&de->list);
1163	io_queue_async_work(req);	1236	/* punt-init is done before queueing for defer */
		1237	__io_queue_async_work(de->req);
		1238	kfree(de);
1164	} while (!list_empty(&ctx->defer_list));	1239	} while (!list_empty(&ctx->defer_list));
1165	}	1240	}
1166		1241
@@ -1168,15 +1243,15 @@ static void io_flush_timeouts(struct io_ring_ctx *ctx) · 1 unmodified line
1168	{	1243	{
1169	while (!list_empty(&ctx->timeout_list)) {	1244	while (!list_empty(&ctx->timeout_list)) {
1170	struct io_kiocb *req = list_first_entry(&ctx->timeout_list,	1245	struct io_kiocb *req = list_first_entry(&ctx->timeout_list,
1171	struct io_kiocb, list);	1246	struct io_kiocb, timeout.list);
1172		1247
1173	if (req->flags & REQ_F_TIMEOUT_NOSEQ)	1248	if (io_is_timeout_noseq(req))
1174	break;	1249	break;
1175	if (req->timeout.target_seq != ctx->cached_cq_tail	1250	if (req->timeout.target_seq != ctx->cached_cq_tail
1176	- atomic_read(&ctx->cq_timeouts))	1251	- atomic_read(&ctx->cq_timeouts))
1177	break;	1252	break;
1178		1253
1179	list_del_init(&req->list);	1254	list_del_init(&req->timeout.list);
1180	io_kill_timeout(req);	1255	io_kill_timeout(req);
1181	}	1256	}
1182	}	1257	}
@@ -1229,6 +1304,15 @@ static void io_cqring_ev_posted(struct io_ring_ctx *ctx) · 46 unmodified lines
1229	eventfd_signal(ctx->cq_ev_fd, 1);	1304	eventfd_signal(ctx->cq_ev_fd, 1);
1230	}	1305	}
1231		1306
		1307	static void io_cqring_mark_overflow(struct io_ring_ctx *ctx)
		1308	{
		1309	if (list_empty(&ctx->cq_overflow_list)) {
		1310	clear_bit(0, &ctx->sq_check_overflow);
		1311	clear_bit(0, &ctx->cq_check_overflow);
		1312	ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW;
		1313	}
		1314	}
		1315
1232	/* Returns true if there are no backlogged entries after the flush */	1316	/* Returns true if there are no backlogged entries after the flush */
1233	static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)	1317	static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
1234	{	1318	{
@@ -1259,13 +1343,13 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) · 24 unmodified lines
1259	break;	1343	break;
1260		1344
1261	req = list_first_entry(&ctx->cq_overflow_list, struct io_kiocb,	1345	req = list_first_entry(&ctx->cq_overflow_list, struct io_kiocb,
1262	list);	1346	compl.list);
1263	list_move(&req->list, &list);	1347	list_move(&req->compl.list, &list);
1264	req->flags &= ~REQ_F_OVERFLOW;	1348	req->flags &= ~REQ_F_OVERFLOW;
1265	if (cqe) {	1349	if (cqe) {
1266	WRITE_ONCE(cqe->user_data, req->user_data);	1350	WRITE_ONCE(cqe->user_data, req->user_data);
1267	WRITE_ONCE(cqe->res, req->result);	1351	WRITE_ONCE(cqe->res, req->result);
1268	WRITE_ONCE(cqe->flags, req->cflags);	1352	WRITE_ONCE(cqe->flags, req->compl.cflags);
1269	} else {	1353	} else {
1270	WRITE_ONCE(ctx->rings->cq_overflow,	1354	WRITE_ONCE(ctx->rings->cq_overflow,
1271	atomic_inc_return(&ctx->cached_cq_overflow));	1355	atomic_inc_return(&ctx->cached_cq_overflow));
@@ -1273,17 +1357,14 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) · 1 unmodified line
1273	}	1357	}
1274		1358
1275	io_commit_cqring(ctx);	1359	io_commit_cqring(ctx);
1276	if (cqe) {	1360	io_cqring_mark_overflow(ctx);
1277	clear_bit(0, &ctx->sq_check_overflow);	1361
1278	clear_bit(0, &ctx->cq_check_overflow);
1279	ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW;
1280	}
1281	spin_unlock_irqrestore(&ctx->completion_lock, flags);	1362	spin_unlock_irqrestore(&ctx->completion_lock, flags);
1282	io_cqring_ev_posted(ctx);	1363	io_cqring_ev_posted(ctx);
1283		1364
1284	while (!list_empty(&list)) {	1365	while (!list_empty(&list)) {
1285	req = list_first_entry(&list, struct io_kiocb, list);	1366	req = list_first_entry(&list, struct io_kiocb, compl.list);
1286	list_del(&req->list);	1367	list_del(&req->compl.list);
1287	io_put_req(req);	1368	io_put_req(req);
1288	}	1369	}
1289		1370
@@ -1316,11 +1397,12 @@ static void __io_cqring_fill_event(struct io_kiocb *req, long res, long cflags) · 26 unmodified lines
1316	set_bit(0, &ctx->cq_check_overflow);	1397	set_bit(0, &ctx->cq_check_overflow);
1317	ctx->rings->sq_flags \|= IORING_SQ_CQ_OVERFLOW;	1398	ctx->rings->sq_flags \|= IORING_SQ_CQ_OVERFLOW;
1318	}	1399	}
		1400	io_clean_op(req);
1319	req->flags \|= REQ_F_OVERFLOW;	1401	req->flags \|= REQ_F_OVERFLOW;
1320	refcount_inc(&req->refs);
1321	req->result = res;	1402	req->result = res;
1322	req->cflags = cflags;	1403	req->compl.cflags = cflags;
1323	list_add_tail(&req->list, &ctx->cq_overflow_list);	1404	refcount_inc(&req->refs);
		1405	list_add_tail(&req->compl.list, &ctx->cq_overflow_list);
1324	}	1406	}
1325	}	1407	}
1326		1408
@@ -1329,7 +1411,7 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res) · 2 unmodified lines
1329	__io_cqring_fill_event(req, res, 0);	1411	__io_cqring_fill_event(req, res, 0);
1330	}	1412	}
1331		1413
1332	static void __io_cqring_add_event(struct io_kiocb *req, long res, long cflags)	1414	static void io_cqring_add_event(struct io_kiocb *req, long res, long cflags)
1333	{	1415	{
1334	struct io_ring_ctx *ctx = req->ctx;	1416	struct io_ring_ctx *ctx = req->ctx;
1335	unsigned long flags;	1417	unsigned long flags;
@@ -1342,9 +1424,52 @@ static void __io_cqring_add_event(struct io_kiocb *req, long res, long cflags) · 6 unmodified lines
1342	io_cqring_ev_posted(ctx);	1424	io_cqring_ev_posted(ctx);
1343	}	1425	}
1344		1426
1345	static void io_cqring_add_event(struct io_kiocb *req, long res)	1427	static void io_submit_flush_completions(struct io_comp_state *cs)
		1428	{
		1429	struct io_ring_ctx *ctx = cs->ctx;
		1430
		1431	spin_lock_irq(&ctx->completion_lock);
		1432	while (!list_empty(&cs->list)) {
		1433	struct io_kiocb *req;
		1434
		1435	req = list_first_entry(&cs->list, struct io_kiocb, compl.list);
		1436	list_del(&req->compl.list);
		1437	__io_cqring_fill_event(req, req->result, req->compl.cflags);
		1438	if (!(req->flags & REQ_F_LINK_HEAD)) {
		1439	req->flags \|= REQ_F_COMP_LOCKED;
		1440	io_put_req(req);
		1441	} else {
		1442	spin_unlock_irq(&ctx->completion_lock);
		1443	io_put_req(req);
		1444	spin_lock_irq(&ctx->completion_lock);
		1445	}
		1446	}
		1447	io_commit_cqring(ctx);
		1448	spin_unlock_irq(&ctx->completion_lock);
		1449
		1450	io_cqring_ev_posted(ctx);
		1451	cs->nr = 0;
		1452	}
		1453
		1454	static void __io_req_complete(struct io_kiocb *req, long res, unsigned cflags,
		1455	struct io_comp_state *cs)
		1456	{
		1457	if (!cs) {
		1458	io_cqring_add_event(req, res, cflags);
		1459	io_put_req(req);
		1460	} else {
		1461	io_clean_op(req);
		1462	req->result = res;
		1463	req->compl.cflags = cflags;
		1464	list_add_tail(&req->compl.list, &cs->list);
		1465	if (++cs->nr >= 32)
		1466	io_submit_flush_completions(cs);
		1467	}
		1468	}
		1469
		1470	static void io_req_complete(struct io_kiocb *req, long res)
1346	{	1471	{
1347	__io_cqring_add_event(req, res, 0);	1472	__io_req_complete(req, res, 0, NULL);
1348	}	1473	}
1349		1474
1350	static inline bool io_is_fallback_req(struct io_kiocb *req)	1475	static inline bool io_is_fallback_req(struct io_kiocb *req)
@@ -1370,11 +1495,7 @@ static struct io_kiocb io_alloc_req(struct io_ring_ctx ctx, · 19 unmodified lines
1370	gfp_t gfp = GFP_KERNEL \| __GFP_NOWARN;	1495	gfp_t gfp = GFP_KERNEL \| __GFP_NOWARN;
1371	struct io_kiocb *req;	1496	struct io_kiocb *req;
1372		1497
1373	if (!state) {	1498	if (!state->free_reqs) {
1374	req = kmem_cache_alloc(req_cachep, gfp);
1375	if (unlikely(!req))
1376	goto fallback;
1377	} else if (!state->free_reqs) {
1378	size_t sz;	1499	size_t sz;
1379	int ret;	1500	int ret;
1380		1501
@@ -1412,21 +1533,15 @@ static inline void io_put_file(struct io_kiocb req, struct file file, · 31 unmodified lines
1412	fput(file);	1533	fput(file);
1413	}	1534	}
1414		1535
1415	static void __io_req_aux_free(struct io_kiocb *req)	1536	static void io_dismantle_req(struct io_kiocb *req)
1416	{	1537	{
1417	if (req->flags & REQ_F_NEED_CLEANUP)	1538	io_clean_op(req);
1418	io_cleanup_req(req);
1419		1539
1420	kfree(req->io);	1540	if (req->io)
		1541	kfree(req->io);
1421	if (req->file)	1542	if (req->file)
1422	io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE));	1543	io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE));
1423	__io_put_req_task(req);	1544	io_req_clean_work(req);
1424	io_req_work_drop_env(req);
1425	}
1426
1427	static void __io_free_req(struct io_kiocb *req)
1428	{
1429	__io_req_aux_free(req);
1430		1545
1431	if (req->flags & REQ_F_INFLIGHT) {	1546	if (req->flags & REQ_F_INFLIGHT) {
1432	struct io_ring_ctx *ctx = req->ctx;	1547	struct io_ring_ctx *ctx = req->ctx;
@@ -1438,57 +1553,20 @@ static void __io_free_req(struct io_kiocb *req) · 5 unmodified lines
1438	wake_up(&ctx->inflight_wait);	1553	wake_up(&ctx->inflight_wait);
1439	spin_unlock_irqrestore(&ctx->inflight_lock, flags);	1554	spin_unlock_irqrestore(&ctx->inflight_lock, flags);
1440	}	1555	}
1441
1442	percpu_ref_put(&req->ctx->refs);
1443	if (likely(!io_is_fallback_req(req)))
1444	kmem_cache_free(req_cachep, req);
1445	else
1446	clear_bit_unlock(0, (unsigned long *) &req->ctx->fallback_req);
1447	}	1556	}
1448		1557
1449	struct req_batch {	1558	static void __io_free_req(struct io_kiocb *req)
1450	void *reqs[IO_IOPOLL_BATCH];
1451	int to_free;
1452	int need_iter;
1453	};
1454
1455	static void io_free_req_many(struct io_ring_ctx ctx, struct req_batch rb)
1456	{	1559	{
1457	if (!rb->to_free)	1560	struct io_ring_ctx *ctx;
1458	return;
1459	if (rb->need_iter) {
1460	int i, inflight = 0;
1461	unsigned long flags;
1462
1463	for (i = 0; i < rb->to_free; i++) {
1464	struct io_kiocb *req = rb->reqs[i];
1465
1466	if (req->flags & REQ_F_INFLIGHT)
1467	inflight++;
1468	__io_req_aux_free(req);
1469	}
1470	if (!inflight)
1471	goto do_free;
1472
1473	spin_lock_irqsave(&ctx->inflight_lock, flags);
1474	for (i = 0; i < rb->to_free; i++) {
1475	struct io_kiocb *req = rb->reqs[i];
1476
1477	if (req->flags & REQ_F_INFLIGHT) {
1478	list_del(&req->inflight_entry);
1479	if (!--inflight)
1480	break;
1481	}
1482	}
1483	spin_unlock_irqrestore(&ctx->inflight_lock, flags);
1484		1561
1485	if (waitqueue_active(&ctx->inflight_wait))	1562	io_dismantle_req(req);
1486	wake_up(&ctx->inflight_wait);	1563	__io_put_req_task(req);
1487	}	1564	ctx = req->ctx;
1488	do_free:	1565	if (likely(!io_is_fallback_req(req)))
1489	kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs);	1566	kmem_cache_free(req_cachep, req);
1490	percpu_ref_put_many(&ctx->refs, rb->to_free);	1567	else
1491	rb->to_free = rb->need_iter = 0;	1568	clear_bit_unlock(0, (unsigned long *) &ctx->fallback_req);
		1569	percpu_ref_put(&ctx->refs);
1492	}	1570	}
1493		1571
1494	static bool io_link_cancel_timeout(struct io_kiocb *req)	1572	static bool io_link_cancel_timeout(struct io_kiocb *req)
@@ -1508,53 +1586,67 @@ static bool io_link_cancel_timeout(struct io_kiocb *req) · 13 unmodified lines
1508	return false;	1586	return false;
1509	}	1587	}
1510		1588
1511	static void io_req_link_next(struct io_kiocb req, struct io_kiocb *nxtptr)	1589	static bool __io_kill_linked_timeout(struct io_kiocb *req)
		1590	{
		1591	struct io_kiocb *link;
		1592	bool wake_ev;
		1593
		1594	if (list_empty(&req->link_list))
		1595	return false;
		1596	link = list_first_entry(&req->link_list, struct io_kiocb, link_list);
		1597	if (link->opcode != IORING_OP_LINK_TIMEOUT)
		1598	return false;
		1599
		1600	list_del_init(&link->link_list);
		1601	wake_ev = io_link_cancel_timeout(link);
		1602	req->flags &= ~REQ_F_LINK_TIMEOUT;
		1603	return wake_ev;
		1604	}
		1605
		1606	static void io_kill_linked_timeout(struct io_kiocb *req)
1512	{	1607	{
1513	struct io_ring_ctx *ctx = req->ctx;	1608	struct io_ring_ctx *ctx = req->ctx;
1514	bool wake_ev = false;	1609	bool wake_ev;
1515		1610
1516	/* Already got next link */	1611	if (!(req->flags & REQ_F_COMP_LOCKED)) {
1517	if (req->flags & REQ_F_LINK_NEXT)	1612	unsigned long flags;
1518	return;	1613
		1614	spin_lock_irqsave(&ctx->completion_lock, flags);
		1615	wake_ev = __io_kill_linked_timeout(req);
		1616	spin_unlock_irqrestore(&ctx->completion_lock, flags);
		1617	} else {
		1618	wake_ev = __io_kill_linked_timeout(req);
		1619	}
		1620
		1621	if (wake_ev)
		1622	io_cqring_ev_posted(ctx);
		1623	}
		1624
		1625	static struct io_kiocb io_req_link_next(struct io_kiocb req)
		1626	{
		1627	struct io_kiocb *nxt;
1519		1628
1520	/*	1629	/*
1521	* The list should never be empty when we are called here. But could	1630	* The list should never be empty when we are called here. But could
1522	* potentially happen if the chain is messed up, check to be on the	1631	* potentially happen if the chain is messed up, check to be on the
1523	* safe side.	1632	* safe side.
1524	*/	1633	*/
1525	while (!list_empty(&req->link_list)) {	1634	if (unlikely(list_empty(&req->link_list)))
1526	struct io_kiocb *nxt = list_first_entry(&req->link_list,	1635	return NULL;
1527	struct io_kiocb, link_list);
1528
1529	if (unlikely((req->flags & REQ_F_LINK_TIMEOUT) &&
1530	(nxt->flags & REQ_F_TIMEOUT))) {
1531	list_del_init(&nxt->link_list);
1532	wake_ev \|= io_link_cancel_timeout(nxt);
1533	req->flags &= ~REQ_F_LINK_TIMEOUT;
1534	continue;
1535	}
1536
1537	list_del_init(&req->link_list);
1538	if (!list_empty(&nxt->link_list))
1539	nxt->flags \|= REQ_F_LINK_HEAD;
1540	*nxtptr = nxt;
1541	break;
1542	}
1543		1636
1544	req->flags \|= REQ_F_LINK_NEXT;	1637	nxt = list_first_entry(&req->link_list, struct io_kiocb, link_list);
1545	if (wake_ev)	1638	list_del_init(&req->link_list);
1546	io_cqring_ev_posted(ctx);	1639	if (!list_empty(&nxt->link_list))
		1640	nxt->flags \|= REQ_F_LINK_HEAD;
		1641	return nxt;
1547	}	1642	}
1548		1643
1549	/*	1644	/*
1550	* Called if REQ_F_LINK_HEAD is set, and we fail the head request	1645	* Called if REQ_F_LINK_HEAD is set, and we fail the head request
1551	*/	1646	*/
1552	static void io_fail_links(struct io_kiocb *req)	1647	static void __io_fail_links(struct io_kiocb *req)
1553	{	1648	{
1554	struct io_ring_ctx *ctx = req->ctx;	1649	struct io_ring_ctx *ctx = req->ctx;
1555	unsigned long flags;
1556
1557	spin_lock_irqsave(&ctx->completion_lock, flags);
1558		1650
1559	while (!list_empty(&req->link_list)) {	1651	while (!list_empty(&req->link_list)) {
1560	struct io_kiocb *link = list_first_entry(&req->link_list,	1652	struct io_kiocb *link = list_first_entry(&req->link_list,
@@ -1563,88 +1655,225 @@ static void io_fail_links(struct io_kiocb *req) · 2 unmodified lines
1563	list_del_init(&link->link_list);	1655	list_del_init(&link->link_list);
1564	trace_io_uring_fail_link(req, link);	1656	trace_io_uring_fail_link(req, link);
1565		1657
1566	if ((req->flags & REQ_F_LINK_TIMEOUT) &&	1658	io_cqring_fill_event(link, -ECANCELED);
1567	link->opcode == IORING_OP_LINK_TIMEOUT) {	1659	__io_double_put_req(link);
1568	io_link_cancel_timeout(link);
1569	} else {
1570	io_cqring_fill_event(link, -ECANCELED);
1571	__io_double_put_req(link);
1572	}
1573	req->flags &= ~REQ_F_LINK_TIMEOUT;	1660	req->flags &= ~REQ_F_LINK_TIMEOUT;
1574	}	1661	}
1575		1662
1576	io_commit_cqring(ctx);	1663	io_commit_cqring(ctx);
1577	spin_unlock_irqrestore(&ctx->completion_lock, flags);
1578	io_cqring_ev_posted(ctx);	1664	io_cqring_ev_posted(ctx);
1579	}	1665	}
1580		1666
1581	static void io_req_find_next(struct io_kiocb req, struct io_kiocb *nxt)	1667	static void io_fail_links(struct io_kiocb *req)
1582	{	1668	{
1583	if (likely(!(req->flags & REQ_F_LINK_HEAD)))	1669	struct io_ring_ctx *ctx = req->ctx;
1584	return;
1585		1670
1586	/*	1671	if (!(req->flags & REQ_F_COMP_LOCKED)) {
1587	* If LINK is set, we have dependent requests in this chain. If we
1588	* didn't fail this request, queue the first one up, moving any other
1589	* dependencies to the next request. In case of failure, fail the rest
1590	* of the chain.
1591	*/
1592	if (req->flags & REQ_F_FAIL_LINK) {
1593	io_fail_links(req);
1594	} else if ((req->flags & (REQ_F_LINK_TIMEOUT \| REQ_F_COMP_LOCKED)) ==
1595	REQ_F_LINK_TIMEOUT) {
1596	struct io_ring_ctx *ctx = req->ctx;
1597	unsigned long flags;	1672	unsigned long flags;
1598		1673
1599	/*
1600	* If this is a timeout link, we could be racing with the
1601	* timeout timer. Grab the completion lock for this case to
1602	* protect against that.
1603	*/
1604	spin_lock_irqsave(&ctx->completion_lock, flags);	1674	spin_lock_irqsave(&ctx->completion_lock, flags);
1605	io_req_link_next(req, nxt);	1675	__io_fail_links(req);
1606	spin_unlock_irqrestore(&ctx->completion_lock, flags);	1676	spin_unlock_irqrestore(&ctx->completion_lock, flags);
1607	} else {	1677	} else {
1608	io_req_link_next(req, nxt);	1678	__io_fail_links(req);
1609	}	1679	}
1610	}
1611		1680
1612	static void io_free_req(struct io_kiocb *req)	1681	io_cqring_ev_posted(ctx);
		1682	}
		1683
		1684	static struct io_kiocb __io_req_find_next(struct io_kiocb req)
1613	{	1685	{
1614	struct io_kiocb *nxt = NULL;	1686	req->flags &= ~REQ_F_LINK_HEAD;
		1687	if (req->flags & REQ_F_LINK_TIMEOUT)
		1688	io_kill_linked_timeout(req);
1615		1689
1616	io_req_find_next(req, &nxt);	1690	/*
1617	__io_free_req(req);	1691	* If LINK is set, we have dependent requests in this chain. If we
		1692	* didn't fail this request, queue the first one up, moving any other
		1693	* dependencies to the next request. In case of failure, fail the rest
		1694	* of the chain.
		1695	*/
		1696	if (likely(!(req->flags & REQ_F_FAIL_LINK)))
		1697	return io_req_link_next(req);
		1698	io_fail_links(req);
		1699	return NULL;
		1700	}
		1701
		1702	static struct io_kiocb io_req_find_next(struct io_kiocb req)
		1703	{
		1704	if (likely(!(req->flags & REQ_F_LINK_HEAD)))
		1705	return NULL;
		1706	return __io_req_find_next(req);
		1707	}
		1708
		1709	static int io_req_task_work_add(struct io_kiocb req, struct callback_head cb)
		1710	{
		1711	struct task_struct *tsk = req->task;
		1712	struct io_ring_ctx *ctx = req->ctx;
		1713	int ret, notify = TWA_RESUME;
		1714
		1715	/*
		1716	* SQPOLL kernel thread doesn't need notification, just a wakeup.
		1717	* If we're not using an eventfd, then TWA_RESUME is always fine,
		1718	* as we won't have dependencies between request completions for
		1719	* other kernel wait conditions.
		1720	*/
		1721	if (ctx->flags & IORING_SETUP_SQPOLL)
		1722	notify = 0;
		1723	else if (ctx->cq_ev_fd)
		1724	notify = TWA_SIGNAL;
		1725
		1726	ret = task_work_add(tsk, cb, notify);
		1727	if (!ret)
		1728	wake_up_process(tsk);
		1729	return ret;
		1730	}
		1731
		1732	static void __io_req_task_cancel(struct io_kiocb *req, int error)
		1733	{
		1734	struct io_ring_ctx *ctx = req->ctx;
		1735
		1736	spin_lock_irq(&ctx->completion_lock);
		1737	io_cqring_fill_event(req, error);
		1738	io_commit_cqring(ctx);
		1739	spin_unlock_irq(&ctx->completion_lock);
		1740
		1741	io_cqring_ev_posted(ctx);
		1742	req_set_fail_links(req);
		1743	io_double_put_req(req);
		1744	}
		1745
		1746	static void io_req_task_cancel(struct callback_head *cb)
		1747	{
		1748	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
		1749
		1750	__io_req_task_cancel(req, -ECANCELED);
		1751	}
		1752
		1753	static void __io_req_task_submit(struct io_kiocb *req)
		1754	{
		1755	struct io_ring_ctx *ctx = req->ctx;
		1756
		1757	if (!__io_sq_thread_acquire_mm(ctx)) {
		1758	mutex_lock(&ctx->uring_lock);
		1759	__io_queue_sqe(req, NULL, NULL);
		1760	mutex_unlock(&ctx->uring_lock);
		1761	} else {
		1762	__io_req_task_cancel(req, -EFAULT);
		1763	}
		1764	}
		1765
		1766	static void io_req_task_submit(struct callback_head *cb)
		1767	{
		1768	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
		1769
		1770	__io_req_task_submit(req);
		1771	}
		1772
		1773	static void io_req_task_queue(struct io_kiocb *req)
		1774	{
		1775	int ret;
		1776
		1777	init_task_work(&req->task_work, io_req_task_submit);
		1778
		1779	ret = io_req_task_work_add(req, &req->task_work);
		1780	if (unlikely(ret)) {
		1781	struct task_struct *tsk;
		1782
		1783	init_task_work(&req->task_work, io_req_task_cancel);
		1784	tsk = io_wq_get_task(req->ctx->io_wq);
		1785	task_work_add(tsk, &req->task_work, 0);
		1786	wake_up_process(tsk);
		1787	}
		1788	}
		1789
		1790	static void io_queue_next(struct io_kiocb *req)
		1791	{
		1792	struct io_kiocb *nxt = io_req_find_next(req);
1618		1793
1619	if (nxt)	1794	if (nxt)
1620	io_queue_async_work(nxt);	1795	io_req_task_queue(nxt);
1621	}	1796	}
1622		1797
1623	static void io_wq_assign_next(struct io_wq_work *workptr, struct io_kiocb nxt)	1798	static void io_free_req(struct io_kiocb *req)
1624	{	1799	{
1625	struct io_kiocb *link;	1800	io_queue_next(req);
1626	const struct io_op_def *def = &io_op_defs[nxt->opcode];	1801	__io_free_req(req);
		1802	}
1627		1803
1628	if ((nxt->flags & REQ_F_ISREG) && def->hash_reg_file)	1804	struct req_batch {
1629	io_wq_hash_work(&nxt->work, file_inode(nxt->file));	1805	void *reqs[IO_IOPOLL_BATCH];
		1806	int to_free;
1630		1807
1631	*workptr = &nxt->work;	1808	struct task_struct *task;
1632	link = io_prep_linked_timeout(nxt);	1809	int task_refs;
1633	if (link)	1810	};
1634	nxt->flags \|= REQ_F_QUEUE_TIMEOUT;	1811
		1812	static inline void io_init_req_batch(struct req_batch *rb)
		1813	{
		1814	rb->to_free = 0;
		1815	rb->task_refs = 0;
		1816	rb->task = NULL;
		1817	}
		1818
		1819	static void __io_req_free_batch_flush(struct io_ring_ctx *ctx,
		1820	struct req_batch *rb)
		1821	{
		1822	kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs);
		1823	percpu_ref_put_many(&ctx->refs, rb->to_free);
		1824	rb->to_free = 0;
		1825	}
		1826
		1827	static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
		1828	struct req_batch *rb)
		1829	{
		1830	if (rb->to_free)
		1831	__io_req_free_batch_flush(ctx, rb);
		1832	if (rb->task) {
		1833	put_task_struct_many(rb->task, rb->task_refs);
		1834	rb->task = NULL;
		1835	}
		1836	}
		1837
		1838	static void io_req_free_batch(struct req_batch rb, struct io_kiocb req)
		1839	{
		1840	if (unlikely(io_is_fallback_req(req))) {
		1841	io_free_req(req);
		1842	return;
		1843	}
		1844	if (req->flags & REQ_F_LINK_HEAD)
		1845	io_queue_next(req);
		1846
		1847	if (req->flags & REQ_F_TASK_PINNED) {
		1848	if (req->task != rb->task) {
		1849	if (rb->task)
		1850	put_task_struct_many(rb->task, rb->task_refs);
		1851	rb->task = req->task;
		1852	rb->task_refs = 0;
		1853	}
		1854	rb->task_refs++;
		1855	req->flags &= ~REQ_F_TASK_PINNED;
		1856	}
		1857
		1858	io_dismantle_req(req);
		1859	rb->reqs[rb->to_free++] = req;
		1860	if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs)))
		1861	__io_req_free_batch_flush(req->ctx, rb);
1635	}	1862	}
1636		1863
1637	/*	1864	/*
1638	* Drop reference to request, return next in chain (if there is one) if this	1865	* Drop reference to request, return next in chain (if there is one) if this
1639	* was the last reference to this request.	1866	* was the last reference to this request.
1640	*/	1867	*/
1641	__attribute__((nonnull))	1868	static struct io_kiocb io_put_req_find_next(struct io_kiocb req)
1642	static void io_put_req_find_next(struct io_kiocb req, struct io_kiocb *nxtptr)
1643	{	1869	{
		1870	struct io_kiocb *nxt = NULL;
		1871
1644	if (refcount_dec_and_test(&req->refs)) {	1872	if (refcount_dec_and_test(&req->refs)) {
1645	io_req_find_next(req, nxtptr);	1873	nxt = io_req_find_next(req);
1646	__io_free_req(req);	1874	__io_free_req(req);
1647	}	1875	}
		1876	return nxt;
1648	}	1877	}
1649		1878
1650	static void io_put_req(struct io_kiocb *req)	1879	static void io_put_req(struct io_kiocb *req)
@@ -1653,24 +1882,20 @@ static void io_put_req(struct io_kiocb *req) · 2 unmodified lines
1653	io_free_req(req);	1882	io_free_req(req);
1654	}	1883	}
1655		1884
1656	static void io_steal_work(struct io_kiocb *req,	1885	static struct io_wq_work io_steal_work(struct io_kiocb req)
1657	struct io_wq_work **workptr)
1658	{	1886	{
		1887	struct io_kiocb *nxt;
		1888
1659	/*	1889	/*
1660	* It's in an io-wq worker, so there always should be at least	1890	* A ref is owned by io-wq in which context we're. So, if that's the
1661	* one reference, which will be dropped in io_put_work() just	1891	* last one, it's safe to steal next work. False negatives are Ok,
1662	* after the current handler returns.	1892	* it just will be re-punted async in io_put_work()
1663	*
1664	* It also means, that if the counter dropped to 1, then there is
1665	* no asynchronous users left, so it's safe to steal the next work.
1666	*/	1893	*/
1667	if (refcount_read(&req->refs) == 1) {	1894	if (refcount_read(&req->refs) != 1)
1668	struct io_kiocb *nxt = NULL;	1895	return NULL;
1669		1896
1670	io_req_find_next(req, &nxt);	1897	nxt = io_req_find_next(req);
1671	if (nxt)	1898	return nxt ? &nxt->work : NULL;
1672	io_wq_assign_next(workptr, nxt);
1673	}
1674	}	1899	}
1675		1900
1676	/*	1901	/*
@@ -1720,31 +1945,34 @@ static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) · 43 unmodified lines
1720	return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;	1945	return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
1721	}	1946	}
1722		1947
1723	static inline bool io_req_multi_free(struct req_batch rb, struct io_kiocb req)	1948	static unsigned int io_put_kbuf(struct io_kiocb req, struct io_buffer kbuf)
1724	{	1949	{
1725	if ((req->flags & REQ_F_LINK_HEAD) \|\| io_is_fallback_req(req))	1950	unsigned int cflags;
1726	return false;
1727		1951
1728	if (req->file \|\| req->io)	1952	cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
1729	rb->need_iter++;	1953	cflags \|= IORING_CQE_F_BUFFER;
1730		1954	req->flags &= ~REQ_F_BUFFER_SELECTED;
1731	rb->reqs[rb->to_free++] = req;	1955	kfree(kbuf);
1732	if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs)))	1956	return cflags;
1733	io_free_req_many(req->ctx, rb);
1734	return true;
1735	}	1957	}
1736		1958
1737	static int io_put_kbuf(struct io_kiocb *req)	1959	static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req)
1738	{	1960	{
1739	struct io_buffer *kbuf;	1961	struct io_buffer *kbuf;
1740	int cflags;
1741		1962
1742	kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;	1963	kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
1743	cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;	1964	return io_put_kbuf(req, kbuf);
1744	cflags \|= IORING_CQE_F_BUFFER;	1965	}
1745	req->rw.addr = 0;	1966
1746	kfree(kbuf);	1967	static inline bool io_run_task_work(void)
1747	return cflags;	1968	{
		1969	if (current->task_works) {
		1970	__set_current_state(TASK_RUNNING);
		1971	task_work_run();
		1972	return true;
		1973	}
		1974
		1975	return false;
1748	}	1976	}
1749		1977
1750	static void io_iopoll_queue(struct list_head *again)	1978	static void io_iopoll_queue(struct list_head *again)
@@ -1752,18 +1980,9 @@ static void io_iopoll_queue(struct list_head *again) · 1 unmodified line
1752	struct io_kiocb *req;	1980	struct io_kiocb *req;
1753		1981
1754	do {	1982	do {
1755	req = list_first_entry(again, struct io_kiocb, list);	1983	req = list_first_entry(again, struct io_kiocb, inflight_entry);
1756	list_del(&req->list);	1984	list_del(&req->inflight_entry);
1757		1985	__io_complete_rw(req, -EAGAIN, 0, NULL);
1758	/* shouldn't happen unless io_uring is dying, cancel reqs */
1759	if (unlikely(!current->mm)) {
1760	io_complete_rw_common(&req->rw.kiocb, -EAGAIN);
1761	io_put_req(req);
1762	continue;
1763	}
1764
1765	refcount_inc(&req->refs);
1766	io_queue_async_work(req);
1767	} while (!list_empty(again));	1986	} while (!list_empty(again));
1768	}	1987	}
1769		1988
@@ -1780,33 +1999,32 @@ static void io_iopoll_complete(struct io_ring_ctx ctx, unsigned int nr_events, · 10 unmodified lines
1780	/* order with ->result store in io_complete_rw_iopoll() */	1999	/* order with ->result store in io_complete_rw_iopoll() */
1781	smp_rmb();	2000	smp_rmb();
1782		2001
1783	rb.to_free = rb.need_iter = 0;	2002	io_init_req_batch(&rb);
1784	while (!list_empty(done)) {	2003	while (!list_empty(done)) {
1785	int cflags = 0;	2004	int cflags = 0;
1786		2005
1787	req = list_first_entry(done, struct io_kiocb, list);	2006	req = list_first_entry(done, struct io_kiocb, inflight_entry);
1788	if (READ_ONCE(req->result) == -EAGAIN) {	2007	if (READ_ONCE(req->result) == -EAGAIN) {
1789	req->iopoll_completed = 0;	2008	req->iopoll_completed = 0;
1790	list_move_tail(&req->list, &again);	2009	list_move_tail(&req->inflight_entry, &again);
1791	continue;	2010	continue;
1792	}	2011	}
1793	list_del(&req->list);	2012	list_del(&req->inflight_entry);
1794		2013
1795	if (req->flags & REQ_F_BUFFER_SELECTED)	2014	if (req->flags & REQ_F_BUFFER_SELECTED)
1796	cflags = io_put_kbuf(req);	2015	cflags = io_put_rw_kbuf(req);
1797		2016
1798	__io_cqring_fill_event(req, req->result, cflags);	2017	__io_cqring_fill_event(req, req->result, cflags);
1799	(*nr_events)++;	2018	(*nr_events)++;
1800		2019
1801	if (refcount_dec_and_test(&req->refs) &&	2020	if (refcount_dec_and_test(&req->refs))
1802	!io_req_multi_free(&rb, req))	2021	io_req_free_batch(&rb, req);
1803	io_free_req(req);
1804	}	2022	}
1805		2023
1806	io_commit_cqring(ctx);	2024	io_commit_cqring(ctx);
1807	if (ctx->flags & IORING_SETUP_SQPOLL)	2025	if (ctx->flags & IORING_SETUP_SQPOLL)
1808	io_cqring_ev_posted(ctx);	2026	io_cqring_ev_posted(ctx);
1809	io_free_req_many(ctx, &rb);	2027	io_req_free_batch_finish(ctx, &rb);
1810		2028
1811	if (!list_empty(&again))	2029	if (!list_empty(&again))
1812	io_iopoll_queue(&again);	2030	io_iopoll_queue(&again);
@@ -1827,7 +2045,7 @@ static int io_do_iopoll(struct io_ring_ctx ctx, unsigned int nr_events, · 14 unmodified lines
1827	spin = !ctx->poll_multi_file && *nr_events < min;	2045	spin = !ctx->poll_multi_file && *nr_events < min;
1828		2046
1829	ret = 0;	2047	ret = 0;
1830	list_for_each_entry_safe(req, tmp, &ctx->poll_list, list) {	2048	list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) {
1831	struct kiocb *kiocb = &req->rw.kiocb;	2049	struct kiocb *kiocb = &req->rw.kiocb;
1832		2050
1833	/*	2051	/*
@@ -1836,7 +2054,7 @@ static int io_do_iopoll(struct io_ring_ctx ctx, unsigned int nr_events, · 2 unmodified lines
1836	* and complete those lists first, if we have entries there.	2054	* and complete those lists first, if we have entries there.
1837	*/	2055	*/
1838	if (READ_ONCE(req->iopoll_completed)) {	2056	if (READ_ONCE(req->iopoll_completed)) {
1839	list_move_tail(&req->list, &done);	2057	list_move_tail(&req->inflight_entry, &done);
1840	continue;	2058	continue;
1841	}	2059	}
1842	if (!list_empty(&done))	2060	if (!list_empty(&done))
@@ -1846,6 +2064,10 @@ static int io_do_iopoll(struct io_ring_ctx ctx, unsigned int nr_events, · 3 unmodified lines
1846	if (ret < 0)	2064	if (ret < 0)
1847	break;	2065	break;
1848		2066
		2067	/* iopoll may have completed current req */
		2068	if (READ_ONCE(req->iopoll_completed))
		2069	list_move_tail(&req->inflight_entry, &done);
		2070
1849	if (ret && spin)	2071	if (ret && spin)
1850	spin = false;	2072	spin = false;
1851	ret = 0;	2073	ret = 0;
@@ -1865,13 +2087,13 @@ static int io_do_iopoll(struct io_ring_ctx ctx, unsigned int nr_events, · 13 unmodified lines
1865	static int io_iopoll_getevents(struct io_ring_ctx ctx, unsigned int nr_events,	2087	static int io_iopoll_getevents(struct io_ring_ctx ctx, unsigned int nr_events,
1866	long min)	2088	long min)
1867	{	2089	{
1868	while (!list_empty(&ctx->poll_list) && !need_resched()) {	2090	while (!list_empty(&ctx->iopoll_list) && !need_resched()) {
1869	int ret;	2091	int ret;
1870		2092
1871	ret = io_do_iopoll(ctx, nr_events, min);	2093	ret = io_do_iopoll(ctx, nr_events, min);
1872	if (ret < 0)	2094	if (ret < 0)
1873	return ret;	2095	return ret;
1874	if (!min \|\| *nr_events >= min)	2096	if (*nr_events >= min)
1875	return 0;	2097	return 0;
1876	}	2098	}
1877		2099
@@ -1882,29 +2104,37 @@ static int io_iopoll_getevents(struct io_ring_ctx ctx, unsigned int nr_events, · 4 unmodified lines
1882	* We can't just wait for polled events to come to us, we have to actively	2104	* We can't just wait for polled events to come to us, we have to actively
1883	* find and complete them.	2105	* find and complete them.
1884	*/	2106	*/
1885	static void io_iopoll_reap_events(struct io_ring_ctx *ctx)	2107	static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
1886	{	2108	{
1887	if (!(ctx->flags & IORING_SETUP_IOPOLL))	2109	if (!(ctx->flags & IORING_SETUP_IOPOLL))
1888	return;	2110	return;
1889		2111
1890	mutex_lock(&ctx->uring_lock);	2112	mutex_lock(&ctx->uring_lock);
1891	while (!list_empty(&ctx->poll_list)) {	2113	while (!list_empty(&ctx->iopoll_list)) {
1892	unsigned int nr_events = 0;	2114	unsigned int nr_events = 0;
1893		2115
1894	io_iopoll_getevents(ctx, &nr_events, 1);	2116	io_do_iopoll(ctx, &nr_events, 0);
1895		2117
		2118	/* let it sleep and repeat later if can't complete a request */
		2119	if (nr_events == 0)
		2120	break;
1896	/*	2121	/*
1897	* Ensure we allow local-to-the-cpu processing to take place,	2122	* Ensure we allow local-to-the-cpu processing to take place,
1898	* in this case we need to ensure that we reap all events.	2123	* in this case we need to ensure that we reap all events.
		2124	* Also let task_work, etc. to progress by releasing the mutex
1899	*/	2125	*/
1900	cond_resched();	2126	if (need_resched()) {
		2127	mutex_unlock(&ctx->uring_lock);
		2128	cond_resched();
		2129	mutex_lock(&ctx->uring_lock);
		2130	}
1901	}	2131	}
1902	mutex_unlock(&ctx->uring_lock);	2132	mutex_unlock(&ctx->uring_lock);
1903	}	2133	}
1904		2134
1905	static int io_iopoll_check(struct io_ring_ctx ctx, unsigned nr_events,	2135	static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
1906	long min)
1907	{	2136	{
		2137	unsigned int nr_events = 0;
1908	int iters = 0, ret = 0;	2138	int iters = 0, ret = 0;
1909		2139
1910	/*	2140	/*
@@ -1914,8 +2144,6 @@ static int io_iopoll_check(struct io_ring_ctx ctx, unsigned nr_events, · 3 unmodified lines
1914	*/	2144	*/
1915	mutex_lock(&ctx->uring_lock);	2145	mutex_lock(&ctx->uring_lock);
1916	do {	2146	do {
1917	int tmin = 0;
1918
1919	/*	2147	/*
1920	* Don't enter poll loop if we already have events pending.	2148	* Don't enter poll loop if we already have events pending.
1921	* If we do, we can potentially be spinning for commands that	2149	* If we do, we can potentially be spinning for commands that
@@ -1936,17 +2164,15 @@ static int io_iopoll_check(struct io_ring_ctx ctx, unsigned nr_events, · 14 unmodified lines
1936	*/	2164	*/
1937	if (!(++iters & 7)) {	2165	if (!(++iters & 7)) {
1938	mutex_unlock(&ctx->uring_lock);	2166	mutex_unlock(&ctx->uring_lock);
		2167	io_run_task_work();
1939	mutex_lock(&ctx->uring_lock);	2168	mutex_lock(&ctx->uring_lock);
1940	}	2169	}
1941		2170
1942	if (*nr_events < min)	2171	ret = io_iopoll_getevents(ctx, &nr_events, min);
1943	tmin = min - *nr_events;
1944
1945	ret = io_iopoll_getevents(ctx, nr_events, tmin);
1946	if (ret <= 0)	2172	if (ret <= 0)
1947	break;	2173	break;
1948	ret = 0;	2174	ret = 0;
1949	} while (min && !*nr_events && !need_resched());	2175	} while (min && !nr_events && !need_resched());
1950		2176
1951	mutex_unlock(&ctx->uring_lock);	2177	mutex_unlock(&ctx->uring_lock);
1952	return ret;	2178	return ret;
@@ -1966,13 +2192,8 @@ static void kiocb_end_write(struct io_kiocb *req) · 13 unmodified lines
1966	file_end_write(req->file);	2192	file_end_write(req->file);
1967	}	2193	}
1968		2194
1969	static inline void req_set_fail_links(struct io_kiocb *req)	2195	static void io_complete_rw_common(struct kiocb *kiocb, long res,
1970	{	2196	struct io_comp_state *cs)
1971	if ((req->flags & (REQ_F_LINK \| REQ_F_HARDLINK)) == REQ_F_LINK)
1972	req->flags \|= REQ_F_FAIL_LINK;
1973	}
1974
1975	static void io_complete_rw_common(struct kiocb *kiocb, long res)
1976	{	2197	{
1977	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);	2198	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1978	int cflags = 0;	2199	int cflags = 0;
@@ -1983,16 +2204,96 @@ static void io_complete_rw_common(struct kiocb *kiocb, long res) · 4 unmodified lines
1983	if (res != req->result)	2204	if (res != req->result)
1984	req_set_fail_links(req);	2205	req_set_fail_links(req);
1985	if (req->flags & REQ_F_BUFFER_SELECTED)	2206	if (req->flags & REQ_F_BUFFER_SELECTED)
1986	cflags = io_put_kbuf(req);	2207	cflags = io_put_rw_kbuf(req);
1987	__io_cqring_add_event(req, res, cflags);	2208	__io_req_complete(req, res, cflags, cs);
		2209	}
		2210
		2211	#ifdef CONFIG_BLOCK
		2212	static bool io_resubmit_prep(struct io_kiocb *req, int error)
		2213	{
		2214	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
		2215	ssize_t ret = -ECANCELED;
		2216	struct iov_iter iter;
		2217	int rw;
		2218
		2219	if (error) {
		2220	ret = error;
		2221	goto end_req;
		2222	}
		2223
		2224	switch (req->opcode) {
		2225	case IORING_OP_READV:
		2226	case IORING_OP_READ_FIXED:
		2227	case IORING_OP_READ:
		2228	rw = READ;
		2229	break;
		2230	case IORING_OP_WRITEV:
		2231	case IORING_OP_WRITE_FIXED:
		2232	case IORING_OP_WRITE:
		2233	rw = WRITE;
		2234	break;
		2235	default:
		2236	printk_once(KERN_WARNING "io_uring: bad opcode in resubmit %d\n",
		2237	req->opcode);
		2238	goto end_req;
		2239	}
		2240
		2241	ret = io_import_iovec(rw, req, &iovec, &iter, false);
		2242	if (ret < 0)
		2243	goto end_req;
		2244	ret = io_setup_async_rw(req, ret, iovec, inline_vecs, &iter);
		2245	if (!ret)
		2246	return true;
		2247	kfree(iovec);
		2248	end_req:
		2249	req_set_fail_links(req);
		2250	io_req_complete(req, ret);
		2251	return false;
		2252	}
		2253
		2254	static void io_rw_resubmit(struct callback_head *cb)
		2255	{
		2256	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
		2257	struct io_ring_ctx *ctx = req->ctx;
		2258	int err;
		2259
		2260	err = io_sq_thread_acquire_mm(ctx, req);
		2261
		2262	if (io_resubmit_prep(req, err)) {
		2263	refcount_inc(&req->refs);
		2264	io_queue_async_work(req);
		2265	}
		2266	}
		2267	#endif
		2268
		2269	static bool io_rw_reissue(struct io_kiocb *req, long res)
		2270	{
		2271	#ifdef CONFIG_BLOCK
		2272	int ret;
		2273
		2274	if ((res != -EAGAIN && res != -EOPNOTSUPP) \|\| io_wq_current_is_worker())
		2275	return false;
		2276
		2277	init_task_work(&req->task_work, io_rw_resubmit);
		2278	ret = io_req_task_work_add(req, &req->task_work);
		2279	if (!ret)
		2280	return true;
		2281	#endif
		2282	return false;
		2283	}
		2284
		2285	static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
		2286	struct io_comp_state *cs)
		2287	{
		2288	if (!io_rw_reissue(req, res))
		2289	io_complete_rw_common(&req->rw.kiocb, res, cs);
1988	}	2290	}
1989		2291
1990	static void io_complete_rw(struct kiocb *kiocb, long res, long res2)	2292	static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
1991	{	2293	{
1992	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);	2294	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1993		2295
1994	io_complete_rw_common(kiocb, res);	2296	__io_complete_rw(req, res, res2, NULL);
1995	io_put_req(req);
1996	}	2297	}
1997		2298
1998	static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)	2299	static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
@@ -2026,13 +2327,13 @@ static void io_iopoll_req_issued(struct io_kiocb *req) · 27 unmodified lines
2026	* how we do polling eventually, not spinning if we're on potentially	2327	* how we do polling eventually, not spinning if we're on potentially
2027	* different devices.	2328	* different devices.
2028	*/	2329	*/
2029	if (list_empty(&ctx->poll_list)) {	2330	if (list_empty(&ctx->iopoll_list)) {
2030	ctx->poll_multi_file = false;	2331	ctx->poll_multi_file = false;
2031	} else if (!ctx->poll_multi_file) {	2332	} else if (!ctx->poll_multi_file) {
2032	struct io_kiocb *list_req;	2333	struct io_kiocb *list_req;
2033		2334
2034	list_req = list_first_entry(&ctx->poll_list, struct io_kiocb,	2335	list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb,
2035	list);	2336	inflight_entry);
2036	if (list_req->file != req->file)	2337	if (list_req->file != req->file)
2037	ctx->poll_multi_file = true;	2338	ctx->poll_multi_file = true;
2038	}	2339	}
@@ -2042,9 +2343,9 @@ static void io_iopoll_req_issued(struct io_kiocb *req) · 3 unmodified lines
2042	* it to the front so we find it first.	2343	* it to the front so we find it first.
2043	*/	2344	*/
2044	if (READ_ONCE(req->iopoll_completed))	2345	if (READ_ONCE(req->iopoll_completed))
2045	list_add(&req->list, &ctx->poll_list);	2346	list_add(&req->inflight_entry, &ctx->iopoll_list);
2046	else	2347	else
2047	list_add_tail(&req->list, &ctx->poll_list);	2348	list_add_tail(&req->inflight_entry, &ctx->iopoll_list);
2048		2349
2049	if ((ctx->flags & IORING_SETUP_SQPOLL) &&	2350	if ((ctx->flags & IORING_SETUP_SQPOLL) &&
2050	wq_has_sleeper(&ctx->sqo_wait))	2351	wq_has_sleeper(&ctx->sqo_wait))
@@ -2053,10 +2354,8 @@ static void io_iopoll_req_issued(struct io_kiocb *req) · 2 unmodified lines
2053		2354
2054	static void __io_state_file_put(struct io_submit_state *state)	2355	static void __io_state_file_put(struct io_submit_state *state)
2055	{	2356	{
2056	int diff = state->has_refs - state->used_refs;	2357	if (state->has_refs)
2057		2358	fput_many(state->file, state->has_refs);
2058	if (diff)
2059	fput_many(state->file, diff);
2060	state->file = NULL;	2359	state->file = NULL;
2061	}	2360	}
2062		2361
@@ -2078,7 +2377,7 @@ static struct file __io_file_get(struct io_submit_state state, int fd) · 15 unmodified lines
2078		2377
2079	if (state->file) {	2378	if (state->file) {
2080	if (state->fd == fd) {	2379	if (state->fd == fd) {
2081	state->used_refs++;	2380	state->has_refs--;
2082	state->ios_left--;	2381	state->ios_left--;
2083	return state->file;	2382	return state->file;
2084	}	2383	}
@@ -2089,12 +2388,20 @@ static struct file __io_file_get(struct io_submit_state state, int fd) · 4 unmodified lines
2089	return NULL;	2388	return NULL;
2090		2389
2091	state->fd = fd;	2390	state->fd = fd;
2092	state->has_refs = state->ios_left;
2093	state->used_refs = 1;
2094	state->ios_left--;	2391	state->ios_left--;
		2392	state->has_refs = state->ios_left;
2095	return state->file;	2393	return state->file;
2096	}	2394	}
2097		2395
		2396	static bool io_bdev_nowait(struct block_device *bdev)
		2397	{
		2398	#ifdef CONFIG_BLOCK
		2399	return !bdev \|\| queue_is_mq(bdev_get_queue(bdev));
		2400	#else
		2401	return true;
		2402	#endif
		2403	}
		2404
2098	/*	2405	/*
2099	* If we tracked the file through the SCM inflight mechanism, we could support	2406	* If we tracked the file through the SCM inflight mechanism, we could support
2100	* any file. For now, just ensure that anything potentially problematic is done	2407	* any file. For now, just ensure that anything potentially problematic is done
@@ -2104,10 +2411,19 @@ static bool io_file_supports_async(struct file *file, int rw) · 3 unmodified lines
2104	{	2411	{
2105	umode_t mode = file_inode(file)->i_mode;	2412	umode_t mode = file_inode(file)->i_mode;
2106		2413
2107	if (S_ISBLK(mode) \|\| S_ISCHR(mode) \|\| S_ISSOCK(mode))	2414	if (S_ISBLK(mode)) {
2108	return true;	2415	if (io_bdev_nowait(file->f_inode->i_bdev))
2109	if (S_ISREG(mode) && file->f_op != &io_uring_fops)	2416	return true;
		2417	return false;
		2418	}
		2419	if (S_ISCHR(mode) \|\| S_ISSOCK(mode))
2110	return true;	2420	return true;
		2421	if (S_ISREG(mode)) {
		2422	if (io_bdev_nowait(file->f_inode->i_sb->s_bdev) &&
		2423	file->f_op != &io_uring_fops)
		2424	return true;
		2425	return false;
		2426	}
2111		2427
2112	/* any ->read/write should understand O_NONBLOCK */	2428	/* any ->read/write should understand O_NONBLOCK */
2113	if (file->f_flags & O_NONBLOCK)	2429	if (file->f_flags & O_NONBLOCK)
@@ -2158,6 +2474,9 @@ static int io_prep_rw(struct io_kiocb req, const struct io_uring_sqe sqe, · 44 unmodified lines
2158	if (kiocb->ki_flags & IOCB_NOWAIT)	2474	if (kiocb->ki_flags & IOCB_NOWAIT)
2159	req->flags \|= REQ_F_NOWAIT;	2475	req->flags \|= REQ_F_NOWAIT;
2160		2476
		2477	if (kiocb->ki_flags & IOCB_DIRECT)
		2478	io_get_req_task(req);
		2479
2161	if (force_nonblock)	2480	if (force_nonblock)
2162	kiocb->ki_flags \|= IOCB_NOWAIT;	2481	kiocb->ki_flags \|= IOCB_NOWAIT;
2163		2482
@@ -2168,8 +2487,8 @@ static int io_prep_rw(struct io_kiocb req, const struct io_uring_sqe sqe, · 4 unmodified lines
2168		2487
2169	kiocb->ki_flags \|= IOCB_HIPRI;	2488	kiocb->ki_flags \|= IOCB_HIPRI;
2170	kiocb->ki_complete = io_complete_rw_iopoll;	2489	kiocb->ki_complete = io_complete_rw_iopoll;
2171	req->result = 0;
2172	req->iopoll_completed = 0;	2490	req->iopoll_completed = 0;
		2491	io_get_req_task(req);
2173	} else {	2492	} else {
2174	if (kiocb->ki_flags & IOCB_HIPRI)	2493	if (kiocb->ki_flags & IOCB_HIPRI)
2175	return -EINVAL;	2494	return -EINVAL;
@@ -2203,14 +2522,15 @@ static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret) · 27 unmodified lines
2203	}	2522	}
2204	}	2523	}
2205		2524
2206	static void kiocb_done(struct kiocb *kiocb, ssize_t ret)	2525	static void kiocb_done(struct kiocb *kiocb, ssize_t ret,
		2526	struct io_comp_state *cs)
2207	{	2527	{
2208	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);	2528	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
2209		2529
2210	if (req->flags & REQ_F_CUR_POS)	2530	if (req->flags & REQ_F_CUR_POS)
2211	req->file->f_pos = kiocb->ki_pos;	2531	req->file->f_pos = kiocb->ki_pos;
2212	if (ret >= 0 && kiocb->ki_complete == io_complete_rw)	2532	if (ret >= 0 && kiocb->ki_complete == io_complete_rw)
2213	io_complete_rw(kiocb, ret, 0);	2533	__io_complete_rw(req, ret, 0, cs);
2214	else	2534	else
2215	io_rw_done(kiocb, ret);	2535	io_rw_done(kiocb, ret);
2216	}	2536	}
@@ -2466,10 +2786,8 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req, · 249 unmodified lines
2466	if (req->io) {	2786	if (req->io) {
2467	struct io_async_rw *iorw = &req->io->rw;	2787	struct io_async_rw *iorw = &req->io->rw;
2468		2788
2469	*iovec = iorw->iov;	2789	iov_iter_init(iter, rw, iorw->iov, iorw->nr_segs, iorw->size);
2470	iov_iter_init(iter, rw, *iovec, iorw->nr_segs, iorw->size);	2790	*iovec = NULL;
2471	if (iorw->iov == iorw->fast_iov)
2472	*iovec = NULL;
2473	return iorw->size;	2791	return iorw->size;
2474	}	2792	}
2475		2793
@@ -2554,15 +2872,17 @@ static void io_req_map_rw(struct io_kiocb *req, ssize_t io_size, · 78 unmodified lines
2554	struct iovec iovec, struct iovec fast_iov,	2872	struct iovec iovec, struct iovec fast_iov,
2555	struct iov_iter *iter)	2873	struct iov_iter *iter)
2556	{	2874	{
2557	req->io->rw.nr_segs = iter->nr_segs;	2875	struct io_async_rw *rw = &req->io->rw;
2558	req->io->rw.size = io_size;	2876
2559	req->io->rw.iov = iovec;	2877	rw->nr_segs = iter->nr_segs;
2560	if (!req->io->rw.iov) {	2878	rw->size = io_size;
2561	req->io->rw.iov = req->io->rw.fast_iov;	2879	if (!iovec) {
2562	if (req->io->rw.iov != fast_iov)	2880	rw->iov = rw->fast_iov;
2563	memcpy(req->io->rw.iov, fast_iov,	2881	if (rw->iov != fast_iov)
		2882	memcpy(rw->iov, fast_iov,
2564	sizeof(struct iovec) * iter->nr_segs);	2883	sizeof(struct iovec) * iter->nr_segs);
2565	} else {	2884	} else {
		2885	rw->iov = iovec;
2566	req->flags \|= REQ_F_NEED_CLEANUP;	2886	req->flags \|= REQ_F_NEED_CLEANUP;
2567	}	2887	}
2568	}	2888	}
@@ -2593,95 +2913,205 @@ static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size, · 24 unmodified lines
2593		2913
2594	io_req_map_rw(req, io_size, iovec, fast_iov, iter);	2914	io_req_map_rw(req, io_size, iovec, fast_iov, iter);
2595	}	2915	}
2596	return 0;	2916	return 0;
		2917	}
		2918
		2919	static inline int io_rw_prep_async(struct io_kiocb *req, int rw,
		2920	bool force_nonblock)
		2921	{
		2922	struct io_async_ctx *io = req->io;
		2923	struct iov_iter iter;
		2924	ssize_t ret;
		2925
		2926	io->rw.iov = io->rw.fast_iov;
		2927	req->io = NULL;
		2928	ret = io_import_iovec(rw, req, &io->rw.iov, &iter, !force_nonblock);
		2929	req->io = io;
		2930	if (unlikely(ret < 0))
		2931	return ret;
		2932
		2933	io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
		2934	return 0;
		2935	}
		2936
		2937	static int io_read_prep(struct io_kiocb req, const struct io_uring_sqe sqe,
		2938	bool force_nonblock)
		2939	{
		2940	ssize_t ret;
		2941
		2942	ret = io_prep_rw(req, sqe, force_nonblock);
		2943	if (ret)
		2944	return ret;
		2945
		2946	if (unlikely(!(req->file->f_mode & FMODE_READ)))
		2947	return -EBADF;
		2948
		2949	/* either don't need iovec imported or already have it */
		2950	if (!req->io \|\| req->flags & REQ_F_NEED_CLEANUP)
		2951	return 0;
		2952	return io_rw_prep_async(req, READ, force_nonblock);
		2953	}
		2954
		2955	static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
		2956	int sync, void *arg)
		2957	{
		2958	struct wait_page_queue *wpq;
		2959	struct io_kiocb *req = wait->private;
		2960	struct wait_page_key *key = arg;
		2961	int ret;
		2962
		2963	wpq = container_of(wait, struct wait_page_queue, wait);
		2964
		2965	if (!wake_page_match(wpq, key))
		2966	return 0;
		2967
		2968	/* Stop waking things up if the page is locked again */
		2969	if (test_bit(key->bit_nr, &key->page->flags))
		2970	return -1;
		2971
		2972	list_del_init(&wait->entry);
		2973
		2974	init_task_work(&req->task_work, io_req_task_submit);
		2975	/* submit ref gets dropped, acquire a new one */
		2976	refcount_inc(&req->refs);
		2977	ret = io_req_task_work_add(req, &req->task_work);
		2978	if (unlikely(ret)) {
		2979	struct task_struct *tsk;
		2980
		2981	/* queue just for cancelation */
		2982	init_task_work(&req->task_work, io_req_task_cancel);
		2983	tsk = io_wq_get_task(req->ctx->io_wq);
		2984	task_work_add(tsk, &req->task_work, 0);
		2985	wake_up_process(tsk);
		2986	}
		2987	return 1;
		2988	}
		2989
		2990	static inline int kiocb_wait_page_queue_init(struct kiocb *kiocb,
		2991	struct wait_page_queue *wait,
		2992	wait_queue_func_t func,
		2993	void *data)
		2994	{
		2995	/* Can't support async wakeup with polled IO */
		2996	if (kiocb->ki_flags & IOCB_HIPRI)
		2997	return -EINVAL;
		2998	if (kiocb->ki_filp->f_mode & FMODE_BUF_RASYNC) {
		2999	wait->wait.func = func;
		3000	wait->wait.private = data;
		3001	wait->wait.flags = 0;
		3002	INIT_LIST_HEAD(&wait->wait.entry);
		3003	kiocb->ki_flags \|= IOCB_WAITQ;
		3004	kiocb->ki_waitq = wait;
		3005	return 0;
		3006	}
		3007
		3008	return -EOPNOTSUPP;
2597	}	3009	}
2598		3010
2599	static int io_read_prep(struct io_kiocb req, const struct io_uring_sqe sqe,	3011
2600	bool force_nonblock)	3012	static bool io_rw_should_retry(struct io_kiocb *req)
2601	{	3013	{
2602	struct io_async_ctx *io;	3014	struct kiocb *kiocb = &req->rw.kiocb;
2603	struct iov_iter iter;	3015	int ret;
2604	ssize_t ret;
2605		3016
2606	ret = io_prep_rw(req, sqe, force_nonblock);	3017	/* never retry for NOWAIT, we just complete with -EAGAIN */
2607	if (ret)	3018	if (req->flags & REQ_F_NOWAIT)
2608	return ret;	3019	return false;
2609		3020
2610	if (unlikely(!(req->file->f_mode & FMODE_READ)))	3021	/* already tried, or we're doing O_DIRECT */
2611	return -EBADF;	3022	if (kiocb->ki_flags & (IOCB_DIRECT \| IOCB_WAITQ))
		3023	return false;
		3024	/*
		3025	* just use poll if we can, and don't attempt if the fs doesn't
		3026	* support callback based unlocks
		3027	*/
		3028	if (file_can_poll(req->file) \|\| !(req->file->f_mode & FMODE_BUF_RASYNC))
		3029	return false;
2612		3030
2613	/* either don't need iovec imported or already have it */	3031	/*
2614	if (!req->io \|\| req->flags & REQ_F_NEED_CLEANUP)	3032	* If request type doesn't require req->io to defer in general,
2615	return 0;	3033	* we need to allocate it here
		3034	*/
		3035	if (!req->io && __io_alloc_async_ctx(req))
		3036	return false;
2616		3037
2617	io = req->io;	3038	ret = kiocb_wait_page_queue_init(kiocb, &req->io->rw.wpq,
2618	io->rw.iov = io->rw.fast_iov;	3039	io_async_buf_func, req);
2619	req->io = NULL;	3040	if (!ret) {
2620	ret = io_import_iovec(READ, req, &io->rw.iov, &iter, !force_nonblock);	3041	io_get_req_task(req);
2621	req->io = io;	3042	return true;
2622	if (ret < 0)	3043	}
2623	return ret;
2624		3044
2625	io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);	3045	return false;
2626	return 0;	3046	}
		3047
		3048	static int io_iter_do_read(struct io_kiocb req, struct iov_iter iter)
		3049	{
		3050	if (req->file->f_op->read_iter)
		3051	return call_read_iter(req->file, &req->rw.kiocb, iter);
		3052	return loop_rw_iter(READ, req->file, &req->rw.kiocb, iter);
2627	}	3053	}
2628		3054
2629	static int io_read(struct io_kiocb *req, bool force_nonblock)	3055	static int io_read(struct io_kiocb *req, bool force_nonblock,
		3056	struct io_comp_state *cs)
2630	{	3057	{
2631	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;	3058	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
2632	struct kiocb *kiocb = &req->rw.kiocb;	3059	struct kiocb *kiocb = &req->rw.kiocb;
2633	struct iov_iter iter;	3060	struct iov_iter iter;
2634	size_t iov_count;	3061	size_t iov_count;
2635	ssize_t io_size, ret;	3062	ssize_t io_size, ret, ret2;
		3063	unsigned long nr_segs;
2636		3064
2637	ret = io_import_iovec(READ, req, &iovec, &iter, !force_nonblock);	3065	ret = io_import_iovec(READ, req, &iovec, &iter, !force_nonblock);
2638	if (ret < 0)	3066	if (ret < 0)
2639	return ret;	3067	return ret;
		3068	io_size = ret;
		3069	req->result = io_size;
2640		3070
2641	/* Ensure we clear previously set non-block flag */	3071	/* Ensure we clear previously set non-block flag */
2642	if (!force_nonblock)	3072	if (!force_nonblock)
2643	kiocb->ki_flags &= ~IOCB_NOWAIT;	3073	kiocb->ki_flags &= ~IOCB_NOWAIT;
2644		3074
2645	req->result = 0;	3075	/* If the file doesn't support async, just async punt */
2646	io_size = ret;
2647	if (req->flags & REQ_F_LINK_HEAD)
2648	req->result = io_size;
2649
2650	/*
2651	* If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
2652	* we know to async punt it even if it was opened O_NONBLOCK
2653	*/
2654	if (force_nonblock && !io_file_supports_async(req->file, READ))	3076	if (force_nonblock && !io_file_supports_async(req->file, READ))
2655	goto copy_iov;	3077	goto copy_iov;
2656		3078
2657	iov_count = iov_iter_count(&iter);	3079	iov_count = iov_iter_count(&iter);
		3080	nr_segs = iter.nr_segs;
2658	ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count);	3081	ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count);
2659	if (!ret) {	3082	if (unlikely(ret))
2660	ssize_t ret2;	3083	goto out_free;
2661		3084
2662	if (req->file->f_op->read_iter)	3085	ret2 = io_iter_do_read(req, &iter);
2663	ret2 = call_read_iter(req->file, kiocb, &iter);
2664	else
2665	ret2 = loop_rw_iter(READ, req->file, kiocb, &iter);
2666		3086
2667	/* Catch -EAGAIN return for forced non-blocking submission */	3087	/* Catch -EAGAIN return for forced non-blocking submission */
2668	if (!force_nonblock \|\| ret2 != -EAGAIN) {	3088	if (!force_nonblock \|\| (ret2 != -EAGAIN && ret2 != -EIO)) {
2669	kiocb_done(kiocb, ret2);	3089	kiocb_done(kiocb, ret2, cs);
2670	} else {	3090	} else {
		3091	iter.count = iov_count;
		3092	iter.nr_segs = nr_segs;
2671	copy_iov:	3093	copy_iov:
2672	ret = io_setup_async_rw(req, io_size, iovec,	3094	ret = io_setup_async_rw(req, io_size, iovec, inline_vecs,
2673	inline_vecs, &iter);	3095	&iter);
2674	if (ret)	3096	if (ret)
		3097	goto out_free;
		3098	/* it's copied and will be cleaned with ->io */
		3099	iovec = NULL;
		3100	/* if we can retry, do so with the callbacks armed */
		3101	if (io_rw_should_retry(req)) {
		3102	ret2 = io_iter_do_read(req, &iter);
		3103	if (ret2 == -EIOCBQUEUED) {
2675	goto out_free;	3104	goto out_free;
2676	/* any defer here is final, must blocking retry */	3105	} else if (ret2 != -EAGAIN) {
2677	if (!(req->flags & REQ_F_NOWAIT) &&	3106	kiocb_done(kiocb, ret2, cs);
2678	!file_can_poll(req->file))	3107	goto out_free;
2679	req->flags \|= REQ_F_MUST_PUNT;	3108	}
2680	return -EAGAIN;
2681	}	3109	}
		3110	kiocb->ki_flags &= ~IOCB_WAITQ;
		3111	return -EAGAIN;
2682	}	3112	}
2683	out_free:	3113	out_free:
2684	if (!(req->flags & REQ_F_NEED_CLEANUP))	3114	if (iovec)
2685	kfree(iovec);	3115	kfree(iovec);
2686	return ret;	3116	return ret;
2687	}	3117	}
@@ -2689,8 +3119,6 @@ out_free: · 1 unmodified line
2689	static int io_write_prep(struct io_kiocb req, const struct io_uring_sqe sqe,	3119	static int io_write_prep(struct io_kiocb req, const struct io_uring_sqe sqe,
2690	bool force_nonblock)	3120	bool force_nonblock)
2691	{	3121	{
2692	struct io_async_ctx *io;
2693	struct iov_iter iter;
2694	ssize_t ret;	3122	ssize_t ret;
2695		3123
2696	ret = io_prep_rw(req, sqe, force_nonblock);	3124	ret = io_prep_rw(req, sqe, force_nonblock);
@@ -2700,49 +3128,33 @@ static int io_write_prep(struct io_kiocb req, const struct io_uring_sqe sqe, · 3 unmodified lines
2700	if (unlikely(!(req->file->f_mode & FMODE_WRITE)))	3128	if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
2701	return -EBADF;	3129	return -EBADF;
2702		3130
2703	req->fsize = rlimit(RLIMIT_FSIZE);
2704
2705	/* either don't need iovec imported or already have it */	3131	/* either don't need iovec imported or already have it */
2706	if (!req->io \|\| req->flags & REQ_F_NEED_CLEANUP)	3132	if (!req->io \|\| req->flags & REQ_F_NEED_CLEANUP)
2707	return 0;	3133	return 0;
2708		3134	return io_rw_prep_async(req, WRITE, force_nonblock);
2709	io = req->io;
2710	io->rw.iov = io->rw.fast_iov;
2711	req->io = NULL;
2712	ret = io_import_iovec(WRITE, req, &io->rw.iov, &iter, !force_nonblock);
2713	req->io = io;
2714	if (ret < 0)
2715	return ret;
2716
2717	io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
2718	return 0;
2719	}	3135	}
2720		3136
2721	static int io_write(struct io_kiocb *req, bool force_nonblock)	3137	static int io_write(struct io_kiocb *req, bool force_nonblock,
		3138	struct io_comp_state *cs)
2722	{	3139	{
2723	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;	3140	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
2724	struct kiocb *kiocb = &req->rw.kiocb;	3141	struct kiocb *kiocb = &req->rw.kiocb;
2725	struct iov_iter iter;	3142	struct iov_iter iter;
2726	size_t iov_count;	3143	size_t iov_count;
2727	ssize_t ret, io_size;	3144	ssize_t ret, ret2, io_size;
		3145	unsigned long nr_segs;
2728		3146
2729	ret = io_import_iovec(WRITE, req, &iovec, &iter, !force_nonblock);	3147	ret = io_import_iovec(WRITE, req, &iovec, &iter, !force_nonblock);
2730	if (ret < 0)	3148	if (ret < 0)
2731	return ret;	3149	return ret;
		3150	io_size = ret;
		3151	req->result = io_size;
2732		3152
2733	/* Ensure we clear previously set non-block flag */	3153	/* Ensure we clear previously set non-block flag */
2734	if (!force_nonblock)	3154	if (!force_nonblock)
2735	req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;	3155	req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
2736		3156
2737	req->result = 0;	3157	/* If the file doesn't support async, just async punt */
2738	io_size = ret;
2739	if (req->flags & REQ_F_LINK_HEAD)
2740	req->result = io_size;
2741
2742	/*
2743	* If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
2744	* we know to async punt it even if it was opened O_NONBLOCK
2745	*/
2746	if (force_nonblock && !io_file_supports_async(req->file, WRITE))	3158	if (force_nonblock && !io_file_supports_async(req->file, WRITE))
2747	goto copy_iov;	3159	goto copy_iov;
2748		3160
@@ -2752,59 +3164,53 @@ static int io_write(struct io_kiocb *req, bool force_nonblock) · 3 unmodified lines
2752	goto copy_iov;	3164	goto copy_iov;
2753		3165
2754	iov_count = iov_iter_count(&iter);	3166	iov_count = iov_iter_count(&iter);
		3167	nr_segs = iter.nr_segs;
2755	ret = rw_verify_area(WRITE, req->file, &kiocb->ki_pos, iov_count);	3168	ret = rw_verify_area(WRITE, req->file, &kiocb->ki_pos, iov_count);
2756	if (!ret) {	3169	if (unlikely(ret))
2757	ssize_t ret2;	3170	goto out_free;
2758
2759	/*
2760	* Open-code file_start_write here to grab freeze protection,
2761	* which will be released by another thread in
2762	* io_complete_rw(). Fool lockdep by telling it the lock got
2763	* released so that it doesn't complain about the held lock when
2764	* we return to userspace.
2765	*/
2766	if (req->flags & REQ_F_ISREG) {
2767	__sb_start_write(file_inode(req->file)->i_sb,
2768	SB_FREEZE_WRITE, true);
2769	__sb_writers_release(file_inode(req->file)->i_sb,
2770	SB_FREEZE_WRITE);
2771	}
2772	kiocb->ki_flags \|= IOCB_WRITE;
2773
2774	if (!force_nonblock)
2775	current->signal->rlim[RLIMIT_FSIZE].rlim_cur = req->fsize;
2776		3171
2777	if (req->file->f_op->write_iter)	3172	/*
2778	ret2 = call_write_iter(req->file, kiocb, &iter);	3173	* Open-code file_start_write here to grab freeze protection,
2779	else	3174	* which will be released by another thread in
2780	ret2 = loop_rw_iter(WRITE, req->file, kiocb, &iter);	3175	* io_complete_rw(). Fool lockdep by telling it the lock got
		3176	* released so that it doesn't complain about the held lock when
		3177	* we return to userspace.
		3178	*/
		3179	if (req->flags & REQ_F_ISREG) {
		3180	__sb_start_write(file_inode(req->file)->i_sb,
		3181	SB_FREEZE_WRITE, true);
		3182	__sb_writers_release(file_inode(req->file)->i_sb,
		3183	SB_FREEZE_WRITE);
		3184	}
		3185	kiocb->ki_flags \|= IOCB_WRITE;
2781		3186
2782	if (!force_nonblock)	3187	if (req->file->f_op->write_iter)
2783	current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY;	3188	ret2 = call_write_iter(req->file, kiocb, &iter);
		3189	else
		3190	ret2 = loop_rw_iter(WRITE, req->file, kiocb, &iter);
2784		3191
2785	/*	3192	/*
2786	* Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just	3193	* Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
2787	* retry them without IOCB_NOWAIT.	3194	* retry them without IOCB_NOWAIT.
2788	*/	3195	*/
2789	if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))	3196	if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
2790	ret2 = -EAGAIN;	3197	ret2 = -EAGAIN;
2791	if (!force_nonblock \|\| ret2 != -EAGAIN) {	3198	if (!force_nonblock \|\| ret2 != -EAGAIN) {
2792	kiocb_done(kiocb, ret2);	3199	kiocb_done(kiocb, ret2, cs);
2793	} else {	3200	} else {
		3201	iter.count = iov_count;
		3202	iter.nr_segs = nr_segs;
2794	copy_iov:	3203	copy_iov:
2795	ret = io_setup_async_rw(req, io_size, iovec,	3204	ret = io_setup_async_rw(req, io_size, iovec, inline_vecs,
2796	inline_vecs, &iter);	3205	&iter);
2797	if (ret)	3206	if (ret)
2798	goto out_free;	3207	goto out_free;
2799	/* any defer here is final, must blocking retry */	3208	/* it's copied and will be cleaned with ->io */
2800	if (!(req->flags & REQ_F_NOWAIT) &&	3209	iovec = NULL;
2801	!file_can_poll(req->file))	3210	return -EAGAIN;
2802	req->flags \|= REQ_F_MUST_PUNT;
2803	return -EAGAIN;
2804	}
2805	}	3211	}
2806	out_free:	3212	out_free:
2807	if (!(req->flags & REQ_F_NEED_CLEANUP))	3213	if (iovec)
2808	kfree(iovec);	3214	kfree(iovec);
2809	return ret;	3215	return ret;
2810	}	3216	}
@@ -2870,10 +3276,9 @@ static int io_tee(struct io_kiocb *req, bool force_nonblock) · 59 unmodified lines
2870	io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));	3276	io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
2871	req->flags &= ~REQ_F_NEED_CLEANUP;	3277	req->flags &= ~REQ_F_NEED_CLEANUP;
2872		3278
2873	io_cqring_add_event(req, ret);
2874	if (ret != sp->len)	3279	if (ret != sp->len)
2875	req_set_fail_links(req);	3280	req_set_fail_links(req);
2876	io_put_req(req);	3281	io_req_complete(req, ret);
2877	return 0;	3282	return 0;
2878	}	3283	}
2879		3284
@@ -2907,25 +3312,23 @@ static int io_splice(struct io_kiocb *req, bool force_nonblock) · 27 unmodified lines
2907	io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));	3312	io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
2908	req->flags &= ~REQ_F_NEED_CLEANUP;	3313	req->flags &= ~REQ_F_NEED_CLEANUP;
2909		3314
2910	io_cqring_add_event(req, ret);
2911	if (ret != sp->len)	3315	if (ret != sp->len)
2912	req_set_fail_links(req);	3316	req_set_fail_links(req);
2913	io_put_req(req);	3317	io_req_complete(req, ret);
2914	return 0;	3318	return 0;
2915	}	3319	}
2916		3320
2917	/*	3321	/*
2918	* IORING_OP_NOP just posts a completion event, nothing else.	3322	* IORING_OP_NOP just posts a completion event, nothing else.
2919	*/	3323	*/
2920	static int io_nop(struct io_kiocb *req)	3324	static int io_nop(struct io_kiocb req, struct io_comp_state cs)
2921	{	3325	{
2922	struct io_ring_ctx *ctx = req->ctx;	3326	struct io_ring_ctx *ctx = req->ctx;
2923		3327
2924	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))	3328	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
2925	return -EINVAL;	3329	return -EINVAL;
2926		3330
2927	io_cqring_add_event(req, 0);	3331	__io_req_complete(req, 0, 0, cs);
2928	io_put_req(req);
2929	return 0;	3332	return 0;
2930	}	3333	}
2931		3334
@@ -2964,8 +3367,7 @@ static int io_fsync(struct io_kiocb *req, bool force_nonblock) · 32 unmodified lines
2964	req->sync.flags & IORING_FSYNC_DATASYNC);	3367	req->sync.flags & IORING_FSYNC_DATASYNC);
2965	if (ret < 0)	3368	if (ret < 0)
2966	req_set_fail_links(req);	3369	req_set_fail_links(req);
2967	io_cqring_add_event(req, ret);	3370	io_req_complete(req, ret);
2968	io_put_req(req);
2969	return 0;	3371	return 0;
2970	}	3372	}
2971		3373
@@ -2980,7 +3382,6 @@ static int io_fallocate_prep(struct io_kiocb *req, · 8 unmodified lines
2980	req->sync.off = READ_ONCE(sqe->off);	3382	req->sync.off = READ_ONCE(sqe->off);
2981	req->sync.len = READ_ONCE(sqe->addr);	3383	req->sync.len = READ_ONCE(sqe->addr);
2982	req->sync.mode = READ_ONCE(sqe->len);	3384	req->sync.mode = READ_ONCE(sqe->len);
2983	req->fsize = rlimit(RLIMIT_FSIZE);
2984	return 0;	3385	return 0;
2985	}	3386	}
2986		3387
@@ -2991,15 +3392,11 @@ static int io_fallocate(struct io_kiocb *req, bool force_nonblock) · 4 unmodified lines
2991	/* fallocate always requiring blocking context */	3392	/* fallocate always requiring blocking context */
2992	if (force_nonblock)	3393	if (force_nonblock)
2993	return -EAGAIN;	3394	return -EAGAIN;
2994
2995	current->signal->rlim[RLIMIT_FSIZE].rlim_cur = req->fsize;
2996	ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,	3395	ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
2997	req->sync.len);	3396	req->sync.len);
2998	current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY;
2999	if (ret < 0)	3397	if (ret < 0)
3000	req_set_fail_links(req);	3398	req_set_fail_links(req);
3001	io_cqring_add_event(req, ret);	3399	io_req_complete(req, ret);
3002	io_put_req(req);
3003	return 0;	3400	return 0;
3004	}	3401	}
3005		3402
@@ -3095,8 +3492,7 @@ err: · 89 unmodified lines
3095	req->flags &= ~REQ_F_NEED_CLEANUP;	3492	req->flags &= ~REQ_F_NEED_CLEANUP;
3096	if (ret < 0)	3493	if (ret < 0)
3097	req_set_fail_links(req);	3494	req_set_fail_links(req);
3098	io_cqring_add_event(req, ret);	3495	io_req_complete(req, ret);
3099	io_put_req(req);
3100	return 0;	3496	return 0;
3101	}	3497	}
3102		3498
@@ -3150,7 +3546,8 @@ static int __io_remove_buffers(struct io_ring_ctx ctx, struct io_buffer buf, · 47 unmodified lines
3150	return i;	3546	return i;
3151	}	3547	}
3152		3548
3153	static int io_remove_buffers(struct io_kiocb *req, bool force_nonblock)	3549	static int io_remove_buffers(struct io_kiocb *req, bool force_nonblock,
		3550	struct io_comp_state *cs)
3154	{	3551	{
3155	struct io_provide_buf *p = &req->pbuf;	3552	struct io_provide_buf *p = &req->pbuf;
3156	struct io_ring_ctx *ctx = req->ctx;	3553	struct io_ring_ctx *ctx = req->ctx;
@@ -3169,8 +3566,7 @@ static int io_remove_buffers(struct io_kiocb *req, bool force_nonblock) · 12 unmodified lines
3169	io_ring_submit_lock(ctx, !force_nonblock);	3566	io_ring_submit_lock(ctx, !force_nonblock);
3170	if (ret < 0)	3567	if (ret < 0)
3171	req_set_fail_links(req);	3568	req_set_fail_links(req);
3172	io_cqring_add_event(req, ret);	3569	__io_req_complete(req, ret, 0, cs);
3173	io_put_req(req);
3174	return 0;	3570	return 0;
3175	}	3571	}
3176		3572
@@ -3228,7 +3624,8 @@ static int io_add_buffers(struct io_provide_buf pbuf, struct io_buffer *head) · 51 unmodified lines
3228	return i ? i : -ENOMEM;	3624	return i ? i : -ENOMEM;
3229	}	3625	}
3230		3626
3231	static int io_provide_buffers(struct io_kiocb *req, bool force_nonblock)	3627	static int io_provide_buffers(struct io_kiocb *req, bool force_nonblock,
		3628	struct io_comp_state *cs)
3232	{	3629	{
3233	struct io_provide_buf *p = &req->pbuf;	3630	struct io_provide_buf *p = &req->pbuf;
3234	struct io_ring_ctx *ctx = req->ctx;	3631	struct io_ring_ctx *ctx = req->ctx;
@@ -3257,8 +3654,7 @@ out: · 22 unmodified lines
3257	io_ring_submit_unlock(ctx, !force_nonblock);	3654	io_ring_submit_unlock(ctx, !force_nonblock);
3258	if (ret < 0)	3655	if (ret < 0)
3259	req_set_fail_links(req);	3656	req_set_fail_links(req);
3260	io_cqring_add_event(req, ret);	3657	__io_req_complete(req, ret, 0, cs);
3261	io_put_req(req);
3262	return 0;	3658	return 0;
3263	}	3659	}
3264		3660
@@ -3289,7 +3685,8 @@ static int io_epoll_ctl_prep(struct io_kiocb *req, · 24 unmodified lines
3289	#endif	3685	#endif
3290	}	3686	}
3291		3687
3292	static int io_epoll_ctl(struct io_kiocb *req, bool force_nonblock)	3688	static int io_epoll_ctl(struct io_kiocb *req, bool force_nonblock,
		3689	struct io_comp_state *cs)
3293	{	3690	{
3294	#if defined(CONFIG_EPOLL)	3691	#if defined(CONFIG_EPOLL)
3295	struct io_epoll *ie = &req->epoll;	3692	struct io_epoll *ie = &req->epoll;
@@ -3301,8 +3698,7 @@ static int io_epoll_ctl(struct io_kiocb *req, bool force_nonblock) · 5 unmodified lines
3301		3698
3302	if (ret < 0)	3699	if (ret < 0)
3303	req_set_fail_links(req);	3700	req_set_fail_links(req);
3304	io_cqring_add_event(req, ret);	3701	__io_req_complete(req, ret, 0, cs);
3305	io_put_req(req);
3306	return 0;	3702	return 0;
3307	#else	3703	#else
3308	return -EOPNOTSUPP;	3704	return -EOPNOTSUPP;
@@ -3338,8 +3734,7 @@ static int io_madvise(struct io_kiocb *req, bool force_nonblock) · 29 unmodified lines
3338	ret = do_madvise(ma->addr, ma->len, ma->advice);	3734	ret = do_madvise(ma->addr, ma->len, ma->advice);
3339	if (ret < 0)	3735	if (ret < 0)
3340	req_set_fail_links(req);	3736	req_set_fail_links(req);
3341	io_cqring_add_event(req, ret);	3737	io_req_complete(req, ret);
3342	io_put_req(req);
3343	return 0;	3738	return 0;
3344	#else	3739	#else
3345	return -EOPNOTSUPP;	3740	return -EOPNOTSUPP;
@@ -3378,8 +3773,7 @@ static int io_fadvise(struct io_kiocb *req, bool force_nonblock) · 32 unmodified lines
3378	ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);	3773	ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
3379	if (ret < 0)	3774	if (ret < 0)
3380	req_set_fail_links(req);	3775	req_set_fail_links(req);
3381	io_cqring_add_event(req, ret);	3776	io_req_complete(req, ret);
3382	io_put_req(req);
3383	return 0;	3777	return 0;
3384	}	3778	}
3385		3779
@@ -3418,8 +3812,7 @@ static int io_statx(struct io_kiocb *req, bool force_nonblock) · 32 unmodified lines
3418		3812
3419	if (ret < 0)	3813	if (ret < 0)
3420	req_set_fail_links(req);	3814	req_set_fail_links(req);
3421	io_cqring_add_event(req, ret);	3815	io_req_complete(req, ret);
3422	io_put_req(req);
3423	return 0;	3816	return 0;
3424	}	3817	}
3425		3818
@@ -3450,7 +3843,8 @@ static int io_close_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 24 unmodified lines
3450	return 0;	3843	return 0;
3451	}	3844	}
3452		3845
3453	static int io_close(struct io_kiocb *req, bool force_nonblock)	3846	static int io_close(struct io_kiocb *req, bool force_nonblock,
		3847	struct io_comp_state *cs)
3454	{	3848	{
3455	struct io_close *close = &req->close;	3849	struct io_close *close = &req->close;
3456	int ret;	3850	int ret;
@@ -3464,8 +3858,10 @@ static int io_close(struct io_kiocb *req, bool force_nonblock) · 7 unmodified lines
3464		3858
3465	/* if the file has a flush method, be safe and punt to async */	3859	/* if the file has a flush method, be safe and punt to async */
3466	if (close->put_file->f_op->flush && force_nonblock) {	3860	if (close->put_file->f_op->flush && force_nonblock) {
		3861	/* was never set, but play safe */
		3862	req->flags &= ~REQ_F_NOWAIT;
3467	/* avoid grabbing files - we don't need the files */	3863	/* avoid grabbing files - we don't need the files */
3468	req->flags \|= REQ_F_NO_FILE_TABLE \| REQ_F_MUST_PUNT;	3864	req->flags \|= REQ_F_NO_FILE_TABLE;
3469	return -EAGAIN;	3865	return -EAGAIN;
3470	}	3866	}
3471		3867
@@ -3473,10 +3869,9 @@ static int io_close(struct io_kiocb *req, bool force_nonblock) · 1 unmodified line
3473	ret = filp_close(close->put_file, req->work.files);	3869	ret = filp_close(close->put_file, req->work.files);
3474	if (ret < 0)	3870	if (ret < 0)
3475	req_set_fail_links(req);	3871	req_set_fail_links(req);
3476	io_cqring_add_event(req, ret);
3477	fput(close->put_file);	3872	fput(close->put_file);
3478	close->put_file = NULL;	3873	close->put_file = NULL;
3479	io_put_req(req);	3874	__io_req_complete(req, ret, 0, cs);
3480	return 0;	3875	return 0;
3481	}	3876	}
3482		3877
@@ -3510,8 +3905,7 @@ static int io_sync_file_range(struct io_kiocb *req, bool force_nonblock) · 27 unmodified lines
3510	req->sync.flags);	3905	req->sync.flags);
3511	if (ret < 0)	3906	if (ret < 0)
3512	req_set_fail_links(req);	3907	req_set_fail_links(req);
3513	io_cqring_add_event(req, ret);	3908	io_req_complete(req, ret);
3514	io_put_req(req);
3515	return 0;	3909	return 0;
3516	}	3910	}
3517		3911
@@ -3531,6 +3925,15 @@ static int io_setup_async_msg(struct io_kiocb *req, · 13 unmodified lines
3531	return -EAGAIN;	3925	return -EAGAIN;
3532	}	3926	}
3533		3927
		3928	static int io_sendmsg_copy_hdr(struct io_kiocb *req,
		3929	struct io_async_msghdr *iomsg)
		3930	{
		3931	iomsg->iov = iomsg->fast_iov;
		3932	iomsg->msg.msg_name = &iomsg->addr;
		3933	return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg,
		3934	req->sr_msg.msg_flags, &iomsg->iov);
		3935	}
		3936
3534	static int io_sendmsg_prep(struct io_kiocb req, const struct io_uring_sqe sqe)	3937	static int io_sendmsg_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
3535	{	3938	{
3536	struct io_sr_msg *sr = &req->sr_msg;	3939	struct io_sr_msg *sr = &req->sr_msg;
@@ -3541,7 +3944,7 @@ static int io_sendmsg_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 4 unmodified lines
3541	return -EINVAL;	3944	return -EINVAL;
3542		3945
3543	sr->msg_flags = READ_ONCE(sqe->msg_flags);	3946	sr->msg_flags = READ_ONCE(sqe->msg_flags);
3544	sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));	3947	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
3545	sr->len = READ_ONCE(sqe->len);	3948	sr->len = READ_ONCE(sqe->len);
3546		3949
3547	#ifdef CONFIG_COMPAT	3950	#ifdef CONFIG_COMPAT
@@ -3555,136 +3958,126 @@ static int io_sendmsg_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 7 unmodified lines
3555	if (req->flags & REQ_F_NEED_CLEANUP)	3958	if (req->flags & REQ_F_NEED_CLEANUP)
3556	return 0;	3959	return 0;
3557		3960
3558	io->msg.msg.msg_name = &io->msg.addr;	3961	ret = io_sendmsg_copy_hdr(req, &io->msg);
3559	io->msg.iov = io->msg.fast_iov;
3560	ret = sendmsg_copy_msghdr(&io->msg.msg, sr->msg, sr->msg_flags,
3561	&io->msg.iov);
3562	if (!ret)	3962	if (!ret)
3563	req->flags \|= REQ_F_NEED_CLEANUP;	3963	req->flags \|= REQ_F_NEED_CLEANUP;
3564	return ret;	3964	return ret;
3565	}	3965	}
3566		3966
3567	static int io_sendmsg(struct io_kiocb *req, bool force_nonblock)	3967	static int io_sendmsg(struct io_kiocb *req, bool force_nonblock,
		3968	struct io_comp_state *cs)
3568	{	3969	{
3569	struct io_async_msghdr *kmsg = NULL;	3970	struct io_async_msghdr iomsg, *kmsg;
3570	struct socket *sock;	3971	struct socket *sock;
		3972	unsigned flags;
3571	int ret;	3973	int ret;
3572		3974
3573	sock = sock_from_file(req->file, &ret);	3975	sock = sock_from_file(req->file, &ret);
3574	if (sock) {	3976	if (unlikely(!sock))
3575	struct io_async_ctx io;	3977	return ret;
3576	unsigned flags;
3577
3578	if (req->io) {
3579	kmsg = &req->io->msg;
3580	kmsg->msg.msg_name = &req->io->msg.addr;
3581	/* if iov is set, it's allocated already */
3582	if (!kmsg->iov)
3583	kmsg->iov = kmsg->fast_iov;
3584	kmsg->msg.msg_iter.iov = kmsg->iov;
3585	} else {
3586	struct io_sr_msg *sr = &req->sr_msg;
3587
3588	kmsg = &io.msg;
3589	kmsg->msg.msg_name = &io.msg.addr;
3590		3978
3591	io.msg.iov = io.msg.fast_iov;	3979	if (req->io) {
3592	ret = sendmsg_copy_msghdr(&io.msg.msg, sr->msg,	3980	kmsg = &req->io->msg;
3593	sr->msg_flags, &io.msg.iov);	3981	kmsg->msg.msg_name = &req->io->msg.addr;
3594	if (ret)	3982	/* if iov is set, it's allocated already */
3595	return ret;	3983	if (!kmsg->iov)
3596	}	3984	kmsg->iov = kmsg->fast_iov;
		3985	kmsg->msg.msg_iter.iov = kmsg->iov;
		3986	} else {
		3987	ret = io_sendmsg_copy_hdr(req, &iomsg);
		3988	if (ret)
		3989	return ret;
		3990	kmsg = &iomsg;
		3991	}
3597		3992
3598	flags = req->sr_msg.msg_flags;	3993	flags = req->sr_msg.msg_flags;
3599	if (flags & MSG_DONTWAIT)	3994	if (flags & MSG_DONTWAIT)
3600	req->flags \|= REQ_F_NOWAIT;	3995	req->flags \|= REQ_F_NOWAIT;
3601	else if (force_nonblock)	3996	else if (force_nonblock)
3602	flags \|= MSG_DONTWAIT;	3997	flags \|= MSG_DONTWAIT;
3603		3998
3604	ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);	3999	ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
3605	if (force_nonblock && ret == -EAGAIN)	4000	if (force_nonblock && ret == -EAGAIN)
3606	return io_setup_async_msg(req, kmsg);	4001	return io_setup_async_msg(req, kmsg);
3607	if (ret == -ERESTARTSYS)	4002	if (ret == -ERESTARTSYS)
3608	ret = -EINTR;	4003	ret = -EINTR;
3609	}
3610		4004
3611	if (kmsg && kmsg->iov != kmsg->fast_iov)	4005	if (kmsg->iov != kmsg->fast_iov)
3612	kfree(kmsg->iov);	4006	kfree(kmsg->iov);
3613	req->flags &= ~REQ_F_NEED_CLEANUP;	4007	req->flags &= ~REQ_F_NEED_CLEANUP;
3614	io_cqring_add_event(req, ret);
3615	if (ret < 0)	4008	if (ret < 0)
3616	req_set_fail_links(req);	4009	req_set_fail_links(req);
3617	io_put_req(req);	4010	__io_req_complete(req, ret, 0, cs);
3618	return 0;	4011	return 0;
3619	}	4012	}
3620		4013
3621	static int io_send(struct io_kiocb *req, bool force_nonblock)	4014	static int io_send(struct io_kiocb *req, bool force_nonblock,
		4015	struct io_comp_state *cs)
3622	{	4016	{
		4017	struct io_sr_msg *sr = &req->sr_msg;
		4018	struct msghdr msg;
		4019	struct iovec iov;
3623	struct socket *sock;	4020	struct socket *sock;
		4021	unsigned flags;
3624	int ret;	4022	int ret;
3625		4023
3626	sock = sock_from_file(req->file, &ret);	4024	sock = sock_from_file(req->file, &ret);
3627	if (sock) {	4025	if (unlikely(!sock))
3628	struct io_sr_msg *sr = &req->sr_msg;	4026	return ret;
3629	struct msghdr msg;
3630	struct iovec iov;
3631	unsigned flags;
3632		4027
3633	ret = import_single_range(WRITE, sr->buf, sr->len, &iov,	4028	ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
3634	&msg.msg_iter);	4029	if (unlikely(ret))
3635	if (ret)	4030	return ret;;
3636	return ret;
3637		4031
3638	msg.msg_name = NULL;	4032	msg.msg_name = NULL;
3639	msg.msg_control = NULL;	4033	msg.msg_control = NULL;
3640	msg.msg_controllen = 0;	4034	msg.msg_controllen = 0;
3641	msg.msg_namelen = 0;	4035	msg.msg_namelen = 0;
3642		4036
3643	flags = req->sr_msg.msg_flags;	4037	flags = req->sr_msg.msg_flags;
3644	if (flags & MSG_DONTWAIT)	4038	if (flags & MSG_DONTWAIT)
3645	req->flags \|= REQ_F_NOWAIT;	4039	req->flags \|= REQ_F_NOWAIT;
3646	else if (force_nonblock)	4040	else if (force_nonblock)
3647	flags \|= MSG_DONTWAIT;	4041	flags \|= MSG_DONTWAIT;
3648		4042
3649	msg.msg_flags = flags;	4043	msg.msg_flags = flags;
3650	ret = sock_sendmsg(sock, &msg);	4044	ret = sock_sendmsg(sock, &msg);
3651	if (force_nonblock && ret == -EAGAIN)	4045	if (force_nonblock && ret == -EAGAIN)
3652	return -EAGAIN;	4046	return -EAGAIN;
3653	if (ret == -ERESTARTSYS)	4047	if (ret == -ERESTARTSYS)
3654	ret = -EINTR;	4048	ret = -EINTR;
3655	}
3656		4049
3657	io_cqring_add_event(req, ret);
3658	if (ret < 0)	4050	if (ret < 0)
3659	req_set_fail_links(req);	4051	req_set_fail_links(req);
3660	io_put_req(req);	4052	__io_req_complete(req, ret, 0, cs);
3661	return 0;	4053	return 0;
3662	}	4054	}
3663		4055
3664	static int __io_recvmsg_copy_hdr(struct io_kiocb req, struct io_async_ctx io)	4056	static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
		4057	struct io_async_msghdr *iomsg)
3665	{	4058	{
3666	struct io_sr_msg *sr = &req->sr_msg;	4059	struct io_sr_msg *sr = &req->sr_msg;
3667	struct iovec __user *uiov;	4060	struct iovec __user *uiov;
3668	size_t iov_len;	4061	size_t iov_len;
3669	int ret;	4062	int ret;
3670		4063
3671	ret = __copy_msghdr_from_user(&io->msg.msg, sr->msg, &io->msg.uaddr,	4064	ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg,
3672	&uiov, &iov_len);	4065	&iomsg->uaddr, &uiov, &iov_len);
3673	if (ret)	4066	if (ret)
3674	return ret;	4067	return ret;
3675		4068
3676	if (req->flags & REQ_F_BUFFER_SELECT) {	4069	if (req->flags & REQ_F_BUFFER_SELECT) {
3677	if (iov_len > 1)	4070	if (iov_len > 1)
3678	return -EINVAL;	4071	return -EINVAL;
3679	if (copy_from_user(io->msg.iov, uiov, sizeof(*uiov)))	4072	if (copy_from_user(iomsg->iov, uiov, sizeof(*uiov)))
3680	return -EFAULT;	4073	return -EFAULT;
3681	sr->len = io->msg.iov[0].iov_len;	4074	sr->len = iomsg->iov[0].iov_len;
3682	iov_iter_init(&io->msg.msg.msg_iter, READ, io->msg.iov, 1,	4075	iov_iter_init(&iomsg->msg.msg_iter, READ, iomsg->iov, 1,
3683	sr->len);	4076	sr->len);
3684	io->msg.iov = NULL;	4077	iomsg->iov = NULL;
3685	} else {	4078	} else {
3686	ret = import_iovec(READ, uiov, iov_len, UIO_FASTIOV,	4079	ret = import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
3687	&io->msg.iov, &io->msg.msg.msg_iter);	4080	&iomsg->iov, &iomsg->msg.msg_iter);
3688	if (ret > 0)	4081	if (ret > 0)
3689	ret = 0;	4082	ret = 0;
3690	}	4083	}
@@ -3694,7 +4087,7 @@ static int __io_recvmsg_copy_hdr(struct io_kiocb req, struct io_async_ctx io) · 3 unmodified lines
3694		4087
3695	#ifdef CONFIG_COMPAT	4088	#ifdef CONFIG_COMPAT
3696	static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,	4089	static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
3697	struct io_async_ctx *io)	4090	struct io_async_msghdr *iomsg)
3698	{	4091	{
3699	struct compat_msghdr __user *msg_compat;	4092	struct compat_msghdr __user *msg_compat;
3700	struct io_sr_msg *sr = &req->sr_msg;	4093	struct io_sr_msg *sr = &req->sr_msg;
@@ -3703,8 +4096,8 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, · 2 unmodified lines
3703	compat_size_t len;	4096	compat_size_t len;
3704	int ret;	4097	int ret;
3705		4098
3706	msg_compat = (struct compat_msghdr __user *) sr->msg;	4099	msg_compat = (struct compat_msghdr __user *) sr->umsg;
3707	ret = __get_compat_msghdr(&io->msg.msg, msg_compat, &io->msg.uaddr,	4100	ret = __get_compat_msghdr(&iomsg->msg, msg_compat, &iomsg->uaddr,
3708	&ptr, &len);	4101	&ptr, &len);
3709	if (ret)	4102	if (ret)
3710	return ret;	4103	return ret;
@@ -3721,12 +4114,12 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, · 10 unmodified lines
3721	return -EFAULT;	4114	return -EFAULT;
3722	if (clen < 0)	4115	if (clen < 0)
3723	return -EINVAL;	4116	return -EINVAL;
3724	sr->len = io->msg.iov[0].iov_len;	4117	sr->len = iomsg->iov[0].iov_len;
3725	io->msg.iov = NULL;	4118	iomsg->iov = NULL;
3726	} else {	4119	} else {
3727	ret = compat_import_iovec(READ, uiov, len, UIO_FASTIOV,	4120	ret = compat_import_iovec(READ, uiov, len, UIO_FASTIOV,
3728	&io->msg.iov,	4121	&iomsg->iov,
3729	&io->msg.msg.msg_iter);	4122	&iomsg->msg.msg_iter);
3730	if (ret < 0)	4123	if (ret < 0)
3731	return ret;	4124	return ret;
3732	}	4125	}
@@ -3735,40 +4128,40 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, · 2 unmodified lines
3735	}	4128	}
3736	#endif	4129	#endif
3737		4130
3738	static int io_recvmsg_copy_hdr(struct io_kiocb req, struct io_async_ctx io)	4131	static int io_recvmsg_copy_hdr(struct io_kiocb *req,
		4132	struct io_async_msghdr *iomsg)
3739	{	4133	{
3740	io->msg.msg.msg_name = &io->msg.addr;	4134	iomsg->msg.msg_name = &iomsg->addr;
3741	io->msg.iov = io->msg.fast_iov;	4135	iomsg->iov = iomsg->fast_iov;
3742		4136
3743	#ifdef CONFIG_COMPAT	4137	#ifdef CONFIG_COMPAT
3744	if (req->ctx->compat)	4138	if (req->ctx->compat)
3745	return __io_compat_recvmsg_copy_hdr(req, io);	4139	return __io_compat_recvmsg_copy_hdr(req, iomsg);
3746	#endif	4140	#endif
3747		4141
3748	return __io_recvmsg_copy_hdr(req, io);	4142	return __io_recvmsg_copy_hdr(req, iomsg);
3749	}	4143	}
3750		4144
3751	static struct io_buffer io_recv_buffer_select(struct io_kiocb req,	4145	static struct io_buffer io_recv_buffer_select(struct io_kiocb req,
3752	int *cflags, bool needs_lock)	4146	bool needs_lock)
3753	{	4147	{
3754	struct io_sr_msg *sr = &req->sr_msg;	4148	struct io_sr_msg *sr = &req->sr_msg;
3755	struct io_buffer *kbuf;	4149	struct io_buffer *kbuf;
3756		4150
3757	if (!(req->flags & REQ_F_BUFFER_SELECT))
3758	return NULL;
3759
3760	kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);	4151	kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
3761	if (IS_ERR(kbuf))	4152	if (IS_ERR(kbuf))
3762	return kbuf;	4153	return kbuf;
3763		4154
3764	sr->kbuf = kbuf;	4155	sr->kbuf = kbuf;
3765	req->flags \|= REQ_F_BUFFER_SELECTED;	4156	req->flags \|= REQ_F_BUFFER_SELECTED;
3766
3767	*cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
3768	*cflags \|= IORING_CQE_F_BUFFER;
3769	return kbuf;	4157	return kbuf;
3770	}	4158	}
3771		4159
		4160	static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req)
		4161	{
		4162	return io_put_kbuf(req, req->sr_msg.kbuf);
		4163	}
		4164
3772	static int io_recvmsg_prep(struct io_kiocb *req,	4165	static int io_recvmsg_prep(struct io_kiocb *req,
3773	const struct io_uring_sqe *sqe)	4166	const struct io_uring_sqe *sqe)
3774	{	4167	{
@@ -3780,7 +4173,7 @@ static int io_recvmsg_prep(struct io_kiocb *req, · 5 unmodified lines
3780	return -EINVAL;	4173	return -EINVAL;
3781		4174
3782	sr->msg_flags = READ_ONCE(sqe->msg_flags);	4175	sr->msg_flags = READ_ONCE(sqe->msg_flags);
3783	sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));	4176	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
3784	sr->len = READ_ONCE(sqe->len);	4177	sr->len = READ_ONCE(sqe->len);
3785	sr->bgid = READ_ONCE(sqe->buf_group);	4178	sr->bgid = READ_ONCE(sqe->buf_group);
3786		4179
@@ -3795,133 +4188,123 @@ static int io_recvmsg_prep(struct io_kiocb *req, · 8 unmodified lines
3795	if (req->flags & REQ_F_NEED_CLEANUP)	4188	if (req->flags & REQ_F_NEED_CLEANUP)
3796	return 0;	4189	return 0;
3797		4190
3798	ret = io_recvmsg_copy_hdr(req, io);	4191	ret = io_recvmsg_copy_hdr(req, &io->msg);
3799	if (!ret)	4192	if (!ret)
3800	req->flags \|= REQ_F_NEED_CLEANUP;	4193	req->flags \|= REQ_F_NEED_CLEANUP;
3801	return ret;	4194	return ret;
3802	}	4195	}
3803		4196
3804	static int io_recvmsg(struct io_kiocb *req, bool force_nonblock)	4197	static int io_recvmsg(struct io_kiocb *req, bool force_nonblock,
		4198	struct io_comp_state *cs)
3805	{	4199	{
3806	struct io_async_msghdr *kmsg = NULL;	4200	struct io_async_msghdr iomsg, *kmsg;
3807	struct socket *sock;	4201	struct socket *sock;
		4202	struct io_buffer *kbuf;
		4203	unsigned flags;
3808	int ret, cflags = 0;	4204	int ret, cflags = 0;
3809		4205
3810	sock = sock_from_file(req->file, &ret);	4206	sock = sock_from_file(req->file, &ret);
3811	if (sock) {	4207	if (unlikely(!sock))
3812	struct io_buffer *kbuf;	4208	return ret;
3813	struct io_async_ctx io;
3814	unsigned flags;
3815
3816	if (req->io) {
3817	kmsg = &req->io->msg;
3818	kmsg->msg.msg_name = &req->io->msg.addr;
3819	/* if iov is set, it's allocated already */
3820	if (!kmsg->iov)
3821	kmsg->iov = kmsg->fast_iov;
3822	kmsg->msg.msg_iter.iov = kmsg->iov;
3823	} else {
3824	kmsg = &io.msg;
3825	kmsg->msg.msg_name = &io.msg.addr;
3826		4209
3827	ret = io_recvmsg_copy_hdr(req, &io);	4210	if (req->io) {
3828	if (ret)	4211	kmsg = &req->io->msg;
3829	return ret;	4212	kmsg->msg.msg_name = &req->io->msg.addr;
3830	}	4213	/* if iov is set, it's allocated already */
		4214	if (!kmsg->iov)
		4215	kmsg->iov = kmsg->fast_iov;
		4216	kmsg->msg.msg_iter.iov = kmsg->iov;
		4217	} else {
		4218	ret = io_recvmsg_copy_hdr(req, &iomsg);
		4219	if (ret)
		4220	return ret;
		4221	kmsg = &iomsg;
		4222	}
3831		4223
3832	kbuf = io_recv_buffer_select(req, &cflags, !force_nonblock);	4224	if (req->flags & REQ_F_BUFFER_SELECT) {
3833	if (IS_ERR(kbuf)) {	4225	kbuf = io_recv_buffer_select(req, !force_nonblock);
		4226	if (IS_ERR(kbuf))
3834	return PTR_ERR(kbuf);	4227	return PTR_ERR(kbuf);
3835	} else if (kbuf) {	4228	kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
3836	kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);	4229	iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->iov,
3837	iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->iov,	4230	1, req->sr_msg.len);
3838	1, req->sr_msg.len);	4231	}
3839	}
3840		4232
3841	flags = req->sr_msg.msg_flags;	4233	flags = req->sr_msg.msg_flags;
3842	if (flags & MSG_DONTWAIT)	4234	if (flags & MSG_DONTWAIT)
3843	req->flags \|= REQ_F_NOWAIT;	4235	req->flags \|= REQ_F_NOWAIT;
3844	else if (force_nonblock)	4236	else if (force_nonblock)
3845	flags \|= MSG_DONTWAIT;	4237	flags \|= MSG_DONTWAIT;
3846		4238
3847	ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.msg,	4239	ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg,
3848	kmsg->uaddr, flags);	4240	kmsg->uaddr, flags);
3849	if (force_nonblock && ret == -EAGAIN) {	4241	if (force_nonblock && ret == -EAGAIN)
3850	ret = io_setup_async_msg(req, kmsg);	4242	return io_setup_async_msg(req, kmsg);
3851	if (ret != -EAGAIN)	4243	if (ret == -ERESTARTSYS)
3852	kfree(kbuf);	4244	ret = -EINTR;
3853	return ret;
3854	}
3855	if (ret == -ERESTARTSYS)
3856	ret = -EINTR;
3857	if (kbuf)
3858	kfree(kbuf);
3859	}
3860		4245
3861	if (kmsg && kmsg->iov != kmsg->fast_iov)	4246	if (req->flags & REQ_F_BUFFER_SELECTED)
		4247	cflags = io_put_recv_kbuf(req);
		4248	if (kmsg->iov != kmsg->fast_iov)
3862	kfree(kmsg->iov);	4249	kfree(kmsg->iov);
3863	req->flags &= ~REQ_F_NEED_CLEANUP;	4250	req->flags &= ~REQ_F_NEED_CLEANUP;
3864	__io_cqring_add_event(req, ret, cflags);
3865	if (ret < 0)	4251	if (ret < 0)
3866	req_set_fail_links(req);	4252	req_set_fail_links(req);
3867	io_put_req(req);	4253	__io_req_complete(req, ret, cflags, cs);
3868	return 0;	4254	return 0;
3869	}	4255	}
3870		4256
3871	static int io_recv(struct io_kiocb *req, bool force_nonblock)	4257	static int io_recv(struct io_kiocb *req, bool force_nonblock,
		4258	struct io_comp_state *cs)
3872	{	4259	{
3873	struct io_buffer *kbuf = NULL;	4260	struct io_buffer *kbuf;
		4261	struct io_sr_msg *sr = &req->sr_msg;
		4262	struct msghdr msg;
		4263	void __user *buf = sr->buf;
3874	struct socket *sock;	4264	struct socket *sock;
		4265	struct iovec iov;
		4266	unsigned flags;
3875	int ret, cflags = 0;	4267	int ret, cflags = 0;
3876		4268
3877	sock = sock_from_file(req->file, &ret);	4269	sock = sock_from_file(req->file, &ret);
3878	if (sock) {	4270	if (unlikely(!sock))
3879	struct io_sr_msg *sr = &req->sr_msg;	4271	return ret;
3880	void __user *buf = sr->buf;
3881	struct msghdr msg;
3882	struct iovec iov;
3883	unsigned flags;
3884		4272
3885	kbuf = io_recv_buffer_select(req, &cflags, !force_nonblock);	4273	if (req->flags & REQ_F_BUFFER_SELECT) {
		4274	kbuf = io_recv_buffer_select(req, !force_nonblock);
3886	if (IS_ERR(kbuf))	4275	if (IS_ERR(kbuf))
3887	return PTR_ERR(kbuf);	4276	return PTR_ERR(kbuf);
3888	else if (kbuf)	4277	buf = u64_to_user_ptr(kbuf->addr);
3889	buf = u64_to_user_ptr(kbuf->addr);	4278	}
3890		4279
3891	ret = import_single_range(READ, buf, sr->len, &iov,	4280	ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter);
3892	&msg.msg_iter);	4281	if (unlikely(ret))
3893	if (ret) {	4282	goto out_free;
3894	kfree(kbuf);
3895	return ret;
3896	}
3897		4283
3898	req->flags \|= REQ_F_NEED_CLEANUP;	4284	msg.msg_name = NULL;
3899	msg.msg_name = NULL;	4285	msg.msg_control = NULL;
3900	msg.msg_control = NULL;	4286	msg.msg_controllen = 0;
3901	msg.msg_controllen = 0;	4287	msg.msg_namelen = 0;
3902	msg.msg_namelen = 0;	4288	msg.msg_iocb = NULL;
3903	msg.msg_iocb = NULL;	4289	msg.msg_flags = 0;
3904	msg.msg_flags = 0;
3905
3906	flags = req->sr_msg.msg_flags;
3907	if (flags & MSG_DONTWAIT)
3908	req->flags \|= REQ_F_NOWAIT;
3909	else if (force_nonblock)
3910	flags \|= MSG_DONTWAIT;
3911
3912	ret = sock_recvmsg(sock, &msg, flags);
3913	if (force_nonblock && ret == -EAGAIN)
3914	return -EAGAIN;
3915	if (ret == -ERESTARTSYS)
3916	ret = -EINTR;
3917	}
3918		4290
3919	kfree(kbuf);	4291	flags = req->sr_msg.msg_flags;
3920	req->flags &= ~REQ_F_NEED_CLEANUP;	4292	if (flags & MSG_DONTWAIT)
3921	__io_cqring_add_event(req, ret, cflags);	4293	req->flags \|= REQ_F_NOWAIT;
		4294	else if (force_nonblock)
		4295	flags \|= MSG_DONTWAIT;
		4296
		4297	ret = sock_recvmsg(sock, &msg, flags);
		4298	if (force_nonblock && ret == -EAGAIN)
		4299	return -EAGAIN;
		4300	if (ret == -ERESTARTSYS)
		4301	ret = -EINTR;
		4302	out_free:
		4303	if (req->flags & REQ_F_BUFFER_SELECTED)
		4304	cflags = io_put_recv_kbuf(req);
3922	if (ret < 0)	4305	if (ret < 0)
3923	req_set_fail_links(req);	4306	req_set_fail_links(req);
3924	io_put_req(req);	4307	__io_req_complete(req, ret, cflags, cs);
3925	return 0;	4308	return 0;
3926	}	4309	}
3927		4310
@@ -3941,7 +4324,8 @@ static int io_accept_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 13 unmodified lines
3941	return 0;	4324	return 0;
3942	}	4325	}
3943		4326
3944	static int io_accept(struct io_kiocb *req, bool force_nonblock)	4327	static int io_accept(struct io_kiocb *req, bool force_nonblock,
		4328	struct io_comp_state *cs)
3945	{	4329	{
3946	struct io_accept *accept = &req->accept;	4330	struct io_accept *accept = &req->accept;
3947	unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;	4331	unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
@@ -3960,8 +4344,7 @@ static int io_accept(struct io_kiocb *req, bool force_nonblock) · 12 unmodified lines
3960	ret = -EINTR;	4344	ret = -EINTR;
3961	req_set_fail_links(req);	4345	req_set_fail_links(req);
3962	}	4346	}
3963	io_cqring_add_event(req, ret);	4347	__io_req_complete(req, ret, 0, cs);
3964	io_put_req(req);
3965	return 0;	4348	return 0;
3966	}	4349	}
3967		4350
@@ -3985,7 +4368,8 @@ static int io_connect_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 17 unmodified lines
3985	&io->connect.address);	4368	&io->connect.address);
3986	}	4369	}
3987		4370
3988	static int io_connect(struct io_kiocb *req, bool force_nonblock)	4371	static int io_connect(struct io_kiocb *req, bool force_nonblock,
		4372	struct io_comp_state *cs)
3989	{	4373	{
3990	struct io_async_ctx __io, *io;	4374	struct io_async_ctx __io, *io;
3991	unsigned file_flags;	4375	unsigned file_flags;
@@ -4021,8 +4405,7 @@ static int io_connect(struct io_kiocb *req, bool force_nonblock) · 29 unmodified lines
4021	out:	4405	out:
4022	if (ret < 0)	4406	if (ret < 0)
4023	req_set_fail_links(req);	4407	req_set_fail_links(req);
4024	io_cqring_add_event(req, ret);	4408	__io_req_complete(req, ret, 0, cs);
4025	io_put_req(req);
4026	return 0;	4409	return 0;
4027	}	4410	}
4028	#else /* !CONFIG_NET */	4411	#else /* !CONFIG_NET */
@@ -4031,12 +4414,14 @@ static int io_sendmsg_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 2 unmodified lines
4031	return -EOPNOTSUPP;	4414	return -EOPNOTSUPP;
4032	}	4415	}
4033		4416
4034	static int io_sendmsg(struct io_kiocb *req, bool force_nonblock)	4417	static int io_sendmsg(struct io_kiocb *req, bool force_nonblock,
		4418	struct io_comp_state *cs)
4035	{	4419	{
4036	return -EOPNOTSUPP;	4420	return -EOPNOTSUPP;
4037	}	4421	}
4038		4422
4039	static int io_send(struct io_kiocb *req, bool force_nonblock)	4423	static int io_send(struct io_kiocb *req, bool force_nonblock,
		4424	struct io_comp_state *cs)
4040	{	4425	{
4041	return -EOPNOTSUPP;	4426	return -EOPNOTSUPP;
4042	}	4427	}
@@ -4047,12 +4432,14 @@ static int io_recvmsg_prep(struct io_kiocb *req, · 4 unmodified lines
4047	return -EOPNOTSUPP;	4432	return -EOPNOTSUPP;
4048	}	4433	}
4049		4434
4050	static int io_recvmsg(struct io_kiocb *req, bool force_nonblock)	4435	static int io_recvmsg(struct io_kiocb *req, bool force_nonblock,
		4436	struct io_comp_state *cs)
4051	{	4437	{
4052	return -EOPNOTSUPP;	4438	return -EOPNOTSUPP;
4053	}	4439	}
4054		4440
4055	static int io_recv(struct io_kiocb *req, bool force_nonblock)	4441	static int io_recv(struct io_kiocb *req, bool force_nonblock,
		4442	struct io_comp_state *cs)
4056	{	4443	{
4057	return -EOPNOTSUPP;	4444	return -EOPNOTSUPP;
4058	}	4445	}
@@ -4062,55 +4449,33 @@ static int io_accept_prep(struct io_kiocb req, const struct io_uring_sqe sqe) · 3 unmodified lines
4062	return -EOPNOTSUPP;	4449	return -EOPNOTSUPP;
4063	}	4450	}
4064		4451
4065	static int io_accept(struct io_kiocb *req, bool force_nonblock)	4452	static int io_accept(struct io_kiocb *req, bool force_nonblock,
4066	{	4453	struct io_comp_state *cs)
4067	return -EOPNOTSUPP;	4454	{
4068	}	4455	return -EOPNOTSUPP;
4069		4456	}
4070	static int io_connect_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
4071	{
4072	return -EOPNOTSUPP;
4073	}
4074
4075	static int io_connect(struct io_kiocb *req, bool force_nonblock)
4076	{
4077	return -EOPNOTSUPP;
4078	}
4079	#endif /* CONFIG_NET */
4080
4081	struct io_poll_table {
4082	struct poll_table_struct pt;
4083	struct io_kiocb *req;
4084	int error;
4085	};
4086
4087	static int io_req_task_work_add(struct io_kiocb req, struct callback_head cb)
4088	{
4089	struct task_struct *tsk = req->task;
4090	struct io_ring_ctx *ctx = req->ctx;
4091	int ret, notify = TWA_RESUME;
4092
4093	/*
4094	* SQPOLL kernel thread doesn't need notification, just a wakeup.
4095	* If we're not using an eventfd, then TWA_RESUME is always fine,
4096	* as we won't have dependencies between request completions for
4097	* other kernel wait conditions.
4098	*/
4099	if (ctx->flags & IORING_SETUP_SQPOLL)
4100	notify = 0;
4101	else if (ctx->cq_ev_fd)
4102	notify = TWA_SIGNAL;
4103		4457
4104	ret = task_work_add(tsk, cb, notify);	4458	static int io_connect_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
4105	if (!ret)	4459	{
4106	wake_up_process(tsk);	4460	return -EOPNOTSUPP;
4107	return ret;	4461	}
		4462
		4463	static int io_connect(struct io_kiocb *req, bool force_nonblock,
		4464	struct io_comp_state *cs)
		4465	{
		4466	return -EOPNOTSUPP;
4108	}	4467	}
		4468	#endif /* CONFIG_NET */
		4469
		4470	struct io_poll_table {
		4471	struct poll_table_struct pt;
		4472	struct io_kiocb *req;
		4473	int error;
		4474	};
4109		4475
4110	static int __io_async_wake(struct io_kiocb req, struct io_poll_iocb poll,	4476	static int __io_async_wake(struct io_kiocb req, struct io_poll_iocb poll,
4111	__poll_t mask, task_work_func_t func)	4477	__poll_t mask, task_work_func_t func)
4112	{	4478	{
4113	struct task_struct *tsk;
4114	int ret;	4479	int ret;
4115		4480
4116	/* for instances that support it check for an event match first: */	4481	/* for instances that support it check for an event match first: */
@@ -4121,7 +4486,6 @@ static int __io_async_wake(struct io_kiocb req, struct io_poll_iocb poll, · 4 unmodified lines
4121		4486
4122	list_del_init(&poll->wait.entry);	4487	list_del_init(&poll->wait.entry);
4123		4488
4124	tsk = req->task;
4125	req->result = mask;	4489	req->result = mask;
4126	init_task_work(&req->task_work, func);	4490	init_task_work(&req->task_work, func);
4127	/*	4491	/*
@@ -4132,6 +4496,8 @@ static int __io_async_wake(struct io_kiocb req, struct io_poll_iocb poll, · 4 unmodified lines
4132	*/	4496	*/
4133	ret = io_req_task_work_add(req, &req->task_work);	4497	ret = io_req_task_work_add(req, &req->task_work);
4134	if (unlikely(ret)) {	4498	if (unlikely(ret)) {
		4499	struct task_struct *tsk;
		4500
4135	WRITE_ONCE(poll->canceled, true);	4501	WRITE_ONCE(poll->canceled, true);
4136	tsk = io_wq_get_task(req->ctx->io_wq);	4502	tsk = io_wq_get_task(req->ctx->io_wq);
4137	task_work_add(tsk, &req->task_work, 0);	4503	task_work_add(tsk, &req->task_work, 0);
@@ -4199,9 +4565,10 @@ static void io_poll_task_handler(struct io_kiocb req, struct io_kiocb *nxt) · 61 unmodified lines
4199		4565
4200	hash_del(&req->hash_node);	4566	hash_del(&req->hash_node);
4201	io_poll_complete(req, req->result, 0);	4567	io_poll_complete(req, req->result, 0);
		4568	req->flags \|= REQ_F_COMP_LOCKED;
		4569	*nxt = io_put_req_find_next(req);
4202	spin_unlock_irq(&ctx->completion_lock);	4570	spin_unlock_irq(&ctx->completion_lock);
4203		4571
4204	io_put_req_find_next(req, nxt);
4205	io_cqring_ev_posted(ctx);	4572	io_cqring_ev_posted(ctx);
4206	}	4573	}
4207		4574
@@ -4211,13 +4578,8 @@ static void io_poll_task_func(struct callback_head *cb) · 3 unmodified lines
4211	struct io_kiocb *nxt = NULL;	4578	struct io_kiocb *nxt = NULL;
4212		4579
4213	io_poll_task_handler(req, &nxt);	4580	io_poll_task_handler(req, &nxt);
4214	if (nxt) {	4581	if (nxt)
4215	struct io_ring_ctx *ctx = nxt->ctx;	4582	__io_req_task_submit(nxt);
4216
4217	mutex_lock(&ctx->uring_lock);
4218	__io_queue_sqe(nxt, NULL);
4219	mutex_unlock(&ctx->uring_lock);
4220	}
4221	}	4583	}
4222		4584
4223	static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode,	4585	static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode,
@@ -4287,7 +4649,11 @@ static void __io_queue_proc(struct io_poll_iocb poll, struct io_poll_table pt, · 63 unmodified lines
4287		4649
4288	pt->error = 0;	4650	pt->error = 0;
4289	poll->head = head;	4651	poll->head = head;
4290	add_wait_queue(head, &poll->wait);	4652
		4653	if (poll->events & EPOLLEXCLUSIVE)
		4654	add_wait_queue_exclusive(head, &poll->wait);
		4655	else
		4656	add_wait_queue(head, &poll->wait);
4291	}	4657	}
4292		4658
4293	static void io_async_queue_proc(struct file file, struct wait_queue_head head,	4659	static void io_async_queue_proc(struct file file, struct wait_queue_head head,
@@ -4299,34 +4665,11 @@ static void io_async_queue_proc(struct file file, struct wait_queue_head head, · 5 unmodified lines
4299	__io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);	4665	__io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);
4300	}	4666	}
4301		4667
4302	static void io_sq_thread_drop_mm(struct io_ring_ctx *ctx)
4303	{
4304	struct mm_struct *mm = current->mm;
4305
4306	if (mm) {
4307	kthread_unuse_mm(mm);
4308	mmput(mm);
4309	}
4310	}
4311
4312	static int io_sq_thread_acquire_mm(struct io_ring_ctx *ctx,
4313	struct io_kiocb *req)
4314	{
4315	if (io_op_defs[req->opcode].needs_mm && !current->mm) {
4316	if (unlikely(!mmget_not_zero(ctx->sqo_mm)))
4317	return -EFAULT;
4318	kthread_use_mm(ctx->sqo_mm);
4319	}
4320
4321	return 0;
4322	}
4323
4324	static void io_async_task_func(struct callback_head *cb)	4668	static void io_async_task_func(struct callback_head *cb)
4325	{	4669	{
4326	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);	4670	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
4327	struct async_poll *apoll = req->apoll;	4671	struct async_poll *apoll = req->apoll;
4328	struct io_ring_ctx *ctx = req->ctx;	4672	struct io_ring_ctx *ctx = req->ctx;
4329	bool canceled = false;
4330		4673
4331	trace_io_uring_task_run(req->ctx, req->opcode, req->user_data);	4674	trace_io_uring_task_run(req->ctx, req->opcode, req->user_data);
4332		4675
@@ -4336,40 +4679,19 @@ static void io_async_task_func(struct callback_head *cb) · 3 unmodified lines
4336	}	4679	}
4337		4680
4338	/* If req is still hashed, it cannot have been canceled. Don't check. */	4681	/* If req is still hashed, it cannot have been canceled. Don't check. */
4339	if (hash_hashed(&req->hash_node)) {	4682	if (hash_hashed(&req->hash_node))
4340	hash_del(&req->hash_node);	4683	hash_del(&req->hash_node);
4341	} else {
4342	canceled = READ_ONCE(apoll->poll.canceled);
4343	if (canceled) {
4344	io_cqring_fill_event(req, -ECANCELED);
4345	io_commit_cqring(ctx);
4346	}
4347	}
4348		4684
4349	io_poll_remove_double(req, apoll->double_poll);	4685	io_poll_remove_double(req, apoll->double_poll);
4350	spin_unlock_irq(&ctx->completion_lock);	4686	spin_unlock_irq(&ctx->completion_lock);
4351		4687
4352	/* restore ->work in case we need to retry again */	4688	if (!READ_ONCE(apoll->poll.canceled))
4353	if (req->flags & REQ_F_WORK_INITIALIZED)	4689	__io_req_task_submit(req);
4354	memcpy(&req->work, &apoll->work, sizeof(req->work));	4690	else
		4691	__io_req_task_cancel(req, -ECANCELED);
		4692
4355	kfree(apoll->double_poll);	4693	kfree(apoll->double_poll);
4356	kfree(apoll);	4694	kfree(apoll);
4357
4358	if (!canceled) {
4359	__set_current_state(TASK_RUNNING);
4360	if (io_sq_thread_acquire_mm(ctx, req)) {
4361	io_cqring_add_event(req, -EFAULT);
4362	goto end_req;
4363	}
4364	mutex_lock(&ctx->uring_lock);
4365	__io_queue_sqe(req, NULL);
4366	mutex_unlock(&ctx->uring_lock);
4367	} else {
4368	io_cqring_ev_posted(ctx);
4369	end_req:
4370	req_set_fail_links(req);
4371	io_double_put_req(req);
4372	}
4373	}	4695	}
4374		4696
4375	static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync,	4697	static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
@@ -4402,8 +4724,8 @@ static __poll_t __io_arm_poll_handler(struct io_kiocb *req, · 26 unmodified lines
4402	struct io_ring_ctx *ctx = req->ctx;	4724	struct io_ring_ctx *ctx = req->ctx;
4403	bool cancel = false;	4725	bool cancel = false;
4404		4726
4405	poll->file = req->file;
4406	io_init_poll_iocb(poll, mask, wake_func);	4727	io_init_poll_iocb(poll, mask, wake_func);
		4728	poll->file = req->file;
4407	poll->wait.private = req;	4729	poll->wait.private = req;
4408		4730
4409	ipt->pt._key = mask;	4731	ipt->pt._key = mask;
@@ -4443,7 +4765,7 @@ static bool io_arm_poll_handler(struct io_kiocb *req) · 33 unmodified lines
4443		4765
4444	if (!req->file \|\| !file_can_poll(req->file))	4766	if (!req->file \|\| !file_can_poll(req->file))
4445	return false;	4767	return false;
4446	if (req->flags & (REQ_F_MUST_PUNT \| REQ_F_POLLED))	4768	if (req->flags & REQ_F_POLLED)
4447	return false;	4769	return false;
4448	if (!def->pollin && !def->pollout)	4770	if (!def->pollin && !def->pollout)
4449	return false;	4771	return false;
@@ -4454,9 +4776,6 @@ static bool io_arm_poll_handler(struct io_kiocb *req) · 4 unmodified lines
4454	apoll->double_poll = NULL;	4776	apoll->double_poll = NULL;
4455		4777
4456	req->flags \|= REQ_F_POLLED;	4778	req->flags \|= REQ_F_POLLED;
4457	if (req->flags & REQ_F_WORK_INITIALIZED)
4458	memcpy(&apoll->work, &req->work, sizeof(req->work));
4459
4460	io_get_req_task(req);	4779	io_get_req_task(req);
4461	req->apoll = apoll;	4780	req->apoll = apoll;
4462	INIT_HLIST_NODE(&req->hash_node);	4781	INIT_HLIST_NODE(&req->hash_node);
@@ -4475,8 +4794,6 @@ static bool io_arm_poll_handler(struct io_kiocb *req) · 12 unmodified lines
4475	if (ret) {	4794	if (ret) {
4476	io_poll_remove_double(req, apoll->double_poll);	4795	io_poll_remove_double(req, apoll->double_poll);
4477	spin_unlock_irq(&ctx->completion_lock);	4796	spin_unlock_irq(&ctx->completion_lock);
4478	if (req->flags & REQ_F_WORK_INITIALIZED)
4479	memcpy(&req->work, &apoll->work, sizeof(req->work));
4480	kfree(apoll->double_poll);	4797	kfree(apoll->double_poll);
4481	kfree(apoll);	4798	kfree(apoll);
4482	return false;	4799	return false;
@@ -4519,14 +4836,6 @@ static bool io_poll_remove_one(struct io_kiocb *req) · 36 unmodified lines
4519	do_complete = __io_poll_remove_one(req, &apoll->poll);	4836	do_complete = __io_poll_remove_one(req, &apoll->poll);
4520	if (do_complete) {	4837	if (do_complete) {
4521	io_put_req(req);	4838	io_put_req(req);
4522	/*
4523	* restore ->work because we will call
4524	* io_req_work_drop_env below when dropping the
4525	* final reference.
4526	*/
4527	if (req->flags & REQ_F_WORK_INITIALIZED)
4528	memcpy(&req->work, &apoll->work,
4529	sizeof(req->work));
4530	kfree(apoll->double_poll);	4839	kfree(apoll->double_poll);
4531	kfree(apoll);	4840	kfree(apoll);
4532	}	4841	}
@@ -4607,10 +4916,9 @@ static int io_poll_remove(struct io_kiocb *req) · 74 unmodified lines
4607	ret = io_poll_cancel(ctx, addr);	4916	ret = io_poll_cancel(ctx, addr);
4608	spin_unlock_irq(&ctx->completion_lock);	4917	spin_unlock_irq(&ctx->completion_lock);
4609		4918
4610	io_cqring_add_event(req, ret);
4611	if (ret < 0)	4919	if (ret < 0)
4612	req_set_fail_links(req);	4920	req_set_fail_links(req);
4613	io_put_req(req);	4921	io_req_complete(req, ret);
4614	return 0;	4922	return 0;
4615	}	4923	}
4616		4924
@@ -4634,7 +4942,7 @@ static void io_poll_queue_proc(struct file file, struct wait_queue_head head, · 17 unmodified lines
4634	static int io_poll_add_prep(struct io_kiocb req, const struct io_uring_sqe sqe)	4942	static int io_poll_add_prep(struct io_kiocb req, const struct io_uring_sqe sqe)
4635	{	4943	{
4636	struct io_poll_iocb *poll = &req->poll;	4944	struct io_poll_iocb *poll = &req->poll;
4637	u16 events;	4945	u32 events;
4638		4946
4639	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))	4947	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4640	return -EINVAL;	4948	return -EINVAL;
@@ -4643,8 +4951,12 @@ static int io_poll_add_prep(struct io_kiocb req, const struct io_uring_sqe sqe · 2 unmodified lines
4643	if (!poll->file)	4951	if (!poll->file)
4644	return -EBADF;	4952	return -EBADF;
4645		4953
4646	events = READ_ONCE(sqe->poll_events);	4954	events = READ_ONCE(sqe->poll32_events);
4647	poll->events = demangle_poll(events) \| EPOLLERR \| EPOLLHUP;	4955	#ifdef __BIG_ENDIAN
		4956	events = swahw32(events);
		4957	#endif
		4958	poll->events = demangle_poll(events) \| EPOLLERR \| EPOLLHUP \|
		4959	(events & EPOLLEXCLUSIVE);
4648		4960
4649	io_get_req_task(req);	4961	io_get_req_task(req);
4650	return 0;	4962	return 0;
@@ -4657,12 +4969,7 @@ static int io_poll_add(struct io_kiocb *req) · 6 unmodified lines
4657	struct io_poll_table ipt;	4969	struct io_poll_table ipt;
4658	__poll_t mask;	4970	__poll_t mask;
4659		4971
4660	/* ->work is in union with hash_node and others */
4661	io_req_work_drop_env(req);
4662	req->flags &= ~REQ_F_WORK_INITIALIZED;
4663
4664	INIT_HLIST_NODE(&req->hash_node);	4972	INIT_HLIST_NODE(&req->hash_node);
4665	INIT_LIST_HEAD(&req->list);
4666	ipt.pt._qproc = io_poll_queue_proc;	4973	ipt.pt._qproc = io_poll_queue_proc;
4667		4974
4668	mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events,	4975	mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events,
@@ -4689,15 +4996,16 @@ static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) · 20 unmodified lines
4689	struct io_ring_ctx *ctx = req->ctx;	4996	struct io_ring_ctx *ctx = req->ctx;
4690	unsigned long flags;	4997	unsigned long flags;
4691		4998
4692	atomic_inc(&ctx->cq_timeouts);
4693
4694	spin_lock_irqsave(&ctx->completion_lock, flags);	4999	spin_lock_irqsave(&ctx->completion_lock, flags);
		5000	atomic_set(&req->ctx->cq_timeouts,
		5001	atomic_read(&req->ctx->cq_timeouts) + 1);
		5002
4695	/*	5003	/*
4696	* We could be racing with timeout deletion. If the list is empty,	5004	* We could be racing with timeout deletion. If the list is empty,
4697	* then timeout lookup already found it and will be handling it.	5005	* then timeout lookup already found it and will be handling it.
4698	*/	5006	*/
4699	if (!list_empty(&req->list))	5007	if (!list_empty(&req->timeout.list))
4700	list_del_init(&req->list);	5008	list_del_init(&req->timeout.list);
4701		5009
4702	io_cqring_fill_event(req, -ETIME);	5010	io_cqring_fill_event(req, -ETIME);
4703	io_commit_cqring(ctx);	5011	io_commit_cqring(ctx);
@@ -4714,9 +5022,9 @@ static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data) · 10 unmodified lines
4714	struct io_kiocb *req;	5022	struct io_kiocb *req;
4715	int ret = -ENOENT;	5023	int ret = -ENOENT;
4716		5024
4717	list_for_each_entry(req, &ctx->timeout_list, list) {	5025	list_for_each_entry(req, &ctx->timeout_list, timeout.list) {
4718	if (user_data == req->user_data) {	5026	if (user_data == req->user_data) {
4719	list_del_init(&req->list);	5027	list_del_init(&req->timeout.list);
4720	ret = 0;	5028	ret = 0;
4721	break;	5029	break;
4722	}	5030	}
@@ -4798,7 +5106,6 @@ static int io_timeout_prep(struct io_kiocb req, const struct io_uring_sqe sqe, · 75 unmodified lines
4798		5106
4799	data = &req->io->timeout;	5107	data = &req->io->timeout;
4800	data->req = req;	5108	data->req = req;
4801	req->flags \|= REQ_F_TIMEOUT;
4802		5109
4803	if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))	5110	if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
4804	return -EFAULT;	5111	return -EFAULT;
@@ -4826,8 +5133,7 @@ static int io_timeout(struct io_kiocb *req) · 21 unmodified lines
4826	* timeout event to be satisfied. If it isn't set, then this is	5133	* timeout event to be satisfied. If it isn't set, then this is
4827	* a pure timeout request, sequence isn't used.	5134	* a pure timeout request, sequence isn't used.
4828	*/	5135	*/
4829	if (!off) {	5136	if (io_is_timeout_noseq(req)) {
4830	req->flags \|= REQ_F_TIMEOUT_NOSEQ;
4831	entry = ctx->timeout_list.prev;	5137	entry = ctx->timeout_list.prev;
4832	goto add;	5138	goto add;
4833	}	5139	}
@@ -4840,16 +5146,17 @@ static int io_timeout(struct io_kiocb *req) · 6 unmodified lines
4840	* the one we need first.	5146	* the one we need first.
4841	*/	5147	*/
4842	list_for_each_prev(entry, &ctx->timeout_list) {	5148	list_for_each_prev(entry, &ctx->timeout_list) {
4843	struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, list);	5149	struct io_kiocb *nxt = list_entry(entry, struct io_kiocb,
		5150	timeout.list);
4844		5151
4845	if (nxt->flags & REQ_F_TIMEOUT_NOSEQ)	5152	if (io_is_timeout_noseq(nxt))
4846	continue;	5153	continue;
4847	/* nxt.seq is behind @tail, otherwise would've been completed */	5154	/* nxt.seq is behind @tail, otherwise would've been completed */
4848	if (off >= nxt->timeout.target_seq - tail)	5155	if (off >= nxt->timeout.target_seq - tail)
4849	break;	5156	break;
4850	}	5157	}
4851	add:	5158	add:
4852	list_add(&req->list, entry);	5159	list_add(&req->timeout.list, entry);
4853	data->timer.function = io_timeout_fn;	5160	data->timer.function = io_timeout_fn;
4854	hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);	5161	hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
4855	spin_unlock_irq(&ctx->completion_lock);	5162	spin_unlock_irq(&ctx->completion_lock);
@@ -4953,7 +5260,8 @@ static int io_files_update_prep(struct io_kiocb *req, · 97 unmodified lines
4953	return 0;	5260	return 0;
4954	}	5261	}
4955		5262
4956	static int io_files_update(struct io_kiocb *req, bool force_nonblock)	5263	static int io_files_update(struct io_kiocb *req, bool force_nonblock,
		5264	struct io_comp_state *cs)
4957	{	5265	{
4958	struct io_ring_ctx *ctx = req->ctx;	5266	struct io_ring_ctx *ctx = req->ctx;
4959	struct io_uring_files_update up;	5267	struct io_uring_files_update up;
@@ -4971,8 +5279,7 @@ static int io_files_update(struct io_kiocb *req, bool force_nonblock) · 11 unmodified lines
4971		5279
4972	if (ret < 0)	5280	if (ret < 0)
4973	req_set_fail_links(req);	5281	req_set_fail_links(req);
4974	io_cqring_add_event(req, ret);	5282	__io_req_complete(req, ret, 0, cs);
4975	io_put_req(req);
4976	return 0;	5283	return 0;
4977	}	5284	}
4978		5285
@@ -4984,15 +5291,11 @@ static int io_req_defer_prep(struct io_kiocb *req, · 5 unmodified lines
4984	if (!sqe)	5291	if (!sqe)
4985	return 0;	5292	return 0;
4986		5293
4987	io_req_init_async(req);	5294	if (io_alloc_async_ctx(req))
4988		5295	return -EAGAIN;
4989	if (io_op_defs[req->opcode].file_table) {	5296	ret = io_prep_work_files(req);
4990	ret = io_grab_files(req);	5297	if (unlikely(ret))
4991	if (unlikely(ret))	5298	return ret;
4992	return ret;
4993	}
4994
4995	io_req_work_grab_env(req, &io_op_defs[req->opcode]);
4996		5299
4997	switch (req->opcode) {	5300	switch (req->opcode) {
4998	case IORING_OP_NOP:	5301	case IORING_OP_NOP:
@@ -5094,86 +5397,117 @@ static int io_req_defer_prep(struct io_kiocb *req, · 95 unmodified lines
5094	return ret;	5397	return ret;
5095	}	5398	}
5096		5399
		5400	static u32 io_get_sequence(struct io_kiocb *req)
		5401	{
		5402	struct io_kiocb *pos;
		5403	struct io_ring_ctx *ctx = req->ctx;
		5404	u32 total_submitted, nr_reqs = 1;
		5405
		5406	if (req->flags & REQ_F_LINK_HEAD)
		5407	list_for_each_entry(pos, &req->link_list, link_list)
		5408	nr_reqs++;
		5409
		5410	total_submitted = ctx->cached_sq_head - ctx->cached_sq_dropped;
		5411	return total_submitted - nr_reqs;
		5412	}
		5413
5097	static int io_req_defer(struct io_kiocb req, const struct io_uring_sqe sqe)	5414	static int io_req_defer(struct io_kiocb req, const struct io_uring_sqe sqe)
5098	{	5415	{
5099	struct io_ring_ctx *ctx = req->ctx;	5416	struct io_ring_ctx *ctx = req->ctx;
		5417	struct io_defer_entry *de;
5100	int ret;	5418	int ret;
		5419	u32 seq;
5101		5420
5102	/* Still need defer if there is pending req in defer list. */	5421	/* Still need defer if there is pending req in defer list. */
5103	if (!req_need_defer(req) && list_empty_careful(&ctx->defer_list))	5422	if (likely(list_empty_careful(&ctx->defer_list) &&
		5423	!(req->flags & REQ_F_IO_DRAIN)))
		5424	return 0;
		5425
		5426	seq = io_get_sequence(req);
		5427	/* Still a chance to pass the sequence check */
		5428	if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list))
5104	return 0;	5429	return 0;
5105		5430
5106	if (!req->io) {	5431	if (!req->io) {
5107	if (io_alloc_async_ctx(req))
5108	return -EAGAIN;
5109	ret = io_req_defer_prep(req, sqe);	5432	ret = io_req_defer_prep(req, sqe);
5110	if (ret < 0)	5433	if (ret)
5111	return ret;	5434	return ret;
5112	}	5435	}
		5436	io_prep_async_link(req);
		5437	de = kmalloc(sizeof(*de), GFP_KERNEL);
		5438	if (!de)
		5439	return -ENOMEM;
5113		5440
5114	spin_lock_irq(&ctx->completion_lock);	5441	spin_lock_irq(&ctx->completion_lock);
5115	if (!req_need_defer(req) && list_empty(&ctx->defer_list)) {	5442	if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
5116	spin_unlock_irq(&ctx->completion_lock);	5443	spin_unlock_irq(&ctx->completion_lock);
5117	return 0;	5444	kfree(de);
		5445	io_queue_async_work(req);
		5446	return -EIOCBQUEUED;
5118	}	5447	}
5119		5448
5120	trace_io_uring_defer(ctx, req, req->user_data);	5449	trace_io_uring_defer(ctx, req, req->user_data);
5121	list_add_tail(&req->list, &ctx->defer_list);	5450	de->req = req;
		5451	de->seq = seq;
		5452	list_add_tail(&de->list, &ctx->defer_list);
5122	spin_unlock_irq(&ctx->completion_lock);	5453	spin_unlock_irq(&ctx->completion_lock);
5123	return -EIOCBQUEUED;	5454	return -EIOCBQUEUED;
5124	}	5455	}
5125		5456
5126	static void io_cleanup_req(struct io_kiocb *req)	5457	static void __io_clean_op(struct io_kiocb *req)
5127	{	5458	{
5128	struct io_async_ctx *io = req->io;	5459	struct io_async_ctx *io = req->io;
5129		5460
5130	switch (req->opcode) {	5461	if (req->flags & REQ_F_BUFFER_SELECTED) {
5131	case IORING_OP_READV:	5462	switch (req->opcode) {
5132	case IORING_OP_READ_FIXED:	5463	case IORING_OP_READV:
5133	case IORING_OP_READ:	5464	case IORING_OP_READ_FIXED:
5134	if (req->flags & REQ_F_BUFFER_SELECTED)	5465	case IORING_OP_READ:
5135	kfree((void *)(unsigned long)req->rw.addr);	5466	kfree((void *)(unsigned long)req->rw.addr);
5136	/* fallthrough */	5467	break;
5137	case IORING_OP_WRITEV:	5468	case IORING_OP_RECVMSG:
5138	case IORING_OP_WRITE_FIXED:	5469	case IORING_OP_RECV:
5139	case IORING_OP_WRITE:
5140	if (io->rw.iov != io->rw.fast_iov)
5141	kfree(io->rw.iov);
5142	break;
5143	case IORING_OP_RECVMSG:
5144	if (req->flags & REQ_F_BUFFER_SELECTED)
5145	kfree(req->sr_msg.kbuf);
5146	/* fallthrough */
5147	case IORING_OP_SENDMSG:
5148	if (io->msg.iov != io->msg.fast_iov)
5149	kfree(io->msg.iov);
5150	break;
5151	case IORING_OP_RECV:
5152	if (req->flags & REQ_F_BUFFER_SELECTED)
5153	kfree(req->sr_msg.kbuf);	5470	kfree(req->sr_msg.kbuf);
5154	break;	5471	break;
5155	case IORING_OP_OPENAT:	5472	}
5156	case IORING_OP_OPENAT2:	5473	req->flags &= ~REQ_F_BUFFER_SELECTED;
5157	break;	5474	}
5158	case IORING_OP_SPLICE:	5475
5159	case IORING_OP_TEE:	5476	if (req->flags & REQ_F_NEED_CLEANUP) {
5160	io_put_file(req, req->splice.file_in,	5477	switch (req->opcode) {
5161	(req->splice.flags & SPLICE_F_FD_IN_FIXED));	5478	case IORING_OP_READV:
5162	break;	5479	case IORING_OP_READ_FIXED:
		5480	case IORING_OP_READ:
		5481	case IORING_OP_WRITEV:
		5482	case IORING_OP_WRITE_FIXED:
		5483	case IORING_OP_WRITE:
		5484	if (io->rw.iov != io->rw.fast_iov)
		5485	kfree(io->rw.iov);
		5486	break;
		5487	case IORING_OP_RECVMSG:
		5488	case IORING_OP_SENDMSG:
		5489	if (io->msg.iov != io->msg.fast_iov)
		5490	kfree(io->msg.iov);
		5491	break;
		5492	case IORING_OP_SPLICE:
		5493	case IORING_OP_TEE:
		5494	io_put_file(req, req->splice.file_in,
		5495	(req->splice.flags & SPLICE_F_FD_IN_FIXED));
		5496	break;
		5497	}
		5498	req->flags &= ~REQ_F_NEED_CLEANUP;
5163	}	5499	}
5164
5165	req->flags &= ~REQ_F_NEED_CLEANUP;
5166	}	5500	}
5167		5501
5168	static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe,	5502	static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe,
5169	bool force_nonblock)	5503	bool force_nonblock, struct io_comp_state *cs)
5170	{	5504	{
5171	struct io_ring_ctx *ctx = req->ctx;	5505	struct io_ring_ctx *ctx = req->ctx;
5172	int ret;	5506	int ret;
5173		5507
5174	switch (req->opcode) {	5508	switch (req->opcode) {
5175	case IORING_OP_NOP:	5509	case IORING_OP_NOP:
5176	ret = io_nop(req);	5510	ret = io_nop(req, cs);
5177	break;	5511	break;
5178	case IORING_OP_READV:	5512	case IORING_OP_READV:
5179	case IORING_OP_READ_FIXED:	5513	case IORING_OP_READ_FIXED:
@@ -5183,7 +5517,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 3 unmodified lines
5183	if (ret < 0)	5517	if (ret < 0)
5184	break;	5518	break;
5185	}	5519	}
5186	ret = io_read(req, force_nonblock);	5520	ret = io_read(req, force_nonblock, cs);
5187	break;	5521	break;
5188	case IORING_OP_WRITEV:	5522	case IORING_OP_WRITEV:
5189	case IORING_OP_WRITE_FIXED:	5523	case IORING_OP_WRITE_FIXED:
@@ -5193,7 +5527,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 3 unmodified lines
5193	if (ret < 0)	5527	if (ret < 0)
5194	break;	5528	break;
5195	}	5529	}
5196	ret = io_write(req, force_nonblock);	5530	ret = io_write(req, force_nonblock, cs);
5197	break;	5531	break;
5198	case IORING_OP_FSYNC:	5532	case IORING_OP_FSYNC:
5199	if (sqe) {	5533	if (sqe) {
@@ -5235,9 +5569,9 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 35 unmodified lines
5235	break;	5569	break;
5236	}	5570	}
5237	if (req->opcode == IORING_OP_SENDMSG)	5571	if (req->opcode == IORING_OP_SENDMSG)
5238	ret = io_sendmsg(req, force_nonblock);	5572	ret = io_sendmsg(req, force_nonblock, cs);
5239	else	5573	else
5240	ret = io_send(req, force_nonblock);	5574	ret = io_send(req, force_nonblock, cs);
5241	break;	5575	break;
5242	case IORING_OP_RECVMSG:	5576	case IORING_OP_RECVMSG:
5243	case IORING_OP_RECV:	5577	case IORING_OP_RECV:
@@ -5247,9 +5581,9 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 3 unmodified lines
5247	break;	5581	break;
5248	}	5582	}
5249	if (req->opcode == IORING_OP_RECVMSG)	5583	if (req->opcode == IORING_OP_RECVMSG)
5250	ret = io_recvmsg(req, force_nonblock);	5584	ret = io_recvmsg(req, force_nonblock, cs);
5251	else	5585	else
5252	ret = io_recv(req, force_nonblock);	5586	ret = io_recv(req, force_nonblock, cs);
5253	break;	5587	break;
5254	case IORING_OP_TIMEOUT:	5588	case IORING_OP_TIMEOUT:
5255	if (sqe) {	5589	if (sqe) {
@@ -5273,7 +5607,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 17 unmodified lines
5273	if (ret)	5607	if (ret)
5274	break;	5608	break;
5275	}	5609	}
5276	ret = io_accept(req, force_nonblock);	5610	ret = io_accept(req, force_nonblock, cs);
5277	break;	5611	break;
5278	case IORING_OP_CONNECT:	5612	case IORING_OP_CONNECT:
5279	if (sqe) {	5613	if (sqe) {
@@ -5281,7 +5615,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
5281	if (ret)	5615	if (ret)
5282	break;	5616	break;
5283	}	5617	}
5284	ret = io_connect(req, force_nonblock);	5618	ret = io_connect(req, force_nonblock, cs);
5285	break;	5619	break;
5286	case IORING_OP_ASYNC_CANCEL:	5620	case IORING_OP_ASYNC_CANCEL:
5287	if (sqe) {	5621	if (sqe) {
@@ -5313,7 +5647,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 25 unmodified lines
5313	if (ret)	5647	if (ret)
5314	break;	5648	break;
5315	}	5649	}
5316	ret = io_close(req, force_nonblock);	5650	ret = io_close(req, force_nonblock, cs);
5317	break;	5651	break;
5318	case IORING_OP_FILES_UPDATE:	5652	case IORING_OP_FILES_UPDATE:
5319	if (sqe) {	5653	if (sqe) {
@@ -5321,7 +5655,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
5321	if (ret)	5655	if (ret)
5322	break;	5656	break;
5323	}	5657	}
5324	ret = io_files_update(req, force_nonblock);	5658	ret = io_files_update(req, force_nonblock, cs);
5325	break;	5659	break;
5326	case IORING_OP_STATX:	5660	case IORING_OP_STATX:
5327	if (sqe) {	5661	if (sqe) {
@@ -5361,7 +5695,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 33 unmodified lines
5361	if (ret)	5695	if (ret)
5362	break;	5696	break;
5363	}	5697	}
5364	ret = io_epoll_ctl(req, force_nonblock);	5698	ret = io_epoll_ctl(req, force_nonblock, cs);
5365	break;	5699	break;
5366	case IORING_OP_SPLICE:	5700	case IORING_OP_SPLICE:
5367	if (sqe) {	5701	if (sqe) {
@@ -5377,7 +5711,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 9 unmodified lines
5377	if (ret)	5711	if (ret)
5378	break;	5712	break;
5379	}	5713	}
5380	ret = io_provide_buffers(req, force_nonblock);	5714	ret = io_provide_buffers(req, force_nonblock, cs);
5381	break;	5715	break;
5382	case IORING_OP_REMOVE_BUFFERS:	5716	case IORING_OP_REMOVE_BUFFERS:
5383	if (sqe) {	5717	if (sqe) {
@@ -5385,7 +5719,7 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 1 unmodified line
5385	if (ret)	5719	if (ret)
5386	break;	5720	break;
5387	}	5721	}
5388	ret = io_remove_buffers(req, force_nonblock);	5722	ret = io_remove_buffers(req, force_nonblock, cs);
5389	break;	5723	break;
5390	case IORING_OP_TEE:	5724	case IORING_OP_TEE:
5391	if (sqe) {	5725	if (sqe) {
@@ -5420,25 +5754,15 @@ static int io_issue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 28 unmodified lines
5420	return 0;	5754	return 0;
5421	}	5755	}
5422		5756
5423	static void io_arm_async_linked_timeout(struct io_kiocb *req)	5757	static struct io_wq_work io_wq_submit_work(struct io_wq_work work)
5424	{
5425	struct io_kiocb *link;
5426
5427	/* link head's timeout is queued in io_queue_async_work() */
5428	if (!(req->flags & REQ_F_QUEUE_TIMEOUT))
5429	return;
5430
5431	link = list_first_entry(&req->link_list, struct io_kiocb, link_list);
5432	io_queue_linked_timeout(link);
5433	}
5434
5435	static void io_wq_submit_work(struct io_wq_work **workptr)
5436	{	5758	{
5437	struct io_wq_work work = workptr;
5438	struct io_kiocb *req = container_of(work, struct io_kiocb, work);	5759	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
		5760	struct io_kiocb *timeout;
5439	int ret = 0;	5761	int ret = 0;
5440		5762
5441	io_arm_async_linked_timeout(req);	5763	timeout = io_prep_linked_timeout(req);
		5764	if (timeout)
		5765	io_queue_linked_timeout(timeout);
5442		5766
5443	/* if NO_CANCEL is set, we must still run the work */	5767	/* if NO_CANCEL is set, we must still run the work */
5444	if ((work->flags & (IO_WQ_WORK_CANCEL\|IO_WQ_WORK_NO_CANCEL)) ==	5768	if ((work->flags & (IO_WQ_WORK_CANCEL\|IO_WQ_WORK_NO_CANCEL)) ==
@@ -5448,7 +5772,7 @@ static void io_wq_submit_work(struct io_wq_work **workptr) · 3 unmodified lines
5448		5772
5449	if (!ret) {	5773	if (!ret) {
5450	do {	5774	do {
5451	ret = io_issue_sqe(req, NULL, false);	5775	ret = io_issue_sqe(req, NULL, false, NULL);
5452	/*	5776	/*
5453	* We can get EAGAIN for polled IO even though we're	5777	* We can get EAGAIN for polled IO even though we're
5454	* forcing a sync submission from here, since we can't	5778	* forcing a sync submission from here, since we can't
@@ -5462,11 +5786,10 @@ static void io_wq_submit_work(struct io_wq_work **workptr) · 7 unmodified lines
5462		5786
5463	if (ret) {	5787	if (ret) {
5464	req_set_fail_links(req);	5788	req_set_fail_links(req);
5465	io_cqring_add_event(req, ret);	5789	io_req_complete(req, ret);
5466	io_put_req(req);
5467	}	5790	}
5468		5791
5469	io_steal_work(req, workptr);	5792	return io_steal_work(req);
5470	}	5793	}
5471		5794
5472	static inline struct file io_file_from_index(struct io_ring_ctx ctx,	5795	static inline struct file io_file_from_index(struct io_ring_ctx ctx,
@@ -5523,6 +5846,8 @@ static int io_grab_files(struct io_kiocb *req) · 50 unmodified lines
5523	int ret = -EBADF;	5846	int ret = -EBADF;
5524	struct io_ring_ctx *ctx = req->ctx;	5847	struct io_ring_ctx *ctx = req->ctx;
5525		5848
		5849	io_req_init_async(req);
		5850
5526	if (req->work.files \|\| (req->flags & REQ_F_NO_FILE_TABLE))	5851	if (req->work.files \|\| (req->flags & REQ_F_NO_FILE_TABLE))
5527	return 0;	5852	return 0;
5528	if (!ctx->ring_file)	5853	if (!ctx->ring_file)
@@ -5548,6 +5873,13 @@ static int io_grab_files(struct io_kiocb *req) · 19 unmodified lines
5548	return ret;	5873	return ret;
5549	}	5874	}
5550		5875
		5876	static inline int io_prep_work_files(struct io_kiocb *req)
		5877	{
		5878	if (!io_op_defs[req->opcode].file_table)
		5879	return 0;
		5880	return io_grab_files(req);
		5881	}
		5882
5551	static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)	5883	static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
5552	{	5884	{
5553	struct io_timeout_data *data = container_of(timer,	5885	struct io_timeout_data *data = container_of(timer,
@@ -5580,8 +5912,7 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) · 26 unmodified lines
5580	io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME);	5912	io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME);
5581	io_put_req(prev);	5913	io_put_req(prev);
5582	} else {	5914	} else {
5583	io_cqring_add_event(req, -ETIME);	5915	io_req_complete(req, -ETIME);
5584	io_put_req(req);
5585	}	5916	}
5586	return HRTIMER_NORESTART;	5917	return HRTIMER_NORESTART;
5587	}	5918	}
@@ -5614,8 +5945,7 @@ static struct io_kiocb io_prep_linked_timeout(struct io_kiocb req) · 26 unmodified lines
5614		5945
5615	if (!(req->flags & REQ_F_LINK_HEAD))	5946	if (!(req->flags & REQ_F_LINK_HEAD))
5616	return NULL;	5947	return NULL;
5617	/* for polled retry, if flag is set, we already went through here */	5948	if (req->flags & REQ_F_LINK_TIMEOUT)
5618	if (req->flags & REQ_F_POLLED)
5619	return NULL;	5949	return NULL;
5620		5950
5621	nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,	5951	nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,
@@ -5627,7 +5957,8 @@ static struct io_kiocb io_prep_linked_timeout(struct io_kiocb req) · 5 unmodified lines
5627	return nxt;	5957	return nxt;
5628	}	5958	}
5629		5959
5630	static void __io_queue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe)	5960	static void __io_queue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe,
		5961	struct io_comp_state *cs)
5631	{	5962	{
5632	struct io_kiocb *linked_timeout;	5963	struct io_kiocb *linked_timeout;
5633	struct io_kiocb *nxt;	5964	struct io_kiocb *nxt;
@@ -5647,54 +5978,45 @@ again: · 13 unmodified lines
5647	old_creds = override_creds(req->work.creds);	5978	old_creds = override_creds(req->work.creds);
5648	}	5979	}
5649		5980
5650	ret = io_issue_sqe(req, sqe, true);	5981	ret = io_issue_sqe(req, sqe, true, cs);
5651		5982
5652	/*	5983	/*
5653	* We async punt it if the file wasn't marked NOWAIT, or if the file	5984	* We async punt it if the file wasn't marked NOWAIT, or if the file
5654	* doesn't support non-blocking read/write attempts	5985	* doesn't support non-blocking read/write attempts
5655	*/	5986	*/
5656	if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) \|\|	5987	if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
5657	(req->flags & REQ_F_MUST_PUNT))) {	5988	if (!io_arm_poll_handler(req)) {
5658	if (io_arm_poll_handler(req)) {
5659	if (linked_timeout)
5660	io_queue_linked_timeout(linked_timeout);
5661	goto exit;
5662	}
5663	punt:	5989	punt:
5664	io_req_init_async(req);	5990	ret = io_prep_work_files(req);
5665		5991	if (unlikely(ret))
5666	if (io_op_defs[req->opcode].file_table) {
5667	ret = io_grab_files(req);
5668	if (ret)
5669	goto err;	5992	goto err;
		5993	/*
		5994	* Queued up for async execution, worker will release
		5995	* submit reference when the iocb is actually submitted.
		5996	*/
		5997	io_queue_async_work(req);
5670	}	5998	}
5671		5999
5672	/*	6000	if (linked_timeout)
5673	* Queued up for async execution, worker will release	6001	io_queue_linked_timeout(linked_timeout);
5674	* submit reference when the iocb is actually submitted.
5675	*/
5676	io_queue_async_work(req);
5677	goto exit;	6002	goto exit;
5678	}	6003	}
5679		6004
		6005	if (unlikely(ret)) {
5680	err:	6006	err:
5681	nxt = NULL;	6007	/* un-prep timeout, so it'll be killed as any other linked */
5682	/* drop submission reference */	6008	req->flags &= ~REQ_F_LINK_TIMEOUT;
5683	io_put_req_find_next(req, &nxt);
5684
5685	if (linked_timeout) {
5686	if (!ret)
5687	io_queue_linked_timeout(linked_timeout);
5688	else
5689	io_put_req(linked_timeout);
5690	}
5691
5692	/* and drop final reference, if we failed */
5693	if (ret) {
5694	io_cqring_add_event(req, ret);
5695	req_set_fail_links(req);	6009	req_set_fail_links(req);
5696	io_put_req(req);	6010	io_put_req(req);
		6011	io_req_complete(req, ret);
		6012	goto exit;
5697	}	6013	}
		6014
		6015	/* drop submission reference */
		6016	nxt = io_put_req_find_next(req);
		6017	if (linked_timeout)
		6018	io_queue_linked_timeout(linked_timeout);
		6019
5698	if (nxt) {	6020	if (nxt) {
5699	req = nxt;	6021	req = nxt;
5700		6022
@@ -5707,7 +6029,8 @@ exit: · 6 unmodified lines
5707	revert_creds(old_creds);	6029	revert_creds(old_creds);
5708	}	6030	}
5709		6031
5710	static void io_queue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe)	6032	static void io_queue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe,
		6033	struct io_comp_state *cs)
5711	{	6034	{
5712	int ret;	6035	int ret;
5713		6036
@@ -5715,17 +6038,14 @@ static void io_queue_sqe(struct io_kiocb req, const struct io_uring_sqe sqe) · 1 unmodified line
5715	if (ret) {	6038	if (ret) {
5716	if (ret != -EIOCBQUEUED) {	6039	if (ret != -EIOCBQUEUED) {
5717	fail_req:	6040	fail_req:
5718	io_cqring_add_event(req, ret);
5719	req_set_fail_links(req);	6041	req_set_fail_links(req);
5720	io_double_put_req(req);	6042	io_put_req(req);
		6043	io_req_complete(req, ret);
5721	}	6044	}
5722	} else if (req->flags & REQ_F_FORCE_ASYNC) {	6045	} else if (req->flags & REQ_F_FORCE_ASYNC) {
5723	if (!req->io) {	6046	if (!req->io) {
5724	ret = -EAGAIN;
5725	if (io_alloc_async_ctx(req))
5726	goto fail_req;
5727	ret = io_req_defer_prep(req, sqe);	6047	ret = io_req_defer_prep(req, sqe);
5728	if (unlikely(ret < 0))	6048	if (unlikely(ret))
5729	goto fail_req;	6049	goto fail_req;
5730	}	6050	}
5731		6051
@@ -5737,21 +6057,22 @@ fail_req: · 5 unmodified lines
5737	req->work.flags \|= IO_WQ_WORK_CONCURRENT;	6057	req->work.flags \|= IO_WQ_WORK_CONCURRENT;
5738	io_queue_async_work(req);	6058	io_queue_async_work(req);
5739	} else {	6059	} else {
5740	__io_queue_sqe(req, sqe);	6060	__io_queue_sqe(req, sqe, cs);
5741	}	6061	}
5742	}	6062	}
5743		6063
5744	static inline void io_queue_link_head(struct io_kiocb *req)	6064	static inline void io_queue_link_head(struct io_kiocb *req,
		6065	struct io_comp_state *cs)
5745	{	6066	{
5746	if (unlikely(req->flags & REQ_F_FAIL_LINK)) {	6067	if (unlikely(req->flags & REQ_F_FAIL_LINK)) {
5747	io_cqring_add_event(req, -ECANCELED);	6068	io_put_req(req);
5748	io_double_put_req(req);	6069	io_req_complete(req, -ECANCELED);
5749	} else	6070	} else
5750	io_queue_sqe(req, NULL);	6071	io_queue_sqe(req, NULL, cs);
5751	}	6072	}
5752		6073
5753	static int io_submit_sqe(struct io_kiocb req, const struct io_uring_sqe sqe,	6074	static int io_submit_sqe(struct io_kiocb req, const struct io_uring_sqe sqe,
5754	struct io_kiocb **link)	6075	struct io_kiocb *link, struct io_comp_state cs)
5755	{	6076	{
5756	struct io_ring_ctx *ctx = req->ctx;	6077	struct io_ring_ctx *ctx = req->ctx;
5757	int ret;	6078	int ret;
@@ -5777,21 +6098,19 @@ static int io_submit_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 19 unmodified lines
5777	head->flags \|= REQ_F_IO_DRAIN;	6098	head->flags \|= REQ_F_IO_DRAIN;
5778	ctx->drain_next = 1;	6099	ctx->drain_next = 1;
5779	}	6100	}
5780	if (io_alloc_async_ctx(req))
5781	return -EAGAIN;
5782
5783	ret = io_req_defer_prep(req, sqe);	6101	ret = io_req_defer_prep(req, sqe);
5784	if (ret) {	6102	if (unlikely(ret)) {
5785	/* fail even hard links since we don't submit */	6103	/* fail even hard links since we don't submit */
5786	head->flags \|= REQ_F_FAIL_LINK;	6104	head->flags \|= REQ_F_FAIL_LINK;
5787	return ret;	6105	return ret;
5788	}	6106	}
5789	trace_io_uring_link(ctx, req, head);	6107	trace_io_uring_link(ctx, req, head);
		6108	io_get_req_task(req);
5790	list_add_tail(&req->link_list, &head->link_list);	6109	list_add_tail(&req->link_list, &head->link_list);
5791		6110
5792	/* last request of a link, enqueue the link */	6111	/* last request of a link, enqueue the link */
5793	if (!(req->flags & (REQ_F_LINK \| REQ_F_HARDLINK))) {	6112	if (!(req->flags & (REQ_F_LINK \| REQ_F_HARDLINK))) {
5794	io_queue_link_head(head);	6113	io_queue_link_head(head, cs);
5795	*link = NULL;	6114	*link = NULL;
5796	}	6115	}
5797	} else {	6116	} else {
@@ -5803,15 +6122,12 @@ static int io_submit_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 5 unmodified lines
5803	req->flags \|= REQ_F_LINK_HEAD;	6122	req->flags \|= REQ_F_LINK_HEAD;
5804	INIT_LIST_HEAD(&req->link_list);	6123	INIT_LIST_HEAD(&req->link_list);
5805		6124
5806	if (io_alloc_async_ctx(req))
5807	return -EAGAIN;
5808
5809	ret = io_req_defer_prep(req, sqe);	6125	ret = io_req_defer_prep(req, sqe);
5810	if (ret)	6126	if (unlikely(ret))
5811	req->flags \|= REQ_F_FAIL_LINK;	6127	req->flags \|= REQ_F_FAIL_LINK;
5812	*link = req;	6128	*link = req;
5813	} else {	6129	} else {
5814	io_queue_sqe(req, sqe);	6130	io_queue_sqe(req, sqe, cs);
5815	}	6131	}
5816	}	6132	}
5817		6133
@@ -5823,6 +6139,8 @@ static int io_submit_sqe(struct io_kiocb req, const struct io_uring_sqe sqe, · 5 unmodified lines
5823	*/	6139	*/
5824	static void io_submit_state_end(struct io_submit_state *state)	6140	static void io_submit_state_end(struct io_submit_state *state)
5825	{	6141	{
		6142	if (!list_empty(&state->comp.list))
		6143	io_submit_flush_completions(&state->comp);
5826	blk_finish_plug(&state->plug);	6144	blk_finish_plug(&state->plug);
5827	io_state_file_put(state);	6145	io_state_file_put(state);
5828	if (state->free_reqs)	6146	if (state->free_reqs)
@@ -5833,9 +6151,15 @@ static void io_submit_state_end(struct io_submit_state *state) · 4 unmodified lines
5833	* Start submission side cache.	6151	* Start submission side cache.
5834	*/	6152	*/
5835	static void io_submit_state_start(struct io_submit_state *state,	6153	static void io_submit_state_start(struct io_submit_state *state,
5836	unsigned int max_ios)	6154	struct io_ring_ctx *ctx, unsigned int max_ios)
5837	{	6155	{
5838	blk_start_plug(&state->plug);	6156	blk_start_plug(&state->plug);
		6157	#ifdef CONFIG_BLOCK
		6158	state->plug.nowait = true;
		6159	#endif
		6160	state->comp.nr = 0;
		6161	INIT_LIST_HEAD(&state->comp.list);
		6162	state->comp.ctx = ctx;
5839	state->free_reqs = 0;	6163	state->free_reqs = 0;
5840	state->file = NULL;	6164	state->file = NULL;
5841	state->ios_left = max_ios;	6165	state->ios_left = max_ios;
@@ -5900,12 +6224,6 @@ static int io_init_req(struct io_ring_ctx ctx, struct io_kiocb req, · 58 unmodified lines
5900	unsigned int sqe_flags;	6224	unsigned int sqe_flags;
5901	int id;	6225	int id;
5902		6226
5903	/*
5904	* All io need record the previous position, if LINK vs DARIN,
5905	* it can be used to mark the position of the first IO in the
5906	* link list.
5907	*/
5908	req->sequence = ctx->cached_sq_head - ctx->cached_sq_dropped;
5909	req->opcode = READ_ONCE(sqe->opcode);	6227	req->opcode = READ_ONCE(sqe->opcode);
5910	req->user_data = READ_ONCE(sqe->user_data);	6228	req->user_data = READ_ONCE(sqe->user_data);
5911	req->io = NULL;	6229	req->io = NULL;
@@ -5953,7 +6271,7 @@ static int io_init_req(struct io_ring_ctx ctx, struct io_kiocb req, · 41 unmodified lines
5953	static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,	6271	static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
5954	struct file *ring_file, int ring_fd)	6272	struct file *ring_file, int ring_fd)
5955	{	6273	{
5956	struct io_submit_state state, *statep = NULL;	6274	struct io_submit_state state;
5957	struct io_kiocb *link = NULL;	6275	struct io_kiocb *link = NULL;
5958	int i, submitted = 0;	6276	int i, submitted = 0;
5959		6277
@@ -5970,10 +6288,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr, · 10 unmodified lines
5970	if (!percpu_ref_tryget_many(&ctx->refs, nr))	6288	if (!percpu_ref_tryget_many(&ctx->refs, nr))
5971	return -EAGAIN;	6289	return -EAGAIN;
5972		6290
5973	if (nr > IO_PLUG_THRESHOLD) {	6291	io_submit_state_start(&state, ctx, nr);
5974	io_submit_state_start(&state, nr);
5975	statep = &state;
5976	}
5977		6292
5978	ctx->ring_fd = ring_fd;	6293	ctx->ring_fd = ring_fd;
5979	ctx->ring_file = ring_file;	6294	ctx->ring_file = ring_file;
@@ -5988,28 +6303,28 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr, · 8 unmodified lines
5988	io_consume_sqe(ctx);	6303	io_consume_sqe(ctx);
5989	break;	6304	break;
5990	}	6305	}
5991	req = io_alloc_req(ctx, statep);	6306	req = io_alloc_req(ctx, &state);
5992	if (unlikely(!req)) {	6307	if (unlikely(!req)) {
5993	if (!submitted)	6308	if (!submitted)
5994	submitted = -EAGAIN;	6309	submitted = -EAGAIN;
5995	break;	6310	break;
5996	}	6311	}
5997		6312
5998	err = io_init_req(ctx, req, sqe, statep);	6313	err = io_init_req(ctx, req, sqe, &state);
5999	io_consume_sqe(ctx);	6314	io_consume_sqe(ctx);
6000	/* will complete beyond this point, count as submitted */	6315	/* will complete beyond this point, count as submitted */
6001	submitted++;	6316	submitted++;
6002		6317
6003	if (unlikely(err)) {	6318	if (unlikely(err)) {
6004	fail_req:	6319	fail_req:
6005	io_cqring_add_event(req, err);	6320	io_put_req(req);
6006	io_double_put_req(req);	6321	io_req_complete(req, err);
6007	break;	6322	break;
6008	}	6323	}
6009		6324
6010	trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,	6325	trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
6011	true, io_async_submit(ctx));	6326	true, io_async_submit(ctx));
6012	err = io_submit_sqe(req, sqe, &link);	6327	err = io_submit_sqe(req, sqe, &link, &state.comp);
6013	if (err)	6328	if (err)
6014	goto fail_req;	6329	goto fail_req;
6015	}	6330	}
@@ -6020,9 +6335,8 @@ fail_req: · 4 unmodified lines
6020	percpu_ref_put_many(&ctx->refs, nr - ref_used);	6335	percpu_ref_put_many(&ctx->refs, nr - ref_used);
6021	}	6336	}
6022	if (link)	6337	if (link)
6023	io_queue_link_head(link);	6338	io_queue_link_head(link, &state.comp);
6024	if (statep)	6339	io_submit_state_end(&state);
6025	io_submit_state_end(&state);
6026		6340
6027	/* Commit SQ ring head once we've consumed and submitted all SQEs */	6341	/* Commit SQ ring head once we've consumed and submitted all SQEs */
6028	io_commit_sqring(ctx);	6342	io_commit_sqring(ctx);
@@ -6030,6 +6344,21 @@ fail_req: · 1 unmodified line
6030	return submitted;	6344	return submitted;
6031	}	6345	}
6032		6346
		6347	static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
		6348	{
		6349	/* Tell userspace we may need a wakeup call */
		6350	spin_lock_irq(&ctx->completion_lock);
		6351	ctx->rings->sq_flags \|= IORING_SQ_NEED_WAKEUP;
		6352	spin_unlock_irq(&ctx->completion_lock);
		6353	}
		6354
		6355	static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
		6356	{
		6357	spin_lock_irq(&ctx->completion_lock);
		6358	ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
		6359	spin_unlock_irq(&ctx->completion_lock);
		6360	}
		6361
6033	static int io_sq_thread(void *data)	6362	static int io_sq_thread(void *data)
6034	{	6363	{
6035	struct io_ring_ctx *ctx = data;	6364	struct io_ring_ctx *ctx = data;
@@ -6046,12 +6375,12 @@ static int io_sq_thread(void *data) · 10 unmodified lines
6046	while (!kthread_should_park()) {	6375	while (!kthread_should_park()) {
6047	unsigned int to_submit;	6376	unsigned int to_submit;
6048		6377
6049	if (!list_empty(&ctx->poll_list)) {	6378	if (!list_empty(&ctx->iopoll_list)) {
6050	unsigned nr_events = 0;	6379	unsigned nr_events = 0;
6051		6380
6052	mutex_lock(&ctx->uring_lock);	6381	mutex_lock(&ctx->uring_lock);
6053	if (!list_empty(&ctx->poll_list))	6382	if (!list_empty(&ctx->iopoll_list) && !need_resched())
6054	io_iopoll_getevents(ctx, &nr_events, 0);	6383	io_do_iopoll(ctx, &nr_events, 0);
6055	else	6384	else
6056	timeout = jiffies + ctx->sq_thread_idle;	6385	timeout = jiffies + ctx->sq_thread_idle;
6057	mutex_unlock(&ctx->uring_lock);	6386	mutex_unlock(&ctx->uring_lock);
@@ -6070,7 +6399,7 @@ static int io_sq_thread(void *data) · 12 unmodified lines
6070	* adding ourselves to the waitqueue, as the unuse/drop	6399	* adding ourselves to the waitqueue, as the unuse/drop
6071	* may sleep.	6400	* may sleep.
6072	*/	6401	*/
6073	io_sq_thread_drop_mm(ctx);	6402	io_sq_thread_drop_mm();
6074		6403
6075	/*	6404	/*
6076	* We're polling. If we're within the defined idle	6405	* We're polling. If we're within the defined idle
@@ -6079,11 +6408,10 @@ static int io_sq_thread(void *data) · 2 unmodified lines
6079	* more IO, we should wait for the application to	6408	* more IO, we should wait for the application to
6080	* reap events and wake us up.	6409	* reap events and wake us up.
6081	*/	6410	*/
6082	if (!list_empty(&ctx->poll_list) \|\| need_resched() \|\|	6411	if (!list_empty(&ctx->iopoll_list) \|\| need_resched() \|\|
6083	(!time_after(jiffies, timeout) && ret != -EBUSY &&	6412	(!time_after(jiffies, timeout) && ret != -EBUSY &&
6084	!percpu_ref_is_dying(&ctx->refs))) {	6413	!percpu_ref_is_dying(&ctx->refs))) {
6085	if (current->task_works)	6414	io_run_task_work();
6086	task_work_run();
6087	cond_resched();	6415	cond_resched();
6088	continue;	6416	continue;
6089	}	6417	}
@@ -6093,21 +6421,18 @@ static int io_sq_thread(void *data) · 3 unmodified lines
6093		6421
6094	/*	6422	/*
6095	* While doing polled IO, before going to sleep, we need	6423	* While doing polled IO, before going to sleep, we need
6096	* to check if there are new reqs added to poll_list, it	6424	* to check if there are new reqs added to iopoll_list,
6097	* is because reqs may have been punted to io worker and	6425	* it is because reqs may have been punted to io worker
6098	* will be added to poll_list later, hence check the	6426	* and will be added to iopoll_list later, hence check
6099	* poll_list again.	6427	* the iopoll_list again.
6100	*/	6428	*/
6101	if ((ctx->flags & IORING_SETUP_IOPOLL) &&	6429	if ((ctx->flags & IORING_SETUP_IOPOLL) &&
6102	!list_empty_careful(&ctx->poll_list)) {	6430	!list_empty_careful(&ctx->iopoll_list)) {
6103	finish_wait(&ctx->sqo_wait, &wait);	6431	finish_wait(&ctx->sqo_wait, &wait);
6104	continue;	6432	continue;
6105	}	6433	}
6106		6434
6107	/* Tell userspace we may need a wakeup call */	6435	io_ring_set_wakeup_flag(ctx);
6108	spin_lock_irq(&ctx->completion_lock);
6109	ctx->rings->sq_flags \|= IORING_SQ_NEED_WAKEUP;
6110	spin_unlock_irq(&ctx->completion_lock);
6111		6436
6112	to_submit = io_sqring_entries(ctx);	6437	to_submit = io_sqring_entries(ctx);
6113	if (!to_submit \|\| ret == -EBUSY) {	6438	if (!to_submit \|\| ret == -EBUSY) {
@@ -6115,9 +6440,9 @@ static int io_sq_thread(void *data) · 1 unmodified line
6115	finish_wait(&ctx->sqo_wait, &wait);	6440	finish_wait(&ctx->sqo_wait, &wait);
6116	break;	6441	break;
6117	}	6442	}
6118	if (current->task_works) {	6443	if (io_run_task_work()) {
6119	task_work_run();
6120	finish_wait(&ctx->sqo_wait, &wait);	6444	finish_wait(&ctx->sqo_wait, &wait);
		6445	io_ring_clear_wakeup_flag(ctx);
6121	continue;	6446	continue;
6122	}	6447	}
6123	if (signal_pending(current))	6448	if (signal_pending(current))
@@ -6125,17 +6450,13 @@ static int io_sq_thread(void *data) · 1 unmodified line
6125	schedule();	6450	schedule();
6126	finish_wait(&ctx->sqo_wait, &wait);	6451	finish_wait(&ctx->sqo_wait, &wait);
6127		6452
6128	spin_lock_irq(&ctx->completion_lock);	6453	io_ring_clear_wakeup_flag(ctx);
6129	ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
6130	spin_unlock_irq(&ctx->completion_lock);
6131	ret = 0;	6454	ret = 0;
6132	continue;	6455	continue;
6133	}	6456	}
6134	finish_wait(&ctx->sqo_wait, &wait);	6457	finish_wait(&ctx->sqo_wait, &wait);
6135		6458
6136	spin_lock_irq(&ctx->completion_lock);	6459	io_ring_clear_wakeup_flag(ctx);
6137	ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
6138	spin_unlock_irq(&ctx->completion_lock);
6139	}	6460	}
6140		6461
6141	mutex_lock(&ctx->uring_lock);	6462	mutex_lock(&ctx->uring_lock);
@@ -6145,10 +6466,9 @@ static int io_sq_thread(void *data) · 3 unmodified lines
6145	timeout = jiffies + ctx->sq_thread_idle;	6466	timeout = jiffies + ctx->sq_thread_idle;
6146	}	6467	}
6147		6468
6148	if (current->task_works)	6469	io_run_task_work();
6149	task_work_run();
6150		6470
6151	io_sq_thread_drop_mm(ctx);	6471	io_sq_thread_drop_mm();
6152	revert_creds(old_cred);	6472	revert_creds(old_cred);
6153		6473
6154	kthread_parkme();	6474	kthread_parkme();
@@ -6211,9 +6531,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 56 unmodified lines
6211	do {	6531	do {
6212	if (io_cqring_events(ctx, false) >= min_events)	6532	if (io_cqring_events(ctx, false) >= min_events)
6213	return 0;	6533	return 0;
6214	if (!current->task_works)	6534	if (!io_run_task_work())
6215	break;	6535	break;
6216	task_work_run();
6217	} while (1);	6536	} while (1);
6218		6537
6219	if (sig) {	6538	if (sig) {
@@ -6235,8 +6554,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 15 unmodified lines
6235	prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,	6554	prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
6236	TASK_INTERRUPTIBLE);	6555	TASK_INTERRUPTIBLE);
6237	/* make sure we run task_work before checking for signals */	6556	/* make sure we run task_work before checking for signals */
6238	if (current->task_works)	6557	if (io_run_task_work())
6239	task_work_run();	6558	continue;
6240	if (signal_pending(current)) {	6559	if (signal_pending(current)) {
6241	if (current->jobctl & JOBCTL_TASK_WORK) {	6560	if (current->jobctl & JOBCTL_TASK_WORK) {
6242	spin_lock_irq(&current->sighand->siglock);	6561	spin_lock_irq(&current->sighand->siglock);
@@ -7022,17 +7341,21 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx, · 779 unmodified lines
7022	return 0;	7341	return 0;
7023	err:	7342	err:
7024	io_finish_async(ctx);	7343	io_finish_async(ctx);
7025	mmdrop(ctx->sqo_mm);	7344	if (ctx->sqo_mm) {
7026	ctx->sqo_mm = NULL;	7345	mmdrop(ctx->sqo_mm);
		7346	ctx->sqo_mm = NULL;
		7347	}
7027	return ret;	7348	return ret;
7028	}	7349	}
7029		7350
7030	static void io_unaccount_mem(struct user_struct *user, unsigned long nr_pages)	7351	static inline void __io_unaccount_mem(struct user_struct *user,
		7352	unsigned long nr_pages)
7031	{	7353	{
7032	atomic_long_sub(nr_pages, &user->locked_vm);	7354	atomic_long_sub(nr_pages, &user->locked_vm);
7033	}	7355	}
7034		7356
7035	static int io_account_mem(struct user_struct *user, unsigned long nr_pages)	7357	static inline int __io_account_mem(struct user_struct *user,
		7358	unsigned long nr_pages)
7036	{	7359	{
7037	unsigned long page_limit, cur_pages, new_pages;	7360	unsigned long page_limit, cur_pages, new_pages;
7038		7361
@@ -7050,6 +7373,41 @@ static int io_account_mem(struct user_struct *user, unsigned long nr_pages) · 11 unmodified lines
7050	return 0;	7373	return 0;
7051	}	7374	}
7052		7375
		7376	static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
		7377	enum io_mem_account acct)
		7378	{
		7379	if (ctx->limit_mem)
		7380	__io_unaccount_mem(ctx->user, nr_pages);
		7381
		7382	if (ctx->sqo_mm) {
		7383	if (acct == ACCT_LOCKED)
		7384	ctx->sqo_mm->locked_vm -= nr_pages;
		7385	else if (acct == ACCT_PINNED)
		7386	atomic64_sub(nr_pages, &ctx->sqo_mm->pinned_vm);
		7387	}
		7388	}
		7389
		7390	static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
		7391	enum io_mem_account acct)
		7392	{
		7393	int ret;
		7394
		7395	if (ctx->limit_mem) {
		7396	ret = __io_account_mem(ctx->user, nr_pages);
		7397	if (ret)
		7398	return ret;
		7399	}
		7400
		7401	if (ctx->sqo_mm) {
		7402	if (acct == ACCT_LOCKED)
		7403	ctx->sqo_mm->locked_vm += nr_pages;
		7404	else if (acct == ACCT_PINNED)
		7405	atomic64_add(nr_pages, &ctx->sqo_mm->pinned_vm);
		7406	}
		7407
		7408	return 0;
		7409	}
		7410
7053	static void io_mem_free(void *ptr)	7411	static void io_mem_free(void *ptr)
7054	{	7412	{
7055	struct page *page;	7413	struct page *page;
@@ -7086,6 +7444,9 @@ static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries, · 30 unmodified lines
7086	return SIZE_MAX;	7444	return SIZE_MAX;
7087	#endif	7445	#endif
7088		7446
		7447	if (sq_offset)
		7448	*sq_offset = off;
		7449
7089	sq_array_size = array_size(sizeof(u32), sq_entries);	7450	sq_array_size = array_size(sizeof(u32), sq_entries);
7090	if (sq_array_size == SIZE_MAX)	7451	if (sq_array_size == SIZE_MAX)
7091	return SIZE_MAX;	7452	return SIZE_MAX;
@@ -7093,9 +7454,6 @@ static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries, · 1 unmodified line
7093	if (check_add_overflow(off, sq_array_size, &off))	7454	if (check_add_overflow(off, sq_array_size, &off))
7094	return SIZE_MAX;	7455	return SIZE_MAX;
7095		7456
7096	if (sq_offset)
7097	*sq_offset = off;
7098
7099	return off;	7457	return off;
7100	}	7458	}
7101		7459
@@ -7124,8 +7482,7 @@ static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx) · 22 unmodified lines
7124	for (j = 0; j < imu->nr_bvecs; j++)	7482	for (j = 0; j < imu->nr_bvecs; j++)
7125	unpin_user_page(imu->bvec[j].bv_page);	7483	unpin_user_page(imu->bvec[j].bv_page);
7126		7484
7127	if (ctx->account_mem)	7485	io_unaccount_mem(ctx, imu->nr_bvecs, ACCT_PINNED);
7128	io_unaccount_mem(ctx->user, imu->nr_bvecs);
7129	kvfree(imu->bvec);	7486	kvfree(imu->bvec);
7130	imu->nr_bvecs = 0;	7487	imu->nr_bvecs = 0;
7131	}	7488	}
@@ -7208,11 +7565,9 @@ static int io_sqe_buffer_register(struct io_ring_ctx ctx, void __user arg, · 76 unmodified lines
7208	start = ubuf >> PAGE_SHIFT;	7565	start = ubuf >> PAGE_SHIFT;
7209	nr_pages = end - start;	7566	nr_pages = end - start;
7210		7567
7211	if (ctx->account_mem) {	7568	ret = io_account_mem(ctx, nr_pages, ACCT_PINNED);
7212	ret = io_account_mem(ctx->user, nr_pages);	7569	if (ret)
7213	if (ret)	7570	goto err;
7214	goto err;
7215	}
7216		7571
7217	ret = 0;	7572	ret = 0;
7218	if (!pages \|\| nr_pages > got_pages) {	7573	if (!pages \|\| nr_pages > got_pages) {
@@ -7225,8 +7580,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx ctx, void __user arg, · 6 unmodified lines
7225	GFP_KERNEL);	7580	GFP_KERNEL);
7226	if (!pages \|\| !vmas) {	7581	if (!pages \|\| !vmas) {
7227	ret = -ENOMEM;	7582	ret = -ENOMEM;
7228	if (ctx->account_mem)	7583	io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
7229	io_unaccount_mem(ctx->user, nr_pages);
7230	goto err;	7584	goto err;
7231	}	7585	}
7232	got_pages = nr_pages;	7586	got_pages = nr_pages;
@@ -7236,8 +7590,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx ctx, void __user arg, · 3 unmodified lines
7236	GFP_KERNEL);	7590	GFP_KERNEL);
7237	ret = -ENOMEM;	7591	ret = -ENOMEM;
7238	if (!imu->bvec) {	7592	if (!imu->bvec) {
7239	if (ctx->account_mem)	7593	io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
7240	io_unaccount_mem(ctx->user, nr_pages);
7241	goto err;	7594	goto err;
7242	}	7595	}
7243		7596
@@ -7268,8 +7621,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx ctx, void __user arg, · 24 unmodified lines
7268	*/	7621	*/
7269	if (pret > 0)	7622	if (pret > 0)
7270	unpin_user_pages(pages, pret);	7623	unpin_user_pages(pages, pret);
7271	if (ctx->account_mem)	7624	io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
7272	io_unaccount_mem(ctx->user, nr_pages);
7273	kvfree(imu->bvec);	7625	kvfree(imu->bvec);
7274	goto err;	7626	goto err;
7275	}	7627	}
@@ -7353,11 +7705,12 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx) · 77 unmodified lines
7353	static void io_ring_ctx_free(struct io_ring_ctx *ctx)	7705	static void io_ring_ctx_free(struct io_ring_ctx *ctx)
7354	{	7706	{
7355	io_finish_async(ctx);	7707	io_finish_async(ctx);
7356	if (ctx->sqo_mm)	7708	io_sqe_buffer_unregister(ctx);
		7709	if (ctx->sqo_mm) {
7357	mmdrop(ctx->sqo_mm);	7710	mmdrop(ctx->sqo_mm);
		7711	ctx->sqo_mm = NULL;
		7712	}
7358		7713
7359	io_iopoll_reap_events(ctx);
7360	io_sqe_buffer_unregister(ctx);
7361	io_sqe_files_unregister(ctx);	7714	io_sqe_files_unregister(ctx);
7362	io_eventfd_unregister(ctx);	7715	io_eventfd_unregister(ctx);
7363	io_destroy_buffers(ctx);	7716	io_destroy_buffers(ctx);
@@ -7421,11 +7774,8 @@ static int io_remove_personalities(int id, void p, void data) · 57 unmodified lines
7421		7774
7422	static void io_ring_exit_work(struct work_struct *work)	7775	static void io_ring_exit_work(struct work_struct *work)
7423	{	7776	{
7424	struct io_ring_ctx *ctx;	7777	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
7425		7778	exit_work);
7426	ctx = container_of(work, struct io_ring_ctx, exit_work);
7427	if (ctx->rings)
7428	io_cqring_overflow_flush(ctx, true);
7429		7779
7430	/*	7780	/*
7431	* If we're doing polled IO and end up having requests being	7781	* If we're doing polled IO and end up having requests being
@@ -7433,11 +7783,11 @@ static void io_ring_exit_work(struct work_struct *work) · 1 unmodified line
7433	* we're waiting for refs to drop. We need to reap these manually,	7783	* we're waiting for refs to drop. We need to reap these manually,
7434	* as nobody else will be looking for them.	7784	* as nobody else will be looking for them.
7435	*/	7785	*/
7436	while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20)) {	7786	do {
7437	io_iopoll_reap_events(ctx);
7438	if (ctx->rings)	7787	if (ctx->rings)
7439	io_cqring_overflow_flush(ctx, true);	7788	io_cqring_overflow_flush(ctx, true);
7440	}	7789	io_iopoll_try_reap_events(ctx);
		7790	} while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20));
7441	io_ring_ctx_free(ctx);	7791	io_ring_ctx_free(ctx);
7442	}	7792	}
7443		7793
@@ -7453,10 +7803,10 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) · 9 unmodified lines
7453	if (ctx->io_wq)	7803	if (ctx->io_wq)
7454	io_wq_cancel_all(ctx->io_wq);	7804	io_wq_cancel_all(ctx->io_wq);
7455		7805
7456	io_iopoll_reap_events(ctx);
7457	/* if we failed setting up the ctx, we might not have any rings */	7806	/* if we failed setting up the ctx, we might not have any rings */
7458	if (ctx->rings)	7807	if (ctx->rings)
7459	io_cqring_overflow_flush(ctx, true);	7808	io_cqring_overflow_flush(ctx, true);
		7809	io_iopoll_try_reap_events(ctx);
7460	idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx);	7810	idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx);
7461		7811
7462	/*	7812	/*
@@ -7464,9 +7814,8 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) · 1 unmodified line
7464	* is closed but resources aren't reaped yet. This can cause	7814	* is closed but resources aren't reaped yet. This can cause
7465	* spurious failure in setting up a new ring.	7815	* spurious failure in setting up a new ring.
7466	*/	7816	*/
7467	if (ctx->account_mem)	7817	io_unaccount_mem(ctx, ring_pages(ctx->sq_entries, ctx->cq_entries),
7468	io_unaccount_mem(ctx->user,	7818	ACCT_LOCKED);
7469	ring_pages(ctx->sq_entries, ctx->cq_entries));
7470		7819
7471	INIT_WORK(&ctx->exit_work, io_ring_exit_work);	7820	INIT_WORK(&ctx->exit_work, io_ring_exit_work);
7472	queue_work(system_wq, &ctx->exit_work);	7821	queue_work(system_wq, &ctx->exit_work);
@@ -7522,17 +7871,14 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx, · 49 unmodified lines
7522		7871
7523	if (cancel_req->flags & REQ_F_OVERFLOW) {	7872	if (cancel_req->flags & REQ_F_OVERFLOW) {
7524	spin_lock_irq(&ctx->completion_lock);	7873	spin_lock_irq(&ctx->completion_lock);
7525	list_del(&cancel_req->list);	7874	list_del(&cancel_req->compl.list);
7526	cancel_req->flags &= ~REQ_F_OVERFLOW;	7875	cancel_req->flags &= ~REQ_F_OVERFLOW;
7527	if (list_empty(&ctx->cq_overflow_list)) {
7528	clear_bit(0, &ctx->sq_check_overflow);
7529	clear_bit(0, &ctx->cq_check_overflow);
7530	ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW;
7531	}
7532	spin_unlock_irq(&ctx->completion_lock);
7533		7876
		7877	io_cqring_mark_overflow(ctx);
7534	WRITE_ONCE(ctx->rings->cq_overflow,	7878	WRITE_ONCE(ctx->rings->cq_overflow,
7535	atomic_inc_return(&ctx->cached_cq_overflow));	7879	atomic_inc_return(&ctx->cached_cq_overflow));
		7880	io_commit_cqring(ctx);
		7881	spin_unlock_irq(&ctx->completion_lock);
7536		7882
7537	/*	7883	/*
7538	* Put inflight ref and overflow ref. If that's	7884	* Put inflight ref and overflow ref. If that's
@@ -7655,8 +8001,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 116 unmodified lines
7655	int submitted = 0;	8001	int submitted = 0;
7656	struct fd f;	8002	struct fd f;
7657		8003
7658	if (current->task_works)	8004	io_run_task_work();
7659	task_work_run();
7660		8005
7661	if (flags & ~(IORING_ENTER_GETEVENTS \| IORING_ENTER_SQ_WAKEUP))	8006	if (flags & ~(IORING_ENTER_GETEVENTS \| IORING_ENTER_SQ_WAKEUP))
7662	return -EINVAL;	8007	return -EINVAL;
@@ -7695,8 +8040,6 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 32 unmodified lines
7695	goto out;	8040	goto out;
7696	}	8041	}
7697	if (flags & IORING_ENTER_GETEVENTS) {	8042	if (flags & IORING_ENTER_GETEVENTS) {
7698	unsigned nr_events = 0;
7699
7700	min_complete = min(min_complete, ctx->cq_entries);	8043	min_complete = min(min_complete, ctx->cq_entries);
7701		8044
7702	/*	8045	/*
@@ -7707,7 +8050,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 4 unmodified lines
7707	*/	8050	*/
7708	if (ctx->flags & IORING_SETUP_IOPOLL &&	8051	if (ctx->flags & IORING_SETUP_IOPOLL &&
7709	!(ctx->flags & IORING_SETUP_SQPOLL)) {	8052	!(ctx->flags & IORING_SETUP_SQPOLL)) {
7710	ret = io_iopoll_check(ctx, &nr_events, min_complete);	8053	ret = io_iopoll_check(ctx, min_complete);
7711	} else {	8054	} else {
7712	ret = io_cqring_wait(ctx, min_complete, sig, sigsz);	8055	ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
7713	}	8056	}
@@ -7912,7 +8255,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, · 198 unmodified lines
7912	{	8255	{
7913	struct user_struct *user = NULL;	8256	struct user_struct *user = NULL;
7914	struct io_ring_ctx *ctx;	8257	struct io_ring_ctx *ctx;
7915	bool account_mem;	8258	bool limit_mem;
7916	int ret;	8259	int ret;
7917		8260
7918	if (!entries)	8261	if (!entries)
@@ -7951,10 +8294,10 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, · 32 unmodified lines
7951	}	8294	}
7952		8295
7953	user = get_uid(current_user());	8296	user = get_uid(current_user());
7954	account_mem = !capable(CAP_IPC_LOCK);	8297	limit_mem = !capable(CAP_IPC_LOCK);
7955		8298
7956	if (account_mem) {	8299	if (limit_mem) {
7957	ret = io_account_mem(user,	8300	ret = __io_account_mem(user,
7958	ring_pages(p->sq_entries, p->cq_entries));	8301	ring_pages(p->sq_entries, p->cq_entries));
7959	if (ret) {	8302	if (ret) {
7960	free_uid(user);	8303	free_uid(user);
@@ -7964,14 +8307,13 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, · 3 unmodified lines
7964		8307
7965	ctx = io_ring_ctx_alloc(p);	8308	ctx = io_ring_ctx_alloc(p);
7966	if (!ctx) {	8309	if (!ctx) {
7967	if (account_mem)	8310	if (limit_mem)
7968	io_unaccount_mem(user, ring_pages(p->sq_entries,	8311	__io_unaccount_mem(user, ring_pages(p->sq_entries,
7969	p->cq_entries));	8312	p->cq_entries));
7970	free_uid(user);	8313	free_uid(user);
7971	return -ENOMEM;	8314	return -ENOMEM;
7972	}	8315	}
7973	ctx->compat = in_compat_syscall();	8316	ctx->compat = in_compat_syscall();
7974	ctx->account_mem = account_mem;
7975	ctx->user = user;	8317	ctx->user = user;
7976	ctx->creds = get_current_cred();	8318	ctx->creds = get_current_cred();
7977		8319
@@ -8003,12 +8345,22 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, · 25 unmodified lines
8003		8345
8004	p->features = IORING_FEAT_SINGLE_MMAP \| IORING_FEAT_NODROP \|	8346	p->features = IORING_FEAT_SINGLE_MMAP \| IORING_FEAT_NODROP \|
8005	IORING_FEAT_SUBMIT_STABLE \| IORING_FEAT_RW_CUR_POS \|	8347	IORING_FEAT_SUBMIT_STABLE \| IORING_FEAT_RW_CUR_POS \|
8006	IORING_FEAT_CUR_PERSONALITY \| IORING_FEAT_FAST_POLL;	8348	IORING_FEAT_CUR_PERSONALITY \| IORING_FEAT_FAST_POLL \|
		8349	IORING_FEAT_POLL_32BITS;
8007		8350
8008	if (copy_to_user(params, p, sizeof(*p))) {	8351	if (copy_to_user(params, p, sizeof(*p))) {
8009	ret = -EFAULT;	8352	ret = -EFAULT;
8010	goto err;	8353	goto err;
8011	}	8354	}
		8355
		8356	/*
		8357	* Account memory _before_ installing the file descriptor. Once
		8358	* the descriptor is installed, it can get closed at any time.
		8359	*/
		8360	io_account_mem(ctx, ring_pages(p->sq_entries, p->cq_entries),
		8361	ACCT_LOCKED);
		8362	ctx->limit_mem = limit_mem;
		8363
8012	/*	8364	/*
8013	* Install ring fd as the very last thing, so we don't risk someone	8365	* Install ring fd as the very last thing, so we don't risk someone
8014	* having closed it before we finish setup	8366	* having closed it before we finish setup
@@ -8292,7 +8644,8 @@ static int __init io_uring_init(void) · 277 unmodified lines
8292	BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);	8644	BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);
8293	BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);	8645	BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
8294	BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags);	8646	BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags);
8295	BUILD_BUG_SQE_ELEM(28, __u16, poll_events);	8647	BUILD_BUG_SQE_ELEM(28, /* compat */ __u16, poll_events);
		8648	BUILD_BUG_SQE_ELEM(28, __u32, poll32_events);
8296	BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags);	8649	BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags);
8297	BUILD_BUG_SQE_ELEM(28, __u32, msg_flags);	8650	BUILD_BUG_SQE_ELEM(28, __u32, msg_flags);
8298	BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags);	8651	BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags);

fs/xfs/xfs_file.c+1−1

@@ -1080,7 +1080,7 @@ xfs_file_open( · 1079 unmodified lines
1080	return -EFBIG;	1080	return -EFBIG;
1081	if (XFS_FORCED_SHUTDOWN(XFS_M(inode->i_sb)))	1081	if (XFS_FORCED_SHUTDOWN(XFS_M(inode->i_sb)))
1082	return -EIO;	1082	return -EIO;
1083	file->f_mode \|= FMODE_NOWAIT;	1083	file->f_mode \|= FMODE_NOWAIT \| FMODE_BUF_RASYNC;
1084	return 0;	1084	return 0;
1085	}	1085	}
1086		1086

include/linux/blkdev.h+1−0

@@ -1180,6 +1180,7 @@ struct blk_plug { · 1179 unmodified lines
1180	struct list_head cb_list; /* md requires an unplug callback */	1180	struct list_head cb_list; /* md requires an unplug callback */
1181	unsigned short rq_count;	1181	unsigned short rq_count;
1182	bool multiple_queues;	1182	bool multiple_queues;
		1183	bool nowait;
1183	};	1184	};
1184	#define BLK_MAX_REQUEST_COUNT 16	1185	#define BLK_MAX_REQUEST_COUNT 16
1185	#define BLK_PLUG_FLUSH_SIZE (128 * 1024)	1186	#define BLK_PLUG_FLUSH_SIZE (128 * 1024)

include/linux/fs.h+20−6

 /* File does not contribute to nr_files count */
 #define FMODE_NOACCOUNT		((__force fmode_t)0x20000000)
+/* File supports async buffered reads */
+#define FMODE_BUF_RASYNC	((__force fmode_t)0x40000000)
 /*
  * Flag for rw_copy_check_uvector and compat_rw_copy_check_uvector
  * that indicates that they should check the contents of the iovec are
 #define IOCB_SYNC		(1 << 5)
 #define IOCB_WRITE		(1 << 6)
 #define IOCB_NOWAIT		(1 << 7)
+/* iocb->ki_waitq is valid */
+#define IOCB_WAITQ		(1 << 8)
 #define IOCB_NOIO		(1 << 9)
 struct kiocb {
 	int			ki_flags;
 	u16			ki_hint;
 	u16			ki_ioprio; /* See linux/ioprio.h */
-	unsigned int		ki_cookie; /* for ->iopoll */
+	union {
+		unsigned int		ki_cookie; /* for ->iopoll */
+		struct wait_page_queue	*ki_waitq; /* for async buffered IO */
+	};
 	randomized_struct_fields_end
 };
 static inline int kiocb_set_rw_flags(struct kiocb *ki, rwf_t flags)
 {
+	int kiocb_flags = 0;
+	if (!flags)
+		return 0;
 	if (unlikely(flags & ~RWF_SUPPORTED))
 		return -EOPNOTSUPP;
 	if (flags & RWF_NOWAIT) {
 		if (!(ki->ki_filp->f_mode & FMODE_NOWAIT))
 			return -EOPNOTSUPP;
-		ki->ki_flags |= IOCB_NOWAIT;
+		kiocb_flags |= IOCB_NOWAIT;
 	}
 	if (flags & RWF_HIPRI)
-		ki->ki_flags |= IOCB_HIPRI;
+		kiocb_flags |= IOCB_HIPRI;
 	if (flags & RWF_DSYNC)
-		ki->ki_flags |= IOCB_DSYNC;
+		kiocb_flags |= IOCB_DSYNC;
 	if (flags & RWF_SYNC)
-		ki->ki_flags |= (IOCB_DSYNC | IOCB_SYNC);
+		kiocb_flags |= (IOCB_DSYNC | IOCB_SYNC);
 	if (flags & RWF_APPEND)
-		ki->ki_flags |= IOCB_APPEND;
+		kiocb_flags |= IOCB_APPEND;
+	ki->ki_flags |= kiocb_flags;
 	return 0;
 }

include/linux/pagemap.h+43−0

@@ -496,8 +496,35 @@ static inline pgoff_t linear_page_index(struct vm_area_struct *vma, · 495 unmodified lines
496	return pgoff;	496	return pgoff;
497	}	497	}
498		498
		499	/* This has the same layout as wait_bit_key - see fs/cachefiles/rdwr.c */
		500	struct wait_page_key {
		501	struct page *page;
		502	int bit_nr;
		503	int page_match;
		504	};
		505
		506	struct wait_page_queue {
		507	struct page *page;
		508	int bit_nr;
		509	wait_queue_entry_t wait;
		510	};
		511
		512	static inline bool wake_page_match(struct wait_page_queue *wait_page,
		513	struct wait_page_key *key)
		514	{
		515	if (wait_page->page != key->page)
		516	return false;
		517	key->page_match = 1;
		518
		519	if (wait_page->bit_nr != key->bit_nr)
		520	return false;
		521
		522	return true;
		523	}
		524
499	extern void __lock_page(struct page *page);	525	extern void __lock_page(struct page *page);
500	extern int __lock_page_killable(struct page *page);	526	extern int __lock_page_killable(struct page *page);
		527	extern int __lock_page_async(struct page page, struct wait_page_queue wait);
501	extern int __lock_page_or_retry(struct page page, struct mm_struct mm,	528	extern int __lock_page_or_retry(struct page page, struct mm_struct mm,
502	unsigned int flags);	529	unsigned int flags);
503	extern void unlock_page(struct page *page);	530	extern void unlock_page(struct page *page);
@@ -534,6 +561,22 @@ static inline int lock_page_killable(struct page *page) · 30 unmodified lines
534	return 0;	561	return 0;
535	}	562	}
536		563
		564	/*
		565	* lock_page_async - Lock the page, unless this would block. If the page
		566	* is already locked, then queue a callback when the page becomes unlocked.
		567	* This callback can then retry the operation.
		568	*
		569	* Returns 0 if the page is locked successfully, or -EIOCBQUEUED if the page
		570	* was already locked and the callback defined in 'wait' was queued.
		571	*/
		572	static inline int lock_page_async(struct page *page,
		573	struct wait_page_queue *wait)
		574	{
		575	if (!trylock_page(page))
		576	return __lock_page_async(page, wait);
		577	return 0;
		578	}
		579
537	/*	580	/*
538	* lock_page_or_retry - Lock the page, unless this would block and the	581	* lock_page_or_retry - Lock the page, unless this would block and the
539	* caller indicated that it can handle a retry.	582	* caller indicated that it can handle a retry.

include/linux/sched/task.h+6−0

 		__put_task_struct(t);
 }
+static inline void put_task_struct_many(struct task_struct *t, int nr)
+{
+	if (refcount_sub_and_test(nr, &t->usage))
+		__put_task_struct(t);
+}
 void put_task_struct_rcu_user(struct task_struct *task);
 #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT

include/uapi/linux/io_uring.h+3−1

 	union {
 		__kernel_rwf_t	rw_flags;
 		__u32		fsync_flags;
-		__u16		poll_events;
+		__u16		poll_events;	/* compatibility */
+		__u32		poll32_events;	/* word-reversed for BE */
 		__u32		sync_range_flags;
 		__u32		msg_flags;
 		__u32		timeout_flags;
 #define IORING_FEAT_RW_CUR_POS		(1U << 3)
 #define IORING_FEAT_CUR_PERSONALITY	(1U << 4)
 #define IORING_FEAT_FAST_POLL		(1U << 5)
+#define IORING_FEAT_POLL_32BITS 	(1U << 6)
 /*
  * io_uring_register(2) opcodes and arguments

mm/filemap.c+64−27

 	page_writeback_init();
 }
-/* This has the same layout as wait_bit_key - see fs/cachefiles/rdwr.c */
-struct wait_page_key {
-	struct page *page;
-	int bit_nr;
-	int page_match;
-};
-struct wait_page_queue {
-	struct page *page;
-	int bit_nr;
-	wait_queue_entry_t wait;
-};
 static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *arg)
 {
 	int ret;
 	struct wait_page_queue *wait_page
 		= container_of(wait, struct wait_page_queue, wait);
-	if (wait_page->page != key->page)
+	if (!wake_page_match(wait_page, key))
-	       return 0;
-	key->page_match = 1;
-	if (wait_page->bit_nr != key->bit_nr)
 		return 0;
 	/*
 }
 EXPORT_SYMBOL(wait_on_page_bit_killable);
+static int __wait_on_page_locked_async(struct page *page,
+				       struct wait_page_queue *wait, bool set)
+{
+	struct wait_queue_head *q = page_waitqueue(page);
+	int ret = 0;
+	wait->page = page;
+	wait->bit_nr = PG_locked;
+	spin_lock_irq(&q->lock);
+	__add_wait_queue_entry_tail(q, &wait->wait);
+	SetPageWaiters(page);
+	if (set)
+		ret = !trylock_page(page);
+	else
+		ret = PageLocked(page);
+	/*
+	 * If we were succesful now, we know we're still on the
+	 * waitqueue as we're still under the lock. This means it's
+	 * safe to remove and return success, we know the callback
+	 * isn't going to trigger.
+	 */
+	if (!ret)
+		__remove_wait_queue(q, &wait->wait);
+	else
+		ret = -EIOCBQUEUED;
+	spin_unlock_irq(&q->lock);
+	return ret;
+}
+static int wait_on_page_locked_async(struct page *page,
+				     struct wait_page_queue *wait)
+{
+	if (!PageLocked(page))
+		return 0;
+	return __wait_on_page_locked_async(compound_head(page), wait, false);
+}
 /**
  * put_and_wait_on_page_locked - Drop a reference and wait for it to be unlocked
  * @page: The page to wait for.
 }
 EXPORT_SYMBOL_GPL(__lock_page_killable);
+int __lock_page_async(struct page *page, struct wait_page_queue *wait)
+{
+	return __wait_on_page_locked_async(page, wait, true);
+}
 /*
  * Return values:
  * 1 - page is locked; mmap_lock is still held.
 		page = find_get_page(mapping, index);
 		if (!page) {
-			if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO))
+			if (iocb->ki_flags & IOCB_NOIO)
 				goto would_block;
 			page_cache_sync_readahead(mapping,
 					ra, filp,
 					index, last_index - index);
 		}
 		if (!PageUptodate(page)) {
-			if (iocb->ki_flags & IOCB_NOWAIT) {
-				put_page(page);
-				goto would_block;
-			}
 			/*
 			 * See comment in do_read_cache_page on why
 			 * wait_on_page_locked is used to avoid unnecessarily
 			 * serialisations and why it's safe.
 			 */
-			error = wait_on_page_locked_killable(page);
+			if (iocb->ki_flags & IOCB_WAITQ) {
+				if (written) {
+					put_page(page);
+					goto out;
+				}
+				error = wait_on_page_locked_async(page,
+								iocb->ki_waitq);
+			} else {
+				if (iocb->ki_flags & IOCB_NOWAIT) {
+					put_page(page);
+					goto would_block;
+				}
+				error = wait_on_page_locked_killable(page);
+			}
 			if (unlikely(error))
 				goto readpage_error;
 			if (PageUptodate(page))
 page_not_up_to_date:
 		/* Get exclusive access to the page ... */
-		error = lock_page_killable(page);
+		if (iocb->ki_flags & IOCB_WAITQ)
+			error = lock_page_async(page, iocb->ki_waitq);
+		else
+			error = lock_page_killable(page);
 		if (unlikely(error))
 			goto readpage_error;
 		}
 readpage:
-		if (iocb->ki_flags & IOCB_NOIO) {
+		if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT)) {
 			unlock_page(page);
 			put_page(page);
 			goto would_block;

tools/io_uring/liburing.h+5−1

 #include <string.h>
 #include "../../include/uapi/linux/io_uring.h"
 #include <inttypes.h>
+#include <linux/swab.h>
 #include "barrier.h"
 /*
 }
 static inline void io_uring_prep_poll_add(struct io_uring_sqe *sqe, int fd,
-					  short poll_mask)
+					  unsigned poll_mask)
 {
 	memset(sqe, 0, sizeof(*sqe));
 	sqe->opcode = IORING_OP_POLL_ADD;
 	sqe->fd = fd;
+#if __BYTE_ORDER == __BIG_ENDIAN
+	poll_mask = __swahw32(poll_mask);
+#endif
 	sqe->poll_events = poll_mask;
 }