Merge tag 'io_uring-5.8-2020-06-19' of git://git.kernel.dk/linux-block

Pull io_uring fixes from Jens Axboe: - Catch a case where io_sq_thread() didn't do proper mm acquire - Ensure poll completions are reaped on shutdown - Async cancelation and run fixes (Pavel) - io-poll race fixes (Xiaoguang) - Request cleanup race fix (Xiaoguang) * tag 'io_uring-5.8-2020-06-19' of git://git.kernel.dk/linux-block: io_uring: fix possible race condition against REQ_F_NEED_CLEANUP io_uring: reap poll completions while waiting for refs to drop on exit io_uring: acquire 'mm' for task_work for SQPOLL io_uring: add memory barrier to synchronize io_kiocb's result and iopoll_completed io_uring: don't fail links for EAGAIN error in IOPOLL mode io_uring: cancel by ->task not pid io_uring: lazy get task io_uring: batch cancel in io_uring_cancel_files() io_uring: cancel all task's requests on exit io-wq: add an option to cancel all matched reqs io-wq: reorder cancellation pending -> running io_uring: fix lazy work init

Linus Torvalds committed 6 years ago

Commit 4333a9b2 parents d2b1c81 6f2cc163 files · +177 −112Browse files

Unified · Split

3 files

fs/io-wq.c +56 −52
fs/io-wq.h +1 −3
fs/io_uring.c +120 −57

fs/io-wq.c+56−52

 struct io_cb_cancel_data {
 	work_cancel_fn *fn;
 	void *data;
+	int nr_running;
+	int nr_pending;
+	bool cancel_all;
 };
 static bool io_wq_worker_cancel(struct io_worker *worker, void *data)
 {
 	struct io_cb_cancel_data *match = data;
 	unsigned long flags;
-	bool ret = false;
 	/*
 	 * Hold the lock to avoid ->cur_work going out of scope, caller
 	    !(worker->cur_work->flags & IO_WQ_WORK_NO_CANCEL) &&
 	    match->fn(worker->cur_work, match->data)) {
 		send_sig(SIGINT, worker->task, 1);
-		ret = true;
+		match->nr_running++;
 	}
 	spin_unlock_irqrestore(&worker->lock, flags);
-	return ret;
+	return match->nr_running && !match->cancel_all;
 }
-static enum io_wq_cancel io_wqe_cancel_work(struct io_wqe *wqe,
+static void io_wqe_cancel_pending_work(struct io_wqe *wqe,
-					    struct io_cb_cancel_data *match)
+				       struct io_cb_cancel_data *match)
 {
 	struct io_wq_work_node *node, *prev;
 	struct io_wq_work *work;
 	unsigned long flags;
-	bool found = false;
-	/*
+retry:
-	 * First check pending list, if we're lucky we can just remove it
-	 * from there. CANCEL_OK means that the work is returned as-new,
-	 * no completion will be posted for it.
-	 */
 	spin_lock_irqsave(&wqe->lock, flags);
 	wq_list_for_each(node, prev, &wqe->work_list) {
 		work = container_of(node, struct io_wq_work, list);
+		if (!match->fn(work, match->data))
+			continue;
-		if (match->fn(work, match->data)) {
+		wq_list_del(&wqe->work_list, node, prev);
-			wq_list_del(&wqe->work_list, node, prev);
+		spin_unlock_irqrestore(&wqe->lock, flags);
-			found = true;
-			break;
-		}
-	}
-	spin_unlock_irqrestore(&wqe->lock, flags);
-	if (found) {
 		io_run_cancel(work, wqe);
-		return IO_WQ_CANCEL_OK;
+		match->nr_pending++;
+		if (!match->cancel_all)
+			return;
+		/* not safe to continue after unlock */
+		goto retry;
 	}
+	spin_unlock_irqrestore(&wqe->lock, flags);
+}
-	/*
+static void io_wqe_cancel_running_work(struct io_wqe *wqe,
-	 * Now check if a free (going busy) or busy worker has the work
+				       struct io_cb_cancel_data *match)
-	 * currently running. If we find it there, we'll return CANCEL_RUNNING
+{
-	 * as an indication that we attempt to signal cancellation. The
-	 * completion will run normally in this case.
-	 */
 	rcu_read_lock();
-	found = io_wq_for_each_worker(wqe, io_wq_worker_cancel, match);
+	io_wq_for_each_worker(wqe, io_wq_worker_cancel, match);
 	rcu_read_unlock();
-	return found ? IO_WQ_CANCEL_RUNNING : IO_WQ_CANCEL_NOTFOUND;
 }
 enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
-				  void *data)
+				  void *data, bool cancel_all)
 {
 	struct io_cb_cancel_data match = {
-		.fn	= cancel,
+		.fn		= cancel,
-		.data	= data,
+		.data		= data,
+		.cancel_all	= cancel_all,
 	};
-	enum io_wq_cancel ret = IO_WQ_CANCEL_NOTFOUND;
 	int node;
+	/*
+	 * First check pending list, if we're lucky we can just remove it
+	 * from there. CANCEL_OK means that the work is returned as-new,
+	 * no completion will be posted for it.
+	 */
 	for_each_node(node) {
 		struct io_wqe *wqe = wq->wqes[node];
-		ret = io_wqe_cancel_work(wqe, &match);
+		io_wqe_cancel_pending_work(wqe, &match);
-		if (ret != IO_WQ_CANCEL_NOTFOUND)
+		if (match.nr_pending && !match.cancel_all)
-			break;
+			return IO_WQ_CANCEL_OK;
 	}
-	return ret;
+	/*
+	 * Now check if a free (going busy) or busy worker has the work
+	 * currently running. If we find it there, we'll return CANCEL_RUNNING
+	 * as an indication that we attempt to signal cancellation. The
+	 * completion will run normally in this case.
+	 */
+	for_each_node(node) {
+		struct io_wqe *wqe = wq->wqes[node];
+		io_wqe_cancel_running_work(wqe, &match);
+		if (match.nr_running && !match.cancel_all)
+			return IO_WQ_CANCEL_RUNNING;
+	}
+	if (match.nr_running)
+		return IO_WQ_CANCEL_RUNNING;
+	if (match.nr_pending)
+		return IO_WQ_CANCEL_OK;
+	return IO_WQ_CANCEL_NOTFOUND;
 }
 static bool io_wq_io_cb_cancel_data(struct io_wq_work *work, void *data)
 enum io_wq_cancel io_wq_cancel_work(struct io_wq *wq, struct io_wq_work *cwork)
 {
-	return io_wq_cancel_cb(wq, io_wq_io_cb_cancel_data, (void *)cwork);
+	return io_wq_cancel_cb(wq, io_wq_io_cb_cancel_data, (void *)cwork, false);
-}
-static bool io_wq_pid_match(struct io_wq_work *work, void *data)
-{
-	pid_t pid = (pid_t) (unsigned long) data;
-	return work->task_pid == pid;
-}
-enum io_wq_cancel io_wq_cancel_pid(struct io_wq *wq, pid_t pid)
-{
-	void *data = (void *) (unsigned long) pid;
-	return io_wq_cancel_cb(wq, io_wq_pid_match, data);
 }
 struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)

fs/io-wq.h+1−3

 	const struct cred *creds;
 	struct fs_struct *fs;
 	unsigned flags;
-	pid_t task_pid;
 };
 static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)
 void io_wq_cancel_all(struct io_wq *wq);
 enum io_wq_cancel io_wq_cancel_work(struct io_wq *wq, struct io_wq_work *cwork);
-enum io_wq_cancel io_wq_cancel_pid(struct io_wq *wq, pid_t pid);
 typedef bool (work_cancel_fn)(struct io_wq_work *, void *);
 enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
-					void *data);
+					void *data, bool cancel_all);
 struct task_struct *io_wq_get_task(struct io_wq *wq);

fs/io_uring.c+120−57

@@ -541,6 +541,7 @@ enum { · 540 unmodified lines
541	REQ_F_NO_FILE_TABLE_BIT,	541	REQ_F_NO_FILE_TABLE_BIT,
542	REQ_F_QUEUE_TIMEOUT_BIT,	542	REQ_F_QUEUE_TIMEOUT_BIT,
543	REQ_F_WORK_INITIALIZED_BIT,	543	REQ_F_WORK_INITIALIZED_BIT,
		544	REQ_F_TASK_PINNED_BIT,
544		545
545	/* not a real bit, just to check we're not overflowing the space */	546	/* not a real bit, just to check we're not overflowing the space */
546	__REQ_F_LAST_BIT,	547	__REQ_F_LAST_BIT,
@@ -598,6 +599,8 @@ enum { · 51 unmodified lines
598	REQ_F_QUEUE_TIMEOUT = BIT(REQ_F_QUEUE_TIMEOUT_BIT),	599	REQ_F_QUEUE_TIMEOUT = BIT(REQ_F_QUEUE_TIMEOUT_BIT),
599	/* io_wq_work is initialized */	600	/* io_wq_work is initialized */
600	REQ_F_WORK_INITIALIZED = BIT(REQ_F_WORK_INITIALIZED_BIT),	601	REQ_F_WORK_INITIALIZED = BIT(REQ_F_WORK_INITIALIZED_BIT),
		602	/* req->task is refcounted */
		603	REQ_F_TASK_PINNED = BIT(REQ_F_TASK_PINNED_BIT),
601	};	604	};
602		605
603	struct async_poll {	606	struct async_poll {
@@ -910,6 +913,21 @@ struct sock io_uring_get_socket(struct file file) · 306 unmodified lines
910	}	913	}
911	EXPORT_SYMBOL(io_uring_get_socket);	914	EXPORT_SYMBOL(io_uring_get_socket);
912		915
		916	static void io_get_req_task(struct io_kiocb *req)
		917	{
		918	if (req->flags & REQ_F_TASK_PINNED)
		919	return;
		920	get_task_struct(req->task);
		921	req->flags \|= REQ_F_TASK_PINNED;
		922	}
		923
		924	/* not idempotent -- it doesn't clear REQ_F_TASK_PINNED */
		925	static void __io_put_req_task(struct io_kiocb *req)
		926	{
		927	if (req->flags & REQ_F_TASK_PINNED)
		928	put_task_struct(req->task);
		929	}
		930
913	static void io_file_put_work(struct work_struct *work);	931	static void io_file_put_work(struct work_struct *work);
914		932
915	/*	933	/*
@@ -1045,8 +1063,6 @@ static inline void io_req_work_grab_env(struct io_kiocb *req, · 129 unmodified lines
1045	}	1063	}
1046	spin_unlock(&current->fs->lock);	1064	spin_unlock(&current->fs->lock);
1047	}	1065	}
1048	if (!req->work.task_pid)
1049	req->work.task_pid = task_pid_vnr(current);
1050	}	1066	}
1051		1067
1052	static inline void io_req_work_drop_env(struct io_kiocb *req)	1068	static inline void io_req_work_drop_env(struct io_kiocb *req)
@@ -1087,6 +1103,7 @@ static inline void io_prep_async_work(struct io_kiocb *req, · 34 unmodified lines
1087	req->work.flags \|= IO_WQ_WORK_UNBOUND;	1103	req->work.flags \|= IO_WQ_WORK_UNBOUND;
1088	}	1104	}
1089		1105
		1106	io_req_init_async(req);
1090	io_req_work_grab_env(req, def);	1107	io_req_work_grab_env(req, def);
1091		1108
1092	*link = io_prep_linked_timeout(req);	1109	*link = io_prep_linked_timeout(req);
@@ -1398,9 +1415,7 @@ static void __io_req_aux_free(struct io_kiocb *req) · 305 unmodified lines
1398	kfree(req->io);	1415	kfree(req->io);
1399	if (req->file)	1416	if (req->file)
1400	io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE));	1417	io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE));
1401	if (req->task)	1418	__io_put_req_task(req);
1402	put_task_struct(req->task);
1403
1404	io_req_work_drop_env(req);	1419	io_req_work_drop_env(req);
1405	}	1420	}
1406		1421
@@ -1727,6 +1742,18 @@ static int io_put_kbuf(struct io_kiocb *req) · 320 unmodified lines
1727	return cflags;	1742	return cflags;
1728	}	1743	}
1729		1744
		1745	static void io_iopoll_queue(struct list_head *again)
		1746	{
		1747	struct io_kiocb *req;
		1748
		1749	do {
		1750	req = list_first_entry(again, struct io_kiocb, list);
		1751	list_del(&req->list);
		1752	refcount_inc(&req->refs);
		1753	io_queue_async_work(req);
		1754	} while (!list_empty(again));
		1755	}
		1756
1730	/*	1757	/*
1731	* Find and free completed poll iocbs	1758	* Find and free completed poll iocbs
1732	*/	1759	*/
@@ -1735,12 +1762,21 @@ static void io_iopoll_complete(struct io_ring_ctx ctx, unsigned int nr_events, · 2 unmodified lines
1735	{	1762	{
1736	struct req_batch rb;	1763	struct req_batch rb;
1737	struct io_kiocb *req;	1764	struct io_kiocb *req;
		1765	LIST_HEAD(again);
		1766
		1767	/* order with ->result store in io_complete_rw_iopoll() */
		1768	smp_rmb();
1738		1769
1739	rb.to_free = rb.need_iter = 0;	1770	rb.to_free = rb.need_iter = 0;
1740	while (!list_empty(done)) {	1771	while (!list_empty(done)) {
1741	int cflags = 0;	1772	int cflags = 0;
1742		1773
1743	req = list_first_entry(done, struct io_kiocb, list);	1774	req = list_first_entry(done, struct io_kiocb, list);
		1775	if (READ_ONCE(req->result) == -EAGAIN) {
		1776	req->iopoll_completed = 0;
		1777	list_move_tail(&req->list, &again);
		1778	continue;
		1779	}
1744	list_del(&req->list);	1780	list_del(&req->list);
1745		1781
1746	if (req->flags & REQ_F_BUFFER_SELECTED)	1782	if (req->flags & REQ_F_BUFFER_SELECTED)
@@ -1758,18 +1794,9 @@ static void io_iopoll_complete(struct io_ring_ctx ctx, unsigned int nr_events, · 11 unmodified lines
1758	if (ctx->flags & IORING_SETUP_SQPOLL)	1794	if (ctx->flags & IORING_SETUP_SQPOLL)
1759	io_cqring_ev_posted(ctx);	1795	io_cqring_ev_posted(ctx);
1760	io_free_req_many(ctx, &rb);	1796	io_free_req_many(ctx, &rb);
1761	}
1762		1797
1763	static void io_iopoll_queue(struct list_head *again)	1798	if (!list_empty(&again))
1764	{	1799	io_iopoll_queue(&again);
1765	struct io_kiocb *req;
1766
1767	do {
1768	req = list_first_entry(again, struct io_kiocb, list);
1769	list_del(&req->list);
1770	refcount_inc(&req->refs);
1771	io_queue_async_work(req);
1772	} while (!list_empty(again));
1773	}	1800	}
1774		1801
1775	static int io_do_iopoll(struct io_ring_ctx ctx, unsigned int nr_events,	1802	static int io_do_iopoll(struct io_ring_ctx ctx, unsigned int nr_events,
@@ -1777,7 +1804,6 @@ static int io_do_iopoll(struct io_ring_ctx ctx, unsigned int nr_events, · 1 unmodified line
1777	{	1804	{
1778	struct io_kiocb req, tmp;	1805	struct io_kiocb req, tmp;
1779	LIST_HEAD(done);	1806	LIST_HEAD(done);
1780	LIST_HEAD(again);
1781	bool spin;	1807	bool spin;
1782	int ret;	1808	int ret;
1783		1809
@@ -1803,13 +1829,6 @@ static int io_do_iopoll(struct io_ring_ctx ctx, unsigned int nr_events, · 19 unmodified lines
1803	if (!list_empty(&done))	1829	if (!list_empty(&done))
1804	break;	1830	break;
1805		1831
1806	if (req->result == -EAGAIN) {
1807	list_move_tail(&req->list, &again);
1808	continue;
1809	}
1810	if (!list_empty(&again))
1811	break;
1812
1813	ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);	1832	ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
1814	if (ret < 0)	1833	if (ret < 0)
1815	break;	1834	break;
@@ -1822,9 +1841,6 @@ static int io_do_iopoll(struct io_ring_ctx ctx, unsigned int nr_events, · 6 unmodified lines
1822	if (!list_empty(&done))	1841	if (!list_empty(&done))
1823	io_iopoll_complete(ctx, nr_events, &done);	1842	io_iopoll_complete(ctx, nr_events, &done);
1824		1843
1825	if (!list_empty(&again))
1826	io_iopoll_queue(&again);
1827
1828	return ret;	1844	return ret;
1829	}	1845	}
1830		1846
@@ -1973,11 +1989,15 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) · 142 unmodified lines
1973	if (kiocb->ki_flags & IOCB_WRITE)	1989	if (kiocb->ki_flags & IOCB_WRITE)
1974	kiocb_end_write(req);	1990	kiocb_end_write(req);
1975		1991
1976	if (res != req->result)	1992	if (res != -EAGAIN && res != req->result)
1977	req_set_fail_links(req);	1993	req_set_fail_links(req);
1978	req->result = res;	1994
1979	if (res != -EAGAIN)	1995	WRITE_ONCE(req->result, res);
		1996	/* order with io_poll_complete() checking ->result */
		1997	if (res != -EAGAIN) {
		1998	smp_wmb();
1980	WRITE_ONCE(req->iopoll_completed, 1);	1999	WRITE_ONCE(req->iopoll_completed, 1);
		2000	}
1981	}	2001	}
1982		2002
1983	/*	2003	/*
@@ -2650,8 +2670,8 @@ copy_iov: · 666 unmodified lines
2650	}	2670	}
2651	}	2671	}
2652	out_free:	2672	out_free:
2653	kfree(iovec);	2673	if (!(req->flags & REQ_F_NEED_CLEANUP))
2654	req->flags &= ~REQ_F_NEED_CLEANUP;	2674	kfree(iovec);
2655	return ret;	2675	return ret;
2656	}	2676	}
2657		2677
@@ -2773,8 +2793,8 @@ copy_iov: · 115 unmodified lines
2773	}	2793	}
2774	}	2794	}
2775	out_free:	2795	out_free:
2776	req->flags &= ~REQ_F_NEED_CLEANUP;	2796	if (!(req->flags & REQ_F_NEED_CLEANUP))
2777	kfree(iovec);	2797	kfree(iovec);
2778	return ret;	2798	return ret;
2779	}	2799	}
2780		2800
@@ -4236,6 +4256,28 @@ static void io_async_queue_proc(struct file file, struct wait_queue_head head, · 1455 unmodified lines
4236	__io_queue_proc(&pt->req->apoll->poll, pt, head);	4256	__io_queue_proc(&pt->req->apoll->poll, pt, head);
4237	}	4257	}
4238		4258
		4259	static void io_sq_thread_drop_mm(struct io_ring_ctx *ctx)
		4260	{
		4261	struct mm_struct *mm = current->mm;
		4262
		4263	if (mm) {
		4264	kthread_unuse_mm(mm);
		4265	mmput(mm);
		4266	}
		4267	}
		4268
		4269	static int io_sq_thread_acquire_mm(struct io_ring_ctx *ctx,
		4270	struct io_kiocb *req)
		4271	{
		4272	if (io_op_defs[req->opcode].needs_mm && !current->mm) {
		4273	if (unlikely(!mmget_not_zero(ctx->sqo_mm)))
		4274	return -EFAULT;
		4275	kthread_use_mm(ctx->sqo_mm);
		4276	}
		4277
		4278	return 0;
		4279	}
		4280
4239	static void io_async_task_func(struct callback_head *cb)	4281	static void io_async_task_func(struct callback_head *cb)
4240	{	4282	{
4241	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);	4283	struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
@@ -4270,11 +4312,16 @@ static void io_async_task_func(struct callback_head *cb) · 28 unmodified lines
4270		4312
4271	if (!canceled) {	4313	if (!canceled) {
4272	__set_current_state(TASK_RUNNING);	4314	__set_current_state(TASK_RUNNING);
		4315	if (io_sq_thread_acquire_mm(ctx, req)) {
		4316	io_cqring_add_event(req, -EFAULT);
		4317	goto end_req;
		4318	}
4273	mutex_lock(&ctx->uring_lock);	4319	mutex_lock(&ctx->uring_lock);
4274	__io_queue_sqe(req, NULL);	4320	__io_queue_sqe(req, NULL);
4275	mutex_unlock(&ctx->uring_lock);	4321	mutex_unlock(&ctx->uring_lock);
4276	} else {	4322	} else {
4277	io_cqring_ev_posted(ctx);	4323	io_cqring_ev_posted(ctx);
		4324	end_req:
4278	req_set_fail_links(req);	4325	req_set_fail_links(req);
4279	io_double_put_req(req);	4326	io_double_put_req(req);
4280	}	4327	}
@@ -4366,8 +4413,7 @@ static bool io_arm_poll_handler(struct io_kiocb *req) · 85 unmodified lines
4366	memcpy(&apoll->work, &req->work, sizeof(req->work));	4413	memcpy(&apoll->work, &req->work, sizeof(req->work));
4367	had_io = req->io != NULL;	4414	had_io = req->io != NULL;
4368		4415
4369	get_task_struct(current);	4416	io_get_req_task(req);
4370	req->task = current;
4371	req->apoll = apoll;	4417	req->apoll = apoll;
4372	INIT_HLIST_NODE(&req->hash_node);	4418	INIT_HLIST_NODE(&req->hash_node);
4373		4419
@@ -4555,8 +4601,7 @@ static int io_poll_add_prep(struct io_kiocb req, const struct io_uring_sqe sqe · 181 unmodified lines
4555	events = READ_ONCE(sqe->poll_events);	4601	events = READ_ONCE(sqe->poll_events);
4556	poll->events = demangle_poll(events) \| EPOLLERR \| EPOLLHUP;	4602	poll->events = demangle_poll(events) \| EPOLLERR \| EPOLLHUP;
4557		4603
4558	get_task_struct(current);	4604	io_get_req_task(req);
4559	req->task = current;
4560	return 0;	4605	return 0;
4561	}	4606	}
4562		4607
@@ -4772,7 +4817,7 @@ static int io_async_cancel_one(struct io_ring_ctx ctx, void sqe_addr) · 209 unmodified lines
4772	enum io_wq_cancel cancel_ret;	4817	enum io_wq_cancel cancel_ret;
4773	int ret = 0;	4818	int ret = 0;
4774		4819
4775	cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr);	4820	cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr, false);
4776	switch (cancel_ret) {	4821	switch (cancel_ret) {
4777	case IO_WQ_CANCEL_OK:	4822	case IO_WQ_CANCEL_OK:
4778	ret = 0;	4823	ret = 0;
@@ -5817,17 +5862,14 @@ static int io_init_req(struct io_ring_ctx ctx, struct io_kiocb req, · 1038 unmodified lines
5817	req->flags = 0;	5862	req->flags = 0;
5818	/* one is dropped after submission, the other at completion */	5863	/* one is dropped after submission, the other at completion */
5819	refcount_set(&req->refs, 2);	5864	refcount_set(&req->refs, 2);
5820	req->task = NULL;	5865	req->task = current;
5821	req->result = 0;	5866	req->result = 0;
5822		5867
5823	if (unlikely(req->opcode >= IORING_OP_LAST))	5868	if (unlikely(req->opcode >= IORING_OP_LAST))
5824	return -EINVAL;	5869	return -EINVAL;
5825		5870
5826	if (io_op_defs[req->opcode].needs_mm && !current->mm) {	5871	if (unlikely(io_sq_thread_acquire_mm(ctx, req)))
5827	if (unlikely(!mmget_not_zero(ctx->sqo_mm)))	5872	return -EFAULT;
5828	return -EFAULT;
5829	kthread_use_mm(ctx->sqo_mm);
5830	}
5831		5873
5832	sqe_flags = READ_ONCE(sqe->flags);	5874	sqe_flags = READ_ONCE(sqe->flags);
5833	/* enforce forwards compatibility on users */	5875	/* enforce forwards compatibility on users */
@@ -5936,16 +5978,6 @@ fail_req: · 102 unmodified lines
5936	return submitted;	5978	return submitted;
5937	}	5979	}
5938		5980
5939	static inline void io_sq_thread_drop_mm(struct io_ring_ctx *ctx)
5940	{
5941	struct mm_struct *mm = current->mm;
5942
5943	if (mm) {
5944	kthread_unuse_mm(mm);
5945	mmput(mm);
5946	}
5947	}
5948
5949	static int io_sq_thread(void *data)	5981	static int io_sq_thread(void *data)
5950	{	5982	{
5951	struct io_ring_ctx *ctx = data;	5983	struct io_ring_ctx *ctx = data;
@@ -7331,7 +7363,17 @@ static void io_ring_exit_work(struct work_struct *work) · 1379 unmodified lines
7331	if (ctx->rings)	7363	if (ctx->rings)
7332	io_cqring_overflow_flush(ctx, true);	7364	io_cqring_overflow_flush(ctx, true);
7333		7365
7334	wait_for_completion(&ctx->ref_comp);	7366	/*
		7367	* If we're doing polled IO and end up having requests being
		7368	* submitted async (out-of-line), then completions can come in while
		7369	* we're waiting for refs to drop. We need to reap these manually,
		7370	* as nobody else will be looking for them.
		7371	*/
		7372	while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20)) {
		7373	io_iopoll_reap_events(ctx);
		7374	if (ctx->rings)
		7375	io_cqring_overflow_flush(ctx, true);
		7376	}
7335	io_ring_ctx_free(ctx);	7377	io_ring_ctx_free(ctx);
7336	}	7378	}
7337		7379
@@ -7365,9 +7407,22 @@ static int io_uring_release(struct inode inode, struct file file) · 27 unmodified lines
7365	return 0;	7407	return 0;
7366	}	7408	}
7367		7409
		7410	static bool io_wq_files_match(struct io_wq_work work, void data)
		7411	{
		7412	struct files_struct *files = data;
		7413
		7414	return work->files == files;
		7415	}
		7416
7368	static void io_uring_cancel_files(struct io_ring_ctx *ctx,	7417	static void io_uring_cancel_files(struct io_ring_ctx *ctx,
7369	struct files_struct *files)	7418	struct files_struct *files)
7370	{	7419	{
		7420	if (list_empty_careful(&ctx->inflight_list))
		7421	return;
		7422
		7423	/* cancel all at once, should be faster than doing it one by one*/
		7424	io_wq_cancel_cb(ctx->io_wq, io_wq_files_match, files, true);
		7425
7371	while (!list_empty_careful(&ctx->inflight_list)) {	7426	while (!list_empty_careful(&ctx->inflight_list)) {
7372	struct io_kiocb cancel_req = NULL, req;	7427	struct io_kiocb cancel_req = NULL, req;
7373	DEFINE_WAIT(wait);	7428	DEFINE_WAIT(wait);
@@ -7423,6 +7478,14 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx, · 49 unmodified lines
7423	}	7478	}
7424	}	7479	}
7425		7480
		7481	static bool io_cancel_task_cb(struct io_wq_work work, void data)
		7482	{
		7483	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
		7484	struct task_struct *task = data;
		7485
		7486	return req->task == task;
		7487	}
		7488
7426	static int io_uring_flush(struct file file, void data)	7489	static int io_uring_flush(struct file file, void data)
7427	{	7490	{
7428	struct io_ring_ctx *ctx = file->private_data;	7491	struct io_ring_ctx *ctx = file->private_data;
@@ -7433,7 +7496,7 @@ static int io_uring_flush(struct file file, void data) · 4 unmodified lines
7433	* If the task is going away, cancel work it may have pending	7496	* If the task is going away, cancel work it may have pending
7434	*/	7497	*/
7435	if (fatal_signal_pending(current) \|\| (current->flags & PF_EXITING))	7498	if (fatal_signal_pending(current) \|\| (current->flags & PF_EXITING))
7436	io_wq_cancel_pid(ctx->io_wq, task_pid_vnr(current));	7499	io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, current, true);
7437		7500
7438	return 0;	7501	return 0;
7439	}	7502	}