History

Merge tag 'io_uring-5.8-2020-06-19' of git://git.kernel.dk/linux-block

Pull io_uring fixes from Jens Axboe: - Catch a case where io_sq_thread() didn't do proper mm acquire - Ensure poll completions are reaped on shutdown - Async cancelation and run fixes (Pavel) - io-poll race fixes (Xiaoguang) - Request cleanup race fix (Xiaoguang) * tag 'io_uring-5.8-2020-06-19' of git://git.kernel.dk/linux-block: io_uring: fix possible race condition against REQ_F_NEED_CLEANUP io_uring: reap poll completions while waiting for refs to drop on exit io_uring: acquire 'mm' for task_work for SQPOLL io_uring: add memory barrier to synchronize io_kiocb's result and iopoll_completed io_uring: don't fail links for EAGAIN error in IOPOLL mode io_uring: cancel by ->task not pid io_uring: lazy get task io_uring: batch cancel in io_uring_cancel_files() io_uring: cancel all task's requests on exit io-wq: add an option to cancel all matched reqs io-wq: reorder cancellation pending -> running io_uring: fix lazy work init

Linus Torvalds committed 6 years ago

Commit 4333a9b2 parents d2b1c81 6f2cc163 files · +177 −112Browse files

Unified · Split

3 files
  1. fs/io-wq.c +56 −52
  2. fs/io-wq.h +1 −3
  3. fs/io_uring.c +120 −57
fs/io-wq.c+56−52
@@ -903,13 +903,15 @@ void io_wq_cancel_all(struct io_wq *wq) · 902 unmodified lines
903903 struct io_cb_cancel_data {
904904 work_cancel_fn *fn;
905905 void *data;
906+ int nr_running;
907+ int nr_pending;
908+ bool cancel_all;
906909 };
907910
908911 static bool io_wq_worker_cancel(struct io_worker *worker, void *data)
909912 {
910913 struct io_cb_cancel_data *match = data;
911914 unsigned long flags;
912− bool ret = false;
913915
914916 /*
915917 * Hold the lock to avoid ->cur_work going out of scope, caller
@@ -920,74 +922,90 @@ static bool io_wq_worker_cancel(struct io_worker *worker, void *data) · 4 unmodified lines
920922 !(worker->cur_work->flags & IO_WQ_WORK_NO_CANCEL) &&
921923 match->fn(worker->cur_work, match->data)) {
922924 send_sig(SIGINT, worker->task, 1);
923− ret = true;
925+ match->nr_running++;
924926 }
925927 spin_unlock_irqrestore(&worker->lock, flags);
926928
927− return ret;
929+ return match->nr_running && !match->cancel_all;
928930 }
929931
930−static enum io_wq_cancel io_wqe_cancel_work(struct io_wqe *wqe,
931− struct io_cb_cancel_data *match)
932+static void io_wqe_cancel_pending_work(struct io_wqe *wqe,
933+ struct io_cb_cancel_data *match)
932934 {
933935 struct io_wq_work_node *node, *prev;
934936 struct io_wq_work *work;
935937 unsigned long flags;
936− bool found = false;
937938
938− /*
939− * First check pending list, if we're lucky we can just remove it
940− * from there. CANCEL_OK means that the work is returned as-new,
941− * no completion will be posted for it.
942− */
939+retry:
943940 spin_lock_irqsave(&wqe->lock, flags);
944941 wq_list_for_each(node, prev, &wqe->work_list) {
945942 work = container_of(node, struct io_wq_work, list);
943+ if (!match->fn(work, match->data))
944+ continue;
946945
947− if (match->fn(work, match->data)) {
948− wq_list_del(&wqe->work_list, node, prev);
949− found = true;
950− break;
951− }
952− }
953− spin_unlock_irqrestore(&wqe->lock, flags);
954−
955− if (found) {
946+ wq_list_del(&wqe->work_list, node, prev);
947+ spin_unlock_irqrestore(&wqe->lock, flags);
956948 io_run_cancel(work, wqe);
957− return IO_WQ_CANCEL_OK;
949+ match->nr_pending++;
950+ if (!match->cancel_all)
951+ return;
952+
953+ /* not safe to continue after unlock */
954+ goto retry;
958955 }
956+ spin_unlock_irqrestore(&wqe->lock, flags);
957+}
959958
960− /*
961− * Now check if a free (going busy) or busy worker has the work
962− * currently running. If we find it there, we'll return CANCEL_RUNNING
963− * as an indication that we attempt to signal cancellation. The
964− * completion will run normally in this case.
965− */
959+static void io_wqe_cancel_running_work(struct io_wqe *wqe,
960+ struct io_cb_cancel_data *match)
961+{
966962 rcu_read_lock();
967− found = io_wq_for_each_worker(wqe, io_wq_worker_cancel, match);
963+ io_wq_for_each_worker(wqe, io_wq_worker_cancel, match);
968964 rcu_read_unlock();
969− return found ? IO_WQ_CANCEL_RUNNING : IO_WQ_CANCEL_NOTFOUND;
970965 }
971966
972967 enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
973− void *data)
968+ void *data, bool cancel_all)
974969 {
975970 struct io_cb_cancel_data match = {
976− .fn = cancel,
977− .data = data,
971+ .fn = cancel,
972+ .data = data,
973+ .cancel_all = cancel_all,
978974 };
979− enum io_wq_cancel ret = IO_WQ_CANCEL_NOTFOUND;
980975 int node;
981976
977+ /*
978+ * First check pending list, if we're lucky we can just remove it
979+ * from there. CANCEL_OK means that the work is returned as-new,
980+ * no completion will be posted for it.
981+ */
982982 for_each_node(node) {
983983 struct io_wqe *wqe = wq->wqes[node];
984984
985− ret = io_wqe_cancel_work(wqe, &match);
986− if (ret != IO_WQ_CANCEL_NOTFOUND)
987− break;
985+ io_wqe_cancel_pending_work(wqe, &match);
986+ if (match.nr_pending && !match.cancel_all)
987+ return IO_WQ_CANCEL_OK;
988988 }
989989
990− return ret;
990+ /*
991+ * Now check if a free (going busy) or busy worker has the work
992+ * currently running. If we find it there, we'll return CANCEL_RUNNING
993+ * as an indication that we attempt to signal cancellation. The
994+ * completion will run normally in this case.
995+ */
996+ for_each_node(node) {
997+ struct io_wqe *wqe = wq->wqes[node];
998+
999+ io_wqe_cancel_running_work(wqe, &match);
1000+ if (match.nr_running && !match.cancel_all)
1001+ return IO_WQ_CANCEL_RUNNING;
1002+ }
1003+
1004+ if (match.nr_running)
1005+ return IO_WQ_CANCEL_RUNNING;
1006+ if (match.nr_pending)
1007+ return IO_WQ_CANCEL_OK;
1008+ return IO_WQ_CANCEL_NOTFOUND;
9911009 }
9921010
9931011 static bool io_wq_io_cb_cancel_data(struct io_wq_work *work, void *data)
@@ -997,21 +1015,7 @@ static bool io_wq_io_cb_cancel_data(struct io_wq_work *work, void *data) · 3 unmodified lines
9971015
9981016 enum io_wq_cancel io_wq_cancel_work(struct io_wq *wq, struct io_wq_work *cwork)
9991017 {
1000− return io_wq_cancel_cb(wq, io_wq_io_cb_cancel_data, (void *)cwork);
1001−}
1002−
1003−static bool io_wq_pid_match(struct io_wq_work *work, void *data)
1004−{
1005− pid_t pid = (pid_t) (unsigned long) data;
1006−
1007− return work->task_pid == pid;
1008−}
1009−
1010−enum io_wq_cancel io_wq_cancel_pid(struct io_wq *wq, pid_t pid)
1011−{
1012− void *data = (void *) (unsigned long) pid;
1013−
1014− return io_wq_cancel_cb(wq, io_wq_pid_match, data);
1018+ return io_wq_cancel_cb(wq, io_wq_io_cb_cancel_data, (void *)cwork, false);
10151019 }
10161020
10171021 struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)
fs/io-wq.h+1−3
@@ -90,7 +90,6 @@ struct io_wq_work { · 89 unmodified lines
9090 const struct cred *creds;
9191 struct fs_struct *fs;
9292 unsigned flags;
93− pid_t task_pid;
9493 };
9594
9695 static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)
@@ -125,12 +124,11 @@ static inline bool io_wq_is_hashed(struct io_wq_work *work) · 28 unmodified lines
125124
126125 void io_wq_cancel_all(struct io_wq *wq);
127126 enum io_wq_cancel io_wq_cancel_work(struct io_wq *wq, struct io_wq_work *cwork);
128−enum io_wq_cancel io_wq_cancel_pid(struct io_wq *wq, pid_t pid);
129127
130128 typedef bool (work_cancel_fn)(struct io_wq_work *, void *);
131129
132130 enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
133− void *data);
131+ void *data, bool cancel_all);
134132
135133 struct task_struct *io_wq_get_task(struct io_wq *wq);
136134
fs/io_uring.c+120−57
@@ -541,6 +541,7 @@ enum { · 540 unmodified lines
541541 REQ_F_NO_FILE_TABLE_BIT,
542542 REQ_F_QUEUE_TIMEOUT_BIT,
543543 REQ_F_WORK_INITIALIZED_BIT,
544+ REQ_F_TASK_PINNED_BIT,
544545
545546 /* not a real bit, just to check we're not overflowing the space */
546547 __REQ_F_LAST_BIT,
@@ -598,6 +599,8 @@ enum { · 51 unmodified lines
598599 REQ_F_QUEUE_TIMEOUT = BIT(REQ_F_QUEUE_TIMEOUT_BIT),
599600 /* io_wq_work is initialized */
600601 REQ_F_WORK_INITIALIZED = BIT(REQ_F_WORK_INITIALIZED_BIT),
602+ /* req->task is refcounted */
603+ REQ_F_TASK_PINNED = BIT(REQ_F_TASK_PINNED_BIT),
601604 };
602605
603606 struct async_poll {
@@ -910,6 +913,21 @@ struct sock *io_uring_get_socket(struct file *file) · 306 unmodified lines
910913 }
911914 EXPORT_SYMBOL(io_uring_get_socket);
912915
916+static void io_get_req_task(struct io_kiocb *req)
917+{
918+ if (req->flags & REQ_F_TASK_PINNED)
919+ return;
920+ get_task_struct(req->task);
921+ req->flags |= REQ_F_TASK_PINNED;
922+}
923+
924+/* not idempotent -- it doesn't clear REQ_F_TASK_PINNED */
925+static void __io_put_req_task(struct io_kiocb *req)
926+{
927+ if (req->flags & REQ_F_TASK_PINNED)
928+ put_task_struct(req->task);
929+}
930+
913931 static void io_file_put_work(struct work_struct *work);
914932
915933 /*
@@ -1045,8 +1063,6 @@ static inline void io_req_work_grab_env(struct io_kiocb *req, · 129 unmodified lines
10451063 }
10461064 spin_unlock(&current->fs->lock);
10471065 }
1048− if (!req->work.task_pid)
1049− req->work.task_pid = task_pid_vnr(current);
10501066 }
10511067
10521068 static inline void io_req_work_drop_env(struct io_kiocb *req)
@@ -1087,6 +1103,7 @@ static inline void io_prep_async_work(struct io_kiocb *req, · 34 unmodified lines
10871103 req->work.flags |= IO_WQ_WORK_UNBOUND;
10881104 }
10891105
1106+ io_req_init_async(req);
10901107 io_req_work_grab_env(req, def);
10911108
10921109 *link = io_prep_linked_timeout(req);
@@ -1398,9 +1415,7 @@ static void __io_req_aux_free(struct io_kiocb *req) · 305 unmodified lines
13981415 kfree(req->io);
13991416 if (req->file)
14001417 io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE));
1401− if (req->task)
1402− put_task_struct(req->task);
1403−
1418+ __io_put_req_task(req);
14041419 io_req_work_drop_env(req);
14051420 }
14061421
@@ -1727,6 +1742,18 @@ static int io_put_kbuf(struct io_kiocb *req) · 320 unmodified lines
17271742 return cflags;
17281743 }
17291744
1745+static void io_iopoll_queue(struct list_head *again)
1746+{
1747+ struct io_kiocb *req;
1748+
1749+ do {
1750+ req = list_first_entry(again, struct io_kiocb, list);
1751+ list_del(&req->list);
1752+ refcount_inc(&req->refs);
1753+ io_queue_async_work(req);
1754+ } while (!list_empty(again));
1755+}
1756+
17301757 /*
17311758 * Find and free completed poll iocbs
17321759 */
@@ -1735,12 +1762,21 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, · 2 unmodified lines
17351762 {
17361763 struct req_batch rb;
17371764 struct io_kiocb *req;
1765+ LIST_HEAD(again);
1766+
1767+ /* order with ->result store in io_complete_rw_iopoll() */
1768+ smp_rmb();
17381769
17391770 rb.to_free = rb.need_iter = 0;
17401771 while (!list_empty(done)) {
17411772 int cflags = 0;
17421773
17431774 req = list_first_entry(done, struct io_kiocb, list);
1775+ if (READ_ONCE(req->result) == -EAGAIN) {
1776+ req->iopoll_completed = 0;
1777+ list_move_tail(&req->list, &again);
1778+ continue;
1779+ }
17441780 list_del(&req->list);
17451781
17461782 if (req->flags & REQ_F_BUFFER_SELECTED)
@@ -1758,18 +1794,9 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, · 11 unmodified lines
17581794 if (ctx->flags & IORING_SETUP_SQPOLL)
17591795 io_cqring_ev_posted(ctx);
17601796 io_free_req_many(ctx, &rb);
1761−}
17621797
1763−static void io_iopoll_queue(struct list_head *again)
1764−{
1765− struct io_kiocb *req;
1766−
1767− do {
1768− req = list_first_entry(again, struct io_kiocb, list);
1769− list_del(&req->list);
1770− refcount_inc(&req->refs);
1771− io_queue_async_work(req);
1772− } while (!list_empty(again));
1798+ if (!list_empty(&again))
1799+ io_iopoll_queue(&again);
17731800 }
17741801
17751802 static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
@@ -1777,7 +1804,6 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, · 1 unmodified line
17771804 {
17781805 struct io_kiocb *req, *tmp;
17791806 LIST_HEAD(done);
1780− LIST_HEAD(again);
17811807 bool spin;
17821808 int ret;
17831809
@@ -1803,13 +1829,6 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, · 19 unmodified lines
18031829 if (!list_empty(&done))
18041830 break;
18051831
1806− if (req->result == -EAGAIN) {
1807− list_move_tail(&req->list, &again);
1808− continue;
1809− }
1810− if (!list_empty(&again))
1811− break;
1812−
18131832 ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
18141833 if (ret < 0)
18151834 break;
@@ -1822,9 +1841,6 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, · 6 unmodified lines
18221841 if (!list_empty(&done))
18231842 io_iopoll_complete(ctx, nr_events, &done);
18241843
1825− if (!list_empty(&again))
1826− io_iopoll_queue(&again);
1827−
18281844 return ret;
18291845 }
18301846
@@ -1973,11 +1989,15 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) · 142 unmodified lines
19731989 if (kiocb->ki_flags & IOCB_WRITE)
19741990 kiocb_end_write(req);
19751991
1976− if (res != req->result)
1992+ if (res != -EAGAIN && res != req->result)
19771993 req_set_fail_links(req);
1978− req->result = res;
1979− if (res != -EAGAIN)
1994+
1995+ WRITE_ONCE(req->result, res);
1996+ /* order with io_poll_complete() checking ->result */
1997+ if (res != -EAGAIN) {
1998+ smp_wmb();
19801999 WRITE_ONCE(req->iopoll_completed, 1);
2000+ }
19812001 }
19822002
19832003 /*
@@ -2650,8 +2670,8 @@ copy_iov: · 666 unmodified lines
26502670 }
26512671 }
26522672 out_free:
2653− kfree(iovec);
2654− req->flags &= ~REQ_F_NEED_CLEANUP;
2673+ if (!(req->flags & REQ_F_NEED_CLEANUP))
2674+ kfree(iovec);
26552675 return ret;
26562676 }
26572677
@@ -2773,8 +2793,8 @@ copy_iov: · 115 unmodified lines
27732793 }
27742794 }
27752795 out_free:
2776− req->flags &= ~REQ_F_NEED_CLEANUP;
2777− kfree(iovec);
2796+ if (!(req->flags & REQ_F_NEED_CLEANUP))
2797+ kfree(iovec);
27782798 return ret;
27792799 }
27802800
@@ -4236,6 +4256,28 @@ static void io_async_queue_proc(struct file *file, struct wait_queue_head *head, · 1455 unmodified lines
42364256 __io_queue_proc(&pt->req->apoll->poll, pt, head);
42374257 }
42384258
4259+static void io_sq_thread_drop_mm(struct io_ring_ctx *ctx)
4260+{
4261+ struct mm_struct *mm = current->mm;
4262+
4263+ if (mm) {
4264+ kthread_unuse_mm(mm);
4265+ mmput(mm);
4266+ }
4267+}
4268+
4269+static int io_sq_thread_acquire_mm(struct io_ring_ctx *ctx,
4270+ struct io_kiocb *req)
4271+{
4272+ if (io_op_defs[req->opcode].needs_mm && !current->mm) {
4273+ if (unlikely(!mmget_not_zero(ctx->sqo_mm)))
4274+ return -EFAULT;
4275+ kthread_use_mm(ctx->sqo_mm);
4276+ }
4277+
4278+ return 0;
4279+}
4280+
42394281 static void io_async_task_func(struct callback_head *cb)
42404282 {
42414283 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
@@ -4270,11 +4312,16 @@ static void io_async_task_func(struct callback_head *cb) · 28 unmodified lines
42704312
42714313 if (!canceled) {
42724314 __set_current_state(TASK_RUNNING);
4315+ if (io_sq_thread_acquire_mm(ctx, req)) {
4316+ io_cqring_add_event(req, -EFAULT);
4317+ goto end_req;
4318+ }
42734319 mutex_lock(&ctx->uring_lock);
42744320 __io_queue_sqe(req, NULL);
42754321 mutex_unlock(&ctx->uring_lock);
42764322 } else {
42774323 io_cqring_ev_posted(ctx);
4324+end_req:
42784325 req_set_fail_links(req);
42794326 io_double_put_req(req);
42804327 }
@@ -4366,8 +4413,7 @@ static bool io_arm_poll_handler(struct io_kiocb *req) · 85 unmodified lines
43664413 memcpy(&apoll->work, &req->work, sizeof(req->work));
43674414 had_io = req->io != NULL;
43684415
4369− get_task_struct(current);
4370− req->task = current;
4416+ io_get_req_task(req);
43714417 req->apoll = apoll;
43724418 INIT_HLIST_NODE(&req->hash_node);
43734419
@@ -4555,8 +4601,7 @@ static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe · 181 unmodified lines
45554601 events = READ_ONCE(sqe->poll_events);
45564602 poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;
45574603
4558− get_task_struct(current);
4559− req->task = current;
4604+ io_get_req_task(req);
45604605 return 0;
45614606 }
45624607
@@ -4772,7 +4817,7 @@ static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr) · 209 unmodified lines
47724817 enum io_wq_cancel cancel_ret;
47734818 int ret = 0;
47744819
4775− cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr);
4820+ cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr, false);
47764821 switch (cancel_ret) {
47774822 case IO_WQ_CANCEL_OK:
47784823 ret = 0;
@@ -5817,17 +5862,14 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, · 1038 unmodified lines
58175862 req->flags = 0;
58185863 /* one is dropped after submission, the other at completion */
58195864 refcount_set(&req->refs, 2);
5820− req->task = NULL;
5865+ req->task = current;
58215866 req->result = 0;
58225867
58235868 if (unlikely(req->opcode >= IORING_OP_LAST))
58245869 return -EINVAL;
58255870
5826− if (io_op_defs[req->opcode].needs_mm && !current->mm) {
5827− if (unlikely(!mmget_not_zero(ctx->sqo_mm)))
5828− return -EFAULT;
5829− kthread_use_mm(ctx->sqo_mm);
5830− }
5871+ if (unlikely(io_sq_thread_acquire_mm(ctx, req)))
5872+ return -EFAULT;
58315873
58325874 sqe_flags = READ_ONCE(sqe->flags);
58335875 /* enforce forwards compatibility on users */
@@ -5936,16 +5978,6 @@ fail_req: · 102 unmodified lines
59365978 return submitted;
59375979 }
59385980
5939−static inline void io_sq_thread_drop_mm(struct io_ring_ctx *ctx)
5940−{
5941− struct mm_struct *mm = current->mm;
5942−
5943− if (mm) {
5944− kthread_unuse_mm(mm);
5945− mmput(mm);
5946− }
5947−}
5948−
59495981 static int io_sq_thread(void *data)
59505982 {
59515983 struct io_ring_ctx *ctx = data;
@@ -7331,7 +7363,17 @@ static void io_ring_exit_work(struct work_struct *work) · 1379 unmodified lines
73317363 if (ctx->rings)
73327364 io_cqring_overflow_flush(ctx, true);
73337365
7334− wait_for_completion(&ctx->ref_comp);
7366+ /*
7367+ * If we're doing polled IO and end up having requests being
7368+ * submitted async (out-of-line), then completions can come in while
7369+ * we're waiting for refs to drop. We need to reap these manually,
7370+ * as nobody else will be looking for them.
7371+ */
7372+ while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20)) {
7373+ io_iopoll_reap_events(ctx);
7374+ if (ctx->rings)
7375+ io_cqring_overflow_flush(ctx, true);
7376+ }
73357377 io_ring_ctx_free(ctx);
73367378 }
73377379
@@ -7365,9 +7407,22 @@ static int io_uring_release(struct inode *inode, struct file *file) · 27 unmodified lines
73657407 return 0;
73667408 }
73677409
7410+static bool io_wq_files_match(struct io_wq_work *work, void *data)
7411+{
7412+ struct files_struct *files = data;
7413+
7414+ return work->files == files;
7415+}
7416+
73687417 static void io_uring_cancel_files(struct io_ring_ctx *ctx,
73697418 struct files_struct *files)
73707419 {
7420+ if (list_empty_careful(&ctx->inflight_list))
7421+ return;
7422+
7423+ /* cancel all at once, should be faster than doing it one by one*/
7424+ io_wq_cancel_cb(ctx->io_wq, io_wq_files_match, files, true);
7425+
73717426 while (!list_empty_careful(&ctx->inflight_list)) {
73727427 struct io_kiocb *cancel_req = NULL, *req;
73737428 DEFINE_WAIT(wait);
@@ -7423,6 +7478,14 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx, · 49 unmodified lines
74237478 }
74247479 }
74257480
7481+static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
7482+{
7483+ struct io_kiocb *req = container_of(work, struct io_kiocb, work);
7484+ struct task_struct *task = data;
7485+
7486+ return req->task == task;
7487+}
7488+
74267489 static int io_uring_flush(struct file *file, void *data)
74277490 {
74287491 struct io_ring_ctx *ctx = file->private_data;
@@ -7433,7 +7496,7 @@ static int io_uring_flush(struct file *file, void *data) · 4 unmodified lines
74337496 * If the task is going away, cancel work it may have pending
74347497 */
74357498 if (fatal_signal_pending(current) || (current->flags & PF_EXITING))
7436− io_wq_cancel_pid(ctx->io_wq, task_pid_vnr(current));
7499+ io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, current, true);
74377500
74387501 return 0;
74397502 }