History

Merge tag 'for-5.18/io_uring-2022-03-18' of git://git.kernel.dk/linux-block

Pull io_uring updates from Jens Axboe: - Fixes for current file position. Still doesn't have the f_pos_lock sorted, but it's a step in the right direction (Dylan) - Tracing updates (Dylan, Stefan) - Improvements to io-wq locking (Hao) - Improvements for provided buffers (me, Pavel) - Support for registered file descriptors (me, Xiaoguang) - Support for ring messages (me) - Poll improvements (me) - Fix for fixed buffers and non-iterator reads/writes (me) - Support for NAPI on sockets (Olivier) - Ring quiesce improvements (Usama) - Misc fixes (Olivier, Pavel) * tag 'for-5.18/io_uring-2022-03-18' of git://git.kernel.dk/linux-block: (42 commits) io_uring: terminate manual loop iterator loop correctly for non-vecs io_uring: don't check unrelated req->open.how in accept request io_uring: manage provided buffers strictly ordered io_uring: fold evfd signalling under a slower path io_uring: thin down io_commit_cqring() io_uring: shuffle io_eventfd_signal() bits around io_uring: remove extra barrier for non-sqpoll iopoll io_uring: fix provided buffer return on failure for kiocb_done() io_uring: extend provided buf return to fails io_uring: refactor timeout cancellation cqe posting io_uring: normilise naming for fill_cqe* io_uring: cache poll/double-poll state with a request flag io_uring: cache req->apoll->events in req->cflags io_uring: move req->poll_refs into previous struct hole io_uring: make tracing format consistent io_uring: recycle apoll_poll entries io_uring: remove duplicated member check for io_msg_ring_prep() io_uring: allow submissions to continue on error io_uring: recycle provided buffers if request goes async io_uring: ensure reads re-import for selected buffers ...

Linus Torvalds committed 4 years ago

Commit af472a92 parents 93e220a 5e929365 files · +1200 −520Browse files

Unified · Split

5 files
  1. fs/io-wq.c +57 −57
  2. fs/io_uring.c +967 −284
  3. include/linux/io_uring.h +4 −1
  4. include/trace/events/io_uring.h +159 −174
  5. include/uapi/linux/io_uring.h +13 −4
fs/io-wq.c+57−57
@@ -76,6 +76,7 @@ struct io_wqe_acct { · 75 unmodified lines
7676 unsigned max_workers;
7777 int index;
7878 atomic_t nr_running;
79+ raw_spinlock_t lock;
7980 struct io_wq_work_list work_list;
8081 unsigned long flags;
8182 };
@@ -91,7 +92,7 @@ enum { · 9 unmodified lines
9192 */
9293 struct io_wqe {
9394 raw_spinlock_t lock;
94− struct io_wqe_acct acct[2];
95+ struct io_wqe_acct acct[IO_WQ_ACCT_NR];
9596
9697 int node;
9798
@@ -224,12 +225,12 @@ static void io_worker_exit(struct io_worker *worker) · 126 unmodified lines
224225 if (worker->flags & IO_WORKER_F_FREE)
225226 hlist_nulls_del_rcu(&worker->nulls_node);
226227 list_del_rcu(&worker->all_list);
227− preempt_disable();
228+ raw_spin_unlock(&wqe->lock);
228229 io_wqe_dec_running(worker);
229230 worker->flags = 0;
231+ preempt_disable();
230232 current->flags &= ~PF_IO_WORKER;
231233 preempt_enable();
232− raw_spin_unlock(&wqe->lock);
233234
234235 kfree_rcu(worker, rcu);
235236 io_worker_ref_put(wqe->wq);
@@ -238,10 +239,15 @@ static void io_worker_exit(struct io_worker *worker) · 2 unmodified lines
238239
239240 static inline bool io_acct_run_queue(struct io_wqe_acct *acct)
240241 {
242+ bool ret = false;
243+
244+ raw_spin_lock(&acct->lock);
241245 if (!wq_list_empty(&acct->work_list) &&
242246 !test_bit(IO_ACCT_STALLED_BIT, &acct->flags))
243− return true;
244− return false;
247+ ret = true;
248+ raw_spin_unlock(&acct->lock);
249+
250+ return ret;
245251 }
246252
247253 /*
@@ -385,7 +391,6 @@ fail: · 137 unmodified lines
385391 }
386392
387393 static void io_wqe_dec_running(struct io_worker *worker)
388− __must_hold(wqe->lock)
389394 {
390395 struct io_wqe_acct *acct = io_wqe_get_acct(worker);
391396 struct io_wqe *wqe = worker->wqe;
@@ -393,13 +398,14 @@ static void io_wqe_dec_running(struct io_worker *worker) · 1 unmodified line
393398 if (!(worker->flags & IO_WORKER_F_UP))
394399 return;
395400
396− if (atomic_dec_and_test(&acct->nr_running) && io_acct_run_queue(acct)) {
397− atomic_inc(&acct->nr_running);
398− atomic_inc(&wqe->wq->worker_refs);
399− raw_spin_unlock(&wqe->lock);
400− io_queue_worker_create(worker, acct, create_worker_cb);
401− raw_spin_lock(&wqe->lock);
402− }
401+ if (!atomic_dec_and_test(&acct->nr_running))
402+ return;
403+ if (!io_acct_run_queue(acct))
404+ return;
405+
406+ atomic_inc(&acct->nr_running);
407+ atomic_inc(&wqe->wq->worker_refs);
408+ io_queue_worker_create(worker, acct, create_worker_cb);
403409 }
404410
405411 /*
@@ -407,11 +413,12 @@ static void io_wqe_dec_running(struct io_worker *worker) · 1 unmodified line
407413 * it's currently on the freelist
408414 */
409415 static void __io_worker_busy(struct io_wqe *wqe, struct io_worker *worker)
410− __must_hold(wqe->lock)
411416 {
412417 if (worker->flags & IO_WORKER_F_FREE) {
413418 worker->flags &= ~IO_WORKER_F_FREE;
419+ raw_spin_lock(&wqe->lock);
414420 hlist_nulls_del_init_rcu(&worker->nulls_node);
421+ raw_spin_unlock(&wqe->lock);
415422 }
416423 }
417424
@@ -456,7 +463,7 @@ static bool io_wait_on_hash(struct io_wqe *wqe, unsigned int hash) · 38 unmodified lines
456463
457464 static struct io_wq_work *io_get_next_work(struct io_wqe_acct *acct,
458465 struct io_worker *worker)
459− __must_hold(wqe->lock)
466+ __must_hold(acct->lock)
460467 {
461468 struct io_wq_work_node *node, *prev;
462469 struct io_wq_work *work, *tail;
@@ -498,9 +505,9 @@ static struct io_wq_work *io_get_next_work(struct io_wqe_acct *acct, · 35 unmodified lines
498505 * work being added and clearing the stalled bit.
499506 */
500507 set_bit(IO_ACCT_STALLED_BIT, &acct->flags);
501− raw_spin_unlock(&wqe->lock);
508+ raw_spin_unlock(&acct->lock);
502509 unstalled = io_wait_on_hash(wqe, stall_hash);
503− raw_spin_lock(&wqe->lock);
510+ raw_spin_lock(&acct->lock);
504511 if (unstalled) {
505512 clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
506513 if (wq_has_sleeper(&wqe->wq->hash->wait))
@@ -538,7 +545,6 @@ static void io_assign_current_work(struct io_worker *worker, · 31 unmodified lines
538545 static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work);
539546
540547 static void io_worker_handle_work(struct io_worker *worker)
541− __releases(wqe->lock)
542548 {
543549 struct io_wqe_acct *acct = io_wqe_get_acct(worker);
544550 struct io_wqe *wqe = worker->wqe;
@@ -555,7 +561,9 @@ static void io_worker_handle_work(struct io_worker *worker) · 10 unmodified lines
555561 * can't make progress, any work completion or insertion will
556562 * clear the stalled flag.
557563 */
564+ raw_spin_lock(&acct->lock);
558565 work = io_get_next_work(acct, worker);
566+ raw_spin_unlock(&acct->lock);
559567 if (work) {
560568 __io_worker_busy(wqe, worker);
561569
@@ -569,10 +577,9 @@ static void io_worker_handle_work(struct io_worker *worker) · 7 unmodified lines
569577 raw_spin_lock(&worker->lock);
570578 worker->next_work = work;
571579 raw_spin_unlock(&worker->lock);
572− }
573− raw_spin_unlock(&wqe->lock);
574− if (!work)
580+ } else {
575581 break;
582+ }
576583 io_assign_current_work(worker, work);
577584 __set_current_state(TASK_RUNNING);
578585
@@ -608,8 +615,6 @@ static void io_worker_handle_work(struct io_worker *worker) · 29 unmodified lines
608615 wake_up(&wq->hash->wait);
609616 }
610617 } while (work);
611−
612− raw_spin_lock(&wqe->lock);
613618 } while (1);
614619 }
615620
@@ -633,12 +638,10 @@ static int io_wqe_worker(void *data) · 17 unmodified lines
633638 long ret;
634639
635640 set_current_state(TASK_INTERRUPTIBLE);
636−loop:
637− raw_spin_lock(&wqe->lock);
638− if (io_acct_run_queue(acct)) {
641+ while (io_acct_run_queue(acct))
639642 io_worker_handle_work(worker);
640− goto loop;
641− }
643+
644+ raw_spin_lock(&wqe->lock);
642645 /* timed out, exit unless we're the last worker */
643646 if (last_timeout && acct->nr_workers > 1) {
644647 acct->nr_workers--;
@@ -662,10 +665,8 @@ loop: · 17 unmodified lines
662665 last_timeout = !ret;
663666 }
664667
665− if (test_bit(IO_WQ_BIT_EXIT, &wq->state)) {
666− raw_spin_lock(&wqe->lock);
668+ if (test_bit(IO_WQ_BIT_EXIT, &wq->state))
667669 io_worker_handle_work(worker);
668− }
669670
670671 audit_free(current);
671672 io_worker_exit(worker);
@@ -705,10 +706,7 @@ void io_wq_worker_sleeping(struct task_struct *tsk) · 33 unmodified lines
705706 return;
706707
707708 worker->flags &= ~IO_WORKER_F_RUNNING;
708−
709− raw_spin_lock(&worker->wqe->lock);
710709 io_wqe_dec_running(worker);
711− raw_spin_unlock(&worker->wqe->lock);
712710 }
713711
714712 static void io_init_new_worker(struct io_wqe *wqe, struct io_worker *worker,
@@ -778,10 +776,12 @@ static void create_worker_cont(struct callback_head *cb) · 63 unmodified lines
778776 .cancel_all = true,
779777 };
780778
779+ raw_spin_unlock(&wqe->lock);
781780 while (io_acct_cancel_pending_work(wqe, acct, &match))
782− raw_spin_lock(&wqe->lock);
781+ ;
782+ } else {
783+ raw_spin_unlock(&wqe->lock);
783784 }
784− raw_spin_unlock(&wqe->lock);
785785 io_worker_ref_put(wqe->wq);
786786 kfree(worker);
787787 return;
@@ -914,6 +914,7 @@ static bool io_wq_work_match_item(struct io_wq_work *work, void *data) · 126 unmodified lines
914914 static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work)
915915 {
916916 struct io_wqe_acct *acct = io_work_get_acct(wqe, work);
917+ struct io_cb_cancel_data match;
917918 unsigned work_flags = work->flags;
918919 bool do_create;
919920
@@ -927,10 +928,12 @@ static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work) · 7 unmodified lines
927928 return;
928929 }
929930
930− raw_spin_lock(&wqe->lock);
931+ raw_spin_lock(&acct->lock);
931932 io_wqe_insert_work(wqe, work);
932933 clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
934+ raw_spin_unlock(&acct->lock);
933935
936+ raw_spin_lock(&wqe->lock);
934937 rcu_read_lock();
935938 do_create = !io_wqe_activate_free_worker(wqe, acct);
936939 rcu_read_unlock();
@@ -946,18 +949,18 @@ static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work) · 9 unmodified lines
946949 return;
947950
948951 raw_spin_lock(&wqe->lock);
949− /* fatal condition, failed to create the first worker */
950− if (!acct->nr_workers) {
951− struct io_cb_cancel_data match = {
952− .fn = io_wq_work_match_item,
953− .data = work,
954− .cancel_all = false,
955− };
956−
957− if (io_acct_cancel_pending_work(wqe, acct, &match))
958− raw_spin_lock(&wqe->lock);
952+ if (acct->nr_workers) {
953+ raw_spin_unlock(&wqe->lock);
954+ return;
959955 }
960956 raw_spin_unlock(&wqe->lock);
957+
958+ /* fatal condition, failed to create the first worker */
959+ match.fn = io_wq_work_match_item,
960+ match.data = work,
961+ match.cancel_all = false,
962+
963+ io_acct_cancel_pending_work(wqe, acct, &match);
961964 }
962965 }
963966
@@ -1032,22 +1035,23 @@ static inline void io_wqe_remove_pending(struct io_wqe *wqe, · 68 unmodified lines
10321035 static bool io_acct_cancel_pending_work(struct io_wqe *wqe,
10331036 struct io_wqe_acct *acct,
10341037 struct io_cb_cancel_data *match)
1035− __releases(wqe->lock)
10361038 {
10371039 struct io_wq_work_node *node, *prev;
10381040 struct io_wq_work *work;
10391041
1042+ raw_spin_lock(&acct->lock);
10401043 wq_list_for_each(node, prev, &acct->work_list) {
10411044 work = container_of(node, struct io_wq_work, list);
10421045 if (!match->fn(work, match->data))
10431046 continue;
10441047 io_wqe_remove_pending(wqe, work, prev);
1045− raw_spin_unlock(&wqe->lock);
1048+ raw_spin_unlock(&acct->lock);
10461049 io_run_cancel(work, wqe);
10471050 match->nr_pending++;
10481051 /* not safe to continue after unlock */
10491052 return true;
10501053 }
1054+ raw_spin_unlock(&acct->lock);
10511055
10521056 return false;
10531057 }
@@ -1061,7 +1065,6 @@ retry: · 7 unmodified lines
10611065 struct io_wqe_acct *acct = io_get_acct(wqe, i == 0);
10621066
10631067 if (io_acct_cancel_pending_work(wqe, acct, match)) {
1064− raw_spin_lock(&wqe->lock);
10651068 if (match->cancel_all)
10661069 goto retry;
10671070 break;
@@ -1103,13 +1106,11 @@ enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel, · 35 unmodified lines
11031106 for_each_node(node) {
11041107 struct io_wqe *wqe = wq->wqes[node];
11051108
1106− raw_spin_lock(&wqe->lock);
11071109 io_wqe_cancel_pending_work(wqe, &match);
1108− if (match.nr_pending && !match.cancel_all) {
1109− raw_spin_unlock(&wqe->lock);
1110+ if (match.nr_pending && !match.cancel_all)
11101111 return IO_WQ_CANCEL_OK;
1111− }
11121112
1113+ raw_spin_lock(&wqe->lock);
11131114 io_wqe_cancel_running_work(wqe, &match);
11141115 raw_spin_unlock(&wqe->lock);
11151116 if (match.nr_running && !match.cancel_all)
@@ -1190,6 +1191,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) · 74 unmodified lines
11901191 acct->index = i;
11911192 atomic_set(&acct->nr_running, 0);
11921193 INIT_WQ_LIST(&acct->work_list);
1194+ raw_spin_lock_init(&acct->lock);
11931195 }
11941196 wqe->wq = wq;
11951197 raw_spin_lock_init(&wqe->lock);
@@ -1282,9 +1284,7 @@ static void io_wq_destroy(struct io_wq *wq) · 86 unmodified lines
12821284 .fn = io_wq_work_match_all,
12831285 .cancel_all = true,
12841286 };
1285− raw_spin_lock(&wqe->lock);
12861287 io_wqe_cancel_pending_work(wqe, &match);
1287− raw_spin_unlock(&wqe->lock);
12881288 free_cpumask_var(wqe->cpu_mask);
12891289 kfree(wqe);
12901290 }
@@ -1376,7 +1376,7 @@ int io_wq_max_workers(struct io_wq *wq, int *new_count) · 85 unmodified lines
13761376 BUILD_BUG_ON((int) IO_WQ_ACCT_UNBOUND != (int) IO_WQ_UNBOUND);
13771377 BUILD_BUG_ON((int) IO_WQ_ACCT_NR != 2);
13781378
1379− for (i = 0; i < 2; i++) {
1379+ for (i = 0; i < IO_WQ_ACCT_NR; i++) {
13801380 if (new_count[i] > task_rlimit(current, RLIMIT_NPROC))
13811381 new_count[i] = task_rlimit(current, RLIMIT_NPROC);
13821382 }
fs/io_uring.c+967−284
@@ -63,6 +63,7 @@ · 62 unmodified lines
6363 #include <net/sock.h>
6464 #include <net/af_unix.h>
6565 #include <net/scm.h>
66+#include <net/busy_poll.h>
6667 #include <linux/anon_inodes.h>
6768 #include <linux/sched/mm.h>
6869 #include <linux/uaccess.h>
@@ -263,11 +264,18 @@ struct io_rsrc_data { · 194 unmodified lines
263264 bool quiesce;
264265 };
265266
267+struct io_buffer_list {
268+ struct list_head list;
269+ struct list_head buf_list;
270+ __u16 bgid;
271+};
272+
266273 struct io_buffer {
267274 struct list_head list;
268275 __u64 addr;
269276 __u32 len;
270277 __u16 bid;
278+ __u16 bgid;
271279 };
272280
273281 struct io_restriction {
@@ -326,6 +334,14 @@ struct io_submit_state { · 52 unmodified lines
326334 struct blk_plug plug;
327335 };
328336
337+struct io_ev_fd {
338+ struct eventfd_ctx *cq_ev_fd;
339+ unsigned int eventfd_async: 1;
340+ struct rcu_head rcu;
341+};
342+
343+#define IO_BUFFERS_HASH_BITS 5
344+
329345 struct io_ring_ctx {
330346 /* const or read-mostly hot data */
331347 struct {
@@ -335,11 +351,11 @@ struct io_ring_ctx { · 3 unmodified lines
335351 unsigned int flags;
336352 unsigned int compat: 1;
337353 unsigned int drain_next: 1;
338− unsigned int eventfd_async: 1;
339354 unsigned int restricted: 1;
340355 unsigned int off_timeout_used: 1;
341356 unsigned int drain_active: 1;
342357 unsigned int drain_disabled: 1;
358+ unsigned int has_evfd: 1;
343359 } ____cacheline_aligned_in_smp;
344360
345361 /* submission data */
@@ -378,7 +394,9 @@ struct io_ring_ctx { · 32 unmodified lines
378394 struct list_head timeout_list;
379395 struct list_head ltimeout_list;
380396 struct list_head cq_overflow_list;
381− struct xarray io_buffers;
397+ struct list_head *io_buffers;
398+ struct list_head io_buffers_cache;
399+ struct list_head apoll_cache;
382400 struct xarray personalities;
383401 u32 pers_next;
384402 unsigned sq_thread_idle;
@@ -395,11 +413,16 @@ struct io_ring_ctx { · 10 unmodified lines
395413 struct list_head sqd_list;
396414
397415 unsigned long check_cq_overflow;
416+#ifdef CONFIG_NET_RX_BUSY_POLL
417+ /* used to track busy poll napi_id */
418+ struct list_head napi_list;
419+ spinlock_t napi_lock; /* napi_list lock */
420+#endif
398421
399422 struct {
400423 unsigned cached_cq_tail;
401424 unsigned cq_entries;
402− struct eventfd_ctx *cq_ev_fd;
425+ struct io_ev_fd __rcu *io_ev_fd;
403426 struct wait_queue_head cq_wait;
404427 unsigned cq_extra;
405428 atomic_t cq_timeouts;
@@ -421,6 +444,8 @@ struct io_ring_ctx { · 15 unmodified lines
421444 struct hlist_head *cancel_hash;
422445 unsigned cancel_hash_bits;
423446 bool poll_multi_queue;
447+
448+ struct list_head io_buffers_comp;
424449 } ____cacheline_aligned_in_smp;
425450
426451 struct io_restriction restrictions;
@@ -436,6 +461,8 @@ struct io_ring_ctx { · 9 unmodified lines
436461 struct llist_head rsrc_put_llist;
437462 struct list_head rsrc_ref_list;
438463 spinlock_t rsrc_ref_lock;
464+
465+ struct list_head io_buffers_pages;
439466 };
440467
441468 /* Keep this last, we don't need it for the fast path */
@@ -461,6 +488,11 @@ struct io_ring_ctx { · 19 unmodified lines
461488 };
462489 };
463490
491+/*
492+ * Arbitrary limit, can be raised if need be
493+ */
494+#define IO_RINGFD_REG_MAX 16
495+
464496 struct io_uring_task {
465497 /* submission side */
466498 int cached_refs;
@@ -476,6 +508,7 @@ struct io_uring_task { · 9 unmodified lines
476508 struct io_wq_work_list task_list;
477509 struct io_wq_work_list prior_task_list;
478510 struct callback_head task_work;
511+ struct file **registered_rings;
479512 bool task_running;
480513 };
481514
@@ -690,6 +723,12 @@ struct io_hardlink { · 208 unmodified lines
690723 int flags;
691724 };
692725
726+struct io_msg {
727+ struct file *file;
728+ u64 user_data;
729+ u32 len;
730+};
731+
693732 struct io_async_connect {
694733 struct sockaddr_storage address;
695734 };
@@ -741,6 +780,8 @@ enum { · 45 unmodified lines
741780 REQ_F_ARM_LTIMEOUT_BIT,
742781 REQ_F_ASYNC_DATA_BIT,
743782 REQ_F_SKIP_LINK_CQES_BIT,
783+ REQ_F_SINGLE_POLL_BIT,
784+ REQ_F_DOUBLE_POLL_BIT,
744785 /* keep async read/write and isreg together and in order */
745786 REQ_F_SUPPORT_NOWAIT_BIT,
746787 REQ_F_ISREG_BIT,
@@ -799,6 +840,10 @@ enum { · 52 unmodified lines
799840 REQ_F_ASYNC_DATA = BIT(REQ_F_ASYNC_DATA_BIT),
800841 /* don't post CQEs while failing linked requests */
801842 REQ_F_SKIP_LINK_CQES = BIT(REQ_F_SKIP_LINK_CQES_BIT),
843+ /* single poll may be active */
844+ REQ_F_SINGLE_POLL = BIT(REQ_F_SINGLE_POLL_BIT),
845+ /* double poll may active */
846+ REQ_F_DOUBLE_POLL = BIT(REQ_F_DOUBLE_POLL_BIT),
802847 };
803848
804849 struct async_poll {
@@ -825,7 +870,7 @@ enum { · 20 unmodified lines
825870 * NOTE! Each of the iocb union members has the file pointer
826871 * as the first entry in their struct definition. So you can
827872 * access the file pointer through any of the sub-structs,
828− * or directly as just 'ki_filp' in this struct.
873+ * or directly as just 'file' in this struct.
829874 */
830875 struct io_kiocb {
831876 union {
@@ -855,6 +900,7 @@ struct io_kiocb { · 23 unmodified lines
855900 struct io_mkdir mkdir;
856901 struct io_symlink symlink;
857902 struct io_hardlink hardlink;
903+ struct io_msg msg;
858904 };
859905
860906 u8 opcode;
@@ -877,6 +923,7 @@ struct io_kiocb { · 16 unmodified lines
877923 /* used by request caches, completion batching and iopoll */
878924 struct io_wq_work_node comp_list;
879925 atomic_t refs;
926+ atomic_t poll_refs;
880927 struct io_kiocb *link;
881928 struct io_task_work io_task_work;
882929 /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
@@ -885,12 +932,11 @@ struct io_kiocb { · 2 unmodified lines
885932 struct async_poll *apoll;
886933 /* opcode allocated if it needs to store data for async defer */
887934 void *async_data;
888− struct io_wq_work work;
889935 /* custom credentials, valid IFF REQ_F_CREDS is set */
890− const struct cred *creds;
891936 /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */
892937 struct io_buffer *kbuf;
893− atomic_t poll_refs;
938+ const struct cred *creds;
939+ struct io_wq_work work;
894940 };
895941
896942 struct io_tctx_node {
@@ -1105,6 +1151,9 @@ static const struct io_op_def io_op_defs[] = { · 208 unmodified lines
11051151 [IORING_OP_MKDIRAT] = {},
11061152 [IORING_OP_SYMLINKAT] = {},
11071153 [IORING_OP_LINKAT] = {},
1154+ [IORING_OP_MSG_RING] = {
1155+ .needs_file = 1,
1156+ },
11081157 };
11091158
11101159 /* requests with any of those set should undergo io_disarm_next() */
@@ -1141,6 +1190,7 @@ static int io_install_fixed_file(struct io_kiocb *req, struct file *file, · 30 unmodified lines
11411190 static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags);
11421191
11431192 static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer);
1193+static void io_eventfd_signal(struct io_ring_ctx *ctx);
11441194
11451195 static struct kmem_cache *req_cachep;
11461196
@@ -1267,36 +1317,88 @@ static inline void io_req_set_rsrc_node(struct io_kiocb *req, · 120 unmodified lines
12671317 }
12681318 }
12691319
1270−static unsigned int __io_put_kbuf(struct io_kiocb *req)
1320+static unsigned int __io_put_kbuf(struct io_kiocb *req, struct list_head *list)
12711321 {
12721322 struct io_buffer *kbuf = req->kbuf;
12731323 unsigned int cflags;
12741324
1275− cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
1276− cflags |= IORING_CQE_F_BUFFER;
1325+ cflags = IORING_CQE_F_BUFFER | (kbuf->bid << IORING_CQE_BUFFER_SHIFT);
12771326 req->flags &= ~REQ_F_BUFFER_SELECTED;
1278− kfree(kbuf);
1327+ list_add(&kbuf->list, list);
12791328 req->kbuf = NULL;
12801329 return cflags;
12811330 }
12821331
1283−static inline unsigned int io_put_kbuf(struct io_kiocb *req)
1332+static inline unsigned int io_put_kbuf_comp(struct io_kiocb *req)
12841333 {
12851334 if (likely(!(req->flags & REQ_F_BUFFER_SELECTED)))
12861335 return 0;
1287− return __io_put_kbuf(req);
1336+ return __io_put_kbuf(req, &req->ctx->io_buffers_comp);
12881337 }
12891338
1290−static void io_refs_resurrect(struct percpu_ref *ref, struct completion *compl)
1339+static inline unsigned int io_put_kbuf(struct io_kiocb *req,
1340+ unsigned issue_flags)
12911341 {
1292− bool got = percpu_ref_tryget(ref);
1342+ unsigned int cflags;
1343+
1344+ if (likely(!(req->flags & REQ_F_BUFFER_SELECTED)))
1345+ return 0;
1346+
1347+ /*
1348+ * We can add this buffer back to two lists:
1349+ *
1350+ * 1) The io_buffers_cache list. This one is protected by the
1351+ * ctx->uring_lock. If we already hold this lock, add back to this
1352+ * list as we can grab it from issue as well.
1353+ * 2) The io_buffers_comp list. This one is protected by the
1354+ * ctx->completion_lock.
1355+ *
1356+ * We migrate buffers from the comp_list to the issue cache list
1357+ * when we need one.
1358+ */
1359+ if (issue_flags & IO_URING_F_UNLOCKED) {
1360+ struct io_ring_ctx *ctx = req->ctx;
12931361
1294− /* already at zero, wait for ->release() */
1295− if (!got)
1296− wait_for_completion(compl);
1297− percpu_ref_resurrect(ref);
1298− if (got)
1299− percpu_ref_put(ref);
1362+ spin_lock(&ctx->completion_lock);
1363+ cflags = __io_put_kbuf(req, &ctx->io_buffers_comp);
1364+ spin_unlock(&ctx->completion_lock);
1365+ } else {
1366+ cflags = __io_put_kbuf(req, &req->ctx->io_buffers_cache);
1367+ }
1368+
1369+ return cflags;
1370+}
1371+
1372+static struct io_buffer_list *io_buffer_get_list(struct io_ring_ctx *ctx,
1373+ unsigned int bgid)
1374+{
1375+ struct list_head *hash_list;
1376+ struct io_buffer_list *bl;
1377+
1378+ hash_list = &ctx->io_buffers[hash_32(bgid, IO_BUFFERS_HASH_BITS)];
1379+ list_for_each_entry(bl, hash_list, list)
1380+ if (bl->bgid == bgid || bgid == -1U)
1381+ return bl;
1382+
1383+ return NULL;
1384+}
1385+
1386+static void io_kbuf_recycle(struct io_kiocb *req)
1387+{
1388+ struct io_ring_ctx *ctx = req->ctx;
1389+ struct io_buffer_list *bl;
1390+ struct io_buffer *buf;
1391+
1392+ if (likely(!(req->flags & REQ_F_BUFFER_SELECTED)))
1393+ return;
1394+
1395+ lockdep_assert_held(&ctx->uring_lock);
1396+
1397+ buf = req->kbuf;
1398+ bl = io_buffer_get_list(ctx, buf->bgid);
1399+ list_add(&buf->list, &bl->buf_list);
1400+ req->flags &= ~REQ_F_BUFFER_SELECTED;
1401+ req->kbuf = NULL;
13001402 }
13011403
13021404 static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
@@ -1409,7 +1511,7 @@ static __cold void io_fallback_req_func(struct work_struct *work) · 106 unmodified lines
14091511 static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
14101512 {
14111513 struct io_ring_ctx *ctx;
1412− int hash_bits;
1514+ int i, hash_bits;
14131515
14141516 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
14151517 if (!ctx)
@@ -1436,6 +1538,13 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) · 20 unmodified lines
14361538 /* set invalid range, so io_import_fixed() fails meeting it */
14371539 ctx->dummy_ubuf->ubuf = -1UL;
14381540
1541+ ctx->io_buffers = kcalloc(1U << IO_BUFFERS_HASH_BITS,
1542+ sizeof(struct list_head), GFP_KERNEL);
1543+ if (!ctx->io_buffers)
1544+ goto err;
1545+ for (i = 0; i < (1U << IO_BUFFERS_HASH_BITS); i++)
1546+ INIT_LIST_HEAD(&ctx->io_buffers[i]);
1547+
14391548 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
14401549 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
14411550 goto err;
@@ -1444,14 +1553,17 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) · 2 unmodified lines
14441553 init_waitqueue_head(&ctx->sqo_sq_wait);
14451554 INIT_LIST_HEAD(&ctx->sqd_list);
14461555 INIT_LIST_HEAD(&ctx->cq_overflow_list);
1556+ INIT_LIST_HEAD(&ctx->io_buffers_cache);
1557+ INIT_LIST_HEAD(&ctx->apoll_cache);
14471558 init_completion(&ctx->ref_comp);
1448− xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1);
14491559 xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
14501560 mutex_init(&ctx->uring_lock);
14511561 init_waitqueue_head(&ctx->cq_wait);
14521562 spin_lock_init(&ctx->completion_lock);
14531563 spin_lock_init(&ctx->timeout_lock);
14541564 INIT_WQ_LIST(&ctx->iopoll_list);
1565+ INIT_LIST_HEAD(&ctx->io_buffers_pages);
1566+ INIT_LIST_HEAD(&ctx->io_buffers_comp);
14551567 INIT_LIST_HEAD(&ctx->defer_list);
14561568 INIT_LIST_HEAD(&ctx->timeout_list);
14571569 INIT_LIST_HEAD(&ctx->ltimeout_list);
@@ -1464,10 +1576,15 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) · 6 unmodified lines
14641576 INIT_WQ_LIST(&ctx->locked_free_list);
14651577 INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);
14661578 INIT_WQ_LIST(&ctx->submit_state.compl_reqs);
1579+#ifdef CONFIG_NET_RX_BUSY_POLL
1580+ INIT_LIST_HEAD(&ctx->napi_list);
1581+ spin_lock_init(&ctx->napi_lock);
1582+#endif
14671583 return ctx;
14681584 err:
14691585 kfree(ctx->dummy_ubuf);
14701586 kfree(ctx->cancel_hash);
1587+ kfree(ctx->io_buffers);
14711588 kfree(ctx);
14721589 return NULL;
14731590 }
@@ -1610,8 +1727,8 @@ static void io_queue_async_work(struct io_kiocb *req, bool *dont_use) · 136 unmodified lines
16101727 if (WARN_ON_ONCE(!same_thread_group(req->task, current)))
16111728 req->work.flags |= IO_WQ_WORK_CANCEL;
16121729
1613− trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
1614− &req->work, req->flags);
1730+ trace_io_uring_queue_async_work(ctx, req, req->user_data, req->opcode, req->flags,
1731+ &req->work, io_wq_is_hashed(&req->work));
16151732 io_wq_enqueue(tctx->io_wq, &req->work);
16161733 if (link)
16171734 io_queue_linked_timeout(link);
@@ -1681,22 +1798,27 @@ static __cold void io_flush_timeouts(struct io_ring_ctx *ctx) · 63 unmodified lines
16811798 spin_unlock_irq(&ctx->timeout_lock);
16821799 }
16831800
1684−static __cold void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
1685−{
1686− if (ctx->off_timeout_used)
1687− io_flush_timeouts(ctx);
1688− if (ctx->drain_active)
1689− io_queue_deferred(ctx);
1690−}
1691−
16921801 static inline void io_commit_cqring(struct io_ring_ctx *ctx)
16931802 {
1694− if (unlikely(ctx->off_timeout_used || ctx->drain_active))
1695− __io_commit_cqring_flush(ctx);
16961803 /* order cqe stores with ring update */
16971804 smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
16981805 }
16991806
1807+static void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
1808+{
1809+ if (ctx->off_timeout_used || ctx->drain_active) {
1810+ spin_lock(&ctx->completion_lock);
1811+ if (ctx->off_timeout_used)
1812+ io_flush_timeouts(ctx);
1813+ if (ctx->drain_active)
1814+ io_queue_deferred(ctx);
1815+ io_commit_cqring(ctx);
1816+ spin_unlock(&ctx->completion_lock);
1817+ }
1818+ if (ctx->has_evfd)
1819+ io_eventfd_signal(ctx);
1820+}
1821+
17001822 static inline bool io_sqring_full(struct io_ring_ctx *ctx)
17011823 {
17021824 struct io_rings *r = ctx->rings;
@@ -1726,23 +1848,34 @@ static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx) · 23 unmodified lines
17261848 return &rings->cqes[tail & mask];
17271849 }
17281850
1729−static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
1851+static void io_eventfd_signal(struct io_ring_ctx *ctx)
17301852 {
1731− if (likely(!ctx->cq_ev_fd))
1732− return false;
1853+ struct io_ev_fd *ev_fd;
1854+
1855+ rcu_read_lock();
1856+ /*
1857+ * rcu_dereference ctx->io_ev_fd once and use it for both for checking
1858+ * and eventfd_signal
1859+ */
1860+ ev_fd = rcu_dereference(ctx->io_ev_fd);
1861+
1862+ /*
1863+ * Check again if ev_fd exists incase an io_eventfd_unregister call
1864+ * completed between the NULL check of ctx->io_ev_fd at the start of
1865+ * the function and rcu_read_lock.
1866+ */
1867+ if (unlikely(!ev_fd))
1868+ goto out;
17331869 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
1734− return false;
1735− return !ctx->eventfd_async || io_wq_current_is_worker();
1870+ goto out;
1871+
1872+ if (!ev_fd->eventfd_async || io_wq_current_is_worker())
1873+ eventfd_signal(ev_fd->cq_ev_fd, 1);
1874+out:
1875+ rcu_read_unlock();
17361876 }
17371877
1738−/*
1739− * This should only get called when at least one event has been posted.
1740− * Some applications rely on the eventfd notification count only changing
1741− * IFF a new CQE has been added to the CQ ring. There's no depedency on
1742− * 1:1 relationship between how many times this function is called (and
1743− * hence the eventfd count) and number of CQEs posted to the CQ ring.
1744− */
1745−static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1878+static inline void io_cqring_wake(struct io_ring_ctx *ctx)
17461879 {
17471880 /*
17481881 * wake_up_all() may seem excessive, but io_wake_function() and
@@ -1751,21 +1884,32 @@ static void io_cqring_ev_posted(struct io_ring_ctx *ctx) · 2 unmodified lines
17511884 */
17521885 if (wq_has_sleeper(&ctx->cq_wait))
17531886 wake_up_all(&ctx->cq_wait);
1754− if (io_should_trigger_evfd(ctx))
1755− eventfd_signal(ctx->cq_ev_fd, 1);
1887+}
1888+
1889+/*
1890+ * This should only get called when at least one event has been posted.
1891+ * Some applications rely on the eventfd notification count only changing
1892+ * IFF a new CQE has been added to the CQ ring. There's no depedency on
1893+ * 1:1 relationship between how many times this function is called (and
1894+ * hence the eventfd count) and number of CQEs posted to the CQ ring.
1895+ */
1896+static inline void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1897+{
1898+ if (unlikely(ctx->off_timeout_used || ctx->drain_active ||
1899+ ctx->has_evfd))
1900+ __io_commit_cqring_flush(ctx);
1901+
1902+ io_cqring_wake(ctx);
17561903 }
17571904
17581905 static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)
17591906 {
1760− /* see waitqueue_active() comment */
1761− smp_mb();
1907+ if (unlikely(ctx->off_timeout_used || ctx->drain_active ||
1908+ ctx->has_evfd))
1909+ __io_commit_cqring_flush(ctx);
17621910
1763− if (ctx->flags & IORING_SETUP_SQPOLL) {
1764− if (waitqueue_active(&ctx->cq_wait))
1765− wake_up_all(&ctx->cq_wait);
1766− }
1767− if (io_should_trigger_evfd(ctx))
1768− eventfd_signal(ctx->cq_ev_fd, 1);
1911+ if (ctx->flags & IORING_SETUP_SQPOLL)
1912+ io_cqring_wake(ctx);
17691913 }
17701914
17711915 /* Returns true if there are no backlogged entries after the flush */
@@ -1905,8 +2049,6 @@ static inline bool __io_fill_cqe(struct io_ring_ctx *ctx, u64 user_data, · 133 unmodified lines
19052049 {
19062050 struct io_uring_cqe *cqe;
19072051
1908− trace_io_uring_complete(ctx, user_data, res, cflags);
1909−
19102052 /*
19112053 * If we can't get a cq entry, userspace overflowed the
19122054 * submission (by quite a lot). Increment the overflow count in
@@ -1922,16 +2064,23 @@ static inline bool __io_fill_cqe(struct io_ring_ctx *ctx, u64 user_data, · 9 unmodified lines
19222064 return io_cqring_event_overflow(ctx, user_data, res, cflags);
19232065 }
19242066
2067+static inline bool __io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)
2068+{
2069+ trace_io_uring_complete(req->ctx, req, req->user_data, res, cflags);
2070+ return __io_fill_cqe(req->ctx, req->user_data, res, cflags);
2071+}
2072+
19252073 static noinline void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)
19262074 {
19272075 if (!(req->flags & REQ_F_CQE_SKIP))
1928− __io_fill_cqe(req->ctx, req->user_data, res, cflags);
2076+ __io_fill_cqe_req(req, res, cflags);
19292077 }
19302078
19312079 static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data,
19322080 s32 res, u32 cflags)
19332081 {
19342082 ctx->cq_extra++;
2083+ trace_io_uring_complete(ctx, NULL, user_data, res, cflags);
19352084 return __io_fill_cqe(ctx, user_data, res, cflags);
19362085 }
19372086
@@ -1941,7 +2090,7 @@ static void __io_req_complete_post(struct io_kiocb *req, s32 res, · 3 unmodified lines
19412090 struct io_ring_ctx *ctx = req->ctx;
19422091
19432092 if (!(req->flags & REQ_F_CQE_SKIP))
1944− __io_fill_cqe(ctx, req->user_data, res, cflags);
2093+ __io_fill_cqe_req(req, res, cflags);
19452094 /*
19462095 * If we're the last reference to this request, add to our locked
19472096 * free_list cache.
@@ -2000,7 +2149,7 @@ static inline void io_req_complete(struct io_kiocb *req, s32 res) · 52 unmodified lines
20002149 static void io_req_complete_failed(struct io_kiocb *req, s32 res)
20012150 {
20022151 req_set_fail(req);
2003− io_req_complete_post(req, res, 0);
2152+ io_req_complete_post(req, res, io_put_kbuf(req, 0));
20042153 }
20052154
20062155 static void io_req_complete_fail_submit(struct io_kiocb *req)
@@ -2183,7 +2332,9 @@ static void io_fail_links(struct io_kiocb *req) · 176 unmodified lines
21832332 nxt = link->link;
21842333 link->link = NULL;
21852334
2186− trace_io_uring_fail_link(req, link);
2335+ trace_io_uring_fail_link(req->ctx, req, req->user_data,
2336+ req->opcode, link);
2337+
21872338 if (!ignore_cqes) {
21882339 link->flags &= ~REQ_F_CQE_SKIP;
21892340 io_fill_cqe_req(link, res, 0);
@@ -2302,7 +2453,8 @@ static void handle_prev_tw_list(struct io_wq_work_node *node, · 112 unmodified lines
23022453 if (likely(*uring_locked))
23032454 req->io_task_work.func(req, uring_locked);
23042455 else
2305− __io_req_complete_post(req, req->result, io_put_kbuf(req));
2456+ __io_req_complete_post(req, req->result,
2457+ io_put_kbuf_comp(req));
23062458 node = next;
23072459 } while (node);
23082460
@@ -2530,8 +2682,16 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx) · 221 unmodified lines
25302682 comp_list);
25312683
25322684 if (!(req->flags & REQ_F_CQE_SKIP))
2533− __io_fill_cqe(ctx, req->user_data, req->result,
2534− req->cflags);
2685+ __io_fill_cqe_req(req, req->result, req->cflags);
2686+ if ((req->flags & REQ_F_POLLED) && req->apoll) {
2687+ struct async_poll *apoll = req->apoll;
2688+
2689+ if (apoll->double_poll)
2690+ kfree(apoll->double_poll);
2691+ list_add(&apoll->poll.wait.entry,
2692+ &ctx->apoll_cache);
2693+ req->flags &= ~REQ_F_POLLED;
2694+ }
25352695 }
25362696
25372697 io_commit_cqring(ctx);
@@ -2653,7 +2813,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) · 115 unmodified lines
26532813 if (unlikely(req->flags & REQ_F_CQE_SKIP))
26542814 continue;
26552815
2656− __io_fill_cqe(ctx, req->user_data, req->result, io_put_kbuf(req));
2816+ __io_fill_cqe_req(req, req->result, io_put_kbuf(req, 0));
26572817 nr_events++;
26582818 }
26592819
@@ -2829,14 +2989,14 @@ static bool __io_complete_rw_common(struct io_kiocb *req, long res) · 169 unmodified lines
28292989
28302990 static inline void io_req_task_complete(struct io_kiocb *req, bool *locked)
28312991 {
2832− unsigned int cflags = io_put_kbuf(req);
28332992 int res = req->result;
28342993
28352994 if (*locked) {
2836− io_req_complete_state(req, res, cflags);
2995+ io_req_complete_state(req, res, io_put_kbuf(req, 0));
28372996 io_req_add_compl_list(req);
28382997 } else {
2839− io_req_complete_post(req, res, cflags);
2998+ io_req_complete_post(req, res,
2999+ io_put_kbuf(req, IO_URING_F_UNLOCKED));
28403000 }
28413001 }
28423002
@@ -2845,7 +3005,8 @@ static void __io_complete_rw(struct io_kiocb *req, long res, · 2 unmodified lines
28453005 {
28463006 if (__io_complete_rw_common(req, res))
28473007 return;
2848− __io_req_complete(req, issue_flags, req->result, io_put_kbuf(req));
3008+ __io_req_complete(req, issue_flags, req->result,
3009+ io_put_kbuf(req, issue_flags));
28493010 }
28503011
28513012 static void io_complete_rw(struct kiocb *kiocb, long res)
@@ -3000,14 +3161,6 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 148 unmodified lines
30003161 req->flags |= io_file_get_flags(file) << REQ_F_SUPPORT_NOWAIT_BIT;
30013162
30023163 kiocb->ki_pos = READ_ONCE(sqe->off);
3003− if (kiocb->ki_pos == -1) {
3004− if (!(file->f_mode & FMODE_STREAM)) {
3005− req->flags |= REQ_F_CUR_POS;
3006− kiocb->ki_pos = file->f_pos;
3007− } else {
3008− kiocb->ki_pos = 0;
3009− }
3010− }
30113164 kiocb->ki_flags = iocb_flags(file);
30123165 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
30133166 if (unlikely(ret))
@@ -3074,6 +3227,24 @@ static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret) · 60 unmodified lines
30743227 }
30753228 }
30763229
3230+static inline loff_t *io_kiocb_update_pos(struct io_kiocb *req)
3231+{
3232+ struct kiocb *kiocb = &req->rw.kiocb;
3233+ bool is_stream = req->file->f_mode & FMODE_STREAM;
3234+
3235+ if (kiocb->ki_pos == -1) {
3236+ if (!is_stream) {
3237+ req->flags |= REQ_F_CUR_POS;
3238+ kiocb->ki_pos = req->file->f_pos;
3239+ return &kiocb->ki_pos;
3240+ } else {
3241+ kiocb->ki_pos = 0;
3242+ return NULL;
3243+ }
3244+ }
3245+ return is_stream ? NULL : &kiocb->ki_pos;
3246+}
3247+
30773248 static void kiocb_done(struct io_kiocb *req, ssize_t ret,
30783249 unsigned int issue_flags)
30793250 {
@@ -3096,14 +3267,10 @@ static void kiocb_done(struct io_kiocb *req, ssize_t ret, · 16 unmodified lines
30963267
30973268 if (req->flags & REQ_F_REISSUE) {
30983269 req->flags &= ~REQ_F_REISSUE;
3099− if (io_resubmit_prep(req)) {
3270+ if (io_resubmit_prep(req))
31003271 io_req_task_queue_reissue(req);
3101− } else {
3102− req_set_fail(req);
3103− req->result = ret;
3104− req->io_task_work.func = io_req_task_complete;
3105− io_req_task_work_add(req, false);
3106− }
3272+ else
3273+ io_req_task_queue_fail(req, ret);
31073274 }
31083275 }
31093276
@@ -3201,30 +3368,36 @@ static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock) · 91 unmodified lines
32013368 mutex_lock(&ctx->uring_lock);
32023369 }
32033370
3371+static void io_buffer_add_list(struct io_ring_ctx *ctx,
3372+ struct io_buffer_list *bl, unsigned int bgid)
3373+{
3374+ struct list_head *list;
3375+
3376+ list = &ctx->io_buffers[hash_32(bgid, IO_BUFFERS_HASH_BITS)];
3377+ INIT_LIST_HEAD(&bl->buf_list);
3378+ bl->bgid = bgid;
3379+ list_add(&bl->list, list);
3380+}
3381+
32043382 static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,
32053383 int bgid, unsigned int issue_flags)
32063384 {
32073385 struct io_buffer *kbuf = req->kbuf;
3208− struct io_buffer *head;
32093386 bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
3387+ struct io_ring_ctx *ctx = req->ctx;
3388+ struct io_buffer_list *bl;
32103389
32113390 if (req->flags & REQ_F_BUFFER_SELECTED)
32123391 return kbuf;
32133392
3214− io_ring_submit_lock(req->ctx, needs_lock);
3393+ io_ring_submit_lock(ctx, needs_lock);
32153394
3216− lockdep_assert_held(&req->ctx->uring_lock);
3395+ lockdep_assert_held(&ctx->uring_lock);
32173396
3218− head = xa_load(&req->ctx->io_buffers, bgid);
3219− if (head) {
3220− if (!list_empty(&head->list)) {
3221− kbuf = list_last_entry(&head->list, struct io_buffer,
3222− list);
3223− list_del(&kbuf->list);
3224− } else {
3225− kbuf = head;
3226− xa_erase(&req->ctx->io_buffers, bgid);
3227− }
3397+ bl = io_buffer_get_list(ctx, bgid);
3398+ if (bl && !list_empty(&bl->buf_list)) {
3399+ kbuf = list_first_entry(&bl->buf_list, struct io_buffer, list);
3400+ list_del(&kbuf->list);
32283401 if (*len > kbuf->len)
32293402 *len = kbuf->len;
32303403 req->flags |= REQ_F_BUFFER_SELECTED;
@@ -3400,6 +3573,7 @@ static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter) · 169 unmodified lines
34003573 struct kiocb *kiocb = &req->rw.kiocb;
34013574 struct file *file = req->file;
34023575 ssize_t ret = 0;
3576+ loff_t *ppos;
34033577
34043578 /*
34053579 * Don't support polled IO through this interface, and we can't
@@ -3412,6 +3586,8 @@ static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter) · 6 unmodified lines
34123586 !(kiocb->ki_filp->f_flags & O_NONBLOCK))
34133587 return -EAGAIN;
34143588
3589+ ppos = io_kiocb_ppos(kiocb);
3590+
34153591 while (iov_iter_count(iter)) {
34163592 struct iovec iovec;
34173593 ssize_t nr;
@@ -3425,10 +3601,10 @@ static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter) · 7 unmodified lines
34253601
34263602 if (rw == READ) {
34273603 nr = file->f_op->read(file, iovec.iov_base,
3428− iovec.iov_len, io_kiocb_ppos(kiocb));
3604+ iovec.iov_len, ppos);
34293605 } else {
34303606 nr = file->f_op->write(file, iovec.iov_base,
3431− iovec.iov_len, io_kiocb_ppos(kiocb));
3607+ iovec.iov_len, ppos);
34323608 }
34333609
34343610 if (nr < 0) {
@@ -3436,13 +3612,15 @@ static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter) · 1 unmodified line
34363612 ret = nr;
34373613 break;
34383614 }
3615+ ret += nr;
34393616 if (!iov_iter_is_bvec(iter)) {
34403617 iov_iter_advance(iter, nr);
34413618 } else {
3442− req->rw.len -= nr;
34433619 req->rw.addr += nr;
3620+ req->rw.len -= nr;
3621+ if (!req->rw.len)
3622+ break;
34443623 }
3445− ret += nr;
34463624 if (nr != iovec.iov_len)
34473625 break;
34483626 }
@@ -3629,12 +3807,23 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) · 180 unmodified lines
36293807 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
36303808 struct io_async_rw *rw;
36313809 ssize_t ret, ret2;
3810+ loff_t *ppos;
36323811
36333812 if (!req_has_async_data(req)) {
36343813 ret = io_import_iovec(READ, req, &iovec, s, issue_flags);
36353814 if (unlikely(ret < 0))
36363815 return ret;
36373816 } else {
3817+ /*
3818+ * Safe and required to re-import if we're using provided
3819+ * buffers, as we dropped the selected one before retry.
3820+ */
3821+ if (req->flags & REQ_F_BUFFER_SELECT) {
3822+ ret = io_import_iovec(READ, req, &iovec, s, issue_flags);
3823+ if (unlikely(ret < 0))
3824+ return ret;
3825+ }
3826+
36383827 rw = req->async_data;
36393828 s = &rw->s;
36403829 /*
@@ -3659,7 +3848,9 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) · 18 unmodified lines
36593848 kiocb->ki_flags &= ~IOCB_NOWAIT;
36603849 }
36613850
3662− ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), req->result);
3851+ ppos = io_kiocb_update_pos(req);
3852+
3853+ ret = rw_verify_area(READ, req->file, ppos, req->result);
36633854 if (unlikely(ret)) {
36643855 kfree(iovec);
36653856 return ret;
@@ -3669,6 +3860,9 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) · 3 unmodified lines
36693860
36703861 if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) {
36713862 req->flags &= ~REQ_F_REISSUE;
3863+ /* if we can poll, just do that */
3864+ if (req->opcode == IORING_OP_READ && file_can_poll(req->file))
3865+ return -EAGAIN;
36723866 /* IOPOLL retry should happen for io-wq threads */
36733867 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
36743868 goto done;
@@ -3758,6 +3952,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) · 83 unmodified lines
37583952 struct kiocb *kiocb = &req->rw.kiocb;
37593953 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
37603954 ssize_t ret, ret2;
3955+ loff_t *ppos;
37613956
37623957 if (!req_has_async_data(req)) {
37633958 ret = io_import_iovec(WRITE, req, &iovec, s, issue_flags);
@@ -3788,7 +3983,9 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) · 24 unmodified lines
37883983 kiocb->ki_flags &= ~IOCB_NOWAIT;
37893984 }
37903985
3791− ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), req->result);
3986+ ppos = io_kiocb_update_pos(req);
3987+
3988+ ret = rw_verify_area(WRITE, req->file, ppos, req->result);
37923989 if (unlikely(ret))
37933990 goto out_free;
37943991
@@ -4235,6 +4432,45 @@ static int io_nop(struct io_kiocb *req, unsigned int issue_flags) · 440 unmodified lines
42354432 return 0;
42364433 }
42374434
4435+static int io_msg_ring_prep(struct io_kiocb *req,
4436+ const struct io_uring_sqe *sqe)
4437+{
4438+ if (unlikely(sqe->addr || sqe->ioprio || sqe->rw_flags ||
4439+ sqe->splice_fd_in || sqe->buf_index || sqe->personality))
4440+ return -EINVAL;
4441+
4442+ if (req->file->f_op != &io_uring_fops)
4443+ return -EBADFD;
4444+
4445+ req->msg.user_data = READ_ONCE(sqe->off);
4446+ req->msg.len = READ_ONCE(sqe->len);
4447+ return 0;
4448+}
4449+
4450+static int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags)
4451+{
4452+ struct io_ring_ctx *target_ctx;
4453+ struct io_msg *msg = &req->msg;
4454+ int ret = -EOVERFLOW;
4455+ bool filled;
4456+
4457+ target_ctx = req->file->private_data;
4458+
4459+ spin_lock(&target_ctx->completion_lock);
4460+ filled = io_fill_cqe_aux(target_ctx, msg->user_data, msg->len,
4461+ IORING_CQE_F_MSG);
4462+ io_commit_cqring(target_ctx);
4463+ spin_unlock(&target_ctx->completion_lock);
4464+
4465+ if (filled) {
4466+ io_cqring_ev_posted(target_ctx);
4467+ ret = 0;
4468+ }
4469+
4470+ __io_req_complete(req, issue_flags, ret, 0);
4471+ return 0;
4472+}
4473+
42384474 static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
42394475 {
42404476 struct io_ring_ctx *ctx = req->ctx;
@@ -4458,8 +4694,8 @@ static int io_remove_buffers_prep(struct io_kiocb *req, · 217 unmodified lines
44584694 return 0;
44594695 }
44604696
4461−static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
4462− int bgid, unsigned nbufs)
4697+static int __io_remove_buffers(struct io_ring_ctx *ctx,
4698+ struct io_buffer_list *bl, unsigned nbufs)
44634699 {
44644700 unsigned i = 0;
44654701
@@ -4468,19 +4704,16 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf, · 2 unmodified lines
44684704 return 0;
44694705
44704706 /* the head kbuf is the list itself */
4471− while (!list_empty(&buf->list)) {
4707+ while (!list_empty(&bl->buf_list)) {
44724708 struct io_buffer *nxt;
44734709
4474− nxt = list_first_entry(&buf->list, struct io_buffer, list);
4710+ nxt = list_first_entry(&bl->buf_list, struct io_buffer, list);
44754711 list_del(&nxt->list);
4476− kfree(nxt);
44774712 if (++i == nbufs)
44784713 return i;
44794714 cond_resched();
44804715 }
44814716 i++;
4482− kfree(buf);
4483− xa_erase(&ctx->io_buffers, bgid);
44844717
44854718 return i;
44864719 }
@@ -4489,7 +4722,7 @@ static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) · 2 unmodified lines
44894722 {
44904723 struct io_provide_buf *p = &req->pbuf;
44914724 struct io_ring_ctx *ctx = req->ctx;
4492− struct io_buffer *head;
4725+ struct io_buffer_list *bl;
44934726 int ret = 0;
44944727 bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
44954728
@@ -4498,9 +4731,9 @@ static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) · 2 unmodified lines
44984731 lockdep_assert_held(&ctx->uring_lock);
44994732
45004733 ret = -ENOENT;
4501− head = xa_load(&ctx->io_buffers, p->bgid);
4502− if (head)
4503− ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);
4734+ bl = io_buffer_get_list(ctx, p->bgid);
4735+ if (bl)
4736+ ret = __io_remove_buffers(ctx, bl, p->nbufs);
45044737 if (ret < 0)
45054738 req_set_fail(req);
45064739
@@ -4545,39 +4778,80 @@ static int io_provide_buffers_prep(struct io_kiocb *req, · 38 unmodified lines
45454778 return 0;
45464779 }
45474780
4548−static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)
4781+static int io_refill_buffer_cache(struct io_ring_ctx *ctx)
4782+{
4783+ struct io_buffer *buf;
4784+ struct page *page;
4785+ int bufs_in_page;
4786+
4787+ /*
4788+ * Completions that don't happen inline (eg not under uring_lock) will
4789+ * add to ->io_buffers_comp. If we don't have any free buffers, check
4790+ * the completion list and splice those entries first.
4791+ */
4792+ if (!list_empty_careful(&ctx->io_buffers_comp)) {
4793+ spin_lock(&ctx->completion_lock);
4794+ if (!list_empty(&ctx->io_buffers_comp)) {
4795+ list_splice_init(&ctx->io_buffers_comp,
4796+ &ctx->io_buffers_cache);
4797+ spin_unlock(&ctx->completion_lock);
4798+ return 0;
4799+ }
4800+ spin_unlock(&ctx->completion_lock);
4801+ }
4802+
4803+ /*
4804+ * No free buffers and no completion entries either. Allocate a new
4805+ * page worth of buffer entries and add those to our freelist.
4806+ */
4807+ page = alloc_page(GFP_KERNEL_ACCOUNT);
4808+ if (!page)
4809+ return -ENOMEM;
4810+
4811+ list_add(&page->lru, &ctx->io_buffers_pages);
4812+
4813+ buf = page_address(page);
4814+ bufs_in_page = PAGE_SIZE / sizeof(*buf);
4815+ while (bufs_in_page) {
4816+ list_add_tail(&buf->list, &ctx->io_buffers_cache);
4817+ buf++;
4818+ bufs_in_page--;
4819+ }
4820+
4821+ return 0;
4822+}
4823+
4824+static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf,
4825+ struct io_buffer_list *bl)
45494826 {
45504827 struct io_buffer *buf;
45514828 u64 addr = pbuf->addr;
45524829 int i, bid = pbuf->bid;
45534830
45544831 for (i = 0; i < pbuf->nbufs; i++) {
4555− buf = kmalloc(sizeof(*buf), GFP_KERNEL_ACCOUNT);
4556− if (!buf)
4832+ if (list_empty(&ctx->io_buffers_cache) &&
4833+ io_refill_buffer_cache(ctx))
45574834 break;
4558−
4835+ buf = list_first_entry(&ctx->io_buffers_cache, struct io_buffer,
4836+ list);
4837+ list_move_tail(&buf->list, &bl->buf_list);
45594838 buf->addr = addr;
45604839 buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT);
45614840 buf->bid = bid;
4841+ buf->bgid = pbuf->bgid;
45624842 addr += pbuf->len;
45634843 bid++;
4564− if (!*head) {
4565− INIT_LIST_HEAD(&buf->list);
4566− *head = buf;
4567− } else {
4568− list_add_tail(&buf->list, &(*head)->list);
4569− }
45704844 cond_resched();
45714845 }
45724846
4573− return i ? i : -ENOMEM;
4847+ return i ? 0 : -ENOMEM;
45744848 }
45754849
45764850 static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
45774851 {
45784852 struct io_provide_buf *p = &req->pbuf;
45794853 struct io_ring_ctx *ctx = req->ctx;
4580− struct io_buffer *head, *list;
4854+ struct io_buffer_list *bl;
45814855 int ret = 0;
45824856 bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
45834857
@@ -4585,14 +4859,18 @@ static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags) · 1 unmodified line
45854859
45864860 lockdep_assert_held(&ctx->uring_lock);
45874861
4588− list = head = xa_load(&ctx->io_buffers, p->bgid);
4589−
4590− ret = io_add_buffers(p, &head);
4591− if (ret >= 0 && !list) {
4592− ret = xa_insert(&ctx->io_buffers, p->bgid, head, GFP_KERNEL);
4593− if (ret < 0)
4594− __io_remove_buffers(ctx, head, p->bgid, -1U);
4862+ bl = io_buffer_get_list(ctx, p->bgid);
4863+ if (unlikely(!bl)) {
4864+ bl = kmalloc(sizeof(*bl), GFP_KERNEL);
4865+ if (!bl) {
4866+ ret = -ENOMEM;
4867+ goto err;
4868+ }
4869+ io_buffer_add_list(ctx, bl, p->bgid);
45954870 }
4871+
4872+ ret = io_add_buffers(ctx, p, bl);
4873+err:
45964874 if (ret < 0)
45974875 req_set_fail(req);
45984876 /* complete before unlock, IOPOLL may need the lock */
@@ -5184,7 +5462,7 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) · 585 unmodified lines
51845462 if (kmsg->free_iov)
51855463 kfree(kmsg->free_iov);
51865464 req->flags &= ~REQ_F_NEED_CLEANUP;
5187− __io_req_complete(req, issue_flags, ret, io_put_kbuf(req));
5465+ __io_req_complete(req, issue_flags, ret, io_put_kbuf(req, issue_flags));
51885466 return 0;
51895467 }
51905468
@@ -5239,7 +5517,8 @@ static int io_recv(struct io_kiocb *req, unsigned int issue_flags) · 48 unmodified lines
52395517 out_free:
52405518 req_set_fail(req);
52415519 }
5242− __io_req_complete(req, issue_flags, ret, io_put_kbuf(req));
5520+
5521+ __io_req_complete(req, issue_flags, ret, io_put_kbuf(req, issue_flags));
52435522 return 0;
52445523 }
52455524
@@ -5258,8 +5537,7 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 12 unmodified lines
52585537 accept->nofile = rlimit(RLIMIT_NOFILE);
52595538
52605539 accept->file_slot = READ_ONCE(sqe->file_index);
5261− if (accept->file_slot && ((req->open.how.flags & O_CLOEXEC) ||
5262− (accept->flags & SOCK_CLOEXEC)))
5540+ if (accept->file_slot && (accept->flags & SOCK_CLOEXEC))
52635541 return -EINVAL;
52645542 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
52655543 return -EINVAL;
@@ -5399,6 +5677,108 @@ IO_NETOP_FN(send); · 133 unmodified lines
53995677 IO_NETOP_FN(recv);
54005678 #endif /* CONFIG_NET */
54015679
5680+#ifdef CONFIG_NET_RX_BUSY_POLL
5681+
5682+#define NAPI_TIMEOUT (60 * SEC_CONVERSION)
5683+
5684+struct napi_entry {
5685+ struct list_head list;
5686+ unsigned int napi_id;
5687+ unsigned long timeout;
5688+};
5689+
5690+/*
5691+ * Add busy poll NAPI ID from sk.
5692+ */
5693+static void io_add_napi(struct file *file, struct io_ring_ctx *ctx)
5694+{
5695+ unsigned int napi_id;
5696+ struct socket *sock;
5697+ struct sock *sk;
5698+ struct napi_entry *ne;
5699+
5700+ if (!net_busy_loop_on())
5701+ return;
5702+
5703+ sock = sock_from_file(file);
5704+ if (!sock)
5705+ return;
5706+
5707+ sk = sock->sk;
5708+ if (!sk)
5709+ return;
5710+
5711+ napi_id = READ_ONCE(sk->sk_napi_id);
5712+
5713+ /* Non-NAPI IDs can be rejected */
5714+ if (napi_id < MIN_NAPI_ID)
5715+ return;
5716+
5717+ spin_lock(&ctx->napi_lock);
5718+ list_for_each_entry(ne, &ctx->napi_list, list) {
5719+ if (ne->napi_id == napi_id) {
5720+ ne->timeout = jiffies + NAPI_TIMEOUT;
5721+ goto out;
5722+ }
5723+ }
5724+
5725+ ne = kmalloc(sizeof(*ne), GFP_NOWAIT);
5726+ if (!ne)
5727+ goto out;
5728+
5729+ ne->napi_id = napi_id;
5730+ ne->timeout = jiffies + NAPI_TIMEOUT;
5731+ list_add_tail(&ne->list, &ctx->napi_list);
5732+out:
5733+ spin_unlock(&ctx->napi_lock);
5734+}
5735+
5736+static inline void io_check_napi_entry_timeout(struct napi_entry *ne)
5737+{
5738+ if (time_after(jiffies, ne->timeout)) {
5739+ list_del(&ne->list);
5740+ kfree(ne);
5741+ }
5742+}
5743+
5744+/*
5745+ * Busy poll if globally on and supporting sockets found
5746+ */
5747+static bool io_napi_busy_loop(struct list_head *napi_list)
5748+{
5749+ struct napi_entry *ne, *n;
5750+
5751+ list_for_each_entry_safe(ne, n, napi_list, list) {
5752+ napi_busy_loop(ne->napi_id, NULL, NULL, true,
5753+ BUSY_POLL_BUDGET);
5754+ io_check_napi_entry_timeout(ne);
5755+ }
5756+ return !list_empty(napi_list);
5757+}
5758+
5759+static void io_free_napi_list(struct io_ring_ctx *ctx)
5760+{
5761+ spin_lock(&ctx->napi_lock);
5762+ while (!list_empty(&ctx->napi_list)) {
5763+ struct napi_entry *ne =
5764+ list_first_entry(&ctx->napi_list, struct napi_entry,
5765+ list);
5766+
5767+ list_del(&ne->list);
5768+ kfree(ne);
5769+ }
5770+ spin_unlock(&ctx->napi_lock);
5771+}
5772+#else
5773+static inline void io_add_napi(struct file *file, struct io_ring_ctx *ctx)
5774+{
5775+}
5776+
5777+static inline void io_free_napi_list(struct io_ring_ctx *ctx)
5778+{
5779+}
5780+#endif /* CONFIG_NET_RX_BUSY_POLL */
5781+
54025782 struct io_poll_table {
54035783 struct poll_table_struct pt;
54045784 struct io_kiocb *req;
@@ -5474,8 +5854,12 @@ static inline void io_poll_remove_entry(struct io_poll_iocb *poll) · 69 unmodified lines
54745854
54755855 static void io_poll_remove_entries(struct io_kiocb *req)
54765856 {
5477− struct io_poll_iocb *poll = io_poll_get_single(req);
5478− struct io_poll_iocb *poll_double = io_poll_get_double(req);
5857+ /*
5858+ * Nothing to do if neither of those flags are set. Avoid dipping
5859+ * into the poll/apoll/double cachelines if we can.
5860+ */
5861+ if (!(req->flags & (REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL)))
5862+ return;
54795863
54805864 /*
54815865 * While we hold the waitqueue lock and the waitqueue is nonempty,
@@ -5493,9 +5877,10 @@ static void io_poll_remove_entries(struct io_kiocb *req) · 11 unmodified lines
54935877 * In that case, only RCU prevents the queue memory from being freed.
54945878 */
54955879 rcu_read_lock();
5496− io_poll_remove_entry(poll);
5497− if (poll_double)
5498− io_poll_remove_entry(poll_double);
5880+ if (req->flags & REQ_F_SINGLE_POLL)
5881+ io_poll_remove_entry(io_poll_get_single(req));
5882+ if (req->flags & REQ_F_DOUBLE_POLL)
5883+ io_poll_remove_entry(io_poll_get_double(req));
54995884 rcu_read_unlock();
55005885 }
55015886
@@ -5527,13 +5912,13 @@ static int io_poll_check_events(struct io_kiocb *req) · 25 unmodified lines
55275912 return -ECANCELED;
55285913
55295914 if (!req->result) {
5530− struct poll_table_struct pt = { ._key = poll->events };
5915+ struct poll_table_struct pt = { ._key = req->cflags };
55315916
5532− req->result = vfs_poll(req->file, &pt) & poll->events;
5917+ req->result = vfs_poll(req->file, &pt) & req->cflags;
55335918 }
55345919
55355920 /* multishot, just fill an CQE and proceed */
5536− if (req->result && !(poll->events & EPOLLONESHOT)) {
5921+ if (req->result && !(req->cflags & EPOLLONESHOT)) {
55375922 __poll_t mask = mangle_poll(req->result & poll->events);
55385923 bool filled;
55395924
@@ -5545,6 +5930,7 @@ static int io_poll_check_events(struct io_kiocb *req) · 5 unmodified lines
55455930 if (unlikely(!filled))
55465931 return -ECANCELED;
55475932 io_cqring_ev_posted(ctx);
5933+ io_add_napi(req->file, ctx);
55485934 } else if (req->result) {
55495935 return 0;
55505936 }
@@ -5603,29 +5989,36 @@ static void io_apoll_task_func(struct io_kiocb *req, bool *locked) · 52 unmodified lines
56035989 io_req_complete_failed(req, ret);
56045990 }
56055991
5606−static void __io_poll_execute(struct io_kiocb *req, int mask)
5992+static void __io_poll_execute(struct io_kiocb *req, int mask, int events)
56075993 {
56085994 req->result = mask;
5995+ /*
5996+ * This is useful for poll that is armed on behalf of another
5997+ * request, and where the wakeup path could be on a different
5998+ * CPU. We want to avoid pulling in req->apoll->events for that
5999+ * case.
6000+ */
6001+ req->cflags = events;
56096002 if (req->opcode == IORING_OP_POLL_ADD)
56106003 req->io_task_work.func = io_poll_task_func;
56116004 else
56126005 req->io_task_work.func = io_apoll_task_func;
56136006
5614− trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);
6007+ trace_io_uring_task_add(req->ctx, req, req->user_data, req->opcode, mask);
56156008 io_req_task_work_add(req, false);
56166009 }
56176010
5618−static inline void io_poll_execute(struct io_kiocb *req, int res)
6011+static inline void io_poll_execute(struct io_kiocb *req, int res, int events)
56196012 {
56206013 if (io_poll_get_ownership(req))
5621− __io_poll_execute(req, res);
6014+ __io_poll_execute(req, res, events);
56226015 }
56236016
56246017 static void io_poll_cancel_req(struct io_kiocb *req)
56256018 {
56266019 io_poll_mark_cancelled(req);
56276020 /* kick tw, which should complete the request */
5628− io_poll_execute(req, 0);
6021+ io_poll_execute(req, 0, 0);
56296022 }
56306023
56316024 static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
@@ -5639,7 +6032,7 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, · 7 unmodified lines
56396032 if (unlikely(mask & POLLFREE)) {
56406033 io_poll_mark_cancelled(req);
56416034 /* we have to kick tw in case it's not already */
5642− io_poll_execute(req, 0);
6035+ io_poll_execute(req, 0, poll->events);
56436036
56446037 /*
56456038 * If the waitqueue is being freed early but someone is already
@@ -5669,8 +6062,9 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, · 23 unmodified lines
56696062 if (mask && poll->events & EPOLLONESHOT) {
56706063 list_del_init(&poll->wait.entry);
56716064 poll->head = NULL;
6065+ req->flags &= ~REQ_F_SINGLE_POLL;
56726066 }
5673− __io_poll_execute(req, mask);
6067+ __io_poll_execute(req, mask, poll->events);
56746068 }
56756069 return 1;
56766070 }
@@ -5705,12 +6099,14 @@ static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt, · 28 unmodified lines
57056099 pt->error = -ENOMEM;
57066100 return;
57076101 }
6102+ req->flags |= REQ_F_DOUBLE_POLL;
57086103 io_init_poll_iocb(poll, first->events, first->wait.func);
57096104 *poll_ptr = poll;
57106105 if (req->opcode == IORING_OP_POLL_ADD)
57116106 req->flags |= REQ_F_ASYNC_DATA;
57126107 }
57136108
6109+ req->flags |= REQ_F_SINGLE_POLL;
57146110 pt->nr_entries++;
57156111 poll->head = head;
57166112 poll->wait.private = req;
@@ -5774,9 +6170,10 @@ static int __io_arm_poll_handler(struct io_kiocb *req, · 57 unmodified lines
57746170 /* can't multishot if failed, just queue the event we've got */
57756171 if (unlikely(ipt->error || !ipt->nr_entries))
57766172 poll->events |= EPOLLONESHOT;
5777− __io_poll_execute(req, mask);
6173+ __io_poll_execute(req, mask, poll->events);
57786174 return 0;
57796175 }
6176+ io_add_napi(req->file, req->ctx);
57806177
57816178 /*
57826179 * Release ownership. If someone tried to queue a tw while it was
@@ -5784,7 +6181,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, · 1 unmodified line
57846181 */
57856182 v = atomic_dec_return(&req->poll_refs);
57866183 if (unlikely(v & IO_POLL_REF_MASK))
5787− __io_poll_execute(req, 0);
6184+ __io_poll_execute(req, 0, poll->events);
57886185 return 0;
57896186 }
57906187
@@ -5803,7 +6200,7 @@ enum { · 12 unmodified lines
58036200 IO_APOLL_READY
58046201 };
58056202
5806−static int io_arm_poll_handler(struct io_kiocb *req)
6203+static int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags)
58076204 {
58086205 const struct io_op_def *def = &io_op_defs[req->opcode];
58096206 struct io_ring_ctx *ctx = req->ctx;
@@ -5828,9 +6225,16 @@ static int io_arm_poll_handler(struct io_kiocb *req) · 18 unmodified lines
58286225 mask |= POLLOUT | POLLWRNORM;
58296226 }
58306227
5831− apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
5832− if (unlikely(!apoll))
5833− return IO_APOLL_ABORTED;
6228+ if (!(issue_flags & IO_URING_F_UNLOCKED) &&
6229+ !list_empty(&ctx->apoll_cache)) {
6230+ apoll = list_first_entry(&ctx->apoll_cache, struct async_poll,
6231+ poll.wait.entry);
6232+ list_del_init(&apoll->poll.wait.entry);
6233+ } else {
6234+ apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
6235+ if (unlikely(!apoll))
6236+ return IO_APOLL_ABORTED;
6237+ }
58346238 apoll->double_poll = NULL;
58356239 req->apoll = apoll;
58366240 req->flags |= REQ_F_POLLED;
@@ -5840,7 +6244,7 @@ static int io_arm_poll_handler(struct io_kiocb *req) · 3 unmodified lines
58406244 if (ret || ipt.error)
58416245 return ret ? IO_APOLL_READY : IO_APOLL_ABORTED;
58426246
5843− trace_io_uring_poll_arm(ctx, req, req->opcode, req->user_data,
6247+ trace_io_uring_poll_arm(ctx, req, req->user_data, req->opcode,
58446248 mask, apoll->poll.events);
58456249 return IO_APOLL_OK;
58466250 }
@@ -5975,7 +6379,7 @@ static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe · 128 unmodified lines
59756379 return -EINVAL;
59766380
59776381 io_req_set_refcount(req);
5978− poll->events = io_poll_parse_events(sqe, flags);
6382+ req->cflags = poll->events = io_poll_parse_events(sqe, flags);
59796383 return 0;
59806384 }
59816385
@@ -6092,10 +6496,7 @@ static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data) · 110 unmodified lines
60926496
60936497 if (IS_ERR(req))
60946498 return PTR_ERR(req);
6095−
6096− req_set_fail(req);
6097− io_fill_cqe_req(req, -ECANCELED, 0);
6098− io_put_req_deferred(req);
6499+ io_req_task_queue_fail(req, -ECANCELED);
60996500 return 0;
61006501 }
61016502
@@ -6568,6 +6969,8 @@ static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 466 unmodified lines
65686969 return io_symlinkat_prep(req, sqe);
65696970 case IORING_OP_LINKAT:
65706971 return io_linkat_prep(req, sqe);
6972+ case IORING_OP_MSG_RING:
6973+ return io_msg_ring_prep(req, sqe);
65716974 }
65726975
65736976 printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
@@ -6649,7 +7052,7 @@ fail: · 75 unmodified lines
66497052 goto queue;
66507053 }
66517054
6652− trace_io_uring_defer(ctx, req, req->user_data);
7055+ trace_io_uring_defer(ctx, req, req->user_data, req->opcode);
66537056 de->req = req;
66547057 de->seq = seq;
66557058 list_add_tail(&de->list, &ctx->defer_list);
@@ -6659,7 +7062,7 @@ fail: · 3 unmodified lines
66597062 static void io_clean_op(struct io_kiocb *req)
66607063 {
66617064 if (req->flags & REQ_F_BUFFER_SELECTED)
6662− io_put_kbuf(req);
7065+ io_put_kbuf_comp(req);
66637066
66647067 if (req->flags & REQ_F_NEED_CLEANUP) {
66657068 switch (req->opcode) {
@@ -6851,6 +7254,9 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) · 185 unmodified lines
68517254 case IORING_OP_LINKAT:
68527255 ret = io_linkat(req, issue_flags);
68537256 break;
7257+ case IORING_OP_MSG_RING:
7258+ ret = io_msg_ring(req, issue_flags);
7259+ break;
68547260 default:
68557261 ret = -EINVAL;
68567262 break;
@@ -6926,7 +7332,7 @@ static void io_wq_submit_work(struct io_wq_work *work) · 69 unmodified lines
69267332 continue;
69277333 }
69287334
6929− if (io_arm_poll_handler(req) == IO_APOLL_OK)
7335+ if (io_arm_poll_handler(req, issue_flags) == IO_APOLL_OK)
69307336 return;
69317337 /* aborted or ready, in either case retry blocking */
69327338 needs_poll = false;
@@ -6983,7 +7389,7 @@ static struct file *io_file_get_normal(struct io_ring_ctx *ctx, · 50 unmodified lines
69837389 {
69847390 struct file *file = fget(fd);
69857391
6986− trace_io_uring_file_get(ctx, fd);
7392+ trace_io_uring_file_get(ctx, req, req->user_data, fd);
69877393
69887394 /* we don't allow fixed io_uring files */
69897395 if (file && unlikely(file->f_op == &io_uring_fops))
@@ -7072,7 +7478,7 @@ static void io_queue_sqe_arm_apoll(struct io_kiocb *req) · 82 unmodified lines
70727478 {
70737479 struct io_kiocb *linked_timeout = io_prep_linked_timeout(req);
70747480
7075− switch (io_arm_poll_handler(req)) {
7481+ switch (io_arm_poll_handler(req, 0)) {
70767482 case IO_APOLL_READY:
70777483 io_req_task_queue(req);
70787484 break;
@@ -7081,8 +7487,12 @@ static void io_queue_sqe_arm_apoll(struct io_kiocb *req) · 2 unmodified lines
70817487 * Queued up for async execution, worker will release
70827488 * submit reference when the iocb is actually submitted.
70837489 */
7490+ io_kbuf_recycle(req);
70847491 io_queue_async_work(req, NULL);
70857492 break;
7493+ case IO_APOLL_OK:
7494+ io_kbuf_recycle(req);
7495+ break;
70867496 }
70877497
70887498 if (linked_timeout)
@@ -7281,7 +7691,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, · 192 unmodified lines
72817691
72827692 ret = io_init_req(ctx, req, sqe);
72837693 if (unlikely(ret)) {
7284− trace_io_uring_req_failed(sqe, ret);
7694+ trace_io_uring_req_failed(sqe, ctx, req, ret);
72857695
72867696 /* fail even hard links since we don't submit */
72877697 if (link->head) {
@@ -7308,7 +7718,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, · 20 unmodified lines
73087718 }
73097719
73107720 /* don't need @sqe from now on */
7311− trace_io_uring_submit_sqe(ctx, req, req->opcode, req->user_data,
7721+ trace_io_uring_submit_sqe(ctx, req, req->user_data, req->opcode,
73127722 req->flags, true,
73137723 ctx->flags & IORING_SETUP_SQPOLL);
73147724
@@ -7451,8 +7861,14 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) · 136 unmodified lines
74517861 }
74527862 /* will complete beyond this point, count as submitted */
74537863 submitted++;
7454− if (io_submit_sqe(ctx, req, sqe))
7455− break;
7864+ if (io_submit_sqe(ctx, req, sqe)) {
7865+ /*
7866+ * Continue submitting even for sqe failure if the
7867+ * ring was setup with IORING_SETUP_SUBMIT_ALL
7868+ */
7869+ if (!(ctx->flags & IORING_SETUP_SUBMIT_ALL))
7870+ break;
7871+ }
74567872 } while (submitted < nr);
74577873
74587874 if (unlikely(submitted != nr)) {
@@ -7519,7 +7935,13 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) · 60 unmodified lines
75197935 !(ctx->flags & IORING_SETUP_R_DISABLED))
75207936 ret = io_submit_sqes(ctx, to_submit);
75217937 mutex_unlock(&ctx->uring_lock);
7522−
7938+#ifdef CONFIG_NET_RX_BUSY_POLL
7939+ spin_lock(&ctx->napi_lock);
7940+ if (!list_empty(&ctx->napi_list) &&
7941+ io_napi_busy_loop(&ctx->napi_list))
7942+ ++ret;
7943+ spin_unlock(&ctx->napi_lock);
7944+#endif
75237945 if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait))
75247946 wake_up(&ctx->sqo_sq_wait);
75257947 if (creds)
@@ -7650,6 +8072,9 @@ struct io_wait_queue { · 124 unmodified lines
76508072 struct io_ring_ctx *ctx;
76518073 unsigned cq_tail;
76528074 unsigned nr_timeouts;
8075+#ifdef CONFIG_NET_RX_BUSY_POLL
8076+ unsigned busy_poll_to;
8077+#endif
76538078 };
76548079
76558080 static inline bool io_should_wake(struct io_wait_queue *iowq)
@@ -7684,11 +8109,11 @@ static int io_run_task_work_sig(void) · 28 unmodified lines
76848109 {
76858110 if (io_run_task_work())
76868111 return 1;
7687− if (!signal_pending(current))
7688− return 0;
76898112 if (test_thread_flag(TIF_NOTIFY_SIGNAL))
76908113 return -ERESTARTSYS;
7691− return -EINTR;
8114+ if (task_sigpending(current))
8115+ return -EINTR;
8116+ return 0;
76928117 }
76938118
76948119 /* when returns >0, the caller should retry */
@@ -7711,6 +8136,87 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, · 16 unmodified lines
77118136 return 1;
77128137 }
77138138
8139+#ifdef CONFIG_NET_RX_BUSY_POLL
8140+static void io_adjust_busy_loop_timeout(struct timespec64 *ts,
8141+ struct io_wait_queue *iowq)
8142+{
8143+ unsigned busy_poll_to = READ_ONCE(sysctl_net_busy_poll);
8144+ struct timespec64 pollto = ns_to_timespec64(1000 * (s64)busy_poll_to);
8145+
8146+ if (timespec64_compare(ts, &pollto) > 0) {
8147+ *ts = timespec64_sub(*ts, pollto);
8148+ iowq->busy_poll_to = busy_poll_to;
8149+ } else {
8150+ u64 to = timespec64_to_ns(ts);
8151+
8152+ do_div(to, 1000);
8153+ iowq->busy_poll_to = to;
8154+ ts->tv_sec = 0;
8155+ ts->tv_nsec = 0;
8156+ }
8157+}
8158+
8159+static inline bool io_busy_loop_timeout(unsigned long start_time,
8160+ unsigned long bp_usec)
8161+{
8162+ if (bp_usec) {
8163+ unsigned long end_time = start_time + bp_usec;
8164+ unsigned long now = busy_loop_current_time();
8165+
8166+ return time_after(now, end_time);
8167+ }
8168+ return true;
8169+}
8170+
8171+static bool io_busy_loop_end(void *p, unsigned long start_time)
8172+{
8173+ struct io_wait_queue *iowq = p;
8174+
8175+ return signal_pending(current) ||
8176+ io_should_wake(iowq) ||
8177+ io_busy_loop_timeout(start_time, iowq->busy_poll_to);
8178+}
8179+
8180+static void io_blocking_napi_busy_loop(struct list_head *napi_list,
8181+ struct io_wait_queue *iowq)
8182+{
8183+ unsigned long start_time =
8184+ list_is_singular(napi_list) ? 0 :
8185+ busy_loop_current_time();
8186+
8187+ do {
8188+ if (list_is_singular(napi_list)) {
8189+ struct napi_entry *ne =
8190+ list_first_entry(napi_list,
8191+ struct napi_entry, list);
8192+
8193+ napi_busy_loop(ne->napi_id, io_busy_loop_end, iowq,
8194+ true, BUSY_POLL_BUDGET);
8195+ io_check_napi_entry_timeout(ne);
8196+ break;
8197+ }
8198+ } while (io_napi_busy_loop(napi_list) &&
8199+ !io_busy_loop_end(iowq, start_time));
8200+}
8201+
8202+static void io_putback_napi_list(struct io_ring_ctx *ctx,
8203+ struct list_head *napi_list)
8204+{
8205+ struct napi_entry *cne, *lne;
8206+
8207+ spin_lock(&ctx->napi_lock);
8208+ list_for_each_entry(cne, &ctx->napi_list, list)
8209+ list_for_each_entry(lne, napi_list, list)
8210+ if (cne->napi_id == lne->napi_id) {
8211+ list_del(&lne->list);
8212+ kfree(lne);
8213+ break;
8214+ }
8215+ list_splice(napi_list, &ctx->napi_list);
8216+ spin_unlock(&ctx->napi_lock);
8217+}
8218+#endif /* CONFIG_NET_RX_BUSY_POLL */
8219+
77148220 /*
77158221 * Wait until events become available, if we don't already have some. The
77168222 * application must reap them itself, as they reside on the shared cq ring.
@@ -7723,6 +8229,9 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 6 unmodified lines
77238229 struct io_rings *rings = ctx->rings;
77248230 ktime_t timeout = KTIME_MAX;
77258231 int ret;
8232+#ifdef CONFIG_NET_RX_BUSY_POLL
8233+ LIST_HEAD(local_napi_list);
8234+#endif
77268235
77278236 do {
77288237 io_cqring_overflow_flush(ctx);
@@ -7732,14 +8241,6 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 3 unmodified lines
77328241 break;
77338242 } while (1);
77348243
7735− if (uts) {
7736− struct timespec64 ts;
7737−
7738− if (get_timespec64(&ts, uts))
7739− return -EFAULT;
7740− timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns());
7741− }
7742−
77438244 if (sig) {
77448245 #ifdef CONFIG_COMPAT
77458246 if (in_compat_syscall())
@@ -7753,6 +8254,30 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 7 unmodified lines
77538254 return ret;
77548255 }
77558256
8257+#ifdef CONFIG_NET_RX_BUSY_POLL
8258+ iowq.busy_poll_to = 0;
8259+ if (!(ctx->flags & IORING_SETUP_SQPOLL)) {
8260+ spin_lock(&ctx->napi_lock);
8261+ list_splice_init(&ctx->napi_list, &local_napi_list);
8262+ spin_unlock(&ctx->napi_lock);
8263+ }
8264+#endif
8265+ if (uts) {
8266+ struct timespec64 ts;
8267+
8268+ if (get_timespec64(&ts, uts))
8269+ return -EFAULT;
8270+#ifdef CONFIG_NET_RX_BUSY_POLL
8271+ if (!list_empty(&local_napi_list))
8272+ io_adjust_busy_loop_timeout(&ts, &iowq);
8273+#endif
8274+ timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns());
8275+ }
8276+#ifdef CONFIG_NET_RX_BUSY_POLL
8277+ else if (!list_empty(&local_napi_list))
8278+ iowq.busy_poll_to = READ_ONCE(sysctl_net_busy_poll);
8279+#endif
8280+
77568281 init_waitqueue_func_entry(&iowq.wq, io_wake_function);
77578282 iowq.wq.private = current;
77588283 INIT_LIST_HEAD(&iowq.wq.entry);
@@ -7761,6 +8286,12 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 2 unmodified lines
77618286 iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events;
77628287
77638288 trace_io_uring_cqring_wait(ctx, min_events);
8289+#ifdef CONFIG_NET_RX_BUSY_POLL
8290+ if (iowq.busy_poll_to)
8291+ io_blocking_napi_busy_loop(&local_napi_list, &iowq);
8292+ if (!list_empty(&local_napi_list))
8293+ io_putback_napi_list(ctx, &local_napi_list);
8294+#endif
77648295 do {
77658296 /* if we can't even flush overflow, don't wait for more */
77668297 if (!io_cqring_overflow_flush(ctx)) {
@@ -8749,8 +9280,16 @@ static __cold int io_uring_alloc_task_context(struct task_struct *task, · 982 unmodified lines
87499280 if (unlikely(!tctx))
87509281 return -ENOMEM;
87519282
9283+ tctx->registered_rings = kcalloc(IO_RINGFD_REG_MAX,
9284+ sizeof(struct file *), GFP_KERNEL);
9285+ if (unlikely(!tctx->registered_rings)) {
9286+ kfree(tctx);
9287+ return -ENOMEM;
9288+ }
9289+
87529290 ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL);
87539291 if (unlikely(ret)) {
9292+ kfree(tctx->registered_rings);
87549293 kfree(tctx);
87559294 return ret;
87569295 }
@@ -8759,6 +9298,7 @@ static __cold int io_uring_alloc_task_context(struct task_struct *task, · 2 unmodified lines
87599298 if (IS_ERR(tctx->io_wq)) {
87609299 ret = PTR_ERR(tctx->io_wq);
87619300 percpu_counter_destroy(&tctx->inflight);
9301+ kfree(tctx->registered_rings);
87629302 kfree(tctx);
87639303 return ret;
87649304 }
@@ -8783,6 +9323,7 @@ void __io_uring_free(struct task_struct *tsk) · 18 unmodified lines
87839323 WARN_ON_ONCE(tctx->io_wq);
87849324 WARN_ON_ONCE(tctx->cached_refs);
87859325
9326+ kfree(tctx->registered_rings);
87869327 percpu_counter_destroy(&tctx->inflight);
87879328 kfree(tctx);
87889329 tsk->io_uring = NULL;
@@ -9359,33 +9900,55 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, · 570 unmodified lines
93599900 return done ? done : err;
93609901 }
93619902
9362−static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
9903+static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg,
9904+ unsigned int eventfd_async)
93639905 {
9906+ struct io_ev_fd *ev_fd;
93649907 __s32 __user *fds = arg;
93659908 int fd;
93669909
9367− if (ctx->cq_ev_fd)
9910+ ev_fd = rcu_dereference_protected(ctx->io_ev_fd,
9911+ lockdep_is_held(&ctx->uring_lock));
9912+ if (ev_fd)
93689913 return -EBUSY;
93699914
93709915 if (copy_from_user(&fd, fds, sizeof(*fds)))
93719916 return -EFAULT;
93729917
9373− ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
9374− if (IS_ERR(ctx->cq_ev_fd)) {
9375− int ret = PTR_ERR(ctx->cq_ev_fd);
9918+ ev_fd = kmalloc(sizeof(*ev_fd), GFP_KERNEL);
9919+ if (!ev_fd)
9920+ return -ENOMEM;
93769921
9377− ctx->cq_ev_fd = NULL;
9922+ ev_fd->cq_ev_fd = eventfd_ctx_fdget(fd);
9923+ if (IS_ERR(ev_fd->cq_ev_fd)) {
9924+ int ret = PTR_ERR(ev_fd->cq_ev_fd);
9925+ kfree(ev_fd);
93789926 return ret;
93799927 }
9380−
9928+ ev_fd->eventfd_async = eventfd_async;
9929+ ctx->has_evfd = true;
9930+ rcu_assign_pointer(ctx->io_ev_fd, ev_fd);
93819931 return 0;
93829932 }
93839933
9934+static void io_eventfd_put(struct rcu_head *rcu)
9935+{
9936+ struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);
9937+
9938+ eventfd_ctx_put(ev_fd->cq_ev_fd);
9939+ kfree(ev_fd);
9940+}
9941+
93849942 static int io_eventfd_unregister(struct io_ring_ctx *ctx)
93859943 {
9386− if (ctx->cq_ev_fd) {
9387− eventfd_ctx_put(ctx->cq_ev_fd);
9388− ctx->cq_ev_fd = NULL;
9944+ struct io_ev_fd *ev_fd;
9945+
9946+ ev_fd = rcu_dereference_protected(ctx->io_ev_fd,
9947+ lockdep_is_held(&ctx->uring_lock));
9948+ if (ev_fd) {
9949+ ctx->has_evfd = false;
9950+ rcu_assign_pointer(ctx->io_ev_fd, NULL);
9951+ call_rcu(&ev_fd->rcu, io_eventfd_put);
93899952 return 0;
93909953 }
93919954
@@ -9394,11 +9957,28 @@ static int io_eventfd_unregister(struct io_ring_ctx *ctx) · 2 unmodified lines
93949957
93959958 static void io_destroy_buffers(struct io_ring_ctx *ctx)
93969959 {
9397− struct io_buffer *buf;
9398− unsigned long index;
9960+ int i;
9961+
9962+ for (i = 0; i < (1U << IO_BUFFERS_HASH_BITS); i++) {
9963+ struct list_head *list = &ctx->io_buffers[i];
9964+
9965+ while (!list_empty(list)) {
9966+ struct io_buffer_list *bl;
9967+
9968+ bl = list_first_entry(list, struct io_buffer_list, list);
9969+ __io_remove_buffers(ctx, bl, -1U);
9970+ list_del(&bl->list);
9971+ kfree(bl);
9972+ }
9973+ }
93999974
9400− xa_for_each(&ctx->io_buffers, index, buf)
9401− __io_remove_buffers(ctx, buf, index, -1U);
9975+ while (!list_empty(&ctx->io_buffers_pages)) {
9976+ struct page *page;
9977+
9978+ page = list_first_entry(&ctx->io_buffers_pages, struct page, lru);
9979+ list_del_init(&page->lru);
9980+ __free_page(page);
9981+ }
94029982 }
94039983
94049984 static void io_req_caches_free(struct io_ring_ctx *ctx)
@@ -9429,6 +10009,18 @@ static void io_wait_rsrc_data(struct io_rsrc_data *data) · 24 unmodified lines
942910009 wait_for_completion(&data->done);
943010010 }
943110011
10012+static void io_flush_apoll_cache(struct io_ring_ctx *ctx)
10013+{
10014+ struct async_poll *apoll;
10015+
10016+ while (!list_empty(&ctx->apoll_cache)) {
10017+ apoll = list_first_entry(&ctx->apoll_cache, struct async_poll,
10018+ poll.wait.entry);
10019+ list_del(&apoll->poll.wait.entry);
10020+ kfree(apoll);
10021+ }
10022+}
10023+
943210024 static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
943310025 {
943410026 io_sq_thread_finish(ctx);
@@ -9450,8 +10042,9 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) · 15 unmodified lines
945010042 __io_sqe_files_unregister(ctx);
945110043 if (ctx->rings)
945210044 __io_cqring_overflow_flush(ctx, true);
9453− mutex_unlock(&ctx->uring_lock);
945410045 io_eventfd_unregister(ctx);
10046+ io_flush_apoll_cache(ctx);
10047+ mutex_unlock(&ctx->uring_lock);
945510048 io_destroy_buffers(ctx);
945610049 if (ctx->sq_creds)
945710050 put_cred(ctx->sq_creds);
@@ -9483,8 +10076,10 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) · 25 unmodified lines
948310076 io_req_caches_free(ctx);
948410077 if (ctx->hash_map)
948510078 io_wq_put_hash(ctx->hash_map);
10079+ io_free_napi_list(ctx);
948610080 kfree(ctx->cancel_hash);
948710081 kfree(ctx->dummy_ubuf);
10082+ kfree(ctx->io_buffers);
948810083 kfree(ctx);
948910084 }
949010085
@@ -9983,6 +10578,139 @@ void __io_uring_cancel(bool cancel_all) · 492 unmodified lines
998310578 io_uring_cancel_generic(cancel_all, NULL);
998410579 }
998510580
10581+void io_uring_unreg_ringfd(void)
10582+{
10583+ struct io_uring_task *tctx = current->io_uring;
10584+ int i;
10585+
10586+ for (i = 0; i < IO_RINGFD_REG_MAX; i++) {
10587+ if (tctx->registered_rings[i]) {
10588+ fput(tctx->registered_rings[i]);
10589+ tctx->registered_rings[i] = NULL;
10590+ }
10591+ }
10592+}
10593+
10594+static int io_ring_add_registered_fd(struct io_uring_task *tctx, int fd,
10595+ int start, int end)
10596+{
10597+ struct file *file;
10598+ int offset;
10599+
10600+ for (offset = start; offset < end; offset++) {
10601+ offset = array_index_nospec(offset, IO_RINGFD_REG_MAX);
10602+ if (tctx->registered_rings[offset])
10603+ continue;
10604+
10605+ file = fget(fd);
10606+ if (!file) {
10607+ return -EBADF;
10608+ } else if (file->f_op != &io_uring_fops) {
10609+ fput(file);
10610+ return -EOPNOTSUPP;
10611+ }
10612+ tctx->registered_rings[offset] = file;
10613+ return offset;
10614+ }
10615+
10616+ return -EBUSY;
10617+}
10618+
10619+/*
10620+ * Register a ring fd to avoid fdget/fdput for each io_uring_enter()
10621+ * invocation. User passes in an array of struct io_uring_rsrc_update
10622+ * with ->data set to the ring_fd, and ->offset given for the desired
10623+ * index. If no index is desired, application may set ->offset == -1U
10624+ * and we'll find an available index. Returns number of entries
10625+ * successfully processed, or < 0 on error if none were processed.
10626+ */
10627+static int io_ringfd_register(struct io_ring_ctx *ctx, void __user *__arg,
10628+ unsigned nr_args)
10629+{
10630+ struct io_uring_rsrc_update __user *arg = __arg;
10631+ struct io_uring_rsrc_update reg;
10632+ struct io_uring_task *tctx;
10633+ int ret, i;
10634+
10635+ if (!nr_args || nr_args > IO_RINGFD_REG_MAX)
10636+ return -EINVAL;
10637+
10638+ mutex_unlock(&ctx->uring_lock);
10639+ ret = io_uring_add_tctx_node(ctx);
10640+ mutex_lock(&ctx->uring_lock);
10641+ if (ret)
10642+ return ret;
10643+
10644+ tctx = current->io_uring;
10645+ for (i = 0; i < nr_args; i++) {
10646+ int start, end;
10647+
10648+ if (copy_from_user(&reg, &arg[i], sizeof(reg))) {
10649+ ret = -EFAULT;
10650+ break;
10651+ }
10652+
10653+ if (reg.offset == -1U) {
10654+ start = 0;
10655+ end = IO_RINGFD_REG_MAX;
10656+ } else {
10657+ if (reg.offset >= IO_RINGFD_REG_MAX) {
10658+ ret = -EINVAL;
10659+ break;
10660+ }
10661+ start = reg.offset;
10662+ end = start + 1;
10663+ }
10664+
10665+ ret = io_ring_add_registered_fd(tctx, reg.data, start, end);
10666+ if (ret < 0)
10667+ break;
10668+
10669+ reg.offset = ret;
10670+ if (copy_to_user(&arg[i], &reg, sizeof(reg))) {
10671+ fput(tctx->registered_rings[reg.offset]);
10672+ tctx->registered_rings[reg.offset] = NULL;
10673+ ret = -EFAULT;
10674+ break;
10675+ }
10676+ }
10677+
10678+ return i ? i : ret;
10679+}
10680+
10681+static int io_ringfd_unregister(struct io_ring_ctx *ctx, void __user *__arg,
10682+ unsigned nr_args)
10683+{
10684+ struct io_uring_rsrc_update __user *arg = __arg;
10685+ struct io_uring_task *tctx = current->io_uring;
10686+ struct io_uring_rsrc_update reg;
10687+ int ret = 0, i;
10688+
10689+ if (!nr_args || nr_args > IO_RINGFD_REG_MAX)
10690+ return -EINVAL;
10691+ if (!tctx)
10692+ return 0;
10693+
10694+ for (i = 0; i < nr_args; i++) {
10695+ if (copy_from_user(&reg, &arg[i], sizeof(reg))) {
10696+ ret = -EFAULT;
10697+ break;
10698+ }
10699+ if (reg.offset >= IO_RINGFD_REG_MAX) {
10700+ ret = -EINVAL;
10701+ break;
10702+ }
10703+
10704+ reg.offset = array_index_nospec(reg.offset, IO_RINGFD_REG_MAX);
10705+ if (tctx->registered_rings[reg.offset]) {
10706+ fput(tctx->registered_rings[reg.offset]);
10707+ tctx->registered_rings[reg.offset] = NULL;
10708+ }
10709+ }
10710+
10711+ return i ? i : ret;
10712+}
10713+
998610714 static void *io_uring_validate_mmap_request(struct file *file,
998710715 loff_t pgoff, size_t sz)
998810716 {
@@ -10113,12 +10841,28 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 124 unmodified lines
1011310841 io_run_task_work();
1011410842
1011510843 if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
10116− IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG)))
10844+ IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG |
10845+ IORING_ENTER_REGISTERED_RING)))
1011710846 return -EINVAL;
1011810847
10119− f = fdget(fd);
10120− if (unlikely(!f.file))
10121− return -EBADF;
10848+ /*
10849+ * Ring fd has been registered via IORING_REGISTER_RING_FDS, we
10850+ * need only dereference our task private array to find it.
10851+ */
10852+ if (flags & IORING_ENTER_REGISTERED_RING) {
10853+ struct io_uring_task *tctx = current->io_uring;
10854+
10855+ if (!tctx || fd >= IO_RINGFD_REG_MAX)
10856+ return -EINVAL;
10857+ fd = array_index_nospec(fd, IO_RINGFD_REG_MAX);
10858+ f.file = tctx->registered_rings[fd];
10859+ if (unlikely(!f.file))
10860+ return -EBADF;
10861+ } else {
10862+ f = fdget(fd);
10863+ if (unlikely(!f.file))
10864+ return -EBADF;
10865+ }
1012210866
1012310867 ret = -EOPNOTSUPP;
1012410868 if (unlikely(f.file->f_op != &io_uring_fops))
@@ -10192,7 +10936,8 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 67 unmodified lines
1019210936 out:
1019310937 percpu_ref_put(&ctx->refs);
1019410938 out_fput:
10195− fdput(f);
10939+ if (!(flags & IORING_ENTER_REGISTERED_RING))
10940+ fdput(f);
1019610941 return submitted ? submitted : ret;
1019710942 }
1019810943
@@ -10610,7 +11355,7 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) · 411 unmodified lines
1061011355 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
1061111356 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
1061211357 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
10613− IORING_SETUP_R_DISABLED))
11358+ IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL))
1061411359 return -EINVAL;
1061511360
1061611361 return io_uring_create(entries, &p, params);
@@ -10960,61 +11705,6 @@ err: · 343 unmodified lines
1096011705 return ret;
1096111706 }
1096211707
10963−static bool io_register_op_must_quiesce(int op)
10964−{
10965− switch (op) {
10966− case IORING_REGISTER_BUFFERS:
10967− case IORING_UNREGISTER_BUFFERS:
10968− case IORING_REGISTER_FILES:
10969− case IORING_UNREGISTER_FILES:
10970− case IORING_REGISTER_FILES_UPDATE:
10971− case IORING_REGISTER_PROBE:
10972− case IORING_REGISTER_PERSONALITY:
10973− case IORING_UNREGISTER_PERSONALITY:
10974− case IORING_REGISTER_FILES2:
10975− case IORING_REGISTER_FILES_UPDATE2:
10976− case IORING_REGISTER_BUFFERS2:
10977− case IORING_REGISTER_BUFFERS_UPDATE:
10978− case IORING_REGISTER_IOWQ_AFF:
10979− case IORING_UNREGISTER_IOWQ_AFF:
10980− case IORING_REGISTER_IOWQ_MAX_WORKERS:
10981− return false;
10982− default:
10983− return true;
10984− }
10985−}
10986−
10987−static __cold int io_ctx_quiesce(struct io_ring_ctx *ctx)
10988−{
10989− long ret;
10990−
10991− percpu_ref_kill(&ctx->refs);
10992−
10993− /*
10994− * Drop uring mutex before waiting for references to exit. If another
10995− * thread is currently inside io_uring_enter() it might need to grab the
10996− * uring_lock to make progress. If we hold it here across the drain
10997− * wait, then we can deadlock. It's safe to drop the mutex here, since
10998− * no new references will come in after we've killed the percpu ref.
10999− */
11000− mutex_unlock(&ctx->uring_lock);
11001− do {
11002− ret = wait_for_completion_interruptible_timeout(&ctx->ref_comp, HZ);
11003− if (ret) {
11004− ret = min(0L, ret);
11005− break;
11006− }
11007−
11008− ret = io_run_task_work_sig();
11009− io_req_caches_free(ctx);
11010− } while (ret >= 0);
11011− mutex_lock(&ctx->uring_lock);
11012−
11013− if (ret)
11014− io_refs_resurrect(&ctx->refs, &ctx->ref_comp);
11015− return ret;
11016−}
11017−
1101811708 static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
1101911709 void __user *arg, unsigned nr_args)
1102011710 __releases(ctx->uring_lock)
@@ -11038,12 +11728,6 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, · 17 unmodified lines
1103811728 return -EACCES;
1103911729 }
1104011730
11041− if (io_register_op_must_quiesce(opcode)) {
11042− ret = io_ctx_quiesce(ctx);
11043− if (ret)
11044− return ret;
11045− }
11046−
1104711731 switch (opcode) {
1104811732 case IORING_REGISTER_BUFFERS:
1104911733 ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL);
@@ -11067,17 +11751,16 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, · 17 unmodified lines
1106711751 ret = io_register_files_update(ctx, arg, nr_args);
1106811752 break;
1106911753 case IORING_REGISTER_EVENTFD:
11070− case IORING_REGISTER_EVENTFD_ASYNC:
1107111754 ret = -EINVAL;
1107211755 if (nr_args != 1)
1107311756 break;
11074− ret = io_eventfd_register(ctx, arg);
11075− if (ret)
11757+ ret = io_eventfd_register(ctx, arg, 0);
11758+ break;
11759+ case IORING_REGISTER_EVENTFD_ASYNC:
11760+ ret = -EINVAL;
11761+ if (nr_args != 1)
1107611762 break;
11077− if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
11078− ctx->eventfd_async = 1;
11079− else
11080− ctx->eventfd_async = 0;
11763+ ret = io_eventfd_register(ctx, arg, 1);
1108111764 break;
1108211765 case IORING_UNREGISTER_EVENTFD:
1108311766 ret = -EINVAL;
@@ -11144,16 +11827,17 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, · 60 unmodified lines
1114411827 break;
1114511828 ret = io_register_iowq_max_workers(ctx, arg);
1114611829 break;
11830+ case IORING_REGISTER_RING_FDS:
11831+ ret = io_ringfd_register(ctx, arg, nr_args);
11832+ break;
11833+ case IORING_UNREGISTER_RING_FDS:
11834+ ret = io_ringfd_unregister(ctx, arg, nr_args);
11835+ break;
1114711836 default:
1114811837 ret = -EINVAL;
1114911838 break;
1115011839 }
1115111840
11152− if (io_register_op_must_quiesce(opcode)) {
11153− /* bring the ctx back to life */
11154− percpu_ref_reinit(&ctx->refs);
11155− reinit_completion(&ctx->ref_comp);
11156− }
1115711841 return ret;
1115811842 }
1115911843
@@ -11179,8 +11863,7 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, · 19 unmodified lines
1117911863 mutex_lock(&ctx->uring_lock);
1118011864 ret = __io_uring_register(ctx, opcode, arg, nr_args);
1118111865 mutex_unlock(&ctx->uring_lock);
11182− trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
11183− ctx->cq_ev_fd != NULL, ret);
11866+ trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs, ret);
1118411867 out_fput:
1118511868 fdput(f);
1118611869 return ret;
include/linux/io_uring.h+4−1
@@ -9,11 +9,14 @@ · 8 unmodified lines
99 struct sock *io_uring_get_socket(struct file *file);
1010 void __io_uring_cancel(bool cancel_all);
1111 void __io_uring_free(struct task_struct *tsk);
12+void io_uring_unreg_ringfd(void);
1213
1314 static inline void io_uring_files_cancel(void)
1415 {
15− if (current->io_uring)
16+ if (current->io_uring) {
17+ io_uring_unreg_ringfd();
1618 __io_uring_cancel(false);
19+ }
1720 }
1821 static inline void io_uring_task_cancel(void)
1922 {
include/trace/events/io_uring.h+159−174
@@ -29,22 +29,22 @@ TRACE_EVENT(io_uring_create, · 28 unmodified lines
2929 TP_ARGS(fd, ctx, sq_entries, cq_entries, flags),
3030
3131 TP_STRUCT__entry (
32− __field( int, fd )
33− __field( void *, ctx )
32+ __field( int, fd )
33+ __field( void *, ctx )
3434 __field( u32, sq_entries )
3535 __field( u32, cq_entries )
3636 __field( u32, flags )
3737 ),
3838
3939 TP_fast_assign(
40− __entry->fd = fd;
40+ __entry->fd = fd;
4141 __entry->ctx = ctx;
4242 __entry->sq_entries = sq_entries;
4343 __entry->cq_entries = cq_entries;
4444 __entry->flags = flags;
4545 ),
4646
47− TP_printk("ring %p, fd %d sq size %d, cq size %d, flags %d",
47+ TP_printk("ring %p, fd %d sq size %d, cq size %d, flags 0x%x",
4848 __entry->ctx, __entry->fd, __entry->sq_entries,
4949 __entry->cq_entries, __entry->flags)
5050 );
@@ -57,10 +57,9 @@ TRACE_EVENT(io_uring_create, · 6 unmodified lines
5757 * @opcode: describes which operation to perform
5858 * @nr_user_files: number of registered files
5959 * @nr_user_bufs: number of registered buffers
60− * @cq_ev_fd: whether eventfs registered or not
6160 * @ret: return code
6261 *
63− * Allows to trace fixed files/buffers/eventfds, that could be registered to
62+ * Allows to trace fixed files/buffers, that could be registered to
6463 * avoid an overhead of getting references to them for every operation. This
6564 * event, together with io_uring_file_get, can provide a full picture of how
6665 * much overhead one can reduce via fixing.
@@ -68,17 +67,16 @@ TRACE_EVENT(io_uring_create, · 1 unmodified line
6867 TRACE_EVENT(io_uring_register,
6968
7069 TP_PROTO(void *ctx, unsigned opcode, unsigned nr_files,
71− unsigned nr_bufs, bool eventfd, long ret),
70+ unsigned nr_bufs, long ret),
7271
73− TP_ARGS(ctx, opcode, nr_files, nr_bufs, eventfd, ret),
72+ TP_ARGS(ctx, opcode, nr_files, nr_bufs, ret),
7473
7574 TP_STRUCT__entry (
76− __field( void *, ctx )
77− __field( unsigned, opcode )
78− __field( unsigned, nr_files )
79− __field( unsigned, nr_bufs )
80− __field( bool, eventfd )
81− __field( long, ret )
75+ __field( void *, ctx )
76+ __field( unsigned, opcode )
77+ __field( unsigned, nr_files)
78+ __field( unsigned, nr_bufs )
79+ __field( long, ret )
8280 ),
8381
8482 TP_fast_assign(
@@ -86,20 +84,21 @@ TRACE_EVENT(io_uring_register, · 1 unmodified line
8684 __entry->opcode = opcode;
8785 __entry->nr_files = nr_files;
8886 __entry->nr_bufs = nr_bufs;
89− __entry->eventfd = eventfd;
9087 __entry->ret = ret;
9188 ),
9289
9390 TP_printk("ring %p, opcode %d, nr_user_files %d, nr_user_bufs %d, "
94− "eventfd %d, ret %ld",
91+ "ret %ld",
9592 __entry->ctx, __entry->opcode, __entry->nr_files,
96− __entry->nr_bufs, __entry->eventfd, __entry->ret)
93+ __entry->nr_bufs, __entry->ret)
9794 );
9895
9996 /**
10097 * io_uring_file_get - called before getting references to an SQE file
10198 *
10299 * @ctx: pointer to a ring context structure
100+ * @req: pointer to a submitted request
101+ * @user_data: user data associated with the request
103102 * @fd: SQE file descriptor
104103 *
105104 * Allows to trace out how often an SQE file reference is obtained, which can
@@ -108,59 +107,71 @@ TRACE_EVENT(io_uring_register, · 2 unmodified lines
108107 */
109108 TRACE_EVENT(io_uring_file_get,
110109
111− TP_PROTO(void *ctx, int fd),
110+ TP_PROTO(void *ctx, void *req, unsigned long long user_data, int fd),
112111
113− TP_ARGS(ctx, fd),
112+ TP_ARGS(ctx, req, user_data, fd),
114113
115114 TP_STRUCT__entry (
116− __field( void *, ctx )
117− __field( int, fd )
115+ __field( void *, ctx )
116+ __field( void *, req )
117+ __field( u64, user_data )
118+ __field( int, fd )
118119 ),
119120
120121 TP_fast_assign(
121− __entry->ctx = ctx;
122+ __entry->ctx = ctx;
123+ __entry->req = req;
124+ __entry->user_data = user_data;
122125 __entry->fd = fd;
123126 ),
124127
125− TP_printk("ring %p, fd %d", __entry->ctx, __entry->fd)
128+ TP_printk("ring %p, req %p, user_data 0x%llx, fd %d",
129+ __entry->ctx, __entry->req, __entry->user_data, __entry->fd)
126130 );
127131
128132 /**
129133 * io_uring_queue_async_work - called before submitting a new async work
130134 *
131135 * @ctx: pointer to a ring context structure
132− * @hashed: type of workqueue, hashed or normal
133136 * @req: pointer to a submitted request
137+ * @user_data: user data associated with the request
138+ * @opcode: opcode of request
139+ * @flags request flags
134140 * @work: pointer to a submitted io_wq_work
141+ * @rw: type of workqueue, hashed or normal
135142 *
136143 * Allows to trace asynchronous work submission.
137144 */
138145 TRACE_EVENT(io_uring_queue_async_work,
139146
140− TP_PROTO(void *ctx, int rw, void * req, struct io_wq_work *work,
141− unsigned int flags),
147+ TP_PROTO(void *ctx, void * req, unsigned long long user_data, u8 opcode,
148+ unsigned int flags, struct io_wq_work *work, int rw),
142149
143− TP_ARGS(ctx, rw, req, work, flags),
150+ TP_ARGS(ctx, req, user_data, flags, opcode, work, rw),
144151
145152 TP_STRUCT__entry (
146− __field( void *, ctx )
147− __field( int, rw )
148− __field( void *, req )
149− __field( struct io_wq_work *, work )
150− __field( unsigned int, flags )
153+ __field( void *, ctx )
154+ __field( void *, req )
155+ __field( u64, user_data )
156+ __field( u8, opcode )
157+ __field( unsigned int, flags )
158+ __field( struct io_wq_work *, work )
159+ __field( int, rw )
151160 ),
152161
153162 TP_fast_assign(
154− __entry->ctx = ctx;
155− __entry->rw = rw;
156− __entry->req = req;
157− __entry->work = work;
158− __entry->flags = flags;
163+ __entry->ctx = ctx;
164+ __entry->req = req;
165+ __entry->user_data = user_data;
166+ __entry->flags = flags;
167+ __entry->opcode = opcode;
168+ __entry->work = work;
169+ __entry->rw = rw;
159170 ),
160171
161− TP_printk("ring %p, request %p, flags %d, %s queue, work %p",
162− __entry->ctx, __entry->req, __entry->flags,
163− __entry->rw ? "hashed" : "normal", __entry->work)
172+ TP_printk("ring %p, request %p, user_data 0x%llx, opcode %d, flags 0x%x, %s queue, work %p",
173+ __entry->ctx, __entry->req, __entry->user_data, __entry->opcode,
174+ __entry->flags, __entry->rw ? "hashed" : "normal", __entry->work)
164175 );
165176
166177 /**
@@ -169,30 +180,33 @@ TRACE_EVENT(io_uring_queue_async_work, · 2 unmodified lines
169180 * @ctx: pointer to a ring context structure
170181 * @req: pointer to a deferred request
171182 * @user_data: user data associated with the request
183+ * @opcode: opcode of request
172184 *
173185 * Allows to track deferred requests, to get an insight about what requests are
174186 * not started immediately.
175187 */
176188 TRACE_EVENT(io_uring_defer,
177189
178− TP_PROTO(void *ctx, void *req, unsigned long long user_data),
190+ TP_PROTO(void *ctx, void *req, unsigned long long user_data, u8 opcode),
179191
180− TP_ARGS(ctx, req, user_data),
192+ TP_ARGS(ctx, req, user_data, opcode),
181193
182194 TP_STRUCT__entry (
183− __field( void *, ctx )
184− __field( void *, req )
185− __field( unsigned long long, data )
195+ __field( void *, ctx )
196+ __field( void *, req )
197+ __field( unsigned long long, data )
198+ __field( u8, opcode )
186199 ),
187200
188201 TP_fast_assign(
189202 __entry->ctx = ctx;
190203 __entry->req = req;
191204 __entry->data = user_data;
205+ __entry->opcode = opcode;
192206 ),
193207
194− TP_printk("ring %p, request %p user_data %llu", __entry->ctx,
195− __entry->req, __entry->data)
208+ TP_printk("ring %p, request %p, user_data 0x%llx, opcode %d",
209+ __entry->ctx, __entry->req, __entry->data, __entry->opcode)
196210 );
197211
198212 /**
@@ -250,7 +264,7 @@ TRACE_EVENT(io_uring_cqring_wait, · 51 unmodified lines
250264 ),
251265
252266 TP_fast_assign(
253− __entry->ctx = ctx;
267+ __entry->ctx = ctx;
254268 __entry->min_events = min_events;
255269 ),
256270
@@ -260,7 +274,10 @@ TRACE_EVENT(io_uring_cqring_wait, · 3 unmodified lines
260274 /**
261275 * io_uring_fail_link - called before failing a linked request
262276 *
277+ * @ctx: pointer to a ring context structure
263278 * @req: request, which links were cancelled
279+ * @user_data: user data associated with the request
280+ * @opcode: opcode of request
264281 * @link: cancelled link
265282 *
266283 * Allows to track linked requests cancellation, to see not only that some work
@@ -268,27 +285,36 @@ TRACE_EVENT(io_uring_cqring_wait, · 1 unmodified line
268285 */
269286 TRACE_EVENT(io_uring_fail_link,
270287
271− TP_PROTO(void *req, void *link),
288+ TP_PROTO(void *ctx, void *req, unsigned long long user_data, u8 opcode, void *link),
272289
273− TP_ARGS(req, link),
290+ TP_ARGS(ctx, req, user_data, opcode, link),
274291
275292 TP_STRUCT__entry (
276− __field( void *, req )
277− __field( void *, link )
293+ __field( void *, ctx )
294+ __field( void *, req )
295+ __field( unsigned long long, user_data )
296+ __field( u8, opcode )
297+ __field( void *, link )
278298 ),
279299
280300 TP_fast_assign(
281− __entry->req = req;
282− __entry->link = link;
301+ __entry->ctx = ctx;
302+ __entry->req = req;
303+ __entry->user_data = user_data;
304+ __entry->opcode = opcode;
305+ __entry->link = link;
283306 ),
284307
285− TP_printk("request %p, link %p", __entry->req, __entry->link)
308+ TP_printk("ring %p, request %p, user_data 0x%llx, opcode %d, link %p",
309+ __entry->ctx, __entry->req, __entry->user_data, __entry->opcode,
310+ __entry->link)
286311 );
287312
288313 /**
289314 * io_uring_complete - called when completing an SQE
290315 *
291316 * @ctx: pointer to a ring context structure
317+ * @req: pointer to a submitted request
292318 * @user_data: user data associated with the request
293319 * @res: result of the request
294320 * @cflags: completion flags
@@ -296,12 +322,13 @@ TRACE_EVENT(io_uring_fail_link, · 1 unmodified line
296322 */
297323 TRACE_EVENT(io_uring_complete,
298324
299− TP_PROTO(void *ctx, u64 user_data, int res, unsigned cflags),
325+ TP_PROTO(void *ctx, void *req, u64 user_data, int res, unsigned cflags),
300326
301− TP_ARGS(ctx, user_data, res, cflags),
327+ TP_ARGS(ctx, req, user_data, res, cflags),
302328
303329 TP_STRUCT__entry (
304330 __field( void *, ctx )
331+ __field( void *, req )
305332 __field( u64, user_data )
306333 __field( int, res )
307334 __field( unsigned, cflags )
@@ -309,14 +336,16 @@ TRACE_EVENT(io_uring_complete, · 1 unmodified line
309336
310337 TP_fast_assign(
311338 __entry->ctx = ctx;
339+ __entry->req = req;
312340 __entry->user_data = user_data;
313341 __entry->res = res;
314342 __entry->cflags = cflags;
315343 ),
316344
317− TP_printk("ring %p, user_data 0x%llx, result %d, cflags %x",
318− __entry->ctx, (unsigned long long)__entry->user_data,
319− __entry->res, __entry->cflags)
345+ TP_printk("ring %p, req %p, user_data 0x%llx, result %d, cflags 0x%x",
346+ __entry->ctx, __entry->req,
347+ __entry->user_data,
348+ __entry->res, __entry->cflags)
320349 );
321350
322351 /**
@@ -324,8 +353,8 @@ TRACE_EVENT(io_uring_complete, · 1 unmodified line
324353 *
325354 * @ctx: pointer to a ring context structure
326355 * @req: pointer to a submitted request
327− * @opcode: opcode of request
328356 * @user_data: user data associated with the request
357+ * @opcode: opcode of request
329358 * @flags request flags
330359 * @force_nonblock: whether a context blocking or not
331360 * @sq_thread: true if sq_thread has submitted this SQE
@@ -335,34 +364,34 @@ TRACE_EVENT(io_uring_complete, · 3 unmodified lines
335364 */
336365 TRACE_EVENT(io_uring_submit_sqe,
337366
338− TP_PROTO(void *ctx, void *req, u8 opcode, u64 user_data, u32 flags,
367+ TP_PROTO(void *ctx, void *req, unsigned long long user_data, u8 opcode, u32 flags,
339368 bool force_nonblock, bool sq_thread),
340369
341− TP_ARGS(ctx, req, opcode, user_data, flags, force_nonblock, sq_thread),
370+ TP_ARGS(ctx, req, user_data, opcode, flags, force_nonblock, sq_thread),
342371
343372 TP_STRUCT__entry (
344− __field( void *, ctx )
345− __field( void *, req )
346− __field( u8, opcode )
347− __field( u64, user_data )
348− __field( u32, flags )
349− __field( bool, force_nonblock )
350− __field( bool, sq_thread )
373+ __field( void *, ctx )
374+ __field( void *, req )
375+ __field( unsigned long long, user_data )
376+ __field( u8, opcode )
377+ __field( u32, flags )
378+ __field( bool, force_nonblock )
379+ __field( bool, sq_thread )
351380 ),
352381
353382 TP_fast_assign(
354383 __entry->ctx = ctx;
355384 __entry->req = req;
356− __entry->opcode = opcode;
357385 __entry->user_data = user_data;
386+ __entry->opcode = opcode;
358387 __entry->flags = flags;
359388 __entry->force_nonblock = force_nonblock;
360389 __entry->sq_thread = sq_thread;
361390 ),
362391
363− TP_printk("ring %p, req %p, op %d, data 0x%llx, flags %u, "
392+ TP_printk("ring %p, req %p, user_data 0x%llx, opcode %d, flags 0x%x, "
364393 "non block %d, sq_thread %d", __entry->ctx, __entry->req,
365− __entry->opcode, (unsigned long long)__entry->user_data,
394+ __entry->user_data, __entry->opcode,
366395 __entry->flags, __entry->force_nonblock, __entry->sq_thread)
367396 );
368397
@@ -371,8 +400,8 @@ TRACE_EVENT(io_uring_submit_sqe, · 2 unmodified lines
371400 *
372401 * @ctx: pointer to a ring context structure
373402 * @req: pointer to the armed request
374− * @opcode: opcode of request
375403 * @user_data: user data associated with the request
404+ * @opcode: opcode of request
376405 * @mask: request poll events mask
377406 * @events: registered events of interest
378407 *
@@ -381,155 +410,110 @@ TRACE_EVENT(io_uring_submit_sqe, · 2 unmodified lines
381410 */
382411 TRACE_EVENT(io_uring_poll_arm,
383412
384− TP_PROTO(void *ctx, void *req, u8 opcode, u64 user_data,
413+ TP_PROTO(void *ctx, void *req, u64 user_data, u8 opcode,
385414 int mask, int events),
386415
387− TP_ARGS(ctx, req, opcode, user_data, mask, events),
416+ TP_ARGS(ctx, req, user_data, opcode, mask, events),
388417
389418 TP_STRUCT__entry (
390− __field( void *, ctx )
391− __field( void *, req )
392− __field( u8, opcode )
393− __field( u64, user_data )
394− __field( int, mask )
395− __field( int, events )
419+ __field( void *, ctx )
420+ __field( void *, req )
421+ __field( unsigned long long, user_data )
422+ __field( u8, opcode )
423+ __field( int, mask )
424+ __field( int, events )
396425 ),
397426
398427 TP_fast_assign(
399428 __entry->ctx = ctx;
400429 __entry->req = req;
401− __entry->opcode = opcode;
402430 __entry->user_data = user_data;
431+ __entry->opcode = opcode;
403432 __entry->mask = mask;
404433 __entry->events = events;
405434 ),
406435
407− TP_printk("ring %p, req %p, op %d, data 0x%llx, mask 0x%x, events 0x%x",
408− __entry->ctx, __entry->req, __entry->opcode,
409− (unsigned long long) __entry->user_data,
436+ TP_printk("ring %p, req %p, user_data 0x%llx, opcode %d, mask 0x%x, events 0x%x",
437+ __entry->ctx, __entry->req, __entry->user_data, __entry->opcode,
410438 __entry->mask, __entry->events)
411439 );
412440
413−TRACE_EVENT(io_uring_poll_wake,
414−
415− TP_PROTO(void *ctx, u8 opcode, u64 user_data, int mask),
416−
417− TP_ARGS(ctx, opcode, user_data, mask),
418−
419− TP_STRUCT__entry (
420− __field( void *, ctx )
421− __field( u8, opcode )
422− __field( u64, user_data )
423− __field( int, mask )
424− ),
425−
426− TP_fast_assign(
427− __entry->ctx = ctx;
428− __entry->opcode = opcode;
429− __entry->user_data = user_data;
430− __entry->mask = mask;
431− ),
432−
433− TP_printk("ring %p, op %d, data 0x%llx, mask 0x%x",
434− __entry->ctx, __entry->opcode,
435− (unsigned long long) __entry->user_data,
436− __entry->mask)
437−);
438−
439−TRACE_EVENT(io_uring_task_add,
440−
441− TP_PROTO(void *ctx, u8 opcode, u64 user_data, int mask),
442−
443− TP_ARGS(ctx, opcode, user_data, mask),
444−
445− TP_STRUCT__entry (
446− __field( void *, ctx )
447− __field( u8, opcode )
448− __field( u64, user_data )
449− __field( int, mask )
450− ),
451−
452− TP_fast_assign(
453− __entry->ctx = ctx;
454− __entry->opcode = opcode;
455− __entry->user_data = user_data;
456− __entry->mask = mask;
457− ),
458−
459− TP_printk("ring %p, op %d, data 0x%llx, mask %x",
460− __entry->ctx, __entry->opcode,
461− (unsigned long long) __entry->user_data,
462− __entry->mask)
463−);
464−
465441 /*
466− * io_uring_task_run - called when task_work_run() executes the poll events
467− * notification callbacks
442+ * io_uring_task_add - called after adding a task
468443 *
469444 * @ctx: pointer to a ring context structure
470− * @req: pointer to the armed request
471− * @opcode: opcode of request
445+ * @req: pointer to request
472446 * @user_data: user data associated with the request
447+ * @opcode: opcode of request
448+ * @mask: request poll events mask
473449 *
474− * Allows to track when notified poll events are processed
475450 */
476−TRACE_EVENT(io_uring_task_run,
451+TRACE_EVENT(io_uring_task_add,
477452
478− TP_PROTO(void *ctx, void *req, u8 opcode, u64 user_data),
453+ TP_PROTO(void *ctx, void *req, unsigned long long user_data, u8 opcode, int mask),
479454
480− TP_ARGS(ctx, req, opcode, user_data),
455+ TP_ARGS(ctx, req, user_data, opcode, mask),
481456
482457 TP_STRUCT__entry (
483− __field( void *, ctx )
484− __field( void *, req )
485− __field( u8, opcode )
486− __field( u64, user_data )
458+ __field( void *, ctx )
459+ __field( void *, req )
460+ __field( unsigned long long, user_data )
461+ __field( u8, opcode )
462+ __field( int, mask )
487463 ),
488464
489465 TP_fast_assign(
490466 __entry->ctx = ctx;
491467 __entry->req = req;
492− __entry->opcode = opcode;
493468 __entry->user_data = user_data;
469+ __entry->opcode = opcode;
470+ __entry->mask = mask;
494471 ),
495472
496− TP_printk("ring %p, req %p, op %d, data 0x%llx",
497− __entry->ctx, __entry->req, __entry->opcode,
498− (unsigned long long) __entry->user_data)
473+ TP_printk("ring %p, req %p, user_data 0x%llx, opcode %d, mask %x",
474+ __entry->ctx, __entry->req, __entry->user_data, __entry->opcode,
475+ __entry->mask)
499476 );
500477
501478 /*
502479 * io_uring_req_failed - called when an sqe is errored dring submission
503480 *
504481 * @sqe: pointer to the io_uring_sqe that failed
482+ * @ctx: pointer to a ring context structure
483+ * @req: pointer to request
505484 * @error: error it failed with
506485 *
507486 * Allows easier diagnosing of malformed requests in production systems.
508487 */
509488 TRACE_EVENT(io_uring_req_failed,
510489
511− TP_PROTO(const struct io_uring_sqe *sqe, int error),
490+ TP_PROTO(const struct io_uring_sqe *sqe, void *ctx, void *req, int error),
512491
513− TP_ARGS(sqe, error),
492+ TP_ARGS(sqe, ctx, req, error),
514493
515494 TP_STRUCT__entry (
516− __field( u8, opcode )
517− __field( u8, flags )
518− __field( u8, ioprio )
519− __field( u64, off )
520− __field( u64, addr )
521− __field( u32, len )
522− __field( u32, op_flags )
523− __field( u64, user_data )
524− __field( u16, buf_index )
525− __field( u16, personality )
526− __field( u32, file_index )
527− __field( u64, pad1 )
528− __field( u64, pad2 )
529− __field( int, error )
495+ __field( void *, ctx )
496+ __field( void *, req )
497+ __field( unsigned long long, user_data )
498+ __field( u8, opcode )
499+ __field( u8, flags )
500+ __field( u8, ioprio )
501+ __field( u64, off )
502+ __field( u64, addr )
503+ __field( u32, len )
504+ __field( u32, op_flags )
505+ __field( u16, buf_index )
506+ __field( u16, personality )
507+ __field( u32, file_index )
508+ __field( u64, pad1 )
509+ __field( u64, pad2 )
510+ __field( int, error )
530511 ),
531512
532513 TP_fast_assign(
514+ __entry->ctx = ctx;
515+ __entry->req = req;
516+ __entry->user_data = sqe->user_data;
533517 __entry->opcode = sqe->opcode;
534518 __entry->flags = sqe->flags;
535519 __entry->ioprio = sqe->ioprio;
@@ -537,7 +521,6 @@ TRACE_EVENT(io_uring_req_failed, · 1 unmodified line
537521 __entry->addr = sqe->addr;
538522 __entry->len = sqe->len;
539523 __entry->op_flags = sqe->rw_flags;
540− __entry->user_data = sqe->user_data;
541524 __entry->buf_index = sqe->buf_index;
542525 __entry->personality = sqe->personality;
543526 __entry->file_index = sqe->file_index;
@@ -546,13 +529,15 @@ TRACE_EVENT(io_uring_req_failed, · 2 unmodified lines
546529 __entry->error = error;
547530 ),
548531
549− TP_printk("op %d, flags=0x%x, prio=%d, off=%llu, addr=%llu, "
550− "len=%u, rw_flags=0x%x, user_data=0x%llx, buf_index=%d, "
532+ TP_printk("ring %p, req %p, user_data 0x%llx, "
533+ "op %d, flags 0x%x, prio=%d, off=%llu, addr=%llu, "
534+ "len=%u, rw_flags=0x%x, buf_index=%d, "
551535 "personality=%d, file_index=%d, pad=0x%llx/%llx, error=%d",
536+ __entry->ctx, __entry->req, __entry->user_data,
552537 __entry->opcode, __entry->flags, __entry->ioprio,
553538 (unsigned long long)__entry->off,
554539 (unsigned long long) __entry->addr, __entry->len,
555− __entry->op_flags, (unsigned long long) __entry->user_data,
540+ __entry->op_flags,
556541 __entry->buf_index, __entry->personality, __entry->file_index,
557542 (unsigned long long) __entry->pad1,
558543 (unsigned long long) __entry->pad2, __entry->error)
include/uapi/linux/io_uring.h+13−4
@@ -101,6 +101,7 @@ enum { · 100 unmodified lines
101101 #define IORING_SETUP_CLAMP (1U << 4) /* clamp SQ/CQ ring sizes */
102102 #define IORING_SETUP_ATTACH_WQ (1U << 5) /* attach to existing wq */
103103 #define IORING_SETUP_R_DISABLED (1U << 6) /* start with ring disabled */
104+#define IORING_SETUP_SUBMIT_ALL (1U << 7) /* continue submit on error */
104105
105106 enum {
106107 IORING_OP_NOP,
@@ -143,6 +144,7 @@ enum { · 36 unmodified lines
143144 IORING_OP_MKDIRAT,
144145 IORING_OP_SYMLINKAT,
145146 IORING_OP_LINKAT,
147+ IORING_OP_MSG_RING,
146148
147149 /* this goes last, obviously */
148150 IORING_OP_LAST,
@@ -199,9 +201,11 @@ struct io_uring_cqe { · 50 unmodified lines
199201 *
200202 * IORING_CQE_F_BUFFER If set, the upper 16 bits are the buffer ID
201203 * IORING_CQE_F_MORE If set, parent SQE will generate more CQE entries
204+ * IORING_CQE_F_MSG If set, CQE was generated with IORING_OP_MSG_RING
202205 */
203206 #define IORING_CQE_F_BUFFER (1U << 0)
204207 #define IORING_CQE_F_MORE (1U << 1)
208+#define IORING_CQE_F_MSG (1U << 2)
205209
206210 enum {
207211 IORING_CQE_BUFFER_SHIFT = 16,
@@ -257,10 +261,11 @@ struct io_cqring_offsets { · 49 unmodified lines
257261 /*
258262 * io_uring_enter(2) flags
259263 */
260−#define IORING_ENTER_GETEVENTS (1U << 0)
261−#define IORING_ENTER_SQ_WAKEUP (1U << 1)
262−#define IORING_ENTER_SQ_WAIT (1U << 2)
263−#define IORING_ENTER_EXT_ARG (1U << 3)
264+#define IORING_ENTER_GETEVENTS (1U << 0)
265+#define IORING_ENTER_SQ_WAKEUP (1U << 1)
266+#define IORING_ENTER_SQ_WAIT (1U << 2)
267+#define IORING_ENTER_EXT_ARG (1U << 3)
268+#define IORING_ENTER_REGISTERED_RING (1U << 4)
264269
265270 /*
266271 * Passed in for io_uring_setup(2). Copied back with updated info on success
@@ -325,6 +330,10 @@ enum { · 58 unmodified lines
325330 /* set/get max number of io-wq workers */
326331 IORING_REGISTER_IOWQ_MAX_WORKERS = 19,
327332
333+ /* register/unregister io_uring fd with the ring */
334+ IORING_REGISTER_RING_FDS = 20,
335+ IORING_UNREGISTER_RING_FDS = 21,
336+
328337 /* this goes last */
329338 IORING_REGISTER_LAST
330339 };