History

Merge tag 'for-5.18/io_uring-2022-03-18' of git://git.kernel.dk/linux-block

Pull io_uring updates from Jens Axboe: - Fixes for current file position. Still doesn't have the f_pos_lock sorted, but it's a step in the right direction (Dylan) - Tracing updates (Dylan, Stefan) - Improvements to io-wq locking (Hao) - Improvements for provided buffers (me, Pavel) - Support for registered file descriptors (me, Xiaoguang) - Support for ring messages (me) - Poll improvements (me) - Fix for fixed buffers and non-iterator reads/writes (me) - Support for NAPI on sockets (Olivier) - Ring quiesce improvements (Usama) - Misc fixes (Olivier, Pavel) * tag 'for-5.18/io_uring-2022-03-18' of git://git.kernel.dk/linux-block: (42 commits) io_uring: terminate manual loop iterator loop correctly for non-vecs io_uring: don't check unrelated req->open.how in accept request io_uring: manage provided buffers strictly ordered io_uring: fold evfd signalling under a slower path io_uring: thin down io_commit_cqring() io_uring: shuffle io_eventfd_signal() bits around io_uring: remove extra barrier for non-sqpoll iopoll io_uring: fix provided buffer return on failure for kiocb_done() io_uring: extend provided buf return to fails io_uring: refactor timeout cancellation cqe posting io_uring: normilise naming for fill_cqe* io_uring: cache poll/double-poll state with a request flag io_uring: cache req->apoll->events in req->cflags io_uring: move req->poll_refs into previous struct hole io_uring: make tracing format consistent io_uring: recycle apoll_poll entries io_uring: remove duplicated member check for io_msg_ring_prep() io_uring: allow submissions to continue on error io_uring: recycle provided buffers if request goes async io_uring: ensure reads re-import for selected buffers ...

Linus Torvalds committed 4 years ago

Commit af472a92 parents 93e220a 5e929365 files · +1200 −520Browse files

Unified · Split

5 files
  1. fs/io-wq.c +57 −57
  2. fs/io_uring.c +967 −284
  3. include/linux/io_uring.h +4 −1
  4. include/trace/events/io_uring.h +159 −174
  5. include/uapi/linux/io_uring.h +13 −4
fs/io-wq.c+57−57
@@ -76,6 +76,7 @@ struct io_wqe_acct { · 75 unmodified lines
76 unsigned max_workers;76 unsigned max_workers;
77 int index;77 int index;
78 atomic_t nr_running;78 atomic_t nr_running;
79 raw_spinlock_t lock;
79 struct io_wq_work_list work_list;80 struct io_wq_work_list work_list;
80 unsigned long flags;81 unsigned long flags;
81};82};
@@ -91,7 +92,7 @@ enum { · 9 unmodified lines
91 */92 */
92struct io_wqe {93struct io_wqe {
93 raw_spinlock_t lock;94 raw_spinlock_t lock;
94 struct io_wqe_acct acct[2];95 struct io_wqe_acct acct[IO_WQ_ACCT_NR];
95 96
96 int node;97 int node;
97 98
@@ -224,12 +225,12 @@ static void io_worker_exit(struct io_worker *worker) · 126 unmodified lines
224 if (worker->flags & IO_WORKER_F_FREE)225 if (worker->flags & IO_WORKER_F_FREE)
225 hlist_nulls_del_rcu(&worker->nulls_node);226 hlist_nulls_del_rcu(&worker->nulls_node);
226 list_del_rcu(&worker->all_list);227 list_del_rcu(&worker->all_list);
227 preempt_disable();228 raw_spin_unlock(&wqe->lock);
228 io_wqe_dec_running(worker);229 io_wqe_dec_running(worker);
229 worker->flags = 0;230 worker->flags = 0;
231 preempt_disable();
230 current->flags &= ~PF_IO_WORKER;232 current->flags &= ~PF_IO_WORKER;
231 preempt_enable();233 preempt_enable();
232 raw_spin_unlock(&wqe->lock);
233 234
234 kfree_rcu(worker, rcu);235 kfree_rcu(worker, rcu);
235 io_worker_ref_put(wqe->wq);236 io_worker_ref_put(wqe->wq);
@@ -238,10 +239,15 @@ static void io_worker_exit(struct io_worker *worker) · 2 unmodified lines
238 239
239static inline bool io_acct_run_queue(struct io_wqe_acct *acct)240static inline bool io_acct_run_queue(struct io_wqe_acct *acct)
240{241{
242 bool ret = false;
243
244 raw_spin_lock(&acct->lock);
241 if (!wq_list_empty(&acct->work_list) &&245 if (!wq_list_empty(&acct->work_list) &&
242 !test_bit(IO_ACCT_STALLED_BIT, &acct->flags))246 !test_bit(IO_ACCT_STALLED_BIT, &acct->flags))
243 return true;247 ret = true;
244 return false;248 raw_spin_unlock(&acct->lock);
249
250 return ret;
245}251}
246 252
247/*253/*
@@ -385,7 +391,6 @@ fail: · 137 unmodified lines
385}391}
386 392
387static void io_wqe_dec_running(struct io_worker *worker)393static void io_wqe_dec_running(struct io_worker *worker)
388 __must_hold(wqe->lock)
389{394{
390 struct io_wqe_acct *acct = io_wqe_get_acct(worker);395 struct io_wqe_acct *acct = io_wqe_get_acct(worker);
391 struct io_wqe *wqe = worker->wqe;396 struct io_wqe *wqe = worker->wqe;
@@ -393,13 +398,14 @@ static void io_wqe_dec_running(struct io_worker *worker) · 1 unmodified line
393 if (!(worker->flags & IO_WORKER_F_UP))398 if (!(worker->flags & IO_WORKER_F_UP))
394 return;399 return;
395 400
396 if (atomic_dec_and_test(&acct->nr_running) && io_acct_run_queue(acct)) {401 if (!atomic_dec_and_test(&acct->nr_running))
397 atomic_inc(&acct->nr_running);402 return;
398 atomic_inc(&wqe->wq->worker_refs);403 if (!io_acct_run_queue(acct))
399 raw_spin_unlock(&wqe->lock);404 return;
400 io_queue_worker_create(worker, acct, create_worker_cb);405
401 raw_spin_lock(&wqe->lock);406 atomic_inc(&acct->nr_running);
402 }407 atomic_inc(&wqe->wq->worker_refs);
408 io_queue_worker_create(worker, acct, create_worker_cb);
403}409}
404 410
405/*411/*
@@ -407,11 +413,12 @@ static void io_wqe_dec_running(struct io_worker *worker) · 1 unmodified line
407 * it's currently on the freelist413 * it's currently on the freelist
408 */414 */
409static void __io_worker_busy(struct io_wqe *wqe, struct io_worker *worker)415static void __io_worker_busy(struct io_wqe *wqe, struct io_worker *worker)
410 __must_hold(wqe->lock)
411{416{
412 if (worker->flags & IO_WORKER_F_FREE) {417 if (worker->flags & IO_WORKER_F_FREE) {
413 worker->flags &= ~IO_WORKER_F_FREE;418 worker->flags &= ~IO_WORKER_F_FREE;
419 raw_spin_lock(&wqe->lock);
414 hlist_nulls_del_init_rcu(&worker->nulls_node);420 hlist_nulls_del_init_rcu(&worker->nulls_node);
421 raw_spin_unlock(&wqe->lock);
415 }422 }
416}423}
417 424
@@ -456,7 +463,7 @@ static bool io_wait_on_hash(struct io_wqe *wqe, unsigned int hash) · 38 unmodified lines
456 463
457static struct io_wq_work *io_get_next_work(struct io_wqe_acct *acct,464static struct io_wq_work *io_get_next_work(struct io_wqe_acct *acct,
458 struct io_worker *worker)465 struct io_worker *worker)
459 __must_hold(wqe->lock)466 __must_hold(acct->lock)
460{467{
461 struct io_wq_work_node *node, *prev;468 struct io_wq_work_node *node, *prev;
462 struct io_wq_work *work, *tail;469 struct io_wq_work *work, *tail;
@@ -498,9 +505,9 @@ static struct io_wq_work *io_get_next_work(struct io_wqe_acct *acct, · 35 unmodified lines
498 * work being added and clearing the stalled bit.505 * work being added and clearing the stalled bit.
499 */506 */
500 set_bit(IO_ACCT_STALLED_BIT, &acct->flags);507 set_bit(IO_ACCT_STALLED_BIT, &acct->flags);
501 raw_spin_unlock(&wqe->lock);508 raw_spin_unlock(&acct->lock);
502 unstalled = io_wait_on_hash(wqe, stall_hash);509 unstalled = io_wait_on_hash(wqe, stall_hash);
503 raw_spin_lock(&wqe->lock);510 raw_spin_lock(&acct->lock);
504 if (unstalled) {511 if (unstalled) {
505 clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);512 clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
506 if (wq_has_sleeper(&wqe->wq->hash->wait))513 if (wq_has_sleeper(&wqe->wq->hash->wait))
@@ -538,7 +545,6 @@ static void io_assign_current_work(struct io_worker *worker, · 31 unmodified lines
538static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work);545static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work);
539 546
540static void io_worker_handle_work(struct io_worker *worker)547static void io_worker_handle_work(struct io_worker *worker)
541 __releases(wqe->lock)
542{548{
543 struct io_wqe_acct *acct = io_wqe_get_acct(worker);549 struct io_wqe_acct *acct = io_wqe_get_acct(worker);
544 struct io_wqe *wqe = worker->wqe;550 struct io_wqe *wqe = worker->wqe;
@@ -555,7 +561,9 @@ static void io_worker_handle_work(struct io_worker *worker) · 10 unmodified lines
555 * can't make progress, any work completion or insertion will561 * can't make progress, any work completion or insertion will
556 * clear the stalled flag.562 * clear the stalled flag.
557 */563 */
564 raw_spin_lock(&acct->lock);
558 work = io_get_next_work(acct, worker);565 work = io_get_next_work(acct, worker);
566 raw_spin_unlock(&acct->lock);
559 if (work) {567 if (work) {
560 __io_worker_busy(wqe, worker);568 __io_worker_busy(wqe, worker);
561 569
@@ -569,10 +577,9 @@ static void io_worker_handle_work(struct io_worker *worker) · 7 unmodified lines
569 raw_spin_lock(&worker->lock);577 raw_spin_lock(&worker->lock);
570 worker->next_work = work;578 worker->next_work = work;
571 raw_spin_unlock(&worker->lock);579 raw_spin_unlock(&worker->lock);
572 }580 } else {
573 raw_spin_unlock(&wqe->lock);
574 if (!work)
575 break;581 break;
582 }
576 io_assign_current_work(worker, work);583 io_assign_current_work(worker, work);
577 __set_current_state(TASK_RUNNING);584 __set_current_state(TASK_RUNNING);
578 585
@@ -608,8 +615,6 @@ static void io_worker_handle_work(struct io_worker *worker) · 29 unmodified lines
608 wake_up(&wq->hash->wait);615 wake_up(&wq->hash->wait);
609 }616 }
610 } while (work);617 } while (work);
611
612 raw_spin_lock(&wqe->lock);
613 } while (1);618 } while (1);
614}619}
615 620
@@ -633,12 +638,10 @@ static int io_wqe_worker(void *data) · 17 unmodified lines
633 long ret;638 long ret;
634 639
635 set_current_state(TASK_INTERRUPTIBLE);640 set_current_state(TASK_INTERRUPTIBLE);
636loop:641 while (io_acct_run_queue(acct))
637 raw_spin_lock(&wqe->lock);
638 if (io_acct_run_queue(acct)) {
639 io_worker_handle_work(worker);642 io_worker_handle_work(worker);
640 goto loop;643
641 }644 raw_spin_lock(&wqe->lock);
642 /* timed out, exit unless we're the last worker */645 /* timed out, exit unless we're the last worker */
643 if (last_timeout && acct->nr_workers > 1) {646 if (last_timeout && acct->nr_workers > 1) {
644 acct->nr_workers--;647 acct->nr_workers--;
@@ -662,10 +665,8 @@ loop: · 17 unmodified lines
662 last_timeout = !ret;665 last_timeout = !ret;
663 }666 }
664 667
665 if (test_bit(IO_WQ_BIT_EXIT, &wq->state)) {668 if (test_bit(IO_WQ_BIT_EXIT, &wq->state))
666 raw_spin_lock(&wqe->lock);
667 io_worker_handle_work(worker);669 io_worker_handle_work(worker);
668 }
669 670
670 audit_free(current);671 audit_free(current);
671 io_worker_exit(worker);672 io_worker_exit(worker);
@@ -705,10 +706,7 @@ void io_wq_worker_sleeping(struct task_struct *tsk) · 33 unmodified lines
705 return;706 return;
706 707
707 worker->flags &= ~IO_WORKER_F_RUNNING;708 worker->flags &= ~IO_WORKER_F_RUNNING;
708
709 raw_spin_lock(&worker->wqe->lock);
710 io_wqe_dec_running(worker);709 io_wqe_dec_running(worker);
711 raw_spin_unlock(&worker->wqe->lock);
712}710}
713 711
714static void io_init_new_worker(struct io_wqe *wqe, struct io_worker *worker,712static void io_init_new_worker(struct io_wqe *wqe, struct io_worker *worker,
@@ -778,10 +776,12 @@ static void create_worker_cont(struct callback_head *cb) · 63 unmodified lines
778 .cancel_all = true,776 .cancel_all = true,
779 };777 };
780 778
779 raw_spin_unlock(&wqe->lock);
781 while (io_acct_cancel_pending_work(wqe, acct, &match))780 while (io_acct_cancel_pending_work(wqe, acct, &match))
782 raw_spin_lock(&wqe->lock);781 ;
782 } else {
783 raw_spin_unlock(&wqe->lock);
783 }784 }
784 raw_spin_unlock(&wqe->lock);
785 io_worker_ref_put(wqe->wq);785 io_worker_ref_put(wqe->wq);
786 kfree(worker);786 kfree(worker);
787 return;787 return;
@@ -914,6 +914,7 @@ static bool io_wq_work_match_item(struct io_wq_work *work, void *data) · 126 unmodified lines
914static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work)914static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work)
915{915{
916 struct io_wqe_acct *acct = io_work_get_acct(wqe, work);916 struct io_wqe_acct *acct = io_work_get_acct(wqe, work);
917 struct io_cb_cancel_data match;
917 unsigned work_flags = work->flags;918 unsigned work_flags = work->flags;
918 bool do_create;919 bool do_create;
919 920
@@ -927,10 +928,12 @@ static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work) · 7 unmodified lines
927 return;928 return;
928 }929 }
929 930
930 raw_spin_lock(&wqe->lock);931 raw_spin_lock(&acct->lock);
931 io_wqe_insert_work(wqe, work);932 io_wqe_insert_work(wqe, work);
932 clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);933 clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
934 raw_spin_unlock(&acct->lock);
933 935
936 raw_spin_lock(&wqe->lock);
934 rcu_read_lock();937 rcu_read_lock();
935 do_create = !io_wqe_activate_free_worker(wqe, acct);938 do_create = !io_wqe_activate_free_worker(wqe, acct);
936 rcu_read_unlock();939 rcu_read_unlock();
@@ -946,18 +949,18 @@ static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work) · 9 unmodified lines
946 return;949 return;
947 950
948 raw_spin_lock(&wqe->lock);951 raw_spin_lock(&wqe->lock);
949 /* fatal condition, failed to create the first worker */952 if (acct->nr_workers) {
950 if (!acct->nr_workers) {953 raw_spin_unlock(&wqe->lock);
951 struct io_cb_cancel_data match = {954 return;
952 .fn = io_wq_work_match_item,
953 .data = work,
954 .cancel_all = false,
955 };
956
957 if (io_acct_cancel_pending_work(wqe, acct, &match))
958 raw_spin_lock(&wqe->lock);
959 }955 }
960 raw_spin_unlock(&wqe->lock);956 raw_spin_unlock(&wqe->lock);
957
958 /* fatal condition, failed to create the first worker */
959 match.fn = io_wq_work_match_item,
960 match.data = work,
961 match.cancel_all = false,
962
963 io_acct_cancel_pending_work(wqe, acct, &match);
961 }964 }
962}965}
963 966
@@ -1032,22 +1035,23 @@ static inline void io_wqe_remove_pending(struct io_wqe *wqe, · 68 unmodified lines
1032static bool io_acct_cancel_pending_work(struct io_wqe *wqe,1035static bool io_acct_cancel_pending_work(struct io_wqe *wqe,
1033 struct io_wqe_acct *acct,1036 struct io_wqe_acct *acct,
1034 struct io_cb_cancel_data *match)1037 struct io_cb_cancel_data *match)
1035 __releases(wqe->lock)
1036{1038{
1037 struct io_wq_work_node *node, *prev;1039 struct io_wq_work_node *node, *prev;
1038 struct io_wq_work *work;1040 struct io_wq_work *work;
1039 1041
1042 raw_spin_lock(&acct->lock);
1040 wq_list_for_each(node, prev, &acct->work_list) {1043 wq_list_for_each(node, prev, &acct->work_list) {
1041 work = container_of(node, struct io_wq_work, list);1044 work = container_of(node, struct io_wq_work, list);
1042 if (!match->fn(work, match->data))1045 if (!match->fn(work, match->data))
1043 continue;1046 continue;
1044 io_wqe_remove_pending(wqe, work, prev);1047 io_wqe_remove_pending(wqe, work, prev);
1045 raw_spin_unlock(&wqe->lock);1048 raw_spin_unlock(&acct->lock);
1046 io_run_cancel(work, wqe);1049 io_run_cancel(work, wqe);
1047 match->nr_pending++;1050 match->nr_pending++;
1048 /* not safe to continue after unlock */1051 /* not safe to continue after unlock */
1049 return true;1052 return true;
1050 }1053 }
1054 raw_spin_unlock(&acct->lock);
1051 1055
1052 return false;1056 return false;
1053}1057}
@@ -1061,7 +1065,6 @@ retry: · 7 unmodified lines
1061 struct io_wqe_acct *acct = io_get_acct(wqe, i == 0);1065 struct io_wqe_acct *acct = io_get_acct(wqe, i == 0);
1062 1066
1063 if (io_acct_cancel_pending_work(wqe, acct, match)) {1067 if (io_acct_cancel_pending_work(wqe, acct, match)) {
1064 raw_spin_lock(&wqe->lock);
1065 if (match->cancel_all)1068 if (match->cancel_all)
1066 goto retry;1069 goto retry;
1067 break;1070 break;
@@ -1103,13 +1106,11 @@ enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel, · 35 unmodified lines
1103 for_each_node(node) {1106 for_each_node(node) {
1104 struct io_wqe *wqe = wq->wqes[node];1107 struct io_wqe *wqe = wq->wqes[node];
1105 1108
1106 raw_spin_lock(&wqe->lock);
1107 io_wqe_cancel_pending_work(wqe, &match);1109 io_wqe_cancel_pending_work(wqe, &match);
1108 if (match.nr_pending && !match.cancel_all) {1110 if (match.nr_pending && !match.cancel_all)
1109 raw_spin_unlock(&wqe->lock);
1110 return IO_WQ_CANCEL_OK;1111 return IO_WQ_CANCEL_OK;
1111 }
1112 1112
1113 raw_spin_lock(&wqe->lock);
1113 io_wqe_cancel_running_work(wqe, &match);1114 io_wqe_cancel_running_work(wqe, &match);
1114 raw_spin_unlock(&wqe->lock);1115 raw_spin_unlock(&wqe->lock);
1115 if (match.nr_running && !match.cancel_all)1116 if (match.nr_running && !match.cancel_all)
@@ -1190,6 +1191,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) · 74 unmodified lines
1190 acct->index = i;1191 acct->index = i;
1191 atomic_set(&acct->nr_running, 0);1192 atomic_set(&acct->nr_running, 0);
1192 INIT_WQ_LIST(&acct->work_list);1193 INIT_WQ_LIST(&acct->work_list);
1194 raw_spin_lock_init(&acct->lock);
1193 }1195 }
1194 wqe->wq = wq;1196 wqe->wq = wq;
1195 raw_spin_lock_init(&wqe->lock);1197 raw_spin_lock_init(&wqe->lock);
@@ -1282,9 +1284,7 @@ static void io_wq_destroy(struct io_wq *wq) · 86 unmodified lines
1282 .fn = io_wq_work_match_all,1284 .fn = io_wq_work_match_all,
1283 .cancel_all = true,1285 .cancel_all = true,
1284 };1286 };
1285 raw_spin_lock(&wqe->lock);
1286 io_wqe_cancel_pending_work(wqe, &match);1287 io_wqe_cancel_pending_work(wqe, &match);
1287 raw_spin_unlock(&wqe->lock);
1288 free_cpumask_var(wqe->cpu_mask);1288 free_cpumask_var(wqe->cpu_mask);
1289 kfree(wqe);1289 kfree(wqe);
1290 }1290 }
@@ -1376,7 +1376,7 @@ int io_wq_max_workers(struct io_wq *wq, int *new_count) · 85 unmodified lines
1376 BUILD_BUG_ON((int) IO_WQ_ACCT_UNBOUND != (int) IO_WQ_UNBOUND);1376 BUILD_BUG_ON((int) IO_WQ_ACCT_UNBOUND != (int) IO_WQ_UNBOUND);
1377 BUILD_BUG_ON((int) IO_WQ_ACCT_NR != 2);1377 BUILD_BUG_ON((int) IO_WQ_ACCT_NR != 2);
1378 1378
1379 for (i = 0; i < 2; i++) {1379 for (i = 0; i < IO_WQ_ACCT_NR; i++) {
1380 if (new_count[i] > task_rlimit(current, RLIMIT_NPROC))1380 if (new_count[i] > task_rlimit(current, RLIMIT_NPROC))
1381 new_count[i] = task_rlimit(current, RLIMIT_NPROC);1381 new_count[i] = task_rlimit(current, RLIMIT_NPROC);
1382 }1382 }
fs/io_uring.c+967−284
@@ -63,6 +63,7 @@ · 62 unmodified lines
63#include <net/sock.h>63#include <net/sock.h>
64#include <net/af_unix.h>64#include <net/af_unix.h>
65#include <net/scm.h>65#include <net/scm.h>
66#include <net/busy_poll.h>
66#include <linux/anon_inodes.h>67#include <linux/anon_inodes.h>
67#include <linux/sched/mm.h>68#include <linux/sched/mm.h>
68#include <linux/uaccess.h>69#include <linux/uaccess.h>
@@ -263,11 +264,18 @@ struct io_rsrc_data { · 194 unmodified lines
263 bool quiesce;264 bool quiesce;
264};265};
265 266
267struct io_buffer_list {
268 struct list_head list;
269 struct list_head buf_list;
270 __u16 bgid;
271};
272
266struct io_buffer {273struct io_buffer {
267 struct list_head list;274 struct list_head list;
268 __u64 addr;275 __u64 addr;
269 __u32 len;276 __u32 len;
270 __u16 bid;277 __u16 bid;
278 __u16 bgid;
271};279};
272 280
273struct io_restriction {281struct io_restriction {
@@ -326,6 +334,14 @@ struct io_submit_state { · 52 unmodified lines
326 struct blk_plug plug;334 struct blk_plug plug;
327};335};
328 336
337struct io_ev_fd {
338 struct eventfd_ctx *cq_ev_fd;
339 unsigned int eventfd_async: 1;
340 struct rcu_head rcu;
341};
342
343#define IO_BUFFERS_HASH_BITS 5
344
329struct io_ring_ctx {345struct io_ring_ctx {
330 /* const or read-mostly hot data */346 /* const or read-mostly hot data */
331 struct {347 struct {
@@ -335,11 +351,11 @@ struct io_ring_ctx { · 3 unmodified lines
335 unsigned int flags;351 unsigned int flags;
336 unsigned int compat: 1;352 unsigned int compat: 1;
337 unsigned int drain_next: 1;353 unsigned int drain_next: 1;
338 unsigned int eventfd_async: 1;
339 unsigned int restricted: 1;354 unsigned int restricted: 1;
340 unsigned int off_timeout_used: 1;355 unsigned int off_timeout_used: 1;
341 unsigned int drain_active: 1;356 unsigned int drain_active: 1;
342 unsigned int drain_disabled: 1;357 unsigned int drain_disabled: 1;
358 unsigned int has_evfd: 1;
343 } ____cacheline_aligned_in_smp;359 } ____cacheline_aligned_in_smp;
344 360
345 /* submission data */361 /* submission data */
@@ -378,7 +394,9 @@ struct io_ring_ctx { · 32 unmodified lines
378 struct list_head timeout_list;394 struct list_head timeout_list;
379 struct list_head ltimeout_list;395 struct list_head ltimeout_list;
380 struct list_head cq_overflow_list;396 struct list_head cq_overflow_list;
381 struct xarray io_buffers;397 struct list_head *io_buffers;
398 struct list_head io_buffers_cache;
399 struct list_head apoll_cache;
382 struct xarray personalities;400 struct xarray personalities;
383 u32 pers_next;401 u32 pers_next;
384 unsigned sq_thread_idle;402 unsigned sq_thread_idle;
@@ -395,11 +413,16 @@ struct io_ring_ctx { · 10 unmodified lines
395 struct list_head sqd_list;413 struct list_head sqd_list;
396 414
397 unsigned long check_cq_overflow;415 unsigned long check_cq_overflow;
416#ifdef CONFIG_NET_RX_BUSY_POLL
417 /* used to track busy poll napi_id */
418 struct list_head napi_list;
419 spinlock_t napi_lock; /* napi_list lock */
420#endif
398 421
399 struct {422 struct {
400 unsigned cached_cq_tail;423 unsigned cached_cq_tail;
401 unsigned cq_entries;424 unsigned cq_entries;
402 struct eventfd_ctx *cq_ev_fd;425 struct io_ev_fd __rcu *io_ev_fd;
403 struct wait_queue_head cq_wait;426 struct wait_queue_head cq_wait;
404 unsigned cq_extra;427 unsigned cq_extra;
405 atomic_t cq_timeouts;428 atomic_t cq_timeouts;
@@ -421,6 +444,8 @@ struct io_ring_ctx { · 15 unmodified lines
421 struct hlist_head *cancel_hash;444 struct hlist_head *cancel_hash;
422 unsigned cancel_hash_bits;445 unsigned cancel_hash_bits;
423 bool poll_multi_queue;446 bool poll_multi_queue;
447
448 struct list_head io_buffers_comp;
424 } ____cacheline_aligned_in_smp;449 } ____cacheline_aligned_in_smp;
425 450
426 struct io_restriction restrictions;451 struct io_restriction restrictions;
@@ -436,6 +461,8 @@ struct io_ring_ctx { · 9 unmodified lines
436 struct llist_head rsrc_put_llist;461 struct llist_head rsrc_put_llist;
437 struct list_head rsrc_ref_list;462 struct list_head rsrc_ref_list;
438 spinlock_t rsrc_ref_lock;463 spinlock_t rsrc_ref_lock;
464
465 struct list_head io_buffers_pages;
439 };466 };
440 467
441 /* Keep this last, we don't need it for the fast path */468 /* Keep this last, we don't need it for the fast path */
@@ -461,6 +488,11 @@ struct io_ring_ctx { · 19 unmodified lines
461 };488 };
462};489};
463 490
491/*
492 * Arbitrary limit, can be raised if need be
493 */
494#define IO_RINGFD_REG_MAX 16
495
464struct io_uring_task {496struct io_uring_task {
465 /* submission side */497 /* submission side */
466 int cached_refs;498 int cached_refs;
@@ -476,6 +508,7 @@ struct io_uring_task { · 9 unmodified lines
476 struct io_wq_work_list task_list;508 struct io_wq_work_list task_list;
477 struct io_wq_work_list prior_task_list;509 struct io_wq_work_list prior_task_list;
478 struct callback_head task_work;510 struct callback_head task_work;
511 struct file **registered_rings;
479 bool task_running;512 bool task_running;
480};513};
481 514
@@ -690,6 +723,12 @@ struct io_hardlink { · 208 unmodified lines
690 int flags;723 int flags;
691};724};
692 725
726struct io_msg {
727 struct file *file;
728 u64 user_data;
729 u32 len;
730};
731
693struct io_async_connect {732struct io_async_connect {
694 struct sockaddr_storage address;733 struct sockaddr_storage address;
695};734};
@@ -741,6 +780,8 @@ enum { · 45 unmodified lines
741 REQ_F_ARM_LTIMEOUT_BIT,780 REQ_F_ARM_LTIMEOUT_BIT,
742 REQ_F_ASYNC_DATA_BIT,781 REQ_F_ASYNC_DATA_BIT,
743 REQ_F_SKIP_LINK_CQES_BIT,782 REQ_F_SKIP_LINK_CQES_BIT,
783 REQ_F_SINGLE_POLL_BIT,
784 REQ_F_DOUBLE_POLL_BIT,
744 /* keep async read/write and isreg together and in order */785 /* keep async read/write and isreg together and in order */
745 REQ_F_SUPPORT_NOWAIT_BIT,786 REQ_F_SUPPORT_NOWAIT_BIT,
746 REQ_F_ISREG_BIT,787 REQ_F_ISREG_BIT,
@@ -799,6 +840,10 @@ enum { · 52 unmodified lines
799 REQ_F_ASYNC_DATA = BIT(REQ_F_ASYNC_DATA_BIT),840 REQ_F_ASYNC_DATA = BIT(REQ_F_ASYNC_DATA_BIT),
800 /* don't post CQEs while failing linked requests */841 /* don't post CQEs while failing linked requests */
801 REQ_F_SKIP_LINK_CQES = BIT(REQ_F_SKIP_LINK_CQES_BIT),842 REQ_F_SKIP_LINK_CQES = BIT(REQ_F_SKIP_LINK_CQES_BIT),
843 /* single poll may be active */
844 REQ_F_SINGLE_POLL = BIT(REQ_F_SINGLE_POLL_BIT),
845 /* double poll may active */
846 REQ_F_DOUBLE_POLL = BIT(REQ_F_DOUBLE_POLL_BIT),
802};847};
803 848
804struct async_poll {849struct async_poll {
@@ -825,7 +870,7 @@ enum { · 20 unmodified lines
825 * NOTE! Each of the iocb union members has the file pointer870 * NOTE! Each of the iocb union members has the file pointer
826 * as the first entry in their struct definition. So you can871 * as the first entry in their struct definition. So you can
827 * access the file pointer through any of the sub-structs,872 * access the file pointer through any of the sub-structs,
828 * or directly as just 'ki_filp' in this struct.873 * or directly as just 'file' in this struct.
829 */874 */
830struct io_kiocb {875struct io_kiocb {
831 union {876 union {
@@ -855,6 +900,7 @@ struct io_kiocb { · 23 unmodified lines
855 struct io_mkdir mkdir;900 struct io_mkdir mkdir;
856 struct io_symlink symlink;901 struct io_symlink symlink;
857 struct io_hardlink hardlink;902 struct io_hardlink hardlink;
903 struct io_msg msg;
858 };904 };
859 905
860 u8 opcode;906 u8 opcode;
@@ -877,6 +923,7 @@ struct io_kiocb { · 16 unmodified lines
877 /* used by request caches, completion batching and iopoll */923 /* used by request caches, completion batching and iopoll */
878 struct io_wq_work_node comp_list;924 struct io_wq_work_node comp_list;
879 atomic_t refs;925 atomic_t refs;
926 atomic_t poll_refs;
880 struct io_kiocb *link;927 struct io_kiocb *link;
881 struct io_task_work io_task_work;928 struct io_task_work io_task_work;
882 /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */929 /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
@@ -885,12 +932,11 @@ struct io_kiocb { · 2 unmodified lines
885 struct async_poll *apoll;932 struct async_poll *apoll;
886 /* opcode allocated if it needs to store data for async defer */933 /* opcode allocated if it needs to store data for async defer */
887 void *async_data;934 void *async_data;
888 struct io_wq_work work;
889 /* custom credentials, valid IFF REQ_F_CREDS is set */935 /* custom credentials, valid IFF REQ_F_CREDS is set */
890 const struct cred *creds;
891 /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */936 /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */
892 struct io_buffer *kbuf;937 struct io_buffer *kbuf;
893 atomic_t poll_refs;938 const struct cred *creds;
939 struct io_wq_work work;
894};940};
895 941
896struct io_tctx_node {942struct io_tctx_node {
@@ -1105,6 +1151,9 @@ static const struct io_op_def io_op_defs[] = { · 208 unmodified lines
1105 [IORING_OP_MKDIRAT] = {},1151 [IORING_OP_MKDIRAT] = {},
1106 [IORING_OP_SYMLINKAT] = {},1152 [IORING_OP_SYMLINKAT] = {},
1107 [IORING_OP_LINKAT] = {},1153 [IORING_OP_LINKAT] = {},
1154 [IORING_OP_MSG_RING] = {
1155 .needs_file = 1,
1156 },
1108};1157};
1109 1158
1110/* requests with any of those set should undergo io_disarm_next() */1159/* requests with any of those set should undergo io_disarm_next() */
@@ -1141,6 +1190,7 @@ static int io_install_fixed_file(struct io_kiocb *req, struct file *file, · 30 unmodified lines
1141static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags);1190static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags);
1142 1191
1143static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer);1192static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer);
1193static void io_eventfd_signal(struct io_ring_ctx *ctx);
1144 1194
1145static struct kmem_cache *req_cachep;1195static struct kmem_cache *req_cachep;
1146 1196
@@ -1267,36 +1317,88 @@ static inline void io_req_set_rsrc_node(struct io_kiocb *req, · 120 unmodified lines
1267 }1317 }
1268}1318}
1269 1319
1270static unsigned int __io_put_kbuf(struct io_kiocb *req)1320static unsigned int __io_put_kbuf(struct io_kiocb *req, struct list_head *list)
1271{1321{
1272 struct io_buffer *kbuf = req->kbuf;1322 struct io_buffer *kbuf = req->kbuf;
1273 unsigned int cflags;1323 unsigned int cflags;
1274 1324
1275 cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;1325 cflags = IORING_CQE_F_BUFFER | (kbuf->bid << IORING_CQE_BUFFER_SHIFT);
1276 cflags |= IORING_CQE_F_BUFFER;
1277 req->flags &= ~REQ_F_BUFFER_SELECTED;1326 req->flags &= ~REQ_F_BUFFER_SELECTED;
1278 kfree(kbuf);1327 list_add(&kbuf->list, list);
1279 req->kbuf = NULL;1328 req->kbuf = NULL;
1280 return cflags;1329 return cflags;
1281}1330}
1282 1331
1283static inline unsigned int io_put_kbuf(struct io_kiocb *req)1332static inline unsigned int io_put_kbuf_comp(struct io_kiocb *req)
1284{1333{
1285 if (likely(!(req->flags & REQ_F_BUFFER_SELECTED)))1334 if (likely(!(req->flags & REQ_F_BUFFER_SELECTED)))
1286 return 0;1335 return 0;
1287 return __io_put_kbuf(req);1336 return __io_put_kbuf(req, &req->ctx->io_buffers_comp);
1288}1337}
1289 1338
1290static void io_refs_resurrect(struct percpu_ref *ref, struct completion *compl)1339static inline unsigned int io_put_kbuf(struct io_kiocb *req,
1340 unsigned issue_flags)
1291{1341{
1292 bool got = percpu_ref_tryget(ref);1342 unsigned int cflags;
1343
1344 if (likely(!(req->flags & REQ_F_BUFFER_SELECTED)))
1345 return 0;
1346
1347 /*
1348 * We can add this buffer back to two lists:
1349 *
1350 * 1) The io_buffers_cache list. This one is protected by the
1351 * ctx->uring_lock. If we already hold this lock, add back to this
1352 * list as we can grab it from issue as well.
1353 * 2) The io_buffers_comp list. This one is protected by the
1354 * ctx->completion_lock.
1355 *
1356 * We migrate buffers from the comp_list to the issue cache list
1357 * when we need one.
1358 */
1359 if (issue_flags & IO_URING_F_UNLOCKED) {
1360 struct io_ring_ctx *ctx = req->ctx;
1293 1361
1294 /* already at zero, wait for ->release() */1362 spin_lock(&ctx->completion_lock);
1295 if (!got)1363 cflags = __io_put_kbuf(req, &ctx->io_buffers_comp);
1296 wait_for_completion(compl);1364 spin_unlock(&ctx->completion_lock);
1297 percpu_ref_resurrect(ref);1365 } else {
1298 if (got)1366 cflags = __io_put_kbuf(req, &req->ctx->io_buffers_cache);
1299 percpu_ref_put(ref);1367 }
1368
1369 return cflags;
1370}
1371
1372static struct io_buffer_list *io_buffer_get_list(struct io_ring_ctx *ctx,
1373 unsigned int bgid)
1374{
1375 struct list_head *hash_list;
1376 struct io_buffer_list *bl;
1377
1378 hash_list = &ctx->io_buffers[hash_32(bgid, IO_BUFFERS_HASH_BITS)];
1379 list_for_each_entry(bl, hash_list, list)
1380 if (bl->bgid == bgid || bgid == -1U)
1381 return bl;
1382
1383 return NULL;
1384}
1385
1386static void io_kbuf_recycle(struct io_kiocb *req)
1387{
1388 struct io_ring_ctx *ctx = req->ctx;
1389 struct io_buffer_list *bl;
1390 struct io_buffer *buf;
1391
1392 if (likely(!(req->flags & REQ_F_BUFFER_SELECTED)))
1393 return;
1394
1395 lockdep_assert_held(&ctx->uring_lock);
1396
1397 buf = req->kbuf;
1398 bl = io_buffer_get_list(ctx, buf->bgid);
1399 list_add(&buf->list, &bl->buf_list);
1400 req->flags &= ~REQ_F_BUFFER_SELECTED;
1401 req->kbuf = NULL;
1300}1402}
1301 1403
1302static bool io_match_task(struct io_kiocb *head, struct task_struct *task,1404static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
@@ -1409,7 +1511,7 @@ static __cold void io_fallback_req_func(struct work_struct *work) · 106 unmodified lines
1409static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)1511static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
1410{1512{
1411 struct io_ring_ctx *ctx;1513 struct io_ring_ctx *ctx;
1412 int hash_bits;1514 int i, hash_bits;
1413 1515
1414 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);1516 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1415 if (!ctx)1517 if (!ctx)
@@ -1436,6 +1538,13 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) · 20 unmodified lines
1436 /* set invalid range, so io_import_fixed() fails meeting it */1538 /* set invalid range, so io_import_fixed() fails meeting it */
1437 ctx->dummy_ubuf->ubuf = -1UL;1539 ctx->dummy_ubuf->ubuf = -1UL;
1438 1540
1541 ctx->io_buffers = kcalloc(1U << IO_BUFFERS_HASH_BITS,
1542 sizeof(struct list_head), GFP_KERNEL);
1543 if (!ctx->io_buffers)
1544 goto err;
1545 for (i = 0; i < (1U << IO_BUFFERS_HASH_BITS); i++)
1546 INIT_LIST_HEAD(&ctx->io_buffers[i]);
1547
1439 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,1548 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
1440 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))1549 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
1441 goto err;1550 goto err;
@@ -1444,14 +1553,17 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) · 2 unmodified lines
1444 init_waitqueue_head(&ctx->sqo_sq_wait);1553 init_waitqueue_head(&ctx->sqo_sq_wait);
1445 INIT_LIST_HEAD(&ctx->sqd_list);1554 INIT_LIST_HEAD(&ctx->sqd_list);
1446 INIT_LIST_HEAD(&ctx->cq_overflow_list);1555 INIT_LIST_HEAD(&ctx->cq_overflow_list);
1556 INIT_LIST_HEAD(&ctx->io_buffers_cache);
1557 INIT_LIST_HEAD(&ctx->apoll_cache);
1447 init_completion(&ctx->ref_comp);1558 init_completion(&ctx->ref_comp);
1448 xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1);
1449 xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);1559 xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
1450 mutex_init(&ctx->uring_lock);1560 mutex_init(&ctx->uring_lock);
1451 init_waitqueue_head(&ctx->cq_wait);1561 init_waitqueue_head(&ctx->cq_wait);
1452 spin_lock_init(&ctx->completion_lock);1562 spin_lock_init(&ctx->completion_lock);
1453 spin_lock_init(&ctx->timeout_lock);1563 spin_lock_init(&ctx->timeout_lock);
1454 INIT_WQ_LIST(&ctx->iopoll_list);1564 INIT_WQ_LIST(&ctx->iopoll_list);
1565 INIT_LIST_HEAD(&ctx->io_buffers_pages);
1566 INIT_LIST_HEAD(&ctx->io_buffers_comp);
1455 INIT_LIST_HEAD(&ctx->defer_list);1567 INIT_LIST_HEAD(&ctx->defer_list);
1456 INIT_LIST_HEAD(&ctx->timeout_list);1568 INIT_LIST_HEAD(&ctx->timeout_list);
1457 INIT_LIST_HEAD(&ctx->ltimeout_list);1569 INIT_LIST_HEAD(&ctx->ltimeout_list);
@@ -1464,10 +1576,15 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) · 6 unmodified lines
1464 INIT_WQ_LIST(&ctx->locked_free_list);1576 INIT_WQ_LIST(&ctx->locked_free_list);
1465 INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);1577 INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);
1466 INIT_WQ_LIST(&ctx->submit_state.compl_reqs);1578 INIT_WQ_LIST(&ctx->submit_state.compl_reqs);
1579#ifdef CONFIG_NET_RX_BUSY_POLL
1580 INIT_LIST_HEAD(&ctx->napi_list);
1581 spin_lock_init(&ctx->napi_lock);
1582#endif
1467 return ctx;1583 return ctx;
1468err:1584err:
1469 kfree(ctx->dummy_ubuf);1585 kfree(ctx->dummy_ubuf);
1470 kfree(ctx->cancel_hash);1586 kfree(ctx->cancel_hash);
1587 kfree(ctx->io_buffers);
1471 kfree(ctx);1588 kfree(ctx);
1472 return NULL;1589 return NULL;
1473}1590}
@@ -1610,8 +1727,8 @@ static void io_queue_async_work(struct io_kiocb *req, bool *dont_use) · 136 unmodified lines
1610 if (WARN_ON_ONCE(!same_thread_group(req->task, current)))1727 if (WARN_ON_ONCE(!same_thread_group(req->task, current)))
1611 req->work.flags |= IO_WQ_WORK_CANCEL;1728 req->work.flags |= IO_WQ_WORK_CANCEL;
1612 1729
1613 trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,1730 trace_io_uring_queue_async_work(ctx, req, req->user_data, req->opcode, req->flags,
1614 &req->work, req->flags);1731 &req->work, io_wq_is_hashed(&req->work));
1615 io_wq_enqueue(tctx->io_wq, &req->work);1732 io_wq_enqueue(tctx->io_wq, &req->work);
1616 if (link)1733 if (link)
1617 io_queue_linked_timeout(link);1734 io_queue_linked_timeout(link);
@@ -1681,22 +1798,27 @@ static __cold void io_flush_timeouts(struct io_ring_ctx *ctx) · 63 unmodified lines
1681 spin_unlock_irq(&ctx->timeout_lock);1798 spin_unlock_irq(&ctx->timeout_lock);
1682}1799}
1683 1800
1684static __cold void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
1685{
1686 if (ctx->off_timeout_used)
1687 io_flush_timeouts(ctx);
1688 if (ctx->drain_active)
1689 io_queue_deferred(ctx);
1690}
1691
1692static inline void io_commit_cqring(struct io_ring_ctx *ctx)1801static inline void io_commit_cqring(struct io_ring_ctx *ctx)
1693{1802{
1694 if (unlikely(ctx->off_timeout_used || ctx->drain_active))
1695 __io_commit_cqring_flush(ctx);
1696 /* order cqe stores with ring update */1803 /* order cqe stores with ring update */
1697 smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);1804 smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
1698}1805}
1699 1806
1807static void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
1808{
1809 if (ctx->off_timeout_used || ctx->drain_active) {
1810 spin_lock(&ctx->completion_lock);
1811 if (ctx->off_timeout_used)
1812 io_flush_timeouts(ctx);
1813 if (ctx->drain_active)
1814 io_queue_deferred(ctx);
1815 io_commit_cqring(ctx);
1816 spin_unlock(&ctx->completion_lock);
1817 }
1818 if (ctx->has_evfd)
1819 io_eventfd_signal(ctx);
1820}
1821
1700static inline bool io_sqring_full(struct io_ring_ctx *ctx)1822static inline bool io_sqring_full(struct io_ring_ctx *ctx)
1701{1823{
1702 struct io_rings *r = ctx->rings;1824 struct io_rings *r = ctx->rings;
@@ -1726,23 +1848,34 @@ static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx) · 23 unmodified lines
1726 return &rings->cqes[tail & mask];1848 return &rings->cqes[tail & mask];
1727}1849}
1728 1850
1729static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)1851static void io_eventfd_signal(struct io_ring_ctx *ctx)
1730{1852{
1731 if (likely(!ctx->cq_ev_fd))1853 struct io_ev_fd *ev_fd;
1732 return false;1854
1855 rcu_read_lock();
1856 /*
1857 * rcu_dereference ctx->io_ev_fd once and use it for both for checking
1858 * and eventfd_signal
1859 */
1860 ev_fd = rcu_dereference(ctx->io_ev_fd);
1861
1862 /*
1863 * Check again if ev_fd exists incase an io_eventfd_unregister call
1864 * completed between the NULL check of ctx->io_ev_fd at the start of
1865 * the function and rcu_read_lock.
1866 */
1867 if (unlikely(!ev_fd))
1868 goto out;
1733 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)1869 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
1734 return false;1870 goto out;
1735 return !ctx->eventfd_async || io_wq_current_is_worker();1871
1872 if (!ev_fd->eventfd_async || io_wq_current_is_worker())
1873 eventfd_signal(ev_fd->cq_ev_fd, 1);
1874out:
1875 rcu_read_unlock();
1736}1876}
1737 1877
1738/*1878static inline void io_cqring_wake(struct io_ring_ctx *ctx)
1739 * This should only get called when at least one event has been posted.
1740 * Some applications rely on the eventfd notification count only changing
1741 * IFF a new CQE has been added to the CQ ring. There's no depedency on
1742 * 1:1 relationship between how many times this function is called (and
1743 * hence the eventfd count) and number of CQEs posted to the CQ ring.
1744 */
1745static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1746{1879{
1747 /*1880 /*
1748 * wake_up_all() may seem excessive, but io_wake_function() and1881 * wake_up_all() may seem excessive, but io_wake_function() and
@@ -1751,21 +1884,32 @@ static void io_cqring_ev_posted(struct io_ring_ctx *ctx) · 2 unmodified lines
1751 */1884 */
1752 if (wq_has_sleeper(&ctx->cq_wait))1885 if (wq_has_sleeper(&ctx->cq_wait))
1753 wake_up_all(&ctx->cq_wait);1886 wake_up_all(&ctx->cq_wait);
1754 if (io_should_trigger_evfd(ctx))1887}
1755 eventfd_signal(ctx->cq_ev_fd, 1);1888
1889/*
1890 * This should only get called when at least one event has been posted.
1891 * Some applications rely on the eventfd notification count only changing
1892 * IFF a new CQE has been added to the CQ ring. There's no depedency on
1893 * 1:1 relationship between how many times this function is called (and
1894 * hence the eventfd count) and number of CQEs posted to the CQ ring.
1895 */
1896static inline void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1897{
1898 if (unlikely(ctx->off_timeout_used || ctx->drain_active ||
1899 ctx->has_evfd))
1900 __io_commit_cqring_flush(ctx);
1901
1902 io_cqring_wake(ctx);
1756}1903}
1757 1904
1758static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)1905static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)
1759{1906{
1760 /* see waitqueue_active() comment */1907 if (unlikely(ctx->off_timeout_used || ctx->drain_active ||
1761 smp_mb();1908 ctx->has_evfd))
1909 __io_commit_cqring_flush(ctx);
1762 1910
1763 if (ctx->flags & IORING_SETUP_SQPOLL) {1911 if (ctx->flags & IORING_SETUP_SQPOLL)
1764 if (waitqueue_active(&ctx->cq_wait))1912 io_cqring_wake(ctx);
1765 wake_up_all(&ctx->cq_wait);
1766 }
1767 if (io_should_trigger_evfd(ctx))
1768 eventfd_signal(ctx->cq_ev_fd, 1);
1769}1913}
1770 1914
1771/* Returns true if there are no backlogged entries after the flush */1915/* Returns true if there are no backlogged entries after the flush */
@@ -1905,8 +2049,6 @@ static inline bool __io_fill_cqe(struct io_ring_ctx *ctx, u64 user_data, · 133 unmodified lines
1905{2049{
1906 struct io_uring_cqe *cqe;2050 struct io_uring_cqe *cqe;
1907 2051
1908 trace_io_uring_complete(ctx, user_data, res, cflags);
1909
1910 /*2052 /*
1911 * If we can't get a cq entry, userspace overflowed the2053 * If we can't get a cq entry, userspace overflowed the
1912 * submission (by quite a lot). Increment the overflow count in2054 * submission (by quite a lot). Increment the overflow count in
@@ -1922,16 +2064,23 @@ static inline bool __io_fill_cqe(struct io_ring_ctx *ctx, u64 user_data, · 9 unmodified lines
1922 return io_cqring_event_overflow(ctx, user_data, res, cflags);2064 return io_cqring_event_overflow(ctx, user_data, res, cflags);
1923}2065}
1924 2066
2067static inline bool __io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)
2068{
2069 trace_io_uring_complete(req->ctx, req, req->user_data, res, cflags);
2070 return __io_fill_cqe(req->ctx, req->user_data, res, cflags);
2071}
2072
1925static noinline void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)2073static noinline void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)
1926{2074{
1927 if (!(req->flags & REQ_F_CQE_SKIP))2075 if (!(req->flags & REQ_F_CQE_SKIP))
1928 __io_fill_cqe(req->ctx, req->user_data, res, cflags);2076 __io_fill_cqe_req(req, res, cflags);
1929}2077}
1930 2078
1931static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data,2079static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data,
1932 s32 res, u32 cflags)2080 s32 res, u32 cflags)
1933{2081{
1934 ctx->cq_extra++;2082 ctx->cq_extra++;
2083 trace_io_uring_complete(ctx, NULL, user_data, res, cflags);
1935 return __io_fill_cqe(ctx, user_data, res, cflags);2084 return __io_fill_cqe(ctx, user_data, res, cflags);
1936}2085}
1937 2086
@@ -1941,7 +2090,7 @@ static void __io_req_complete_post(struct io_kiocb *req, s32 res, · 3 unmodified lines
1941 struct io_ring_ctx *ctx = req->ctx;2090 struct io_ring_ctx *ctx = req->ctx;
1942 2091
1943 if (!(req->flags & REQ_F_CQE_SKIP))2092 if (!(req->flags & REQ_F_CQE_SKIP))
1944 __io_fill_cqe(ctx, req->user_data, res, cflags);2093 __io_fill_cqe_req(req, res, cflags);
1945 /*2094 /*
1946 * If we're the last reference to this request, add to our locked2095 * If we're the last reference to this request, add to our locked
1947 * free_list cache.2096 * free_list cache.
@@ -2000,7 +2149,7 @@ static inline void io_req_complete(struct io_kiocb *req, s32 res) · 52 unmodified lines
2000static void io_req_complete_failed(struct io_kiocb *req, s32 res)2149static void io_req_complete_failed(struct io_kiocb *req, s32 res)
2001{2150{
2002 req_set_fail(req);2151 req_set_fail(req);
2003 io_req_complete_post(req, res, 0);2152 io_req_complete_post(req, res, io_put_kbuf(req, 0));
2004}2153}
2005 2154
2006static void io_req_complete_fail_submit(struct io_kiocb *req)2155static void io_req_complete_fail_submit(struct io_kiocb *req)
@@ -2183,7 +2332,9 @@ static void io_fail_links(struct io_kiocb *req) · 176 unmodified lines
2183 nxt = link->link;2332 nxt = link->link;
2184 link->link = NULL;2333 link->link = NULL;
2185 2334
2186 trace_io_uring_fail_link(req, link);2335 trace_io_uring_fail_link(req->ctx, req, req->user_data,
2336 req->opcode, link);
2337
2187 if (!ignore_cqes) {2338 if (!ignore_cqes) {
2188 link->flags &= ~REQ_F_CQE_SKIP;2339 link->flags &= ~REQ_F_CQE_SKIP;
2189 io_fill_cqe_req(link, res, 0);2340 io_fill_cqe_req(link, res, 0);
@@ -2302,7 +2453,8 @@ static void handle_prev_tw_list(struct io_wq_work_node *node, · 112 unmodified lines
2302 if (likely(*uring_locked))2453 if (likely(*uring_locked))
2303 req->io_task_work.func(req, uring_locked);2454 req->io_task_work.func(req, uring_locked);
2304 else2455 else
2305 __io_req_complete_post(req, req->result, io_put_kbuf(req));2456 __io_req_complete_post(req, req->result,
2457 io_put_kbuf_comp(req));
2306 node = next;2458 node = next;
2307 } while (node);2459 } while (node);
2308 2460
@@ -2530,8 +2682,16 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx) · 221 unmodified lines
2530 comp_list);2682 comp_list);
2531 2683
2532 if (!(req->flags & REQ_F_CQE_SKIP))2684 if (!(req->flags & REQ_F_CQE_SKIP))
2533 __io_fill_cqe(ctx, req->user_data, req->result,2685 __io_fill_cqe_req(req, req->result, req->cflags);
2534 req->cflags);2686 if ((req->flags & REQ_F_POLLED) && req->apoll) {
2687 struct async_poll *apoll = req->apoll;
2688
2689 if (apoll->double_poll)
2690 kfree(apoll->double_poll);
2691 list_add(&apoll->poll.wait.entry,
2692 &ctx->apoll_cache);
2693 req->flags &= ~REQ_F_POLLED;
2694 }
2535 }2695 }
2536 2696
2537 io_commit_cqring(ctx);2697 io_commit_cqring(ctx);
@@ -2653,7 +2813,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) · 115 unmodified lines
2653 if (unlikely(req->flags & REQ_F_CQE_SKIP))2813 if (unlikely(req->flags & REQ_F_CQE_SKIP))
2654 continue;2814 continue;
2655 2815
2656 __io_fill_cqe(ctx, req->user_data, req->result, io_put_kbuf(req));2816 __io_fill_cqe_req(req, req->result, io_put_kbuf(req, 0));
2657 nr_events++;2817 nr_events++;
2658 }2818 }
2659 2819
@@ -2829,14 +2989,14 @@ static bool __io_complete_rw_common(struct io_kiocb *req, long res) · 169 unmodified lines
2829 2989
2830static inline void io_req_task_complete(struct io_kiocb *req, bool *locked)2990static inline void io_req_task_complete(struct io_kiocb *req, bool *locked)
2831{2991{
2832 unsigned int cflags = io_put_kbuf(req);
2833 int res = req->result;2992 int res = req->result;
2834 2993
2835 if (*locked) {2994 if (*locked) {
2836 io_req_complete_state(req, res, cflags);2995 io_req_complete_state(req, res, io_put_kbuf(req, 0));
2837 io_req_add_compl_list(req);2996 io_req_add_compl_list(req);
2838 } else {2997 } else {
2839 io_req_complete_post(req, res, cflags);2998 io_req_complete_post(req, res,
2999 io_put_kbuf(req, IO_URING_F_UNLOCKED));
2840 }3000 }
2841}3001}
2842 3002
@@ -2845,7 +3005,8 @@ static void __io_complete_rw(struct io_kiocb *req, long res, · 2 unmodified lines
2845{3005{
2846 if (__io_complete_rw_common(req, res))3006 if (__io_complete_rw_common(req, res))
2847 return;3007 return;
2848 __io_req_complete(req, issue_flags, req->result, io_put_kbuf(req));3008 __io_req_complete(req, issue_flags, req->result,
3009 io_put_kbuf(req, issue_flags));
2849}3010}
2850 3011
2851static void io_complete_rw(struct kiocb *kiocb, long res)3012static void io_complete_rw(struct kiocb *kiocb, long res)
@@ -3000,14 +3161,6 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 148 unmodified lines
3000 req->flags |= io_file_get_flags(file) << REQ_F_SUPPORT_NOWAIT_BIT;3161 req->flags |= io_file_get_flags(file) << REQ_F_SUPPORT_NOWAIT_BIT;
3001 3162
3002 kiocb->ki_pos = READ_ONCE(sqe->off);3163 kiocb->ki_pos = READ_ONCE(sqe->off);
3003 if (kiocb->ki_pos == -1) {
3004 if (!(file->f_mode & FMODE_STREAM)) {
3005 req->flags |= REQ_F_CUR_POS;
3006 kiocb->ki_pos = file->f_pos;
3007 } else {
3008 kiocb->ki_pos = 0;
3009 }
3010 }
3011 kiocb->ki_flags = iocb_flags(file);3164 kiocb->ki_flags = iocb_flags(file);
3012 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));3165 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
3013 if (unlikely(ret))3166 if (unlikely(ret))
@@ -3074,6 +3227,24 @@ static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret) · 60 unmodified lines
3074 }3227 }
3075}3228}
3076 3229
3230static inline loff_t *io_kiocb_update_pos(struct io_kiocb *req)
3231{
3232 struct kiocb *kiocb = &req->rw.kiocb;
3233 bool is_stream = req->file->f_mode & FMODE_STREAM;
3234
3235 if (kiocb->ki_pos == -1) {
3236 if (!is_stream) {
3237 req->flags |= REQ_F_CUR_POS;
3238 kiocb->ki_pos = req->file->f_pos;
3239 return &kiocb->ki_pos;
3240 } else {
3241 kiocb->ki_pos = 0;
3242 return NULL;
3243 }
3244 }
3245 return is_stream ? NULL : &kiocb->ki_pos;
3246}
3247
3077static void kiocb_done(struct io_kiocb *req, ssize_t ret,3248static void kiocb_done(struct io_kiocb *req, ssize_t ret,
3078 unsigned int issue_flags)3249 unsigned int issue_flags)
3079{3250{
@@ -3096,14 +3267,10 @@ static void kiocb_done(struct io_kiocb *req, ssize_t ret, · 16 unmodified lines
3096 3267
3097 if (req->flags & REQ_F_REISSUE) {3268 if (req->flags & REQ_F_REISSUE) {
3098 req->flags &= ~REQ_F_REISSUE;3269 req->flags &= ~REQ_F_REISSUE;
3099 if (io_resubmit_prep(req)) {3270 if (io_resubmit_prep(req))
3100 io_req_task_queue_reissue(req);3271 io_req_task_queue_reissue(req);
3101 } else {3272 else
3102 req_set_fail(req);3273 io_req_task_queue_fail(req, ret);
3103 req->result = ret;
3104 req->io_task_work.func = io_req_task_complete;
3105 io_req_task_work_add(req, false);
3106 }
3107 }3274 }
3108}3275}
3109 3276
@@ -3201,30 +3368,36 @@ static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock) · 91 unmodified lines
3201 mutex_lock(&ctx->uring_lock);3368 mutex_lock(&ctx->uring_lock);
3202}3369}
3203 3370
3371static void io_buffer_add_list(struct io_ring_ctx *ctx,
3372 struct io_buffer_list *bl, unsigned int bgid)
3373{
3374 struct list_head *list;
3375
3376 list = &ctx->io_buffers[hash_32(bgid, IO_BUFFERS_HASH_BITS)];
3377 INIT_LIST_HEAD(&bl->buf_list);
3378 bl->bgid = bgid;
3379 list_add(&bl->list, list);
3380}
3381
3204static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,3382static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,
3205 int bgid, unsigned int issue_flags)3383 int bgid, unsigned int issue_flags)
3206{3384{
3207 struct io_buffer *kbuf = req->kbuf;3385 struct io_buffer *kbuf = req->kbuf;
3208 struct io_buffer *head;
3209 bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;3386 bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
3387 struct io_ring_ctx *ctx = req->ctx;
3388 struct io_buffer_list *bl;
3210 3389
3211 if (req->flags & REQ_F_BUFFER_SELECTED)3390 if (req->flags & REQ_F_BUFFER_SELECTED)
3212 return kbuf;3391 return kbuf;
3213 3392
3214 io_ring_submit_lock(req->ctx, needs_lock);3393 io_ring_submit_lock(ctx, needs_lock);
3215 3394
3216 lockdep_assert_held(&req->ctx->uring_lock);3395 lockdep_assert_held(&ctx->uring_lock);
3217 3396
3218 head = xa_load(&req->ctx->io_buffers, bgid);3397 bl = io_buffer_get_list(ctx, bgid);
3219 if (head) {3398 if (bl && !list_empty(&bl->buf_list)) {
3220 if (!list_empty(&head->list)) {3399 kbuf = list_first_entry(&bl->buf_list, struct io_buffer, list);
3221 kbuf = list_last_entry(&head->list, struct io_buffer,3400 list_del(&kbuf->list);
3222 list);
3223 list_del(&kbuf->list);
3224 } else {
3225 kbuf = head;
3226 xa_erase(&req->ctx->io_buffers, bgid);
3227 }
3228 if (*len > kbuf->len)3401 if (*len > kbuf->len)
3229 *len = kbuf->len;3402 *len = kbuf->len;
3230 req->flags |= REQ_F_BUFFER_SELECTED;3403 req->flags |= REQ_F_BUFFER_SELECTED;
@@ -3400,6 +3573,7 @@ static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter) · 169 unmodified lines
3400 struct kiocb *kiocb = &req->rw.kiocb;3573 struct kiocb *kiocb = &req->rw.kiocb;
3401 struct file *file = req->file;3574 struct file *file = req->file;
3402 ssize_t ret = 0;3575 ssize_t ret = 0;
3576 loff_t *ppos;
3403 3577
3404 /*3578 /*
3405 * Don't support polled IO through this interface, and we can't3579 * Don't support polled IO through this interface, and we can't
@@ -3412,6 +3586,8 @@ static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter) · 6 unmodified lines
3412 !(kiocb->ki_filp->f_flags & O_NONBLOCK))3586 !(kiocb->ki_filp->f_flags & O_NONBLOCK))
3413 return -EAGAIN;3587 return -EAGAIN;
3414 3588
3589 ppos = io_kiocb_ppos(kiocb);
3590
3415 while (iov_iter_count(iter)) {3591 while (iov_iter_count(iter)) {
3416 struct iovec iovec;3592 struct iovec iovec;
3417 ssize_t nr;3593 ssize_t nr;
@@ -3425,10 +3601,10 @@ static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter) · 7 unmodified lines
3425 3601
3426 if (rw == READ) {3602 if (rw == READ) {
3427 nr = file->f_op->read(file, iovec.iov_base,3603 nr = file->f_op->read(file, iovec.iov_base,
3428 iovec.iov_len, io_kiocb_ppos(kiocb));3604 iovec.iov_len, ppos);
3429 } else {3605 } else {
3430 nr = file->f_op->write(file, iovec.iov_base,3606 nr = file->f_op->write(file, iovec.iov_base,
3431 iovec.iov_len, io_kiocb_ppos(kiocb));3607 iovec.iov_len, ppos);
3432 }3608 }
3433 3609
3434 if (nr < 0) {3610 if (nr < 0) {
@@ -3436,13 +3612,15 @@ static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter) · 1 unmodified line
3436 ret = nr;3612 ret = nr;
3437 break;3613 break;
3438 }3614 }
3615 ret += nr;
3439 if (!iov_iter_is_bvec(iter)) {3616 if (!iov_iter_is_bvec(iter)) {
3440 iov_iter_advance(iter, nr);3617 iov_iter_advance(iter, nr);
3441 } else {3618 } else {
3442 req->rw.len -= nr;
3443 req->rw.addr += nr;3619 req->rw.addr += nr;
3620 req->rw.len -= nr;
3621 if (!req->rw.len)
3622 break;
3444 }3623 }
3445 ret += nr;
3446 if (nr != iovec.iov_len)3624 if (nr != iovec.iov_len)
3447 break;3625 break;
3448 }3626 }
@@ -3629,12 +3807,23 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) · 180 unmodified lines
3629 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;3807 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
3630 struct io_async_rw *rw;3808 struct io_async_rw *rw;
3631 ssize_t ret, ret2;3809 ssize_t ret, ret2;
3810 loff_t *ppos;
3632 3811
3633 if (!req_has_async_data(req)) {3812 if (!req_has_async_data(req)) {
3634 ret = io_import_iovec(READ, req, &iovec, s, issue_flags);3813 ret = io_import_iovec(READ, req, &iovec, s, issue_flags);
3635 if (unlikely(ret < 0))3814 if (unlikely(ret < 0))
3636 return ret;3815 return ret;
3637 } else {3816 } else {
3817 /*
3818 * Safe and required to re-import if we're using provided
3819 * buffers, as we dropped the selected one before retry.
3820 */
3821 if (req->flags & REQ_F_BUFFER_SELECT) {
3822 ret = io_import_iovec(READ, req, &iovec, s, issue_flags);
3823 if (unlikely(ret < 0))
3824 return ret;
3825 }
3826
3638 rw = req->async_data;3827 rw = req->async_data;
3639 s = &rw->s;3828 s = &rw->s;
3640 /*3829 /*
@@ -3659,7 +3848,9 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) · 18 unmodified lines
3659 kiocb->ki_flags &= ~IOCB_NOWAIT;3848 kiocb->ki_flags &= ~IOCB_NOWAIT;
3660 }3849 }
3661 3850
3662 ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), req->result);3851 ppos = io_kiocb_update_pos(req);
3852
3853 ret = rw_verify_area(READ, req->file, ppos, req->result);
3663 if (unlikely(ret)) {3854 if (unlikely(ret)) {
3664 kfree(iovec);3855 kfree(iovec);
3665 return ret;3856 return ret;
@@ -3669,6 +3860,9 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) · 3 unmodified lines
3669 3860
3670 if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) {3861 if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) {
3671 req->flags &= ~REQ_F_REISSUE;3862 req->flags &= ~REQ_F_REISSUE;
3863 /* if we can poll, just do that */
3864 if (req->opcode == IORING_OP_READ && file_can_poll(req->file))
3865 return -EAGAIN;
3672 /* IOPOLL retry should happen for io-wq threads */3866 /* IOPOLL retry should happen for io-wq threads */
3673 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))3867 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
3674 goto done;3868 goto done;
@@ -3758,6 +3952,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) · 83 unmodified lines
3758 struct kiocb *kiocb = &req->rw.kiocb;3952 struct kiocb *kiocb = &req->rw.kiocb;
3759 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;3953 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
3760 ssize_t ret, ret2;3954 ssize_t ret, ret2;
3955 loff_t *ppos;
3761 3956
3762 if (!req_has_async_data(req)) {3957 if (!req_has_async_data(req)) {
3763 ret = io_import_iovec(WRITE, req, &iovec, s, issue_flags);3958 ret = io_import_iovec(WRITE, req, &iovec, s, issue_flags);
@@ -3788,7 +3983,9 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) · 24 unmodified lines
3788 kiocb->ki_flags &= ~IOCB_NOWAIT;3983 kiocb->ki_flags &= ~IOCB_NOWAIT;
3789 }3984 }
3790 3985
3791 ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), req->result);3986 ppos = io_kiocb_update_pos(req);
3987
3988 ret = rw_verify_area(WRITE, req->file, ppos, req->result);
3792 if (unlikely(ret))3989 if (unlikely(ret))
3793 goto out_free;3990 goto out_free;
3794 3991
@@ -4235,6 +4432,45 @@ static int io_nop(struct io_kiocb *req, unsigned int issue_flags) · 440 unmodified lines
4235 return 0;4432 return 0;
4236}4433}
4237 4434
4435static int io_msg_ring_prep(struct io_kiocb *req,
4436 const struct io_uring_sqe *sqe)
4437{
4438 if (unlikely(sqe->addr || sqe->ioprio || sqe->rw_flags ||
4439 sqe->splice_fd_in || sqe->buf_index || sqe->personality))
4440 return -EINVAL;
4441
4442 if (req->file->f_op != &io_uring_fops)
4443 return -EBADFD;
4444
4445 req->msg.user_data = READ_ONCE(sqe->off);
4446 req->msg.len = READ_ONCE(sqe->len);
4447 return 0;
4448}
4449
4450static int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags)
4451{
4452 struct io_ring_ctx *target_ctx;
4453 struct io_msg *msg = &req->msg;
4454 int ret = -EOVERFLOW;
4455 bool filled;
4456
4457 target_ctx = req->file->private_data;
4458
4459 spin_lock(&target_ctx->completion_lock);
4460 filled = io_fill_cqe_aux(target_ctx, msg->user_data, msg->len,
4461 IORING_CQE_F_MSG);
4462 io_commit_cqring(target_ctx);
4463 spin_unlock(&target_ctx->completion_lock);
4464
4465 if (filled) {
4466 io_cqring_ev_posted(target_ctx);
4467 ret = 0;
4468 }
4469
4470 __io_req_complete(req, issue_flags, ret, 0);
4471 return 0;
4472}
4473
4238static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)4474static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4239{4475{
4240 struct io_ring_ctx *ctx = req->ctx;4476 struct io_ring_ctx *ctx = req->ctx;
@@ -4458,8 +4694,8 @@ static int io_remove_buffers_prep(struct io_kiocb *req, · 217 unmodified lines
4458 return 0;4694 return 0;
4459}4695}
4460 4696
4461static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,4697static int __io_remove_buffers(struct io_ring_ctx *ctx,
4462 int bgid, unsigned nbufs)4698 struct io_buffer_list *bl, unsigned nbufs)
4463{4699{
4464 unsigned i = 0;4700 unsigned i = 0;
4465 4701
@@ -4468,19 +4704,16 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf, · 2 unmodified lines
4468 return 0;4704 return 0;
4469 4705
4470 /* the head kbuf is the list itself */4706 /* the head kbuf is the list itself */
4471 while (!list_empty(&buf->list)) {4707 while (!list_empty(&bl->buf_list)) {
4472 struct io_buffer *nxt;4708 struct io_buffer *nxt;
4473 4709
4474 nxt = list_first_entry(&buf->list, struct io_buffer, list);4710 nxt = list_first_entry(&bl->buf_list, struct io_buffer, list);
4475 list_del(&nxt->list);4711 list_del(&nxt->list);
4476 kfree(nxt);
4477 if (++i == nbufs)4712 if (++i == nbufs)
4478 return i;4713 return i;
4479 cond_resched();4714 cond_resched();
4480 }4715 }
4481 i++;4716 i++;
4482 kfree(buf);
4483 xa_erase(&ctx->io_buffers, bgid);
4484 4717
4485 return i;4718 return i;
4486}4719}
@@ -4489,7 +4722,7 @@ static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) · 2 unmodified lines
4489{4722{
4490 struct io_provide_buf *p = &req->pbuf;4723 struct io_provide_buf *p = &req->pbuf;
4491 struct io_ring_ctx *ctx = req->ctx;4724 struct io_ring_ctx *ctx = req->ctx;
4492 struct io_buffer *head;4725 struct io_buffer_list *bl;
4493 int ret = 0;4726 int ret = 0;
4494 bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;4727 bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
4495 4728
@@ -4498,9 +4731,9 @@ static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) · 2 unmodified lines
4498 lockdep_assert_held(&ctx->uring_lock);4731 lockdep_assert_held(&ctx->uring_lock);
4499 4732
4500 ret = -ENOENT;4733 ret = -ENOENT;
4501 head = xa_load(&ctx->io_buffers, p->bgid);4734 bl = io_buffer_get_list(ctx, p->bgid);
4502 if (head)4735 if (bl)
4503 ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);4736 ret = __io_remove_buffers(ctx, bl, p->nbufs);
4504 if (ret < 0)4737 if (ret < 0)
4505 req_set_fail(req);4738 req_set_fail(req);
4506 4739
@@ -4545,39 +4778,80 @@ static int io_provide_buffers_prep(struct io_kiocb *req, · 38 unmodified lines
4545 return 0;4778 return 0;
4546}4779}
4547 4780
4548static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)4781static int io_refill_buffer_cache(struct io_ring_ctx *ctx)
4782{
4783 struct io_buffer *buf;
4784 struct page *page;
4785 int bufs_in_page;
4786
4787 /*
4788 * Completions that don't happen inline (eg not under uring_lock) will
4789 * add to ->io_buffers_comp. If we don't have any free buffers, check
4790 * the completion list and splice those entries first.
4791 */
4792 if (!list_empty_careful(&ctx->io_buffers_comp)) {
4793 spin_lock(&ctx->completion_lock);
4794 if (!list_empty(&ctx->io_buffers_comp)) {
4795 list_splice_init(&ctx->io_buffers_comp,
4796 &ctx->io_buffers_cache);
4797 spin_unlock(&ctx->completion_lock);
4798 return 0;
4799 }
4800 spin_unlock(&ctx->completion_lock);
4801 }
4802
4803 /*
4804 * No free buffers and no completion entries either. Allocate a new
4805 * page worth of buffer entries and add those to our freelist.
4806 */
4807 page = alloc_page(GFP_KERNEL_ACCOUNT);
4808 if (!page)
4809 return -ENOMEM;
4810
4811 list_add(&page->lru, &ctx->io_buffers_pages);
4812
4813 buf = page_address(page);
4814 bufs_in_page = PAGE_SIZE / sizeof(*buf);
4815 while (bufs_in_page) {
4816 list_add_tail(&buf->list, &ctx->io_buffers_cache);
4817 buf++;
4818 bufs_in_page--;
4819 }
4820
4821 return 0;
4822}
4823
4824static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf,
4825 struct io_buffer_list *bl)
4549{4826{
4550 struct io_buffer *buf;4827 struct io_buffer *buf;
4551 u64 addr = pbuf->addr;4828 u64 addr = pbuf->addr;
4552 int i, bid = pbuf->bid;4829 int i, bid = pbuf->bid;
4553 4830
4554 for (i = 0; i < pbuf->nbufs; i++) {4831 for (i = 0; i < pbuf->nbufs; i++) {
4555 buf = kmalloc(sizeof(*buf), GFP_KERNEL_ACCOUNT);4832 if (list_empty(&ctx->io_buffers_cache) &&
4556 if (!buf)4833 io_refill_buffer_cache(ctx))
4557 break;4834 break;
4558 4835 buf = list_first_entry(&ctx->io_buffers_cache, struct io_buffer,
4836 list);
4837 list_move_tail(&buf->list, &bl->buf_list);
4559 buf->addr = addr;4838 buf->addr = addr;
4560 buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT);4839 buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT);
4561 buf->bid = bid;4840 buf->bid = bid;
4841 buf->bgid = pbuf->bgid;
4562 addr += pbuf->len;4842 addr += pbuf->len;
4563 bid++;4843 bid++;
4564 if (!*head) {
4565 INIT_LIST_HEAD(&buf->list);
4566 *head = buf;
4567 } else {
4568 list_add_tail(&buf->list, &(*head)->list);
4569 }
4570 cond_resched();4844 cond_resched();
4571 }4845 }
4572 4846
4573 return i ? i : -ENOMEM;4847 return i ? 0 : -ENOMEM;
4574}4848}
4575 4849
4576static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)4850static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
4577{4851{
4578 struct io_provide_buf *p = &req->pbuf;4852 struct io_provide_buf *p = &req->pbuf;
4579 struct io_ring_ctx *ctx = req->ctx;4853 struct io_ring_ctx *ctx = req->ctx;
4580 struct io_buffer *head, *list;4854 struct io_buffer_list *bl;
4581 int ret = 0;4855 int ret = 0;
4582 bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;4856 bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
4583 4857
@@ -4585,14 +4859,18 @@ static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags) · 1 unmodified line
4585 4859
4586 lockdep_assert_held(&ctx->uring_lock);4860 lockdep_assert_held(&ctx->uring_lock);
4587 4861
4588 list = head = xa_load(&ctx->io_buffers, p->bgid);4862 bl = io_buffer_get_list(ctx, p->bgid);
4589 4863 if (unlikely(!bl)) {
4590 ret = io_add_buffers(p, &head);4864 bl = kmalloc(sizeof(*bl), GFP_KERNEL);
4591 if (ret >= 0 && !list) {4865 if (!bl) {
4592 ret = xa_insert(&ctx->io_buffers, p->bgid, head, GFP_KERNEL);4866 ret = -ENOMEM;
4593 if (ret < 0)4867 goto err;
4594 __io_remove_buffers(ctx, head, p->bgid, -1U);4868 }
4869 io_buffer_add_list(ctx, bl, p->bgid);
4595 }4870 }
4871
4872 ret = io_add_buffers(ctx, p, bl);
4873err:
4596 if (ret < 0)4874 if (ret < 0)
4597 req_set_fail(req);4875 req_set_fail(req);
4598 /* complete before unlock, IOPOLL may need the lock */4876 /* complete before unlock, IOPOLL may need the lock */
@@ -5184,7 +5462,7 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) · 585 unmodified lines
5184 if (kmsg->free_iov)5462 if (kmsg->free_iov)
5185 kfree(kmsg->free_iov);5463 kfree(kmsg->free_iov);
5186 req->flags &= ~REQ_F_NEED_CLEANUP;5464 req->flags &= ~REQ_F_NEED_CLEANUP;
5187 __io_req_complete(req, issue_flags, ret, io_put_kbuf(req));5465 __io_req_complete(req, issue_flags, ret, io_put_kbuf(req, issue_flags));
5188 return 0;5466 return 0;
5189}5467}
5190 5468
@@ -5239,7 +5517,8 @@ static int io_recv(struct io_kiocb *req, unsigned int issue_flags) · 48 unmodified lines
5239out_free:5517out_free:
5240 req_set_fail(req);5518 req_set_fail(req);
5241 }5519 }
5242 __io_req_complete(req, issue_flags, ret, io_put_kbuf(req));5520
5521 __io_req_complete(req, issue_flags, ret, io_put_kbuf(req, issue_flags));
5243 return 0;5522 return 0;
5244}5523}
5245 5524
@@ -5258,8 +5537,7 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 12 unmodified lines
5258 accept->nofile = rlimit(RLIMIT_NOFILE);5537 accept->nofile = rlimit(RLIMIT_NOFILE);
5259 5538
5260 accept->file_slot = READ_ONCE(sqe->file_index);5539 accept->file_slot = READ_ONCE(sqe->file_index);
5261 if (accept->file_slot && ((req->open.how.flags & O_CLOEXEC) ||5540 if (accept->file_slot && (accept->flags & SOCK_CLOEXEC))
5262 (accept->flags & SOCK_CLOEXEC)))
5263 return -EINVAL;5541 return -EINVAL;
5264 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))5542 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
5265 return -EINVAL;5543 return -EINVAL;
@@ -5399,6 +5677,108 @@ IO_NETOP_FN(send); · 133 unmodified lines
5399IO_NETOP_FN(recv);5677IO_NETOP_FN(recv);
5400#endif /* CONFIG_NET */5678#endif /* CONFIG_NET */
5401 5679
5680#ifdef CONFIG_NET_RX_BUSY_POLL
5681
5682#define NAPI_TIMEOUT (60 * SEC_CONVERSION)
5683
5684struct napi_entry {
5685 struct list_head list;
5686 unsigned int napi_id;
5687 unsigned long timeout;
5688};
5689
5690/*
5691 * Add busy poll NAPI ID from sk.
5692 */
5693static void io_add_napi(struct file *file, struct io_ring_ctx *ctx)
5694{
5695 unsigned int napi_id;
5696 struct socket *sock;
5697 struct sock *sk;
5698 struct napi_entry *ne;
5699
5700 if (!net_busy_loop_on())
5701 return;
5702
5703 sock = sock_from_file(file);
5704 if (!sock)
5705 return;
5706
5707 sk = sock->sk;
5708 if (!sk)
5709 return;
5710
5711 napi_id = READ_ONCE(sk->sk_napi_id);
5712
5713 /* Non-NAPI IDs can be rejected */
5714 if (napi_id < MIN_NAPI_ID)
5715 return;
5716
5717 spin_lock(&ctx->napi_lock);
5718 list_for_each_entry(ne, &ctx->napi_list, list) {
5719 if (ne->napi_id == napi_id) {
5720 ne->timeout = jiffies + NAPI_TIMEOUT;
5721 goto out;
5722 }
5723 }
5724
5725 ne = kmalloc(sizeof(*ne), GFP_NOWAIT);
5726 if (!ne)
5727 goto out;
5728
5729 ne->napi_id = napi_id;
5730 ne->timeout = jiffies + NAPI_TIMEOUT;
5731 list_add_tail(&ne->list, &ctx->napi_list);
5732out:
5733 spin_unlock(&ctx->napi_lock);
5734}
5735
5736static inline void io_check_napi_entry_timeout(struct napi_entry *ne)
5737{
5738 if (time_after(jiffies, ne->timeout)) {
5739 list_del(&ne->list);
5740 kfree(ne);
5741 }
5742}
5743
5744/*
5745 * Busy poll if globally on and supporting sockets found
5746 */
5747static bool io_napi_busy_loop(struct list_head *napi_list)
5748{
5749 struct napi_entry *ne, *n;
5750
5751 list_for_each_entry_safe(ne, n, napi_list, list) {
5752 napi_busy_loop(ne->napi_id, NULL, NULL, true,
5753 BUSY_POLL_BUDGET);
5754 io_check_napi_entry_timeout(ne);
5755 }
5756 return !list_empty(napi_list);
5757}
5758
5759static void io_free_napi_list(struct io_ring_ctx *ctx)
5760{
5761 spin_lock(&ctx->napi_lock);
5762 while (!list_empty(&ctx->napi_list)) {
5763 struct napi_entry *ne =
5764 list_first_entry(&ctx->napi_list, struct napi_entry,
5765 list);
5766
5767 list_del(&ne->list);
5768 kfree(ne);
5769 }
5770 spin_unlock(&ctx->napi_lock);
5771}
5772#else
5773static inline void io_add_napi(struct file *file, struct io_ring_ctx *ctx)
5774{
5775}
5776
5777static inline void io_free_napi_list(struct io_ring_ctx *ctx)
5778{
5779}
5780#endif /* CONFIG_NET_RX_BUSY_POLL */
5781
5402struct io_poll_table {5782struct io_poll_table {
5403 struct poll_table_struct pt;5783 struct poll_table_struct pt;
5404 struct io_kiocb *req;5784 struct io_kiocb *req;
@@ -5474,8 +5854,12 @@ static inline void io_poll_remove_entry(struct io_poll_iocb *poll) · 69 unmodified lines
5474 5854
5475static void io_poll_remove_entries(struct io_kiocb *req)5855static void io_poll_remove_entries(struct io_kiocb *req)
5476{5856{
5477 struct io_poll_iocb *poll = io_poll_get_single(req);5857 /*
5478 struct io_poll_iocb *poll_double = io_poll_get_double(req);5858 * Nothing to do if neither of those flags are set. Avoid dipping
5859 * into the poll/apoll/double cachelines if we can.
5860 */
5861 if (!(req->flags & (REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL)))
5862 return;
5479 5863
5480 /*5864 /*
5481 * While we hold the waitqueue lock and the waitqueue is nonempty,5865 * While we hold the waitqueue lock and the waitqueue is nonempty,
@@ -5493,9 +5877,10 @@ static void io_poll_remove_entries(struct io_kiocb *req) · 11 unmodified lines
5493 * In that case, only RCU prevents the queue memory from being freed.5877 * In that case, only RCU prevents the queue memory from being freed.
5494 */5878 */
5495 rcu_read_lock();5879 rcu_read_lock();
5496 io_poll_remove_entry(poll);5880 if (req->flags & REQ_F_SINGLE_POLL)
5497 if (poll_double)5881 io_poll_remove_entry(io_poll_get_single(req));
5498 io_poll_remove_entry(poll_double);5882 if (req->flags & REQ_F_DOUBLE_POLL)
5883 io_poll_remove_entry(io_poll_get_double(req));
5499 rcu_read_unlock();5884 rcu_read_unlock();
5500}5885}
5501 5886
@@ -5527,13 +5912,13 @@ static int io_poll_check_events(struct io_kiocb *req) · 25 unmodified lines
5527 return -ECANCELED;5912 return -ECANCELED;
5528 5913
5529 if (!req->result) {5914 if (!req->result) {
5530 struct poll_table_struct pt = { ._key = poll->events };5915 struct poll_table_struct pt = { ._key = req->cflags };
5531 5916
5532 req->result = vfs_poll(req->file, &pt) & poll->events;5917 req->result = vfs_poll(req->file, &pt) & req->cflags;
5533 }5918 }
5534 5919
5535 /* multishot, just fill an CQE and proceed */5920 /* multishot, just fill an CQE and proceed */
5536 if (req->result && !(poll->events & EPOLLONESHOT)) {5921 if (req->result && !(req->cflags & EPOLLONESHOT)) {
5537 __poll_t mask = mangle_poll(req->result & poll->events);5922 __poll_t mask = mangle_poll(req->result & poll->events);
5538 bool filled;5923 bool filled;
5539 5924
@@ -5545,6 +5930,7 @@ static int io_poll_check_events(struct io_kiocb *req) · 5 unmodified lines
5545 if (unlikely(!filled))5930 if (unlikely(!filled))
5546 return -ECANCELED;5931 return -ECANCELED;
5547 io_cqring_ev_posted(ctx);5932 io_cqring_ev_posted(ctx);
5933 io_add_napi(req->file, ctx);
5548 } else if (req->result) {5934 } else if (req->result) {
5549 return 0;5935 return 0;
5550 }5936 }
@@ -5603,29 +5989,36 @@ static void io_apoll_task_func(struct io_kiocb *req, bool *locked) · 52 unmodified lines
5603 io_req_complete_failed(req, ret);5989 io_req_complete_failed(req, ret);
5604}5990}
5605 5991
5606static void __io_poll_execute(struct io_kiocb *req, int mask)5992static void __io_poll_execute(struct io_kiocb *req, int mask, int events)
5607{5993{
5608 req->result = mask;5994 req->result = mask;
5995 /*
5996 * This is useful for poll that is armed on behalf of another
5997 * request, and where the wakeup path could be on a different
5998 * CPU. We want to avoid pulling in req->apoll->events for that
5999 * case.
6000 */
6001 req->cflags = events;
5609 if (req->opcode == IORING_OP_POLL_ADD)6002 if (req->opcode == IORING_OP_POLL_ADD)
5610 req->io_task_work.func = io_poll_task_func;6003 req->io_task_work.func = io_poll_task_func;
5611 else6004 else
5612 req->io_task_work.func = io_apoll_task_func;6005 req->io_task_work.func = io_apoll_task_func;
5613 6006
5614 trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);6007 trace_io_uring_task_add(req->ctx, req, req->user_data, req->opcode, mask);
5615 io_req_task_work_add(req, false);6008 io_req_task_work_add(req, false);
5616}6009}
5617 6010
5618static inline void io_poll_execute(struct io_kiocb *req, int res)6011static inline void io_poll_execute(struct io_kiocb *req, int res, int events)
5619{6012{
5620 if (io_poll_get_ownership(req))6013 if (io_poll_get_ownership(req))
5621 __io_poll_execute(req, res);6014 __io_poll_execute(req, res, events);
5622}6015}
5623 6016
5624static void io_poll_cancel_req(struct io_kiocb *req)6017static void io_poll_cancel_req(struct io_kiocb *req)
5625{6018{
5626 io_poll_mark_cancelled(req);6019 io_poll_mark_cancelled(req);
5627 /* kick tw, which should complete the request */6020 /* kick tw, which should complete the request */
5628 io_poll_execute(req, 0);6021 io_poll_execute(req, 0, 0);
5629}6022}
5630 6023
5631static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,6024static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
@@ -5639,7 +6032,7 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, · 7 unmodified lines
5639 if (unlikely(mask & POLLFREE)) {6032 if (unlikely(mask & POLLFREE)) {
5640 io_poll_mark_cancelled(req);6033 io_poll_mark_cancelled(req);
5641 /* we have to kick tw in case it's not already */6034 /* we have to kick tw in case it's not already */
5642 io_poll_execute(req, 0);6035 io_poll_execute(req, 0, poll->events);
5643 6036
5644 /*6037 /*
5645 * If the waitqueue is being freed early but someone is already6038 * If the waitqueue is being freed early but someone is already
@@ -5669,8 +6062,9 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, · 23 unmodified lines
5669 if (mask && poll->events & EPOLLONESHOT) {6062 if (mask && poll->events & EPOLLONESHOT) {
5670 list_del_init(&poll->wait.entry);6063 list_del_init(&poll->wait.entry);
5671 poll->head = NULL;6064 poll->head = NULL;
6065 req->flags &= ~REQ_F_SINGLE_POLL;
5672 }6066 }
5673 __io_poll_execute(req, mask);6067 __io_poll_execute(req, mask, poll->events);
5674 }6068 }
5675 return 1;6069 return 1;
5676}6070}
@@ -5705,12 +6099,14 @@ static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt, · 28 unmodified lines
5705 pt->error = -ENOMEM;6099 pt->error = -ENOMEM;
5706 return;6100 return;
5707 }6101 }
6102 req->flags |= REQ_F_DOUBLE_POLL;
5708 io_init_poll_iocb(poll, first->events, first->wait.func);6103 io_init_poll_iocb(poll, first->events, first->wait.func);
5709 *poll_ptr = poll;6104 *poll_ptr = poll;
5710 if (req->opcode == IORING_OP_POLL_ADD)6105 if (req->opcode == IORING_OP_POLL_ADD)
5711 req->flags |= REQ_F_ASYNC_DATA;6106 req->flags |= REQ_F_ASYNC_DATA;
5712 }6107 }
5713 6108
6109 req->flags |= REQ_F_SINGLE_POLL;
5714 pt->nr_entries++;6110 pt->nr_entries++;
5715 poll->head = head;6111 poll->head = head;
5716 poll->wait.private = req;6112 poll->wait.private = req;
@@ -5774,9 +6170,10 @@ static int __io_arm_poll_handler(struct io_kiocb *req, · 57 unmodified lines
5774 /* can't multishot if failed, just queue the event we've got */6170 /* can't multishot if failed, just queue the event we've got */
5775 if (unlikely(ipt->error || !ipt->nr_entries))6171 if (unlikely(ipt->error || !ipt->nr_entries))
5776 poll->events |= EPOLLONESHOT;6172 poll->events |= EPOLLONESHOT;
5777 __io_poll_execute(req, mask);6173 __io_poll_execute(req, mask, poll->events);
5778 return 0;6174 return 0;
5779 }6175 }
6176 io_add_napi(req->file, req->ctx);
5780 6177
5781 /*6178 /*
5782 * Release ownership. If someone tried to queue a tw while it was6179 * Release ownership. If someone tried to queue a tw while it was
@@ -5784,7 +6181,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, · 1 unmodified line
5784 */6181 */
5785 v = atomic_dec_return(&req->poll_refs);6182 v = atomic_dec_return(&req->poll_refs);
5786 if (unlikely(v & IO_POLL_REF_MASK))6183 if (unlikely(v & IO_POLL_REF_MASK))
5787 __io_poll_execute(req, 0);6184 __io_poll_execute(req, 0, poll->events);
5788 return 0;6185 return 0;
5789}6186}
5790 6187
@@ -5803,7 +6200,7 @@ enum { · 12 unmodified lines
5803 IO_APOLL_READY6200 IO_APOLL_READY
5804};6201};
5805 6202
5806static int io_arm_poll_handler(struct io_kiocb *req)6203static int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags)
5807{6204{
5808 const struct io_op_def *def = &io_op_defs[req->opcode];6205 const struct io_op_def *def = &io_op_defs[req->opcode];
5809 struct io_ring_ctx *ctx = req->ctx;6206 struct io_ring_ctx *ctx = req->ctx;
@@ -5828,9 +6225,16 @@ static int io_arm_poll_handler(struct io_kiocb *req) · 18 unmodified lines
5828 mask |= POLLOUT | POLLWRNORM;6225 mask |= POLLOUT | POLLWRNORM;
5829 }6226 }
5830 6227
5831 apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);6228 if (!(issue_flags & IO_URING_F_UNLOCKED) &&
5832 if (unlikely(!apoll))6229 !list_empty(&ctx->apoll_cache)) {
5833 return IO_APOLL_ABORTED;6230 apoll = list_first_entry(&ctx->apoll_cache, struct async_poll,
6231 poll.wait.entry);
6232 list_del_init(&apoll->poll.wait.entry);
6233 } else {
6234 apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
6235 if (unlikely(!apoll))
6236 return IO_APOLL_ABORTED;
6237 }
5834 apoll->double_poll = NULL;6238 apoll->double_poll = NULL;
5835 req->apoll = apoll;6239 req->apoll = apoll;
5836 req->flags |= REQ_F_POLLED;6240 req->flags |= REQ_F_POLLED;
@@ -5840,7 +6244,7 @@ static int io_arm_poll_handler(struct io_kiocb *req) · 3 unmodified lines
5840 if (ret || ipt.error)6244 if (ret || ipt.error)
5841 return ret ? IO_APOLL_READY : IO_APOLL_ABORTED;6245 return ret ? IO_APOLL_READY : IO_APOLL_ABORTED;
5842 6246
5843 trace_io_uring_poll_arm(ctx, req, req->opcode, req->user_data,6247 trace_io_uring_poll_arm(ctx, req, req->user_data, req->opcode,
5844 mask, apoll->poll.events);6248 mask, apoll->poll.events);
5845 return IO_APOLL_OK;6249 return IO_APOLL_OK;
5846}6250}
@@ -5975,7 +6379,7 @@ static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe · 128 unmodified lines
5975 return -EINVAL;6379 return -EINVAL;
5976 6380
5977 io_req_set_refcount(req);6381 io_req_set_refcount(req);
5978 poll->events = io_poll_parse_events(sqe, flags);6382 req->cflags = poll->events = io_poll_parse_events(sqe, flags);
5979 return 0;6383 return 0;
5980}6384}
5981 6385
@@ -6092,10 +6496,7 @@ static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data) · 110 unmodified lines
6092 6496
6093 if (IS_ERR(req))6497 if (IS_ERR(req))
6094 return PTR_ERR(req);6498 return PTR_ERR(req);
6095 6499 io_req_task_queue_fail(req, -ECANCELED);
6096 req_set_fail(req);
6097 io_fill_cqe_req(req, -ECANCELED, 0);
6098 io_put_req_deferred(req);
6099 return 0;6500 return 0;
6100}6501}
6101 6502
@@ -6568,6 +6969,8 @@ static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 466 unmodified lines
6568 return io_symlinkat_prep(req, sqe);6969 return io_symlinkat_prep(req, sqe);
6569 case IORING_OP_LINKAT:6970 case IORING_OP_LINKAT:
6570 return io_linkat_prep(req, sqe);6971 return io_linkat_prep(req, sqe);
6972 case IORING_OP_MSG_RING:
6973 return io_msg_ring_prep(req, sqe);
6571 }6974 }
6572 6975
6573 printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",6976 printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
@@ -6649,7 +7052,7 @@ fail: · 75 unmodified lines
6649 goto queue;7052 goto queue;
6650 }7053 }
6651 7054
6652 trace_io_uring_defer(ctx, req, req->user_data);7055 trace_io_uring_defer(ctx, req, req->user_data, req->opcode);
6653 de->req = req;7056 de->req = req;
6654 de->seq = seq;7057 de->seq = seq;
6655 list_add_tail(&de->list, &ctx->defer_list);7058 list_add_tail(&de->list, &ctx->defer_list);
@@ -6659,7 +7062,7 @@ fail: · 3 unmodified lines
6659static void io_clean_op(struct io_kiocb *req)7062static void io_clean_op(struct io_kiocb *req)
6660{7063{
6661 if (req->flags & REQ_F_BUFFER_SELECTED)7064 if (req->flags & REQ_F_BUFFER_SELECTED)
6662 io_put_kbuf(req);7065 io_put_kbuf_comp(req);
6663 7066
6664 if (req->flags & REQ_F_NEED_CLEANUP) {7067 if (req->flags & REQ_F_NEED_CLEANUP) {
6665 switch (req->opcode) {7068 switch (req->opcode) {
@@ -6851,6 +7254,9 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) · 185 unmodified lines
6851 case IORING_OP_LINKAT:7254 case IORING_OP_LINKAT:
6852 ret = io_linkat(req, issue_flags);7255 ret = io_linkat(req, issue_flags);
6853 break;7256 break;
7257 case IORING_OP_MSG_RING:
7258 ret = io_msg_ring(req, issue_flags);
7259 break;
6854 default:7260 default:
6855 ret = -EINVAL;7261 ret = -EINVAL;
6856 break;7262 break;
@@ -6926,7 +7332,7 @@ static void io_wq_submit_work(struct io_wq_work *work) · 69 unmodified lines
6926 continue;7332 continue;
6927 }7333 }
6928 7334
6929 if (io_arm_poll_handler(req) == IO_APOLL_OK)7335 if (io_arm_poll_handler(req, issue_flags) == IO_APOLL_OK)
6930 return;7336 return;
6931 /* aborted or ready, in either case retry blocking */7337 /* aborted or ready, in either case retry blocking */
6932 needs_poll = false;7338 needs_poll = false;
@@ -6983,7 +7389,7 @@ static struct file *io_file_get_normal(struct io_ring_ctx *ctx, · 50 unmodified lines
6983{7389{
6984 struct file *file = fget(fd);7390 struct file *file = fget(fd);
6985 7391
6986 trace_io_uring_file_get(ctx, fd);7392 trace_io_uring_file_get(ctx, req, req->user_data, fd);
6987 7393
6988 /* we don't allow fixed io_uring files */7394 /* we don't allow fixed io_uring files */
6989 if (file && unlikely(file->f_op == &io_uring_fops))7395 if (file && unlikely(file->f_op == &io_uring_fops))
@@ -7072,7 +7478,7 @@ static void io_queue_sqe_arm_apoll(struct io_kiocb *req) · 82 unmodified lines
7072{7478{
7073 struct io_kiocb *linked_timeout = io_prep_linked_timeout(req);7479 struct io_kiocb *linked_timeout = io_prep_linked_timeout(req);
7074 7480
7075 switch (io_arm_poll_handler(req)) {7481 switch (io_arm_poll_handler(req, 0)) {
7076 case IO_APOLL_READY:7482 case IO_APOLL_READY:
7077 io_req_task_queue(req);7483 io_req_task_queue(req);
7078 break;7484 break;
@@ -7081,8 +7487,12 @@ static void io_queue_sqe_arm_apoll(struct io_kiocb *req) · 2 unmodified lines
7081 * Queued up for async execution, worker will release7487 * Queued up for async execution, worker will release
7082 * submit reference when the iocb is actually submitted.7488 * submit reference when the iocb is actually submitted.
7083 */7489 */
7490 io_kbuf_recycle(req);
7084 io_queue_async_work(req, NULL);7491 io_queue_async_work(req, NULL);
7085 break;7492 break;
7493 case IO_APOLL_OK:
7494 io_kbuf_recycle(req);
7495 break;
7086 }7496 }
7087 7497
7088 if (linked_timeout)7498 if (linked_timeout)
@@ -7281,7 +7691,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, · 192 unmodified lines
7281 7691
7282 ret = io_init_req(ctx, req, sqe);7692 ret = io_init_req(ctx, req, sqe);
7283 if (unlikely(ret)) {7693 if (unlikely(ret)) {
7284 trace_io_uring_req_failed(sqe, ret);7694 trace_io_uring_req_failed(sqe, ctx, req, ret);
7285 7695
7286 /* fail even hard links since we don't submit */7696 /* fail even hard links since we don't submit */
7287 if (link->head) {7697 if (link->head) {
@@ -7308,7 +7718,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, · 20 unmodified lines
7308 }7718 }
7309 7719
7310 /* don't need @sqe from now on */7720 /* don't need @sqe from now on */
7311 trace_io_uring_submit_sqe(ctx, req, req->opcode, req->user_data,7721 trace_io_uring_submit_sqe(ctx, req, req->user_data, req->opcode,
7312 req->flags, true,7722 req->flags, true,
7313 ctx->flags & IORING_SETUP_SQPOLL);7723 ctx->flags & IORING_SETUP_SQPOLL);
7314 7724
@@ -7451,8 +7861,14 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) · 136 unmodified lines
7451 }7861 }
7452 /* will complete beyond this point, count as submitted */7862 /* will complete beyond this point, count as submitted */
7453 submitted++;7863 submitted++;
7454 if (io_submit_sqe(ctx, req, sqe))7864 if (io_submit_sqe(ctx, req, sqe)) {
7455 break;7865 /*
7866 * Continue submitting even for sqe failure if the
7867 * ring was setup with IORING_SETUP_SUBMIT_ALL
7868 */
7869 if (!(ctx->flags & IORING_SETUP_SUBMIT_ALL))
7870 break;
7871 }
7456 } while (submitted < nr);7872 } while (submitted < nr);
7457 7873
7458 if (unlikely(submitted != nr)) {7874 if (unlikely(submitted != nr)) {
@@ -7519,7 +7935,13 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) · 60 unmodified lines
7519 !(ctx->flags & IORING_SETUP_R_DISABLED))7935 !(ctx->flags & IORING_SETUP_R_DISABLED))
7520 ret = io_submit_sqes(ctx, to_submit);7936 ret = io_submit_sqes(ctx, to_submit);
7521 mutex_unlock(&ctx->uring_lock);7937 mutex_unlock(&ctx->uring_lock);
7522 7938#ifdef CONFIG_NET_RX_BUSY_POLL
7939 spin_lock(&ctx->napi_lock);
7940 if (!list_empty(&ctx->napi_list) &&
7941 io_napi_busy_loop(&ctx->napi_list))
7942 ++ret;
7943 spin_unlock(&ctx->napi_lock);
7944#endif
7523 if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait))7945 if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait))
7524 wake_up(&ctx->sqo_sq_wait);7946 wake_up(&ctx->sqo_sq_wait);
7525 if (creds)7947 if (creds)
@@ -7650,6 +8072,9 @@ struct io_wait_queue { · 124 unmodified lines
7650 struct io_ring_ctx *ctx;8072 struct io_ring_ctx *ctx;
7651 unsigned cq_tail;8073 unsigned cq_tail;
7652 unsigned nr_timeouts;8074 unsigned nr_timeouts;
8075#ifdef CONFIG_NET_RX_BUSY_POLL
8076 unsigned busy_poll_to;
8077#endif
7653};8078};
7654 8079
7655static inline bool io_should_wake(struct io_wait_queue *iowq)8080static inline bool io_should_wake(struct io_wait_queue *iowq)
@@ -7684,11 +8109,11 @@ static int io_run_task_work_sig(void) · 28 unmodified lines
7684{8109{
7685 if (io_run_task_work())8110 if (io_run_task_work())
7686 return 1;8111 return 1;
7687 if (!signal_pending(current))
7688 return 0;
7689 if (test_thread_flag(TIF_NOTIFY_SIGNAL))8112 if (test_thread_flag(TIF_NOTIFY_SIGNAL))
7690 return -ERESTARTSYS;8113 return -ERESTARTSYS;
7691 return -EINTR;8114 if (task_sigpending(current))
8115 return -EINTR;
8116 return 0;
7692}8117}
7693 8118
7694/* when returns >0, the caller should retry */8119/* when returns >0, the caller should retry */
@@ -7711,6 +8136,87 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, · 16 unmodified lines
7711 return 1;8136 return 1;
7712}8137}
7713 8138
8139#ifdef CONFIG_NET_RX_BUSY_POLL
8140static void io_adjust_busy_loop_timeout(struct timespec64 *ts,
8141 struct io_wait_queue *iowq)
8142{
8143 unsigned busy_poll_to = READ_ONCE(sysctl_net_busy_poll);
8144 struct timespec64 pollto = ns_to_timespec64(1000 * (s64)busy_poll_to);
8145
8146 if (timespec64_compare(ts, &pollto) > 0) {
8147 *ts = timespec64_sub(*ts, pollto);
8148 iowq->busy_poll_to = busy_poll_to;
8149 } else {
8150 u64 to = timespec64_to_ns(ts);
8151
8152 do_div(to, 1000);
8153 iowq->busy_poll_to = to;
8154 ts->tv_sec = 0;
8155 ts->tv_nsec = 0;
8156 }
8157}
8158
8159static inline bool io_busy_loop_timeout(unsigned long start_time,
8160 unsigned long bp_usec)
8161{
8162 if (bp_usec) {
8163 unsigned long end_time = start_time + bp_usec;
8164 unsigned long now = busy_loop_current_time();
8165
8166 return time_after(now, end_time);
8167 }
8168 return true;
8169}
8170
8171static bool io_busy_loop_end(void *p, unsigned long start_time)
8172{
8173 struct io_wait_queue *iowq = p;
8174
8175 return signal_pending(current) ||
8176 io_should_wake(iowq) ||
8177 io_busy_loop_timeout(start_time, iowq->busy_poll_to);
8178}
8179
8180static void io_blocking_napi_busy_loop(struct list_head *napi_list,
8181 struct io_wait_queue *iowq)
8182{
8183 unsigned long start_time =
8184 list_is_singular(napi_list) ? 0 :
8185 busy_loop_current_time();
8186
8187 do {
8188 if (list_is_singular(napi_list)) {
8189 struct napi_entry *ne =
8190 list_first_entry(napi_list,
8191 struct napi_entry, list);
8192
8193 napi_busy_loop(ne->napi_id, io_busy_loop_end, iowq,
8194 true, BUSY_POLL_BUDGET);
8195 io_check_napi_entry_timeout(ne);
8196 break;
8197 }
8198 } while (io_napi_busy_loop(napi_list) &&
8199 !io_busy_loop_end(iowq, start_time));
8200}
8201
8202static void io_putback_napi_list(struct io_ring_ctx *ctx,
8203 struct list_head *napi_list)
8204{
8205 struct napi_entry *cne, *lne;
8206
8207 spin_lock(&ctx->napi_lock);
8208 list_for_each_entry(cne, &ctx->napi_list, list)
8209 list_for_each_entry(lne, napi_list, list)
8210 if (cne->napi_id == lne->napi_id) {
8211 list_del(&lne->list);
8212 kfree(lne);
8213 break;
8214 }
8215 list_splice(napi_list, &ctx->napi_list);
8216 spin_unlock(&ctx->napi_lock);
8217}
8218#endif /* CONFIG_NET_RX_BUSY_POLL */
8219
7714/*8220/*
7715 * Wait until events become available, if we don't already have some. The8221 * Wait until events become available, if we don't already have some. The
7716 * application must reap them itself, as they reside on the shared cq ring.8222 * application must reap them itself, as they reside on the shared cq ring.
@@ -7723,6 +8229,9 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 6 unmodified lines
7723 struct io_rings *rings = ctx->rings;8229 struct io_rings *rings = ctx->rings;
7724 ktime_t timeout = KTIME_MAX;8230 ktime_t timeout = KTIME_MAX;
7725 int ret;8231 int ret;
8232#ifdef CONFIG_NET_RX_BUSY_POLL
8233 LIST_HEAD(local_napi_list);
8234#endif
7726 8235
7727 do {8236 do {
7728 io_cqring_overflow_flush(ctx);8237 io_cqring_overflow_flush(ctx);
@@ -7732,14 +8241,6 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 3 unmodified lines
7732 break;8241 break;
7733 } while (1);8242 } while (1);
7734 8243
7735 if (uts) {
7736 struct timespec64 ts;
7737
7738 if (get_timespec64(&ts, uts))
7739 return -EFAULT;
7740 timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns());
7741 }
7742
7743 if (sig) {8244 if (sig) {
7744#ifdef CONFIG_COMPAT8245#ifdef CONFIG_COMPAT
7745 if (in_compat_syscall())8246 if (in_compat_syscall())
@@ -7753,6 +8254,30 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 7 unmodified lines
7753 return ret;8254 return ret;
7754 }8255 }
7755 8256
8257#ifdef CONFIG_NET_RX_BUSY_POLL
8258 iowq.busy_poll_to = 0;
8259 if (!(ctx->flags & IORING_SETUP_SQPOLL)) {
8260 spin_lock(&ctx->napi_lock);
8261 list_splice_init(&ctx->napi_list, &local_napi_list);
8262 spin_unlock(&ctx->napi_lock);
8263 }
8264#endif
8265 if (uts) {
8266 struct timespec64 ts;
8267
8268 if (get_timespec64(&ts, uts))
8269 return -EFAULT;
8270#ifdef CONFIG_NET_RX_BUSY_POLL
8271 if (!list_empty(&local_napi_list))
8272 io_adjust_busy_loop_timeout(&ts, &iowq);
8273#endif
8274 timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns());
8275 }
8276#ifdef CONFIG_NET_RX_BUSY_POLL
8277 else if (!list_empty(&local_napi_list))
8278 iowq.busy_poll_to = READ_ONCE(sysctl_net_busy_poll);
8279#endif
8280
7756 init_waitqueue_func_entry(&iowq.wq, io_wake_function);8281 init_waitqueue_func_entry(&iowq.wq, io_wake_function);
7757 iowq.wq.private = current;8282 iowq.wq.private = current;
7758 INIT_LIST_HEAD(&iowq.wq.entry);8283 INIT_LIST_HEAD(&iowq.wq.entry);
@@ -7761,6 +8286,12 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 2 unmodified lines
7761 iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events;8286 iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events;
7762 8287
7763 trace_io_uring_cqring_wait(ctx, min_events);8288 trace_io_uring_cqring_wait(ctx, min_events);
8289#ifdef CONFIG_NET_RX_BUSY_POLL
8290 if (iowq.busy_poll_to)
8291 io_blocking_napi_busy_loop(&local_napi_list, &iowq);
8292 if (!list_empty(&local_napi_list))
8293 io_putback_napi_list(ctx, &local_napi_list);
8294#endif
7764 do {8295 do {
7765 /* if we can't even flush overflow, don't wait for more */8296 /* if we can't even flush overflow, don't wait for more */
7766 if (!io_cqring_overflow_flush(ctx)) {8297 if (!io_cqring_overflow_flush(ctx)) {
@@ -8749,8 +9280,16 @@ static __cold int io_uring_alloc_task_context(struct task_struct *task, · 982 unmodified lines
8749 if (unlikely(!tctx))9280 if (unlikely(!tctx))
8750 return -ENOMEM;9281 return -ENOMEM;
8751 9282
9283 tctx->registered_rings = kcalloc(IO_RINGFD_REG_MAX,
9284 sizeof(struct file *), GFP_KERNEL);
9285 if (unlikely(!tctx->registered_rings)) {
9286 kfree(tctx);
9287 return -ENOMEM;
9288 }
9289
8752 ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL);9290 ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL);
8753 if (unlikely(ret)) {9291 if (unlikely(ret)) {
9292 kfree(tctx->registered_rings);
8754 kfree(tctx);9293 kfree(tctx);
8755 return ret;9294 return ret;
8756 }9295 }
@@ -8759,6 +9298,7 @@ static __cold int io_uring_alloc_task_context(struct task_struct *task, · 2 unmodified lines
8759 if (IS_ERR(tctx->io_wq)) {9298 if (IS_ERR(tctx->io_wq)) {
8760 ret = PTR_ERR(tctx->io_wq);9299 ret = PTR_ERR(tctx->io_wq);
8761 percpu_counter_destroy(&tctx->inflight);9300 percpu_counter_destroy(&tctx->inflight);
9301 kfree(tctx->registered_rings);
8762 kfree(tctx);9302 kfree(tctx);
8763 return ret;9303 return ret;
8764 }9304 }
@@ -8783,6 +9323,7 @@ void __io_uring_free(struct task_struct *tsk) · 18 unmodified lines
8783 WARN_ON_ONCE(tctx->io_wq);9323 WARN_ON_ONCE(tctx->io_wq);
8784 WARN_ON_ONCE(tctx->cached_refs);9324 WARN_ON_ONCE(tctx->cached_refs);
8785 9325
9326 kfree(tctx->registered_rings);
8786 percpu_counter_destroy(&tctx->inflight);9327 percpu_counter_destroy(&tctx->inflight);
8787 kfree(tctx);9328 kfree(tctx);
8788 tsk->io_uring = NULL;9329 tsk->io_uring = NULL;
@@ -9359,33 +9900,55 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, · 570 unmodified lines
9359 return done ? done : err;9900 return done ? done : err;
9360}9901}
9361 9902
9362static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)9903static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg,
9904 unsigned int eventfd_async)
9363{9905{
9906 struct io_ev_fd *ev_fd;
9364 __s32 __user *fds = arg;9907 __s32 __user *fds = arg;
9365 int fd;9908 int fd;
9366 9909
9367 if (ctx->cq_ev_fd)9910 ev_fd = rcu_dereference_protected(ctx->io_ev_fd,
9911 lockdep_is_held(&ctx->uring_lock));
9912 if (ev_fd)
9368 return -EBUSY;9913 return -EBUSY;
9369 9914
9370 if (copy_from_user(&fd, fds, sizeof(*fds)))9915 if (copy_from_user(&fd, fds, sizeof(*fds)))
9371 return -EFAULT;9916 return -EFAULT;
9372 9917
9373 ctx->cq_ev_fd = eventfd_ctx_fdget(fd);9918 ev_fd = kmalloc(sizeof(*ev_fd), GFP_KERNEL);
9374 if (IS_ERR(ctx->cq_ev_fd)) {9919 if (!ev_fd)
9375 int ret = PTR_ERR(ctx->cq_ev_fd);9920 return -ENOMEM;
9376 9921
9377 ctx->cq_ev_fd = NULL;9922 ev_fd->cq_ev_fd = eventfd_ctx_fdget(fd);
9923 if (IS_ERR(ev_fd->cq_ev_fd)) {
9924 int ret = PTR_ERR(ev_fd->cq_ev_fd);
9925 kfree(ev_fd);
9378 return ret;9926 return ret;
9379 }9927 }
9380 9928 ev_fd->eventfd_async = eventfd_async;
9929 ctx->has_evfd = true;
9930 rcu_assign_pointer(ctx->io_ev_fd, ev_fd);
9381 return 0;9931 return 0;
9382}9932}
9383 9933
9934static void io_eventfd_put(struct rcu_head *rcu)
9935{
9936 struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);
9937
9938 eventfd_ctx_put(ev_fd->cq_ev_fd);
9939 kfree(ev_fd);
9940}
9941
9384static int io_eventfd_unregister(struct io_ring_ctx *ctx)9942static int io_eventfd_unregister(struct io_ring_ctx *ctx)
9385{9943{
9386 if (ctx->cq_ev_fd) {9944 struct io_ev_fd *ev_fd;
9387 eventfd_ctx_put(ctx->cq_ev_fd);9945
9388 ctx->cq_ev_fd = NULL;9946 ev_fd = rcu_dereference_protected(ctx->io_ev_fd,
9947 lockdep_is_held(&ctx->uring_lock));
9948 if (ev_fd) {
9949 ctx->has_evfd = false;
9950 rcu_assign_pointer(ctx->io_ev_fd, NULL);
9951 call_rcu(&ev_fd->rcu, io_eventfd_put);
9389 return 0;9952 return 0;
9390 }9953 }
9391 9954
@@ -9394,11 +9957,28 @@ static int io_eventfd_unregister(struct io_ring_ctx *ctx) · 2 unmodified lines
9394 9957
9395static void io_destroy_buffers(struct io_ring_ctx *ctx)9958static void io_destroy_buffers(struct io_ring_ctx *ctx)
9396{9959{
9397 struct io_buffer *buf;9960 int i;
9398 unsigned long index;9961
9962 for (i = 0; i < (1U << IO_BUFFERS_HASH_BITS); i++) {
9963 struct list_head *list = &ctx->io_buffers[i];
9964
9965 while (!list_empty(list)) {
9966 struct io_buffer_list *bl;
9967
9968 bl = list_first_entry(list, struct io_buffer_list, list);
9969 __io_remove_buffers(ctx, bl, -1U);
9970 list_del(&bl->list);
9971 kfree(bl);
9972 }
9973 }
9399 9974
9400 xa_for_each(&ctx->io_buffers, index, buf)9975 while (!list_empty(&ctx->io_buffers_pages)) {
9401 __io_remove_buffers(ctx, buf, index, -1U);9976 struct page *page;
9977
9978 page = list_first_entry(&ctx->io_buffers_pages, struct page, lru);
9979 list_del_init(&page->lru);
9980 __free_page(page);
9981 }
9402}9982}
9403 9983
9404static void io_req_caches_free(struct io_ring_ctx *ctx)9984static void io_req_caches_free(struct io_ring_ctx *ctx)
@@ -9429,6 +10009,18 @@ static void io_wait_rsrc_data(struct io_rsrc_data *data) · 24 unmodified lines
9429 wait_for_completion(&data->done);10009 wait_for_completion(&data->done);
9430}10010}
9431 10011
10012static void io_flush_apoll_cache(struct io_ring_ctx *ctx)
10013{
10014 struct async_poll *apoll;
10015
10016 while (!list_empty(&ctx->apoll_cache)) {
10017 apoll = list_first_entry(&ctx->apoll_cache, struct async_poll,
10018 poll.wait.entry);
10019 list_del(&apoll->poll.wait.entry);
10020 kfree(apoll);
10021 }
10022}
10023
9432static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)10024static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
9433{10025{
9434 io_sq_thread_finish(ctx);10026 io_sq_thread_finish(ctx);
@@ -9450,8 +10042,9 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) · 15 unmodified lines
9450 __io_sqe_files_unregister(ctx);10042 __io_sqe_files_unregister(ctx);
9451 if (ctx->rings)10043 if (ctx->rings)
9452 __io_cqring_overflow_flush(ctx, true);10044 __io_cqring_overflow_flush(ctx, true);
9453 mutex_unlock(&ctx->uring_lock);
9454 io_eventfd_unregister(ctx);10045 io_eventfd_unregister(ctx);
10046 io_flush_apoll_cache(ctx);
10047 mutex_unlock(&ctx->uring_lock);
9455 io_destroy_buffers(ctx);10048 io_destroy_buffers(ctx);
9456 if (ctx->sq_creds)10049 if (ctx->sq_creds)
9457 put_cred(ctx->sq_creds);10050 put_cred(ctx->sq_creds);
@@ -9483,8 +10076,10 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) · 25 unmodified lines
9483 io_req_caches_free(ctx);10076 io_req_caches_free(ctx);
9484 if (ctx->hash_map)10077 if (ctx->hash_map)
9485 io_wq_put_hash(ctx->hash_map);10078 io_wq_put_hash(ctx->hash_map);
10079 io_free_napi_list(ctx);
9486 kfree(ctx->cancel_hash);10080 kfree(ctx->cancel_hash);
9487 kfree(ctx->dummy_ubuf);10081 kfree(ctx->dummy_ubuf);
10082 kfree(ctx->io_buffers);
9488 kfree(ctx);10083 kfree(ctx);
9489}10084}
9490 10085
@@ -9983,6 +10578,139 @@ void __io_uring_cancel(bool cancel_all) · 492 unmodified lines
9983 io_uring_cancel_generic(cancel_all, NULL);10578 io_uring_cancel_generic(cancel_all, NULL);
9984}10579}
9985 10580
10581void io_uring_unreg_ringfd(void)
10582{
10583 struct io_uring_task *tctx = current->io_uring;
10584 int i;
10585
10586 for (i = 0; i < IO_RINGFD_REG_MAX; i++) {
10587 if (tctx->registered_rings[i]) {
10588 fput(tctx->registered_rings[i]);
10589 tctx->registered_rings[i] = NULL;
10590 }
10591 }
10592}
10593
10594static int io_ring_add_registered_fd(struct io_uring_task *tctx, int fd,
10595 int start, int end)
10596{
10597 struct file *file;
10598 int offset;
10599
10600 for (offset = start; offset < end; offset++) {
10601 offset = array_index_nospec(offset, IO_RINGFD_REG_MAX);
10602 if (tctx->registered_rings[offset])
10603 continue;
10604
10605 file = fget(fd);
10606 if (!file) {
10607 return -EBADF;
10608 } else if (file->f_op != &io_uring_fops) {
10609 fput(file);
10610 return -EOPNOTSUPP;
10611 }
10612 tctx->registered_rings[offset] = file;
10613 return offset;
10614 }
10615
10616 return -EBUSY;
10617}
10618
10619/*
10620 * Register a ring fd to avoid fdget/fdput for each io_uring_enter()
10621 * invocation. User passes in an array of struct io_uring_rsrc_update
10622 * with ->data set to the ring_fd, and ->offset given for the desired
10623 * index. If no index is desired, application may set ->offset == -1U
10624 * and we'll find an available index. Returns number of entries
10625 * successfully processed, or < 0 on error if none were processed.
10626 */
10627static int io_ringfd_register(struct io_ring_ctx *ctx, void __user *__arg,
10628 unsigned nr_args)
10629{
10630 struct io_uring_rsrc_update __user *arg = __arg;
10631 struct io_uring_rsrc_update reg;
10632 struct io_uring_task *tctx;
10633 int ret, i;
10634
10635 if (!nr_args || nr_args > IO_RINGFD_REG_MAX)
10636 return -EINVAL;
10637
10638 mutex_unlock(&ctx->uring_lock);
10639 ret = io_uring_add_tctx_node(ctx);
10640 mutex_lock(&ctx->uring_lock);
10641 if (ret)
10642 return ret;
10643
10644 tctx = current->io_uring;
10645 for (i = 0; i < nr_args; i++) {
10646 int start, end;
10647
10648 if (copy_from_user(&reg, &arg[i], sizeof(reg))) {
10649 ret = -EFAULT;
10650 break;
10651 }
10652
10653 if (reg.offset == -1U) {
10654 start = 0;
10655 end = IO_RINGFD_REG_MAX;
10656 } else {
10657 if (reg.offset >= IO_RINGFD_REG_MAX) {
10658 ret = -EINVAL;
10659 break;
10660 }
10661 start = reg.offset;
10662 end = start + 1;
10663 }
10664
10665 ret = io_ring_add_registered_fd(tctx, reg.data, start, end);
10666 if (ret < 0)
10667 break;
10668
10669 reg.offset = ret;
10670 if (copy_to_user(&arg[i], &reg, sizeof(reg))) {
10671 fput(tctx->registered_rings[reg.offset]);
10672 tctx->registered_rings[reg.offset] = NULL;
10673 ret = -EFAULT;
10674 break;
10675 }
10676 }
10677
10678 return i ? i : ret;
10679}
10680
10681static int io_ringfd_unregister(struct io_ring_ctx *ctx, void __user *__arg,
10682 unsigned nr_args)
10683{
10684 struct io_uring_rsrc_update __user *arg = __arg;
10685 struct io_uring_task *tctx = current->io_uring;
10686 struct io_uring_rsrc_update reg;
10687 int ret = 0, i;
10688
10689 if (!nr_args || nr_args > IO_RINGFD_REG_MAX)
10690 return -EINVAL;
10691 if (!tctx)
10692 return 0;
10693
10694 for (i = 0; i < nr_args; i++) {
10695 if (copy_from_user(&reg, &arg[i], sizeof(reg))) {
10696 ret = -EFAULT;
10697 break;
10698 }
10699 if (reg.offset >= IO_RINGFD_REG_MAX) {
10700 ret = -EINVAL;
10701 break;
10702 }
10703
10704 reg.offset = array_index_nospec(reg.offset, IO_RINGFD_REG_MAX);
10705 if (tctx->registered_rings[reg.offset]) {
10706 fput(tctx->registered_rings[reg.offset]);
10707 tctx->registered_rings[reg.offset] = NULL;
10708 }
10709 }
10710
10711 return i ? i : ret;
10712}
10713
9986static void *io_uring_validate_mmap_request(struct file *file,10714static void *io_uring_validate_mmap_request(struct file *file,
9987 loff_t pgoff, size_t sz)10715 loff_t pgoff, size_t sz)
9988{10716{
@@ -10113,12 +10841,28 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 124 unmodified lines
10113 io_run_task_work();10841 io_run_task_work();
10114 10842
10115 if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |10843 if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
10116 IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG)))10844 IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG |
10845 IORING_ENTER_REGISTERED_RING)))
10117 return -EINVAL;10846 return -EINVAL;
10118 10847
10119 f = fdget(fd);10848 /*
10120 if (unlikely(!f.file))10849 * Ring fd has been registered via IORING_REGISTER_RING_FDS, we
10121 return -EBADF;10850 * need only dereference our task private array to find it.
10851 */
10852 if (flags & IORING_ENTER_REGISTERED_RING) {
10853 struct io_uring_task *tctx = current->io_uring;
10854
10855 if (!tctx || fd >= IO_RINGFD_REG_MAX)
10856 return -EINVAL;
10857 fd = array_index_nospec(fd, IO_RINGFD_REG_MAX);
10858 f.file = tctx->registered_rings[fd];
10859 if (unlikely(!f.file))
10860 return -EBADF;
10861 } else {
10862 f = fdget(fd);
10863 if (unlikely(!f.file))
10864 return -EBADF;
10865 }
10122 10866
10123 ret = -EOPNOTSUPP;10867 ret = -EOPNOTSUPP;
10124 if (unlikely(f.file->f_op != &io_uring_fops))10868 if (unlikely(f.file->f_op != &io_uring_fops))
@@ -10192,7 +10936,8 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 67 unmodified lines
10192out:10936out:
10193 percpu_ref_put(&ctx->refs);10937 percpu_ref_put(&ctx->refs);
10194out_fput:10938out_fput:
10195 fdput(f);10939 if (!(flags & IORING_ENTER_REGISTERED_RING))
10940 fdput(f);
10196 return submitted ? submitted : ret;10941 return submitted ? submitted : ret;
10197}10942}
10198 10943
@@ -10610,7 +11355,7 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) · 411 unmodified lines
10610 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |11355 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
10611 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |11356 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
10612 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |11357 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
10613 IORING_SETUP_R_DISABLED))11358 IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL))
10614 return -EINVAL;11359 return -EINVAL;
10615 11360
10616 return io_uring_create(entries, &p, params);11361 return io_uring_create(entries, &p, params);
@@ -10960,61 +11705,6 @@ err: · 343 unmodified lines
10960 return ret;11705 return ret;
10961}11706}
10962 11707
10963static bool io_register_op_must_quiesce(int op)
10964{
10965 switch (op) {
10966 case IORING_REGISTER_BUFFERS:
10967 case IORING_UNREGISTER_BUFFERS:
10968 case IORING_REGISTER_FILES:
10969 case IORING_UNREGISTER_FILES:
10970 case IORING_REGISTER_FILES_UPDATE:
10971 case IORING_REGISTER_PROBE:
10972 case IORING_REGISTER_PERSONALITY:
10973 case IORING_UNREGISTER_PERSONALITY:
10974 case IORING_REGISTER_FILES2:
10975 case IORING_REGISTER_FILES_UPDATE2:
10976 case IORING_REGISTER_BUFFERS2:
10977 case IORING_REGISTER_BUFFERS_UPDATE:
10978 case IORING_REGISTER_IOWQ_AFF:
10979 case IORING_UNREGISTER_IOWQ_AFF:
10980 case IORING_REGISTER_IOWQ_MAX_WORKERS:
10981 return false;
10982 default:
10983 return true;
10984 }
10985}
10986
10987static __cold int io_ctx_quiesce(struct io_ring_ctx *ctx)
10988{
10989 long ret;
10990
10991 percpu_ref_kill(&ctx->refs);
10992
10993 /*
10994 * Drop uring mutex before waiting for references to exit. If another
10995 * thread is currently inside io_uring_enter() it might need to grab the
10996 * uring_lock to make progress. If we hold it here across the drain
10997 * wait, then we can deadlock. It's safe to drop the mutex here, since
10998 * no new references will come in after we've killed the percpu ref.
10999 */
11000 mutex_unlock(&ctx->uring_lock);
11001 do {
11002 ret = wait_for_completion_interruptible_timeout(&ctx->ref_comp, HZ);
11003 if (ret) {
11004 ret = min(0L, ret);
11005 break;
11006 }
11007
11008 ret = io_run_task_work_sig();
11009 io_req_caches_free(ctx);
11010 } while (ret >= 0);
11011 mutex_lock(&ctx->uring_lock);
11012
11013 if (ret)
11014 io_refs_resurrect(&ctx->refs, &ctx->ref_comp);
11015 return ret;
11016}
11017
11018static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,11708static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
11019 void __user *arg, unsigned nr_args)11709 void __user *arg, unsigned nr_args)
11020 __releases(ctx->uring_lock)11710 __releases(ctx->uring_lock)
@@ -11038,12 +11728,6 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, · 17 unmodified lines
11038 return -EACCES;11728 return -EACCES;
11039 }11729 }
11040 11730
11041 if (io_register_op_must_quiesce(opcode)) {
11042 ret = io_ctx_quiesce(ctx);
11043 if (ret)
11044 return ret;
11045 }
11046
11047 switch (opcode) {11731 switch (opcode) {
11048 case IORING_REGISTER_BUFFERS:11732 case IORING_REGISTER_BUFFERS:
11049 ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL);11733 ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL);
@@ -11067,17 +11751,16 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, · 17 unmodified lines
11067 ret = io_register_files_update(ctx, arg, nr_args);11751 ret = io_register_files_update(ctx, arg, nr_args);
11068 break;11752 break;
11069 case IORING_REGISTER_EVENTFD:11753 case IORING_REGISTER_EVENTFD:
11070 case IORING_REGISTER_EVENTFD_ASYNC:
11071 ret = -EINVAL;11754 ret = -EINVAL;
11072 if (nr_args != 1)11755 if (nr_args != 1)
11073 break;11756 break;
11074 ret = io_eventfd_register(ctx, arg);11757 ret = io_eventfd_register(ctx, arg, 0);
11075 if (ret)11758 break;
11759 case IORING_REGISTER_EVENTFD_ASYNC:
11760 ret = -EINVAL;
11761 if (nr_args != 1)
11076 break;11762 break;
11077 if (opcode == IORING_REGISTER_EVENTFD_ASYNC)11763 ret = io_eventfd_register(ctx, arg, 1);
11078 ctx->eventfd_async = 1;
11079 else
11080 ctx->eventfd_async = 0;
11081 break;11764 break;
11082 case IORING_UNREGISTER_EVENTFD:11765 case IORING_UNREGISTER_EVENTFD:
11083 ret = -EINVAL;11766 ret = -EINVAL;
@@ -11144,16 +11827,17 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, · 60 unmodified lines
11144 break;11827 break;
11145 ret = io_register_iowq_max_workers(ctx, arg);11828 ret = io_register_iowq_max_workers(ctx, arg);
11146 break;11829 break;
11830 case IORING_REGISTER_RING_FDS:
11831 ret = io_ringfd_register(ctx, arg, nr_args);
11832 break;
11833 case IORING_UNREGISTER_RING_FDS:
11834 ret = io_ringfd_unregister(ctx, arg, nr_args);
11835 break;
11147 default:11836 default:
11148 ret = -EINVAL;11837 ret = -EINVAL;
11149 break;11838 break;
11150 }11839 }
11151 11840
11152 if (io_register_op_must_quiesce(opcode)) {
11153 /* bring the ctx back to life */
11154 percpu_ref_reinit(&ctx->refs);
11155 reinit_completion(&ctx->ref_comp);
11156 }
11157 return ret;11841 return ret;
11158}11842}
11159 11843
@@ -11179,8 +11863,7 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, · 19 unmodified lines
11179 mutex_lock(&ctx->uring_lock);11863 mutex_lock(&ctx->uring_lock);
11180 ret = __io_uring_register(ctx, opcode, arg, nr_args);11864 ret = __io_uring_register(ctx, opcode, arg, nr_args);
11181 mutex_unlock(&ctx->uring_lock);11865 mutex_unlock(&ctx->uring_lock);
11182 trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,11866 trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs, ret);
11183 ctx->cq_ev_fd != NULL, ret);
11184out_fput:11867out_fput:
11185 fdput(f);11868 fdput(f);
11186 return ret;11869 return ret;
include/linux/io_uring.h+4−1
@@ -9,11 +9,14 @@ · 8 unmodified lines
9struct sock *io_uring_get_socket(struct file *file);9struct sock *io_uring_get_socket(struct file *file);
10void __io_uring_cancel(bool cancel_all);10void __io_uring_cancel(bool cancel_all);
11void __io_uring_free(struct task_struct *tsk);11void __io_uring_free(struct task_struct *tsk);
12void io_uring_unreg_ringfd(void);
12 13
13static inline void io_uring_files_cancel(void)14static inline void io_uring_files_cancel(void)
14{15{
15 if (current->io_uring)16 if (current->io_uring) {
17 io_uring_unreg_ringfd();
16 __io_uring_cancel(false);18 __io_uring_cancel(false);
19 }
17}20}
18static inline void io_uring_task_cancel(void)21static inline void io_uring_task_cancel(void)
19{22{
include/trace/events/io_uring.h+159−174
@@ -29,22 +29,22 @@ TRACE_EVENT(io_uring_create, · 28 unmodified lines
29 TP_ARGS(fd, ctx, sq_entries, cq_entries, flags),29 TP_ARGS(fd, ctx, sq_entries, cq_entries, flags),
30 30
31 TP_STRUCT__entry (31 TP_STRUCT__entry (
32 __field( int, fd )32 __field( int, fd )
33 __field( void *, ctx )33 __field( void *, ctx )
34 __field( u32, sq_entries )34 __field( u32, sq_entries )
35 __field( u32, cq_entries )35 __field( u32, cq_entries )
36 __field( u32, flags )36 __field( u32, flags )
37 ),37 ),
38 38
39 TP_fast_assign(39 TP_fast_assign(
40 __entry->fd = fd;40 __entry->fd = fd;
41 __entry->ctx = ctx;41 __entry->ctx = ctx;
42 __entry->sq_entries = sq_entries;42 __entry->sq_entries = sq_entries;
43 __entry->cq_entries = cq_entries;43 __entry->cq_entries = cq_entries;
44 __entry->flags = flags;44 __entry->flags = flags;
45 ),45 ),
46 46
47 TP_printk("ring %p, fd %d sq size %d, cq size %d, flags %d",47 TP_printk("ring %p, fd %d sq size %d, cq size %d, flags 0x%x",
48 __entry->ctx, __entry->fd, __entry->sq_entries,48 __entry->ctx, __entry->fd, __entry->sq_entries,
49 __entry->cq_entries, __entry->flags)49 __entry->cq_entries, __entry->flags)
50);50);
@@ -57,10 +57,9 @@ TRACE_EVENT(io_uring_create, · 6 unmodified lines
57 * @opcode: describes which operation to perform57 * @opcode: describes which operation to perform
58 * @nr_user_files: number of registered files58 * @nr_user_files: number of registered files
59 * @nr_user_bufs: number of registered buffers59 * @nr_user_bufs: number of registered buffers
60 * @cq_ev_fd: whether eventfs registered or not
61 * @ret: return code60 * @ret: return code
62 *61 *
63 * Allows to trace fixed files/buffers/eventfds, that could be registered to62 * Allows to trace fixed files/buffers, that could be registered to
64 * avoid an overhead of getting references to them for every operation. This63 * avoid an overhead of getting references to them for every operation. This
65 * event, together with io_uring_file_get, can provide a full picture of how64 * event, together with io_uring_file_get, can provide a full picture of how
66 * much overhead one can reduce via fixing.65 * much overhead one can reduce via fixing.
@@ -68,17 +67,16 @@ TRACE_EVENT(io_uring_create, · 1 unmodified line
68TRACE_EVENT(io_uring_register,67TRACE_EVENT(io_uring_register,
69 68
70 TP_PROTO(void *ctx, unsigned opcode, unsigned nr_files,69 TP_PROTO(void *ctx, unsigned opcode, unsigned nr_files,
71 unsigned nr_bufs, bool eventfd, long ret),70 unsigned nr_bufs, long ret),
72 71
73 TP_ARGS(ctx, opcode, nr_files, nr_bufs, eventfd, ret),72 TP_ARGS(ctx, opcode, nr_files, nr_bufs, ret),
74 73
75 TP_STRUCT__entry (74 TP_STRUCT__entry (
76 __field( void *, ctx )75 __field( void *, ctx )
77 __field( unsigned, opcode )76 __field( unsigned, opcode )
78 __field( unsigned, nr_files )77 __field( unsigned, nr_files)
79 __field( unsigned, nr_bufs )78 __field( unsigned, nr_bufs )
80 __field( bool, eventfd )79 __field( long, ret )
81 __field( long, ret )
82 ),80 ),
83 81
84 TP_fast_assign(82 TP_fast_assign(
@@ -86,20 +84,21 @@ TRACE_EVENT(io_uring_register, · 1 unmodified line
86 __entry->opcode = opcode;84 __entry->opcode = opcode;
87 __entry->nr_files = nr_files;85 __entry->nr_files = nr_files;
88 __entry->nr_bufs = nr_bufs;86 __entry->nr_bufs = nr_bufs;
89 __entry->eventfd = eventfd;
90 __entry->ret = ret;87 __entry->ret = ret;
91 ),88 ),
92 89
93 TP_printk("ring %p, opcode %d, nr_user_files %d, nr_user_bufs %d, "90 TP_printk("ring %p, opcode %d, nr_user_files %d, nr_user_bufs %d, "
94 "eventfd %d, ret %ld",91 "ret %ld",
95 __entry->ctx, __entry->opcode, __entry->nr_files,92 __entry->ctx, __entry->opcode, __entry->nr_files,
96 __entry->nr_bufs, __entry->eventfd, __entry->ret)93 __entry->nr_bufs, __entry->ret)
97);94);
98 95
99/**96/**
100 * io_uring_file_get - called before getting references to an SQE file97 * io_uring_file_get - called before getting references to an SQE file
101 *98 *
102 * @ctx: pointer to a ring context structure99 * @ctx: pointer to a ring context structure
100 * @req: pointer to a submitted request
101 * @user_data: user data associated with the request
103 * @fd: SQE file descriptor102 * @fd: SQE file descriptor
104 *103 *
105 * Allows to trace out how often an SQE file reference is obtained, which can104 * Allows to trace out how often an SQE file reference is obtained, which can
@@ -108,59 +107,71 @@ TRACE_EVENT(io_uring_register, · 2 unmodified lines
108 */107 */
109TRACE_EVENT(io_uring_file_get,108TRACE_EVENT(io_uring_file_get,
110 109
111 TP_PROTO(void *ctx, int fd),110 TP_PROTO(void *ctx, void *req, unsigned long long user_data, int fd),
112 111
113 TP_ARGS(ctx, fd),112 TP_ARGS(ctx, req, user_data, fd),
114 113
115 TP_STRUCT__entry (114 TP_STRUCT__entry (
116 __field( void *, ctx )115 __field( void *, ctx )
117 __field( int, fd )116 __field( void *, req )
117 __field( u64, user_data )
118 __field( int, fd )
118 ),119 ),
119 120
120 TP_fast_assign(121 TP_fast_assign(
121 __entry->ctx = ctx;122 __entry->ctx = ctx;
123 __entry->req = req;
124 __entry->user_data = user_data;
122 __entry->fd = fd;125 __entry->fd = fd;
123 ),126 ),
124 127
125 TP_printk("ring %p, fd %d", __entry->ctx, __entry->fd)128 TP_printk("ring %p, req %p, user_data 0x%llx, fd %d",
129 __entry->ctx, __entry->req, __entry->user_data, __entry->fd)
126);130);
127 131
128/**132/**
129 * io_uring_queue_async_work - called before submitting a new async work133 * io_uring_queue_async_work - called before submitting a new async work
130 *134 *
131 * @ctx: pointer to a ring context structure135 * @ctx: pointer to a ring context structure
132 * @hashed: type of workqueue, hashed or normal
133 * @req: pointer to a submitted request136 * @req: pointer to a submitted request
137 * @user_data: user data associated with the request
138 * @opcode: opcode of request
139 * @flags request flags
134 * @work: pointer to a submitted io_wq_work140 * @work: pointer to a submitted io_wq_work
141 * @rw: type of workqueue, hashed or normal
135 *142 *
136 * Allows to trace asynchronous work submission.143 * Allows to trace asynchronous work submission.
137 */144 */
138TRACE_EVENT(io_uring_queue_async_work,145TRACE_EVENT(io_uring_queue_async_work,
139 146
140 TP_PROTO(void *ctx, int rw, void * req, struct io_wq_work *work,147 TP_PROTO(void *ctx, void * req, unsigned long long user_data, u8 opcode,
141 unsigned int flags),148 unsigned int flags, struct io_wq_work *work, int rw),
142 149
143 TP_ARGS(ctx, rw, req, work, flags),150 TP_ARGS(ctx, req, user_data, flags, opcode, work, rw),
144 151
145 TP_STRUCT__entry (152 TP_STRUCT__entry (
146 __field( void *, ctx )153 __field( void *, ctx )
147 __field( int, rw )154 __field( void *, req )
148 __field( void *, req )155 __field( u64, user_data )
149 __field( struct io_wq_work *, work )156 __field( u8, opcode )
150 __field( unsigned int, flags )157 __field( unsigned int, flags )
158 __field( struct io_wq_work *, work )
159 __field( int, rw )
151 ),160 ),
152 161
153 TP_fast_assign(162 TP_fast_assign(
154 __entry->ctx = ctx;163 __entry->ctx = ctx;
155 __entry->rw = rw;164 __entry->req = req;
156 __entry->req = req;165 __entry->user_data = user_data;
157 __entry->work = work;166 __entry->flags = flags;
158 __entry->flags = flags;167 __entry->opcode = opcode;
168 __entry->work = work;
169 __entry->rw = rw;
159 ),170 ),
160 171
161 TP_printk("ring %p, request %p, flags %d, %s queue, work %p",172 TP_printk("ring %p, request %p, user_data 0x%llx, opcode %d, flags 0x%x, %s queue, work %p",
162 __entry->ctx, __entry->req, __entry->flags,173 __entry->ctx, __entry->req, __entry->user_data, __entry->opcode,
163 __entry->rw ? "hashed" : "normal", __entry->work)174 __entry->flags, __entry->rw ? "hashed" : "normal", __entry->work)
164);175);
165 176
166/**177/**
@@ -169,30 +180,33 @@ TRACE_EVENT(io_uring_queue_async_work, · 2 unmodified lines
169 * @ctx: pointer to a ring context structure180 * @ctx: pointer to a ring context structure
170 * @req: pointer to a deferred request181 * @req: pointer to a deferred request
171 * @user_data: user data associated with the request182 * @user_data: user data associated with the request
183 * @opcode: opcode of request
172 *184 *
173 * Allows to track deferred requests, to get an insight about what requests are185 * Allows to track deferred requests, to get an insight about what requests are
174 * not started immediately.186 * not started immediately.
175 */187 */
176TRACE_EVENT(io_uring_defer,188TRACE_EVENT(io_uring_defer,
177 189
178 TP_PROTO(void *ctx, void *req, unsigned long long user_data),190 TP_PROTO(void *ctx, void *req, unsigned long long user_data, u8 opcode),
179 191
180 TP_ARGS(ctx, req, user_data),192 TP_ARGS(ctx, req, user_data, opcode),
181 193
182 TP_STRUCT__entry (194 TP_STRUCT__entry (
183 __field( void *, ctx )195 __field( void *, ctx )
184 __field( void *, req )196 __field( void *, req )
185 __field( unsigned long long, data )197 __field( unsigned long long, data )
198 __field( u8, opcode )
186 ),199 ),
187 200
188 TP_fast_assign(201 TP_fast_assign(
189 __entry->ctx = ctx;202 __entry->ctx = ctx;
190 __entry->req = req;203 __entry->req = req;
191 __entry->data = user_data;204 __entry->data = user_data;
205 __entry->opcode = opcode;
192 ),206 ),
193 207
194 TP_printk("ring %p, request %p user_data %llu", __entry->ctx,208 TP_printk("ring %p, request %p, user_data 0x%llx, opcode %d",
195 __entry->req, __entry->data)209 __entry->ctx, __entry->req, __entry->data, __entry->opcode)
196);210);
197 211
198/**212/**
@@ -250,7 +264,7 @@ TRACE_EVENT(io_uring_cqring_wait, · 51 unmodified lines
250 ),264 ),
251 265
252 TP_fast_assign(266 TP_fast_assign(
253 __entry->ctx = ctx;267 __entry->ctx = ctx;
254 __entry->min_events = min_events;268 __entry->min_events = min_events;
255 ),269 ),
256 270
@@ -260,7 +274,10 @@ TRACE_EVENT(io_uring_cqring_wait, · 3 unmodified lines
260/**274/**
261 * io_uring_fail_link - called before failing a linked request275 * io_uring_fail_link - called before failing a linked request
262 *276 *
277 * @ctx: pointer to a ring context structure
263 * @req: request, which links were cancelled278 * @req: request, which links were cancelled
279 * @user_data: user data associated with the request
280 * @opcode: opcode of request
264 * @link: cancelled link281 * @link: cancelled link
265 *282 *
266 * Allows to track linked requests cancellation, to see not only that some work283 * Allows to track linked requests cancellation, to see not only that some work
@@ -268,27 +285,36 @@ TRACE_EVENT(io_uring_cqring_wait, · 1 unmodified line
268 */285 */
269TRACE_EVENT(io_uring_fail_link,286TRACE_EVENT(io_uring_fail_link,
270 287
271 TP_PROTO(void *req, void *link),288 TP_PROTO(void *ctx, void *req, unsigned long long user_data, u8 opcode, void *link),
272 289
273 TP_ARGS(req, link),290 TP_ARGS(ctx, req, user_data, opcode, link),
274 291
275 TP_STRUCT__entry (292 TP_STRUCT__entry (
276 __field( void *, req )293 __field( void *, ctx )
277 __field( void *, link )294 __field( void *, req )
295 __field( unsigned long long, user_data )
296 __field( u8, opcode )
297 __field( void *, link )
278 ),298 ),
279 299
280 TP_fast_assign(300 TP_fast_assign(
281 __entry->req = req;301 __entry->ctx = ctx;
282 __entry->link = link;302 __entry->req = req;
303 __entry->user_data = user_data;
304 __entry->opcode = opcode;
305 __entry->link = link;
283 ),306 ),
284 307
285 TP_printk("request %p, link %p", __entry->req, __entry->link)308 TP_printk("ring %p, request %p, user_data 0x%llx, opcode %d, link %p",
309 __entry->ctx, __entry->req, __entry->user_data, __entry->opcode,
310 __entry->link)
286);311);
287 312
288/**313/**
289 * io_uring_complete - called when completing an SQE314 * io_uring_complete - called when completing an SQE
290 *315 *
291 * @ctx: pointer to a ring context structure316 * @ctx: pointer to a ring context structure
317 * @req: pointer to a submitted request
292 * @user_data: user data associated with the request318 * @user_data: user data associated with the request
293 * @res: result of the request319 * @res: result of the request
294 * @cflags: completion flags320 * @cflags: completion flags
@@ -296,12 +322,13 @@ TRACE_EVENT(io_uring_fail_link, · 1 unmodified line
296 */322 */
297TRACE_EVENT(io_uring_complete,323TRACE_EVENT(io_uring_complete,
298 324
299 TP_PROTO(void *ctx, u64 user_data, int res, unsigned cflags),325 TP_PROTO(void *ctx, void *req, u64 user_data, int res, unsigned cflags),
300 326
301 TP_ARGS(ctx, user_data, res, cflags),327 TP_ARGS(ctx, req, user_data, res, cflags),
302 328
303 TP_STRUCT__entry (329 TP_STRUCT__entry (
304 __field( void *, ctx )330 __field( void *, ctx )
331 __field( void *, req )
305 __field( u64, user_data )332 __field( u64, user_data )
306 __field( int, res )333 __field( int, res )
307 __field( unsigned, cflags )334 __field( unsigned, cflags )
@@ -309,14 +336,16 @@ TRACE_EVENT(io_uring_complete, · 1 unmodified line
309 336
310 TP_fast_assign(337 TP_fast_assign(
311 __entry->ctx = ctx;338 __entry->ctx = ctx;
339 __entry->req = req;
312 __entry->user_data = user_data;340 __entry->user_data = user_data;
313 __entry->res = res;341 __entry->res = res;
314 __entry->cflags = cflags;342 __entry->cflags = cflags;
315 ),343 ),
316 344
317 TP_printk("ring %p, user_data 0x%llx, result %d, cflags %x",345 TP_printk("ring %p, req %p, user_data 0x%llx, result %d, cflags 0x%x",
318 __entry->ctx, (unsigned long long)__entry->user_data,346 __entry->ctx, __entry->req,
319 __entry->res, __entry->cflags)347 __entry->user_data,
348 __entry->res, __entry->cflags)
320);349);
321 350
322/**351/**
@@ -324,8 +353,8 @@ TRACE_EVENT(io_uring_complete, · 1 unmodified line
324 *353 *
325 * @ctx: pointer to a ring context structure354 * @ctx: pointer to a ring context structure
326 * @req: pointer to a submitted request355 * @req: pointer to a submitted request
327 * @opcode: opcode of request
328 * @user_data: user data associated with the request356 * @user_data: user data associated with the request
357 * @opcode: opcode of request
329 * @flags request flags358 * @flags request flags
330 * @force_nonblock: whether a context blocking or not359 * @force_nonblock: whether a context blocking or not
331 * @sq_thread: true if sq_thread has submitted this SQE360 * @sq_thread: true if sq_thread has submitted this SQE
@@ -335,34 +364,34 @@ TRACE_EVENT(io_uring_complete, · 3 unmodified lines
335 */364 */
336TRACE_EVENT(io_uring_submit_sqe,365TRACE_EVENT(io_uring_submit_sqe,
337 366
338 TP_PROTO(void *ctx, void *req, u8 opcode, u64 user_data, u32 flags,367 TP_PROTO(void *ctx, void *req, unsigned long long user_data, u8 opcode, u32 flags,
339 bool force_nonblock, bool sq_thread),368 bool force_nonblock, bool sq_thread),
340 369
341 TP_ARGS(ctx, req, opcode, user_data, flags, force_nonblock, sq_thread),370 TP_ARGS(ctx, req, user_data, opcode, flags, force_nonblock, sq_thread),
342 371
343 TP_STRUCT__entry (372 TP_STRUCT__entry (
344 __field( void *, ctx )373 __field( void *, ctx )
345 __field( void *, req )374 __field( void *, req )
346 __field( u8, opcode )375 __field( unsigned long long, user_data )
347 __field( u64, user_data )376 __field( u8, opcode )
348 __field( u32, flags )377 __field( u32, flags )
349 __field( bool, force_nonblock )378 __field( bool, force_nonblock )
350 __field( bool, sq_thread )379 __field( bool, sq_thread )
351 ),380 ),
352 381
353 TP_fast_assign(382 TP_fast_assign(
354 __entry->ctx = ctx;383 __entry->ctx = ctx;
355 __entry->req = req;384 __entry->req = req;
356 __entry->opcode = opcode;
357 __entry->user_data = user_data;385 __entry->user_data = user_data;
386 __entry->opcode = opcode;
358 __entry->flags = flags;387 __entry->flags = flags;
359 __entry->force_nonblock = force_nonblock;388 __entry->force_nonblock = force_nonblock;
360 __entry->sq_thread = sq_thread;389 __entry->sq_thread = sq_thread;
361 ),390 ),
362 391
363 TP_printk("ring %p, req %p, op %d, data 0x%llx, flags %u, "392 TP_printk("ring %p, req %p, user_data 0x%llx, opcode %d, flags 0x%x, "
364 "non block %d, sq_thread %d", __entry->ctx, __entry->req,393 "non block %d, sq_thread %d", __entry->ctx, __entry->req,
365 __entry->opcode, (unsigned long long)__entry->user_data,394 __entry->user_data, __entry->opcode,
366 __entry->flags, __entry->force_nonblock, __entry->sq_thread)395 __entry->flags, __entry->force_nonblock, __entry->sq_thread)
367);396);
368 397
@@ -371,8 +400,8 @@ TRACE_EVENT(io_uring_submit_sqe, · 2 unmodified lines
371 *400 *
372 * @ctx: pointer to a ring context structure401 * @ctx: pointer to a ring context structure
373 * @req: pointer to the armed request402 * @req: pointer to the armed request
374 * @opcode: opcode of request
375 * @user_data: user data associated with the request403 * @user_data: user data associated with the request
404 * @opcode: opcode of request
376 * @mask: request poll events mask405 * @mask: request poll events mask
377 * @events: registered events of interest406 * @events: registered events of interest
378 *407 *
@@ -381,155 +410,110 @@ TRACE_EVENT(io_uring_submit_sqe, · 2 unmodified lines
381 */410 */
382TRACE_EVENT(io_uring_poll_arm,411TRACE_EVENT(io_uring_poll_arm,
383 412
384 TP_PROTO(void *ctx, void *req, u8 opcode, u64 user_data,413 TP_PROTO(void *ctx, void *req, u64 user_data, u8 opcode,
385 int mask, int events),414 int mask, int events),
386 415
387 TP_ARGS(ctx, req, opcode, user_data, mask, events),416 TP_ARGS(ctx, req, user_data, opcode, mask, events),
388 417
389 TP_STRUCT__entry (418 TP_STRUCT__entry (
390 __field( void *, ctx )419 __field( void *, ctx )
391 __field( void *, req )420 __field( void *, req )
392 __field( u8, opcode )421 __field( unsigned long long, user_data )
393 __field( u64, user_data )422 __field( u8, opcode )
394 __field( int, mask )423 __field( int, mask )
395 __field( int, events )424 __field( int, events )
396 ),425 ),
397 426
398 TP_fast_assign(427 TP_fast_assign(
399 __entry->ctx = ctx;428 __entry->ctx = ctx;
400 __entry->req = req;429 __entry->req = req;
401 __entry->opcode = opcode;
402 __entry->user_data = user_data;430 __entry->user_data = user_data;
431 __entry->opcode = opcode;
403 __entry->mask = mask;432 __entry->mask = mask;
404 __entry->events = events;433 __entry->events = events;
405 ),434 ),
406 435
407 TP_printk("ring %p, req %p, op %d, data 0x%llx, mask 0x%x, events 0x%x",436 TP_printk("ring %p, req %p, user_data 0x%llx, opcode %d, mask 0x%x, events 0x%x",
408 __entry->ctx, __entry->req, __entry->opcode,437 __entry->ctx, __entry->req, __entry->user_data, __entry->opcode,
409 (unsigned long long) __entry->user_data,
410 __entry->mask, __entry->events)438 __entry->mask, __entry->events)
411);439);
412 440
413TRACE_EVENT(io_uring_poll_wake,
414
415 TP_PROTO(void *ctx, u8 opcode, u64 user_data, int mask),
416
417 TP_ARGS(ctx, opcode, user_data, mask),
418
419 TP_STRUCT__entry (
420 __field( void *, ctx )
421 __field( u8, opcode )
422 __field( u64, user_data )
423 __field( int, mask )
424 ),
425
426 TP_fast_assign(
427 __entry->ctx = ctx;
428 __entry->opcode = opcode;
429 __entry->user_data = user_data;
430 __entry->mask = mask;
431 ),
432
433 TP_printk("ring %p, op %d, data 0x%llx, mask 0x%x",
434 __entry->ctx, __entry->opcode,
435 (unsigned long long) __entry->user_data,
436 __entry->mask)
437);
438
439TRACE_EVENT(io_uring_task_add,
440
441 TP_PROTO(void *ctx, u8 opcode, u64 user_data, int mask),
442
443 TP_ARGS(ctx, opcode, user_data, mask),
444
445 TP_STRUCT__entry (
446 __field( void *, ctx )
447 __field( u8, opcode )
448 __field( u64, user_data )
449 __field( int, mask )
450 ),
451
452 TP_fast_assign(
453 __entry->ctx = ctx;
454 __entry->opcode = opcode;
455 __entry->user_data = user_data;
456 __entry->mask = mask;
457 ),
458
459 TP_printk("ring %p, op %d, data 0x%llx, mask %x",
460 __entry->ctx, __entry->opcode,
461 (unsigned long long) __entry->user_data,
462 __entry->mask)
463);
464
465/*441/*
466 * io_uring_task_run - called when task_work_run() executes the poll events442 * io_uring_task_add - called after adding a task
467 * notification callbacks
468 *443 *
469 * @ctx: pointer to a ring context structure444 * @ctx: pointer to a ring context structure
470 * @req: pointer to the armed request445 * @req: pointer to request
471 * @opcode: opcode of request
472 * @user_data: user data associated with the request446 * @user_data: user data associated with the request
447 * @opcode: opcode of request
448 * @mask: request poll events mask
473 *449 *
474 * Allows to track when notified poll events are processed
475 */450 */
476TRACE_EVENT(io_uring_task_run,451TRACE_EVENT(io_uring_task_add,
477 452
478 TP_PROTO(void *ctx, void *req, u8 opcode, u64 user_data),453 TP_PROTO(void *ctx, void *req, unsigned long long user_data, u8 opcode, int mask),
479 454
480 TP_ARGS(ctx, req, opcode, user_data),455 TP_ARGS(ctx, req, user_data, opcode, mask),
481 456
482 TP_STRUCT__entry (457 TP_STRUCT__entry (
483 __field( void *, ctx )458 __field( void *, ctx )
484 __field( void *, req )459 __field( void *, req )
485 __field( u8, opcode )460 __field( unsigned long long, user_data )
486 __field( u64, user_data )461 __field( u8, opcode )
462 __field( int, mask )
487 ),463 ),
488 464
489 TP_fast_assign(465 TP_fast_assign(
490 __entry->ctx = ctx;466 __entry->ctx = ctx;
491 __entry->req = req;467 __entry->req = req;
492 __entry->opcode = opcode;
493 __entry->user_data = user_data;468 __entry->user_data = user_data;
469 __entry->opcode = opcode;
470 __entry->mask = mask;
494 ),471 ),
495 472
496 TP_printk("ring %p, req %p, op %d, data 0x%llx",473 TP_printk("ring %p, req %p, user_data 0x%llx, opcode %d, mask %x",
497 __entry->ctx, __entry->req, __entry->opcode,474 __entry->ctx, __entry->req, __entry->user_data, __entry->opcode,
498 (unsigned long long) __entry->user_data)475 __entry->mask)
499);476);
500 477
501/*478/*
502 * io_uring_req_failed - called when an sqe is errored dring submission479 * io_uring_req_failed - called when an sqe is errored dring submission
503 *480 *
504 * @sqe: pointer to the io_uring_sqe that failed481 * @sqe: pointer to the io_uring_sqe that failed
482 * @ctx: pointer to a ring context structure
483 * @req: pointer to request
505 * @error: error it failed with484 * @error: error it failed with
506 *485 *
507 * Allows easier diagnosing of malformed requests in production systems.486 * Allows easier diagnosing of malformed requests in production systems.
508 */487 */
509TRACE_EVENT(io_uring_req_failed,488TRACE_EVENT(io_uring_req_failed,
510 489
511 TP_PROTO(const struct io_uring_sqe *sqe, int error),490 TP_PROTO(const struct io_uring_sqe *sqe, void *ctx, void *req, int error),
512 491
513 TP_ARGS(sqe, error),492 TP_ARGS(sqe, ctx, req, error),
514 493
515 TP_STRUCT__entry (494 TP_STRUCT__entry (
516 __field( u8, opcode )495 __field( void *, ctx )
517 __field( u8, flags )496 __field( void *, req )
518 __field( u8, ioprio )497 __field( unsigned long long, user_data )
519 __field( u64, off )498 __field( u8, opcode )
520 __field( u64, addr )499 __field( u8, flags )
521 __field( u32, len )500 __field( u8, ioprio )
522 __field( u32, op_flags )501 __field( u64, off )
523 __field( u64, user_data )502 __field( u64, addr )
524 __field( u16, buf_index )503 __field( u32, len )
525 __field( u16, personality )504 __field( u32, op_flags )
526 __field( u32, file_index )505 __field( u16, buf_index )
527 __field( u64, pad1 )506 __field( u16, personality )
528 __field( u64, pad2 )507 __field( u32, file_index )
529 __field( int, error )508 __field( u64, pad1 )
509 __field( u64, pad2 )
510 __field( int, error )
530 ),511 ),
531 512
532 TP_fast_assign(513 TP_fast_assign(
514 __entry->ctx = ctx;
515 __entry->req = req;
516 __entry->user_data = sqe->user_data;
533 __entry->opcode = sqe->opcode;517 __entry->opcode = sqe->opcode;
534 __entry->flags = sqe->flags;518 __entry->flags = sqe->flags;
535 __entry->ioprio = sqe->ioprio;519 __entry->ioprio = sqe->ioprio;
@@ -537,7 +521,6 @@ TRACE_EVENT(io_uring_req_failed, · 1 unmodified line
537 __entry->addr = sqe->addr;521 __entry->addr = sqe->addr;
538 __entry->len = sqe->len;522 __entry->len = sqe->len;
539 __entry->op_flags = sqe->rw_flags;523 __entry->op_flags = sqe->rw_flags;
540 __entry->user_data = sqe->user_data;
541 __entry->buf_index = sqe->buf_index;524 __entry->buf_index = sqe->buf_index;
542 __entry->personality = sqe->personality;525 __entry->personality = sqe->personality;
543 __entry->file_index = sqe->file_index;526 __entry->file_index = sqe->file_index;
@@ -546,13 +529,15 @@ TRACE_EVENT(io_uring_req_failed, · 2 unmodified lines
546 __entry->error = error;529 __entry->error = error;
547 ),530 ),
548 531
549 TP_printk("op %d, flags=0x%x, prio=%d, off=%llu, addr=%llu, "532 TP_printk("ring %p, req %p, user_data 0x%llx, "
550 "len=%u, rw_flags=0x%x, user_data=0x%llx, buf_index=%d, "533 "op %d, flags 0x%x, prio=%d, off=%llu, addr=%llu, "
534 "len=%u, rw_flags=0x%x, buf_index=%d, "
551 "personality=%d, file_index=%d, pad=0x%llx/%llx, error=%d",535 "personality=%d, file_index=%d, pad=0x%llx/%llx, error=%d",
536 __entry->ctx, __entry->req, __entry->user_data,
552 __entry->opcode, __entry->flags, __entry->ioprio,537 __entry->opcode, __entry->flags, __entry->ioprio,
553 (unsigned long long)__entry->off,538 (unsigned long long)__entry->off,
554 (unsigned long long) __entry->addr, __entry->len,539 (unsigned long long) __entry->addr, __entry->len,
555 __entry->op_flags, (unsigned long long) __entry->user_data,540 __entry->op_flags,
556 __entry->buf_index, __entry->personality, __entry->file_index,541 __entry->buf_index, __entry->personality, __entry->file_index,
557 (unsigned long long) __entry->pad1,542 (unsigned long long) __entry->pad1,
558 (unsigned long long) __entry->pad2, __entry->error)543 (unsigned long long) __entry->pad2, __entry->error)
include/uapi/linux/io_uring.h+13−4
@@ -101,6 +101,7 @@ enum { · 100 unmodified lines
101#define IORING_SETUP_CLAMP (1U << 4) /* clamp SQ/CQ ring sizes */101#define IORING_SETUP_CLAMP (1U << 4) /* clamp SQ/CQ ring sizes */
102#define IORING_SETUP_ATTACH_WQ (1U << 5) /* attach to existing wq */102#define IORING_SETUP_ATTACH_WQ (1U << 5) /* attach to existing wq */
103#define IORING_SETUP_R_DISABLED (1U << 6) /* start with ring disabled */103#define IORING_SETUP_R_DISABLED (1U << 6) /* start with ring disabled */
104#define IORING_SETUP_SUBMIT_ALL (1U << 7) /* continue submit on error */
104 105
105enum {106enum {
106 IORING_OP_NOP,107 IORING_OP_NOP,
@@ -143,6 +144,7 @@ enum { · 36 unmodified lines
143 IORING_OP_MKDIRAT,144 IORING_OP_MKDIRAT,
144 IORING_OP_SYMLINKAT,145 IORING_OP_SYMLINKAT,
145 IORING_OP_LINKAT,146 IORING_OP_LINKAT,
147 IORING_OP_MSG_RING,
146 148
147 /* this goes last, obviously */149 /* this goes last, obviously */
148 IORING_OP_LAST,150 IORING_OP_LAST,
@@ -199,9 +201,11 @@ struct io_uring_cqe { · 50 unmodified lines
199 *201 *
200 * IORING_CQE_F_BUFFER If set, the upper 16 bits are the buffer ID202 * IORING_CQE_F_BUFFER If set, the upper 16 bits are the buffer ID
201 * IORING_CQE_F_MORE If set, parent SQE will generate more CQE entries203 * IORING_CQE_F_MORE If set, parent SQE will generate more CQE entries
204 * IORING_CQE_F_MSG If set, CQE was generated with IORING_OP_MSG_RING
202 */205 */
203#define IORING_CQE_F_BUFFER (1U << 0)206#define IORING_CQE_F_BUFFER (1U << 0)
204#define IORING_CQE_F_MORE (1U << 1)207#define IORING_CQE_F_MORE (1U << 1)
208#define IORING_CQE_F_MSG (1U << 2)
205 209
206enum {210enum {
207 IORING_CQE_BUFFER_SHIFT = 16,211 IORING_CQE_BUFFER_SHIFT = 16,
@@ -257,10 +261,11 @@ struct io_cqring_offsets { · 49 unmodified lines
257/*261/*
258 * io_uring_enter(2) flags262 * io_uring_enter(2) flags
259 */263 */
260#define IORING_ENTER_GETEVENTS (1U << 0)264#define IORING_ENTER_GETEVENTS (1U << 0)
261#define IORING_ENTER_SQ_WAKEUP (1U << 1)265#define IORING_ENTER_SQ_WAKEUP (1U << 1)
262#define IORING_ENTER_SQ_WAIT (1U << 2)266#define IORING_ENTER_SQ_WAIT (1U << 2)
263#define IORING_ENTER_EXT_ARG (1U << 3)267#define IORING_ENTER_EXT_ARG (1U << 3)
268#define IORING_ENTER_REGISTERED_RING (1U << 4)
264 269
265/*270/*
266 * Passed in for io_uring_setup(2). Copied back with updated info on success271 * Passed in for io_uring_setup(2). Copied back with updated info on success
@@ -325,6 +330,10 @@ enum { · 58 unmodified lines
325 /* set/get max number of io-wq workers */330 /* set/get max number of io-wq workers */
326 IORING_REGISTER_IOWQ_MAX_WORKERS = 19,331 IORING_REGISTER_IOWQ_MAX_WORKERS = 19,
327 332
333 /* register/unregister io_uring fd with the ring */
334 IORING_REGISTER_RING_FDS = 20,
335 IORING_UNREGISTER_RING_FDS = 21,
336
328 /* this goes last */337 /* this goes last */
329 IORING_REGISTER_LAST338 IORING_REGISTER_LAST
330};339};