History

Merge tag 'for-5.7/io_uring-2020-03-29' of git://git.kernel.dk/linux-block

Pull io_uring updates from Jens Axboe: "Here are the io_uring changes for this merge window. Light on new features this time around (just splice + buffer selection), lots of cleanups, fixes, and improvements to existing support. In particular, this contains: - Cleanup fixed file update handling for stack fallback (Hillf) - Re-work of how pollable async IO is handled, we no longer require thread offload to handle that. Instead we rely using poll to drive this, with task_work execution. - In conjunction with the above, allow expendable buffer selection, so that poll+recv (for example) no longer has to be a split operation. - Make sure we honor RLIMIT_FSIZE for buffered writes - Add support for splice (Pavel) - Linked work inheritance fixes and optimizations (Pavel) - Async work fixes and cleanups (Pavel) - Improve io-wq locking (Pavel) - Hashed link write improvements (Pavel) - SETUP_IOPOLL|SETUP_SQPOLL improvements (Xiaoguang)" * tag 'for-5.7/io_uring-2020-03-29' of git://git.kernel.dk/linux-block: (54 commits) io_uring: cleanup io_alloc_async_ctx() io_uring: fix missing 'return' in comment io-wq: handle hashed writes in chains io-uring: drop 'free_pfile' in struct io_file_put io-uring: drop completion when removing file io_uring: Fix ->data corruption on re-enqueue io-wq: close cancel gap for hashed linked work io_uring: make spdxcheck.py happy io_uring: honor original task RLIMIT_FSIZE io-wq: hash dependent work io-wq: split hashing and enqueueing io-wq: don't resched if there is no work io-wq: remove duplicated cancel code io_uring: fix truncated async read/readv and write/writev retry io_uring: dual license io_uring.h uapi header io_uring: io_uring_enter(2) don't poll while SETUP_IOPOLL|SETUP_SQPOLL enabled io_uring: Fix unused function warnings io_uring: add end-of-bits marker and build time verify it io_uring: provide means of removing buffers io_uring: add IOSQE_BUFFER_SELECT support for IORING_OP_RECVMSG ...

Linus Torvalds committed 6 years ago

Commit e59cd882 parents 1592614 3d9932a12 files · +1826 −856Browse files

Unified · Split

12 files
  1. fs/io-wq.c +162 −206
  2. fs/io-wq.h +46 −19
  3. fs/io_uring.c +1406 −609
  4. fs/splice.c +3 −3
  5. include/linux/socket.h +4 −0
  6. include/linux/splice.h +3 −0
  7. include/net/compat.h +3 −0
  8. include/trace/events/io_uring.h +103 −0
  9. include/uapi/linux/io_uring.h +38 −4
  10. kernel/task_work.c +14 −4
  11. net/compat.c +23 −7
  12. net/socket.c +21 −4
fs/io-wq.c+162−206
@@ -69,6 +69,8 @@ struct io_worker { · 68 unmodified lines
69#define IO_WQ_HASH_ORDER 569#define IO_WQ_HASH_ORDER 5
70#endif70#endif
71 71
72#define IO_WQ_NR_HASH_BUCKETS (1u << IO_WQ_HASH_ORDER)
73
72struct io_wqe_acct {74struct io_wqe_acct {
73 unsigned nr_workers;75 unsigned nr_workers;
74 unsigned max_workers;76 unsigned max_workers;
@@ -98,6 +100,7 @@ struct io_wqe { · 23 unmodified lines
98 struct list_head all_list;100 struct list_head all_list;
99 101
100 struct io_wq *wq;102 struct io_wq *wq;
103 struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS];
101};104};
102 105
103/*106/*
@@ -107,8 +110,7 @@ struct io_wq { · 3 unmodified lines
107 struct io_wqe **wqes;110 struct io_wqe **wqes;
108 unsigned long state;111 unsigned long state;
109 112
110 get_work_fn *get_work;113 free_work_fn *free_work;
111 put_work_fn *put_work;
112 114
113 struct task_struct *manager;115 struct task_struct *manager;
114 struct user_struct *user;116 struct user_struct *user;
@@ -376,26 +378,35 @@ static bool __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker) · 261 unmodified lines
376 return __io_worker_unuse(wqe, worker);378 return __io_worker_unuse(wqe, worker);
377}379}
378 380
379static struct io_wq_work *io_get_next_work(struct io_wqe *wqe, unsigned *hash)381static inline unsigned int io_get_work_hash(struct io_wq_work *work)
382{
383 return work->flags >> IO_WQ_HASH_SHIFT;
384}
385
386static struct io_wq_work *io_get_next_work(struct io_wqe *wqe)
380 __must_hold(wqe->lock)387 __must_hold(wqe->lock)
381{388{
382 struct io_wq_work_node *node, *prev;389 struct io_wq_work_node *node, *prev;
383 struct io_wq_work *work;390 struct io_wq_work *work, *tail;
391 unsigned int hash;
384 392
385 wq_list_for_each(node, prev, &wqe->work_list) {393 wq_list_for_each(node, prev, &wqe->work_list) {
386 work = container_of(node, struct io_wq_work, list);394 work = container_of(node, struct io_wq_work, list);
387 395
388 /* not hashed, can run anytime */396 /* not hashed, can run anytime */
389 if (!(work->flags & IO_WQ_WORK_HASHED)) {397 if (!io_wq_is_hashed(work)) {
390 wq_node_del(&wqe->work_list, node, prev);398 wq_list_del(&wqe->work_list, node, prev);
391 return work;399 return work;
392 }400 }
393 401
394 /* hashed, can run if not already running */402 /* hashed, can run if not already running */
395 *hash = work->flags >> IO_WQ_HASH_SHIFT;403 hash = io_get_work_hash(work);
396 if (!(wqe->hash_map & BIT_ULL(*hash))) {404 if (!(wqe->hash_map & BIT(hash))) {
397 wqe->hash_map |= BIT_ULL(*hash);405 wqe->hash_map |= BIT(hash);
398 wq_node_del(&wqe->work_list, node, prev);406 /* all items with this hash lie in [work, tail] */
407 tail = wqe->hash_tail[hash];
408 wqe->hash_tail[hash] = NULL;
409 wq_list_cut(&wqe->work_list, &tail->list, prev);
399 return work;410 return work;
400 }411 }
401 }412 }
@@ -440,16 +451,49 @@ static void io_wq_switch_creds(struct io_worker *worker, · 38 unmodified lines
440 worker->saved_creds = old_creds;451 worker->saved_creds = old_creds;
441}452}
442 453
454static void io_impersonate_work(struct io_worker *worker,
455 struct io_wq_work *work)
456{
457 if (work->files && current->files != work->files) {
458 task_lock(current);
459 current->files = work->files;
460 task_unlock(current);
461 }
462 if (work->fs && current->fs != work->fs)
463 current->fs = work->fs;
464 if (work->mm != worker->mm)
465 io_wq_switch_mm(worker, work);
466 if (worker->cur_creds != work->creds)
467 io_wq_switch_creds(worker, work);
468}
469
470static void io_assign_current_work(struct io_worker *worker,
471 struct io_wq_work *work)
472{
473 if (work) {
474 /* flush pending signals before assigning new work */
475 if (signal_pending(current))
476 flush_signals(current);
477 cond_resched();
478 }
479
480 spin_lock_irq(&worker->lock);
481 worker->cur_work = work;
482 spin_unlock_irq(&worker->lock);
483}
484
485static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work);
486
443static void io_worker_handle_work(struct io_worker *worker)487static void io_worker_handle_work(struct io_worker *worker)
444 __releases(wqe->lock)488 __releases(wqe->lock)
445{489{
446 struct io_wq_work *work, *old_work = NULL, *put_work = NULL;
447 struct io_wqe *wqe = worker->wqe;490 struct io_wqe *wqe = worker->wqe;
448 struct io_wq *wq = wqe->wq;491 struct io_wq *wq = wqe->wq;
449 492
450 do {493 do {
451 unsigned hash = -1U;494 struct io_wq_work *work;
452 495 unsigned int hash;
496get_next:
453 /*497 /*
454 * If we got some work, mark us as busy. If we didn't, but498 * If we got some work, mark us as busy. If we didn't, but
455 * the list isn't empty, it means we stalled on hashed work.499 * the list isn't empty, it means we stalled on hashed work.
@@ -457,81 +501,60 @@ static void io_worker_handle_work(struct io_worker *worker) · 1 unmodified line
457 * can't make progress, any work completion or insertion will501 * can't make progress, any work completion or insertion will
458 * clear the stalled flag.502 * clear the stalled flag.
459 */503 */
460 work = io_get_next_work(wqe, &hash);504 work = io_get_next_work(wqe);
461 if (work)505 if (work)
462 __io_worker_busy(wqe, worker, work);506 __io_worker_busy(wqe, worker, work);
463 else if (!wq_list_empty(&wqe->work_list))507 else if (!wq_list_empty(&wqe->work_list))
464 wqe->flags |= IO_WQE_FLAG_STALLED;508 wqe->flags |= IO_WQE_FLAG_STALLED;
465 509
466 spin_unlock_irq(&wqe->lock);510 spin_unlock_irq(&wqe->lock);
467 if (put_work && wq->put_work)
468 wq->put_work(old_work);
469 if (!work)511 if (!work)
470 break;512 break;
471next:513 io_assign_current_work(worker, work);
472 /* flush any pending signals before assigning new work */514
473 if (signal_pending(current))515 /* handle a whole dependent link */
474 flush_signals(current);516 do {
475 517 struct io_wq_work *old_work, *next_hashed, *linked;
476 cond_resched();518
477 519 next_hashed = wq_next_work(work);
478 spin_lock_irq(&worker->lock);520 io_impersonate_work(worker, work);
479 worker->cur_work = work;521 /*
480 spin_unlock_irq(&worker->lock);522 * OK to set IO_WQ_WORK_CANCEL even for uncancellable
481 523 * work, the worker function will do the right thing.
482 if (work->flags & IO_WQ_WORK_CB)524 */
483 work->func(&work);525 if (test_bit(IO_WQ_BIT_CANCEL, &wq->state))
484 526 work->flags |= IO_WQ_WORK_CANCEL;
485 if (work->files && current->files != work->files) {527
486 task_lock(current);528 hash = io_get_work_hash(work);
487 current->files = work->files;529 linked = old_work = work;
488 task_unlock(current);530 linked->func(&linked);
489 }531 linked = (old_work == linked) ? NULL : linked;
490 if (work->fs && current->fs != work->fs)532
491 current->fs = work->fs;533 work = next_hashed;
492 if (work->mm != worker->mm)534 if (!work && linked && !io_wq_is_hashed(linked)) {
493 io_wq_switch_mm(worker, work);535 work = linked;
494 if (worker->cur_creds != work->creds)536 linked = NULL;
495 io_wq_switch_creds(worker, work);
496 /*
497 * OK to set IO_WQ_WORK_CANCEL even for uncancellable work,
498 * the worker function will do the right thing.
499 */
500 if (test_bit(IO_WQ_BIT_CANCEL, &wq->state))
501 work->flags |= IO_WQ_WORK_CANCEL;
502 if (worker->mm)
503 work->flags |= IO_WQ_WORK_HAS_MM;
504
505 if (wq->get_work) {
506 put_work = work;
507 wq->get_work(work);
508 }
509
510 old_work = work;
511 work->func(&work);
512
513 spin_lock_irq(&worker->lock);
514 worker->cur_work = NULL;
515 spin_unlock_irq(&worker->lock);
516
517 spin_lock_irq(&wqe->lock);
518
519 if (hash != -1U) {
520 wqe->hash_map &= ~BIT_ULL(hash);
521 wqe->flags &= ~IO_WQE_FLAG_STALLED;
522 }
523 if (work && work != old_work) {
524 spin_unlock_irq(&wqe->lock);
525
526 if (put_work && wq->put_work) {
527 wq->put_work(put_work);
528 put_work = NULL;
529 }537 }
538 io_assign_current_work(worker, work);
539 wq->free_work(old_work);
540
541 if (linked)
542 io_wqe_enqueue(wqe, linked);
543
544 if (hash != -1U && !next_hashed) {
545 spin_lock_irq(&wqe->lock);
546 wqe->hash_map &= ~BIT_ULL(hash);
547 wqe->flags &= ~IO_WQE_FLAG_STALLED;
548 /* dependent work is not hashed */
549 hash = -1U;
550 /* skip unnecessary unlock-lock wqe->lock */
551 if (!work)
552 goto get_next;
553 spin_unlock_irq(&wqe->lock);
554 }
555 } while (work);
530 556
531 /* dependent work not hashed */557 spin_lock_irq(&wqe->lock);
532 hash = -1U;
533 goto next;
534 }
535 } while (1);558 } while (1);
536}559}
537 560
@@ -747,17 +770,40 @@ static bool io_wq_can_queue(struct io_wqe *wqe, struct io_wqe_acct *acct, · 209 unmodified lines
747 return true;770 return true;
748}771}
749 772
750static void io_run_cancel(struct io_wq_work *work)773static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe)
751{774{
775 struct io_wq *wq = wqe->wq;
776
752 do {777 do {
753 struct io_wq_work *old_work = work;778 struct io_wq_work *old_work = work;
754 779
755 work->flags |= IO_WQ_WORK_CANCEL;780 work->flags |= IO_WQ_WORK_CANCEL;
756 work->func(&work);781 work->func(&work);
757 work = (work == old_work) ? NULL : work;782 work = (work == old_work) ? NULL : work;
783 wq->free_work(old_work);
758 } while (work);784 } while (work);
759}785}
760 786
787static void io_wqe_insert_work(struct io_wqe *wqe, struct io_wq_work *work)
788{
789 unsigned int hash;
790 struct io_wq_work *tail;
791
792 if (!io_wq_is_hashed(work)) {
793append:
794 wq_list_add_tail(&work->list, &wqe->work_list);
795 return;
796 }
797
798 hash = io_get_work_hash(work);
799 tail = wqe->hash_tail[hash];
800 wqe->hash_tail[hash] = work;
801 if (!tail)
802 goto append;
803
804 wq_list_add_after(&work->list, &tail->list, &wqe->work_list);
805}
806
761static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work)807static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work)
762{808{
763 struct io_wqe_acct *acct = io_work_get_acct(wqe, work);809 struct io_wqe_acct *acct = io_work_get_acct(wqe, work);
@@ -771,13 +817,13 @@ static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work) · 7 unmodified lines
771 * It's close enough to not be an issue, fork() has the same delay.817 * It's close enough to not be an issue, fork() has the same delay.
772 */818 */
773 if (unlikely(!io_wq_can_queue(wqe, acct, work))) {819 if (unlikely(!io_wq_can_queue(wqe, acct, work))) {
774 io_run_cancel(work);820 io_run_cancel(work, wqe);
775 return;821 return;
776 }822 }
777 823
778 work_flags = work->flags;824 work_flags = work->flags;
779 spin_lock_irqsave(&wqe->lock, flags);825 spin_lock_irqsave(&wqe->lock, flags);
780 wq_list_add_tail(&work->list, &wqe->work_list);826 io_wqe_insert_work(wqe, work);
781 wqe->flags &= ~IO_WQE_FLAG_STALLED;827 wqe->flags &= ~IO_WQE_FLAG_STALLED;
782 spin_unlock_irqrestore(&wqe->lock, flags);828 spin_unlock_irqrestore(&wqe->lock, flags);
783 829
@@ -794,19 +840,15 @@ void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work) · 10 unmodified lines
794}840}
795 841
796/*842/*
797 * Enqueue work, hashed by some key. Work items that hash to the same value843 * Work items that hash to the same value will not be done in parallel.
798 * will not be done in parallel. Used to limit concurrent writes, generally844 * Used to limit concurrent writes, generally hashed by inode.
799 * hashed by inode.
800 */845 */
801void io_wq_enqueue_hashed(struct io_wq *wq, struct io_wq_work *work, void *val)846void io_wq_hash_work(struct io_wq_work *work, void *val)
802{847{
803 struct io_wqe *wqe = wq->wqes[numa_node_id()];848 unsigned int bit;
804 unsigned bit;
805
806 849
807 bit = hash_ptr(val, IO_WQ_HASH_ORDER);850 bit = hash_ptr(val, IO_WQ_HASH_ORDER);
808 work->flags |= (IO_WQ_WORK_HASHED | (bit << IO_WQ_HASH_SHIFT));851 work->flags |= (IO_WQ_WORK_HASHED | (bit << IO_WQ_HASH_SHIFT));
809 io_wqe_enqueue(wqe, work);
810}852}
811 853
812static bool io_wqe_worker_send_sig(struct io_worker *worker, void *data)854static bool io_wqe_worker_send_sig(struct io_worker *worker, void *data)
@@ -856,14 +898,13 @@ void io_wq_cancel_all(struct io_wq *wq) · 43 unmodified lines
856}898}
857 899
858struct io_cb_cancel_data {900struct io_cb_cancel_data {
859 struct io_wqe *wqe;901 work_cancel_fn *fn;
860 work_cancel_fn *cancel;902 void *data;
861 void *caller_data;
862};903};
863 904
864static bool io_work_cancel(struct io_worker *worker, void *cancel_data)905static bool io_wq_worker_cancel(struct io_worker *worker, void *data)
865{906{
866 struct io_cb_cancel_data *data = cancel_data;907 struct io_cb_cancel_data *match = data;
867 unsigned long flags;908 unsigned long flags;
868 bool ret = false;909 bool ret = false;
869 910
@@ -874,83 +915,7 @@ static bool io_work_cancel(struct io_worker *worker, void *cancel_data) · 4 unmodified lines
874 spin_lock_irqsave(&worker->lock, flags);915 spin_lock_irqsave(&worker->lock, flags);
875 if (worker->cur_work &&916 if (worker->cur_work &&
876 !(worker->cur_work->flags & IO_WQ_WORK_NO_CANCEL) &&917 !(worker->cur_work->flags & IO_WQ_WORK_NO_CANCEL) &&
877 data->cancel(worker->cur_work, data->caller_data)) {918 match->fn(worker->cur_work, match->data)) {
878 send_sig(SIGINT, worker->task, 1);
879 ret = true;
880 }
881 spin_unlock_irqrestore(&worker->lock, flags);
882
883 return ret;
884}
885
886static enum io_wq_cancel io_wqe_cancel_cb_work(struct io_wqe *wqe,
887 work_cancel_fn *cancel,
888 void *cancel_data)
889{
890 struct io_cb_cancel_data data = {
891 .wqe = wqe,
892 .cancel = cancel,
893 .caller_data = cancel_data,
894 };
895 struct io_wq_work_node *node, *prev;
896 struct io_wq_work *work;
897 unsigned long flags;
898 bool found = false;
899
900 spin_lock_irqsave(&wqe->lock, flags);
901 wq_list_for_each(node, prev, &wqe->work_list) {
902 work = container_of(node, struct io_wq_work, list);
903
904 if (cancel(work, cancel_data)) {
905 wq_node_del(&wqe->work_list, node, prev);
906 found = true;
907 break;
908 }
909 }
910 spin_unlock_irqrestore(&wqe->lock, flags);
911
912 if (found) {
913 io_run_cancel(work);
914 return IO_WQ_CANCEL_OK;
915 }
916
917 rcu_read_lock();
918 found = io_wq_for_each_worker(wqe, io_work_cancel, &data);
919 rcu_read_unlock();
920 return found ? IO_WQ_CANCEL_RUNNING : IO_WQ_CANCEL_NOTFOUND;
921}
922
923enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
924 void *data)
925{
926 enum io_wq_cancel ret = IO_WQ_CANCEL_NOTFOUND;
927 int node;
928
929 for_each_node(node) {
930 struct io_wqe *wqe = wq->wqes[node];
931
932 ret = io_wqe_cancel_cb_work(wqe, cancel, data);
933 if (ret != IO_WQ_CANCEL_NOTFOUND)
934 break;
935 }
936
937 return ret;
938}
939
940struct work_match {
941 bool (*fn)(struct io_wq_work *, void *data);
942 void *data;
943};
944
945static bool io_wq_worker_cancel(struct io_worker *worker, void *data)
946{
947 struct work_match *match = data;
948 unsigned long flags;
949 bool ret = false;
950
951 spin_lock_irqsave(&worker->lock, flags);
952 if (match->fn(worker->cur_work, match->data) &&
953 !(worker->cur_work->flags & IO_WQ_WORK_NO_CANCEL)) {
954 send_sig(SIGINT, worker->task, 1);919 send_sig(SIGINT, worker->task, 1);
955 ret = true;920 ret = true;
956 }921 }
@@ -960,7 +925,7 @@ static bool io_wq_worker_cancel(struct io_worker *worker, void *data) · 3 unmodified lines
960}925}
961 926
962static enum io_wq_cancel io_wqe_cancel_work(struct io_wqe *wqe,927static enum io_wq_cancel io_wqe_cancel_work(struct io_wqe *wqe,
963 struct work_match *match)928 struct io_cb_cancel_data *match)
964{929{
965 struct io_wq_work_node *node, *prev;930 struct io_wq_work_node *node, *prev;
966 struct io_wq_work *work;931 struct io_wq_work *work;
@@ -977,7 +942,7 @@ static enum io_wq_cancel io_wqe_cancel_work(struct io_wqe *wqe, · 10 unmodified lines
977 work = container_of(node, struct io_wq_work, list);942 work = container_of(node, struct io_wq_work, list);
978 943
979 if (match->fn(work, match->data)) {944 if (match->fn(work, match->data)) {
980 wq_node_del(&wqe->work_list, node, prev);945 wq_list_del(&wqe->work_list, node, prev);
981 found = true;946 found = true;
982 break;947 break;
983 }948 }
@@ -985,7 +950,7 @@ static enum io_wq_cancel io_wqe_cancel_work(struct io_wqe *wqe, · 1 unmodified line
985 spin_unlock_irqrestore(&wqe->lock, flags);950 spin_unlock_irqrestore(&wqe->lock, flags);
986 951
987 if (found) {952 if (found) {
988 io_run_cancel(work);953 io_run_cancel(work, wqe);
989 return IO_WQ_CANCEL_OK;954 return IO_WQ_CANCEL_OK;
990 }955 }
991 956
@@ -1001,22 +966,16 @@ static enum io_wq_cancel io_wqe_cancel_work(struct io_wqe *wqe, · 9 unmodified lines
1001 return found ? IO_WQ_CANCEL_RUNNING : IO_WQ_CANCEL_NOTFOUND;966 return found ? IO_WQ_CANCEL_RUNNING : IO_WQ_CANCEL_NOTFOUND;
1002}967}
1003 968
1004static bool io_wq_work_match(struct io_wq_work *work, void *data)969enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
1005{970 void *data)
1006 return work == data;
1007}
1008
1009enum io_wq_cancel io_wq_cancel_work(struct io_wq *wq, struct io_wq_work *cwork)
1010{971{
1011 struct work_match match = {972 struct io_cb_cancel_data match = {
1012 .fn = io_wq_work_match,973 .fn = cancel,
1013 .data = cwork974 .data = data,
1014 };975 };
1015 enum io_wq_cancel ret = IO_WQ_CANCEL_NOTFOUND;976 enum io_wq_cancel ret = IO_WQ_CANCEL_NOTFOUND;
1016 int node;977 int node;
1017 978
1018 cwork->flags |= IO_WQ_WORK_CANCEL;
1019
1020 for_each_node(node) {979 for_each_node(node) {
1021 struct io_wqe *wqe = wq->wqes[node];980 struct io_wqe *wqe = wq->wqes[node];
1022 981
@@ -1028,33 +987,28 @@ enum io_wq_cancel io_wq_cancel_work(struct io_wq *wq, struct io_wq_work *cwork) · 5 unmodified lines
1028 return ret;987 return ret;
1029}988}
1030 989
990static bool io_wq_io_cb_cancel_data(struct io_wq_work *work, void *data)
991{
992 return work == data;
993}
994
995enum io_wq_cancel io_wq_cancel_work(struct io_wq *wq, struct io_wq_work *cwork)
996{
997 return io_wq_cancel_cb(wq, io_wq_io_cb_cancel_data, (void *)cwork);
998}
999
1031static bool io_wq_pid_match(struct io_wq_work *work, void *data)1000static bool io_wq_pid_match(struct io_wq_work *work, void *data)
1032{1001{
1033 pid_t pid = (pid_t) (unsigned long) data;1002 pid_t pid = (pid_t) (unsigned long) data;
1034 1003
1035 if (work)1004 return work->task_pid == pid;
1036 return work->task_pid == pid;
1037 return false;
1038}1005}
1039 1006
1040enum io_wq_cancel io_wq_cancel_pid(struct io_wq *wq, pid_t pid)1007enum io_wq_cancel io_wq_cancel_pid(struct io_wq *wq, pid_t pid)
1041{1008{
1042 struct work_match match = {1009 void *data = (void *) (unsigned long) pid;
1043 .fn = io_wq_pid_match,
1044 .data = (void *) (unsigned long) pid
1045 };
1046 enum io_wq_cancel ret = IO_WQ_CANCEL_NOTFOUND;
1047 int node;
1048
1049 for_each_node(node) {
1050 struct io_wqe *wqe = wq->wqes[node];
1051 1010
1052 ret = io_wqe_cancel_work(wqe, &match);1011 return io_wq_cancel_cb(wq, io_wq_pid_match, data);
1053 if (ret != IO_WQ_CANCEL_NOTFOUND)
1054 break;
1055 }
1056
1057 return ret;
1058}1012}
1059 1013
1060struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)1014struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)
@@ -1062,6 +1016,9 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) · 1 unmodified line
1062 int ret = -ENOMEM, node;1016 int ret = -ENOMEM, node;
1063 struct io_wq *wq;1017 struct io_wq *wq;
1064 1018
1019 if (WARN_ON_ONCE(!data->free_work))
1020 return ERR_PTR(-EINVAL);
1021
1065 wq = kzalloc(sizeof(*wq), GFP_KERNEL);1022 wq = kzalloc(sizeof(*wq), GFP_KERNEL);
1066 if (!wq)1023 if (!wq)
1067 return ERR_PTR(-ENOMEM);1024 return ERR_PTR(-ENOMEM);
@@ -1072,8 +1029,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) · 4 unmodified lines
1072 return ERR_PTR(-ENOMEM);1029 return ERR_PTR(-ENOMEM);
1073 }1030 }
1074 1031
1075 wq->get_work = data->get_work;1032 wq->free_work = data->free_work;
1076 wq->put_work = data->put_work;
1077 1033
1078 /* caller must already hold a reference to this */1034 /* caller must already hold a reference to this */
1079 wq->user = data->user;1035 wq->user = data->user;
@@ -1130,7 +1086,7 @@ err: · 50 unmodified lines
1130 1086
1131bool io_wq_get(struct io_wq *wq, struct io_wq_data *data)1087bool io_wq_get(struct io_wq *wq, struct io_wq_data *data)
1132{1088{
1133 if (data->get_work != wq->get_work || data->put_work != wq->put_work)1089 if (data->free_work != wq->free_work)
1134 return false;1090 return false;
1135 1091
1136 return refcount_inc_not_zero(&wq->use_refs);1092 return refcount_inc_not_zero(&wq->use_refs);
fs/io-wq.h+46−19
@@ -5,10 +5,8 @@ struct io_wq; · 4 unmodified lines
5 5
6enum {6enum {
7 IO_WQ_WORK_CANCEL = 1,7 IO_WQ_WORK_CANCEL = 1,
8 IO_WQ_WORK_HAS_MM = 2,
9 IO_WQ_WORK_HASHED = 4,8 IO_WQ_WORK_HASHED = 4,
10 IO_WQ_WORK_UNBOUND = 32,9 IO_WQ_WORK_UNBOUND = 32,
11 IO_WQ_WORK_CB = 128,
12 IO_WQ_WORK_NO_CANCEL = 256,10 IO_WQ_WORK_NO_CANCEL = 256,
13 IO_WQ_WORK_CONCURRENT = 512,11 IO_WQ_WORK_CONCURRENT = 512,
14 12
@@ -30,6 +28,18 @@ struct io_wq_work_list { · 15 unmodified lines
30 struct io_wq_work_node *last;28 struct io_wq_work_node *last;
31};29};
32 30
31static inline void wq_list_add_after(struct io_wq_work_node *node,
32 struct io_wq_work_node *pos,
33 struct io_wq_work_list *list)
34{
35 struct io_wq_work_node *next = pos->next;
36
37 pos->next = node;
38 node->next = next;
39 if (!next)
40 list->last = node;
41}
42
33static inline void wq_list_add_tail(struct io_wq_work_node *node,43static inline void wq_list_add_tail(struct io_wq_work_node *node,
34 struct io_wq_work_list *list)44 struct io_wq_work_list *list)
35{45{
@@ -42,17 +52,26 @@ static inline void wq_list_add_tail(struct io_wq_work_node *node, · 6 unmodified lines
42 }52 }
43}53}
44 54
45static inline void wq_node_del(struct io_wq_work_list *list,55static inline void wq_list_cut(struct io_wq_work_list *list,
46 struct io_wq_work_node *node,56 struct io_wq_work_node *last,
47 struct io_wq_work_node *prev)57 struct io_wq_work_node *prev)
48{58{
49 if (node == list->first)59 /* first in the list, if prev==NULL */
50 WRITE_ONCE(list->first, node->next);60 if (!prev)
51 if (node == list->last)61 WRITE_ONCE(list->first, last->next);
62 else
63 prev->next = last->next;
64
65 if (last == list->last)
52 list->last = prev;66 list->last = prev;
53 if (prev)67 last->next = NULL;
54 prev->next = node->next;68}
55 node->next = NULL;69
70static inline void wq_list_del(struct io_wq_work_list *list,
71 struct io_wq_work_node *node,
72 struct io_wq_work_node *prev)
73{
74 wq_list_cut(list, node, prev);
56}75}
57 76
58#define wq_list_for_each(pos, prv, head) \77#define wq_list_for_each(pos, prv, head) \
@@ -65,10 +84,7 @@ static inline void wq_node_del(struct io_wq_work_list *list, · 6 unmodified lines
65} while (0)84} while (0)
66 85
67struct io_wq_work {86struct io_wq_work {
68 union {87 struct io_wq_work_node list;
69 struct io_wq_work_node list;
70 void *data;
71 };
72 void (*func)(struct io_wq_work **);88 void (*func)(struct io_wq_work **);
73 struct files_struct *files;89 struct files_struct *files;
74 struct mm_struct *mm;90 struct mm_struct *mm;
@@ -83,14 +99,20 @@ struct io_wq_work { · 8 unmodified lines
83 *(work) = (struct io_wq_work){ .func = _func }; \99 *(work) = (struct io_wq_work){ .func = _func }; \
84 } while (0) \100 } while (0) \
85 101
86typedef void (get_work_fn)(struct io_wq_work *);102static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)
87typedef void (put_work_fn)(struct io_wq_work *);103{
104 if (!work->list.next)
105 return NULL;
106
107 return container_of(work->list.next, struct io_wq_work, list);
108}
109
110typedef void (free_work_fn)(struct io_wq_work *);
88 111
89struct io_wq_data {112struct io_wq_data {
90 struct user_struct *user;113 struct user_struct *user;
91 114
92 get_work_fn *get_work;115 free_work_fn *free_work;
93 put_work_fn *put_work;
94};116};
95 117
96struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data);118struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data);
@@ -98,7 +120,12 @@ bool io_wq_get(struct io_wq *wq, struct io_wq_data *data); · 1 unmodified line
98void io_wq_destroy(struct io_wq *wq);120void io_wq_destroy(struct io_wq *wq);
99 121
100void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work);122void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work);
101void io_wq_enqueue_hashed(struct io_wq *wq, struct io_wq_work *work, void *val);123void io_wq_hash_work(struct io_wq_work *work, void *val);
124
125static inline bool io_wq_is_hashed(struct io_wq_work *work)
126{
127 return work->flags & IO_WQ_WORK_HASHED;
128}
102 129
103void io_wq_cancel_all(struct io_wq *wq);130void io_wq_cancel_all(struct io_wq *wq);
104enum io_wq_cancel io_wq_cancel_work(struct io_wq *wq, struct io_wq_work *cwork);131enum io_wq_cancel io_wq_cancel_work(struct io_wq *wq, struct io_wq_work *cwork);
fs/io_uring.c+1406−609
@@ -44,6 +44,7 @@ · 43 unmodified lines
44#include <linux/errno.h>44#include <linux/errno.h>
45#include <linux/syscalls.h>45#include <linux/syscalls.h>
46#include <linux/compat.h>46#include <linux/compat.h>
47#include <net/compat.h>
47#include <linux/refcount.h>48#include <linux/refcount.h>
48#include <linux/uio.h>49#include <linux/uio.h>
49#include <linux/bits.h>50#include <linux/bits.h>
@@ -76,6 +77,8 @@ · 26 unmodified lines
76#include <linux/fadvise.h>77#include <linux/fadvise.h>
77#include <linux/eventpoll.h>78#include <linux/eventpoll.h>
78#include <linux/fs_struct.h>79#include <linux/fs_struct.h>
80#include <linux/splice.h>
81#include <linux/task_work.h>
79 82
80#define CREATE_TRACE_POINTS83#define CREATE_TRACE_POINTS
81#include <trace/events/io_uring.h>84#include <trace/events/io_uring.h>
@@ -193,6 +196,13 @@ struct fixed_file_data { · 111 unmodified lines
193 struct completion done;196 struct completion done;
194};197};
195 198
199struct io_buffer {
200 struct list_head list;
201 __u64 addr;
202 __s32 len;
203 __u16 bid;
204};
205
196struct io_ring_ctx {206struct io_ring_ctx {
197 struct {207 struct {
198 struct percpu_ref refs;208 struct percpu_ref refs;
@@ -270,6 +280,8 @@ struct io_ring_ctx { · 71 unmodified lines
270 struct socket *ring_sock;280 struct socket *ring_sock;
271#endif281#endif
272 282
283 struct idr io_buffer_idr;
284
273 struct idr personality_idr;285 struct idr personality_idr;
274 286
275 struct {287 struct {
@@ -290,7 +302,6 @@ struct io_ring_ctx { · 14 unmodified lines
290 302
291 struct {303 struct {
292 spinlock_t completion_lock;304 spinlock_t completion_lock;
293 struct llist_head poll_llist;
294 305
295 /*306 /*
296 * ->poll_list is protected by the ctx->uring_lock for307 * ->poll_list is protected by the ctx->uring_lock for
@@ -386,7 +397,9 @@ struct io_sr_msg { · 89 unmodified lines
386 void __user *buf;397 void __user *buf;
387 };398 };
388 int msg_flags;399 int msg_flags;
400 int bgid;
389 size_t len;401 size_t len;
402 struct io_buffer *kbuf;
390};403};
391 404
392struct io_open {405struct io_open {
@@ -430,6 +443,24 @@ struct io_epoll { · 37 unmodified lines
430 struct epoll_event event;443 struct epoll_event event;
431};444};
432 445
446struct io_splice {
447 struct file *file_out;
448 struct file *file_in;
449 loff_t off_out;
450 loff_t off_in;
451 u64 len;
452 unsigned int flags;
453};
454
455struct io_provide_buf {
456 struct file *file;
457 __u64 addr;
458 __s32 len;
459 __u32 bgid;
460 __u16 nbufs;
461 __u16 bid;
462};
463
433struct io_async_connect {464struct io_async_connect {
434 struct sockaddr_storage address;465 struct sockaddr_storage address;
435};466};
@@ -464,6 +495,7 @@ enum { · 28 unmodified lines
464 REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,495 REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
465 REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,496 REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
466 REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,497 REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
498 REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,
467 499
468 REQ_F_LINK_NEXT_BIT,500 REQ_F_LINK_NEXT_BIT,
469 REQ_F_FAIL_LINK_BIT,501 REQ_F_FAIL_LINK_BIT,
@@ -479,6 +511,11 @@ enum { · 9 unmodified lines
479 REQ_F_COMP_LOCKED_BIT,511 REQ_F_COMP_LOCKED_BIT,
480 REQ_F_NEED_CLEANUP_BIT,512 REQ_F_NEED_CLEANUP_BIT,
481 REQ_F_OVERFLOW_BIT,513 REQ_F_OVERFLOW_BIT,
514 REQ_F_POLLED_BIT,
515 REQ_F_BUFFER_SELECTED_BIT,
516
517 /* not a real bit, just to check we're not overflowing the space */
518 __REQ_F_LAST_BIT,
482};519};
483 520
484enum {521enum {
@@ -492,6 +529,8 @@ enum { · 7 unmodified lines
492 REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),529 REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
493 /* IOSQE_ASYNC */530 /* IOSQE_ASYNC */
494 REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),531 REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
532 /* IOSQE_BUFFER_SELECT */
533 REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT),
495 534
496 /* already grabbed next link */535 /* already grabbed next link */
497 REQ_F_LINK_NEXT = BIT(REQ_F_LINK_NEXT_BIT),536 REQ_F_LINK_NEXT = BIT(REQ_F_LINK_NEXT_BIT),
@@ -521,6 +560,15 @@ enum { · 23 unmodified lines
521 REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT),560 REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT),
522 /* in overflow list */561 /* in overflow list */
523 REQ_F_OVERFLOW = BIT(REQ_F_OVERFLOW_BIT),562 REQ_F_OVERFLOW = BIT(REQ_F_OVERFLOW_BIT),
563 /* already went through poll handler */
564 REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),
565 /* buffer already selected */
566 REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
567};
568
569struct async_poll {
570 struct io_poll_iocb poll;
571 struct io_wq_work work;
524};572};
525 573
526/*574/*
@@ -546,32 +594,45 @@ struct io_kiocb { · 19 unmodified lines
546 struct io_fadvise fadvise;594 struct io_fadvise fadvise;
547 struct io_madvise madvise;595 struct io_madvise madvise;
548 struct io_epoll epoll;596 struct io_epoll epoll;
597 struct io_splice splice;
598 struct io_provide_buf pbuf;
549 };599 };
550 600
551 struct io_async_ctx *io;601 struct io_async_ctx *io;
552 /*
553 * llist_node is only used for poll deferred completions
554 */
555 struct llist_node llist_node;
556 bool in_async;
557 bool needs_fixed_file;602 bool needs_fixed_file;
558 u8 opcode;603 u8 opcode;
559 604
560 struct io_ring_ctx *ctx;605 struct io_ring_ctx *ctx;
561 union {606 struct list_head list;
562 struct list_head list;
563 struct hlist_node hash_node;
564 };
565 struct list_head link_list;
566 unsigned int flags;607 unsigned int flags;
567 refcount_t refs;608 refcount_t refs;
609 union {
610 struct task_struct *task;
611 unsigned long fsize;
612 };
568 u64 user_data;613 u64 user_data;
569 u32 result;614 u32 result;
570 u32 sequence;615 u32 sequence;
571 616
617 struct list_head link_list;
618
572 struct list_head inflight_entry;619 struct list_head inflight_entry;
573 620
574 struct io_wq_work work;621 union {
622 /*
623 * Only commands that never go async can use the below fields,
624 * obviously. Right now only IORING_OP_POLL_ADD uses them, and
625 * async armed poll handlers for regular commands. The latter
626 * restore the work, if needed.
627 */
628 struct {
629 struct callback_head task_work;
630 struct hlist_node hash_node;
631 struct async_poll *apoll;
632 int cflags;
633 };
634 struct io_wq_work work;
635 };
575};636};
576 637
577#define IO_PLUG_THRESHOLD 2638#define IO_PLUG_THRESHOLD 2
@@ -615,6 +676,11 @@ struct io_op_def { · 37 unmodified lines
615 unsigned file_table : 1;676 unsigned file_table : 1;
616 /* needs ->fs */677 /* needs ->fs */
617 unsigned needs_fs : 1;678 unsigned needs_fs : 1;
679 /* set if opcode supports polled "wait" */
680 unsigned pollin : 1;
681 unsigned pollout : 1;
682 /* op supports buffer selection */
683 unsigned buffer_select : 1;
618};684};
619 685
620static const struct io_op_def io_op_defs[] = {686static const struct io_op_def io_op_defs[] = {
@@ -624,6 +690,8 @@ static const struct io_op_def io_op_defs[] = { · 3 unmodified lines
624 .needs_mm = 1,690 .needs_mm = 1,
625 .needs_file = 1,691 .needs_file = 1,
626 .unbound_nonreg_file = 1,692 .unbound_nonreg_file = 1,
693 .pollin = 1,
694 .buffer_select = 1,
627 },695 },
628 [IORING_OP_WRITEV] = {696 [IORING_OP_WRITEV] = {
629 .async_ctx = 1,697 .async_ctx = 1,
@@ -631,6 +699,7 @@ static const struct io_op_def io_op_defs[] = { · 1 unmodified line
631 .needs_file = 1,699 .needs_file = 1,
632 .hash_reg_file = 1,700 .hash_reg_file = 1,
633 .unbound_nonreg_file = 1,701 .unbound_nonreg_file = 1,
702 .pollout = 1,
634 },703 },
635 [IORING_OP_FSYNC] = {704 [IORING_OP_FSYNC] = {
636 .needs_file = 1,705 .needs_file = 1,
@@ -638,11 +707,13 @@ static const struct io_op_def io_op_defs[] = { · 1 unmodified line
638 [IORING_OP_READ_FIXED] = {707 [IORING_OP_READ_FIXED] = {
639 .needs_file = 1,708 .needs_file = 1,
640 .unbound_nonreg_file = 1,709 .unbound_nonreg_file = 1,
710 .pollin = 1,
641 },711 },
642 [IORING_OP_WRITE_FIXED] = {712 [IORING_OP_WRITE_FIXED] = {
643 .needs_file = 1,713 .needs_file = 1,
644 .hash_reg_file = 1,714 .hash_reg_file = 1,
645 .unbound_nonreg_file = 1,715 .unbound_nonreg_file = 1,
716 .pollout = 1,
646 },717 },
647 [IORING_OP_POLL_ADD] = {718 [IORING_OP_POLL_ADD] = {
648 .needs_file = 1,719 .needs_file = 1,
@@ -658,6 +729,7 @@ static const struct io_op_def io_op_defs[] = { · 9 unmodified lines
658 .needs_file = 1,729 .needs_file = 1,
659 .unbound_nonreg_file = 1,730 .unbound_nonreg_file = 1,
660 .needs_fs = 1,731 .needs_fs = 1,
732 .pollout = 1,
661 },733 },
662 [IORING_OP_RECVMSG] = {734 [IORING_OP_RECVMSG] = {
663 .async_ctx = 1,735 .async_ctx = 1,
@@ -665,6 +737,8 @@ static const struct io_op_def io_op_defs[] = { · 1 unmodified line
665 .needs_file = 1,737 .needs_file = 1,
666 .unbound_nonreg_file = 1,738 .unbound_nonreg_file = 1,
667 .needs_fs = 1,739 .needs_fs = 1,
740 .pollin = 1,
741 .buffer_select = 1,
668 },742 },
669 [IORING_OP_TIMEOUT] = {743 [IORING_OP_TIMEOUT] = {
670 .async_ctx = 1,744 .async_ctx = 1,
@@ -676,6 +750,7 @@ static const struct io_op_def io_op_defs[] = { · 5 unmodified lines
676 .needs_file = 1,750 .needs_file = 1,
677 .unbound_nonreg_file = 1,751 .unbound_nonreg_file = 1,
678 .file_table = 1,752 .file_table = 1,
753 .pollin = 1,
679 },754 },
680 [IORING_OP_ASYNC_CANCEL] = {},755 [IORING_OP_ASYNC_CANCEL] = {},
681 [IORING_OP_LINK_TIMEOUT] = {756 [IORING_OP_LINK_TIMEOUT] = {
@@ -687,6 +762,7 @@ static const struct io_op_def io_op_defs[] = { · 5 unmodified lines
687 .needs_mm = 1,762 .needs_mm = 1,
688 .needs_file = 1,763 .needs_file = 1,
689 .unbound_nonreg_file = 1,764 .unbound_nonreg_file = 1,
765 .pollout = 1,
690 },766 },
691 [IORING_OP_FALLOCATE] = {767 [IORING_OP_FALLOCATE] = {
692 .needs_file = 1,768 .needs_file = 1,
@@ -715,11 +791,14 @@ static const struct io_op_def io_op_defs[] = { · 22 unmodified lines
715 .needs_mm = 1,791 .needs_mm = 1,
716 .needs_file = 1,792 .needs_file = 1,
717 .unbound_nonreg_file = 1,793 .unbound_nonreg_file = 1,
794 .pollin = 1,
795 .buffer_select = 1,
718 },796 },
719 [IORING_OP_WRITE] = {797 [IORING_OP_WRITE] = {
720 .needs_mm = 1,798 .needs_mm = 1,
721 .needs_file = 1,799 .needs_file = 1,
722 .unbound_nonreg_file = 1,800 .unbound_nonreg_file = 1,
801 .pollout = 1,
723 },802 },
724 [IORING_OP_FADVISE] = {803 [IORING_OP_FADVISE] = {
725 .needs_file = 1,804 .needs_file = 1,
@@ -731,11 +810,14 @@ static const struct io_op_def io_op_defs[] = { · 5 unmodified lines
731 .needs_mm = 1,810 .needs_mm = 1,
732 .needs_file = 1,811 .needs_file = 1,
733 .unbound_nonreg_file = 1,812 .unbound_nonreg_file = 1,
813 .pollout = 1,
734 },814 },
735 [IORING_OP_RECV] = {815 [IORING_OP_RECV] = {
736 .needs_mm = 1,816 .needs_mm = 1,
737 .needs_file = 1,817 .needs_file = 1,
738 .unbound_nonreg_file = 1,818 .unbound_nonreg_file = 1,
819 .pollin = 1,
820 .buffer_select = 1,
739 },821 },
740 [IORING_OP_OPENAT2] = {822 [IORING_OP_OPENAT2] = {
741 .needs_file = 1,823 .needs_file = 1,
@@ -747,6 +829,13 @@ static const struct io_op_def io_op_defs[] = { · 5 unmodified lines
747 .unbound_nonreg_file = 1,829 .unbound_nonreg_file = 1,
748 .file_table = 1,830 .file_table = 1,
749 },831 },
832 [IORING_OP_SPLICE] = {
833 .needs_file = 1,
834 .hash_reg_file = 1,
835 .unbound_nonreg_file = 1,
836 },
837 [IORING_OP_PROVIDE_BUFFERS] = {},
838 [IORING_OP_REMOVE_BUFFERS] = {},
750};839};
751 840
752static void io_wq_submit_work(struct io_wq_work **workptr);841static void io_wq_submit_work(struct io_wq_work **workptr);
@@ -761,6 +850,10 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, · 8 unmodified lines
761static int io_grab_files(struct io_kiocb *req);850static int io_grab_files(struct io_kiocb *req);
762static void io_ring_file_ref_flush(struct fixed_file_data *data);851static void io_ring_file_ref_flush(struct fixed_file_data *data);
763static void io_cleanup_req(struct io_kiocb *req);852static void io_cleanup_req(struct io_kiocb *req);
853static int io_file_get(struct io_submit_state *state, struct io_kiocb *req,
854 int fd, struct file **out_file, bool fixed);
855static void __io_queue_sqe(struct io_kiocb *req,
856 const struct io_uring_sqe *sqe);
764 857
765static struct kmem_cache *req_cachep;858static struct kmem_cache *req_cachep;
766 859
@@ -827,11 +920,11 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) · 60 unmodified lines
827 INIT_LIST_HEAD(&ctx->cq_overflow_list);920 INIT_LIST_HEAD(&ctx->cq_overflow_list);
828 init_completion(&ctx->completions[0]);921 init_completion(&ctx->completions[0]);
829 init_completion(&ctx->completions[1]);922 init_completion(&ctx->completions[1]);
923 idr_init(&ctx->io_buffer_idr);
830 idr_init(&ctx->personality_idr);924 idr_init(&ctx->personality_idr);
831 mutex_init(&ctx->uring_lock);925 mutex_init(&ctx->uring_lock);
832 init_waitqueue_head(&ctx->wait);926 init_waitqueue_head(&ctx->wait);
833 spin_lock_init(&ctx->completion_lock);927 spin_lock_init(&ctx->completion_lock);
834 init_llist_head(&ctx->poll_llist);
835 INIT_LIST_HEAD(&ctx->poll_list);928 INIT_LIST_HEAD(&ctx->poll_list);
836 INIT_LIST_HEAD(&ctx->defer_list);929 INIT_LIST_HEAD(&ctx->defer_list);
837 INIT_LIST_HEAD(&ctx->timeout_list);930 INIT_LIST_HEAD(&ctx->timeout_list);
@@ -952,15 +1045,14 @@ static inline void io_req_work_drop_env(struct io_kiocb *req) · 114 unmodified lines
952 }1045 }
953}1046}
954 1047
955static inline bool io_prep_async_work(struct io_kiocb *req,1048static inline void io_prep_async_work(struct io_kiocb *req,
956 struct io_kiocb **link)1049 struct io_kiocb **link)
957{1050{
958 const struct io_op_def *def = &io_op_defs[req->opcode];1051 const struct io_op_def *def = &io_op_defs[req->opcode];
959 bool do_hashed = false;
960 1052
961 if (req->flags & REQ_F_ISREG) {1053 if (req->flags & REQ_F_ISREG) {
962 if (def->hash_reg_file)1054 if (def->hash_reg_file)
963 do_hashed = true;1055 io_wq_hash_work(&req->work, file_inode(req->file));
964 } else {1056 } else {
965 if (def->unbound_nonreg_file)1057 if (def->unbound_nonreg_file)
966 req->work.flags |= IO_WQ_WORK_UNBOUND;1058 req->work.flags |= IO_WQ_WORK_UNBOUND;
@@ -969,25 +1061,18 @@ static inline bool io_prep_async_work(struct io_kiocb *req, · 2 unmodified lines
969 io_req_work_grab_env(req, def);1061 io_req_work_grab_env(req, def);
970 1062
971 *link = io_prep_linked_timeout(req);1063 *link = io_prep_linked_timeout(req);
972 return do_hashed;
973}1064}
974 1065
975static inline void io_queue_async_work(struct io_kiocb *req)1066static inline void io_queue_async_work(struct io_kiocb *req)
976{1067{
977 struct io_ring_ctx *ctx = req->ctx;1068 struct io_ring_ctx *ctx = req->ctx;
978 struct io_kiocb *link;1069 struct io_kiocb *link;
979 bool do_hashed;
980 1070
981 do_hashed = io_prep_async_work(req, &link);1071 io_prep_async_work(req, &link);
982 1072
983 trace_io_uring_queue_async_work(ctx, do_hashed, req, &req->work,1073 trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
984 req->flags);1074 &req->work, req->flags);
985 if (!do_hashed) {1075 io_wq_enqueue(ctx->io_wq, &req->work);
986 io_wq_enqueue(ctx->io_wq, &req->work);
987 } else {
988 io_wq_enqueue_hashed(ctx->io_wq, &req->work,
989 file_inode(req->file));
990 }
991 1076
992 if (link)1077 if (link)
993 io_queue_linked_timeout(link);1078 io_queue_linked_timeout(link);
@@ -1054,24 +1139,19 @@ static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx) · 60 unmodified lines
1054 return false;1139 return false;
1055 if (!ctx->eventfd_async)1140 if (!ctx->eventfd_async)
1056 return true;1141 return true;
1057 return io_wq_current_is_worker() || in_interrupt();1142 return io_wq_current_is_worker();
1058}1143}
1059 1144
1060static void __io_cqring_ev_posted(struct io_ring_ctx *ctx, bool trigger_ev)1145static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1061{1146{
1062 if (waitqueue_active(&ctx->wait))1147 if (waitqueue_active(&ctx->wait))
1063 wake_up(&ctx->wait);1148 wake_up(&ctx->wait);
1064 if (waitqueue_active(&ctx->sqo_wait))1149 if (waitqueue_active(&ctx->sqo_wait))
1065 wake_up(&ctx->sqo_wait);1150 wake_up(&ctx->sqo_wait);
1066 if (trigger_ev)1151 if (io_should_trigger_evfd(ctx))
1067 eventfd_signal(ctx->cq_ev_fd, 1);1152 eventfd_signal(ctx->cq_ev_fd, 1);
1068}1153}
1069 1154
1070static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1071{
1072 __io_cqring_ev_posted(ctx, io_should_trigger_evfd(ctx));
1073}
1074
1075/* Returns true if there are no backlogged entries after the flush */1155/* Returns true if there are no backlogged entries after the flush */
1076static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)1156static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
1077{1157{
@@ -1108,7 +1188,7 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) · 30 unmodified lines
1108 if (cqe) {1188 if (cqe) {
1109 WRITE_ONCE(cqe->user_data, req->user_data);1189 WRITE_ONCE(cqe->user_data, req->user_data);
1110 WRITE_ONCE(cqe->res, req->result);1190 WRITE_ONCE(cqe->res, req->result);
1111 WRITE_ONCE(cqe->flags, 0);1191 WRITE_ONCE(cqe->flags, req->cflags);
1112 } else {1192 } else {
1113 WRITE_ONCE(ctx->rings->cq_overflow,1193 WRITE_ONCE(ctx->rings->cq_overflow,
1114 atomic_inc_return(&ctx->cached_cq_overflow));1194 atomic_inc_return(&ctx->cached_cq_overflow));
@@ -1132,7 +1212,7 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) · 17 unmodified lines
1132 return cqe != NULL;1212 return cqe != NULL;
1133}1213}
1134 1214
1135static void io_cqring_fill_event(struct io_kiocb *req, long res)1215static void __io_cqring_fill_event(struct io_kiocb *req, long res, long cflags)
1136{1216{
1137 struct io_ring_ctx *ctx = req->ctx;1217 struct io_ring_ctx *ctx = req->ctx;
1138 struct io_uring_cqe *cqe;1218 struct io_uring_cqe *cqe;
@@ -1148,7 +1228,7 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res) · 9 unmodified lines
1148 if (likely(cqe)) {1228 if (likely(cqe)) {
1149 WRITE_ONCE(cqe->user_data, req->user_data);1229 WRITE_ONCE(cqe->user_data, req->user_data);
1150 WRITE_ONCE(cqe->res, res);1230 WRITE_ONCE(cqe->res, res);
1151 WRITE_ONCE(cqe->flags, 0);1231 WRITE_ONCE(cqe->flags, cflags);
1152 } else if (ctx->cq_overflow_flushed) {1232 } else if (ctx->cq_overflow_flushed) {
1153 WRITE_ONCE(ctx->rings->cq_overflow,1233 WRITE_ONCE(ctx->rings->cq_overflow,
1154 atomic_inc_return(&ctx->cached_cq_overflow));1234 atomic_inc_return(&ctx->cached_cq_overflow));
@@ -1160,23 +1240,34 @@ static void io_cqring_fill_event(struct io_kiocb *req, long res) · 5 unmodified lines
1160 req->flags |= REQ_F_OVERFLOW;1240 req->flags |= REQ_F_OVERFLOW;
1161 refcount_inc(&req->refs);1241 refcount_inc(&req->refs);
1162 req->result = res;1242 req->result = res;
1243 req->cflags = cflags;
1163 list_add_tail(&req->list, &ctx->cq_overflow_list);1244 list_add_tail(&req->list, &ctx->cq_overflow_list);
1164 }1245 }
1165}1246}
1166 1247
1167static void io_cqring_add_event(struct io_kiocb *req, long res)1248static void io_cqring_fill_event(struct io_kiocb *req, long res)
1249{
1250 __io_cqring_fill_event(req, res, 0);
1251}
1252
1253static void __io_cqring_add_event(struct io_kiocb *req, long res, long cflags)
1168{1254{
1169 struct io_ring_ctx *ctx = req->ctx;1255 struct io_ring_ctx *ctx = req->ctx;
1170 unsigned long flags;1256 unsigned long flags;
1171 1257
1172 spin_lock_irqsave(&ctx->completion_lock, flags);1258 spin_lock_irqsave(&ctx->completion_lock, flags);
1173 io_cqring_fill_event(req, res);1259 __io_cqring_fill_event(req, res, cflags);
1174 io_commit_cqring(ctx);1260 io_commit_cqring(ctx);
1175 spin_unlock_irqrestore(&ctx->completion_lock, flags);1261 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1176 1262
1177 io_cqring_ev_posted(ctx);1263 io_cqring_ev_posted(ctx);
1178}1264}
1179 1265
1266static void io_cqring_add_event(struct io_kiocb *req, long res)
1267{
1268 __io_cqring_add_event(req, res, 0);
1269}
1270
1180static inline bool io_is_fallback_req(struct io_kiocb *req)1271static inline bool io_is_fallback_req(struct io_kiocb *req)
1181{1272{
1182 return req == (struct io_kiocb *)1273 return req == (struct io_kiocb *)
@@ -1246,6 +1337,15 @@ fallback: · 63 unmodified lines
1246 return NULL;1337 return NULL;
1247}1338}
1248 1339
1340static inline void io_put_file(struct io_kiocb *req, struct file *file,
1341 bool fixed)
1342{
1343 if (fixed)
1344 percpu_ref_put(&req->ctx->file_data->refs);
1345 else
1346 fput(file);
1347}
1348
1249static void __io_req_do_free(struct io_kiocb *req)1349static void __io_req_do_free(struct io_kiocb *req)
1250{1350{
1251 if (likely(!io_is_fallback_req(req)))1351 if (likely(!io_is_fallback_req(req)))
@@ -1256,18 +1356,12 @@ static void __io_req_do_free(struct io_kiocb *req) · 4 unmodified lines
1256 1356
1257static void __io_req_aux_free(struct io_kiocb *req)1357static void __io_req_aux_free(struct io_kiocb *req)
1258{1358{
1259 struct io_ring_ctx *ctx = req->ctx;
1260
1261 if (req->flags & REQ_F_NEED_CLEANUP)1359 if (req->flags & REQ_F_NEED_CLEANUP)
1262 io_cleanup_req(req);1360 io_cleanup_req(req);
1263 1361
1264 kfree(req->io);1362 kfree(req->io);
1265 if (req->file) {1363 if (req->file)
1266 if (req->flags & REQ_F_FIXED_FILE)1364 io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE));
1267 percpu_ref_put(&ctx->file_data->refs);
1268 else
1269 fput(req->file);
1270 }
1271 1365
1272 io_req_work_drop_env(req);1366 io_req_work_drop_env(req);
1273}1367}
@@ -1474,6 +1568,30 @@ static void io_free_req(struct io_kiocb *req) · 200 unmodified lines
1474 io_queue_async_work(nxt);1568 io_queue_async_work(nxt);
1475}1569}
1476 1570
1571static void io_link_work_cb(struct io_wq_work **workptr)
1572{
1573 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
1574 struct io_kiocb *link;
1575
1576 link = list_first_entry(&req->link_list, struct io_kiocb, link_list);
1577 io_queue_linked_timeout(link);
1578 io_wq_submit_work(workptr);
1579}
1580
1581static void io_wq_assign_next(struct io_wq_work **workptr, struct io_kiocb *nxt)
1582{
1583 struct io_kiocb *link;
1584 const struct io_op_def *def = &io_op_defs[nxt->opcode];
1585
1586 if ((nxt->flags & REQ_F_ISREG) && def->hash_reg_file)
1587 io_wq_hash_work(&nxt->work, file_inode(nxt->file));
1588
1589 *workptr = &nxt->work;
1590 link = io_prep_linked_timeout(nxt);
1591 if (link)
1592 nxt->work.func = io_link_work_cb;
1593}
1594
1477/*1595/*
1478 * Drop reference to request, return next in chain (if there is one) if this1596 * Drop reference to request, return next in chain (if there is one) if this
1479 * was the last reference to this request.1597 * was the last reference to this request.
@@ -1493,6 +1611,26 @@ static void io_put_req(struct io_kiocb *req) · 13 unmodified lines
1493 io_free_req(req);1611 io_free_req(req);
1494}1612}
1495 1613
1614static void io_steal_work(struct io_kiocb *req,
1615 struct io_wq_work **workptr)
1616{
1617 /*
1618 * It's in an io-wq worker, so there always should be at least
1619 * one reference, which will be dropped in io_put_work() just
1620 * after the current handler returns.
1621 *
1622 * It also means, that if the counter dropped to 1, then there is
1623 * no asynchronous users left, so it's safe to steal the next work.
1624 */
1625 if (refcount_read(&req->refs) == 1) {
1626 struct io_kiocb *nxt = NULL;
1627
1628 io_req_find_next(req, &nxt);
1629 if (nxt)
1630 io_wq_assign_next(workptr, nxt);
1631 }
1632}
1633
1496/*1634/*
1497 * Must only be used if we don't need to care about links, usually from1635 * Must only be used if we don't need to care about links, usually from
1498 * within the completion handling itself.1636 * within the completion handling itself.
@@ -1554,6 +1692,19 @@ static inline bool io_req_multi_free(struct req_batch *rb, struct io_kiocb *req) · 55 unmodified lines
1554 return true;1692 return true;
1555}1693}
1556 1694
1695static int io_put_kbuf(struct io_kiocb *req)
1696{
1697 struct io_buffer *kbuf;
1698 int cflags;
1699
1700 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
1701 cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
1702 cflags |= IORING_CQE_F_BUFFER;
1703 req->rw.addr = 0;
1704 kfree(kbuf);
1705 return cflags;
1706}
1707
1557/*1708/*
1558 * Find and free completed poll iocbs1709 * Find and free completed poll iocbs
1559 */1710 */
@@ -1565,10 +1716,15 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, · 5 unmodified lines
1565 1716
1566 rb.to_free = rb.need_iter = 0;1717 rb.to_free = rb.need_iter = 0;
1567 while (!list_empty(done)) {1718 while (!list_empty(done)) {
1719 int cflags = 0;
1720
1568 req = list_first_entry(done, struct io_kiocb, list);1721 req = list_first_entry(done, struct io_kiocb, list);
1569 list_del(&req->list);1722 list_del(&req->list);
1570 1723
1571 io_cqring_fill_event(req, req->result);1724 if (req->flags & REQ_F_BUFFER_SELECTED)
1725 cflags = io_put_kbuf(req);
1726
1727 __io_cqring_fill_event(req, req->result, cflags);
1572 (*nr_events)++;1728 (*nr_events)++;
1573 1729
1574 if (refcount_dec_and_test(&req->refs) &&1730 if (refcount_dec_and_test(&req->refs) &&
@@ -1577,6 +1733,8 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, · 2 unmodified lines
1577 }1733 }
1578 1734
1579 io_commit_cqring(ctx);1735 io_commit_cqring(ctx);
1736 if (ctx->flags & IORING_SETUP_SQPOLL)
1737 io_cqring_ev_posted(ctx);
1580 io_free_req_many(ctx, &rb);1738 io_free_req_many(ctx, &rb);
1581}1739}
1582 1740
@@ -1743,13 +1901,16 @@ static inline void req_set_fail_links(struct io_kiocb *req) · 160 unmodified lines
1743static void io_complete_rw_common(struct kiocb *kiocb, long res)1901static void io_complete_rw_common(struct kiocb *kiocb, long res)
1744{1902{
1745 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);1903 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1904 int cflags = 0;
1746 1905
1747 if (kiocb->ki_flags & IOCB_WRITE)1906 if (kiocb->ki_flags & IOCB_WRITE)
1748 kiocb_end_write(req);1907 kiocb_end_write(req);
1749 1908
1750 if (res != req->result)1909 if (res != req->result)
1751 req_set_fail_links(req);1910 req_set_fail_links(req);
1752 io_cqring_add_event(req, res);1911 if (req->flags & REQ_F_BUFFER_SELECTED)
1912 cflags = io_put_kbuf(req);
1913 __io_cqring_add_event(req, res, cflags);
1753}1914}
1754 1915
1755static void io_complete_rw(struct kiocb *kiocb, long res, long res2)1916static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
@@ -1760,17 +1921,6 @@ static void io_complete_rw(struct kiocb *kiocb, long res, long res2) · 4 unmodified lines
1760 io_put_req(req);1921 io_put_req(req);
1761}1922}
1762 1923
1763static struct io_kiocb *__io_complete_rw(struct kiocb *kiocb, long res)
1764{
1765 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1766 struct io_kiocb *nxt = NULL;
1767
1768 io_complete_rw_common(kiocb, res);
1769 io_put_req_find_next(req, &nxt);
1770
1771 return nxt;
1772}
1773
1774static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)1924static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
1775{1925{
1776 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);1926 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
@@ -1841,7 +1991,7 @@ static void io_file_put(struct io_submit_state *state) · 64 unmodified lines
1841 * assuming most submissions are for one file, or at least that each file1991 * assuming most submissions are for one file, or at least that each file
1842 * has more than one submission.1992 * has more than one submission.
1843 */1993 */
1844static struct file *io_file_get(struct io_submit_state *state, int fd)1994static struct file *__io_file_get(struct io_submit_state *state, int fd)
1845{1995{
1846 if (!state)1996 if (!state)
1847 return fget(fd);1997 return fget(fd);
@@ -1938,7 +2088,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 90 unmodified lines
1938 2088
1939 req->rw.addr = READ_ONCE(sqe->addr);2089 req->rw.addr = READ_ONCE(sqe->addr);
1940 req->rw.len = READ_ONCE(sqe->len);2090 req->rw.len = READ_ONCE(sqe->len);
1941 /* we own ->private, reuse it for the buffer index */2091 /* we own ->private, reuse it for the buffer index / buffer ID */
1942 req->rw.kiocb.private = (void *) (unsigned long)2092 req->rw.kiocb.private = (void *) (unsigned long)
1943 READ_ONCE(sqe->buf_index);2093 READ_ONCE(sqe->buf_index);
1944 return 0;2094 return 0;
@@ -1965,15 +2115,14 @@ static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret) · 20 unmodified lines
1965 }2115 }
1966}2116}
1967 2117
1968static void kiocb_done(struct kiocb *kiocb, ssize_t ret, struct io_kiocb **nxt,2118static void kiocb_done(struct kiocb *kiocb, ssize_t ret)
1969 bool in_async)
1970{2119{
1971 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);2120 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1972 2121
1973 if (req->flags & REQ_F_CUR_POS)2122 if (req->flags & REQ_F_CUR_POS)
1974 req->file->f_pos = kiocb->ki_pos;2123 req->file->f_pos = kiocb->ki_pos;
1975 if (in_async && ret >= 0 && kiocb->ki_complete == io_complete_rw)2124 if (ret >= 0 && kiocb->ki_complete == io_complete_rw)
1976 *nxt = __io_complete_rw(kiocb, ret);2125 io_complete_rw(kiocb, ret, 0);
1977 else2126 else
1978 io_rw_done(kiocb, ret);2127 io_rw_done(kiocb, ret);
1979}2128}
@@ -2052,11 +2201,147 @@ static ssize_t io_import_fixed(struct io_kiocb *req, int rw, · 72 unmodified lines
2052 return len;2201 return len;
2053}2202}
2054 2203
2204static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
2205{
2206 if (needs_lock)
2207 mutex_unlock(&ctx->uring_lock);
2208}
2209
2210static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
2211{
2212 /*
2213 * "Normal" inline submissions always hold the uring_lock, since we
2214 * grab it from the system call. Same is true for the SQPOLL offload.
2215 * The only exception is when we've detached the request and issue it
2216 * from an async worker thread, grab the lock for that case.
2217 */
2218 if (needs_lock)
2219 mutex_lock(&ctx->uring_lock);
2220}
2221
2222static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,
2223 int bgid, struct io_buffer *kbuf,
2224 bool needs_lock)
2225{
2226 struct io_buffer *head;
2227
2228 if (req->flags & REQ_F_BUFFER_SELECTED)
2229 return kbuf;
2230
2231 io_ring_submit_lock(req->ctx, needs_lock);
2232
2233 lockdep_assert_held(&req->ctx->uring_lock);
2234
2235 head = idr_find(&req->ctx->io_buffer_idr, bgid);
2236 if (head) {
2237 if (!list_empty(&head->list)) {
2238 kbuf = list_last_entry(&head->list, struct io_buffer,
2239 list);
2240 list_del(&kbuf->list);
2241 } else {
2242 kbuf = head;
2243 idr_remove(&req->ctx->io_buffer_idr, bgid);
2244 }
2245 if (*len > kbuf->len)
2246 *len = kbuf->len;
2247 } else {
2248 kbuf = ERR_PTR(-ENOBUFS);
2249 }
2250
2251 io_ring_submit_unlock(req->ctx, needs_lock);
2252
2253 return kbuf;
2254}
2255
2256static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,
2257 bool needs_lock)
2258{
2259 struct io_buffer *kbuf;
2260 int bgid;
2261
2262 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2263 bgid = (int) (unsigned long) req->rw.kiocb.private;
2264 kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock);
2265 if (IS_ERR(kbuf))
2266 return kbuf;
2267 req->rw.addr = (u64) (unsigned long) kbuf;
2268 req->flags |= REQ_F_BUFFER_SELECTED;
2269 return u64_to_user_ptr(kbuf->addr);
2270}
2271
2272#ifdef CONFIG_COMPAT
2273static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,
2274 bool needs_lock)
2275{
2276 struct compat_iovec __user *uiov;
2277 compat_ssize_t clen;
2278 void __user *buf;
2279 ssize_t len;
2280
2281 uiov = u64_to_user_ptr(req->rw.addr);
2282 if (!access_ok(uiov, sizeof(*uiov)))
2283 return -EFAULT;
2284 if (__get_user(clen, &uiov->iov_len))
2285 return -EFAULT;
2286 if (clen < 0)
2287 return -EINVAL;
2288
2289 len = clen;
2290 buf = io_rw_buffer_select(req, &len, needs_lock);
2291 if (IS_ERR(buf))
2292 return PTR_ERR(buf);
2293 iov[0].iov_base = buf;
2294 iov[0].iov_len = (compat_size_t) len;
2295 return 0;
2296}
2297#endif
2298
2299static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
2300 bool needs_lock)
2301{
2302 struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr);
2303 void __user *buf;
2304 ssize_t len;
2305
2306 if (copy_from_user(iov, uiov, sizeof(*uiov)))
2307 return -EFAULT;
2308
2309 len = iov[0].iov_len;
2310 if (len < 0)
2311 return -EINVAL;
2312 buf = io_rw_buffer_select(req, &len, needs_lock);
2313 if (IS_ERR(buf))
2314 return PTR_ERR(buf);
2315 iov[0].iov_base = buf;
2316 iov[0].iov_len = len;
2317 return 0;
2318}
2319
2320static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
2321 bool needs_lock)
2322{
2323 if (req->flags & REQ_F_BUFFER_SELECTED)
2324 return 0;
2325 if (!req->rw.len)
2326 return 0;
2327 else if (req->rw.len > 1)
2328 return -EINVAL;
2329
2330#ifdef CONFIG_COMPAT
2331 if (req->ctx->compat)
2332 return io_compat_import(req, iov, needs_lock);
2333#endif
2334
2335 return __io_iov_buffer_select(req, iov, needs_lock);
2336}
2337
2055static ssize_t io_import_iovec(int rw, struct io_kiocb *req,2338static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
2056 struct iovec **iovec, struct iov_iter *iter)2339 struct iovec **iovec, struct iov_iter *iter,
2340 bool needs_lock)
2057{2341{
2058 void __user *buf = u64_to_user_ptr(req->rw.addr);2342 void __user *buf = u64_to_user_ptr(req->rw.addr);
2059 size_t sqe_len = req->rw.len;2343 size_t sqe_len = req->rw.len;
2344 ssize_t ret;
2060 u8 opcode;2345 u8 opcode;
2061 2346
2062 opcode = req->opcode;2347 opcode = req->opcode;
@@ -2065,12 +2350,20 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req, · 2 unmodified lines
2065 return io_import_fixed(req, rw, iter);2350 return io_import_fixed(req, rw, iter);
2066 }2351 }
2067 2352
2068 /* buffer index only valid with fixed read/write */2353 /* buffer index only valid with fixed read/write, or buffer select */
2069 if (req->rw.kiocb.private)2354 if (req->rw.kiocb.private && !(req->flags & REQ_F_BUFFER_SELECT))
2070 return -EINVAL;2355 return -EINVAL;
2071 2356
2072 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {2357 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
2073 ssize_t ret;2358 if (req->flags & REQ_F_BUFFER_SELECT) {
2359 buf = io_rw_buffer_select(req, &sqe_len, needs_lock);
2360 if (IS_ERR(buf)) {
2361 *iovec = NULL;
2362 return PTR_ERR(buf);
2363 }
2364 req->rw.len = sqe_len;
2365 }
2366
2074 ret = import_single_range(rw, buf, sqe_len, *iovec, iter);2367 ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
2075 *iovec = NULL;2368 *iovec = NULL;
2076 return ret < 0 ? ret : sqe_len;2369 return ret < 0 ? ret : sqe_len;
@@ -2086,6 +2379,16 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req, · 9 unmodified lines
2086 return iorw->size;2379 return iorw->size;
2087 }2380 }
2088 2381
2382 if (req->flags & REQ_F_BUFFER_SELECT) {
2383 ret = io_iov_buffer_select(req, *iovec, needs_lock);
2384 if (!ret) {
2385 ret = (*iovec)->iov_len;
2386 iov_iter_init(iter, rw, *iovec, 1, ret);
2387 }
2388 *iovec = NULL;
2389 return ret;
2390 }
2391
2089#ifdef CONFIG_COMPAT2392#ifdef CONFIG_COMPAT
2090 if (req->ctx->compat)2393 if (req->ctx->compat)
2091 return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,2394 return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
@@ -2169,12 +2472,18 @@ static void io_req_map_rw(struct io_kiocb *req, ssize_t io_size, · 77 unmodified lines
2169 }2472 }
2170}2473}
2171 2474
2475static inline int __io_alloc_async_ctx(struct io_kiocb *req)
2476{
2477 req->io = kmalloc(sizeof(*req->io), GFP_KERNEL);
2478 return req->io == NULL;
2479}
2480
2172static int io_alloc_async_ctx(struct io_kiocb *req)2481static int io_alloc_async_ctx(struct io_kiocb *req)
2173{2482{
2174 if (!io_op_defs[req->opcode].async_ctx)2483 if (!io_op_defs[req->opcode].async_ctx)
2175 return 0;2484 return 0;
2176 req->io = kmalloc(sizeof(*req->io), GFP_KERNEL);2485
2177 return req->io == NULL;2486 return __io_alloc_async_ctx(req);
2178}2487}
2179 2488
2180static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size,2489static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size,
@@ -2184,7 +2493,7 @@ static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size, · 3 unmodified lines
2184 if (!io_op_defs[req->opcode].async_ctx)2493 if (!io_op_defs[req->opcode].async_ctx)
2185 return 0;2494 return 0;
2186 if (!req->io) {2495 if (!req->io) {
2187 if (io_alloc_async_ctx(req))2496 if (__io_alloc_async_ctx(req))
2188 return -ENOMEM;2497 return -ENOMEM;
2189 2498
2190 io_req_map_rw(req, io_size, iovec, fast_iov, iter);2499 io_req_map_rw(req, io_size, iovec, fast_iov, iter);
@@ -2213,7 +2522,7 @@ static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 22 unmodified lines
2213 io = req->io;2522 io = req->io;
2214 io->rw.iov = io->rw.fast_iov;2523 io->rw.iov = io->rw.fast_iov;
2215 req->io = NULL;2524 req->io = NULL;
2216 ret = io_import_iovec(READ, req, &io->rw.iov, &iter);2525 ret = io_import_iovec(READ, req, &io->rw.iov, &iter, !force_nonblock);
2217 req->io = io;2526 req->io = io;
2218 if (ret < 0)2527 if (ret < 0)
2219 return ret;2528 return ret;
@@ -2222,8 +2531,7 @@ static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 2 unmodified lines
2222 return 0;2531 return 0;
2223}2532}
2224 2533
2225static int io_read(struct io_kiocb *req, struct io_kiocb **nxt,2534static int io_read(struct io_kiocb *req, bool force_nonblock)
2226 bool force_nonblock)
2227{2535{
2228 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;2536 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
2229 struct kiocb *kiocb = &req->rw.kiocb;2537 struct kiocb *kiocb = &req->rw.kiocb;
@@ -2231,13 +2539,13 @@ static int io_read(struct io_kiocb *req, struct io_kiocb **nxt, · 1 unmodified line
2231 size_t iov_count;2539 size_t iov_count;
2232 ssize_t io_size, ret;2540 ssize_t io_size, ret;
2233 2541
2234 ret = io_import_iovec(READ, req, &iovec, &iter);2542 ret = io_import_iovec(READ, req, &iovec, &iter, !force_nonblock);
2235 if (ret < 0)2543 if (ret < 0)
2236 return ret;2544 return ret;
2237 2545
2238 /* Ensure we clear previously set non-block flag */2546 /* Ensure we clear previously set non-block flag */
2239 if (!force_nonblock)2547 if (!force_nonblock)
2240 req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;2548 kiocb->ki_flags &= ~IOCB_NOWAIT;
2241 2549
2242 req->result = 0;2550 req->result = 0;
2243 io_size = ret;2551 io_size = ret;
@@ -2248,10 +2556,8 @@ static int io_read(struct io_kiocb *req, struct io_kiocb **nxt, · 4 unmodified lines
2248 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so2556 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
2249 * we know to async punt it even if it was opened O_NONBLOCK2557 * we know to async punt it even if it was opened O_NONBLOCK
2250 */2558 */
2251 if (force_nonblock && !io_file_supports_async(req->file)) {2559 if (force_nonblock && !io_file_supports_async(req->file))
2252 req->flags |= REQ_F_MUST_PUNT;
2253 goto copy_iov;2560 goto copy_iov;
2254 }
2255 2561
2256 iov_count = iov_iter_count(&iter);2562 iov_count = iov_iter_count(&iter);
2257 ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count);2563 ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count);
@@ -2265,13 +2571,16 @@ static int io_read(struct io_kiocb *req, struct io_kiocb **nxt, · 7 unmodified lines
2265 2571
2266 /* Catch -EAGAIN return for forced non-blocking submission */2572 /* Catch -EAGAIN return for forced non-blocking submission */
2267 if (!force_nonblock || ret2 != -EAGAIN) {2573 if (!force_nonblock || ret2 != -EAGAIN) {
2268 kiocb_done(kiocb, ret2, nxt, req->in_async);2574 kiocb_done(kiocb, ret2);
2269 } else {2575 } else {
2270copy_iov:2576copy_iov:
2271 ret = io_setup_async_rw(req, io_size, iovec,2577 ret = io_setup_async_rw(req, io_size, iovec,
2272 inline_vecs, &iter);2578 inline_vecs, &iter);
2273 if (ret)2579 if (ret)
2274 goto out_free;2580 goto out_free;
2581 /* any defer here is final, must blocking retry */
2582 if (!(req->flags & REQ_F_NOWAIT))
2583 req->flags |= REQ_F_MUST_PUNT;
2275 return -EAGAIN;2584 return -EAGAIN;
2276 }2585 }
2277 }2586 }
@@ -2295,6 +2604,8 @@ static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 17 unmodified lines
2295 if (unlikely(!(req->file->f_mode & FMODE_WRITE)))2604 if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
2296 return -EBADF;2605 return -EBADF;
2297 2606
2607 req->fsize = rlimit(RLIMIT_FSIZE);
2608
2298 /* either don't need iovec imported or already have it */2609 /* either don't need iovec imported or already have it */
2299 if (!req->io || req->flags & REQ_F_NEED_CLEANUP)2610 if (!req->io || req->flags & REQ_F_NEED_CLEANUP)
2300 return 0;2611 return 0;
@@ -2302,7 +2613,7 @@ static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 1 unmodified line
2302 io = req->io;2613 io = req->io;
2303 io->rw.iov = io->rw.fast_iov;2614 io->rw.iov = io->rw.fast_iov;
2304 req->io = NULL;2615 req->io = NULL;
2305 ret = io_import_iovec(WRITE, req, &io->rw.iov, &iter);2616 ret = io_import_iovec(WRITE, req, &io->rw.iov, &iter, !force_nonblock);
2306 req->io = io;2617 req->io = io;
2307 if (ret < 0)2618 if (ret < 0)
2308 return ret;2619 return ret;
@@ -2311,8 +2622,7 @@ static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 2 unmodified lines
2311 return 0;2622 return 0;
2312}2623}
2313 2624
2314static int io_write(struct io_kiocb *req, struct io_kiocb **nxt,2625static int io_write(struct io_kiocb *req, bool force_nonblock)
2315 bool force_nonblock)
2316{2626{
2317 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;2627 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
2318 struct kiocb *kiocb = &req->rw.kiocb;2628 struct kiocb *kiocb = &req->rw.kiocb;
@@ -2320,7 +2630,7 @@ static int io_write(struct io_kiocb *req, struct io_kiocb **nxt, · 1 unmodified line
2320 size_t iov_count;2630 size_t iov_count;
2321 ssize_t ret, io_size;2631 ssize_t ret, io_size;
2322 2632
2323 ret = io_import_iovec(WRITE, req, &iovec, &iter);2633 ret = io_import_iovec(WRITE, req, &iovec, &iter, !force_nonblock);
2324 if (ret < 0)2634 if (ret < 0)
2325 return ret;2635 return ret;
2326 2636
@@ -2337,10 +2647,8 @@ static int io_write(struct io_kiocb *req, struct io_kiocb **nxt, · 10 unmodified lines
2337 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so2647 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
2338 * we know to async punt it even if it was opened O_NONBLOCK2648 * we know to async punt it even if it was opened O_NONBLOCK
2339 */2649 */
2340 if (force_nonblock && !io_file_supports_async(req->file)) {2650 if (force_nonblock && !io_file_supports_async(req->file))
2341 req->flags |= REQ_F_MUST_PUNT;
2342 goto copy_iov;2651 goto copy_iov;
2343 }
2344 2652
2345 /* file path doesn't support NOWAIT for non-direct_IO */2653 /* file path doesn't support NOWAIT for non-direct_IO */
2346 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&2654 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
@@ -2367,24 +2675,33 @@ static int io_write(struct io_kiocb *req, struct io_kiocb **nxt, · 20 unmodified lines
2367 }2675 }
2368 kiocb->ki_flags |= IOCB_WRITE;2676 kiocb->ki_flags |= IOCB_WRITE;
2369 2677
2678 if (!force_nonblock)
2679 current->signal->rlim[RLIMIT_FSIZE].rlim_cur = req->fsize;
2680
2370 if (req->file->f_op->write_iter)2681 if (req->file->f_op->write_iter)
2371 ret2 = call_write_iter(req->file, kiocb, &iter);2682 ret2 = call_write_iter(req->file, kiocb, &iter);
2372 else2683 else
2373 ret2 = loop_rw_iter(WRITE, req->file, kiocb, &iter);2684 ret2 = loop_rw_iter(WRITE, req->file, kiocb, &iter);
2685
2686 if (!force_nonblock)
2687 current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY;
2688
2374 /*2689 /*
2375 * Raw bdev writes will -EOPNOTSUPP for IOCB_NOWAIT. Just2690 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
2376 * retry them without IOCB_NOWAIT.2691 * retry them without IOCB_NOWAIT.
2377 */2692 */
2378 if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))2693 if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
2379 ret2 = -EAGAIN;2694 ret2 = -EAGAIN;
2380 if (!force_nonblock || ret2 != -EAGAIN) {2695 if (!force_nonblock || ret2 != -EAGAIN) {
2381 kiocb_done(kiocb, ret2, nxt, req->in_async);2696 kiocb_done(kiocb, ret2);
2382 } else {2697 } else {
2383copy_iov:2698copy_iov:
2384 ret = io_setup_async_rw(req, io_size, iovec,2699 ret = io_setup_async_rw(req, io_size, iovec,
2385 inline_vecs, &iter);2700 inline_vecs, &iter);
2386 if (ret)2701 if (ret)
2387 goto out_free;2702 goto out_free;
2703 /* any defer here is final, must blocking retry */
2704 req->flags |= REQ_F_MUST_PUNT;
2388 return -EAGAIN;2705 return -EAGAIN;
2389 }2706 }
2390 }2707 }
@@ -2394,6 +2711,76 @@ out_free: · 3 unmodified lines
2394 return ret;2711 return ret;
2395}2712}
2396 2713
2714static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2715{
2716 struct io_splice* sp = &req->splice;
2717 unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
2718 int ret;
2719
2720 if (req->flags & REQ_F_NEED_CLEANUP)
2721 return 0;
2722
2723 sp->file_in = NULL;
2724 sp->off_in = READ_ONCE(sqe->splice_off_in);
2725 sp->off_out = READ_ONCE(sqe->off);
2726 sp->len = READ_ONCE(sqe->len);
2727 sp->flags = READ_ONCE(sqe->splice_flags);
2728
2729 if (unlikely(sp->flags & ~valid_flags))
2730 return -EINVAL;
2731
2732 ret = io_file_get(NULL, req, READ_ONCE(sqe->splice_fd_in), &sp->file_in,
2733 (sp->flags & SPLICE_F_FD_IN_FIXED));
2734 if (ret)
2735 return ret;
2736 req->flags |= REQ_F_NEED_CLEANUP;
2737
2738 if (!S_ISREG(file_inode(sp->file_in)->i_mode))
2739 req->work.flags |= IO_WQ_WORK_UNBOUND;
2740
2741 return 0;
2742}
2743
2744static bool io_splice_punt(struct file *file)
2745{
2746 if (get_pipe_info(file))
2747 return false;
2748 if (!io_file_supports_async(file))
2749 return true;
2750 return !(file->f_mode & O_NONBLOCK);
2751}
2752
2753static int io_splice(struct io_kiocb *req, bool force_nonblock)
2754{
2755 struct io_splice *sp = &req->splice;
2756 struct file *in = sp->file_in;
2757 struct file *out = sp->file_out;
2758 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
2759 loff_t *poff_in, *poff_out;
2760 long ret;
2761
2762 if (force_nonblock) {
2763 if (io_splice_punt(in) || io_splice_punt(out))
2764 return -EAGAIN;
2765 flags |= SPLICE_F_NONBLOCK;
2766 }
2767
2768 poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
2769 poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
2770 ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
2771 if (force_nonblock && ret == -EAGAIN)
2772 return -EAGAIN;
2773
2774 io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
2775 req->flags &= ~REQ_F_NEED_CLEANUP;
2776
2777 io_cqring_add_event(req, ret);
2778 if (ret != sp->len)
2779 req_set_fail_links(req);
2780 io_put_req(req);
2781 return 0;
2782}
2783
2397/*2784/*
2398 * IORING_OP_NOP just posts a completion event, nothing else.2785 * IORING_OP_NOP just posts a completion event, nothing else.
2399 */2786 */
@@ -2442,85 +2829,63 @@ static bool io_req_cancelled(struct io_kiocb *req) · 42 unmodified lines
2442 return false;2829 return false;
2443}2830}
2444 2831
2445static void io_link_work_cb(struct io_wq_work **workptr)2832static void __io_fsync(struct io_kiocb *req)
2446{
2447 struct io_wq_work *work = *workptr;
2448 struct io_kiocb *link = work->data;
2449
2450 io_queue_linked_timeout(link);
2451 work->func = io_wq_submit_work;
2452}
2453
2454static void io_wq_assign_next(struct io_wq_work **workptr, struct io_kiocb *nxt)
2455{
2456 struct io_kiocb *link;
2457
2458 io_prep_async_work(nxt, &link);
2459 *workptr = &nxt->work;
2460 if (link) {
2461 nxt->work.flags |= IO_WQ_WORK_CB;
2462 nxt->work.func = io_link_work_cb;
2463 nxt->work.data = link;
2464 }
2465}
2466
2467static void io_fsync_finish(struct io_wq_work **workptr)
2468{2833{
2469 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2470 loff_t end = req->sync.off + req->sync.len;2834 loff_t end = req->sync.off + req->sync.len;
2471 struct io_kiocb *nxt = NULL;
2472 int ret;2835 int ret;
2473 2836
2474 if (io_req_cancelled(req))
2475 return;
2476
2477 ret = vfs_fsync_range(req->file, req->sync.off,2837 ret = vfs_fsync_range(req->file, req->sync.off,
2478 end > 0 ? end : LLONG_MAX,2838 end > 0 ? end : LLONG_MAX,
2479 req->sync.flags & IORING_FSYNC_DATASYNC);2839 req->sync.flags & IORING_FSYNC_DATASYNC);
2480 if (ret < 0)2840 if (ret < 0)
2481 req_set_fail_links(req);2841 req_set_fail_links(req);
2482 io_cqring_add_event(req, ret);2842 io_cqring_add_event(req, ret);
2483 io_put_req_find_next(req, &nxt);2843 io_put_req(req);
2484 if (nxt)
2485 io_wq_assign_next(workptr, nxt);
2486}2844}
2487 2845
2488static int io_fsync(struct io_kiocb *req, struct io_kiocb **nxt,2846static void io_fsync_finish(struct io_wq_work **workptr)
2489 bool force_nonblock)
2490{2847{
2491 struct io_wq_work *work, *old_work;2848 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2849
2850 if (io_req_cancelled(req))
2851 return;
2852 __io_fsync(req);
2853 io_steal_work(req, workptr);
2854}
2492 2855
2856static int io_fsync(struct io_kiocb *req, bool force_nonblock)
2857{
2493 /* fsync always requires a blocking context */2858 /* fsync always requires a blocking context */
2494 if (force_nonblock) {2859 if (force_nonblock) {
2495 io_put_req(req);
2496 req->work.func = io_fsync_finish;2860 req->work.func = io_fsync_finish;
2497 return -EAGAIN;2861 return -EAGAIN;
2498 }2862 }
2499 2863 __io_fsync(req);
2500 work = old_work = &req->work;
2501 io_fsync_finish(&work);
2502 if (work && work != old_work)
2503 *nxt = container_of(work, struct io_kiocb, work);
2504 return 0;2864 return 0;
2505}2865}
2506 2866
2507static void io_fallocate_finish(struct io_wq_work **workptr)2867static void __io_fallocate(struct io_kiocb *req)
2508{2868{
2509 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2510 struct io_kiocb *nxt = NULL;
2511 int ret;2869 int ret;
2512 2870
2513 if (io_req_cancelled(req))2871 current->signal->rlim[RLIMIT_FSIZE].rlim_cur = req->fsize;
2514 return;
2515
2516 ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,2872 ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
2517 req->sync.len);2873 req->sync.len);
2874 current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY;
2518 if (ret < 0)2875 if (ret < 0)
2519 req_set_fail_links(req);2876 req_set_fail_links(req);
2520 io_cqring_add_event(req, ret);2877 io_cqring_add_event(req, ret);
2521 io_put_req_find_next(req, &nxt);2878 io_put_req(req);
2522 if (nxt)2879}
2523 io_wq_assign_next(workptr, nxt);2880
2881static void io_fallocate_finish(struct io_wq_work **workptr)
2882{
2883 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2884
2885 if (io_req_cancelled(req))
2886 return;
2887 __io_fallocate(req);
2888 io_steal_work(req, workptr);
2524}2889}
2525 2890
2526static int io_fallocate_prep(struct io_kiocb *req,2891static int io_fallocate_prep(struct io_kiocb *req,
@@ -2532,26 +2897,19 @@ static int io_fallocate_prep(struct io_kiocb *req, · 5 unmodified lines
2532 req->sync.off = READ_ONCE(sqe->off);2897 req->sync.off = READ_ONCE(sqe->off);
2533 req->sync.len = READ_ONCE(sqe->addr);2898 req->sync.len = READ_ONCE(sqe->addr);
2534 req->sync.mode = READ_ONCE(sqe->len);2899 req->sync.mode = READ_ONCE(sqe->len);
2900 req->fsize = rlimit(RLIMIT_FSIZE);
2535 return 0;2901 return 0;
2536}2902}
2537 2903
2538static int io_fallocate(struct io_kiocb *req, struct io_kiocb **nxt,2904static int io_fallocate(struct io_kiocb *req, bool force_nonblock)
2539 bool force_nonblock)
2540{2905{
2541 struct io_wq_work *work, *old_work;
2542
2543 /* fallocate always requiring blocking context */2906 /* fallocate always requiring blocking context */
2544 if (force_nonblock) {2907 if (force_nonblock) {
2545 io_put_req(req);
2546 req->work.func = io_fallocate_finish;2908 req->work.func = io_fallocate_finish;
2547 return -EAGAIN;2909 return -EAGAIN;
2548 }2910 }
2549 2911
2550 work = old_work = &req->work;2912 __io_fallocate(req);
2551 io_fallocate_finish(&work);
2552 if (work && work != old_work)
2553 *nxt = container_of(work, struct io_kiocb, work);
2554
2555 return 0;2913 return 0;
2556}2914}
2557 2915
@@ -2626,8 +2984,7 @@ static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 68 unmodified lines
2626 return 0;2984 return 0;
2627}2985}
2628 2986
2629static int io_openat2(struct io_kiocb *req, struct io_kiocb **nxt,2987static int io_openat2(struct io_kiocb *req, bool force_nonblock)
2630 bool force_nonblock)
2631{2988{
2632 struct open_flags op;2989 struct open_flags op;
2633 struct file *file;2990 struct file *file;
@@ -2658,15 +3015,171 @@ err: · 24 unmodified lines
2658 if (ret < 0)3015 if (ret < 0)
2659 req_set_fail_links(req);3016 req_set_fail_links(req);
2660 io_cqring_add_event(req, ret);3017 io_cqring_add_event(req, ret);
2661 io_put_req_find_next(req, nxt);3018 io_put_req(req);
2662 return 0;3019 return 0;
2663}3020}
2664 3021
2665static int io_openat(struct io_kiocb *req, struct io_kiocb **nxt,3022static int io_openat(struct io_kiocb *req, bool force_nonblock)
2666 bool force_nonblock)
2667{3023{
2668 req->open.how = build_open_how(req->open.how.flags, req->open.how.mode);3024 req->open.how = build_open_how(req->open.how.flags, req->open.how.mode);
2669 return io_openat2(req, nxt, force_nonblock);3025 return io_openat2(req, force_nonblock);
3026}
3027
3028static int io_remove_buffers_prep(struct io_kiocb *req,
3029 const struct io_uring_sqe *sqe)
3030{
3031 struct io_provide_buf *p = &req->pbuf;
3032 u64 tmp;
3033
3034 if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off)
3035 return -EINVAL;
3036
3037 tmp = READ_ONCE(sqe->fd);
3038 if (!tmp || tmp > USHRT_MAX)
3039 return -EINVAL;
3040
3041 memset(p, 0, sizeof(*p));
3042 p->nbufs = tmp;
3043 p->bgid = READ_ONCE(sqe->buf_group);
3044 return 0;
3045}
3046
3047static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
3048 int bgid, unsigned nbufs)
3049{
3050 unsigned i = 0;
3051
3052 /* shouldn't happen */
3053 if (!nbufs)
3054 return 0;
3055
3056 /* the head kbuf is the list itself */
3057 while (!list_empty(&buf->list)) {
3058 struct io_buffer *nxt;
3059
3060 nxt = list_first_entry(&buf->list, struct io_buffer, list);
3061 list_del(&nxt->list);
3062 kfree(nxt);
3063 if (++i == nbufs)
3064 return i;
3065 }
3066 i++;
3067 kfree(buf);
3068 idr_remove(&ctx->io_buffer_idr, bgid);
3069
3070 return i;
3071}
3072
3073static int io_remove_buffers(struct io_kiocb *req, bool force_nonblock)
3074{
3075 struct io_provide_buf *p = &req->pbuf;
3076 struct io_ring_ctx *ctx = req->ctx;
3077 struct io_buffer *head;
3078 int ret = 0;
3079
3080 io_ring_submit_lock(ctx, !force_nonblock);
3081
3082 lockdep_assert_held(&ctx->uring_lock);
3083
3084 ret = -ENOENT;
3085 head = idr_find(&ctx->io_buffer_idr, p->bgid);
3086 if (head)
3087 ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);
3088
3089 io_ring_submit_lock(ctx, !force_nonblock);
3090 if (ret < 0)
3091 req_set_fail_links(req);
3092 io_cqring_add_event(req, ret);
3093 io_put_req(req);
3094 return 0;
3095}
3096
3097static int io_provide_buffers_prep(struct io_kiocb *req,
3098 const struct io_uring_sqe *sqe)
3099{
3100 struct io_provide_buf *p = &req->pbuf;
3101 u64 tmp;
3102
3103 if (sqe->ioprio || sqe->rw_flags)
3104 return -EINVAL;
3105
3106 tmp = READ_ONCE(sqe->fd);
3107 if (!tmp || tmp > USHRT_MAX)
3108 return -E2BIG;
3109 p->nbufs = tmp;
3110 p->addr = READ_ONCE(sqe->addr);
3111 p->len = READ_ONCE(sqe->len);
3112
3113 if (!access_ok(u64_to_user_ptr(p->addr), p->len))
3114 return -EFAULT;
3115
3116 p->bgid = READ_ONCE(sqe->buf_group);
3117 tmp = READ_ONCE(sqe->off);
3118 if (tmp > USHRT_MAX)
3119 return -E2BIG;
3120 p->bid = tmp;
3121 return 0;
3122}
3123
3124static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)
3125{
3126 struct io_buffer *buf;
3127 u64 addr = pbuf->addr;
3128 int i, bid = pbuf->bid;
3129
3130 for (i = 0; i < pbuf->nbufs; i++) {
3131 buf = kmalloc(sizeof(*buf), GFP_KERNEL);
3132 if (!buf)
3133 break;
3134
3135 buf->addr = addr;
3136 buf->len = pbuf->len;
3137 buf->bid = bid;
3138 addr += pbuf->len;
3139 bid++;
3140 if (!*head) {
3141 INIT_LIST_HEAD(&buf->list);
3142 *head = buf;
3143 } else {
3144 list_add_tail(&buf->list, &(*head)->list);
3145 }
3146 }
3147
3148 return i ? i : -ENOMEM;
3149}
3150
3151static int io_provide_buffers(struct io_kiocb *req, bool force_nonblock)
3152{
3153 struct io_provide_buf *p = &req->pbuf;
3154 struct io_ring_ctx *ctx = req->ctx;
3155 struct io_buffer *head, *list;
3156 int ret = 0;
3157
3158 io_ring_submit_lock(ctx, !force_nonblock);
3159
3160 lockdep_assert_held(&ctx->uring_lock);
3161
3162 list = head = idr_find(&ctx->io_buffer_idr, p->bgid);
3163
3164 ret = io_add_buffers(p, &head);
3165 if (ret < 0)
3166 goto out;
3167
3168 if (!list) {
3169 ret = idr_alloc(&ctx->io_buffer_idr, head, p->bgid, p->bgid + 1,
3170 GFP_KERNEL);
3171 if (ret < 0) {
3172 __io_remove_buffers(ctx, head, p->bgid, -1U);
3173 goto out;
3174 }
3175 }
3176out:
3177 io_ring_submit_unlock(ctx, !force_nonblock);
3178 if (ret < 0)
3179 req_set_fail_links(req);
3180 io_cqring_add_event(req, ret);
3181 io_put_req(req);
3182 return 0;
2670}3183}
2671 3184
2672static int io_epoll_ctl_prep(struct io_kiocb *req,3185static int io_epoll_ctl_prep(struct io_kiocb *req,
@@ -2694,8 +3207,7 @@ static int io_epoll_ctl_prep(struct io_kiocb *req, · 21 unmodified lines
2694#endif3207#endif
2695}3208}
2696 3209
2697static int io_epoll_ctl(struct io_kiocb *req, struct io_kiocb **nxt,3210static int io_epoll_ctl(struct io_kiocb *req, bool force_nonblock)
2698 bool force_nonblock)
2699{3211{
2700#if defined(CONFIG_EPOLL)3212#if defined(CONFIG_EPOLL)
2701 struct io_epoll *ie = &req->epoll;3213 struct io_epoll *ie = &req->epoll;
@@ -2708,7 +3220,7 @@ static int io_epoll_ctl(struct io_kiocb *req, struct io_kiocb **nxt, · 6 unmodified lines
2708 if (ret < 0)3220 if (ret < 0)
2709 req_set_fail_links(req);3221 req_set_fail_links(req);
2710 io_cqring_add_event(req, ret);3222 io_cqring_add_event(req, ret);
2711 io_put_req_find_next(req, nxt);3223 io_put_req(req);
2712 return 0;3224 return 0;
2713#else3225#else
2714 return -EOPNOTSUPP;3226 return -EOPNOTSUPP;
@@ -2730,8 +3242,7 @@ static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 15 unmodified lines
2730#endif3242#endif
2731}3243}
2732 3244
2733static int io_madvise(struct io_kiocb *req, struct io_kiocb **nxt,3245static int io_madvise(struct io_kiocb *req, bool force_nonblock)
2734 bool force_nonblock)
2735{3246{
2736#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)3247#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
2737 struct io_madvise *ma = &req->madvise;3248 struct io_madvise *ma = &req->madvise;
@@ -2744,7 +3255,7 @@ static int io_madvise(struct io_kiocb *req, struct io_kiocb **nxt, · 6 unmodified lines
2744 if (ret < 0)3255 if (ret < 0)
2745 req_set_fail_links(req);3256 req_set_fail_links(req);
2746 io_cqring_add_event(req, ret);3257 io_cqring_add_event(req, ret);
2747 io_put_req_find_next(req, nxt);3258 io_put_req(req);
2748 return 0;3259 return 0;
2749#else3260#else
2750 return -EOPNOTSUPP;3261 return -EOPNOTSUPP;
@@ -2762,8 +3273,7 @@ static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 11 unmodified lines
2762 return 0;3273 return 0;
2763}3274}
2764 3275
2765static int io_fadvise(struct io_kiocb *req, struct io_kiocb **nxt,3276static int io_fadvise(struct io_kiocb *req, bool force_nonblock)
2766 bool force_nonblock)
2767{3277{
2768 struct io_fadvise *fa = &req->fadvise;3278 struct io_fadvise *fa = &req->fadvise;
2769 int ret;3279 int ret;
@@ -2783,7 +3293,7 @@ static int io_fadvise(struct io_kiocb *req, struct io_kiocb **nxt, · 13 unmodified lines
2783 if (ret < 0)3293 if (ret < 0)
2784 req_set_fail_links(req);3294 req_set_fail_links(req);
2785 io_cqring_add_event(req, ret);3295 io_cqring_add_event(req, ret);
2786 io_put_req_find_next(req, nxt);3296 io_put_req(req);
2787 return 0;3297 return 0;
2788}3298}
2789 3299
@@ -2820,8 +3330,7 @@ static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 30 unmodified lines
2820 return 0;3330 return 0;
2821}3331}
2822 3332
2823static int io_statx(struct io_kiocb *req, struct io_kiocb **nxt,3333static int io_statx(struct io_kiocb *req, bool force_nonblock)
2824 bool force_nonblock)
2825{3334{
2826 struct io_open *ctx = &req->open;3335 struct io_open *ctx = &req->open;
2827 unsigned lookup_flags;3336 unsigned lookup_flags;
@@ -2858,7 +3367,7 @@ err: · 30 unmodified lines
2858 if (ret < 0)3367 if (ret < 0)
2859 req_set_fail_links(req);3368 req_set_fail_links(req);
2860 io_cqring_add_event(req, ret);3369 io_cqring_add_event(req, ret);
2861 io_put_req_find_next(req, nxt);3370 io_put_req(req);
2862 return 0;3371 return 0;
2863}3372}
2864 3373
@@ -2885,7 +3394,7 @@ static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 20 unmodified lines
2885}3394}
2886 3395
2887/* only called when __close_fd_get_file() is done */3396/* only called when __close_fd_get_file() is done */
2888static void __io_close_finish(struct io_kiocb *req, struct io_kiocb **nxt)3397static void __io_close_finish(struct io_kiocb *req)
2889{3398{
2890 int ret;3399 int ret;
2891 3400
@@ -2894,22 +3403,19 @@ static void __io_close_finish(struct io_kiocb *req, struct io_kiocb **nxt) · 2 unmodified lines
2894 req_set_fail_links(req);3403 req_set_fail_links(req);
2895 io_cqring_add_event(req, ret);3404 io_cqring_add_event(req, ret);
2896 fput(req->close.put_file);3405 fput(req->close.put_file);
2897 io_put_req_find_next(req, nxt);3406 io_put_req(req);
2898}3407}
2899 3408
2900static void io_close_finish(struct io_wq_work **workptr)3409static void io_close_finish(struct io_wq_work **workptr)
2901{3410{
2902 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);3411 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2903 struct io_kiocb *nxt = NULL;
2904 3412
2905 /* not cancellable, don't do io_req_cancelled() */3413 /* not cancellable, don't do io_req_cancelled() */
2906 __io_close_finish(req, &nxt);3414 __io_close_finish(req);
2907 if (nxt)3415 io_steal_work(req, workptr);
2908 io_wq_assign_next(workptr, nxt);
2909}3416}
2910 3417
2911static int io_close(struct io_kiocb *req, struct io_kiocb **nxt,3418static int io_close(struct io_kiocb *req, bool force_nonblock)
2912 bool force_nonblock)
2913{3419{
2914 int ret;3420 int ret;
2915 3421
@@ -2919,23 +3425,25 @@ static int io_close(struct io_kiocb *req, struct io_kiocb **nxt, · 3 unmodified lines
2919 return ret;3425 return ret;
2920 3426
2921 /* if the file has a flush method, be safe and punt to async */3427 /* if the file has a flush method, be safe and punt to async */
2922 if (req->close.put_file->f_op->flush && !io_wq_current_is_worker())3428 if (req->close.put_file->f_op->flush && force_nonblock) {
2923 goto eagain;3429 /* submission ref will be dropped, take it for async */
3430 refcount_inc(&req->refs);
3431
3432 req->work.func = io_close_finish;
3433 /*
3434 * Do manual async queue here to avoid grabbing files - we don't
3435 * need the files, and it'll cause io_close_finish() to close
3436 * the file again and cause a double CQE entry for this request
3437 */
3438 io_queue_async_work(req);
3439 return 0;
3440 }
2924 3441
2925 /*3442 /*
2926 * No ->flush(), safely close from here and just punt the3443 * No ->flush(), safely close from here and just punt the
2927 * fput() to async context.3444 * fput() to async context.
2928 */3445 */
2929 __io_close_finish(req, nxt);3446 __io_close_finish(req);
2930 return 0;
2931eagain:
2932 req->work.func = io_close_finish;
2933 /*
2934 * Do manual async queue here to avoid grabbing files - we don't
2935 * need the files, and it'll cause io_close_finish() to close
2936 * the file again and cause a double CQE entry for this request
2937 */
2938 io_queue_async_work(req);
2939 return 0;3447 return 0;
2940}3448}
2941 3449
@@ -2957,47 +3465,62 @@ static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 15 unmodified lines
2957 return 0;3465 return 0;
2958}3466}
2959 3467
2960static void io_sync_file_range_finish(struct io_wq_work **workptr)3468static void __io_sync_file_range(struct io_kiocb *req)
2961{3469{
2962 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2963 struct io_kiocb *nxt = NULL;
2964 int ret;3470 int ret;
2965 3471
2966 if (io_req_cancelled(req))
2967 return;
2968
2969 ret = sync_file_range(req->file, req->sync.off, req->sync.len,3472 ret = sync_file_range(req->file, req->sync.off, req->sync.len,
2970 req->sync.flags);3473 req->sync.flags);
2971 if (ret < 0)3474 if (ret < 0)
2972 req_set_fail_links(req);3475 req_set_fail_links(req);
2973 io_cqring_add_event(req, ret);3476 io_cqring_add_event(req, ret);
2974 io_put_req_find_next(req, &nxt);3477 io_put_req(req);
3478}
3479
3480
3481static void io_sync_file_range_finish(struct io_wq_work **workptr)
3482{
3483 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
3484 struct io_kiocb *nxt = NULL;
3485
3486 if (io_req_cancelled(req))
3487 return;
3488 __io_sync_file_range(req);
3489 io_put_req(req); /* put submission ref */
2975 if (nxt)3490 if (nxt)
2976 io_wq_assign_next(workptr, nxt);3491 io_wq_assign_next(workptr, nxt);
2977}3492}
2978 3493
2979static int io_sync_file_range(struct io_kiocb *req, struct io_kiocb **nxt,3494static int io_sync_file_range(struct io_kiocb *req, bool force_nonblock)
2980 bool force_nonblock)
2981{3495{
2982 struct io_wq_work *work, *old_work;
2983
2984 /* sync_file_range always requires a blocking context */3496 /* sync_file_range always requires a blocking context */
2985 if (force_nonblock) {3497 if (force_nonblock) {
2986 io_put_req(req);
2987 req->work.func = io_sync_file_range_finish;3498 req->work.func = io_sync_file_range_finish;
2988 return -EAGAIN;3499 return -EAGAIN;
2989 }3500 }
2990 3501
2991 work = old_work = &req->work;3502 __io_sync_file_range(req);
2992 io_sync_file_range_finish(&work);
2993 if (work && work != old_work)
2994 *nxt = container_of(work, struct io_kiocb, work);
2995 return 0;3503 return 0;
2996}3504}
2997 3505
3506#if defined(CONFIG_NET)
3507static int io_setup_async_msg(struct io_kiocb *req,
3508 struct io_async_msghdr *kmsg)
3509{
3510 if (req->io)
3511 return -EAGAIN;
3512 if (io_alloc_async_ctx(req)) {
3513 if (kmsg->iov != kmsg->fast_iov)
3514 kfree(kmsg->iov);
3515 return -ENOMEM;
3516 }
3517 req->flags |= REQ_F_NEED_CLEANUP;
3518 memcpy(&req->io->msg, kmsg, sizeof(*kmsg));
3519 return -EAGAIN;
3520}
3521
2998static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)3522static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2999{3523{
3000#if defined(CONFIG_NET)
3001 struct io_sr_msg *sr = &req->sr_msg;3524 struct io_sr_msg *sr = &req->sr_msg;
3002 struct io_async_ctx *io = req->io;3525 struct io_async_ctx *io = req->io;
3003 int ret;3526 int ret;
@@ -3023,15 +3546,10 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 19 unmodified lines
3023 if (!ret)3546 if (!ret)
3024 req->flags |= REQ_F_NEED_CLEANUP;3547 req->flags |= REQ_F_NEED_CLEANUP;
3025 return ret;3548 return ret;
3026#else
3027 return -EOPNOTSUPP;
3028#endif
3029}3549}
3030 3550
3031static int io_sendmsg(struct io_kiocb *req, struct io_kiocb **nxt,3551static int io_sendmsg(struct io_kiocb *req, bool force_nonblock)
3032 bool force_nonblock)
3033{3552{
3034#if defined(CONFIG_NET)
3035 struct io_async_msghdr *kmsg = NULL;3553 struct io_async_msghdr *kmsg = NULL;
3036 struct socket *sock;3554 struct socket *sock;
3037 int ret;3555 int ret;
@@ -3071,18 +3589,8 @@ static int io_sendmsg(struct io_kiocb *req, struct io_kiocb **nxt, · 33 unmodified lines
3071 flags |= MSG_DONTWAIT;3589 flags |= MSG_DONTWAIT;
3072 3590
3073 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);3591 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
3074 if (force_nonblock && ret == -EAGAIN) {3592 if (force_nonblock && ret == -EAGAIN)
3075 if (req->io)3593 return io_setup_async_msg(req, kmsg);
3076 return -EAGAIN;
3077 if (io_alloc_async_ctx(req)) {
3078 if (kmsg->iov != kmsg->fast_iov)
3079 kfree(kmsg->iov);
3080 return -ENOMEM;
3081 }
3082 req->flags |= REQ_F_NEED_CLEANUP;
3083 memcpy(&req->io->msg, &io.msg, sizeof(io.msg));
3084 return -EAGAIN;
3085 }
3086 if (ret == -ERESTARTSYS)3594 if (ret == -ERESTARTSYS)
3087 ret = -EINTR;3595 ret = -EINTR;
3088 }3596 }
@@ -3093,17 +3601,12 @@ static int io_sendmsg(struct io_kiocb *req, struct io_kiocb **nxt, · 4 unmodified lines
3093 io_cqring_add_event(req, ret);3601 io_cqring_add_event(req, ret);
3094 if (ret < 0)3602 if (ret < 0)
3095 req_set_fail_links(req);3603 req_set_fail_links(req);
3096 io_put_req_find_next(req, nxt);3604 io_put_req(req);
3097 return 0;3605 return 0;
3098#else
3099 return -EOPNOTSUPP;
3100#endif
3101}3606}
3102 3607
3103static int io_send(struct io_kiocb *req, struct io_kiocb **nxt,3608static int io_send(struct io_kiocb *req, bool force_nonblock)
3104 bool force_nonblock)
3105{3609{
3106#if defined(CONFIG_NET)
3107 struct socket *sock;3610 struct socket *sock;
3108 int ret;3611 int ret;
3109 3612
@@ -3144,17 +3647,120 @@ static int io_send(struct io_kiocb *req, struct io_kiocb **nxt, · 34 unmodified lines
3144 io_cqring_add_event(req, ret);3647 io_cqring_add_event(req, ret);
3145 if (ret < 0)3648 if (ret < 0)
3146 req_set_fail_links(req);3649 req_set_fail_links(req);
3147 io_put_req_find_next(req, nxt);3650 io_put_req(req);
3148 return 0;3651 return 0;
3149#else3652}
3150 return -EOPNOTSUPP;3653
3654static int __io_recvmsg_copy_hdr(struct io_kiocb *req, struct io_async_ctx *io)
3655{
3656 struct io_sr_msg *sr = &req->sr_msg;
3657 struct iovec __user *uiov;
3658 size_t iov_len;
3659 int ret;
3660
3661 ret = __copy_msghdr_from_user(&io->msg.msg, sr->msg, &io->msg.uaddr,
3662 &uiov, &iov_len);
3663 if (ret)
3664 return ret;
3665
3666 if (req->flags & REQ_F_BUFFER_SELECT) {
3667 if (iov_len > 1)
3668 return -EINVAL;
3669 if (copy_from_user(io->msg.iov, uiov, sizeof(*uiov)))
3670 return -EFAULT;
3671 sr->len = io->msg.iov[0].iov_len;
3672 iov_iter_init(&io->msg.msg.msg_iter, READ, io->msg.iov, 1,
3673 sr->len);
3674 io->msg.iov = NULL;
3675 } else {
3676 ret = import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
3677 &io->msg.iov, &io->msg.msg.msg_iter);
3678 if (ret > 0)
3679 ret = 0;
3680 }
3681
3682 return ret;
3683}
3684
3685#ifdef CONFIG_COMPAT
3686static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
3687 struct io_async_ctx *io)
3688{
3689 struct compat_msghdr __user *msg_compat;
3690 struct io_sr_msg *sr = &req->sr_msg;
3691 struct compat_iovec __user *uiov;
3692 compat_uptr_t ptr;
3693 compat_size_t len;
3694 int ret;
3695
3696 msg_compat = (struct compat_msghdr __user *) sr->msg;
3697 ret = __get_compat_msghdr(&io->msg.msg, msg_compat, &io->msg.uaddr,
3698 &ptr, &len);
3699 if (ret)
3700 return ret;
3701
3702 uiov = compat_ptr(ptr);
3703 if (req->flags & REQ_F_BUFFER_SELECT) {
3704 compat_ssize_t clen;
3705
3706 if (len > 1)
3707 return -EINVAL;
3708 if (!access_ok(uiov, sizeof(*uiov)))
3709 return -EFAULT;
3710 if (__get_user(clen, &uiov->iov_len))
3711 return -EFAULT;
3712 if (clen < 0)
3713 return -EINVAL;
3714 sr->len = io->msg.iov[0].iov_len;
3715 io->msg.iov = NULL;
3716 } else {
3717 ret = compat_import_iovec(READ, uiov, len, UIO_FASTIOV,
3718 &io->msg.iov,
3719 &io->msg.msg.msg_iter);
3720 if (ret < 0)
3721 return ret;
3722 }
3723
3724 return 0;
3725}
3726#endif
3727
3728static int io_recvmsg_copy_hdr(struct io_kiocb *req, struct io_async_ctx *io)
3729{
3730 io->msg.iov = io->msg.fast_iov;
3731
3732#ifdef CONFIG_COMPAT
3733 if (req->ctx->compat)
3734 return __io_compat_recvmsg_copy_hdr(req, io);
3151#endif3735#endif
3736
3737 return __io_recvmsg_copy_hdr(req, io);
3738}
3739
3740static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,
3741 int *cflags, bool needs_lock)
3742{
3743 struct io_sr_msg *sr = &req->sr_msg;
3744 struct io_buffer *kbuf;
3745
3746 if (!(req->flags & REQ_F_BUFFER_SELECT))
3747 return NULL;
3748
3749 kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
3750 if (IS_ERR(kbuf))
3751 return kbuf;
3752
3753 sr->kbuf = kbuf;
3754 req->flags |= REQ_F_BUFFER_SELECTED;
3755
3756 *cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
3757 *cflags |= IORING_CQE_F_BUFFER;
3758 return kbuf;
3152}3759}
3153 3760
3154static int io_recvmsg_prep(struct io_kiocb *req,3761static int io_recvmsg_prep(struct io_kiocb *req,
3155 const struct io_uring_sqe *sqe)3762 const struct io_uring_sqe *sqe)
3156{3763{
3157#if defined(CONFIG_NET)
3158 struct io_sr_msg *sr = &req->sr_msg;3764 struct io_sr_msg *sr = &req->sr_msg;
3159 struct io_async_ctx *io = req->io;3765 struct io_async_ctx *io = req->io;
3160 int ret;3766 int ret;
@@ -3162,6 +3768,7 @@ static int io_recvmsg_prep(struct io_kiocb *req, · 1 unmodified line
3162 sr->msg_flags = READ_ONCE(sqe->msg_flags);3768 sr->msg_flags = READ_ONCE(sqe->msg_flags);
3163 sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));3769 sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
3164 sr->len = READ_ONCE(sqe->len);3770 sr->len = READ_ONCE(sqe->len);
3771 sr->bgid = READ_ONCE(sqe->buf_group);
3165 3772
3166#ifdef CONFIG_COMPAT3773#ifdef CONFIG_COMPAT
3167 if (req->ctx->compat)3774 if (req->ctx->compat)
@@ -3174,30 +3781,24 @@ static int io_recvmsg_prep(struct io_kiocb *req, · 6 unmodified lines
3174 if (req->flags & REQ_F_NEED_CLEANUP)3781 if (req->flags & REQ_F_NEED_CLEANUP)
3175 return 0;3782 return 0;
3176 3783
3177 io->msg.iov = io->msg.fast_iov;3784 ret = io_recvmsg_copy_hdr(req, io);
3178 ret = recvmsg_copy_msghdr(&io->msg.msg, sr->msg, sr->msg_flags,
3179 &io->msg.uaddr, &io->msg.iov);
3180 if (!ret)3785 if (!ret)
3181 req->flags |= REQ_F_NEED_CLEANUP;3786 req->flags |= REQ_F_NEED_CLEANUP;
3182 return ret;3787 return ret;
3183#else
3184 return -EOPNOTSUPP;
3185#endif
3186}3788}
3187 3789
3188static int io_recvmsg(struct io_kiocb *req, struct io_kiocb **nxt,3790static int io_recvmsg(struct io_kiocb *req, bool force_nonblock)
3189 bool force_nonblock)
3190{3791{
3191#if defined(CONFIG_NET)
3192 struct io_async_msghdr *kmsg = NULL;3792 struct io_async_msghdr *kmsg = NULL;
3193 struct socket *sock;3793 struct socket *sock;
3194 int ret;3794 int ret, cflags = 0;
3195 3795
3196 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))3796 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3197 return -EINVAL;3797 return -EINVAL;
3198 3798
3199 sock = sock_from_file(req->file, &ret);3799 sock = sock_from_file(req->file, &ret);
3200 if (sock) {3800 if (sock) {
3801 struct io_buffer *kbuf;
3201 struct io_async_ctx io;3802 struct io_async_ctx io;
3202 unsigned flags;3803 unsigned flags;
3203 3804
@@ -3209,19 +3810,23 @@ static int io_recvmsg(struct io_kiocb *req, struct io_kiocb **nxt, · 5 unmodified lines
3209 kmsg->iov = kmsg->fast_iov;3810 kmsg->iov = kmsg->fast_iov;
3210 kmsg->msg.msg_iter.iov = kmsg->iov;3811 kmsg->msg.msg_iter.iov = kmsg->iov;
3211 } else {3812 } else {
3212 struct io_sr_msg *sr = &req->sr_msg;
3213
3214 kmsg = &io.msg;3813 kmsg = &io.msg;
3215 kmsg->msg.msg_name = &io.msg.addr;3814 kmsg->msg.msg_name = &io.msg.addr;
3216 3815
3217 io.msg.iov = io.msg.fast_iov;3816 ret = io_recvmsg_copy_hdr(req, &io);
3218 ret = recvmsg_copy_msghdr(&io.msg.msg, sr->msg,
3219 sr->msg_flags, &io.msg.uaddr,
3220 &io.msg.iov);
3221 if (ret)3817 if (ret)
3222 return ret;3818 return ret;
3223 }3819 }
3224 3820
3821 kbuf = io_recv_buffer_select(req, &cflags, !force_nonblock);
3822 if (IS_ERR(kbuf)) {
3823 return PTR_ERR(kbuf);
3824 } else if (kbuf) {
3825 kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
3826 iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->iov,
3827 1, req->sr_msg.len);
3828 }
3829
3225 flags = req->sr_msg.msg_flags;3830 flags = req->sr_msg.msg_flags;
3226 if (flags & MSG_DONTWAIT)3831 if (flags & MSG_DONTWAIT)
3227 req->flags |= REQ_F_NOWAIT;3832 req->flags |= REQ_F_NOWAIT;
@@ -3230,18 +3835,8 @@ static int io_recvmsg(struct io_kiocb *req, struct io_kiocb **nxt, · 2 unmodified lines
3230 3835
3231 ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.msg,3836 ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.msg,
3232 kmsg->uaddr, flags);3837 kmsg->uaddr, flags);
3233 if (force_nonblock && ret == -EAGAIN) {3838 if (force_nonblock && ret == -EAGAIN)
3234 if (req->io)3839 return io_setup_async_msg(req, kmsg);
3235 return -EAGAIN;
3236 if (io_alloc_async_ctx(req)) {
3237 if (kmsg->iov != kmsg->fast_iov)
3238 kfree(kmsg->iov);
3239 return -ENOMEM;
3240 }
3241 memcpy(&req->io->msg, &io.msg, sizeof(io.msg));
3242 req->flags |= REQ_F_NEED_CLEANUP;
3243 return -EAGAIN;
3244 }
3245 if (ret == -ERESTARTSYS)3840 if (ret == -ERESTARTSYS)
3246 ret = -EINTR;3841 ret = -EINTR;
3247 }3842 }
@@ -3249,22 +3844,18 @@ static int io_recvmsg(struct io_kiocb *req, struct io_kiocb **nxt, · 1 unmodified line
3249 if (kmsg && kmsg->iov != kmsg->fast_iov)3844 if (kmsg && kmsg->iov != kmsg->fast_iov)
3250 kfree(kmsg->iov);3845 kfree(kmsg->iov);
3251 req->flags &= ~REQ_F_NEED_CLEANUP;3846 req->flags &= ~REQ_F_NEED_CLEANUP;
3252 io_cqring_add_event(req, ret);3847 __io_cqring_add_event(req, ret, cflags);
3253 if (ret < 0)3848 if (ret < 0)
3254 req_set_fail_links(req);3849 req_set_fail_links(req);
3255 io_put_req_find_next(req, nxt);3850 io_put_req(req);
3256 return 0;3851 return 0;
3257#else
3258 return -EOPNOTSUPP;
3259#endif
3260}3852}
3261 3853
3262static int io_recv(struct io_kiocb *req, struct io_kiocb **nxt,3854static int io_recv(struct io_kiocb *req, bool force_nonblock)
3263 bool force_nonblock)
3264{3855{
3265#if defined(CONFIG_NET)3856 struct io_buffer *kbuf = NULL;
3266 struct socket *sock;3857 struct socket *sock;
3267 int ret;3858 int ret, cflags = 0;
3268 3859
3269 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))3860 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3270 return -EINVAL;3861 return -EINVAL;
@@ -3272,15 +3863,25 @@ static int io_recv(struct io_kiocb *req, struct io_kiocb **nxt, · 1 unmodified line
3272 sock = sock_from_file(req->file, &ret);3863 sock = sock_from_file(req->file, &ret);
3273 if (sock) {3864 if (sock) {
3274 struct io_sr_msg *sr = &req->sr_msg;3865 struct io_sr_msg *sr = &req->sr_msg;
3866 void __user *buf = sr->buf;
3275 struct msghdr msg;3867 struct msghdr msg;
3276 struct iovec iov;3868 struct iovec iov;
3277 unsigned flags;3869 unsigned flags;
3278 3870
3279 ret = import_single_range(READ, sr->buf, sr->len, &iov,3871 kbuf = io_recv_buffer_select(req, &cflags, !force_nonblock);
3872 if (IS_ERR(kbuf))
3873 return PTR_ERR(kbuf);
3874 else if (kbuf)
3875 buf = u64_to_user_ptr(kbuf->addr);
3876
3877 ret = import_single_range(READ, buf, sr->len, &iov,
3280 &msg.msg_iter);3878 &msg.msg_iter);
3281 if (ret)3879 if (ret) {
3880 kfree(kbuf);
3282 return ret;3881 return ret;
3882 }
3283 3883
3884 req->flags |= REQ_F_NEED_CLEANUP;
3284 msg.msg_name = NULL;3885 msg.msg_name = NULL;
3285 msg.msg_control = NULL;3886 msg.msg_control = NULL;
3286 msg.msg_controllen = 0;3887 msg.msg_controllen = 0;
@@ -3301,20 +3902,17 @@ static int io_recv(struct io_kiocb *req, struct io_kiocb **nxt, · 14 unmodified lines
3301 ret = -EINTR;3902 ret = -EINTR;
3302 }3903 }
3303 3904
3304 io_cqring_add_event(req, ret);3905 kfree(kbuf);
3906 req->flags &= ~REQ_F_NEED_CLEANUP;
3907 __io_cqring_add_event(req, ret, cflags);
3305 if (ret < 0)3908 if (ret < 0)
3306 req_set_fail_links(req);3909 req_set_fail_links(req);
3307 io_put_req_find_next(req, nxt);3910 io_put_req(req);
3308 return 0;3911 return 0;
3309#else
3310 return -EOPNOTSUPP;
3311#endif
3312}3912}
3313 3913
3314
3315static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)3914static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3316{3915{
3317#if defined(CONFIG_NET)
3318 struct io_accept *accept = &req->accept;3916 struct io_accept *accept = &req->accept;
3319 3917
3320 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))3918 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
@@ -3327,14 +3925,9 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 6 unmodified lines
3327 accept->flags = READ_ONCE(sqe->accept_flags);3925 accept->flags = READ_ONCE(sqe->accept_flags);
3328 accept->nofile = rlimit(RLIMIT_NOFILE);3926 accept->nofile = rlimit(RLIMIT_NOFILE);
3329 return 0;3927 return 0;
3330#else
3331 return -EOPNOTSUPP;
3332#endif
3333}3928}
3334 3929
3335#if defined(CONFIG_NET)3930static int __io_accept(struct io_kiocb *req, bool force_nonblock)
3336static int __io_accept(struct io_kiocb *req, struct io_kiocb **nxt,
3337 bool force_nonblock)
3338{3931{
3339 struct io_accept *accept = &req->accept;3932 struct io_accept *accept = &req->accept;
3340 unsigned file_flags;3933 unsigned file_flags;
@@ -3351,44 +3944,34 @@ static int __io_accept(struct io_kiocb *req, struct io_kiocb **nxt, · 10 unmodified lines
3351 if (ret < 0)3944 if (ret < 0)
3352 req_set_fail_links(req);3945 req_set_fail_links(req);
3353 io_cqring_add_event(req, ret);3946 io_cqring_add_event(req, ret);
3354 io_put_req_find_next(req, nxt);3947 io_put_req(req);
3355 return 0;3948 return 0;
3356}3949}
3357 3950
3358static void io_accept_finish(struct io_wq_work **workptr)3951static void io_accept_finish(struct io_wq_work **workptr)
3359{3952{
3360 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);3953 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
3361 struct io_kiocb *nxt = NULL;
3362 3954
3363 if (io_req_cancelled(req))3955 if (io_req_cancelled(req))
3364 return;3956 return;
3365 __io_accept(req, &nxt, false);3957 __io_accept(req, false);
3366 if (nxt)3958 io_steal_work(req, workptr);
3367 io_wq_assign_next(workptr, nxt);
3368}3959}
3369#endif
3370 3960
3371static int io_accept(struct io_kiocb *req, struct io_kiocb **nxt,3961static int io_accept(struct io_kiocb *req, bool force_nonblock)
3372 bool force_nonblock)
3373{3962{
3374#if defined(CONFIG_NET)
3375 int ret;3963 int ret;
3376 3964
3377 ret = __io_accept(req, nxt, force_nonblock);3965 ret = __io_accept(req, force_nonblock);
3378 if (ret == -EAGAIN && force_nonblock) {3966 if (ret == -EAGAIN && force_nonblock) {
3379 req->work.func = io_accept_finish;3967 req->work.func = io_accept_finish;
3380 io_put_req(req);
3381 return -EAGAIN;3968 return -EAGAIN;
3382 }3969 }
3383 return 0;3970 return 0;
3384#else
3385 return -EOPNOTSUPP;
3386#endif
3387}3971}
3388 3972
3389static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)3973static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3390{3974{
3391#if defined(CONFIG_NET)
3392 struct io_connect *conn = &req->connect;3975 struct io_connect *conn = &req->connect;
3393 struct io_async_ctx *io = req->io;3976 struct io_async_ctx *io = req->io;
3394 3977
@@ -3405,15 +3988,10 @@ static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 10 unmodified lines
3405 3988
3406 return move_addr_to_kernel(conn->addr, conn->addr_len,3989 return move_addr_to_kernel(conn->addr, conn->addr_len,
3407 &io->connect.address);3990 &io->connect.address);
3408#else
3409 return -EOPNOTSUPP;
3410#endif
3411}3991}
3412 3992
3413static int io_connect(struct io_kiocb *req, struct io_kiocb **nxt,3993static int io_connect(struct io_kiocb *req, bool force_nonblock)
3414 bool force_nonblock)
3415{3994{
3416#if defined(CONFIG_NET)
3417 struct io_async_ctx __io, *io;3995 struct io_async_ctx __io, *io;
3418 unsigned file_flags;3996 unsigned file_flags;
3419 int ret;3997 int ret;
@@ -3449,25 +4027,301 @@ out: · 29 unmodified lines
3449 if (ret < 0)4027 if (ret < 0)
3450 req_set_fail_links(req);4028 req_set_fail_links(req);
3451 io_cqring_add_event(req, ret);4029 io_cqring_add_event(req, ret);
3452 io_put_req_find_next(req, nxt);4030 io_put_req(req);
3453 return 0;4031 return 0;
3454#else4032}
4033#else /* !CONFIG_NET */
4034static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4035{
3455 return -EOPNOTSUPP;4036 return -EOPNOTSUPP;
3456#endif
3457}4037}
3458 4038
3459static void io_poll_remove_one(struct io_kiocb *req)4039static int io_sendmsg(struct io_kiocb *req, bool force_nonblock)
4040{
4041 return -EOPNOTSUPP;
4042}
4043
4044static int io_send(struct io_kiocb *req, bool force_nonblock)
4045{
4046 return -EOPNOTSUPP;
4047}
4048
4049static int io_recvmsg_prep(struct io_kiocb *req,
4050 const struct io_uring_sqe *sqe)
4051{
4052 return -EOPNOTSUPP;
4053}
4054
4055static int io_recvmsg(struct io_kiocb *req, bool force_nonblock)
4056{
4057 return -EOPNOTSUPP;
4058}
4059
4060static int io_recv(struct io_kiocb *req, bool force_nonblock)
4061{
4062 return -EOPNOTSUPP;
4063}
4064
4065static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4066{
4067 return -EOPNOTSUPP;
4068}
4069
4070static int io_accept(struct io_kiocb *req, bool force_nonblock)
4071{
4072 return -EOPNOTSUPP;
4073}
4074
4075static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4076{
4077 return -EOPNOTSUPP;
4078}
4079
4080static int io_connect(struct io_kiocb *req, bool force_nonblock)
4081{
4082 return -EOPNOTSUPP;
4083}
4084#endif /* CONFIG_NET */
4085
4086struct io_poll_table {
4087 struct poll_table_struct pt;
4088 struct io_kiocb *req;
4089 int error;
4090};
4091
4092static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
4093 struct wait_queue_head *head)
4094{
4095 if (unlikely(poll->head)) {
4096 pt->error = -EINVAL;
4097 return;
4098 }
4099
4100 pt->error = 0;
4101 poll->head = head;
4102 add_wait_queue(head, &poll->wait);
4103}
4104
4105static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
4106 struct poll_table_struct *p)
4107{
4108 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
4109
4110 __io_queue_proc(&pt->req->apoll->poll, pt, head);
4111}
4112
4113static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
4114 __poll_t mask, task_work_func_t func)
4115{
4116 struct task_struct *tsk;
4117
4118 /* for instances that support it check for an event match first: */
4119 if (mask && !(mask & poll->events))
4120 return 0;
4121
4122 trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);
4123
4124 list_del_init(&poll->wait.entry);
4125
4126 tsk = req->task;
4127 req->result = mask;
4128 init_task_work(&req->task_work, func);
4129 /*
4130 * If this fails, then the task is exiting. If that is the case, then
4131 * the exit check will ultimately cancel these work items. Hence we
4132 * don't need to check here and handle it specifically.
4133 */
4134 task_work_add(tsk, &req->task_work, true);
4135 wake_up_process(tsk);
4136 return 1;
4137}
4138
4139static void io_async_task_func(struct callback_head *cb)
4140{
4141 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
4142 struct async_poll *apoll = req->apoll;
4143 struct io_ring_ctx *ctx = req->ctx;
4144
4145 trace_io_uring_task_run(req->ctx, req->opcode, req->user_data);
4146
4147 WARN_ON_ONCE(!list_empty(&req->apoll->poll.wait.entry));
4148
4149 if (hash_hashed(&req->hash_node)) {
4150 spin_lock_irq(&ctx->completion_lock);
4151 hash_del(&req->hash_node);
4152 spin_unlock_irq(&ctx->completion_lock);
4153 }
4154
4155 /* restore ->work in case we need to retry again */
4156 memcpy(&req->work, &apoll->work, sizeof(req->work));
4157
4158 __set_current_state(TASK_RUNNING);
4159 mutex_lock(&ctx->uring_lock);
4160 __io_queue_sqe(req, NULL);
4161 mutex_unlock(&ctx->uring_lock);
4162
4163 kfree(apoll);
4164}
4165
4166static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
4167 void *key)
4168{
4169 struct io_kiocb *req = wait->private;
4170 struct io_poll_iocb *poll = &req->apoll->poll;
4171
4172 trace_io_uring_poll_wake(req->ctx, req->opcode, req->user_data,
4173 key_to_poll(key));
4174
4175 return __io_async_wake(req, poll, key_to_poll(key), io_async_task_func);
4176}
4177
4178static void io_poll_req_insert(struct io_kiocb *req)
4179{
4180 struct io_ring_ctx *ctx = req->ctx;
4181 struct hlist_head *list;
4182
4183 list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
4184 hlist_add_head(&req->hash_node, list);
4185}
4186
4187static __poll_t __io_arm_poll_handler(struct io_kiocb *req,
4188 struct io_poll_iocb *poll,
4189 struct io_poll_table *ipt, __poll_t mask,
4190 wait_queue_func_t wake_func)
4191 __acquires(&ctx->completion_lock)
4192{
4193 struct io_ring_ctx *ctx = req->ctx;
4194 bool cancel = false;
4195
4196 poll->file = req->file;
4197 poll->head = NULL;
4198 poll->done = poll->canceled = false;
4199 poll->events = mask;
4200
4201 ipt->pt._key = mask;
4202 ipt->req = req;
4203 ipt->error = -EINVAL;
4204
4205 INIT_LIST_HEAD(&poll->wait.entry);
4206 init_waitqueue_func_entry(&poll->wait, wake_func);
4207 poll->wait.private = req;
4208
4209 mask = vfs_poll(req->file, &ipt->pt) & poll->events;
4210
4211 spin_lock_irq(&ctx->completion_lock);
4212 if (likely(poll->head)) {
4213 spin_lock(&poll->head->lock);
4214 if (unlikely(list_empty(&poll->wait.entry))) {
4215 if (ipt->error)
4216 cancel = true;
4217 ipt->error = 0;
4218 mask = 0;
4219 }
4220 if (mask || ipt->error)
4221 list_del_init(&poll->wait.entry);
4222 else if (cancel)
4223 WRITE_ONCE(poll->canceled, true);
4224 else if (!poll->done) /* actually waiting for an event */
4225 io_poll_req_insert(req);
4226 spin_unlock(&poll->head->lock);
4227 }
4228
4229 return mask;
4230}
4231
4232static bool io_arm_poll_handler(struct io_kiocb *req)
4233{
4234 const struct io_op_def *def = &io_op_defs[req->opcode];
4235 struct io_ring_ctx *ctx = req->ctx;
4236 struct async_poll *apoll;
4237 struct io_poll_table ipt;
4238 __poll_t mask, ret;
4239
4240 if (!req->file || !file_can_poll(req->file))
4241 return false;
4242 if (req->flags & (REQ_F_MUST_PUNT | REQ_F_POLLED))
4243 return false;
4244 if (!def->pollin && !def->pollout)
4245 return false;
4246
4247 apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
4248 if (unlikely(!apoll))
4249 return false;
4250
4251 req->flags |= REQ_F_POLLED;
4252 memcpy(&apoll->work, &req->work, sizeof(req->work));
4253
4254 /*
4255 * Don't need a reference here, as we're adding it to the task
4256 * task_works list. If the task exits, the list is pruned.
4257 */
4258 req->task = current;
4259 req->apoll = apoll;
4260 INIT_HLIST_NODE(&req->hash_node);
4261
4262 mask = 0;
4263 if (def->pollin)
4264 mask |= POLLIN | POLLRDNORM;
4265 if (def->pollout)
4266 mask |= POLLOUT | POLLWRNORM;
4267 mask |= POLLERR | POLLPRI;
4268
4269 ipt.pt._qproc = io_async_queue_proc;
4270
4271 ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask,
4272 io_async_wake);
4273 if (ret) {
4274 ipt.error = 0;
4275 apoll->poll.done = true;
4276 spin_unlock_irq(&ctx->completion_lock);
4277 memcpy(&req->work, &apoll->work, sizeof(req->work));
4278 kfree(apoll);
4279 return false;
4280 }
4281 spin_unlock_irq(&ctx->completion_lock);
4282 trace_io_uring_poll_arm(ctx, req->opcode, req->user_data, mask,
4283 apoll->poll.events);
4284 return true;
4285}
4286
4287static bool __io_poll_remove_one(struct io_kiocb *req,
4288 struct io_poll_iocb *poll)
3460{4289{
3461 struct io_poll_iocb *poll = &req->poll;4290 bool do_complete = false;
3462 4291
3463 spin_lock(&poll->head->lock);4292 spin_lock(&poll->head->lock);
3464 WRITE_ONCE(poll->canceled, true);4293 WRITE_ONCE(poll->canceled, true);
3465 if (!list_empty(&poll->wait.entry)) {4294 if (!list_empty(&poll->wait.entry)) {
3466 list_del_init(&poll->wait.entry);4295 list_del_init(&poll->wait.entry);
3467 io_queue_async_work(req);4296 do_complete = true;
3468 }4297 }
3469 spin_unlock(&poll->head->lock);4298 spin_unlock(&poll->head->lock);
4299 return do_complete;
4300}
4301
4302static bool io_poll_remove_one(struct io_kiocb *req)
4303{
4304 bool do_complete;
4305
4306 if (req->opcode == IORING_OP_POLL_ADD) {
4307 do_complete = __io_poll_remove_one(req, &req->poll);
4308 } else {
4309 /* non-poll requests have submit ref still */
4310 do_complete = __io_poll_remove_one(req, &req->apoll->poll);
4311 if (do_complete)
4312 io_put_req(req);
4313 }
4314
3470 hash_del(&req->hash_node);4315 hash_del(&req->hash_node);
4316
4317 if (do_complete) {
4318 io_cqring_fill_event(req, -ECANCELED);
4319 io_commit_cqring(req->ctx);
4320 req->flags |= REQ_F_COMP_LOCKED;
4321 io_put_req(req);
4322 }
4323
4324 return do_complete;
3471}4325}
3472 4326
3473static void io_poll_remove_all(struct io_ring_ctx *ctx)4327static void io_poll_remove_all(struct io_ring_ctx *ctx)
@@ -3485,6 +4339,8 @@ static void io_poll_remove_all(struct io_ring_ctx *ctx) · 11 unmodified lines
3485 io_poll_remove_one(req);4339 io_poll_remove_one(req);
3486 }4340 }
3487 spin_unlock_irq(&ctx->completion_lock);4341 spin_unlock_irq(&ctx->completion_lock);
4342
4343 io_cqring_ev_posted(ctx);
3488}4344}
3489 4345
3490static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr)4346static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr)
@@ -3494,10 +4350,11 @@ static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr) · 3 unmodified lines
3494 4350
3495 list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];4351 list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
3496 hlist_for_each_entry(req, list, hash_node) {4352 hlist_for_each_entry(req, list, hash_node) {
3497 if (sqe_addr == req->user_data) {4353 if (sqe_addr != req->user_data)
3498 io_poll_remove_one(req);4354 continue;
4355 if (io_poll_remove_one(req))
3499 return 0;4356 return 0;
3500 }4357 return -EALREADY;
3501 }4358 }
3502 4359
3503 return -ENOENT;4360 return -ENOENT;
@@ -3543,186 +4400,54 @@ static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error) · 39 unmodified lines
3543 struct io_ring_ctx *ctx = req->ctx;4400 struct io_ring_ctx *ctx = req->ctx;
3544 4401
3545 req->poll.done = true;4402 req->poll.done = true;
3546 if (error)4403 io_cqring_fill_event(req, error ? error : mangle_poll(mask));
3547 io_cqring_fill_event(req, error);
3548 else
3549 io_cqring_fill_event(req, mangle_poll(mask));
3550 io_commit_cqring(ctx);4404 io_commit_cqring(ctx);
3551}4405}
3552 4406
3553static void io_poll_complete_work(struct io_wq_work **workptr)4407static void io_poll_task_handler(struct io_kiocb *req, struct io_kiocb **nxt)
3554{4408{
3555 struct io_wq_work *work = *workptr;
3556 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
3557 struct io_poll_iocb *poll = &req->poll;
3558 struct poll_table_struct pt = { ._key = poll->events };
3559 struct io_ring_ctx *ctx = req->ctx;4409 struct io_ring_ctx *ctx = req->ctx;
3560 struct io_kiocb *nxt = NULL;
3561 __poll_t mask = 0;
3562 int ret = 0;
3563
3564 if (work->flags & IO_WQ_WORK_CANCEL) {
3565 WRITE_ONCE(poll->canceled, true);
3566 ret = -ECANCELED;
3567 } else if (READ_ONCE(poll->canceled)) {
3568 ret = -ECANCELED;
3569 }
3570
3571 if (ret != -ECANCELED)
3572 mask = vfs_poll(poll->file, &pt) & poll->events;
3573 4410
3574 /*
3575 * Note that ->ki_cancel callers also delete iocb from active_reqs after
3576 * calling ->ki_cancel. We need the ctx_lock roundtrip here to
3577 * synchronize with them. In the cancellation case the list_del_init
3578 * itself is not actually needed, but harmless so we keep it in to
3579 * avoid further branches in the fast path.
3580 */
3581 spin_lock_irq(&ctx->completion_lock);4411 spin_lock_irq(&ctx->completion_lock);
3582 if (!mask && ret != -ECANCELED) {
3583 add_wait_queue(poll->head, &poll->wait);
3584 spin_unlock_irq(&ctx->completion_lock);
3585 return;
3586 }
3587 hash_del(&req->hash_node);4412 hash_del(&req->hash_node);
3588 io_poll_complete(req, mask, ret);4413 io_poll_complete(req, req->result, 0);
4414 req->flags |= REQ_F_COMP_LOCKED;
4415 io_put_req_find_next(req, nxt);
3589 spin_unlock_irq(&ctx->completion_lock);4416 spin_unlock_irq(&ctx->completion_lock);
3590 4417
3591 io_cqring_ev_posted(ctx);4418 io_cqring_ev_posted(ctx);
3592
3593 if (ret < 0)
3594 req_set_fail_links(req);
3595 io_put_req_find_next(req, &nxt);
3596 if (nxt)
3597 io_wq_assign_next(workptr, nxt);
3598}4419}
3599 4420
3600static void __io_poll_flush(struct io_ring_ctx *ctx, struct llist_node *nodes)4421static void io_poll_task_func(struct callback_head *cb)
3601{4422{
3602 struct io_kiocb *req, *tmp;4423 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
3603 struct req_batch rb;4424 struct io_kiocb *nxt = NULL;
3604 4425
3605 rb.to_free = rb.need_iter = 0;4426 io_poll_task_handler(req, &nxt);
3606 spin_lock_irq(&ctx->completion_lock);4427 if (nxt) {
3607 llist_for_each_entry_safe(req, tmp, nodes, llist_node) {4428 struct io_ring_ctx *ctx = nxt->ctx;
3608 hash_del(&req->hash_node);
3609 io_poll_complete(req, req->result, 0);
3610 4429
3611 if (refcount_dec_and_test(&req->refs) &&4430 mutex_lock(&ctx->uring_lock);
3612 !io_req_multi_free(&rb, req)) {4431 __io_queue_sqe(nxt, NULL);
3613 req->flags |= REQ_F_COMP_LOCKED;4432 mutex_unlock(&ctx->uring_lock);
3614 io_free_req(req);
3615 }
3616 }4433 }
3617 spin_unlock_irq(&ctx->completion_lock);
3618
3619 io_cqring_ev_posted(ctx);
3620 io_free_req_many(ctx, &rb);
3621}
3622
3623static void io_poll_flush(struct io_wq_work **workptr)
3624{
3625 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
3626 struct llist_node *nodes;
3627
3628 nodes = llist_del_all(&req->ctx->poll_llist);
3629 if (nodes)
3630 __io_poll_flush(req->ctx, nodes);
3631}
3632
3633static void io_poll_trigger_evfd(struct io_wq_work **workptr)
3634{
3635 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
3636
3637 eventfd_signal(req->ctx->cq_ev_fd, 1);
3638 io_put_req(req);
3639}4434}
3640 4435
3641static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,4436static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
3642 void *key)4437 void *key)
3643{4438{
3644 struct io_poll_iocb *poll = wait->private;4439 struct io_kiocb *req = wait->private;
3645 struct io_kiocb *req = container_of(poll, struct io_kiocb, poll);4440 struct io_poll_iocb *poll = &req->poll;
3646 struct io_ring_ctx *ctx = req->ctx;
3647 __poll_t mask = key_to_poll(key);
3648
3649 /* for instances that support it check for an event match first: */
3650 if (mask && !(mask & poll->events))
3651 return 0;
3652
3653 list_del_init(&poll->wait.entry);
3654
3655 /*
3656 * Run completion inline if we can. We're using trylock here because
3657 * we are violating the completion_lock -> poll wq lock ordering.
3658 * If we have a link timeout we're going to need the completion_lock
3659 * for finalizing the request, mark us as having grabbed that already.
3660 */
3661 if (mask) {
3662 unsigned long flags;
3663
3664 if (llist_empty(&ctx->poll_llist) &&
3665 spin_trylock_irqsave(&ctx->completion_lock, flags)) {
3666 bool trigger_ev;
3667
3668 hash_del(&req->hash_node);
3669 io_poll_complete(req, mask, 0);
3670
3671 trigger_ev = io_should_trigger_evfd(ctx);
3672 if (trigger_ev && eventfd_signal_count()) {
3673 trigger_ev = false;
3674 req->work.func = io_poll_trigger_evfd;
3675 } else {
3676 req->flags |= REQ_F_COMP_LOCKED;
3677 io_put_req(req);
3678 req = NULL;
3679 }
3680 spin_unlock_irqrestore(&ctx->completion_lock, flags);
3681 __io_cqring_ev_posted(ctx, trigger_ev);
3682 } else {
3683 req->result = mask;
3684 req->llist_node.next = NULL;
3685 /* if the list wasn't empty, we're done */
3686 if (!llist_add(&req->llist_node, &ctx->poll_llist))
3687 req = NULL;
3688 else
3689 req->work.func = io_poll_flush;
3690 }
3691 }
3692 if (req)
3693 io_queue_async_work(req);
3694 4441
3695 return 1;4442 return __io_async_wake(req, poll, key_to_poll(key), io_poll_task_func);
3696}4443}
3697 4444
3698struct io_poll_table {
3699 struct poll_table_struct pt;
3700 struct io_kiocb *req;
3701 int error;
3702};
3703
3704static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,4445static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
3705 struct poll_table_struct *p)4446 struct poll_table_struct *p)
3706{4447{
3707 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);4448 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
3708 4449
3709 if (unlikely(pt->req->poll.head)) {4450 __io_queue_proc(&pt->req->poll, pt, head);
3710 pt->error = -EINVAL;
3711 return;
3712 }
3713
3714 pt->error = 0;
3715 pt->req->poll.head = head;
3716 add_wait_queue(head, &pt->req->poll.wait);
3717}
3718
3719static void io_poll_req_insert(struct io_kiocb *req)
3720{
3721 struct io_ring_ctx *ctx = req->ctx;
3722 struct hlist_head *list;
3723
3724 list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
3725 hlist_add_head(&req->hash_node, list);
3726}4451}
3727 4452
3728static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)4453static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
@@ -3739,55 +4464,29 @@ static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe · 10 unmodified lines
3739 4464
3740 events = READ_ONCE(sqe->poll_events);4465 events = READ_ONCE(sqe->poll_events);
3741 poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;4466 poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;
4467
4468 /*
4469 * Don't need a reference here, as we're adding it to the task
4470 * task_works list. If the task exits, the list is pruned.
4471 */
4472 req->task = current;
3742 return 0;4473 return 0;
3743}4474}
3744 4475
3745static int io_poll_add(struct io_kiocb *req, struct io_kiocb **nxt)4476static int io_poll_add(struct io_kiocb *req)
3746{4477{
3747 struct io_poll_iocb *poll = &req->poll;4478 struct io_poll_iocb *poll = &req->poll;
3748 struct io_ring_ctx *ctx = req->ctx;4479 struct io_ring_ctx *ctx = req->ctx;
3749 struct io_poll_table ipt;4480 struct io_poll_table ipt;
3750 bool cancel = false;
3751 __poll_t mask;4481 __poll_t mask;
3752 4482
3753 INIT_IO_WORK(&req->work, io_poll_complete_work);
3754 INIT_HLIST_NODE(&req->hash_node);4483 INIT_HLIST_NODE(&req->hash_node);
3755
3756 poll->head = NULL;
3757 poll->done = false;
3758 poll->canceled = false;
3759
3760 ipt.pt._qproc = io_poll_queue_proc;
3761 ipt.pt._key = poll->events;
3762 ipt.req = req;
3763 ipt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */
3764
3765 /* initialized the list so that we can do list_empty checks */
3766 INIT_LIST_HEAD(&poll->wait.entry);
3767 init_waitqueue_func_entry(&poll->wait, io_poll_wake);
3768 poll->wait.private = poll;
3769
3770 INIT_LIST_HEAD(&req->list);4484 INIT_LIST_HEAD(&req->list);
4485 ipt.pt._qproc = io_poll_queue_proc;
3771 4486
3772 mask = vfs_poll(poll->file, &ipt.pt) & poll->events;4487 mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events,
4488 io_poll_wake);
3773 4489
3774 spin_lock_irq(&ctx->completion_lock);
3775 if (likely(poll->head)) {
3776 spin_lock(&poll->head->lock);
3777 if (unlikely(list_empty(&poll->wait.entry))) {
3778 if (ipt.error)
3779 cancel = true;
3780 ipt.error = 0;
3781 mask = 0;
3782 }
3783 if (mask || ipt.error)
3784 list_del_init(&poll->wait.entry);
3785 else if (cancel)
3786 WRITE_ONCE(poll->canceled, true);
3787 else if (!poll->done) /* actually waiting for an event */
3788 io_poll_req_insert(req);
3789 spin_unlock(&poll->head->lock);
3790 }
3791 if (mask) { /* no async, we'd stolen it */4490 if (mask) { /* no async, we'd stolen it */
3792 ipt.error = 0;4491 ipt.error = 0;
3793 io_poll_complete(req, mask, 0);4492 io_poll_complete(req, mask, 0);
@@ -3796,7 +4495,7 @@ static int io_poll_add(struct io_kiocb *req, struct io_kiocb **nxt) · 2 unmodified lines
3796 4495
3797 if (mask) {4496 if (mask) {
3798 io_cqring_ev_posted(ctx);4497 io_cqring_ev_posted(ctx);
3799 io_put_req_find_next(req, nxt);4498 io_put_req(req);
3800 }4499 }
3801 return ipt.error;4500 return ipt.error;
3802}4501}
@@ -4045,7 +4744,7 @@ static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr) · 242 unmodified lines
4045 4744
4046static void io_async_find_and_cancel(struct io_ring_ctx *ctx,4745static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
4047 struct io_kiocb *req, __u64 sqe_addr,4746 struct io_kiocb *req, __u64 sqe_addr,
4048 struct io_kiocb **nxt, int success_ret)4747 int success_ret)
4049{4748{
4050 unsigned long flags;4749 unsigned long flags;
4051 int ret;4750 int ret;
@@ -4071,7 +4770,7 @@ done: · 19 unmodified lines
4071 4770
4072 if (ret < 0)4771 if (ret < 0)
4073 req_set_fail_links(req);4772 req_set_fail_links(req);
4074 io_put_req_find_next(req, nxt);4773 io_put_req(req);
4075}4774}
4076 4775
4077static int io_async_cancel_prep(struct io_kiocb *req,4776static int io_async_cancel_prep(struct io_kiocb *req,
@@ -4087,11 +4786,11 @@ static int io_async_cancel_prep(struct io_kiocb *req, · 9 unmodified lines
4087 return 0;4786 return 0;
4088}4787}
4089 4788
4090static int io_async_cancel(struct io_kiocb *req, struct io_kiocb **nxt)4789static int io_async_cancel(struct io_kiocb *req)
4091{4790{
4092 struct io_ring_ctx *ctx = req->ctx;4791 struct io_ring_ctx *ctx = req->ctx;
4093 4792
4094 io_async_find_and_cancel(ctx, req, req->cancel.addr, nxt, 0);4793 io_async_find_and_cancel(ctx, req, req->cancel.addr, 0);
4095 return 0;4794 return 0;
4096}4795}
4097 4796
@@ -4226,6 +4925,15 @@ static int io_req_defer_prep(struct io_kiocb *req, · 128 unmodified lines
4226 case IORING_OP_EPOLL_CTL:4925 case IORING_OP_EPOLL_CTL:
4227 ret = io_epoll_ctl_prep(req, sqe);4926 ret = io_epoll_ctl_prep(req, sqe);
4228 break;4927 break;
4928 case IORING_OP_SPLICE:
4929 ret = io_splice_prep(req, sqe);
4930 break;
4931 case IORING_OP_PROVIDE_BUFFERS:
4932 ret = io_provide_buffers_prep(req, sqe);
4933 break;
4934 case IORING_OP_REMOVE_BUFFERS:
4935 ret = io_remove_buffers_prep(req, sqe);
4936 break;
4229 default:4937 default:
4230 printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",4938 printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
4231 req->opcode);4939 req->opcode);
@@ -4272,29 +4980,43 @@ static void io_cleanup_req(struct io_kiocb *req) · 40 unmodified lines
4272 case IORING_OP_READV:4980 case IORING_OP_READV:
4273 case IORING_OP_READ_FIXED:4981 case IORING_OP_READ_FIXED:
4274 case IORING_OP_READ:4982 case IORING_OP_READ:
4983 if (req->flags & REQ_F_BUFFER_SELECTED)
4984 kfree((void *)(unsigned long)req->rw.addr);
4985 /* fallthrough */
4275 case IORING_OP_WRITEV:4986 case IORING_OP_WRITEV:
4276 case IORING_OP_WRITE_FIXED:4987 case IORING_OP_WRITE_FIXED:
4277 case IORING_OP_WRITE:4988 case IORING_OP_WRITE:
4278 if (io->rw.iov != io->rw.fast_iov)4989 if (io->rw.iov != io->rw.fast_iov)
4279 kfree(io->rw.iov);4990 kfree(io->rw.iov);
4280 break;4991 break;
4281 case IORING_OP_SENDMSG:
4282 case IORING_OP_RECVMSG:4992 case IORING_OP_RECVMSG:
4993 if (req->flags & REQ_F_BUFFER_SELECTED)
4994 kfree(req->sr_msg.kbuf);
4995 /* fallthrough */
4996 case IORING_OP_SENDMSG:
4283 if (io->msg.iov != io->msg.fast_iov)4997 if (io->msg.iov != io->msg.fast_iov)
4284 kfree(io->msg.iov);4998 kfree(io->msg.iov);
4285 break;4999 break;
5000 case IORING_OP_RECV:
5001 if (req->flags & REQ_F_BUFFER_SELECTED)
5002 kfree(req->sr_msg.kbuf);
5003 break;
4286 case IORING_OP_OPENAT:5004 case IORING_OP_OPENAT:
4287 case IORING_OP_OPENAT2:5005 case IORING_OP_OPENAT2:
4288 case IORING_OP_STATX:5006 case IORING_OP_STATX:
4289 putname(req->open.filename);5007 putname(req->open.filename);
4290 break;5008 break;
5009 case IORING_OP_SPLICE:
5010 io_put_file(req, req->splice.file_in,
5011 (req->splice.flags & SPLICE_F_FD_IN_FIXED));
5012 break;
4291 }5013 }
4292 5014
4293 req->flags &= ~REQ_F_NEED_CLEANUP;5015 req->flags &= ~REQ_F_NEED_CLEANUP;
4294}5016}
4295 5017
4296static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,5018static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
4297 struct io_kiocb **nxt, bool force_nonblock)5019 bool force_nonblock)
4298{5020{
4299 struct io_ring_ctx *ctx = req->ctx;5021 struct io_ring_ctx *ctx = req->ctx;
4300 int ret;5022 int ret;
@@ -4311,7 +5033,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 10 unmodified lines
4311 if (ret < 0)5033 if (ret < 0)
4312 break;5034 break;
4313 }5035 }
4314 ret = io_read(req, nxt, force_nonblock);5036 ret = io_read(req, force_nonblock);
4315 break;5037 break;
4316 case IORING_OP_WRITEV:5038 case IORING_OP_WRITEV:
4317 case IORING_OP_WRITE_FIXED:5039 case IORING_OP_WRITE_FIXED:
@@ -4321,7 +5043,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 3 unmodified lines
4321 if (ret < 0)5043 if (ret < 0)
4322 break;5044 break;
4323 }5045 }
4324 ret = io_write(req, nxt, force_nonblock);5046 ret = io_write(req, force_nonblock);
4325 break;5047 break;
4326 case IORING_OP_FSYNC:5048 case IORING_OP_FSYNC:
4327 if (sqe) {5049 if (sqe) {
@@ -4329,7 +5051,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 1 unmodified line
4329 if (ret < 0)5051 if (ret < 0)
4330 break;5052 break;
4331 }5053 }
4332 ret = io_fsync(req, nxt, force_nonblock);5054 ret = io_fsync(req, force_nonblock);
4333 break;5055 break;
4334 case IORING_OP_POLL_ADD:5056 case IORING_OP_POLL_ADD:
4335 if (sqe) {5057 if (sqe) {
@@ -4337,7 +5059,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 1 unmodified line
4337 if (ret)5059 if (ret)
4338 break;5060 break;
4339 }5061 }
4340 ret = io_poll_add(req, nxt);5062 ret = io_poll_add(req);
4341 break;5063 break;
4342 case IORING_OP_POLL_REMOVE:5064 case IORING_OP_POLL_REMOVE:
4343 if (sqe) {5065 if (sqe) {
@@ -4353,7 +5075,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 9 unmodified lines
4353 if (ret < 0)5075 if (ret < 0)
4354 break;5076 break;
4355 }5077 }
4356 ret = io_sync_file_range(req, nxt, force_nonblock);5078 ret = io_sync_file_range(req, force_nonblock);
4357 break;5079 break;
4358 case IORING_OP_SENDMSG:5080 case IORING_OP_SENDMSG:
4359 case IORING_OP_SEND:5081 case IORING_OP_SEND:
@@ -4363,9 +5085,9 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 3 unmodified lines
4363 break;5085 break;
4364 }5086 }
4365 if (req->opcode == IORING_OP_SENDMSG)5087 if (req->opcode == IORING_OP_SENDMSG)
4366 ret = io_sendmsg(req, nxt, force_nonblock);5088 ret = io_sendmsg(req, force_nonblock);
4367 else5089 else
4368 ret = io_send(req, nxt, force_nonblock);5090 ret = io_send(req, force_nonblock);
4369 break;5091 break;
4370 case IORING_OP_RECVMSG:5092 case IORING_OP_RECVMSG:
4371 case IORING_OP_RECV:5093 case IORING_OP_RECV:
@@ -4375,9 +5097,9 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 3 unmodified lines
4375 break;5097 break;
4376 }5098 }
4377 if (req->opcode == IORING_OP_RECVMSG)5099 if (req->opcode == IORING_OP_RECVMSG)
4378 ret = io_recvmsg(req, nxt, force_nonblock);5100 ret = io_recvmsg(req, force_nonblock);
4379 else5101 else
4380 ret = io_recv(req, nxt, force_nonblock);5102 ret = io_recv(req, force_nonblock);
4381 break;5103 break;
4382 case IORING_OP_TIMEOUT:5104 case IORING_OP_TIMEOUT:
4383 if (sqe) {5105 if (sqe) {
@@ -4401,7 +5123,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 17 unmodified lines
4401 if (ret)5123 if (ret)
4402 break;5124 break;
4403 }5125 }
4404 ret = io_accept(req, nxt, force_nonblock);5126 ret = io_accept(req, force_nonblock);
4405 break;5127 break;
4406 case IORING_OP_CONNECT:5128 case IORING_OP_CONNECT:
4407 if (sqe) {5129 if (sqe) {
@@ -4409,7 +5131,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 1 unmodified line
4409 if (ret)5131 if (ret)
4410 break;5132 break;
4411 }5133 }
4412 ret = io_connect(req, nxt, force_nonblock);5134 ret = io_connect(req, force_nonblock);
4413 break;5135 break;
4414 case IORING_OP_ASYNC_CANCEL:5136 case IORING_OP_ASYNC_CANCEL:
4415 if (sqe) {5137 if (sqe) {
@@ -4417,7 +5139,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 1 unmodified line
4417 if (ret)5139 if (ret)
4418 break;5140 break;
4419 }5141 }
4420 ret = io_async_cancel(req, nxt);5142 ret = io_async_cancel(req);
4421 break;5143 break;
4422 case IORING_OP_FALLOCATE:5144 case IORING_OP_FALLOCATE:
4423 if (sqe) {5145 if (sqe) {
@@ -4425,7 +5147,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 1 unmodified line
4425 if (ret)5147 if (ret)
4426 break;5148 break;
4427 }5149 }
4428 ret = io_fallocate(req, nxt, force_nonblock);5150 ret = io_fallocate(req, force_nonblock);
4429 break;5151 break;
4430 case IORING_OP_OPENAT:5152 case IORING_OP_OPENAT:
4431 if (sqe) {5153 if (sqe) {
@@ -4433,7 +5155,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 1 unmodified line
4433 if (ret)5155 if (ret)
4434 break;5156 break;
4435 }5157 }
4436 ret = io_openat(req, nxt, force_nonblock);5158 ret = io_openat(req, force_nonblock);
4437 break;5159 break;
4438 case IORING_OP_CLOSE:5160 case IORING_OP_CLOSE:
4439 if (sqe) {5161 if (sqe) {
@@ -4441,7 +5163,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 1 unmodified line
4441 if (ret)5163 if (ret)
4442 break;5164 break;
4443 }5165 }
4444 ret = io_close(req, nxt, force_nonblock);5166 ret = io_close(req, force_nonblock);
4445 break;5167 break;
4446 case IORING_OP_FILES_UPDATE:5168 case IORING_OP_FILES_UPDATE:
4447 if (sqe) {5169 if (sqe) {
@@ -4457,7 +5179,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 9 unmodified lines
4457 if (ret)5179 if (ret)
4458 break;5180 break;
4459 }5181 }
4460 ret = io_statx(req, nxt, force_nonblock);5182 ret = io_statx(req, force_nonblock);
4461 break;5183 break;
4462 case IORING_OP_FADVISE:5184 case IORING_OP_FADVISE:
4463 if (sqe) {5185 if (sqe) {
@@ -4465,7 +5187,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 1 unmodified line
4465 if (ret)5187 if (ret)
4466 break;5188 break;
4467 }5189 }
4468 ret = io_fadvise(req, nxt, force_nonblock);5190 ret = io_fadvise(req, force_nonblock);
4469 break;5191 break;
4470 case IORING_OP_MADVISE:5192 case IORING_OP_MADVISE:
4471 if (sqe) {5193 if (sqe) {
@@ -4473,7 +5195,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 1 unmodified line
4473 if (ret)5195 if (ret)
4474 break;5196 break;
4475 }5197 }
4476 ret = io_madvise(req, nxt, force_nonblock);5198 ret = io_madvise(req, force_nonblock);
4477 break;5199 break;
4478 case IORING_OP_OPENAT2:5200 case IORING_OP_OPENAT2:
4479 if (sqe) {5201 if (sqe) {
@@ -4481,7 +5203,7 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 1 unmodified line
4481 if (ret)5203 if (ret)
4482 break;5204 break;
4483 }5205 }
4484 ret = io_openat2(req, nxt, force_nonblock);5206 ret = io_openat2(req, force_nonblock);
4485 break;5207 break;
4486 case IORING_OP_EPOLL_CTL:5208 case IORING_OP_EPOLL_CTL:
4487 if (sqe) {5209 if (sqe) {
@@ -4489,7 +5211,31 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 1 unmodified line
4489 if (ret)5211 if (ret)
4490 break;5212 break;
4491 }5213 }
4492 ret = io_epoll_ctl(req, nxt, force_nonblock);5214 ret = io_epoll_ctl(req, force_nonblock);
5215 break;
5216 case IORING_OP_SPLICE:
5217 if (sqe) {
5218 ret = io_splice_prep(req, sqe);
5219 if (ret < 0)
5220 break;
5221 }
5222 ret = io_splice(req, force_nonblock);
5223 break;
5224 case IORING_OP_PROVIDE_BUFFERS:
5225 if (sqe) {
5226 ret = io_provide_buffers_prep(req, sqe);
5227 if (ret)
5228 break;
5229 }
5230 ret = io_provide_buffers(req, force_nonblock);
5231 break;
5232 case IORING_OP_REMOVE_BUFFERS:
5233 if (sqe) {
5234 ret = io_remove_buffers_prep(req, sqe);
5235 if (ret)
5236 break;
5237 }
5238 ret = io_remove_buffers(req, force_nonblock);
4493 break;5239 break;
4494 default:5240 default:
4495 ret = -EINVAL;5241 ret = -EINVAL;
@@ -4522,7 +5268,6 @@ static void io_wq_submit_work(struct io_wq_work **workptr) · 26 unmodified lines
4522{5268{
4523 struct io_wq_work *work = *workptr;5269 struct io_wq_work *work = *workptr;
4524 struct io_kiocb *req = container_of(work, struct io_kiocb, work);5270 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
4525 struct io_kiocb *nxt = NULL;
4526 int ret = 0;5271 int ret = 0;
4527 5272
4528 /* if NO_CANCEL is set, we must still run the work */5273 /* if NO_CANCEL is set, we must still run the work */
@@ -4532,9 +5277,8 @@ static void io_wq_submit_work(struct io_wq_work **workptr) · 3 unmodified lines
4532 }5277 }
4533 5278
4534 if (!ret) {5279 if (!ret) {
4535 req->in_async = true;
4536 do {5280 do {
4537 ret = io_issue_sqe(req, NULL, &nxt, false);5281 ret = io_issue_sqe(req, NULL, false);
4538 /*5282 /*
4539 * We can get EAGAIN for polled IO even though we're5283 * We can get EAGAIN for polled IO even though we're
4540 * forcing a sync submission from here, since we can't5284 * forcing a sync submission from here, since we can't
@@ -4546,18 +5290,13 @@ static void io_wq_submit_work(struct io_wq_work **workptr) · 5 unmodified lines
4546 } while (1);5290 } while (1);
4547 }5291 }
4548 5292
4549 /* drop submission reference */
4550 io_put_req(req);
4551
4552 if (ret) {5293 if (ret) {
4553 req_set_fail_links(req);5294 req_set_fail_links(req);
4554 io_cqring_add_event(req, ret);5295 io_cqring_add_event(req, ret);
4555 io_put_req(req);5296 io_put_req(req);
4556 }5297 }
4557 5298
4558 /* if a dependent link is ready, pass it back */5299 io_steal_work(req, workptr);
4559 if (!ret && nxt)
4560 io_wq_assign_next(workptr, nxt);
4561}5300}
4562 5301
4563static int io_req_needs_file(struct io_kiocb *req, int fd)5302static int io_req_needs_file(struct io_kiocb *req, int fd)
@@ -4578,41 +5317,52 @@ static inline struct file *io_file_from_index(struct io_ring_ctx *ctx, · 14 unmodified lines
4578 return table->files[index & IORING_FILE_TABLE_MASK];;5317 return table->files[index & IORING_FILE_TABLE_MASK];;
4579}5318}
4580 5319
4581static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req,5320static int io_file_get(struct io_submit_state *state, struct io_kiocb *req,
4582 const struct io_uring_sqe *sqe)5321 int fd, struct file **out_file, bool fixed)
4583{5322{
4584 struct io_ring_ctx *ctx = req->ctx;5323 struct io_ring_ctx *ctx = req->ctx;
4585 unsigned flags;5324 struct file *file;
4586 int fd;
4587
4588 flags = READ_ONCE(sqe->flags);
4589 fd = READ_ONCE(sqe->fd);
4590
4591 if (!io_req_needs_file(req, fd))
4592 return 0;
4593 5325
4594 if (flags & IOSQE_FIXED_FILE) {5326 if (fixed) {
4595 if (unlikely(!ctx->file_data ||5327 if (unlikely(!ctx->file_data ||
4596 (unsigned) fd >= ctx->nr_user_files))5328 (unsigned) fd >= ctx->nr_user_files))
4597 return -EBADF;5329 return -EBADF;
4598 fd = array_index_nospec(fd, ctx->nr_user_files);5330 fd = array_index_nospec(fd, ctx->nr_user_files);
4599 req->file = io_file_from_index(ctx, fd);5331 file = io_file_from_index(ctx, fd);
4600 if (!req->file)5332 if (!file)
4601 return -EBADF;5333 return -EBADF;
4602 req->flags |= REQ_F_FIXED_FILE;
4603 percpu_ref_get(&ctx->file_data->refs);5334 percpu_ref_get(&ctx->file_data->refs);
4604 } else {5335 } else {
4605 if (req->needs_fixed_file)
4606 return -EBADF;
4607 trace_io_uring_file_get(ctx, fd);5336 trace_io_uring_file_get(ctx, fd);
4608 req->file = io_file_get(state, fd);5337 file = __io_file_get(state, fd);
4609 if (unlikely(!req->file))5338 if (unlikely(!file))
4610 return -EBADF;5339 return -EBADF;
4611 }5340 }
4612 5341
5342 *out_file = file;
4613 return 0;5343 return 0;
4614}5344}
4615 5345
5346static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req,
5347 const struct io_uring_sqe *sqe)
5348{
5349 unsigned flags;
5350 int fd;
5351 bool fixed;
5352
5353 flags = READ_ONCE(sqe->flags);
5354 fd = READ_ONCE(sqe->fd);
5355
5356 if (!io_req_needs_file(req, fd))
5357 return 0;
5358
5359 fixed = (flags & IOSQE_FIXED_FILE);
5360 if (unlikely(!fixed && req->needs_fixed_file))
5361 return -EBADF;
5362
5363 return io_file_get(state, req, fd, &req->file, fixed);
5364}
5365
4616static int io_grab_files(struct io_kiocb *req)5366static int io_grab_files(struct io_kiocb *req)
4617{5367{
4618 int ret = -EBADF;5368 int ret = -EBADF;
@@ -4672,8 +5422,7 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) · 53 unmodified lines
4672 5422
4673 if (prev) {5423 if (prev) {
4674 req_set_fail_links(prev);5424 req_set_fail_links(prev);
4675 io_async_find_and_cancel(ctx, req, prev->user_data, NULL,5425 io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME);
4676 -ETIME);
4677 io_put_req(prev);5426 io_put_req(prev);
4678 } else {5427 } else {
4679 io_cqring_add_event(req, -ETIME);5428 io_cqring_add_event(req, -ETIME);
@@ -4710,6 +5459,9 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) · 30 unmodified lines
4710 5459
4711 if (!(req->flags & REQ_F_LINK))5460 if (!(req->flags & REQ_F_LINK))
4712 return NULL;5461 return NULL;
5462 /* for polled retry, if flag is set, we already went through here */
5463 if (req->flags & REQ_F_POLLED)
5464 return NULL;
4713 5465
4714 nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,5466 nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,
4715 link_list);5467 link_list);
@@ -4723,7 +5475,7 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) · 7 unmodified lines
4723static void __io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe)5475static void __io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4724{5476{
4725 struct io_kiocb *linked_timeout;5477 struct io_kiocb *linked_timeout;
4726 struct io_kiocb *nxt = NULL;5478 struct io_kiocb *nxt;
4727 const struct cred *old_creds = NULL;5479 const struct cred *old_creds = NULL;
4728 int ret;5480 int ret;
4729 5481
@@ -4739,7 +5491,7 @@ again: · 9 unmodified lines
4739 old_creds = override_creds(req->work.creds);5491 old_creds = override_creds(req->work.creds);
4740 }5492 }
4741 5493
4742 ret = io_issue_sqe(req, sqe, &nxt, true);5494 ret = io_issue_sqe(req, sqe, true);
4743 5495
4744 /*5496 /*
4745 * We async punt it if the file wasn't marked NOWAIT, or if the file5497 * We async punt it if the file wasn't marked NOWAIT, or if the file
@@ -4747,6 +5499,11 @@ again: · 1 unmodified line
4747 */5499 */
4748 if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) ||5500 if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) ||
4749 (req->flags & REQ_F_MUST_PUNT))) {5501 (req->flags & REQ_F_MUST_PUNT))) {
5502 if (io_arm_poll_handler(req)) {
5503 if (linked_timeout)
5504 io_queue_linked_timeout(linked_timeout);
5505 goto exit;
5506 }
4750punt:5507punt:
4751 if (io_op_defs[req->opcode].file_table) {5508 if (io_op_defs[req->opcode].file_table) {
4752 ret = io_grab_files(req);5509 ret = io_grab_files(req);
@@ -4759,10 +5516,11 @@ punt: · 6 unmodified lines
4759 * submit reference when the iocb is actually submitted.5516 * submit reference when the iocb is actually submitted.
4760 */5517 */
4761 io_queue_async_work(req);5518 io_queue_async_work(req);
4762 goto done_req;5519 goto exit;
4763 }5520 }
4764 5521
4765err:5522err:
5523 nxt = NULL;
4766 /* drop submission reference */5524 /* drop submission reference */
4767 io_put_req_find_next(req, &nxt);5525 io_put_req_find_next(req, &nxt);
4768 5526
@@ -4779,15 +5537,14 @@ err: · 10 unmodified lines
4779 req_set_fail_links(req);5537 req_set_fail_links(req);
4780 io_put_req(req);5538 io_put_req(req);
4781 }5539 }
4782done_req:
4783 if (nxt) {5540 if (nxt) {
4784 req = nxt;5541 req = nxt;
4785 nxt = NULL;
4786 5542
4787 if (req->flags & REQ_F_FORCE_ASYNC)5543 if (req->flags & REQ_F_FORCE_ASYNC)
4788 goto punt;5544 goto punt;
4789 goto again;5545 goto again;
4790 }5546 }
5547exit:
4791 if (old_creds)5548 if (old_creds)
4792 revert_creds(old_creds);5549 revert_creds(old_creds);
4793}5550}
@@ -4829,7 +5586,8 @@ static inline void io_queue_link_head(struct io_kiocb *req) · 35 unmodified lines
4829}5586}
4830 5587
4831#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \5588#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
4832 IOSQE_IO_HARDLINK | IOSQE_ASYNC)5589 IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
5590 IOSQE_BUFFER_SELECT)
4833 5591
4834static bool io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,5592static bool io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
4835 struct io_submit_state *state, struct io_kiocb **link)5593 struct io_submit_state *state, struct io_kiocb **link)
@@ -4846,6 +5604,12 @@ static bool io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 10 unmodified lines
4846 goto err_req;5604 goto err_req;
4847 }5605 }
4848 5606
5607 if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
5608 !io_op_defs[req->opcode].buffer_select) {
5609 ret = -EOPNOTSUPP;
5610 goto err_req;
5611 }
5612
4849 id = READ_ONCE(sqe->personality);5613 id = READ_ONCE(sqe->personality);
4850 if (id) {5614 if (id) {
4851 req->work.creds = idr_find(&ctx->personality_idr, id);5615 req->work.creds = idr_find(&ctx->personality_idr, id);
@@ -4857,8 +5621,9 @@ static bool io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, · 5 unmodified lines
4857 }5621 }
4858 5622
4859 /* same numerical values with corresponding REQ_F_*, safe to copy */5623 /* same numerical values with corresponding REQ_F_*, safe to copy */
4860 req->flags |= sqe_flags & (IOSQE_IO_DRAIN|IOSQE_IO_HARDLINK|5624 req->flags |= sqe_flags & (IOSQE_IO_DRAIN | IOSQE_IO_HARDLINK |
4861 IOSQE_ASYNC);5625 IOSQE_ASYNC | IOSQE_FIXED_FILE |
5626 IOSQE_BUFFER_SELECT);
4862 5627
4863 ret = io_req_set_file(state, req, sqe);5628 ret = io_req_set_file(state, req, sqe);
4864 if (unlikely(ret)) {5629 if (unlikely(ret)) {
@@ -5079,7 +5844,6 @@ fail_req: · 214 unmodified lines
5079 *mm = ctx->sqo_mm;5844 *mm = ctx->sqo_mm;
5080 }5845 }
5081 5846
5082 req->in_async = async;
5083 req->needs_fixed_file = async;5847 req->needs_fixed_file = async;
5084 trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,5848 trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
5085 true, async);5849 true, async);
@@ -5163,6 +5927,8 @@ static int io_sq_thread(void *data) · 77 unmodified lines
5163 if (!list_empty(&ctx->poll_list) ||5927 if (!list_empty(&ctx->poll_list) ||
5164 (!time_after(jiffies, timeout) && ret != -EBUSY &&5928 (!time_after(jiffies, timeout) && ret != -EBUSY &&
5165 !percpu_ref_is_dying(&ctx->refs))) {5929 !percpu_ref_is_dying(&ctx->refs))) {
5930 if (current->task_works)
5931 task_work_run();
5166 cond_resched();5932 cond_resched();
5167 continue;5933 continue;
5168 }5934 }
@@ -5194,6 +5960,10 @@ static int io_sq_thread(void *data) · 25 unmodified lines
5194 finish_wait(&ctx->sqo_wait, &wait);5960 finish_wait(&ctx->sqo_wait, &wait);
5195 break;5961 break;
5196 }5962 }
5963 if (current->task_works) {
5964 task_work_run();
5965 continue;
5966 }
5197 if (signal_pending(current))5967 if (signal_pending(current))
5198 flush_signals(current);5968 flush_signals(current);
5199 schedule();5969 schedule();
@@ -5213,6 +5983,9 @@ static int io_sq_thread(void *data) · 13 unmodified lines
5213 timeout = jiffies + ctx->sq_thread_idle;5983 timeout = jiffies + ctx->sq_thread_idle;
5214 }5984 }
5215 5985
5986 if (current->task_works)
5987 task_work_run();
5988
5216 set_fs(old_fs);5989 set_fs(old_fs);
5217 if (cur_mm) {5990 if (cur_mm) {
5218 unuse_mm(cur_mm);5991 unuse_mm(cur_mm);
@@ -5277,8 +6050,13 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 58 unmodified lines
5277 struct io_rings *rings = ctx->rings;6050 struct io_rings *rings = ctx->rings;
5278 int ret = 0;6051 int ret = 0;
5279 6052
5280 if (io_cqring_events(ctx, false) >= min_events)6053 do {
5281 return 0;6054 if (io_cqring_events(ctx, false) >= min_events)
6055 return 0;
6056 if (!current->task_works)
6057 break;
6058 task_work_run();
6059 } while (1);
5282 6060
5283 if (sig) {6061 if (sig) {
5284#ifdef CONFIG_COMPAT6062#ifdef CONFIG_COMPAT
@@ -5298,6 +6076,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 13 unmodified lines
5298 do {6076 do {
5299 prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,6077 prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
5300 TASK_INTERRUPTIBLE);6078 TASK_INTERRUPTIBLE);
6079 if (current->task_works)
6080 task_work_run();
5301 if (io_should_wake(&iowq, false))6081 if (io_should_wake(&iowq, false))
5302 break;6082 break;
5303 schedule();6083 schedule();
@@ -5607,7 +6387,6 @@ static void io_ring_file_put(struct io_ring_ctx *ctx, struct file *file) · 303 unmodified lines
5607struct io_file_put {6387struct io_file_put {
5608 struct llist_node llist;6388 struct llist_node llist;
5609 struct file *file;6389 struct file *file;
5610 struct completion *done;
5611};6390};
5612 6391
5613static void io_ring_file_ref_flush(struct fixed_file_data *data)6392static void io_ring_file_ref_flush(struct fixed_file_data *data)
@@ -5618,10 +6397,7 @@ static void io_ring_file_ref_flush(struct fixed_file_data *data) · 4 unmodified lines
5618 while ((node = llist_del_all(&data->put_llist)) != NULL) {6397 while ((node = llist_del_all(&data->put_llist)) != NULL) {
5619 llist_for_each_entry_safe(pfile, tmp, node, llist) {6398 llist_for_each_entry_safe(pfile, tmp, node, llist) {
5620 io_ring_file_put(data->ctx, pfile->file);6399 io_ring_file_put(data->ctx, pfile->file);
5621 if (pfile->done)6400 kfree(pfile);
5622 complete(pfile->done);
5623 else
5624 kfree(pfile);
5625 }6401 }
5626 }6402 }
5627}6403}
@@ -5816,33 +6592,18 @@ static void io_atomic_switch(struct percpu_ref *ref) · 188 unmodified lines
5816 percpu_ref_get(&data->refs);6592 percpu_ref_get(&data->refs);
5817}6593}
5818 6594
5819static bool io_queue_file_removal(struct fixed_file_data *data,6595static int io_queue_file_removal(struct fixed_file_data *data,
5820 struct file *file)6596 struct file *file)
5821{6597{
5822 struct io_file_put *pfile, pfile_stack;6598 struct io_file_put *pfile;
5823 DECLARE_COMPLETION_ONSTACK(done);
5824 6599
5825 /*
5826 * If we fail allocating the struct we need for doing async reomval
5827 * of this file, just punt to sync and wait for it.
5828 */
5829 pfile = kzalloc(sizeof(*pfile), GFP_KERNEL);6600 pfile = kzalloc(sizeof(*pfile), GFP_KERNEL);
5830 if (!pfile) {6601 if (!pfile)
5831 pfile = &pfile_stack;6602 return -ENOMEM;
5832 pfile->done = &done;
5833 }
5834 6603
5835 pfile->file = file;6604 pfile->file = file;
5836 llist_add(&pfile->llist, &data->put_llist);6605 llist_add(&pfile->llist, &data->put_llist);
5837 6606 return 0;
5838 if (pfile == &pfile_stack) {
5839 percpu_ref_switch_to_atomic(&data->refs, io_atomic_switch);
5840 wait_for_completion(&done);
5841 flush_work(&data->ref_work);
5842 return false;
5843 }
5844
5845 return true;
5846}6607}
5847 6608
5848static int __io_sqe_files_update(struct io_ring_ctx *ctx,6609static int __io_sqe_files_update(struct io_ring_ctx *ctx,
@@ -5877,9 +6638,11 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, · 28 unmodified lines
5877 index = i & IORING_FILE_TABLE_MASK;6638 index = i & IORING_FILE_TABLE_MASK;
5878 if (table->files[index]) {6639 if (table->files[index]) {
5879 file = io_file_from_index(ctx, index);6640 file = io_file_from_index(ctx, index);
6641 err = io_queue_file_removal(data, file);
6642 if (err)
6643 break;
5880 table->files[index] = NULL;6644 table->files[index] = NULL;
5881 if (io_queue_file_removal(data, file))6645 ref_switch = true;
5882 ref_switch = true;
5883 }6646 }
5884 if (fd != -1) {6647 if (fd != -1) {
5885 file = fget(fd);6648 file = fget(fd);
@@ -5932,20 +6695,14 @@ static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg, · 46 unmodified lines
5932 return __io_sqe_files_update(ctx, &up, nr_args);6695 return __io_sqe_files_update(ctx, &up, nr_args);
5933}6696}
5934 6697
5935static void io_put_work(struct io_wq_work *work)6698static void io_free_work(struct io_wq_work *work)
5936{6699{
5937 struct io_kiocb *req = container_of(work, struct io_kiocb, work);6700 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
5938 6701
6702 /* Consider that io_steal_work() relies on this ref */
5939 io_put_req(req);6703 io_put_req(req);
5940}6704}
5941 6705
5942static void io_get_work(struct io_wq_work *work)
5943{
5944 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
5945
5946 refcount_inc(&req->refs);
5947}
5948
5949static int io_init_wq_offload(struct io_ring_ctx *ctx,6706static int io_init_wq_offload(struct io_ring_ctx *ctx,
5950 struct io_uring_params *p)6707 struct io_uring_params *p)
5951{6708{
@@ -5956,8 +6713,7 @@ static int io_init_wq_offload(struct io_ring_ctx *ctx, · 4 unmodified lines
5956 int ret = 0;6713 int ret = 0;
5957 6714
5958 data.user = ctx->user;6715 data.user = ctx->user;
5959 data.get_work = io_get_work;6716 data.free_work = io_free_work;
5960 data.put_work = io_put_work;
5961 6717
5962 if (!(p->flags & IORING_SETUP_ATTACH_WQ)) {6718 if (!(p->flags & IORING_SETUP_ATTACH_WQ)) {
5963 /* Do QD, or 4 * CPUS, whatever is smallest */6719 /* Do QD, or 4 * CPUS, whatever is smallest */
@@ -6359,6 +7115,21 @@ static int io_eventfd_unregister(struct io_ring_ctx *ctx) · 395 unmodified lines
6359 return -ENXIO;7115 return -ENXIO;
6360}7116}
6361 7117
7118static int __io_destroy_buffers(int id, void *p, void *data)
7119{
7120 struct io_ring_ctx *ctx = data;
7121 struct io_buffer *buf = p;
7122
7123 __io_remove_buffers(ctx, buf, id, -1U);
7124 return 0;
7125}
7126
7127static void io_destroy_buffers(struct io_ring_ctx *ctx)
7128{
7129 idr_for_each(&ctx->io_buffer_idr, __io_destroy_buffers, ctx);
7130 idr_destroy(&ctx->io_buffer_idr);
7131}
7132
6362static void io_ring_ctx_free(struct io_ring_ctx *ctx)7133static void io_ring_ctx_free(struct io_ring_ctx *ctx)
6363{7134{
6364 io_finish_async(ctx);7135 io_finish_async(ctx);
@@ -6369,6 +7140,7 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) · 4 unmodified lines
6369 io_sqe_buffer_unregister(ctx);7140 io_sqe_buffer_unregister(ctx);
6370 io_sqe_files_unregister(ctx);7141 io_sqe_files_unregister(ctx);
6371 io_eventfd_unregister(ctx);7142 io_eventfd_unregister(ctx);
7143 io_destroy_buffers(ctx);
6372 idr_destroy(&ctx->personality_idr);7144 idr_destroy(&ctx->personality_idr);
6373 7145
6374#if defined(CONFIG_UNIX)7146#if defined(CONFIG_UNIX)
@@ -6623,6 +7395,9 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 248 unmodified lines
6623 int submitted = 0;7395 int submitted = 0;
6624 struct fd f;7396 struct fd f;
6625 7397
7398 if (current->task_works)
7399 task_work_run();
7400
6626 if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))7401 if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
6627 return -EINVAL;7402 return -EINVAL;
6628 7403
@@ -6669,7 +7444,14 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 40 unmodified lines
6669 7444
6670 min_complete = min(min_complete, ctx->cq_entries);7445 min_complete = min(min_complete, ctx->cq_entries);
6671 7446
6672 if (ctx->flags & IORING_SETUP_IOPOLL) {7447 /*
7448 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
7449 * space applications don't need to do io completion events
7450 * polling again, they can rely on io_sq_thread to do polling
7451 * work, which can reduce cpu usage and uring_lock contention.
7452 */
7453 if (ctx->flags & IORING_SETUP_IOPOLL &&
7454 !(ctx->flags & IORING_SETUP_SQPOLL)) {
6673 ret = io_iopoll_check(ctx, &nr_events, min_complete);7455 ret = io_iopoll_check(ctx, &nr_events, min_complete);
6674 } else {7456 } else {
6675 ret = io_cqring_wait(ctx, min_complete, sig, sigsz);7457 ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
@@ -6745,6 +7527,17 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) · 69 unmodified lines
6745 seq_printf(m, "Personalities:\n");7527 seq_printf(m, "Personalities:\n");
6746 idr_for_each(&ctx->personality_idr, io_uring_show_cred, m);7528 idr_for_each(&ctx->personality_idr, io_uring_show_cred, m);
6747 }7529 }
7530 seq_printf(m, "PollList:\n");
7531 spin_lock_irq(&ctx->completion_lock);
7532 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
7533 struct hlist_head *list = &ctx->cancel_hash[i];
7534 struct io_kiocb *req;
7535
7536 hlist_for_each_entry(req, list, hash_node)
7537 seq_printf(m, " op=%d, task_works=%d\n", req->opcode,
7538 req->task->task_works != NULL);
7539 }
7540 spin_unlock_irq(&ctx->completion_lock);
6748 mutex_unlock(&ctx->uring_lock);7541 mutex_unlock(&ctx->uring_lock);
6749}7542}
6750 7543
@@ -6961,7 +7754,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p) · 210 unmodified lines
6961 7754
6962 p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |7755 p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
6963 IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |7756 IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
6964 IORING_FEAT_CUR_PERSONALITY;7757 IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL;
6965 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);7758 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
6966 return ret;7759 return ret;
6967err:7760err:
@@ -7239,6 +8032,7 @@ static int __init io_uring_init(void) · 271 unmodified lines
7239 BUILD_BUG_SQE_ELEM(8, __u64, off);8032 BUILD_BUG_SQE_ELEM(8, __u64, off);
7240 BUILD_BUG_SQE_ELEM(8, __u64, addr2);8033 BUILD_BUG_SQE_ELEM(8, __u64, addr2);
7241 BUILD_BUG_SQE_ELEM(16, __u64, addr);8034 BUILD_BUG_SQE_ELEM(16, __u64, addr);
8035 BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in);
7242 BUILD_BUG_SQE_ELEM(24, __u32, len);8036 BUILD_BUG_SQE_ELEM(24, __u32, len);
7243 BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags);8037 BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags);
7244 BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);8038 BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);
@@ -7253,11 +8047,14 @@ static int __init io_uring_init(void) · 8 unmodified lines
7253 BUILD_BUG_SQE_ELEM(28, __u32, open_flags);8047 BUILD_BUG_SQE_ELEM(28, __u32, open_flags);
7254 BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);8048 BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);
7255 BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);8049 BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);
8050 BUILD_BUG_SQE_ELEM(28, __u32, splice_flags);
7256 BUILD_BUG_SQE_ELEM(32, __u64, user_data);8051 BUILD_BUG_SQE_ELEM(32, __u64, user_data);
7257 BUILD_BUG_SQE_ELEM(40, __u16, buf_index);8052 BUILD_BUG_SQE_ELEM(40, __u16, buf_index);
7258 BUILD_BUG_SQE_ELEM(42, __u16, personality);8053 BUILD_BUG_SQE_ELEM(42, __u16, personality);
8054 BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in);
7259 8055
7260 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);8056 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
8057 BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int));
7261 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);8058 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
7262 return 0;8059 return 0;
7263};8060};
fs/splice.c+3−3
@@ -1109,9 +1109,9 @@ static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, · 1108 unmodified lines
1109/*1109/*
1110 * Determine where to splice to/from.1110 * Determine where to splice to/from.
1111 */1111 */
1112static long do_splice(struct file *in, loff_t __user *off_in,1112long do_splice(struct file *in, loff_t __user *off_in,
1113 struct file *out, loff_t __user *off_out,1113 struct file *out, loff_t __user *off_out,
1114 size_t len, unsigned int flags)1114 size_t len, unsigned int flags)
1115{1115{
1116 struct pipe_inode_info *ipipe;1116 struct pipe_inode_info *ipipe;
1117 struct pipe_inode_info *opipe;1117 struct pipe_inode_info *opipe;
include/linux/socket.h+4−0
@@ -391,6 +391,10 @@ extern int recvmsg_copy_msghdr(struct msghdr *msg, · 390 unmodified lines
391 struct user_msghdr __user *umsg, unsigned flags,391 struct user_msghdr __user *umsg, unsigned flags,
392 struct sockaddr __user **uaddr,392 struct sockaddr __user **uaddr,
393 struct iovec **iov);393 struct iovec **iov);
394extern int __copy_msghdr_from_user(struct msghdr *kmsg,
395 struct user_msghdr __user *umsg,
396 struct sockaddr __user **save_addr,
397 struct iovec __user **uiov, size_t *nsegs);
394 398
395/* helpers which do the actual work for syscalls */399/* helpers which do the actual work for syscalls */
396extern int __sys_recvfrom(int fd, void __user *ubuf, size_t size,400extern int __sys_recvfrom(int fd, void __user *ubuf, size_t size,
include/linux/splice.h+3−0
@@ -78,6 +78,9 @@ extern ssize_t add_to_pipe(struct pipe_inode_info *, · 77 unmodified lines
78 struct pipe_buffer *);78 struct pipe_buffer *);
79extern ssize_t splice_direct_to_actor(struct file *, struct splice_desc *,79extern ssize_t splice_direct_to_actor(struct file *, struct splice_desc *,
80 splice_direct_actor *);80 splice_direct_actor *);
81extern long do_splice(struct file *in, loff_t __user *off_in,
82 struct file *out, loff_t __user *off_out,
83 size_t len, unsigned int flags);
81 84
82/*85/*
83 * for dynamic pipe sizing86 * for dynamic pipe sizing
include/net/compat.h+3−0
@@ -38,6 +38,9 @@ struct compat_cmsghdr { · 37 unmodified lines
38#define compat_mmsghdr mmsghdr38#define compat_mmsghdr mmsghdr
39#endif /* defined(CONFIG_COMPAT) */39#endif /* defined(CONFIG_COMPAT) */
40 40
41int __get_compat_msghdr(struct msghdr *kmsg, struct compat_msghdr __user *umsg,
42 struct sockaddr __user **save_addr, compat_uptr_t *ptr,
43 compat_size_t *len);
41int get_compat_msghdr(struct msghdr *, struct compat_msghdr __user *,44int get_compat_msghdr(struct msghdr *, struct compat_msghdr __user *,
42 struct sockaddr __user **, struct iovec **);45 struct sockaddr __user **, struct iovec **);
43struct sock_fprog __user *get_compat_bpf_fprog(char __user *optval);46struct sock_fprog __user *get_compat_bpf_fprog(char __user *optval);
include/trace/events/io_uring.h+103−0
@@ -357,6 +357,109 @@ TRACE_EVENT(io_uring_submit_sqe, · 356 unmodified lines
357 __entry->force_nonblock, __entry->sq_thread)357 __entry->force_nonblock, __entry->sq_thread)
358);358);
359 359
360TRACE_EVENT(io_uring_poll_arm,
361
362 TP_PROTO(void *ctx, u8 opcode, u64 user_data, int mask, int events),
363
364 TP_ARGS(ctx, opcode, user_data, mask, events),
365
366 TP_STRUCT__entry (
367 __field( void *, ctx )
368 __field( u8, opcode )
369 __field( u64, user_data )
370 __field( int, mask )
371 __field( int, events )
372 ),
373
374 TP_fast_assign(
375 __entry->ctx = ctx;
376 __entry->opcode = opcode;
377 __entry->user_data = user_data;
378 __entry->mask = mask;
379 __entry->events = events;
380 ),
381
382 TP_printk("ring %p, op %d, data 0x%llx, mask 0x%x, events 0x%x",
383 __entry->ctx, __entry->opcode,
384 (unsigned long long) __entry->user_data,
385 __entry->mask, __entry->events)
386);
387
388TRACE_EVENT(io_uring_poll_wake,
389
390 TP_PROTO(void *ctx, u8 opcode, u64 user_data, int mask),
391
392 TP_ARGS(ctx, opcode, user_data, mask),
393
394 TP_STRUCT__entry (
395 __field( void *, ctx )
396 __field( u8, opcode )
397 __field( u64, user_data )
398 __field( int, mask )
399 ),
400
401 TP_fast_assign(
402 __entry->ctx = ctx;
403 __entry->opcode = opcode;
404 __entry->user_data = user_data;
405 __entry->mask = mask;
406 ),
407
408 TP_printk("ring %p, op %d, data 0x%llx, mask 0x%x",
409 __entry->ctx, __entry->opcode,
410 (unsigned long long) __entry->user_data,
411 __entry->mask)
412);
413
414TRACE_EVENT(io_uring_task_add,
415
416 TP_PROTO(void *ctx, u8 opcode, u64 user_data, int mask),
417
418 TP_ARGS(ctx, opcode, user_data, mask),
419
420 TP_STRUCT__entry (
421 __field( void *, ctx )
422 __field( u8, opcode )
423 __field( u64, user_data )
424 __field( int, mask )
425 ),
426
427 TP_fast_assign(
428 __entry->ctx = ctx;
429 __entry->opcode = opcode;
430 __entry->user_data = user_data;
431 __entry->mask = mask;
432 ),
433
434 TP_printk("ring %p, op %d, data 0x%llx, mask %x",
435 __entry->ctx, __entry->opcode,
436 (unsigned long long) __entry->user_data,
437 __entry->mask)
438);
439
440TRACE_EVENT(io_uring_task_run,
441
442 TP_PROTO(void *ctx, u8 opcode, u64 user_data),
443
444 TP_ARGS(ctx, opcode, user_data),
445
446 TP_STRUCT__entry (
447 __field( void *, ctx )
448 __field( u8, opcode )
449 __field( u64, user_data )
450 ),
451
452 TP_fast_assign(
453 __entry->ctx = ctx;
454 __entry->opcode = opcode;
455 __entry->user_data = user_data;
456 ),
457
458 TP_printk("ring %p, op %d, data 0x%llx",
459 __entry->ctx, __entry->opcode,
460 (unsigned long long) __entry->user_data)
461);
462
360#endif /* _TRACE_IO_URING_H */463#endif /* _TRACE_IO_URING_H */
361 464
362/* This part must be outside protection */465/* This part must be outside protection */
include/uapi/linux/io_uring.h+38−4
@@ -1,4 +1,4 @@
1/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */1/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */
2/*2/*
3 * Header file for the io_uring interface.3 * Header file for the io_uring interface.
4 *4 *
@@ -23,7 +23,10 @@ struct io_uring_sqe { · 18 unmodified lines
23 __u64 off; /* offset into file */23 __u64 off; /* offset into file */
24 __u64 addr2;24 __u64 addr2;
25 };25 };
26 __u64 addr; /* pointer to buffer or iovecs */26 union {
27 __u64 addr; /* pointer to buffer or iovecs */
28 __u64 splice_off_in;
29 };
27 __u32 len; /* buffer size or number of iovecs */30 __u32 len; /* buffer size or number of iovecs */
28 union {31 union {
29 __kernel_rwf_t rw_flags;32 __kernel_rwf_t rw_flags;
@@ -37,14 +40,21 @@ struct io_uring_sqe { · 7 unmodified lines
37 __u32 open_flags;40 __u32 open_flags;
38 __u32 statx_flags;41 __u32 statx_flags;
39 __u32 fadvise_advice;42 __u32 fadvise_advice;
43 __u32 splice_flags;
40 };44 };
41 __u64 user_data; /* data to be passed back at completion time */45 __u64 user_data; /* data to be passed back at completion time */
42 union {46 union {
43 struct {47 struct {
44 /* index into fixed buffers, if used */48 /* pack this to avoid bogus arm OABI complaints */
45 __u16 buf_index;49 union {
50 /* index into fixed buffers, if used */
51 __u16 buf_index;
52 /* for grouped buffer selection */
53 __u16 buf_group;
54 } __attribute__((packed));
46 /* personality to use, if used */55 /* personality to use, if used */
47 __u16 personality;56 __u16 personality;
57 __s32 splice_fd_in;
48 };58 };
49 __u64 __pad2[3];59 __u64 __pad2[3];
50 };60 };
@@ -56,6 +66,7 @@ enum { · 5 unmodified lines
56 IOSQE_IO_LINK_BIT,66 IOSQE_IO_LINK_BIT,
57 IOSQE_IO_HARDLINK_BIT,67 IOSQE_IO_HARDLINK_BIT,
58 IOSQE_ASYNC_BIT,68 IOSQE_ASYNC_BIT,
69 IOSQE_BUFFER_SELECT_BIT,
59};70};
60 71
61/*72/*
@@ -71,6 +82,8 @@ enum { · 9 unmodified lines
71#define IOSQE_IO_HARDLINK (1U << IOSQE_IO_HARDLINK_BIT)82#define IOSQE_IO_HARDLINK (1U << IOSQE_IO_HARDLINK_BIT)
72/* always go async */83/* always go async */
73#define IOSQE_ASYNC (1U << IOSQE_ASYNC_BIT)84#define IOSQE_ASYNC (1U << IOSQE_ASYNC_BIT)
85/* select buffer from sqe->buf_group */
86#define IOSQE_BUFFER_SELECT (1U << IOSQE_BUFFER_SELECT_BIT)
74 87
75/*88/*
76 * io_uring_setup() flags89 * io_uring_setup() flags
@@ -113,6 +126,9 @@ enum { · 36 unmodified lines
113 IORING_OP_RECV,126 IORING_OP_RECV,
114 IORING_OP_OPENAT2,127 IORING_OP_OPENAT2,
115 IORING_OP_EPOLL_CTL,128 IORING_OP_EPOLL_CTL,
129 IORING_OP_SPLICE,
130 IORING_OP_PROVIDE_BUFFERS,
131 IORING_OP_REMOVE_BUFFERS,
116 132
117 /* this goes last, obviously */133 /* this goes last, obviously */
118 IORING_OP_LAST,134 IORING_OP_LAST,
@@ -128,6 +144,12 @@ enum { · 9 unmodified lines
128 */144 */
129#define IORING_TIMEOUT_ABS (1U << 0)145#define IORING_TIMEOUT_ABS (1U << 0)
130 146
147/*
148 * sqe->splice_flags
149 * extends splice(2) flags
150 */
151#define SPLICE_F_FD_IN_FIXED (1U << 31) /* the last bit of __u32 */
152
131/*153/*
132 * IO completion data structure (Completion Queue Entry)154 * IO completion data structure (Completion Queue Entry)
133 */155 */
@@ -137,6 +159,17 @@ struct io_uring_cqe { · 3 unmodified lines
137 __u32 flags;159 __u32 flags;
138};160};
139 161
162/*
163 * cqe->flags
164 *
165 * IORING_CQE_F_BUFFER If set, the upper 16 bits are the buffer ID
166 */
167#define IORING_CQE_F_BUFFER (1U << 0)
168
169enum {
170 IORING_CQE_BUFFER_SHIFT = 16,
171};
172
140/*173/*
141 * Magic offsets for the application to mmap the data it needs174 * Magic offsets for the application to mmap the data it needs
142 */175 */
@@ -204,6 +237,7 @@ struct io_uring_params { · 61 unmodified lines
204#define IORING_FEAT_SUBMIT_STABLE (1U << 2)237#define IORING_FEAT_SUBMIT_STABLE (1U << 2)
205#define IORING_FEAT_RW_CUR_POS (1U << 3)238#define IORING_FEAT_RW_CUR_POS (1U << 3)
206#define IORING_FEAT_CUR_PERSONALITY (1U << 4)239#define IORING_FEAT_CUR_PERSONALITY (1U << 4)
240#define IORING_FEAT_FAST_POLL (1U << 5)
207 241
208/*242/*
209 * io_uring_register(2) opcodes and arguments243 * io_uring_register(2) opcodes and arguments
kernel/task_work.c+14−4
@@ -97,16 +97,26 @@ void task_work_run(void) · 96 unmodified lines
97 * work->func() can do task_work_add(), do not set97 * work->func() can do task_work_add(), do not set
98 * work_exited unless the list is empty.98 * work_exited unless the list is empty.
99 */99 */
100 raw_spin_lock_irq(&task->pi_lock);
101 do {100 do {
101 head = NULL;
102 work = READ_ONCE(task->task_works);102 work = READ_ONCE(task->task_works);
103 head = !work && (task->flags & PF_EXITING) ?103 if (!work) {
104 &work_exited : NULL;104 if (task->flags & PF_EXITING)
105 head = &work_exited;
106 else
107 break;
108 }
105 } while (cmpxchg(&task->task_works, work, head) != work);109 } while (cmpxchg(&task->task_works, work, head) != work);
106 raw_spin_unlock_irq(&task->pi_lock);
107 110
108 if (!work)111 if (!work)
109 break;112 break;
113 /*
114 * Synchronize with task_work_cancel(). It can not remove
115 * the first entry == work, cmpxchg(task_works) must fail.
116 * But it can remove another entry from the ->next list.
117 */
118 raw_spin_lock_irq(&task->pi_lock);
119 raw_spin_unlock_irq(&task->pi_lock);
110 120
111 do {121 do {
112 next = work->next;122 next = work->next;
net/compat.c+23−7
@@ -33,10 +33,10 @@ · 32 unmodified lines
33#include <linux/uaccess.h>33#include <linux/uaccess.h>
34#include <net/compat.h>34#include <net/compat.h>
35 35
36int get_compat_msghdr(struct msghdr *kmsg,36int __get_compat_msghdr(struct msghdr *kmsg,
37 struct compat_msghdr __user *umsg,37 struct compat_msghdr __user *umsg,
38 struct sockaddr __user **save_addr,38 struct sockaddr __user **save_addr,
39 struct iovec **iov)39 compat_uptr_t *ptr, compat_size_t *len)
40{40{
41 struct compat_msghdr msg;41 struct compat_msghdr msg;
42 ssize_t err;42 ssize_t err;
@@ -79,10 +79,26 @@ int get_compat_msghdr(struct msghdr *kmsg, · 36 unmodified lines
79 return -EMSGSIZE;79 return -EMSGSIZE;
80 80
81 kmsg->msg_iocb = NULL;81 kmsg->msg_iocb = NULL;
82 *ptr = msg.msg_iov;
83 *len = msg.msg_iovlen;
84 return 0;
85}
86
87int get_compat_msghdr(struct msghdr *kmsg,
88 struct compat_msghdr __user *umsg,
89 struct sockaddr __user **save_addr,
90 struct iovec **iov)
91{
92 compat_uptr_t ptr;
93 compat_size_t len;
94 ssize_t err;
95
96 err = __get_compat_msghdr(kmsg, umsg, save_addr, &ptr, &len);
97 if (err)
98 return err;
82 99
83 err = compat_import_iovec(save_addr ? READ : WRITE,100 err = compat_import_iovec(save_addr ? READ : WRITE, compat_ptr(ptr),
84 compat_ptr(msg.msg_iov), msg.msg_iovlen,101 len, UIO_FASTIOV, iov, &kmsg->msg_iter);
85 UIO_FASTIOV, iov, &kmsg->msg_iter);
86 return err < 0 ? err : 0;102 return err < 0 ? err : 0;
87}103}
88 104
net/socket.c+21−4
@@ -2228,10 +2228,10 @@ struct used_address { · 2227 unmodified lines
2228 unsigned int name_len;2228 unsigned int name_len;
2229};2229};
2230 2230
2231static int copy_msghdr_from_user(struct msghdr *kmsg,2231int __copy_msghdr_from_user(struct msghdr *kmsg,
2232 struct user_msghdr __user *umsg,2232 struct user_msghdr __user *umsg,
2233 struct sockaddr __user **save_addr,2233 struct sockaddr __user **save_addr,
2234 struct iovec **iov)2234 struct iovec __user **uiov, size_t *nsegs)
2235{2235{
2236 struct user_msghdr msg;2236 struct user_msghdr msg;
2237 ssize_t err;2237 ssize_t err;
@@ -2273,6 +2273,23 @@ static int copy_msghdr_from_user(struct msghdr *kmsg, · 35 unmodified lines
2273 return -EMSGSIZE;2273 return -EMSGSIZE;
2274 2274
2275 kmsg->msg_iocb = NULL;2275 kmsg->msg_iocb = NULL;
2276 *uiov = msg.msg_iov;
2277 *nsegs = msg.msg_iovlen;
2278 return 0;
2279}
2280
2281static int copy_msghdr_from_user(struct msghdr *kmsg,
2282 struct user_msghdr __user *umsg,
2283 struct sockaddr __user **save_addr,
2284 struct iovec **iov)
2285{
2286 struct user_msghdr msg;
2287 ssize_t err;
2288
2289 err = __copy_msghdr_from_user(kmsg, umsg, save_addr, &msg.msg_iov,
2290 &msg.msg_iovlen);
2291 if (err)
2292 return err;
2276 2293
2277 err = import_iovec(save_addr ? READ : WRITE,2294 err = import_iovec(save_addr ? READ : WRITE,
2278 msg.msg_iov, msg.msg_iovlen,2295 msg.msg_iov, msg.msg_iovlen,