History

Merge tag 'for-5.9/io_uring-20200802' of git://git.kernel.dk/linux-block

Pull io_uring updates from Jens Axboe: "Lots of cleanups in here, hardening the code and/or making it easier to read and fixing bugs, but a core feature/change too adding support for real async buffered reads. With the latter in place, we just need buffered write async support and we're done relying on kthreads for the fast path. In detail: - Cleanup how memory accounting is done on ring setup/free (Bijan) - sq array offset calculation fixup (Dmitry) - Consistently handle blocking off O_DIRECT submission path (me) - Support proper async buffered reads, instead of relying on kthread offload for that. This uses the page waitqueue to drive retries from task_work, like we handle poll based retry. (me) - IO completion optimizations (me) - Fix race with accounting and ring fd install (me) - Support EPOLLEXCLUSIVE (Jiufei) - Get rid of the io_kiocb unionizing, made possible by shrinking other bits (Pavel) - Completion side cleanups (Pavel) - Cleanup REQ_F_ flags handling, and kill off many of them (Pavel) - Request environment grabbing cleanups (Pavel) - File and socket read/write cleanups (Pavel) - Improve kiocb_set_rw_flags() (Pavel) - Tons of fixes and cleanups (Pavel) - IORING_SQ_NEED_WAKEUP clear fix (Xiaoguang)" * tag 'for-5.9/io_uring-20200802' of git://git.kernel.dk/linux-block: (127 commits) io_uring: flip if handling after io_setup_async_rw fs: optimise kiocb_set_rw_flags() io_uring: don't touch 'ctx' after installing file descriptor io_uring: get rid of atomic FAA for cq_timeouts io_uring: consolidate *_check_overflow accounting io_uring: fix stalled deferred requests io_uring: fix racy overflow count reporting io_uring: deduplicate __io_complete_rw() io_uring: de-unionise io_kiocb io-wq: update hash bits io_uring: fix missing io_queue_linked_timeout() io_uring: mark ->work uninitialised after cleanup io_uring: deduplicate io_grab_files() calls io_uring: don't do opcode prep twice io_uring: clear IORING_SQ_NEED_WAKEUP after executing task works io_uring: batch put_task_struct() tasks: add put_task_struct_many() io_uring: return locked and pinned page accounting io_uring: don't miscount pinned memory io_uring: don't open-code recv kbuf managment ...

Linus Torvalds committed 6 years ago

Commit cdc8fcb2 parents 382625d fa15baf14 files · +1644 −1181Browse files

Showing the changes to include/linux (4 of 14 files). Show all

Unified · Split

4 files
  1. include/linux/blkdev.h +1 −0
  2. include/linux/fs.h +20 −6
  3. include/linux/pagemap.h +43 −0
  4. include/linux/sched/task.h +6 −0
include/linux/blkdev.h+1−0
@@ -1180,6 +1180,7 @@ struct blk_plug { · 1179 unmodified lines
1180 struct list_head cb_list; /* md requires an unplug callback */1180 struct list_head cb_list; /* md requires an unplug callback */
1181 unsigned short rq_count;1181 unsigned short rq_count;
1182 bool multiple_queues;1182 bool multiple_queues;
1183 bool nowait;
1183};1184};
1184#define BLK_MAX_REQUEST_COUNT 161185#define BLK_MAX_REQUEST_COUNT 16
1185#define BLK_PLUG_FLUSH_SIZE (128 * 1024)1186#define BLK_PLUG_FLUSH_SIZE (128 * 1024)
include/linux/fs.h+20−6
@@ -175,6 +175,9 @@ typedef int (dio_iodone_t)(struct kiocb *iocb, loff_t offset, · 174 unmodified lines
175/* File does not contribute to nr_files count */175/* File does not contribute to nr_files count */
176#define FMODE_NOACCOUNT ((__force fmode_t)0x20000000)176#define FMODE_NOACCOUNT ((__force fmode_t)0x20000000)
177 177
178/* File supports async buffered reads */
179#define FMODE_BUF_RASYNC ((__force fmode_t)0x40000000)
180
178/*181/*
179 * Flag for rw_copy_check_uvector and compat_rw_copy_check_uvector182 * Flag for rw_copy_check_uvector and compat_rw_copy_check_uvector
180 * that indicates that they should check the contents of the iovec are183 * that indicates that they should check the contents of the iovec are
@@ -315,6 +318,8 @@ enum rw_hint { · 134 unmodified lines
315#define IOCB_SYNC (1 << 5)318#define IOCB_SYNC (1 << 5)
316#define IOCB_WRITE (1 << 6)319#define IOCB_WRITE (1 << 6)
317#define IOCB_NOWAIT (1 << 7)320#define IOCB_NOWAIT (1 << 7)
321/* iocb->ki_waitq is valid */
322#define IOCB_WAITQ (1 << 8)
318#define IOCB_NOIO (1 << 9)323#define IOCB_NOIO (1 << 9)
319 324
320struct kiocb {325struct kiocb {
@@ -329,7 +334,10 @@ struct kiocb { · 8 unmodified lines
329 int ki_flags;334 int ki_flags;
330 u16 ki_hint;335 u16 ki_hint;
331 u16 ki_ioprio; /* See linux/ioprio.h */336 u16 ki_ioprio; /* See linux/ioprio.h */
332 unsigned int ki_cookie; /* for ->iopoll */337 union {
338 unsigned int ki_cookie; /* for ->iopoll */
339 struct wait_page_queue *ki_waitq; /* for async buffered IO */
340 };
333 341
334 randomized_struct_fields_end342 randomized_struct_fields_end
335};343};
@@ -3275,22 +3283,28 @@ static inline int iocb_flags(struct file *file) · 2939 unmodified lines
3275 3283
3276static inline int kiocb_set_rw_flags(struct kiocb *ki, rwf_t flags)3284static inline int kiocb_set_rw_flags(struct kiocb *ki, rwf_t flags)
3277{3285{
3286 int kiocb_flags = 0;
3287
3288 if (!flags)
3289 return 0;
3278 if (unlikely(flags & ~RWF_SUPPORTED))3290 if (unlikely(flags & ~RWF_SUPPORTED))
3279 return -EOPNOTSUPP;3291 return -EOPNOTSUPP;
3280 3292
3281 if (flags & RWF_NOWAIT) {3293 if (flags & RWF_NOWAIT) {
3282 if (!(ki->ki_filp->f_mode & FMODE_NOWAIT))3294 if (!(ki->ki_filp->f_mode & FMODE_NOWAIT))
3283 return -EOPNOTSUPP;3295 return -EOPNOTSUPP;
3284 ki->ki_flags |= IOCB_NOWAIT;3296 kiocb_flags |= IOCB_NOWAIT;
3285 }3297 }
3286 if (flags & RWF_HIPRI)3298 if (flags & RWF_HIPRI)
3287 ki->ki_flags |= IOCB_HIPRI;3299 kiocb_flags |= IOCB_HIPRI;
3288 if (flags & RWF_DSYNC)3300 if (flags & RWF_DSYNC)
3289 ki->ki_flags |= IOCB_DSYNC;3301 kiocb_flags |= IOCB_DSYNC;
3290 if (flags & RWF_SYNC)3302 if (flags & RWF_SYNC)
3291 ki->ki_flags |= (IOCB_DSYNC | IOCB_SYNC);3303 kiocb_flags |= (IOCB_DSYNC | IOCB_SYNC);
3292 if (flags & RWF_APPEND)3304 if (flags & RWF_APPEND)
3293 ki->ki_flags |= IOCB_APPEND;3305 kiocb_flags |= IOCB_APPEND;
3306
3307 ki->ki_flags |= kiocb_flags;
3294 return 0;3308 return 0;
3295}3309}
3296 3310
include/linux/pagemap.h+43−0
@@ -496,8 +496,35 @@ static inline pgoff_t linear_page_index(struct vm_area_struct *vma, · 495 unmodified lines
496 return pgoff;496 return pgoff;
497}497}
498 498
499/* This has the same layout as wait_bit_key - see fs/cachefiles/rdwr.c */
500struct wait_page_key {
501 struct page *page;
502 int bit_nr;
503 int page_match;
504};
505
506struct wait_page_queue {
507 struct page *page;
508 int bit_nr;
509 wait_queue_entry_t wait;
510};
511
512static inline bool wake_page_match(struct wait_page_queue *wait_page,
513 struct wait_page_key *key)
514{
515 if (wait_page->page != key->page)
516 return false;
517 key->page_match = 1;
518
519 if (wait_page->bit_nr != key->bit_nr)
520 return false;
521
522 return true;
523}
524
499extern void __lock_page(struct page *page);525extern void __lock_page(struct page *page);
500extern int __lock_page_killable(struct page *page);526extern int __lock_page_killable(struct page *page);
527extern int __lock_page_async(struct page *page, struct wait_page_queue *wait);
501extern int __lock_page_or_retry(struct page *page, struct mm_struct *mm,528extern int __lock_page_or_retry(struct page *page, struct mm_struct *mm,
502 unsigned int flags);529 unsigned int flags);
503extern void unlock_page(struct page *page);530extern void unlock_page(struct page *page);
@@ -534,6 +561,22 @@ static inline int lock_page_killable(struct page *page) · 30 unmodified lines
534 return 0;561 return 0;
535}562}
536 563
564/*
565 * lock_page_async - Lock the page, unless this would block. If the page
566 * is already locked, then queue a callback when the page becomes unlocked.
567 * This callback can then retry the operation.
568 *
569 * Returns 0 if the page is locked successfully, or -EIOCBQUEUED if the page
570 * was already locked and the callback defined in 'wait' was queued.
571 */
572static inline int lock_page_async(struct page *page,
573 struct wait_page_queue *wait)
574{
575 if (!trylock_page(page))
576 return __lock_page_async(page, wait);
577 return 0;
578}
579
537/*580/*
538 * lock_page_or_retry - Lock the page, unless this would block and the581 * lock_page_or_retry - Lock the page, unless this would block and the
539 * caller indicated that it can handle a retry.582 * caller indicated that it can handle a retry.
include/linux/sched/task.h+6−0
@@ -126,6 +126,12 @@ static inline void put_task_struct(struct task_struct *t) · 125 unmodified lines
126 __put_task_struct(t);126 __put_task_struct(t);
127}127}
128 128
129static inline void put_task_struct_many(struct task_struct *t, int nr)
130{
131 if (refcount_sub_and_test(nr, &t->usage))
132 __put_task_struct(t);
133}
134
129void put_task_struct_rcu_user(struct task_struct *task);135void put_task_struct_rcu_user(struct task_struct *task);
130 136
131#ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT137#ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT