Pull io_uring updates from Jens Axboe: "Lots of cleanups in here, hardening the code and/or making it easier to read and fixing bugs, but a core feature/change too adding support for real async buffered reads. With the latter in place, we just need buffered write async support and we're done relying on kthreads for the fast path. In detail: - Cleanup how memory accounting is done on ring setup/free (Bijan) - sq array offset calculation fixup (Dmitry) - Consistently handle blocking off O_DIRECT submission path (me) - Support proper async buffered reads, instead of relying on kthread offload for that. This uses the page waitqueue to drive retries from task_work, like we handle poll based retry. (me) - IO completion optimizations (me) - Fix race with accounting and ring fd install (me) - Support EPOLLEXCLUSIVE (Jiufei) - Get rid of the io_kiocb unionizing, made possible by shrinking other bits (Pavel) - Completion side cleanups (Pavel) - Cleanup REQ_F_ flags handling, and kill off many of them (Pavel) - Request environment grabbing cleanups (Pavel) - File and socket read/write cleanups (Pavel) - Improve kiocb_set_rw_flags() (Pavel) - Tons of fixes and cleanups (Pavel) - IORING_SQ_NEED_WAKEUP clear fix (Xiaoguang)" * tag 'for-5.9/io_uring-20200802' of git://git.kernel.dk/linux-block: (127 commits) io_uring: flip if handling after io_setup_async_rw fs: optimise kiocb_set_rw_flags() io_uring: don't touch 'ctx' after installing file descriptor io_uring: get rid of atomic FAA for cq_timeouts io_uring: consolidate *_check_overflow accounting io_uring: fix stalled deferred requests io_uring: fix racy overflow count reporting io_uring: deduplicate __io_complete_rw() io_uring: de-unionise io_kiocb io-wq: update hash bits io_uring: fix missing io_queue_linked_timeout() io_uring: mark ->work uninitialised after cleanup io_uring: deduplicate io_grab_files() calls io_uring: don't do opcode prep twice io_uring: clear IORING_SQ_NEED_WAKEUP after executing task works io_uring: batch put_task_struct() tasks: add put_task_struct_many() io_uring: return locked and pinned page accounting io_uring: don't miscount pinned memory io_uring: don't open-code recv kbuf managment ...
Showing the changes to include/linux (4 of 14 files). Show all
Unified · Split
| @@ -1180,6 +1180,7 @@ struct blk_plug { · 1179 unmodified lines | ||
| 1180 | 1180 | struct list_head cb_list; /* md requires an unplug callback */ |
| 1181 | 1181 | unsigned short rq_count; |
| 1182 | 1182 | bool multiple_queues; |
| 1183 | + bool nowait; | |
| 1183 | 1184 | }; |
| 1184 | 1185 | #define BLK_MAX_REQUEST_COUNT 16 |
| 1185 | 1186 | #define BLK_PLUG_FLUSH_SIZE (128 * 1024) |
| @@ -175,6 +175,9 @@ typedef int (dio_iodone_t)(struct kiocb *iocb, loff_t offset, · 174 unmodified lines | ||
| 175 | 175 | /* File does not contribute to nr_files count */ |
| 176 | 176 | #define FMODE_NOACCOUNT ((__force fmode_t)0x20000000) |
| 177 | 177 | |
| 178 | +/* File supports async buffered reads */ | |
| 179 | +#define FMODE_BUF_RASYNC ((__force fmode_t)0x40000000) | |
| 180 | + | |
| 178 | 181 | /* |
| 179 | 182 | * Flag for rw_copy_check_uvector and compat_rw_copy_check_uvector |
| 180 | 183 | * that indicates that they should check the contents of the iovec are |
| @@ -315,6 +318,8 @@ enum rw_hint { · 134 unmodified lines | ||
| 315 | 318 | #define IOCB_SYNC (1 << 5) |
| 316 | 319 | #define IOCB_WRITE (1 << 6) |
| 317 | 320 | #define IOCB_NOWAIT (1 << 7) |
| 321 | +/* iocb->ki_waitq is valid */ | |
| 322 | +#define IOCB_WAITQ (1 << 8) | |
| 318 | 323 | #define IOCB_NOIO (1 << 9) |
| 319 | 324 | |
| 320 | 325 | struct kiocb { |
| @@ -329,7 +334,10 @@ struct kiocb { · 8 unmodified lines | ||
| 329 | 334 | int ki_flags; |
| 330 | 335 | u16 ki_hint; |
| 331 | 336 | u16 ki_ioprio; /* See linux/ioprio.h */ |
| 332 | − unsigned int ki_cookie; /* for ->iopoll */ | |
| 337 | + union { | |
| 338 | + unsigned int ki_cookie; /* for ->iopoll */ | |
| 339 | + struct wait_page_queue *ki_waitq; /* for async buffered IO */ | |
| 340 | + }; | |
| 333 | 341 | |
| 334 | 342 | randomized_struct_fields_end |
| 335 | 343 | }; |
| @@ -3275,22 +3283,28 @@ static inline int iocb_flags(struct file *file) · 2939 unmodified lines | ||
| 3275 | 3283 | |
| 3276 | 3284 | static inline int kiocb_set_rw_flags(struct kiocb *ki, rwf_t flags) |
| 3277 | 3285 | { |
| 3286 | + int kiocb_flags = 0; | |
| 3287 | + | |
| 3288 | + if (!flags) | |
| 3289 | + return 0; | |
| 3278 | 3290 | if (unlikely(flags & ~RWF_SUPPORTED)) |
| 3279 | 3291 | return -EOPNOTSUPP; |
| 3280 | 3292 | |
| 3281 | 3293 | if (flags & RWF_NOWAIT) { |
| 3282 | 3294 | if (!(ki->ki_filp->f_mode & FMODE_NOWAIT)) |
| 3283 | 3295 | return -EOPNOTSUPP; |
| 3284 | − ki->ki_flags |= IOCB_NOWAIT; | |
| 3296 | + kiocb_flags |= IOCB_NOWAIT; | |
| 3285 | 3297 | } |
| 3286 | 3298 | if (flags & RWF_HIPRI) |
| 3287 | − ki->ki_flags |= IOCB_HIPRI; | |
| 3299 | + kiocb_flags |= IOCB_HIPRI; | |
| 3288 | 3300 | if (flags & RWF_DSYNC) |
| 3289 | − ki->ki_flags |= IOCB_DSYNC; | |
| 3301 | + kiocb_flags |= IOCB_DSYNC; | |
| 3290 | 3302 | if (flags & RWF_SYNC) |
| 3291 | − ki->ki_flags |= (IOCB_DSYNC | IOCB_SYNC); | |
| 3303 | + kiocb_flags |= (IOCB_DSYNC | IOCB_SYNC); | |
| 3292 | 3304 | if (flags & RWF_APPEND) |
| 3293 | − ki->ki_flags |= IOCB_APPEND; | |
| 3305 | + kiocb_flags |= IOCB_APPEND; | |
| 3306 | + | |
| 3307 | + ki->ki_flags |= kiocb_flags; | |
| 3294 | 3308 | return 0; |
| 3295 | 3309 | } |
| 3296 | 3310 | |
| @@ -496,8 +496,35 @@ static inline pgoff_t linear_page_index(struct vm_area_struct *vma, · 495 unmodified lines | ||
| 496 | 496 | return pgoff; |
| 497 | 497 | } |
| 498 | 498 | |
| 499 | +/* This has the same layout as wait_bit_key - see fs/cachefiles/rdwr.c */ | |
| 500 | +struct wait_page_key { | |
| 501 | + struct page *page; | |
| 502 | + int bit_nr; | |
| 503 | + int page_match; | |
| 504 | +}; | |
| 505 | + | |
| 506 | +struct wait_page_queue { | |
| 507 | + struct page *page; | |
| 508 | + int bit_nr; | |
| 509 | + wait_queue_entry_t wait; | |
| 510 | +}; | |
| 511 | + | |
| 512 | +static inline bool wake_page_match(struct wait_page_queue *wait_page, | |
| 513 | + struct wait_page_key *key) | |
| 514 | +{ | |
| 515 | + if (wait_page->page != key->page) | |
| 516 | + return false; | |
| 517 | + key->page_match = 1; | |
| 518 | + | |
| 519 | + if (wait_page->bit_nr != key->bit_nr) | |
| 520 | + return false; | |
| 521 | + | |
| 522 | + return true; | |
| 523 | +} | |
| 524 | + | |
| 499 | 525 | extern void __lock_page(struct page *page); |
| 500 | 526 | extern int __lock_page_killable(struct page *page); |
| 527 | +extern int __lock_page_async(struct page *page, struct wait_page_queue *wait); | |
| 501 | 528 | extern int __lock_page_or_retry(struct page *page, struct mm_struct *mm, |
| 502 | 529 | unsigned int flags); |
| 503 | 530 | extern void unlock_page(struct page *page); |
| @@ -534,6 +561,22 @@ static inline int lock_page_killable(struct page *page) · 30 unmodified lines | ||
| 534 | 561 | return 0; |
| 535 | 562 | } |
| 536 | 563 | |
| 564 | +/* | |
| 565 | + * lock_page_async - Lock the page, unless this would block. If the page | |
| 566 | + * is already locked, then queue a callback when the page becomes unlocked. | |
| 567 | + * This callback can then retry the operation. | |
| 568 | + * | |
| 569 | + * Returns 0 if the page is locked successfully, or -EIOCBQUEUED if the page | |
| 570 | + * was already locked and the callback defined in 'wait' was queued. | |
| 571 | + */ | |
| 572 | +static inline int lock_page_async(struct page *page, | |
| 573 | + struct wait_page_queue *wait) | |
| 574 | +{ | |
| 575 | + if (!trylock_page(page)) | |
| 576 | + return __lock_page_async(page, wait); | |
| 577 | + return 0; | |
| 578 | +} | |
| 579 | + | |
| 537 | 580 | /* |
| 538 | 581 | * lock_page_or_retry - Lock the page, unless this would block and the |
| 539 | 582 | * caller indicated that it can handle a retry. |
| @@ -126,6 +126,12 @@ static inline void put_task_struct(struct task_struct *t) · 125 unmodified lines | ||
| 126 | 126 | __put_task_struct(t); |
| 127 | 127 | } |
| 128 | 128 | |
| 129 | +static inline void put_task_struct_many(struct task_struct *t, int nr) | |
| 130 | +{ | |
| 131 | + if (refcount_sub_and_test(nr, &t->usage)) | |
| 132 | + __put_task_struct(t); | |
| 133 | +} | |
| 134 | + | |
| 129 | 135 | void put_task_struct_rcu_user(struct task_struct *task); |
| 130 | 136 | |
| 131 | 137 | #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT |