Pull io_uring updates from Jens Axboe: "Here are the main io_uring changes for 5.19. This contains: - Fixes for sparse type warnings (Christoph, Vasily) - Support for multi-shot accept (Hao) - Support for io_uring managed fixed files, rather than always needing the applicationt o manage the indices (me) - Fix for a spurious poll wakeup (Dylan) - CQE overflow fixes (Dylan) - Support more types of cancelations (me) - Support for co-operative task_work signaling, rather than always forcing an IPI (me) - Support for doing poll first when appropriate, rather than always attempting a transfer first (me) - Provided buffer cleanups and support for mapped buffers (me) - Improve how io_uring handles inflight SCM files (Pavel) - Speedups for registered files (Pavel, me) - Organize the completion data in a struct in io_kiocb rather than keep it in separate spots (Pavel) - task_work improvements (Pavel) - Cleanup and optimize the submission path, in general and for handling links (Pavel) - Speedups for registered resource handling (Pavel) - Support sparse buffers and file maps (Pavel, me) - Various fixes and cleanups (Almog, Pavel, me)" * tag 'for-5.19/io_uring-2022-05-22' of git://git.kernel.dk/linux-block: (111 commits) io_uring: fix incorrect __kernel_rwf_t cast io_uring: disallow mixed provided buffer group registrations io_uring: initialize io_buffer_list head when shared ring is unregistered io_uring: add fully sparse buffer registration io_uring: use rcu_dereference in io_close io_uring: consistently use the EPOLL* defines io_uring: make apoll_events a __poll_t io_uring: drop a spurious inline on a forward declaration io_uring: don't use ERR_PTR for user pointers io_uring: use a rwf_t for io_rw.flags io_uring: add support for ring mapped supplied buffers io_uring: add io_pin_pages() helper io_uring: add buffer selection support to IORING_OP_NOP io_uring: fix locking state for empty buffer group io_uring: implement multishot mode for accept io_uring: let fast poll support multishot io_uring: add REQ_F_APOLL_MULTISHOT for requests io_uring: add IORING_ACCEPT_MULTISHOT for accept io_uring: only wake when the correct events are set io_uring: avoid io-wq -EAGAIN looping for !IOPOLL ...
Showing the changes to include/uapi (1 of 8 files). Show all
Unified · Split
| @@ -63,6 +63,15 @@ struct io_uring_sqe { · 62 unmodified lines | |||
| 63 | __u64 __pad2[2]; | 63 | __u64 __pad2[2]; |
| 64 | }; | 64 | }; |
| 65 | 65 | ||
| 66 | /* | ||
| 67 | * If sqe->file_index is set to this for opcodes that instantiate a new | ||
| 68 | * direct descriptor (like openat/openat2/accept), then io_uring will allocate | ||
| 69 | * an available direct descriptor instead of having the application pass one | ||
| 70 | * in. The picked direct descriptor will be returned in cqe->res, or -ENFILE | ||
| 71 | * if the space is full. | ||
| 72 | */ | ||
| 73 | #define IORING_FILE_INDEX_ALLOC (~0U) | ||
| 74 | |||
| 66 | enum { | 75 | enum { |
| 67 | IOSQE_FIXED_FILE_BIT, | 76 | IOSQE_FIXED_FILE_BIT, |
| 68 | IOSQE_IO_DRAIN_BIT, | 77 | IOSQE_IO_DRAIN_BIT, |
| @@ -102,6 +111,20 @@ enum { · 33 unmodified lines | |||
| 102 | #define IORING_SETUP_ATTACH_WQ (1U << 5) /* attach to existing wq */ | 111 | #define IORING_SETUP_ATTACH_WQ (1U << 5) /* attach to existing wq */ |
| 103 | #define IORING_SETUP_R_DISABLED (1U << 6) /* start with ring disabled */ | 112 | #define IORING_SETUP_R_DISABLED (1U << 6) /* start with ring disabled */ |
| 104 | #define IORING_SETUP_SUBMIT_ALL (1U << 7) /* continue submit on error */ | 113 | #define IORING_SETUP_SUBMIT_ALL (1U << 7) /* continue submit on error */ |
| 114 | /* | ||
| 115 | * Cooperative task running. When requests complete, they often require | ||
| 116 | * forcing the submitter to transition to the kernel to complete. If this | ||
| 117 | * flag is set, work will be done when the task transitions anyway, rather | ||
| 118 | * than force an inter-processor interrupt reschedule. This avoids interrupting | ||
| 119 | * a task running in userspace, and saves an IPI. | ||
| 120 | */ | ||
| 121 | #define IORING_SETUP_COOP_TASKRUN (1U << 8) | ||
| 122 | /* | ||
| 123 | * If COOP_TASKRUN is set, get notified if task work is available for | ||
| 124 | * running and a kernel transition would be needed to run it. This sets | ||
| 125 | * IORING_SQ_TASKRUN in the sq ring flags. Not valid with COOP_TASKRUN. | ||
| 126 | */ | ||
| 127 | #define IORING_SETUP_TASKRUN_FLAG (1U << 9) | ||
| 105 | 128 | ||
| 106 | enum { | 129 | enum { |
| 107 | IORING_OP_NOP, | 130 | IORING_OP_NOP, |
| @@ -187,6 +210,33 @@ enum { · 79 unmodified lines | |||
| 187 | #define IORING_POLL_UPDATE_EVENTS (1U << 1) | 210 | #define IORING_POLL_UPDATE_EVENTS (1U << 1) |
| 188 | #define IORING_POLL_UPDATE_USER_DATA (1U << 2) | 211 | #define IORING_POLL_UPDATE_USER_DATA (1U << 2) |
| 189 | 212 | ||
| 213 | /* | ||
| 214 | * ASYNC_CANCEL flags. | ||
| 215 | * | ||
| 216 | * IORING_ASYNC_CANCEL_ALL Cancel all requests that match the given key | ||
| 217 | * IORING_ASYNC_CANCEL_FD Key off 'fd' for cancelation rather than the | ||
| 218 | * request 'user_data' | ||
| 219 | * IORING_ASYNC_CANCEL_ANY Match any request | ||
| 220 | */ | ||
| 221 | #define IORING_ASYNC_CANCEL_ALL (1U << 0) | ||
| 222 | #define IORING_ASYNC_CANCEL_FD (1U << 1) | ||
| 223 | #define IORING_ASYNC_CANCEL_ANY (1U << 2) | ||
| 224 | |||
| 225 | /* | ||
| 226 | * send/sendmsg and recv/recvmsg flags (sqe->addr2) | ||
| 227 | * | ||
| 228 | * IORING_RECVSEND_POLL_FIRST If set, instead of first attempting to send | ||
| 229 | * or receive and arm poll if that yields an | ||
| 230 | * -EAGAIN result, arm poll upfront and skip | ||
| 231 | * the initial transfer attempt. | ||
| 232 | */ | ||
| 233 | #define IORING_RECVSEND_POLL_FIRST (1U << 0) | ||
| 234 | |||
| 235 | /* | ||
| 236 | * accept flags stored in sqe->ioprio | ||
| 237 | */ | ||
| 238 | #define IORING_ACCEPT_MULTISHOT (1U << 0) | ||
| 239 | |||
| 190 | /* | 240 | /* |
| 191 | * IO completion data structure (Completion Queue Entry) | 241 | * IO completion data structure (Completion Queue Entry) |
| 192 | */ | 242 | */ |
| @@ -236,6 +286,7 @@ struct io_sqring_offsets { · 43 unmodified lines | |||
| 236 | */ | 286 | */ |
| 237 | #define IORING_SQ_NEED_WAKEUP (1U << 0) /* needs io_uring_enter wakeup */ | 287 | #define IORING_SQ_NEED_WAKEUP (1U << 0) /* needs io_uring_enter wakeup */ |
| 238 | #define IORING_SQ_CQ_OVERFLOW (1U << 1) /* CQ ring is overflown */ | 288 | #define IORING_SQ_CQ_OVERFLOW (1U << 1) /* CQ ring is overflown */ |
| 289 | #define IORING_SQ_TASKRUN (1U << 2) /* task should enter the kernel */ | ||
| 239 | 290 | ||
| 240 | struct io_cqring_offsets { | 291 | struct io_cqring_offsets { |
| 241 | __u32 head; | 292 | __u32 head; |
| @@ -333,6 +384,10 @@ enum { · 91 unmodified lines | |||
| 333 | IORING_REGISTER_RING_FDS = 20, | 384 | IORING_REGISTER_RING_FDS = 20, |
| 334 | IORING_UNREGISTER_RING_FDS = 21, | 385 | IORING_UNREGISTER_RING_FDS = 21, |
| 335 | 386 | ||
| 387 | /* register ring based provide buffer group */ | ||
| 388 | IORING_REGISTER_PBUF_RING = 22, | ||
| 389 | IORING_UNREGISTER_PBUF_RING = 23, | ||
| 390 | |||
| 336 | /* this goes last */ | 391 | /* this goes last */ |
| 337 | IORING_REGISTER_LAST | 392 | IORING_REGISTER_LAST |
| 338 | }; | 393 | }; |
| @@ -350,9 +405,15 @@ struct io_uring_files_update { · 11 unmodified lines | |||
| 350 | __aligned_u64 /* __s32 * */ fds; | 405 | __aligned_u64 /* __s32 * */ fds; |
| 351 | }; | 406 | }; |
| 352 | 407 | ||
| 408 | /* | ||
| 409 | * Register a fully sparse file space, rather than pass in an array of all | ||
| 410 | * -1 file descriptors. | ||
| 411 | */ | ||
| 412 | #define IORING_RSRC_REGISTER_SPARSE (1U << 0) | ||
| 413 | |||
| 353 | struct io_uring_rsrc_register { | 414 | struct io_uring_rsrc_register { |
| 354 | __u32 nr; | 415 | __u32 nr; |
| 355 | __u32 resv; | 416 | __u32 flags; |
| 356 | __u64 resv2; | 417 | __u64 resv2; |
| 357 | __aligned_u64 data; | 418 | __aligned_u64 data; |
| 358 | __aligned_u64 tags; | 419 | __aligned_u64 tags; |
| @@ -404,6 +465,38 @@ struct io_uring_restriction { · 45 unmodified lines | |||
| 404 | __u32 resv2[3]; | 465 | __u32 resv2[3]; |
| 405 | }; | 466 | }; |
| 406 | 467 | ||
| 468 | struct io_uring_buf { | ||
| 469 | __u64 addr; | ||
| 470 | __u32 len; | ||
| 471 | __u16 bid; | ||
| 472 | __u16 resv; | ||
| 473 | }; | ||
| 474 | |||
| 475 | struct io_uring_buf_ring { | ||
| 476 | union { | ||
| 477 | /* | ||
| 478 | * To avoid spilling into more pages than we need to, the | ||
| 479 | * ring tail is overlaid with the io_uring_buf->resv field. | ||
| 480 | */ | ||
| 481 | struct { | ||
| 482 | __u64 resv1; | ||
| 483 | __u32 resv2; | ||
| 484 | __u16 resv3; | ||
| 485 | __u16 tail; | ||
| 486 | }; | ||
| 487 | struct io_uring_buf bufs[0]; | ||
| 488 | }; | ||
| 489 | }; | ||
| 490 | |||
| 491 | /* argument for IORING_(UN)REGISTER_PBUF_RING */ | ||
| 492 | struct io_uring_buf_reg { | ||
| 493 | __u64 ring_addr; | ||
| 494 | __u32 ring_entries; | ||
| 495 | __u16 bgid; | ||
| 496 | __u16 pad; | ||
| 497 | __u64 resv[3]; | ||
| 498 | }; | ||
| 499 | |||
| 407 | /* | 500 | /* |
| 408 | * io_uring_restriction->opcode values | 501 | * io_uring_restriction->opcode values |
| 409 | */ | 502 | */ |