Pull io_uring zerocopy support from Jens Axboe: "This adds support for efficient support for zerocopy sends through io_uring. Both ipv4 and ipv6 is supported, as well as both TCP and UDP. The core network changes to support this is in a stable branch from Jakub that both io_uring and net-next has pulled in, and the io_uring changes are layered on top of that. All of the work has been done by Pavel" * tag 'for-5.20/io_uring-zerocopy-send-2022-07-29' of git://git.kernel.dk/linux-block: (34 commits) io_uring: notification completion optimisation io_uring: export req alloc from core io_uring/net: use unsigned for flags io_uring/net: make page accounting more consistent io_uring/net: checks errors of zc mem accounting io_uring/net: improve io_get_notif_slot types selftests/io_uring: test zerocopy send io_uring: enable managed frags with register buffers io_uring: add zc notification flush requests io_uring: rename IORING_OP_FILES_UPDATE io_uring: flush notifiers after sendzc io_uring: sendzc with fixed buffers io_uring: allow to pass addr into sendzc io_uring: account locked pages for non-fixed zc io_uring: wire send zc request type io_uring: add notification slot registration io_uring: add rsrc referencing for notifiers io_uring: complete notifiers in tw io_uring: cache struct io_notif io_uring: add zc notification infrastructure ...
Showing the changes to include/uapi (1 of 25 files). Show all
Unified · Split
| @@ -66,6 +66,10 @@ struct io_uring_sqe { · 65 unmodified lines | ||
| 66 | 66 | union { |
| 67 | 67 | __s32 splice_fd_in; |
| 68 | 68 | __u32 file_index; |
| 69 | + struct { | |
| 70 | + __u16 notification_idx; | |
| 71 | + __u16 addr_len; | |
| 72 | + }; | |
| 69 | 73 | }; |
| 70 | 74 | union { |
| 71 | 75 | struct { |
| @@ -170,7 +174,8 @@ enum io_uring_op { · 98 unmodified lines | ||
| 170 | 174 | IORING_OP_FALLOCATE, |
| 171 | 175 | IORING_OP_OPENAT, |
| 172 | 176 | IORING_OP_CLOSE, |
| 173 | − IORING_OP_FILES_UPDATE, | |
| 177 | + IORING_OP_RSRC_UPDATE, | |
| 178 | + IORING_OP_FILES_UPDATE = IORING_OP_RSRC_UPDATE, | |
| 174 | 179 | IORING_OP_STATX, |
| 175 | 180 | IORING_OP_READ, |
| 176 | 181 | IORING_OP_WRITE, |
| @@ -197,6 +202,7 @@ enum io_uring_op { · 20 unmodified lines | ||
| 197 | 202 | IORING_OP_GETXATTR, |
| 198 | 203 | IORING_OP_SOCKET, |
| 199 | 204 | IORING_OP_URING_CMD, |
| 205 | + IORING_OP_SENDZC_NOTIF, | |
| 200 | 206 | |
| 201 | 207 | /* this goes last, obviously */ |
| 202 | 208 | IORING_OP_LAST, |
| @@ -218,6 +224,7 @@ enum io_uring_op { · 15 unmodified lines | ||
| 218 | 224 | #define IORING_TIMEOUT_ETIME_SUCCESS (1U << 5) |
| 219 | 225 | #define IORING_TIMEOUT_CLOCK_MASK (IORING_TIMEOUT_BOOTTIME | IORING_TIMEOUT_REALTIME) |
| 220 | 226 | #define IORING_TIMEOUT_UPDATE_MASK (IORING_TIMEOUT_UPDATE | IORING_LINK_TIMEOUT_UPDATE) |
| 227 | + | |
| 221 | 228 | /* |
| 222 | 229 | * sqe->splice_flags |
| 223 | 230 | * extends splice(2) flags |
| @@ -267,15 +274,32 @@ enum io_uring_op { · 43 unmodified lines | ||
| 267 | 274 | * IORING_RECV_MULTISHOT Multishot recv. Sets IORING_CQE_F_MORE if |
| 268 | 275 | * the handler will continue to report |
| 269 | 276 | * CQEs on behalf of the same SQE. |
| 277 | + * | |
| 278 | + * IORING_RECVSEND_FIXED_BUF Use registered buffers, the index is stored in | |
| 279 | + * the buf_index field. | |
| 280 | + * | |
| 281 | + * IORING_RECVSEND_NOTIF_FLUSH Flush a notification after a successful | |
| 282 | + * successful. Only for zerocopy sends. | |
| 270 | 283 | */ |
| 271 | 284 | #define IORING_RECVSEND_POLL_FIRST (1U << 0) |
| 272 | −#define IORING_RECV_MULTISHOT (1U << 1) | |
| 285 | +#define IORING_RECV_MULTISHOT (1U << 1) | |
| 286 | +#define IORING_RECVSEND_FIXED_BUF (1U << 2) | |
| 287 | +#define IORING_RECVSEND_NOTIF_FLUSH (1U << 3) | |
| 273 | 288 | |
| 274 | 289 | /* |
| 275 | 290 | * accept flags stored in sqe->ioprio |
| 276 | 291 | */ |
| 277 | 292 | #define IORING_ACCEPT_MULTISHOT (1U << 0) |
| 278 | 293 | |
| 294 | + | |
| 295 | +/* | |
| 296 | + * IORING_OP_RSRC_UPDATE flags | |
| 297 | + */ | |
| 298 | +enum { | |
| 299 | + IORING_RSRC_UPDATE_FILES, | |
| 300 | + IORING_RSRC_UPDATE_NOTIF, | |
| 301 | +}; | |
| 302 | + | |
| 279 | 303 | /* |
| 280 | 304 | * IORING_OP_MSG_RING command types, stored in sqe->addr |
| 281 | 305 | */ |
| @@ -457,6 +481,10 @@ enum { · 175 unmodified lines | ||
| 457 | 481 | /* register a range of fixed file slots for automatic slot allocation */ |
| 458 | 482 | IORING_REGISTER_FILE_ALLOC_RANGE = 25, |
| 459 | 483 | |
| 484 | + /* zerocopy notification API */ | |
| 485 | + IORING_REGISTER_NOTIFIERS = 26, | |
| 486 | + IORING_UNREGISTER_NOTIFIERS = 27, | |
| 487 | + | |
| 460 | 488 | /* this goes last */ |
| 461 | 489 | IORING_REGISTER_LAST |
| 462 | 490 | }; |
| @@ -503,6 +531,19 @@ struct io_uring_rsrc_update2 { · 40 unmodified lines | ||
| 503 | 531 | __u32 resv2; |
| 504 | 532 | }; |
| 505 | 533 | |
| 534 | +struct io_uring_notification_slot { | |
| 535 | + __u64 tag; | |
| 536 | + __u64 resv[3]; | |
| 537 | +}; | |
| 538 | + | |
| 539 | +struct io_uring_notification_register { | |
| 540 | + __u32 nr_slots; | |
| 541 | + __u32 resv; | |
| 542 | + __u64 resv2; | |
| 543 | + __u64 data; | |
| 544 | + __u64 resv3; | |
| 545 | +}; | |
| 546 | + | |
| 506 | 547 | /* Skip updating fd indexes set to this value in the fd table */ |
| 507 | 548 | #define IORING_REGISTER_FILES_SKIP (-2) |
| 508 | 549 | |