History

io_uring: add submission polling

This enables an application to do IO, without ever entering the kernel. By using the SQ ring to fill in new sqes and watching for completions on the CQ ring, we can submit and reap IOs without doing a single system call. The kernel side thread will poll for new submissions, and in case of HIPRI/polled IO, it'll also poll for completions. By default, we allow 1 second of active spinning. This can by changed by passing in a different grace period at io_uring_register(2) time. If the thread exceeds this idle time without having any work to do, it will set: sq_ring->flags |= IORING_SQ_NEED_WAKEUP. The application will have to call io_uring_enter() to start things back up again. If IO is kept busy, that will never be needed. Basically an application that has this feature enabled will guard it's io_uring_enter(2) call with: read_barrier(); if (*sq_ring->flags & IORING_SQ_NEED_WAKEUP) io_uring_enter(fd, 0, 0, IORING_ENTER_SQ_WAKEUP); instead of calling it unconditionally. It's mandatory to use fixed files with this feature. Failure to do so will result in the application getting an -EBADF CQ entry when submitting IO.

Jens Axboe committed 7 years ago

Commit 6c271ce1 parent 6b063142 files · +253 −8Browse files

Showing the changes to include/uapi/linux (1 of 2 files). Show all

Unified · Split

include/uapi/linux/io_uring.h+11−1
@@ -42,6 +42,8 @@ struct io_uring_sqe { · 41 unmodified lines
42 * io_uring_setup() flags42 * io_uring_setup() flags
43 */43 */
44#define IORING_SETUP_IOPOLL (1U << 0) /* io_context is polled */44#define IORING_SETUP_IOPOLL (1U << 0) /* io_context is polled */
45#define IORING_SETUP_SQPOLL (1U << 1) /* SQ poll thread */
46#define IORING_SETUP_SQ_AFF (1U << 2) /* sq_thread_cpu is valid */
45 47
46#define IORING_OP_NOP 048#define IORING_OP_NOP 0
47#define IORING_OP_READV 149#define IORING_OP_READV 1
@@ -86,6 +88,11 @@ struct io_sqring_offsets { · 38 unmodified lines
86 __u64 resv2;88 __u64 resv2;
87};89};
88 90
91/*
92 * sq_ring->flags
93 */
94#define IORING_SQ_NEED_WAKEUP (1U << 0) /* needs io_uring_enter wakeup */
95
89struct io_cqring_offsets {96struct io_cqring_offsets {
90 __u32 head;97 __u32 head;
91 __u32 tail;98 __u32 tail;
@@ -100,6 +107,7 @@ struct io_cqring_offsets { · 8 unmodified lines
100 * io_uring_enter(2) flags107 * io_uring_enter(2) flags
101 */108 */
102#define IORING_ENTER_GETEVENTS (1U << 0)109#define IORING_ENTER_GETEVENTS (1U << 0)
110#define IORING_ENTER_SQ_WAKEUP (1U << 1)
103 111
104/*112/*
105 * Passed in for io_uring_setup(2). Copied back with updated info on success113 * Passed in for io_uring_setup(2). Copied back with updated info on success
@@ -108,7 +116,9 @@ struct io_uring_params { · 2 unmodified lines
108 __u32 sq_entries;116 __u32 sq_entries;
109 __u32 cq_entries;117 __u32 cq_entries;
110 __u32 flags;118 __u32 flags;
111 __u32 resv[7];119 __u32 sq_thread_cpu;
120 __u32 sq_thread_idle;
121 __u32 resv[5];
112 struct io_sqring_offsets sq_off;122 struct io_sqring_offsets sq_off;
113 struct io_cqring_offsets cq_off;123 struct io_cqring_offsets cq_off;
114};124};