io_uring: add submission polling

This enables an application to do IO, without ever entering the kernel. By using the SQ ring to fill in new sqes and watching for completions on the CQ ring, we can submit and reap IOs without doing a single system call. The kernel side thread will poll for new submissions, and in case of HIPRI/polled IO, it'll also poll for completions. By default, we allow 1 second of active spinning. This can by changed by passing in a different grace period at io_uring_register(2) time. If the thread exceeds this idle time without having any work to do, it will set: sq_ring->flags |= IORING_SQ_NEED_WAKEUP. The application will have to call io_uring_enter() to start things back up again. If IO is kept busy, that will never be needed. Basically an application that has this feature enabled will guard it's io_uring_enter(2) call with: read_barrier(); if (*sq_ring->flags & IORING_SQ_NEED_WAKEUP) io_uring_enter(fd, 0, 0, IORING_ENTER_SQ_WAKEUP); instead of calling it unconditionally. It's mandatory to use fixed files with this feature. Failure to do so will result in the application getting an -EBADF CQ entry when submitting IO.

Jens Axboe committed 7 years ago

Commit 6c271ce1 parent 6b063142 files · +253 −8Browse files

Showing the changes to include/uapi/linux (1 of 2 files). Show all

Unified · Split

include/uapi/linux/io_uring.h+11−1

@@ -42,6 +42,8 @@ struct io_uring_sqe { · 41 unmodified lines
42	42	* io_uring_setup() flags
43	43	*/
44	44	#define IORING_SETUP_IOPOLL (1U << 0) /* io_context is polled */
	45	+#define IORING_SETUP_SQPOLL (1U << 1) /* SQ poll thread */
	46	+#define IORING_SETUP_SQ_AFF (1U << 2) /* sq_thread_cpu is valid */
45	47
46	48	#define IORING_OP_NOP 0
47	49	#define IORING_OP_READV 1
@@ -86,6 +88,11 @@ struct io_sqring_offsets { · 38 unmodified lines
86	88	__u64 resv2;
87	89	};
88	90
	91	+/*
	92	+ * sq_ring->flags
	93	+ */
	94	+#define IORING_SQ_NEED_WAKEUP (1U << 0) /* needs io_uring_enter wakeup */
	95	+
89	96	struct io_cqring_offsets {
90	97	__u32 head;
91	98	__u32 tail;
@@ -100,6 +107,7 @@ struct io_cqring_offsets { · 8 unmodified lines
100	107	* io_uring_enter(2) flags
101	108	*/
102	109	#define IORING_ENTER_GETEVENTS (1U << 0)
	110	+#define IORING_ENTER_SQ_WAKEUP (1U << 1)
103	111
104	112	/*
105	113	* Passed in for io_uring_setup(2). Copied back with updated info on success
@@ -108,7 +116,9 @@ struct io_uring_params { · 2 unmodified lines
108	116	__u32 sq_entries;
109	117	__u32 cq_entries;
110	118	__u32 flags;
111		− __u32 resv[7];
	119	+ __u32 sq_thread_cpu;
	120	+ __u32 sq_thread_idle;
	121	+ __u32 resv[5];
112	122	struct io_sqring_offsets sq_off;
113	123	struct io_cqring_offsets cq_off;
114	124	};