History

io_uring: add timeout support for io_uring_enter()

Now users who want to get woken when waiting for events should submit a timeout command first. It is not safe for applications that split SQ and CQ handling between two threads, such as mysql. Users should synchronize the two threads explicitly to protect SQ and that will impact the performance. This patch adds support for timeout to existing io_uring_enter(). To avoid overloading arguments, it introduces a new parameter structure which contains sigmask and timeout. I have tested the workloads with one thread submiting nop requests while the other reaping the cqe with timeout. It shows 1.8~2x faster when the iodepth is 16.

Hao Xu committed 6 years ago

Commit c73ebb61 parent 27926b63 files · +72 −8Browse files

Unified · Split

3 files
  1. fs/io_uring.c +62 −7
  2. include/linux/syscalls.h +1 −1
  3. include/uapi/linux/io_uring.h +9 −0
fs/io_uring.c+62−7
@@ -7118,7 +7118,8 @@ static int io_run_task_work_sig(void) · 7117 unmodified lines
71187118 * application must reap them itself, as they reside on the shared cq ring.
71197119 */
71207120 static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
7121− const sigset_t __user *sig, size_t sigsz)
7121+ const sigset_t __user *sig, size_t sigsz,
7122+ struct __kernel_timespec __user *uts)
71227123 {
71237124 struct io_wait_queue iowq = {
71247125 .wq = {
@@ -7130,6 +7131,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 5 unmodified lines
71307131 .to_wait = min_events,
71317132 };
71327133 struct io_rings *rings = ctx->rings;
7134+ struct timespec64 ts;
7135+ signed long timeout = 0;
71337136 int ret = 0;
71347137
71357138 do {
@@ -7152,6 +7155,12 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 16 unmodified lines
71527155 return ret;
71537156 }
71547157
7158+ if (uts) {
7159+ if (get_timespec64(&ts, uts))
7160+ return -EFAULT;
7161+ timeout = timespec64_to_jiffies(&ts);
7162+ }
7163+
71557164 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
71567165 trace_io_uring_cqring_wait(ctx, min_events);
71577166 do {
@@ -7165,7 +7174,15 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · 7 unmodified lines
71657174 break;
71667175 if (io_should_wake(&iowq, false))
71677176 break;
7168− schedule();
7177+ if (uts) {
7178+ timeout = schedule_timeout(timeout);
7179+ if (timeout == 0) {
7180+ ret = -ETIME;
7181+ break;
7182+ }
7183+ } else {
7184+ schedule();
7185+ }
71697186 } while (1);
71707187 finish_wait(&ctx->wait, &iowq.wq);
71717188
@@ -9167,9 +9184,39 @@ static void io_sqpoll_wait_sq(struct io_ring_ctx *ctx) · 1995 unmodified lines
91679184 finish_wait(&ctx->sqo_sq_wait, &wait);
91689185 }
91699186
9187+static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz,
9188+ struct __kernel_timespec __user **ts,
9189+ const sigset_t __user **sig)
9190+{
9191+ struct io_uring_getevents_arg arg;
9192+
9193+ /*
9194+ * If EXT_ARG isn't set, then we have no timespec and the argp pointer
9195+ * is just a pointer to the sigset_t.
9196+ */
9197+ if (!(flags & IORING_ENTER_EXT_ARG)) {
9198+ *sig = (const sigset_t __user *) argp;
9199+ *ts = NULL;
9200+ return 0;
9201+ }
9202+
9203+ /*
9204+ * EXT_ARG is set - ensure we agree on the size of it and copy in our
9205+ * timespec and sigset_t pointers if good.
9206+ */
9207+ if (*argsz != sizeof(arg))
9208+ return -EINVAL;
9209+ if (copy_from_user(&arg, argp, sizeof(arg)))
9210+ return -EFAULT;
9211+ *sig = u64_to_user_ptr(arg.sigmask);
9212+ *argsz = arg.sigmask_sz;
9213+ *ts = u64_to_user_ptr(arg.ts);
9214+ return 0;
9215+}
9216+
91709217 SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
9171− u32, min_complete, u32, flags, const sigset_t __user *, sig,
9172− size_t, sigsz)
9218+ u32, min_complete, u32, flags, const void __user *, argp,
9219+ size_t, argsz)
91739220 {
91749221 struct io_ring_ctx *ctx;
91759222 long ret = -EBADF;
@@ -9179,7 +9226,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 3 unmodified lines
91799226 io_run_task_work();
91809227
91819228 if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
9182− IORING_ENTER_SQ_WAIT))
9229+ IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG))
91839230 return -EINVAL;
91849231
91859232 f = fdget(fd);
@@ -9225,6 +9272,13 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 39 unmodified lines
92259272 goto out;
92269273 }
92279274 if (flags & IORING_ENTER_GETEVENTS) {
9275+ const sigset_t __user *sig;
9276+ struct __kernel_timespec __user *ts;
9277+
9278+ ret = io_get_ext_arg(flags, argp, &argsz, &ts, &sig);
9279+ if (unlikely(ret))
9280+ goto out;
9281+
92289282 min_complete = min(min_complete, ctx->cq_entries);
92299283
92309284 /*
@@ -9237,7 +9291,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, · 6 unmodified lines
92379291 !(ctx->flags & IORING_SETUP_SQPOLL)) {
92389292 ret = io_iopoll_check(ctx, min_complete);
92399293 } else {
9240− ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
9294+ ret = io_cqring_wait(ctx, min_complete, sig, argsz, ts);
92419295 }
92429296 }
92439297
@@ -9600,7 +9654,8 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, · 356 unmodified lines
96009654 p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
96019655 IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
96029656 IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
9603− IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED;
9657+ IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED |
9658+ IORING_FEAT_EXT_ARG;
96049659
96059660 if (copy_to_user(params, p, sizeof(*p))) {
96069661 ret = -EFAULT;
include/linux/syscalls.h+1−1
@@ -317,7 +317,7 @@ asmlinkage long sys_io_uring_setup(u32 entries, · 316 unmodified lines
317317 struct io_uring_params __user *p);
318318 asmlinkage long sys_io_uring_enter(unsigned int fd, u32 to_submit,
319319 u32 min_complete, u32 flags,
320− const sigset_t __user *sig, size_t sigsz);
320+ const void __user *argp, size_t argsz);
321321 asmlinkage long sys_io_uring_register(unsigned int fd, unsigned int op,
322322 void __user *arg, unsigned int nr_args);
323323
include/uapi/linux/io_uring.h+9−0
@@ -231,6 +231,7 @@ struct io_cqring_offsets { · 230 unmodified lines
231231 #define IORING_ENTER_GETEVENTS (1U << 0)
232232 #define IORING_ENTER_SQ_WAKEUP (1U << 1)
233233 #define IORING_ENTER_SQ_WAIT (1U << 2)
234+#define IORING_ENTER_EXT_ARG (1U << 3)
234235
235236 /*
236237 * Passed in for io_uring_setup(2). Copied back with updated info on success
@@ -259,6 +260,7 @@ struct io_uring_params { · 22 unmodified lines
259260 #define IORING_FEAT_FAST_POLL (1U << 5)
260261 #define IORING_FEAT_POLL_32BITS (1U << 6)
261262 #define IORING_FEAT_SQPOLL_NONFIXED (1U << 7)
263+#define IORING_FEAT_EXT_ARG (1U << 8)
262264
263265 /*
264266 * io_uring_register(2) opcodes and arguments
@@ -335,4 +337,11 @@ enum { · 70 unmodified lines
335337 IORING_RESTRICTION_LAST
336338 };
337339
340+struct io_uring_getevents_arg {
341+ __u64 sigmask;
342+ __u32 sigmask_sz;
343+ __u32 pad;
344+ __u64 ts;
345+};
346+
338347 #endif