History

io_uring: Add io_uring_setup flag to pre-register ring fd and never install it

With IORING_REGISTER_USE_REGISTERED_RING, an application can register the ring fd and use it via registered index rather than installed fd. This allows using a registered ring for everything *except* the initial mmap. With IORING_SETUP_NO_MMAP, io_uring_setup uses buffers allocated by the user, rather than requiring a subsequent mmap. The combination of the two allows a user to operate *entirely* via a registered ring fd, making it unnecessary to ever install the fd in the first place. So, add a flag IORING_SETUP_REGISTERED_FD_ONLY to make io_uring_setup register the fd and return a registered index, without installing the fd. This allows an application to avoid touching the fd table at all, and allows a library to never even momentarily install a file descriptor. This splits out an io_ring_add_registered_file helper from io_ring_add_registered_fd, for use by io_uring_setup.

Josh Triplett committed 3 years ago

Commit 6e76ac51 parent 03d89a24 files · +52 −26Browse files

Unified · Split

4 files
  1. include/uapi/linux/io_uring.h +7 −0
  2. io_uring/io_uring.c +22 −15
  3. io_uring/io_uring.h +3 −0
  4. io_uring/tctx.c +20 −11
include/uapi/linux/io_uring.h+7−0
@@ -178,6 +178,13 @@ enum { · 177 unmodified lines
178 */178 */
179#define IORING_SETUP_NO_MMAP (1U << 14)179#define IORING_SETUP_NO_MMAP (1U << 14)
180 180
181/*
182 * Register the ring fd in itself for use with
183 * IORING_REGISTER_USE_REGISTERED_RING; return a registered fd index rather
184 * than an fd.
185 */
186#define IORING_SETUP_REGISTERED_FD_ONLY (1U << 15)
187
181enum io_uring_op {188enum io_uring_op {
182 IORING_OP_NOP,189 IORING_OP_NOP,
183 IORING_OP_READV,190 IORING_OP_READV,
io_uring/io_uring.c+22−15
@@ -3788,19 +3788,13 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, · 3787 unmodified lines
3788 return 0;3788 return 0;
3789}3789}
3790 3790
3791static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file)3791static int io_uring_install_fd(struct file *file)
3792{3792{
3793 int ret, fd;3793 int fd;
3794 3794
3795 fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);3795 fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
3796 if (fd < 0)3796 if (fd < 0)
3797 return fd;3797 return fd;
3798
3799 ret = __io_uring_add_tctx_node(ctx);
3800 if (ret) {
3801 put_unused_fd(fd);
3802 return ret;
3803 }
3804 fd_install(fd, file);3798 fd_install(fd, file);
3805 return fd;3799 return fd;
3806}3800}
@@ -3840,6 +3834,7 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, · 33 unmodified lines
3840 struct io_uring_params __user *params)3834 struct io_uring_params __user *params)
3841{3835{
3842 struct io_ring_ctx *ctx;3836 struct io_ring_ctx *ctx;
3837 struct io_uring_task *tctx;
3843 struct file *file;3838 struct file *file;
3844 int ret;3839 int ret;
3845 3840
@@ -3851,6 +3846,10 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, · 5 unmodified lines
3851 entries = IORING_MAX_ENTRIES;3846 entries = IORING_MAX_ENTRIES;
3852 }3847 }
3853 3848
3849 if ((p->flags & IORING_SETUP_REGISTERED_FD_ONLY)
3850 && !(p->flags & IORING_SETUP_NO_MMAP))
3851 return -EINVAL;
3852
3854 /*3853 /*
3855 * Use twice as many entries for the CQ ring. It's possible for the3854 * Use twice as many entries for the CQ ring. It's possible for the
3856 * application to drive a higher depth than the size of the SQ ring,3855 * application to drive a higher depth than the size of the SQ ring,
@@ -4007,22 +4006,30 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, · 150 unmodified lines
4007 goto err;4006 goto err;
4008 }4007 }
4009 4008
4009 ret = __io_uring_add_tctx_node(ctx);
4010 if (ret)
4011 goto err_fput;
4012 tctx = current->io_uring;
4013
4010 /*4014 /*
4011 * Install ring fd as the very last thing, so we don't risk someone4015 * Install ring fd as the very last thing, so we don't risk someone
4012 * having closed it before we finish setup4016 * having closed it before we finish setup
4013 */4017 */
4014 ret = io_uring_install_fd(ctx, file);4018 if (p->flags & IORING_SETUP_REGISTERED_FD_ONLY)
4015 if (ret < 0) {4019 ret = io_ring_add_registered_file(tctx, file, 0, IO_RINGFD_REG_MAX);
4016 /* fput will clean it up */4020 else
4017 fput(file);4021 ret = io_uring_install_fd(file);
4018 return ret;4022 if (ret < 0)
4019 }4023 goto err_fput;
4020 4024
4021 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);4025 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
4022 return ret;4026 return ret;
4023err:4027err:
4024 io_ring_ctx_wait_and_kill(ctx);4028 io_ring_ctx_wait_and_kill(ctx);
4025 return ret;4029 return ret;
4030err_fput:
4031 fput(file);
4032 return ret;
4026}4033}
4027 4034
4028/*4035/*
@@ -4049,7 +4056,7 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) · 20 unmodified lines
4049 IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG |4056 IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG |
4050 IORING_SETUP_SQE128 | IORING_SETUP_CQE32 |4057 IORING_SETUP_SQE128 | IORING_SETUP_CQE32 |
4051 IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN |4058 IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN |
4052 IORING_SETUP_NO_MMAP))4059 IORING_SETUP_NO_MMAP | IORING_SETUP_REGISTERED_FD_ONLY))
4053 return -EINVAL;4060 return -EINVAL;
4054 4061
4055 return io_uring_create(entries, &p, params);4062 return io_uring_create(entries, &p, params);
io_uring/io_uring.h+3−0
@@ -75,6 +75,9 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd); · 74 unmodified lines
75int io_uring_alloc_task_context(struct task_struct *task,75int io_uring_alloc_task_context(struct task_struct *task,
76 struct io_ring_ctx *ctx);76 struct io_ring_ctx *ctx);
77 77
78int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file,
79 int start, int end);
80
78int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts);81int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts);
79int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr);82int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr);
80int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin);83int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin);
io_uring/tctx.c+20−11
@@ -208,31 +208,40 @@ void io_uring_unreg_ringfd(void) · 207 unmodified lines
208 }208 }
209}209}
210 210
211static int io_ring_add_registered_fd(struct io_uring_task *tctx, int fd,211int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file,
212 int start, int end)212 int start, int end)
213{213{
214 struct file *file;
215 int offset;214 int offset;
216
217 for (offset = start; offset < end; offset++) {215 for (offset = start; offset < end; offset++) {
218 offset = array_index_nospec(offset, IO_RINGFD_REG_MAX);216 offset = array_index_nospec(offset, IO_RINGFD_REG_MAX);
219 if (tctx->registered_rings[offset])217 if (tctx->registered_rings[offset])
220 continue;218 continue;
221 219
222 file = fget(fd);
223 if (!file) {
224 return -EBADF;
225 } else if (!io_is_uring_fops(file)) {
226 fput(file);
227 return -EOPNOTSUPP;
228 }
229 tctx->registered_rings[offset] = file;220 tctx->registered_rings[offset] = file;
230 return offset;221 return offset;
231 }222 }
232
233 return -EBUSY;223 return -EBUSY;
234}224}
235 225
226static int io_ring_add_registered_fd(struct io_uring_task *tctx, int fd,
227 int start, int end)
228{
229 struct file *file;
230 int offset;
231
232 file = fget(fd);
233 if (!file) {
234 return -EBADF;
235 } else if (!io_is_uring_fops(file)) {
236 fput(file);
237 return -EOPNOTSUPP;
238 }
239 offset = io_ring_add_registered_file(tctx, file, start, end);
240 if (offset < 0)
241 fput(file);
242 return offset;
243}
244
236/*245/*
237 * Register a ring fd to avoid fdget/fdput for each io_uring_enter()246 * Register a ring fd to avoid fdget/fdput for each io_uring_enter()
238 * invocation. User passes in an array of struct io_uring_rsrc_update247 * invocation. User passes in an array of struct io_uring_rsrc_update

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t