With IORING_REGISTER_USE_REGISTERED_RING, an application can register the ring fd and use it via registered index rather than installed fd. This allows using a registered ring for everything *except* the initial mmap. With IORING_SETUP_NO_MMAP, io_uring_setup uses buffers allocated by the user, rather than requiring a subsequent mmap. The combination of the two allows a user to operate *entirely* via a registered ring fd, making it unnecessary to ever install the fd in the first place. So, add a flag IORING_SETUP_REGISTERED_FD_ONLY to make io_uring_setup register the fd and return a registered index, without installing the fd. This allows an application to avoid touching the fd table at all, and allows a library to never even momentarily install a file descriptor. This splits out an io_ring_add_registered_file helper from io_ring_add_registered_fd, for use by io_uring_setup.
Unified · Split
| @@ -178,6 +178,13 @@ enum { · 177 unmodified lines | |||
| 178 | */ | 178 | */ |
| 179 | #define IORING_SETUP_NO_MMAP (1U << 14) | 179 | #define IORING_SETUP_NO_MMAP (1U << 14) |
| 180 | 180 | ||
| 181 | /* | ||
| 182 | * Register the ring fd in itself for use with | ||
| 183 | * IORING_REGISTER_USE_REGISTERED_RING; return a registered fd index rather | ||
| 184 | * than an fd. | ||
| 185 | */ | ||
| 186 | #define IORING_SETUP_REGISTERED_FD_ONLY (1U << 15) | ||
| 187 | |||
| 181 | enum io_uring_op { | 188 | enum io_uring_op { |
| 182 | IORING_OP_NOP, | 189 | IORING_OP_NOP, |
| 183 | IORING_OP_READV, | 190 | IORING_OP_READV, |
| @@ -3788,19 +3788,13 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, · 3787 unmodified lines | |||
| 3788 | return 0; | 3788 | return 0; |
| 3789 | } | 3789 | } |
| 3790 | 3790 | ||
| 3791 | static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file) | 3791 | static int io_uring_install_fd(struct file *file) |
| 3792 | { | 3792 | { |
| 3793 | int ret, fd; | 3793 | int fd; |
| 3794 | 3794 | ||
| 3795 | fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); | 3795 | fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); |
| 3796 | if (fd < 0) | 3796 | if (fd < 0) |
| 3797 | return fd; | 3797 | return fd; |
| 3798 | |||
| 3799 | ret = __io_uring_add_tctx_node(ctx); | ||
| 3800 | if (ret) { | ||
| 3801 | put_unused_fd(fd); | ||
| 3802 | return ret; | ||
| 3803 | } | ||
| 3804 | fd_install(fd, file); | 3798 | fd_install(fd, file); |
| 3805 | return fd; | 3799 | return fd; |
| 3806 | } | 3800 | } |
| @@ -3840,6 +3834,7 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, · 33 unmodified lines | |||
| 3840 | struct io_uring_params __user *params) | 3834 | struct io_uring_params __user *params) |
| 3841 | { | 3835 | { |
| 3842 | struct io_ring_ctx *ctx; | 3836 | struct io_ring_ctx *ctx; |
| 3837 | struct io_uring_task *tctx; | ||
| 3843 | struct file *file; | 3838 | struct file *file; |
| 3844 | int ret; | 3839 | int ret; |
| 3845 | 3840 | ||
| @@ -3851,6 +3846,10 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, · 5 unmodified lines | |||
| 3851 | entries = IORING_MAX_ENTRIES; | 3846 | entries = IORING_MAX_ENTRIES; |
| 3852 | } | 3847 | } |
| 3853 | 3848 | ||
| 3849 | if ((p->flags & IORING_SETUP_REGISTERED_FD_ONLY) | ||
| 3850 | && !(p->flags & IORING_SETUP_NO_MMAP)) | ||
| 3851 | return -EINVAL; | ||
| 3852 | |||
| 3854 | /* | 3853 | /* |
| 3855 | * Use twice as many entries for the CQ ring. It's possible for the | 3854 | * Use twice as many entries for the CQ ring. It's possible for the |
| 3856 | * application to drive a higher depth than the size of the SQ ring, | 3855 | * application to drive a higher depth than the size of the SQ ring, |
| @@ -4007,22 +4006,30 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, · 150 unmodified lines | |||
| 4007 | goto err; | 4006 | goto err; |
| 4008 | } | 4007 | } |
| 4009 | 4008 | ||
| 4009 | ret = __io_uring_add_tctx_node(ctx); | ||
| 4010 | if (ret) | ||
| 4011 | goto err_fput; | ||
| 4012 | tctx = current->io_uring; | ||
| 4013 | |||
| 4010 | /* | 4014 | /* |
| 4011 | * Install ring fd as the very last thing, so we don't risk someone | 4015 | * Install ring fd as the very last thing, so we don't risk someone |
| 4012 | * having closed it before we finish setup | 4016 | * having closed it before we finish setup |
| 4013 | */ | 4017 | */ |
| 4014 | ret = io_uring_install_fd(ctx, file); | 4018 | if (p->flags & IORING_SETUP_REGISTERED_FD_ONLY) |
| 4015 | if (ret < 0) { | 4019 | ret = io_ring_add_registered_file(tctx, file, 0, IO_RINGFD_REG_MAX); |
| 4016 | /* fput will clean it up */ | 4020 | else |
| 4017 | fput(file); | 4021 | ret = io_uring_install_fd(file); |
| 4018 | return ret; | 4022 | if (ret < 0) |
| 4019 | } | 4023 | goto err_fput; |
| 4020 | 4024 | ||
| 4021 | trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags); | 4025 | trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags); |
| 4022 | return ret; | 4026 | return ret; |
| 4023 | err: | 4027 | err: |
| 4024 | io_ring_ctx_wait_and_kill(ctx); | 4028 | io_ring_ctx_wait_and_kill(ctx); |
| 4025 | return ret; | 4029 | return ret; |
| 4030 | err_fput: | ||
| 4031 | fput(file); | ||
| 4032 | return ret; | ||
| 4026 | } | 4033 | } |
| 4027 | 4034 | ||
| 4028 | /* | 4035 | /* |
| @@ -4049,7 +4056,7 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) · 20 unmodified lines | |||
| 4049 | IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG | | 4056 | IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG | |
| 4050 | IORING_SETUP_SQE128 | IORING_SETUP_CQE32 | | 4057 | IORING_SETUP_SQE128 | IORING_SETUP_CQE32 | |
| 4051 | IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN | | 4058 | IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN | |
| 4052 | IORING_SETUP_NO_MMAP)) | 4059 | IORING_SETUP_NO_MMAP | IORING_SETUP_REGISTERED_FD_ONLY)) |
| 4053 | return -EINVAL; | 4060 | return -EINVAL; |
| 4054 | 4061 | ||
| 4055 | return io_uring_create(entries, &p, params); | 4062 | return io_uring_create(entries, &p, params); |
| @@ -75,6 +75,9 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd); · 74 unmodified lines | |||
| 75 | int io_uring_alloc_task_context(struct task_struct *task, | 75 | int io_uring_alloc_task_context(struct task_struct *task, |
| 76 | struct io_ring_ctx *ctx); | 76 | struct io_ring_ctx *ctx); |
| 77 | 77 | ||
| 78 | int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file, | ||
| 79 | int start, int end); | ||
| 80 | |||
| 78 | int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts); | 81 | int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts); |
| 79 | int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr); | 82 | int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr); |
| 80 | int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin); | 83 | int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin); |
| @@ -208,31 +208,40 @@ void io_uring_unreg_ringfd(void) · 207 unmodified lines | |||
| 208 | } | 208 | } |
| 209 | } | 209 | } |
| 210 | 210 | ||
| 211 | static int io_ring_add_registered_fd(struct io_uring_task *tctx, int fd, | 211 | int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file, |
| 212 | int start, int end) | 212 | int start, int end) |
| 213 | { | 213 | { |
| 214 | struct file *file; | ||
| 215 | int offset; | 214 | int offset; |
| 216 | |||
| 217 | for (offset = start; offset < end; offset++) { | 215 | for (offset = start; offset < end; offset++) { |
| 218 | offset = array_index_nospec(offset, IO_RINGFD_REG_MAX); | 216 | offset = array_index_nospec(offset, IO_RINGFD_REG_MAX); |
| 219 | if (tctx->registered_rings[offset]) | 217 | if (tctx->registered_rings[offset]) |
| 220 | continue; | 218 | continue; |
| 221 | 219 | ||
| 222 | file = fget(fd); | ||
| 223 | if (!file) { | ||
| 224 | return -EBADF; | ||
| 225 | } else if (!io_is_uring_fops(file)) { | ||
| 226 | fput(file); | ||
| 227 | return -EOPNOTSUPP; | ||
| 228 | } | ||
| 229 | tctx->registered_rings[offset] = file; | 220 | tctx->registered_rings[offset] = file; |
| 230 | return offset; | 221 | return offset; |
| 231 | } | 222 | } |
| 232 | |||
| 233 | return -EBUSY; | 223 | return -EBUSY; |
| 234 | } | 224 | } |
| 235 | 225 | ||
| 226 | static int io_ring_add_registered_fd(struct io_uring_task *tctx, int fd, | ||
| 227 | int start, int end) | ||
| 228 | { | ||
| 229 | struct file *file; | ||
| 230 | int offset; | ||
| 231 | |||
| 232 | file = fget(fd); | ||
| 233 | if (!file) { | ||
| 234 | return -EBADF; | ||
| 235 | } else if (!io_is_uring_fops(file)) { | ||
| 236 | fput(file); | ||
| 237 | return -EOPNOTSUPP; | ||
| 238 | } | ||
| 239 | offset = io_ring_add_registered_file(tctx, file, start, end); | ||
| 240 | if (offset < 0) | ||
| 241 | fput(file); | ||
| 242 | return offset; | ||
| 243 | } | ||
| 244 | |||
| 236 | /* | 245 | /* |
| 237 | * Register a ring fd to avoid fdget/fdput for each io_uring_enter() | 246 | * Register a ring fd to avoid fdget/fdput for each io_uring_enter() |
| 238 | * invocation. User passes in an array of struct io_uring_rsrc_update | 247 | * invocation. User passes in an array of struct io_uring_rsrc_update |