History

Merge branch 'io_uring/io_uring-5.19' of https://github.com/isilence/linux into io_uring-5.19

Pull io_uring fixes from Pavel. * 'io_uring/io_uring-5.19' of https://github.com/isilence/linux: io_uring: fix double unlock for pbuf select io_uring: kbuf: fix bug of not consuming ring buffer in partial io case io_uring: openclose: fix bug of closing wrong fixed file io_uring: fix not locked access to fixed buf table io_uring: fix races with buffer table unregister io_uring: fix races with file table unregister

Jens Axboe committed 4 years ago

Commit feaf6252 parents f9437ac fc9375e1 file · +50 −25Browse files

Unified · Split

fs/io_uring.c+50−25
@@ -1729,9 +1729,16 @@ static void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) · 1728 unmodified lines
17291729
17301730 if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))
17311731 return;
1732− /* don't recycle if we already did IO to this buffer */
1733− if (req->flags & REQ_F_PARTIAL_IO)
1732+ /*
1733+ * For legacy provided buffer mode, don't recycle if we already did
1734+ * IO to this buffer. For ring-mapped provided buffer mode, we should
1735+ * increment ring->head to explicitly monopolize the buffer to avoid
1736+ * multiple use.
1737+ */
1738+ if ((req->flags & REQ_F_BUFFER_SELECTED) &&
1739+ (req->flags & REQ_F_PARTIAL_IO))
17341740 return;
1741+
17351742 /*
17361743 * We don't need to recycle for REQ_F_BUFFER_RING, we can just clear
17371744 * the flag and hence ensure that bl->head doesn't get incremented.
@@ -1739,8 +1746,13 @@ static void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) · 1 unmodified line
17391746 */
17401747 if (req->flags & REQ_F_BUFFER_RING) {
17411748 if (req->buf_list) {
1742− req->buf_index = req->buf_list->bgid;
1743− req->flags &= ~REQ_F_BUFFER_RING;
1749+ if (req->flags & REQ_F_PARTIAL_IO) {
1750+ req->buf_list->head++;
1751+ req->buf_list = NULL;
1752+ } else {
1753+ req->buf_index = req->buf_list->bgid;
1754+ req->flags &= ~REQ_F_BUFFER_RING;
1755+ }
17441756 }
17451757 return;
17461758 }
@@ -3677,6 +3689,20 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 1930 unmodified lines
36773689 int ret;
36783690
36793691 kiocb->ki_pos = READ_ONCE(sqe->off);
3692+ /* used for fixed read/write too - just read unconditionally */
3693+ req->buf_index = READ_ONCE(sqe->buf_index);
3694+
3695+ if (req->opcode == IORING_OP_READ_FIXED ||
3696+ req->opcode == IORING_OP_WRITE_FIXED) {
3697+ struct io_ring_ctx *ctx = req->ctx;
3698+ u16 index;
3699+
3700+ if (unlikely(req->buf_index >= ctx->nr_user_bufs))
3701+ return -EFAULT;
3702+ index = array_index_nospec(req->buf_index, ctx->nr_user_bufs);
3703+ req->imu = ctx->user_bufs[index];
3704+ io_req_set_rsrc_node(req, ctx, 0);
3705+ }
36803706
36813707 ioprio = READ_ONCE(sqe->ioprio);
36823708 if (ioprio) {
@@ -3689,12 +3715,9 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 6 unmodified lines
36893715 kiocb->ki_ioprio = get_current_ioprio();
36903716 }
36913717
3692− req->imu = NULL;
36933718 req->rw.addr = READ_ONCE(sqe->addr);
36943719 req->rw.len = READ_ONCE(sqe->len);
36953720 req->rw.flags = READ_ONCE(sqe->rw_flags);
3696− /* used for fixed read/write too - just read unconditionally */
3697− req->buf_index = READ_ONCE(sqe->buf_index);
36983721 return 0;
36993722 }
37003723
@@ -3826,20 +3849,9 @@ static int __io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter · 125 unmodified lines
38263849 static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter,
38273850 unsigned int issue_flags)
38283851 {
3829− struct io_mapped_ubuf *imu = req->imu;
3830− u16 index, buf_index = req->buf_index;
3831−
3832− if (likely(!imu)) {
3833− struct io_ring_ctx *ctx = req->ctx;
3834−
3835− if (unlikely(buf_index >= ctx->nr_user_bufs))
3836− return -EFAULT;
3837− io_req_set_rsrc_node(req, ctx, issue_flags);
3838− index = array_index_nospec(buf_index, ctx->nr_user_bufs);
3839− imu = READ_ONCE(ctx->user_bufs[index]);
3840− req->imu = imu;
3841− }
3842− return __io_import_fixed(req, rw, iter, imu);
3852+ if (WARN_ON_ONCE(!req->imu))
3853+ return -EFAULT;
3854+ return __io_import_fixed(req, rw, iter, req->imu);
38433855 }
38443856
38453857 static int io_buffer_add_list(struct io_ring_ctx *ctx,
@@ -3878,10 +3890,8 @@ static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len, · 32 unmodified lines
38783890 struct io_uring_buf *buf;
38793891 __u16 head = bl->head;
38803892
3881− if (unlikely(smp_load_acquire(&br->tail) == head)) {
3882− io_ring_submit_unlock(req->ctx, issue_flags);
3893+ if (unlikely(smp_load_acquire(&br->tail) == head))
38833894 return NULL;
3884− }
38853895
38863896 head &= bl->mask;
38873897 if (head < IO_BUFFER_LIST_BUF_PER_PAGE) {
@@ -8063,8 +8073,8 @@ static int io_files_update_with_index_alloc(struct io_kiocb *req, · 4175 unmodified lines
80638073 if (ret < 0)
80648074 break;
80658075 if (copy_to_user(&fds[done], &ret, sizeof(ret))) {
8066− ret = -EFAULT;
80678076 __io_close_fixed(req, issue_flags, ret);
8077+ ret = -EFAULT;
80688078 break;
80698079 }
80708080 }
@@ -9788,11 +9798,19 @@ static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) · 1717 unmodified lines
97889798
97899799 static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
97909800 {
9801+ unsigned nr = ctx->nr_user_files;
97919802 int ret;
97929803
97939804 if (!ctx->file_data)
97949805 return -ENXIO;
9806+
9807+ /*
9808+ * Quiesce may unlock ->uring_lock, and while it's not held
9809+ * prevent new requests using the table.
9810+ */
9811+ ctx->nr_user_files = 0;
97959812 ret = io_rsrc_ref_quiesce(ctx->file_data, ctx);
9813+ ctx->nr_user_files = nr;
97969814 if (!ret)
97979815 __io_sqe_files_unregister(ctx);
97989816 return ret;
@@ -10690,12 +10708,19 @@ static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx) · 891 unmodified lines
1069010708
1069110709 static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
1069210710 {
10711+ unsigned nr = ctx->nr_user_bufs;
1069310712 int ret;
1069410713
1069510714 if (!ctx->buf_data)
1069610715 return -ENXIO;
1069710716
10717+ /*
10718+ * Quiesce may unlock ->uring_lock, and while it's not held
10719+ * prevent new requests using the table.
10720+ */
10721+ ctx->nr_user_bufs = 0;
1069810722 ret = io_rsrc_ref_quiesce(ctx->buf_data, ctx);
10723+ ctx->nr_user_bufs = nr;
1069910724 if (!ret)
1070010725 __io_sqe_buffers_unregister(ctx);
1070110726 return ret;