History

Merge branch 'io_uring/io_uring-5.19' of https://github.com/isilence/linux into io_uring-5.19

Pull io_uring fixes from Pavel. * 'io_uring/io_uring-5.19' of https://github.com/isilence/linux: io_uring: fix double unlock for pbuf select io_uring: kbuf: fix bug of not consuming ring buffer in partial io case io_uring: openclose: fix bug of closing wrong fixed file io_uring: fix not locked access to fixed buf table io_uring: fix races with buffer table unregister io_uring: fix races with file table unregister

Jens Axboe committed 4 years ago

Commit feaf6252 parents f9437ac fc9375e1 file · +50 −25Browse files

Unified · Split

fs/io_uring.c+50−25
@@ -1729,9 +1729,16 @@ static void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) · 1728 unmodified lines
1729 1729
1730 if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))1730 if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))
1731 return;1731 return;
1732 /* don't recycle if we already did IO to this buffer */1732 /*
1733 if (req->flags & REQ_F_PARTIAL_IO)1733 * For legacy provided buffer mode, don't recycle if we already did
1734 * IO to this buffer. For ring-mapped provided buffer mode, we should
1735 * increment ring->head to explicitly monopolize the buffer to avoid
1736 * multiple use.
1737 */
1738 if ((req->flags & REQ_F_BUFFER_SELECTED) &&
1739 (req->flags & REQ_F_PARTIAL_IO))
1734 return;1740 return;
1741
1735 /*1742 /*
1736 * We don't need to recycle for REQ_F_BUFFER_RING, we can just clear1743 * We don't need to recycle for REQ_F_BUFFER_RING, we can just clear
1737 * the flag and hence ensure that bl->head doesn't get incremented.1744 * the flag and hence ensure that bl->head doesn't get incremented.
@@ -1739,8 +1746,13 @@ static void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) · 1 unmodified line
1739 */1746 */
1740 if (req->flags & REQ_F_BUFFER_RING) {1747 if (req->flags & REQ_F_BUFFER_RING) {
1741 if (req->buf_list) {1748 if (req->buf_list) {
1742 req->buf_index = req->buf_list->bgid;1749 if (req->flags & REQ_F_PARTIAL_IO) {
1743 req->flags &= ~REQ_F_BUFFER_RING;1750 req->buf_list->head++;
1751 req->buf_list = NULL;
1752 } else {
1753 req->buf_index = req->buf_list->bgid;
1754 req->flags &= ~REQ_F_BUFFER_RING;
1755 }
1744 }1756 }
1745 return;1757 return;
1746 }1758 }
@@ -3677,6 +3689,20 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 1930 unmodified lines
3677 int ret;3689 int ret;
3678 3690
3679 kiocb->ki_pos = READ_ONCE(sqe->off);3691 kiocb->ki_pos = READ_ONCE(sqe->off);
3692 /* used for fixed read/write too - just read unconditionally */
3693 req->buf_index = READ_ONCE(sqe->buf_index);
3694
3695 if (req->opcode == IORING_OP_READ_FIXED ||
3696 req->opcode == IORING_OP_WRITE_FIXED) {
3697 struct io_ring_ctx *ctx = req->ctx;
3698 u16 index;
3699
3700 if (unlikely(req->buf_index >= ctx->nr_user_bufs))
3701 return -EFAULT;
3702 index = array_index_nospec(req->buf_index, ctx->nr_user_bufs);
3703 req->imu = ctx->user_bufs[index];
3704 io_req_set_rsrc_node(req, ctx, 0);
3705 }
3680 3706
3681 ioprio = READ_ONCE(sqe->ioprio);3707 ioprio = READ_ONCE(sqe->ioprio);
3682 if (ioprio) {3708 if (ioprio) {
@@ -3689,12 +3715,9 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 6 unmodified lines
3689 kiocb->ki_ioprio = get_current_ioprio();3715 kiocb->ki_ioprio = get_current_ioprio();
3690 }3716 }
3691 3717
3692 req->imu = NULL;
3693 req->rw.addr = READ_ONCE(sqe->addr);3718 req->rw.addr = READ_ONCE(sqe->addr);
3694 req->rw.len = READ_ONCE(sqe->len);3719 req->rw.len = READ_ONCE(sqe->len);
3695 req->rw.flags = READ_ONCE(sqe->rw_flags);3720 req->rw.flags = READ_ONCE(sqe->rw_flags);
3696 /* used for fixed read/write too - just read unconditionally */
3697 req->buf_index = READ_ONCE(sqe->buf_index);
3698 return 0;3721 return 0;
3699}3722}
3700 3723
@@ -3826,20 +3849,9 @@ static int __io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter · 125 unmodified lines
3826static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter,3849static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter,
3827 unsigned int issue_flags)3850 unsigned int issue_flags)
3828{3851{
3829 struct io_mapped_ubuf *imu = req->imu;3852 if (WARN_ON_ONCE(!req->imu))
3830 u16 index, buf_index = req->buf_index;3853 return -EFAULT;
3831 3854 return __io_import_fixed(req, rw, iter, req->imu);
3832 if (likely(!imu)) {
3833 struct io_ring_ctx *ctx = req->ctx;
3834
3835 if (unlikely(buf_index >= ctx->nr_user_bufs))
3836 return -EFAULT;
3837 io_req_set_rsrc_node(req, ctx, issue_flags);
3838 index = array_index_nospec(buf_index, ctx->nr_user_bufs);
3839 imu = READ_ONCE(ctx->user_bufs[index]);
3840 req->imu = imu;
3841 }
3842 return __io_import_fixed(req, rw, iter, imu);
3843}3855}
3844 3856
3845static int io_buffer_add_list(struct io_ring_ctx *ctx,3857static int io_buffer_add_list(struct io_ring_ctx *ctx,
@@ -3878,10 +3890,8 @@ static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len, · 32 unmodified lines
3878 struct io_uring_buf *buf;3890 struct io_uring_buf *buf;
3879 __u16 head = bl->head;3891 __u16 head = bl->head;
3880 3892
3881 if (unlikely(smp_load_acquire(&br->tail) == head)) {3893 if (unlikely(smp_load_acquire(&br->tail) == head))
3882 io_ring_submit_unlock(req->ctx, issue_flags);
3883 return NULL;3894 return NULL;
3884 }
3885 3895
3886 head &= bl->mask;3896 head &= bl->mask;
3887 if (head < IO_BUFFER_LIST_BUF_PER_PAGE) {3897 if (head < IO_BUFFER_LIST_BUF_PER_PAGE) {
@@ -8063,8 +8073,8 @@ static int io_files_update_with_index_alloc(struct io_kiocb *req, · 4175 unmodified lines
8063 if (ret < 0)8073 if (ret < 0)
8064 break;8074 break;
8065 if (copy_to_user(&fds[done], &ret, sizeof(ret))) {8075 if (copy_to_user(&fds[done], &ret, sizeof(ret))) {
8066 ret = -EFAULT;
8067 __io_close_fixed(req, issue_flags, ret);8076 __io_close_fixed(req, issue_flags, ret);
8077 ret = -EFAULT;
8068 break;8078 break;
8069 }8079 }
8070 }8080 }
@@ -9788,11 +9798,19 @@ static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) · 1717 unmodified lines
9788 9798
9789static int io_sqe_files_unregister(struct io_ring_ctx *ctx)9799static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
9790{9800{
9801 unsigned nr = ctx->nr_user_files;
9791 int ret;9802 int ret;
9792 9803
9793 if (!ctx->file_data)9804 if (!ctx->file_data)
9794 return -ENXIO;9805 return -ENXIO;
9806
9807 /*
9808 * Quiesce may unlock ->uring_lock, and while it's not held
9809 * prevent new requests using the table.
9810 */
9811 ctx->nr_user_files = 0;
9795 ret = io_rsrc_ref_quiesce(ctx->file_data, ctx);9812 ret = io_rsrc_ref_quiesce(ctx->file_data, ctx);
9813 ctx->nr_user_files = nr;
9796 if (!ret)9814 if (!ret)
9797 __io_sqe_files_unregister(ctx);9815 __io_sqe_files_unregister(ctx);
9798 return ret;9816 return ret;
@@ -10690,12 +10708,19 @@ static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx) · 891 unmodified lines
10690 10708
10691static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)10709static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
10692{10710{
10711 unsigned nr = ctx->nr_user_bufs;
10693 int ret;10712 int ret;
10694 10713
10695 if (!ctx->buf_data)10714 if (!ctx->buf_data)
10696 return -ENXIO;10715 return -ENXIO;
10697 10716
10717 /*
10718 * Quiesce may unlock ->uring_lock, and while it's not held
10719 * prevent new requests using the table.
10720 */
10721 ctx->nr_user_bufs = 0;
10698 ret = io_rsrc_ref_quiesce(ctx->buf_data, ctx);10722 ret = io_rsrc_ref_quiesce(ctx->buf_data, ctx);
10723 ctx->nr_user_bufs = nr;
10699 if (!ret)10724 if (!ret)
10700 __io_sqe_buffers_unregister(ctx);10725 __io_sqe_buffers_unregister(ctx);
10701 return ret;10726 return ret;