Pull io_uring fixes from Pavel. * 'io_uring/io_uring-5.19' of https://github.com/isilence/linux: io_uring: fix double unlock for pbuf select io_uring: kbuf: fix bug of not consuming ring buffer in partial io case io_uring: openclose: fix bug of closing wrong fixed file io_uring: fix not locked access to fixed buf table io_uring: fix races with buffer table unregister io_uring: fix races with file table unregister
Unified · Split
| @@ -1729,9 +1729,16 @@ static void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) · 1728 unmodified lines | ||
| 1729 | 1729 | |
| 1730 | 1730 | if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING))) |
| 1731 | 1731 | return; |
| 1732 | − /* don't recycle if we already did IO to this buffer */ | |
| 1733 | − if (req->flags & REQ_F_PARTIAL_IO) | |
| 1732 | + /* | |
| 1733 | + * For legacy provided buffer mode, don't recycle if we already did | |
| 1734 | + * IO to this buffer. For ring-mapped provided buffer mode, we should | |
| 1735 | + * increment ring->head to explicitly monopolize the buffer to avoid | |
| 1736 | + * multiple use. | |
| 1737 | + */ | |
| 1738 | + if ((req->flags & REQ_F_BUFFER_SELECTED) && | |
| 1739 | + (req->flags & REQ_F_PARTIAL_IO)) | |
| 1734 | 1740 | return; |
| 1741 | + | |
| 1735 | 1742 | /* |
| 1736 | 1743 | * We don't need to recycle for REQ_F_BUFFER_RING, we can just clear |
| 1737 | 1744 | * the flag and hence ensure that bl->head doesn't get incremented. |
| @@ -1739,8 +1746,13 @@ static void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) · 1 unmodified line | ||
| 1739 | 1746 | */ |
| 1740 | 1747 | if (req->flags & REQ_F_BUFFER_RING) { |
| 1741 | 1748 | if (req->buf_list) { |
| 1742 | − req->buf_index = req->buf_list->bgid; | |
| 1743 | − req->flags &= ~REQ_F_BUFFER_RING; | |
| 1749 | + if (req->flags & REQ_F_PARTIAL_IO) { | |
| 1750 | + req->buf_list->head++; | |
| 1751 | + req->buf_list = NULL; | |
| 1752 | + } else { | |
| 1753 | + req->buf_index = req->buf_list->bgid; | |
| 1754 | + req->flags &= ~REQ_F_BUFFER_RING; | |
| 1755 | + } | |
| 1744 | 1756 | } |
| 1745 | 1757 | return; |
| 1746 | 1758 | } |
| @@ -3677,6 +3689,20 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 1930 unmodified lines | ||
| 3677 | 3689 | int ret; |
| 3678 | 3690 | |
| 3679 | 3691 | kiocb->ki_pos = READ_ONCE(sqe->off); |
| 3692 | + /* used for fixed read/write too - just read unconditionally */ | |
| 3693 | + req->buf_index = READ_ONCE(sqe->buf_index); | |
| 3694 | + | |
| 3695 | + if (req->opcode == IORING_OP_READ_FIXED || | |
| 3696 | + req->opcode == IORING_OP_WRITE_FIXED) { | |
| 3697 | + struct io_ring_ctx *ctx = req->ctx; | |
| 3698 | + u16 index; | |
| 3699 | + | |
| 3700 | + if (unlikely(req->buf_index >= ctx->nr_user_bufs)) | |
| 3701 | + return -EFAULT; | |
| 3702 | + index = array_index_nospec(req->buf_index, ctx->nr_user_bufs); | |
| 3703 | + req->imu = ctx->user_bufs[index]; | |
| 3704 | + io_req_set_rsrc_node(req, ctx, 0); | |
| 3705 | + } | |
| 3680 | 3706 | |
| 3681 | 3707 | ioprio = READ_ONCE(sqe->ioprio); |
| 3682 | 3708 | if (ioprio) { |
| @@ -3689,12 +3715,9 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 6 unmodified lines | ||
| 3689 | 3715 | kiocb->ki_ioprio = get_current_ioprio(); |
| 3690 | 3716 | } |
| 3691 | 3717 | |
| 3692 | − req->imu = NULL; | |
| 3693 | 3718 | req->rw.addr = READ_ONCE(sqe->addr); |
| 3694 | 3719 | req->rw.len = READ_ONCE(sqe->len); |
| 3695 | 3720 | req->rw.flags = READ_ONCE(sqe->rw_flags); |
| 3696 | − /* used for fixed read/write too - just read unconditionally */ | |
| 3697 | − req->buf_index = READ_ONCE(sqe->buf_index); | |
| 3698 | 3721 | return 0; |
| 3699 | 3722 | } |
| 3700 | 3723 | |
| @@ -3826,20 +3849,9 @@ static int __io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter · 125 unmodified lines | ||
| 3826 | 3849 | static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter, |
| 3827 | 3850 | unsigned int issue_flags) |
| 3828 | 3851 | { |
| 3829 | − struct io_mapped_ubuf *imu = req->imu; | |
| 3830 | − u16 index, buf_index = req->buf_index; | |
| 3831 | − | |
| 3832 | − if (likely(!imu)) { | |
| 3833 | − struct io_ring_ctx *ctx = req->ctx; | |
| 3834 | − | |
| 3835 | − if (unlikely(buf_index >= ctx->nr_user_bufs)) | |
| 3836 | − return -EFAULT; | |
| 3837 | − io_req_set_rsrc_node(req, ctx, issue_flags); | |
| 3838 | − index = array_index_nospec(buf_index, ctx->nr_user_bufs); | |
| 3839 | − imu = READ_ONCE(ctx->user_bufs[index]); | |
| 3840 | − req->imu = imu; | |
| 3841 | − } | |
| 3842 | − return __io_import_fixed(req, rw, iter, imu); | |
| 3852 | + if (WARN_ON_ONCE(!req->imu)) | |
| 3853 | + return -EFAULT; | |
| 3854 | + return __io_import_fixed(req, rw, iter, req->imu); | |
| 3843 | 3855 | } |
| 3844 | 3856 | |
| 3845 | 3857 | static int io_buffer_add_list(struct io_ring_ctx *ctx, |
| @@ -3878,10 +3890,8 @@ static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len, · 32 unmodified lines | ||
| 3878 | 3890 | struct io_uring_buf *buf; |
| 3879 | 3891 | __u16 head = bl->head; |
| 3880 | 3892 | |
| 3881 | − if (unlikely(smp_load_acquire(&br->tail) == head)) { | |
| 3882 | − io_ring_submit_unlock(req->ctx, issue_flags); | |
| 3893 | + if (unlikely(smp_load_acquire(&br->tail) == head)) | |
| 3883 | 3894 | return NULL; |
| 3884 | − } | |
| 3885 | 3895 | |
| 3886 | 3896 | head &= bl->mask; |
| 3887 | 3897 | if (head < IO_BUFFER_LIST_BUF_PER_PAGE) { |
| @@ -8063,8 +8073,8 @@ static int io_files_update_with_index_alloc(struct io_kiocb *req, · 4175 unmodified lines | ||
| 8063 | 8073 | if (ret < 0) |
| 8064 | 8074 | break; |
| 8065 | 8075 | if (copy_to_user(&fds[done], &ret, sizeof(ret))) { |
| 8066 | − ret = -EFAULT; | |
| 8067 | 8076 | __io_close_fixed(req, issue_flags, ret); |
| 8077 | + ret = -EFAULT; | |
| 8068 | 8078 | break; |
| 8069 | 8079 | } |
| 8070 | 8080 | } |
| @@ -9788,11 +9798,19 @@ static void __io_sqe_files_unregister(struct io_ring_ctx *ctx) · 1717 unmodified lines | ||
| 9788 | 9798 | |
| 9789 | 9799 | static int io_sqe_files_unregister(struct io_ring_ctx *ctx) |
| 9790 | 9800 | { |
| 9801 | + unsigned nr = ctx->nr_user_files; | |
| 9791 | 9802 | int ret; |
| 9792 | 9803 | |
| 9793 | 9804 | if (!ctx->file_data) |
| 9794 | 9805 | return -ENXIO; |
| 9806 | + | |
| 9807 | + /* | |
| 9808 | + * Quiesce may unlock ->uring_lock, and while it's not held | |
| 9809 | + * prevent new requests using the table. | |
| 9810 | + */ | |
| 9811 | + ctx->nr_user_files = 0; | |
| 9795 | 9812 | ret = io_rsrc_ref_quiesce(ctx->file_data, ctx); |
| 9813 | + ctx->nr_user_files = nr; | |
| 9796 | 9814 | if (!ret) |
| 9797 | 9815 | __io_sqe_files_unregister(ctx); |
| 9798 | 9816 | return ret; |
| @@ -10690,12 +10708,19 @@ static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx) · 891 unmodified lines | ||
| 10690 | 10708 | |
| 10691 | 10709 | static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx) |
| 10692 | 10710 | { |
| 10711 | + unsigned nr = ctx->nr_user_bufs; | |
| 10693 | 10712 | int ret; |
| 10694 | 10713 | |
| 10695 | 10714 | if (!ctx->buf_data) |
| 10696 | 10715 | return -ENXIO; |
| 10697 | 10716 | |
| 10717 | + /* | |
| 10718 | + * Quiesce may unlock ->uring_lock, and while it's not held | |
| 10719 | + * prevent new requests using the table. | |
| 10720 | + */ | |
| 10721 | + ctx->nr_user_bufs = 0; | |
| 10698 | 10722 | ret = io_rsrc_ref_quiesce(ctx->buf_data, ctx); |
| 10723 | + ctx->nr_user_bufs = nr; | |
| 10699 | 10724 | if (!ret) |
| 10700 | 10725 | __io_sqe_buffers_unregister(ctx); |
| 10701 | 10726 | return ret; |