// SPDX-License-Identifier: GPL-2.0#include <linux/kernel.h>#include <linux/errno.h>#include <linux/fs.h>#include <linux/file.h>#include <linux/blk-mq.h>#include <linux/mm.h>#include <linux/slab.h>#include <linux/fsnotify.h>#include <linux/poll.h>#include <linux/nospec.h>#include <linux/io_uring/cmd.h>#include <linux/indirect_call_wrapper.h> #include <uapi/linux/io_uring.h> #include "filetable.h"#include "io_uring.h"#include "opdef.h"#include "kbuf.h"#include "alloc_cache.h"#include "rsrc.h"#include "poll.h"#include "rw.h" static void io_complete_rw(struct kiocb *kiocb, long res);static void io_complete_rw_iopoll(struct kiocb *kiocb, long res); struct io_rw { /* NOTE: kiocb has the file as the first member, so don't do it here */ struct kiocb kiocb; u64 addr; u32 len; rwf_t flags;}; static bool io_file_supports_nowait(struct io_kiocb *req, __poll_t mask){ /* If FMODE_NOWAIT is set for a file, we're golden */ if (req->flags & REQ_F_SUPPORT_NOWAIT) return true; /* No FMODE_NOWAIT, if we can poll, check the status */ if (io_file_can_poll(req)) { struct poll_table_struct pt = { ._key = mask }; return vfs_poll(req->file, &pt) & mask; } /* No FMODE_NOWAIT support, and file isn't pollable. Tough luck. */ return false;} static int io_iov_buffer_select_prep(struct io_kiocb *req){ struct iovec __user *uiov; struct iovec fast_iov, *iov; struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); if (rw->len != 1) return -EINVAL; uiov = u64_to_user_ptr(rw->addr); iov = iovec_from_user(uiov, 1, 1, &fast_iov, io_is_compat(req->ctx)); if (IS_ERR(iov)) return PTR_ERR(iov); rw->len = iov->iov_len; return 0;} static int io_import_vec(int ddir, struct io_kiocb *req, struct io_async_rw *io, const struct iovec __user *uvec, size_t uvec_segs){ int ret, nr_segs; struct iovec *iov; if (io->vec.iovec) { nr_segs = io->vec.nr; iov = io->vec.iovec; } else { nr_segs = 1; iov = &io->fast_iov; } ret = __import_iovec(ddir, uvec, uvec_segs, nr_segs, &iov, &io->iter, io_is_compat(req->ctx)); if (unlikely(ret < 0)) return ret; if (iov) { req->flags |= REQ_F_NEED_CLEANUP; io_vec_reset_iovec(&io->vec, iov, io->iter.nr_segs); } return 0;} static int __io_import_rw_buffer(int ddir, struct io_kiocb *req, struct io_async_rw *io, struct io_br_sel *sel, unsigned int issue_flags){ const struct io_issue_def *def = &io_issue_defs[req->opcode]; struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); size_t sqe_len = rw->len; sel->addr = u64_to_user_ptr(rw->addr); if (def->vectored && !(req->flags & REQ_F_BUFFER_SELECT)) return io_import_vec(ddir, req, io, sel->addr, sqe_len); if (io_do_buffer_select(req)) { *sel = io_buffer_select(req, &sqe_len, io->buf_group, issue_flags); if (!sel->addr) return -ENOBUFS; rw->addr = (unsigned long) sel->addr; rw->len = sqe_len; } return import_ubuf(ddir, sel->addr, sqe_len, &io->iter);} static inline int io_import_rw_buffer(int rw, struct io_kiocb *req, struct io_async_rw *io, struct io_br_sel *sel, unsigned int issue_flags){ int ret; ret = __io_import_rw_buffer(rw, req, io, sel, issue_flags); if (unlikely(ret < 0)) return ret; iov_iter_save_state(&io->iter, &io->iter_state); return 0;} static bool io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags){ struct io_async_rw *rw = req->async_data; if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) return false; io_alloc_cache_vec_kasan(&rw->vec); if (rw->vec.nr > IO_VEC_CACHE_SOFT_CAP) io_vec_free(&rw->vec); if (io_alloc_cache_put(&req->ctx->rw_cache, rw)) { io_req_async_data_clear(req, 0); return true; } return false;} static void io_req_rw_cleanup(struct io_kiocb *req, unsigned int issue_flags){ /* * Disable quick recycling for anything that's gone through io-wq. * In theory, this should be fine to cleanup. However, some read or * write iter handling touches the iovec AFTER having called into the * handler, eg to reexpand or revert. This means we can have: * * task io-wq * issue * punt to io-wq * issue * blkdev_write_iter() * ->ki_complete() * io_complete_rw() * queue tw complete * run tw * req_rw_cleanup * iov_iter_count() <- look at iov_iter again * * which can lead to a UAF. This is only possible for io-wq offload * as the cleanup can run in parallel. As io-wq is not the fast path, * just leave cleanup to the end. * * This is really a bug in the core code that does this, any issue * path should assume that a successful (or -EIOCBQUEUED) return can * mean that the underlying data can be gone at any time. But that * should be fixed separately, and then this check could be killed. */ if (!(req->flags & (REQ_F_REISSUE | REQ_F_REFCOUNT))) { req->flags &= ~REQ_F_NEED_CLEANUP; if (!io_rw_recycle(req, issue_flags)) { struct io_async_rw *rw = req->async_data; io_vec_free(&rw->vec); } }} static int io_rw_alloc_async(struct io_kiocb *req){ struct io_ring_ctx *ctx = req->ctx; struct io_async_rw *rw; rw = io_uring_alloc_async_data(&ctx->rw_cache, req); if (!rw) return -ENOMEM; if (rw->vec.iovec) req->flags |= REQ_F_NEED_CLEANUP; rw->bytes_done = 0; return 0;} static inline void io_meta_save_state(struct io_async_rw *io){ io->meta_state.seed = io->meta.seed; iov_iter_save_state(&io->meta.iter, &io->meta_state.iter_meta);} static inline void io_meta_restore(struct io_async_rw *io, struct kiocb *kiocb){ if (kiocb->ki_flags & IOCB_HAS_METADATA) { io->meta.seed = io->meta_state.seed; iov_iter_restore(&io->meta.iter, &io->meta_state.iter_meta); }} static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir, u64 attr_ptr){ struct io_uring_attr_pi pi_attr; struct io_async_rw *io; int ret; if (copy_from_user(&pi_attr, u64_to_user_ptr(attr_ptr), sizeof(pi_attr))) return -EFAULT; if (pi_attr.rsvd) return -EINVAL; io = req->async_data; io->meta.flags = pi_attr.flags; io->meta.app_tag = pi_attr.app_tag; io->meta.seed = pi_attr.seed; ret = import_ubuf(ddir, u64_to_user_ptr(pi_attr.addr), pi_attr.len, &io->meta.iter); if (unlikely(ret < 0)) return ret; req->flags |= REQ_F_HAS_METADATA; io_meta_save_state(io); return ret;} static int __io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, int ddir){ struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct io_async_rw *io; unsigned ioprio; u64 attr_type_mask; int ret; if (io_rw_alloc_async(req)) return -ENOMEM; io = req->async_data; rw->kiocb.ki_pos = READ_ONCE(sqe->off); /* used for fixed read/write too - just read unconditionally */ req->buf_index = READ_ONCE(sqe->buf_index); io->buf_group = req->buf_index; ioprio = READ_ONCE(sqe->ioprio); if (ioprio) { ret = ioprio_check_cap(ioprio); if (ret) return ret; rw->kiocb.ki_ioprio = ioprio; } else { rw->kiocb.ki_ioprio = get_current_ioprio(); } rw->kiocb.ki_flags = 0; rw->kiocb.ki_write_stream = READ_ONCE(sqe->write_stream); if (req->ctx->flags & IORING_SETUP_IOPOLL) rw->kiocb.ki_complete = io_complete_rw_iopoll; else rw->kiocb.ki_complete = io_complete_rw; rw->addr = READ_ONCE(sqe->addr); rw->len = READ_ONCE(sqe->len); rw->flags = (__force rwf_t) READ_ONCE(sqe->rw_flags); attr_type_mask = READ_ONCE(sqe->attr_type_mask); if (attr_type_mask) { u64 attr_ptr; /* only PI attribute is supported currently */ if (attr_type_mask != IORING_RW_ATTR_FLAG_PI) return -EINVAL; attr_ptr = READ_ONCE(sqe->attr_ptr); return io_prep_rw_pi(req, rw, ddir, attr_ptr); } return 0;} static int io_rw_do_import(struct io_kiocb *req, int ddir){ struct io_br_sel sel = { }; if (io_do_buffer_select(req)) return 0; return io_import_rw_buffer(ddir, req, req->async_data, &sel, 0);} static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, int ddir){ int ret; ret = __io_prep_rw(req, sqe, ddir); if (unlikely(ret)) return ret; return io_rw_do_import(req, ddir);} int io_prep_read(struct io_kiocb *req, const struct io_uring_sqe *sqe){ return io_prep_rw(req, sqe, ITER_DEST);} int io_prep_write(struct io_kiocb *req, const struct io_uring_sqe *sqe){ return io_prep_rw(req, sqe, ITER_SOURCE);} static int io_prep_rwv(struct io_kiocb *req, const struct io_uring_sqe *sqe, int ddir){ int ret; ret = io_prep_rw(req, sqe, ddir); if (unlikely(ret)) return ret; if (!(req->flags & REQ_F_BUFFER_SELECT)) return 0; /* * Have to do this validation here, as this is in io_read() rw->len * might have changed due to buffer selection */ return io_iov_buffer_select_prep(req);} int io_prep_readv(struct io_kiocb *req, const struct io_uring_sqe *sqe){ return io_prep_rwv(req, sqe, ITER_DEST);} int io_prep_writev(struct io_kiocb *req, const struct io_uring_sqe *sqe){ return io_prep_rwv(req, sqe, ITER_SOURCE);} static int io_init_rw_fixed(struct io_kiocb *req, unsigned int issue_flags, int ddir){ struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct io_async_rw *io = req->async_data; int ret; if (io->bytes_done) return 0; ret = io_import_reg_buf(req, &io->iter, rw->addr, rw->len, ddir, issue_flags); iov_iter_save_state(&io->iter, &io->iter_state); return ret;} int io_prep_read_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe){ return __io_prep_rw(req, sqe, ITER_DEST);} int io_prep_write_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe){ return __io_prep_rw(req, sqe, ITER_SOURCE);} static int io_rw_import_reg_vec(struct io_kiocb *req, struct io_async_rw *io, int ddir, unsigned int issue_flags){ struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); unsigned uvec_segs = rw->len; int ret; ret = io_import_reg_vec(ddir, &io->iter, req, &io->vec, uvec_segs, issue_flags); if (unlikely(ret)) return ret; iov_iter_save_state(&io->iter, &io->iter_state); req->flags &= ~REQ_F_IMPORT_BUFFER; return 0;} static int io_rw_prep_reg_vec(struct io_kiocb *req){ struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct io_async_rw *io = req->async_data; const struct iovec __user *uvec; uvec = u64_to_user_ptr(rw->addr); return io_prep_reg_iovec(req, &io->vec, uvec, rw->len);} int io_prep_readv_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe){ int ret; ret = __io_prep_rw(req, sqe, ITER_DEST); if (unlikely(ret)) return ret; return io_rw_prep_reg_vec(req);} int io_prep_writev_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe){ int ret; ret = __io_prep_rw(req, sqe, ITER_SOURCE); if (unlikely(ret)) return ret; return io_rw_prep_reg_vec(req);} /* * Multishot read is prepared just like a normal read/write request, only * difference is that we set the MULTISHOT flag. */int io_read_mshot_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe){ struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); int ret; /* must be used with provided buffers */ if (!(req->flags & REQ_F_BUFFER_SELECT)) return -EINVAL; ret = __io_prep_rw(req, sqe, ITER_DEST); if (unlikely(ret)) return ret; if (rw->addr || rw->len) return -EINVAL; req->flags |= REQ_F_APOLL_MULTISHOT; return 0;} void io_readv_writev_cleanup(struct io_kiocb *req){ struct io_async_rw *rw = req->async_data; lockdep_assert_held(&req->ctx->uring_lock); io_vec_free(&rw->vec); io_rw_recycle(req, 0);} static inline loff_t *io_kiocb_update_pos(struct io_kiocb *req){ struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); if (rw->kiocb.ki_pos != -1) return &rw->kiocb.ki_pos; if (!(req->file->f_mode & FMODE_STREAM)) { req->flags |= REQ_F_CUR_POS; rw->kiocb.ki_pos = req->file->f_pos; return &rw->kiocb.ki_pos; } rw->kiocb.ki_pos = 0; return NULL;} static bool io_rw_should_reissue(struct io_kiocb *req){#ifdef CONFIG_BLOCK struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); umode_t mode = file_inode(req->file)->i_mode; struct io_async_rw *io = req->async_data; struct io_ring_ctx *ctx = req->ctx; if (!S_ISBLK(mode) && !S_ISREG(mode)) return false; if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() && !(req->flags & REQ_F_IOPOLL))) return false; /* * If ref is dying, we might be running poll reap from the exit work. * Don't attempt to reissue from that path, just let it fail with * -EAGAIN. */ if (percpu_ref_is_dying(&ctx->refs))Lines 1–500 of 1391. Load more
// SPDX-License-Identifier: GPL-2.0 #include <linux/kernel.h> #include <linux/errno.h> #include <linux/fs.h> #include <linux/file.h> #include <linux/blk-mq.h> #include <linux/mm.h> #include <linux/slab.h> #include <linux/fsnotify.h> #include <linux/poll.h> #include <linux/nospec.h> #include <linux/io_uring/cmd.h> #include <linux/indirect_call_wrapper.h> #include <uapi/linux/io_uring.h> #include "filetable.h" #include "io_uring.h" #include "opdef.h" #include "kbuf.h" #include "alloc_cache.h" #include "rsrc.h" #include "poll.h" #include "rw.h" static void io_complete_rw(struct kiocb *kiocb, long res); static void io_complete_rw_iopoll(struct kiocb *kiocb, long res); struct io_rw { /* NOTE: kiocb has the file as the first member, so don't do it here */ struct kiocb kiocb; u64 addr; u32 len; rwf_t flags; }; static bool io_file_supports_nowait(struct io_kiocb *req, __poll_t mask) { /* If FMODE_NOWAIT is set for a file, we're golden */ if (req->flags & REQ_F_SUPPORT_NOWAIT) return true; /* No FMODE_NOWAIT, if we can poll, check the status */ if (io_file_can_poll(req)) { struct poll_table_struct pt = { ._key = mask }; return vfs_poll(req->file, &pt) & mask; } /* No FMODE_NOWAIT support, and file isn't pollable. Tough luck. */ return false; } static int io_iov_buffer_select_prep(struct io_kiocb *req) { struct iovec __user *uiov; struct iovec fast_iov, *iov; struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); if (rw->len != 1) return -EINVAL; uiov = u64_to_user_ptr(rw->addr); iov = iovec_from_user(uiov, 1, 1, &fast_iov, io_is_compat(req->ctx)); if (IS_ERR(iov)) return PTR_ERR(iov); rw->len = iov->iov_len; return 0; } static int io_import_vec(int ddir, struct io_kiocb *req, struct io_async_rw *io, const struct iovec __user *uvec, size_t uvec_segs) { int ret, nr_segs; struct iovec *iov; if (io->vec.iovec) { nr_segs = io->vec.nr; iov = io->vec.iovec; } else { nr_segs = 1; iov = &io->fast_iov; } ret = __import_iovec(ddir, uvec, uvec_segs, nr_segs, &iov, &io->iter, io_is_compat(req->ctx)); if (unlikely(ret < 0)) return ret; if (iov) { req->flags |= REQ_F_NEED_CLEANUP; io_vec_reset_iovec(&io->vec, iov, io->iter.nr_segs); } return 0; } static int __io_import_rw_buffer(int ddir, struct io_kiocb *req, struct io_async_rw *io, struct io_br_sel *sel, unsigned int issue_flags) { const struct io_issue_def *def = &io_issue_defs[req->opcode]; struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); size_t sqe_len = rw->len; sel->addr = u64_to_user_ptr(rw->addr); if (def->vectored && !(req->flags & REQ_F_BUFFER_SELECT)) return io_import_vec(ddir, req, io, sel->addr, sqe_len); if (io_do_buffer_select(req)) { *sel = io_buffer_select(req, &sqe_len, io->buf_group, issue_flags); if (!sel->addr) return -ENOBUFS; rw->addr = (unsigned long) sel->addr; rw->len = sqe_len; } return import_ubuf(ddir, sel->addr, sqe_len, &io->iter); } static inline int io_import_rw_buffer(int rw, struct io_kiocb *req, struct io_async_rw *io, struct io_br_sel *sel, unsigned int issue_flags) { int ret; ret = __io_import_rw_buffer(rw, req, io, sel, issue_flags); if (unlikely(ret < 0)) return ret; iov_iter_save_state(&io->iter, &io->iter_state); return 0; } static bool io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags) { struct io_async_rw *rw = req->async_data; if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) return false; io_alloc_cache_vec_kasan(&rw->vec); if (rw->vec.nr > IO_VEC_CACHE_SOFT_CAP) io_vec_free(&rw->vec); if (io_alloc_cache_put(&req->ctx->rw_cache, rw)) { io_req_async_data_clear(req, 0); return true; } return false; } static void io_req_rw_cleanup(struct io_kiocb *req, unsigned int issue_flags) { /* * Disable quick recycling for anything that's gone through io-wq. * In theory, this should be fine to cleanup. However, some read or * write iter handling touches the iovec AFTER having called into the * handler, eg to reexpand or revert. This means we can have: * * task io-wq * issue * punt to io-wq * issue * blkdev_write_iter() * ->ki_complete() * io_complete_rw() * queue tw complete * run tw * req_rw_cleanup * iov_iter_count() <- look at iov_iter again * * which can lead to a UAF. This is only possible for io-wq offload * as the cleanup can run in parallel. As io-wq is not the fast path, * just leave cleanup to the end. * * This is really a bug in the core code that does this, any issue * path should assume that a successful (or -EIOCBQUEUED) return can * mean that the underlying data can be gone at any time. But that * should be fixed separately, and then this check could be killed. */ if (!(req->flags & (REQ_F_REISSUE | REQ_F_REFCOUNT))) { req->flags &= ~REQ_F_NEED_CLEANUP; if (!io_rw_recycle(req, issue_flags)) { struct io_async_rw *rw = req->async_data; io_vec_free(&rw->vec); } } } static int io_rw_alloc_async(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; struct io_async_rw *rw; rw = io_uring_alloc_async_data(&ctx->rw_cache, req); if (!rw) return -ENOMEM; if (rw->vec.iovec) req->flags |= REQ_F_NEED_CLEANUP; rw->bytes_done = 0; return 0; } static inline void io_meta_save_state(struct io_async_rw *io) { io->meta_state.seed = io->meta.seed; iov_iter_save_state(&io->meta.iter, &io->meta_state.iter_meta); } static inline void io_meta_restore(struct io_async_rw *io, struct kiocb *kiocb) { if (kiocb->ki_flags & IOCB_HAS_METADATA) { io->meta.seed = io->meta_state.seed; iov_iter_restore(&io->meta.iter, &io->meta_state.iter_meta); } } static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir, u64 attr_ptr) { struct io_uring_attr_pi pi_attr; struct io_async_rw *io; int ret; if (copy_from_user(&pi_attr, u64_to_user_ptr(attr_ptr), sizeof(pi_attr))) return -EFAULT; if (pi_attr.rsvd) return -EINVAL; io = req->async_data; io->meta.flags = pi_attr.flags; io->meta.app_tag = pi_attr.app_tag; io->meta.seed = pi_attr.seed; ret = import_ubuf(ddir, u64_to_user_ptr(pi_attr.addr), pi_attr.len, &io->meta.iter); if (unlikely(ret < 0)) return ret; req->flags |= REQ_F_HAS_METADATA; io_meta_save_state(io); return ret; } static int __io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, int ddir) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct io_async_rw *io; unsigned ioprio; u64 attr_type_mask; int ret; if (io_rw_alloc_async(req)) return -ENOMEM; io = req->async_data; rw->kiocb.ki_pos = READ_ONCE(sqe->off); /* used for fixed read/write too - just read unconditionally */ req->buf_index = READ_ONCE(sqe->buf_index); io->buf_group = req->buf_index; ioprio = READ_ONCE(sqe->ioprio); if (ioprio) { ret = ioprio_check_cap(ioprio); if (ret) return ret; rw->kiocb.ki_ioprio = ioprio; } else { rw->kiocb.ki_ioprio = get_current_ioprio(); } rw->kiocb.ki_flags = 0; rw->kiocb.ki_write_stream = READ_ONCE(sqe->write_stream); if (req->ctx->flags & IORING_SETUP_IOPOLL) rw->kiocb.ki_complete = io_complete_rw_iopoll; else rw->kiocb.ki_complete = io_complete_rw; rw->addr = READ_ONCE(sqe->addr); rw->len = READ_ONCE(sqe->len); rw->flags = (__force rwf_t) READ_ONCE(sqe->rw_flags); attr_type_mask = READ_ONCE(sqe->attr_type_mask); if (attr_type_mask) { u64 attr_ptr; /* only PI attribute is supported currently */ if (attr_type_mask != IORING_RW_ATTR_FLAG_PI) return -EINVAL; attr_ptr = READ_ONCE(sqe->attr_ptr); return io_prep_rw_pi(req, rw, ddir, attr_ptr); } return 0; } static int io_rw_do_import(struct io_kiocb *req, int ddir) { struct io_br_sel sel = { }; if (io_do_buffer_select(req)) return 0; return io_import_rw_buffer(ddir, req, req->async_data, &sel, 0); } static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, int ddir) { int ret; ret = __io_prep_rw(req, sqe, ddir); if (unlikely(ret)) return ret; return io_rw_do_import(req, ddir); } int io_prep_read(struct io_kiocb *req, const struct io_uring_sqe *sqe) { return io_prep_rw(req, sqe, ITER_DEST); } int io_prep_write(struct io_kiocb *req, const struct io_uring_sqe *sqe) { return io_prep_rw(req, sqe, ITER_SOURCE); } static int io_prep_rwv(struct io_kiocb *req, const struct io_uring_sqe *sqe, int ddir) { int ret; ret = io_prep_rw(req, sqe, ddir); if (unlikely(ret)) return ret; if (!(req->flags & REQ_F_BUFFER_SELECT)) return 0; /* * Have to do this validation here, as this is in io_read() rw->len * might have changed due to buffer selection */ return io_iov_buffer_select_prep(req); } int io_prep_readv(struct io_kiocb *req, const struct io_uring_sqe *sqe) { return io_prep_rwv(req, sqe, ITER_DEST); } int io_prep_writev(struct io_kiocb *req, const struct io_uring_sqe *sqe) { return io_prep_rwv(req, sqe, ITER_SOURCE); } static int io_init_rw_fixed(struct io_kiocb *req, unsigned int issue_flags, int ddir) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct io_async_rw *io = req->async_data; int ret; if (io->bytes_done) return 0; ret = io_import_reg_buf(req, &io->iter, rw->addr, rw->len, ddir, issue_flags); iov_iter_save_state(&io->iter, &io->iter_state); return ret; } int io_prep_read_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe) { return __io_prep_rw(req, sqe, ITER_DEST); } int io_prep_write_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe) { return __io_prep_rw(req, sqe, ITER_SOURCE); } static int io_rw_import_reg_vec(struct io_kiocb *req, struct io_async_rw *io, int ddir, unsigned int issue_flags) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); unsigned uvec_segs = rw->len; int ret; ret = io_import_reg_vec(ddir, &io->iter, req, &io->vec, uvec_segs, issue_flags); if (unlikely(ret)) return ret; iov_iter_save_state(&io->iter, &io->iter_state); req->flags &= ~REQ_F_IMPORT_BUFFER; return 0; } static int io_rw_prep_reg_vec(struct io_kiocb *req) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct io_async_rw *io = req->async_data; const struct iovec __user *uvec; uvec = u64_to_user_ptr(rw->addr); return io_prep_reg_iovec(req, &io->vec, uvec, rw->len); } int io_prep_readv_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe) { int ret; ret = __io_prep_rw(req, sqe, ITER_DEST); if (unlikely(ret)) return ret; return io_rw_prep_reg_vec(req); } int io_prep_writev_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe) { int ret; ret = __io_prep_rw(req, sqe, ITER_SOURCE); if (unlikely(ret)) return ret; return io_rw_prep_reg_vec(req); } /* * Multishot read is prepared just like a normal read/write request, only * difference is that we set the MULTISHOT flag. */ int io_read_mshot_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); int ret; /* must be used with provided buffers */ if (!(req->flags & REQ_F_BUFFER_SELECT)) return -EINVAL; ret = __io_prep_rw(req, sqe, ITER_DEST); if (unlikely(ret)) return ret; if (rw->addr || rw->len) return -EINVAL; req->flags |= REQ_F_APOLL_MULTISHOT; return 0; } void io_readv_writev_cleanup(struct io_kiocb *req) { struct io_async_rw *rw = req->async_data; lockdep_assert_held(&req->ctx->uring_lock); io_vec_free(&rw->vec); io_rw_recycle(req, 0); } static inline loff_t *io_kiocb_update_pos(struct io_kiocb *req) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); if (rw->kiocb.ki_pos != -1) return &rw->kiocb.ki_pos; if (!(req->file->f_mode & FMODE_STREAM)) { req->flags |= REQ_F_CUR_POS; rw->kiocb.ki_pos = req->file->f_pos; return &rw->kiocb.ki_pos; } rw->kiocb.ki_pos = 0; return NULL; } static bool io_rw_should_reissue(struct io_kiocb *req) { #ifdef CONFIG_BLOCK struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); umode_t mode = file_inode(req->file)->i_mode; struct io_async_rw *io = req->async_data; struct io_ring_ctx *ctx = req->ctx; if (!S_ISBLK(mode) && !S_ISREG(mode)) return false; if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() && !(req->flags & REQ_F_IOPOLL))) return false; /* * If ref is dying, we might be running poll reap from the exit work. * Don't attempt to reissue from that path, just let it fail with * -EAGAIN. */ if (percpu_ref_is_dying(&ctx->refs))