This is cleaner than gating on the opcode type, particularly as more read/write type opcodes may be added. Then we can use that for the data import, and for __io_read() on whether or not we need to copy state.
Unified · Split
| @@ -63,6 +63,7 @@ const struct io_issue_def io_issue_defs[] = { · 62 unmodified lines | |||
| 63 | .ioprio = 1, | 63 | .ioprio = 1, |
| 64 | .iopoll = 1, | 64 | .iopoll = 1, |
| 65 | .iopoll_queue = 1, | 65 | .iopoll_queue = 1, |
| 66 | .vectored = 1, | ||
| 66 | .prep = io_prep_rw, | 67 | .prep = io_prep_rw, |
| 67 | .issue = io_read, | 68 | .issue = io_read, |
| 68 | }, | 69 | }, |
| @@ -76,6 +77,7 @@ const struct io_issue_def io_issue_defs[] = { · 7 unmodified lines | |||
| 76 | .ioprio = 1, | 77 | .ioprio = 1, |
| 77 | .iopoll = 1, | 78 | .iopoll = 1, |
| 78 | .iopoll_queue = 1, | 79 | .iopoll_queue = 1, |
| 80 | .vectored = 1, | ||
| 79 | .prep = io_prep_rw, | 81 | .prep = io_prep_rw, |
| 80 | .issue = io_write, | 82 | .issue = io_write, |
| 81 | }, | 83 | }, |
| @@ -29,6 +29,8 @@ struct io_issue_def { · 28 unmodified lines | |||
| 29 | unsigned iopoll_queue : 1; | 29 | unsigned iopoll_queue : 1; |
| 30 | /* opcode specific path will handle ->async_data allocation if needed */ | 30 | /* opcode specific path will handle ->async_data allocation if needed */ |
| 31 | unsigned manual_alloc : 1; | 31 | unsigned manual_alloc : 1; |
| 32 | /* vectored opcode, set if 1) vectored, and 2) handler needs to know */ | ||
| 33 | unsigned vectored : 1; | ||
| 32 | 34 | ||
| 33 | int (*issue)(struct io_kiocb *, unsigned int); | 35 | int (*issue)(struct io_kiocb *, unsigned int); |
| 34 | int (*prep)(struct io_kiocb *, const struct io_uring_sqe *); | 36 | int (*prep)(struct io_kiocb *, const struct io_uring_sqe *); |
| @@ -388,8 +388,7 @@ static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, · 387 unmodified lines | |||
| 388 | buf = u64_to_user_ptr(rw->addr); | 388 | buf = u64_to_user_ptr(rw->addr); |
| 389 | sqe_len = rw->len; | 389 | sqe_len = rw->len; |
| 390 | 390 | ||
| 391 | if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE || | 391 | if (!io_issue_defs[opcode].vectored || req->flags & REQ_F_BUFFER_SELECT) { |
| 392 | (req->flags & REQ_F_BUFFER_SELECT)) { | ||
| 393 | if (io_do_buffer_select(req)) { | 392 | if (io_do_buffer_select(req)) { |
| 394 | buf = io_buffer_select(req, &sqe_len, issue_flags); | 393 | buf = io_buffer_select(req, &sqe_len, issue_flags); |
| 395 | if (!buf) | 394 | if (!buf) |
| @@ -776,8 +775,11 @@ static int __io_read(struct io_kiocb *req, unsigned int issue_flags) · 380 unmodified lines | |||
| 776 | 775 | ||
| 777 | if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) { | 776 | if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) { |
| 778 | req->flags &= ~REQ_F_REISSUE; | 777 | req->flags &= ~REQ_F_REISSUE; |
| 779 | /* if we can poll, just do that */ | 778 | /* |
| 780 | if (req->opcode == IORING_OP_READ && file_can_poll(req->file)) | 779 | * If we can poll, just do that. For a vectored read, we'll |
| 780 | * need to copy state first. | ||
| 781 | */ | ||
| 782 | if (file_can_poll(req->file) && !io_issue_defs[req->opcode].vectored) | ||
| 781 | return -EAGAIN; | 783 | return -EAGAIN; |
| 782 | /* IOPOLL retry should happen for io-wq threads */ | 784 | /* IOPOLL retry should happen for io-wq threads */ |
| 783 | if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) | 785 | if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) |