Pull io_uring fixes from Jens Axboe: - Removed function that became unused after last week's merge (Jiapeng) - Two small fixes for kbuf recycling (Pavel) - Include address copy for zc send for POLLFIRST (Pavel) - Fix for short IO handling in the normal read/write path (Pavel) * tag 'io_uring-6.0-2022-09-09' of git://git.kernel.dk/linux-block: io_uring/rw: fix short rw error handling io_uring/net: copy addr for zc on POLL_FIRST io_uring: recycle kbuf recycle on tw requeue io_uring/kbuf: fix not advancing READV kbuf ring io_uring/notif: Remove the unused function io_notif_complete()
Unified · Split
| @@ -1728,6 +1728,7 @@ static void io_queue_async(struct io_kiocb *req, int ret) · 1727 unmodified lines | ||
| 1728 | 1728 | |
| 1729 | 1729 | switch (io_arm_poll_handler(req, 0)) { |
| 1730 | 1730 | case IO_APOLL_READY: |
| 1731 | + io_kbuf_recycle(req, 0); | |
| 1731 | 1732 | io_req_task_queue(req); |
| 1732 | 1733 | break; |
| 1733 | 1734 | case IO_APOLL_ABORTED: |
| @@ -91,9 +91,13 @@ static inline void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) · 90 unmodified lines | ||
| 91 | 91 | * buffer data. However if that buffer is recycled the original request |
| 92 | 92 | * data stored in addr is lost. Therefore forbid recycling for now. |
| 93 | 93 | */ |
| 94 | − if (req->opcode == IORING_OP_READV) | |
| 94 | + if (req->opcode == IORING_OP_READV) { | |
| 95 | + if ((req->flags & REQ_F_BUFFER_RING) && req->buf_list) { | |
| 96 | + req->buf_list->head++; | |
| 97 | + req->buf_list = NULL; | |
| 98 | + } | |
| 95 | 99 | return; |
| 96 | − | |
| 100 | + } | |
| 97 | 101 | if (req->flags & REQ_F_BUFFER_SELECTED) |
| 98 | 102 | io_kbuf_recycle_legacy(req, issue_flags); |
| 99 | 103 | if (req->flags & REQ_F_BUFFER_RING) |
| @@ -1003,9 +1003,6 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) · 1002 unmodified lines | ||
| 1003 | 1003 | unsigned msg_flags, cflags; |
| 1004 | 1004 | int ret, min_ret = 0; |
| 1005 | 1005 | |
| 1006 | − if (!(req->flags & REQ_F_POLLED) && | |
| 1007 | − (zc->flags & IORING_RECVSEND_POLL_FIRST)) | |
| 1008 | − return -EAGAIN; | |
| 1009 | 1006 | sock = sock_from_file(req->file); |
| 1010 | 1007 | if (unlikely(!sock)) |
| 1011 | 1008 | return -ENOTSOCK; |
| @@ -1030,6 +1027,10 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) · 18 unmodified lines | ||
| 1030 | 1027 | msg.msg_namelen = zc->addr_len; |
| 1031 | 1028 | } |
| 1032 | 1029 | |
| 1030 | + if (!(req->flags & REQ_F_POLLED) && | |
| 1031 | + (zc->flags & IORING_RECVSEND_POLL_FIRST)) | |
| 1032 | + return io_setup_async_addr(req, addr, issue_flags); | |
| 1033 | + | |
| 1033 | 1034 | if (zc->flags & IORING_RECVSEND_FIXED_BUF) { |
| 1034 | 1035 | ret = io_import_fixed(WRITE, &msg.msg_iter, req->imu, |
| 1035 | 1036 | (u64)(uintptr_t)zc->buf, zc->len); |
| @@ -21,14 +21,6 @@ static void __io_notif_complete_tw(struct io_kiocb *notif, bool *locked) · 20 unmodified lines | ||
| 21 | 21 | io_req_task_complete(notif, locked); |
| 22 | 22 | } |
| 23 | 23 | |
| 24 | −static inline void io_notif_complete(struct io_kiocb *notif) | |
| 25 | − __must_hold(¬if->ctx->uring_lock) | |
| 26 | −{ | |
| 27 | − bool locked = true; | |
| 28 | − | |
| 29 | − __io_notif_complete_tw(notif, &locked); | |
| 30 | −} | |
| 31 | − | |
| 32 | 24 | static void io_uring_tx_zerocopy_callback(struct sk_buff *skb, |
| 33 | 25 | struct ubuf_info *uarg, |
| 34 | 26 | bool success) |
| @@ -206,6 +206,20 @@ static bool __io_complete_rw_common(struct io_kiocb *req, long res) · 205 unmodified lines | ||
| 206 | 206 | return false; |
| 207 | 207 | } |
| 208 | 208 | |
| 209 | +static inline unsigned io_fixup_rw_res(struct io_kiocb *req, unsigned res) | |
| 210 | +{ | |
| 211 | + struct io_async_rw *io = req->async_data; | |
| 212 | + | |
| 213 | + /* add previously done IO, if any */ | |
| 214 | + if (req_has_async_data(req) && io->bytes_done > 0) { | |
| 215 | + if (res < 0) | |
| 216 | + res = io->bytes_done; | |
| 217 | + else | |
| 218 | + res += io->bytes_done; | |
| 219 | + } | |
| 220 | + return res; | |
| 221 | +} | |
| 222 | + | |
| 209 | 223 | static void io_complete_rw(struct kiocb *kiocb, long res) |
| 210 | 224 | { |
| 211 | 225 | struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb); |
| @@ -213,7 +227,7 @@ static void io_complete_rw(struct kiocb *kiocb, long res) · 1 unmodified line | ||
| 213 | 227 | |
| 214 | 228 | if (__io_complete_rw_common(req, res)) |
| 215 | 229 | return; |
| 216 | − io_req_set_res(req, res, 0); | |
| 230 | + io_req_set_res(req, io_fixup_rw_res(req, res), 0); | |
| 217 | 231 | req->io_task_work.func = io_req_task_complete; |
| 218 | 232 | io_req_task_work_add(req); |
| 219 | 233 | } |
| @@ -240,22 +254,14 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res) · 20 unmodified lines | ||
| 240 | 254 | static int kiocb_done(struct io_kiocb *req, ssize_t ret, |
| 241 | 255 | unsigned int issue_flags) |
| 242 | 256 | { |
| 243 | − struct io_async_rw *io = req->async_data; | |
| 244 | 257 | struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); |
| 245 | − | |
| 246 | − /* add previously done IO, if any */ | |
| 247 | − if (req_has_async_data(req) && io->bytes_done > 0) { | |
| 248 | − if (ret < 0) | |
| 249 | − ret = io->bytes_done; | |
| 250 | − else | |
| 251 | − ret += io->bytes_done; | |
| 252 | − } | |
| 258 | + unsigned final_ret = io_fixup_rw_res(req, ret); | |
| 253 | 259 | |
| 254 | 260 | if (req->flags & REQ_F_CUR_POS) |
| 255 | 261 | req->file->f_pos = rw->kiocb.ki_pos; |
| 256 | 262 | if (ret >= 0 && (rw->kiocb.ki_complete == io_complete_rw)) { |
| 257 | 263 | if (!__io_complete_rw_common(req, ret)) { |
| 258 | − io_req_set_res(req, req->cqe.res, | |
| 264 | + io_req_set_res(req, final_ret, | |
| 259 | 265 | io_put_kbuf(req, issue_flags)); |
| 260 | 266 | return IOU_OK; |
| 261 | 267 | } |
| @@ -268,7 +274,7 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret, · 6 unmodified lines | ||
| 268 | 274 | if (io_resubmit_prep(req)) |
| 269 | 275 | io_req_task_queue_reissue(req); |
| 270 | 276 | else |
| 271 | − io_req_task_queue_fail(req, ret); | |
| 277 | + io_req_task_queue_fail(req, final_ret); | |
| 272 | 278 | } |
| 273 | 279 | return IOU_ISSUE_SKIP_COMPLETE; |
| 274 | 280 | } |