uring_cmd currently copies the full SQE at prep time, just in case it needs it to be stable. However, for inline completions or requests that get queued up on the device side, there's no need to ever copy the SQE. This is particularly important, as various use cases of uring_cmd will be using 128b sized SQEs. Opt in to using ->sqe_copy() to let the core of io_uring decide when to copy SQEs. This callback will only be called if it is safe to do so.
Unified · Split
| @@ -759,6 +759,7 @@ const struct io_cold_def io_cold_defs[] = { · 758 unmodified lines | |||
| 759 | }, | 759 | }, |
| 760 | [IORING_OP_URING_CMD] = { | 760 | [IORING_OP_URING_CMD] = { |
| 761 | .name = "URING_CMD", | 761 | .name = "URING_CMD", |
| 762 | .sqe_copy = io_uring_cmd_sqe_copy, | ||
| 762 | .cleanup = io_uring_cmd_cleanup, | 763 | .cleanup = io_uring_cmd_cleanup, |
| 763 | }, | 764 | }, |
| 764 | [IORING_OP_SEND_ZC] = { | 765 | [IORING_OP_SEND_ZC] = { |
| @@ -205,17 +205,20 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 204 unmodified lines | |||
| 205 | if (!ac) | 205 | if (!ac) |
| 206 | return -ENOMEM; | 206 | return -ENOMEM; |
| 207 | ac->data.op_data = NULL; | 207 | ac->data.op_data = NULL; |
| 208 | ioucmd->sqe = sqe; | ||
| 209 | return 0; | ||
| 210 | } | ||
| 211 | |||
| 212 | void io_uring_cmd_sqe_copy(struct io_kiocb *req) | ||
| 213 | { | ||
| 214 | struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); | ||
| 215 | struct io_async_cmd *ac = req->async_data; | ||
| 208 | 216 | ||
| 209 | /* | 217 | /* Should not happen, as REQ_F_SQE_COPIED covers this */ |
| 210 | * Unconditionally cache the SQE for now - this is only needed for | 218 | if (WARN_ON_ONCE(ioucmd->sqe == ac->sqes)) |
| 211 | * requests that go async, but prep handlers must ensure that any | 219 | return; |
| 212 | * sqe data is stable beyond prep. Since uring_cmd is special in | 220 | memcpy(ac->sqes, ioucmd->sqe, uring_sqe_size(req->ctx)); |
| 213 | * that it doesn't read in per-op data, play it safe and ensure that | ||
| 214 | * any SQE data is stable beyond prep. This can later get relaxed. | ||
| 215 | */ | ||
| 216 | memcpy(ac->sqes, sqe, uring_sqe_size(req->ctx)); | ||
| 217 | ioucmd->sqe = ac->sqes; | 221 | ioucmd->sqe = ac->sqes; |
| 218 | return 0; | ||
| 219 | } | 222 | } |
| 220 | 223 | ||
| 221 | int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) | 224 | int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) |
| @@ -11,6 +11,7 @@ struct io_async_cmd { · 10 unmodified lines | |||
| 11 | 11 | ||
| 12 | int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags); | 12 | int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags); |
| 13 | int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); | 13 | int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); |
| 14 | void io_uring_cmd_sqe_copy(struct io_kiocb *req); | ||
| 14 | void io_uring_cmd_cleanup(struct io_kiocb *req); | 15 | void io_uring_cmd_cleanup(struct io_kiocb *req); |
| 15 | 16 | ||
| 16 | bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx, | 17 | bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx, |