History

Merge tag 'for-6.4/io_uring-2023-05-07' of git://git.kernel.dk/linux

Pull more io_uring updates from Jens Axboe: "Nothing major in here, just two different parts: - A small series from Breno that enables passing the full SQE down for ->uring_cmd(). This is a prerequisite for enabling full network socket operations. Queued up a bit late because of some stylistic concerns that got resolved, would be nice to have this in 6.4-rc1 so the dependent work will be easier to handle for 6.5. - Fix for the huge page coalescing, which was a regression introduced in the 6.3 kernel release (Tobias)" * tag 'for-6.4/io_uring-2023-05-07' of git://git.kernel.dk/linux: io_uring: Remove unnecessary BUILD_BUG_ON io_uring: Pass whole sqe to commands io_uring: Create a helper to return the SQE size io_uring/rsrc: check for nonconsecutive pages

Linus Torvalds committed 3 years ago

Commit 03e5cb72 parents fc4354c d2b7fa68 files · +47 −41Browse files

Unified · Split

8 files
  1. drivers/block/ublk_drv.c +20 −20
  2. drivers/nvme/host/ioctl.c +1 −1
  3. include/linux/io_uring.h +6 −1
  4. io_uring/io_uring.h +10 −0
  5. io_uring/opdef.c +1 −1
  6. io_uring/rsrc.c +6 −1
  7. io_uring/uring_cmd.c +3 −9
  8. io_uring/uring_cmd.h +0 −8
drivers/block/ublk_drv.c+20−20
@@ -1035,7 +1035,7 @@ static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma) · 1034 unmodified lines
10351035 }
10361036
10371037 static void ublk_commit_completion(struct ublk_device *ub,
1038− struct ublksrv_io_cmd *ub_cmd)
1038+ const struct ublksrv_io_cmd *ub_cmd)
10391039 {
10401040 u32 qid = ub_cmd->q_id, tag = ub_cmd->tag;
10411041 struct ublk_queue *ubq = ublk_get_queue(ub, qid);
@@ -1292,7 +1292,7 @@ static inline int ublk_check_cmd_op(u32 cmd_op) · 250 unmodified lines
12921292
12931293 static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,
12941294 unsigned int issue_flags,
1295− struct ublksrv_io_cmd *ub_cmd)
1295+ const struct ublksrv_io_cmd *ub_cmd)
12961296 {
12971297 struct ublk_device *ub = cmd->file->private_data;
12981298 struct ublk_queue *ubq;
@@ -1399,17 +1399,17 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, · 100 unmodified lines
13991399
14001400 static int ublk_ch_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
14011401 {
1402− struct ublksrv_io_cmd *ub_src = (struct ublksrv_io_cmd *) cmd->cmd;
1403− struct ublksrv_io_cmd ub_cmd;
1404−
14051402 /*
14061403 * Not necessary for async retry, but let's keep it simple and always
14071404 * copy the values to avoid any potential reuse.
14081405 */
1409− ub_cmd.q_id = READ_ONCE(ub_src->q_id);
1410− ub_cmd.tag = READ_ONCE(ub_src->tag);
1411− ub_cmd.result = READ_ONCE(ub_src->result);
1412− ub_cmd.addr = READ_ONCE(ub_src->addr);
1406+ const struct ublksrv_io_cmd *ub_src = io_uring_sqe_cmd(cmd->sqe);
1407+ const struct ublksrv_io_cmd ub_cmd = {
1408+ .q_id = READ_ONCE(ub_src->q_id),
1409+ .tag = READ_ONCE(ub_src->tag),
1410+ .result = READ_ONCE(ub_src->result),
1411+ .addr = READ_ONCE(ub_src->addr)
1412+ };
14131413
14141414 return __ublk_ch_uring_cmd(cmd, issue_flags, &ub_cmd);
14151415 }
@@ -1619,7 +1619,7 @@ static struct ublk_device *ublk_get_device_from_id(int idx) · 203 unmodified lines
16191619
16201620 static int ublk_ctrl_start_dev(struct ublk_device *ub, struct io_uring_cmd *cmd)
16211621 {
1622− struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1622+ const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
16231623 int ublksrv_pid = (int)header->data[0];
16241624 struct gendisk *disk;
16251625 int ret = -EINVAL;
@@ -1682,7 +1682,7 @@ out_unlock: · 56 unmodified lines
16821682 static int ublk_ctrl_get_queue_affinity(struct ublk_device *ub,
16831683 struct io_uring_cmd *cmd)
16841684 {
1685− struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1685+ const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
16861686 void __user *argp = (void __user *)(unsigned long)header->addr;
16871687 cpumask_var_t cpumask;
16881688 unsigned long queue;
@@ -1733,7 +1733,7 @@ static inline void ublk_dump_dev_info(struct ublksrv_ctrl_dev_info *info) · 44 unmodified lines
17331733
17341734 static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd)
17351735 {
1736− struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1736+ const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
17371737 void __user *argp = (void __user *)(unsigned long)header->addr;
17381738 struct ublksrv_ctrl_dev_info info;
17391739 struct ublk_device *ub;
@@ -1910,7 +1910,7 @@ static int ublk_ctrl_del_dev(struct ublk_device **p_ub) · 170 unmodified lines
19101910
19111911 static inline void ublk_ctrl_cmd_dump(struct io_uring_cmd *cmd)
19121912 {
1913− struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1913+ const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
19141914
19151915 pr_devel("%s: cmd_op %x, dev id %d qid %d data %llx buf %llx len %u\n",
19161916 __func__, cmd->cmd_op, header->dev_id, header->queue_id,
@@ -1929,7 +1929,7 @@ static int ublk_ctrl_stop_dev(struct ublk_device *ub) · 12 unmodified lines
19291929 static int ublk_ctrl_get_dev_info(struct ublk_device *ub,
19301930 struct io_uring_cmd *cmd)
19311931 {
1932− struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1932+ const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
19331933 void __user *argp = (void __user *)(unsigned long)header->addr;
19341934
19351935 if (header->len < sizeof(struct ublksrv_ctrl_dev_info) || !header->addr)
@@ -1960,7 +1960,7 @@ static void ublk_ctrl_fill_params_devt(struct ublk_device *ub) · 24 unmodified lines
19601960 static int ublk_ctrl_get_params(struct ublk_device *ub,
19611961 struct io_uring_cmd *cmd)
19621962 {
1963− struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1963+ const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
19641964 void __user *argp = (void __user *)(unsigned long)header->addr;
19651965 struct ublk_params_header ph;
19661966 int ret;
@@ -1991,7 +1991,7 @@ static int ublk_ctrl_get_params(struct ublk_device *ub, · 24 unmodified lines
19911991 static int ublk_ctrl_set_params(struct ublk_device *ub,
19921992 struct io_uring_cmd *cmd)
19931993 {
1994− struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
1994+ const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
19951995 void __user *argp = (void __user *)(unsigned long)header->addr;
19961996 struct ublk_params_header ph;
19971997 int ret = -EFAULT;
@@ -2052,7 +2052,7 @@ static void ublk_queue_reinit(struct ublk_device *ub, struct ublk_queue *ubq) · 54 unmodified lines
20522052 static int ublk_ctrl_start_recovery(struct ublk_device *ub,
20532053 struct io_uring_cmd *cmd)
20542054 {
2055− struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
2055+ const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
20562056 int ret = -EINVAL;
20572057 int i;
20582058
@@ -2094,7 +2094,7 @@ static int ublk_ctrl_start_recovery(struct ublk_device *ub, · 35 unmodified lines
20942094 static int ublk_ctrl_end_recovery(struct ublk_device *ub,
20952095 struct io_uring_cmd *cmd)
20962096 {
2097− struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
2097+ const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
20982098 int ublksrv_pid = (int)header->data[0];
20992099 int ret = -EINVAL;
21002100
@@ -2161,7 +2161,7 @@ exit: · 60 unmodified lines
21612161 static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub,
21622162 struct io_uring_cmd *cmd)
21632163 {
2164− struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
2164+ struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)io_uring_sqe_cmd(cmd->sqe);
21652165 bool unprivileged = ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV;
21662166 void __user *argp = (void __user *)(unsigned long)header->addr;
21672167 char *dev_path = NULL;
@@ -2240,7 +2240,7 @@ exit: · 72 unmodified lines
22402240 static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd,
22412241 unsigned int issue_flags)
22422242 {
2243− struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;
2243+ const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
22442244 struct ublk_device *ub = NULL;
22452245 u32 cmd_op = cmd->cmd_op;
22462246 int ret = -EINVAL;
drivers/nvme/host/ioctl.c+1−1
@@ -552,7 +552,7 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, · 551 unmodified lines
552552 struct io_uring_cmd *ioucmd, unsigned int issue_flags, bool vec)
553553 {
554554 struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
555− const struct nvme_uring_cmd *cmd = ioucmd->cmd;
555+ const struct nvme_uring_cmd *cmd = io_uring_sqe_cmd(ioucmd->sqe);
556556 struct request_queue *q = ns ? ns->queue : ctrl->admin_q;
557557 struct nvme_uring_data d;
558558 struct nvme_command c;
include/linux/io_uring.h+6−1
@@ -24,7 +24,7 @@ enum io_uring_cmd_flags { · 23 unmodified lines
2424
2525 struct io_uring_cmd {
2626 struct file *file;
27− const void *cmd;
27+ const struct io_uring_sqe *sqe;
2828 union {
2929 /* callback to defer completions to task context */
3030 void (*task_work_cb)(struct io_uring_cmd *cmd, unsigned);
@@ -66,6 +66,11 @@ static inline void io_uring_free(struct task_struct *tsk) · 35 unmodified lines
6666 if (tsk->io_uring)
6767 __io_uring_free(tsk);
6868 }
69+
70+static inline const void *io_uring_sqe_cmd(const struct io_uring_sqe *sqe)
71+{
72+ return sqe->cmd;
73+}
6974 #else
7075 static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
7176 struct iov_iter *iter, void *ioucmd)
io_uring/io_uring.h+10−0
@@ -394,4 +394,14 @@ static inline void io_req_queue_tw_complete(struct io_kiocb *req, s32 res) · 393 unmodified lines
394394 io_req_task_work_add(req);
395395 }
396396
397+/*
398+ * IORING_SETUP_SQE128 contexts allocate twice the normal SQE size for each
399+ * slot.
400+ */
401+static inline size_t uring_sqe_size(struct io_ring_ctx *ctx)
402+{
403+ if (ctx->flags & IORING_SETUP_SQE128)
404+ return 2 * sizeof(struct io_uring_sqe);
405+ return sizeof(struct io_uring_sqe);
406+}
397407 #endif
io_uring/opdef.c+1−1
@@ -627,7 +627,7 @@ const struct io_cold_def io_cold_defs[] = { · 626 unmodified lines
627627 },
628628 [IORING_OP_URING_CMD] = {
629629 .name = "URING_CMD",
630− .async_size = uring_cmd_pdu_size(1),
630+ .async_size = 2 * sizeof(struct io_uring_sqe),
631631 .prep_async = io_uring_cmd_prep_async,
632632 },
633633 [IORING_OP_SEND_ZC] = {
io_uring/rsrc.c+6−1
@@ -1116,7 +1116,12 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, · 1115 unmodified lines
11161116 if (nr_pages > 1) {
11171117 folio = page_folio(pages[0]);
11181118 for (i = 1; i < nr_pages; i++) {
1119− if (page_folio(pages[i]) != folio) {
1119+ /*
1120+ * Pages must be consecutive and on the same folio for
1121+ * this to work
1122+ */
1123+ if (page_folio(pages[i]) != folio ||
1124+ pages[i] != pages[i - 1] + 1) {
11201125 folio = NULL;
11211126 break;
11221127 }
io_uring/uring_cmd.c+3−9
@@ -69,15 +69,9 @@ EXPORT_SYMBOL_GPL(io_uring_cmd_done); · 68 unmodified lines
6969 int io_uring_cmd_prep_async(struct io_kiocb *req)
7070 {
7171 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
72− size_t cmd_size;
7372
74− BUILD_BUG_ON(uring_cmd_pdu_size(0) != 16);
75− BUILD_BUG_ON(uring_cmd_pdu_size(1) != 80);
76−
77− cmd_size = uring_cmd_pdu_size(req->ctx->flags & IORING_SETUP_SQE128);
78−
79− memcpy(req->async_data, ioucmd->cmd, cmd_size);
80− ioucmd->cmd = req->async_data;
73+ memcpy(req->async_data, ioucmd->sqe, uring_sqe_size(req->ctx));
74+ ioucmd->sqe = req->async_data;
8175 return 0;
8276 }
8377
@@ -103,7 +97,7 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 19 unmodified lines
10397 req->imu = ctx->user_bufs[index];
10498 io_req_set_rsrc_node(req, ctx, 0);
10599 }
106− ioucmd->cmd = sqe->cmd;
100+ ioucmd->sqe = sqe;
107101 ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
108102 return 0;
109103 }
io_uring/uring_cmd.h+0−8
@@ -3,11 +3,3 @@ · 2 unmodified lines
33 int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags);
44 int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
55 int io_uring_cmd_prep_async(struct io_kiocb *req);
6−
7−/*
8− * The URING_CMD payload starts at 'cmd' in the first sqe, and continues into
9− * the following sqe if SQE128 is used.
10− */
11−#define uring_cmd_pdu_size(is_sqe128) \
12− ((1 + !!(is_sqe128)) * sizeof(struct io_uring_sqe) - \
13− offsetof(struct io_uring_sqe, cmd))

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t