History

Merge tag 'for-6.4/io_uring-2023-05-07' of git://git.kernel.dk/linux

Pull more io_uring updates from Jens Axboe: "Nothing major in here, just two different parts: - A small series from Breno that enables passing the full SQE down for ->uring_cmd(). This is a prerequisite for enabling full network socket operations. Queued up a bit late because of some stylistic concerns that got resolved, would be nice to have this in 6.4-rc1 so the dependent work will be easier to handle for 6.5. - Fix for the huge page coalescing, which was a regression introduced in the 6.3 kernel release (Tobias)" * tag 'for-6.4/io_uring-2023-05-07' of git://git.kernel.dk/linux: io_uring: Remove unnecessary BUILD_BUG_ON io_uring: Pass whole sqe to commands io_uring: Create a helper to return the SQE size io_uring/rsrc: check for nonconsecutive pages

Linus Torvalds committed 3 years ago

Commit 03e5cb72 parents fc4354c d2b7fa68 files · +47 −41Browse files

Unified · Split

8 files
  1. drivers/block/ublk_drv.c +20 −20
  2. drivers/nvme/host/ioctl.c +1 −1
  3. include/linux/io_uring.h +6 −1
  4. io_uring/io_uring.h +10 −0
  5. io_uring/opdef.c +1 −1
  6. io_uring/rsrc.c +6 −1
  7. io_uring/uring_cmd.c +3 −9
  8. io_uring/uring_cmd.h +0 −8
drivers/block/ublk_drv.c+20−20
@@ -1035,7 +1035,7 @@ static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma) · 1034 unmodified lines
1035}1035}
1036 1036
1037static void ublk_commit_completion(struct ublk_device *ub,1037static void ublk_commit_completion(struct ublk_device *ub,
1038 struct ublksrv_io_cmd *ub_cmd)1038 const struct ublksrv_io_cmd *ub_cmd)
1039{1039{
1040 u32 qid = ub_cmd->q_id, tag = ub_cmd->tag;1040 u32 qid = ub_cmd->q_id, tag = ub_cmd->tag;
1041 struct ublk_queue *ubq = ublk_get_queue(ub, qid);1041 struct ublk_queue *ubq = ublk_get_queue(ub, qid);
@@ -1292,7 +1292,7 @@ static inline int ublk_check_cmd_op(u32 cmd_op) · 250 unmodified lines
1292 1292
1293static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,1293static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,
1294 unsigned int issue_flags,1294 unsigned int issue_flags,
1295 struct ublksrv_io_cmd *ub_cmd)1295 const struct ublksrv_io_cmd *ub_cmd)
1296{1296{
1297 struct ublk_device *ub = cmd->file->private_data;1297 struct ublk_device *ub = cmd->file->private_data;
1298 struct ublk_queue *ubq;1298 struct ublk_queue *ubq;
@@ -1399,17 +1399,17 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, · 100 unmodified lines
1399 1399
1400static int ublk_ch_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)1400static int ublk_ch_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
1401{1401{
1402 struct ublksrv_io_cmd *ub_src = (struct ublksrv_io_cmd *) cmd->cmd;
1403 struct ublksrv_io_cmd ub_cmd;
1404
1405 /*1402 /*
1406 * Not necessary for async retry, but let's keep it simple and always1403 * Not necessary for async retry, but let's keep it simple and always
1407 * copy the values to avoid any potential reuse.1404 * copy the values to avoid any potential reuse.
1408 */1405 */
1409 ub_cmd.q_id = READ_ONCE(ub_src->q_id);1406 const struct ublksrv_io_cmd *ub_src = io_uring_sqe_cmd(cmd->sqe);
1410 ub_cmd.tag = READ_ONCE(ub_src->tag);1407 const struct ublksrv_io_cmd ub_cmd = {
1411 ub_cmd.result = READ_ONCE(ub_src->result);1408 .q_id = READ_ONCE(ub_src->q_id),
1412 ub_cmd.addr = READ_ONCE(ub_src->addr);1409 .tag = READ_ONCE(ub_src->tag),
1410 .result = READ_ONCE(ub_src->result),
1411 .addr = READ_ONCE(ub_src->addr)
1412 };
1413 1413
1414 return __ublk_ch_uring_cmd(cmd, issue_flags, &ub_cmd);1414 return __ublk_ch_uring_cmd(cmd, issue_flags, &ub_cmd);
1415}1415}
@@ -1619,7 +1619,7 @@ static struct ublk_device *ublk_get_device_from_id(int idx) · 203 unmodified lines
1619 1619
1620static int ublk_ctrl_start_dev(struct ublk_device *ub, struct io_uring_cmd *cmd)1620static int ublk_ctrl_start_dev(struct ublk_device *ub, struct io_uring_cmd *cmd)
1621{1621{
1622 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;1622 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
1623 int ublksrv_pid = (int)header->data[0];1623 int ublksrv_pid = (int)header->data[0];
1624 struct gendisk *disk;1624 struct gendisk *disk;
1625 int ret = -EINVAL;1625 int ret = -EINVAL;
@@ -1682,7 +1682,7 @@ out_unlock: · 56 unmodified lines
1682static int ublk_ctrl_get_queue_affinity(struct ublk_device *ub,1682static int ublk_ctrl_get_queue_affinity(struct ublk_device *ub,
1683 struct io_uring_cmd *cmd)1683 struct io_uring_cmd *cmd)
1684{1684{
1685 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;1685 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
1686 void __user *argp = (void __user *)(unsigned long)header->addr;1686 void __user *argp = (void __user *)(unsigned long)header->addr;
1687 cpumask_var_t cpumask;1687 cpumask_var_t cpumask;
1688 unsigned long queue;1688 unsigned long queue;
@@ -1733,7 +1733,7 @@ static inline void ublk_dump_dev_info(struct ublksrv_ctrl_dev_info *info) · 44 unmodified lines
1733 1733
1734static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd)1734static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd)
1735{1735{
1736 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;1736 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
1737 void __user *argp = (void __user *)(unsigned long)header->addr;1737 void __user *argp = (void __user *)(unsigned long)header->addr;
1738 struct ublksrv_ctrl_dev_info info;1738 struct ublksrv_ctrl_dev_info info;
1739 struct ublk_device *ub;1739 struct ublk_device *ub;
@@ -1910,7 +1910,7 @@ static int ublk_ctrl_del_dev(struct ublk_device **p_ub) · 170 unmodified lines
1910 1910
1911static inline void ublk_ctrl_cmd_dump(struct io_uring_cmd *cmd)1911static inline void ublk_ctrl_cmd_dump(struct io_uring_cmd *cmd)
1912{1912{
1913 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;1913 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
1914 1914
1915 pr_devel("%s: cmd_op %x, dev id %d qid %d data %llx buf %llx len %u\n",1915 pr_devel("%s: cmd_op %x, dev id %d qid %d data %llx buf %llx len %u\n",
1916 __func__, cmd->cmd_op, header->dev_id, header->queue_id,1916 __func__, cmd->cmd_op, header->dev_id, header->queue_id,
@@ -1929,7 +1929,7 @@ static int ublk_ctrl_stop_dev(struct ublk_device *ub) · 12 unmodified lines
1929static int ublk_ctrl_get_dev_info(struct ublk_device *ub,1929static int ublk_ctrl_get_dev_info(struct ublk_device *ub,
1930 struct io_uring_cmd *cmd)1930 struct io_uring_cmd *cmd)
1931{1931{
1932 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;1932 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
1933 void __user *argp = (void __user *)(unsigned long)header->addr;1933 void __user *argp = (void __user *)(unsigned long)header->addr;
1934 1934
1935 if (header->len < sizeof(struct ublksrv_ctrl_dev_info) || !header->addr)1935 if (header->len < sizeof(struct ublksrv_ctrl_dev_info) || !header->addr)
@@ -1960,7 +1960,7 @@ static void ublk_ctrl_fill_params_devt(struct ublk_device *ub) · 24 unmodified lines
1960static int ublk_ctrl_get_params(struct ublk_device *ub,1960static int ublk_ctrl_get_params(struct ublk_device *ub,
1961 struct io_uring_cmd *cmd)1961 struct io_uring_cmd *cmd)
1962{1962{
1963 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;1963 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
1964 void __user *argp = (void __user *)(unsigned long)header->addr;1964 void __user *argp = (void __user *)(unsigned long)header->addr;
1965 struct ublk_params_header ph;1965 struct ublk_params_header ph;
1966 int ret;1966 int ret;
@@ -1991,7 +1991,7 @@ static int ublk_ctrl_get_params(struct ublk_device *ub, · 24 unmodified lines
1991static int ublk_ctrl_set_params(struct ublk_device *ub,1991static int ublk_ctrl_set_params(struct ublk_device *ub,
1992 struct io_uring_cmd *cmd)1992 struct io_uring_cmd *cmd)
1993{1993{
1994 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;1994 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
1995 void __user *argp = (void __user *)(unsigned long)header->addr;1995 void __user *argp = (void __user *)(unsigned long)header->addr;
1996 struct ublk_params_header ph;1996 struct ublk_params_header ph;
1997 int ret = -EFAULT;1997 int ret = -EFAULT;
@@ -2052,7 +2052,7 @@ static void ublk_queue_reinit(struct ublk_device *ub, struct ublk_queue *ubq) · 54 unmodified lines
2052static int ublk_ctrl_start_recovery(struct ublk_device *ub,2052static int ublk_ctrl_start_recovery(struct ublk_device *ub,
2053 struct io_uring_cmd *cmd)2053 struct io_uring_cmd *cmd)
2054{2054{
2055 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;2055 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
2056 int ret = -EINVAL;2056 int ret = -EINVAL;
2057 int i;2057 int i;
2058 2058
@@ -2094,7 +2094,7 @@ static int ublk_ctrl_start_recovery(struct ublk_device *ub, · 35 unmodified lines
2094static int ublk_ctrl_end_recovery(struct ublk_device *ub,2094static int ublk_ctrl_end_recovery(struct ublk_device *ub,
2095 struct io_uring_cmd *cmd)2095 struct io_uring_cmd *cmd)
2096{2096{
2097 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;2097 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
2098 int ublksrv_pid = (int)header->data[0];2098 int ublksrv_pid = (int)header->data[0];
2099 int ret = -EINVAL;2099 int ret = -EINVAL;
2100 2100
@@ -2161,7 +2161,7 @@ exit: · 60 unmodified lines
2161static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub,2161static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub,
2162 struct io_uring_cmd *cmd)2162 struct io_uring_cmd *cmd)
2163{2163{
2164 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;2164 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)io_uring_sqe_cmd(cmd->sqe);
2165 bool unprivileged = ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV;2165 bool unprivileged = ub->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV;
2166 void __user *argp = (void __user *)(unsigned long)header->addr;2166 void __user *argp = (void __user *)(unsigned long)header->addr;
2167 char *dev_path = NULL;2167 char *dev_path = NULL;
@@ -2240,7 +2240,7 @@ exit: · 72 unmodified lines
2240static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd,2240static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd,
2241 unsigned int issue_flags)2241 unsigned int issue_flags)
2242{2242{
2243 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd;2243 const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
2244 struct ublk_device *ub = NULL;2244 struct ublk_device *ub = NULL;
2245 u32 cmd_op = cmd->cmd_op;2245 u32 cmd_op = cmd->cmd_op;
2246 int ret = -EINVAL;2246 int ret = -EINVAL;
drivers/nvme/host/ioctl.c+1−1
@@ -552,7 +552,7 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, · 551 unmodified lines
552 struct io_uring_cmd *ioucmd, unsigned int issue_flags, bool vec)552 struct io_uring_cmd *ioucmd, unsigned int issue_flags, bool vec)
553{553{
554 struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);554 struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
555 const struct nvme_uring_cmd *cmd = ioucmd->cmd;555 const struct nvme_uring_cmd *cmd = io_uring_sqe_cmd(ioucmd->sqe);
556 struct request_queue *q = ns ? ns->queue : ctrl->admin_q;556 struct request_queue *q = ns ? ns->queue : ctrl->admin_q;
557 struct nvme_uring_data d;557 struct nvme_uring_data d;
558 struct nvme_command c;558 struct nvme_command c;
include/linux/io_uring.h+6−1
@@ -24,7 +24,7 @@ enum io_uring_cmd_flags { · 23 unmodified lines
24 24
25struct io_uring_cmd {25struct io_uring_cmd {
26 struct file *file;26 struct file *file;
27 const void *cmd;27 const struct io_uring_sqe *sqe;
28 union {28 union {
29 /* callback to defer completions to task context */29 /* callback to defer completions to task context */
30 void (*task_work_cb)(struct io_uring_cmd *cmd, unsigned);30 void (*task_work_cb)(struct io_uring_cmd *cmd, unsigned);
@@ -66,6 +66,11 @@ static inline void io_uring_free(struct task_struct *tsk) · 35 unmodified lines
66 if (tsk->io_uring)66 if (tsk->io_uring)
67 __io_uring_free(tsk);67 __io_uring_free(tsk);
68}68}
69
70static inline const void *io_uring_sqe_cmd(const struct io_uring_sqe *sqe)
71{
72 return sqe->cmd;
73}
69#else74#else
70static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,75static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
71 struct iov_iter *iter, void *ioucmd)76 struct iov_iter *iter, void *ioucmd)
io_uring/io_uring.h+10−0
@@ -394,4 +394,14 @@ static inline void io_req_queue_tw_complete(struct io_kiocb *req, s32 res) · 393 unmodified lines
394 io_req_task_work_add(req);394 io_req_task_work_add(req);
395}395}
396 396
397/*
398 * IORING_SETUP_SQE128 contexts allocate twice the normal SQE size for each
399 * slot.
400 */
401static inline size_t uring_sqe_size(struct io_ring_ctx *ctx)
402{
403 if (ctx->flags & IORING_SETUP_SQE128)
404 return 2 * sizeof(struct io_uring_sqe);
405 return sizeof(struct io_uring_sqe);
406}
397#endif407#endif
io_uring/opdef.c+1−1
@@ -627,7 +627,7 @@ const struct io_cold_def io_cold_defs[] = { · 626 unmodified lines
627 },627 },
628 [IORING_OP_URING_CMD] = {628 [IORING_OP_URING_CMD] = {
629 .name = "URING_CMD",629 .name = "URING_CMD",
630 .async_size = uring_cmd_pdu_size(1),630 .async_size = 2 * sizeof(struct io_uring_sqe),
631 .prep_async = io_uring_cmd_prep_async,631 .prep_async = io_uring_cmd_prep_async,
632 },632 },
633 [IORING_OP_SEND_ZC] = {633 [IORING_OP_SEND_ZC] = {
io_uring/rsrc.c+6−1
@@ -1116,7 +1116,12 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, · 1115 unmodified lines
1116 if (nr_pages > 1) {1116 if (nr_pages > 1) {
1117 folio = page_folio(pages[0]);1117 folio = page_folio(pages[0]);
1118 for (i = 1; i < nr_pages; i++) {1118 for (i = 1; i < nr_pages; i++) {
1119 if (page_folio(pages[i]) != folio) {1119 /*
1120 * Pages must be consecutive and on the same folio for
1121 * this to work
1122 */
1123 if (page_folio(pages[i]) != folio ||
1124 pages[i] != pages[i - 1] + 1) {
1120 folio = NULL;1125 folio = NULL;
1121 break;1126 break;
1122 }1127 }
io_uring/uring_cmd.c+3−9
@@ -69,15 +69,9 @@ EXPORT_SYMBOL_GPL(io_uring_cmd_done); · 68 unmodified lines
69int io_uring_cmd_prep_async(struct io_kiocb *req)69int io_uring_cmd_prep_async(struct io_kiocb *req)
70{70{
71 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);71 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
72 size_t cmd_size;
73 72
74 BUILD_BUG_ON(uring_cmd_pdu_size(0) != 16);73 memcpy(req->async_data, ioucmd->sqe, uring_sqe_size(req->ctx));
75 BUILD_BUG_ON(uring_cmd_pdu_size(1) != 80);74 ioucmd->sqe = req->async_data;
76
77 cmd_size = uring_cmd_pdu_size(req->ctx->flags & IORING_SETUP_SQE128);
78
79 memcpy(req->async_data, ioucmd->cmd, cmd_size);
80 ioucmd->cmd = req->async_data;
81 return 0;75 return 0;
82}76}
83 77
@@ -103,7 +97,7 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 19 unmodified lines
103 req->imu = ctx->user_bufs[index];97 req->imu = ctx->user_bufs[index];
104 io_req_set_rsrc_node(req, ctx, 0);98 io_req_set_rsrc_node(req, ctx, 0);
105 }99 }
106 ioucmd->cmd = sqe->cmd;100 ioucmd->sqe = sqe;
107 ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);101 ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
108 return 0;102 return 0;
109}103}
io_uring/uring_cmd.h+0−8
@@ -3,11 +3,3 @@ · 2 unmodified lines
3int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags);3int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags);
4int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);4int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
5int io_uring_cmd_prep_async(struct io_kiocb *req);5int io_uring_cmd_prep_async(struct io_kiocb *req);
6
7/*
8 * The URING_CMD payload starts at 'cmd' in the first sqe, and continues into
9 * the following sqe if SQE128 is used.
10 */
11#define uring_cmd_pdu_size(is_sqe128) \
12 ((1 + !!(is_sqe128)) * sizeof(struct io_uring_sqe) - \
13 offsetof(struct io_uring_sqe, cmd))

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t