Currently, UBLK_IO_REGISTER_IO_BUF and UBLK_IO_UNREGISTER_IO_BUF are only permitted on the ublk_io's daemon task. But this restriction is unnecessary. ublk_register_io_buf() calls __ublk_check_and_get_req() to look up the request from the tagset and atomically take a reference on the request without accessing the ublk_io. ublk_unregister_io_buf() doesn't use the q_id or tag at all. So allow these opcodes even on tasks other than io->task. Handle UBLK_IO_UNREGISTER_IO_BUF before obtaining the ubq and io since the buffer index being unregistered is not necessarily related to the specified q_id and tag. Add a feature flag UBLK_F_BUF_REG_OFF_DAEMON that userspace can use to determine whether the kernel supports off-daemon buffer registration.
Unified · Split
| @@ -70,7 +70,8 @@ · 69 unmodified lines | |||
| 70 | | UBLK_F_UPDATE_SIZE \ | 70 | | UBLK_F_UPDATE_SIZE \ |
| 71 | | UBLK_F_AUTO_BUF_REG \ | 71 | | UBLK_F_AUTO_BUF_REG \ |
| 72 | | UBLK_F_QUIESCE \ | 72 | | UBLK_F_QUIESCE \ |
| 73 | | UBLK_F_PER_IO_DAEMON) | 73 | | UBLK_F_PER_IO_DAEMON \ |
| 74 | | UBLK_F_BUF_REG_OFF_DAEMON) | ||
| 74 | 75 | ||
| 75 | #define UBLK_F_ALL_RECOVERY_FLAGS (UBLK_F_USER_RECOVERY \ | 76 | #define UBLK_F_ALL_RECOVERY_FLAGS (UBLK_F_USER_RECOVERY \ |
| 76 | | UBLK_F_USER_RECOVERY_REISSUE \ | 77 | | UBLK_F_USER_RECOVERY_REISSUE \ |
| @@ -2204,6 +2205,14 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, · 2127 unmodified lines | |||
| 2204 | if (ret) | 2205 | if (ret) |
| 2205 | goto out; | 2206 | goto out; |
| 2206 | 2207 | ||
| 2208 | /* | ||
| 2209 | * io_buffer_unregister_bvec() doesn't access the ubq or io, | ||
| 2210 | * so no need to validate the q_id, tag, or task | ||
| 2211 | */ | ||
| 2212 | if (_IOC_NR(cmd_op) == UBLK_IO_UNREGISTER_IO_BUF) | ||
| 2213 | return ublk_unregister_io_buf(cmd, ub, ub_cmd->addr, | ||
| 2214 | issue_flags); | ||
| 2215 | |||
| 2207 | ret = -EINVAL; | 2216 | ret = -EINVAL; |
| 2208 | if (ub_cmd->q_id >= ub->dev_info.nr_hw_queues) | 2217 | if (ub_cmd->q_id >= ub->dev_info.nr_hw_queues) |
| 2209 | goto out; | 2218 | goto out; |
| @@ -2224,8 +2233,17 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, · 14 unmodified lines | |||
| 2224 | return -EIOCBQUEUED; | 2233 | return -EIOCBQUEUED; |
| 2225 | } | 2234 | } |
| 2226 | 2235 | ||
| 2227 | if (READ_ONCE(io->task) != current) | 2236 | if (READ_ONCE(io->task) != current) { |
| 2237 | /* | ||
| 2238 | * ublk_register_io_buf() accesses only the io's refcount, | ||
| 2239 | * so can be handled on any task | ||
| 2240 | */ | ||
| 2241 | if (_IOC_NR(cmd_op) == UBLK_IO_REGISTER_IO_BUF) | ||
| 2242 | return ublk_register_io_buf(cmd, ubq, io, ub_cmd->addr, | ||
| 2243 | issue_flags); | ||
| 2244 | |||
| 2228 | goto out; | 2245 | goto out; |
| 2246 | } | ||
| 2229 | 2247 | ||
| 2230 | /* there is pending io cmd, something must be wrong */ | 2248 | /* there is pending io cmd, something must be wrong */ |
| 2231 | if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)) { | 2249 | if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)) { |
| @@ -2244,8 +2262,6 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, · 12 unmodified lines | |||
| 2244 | switch (_IOC_NR(cmd_op)) { | 2262 | switch (_IOC_NR(cmd_op)) { |
| 2245 | case UBLK_IO_REGISTER_IO_BUF: | 2263 | case UBLK_IO_REGISTER_IO_BUF: |
| 2246 | return ublk_register_io_buf(cmd, ubq, io, ub_cmd->addr, issue_flags); | 2264 | return ublk_register_io_buf(cmd, ubq, io, ub_cmd->addr, issue_flags); |
| 2247 | case UBLK_IO_UNREGISTER_IO_BUF: | ||
| 2248 | return ublk_unregister_io_buf(cmd, ub, ub_cmd->addr, issue_flags); | ||
| 2249 | case UBLK_IO_COMMIT_AND_FETCH_REQ: | 2265 | case UBLK_IO_COMMIT_AND_FETCH_REQ: |
| 2250 | ret = ublk_commit_and_fetch(ubq, io, cmd, ub_cmd, issue_flags); | 2266 | ret = ublk_commit_and_fetch(ubq, io, cmd, ub_cmd, issue_flags); |
| 2251 | if (ret) | 2267 | if (ret) |
| @@ -2961,7 +2977,8 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) · 709 unmodified lines | |||
| 2961 | 2977 | ||
| 2962 | ub->dev_info.flags |= UBLK_F_CMD_IOCTL_ENCODE | | 2978 | ub->dev_info.flags |= UBLK_F_CMD_IOCTL_ENCODE | |
| 2963 | UBLK_F_URING_CMD_COMP_IN_TASK | | 2979 | UBLK_F_URING_CMD_COMP_IN_TASK | |
| 2964 | UBLK_F_PER_IO_DAEMON; | 2980 | UBLK_F_PER_IO_DAEMON | |
| 2981 | UBLK_F_BUF_REG_OFF_DAEMON; | ||
| 2965 | 2982 | ||
| 2966 | /* GET_DATA isn't needed any more with USER_COPY or ZERO COPY */ | 2983 | /* GET_DATA isn't needed any more with USER_COPY or ZERO COPY */ |
| 2967 | if (ub->dev_info.flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY | | 2984 | if (ub->dev_info.flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY | |
| @@ -301,6 +301,16 @@ · 300 unmodified lines | |||
| 301 | */ | 301 | */ |
| 302 | #define UBLK_F_PER_IO_DAEMON (1ULL << 13) | 302 | #define UBLK_F_PER_IO_DAEMON (1ULL << 13) |
| 303 | 303 | ||
| 304 | /* | ||
| 305 | * If this feature is set, UBLK_U_IO_REGISTER_IO_BUF/UBLK_U_IO_UNREGISTER_IO_BUF | ||
| 306 | * can be issued for an I/O on any task. q_id and tag are also ignored in | ||
| 307 | * UBLK_U_IO_UNREGISTER_IO_BUF's ublksrv_io_cmd. | ||
| 308 | * If it is unset, zero-copy buffers can only be registered and unregistered by | ||
| 309 | * the I/O's daemon task. The q_id and tag of the registered buffer are required | ||
| 310 | * in UBLK_U_IO_UNREGISTER_IO_BUF's ublksrv_io_cmd. | ||
| 311 | */ | ||
| 312 | #define UBLK_F_BUF_REG_OFF_DAEMON (1ULL << 14) | ||
| 313 | |||
| 304 | /* device state */ | 314 | /* device state */ |
| 305 | #define UBLK_S_DEV_DEAD 0 | 315 | #define UBLK_S_DEV_DEAD 0 |
| 306 | #define UBLK_S_DEV_LIVE 1 | 316 | #define UBLK_S_DEV_LIVE 1 |