Currently, UBLK_IO_REGISTER_IO_BUF and UBLK_IO_UNREGISTER_IO_BUF are only permitted on the ublk_io's daemon task. But this restriction is unnecessary. ublk_register_io_buf() calls __ublk_check_and_get_req() to look up the request from the tagset and atomically take a reference on the request without accessing the ublk_io. ublk_unregister_io_buf() doesn't use the q_id or tag at all. So allow these opcodes even on tasks other than io->task. Handle UBLK_IO_UNREGISTER_IO_BUF before obtaining the ubq and io since the buffer index being unregistered is not necessarily related to the specified q_id and tag. Add a feature flag UBLK_F_BUF_REG_OFF_DAEMON that userspace can use to determine whether the kernel supports off-daemon buffer registration.
Unified · Split
| @@ -70,7 +70,8 @@ · 69 unmodified lines | ||
| 70 | 70 | | UBLK_F_UPDATE_SIZE \ |
| 71 | 71 | | UBLK_F_AUTO_BUF_REG \ |
| 72 | 72 | | UBLK_F_QUIESCE \ |
| 73 | − | UBLK_F_PER_IO_DAEMON) | |
| 73 | + | UBLK_F_PER_IO_DAEMON \ | |
| 74 | + | UBLK_F_BUF_REG_OFF_DAEMON) | |
| 74 | 75 | |
| 75 | 76 | #define UBLK_F_ALL_RECOVERY_FLAGS (UBLK_F_USER_RECOVERY \ |
| 76 | 77 | | UBLK_F_USER_RECOVERY_REISSUE \ |
| @@ -2204,6 +2205,14 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, · 2127 unmodified lines | ||
| 2204 | 2205 | if (ret) |
| 2205 | 2206 | goto out; |
| 2206 | 2207 | |
| 2208 | + /* | |
| 2209 | + * io_buffer_unregister_bvec() doesn't access the ubq or io, | |
| 2210 | + * so no need to validate the q_id, tag, or task | |
| 2211 | + */ | |
| 2212 | + if (_IOC_NR(cmd_op) == UBLK_IO_UNREGISTER_IO_BUF) | |
| 2213 | + return ublk_unregister_io_buf(cmd, ub, ub_cmd->addr, | |
| 2214 | + issue_flags); | |
| 2215 | + | |
| 2207 | 2216 | ret = -EINVAL; |
| 2208 | 2217 | if (ub_cmd->q_id >= ub->dev_info.nr_hw_queues) |
| 2209 | 2218 | goto out; |
| @@ -2224,8 +2233,17 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, · 14 unmodified lines | ||
| 2224 | 2233 | return -EIOCBQUEUED; |
| 2225 | 2234 | } |
| 2226 | 2235 | |
| 2227 | − if (READ_ONCE(io->task) != current) | |
| 2236 | + if (READ_ONCE(io->task) != current) { | |
| 2237 | + /* | |
| 2238 | + * ublk_register_io_buf() accesses only the io's refcount, | |
| 2239 | + * so can be handled on any task | |
| 2240 | + */ | |
| 2241 | + if (_IOC_NR(cmd_op) == UBLK_IO_REGISTER_IO_BUF) | |
| 2242 | + return ublk_register_io_buf(cmd, ubq, io, ub_cmd->addr, | |
| 2243 | + issue_flags); | |
| 2244 | + | |
| 2228 | 2245 | goto out; |
| 2246 | + } | |
| 2229 | 2247 | |
| 2230 | 2248 | /* there is pending io cmd, something must be wrong */ |
| 2231 | 2249 | if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)) { |
| @@ -2244,8 +2262,6 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd, · 12 unmodified lines | ||
| 2244 | 2262 | switch (_IOC_NR(cmd_op)) { |
| 2245 | 2263 | case UBLK_IO_REGISTER_IO_BUF: |
| 2246 | 2264 | return ublk_register_io_buf(cmd, ubq, io, ub_cmd->addr, issue_flags); |
| 2247 | − case UBLK_IO_UNREGISTER_IO_BUF: | |
| 2248 | − return ublk_unregister_io_buf(cmd, ub, ub_cmd->addr, issue_flags); | |
| 2249 | 2265 | case UBLK_IO_COMMIT_AND_FETCH_REQ: |
| 2250 | 2266 | ret = ublk_commit_and_fetch(ubq, io, cmd, ub_cmd, issue_flags); |
| 2251 | 2267 | if (ret) |
| @@ -2961,7 +2977,8 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) · 709 unmodified lines | ||
| 2961 | 2977 | |
| 2962 | 2978 | ub->dev_info.flags |= UBLK_F_CMD_IOCTL_ENCODE | |
| 2963 | 2979 | UBLK_F_URING_CMD_COMP_IN_TASK | |
| 2964 | − UBLK_F_PER_IO_DAEMON; | |
| 2980 | + UBLK_F_PER_IO_DAEMON | | |
| 2981 | + UBLK_F_BUF_REG_OFF_DAEMON; | |
| 2965 | 2982 | |
| 2966 | 2983 | /* GET_DATA isn't needed any more with USER_COPY or ZERO COPY */ |
| 2967 | 2984 | if (ub->dev_info.flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY | |
| @@ -301,6 +301,16 @@ · 300 unmodified lines | ||
| 301 | 301 | */ |
| 302 | 302 | #define UBLK_F_PER_IO_DAEMON (1ULL << 13) |
| 303 | 303 | |
| 304 | +/* | |
| 305 | + * If this feature is set, UBLK_U_IO_REGISTER_IO_BUF/UBLK_U_IO_UNREGISTER_IO_BUF | |
| 306 | + * can be issued for an I/O on any task. q_id and tag are also ignored in | |
| 307 | + * UBLK_U_IO_UNREGISTER_IO_BUF's ublksrv_io_cmd. | |
| 308 | + * If it is unset, zero-copy buffers can only be registered and unregistered by | |
| 309 | + * the I/O's daemon task. The q_id and tag of the registered buffer are required | |
| 310 | + * in UBLK_U_IO_UNREGISTER_IO_BUF's ublksrv_io_cmd. | |
| 311 | + */ | |
| 312 | +#define UBLK_F_BUF_REG_OFF_DAEMON (1ULL << 14) | |
| 313 | + | |
| 304 | 314 | /* device state */ |
| 305 | 315 | #define UBLK_S_DEV_DEAD 0 |
| 306 | 316 | #define UBLK_S_DEV_LIVE 1 |