| @@ -0,0 +1,458 @@ |
| | 1 | // SPDX-License-Identifier: GPL-2.0 |
| | 2 | /* |
| | 3 | * NVMe Over Fabrics Target Passthrough command implementation. |
| | 4 | * |
| | 5 | * Copyright (c) 2017-2018 Western Digital Corporation or its |
| | 6 | * affiliates. |
| | 7 | * Copyright (c) 2019-2020, Eideticom Inc. |
| | 8 | * |
| | 9 | */ |
| | 10 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
| | 11 | #include <linux/module.h> |
| | 12 | |
| | 13 | #include "../host/nvme.h" |
| | 14 | #include "nvmet.h" |
| | 15 | |
| | 16 | MODULE_IMPORT_NS(NVME_TARGET_PASSTHRU); |
| | 17 | |
| | 18 | static u16 nvmet_passthru_override_id_ctrl(struct nvmet_req *req) |
| | 19 | { |
| | 20 | struct nvmet_ctrl *ctrl = req->sq->ctrl; |
| | 21 | struct nvme_ctrl *pctrl = ctrl->subsys->passthru_ctrl; |
| | 22 | u16 status = NVME_SC_SUCCESS; |
| | 23 | struct nvme_id_ctrl *id; |
| | 24 | u32 max_hw_sectors; |
| | 25 | int page_shift; |
| | 26 | |
| | 27 | id = kzalloc(sizeof(*id), GFP_KERNEL); |
| | 28 | if (!id) |
| | 29 | return NVME_SC_INTERNAL; |
| | 30 | |
| | 31 | status = nvmet_copy_from_sgl(req, 0, id, sizeof(*id)); |
| | 32 | if (status) |
| | 33 | goto out_free; |
| | 34 | |
| | 35 | id->cntlid = cpu_to_le16(ctrl->cntlid); |
| | 36 | id->ver = cpu_to_le32(ctrl->subsys->ver); |
| | 37 | |
| | 38 | /* |
| | 39 | * The passthru NVMe driver may have a limit on the number of segments |
| | 40 | * which depends on the host's memory fragementation. To solve this, |
| | 41 | * ensure mdts is limited to the pages equal to the number of segments. |
| | 42 | */ |
| | 43 | max_hw_sectors = min_not_zero(pctrl->max_segments << (PAGE_SHIFT - 9), |
| | 44 | pctrl->max_hw_sectors); |
| | 45 | |
| | 46 | page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12; |
| | 47 | |
| | 48 | id->mdts = ilog2(max_hw_sectors) + 9 - page_shift; |
| | 49 | |
| | 50 | id->acl = 3; |
| | 51 | /* |
| | 52 | * We export aerl limit for the fabrics controller, update this when |
| | 53 | * passthru based aerl support is added. |
| | 54 | */ |
| | 55 | id->aerl = NVMET_ASYNC_EVENTS - 1; |
| | 56 | |
| | 57 | /* emulate kas as most of the PCIe ctrl don't have a support for kas */ |
| | 58 | id->kas = cpu_to_le16(NVMET_KAS); |
| | 59 | |
| | 60 | /* don't support host memory buffer */ |
| | 61 | id->hmpre = 0; |
| | 62 | id->hmmin = 0; |
| | 63 | |
| | 64 | id->sqes = min_t(__u8, ((0x6 << 4) | 0x6), id->sqes); |
| | 65 | id->cqes = min_t(__u8, ((0x4 << 4) | 0x4), id->cqes); |
| | 66 | id->maxcmd = cpu_to_le16(NVMET_MAX_CMD); |
| | 67 | |
| | 68 | /* don't support fuse commands */ |
| | 69 | id->fuses = 0; |
| | 70 | |
| | 71 | id->sgls = cpu_to_le32(1 << 0); /* we always support SGLs */ |
| | 72 | if (ctrl->ops->flags & NVMF_KEYED_SGLS) |
| | 73 | id->sgls |= cpu_to_le32(1 << 2); |
| | 74 | if (req->port->inline_data_size) |
| | 75 | id->sgls |= cpu_to_le32(1 << 20); |
| | 76 | |
| | 77 | /* |
| | 78 | * When passsthru controller is setup using nvme-loop transport it will |
| | 79 | * export the passthru ctrl subsysnqn (PCIe NVMe ctrl) and will fail in |
| | 80 | * the nvme/host/core.c in the nvme_init_subsystem()->nvme_active_ctrl() |
| | 81 | * code path with duplicate ctr subsynqn. In order to prevent that we |
| | 82 | * mask the passthru-ctrl subsysnqn with the target ctrl subsysnqn. |
| | 83 | */ |
| | 84 | memcpy(id->subnqn, ctrl->subsysnqn, sizeof(id->subnqn)); |
| | 85 | |
| | 86 | /* use fabric id-ctrl values */ |
| | 87 | id->ioccsz = cpu_to_le32((sizeof(struct nvme_command) + |
| | 88 | req->port->inline_data_size) / 16); |
| | 89 | id->iorcsz = cpu_to_le32(sizeof(struct nvme_completion) / 16); |
| | 90 | |
| | 91 | id->msdbd = ctrl->ops->msdbd; |
| | 92 | |
| | 93 | /* Support multipath connections with fabrics */ |
| | 94 | id->cmic |= 1 << 1; |
| | 95 | |
| | 96 | /* Disable reservations, see nvmet_parse_passthru_io_cmd() */ |
| | 97 | id->oncs &= cpu_to_le16(~NVME_CTRL_ONCS_RESERVATIONS); |
| | 98 | |
| | 99 | status = nvmet_copy_to_sgl(req, 0, id, sizeof(struct nvme_id_ctrl)); |
| | 100 | |
| | 101 | out_free: |
| | 102 | kfree(id); |
| | 103 | return status; |
| | 104 | } |
| | 105 | |
| | 106 | static u16 nvmet_passthru_override_id_ns(struct nvmet_req *req) |
| | 107 | { |
| | 108 | u16 status = NVME_SC_SUCCESS; |
| | 109 | struct nvme_id_ns *id; |
| | 110 | int i; |
| | 111 | |
| | 112 | id = kzalloc(sizeof(*id), GFP_KERNEL); |
| | 113 | if (!id) |
| | 114 | return NVME_SC_INTERNAL; |
| | 115 | |
| | 116 | status = nvmet_copy_from_sgl(req, 0, id, sizeof(struct nvme_id_ns)); |
| | 117 | if (status) |
| | 118 | goto out_free; |
| | 119 | |
| | 120 | for (i = 0; i < (id->nlbaf + 1); i++) |
| | 121 | if (id->lbaf[i].ms) |
| | 122 | memset(&id->lbaf[i], 0, sizeof(id->lbaf[i])); |
| | 123 | |
| | 124 | id->flbas = id->flbas & ~(1 << 4); |
| | 125 | |
| | 126 | /* |
| | 127 | * Presently the NVMEof target code does not support sending |
| | 128 | * metadata, so we must disable it here. This should be updated |
| | 129 | * once target starts supporting metadata. |
| | 130 | */ |
| | 131 | id->mc = 0; |
| | 132 | |
| | 133 | status = nvmet_copy_to_sgl(req, 0, id, sizeof(*id)); |
| | 134 | |
| | 135 | out_free: |
| | 136 | kfree(id); |
| | 137 | return status; |
| | 138 | } |
| | 139 | |
| | 140 | static void nvmet_passthru_execute_cmd_work(struct work_struct *w) |
| | 141 | { |
| | 142 | struct nvmet_req *req = container_of(w, struct nvmet_req, p.work); |
| | 143 | struct request *rq = req->p.rq; |
| | 144 | u16 status; |
| | 145 | |
| | 146 | nvme_execute_passthru_rq(rq); |
| | 147 | |
| | 148 | status = nvme_req(rq)->status; |
| | 149 | if (status == NVME_SC_SUCCESS && |
| | 150 | req->cmd->common.opcode == nvme_admin_identify) { |
| | 151 | switch (req->cmd->identify.cns) { |
| | 152 | case NVME_ID_CNS_CTRL: |
| | 153 | nvmet_passthru_override_id_ctrl(req); |
| | 154 | break; |
| | 155 | case NVME_ID_CNS_NS: |
| | 156 | nvmet_passthru_override_id_ns(req); |
| | 157 | break; |
| | 158 | } |
| | 159 | } |
| | 160 | |
| | 161 | req->cqe->result = nvme_req(rq)->result; |
| | 162 | nvmet_req_complete(req, status); |
| | 163 | blk_put_request(rq); |
| | 164 | } |
| | 165 | |
| | 166 | static void nvmet_passthru_req_done(struct request *rq, |
| | 167 | blk_status_t blk_status) |
| | 168 | { |
| | 169 | struct nvmet_req *req = rq->end_io_data; |
| | 170 | |
| | 171 | req->cqe->result = nvme_req(rq)->result; |
| | 172 | nvmet_req_complete(req, nvme_req(rq)->status); |
| | 173 | blk_put_request(rq); |
| | 174 | } |
| | 175 | |
| | 176 | static int nvmet_passthru_map_sg(struct nvmet_req *req, struct request *rq) |
| | 177 | { |
| | 178 | int sg_cnt = req->sg_cnt; |
| | 179 | struct scatterlist *sg; |
| | 180 | int op_flags = 0; |
| | 181 | struct bio *bio; |
| | 182 | int i, ret; |
| | 183 | |
| | 184 | if (req->cmd->common.opcode == nvme_cmd_flush) |
| | 185 | op_flags = REQ_FUA; |
| | 186 | else if (nvme_is_write(req->cmd)) |
| | 187 | op_flags = REQ_SYNC | REQ_IDLE; |
| | 188 | |
| | 189 | bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES)); |
| | 190 | bio->bi_end_io = bio_put; |
| | 191 | bio->bi_opf = req_op(rq) | op_flags; |
| | 192 | |
| | 193 | for_each_sg(req->sg, sg, req->sg_cnt, i) { |
| | 194 | if (bio_add_pc_page(rq->q, bio, sg_page(sg), sg->length, |
| | 195 | sg->offset) < sg->length) { |
| | 196 | bio_put(bio); |
| | 197 | return -EINVAL; |
| | 198 | } |
| | 199 | sg_cnt--; |
| | 200 | } |
| | 201 | |
| | 202 | ret = blk_rq_append_bio(rq, &bio); |
| | 203 | if (unlikely(ret)) { |
| | 204 | bio_put(bio); |
| | 205 | return ret; |
| | 206 | } |
| | 207 | |
| | 208 | return 0; |
| | 209 | } |
| | 210 | |
| | 211 | static void nvmet_passthru_execute_cmd(struct nvmet_req *req) |
| | 212 | { |
| | 213 | struct nvme_ctrl *ctrl = nvmet_req_passthru_ctrl(req); |
| | 214 | struct request_queue *q = ctrl->admin_q; |
| | 215 | struct nvme_ns *ns = NULL; |
| | 216 | struct request *rq = NULL; |
| | 217 | u32 effects; |
| | 218 | u16 status; |
| | 219 | int ret; |
| | 220 | |
| | 221 | if (likely(req->sq->qid != 0)) { |
| | 222 | u32 nsid = le32_to_cpu(req->cmd->common.nsid); |
| | 223 | |
| | 224 | ns = nvme_find_get_ns(ctrl, nsid); |
| | 225 | if (unlikely(!ns)) { |
| | 226 | pr_err("failed to get passthru ns nsid:%u\n", nsid); |
| | 227 | status = NVME_SC_INVALID_NS | NVME_SC_DNR; |
| | 228 | goto fail_out; |
| | 229 | } |
| | 230 | |
| | 231 | q = ns->queue; |
| | 232 | } |
| | 233 | |
| | 234 | rq = nvme_alloc_request(q, req->cmd, BLK_MQ_REQ_NOWAIT, NVME_QID_ANY); |
| | 235 | if (IS_ERR(rq)) { |
| | 236 | rq = NULL; |
| | 237 | status = NVME_SC_INTERNAL; |
| | 238 | goto fail_out; |
| | 239 | } |
| | 240 | |
| | 241 | if (req->sg_cnt) { |
| | 242 | ret = nvmet_passthru_map_sg(req, rq); |
| | 243 | if (unlikely(ret)) { |
| | 244 | status = NVME_SC_INTERNAL; |
| | 245 | goto fail_out; |
| | 246 | } |
| | 247 | } |
| | 248 | |
| | 249 | /* |
| | 250 | * If there are effects for the command we are about to execute, or |
| | 251 | * an end_req function we need to use nvme_execute_passthru_rq() |
| | 252 | * synchronously in a work item seeing the end_req function and |
| | 253 | * nvme_passthru_end() can't be called in the request done callback |
| | 254 | * which is typically in interrupt context. |
| | 255 | */ |
| | 256 | effects = nvme_command_effects(ctrl, ns, req->cmd->common.opcode); |
| | 257 | if (req->p.use_workqueue || effects) { |
| | 258 | INIT_WORK(&req->p.work, nvmet_passthru_execute_cmd_work); |
| | 259 | req->p.rq = rq; |
| | 260 | schedule_work(&req->p.work); |
| | 261 | } else { |
| | 262 | rq->end_io_data = req; |
| | 263 | blk_execute_rq_nowait(rq->q, ns ? ns->disk : NULL, rq, 0, |
| | 264 | nvmet_passthru_req_done); |
| | 265 | } |
| | 266 | |
| | 267 | if (ns) |
| | 268 | nvme_put_ns(ns); |
| | 269 | |
| | 270 | return; |
| | 271 | |
| | 272 | fail_out: |
| | 273 | if (ns) |
| | 274 | nvme_put_ns(ns); |
| | 275 | nvmet_req_complete(req, status); |
| | 276 | blk_put_request(rq); |
| | 277 | } |
| | 278 | |
| | 279 | /* |
| | 280 | * We need to emulate set host behaviour to ensure that any requested |
| | 281 | * behaviour of the target's host matches the requested behaviour |
| | 282 | * of the device's host and fail otherwise. |
| | 283 | */ |
| | 284 | static void nvmet_passthru_set_host_behaviour(struct nvmet_req *req) |
| | 285 | { |
| | 286 | struct nvme_ctrl *ctrl = nvmet_req_passthru_ctrl(req); |
| | 287 | struct nvme_feat_host_behavior *host; |
| | 288 | u16 status = NVME_SC_INTERNAL; |
| | 289 | int ret; |
| | 290 | |
| | 291 | host = kzalloc(sizeof(*host) * 2, GFP_KERNEL); |
| | 292 | if (!host) |
| | 293 | goto out_complete_req; |
| | 294 | |
| | 295 | ret = nvme_get_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0, |
| | 296 | host, sizeof(*host), NULL); |
| | 297 | if (ret) |
| | 298 | goto out_free_host; |
| | 299 | |
| | 300 | status = nvmet_copy_from_sgl(req, 0, &host[1], sizeof(*host)); |
| | 301 | if (status) |
| | 302 | goto out_free_host; |
| | 303 | |
| | 304 | if (memcmp(&host[0], &host[1], sizeof(host[0]))) { |
| | 305 | pr_warn("target host has requested different behaviour from the local host\n"); |
| | 306 | status = NVME_SC_INTERNAL; |
| | 307 | } |
| | 308 | |
| | 309 | out_free_host: |
| | 310 | kfree(host); |
| | 311 | out_complete_req: |
| | 312 | nvmet_req_complete(req, status); |
| | 313 | } |
| | 314 | |
| | 315 | static u16 nvmet_setup_passthru_command(struct nvmet_req *req) |
| | 316 | { |
| | 317 | req->p.use_workqueue = false; |
| | 318 | req->execute = nvmet_passthru_execute_cmd; |
| | 319 | return NVME_SC_SUCCESS; |
| | 320 | } |
| | 321 | |
| | 322 | u16 nvmet_parse_passthru_io_cmd(struct nvmet_req *req) |
| | 323 | { |
| | 324 | switch (req->cmd->common.opcode) { |
| | 325 | case nvme_cmd_resv_register: |
| | 326 | case nvme_cmd_resv_report: |
| | 327 | case nvme_cmd_resv_acquire: |
| | 328 | case nvme_cmd_resv_release: |
| | 329 | /* |
| | 330 | * Reservations cannot be supported properly because the |
| | 331 | * underlying device has no way of differentiating different |
| | 332 | * hosts that connect via fabrics. This could potentially be |
| | 333 | * emulated in the future if regular targets grow support for |
| | 334 | * this feature. |
| | 335 | */ |
| | 336 | return NVME_SC_INVALID_OPCODE | NVME_SC_DNR; |
| | 337 | } |
| | 338 | |
| | 339 | return nvmet_setup_passthru_command(req); |
| | 340 | } |
| | 341 | |
| | 342 | /* |
| | 343 | * Only features that are emulated or specifically allowed in the list are |
| | 344 | * passed down to the controller. This function implements the allow list for |
| | 345 | * both get and set features. |
| | 346 | */ |
| | 347 | static u16 nvmet_passthru_get_set_features(struct nvmet_req *req) |
| | 348 | { |
| | 349 | switch (le32_to_cpu(req->cmd->features.fid)) { |
| | 350 | case NVME_FEAT_ARBITRATION: |
| | 351 | case NVME_FEAT_POWER_MGMT: |
| | 352 | case NVME_FEAT_LBA_RANGE: |
| | 353 | case NVME_FEAT_TEMP_THRESH: |
| | 354 | case NVME_FEAT_ERR_RECOVERY: |
| | 355 | case NVME_FEAT_VOLATILE_WC: |
| | 356 | case NVME_FEAT_WRITE_ATOMIC: |
| | 357 | case NVME_FEAT_AUTO_PST: |
| | 358 | case NVME_FEAT_TIMESTAMP: |
| | 359 | case NVME_FEAT_HCTM: |
| | 360 | case NVME_FEAT_NOPSC: |
| | 361 | case NVME_FEAT_RRL: |
| | 362 | case NVME_FEAT_PLM_CONFIG: |
| | 363 | case NVME_FEAT_PLM_WINDOW: |
| | 364 | case NVME_FEAT_HOST_BEHAVIOR: |
| | 365 | case NVME_FEAT_SANITIZE: |
| | 366 | case NVME_FEAT_VENDOR_START ... NVME_FEAT_VENDOR_END: |
| | 367 | return nvmet_setup_passthru_command(req); |
| | 368 | |
| | 369 | case NVME_FEAT_ASYNC_EVENT: |
| | 370 | /* There is no support for forwarding ASYNC events */ |
| | 371 | case NVME_FEAT_IRQ_COALESCE: |
| | 372 | case NVME_FEAT_IRQ_CONFIG: |
| | 373 | /* The IRQ settings will not apply to the target controller */ |
| | 374 | case NVME_FEAT_HOST_MEM_BUF: |
| | 375 | /* |
| | 376 | * Any HMB that's set will not be passed through and will |
| | 377 | * not work as expected |
| | 378 | */ |
| | 379 | case NVME_FEAT_SW_PROGRESS: |
| | 380 | /* |
| | 381 | * The Pre-Boot Software Load Count doesn't make much |
| | 382 | * sense for a target to export |
| | 383 | */ |
| | 384 | case NVME_FEAT_RESV_MASK: |
| | 385 | case NVME_FEAT_RESV_PERSIST: |
| | 386 | /* No reservations, see nvmet_parse_passthru_io_cmd() */ |
| | 387 | default: |
| | 388 | return NVME_SC_INVALID_OPCODE | NVME_SC_DNR; |
| | 389 | } |
| | 390 | } |
| | 391 | |
| | 392 | u16 nvmet_parse_passthru_admin_cmd(struct nvmet_req *req) |
| | 393 | { |
| | 394 | /* |
| | 395 | * Passthru all vendor specific commands |
| | 396 | */ |
| | 397 | if (req->cmd->common.opcode >= nvme_admin_vendor_start) |
| | 398 | return nvmet_setup_passthru_command(req); |
| | 399 | |
| | 400 | switch (req->cmd->common.opcode) { |
| | 401 | case nvme_admin_async_event: |
| | 402 | req->execute = nvmet_execute_async_event; |
| | 403 | return NVME_SC_SUCCESS; |
| | 404 | case nvme_admin_keep_alive: |
| | 405 | /* |
| | 406 | * Most PCIe ctrls don't support keep alive cmd, we route keep |
| | 407 | * alive to the non-passthru mode. In future please change this |
| | 408 | * code when PCIe ctrls with keep alive support available. |
| | 409 | */ |
| | 410 | req->execute = nvmet_execute_keep_alive; |
| | 411 | return NVME_SC_SUCCESS; |
| | 412 | case nvme_admin_set_features: |
| | 413 | switch (le32_to_cpu(req->cmd->features.fid)) { |
| | 414 | case NVME_FEAT_ASYNC_EVENT: |
| | 415 | case NVME_FEAT_KATO: |
| | 416 | case NVME_FEAT_NUM_QUEUES: |
| | 417 | case NVME_FEAT_HOST_ID: |
| | 418 | req->execute = nvmet_execute_set_features; |
| | 419 | return NVME_SC_SUCCESS; |
| | 420 | case NVME_FEAT_HOST_BEHAVIOR: |
| | 421 | req->execute = nvmet_passthru_set_host_behaviour; |
| | 422 | return NVME_SC_SUCCESS; |
| | 423 | default: |
| | 424 | return nvmet_passthru_get_set_features(req); |
| | 425 | } |
| | 426 | break; |
| | 427 | case nvme_admin_get_features: |
| | 428 | switch (le32_to_cpu(req->cmd->features.fid)) { |
| | 429 | case NVME_FEAT_ASYNC_EVENT: |
| | 430 | case NVME_FEAT_KATO: |
| | 431 | case NVME_FEAT_NUM_QUEUES: |
| | 432 | case NVME_FEAT_HOST_ID: |
| | 433 | req->execute = nvmet_execute_get_features; |
| | 434 | return NVME_SC_SUCCESS; |
| | 435 | default: |
| | 436 | return nvmet_passthru_get_set_features(req); |
| | 437 | } |
| | 438 | break; |
| | 439 | case nvme_admin_identify: |
| | 440 | switch (req->cmd->identify.cns) { |
| | 441 | case NVME_ID_CNS_CTRL: |
| | 442 | req->execute = nvmet_passthru_execute_cmd; |
| | 443 | req->p.use_workqueue = true; |
| | 444 | return NVME_SC_SUCCESS; |
| | 445 | case NVME_ID_CNS_NS: |
| | 446 | req->execute = nvmet_passthru_execute_cmd; |
| | 447 | req->p.use_workqueue = true; |
| | 448 | return NVME_SC_SUCCESS; |
| | 449 | default: |
| | 450 | return nvmet_setup_passthru_command(req); |
| | 451 | } |
| | 452 | case nvme_admin_get_log_page: |
| | 453 | return nvmet_setup_passthru_command(req); |
| | 454 | default: |
| | 455 | /* Reject commands not in the allowlist above */ |
| | 456 | return NVME_SC_INVALID_OPCODE | NVME_SC_DNR; |
| | 457 | } |
| | 458 | } |