| @@ -0,0 +1,458 @@ |
| 1 | +// SPDX-License-Identifier: GPL-2.0 |
| 2 | +/* |
| 3 | + * NVMe Over Fabrics Target Passthrough command implementation. |
| 4 | + * |
| 5 | + * Copyright (c) 2017-2018 Western Digital Corporation or its |
| 6 | + * affiliates. |
| 7 | + * Copyright (c) 2019-2020, Eideticom Inc. |
| 8 | + * |
| 9 | + */ |
| 10 | +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
| 11 | +#include <linux/module.h> |
| 12 | + |
| 13 | +#include "../host/nvme.h" |
| 14 | +#include "nvmet.h" |
| 15 | + |
| 16 | +MODULE_IMPORT_NS(NVME_TARGET_PASSTHRU); |
| 17 | + |
| 18 | +static u16 nvmet_passthru_override_id_ctrl(struct nvmet_req *req) |
| 19 | +{ |
| 20 | + struct nvmet_ctrl *ctrl = req->sq->ctrl; |
| 21 | + struct nvme_ctrl *pctrl = ctrl->subsys->passthru_ctrl; |
| 22 | + u16 status = NVME_SC_SUCCESS; |
| 23 | + struct nvme_id_ctrl *id; |
| 24 | + u32 max_hw_sectors; |
| 25 | + int page_shift; |
| 26 | + |
| 27 | + id = kzalloc(sizeof(*id), GFP_KERNEL); |
| 28 | + if (!id) |
| 29 | + return NVME_SC_INTERNAL; |
| 30 | + |
| 31 | + status = nvmet_copy_from_sgl(req, 0, id, sizeof(*id)); |
| 32 | + if (status) |
| 33 | + goto out_free; |
| 34 | + |
| 35 | + id->cntlid = cpu_to_le16(ctrl->cntlid); |
| 36 | + id->ver = cpu_to_le32(ctrl->subsys->ver); |
| 37 | + |
| 38 | + /* |
| 39 | + * The passthru NVMe driver may have a limit on the number of segments |
| 40 | + * which depends on the host's memory fragementation. To solve this, |
| 41 | + * ensure mdts is limited to the pages equal to the number of segments. |
| 42 | + */ |
| 43 | + max_hw_sectors = min_not_zero(pctrl->max_segments << (PAGE_SHIFT - 9), |
| 44 | + pctrl->max_hw_sectors); |
| 45 | + |
| 46 | + page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12; |
| 47 | + |
| 48 | + id->mdts = ilog2(max_hw_sectors) + 9 - page_shift; |
| 49 | + |
| 50 | + id->acl = 3; |
| 51 | + /* |
| 52 | + * We export aerl limit for the fabrics controller, update this when |
| 53 | + * passthru based aerl support is added. |
| 54 | + */ |
| 55 | + id->aerl = NVMET_ASYNC_EVENTS - 1; |
| 56 | + |
| 57 | + /* emulate kas as most of the PCIe ctrl don't have a support for kas */ |
| 58 | + id->kas = cpu_to_le16(NVMET_KAS); |
| 59 | + |
| 60 | + /* don't support host memory buffer */ |
| 61 | + id->hmpre = 0; |
| 62 | + id->hmmin = 0; |
| 63 | + |
| 64 | + id->sqes = min_t(__u8, ((0x6 << 4) | 0x6), id->sqes); |
| 65 | + id->cqes = min_t(__u8, ((0x4 << 4) | 0x4), id->cqes); |
| 66 | + id->maxcmd = cpu_to_le16(NVMET_MAX_CMD); |
| 67 | + |
| 68 | + /* don't support fuse commands */ |
| 69 | + id->fuses = 0; |
| 70 | + |
| 71 | + id->sgls = cpu_to_le32(1 << 0); /* we always support SGLs */ |
| 72 | + if (ctrl->ops->flags & NVMF_KEYED_SGLS) |
| 73 | + id->sgls |= cpu_to_le32(1 << 2); |
| 74 | + if (req->port->inline_data_size) |
| 75 | + id->sgls |= cpu_to_le32(1 << 20); |
| 76 | + |
| 77 | + /* |
| 78 | + * When passsthru controller is setup using nvme-loop transport it will |
| 79 | + * export the passthru ctrl subsysnqn (PCIe NVMe ctrl) and will fail in |
| 80 | + * the nvme/host/core.c in the nvme_init_subsystem()->nvme_active_ctrl() |
| 81 | + * code path with duplicate ctr subsynqn. In order to prevent that we |
| 82 | + * mask the passthru-ctrl subsysnqn with the target ctrl subsysnqn. |
| 83 | + */ |
| 84 | + memcpy(id->subnqn, ctrl->subsysnqn, sizeof(id->subnqn)); |
| 85 | + |
| 86 | + /* use fabric id-ctrl values */ |
| 87 | + id->ioccsz = cpu_to_le32((sizeof(struct nvme_command) + |
| 88 | + req->port->inline_data_size) / 16); |
| 89 | + id->iorcsz = cpu_to_le32(sizeof(struct nvme_completion) / 16); |
| 90 | + |
| 91 | + id->msdbd = ctrl->ops->msdbd; |
| 92 | + |
| 93 | + /* Support multipath connections with fabrics */ |
| 94 | + id->cmic |= 1 << 1; |
| 95 | + |
| 96 | + /* Disable reservations, see nvmet_parse_passthru_io_cmd() */ |
| 97 | + id->oncs &= cpu_to_le16(~NVME_CTRL_ONCS_RESERVATIONS); |
| 98 | + |
| 99 | + status = nvmet_copy_to_sgl(req, 0, id, sizeof(struct nvme_id_ctrl)); |
| 100 | + |
| 101 | +out_free: |
| 102 | + kfree(id); |
| 103 | + return status; |
| 104 | +} |
| 105 | + |
| 106 | +static u16 nvmet_passthru_override_id_ns(struct nvmet_req *req) |
| 107 | +{ |
| 108 | + u16 status = NVME_SC_SUCCESS; |
| 109 | + struct nvme_id_ns *id; |
| 110 | + int i; |
| 111 | + |
| 112 | + id = kzalloc(sizeof(*id), GFP_KERNEL); |
| 113 | + if (!id) |
| 114 | + return NVME_SC_INTERNAL; |
| 115 | + |
| 116 | + status = nvmet_copy_from_sgl(req, 0, id, sizeof(struct nvme_id_ns)); |
| 117 | + if (status) |
| 118 | + goto out_free; |
| 119 | + |
| 120 | + for (i = 0; i < (id->nlbaf + 1); i++) |
| 121 | + if (id->lbaf[i].ms) |
| 122 | + memset(&id->lbaf[i], 0, sizeof(id->lbaf[i])); |
| 123 | + |
| 124 | + id->flbas = id->flbas & ~(1 << 4); |
| 125 | + |
| 126 | + /* |
| 127 | + * Presently the NVMEof target code does not support sending |
| 128 | + * metadata, so we must disable it here. This should be updated |
| 129 | + * once target starts supporting metadata. |
| 130 | + */ |
| 131 | + id->mc = 0; |
| 132 | + |
| 133 | + status = nvmet_copy_to_sgl(req, 0, id, sizeof(*id)); |
| 134 | + |
| 135 | +out_free: |
| 136 | + kfree(id); |
| 137 | + return status; |
| 138 | +} |
| 139 | + |
| 140 | +static void nvmet_passthru_execute_cmd_work(struct work_struct *w) |
| 141 | +{ |
| 142 | + struct nvmet_req *req = container_of(w, struct nvmet_req, p.work); |
| 143 | + struct request *rq = req->p.rq; |
| 144 | + u16 status; |
| 145 | + |
| 146 | + nvme_execute_passthru_rq(rq); |
| 147 | + |
| 148 | + status = nvme_req(rq)->status; |
| 149 | + if (status == NVME_SC_SUCCESS && |
| 150 | + req->cmd->common.opcode == nvme_admin_identify) { |
| 151 | + switch (req->cmd->identify.cns) { |
| 152 | + case NVME_ID_CNS_CTRL: |
| 153 | + nvmet_passthru_override_id_ctrl(req); |
| 154 | + break; |
| 155 | + case NVME_ID_CNS_NS: |
| 156 | + nvmet_passthru_override_id_ns(req); |
| 157 | + break; |
| 158 | + } |
| 159 | + } |
| 160 | + |
| 161 | + req->cqe->result = nvme_req(rq)->result; |
| 162 | + nvmet_req_complete(req, status); |
| 163 | + blk_put_request(rq); |
| 164 | +} |
| 165 | + |
| 166 | +static void nvmet_passthru_req_done(struct request *rq, |
| 167 | + blk_status_t blk_status) |
| 168 | +{ |
| 169 | + struct nvmet_req *req = rq->end_io_data; |
| 170 | + |
| 171 | + req->cqe->result = nvme_req(rq)->result; |
| 172 | + nvmet_req_complete(req, nvme_req(rq)->status); |
| 173 | + blk_put_request(rq); |
| 174 | +} |
| 175 | + |
| 176 | +static int nvmet_passthru_map_sg(struct nvmet_req *req, struct request *rq) |
| 177 | +{ |
| 178 | + int sg_cnt = req->sg_cnt; |
| 179 | + struct scatterlist *sg; |
| 180 | + int op_flags = 0; |
| 181 | + struct bio *bio; |
| 182 | + int i, ret; |
| 183 | + |
| 184 | + if (req->cmd->common.opcode == nvme_cmd_flush) |
| 185 | + op_flags = REQ_FUA; |
| 186 | + else if (nvme_is_write(req->cmd)) |
| 187 | + op_flags = REQ_SYNC | REQ_IDLE; |
| 188 | + |
| 189 | + bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES)); |
| 190 | + bio->bi_end_io = bio_put; |
| 191 | + bio->bi_opf = req_op(rq) | op_flags; |
| 192 | + |
| 193 | + for_each_sg(req->sg, sg, req->sg_cnt, i) { |
| 194 | + if (bio_add_pc_page(rq->q, bio, sg_page(sg), sg->length, |
| 195 | + sg->offset) < sg->length) { |
| 196 | + bio_put(bio); |
| 197 | + return -EINVAL; |
| 198 | + } |
| 199 | + sg_cnt--; |
| 200 | + } |
| 201 | + |
| 202 | + ret = blk_rq_append_bio(rq, &bio); |
| 203 | + if (unlikely(ret)) { |
| 204 | + bio_put(bio); |
| 205 | + return ret; |
| 206 | + } |
| 207 | + |
| 208 | + return 0; |
| 209 | +} |
| 210 | + |
| 211 | +static void nvmet_passthru_execute_cmd(struct nvmet_req *req) |
| 212 | +{ |
| 213 | + struct nvme_ctrl *ctrl = nvmet_req_passthru_ctrl(req); |
| 214 | + struct request_queue *q = ctrl->admin_q; |
| 215 | + struct nvme_ns *ns = NULL; |
| 216 | + struct request *rq = NULL; |
| 217 | + u32 effects; |
| 218 | + u16 status; |
| 219 | + int ret; |
| 220 | + |
| 221 | + if (likely(req->sq->qid != 0)) { |
| 222 | + u32 nsid = le32_to_cpu(req->cmd->common.nsid); |
| 223 | + |
| 224 | + ns = nvme_find_get_ns(ctrl, nsid); |
| 225 | + if (unlikely(!ns)) { |
| 226 | + pr_err("failed to get passthru ns nsid:%u\n", nsid); |
| 227 | + status = NVME_SC_INVALID_NS | NVME_SC_DNR; |
| 228 | + goto fail_out; |
| 229 | + } |
| 230 | + |
| 231 | + q = ns->queue; |
| 232 | + } |
| 233 | + |
| 234 | + rq = nvme_alloc_request(q, req->cmd, BLK_MQ_REQ_NOWAIT, NVME_QID_ANY); |
| 235 | + if (IS_ERR(rq)) { |
| 236 | + rq = NULL; |
| 237 | + status = NVME_SC_INTERNAL; |
| 238 | + goto fail_out; |
| 239 | + } |
| 240 | + |
| 241 | + if (req->sg_cnt) { |
| 242 | + ret = nvmet_passthru_map_sg(req, rq); |
| 243 | + if (unlikely(ret)) { |
| 244 | + status = NVME_SC_INTERNAL; |
| 245 | + goto fail_out; |
| 246 | + } |
| 247 | + } |
| 248 | + |
| 249 | + /* |
| 250 | + * If there are effects for the command we are about to execute, or |
| 251 | + * an end_req function we need to use nvme_execute_passthru_rq() |
| 252 | + * synchronously in a work item seeing the end_req function and |
| 253 | + * nvme_passthru_end() can't be called in the request done callback |
| 254 | + * which is typically in interrupt context. |
| 255 | + */ |
| 256 | + effects = nvme_command_effects(ctrl, ns, req->cmd->common.opcode); |
| 257 | + if (req->p.use_workqueue || effects) { |
| 258 | + INIT_WORK(&req->p.work, nvmet_passthru_execute_cmd_work); |
| 259 | + req->p.rq = rq; |
| 260 | + schedule_work(&req->p.work); |
| 261 | + } else { |
| 262 | + rq->end_io_data = req; |
| 263 | + blk_execute_rq_nowait(rq->q, ns ? ns->disk : NULL, rq, 0, |
| 264 | + nvmet_passthru_req_done); |
| 265 | + } |
| 266 | + |
| 267 | + if (ns) |
| 268 | + nvme_put_ns(ns); |
| 269 | + |
| 270 | + return; |
| 271 | + |
| 272 | +fail_out: |
| 273 | + if (ns) |
| 274 | + nvme_put_ns(ns); |
| 275 | + nvmet_req_complete(req, status); |
| 276 | + blk_put_request(rq); |
| 277 | +} |
| 278 | + |
| 279 | +/* |
| 280 | + * We need to emulate set host behaviour to ensure that any requested |
| 281 | + * behaviour of the target's host matches the requested behaviour |
| 282 | + * of the device's host and fail otherwise. |
| 283 | + */ |
| 284 | +static void nvmet_passthru_set_host_behaviour(struct nvmet_req *req) |
| 285 | +{ |
| 286 | + struct nvme_ctrl *ctrl = nvmet_req_passthru_ctrl(req); |
| 287 | + struct nvme_feat_host_behavior *host; |
| 288 | + u16 status = NVME_SC_INTERNAL; |
| 289 | + int ret; |
| 290 | + |
| 291 | + host = kzalloc(sizeof(*host) * 2, GFP_KERNEL); |
| 292 | + if (!host) |
| 293 | + goto out_complete_req; |
| 294 | + |
| 295 | + ret = nvme_get_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0, |
| 296 | + host, sizeof(*host), NULL); |
| 297 | + if (ret) |
| 298 | + goto out_free_host; |
| 299 | + |
| 300 | + status = nvmet_copy_from_sgl(req, 0, &host[1], sizeof(*host)); |
| 301 | + if (status) |
| 302 | + goto out_free_host; |
| 303 | + |
| 304 | + if (memcmp(&host[0], &host[1], sizeof(host[0]))) { |
| 305 | + pr_warn("target host has requested different behaviour from the local host\n"); |
| 306 | + status = NVME_SC_INTERNAL; |
| 307 | + } |
| 308 | + |
| 309 | +out_free_host: |
| 310 | + kfree(host); |
| 311 | +out_complete_req: |
| 312 | + nvmet_req_complete(req, status); |
| 313 | +} |
| 314 | + |
| 315 | +static u16 nvmet_setup_passthru_command(struct nvmet_req *req) |
| 316 | +{ |
| 317 | + req->p.use_workqueue = false; |
| 318 | + req->execute = nvmet_passthru_execute_cmd; |
| 319 | + return NVME_SC_SUCCESS; |
| 320 | +} |
| 321 | + |
| 322 | +u16 nvmet_parse_passthru_io_cmd(struct nvmet_req *req) |
| 323 | +{ |
| 324 | + switch (req->cmd->common.opcode) { |
| 325 | + case nvme_cmd_resv_register: |
| 326 | + case nvme_cmd_resv_report: |
| 327 | + case nvme_cmd_resv_acquire: |
| 328 | + case nvme_cmd_resv_release: |
| 329 | + /* |
| 330 | + * Reservations cannot be supported properly because the |
| 331 | + * underlying device has no way of differentiating different |
| 332 | + * hosts that connect via fabrics. This could potentially be |
| 333 | + * emulated in the future if regular targets grow support for |
| 334 | + * this feature. |
| 335 | + */ |
| 336 | + return NVME_SC_INVALID_OPCODE | NVME_SC_DNR; |
| 337 | + } |
| 338 | + |
| 339 | + return nvmet_setup_passthru_command(req); |
| 340 | +} |
| 341 | + |
| 342 | +/* |
| 343 | + * Only features that are emulated or specifically allowed in the list are |
| 344 | + * passed down to the controller. This function implements the allow list for |
| 345 | + * both get and set features. |
| 346 | + */ |
| 347 | +static u16 nvmet_passthru_get_set_features(struct nvmet_req *req) |
| 348 | +{ |
| 349 | + switch (le32_to_cpu(req->cmd->features.fid)) { |
| 350 | + case NVME_FEAT_ARBITRATION: |
| 351 | + case NVME_FEAT_POWER_MGMT: |
| 352 | + case NVME_FEAT_LBA_RANGE: |
| 353 | + case NVME_FEAT_TEMP_THRESH: |
| 354 | + case NVME_FEAT_ERR_RECOVERY: |
| 355 | + case NVME_FEAT_VOLATILE_WC: |
| 356 | + case NVME_FEAT_WRITE_ATOMIC: |
| 357 | + case NVME_FEAT_AUTO_PST: |
| 358 | + case NVME_FEAT_TIMESTAMP: |
| 359 | + case NVME_FEAT_HCTM: |
| 360 | + case NVME_FEAT_NOPSC: |
| 361 | + case NVME_FEAT_RRL: |
| 362 | + case NVME_FEAT_PLM_CONFIG: |
| 363 | + case NVME_FEAT_PLM_WINDOW: |
| 364 | + case NVME_FEAT_HOST_BEHAVIOR: |
| 365 | + case NVME_FEAT_SANITIZE: |
| 366 | + case NVME_FEAT_VENDOR_START ... NVME_FEAT_VENDOR_END: |
| 367 | + return nvmet_setup_passthru_command(req); |
| 368 | + |
| 369 | + case NVME_FEAT_ASYNC_EVENT: |
| 370 | + /* There is no support for forwarding ASYNC events */ |
| 371 | + case NVME_FEAT_IRQ_COALESCE: |
| 372 | + case NVME_FEAT_IRQ_CONFIG: |
| 373 | + /* The IRQ settings will not apply to the target controller */ |
| 374 | + case NVME_FEAT_HOST_MEM_BUF: |
| 375 | + /* |
| 376 | + * Any HMB that's set will not be passed through and will |
| 377 | + * not work as expected |
| 378 | + */ |
| 379 | + case NVME_FEAT_SW_PROGRESS: |
| 380 | + /* |
| 381 | + * The Pre-Boot Software Load Count doesn't make much |
| 382 | + * sense for a target to export |
| 383 | + */ |
| 384 | + case NVME_FEAT_RESV_MASK: |
| 385 | + case NVME_FEAT_RESV_PERSIST: |
| 386 | + /* No reservations, see nvmet_parse_passthru_io_cmd() */ |
| 387 | + default: |
| 388 | + return NVME_SC_INVALID_OPCODE | NVME_SC_DNR; |
| 389 | + } |
| 390 | +} |
| 391 | + |
| 392 | +u16 nvmet_parse_passthru_admin_cmd(struct nvmet_req *req) |
| 393 | +{ |
| 394 | + /* |
| 395 | + * Passthru all vendor specific commands |
| 396 | + */ |
| 397 | + if (req->cmd->common.opcode >= nvme_admin_vendor_start) |
| 398 | + return nvmet_setup_passthru_command(req); |
| 399 | + |
| 400 | + switch (req->cmd->common.opcode) { |
| 401 | + case nvme_admin_async_event: |
| 402 | + req->execute = nvmet_execute_async_event; |
| 403 | + return NVME_SC_SUCCESS; |
| 404 | + case nvme_admin_keep_alive: |
| 405 | + /* |
| 406 | + * Most PCIe ctrls don't support keep alive cmd, we route keep |
| 407 | + * alive to the non-passthru mode. In future please change this |
| 408 | + * code when PCIe ctrls with keep alive support available. |
| 409 | + */ |
| 410 | + req->execute = nvmet_execute_keep_alive; |
| 411 | + return NVME_SC_SUCCESS; |
| 412 | + case nvme_admin_set_features: |
| 413 | + switch (le32_to_cpu(req->cmd->features.fid)) { |
| 414 | + case NVME_FEAT_ASYNC_EVENT: |
| 415 | + case NVME_FEAT_KATO: |
| 416 | + case NVME_FEAT_NUM_QUEUES: |
| 417 | + case NVME_FEAT_HOST_ID: |
| 418 | + req->execute = nvmet_execute_set_features; |
| 419 | + return NVME_SC_SUCCESS; |
| 420 | + case NVME_FEAT_HOST_BEHAVIOR: |
| 421 | + req->execute = nvmet_passthru_set_host_behaviour; |
| 422 | + return NVME_SC_SUCCESS; |
| 423 | + default: |
| 424 | + return nvmet_passthru_get_set_features(req); |
| 425 | + } |
| 426 | + break; |
| 427 | + case nvme_admin_get_features: |
| 428 | + switch (le32_to_cpu(req->cmd->features.fid)) { |
| 429 | + case NVME_FEAT_ASYNC_EVENT: |
| 430 | + case NVME_FEAT_KATO: |
| 431 | + case NVME_FEAT_NUM_QUEUES: |
| 432 | + case NVME_FEAT_HOST_ID: |
| 433 | + req->execute = nvmet_execute_get_features; |
| 434 | + return NVME_SC_SUCCESS; |
| 435 | + default: |
| 436 | + return nvmet_passthru_get_set_features(req); |
| 437 | + } |
| 438 | + break; |
| 439 | + case nvme_admin_identify: |
| 440 | + switch (req->cmd->identify.cns) { |
| 441 | + case NVME_ID_CNS_CTRL: |
| 442 | + req->execute = nvmet_passthru_execute_cmd; |
| 443 | + req->p.use_workqueue = true; |
| 444 | + return NVME_SC_SUCCESS; |
| 445 | + case NVME_ID_CNS_NS: |
| 446 | + req->execute = nvmet_passthru_execute_cmd; |
| 447 | + req->p.use_workqueue = true; |
| 448 | + return NVME_SC_SUCCESS; |
| 449 | + default: |
| 450 | + return nvmet_setup_passthru_command(req); |
| 451 | + } |
| 452 | + case nvme_admin_get_log_page: |
| 453 | + return nvmet_setup_passthru_command(req); |
| 454 | + default: |
| 455 | + /* Reject commands not in the allowlist above */ |
| 456 | + return NVME_SC_INVALID_OPCODE | NVME_SC_DNR; |
| 457 | + } |
| 458 | +} |