/* * NVMe over Fabrics loopback device. * Copyright (c) 2015-2016 HGST, a Western Digital Company. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. * * This program is distributed in the hope it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. */#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt#include <linux/scatterlist.h>#include <linux/blk-mq.h>#include <linux/nvme.h>#include <linux/module.h>#include <linux/parser.h>#include "nvmet.h"#include "../host/nvme.h"#include "../host/fabrics.h" #define NVME_LOOP_MAX_SEGMENTS 256 /* * We handle AEN commands ourselves and don't even let the * block layer know about them. */#define NVME_LOOP_NR_AEN_COMMANDS 1#define NVME_LOOP_AQ_BLKMQ_DEPTH \ (NVME_AQ_DEPTH - NVME_LOOP_NR_AEN_COMMANDS) struct nvme_loop_iod { struct nvme_request nvme_req; struct nvme_command cmd; struct nvme_completion rsp; struct nvmet_req req; struct nvme_loop_queue *queue; struct work_struct work; struct sg_table sg_table; struct scatterlist first_sgl[];}; struct nvme_loop_ctrl { struct nvme_loop_queue *queues; struct blk_mq_tag_set admin_tag_set; struct list_head list; struct blk_mq_tag_set tag_set; struct nvme_loop_iod async_event_iod; struct nvme_ctrl ctrl; struct nvmet_ctrl *target_ctrl; struct work_struct delete_work;}; static inline struct nvme_loop_ctrl *to_loop_ctrl(struct nvme_ctrl *ctrl){ return container_of(ctrl, struct nvme_loop_ctrl, ctrl);} struct nvme_loop_queue { struct nvmet_cq nvme_cq; struct nvmet_sq nvme_sq; struct nvme_loop_ctrl *ctrl;}; static struct nvmet_port *nvmet_loop_port; static LIST_HEAD(nvme_loop_ctrl_list);static DEFINE_MUTEX(nvme_loop_ctrl_mutex); static void nvme_loop_queue_response(struct nvmet_req *nvme_req);static void nvme_loop_delete_ctrl(struct nvmet_ctrl *ctrl); static struct nvmet_fabrics_ops nvme_loop_ops; static inline int nvme_loop_queue_idx(struct nvme_loop_queue *queue){ return queue - queue->ctrl->queues;} static void nvme_loop_complete_rq(struct request *req){ struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req); nvme_cleanup_cmd(req); sg_free_table_chained(&iod->sg_table, true); nvme_complete_rq(req);} static struct blk_mq_tags *nvme_loop_tagset(struct nvme_loop_queue *queue){ u32 queue_idx = nvme_loop_queue_idx(queue); if (queue_idx == 0) return queue->ctrl->admin_tag_set.tags[queue_idx]; return queue->ctrl->tag_set.tags[queue_idx - 1];} static void nvme_loop_queue_response(struct nvmet_req *req){ struct nvme_loop_queue *queue = container_of(req->sq, struct nvme_loop_queue, nvme_sq); struct nvme_completion *cqe = req->rsp; /* * AEN requests are special as they don't time out and can * survive any kind of queue freeze and often don't respond to * aborts. We don't even bother to allocate a struct request * for them but rather special case them here. */ if (unlikely(nvme_loop_queue_idx(queue) == 0 && cqe->command_id >= NVME_LOOP_AQ_BLKMQ_DEPTH)) { nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status, &cqe->result); } else { struct request *rq; rq = blk_mq_tag_to_rq(nvme_loop_tagset(queue), cqe->command_id); if (!rq) { dev_err(queue->ctrl->ctrl.device, "tag 0x%x on queue %d not found\n", cqe->command_id, nvme_loop_queue_idx(queue)); return; } nvme_end_request(rq, cqe->status, cqe->result); }} static void nvme_loop_execute_work(struct work_struct *work){ struct nvme_loop_iod *iod = container_of(work, struct nvme_loop_iod, work); iod->req.execute(&iod->req);} static enum blk_eh_timer_returnnvme_loop_timeout(struct request *rq, bool reserved){ struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(rq); /* queue error recovery */ nvme_reset_ctrl(&iod->queue->ctrl->ctrl); /* fail with DNR on admin cmd timeout */ nvme_req(rq)->status = NVME_SC_ABORT_REQ | NVME_SC_DNR; return BLK_EH_HANDLED;} static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd){ struct nvme_ns *ns = hctx->queue->queuedata; struct nvme_loop_queue *queue = hctx->driver_data; struct request *req = bd->rq; struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req); blk_status_t ret; ret = nvme_setup_cmd(ns, req, &iod->cmd); if (ret) return ret; iod->cmd.common.flags |= NVME_CMD_SGL_METABUF; iod->req.port = nvmet_loop_port; if (!nvmet_req_init(&iod->req, &queue->nvme_cq, &queue->nvme_sq, &nvme_loop_ops)) { nvme_cleanup_cmd(req); blk_mq_start_request(req); nvme_loop_queue_response(&iod->req); return BLK_STS_OK; } if (blk_rq_bytes(req)) { iod->sg_table.sgl = iod->first_sgl; if (sg_alloc_table_chained(&iod->sg_table, blk_rq_nr_phys_segments(req), iod->sg_table.sgl)) return BLK_STS_RESOURCE; iod->req.sg = iod->sg_table.sgl; iod->req.sg_cnt = blk_rq_map_sg(req->q, req, iod->sg_table.sgl); } blk_mq_start_request(req); schedule_work(&iod->work); return BLK_STS_OK;} static void nvme_loop_submit_async_event(struct nvme_ctrl *arg, int aer_idx){ struct nvme_loop_ctrl *ctrl = to_loop_ctrl(arg); struct nvme_loop_queue *queue = &ctrl->queues[0]; struct nvme_loop_iod *iod = &ctrl->async_event_iod; memset(&iod->cmd, 0, sizeof(iod->cmd)); iod->cmd.common.opcode = nvme_admin_async_event; iod->cmd.common.command_id = NVME_LOOP_AQ_BLKMQ_DEPTH; iod->cmd.common.flags |= NVME_CMD_SGL_METABUF; if (!nvmet_req_init(&iod->req, &queue->nvme_cq, &queue->nvme_sq, &nvme_loop_ops)) { dev_err(ctrl->ctrl.device, "failed async event work\n"); return; } schedule_work(&iod->work);} static int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl, struct nvme_loop_iod *iod, unsigned int queue_idx){ iod->req.cmd = &iod->cmd; iod->req.rsp = &iod->rsp; iod->queue = &ctrl->queues[queue_idx]; INIT_WORK(&iod->work, nvme_loop_execute_work); return 0;} static int nvme_loop_init_request(struct blk_mq_tag_set *set, struct request *req, unsigned int hctx_idx, unsigned int numa_node){ struct nvme_loop_ctrl *ctrl = set->driver_data; return nvme_loop_init_iod(ctrl, blk_mq_rq_to_pdu(req), (set == &ctrl->tag_set) ? hctx_idx + 1 : 0);} static int nvme_loop_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx){ struct nvme_loop_ctrl *ctrl = data; struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1]; BUG_ON(hctx_idx >= ctrl->ctrl.queue_count); hctx->driver_data = queue; return 0;} static int nvme_loop_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx){ struct nvme_loop_ctrl *ctrl = data; struct nvme_loop_queue *queue = &ctrl->queues[0]; BUG_ON(hctx_idx != 0); hctx->driver_data = queue; return 0;} static const struct blk_mq_ops nvme_loop_mq_ops = { .queue_rq = nvme_loop_queue_rq, .complete = nvme_loop_complete_rq, .init_request = nvme_loop_init_request, .init_hctx = nvme_loop_init_hctx, .timeout = nvme_loop_timeout,}; static const struct blk_mq_ops nvme_loop_admin_mq_ops = { .queue_rq = nvme_loop_queue_rq, .complete = nvme_loop_complete_rq, .init_request = nvme_loop_init_request, .init_hctx = nvme_loop_init_admin_hctx, .timeout = nvme_loop_timeout,}; static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl){ nvmet_sq_destroy(&ctrl->queues[0].nvme_sq); blk_cleanup_queue(ctrl->ctrl.admin_q); blk_mq_free_tag_set(&ctrl->admin_tag_set);} static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl){ struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl); if (list_empty(&ctrl->list)) goto free_ctrl; mutex_lock(&nvme_loop_ctrl_mutex); list_del(&ctrl->list); mutex_unlock(&nvme_loop_ctrl_mutex); if (nctrl->tagset) { blk_cleanup_queue(ctrl->ctrl.connect_q); blk_mq_free_tag_set(&ctrl->tag_set); } kfree(ctrl->queues); nvmf_free_options(nctrl->opts);free_ctrl: kfree(ctrl);} static void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl){ int i; for (i = 1; i < ctrl->ctrl.queue_count; i++) nvmet_sq_destroy(&ctrl->queues[i].nvme_sq);} static int nvme_loop_init_io_queues(struct nvme_loop_ctrl *ctrl){ struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; unsigned int nr_io_queues; int ret, i; nr_io_queues = min(opts->nr_io_queues, num_online_cpus()); ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues); if (ret || !nr_io_queues) return ret; dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues); for (i = 1; i <= nr_io_queues; i++) { ctrl->queues[i].ctrl = ctrl; ret = nvmet_sq_init(&ctrl->queues[i].nvme_sq); if (ret) goto out_destroy_queues; ctrl->ctrl.queue_count++; } return 0; out_destroy_queues: nvme_loop_destroy_io_queues(ctrl); return ret;} static int nvme_loop_connect_io_queues(struct nvme_loop_ctrl *ctrl){ int i, ret; for (i = 1; i < ctrl->ctrl.queue_count; i++) { ret = nvmf_connect_io_queue(&ctrl->ctrl, i); if (ret) return ret; } return 0;} static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl){ int error; memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set)); ctrl->admin_tag_set.ops = &nvme_loop_admin_mq_ops; ctrl->admin_tag_set.queue_depth = NVME_LOOP_AQ_BLKMQ_DEPTH; ctrl->admin_tag_set.reserved_tags = 2; /* connect + keep-alive */ ctrl->admin_tag_set.numa_node = NUMA_NO_NODE; ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_loop_iod) + SG_CHUNK_SIZE * sizeof(struct scatterlist); ctrl->admin_tag_set.driver_data = ctrl; ctrl->admin_tag_set.nr_hw_queues = 1; ctrl->admin_tag_set.timeout = ADMIN_TIMEOUT; ctrl->queues[0].ctrl = ctrl; error = nvmet_sq_init(&ctrl->queues[0].nvme_sq); if (error) return error; ctrl->ctrl.queue_count = 1; error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set); if (error) goto out_free_sq; ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set); if (IS_ERR(ctrl->ctrl.admin_q)) { error = PTR_ERR(ctrl->ctrl.admin_q); goto out_free_tagset; } error = nvmf_connect_admin_queue(&ctrl->ctrl); if (error) goto out_cleanup_queue; error = nvmf_reg_read64(&ctrl->ctrl, NVME_REG_CAP, &ctrl->ctrl.cap); if (error) { dev_err(ctrl->ctrl.device, "prop_get NVME_REG_CAP failed\n"); goto out_cleanup_queue; } ctrl->ctrl.sqsize = min_t(int, NVME_CAP_MQES(ctrl->ctrl.cap), ctrl->ctrl.sqsize); error = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap); if (error) goto out_cleanup_queue; ctrl->ctrl.max_hw_sectors = (NVME_LOOP_MAX_SEGMENTS - 1) << (PAGE_SHIFT - 9); error = nvme_init_identify(&ctrl->ctrl); if (error) goto out_cleanup_queue; return 0; out_cleanup_queue: blk_cleanup_queue(ctrl->ctrl.admin_q);out_free_tagset: blk_mq_free_tag_set(&ctrl->admin_tag_set);out_free_sq: nvmet_sq_destroy(&ctrl->queues[0].nvme_sq); return error;} static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl){ if (ctrl->ctrl.queue_count > 1) { nvme_stop_queues(&ctrl->ctrl); blk_mq_tagset_busy_iter(&ctrl->tag_set, nvme_cancel_request, &ctrl->ctrl); nvme_loop_destroy_io_queues(ctrl); } if (ctrl->ctrl.state == NVME_CTRL_LIVE) nvme_shutdown_ctrl(&ctrl->ctrl); blk_mq_quiesce_queue(ctrl->ctrl.admin_q); blk_mq_tagset_busy_iter(&ctrl->admin_tag_set, nvme_cancel_request, &ctrl->ctrl); blk_mq_unquiesce_queue(ctrl->ctrl.admin_q); nvme_loop_destroy_admin_queue(ctrl);} static void nvme_loop_del_ctrl_work(struct work_struct *work){ struct nvme_loop_ctrl *ctrl = container_of(work, struct nvme_loop_ctrl, delete_work); nvme_stop_ctrl(&ctrl->ctrl); nvme_remove_namespaces(&ctrl->ctrl); nvme_loop_shutdown_ctrl(ctrl); nvme_uninit_ctrl(&ctrl->ctrl); nvme_put_ctrl(&ctrl->ctrl);} static int __nvme_loop_del_ctrl(struct nvme_loop_ctrl *ctrl){ if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING)) return -EBUSY; if (!queue_work(nvme_wq, &ctrl->delete_work)) return -EBUSY; return 0;} static int nvme_loop_del_ctrl(struct nvme_ctrl *nctrl){ struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl); int ret; ret = __nvme_loop_del_ctrl(ctrl); if (ret) return ret; flush_work(&ctrl->delete_work); return 0;} static void nvme_loop_delete_ctrl(struct nvmet_ctrl *nctrl){ struct nvme_loop_ctrl *ctrl; mutex_lock(&nvme_loop_ctrl_mutex); list_for_each_entry(ctrl, &nvme_loop_ctrl_list, list) { if (ctrl->ctrl.cntlid == nctrl->cntlid) __nvme_loop_del_ctrl(ctrl); } mutex_unlock(&nvme_loop_ctrl_mutex);} static void nvme_loop_reset_ctrl_work(struct work_struct *work){ struct nvme_loop_ctrl *ctrl = container_of(work, struct nvme_loop_ctrl, ctrl.reset_work); bool changed; int ret; nvme_stop_ctrl(&ctrl->ctrl); nvme_loop_shutdown_ctrl(ctrl); ret = nvme_loop_configure_admin_queue(ctrl); if (ret)Lines 1–500 of 742. Load more
/* * NVMe over Fabrics loopback device. * Copyright (c) 2015-2016 HGST, a Western Digital Company. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. * * This program is distributed in the hope it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/scatterlist.h> #include <linux/blk-mq.h> #include <linux/nvme.h> #include <linux/module.h> #include <linux/parser.h> #include "nvmet.h" #include "../host/nvme.h" #include "../host/fabrics.h" #define NVME_LOOP_MAX_SEGMENTS 256 /* * We handle AEN commands ourselves and don't even let the * block layer know about them. */ #define NVME_LOOP_NR_AEN_COMMANDS 1 #define NVME_LOOP_AQ_BLKMQ_DEPTH \ (NVME_AQ_DEPTH - NVME_LOOP_NR_AEN_COMMANDS) struct nvme_loop_iod { struct nvme_request nvme_req; struct nvme_command cmd; struct nvme_completion rsp; struct nvmet_req req; struct nvme_loop_queue *queue; struct work_struct work; struct sg_table sg_table; struct scatterlist first_sgl[]; }; struct nvme_loop_ctrl { struct nvme_loop_queue *queues; struct blk_mq_tag_set admin_tag_set; struct list_head list; struct blk_mq_tag_set tag_set; struct nvme_loop_iod async_event_iod; struct nvme_ctrl ctrl; struct nvmet_ctrl *target_ctrl; struct work_struct delete_work; }; static inline struct nvme_loop_ctrl *to_loop_ctrl(struct nvme_ctrl *ctrl) { return container_of(ctrl, struct nvme_loop_ctrl, ctrl); } struct nvme_loop_queue { struct nvmet_cq nvme_cq; struct nvmet_sq nvme_sq; struct nvme_loop_ctrl *ctrl; }; static struct nvmet_port *nvmet_loop_port; static LIST_HEAD(nvme_loop_ctrl_list); static DEFINE_MUTEX(nvme_loop_ctrl_mutex); static void nvme_loop_queue_response(struct nvmet_req *nvme_req); static void nvme_loop_delete_ctrl(struct nvmet_ctrl *ctrl); static struct nvmet_fabrics_ops nvme_loop_ops; static inline int nvme_loop_queue_idx(struct nvme_loop_queue *queue) { return queue - queue->ctrl->queues; } static void nvme_loop_complete_rq(struct request *req) { struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req); nvme_cleanup_cmd(req); sg_free_table_chained(&iod->sg_table, true); nvme_complete_rq(req); } static struct blk_mq_tags *nvme_loop_tagset(struct nvme_loop_queue *queue) { u32 queue_idx = nvme_loop_queue_idx(queue); if (queue_idx == 0) return queue->ctrl->admin_tag_set.tags[queue_idx]; return queue->ctrl->tag_set.tags[queue_idx - 1]; } static void nvme_loop_queue_response(struct nvmet_req *req) { struct nvme_loop_queue *queue = container_of(req->sq, struct nvme_loop_queue, nvme_sq); struct nvme_completion *cqe = req->rsp; /* * AEN requests are special as they don't time out and can * survive any kind of queue freeze and often don't respond to * aborts. We don't even bother to allocate a struct request * for them but rather special case them here. */ if (unlikely(nvme_loop_queue_idx(queue) == 0 && cqe->command_id >= NVME_LOOP_AQ_BLKMQ_DEPTH)) { nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status, &cqe->result); } else { struct request *rq; rq = blk_mq_tag_to_rq(nvme_loop_tagset(queue), cqe->command_id); if (!rq) { dev_err(queue->ctrl->ctrl.device, "tag 0x%x on queue %d not found\n", cqe->command_id, nvme_loop_queue_idx(queue)); return; } nvme_end_request(rq, cqe->status, cqe->result); } } static void nvme_loop_execute_work(struct work_struct *work) { struct nvme_loop_iod *iod = container_of(work, struct nvme_loop_iod, work); iod->req.execute(&iod->req); } static enum blk_eh_timer_return nvme_loop_timeout(struct request *rq, bool reserved) { struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(rq); /* queue error recovery */ nvme_reset_ctrl(&iod->queue->ctrl->ctrl); /* fail with DNR on admin cmd timeout */ nvme_req(rq)->status = NVME_SC_ABORT_REQ | NVME_SC_DNR; return BLK_EH_HANDLED; } static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { struct nvme_ns *ns = hctx->queue->queuedata; struct nvme_loop_queue *queue = hctx->driver_data; struct request *req = bd->rq; struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req); blk_status_t ret; ret = nvme_setup_cmd(ns, req, &iod->cmd); if (ret) return ret; iod->cmd.common.flags |= NVME_CMD_SGL_METABUF; iod->req.port = nvmet_loop_port; if (!nvmet_req_init(&iod->req, &queue->nvme_cq, &queue->nvme_sq, &nvme_loop_ops)) { nvme_cleanup_cmd(req); blk_mq_start_request(req); nvme_loop_queue_response(&iod->req); return BLK_STS_OK; } if (blk_rq_bytes(req)) { iod->sg_table.sgl = iod->first_sgl; if (sg_alloc_table_chained(&iod->sg_table, blk_rq_nr_phys_segments(req), iod->sg_table.sgl)) return BLK_STS_RESOURCE; iod->req.sg = iod->sg_table.sgl; iod->req.sg_cnt = blk_rq_map_sg(req->q, req, iod->sg_table.sgl); } blk_mq_start_request(req); schedule_work(&iod->work); return BLK_STS_OK; } static void nvme_loop_submit_async_event(struct nvme_ctrl *arg, int aer_idx) { struct nvme_loop_ctrl *ctrl = to_loop_ctrl(arg); struct nvme_loop_queue *queue = &ctrl->queues[0]; struct nvme_loop_iod *iod = &ctrl->async_event_iod; memset(&iod->cmd, 0, sizeof(iod->cmd)); iod->cmd.common.opcode = nvme_admin_async_event; iod->cmd.common.command_id = NVME_LOOP_AQ_BLKMQ_DEPTH; iod->cmd.common.flags |= NVME_CMD_SGL_METABUF; if (!nvmet_req_init(&iod->req, &queue->nvme_cq, &queue->nvme_sq, &nvme_loop_ops)) { dev_err(ctrl->ctrl.device, "failed async event work\n"); return; } schedule_work(&iod->work); } static int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl, struct nvme_loop_iod *iod, unsigned int queue_idx) { iod->req.cmd = &iod->cmd; iod->req.rsp = &iod->rsp; iod->queue = &ctrl->queues[queue_idx]; INIT_WORK(&iod->work, nvme_loop_execute_work); return 0; } static int nvme_loop_init_request(struct blk_mq_tag_set *set, struct request *req, unsigned int hctx_idx, unsigned int numa_node) { struct nvme_loop_ctrl *ctrl = set->driver_data; return nvme_loop_init_iod(ctrl, blk_mq_rq_to_pdu(req), (set == &ctrl->tag_set) ? hctx_idx + 1 : 0); } static int nvme_loop_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx) { struct nvme_loop_ctrl *ctrl = data; struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1]; BUG_ON(hctx_idx >= ctrl->ctrl.queue_count); hctx->driver_data = queue; return 0; } static int nvme_loop_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx) { struct nvme_loop_ctrl *ctrl = data; struct nvme_loop_queue *queue = &ctrl->queues[0]; BUG_ON(hctx_idx != 0); hctx->driver_data = queue; return 0; } static const struct blk_mq_ops nvme_loop_mq_ops = { .queue_rq = nvme_loop_queue_rq, .complete = nvme_loop_complete_rq, .init_request = nvme_loop_init_request, .init_hctx = nvme_loop_init_hctx, .timeout = nvme_loop_timeout, }; static const struct blk_mq_ops nvme_loop_admin_mq_ops = { .queue_rq = nvme_loop_queue_rq, .complete = nvme_loop_complete_rq, .init_request = nvme_loop_init_request, .init_hctx = nvme_loop_init_admin_hctx, .timeout = nvme_loop_timeout, }; static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl) { nvmet_sq_destroy(&ctrl->queues[0].nvme_sq); blk_cleanup_queue(ctrl->ctrl.admin_q); blk_mq_free_tag_set(&ctrl->admin_tag_set); } static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl) { struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl); if (list_empty(&ctrl->list)) goto free_ctrl; mutex_lock(&nvme_loop_ctrl_mutex); list_del(&ctrl->list); mutex_unlock(&nvme_loop_ctrl_mutex); if (nctrl->tagset) { blk_cleanup_queue(ctrl->ctrl.connect_q); blk_mq_free_tag_set(&ctrl->tag_set); } kfree(ctrl->queues); nvmf_free_options(nctrl->opts); free_ctrl: kfree(ctrl); } static void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl) { int i; for (i = 1; i < ctrl->ctrl.queue_count; i++) nvmet_sq_destroy(&ctrl->queues[i].nvme_sq); } static int nvme_loop_init_io_queues(struct nvme_loop_ctrl *ctrl) { struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; unsigned int nr_io_queues; int ret, i; nr_io_queues = min(opts->nr_io_queues, num_online_cpus()); ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues); if (ret || !nr_io_queues) return ret; dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues); for (i = 1; i <= nr_io_queues; i++) { ctrl->queues[i].ctrl = ctrl; ret = nvmet_sq_init(&ctrl->queues[i].nvme_sq); if (ret) goto out_destroy_queues; ctrl->ctrl.queue_count++; } return 0; out_destroy_queues: nvme_loop_destroy_io_queues(ctrl); return ret; } static int nvme_loop_connect_io_queues(struct nvme_loop_ctrl *ctrl) { int i, ret; for (i = 1; i < ctrl->ctrl.queue_count; i++) { ret = nvmf_connect_io_queue(&ctrl->ctrl, i); if (ret) return ret; } return 0; } static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl) { int error; memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set)); ctrl->admin_tag_set.ops = &nvme_loop_admin_mq_ops; ctrl->admin_tag_set.queue_depth = NVME_LOOP_AQ_BLKMQ_DEPTH; ctrl->admin_tag_set.reserved_tags = 2; /* connect + keep-alive */ ctrl->admin_tag_set.numa_node = NUMA_NO_NODE; ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_loop_iod) + SG_CHUNK_SIZE * sizeof(struct scatterlist); ctrl->admin_tag_set.driver_data = ctrl; ctrl->admin_tag_set.nr_hw_queues = 1; ctrl->admin_tag_set.timeout = ADMIN_TIMEOUT; ctrl->queues[0].ctrl = ctrl; error = nvmet_sq_init(&ctrl->queues[0].nvme_sq); if (error) return error; ctrl->ctrl.queue_count = 1; error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set); if (error) goto out_free_sq; ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set); if (IS_ERR(ctrl->ctrl.admin_q)) { error = PTR_ERR(ctrl->ctrl.admin_q); goto out_free_tagset; } error = nvmf_connect_admin_queue(&ctrl->ctrl); if (error) goto out_cleanup_queue; error = nvmf_reg_read64(&ctrl->ctrl, NVME_REG_CAP, &ctrl->ctrl.cap); if (error) { dev_err(ctrl->ctrl.device, "prop_get NVME_REG_CAP failed\n"); goto out_cleanup_queue; } ctrl->ctrl.sqsize = min_t(int, NVME_CAP_MQES(ctrl->ctrl.cap), ctrl->ctrl.sqsize); error = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap); if (error) goto out_cleanup_queue; ctrl->ctrl.max_hw_sectors = (NVME_LOOP_MAX_SEGMENTS - 1) << (PAGE_SHIFT - 9); error = nvme_init_identify(&ctrl->ctrl); if (error) goto out_cleanup_queue; return 0; out_cleanup_queue: blk_cleanup_queue(ctrl->ctrl.admin_q); out_free_tagset: blk_mq_free_tag_set(&ctrl->admin_tag_set); out_free_sq: nvmet_sq_destroy(&ctrl->queues[0].nvme_sq); return error; } static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl) { if (ctrl->ctrl.queue_count > 1) { nvme_stop_queues(&ctrl->ctrl); blk_mq_tagset_busy_iter(&ctrl->tag_set, nvme_cancel_request, &ctrl->ctrl); nvme_loop_destroy_io_queues(ctrl); } if (ctrl->ctrl.state == NVME_CTRL_LIVE) nvme_shutdown_ctrl(&ctrl->ctrl); blk_mq_quiesce_queue(ctrl->ctrl.admin_q); blk_mq_tagset_busy_iter(&ctrl->admin_tag_set, nvme_cancel_request, &ctrl->ctrl); blk_mq_unquiesce_queue(ctrl->ctrl.admin_q); nvme_loop_destroy_admin_queue(ctrl); } static void nvme_loop_del_ctrl_work(struct work_struct *work) { struct nvme_loop_ctrl *ctrl = container_of(work, struct nvme_loop_ctrl, delete_work); nvme_stop_ctrl(&ctrl->ctrl); nvme_remove_namespaces(&ctrl->ctrl); nvme_loop_shutdown_ctrl(ctrl); nvme_uninit_ctrl(&ctrl->ctrl); nvme_put_ctrl(&ctrl->ctrl); } static int __nvme_loop_del_ctrl(struct nvme_loop_ctrl *ctrl) { if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING)) return -EBUSY; if (!queue_work(nvme_wq, &ctrl->delete_work)) return -EBUSY; return 0; } static int nvme_loop_del_ctrl(struct nvme_ctrl *nctrl) { struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl); int ret; ret = __nvme_loop_del_ctrl(ctrl); if (ret) return ret; flush_work(&ctrl->delete_work); return 0; } static void nvme_loop_delete_ctrl(struct nvmet_ctrl *nctrl) { struct nvme_loop_ctrl *ctrl; mutex_lock(&nvme_loop_ctrl_mutex); list_for_each_entry(ctrl, &nvme_loop_ctrl_list, list) { if (ctrl->ctrl.cntlid == nctrl->cntlid) __nvme_loop_del_ctrl(ctrl); } mutex_unlock(&nvme_loop_ctrl_mutex); } static void nvme_loop_reset_ctrl_work(struct work_struct *work) { struct nvme_loop_ctrl *ctrl = container_of(work, struct nvme_loop_ctrl, ctrl.reset_work); bool changed; int ret; nvme_stop_ctrl(&ctrl->ctrl); nvme_loop_shutdown_ctrl(ctrl); ret = nvme_loop_configure_admin_queue(ctrl); if (ret)