// SPDX-License-Identifier: GPL-2.0/* * NVMe over Fabrics loopback device. * Copyright (c) 2015-2016 HGST, a Western Digital Company. */#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt#include <linux/scatterlist.h>#include <linux/blk-mq.h>#include <linux/nvme.h>#include <linux/module.h>#include <linux/parser.h>#include "nvmet.h"#include "../host/nvme.h"#include "../host/fabrics.h" #define NVME_LOOP_MAX_SEGMENTS 256 struct nvme_loop_iod { struct nvme_request nvme_req; struct nvme_command cmd; struct nvme_completion cqe; struct nvmet_req req; struct nvme_loop_queue *queue; struct work_struct work; struct sg_table sg_table; struct scatterlist first_sgl[];}; struct nvme_loop_ctrl { struct nvme_loop_queue *queues; struct blk_mq_tag_set admin_tag_set; struct list_head list; struct blk_mq_tag_set tag_set; struct nvme_loop_iod async_event_iod; struct nvme_ctrl ctrl; struct nvmet_port *port;}; static inline struct nvme_loop_ctrl *to_loop_ctrl(struct nvme_ctrl *ctrl){ return container_of(ctrl, struct nvme_loop_ctrl, ctrl);} enum nvme_loop_queue_flags { NVME_LOOP_Q_LIVE = 0,}; struct nvme_loop_queue { struct nvmet_cq nvme_cq; struct nvmet_sq nvme_sq; struct nvme_loop_ctrl *ctrl; unsigned long flags;}; static LIST_HEAD(nvme_loop_ports);static DEFINE_MUTEX(nvme_loop_ports_mutex); static LIST_HEAD(nvme_loop_ctrl_list);static DEFINE_MUTEX(nvme_loop_ctrl_mutex); static void nvme_loop_queue_response(struct nvmet_req *nvme_req);static void nvme_loop_delete_ctrl(struct nvmet_ctrl *ctrl); static const struct nvmet_fabrics_ops nvme_loop_ops; static inline int nvme_loop_queue_idx(struct nvme_loop_queue *queue){ return queue - queue->ctrl->queues;} static void nvme_loop_complete_rq(struct request *req){ struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req); sg_free_table_chained(&iod->sg_table, NVME_INLINE_SG_CNT); nvme_complete_rq(req);} static struct blk_mq_tags *nvme_loop_tagset(struct nvme_loop_queue *queue){ u32 queue_idx = nvme_loop_queue_idx(queue); if (queue_idx == 0) return queue->ctrl->admin_tag_set.tags[queue_idx]; return queue->ctrl->tag_set.tags[queue_idx - 1];} static void nvme_loop_queue_response(struct nvmet_req *req){ struct nvme_loop_queue *queue = container_of(req->sq, struct nvme_loop_queue, nvme_sq); struct nvme_completion *cqe = req->cqe; /* * AEN requests are special as they don't time out and can * survive any kind of queue freeze and often don't respond to * aborts. We don't even bother to allocate a struct request * for them but rather special case them here. */ if (unlikely(nvme_is_aen_req(nvme_loop_queue_idx(queue), cqe->command_id))) { nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status, &cqe->result); } else { struct request *rq; rq = blk_mq_tag_to_rq(nvme_loop_tagset(queue), cqe->command_id); if (!rq) { dev_err(queue->ctrl->ctrl.device, "tag 0x%x on queue %d not found\n", cqe->command_id, nvme_loop_queue_idx(queue)); return; } if (!nvme_try_complete_req(rq, cqe->status, cqe->result)) nvme_loop_complete_rq(rq); }} static void nvme_loop_execute_work(struct work_struct *work){ struct nvme_loop_iod *iod = container_of(work, struct nvme_loop_iod, work); iod->req.execute(&iod->req);} static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd){ struct nvme_ns *ns = hctx->queue->queuedata; struct nvme_loop_queue *queue = hctx->driver_data; struct request *req = bd->rq; struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req); bool queue_ready = test_bit(NVME_LOOP_Q_LIVE, &queue->flags); blk_status_t ret; if (!nvme_check_ready(&queue->ctrl->ctrl, req, queue_ready)) return nvme_fail_nonready_command(&queue->ctrl->ctrl, req); ret = nvme_setup_cmd(ns, req); if (ret) return ret; blk_mq_start_request(req); iod->cmd.common.flags |= NVME_CMD_SGL_METABUF; iod->req.port = queue->ctrl->port; if (!nvmet_req_init(&iod->req, &queue->nvme_cq, &queue->nvme_sq, &nvme_loop_ops)) return BLK_STS_OK; if (blk_rq_nr_phys_segments(req)) { iod->sg_table.sgl = iod->first_sgl; if (sg_alloc_table_chained(&iod->sg_table, blk_rq_nr_phys_segments(req), iod->sg_table.sgl, NVME_INLINE_SG_CNT)) { nvme_cleanup_cmd(req); return BLK_STS_RESOURCE; } iod->req.sg = iod->sg_table.sgl; iod->req.sg_cnt = blk_rq_map_sg(req->q, req, iod->sg_table.sgl); iod->req.transfer_len = blk_rq_payload_bytes(req); } schedule_work(&iod->work); return BLK_STS_OK;} static void nvme_loop_submit_async_event(struct nvme_ctrl *arg){ struct nvme_loop_ctrl *ctrl = to_loop_ctrl(arg); struct nvme_loop_queue *queue = &ctrl->queues[0]; struct nvme_loop_iod *iod = &ctrl->async_event_iod; memset(&iod->cmd, 0, sizeof(iod->cmd)); iod->cmd.common.opcode = nvme_admin_async_event; iod->cmd.common.command_id = NVME_AQ_BLK_MQ_DEPTH; iod->cmd.common.flags |= NVME_CMD_SGL_METABUF; if (!nvmet_req_init(&iod->req, &queue->nvme_cq, &queue->nvme_sq, &nvme_loop_ops)) { dev_err(ctrl->ctrl.device, "failed async event work\n"); return; } schedule_work(&iod->work);} static int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl, struct nvme_loop_iod *iod, unsigned int queue_idx){ iod->req.cmd = &iod->cmd; iod->req.cqe = &iod->cqe; iod->queue = &ctrl->queues[queue_idx]; INIT_WORK(&iod->work, nvme_loop_execute_work); return 0;} static int nvme_loop_init_request(struct blk_mq_tag_set *set, struct request *req, unsigned int hctx_idx, unsigned int numa_node){ struct nvme_loop_ctrl *ctrl = set->driver_data; struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req); nvme_req(req)->ctrl = &ctrl->ctrl; nvme_req(req)->cmd = &iod->cmd; return nvme_loop_init_iod(ctrl, blk_mq_rq_to_pdu(req), (set == &ctrl->tag_set) ? hctx_idx + 1 : 0);} static struct lock_class_key loop_hctx_fq_lock_key; static int nvme_loop_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx){ struct nvme_loop_ctrl *ctrl = data; struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1]; BUG_ON(hctx_idx >= ctrl->ctrl.queue_count); /* * flush_end_io() can be called recursively for us, so use our own * lock class key for avoiding lockdep possible recursive locking, * then we can remove the dynamically allocated lock class for each * flush queue, that way may cause horrible boot delay. */ blk_mq_hctx_set_fq_lock_class(hctx, &loop_hctx_fq_lock_key); hctx->driver_data = queue; return 0;} static int nvme_loop_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx){ struct nvme_loop_ctrl *ctrl = data; struct nvme_loop_queue *queue = &ctrl->queues[0]; BUG_ON(hctx_idx != 0); hctx->driver_data = queue; return 0;} static const struct blk_mq_ops nvme_loop_mq_ops = { .queue_rq = nvme_loop_queue_rq, .complete = nvme_loop_complete_rq, .init_request = nvme_loop_init_request, .init_hctx = nvme_loop_init_hctx,}; static const struct blk_mq_ops nvme_loop_admin_mq_ops = { .queue_rq = nvme_loop_queue_rq, .complete = nvme_loop_complete_rq, .init_request = nvme_loop_init_request, .init_hctx = nvme_loop_init_admin_hctx,}; static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl){ if (!test_and_clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags)) return; nvmet_sq_destroy(&ctrl->queues[0].nvme_sq); blk_cleanup_queue(ctrl->ctrl.admin_q); blk_cleanup_queue(ctrl->ctrl.fabrics_q); blk_mq_free_tag_set(&ctrl->admin_tag_set);} static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl){ struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl); if (list_empty(&ctrl->list)) goto free_ctrl; mutex_lock(&nvme_loop_ctrl_mutex); list_del(&ctrl->list); mutex_unlock(&nvme_loop_ctrl_mutex); if (nctrl->tagset) { blk_cleanup_queue(ctrl->ctrl.connect_q); blk_mq_free_tag_set(&ctrl->tag_set); } kfree(ctrl->queues); nvmf_free_options(nctrl->opts);free_ctrl: kfree(ctrl);} static void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl){ int i; for (i = 1; i < ctrl->ctrl.queue_count; i++) { clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags); nvmet_sq_destroy(&ctrl->queues[i].nvme_sq); } ctrl->ctrl.queue_count = 1;} static int nvme_loop_init_io_queues(struct nvme_loop_ctrl *ctrl){ struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; unsigned int nr_io_queues; int ret, i; nr_io_queues = min(opts->nr_io_queues, num_online_cpus()); ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues); if (ret || !nr_io_queues) return ret; dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues); for (i = 1; i <= nr_io_queues; i++) { ctrl->queues[i].ctrl = ctrl; ret = nvmet_sq_init(&ctrl->queues[i].nvme_sq); if (ret) goto out_destroy_queues; ctrl->ctrl.queue_count++; } return 0; out_destroy_queues: nvme_loop_destroy_io_queues(ctrl); return ret;} static int nvme_loop_connect_io_queues(struct nvme_loop_ctrl *ctrl){ int i, ret; for (i = 1; i < ctrl->ctrl.queue_count; i++) { ret = nvmf_connect_io_queue(&ctrl->ctrl, i, false); if (ret) return ret; set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags); } return 0;} static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl){ int error; memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set)); ctrl->admin_tag_set.ops = &nvme_loop_admin_mq_ops; ctrl->admin_tag_set.queue_depth = NVME_AQ_MQ_TAG_DEPTH; ctrl->admin_tag_set.reserved_tags = NVMF_RESERVED_TAGS; ctrl->admin_tag_set.numa_node = ctrl->ctrl.numa_node; ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_loop_iod) + NVME_INLINE_SG_CNT * sizeof(struct scatterlist); ctrl->admin_tag_set.driver_data = ctrl; ctrl->admin_tag_set.nr_hw_queues = 1; ctrl->admin_tag_set.timeout = NVME_ADMIN_TIMEOUT; ctrl->admin_tag_set.flags = BLK_MQ_F_NO_SCHED; ctrl->queues[0].ctrl = ctrl; error = nvmet_sq_init(&ctrl->queues[0].nvme_sq); if (error) return error; ctrl->ctrl.queue_count = 1; error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set); if (error) goto out_free_sq; ctrl->ctrl.admin_tagset = &ctrl->admin_tag_set; ctrl->ctrl.fabrics_q = blk_mq_init_queue(&ctrl->admin_tag_set); if (IS_ERR(ctrl->ctrl.fabrics_q)) { error = PTR_ERR(ctrl->ctrl.fabrics_q); goto out_free_tagset; } ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set); if (IS_ERR(ctrl->ctrl.admin_q)) { error = PTR_ERR(ctrl->ctrl.admin_q); goto out_cleanup_fabrics_q; } error = nvmf_connect_admin_queue(&ctrl->ctrl); if (error) goto out_cleanup_queue; set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags); error = nvme_enable_ctrl(&ctrl->ctrl); if (error) goto out_cleanup_queue; ctrl->ctrl.max_hw_sectors = (NVME_LOOP_MAX_SEGMENTS - 1) << (PAGE_SHIFT - 9); blk_mq_unquiesce_queue(ctrl->ctrl.admin_q); error = nvme_init_ctrl_finish(&ctrl->ctrl); if (error) goto out_cleanup_queue; return 0; out_cleanup_queue: clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags); blk_cleanup_queue(ctrl->ctrl.admin_q);out_cleanup_fabrics_q: blk_cleanup_queue(ctrl->ctrl.fabrics_q);out_free_tagset: blk_mq_free_tag_set(&ctrl->admin_tag_set);out_free_sq: nvmet_sq_destroy(&ctrl->queues[0].nvme_sq); return error;} static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl){ if (ctrl->ctrl.queue_count > 1) { nvme_stop_queues(&ctrl->ctrl); blk_mq_tagset_busy_iter(&ctrl->tag_set, nvme_cancel_request, &ctrl->ctrl); blk_mq_tagset_wait_completed_request(&ctrl->tag_set); nvme_loop_destroy_io_queues(ctrl); } blk_mq_quiesce_queue(ctrl->ctrl.admin_q); if (ctrl->ctrl.state == NVME_CTRL_LIVE) nvme_shutdown_ctrl(&ctrl->ctrl); blk_mq_tagset_busy_iter(&ctrl->admin_tag_set, nvme_cancel_request, &ctrl->ctrl); blk_mq_tagset_wait_completed_request(&ctrl->admin_tag_set); nvme_loop_destroy_admin_queue(ctrl);} static void nvme_loop_delete_ctrl_host(struct nvme_ctrl *ctrl){ nvme_loop_shutdown_ctrl(to_loop_ctrl(ctrl));} static void nvme_loop_delete_ctrl(struct nvmet_ctrl *nctrl){ struct nvme_loop_ctrl *ctrl; mutex_lock(&nvme_loop_ctrl_mutex); list_for_each_entry(ctrl, &nvme_loop_ctrl_list, list) { if (ctrl->ctrl.cntlid == nctrl->cntlid) nvme_delete_ctrl(&ctrl->ctrl); } mutex_unlock(&nvme_loop_ctrl_mutex);} static void nvme_loop_reset_ctrl_work(struct work_struct *work){ struct nvme_loop_ctrl *ctrl = container_of(work, struct nvme_loop_ctrl, ctrl.reset_work); int ret; nvme_stop_ctrl(&ctrl->ctrl); nvme_loop_shutdown_ctrl(ctrl); if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) { if (ctrl->ctrl.state != NVME_CTRL_DELETING && ctrl->ctrl.state != NVME_CTRL_DELETING_NOIO) /* state change failure for non-deleted ctrl? */ WARN_ON_ONCE(1); return; } ret = nvme_loop_configure_admin_queue(ctrl); if (ret) goto out_disable; ret = nvme_loop_init_io_queues(ctrl); if (ret) goto out_destroy_admin; ret = nvme_loop_connect_io_queues(ctrl); if (ret) goto out_destroy_io; blk_mq_update_nr_hw_queues(&ctrl->tag_set, ctrl->ctrl.queue_count - 1); if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE)) WARN_ON_ONCE(1); nvme_start_ctrl(&ctrl->ctrl); return; out_destroy_io: nvme_loop_destroy_io_queues(ctrl);out_destroy_admin: nvme_loop_destroy_admin_queue(ctrl);Lines 1–500 of 737. Load more
// SPDX-License-Identifier: GPL-2.0 /* * NVMe over Fabrics loopback device. * Copyright (c) 2015-2016 HGST, a Western Digital Company. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/scatterlist.h> #include <linux/blk-mq.h> #include <linux/nvme.h> #include <linux/module.h> #include <linux/parser.h> #include "nvmet.h" #include "../host/nvme.h" #include "../host/fabrics.h" #define NVME_LOOP_MAX_SEGMENTS 256 struct nvme_loop_iod { struct nvme_request nvme_req; struct nvme_command cmd; struct nvme_completion cqe; struct nvmet_req req; struct nvme_loop_queue *queue; struct work_struct work; struct sg_table sg_table; struct scatterlist first_sgl[]; }; struct nvme_loop_ctrl { struct nvme_loop_queue *queues; struct blk_mq_tag_set admin_tag_set; struct list_head list; struct blk_mq_tag_set tag_set; struct nvme_loop_iod async_event_iod; struct nvme_ctrl ctrl; struct nvmet_port *port; }; static inline struct nvme_loop_ctrl *to_loop_ctrl(struct nvme_ctrl *ctrl) { return container_of(ctrl, struct nvme_loop_ctrl, ctrl); } enum nvme_loop_queue_flags { NVME_LOOP_Q_LIVE = 0, }; struct nvme_loop_queue { struct nvmet_cq nvme_cq; struct nvmet_sq nvme_sq; struct nvme_loop_ctrl *ctrl; unsigned long flags; }; static LIST_HEAD(nvme_loop_ports); static DEFINE_MUTEX(nvme_loop_ports_mutex); static LIST_HEAD(nvme_loop_ctrl_list); static DEFINE_MUTEX(nvme_loop_ctrl_mutex); static void nvme_loop_queue_response(struct nvmet_req *nvme_req); static void nvme_loop_delete_ctrl(struct nvmet_ctrl *ctrl); static const struct nvmet_fabrics_ops nvme_loop_ops; static inline int nvme_loop_queue_idx(struct nvme_loop_queue *queue) { return queue - queue->ctrl->queues; } static void nvme_loop_complete_rq(struct request *req) { struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req); sg_free_table_chained(&iod->sg_table, NVME_INLINE_SG_CNT); nvme_complete_rq(req); } static struct blk_mq_tags *nvme_loop_tagset(struct nvme_loop_queue *queue) { u32 queue_idx = nvme_loop_queue_idx(queue); if (queue_idx == 0) return queue->ctrl->admin_tag_set.tags[queue_idx]; return queue->ctrl->tag_set.tags[queue_idx - 1]; } static void nvme_loop_queue_response(struct nvmet_req *req) { struct nvme_loop_queue *queue = container_of(req->sq, struct nvme_loop_queue, nvme_sq); struct nvme_completion *cqe = req->cqe; /* * AEN requests are special as they don't time out and can * survive any kind of queue freeze and often don't respond to * aborts. We don't even bother to allocate a struct request * for them but rather special case them here. */ if (unlikely(nvme_is_aen_req(nvme_loop_queue_idx(queue), cqe->command_id))) { nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status, &cqe->result); } else { struct request *rq; rq = blk_mq_tag_to_rq(nvme_loop_tagset(queue), cqe->command_id); if (!rq) { dev_err(queue->ctrl->ctrl.device, "tag 0x%x on queue %d not found\n", cqe->command_id, nvme_loop_queue_idx(queue)); return; } if (!nvme_try_complete_req(rq, cqe->status, cqe->result)) nvme_loop_complete_rq(rq); } } static void nvme_loop_execute_work(struct work_struct *work) { struct nvme_loop_iod *iod = container_of(work, struct nvme_loop_iod, work); iod->req.execute(&iod->req); } static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { struct nvme_ns *ns = hctx->queue->queuedata; struct nvme_loop_queue *queue = hctx->driver_data; struct request *req = bd->rq; struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req); bool queue_ready = test_bit(NVME_LOOP_Q_LIVE, &queue->flags); blk_status_t ret; if (!nvme_check_ready(&queue->ctrl->ctrl, req, queue_ready)) return nvme_fail_nonready_command(&queue->ctrl->ctrl, req); ret = nvme_setup_cmd(ns, req); if (ret) return ret; blk_mq_start_request(req); iod->cmd.common.flags |= NVME_CMD_SGL_METABUF; iod->req.port = queue->ctrl->port; if (!nvmet_req_init(&iod->req, &queue->nvme_cq, &queue->nvme_sq, &nvme_loop_ops)) return BLK_STS_OK; if (blk_rq_nr_phys_segments(req)) { iod->sg_table.sgl = iod->first_sgl; if (sg_alloc_table_chained(&iod->sg_table, blk_rq_nr_phys_segments(req), iod->sg_table.sgl, NVME_INLINE_SG_CNT)) { nvme_cleanup_cmd(req); return BLK_STS_RESOURCE; } iod->req.sg = iod->sg_table.sgl; iod->req.sg_cnt = blk_rq_map_sg(req->q, req, iod->sg_table.sgl); iod->req.transfer_len = blk_rq_payload_bytes(req); } schedule_work(&iod->work); return BLK_STS_OK; } static void nvme_loop_submit_async_event(struct nvme_ctrl *arg) { struct nvme_loop_ctrl *ctrl = to_loop_ctrl(arg); struct nvme_loop_queue *queue = &ctrl->queues[0]; struct nvme_loop_iod *iod = &ctrl->async_event_iod; memset(&iod->cmd, 0, sizeof(iod->cmd)); iod->cmd.common.opcode = nvme_admin_async_event; iod->cmd.common.command_id = NVME_AQ_BLK_MQ_DEPTH; iod->cmd.common.flags |= NVME_CMD_SGL_METABUF; if (!nvmet_req_init(&iod->req, &queue->nvme_cq, &queue->nvme_sq, &nvme_loop_ops)) { dev_err(ctrl->ctrl.device, "failed async event work\n"); return; } schedule_work(&iod->work); } static int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl, struct nvme_loop_iod *iod, unsigned int queue_idx) { iod->req.cmd = &iod->cmd; iod->req.cqe = &iod->cqe; iod->queue = &ctrl->queues[queue_idx]; INIT_WORK(&iod->work, nvme_loop_execute_work); return 0; } static int nvme_loop_init_request(struct blk_mq_tag_set *set, struct request *req, unsigned int hctx_idx, unsigned int numa_node) { struct nvme_loop_ctrl *ctrl = set->driver_data; struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req); nvme_req(req)->ctrl = &ctrl->ctrl; nvme_req(req)->cmd = &iod->cmd; return nvme_loop_init_iod(ctrl, blk_mq_rq_to_pdu(req), (set == &ctrl->tag_set) ? hctx_idx + 1 : 0); } static struct lock_class_key loop_hctx_fq_lock_key; static int nvme_loop_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx) { struct nvme_loop_ctrl *ctrl = data; struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1]; BUG_ON(hctx_idx >= ctrl->ctrl.queue_count); /* * flush_end_io() can be called recursively for us, so use our own * lock class key for avoiding lockdep possible recursive locking, * then we can remove the dynamically allocated lock class for each * flush queue, that way may cause horrible boot delay. */ blk_mq_hctx_set_fq_lock_class(hctx, &loop_hctx_fq_lock_key); hctx->driver_data = queue; return 0; } static int nvme_loop_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx) { struct nvme_loop_ctrl *ctrl = data; struct nvme_loop_queue *queue = &ctrl->queues[0]; BUG_ON(hctx_idx != 0); hctx->driver_data = queue; return 0; } static const struct blk_mq_ops nvme_loop_mq_ops = { .queue_rq = nvme_loop_queue_rq, .complete = nvme_loop_complete_rq, .init_request = nvme_loop_init_request, .init_hctx = nvme_loop_init_hctx, }; static const struct blk_mq_ops nvme_loop_admin_mq_ops = { .queue_rq = nvme_loop_queue_rq, .complete = nvme_loop_complete_rq, .init_request = nvme_loop_init_request, .init_hctx = nvme_loop_init_admin_hctx, }; static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl) { if (!test_and_clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags)) return; nvmet_sq_destroy(&ctrl->queues[0].nvme_sq); blk_cleanup_queue(ctrl->ctrl.admin_q); blk_cleanup_queue(ctrl->ctrl.fabrics_q); blk_mq_free_tag_set(&ctrl->admin_tag_set); } static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl) { struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl); if (list_empty(&ctrl->list)) goto free_ctrl; mutex_lock(&nvme_loop_ctrl_mutex); list_del(&ctrl->list); mutex_unlock(&nvme_loop_ctrl_mutex); if (nctrl->tagset) { blk_cleanup_queue(ctrl->ctrl.connect_q); blk_mq_free_tag_set(&ctrl->tag_set); } kfree(ctrl->queues); nvmf_free_options(nctrl->opts); free_ctrl: kfree(ctrl); } static void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl) { int i; for (i = 1; i < ctrl->ctrl.queue_count; i++) { clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags); nvmet_sq_destroy(&ctrl->queues[i].nvme_sq); } ctrl->ctrl.queue_count = 1; } static int nvme_loop_init_io_queues(struct nvme_loop_ctrl *ctrl) { struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; unsigned int nr_io_queues; int ret, i; nr_io_queues = min(opts->nr_io_queues, num_online_cpus()); ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues); if (ret || !nr_io_queues) return ret; dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues); for (i = 1; i <= nr_io_queues; i++) { ctrl->queues[i].ctrl = ctrl; ret = nvmet_sq_init(&ctrl->queues[i].nvme_sq); if (ret) goto out_destroy_queues; ctrl->ctrl.queue_count++; } return 0; out_destroy_queues: nvme_loop_destroy_io_queues(ctrl); return ret; } static int nvme_loop_connect_io_queues(struct nvme_loop_ctrl *ctrl) { int i, ret; for (i = 1; i < ctrl->ctrl.queue_count; i++) { ret = nvmf_connect_io_queue(&ctrl->ctrl, i, false); if (ret) return ret; set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags); } return 0; } static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl) { int error; memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set)); ctrl->admin_tag_set.ops = &nvme_loop_admin_mq_ops; ctrl->admin_tag_set.queue_depth = NVME_AQ_MQ_TAG_DEPTH; ctrl->admin_tag_set.reserved_tags = NVMF_RESERVED_TAGS; ctrl->admin_tag_set.numa_node = ctrl->ctrl.numa_node; ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_loop_iod) + NVME_INLINE_SG_CNT * sizeof(struct scatterlist); ctrl->admin_tag_set.driver_data = ctrl; ctrl->admin_tag_set.nr_hw_queues = 1; ctrl->admin_tag_set.timeout = NVME_ADMIN_TIMEOUT; ctrl->admin_tag_set.flags = BLK_MQ_F_NO_SCHED; ctrl->queues[0].ctrl = ctrl; error = nvmet_sq_init(&ctrl->queues[0].nvme_sq); if (error) return error; ctrl->ctrl.queue_count = 1; error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set); if (error) goto out_free_sq; ctrl->ctrl.admin_tagset = &ctrl->admin_tag_set; ctrl->ctrl.fabrics_q = blk_mq_init_queue(&ctrl->admin_tag_set); if (IS_ERR(ctrl->ctrl.fabrics_q)) { error = PTR_ERR(ctrl->ctrl.fabrics_q); goto out_free_tagset; } ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set); if (IS_ERR(ctrl->ctrl.admin_q)) { error = PTR_ERR(ctrl->ctrl.admin_q); goto out_cleanup_fabrics_q; } error = nvmf_connect_admin_queue(&ctrl->ctrl); if (error) goto out_cleanup_queue; set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags); error = nvme_enable_ctrl(&ctrl->ctrl); if (error) goto out_cleanup_queue; ctrl->ctrl.max_hw_sectors = (NVME_LOOP_MAX_SEGMENTS - 1) << (PAGE_SHIFT - 9); blk_mq_unquiesce_queue(ctrl->ctrl.admin_q); error = nvme_init_ctrl_finish(&ctrl->ctrl); if (error) goto out_cleanup_queue; return 0; out_cleanup_queue: clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags); blk_cleanup_queue(ctrl->ctrl.admin_q); out_cleanup_fabrics_q: blk_cleanup_queue(ctrl->ctrl.fabrics_q); out_free_tagset: blk_mq_free_tag_set(&ctrl->admin_tag_set); out_free_sq: nvmet_sq_destroy(&ctrl->queues[0].nvme_sq); return error; } static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl) { if (ctrl->ctrl.queue_count > 1) { nvme_stop_queues(&ctrl->ctrl); blk_mq_tagset_busy_iter(&ctrl->tag_set, nvme_cancel_request, &ctrl->ctrl); blk_mq_tagset_wait_completed_request(&ctrl->tag_set); nvme_loop_destroy_io_queues(ctrl); } blk_mq_quiesce_queue(ctrl->ctrl.admin_q); if (ctrl->ctrl.state == NVME_CTRL_LIVE) nvme_shutdown_ctrl(&ctrl->ctrl); blk_mq_tagset_busy_iter(&ctrl->admin_tag_set, nvme_cancel_request, &ctrl->ctrl); blk_mq_tagset_wait_completed_request(&ctrl->admin_tag_set); nvme_loop_destroy_admin_queue(ctrl); } static void nvme_loop_delete_ctrl_host(struct nvme_ctrl *ctrl) { nvme_loop_shutdown_ctrl(to_loop_ctrl(ctrl)); } static void nvme_loop_delete_ctrl(struct nvmet_ctrl *nctrl) { struct nvme_loop_ctrl *ctrl; mutex_lock(&nvme_loop_ctrl_mutex); list_for_each_entry(ctrl, &nvme_loop_ctrl_list, list) { if (ctrl->ctrl.cntlid == nctrl->cntlid) nvme_delete_ctrl(&ctrl->ctrl); } mutex_unlock(&nvme_loop_ctrl_mutex); } static void nvme_loop_reset_ctrl_work(struct work_struct *work) { struct nvme_loop_ctrl *ctrl = container_of(work, struct nvme_loop_ctrl, ctrl.reset_work); int ret; nvme_stop_ctrl(&ctrl->ctrl); nvme_loop_shutdown_ctrl(ctrl); if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) { if (ctrl->ctrl.state != NVME_CTRL_DELETING && ctrl->ctrl.state != NVME_CTRL_DELETING_NOIO) /* state change failure for non-deleted ctrl? */ WARN_ON_ONCE(1); return; } ret = nvme_loop_configure_admin_queue(ctrl); if (ret) goto out_disable; ret = nvme_loop_init_io_queues(ctrl); if (ret) goto out_destroy_admin; ret = nvme_loop_connect_io_queues(ctrl); if (ret) goto out_destroy_io; blk_mq_update_nr_hw_queues(&ctrl->tag_set, ctrl->ctrl.queue_count - 1); if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE)) WARN_ON_ONCE(1); nvme_start_ctrl(&ctrl->ctrl); return; out_destroy_io: nvme_loop_destroy_io_queues(ctrl); out_destroy_admin: nvme_loop_destroy_admin_queue(ctrl);