// SPDX-License-Identifier: GPL-2.0/* * NVM Express device driver * Copyright (c) 2011-2014, Intel Corporation. */ #include <linux/async.h>#include <linux/blkdev.h>#include <linux/blk-mq.h>#include <linux/blk-integrity.h>#include <linux/compat.h>#include <linux/delay.h>#include <linux/errno.h>#include <linux/hdreg.h>#include <linux/kernel.h>#include <linux/module.h>#include <linux/backing-dev.h>#include <linux/slab.h>#include <linux/types.h>#include <linux/pr.h>#include <linux/ptrace.h>#include <linux/nvme_ioctl.h>#include <linux/pm_qos.h>#include <linux/ratelimit.h>#include <linux/unaligned.h> #include "nvme.h"#include "fabrics.h"#include <linux/nvme-auth.h> #define CREATE_TRACE_POINTS#include "trace.h" #define NVME_MINORS (1U << MINORBITS) /* * Write hints (bio->bi_write_stream) are u8, so FDP placement handles beyond * U8_MAX can never be selected. Cap the handle count to bound both the RUH * status buffer and the per-head plids array. */#define NVME_MAX_PLIDS U8_MAX struct nvme_ns_info { struct nvme_ns_ids ids; u32 nsid; __le32 anagrpid; u8 pi_offset; u16 endgid; u64 runs; bool is_shared; bool is_readonly; bool is_ready; bool is_removed; bool is_rotational; bool no_vwc;}; unsigned int admin_timeout = 60;module_param(admin_timeout, uint, 0644);MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");EXPORT_SYMBOL_GPL(admin_timeout); unsigned int nvme_io_timeout = 30;module_param_named(io_timeout, nvme_io_timeout, uint, 0644);MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");EXPORT_SYMBOL_GPL(nvme_io_timeout); static unsigned char shutdown_timeout = 5;module_param(shutdown_timeout, byte, 0644);MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown"); static u8 nvme_max_retries = 5;module_param_named(max_retries, nvme_max_retries, byte, 0644);MODULE_PARM_DESC(max_retries, "max number of retries a command may have"); static unsigned long default_ps_max_latency_us = 100000;module_param(default_ps_max_latency_us, ulong, 0644);MODULE_PARM_DESC(default_ps_max_latency_us, "max power saving latency for new devices; use PM QOS to change per device"); static bool force_apst;module_param(force_apst, bool, 0644);MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off"); static unsigned long apst_primary_timeout_ms = 100;module_param(apst_primary_timeout_ms, ulong, 0644);MODULE_PARM_DESC(apst_primary_timeout_ms, "primary APST timeout in ms"); static unsigned long apst_secondary_timeout_ms = 2000;module_param(apst_secondary_timeout_ms, ulong, 0644);MODULE_PARM_DESC(apst_secondary_timeout_ms, "secondary APST timeout in ms"); static unsigned long apst_primary_latency_tol_us = 15000;module_param(apst_primary_latency_tol_us, ulong, 0644);MODULE_PARM_DESC(apst_primary_latency_tol_us, "primary APST latency tolerance in us"); static unsigned long apst_secondary_latency_tol_us = 100000;module_param(apst_secondary_latency_tol_us, ulong, 0644);MODULE_PARM_DESC(apst_secondary_latency_tol_us, "secondary APST latency tolerance in us"); /* * Older kernels didn't enable protection information if it was at an offset. * Newer kernels do, so it breaks reads on the upgrade if such formats were * used in prior kernels since the metadata written did not contain a valid * checksum. */static bool disable_pi_offsets = false;module_param(disable_pi_offsets, bool, 0444);MODULE_PARM_DESC(disable_pi_offsets, "disable protection information if it has an offset"); /* * nvme_wq - hosts nvme related works that are not reset or delete * nvme_reset_wq - hosts nvme reset works * nvme_delete_wq - hosts nvme delete works * * nvme_wq will host works such as scan, aen handling, fw activation, * keep-alive, periodic reconnects etc. nvme_reset_wq * runs reset works which also flush works hosted on nvme_wq for * serialization purposes. nvme_delete_wq host controller deletion * works which flush reset works for serialization. */struct workqueue_struct *nvme_wq;EXPORT_SYMBOL_GPL(nvme_wq); struct workqueue_struct *nvme_reset_wq;EXPORT_SYMBOL_GPL(nvme_reset_wq); struct workqueue_struct *nvme_delete_wq;EXPORT_SYMBOL_GPL(nvme_delete_wq); DEFINE_MUTEX(nvme_subsystems_lock);static LIST_HEAD_GUARDED(nvme_subsystems, nvme_subsystems_lock); static DEFINE_IDA(nvme_instance_ida);static dev_t nvme_ctrl_base_chr_devt;static int nvme_class_uevent(const struct device *dev, struct kobj_uevent_env *env);static const struct class nvme_class = { .name = "nvme", .dev_uevent = nvme_class_uevent,}; static const struct class nvme_subsys_class = { .name = "nvme-subsystem",}; static DEFINE_IDA(nvme_ns_chr_minor_ida);static dev_t nvme_ns_chr_devt;static const struct class nvme_ns_chr_class = { .name = "nvme-generic",}; static void nvme_put_subsystem(struct nvme_subsystem *subsys);static void nvme_update_keep_alive(struct nvme_ctrl *ctrl, struct nvme_command *cmd);static int nvme_get_log_lsi(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi, void *log, size_t size, u64 offset, u16 lsi); void nvme_queue_scan(struct nvme_ctrl *ctrl){ /* * Only new queue scan work when admin and IO queues are both alive */ if (nvme_ctrl_state(ctrl) == NVME_CTRL_LIVE && ctrl->tagset) queue_work(nvme_wq, &ctrl->scan_work);} /* * Use this function to proceed with scheduling reset_work for a controller * that had previously been set to the resetting state. This is intended for * code paths that can't be interrupted by other reset attempts. A hot removal * may prevent this from succeeding. */int nvme_try_sched_reset(struct nvme_ctrl *ctrl){ if (nvme_ctrl_state(ctrl) != NVME_CTRL_RESETTING) return -EBUSY; if (!queue_work(nvme_reset_wq, &ctrl->reset_work)) return -EBUSY; return 0;}EXPORT_SYMBOL_GPL(nvme_try_sched_reset); static void nvme_failfast_work(struct work_struct *work){ struct nvme_ctrl *ctrl = container_of(to_delayed_work(work), struct nvme_ctrl, failfast_work); if (nvme_ctrl_state(ctrl) != NVME_CTRL_CONNECTING) return; set_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags); dev_info(ctrl->device, "failfast expired\n"); nvme_kick_requeue_lists(ctrl);} static inline void nvme_start_failfast_work(struct nvme_ctrl *ctrl){ if (!ctrl->opts || ctrl->opts->fast_io_fail_tmo == -1) return; schedule_delayed_work(&ctrl->failfast_work, ctrl->opts->fast_io_fail_tmo * HZ);} static inline void nvme_stop_failfast_work(struct nvme_ctrl *ctrl){ if (!ctrl->opts) return; cancel_delayed_work_sync(&ctrl->failfast_work); clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);} int nvme_reset_ctrl(struct nvme_ctrl *ctrl){ if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) return -EBUSY; if (!queue_work(nvme_reset_wq, &ctrl->reset_work)) return -EBUSY; return 0;}EXPORT_SYMBOL_GPL(nvme_reset_ctrl); int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl){ int ret; ret = nvme_reset_ctrl(ctrl); if (!ret) { flush_work(&ctrl->reset_work); if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) ret = -ENETRESET; } return ret;} static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl){ dev_info(ctrl->device, "Removing ctrl: NQN \"%s\"\n", nvmf_ctrl_subsysnqn(ctrl)); flush_work(&ctrl->reset_work); nvme_stop_ctrl(ctrl); nvme_remove_namespaces(ctrl); ctrl->ops->delete_ctrl(ctrl); nvme_uninit_ctrl(ctrl);} static void nvme_delete_ctrl_work(struct work_struct *work){ struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, delete_work); nvme_do_delete_ctrl(ctrl);} int nvme_delete_ctrl(struct nvme_ctrl *ctrl){ if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING)) return -EBUSY; if (!queue_work(nvme_delete_wq, &ctrl->delete_work)) return -EBUSY; return 0;}EXPORT_SYMBOL_GPL(nvme_delete_ctrl); void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl){ /* * Keep a reference until nvme_do_delete_ctrl() complete, * since ->delete_ctrl can free the controller. */ nvme_get_ctrl(ctrl); if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING)) nvme_do_delete_ctrl(ctrl); nvme_put_ctrl(ctrl);} static blk_status_t nvme_error_status(u16 status){ switch (status & NVME_SCT_SC_MASK) { case NVME_SC_SUCCESS: return BLK_STS_OK; case NVME_SC_CAP_EXCEEDED: return BLK_STS_NOSPC; case NVME_SC_LBA_RANGE: case NVME_SC_CMD_INTERRUPTED: case NVME_SC_NS_NOT_READY: return BLK_STS_TARGET; case NVME_SC_BAD_ATTRIBUTES: case NVME_SC_INVALID_OPCODE: case NVME_SC_INVALID_FIELD: case NVME_SC_INVALID_NS: return BLK_STS_NOTSUPP; case NVME_SC_WRITE_FAULT: case NVME_SC_READ_ERROR: case NVME_SC_UNWRITTEN_BLOCK: case NVME_SC_ACCESS_DENIED: case NVME_SC_READ_ONLY: case NVME_SC_COMPARE_FAILED: return BLK_STS_MEDIUM; case NVME_SC_GUARD_CHECK: case NVME_SC_APPTAG_CHECK: case NVME_SC_REFTAG_CHECK: case NVME_SC_INVALID_PI: return BLK_STS_PROTECTION; case NVME_SC_RESERVATION_CONFLICT: return BLK_STS_RESV_CONFLICT; case NVME_SC_HOST_PATH_ERROR: return BLK_STS_TRANSPORT; case NVME_SC_ZONE_TOO_MANY_ACTIVE: return BLK_STS_ZONE_ACTIVE_RESOURCE; case NVME_SC_ZONE_TOO_MANY_OPEN: return BLK_STS_ZONE_OPEN_RESOURCE; default: return BLK_STS_IOERR; }} static void nvme_retry_req(struct request *req){ unsigned long delay = 0; u16 crd; struct nvme_ns *ns = req->q->queuedata; /* The mask and shift result must be <= 3 */ crd = (nvme_req(req)->status & NVME_STATUS_CRD) >> 11; if (crd) delay = nvme_req(req)->ctrl->crdt[crd - 1] * 100; nvme_req(req)->retries++; if (ns) atomic_long_inc(&ns->retries); blk_mq_requeue_request(req, false); blk_mq_delay_kick_requeue_list(req->q, delay);} static void nvme_log_error(struct request *req){ struct nvme_ns *ns = req->q->queuedata; struct nvme_request *nr = nvme_req(req); if (ns) { pr_err_ratelimited("%s: %s(0x%x) @ LBA %llu, %u blocks, %s (sct 0x%x / sc 0x%x) %s%s\n", ns->disk ? ns->disk->disk_name : "?", nvme_get_opcode_str(nr->cmd->common.opcode), nr->cmd->common.opcode, nvme_sect_to_lba(ns->head, blk_rq_pos(req)), blk_rq_bytes(req) >> ns->head->lba_shift, nvme_get_error_status_str(nr->status), NVME_SCT(nr->status), /* Status Code Type */ nr->status & NVME_SC_MASK, /* Status Code */ nr->status & NVME_STATUS_MORE ? "MORE " : "", nr->status & NVME_STATUS_DNR ? "DNR " : ""); return; } pr_err_ratelimited("%s: %s(0x%x), %s (sct 0x%x / sc 0x%x) %s%s\n", dev_name(nr->ctrl->device), nvme_get_admin_opcode_str(nr->cmd->common.opcode), nr->cmd->common.opcode, nvme_get_error_status_str(nr->status), NVME_SCT(nr->status), /* Status Code Type */ nr->status & NVME_SC_MASK, /* Status Code */ nr->status & NVME_STATUS_MORE ? "MORE " : "", nr->status & NVME_STATUS_DNR ? "DNR " : "");} static void nvme_log_err_passthru(struct request *req){ struct nvme_ns *ns = req->q->queuedata; struct nvme_request *nr = nvme_req(req); pr_err_ratelimited("%s: %s(0x%x), %s (sct 0x%x / sc 0x%x) %s%s" "cdw10=0x%x cdw11=0x%x cdw12=0x%x cdw13=0x%x cdw14=0x%x cdw15=0x%x\n", ns ? ns->disk->disk_name : dev_name(nr->ctrl->device), ns ? nvme_get_opcode_str(nr->cmd->common.opcode) : nvme_get_admin_opcode_str(nr->cmd->common.opcode), nr->cmd->common.opcode, nvme_get_error_status_str(nr->status), NVME_SCT(nr->status), /* Status Code Type */ nr->status & NVME_SC_MASK, /* Status Code */ nr->status & NVME_STATUS_MORE ? "MORE " : "", nr->status & NVME_STATUS_DNR ? "DNR " : "", le32_to_cpu(nr->cmd->common.cdw10), le32_to_cpu(nr->cmd->common.cdw11), le32_to_cpu(nr->cmd->common.cdw12), le32_to_cpu(nr->cmd->common.cdw13), le32_to_cpu(nr->cmd->common.cdw14), le32_to_cpu(nr->cmd->common.cdw15));} enum nvme_disposition { COMPLETE, RETRY, FAILOVER, AUTHENTICATE,}; static inline enum nvme_disposition nvme_decide_disposition(struct request *req){ if (likely(nvme_req(req)->status == 0)) return COMPLETE; if (blk_noretry_request(req) || (nvme_req(req)->status & NVME_STATUS_DNR) || nvme_req(req)->retries >= nvme_max_retries) return COMPLETE; if ((nvme_req(req)->status & NVME_SCT_SC_MASK) == NVME_SC_AUTH_REQUIRED) return AUTHENTICATE; if (req->cmd_flags & REQ_NVME_MPATH) { if (nvme_is_path_error(nvme_req(req)->status) || blk_queue_dying(req->q)) return FAILOVER; } else { if (blk_queue_dying(req->q)) return COMPLETE; } return RETRY;} static inline void nvme_end_req_zoned(struct request *req){ if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) && req_op(req) == REQ_OP_ZONE_APPEND) { struct nvme_ns *ns = req->q->queuedata; req->__sector = nvme_lba_to_sect(ns->head, le64_to_cpu(nvme_req(req)->result.u64)); }} static inline void __nvme_end_req(struct request *req){ struct nvme_ns *ns = req->q->queuedata; struct nvme_request *nr = nvme_req(req); if (unlikely(nr->status && !(req->rq_flags & RQF_QUIET))) { if (blk_rq_is_passthrough(req)) nvme_log_err_passthru(req); else nvme_log_error(req); if (ns) atomic_long_inc(&ns->errors); else atomic_long_inc(&nr->ctrl->errors); } nvme_end_req_zoned(req); nvme_trace_bio_complete(req); if (req->cmd_flags & REQ_NVME_MPATH) nvme_mpath_end_request(req);} void nvme_end_req(struct request *req){ blk_status_t status = nvme_error_status(nvme_req(req)->status); __nvme_end_req(req); blk_mq_end_request(req, status);} static void __nvme_complete_rq(struct request *req){ struct nvme_ctrl *ctrl = nvme_req(req)->ctrl; nvme_cleanup_cmd(req); /* * Completions of long-running commands should not be able to * defer sending of periodic keep alives, since the controller * may have completed processing such commands a long time ago * (arbitrarily close to command submission time). * req->deadline - req->timeout is the command submission time * in jiffies. */ if (ctrl->kas && req->deadline - req->timeout >= ctrl->ka_last_check_time) ctrl->comp_seen = true; switch (nvme_decide_disposition(req)) { case COMPLETE: nvme_end_req(req); return; case RETRY: nvme_retry_req(req); return; case FAILOVER: nvme_failover_req(req);Lines 1–500 of 5587. Load more
// SPDX-License-Identifier: GPL-2.0 /* * NVM Express device driver * Copyright (c) 2011-2014, Intel Corporation. */ #include <linux/async.h> #include <linux/blkdev.h> #include <linux/blk-mq.h> #include <linux/blk-integrity.h> #include <linux/compat.h> #include <linux/delay.h> #include <linux/errno.h> #include <linux/hdreg.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/backing-dev.h> #include <linux/slab.h> #include <linux/types.h> #include <linux/pr.h> #include <linux/ptrace.h> #include <linux/nvme_ioctl.h> #include <linux/pm_qos.h> #include <linux/ratelimit.h> #include <linux/unaligned.h> #include "nvme.h" #include "fabrics.h" #include <linux/nvme-auth.h> #define CREATE_TRACE_POINTS #include "trace.h" #define NVME_MINORS (1U << MINORBITS) /* * Write hints (bio->bi_write_stream) are u8, so FDP placement handles beyond * U8_MAX can never be selected. Cap the handle count to bound both the RUH * status buffer and the per-head plids array. */ #define NVME_MAX_PLIDS U8_MAX struct nvme_ns_info { struct nvme_ns_ids ids; u32 nsid; __le32 anagrpid; u8 pi_offset; u16 endgid; u64 runs; bool is_shared; bool is_readonly; bool is_ready; bool is_removed; bool is_rotational; bool no_vwc; }; unsigned int admin_timeout = 60; module_param(admin_timeout, uint, 0644); MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands"); EXPORT_SYMBOL_GPL(admin_timeout); unsigned int nvme_io_timeout = 30; module_param_named(io_timeout, nvme_io_timeout, uint, 0644); MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O"); EXPORT_SYMBOL_GPL(nvme_io_timeout); static unsigned char shutdown_timeout = 5; module_param(shutdown_timeout, byte, 0644); MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown"); static u8 nvme_max_retries = 5; module_param_named(max_retries, nvme_max_retries, byte, 0644); MODULE_PARM_DESC(max_retries, "max number of retries a command may have"); static unsigned long default_ps_max_latency_us = 100000; module_param(default_ps_max_latency_us, ulong, 0644); MODULE_PARM_DESC(default_ps_max_latency_us, "max power saving latency for new devices; use PM QOS to change per device"); static bool force_apst; module_param(force_apst, bool, 0644); MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off"); static unsigned long apst_primary_timeout_ms = 100; module_param(apst_primary_timeout_ms, ulong, 0644); MODULE_PARM_DESC(apst_primary_timeout_ms, "primary APST timeout in ms"); static unsigned long apst_secondary_timeout_ms = 2000; module_param(apst_secondary_timeout_ms, ulong, 0644); MODULE_PARM_DESC(apst_secondary_timeout_ms, "secondary APST timeout in ms"); static unsigned long apst_primary_latency_tol_us = 15000; module_param(apst_primary_latency_tol_us, ulong, 0644); MODULE_PARM_DESC(apst_primary_latency_tol_us, "primary APST latency tolerance in us"); static unsigned long apst_secondary_latency_tol_us = 100000; module_param(apst_secondary_latency_tol_us, ulong, 0644); MODULE_PARM_DESC(apst_secondary_latency_tol_us, "secondary APST latency tolerance in us"); /* * Older kernels didn't enable protection information if it was at an offset. * Newer kernels do, so it breaks reads on the upgrade if such formats were * used in prior kernels since the metadata written did not contain a valid * checksum. */ static bool disable_pi_offsets = false; module_param(disable_pi_offsets, bool, 0444); MODULE_PARM_DESC(disable_pi_offsets, "disable protection information if it has an offset"); /* * nvme_wq - hosts nvme related works that are not reset or delete * nvme_reset_wq - hosts nvme reset works * nvme_delete_wq - hosts nvme delete works * * nvme_wq will host works such as scan, aen handling, fw activation, * keep-alive, periodic reconnects etc. nvme_reset_wq * runs reset works which also flush works hosted on nvme_wq for * serialization purposes. nvme_delete_wq host controller deletion * works which flush reset works for serialization. */ struct workqueue_struct *nvme_wq; EXPORT_SYMBOL_GPL(nvme_wq); struct workqueue_struct *nvme_reset_wq; EXPORT_SYMBOL_GPL(nvme_reset_wq); struct workqueue_struct *nvme_delete_wq; EXPORT_SYMBOL_GPL(nvme_delete_wq); DEFINE_MUTEX(nvme_subsystems_lock); static LIST_HEAD_GUARDED(nvme_subsystems, nvme_subsystems_lock); static DEFINE_IDA(nvme_instance_ida); static dev_t nvme_ctrl_base_chr_devt; static int nvme_class_uevent(const struct device *dev, struct kobj_uevent_env *env); static const struct class nvme_class = { .name = "nvme", .dev_uevent = nvme_class_uevent, }; static const struct class nvme_subsys_class = { .name = "nvme-subsystem", }; static DEFINE_IDA(nvme_ns_chr_minor_ida); static dev_t nvme_ns_chr_devt; static const struct class nvme_ns_chr_class = { .name = "nvme-generic", }; static void nvme_put_subsystem(struct nvme_subsystem *subsys); static void nvme_update_keep_alive(struct nvme_ctrl *ctrl, struct nvme_command *cmd); static int nvme_get_log_lsi(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi, void *log, size_t size, u64 offset, u16 lsi); void nvme_queue_scan(struct nvme_ctrl *ctrl) { /* * Only new queue scan work when admin and IO queues are both alive */ if (nvme_ctrl_state(ctrl) == NVME_CTRL_LIVE && ctrl->tagset) queue_work(nvme_wq, &ctrl->scan_work); } /* * Use this function to proceed with scheduling reset_work for a controller * that had previously been set to the resetting state. This is intended for * code paths that can't be interrupted by other reset attempts. A hot removal * may prevent this from succeeding. */ int nvme_try_sched_reset(struct nvme_ctrl *ctrl) { if (nvme_ctrl_state(ctrl) != NVME_CTRL_RESETTING) return -EBUSY; if (!queue_work(nvme_reset_wq, &ctrl->reset_work)) return -EBUSY; return 0; } EXPORT_SYMBOL_GPL(nvme_try_sched_reset); static void nvme_failfast_work(struct work_struct *work) { struct nvme_ctrl *ctrl = container_of(to_delayed_work(work), struct nvme_ctrl, failfast_work); if (nvme_ctrl_state(ctrl) != NVME_CTRL_CONNECTING) return; set_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags); dev_info(ctrl->device, "failfast expired\n"); nvme_kick_requeue_lists(ctrl); } static inline void nvme_start_failfast_work(struct nvme_ctrl *ctrl) { if (!ctrl->opts || ctrl->opts->fast_io_fail_tmo == -1) return; schedule_delayed_work(&ctrl->failfast_work, ctrl->opts->fast_io_fail_tmo * HZ); } static inline void nvme_stop_failfast_work(struct nvme_ctrl *ctrl) { if (!ctrl->opts) return; cancel_delayed_work_sync(&ctrl->failfast_work); clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags); } int nvme_reset_ctrl(struct nvme_ctrl *ctrl) { if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) return -EBUSY; if (!queue_work(nvme_reset_wq, &ctrl->reset_work)) return -EBUSY; return 0; } EXPORT_SYMBOL_GPL(nvme_reset_ctrl); int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl) { int ret; ret = nvme_reset_ctrl(ctrl); if (!ret) { flush_work(&ctrl->reset_work); if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) ret = -ENETRESET; } return ret; } static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl) { dev_info(ctrl->device, "Removing ctrl: NQN \"%s\"\n", nvmf_ctrl_subsysnqn(ctrl)); flush_work(&ctrl->reset_work); nvme_stop_ctrl(ctrl); nvme_remove_namespaces(ctrl); ctrl->ops->delete_ctrl(ctrl); nvme_uninit_ctrl(ctrl); } static void nvme_delete_ctrl_work(struct work_struct *work) { struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, delete_work); nvme_do_delete_ctrl(ctrl); } int nvme_delete_ctrl(struct nvme_ctrl *ctrl) { if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING)) return -EBUSY; if (!queue_work(nvme_delete_wq, &ctrl->delete_work)) return -EBUSY; return 0; } EXPORT_SYMBOL_GPL(nvme_delete_ctrl); void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl) { /* * Keep a reference until nvme_do_delete_ctrl() complete, * since ->delete_ctrl can free the controller. */ nvme_get_ctrl(ctrl); if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING)) nvme_do_delete_ctrl(ctrl); nvme_put_ctrl(ctrl); } static blk_status_t nvme_error_status(u16 status) { switch (status & NVME_SCT_SC_MASK) { case NVME_SC_SUCCESS: return BLK_STS_OK; case NVME_SC_CAP_EXCEEDED: return BLK_STS_NOSPC; case NVME_SC_LBA_RANGE: case NVME_SC_CMD_INTERRUPTED: case NVME_SC_NS_NOT_READY: return BLK_STS_TARGET; case NVME_SC_BAD_ATTRIBUTES: case NVME_SC_INVALID_OPCODE: case NVME_SC_INVALID_FIELD: case NVME_SC_INVALID_NS: return BLK_STS_NOTSUPP; case NVME_SC_WRITE_FAULT: case NVME_SC_READ_ERROR: case NVME_SC_UNWRITTEN_BLOCK: case NVME_SC_ACCESS_DENIED: case NVME_SC_READ_ONLY: case NVME_SC_COMPARE_FAILED: return BLK_STS_MEDIUM; case NVME_SC_GUARD_CHECK: case NVME_SC_APPTAG_CHECK: case NVME_SC_REFTAG_CHECK: case NVME_SC_INVALID_PI: return BLK_STS_PROTECTION; case NVME_SC_RESERVATION_CONFLICT: return BLK_STS_RESV_CONFLICT; case NVME_SC_HOST_PATH_ERROR: return BLK_STS_TRANSPORT; case NVME_SC_ZONE_TOO_MANY_ACTIVE: return BLK_STS_ZONE_ACTIVE_RESOURCE; case NVME_SC_ZONE_TOO_MANY_OPEN: return BLK_STS_ZONE_OPEN_RESOURCE; default: return BLK_STS_IOERR; } } static void nvme_retry_req(struct request *req) { unsigned long delay = 0; u16 crd; struct nvme_ns *ns = req->q->queuedata; /* The mask and shift result must be <= 3 */ crd = (nvme_req(req)->status & NVME_STATUS_CRD) >> 11; if (crd) delay = nvme_req(req)->ctrl->crdt[crd - 1] * 100; nvme_req(req)->retries++; if (ns) atomic_long_inc(&ns->retries); blk_mq_requeue_request(req, false); blk_mq_delay_kick_requeue_list(req->q, delay); } static void nvme_log_error(struct request *req) { struct nvme_ns *ns = req->q->queuedata; struct nvme_request *nr = nvme_req(req); if (ns) { pr_err_ratelimited("%s: %s(0x%x) @ LBA %llu, %u blocks, %s (sct 0x%x / sc 0x%x) %s%s\n", ns->disk ? ns->disk->disk_name : "?", nvme_get_opcode_str(nr->cmd->common.opcode), nr->cmd->common.opcode, nvme_sect_to_lba(ns->head, blk_rq_pos(req)), blk_rq_bytes(req) >> ns->head->lba_shift, nvme_get_error_status_str(nr->status), NVME_SCT(nr->status), /* Status Code Type */ nr->status & NVME_SC_MASK, /* Status Code */ nr->status & NVME_STATUS_MORE ? "MORE " : "", nr->status & NVME_STATUS_DNR ? "DNR " : ""); return; } pr_err_ratelimited("%s: %s(0x%x), %s (sct 0x%x / sc 0x%x) %s%s\n", dev_name(nr->ctrl->device), nvme_get_admin_opcode_str(nr->cmd->common.opcode), nr->cmd->common.opcode, nvme_get_error_status_str(nr->status), NVME_SCT(nr->status), /* Status Code Type */ nr->status & NVME_SC_MASK, /* Status Code */ nr->status & NVME_STATUS_MORE ? "MORE " : "", nr->status & NVME_STATUS_DNR ? "DNR " : ""); } static void nvme_log_err_passthru(struct request *req) { struct nvme_ns *ns = req->q->queuedata; struct nvme_request *nr = nvme_req(req); pr_err_ratelimited("%s: %s(0x%x), %s (sct 0x%x / sc 0x%x) %s%s" "cdw10=0x%x cdw11=0x%x cdw12=0x%x cdw13=0x%x cdw14=0x%x cdw15=0x%x\n", ns ? ns->disk->disk_name : dev_name(nr->ctrl->device), ns ? nvme_get_opcode_str(nr->cmd->common.opcode) : nvme_get_admin_opcode_str(nr->cmd->common.opcode), nr->cmd->common.opcode, nvme_get_error_status_str(nr->status), NVME_SCT(nr->status), /* Status Code Type */ nr->status & NVME_SC_MASK, /* Status Code */ nr->status & NVME_STATUS_MORE ? "MORE " : "", nr->status & NVME_STATUS_DNR ? "DNR " : "", le32_to_cpu(nr->cmd->common.cdw10), le32_to_cpu(nr->cmd->common.cdw11), le32_to_cpu(nr->cmd->common.cdw12), le32_to_cpu(nr->cmd->common.cdw13), le32_to_cpu(nr->cmd->common.cdw14), le32_to_cpu(nr->cmd->common.cdw15)); } enum nvme_disposition { COMPLETE, RETRY, FAILOVER, AUTHENTICATE, }; static inline enum nvme_disposition nvme_decide_disposition(struct request *req) { if (likely(nvme_req(req)->status == 0)) return COMPLETE; if (blk_noretry_request(req) || (nvme_req(req)->status & NVME_STATUS_DNR) || nvme_req(req)->retries >= nvme_max_retries) return COMPLETE; if ((nvme_req(req)->status & NVME_SCT_SC_MASK) == NVME_SC_AUTH_REQUIRED) return AUTHENTICATE; if (req->cmd_flags & REQ_NVME_MPATH) { if (nvme_is_path_error(nvme_req(req)->status) || blk_queue_dying(req->q)) return FAILOVER; } else { if (blk_queue_dying(req->q)) return COMPLETE; } return RETRY; } static inline void nvme_end_req_zoned(struct request *req) { if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) && req_op(req) == REQ_OP_ZONE_APPEND) { struct nvme_ns *ns = req->q->queuedata; req->__sector = nvme_lba_to_sect(ns->head, le64_to_cpu(nvme_req(req)->result.u64)); } } static inline void __nvme_end_req(struct request *req) { struct nvme_ns *ns = req->q->queuedata; struct nvme_request *nr = nvme_req(req); if (unlikely(nr->status && !(req->rq_flags & RQF_QUIET))) { if (blk_rq_is_passthrough(req)) nvme_log_err_passthru(req); else nvme_log_error(req); if (ns) atomic_long_inc(&ns->errors); else atomic_long_inc(&nr->ctrl->errors); } nvme_end_req_zoned(req); nvme_trace_bio_complete(req); if (req->cmd_flags & REQ_NVME_MPATH) nvme_mpath_end_request(req); } void nvme_end_req(struct request *req) { blk_status_t status = nvme_error_status(nvme_req(req)->status); __nvme_end_req(req); blk_mq_end_request(req, status); } static void __nvme_complete_rq(struct request *req) { struct nvme_ctrl *ctrl = nvme_req(req)->ctrl; nvme_cleanup_cmd(req); /* * Completions of long-running commands should not be able to * defer sending of periodic keep alives, since the controller * may have completed processing such commands a long time ago * (arbitrarily close to command submission time). * req->deadline - req->timeout is the command submission time * in jiffies. */ if (ctrl->kas && req->deadline - req->timeout >= ctrl->ka_last_check_time) ctrl->comp_seen = true; switch (nvme_decide_disposition(req)) { case COMPLETE: nvme_end_req(req); return; case RETRY: nvme_retry_req(req); return; case FAILOVER: nvme_failover_req(req);