5587 lines · 148.3 KB · C
RawDownload
// SPDX-License-Identifier: GPL-2.0/* * NVM Express device driver * Copyright (c) 2011-2014, Intel Corporation. */ #include <linux/async.h>#include <linux/blkdev.h>#include <linux/blk-mq.h>#include <linux/blk-integrity.h>#include <linux/compat.h>#include <linux/delay.h>#include <linux/errno.h>#include <linux/hdreg.h>#include <linux/kernel.h>#include <linux/module.h>#include <linux/backing-dev.h>#include <linux/slab.h>#include <linux/types.h>#include <linux/pr.h>#include <linux/ptrace.h>#include <linux/nvme_ioctl.h>#include <linux/pm_qos.h>#include <linux/ratelimit.h>#include <linux/unaligned.h> #include "nvme.h"#include "fabrics.h"#include <linux/nvme-auth.h> #define CREATE_TRACE_POINTS#include "trace.h" #define NVME_MINORS		(1U << MINORBITS) /* * Write hints (bio->bi_write_stream) are u8, so FDP placement handles beyond * U8_MAX can never be selected. Cap the handle count to bound both the RUH * status buffer and the per-head plids array. */#define NVME_MAX_PLIDS		U8_MAX struct nvme_ns_info {	struct nvme_ns_ids ids;	u32 nsid;	__le32 anagrpid;	u8 pi_offset;	u16 endgid;	u64 runs;	bool is_shared;	bool is_readonly;	bool is_ready;	bool is_removed;	bool is_rotational;	bool no_vwc;}; unsigned int admin_timeout = 60;module_param(admin_timeout, uint, 0644);MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");EXPORT_SYMBOL_GPL(admin_timeout); unsigned int nvme_io_timeout = 30;module_param_named(io_timeout, nvme_io_timeout, uint, 0644);MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");EXPORT_SYMBOL_GPL(nvme_io_timeout); static unsigned char shutdown_timeout = 5;module_param(shutdown_timeout, byte, 0644);MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown"); static u8 nvme_max_retries = 5;module_param_named(max_retries, nvme_max_retries, byte, 0644);MODULE_PARM_DESC(max_retries, "max number of retries a command may have"); static unsigned long default_ps_max_latency_us = 100000;module_param(default_ps_max_latency_us, ulong, 0644);MODULE_PARM_DESC(default_ps_max_latency_us,		 "max power saving latency for new devices; use PM QOS to change per device"); static bool force_apst;module_param(force_apst, bool, 0644);MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off"); static unsigned long apst_primary_timeout_ms = 100;module_param(apst_primary_timeout_ms, ulong, 0644);MODULE_PARM_DESC(apst_primary_timeout_ms,	"primary APST timeout in ms"); static unsigned long apst_secondary_timeout_ms = 2000;module_param(apst_secondary_timeout_ms, ulong, 0644);MODULE_PARM_DESC(apst_secondary_timeout_ms,	"secondary APST timeout in ms"); static unsigned long apst_primary_latency_tol_us = 15000;module_param(apst_primary_latency_tol_us, ulong, 0644);MODULE_PARM_DESC(apst_primary_latency_tol_us,	"primary APST latency tolerance in us"); static unsigned long apst_secondary_latency_tol_us = 100000;module_param(apst_secondary_latency_tol_us, ulong, 0644);MODULE_PARM_DESC(apst_secondary_latency_tol_us,	"secondary APST latency tolerance in us"); /* * Older kernels didn't enable protection information if it was at an offset. * Newer kernels do, so it breaks reads on the upgrade if such formats were * used in prior kernels since the metadata written did not contain a valid * checksum. */static bool disable_pi_offsets = false;module_param(disable_pi_offsets, bool, 0444);MODULE_PARM_DESC(disable_pi_offsets,	"disable protection information if it has an offset"); /* * nvme_wq - hosts nvme related works that are not reset or delete * nvme_reset_wq - hosts nvme reset works * nvme_delete_wq - hosts nvme delete works * * nvme_wq will host works such as scan, aen handling, fw activation, * keep-alive, periodic reconnects etc. nvme_reset_wq * runs reset works which also flush works hosted on nvme_wq for * serialization purposes. nvme_delete_wq host controller deletion * works which flush reset works for serialization. */struct workqueue_struct *nvme_wq;EXPORT_SYMBOL_GPL(nvme_wq); struct workqueue_struct *nvme_reset_wq;EXPORT_SYMBOL_GPL(nvme_reset_wq); struct workqueue_struct *nvme_delete_wq;EXPORT_SYMBOL_GPL(nvme_delete_wq); DEFINE_MUTEX(nvme_subsystems_lock);static LIST_HEAD_GUARDED(nvme_subsystems, nvme_subsystems_lock); static DEFINE_IDA(nvme_instance_ida);static dev_t nvme_ctrl_base_chr_devt;static int nvme_class_uevent(const struct device *dev, struct kobj_uevent_env *env);static const struct class nvme_class = {	.name = "nvme",	.dev_uevent = nvme_class_uevent,}; static const struct class nvme_subsys_class = {	.name = "nvme-subsystem",}; static DEFINE_IDA(nvme_ns_chr_minor_ida);static dev_t nvme_ns_chr_devt;static const struct class nvme_ns_chr_class = {	.name = "nvme-generic",}; static void nvme_put_subsystem(struct nvme_subsystem *subsys);static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,				   struct nvme_command *cmd);static int nvme_get_log_lsi(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page,		u8 lsp, u8 csi, void *log, size_t size, u64 offset, u16 lsi); void nvme_queue_scan(struct nvme_ctrl *ctrl){	/*	 * Only new queue scan work when admin and IO queues are both alive	 */	if (nvme_ctrl_state(ctrl) == NVME_CTRL_LIVE && ctrl->tagset)		queue_work(nvme_wq, &ctrl->scan_work);} /* * Use this function to proceed with scheduling reset_work for a controller * that had previously been set to the resetting state. This is intended for * code paths that can't be interrupted by other reset attempts. A hot removal * may prevent this from succeeding. */int nvme_try_sched_reset(struct nvme_ctrl *ctrl){	if (nvme_ctrl_state(ctrl) != NVME_CTRL_RESETTING)		return -EBUSY;	if (!queue_work(nvme_reset_wq, &ctrl->reset_work))		return -EBUSY;	return 0;}EXPORT_SYMBOL_GPL(nvme_try_sched_reset); static void nvme_failfast_work(struct work_struct *work){	struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),			struct nvme_ctrl, failfast_work); 	if (nvme_ctrl_state(ctrl) != NVME_CTRL_CONNECTING)		return; 	set_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);	dev_info(ctrl->device, "failfast expired\n");	nvme_kick_requeue_lists(ctrl);} static inline void nvme_start_failfast_work(struct nvme_ctrl *ctrl){	if (!ctrl->opts || ctrl->opts->fast_io_fail_tmo == -1)		return; 	schedule_delayed_work(&ctrl->failfast_work,			      ctrl->opts->fast_io_fail_tmo * HZ);} static inline void nvme_stop_failfast_work(struct nvme_ctrl *ctrl){	if (!ctrl->opts)		return; 	cancel_delayed_work_sync(&ctrl->failfast_work);	clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);}  int nvme_reset_ctrl(struct nvme_ctrl *ctrl){	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))		return -EBUSY;	if (!queue_work(nvme_reset_wq, &ctrl->reset_work))		return -EBUSY;	return 0;}EXPORT_SYMBOL_GPL(nvme_reset_ctrl); int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl){	int ret; 	ret = nvme_reset_ctrl(ctrl);	if (!ret) {		flush_work(&ctrl->reset_work);		if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)			ret = -ENETRESET;	} 	return ret;} static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl){	dev_info(ctrl->device,		 "Removing ctrl: NQN \"%s\"\n", nvmf_ctrl_subsysnqn(ctrl)); 	flush_work(&ctrl->reset_work);	nvme_stop_ctrl(ctrl);	nvme_remove_namespaces(ctrl);	ctrl->ops->delete_ctrl(ctrl);	nvme_uninit_ctrl(ctrl);} static void nvme_delete_ctrl_work(struct work_struct *work){	struct nvme_ctrl *ctrl =		container_of(work, struct nvme_ctrl, delete_work); 	nvme_do_delete_ctrl(ctrl);} int nvme_delete_ctrl(struct nvme_ctrl *ctrl){	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))		return -EBUSY;	if (!queue_work(nvme_delete_wq, &ctrl->delete_work))		return -EBUSY;	return 0;}EXPORT_SYMBOL_GPL(nvme_delete_ctrl); void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl){	/*	 * Keep a reference until nvme_do_delete_ctrl() complete,	 * since ->delete_ctrl can free the controller.	 */	nvme_get_ctrl(ctrl);	if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))		nvme_do_delete_ctrl(ctrl);	nvme_put_ctrl(ctrl);} static blk_status_t nvme_error_status(u16 status){	switch (status & NVME_SCT_SC_MASK) {	case NVME_SC_SUCCESS:		return BLK_STS_OK;	case NVME_SC_CAP_EXCEEDED:		return BLK_STS_NOSPC;	case NVME_SC_LBA_RANGE:	case NVME_SC_CMD_INTERRUPTED:	case NVME_SC_NS_NOT_READY:		return BLK_STS_TARGET;	case NVME_SC_BAD_ATTRIBUTES:	case NVME_SC_INVALID_OPCODE:	case NVME_SC_INVALID_FIELD:	case NVME_SC_INVALID_NS:		return BLK_STS_NOTSUPP;	case NVME_SC_WRITE_FAULT:	case NVME_SC_READ_ERROR:	case NVME_SC_UNWRITTEN_BLOCK:	case NVME_SC_ACCESS_DENIED:	case NVME_SC_READ_ONLY:	case NVME_SC_COMPARE_FAILED:		return BLK_STS_MEDIUM;	case NVME_SC_GUARD_CHECK:	case NVME_SC_APPTAG_CHECK:	case NVME_SC_REFTAG_CHECK:	case NVME_SC_INVALID_PI:		return BLK_STS_PROTECTION;	case NVME_SC_RESERVATION_CONFLICT:		return BLK_STS_RESV_CONFLICT;	case NVME_SC_HOST_PATH_ERROR:		return BLK_STS_TRANSPORT;	case NVME_SC_ZONE_TOO_MANY_ACTIVE:		return BLK_STS_ZONE_ACTIVE_RESOURCE;	case NVME_SC_ZONE_TOO_MANY_OPEN:		return BLK_STS_ZONE_OPEN_RESOURCE;	default:		return BLK_STS_IOERR;	}} static void nvme_retry_req(struct request *req){	unsigned long delay = 0;	u16 crd;	struct nvme_ns *ns = req->q->queuedata; 	/* The mask and shift result must be <= 3 */	crd = (nvme_req(req)->status & NVME_STATUS_CRD) >> 11;	if (crd)		delay = nvme_req(req)->ctrl->crdt[crd - 1] * 100; 	nvme_req(req)->retries++;	if (ns)		atomic_long_inc(&ns->retries); 	blk_mq_requeue_request(req, false);	blk_mq_delay_kick_requeue_list(req->q, delay);} static void nvme_log_error(struct request *req){	struct nvme_ns *ns = req->q->queuedata;	struct nvme_request *nr = nvme_req(req); 	if (ns) {		pr_err_ratelimited("%s: %s(0x%x) @ LBA %llu, %u blocks, %s (sct 0x%x / sc 0x%x) %s%s\n",		       ns->disk ? ns->disk->disk_name : "?",		       nvme_get_opcode_str(nr->cmd->common.opcode),		       nr->cmd->common.opcode,		       nvme_sect_to_lba(ns->head, blk_rq_pos(req)),		       blk_rq_bytes(req) >> ns->head->lba_shift,		       nvme_get_error_status_str(nr->status),		       NVME_SCT(nr->status),		/* Status Code Type */		       nr->status & NVME_SC_MASK,	/* Status Code */		       nr->status & NVME_STATUS_MORE ? "MORE " : "",		       nr->status & NVME_STATUS_DNR  ? "DNR "  : "");		return;	} 	pr_err_ratelimited("%s: %s(0x%x), %s (sct 0x%x / sc 0x%x) %s%s\n",			   dev_name(nr->ctrl->device),			   nvme_get_admin_opcode_str(nr->cmd->common.opcode),			   nr->cmd->common.opcode,			   nvme_get_error_status_str(nr->status),			   NVME_SCT(nr->status),	/* Status Code Type */			   nr->status & NVME_SC_MASK,	/* Status Code */			   nr->status & NVME_STATUS_MORE ? "MORE " : "",			   nr->status & NVME_STATUS_DNR  ? "DNR "  : "");} static void nvme_log_err_passthru(struct request *req){	struct nvme_ns *ns = req->q->queuedata;	struct nvme_request *nr = nvme_req(req); 	pr_err_ratelimited("%s: %s(0x%x), %s (sct 0x%x / sc 0x%x) %s%s"		"cdw10=0x%x cdw11=0x%x cdw12=0x%x cdw13=0x%x cdw14=0x%x cdw15=0x%x\n",		ns ? ns->disk->disk_name : dev_name(nr->ctrl->device),		ns ? nvme_get_opcode_str(nr->cmd->common.opcode) :		     nvme_get_admin_opcode_str(nr->cmd->common.opcode),		nr->cmd->common.opcode,		nvme_get_error_status_str(nr->status),		NVME_SCT(nr->status),		/* Status Code Type */		nr->status & NVME_SC_MASK,	/* Status Code */		nr->status & NVME_STATUS_MORE ? "MORE " : "",		nr->status & NVME_STATUS_DNR  ? "DNR "  : "",		le32_to_cpu(nr->cmd->common.cdw10),		le32_to_cpu(nr->cmd->common.cdw11),		le32_to_cpu(nr->cmd->common.cdw12),		le32_to_cpu(nr->cmd->common.cdw13),		le32_to_cpu(nr->cmd->common.cdw14),		le32_to_cpu(nr->cmd->common.cdw15));} enum nvme_disposition {	COMPLETE,	RETRY,	FAILOVER,	AUTHENTICATE,}; static inline enum nvme_disposition nvme_decide_disposition(struct request *req){	if (likely(nvme_req(req)->status == 0))		return COMPLETE; 	if (blk_noretry_request(req) ||	    (nvme_req(req)->status & NVME_STATUS_DNR) ||	    nvme_req(req)->retries >= nvme_max_retries)		return COMPLETE; 	if ((nvme_req(req)->status & NVME_SCT_SC_MASK) == NVME_SC_AUTH_REQUIRED)		return AUTHENTICATE; 	if (req->cmd_flags & REQ_NVME_MPATH) {		if (nvme_is_path_error(nvme_req(req)->status) ||		    blk_queue_dying(req->q))			return FAILOVER;	} else {		if (blk_queue_dying(req->q))			return COMPLETE;	} 	return RETRY;} static inline void nvme_end_req_zoned(struct request *req){	if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&	    req_op(req) == REQ_OP_ZONE_APPEND) {		struct nvme_ns *ns = req->q->queuedata; 		req->__sector = nvme_lba_to_sect(ns->head,			le64_to_cpu(nvme_req(req)->result.u64));	}} static inline void __nvme_end_req(struct request *req){	struct nvme_ns *ns = req->q->queuedata;	struct nvme_request *nr = nvme_req(req); 	if (unlikely(nr->status && !(req->rq_flags & RQF_QUIET))) {		if (blk_rq_is_passthrough(req))			nvme_log_err_passthru(req);		else			nvme_log_error(req); 		if (ns)			atomic_long_inc(&ns->errors);		else			atomic_long_inc(&nr->ctrl->errors);	}	nvme_end_req_zoned(req);	nvme_trace_bio_complete(req);	if (req->cmd_flags & REQ_NVME_MPATH)		nvme_mpath_end_request(req);} void nvme_end_req(struct request *req){	blk_status_t status = nvme_error_status(nvme_req(req)->status); 	__nvme_end_req(req);	blk_mq_end_request(req, status);} static void __nvme_complete_rq(struct request *req){	struct nvme_ctrl *ctrl = nvme_req(req)->ctrl; 	nvme_cleanup_cmd(req); 	/*	 * Completions of long-running commands should not be able to	 * defer sending of periodic keep alives, since the controller	 * may have completed processing such commands a long time ago	 * (arbitrarily close to command submission time).	 * req->deadline - req->timeout is the command submission time	 * in jiffies.	 */	if (ctrl->kas &&	    req->deadline - req->timeout >= ctrl->ka_last_check_time)		ctrl->comp_seen = true; 	switch (nvme_decide_disposition(req)) {	case COMPLETE:		nvme_end_req(req);		return;	case RETRY:		nvme_retry_req(req);		return;	case FAILOVER:		nvme_failover_req(req);

Lines 1–500 of 5587.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t