// SPDX-License-Identifier: GPL-2.0/* * Copyright (C) 1991, 1992 Linus Torvalds * Copyright (C) 1994,      Karl Keyte: Added support for disk statistics * Elevator latency, (C) 2000  Andrea Arcangeli <andrea@suse.de> SuSE * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de> * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au> *	-  July2000 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001 */ /* * This handles all read/write requests to block devices */#include <linux/kernel.h>#include <linux/module.h>#include <linux/bio.h>#include <linux/blkdev.h>#include <linux/blk-pm.h>#include <linux/blk-integrity.h>#include <linux/highmem.h>#include <linux/mm.h>#include <linux/pagemap.h>#include <linux/kernel_stat.h>#include <linux/string.h>#include <linux/init.h>#include <linux/completion.h>#include <linux/slab.h>#include <linux/swap.h>#include <linux/writeback.h>#include <linux/task_io_accounting_ops.h>#include <linux/fault-inject.h>#include <linux/list_sort.h>#include <linux/delay.h>#include <linux/ratelimit.h>#include <linux/pm_runtime.h>#include <linux/t10-pi.h>#include <linux/debugfs.h>#include <linux/bpf.h>#include <linux/part_stat.h>#include <linux/sched/sysctl.h>#include <linux/blk-crypto.h> #define CREATE_TRACE_POINTS#include <trace/events/block.h> #include "blk.h"#include "blk-mq-sched.h"#include "blk-pm.h"#include "blk-cgroup.h"#include "blk-throttle.h"#include "blk-ioprio.h"#include "error-injection.h" struct dentry *blk_debugfs_root; EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);EXPORT_TRACEPOINT_SYMBOL_GPL(block_split);EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_insert); static DEFINE_IDA(blk_queue_ida); /* * For queue allocation */static struct kmem_cache *blk_requestq_cachep; /* * Controlling structure to kblockd */static struct workqueue_struct *kblockd_workqueue; /** * blk_queue_flag_set - atomically set a queue flag * @flag: flag to be set * @q: request queue */void blk_queue_flag_set(unsigned int flag, struct request_queue *q){	set_bit(flag, &q->queue_flags);}EXPORT_SYMBOL(blk_queue_flag_set); /** * blk_queue_flag_clear - atomically clear a queue flag * @flag: flag to be cleared * @q: request queue */void blk_queue_flag_clear(unsigned int flag, struct request_queue *q){	clear_bit(flag, &q->queue_flags);}EXPORT_SYMBOL(blk_queue_flag_clear); #define REQ_OP_NAME(name) [REQ_OP_##name] = #namestatic const char *const blk_op_name[] = {	REQ_OP_NAME(READ),	REQ_OP_NAME(WRITE),	REQ_OP_NAME(FLUSH),	REQ_OP_NAME(DISCARD),	REQ_OP_NAME(SECURE_ERASE),	REQ_OP_NAME(ZONE_RESET),	REQ_OP_NAME(ZONE_RESET_ALL),	REQ_OP_NAME(ZONE_OPEN),	REQ_OP_NAME(ZONE_CLOSE),	REQ_OP_NAME(ZONE_FINISH),	REQ_OP_NAME(ZONE_APPEND),	REQ_OP_NAME(WRITE_ZEROES),	REQ_OP_NAME(DRV_IN),	REQ_OP_NAME(DRV_OUT),};#undef REQ_OP_NAME /** * blk_op_str - Return the string "name" for an operation REQ_OP_name. * @op: a request operation. * * Convert a request operation REQ_OP_name into the string "name". Useful for * debugging and tracing BIOs and requests. For an invalid request operation * code, the string "UNKNOWN" is returned. */inline const char *blk_op_str(enum req_op op){	const char *op_str = "UNKNOWN"; 	if (op < ARRAY_SIZE(blk_op_name) && blk_op_name[op])		op_str = blk_op_name[op]; 	return op_str;}EXPORT_SYMBOL_GPL(blk_op_str); enum req_op str_to_blk_op(const char *op){	int i; 	for (i = 0; i < ARRAY_SIZE(blk_op_name); i++)		if (blk_op_name[i] && !strcmp(blk_op_name[i], op))			return (enum req_op)i;	return REQ_OP_LAST;} #define ENT(_tag, _errno, _desc)	\[BLK_STS_##_tag] = {				\	.errno		= _errno,		\	.tag		= __stringify(_tag),	\	.name		= _desc,		\}static const struct {	int		errno;	const char	*tag;	const char	*name;} blk_errors[] = {	ENT(OK,			0,		""),	ENT(NOTSUPP,		-EOPNOTSUPP,	"operation not supported"),	ENT(TIMEOUT,		-ETIMEDOUT,	"timeout"),	ENT(NOSPC,		-ENOSPC,	"critical space allocation"),	ENT(TRANSPORT,		-ENOLINK,	"recoverable transport"),	ENT(TARGET,		-EREMOTEIO,	"critical target"),	ENT(RESV_CONFLICT,	-EBADE,		"reservation conflict"),	ENT(MEDIUM,		-ENODATA,	"critical medium"),	ENT(PROTECTION,		-EILSEQ,	"protection"),	ENT(RESOURCE,		-ENOMEM,	"kernel resource"),	ENT(DEV_RESOURCE,	-EBUSY,		"device resource"),	ENT(AGAIN,		-EAGAIN,	"nonblocking retry"),	ENT(OFFLINE,		-ENODEV,	"device offline"), 	/* device mapper special case, should not leak out: */	ENT(DM_REQUEUE,		-EREMCHG,	"dm internal retry"), 	/* zone device specific errors */	ENT(ZONE_OPEN_RESOURCE, -ETOOMANYREFS,	"open zones exceeded"),	ENT(ZONE_ACTIVE_RESOURCE, -EOVERFLOW,	"active zones exceeded"), 	/* Command duration limit device-side timeout */	ENT(DURATION_LIMIT,	-ETIME,		"duration limit exceeded"),	ENT(INVAL,		-EINVAL,	"invalid"), 	/* everything else not covered above: */	ENT(IOERR,		-EIO,		"I/O"),};#undef ENT blk_status_t errno_to_blk_status(int errno){	int i; 	for (i = 0; i < ARRAY_SIZE(blk_errors); i++) {		if (blk_errors[i].errno == errno)			return (__force blk_status_t)i;	} 	return BLK_STS_IOERR;}EXPORT_SYMBOL_GPL(errno_to_blk_status); int blk_status_to_errno(blk_status_t status){	int idx = (__force int)status; 	if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))		return -EIO;	return blk_errors[idx].errno;}EXPORT_SYMBOL_GPL(blk_status_to_errno); const char *blk_status_to_str(blk_status_t status){	int idx = (__force int)status; 	if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))		return "<null>";	return blk_errors[idx].name;} const char *blk_status_to_tag(blk_status_t status){	int idx = (__force int)status; 	if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors) || !blk_errors[idx].tag))		return "<null>";	return blk_errors[idx].tag;} blk_status_t tag_to_blk_status(const char *tag){	int i; 	for (i = 0; i < ARRAY_SIZE(blk_errors); i++) {		if (blk_errors[i].tag &&		    !strcmp(blk_errors[i].tag, tag))			return (__force blk_status_t)i;	} 	/*	 * Return BLK_STS_OK for mismatches as this function is intended to	 * parse error status values.	 */	return BLK_STS_OK;} /** * blk_sync_queue - cancel any pending callbacks on a queue * @q: the queue * * Description: *     The block layer may perform asynchronous callback activity *     on a queue, such as calling the unplug function after a timeout. *     A block device may call blk_sync_queue to ensure that any *     such activity is cancelled, thus allowing it to release resources *     that the callbacks might use. The caller must already have made sure *     that its ->submit_bio will not re-add plugging prior to calling *     this function. * *     This function does not cancel any asynchronous activity arising *     out of elevator or throttling code. That would require elevator_exit() *     and blkcg_exit_queue() to be called with queue lock initialized. * */void blk_sync_queue(struct request_queue *q){	timer_delete_sync(&q->timeout);	cancel_work_sync(&q->timeout_work);}EXPORT_SYMBOL(blk_sync_queue); /** * blk_set_pm_only - increment pm_only counter * @q: request queue pointer */void blk_set_pm_only(struct request_queue *q){	atomic_inc(&q->pm_only);}EXPORT_SYMBOL_GPL(blk_set_pm_only); void blk_clear_pm_only(struct request_queue *q){	int pm_only; 	pm_only = atomic_dec_return(&q->pm_only);	WARN_ON_ONCE(pm_only < 0);	if (pm_only == 0)		wake_up_all(&q->mq_freeze_wq);}EXPORT_SYMBOL_GPL(blk_clear_pm_only); static void blk_free_queue_rcu(struct rcu_head *rcu_head){	struct request_queue *q = container_of(rcu_head,			struct request_queue, rcu_head); 	percpu_ref_exit(&q->q_usage_counter);	kmem_cache_free(blk_requestq_cachep, q);} static void blk_free_queue(struct request_queue *q){	blk_free_queue_stats(q->stats);	if (queue_is_mq(q))		blk_mq_release(q); 	ida_free(&blk_queue_ida, q->id);	lockdep_unregister_key(&q->io_lock_cls_key);	lockdep_unregister_key(&q->q_lock_cls_key);	call_rcu(&q->rcu_head, blk_free_queue_rcu);} /** * blk_put_queue - decrement the request_queue refcount * @q: the request_queue structure to decrement the refcount for * * Decrements the refcount of the request_queue and free it when the refcount * reaches 0. */void blk_put_queue(struct request_queue *q){	if (refcount_dec_and_test(&q->refs))		blk_free_queue(q);}EXPORT_SYMBOL(blk_put_queue); bool blk_queue_start_drain(struct request_queue *q){	/*	 * When queue DYING flag is set, we need to block new req	 * entering queue, so we call blk_freeze_queue_start() to	 * prevent I/O from crossing blk_queue_enter().	 */	bool freeze = __blk_freeze_queue_start(q, current);	if (queue_is_mq(q))		blk_mq_wake_waiters(q);	/* Make blk_queue_enter() reexamine the DYING flag. */	wake_up_all(&q->mq_freeze_wq); 	return freeze;} /** * blk_queue_enter() - try to increase q->q_usage_counter * @q: request queue pointer * @flags: BLK_MQ_REQ_NOWAIT and/or BLK_MQ_REQ_PM */int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags){	const bool pm = flags & BLK_MQ_REQ_PM; 	while (!blk_try_enter_queue(q, pm)) {		if (flags & BLK_MQ_REQ_NOWAIT)			return -EAGAIN; 		/*		 * read pair of barrier in blk_freeze_queue_start(), we need to		 * order reading __PERCPU_REF_DEAD flag of .q_usage_counter and		 * reading .mq_freeze_depth or queue dying flag, otherwise the		 * following wait may never return if the two reads are		 * reordered.		 */		smp_rmb();		wait_event(q->mq_freeze_wq,			   (!q->mq_freeze_depth &&			    blk_pm_resume_queue(pm, q)) ||			   blk_queue_dying(q));		if (blk_queue_dying(q))			return -ENODEV;	} 	rwsem_acquire_read(&q->q_lockdep_map, 0, 0, _RET_IP_);	rwsem_release(&q->q_lockdep_map, _RET_IP_);	return 0;} int __bio_queue_enter(struct request_queue *q, struct bio *bio){	while (!blk_try_enter_queue(q, false)) {		struct gendisk *disk = bio->bi_bdev->bd_disk; 		if (bio->bi_opf & REQ_NOWAIT) {			if (test_bit(GD_DEAD, &disk->state))				goto dead;			bio_wouldblock_error(bio);			return -EAGAIN;		} 		/*		 * read pair of barrier in blk_freeze_queue_start(), we need to		 * order reading __PERCPU_REF_DEAD flag of .q_usage_counter and		 * reading .mq_freeze_depth or queue dying flag, otherwise the		 * following wait may never return if the two reads are		 * reordered.		 */		smp_rmb();		wait_event(q->mq_freeze_wq,			   (!q->mq_freeze_depth &&			    blk_pm_resume_queue(false, q)) ||			   test_bit(GD_DEAD, &disk->state));		if (test_bit(GD_DEAD, &disk->state))			goto dead;	} 	rwsem_acquire_read(&q->io_lockdep_map, 0, 0, _RET_IP_);	rwsem_release(&q->io_lockdep_map, _RET_IP_);	return 0;dead:	bio_io_error(bio);	return -ENODEV;} void blk_queue_exit(struct request_queue *q){	percpu_ref_put(&q->q_usage_counter);} static void blk_queue_usage_counter_release(struct percpu_ref *ref){	struct request_queue *q =		container_of(ref, struct request_queue, q_usage_counter); 	wake_up_all(&q->mq_freeze_wq);} static void blk_rq_timed_out_timer(struct timer_list *t){	struct request_queue *q = timer_container_of(q, t, timeout); 	kblockd_schedule_work(&q->timeout_work);} static void blk_timeout_work(struct work_struct *work){} struct request_queue *blk_alloc_queue(struct queue_limits *lim, int node_id){	struct request_queue *q;	int error; 	q = kmem_cache_alloc_node(blk_requestq_cachep, GFP_KERNEL | __GFP_ZERO,				  node_id);	if (!q)		return ERR_PTR(-ENOMEM); 	q->last_merge = NULL; 	q->id = ida_alloc(&blk_queue_ida, GFP_KERNEL);	if (q->id < 0) {		error = q->id;		goto fail_q;	} 	q->stats = blk_alloc_queue_stats();	if (!q->stats) {		error = -ENOMEM;		goto fail_id;	} 	error = blk_set_default_limits(lim);	if (error)		goto fail_stats;	q->limits = *lim; 	q->node = node_id; 	atomic_set(&q->nr_active_requests_shared_tags, 0); 	timer_setup(&q->timeout, blk_rq_timed_out_timer, 0);	INIT_WORK(&q->timeout_work, blk_timeout_work);	INIT_LIST_HEAD(&q->icq_list); 	refcount_set(&q->refs, 1);	mutex_init(&q->debugfs_mutex);	mutex_init(&q->elevator_lock);	mutex_init(&q->sysfs_lock);	mutex_init(&q->limits_lock);	mutex_init(&q->rq_qos_mutex);	spin_lock_init(&q->queue_lock); 	init_waitqueue_head(&q->mq_freeze_wq);	mutex_init(&q->mq_freeze_lock); 	blkg_init_queue(q); 	/*	 * Init percpu_ref in atomic mode so that it's faster to shutdown.	 * See blk_register_queue() for details.	 */	error = percpu_ref_init(&q->q_usage_counter,				blk_queue_usage_counter_release,				PERCPU_REF_INIT_ATOMIC, GFP_KERNEL);	if (error)		goto fail_stats;	lockdep_register_key(&q->io_lock_cls_key);	lockdep_register_key(&q->q_lock_cls_key);	lockdep_init_map(&q->io_lockdep_map, "&q->q_usage_counter(io)",			 &q->io_lock_cls_key, 0);	lockdep_init_map(&q->q_lockdep_map, "&q->q_usage_counter(queue)",			 &q->q_lock_cls_key, 0);

Lines 1–500 of 1333. Load more