lael-labs/linux

Documentation/ABI/testing/procfs-diskstats+5−0

@@ -29,4 +29,9 @@ Description: · 28 unmodified lines
29	17 - sectors discarded	29	17 - sectors discarded
30	18 - time spent discarding	30	18 - time spent discarding
31		31
		32	Kernel 5.5+ appends two more fields for flush requests:
		33
		34	19 - flush requests completed successfully
		35	20 - time spent flushing
		36
32	For more details refer to Documentation/admin-guide/iostats.rst	37	For more details refer to Documentation/admin-guide/iostats.rst

Documentation/ABI/testing/sysfs-block+6−0

 		 9 - I/Os currently in progress
 		10 - time spent doing I/Os (ms)
 		11 - weighted time spent doing I/Os (ms)
+		12 - discards completed
+		13 - discards merged
+		14 - sectors discarded
+		15 - time spent discarding (ms)
+		16 - flush requests completed
+		17 - time spent flushing (ms)
 		For more details refer Documentation/admin-guide/iostats.rst

Documentation/admin-guide/iostats.rst+9−0

@@ -121,6 +121,15 @@ Field 15 -- # of milliseconds spent discarding · 120 unmodified lines
121	This is the total number of milliseconds spent by all discards (as	121	This is the total number of milliseconds spent by all discards (as
122	measured from __make_request() to end_that_request_last()).	122	measured from __make_request() to end_that_request_last()).
123		123
		124	Field 16 -- # of flush requests completed
		125	This is the total number of flush requests completed successfully.
		126
		127	Block layer combines flush requests and executes at most one at a time.
		128	This counts flush requests executed by disk. Not tracked for partitions.
		129
		130	Field 17 -- # of milliseconds spent flushing
		131	This is the total number of milliseconds spent by all flush requests.
		132
124	To avoid introducing performance bottlenecks, no locks are held while	133	To avoid introducing performance bottlenecks, no locks are held while
125	modifying these counters. This implies that minor inaccuracies may be	134	modifying these counters. This implies that minor inaccuracies may be
126	introduced when changes collide, so (for instance) adding up all the	135	introduced when changes collide, so (for instance) adding up all the

Documentation/block/stat.rst+12−2

@@ -41,6 +41,8 @@ discard I/Os requests number of discard I/Os processed · 40 unmodified lines
41	discard merges requests number of discard I/Os merged with in-queue I/O	41	discard merges requests number of discard I/Os merged with in-queue I/O
42	discard sectors sectors number of sectors discarded	42	discard sectors sectors number of sectors discarded
43	discard ticks milliseconds total wait time for discard requests	43	discard ticks milliseconds total wait time for discard requests
		44	flush I/Os requests number of flush I/Os processed
		45	flush ticks milliseconds total wait time for flush requests
44	=============== ============= =================================================	46	=============== ============= =================================================
45		47
46	read I/Os, write I/Os, discard I/0s	48	read I/Os, write I/Os, discard I/0s
@@ -48,6 +50,14 @@ read I/Os, write I/Os, discard I/0s · 1 unmodified line
48		50
49	These values increment when an I/O request completes.	51	These values increment when an I/O request completes.
50		52
		53	flush I/Os
		54	==========
		55
		56	These values increment when an flush I/O request completes.
		57
		58	Block layer combines flush requests and executes at most one at a time.
		59	This counts flush requests executed by disk. Not tracked for partitions.
		60
51	read merges, write merges, discard merges	61	read merges, write merges, discard merges
52	=========================================	62	=========================================
53		63
@@ -62,8 +72,8 @@ discarded from this block device. The "sectors" in question are the · 8 unmodified lines
62	standard UNIX 512-byte sectors, not any device- or filesystem-specific	72	standard UNIX 512-byte sectors, not any device- or filesystem-specific
63	block size. The counters are incremented when the I/O completes.	73	block size. The counters are incremented when the I/O completes.
64		74
65	read ticks, write ticks, discard ticks	75	read ticks, write ticks, discard ticks, flush ticks
66	======================================	76	===================================================
67		77
68	These values count the number of milliseconds that I/O requests have	78	These values count the number of milliseconds that I/O requests have
69	waited on this block device. If there are multiple I/O requests waiting,	79	waited on this block device. If there are multiple I/O requests waiting,

block/Kconfig+4−0

 config BLK_SCSI_REQUEST
 	bool
+config BLK_CGROUP_RWSTAT
+	bool
 config BLK_DEV_BSG
 	bool "Block layer SG support v4"
 	default y
 config BLK_DEV_THROTTLING
 	bool "Block layer bio throttling support"
 	depends on BLK_CGROUP=y
+	select BLK_CGROUP_RWSTAT
 	---help---
 	Block layer bio throttling support. It can be used to limit
 	the IO rate to a device. IO rate policies are per cgroup and

block/Kconfig.iosched+1−0

@@ -31,6 +31,7 @@ config IOSCHED_BFQ · 30 unmodified lines
31	config BFQ_GROUP_IOSCHED	31	config BFQ_GROUP_IOSCHED
32	bool "BFQ hierarchical scheduling support"	32	bool "BFQ hierarchical scheduling support"
33	depends on IOSCHED_BFQ && BLK_CGROUP	33	depends on IOSCHED_BFQ && BLK_CGROUP
		34	select BLK_CGROUP_RWSTAT
34	---help---	35	---help---
35		36
36	Enable hierarchical scheduling in BFQ, using the blkio	37	Enable hierarchical scheduling in BFQ, using the blkio

block/Makefile+1−0

@@ -16,6 +16,7 @@ obj-$(CONFIG_BLK_SCSI_REQUEST) += scsi_ioctl.o · 15 unmodified lines
16	obj-$(CONFIG_BLK_DEV_BSG) += bsg.o	16	obj-$(CONFIG_BLK_DEV_BSG) += bsg.o
17	obj-$(CONFIG_BLK_DEV_BSGLIB) += bsg-lib.o	17	obj-$(CONFIG_BLK_DEV_BSGLIB) += bsg-lib.o
18	obj-$(CONFIG_BLK_CGROUP) += blk-cgroup.o	18	obj-$(CONFIG_BLK_CGROUP) += blk-cgroup.o
		19	obj-$(CONFIG_BLK_CGROUP_RWSTAT) += blk-cgroup-rwstat.o
19	obj-$(CONFIG_BLK_DEV_THROTTLING) += blk-throttle.o	20	obj-$(CONFIG_BLK_DEV_THROTTLING) += blk-throttle.o
20	obj-$(CONFIG_BLK_CGROUP_IOLATENCY) += blk-iolatency.o	21	obj-$(CONFIG_BLK_CGROUP_IOLATENCY) += blk-iolatency.o
21	obj-$(CONFIG_BLK_CGROUP_IOCOST) += blk-iocost.o	22	obj-$(CONFIG_BLK_CGROUP_IOCOST) += blk-iocost.o

block/bfq-cgroup.c+50−35

@@ -347,6 +347,14 @@ void bfqg_and_blkg_put(struct bfq_group *bfqg) · 346 unmodified lines
347	bfqg_put(bfqg);	347	bfqg_put(bfqg);
348	}	348	}
349		349
		350	void bfqg_stats_update_legacy_io(struct request_queue q, struct request rq)
		351	{
		352	struct bfq_group *bfqg = blkg_to_bfqg(rq->bio->bi_blkg);
		353
		354	blkg_rwstat_add(&bfqg->stats.bytes, rq->cmd_flags, blk_rq_bytes(rq));
		355	blkg_rwstat_add(&bfqg->stats.ios, rq->cmd_flags, 1);
		356	}
		357
350	/* @stats = 0 */	358	/* @stats = 0 */
351	static void bfqg_stats_reset(struct bfqg_stats *stats)	359	static void bfqg_stats_reset(struct bfqg_stats *stats)
352	{	360	{
@@ -431,6 +439,8 @@ void bfq_init_entity(struct bfq_entity entity, struct bfq_group bfqg) · 78 unmodified lines
431		439
432	static void bfqg_stats_exit(struct bfqg_stats *stats)	440	static void bfqg_stats_exit(struct bfqg_stats *stats)
433	{	441	{
		442	blkg_rwstat_exit(&stats->bytes);
		443	blkg_rwstat_exit(&stats->ios);
434	#ifdef CONFIG_BFQ_CGROUP_DEBUG	444	#ifdef CONFIG_BFQ_CGROUP_DEBUG
435	blkg_rwstat_exit(&stats->merged);	445	blkg_rwstat_exit(&stats->merged);
436	blkg_rwstat_exit(&stats->service_time);	446	blkg_rwstat_exit(&stats->service_time);
@@ -448,6 +458,10 @@ static void bfqg_stats_exit(struct bfqg_stats *stats) · 11 unmodified lines
448		458
449	static int bfqg_stats_init(struct bfqg_stats *stats, gfp_t gfp)	459	static int bfqg_stats_init(struct bfqg_stats *stats, gfp_t gfp)
450	{	460	{
		461	if (blkg_rwstat_init(&stats->bytes, gfp) \|\|
		462	blkg_rwstat_init(&stats->ios, gfp))
		463	return -ENOMEM;
		464
451	#ifdef CONFIG_BFQ_CGROUP_DEBUG	465	#ifdef CONFIG_BFQ_CGROUP_DEBUG
452	if (blkg_rwstat_init(&stats->merged, gfp) \|\|	466	if (blkg_rwstat_init(&stats->merged, gfp) \|\|
453	blkg_rwstat_init(&stats->service_time, gfp) \|\|	467	blkg_rwstat_init(&stats->service_time, gfp) \|\|
@@ -1057,18 +1071,35 @@ static ssize_t bfq_io_set_weight(struct kernfs_open_file *of, · 603 unmodified lines
1057	return bfq_io_set_device_weight(of, buf, nbytes, off);	1071	return bfq_io_set_device_weight(of, buf, nbytes, off);
1058	}	1072	}
1059		1073
1060	#ifdef CONFIG_BFQ_CGROUP_DEBUG	1074	static int bfqg_print_rwstat(struct seq_file sf, void v)
1061	static int bfqg_print_stat(struct seq_file sf, void v)
1062	{	1075	{
1063	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_stat,	1076	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_rwstat,
1064	&blkcg_policy_bfq, seq_cft(sf)->private, false);	1077	&blkcg_policy_bfq, seq_cft(sf)->private, true);
1065	return 0;	1078	return 0;
1066	}	1079	}
1067		1080
1068	static int bfqg_print_rwstat(struct seq_file sf, void v)	1081	static u64 bfqg_prfill_rwstat_recursive(struct seq_file *sf,
		1082	struct blkg_policy_data *pd, int off)
1069	{	1083	{
1070	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_rwstat,	1084	struct blkg_rwstat_sample sum;
1071	&blkcg_policy_bfq, seq_cft(sf)->private, true);	1085
		1086	blkg_rwstat_recursive_sum(pd_to_blkg(pd), &blkcg_policy_bfq, off, &sum);
		1087	return __blkg_prfill_rwstat(sf, pd, &sum);
		1088	}
		1089
		1090	static int bfqg_print_rwstat_recursive(struct seq_file sf, void v)
		1091	{
		1092	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
		1093	bfqg_prfill_rwstat_recursive, &blkcg_policy_bfq,
		1094	seq_cft(sf)->private, true);
		1095	return 0;
		1096	}
		1097
		1098	#ifdef CONFIG_BFQ_CGROUP_DEBUG
		1099	static int bfqg_print_stat(struct seq_file sf, void v)
		1100	{
		1101	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_stat,
		1102	&blkcg_policy_bfq, seq_cft(sf)->private, false);
1072	return 0;	1103	return 0;
1073	}	1104	}
1074		1105
@@ -1097,15 +1128,6 @@ static u64 bfqg_prfill_stat_recursive(struct seq_file *sf, · 22 unmodified lines
1097	return __blkg_prfill_u64(sf, pd, sum);	1128	return __blkg_prfill_u64(sf, pd, sum);
1098	}	1129	}
1099		1130
1100	static u64 bfqg_prfill_rwstat_recursive(struct seq_file *sf,
1101	struct blkg_policy_data *pd, int off)
1102	{
1103	struct blkg_rwstat_sample sum;
1104
1105	blkg_rwstat_recursive_sum(pd_to_blkg(pd), &blkcg_policy_bfq, off, &sum);
1106	return __blkg_prfill_rwstat(sf, pd, &sum);
1107	}
1108
1109	static int bfqg_print_stat_recursive(struct seq_file sf, void v)	1131	static int bfqg_print_stat_recursive(struct seq_file sf, void v)
1110	{	1132	{
1111	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),	1133	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
@@ -1114,18 +1136,11 @@ static int bfqg_print_stat_recursive(struct seq_file sf, void v) · 2 unmodified lines
1114	return 0;	1136	return 0;
1115	}	1137	}
1116		1138
1117	static int bfqg_print_rwstat_recursive(struct seq_file sf, void v)
1118	{
1119	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1120	bfqg_prfill_rwstat_recursive, &blkcg_policy_bfq,
1121	seq_cft(sf)->private, true);
1122	return 0;
1123	}
1124
1125	static u64 bfqg_prfill_sectors(struct seq_file sf, struct blkg_policy_data pd,	1139	static u64 bfqg_prfill_sectors(struct seq_file sf, struct blkg_policy_data pd,
1126	int off)	1140	int off)
1127	{	1141	{
1128	u64 sum = blkg_rwstat_total(&pd->blkg->stat_bytes);	1142	struct bfq_group *bfqg = blkg_to_bfqg(pd->blkg);
		1143	u64 sum = blkg_rwstat_total(&bfqg->stats.bytes);
1129		1144
1130	return __blkg_prfill_u64(sf, pd, sum >> 9);	1145	return __blkg_prfill_u64(sf, pd, sum >> 9);
1131	}	1146	}
@@ -1142,8 +1157,8 @@ static u64 bfqg_prfill_sectors_recursive(struct seq_file *sf, · 10 unmodified lines
1142	{	1157	{
1143	struct blkg_rwstat_sample tmp;	1158	struct blkg_rwstat_sample tmp;
1144		1159
1145	blkg_rwstat_recursive_sum(pd->blkg, NULL,	1160	blkg_rwstat_recursive_sum(pd->blkg, &blkcg_policy_bfq,
1146	offsetof(struct blkcg_gq, stat_bytes), &tmp);	1161	offsetof(struct bfq_group, stats.bytes), &tmp);
1147		1162
1148	return __blkg_prfill_u64(sf, pd,	1163	return __blkg_prfill_u64(sf, pd,
1149	(tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE]) >> 9);	1164	(tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE]) >> 9);
@@ -1226,13 +1241,13 @@ struct cftype bfq_blkcg_legacy_files[] = { · 76 unmodified lines
1226	/* statistics, covers only the tasks in the bfqg */	1241	/* statistics, covers only the tasks in the bfqg */
1227	{	1242	{
1228	.name = "bfq.io_service_bytes",	1243	.name = "bfq.io_service_bytes",
1229	.private = (unsigned long)&blkcg_policy_bfq,	1244	.private = offsetof(struct bfq_group, stats.bytes),
1230	.seq_show = blkg_print_stat_bytes,	1245	.seq_show = bfqg_print_rwstat,
1231	},	1246	},
1232	{	1247	{
1233	.name = "bfq.io_serviced",	1248	.name = "bfq.io_serviced",
1234	.private = (unsigned long)&blkcg_policy_bfq,	1249	.private = offsetof(struct bfq_group, stats.ios),
1235	.seq_show = blkg_print_stat_ios,	1250	.seq_show = bfqg_print_rwstat,
1236	},	1251	},
1237	#ifdef CONFIG_BFQ_CGROUP_DEBUG	1252	#ifdef CONFIG_BFQ_CGROUP_DEBUG
1238	{	1253	{
@@ -1269,13 +1284,13 @@ struct cftype bfq_blkcg_legacy_files[] = { · 30 unmodified lines
1269	/* the same statistics which cover the bfqg and its descendants */	1284	/* the same statistics which cover the bfqg and its descendants */
1270	{	1285	{
1271	.name = "bfq.io_service_bytes_recursive",	1286	.name = "bfq.io_service_bytes_recursive",
1272	.private = (unsigned long)&blkcg_policy_bfq,	1287	.private = offsetof(struct bfq_group, stats.bytes),
1273	.seq_show = blkg_print_stat_bytes_recursive,	1288	.seq_show = bfqg_print_rwstat_recursive,
1274	},	1289	},
1275	{	1290	{
1276	.name = "bfq.io_serviced_recursive",	1291	.name = "bfq.io_serviced_recursive",
1277	.private = (unsigned long)&blkcg_policy_bfq,	1292	.private = offsetof(struct bfq_group, stats.ios),
1278	.seq_show = blkg_print_stat_ios_recursive,	1293	.seq_show = bfqg_print_rwstat_recursive,
1279	},	1294	},
1280	#ifdef CONFIG_BFQ_CGROUP_DEBUG	1295	#ifdef CONFIG_BFQ_CGROUP_DEBUG
1281	{	1296	{

block/bfq-iosched.c+4−0

 	bool idle_timer_disabled = false;
 	unsigned int cmd_flags;
+#ifdef CONFIG_BFQ_GROUP_IOSCHED
+	if (!cgroup_subsys_on_dfl(io_cgrp_subsys) && rq->bio)
+		bfqg_stats_update_legacy_io(q, rq);
+#endif
 	spin_lock_irq(&bfqd->lock);
 	if (blk_mq_sched_try_insert_merge(q, rq)) {
 		spin_unlock_irq(&bfqd->lock);

block/bfq-iosched.h+10−0

 #include <linux/hrtimer.h>
 #include <linux/blk-cgroup.h>
+#include "blk-cgroup-rwstat.h"
 #define BFQ_IOPRIO_CLASSES	3
 #define BFQ_CL_IDLE_TIMEOUT	(HZ/5)
 };
 struct bfqg_stats {
+	/* basic stats */
+	struct blkg_rwstat		bytes;
+	struct blkg_rwstat		ios;
 #ifdef CONFIG_BFQ_CGROUP_DEBUG
 	/* number of ios merged */
 	struct blkg_rwstat		merged;
 /* ---------------- cgroups-support interface ---------------- */
+void bfqg_stats_update_legacy_io(struct request_queue *q, struct request *rq);
 void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
 			      unsigned int op);
 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op);
 #define bfq_log_bfqq(bfqd, bfqq, fmt, args...)	do {			\
 	char pid_str[MAX_PID_STR_LENGTH];	\
+	if (likely(!blk_trace_note_message_enabled((bfqd)->queue)))	\
+		break;							\
 	bfq_pid_to_str((bfqq)->pid, pid_str, MAX_PID_STR_LENGTH);	\
 	blk_add_cgroup_trace_msg((bfqd)->queue,				\
 			bfqg_to_blkg(bfqq_group(bfqq))->blkcg,		\
 #define bfq_log_bfqq(bfqd, bfqq, fmt, args...) do {	\
 	char pid_str[MAX_PID_STR_LENGTH];	\
+	if (likely(!blk_trace_note_message_enabled((bfqd)->queue)))	\
+		break;							\
 	bfq_pid_to_str((bfqq)->pid, pid_str, MAX_PID_STR_LENGTH);	\
 	blk_add_trace_msg((bfqd)->queue, "bfq%s%c " fmt, pid_str,	\
 			bfq_bfqq_sync((bfqq)) ? 'S' : 'A',		\

block/blk-cgroup-rwstat.c+129−0

@@ -0,0 +1,129 @@
	1	/* SPDX-License-Identifier: GPL-2.0
	2	*
	3	* Legacy blkg rwstat helpers enabled by CONFIG_BLK_CGROUP_RWSTAT.
	4	* Do not use in new code.
	5	*/
	6	#include "blk-cgroup-rwstat.h"
	7
	8	int blkg_rwstat_init(struct blkg_rwstat *rwstat, gfp_t gfp)
	9	{
	10	int i, ret;
	11
	12	for (i = 0; i < BLKG_RWSTAT_NR; i++) {
	13	ret = percpu_counter_init(&rwstat->cpu_cnt[i], 0, gfp);
	14	if (ret) {
	15	while (--i >= 0)
	16	percpu_counter_destroy(&rwstat->cpu_cnt[i]);
	17	return ret;
	18	}
	19	atomic64_set(&rwstat->aux_cnt[i], 0);
	20	}
	21	return 0;
	22	}
	23	EXPORT_SYMBOL_GPL(blkg_rwstat_init);
	24
	25	void blkg_rwstat_exit(struct blkg_rwstat *rwstat)
	26	{
	27	int i;
	28
	29	for (i = 0; i < BLKG_RWSTAT_NR; i++)
	30	percpu_counter_destroy(&rwstat->cpu_cnt[i]);
	31	}
	32	EXPORT_SYMBOL_GPL(blkg_rwstat_exit);
	33
	34	/**
	35	* __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
	36	* @sf: seq_file to print to
	37	* @pd: policy private data of interest
	38	* @rwstat: rwstat to print
	39	*
	40	* Print @rwstat to @sf for the device assocaited with @pd.
	41	*/
	42	u64 __blkg_prfill_rwstat(struct seq_file sf, struct blkg_policy_data pd,
	43	const struct blkg_rwstat_sample *rwstat)
	44	{
	45	static const char *rwstr[] = {
	46	[BLKG_RWSTAT_READ] = "Read",
	47	[BLKG_RWSTAT_WRITE] = "Write",
	48	[BLKG_RWSTAT_SYNC] = "Sync",
	49	[BLKG_RWSTAT_ASYNC] = "Async",
	50	[BLKG_RWSTAT_DISCARD] = "Discard",
	51	};
	52	const char *dname = blkg_dev_name(pd->blkg);
	53	u64 v;
	54	int i;
	55
	56	if (!dname)
	57	return 0;
	58
	59	for (i = 0; i < BLKG_RWSTAT_NR; i++)
	60	seq_printf(sf, "%s %s %llu\n", dname, rwstr[i],
	61	rwstat->cnt[i]);
	62
	63	v = rwstat->cnt[BLKG_RWSTAT_READ] +
	64	rwstat->cnt[BLKG_RWSTAT_WRITE] +
	65	rwstat->cnt[BLKG_RWSTAT_DISCARD];
	66	seq_printf(sf, "%s Total %llu\n", dname, v);
	67	return v;
	68	}
	69	EXPORT_SYMBOL_GPL(__blkg_prfill_rwstat);
	70
	71	/**
	72	* blkg_prfill_rwstat - prfill callback for blkg_rwstat
	73	* @sf: seq_file to print to
	74	* @pd: policy private data of interest
	75	* @off: offset to the blkg_rwstat in @pd
	76	*
	77	* prfill callback for printing a blkg_rwstat.
	78	*/
	79	u64 blkg_prfill_rwstat(struct seq_file sf, struct blkg_policy_data pd,
	80	int off)
	81	{
	82	struct blkg_rwstat_sample rwstat = { };
	83
	84	blkg_rwstat_read((void *)pd + off, &rwstat);
	85	return __blkg_prfill_rwstat(sf, pd, &rwstat);
	86	}
	87	EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
	88
	89	/**
	90	* blkg_rwstat_recursive_sum - collect hierarchical blkg_rwstat
	91	* @blkg: blkg of interest
	92	* @pol: blkcg_policy which contains the blkg_rwstat
	93	* @off: offset to the blkg_rwstat in blkg_policy_data or @blkg
	94	* @sum: blkg_rwstat_sample structure containing the results
	95	*
	96	* Collect the blkg_rwstat specified by @blkg, @pol and @off and all its
	97	* online descendants and their aux counts. The caller must be holding the
	98	* queue lock for online tests.
	99	*
	100	* If @pol is NULL, blkg_rwstat is at @off bytes into @blkg; otherwise, it
	101	* is at @off bytes into @blkg's blkg_policy_data of the policy.
	102	*/
	103	void blkg_rwstat_recursive_sum(struct blkcg_gq blkg, struct blkcg_policy pol,
	104	int off, struct blkg_rwstat_sample *sum)
	105	{
	106	struct blkcg_gq *pos_blkg;
	107	struct cgroup_subsys_state *pos_css;
	108	unsigned int i;
	109
	110	lockdep_assert_held(&blkg->q->queue_lock);
	111
	112	rcu_read_lock();
	113	blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
	114	struct blkg_rwstat *rwstat;
	115
	116	if (!pos_blkg->online)
	117	continue;
	118
	119	if (pol)
	120	rwstat = (void *)blkg_to_pd(pos_blkg, pol) + off;
	121	else
	122	rwstat = (void *)pos_blkg + off;
	123
	124	for (i = 0; i < BLKG_RWSTAT_NR; i++)
	125	sum->cnt[i] = blkg_rwstat_read_counter(rwstat, i);
	126	}
	127	rcu_read_unlock();
	128	}
	129	EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum);

block/blk-cgroup-rwstat.h+149−0

@@ -0,0 +1,149 @@
	1	/* SPDX-License-Identifier: GPL-2.0
	2	*
	3	* Legacy blkg rwstat helpers enabled by CONFIG_BLK_CGROUP_RWSTAT.
	4	* Do not use in new code.
	5	*/
	6	#ifndef _BLK_CGROUP_RWSTAT_H
	7	#define _BLK_CGROUP_RWSTAT_H
	8
	9	#include <linux/blk-cgroup.h>
	10
	11	enum blkg_rwstat_type {
	12	BLKG_RWSTAT_READ,
	13	BLKG_RWSTAT_WRITE,
	14	BLKG_RWSTAT_SYNC,
	15	BLKG_RWSTAT_ASYNC,
	16	BLKG_RWSTAT_DISCARD,
	17
	18	BLKG_RWSTAT_NR,
	19	BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
	20	};
	21
	22	/*
	23	* blkg_[rw]stat->aux_cnt is excluded for local stats but included for
	24	* recursive. Used to carry stats of dead children.
	25	*/
	26	struct blkg_rwstat {
	27	struct percpu_counter cpu_cnt[BLKG_RWSTAT_NR];
	28	atomic64_t aux_cnt[BLKG_RWSTAT_NR];
	29	};
	30
	31	struct blkg_rwstat_sample {
	32	u64 cnt[BLKG_RWSTAT_NR];
	33	};
	34
	35	static inline u64 blkg_rwstat_read_counter(struct blkg_rwstat *rwstat,
	36	unsigned int idx)
	37	{
	38	return atomic64_read(&rwstat->aux_cnt[idx]) +
	39	percpu_counter_sum_positive(&rwstat->cpu_cnt[idx]);
	40	}
	41
	42	int blkg_rwstat_init(struct blkg_rwstat *rwstat, gfp_t gfp);
	43	void blkg_rwstat_exit(struct blkg_rwstat *rwstat);
	44	u64 __blkg_prfill_rwstat(struct seq_file sf, struct blkg_policy_data pd,
	45	const struct blkg_rwstat_sample *rwstat);
	46	u64 blkg_prfill_rwstat(struct seq_file sf, struct blkg_policy_data pd,
	47	int off);
	48	void blkg_rwstat_recursive_sum(struct blkcg_gq blkg, struct blkcg_policy pol,
	49	int off, struct blkg_rwstat_sample *sum);
	50
	51
	52	/**
	53	* blkg_rwstat_add - add a value to a blkg_rwstat
	54	* @rwstat: target blkg_rwstat
	55	* @op: REQ_OP and flags
	56	* @val: value to add
	57	*
	58	* Add @val to @rwstat. The counters are chosen according to @rw. The
	59	* caller is responsible for synchronizing calls to this function.
	60	*/
	61	static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
	62	unsigned int op, uint64_t val)
	63	{
	64	struct percpu_counter *cnt;
	65
	66	if (op_is_discard(op))
	67	cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_DISCARD];
	68	else if (op_is_write(op))
	69	cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_WRITE];
	70	else
	71	cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_READ];
	72
	73	percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
	74
	75	if (op_is_sync(op))
	76	cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC];
	77	else
	78	cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC];
	79
	80	percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
	81	}
	82
	83	/**
	84	* blkg_rwstat_read - read the current values of a blkg_rwstat
	85	* @rwstat: blkg_rwstat to read
	86	*
	87	* Read the current snapshot of @rwstat and return it in the aux counts.
	88	*/
	89	static inline void blkg_rwstat_read(struct blkg_rwstat *rwstat,
	90	struct blkg_rwstat_sample *result)
	91	{
	92	int i;
	93
	94	for (i = 0; i < BLKG_RWSTAT_NR; i++)
	95	result->cnt[i] =
	96	percpu_counter_sum_positive(&rwstat->cpu_cnt[i]);
	97	}
	98
	99	/**
	100	* blkg_rwstat_total - read the total count of a blkg_rwstat
	101	* @rwstat: blkg_rwstat to read
	102	*
	103	* Return the total count of @rwstat regardless of the IO direction. This
	104	* function can be called without synchronization and takes care of u64
	105	* atomicity.
	106	*/
	107	static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat)
	108	{
	109	struct blkg_rwstat_sample tmp = { };
	110
	111	blkg_rwstat_read(rwstat, &tmp);
	112	return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
	113	}
	114
	115	/**
	116	* blkg_rwstat_reset - reset a blkg_rwstat
	117	* @rwstat: blkg_rwstat to reset
	118	*/
	119	static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
	120	{
	121	int i;
	122
	123	for (i = 0; i < BLKG_RWSTAT_NR; i++) {
	124	percpu_counter_set(&rwstat->cpu_cnt[i], 0);
	125	atomic64_set(&rwstat->aux_cnt[i], 0);
	126	}
	127	}
	128
	129	/**
	130	* blkg_rwstat_add_aux - add a blkg_rwstat into another's aux count
	131	* @to: the destination blkg_rwstat
	132	* @from: the source
	133	*
	134	* Add @from's count including the aux one to @to's aux count.
	135	*/
	136	static inline void blkg_rwstat_add_aux(struct blkg_rwstat *to,
	137	struct blkg_rwstat *from)
	138	{
	139	u64 sum[BLKG_RWSTAT_NR];
	140	int i;
	141
	142	for (i = 0; i < BLKG_RWSTAT_NR; i++)
	143	sum[i] = percpu_counter_sum_positive(&from->cpu_cnt[i]);
	144
	145	for (i = 0; i < BLKG_RWSTAT_NR; i++)
	146	atomic64_add(sum[i] + atomic64_read(&from->aux_cnt[i]),
	147	&to->aux_cnt[i]);
	148	}
	149	#endif /* _BLK_CGROUP_RWSTAT_H */

block/blk-cgroup.c+99−205

@@ -80,8 +80,7 @@ static void blkg_free(struct blkcg_gq *blkg) · 79 unmodified lines
80	if (blkg->pd[i])	80	if (blkg->pd[i])
81	blkcg_policy[i]->pd_free_fn(blkg->pd[i]);	81	blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
82		82
83	blkg_rwstat_exit(&blkg->stat_ios);	83	free_percpu(blkg->iostat_cpu);
84	blkg_rwstat_exit(&blkg->stat_bytes);
85	percpu_ref_exit(&blkg->refcnt);	84	percpu_ref_exit(&blkg->refcnt);
86	kfree(blkg);	85	kfree(blkg);
87	}	86	}
@@ -146,7 +145,7 @@ static struct blkcg_gq blkg_alloc(struct blkcg blkcg, struct request_queue *q, · 58 unmodified lines
146	gfp_t gfp_mask)	145	gfp_t gfp_mask)
147	{	146	{
148	struct blkcg_gq *blkg;	147	struct blkcg_gq *blkg;
149	int i;	148	int i, cpu;
150		149
151	/* alloc and init base part */	150	/* alloc and init base part */
152	blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);	151	blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);
@@ -156,8 +155,8 @@ static struct blkcg_gq blkg_alloc(struct blkcg blkcg, struct request_queue *q, · 3 unmodified lines
156	if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask))	155	if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask))
157	goto err_free;	156	goto err_free;
158		157
159	if (blkg_rwstat_init(&blkg->stat_bytes, gfp_mask) \|\|	158	blkg->iostat_cpu = alloc_percpu_gfp(struct blkg_iostat_set, gfp_mask);
160	blkg_rwstat_init(&blkg->stat_ios, gfp_mask))	159	if (!blkg->iostat_cpu)
161	goto err_free;	160	goto err_free;
162		161
163	blkg->q = q;	162	blkg->q = q;
@@ -167,6 +166,10 @@ static struct blkcg_gq blkg_alloc(struct blkcg blkcg, struct request_queue *q, · 3 unmodified lines
167	INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);	166	INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);
168	blkg->blkcg = blkcg;	167	blkg->blkcg = blkcg;
169		168
		169	u64_stats_init(&blkg->iostat.sync);
		170	for_each_possible_cpu(cpu)
		171	u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync);
		172
170	for (i = 0; i < BLKCG_MAX_POLS; i++) {	173	for (i = 0; i < BLKCG_MAX_POLS; i++) {
171	struct blkcg_policy *pol = blkcg_policy[i];	174	struct blkcg_policy *pol = blkcg_policy[i];
172	struct blkg_policy_data *pd;	175	struct blkg_policy_data *pd;
@@ -393,7 +396,6 @@ struct blkcg_gq blkg_lookup_create(struct blkcg blkcg, · 220 unmodified lines
393	static void blkg_destroy(struct blkcg_gq *blkg)	396	static void blkg_destroy(struct blkcg_gq *blkg)
394	{	397	{
395	struct blkcg *blkcg = blkg->blkcg;	398	struct blkcg *blkcg = blkg->blkcg;
396	struct blkcg_gq *parent = blkg->parent;
397	int i;	399	int i;
398		400
399	lockdep_assert_held(&blkg->q->queue_lock);	401	lockdep_assert_held(&blkg->q->queue_lock);
@@ -410,11 +412,6 @@ static void blkg_destroy(struct blkcg_gq *blkg) · 10 unmodified lines
410	pol->pd_offline_fn(blkg->pd[i]);	412	pol->pd_offline_fn(blkg->pd[i]);
411	}	413	}
412		414
413	if (parent) {
414	blkg_rwstat_add_aux(&parent->stat_bytes, &blkg->stat_bytes);
415	blkg_rwstat_add_aux(&parent->stat_ios, &blkg->stat_ios);
416	}
417
418	blkg->online = false;	415	blkg->online = false;
419		416
420	radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);	417	radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
@@ -464,7 +461,7 @@ static int blkcg_reset_stats(struct cgroup_subsys_state *css, · 43 unmodified lines
464	{	461	{
465	struct blkcg *blkcg = css_to_blkcg(css);	462	struct blkcg *blkcg = css_to_blkcg(css);
466	struct blkcg_gq *blkg;	463	struct blkcg_gq *blkg;
467	int i;	464	int i, cpu;
468		465
469	mutex_lock(&blkcg_pol_mutex);	466	mutex_lock(&blkcg_pol_mutex);
470	spin_lock_irq(&blkcg->lock);	467	spin_lock_irq(&blkcg->lock);
@@ -475,8 +472,12 @@ static int blkcg_reset_stats(struct cgroup_subsys_state *css, · 4 unmodified lines
475	* anyway. If you get hit by a race, retry.	472	* anyway. If you get hit by a race, retry.
476	*/	473	*/
477	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {	474	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
478	blkg_rwstat_reset(&blkg->stat_bytes);	475	for_each_possible_cpu(cpu) {
479	blkg_rwstat_reset(&blkg->stat_ios);	476	struct blkg_iostat_set *bis =
		477	per_cpu_ptr(blkg->iostat_cpu, cpu);
		478	memset(bis, 0, sizeof(*bis));
		479	}
		480	memset(&blkg->iostat, 0, sizeof(blkg->iostat));
480		481
481	for (i = 0; i < BLKCG_MAX_POLS; i++) {	482	for (i = 0; i < BLKCG_MAX_POLS; i++) {
482	struct blkcg_policy *pol = blkcg_policy[i];	483	struct blkcg_policy *pol = blkcg_policy[i];
@@ -560,186 +561,6 @@ u64 __blkg_prfill_u64(struct seq_file sf, struct blkg_policy_data pd, u64 v) · 77 unmodified lines
560	}	561	}
561	EXPORT_SYMBOL_GPL(__blkg_prfill_u64);	562	EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
562		563
563	/**
564	* __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
565	* @sf: seq_file to print to
566	* @pd: policy private data of interest
567	* @rwstat: rwstat to print
568	*
569	* Print @rwstat to @sf for the device assocaited with @pd.
570	*/
571	u64 __blkg_prfill_rwstat(struct seq_file sf, struct blkg_policy_data pd,
572	const struct blkg_rwstat_sample *rwstat)
573	{
574	static const char *rwstr[] = {
575	[BLKG_RWSTAT_READ] = "Read",
576	[BLKG_RWSTAT_WRITE] = "Write",
577	[BLKG_RWSTAT_SYNC] = "Sync",
578	[BLKG_RWSTAT_ASYNC] = "Async",
579	[BLKG_RWSTAT_DISCARD] = "Discard",
580	};
581	const char *dname = blkg_dev_name(pd->blkg);
582	u64 v;
583	int i;
584
585	if (!dname)
586	return 0;
587
588	for (i = 0; i < BLKG_RWSTAT_NR; i++)
589	seq_printf(sf, "%s %s %llu\n", dname, rwstr[i],
590	rwstat->cnt[i]);
591
592	v = rwstat->cnt[BLKG_RWSTAT_READ] +
593	rwstat->cnt[BLKG_RWSTAT_WRITE] +
594	rwstat->cnt[BLKG_RWSTAT_DISCARD];
595	seq_printf(sf, "%s Total %llu\n", dname, v);
596	return v;
597	}
598	EXPORT_SYMBOL_GPL(__blkg_prfill_rwstat);
599
600	/**
601	* blkg_prfill_rwstat - prfill callback for blkg_rwstat
602	* @sf: seq_file to print to
603	* @pd: policy private data of interest
604	* @off: offset to the blkg_rwstat in @pd
605	*
606	* prfill callback for printing a blkg_rwstat.
607	*/
608	u64 blkg_prfill_rwstat(struct seq_file sf, struct blkg_policy_data pd,
609	int off)
610	{
611	struct blkg_rwstat_sample rwstat = { };
612
613	blkg_rwstat_read((void *)pd + off, &rwstat);
614	return __blkg_prfill_rwstat(sf, pd, &rwstat);
615	}
616	EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
617
618	static u64 blkg_prfill_rwstat_field(struct seq_file *sf,
619	struct blkg_policy_data *pd, int off)
620	{
621	struct blkg_rwstat_sample rwstat = { };
622
623	blkg_rwstat_read((void *)pd->blkg + off, &rwstat);
624	return __blkg_prfill_rwstat(sf, pd, &rwstat);
625	}
626
627	/**
628	* blkg_print_stat_bytes - seq_show callback for blkg->stat_bytes
629	* @sf: seq_file to print to
630	* @v: unused
631	*
632	* To be used as cftype->seq_show to print blkg->stat_bytes.
633	* cftype->private must be set to the blkcg_policy.
634	*/
635	int blkg_print_stat_bytes(struct seq_file sf, void v)
636	{
637	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
638	blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
639	offsetof(struct blkcg_gq, stat_bytes), true);
640	return 0;
641	}
642	EXPORT_SYMBOL_GPL(blkg_print_stat_bytes);
643
644	/**
645	* blkg_print_stat_bytes - seq_show callback for blkg->stat_ios
646	* @sf: seq_file to print to
647	* @v: unused
648	*
649	* To be used as cftype->seq_show to print blkg->stat_ios. cftype->private
650	* must be set to the blkcg_policy.
651	*/
652	int blkg_print_stat_ios(struct seq_file sf, void v)
653	{
654	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
655	blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
656	offsetof(struct blkcg_gq, stat_ios), true);
657	return 0;
658	}
659	EXPORT_SYMBOL_GPL(blkg_print_stat_ios);
660
661	static u64 blkg_prfill_rwstat_field_recursive(struct seq_file *sf,
662	struct blkg_policy_data *pd,
663	int off)
664	{
665	struct blkg_rwstat_sample rwstat;
666
667	blkg_rwstat_recursive_sum(pd->blkg, NULL, off, &rwstat);
668	return __blkg_prfill_rwstat(sf, pd, &rwstat);
669	}
670
671	/**
672	* blkg_print_stat_bytes_recursive - recursive version of blkg_print_stat_bytes
673	* @sf: seq_file to print to
674	* @v: unused
675	*/
676	int blkg_print_stat_bytes_recursive(struct seq_file sf, void v)
677	{
678	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
679	blkg_prfill_rwstat_field_recursive,
680	(void *)seq_cft(sf)->private,
681	offsetof(struct blkcg_gq, stat_bytes), true);
682	return 0;
683	}
684	EXPORT_SYMBOL_GPL(blkg_print_stat_bytes_recursive);
685
686	/**
687	* blkg_print_stat_ios_recursive - recursive version of blkg_print_stat_ios
688	* @sf: seq_file to print to
689	* @v: unused
690	*/
691	int blkg_print_stat_ios_recursive(struct seq_file sf, void v)
692	{
693	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
694	blkg_prfill_rwstat_field_recursive,
695	(void *)seq_cft(sf)->private,
696	offsetof(struct blkcg_gq, stat_ios), true);
697	return 0;
698	}
699	EXPORT_SYMBOL_GPL(blkg_print_stat_ios_recursive);
700
701	/**
702	* blkg_rwstat_recursive_sum - collect hierarchical blkg_rwstat
703	* @blkg: blkg of interest
704	* @pol: blkcg_policy which contains the blkg_rwstat
705	* @off: offset to the blkg_rwstat in blkg_policy_data or @blkg
706	* @sum: blkg_rwstat_sample structure containing the results
707	*
708	* Collect the blkg_rwstat specified by @blkg, @pol and @off and all its
709	* online descendants and their aux counts. The caller must be holding the
710	* queue lock for online tests.
711	*
712	* If @pol is NULL, blkg_rwstat is at @off bytes into @blkg; otherwise, it
713	* is at @off bytes into @blkg's blkg_policy_data of the policy.
714	*/
715	void blkg_rwstat_recursive_sum(struct blkcg_gq blkg, struct blkcg_policy pol,
716	int off, struct blkg_rwstat_sample *sum)
717	{
718	struct blkcg_gq *pos_blkg;
719	struct cgroup_subsys_state *pos_css;
720	unsigned int i;
721
722	lockdep_assert_held(&blkg->q->queue_lock);
723
724	rcu_read_lock();
725	blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
726	struct blkg_rwstat *rwstat;
727
728	if (!pos_blkg->online)
729	continue;
730
731	if (pol)
732	rwstat = (void *)blkg_to_pd(pos_blkg, pol) + off;
733	else
734	rwstat = (void *)pos_blkg + off;
735
736	for (i = 0; i < BLKG_RWSTAT_NR; i++)
737	sum->cnt[i] = blkg_rwstat_read_counter(rwstat, i);
738	}
739	rcu_read_unlock();
740	}
741	EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum);
742
743	/* Performs queue bypass and policy enabled checks then looks up blkg. */	564	/* Performs queue bypass and policy enabled checks then looks up blkg. */
744	static struct blkcg_gq blkg_lookup_check(struct blkcg blkcg,	565	static struct blkcg_gq blkg_lookup_check(struct blkcg blkcg,
745	const struct blkcg_policy *pol,	566	const struct blkcg_policy *pol,
@@ -923,16 +744,18 @@ static int blkcg_print_stat(struct seq_file sf, void v) · 177 unmodified lines
923	struct blkcg *blkcg = css_to_blkcg(seq_css(sf));	744	struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
924	struct blkcg_gq *blkg;	745	struct blkcg_gq *blkg;
925		746
		747	cgroup_rstat_flush(blkcg->css.cgroup);
926	rcu_read_lock();	748	rcu_read_lock();
927		749
928	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {	750	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
		751	struct blkg_iostat_set *bis = &blkg->iostat;
929	const char *dname;	752	const char *dname;
930	char *buf;	753	char *buf;
931	struct blkg_rwstat_sample rwstat;
932	u64 rbytes, wbytes, rios, wios, dbytes, dios;	754	u64 rbytes, wbytes, rios, wios, dbytes, dios;
933	size_t size = seq_get_buf(sf, &buf), off = 0;	755	size_t size = seq_get_buf(sf, &buf), off = 0;
934	int i;	756	int i;
935	bool has_stats = false;	757	bool has_stats = false;
		758	unsigned seq;
936		759
937	spin_lock_irq(&blkg->q->queue_lock);	760	spin_lock_irq(&blkg->q->queue_lock);
938		761
@@ -951,17 +774,16 @@ static int blkcg_print_stat(struct seq_file sf, void v) · 12 unmodified lines
951	*/	774	*/
952	off += scnprintf(buf+off, size-off, "%s ", dname);	775	off += scnprintf(buf+off, size-off, "%s ", dname);
953		776
954	blkg_rwstat_recursive_sum(blkg, NULL,	777	do {
955	offsetof(struct blkcg_gq, stat_bytes), &rwstat);	778	seq = u64_stats_fetch_begin(&bis->sync);
956	rbytes = rwstat.cnt[BLKG_RWSTAT_READ];
957	wbytes = rwstat.cnt[BLKG_RWSTAT_WRITE];
958	dbytes = rwstat.cnt[BLKG_RWSTAT_DISCARD];
959		779
960	blkg_rwstat_recursive_sum(blkg, NULL,	780	rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
961	offsetof(struct blkcg_gq, stat_ios), &rwstat);	781	wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
962	rios = rwstat.cnt[BLKG_RWSTAT_READ];	782	dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
963	wios = rwstat.cnt[BLKG_RWSTAT_WRITE];	783	rios = bis->cur.ios[BLKG_IOSTAT_READ];
964	dios = rwstat.cnt[BLKG_RWSTAT_DISCARD];	784	wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
		785	dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
		786	} while (u64_stats_fetch_retry(&bis->sync, seq));
965		787
966	if (rbytes \|\| wbytes \|\| rios \|\| wios) {	788	if (rbytes \|\| wbytes \|\| rios \|\| wios) {
967	has_stats = true;	789	has_stats = true;
@@ -1297,6 +1119,77 @@ static int blkcg_can_attach(struct cgroup_taskset *tset) · 329 unmodified lines
1297	return ret;	1119	return ret;
1298	}	1120	}
1299		1121
		1122	static void blkg_iostat_set(struct blkg_iostat dst, struct blkg_iostat src)
		1123	{
		1124	int i;
		1125
		1126	for (i = 0; i < BLKG_IOSTAT_NR; i++) {
		1127	dst->bytes[i] = src->bytes[i];
		1128	dst->ios[i] = src->ios[i];
		1129	}
		1130	}
		1131
		1132	static void blkg_iostat_add(struct blkg_iostat dst, struct blkg_iostat src)
		1133	{
		1134	int i;
		1135
		1136	for (i = 0; i < BLKG_IOSTAT_NR; i++) {
		1137	dst->bytes[i] += src->bytes[i];
		1138	dst->ios[i] += src->ios[i];
		1139	}
		1140	}
		1141
		1142	static void blkg_iostat_sub(struct blkg_iostat dst, struct blkg_iostat src)
		1143	{
		1144	int i;
		1145
		1146	for (i = 0; i < BLKG_IOSTAT_NR; i++) {
		1147	dst->bytes[i] -= src->bytes[i];
		1148	dst->ios[i] -= src->ios[i];
		1149	}
		1150	}
		1151
		1152	static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
		1153	{
		1154	struct blkcg *blkcg = css_to_blkcg(css);
		1155	struct blkcg_gq *blkg;
		1156
		1157	rcu_read_lock();
		1158
		1159	hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
		1160	struct blkcg_gq *parent = blkg->parent;
		1161	struct blkg_iostat_set *bisc = per_cpu_ptr(blkg->iostat_cpu, cpu);
		1162	struct blkg_iostat cur, delta;
		1163	unsigned seq;
		1164
		1165	/* fetch the current per-cpu values */
		1166	do {
		1167	seq = u64_stats_fetch_begin(&bisc->sync);
		1168	blkg_iostat_set(&cur, &bisc->cur);
		1169	} while (u64_stats_fetch_retry(&bisc->sync, seq));
		1170
		1171	/* propagate percpu delta to global */
		1172	u64_stats_update_begin(&blkg->iostat.sync);
		1173	blkg_iostat_set(&delta, &cur);
		1174	blkg_iostat_sub(&delta, &bisc->last);
		1175	blkg_iostat_add(&blkg->iostat.cur, &delta);
		1176	blkg_iostat_add(&bisc->last, &delta);
		1177	u64_stats_update_end(&blkg->iostat.sync);
		1178
		1179	/* propagate global delta to parent */
		1180	if (parent) {
		1181	u64_stats_update_begin(&parent->iostat.sync);
		1182	blkg_iostat_set(&delta, &blkg->iostat.cur);
		1183	blkg_iostat_sub(&delta, &blkg->iostat.last);
		1184	blkg_iostat_add(&parent->iostat.cur, &delta);
		1185	blkg_iostat_add(&blkg->iostat.last, &delta);
		1186	u64_stats_update_end(&parent->iostat.sync);
		1187	}
		1188	}
		1189
		1190	rcu_read_unlock();
		1191	}
		1192
1300	static void blkcg_bind(struct cgroup_subsys_state *root_css)	1193	static void blkcg_bind(struct cgroup_subsys_state *root_css)
1301	{	1194	{
1302	int i;	1195	int i;
@@ -1329,6 +1222,7 @@ struct cgroup_subsys io_cgrp_subsys = { · 26 unmodified lines
1329	.css_offline = blkcg_css_offline,	1222	.css_offline = blkcg_css_offline,
1330	.css_free = blkcg_css_free,	1223	.css_free = blkcg_css_free,
1331	.can_attach = blkcg_can_attach,	1224	.can_attach = blkcg_can_attach,
		1225	.css_rstat_flush = blkcg_rstat_flush,
1332	.bind = blkcg_bind,	1226	.bind = blkcg_bind,
1333	.dfl_cftypes = blkcg_files,	1227	.dfl_cftypes = blkcg_files,
1334	.legacy_cftypes = blkcg_legacy_files,	1228	.legacy_cftypes = blkcg_legacy_files,

block/blk-core.c+11−5

 	REQ_OP_NAME(SECURE_ERASE),
 	REQ_OP_NAME(ZONE_RESET),
 	REQ_OP_NAME(ZONE_RESET_ALL),
+	REQ_OP_NAME(ZONE_OPEN),
+	REQ_OP_NAME(ZONE_CLOSE),
+	REQ_OP_NAME(ZONE_FINISH),
 	REQ_OP_NAME(WRITE_SAME),
 	REQ_OP_NAME(WRITE_ZEROES),
 	REQ_OP_NAME(SCSI_IN),
  */
 void blk_cleanup_queue(struct request_queue *q)
 {
+	WARN_ON_ONCE(blk_queue_registered(q));
 	/* mark @q DYING, no new request or merges will be allowed afterwards */
-	mutex_lock(&q->sysfs_lock);
 	blk_set_queue_dying(q);
 	blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q);
 	blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
 	blk_queue_flag_set(QUEUE_FLAG_DYING, q);
-	mutex_unlock(&q->sysfs_lock);
 	/*
 	 * Drain all requests queued before DYING marking. Set DEAD flag to
 		goto out;
 	/*
-	 * Zone reset does not include bi_size so bio_sectors() is always 0.
+	 * Zone management bios do not have a sector count but they do have
-	 * Include a test for the reset op code and perform the remap if needed.
+	 * a start sector filled out and need to be remapped.
 	 */
-	if (bio_sectors(bio) || bio_op(bio) == REQ_OP_ZONE_RESET) {
+	if (bio_sectors(bio) || op_is_zone_mgmt(bio_op(bio))) {
 		if (bio_check_eod(bio, part_nr_sects_read(p)))
 			goto out;
 		bio->bi_iter.bi_sector += p->start_sect;
 			goto not_supported;
 		break;
 	case REQ_OP_ZONE_RESET:
+	case REQ_OP_ZONE_OPEN:
+	case REQ_OP_ZONE_CLOSE:
+	case REQ_OP_ZONE_FINISH:
 		if (!blk_queue_is_zoned(q))
 			goto not_supported;
 		break;

block/blk-exec.c+2−0

@@ -55,6 +55,8 @@ void blk_execute_rq_nowait(struct request_queue q, struct gendisk bd_disk, · 54 unmodified lines
55	rq->rq_disk = bd_disk;	55	rq->rq_disk = bd_disk;
56	rq->end_io = done;	56	rq->end_io = done;
57		57
		58	blk_account_io_start(rq, true);
		59
58	/*	60	/*
59	* don't check dying flag for MQ because the request won't	61	* don't check dying flag for MQ because the request won't
60	* be reused after dying flag is set	62	* be reused after dying flag is set

block/blk-flush.c+14−1

 	blk_mq_add_to_requeue_list(rq, add_front, true);
 }
+static void blk_account_io_flush(struct request *rq)
+{
+	struct hd_struct *part = &rq->rq_disk->part0;
+	part_stat_lock();
+	part_stat_inc(part, ios[STAT_FLUSH]);
+	part_stat_add(part, nsecs[STAT_FLUSH],
+		      ktime_get_ns() - rq->start_time_ns);
+	part_stat_unlock();
+}
 /**
  * blk_flush_complete_seq - complete flush sequence
  * @rq: PREFLUSH/FUA request being sequenced
 	case REQ_FSEQ_DONE:
 		/*
-		 * @rq was previously adjusted by blk_flush_issue() for
+		 * @rq was previously adjusted by blk_insert_flush() for
 		 * flush sequencing and may already have gone through the
 		 * flush data request completion path.  Restore @rq for
 		 * normal completion and end it.
 	struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx);
 	struct blk_mq_hw_ctx *hctx;
+	blk_account_io_flush(flush_rq);
 	/* release the tag's ownership to the req cloned from */
 	spin_lock_irqsave(&fq->mq_flush_lock, flags);

block/blk-merge.c+16−1

@@ -293,7 +293,7 @@ split: · 292 unmodified lines
293	void __blk_queue_split(struct request_queue q, struct bio *bio,	293	void __blk_queue_split(struct request_queue q, struct bio *bio,
294	unsigned int *nr_segs)	294	unsigned int *nr_segs)
295	{	295	{
296	struct bio *split;	296	struct bio *split = NULL;
297		297
298	switch (bio_op(*bio)) {	298	switch (bio_op(*bio)) {
299	case REQ_OP_DISCARD:	299	case REQ_OP_DISCARD:
@@ -309,6 +309,21 @@ void __blk_queue_split(struct request_queue q, struct bio *bio, · 9 unmodified lines
309	nr_segs);	309	nr_segs);
310	break;	310	break;
311	default:	311	default:
		312	/*
		313	* All drivers must accept single-segments bios that are <=
		314	* PAGE_SIZE. This is a quick and dirty check that relies on
		315	* the fact that bi_io_vec[0] is always valid if a bio has data.
		316	* The check might lead to occasional false negatives when bios
		317	* are cloned, but compared to the performance impact of cloned
		318	* bios themselves the loop below doesn't matter anyway.
		319	*/
		320	if (!q->limits.chunk_sectors &&
		321	(*bio)->bi_vcnt == 1 &&
		322	((*bio)->bi_io_vec[0].bv_len +
		323	(*bio)->bi_io_vec[0].bv_offset) <= PAGE_SIZE) {
		324	*nr_segs = 1;
		325	break;
		326	}
312	split = blk_bio_segment_split(q, *bio, &q->bio_split, nr_segs);	327	split = blk_bio_segment_split(q, *bio, &q->bio_split, nr_segs);
313	break;	328	break;
314	}	329	}

block/blk-mq-sysfs.c+14−17

 	if (!entry->show)
 		return -EIO;
-	res = -ENOENT;
 	mutex_lock(&q->sysfs_lock);
-	if (!blk_queue_dying(q))
+	res = entry->show(ctx, page);
-		res = entry->show(ctx, page);
 	mutex_unlock(&q->sysfs_lock);
 	return res;
 }
 	if (!entry->store)
 		return -EIO;
-	res = -ENOENT;
 	mutex_lock(&q->sysfs_lock);
-	if (!blk_queue_dying(q))
+	res = entry->store(ctx, page, length);
-		res = entry->store(ctx, page, length);
 	mutex_unlock(&q->sysfs_lock);
 	return res;
 }
 	if (!entry->show)
 		return -EIO;
-	res = -ENOENT;
 	mutex_lock(&q->sysfs_lock);
-	if (!blk_queue_dying(q))
+	res = entry->show(hctx, page);
-		res = entry->show(hctx, page);
 	mutex_unlock(&q->sysfs_lock);
 	return res;
 }
 	if (!entry->store)
 		return -EIO;
-	res = -ENOENT;
 	mutex_lock(&q->sysfs_lock);
-	if (!blk_queue_dying(q))
+	res = entry->store(hctx, page, length);
-		res = entry->store(hctx, page, length);
 	mutex_unlock(&q->sysfs_lock);
 	return res;
 }
 static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page)
 {
+	const size_t size = PAGE_SIZE - 1;
 	unsigned int i, first = 1;
-	ssize_t ret = 0;
+	int ret = 0, pos = 0;
 	for_each_cpu(i, hctx->cpumask) {
 		if (first)
-			ret += sprintf(ret + page, "%u", i);
+			ret = snprintf(pos + page, size - pos, "%u", i);
 		else
-			ret += sprintf(ret + page, ", %u", i);
+			ret = snprintf(pos + page, size - pos, ", %u", i);
+		if (ret >= size - pos)
+			break;
 		first = 0;
+		pos += ret;
 	}
-	ret += sprintf(ret + page, "\n");
+	ret = snprintf(pos + page, size + 1 - pos, "\n");
-	return ret;
+	return pos + ret;
 }
 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {

block/blk-mq-tag.c+0−8

@@ -15,14 +15,6 @@ · 14 unmodified lines
15	#include "blk-mq.h"	15	#include "blk-mq.h"
16	#include "blk-mq-tag.h"	16	#include "blk-mq-tag.h"
17		17
18	bool blk_mq_has_free_tags(struct blk_mq_tags *tags)
19	{
20	if (!tags)
21	return true;
22
23	return sbitmap_any_bit_clear(&tags->bitmap_tags.sb);
24	}
25
26	/*	18	/*
27	* If a previously inactive queue goes active, bump the active user count.	19	* If a previously inactive queue goes active, bump the active user count.
28	* We need to do this before try to allocate driver tag, then even if fail	20	* We need to do this before try to allocate driver tag, then even if fail

block/blk-mq-tag.h+0−1

@@ -28,7 +28,6 @@ extern void blk_mq_free_tags(struct blk_mq_tags *tags); · 27 unmodified lines
28	extern unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data);	28	extern unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data);
29	extern void blk_mq_put_tag(struct blk_mq_hw_ctx hctx, struct blk_mq_tags tags,	29	extern void blk_mq_put_tag(struct blk_mq_hw_ctx hctx, struct blk_mq_tags tags,
30	struct blk_mq_ctx *ctx, unsigned int tag);	30	struct blk_mq_ctx *ctx, unsigned int tag);
31	extern bool blk_mq_has_free_tags(struct blk_mq_tags *tags);
32	extern int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx,	31	extern int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx,
33	struct blk_mq_tags **tags,	32	struct blk_mq_tags **tags,
34	unsigned int depth, bool can_grow);	33	unsigned int depth, bool can_grow);

block/blk-mq.c+58−78

 struct mq_inflight {
 	struct hd_struct *part;
-	unsigned int *inflight;
+	unsigned int inflight[2];
 };
 static bool blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx,
 {
 	struct mq_inflight *mi = priv;
-	/*
-	 * index[0] counts the specific partition that was asked for.
-	 */
 	if (rq->part == mi->part)
-		mi->inflight[0]++;
+		mi->inflight[rq_data_dir(rq)]++;
 	return true;
 }
 unsigned int blk_mq_in_flight(struct request_queue *q, struct hd_struct *part)
 {
-	unsigned inflight[2];
+	struct mq_inflight mi = { .part = part };
-	struct mq_inflight mi = { .part = part, .inflight = inflight, };
-	inflight[0] = inflight[1] = 0;
 	blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
-	return inflight[0];
+	return mi.inflight[0] + mi.inflight[1];
-}
-static bool blk_mq_check_inflight_rw(struct blk_mq_hw_ctx *hctx,
-				     struct request *rq, void *priv,
-				     bool reserved)
-{
-	struct mq_inflight *mi = priv;
-	if (rq->part == mi->part)
-		mi->inflight[rq_data_dir(rq)]++;
-	return true;
 }
 void blk_mq_in_flight_rw(struct request_queue *q, struct hd_struct *part,
 			 unsigned int inflight[2])
 {
-	struct mq_inflight mi = { .part = part, .inflight = inflight, };
+	struct mq_inflight mi = { .part = part };
-	inflight[0] = inflight[1] = 0;
+	blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
-	blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight_rw, &mi);
+	inflight[0] = mi.inflight[0];
+	inflight[1] = mi.inflight[1];
 }
 void blk_freeze_queue_start(struct request_queue *q)
 			blk_mq_tag_wakeup_all(hctx->tags, true);
 }
-bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
-{
-	return blk_mq_has_free_tags(hctx->tags);
-}
-EXPORT_SYMBOL(blk_mq_can_queue);
 /*
  * Only need start/end time stamping if we have iostat or
  * blk stats enabled, or using an IO scheduler.
 }
 EXPORT_SYMBOL(blk_mq_complete_request);
-int blk_mq_request_started(struct request *rq)
-{
-	return blk_mq_rq_state(rq) != MQ_RQ_IDLE;
-}
-EXPORT_SYMBOL_GPL(blk_mq_request_started);
-int blk_mq_request_completed(struct request *rq)
-{
-	return blk_mq_rq_state(rq) == MQ_RQ_COMPLETE;
-}
-EXPORT_SYMBOL_GPL(blk_mq_request_completed);
 void blk_mq_start_request(struct request *rq)
 {
 	struct request_queue *q = rq->q;
 	bool shared;
 	if (rq->tag != -1)
-		goto done;
+		return true;
 	if (blk_mq_tag_is_reserved(data.hctx->sched_tags, rq->internal_tag))
 		data.flags |= BLK_MQ_REQ_RESERVED;
 		data.hctx->tags->rqs[rq->tag] = rq;
 	}
-done:
 	return rq->tag != -1;
 }
 }
 EXPORT_SYMBOL(blk_mq_delay_run_hw_queue);
-bool blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
+void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
 {
 	int srcu_idx;
 	bool need_run;
 		blk_mq_hctx_has_pending(hctx);
 	hctx_unlock(hctx, srcu_idx);
-	if (need_run) {
+	if (need_run)
 		__blk_mq_delay_run_hw_queue(hctx, async, 0);
-		return true;
-	}
-	return false;
 }
 EXPORT_SYMBOL(blk_mq_run_hw_queue);
 	int i, j, end;
 	struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
+	if (q->nr_hw_queues < set->nr_hw_queues) {
+		struct blk_mq_hw_ctx **new_hctxs;
+		new_hctxs = kcalloc_node(set->nr_hw_queues,
+				       sizeof(*new_hctxs), GFP_KERNEL,
+				       set->numa_node);
+		if (!new_hctxs)
+			return;
+		if (hctxs)
+			memcpy(new_hctxs, hctxs, q->nr_hw_queues *
+			       sizeof(*hctxs));
+		q->queue_hw_ctx = new_hctxs;
+		q->nr_hw_queues = set->nr_hw_queues;
+		kfree(hctxs);
+		hctxs = new_hctxs;
+	}
 	/* protect against switching io scheduler  */
 	mutex_lock(&q->sysfs_lock);
 	for (i = 0; i < set->nr_hw_queues; i++) {
 	mutex_unlock(&q->sysfs_lock);
 }
-/*
- * Maximum number of hardware queues we support. For single sets, we'll never
- * have more than the CPUs (software queues). For multiple sets, the tag_set
- * user may have set ->nr_hw_queues larger.
- */
-static unsigned int nr_hw_queues(struct blk_mq_tag_set *set)
-{
-	if (set->nr_maps == 1)
-		return nr_cpu_ids;
-	return max(set->nr_hw_queues, nr_cpu_ids);
-}
 struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
 						  struct request_queue *q,
 						  bool elevator_init)
 	/* init q->mq_kobj and sw queues' kobjects */
 	blk_mq_sysfs_init(q);
-	q->nr_queues = nr_hw_queues(set);
-	q->queue_hw_ctx = kcalloc_node(q->nr_queues, sizeof(*(q->queue_hw_ctx)),
-						GFP_KERNEL, set->numa_node);
-	if (!q->queue_hw_ctx)
-		goto err_sys_init;
 	INIT_LIST_HEAD(&q->unused_hctx_list);
 	spin_lock_init(&q->unused_hctx_lock);
 err_hctxs:
 	kfree(q->queue_hw_ctx);
 	q->nr_hw_queues = 0;
-err_sys_init:
 	blk_mq_sysfs_deinit(q);
 err_poll:
 	blk_stat_free_callback(q->poll_cb);
 	}
 }
+static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set,
+				  int cur_nr_hw_queues, int new_nr_hw_queues)
+{
+	struct blk_mq_tags **new_tags;
+	if (cur_nr_hw_queues >= new_nr_hw_queues)
+		return 0;
+	new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *),
+				GFP_KERNEL, set->numa_node);
+	if (!new_tags)
+		return -ENOMEM;
+	if (set->tags)
+		memcpy(new_tags, set->tags, cur_nr_hw_queues *
+		       sizeof(*set->tags));
+	kfree(set->tags);
+	set->tags = new_tags;
+	set->nr_hw_queues = new_nr_hw_queues;
+	return 0;
+}
 /*
  * Alloc a tag set to be associated with one or more request queues.
  * May fail with EINVAL for various error conditions. May adjust the
 	if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids)
 		set->nr_hw_queues = nr_cpu_ids;
-	set->tags = kcalloc_node(nr_hw_queues(set), sizeof(struct blk_mq_tags *),
+	if (blk_mq_realloc_tag_set_tags(set, 0, set->nr_hw_queues) < 0)
-				 GFP_KERNEL, set->numa_node);
-	if (!set->tags)
 		return -ENOMEM;
 	ret = -ENOMEM;
 {
 	int i, j;
-	for (i = 0; i < nr_hw_queues(set); i++)
+	for (i = 0; i < set->nr_hw_queues; i++)
 		blk_mq_free_map_and_requests(set, i);
 	for (j = 0; j < set->nr_maps; j++) {
 	list_for_each_entry(q, &set->tag_list, tag_set_list)
 		blk_mq_freeze_queue(q);
-	/*
-	 * Sync with blk_mq_queue_tag_busy_iter.
-	 */
-	synchronize_rcu();
 	/*
 	 * Switch IO scheduler to 'none', cleaning up the data associated
 	 * with the previous scheduler. We will switch back once we are done
 		blk_mq_sysfs_unregister(q);
 	}
+	if (blk_mq_realloc_tag_set_tags(set, set->nr_hw_queues, nr_hw_queues) <
+	    0)
+		goto reregister;
 	prev_nr_hw_queues = set->nr_hw_queues;
 	set->nr_hw_queues = nr_hw_queues;
 	blk_mq_update_queue_map(set);
 		blk_mq_map_swqueue(q);
 	}
+reregister:
 	list_for_each_entry(q, &set->tag_list, tag_set_list) {
 		blk_mq_sysfs_register(q);
 		blk_mq_debugfs_register_hctxs(q);

block/blk-mq.h+0−9

@@ -128,15 +128,6 @@ extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx); · 127 unmodified lines
128		128
129	void blk_mq_release(struct request_queue *q);	129	void blk_mq_release(struct request_queue *q);
130		130
131	/**
132	* blk_mq_rq_state() - read the current MQ_RQ_* state of a request
133	* @rq: target request.
134	*/
135	static inline enum mq_rq_state blk_mq_rq_state(struct request *rq)
136	{
137	return READ_ONCE(rq->state);
138	}
139
140	static inline struct blk_mq_ctx __blk_mq_get_ctx(struct request_queue q,	131	static inline struct blk_mq_ctx __blk_mq_get_ctx(struct request_queue q,
141	unsigned int cpu)	132	unsigned int cpu)
142	{	133	{

block/blk-softirq.c+0−4

 static void trigger_softirq(void *data)
 {
 	struct request *rq = data;
-	unsigned long flags;
 	struct list_head *list;
-	local_irq_save(flags);
 	list = this_cpu_ptr(&blk_cpu_done);
 	list_add_tail(&rq->ipi_list, list);
 	if (list->next == &rq->ipi_list)
 		raise_softirq_irqoff(BLOCK_SOFTIRQ);
-	local_irq_restore(flags);
 }
 /*

block/blk-stat.c+4−3

 	struct request_queue *q = rq->q;
 	struct blk_stat_callback *cb;
 	struct blk_rq_stat *stat;
-	int bucket;
+	int bucket, cpu;
 	u64 value;
 	value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0;
 	blk_throtl_stat_add(rq, value);
 	rcu_read_lock();
+	cpu = get_cpu();
 	list_for_each_entry_rcu(cb, &q->stats->callbacks, list) {
 		if (!blk_stat_is_active(cb))
 			continue;
 		if (bucket < 0)
 			continue;
-		stat = &get_cpu_ptr(cb->cpu_stat)[bucket];
+		stat = &per_cpu_ptr(cb->cpu_stat, cpu)[bucket];
 		blk_rq_stat_add(stat, value);
-		put_cpu_ptr(cb->cpu_stat);
 	}
+	put_cpu();
 	rcu_read_unlock();
 }

block/blk-sysfs.c+0−8

 	if (!entry->show)
 		return -EIO;
 	mutex_lock(&q->sysfs_lock);
-	if (blk_queue_dying(q)) {
-		mutex_unlock(&q->sysfs_lock);
-		return -ENOENT;
-	}
 	res = entry->show(q, page);
 	mutex_unlock(&q->sysfs_lock);
 	return res;
 	q = container_of(kobj, struct request_queue, kobj);
 	mutex_lock(&q->sysfs_lock);
-	if (blk_queue_dying(q)) {
-		mutex_unlock(&q->sysfs_lock);
-		return -ENOENT;
-	}
 	res = entry->store(q, page, length);
 	mutex_unlock(&q->sysfs_lock);
 	return res;

block/blk-throttle.c+62−9

 #include <linux/blktrace_api.h>
 #include <linux/blk-cgroup.h>
 #include "blk.h"
+#include "blk-cgroup-rwstat.h"
 /* Max dispatch from a group in 1 round */
 static int throtl_grp_quantum = 8;
 	unsigned int bio_cnt; /* total bios */
 	unsigned int bad_bio_cnt; /* bios exceeding latency threshold */
 	unsigned long bio_cnt_reset_time;
+	struct blkg_rwstat stat_bytes;
+	struct blkg_rwstat stat_ios;
 };
 /* We measure latency for request size from <= 4k to >= 1M */
 	if (!tg)
 		return NULL;
+	if (blkg_rwstat_init(&tg->stat_bytes, gfp))
+		goto err_free_tg;
+	if (blkg_rwstat_init(&tg->stat_ios, gfp))
+		goto err_exit_stat_bytes;
 	throtl_service_queue_init(&tg->service_queue);
 	for (rw = READ; rw <= WRITE; rw++) {
 	tg->idletime_threshold_conf = DFL_IDLE_THRESHOLD;
 	return &tg->pd;
+err_exit_stat_bytes:
+	blkg_rwstat_exit(&tg->stat_bytes);
+err_free_tg:
+	kfree(tg);
+	return NULL;
 }
 static void throtl_pd_init(struct blkg_policy_data *pd)
 	struct throtl_grp *tg = pd_to_tg(pd);
 	del_timer_sync(&tg->service_queue.pending_timer);
+	blkg_rwstat_exit(&tg->stat_bytes);
+	blkg_rwstat_exit(&tg->stat_ios);
 	kfree(tg);
 }
 	return tg_set_conf(of, buf, nbytes, off, false);
 }
+static int tg_print_rwstat(struct seq_file *sf, void *v)
+{
+	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
+			  blkg_prfill_rwstat, &blkcg_policy_throtl,
+			  seq_cft(sf)->private, true);
+	return 0;
+}
+static u64 tg_prfill_rwstat_recursive(struct seq_file *sf,
+				      struct blkg_policy_data *pd, int off)
+{
+	struct blkg_rwstat_sample sum;
+	blkg_rwstat_recursive_sum(pd_to_blkg(pd), &blkcg_policy_throtl, off,
+				  &sum);
+	return __blkg_prfill_rwstat(sf, pd, &sum);
+}
+static int tg_print_rwstat_recursive(struct seq_file *sf, void *v)
+{
+	blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
+			  tg_prfill_rwstat_recursive, &blkcg_policy_throtl,
+			  seq_cft(sf)->private, true);
+	return 0;
+}
 static struct cftype throtl_legacy_files[] = {
 	{
 		.name = "throttle.read_bps_device",
 	},
 	{
 		.name = "throttle.io_service_bytes",
-		.private = (unsigned long)&blkcg_policy_throtl,
+		.private = offsetof(struct throtl_grp, stat_bytes),
-		.seq_show = blkg_print_stat_bytes,
+		.seq_show = tg_print_rwstat,
 	},
 	{
 		.name = "throttle.io_service_bytes_recursive",
-		.private = (unsigned long)&blkcg_policy_throtl,
+		.private = offsetof(struct throtl_grp, stat_bytes),
-		.seq_show = blkg_print_stat_bytes_recursive,
+		.seq_show = tg_print_rwstat_recursive,
 	},
 	{
 		.name = "throttle.io_serviced",
-		.private = (unsigned long)&blkcg_policy_throtl,
+		.private = offsetof(struct throtl_grp, stat_ios),
-		.seq_show = blkg_print_stat_ios,
+		.seq_show = tg_print_rwstat,
 	},
 	{
 		.name = "throttle.io_serviced_recursive",
-		.private = (unsigned long)&blkcg_policy_throtl,
+		.private = offsetof(struct throtl_grp, stat_ios),
-		.seq_show = blkg_print_stat_ios_recursive,
+		.seq_show = tg_print_rwstat_recursive,
 	},
 	{ }	/* terminate */
 };
 	WARN_ON_ONCE(!rcu_read_lock_held());
 	/* see throtl_charge_bio() */
-	if (bio_flagged(bio, BIO_THROTTLED) || !tg->has_rules[rw])
+	if (bio_flagged(bio, BIO_THROTTLED))
+		goto out;
+	if (!cgroup_subsys_on_dfl(io_cgrp_subsys)) {
+		blkg_rwstat_add(&tg->stat_bytes, bio->bi_opf,
+				bio->bi_iter.bi_size);
+		blkg_rwstat_add(&tg->stat_ios, bio->bi_opf, 1);
+	}
+	if (!tg->has_rules[rw])
 		goto out;
 	spin_lock_irq(&q->queue_lock);

block/blk-zoned.c+53−46

 }
 EXPORT_SYMBOL_GPL(blkdev_report_zones);
-/*
- * Special case of zone reset operation to reset all zones in one command,
- * useful for applications like mkfs.
- */
-static int __blkdev_reset_all_zones(struct block_device *bdev, gfp_t gfp_mask)
-{
-	struct bio *bio = bio_alloc(gfp_mask, 0);
-	int ret;
-	/* across the zones operations, don't need any sectors */
-	bio_set_dev(bio, bdev);
-	bio_set_op_attrs(bio, REQ_OP_ZONE_RESET_ALL, 0);
-	ret = submit_bio_wait(bio);
-	bio_put(bio);
-	return ret;
-}
 static inline bool blkdev_allow_reset_all_zones(struct block_device *bdev,
+						sector_t sector,
 						sector_t nr_sectors)
 {
 	if (!blk_queue_zone_resetall(bdev_get_queue(bdev)))
 		return false;
-	if (nr_sectors != part_nr_sects_read(bdev->bd_part))
+	if (sector || nr_sectors != part_nr_sects_read(bdev->bd_part))
 		return false;
 	/*
 	 * REQ_OP_ZONE_RESET_ALL can be executed only if the block device is
 }
 /**
- * blkdev_reset_zones - Reset zones write pointer
+ * blkdev_zone_mgmt - Execute a zone management operation on a range of zones
  * @bdev:	Target block device
- * @sector:	Start sector of the first zone to reset
+ * @op:		Operation to be performed on the zones
- * @nr_sectors:	Number of sectors, at least the length of one zone
+ * @sector:	Start sector of the first zone to operate on
+ * @nr_sectors:	Number of sectors, should be at least the length of one zone and
+ *		must be zone size aligned.
  * @gfp_mask:	Memory allocation flags (for bio_alloc)
  *
  * Description:
- *    Reset the write pointer of the zones contained in the range
+ *    Perform the specified operation on the range of zones specified by
  *    @sector..@sector+@nr_sectors. Specifying the entire disk sector range
  *    is valid, but the specified range should not contain conventional zones.
+ *    The operation to execute on each zone can be a zone reset, open, close
+ *    or finish request.
  */
-int blkdev_reset_zones(struct block_device *bdev,
+int blkdev_zone_mgmt(struct block_device *bdev, enum req_opf op,
-		       sector_t sector, sector_t nr_sectors,
+		     sector_t sector, sector_t nr_sectors,
-		       gfp_t gfp_mask)
+		     gfp_t gfp_mask)
 {
 	struct request_queue *q = bdev_get_queue(bdev);
-	sector_t zone_sectors;
+	sector_t zone_sectors = blk_queue_zone_sectors(q);
 	sector_t end_sector = sector + nr_sectors;
 	struct bio *bio = NULL;
-	struct blk_plug plug;
 	int ret;
 	if (!blk_queue_is_zoned(q))
 	if (bdev_read_only(bdev))
 		return -EPERM;
+	if (!op_is_zone_mgmt(op))
+		return -EOPNOTSUPP;
 	if (!nr_sectors || end_sector > bdev->bd_part->nr_sects)
 		/* Out of range */
 		return -EINVAL;
-	if (blkdev_allow_reset_all_zones(bdev, nr_sectors))
-		return  __blkdev_reset_all_zones(bdev, gfp_mask);
 	/* Check alignment (handle eventual smaller last zone) */
-	zone_sectors = blk_queue_zone_sectors(q);
 	if (sector & (zone_sectors - 1))
 		return -EINVAL;
 	    end_sector != bdev->bd_part->nr_sects)
 		return -EINVAL;
-	blk_start_plug(&plug);
 	while (sector < end_sector) {
 		bio = blk_next_bio(bio, 0, gfp_mask);
-		bio->bi_iter.bi_sector = sector;
 		bio_set_dev(bio, bdev);
-		bio_set_op_attrs(bio, REQ_OP_ZONE_RESET, 0);
+		/*
+		 * Special case for the zone reset operation that reset all
+		 * zones, this is useful for applications like mkfs.
+		 */
+		if (op == REQ_OP_ZONE_RESET &&
+		    blkdev_allow_reset_all_zones(bdev, sector, nr_sectors)) {
+			bio->bi_opf = REQ_OP_ZONE_RESET_ALL;
+			break;
+		}
+		bio->bi_opf = op;
+		bio->bi_iter.bi_sector = sector;
 		sector += zone_sectors;
 		/* This may take a while, so be nice to others */
 		cond_resched();
 	}
 	ret = submit_bio_wait(bio);
 	bio_put(bio);
-	blk_finish_plug(&plug);
 	return ret;
 }
-EXPORT_SYMBOL_GPL(blkdev_reset_zones);
+EXPORT_SYMBOL_GPL(blkdev_zone_mgmt);
 /*
  * BLKREPORTZONE ioctl processing.
 }
 /*
- * BLKRESETZONE ioctl processing.
+ * BLKRESETZONE, BLKOPENZONE, BLKCLOSEZONE and BLKFINISHZONE ioctl processing.
  * Called from blkdev_ioctl.
  */
-int blkdev_reset_zones_ioctl(struct block_device *bdev, fmode_t mode,
+int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode,
-			     unsigned int cmd, unsigned long arg)
+			   unsigned int cmd, unsigned long arg)
 {
 	void __user *argp = (void __user *)arg;
 	struct request_queue *q;
 	struct blk_zone_range zrange;
+	enum req_opf op;
 	if (!argp)
 		return -EINVAL;
 	if (copy_from_user(&zrange, argp, sizeof(struct blk_zone_range)))
 		return -EFAULT;
-	return blkdev_reset_zones(bdev, zrange.sector, zrange.nr_sectors,
+	switch (cmd) {
-				  GFP_KERNEL);
+	case BLKRESETZONE:
+		op = REQ_OP_ZONE_RESET;
+		break;
+	case BLKOPENZONE:
+		op = REQ_OP_ZONE_OPEN;
+		break;
+	case BLKCLOSEZONE:
+		op = REQ_OP_ZONE_CLOSE;
+		break;
+	case BLKFINISHZONE:
+		op = REQ_OP_ZONE_FINISH;
+		break;
+	default:
+		return -ENOTTY;
+	}
+	return blkdev_zone_mgmt(bdev, op, zrange.sector, zrange.nr_sectors,
+				GFP_KERNEL);
 }
 static inline unsigned long *blk_alloc_zone_bitmap(int node,

block/blk.h+2−5

  * Contribute to IO statistics IFF:
  *
  *	a) it's attached to a gendisk, and
- *	b) the queue had IO stats enabled when this request was started, and
+ *	b) the queue had IO stats enabled when this request was started
- *	c) it's a file system request
  */
 static inline bool blk_do_io_stat(struct request *rq)
 {
-	return rq->rq_disk &&
+	return rq->rq_disk && (rq->rq_flags & RQF_IO_STAT);
-	       (rq->rq_flags & RQF_IO_STAT) &&
-		!blk_rq_is_passthrough(rq);
 }
 static inline void req_set_nomerge(struct request_queue *q, struct request *req)

block/elevator.c+9−0

@@ -832,3 +832,12 @@ struct request elv_rb_latter_request(struct request_queue q, · 831 unmodified lines
832	return NULL;	832	return NULL;
833	}	833	}
834	EXPORT_SYMBOL(elv_rb_latter_request);	834	EXPORT_SYMBOL(elv_rb_latter_request);
		835
		836	static int __init elevator_setup(char *str)
		837	{
		838	pr_warn("Kernel parameter elevator= does not have any effect anymore.\n"
		839	"Please use sysfs to set IO scheduler for individual devices.\n");
		840	return 1;
		841	}
		842
		843	__setup("elevator=", elevator_setup);

block/genhd.c+6−2

 			   "%lu %lu %lu %u "
 			   "%lu %lu %lu %u "
 			   "%u %u %u "
-			   "%lu %lu %lu %u\n",
+			   "%lu %lu %lu %u "
+			   "%lu %u"
+			   "\n",
 			   MAJOR(part_devt(hd)), MINOR(part_devt(hd)),
 			   disk_name(gp, hd->partno, buf),
 			   part_stat_read(hd, ios[STAT_READ]),
 			   part_stat_read(hd, ios[STAT_DISCARD]),
 			   part_stat_read(hd, merges[STAT_DISCARD]),
 			   part_stat_read(hd, sectors[STAT_DISCARD]),
-			   (unsigned int)part_stat_read_msecs(hd, STAT_DISCARD)
+			   (unsigned int)part_stat_read_msecs(hd, STAT_DISCARD),
+			   part_stat_read(hd, ios[STAT_FLUSH]),
+			   (unsigned int)part_stat_read_msecs(hd, STAT_FLUSH)
 			);
 	}
 	disk_part_iter_exit(&piter);

block/ioctl.c+4−1

 	case BLKREPORTZONE:
 		return blkdev_report_zones_ioctl(bdev, mode, cmd, arg);
 	case BLKRESETZONE:
-		return blkdev_reset_zones_ioctl(bdev, mode, cmd, arg);
+	case BLKOPENZONE:
+	case BLKCLOSEZONE:
+	case BLKFINISHZONE:
+		return blkdev_zone_mgmt_ioctl(bdev, mode, cmd, arg);
 	case BLKGETZONESZ:
 		return put_uint(arg, bdev_zone_sectors(bdev));
 	case BLKGETNRZONES:

block/opal_proto.h+5−1

  * Derived from: TCG_Storage_Architecture_Core_Spec_v2.01_r1.00
  * Section: 6.3 Assigned UIDs
  */
-#define OPAL_UID_LENGTH 8
 #define OPAL_METHOD_LENGTH 8
 #define OPAL_MSID_KEYLEN 15
 #define OPAL_UID_LENGTH_HALF 4
 	OPAL_C_PIN_TABLE,
 	OPAL_LOCKING_INFO_TABLE,
 	OPAL_ENTERPRISE_LOCKING_INFO_TABLE,
+	OPAL_DATASTORE,
 	/* C_PIN_TABLE object ID's */
 	OPAL_C_PIN_MSID,
 	OPAL_C_PIN_SID,
 	OPAL_LOCKING_LOCKED = 0x03,
 };
+enum opal_parameter {
+	OPAL_SUM_SET_LIST = 0x060000,
+};
 /* Packets derived from:
  * TCG_Storage_Architecture_Core_Spec_v2.01_r1.00
  * Secion: 3.2.3 ComPackets, Packets & Subpackets

block/partition-generic.c+5−2

 		"%8lu %8lu %8llu %8u "
 		"%8lu %8lu %8llu %8u "
 		"%8u %8u %8u "
-		"%8lu %8lu %8llu %8u"
+		"%8lu %8lu %8llu %8u "
+		"%8lu %8u"
 		"\n",
 		part_stat_read(p, ios[STAT_READ]),
 		part_stat_read(p, merges[STAT_READ]),
 		part_stat_read(p, ios[STAT_DISCARD]),
 		part_stat_read(p, merges[STAT_DISCARD]),
 		(unsigned long long)part_stat_read(p, sectors[STAT_DISCARD]),
-		(unsigned int)part_stat_read_msecs(p, STAT_DISCARD));
+		(unsigned int)part_stat_read_msecs(p, STAT_DISCARD),
+		part_stat_read(p, ios[STAT_FLUSH]),
+		(unsigned int)part_stat_read_msecs(p, STAT_FLUSH));
 }
 ssize_t part_inflight_show(struct device *dev, struct device_attribute *attr,

block/sed-opal.c+249−69

@@ -149,6 +149,8 @@ static const u8 opaluid[][OPAL_UID_LENGTH] = { · 148 unmodified lines
149	{ 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, 0x00, 0x01 },	149	{ 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, 0x00, 0x01 },
150	[OPAL_ENTERPRISE_LOCKING_INFO_TABLE] =	150	[OPAL_ENTERPRISE_LOCKING_INFO_TABLE] =
151	{ 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, 0x00, 0x00 },	151	{ 0x00, 0x00, 0x08, 0x01, 0x00, 0x00, 0x00, 0x00 },
		152	[OPAL_DATASTORE] =
		153	{ 0x00, 0x00, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00 },
152		154
153	/* C_PIN_TABLE object ID's */	155	/* C_PIN_TABLE object ID's */
154	[OPAL_C_PIN_MSID] =	156	[OPAL_C_PIN_MSID] =
@@ -1139,11 +1141,11 @@ static int generic_get_column(struct opal_dev dev, const u8 table, · 984 unmodified lines
1139	*	1141	*
1140	* the result is provided in dev->resp->tok[4]	1142	* the result is provided in dev->resp->tok[4]
1141	*/	1143	*/
1142	static int generic_get_table_info(struct opal_dev *dev, enum opal_uid table,	1144	static int generic_get_table_info(struct opal_dev dev, const u8 table_uid,
1143	u64 column)	1145	u64 column)
1144	{	1146	{
1145	u8 uid[OPAL_UID_LENGTH];	1147	u8 uid[OPAL_UID_LENGTH];
1146	const unsigned int half = OPAL_UID_LENGTH/2;	1148	const unsigned int half = OPAL_UID_LENGTH_HALF;
1147		1149
1148	/* sed-opal UIDs can be split in two halves:	1150	/* sed-opal UIDs can be split in two halves:
1149	* first: actual table index	1151	* first: actual table index
@@ -1152,7 +1154,7 @@ static int generic_get_table_info(struct opal_dev *dev, enum opal_uid table, · 2 unmodified lines
1152	* first part of the target table as relative index into that table	1154	* first part of the target table as relative index into that table
1153	*/	1155	*/
1154	memcpy(uid, opaluid[OPAL_TABLE_TABLE], half);	1156	memcpy(uid, opaluid[OPAL_TABLE_TABLE], half);
1155	memcpy(uid+half, opaluid[table], half);	1157	memcpy(uid + half, table_uid, half);
1156		1158
1157	return generic_get_column(dev, uid, column);	1159	return generic_get_column(dev, uid, column);
1158	}	1160	}
@@ -1221,6 +1223,75 @@ static int get_active_key(struct opal_dev dev, void data) · 62 unmodified lines
1221	return get_active_key_cont(dev);	1223	return get_active_key_cont(dev);
1222	}	1224	}
1223		1225
		1226	static int generic_table_write_data(struct opal_dev *dev, const u64 data,
		1227	u64 offset, u64 size, const u8 *uid)
		1228	{
		1229	const u8 __user src = (u8 __user )(uintptr_t)data;
		1230	u8 *dst;
		1231	u64 len;
		1232	size_t off = 0;
		1233	int err;
		1234
		1235	/* do we fit in the available space? */
		1236	err = generic_get_table_info(dev, uid, OPAL_TABLE_ROWS);
		1237	if (err) {
		1238	pr_debug("Couldn't get the table size\n");
		1239	return err;
		1240	}
		1241
		1242	len = response_get_u64(&dev->parsed, 4);
		1243	if (size > len \|\| offset > len - size) {
		1244	pr_debug("Does not fit in the table (%llu vs. %llu)\n",
		1245	offset + size, len);
		1246	return -ENOSPC;
		1247	}
		1248
		1249	/* do the actual transmission(s) */
		1250	while (off < size) {
		1251	err = cmd_start(dev, uid, opalmethod[OPAL_SET]);
		1252	add_token_u8(&err, dev, OPAL_STARTNAME);
		1253	add_token_u8(&err, dev, OPAL_WHERE);
		1254	add_token_u64(&err, dev, offset + off);
		1255	add_token_u8(&err, dev, OPAL_ENDNAME);
		1256
		1257	add_token_u8(&err, dev, OPAL_STARTNAME);
		1258	add_token_u8(&err, dev, OPAL_VALUES);
		1259
		1260	/*
		1261	* The bytestring header is either 1 or 2 bytes, so assume 2.
		1262	* There also needs to be enough space to accommodate the
		1263	* trailing OPAL_ENDNAME (1 byte) and tokens added by
		1264	* cmd_finalize.
		1265	*/
		1266	len = min(remaining_size(dev) - (2+1+CMD_FINALIZE_BYTES_NEEDED),
		1267	(size_t)(size - off));
		1268	pr_debug("Write bytes %zu+%llu/%llu\n", off, len, size);
		1269
		1270	dst = add_bytestring_header(&err, dev, len);
		1271	if (!dst)
		1272	break;
		1273
		1274	if (copy_from_user(dst, src + off, len)) {
		1275	err = -EFAULT;
		1276	break;
		1277	}
		1278
		1279	dev->pos += len;
		1280
		1281	add_token_u8(&err, dev, OPAL_ENDNAME);
		1282	if (err)
		1283	break;
		1284
		1285	err = finalize_and_send(dev, parse_and_check_status);
		1286	if (err)
		1287	break;
		1288
		1289	off += len;
		1290	}
		1291
		1292	return err;
		1293	}
		1294
1224	static int generic_lr_enable_disable(struct opal_dev *dev,	1295	static int generic_lr_enable_disable(struct opal_dev *dev,
1225	u8 *uid, bool rle, bool wle,	1296	u8 *uid, bool rle, bool wle,
1226	bool rl, bool wl)	1297	bool rl, bool wl)
@@ -1583,68 +1654,9 @@ static int set_mbr_enable_disable(struct opal_dev dev, void data) · 356 unmodified lines
1583	static int write_shadow_mbr(struct opal_dev dev, void data)	1654	static int write_shadow_mbr(struct opal_dev dev, void data)
1584	{	1655	{
1585	struct opal_shadow_mbr *shadow = data;	1656	struct opal_shadow_mbr *shadow = data;
1586	const u8 __user *src;
1587	u8 *dst;
1588	size_t off = 0;
1589	u64 len;
1590	int err = 0;
1591
1592	/* do we fit in the available shadow mbr space? */
1593	err = generic_get_table_info(dev, OPAL_MBR, OPAL_TABLE_ROWS);
1594	if (err) {
1595	pr_debug("MBR: could not get shadow size\n");
1596	return err;
1597	}
1598
1599	len = response_get_u64(&dev->parsed, 4);
1600	if (shadow->size > len \|\| shadow->offset > len - shadow->size) {
1601	pr_debug("MBR: does not fit in shadow (%llu vs. %llu)\n",
1602	shadow->offset + shadow->size, len);
1603	return -ENOSPC;
1604	}
1605
1606	/* do the actual transmission(s) */
1607	src = (u8 __user *)(uintptr_t)shadow->data;
1608	while (off < shadow->size) {
1609	err = cmd_start(dev, opaluid[OPAL_MBR], opalmethod[OPAL_SET]);
1610	add_token_u8(&err, dev, OPAL_STARTNAME);
1611	add_token_u8(&err, dev, OPAL_WHERE);
1612	add_token_u64(&err, dev, shadow->offset + off);
1613	add_token_u8(&err, dev, OPAL_ENDNAME);
1614
1615	add_token_u8(&err, dev, OPAL_STARTNAME);
1616	add_token_u8(&err, dev, OPAL_VALUES);
1617
1618	/*
1619	* The bytestring header is either 1 or 2 bytes, so assume 2.
1620	* There also needs to be enough space to accommodate the
1621	* trailing OPAL_ENDNAME (1 byte) and tokens added by
1622	* cmd_finalize.
1623	*/
1624	len = min(remaining_size(dev) - (2+1+CMD_FINALIZE_BYTES_NEEDED),
1625	(size_t)(shadow->size - off));
1626	pr_debug("MBR: write bytes %zu+%llu/%llu\n",
1627	off, len, shadow->size);
1628
1629	dst = add_bytestring_header(&err, dev, len);
1630	if (!dst)
1631	break;
1632	if (copy_from_user(dst, src + off, len))
1633	err = -EFAULT;
1634	dev->pos += len;
1635
1636	add_token_u8(&err, dev, OPAL_ENDNAME);
1637	if (err)
1638	break;
1639
1640	err = finalize_and_send(dev, parse_and_check_status);
1641	if (err)
1642	break;
1643
1644	off += len;
1645	}
1646		1657
1647	return err;	1658	return generic_table_write_data(dev, shadow->data, shadow->offset,
		1659	shadow->size, opaluid[OPAL_MBR]);
1648	}	1660	}
1649		1661
1650	static int generic_pw_cmd(u8 key, size_t key_len, u8 cpin_uid,	1662	static int generic_pw_cmd(u8 key, size_t key_len, u8 cpin_uid,
@@ -1874,7 +1886,6 @@ static int activate_lsp(struct opal_dev dev, void data) · 223 unmodified lines
1874	{	1886	{
1875	struct opal_lr_act *opal_act = data;	1887	struct opal_lr_act *opal_act = data;
1876	u8 user_lr[OPAL_UID_LENGTH];	1888	u8 user_lr[OPAL_UID_LENGTH];
1877	u8 uint_3 = 0x83;
1878	int err, i;	1889	int err, i;
1879		1890
1880	err = cmd_start(dev, opaluid[OPAL_LOCKINGSP_UID],	1891	err = cmd_start(dev, opaluid[OPAL_LOCKINGSP_UID],
@@ -1887,10 +1898,7 @@ static int activate_lsp(struct opal_dev dev, void data) · 6 unmodified lines
1887	return err;	1898	return err;
1888		1899
1889	add_token_u8(&err, dev, OPAL_STARTNAME);	1900	add_token_u8(&err, dev, OPAL_STARTNAME);
1890	add_token_u8(&err, dev, uint_3);	1901	add_token_u64(&err, dev, OPAL_SUM_SET_LIST);
1891	add_token_u8(&err, dev, 6);
1892	add_token_u8(&err, dev, 0);
1893	add_token_u8(&err, dev, 0);
1894		1902
1895	add_token_u8(&err, dev, OPAL_STARTLIST);	1903	add_token_u8(&err, dev, OPAL_STARTLIST);
1896	add_token_bytestring(&err, dev, user_lr, OPAL_UID_LENGTH);	1904	add_token_bytestring(&err, dev, user_lr, OPAL_UID_LENGTH);
@@ -1957,6 +1965,113 @@ static int get_msid_cpin_pin(struct opal_dev dev, void data) · 60 unmodified lines
1957	return 0;	1965	return 0;
1958	}	1966	}
1959		1967
		1968	static int write_table_data(struct opal_dev dev, void data)
		1969	{
		1970	struct opal_read_write_table *write_tbl = data;
		1971
		1972	return generic_table_write_data(dev, write_tbl->data, write_tbl->offset,
		1973	write_tbl->size, write_tbl->table_uid);
		1974	}
		1975
		1976	static int read_table_data_cont(struct opal_dev *dev)
		1977	{
		1978	int err;
		1979	const char *data_read;
		1980
		1981	err = parse_and_check_status(dev);
		1982	if (err)
		1983	return err;
		1984
		1985	dev->prev_d_len = response_get_string(&dev->parsed, 1, &data_read);
		1986	dev->prev_data = (void *)data_read;
		1987	if (!dev->prev_data) {
		1988	pr_debug("%s: Couldn't read data from the table.\n", __func__);
		1989	return OPAL_INVAL_PARAM;
		1990	}
		1991
		1992	return 0;
		1993	}
		1994
		1995	/*
		1996	* IO_BUFFER_LENGTH = 2048
		1997	* sizeof(header) = 56
		1998	* No. of Token Bytes in the Response = 11
		1999	* MAX size of data that can be carried in response buffer
		2000	* at a time is : 2048 - (56 + 11) = 1981 = 0x7BD.
		2001	*/
		2002	#define OPAL_MAX_READ_TABLE (0x7BD)
		2003
		2004	static int read_table_data(struct opal_dev dev, void data)
		2005	{
		2006	struct opal_read_write_table *read_tbl = data;
		2007	int err;
		2008	size_t off = 0, max_read_size = OPAL_MAX_READ_TABLE;
		2009	u64 table_len, len;
		2010	u64 offset = read_tbl->offset, read_size = read_tbl->size - 1;
		2011	u8 __user *dst;
		2012
		2013	err = generic_get_table_info(dev, read_tbl->table_uid, OPAL_TABLE_ROWS);
		2014	if (err) {
		2015	pr_debug("Couldn't get the table size\n");
		2016	return err;
		2017	}
		2018
		2019	table_len = response_get_u64(&dev->parsed, 4);
		2020
		2021	/* Check if the user is trying to read from the table limits */
		2022	if (read_size > table_len \|\| offset > table_len - read_size) {
		2023	pr_debug("Read size exceeds the Table size limits (%llu vs. %llu)\n",
		2024	offset + read_size, table_len);
		2025	return -EINVAL;
		2026	}
		2027
		2028	while (off < read_size) {
		2029	err = cmd_start(dev, read_tbl->table_uid, opalmethod[OPAL_GET]);
		2030
		2031	add_token_u8(&err, dev, OPAL_STARTLIST);
		2032	add_token_u8(&err, dev, OPAL_STARTNAME);
		2033	add_token_u8(&err, dev, OPAL_STARTROW);
		2034	add_token_u64(&err, dev, offset + off); /* start row value */
		2035	add_token_u8(&err, dev, OPAL_ENDNAME);
		2036
		2037	add_token_u8(&err, dev, OPAL_STARTNAME);
		2038	add_token_u8(&err, dev, OPAL_ENDROW);
		2039
		2040	len = min(max_read_size, (size_t)(read_size - off));
		2041	add_token_u64(&err, dev, offset + off + len); /* end row value
		2042	*/
		2043	add_token_u8(&err, dev, OPAL_ENDNAME);
		2044	add_token_u8(&err, dev, OPAL_ENDLIST);
		2045
		2046	if (err) {
		2047	pr_debug("Error building read table data command.\n");
		2048	break;
		2049	}
		2050
		2051	err = finalize_and_send(dev, read_table_data_cont);
		2052	if (err)
		2053	break;
		2054
		2055	/* len+1: This includes the NULL terminator at the end*/
		2056	if (dev->prev_d_len > len + 1) {
		2057	err = -EOVERFLOW;
		2058	break;
		2059	}
		2060
		2061	dst = (u8 __user *)(uintptr_t)read_tbl->data;
		2062	if (copy_to_user(dst + off, dev->prev_data, dev->prev_d_len)) {
		2063	pr_debug("Error copying data to userspace\n");
		2064	err = -EFAULT;
		2065	break;
		2066	}
		2067	dev->prev_data = NULL;
		2068
		2069	off += len;
		2070	}
		2071
		2072	return err;
		2073	}
		2074
1960	static int end_opal_session(struct opal_dev dev, void data)	2075	static int end_opal_session(struct opal_dev dev, void data)
1961	{	2076	{
1962	int err = 0;	2077	int err = 0;
@@ -2443,6 +2558,68 @@ bool opal_unlock_from_suspend(struct opal_dev *dev) · 480 unmodified lines
2443	}	2558	}
2444	EXPORT_SYMBOL(opal_unlock_from_suspend);	2559	EXPORT_SYMBOL(opal_unlock_from_suspend);
2445		2560
		2561	static int opal_read_table(struct opal_dev *dev,
		2562	struct opal_read_write_table *rw_tbl)
		2563	{
		2564	const struct opal_step read_table_steps[] = {
		2565	{ start_admin1LSP_opal_session, &rw_tbl->key },
		2566	{ read_table_data, rw_tbl },
		2567	{ end_opal_session, }
		2568	};
		2569	int ret = 0;
		2570
		2571	if (!rw_tbl->size)
		2572	return ret;
		2573
		2574	return execute_steps(dev, read_table_steps,
		2575	ARRAY_SIZE(read_table_steps));
		2576	}
		2577
		2578	static int opal_write_table(struct opal_dev *dev,
		2579	struct opal_read_write_table *rw_tbl)
		2580	{
		2581	const struct opal_step write_table_steps[] = {
		2582	{ start_admin1LSP_opal_session, &rw_tbl->key },
		2583	{ write_table_data, rw_tbl },
		2584	{ end_opal_session, }
		2585	};
		2586	int ret = 0;
		2587
		2588	if (!rw_tbl->size)
		2589	return ret;
		2590
		2591	return execute_steps(dev, write_table_steps,
		2592	ARRAY_SIZE(write_table_steps));
		2593	}
		2594
		2595	static int opal_generic_read_write_table(struct opal_dev *dev,
		2596	struct opal_read_write_table *rw_tbl)
		2597	{
		2598	int ret, bit_set;
		2599
		2600	mutex_lock(&dev->dev_lock);
		2601	setup_opal_dev(dev);
		2602
		2603	bit_set = fls64(rw_tbl->flags) - 1;
		2604	switch (bit_set) {
		2605	case OPAL_READ_TABLE:
		2606	ret = opal_read_table(dev, rw_tbl);
		2607	break;
		2608	case OPAL_WRITE_TABLE:
		2609	ret = opal_write_table(dev, rw_tbl);
		2610	break;
		2611	default:
		2612	pr_debug("Invalid bit set in the flag (%016llx).\n",
		2613	rw_tbl->flags);
		2614	ret = -EINVAL;
		2615	break;
		2616	}
		2617
		2618	mutex_unlock(&dev->dev_lock);
		2619
		2620	return ret;
		2621	}
		2622
2446	int sed_ioctl(struct opal_dev dev, unsigned int cmd, void __user arg)	2623	int sed_ioctl(struct opal_dev dev, unsigned int cmd, void __user arg)
2447	{	2624	{
2448	void *p;	2625	void *p;
@@ -2505,6 +2682,9 @@ int sed_ioctl(struct opal_dev dev, unsigned int cmd, void __user arg) · 56 unmodified lines
2505	case IOC_OPAL_PSID_REVERT_TPR:	2682	case IOC_OPAL_PSID_REVERT_TPR:
2506	ret = opal_reverttper(dev, p, true);	2683	ret = opal_reverttper(dev, p, true);
2507	break;	2684	break;
		2685	case IOC_OPAL_GENERIC_TABLE_RW:
		2686	ret = opal_generic_read_write_table(dev, p);
		2687	break;
2508	default:	2688	default:
2509	break;	2689	break;
2510	}	2690	}

block/t10-pi.c+2−6

 	return t10_pi_verify(iter, t10_pi_ip_fn, T10_PI_TYPE3_PROTECTION);
 }
-/**
+/* Type 3 does not have a reference tag so no remapping is required. */
- * Type 3 does not have a reference tag so no remapping is required.
- */
 static void t10_pi_type3_prepare(struct request *rq)
 {
 }
-/**
+/* Type 3 does not have a reference tag so no remapping is required. */
- * Type 3 does not have a reference tag so no remapping is required.
- */
 static void t10_pi_type3_complete(struct request *rq, unsigned int nr_bytes)
 {
 }

drivers/md/dm-zoned-metadata.c+3−3

@@ -1312,9 +1312,9 @@ static int dmz_reset_zone(struct dmz_metadata zmd, struct dm_zone zone) · 1311 unmodified lines
1312	if (!dmz_is_empty(zone) \|\| dmz_seq_write_err(zone)) {	1312	if (!dmz_is_empty(zone) \|\| dmz_seq_write_err(zone)) {
1313	struct dmz_dev *dev = zmd->dev;	1313	struct dmz_dev *dev = zmd->dev;
1314		1314
1315	ret = blkdev_reset_zones(dev->bdev,	1315	ret = blkdev_zone_mgmt(dev->bdev, REQ_OP_ZONE_RESET,
1316	dmz_start_sect(zmd, zone),	1316	dmz_start_sect(zmd, zone),
1317	dev->zone_nr_sectors, GFP_NOIO);	1317	dev->zone_nr_sectors, GFP_NOIO);
1318	if (ret) {	1318	if (ret) {
1319	dmz_dev_err(dev, "Reset zone %u failed %d",	1319	dmz_dev_err(dev, "Reset zone %u failed %d",
1320	dmz_id(zmd, zone), ret);	1320	dmz_id(zmd, zone), ret);

fs/block_dev.c+20−17

@@ -1403,11 +1403,7 @@ static void flush_disk(struct block_device *bdev, bool kill_dirty) · 1402 unmodified lines
1403	"resized disk %s\n",	1403	"resized disk %s\n",
1404	bdev->bd_disk ? bdev->bd_disk->disk_name : "");	1404	bdev->bd_disk ? bdev->bd_disk->disk_name : "");
1405	}	1405	}
1406		1406	bdev->bd_invalidated = 1;
1407	if (!bdev->bd_disk)
1408	return;
1409	if (disk_part_scan_enabled(bdev->bd_disk))
1410	bdev->bd_invalidated = 1;
1411	}	1407	}
1412		1408
1413	/**	1409	/**
@@ -1512,6 +1508,19 @@ EXPORT_SYMBOL(bd_set_size); · 98 unmodified lines
1512		1508
1513	static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part);	1509	static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part);
1514		1510
		1511	static void bdev_disk_changed(struct block_device *bdev, bool invalidate)
		1512	{
		1513	if (disk_part_scan_enabled(bdev->bd_disk)) {
		1514	if (invalidate)
		1515	invalidate_partitions(bdev->bd_disk, bdev);
		1516	else
		1517	rescan_partitions(bdev->bd_disk, bdev);
		1518	} else {
		1519	check_disk_size_change(bdev->bd_disk, bdev, !invalidate);
		1520	bdev->bd_invalidated = 0;
		1521	}
		1522	}
		1523
1515	/*	1524	/*
1516	* bd_mutex locking:	1525	* bd_mutex locking:
1517	*	1526	*
@@ -1594,12 +1603,9 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part) · 76 unmodified lines
1594	* The latter is necessary to prevent ghost	1603	* The latter is necessary to prevent ghost
1595	* partitions on a removed medium.	1604	* partitions on a removed medium.
1596	*/	1605	*/
1597	if (bdev->bd_invalidated) {	1606	if (bdev->bd_invalidated &&
1598	if (!ret)	1607	(!ret \|\| ret == -ENOMEDIUM))
1599	rescan_partitions(disk, bdev);	1608	bdev_disk_changed(bdev, ret == -ENOMEDIUM);
1600	else if (ret == -ENOMEDIUM)
1601	invalidate_partitions(disk, bdev);
1602	}
1603		1609
1604	if (ret)	1610	if (ret)
1605	goto out_clear;	1611	goto out_clear;
@@ -1632,12 +1638,9 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part) · 26 unmodified lines
1632	if (bdev->bd_disk->fops->open)	1638	if (bdev->bd_disk->fops->open)
1633	ret = bdev->bd_disk->fops->open(bdev, mode);	1639	ret = bdev->bd_disk->fops->open(bdev, mode);
1634	/* the same as first opener case, read comment there */	1640	/* the same as first opener case, read comment there */
1635	if (bdev->bd_invalidated) {	1641	if (bdev->bd_invalidated &&
1636	if (!ret)	1642	(!ret \|\| ret == -ENOMEDIUM))
1637	rescan_partitions(bdev->bd_disk, bdev);	1643	bdev_disk_changed(bdev, ret == -ENOMEDIUM);
1638	else if (ret == -ENOMEDIUM)
1639	invalidate_partitions(bdev->bd_disk, bdev);
1640	}
1641	if (ret)	1644	if (ret)
1642	goto out_unlock_bdev;	1645	goto out_unlock_bdev;
1643	}	1646	}

fs/f2fs/segment.c+2−1

@@ -1771,7 +1771,8 @@ static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi, · 1770 unmodified lines
1771	return -EIO;	1771	return -EIO;
1772	}	1772	}
1773	trace_f2fs_issue_reset_zone(bdev, blkstart);	1773	trace_f2fs_issue_reset_zone(bdev, blkstart);
1774	return blkdev_reset_zones(bdev, sector, nr_sects, GFP_NOFS);	1774	return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET,
		1775	sector, nr_sects, GFP_NOFS);
1775	}	1776	}
1776		1777
1777	/* For conventional zones, use regular discard if supported */	1778	/* For conventional zones, use regular discard if supported */

fs/fcntl.c+1−1

@@ -261,7 +261,7 @@ static int f_getowner_uids(struct file *filp, unsigned long arg) · 260 unmodified lines
261	static bool rw_hint_valid(enum rw_hint hint)	261	static bool rw_hint_valid(enum rw_hint hint)
262	{	262	{
263	switch (hint) {	263	switch (hint) {
264	case RWF_WRITE_LIFE_NOT_SET:	264	case RWH_WRITE_LIFE_NOT_SET:
265	case RWH_WRITE_LIFE_NONE:	265	case RWH_WRITE_LIFE_NONE:
266	case RWH_WRITE_LIFE_SHORT:	266	case RWH_WRITE_LIFE_SHORT:
267	case RWH_WRITE_LIFE_MEDIUM:	267	case RWH_WRITE_LIFE_MEDIUM:

include/linux/blk-cgroup.h+37−162

@@ -15,7 +15,9 @@ · 14 unmodified lines
15	*/	15	*/
16		16
17	#include <linux/cgroup.h>	17	#include <linux/cgroup.h>
		18	#include <linux/percpu.h>
18	#include <linux/percpu_counter.h>	19	#include <linux/percpu_counter.h>
		20	#include <linux/u64_stats_sync.h>
19	#include <linux/seq_file.h>	21	#include <linux/seq_file.h>
20	#include <linux/radix-tree.h>	22	#include <linux/radix-tree.h>
21	#include <linux/blkdev.h>	23	#include <linux/blkdev.h>
@@ -31,15 +33,12 @@ · 9 unmodified lines
31		33
32	#ifdef CONFIG_BLK_CGROUP	34	#ifdef CONFIG_BLK_CGROUP
33		35
34	enum blkg_rwstat_type {	36	enum blkg_iostat_type {
35	BLKG_RWSTAT_READ,	37	BLKG_IOSTAT_READ,
36	BLKG_RWSTAT_WRITE,	38	BLKG_IOSTAT_WRITE,
37	BLKG_RWSTAT_SYNC,	39	BLKG_IOSTAT_DISCARD,
38	BLKG_RWSTAT_ASYNC,
39	BLKG_RWSTAT_DISCARD,
40		40
41	BLKG_RWSTAT_NR,	41	BLKG_IOSTAT_NR,
42	BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
43	};	42	};
44		43
45	struct blkcg_gq;	44	struct blkcg_gq;
@@ -61,17 +60,15 @@ struct blkcg { · 15 unmodified lines
61	#endif	60	#endif
62	};	61	};
63		62
64	/*	63	struct blkg_iostat {
65	* blkg_[rw]stat->aux_cnt is excluded for local stats but included for	64	u64 bytes[BLKG_IOSTAT_NR];
66	* recursive. Used to carry stats of dead children.	65	u64 ios[BLKG_IOSTAT_NR];
67	*/
68	struct blkg_rwstat {
69	struct percpu_counter cpu_cnt[BLKG_RWSTAT_NR];
70	atomic64_t aux_cnt[BLKG_RWSTAT_NR];
71	};	66	};
72		67
73	struct blkg_rwstat_sample {	68	struct blkg_iostat_set {
74	u64 cnt[BLKG_RWSTAT_NR];	69	struct u64_stats_sync sync;
		70	struct blkg_iostat cur;
		71	struct blkg_iostat last;
75	};	72	};
76		73
77	/*	74	/*
@@ -127,8 +124,8 @@ struct blkcg_gq { · 49 unmodified lines
127	/* is this blkg online? protected by both blkcg and q locks */	124	/* is this blkg online? protected by both blkcg and q locks */
128	bool online;	125	bool online;
129		126
130	struct blkg_rwstat stat_bytes;	127	struct blkg_iostat_set __percpu *iostat_cpu;
131	struct blkg_rwstat stat_ios;	128	struct blkg_iostat_set iostat;
132		129
133	struct blkg_policy_data *pd[BLKCG_MAX_POLS];	130	struct blkg_policy_data *pd[BLKCG_MAX_POLS];
134		131
@@ -202,13 +199,6 @@ int blkcg_activate_policy(struct request_queue *q, · 67 unmodified lines
202	void blkcg_deactivate_policy(struct request_queue *q,	199	void blkcg_deactivate_policy(struct request_queue *q,
203	const struct blkcg_policy *pol);	200	const struct blkcg_policy *pol);
204		201
205	static inline u64 blkg_rwstat_read_counter(struct blkg_rwstat *rwstat,
206	unsigned int idx)
207	{
208	return atomic64_read(&rwstat->aux_cnt[idx]) +
209	percpu_counter_sum_positive(&rwstat->cpu_cnt[idx]);
210	}
211
212	const char blkg_dev_name(struct blkcg_gq blkg);	202	const char blkg_dev_name(struct blkcg_gq blkg);
213	void blkcg_print_blkgs(struct seq_file sf, struct blkcg blkcg,	203	void blkcg_print_blkgs(struct seq_file sf, struct blkcg blkcg,
214	u64 (prfill)(struct seq_file ,	204	u64 (prfill)(struct seq_file ,
@@ -216,17 +206,6 @@ void blkcg_print_blkgs(struct seq_file sf, struct blkcg blkcg, · 1 unmodified line
216	const struct blkcg_policy *pol, int data,	206	const struct blkcg_policy *pol, int data,
217	bool show_total);	207	bool show_total);
218	u64 __blkg_prfill_u64(struct seq_file sf, struct blkg_policy_data pd, u64 v);	208	u64 __blkg_prfill_u64(struct seq_file sf, struct blkg_policy_data pd, u64 v);
219	u64 __blkg_prfill_rwstat(struct seq_file sf, struct blkg_policy_data pd,
220	const struct blkg_rwstat_sample *rwstat);
221	u64 blkg_prfill_rwstat(struct seq_file sf, struct blkg_policy_data pd,
222	int off);
223	int blkg_print_stat_bytes(struct seq_file sf, void v);
224	int blkg_print_stat_ios(struct seq_file sf, void v);
225	int blkg_print_stat_bytes_recursive(struct seq_file sf, void v);
226	int blkg_print_stat_ios_recursive(struct seq_file sf, void v);
227
228	void blkg_rwstat_recursive_sum(struct blkcg_gq blkg, struct blkcg_policy pol,
229	int off, struct blkg_rwstat_sample *sum);
230		209
231	struct blkg_conf_ctx {	210	struct blkg_conf_ctx {
232	struct gendisk *disk;	211	struct gendisk *disk;
@@ -578,128 +557,6 @@ static inline void blkg_put(struct blkcg_gq *blkg) · 345 unmodified lines
578	if (((d_blkg) = __blkg_lookup(css_to_blkcg(pos_css), \	557	if (((d_blkg) = __blkg_lookup(css_to_blkcg(pos_css), \
579	(p_blkg)->q, false)))	558	(p_blkg)->q, false)))
580		559
581	static inline int blkg_rwstat_init(struct blkg_rwstat *rwstat, gfp_t gfp)
582	{
583	int i, ret;
584
585	for (i = 0; i < BLKG_RWSTAT_NR; i++) {
586	ret = percpu_counter_init(&rwstat->cpu_cnt[i], 0, gfp);
587	if (ret) {
588	while (--i >= 0)
589	percpu_counter_destroy(&rwstat->cpu_cnt[i]);
590	return ret;
591	}
592	atomic64_set(&rwstat->aux_cnt[i], 0);
593	}
594	return 0;
595	}
596
597	static inline void blkg_rwstat_exit(struct blkg_rwstat *rwstat)
598	{
599	int i;
600
601	for (i = 0; i < BLKG_RWSTAT_NR; i++)
602	percpu_counter_destroy(&rwstat->cpu_cnt[i]);
603	}
604
605	/**
606	* blkg_rwstat_add - add a value to a blkg_rwstat
607	* @rwstat: target blkg_rwstat
608	* @op: REQ_OP and flags
609	* @val: value to add
610	*
611	* Add @val to @rwstat. The counters are chosen according to @rw. The
612	* caller is responsible for synchronizing calls to this function.
613	*/
614	static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
615	unsigned int op, uint64_t val)
616	{
617	struct percpu_counter *cnt;
618
619	if (op_is_discard(op))
620	cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_DISCARD];
621	else if (op_is_write(op))
622	cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_WRITE];
623	else
624	cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_READ];
625
626	percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
627
628	if (op_is_sync(op))
629	cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC];
630	else
631	cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC];
632
633	percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
634	}
635
636	/**
637	* blkg_rwstat_read - read the current values of a blkg_rwstat
638	* @rwstat: blkg_rwstat to read
639	*
640	* Read the current snapshot of @rwstat and return it in the aux counts.
641	*/
642	static inline void blkg_rwstat_read(struct blkg_rwstat *rwstat,
643	struct blkg_rwstat_sample *result)
644	{
645	int i;
646
647	for (i = 0; i < BLKG_RWSTAT_NR; i++)
648	result->cnt[i] =
649	percpu_counter_sum_positive(&rwstat->cpu_cnt[i]);
650	}
651
652	/**
653	* blkg_rwstat_total - read the total count of a blkg_rwstat
654	* @rwstat: blkg_rwstat to read
655	*
656	* Return the total count of @rwstat regardless of the IO direction. This
657	* function can be called without synchronization and takes care of u64
658	* atomicity.
659	*/
660	static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat)
661	{
662	struct blkg_rwstat_sample tmp = { };
663
664	blkg_rwstat_read(rwstat, &tmp);
665	return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
666	}
667
668	/**
669	* blkg_rwstat_reset - reset a blkg_rwstat
670	* @rwstat: blkg_rwstat to reset
671	*/
672	static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
673	{
674	int i;
675
676	for (i = 0; i < BLKG_RWSTAT_NR; i++) {
677	percpu_counter_set(&rwstat->cpu_cnt[i], 0);
678	atomic64_set(&rwstat->aux_cnt[i], 0);
679	}
680	}
681
682	/**
683	* blkg_rwstat_add_aux - add a blkg_rwstat into another's aux count
684	* @to: the destination blkg_rwstat
685	* @from: the source
686	*
687	* Add @from's count including the aux one to @to's aux count.
688	*/
689	static inline void blkg_rwstat_add_aux(struct blkg_rwstat *to,
690	struct blkg_rwstat *from)
691	{
692	u64 sum[BLKG_RWSTAT_NR];
693	int i;
694
695	for (i = 0; i < BLKG_RWSTAT_NR; i++)
696	sum[i] = percpu_counter_sum_positive(&from->cpu_cnt[i]);
697
698	for (i = 0; i < BLKG_RWSTAT_NR; i++)
699	atomic64_add(sum[i] + atomic64_read(&from->aux_cnt[i]),
700	&to->aux_cnt[i]);
701	}
702
703	#ifdef CONFIG_BLK_DEV_THROTTLING	560	#ifdef CONFIG_BLK_DEV_THROTTLING
704	extern bool blk_throtl_bio(struct request_queue q, struct blkcg_gq blkg,	561	extern bool blk_throtl_bio(struct request_queue q, struct blkcg_gq blkg,
705	struct bio *bio);	562	struct bio *bio);
@@ -745,15 +602,33 @@ static inline bool blkcg_bio_issue_check(struct request_queue *q, · 39 unmodified lines
745	throtl = blk_throtl_bio(q, blkg, bio);	602	throtl = blk_throtl_bio(q, blkg, bio);
746		603
747	if (!throtl) {	604	if (!throtl) {
		605	struct blkg_iostat_set *bis;
		606	int rwd, cpu;
		607
		608	if (op_is_discard(bio->bi_opf))
		609	rwd = BLKG_IOSTAT_DISCARD;
		610	else if (op_is_write(bio->bi_opf))
		611	rwd = BLKG_IOSTAT_WRITE;
		612	else
		613	rwd = BLKG_IOSTAT_READ;
		614
		615	cpu = get_cpu();
		616	bis = per_cpu_ptr(blkg->iostat_cpu, cpu);
		617	u64_stats_update_begin(&bis->sync);
		618
748	/*	619	/*
749	* If the bio is flagged with BIO_QUEUE_ENTERED it means this	620	* If the bio is flagged with BIO_QUEUE_ENTERED it means this
750	* is a split bio and we would have already accounted for the	621	* is a split bio and we would have already accounted for the
751	* size of the bio.	622	* size of the bio.
752	*/	623	*/
753	if (!bio_flagged(bio, BIO_QUEUE_ENTERED))	624	if (!bio_flagged(bio, BIO_QUEUE_ENTERED))
754	blkg_rwstat_add(&blkg->stat_bytes, bio->bi_opf,	625	bis->cur.bytes[rwd] += bio->bi_iter.bi_size;
755	bio->bi_iter.bi_size);	626	bis->cur.ios[rwd]++;
756	blkg_rwstat_add(&blkg->stat_ios, bio->bi_opf, 1);	627
		628	u64_stats_update_end(&bis->sync);
		629	if (cgroup_subsys_on_dfl(io_cgrp_subsys))
		630	cgroup_rstat_updated(blkg->blkcg->css.cgroup, cpu);
		631	put_cpu();
757	}	632	}
758		633
759	blkcg_bio_issue_init(bio);	634	blkcg_bio_issue_init(bio);

include/linux/blk-mq.h+246−54

@@ -10,103 +10,239 @@ struct blk_mq_tags; · 9 unmodified lines
10	struct blk_flush_queue;	10	struct blk_flush_queue;
11		11
12	/**	12	/**
13	* struct blk_mq_hw_ctx - State for a hardware queue facing the hardware block device	13	* struct blk_mq_hw_ctx - State for a hardware queue facing the hardware
		14	* block device
14	*/	15	*/
15	struct blk_mq_hw_ctx {	16	struct blk_mq_hw_ctx {
16	struct {	17	struct {
		18	/** @lock: Protects the dispatch list. */
17	spinlock_t lock;	19	spinlock_t lock;
		20	/**
		21	* @dispatch: Used for requests that are ready to be
		22	* dispatched to the hardware but for some reason (e.g. lack of
		23	* resources) could not be sent to the hardware. As soon as the
		24	* driver can send new requests, requests at this list will
		25	* be sent first for a fairer dispatch.
		26	*/
18	struct list_head dispatch;	27	struct list_head dispatch;
19	unsigned long state; /* BLK_MQ_S_* flags */	28	/**
		29	* @state: BLK_MQ_S_* flags. Defines the state of the hw
		30	* queue (active, scheduled to restart, stopped).
		31	*/
		32	unsigned long state;
20	} ____cacheline_aligned_in_smp;	33	} ____cacheline_aligned_in_smp;
21		34
		35	/**
		36	* @run_work: Used for scheduling a hardware queue run at a later time.
		37	*/
22	struct delayed_work run_work;	38	struct delayed_work run_work;
		39	/** @cpumask: Map of available CPUs where this hctx can run. */
23	cpumask_var_t cpumask;	40	cpumask_var_t cpumask;
		41	/**
		42	* @next_cpu: Used by blk_mq_hctx_next_cpu() for round-robin CPU
		43	* selection from @cpumask.
		44	*/
24	int next_cpu;	45	int next_cpu;
		46	/**
		47	* @next_cpu_batch: Counter of how many works left in the batch before
		48	* changing to the next CPU.
		49	*/
25	int next_cpu_batch;	50	int next_cpu_batch;
26		51
27	unsigned long flags; /* BLK_MQ_F_* flags */	52	/** @flags: BLK_MQ_F_* flags. Defines the behaviour of the queue. */
		53	unsigned long flags;
28		54
		55	/**
		56	* @sched_data: Pointer owned by the IO scheduler attached to a request
		57	* queue. It's up to the IO scheduler how to use this pointer.
		58	*/
29	void *sched_data;	59	void *sched_data;
		60	/**
		61	* @queue: Pointer to the request queue that owns this hardware context.
		62	*/
30	struct request_queue *queue;	63	struct request_queue *queue;
		64	/** @fq: Queue of requests that need to perform a flush operation. */
31	struct blk_flush_queue *fq;	65	struct blk_flush_queue *fq;
32		66
		67	/**
		68	* @driver_data: Pointer to data owned by the block driver that created
		69	* this hctx
		70	*/
33	void *driver_data;	71	void *driver_data;
34		72
		73	/**
		74	* @ctx_map: Bitmap for each software queue. If bit is on, there is a
		75	* pending request in that software queue.
		76	*/
35	struct sbitmap ctx_map;	77	struct sbitmap ctx_map;
36		78
		79	/**
		80	* @dispatch_from: Software queue to be used when no scheduler was
		81	* selected.
		82	*/
37	struct blk_mq_ctx *dispatch_from;	83	struct blk_mq_ctx *dispatch_from;
		84	/**
		85	* @dispatch_busy: Number used by blk_mq_update_dispatch_busy() to
		86	* decide if the hw_queue is busy using Exponential Weighted Moving
		87	* Average algorithm.
		88	*/
38	unsigned int dispatch_busy;	89	unsigned int dispatch_busy;
39		90
		91	/** @type: HCTX_TYPE_* flags. Type of hardware queue. */
40	unsigned short type;	92	unsigned short type;
		93	/** @nr_ctx: Number of software queues. */
41	unsigned short nr_ctx;	94	unsigned short nr_ctx;
		95	/** @ctxs: Array of software queues. */
42	struct blk_mq_ctx **ctxs;	96	struct blk_mq_ctx **ctxs;
43		97
		98	/** @dispatch_wait_lock: Lock for dispatch_wait queue. */
44	spinlock_t dispatch_wait_lock;	99	spinlock_t dispatch_wait_lock;
		100	/**
		101	* @dispatch_wait: Waitqueue to put requests when there is no tag
		102	* available at the moment, to wait for another try in the future.
		103	*/
45	wait_queue_entry_t dispatch_wait;	104	wait_queue_entry_t dispatch_wait;
		105
		106	/**
		107	* @wait_index: Index of next available dispatch_wait queue to insert
		108	* requests.
		109	*/
46	atomic_t wait_index;	110	atomic_t wait_index;
47		111
		112	/**
		113	* @tags: Tags owned by the block driver. A tag at this set is only
		114	* assigned when a request is dispatched from a hardware queue.
		115	*/
48	struct blk_mq_tags *tags;	116	struct blk_mq_tags *tags;
		117	/**
		118	* @sched_tags: Tags owned by I/O scheduler. If there is an I/O
		119	* scheduler associated with a request queue, a tag is assigned when
		120	* that request is allocated. Else, this member is not used.
		121	*/
49	struct blk_mq_tags *sched_tags;	122	struct blk_mq_tags *sched_tags;
50		123
		124	/** @queued: Number of queued requests. */
51	unsigned long queued;	125	unsigned long queued;
		126	/** @run: Number of dispatched requests. */
52	unsigned long run;	127	unsigned long run;
53	#define BLK_MQ_MAX_DISPATCH_ORDER 7	128	#define BLK_MQ_MAX_DISPATCH_ORDER 7
		129	/** @dispatched: Number of dispatch requests by queue. */
54	unsigned long dispatched[BLK_MQ_MAX_DISPATCH_ORDER];	130	unsigned long dispatched[BLK_MQ_MAX_DISPATCH_ORDER];
55		131
		132	/** @numa_node: NUMA node the storage adapter has been connected to. */
56	unsigned int numa_node;	133	unsigned int numa_node;
		134	/** @queue_num: Index of this hardware queue. */
57	unsigned int queue_num;	135	unsigned int queue_num;
58		136
		137	/**
		138	* @nr_active: Number of active requests. Only used when a tag set is
		139	* shared across request queues.
		140	*/
59	atomic_t nr_active;	141	atomic_t nr_active;
60		142
		143	/** @cpuhp_dead: List to store request if some CPU die. */
61	struct hlist_node cpuhp_dead;	144	struct hlist_node cpuhp_dead;
		145	/** @kobj: Kernel object for sysfs. */
62	struct kobject kobj;	146	struct kobject kobj;
63		147
		148	/** @poll_considered: Count times blk_poll() was called. */
64	unsigned long poll_considered;	149	unsigned long poll_considered;
		150	/** @poll_invoked: Count how many requests blk_poll() polled. */
65	unsigned long poll_invoked;	151	unsigned long poll_invoked;
		152	/** @poll_success: Count how many polled requests were completed. */
66	unsigned long poll_success;	153	unsigned long poll_success;
67		154
68	#ifdef CONFIG_BLK_DEBUG_FS	155	#ifdef CONFIG_BLK_DEBUG_FS
		156	/**
		157	* @debugfs_dir: debugfs directory for this hardware queue. Named
		158	* as cpu<cpu_number>.
		159	*/
69	struct dentry *debugfs_dir;	160	struct dentry *debugfs_dir;
		161	/** @sched_debugfs_dir: debugfs directory for the scheduler. */
70	struct dentry *sched_debugfs_dir;	162	struct dentry *sched_debugfs_dir;
71	#endif	163	#endif
72		164
		165	/** @hctx_list: List of all hardware queues. */
73	struct list_head hctx_list;	166	struct list_head hctx_list;
74		167
75	/* Must be the last member - see also blk_mq_hw_ctx_size(). */	168	/**
		169	* @srcu: Sleepable RCU. Use as lock when type of the hardware queue is
		170	* blocking (BLK_MQ_F_BLOCKING). Must be the last member - see also
		171	* blk_mq_hw_ctx_size().
		172	*/
76	struct srcu_struct srcu[0];	173	struct srcu_struct srcu[0];
77	};	174	};
78		175
		176	/**
		177	* struct blk_mq_queue_map - Map software queues to hardware queues
		178	* @mq_map: CPU ID to hardware queue index map. This is an array
		179	* with nr_cpu_ids elements. Each element has a value in the range
		180	* [@queue_offset, @queue_offset + @nr_queues).
		181	* @nr_queues: Number of hardware queues to map CPU IDs onto.
		182	* @queue_offset: First hardware queue to map onto. Used by the PCIe NVMe
		183	* driver to map each hardware queue type (enum hctx_type) onto a distinct
		184	* set of hardware queues.
		185	*/
79	struct blk_mq_queue_map {	186	struct blk_mq_queue_map {
80	unsigned int *mq_map;	187	unsigned int *mq_map;
81	unsigned int nr_queues;	188	unsigned int nr_queues;
82	unsigned int queue_offset;	189	unsigned int queue_offset;
83	};	190	};
84		191
		192	/**
		193	* enum hctx_type - Type of hardware queue
		194	* @HCTX_TYPE_DEFAULT: All I/O not otherwise accounted for.
		195	* @HCTX_TYPE_READ: Just for READ I/O.
		196	* @HCTX_TYPE_POLL: Polled I/O of any kind.
		197	* @HCTX_MAX_TYPES: Number of types of hctx.
		198	*/
85	enum hctx_type {	199	enum hctx_type {
86	HCTX_TYPE_DEFAULT, /* all I/O not otherwise accounted for */	200	HCTX_TYPE_DEFAULT,
87	HCTX_TYPE_READ, /* just for READ I/O */	201	HCTX_TYPE_READ,
88	HCTX_TYPE_POLL, /* polled I/O of any kind */	202	HCTX_TYPE_POLL,
89		203
90	HCTX_MAX_TYPES,	204	HCTX_MAX_TYPES,
91	};	205	};
92		206
		207	/**
		208	* struct blk_mq_tag_set - tag set that can be shared between request queues
		209	* @map: One or more ctx -> hctx mappings. One map exists for each
		210	* hardware queue type (enum hctx_type) that the driver wishes
		211	* to support. There are no restrictions on maps being of the
		212	* same size, and it's perfectly legal to share maps between
		213	* types.
		214	* @nr_maps: Number of elements in the @map array. A number in the range
		215	* [1, HCTX_MAX_TYPES].
		216	* @ops: Pointers to functions that implement block driver behavior.
		217	* @nr_hw_queues: Number of hardware queues supported by the block driver that
		218	* owns this data structure.
		219	* @queue_depth: Number of tags per hardware queue, reserved tags included.
		220	* @reserved_tags: Number of tags to set aside for BLK_MQ_REQ_RESERVED tag
		221	* allocations.
		222	* @cmd_size: Number of additional bytes to allocate per request. The block
		223	* driver owns these additional bytes.
		224	* @numa_node: NUMA node the storage adapter has been connected to.
		225	* @timeout: Request processing timeout in jiffies.
		226	* @flags: Zero or more BLK_MQ_F_* flags.
		227	* @driver_data: Pointer to data owned by the block driver that created this
		228	* tag set.
		229	* @tags: Tag sets. One tag set per hardware queue. Has @nr_hw_queues
		230	* elements.
		231	* @tag_list_lock: Serializes tag_list accesses.
		232	* @tag_list: List of the request queues that use this tag set. See also
		233	* request_queue.tag_set_list.
		234	*/
93	struct blk_mq_tag_set {	235	struct blk_mq_tag_set {
94	/*
95	* map[] holds ctx -> hctx mappings, one map exists for each type
96	* that the driver wishes to support. There are no restrictions
97	* on maps being of the same size, and it's perfectly legal to
98	* share maps between types.
99	*/
100	struct blk_mq_queue_map map[HCTX_MAX_TYPES];	236	struct blk_mq_queue_map map[HCTX_MAX_TYPES];
101	unsigned int nr_maps; /* nr entries in map[] */	237	unsigned int nr_maps;
102	const struct blk_mq_ops *ops;	238	const struct blk_mq_ops *ops;
103	unsigned int nr_hw_queues; /* nr hw queues across maps */	239	unsigned int nr_hw_queues;
104	unsigned int queue_depth; /* max hw supported */	240	unsigned int queue_depth;
105	unsigned int reserved_tags;	241	unsigned int reserved_tags;
106	unsigned int cmd_size; /* per-request extra data */	242	unsigned int cmd_size;
107	int numa_node;	243	int numa_node;
108	unsigned int timeout;	244	unsigned int timeout;
109	unsigned int flags; /* BLK_MQ_F_* */	245	unsigned int flags;
110	void *driver_data;	246	void *driver_data;
111		247
112	struct blk_mq_tags **tags;	248	struct blk_mq_tags **tags;
@@ -115,6 +251,12 @@ struct blk_mq_tag_set { · 2 unmodified lines
115	struct list_head tag_list;	251	struct list_head tag_list;
116	};	252	};
117		253
		254	/**
		255	* struct blk_mq_queue_data - Data about a request inserted in a queue
		256	*
		257	* @rq: Request pointer.
		258	* @last: If it is the last request in the queue.
		259	*/
118	struct blk_mq_queue_data {	260	struct blk_mq_queue_data {
119	struct request *rq;	261	struct request *rq;
120	bool last;	262	bool last;
@@ -142,81 +284,101 @@ typedef bool (busy_fn)(struct request_queue *); · 21 unmodified lines
142	typedef void (complete_fn)(struct request *);	284	typedef void (complete_fn)(struct request *);
143	typedef void (cleanup_rq_fn)(struct request *);	285	typedef void (cleanup_rq_fn)(struct request *);
144		286
145		287	/**
		288	* struct blk_mq_ops - Callback functions that implements block driver
		289	* behaviour.
		290	*/
146	struct blk_mq_ops {	291	struct blk_mq_ops {
147	/*	292	/**
148	* Queue request	293	* @queue_rq: Queue a new request from block IO.
149	*/	294	*/
150	queue_rq_fn *queue_rq;	295	queue_rq_fn *queue_rq;
151		296
152	/*	297	/**
153	* If a driver uses bd->last to judge when to submit requests to	298	* @commit_rqs: If a driver uses bd->last to judge when to submit
154	* hardware, it must define this function. In case of errors that	299	* requests to hardware, it must define this function. In case of errors
155	* make us stop issuing further requests, this hook serves the	300	* that make us stop issuing further requests, this hook serves the
156	* purpose of kicking the hardware (which the last request otherwise	301	* purpose of kicking the hardware (which the last request otherwise
157	* would have done).	302	* would have done).
158	*/	303	*/
159	commit_rqs_fn *commit_rqs;	304	commit_rqs_fn *commit_rqs;
160		305
161	/*	306	/**
162	* Reserve budget before queue request, once .queue_rq is	307	* @get_budget: Reserve budget before queue request, once .queue_rq is
163	* run, it is driver's responsibility to release the	308	* run, it is driver's responsibility to release the
164	* reserved budget. Also we have to handle failure case	309	* reserved budget. Also we have to handle failure case
165	* of .get_budget for avoiding I/O deadlock.	310	* of .get_budget for avoiding I/O deadlock.
166	*/	311	*/
167	get_budget_fn *get_budget;	312	get_budget_fn *get_budget;
		313	/**
		314	* @put_budget: Release the reserved budget.
		315	*/
168	put_budget_fn *put_budget;	316	put_budget_fn *put_budget;
169		317
170	/*	318	/**
171	* Called on request timeout	319	* @timeout: Called on request timeout.
172	*/	320	*/
173	timeout_fn *timeout;	321	timeout_fn *timeout;
174		322
175	/*	323	/**
176	* Called to poll for completion of a specific tag.	324	* @poll: Called to poll for completion of a specific tag.
177	*/	325	*/
178	poll_fn *poll;	326	poll_fn *poll;
179		327
		328	/**
		329	* @complete: Mark the request as complete.
		330	*/
180	complete_fn *complete;	331	complete_fn *complete;
181		332
182	/*	333	/**
183	* Called when the block layer side of a hardware queue has been	334	* @init_hctx: Called when the block layer side of a hardware queue has
184	* set up, allowing the driver to allocate/init matching structures.	335	* been set up, allowing the driver to allocate/init matching
185	* Ditto for exit/teardown.	336	* structures.
186	*/	337	*/
187	init_hctx_fn *init_hctx;	338	init_hctx_fn *init_hctx;
		339	/**
		340	* @exit_hctx: Ditto for exit/teardown.
		341	*/
188	exit_hctx_fn *exit_hctx;	342	exit_hctx_fn *exit_hctx;
189		343
190	/*	344	/**
191	* Called for every command allocated by the block layer to allow	345	* @init_request: Called for every command allocated by the block layer
192	* the driver to set up driver specific data.	346	* to allow the driver to set up driver specific data.
193	*	347	*
194	* Tag greater than or equal to queue_depth is for setting up	348	* Tag greater than or equal to queue_depth is for setting up
195	* flush request.	349	* flush request.
196	*
197	* Ditto for exit/teardown.
198	*/	350	*/
199	init_request_fn *init_request;	351	init_request_fn *init_request;
		352	/**
		353	* @exit_request: Ditto for exit/teardown.
		354	*/
200	exit_request_fn *exit_request;	355	exit_request_fn *exit_request;
201	/* Called from inside blk_get_request() */	356
		357	/**
		358	* @initialize_rq_fn: Called from inside blk_get_request().
		359	*/
202	void (initialize_rq_fn)(struct request rq);	360	void (initialize_rq_fn)(struct request rq);
203		361
204	/*	362	/**
205	* Called before freeing one request which isn't completed yet,	363	* @cleanup_rq: Called before freeing one request which isn't completed
206	* and usually for freeing the driver private data	364	* yet, and usually for freeing the driver private data.
207	*/	365	*/
208	cleanup_rq_fn *cleanup_rq;	366	cleanup_rq_fn *cleanup_rq;
209		367
210	/*	368	/**
211	* If set, returns whether or not this queue currently is busy	369	* @busy: If set, returns whether or not this queue currently is busy.
212	*/	370	*/
213	busy_fn *busy;	371	busy_fn *busy;
214		372
		373	/**
		374	* @map_queues: This allows drivers specify their own queue mapping by
		375	* overriding the setup-time function that builds the mq_map.
		376	*/
215	map_queues_fn *map_queues;	377	map_queues_fn *map_queues;
216		378
217	#ifdef CONFIG_BLK_DEBUG_FS	379	#ifdef CONFIG_BLK_DEBUG_FS
218	/*	380	/**
219	* Used by the debugfs implementation to show driver-specific	381	* @show_rq: Used by the debugfs implementation to show driver-specific
220	* information about a request.	382	* information about a request.
221	*/	383	*/
222	void (show_rq)(struct seq_file m, struct request *rq);	384	void (show_rq)(struct seq_file m, struct request *rq);
@@ -262,7 +424,6 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set); · 39 unmodified lines
262	void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule);	424	void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule);
263		425
264	void blk_mq_free_request(struct request *rq);	426	void blk_mq_free_request(struct request *rq);
265	bool blk_mq_can_queue(struct blk_mq_hw_ctx *);
266		427
267	bool blk_mq_queue_inflight(struct request_queue *q);	428	bool blk_mq_queue_inflight(struct request_queue *q);
268		429
@@ -301,9 +462,25 @@ static inline u16 blk_mq_unique_tag_to_tag(u32 unique_tag) · 32 unmodified lines
301	return unique_tag & BLK_MQ_UNIQUE_TAG_MASK;	462	return unique_tag & BLK_MQ_UNIQUE_TAG_MASK;
302	}	463	}
303		464
		465	/**
		466	* blk_mq_rq_state() - read the current MQ_RQ_* state of a request
		467	* @rq: target request.
		468	*/
		469	static inline enum mq_rq_state blk_mq_rq_state(struct request *rq)
		470	{
		471	return READ_ONCE(rq->state);
		472	}
		473
		474	static inline int blk_mq_request_started(struct request *rq)
		475	{
		476	return blk_mq_rq_state(rq) != MQ_RQ_IDLE;
		477	}
		478
		479	static inline int blk_mq_request_completed(struct request *rq)
		480	{
		481	return blk_mq_rq_state(rq) == MQ_RQ_COMPLETE;
		482	}
304		483
305	int blk_mq_request_started(struct request *rq);
306	int blk_mq_request_completed(struct request *rq);
307	void blk_mq_start_request(struct request *rq);	484	void blk_mq_start_request(struct request *rq);
308	void blk_mq_end_request(struct request *rq, blk_status_t error);	485	void blk_mq_end_request(struct request *rq, blk_status_t error);
309	void __blk_mq_end_request(struct request *rq, blk_status_t error);	486	void __blk_mq_end_request(struct request *rq, blk_status_t error);
@@ -324,7 +501,7 @@ void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async); · 14 unmodified lines
324	void blk_mq_quiesce_queue(struct request_queue *q);	501	void blk_mq_quiesce_queue(struct request_queue *q);
325	void blk_mq_unquiesce_queue(struct request_queue *q);	502	void blk_mq_unquiesce_queue(struct request_queue *q);
326	void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs);	503	void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs);
327	bool blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);	504	void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
328	void blk_mq_run_hw_queues(struct request_queue *q, bool async);	505	void blk_mq_run_hw_queues(struct request_queue *q, bool async);
329	void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,	506	void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
330	busy_tag_iter_fn fn, void priv);	507	busy_tag_iter_fn fn, void priv);
@@ -343,14 +520,29 @@ void blk_mq_quiesce_queue_nowait(struct request_queue *q); · 12 unmodified lines
343		520
344	unsigned int blk_mq_rq_cpu(struct request *rq);	521	unsigned int blk_mq_rq_cpu(struct request *rq);
345		522
346	/*	523	/**
		524	* blk_mq_rq_from_pdu - cast a PDU to a request
		525	* @pdu: the PDU (Protocol Data Unit) to be casted
		526	*
		527	* Return: request
		528	*
347	* Driver command data is immediately after the request. So subtract request	529	* Driver command data is immediately after the request. So subtract request
348	* size to get back to the original request, add request size to get the PDU.	530	* size to get back to the original request.
349	*/	531	*/
350	static inline struct request blk_mq_rq_from_pdu(void pdu)	532	static inline struct request blk_mq_rq_from_pdu(void pdu)
351	{	533	{
352	return pdu - sizeof(struct request);	534	return pdu - sizeof(struct request);
353	}	535	}
		536
		537	/**
		538	* blk_mq_rq_to_pdu - cast a request to a PDU
		539	* @rq: the request to be casted
		540	*
		541	* Return: pointer to the PDU
		542	*
		543	* Driver command data is immediately after the request. So add request to get
		544	* the PDU.
		545	*/
354	static inline void blk_mq_rq_to_pdu(struct request rq)	546	static inline void blk_mq_rq_to_pdu(struct request rq)
355	{	547	{
356	return rq + 1;	548	return rq + 1;

include/linux/blk_types.h+27−1

 	unsigned short		bi_write_hint;
 	blk_status_t		bi_status;
 	u8			bi_partno;
+	atomic_t		__bi_remaining;
 	struct bvec_iter	bi_iter;
-	atomic_t		__bi_remaining;
 	bio_end_io_t		*bi_end_io;
 	void			*bi_private;
 	REQ_OP_ZONE_RESET_ALL	= 8,
 	/* write the zero filled sector many times */
 	REQ_OP_WRITE_ZEROES	= 9,
+	/* Open a zone */
+	REQ_OP_ZONE_OPEN	= 10,
+	/* Close a zone */
+	REQ_OP_ZONE_CLOSE	= 11,
+	/* Transition a zone to full */
+	REQ_OP_ZONE_FINISH	= 12,
 	/* SCSI passthrough using struct scsi_request */
 	REQ_OP_SCSI_IN		= 32,
 	STAT_READ,
 	STAT_WRITE,
 	STAT_DISCARD,
+	STAT_FLUSH,
 	NR_STAT_GROUPS
 };
 	return (op & REQ_OP_MASK) == REQ_OP_DISCARD;
 }
+/*
+ * Check if a bio or request operation is a zone management operation, with
+ * the exception of REQ_OP_ZONE_RESET_ALL which is treated as a special case
+ * due to its different handling in the block layer and device response in
+ * case of command failure.
+ */
+static inline bool op_is_zone_mgmt(enum req_opf op)
+{
+	switch (op & REQ_OP_MASK) {
+	case REQ_OP_ZONE_RESET:
+	case REQ_OP_ZONE_OPEN:
+	case REQ_OP_ZONE_CLOSE:
+	case REQ_OP_ZONE_FINISH:
+		return true;
+	default:
+		return false;
+	}
+}
 static inline int op_stat_group(unsigned int op)
 {
 	if (op_is_discard(op))

include/linux/blkdev.h+8−8

 extern int blkdev_report_zones(struct block_device *bdev,
 			       sector_t sector, struct blk_zone *zones,
 			       unsigned int *nr_zones);
-extern int blkdev_reset_zones(struct block_device *bdev, sector_t sectors,
+extern int blkdev_zone_mgmt(struct block_device *bdev, enum req_opf op,
-			      sector_t nr_sectors, gfp_t gfp_mask);
+			    sector_t sectors, sector_t nr_sectors,
+			    gfp_t gfp_mask);
 extern int blk_revalidate_disk_zones(struct gendisk *disk);
 extern int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode,
 				     unsigned int cmd, unsigned long arg);
-extern int blkdev_reset_zones_ioctl(struct block_device *bdev, fmode_t mode,
+extern int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode,
-				    unsigned int cmd, unsigned long arg);
+				  unsigned int cmd, unsigned long arg);
 #else /* CONFIG_BLK_DEV_ZONED */
 	return -ENOTTY;
 }
-static inline int blkdev_reset_zones_ioctl(struct block_device *bdev,
+static inline int blkdev_zone_mgmt_ioctl(struct block_device *bdev,
-					   fmode_t mode, unsigned int cmd,
+					 fmode_t mode, unsigned int cmd,
-					   unsigned long arg)
+					 unsigned long arg)
 {
 	return -ENOTTY;
 }
 	/* sw queues */
 	struct blk_mq_ctx __percpu	*queue_ctx;
-	unsigned int		nr_queues;
 	unsigned int		queue_depth;

include/linux/sbitmap.h+0−9

@@ -216,15 +216,6 @@ int sbitmap_get_shallow(struct sbitmap *sb, unsigned int alloc_hint, · 215 unmodified lines
216	*/	216	*/
217	bool sbitmap_any_bit_set(const struct sbitmap *sb);	217	bool sbitmap_any_bit_set(const struct sbitmap *sb);
218		218
219	/**
220	* sbitmap_any_bit_clear() - Check for an unset bit in a &struct
221	* sbitmap.
222	* @sb: Bitmap to check.
223	*
224	* Return: true if any bit in the bitmap is clear, false otherwise.
225	*/
226	bool sbitmap_any_bit_clear(const struct sbitmap *sb);
227
228	#define SB_NR_TO_INDEX(sb, bitnr) ((bitnr) >> (sb)->shift)	219	#define SB_NR_TO_INDEX(sb, bitnr) ((bitnr) >> (sb)->shift)
229	#define SB_NR_TO_BIT(sb, bitnr) ((bitnr) & ((1U << (sb)->shift) - 1U))	220	#define SB_NR_TO_BIT(sb, bitnr) ((bitnr) & ((1U << (sb)->shift) - 1U))
230		221

include/linux/sed-opal.h+1−0

@@ -42,6 +42,7 @@ static inline bool is_sed_ioctl(unsigned int cmd) · 41 unmodified lines
42	case IOC_OPAL_PSID_REVERT_TPR:	42	case IOC_OPAL_PSID_REVERT_TPR:
43	case IOC_OPAL_MBR_DONE:	43	case IOC_OPAL_MBR_DONE:
44	case IOC_OPAL_WRITE_SHADOW_MBR:	44	case IOC_OPAL_WRITE_SHADOW_MBR:
		45	case IOC_OPAL_GENERIC_TABLE_RW:
45	return true;	46	return true;
46	}	47	}
47	return false;	48	return false;

include/trace/events/wbt.h+8−4

 	),
 	TP_fast_assign(
-		strncpy(__entry->name, dev_name(bdi->dev), 32);
+		strlcpy(__entry->name, dev_name(bdi->dev),
+			ARRAY_SIZE(__entry->name));
 		__entry->rmean		= stat[0].mean;
 		__entry->rmin		= stat[0].min;
 		__entry->rmax		= stat[0].max;
 	),
 	TP_fast_assign(
-		strncpy(__entry->name, dev_name(bdi->dev), 32);
+		strlcpy(__entry->name, dev_name(bdi->dev),
+			ARRAY_SIZE(__entry->name));
 		__entry->lat = div_u64(lat, 1000);
 	),
 	),
 	TP_fast_assign(
-		strncpy(__entry->name, dev_name(bdi->dev), 32);
+		strlcpy(__entry->name, dev_name(bdi->dev),
+			ARRAY_SIZE(__entry->name));
 		__entry->msg	= msg;
 		__entry->step	= step;
 		__entry->window	= div_u64(window, 1000);
 	),
 	TP_fast_assign(
-		strncpy(__entry->name, dev_name(bdi->dev), 32);
+		strlcpy(__entry->name, dev_name(bdi->dev),
+			ARRAY_SIZE(__entry->name));
 		__entry->status		= status;
 		__entry->step		= step;
 		__entry->inflight	= inflight;

include/uapi/linux/blkzoned.h+14−3

 };
 /**
- * struct blk_zone_range - BLKRESETZONE ioctl request
+ * struct blk_zone_range - BLKRESETZONE/BLKOPENZONE/
- * @sector: starting sector of the first zone to issue reset write pointer
+ *                         BLKCLOSEZONE/BLKFINISHZONE ioctl
- * @nr_sectors: Total number of sectors of 1 or more zones to reset
+ *                         requests
+ * @sector: Starting sector of the first zone to operate on.
+ * @nr_sectors: Total number of sectors of all zones to operate on.
  */
 struct blk_zone_range {
 	__u64		sector;
  *                sector range. The sector range must be zone aligned.
  * @BLKGETZONESZ: Get the device zone size in number of 512 B sectors.
  * @BLKGETNRZONES: Get the total number of zones of the device.
+ * @BLKOPENZONE: Open the zones in the specified sector range.
+ *               The 512 B sector range must be zone aligned.
+ * @BLKCLOSEZONE: Close the zones in the specified sector range.
+ *                The 512 B sector range must be zone aligned.
+ * @BLKFINISHZONE: Mark the zones as full in the specified sector range.
+ *                 The 512 B sector range must be zone aligned.
  */
 #define BLKREPORTZONE	_IOWR(0x12, 130, struct blk_zone_report)
 #define BLKRESETZONE	_IOW(0x12, 131, struct blk_zone_range)
 #define BLKGETZONESZ	_IOR(0x12, 132, __u32)
 #define BLKGETNRZONES	_IOR(0x12, 133, __u32)
+#define BLKOPENZONE	_IOW(0x12, 134, struct blk_zone_range)
+#define BLKCLOSEZONE	_IOW(0x12, 135, struct blk_zone_range)
+#define BLKFINISHZONE	_IOW(0x12, 136, struct blk_zone_range)
 #endif /* _UAPI_BLKZONED_H */

include/uapi/linux/fcntl.h+8−1

  * Valid hint values for F_{GET,SET}_RW_HINT. 0 is "not set", or can be
  * used to clear any hints previously set.
  */
-#define RWF_WRITE_LIFE_NOT_SET	0
+#define RWH_WRITE_LIFE_NOT_SET	0
 #define RWH_WRITE_LIFE_NONE	1
 #define RWH_WRITE_LIFE_SHORT	2
 #define RWH_WRITE_LIFE_MEDIUM	3
 #define RWH_WRITE_LIFE_LONG	4
 #define RWH_WRITE_LIFE_EXTREME	5
+/*
+ * The originally introduced spelling is remained from the first
+ * versions of the patch set that introduced the feature, see commit
+ * v4.13-rc1~212^2~51.
+ */
+#define RWF_WRITE_LIFE_NOT_SET	RWH_WRITE_LIFE_NOT_SET
 /*
  * Types of directory notifications that may be requested.
  */

include/uapi/linux/sed-opal.h+20−0

@@ -113,6 +113,25 @@ struct opal_shadow_mbr { · 112 unmodified lines
113	__u64 size;	113	__u64 size;
114	};	114	};
115		115
		116	/* Opal table operations */
		117	enum opal_table_ops {
		118	OPAL_READ_TABLE,
		119	OPAL_WRITE_TABLE,
		120	};
		121
		122	#define OPAL_UID_LENGTH 8
		123	struct opal_read_write_table {
		124	struct opal_key key;
		125	const __u64 data;
		126	const __u8 table_uid[OPAL_UID_LENGTH];
		127	__u64 offset;
		128	__u64 size;
		129	#define OPAL_TABLE_READ (1 << OPAL_READ_TABLE)
		130	#define OPAL_TABLE_WRITE (1 << OPAL_WRITE_TABLE)
		131	__u64 flags;
		132	__u64 priv;
		133	};
		134
116	#define IOC_OPAL_SAVE _IOW('p', 220, struct opal_lock_unlock)	135	#define IOC_OPAL_SAVE _IOW('p', 220, struct opal_lock_unlock)
117	#define IOC_OPAL_LOCK_UNLOCK _IOW('p', 221, struct opal_lock_unlock)	136	#define IOC_OPAL_LOCK_UNLOCK _IOW('p', 221, struct opal_lock_unlock)
118	#define IOC_OPAL_TAKE_OWNERSHIP _IOW('p', 222, struct opal_key)	137	#define IOC_OPAL_TAKE_OWNERSHIP _IOW('p', 222, struct opal_key)
@@ -128,5 +147,6 @@ struct opal_shadow_mbr { · 9 unmodified lines
128	#define IOC_OPAL_PSID_REVERT_TPR _IOW('p', 232, struct opal_key)	147	#define IOC_OPAL_PSID_REVERT_TPR _IOW('p', 232, struct opal_key)
129	#define IOC_OPAL_MBR_DONE _IOW('p', 233, struct opal_mbr_done)	148	#define IOC_OPAL_MBR_DONE _IOW('p', 233, struct opal_mbr_done)
130	#define IOC_OPAL_WRITE_SHADOW_MBR _IOW('p', 234, struct opal_shadow_mbr)	149	#define IOC_OPAL_WRITE_SHADOW_MBR _IOW('p', 234, struct opal_shadow_mbr)
		150	#define IOC_OPAL_GENERIC_TABLE_RW _IOW('p', 235, struct opal_read_write_table)
131		151
132	#endif /* _UAPI_SED_OPAL_H */	152	#endif /* _UAPI_SED_OPAL_H */

lib/sbitmap.c+0−17

@@ -236,23 +236,6 @@ bool sbitmap_any_bit_set(const struct sbitmap *sb) · 235 unmodified lines
236	}	236	}
237	EXPORT_SYMBOL_GPL(sbitmap_any_bit_set);	237	EXPORT_SYMBOL_GPL(sbitmap_any_bit_set);
238		238
239	bool sbitmap_any_bit_clear(const struct sbitmap *sb)
240	{
241	unsigned int i;
242
243	for (i = 0; i < sb->map_nr; i++) {
244	const struct sbitmap_word *word = &sb->map[i];
245	unsigned long mask = word->word & ~word->cleared;
246	unsigned long ret;
247
248	ret = find_first_zero_bit(&mask, word->depth);
249	if (ret < word->depth)
250	return true;
251	}
252	return false;
253	}
254	EXPORT_SYMBOL_GPL(sbitmap_any_bit_clear);
255
256	static unsigned int __sbitmap_weight(const struct sbitmap *sb, bool set)	239	static unsigned int __sbitmap_weight(const struct sbitmap *sb, bool set)
257	{	240	{
258	unsigned int i, weight = 0;	241	unsigned int i, weight = 0;

tools/include/uapi/linux/fcntl.h+8−1

  * Valid hint values for F_{GET,SET}_RW_HINT. 0 is "not set", or can be
  * used to clear any hints previously set.
  */
-#define RWF_WRITE_LIFE_NOT_SET	0
+#define RWH_WRITE_LIFE_NOT_SET	0
 #define RWH_WRITE_LIFE_NONE	1
 #define RWH_WRITE_LIFE_SHORT	2
 #define RWH_WRITE_LIFE_MEDIUM	3
 #define RWH_WRITE_LIFE_LONG	4
 #define RWH_WRITE_LIFE_EXTREME	5
+/*
+ * The originally introduced spelling is remained from the first
+ * versions of the patch set that introduced the feature, see commit
+ * v4.13-rc1~212^2~51.
+ */
+#define RWF_WRITE_LIFE_NOT_SET	RWH_WRITE_LIFE_NOT_SET
 /*
  * Types of directory notifications that may be requested.
  */