Merge tag 'for-6.0/dm-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm

Pull device mapper updates from Mike Snitzer: - Refactor DM core's mempool allocation so that it clearer by not being split acorss files. - Improve DM core's BLK_STS_DM_REQUEUE and BLK_STS_AGAIN handling. - Optimize DM core's more common bio splitting by eliminating the use of bio cloning with bio_split+bio_chain. Shift that cloning cost to the relatively unlikely dm_io requeue case that only occurs during error handling. Introduces dm_io_rewind() that will clone a bio that reflects the subset of the original bio that must be requeued. - Remove DM core's dm_table_get_num_targets() wrapper and audit all dm_table_get_target() callers. - Fix potential for OOM with DM writecache target by setting a default MAX_WRITEBACK_JOBS (set to 256MiB or 1/16 of total system memory, whichever is smaller). - Fix DM writecache target's stats that are reported through DM-specific table info. - Fix use-after-free crash in dm_sm_register_threshold_callback(). - Refine DM core's Persistent Reservation handling in preparation for broader work Mike Christie is doing to add compatibility with Microsoft Windows Failover Cluster. - Fix various KASAN reported bugs in the DM raid target. - Fix DM raid target crash due to md_handle_request() bio splitting that recurses to block core without properly initializing the bio's bi_dev. - Fix some code comment typos and fix some Documentation formatting. * tag 'for-6.0/dm-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm: (29 commits) dm: fix dm-raid crash if md_handle_request() splits bio dm raid: fix address sanitizer warning in raid_resume dm raid: fix address sanitizer warning in raid_status dm: Start pr_preempt from the same starting path dm: Fix PR release handling for non All Registrants dm: Start pr_reserve from the same starting path dm: Allow dm_call_pr to be used for path searches dm: return early from dm_pr_call() if DM device is suspended dm thin: fix use-after-free crash in dm_sm_register_threshold_callback dm writecache: count number of blocks discarded, not number of discard bios dm writecache: count number of blocks written, not number of write bios dm writecache: count number of blocks read, not number of read bios dm writecache: return void from functions dm kcopyd: use __GFP_HIGHMEM when allocating pages dm writecache: set a default MAX_WRITEBACK_JOBS Documentation: dm writecache: Render status list as list Documentation: dm writecache: add blank line before optional parameters dm snapshot: fix typo in snapshot_map() comment dm raid: remove redundant "the" in parse_raid_params() comment dm cache: fix typo in 2 comment blocks ...

Linus Torvalds committed 4 years ago

Commit 8374cfe2 parents c013d0a 9dd1cd322 files · +694 −405Browse files

Unified · Split

22 files

Documentation/admin-guide/device-mapper/writecache.rst +10 −8
drivers/md/Makefile +1 −1
drivers/md/dm-cache-metadata.h +1 −1
drivers/md/dm-cache-target.c +1 −1
drivers/md/dm-core.h +22 −1
drivers/md/dm-ima.c +1 −4
drivers/md/dm-io-rewind.c +166 −0
drivers/md/dm-ioctl.c +3 −3
drivers/md/dm-kcopyd.c +1 −1
drivers/md/dm-raid.c +4 −3
drivers/md/dm-rq.c +0 −1
drivers/md/dm-snap.c +1 −1
drivers/md/dm-table.c +146 −172
drivers/md/dm-thin-metadata.c +5 −2
drivers/md/dm-thin.c +3 −1
drivers/md/dm-verity-target.c +3 −4
drivers/md/dm-writecache.c +25 −18
drivers/md/dm-zone.c +2 −5
drivers/md/dm.c +291 −171
drivers/md/dm.h +0 −4
include/linux/device-mapper.h +6 −1
include/uapi/linux/dm-ioctl.h +2 −2

Documentation/admin-guide/device-mapper/writecache.rst+10−8

@@ -20,6 +20,7 @@ Constructor parameters: · 19 unmodified lines
20	size)	20	size)
21	5. the number of optional parameters (the parameters with an argument	21	5. the number of optional parameters (the parameters with an argument
22	count as two)	22	count as two)
		23
23	start_sector n (default: 0)	24	start_sector n (default: 0)
24	offset from the start of cache device in 512-byte sectors	25	offset from the start of cache device in 512-byte sectors
25	high_watermark n (default: 50)	26	high_watermark n (default: 50)
@@ -74,20 +75,21 @@ Constructor parameters: · 48 unmodified lines
74	the origin volume in the last n milliseconds	75	the origin volume in the last n milliseconds
75		76
76	Status:	77	Status:
		78
77	1. error indicator - 0 if there was no error, otherwise error number	79	1. error indicator - 0 if there was no error, otherwise error number
78	2. the number of blocks	80	2. the number of blocks
79	3. the number of free blocks	81	3. the number of free blocks
80	4. the number of blocks under writeback	82	4. the number of blocks under writeback
81	5. the number of read requests	83	5. the number of read blocks
82	6. the number of read requests that hit the cache	84	6. the number of read blocks that hit the cache
83	7. the number of write requests	85	7. the number of write blocks
84	8. the number of write requests that hit uncommitted block	86	8. the number of write blocks that hit uncommitted block
85	9. the number of write requests that hit committed block	87	9. the number of write blocks that hit committed block
86	10. the number of write requests that bypass the cache	88	10. the number of write blocks that bypass the cache
87	11. the number of write requests that are allocated in the cache	89	11. the number of write blocks that are allocated in the cache
88	12. the number of write requests that are blocked on the freelist	90	12. the number of write requests that are blocked on the freelist
89	13. the number of flush requests	91	13. the number of flush requests
90	14. the number of discard requests	92	14. the number of discarded blocks
91		93
92	Messages:	94	Messages:
93	flush	95	flush

drivers/md/Makefile+1−1

@@ -5,7 +5,7 @@ · 4 unmodified lines
5		5
6	dm-mod-y += dm.o dm-table.o dm-target.o dm-linear.o dm-stripe.o \	6	dm-mod-y += dm.o dm-table.o dm-target.o dm-linear.o dm-stripe.o \
7	dm-ioctl.o dm-io.o dm-kcopyd.o dm-sysfs.o dm-stats.o \	7	dm-ioctl.o dm-io.o dm-kcopyd.o dm-sysfs.o dm-stats.o \
8	dm-rq.o	8	dm-rq.o dm-io-rewind.o
9	dm-multipath-y += dm-path-selector.o dm-mpath.o	9	dm-multipath-y += dm-path-selector.o dm-mpath.o
10	dm-historical-service-time-y += dm-ps-historical-service-time.o	10	dm-historical-service-time-y += dm-ps-historical-service-time.o
11	dm-io-affinity-y += dm-ps-io-affinity.o	11	dm-io-affinity-y += dm-ps-io-affinity.o

drivers/md/dm-cache-metadata.h+1−1

@@ -131,7 +131,7 @@ void dm_cache_dump(struct dm_cache_metadata *cmd); · 130 unmodified lines
131	* hints will be lost.	131	* hints will be lost.
132	*	132	*
133	* The hints are indexed by the cblock, but many policies will not	133	* The hints are indexed by the cblock, but many policies will not
134	* neccessarily have a fast way of accessing efficiently via cblock. So	134	* necessarily have a fast way of accessing efficiently via cblock. So
135	* rather than querying the policy for each cblock, we let it walk its data	135	* rather than querying the policy for each cblock, we let it walk its data
136	* structures and fill in the hints in whatever order it wishes.	136	* structures and fill in the hints in whatever order it wishes.
137	*/	137	*/

drivers/md/dm-cache-target.c+1−1

@@ -2775,7 +2775,7 @@ static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock, · 2774 unmodified lines
2775		2775
2776	/*	2776	/*
2777	* The discard block size in the on disk metadata is not	2777	* The discard block size in the on disk metadata is not
2778	* neccessarily the same as we're currently using. So we have to	2778	* necessarily the same as we're currently using. So we have to
2779	* be careful to only set the discarded attribute if we know it	2779	* be careful to only set the discarded attribute if we know it
2780	* covers a complete block of the new size.	2780	* covers a complete block of the new size.
2781	*/	2781	*/

drivers/md/dm-core.h+22−1

 #define DM_RESERVED_MAX_IOS		1024
+struct dm_io;
 struct dm_kobject_holder {
 	struct kobject kobj;
 	struct completion completion;
 	spinlock_t deferred_lock;
 	struct bio_list deferred;
+	/*
+	 * requeue work context is needed for cloning one new bio
+	 * to represent the dm_io to be requeued, since each
+	 * dm_io may point to the original bio from FS.
+	 */
+	struct work_struct requeue_work;
+	struct dm_io *requeue_list;
 	void *interface_ptr;
 	/*
 #endif
 };
+static inline struct dm_target *dm_table_get_target(struct dm_table *t,
+						    unsigned int index)
+{
+	BUG_ON(index >= t->num_targets);
+	return t->targets + index;
+}
 /*
  * One of these is allocated per clone bio.
  */
 	sector_t old_sector;
 	struct bio clone;
 };
+#define DM_TARGET_IO_BIO_OFFSET (offsetof(struct dm_target_io, clone))
+#define DM_IO_BIO_OFFSET \
+	(offsetof(struct dm_target_io, clone) + offsetof(struct dm_io, tio))
 /*
  * dm_target_io flags
 	atomic_t io_count;
 	struct mapped_device *md;
-	struct bio *split_bio;
 	/* The three fields represent mapped part of original bio */
 	struct bio *orig_bio;
 	unsigned int sector_offset; /* offset to end of orig_bio */
 	io->flags |= (1U << bit);
 }
+void dm_io_rewind(struct dm_io *io, struct bio_set *bs);
 static inline struct completion *dm_get_completion_from_kobject(struct kobject *kobj)
 {
 	return &container_of(kobj, struct dm_kobject_holder, kobj)->completion;

drivers/md/dm-ima.c+1−4

 	if (!target_data_buf)
 		goto error;
-	num_targets = dm_table_get_num_targets(table);
+	num_targets = table->num_targets;
 	if (dm_ima_alloc_and_copy_device_data(table->md, &device_data_buf, num_targets, noio))
 		goto error;
 	for (i = 0; i < num_targets; i++) {
 		struct dm_target *ti = dm_table_get_target(table, i);
-		if (!ti)
-			goto error;
 		last_target_measured = 0;
 		/*

drivers/md/dm-io-rewind.c+166−0

@@ -0,0 +1,166 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/*
	3	* Copyright 2022 Red Hat, Inc.
	4	*/
	5
	6	#include <linux/bio.h>
	7	#include <linux/blk-crypto.h>
	8	#include <linux/blk-integrity.h>
	9
	10	#include "dm-core.h"
	11
	12	static inline bool dm_bvec_iter_rewind(const struct bio_vec *bv,
	13	struct bvec_iter *iter,
	14	unsigned int bytes)
	15	{
	16	int idx;
	17
	18	iter->bi_size += bytes;
	19	if (bytes <= iter->bi_bvec_done) {
	20	iter->bi_bvec_done -= bytes;
	21	return true;
	22	}
	23
	24	bytes -= iter->bi_bvec_done;
	25	idx = iter->bi_idx - 1;
	26
	27	while (idx >= 0 && bytes && bytes > bv[idx].bv_len) {
	28	bytes -= bv[idx].bv_len;
	29	idx--;
	30	}
	31
	32	if (WARN_ONCE(idx < 0 && bytes,
	33	"Attempted to rewind iter beyond bvec's boundaries\n")) {
	34	iter->bi_size -= bytes;
	35	iter->bi_bvec_done = 0;
	36	iter->bi_idx = 0;
	37	return false;
	38	}
	39
	40	iter->bi_idx = idx;
	41	iter->bi_bvec_done = bv[idx].bv_len - bytes;
	42	return true;
	43	}
	44
	45	#if defined(CONFIG_BLK_DEV_INTEGRITY)
	46
	47	/**
	48	* dm_bio_integrity_rewind - Rewind integrity vector
	49	* @bio: bio whose integrity vector to update
	50	* @bytes_done: number of data bytes to rewind
	51	*
	52	* Description: This function calculates how many integrity bytes the
	53	* number of completed data bytes correspond to and rewind the
	54	* integrity vector accordingly.
	55	*/
	56	static void dm_bio_integrity_rewind(struct bio *bio, unsigned int bytes_done)
	57	{
	58	struct bio_integrity_payload *bip = bio_integrity(bio);
	59	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
	60	unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9);
	61
	62	bip->bip_iter.bi_sector -= bio_integrity_intervals(bi, bytes_done >> 9);
	63	dm_bvec_iter_rewind(bip->bip_vec, &bip->bip_iter, bytes);
	64	}
	65
	66	#else /* CONFIG_BLK_DEV_INTEGRITY */
	67
	68	static inline void dm_bio_integrity_rewind(struct bio *bio,
	69	unsigned int bytes_done)
	70	{
	71	return;
	72	}
	73
	74	#endif
	75
	76	#if defined(CONFIG_BLK_INLINE_ENCRYPTION)
	77
	78	/* Decrements @dun by @dec, treating @dun as a multi-limb integer. */
	79	static void dm_bio_crypt_dun_decrement(u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE],
	80	unsigned int dec)
	81	{
	82	int i;
	83
	84	for (i = 0; dec && i < BLK_CRYPTO_DUN_ARRAY_SIZE; i++) {
	85	u64 prev = dun[i];
	86
	87	dun[i] -= dec;
	88	if (dun[i] > prev)
	89	dec = 1;
	90	else
	91	dec = 0;
	92	}
	93	}
	94
	95	static void dm_bio_crypt_rewind(struct bio *bio, unsigned int bytes)
	96	{
	97	struct bio_crypt_ctx *bc = bio->bi_crypt_context;
	98
	99	dm_bio_crypt_dun_decrement(bc->bc_dun,
	100	bytes >> bc->bc_key->data_unit_size_bits);
	101	}
	102
	103	#else /* CONFIG_BLK_INLINE_ENCRYPTION */
	104
	105	static inline void dm_bio_crypt_rewind(struct bio *bio, unsigned int bytes)
	106	{
	107	return;
	108	}
	109
	110	#endif
	111
	112	static inline void dm_bio_rewind_iter(const struct bio *bio,
	113	struct bvec_iter *iter, unsigned int bytes)
	114	{
	115	iter->bi_sector -= bytes >> 9;
	116
	117	/* No advance means no rewind */
	118	if (bio_no_advance_iter(bio))
	119	iter->bi_size += bytes;
	120	else
	121	dm_bvec_iter_rewind(bio->bi_io_vec, iter, bytes);
	122	}
	123
	124	/**
	125	* dm_bio_rewind - update ->bi_iter of @bio by rewinding @bytes.
	126	* @bio: bio to rewind
	127	* @bytes: how many bytes to rewind
	128	*
	129	* WARNING:
	130	* Caller must ensure that @bio has a fixed end sector, to allow
	131	* rewinding from end of bio and restoring its original position.
	132	* Caller is also responsibile for restoring bio's size.
	133	*/
	134	static void dm_bio_rewind(struct bio *bio, unsigned bytes)
	135	{
	136	if (bio_integrity(bio))
	137	dm_bio_integrity_rewind(bio, bytes);
	138
	139	if (bio_has_crypt_ctx(bio))
	140	dm_bio_crypt_rewind(bio, bytes);
	141
	142	dm_bio_rewind_iter(bio, &bio->bi_iter, bytes);
	143	}
	144
	145	void dm_io_rewind(struct dm_io io, struct bio_set bs)
	146	{
	147	struct bio *orig = io->orig_bio;
	148	struct bio *new_orig = bio_alloc_clone(orig->bi_bdev, orig,
	149	GFP_NOIO, bs);
	150	/*
	151	* dm_bio_rewind can restore to previous position since the
	152	* end sector is fixed for original bio, but we still need
	153	* to restore bio's size manually (using io->sectors).
	154	*/
	155	dm_bio_rewind(new_orig, ((io->sector_offset << 9) -
	156	orig->bi_iter.bi_size));
	157	bio_trim(new_orig, 0, io->sectors);
	158
	159	bio_chain(new_orig, orig);
	160	/*
	161	* __bi_remaining was increased (by dm_split_and_process_bio),
	162	* so must drop the one added in bio_chain.
	163	*/
	164	atomic_dec(&orig->__bi_remaining);
	165	io->orig_bio = new_orig;
	166	}

drivers/md/dm-ioctl.c+3−3

@@ -832,7 +832,7 @@ static void __dev_status(struct mapped_device md, struct dm_ioctl param) · 831 unmodified lines
832	if (!(param->flags & DM_QUERY_INACTIVE_TABLE_FLAG)) {	832	if (!(param->flags & DM_QUERY_INACTIVE_TABLE_FLAG)) {
833	if (get_disk_ro(disk))	833	if (get_disk_ro(disk))
834	param->flags \|= DM_READONLY_FLAG;	834	param->flags \|= DM_READONLY_FLAG;
835	param->target_count = dm_table_get_num_targets(table);	835	param->target_count = table->num_targets;
836	}	836	}
837		837
838	param->flags \|= DM_ACTIVE_PRESENT_FLAG;	838	param->flags \|= DM_ACTIVE_PRESENT_FLAG;
@@ -845,7 +845,7 @@ static void __dev_status(struct mapped_device md, struct dm_ioctl param) · 6 unmodified lines
845	if (table) {	845	if (table) {
846	if (!(dm_table_get_mode(table) & FMODE_WRITE))	846	if (!(dm_table_get_mode(table) & FMODE_WRITE))
847	param->flags \|= DM_READONLY_FLAG;	847	param->flags \|= DM_READONLY_FLAG;
848	param->target_count = dm_table_get_num_targets(table);	848	param->target_count = table->num_targets;
849	}	849	}
850	dm_put_live_table(md, srcu_idx);	850	dm_put_live_table(md, srcu_idx);
851	}	851	}
@@ -1248,7 +1248,7 @@ static void retrieve_status(struct dm_table *table, · 396 unmodified lines
1248	type = STATUSTYPE_INFO;	1248	type = STATUSTYPE_INFO;
1249		1249
1250	/* Get all the target info */	1250	/* Get all the target info */
1251	num_targets = dm_table_get_num_targets(table);	1251	num_targets = table->num_targets;
1252	for (i = 0; i < num_targets; i++) {	1252	for (i = 0; i < num_targets; i++) {
1253	struct dm_target *ti = dm_table_get_target(table, i);	1253	struct dm_target *ti = dm_table_get_target(table, i);
1254	size_t l;	1254	size_t l;

drivers/md/dm-kcopyd.c+1−1

@@ -219,7 +219,7 @@ static struct page_list *alloc_pl(gfp_t gfp) · 218 unmodified lines
219	if (!pl)	219	if (!pl)
220	return NULL;	220	return NULL;
221		221
222	pl->page = alloc_page(gfp);	222	pl->page = alloc_page(gfp \| __GFP_HIGHMEM);
223	if (!pl->page) {	223	if (!pl->page) {
224	kfree(pl);	224	kfree(pl);
225	return NULL;	225	return NULL;

drivers/md/dm-raid.c+4−3

 			}
 			rs->md.bitmap_info.daemon_sleep = value;
 		} else if (!strcasecmp(key, dm_raid_arg_name_by_flag(CTR_FLAG_DATA_OFFSET))) {
-			/* Userspace passes new data_offset after having extended the the data image LV */
+			/* Userspace passes new data_offset after having extended the data image LV */
 			if (test_and_set_bit(__CTR_FLAG_DATA_OFFSET, &rs->ctr_flags)) {
 				rs->ti->error = "Only one data_offset argument pair allowed";
 				return -EINVAL;
 	INIT_WORK(&rs->md.event_work, do_table_event);
 	ti->private = rs;
 	ti->num_flush_bios = 1;
+	ti->needs_bio_set_dev = true;
 	/* Restore any requested new layout for conversion decision */
 	rs_config_restore(rs, &rs_layout);
 {
 	struct raid_set *rs = ti->private;
 	struct mddev *mddev = &rs->md;
-	struct r5conf *conf = mddev->private;
+	struct r5conf *conf = rs_is_raid456(rs) ? mddev->private : NULL;
 	int i, max_nr_stripes = conf ? conf->max_nr_stripes : 0;
 	unsigned long recovery;
 	unsigned int raid_param_cnt = 1; /* at least 1 for chunksize */
 	memset(cleared_failed_devices, 0, sizeof(cleared_failed_devices));
-	for (i = 0; i < mddev->raid_disks; i++) {
+	for (i = 0; i < rs->raid_disks; i++) {
 		r = &rs->dev[i].rdev;
 		/* HM FIXME: enhance journal device recovery processing */
 		if (test_bit(Journal, &r->flags))

drivers/md/dm-rq.c+0−1

@@ -43,7 +43,6 @@ unsigned dm_get_reserved_rq_based_ios(void) · 42 unmodified lines
43	return __dm_get_module_param(&reserved_rq_based_ios,	43	return __dm_get_module_param(&reserved_rq_based_ios,
44	RESERVED_REQUEST_BASED_IOS, DM_RESERVED_MAX_IOS);	44	RESERVED_REQUEST_BASED_IOS, DM_RESERVED_MAX_IOS);
45	}	45	}
46	EXPORT_SYMBOL_GPL(dm_get_reserved_rq_based_ios);
47		46
48	static unsigned dm_get_blk_mq_nr_hw_queues(void)	47	static unsigned dm_get_blk_mq_nr_hw_queues(void)
49	{	48	{

drivers/md/dm-snap.c+1−1

@@ -2026,7 +2026,7 @@ static int snapshot_map(struct dm_target ti, struct bio bio) · 2025 unmodified lines
2026	/*	2026	/*
2027	* Write to snapshot - higher level takes care of RW/RO	2027	* Write to snapshot - higher level takes care of RW/RO
2028	* flags so we should only get this if we are	2028	* flags so we should only get this if we are
2029	* writeable.	2029	* writable.
2030	*/	2030	*/
2031	if (bio_data_dir(bio) == WRITE) {	2031	if (bio_data_dir(bio) == WRITE) {
2032	pe = __lookup_pending_exception(s, chunk);	2032	pe = __lookup_pending_exception(s, chunk);

drivers/md/dm-table.c+146−172

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	*/	6	*/
7		7
8	#include "dm-core.h"	8	#include "dm-core.h"
		9	#include "dm-rq.h"
9		10
10	#include <linux/module.h>	11	#include <linux/module.h>
11	#include <linux/vmalloc.h>	12	#include <linux/vmalloc.h>
@@ -174,8 +175,6 @@ static void dm_table_destroy_crypto_profile(struct dm_table *t); · 162 unmodified lines
174		175
175	void dm_table_destroy(struct dm_table *t)	176	void dm_table_destroy(struct dm_table *t)
176	{	177	{
177	unsigned int i;
178
179	if (!t)	178	if (!t)
180	return;	179	return;
181		180
@@ -184,13 +183,13 @@ void dm_table_destroy(struct dm_table *t) · 2 unmodified lines
184	kvfree(t->index[t->depth - 2]);	183	kvfree(t->index[t->depth - 2]);
185		184
186	/* free the targets */	185	/* free the targets */
187	for (i = 0; i < t->num_targets; i++) {	186	for (unsigned int i = 0; i < t->num_targets; i++) {
188	struct dm_target *tgt = t->targets + i;	187	struct dm_target *ti = dm_table_get_target(t, i);
189		188
190	if (tgt->type->dtr)	189	if (ti->type->dtr)
191	tgt->type->dtr(tgt);	190	ti->type->dtr(ti);
192		191
193	dm_put_target_type(tgt->type);	192	dm_put_target_type(ti->type);
194	}	193	}
195		194
196	kvfree(t->highs);	195	kvfree(t->highs);
@@ -450,14 +449,14 @@ EXPORT_SYMBOL(dm_put_device); · 253 unmodified lines
450	/*	449	/*
451	* Checks to see if the target joins onto the end of the table.	450	* Checks to see if the target joins onto the end of the table.
452	*/	451	*/
453	static int adjoin(struct dm_table table, struct dm_target ti)	452	static int adjoin(struct dm_table t, struct dm_target ti)
454	{	453	{
455	struct dm_target *prev;	454	struct dm_target *prev;
456		455
457	if (!table->num_targets)	456	if (!t->num_targets)
458	return !ti->begin;	457	return !ti->begin;
459		458
460	prev = &table->targets[table->num_targets - 1];	459	prev = &t->targets[t->num_targets - 1];
461	return (ti->begin == (prev->begin + prev->len));	460	return (ti->begin == (prev->begin + prev->len));
462	}	461	}
463		462
@@ -564,8 +563,8 @@ int dm_split_args(int argc, char *argvp, char input) · 100 unmodified lines
564	* two or more targets, the size of each piece it gets split into must	563	* two or more targets, the size of each piece it gets split into must
565	* be compatible with the logical_block_size of the target processing it.	564	* be compatible with the logical_block_size of the target processing it.
566	*/	565	*/
567	static int validate_hardware_logical_block_alignment(struct dm_table *table,	566	static int validate_hardware_logical_block_alignment(struct dm_table *t,
568	struct queue_limits *limits)	567	struct queue_limits *limits)
569	{	568	{
570	/*	569	/*
571	* This function uses arithmetic modulo the logical_block_size	570	* This function uses arithmetic modulo the logical_block_size
@@ -587,13 +586,13 @@ static int validate_hardware_logical_block_alignment(struct dm_table *table, · 15 unmodified lines
587		586
588	struct dm_target *ti;	587	struct dm_target *ti;
589	struct queue_limits ti_limits;	588	struct queue_limits ti_limits;
590	unsigned i;	589	unsigned int i;
591		590
592	/*	591	/*
593	* Check each entry in the table in turn.	592	* Check each entry in the table in turn.
594	*/	593	*/
595	for (i = 0; i < dm_table_get_num_targets(table); i++) {	594	for (i = 0; i < t->num_targets; i++) {
596	ti = dm_table_get_target(table, i);	595	ti = dm_table_get_target(t, i);
597		596
598	blk_set_stacking_limits(&ti_limits);	597	blk_set_stacking_limits(&ti_limits);
599		598
@@ -621,7 +620,7 @@ static int validate_hardware_logical_block_alignment(struct dm_table *table, · 21 unmodified lines
621	if (remaining) {	620	if (remaining) {
622	DMWARN("%s: table line %u (start sect %llu len %llu) "	621	DMWARN("%s: table line %u (start sect %llu len %llu) "
623	"not aligned to h/w logical block size %u",	622	"not aligned to h/w logical block size %u",
624	dm_device_name(table->md), i,	623	dm_device_name(t->md), i,
625	(unsigned long long) ti->begin,	624	(unsigned long long) ti->begin,
626	(unsigned long long) ti->len,	625	(unsigned long long) ti->len,
627	limits->logical_block_size);	626	limits->logical_block_size);
@@ -636,7 +635,7 @@ int dm_table_add_target(struct dm_table t, const char type, · 8 unmodified lines
636	{	635	{
637	int r = -EINVAL, argc;	636	int r = -EINVAL, argc;
638	char **argv;	637	char **argv;
639	struct dm_target *tgt;	638	struct dm_target *ti;
640		639
641	if (t->singleton) {	640	if (t->singleton) {
642	DMERR("%s: target type %s must appear alone in table",	641	DMERR("%s: target type %s must appear alone in table",
@@ -646,87 +645,87 @@ int dm_table_add_target(struct dm_table t, const char type, · 3 unmodified lines
646		645
647	BUG_ON(t->num_targets >= t->num_allocated);	646	BUG_ON(t->num_targets >= t->num_allocated);
648		647
649	tgt = t->targets + t->num_targets;	648	ti = t->targets + t->num_targets;
650	memset(tgt, 0, sizeof(*tgt));	649	memset(ti, 0, sizeof(*ti));
651		650
652	if (!len) {	651	if (!len) {
653	DMERR("%s: zero-length target", dm_device_name(t->md));	652	DMERR("%s: zero-length target", dm_device_name(t->md));
654	return -EINVAL;	653	return -EINVAL;
655	}	654	}
656		655
657	tgt->type = dm_get_target_type(type);	656	ti->type = dm_get_target_type(type);
658	if (!tgt->type) {	657	if (!ti->type) {
659	DMERR("%s: %s: unknown target type", dm_device_name(t->md), type);	658	DMERR("%s: %s: unknown target type", dm_device_name(t->md), type);
660	return -EINVAL;	659	return -EINVAL;
661	}	660	}
662		661
663	if (dm_target_needs_singleton(tgt->type)) {	662	if (dm_target_needs_singleton(ti->type)) {
664	if (t->num_targets) {	663	if (t->num_targets) {
665	tgt->error = "singleton target type must appear alone in table";	664	ti->error = "singleton target type must appear alone in table";
666	goto bad;	665	goto bad;
667	}	666	}
668	t->singleton = true;	667	t->singleton = true;
669	}	668	}
670		669
671	if (dm_target_always_writeable(tgt->type) && !(t->mode & FMODE_WRITE)) {	670	if (dm_target_always_writeable(ti->type) && !(t->mode & FMODE_WRITE)) {
672	tgt->error = "target type may not be included in a read-only table";	671	ti->error = "target type may not be included in a read-only table";
673	goto bad;	672	goto bad;
674	}	673	}
675		674
676	if (t->immutable_target_type) {	675	if (t->immutable_target_type) {
677	if (t->immutable_target_type != tgt->type) {	676	if (t->immutable_target_type != ti->type) {
678	tgt->error = "immutable target type cannot be mixed with other target types";	677	ti->error = "immutable target type cannot be mixed with other target types";
679	goto bad;	678	goto bad;
680	}	679	}
681	} else if (dm_target_is_immutable(tgt->type)) {	680	} else if (dm_target_is_immutable(ti->type)) {
682	if (t->num_targets) {	681	if (t->num_targets) {
683	tgt->error = "immutable target type cannot be mixed with other target types";	682	ti->error = "immutable target type cannot be mixed with other target types";
684	goto bad;	683	goto bad;
685	}	684	}
686	t->immutable_target_type = tgt->type;	685	t->immutable_target_type = ti->type;
687	}	686	}
688		687
689	if (dm_target_has_integrity(tgt->type))	688	if (dm_target_has_integrity(ti->type))
690	t->integrity_added = 1;	689	t->integrity_added = 1;
691		690
692	tgt->table = t;	691	ti->table = t;
693	tgt->begin = start;	692	ti->begin = start;
694	tgt->len = len;	693	ti->len = len;
695	tgt->error = "Unknown error";	694	ti->error = "Unknown error";
696		695
697	/*	696	/*
698	* Does this target adjoin the previous one ?	697	* Does this target adjoin the previous one ?
699	*/	698	*/
700	if (!adjoin(t, tgt)) {	699	if (!adjoin(t, ti)) {
701	tgt->error = "Gap in table";	700	ti->error = "Gap in table";
702	goto bad;	701	goto bad;
703	}	702	}
704		703
705	r = dm_split_args(&argc, &argv, params);	704	r = dm_split_args(&argc, &argv, params);
706	if (r) {	705	if (r) {
707	tgt->error = "couldn't split parameters";	706	ti->error = "couldn't split parameters";
708	goto bad;	707	goto bad;
709	}	708	}
710		709
711	r = tgt->type->ctr(tgt, argc, argv);	710	r = ti->type->ctr(ti, argc, argv);
712	kfree(argv);	711	kfree(argv);
713	if (r)	712	if (r)
714	goto bad;	713	goto bad;
715		714
716	t->highs[t->num_targets++] = tgt->begin + tgt->len - 1;	715	t->highs[t->num_targets++] = ti->begin + ti->len - 1;
717		716
718	if (!tgt->num_discard_bios && tgt->discards_supported)	717	if (!ti->num_discard_bios && ti->discards_supported)
719	DMWARN("%s: %s: ignoring discards_supported because num_discard_bios is zero.",	718	DMWARN("%s: %s: ignoring discards_supported because num_discard_bios is zero.",
720	dm_device_name(t->md), type);	719	dm_device_name(t->md), type);
721		720
722	if (tgt->limit_swap_bios && !static_key_enabled(&swap_bios_enabled.key))	721	if (ti->limit_swap_bios && !static_key_enabled(&swap_bios_enabled.key))
723	static_branch_enable(&swap_bios_enabled);	722	static_branch_enable(&swap_bios_enabled);
724		723
725	return 0;	724	return 0;
726		725
727	bad:	726	bad:
728	DMERR("%s: %s: %s (%pe)", dm_device_name(t->md), type, tgt->error, ERR_PTR(r));	727	DMERR("%s: %s: %s (%pe)", dm_device_name(t->md), type, ti->error, ERR_PTR(r));
729	dm_put_target_type(tgt->type);	728	dm_put_target_type(ti->type);
730	return r;	729	return r;
731	}	730	}
732		731
@@ -825,14 +824,11 @@ static int device_not_dax_synchronous_capable(struct dm_target *ti, struct dm_de · 92 unmodified lines
825	}	824	}
826		825
827	static bool dm_table_supports_dax(struct dm_table *t,	826	static bool dm_table_supports_dax(struct dm_table *t,
828	iterate_devices_callout_fn iterate_fn)	827	iterate_devices_callout_fn iterate_fn)
829	{	828	{
830	struct dm_target *ti;
831	unsigned i;
832
833	/* Ensure that all targets support DAX. */	829	/* Ensure that all targets support DAX. */
834	for (i = 0; i < dm_table_get_num_targets(t); i++) {	830	for (unsigned int i = 0; i < t->num_targets; i++) {
835	ti = dm_table_get_target(t, i);	831	struct dm_target *ti = dm_table_get_target(t, i);
836		832
837	if (!ti->type->direct_access)	833	if (!ti->type->direct_access)
838	return false;	834	return false;
@@ -860,9 +856,8 @@ static int device_is_rq_stackable(struct dm_target ti, struct dm_dev dev, · 21 unmodified lines
860		856
861	static int dm_table_determine_type(struct dm_table *t)	857	static int dm_table_determine_type(struct dm_table *t)
862	{	858	{
863	unsigned i;
864	unsigned bio_based = 0, request_based = 0, hybrid = 0;	859	unsigned bio_based = 0, request_based = 0, hybrid = 0;
865	struct dm_target *tgt;	860	struct dm_target *ti;
866	struct list_head *devices = dm_table_get_devices(t);	861	struct list_head *devices = dm_table_get_devices(t);
867	enum dm_queue_mode live_md_type = dm_get_md_type(t->md);	862	enum dm_queue_mode live_md_type = dm_get_md_type(t->md);
868		863
@@ -876,11 +871,11 @@ static int dm_table_determine_type(struct dm_table *t) · 7 unmodified lines
876	goto verify_rq_based;	871	goto verify_rq_based;
877	}	872	}
878		873
879	for (i = 0; i < t->num_targets; i++) {	874	for (unsigned int i = 0; i < t->num_targets; i++) {
880	tgt = t->targets + i;	875	ti = dm_table_get_target(t, i);
881	if (dm_target_hybrid(tgt))	876	if (dm_target_hybrid(ti))
882	hybrid = 1;	877	hybrid = 1;
883	else if (dm_target_request_based(tgt))	878	else if (dm_target_request_based(ti))
884	request_based = 1;	879	request_based = 1;
885	else	880	else
886	bio_based = 1;	881	bio_based = 1;
@@ -942,18 +937,18 @@ verify_rq_based: · 55 unmodified lines
942	return 0;	937	return 0;
943	}	938	}
944		939
945	tgt = dm_table_get_immutable_target(t);	940	ti = dm_table_get_immutable_target(t);
946	if (!tgt) {	941	if (!ti) {
947	DMERR("table load rejected: immutable target is required");	942	DMERR("table load rejected: immutable target is required");
948	return -EINVAL;	943	return -EINVAL;
949	} else if (tgt->max_io_len) {	944	} else if (ti->max_io_len) {
950	DMERR("table load rejected: immutable target that splits IO is not supported");	945	DMERR("table load rejected: immutable target that splits IO is not supported");
951	return -EINVAL;	946	return -EINVAL;
952	}	947	}
953		948
954	/* Non-request-stackable devices can't be used for request-based dm */	949	/* Non-request-stackable devices can't be used for request-based dm */
955	if (!tgt->type->iterate_devices \|\|	950	if (!ti->type->iterate_devices \|\|
956	!tgt->type->iterate_devices(tgt, device_is_rq_stackable, NULL)) {	951	!ti->type->iterate_devices(ti, device_is_rq_stackable, NULL)) {
957	DMERR("table load rejected: including non-request-stackable devices");	952	DMERR("table load rejected: including non-request-stackable devices");
958	return -EINVAL;	953	return -EINVAL;
959	}	954	}
@@ -983,11 +978,9 @@ struct dm_target dm_table_get_immutable_target(struct dm_table t) · 23 unmodified lines
983		978
984	struct dm_target dm_table_get_wildcard_target(struct dm_table t)	979	struct dm_target dm_table_get_wildcard_target(struct dm_table t)
985	{	980	{
986	struct dm_target *ti;	981	for (unsigned int i = 0; i < t->num_targets; i++) {
987	unsigned i;	982	struct dm_target *ti = dm_table_get_target(t, i);
988		983
989	for (i = 0; i < dm_table_get_num_targets(t); i++) {
990	ti = dm_table_get_target(t, i);
991	if (dm_target_is_wildcard(ti->type))	984	if (dm_target_is_wildcard(ti->type))
992	return ti;	985	return ti;
993	}	986	}
@@ -1010,32 +1003,56 @@ static bool dm_table_supports_poll(struct dm_table *t); · 16 unmodified lines
1010	static int dm_table_alloc_md_mempools(struct dm_table t, struct mapped_device md)	1003	static int dm_table_alloc_md_mempools(struct dm_table t, struct mapped_device md)
1011	{	1004	{
1012	enum dm_queue_mode type = dm_table_get_type(t);	1005	enum dm_queue_mode type = dm_table_get_type(t);
1013	unsigned per_io_data_size = 0;	1006	unsigned int per_io_data_size = 0, front_pad, io_front_pad;
1014	unsigned min_pool_size = 0;	1007	unsigned int min_pool_size = 0, pool_size;
1015	struct dm_target *ti;	1008	struct dm_md_mempools *pools;
1016	unsigned i;
1017	bool poll_supported = false;
1018		1009
1019	if (unlikely(type == DM_TYPE_NONE)) {	1010	if (unlikely(type == DM_TYPE_NONE)) {
1020	DMWARN("no table type is set, can't allocate mempools");	1011	DMWARN("no table type is set, can't allocate mempools");
1021	return -EINVAL;	1012	return -EINVAL;
1022	}	1013	}
1023		1014
1024	if (__table_type_bio_based(type)) {	1015	pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
1025	for (i = 0; i < t->num_targets; i++) {	1016	if (!pools)
1026	ti = t->targets + i;	1017	return -ENOMEM;
1027	per_io_data_size = max(per_io_data_size, ti->per_io_data_size);	1018
1028	min_pool_size = max(min_pool_size, ti->num_flush_bios);	1019	if (type == DM_TYPE_REQUEST_BASED) {
1029	}	1020	pool_size = dm_get_reserved_rq_based_ios();
1030	poll_supported = dm_table_supports_poll(t);	1021	front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
		1022	goto init_bs;
1031	}	1023	}
1032		1024
1033	t->mempools = dm_alloc_md_mempools(md, type, per_io_data_size, min_pool_size,	1025	for (unsigned int i = 0; i < t->num_targets; i++) {
1034	t->integrity_supported, poll_supported);	1026	struct dm_target *ti = dm_table_get_target(t, i);
1035	if (!t->mempools)
1036	return -ENOMEM;
1037		1027
		1028	per_io_data_size = max(per_io_data_size, ti->per_io_data_size);
		1029	min_pool_size = max(min_pool_size, ti->num_flush_bios);
		1030	}
		1031	pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
		1032	front_pad = roundup(per_io_data_size,
		1033	__alignof__(struct dm_target_io)) + DM_TARGET_IO_BIO_OFFSET;
		1034
		1035	io_front_pad = roundup(per_io_data_size,
		1036	__alignof__(struct dm_io)) + DM_IO_BIO_OFFSET;
		1037	if (bioset_init(&pools->io_bs, pool_size, io_front_pad,
		1038	dm_table_supports_poll(t) ? BIOSET_PERCPU_CACHE : 0))
		1039	goto out_free_pools;
		1040	if (t->integrity_supported &&
		1041	bioset_integrity_create(&pools->io_bs, pool_size))
		1042	goto out_free_pools;
		1043	init_bs:
		1044	if (bioset_init(&pools->bs, pool_size, front_pad, 0))
		1045	goto out_free_pools;
		1046	if (t->integrity_supported &&
		1047	bioset_integrity_create(&pools->bs, pool_size))
		1048	goto out_free_pools;
		1049
		1050	t->mempools = pools;
1038	return 0;	1051	return 0;
		1052
		1053	out_free_pools:
		1054	dm_free_md_mempools(pools);
		1055	return -ENOMEM;
1039	}	1056	}
1040		1057
1041	static int setup_indexes(struct dm_table *t)	1058	static int setup_indexes(struct dm_table *t)
@@ -1100,10 +1117,10 @@ static struct gendisk * dm_table_get_integrity_disk(struct dm_table *t) · 58 unmodified lines
1100	struct list_head *devices = dm_table_get_devices(t);	1117	struct list_head *devices = dm_table_get_devices(t);
1101	struct dm_dev_internal *dd = NULL;	1118	struct dm_dev_internal *dd = NULL;
1102	struct gendisk prev_disk = NULL, template_disk = NULL;	1119	struct gendisk prev_disk = NULL, template_disk = NULL;
1103	unsigned i;
1104		1120
1105	for (i = 0; i < dm_table_get_num_targets(t); i++) {	1121	for (unsigned int i = 0; i < t->num_targets; i++) {
1106	struct dm_target *ti = dm_table_get_target(t, i);	1122	struct dm_target *ti = dm_table_get_target(t, i);
		1123
1107	if (!dm_target_passes_integrity(ti->type))	1124	if (!dm_target_passes_integrity(ti->type))
1108	goto no_integrity;	1125	goto no_integrity;
1109	}	1126	}
@@ -1217,18 +1234,19 @@ static int dm_keyslot_evict(struct blk_crypto_profile *profile, · 107 unmodified lines
1217	struct dm_keyslot_evict_args args = { key };	1234	struct dm_keyslot_evict_args args = { key };
1218	struct dm_table *t;	1235	struct dm_table *t;
1219	int srcu_idx;	1236	int srcu_idx;
1220	int i;
1221	struct dm_target *ti;
1222		1237
1223	t = dm_get_live_table(md, &srcu_idx);	1238	t = dm_get_live_table(md, &srcu_idx);
1224	if (!t)	1239	if (!t)
1225	return 0;	1240	return 0;
1226	for (i = 0; i < dm_table_get_num_targets(t); i++) {	1241
1227	ti = dm_table_get_target(t, i);	1242	for (unsigned int i = 0; i < t->num_targets; i++) {
		1243	struct dm_target *ti = dm_table_get_target(t, i);
		1244
1228	if (!ti->type->iterate_devices)	1245	if (!ti->type->iterate_devices)
1229	continue;	1246	continue;
1230	ti->type->iterate_devices(ti, dm_keyslot_evict_callback, &args);	1247	ti->type->iterate_devices(ti, dm_keyslot_evict_callback, &args);
1231	}	1248	}
		1249
1232	dm_put_live_table(md, srcu_idx);	1250	dm_put_live_table(md, srcu_idx);
1233	return args.err;	1251	return args.err;
1234	}	1252	}
@@ -1277,7 +1295,6 @@ static int dm_table_construct_crypto_profile(struct dm_table *t) · 42 unmodified lines
1277	{	1295	{
1278	struct dm_crypto_profile *dmcp;	1296	struct dm_crypto_profile *dmcp;
1279	struct blk_crypto_profile *profile;	1297	struct blk_crypto_profile *profile;
1280	struct dm_target *ti;
1281	unsigned int i;	1298	unsigned int i;
1282	bool empty_profile = true;	1299	bool empty_profile = true;
1283		1300
@@ -1293,8 +1310,8 @@ static int dm_table_construct_crypto_profile(struct dm_table *t) · 9 unmodified lines
1293	memset(profile->modes_supported, 0xFF,	1310	memset(profile->modes_supported, 0xFF,
1294	sizeof(profile->modes_supported));	1311	sizeof(profile->modes_supported));
1295		1312
1296	for (i = 0; i < dm_table_get_num_targets(t); i++) {	1313	for (i = 0; i < t->num_targets; i++) {
1297	ti = dm_table_get_target(t, i);	1314	struct dm_target *ti = dm_table_get_target(t, i);
1298		1315
1299	if (!dm_target_passes_crypto(ti->type)) {	1316	if (!dm_target_passes_crypto(ti->type)) {
1300	blk_crypto_intersect_capabilities(profile, NULL);	1317	blk_crypto_intersect_capabilities(profile, NULL);
@@ -1444,14 +1461,6 @@ inline sector_t dm_table_get_size(struct dm_table *t) · 143 unmodified lines
1444	}	1461	}
1445	EXPORT_SYMBOL(dm_table_get_size);	1462	EXPORT_SYMBOL(dm_table_get_size);
1446		1463
1447	struct dm_target dm_table_get_target(struct dm_table t, unsigned int index)
1448	{
1449	if (index >= t->num_targets)
1450	return NULL;
1451
1452	return t->targets + index;
1453	}
1454
1455	/*	1464	/*
1456	* Search the btree for the correct target.	1465	* Search the btree for the correct target.
1457	*	1466	*
@@ -1512,11 +1521,8 @@ static int device_not_poll_capable(struct dm_target ti, struct dm_dev dev, · 54 unmodified lines
1512	static bool dm_table_any_dev_attr(struct dm_table *t,	1521	static bool dm_table_any_dev_attr(struct dm_table *t,
1513	iterate_devices_callout_fn func, void *data)	1522	iterate_devices_callout_fn func, void *data)
1514	{	1523	{
1515	struct dm_target *ti;	1524	for (unsigned int i = 0; i < t->num_targets; i++) {
1516	unsigned int i;	1525	struct dm_target *ti = dm_table_get_target(t, i);
1517
1518	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1519	ti = dm_table_get_target(t, i);
1520		1526
1521	if (ti->type->iterate_devices &&	1527	if (ti->type->iterate_devices &&
1522	ti->type->iterate_devices(ti, func, data))	1528	ti->type->iterate_devices(ti, func, data))
@@ -1538,11 +1544,8 @@ static int count_device(struct dm_target ti, struct dm_dev dev, · 15 unmodified lines
1538		1544
1539	static bool dm_table_supports_poll(struct dm_table *t)	1545	static bool dm_table_supports_poll(struct dm_table *t)
1540	{	1546	{
1541	struct dm_target *ti;	1547	for (unsigned int i = 0; i < t->num_targets; i++) {
1542	unsigned i = 0;	1548	struct dm_target *ti = dm_table_get_target(t, i);
1543
1544	while (i < dm_table_get_num_targets(t)) {
1545	ti = dm_table_get_target(t, i++);
1546		1549
1547	if (!ti->type->iterate_devices \|\|	1550	if (!ti->type->iterate_devices \|\|
1548	ti->type->iterate_devices(ti, device_not_poll_capable, NULL))	1551	ti->type->iterate_devices(ti, device_not_poll_capable, NULL))
@@ -1558,18 +1561,15 @@ static bool dm_table_supports_poll(struct dm_table *t) · 9 unmodified lines
1558	* Returns false if the result is unknown because a target doesn't	1561	* Returns false if the result is unknown because a target doesn't
1559	* support iterate_devices.	1562	* support iterate_devices.
1560	*/	1563	*/
1561	bool dm_table_has_no_data_devices(struct dm_table *table)	1564	bool dm_table_has_no_data_devices(struct dm_table *t)
1562	{	1565	{
1563	struct dm_target *ti;	1566	for (unsigned int i = 0; i < t->num_targets; i++) {
1564	unsigned i, num_devices;	1567	struct dm_target *ti = dm_table_get_target(t, i);
1565		1568	unsigned num_devices = 0;
1566	for (i = 0; i < dm_table_get_num_targets(table); i++) {
1567	ti = dm_table_get_target(table, i);
1568		1569
1569	if (!ti->type->iterate_devices)	1570	if (!ti->type->iterate_devices)
1570	return false;	1571	return false;
1571		1572
1572	num_devices = 0;
1573	ti->type->iterate_devices(ti, count_device, &num_devices);	1573	ti->type->iterate_devices(ti, count_device, &num_devices);
1574	if (num_devices)	1574	if (num_devices)
1575	return false;	1575	return false;
@@ -1597,11 +1597,8 @@ static int device_not_zoned_model(struct dm_target ti, struct dm_dev dev, · 21 unmodified lines
1597	static bool dm_table_supports_zoned_model(struct dm_table *t,	1597	static bool dm_table_supports_zoned_model(struct dm_table *t,
1598	enum blk_zoned_model zoned_model)	1598	enum blk_zoned_model zoned_model)
1599	{	1599	{
1600	struct dm_target *ti;	1600	for (unsigned int i = 0; i < t->num_targets; i++) {
1601	unsigned i;	1601	struct dm_target *ti = dm_table_get_target(t, i);
1602
1603	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1604	ti = dm_table_get_target(t, i);
1605		1602
1606	if (dm_target_supports_zoned_hm(ti->type)) {	1603	if (dm_target_supports_zoned_hm(ti->type)) {
1607	if (!ti->type->iterate_devices \|\|	1604	if (!ti->type->iterate_devices \|\|
@@ -1632,16 +1629,16 @@ static int device_not_matches_zone_sectors(struct dm_target ti, struct dm_dev · 24 unmodified lines
1632	* zone sectors, if the destination device is a zoned block device, it shall	1629	* zone sectors, if the destination device is a zoned block device, it shall
1633	* have the specified zone_sectors.	1630	* have the specified zone_sectors.
1634	*/	1631	*/
1635	static int validate_hardware_zoned_model(struct dm_table *table,	1632	static int validate_hardware_zoned_model(struct dm_table *t,
1636	enum blk_zoned_model zoned_model,	1633	enum blk_zoned_model zoned_model,
1637	unsigned int zone_sectors)	1634	unsigned int zone_sectors)
1638	{	1635	{
1639	if (zoned_model == BLK_ZONED_NONE)	1636	if (zoned_model == BLK_ZONED_NONE)
1640	return 0;	1637	return 0;
1641		1638
1642	if (!dm_table_supports_zoned_model(table, zoned_model)) {	1639	if (!dm_table_supports_zoned_model(t, zoned_model)) {
1643	DMERR("%s: zoned model is not consistent across all devices",	1640	DMERR("%s: zoned model is not consistent across all devices",
1644	dm_device_name(table->md));	1641	dm_device_name(t->md));
1645	return -EINVAL;	1642	return -EINVAL;
1646	}	1643	}
1647		1644
@@ -1649,9 +1646,9 @@ static int validate_hardware_zoned_model(struct dm_table *table, · 1 unmodified line
1649	if (!zone_sectors \|\| !is_power_of_2(zone_sectors))	1646	if (!zone_sectors \|\| !is_power_of_2(zone_sectors))
1650	return -EINVAL;	1647	return -EINVAL;
1651		1648
1652	if (dm_table_any_dev_attr(table, device_not_matches_zone_sectors, &zone_sectors)) {	1649	if (dm_table_any_dev_attr(t, device_not_matches_zone_sectors, &zone_sectors)) {
1653	DMERR("%s: zone sectors is not consistent across all zoned devices",	1650	DMERR("%s: zone sectors is not consistent across all zoned devices",
1654	dm_device_name(table->md));	1651	dm_device_name(t->md));
1655	return -EINVAL;	1652	return -EINVAL;
1656	}	1653	}
1657		1654
@@ -1661,21 +1658,19 @@ static int validate_hardware_zoned_model(struct dm_table *table, · 3 unmodified lines
1661	/*	1658	/*
1662	* Establish the new table's queue_limits and validate them.	1659	* Establish the new table's queue_limits and validate them.
1663	*/	1660	*/
1664	int dm_calculate_queue_limits(struct dm_table *table,	1661	int dm_calculate_queue_limits(struct dm_table *t,
1665	struct queue_limits *limits)	1662	struct queue_limits *limits)
1666	{	1663	{
1667	struct dm_target *ti;
1668	struct queue_limits ti_limits;	1664	struct queue_limits ti_limits;
1669	unsigned i;
1670	enum blk_zoned_model zoned_model = BLK_ZONED_NONE;	1665	enum blk_zoned_model zoned_model = BLK_ZONED_NONE;
1671	unsigned int zone_sectors = 0;	1666	unsigned int zone_sectors = 0;
1672		1667
1673	blk_set_stacking_limits(limits);	1668	blk_set_stacking_limits(limits);
1674		1669
1675	for (i = 0; i < dm_table_get_num_targets(table); i++) {	1670	for (unsigned int i = 0; i < t->num_targets; i++) {
1676	blk_set_stacking_limits(&ti_limits);	1671	struct dm_target *ti = dm_table_get_target(t, i);
1677		1672
1678	ti = dm_table_get_target(table, i);	1673	blk_set_stacking_limits(&ti_limits);
1679		1674
1680	if (!ti->type->iterate_devices)	1675	if (!ti->type->iterate_devices)
1681	goto combine_limits;	1676	goto combine_limits;
@@ -1716,7 +1711,7 @@ combine_limits: · 34 unmodified lines
1716	DMWARN("%s: adding target device "	1711	DMWARN("%s: adding target device "
1717	"(start sect %llu len %llu) "	1712	"(start sect %llu len %llu) "
1718	"caused an alignment inconsistency",	1713	"caused an alignment inconsistency",
1719	dm_device_name(table->md),	1714	dm_device_name(t->md),
1720	(unsigned long long) ti->begin,	1715	(unsigned long long) ti->begin,
1721	(unsigned long long) ti->len);	1716	(unsigned long long) ti->len);
1722	}	1717	}
@@ -1736,10 +1731,10 @@ combine_limits: · 13 unmodified lines
1736	zoned_model = limits->zoned;	1731	zoned_model = limits->zoned;
1737	zone_sectors = limits->chunk_sectors;	1732	zone_sectors = limits->chunk_sectors;
1738	}	1733	}
1739	if (validate_hardware_zoned_model(table, zoned_model, zone_sectors))	1734	if (validate_hardware_zoned_model(t, zoned_model, zone_sectors))
1740	return -EINVAL;	1735	return -EINVAL;
1741		1736
1742	return validate_hardware_logical_block_alignment(table, limits);	1737	return validate_hardware_logical_block_alignment(t, limits);
1743	}	1738	}
1744		1739
1745	/*	1740	/*
@@ -1783,17 +1778,14 @@ static int device_flush_capable(struct dm_target ti, struct dm_dev dev, · 37 unmodified lines
1783		1778
1784	static bool dm_table_supports_flush(struct dm_table *t, unsigned long flush)	1779	static bool dm_table_supports_flush(struct dm_table *t, unsigned long flush)
1785	{	1780	{
1786	struct dm_target *ti;
1787	unsigned i;
1788
1789	/*	1781	/*
1790	* Require at least one underlying device to support flushes.	1782	* Require at least one underlying device to support flushes.
1791	* t->devices includes internal dm devices such as mirror logs	1783	* t->devices includes internal dm devices such as mirror logs
1792	* so we need to use iterate_devices here, which targets	1784	* so we need to use iterate_devices here, which targets
1793	* supporting flushes must provide.	1785	* supporting flushes must provide.
1794	*/	1786	*/
1795	for (i = 0; i < dm_table_get_num_targets(t); i++) {	1787	for (unsigned int i = 0; i < t->num_targets; i++) {
1796	ti = dm_table_get_target(t, i);	1788	struct dm_target *ti = dm_table_get_target(t, i);
1797		1789
1798	if (!ti->num_flush_bios)	1790	if (!ti->num_flush_bios)
1799	continue;	1791	continue;
@@ -1847,11 +1839,8 @@ static int device_not_write_zeroes_capable(struct dm_target ti, struct dm_dev · 47 unmodified lines
1847		1839
1848	static bool dm_table_supports_write_zeroes(struct dm_table *t)	1840	static bool dm_table_supports_write_zeroes(struct dm_table *t)
1849	{	1841	{
1850	struct dm_target *ti;	1842	for (unsigned int i = 0; i < t->num_targets; i++) {
1851	unsigned i = 0;	1843	struct dm_target *ti = dm_table_get_target(t, i);
1852
1853	while (i < dm_table_get_num_targets(t)) {
1854	ti = dm_table_get_target(t, i++);
1855		1844
1856	if (!ti->num_write_zeroes_bios)	1845	if (!ti->num_write_zeroes_bios)
1857	return false;	1846	return false;
@@ -1874,11 +1863,8 @@ static int device_not_nowait_capable(struct dm_target ti, struct dm_dev dev, · 16 unmodified lines
1874		1863
1875	static bool dm_table_supports_nowait(struct dm_table *t)	1864	static bool dm_table_supports_nowait(struct dm_table *t)
1876	{	1865	{
1877	struct dm_target *ti;	1866	for (unsigned int i = 0; i < t->num_targets; i++) {
1878	unsigned i = 0;	1867	struct dm_target *ti = dm_table_get_target(t, i);
1879
1880	while (i < dm_table_get_num_targets(t)) {
1881	ti = dm_table_get_target(t, i++);
1882		1868
1883	if (!dm_target_supports_nowait(ti->type))	1869	if (!dm_target_supports_nowait(ti->type))
1884	return false;	1870	return false;
@@ -1899,11 +1885,8 @@ static int device_not_discard_capable(struct dm_target ti, struct dm_dev dev, · 14 unmodified lines
1899		1885
1900	static bool dm_table_supports_discards(struct dm_table *t)	1886	static bool dm_table_supports_discards(struct dm_table *t)
1901	{	1887	{
1902	struct dm_target *ti;	1888	for (unsigned int i = 0; i < t->num_targets; i++) {
1903	unsigned i;	1889	struct dm_target *ti = dm_table_get_target(t, i);
1904
1905	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1906	ti = dm_table_get_target(t, i);
1907		1890
1908	if (!ti->num_discard_bios)	1891	if (!ti->num_discard_bios)
1909	return false;	1892	return false;
@@ -1931,11 +1914,8 @@ static int device_not_secure_erase_capable(struct dm_target *ti, · 21 unmodified lines
1931		1914
1932	static bool dm_table_supports_secure_erase(struct dm_table *t)	1915	static bool dm_table_supports_secure_erase(struct dm_table *t)
1933	{	1916	{
1934	struct dm_target *ti;	1917	for (unsigned int i = 0; i < t->num_targets; i++) {
1935	unsigned int i;	1918	struct dm_target *ti = dm_table_get_target(t, i);
1936
1937	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1938	ti = dm_table_get_target(t, i);
1939		1919
1940	if (!ti->num_secure_erase_bios)	1920	if (!ti->num_secure_erase_bios)
1941	return false;	1921	return false;
@@ -2065,11 +2045,6 @@ int dm_table_set_restrictions(struct dm_table t, struct request_queue q, · 123 unmodified lines
2065	return 0;	2045	return 0;
2066	}	2046	}
2067		2047
2068	unsigned int dm_table_get_num_targets(struct dm_table *t)
2069	{
2070	return t->num_targets;
2071	}
2072
2073	struct list_head dm_table_get_devices(struct dm_table t)	2048	struct list_head dm_table_get_devices(struct dm_table t)
2074	{	2049	{
2075	return &t->devices;	2050	return &t->devices;
@@ -2089,12 +2064,11 @@ enum suspend_mode { · 13 unmodified lines
2089		2064
2090	static void suspend_targets(struct dm_table *t, enum suspend_mode mode)	2065	static void suspend_targets(struct dm_table *t, enum suspend_mode mode)
2091	{	2066	{
2092	int i = t->num_targets;
2093	struct dm_target *ti = t->targets;
2094
2095	lockdep_assert_held(&t->md->suspend_lock);	2067	lockdep_assert_held(&t->md->suspend_lock);
2096		2068
2097	while (i--) {	2069	for (unsigned int i = 0; i < t->num_targets; i++) {
		2070	struct dm_target *ti = dm_table_get_target(t, i);
		2071
2098	switch (mode) {	2072	switch (mode) {
2099	case PRESUSPEND:	2073	case PRESUSPEND:
2100	if (ti->type->presuspend)	2074	if (ti->type->presuspend)
@@ -2109,7 +2083,6 @@ static void suspend_targets(struct dm_table *t, enum suspend_mode mode) · 8 unmodified lines
2109	ti->type->postsuspend(ti);	2083	ti->type->postsuspend(ti);
2110	break;	2084	break;
2111	}	2085	}
2112	ti++;
2113	}	2086	}
2114	}	2087	}
2115		2088
@@ -2139,12 +2112,13 @@ void dm_table_postsuspend_targets(struct dm_table *t) · 23 unmodified lines
2139		2112
2140	int dm_table_resume_targets(struct dm_table *t)	2113	int dm_table_resume_targets(struct dm_table *t)
2141	{	2114	{
2142	int i, r = 0;	2115	unsigned int i;
		2116	int r = 0;
2143		2117
2144	lockdep_assert_held(&t->md->suspend_lock);	2118	lockdep_assert_held(&t->md->suspend_lock);
2145		2119
2146	for (i = 0; i < t->num_targets; i++) {	2120	for (i = 0; i < t->num_targets; i++) {
2147	struct dm_target *ti = t->targets + i;	2121	struct dm_target *ti = dm_table_get_target(t, i);
2148		2122
2149	if (!ti->type->preresume)	2123	if (!ti->type->preresume)
2150	continue;	2124	continue;
@@ -2158,7 +2132,7 @@ int dm_table_resume_targets(struct dm_table *t) · 7 unmodified lines
2158	}	2132	}
2159		2133
2160	for (i = 0; i < t->num_targets; i++) {	2134	for (i = 0; i < t->num_targets; i++) {
2161	struct dm_target *ti = t->targets + i;	2135	struct dm_target *ti = dm_table_get_target(t, i);
2162		2136
2163	if (ti->type->resume)	2137	if (ti->type->resume)
2164	ti->type->resume(ti);	2138	ti->type->resume(ti);

drivers/md/dm-thin-metadata.c+5−2

 					dm_sm_threshold_fn fn,
 					void *context)
 {
-	int r;
+	int r = -EINVAL;
 	pmd_write_lock_in_core(pmd);
-	r = dm_sm_register_threshold_callback(pmd->metadata_sm, threshold, fn, context);
+	if (!pmd->fail_io) {
+		r = dm_sm_register_threshold_callback(pmd->metadata_sm,
+						      threshold, fn, context);
+	}
 	pmd_write_unlock(pmd);
 	return r;

drivers/md/dm-thin.c+3−1

 						calc_metadata_threshold(pt),
 						metadata_low_callback,
 						pool);
-	if (r)
+	if (r) {
+		ti->error = "Error registering metadata threshold";
 		goto out_flags_changed;
+	}
 	dm_pool_register_pre_commit_callback(pool->pmd,
 					     metadata_pre_commit_callback, pool);

drivers/md/dm-verity-target.c+3−4

 			if (v->validated_blocks)
 				set_bit(cur_block, v->validated_blocks);
 			continue;
-		}
+		} else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA,
-		else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA,
+					   cur_block, NULL, &start) == 0) {
-					   cur_block, NULL, &start) == 0)
 			continue;
-		else {
+		} else {
 			if (bio->bi_status) {
 				/*
 				 * Error correction failed; Just return error

drivers/md/dm-writecache.c+25−18

 #define HIGH_WATERMARK			50
 #define LOW_WATERMARK			45
-#define MAX_WRITEBACK_JOBS		0
+#define MAX_WRITEBACK_JOBS		min(0x10000000 / PAGE_SIZE, totalram_pages() / 16)
 #define ENDIO_LATENCY			16
 #define WRITEBACK_LATENCY		64
 #define AUTOCOMMIT_BLOCKS_SSD		65536
 	WC_MAP_ERROR,
 };
-static enum wc_map_op writecache_map_remap_origin(struct dm_writecache *wc, struct bio *bio,
+static void writecache_map_remap_origin(struct dm_writecache *wc, struct bio *bio,
-						  struct wc_entry *e)
+					struct wc_entry *e)
 {
 	if (e) {
 		sector_t next_boundary =
 		if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT)
 			dm_accept_partial_bio(bio, next_boundary);
 	}
-	return WC_MAP_REMAP_ORIGIN;
 }
 static enum wc_map_op writecache_map_read(struct dm_writecache *wc, struct bio *bio)
 			map_op = WC_MAP_REMAP;
 		}
 	} else {
-		map_op = writecache_map_remap_origin(wc, bio, e);
+		writecache_map_remap_origin(wc, bio, e);
+		wc->stats.reads += (bio->bi_iter.bi_size - wc->block_size) >> wc->block_size_bits;
+		map_op = WC_MAP_REMAP_ORIGIN;
 	}
 	return map_op;
 }
-static enum wc_map_op writecache_bio_copy_ssd(struct dm_writecache *wc, struct bio *bio,
+static void writecache_bio_copy_ssd(struct dm_writecache *wc, struct bio *bio,
-					      struct wc_entry *e, bool search_used)
+				    struct wc_entry *e, bool search_used)
 {
 	unsigned bio_size = wc->block_size;
 	sector_t start_cache_sec = cache_sector(wc, e);
 	bio->bi_iter.bi_sector = start_cache_sec;
 	dm_accept_partial_bio(bio, bio_size >> SECTOR_SHIFT);
+	wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
+	wc->stats.writes_allocate += (bio->bi_iter.bi_size - wc->block_size) >> wc->block_size_bits;
 	if (unlikely(wc->uncommitted_blocks >= wc->autocommit_blocks)) {
 		wc->uncommitted_blocks = 0;
 		queue_work(wc->writeback_wq, &wc->flush_work);
 	} else {
 		writecache_schedule_autocommit(wc);
 	}
-	return WC_MAP_REMAP;
 }
 static enum wc_map_op writecache_map_write(struct dm_writecache *wc, struct bio *bio)
 	do {
 		bool found_entry = false;
 		bool search_used = false;
-		wc->stats.writes++;
+		if (writecache_has_error(wc)) {
-		if (writecache_has_error(wc))
+			wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
 			return WC_MAP_ERROR;
+		}
 		e = writecache_find_entry(wc, bio->bi_iter.bi_sector, 0);
 		if (e) {
 			if (!writecache_entry_is_committed(wc, e)) {
 		if (unlikely(!e)) {
 			if (!WC_MODE_PMEM(wc) && !found_entry) {
 direct_write:
-				wc->stats.writes_around++;
 				e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
-				return writecache_map_remap_origin(wc, bio, e);
+				writecache_map_remap_origin(wc, bio, e);
+				wc->stats.writes_around += bio->bi_iter.bi_size >> wc->block_size_bits;
+				wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
+				return WC_MAP_REMAP_ORIGIN;
 			}
 			wc->stats.writes_blocked_on_freelist++;
 			writecache_wait_on_freelist(wc);
 		wc->uncommitted_blocks++;
 		wc->stats.writes_allocate++;
 bio_copy:
-		if (WC_MODE_PMEM(wc))
+		if (WC_MODE_PMEM(wc)) {
 			bio_copy_block(wc, bio, memory_data(wc, e));
-		else
+			wc->stats.writes++;
-			return writecache_bio_copy_ssd(wc, bio, e, search_used);
+		} else {
+			writecache_bio_copy_ssd(wc, bio, e, search_used);
+			return WC_MAP_REMAP;
+		}
 	} while (bio->bi_iter.bi_size);
 	if (unlikely(bio->bi_opf & REQ_FUA || wc->uncommitted_blocks >= wc->autocommit_blocks))
 static enum wc_map_op writecache_map_discard(struct dm_writecache *wc, struct bio *bio)
 {
-	wc->stats.discards++;
+	wc->stats.discards += bio->bi_iter.bi_size >> wc->block_size_bits;
 	if (writecache_has_error(wc))
 		return WC_MAP_ERROR;

drivers/md/dm-zone.c+2−5

 static bool dm_table_supports_zone_append(struct dm_table *t)
 {
-	struct dm_target *ti;
+	for (unsigned int i = 0; i < t->num_targets; i++) {
-	unsigned int i;
+		struct dm_target *ti = dm_table_get_target(t, i);
-	for (i = 0; i < dm_table_get_num_targets(t); i++) {
-		ti = dm_table_get_target(t, i);
 		if (ti->emulate_zone_append)
 			return false;

drivers/md/dm.c+291−171

@@ -88,10 +88,6 @@ struct clone_info { · 87 unmodified lines
88	bool submit_as_polled:1;	88	bool submit_as_polled:1;
89	};	89	};
90		90
91	#define DM_TARGET_IO_BIO_OFFSET (offsetof(struct dm_target_io, clone))
92	#define DM_IO_BIO_OFFSET \
93	(offsetof(struct dm_target_io, clone) + offsetof(struct dm_io, tio))
94
95	static inline struct dm_target_io clone_to_tio(struct bio clone)	91	static inline struct dm_target_io clone_to_tio(struct bio clone)
96	{	92	{
97	return container_of(clone, struct dm_target_io, clone);	93	return container_of(clone, struct dm_target_io, clone);
@@ -415,7 +411,7 @@ static int dm_blk_getgeo(struct block_device bdev, struct hd_geometry geo) · 317 unmodified lines
415	static int dm_prepare_ioctl(struct mapped_device md, int srcu_idx,	411	static int dm_prepare_ioctl(struct mapped_device md, int srcu_idx,
416	struct block_device **bdev)	412	struct block_device **bdev)
417	{	413	{
418	struct dm_target *tgt;	414	struct dm_target *ti;
419	struct dm_table *map;	415	struct dm_table *map;
420	int r;	416	int r;
421		417
@@ -426,17 +422,17 @@ retry: · 4 unmodified lines
426	return r;	422	return r;
427		423
428	/* We only support devices that have a single target */	424	/* We only support devices that have a single target */
429	if (dm_table_get_num_targets(map) != 1)	425	if (map->num_targets != 1)
430	return r;	426	return r;
431		427
432	tgt = dm_table_get_target(map, 0);	428	ti = dm_table_get_target(map, 0);
433	if (!tgt->type->prepare_ioctl)	429	if (!ti->type->prepare_ioctl)
434	return r;	430	return r;
435		431
436	if (dm_suspended_md(md))	432	if (dm_suspended_md(md))
437	return -EAGAIN;	433	return -EAGAIN;
438		434
439	r = tgt->type->prepare_ioctl(tgt, bdev);	435	r = ti->type->prepare_ioctl(ti, bdev);
440	if (r == -ENOTCONN && !fatal_signal_pending(current)) {	436	if (r == -ENOTCONN && !fatal_signal_pending(current)) {
441	dm_put_live_table(md, *srcu_idx);	437	dm_put_live_table(md, *srcu_idx);
442	msleep(10);	438	msleep(10);
@@ -578,9 +574,6 @@ static struct dm_io alloc_io(struct mapped_device md, struct bio *bio) · 135 unmodified lines
578	struct bio *clone;	574	struct bio *clone;
579		575
580	clone = bio_alloc_clone(NULL, bio, GFP_NOIO, &md->mempools->io_bs);	576	clone = bio_alloc_clone(NULL, bio, GFP_NOIO, &md->mempools->io_bs);
581	/* Set default bdev, but target must bio_set_dev() before issuing IO */
582	clone->bi_bdev = md->disk->part0;
583
584	tio = clone_to_tio(clone);	577	tio = clone_to_tio(clone);
585	tio->flags = 0;	578	tio->flags = 0;
586	dm_tio_set_flag(tio, DM_TIO_INSIDE_DM_IO);	579	dm_tio_set_flag(tio, DM_TIO_INSIDE_DM_IO);
@@ -594,7 +587,6 @@ static struct dm_io alloc_io(struct mapped_device md, struct bio *bio) · 7 unmodified lines
594	atomic_set(&io->io_count, 2);	587	atomic_set(&io->io_count, 2);
595	this_cpu_inc(*md->pending_io);	588	this_cpu_inc(*md->pending_io);
596	io->orig_bio = bio;	589	io->orig_bio = bio;
597	io->split_bio = NULL;
598	io->md = md;	590	io->md = md;
599	spin_lock_init(&io->lock);	591	spin_lock_init(&io->lock);
600	io->start_time = jiffies;	592	io->start_time = jiffies;
@@ -614,6 +606,7 @@ static void free_io(struct dm_io *io) · 13 unmodified lines
614	static struct bio alloc_tio(struct clone_info ci, struct dm_target *ti,	606	static struct bio alloc_tio(struct clone_info ci, struct dm_target *ti,
615	unsigned target_bio_nr, unsigned *len, gfp_t gfp_mask)	607	unsigned target_bio_nr, unsigned *len, gfp_t gfp_mask)
616	{	608	{
		609	struct mapped_device *md = ci->io->md;
617	struct dm_target_io *tio;	610	struct dm_target_io *tio;
618	struct bio *clone;	611	struct bio *clone;
619		612
@@ -623,14 +616,10 @@ static struct bio alloc_tio(struct clone_info ci, struct dm_target *ti, · 3 unmodified lines
623	/* alloc_io() already initialized embedded clone */	616	/* alloc_io() already initialized embedded clone */
624	clone = &tio->clone;	617	clone = &tio->clone;
625	} else {	618	} else {
626	struct mapped_device *md = ci->io->md;
627
628	clone = bio_alloc_clone(NULL, ci->bio, gfp_mask,	619	clone = bio_alloc_clone(NULL, ci->bio, gfp_mask,
629	&md->mempools->bs);	620	&md->mempools->bs);
630	if (!clone)	621	if (!clone)
631	return NULL;	622	return NULL;
632	/* Set default bdev, but target must bio_set_dev() before issuing IO */
633	clone->bi_bdev = md->disk->part0;
634		623
635	/* REQ_DM_POLL_LIST shouldn't be inherited */	624	/* REQ_DM_POLL_LIST shouldn't be inherited */
636	clone->bi_opf &= ~REQ_DM_POLL_LIST;	625	clone->bi_opf &= ~REQ_DM_POLL_LIST;
@@ -646,6 +635,11 @@ static struct bio alloc_tio(struct clone_info ci, struct dm_target *ti, · 9 unmodified lines
646	tio->len_ptr = len;	635	tio->len_ptr = len;
647	tio->old_sector = 0;	636	tio->old_sector = 0;
648		637
		638	/* Set default bdev, but target must bio_set_dev() before issuing IO */
		639	clone->bi_bdev = md->disk->part0;
		640	if (unlikely(ti->needs_bio_set_dev))
		641	bio_set_dev(clone, md->disk->part0);
		642
649	if (len) {	643	if (len) {
650	clone->bi_iter.bi_size = to_bytes(*len);	644	clone->bi_iter.bi_size = to_bytes(*len);
651	if (bio_integrity(clone))	645	if (bio_integrity(clone))
@@ -884,22 +878,63 @@ static int __noflush_suspending(struct mapped_device *md) · 232 unmodified lines
884	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);	878	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
885	}	879	}
886		880
887	static void dm_io_complete(struct dm_io *io)	881	static void dm_requeue_add_io(struct dm_io *io, bool first_stage)
888	{	882	{
889	blk_status_t io_error;
890	struct mapped_device *md = io->md;	883	struct mapped_device *md = io->md;
891	struct bio *bio = io->split_bio ? io->split_bio : io->orig_bio;
892		884
893	if (io->status == BLK_STS_DM_REQUEUE) {	885	if (first_stage) {
		886	struct dm_io *next = md->requeue_list;
		887
		888	md->requeue_list = io;
		889	io->next = next;
		890	} else {
		891	bio_list_add_head(&md->deferred, io->orig_bio);
		892	}
		893	}
		894
		895	static void dm_kick_requeue(struct mapped_device *md, bool first_stage)
		896	{
		897	if (first_stage)
		898	queue_work(md->wq, &md->requeue_work);
		899	else
		900	queue_work(md->wq, &md->work);
		901	}
		902
		903	/*
		904	* Return true if the dm_io's original bio is requeued.
		905	* io->status is updated with error if requeue disallowed.
		906	*/
		907	static bool dm_handle_requeue(struct dm_io *io, bool first_stage)
		908	{
		909	struct bio *bio = io->orig_bio;
		910	bool handle_requeue = (io->status == BLK_STS_DM_REQUEUE);
		911	bool handle_polled_eagain = ((io->status == BLK_STS_AGAIN) &&
		912	(bio->bi_opf & REQ_POLLED));
		913	struct mapped_device *md = io->md;
		914	bool requeued = false;
		915
		916	if (handle_requeue \|\| handle_polled_eagain) {
894	unsigned long flags;	917	unsigned long flags;
		918
		919	if (bio->bi_opf & REQ_POLLED) {
		920	/*
		921	* Upper layer won't help us poll split bio
		922	* (io->orig_bio may only reflect a subset of the
		923	* pre-split original) so clear REQ_POLLED.
		924	*/
		925	bio_clear_polled(bio);
		926	}
		927
895	/*	928	/*
896	* Target requested pushing back the I/O.	929	* Target requested pushing back the I/O or
		930	* polled IO hit BLK_STS_AGAIN.
897	*/	931	*/
898	spin_lock_irqsave(&md->deferred_lock, flags);	932	spin_lock_irqsave(&md->deferred_lock, flags);
899	if (__noflush_suspending(md) &&	933	if ((__noflush_suspending(md) &&
900	!WARN_ON_ONCE(dm_is_zone_write(md, bio))) {	934	!WARN_ON_ONCE(dm_is_zone_write(md, bio))) \|\|
901	/* NOTE early return due to BLK_STS_DM_REQUEUE below */	935	handle_polled_eagain \|\| first_stage) {
902	bio_list_add_head(&md->deferred, bio);	936	dm_requeue_add_io(io, first_stage);
		937	requeued = true;
903	} else {	938	} else {
904	/*	939	/*
905	* noflush suspend was interrupted or this is	940	* noflush suspend was interrupted or this is
@@ -910,6 +945,23 @@ static void dm_io_complete(struct dm_io *io) · 4 unmodified lines
910	spin_unlock_irqrestore(&md->deferred_lock, flags);	945	spin_unlock_irqrestore(&md->deferred_lock, flags);
911	}	946	}
912		947
		948	if (requeued)
		949	dm_kick_requeue(md, first_stage);
		950
		951	return requeued;
		952	}
		953
		954	static void __dm_io_complete(struct dm_io *io, bool first_stage)
		955	{
		956	struct bio *bio = io->orig_bio;
		957	struct mapped_device *md = io->md;
		958	blk_status_t io_error;
		959	bool requeued;
		960
		961	requeued = dm_handle_requeue(io, first_stage);
		962	if (requeued && first_stage)
		963	return;
		964
913	io_error = io->status;	965	io_error = io->status;
914	if (dm_io_flagged(io, DM_IO_ACCOUNTED))	966	if (dm_io_flagged(io, DM_IO_ACCOUNTED))
915	dm_end_io_acct(io);	967	dm_end_io_acct(io);
@@ -929,23 +981,9 @@ static void dm_io_complete(struct dm_io *io) · 13 unmodified lines
929	if (unlikely(wq_has_sleeper(&md->wait)))	981	if (unlikely(wq_has_sleeper(&md->wait)))
930	wake_up(&md->wait);	982	wake_up(&md->wait);
931		983
932	if (io_error == BLK_STS_DM_REQUEUE \|\| io_error == BLK_STS_AGAIN) {	984	/* Return early if the original bio was requeued */
933	if (bio->bi_opf & REQ_POLLED) {	985	if (requeued)
934	/*	986	return;
935	* Upper layer won't help us poll split bio (io->orig_bio
936	* may only reflect a subset of the pre-split original)
937	* so clear REQ_POLLED in case of requeue.
938	*/
939	bio_clear_polled(bio);
940	if (io_error == BLK_STS_AGAIN) {
941	/* io_uring doesn't handle BLK_STS_AGAIN (yet) */
942	queue_io(md, bio);
943	return;
944	}
945	}
946	if (io_error == BLK_STS_DM_REQUEUE)
947	return;
948	}
949		987
950	if (bio_is_flush_with_data(bio)) {	988	if (bio_is_flush_with_data(bio)) {
951	/*	989	/*
@@ -962,6 +1000,58 @@ static void dm_io_complete(struct dm_io *io) · 10 unmodified lines
962	}	1000	}
963	}	1001	}
964		1002
		1003	static void dm_wq_requeue_work(struct work_struct *work)
		1004	{
		1005	struct mapped_device *md = container_of(work, struct mapped_device,
		1006	requeue_work);
		1007	unsigned long flags;
		1008	struct dm_io *io;
		1009
		1010	/* reuse deferred lock to simplify dm_handle_requeue */
		1011	spin_lock_irqsave(&md->deferred_lock, flags);
		1012	io = md->requeue_list;
		1013	md->requeue_list = NULL;
		1014	spin_unlock_irqrestore(&md->deferred_lock, flags);
		1015
		1016	while (io) {
		1017	struct dm_io *next = io->next;
		1018
		1019	dm_io_rewind(io, &md->queue->bio_split);
		1020
		1021	io->next = NULL;
		1022	__dm_io_complete(io, false);
		1023	io = next;
		1024	}
		1025	}
		1026
		1027	/*
		1028	* Two staged requeue:
		1029	*
		1030	* 1) io->orig_bio points to the real original bio, and the part mapped to
		1031	* this io must be requeued, instead of other parts of the original bio.
		1032	*
		1033	* 2) io->orig_bio points to new cloned bio which matches the requeued dm_io.
		1034	*/
		1035	static void dm_io_complete(struct dm_io *io)
		1036	{
		1037	bool first_requeue;
		1038
		1039	/*
		1040	* Only dm_io that has been split needs two stage requeue, otherwise
		1041	* we may run into long bio clone chain during suspend and OOM could
		1042	* be triggered.
		1043	*
		1044	* Also flush data dm_io won't be marked as DM_IO_WAS_SPLIT, so they
		1045	* also aren't handled via the first stage requeue.
		1046	*/
		1047	if (dm_io_flagged(io, DM_IO_WAS_SPLIT))
		1048	first_requeue = true;
		1049	else
		1050	first_requeue = false;
		1051
		1052	__dm_io_complete(io, first_requeue);
		1053	}
		1054
965	/*	1055	/*
966	* Decrements the number of outstanding ios that a bio has been	1056	* Decrements the number of outstanding ios that a bio has been
967	* cloned into, completing the original io if necc.	1057	* cloned into, completing the original io if necc.
@@ -1240,6 +1330,7 @@ out: · 272 unmodified lines
1240	void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)	1330	void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
1241	{	1331	{
1242	struct dm_target_io *tio = clone_to_tio(bio);	1332	struct dm_target_io *tio = clone_to_tio(bio);
		1333	struct dm_io *io = tio->io;
1243	unsigned bio_sectors = bio_sectors(bio);	1334	unsigned bio_sectors = bio_sectors(bio);
1244		1335
1245	BUG_ON(dm_tio_flagged(tio, DM_TIO_IS_DUPLICATE_BIO));	1336	BUG_ON(dm_tio_flagged(tio, DM_TIO_IS_DUPLICATE_BIO));
@@ -1255,8 +1346,9 @@ void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors) · 9 unmodified lines
1255	* __split_and_process_bio() may have already saved mapped part	1346	* __split_and_process_bio() may have already saved mapped part
1256	* for accounting but it is being reduced so update accordingly.	1347	* for accounting but it is being reduced so update accordingly.
1257	*/	1348	*/
1258	dm_io_set_flag(tio->io, DM_IO_WAS_SPLIT);	1349	dm_io_set_flag(io, DM_IO_WAS_SPLIT);
1259	tio->io->sectors = n_sectors;	1350	io->sectors = n_sectors;
		1351	io->sector_offset = bio_sectors(io->orig_bio);
1260	}	1352	}
1261	EXPORT_SYMBOL_GPL(dm_accept_partial_bio);	1353	EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
1262		1354
@@ -1379,17 +1471,7 @@ static void setup_split_accounting(struct clone_info *ci, unsigned len) · 116 unmodified lines
1379	*/	1471	*/
1380	dm_io_set_flag(io, DM_IO_WAS_SPLIT);	1472	dm_io_set_flag(io, DM_IO_WAS_SPLIT);
1381	io->sectors = len;	1473	io->sectors = len;
1382	}	1474	io->sector_offset = bio_sectors(ci->bio);
1383
1384	if (static_branch_unlikely(&stats_enabled) &&
1385	unlikely(dm_stats_used(&io->md->stats))) {
1386	/*
1387	* Save bi_sector in terms of its offset from end of
1388	* original bio, only needed for DM-stats' benefit.
1389	* - saved regardless of whether split needed so that
1390	* dm_accept_partial_bio() doesn't need to.
1391	*/
1392	io->sector_offset = bio_end_sector(ci->bio) - ci->sector;
1393	}	1475	}
1394	}	1476	}
1395		1477
@@ -1423,11 +1505,11 @@ static void alloc_multiple_bios(struct bio_list blist, struct clone_info ci, · 27 unmodified lines
1423	}	1505	}
1424		1506
1425	static int __send_duplicate_bios(struct clone_info ci, struct dm_target ti,	1507	static int __send_duplicate_bios(struct clone_info ci, struct dm_target ti,
1426	unsigned num_bios, unsigned *len)	1508	unsigned int num_bios, unsigned *len)
1427	{	1509	{
1428	struct bio_list blist = BIO_EMPTY_LIST;	1510	struct bio_list blist = BIO_EMPTY_LIST;
1429	struct bio *clone;	1511	struct bio *clone;
1430	int ret = 0;	1512	unsigned int ret = 0;
1431		1513
1432	switch (num_bios) {	1514	switch (num_bios) {
1433	case 0:	1515	case 0:
@@ -1455,8 +1537,7 @@ static int __send_duplicate_bios(struct clone_info ci, struct dm_target ti, · 21 unmodified lines
1455		1537
1456	static void __send_empty_flush(struct clone_info *ci)	1538	static void __send_empty_flush(struct clone_info *ci)
1457	{	1539	{
1458	unsigned target_nr = 0;	1540	struct dm_table *t = ci->map;
1459	struct dm_target *ti;
1460	struct bio flush_bio;	1541	struct bio flush_bio;
1461		1542
1462	/*	1543	/*
@@ -1471,8 +1552,9 @@ static void __send_empty_flush(struct clone_info *ci) · 8 unmodified lines
1471	ci->sector_count = 0;	1552	ci->sector_count = 0;
1472	ci->io->tio.clone.bi_iter.bi_size = 0;	1553	ci->io->tio.clone.bi_iter.bi_size = 0;
1473		1554
1474	while ((ti = dm_table_get_target(ci->map, target_nr++))) {	1555	for (unsigned int i = 0; i < t->num_targets; i++) {
1475	int bios;	1556	unsigned int bios;
		1557	struct dm_target *ti = dm_table_get_target(t, i);
1476		1558
1477	atomic_add(ti->num_flush_bios, &ci->io->io_count);	1559	atomic_add(ti->num_flush_bios, &ci->io->io_count);
1478	bios = __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);	1560	bios = __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
@@ -1492,7 +1574,7 @@ static void __send_changing_extent_only(struct clone_info *ci, struct dm_target · 13 unmodified lines
1492	unsigned num_bios)	1574	unsigned num_bios)
1493	{	1575	{
1494	unsigned len;	1576	unsigned len;
1495	int bios;	1577	unsigned int bios;
1496		1578
1497	len = min_t(sector_t, ci->sector_count,	1579	len = min_t(sector_t, ci->sector_count,
1498	max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector)));	1580	max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector)));
@@ -1691,11 +1773,9 @@ static void dm_split_and_process_bio(struct mapped_device *md, · 192 unmodified lines
1691	* Remainder must be passed to submit_bio_noacct() so it gets handled	1773	* Remainder must be passed to submit_bio_noacct() so it gets handled
1692	* after bios already submitted have been completely processed.	1774	* after bios already submitted have been completely processed.
1693	*/	1775	*/
1694	WARN_ON_ONCE(!dm_io_flagged(io, DM_IO_WAS_SPLIT));	1776	bio_trim(bio, io->sectors, ci.sector_count);
1695	io->split_bio = bio_split(bio, io->sectors, GFP_NOIO,	1777	trace_block_split(bio, bio->bi_iter.bi_sector);
1696	&md->queue->bio_split);	1778	bio_inc_remaining(bio);
1697	bio_chain(io->split_bio, bio);
1698	trace_block_split(io->split_bio, bio->bi_iter.bi_sector);
1699	submit_bio_noacct(bio);	1779	submit_bio_noacct(bio);
1700	out:	1780	out:
1701	/*	1781	/*
@@ -1971,9 +2051,11 @@ static struct mapped_device *alloc_dev(int minor) · 269 unmodified lines
1971		2051
1972	init_waitqueue_head(&md->wait);	2052	init_waitqueue_head(&md->wait);
1973	INIT_WORK(&md->work, dm_wq_work);	2053	INIT_WORK(&md->work, dm_wq_work);
		2054	INIT_WORK(&md->requeue_work, dm_wq_requeue_work);
1974	init_waitqueue_head(&md->eventq);	2055	init_waitqueue_head(&md->eventq);
1975	init_completion(&md->kobj_holder.completion);	2056	init_completion(&md->kobj_holder.completion);
1976		2057
		2058	md->requeue_list = NULL;
1977	md->swap_bios = get_swap_bios();	2059	md->swap_bios = get_swap_bios();
1978	sema_init(&md->swap_bios_semaphore, md->swap_bios);	2060	sema_init(&md->swap_bios_semaphore, md->swap_bios);
1979	mutex_init(&md->swap_bios_lock);	2061	mutex_init(&md->swap_bios_lock);
@@ -2980,54 +3062,6 @@ int dm_noflush_suspending(struct dm_target *ti) · 1000 unmodified lines
2980	}	3062	}
2981	EXPORT_SYMBOL_GPL(dm_noflush_suspending);	3063	EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2982		3064
2983	struct dm_md_mempools dm_alloc_md_mempools(struct mapped_device md, enum dm_queue_mode type,
2984	unsigned per_io_data_size, unsigned min_pool_size,
2985	bool integrity, bool poll)
2986	{
2987	struct dm_md_mempools pools = kzalloc_node(sizeof(pools), GFP_KERNEL, md->numa_node_id);
2988	unsigned int pool_size = 0;
2989	unsigned int front_pad, io_front_pad;
2990	int ret;
2991
2992	if (!pools)
2993	return NULL;
2994
2995	switch (type) {
2996	case DM_TYPE_BIO_BASED:
2997	case DM_TYPE_DAX_BIO_BASED:
2998	pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
2999	front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + DM_TARGET_IO_BIO_OFFSET;
3000	io_front_pad = roundup(per_io_data_size, __alignof__(struct dm_io)) + DM_IO_BIO_OFFSET;
3001	ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, poll ? BIOSET_PERCPU_CACHE : 0);
3002	if (ret)
3003	goto out;
3004	if (integrity && bioset_integrity_create(&pools->io_bs, pool_size))
3005	goto out;
3006	break;
3007	case DM_TYPE_REQUEST_BASED:
3008	pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size);
3009	front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
3010	/* per_io_data_size is used for blk-mq pdu at queue allocation */
3011	break;
3012	default:
3013	BUG();
3014	}
3015
3016	ret = bioset_init(&pools->bs, pool_size, front_pad, 0);
3017	if (ret)
3018	goto out;
3019
3020	if (integrity && bioset_integrity_create(&pools->bs, pool_size))
3021	goto out;
3022
3023	return pools;
3024
3025	out:
3026	dm_free_md_mempools(pools);
3027
3028	return NULL;
3029	}
3030
3031	void dm_free_md_mempools(struct dm_md_mempools *pools)	3065	void dm_free_md_mempools(struct dm_md_mempools *pools)
3032	{	3066	{
3033	if (!pools)	3067	if (!pools)
@@ -3043,11 +3077,14 @@ struct dm_pr { · 9 unmodified lines
3043	u64 old_key;	3077	u64 old_key;
3044	u64 new_key;	3078	u64 new_key;
3045	u32 flags;	3079	u32 flags;
		3080	bool abort;
3046	bool fail_early;	3081	bool fail_early;
		3082	int ret;
		3083	enum pr_type type;
3047	};	3084	};
3048		3085
3049	static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,	3086	static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
3050	void *data)	3087	struct dm_pr *pr)
3051	{	3088	{
3052	struct mapped_device *md = bdev->bd_disk->private_data;	3089	struct mapped_device *md = bdev->bd_disk->private_data;
3053	struct dm_table *table;	3090	struct dm_table *table;
@@ -3059,15 +3096,21 @@ static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn, · 5 unmodified lines
3059	goto out;	3096	goto out;
3060		3097
3061	/* We only support devices that have a single target */	3098	/* We only support devices that have a single target */
3062	if (dm_table_get_num_targets(table) != 1)	3099	if (table->num_targets != 1)
3063	goto out;	3100	goto out;
3064	ti = dm_table_get_target(table, 0);	3101	ti = dm_table_get_target(table, 0);
3065		3102
		3103	if (dm_suspended_md(md)) {
		3104	ret = -EAGAIN;
		3105	goto out;
		3106	}
		3107
3066	ret = -EINVAL;	3108	ret = -EINVAL;
3067	if (!ti->type->iterate_devices)	3109	if (!ti->type->iterate_devices)
3068	goto out;	3110	goto out;
3069		3111
3070	ret = ti->type->iterate_devices(ti, fn, data);	3112	ti->type->iterate_devices(ti, fn, pr);
		3113	ret = 0;
3071	out:	3114	out:
3072	dm_put_live_table(md, srcu_idx);	3115	dm_put_live_table(md, srcu_idx);
3073	return ret;	3116	return ret;
@@ -3081,10 +3124,24 @@ static int __dm_pr_register(struct dm_target ti, struct dm_dev dev, · 7 unmodified lines
3081	{	3124	{
3082	struct dm_pr *pr = data;	3125	struct dm_pr *pr = data;
3083	const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;	3126	const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
		3127	int ret;
		3128
		3129	if (!ops \|\| !ops->pr_register) {
		3130	pr->ret = -EOPNOTSUPP;
		3131	return -1;
		3132	}
3084		3133
3085	if (!ops \|\| !ops->pr_register)	3134	ret = ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
3086	return -EOPNOTSUPP;	3135	if (!ret)
3087	return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);	3136	return 0;
		3137
		3138	if (!pr->ret)
		3139	pr->ret = ret;
		3140
		3141	if (pr->fail_early)
		3142	return -1;
		3143
		3144	return 0;
3088	}	3145	}
3089		3146
3090	static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,	3147	static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
@@ -3095,82 +3152,145 @@ static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key, · 4 unmodified lines
3095	.new_key = new_key,	3152	.new_key = new_key,
3096	.flags = flags,	3153	.flags = flags,
3097	.fail_early = true,	3154	.fail_early = true,
		3155	.ret = 0,
3098	};	3156	};
3099	int ret;	3157	int ret;
3100		3158
3101	ret = dm_call_pr(bdev, __dm_pr_register, &pr);	3159	ret = dm_call_pr(bdev, __dm_pr_register, &pr);
3102	if (ret && new_key) {	3160	if (ret) {
3103	/* unregister all paths if we failed to register any path */	3161	/* Didn't even get to register a path */
3104	pr.old_key = new_key;	3162	return ret;
3105	pr.new_key = 0;
3106	pr.flags = 0;
3107	pr.fail_early = false;
3108	dm_call_pr(bdev, __dm_pr_register, &pr);
3109	}	3163	}
3110		3164
		3165	if (!pr.ret)
		3166	return 0;
		3167	ret = pr.ret;
		3168
		3169	if (!new_key)
		3170	return ret;
		3171
		3172	/* unregister all paths if we failed to register any path */
		3173	pr.old_key = new_key;
		3174	pr.new_key = 0;
		3175	pr.flags = 0;
		3176	pr.fail_early = false;
		3177	(void) dm_call_pr(bdev, __dm_pr_register, &pr);
3111	return ret;	3178	return ret;
3112	}	3179	}
3113		3180
		3181
		3182	static int __dm_pr_reserve(struct dm_target ti, struct dm_dev dev,
		3183	sector_t start, sector_t len, void *data)
		3184	{
		3185	struct dm_pr *pr = data;
		3186	const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
		3187
		3188	if (!ops \|\| !ops->pr_reserve) {
		3189	pr->ret = -EOPNOTSUPP;
		3190	return -1;
		3191	}
		3192
		3193	pr->ret = ops->pr_reserve(dev->bdev, pr->old_key, pr->type, pr->flags);
		3194	if (!pr->ret)
		3195	return -1;
		3196
		3197	return 0;
		3198	}
		3199
3114	static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,	3200	static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
3115	u32 flags)	3201	u32 flags)
3116	{	3202	{
3117	struct mapped_device *md = bdev->bd_disk->private_data;	3203	struct dm_pr pr = {
3118	const struct pr_ops *ops;	3204	.old_key = key,
3119	int r, srcu_idx;	3205	.flags = flags,
		3206	.type = type,
		3207	.fail_early = false,
		3208	.ret = 0,
		3209	};
		3210	int ret;
3120		3211
3121	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);	3212	ret = dm_call_pr(bdev, __dm_pr_reserve, &pr);
3122	if (r < 0)	3213	if (ret)
3123	goto out;	3214	return ret;
3124		3215
3125	ops = bdev->bd_disk->fops->pr_ops;	3216	return pr.ret;
3126	if (ops && ops->pr_reserve)	3217	}
3127	r = ops->pr_reserve(bdev, key, type, flags);	3218
3128	else	3219	/*
3129	r = -EOPNOTSUPP;	3220	* If there is a non-All Registrants type of reservation, the release must be
3130	out:	3221	* sent down the holding path. For the cases where there is no reservation or
3131	dm_unprepare_ioctl(md, srcu_idx);	3222	* the path is not the holder the device will also return success, so we must
3132	return r;	3223	* try each path to make sure we got the correct path.
		3224	*/
		3225	static int __dm_pr_release(struct dm_target ti, struct dm_dev dev,
		3226	sector_t start, sector_t len, void *data)
		3227	{
		3228	struct dm_pr *pr = data;
		3229	const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
		3230
		3231	if (!ops \|\| !ops->pr_release) {
		3232	pr->ret = -EOPNOTSUPP;
		3233	return -1;
		3234	}
		3235
		3236	pr->ret = ops->pr_release(dev->bdev, pr->old_key, pr->type);
		3237	if (pr->ret)
		3238	return -1;
		3239
		3240	return 0;
3133	}	3241	}
3134		3242
3135	static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)	3243	static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
3136	{	3244	{
3137	struct mapped_device *md = bdev->bd_disk->private_data;	3245	struct dm_pr pr = {
3138	const struct pr_ops *ops;	3246	.old_key = key,
3139	int r, srcu_idx;	3247	.type = type,
		3248	.fail_early = false,
		3249	};
		3250	int ret;
3140		3251
3141	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);	3252	ret = dm_call_pr(bdev, __dm_pr_release, &pr);
3142	if (r < 0)	3253	if (ret)
3143	goto out;	3254	return ret;
3144		3255
3145	ops = bdev->bd_disk->fops->pr_ops;	3256	return pr.ret;
3146	if (ops && ops->pr_release)	3257	}
3147	r = ops->pr_release(bdev, key, type);	3258
3148	else	3259	static int __dm_pr_preempt(struct dm_target ti, struct dm_dev dev,
3149	r = -EOPNOTSUPP;	3260	sector_t start, sector_t len, void *data)
3150	out:	3261	{
3151	dm_unprepare_ioctl(md, srcu_idx);	3262	struct dm_pr *pr = data;
3152	return r;	3263	const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
		3264
		3265	if (!ops \|\| !ops->pr_preempt) {
		3266	pr->ret = -EOPNOTSUPP;
		3267	return -1;
		3268	}
		3269
		3270	pr->ret = ops->pr_preempt(dev->bdev, pr->old_key, pr->new_key, pr->type,
		3271	pr->abort);
		3272	if (!pr->ret)
		3273	return -1;
		3274
		3275	return 0;
3153	}	3276	}
3154		3277
3155	static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,	3278	static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
3156	enum pr_type type, bool abort)	3279	enum pr_type type, bool abort)
3157	{	3280	{
3158	struct mapped_device *md = bdev->bd_disk->private_data;	3281	struct dm_pr pr = {
3159	const struct pr_ops *ops;	3282	.new_key = new_key,
3160	int r, srcu_idx;	3283	.old_key = old_key,
		3284	.type = type,
		3285	.fail_early = false,
		3286	};
		3287	int ret;
3161		3288
3162	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);	3289	ret = dm_call_pr(bdev, __dm_pr_preempt, &pr);
3163	if (r < 0)	3290	if (ret)
3164	goto out;	3291	return ret;
3165		3292
3166	ops = bdev->bd_disk->fops->pr_ops;	3293	return pr.ret;
3167	if (ops && ops->pr_preempt)
3168	r = ops->pr_preempt(bdev, old_key, new_key, type, abort);
3169	else
3170	r = -EOPNOTSUPP;
3171	out:
3172	dm_unprepare_ioctl(md, srcu_idx);
3173	return r;
3174	}	3294	}
3175		3295
3176	static int dm_pr_clear(struct block_device *bdev, u64 key)	3296	static int dm_pr_clear(struct block_device *bdev, u64 key)

drivers/md/dm.h+0−4

  *---------------------------------------------------------------*/
 void dm_table_event_callback(struct dm_table *t,
 			     void (*fn)(void *), void *context);
-struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index);
 struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector);
 bool dm_table_has_no_data_devices(struct dm_table *table);
 int dm_calculate_queue_limits(struct dm_table *table,
 /*
  * Mempool operations
  */
-struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type,
-					    unsigned per_io_data_size, unsigned min_pool_size,
-					    bool integrity, bool poll);
 void dm_free_md_mempools(struct dm_md_mempools *pools);
 /*

include/linux/device-mapper.h+6−1

 	 * after returning DM_MAPIO_SUBMITTED from its map function.
 	 */
 	bool accounts_remapped_io:1;
+	/*
+	 * Set if the target will submit the DM bio without first calling
+	 * bio_set_dev(). NOTE: ideally a target should _not_ need this.
+	 */
+	bool needs_bio_set_dev:1;
 };
 void *dm_per_bio_data(struct bio *bio, size_t data_size);
  * Queries
  */
 sector_t dm_table_get_size(struct dm_table *t);
-unsigned int dm_table_get_num_targets(struct dm_table *t);
 fmode_t dm_table_get_mode(struct dm_table *t);
 struct mapped_device *dm_table_get_md(struct dm_table *t);
 const char *dm_table_device_name(struct dm_table *t);

include/uapi/linux/dm-ioctl.h+2−2

@@ -286,9 +286,9 @@ enum { · 285 unmodified lines
286	#define DM_DEV_SET_GEOMETRY _IOWR(DM_IOCTL, DM_DEV_SET_GEOMETRY_CMD, struct dm_ioctl)	286	#define DM_DEV_SET_GEOMETRY _IOWR(DM_IOCTL, DM_DEV_SET_GEOMETRY_CMD, struct dm_ioctl)
287		287
288	#define DM_VERSION_MAJOR 4	288	#define DM_VERSION_MAJOR 4
289	#define DM_VERSION_MINOR 46	289	#define DM_VERSION_MINOR 47
290	#define DM_VERSION_PATCHLEVEL 0	290	#define DM_VERSION_PATCHLEVEL 0
291	#define DM_VERSION_EXTRA "-ioctl (2022-02-22)"	291	#define DM_VERSION_EXTRA "-ioctl (2022-07-28)"
292		292
293	/* Status bits */	293	/* Status bits */
294	#define DM_READONLY_FLAG (1 << 0) /* In/Out */	294	#define DM_READONLY_FLAG (1 << 0) /* In/Out */