Merge tag 'for-6.0/dm-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm

Pull device mapper updates from Mike Snitzer: - Refactor DM core's mempool allocation so that it clearer by not being split acorss files. - Improve DM core's BLK_STS_DM_REQUEUE and BLK_STS_AGAIN handling. - Optimize DM core's more common bio splitting by eliminating the use of bio cloning with bio_split+bio_chain. Shift that cloning cost to the relatively unlikely dm_io requeue case that only occurs during error handling. Introduces dm_io_rewind() that will clone a bio that reflects the subset of the original bio that must be requeued. - Remove DM core's dm_table_get_num_targets() wrapper and audit all dm_table_get_target() callers. - Fix potential for OOM with DM writecache target by setting a default MAX_WRITEBACK_JOBS (set to 256MiB or 1/16 of total system memory, whichever is smaller). - Fix DM writecache target's stats that are reported through DM-specific table info. - Fix use-after-free crash in dm_sm_register_threshold_callback(). - Refine DM core's Persistent Reservation handling in preparation for broader work Mike Christie is doing to add compatibility with Microsoft Windows Failover Cluster. - Fix various KASAN reported bugs in the DM raid target. - Fix DM raid target crash due to md_handle_request() bio splitting that recurses to block core without properly initializing the bio's bi_dev. - Fix some code comment typos and fix some Documentation formatting. * tag 'for-6.0/dm-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm: (29 commits) dm: fix dm-raid crash if md_handle_request() splits bio dm raid: fix address sanitizer warning in raid_resume dm raid: fix address sanitizer warning in raid_status dm: Start pr_preempt from the same starting path dm: Fix PR release handling for non All Registrants dm: Start pr_reserve from the same starting path dm: Allow dm_call_pr to be used for path searches dm: return early from dm_pr_call() if DM device is suspended dm thin: fix use-after-free crash in dm_sm_register_threshold_callback dm writecache: count number of blocks discarded, not number of discard bios dm writecache: count number of blocks written, not number of write bios dm writecache: count number of blocks read, not number of read bios dm writecache: return void from functions dm kcopyd: use __GFP_HIGHMEM when allocating pages dm writecache: set a default MAX_WRITEBACK_JOBS Documentation: dm writecache: Render status list as list Documentation: dm writecache: add blank line before optional parameters dm snapshot: fix typo in snapshot_map() comment dm raid: remove redundant "the" in parse_raid_params() comment dm cache: fix typo in 2 comment blocks ...

Linus Torvalds committed 4 years ago

Commit 8374cfe2 parents c013d0a 9dd1cd322 files · +694 −405Browse files

Unified · Split

22 files

Documentation/admin-guide/device-mapper/writecache.rst +10 −8
drivers/md/Makefile +1 −1
drivers/md/dm-cache-metadata.h +1 −1
drivers/md/dm-cache-target.c +1 −1
drivers/md/dm-core.h +22 −1
drivers/md/dm-ima.c +1 −4
drivers/md/dm-io-rewind.c +166 −0
drivers/md/dm-ioctl.c +3 −3
drivers/md/dm-kcopyd.c +1 −1
drivers/md/dm-raid.c +4 −3
drivers/md/dm-rq.c +0 −1
drivers/md/dm-snap.c +1 −1
drivers/md/dm-table.c +146 −172
drivers/md/dm-thin-metadata.c +5 −2
drivers/md/dm-thin.c +3 −1
drivers/md/dm-verity-target.c +3 −4
drivers/md/dm-writecache.c +25 −18
drivers/md/dm-zone.c +2 −5
drivers/md/dm.c +291 −171
drivers/md/dm.h +0 −4
include/linux/device-mapper.h +6 −1
include/uapi/linux/dm-ioctl.h +2 −2

Documentation/admin-guide/device-mapper/writecache.rst+10−8

@@ -20,6 +20,7 @@ Constructor parameters: · 19 unmodified lines
20	20	size)
21	21	5. the number of optional parameters (the parameters with an argument
22	22	count as two)
	23	+
23	24	start_sector n (default: 0)
24	25	offset from the start of cache device in 512-byte sectors
25	26	high_watermark n (default: 50)
@@ -74,20 +75,21 @@ Constructor parameters: · 48 unmodified lines
74	75	the origin volume in the last n milliseconds
75	76
76	77	Status:
	78	+
77	79	1. error indicator - 0 if there was no error, otherwise error number
78	80	2. the number of blocks
79	81	3. the number of free blocks
80	82	4. the number of blocks under writeback
81		−5. the number of read requests
82		−6. the number of read requests that hit the cache
83		−7. the number of write requests
84		−8. the number of write requests that hit uncommitted block
85		−9. the number of write requests that hit committed block
86		−10. the number of write requests that bypass the cache
87		−11. the number of write requests that are allocated in the cache
	83	+5. the number of read blocks
	84	+6. the number of read blocks that hit the cache
	85	+7. the number of write blocks
	86	+8. the number of write blocks that hit uncommitted block
	87	+9. the number of write blocks that hit committed block
	88	+10. the number of write blocks that bypass the cache
	89	+11. the number of write blocks that are allocated in the cache
88	90	12. the number of write requests that are blocked on the freelist
89	91	13. the number of flush requests
90		−14. the number of discard requests
	92	+14. the number of discarded blocks
91	93
92	94	Messages:
93	95	flush

drivers/md/Makefile+1−1

@@ -5,7 +5,7 @@ · 4 unmodified lines
5	5
6	6	dm-mod-y += dm.o dm-table.o dm-target.o dm-linear.o dm-stripe.o \
7	7	dm-ioctl.o dm-io.o dm-kcopyd.o dm-sysfs.o dm-stats.o \
8		− dm-rq.o
	8	+ dm-rq.o dm-io-rewind.o
9	9	dm-multipath-y += dm-path-selector.o dm-mpath.o
10	10	dm-historical-service-time-y += dm-ps-historical-service-time.o
11	11	dm-io-affinity-y += dm-ps-io-affinity.o

drivers/md/dm-cache-metadata.h+1−1

@@ -131,7 +131,7 @@ void dm_cache_dump(struct dm_cache_metadata *cmd); · 130 unmodified lines
131	131	* hints will be lost.
132	132	*
133	133	* The hints are indexed by the cblock, but many policies will not
134		− * neccessarily have a fast way of accessing efficiently via cblock. So
	134	+ * necessarily have a fast way of accessing efficiently via cblock. So
135	135	* rather than querying the policy for each cblock, we let it walk its data
136	136	* structures and fill in the hints in whatever order it wishes.
137	137	*/

drivers/md/dm-cache-target.c+1−1

@@ -2775,7 +2775,7 @@ static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock, · 2774 unmodified lines
2775	2775
2776	2776	/*
2777	2777	* The discard block size in the on disk metadata is not
2778		− * neccessarily the same as we're currently using. So we have to
	2778	+ * necessarily the same as we're currently using. So we have to
2779	2779	* be careful to only set the discarded attribute if we know it
2780	2780	* covers a complete block of the new size.
2781	2781	*/

drivers/md/dm-core.h+22−1

@@ -22,6 +22,8 @@ · 21 unmodified lines
22	22
23	23	#define DM_RESERVED_MAX_IOS 1024
24	24
	25	+struct dm_io;
	26	+
25	27	struct dm_kobject_holder {
26	28	struct kobject kobj;
27	29	struct completion completion;
@@ -91,6 +93,14 @@ struct mapped_device { · 63 unmodified lines
91	93	spinlock_t deferred_lock;
92	94	struct bio_list deferred;
93	95
	96	+ /*
	97	+ * requeue work context is needed for cloning one new bio
	98	+ * to represent the dm_io to be requeued, since each
	99	+ * dm_io may point to the original bio from FS.
	100	+ */
	101	+ struct work_struct requeue_work;
	102	+ struct dm_io *requeue_list;
	103	+
94	104	void *interface_ptr;
95	105
96	106	/*
@@ -216,6 +226,13 @@ struct dm_table { · 119 unmodified lines
216	226	#endif
217	227	};
218	228
	229	+static inline struct dm_target dm_table_get_target(struct dm_table t,
	230	+ unsigned int index)
	231	+{
	232	+ BUG_ON(index >= t->num_targets);
	233	+ return t->targets + index;
	234	+}
	235	+
219	236	/*
220	237	* One of these is allocated per clone bio.
221	238	*/
@@ -230,6 +247,9 @@ struct dm_target_io { · 8 unmodified lines
230	247	sector_t old_sector;
231	248	struct bio clone;
232	249	};
	250	+#define DM_TARGET_IO_BIO_OFFSET (offsetof(struct dm_target_io, clone))
	251	+#define DM_IO_BIO_OFFSET \
	252	+ (offsetof(struct dm_target_io, clone) + offsetof(struct dm_io, tio))
233	253
234	254	/*
235	255	* dm_target_io flags
@@ -272,7 +292,6 @@ struct dm_io { · 36 unmodified lines
272	292	atomic_t io_count;
273	293	struct mapped_device *md;
274	294
275		− struct bio *split_bio;
276	295	/* The three fields represent mapped part of original bio */
277	296	struct bio *orig_bio;
278	297	unsigned int sector_offset; /* offset to end of orig_bio */
@@ -300,6 +319,8 @@ static inline void dm_io_set_flag(struct dm_io *io, unsigned int bit) · 21 unmodified lines
300	319	io->flags \|= (1U << bit);
301	320	}
302	321
	322	+void dm_io_rewind(struct dm_io io, struct bio_set bs);
	323	+
303	324	static inline struct completion dm_get_completion_from_kobject(struct kobject kobj)
304	325	{
305	326	return &container_of(kobj, struct dm_kobject_holder, kobj)->completion;

drivers/md/dm-ima.c+1−4

@@ -208,7 +208,7 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl · 207 unmodified lines
208	208	if (!target_data_buf)
209	209	goto error;
210	210
211		− num_targets = dm_table_get_num_targets(table);
	211	+ num_targets = table->num_targets;
212	212
213	213	if (dm_ima_alloc_and_copy_device_data(table->md, &device_data_buf, num_targets, noio))
214	214	goto error;
@@ -237,9 +237,6 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl · 22 unmodified lines
237	237	for (i = 0; i < num_targets; i++) {
238	238	struct dm_target *ti = dm_table_get_target(table, i);
239	239
240		− if (!ti)
241		− goto error;
242		−
243	240	last_target_measured = 0;
244	241
245	242	/*

drivers/md/dm-io-rewind.c+166−0

@@ -0,0 +1,166 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+/*
	3	+ * Copyright 2022 Red Hat, Inc.
	4	+ */
	5	+
	6	+#include <linux/bio.h>
	7	+#include <linux/blk-crypto.h>
	8	+#include <linux/blk-integrity.h>
	9	+
	10	+#include "dm-core.h"
	11	+
	12	+static inline bool dm_bvec_iter_rewind(const struct bio_vec *bv,
	13	+ struct bvec_iter *iter,
	14	+ unsigned int bytes)
	15	+{
	16	+ int idx;
	17	+
	18	+ iter->bi_size += bytes;
	19	+ if (bytes <= iter->bi_bvec_done) {
	20	+ iter->bi_bvec_done -= bytes;
	21	+ return true;
	22	+ }
	23	+
	24	+ bytes -= iter->bi_bvec_done;
	25	+ idx = iter->bi_idx - 1;
	26	+
	27	+ while (idx >= 0 && bytes && bytes > bv[idx].bv_len) {
	28	+ bytes -= bv[idx].bv_len;
	29	+ idx--;
	30	+ }
	31	+
	32	+ if (WARN_ONCE(idx < 0 && bytes,
	33	+ "Attempted to rewind iter beyond bvec's boundaries\n")) {
	34	+ iter->bi_size -= bytes;
	35	+ iter->bi_bvec_done = 0;
	36	+ iter->bi_idx = 0;
	37	+ return false;
	38	+ }
	39	+
	40	+ iter->bi_idx = idx;
	41	+ iter->bi_bvec_done = bv[idx].bv_len - bytes;
	42	+ return true;
	43	+}
	44	+
	45	+#if defined(CONFIG_BLK_DEV_INTEGRITY)
	46	+
	47	+/**
	48	+ * dm_bio_integrity_rewind - Rewind integrity vector
	49	+ * @bio: bio whose integrity vector to update
	50	+ * @bytes_done: number of data bytes to rewind
	51	+ *
	52	+ * Description: This function calculates how many integrity bytes the
	53	+ * number of completed data bytes correspond to and rewind the
	54	+ * integrity vector accordingly.
	55	+ */
	56	+static void dm_bio_integrity_rewind(struct bio *bio, unsigned int bytes_done)
	57	+{
	58	+ struct bio_integrity_payload *bip = bio_integrity(bio);
	59	+ struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
	60	+ unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9);
	61	+
	62	+ bip->bip_iter.bi_sector -= bio_integrity_intervals(bi, bytes_done >> 9);
	63	+ dm_bvec_iter_rewind(bip->bip_vec, &bip->bip_iter, bytes);
	64	+}
	65	+
	66	+#else /* CONFIG_BLK_DEV_INTEGRITY */
	67	+
	68	+static inline void dm_bio_integrity_rewind(struct bio *bio,
	69	+ unsigned int bytes_done)
	70	+{
	71	+ return;
	72	+}
	73	+
	74	+#endif
	75	+
	76	+#if defined(CONFIG_BLK_INLINE_ENCRYPTION)
	77	+
	78	+/* Decrements @dun by @dec, treating @dun as a multi-limb integer. */
	79	+static void dm_bio_crypt_dun_decrement(u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE],
	80	+ unsigned int dec)
	81	+{
	82	+ int i;
	83	+
	84	+ for (i = 0; dec && i < BLK_CRYPTO_DUN_ARRAY_SIZE; i++) {
	85	+ u64 prev = dun[i];
	86	+
	87	+ dun[i] -= dec;
	88	+ if (dun[i] > prev)
	89	+ dec = 1;
	90	+ else
	91	+ dec = 0;
	92	+ }
	93	+}
	94	+
	95	+static void dm_bio_crypt_rewind(struct bio *bio, unsigned int bytes)
	96	+{
	97	+ struct bio_crypt_ctx *bc = bio->bi_crypt_context;
	98	+
	99	+ dm_bio_crypt_dun_decrement(bc->bc_dun,
	100	+ bytes >> bc->bc_key->data_unit_size_bits);
	101	+}
	102	+
	103	+#else /* CONFIG_BLK_INLINE_ENCRYPTION */
	104	+
	105	+static inline void dm_bio_crypt_rewind(struct bio *bio, unsigned int bytes)
	106	+{
	107	+ return;
	108	+}
	109	+
	110	+#endif
	111	+
	112	+static inline void dm_bio_rewind_iter(const struct bio *bio,
	113	+ struct bvec_iter *iter, unsigned int bytes)
	114	+{
	115	+ iter->bi_sector -= bytes >> 9;
	116	+
	117	+ /* No advance means no rewind */
	118	+ if (bio_no_advance_iter(bio))
	119	+ iter->bi_size += bytes;
	120	+ else
	121	+ dm_bvec_iter_rewind(bio->bi_io_vec, iter, bytes);
	122	+}
	123	+
	124	+/**
	125	+ * dm_bio_rewind - update ->bi_iter of @bio by rewinding @bytes.
	126	+ * @bio: bio to rewind
	127	+ * @bytes: how many bytes to rewind
	128	+ *
	129	+ * WARNING:
	130	+ * Caller must ensure that @bio has a fixed end sector, to allow
	131	+ * rewinding from end of bio and restoring its original position.
	132	+ * Caller is also responsibile for restoring bio's size.
	133	+ */
	134	+static void dm_bio_rewind(struct bio *bio, unsigned bytes)
	135	+{
	136	+ if (bio_integrity(bio))
	137	+ dm_bio_integrity_rewind(bio, bytes);
	138	+
	139	+ if (bio_has_crypt_ctx(bio))
	140	+ dm_bio_crypt_rewind(bio, bytes);
	141	+
	142	+ dm_bio_rewind_iter(bio, &bio->bi_iter, bytes);
	143	+}
	144	+
	145	+void dm_io_rewind(struct dm_io io, struct bio_set bs)
	146	+{
	147	+ struct bio *orig = io->orig_bio;
	148	+ struct bio *new_orig = bio_alloc_clone(orig->bi_bdev, orig,
	149	+ GFP_NOIO, bs);
	150	+ /*
	151	+ * dm_bio_rewind can restore to previous position since the
	152	+ * end sector is fixed for original bio, but we still need
	153	+ * to restore bio's size manually (using io->sectors).
	154	+ */
	155	+ dm_bio_rewind(new_orig, ((io->sector_offset << 9) -
	156	+ orig->bi_iter.bi_size));
	157	+ bio_trim(new_orig, 0, io->sectors);
	158	+
	159	+ bio_chain(new_orig, orig);
	160	+ /*
	161	+ * __bi_remaining was increased (by dm_split_and_process_bio),
	162	+ * so must drop the one added in bio_chain.
	163	+ */
	164	+ atomic_dec(&orig->__bi_remaining);
	165	+ io->orig_bio = new_orig;
	166	+}

drivers/md/dm-ioctl.c+3−3

@@ -832,7 +832,7 @@ static void __dev_status(struct mapped_device md, struct dm_ioctl param) · 831 unmodified lines
832	832	if (!(param->flags & DM_QUERY_INACTIVE_TABLE_FLAG)) {
833	833	if (get_disk_ro(disk))
834	834	param->flags \|= DM_READONLY_FLAG;
835		− param->target_count = dm_table_get_num_targets(table);
	835	+ param->target_count = table->num_targets;
836	836	}
837	837
838	838	param->flags \|= DM_ACTIVE_PRESENT_FLAG;
@@ -845,7 +845,7 @@ static void __dev_status(struct mapped_device md, struct dm_ioctl param) · 6 unmodified lines
845	845	if (table) {
846	846	if (!(dm_table_get_mode(table) & FMODE_WRITE))
847	847	param->flags \|= DM_READONLY_FLAG;
848		− param->target_count = dm_table_get_num_targets(table);
	848	+ param->target_count = table->num_targets;
849	849	}
850	850	dm_put_live_table(md, srcu_idx);
851	851	}
@@ -1248,7 +1248,7 @@ static void retrieve_status(struct dm_table *table, · 396 unmodified lines
1248	1248	type = STATUSTYPE_INFO;
1249	1249
1250	1250	/* Get all the target info */
1251		− num_targets = dm_table_get_num_targets(table);
	1251	+ num_targets = table->num_targets;
1252	1252	for (i = 0; i < num_targets; i++) {
1253	1253	struct dm_target *ti = dm_table_get_target(table, i);
1254	1254	size_t l;

drivers/md/dm-kcopyd.c+1−1

@@ -219,7 +219,7 @@ static struct page_list *alloc_pl(gfp_t gfp) · 218 unmodified lines
219	219	if (!pl)
220	220	return NULL;
221	221
222		− pl->page = alloc_page(gfp);
	222	+ pl->page = alloc_page(gfp \| __GFP_HIGHMEM);
223	223	if (!pl->page) {
224	224	kfree(pl);
225	225	return NULL;

drivers/md/dm-raid.c+4−3

@@ -1369,7 +1369,7 @@ static int parse_raid_params(struct raid_set rs, struct dm_arg_set as, · 1368 unmodified lines
1369	1369	}
1370	1370	rs->md.bitmap_info.daemon_sleep = value;
1371	1371	} else if (!strcasecmp(key, dm_raid_arg_name_by_flag(CTR_FLAG_DATA_OFFSET))) {
1372		− /* Userspace passes new data_offset after having extended the the data image LV */
	1372	+ /* Userspace passes new data_offset after having extended the data image LV */
1373	1373	if (test_and_set_bit(__CTR_FLAG_DATA_OFFSET, &rs->ctr_flags)) {
1374	1374	rs->ti->error = "Only one data_offset argument pair allowed";
1375	1375	return -EINVAL;
@@ -3097,6 +3097,7 @@ static int raid_ctr(struct dm_target ti, unsigned int argc, char *argv) · 1721 unmodified lines
3097	3097	INIT_WORK(&rs->md.event_work, do_table_event);
3098	3098	ti->private = rs;
3099	3099	ti->num_flush_bios = 1;
	3100	+ ti->needs_bio_set_dev = true;
3100	3101
3101	3102	/* Restore any requested new layout for conversion decision */
3102	3103	rs_config_restore(rs, &rs_layout);
@@ -3509,7 +3510,7 @@ static void raid_status(struct dm_target *ti, status_type_t type, · 406 unmodified lines
3509	3510	{
3510	3511	struct raid_set *rs = ti->private;
3511	3512	struct mddev *mddev = &rs->md;
3512		− struct r5conf *conf = mddev->private;
	3513	+ struct r5conf *conf = rs_is_raid456(rs) ? mddev->private : NULL;
3513	3514	int i, max_nr_stripes = conf ? conf->max_nr_stripes : 0;
3514	3515	unsigned long recovery;
3515	3516	unsigned int raid_param_cnt = 1; /* at least 1 for chunksize */
@@ -3819,7 +3820,7 @@ static void attempt_restore_of_faulty_devices(struct raid_set *rs) · 303 unmodified lines
3819	3820
3820	3821	memset(cleared_failed_devices, 0, sizeof(cleared_failed_devices));
3821	3822
3822		− for (i = 0; i < mddev->raid_disks; i++) {
	3823	+ for (i = 0; i < rs->raid_disks; i++) {
3823	3824	r = &rs->dev[i].rdev;
3824	3825	/* HM FIXME: enhance journal device recovery processing */
3825	3826	if (test_bit(Journal, &r->flags))

drivers/md/dm-rq.c+0−1

@@ -43,7 +43,6 @@ unsigned dm_get_reserved_rq_based_ios(void) · 42 unmodified lines
43	43	return __dm_get_module_param(&reserved_rq_based_ios,
44	44	RESERVED_REQUEST_BASED_IOS, DM_RESERVED_MAX_IOS);
45	45	}
46		−EXPORT_SYMBOL_GPL(dm_get_reserved_rq_based_ios);
47	46
48	47	static unsigned dm_get_blk_mq_nr_hw_queues(void)
49	48	{

drivers/md/dm-snap.c+1−1

@@ -2026,7 +2026,7 @@ static int snapshot_map(struct dm_target ti, struct bio bio) · 2025 unmodified lines
2026	2026	/*
2027	2027	* Write to snapshot - higher level takes care of RW/RO
2028	2028	* flags so we should only get this if we are
2029		− * writeable.
	2029	+ * writable.
2030	2030	*/
2031	2031	if (bio_data_dir(bio) == WRITE) {
2032	2032	pe = __lookup_pending_exception(s, chunk);

drivers/md/dm-table.c+146−172

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6	*/
7	7
8	8	#include "dm-core.h"
	9	+#include "dm-rq.h"
9	10
10	11	#include <linux/module.h>
11	12	#include <linux/vmalloc.h>
@@ -174,8 +175,6 @@ static void dm_table_destroy_crypto_profile(struct dm_table *t); · 162 unmodified lines
174	175
175	176	void dm_table_destroy(struct dm_table *t)
176	177	{
177		− unsigned int i;
178		−
179	178	if (!t)
180	179	return;
181	180
@@ -184,13 +183,13 @@ void dm_table_destroy(struct dm_table *t) · 2 unmodified lines
184	183	kvfree(t->index[t->depth - 2]);
185	184
186	185	/* free the targets */
187		− for (i = 0; i < t->num_targets; i++) {
188		− struct dm_target *tgt = t->targets + i;
	186	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	187	+ struct dm_target *ti = dm_table_get_target(t, i);
189	188
190		− if (tgt->type->dtr)
191		− tgt->type->dtr(tgt);
	189	+ if (ti->type->dtr)
	190	+ ti->type->dtr(ti);
192	191
193		− dm_put_target_type(tgt->type);
	192	+ dm_put_target_type(ti->type);
194	193	}
195	194
196	195	kvfree(t->highs);
@@ -450,14 +449,14 @@ EXPORT_SYMBOL(dm_put_device); · 253 unmodified lines
450	449	/*
451	450	* Checks to see if the target joins onto the end of the table.
452	451	*/
453		−static int adjoin(struct dm_table table, struct dm_target ti)
	452	+static int adjoin(struct dm_table t, struct dm_target ti)
454	453	{
455	454	struct dm_target *prev;
456	455
457		− if (!table->num_targets)
	456	+ if (!t->num_targets)
458	457	return !ti->begin;
459	458
460		− prev = &table->targets[table->num_targets - 1];
	459	+ prev = &t->targets[t->num_targets - 1];
461	460	return (ti->begin == (prev->begin + prev->len));
462	461	}
463	462
@@ -564,8 +563,8 @@ int dm_split_args(int argc, char *argvp, char input) · 100 unmodified lines
564	563	* two or more targets, the size of each piece it gets split into must
565	564	* be compatible with the logical_block_size of the target processing it.
566	565	*/
567		−static int validate_hardware_logical_block_alignment(struct dm_table *table,
568		− struct queue_limits *limits)
	566	+static int validate_hardware_logical_block_alignment(struct dm_table *t,
	567	+ struct queue_limits *limits)
569	568	{
570	569	/*
571	570	* This function uses arithmetic modulo the logical_block_size
@@ -587,13 +586,13 @@ static int validate_hardware_logical_block_alignment(struct dm_table *table, · 15 unmodified lines
587	586
588	587	struct dm_target *ti;
589	588	struct queue_limits ti_limits;
590		− unsigned i;
	589	+ unsigned int i;
591	590
592	591	/*
593	592	* Check each entry in the table in turn.
594	593	*/
595		− for (i = 0; i < dm_table_get_num_targets(table); i++) {
596		− ti = dm_table_get_target(table, i);
	594	+ for (i = 0; i < t->num_targets; i++) {
	595	+ ti = dm_table_get_target(t, i);
597	596
598	597	blk_set_stacking_limits(&ti_limits);
599	598
@@ -621,7 +620,7 @@ static int validate_hardware_logical_block_alignment(struct dm_table *table, · 21 unmodified lines
621	620	if (remaining) {
622	621	DMWARN("%s: table line %u (start sect %llu len %llu) "
623	622	"not aligned to h/w logical block size %u",
624		− dm_device_name(table->md), i,
	623	+ dm_device_name(t->md), i,
625	624	(unsigned long long) ti->begin,
626	625	(unsigned long long) ti->len,
627	626	limits->logical_block_size);
@@ -636,7 +635,7 @@ int dm_table_add_target(struct dm_table t, const char type, · 8 unmodified lines
636	635	{
637	636	int r = -EINVAL, argc;
638	637	char **argv;
639		− struct dm_target *tgt;
	638	+ struct dm_target *ti;
640	639
641	640	if (t->singleton) {
642	641	DMERR("%s: target type %s must appear alone in table",
@@ -646,87 +645,87 @@ int dm_table_add_target(struct dm_table t, const char type, · 3 unmodified lines
646	645
647	646	BUG_ON(t->num_targets >= t->num_allocated);
648	647
649		− tgt = t->targets + t->num_targets;
650		− memset(tgt, 0, sizeof(*tgt));
	648	+ ti = t->targets + t->num_targets;
	649	+ memset(ti, 0, sizeof(*ti));
651	650
652	651	if (!len) {
653	652	DMERR("%s: zero-length target", dm_device_name(t->md));
654	653	return -EINVAL;
655	654	}
656	655
657		− tgt->type = dm_get_target_type(type);
658		− if (!tgt->type) {
	656	+ ti->type = dm_get_target_type(type);
	657	+ if (!ti->type) {
659	658	DMERR("%s: %s: unknown target type", dm_device_name(t->md), type);
660	659	return -EINVAL;
661	660	}
662	661
663		− if (dm_target_needs_singleton(tgt->type)) {
	662	+ if (dm_target_needs_singleton(ti->type)) {
664	663	if (t->num_targets) {
665		− tgt->error = "singleton target type must appear alone in table";
	664	+ ti->error = "singleton target type must appear alone in table";
666	665	goto bad;
667	666	}
668	667	t->singleton = true;
669	668	}
670	669
671		− if (dm_target_always_writeable(tgt->type) && !(t->mode & FMODE_WRITE)) {
672		− tgt->error = "target type may not be included in a read-only table";
	670	+ if (dm_target_always_writeable(ti->type) && !(t->mode & FMODE_WRITE)) {
	671	+ ti->error = "target type may not be included in a read-only table";
673	672	goto bad;
674	673	}
675	674
676	675	if (t->immutable_target_type) {
677		− if (t->immutable_target_type != tgt->type) {
678		− tgt->error = "immutable target type cannot be mixed with other target types";
	676	+ if (t->immutable_target_type != ti->type) {
	677	+ ti->error = "immutable target type cannot be mixed with other target types";
679	678	goto bad;
680	679	}
681		− } else if (dm_target_is_immutable(tgt->type)) {
	680	+ } else if (dm_target_is_immutable(ti->type)) {
682	681	if (t->num_targets) {
683		− tgt->error = "immutable target type cannot be mixed with other target types";
	682	+ ti->error = "immutable target type cannot be mixed with other target types";
684	683	goto bad;
685	684	}
686		− t->immutable_target_type = tgt->type;
	685	+ t->immutable_target_type = ti->type;
687	686	}
688	687
689		− if (dm_target_has_integrity(tgt->type))
	688	+ if (dm_target_has_integrity(ti->type))
690	689	t->integrity_added = 1;
691	690
692		− tgt->table = t;
693		− tgt->begin = start;
694		− tgt->len = len;
695		− tgt->error = "Unknown error";
	691	+ ti->table = t;
	692	+ ti->begin = start;
	693	+ ti->len = len;
	694	+ ti->error = "Unknown error";
696	695
697	696	/*
698	697	* Does this target adjoin the previous one ?
699	698	*/
700		− if (!adjoin(t, tgt)) {
701		− tgt->error = "Gap in table";
	699	+ if (!adjoin(t, ti)) {
	700	+ ti->error = "Gap in table";
702	701	goto bad;
703	702	}
704	703
705	704	r = dm_split_args(&argc, &argv, params);
706	705	if (r) {
707		− tgt->error = "couldn't split parameters";
	706	+ ti->error = "couldn't split parameters";
708	707	goto bad;
709	708	}
710	709
711		− r = tgt->type->ctr(tgt, argc, argv);
	710	+ r = ti->type->ctr(ti, argc, argv);
712	711	kfree(argv);
713	712	if (r)
714	713	goto bad;
715	714
716		− t->highs[t->num_targets++] = tgt->begin + tgt->len - 1;
	715	+ t->highs[t->num_targets++] = ti->begin + ti->len - 1;
717	716
718		− if (!tgt->num_discard_bios && tgt->discards_supported)
	717	+ if (!ti->num_discard_bios && ti->discards_supported)
719	718	DMWARN("%s: %s: ignoring discards_supported because num_discard_bios is zero.",
720	719	dm_device_name(t->md), type);
721	720
722		− if (tgt->limit_swap_bios && !static_key_enabled(&swap_bios_enabled.key))
	721	+ if (ti->limit_swap_bios && !static_key_enabled(&swap_bios_enabled.key))
723	722	static_branch_enable(&swap_bios_enabled);
724	723
725	724	return 0;
726	725
727	726	bad:
728		− DMERR("%s: %s: %s (%pe)", dm_device_name(t->md), type, tgt->error, ERR_PTR(r));
729		− dm_put_target_type(tgt->type);
	727	+ DMERR("%s: %s: %s (%pe)", dm_device_name(t->md), type, ti->error, ERR_PTR(r));
	728	+ dm_put_target_type(ti->type);
730	729	return r;
731	730	}
732	731
@@ -825,14 +824,11 @@ static int device_not_dax_synchronous_capable(struct dm_target *ti, struct dm_de · 92 unmodified lines
825	824	}
826	825
827	826	static bool dm_table_supports_dax(struct dm_table *t,
828		− iterate_devices_callout_fn iterate_fn)
	827	+ iterate_devices_callout_fn iterate_fn)
829	828	{
830		− struct dm_target *ti;
831		− unsigned i;
832		−
833	829	/* Ensure that all targets support DAX. */
834		− for (i = 0; i < dm_table_get_num_targets(t); i++) {
835		− ti = dm_table_get_target(t, i);
	830	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	831	+ struct dm_target *ti = dm_table_get_target(t, i);
836	832
837	833	if (!ti->type->direct_access)
838	834	return false;
@@ -860,9 +856,8 @@ static int device_is_rq_stackable(struct dm_target ti, struct dm_dev dev, · 21 unmodified lines
860	856
861	857	static int dm_table_determine_type(struct dm_table *t)
862	858	{
863		− unsigned i;
864	859	unsigned bio_based = 0, request_based = 0, hybrid = 0;
865		− struct dm_target *tgt;
	860	+ struct dm_target *ti;
866	861	struct list_head *devices = dm_table_get_devices(t);
867	862	enum dm_queue_mode live_md_type = dm_get_md_type(t->md);
868	863
@@ -876,11 +871,11 @@ static int dm_table_determine_type(struct dm_table *t) · 7 unmodified lines
876	871	goto verify_rq_based;
877	872	}
878	873
879		− for (i = 0; i < t->num_targets; i++) {
880		− tgt = t->targets + i;
881		− if (dm_target_hybrid(tgt))
	874	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	875	+ ti = dm_table_get_target(t, i);
	876	+ if (dm_target_hybrid(ti))
882	877	hybrid = 1;
883		− else if (dm_target_request_based(tgt))
	878	+ else if (dm_target_request_based(ti))
884	879	request_based = 1;
885	880	else
886	881	bio_based = 1;
@@ -942,18 +937,18 @@ verify_rq_based: · 55 unmodified lines
942	937	return 0;
943	938	}
944	939
945		− tgt = dm_table_get_immutable_target(t);
946		− if (!tgt) {
	940	+ ti = dm_table_get_immutable_target(t);
	941	+ if (!ti) {
947	942	DMERR("table load rejected: immutable target is required");
948	943	return -EINVAL;
949		− } else if (tgt->max_io_len) {
	944	+ } else if (ti->max_io_len) {
950	945	DMERR("table load rejected: immutable target that splits IO is not supported");
951	946	return -EINVAL;
952	947	}
953	948
954	949	/* Non-request-stackable devices can't be used for request-based dm */
955		− if (!tgt->type->iterate_devices \|\|
956		− !tgt->type->iterate_devices(tgt, device_is_rq_stackable, NULL)) {
	950	+ if (!ti->type->iterate_devices \|\|
	951	+ !ti->type->iterate_devices(ti, device_is_rq_stackable, NULL)) {
957	952	DMERR("table load rejected: including non-request-stackable devices");
958	953	return -EINVAL;
959	954	}
@@ -983,11 +978,9 @@ struct dm_target dm_table_get_immutable_target(struct dm_table t) · 23 unmodified lines
983	978
984	979	struct dm_target dm_table_get_wildcard_target(struct dm_table t)
985	980	{
986		− struct dm_target *ti;
987		− unsigned i;
	981	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	982	+ struct dm_target *ti = dm_table_get_target(t, i);
988	983
989		− for (i = 0; i < dm_table_get_num_targets(t); i++) {
990		− ti = dm_table_get_target(t, i);
991	984	if (dm_target_is_wildcard(ti->type))
992	985	return ti;
993	986	}
@@ -1010,32 +1003,56 @@ static bool dm_table_supports_poll(struct dm_table *t); · 16 unmodified lines
1010	1003	static int dm_table_alloc_md_mempools(struct dm_table t, struct mapped_device md)
1011	1004	{
1012	1005	enum dm_queue_mode type = dm_table_get_type(t);
1013		− unsigned per_io_data_size = 0;
1014		− unsigned min_pool_size = 0;
1015		− struct dm_target *ti;
1016		− unsigned i;
1017		− bool poll_supported = false;
	1006	+ unsigned int per_io_data_size = 0, front_pad, io_front_pad;
	1007	+ unsigned int min_pool_size = 0, pool_size;
	1008	+ struct dm_md_mempools *pools;
1018	1009
1019	1010	if (unlikely(type == DM_TYPE_NONE)) {
1020	1011	DMWARN("no table type is set, can't allocate mempools");
1021	1012	return -EINVAL;
1022	1013	}
1023	1014
1024		− if (__table_type_bio_based(type)) {
1025		− for (i = 0; i < t->num_targets; i++) {
1026		− ti = t->targets + i;
1027		− per_io_data_size = max(per_io_data_size, ti->per_io_data_size);
1028		− min_pool_size = max(min_pool_size, ti->num_flush_bios);
1029		− }
1030		− poll_supported = dm_table_supports_poll(t);
	1015	+ pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
	1016	+ if (!pools)
	1017	+ return -ENOMEM;
	1018	+
	1019	+ if (type == DM_TYPE_REQUEST_BASED) {
	1020	+ pool_size = dm_get_reserved_rq_based_ios();
	1021	+ front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
	1022	+ goto init_bs;
1031	1023	}
1032	1024
1033		− t->mempools = dm_alloc_md_mempools(md, type, per_io_data_size, min_pool_size,
1034		− t->integrity_supported, poll_supported);
1035		− if (!t->mempools)
1036		− return -ENOMEM;
	1025	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	1026	+ struct dm_target *ti = dm_table_get_target(t, i);
1037	1027
	1028	+ per_io_data_size = max(per_io_data_size, ti->per_io_data_size);
	1029	+ min_pool_size = max(min_pool_size, ti->num_flush_bios);
	1030	+ }
	1031	+ pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
	1032	+ front_pad = roundup(per_io_data_size,
	1033	+ __alignof__(struct dm_target_io)) + DM_TARGET_IO_BIO_OFFSET;
	1034	+
	1035	+ io_front_pad = roundup(per_io_data_size,
	1036	+ __alignof__(struct dm_io)) + DM_IO_BIO_OFFSET;
	1037	+ if (bioset_init(&pools->io_bs, pool_size, io_front_pad,
	1038	+ dm_table_supports_poll(t) ? BIOSET_PERCPU_CACHE : 0))
	1039	+ goto out_free_pools;
	1040	+ if (t->integrity_supported &&
	1041	+ bioset_integrity_create(&pools->io_bs, pool_size))
	1042	+ goto out_free_pools;
	1043	+init_bs:
	1044	+ if (bioset_init(&pools->bs, pool_size, front_pad, 0))
	1045	+ goto out_free_pools;
	1046	+ if (t->integrity_supported &&
	1047	+ bioset_integrity_create(&pools->bs, pool_size))
	1048	+ goto out_free_pools;
	1049	+
	1050	+ t->mempools = pools;
1038	1051	return 0;
	1052	+
	1053	+out_free_pools:
	1054	+ dm_free_md_mempools(pools);
	1055	+ return -ENOMEM;
1039	1056	}
1040	1057
1041	1058	static int setup_indexes(struct dm_table *t)
@@ -1100,10 +1117,10 @@ static struct gendisk * dm_table_get_integrity_disk(struct dm_table *t) · 58 unmodified lines
1100	1117	struct list_head *devices = dm_table_get_devices(t);
1101	1118	struct dm_dev_internal *dd = NULL;
1102	1119	struct gendisk prev_disk = NULL, template_disk = NULL;
1103		− unsigned i;
1104	1120
1105		− for (i = 0; i < dm_table_get_num_targets(t); i++) {
	1121	+ for (unsigned int i = 0; i < t->num_targets; i++) {
1106	1122	struct dm_target *ti = dm_table_get_target(t, i);
	1123	+
1107	1124	if (!dm_target_passes_integrity(ti->type))
1108	1125	goto no_integrity;
1109	1126	}
@@ -1217,18 +1234,19 @@ static int dm_keyslot_evict(struct blk_crypto_profile *profile, · 107 unmodified lines
1217	1234	struct dm_keyslot_evict_args args = { key };
1218	1235	struct dm_table *t;
1219	1236	int srcu_idx;
1220		− int i;
1221		− struct dm_target *ti;
1222	1237
1223	1238	t = dm_get_live_table(md, &srcu_idx);
1224	1239	if (!t)
1225	1240	return 0;
1226		− for (i = 0; i < dm_table_get_num_targets(t); i++) {
1227		− ti = dm_table_get_target(t, i);
	1241	+
	1242	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	1243	+ struct dm_target *ti = dm_table_get_target(t, i);
	1244	+
1228	1245	if (!ti->type->iterate_devices)
1229	1246	continue;
1230	1247	ti->type->iterate_devices(ti, dm_keyslot_evict_callback, &args);
1231	1248	}
	1249	+
1232	1250	dm_put_live_table(md, srcu_idx);
1233	1251	return args.err;
1234	1252	}
@@ -1277,7 +1295,6 @@ static int dm_table_construct_crypto_profile(struct dm_table *t) · 42 unmodified lines
1277	1295	{
1278	1296	struct dm_crypto_profile *dmcp;
1279	1297	struct blk_crypto_profile *profile;
1280		− struct dm_target *ti;
1281	1298	unsigned int i;
1282	1299	bool empty_profile = true;
1283	1300
@@ -1293,8 +1310,8 @@ static int dm_table_construct_crypto_profile(struct dm_table *t) · 9 unmodified lines
1293	1310	memset(profile->modes_supported, 0xFF,
1294	1311	sizeof(profile->modes_supported));
1295	1312
1296		− for (i = 0; i < dm_table_get_num_targets(t); i++) {
1297		− ti = dm_table_get_target(t, i);
	1313	+ for (i = 0; i < t->num_targets; i++) {
	1314	+ struct dm_target *ti = dm_table_get_target(t, i);
1298	1315
1299	1316	if (!dm_target_passes_crypto(ti->type)) {
1300	1317	blk_crypto_intersect_capabilities(profile, NULL);
@@ -1444,14 +1461,6 @@ inline sector_t dm_table_get_size(struct dm_table *t) · 143 unmodified lines
1444	1461	}
1445	1462	EXPORT_SYMBOL(dm_table_get_size);
1446	1463
1447		−struct dm_target dm_table_get_target(struct dm_table t, unsigned int index)
1448		−{
1449		− if (index >= t->num_targets)
1450		− return NULL;
1451		−
1452		− return t->targets + index;
1453		−}
1454		−
1455	1464	/*
1456	1465	* Search the btree for the correct target.
1457	1466	*
@@ -1512,11 +1521,8 @@ static int device_not_poll_capable(struct dm_target ti, struct dm_dev dev, · 54 unmodified lines
1512	1521	static bool dm_table_any_dev_attr(struct dm_table *t,
1513	1522	iterate_devices_callout_fn func, void *data)
1514	1523	{
1515		− struct dm_target *ti;
1516		− unsigned int i;
1517		−
1518		− for (i = 0; i < dm_table_get_num_targets(t); i++) {
1519		− ti = dm_table_get_target(t, i);
	1524	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	1525	+ struct dm_target *ti = dm_table_get_target(t, i);
1520	1526
1521	1527	if (ti->type->iterate_devices &&
1522	1528	ti->type->iterate_devices(ti, func, data))
@@ -1538,11 +1544,8 @@ static int count_device(struct dm_target ti, struct dm_dev dev, · 15 unmodified lines
1538	1544
1539	1545	static bool dm_table_supports_poll(struct dm_table *t)
1540	1546	{
1541		− struct dm_target *ti;
1542		− unsigned i = 0;
1543		−
1544		− while (i < dm_table_get_num_targets(t)) {
1545		− ti = dm_table_get_target(t, i++);
	1547	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	1548	+ struct dm_target *ti = dm_table_get_target(t, i);
1546	1549
1547	1550	if (!ti->type->iterate_devices \|\|
1548	1551	ti->type->iterate_devices(ti, device_not_poll_capable, NULL))
@@ -1558,18 +1561,15 @@ static bool dm_table_supports_poll(struct dm_table *t) · 9 unmodified lines
1558	1561	* Returns false if the result is unknown because a target doesn't
1559	1562	* support iterate_devices.
1560	1563	*/
1561		−bool dm_table_has_no_data_devices(struct dm_table *table)
	1564	+bool dm_table_has_no_data_devices(struct dm_table *t)
1562	1565	{
1563		− struct dm_target *ti;
1564		− unsigned i, num_devices;
1565		−
1566		− for (i = 0; i < dm_table_get_num_targets(table); i++) {
1567		− ti = dm_table_get_target(table, i);
	1566	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	1567	+ struct dm_target *ti = dm_table_get_target(t, i);
	1568	+ unsigned num_devices = 0;
1568	1569
1569	1570	if (!ti->type->iterate_devices)
1570	1571	return false;
1571	1572
1572		− num_devices = 0;
1573	1573	ti->type->iterate_devices(ti, count_device, &num_devices);
1574	1574	if (num_devices)
1575	1575	return false;
@@ -1597,11 +1597,8 @@ static int device_not_zoned_model(struct dm_target ti, struct dm_dev dev, · 21 unmodified lines
1597	1597	static bool dm_table_supports_zoned_model(struct dm_table *t,
1598	1598	enum blk_zoned_model zoned_model)
1599	1599	{
1600		− struct dm_target *ti;
1601		− unsigned i;
1602		−
1603		− for (i = 0; i < dm_table_get_num_targets(t); i++) {
1604		− ti = dm_table_get_target(t, i);
	1600	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	1601	+ struct dm_target *ti = dm_table_get_target(t, i);
1605	1602
1606	1603	if (dm_target_supports_zoned_hm(ti->type)) {
1607	1604	if (!ti->type->iterate_devices \|\|
@@ -1632,16 +1629,16 @@ static int device_not_matches_zone_sectors(struct dm_target ti, struct dm_dev · 24 unmodified lines
1632	1629	* zone sectors, if the destination device is a zoned block device, it shall
1633	1630	* have the specified zone_sectors.
1634	1631	*/
1635		−static int validate_hardware_zoned_model(struct dm_table *table,
	1632	+static int validate_hardware_zoned_model(struct dm_table *t,
1636	1633	enum blk_zoned_model zoned_model,
1637	1634	unsigned int zone_sectors)
1638	1635	{
1639	1636	if (zoned_model == BLK_ZONED_NONE)
1640	1637	return 0;
1641	1638
1642		− if (!dm_table_supports_zoned_model(table, zoned_model)) {
	1639	+ if (!dm_table_supports_zoned_model(t, zoned_model)) {
1643	1640	DMERR("%s: zoned model is not consistent across all devices",
1644		− dm_device_name(table->md));
	1641	+ dm_device_name(t->md));
1645	1642	return -EINVAL;
1646	1643	}
1647	1644
@@ -1649,9 +1646,9 @@ static int validate_hardware_zoned_model(struct dm_table *table, · 1 unmodified line
1649	1646	if (!zone_sectors \|\| !is_power_of_2(zone_sectors))
1650	1647	return -EINVAL;
1651	1648
1652		− if (dm_table_any_dev_attr(table, device_not_matches_zone_sectors, &zone_sectors)) {
	1649	+ if (dm_table_any_dev_attr(t, device_not_matches_zone_sectors, &zone_sectors)) {
1653	1650	DMERR("%s: zone sectors is not consistent across all zoned devices",
1654		− dm_device_name(table->md));
	1651	+ dm_device_name(t->md));
1655	1652	return -EINVAL;
1656	1653	}
1657	1654
@@ -1661,21 +1658,19 @@ static int validate_hardware_zoned_model(struct dm_table *table, · 3 unmodified lines
1661	1658	/*
1662	1659	* Establish the new table's queue_limits and validate them.
1663	1660	*/
1664		−int dm_calculate_queue_limits(struct dm_table *table,
	1661	+int dm_calculate_queue_limits(struct dm_table *t,
1665	1662	struct queue_limits *limits)
1666	1663	{
1667		− struct dm_target *ti;
1668	1664	struct queue_limits ti_limits;
1669		− unsigned i;
1670	1665	enum blk_zoned_model zoned_model = BLK_ZONED_NONE;
1671	1666	unsigned int zone_sectors = 0;
1672	1667
1673	1668	blk_set_stacking_limits(limits);
1674	1669
1675		− for (i = 0; i < dm_table_get_num_targets(table); i++) {
1676		− blk_set_stacking_limits(&ti_limits);
	1670	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	1671	+ struct dm_target *ti = dm_table_get_target(t, i);
1677	1672
1678		− ti = dm_table_get_target(table, i);
	1673	+ blk_set_stacking_limits(&ti_limits);
1679	1674
1680	1675	if (!ti->type->iterate_devices)
1681	1676	goto combine_limits;
@@ -1716,7 +1711,7 @@ combine_limits: · 34 unmodified lines
1716	1711	DMWARN("%s: adding target device "
1717	1712	"(start sect %llu len %llu) "
1718	1713	"caused an alignment inconsistency",
1719		− dm_device_name(table->md),
	1714	+ dm_device_name(t->md),
1720	1715	(unsigned long long) ti->begin,
1721	1716	(unsigned long long) ti->len);
1722	1717	}
@@ -1736,10 +1731,10 @@ combine_limits: · 13 unmodified lines
1736	1731	zoned_model = limits->zoned;
1737	1732	zone_sectors = limits->chunk_sectors;
1738	1733	}
1739		− if (validate_hardware_zoned_model(table, zoned_model, zone_sectors))
	1734	+ if (validate_hardware_zoned_model(t, zoned_model, zone_sectors))
1740	1735	return -EINVAL;
1741	1736
1742		− return validate_hardware_logical_block_alignment(table, limits);
	1737	+ return validate_hardware_logical_block_alignment(t, limits);
1743	1738	}
1744	1739
1745	1740	/*
@@ -1783,17 +1778,14 @@ static int device_flush_capable(struct dm_target ti, struct dm_dev dev, · 37 unmodified lines
1783	1778
1784	1779	static bool dm_table_supports_flush(struct dm_table *t, unsigned long flush)
1785	1780	{
1786		− struct dm_target *ti;
1787		− unsigned i;
1788		−
1789	1781	/*
1790	1782	* Require at least one underlying device to support flushes.
1791	1783	* t->devices includes internal dm devices such as mirror logs
1792	1784	* so we need to use iterate_devices here, which targets
1793	1785	* supporting flushes must provide.
1794	1786	*/
1795		− for (i = 0; i < dm_table_get_num_targets(t); i++) {
1796		− ti = dm_table_get_target(t, i);
	1787	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	1788	+ struct dm_target *ti = dm_table_get_target(t, i);
1797	1789
1798	1790	if (!ti->num_flush_bios)
1799	1791	continue;
@@ -1847,11 +1839,8 @@ static int device_not_write_zeroes_capable(struct dm_target ti, struct dm_dev · 47 unmodified lines
1847	1839
1848	1840	static bool dm_table_supports_write_zeroes(struct dm_table *t)
1849	1841	{
1850		− struct dm_target *ti;
1851		− unsigned i = 0;
1852		−
1853		− while (i < dm_table_get_num_targets(t)) {
1854		− ti = dm_table_get_target(t, i++);
	1842	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	1843	+ struct dm_target *ti = dm_table_get_target(t, i);
1855	1844
1856	1845	if (!ti->num_write_zeroes_bios)
1857	1846	return false;
@@ -1874,11 +1863,8 @@ static int device_not_nowait_capable(struct dm_target ti, struct dm_dev dev, · 16 unmodified lines
1874	1863
1875	1864	static bool dm_table_supports_nowait(struct dm_table *t)
1876	1865	{
1877		− struct dm_target *ti;
1878		− unsigned i = 0;
1879		−
1880		− while (i < dm_table_get_num_targets(t)) {
1881		− ti = dm_table_get_target(t, i++);
	1866	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	1867	+ struct dm_target *ti = dm_table_get_target(t, i);
1882	1868
1883	1869	if (!dm_target_supports_nowait(ti->type))
1884	1870	return false;
@@ -1899,11 +1885,8 @@ static int device_not_discard_capable(struct dm_target ti, struct dm_dev dev, · 14 unmodified lines
1899	1885
1900	1886	static bool dm_table_supports_discards(struct dm_table *t)
1901	1887	{
1902		− struct dm_target *ti;
1903		− unsigned i;
1904		−
1905		− for (i = 0; i < dm_table_get_num_targets(t); i++) {
1906		− ti = dm_table_get_target(t, i);
	1888	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	1889	+ struct dm_target *ti = dm_table_get_target(t, i);
1907	1890
1908	1891	if (!ti->num_discard_bios)
1909	1892	return false;
@@ -1931,11 +1914,8 @@ static int device_not_secure_erase_capable(struct dm_target *ti, · 21 unmodified lines
1931	1914
1932	1915	static bool dm_table_supports_secure_erase(struct dm_table *t)
1933	1916	{
1934		− struct dm_target *ti;
1935		− unsigned int i;
1936		−
1937		− for (i = 0; i < dm_table_get_num_targets(t); i++) {
1938		− ti = dm_table_get_target(t, i);
	1917	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	1918	+ struct dm_target *ti = dm_table_get_target(t, i);
1939	1919
1940	1920	if (!ti->num_secure_erase_bios)
1941	1921	return false;
@@ -2065,11 +2045,6 @@ int dm_table_set_restrictions(struct dm_table t, struct request_queue q, · 123 unmodified lines
2065	2045	return 0;
2066	2046	}
2067	2047
2068		−unsigned int dm_table_get_num_targets(struct dm_table *t)
2069		−{
2070		− return t->num_targets;
2071		−}
2072		−
2073	2048	struct list_head dm_table_get_devices(struct dm_table t)
2074	2049	{
2075	2050	return &t->devices;
@@ -2089,12 +2064,11 @@ enum suspend_mode { · 13 unmodified lines
2089	2064
2090	2065	static void suspend_targets(struct dm_table *t, enum suspend_mode mode)
2091	2066	{
2092		− int i = t->num_targets;
2093		− struct dm_target *ti = t->targets;
2094		−
2095	2067	lockdep_assert_held(&t->md->suspend_lock);
2096	2068
2097		− while (i--) {
	2069	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	2070	+ struct dm_target *ti = dm_table_get_target(t, i);
	2071	+
2098	2072	switch (mode) {
2099	2073	case PRESUSPEND:
2100	2074	if (ti->type->presuspend)
@@ -2109,7 +2083,6 @@ static void suspend_targets(struct dm_table *t, enum suspend_mode mode) · 8 unmodified lines
2109	2083	ti->type->postsuspend(ti);
2110	2084	break;
2111	2085	}
2112		− ti++;
2113	2086	}
2114	2087	}
2115	2088
@@ -2139,12 +2112,13 @@ void dm_table_postsuspend_targets(struct dm_table *t) · 23 unmodified lines
2139	2112
2140	2113	int dm_table_resume_targets(struct dm_table *t)
2141	2114	{
2142		− int i, r = 0;
	2115	+ unsigned int i;
	2116	+ int r = 0;
2143	2117
2144	2118	lockdep_assert_held(&t->md->suspend_lock);
2145	2119
2146	2120	for (i = 0; i < t->num_targets; i++) {
2147		− struct dm_target *ti = t->targets + i;
	2121	+ struct dm_target *ti = dm_table_get_target(t, i);
2148	2122
2149	2123	if (!ti->type->preresume)
2150	2124	continue;
@@ -2158,7 +2132,7 @@ int dm_table_resume_targets(struct dm_table *t) · 7 unmodified lines
2158	2132	}
2159	2133
2160	2134	for (i = 0; i < t->num_targets; i++) {
2161		− struct dm_target *ti = t->targets + i;
	2135	+ struct dm_target *ti = dm_table_get_target(t, i);
2162	2136
2163	2137	if (ti->type->resume)
2164	2138	ti->type->resume(ti);

drivers/md/dm-thin-metadata.c+5−2

@@ -2045,10 +2045,13 @@ int dm_pool_register_metadata_threshold(struct dm_pool_metadata *pmd, · 2044 unmodified lines
2045	2045	dm_sm_threshold_fn fn,
2046	2046	void *context)
2047	2047	{
2048		− int r;
	2048	+ int r = -EINVAL;
2049	2049
2050	2050	pmd_write_lock_in_core(pmd);
2051		− r = dm_sm_register_threshold_callback(pmd->metadata_sm, threshold, fn, context);
	2051	+ if (!pmd->fail_io) {
	2052	+ r = dm_sm_register_threshold_callback(pmd->metadata_sm,
	2053	+ threshold, fn, context);
	2054	+ }
2052	2055	pmd_write_unlock(pmd);
2053	2056
2054	2057	return r;

drivers/md/dm-thin.c+3−1

@@ -3375,8 +3375,10 @@ static int pool_ctr(struct dm_target ti, unsigned argc, char *argv) · 3374 unmodified lines
3375	3375	calc_metadata_threshold(pt),
3376	3376	metadata_low_callback,
3377	3377	pool);
3378		− if (r)
	3378	+ if (r) {
	3379	+ ti->error = "Error registering metadata threshold";
3379	3380	goto out_flags_changed;
	3381	+ }
3380	3382
3381	3383	dm_pool_register_pre_commit_callback(pool->pmd,
3382	3384	metadata_pre_commit_callback, pool);

drivers/md/dm-verity-target.c+3−4

@@ -527,11 +527,10 @@ static int verity_verify_io(struct dm_verity_io *io) · 526 unmodified lines
527	527	if (v->validated_blocks)
528	528	set_bit(cur_block, v->validated_blocks);
529	529	continue;
530		− }
531		− else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA,
532		− cur_block, NULL, &start) == 0)
	530	+ } else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA,
	531	+ cur_block, NULL, &start) == 0) {
533	532	continue;
534		− else {
	533	+ } else {
535	534	if (bio->bi_status) {
536	535	/*
537	536	* Error correction failed; Just return error

drivers/md/dm-writecache.c+25−18

@@ -22,7 +22,7 @@ · 21 unmodified lines
22	22
23	23	#define HIGH_WATERMARK 50
24	24	#define LOW_WATERMARK 45
25		−#define MAX_WRITEBACK_JOBS 0
	25	+#define MAX_WRITEBACK_JOBS min(0x10000000 / PAGE_SIZE, totalram_pages() / 16)
26	26	#define ENDIO_LATENCY 16
27	27	#define WRITEBACK_LATENCY 64
28	28	#define AUTOCOMMIT_BLOCKS_SSD 65536
@@ -1325,8 +1325,8 @@ enum wc_map_op { · 1296 unmodified lines
1325	1325	WC_MAP_ERROR,
1326	1326	};
1327	1327
1328		−static enum wc_map_op writecache_map_remap_origin(struct dm_writecache wc, struct bio bio,
1329		− struct wc_entry *e)
	1328	+static void writecache_map_remap_origin(struct dm_writecache wc, struct bio bio,
	1329	+ struct wc_entry *e)
1330	1330	{
1331	1331	if (e) {
1332	1332	sector_t next_boundary =
@@ -1334,8 +1334,6 @@ static enum wc_map_op writecache_map_remap_origin(struct dm_writecache *wc, stru · 1 unmodified line
1334	1334	if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT)
1335	1335	dm_accept_partial_bio(bio, next_boundary);
1336	1336	}
1337		−
1338		− return WC_MAP_REMAP_ORIGIN;
1339	1337	}
1340	1338
1341	1339	static enum wc_map_op writecache_map_read(struct dm_writecache wc, struct bio bio)
@@ -1362,14 +1360,16 @@ read_next_block: · 20 unmodified lines
1362	1360	map_op = WC_MAP_REMAP;
1363	1361	}
1364	1362	} else {
1365		− map_op = writecache_map_remap_origin(wc, bio, e);
	1363	+ writecache_map_remap_origin(wc, bio, e);
	1364	+ wc->stats.reads += (bio->bi_iter.bi_size - wc->block_size) >> wc->block_size_bits;
	1365	+ map_op = WC_MAP_REMAP_ORIGIN;
1366	1366	}
1367	1367
1368	1368	return map_op;
1369	1369	}
1370	1370
1371		−static enum wc_map_op writecache_bio_copy_ssd(struct dm_writecache wc, struct bio bio,
1372		− struct wc_entry *e, bool search_used)
	1371	+static void writecache_bio_copy_ssd(struct dm_writecache wc, struct bio bio,
	1372	+ struct wc_entry *e, bool search_used)
1373	1373	{
1374	1374	unsigned bio_size = wc->block_size;
1375	1375	sector_t start_cache_sec = cache_sector(wc, e);
@@ -1409,14 +1409,15 @@ static enum wc_map_op writecache_bio_copy_ssd(struct dm_writecache *wc, struct b · 33 unmodified lines
1409	1409	bio->bi_iter.bi_sector = start_cache_sec;
1410	1410	dm_accept_partial_bio(bio, bio_size >> SECTOR_SHIFT);
1411	1411
	1412	+ wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
	1413	+ wc->stats.writes_allocate += (bio->bi_iter.bi_size - wc->block_size) >> wc->block_size_bits;
	1414	+
1412	1415	if (unlikely(wc->uncommitted_blocks >= wc->autocommit_blocks)) {
1413	1416	wc->uncommitted_blocks = 0;
1414	1417	queue_work(wc->writeback_wq, &wc->flush_work);
1415	1418	} else {
1416	1419	writecache_schedule_autocommit(wc);
1417	1420	}
1418		−
1419		− return WC_MAP_REMAP;
1420	1421	}
1421	1422
1422	1423	static enum wc_map_op writecache_map_write(struct dm_writecache wc, struct bio bio)
@@ -1426,9 +1427,10 @@ static enum wc_map_op writecache_map_write(struct dm_writecache *wc, struct bio · 3 unmodified lines
1426	1427	do {
1427	1428	bool found_entry = false;
1428	1429	bool search_used = false;
1429		− wc->stats.writes++;
1430		− if (writecache_has_error(wc))
	1430	+ if (writecache_has_error(wc)) {
	1431	+ wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
1431	1432	return WC_MAP_ERROR;
	1433	+ }
1432	1434	e = writecache_find_entry(wc, bio->bi_iter.bi_sector, 0);
1433	1435	if (e) {
1434	1436	if (!writecache_entry_is_committed(wc, e)) {
@@ -1452,9 +1454,11 @@ static enum wc_map_op writecache_map_write(struct dm_writecache *wc, struct bio · 17 unmodified lines
1452	1454	if (unlikely(!e)) {
1453	1455	if (!WC_MODE_PMEM(wc) && !found_entry) {
1454	1456	direct_write:
1455		− wc->stats.writes_around++;
1456	1457	e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
1457		− return writecache_map_remap_origin(wc, bio, e);
	1458	+ writecache_map_remap_origin(wc, bio, e);
	1459	+ wc->stats.writes_around += bio->bi_iter.bi_size >> wc->block_size_bits;
	1460	+ wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
	1461	+ return WC_MAP_REMAP_ORIGIN;
1458	1462	}
1459	1463	wc->stats.writes_blocked_on_freelist++;
1460	1464	writecache_wait_on_freelist(wc);
@@ -1465,10 +1469,13 @@ direct_write: · 4 unmodified lines
1465	1469	wc->uncommitted_blocks++;
1466	1470	wc->stats.writes_allocate++;
1467	1471	bio_copy:
1468		− if (WC_MODE_PMEM(wc))
	1472	+ if (WC_MODE_PMEM(wc)) {
1469	1473	bio_copy_block(wc, bio, memory_data(wc, e));
1470		− else
1471		− return writecache_bio_copy_ssd(wc, bio, e, search_used);
	1474	+ wc->stats.writes++;
	1475	+ } else {
	1476	+ writecache_bio_copy_ssd(wc, bio, e, search_used);
	1477	+ return WC_MAP_REMAP;
	1478	+ }
1472	1479	} while (bio->bi_iter.bi_size);
1473	1480
1474	1481	if (unlikely(bio->bi_opf & REQ_FUA \|\| wc->uncommitted_blocks >= wc->autocommit_blocks))
@@ -1503,7 +1510,7 @@ static enum wc_map_op writecache_map_flush(struct dm_writecache *wc, struct bio · 28 unmodified lines
1503	1510
1504	1511	static enum wc_map_op writecache_map_discard(struct dm_writecache wc, struct bio bio)
1505	1512	{
1506		− wc->stats.discards++;
	1513	+ wc->stats.discards += bio->bi_iter.bi_size >> wc->block_size_bits;
1507	1514
1508	1515	if (writecache_has_error(wc))
1509	1516	return WC_MAP_ERROR;

drivers/md/dm-zone.c+2−5

@@ -273,11 +273,8 @@ static int device_not_zone_append_capable(struct dm_target *ti, · 272 unmodified lines
273	273
274	274	static bool dm_table_supports_zone_append(struct dm_table *t)
275	275	{
276		− struct dm_target *ti;
277		− unsigned int i;
278		−
279		− for (i = 0; i < dm_table_get_num_targets(t); i++) {
280		− ti = dm_table_get_target(t, i);
	276	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	277	+ struct dm_target *ti = dm_table_get_target(t, i);
281	278
282	279	if (ti->emulate_zone_append)
283	280	return false;

drivers/md/dm.c+291−171

@@ -88,10 +88,6 @@ struct clone_info { · 87 unmodified lines
88	88	bool submit_as_polled:1;
89	89	};
90	90
91		−#define DM_TARGET_IO_BIO_OFFSET (offsetof(struct dm_target_io, clone))
92		−#define DM_IO_BIO_OFFSET \
93		− (offsetof(struct dm_target_io, clone) + offsetof(struct dm_io, tio))
94		−
95	91	static inline struct dm_target_io clone_to_tio(struct bio clone)
96	92	{
97	93	return container_of(clone, struct dm_target_io, clone);
@@ -415,7 +411,7 @@ static int dm_blk_getgeo(struct block_device bdev, struct hd_geometry geo) · 317 unmodified lines
415	411	static int dm_prepare_ioctl(struct mapped_device md, int srcu_idx,
416	412	struct block_device **bdev)
417	413	{
418		− struct dm_target *tgt;
	414	+ struct dm_target *ti;
419	415	struct dm_table *map;
420	416	int r;
421	417
@@ -426,17 +422,17 @@ retry: · 4 unmodified lines
426	422	return r;
427	423
428	424	/* We only support devices that have a single target */
429		− if (dm_table_get_num_targets(map) != 1)
	425	+ if (map->num_targets != 1)
430	426	return r;
431	427
432		− tgt = dm_table_get_target(map, 0);
433		− if (!tgt->type->prepare_ioctl)
	428	+ ti = dm_table_get_target(map, 0);
	429	+ if (!ti->type->prepare_ioctl)
434	430	return r;
435	431
436	432	if (dm_suspended_md(md))
437	433	return -EAGAIN;
438	434
439		− r = tgt->type->prepare_ioctl(tgt, bdev);
	435	+ r = ti->type->prepare_ioctl(ti, bdev);
440	436	if (r == -ENOTCONN && !fatal_signal_pending(current)) {
441	437	dm_put_live_table(md, *srcu_idx);
442	438	msleep(10);
@@ -578,9 +574,6 @@ static struct dm_io alloc_io(struct mapped_device md, struct bio *bio) · 135 unmodified lines
578	574	struct bio *clone;
579	575
580	576	clone = bio_alloc_clone(NULL, bio, GFP_NOIO, &md->mempools->io_bs);
581		− /* Set default bdev, but target must bio_set_dev() before issuing IO */
582		− clone->bi_bdev = md->disk->part0;
583		−
584	577	tio = clone_to_tio(clone);
585	578	tio->flags = 0;
586	579	dm_tio_set_flag(tio, DM_TIO_INSIDE_DM_IO);
@@ -594,7 +587,6 @@ static struct dm_io alloc_io(struct mapped_device md, struct bio *bio) · 7 unmodified lines
594	587	atomic_set(&io->io_count, 2);
595	588	this_cpu_inc(*md->pending_io);
596	589	io->orig_bio = bio;
597		− io->split_bio = NULL;
598	590	io->md = md;
599	591	spin_lock_init(&io->lock);
600	592	io->start_time = jiffies;
@@ -614,6 +606,7 @@ static void free_io(struct dm_io *io) · 13 unmodified lines
614	606	static struct bio alloc_tio(struct clone_info ci, struct dm_target *ti,
615	607	unsigned target_bio_nr, unsigned *len, gfp_t gfp_mask)
616	608	{
	609	+ struct mapped_device *md = ci->io->md;
617	610	struct dm_target_io *tio;
618	611	struct bio *clone;
619	612
@@ -623,14 +616,10 @@ static struct bio alloc_tio(struct clone_info ci, struct dm_target *ti, · 3 unmodified lines
623	616	/* alloc_io() already initialized embedded clone */
624	617	clone = &tio->clone;
625	618	} else {
626		− struct mapped_device *md = ci->io->md;
627		−
628	619	clone = bio_alloc_clone(NULL, ci->bio, gfp_mask,
629	620	&md->mempools->bs);
630	621	if (!clone)
631	622	return NULL;
632		− /* Set default bdev, but target must bio_set_dev() before issuing IO */
633		− clone->bi_bdev = md->disk->part0;
634	623
635	624	/* REQ_DM_POLL_LIST shouldn't be inherited */
636	625	clone->bi_opf &= ~REQ_DM_POLL_LIST;
@@ -646,6 +635,11 @@ static struct bio alloc_tio(struct clone_info ci, struct dm_target *ti, · 9 unmodified lines
646	635	tio->len_ptr = len;
647	636	tio->old_sector = 0;
648	637
	638	+ /* Set default bdev, but target must bio_set_dev() before issuing IO */
	639	+ clone->bi_bdev = md->disk->part0;
	640	+ if (unlikely(ti->needs_bio_set_dev))
	641	+ bio_set_dev(clone, md->disk->part0);
	642	+
649	643	if (len) {
650	644	clone->bi_iter.bi_size = to_bytes(*len);
651	645	if (bio_integrity(clone))
@@ -884,22 +878,63 @@ static int __noflush_suspending(struct mapped_device *md) · 232 unmodified lines
884	878	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
885	879	}
886	880
887		−static void dm_io_complete(struct dm_io *io)
	881	+static void dm_requeue_add_io(struct dm_io *io, bool first_stage)
888	882	{
889		− blk_status_t io_error;
890	883	struct mapped_device *md = io->md;
891		− struct bio *bio = io->split_bio ? io->split_bio : io->orig_bio;
892	884
893		− if (io->status == BLK_STS_DM_REQUEUE) {
	885	+ if (first_stage) {
	886	+ struct dm_io *next = md->requeue_list;
	887	+
	888	+ md->requeue_list = io;
	889	+ io->next = next;
	890	+ } else {
	891	+ bio_list_add_head(&md->deferred, io->orig_bio);
	892	+ }
	893	+}
	894	+
	895	+static void dm_kick_requeue(struct mapped_device *md, bool first_stage)
	896	+{
	897	+ if (first_stage)
	898	+ queue_work(md->wq, &md->requeue_work);
	899	+ else
	900	+ queue_work(md->wq, &md->work);
	901	+}
	902	+
	903	+/*
	904	+ * Return true if the dm_io's original bio is requeued.
	905	+ * io->status is updated with error if requeue disallowed.
	906	+ */
	907	+static bool dm_handle_requeue(struct dm_io *io, bool first_stage)
	908	+{
	909	+ struct bio *bio = io->orig_bio;
	910	+ bool handle_requeue = (io->status == BLK_STS_DM_REQUEUE);
	911	+ bool handle_polled_eagain = ((io->status == BLK_STS_AGAIN) &&
	912	+ (bio->bi_opf & REQ_POLLED));
	913	+ struct mapped_device *md = io->md;
	914	+ bool requeued = false;
	915	+
	916	+ if (handle_requeue \|\| handle_polled_eagain) {
894	917	unsigned long flags;
	918	+
	919	+ if (bio->bi_opf & REQ_POLLED) {
	920	+ /*
	921	+ * Upper layer won't help us poll split bio
	922	+ * (io->orig_bio may only reflect a subset of the
	923	+ * pre-split original) so clear REQ_POLLED.
	924	+ */
	925	+ bio_clear_polled(bio);
	926	+ }
	927	+
895	928	/*
896		− * Target requested pushing back the I/O.
	929	+ * Target requested pushing back the I/O or
	930	+ * polled IO hit BLK_STS_AGAIN.
897	931	*/
898	932	spin_lock_irqsave(&md->deferred_lock, flags);
899		− if (__noflush_suspending(md) &&
900		− !WARN_ON_ONCE(dm_is_zone_write(md, bio))) {
901		− /* NOTE early return due to BLK_STS_DM_REQUEUE below */
902		− bio_list_add_head(&md->deferred, bio);
	933	+ if ((__noflush_suspending(md) &&
	934	+ !WARN_ON_ONCE(dm_is_zone_write(md, bio))) \|\|
	935	+ handle_polled_eagain \|\| first_stage) {
	936	+ dm_requeue_add_io(io, first_stage);
	937	+ requeued = true;
903	938	} else {
904	939	/*
905	940	* noflush suspend was interrupted or this is
@@ -910,6 +945,23 @@ static void dm_io_complete(struct dm_io *io) · 4 unmodified lines
910	945	spin_unlock_irqrestore(&md->deferred_lock, flags);
911	946	}
912	947
	948	+ if (requeued)
	949	+ dm_kick_requeue(md, first_stage);
	950	+
	951	+ return requeued;
	952	+}
	953	+
	954	+static void __dm_io_complete(struct dm_io *io, bool first_stage)
	955	+{
	956	+ struct bio *bio = io->orig_bio;
	957	+ struct mapped_device *md = io->md;
	958	+ blk_status_t io_error;
	959	+ bool requeued;
	960	+
	961	+ requeued = dm_handle_requeue(io, first_stage);
	962	+ if (requeued && first_stage)
	963	+ return;
	964	+
913	965	io_error = io->status;
914	966	if (dm_io_flagged(io, DM_IO_ACCOUNTED))
915	967	dm_end_io_acct(io);
@@ -929,23 +981,9 @@ static void dm_io_complete(struct dm_io *io) · 13 unmodified lines
929	981	if (unlikely(wq_has_sleeper(&md->wait)))
930	982	wake_up(&md->wait);
931	983
932		− if (io_error == BLK_STS_DM_REQUEUE \|\| io_error == BLK_STS_AGAIN) {
933		− if (bio->bi_opf & REQ_POLLED) {
934		− /*
935		− * Upper layer won't help us poll split bio (io->orig_bio
936		− * may only reflect a subset of the pre-split original)
937		− * so clear REQ_POLLED in case of requeue.
938		− */
939		− bio_clear_polled(bio);
940		− if (io_error == BLK_STS_AGAIN) {
941		− /* io_uring doesn't handle BLK_STS_AGAIN (yet) */
942		− queue_io(md, bio);
943		− return;
944		− }
945		− }
946		− if (io_error == BLK_STS_DM_REQUEUE)
947		− return;
948		− }
	984	+ /* Return early if the original bio was requeued */
	985	+ if (requeued)
	986	+ return;
949	987
950	988	if (bio_is_flush_with_data(bio)) {
951	989	/*
@@ -962,6 +1000,58 @@ static void dm_io_complete(struct dm_io *io) · 10 unmodified lines
962	1000	}
963	1001	}
964	1002
	1003	+static void dm_wq_requeue_work(struct work_struct *work)
	1004	+{
	1005	+ struct mapped_device *md = container_of(work, struct mapped_device,
	1006	+ requeue_work);
	1007	+ unsigned long flags;
	1008	+ struct dm_io *io;
	1009	+
	1010	+ /* reuse deferred lock to simplify dm_handle_requeue */
	1011	+ spin_lock_irqsave(&md->deferred_lock, flags);
	1012	+ io = md->requeue_list;
	1013	+ md->requeue_list = NULL;
	1014	+ spin_unlock_irqrestore(&md->deferred_lock, flags);
	1015	+
	1016	+ while (io) {
	1017	+ struct dm_io *next = io->next;
	1018	+
	1019	+ dm_io_rewind(io, &md->queue->bio_split);
	1020	+
	1021	+ io->next = NULL;
	1022	+ __dm_io_complete(io, false);
	1023	+ io = next;
	1024	+ }
	1025	+}
	1026	+
	1027	+/*
	1028	+ * Two staged requeue:
	1029	+ *
	1030	+ * 1) io->orig_bio points to the real original bio, and the part mapped to
	1031	+ * this io must be requeued, instead of other parts of the original bio.
	1032	+ *
	1033	+ * 2) io->orig_bio points to new cloned bio which matches the requeued dm_io.
	1034	+ */
	1035	+static void dm_io_complete(struct dm_io *io)
	1036	+{
	1037	+ bool first_requeue;
	1038	+
	1039	+ /*
	1040	+ * Only dm_io that has been split needs two stage requeue, otherwise
	1041	+ * we may run into long bio clone chain during suspend and OOM could
	1042	+ * be triggered.
	1043	+ *
	1044	+ * Also flush data dm_io won't be marked as DM_IO_WAS_SPLIT, so they
	1045	+ * also aren't handled via the first stage requeue.
	1046	+ */
	1047	+ if (dm_io_flagged(io, DM_IO_WAS_SPLIT))
	1048	+ first_requeue = true;
	1049	+ else
	1050	+ first_requeue = false;
	1051	+
	1052	+ __dm_io_complete(io, first_requeue);
	1053	+}
	1054	+
965	1055	/*
966	1056	* Decrements the number of outstanding ios that a bio has been
967	1057	* cloned into, completing the original io if necc.
@@ -1240,6 +1330,7 @@ out: · 272 unmodified lines
1240	1330	void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
1241	1331	{
1242	1332	struct dm_target_io *tio = clone_to_tio(bio);
	1333	+ struct dm_io *io = tio->io;
1243	1334	unsigned bio_sectors = bio_sectors(bio);
1244	1335
1245	1336	BUG_ON(dm_tio_flagged(tio, DM_TIO_IS_DUPLICATE_BIO));
@@ -1255,8 +1346,9 @@ void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors) · 9 unmodified lines
1255	1346	* __split_and_process_bio() may have already saved mapped part
1256	1347	* for accounting but it is being reduced so update accordingly.
1257	1348	*/
1258		− dm_io_set_flag(tio->io, DM_IO_WAS_SPLIT);
1259		− tio->io->sectors = n_sectors;
	1349	+ dm_io_set_flag(io, DM_IO_WAS_SPLIT);
	1350	+ io->sectors = n_sectors;
	1351	+ io->sector_offset = bio_sectors(io->orig_bio);
1260	1352	}
1261	1353	EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
1262	1354
@@ -1379,17 +1471,7 @@ static void setup_split_accounting(struct clone_info *ci, unsigned len) · 116 unmodified lines
1379	1471	*/
1380	1472	dm_io_set_flag(io, DM_IO_WAS_SPLIT);
1381	1473	io->sectors = len;
1382		− }
1383		−
1384		− if (static_branch_unlikely(&stats_enabled) &&
1385		− unlikely(dm_stats_used(&io->md->stats))) {
1386		− /*
1387		− * Save bi_sector in terms of its offset from end of
1388		− * original bio, only needed for DM-stats' benefit.
1389		− * - saved regardless of whether split needed so that
1390		− * dm_accept_partial_bio() doesn't need to.
1391		− */
1392		− io->sector_offset = bio_end_sector(ci->bio) - ci->sector;
	1474	+ io->sector_offset = bio_sectors(ci->bio);
1393	1475	}
1394	1476	}
1395	1477
@@ -1423,11 +1505,11 @@ static void alloc_multiple_bios(struct bio_list blist, struct clone_info ci, · 27 unmodified lines
1423	1505	}
1424	1506
1425	1507	static int __send_duplicate_bios(struct clone_info ci, struct dm_target ti,
1426		− unsigned num_bios, unsigned *len)
	1508	+ unsigned int num_bios, unsigned *len)
1427	1509	{
1428	1510	struct bio_list blist = BIO_EMPTY_LIST;
1429	1511	struct bio *clone;
1430		− int ret = 0;
	1512	+ unsigned int ret = 0;
1431	1513
1432	1514	switch (num_bios) {
1433	1515	case 0:
@@ -1455,8 +1537,7 @@ static int __send_duplicate_bios(struct clone_info ci, struct dm_target ti, · 21 unmodified lines
1455	1537
1456	1538	static void __send_empty_flush(struct clone_info *ci)
1457	1539	{
1458		− unsigned target_nr = 0;
1459		− struct dm_target *ti;
	1540	+ struct dm_table *t = ci->map;
1460	1541	struct bio flush_bio;
1461	1542
1462	1543	/*
@@ -1471,8 +1552,9 @@ static void __send_empty_flush(struct clone_info *ci) · 8 unmodified lines
1471	1552	ci->sector_count = 0;
1472	1553	ci->io->tio.clone.bi_iter.bi_size = 0;
1473	1554
1474		− while ((ti = dm_table_get_target(ci->map, target_nr++))) {
1475		− int bios;
	1555	+ for (unsigned int i = 0; i < t->num_targets; i++) {
	1556	+ unsigned int bios;
	1557	+ struct dm_target *ti = dm_table_get_target(t, i);
1476	1558
1477	1559	atomic_add(ti->num_flush_bios, &ci->io->io_count);
1478	1560	bios = __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
@@ -1492,7 +1574,7 @@ static void __send_changing_extent_only(struct clone_info *ci, struct dm_target · 13 unmodified lines
1492	1574	unsigned num_bios)
1493	1575	{
1494	1576	unsigned len;
1495		− int bios;
	1577	+ unsigned int bios;
1496	1578
1497	1579	len = min_t(sector_t, ci->sector_count,
1498	1580	max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector)));
@@ -1691,11 +1773,9 @@ static void dm_split_and_process_bio(struct mapped_device *md, · 192 unmodified lines
1691	1773	* Remainder must be passed to submit_bio_noacct() so it gets handled
1692	1774	* after bios already submitted have been completely processed.
1693	1775	*/
1694		− WARN_ON_ONCE(!dm_io_flagged(io, DM_IO_WAS_SPLIT));
1695		− io->split_bio = bio_split(bio, io->sectors, GFP_NOIO,
1696		− &md->queue->bio_split);
1697		− bio_chain(io->split_bio, bio);
1698		− trace_block_split(io->split_bio, bio->bi_iter.bi_sector);
	1776	+ bio_trim(bio, io->sectors, ci.sector_count);
	1777	+ trace_block_split(bio, bio->bi_iter.bi_sector);
	1778	+ bio_inc_remaining(bio);
1699	1779	submit_bio_noacct(bio);
1700	1780	out:
1701	1781	/*
@@ -1971,9 +2051,11 @@ static struct mapped_device *alloc_dev(int minor) · 269 unmodified lines
1971	2051
1972	2052	init_waitqueue_head(&md->wait);
1973	2053	INIT_WORK(&md->work, dm_wq_work);
	2054	+ INIT_WORK(&md->requeue_work, dm_wq_requeue_work);
1974	2055	init_waitqueue_head(&md->eventq);
1975	2056	init_completion(&md->kobj_holder.completion);
1976	2057
	2058	+ md->requeue_list = NULL;
1977	2059	md->swap_bios = get_swap_bios();
1978	2060	sema_init(&md->swap_bios_semaphore, md->swap_bios);
1979	2061	mutex_init(&md->swap_bios_lock);
@@ -2980,54 +3062,6 @@ int dm_noflush_suspending(struct dm_target *ti) · 1000 unmodified lines
2980	3062	}
2981	3063	EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2982	3064
2983		−struct dm_md_mempools dm_alloc_md_mempools(struct mapped_device md, enum dm_queue_mode type,
2984		− unsigned per_io_data_size, unsigned min_pool_size,
2985		− bool integrity, bool poll)
2986		−{
2987		− struct dm_md_mempools pools = kzalloc_node(sizeof(pools), GFP_KERNEL, md->numa_node_id);
2988		− unsigned int pool_size = 0;
2989		− unsigned int front_pad, io_front_pad;
2990		− int ret;
2991		−
2992		− if (!pools)
2993		− return NULL;
2994		−
2995		− switch (type) {
2996		− case DM_TYPE_BIO_BASED:
2997		− case DM_TYPE_DAX_BIO_BASED:
2998		− pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
2999		− front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + DM_TARGET_IO_BIO_OFFSET;
3000		− io_front_pad = roundup(per_io_data_size, __alignof__(struct dm_io)) + DM_IO_BIO_OFFSET;
3001		− ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, poll ? BIOSET_PERCPU_CACHE : 0);
3002		− if (ret)
3003		− goto out;
3004		− if (integrity && bioset_integrity_create(&pools->io_bs, pool_size))
3005		− goto out;
3006		− break;
3007		− case DM_TYPE_REQUEST_BASED:
3008		− pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size);
3009		− front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
3010		− /* per_io_data_size is used for blk-mq pdu at queue allocation */
3011		− break;
3012		− default:
3013		− BUG();
3014		− }
3015		−
3016		− ret = bioset_init(&pools->bs, pool_size, front_pad, 0);
3017		− if (ret)
3018		− goto out;
3019		−
3020		− if (integrity && bioset_integrity_create(&pools->bs, pool_size))
3021		− goto out;
3022		−
3023		− return pools;
3024		−
3025		−out:
3026		− dm_free_md_mempools(pools);
3027		−
3028		− return NULL;
3029		−}
3030		−
3031	3065	void dm_free_md_mempools(struct dm_md_mempools *pools)
3032	3066	{
3033	3067	if (!pools)
@@ -3043,11 +3077,14 @@ struct dm_pr { · 9 unmodified lines
3043	3077	u64 old_key;
3044	3078	u64 new_key;
3045	3079	u32 flags;
	3080	+ bool abort;
3046	3081	bool fail_early;
	3082	+ int ret;
	3083	+ enum pr_type type;
3047	3084	};
3048	3085
3049	3086	static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
3050		− void *data)
	3087	+ struct dm_pr *pr)
3051	3088	{
3052	3089	struct mapped_device *md = bdev->bd_disk->private_data;
3053	3090	struct dm_table *table;
@@ -3059,15 +3096,21 @@ static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn, · 5 unmodified lines
3059	3096	goto out;
3060	3097
3061	3098	/* We only support devices that have a single target */
3062		− if (dm_table_get_num_targets(table) != 1)
	3099	+ if (table->num_targets != 1)
3063	3100	goto out;
3064	3101	ti = dm_table_get_target(table, 0);
3065	3102
	3103	+ if (dm_suspended_md(md)) {
	3104	+ ret = -EAGAIN;
	3105	+ goto out;
	3106	+ }
	3107	+
3066	3108	ret = -EINVAL;
3067	3109	if (!ti->type->iterate_devices)
3068	3110	goto out;
3069	3111
3070		− ret = ti->type->iterate_devices(ti, fn, data);
	3112	+ ti->type->iterate_devices(ti, fn, pr);
	3113	+ ret = 0;
3071	3114	out:
3072	3115	dm_put_live_table(md, srcu_idx);
3073	3116	return ret;
@@ -3081,10 +3124,24 @@ static int __dm_pr_register(struct dm_target ti, struct dm_dev dev, · 7 unmodified lines
3081	3124	{
3082	3125	struct dm_pr *pr = data;
3083	3126	const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
	3127	+ int ret;
	3128	+
	3129	+ if (!ops \|\| !ops->pr_register) {
	3130	+ pr->ret = -EOPNOTSUPP;
	3131	+ return -1;
	3132	+ }
3084	3133
3085		− if (!ops \|\| !ops->pr_register)
3086		− return -EOPNOTSUPP;
3087		− return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
	3134	+ ret = ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
	3135	+ if (!ret)
	3136	+ return 0;
	3137	+
	3138	+ if (!pr->ret)
	3139	+ pr->ret = ret;
	3140	+
	3141	+ if (pr->fail_early)
	3142	+ return -1;
	3143	+
	3144	+ return 0;
3088	3145	}
3089	3146
3090	3147	static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
@@ -3095,82 +3152,145 @@ static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key, · 4 unmodified lines
3095	3152	.new_key = new_key,
3096	3153	.flags = flags,
3097	3154	.fail_early = true,
	3155	+ .ret = 0,
3098	3156	};
3099	3157	int ret;
3100	3158
3101	3159	ret = dm_call_pr(bdev, __dm_pr_register, &pr);
3102		− if (ret && new_key) {
3103		− /* unregister all paths if we failed to register any path */
3104		− pr.old_key = new_key;
3105		− pr.new_key = 0;
3106		− pr.flags = 0;
3107		− pr.fail_early = false;
3108		− dm_call_pr(bdev, __dm_pr_register, &pr);
	3160	+ if (ret) {
	3161	+ /* Didn't even get to register a path */
	3162	+ return ret;
3109	3163	}
3110	3164
	3165	+ if (!pr.ret)
	3166	+ return 0;
	3167	+ ret = pr.ret;
	3168	+
	3169	+ if (!new_key)
	3170	+ return ret;
	3171	+
	3172	+ /* unregister all paths if we failed to register any path */
	3173	+ pr.old_key = new_key;
	3174	+ pr.new_key = 0;
	3175	+ pr.flags = 0;
	3176	+ pr.fail_early = false;
	3177	+ (void) dm_call_pr(bdev, __dm_pr_register, &pr);
3111	3178	return ret;
3112	3179	}
3113	3180
	3181	+
	3182	+static int __dm_pr_reserve(struct dm_target ti, struct dm_dev dev,
	3183	+ sector_t start, sector_t len, void *data)
	3184	+{
	3185	+ struct dm_pr *pr = data;
	3186	+ const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
	3187	+
	3188	+ if (!ops \|\| !ops->pr_reserve) {
	3189	+ pr->ret = -EOPNOTSUPP;
	3190	+ return -1;
	3191	+ }
	3192	+
	3193	+ pr->ret = ops->pr_reserve(dev->bdev, pr->old_key, pr->type, pr->flags);
	3194	+ if (!pr->ret)
	3195	+ return -1;
	3196	+
	3197	+ return 0;
	3198	+}
	3199	+
3114	3200	static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
3115	3201	u32 flags)
3116	3202	{
3117		− struct mapped_device *md = bdev->bd_disk->private_data;
3118		− const struct pr_ops *ops;
3119		− int r, srcu_idx;
	3203	+ struct dm_pr pr = {
	3204	+ .old_key = key,
	3205	+ .flags = flags,
	3206	+ .type = type,
	3207	+ .fail_early = false,
	3208	+ .ret = 0,
	3209	+ };
	3210	+ int ret;
3120	3211
3121		− r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3122		− if (r < 0)
3123		− goto out;
	3212	+ ret = dm_call_pr(bdev, __dm_pr_reserve, &pr);
	3213	+ if (ret)
	3214	+ return ret;
3124	3215
3125		− ops = bdev->bd_disk->fops->pr_ops;
3126		− if (ops && ops->pr_reserve)
3127		− r = ops->pr_reserve(bdev, key, type, flags);
3128		− else
3129		− r = -EOPNOTSUPP;
3130		−out:
3131		− dm_unprepare_ioctl(md, srcu_idx);
3132		− return r;
	3216	+ return pr.ret;
	3217	+}
	3218	+
	3219	+/*
	3220	+ * If there is a non-All Registrants type of reservation, the release must be
	3221	+ * sent down the holding path. For the cases where there is no reservation or
	3222	+ * the path is not the holder the device will also return success, so we must
	3223	+ * try each path to make sure we got the correct path.
	3224	+ */
	3225	+static int __dm_pr_release(struct dm_target ti, struct dm_dev dev,
	3226	+ sector_t start, sector_t len, void *data)
	3227	+{
	3228	+ struct dm_pr *pr = data;
	3229	+ const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
	3230	+
	3231	+ if (!ops \|\| !ops->pr_release) {
	3232	+ pr->ret = -EOPNOTSUPP;
	3233	+ return -1;
	3234	+ }
	3235	+
	3236	+ pr->ret = ops->pr_release(dev->bdev, pr->old_key, pr->type);
	3237	+ if (pr->ret)
	3238	+ return -1;
	3239	+
	3240	+ return 0;
3133	3241	}
3134	3242
3135	3243	static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
3136	3244	{
3137		− struct mapped_device *md = bdev->bd_disk->private_data;
3138		− const struct pr_ops *ops;
3139		− int r, srcu_idx;
	3245	+ struct dm_pr pr = {
	3246	+ .old_key = key,
	3247	+ .type = type,
	3248	+ .fail_early = false,
	3249	+ };
	3250	+ int ret;
3140	3251
3141		− r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3142		− if (r < 0)
3143		− goto out;
	3252	+ ret = dm_call_pr(bdev, __dm_pr_release, &pr);
	3253	+ if (ret)
	3254	+ return ret;
3144	3255
3145		− ops = bdev->bd_disk->fops->pr_ops;
3146		− if (ops && ops->pr_release)
3147		− r = ops->pr_release(bdev, key, type);
3148		− else
3149		− r = -EOPNOTSUPP;
3150		−out:
3151		− dm_unprepare_ioctl(md, srcu_idx);
3152		− return r;
	3256	+ return pr.ret;
	3257	+}
	3258	+
	3259	+static int __dm_pr_preempt(struct dm_target ti, struct dm_dev dev,
	3260	+ sector_t start, sector_t len, void *data)
	3261	+{
	3262	+ struct dm_pr *pr = data;
	3263	+ const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
	3264	+
	3265	+ if (!ops \|\| !ops->pr_preempt) {
	3266	+ pr->ret = -EOPNOTSUPP;
	3267	+ return -1;
	3268	+ }
	3269	+
	3270	+ pr->ret = ops->pr_preempt(dev->bdev, pr->old_key, pr->new_key, pr->type,
	3271	+ pr->abort);
	3272	+ if (!pr->ret)
	3273	+ return -1;
	3274	+
	3275	+ return 0;
3153	3276	}
3154	3277
3155	3278	static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
3156	3279	enum pr_type type, bool abort)
3157	3280	{
3158		− struct mapped_device *md = bdev->bd_disk->private_data;
3159		− const struct pr_ops *ops;
3160		− int r, srcu_idx;
	3281	+ struct dm_pr pr = {
	3282	+ .new_key = new_key,
	3283	+ .old_key = old_key,
	3284	+ .type = type,
	3285	+ .fail_early = false,
	3286	+ };
	3287	+ int ret;
3161	3288
3162		− r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3163		− if (r < 0)
3164		− goto out;
	3289	+ ret = dm_call_pr(bdev, __dm_pr_preempt, &pr);
	3290	+ if (ret)
	3291	+ return ret;
3165	3292
3166		− ops = bdev->bd_disk->fops->pr_ops;
3167		− if (ops && ops->pr_preempt)
3168		− r = ops->pr_preempt(bdev, old_key, new_key, type, abort);
3169		− else
3170		− r = -EOPNOTSUPP;
3171		−out:
3172		− dm_unprepare_ioctl(md, srcu_idx);
3173		− return r;
	3293	+ return pr.ret;
3174	3294	}
3175	3295
3176	3296	static int dm_pr_clear(struct block_device *bdev, u64 key)

drivers/md/dm.h+0−4

@@ -53,7 +53,6 @@ struct dm_io; · 52 unmodified lines
53	53	---------------------------------------------------------------/
54	54	void dm_table_event_callback(struct dm_table *t,
55	55	void (fn)(void ), void *context);
56		−struct dm_target dm_table_get_target(struct dm_table t, unsigned int index);
57	56	struct dm_target dm_table_find_target(struct dm_table t, sector_t sector);
58	57	bool dm_table_has_no_data_devices(struct dm_table *table);
59	58	int dm_calculate_queue_limits(struct dm_table *table,
@@ -218,9 +217,6 @@ void dm_kcopyd_exit(void); · 158 unmodified lines
218	217	/*
219	218	* Mempool operations
220	219	*/
221		−struct dm_md_mempools dm_alloc_md_mempools(struct mapped_device md, enum dm_queue_mode type,
222		− unsigned per_io_data_size, unsigned min_pool_size,
223		− bool integrity, bool poll);
224	220	void dm_free_md_mempools(struct dm_md_mempools *pools);
225	221
226	222	/*

include/linux/device-mapper.h+6−1

@@ -373,6 +373,12 @@ struct dm_target { · 372 unmodified lines
373	373	* after returning DM_MAPIO_SUBMITTED from its map function.
374	374	*/
375	375	bool accounts_remapped_io:1;
	376	+
	377	+ /*
	378	+ * Set if the target will submit the DM bio without first calling
	379	+ * bio_set_dev(). NOTE: ideally a target should _not_ need this.
	380	+ */
	381	+ bool needs_bio_set_dev:1;
376	382	};
377	383
378	384	void dm_per_bio_data(struct bio bio, size_t data_size);
@@ -561,7 +567,6 @@ void dm_sync_table(struct mapped_device *md); · 182 unmodified lines
561	567	* Queries
562	568	*/
563	569	sector_t dm_table_get_size(struct dm_table *t);
564		−unsigned int dm_table_get_num_targets(struct dm_table *t);
565	570	fmode_t dm_table_get_mode(struct dm_table *t);
566	571	struct mapped_device dm_table_get_md(struct dm_table t);
567	572	const char dm_table_device_name(struct dm_table t);

include/uapi/linux/dm-ioctl.h+2−2

@@ -286,9 +286,9 @@ enum { · 285 unmodified lines
286	286	#define DM_DEV_SET_GEOMETRY _IOWR(DM_IOCTL, DM_DEV_SET_GEOMETRY_CMD, struct dm_ioctl)
287	287
288	288	#define DM_VERSION_MAJOR 4
289		−#define DM_VERSION_MINOR 46
	289	+#define DM_VERSION_MINOR 47
290	290	#define DM_VERSION_PATCHLEVEL 0
291		−#define DM_VERSION_EXTRA "-ioctl (2022-02-22)"
	291	+#define DM_VERSION_EXTRA "-ioctl (2022-07-28)"
292	292
293	293	/* Status bits */
294	294	#define DM_READONLY_FLAG (1 << 0) /* In/Out */