Merge tag 'dm-3.19-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm

Pull device mapper updates from Mike Snitzer: - Significant DM thin-provisioning performance improvements to meet performance requirements that were requested by the Gluster distributed filesystem. Specifically, dm-thinp now takes care to aggregate IO that will be issued to the same thinp block before issuing IO to the underlying devices. This really helps improve performance on HW RAID6 devices that have a writeback cache because it avoids RMW in the HW RAID controller. - Some stable fixes: fix leak in DM bufio if integrity profiles were enabled, use memzero_explicit in DM crypt to avoid any potential for information leak, and a DM cache fix to properly mark a cache block dirty if it was promoted to the cache via the overwrite optimization. - A few simple DM persistent data library fixes - DM cache multiqueue policy block promotion improvements. - DM cache discard improvements that take advantage of range (multiblock) discard support in the DM bio-prison. This allows for much more efficient bulk discard processing (e.g. when mkfs.xfs discards the entire device). - Some small optimizations in DM core and RCU deference cleanups - DM core changes to suspend/resume code to introduce the new internal suspend/resume interface that the DM thin-pool target now uses to suspend/resume active thin devices when the thin-pool must suspend/resume. This avoids forcing userspace to track all active thin volumes in a thin-pool when the thin-pool is suspended for the purposes of metadata or data space resize. * tag 'dm-3.19-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm: (49 commits) dm crypt: use memzero_explicit for on-stack buffer dm space map metadata: fix sm_bootstrap_get_count() dm space map metadata: fix sm_bootstrap_get_nr_blocks() dm bufio: fix memleak when using a dm_buffer's inline bio dm cache: fix spurious cell_defer when dealing with partial block at end of device dm cache: dirty flag was mistakenly being cleared when promoting via overwrite dm cache: only use overwrite optimisation for promotion when in writeback mode dm cache: discard block size must be a multiple of cache block size dm cache: fix a harmless race when working out if a block is discarded dm cache: when reloading a discard bitset allow for a different discard block size dm cache: fix some issues with the new discard range support dm array: if resizing the array is a noop set the new root to the old one dm: use rcu_dereference_protected instead of rcu_dereference dm thin: fix pool_io_hints to avoid looking at max_hw_sectors dm thin: suspend/resume active thin devices when reloading thin-pool dm: enhance internal suspend and resume interface dm thin: do not allow thin device activation while pool is suspended dm: add presuspend_undo hook to target_type dm: return earlier from dm_blk_ioctl if target doesn't implement .ioctl dm thin: remove stale 'trim' message in block comment above pool_message ...

Linus Torvalds committed 11 years ago

Commit 140dfc92 parents f94784b 1a71d6f24 files · +1634 −580Browse files

Unified · Split

24 files

Documentation/device-mapper/cache-policies.txt +15 −9
drivers/md/dm-bio-prison.c +87 −99
drivers/md/dm-bio-prison.h +21 −7
drivers/md/dm-bufio.c +148 −78
drivers/md/dm-cache-block-types.h +11 −0
drivers/md/dm-cache-metadata.c +17 −17
drivers/md/dm-cache-metadata.h +3 −3
drivers/md/dm-cache-policy-mq.c +54 −28
drivers/md/dm-cache-target.c +301 −77
drivers/md/dm-crypt.c +1 −1
drivers/md/dm-ioctl.c +4 −1
drivers/md/dm-stats.c +1 −1
drivers/md/dm-table.c +29 −7
drivers/md/dm-thin-metadata.c +18 −17
drivers/md/dm-thin-metadata.h +7 −2
drivers/md/dm-thin.c +607 −153
drivers/md/dm.c +200 −73
drivers/md/dm.h +10 −0
drivers/md/persistent-data/dm-array.c +3 −1
drivers/md/persistent-data/dm-space-map-metadata.c +6 −2
drivers/md/persistent-data/dm-transaction-manager.c +75 −2
drivers/md/persistent-data/dm-transaction-manager.h +7 −0
include/linux/device-mapper.h +2 −0
include/uapi/linux/dm-ioctl.h +7 −2

Documentation/device-mapper/cache-policies.txt+15−9

@@ -47,20 +47,26 @@ Message and constructor argument pairs are: · 46 unmodified lines
47	47	'discard_promote_adjustment <value>'
48	48
49	49	The sequential threshold indicates the number of contiguous I/Os
50		−required before a stream is treated as sequential. The random threshold
	50	+required before a stream is treated as sequential. Once a stream is
	51	+considered sequential it will bypass the cache. The random threshold
51	52	is the number of intervening non-contiguous I/Os that must be seen
52	53	before the stream is treated as random again.
53	54
54	55	The sequential and random thresholds default to 512 and 4 respectively.
55	56
56		−Large, sequential ios are probably better left on the origin device
57		−since spindles tend to have good bandwidth. The io_tracker counts
58		−contiguous I/Os to try to spot when the io is in one of these sequential
59		−modes.
60		−
61		−Internally the mq policy maintains a promotion threshold variable. If
62		−the hit count of a block not in the cache goes above this threshold it
63		−gets promoted to the cache. The read, write and discard promote adjustment
	57	+Large, sequential I/Os are probably better left on the origin device
	58	+since spindles tend to have good sequential I/O bandwidth. The
	59	+io_tracker counts contiguous I/Os to try to spot when the I/O is in one
	60	+of these sequential modes. But there are use-cases for wanting to
	61	+promote sequential blocks to the cache (e.g. fast application startup).
	62	+If sequential threshold is set to 0 the sequential I/O detection is
	63	+disabled and sequential I/O will no longer implicitly bypass the cache.
	64	+Setting the random threshold to 0 does _not_ disable the random I/O
	65	+stream detection.
	66	+
	67	+Internally the mq policy determines a promotion threshold. If the hit
	68	+count of a block not in the cache goes above this threshold it gets
	69	+promoted to the cache. The read, write and discard promote adjustment
64	70	tunables allow you to tweak the promotion threshold by adding a small
65	71	value based on the io type. They default to 4, 8 and 1 respectively.
66	72	If you're trying to quickly warm a new cache device you may wish to

drivers/md/dm-bio-prison.c+87−99

@@ -14,68 +14,38 @@ · 13 unmodified lines
14	14
15	15	/----------------------------------------------------------------/
16	16
17		−struct bucket {
18		− spinlock_t lock;
19		− struct hlist_head cells;
20		−};
	17	+#define MIN_CELLS 1024
21	18
22	19	struct dm_bio_prison {
	20	+ spinlock_t lock;
23	21	mempool_t *cell_pool;
24		−
25		− unsigned nr_buckets;
26		− unsigned hash_mask;
27		− struct bucket *buckets;
	22	+ struct rb_root cells;
28	23	};
29	24
30		−/----------------------------------------------------------------/
31		−
32		−static uint32_t calc_nr_buckets(unsigned nr_cells)
33		−{
34		− uint32_t n = 128;
35		−
36		− nr_cells /= 4;
37		− nr_cells = min(nr_cells, 8192u);
38		−
39		− while (n < nr_cells)
40		− n <<= 1;
41		−
42		− return n;
43		−}
44		−
45	25	static struct kmem_cache *_cell_cache;
46	26
47		−static void init_bucket(struct bucket *b)
48		−{
49		− spin_lock_init(&b->lock);
50		− INIT_HLIST_HEAD(&b->cells);
51		−}
	27	+/----------------------------------------------------------------/
52	28
53	29	/*
54	30	* @nr_cells should be the number of cells you want in use _concurrently_.
55	31	* Don't confuse it with the number of distinct keys.
56	32	*/
57		−struct dm_bio_prison *dm_bio_prison_create(unsigned nr_cells)
	33	+struct dm_bio_prison *dm_bio_prison_create(void)
58	34	{
59		− unsigned i;
60		− uint32_t nr_buckets = calc_nr_buckets(nr_cells);
61		− size_t len = sizeof(struct dm_bio_prison) +
62		− (sizeof(struct bucket) * nr_buckets);
63		− struct dm_bio_prison *prison = kmalloc(len, GFP_KERNEL);
	35	+ struct dm_bio_prison prison = kmalloc(sizeof(prison), GFP_KERNEL);
64	36
65	37	if (!prison)
66	38	return NULL;
67	39
68		− prison->cell_pool = mempool_create_slab_pool(nr_cells, _cell_cache);
	40	+ spin_lock_init(&prison->lock);
	41	+
	42	+ prison->cell_pool = mempool_create_slab_pool(MIN_CELLS, _cell_cache);
69	43	if (!prison->cell_pool) {
70	44	kfree(prison);
71	45	return NULL;
72	46	}
73	47
74		− prison->nr_buckets = nr_buckets;
75		− prison->hash_mask = nr_buckets - 1;
76		− prison->buckets = (struct bucket *) (prison + 1);
77		− for (i = 0; i < nr_buckets; i++)
78		− init_bucket(prison->buckets + i);
	48	+ prison->cells = RB_ROOT;
79	49
80	50	return prison;
81	51	}
@@ -101,68 +71,73 @@ void dm_bio_prison_free_cell(struct dm_bio_prison *prison, · 19 unmodified lines
101	71	}
102	72	EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell);
103	73
104		−static uint32_t hash_key(struct dm_bio_prison prison, struct dm_cell_key key)
	74	+static void __setup_new_cell(struct dm_cell_key *key,
	75	+ struct bio *holder,
	76	+ struct dm_bio_prison_cell *cell)
105	77	{
106		− const unsigned long BIG_PRIME = 4294967291UL;
107		− uint64_t hash = key->block * BIG_PRIME;
108		−
109		− return (uint32_t) (hash & prison->hash_mask);
	78	+ memcpy(&cell->key, key, sizeof(cell->key));
	79	+ cell->holder = holder;
	80	+ bio_list_init(&cell->bios);
110	81	}
111	82
112		−static int keys_equal(struct dm_cell_key lhs, struct dm_cell_key rhs)
	83	+static int cmp_keys(struct dm_cell_key *lhs,
	84	+ struct dm_cell_key *rhs)
113	85	{
114		− return (lhs->virtual == rhs->virtual) &&
115		− (lhs->dev == rhs->dev) &&
116		− (lhs->block == rhs->block);
117		−}
	86	+ if (lhs->virtual < rhs->virtual)
	87	+ return -1;
118	88
119		−static struct bucket get_bucket(struct dm_bio_prison prison,
120		− struct dm_cell_key *key)
121		−{
122		− return prison->buckets + hash_key(prison, key);
123		−}
	89	+ if (lhs->virtual > rhs->virtual)
	90	+ return 1;
124	91
125		−static struct dm_bio_prison_cell __search_bucket(struct bucket b,
126		− struct dm_cell_key *key)
127		−{
128		− struct dm_bio_prison_cell *cell;
	92	+ if (lhs->dev < rhs->dev)
	93	+ return -1;
129	94
130		− hlist_for_each_entry(cell, &b->cells, list)
131		− if (keys_equal(&cell->key, key))
132		− return cell;
	95	+ if (lhs->dev > rhs->dev)
	96	+ return 1;
133	97
134		− return NULL;
135		−}
	98	+ if (lhs->block_end <= rhs->block_begin)
	99	+ return -1;
136	100
137		−static void __setup_new_cell(struct bucket *b,
138		− struct dm_cell_key *key,
139		− struct bio *holder,
140		− struct dm_bio_prison_cell *cell)
141		−{
142		− memcpy(&cell->key, key, sizeof(cell->key));
143		− cell->holder = holder;
144		− bio_list_init(&cell->bios);
145		− hlist_add_head(&cell->list, &b->cells);
	101	+ if (lhs->block_begin >= rhs->block_end)
	102	+ return 1;
	103	+
	104	+ return 0;
146	105	}
147	106
148		−static int __bio_detain(struct bucket *b,
	107	+static int __bio_detain(struct dm_bio_prison *prison,
149	108	struct dm_cell_key *key,
150	109	struct bio *inmate,
151	110	struct dm_bio_prison_cell *cell_prealloc,
152	111	struct dm_bio_prison_cell **cell_result)
153	112	{
154		− struct dm_bio_prison_cell *cell;
155		−
156		− cell = __search_bucket(b, key);
157		− if (cell) {
158		− if (inmate)
159		− bio_list_add(&cell->bios, inmate);
160		− *cell_result = cell;
161		− return 1;
	113	+ int r;
	114	+ struct rb_node *new = &prison->cells.rb_node, parent = NULL;
	115	+
	116	+ while (*new) {
	117	+ struct dm_bio_prison_cell *cell =
	118	+ container_of(*new, struct dm_bio_prison_cell, node);
	119	+
	120	+ r = cmp_keys(key, &cell->key);
	121	+
	122	+ parent = *new;
	123	+ if (r < 0)
	124	+ new = &((*new)->rb_left);
	125	+ else if (r > 0)
	126	+ new = &((*new)->rb_right);
	127	+ else {
	128	+ if (inmate)
	129	+ bio_list_add(&cell->bios, inmate);
	130	+ *cell_result = cell;
	131	+ return 1;
	132	+ }
162	133	}
163	134
164		− __setup_new_cell(b, key, inmate, cell_prealloc);
	135	+ __setup_new_cell(key, inmate, cell_prealloc);
165	136	*cell_result = cell_prealloc;
	137	+
	138	+ rb_link_node(&cell_prealloc->node, parent, new);
	139	+ rb_insert_color(&cell_prealloc->node, &prison->cells);
	140	+
166	141	return 0;
167	142	}
168	143
@@ -174,11 +149,10 @@ static int bio_detain(struct dm_bio_prison *prison, · 5 unmodified lines
174	149	{
175	150	int r;
176	151	unsigned long flags;
177		− struct bucket *b = get_bucket(prison, key);
178	152
179		− spin_lock_irqsave(&b->lock, flags);
180		− r = __bio_detain(b, key, inmate, cell_prealloc, cell_result);
181		− spin_unlock_irqrestore(&b->lock, flags);
	153	+ spin_lock_irqsave(&prison->lock, flags);
	154	+ r = __bio_detain(prison, key, inmate, cell_prealloc, cell_result);
	155	+ spin_unlock_irqrestore(&prison->lock, flags);
182	156
183	157	return r;
184	158	}
@@ -205,10 +179,11 @@ EXPORT_SYMBOL_GPL(dm_get_cell); · 20 unmodified lines
205	179	/*
206	180	* @inmates must have been initialised prior to this call
207	181	*/
208		−static void __cell_release(struct dm_bio_prison_cell *cell,
	182	+static void __cell_release(struct dm_bio_prison *prison,
	183	+ struct dm_bio_prison_cell *cell,
209	184	struct bio_list *inmates)
210	185	{
211		− hlist_del(&cell->list);
	186	+ rb_erase(&cell->node, &prison->cells);
212	187
213	188	if (inmates) {
214	189	if (cell->holder)
@@ -222,21 +197,21 @@ void dm_cell_release(struct dm_bio_prison *prison, · 7 unmodified lines
222	197	struct bio_list *bios)
223	198	{
224	199	unsigned long flags;
225		− struct bucket *b = get_bucket(prison, &cell->key);
226	200
227		− spin_lock_irqsave(&b->lock, flags);
228		− __cell_release(cell, bios);
229		− spin_unlock_irqrestore(&b->lock, flags);
	201	+ spin_lock_irqsave(&prison->lock, flags);
	202	+ __cell_release(prison, cell, bios);
	203	+ spin_unlock_irqrestore(&prison->lock, flags);
230	204	}
231	205	EXPORT_SYMBOL_GPL(dm_cell_release);
232	206
233	207	/*
234	208	* Sometimes we don't want the holder, just the additional bios.
235	209	*/
236		−static void __cell_release_no_holder(struct dm_bio_prison_cell *cell,
	210	+static void __cell_release_no_holder(struct dm_bio_prison *prison,
	211	+ struct dm_bio_prison_cell *cell,
237	212	struct bio_list *inmates)
238	213	{
239		− hlist_del(&cell->list);
	214	+ rb_erase(&cell->node, &prison->cells);
240	215	bio_list_merge(inmates, &cell->bios);
241	216	}
242	217
@@ -245,11 +220,10 @@ void dm_cell_release_no_holder(struct dm_bio_prison *prison, · 2 unmodified lines
245	220	struct bio_list *inmates)
246	221	{
247	222	unsigned long flags;
248		− struct bucket *b = get_bucket(prison, &cell->key);
249	223
250		− spin_lock_irqsave(&b->lock, flags);
251		− __cell_release_no_holder(cell, inmates);
252		− spin_unlock_irqrestore(&b->lock, flags);
	224	+ spin_lock_irqsave(&prison->lock, flags);
	225	+ __cell_release_no_holder(prison, cell, inmates);
	226	+ spin_unlock_irqrestore(&prison->lock, flags);
253	227	}
254	228	EXPORT_SYMBOL_GPL(dm_cell_release_no_holder);
255	229
@@ -267,6 +241,20 @@ void dm_cell_error(struct dm_bio_prison *prison, · 11 unmodified lines
267	241	}
268	242	EXPORT_SYMBOL_GPL(dm_cell_error);
269	243
	244	+void dm_cell_visit_release(struct dm_bio_prison *prison,
	245	+ void (visit_fn)(void , struct dm_bio_prison_cell *),
	246	+ void *context,
	247	+ struct dm_bio_prison_cell *cell)
	248	+{
	249	+ unsigned long flags;
	250	+
	251	+ spin_lock_irqsave(&prison->lock, flags);
	252	+ visit_fn(context, cell);
	253	+ rb_erase(&cell->node, &prison->cells);
	254	+ spin_unlock_irqrestore(&prison->lock, flags);
	255	+}
	256	+EXPORT_SYMBOL_GPL(dm_cell_visit_release);
	257	+
270	258	/----------------------------------------------------------------/
271	259
272	260	#define DEFERRED_SET_SIZE 64

drivers/md/dm-bio-prison.h+21−7

@@ -10,8 +10,8 @@ · 9 unmodified lines
10	10	#include "persistent-data/dm-block-manager.h" /* FIXME: for dm_block_t */
11	11	#include "dm-thin-metadata.h" /* FIXME: for dm_thin_id */
12	12
13		−#include <linux/list.h>
14	13	#include <linux/bio.h>
	14	+#include <linux/rbtree.h>
15	15
16	16	/----------------------------------------------------------------/
17	17
@@ -23,11 +23,14 @@ · 5 unmodified lines
23	23	*/
24	24	struct dm_bio_prison;
25	25
26		−/* FIXME: this needs to be more abstract */
	26	+/*
	27	+ * Keys define a range of blocks within either a virtual or physical
	28	+ * device.
	29	+ */
27	30	struct dm_cell_key {
28	31	int virtual;
29	32	dm_thin_id dev;
30		− dm_block_t block;
	33	+ dm_block_t block_begin, block_end;
31	34	};
32	35
33	36	/*
@@ -35,13 +38,15 @@ struct dm_cell_key { · 1 unmodified line
35	38	* themselves.
36	39	*/
37	40	struct dm_bio_prison_cell {
38		− struct hlist_node list;
	41	+ struct list_head user_list; /* for client use */
	42	+ struct rb_node node;
	43	+
39	44	struct dm_cell_key key;
40	45	struct bio *holder;
41	46	struct bio_list bios;
42	47	};
43	48
44		−struct dm_bio_prison *dm_bio_prison_create(unsigned nr_cells);
	49	+struct dm_bio_prison *dm_bio_prison_create(void);
45	50	void dm_bio_prison_destroy(struct dm_bio_prison *prison);
46	51
47	52	/*
@@ -57,7 +62,7 @@ void dm_bio_prison_free_cell(struct dm_bio_prison *prison, · 9 unmodified lines
57	62	struct dm_bio_prison_cell *cell);
58	63
59	64	/*
60		− * Creates, or retrieves a cell for the given key.
	65	+ * Creates, or retrieves a cell that overlaps the given key.
61	66	*
62	67	* Returns 1 if pre-existing cell returned, zero if new cell created using
63	68	* @cell_prealloc.
@@ -68,7 +73,8 @@ int dm_get_cell(struct dm_bio_prison *prison, · 4 unmodified lines
68	73	struct dm_bio_prison_cell **cell_result);
69	74
70	75	/*
71		− * An atomic op that combines retrieving a cell, and adding a bio to it.
	76	+ * An atomic op that combines retrieving or creating a cell, and adding a
	77	+ * bio to it.
72	78	*
73	79	* Returns 1 if the cell was already held, 0 if @inmate is the new holder.
74	80	*/
@@ -87,6 +93,14 @@ void dm_cell_release_no_holder(struct dm_bio_prison *prison, · 12 unmodified lines
87	93	void dm_cell_error(struct dm_bio_prison *prison,
88	94	struct dm_bio_prison_cell *cell, int error);
89	95
	96	+/*
	97	+ * Visits the cell and then releases. Guarantees no new inmates are
	98	+ * inserted between the visit and release.
	99	+ */
	100	+void dm_cell_visit_release(struct dm_bio_prison *prison,
	101	+ void (visit_fn)(void , struct dm_bio_prison_cell *),
	102	+ void context, struct dm_bio_prison_cell cell);
	103	+
90	104	/----------------------------------------------------------------/
91	105
92	106	/*

drivers/md/dm-bufio.c+148−78

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	#include <linux/vmalloc.h>
15	15	#include <linux/shrinker.h>
16	16	#include <linux/module.h>
	17	+#include <linux/rbtree.h>
17	18
18	19	#define DM_MSG_PREFIX "bufio"
19	20
@@ -34,26 +35,23 @@ · 14 unmodified lines
34	35	/*
35	36	* Check buffer ages in this interval (seconds)
36	37	*/
37		−#define DM_BUFIO_WORK_TIMER_SECS 10
	38	+#define DM_BUFIO_WORK_TIMER_SECS 30
38	39
39	40	/*
40	41	* Free buffers when they are older than this (seconds)
41	42	*/
42		−#define DM_BUFIO_DEFAULT_AGE_SECS 60
	43	+#define DM_BUFIO_DEFAULT_AGE_SECS 300
43	44
44	45	/*
45		− * The number of bvec entries that are embedded directly in the buffer.
46		− * If the chunk size is larger, dm-io is used to do the io.
	46	+ * The nr of bytes of cached data to keep around.
47	47	*/
48		−#define DM_BUFIO_INLINE_VECS 16
	48	+#define DM_BUFIO_DEFAULT_RETAIN_BYTES (256 * 1024)
49	49
50	50	/*
51		− * Buffer hash
	51	+ * The number of bvec entries that are embedded directly in the buffer.
	52	+ * If the chunk size is larger, dm-io is used to do the io.
52	53	*/
53		−#define DM_BUFIO_HASH_BITS 20
54		−#define DM_BUFIO_HASH(block) \
55		− ((((block) >> DM_BUFIO_HASH_BITS) ^ (block)) & \
56		− ((1 << DM_BUFIO_HASH_BITS) - 1))
	54	+#define DM_BUFIO_INLINE_VECS 16
57	55
58	56	/*
59	57	* Don't try to use kmem_cache_alloc for blocks larger than this.
@@ -106,7 +104,7 @@ struct dm_bufio_client { · 46 unmodified lines
106	104
107	105	unsigned minimum_buffers;
108	106
109		− struct hlist_head *cache_hash;
	107	+ struct rb_root buffer_tree;
110	108	wait_queue_head_t free_buffer_wait;
111	109
112	110	int async_write_error;
@@ -135,7 +133,7 @@ enum data_mode { · 22 unmodified lines
135	133	};
136	134
137	135	struct dm_buffer {
138		− struct hlist_node hash_list;
	136	+ struct rb_node node;
139	137	struct list_head lru_list;
140	138	sector_t block;
141	139	void *data;
@@ -223,6 +221,7 @@ static DEFINE_SPINLOCK(param_spinlock); · 81 unmodified lines
223	221	* Buffers are freed after this timeout
224	222	*/
225	223	static unsigned dm_bufio_max_age = DM_BUFIO_DEFAULT_AGE_SECS;
	224	+static unsigned dm_bufio_retain_bytes = DM_BUFIO_DEFAULT_RETAIN_BYTES;
226	225
227	226	static unsigned long dm_bufio_peak_allocated;
228	227	static unsigned long dm_bufio_allocated_kmem_cache;
@@ -253,6 +252,53 @@ static LIST_HEAD(dm_bufio_all_clients); · 24 unmodified lines
253	252	*/
254	253	static DEFINE_MUTEX(dm_bufio_clients_lock);
255	254
	255	+/*----------------------------------------------------------------
	256	+ * A red/black tree acts as an index for all the buffers.
	257	+ --------------------------------------------------------------/
	258	+static struct dm_buffer __find(struct dm_bufio_client c, sector_t block)
	259	+{
	260	+ struct rb_node *n = c->buffer_tree.rb_node;
	261	+ struct dm_buffer *b;
	262	+
	263	+ while (n) {
	264	+ b = container_of(n, struct dm_buffer, node);
	265	+
	266	+ if (b->block == block)
	267	+ return b;
	268	+
	269	+ n = (b->block < block) ? n->rb_left : n->rb_right;
	270	+ }
	271	+
	272	+ return NULL;
	273	+}
	274	+
	275	+static void __insert(struct dm_bufio_client c, struct dm_buffer b)
	276	+{
	277	+ struct rb_node *new = &c->buffer_tree.rb_node, parent = NULL;
	278	+ struct dm_buffer *found;
	279	+
	280	+ while (*new) {
	281	+ found = container_of(*new, struct dm_buffer, node);
	282	+
	283	+ if (found->block == b->block) {
	284	+ BUG_ON(found != b);
	285	+ return;
	286	+ }
	287	+
	288	+ parent = *new;
	289	+ new = (found->block < b->block) ?
	290	+ &((new)->rb_left) : &((new)->rb_right);
	291	+ }
	292	+
	293	+ rb_link_node(&b->node, parent, new);
	294	+ rb_insert_color(&b->node, &c->buffer_tree);
	295	+}
	296	+
	297	+static void __remove(struct dm_bufio_client c, struct dm_buffer b)
	298	+{
	299	+ rb_erase(&b->node, &c->buffer_tree);
	300	+}
	301	+
256	302	/----------------------------------------------------------------/
257	303
258	304	static void adjust_total_allocated(enum data_mode data_mode, long diff)
@@ -434,7 +480,7 @@ static void __link_buffer(struct dm_buffer *b, sector_t block, int dirty) · 175 unmodified lines
434	480	b->block = block;
435	481	b->list_mode = dirty;
436	482	list_add(&b->lru_list, &c->lru[dirty]);
437		− hlist_add_head(&b->hash_list, &c->cache_hash[DM_BUFIO_HASH(block)]);
	483	+ __insert(b->c, b);
438	484	b->last_accessed = jiffies;
439	485	}
440	486
@@ -448,7 +494,7 @@ static void __unlink_buffer(struct dm_buffer *b) · 7 unmodified lines
448	494	BUG_ON(!c->n_buffers[b->list_mode]);
449	495
450	496	c->n_buffers[b->list_mode]--;
451		− hlist_del(&b->hash_list);
	497	+ __remove(b->c, b);
452	498	list_del(&b->lru_list);
453	499	}
454	500
@@ -532,6 +578,19 @@ static void use_dmio(struct dm_buffer *b, int rw, sector_t block, · 77 unmodified lines
532	578	end_io(&b->bio, r);
533	579	}
534	580
	581	+static void inline_endio(struct bio *bio, int error)
	582	+{
	583	+ bio_end_io_t *end_fn = bio->bi_private;
	584	+
	585	+ /*
	586	+ * Reset the bio to free any attached resources
	587	+ * (e.g. bio integrity profiles).
	588	+ */
	589	+ bio_reset(bio);
	590	+
	591	+ end_fn(bio, error);
	592	+}
	593	+
535	594	static void use_inline_bio(struct dm_buffer *b, int rw, sector_t block,
536	595	bio_end_io_t *end_io)
537	596	{
@@ -543,7 +602,12 @@ static void use_inline_bio(struct dm_buffer *b, int rw, sector_t block, · 5 unmodified lines
543	602	b->bio.bi_max_vecs = DM_BUFIO_INLINE_VECS;
544	603	b->bio.bi_iter.bi_sector = block << b->c->sectors_per_block_bits;
545	604	b->bio.bi_bdev = b->c->bdev;
546		− b->bio.bi_end_io = end_io;
	605	+ b->bio.bi_end_io = inline_endio;
	606	+ /*
	607	+ * Use of .bi_private isn't a problem here because
	608	+ * the dm_buffer's inline bio is local to bufio.
	609	+ */
	610	+ b->bio.bi_private = end_io;
547	611
548	612	/*
549	613	* We assume that if len >= PAGE_SIZE ptr is page-aligned.
@@ -887,23 +951,6 @@ static void __check_watermark(struct dm_bufio_client *c, · 337 unmodified lines
887	951	__write_dirty_buffers_async(c, 1, write_list);
888	952	}
889	953
890		−/*
891		− * Find a buffer in the hash.
892		− */
893		−static struct dm_buffer __find(struct dm_bufio_client c, sector_t block)
894		−{
895		− struct dm_buffer *b;
896		−
897		− hlist_for_each_entry(b, &c->cache_hash[DM_BUFIO_HASH(block)],
898		− hash_list) {
899		− dm_bufio_cond_resched();
900		− if (b->block == block)
901		− return b;
902		− }
903		−
904		− return NULL;
905		−}
906		−
907	954	/*----------------------------------------------------------------
908	955	* Getting a buffer
909	956	--------------------------------------------------------------/
@@ -1433,45 +1480,52 @@ static void drop_buffers(struct dm_bufio_client *c) · 523 unmodified lines
1433	1480	}
1434	1481
1435	1482	/*
1436		− * Test if the buffer is unused and too old, and commit it.
	1483	+ * We may not be able to evict this buffer if IO pending or the client
	1484	+ * is still using it. Caller is expected to know buffer is too old.
	1485	+ *
1437	1486	* And if GFP_NOFS is used, we must not do any I/O because we hold
1438	1487	* dm_bufio_clients_lock and we would risk deadlock if the I/O gets
1439	1488	* rerouted to different bufio client.
1440	1489	*/
1441		−static int __cleanup_old_buffer(struct dm_buffer *b, gfp_t gfp,
1442		− unsigned long max_jiffies)
	1490	+static bool __try_evict_buffer(struct dm_buffer *b, gfp_t gfp)
1443	1491	{
1444		− if (jiffies - b->last_accessed < max_jiffies)
1445		− return 0;
1446		−
1447	1492	if (!(gfp & __GFP_FS)) {
1448	1493	if (test_bit(B_READING, &b->state) \|\|
1449	1494	test_bit(B_WRITING, &b->state) \|\|
1450	1495	test_bit(B_DIRTY, &b->state))
1451		− return 0;
	1496	+ return false;
1452	1497	}
1453	1498
1454	1499	if (b->hold_count)
1455		− return 0;
	1500	+ return false;
1456	1501
1457	1502	__make_buffer_clean(b);
1458	1503	__unlink_buffer(b);
1459	1504	__free_buffer_wake(b);
1460	1505
1461		− return 1;
	1506	+ return true;
1462	1507	}
1463	1508
1464		−static long __scan(struct dm_bufio_client *c, unsigned long nr_to_scan,
1465		− gfp_t gfp_mask)
	1509	+static unsigned get_retain_buffers(struct dm_bufio_client *c)
	1510	+{
	1511	+ unsigned retain_bytes = ACCESS_ONCE(dm_bufio_retain_bytes);
	1512	+ return retain_bytes / c->block_size;
	1513	+}
	1514	+
	1515	+static unsigned long __scan(struct dm_bufio_client *c, unsigned long nr_to_scan,
	1516	+ gfp_t gfp_mask)
1466	1517	{
1467	1518	int l;
1468	1519	struct dm_buffer b, tmp;
1469		− long freed = 0;
	1520	+ unsigned long freed = 0;
	1521	+ unsigned long count = nr_to_scan;
	1522	+ unsigned retain_target = get_retain_buffers(c);
1470	1523
1471	1524	for (l = 0; l < LIST_SIZE; l++) {
1472	1525	list_for_each_entry_safe_reverse(b, tmp, &c->lru[l], lru_list) {
1473		− freed += __cleanup_old_buffer(b, gfp_mask, 0);
1474		− if (!--nr_to_scan)
	1526	+ if (__try_evict_buffer(b, gfp_mask))
	1527	+ freed++;
	1528	+ if (!--nr_to_scan \|\| ((count - freed) <= retain_target))
1475	1529	return freed;
1476	1530	dm_bufio_cond_resched();
1477	1531	}
@@ -1533,11 +1587,7 @@ struct dm_bufio_client dm_bufio_client_create(struct block_device bdev, unsign · 55 unmodified lines
1533	1587	r = -ENOMEM;
1534	1588	goto bad_client;
1535	1589	}
1536		− c->cache_hash = vmalloc(sizeof(struct hlist_head) << DM_BUFIO_HASH_BITS);
1537		− if (!c->cache_hash) {
1538		− r = -ENOMEM;
1539		− goto bad_hash;
1540		− }
	1590	+ c->buffer_tree = RB_ROOT;
1541	1591
1542	1592	c->bdev = bdev;
1543	1593	c->block_size = block_size;
@@ -1556,9 +1606,6 @@ struct dm_bufio_client dm_bufio_client_create(struct block_device bdev, unsign · 12 unmodified lines
1556	1606	c->n_buffers[i] = 0;
1557	1607	}
1558	1608
1559		− for (i = 0; i < 1 << DM_BUFIO_HASH_BITS; i++)
1560		− INIT_HLIST_HEAD(&c->cache_hash[i]);
1561		−
1562	1609	mutex_init(&c->lock);
1563	1610	INIT_LIST_HEAD(&c->reserved_buffers);
1564	1611	c->need_reserved_buffers = reserved_buffers;
@@ -1632,8 +1679,6 @@ bad_cache: · 67 unmodified lines
1632	1679	}
1633	1680	dm_io_client_destroy(c->dm_io);
1634	1681	bad_dm_io:
1635		− vfree(c->cache_hash);
1636		−bad_hash:
1637	1682	kfree(c);
1638	1683	bad_client:
1639	1684	return ERR_PTR(r);
@@ -1660,9 +1705,7 @@ void dm_bufio_client_destroy(struct dm_bufio_client *c) · 20 unmodified lines
1660	1705
1661	1706	mutex_unlock(&dm_bufio_clients_lock);
1662	1707
1663		− for (i = 0; i < 1 << DM_BUFIO_HASH_BITS; i++)
1664		− BUG_ON(!hlist_empty(&c->cache_hash[i]));
1665		−
	1708	+ BUG_ON(!RB_EMPTY_ROOT(&c->buffer_tree));
1666	1709	BUG_ON(c->need_reserved_buffers);
1667	1710
1668	1711	while (!list_empty(&c->reserved_buffers)) {
@@ -1680,36 +1723,60 @@ void dm_bufio_client_destroy(struct dm_bufio_client *c) · 11 unmodified lines
1680	1723	BUG_ON(c->n_buffers[i]);
1681	1724
1682	1725	dm_io_client_destroy(c->dm_io);
1683		− vfree(c->cache_hash);
1684	1726	kfree(c);
1685	1727	}
1686	1728	EXPORT_SYMBOL_GPL(dm_bufio_client_destroy);
1687	1729
1688		−static void cleanup_old_buffers(void)
	1730	+static unsigned get_max_age_hz(void)
1689	1731	{
1690		− unsigned long max_age = ACCESS_ONCE(dm_bufio_max_age);
1691		− struct dm_bufio_client *c;
	1732	+ unsigned max_age = ACCESS_ONCE(dm_bufio_max_age);
1692	1733
1693		− if (max_age > ULONG_MAX / HZ)
1694		− max_age = ULONG_MAX / HZ;
	1734	+ if (max_age > UINT_MAX / HZ)
	1735	+ max_age = UINT_MAX / HZ;
1695	1736
1696		− mutex_lock(&dm_bufio_clients_lock);
1697		− list_for_each_entry(c, &dm_bufio_all_clients, client_list) {
1698		− if (!dm_bufio_trylock(c))
1699		− continue;
	1737	+ return max_age * HZ;
	1738	+}
1700	1739
1701		− while (!list_empty(&c->lru[LIST_CLEAN])) {
1702		− struct dm_buffer *b;
1703		− b = list_entry(c->lru[LIST_CLEAN].prev,
1704		− struct dm_buffer, lru_list);
1705		− if (!__cleanup_old_buffer(b, 0, max_age * HZ))
1706		− break;
1707		− dm_bufio_cond_resched();
1708		− }
	1740	+static bool older_than(struct dm_buffer *b, unsigned long age_hz)
	1741	+{
	1742	+ return (jiffies - b->last_accessed) >= age_hz;
	1743	+}
	1744	+
	1745	+static void __evict_old_buffers(struct dm_bufio_client *c, unsigned long age_hz)
	1746	+{
	1747	+ struct dm_buffer b, tmp;
	1748	+ unsigned retain_target = get_retain_buffers(c);
	1749	+ unsigned count;
	1750	+
	1751	+ dm_bufio_lock(c);
	1752	+
	1753	+ count = c->n_buffers[LIST_CLEAN] + c->n_buffers[LIST_DIRTY];
	1754	+ list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_CLEAN], lru_list) {
	1755	+ if (count <= retain_target)
	1756	+ break;
	1757	+
	1758	+ if (!older_than(b, age_hz))
	1759	+ break;
	1760	+
	1761	+ if (__try_evict_buffer(b, 0))
	1762	+ count--;
1709	1763
1710		− dm_bufio_unlock(c);
1711	1764	dm_bufio_cond_resched();
1712	1765	}
	1766	+
	1767	+ dm_bufio_unlock(c);
	1768	+}
	1769	+
	1770	+static void cleanup_old_buffers(void)
	1771	+{
	1772	+ unsigned long max_age_hz = get_max_age_hz();
	1773	+ struct dm_bufio_client *c;
	1774	+
	1775	+ mutex_lock(&dm_bufio_clients_lock);
	1776	+
	1777	+ list_for_each_entry(c, &dm_bufio_all_clients, client_list)
	1778	+ __evict_old_buffers(c, max_age_hz);
	1779	+
1713	1780	mutex_unlock(&dm_bufio_clients_lock);
1714	1781	}
1715	1782
@@ -1834,6 +1901,9 @@ MODULE_PARM_DESC(max_cache_size_bytes, "Size of metadata cache"); · 118 unmodified lines
1834	1901	module_param_named(max_age_seconds, dm_bufio_max_age, uint, S_IRUGO \| S_IWUSR);
1835	1902	MODULE_PARM_DESC(max_age_seconds, "Max age of a buffer in seconds");
1836	1903
	1904	+module_param_named(retain_bytes, dm_bufio_retain_bytes, uint, S_IRUGO \| S_IWUSR);
	1905	+MODULE_PARM_DESC(retain_bytes, "Try to keep at least this many bytes cached in memory");
	1906	+
1837	1907	module_param_named(peak_allocated_bytes, dm_bufio_peak_allocated, ulong, S_IRUGO \| S_IWUSR);
1838	1908	MODULE_PARM_DESC(peak_allocated_bytes, "Tracks the maximum allocated memory");
1839	1909

drivers/md/dm-cache-block-types.h+11−0

@@ -19,6 +19,7 @@ · 18 unmodified lines
19	19
20	20	typedef dm_block_t __bitwise__ dm_oblock_t;
21	21	typedef uint32_t __bitwise__ dm_cblock_t;
	22	+typedef dm_block_t __bitwise__ dm_dblock_t;
22	23
23	24	static inline dm_oblock_t to_oblock(dm_block_t b)
24	25	{
@@ -40,4 +41,14 @@ static inline uint32_t from_cblock(dm_cblock_t b) · 15 unmodified lines
40	41	return (__force uint32_t) b;
41	42	}
42	43
	44	+static inline dm_dblock_t to_dblock(dm_block_t b)
	45	+{
	46	+ return (__force dm_dblock_t) b;
	47	+}
	48	+
	49	+static inline dm_block_t from_dblock(dm_dblock_t b)
	50	+{
	51	+ return (__force dm_block_t) b;
	52	+}
	53	+
43	54	#endif /* DM_CACHE_BLOCK_TYPES_H */

drivers/md/dm-cache-metadata.c+17−17

@@ -109,7 +109,7 @@ struct dm_cache_metadata { · 108 unmodified lines
109	109	dm_block_t discard_root;
110	110
111	111	sector_t discard_block_size;
112		− dm_oblock_t discard_nr_blocks;
	112	+ dm_dblock_t discard_nr_blocks;
113	113
114	114	sector_t data_block_size;
115	115	dm_cblock_t cache_blocks;
@@ -329,7 +329,7 @@ static int __write_initial_superblock(struct dm_cache_metadata *cmd) · 213 unmodified lines
329	329	disk_super->hint_root = cpu_to_le64(cmd->hint_root);
330	330	disk_super->discard_root = cpu_to_le64(cmd->discard_root);
331	331	disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);
332		− disk_super->discard_nr_blocks = cpu_to_le64(from_oblock(cmd->discard_nr_blocks));
	332	+ disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));
333	333	disk_super->metadata_block_size = cpu_to_le32(DM_CACHE_METADATA_BLOCK_SIZE);
334	334	disk_super->data_block_size = cpu_to_le32(cmd->data_block_size);
335	335	disk_super->cache_blocks = cpu_to_le32(0);
@@ -528,7 +528,7 @@ static void read_superblock_fields(struct dm_cache_metadata *cmd, · 192 unmodified lines
528	528	cmd->hint_root = le64_to_cpu(disk_super->hint_root);
529	529	cmd->discard_root = le64_to_cpu(disk_super->discard_root);
530	530	cmd->discard_block_size = le64_to_cpu(disk_super->discard_block_size);
531		− cmd->discard_nr_blocks = to_oblock(le64_to_cpu(disk_super->discard_nr_blocks));
	531	+ cmd->discard_nr_blocks = to_dblock(le64_to_cpu(disk_super->discard_nr_blocks));
532	532	cmd->data_block_size = le32_to_cpu(disk_super->data_block_size);
533	533	cmd->cache_blocks = to_cblock(le32_to_cpu(disk_super->cache_blocks));
534	534	strncpy(cmd->policy_name, disk_super->policy_name, sizeof(cmd->policy_name));
@@ -626,7 +626,7 @@ static int __commit_transaction(struct dm_cache_metadata *cmd, · 91 unmodified lines
626	626	disk_super->hint_root = cpu_to_le64(cmd->hint_root);
627	627	disk_super->discard_root = cpu_to_le64(cmd->discard_root);
628	628	disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);
629		− disk_super->discard_nr_blocks = cpu_to_le64(from_oblock(cmd->discard_nr_blocks));
	629	+ disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));
630	630	disk_super->cache_blocks = cpu_to_le32(from_cblock(cmd->cache_blocks));
631	631	strncpy(disk_super->policy_name, cmd->policy_name, sizeof(disk_super->policy_name));
632	632	disk_super->policy_version[0] = cpu_to_le32(cmd->policy_version[0]);
@@ -797,15 +797,15 @@ out: · 164 unmodified lines
797	797
798	798	int dm_cache_discard_bitset_resize(struct dm_cache_metadata *cmd,
799	799	sector_t discard_block_size,
800		− dm_oblock_t new_nr_entries)
	800	+ dm_dblock_t new_nr_entries)
801	801	{
802	802	int r;
803	803
804	804	down_write(&cmd->root_lock);
805	805	r = dm_bitset_resize(&cmd->discard_info,
806	806	cmd->discard_root,
807		− from_oblock(cmd->discard_nr_blocks),
808		− from_oblock(new_nr_entries),
	807	+ from_dblock(cmd->discard_nr_blocks),
	808	+ from_dblock(new_nr_entries),
809	809	false, &cmd->discard_root);
810	810	if (!r) {
811	811	cmd->discard_block_size = discard_block_size;
@@ -818,28 +818,28 @@ int dm_cache_discard_bitset_resize(struct dm_cache_metadata *cmd, · 6 unmodified lines
818	818	return r;
819	819	}
820	820
821		−static int __set_discard(struct dm_cache_metadata *cmd, dm_oblock_t b)
	821	+static int __set_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)
822	822	{
823	823	return dm_bitset_set_bit(&cmd->discard_info, cmd->discard_root,
824		− from_oblock(b), &cmd->discard_root);
	824	+ from_dblock(b), &cmd->discard_root);
825	825	}
826	826
827		−static int __clear_discard(struct dm_cache_metadata *cmd, dm_oblock_t b)
	827	+static int __clear_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)
828	828	{
829	829	return dm_bitset_clear_bit(&cmd->discard_info, cmd->discard_root,
830		− from_oblock(b), &cmd->discard_root);
	830	+ from_dblock(b), &cmd->discard_root);
831	831	}
832	832
833		−static int __is_discarded(struct dm_cache_metadata *cmd, dm_oblock_t b,
	833	+static int __is_discarded(struct dm_cache_metadata *cmd, dm_dblock_t b,
834	834	bool *is_discarded)
835	835	{
836	836	return dm_bitset_test_bit(&cmd->discard_info, cmd->discard_root,
837		− from_oblock(b), &cmd->discard_root,
	837	+ from_dblock(b), &cmd->discard_root,
838	838	is_discarded);
839	839	}
840	840
841	841	static int __discard(struct dm_cache_metadata *cmd,
842		− dm_oblock_t dblock, bool discard)
	842	+ dm_dblock_t dblock, bool discard)
843	843	{
844	844	int r;
845	845
@@ -852,7 +852,7 @@ static int __discard(struct dm_cache_metadata *cmd, · 6 unmodified lines
852	852	}
853	853
854	854	int dm_cache_set_discard(struct dm_cache_metadata *cmd,
855		− dm_oblock_t dblock, bool discard)
	855	+ dm_dblock_t dblock, bool discard)
856	856	{
857	857	int r;
858	858
@@ -870,8 +870,8 @@ static int __load_discards(struct dm_cache_metadata *cmd, · 11 unmodified lines
870	870	dm_block_t b;
871	871	bool discard;
872	872
873		− for (b = 0; b < from_oblock(cmd->discard_nr_blocks); b++) {
874		− dm_oblock_t dblock = to_oblock(b);
	873	+ for (b = 0; b < from_dblock(cmd->discard_nr_blocks); b++) {
	874	+ dm_dblock_t dblock = to_dblock(b);
875	875
876	876	if (cmd->clean_when_opened) {
877	877	r = __is_discarded(cmd, dblock, &discard);

drivers/md/dm-cache-metadata.h+3−3

@@ -70,14 +70,14 @@ dm_cblock_t dm_cache_size(struct dm_cache_metadata *cmd); · 69 unmodified lines
70	70
71	71	int dm_cache_discard_bitset_resize(struct dm_cache_metadata *cmd,
72	72	sector_t discard_block_size,
73		− dm_oblock_t new_nr_entries);
	73	+ dm_dblock_t new_nr_entries);
74	74
75	75	typedef int (load_discard_fn)(void context, sector_t discard_block_size,
76		− dm_oblock_t dblock, bool discarded);
	76	+ dm_dblock_t dblock, bool discarded);
77	77	int dm_cache_load_discards(struct dm_cache_metadata *cmd,
78	78	load_discard_fn fn, void *context);
79	79
80		−int dm_cache_set_discard(struct dm_cache_metadata *cmd, dm_oblock_t dblock, bool discard);
	80	+int dm_cache_set_discard(struct dm_cache_metadata *cmd, dm_dblock_t dblock, bool discard);
81	81
82	82	int dm_cache_remove_mapping(struct dm_cache_metadata *cmd, dm_cblock_t cblock);
83	83	int dm_cache_insert_mapping(struct dm_cache_metadata *cmd, dm_cblock_t cblock, dm_oblock_t oblock);

drivers/md/dm-cache-policy-mq.c+54−28

@@ -181,24 +181,30 @@ static void queue_shift_down(struct queue *q) · 180 unmodified lines
181	181	* Gives us the oldest entry of the lowest popoulated level. If the first
182	182	* level is emptied then we shift down one level.
183	183	*/
184		−static struct list_head queue_pop(struct queue q)
	184	+static struct list_head queue_peek(struct queue q)
185	185	{
186	186	unsigned level;
187		− struct list_head *r;
188	187
189	188	for (level = 0; level < NR_QUEUE_LEVELS; level++)
190		− if (!list_empty(q->qs + level)) {
191		− r = q->qs[level].next;
192		− list_del(r);
	189	+ if (!list_empty(q->qs + level))
	190	+ return q->qs[level].next;
193	191
194		− /* have we just emptied the bottom level? */
195		− if (level == 0 && list_empty(q->qs))
196		− queue_shift_down(q);
	192	+ return NULL;
	193	+}
197	194
198		− return r;
199		− }
	195	+static struct list_head queue_pop(struct queue q)
	196	+{
	197	+ struct list_head *r = queue_peek(q);
200	198
201		− return NULL;
	199	+ if (r) {
	200	+ list_del(r);
	201	+
	202	+ /* have we just emptied the bottom level? */
	203	+ if (list_empty(q->qs))
	204	+ queue_shift_down(q);
	205	+ }
	206	+
	207	+ return r;
202	208	}
203	209
204	210	static struct list_head list_pop(struct list_head lh)
@@ -383,13 +389,6 @@ struct mq_policy { · 178 unmodified lines
383	389	unsigned generation;
384	390	unsigned generation_period; /* in lookups (will probably change) */
385	391
386		− /*
387		− * Entries in the pre_cache whose hit count passes the promotion
388		− * threshold move to the cache proper. Working out the correct
389		− * value for the promotion_threshold is crucial to this policy.
390		− */
391		− unsigned promote_threshold;
392		−
393	392	unsigned discard_promote_adjustment;
394	393	unsigned read_promote_adjustment;
395	394	unsigned write_promote_adjustment;
@@ -406,6 +405,7 @@ struct mq_policy { · 10 unmodified lines
406	405	#define DEFAULT_DISCARD_PROMOTE_ADJUSTMENT 1
407	406	#define DEFAULT_READ_PROMOTE_ADJUSTMENT 4
408	407	#define DEFAULT_WRITE_PROMOTE_ADJUSTMENT 8
	408	+#define DISCOURAGE_DEMOTING_DIRTY_THRESHOLD 128
409	409
410	410	/----------------------------------------------------------------/
411	411
@@ -518,6 +518,12 @@ static struct entry pop(struct mq_policy mq, struct queue *q) · 106 unmodified lines
518	518	return e;
519	519	}
520	520
	521	+static struct entry peek(struct queue q)
	522	+{
	523	+ struct list_head *h = queue_peek(q);
	524	+ return h ? container_of(h, struct entry, list) : NULL;
	525	+}
	526	+
521	527	/*
522	528	* Has this entry already been updated?
523	529	*/
@@ -570,10 +576,6 @@ static void check_generation(struct mq_policy *mq) · 46 unmodified lines
570	576	break;
571	577	}
572	578	}
573		−
574		− mq->promote_threshold = nr ? total / nr : 1;
575		− if (mq->promote_threshold * nr < total)
576		− mq->promote_threshold++;
577	579	}
578	580	}
579	581
@@ -640,6 +642,30 @@ static int demote_cblock(struct mq_policy mq, dm_oblock_t oblock) · 60 unmodified lines
640	642	return 0;
641	643	}
642	644
	645	+/*
	646	+ * Entries in the pre_cache whose hit count passes the promotion
	647	+ * threshold move to the cache proper. Working out the correct
	648	+ * value for the promotion_threshold is crucial to this policy.
	649	+ */
	650	+static unsigned promote_threshold(struct mq_policy *mq)
	651	+{
	652	+ struct entry *e;
	653	+
	654	+ if (any_free_cblocks(mq))
	655	+ return 0;
	656	+
	657	+ e = peek(&mq->cache_clean);
	658	+ if (e)
	659	+ return e->hit_count;
	660	+
	661	+ e = peek(&mq->cache_dirty);
	662	+ if (e)
	663	+ return e->hit_count + DISCOURAGE_DEMOTING_DIRTY_THRESHOLD;
	664	+
	665	+ /* This should never happen */
	666	+ return 0;
	667	+}
	668	+
643	669	/*
644	670	* We modify the basic promotion_threshold depending on the specific io.
645	671	*
@@ -653,7 +679,7 @@ static unsigned adjusted_promote_threshold(struct mq_policy *mq, · 7 unmodified lines
653	679	bool discarded_oblock, int data_dir)
654	680	{
655	681	if (data_dir == READ)
656		− return mq->promote_threshold + mq->read_promote_adjustment;
	682	+ return promote_threshold(mq) + mq->read_promote_adjustment;
657	683
658	684	if (discarded_oblock && (any_free_cblocks(mq) \|\| any_clean_cblocks(mq))) {
659	685	/*
@@ -663,7 +689,7 @@ static unsigned adjusted_promote_threshold(struct mq_policy *mq, · 3 unmodified lines
663	689	return mq->discard_promote_adjustment;
664	690	}
665	691
666		− return mq->promote_threshold + mq->write_promote_adjustment;
	692	+ return promote_threshold(mq) + mq->write_promote_adjustment;
667	693	}
668	694
669	695	static bool should_promote(struct mq_policy mq, struct entry e,
@@ -839,7 +865,8 @@ static int map(struct mq_policy *mq, dm_oblock_t oblock, · 169 unmodified lines
839	865	if (e && in_cache(mq, e))
840	866	r = cache_entry_found(mq, e, result);
841	867
842		− else if (iot_pattern(&mq->tracker) == PATTERN_SEQUENTIAL)
	868	+ else if (mq->tracker.thresholds[PATTERN_SEQUENTIAL] &&
	869	+ iot_pattern(&mq->tracker) == PATTERN_SEQUENTIAL)
843	870	result->op = POLICY_MISS;
844	871
845	872	else if (e)
@@ -1230,7 +1257,6 @@ static struct dm_cache_policy *mq_create(dm_cblock_t cache_size, · 384 unmodified lines
1230	1257	mq->tick = 0;
1231	1258	mq->hit_count = 0;
1232	1259	mq->generation = 0;
1233		− mq->promote_threshold = 0;
1234	1260	mq->discard_promote_adjustment = DEFAULT_DISCARD_PROMOTE_ADJUSTMENT;
1235	1261	mq->read_promote_adjustment = DEFAULT_READ_PROMOTE_ADJUSTMENT;
1236	1262	mq->write_promote_adjustment = DEFAULT_WRITE_PROMOTE_ADJUSTMENT;
@@ -1265,7 +1291,7 @@ bad_pre_cache_init: · 28 unmodified lines
1265	1291
1266	1292	static struct dm_cache_policy_type mq_policy_type = {
1267	1293	.name = "mq",
1268		− .version = {1, 2, 0},
	1294	+ .version = {1, 3, 0},
1269	1295	.hint_size = 4,
1270	1296	.owner = THIS_MODULE,
1271	1297	.create = mq_create
@@ -1273,7 +1299,7 @@ static struct dm_cache_policy_type mq_policy_type = { · 1 unmodified line
1273	1299
1274	1300	static struct dm_cache_policy_type default_policy_type = {
1275	1301	.name = "default",
1276		− .version = {1, 2, 0},
	1302	+ .version = {1, 3, 0},
1277	1303	.hint_size = 4,
1278	1304	.owner = THIS_MODULE,
1279	1305	.create = mq_create,

drivers/md/dm-cache-target.c+301−77

@@ -95,7 +95,6 @@ static void dm_unhook_bio(struct dm_hook_info h, struct bio bio) · 94 unmodified lines
95	95
96	96	/----------------------------------------------------------------/
97	97
98		−#define PRISON_CELLS 1024
99	98	#define MIGRATION_POOL_SIZE 128
100	99	#define COMMIT_PERIOD HZ
101	100	#define MIGRATION_COUNT_WINDOW 10
@@ -237,8 +236,9 @@ struct cache { · 135 unmodified lines
237	236	/*
238	237	* origin_blocks entries, discarded if set.
239	238	*/
240		− dm_oblock_t discard_nr_blocks;
	239	+ dm_dblock_t discard_nr_blocks;
241	240	unsigned long *discard_bitset;
	241	+ uint32_t discard_block_size; /* a power of 2 times sectors per block */
242	242
243	243	/*
244	244	* Rather than reconstructing the table line for the status we just
@@ -310,6 +310,7 @@ struct dm_cache_migration { · 65 unmodified lines
310	310	dm_cblock_t cblock;
311	311
312	312	bool err:1;
	313	+ bool discard:1;
313	314	bool writeback:1;
314	315	bool demote:1;
315	316	bool promote:1;
@@ -433,11 +434,12 @@ static void prealloc_put_cell(struct prealloc p, struct dm_bio_prison_cell cel · 117 unmodified lines
433	434
434	435	/----------------------------------------------------------------/
435	436
436		−static void build_key(dm_oblock_t oblock, struct dm_cell_key *key)
	437	+static void build_key(dm_oblock_t begin, dm_oblock_t end, struct dm_cell_key *key)
437	438	{
438	439	key->virtual = 0;
439	440	key->dev = 0;
440		− key->block = from_oblock(oblock);
	441	+ key->block_begin = from_oblock(begin);
	442	+ key->block_end = from_oblock(end);
441	443	}
442	444
443	445	/*
@@ -447,15 +449,15 @@ static void build_key(dm_oblock_t oblock, struct dm_cell_key *key) · 3 unmodified lines
447	449	*/
448	450	typedef void (cell_free_fn)(void context, struct dm_bio_prison_cell *cell);
449	451
450		−static int bio_detain(struct cache *cache, dm_oblock_t oblock,
451		− struct bio bio, struct dm_bio_prison_cell cell_prealloc,
452		− cell_free_fn free_fn, void *free_context,
453		− struct dm_bio_prison_cell **cell_result)
	452	+static int bio_detain_range(struct cache *cache, dm_oblock_t oblock_begin, dm_oblock_t oblock_end,
	453	+ struct bio bio, struct dm_bio_prison_cell cell_prealloc,
	454	+ cell_free_fn free_fn, void *free_context,
	455	+ struct dm_bio_prison_cell **cell_result)
454	456	{
455	457	int r;
456	458	struct dm_cell_key key;
457	459
458		− build_key(oblock, &key);
	460	+ build_key(oblock_begin, oblock_end, &key);
459	461	r = dm_bio_detain(cache->prison, &key, bio, cell_prealloc, cell_result);
460	462	if (r)
461	463	free_fn(free_context, cell_prealloc);
@@ -463,6 +465,16 @@ static int bio_detain(struct cache *cache, dm_oblock_t oblock, · 1 unmodified line
463	465	return r;
464	466	}
465	467
	468	+static int bio_detain(struct cache *cache, dm_oblock_t oblock,
	469	+ struct bio bio, struct dm_bio_prison_cell cell_prealloc,
	470	+ cell_free_fn free_fn, void *free_context,
	471	+ struct dm_bio_prison_cell **cell_result)
	472	+{
	473	+ dm_oblock_t end = to_oblock(from_oblock(oblock) + 1ULL);
	474	+ return bio_detain_range(cache, oblock, end, bio,
	475	+ cell_prealloc, free_fn, free_context, cell_result);
	476	+}
	477	+
466	478	static int get_cell(struct cache *cache,
467	479	dm_oblock_t oblock,
468	480	struct prealloc *structs,
@@ -474,7 +486,7 @@ static int get_cell(struct cache *cache, · 5 unmodified lines
474	486
475	487	cell_prealloc = prealloc_get_cell(structs);
476	488
477		− build_key(oblock, &key);
	489	+ build_key(oblock, to_oblock(from_oblock(oblock) + 1ULL), &key);
478	490	r = dm_get_cell(cache->prison, &key, cell_prealloc, cell_result);
479	491	if (r)
480	492	prealloc_put_cell(structs, cell_prealloc);
@@ -524,33 +536,57 @@ static dm_block_t block_div(dm_block_t b, uint32_t n) · 43 unmodified lines
524	536	return b;
525	537	}
526	538
527		−static void set_discard(struct cache *cache, dm_oblock_t b)
	539	+static dm_block_t oblocks_per_dblock(struct cache *cache)
	540	+{
	541	+ dm_block_t oblocks = cache->discard_block_size;
	542	+
	543	+ if (block_size_is_power_of_two(cache))
	544	+ oblocks >>= cache->sectors_per_block_shift;
	545	+ else
	546	+ oblocks = block_div(oblocks, cache->sectors_per_block);
	547	+
	548	+ return oblocks;
	549	+}
	550	+
	551	+static dm_dblock_t oblock_to_dblock(struct cache *cache, dm_oblock_t oblock)
	552	+{
	553	+ return to_dblock(block_div(from_oblock(oblock),
	554	+ oblocks_per_dblock(cache)));
	555	+}
	556	+
	557	+static dm_oblock_t dblock_to_oblock(struct cache *cache, dm_dblock_t dblock)
	558	+{
	559	+ return to_oblock(from_dblock(dblock) * oblocks_per_dblock(cache));
	560	+}
	561	+
	562	+static void set_discard(struct cache *cache, dm_dblock_t b)
528	563	{
529	564	unsigned long flags;
530	565
	566	+ BUG_ON(from_dblock(b) >= from_dblock(cache->discard_nr_blocks));
531	567	atomic_inc(&cache->stats.discard_count);
532	568
533	569	spin_lock_irqsave(&cache->lock, flags);
534		− set_bit(from_oblock(b), cache->discard_bitset);
	570	+ set_bit(from_dblock(b), cache->discard_bitset);
535	571	spin_unlock_irqrestore(&cache->lock, flags);
536	572	}
537	573
538		−static void clear_discard(struct cache *cache, dm_oblock_t b)
	574	+static void clear_discard(struct cache *cache, dm_dblock_t b)
539	575	{
540	576	unsigned long flags;
541	577
542	578	spin_lock_irqsave(&cache->lock, flags);
543		− clear_bit(from_oblock(b), cache->discard_bitset);
	579	+ clear_bit(from_dblock(b), cache->discard_bitset);
544	580	spin_unlock_irqrestore(&cache->lock, flags);
545	581	}
546	582
547		−static bool is_discarded(struct cache *cache, dm_oblock_t b)
	583	+static bool is_discarded(struct cache *cache, dm_dblock_t b)
548	584	{
549	585	int r;
550	586	unsigned long flags;
551	587
552	588	spin_lock_irqsave(&cache->lock, flags);
553		− r = test_bit(from_oblock(b), cache->discard_bitset);
	589	+ r = test_bit(from_dblock(b), cache->discard_bitset);
554	590	spin_unlock_irqrestore(&cache->lock, flags);
555	591
556	592	return r;
@@ -562,7 +598,8 @@ static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b) · 5 unmodified lines
562	598	unsigned long flags;
563	599
564	600	spin_lock_irqsave(&cache->lock, flags);
565		− r = test_bit(from_oblock(b), cache->discard_bitset);
	601	+ r = test_bit(from_dblock(oblock_to_dblock(cache, b)),
	602	+ cache->discard_bitset);
566	603	spin_unlock_irqrestore(&cache->lock, flags);
567	604
568	605	return r;
@@ -687,7 +724,7 @@ static void remap_to_origin_clear_discard(struct cache cache, struct bio bio, · 118 unmodified lines
687	724	check_if_tick_bio_needed(cache, bio);
688	725	remap_to_origin(cache, bio);
689	726	if (bio_data_dir(bio) == WRITE)
690		− clear_discard(cache, oblock);
	727	+ clear_discard(cache, oblock_to_dblock(cache, oblock));
691	728	}
692	729
693	730	static void remap_to_cache_dirty(struct cache cache, struct bio bio,
@@ -697,7 +734,7 @@ static void remap_to_cache_dirty(struct cache cache, struct bio bio, · 3 unmodified lines
697	734	remap_to_cache(cache, bio, cblock);
698	735	if (bio_data_dir(bio) == WRITE) {
699	736	set_dirty(cache, oblock, cblock);
700		− clear_discard(cache, oblock);
	737	+ clear_discard(cache, oblock_to_dblock(cache, oblock));
701	738	}
702	739	}
703	740
@@ -951,10 +988,14 @@ static void migration_success_post_commit(struct dm_cache_migration *mg) · 247 unmodified lines
951	988	}
952	989
953	990	} else {
954		− clear_dirty(cache, mg->new_oblock, mg->cblock);
955		− if (mg->requeue_holder)
	991	+ if (mg->requeue_holder) {
	992	+ clear_dirty(cache, mg->new_oblock, mg->cblock);
956	993	cell_defer(cache, mg->new_ocell, true);
957		− else {
	994	+ } else {
	995	+ /*
	996	+ * The block was promoted via an overwrite, so it's dirty.
	997	+ */
	998	+ set_dirty(cache, mg->new_oblock, mg->cblock);
958	999	bio_endio(mg->new_ocell->holder, 0);
959	1000	cell_defer(cache, mg->new_ocell, false);
960	1001	}
@@ -978,7 +1019,7 @@ static void copy_complete(int read_err, unsigned long write_err, void *context) · 17 unmodified lines
978	1019	wake_worker(cache);
979	1020	}
980	1021
981		−static void issue_copy_real(struct dm_cache_migration *mg)
	1022	+static void issue_copy(struct dm_cache_migration *mg)
982	1023	{
983	1024	int r;
984	1025	struct dm_io_region o_region, c_region;
@@ -1057,11 +1098,46 @@ static void avoid_copy(struct dm_cache_migration *mg) · 72 unmodified lines
1057	1098	migration_success_pre_commit(mg);
1058	1099	}
1059	1100
1060		−static void issue_copy(struct dm_cache_migration *mg)
	1101	+static void calc_discard_block_range(struct cache cache, struct bio bio,
	1102	+ dm_dblock_t b, dm_dblock_t e)
	1103	+{
	1104	+ sector_t sb = bio->bi_iter.bi_sector;
	1105	+ sector_t se = bio_end_sector(bio);
	1106	+
	1107	+ *b = to_dblock(dm_sector_div_up(sb, cache->discard_block_size));
	1108	+
	1109	+ if (se - sb < cache->discard_block_size)
	1110	+ e = b;
	1111	+ else
	1112	+ *e = to_dblock(block_div(se, cache->discard_block_size));
	1113	+}
	1114	+
	1115	+static void issue_discard(struct dm_cache_migration *mg)
	1116	+{
	1117	+ dm_dblock_t b, e;
	1118	+ struct bio *bio = mg->new_ocell->holder;
	1119	+
	1120	+ calc_discard_block_range(mg->cache, bio, &b, &e);
	1121	+ while (b != e) {
	1122	+ set_discard(mg->cache, b);
	1123	+ b = to_dblock(from_dblock(b) + 1);
	1124	+ }
	1125	+
	1126	+ bio_endio(bio, 0);
	1127	+ cell_defer(mg->cache, mg->new_ocell, false);
	1128	+ free_migration(mg);
	1129	+}
	1130	+
	1131	+static void issue_copy_or_discard(struct dm_cache_migration *mg)
1061	1132	{
1062	1133	bool avoid;
1063	1134	struct cache *cache = mg->cache;
1064	1135
	1136	+ if (mg->discard) {
	1137	+ issue_discard(mg);
	1138	+ return;
	1139	+ }
	1140	+
1065	1141	if (mg->writeback \|\| mg->demote)
1066	1142	avoid = !is_dirty(cache, mg->cblock) \|\|
1067	1143	is_discarded_oblock(cache, mg->old_oblock);
@@ -1070,13 +1146,14 @@ static void issue_copy(struct dm_cache_migration *mg) · 2 unmodified lines
1070	1146
1071	1147	avoid = is_discarded_oblock(cache, mg->new_oblock);
1072	1148
1073		− if (!avoid && bio_writes_complete_block(cache, bio)) {
	1149	+ if (writeback_mode(&cache->features) &&
	1150	+ !avoid && bio_writes_complete_block(cache, bio)) {
1074	1151	issue_overwrite(mg, bio);
1075	1152	return;
1076	1153	}
1077	1154	}
1078	1155
1079		− avoid ? avoid_copy(mg) : issue_copy_real(mg);
	1156	+ avoid ? avoid_copy(mg) : issue_copy(mg);
1080	1157	}
1081	1158
1082	1159	static void complete_migration(struct dm_cache_migration *mg)
@@ -1161,6 +1238,7 @@ static void promote(struct cache cache, struct prealloc structs, · 78 unmodified lines
1161	1238	struct dm_cache_migration *mg = prealloc_get_migration(structs);
1162	1239
1163	1240	mg->err = false;
	1241	+ mg->discard = false;
1164	1242	mg->writeback = false;
1165	1243	mg->demote = false;
1166	1244	mg->promote = true;
@@ -1184,6 +1262,7 @@ static void writeback(struct cache cache, struct prealloc structs, · 17 unmodified lines
1184	1262	struct dm_cache_migration *mg = prealloc_get_migration(structs);
1185	1263
1186	1264	mg->err = false;
	1265	+ mg->discard = false;
1187	1266	mg->writeback = true;
1188	1267	mg->demote = false;
1189	1268	mg->promote = false;
@@ -1209,6 +1288,7 @@ static void demote_then_promote(struct cache cache, struct prealloc structs, · 19 unmodified lines
1209	1288	struct dm_cache_migration *mg = prealloc_get_migration(structs);
1210	1289
1211	1290	mg->err = false;
	1291	+ mg->discard = false;
1212	1292	mg->writeback = false;
1213	1293	mg->demote = true;
1214	1294	mg->promote = true;
@@ -1237,6 +1317,7 @@ static void invalidate(struct cache cache, struct prealloc structs, · 22 unmodified lines
1237	1317	struct dm_cache_migration *mg = prealloc_get_migration(structs);
1238	1318
1239	1319	mg->err = false;
	1320	+ mg->discard = false;
1240	1321	mg->writeback = false;
1241	1322	mg->demote = true;
1242	1323	mg->promote = false;
@@ -1253,6 +1334,26 @@ static void invalidate(struct cache cache, struct prealloc structs, · 10 unmodified lines
1253	1334	quiesce_migration(mg);
1254	1335	}
1255	1336
	1337	+static void discard(struct cache cache, struct prealloc structs,
	1338	+ struct dm_bio_prison_cell *cell)
	1339	+{
	1340	+ struct dm_cache_migration *mg = prealloc_get_migration(structs);
	1341	+
	1342	+ mg->err = false;
	1343	+ mg->discard = true;
	1344	+ mg->writeback = false;
	1345	+ mg->demote = false;
	1346	+ mg->promote = false;
	1347	+ mg->requeue_holder = false;
	1348	+ mg->invalidate = false;
	1349	+ mg->cache = cache;
	1350	+ mg->old_ocell = NULL;
	1351	+ mg->new_ocell = cell;
	1352	+ mg->start_jiffies = jiffies;
	1353	+
	1354	+ quiesce_migration(mg);
	1355	+}
	1356	+
1256	1357	/*----------------------------------------------------------------
1257	1358	* bio processing
1258	1359	--------------------------------------------------------------/
@@ -1286,31 +1387,27 @@ static void process_flush_bio(struct cache cache, struct bio bio) · 27 unmodified lines
1286	1387	issue(cache, bio);
1287	1388	}
1288	1389
1289		−/*
1290		− * People generally discard large parts of a device, eg, the whole device
1291		− * when formatting. Splitting these large discards up into cache block
1292		− * sized ios and then quiescing (always neccessary for discard) takes too
1293		− * long.
1294		− *
1295		− * We keep it simple, and allow any size of discard to come in, and just
1296		− * mark off blocks on the discard bitset. No passdown occurs!
1297		− *
1298		− * To implement passdown we need to change the bio_prison such that a cell
1299		− * can have a key that spans many blocks.
1300		− */
1301		−static void process_discard_bio(struct cache cache, struct bio bio)
	1390	+static void process_discard_bio(struct cache cache, struct prealloc structs,
	1391	+ struct bio *bio)
1302	1392	{
1303		− dm_block_t start_block = dm_sector_div_up(bio->bi_iter.bi_sector,
1304		− cache->sectors_per_block);
1305		− dm_block_t end_block = bio_end_sector(bio);
1306		− dm_block_t b;
	1393	+ int r;
	1394	+ dm_dblock_t b, e;
	1395	+ struct dm_bio_prison_cell cell_prealloc, new_ocell;
1307	1396
1308		− end_block = block_div(end_block, cache->sectors_per_block);
	1397	+ calc_discard_block_range(cache, bio, &b, &e);
	1398	+ if (b == e) {
	1399	+ bio_endio(bio, 0);
	1400	+ return;
	1401	+ }
1309	1402
1310		− for (b = start_block; b < end_block; b++)
1311		− set_discard(cache, to_oblock(b));
	1403	+ cell_prealloc = prealloc_get_cell(structs);
	1404	+ r = bio_detain_range(cache, dblock_to_oblock(cache, b), dblock_to_oblock(cache, e), bio, cell_prealloc,
	1405	+ (cell_free_fn) prealloc_put_cell,
	1406	+ structs, &new_ocell);
	1407	+ if (r > 0)
	1408	+ return;
1312	1409
1313		− bio_endio(bio, 0);
	1410	+ discard(cache, structs, new_ocell);
1314	1411	}
1315	1412
1316	1413	static bool spare_migration_bandwidth(struct cache *cache)
@@ -1340,9 +1437,8 @@ static void process_bio(struct cache cache, struct prealloc structs, · 23 unmodified lines
1340	1437	dm_oblock_t block = get_bio_block(cache, bio);
1341	1438	struct dm_bio_prison_cell cell_prealloc, old_ocell, *new_ocell;
1342	1439	struct policy_result lookup_result;
1343		− bool discarded_block = is_discarded_oblock(cache, block);
1344	1440	bool passthrough = passthrough_mode(&cache->features);
1345		− bool can_migrate = !passthrough && (discarded_block \|\| spare_migration_bandwidth(cache));
	1441	+ bool discarded_block, can_migrate;
1346	1442
1347	1443	/*
1348	1444	* Check to see if that block is currently migrating.
@@ -1354,6 +1450,9 @@ static void process_bio(struct cache cache, struct prealloc structs, · 5 unmodified lines
1354	1450	if (r > 0)
1355	1451	return;
1356	1452
	1453	+ discarded_block = is_discarded_oblock(cache, block);
	1454	+ can_migrate = !passthrough && (discarded_block \|\| spare_migration_bandwidth(cache));
	1455	+
1357	1456	r = policy_map(cache->policy, block, true, can_migrate, discarded_block,
1358	1457	bio, &lookup_result);
1359	1458
@@ -1500,7 +1599,7 @@ static void process_deferred_bios(struct cache *cache) · 140 unmodified lines
1500	1599	if (bio->bi_rw & REQ_FLUSH)
1501	1600	process_flush_bio(cache, bio);
1502	1601	else if (bio->bi_rw & REQ_DISCARD)
1503		− process_discard_bio(cache, bio);
	1602	+ process_discard_bio(cache, &structs, bio);
1504	1603	else
1505	1604	process_bio(cache, &structs, bio);
1506	1605	}
@@ -1715,7 +1814,7 @@ static void do_worker(struct work_struct *ws) · 208 unmodified lines
1715	1814	process_invalidation_requests(cache);
1716	1815	}
1717	1816
1718		− process_migrations(cache, &cache->quiesced_migrations, issue_copy);
	1817	+ process_migrations(cache, &cache->quiesced_migrations, issue_copy_or_discard);
1719	1818	process_migrations(cache, &cache->completed_migrations, complete_migration);
1720	1819
1721	1820	if (commit_if_needed(cache)) {
@@ -2180,6 +2279,45 @@ static int create_cache_policy(struct cache cache, struct cache_args ca, · 458 unmodified lines
2180	2279	return 0;
2181	2280	}
2182	2281
	2282	+/*
	2283	+ * We want the discard block size to be at least the size of the cache
	2284	+ * block size and have no more than 2^14 discard blocks across the origin.
	2285	+ */
	2286	+#define MAX_DISCARD_BLOCKS (1 << 14)
	2287	+
	2288	+static bool too_many_discard_blocks(sector_t discard_block_size,
	2289	+ sector_t origin_size)
	2290	+{
	2291	+ (void) sector_div(origin_size, discard_block_size);
	2292	+
	2293	+ return origin_size > MAX_DISCARD_BLOCKS;
	2294	+}
	2295	+
	2296	+static sector_t calculate_discard_block_size(sector_t cache_block_size,
	2297	+ sector_t origin_size)
	2298	+{
	2299	+ sector_t discard_block_size = cache_block_size;
	2300	+
	2301	+ if (origin_size)
	2302	+ while (too_many_discard_blocks(discard_block_size, origin_size))
	2303	+ discard_block_size *= 2;
	2304	+
	2305	+ return discard_block_size;
	2306	+}
	2307	+
	2308	+static void set_cache_size(struct cache *cache, dm_cblock_t size)
	2309	+{
	2310	+ dm_block_t nr_blocks = from_cblock(size);
	2311	+
	2312	+ if (nr_blocks > (1 << 20) && cache->cache_size != size)
	2313	+ DMWARN_LIMIT("You have created a cache device with a lot of individual cache blocks (%llu)\n"
	2314	+ "All these mappings can consume a lot of kernel memory, and take some time to read/write.\n"
	2315	+ "Please consider increasing the cache block size to reduce the overall cache block count.",
	2316	+ (unsigned long long) nr_blocks);
	2317	+
	2318	+ cache->cache_size = size;
	2319	+}
	2320	+
2183	2321	#define DEFAULT_MIGRATION_THRESHOLD 2048
2184	2322
2185	2323	static int cache_create(struct cache_args ca, struct cache *result)
@@ -2204,8 +2342,7 @@ static int cache_create(struct cache_args ca, struct cache *result) · 18 unmodified lines
2204	2342	ti->num_discard_bios = 1;
2205	2343	ti->discards_supported = true;
2206	2344	ti->discard_zeroes_data_unsupported = true;
2207		− /* Discard bios must be split on a block boundary */
2208		− ti->split_discard_bios = true;
	2345	+ ti->split_discard_bios = false;
2209	2346
2210	2347	cache->features = ca->features;
2211	2348	ti->per_bio_data_size = get_per_bio_data_size(cache);
@@ -2235,10 +2372,10 @@ static int cache_create(struct cache_args ca, struct cache *result) · 23 unmodified lines
2235	2372
2236	2373	cache->sectors_per_block_shift = -1;
2237	2374	cache_size = block_div(cache_size, ca->block_size);
2238		− cache->cache_size = to_cblock(cache_size);
	2375	+ set_cache_size(cache, to_cblock(cache_size));
2239	2376	} else {
2240	2377	cache->sectors_per_block_shift = __ffs(ca->block_size);
2241		− cache->cache_size = to_cblock(ca->cache_sectors >> cache->sectors_per_block_shift);
	2378	+ set_cache_size(cache, to_cblock(ca->cache_sectors >> cache->sectors_per_block_shift));
2242	2379	}
2243	2380
2244	2381	r = create_cache_policy(cache, ca, error);
@@ -2303,13 +2440,17 @@ static int cache_create(struct cache_args ca, struct cache *result) · 58 unmodified lines
2303	2440	}
2304	2441	clear_bitset(cache->dirty_bitset, from_cblock(cache->cache_size));
2305	2442
2306		− cache->discard_nr_blocks = cache->origin_blocks;
2307		− cache->discard_bitset = alloc_bitset(from_oblock(cache->discard_nr_blocks));
	2443	+ cache->discard_block_size =
	2444	+ calculate_discard_block_size(cache->sectors_per_block,
	2445	+ cache->origin_sectors);
	2446	+ cache->discard_nr_blocks = to_dblock(dm_sector_div_up(cache->origin_sectors,
	2447	+ cache->discard_block_size));
	2448	+ cache->discard_bitset = alloc_bitset(from_dblock(cache->discard_nr_blocks));
2308	2449	if (!cache->discard_bitset) {
2309	2450	*error = "could not allocate discard bitset";
2310	2451	goto bad;
2311	2452	}
2312		− clear_bitset(cache->discard_bitset, from_oblock(cache->discard_nr_blocks));
	2453	+ clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
2313	2454
2314	2455	cache->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);
2315	2456	if (IS_ERR(cache->copier)) {
@@ -2327,7 +2468,7 @@ static int cache_create(struct cache_args ca, struct cache *result) · 11 unmodified lines
2327	2468	INIT_DELAYED_WORK(&cache->waker, do_waker);
2328	2469	cache->last_commit_jiffies = jiffies;
2329	2470
2330		− cache->prison = dm_bio_prison_create(PRISON_CELLS);
	2471	+ cache->prison = dm_bio_prison_create();
2331	2472	if (!cache->prison) {
2332	2473	*error = "could not create bio prison";
2333	2474	goto bad;
@@ -2549,11 +2690,11 @@ static int __cache_map(struct cache cache, struct bio bio, struct dm_bio_priso · 215 unmodified lines
2549	2690	static int cache_map(struct dm_target ti, struct bio bio)
2550	2691	{
2551	2692	int r;
2552		− struct dm_bio_prison_cell *cell;
	2693	+ struct dm_bio_prison_cell *cell = NULL;
2553	2694	struct cache *cache = ti->private;
2554	2695
2555	2696	r = __cache_map(cache, bio, &cell);
2556		− if (r == DM_MAPIO_REMAPPED) {
	2697	+ if (r == DM_MAPIO_REMAPPED && cell) {
2557	2698	inc_ds(cache, bio, cell);
2558	2699	cell_defer(cache, cell, false);
2559	2700	}
@@ -2599,16 +2740,16 @@ static int write_discard_bitset(struct cache *cache) · 39 unmodified lines
2599	2740	{
2600	2741	unsigned i, r;
2601	2742
2602		− r = dm_cache_discard_bitset_resize(cache->cmd, cache->sectors_per_block,
2603		− cache->origin_blocks);
	2743	+ r = dm_cache_discard_bitset_resize(cache->cmd, cache->discard_block_size,
	2744	+ cache->discard_nr_blocks);
2604	2745	if (r) {
2605	2746	DMERR("could not resize on-disk discard bitset");
2606	2747	return r;
2607	2748	}
2608	2749
2609		− for (i = 0; i < from_oblock(cache->discard_nr_blocks); i++) {
2610		− r = dm_cache_set_discard(cache->cmd, to_oblock(i),
2611		− is_discarded(cache, to_oblock(i)));
	2750	+ for (i = 0; i < from_dblock(cache->discard_nr_blocks); i++) {
	2751	+ r = dm_cache_set_discard(cache->cmd, to_dblock(i),
	2752	+ is_discarded(cache, to_dblock(i)));
2612	2753	if (r)
2613	2754	return r;
2614	2755	}
@@ -2680,15 +2821,86 @@ static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock, · 65 unmodified lines
2680	2821	return 0;
2681	2822	}
2682	2823
	2824	+/*
	2825	+ * The discard block size in the on disk metadata is not
	2826	+ * neccessarily the same as we're currently using. So we have to
	2827	+ * be careful to only set the discarded attribute if we know it
	2828	+ * covers a complete block of the new size.
	2829	+ */
	2830	+struct discard_load_info {
	2831	+ struct cache *cache;
	2832	+
	2833	+ /*
	2834	+ * These blocks are sized using the on disk dblock size, rather
	2835	+ * than the current one.
	2836	+ */
	2837	+ dm_block_t block_size;
	2838	+ dm_block_t discard_begin, discard_end;
	2839	+};
	2840	+
	2841	+static void discard_load_info_init(struct cache *cache,
	2842	+ struct discard_load_info *li)
	2843	+{
	2844	+ li->cache = cache;
	2845	+ li->discard_begin = li->discard_end = 0;
	2846	+}
	2847	+
	2848	+static void set_discard_range(struct discard_load_info *li)
	2849	+{
	2850	+ sector_t b, e;
	2851	+
	2852	+ if (li->discard_begin == li->discard_end)
	2853	+ return;
	2854	+
	2855	+ /*
	2856	+ * Convert to sectors.
	2857	+ */
	2858	+ b = li->discard_begin * li->block_size;
	2859	+ e = li->discard_end * li->block_size;
	2860	+
	2861	+ /*
	2862	+ * Then convert back to the current dblock size.
	2863	+ */
	2864	+ b = dm_sector_div_up(b, li->cache->discard_block_size);
	2865	+ sector_div(e, li->cache->discard_block_size);
	2866	+
	2867	+ /*
	2868	+ * The origin may have shrunk, so we need to check we're still in
	2869	+ * bounds.
	2870	+ */
	2871	+ if (e > from_dblock(li->cache->discard_nr_blocks))
	2872	+ e = from_dblock(li->cache->discard_nr_blocks);
	2873	+
	2874	+ for (; b < e; b++)
	2875	+ set_discard(li->cache, to_dblock(b));
	2876	+}
	2877	+
2683	2878	static int load_discard(void *context, sector_t discard_block_size,
2684		− dm_oblock_t oblock, bool discard)
	2879	+ dm_dblock_t dblock, bool discard)
2685	2880	{
2686		− struct cache *cache = context;
	2881	+ struct discard_load_info *li = context;
2687	2882
2688		− if (discard)
2689		− set_discard(cache, oblock);
2690		− else
2691		− clear_discard(cache, oblock);
	2883	+ li->block_size = discard_block_size;
	2884	+
	2885	+ if (discard) {
	2886	+ if (from_dblock(dblock) == li->discard_end)
	2887	+ /*
	2888	+ * We're already in a discard range, just extend it.
	2889	+ */
	2890	+ li->discard_end = li->discard_end + 1ULL;
	2891	+
	2892	+ else {
	2893	+ /*
	2894	+ * Emit the old range and start a new one.
	2895	+ */
	2896	+ set_discard_range(li);
	2897	+ li->discard_begin = from_dblock(dblock);
	2898	+ li->discard_end = li->discard_begin + 1ULL;
	2899	+ }
	2900	+ } else {
	2901	+ set_discard_range(li);
	2902	+ li->discard_begin = li->discard_end = 0;
	2903	+ }
2692	2904
2693	2905	return 0;
2694	2906	}
@@ -2730,7 +2942,7 @@ static int resize_cache_dev(struct cache *cache, dm_cblock_t new_size) · 35 unmodified lines
2730	2942	return r;
2731	2943	}
2732	2944
2733		− cache->cache_size = new_size;
	2945	+ set_cache_size(cache, new_size);
2734	2946
2735	2947	return 0;
2736	2948	}
@@ -2772,11 +2984,22 @@ static int cache_preresume(struct dm_target *ti) · 35 unmodified lines
2772	2984	}
2773	2985
2774	2986	if (!cache->loaded_discards) {
2775		− r = dm_cache_load_discards(cache->cmd, load_discard, cache);
	2987	+ struct discard_load_info li;
	2988	+
	2989	+ /*
	2990	+ * The discard bitset could have been resized, or the
	2991	+ * discard block size changed. To be safe we start by
	2992	+ * setting every dblock to not discarded.
	2993	+ */
	2994	+ clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
	2995	+
	2996	+ discard_load_info_init(cache, &li);
	2997	+ r = dm_cache_load_discards(cache->cmd, load_discard, &li);
2776	2998	if (r) {
2777	2999	DMERR("could not load origin discards");
2778	3000	return r;
2779	3001	}
	3002	+ set_discard_range(&li);
2780	3003
2781	3004	cache->loaded_discards = true;
2782	3005	}
@@ -3079,8 +3302,9 @@ static void set_discard_limits(struct cache cache, struct queue_limits limits) · 296 unmodified lines
3079	3302	/*
3080	3303	* FIXME: these limits may be incompatible with the cache device
3081	3304	*/
3082		− limits->max_discard_sectors = cache->sectors_per_block;
3083		− limits->discard_granularity = cache->sectors_per_block << SECTOR_SHIFT;
	3305	+ limits->max_discard_sectors = min_t(sector_t, cache->discard_block_size * 1024,
	3306	+ cache->origin_sectors);
	3307	+ limits->discard_granularity = cache->discard_block_size << SECTOR_SHIFT;
3084	3308	}
3085	3309
3086	3310	static void cache_io_hints(struct dm_target ti, struct queue_limits limits)
@@ -3104,7 +3328,7 @@ static void cache_io_hints(struct dm_target ti, struct queue_limits limits) · 17 unmodified lines
3104	3328
3105	3329	static struct target_type cache_target = {
3106	3330	.name = "cache",
3107		− .version = {1, 5, 0},
	3331	+ .version = {1, 6, 0},
3108	3332	.module = THIS_MODULE,
3109	3333	.ctr = cache_ctr,
3110	3334	.dtr = cache_dtr,

drivers/md/dm-crypt.c+1−1

@@ -705,7 +705,7 @@ static int crypt_iv_tcw_whitening(struct crypt_config *cc, · 704 unmodified lines
705	705	for (i = 0; i < ((1 << SECTOR_SHIFT) / 8); i++)
706	706	crypto_xor(data + i * 8, buf, 8);
707	707	out:
708		− memset(buf, 0, sizeof(buf));
	708	+ memzero_explicit(buf, sizeof(buf));
709	709	return r;
710	710	}
711	711

drivers/md/dm-ioctl.c+4−1

@@ -684,11 +684,14 @@ static void __dev_status(struct mapped_device md, struct dm_ioctl param) · 683 unmodified lines
684	684	int srcu_idx;
685	685
686	686	param->flags &= ~(DM_SUSPEND_FLAG \| DM_READONLY_FLAG \|
687		− DM_ACTIVE_PRESENT_FLAG);
	687	+ DM_ACTIVE_PRESENT_FLAG \| DM_INTERNAL_SUSPEND_FLAG);
688	688
689	689	if (dm_suspended_md(md))
690	690	param->flags \|= DM_SUSPEND_FLAG;
691	691
	692	+ if (dm_suspended_internally_md(md))
	693	+ param->flags \|= DM_INTERNAL_SUSPEND_FLAG;
	694	+
692	695	if (dm_test_deferred_remove_flag(md))
693	696	param->flags \|= DM_DEFERRED_REMOVE;
694	697

drivers/md/dm-stats.c+1−1

@@ -824,7 +824,7 @@ static int message_stats_create(struct mapped_device *md, · 823 unmodified lines
824	824	return 1;
825	825
826	826	id = dm_stats_create(dm_get_stats(md), start, end, step, program_id, aux_data,
827		− dm_internal_suspend, dm_internal_resume, md);
	827	+ dm_internal_suspend_fast, dm_internal_resume_fast, md);
828	828	if (id < 0)
829	829	return id;
830	830

drivers/md/dm-table.c+29−7

@@ -1521,18 +1521,32 @@ fmode_t dm_table_get_mode(struct dm_table *t) · 1520 unmodified lines
1521	1521	}
1522	1522	EXPORT_SYMBOL(dm_table_get_mode);
1523	1523
1524		−static void suspend_targets(struct dm_table *t, unsigned postsuspend)
	1524	+enum suspend_mode {
	1525	+ PRESUSPEND,
	1526	+ PRESUSPEND_UNDO,
	1527	+ POSTSUSPEND,
	1528	+};
	1529	+
	1530	+static void suspend_targets(struct dm_table *t, enum suspend_mode mode)
1525	1531	{
1526	1532	int i = t->num_targets;
1527	1533	struct dm_target *ti = t->targets;
1528	1534
1529	1535	while (i--) {
1530		− if (postsuspend) {
	1536	+ switch (mode) {
	1537	+ case PRESUSPEND:
	1538	+ if (ti->type->presuspend)
	1539	+ ti->type->presuspend(ti);
	1540	+ break;
	1541	+ case PRESUSPEND_UNDO:
	1542	+ if (ti->type->presuspend_undo)
	1543	+ ti->type->presuspend_undo(ti);
	1544	+ break;
	1545	+ case POSTSUSPEND:
1531	1546	if (ti->type->postsuspend)
1532	1547	ti->type->postsuspend(ti);
1533		− } else if (ti->type->presuspend)
1534		− ti->type->presuspend(ti);
1535		−
	1548	+ break;
	1549	+ }
1536	1550	ti++;
1537	1551	}
1538	1552	}
@@ -1542,7 +1556,15 @@ void dm_table_presuspend_targets(struct dm_table *t) · 3 unmodified lines
1542	1556	if (!t)
1543	1557	return;
1544	1558
1545		− suspend_targets(t, 0);
	1559	+ suspend_targets(t, PRESUSPEND);
	1560	+}
	1561	+
	1562	+void dm_table_presuspend_undo_targets(struct dm_table *t)
	1563	+{
	1564	+ if (!t)
	1565	+ return;
	1566	+
	1567	+ suspend_targets(t, PRESUSPEND_UNDO);
1546	1568	}
1547	1569
1548	1570	void dm_table_postsuspend_targets(struct dm_table *t)
@@ -1550,7 +1572,7 @@ void dm_table_postsuspend_targets(struct dm_table *t) · 1 unmodified line
1550	1572	if (!t)
1551	1573	return;
1552	1574
1553		− suspend_targets(t, 1);
	1575	+ suspend_targets(t, POSTSUSPEND);
1554	1576	}
1555	1577
1556	1578	int dm_table_resume_targets(struct dm_table *t)

drivers/md/dm-thin-metadata.c+18−17

@@ -1384,42 +1384,38 @@ static bool __snapshotted_since(struct dm_thin_device *td, uint32_t time) · 1383 unmodified lines
1384	1384	}
1385	1385
1386	1386	int dm_thin_find_block(struct dm_thin_device *td, dm_block_t block,
1387		− int can_block, struct dm_thin_lookup_result *result)
	1387	+ int can_issue_io, struct dm_thin_lookup_result *result)
1388	1388	{
1389		− int r = -EINVAL;
1390		− uint64_t block_time = 0;
	1389	+ int r;
1391	1390	__le64 value;
1392	1391	struct dm_pool_metadata *pmd = td->pmd;
1393	1392	dm_block_t keys[2] = { td->id, block };
1394	1393	struct dm_btree_info *info;
1395	1394
1396		− if (can_block) {
1397		− down_read(&pmd->root_lock);
1398		− info = &pmd->info;
1399		− } else if (down_read_trylock(&pmd->root_lock))
1400		− info = &pmd->nb_info;
1401		− else
1402		− return -EWOULDBLOCK;
1403		−
1404	1395	if (pmd->fail_io)
1405		− goto out;
	1396	+ return -EINVAL;
1406	1397
1407		− r = dm_btree_lookup(info, pmd->root, keys, &value);
1408		− if (!r)
1409		− block_time = le64_to_cpu(value);
	1398	+ down_read(&pmd->root_lock);
1410	1399
1411		−out:
1412		− up_read(&pmd->root_lock);
	1400	+ if (can_issue_io) {
	1401	+ info = &pmd->info;
	1402	+ } else
	1403	+ info = &pmd->nb_info;
1413	1404
	1405	+ r = dm_btree_lookup(info, pmd->root, keys, &value);
1414	1406	if (!r) {
	1407	+ uint64_t block_time = 0;
1415	1408	dm_block_t exception_block;
1416	1409	uint32_t exception_time;
	1410	+
	1411	+ block_time = le64_to_cpu(value);
1417	1412	unpack_block_time(block_time, &exception_block,
1418	1413	&exception_time);
1419	1414	result->block = exception_block;
1420	1415	result->shared = __snapshotted_since(td, exception_time);
1421	1416	}
1422	1417
	1418	+ up_read(&pmd->root_lock);
1423	1419	return r;
1424	1420	}
1425	1421
@@ -1813,3 +1809,8 @@ bool dm_pool_metadata_needs_check(struct dm_pool_metadata *pmd) · 387 unmodified lines
1813	1809
1814	1810	return needs_check;
1815	1811	}
	1812	+
	1813	+void dm_pool_issue_prefetches(struct dm_pool_metadata *pmd)
	1814	+{
	1815	+ dm_tm_issue_prefetches(pmd->tm);
	1816	+}

drivers/md/dm-thin-metadata.h+7−2

@@ -139,12 +139,12 @@ struct dm_thin_lookup_result { · 138 unmodified lines
139	139
140	140	/*
141	141	* Returns:
142		− * -EWOULDBLOCK iff @can_block is set and would block.
	142	+ * -EWOULDBLOCK iff @can_issue_io is set and would issue IO
143	143	* -ENODATA iff that mapping is not present.
144	144	* 0 success
145	145	*/
146	146	int dm_thin_find_block(struct dm_thin_device *td, dm_block_t block,
147		− int can_block, struct dm_thin_lookup_result *result);
	147	+ int can_issue_io, struct dm_thin_lookup_result *result);
148	148
149	149	/*
150	150	* Obtain an unused block.
@@ -213,6 +213,11 @@ int dm_pool_register_metadata_threshold(struct dm_pool_metadata *pmd, · 62 unmodified lines
213	213	int dm_pool_metadata_set_needs_check(struct dm_pool_metadata *pmd);
214	214	bool dm_pool_metadata_needs_check(struct dm_pool_metadata *pmd);
215	215
	216	+/*
	217	+ * Issue any prefetches that may be useful.
	218	+ */
	219	+void dm_pool_issue_prefetches(struct dm_pool_metadata *pmd);
	220	+
216	221	/----------------------------------------------------------------/
217	222
218	223	#endif

drivers/md/dm-thin.c+607−153

@@ -11,11 +11,13 @@ · 10 unmodified lines
11	11	#include <linux/device-mapper.h>
12	12	#include <linux/dm-io.h>
13	13	#include <linux/dm-kcopyd.h>
	14	+#include <linux/log2.h>
14	15	#include <linux/list.h>
15	16	#include <linux/rculist.h>
16	17	#include <linux/init.h>
17	18	#include <linux/module.h>
18	19	#include <linux/slab.h>
	20	+#include <linux/sort.h>
19	21	#include <linux/rbtree.h>
20	22
21	23	#define DM_MSG_PREFIX "thin"
@@ -25,7 +27,6 @@ · 3 unmodified lines
25	27	*/
26	28	#define ENDIO_HOOK_POOL_SIZE 1024
27	29	#define MAPPING_POOL_SIZE 1024
28		−#define PRISON_CELLS 1024
29	30	#define COMMIT_PERIOD HZ
30	31	#define NO_SPACE_TIMEOUT_SECS 60
31	32
@@ -114,7 +115,8 @@ static void build_data_key(struct dm_thin_device *td, · 82 unmodified lines
114	115	{
115	116	key->virtual = 0;
116	117	key->dev = dm_thin_dev_id(td);
117		− key->block = b;
	118	+ key->block_begin = b;
	119	+ key->block_end = b + 1ULL;
118	120	}
119	121
120	122	static void build_virtual_key(struct dm_thin_device *td, dm_block_t b,
@@ -122,7 +124,55 @@ static void build_virtual_key(struct dm_thin_device *td, dm_block_t b, · 1 unmodified line
122	124	{
123	125	key->virtual = 1;
124	126	key->dev = dm_thin_dev_id(td);
125		− key->block = b;
	127	+ key->block_begin = b;
	128	+ key->block_end = b + 1ULL;
	129	+}
	130	+
	131	+/----------------------------------------------------------------/
	132	+
	133	+#define THROTTLE_THRESHOLD (1 * HZ)
	134	+
	135	+struct throttle {
	136	+ struct rw_semaphore lock;
	137	+ unsigned long threshold;
	138	+ bool throttle_applied;
	139	+};
	140	+
	141	+static void throttle_init(struct throttle *t)
	142	+{
	143	+ init_rwsem(&t->lock);
	144	+ t->throttle_applied = false;
	145	+}
	146	+
	147	+static void throttle_work_start(struct throttle *t)
	148	+{
	149	+ t->threshold = jiffies + THROTTLE_THRESHOLD;
	150	+}
	151	+
	152	+static void throttle_work_update(struct throttle *t)
	153	+{
	154	+ if (!t->throttle_applied && jiffies > t->threshold) {
	155	+ down_write(&t->lock);
	156	+ t->throttle_applied = true;
	157	+ }
	158	+}
	159	+
	160	+static void throttle_work_complete(struct throttle *t)
	161	+{
	162	+ if (t->throttle_applied) {
	163	+ t->throttle_applied = false;
	164	+ up_write(&t->lock);
	165	+ }
	166	+}
	167	+
	168	+static void throttle_lock(struct throttle *t)
	169	+{
	170	+ down_read(&t->lock);
	171	+}
	172	+
	173	+static void throttle_unlock(struct throttle *t)
	174	+{
	175	+ up_read(&t->lock);
126	176	}
127	177
128	178	/----------------------------------------------------------------/
@@ -155,8 +205,11 @@ struct pool_features { · 26 unmodified lines
155	205
156	206	struct thin_c;
157	207	typedef void (process_bio_fn)(struct thin_c tc, struct bio *bio);
	208	+typedef void (process_cell_fn)(struct thin_c tc, struct dm_bio_prison_cell *cell);
158	209	typedef void (process_mapping_fn)(struct dm_thin_new_mapping m);
159	210
	211	+#define CELL_SORT_ARRAY_SIZE 8192
	212	+
160	213	struct pool {
161	214	struct list_head list;
162	215	struct dm_target ti; / Only set if a pool target is bound */
@@ -171,11 +224,13 @@ struct pool { · 8 unmodified lines
171	224
172	225	struct pool_features pf;
173	226	bool low_water_triggered:1; /* A dm event has been sent */
	227	+ bool suspended:1;
174	228
175	229	struct dm_bio_prison *prison;
176	230	struct dm_kcopyd_client *copier;
177	231
178	232	struct workqueue_struct *wq;
	233	+ struct throttle throttle;
179	234	struct work_struct worker;
180	235	struct delayed_work waker;
181	236	struct delayed_work no_space_timeout;
@@ -198,8 +253,13 @@ struct pool { · 16 unmodified lines
198	253	process_bio_fn process_bio;
199	254	process_bio_fn process_discard;
200	255
	256	+ process_cell_fn process_cell;
	257	+ process_cell_fn process_discard_cell;
	258	+
201	259	process_mapping_fn process_prepared_mapping;
202	260	process_mapping_fn process_prepared_discard;
	261	+
	262	+ struct dm_bio_prison_cell *cell_sort_array[CELL_SORT_ARRAY_SIZE];
203	263	};
204	264
205	265	static enum pool_mode get_pool_mode(struct pool *pool);
@@ -232,8 +292,11 @@ struct thin_c { · 26 unmodified lines
232	292
233	293	struct pool *pool;
234	294	struct dm_thin_device *td;
	295	+ struct mapped_device *thin_md;
	296	+
235	297	bool requeue_mode:1;
236	298	spinlock_t lock;
	299	+ struct list_head deferred_cells;
237	300	struct bio_list deferred_bio_list;
238	301	struct bio_list retry_on_resume_list;
239	302	struct rb_root sort_bio_list; /* sorted list of deferred bios */
@@ -290,6 +353,15 @@ static void cell_release(struct pool *pool, · 50 unmodified lines
290	353	dm_bio_prison_free_cell(pool->prison, cell);
291	354	}
292	355
	356	+static void cell_visit_release(struct pool *pool,
	357	+ void (fn)(void , struct dm_bio_prison_cell *),
	358	+ void *context,
	359	+ struct dm_bio_prison_cell *cell)
	360	+{
	361	+ dm_cell_visit_release(pool->prison, fn, context, cell);
	362	+ dm_bio_prison_free_cell(pool->prison, cell);
	363	+}
	364	+
293	365	static void cell_release_no_holder(struct pool *pool,
294	366	struct dm_bio_prison_cell *cell,
295	367	struct bio_list *bios)
@@ -298,19 +370,6 @@ static void cell_release_no_holder(struct pool *pool, · 2 unmodified lines
298	370	dm_bio_prison_free_cell(pool->prison, cell);
299	371	}
300	372
301		−static void cell_defer_no_holder_no_free(struct thin_c *tc,
302		− struct dm_bio_prison_cell *cell)
303		−{
304		− struct pool *pool = tc->pool;
305		− unsigned long flags;
306		−
307		− spin_lock_irqsave(&tc->lock, flags);
308		− dm_cell_release_no_holder(pool->prison, cell, &tc->deferred_bio_list);
309		− spin_unlock_irqrestore(&tc->lock, flags);
310		−
311		− wake_worker(pool);
312		−}
313		−
314	373	static void cell_error_with_code(struct pool *pool,
315	374	struct dm_bio_prison_cell *cell, int error_code)
316	375	{
@@ -323,6 +382,16 @@ static void cell_error(struct pool pool, struct dm_bio_prison_cell cell) · 6 unmodified lines
323	382	cell_error_with_code(pool, cell, -EIO);
324	383	}
325	384
	385	+static void cell_success(struct pool pool, struct dm_bio_prison_cell cell)
	386	+{
	387	+ cell_error_with_code(pool, cell, 0);
	388	+}
	389	+
	390	+static void cell_requeue(struct pool pool, struct dm_bio_prison_cell cell)
	391	+{
	392	+ cell_error_with_code(pool, cell, DM_ENDIO_REQUEUE);
	393	+}
	394	+
326	395	/----------------------------------------------------------------/
327	396
328	397	/*
@@ -393,44 +462,65 @@ struct dm_thin_endio_hook { · 64 unmodified lines
393	462	struct rb_node rb_node;
394	463	};
395	464
396		−static void requeue_bio_list(struct thin_c tc, struct bio_list master)
	465	+static void __merge_bio_list(struct bio_list bios, struct bio_list master)
	466	+{
	467	+ bio_list_merge(bios, master);
	468	+ bio_list_init(master);
	469	+}
	470	+
	471	+static void error_bio_list(struct bio_list *bios, int error)
397	472	{
398	473	struct bio *bio;
	474	+
	475	+ while ((bio = bio_list_pop(bios)))
	476	+ bio_endio(bio, error);
	477	+}
	478	+
	479	+static void error_thin_bio_list(struct thin_c tc, struct bio_list master, int error)
	480	+{
399	481	struct bio_list bios;
400	482	unsigned long flags;
401	483
402	484	bio_list_init(&bios);
403	485
404	486	spin_lock_irqsave(&tc->lock, flags);
405		− bio_list_merge(&bios, master);
406		− bio_list_init(master);
	487	+ __merge_bio_list(&bios, master);
407	488	spin_unlock_irqrestore(&tc->lock, flags);
408	489
409		− while ((bio = bio_list_pop(&bios)))
410		− bio_endio(bio, DM_ENDIO_REQUEUE);
	490	+ error_bio_list(&bios, error);
411	491	}
412	492
413		−static void requeue_io(struct thin_c *tc)
	493	+static void requeue_deferred_cells(struct thin_c *tc)
414	494	{
415		− requeue_bio_list(tc, &tc->deferred_bio_list);
416		− requeue_bio_list(tc, &tc->retry_on_resume_list);
	495	+ struct pool *pool = tc->pool;
	496	+ unsigned long flags;
	497	+ struct list_head cells;
	498	+ struct dm_bio_prison_cell cell, tmp;
	499	+
	500	+ INIT_LIST_HEAD(&cells);
	501	+
	502	+ spin_lock_irqsave(&tc->lock, flags);
	503	+ list_splice_init(&tc->deferred_cells, &cells);
	504	+ spin_unlock_irqrestore(&tc->lock, flags);
	505	+
	506	+ list_for_each_entry_safe(cell, tmp, &cells, user_list)
	507	+ cell_requeue(pool, cell);
417	508	}
418	509
419		−static void error_thin_retry_list(struct thin_c *tc)
	510	+static void requeue_io(struct thin_c *tc)
420	511	{
421		− struct bio *bio;
422		− unsigned long flags;
423	512	struct bio_list bios;
	513	+ unsigned long flags;
424	514
425	515	bio_list_init(&bios);
426	516
427	517	spin_lock_irqsave(&tc->lock, flags);
428		− bio_list_merge(&bios, &tc->retry_on_resume_list);
429		− bio_list_init(&tc->retry_on_resume_list);
	518	+ __merge_bio_list(&bios, &tc->deferred_bio_list);
	519	+ __merge_bio_list(&bios, &tc->retry_on_resume_list);
430	520	spin_unlock_irqrestore(&tc->lock, flags);
431	521
432		− while ((bio = bio_list_pop(&bios)))
433		− bio_io_error(bio);
	522	+ error_bio_list(&bios, DM_ENDIO_REQUEUE);
	523	+ requeue_deferred_cells(tc);
434	524	}
435	525
436	526	static void error_retry_list(struct pool *pool)
@@ -439,7 +529,7 @@ static void error_retry_list(struct pool *pool) · 2 unmodified lines
439	529
440	530	rcu_read_lock();
441	531	list_for_each_entry_rcu(tc, &pool->active_thins, list)
442		− error_thin_retry_list(tc);
	532	+ error_thin_bio_list(tc, &tc->retry_on_resume_list, -EIO);
443	533	rcu_read_unlock();
444	534	}
445	535
@@ -629,33 +719,75 @@ static void overwrite_endio(struct bio *bio, int err) · 183 unmodified lines
629	719	*/
630	720
631	721	/*
632		− * This sends the bios in the cell back to the deferred_bios list.
	722	+ * This sends the bios in the cell, except the original holder, back
	723	+ * to the deferred_bios list.
633	724	*/
634		−static void cell_defer(struct thin_c tc, struct dm_bio_prison_cell cell)
	725	+static void cell_defer_no_holder(struct thin_c tc, struct dm_bio_prison_cell cell)
635	726	{
636	727	struct pool *pool = tc->pool;
637	728	unsigned long flags;
638	729
639	730	spin_lock_irqsave(&tc->lock, flags);
640		− cell_release(pool, cell, &tc->deferred_bio_list);
	731	+ cell_release_no_holder(pool, cell, &tc->deferred_bio_list);
641	732	spin_unlock_irqrestore(&tc->lock, flags);
642	733
643	734	wake_worker(pool);
644	735	}
645	736
646		−/*
647		− * Same as cell_defer above, except it omits the original holder of the cell.
648		− */
649		−static void cell_defer_no_holder(struct thin_c tc, struct dm_bio_prison_cell cell)
	737	+static void thin_defer_bio(struct thin_c tc, struct bio bio);
	738	+
	739	+struct remap_info {
	740	+ struct thin_c *tc;
	741	+ struct bio_list defer_bios;
	742	+ struct bio_list issue_bios;
	743	+};
	744	+
	745	+static void __inc_remap_and_issue_cell(void *context,
	746	+ struct dm_bio_prison_cell *cell)
650	747	{
651		− struct pool *pool = tc->pool;
652		− unsigned long flags;
	748	+ struct remap_info *info = context;
	749	+ struct bio *bio;
653	750
654		− spin_lock_irqsave(&tc->lock, flags);
655		− cell_release_no_holder(pool, cell, &tc->deferred_bio_list);
656		− spin_unlock_irqrestore(&tc->lock, flags);
	751	+ while ((bio = bio_list_pop(&cell->bios))) {
	752	+ if (bio->bi_rw & (REQ_DISCARD \| REQ_FLUSH \| REQ_FUA))
	753	+ bio_list_add(&info->defer_bios, bio);
	754	+ else {
	755	+ inc_all_io_entry(info->tc->pool, bio);
657	756
658		− wake_worker(pool);
	757	+ /*
	758	+ * We can't issue the bios with the bio prison lock
	759	+ * held, so we add them to a list to issue on
	760	+ * return from this function.
	761	+ */
	762	+ bio_list_add(&info->issue_bios, bio);
	763	+ }
	764	+ }
	765	+}
	766	+
	767	+static void inc_remap_and_issue_cell(struct thin_c *tc,
	768	+ struct dm_bio_prison_cell *cell,
	769	+ dm_block_t block)
	770	+{
	771	+ struct bio *bio;
	772	+ struct remap_info info;
	773	+
	774	+ info.tc = tc;
	775	+ bio_list_init(&info.defer_bios);
	776	+ bio_list_init(&info.issue_bios);
	777	+
	778	+ /*
	779	+ * We have to be careful to inc any bios we're about to issue
	780	+ * before the cell is released, and avoid a race with new bios
	781	+ * being added to the cell.
	782	+ */
	783	+ cell_visit_release(tc->pool, __inc_remap_and_issue_cell,
	784	+ &info, cell);
	785	+
	786	+ while ((bio = bio_list_pop(&info.defer_bios)))
	787	+ thin_defer_bio(tc, bio);
	788	+
	789	+ while ((bio = bio_list_pop(&info.issue_bios)))
	790	+ remap_and_issue(info.tc, bio, block);
659	791	}
660	792
661	793	static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m)
@@ -706,10 +838,13 @@ static void process_prepared_mapping(struct dm_thin_new_mapping *m) · 44 unmodified lines
706	838	* the bios in the cell.
707	839	*/
708	840	if (bio) {
709		− cell_defer_no_holder(tc, m->cell);
	841	+ inc_remap_and_issue_cell(tc, m->cell, m->data_block);
710	842	bio_endio(bio, 0);
711		− } else
712		− cell_defer(tc, m->cell);
	843	+ } else {
	844	+ inc_all_io_entry(tc->pool, m->cell->holder);
	845	+ remap_and_issue(tc, m->cell->holder, m->data_block);
	846	+ inc_remap_and_issue_cell(tc, m->cell, m->data_block);
	847	+ }
713	848
714	849	out:
715	850	list_del(&m->list);
@@ -842,6 +977,20 @@ static void ll_zero(struct thin_c tc, struct dm_thin_new_mapping m, · 126 unmodified lines
842	977	}
843	978	}
844	979
	980	+static void remap_and_issue_overwrite(struct thin_c tc, struct bio bio,
	981	+ dm_block_t data_block,
	982	+ struct dm_thin_new_mapping *m)
	983	+{
	984	+ struct pool *pool = tc->pool;
	985	+ struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
	986	+
	987	+ h->overwrite_mapping = m;
	988	+ m->bio = bio;
	989	+ save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
	990	+ inc_all_io_entry(pool, bio);
	991	+ remap_and_issue(tc, bio, data_block);
	992	+}
	993	+
845	994	/*
846	995	* A partial copy also needs to zero the uncopied region.
847	996	*/
@@ -876,15 +1025,9 @@ static void schedule_copy(struct thin_c *tc, dm_block_t virt_block, · 28 unmodified lines
876	1025	* If the whole block of data is being overwritten, we can issue the
877	1026	* bio immediately. Otherwise we use kcopyd to clone the data first.
878	1027	*/
879		− if (io_overwrites_block(pool, bio)) {
880		− struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
881		−
882		− h->overwrite_mapping = m;
883		− m->bio = bio;
884		− save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
885		− inc_all_io_entry(pool, bio);
886		− remap_and_issue(tc, bio, data_dest);
887		− } else {
	1028	+ if (io_overwrites_block(pool, bio))
	1029	+ remap_and_issue_overwrite(tc, bio, data_dest, m);
	1030	+ else {
888	1031	struct dm_io_region from, to;
889	1032
890	1033	from.bdev = origin->bdev;
@@ -953,16 +1096,10 @@ static void schedule_zero(struct thin_c *tc, dm_block_t virt_block, · 62 unmodified lines
953	1096	if (!pool->pf.zero_new_blocks)
954	1097	process_prepared_mapping(m);
955	1098
956		− else if (io_overwrites_block(pool, bio)) {
957		− struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
958		−
959		− h->overwrite_mapping = m;
960		− m->bio = bio;
961		− save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
962		− inc_all_io_entry(pool, bio);
963		− remap_and_issue(tc, bio, data_block);
	1099	+ else if (io_overwrites_block(pool, bio))
	1100	+ remap_and_issue_overwrite(tc, bio, data_block, m);
964	1101
965		− } else
	1102	+ else
966	1103	ll_zero(tc, m,
967	1104	data_block * pool->sectors_per_block,
968	1105	(data_block + 1) * pool->sectors_per_block);
@@ -1134,29 +1271,25 @@ static void retry_bios_on_resume(struct pool pool, struct dm_bio_prison_cell c · 165 unmodified lines
1134	1271	bio_list_init(&bios);
1135	1272	cell_release(pool, cell, &bios);
1136	1273
1137		− error = should_error_unserviceable_bio(pool);
1138		− if (error)
1139		− while ((bio = bio_list_pop(&bios)))
1140		− bio_endio(bio, error);
1141		− else
1142		− while ((bio = bio_list_pop(&bios)))
1143		− retry_on_resume(bio);
	1274	+ while ((bio = bio_list_pop(&bios)))
	1275	+ retry_on_resume(bio);
1144	1276	}
1145	1277
1146		−static void process_discard(struct thin_c tc, struct bio bio)
	1278	+static void process_discard_cell(struct thin_c tc, struct dm_bio_prison_cell cell)
1147	1279	{
1148	1280	int r;
1149		− unsigned long flags;
	1281	+ struct bio *bio = cell->holder;
1150	1282	struct pool *pool = tc->pool;
1151		− struct dm_bio_prison_cell cell, cell2;
1152		− struct dm_cell_key key, key2;
	1283	+ struct dm_bio_prison_cell *cell2;
	1284	+ struct dm_cell_key key2;
1153	1285	dm_block_t block = get_bio_block(tc, bio);
1154	1286	struct dm_thin_lookup_result lookup_result;
1155	1287	struct dm_thin_new_mapping *m;
1156	1288
1157		− build_virtual_key(tc->td, block, &key);
1158		− if (bio_detain(tc->pool, &key, bio, &cell))
	1289	+ if (tc->requeue_mode) {
	1290	+ cell_requeue(pool, cell);
1159	1291	return;
	1292	+ }
1160	1293
1161	1294	r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1162	1295	switch (r) {
@@ -1187,12 +1320,9 @@ static void process_discard(struct thin_c tc, struct bio bio) · 24 unmodified lines
1187	1320	m->cell2 = cell2;
1188	1321	m->bio = bio;
1189	1322
1190		− if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list)) {
1191		− spin_lock_irqsave(&pool->lock, flags);
1192		− list_add_tail(&m->list, &pool->prepared_discards);
1193		− spin_unlock_irqrestore(&pool->lock, flags);
1194		− wake_worker(pool);
1195		− }
	1323	+ if (!dm_deferred_set_add_work(pool->all_io_ds, &m->list))
	1324	+ pool->process_prepared_discard(m);
	1325	+
1196	1326	} else {
1197	1327	inc_all_io_entry(pool, bio);
1198	1328	cell_defer_no_holder(tc, cell);
@@ -1227,6 +1357,19 @@ static void process_discard(struct thin_c tc, struct bio bio) · 28 unmodified lines
1227	1357	}
1228	1358	}
1229	1359
	1360	+static void process_discard_bio(struct thin_c tc, struct bio bio)
	1361	+{
	1362	+ struct dm_bio_prison_cell *cell;
	1363	+ struct dm_cell_key key;
	1364	+ dm_block_t block = get_bio_block(tc, bio);
	1365	+
	1366	+ build_virtual_key(tc->td, block, &key);
	1367	+ if (bio_detain(tc->pool, &key, bio, &cell))
	1368	+ return;
	1369	+
	1370	+ process_discard_cell(tc, cell);
	1371	+}
	1372	+
1230	1373	static void break_sharing(struct thin_c tc, struct bio bio, dm_block_t block,
1231	1374	struct dm_cell_key *key,
1232	1375	struct dm_thin_lookup_result *lookup_result,
@@ -1255,11 +1398,53 @@ static void break_sharing(struct thin_c tc, struct bio bio, dm_block_t block, · 22 unmodified lines
1255	1398	}
1256	1399	}
1257	1400
	1401	+static void __remap_and_issue_shared_cell(void *context,
	1402	+ struct dm_bio_prison_cell *cell)
	1403	+{
	1404	+ struct remap_info *info = context;
	1405	+ struct bio *bio;
	1406	+
	1407	+ while ((bio = bio_list_pop(&cell->bios))) {
	1408	+ if ((bio_data_dir(bio) == WRITE) \|\|
	1409	+ (bio->bi_rw & (REQ_DISCARD \| REQ_FLUSH \| REQ_FUA)))
	1410	+ bio_list_add(&info->defer_bios, bio);
	1411	+ else {
	1412	+ struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));;
	1413	+
	1414	+ h->shared_read_entry = dm_deferred_entry_inc(info->tc->pool->shared_read_ds);
	1415	+ inc_all_io_entry(info->tc->pool, bio);
	1416	+ bio_list_add(&info->issue_bios, bio);
	1417	+ }
	1418	+ }
	1419	+}
	1420	+
	1421	+static void remap_and_issue_shared_cell(struct thin_c *tc,
	1422	+ struct dm_bio_prison_cell *cell,
	1423	+ dm_block_t block)
	1424	+{
	1425	+ struct bio *bio;
	1426	+ struct remap_info info;
	1427	+
	1428	+ info.tc = tc;
	1429	+ bio_list_init(&info.defer_bios);
	1430	+ bio_list_init(&info.issue_bios);
	1431	+
	1432	+ cell_visit_release(tc->pool, __remap_and_issue_shared_cell,
	1433	+ &info, cell);
	1434	+
	1435	+ while ((bio = bio_list_pop(&info.defer_bios)))
	1436	+ thin_defer_bio(tc, bio);
	1437	+
	1438	+ while ((bio = bio_list_pop(&info.issue_bios)))
	1439	+ remap_and_issue(tc, bio, block);
	1440	+}
	1441	+
1258	1442	static void process_shared_bio(struct thin_c tc, struct bio bio,
1259	1443	dm_block_t block,
1260		− struct dm_thin_lookup_result *lookup_result)
	1444	+ struct dm_thin_lookup_result *lookup_result,
	1445	+ struct dm_bio_prison_cell *virt_cell)
1261	1446	{
1262		− struct dm_bio_prison_cell *cell;
	1447	+ struct dm_bio_prison_cell *data_cell;
1263	1448	struct pool *pool = tc->pool;
1264	1449	struct dm_cell_key key;
1265	1450
@@ -1268,19 +1453,23 @@ static void process_shared_bio(struct thin_c tc, struct bio bio, · 2 unmodified lines
1268	1453	* of being broken so we have nothing further to do here.
1269	1454	*/
1270	1455	build_data_key(tc->td, lookup_result->block, &key);
1271		− if (bio_detain(pool, &key, bio, &cell))
	1456	+ if (bio_detain(pool, &key, bio, &data_cell)) {
	1457	+ cell_defer_no_holder(tc, virt_cell);
1272	1458	return;
	1459	+ }
1273	1460
1274		− if (bio_data_dir(bio) == WRITE && bio->bi_iter.bi_size)
1275		− break_sharing(tc, bio, block, &key, lookup_result, cell);
1276		− else {
	1461	+ if (bio_data_dir(bio) == WRITE && bio->bi_iter.bi_size) {
	1462	+ break_sharing(tc, bio, block, &key, lookup_result, data_cell);
	1463	+ cell_defer_no_holder(tc, virt_cell);
	1464	+ } else {
1277	1465	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
1278	1466
1279	1467	h->shared_read_entry = dm_deferred_entry_inc(pool->shared_read_ds);
1280	1468	inc_all_io_entry(pool, bio);
1281		− cell_defer_no_holder(tc, cell);
1282		−
1283	1469	remap_and_issue(tc, bio, lookup_result->block);
	1470	+
	1471	+ remap_and_issue_shared_cell(tc, data_cell, lookup_result->block);
	1472	+ remap_and_issue_shared_cell(tc, virt_cell, lookup_result->block);
1284	1473	}
1285	1474	}
1286	1475
@@ -1333,34 +1522,28 @@ static void provision_block(struct thin_c tc, struct bio bio, dm_block_t block · 46 unmodified lines
1333	1522	}
1334	1523	}
1335	1524
1336		−static void process_bio(struct thin_c tc, struct bio bio)
	1525	+static void process_cell(struct thin_c tc, struct dm_bio_prison_cell cell)
1337	1526	{
1338	1527	int r;
1339	1528	struct pool *pool = tc->pool;
	1529	+ struct bio *bio = cell->holder;
1340	1530	dm_block_t block = get_bio_block(tc, bio);
1341		− struct dm_bio_prison_cell *cell;
1342		− struct dm_cell_key key;
1343	1531	struct dm_thin_lookup_result lookup_result;
1344	1532
1345		− /*
1346		− * If cell is already occupied, then the block is already
1347		− * being provisioned so we have nothing further to do here.
1348		− */
1349		− build_virtual_key(tc->td, block, &key);
1350		− if (bio_detain(pool, &key, bio, &cell))
	1533	+ if (tc->requeue_mode) {
	1534	+ cell_requeue(pool, cell);
1351	1535	return;
	1536	+ }
1352	1537
1353	1538	r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1354	1539	switch (r) {
1355	1540	case 0:
1356		− if (lookup_result.shared) {
1357		− process_shared_bio(tc, bio, block, &lookup_result);
1358		− cell_defer_no_holder(tc, cell); /* FIXME: pass this cell into process_shared? */
1359		− } else {
	1541	+ if (lookup_result.shared)
	1542	+ process_shared_bio(tc, bio, block, &lookup_result, cell);
	1543	+ else {
1360	1544	inc_all_io_entry(pool, bio);
1361		− cell_defer_no_holder(tc, cell);
1362		−
1363	1545	remap_and_issue(tc, bio, lookup_result.block);
	1546	+ inc_remap_and_issue_cell(tc, cell, lookup_result.block);
1364	1547	}
1365	1548	break;
1366	1549
@@ -1394,7 +1577,26 @@ static void process_bio(struct thin_c tc, struct bio bio) · 27 unmodified lines
1394	1577	}
1395	1578	}
1396	1579
1397		−static void process_bio_read_only(struct thin_c tc, struct bio bio)
	1580	+static void process_bio(struct thin_c tc, struct bio bio)
	1581	+{
	1582	+ struct pool *pool = tc->pool;
	1583	+ dm_block_t block = get_bio_block(tc, bio);
	1584	+ struct dm_bio_prison_cell *cell;
	1585	+ struct dm_cell_key key;
	1586	+
	1587	+ /*
	1588	+ * If cell is already occupied, then the block is already
	1589	+ * being provisioned so we have nothing further to do here.
	1590	+ */
	1591	+ build_virtual_key(tc->td, block, &key);
	1592	+ if (bio_detain(pool, &key, bio, &cell))
	1593	+ return;
	1594	+
	1595	+ process_cell(tc, cell);
	1596	+}
	1597	+
	1598	+static void __process_bio_read_only(struct thin_c tc, struct bio bio,
	1599	+ struct dm_bio_prison_cell *cell)
1398	1600	{
1399	1601	int r;
1400	1602	int rw = bio_data_dir(bio);
@@ -1404,15 +1606,21 @@ static void process_bio_read_only(struct thin_c tc, struct bio bio) · 3 unmodified lines
1404	1606	r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1405	1607	switch (r) {
1406	1608	case 0:
1407		− if (lookup_result.shared && (rw == WRITE) && bio->bi_iter.bi_size)
	1609	+ if (lookup_result.shared && (rw == WRITE) && bio->bi_iter.bi_size) {
1408	1610	handle_unserviceable_bio(tc->pool, bio);
1409		− else {
	1611	+ if (cell)
	1612	+ cell_defer_no_holder(tc, cell);
	1613	+ } else {
1410	1614	inc_all_io_entry(tc->pool, bio);
1411	1615	remap_and_issue(tc, bio, lookup_result.block);
	1616	+ if (cell)
	1617	+ inc_remap_and_issue_cell(tc, cell, lookup_result.block);
1412	1618	}
1413	1619	break;
1414	1620
1415	1621	case -ENODATA:
	1622	+ if (cell)
	1623	+ cell_defer_no_holder(tc, cell);
1416	1624	if (rw != READ) {
1417	1625	handle_unserviceable_bio(tc->pool, bio);
1418	1626	break;
@@ -1431,11 +1639,23 @@ static void process_bio_read_only(struct thin_c tc, struct bio bio) · 12 unmodified lines
1431	1639	default:
1432	1640	DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
1433	1641	__func__, r);
	1642	+ if (cell)
	1643	+ cell_defer_no_holder(tc, cell);
1434	1644	bio_io_error(bio);
1435	1645	break;
1436	1646	}
1437	1647	}
1438	1648
	1649	+static void process_bio_read_only(struct thin_c tc, struct bio bio)
	1650	+{
	1651	+ __process_bio_read_only(tc, bio, NULL);
	1652	+}
	1653	+
	1654	+static void process_cell_read_only(struct thin_c tc, struct dm_bio_prison_cell cell)
	1655	+{
	1656	+ __process_bio_read_only(tc, cell->holder, cell);
	1657	+}
	1658	+
1439	1659	static void process_bio_success(struct thin_c tc, struct bio bio)
1440	1660	{
1441	1661	bio_endio(bio, 0);
@@ -1446,6 +1666,16 @@ static void process_bio_fail(struct thin_c tc, struct bio bio) · 4 unmodified lines
1446	1666	bio_io_error(bio);
1447	1667	}
1448	1668
	1669	+static void process_cell_success(struct thin_c tc, struct dm_bio_prison_cell cell)
	1670	+{
	1671	+ cell_success(tc->pool, cell);
	1672	+}
	1673	+
	1674	+static void process_cell_fail(struct thin_c tc, struct dm_bio_prison_cell cell)
	1675	+{
	1676	+ cell_error(tc->pool, cell);
	1677	+}
	1678	+
1449	1679	/*
1450	1680	* FIXME: should we also commit due to size of transaction, measured in
1451	1681	* metadata blocks?
@@ -1527,9 +1757,10 @@ static void process_thin_deferred_bios(struct thin_c *tc) · 75 unmodified lines
1527	1757	struct bio *bio;
1528	1758	struct bio_list bios;
1529	1759	struct blk_plug plug;
	1760	+ unsigned count = 0;
1530	1761
1531	1762	if (tc->requeue_mode) {
1532		− requeue_bio_list(tc, &tc->deferred_bio_list);
	1763	+ error_thin_bio_list(tc, &tc->deferred_bio_list, DM_ENDIO_REQUEUE);
1533	1764	return;
1534	1765	}
1535	1766
@@ -1568,10 +1799,97 @@ static void process_thin_deferred_bios(struct thin_c *tc) · 32 unmodified lines
1568	1799	pool->process_discard(tc, bio);
1569	1800	else
1570	1801	pool->process_bio(tc, bio);
	1802	+
	1803	+ if ((count++ & 127) == 0) {
	1804	+ throttle_work_update(&pool->throttle);
	1805	+ dm_pool_issue_prefetches(pool->pmd);
	1806	+ }
1571	1807	}
1572	1808	blk_finish_plug(&plug);
1573	1809	}
1574	1810
	1811	+static int cmp_cells(const void lhs, const void rhs)
	1812	+{
	1813	+ struct dm_bio_prison_cell lhs_cell = ((struct dm_bio_prison_cell **) lhs);
	1814	+ struct dm_bio_prison_cell rhs_cell = ((struct dm_bio_prison_cell **) rhs);
	1815	+
	1816	+ BUG_ON(!lhs_cell->holder);
	1817	+ BUG_ON(!rhs_cell->holder);
	1818	+
	1819	+ if (lhs_cell->holder->bi_iter.bi_sector < rhs_cell->holder->bi_iter.bi_sector)
	1820	+ return -1;
	1821	+
	1822	+ if (lhs_cell->holder->bi_iter.bi_sector > rhs_cell->holder->bi_iter.bi_sector)
	1823	+ return 1;
	1824	+
	1825	+ return 0;
	1826	+}
	1827	+
	1828	+static unsigned sort_cells(struct pool pool, struct list_head cells)
	1829	+{
	1830	+ unsigned count = 0;
	1831	+ struct dm_bio_prison_cell cell, tmp;
	1832	+
	1833	+ list_for_each_entry_safe(cell, tmp, cells, user_list) {
	1834	+ if (count >= CELL_SORT_ARRAY_SIZE)
	1835	+ break;
	1836	+
	1837	+ pool->cell_sort_array[count++] = cell;
	1838	+ list_del(&cell->user_list);
	1839	+ }
	1840	+
	1841	+ sort(pool->cell_sort_array, count, sizeof(cell), cmp_cells, NULL);
	1842	+
	1843	+ return count;
	1844	+}
	1845	+
	1846	+static void process_thin_deferred_cells(struct thin_c *tc)
	1847	+{
	1848	+ struct pool *pool = tc->pool;
	1849	+ unsigned long flags;
	1850	+ struct list_head cells;
	1851	+ struct dm_bio_prison_cell *cell;
	1852	+ unsigned i, j, count;
	1853	+
	1854	+ INIT_LIST_HEAD(&cells);
	1855	+
	1856	+ spin_lock_irqsave(&tc->lock, flags);
	1857	+ list_splice_init(&tc->deferred_cells, &cells);
	1858	+ spin_unlock_irqrestore(&tc->lock, flags);
	1859	+
	1860	+ if (list_empty(&cells))
	1861	+ return;
	1862	+
	1863	+ do {
	1864	+ count = sort_cells(tc->pool, &cells);
	1865	+
	1866	+ for (i = 0; i < count; i++) {
	1867	+ cell = pool->cell_sort_array[i];
	1868	+ BUG_ON(!cell->holder);
	1869	+
	1870	+ /*
	1871	+ * If we've got no free new_mapping structs, and processing
	1872	+ * this bio might require one, we pause until there are some
	1873	+ * prepared mappings to process.
	1874	+ */
	1875	+ if (ensure_next_mapping(pool)) {
	1876	+ for (j = i; j < count; j++)
	1877	+ list_add(&pool->cell_sort_array[j]->user_list, &cells);
	1878	+
	1879	+ spin_lock_irqsave(&tc->lock, flags);
	1880	+ list_splice(&cells, &tc->deferred_cells);
	1881	+ spin_unlock_irqrestore(&tc->lock, flags);
	1882	+ return;
	1883	+ }
	1884	+
	1885	+ if (cell->holder->bi_rw & REQ_DISCARD)
	1886	+ pool->process_discard_cell(tc, cell);
	1887	+ else
	1888	+ pool->process_cell(tc, cell);
	1889	+ }
	1890	+ } while (!list_empty(&cells));
	1891	+}
	1892	+
1575	1893	static void thin_get(struct thin_c *tc);
1576	1894	static void thin_put(struct thin_c *tc);
1577	1895
@@ -1620,6 +1938,7 @@ static void process_deferred_bios(struct pool *pool) · 42 unmodified lines
1620	1938
1621	1939	tc = get_first_thin(pool);
1622	1940	while (tc) {
	1941	+ process_thin_deferred_cells(tc);
1623	1942	process_thin_deferred_bios(tc);
1624	1943	tc = get_next_thin(pool, tc);
1625	1944	}
@@ -1653,9 +1972,15 @@ static void do_worker(struct work_struct *ws) · 27 unmodified lines
1653	1972	{
1654	1973	struct pool *pool = container_of(ws, struct pool, worker);
1655	1974
	1975	+ throttle_work_start(&pool->throttle);
	1976	+ dm_pool_issue_prefetches(pool->pmd);
	1977	+ throttle_work_update(&pool->throttle);
1656	1978	process_prepared(pool, &pool->prepared_mappings, &pool->process_prepared_mapping);
	1979	+ throttle_work_update(&pool->throttle);
1657	1980	process_prepared(pool, &pool->prepared_discards, &pool->process_prepared_discard);
	1981	+ throttle_work_update(&pool->throttle);
1658	1982	process_deferred_bios(pool);
	1983	+ throttle_work_complete(&pool->throttle);
1659	1984	}
1660	1985
1661	1986	/*
@@ -1792,6 +2117,8 @@ static void set_pool_mode(struct pool *pool, enum pool_mode new_mode) · 130 unmodified lines
1792	2117	dm_pool_metadata_read_only(pool->pmd);
1793	2118	pool->process_bio = process_bio_fail;
1794	2119	pool->process_discard = process_bio_fail;
	2120	+ pool->process_cell = process_cell_fail;
	2121	+ pool->process_discard_cell = process_cell_fail;
1795	2122	pool->process_prepared_mapping = process_prepared_mapping_fail;
1796	2123	pool->process_prepared_discard = process_prepared_discard_fail;
1797	2124
@@ -1804,6 +2131,8 @@ static void set_pool_mode(struct pool *pool, enum pool_mode new_mode) · 6 unmodified lines
1804	2131	dm_pool_metadata_read_only(pool->pmd);
1805	2132	pool->process_bio = process_bio_read_only;
1806	2133	pool->process_discard = process_bio_success;
	2134	+ pool->process_cell = process_cell_read_only;
	2135	+ pool->process_discard_cell = process_cell_success;
1807	2136	pool->process_prepared_mapping = process_prepared_mapping_fail;
1808	2137	pool->process_prepared_discard = process_prepared_discard_passdown;
1809	2138
@@ -1822,7 +2151,9 @@ static void set_pool_mode(struct pool *pool, enum pool_mode new_mode) · 12 unmodified lines
1822	2151	if (old_mode != new_mode)
1823	2152	notify_of_pool_mode_change(pool, "out-of-data-space");
1824	2153	pool->process_bio = process_bio_read_only;
1825		− pool->process_discard = process_discard;
	2154	+ pool->process_discard = process_discard_bio;
	2155	+ pool->process_cell = process_cell_read_only;
	2156	+ pool->process_discard_cell = process_discard_cell;
1826	2157	pool->process_prepared_mapping = process_prepared_mapping;
1827	2158	pool->process_prepared_discard = process_prepared_discard_passdown;
1828	2159
@@ -1835,7 +2166,9 @@ static void set_pool_mode(struct pool *pool, enum pool_mode new_mode) · 6 unmodified lines
1835	2166	notify_of_pool_mode_change(pool, "write");
1836	2167	dm_pool_metadata_read_write(pool->pmd);
1837	2168	pool->process_bio = process_bio;
1838		− pool->process_discard = process_discard;
	2169	+ pool->process_discard = process_discard_bio;
	2170	+ pool->process_cell = process_cell;
	2171	+ pool->process_discard_cell = process_discard_cell;
1839	2172	pool->process_prepared_mapping = process_prepared_mapping;
1840	2173	pool->process_prepared_discard = process_prepared_discard;
1841	2174	break;
@@ -1895,6 +2228,29 @@ static void thin_defer_bio(struct thin_c tc, struct bio bio) · 53 unmodified lines
1895	2228	wake_worker(pool);
1896	2229	}
1897	2230
	2231	+static void thin_defer_bio_with_throttle(struct thin_c tc, struct bio bio)
	2232	+{
	2233	+ struct pool *pool = tc->pool;
	2234	+
	2235	+ throttle_lock(&pool->throttle);
	2236	+ thin_defer_bio(tc, bio);
	2237	+ throttle_unlock(&pool->throttle);
	2238	+}
	2239	+
	2240	+static void thin_defer_cell(struct thin_c tc, struct dm_bio_prison_cell cell)
	2241	+{
	2242	+ unsigned long flags;
	2243	+ struct pool *pool = tc->pool;
	2244	+
	2245	+ throttle_lock(&pool->throttle);
	2246	+ spin_lock_irqsave(&tc->lock, flags);
	2247	+ list_add_tail(&cell->user_list, &tc->deferred_cells);
	2248	+ spin_unlock_irqrestore(&tc->lock, flags);
	2249	+ throttle_unlock(&pool->throttle);
	2250	+
	2251	+ wake_worker(pool);
	2252	+}
	2253	+
1898	2254	static void thin_hook_bio(struct thin_c tc, struct bio bio)
1899	2255	{
1900	2256	struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
@@ -1915,8 +2271,7 @@ static int thin_bio_map(struct dm_target ti, struct bio bio) · 14 unmodified lines
1915	2271	dm_block_t block = get_bio_block(tc, bio);
1916	2272	struct dm_thin_device *td = tc->td;
1917	2273	struct dm_thin_lookup_result result;
1918		− struct dm_bio_prison_cell cell1, cell2;
1919		− struct dm_bio_prison_cell *cell_result;
	2274	+ struct dm_bio_prison_cell virt_cell, data_cell;
1920	2275	struct dm_cell_key key;
1921	2276
1922	2277	thin_hook_bio(tc, bio);
@@ -1932,7 +2287,7 @@ static int thin_bio_map(struct dm_target ti, struct bio bio) · 9 unmodified lines
1932	2287	}
1933	2288
1934	2289	if (bio->bi_rw & (REQ_DISCARD \| REQ_FLUSH \| REQ_FUA)) {
1935		− thin_defer_bio(tc, bio);
	2290	+ thin_defer_bio_with_throttle(tc, bio);
1936	2291	return DM_MAPIO_SUBMITTED;
1937	2292	}
1938	2293
@@ -1941,7 +2296,7 @@ static int thin_bio_map(struct dm_target ti, struct bio bio) · 2 unmodified lines
1941	2296	* there's a race with discard.
1942	2297	*/
1943	2298	build_virtual_key(tc->td, block, &key);
1944		− if (dm_bio_detain(tc->pool->prison, &key, bio, &cell1, &cell_result))
	2299	+ if (bio_detain(tc->pool, &key, bio, &virt_cell))
1945	2300	return DM_MAPIO_SUBMITTED;
1946	2301
1947	2302	r = dm_thin_find_block(td, block, 0, &result);
@@ -1966,20 +2321,19 @@ static int thin_bio_map(struct dm_target ti, struct bio bio) · 18 unmodified lines
1966	2321	* More distant ancestors are irrelevant. The
1967	2322	* shared flag will be set in their case.
1968	2323	*/
1969		− thin_defer_bio(tc, bio);
1970		− cell_defer_no_holder_no_free(tc, &cell1);
	2324	+ thin_defer_cell(tc, virt_cell);
1971	2325	return DM_MAPIO_SUBMITTED;
1972	2326	}
1973	2327
1974	2328	build_data_key(tc->td, result.block, &key);
1975		− if (dm_bio_detain(tc->pool->prison, &key, bio, &cell2, &cell_result)) {
1976		− cell_defer_no_holder_no_free(tc, &cell1);
	2329	+ if (bio_detain(tc->pool, &key, bio, &data_cell)) {
	2330	+ cell_defer_no_holder(tc, virt_cell);
1977	2331	return DM_MAPIO_SUBMITTED;
1978	2332	}
1979	2333
1980	2334	inc_all_io_entry(tc->pool, bio);
1981		− cell_defer_no_holder_no_free(tc, &cell2);
1982		− cell_defer_no_holder_no_free(tc, &cell1);
	2335	+ cell_defer_no_holder(tc, data_cell);
	2336	+ cell_defer_no_holder(tc, virt_cell);
1983	2337
1984	2338	remap(tc, bio, result.block);
1985	2339	return DM_MAPIO_REMAPPED;
@@ -1991,18 +2345,13 @@ static int thin_bio_map(struct dm_target ti, struct bio bio) · 5 unmodified lines
1991	2345	* of doing so.
1992	2346	*/
1993	2347	handle_unserviceable_bio(tc->pool, bio);
1994		− cell_defer_no_holder_no_free(tc, &cell1);
	2348	+ cell_defer_no_holder(tc, virt_cell);
1995	2349	return DM_MAPIO_SUBMITTED;
1996	2350	}
1997	2351	/* fall through */
1998	2352
1999	2353	case -EWOULDBLOCK:
2000		− /*
2001		− * In future, the failed dm_thin_find_block above could
2002		− * provide the hint to load the metadata into cache.
2003		− */
2004		− thin_defer_bio(tc, bio);
2005		− cell_defer_no_holder_no_free(tc, &cell1);
	2354	+ thin_defer_cell(tc, virt_cell);
2006	2355	return DM_MAPIO_SUBMITTED;
2007	2356
2008	2357	default:
@@ -2012,7 +2361,7 @@ static int thin_bio_map(struct dm_target ti, struct bio bio) · 3 unmodified lines
2012	2361	* pool is switched to fail-io mode.
2013	2362	*/
2014	2363	bio_io_error(bio);
2015		− cell_defer_no_holder_no_free(tc, &cell1);
	2364	+ cell_defer_no_holder(tc, virt_cell);
2016	2365	return DM_MAPIO_SUBMITTED;
2017	2366	}
2018	2367	}
@@ -2193,7 +2542,7 @@ static struct pool pool_create(struct mapped_device pool_md, · 174 unmodified lines
2193	2542	pool->sectors_per_block_shift = __ffs(block_size);
2194	2543	pool->low_water_blocks = 0;
2195	2544	pool_features_init(&pool->pf);
2196		− pool->prison = dm_bio_prison_create(PRISON_CELLS);
	2545	+ pool->prison = dm_bio_prison_create();
2197	2546	if (!pool->prison) {
2198	2547	*error = "Error creating pool's bio prison";
2199	2548	err_p = ERR_PTR(-ENOMEM);
@@ -2219,6 +2568,7 @@ static struct pool pool_create(struct mapped_device pool_md, · 19 unmodified lines
2219	2568	goto bad_wq;
2220	2569	}
2221	2570
	2571	+ throttle_init(&pool->throttle);
2222	2572	INIT_WORK(&pool->worker, do_worker);
2223	2573	INIT_DELAYED_WORK(&pool->waker, do_waker);
2224	2574	INIT_DELAYED_WORK(&pool->no_space_timeout, do_no_space_timeout);
@@ -2228,6 +2578,7 @@ static struct pool pool_create(struct mapped_device pool_md, · 3 unmodified lines
2228	2578	INIT_LIST_HEAD(&pool->prepared_discards);
2229	2579	INIT_LIST_HEAD(&pool->active_thins);
2230	2580	pool->low_water_triggered = false;
	2581	+ pool->suspended = true;
2231	2582
2232	2583	pool->shared_read_ds = dm_deferred_set_create();
2233	2584	if (!pool->shared_read_ds) {
@@ -2764,20 +3115,77 @@ static int pool_preresume(struct dm_target *ti) · 530 unmodified lines
2764	3115	return 0;
2765	3116	}
2766	3117
	3118	+static void pool_suspend_active_thins(struct pool *pool)
	3119	+{
	3120	+ struct thin_c *tc;
	3121	+
	3122	+ /* Suspend all active thin devices */
	3123	+ tc = get_first_thin(pool);
	3124	+ while (tc) {
	3125	+ dm_internal_suspend_noflush(tc->thin_md);
	3126	+ tc = get_next_thin(pool, tc);
	3127	+ }
	3128	+}
	3129	+
	3130	+static void pool_resume_active_thins(struct pool *pool)
	3131	+{
	3132	+ struct thin_c *tc;
	3133	+
	3134	+ /* Resume all active thin devices */
	3135	+ tc = get_first_thin(pool);
	3136	+ while (tc) {
	3137	+ dm_internal_resume(tc->thin_md);
	3138	+ tc = get_next_thin(pool, tc);
	3139	+ }
	3140	+}
	3141	+
2767	3142	static void pool_resume(struct dm_target *ti)
2768	3143	{
2769	3144	struct pool_c *pt = ti->private;
2770	3145	struct pool *pool = pt->pool;
2771	3146	unsigned long flags;
2772	3147
	3148	+ /*
	3149	+ * Must requeue active_thins' bios and then resume
	3150	+ * active_thins _before_ clearing 'suspend' flag.
	3151	+ */
	3152	+ requeue_bios(pool);
	3153	+ pool_resume_active_thins(pool);
	3154	+
2773	3155	spin_lock_irqsave(&pool->lock, flags);
2774	3156	pool->low_water_triggered = false;
	3157	+ pool->suspended = false;
2775	3158	spin_unlock_irqrestore(&pool->lock, flags);
2776		− requeue_bios(pool);
2777	3159
2778	3160	do_waker(&pool->waker.work);
2779	3161	}
2780	3162
	3163	+static void pool_presuspend(struct dm_target *ti)
	3164	+{
	3165	+ struct pool_c *pt = ti->private;
	3166	+ struct pool *pool = pt->pool;
	3167	+ unsigned long flags;
	3168	+
	3169	+ spin_lock_irqsave(&pool->lock, flags);
	3170	+ pool->suspended = true;
	3171	+ spin_unlock_irqrestore(&pool->lock, flags);
	3172	+
	3173	+ pool_suspend_active_thins(pool);
	3174	+}
	3175	+
	3176	+static void pool_presuspend_undo(struct dm_target *ti)
	3177	+{
	3178	+ struct pool_c *pt = ti->private;
	3179	+ struct pool *pool = pt->pool;
	3180	+ unsigned long flags;
	3181	+
	3182	+ pool_resume_active_thins(pool);
	3183	+
	3184	+ spin_lock_irqsave(&pool->lock, flags);
	3185	+ pool->suspended = false;
	3186	+ spin_unlock_irqrestore(&pool->lock, flags);
	3187	+}
	3188	+
2781	3189	static void pool_postsuspend(struct dm_target *ti)
2782	3190	{
2783	3191	struct pool_c *pt = ti->private;
@@ -2949,7 +3357,6 @@ static int process_release_metadata_snap_mesg(unsigned argc, char **argv, struct · 165 unmodified lines
2949	3357	* create_thin <dev_id>
2950	3358	* create_snap <dev_id> <origin_id>
2951	3359	* delete <dev_id>
2952		− * trim <dev_id> <new_size_in_sectors>
2953	3360	* set_transaction_id <current_trans_id> <new_trans_id>
2954	3361	* reserve_metadata_snap
2955	3362	* release_metadata_snap
@@ -3177,15 +3584,35 @@ static void pool_io_hints(struct dm_target ti, struct queue_limits limits) · 221 unmodified lines
3177	3584	{
3178	3585	struct pool_c *pt = ti->private;
3179	3586	struct pool *pool = pt->pool;
3180		− uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
	3587	+ sector_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
	3588	+
	3589	+ /*
	3590	+ * If max_sectors is smaller than pool->sectors_per_block adjust it
	3591	+ * to the highest possible power-of-2 factor of pool->sectors_per_block.
	3592	+ * This is especially beneficial when the pool's data device is a RAID
	3593	+ * device that has a full stripe width that matches pool->sectors_per_block
	3594	+ * -- because even though partial RAID stripe-sized IOs will be issued to a
	3595	+ * single RAID stripe; when aggregated they will end on a full RAID stripe
	3596	+ * boundary.. which avoids additional partial RAID stripe writes cascading
	3597	+ */
	3598	+ if (limits->max_sectors < pool->sectors_per_block) {
	3599	+ while (!is_factor(pool->sectors_per_block, limits->max_sectors)) {
	3600	+ if ((limits->max_sectors & (limits->max_sectors - 1)) == 0)
	3601	+ limits->max_sectors--;
	3602	+ limits->max_sectors = rounddown_pow_of_two(limits->max_sectors);
	3603	+ }
	3604	+ }
3181	3605
3182	3606	/*
3183	3607	* If the system-determined stacked limits are compatible with the
3184	3608	* pool's blocksize (io_opt is a factor) do not override them.
3185	3609	*/
3186	3610	if (io_opt_sectors < pool->sectors_per_block \|\|
3187		− do_div(io_opt_sectors, pool->sectors_per_block)) {
3188		− blk_limits_io_min(limits, pool->sectors_per_block << SECTOR_SHIFT);
	3611	+ !is_factor(io_opt_sectors, pool->sectors_per_block)) {
	3612	+ if (is_factor(pool->sectors_per_block, limits->max_sectors))
	3613	+ blk_limits_io_min(limits, limits->max_sectors << SECTOR_SHIFT);
	3614	+ else
	3615	+ blk_limits_io_min(limits, pool->sectors_per_block << SECTOR_SHIFT);
3189	3616	blk_limits_io_opt(limits, pool->sectors_per_block << SECTOR_SHIFT);
3190	3617	}
3191	3618
@@ -3214,11 +3641,13 @@ static struct target_type pool_target = { · 22 unmodified lines
3214	3641	.name = "thin-pool",
3215	3642	.features = DM_TARGET_SINGLETON \| DM_TARGET_ALWAYS_WRITEABLE \|
3216	3643	DM_TARGET_IMMUTABLE,
3217		− .version = {1, 13, 0},
	3644	+ .version = {1, 14, 0},
3218	3645	.module = THIS_MODULE,
3219	3646	.ctr = pool_ctr,
3220	3647	.dtr = pool_dtr,
3221	3648	.map = pool_map,
	3649	+ .presuspend = pool_presuspend,
	3650	+ .presuspend_undo = pool_presuspend_undo,
3222	3651	.postsuspend = pool_postsuspend,
3223	3652	.preresume = pool_preresume,
3224	3653	.resume = pool_resume,
@@ -3248,14 +3677,14 @@ static void thin_dtr(struct dm_target *ti) · 23 unmodified lines
3248	3677	struct thin_c *tc = ti->private;
3249	3678	unsigned long flags;
3250	3679
3251		− thin_put(tc);
3252		− wait_for_completion(&tc->can_destroy);
3253		−
3254	3680	spin_lock_irqsave(&tc->pool->lock, flags);
3255	3681	list_del_rcu(&tc->list);
3256	3682	spin_unlock_irqrestore(&tc->pool->lock, flags);
3257	3683	synchronize_rcu();
3258	3684
	3685	+ thin_put(tc);
	3686	+ wait_for_completion(&tc->can_destroy);
	3687	+
3259	3688	mutex_lock(&dm_thin_pool_table.mutex);
3260	3689
3261	3690	__pool_dec(tc->pool);
@@ -3302,7 +3731,9 @@ static int thin_ctr(struct dm_target ti, unsigned argc, char *argv) · 40 unmodified lines
3302	3731	r = -ENOMEM;
3303	3732	goto out_unlock;
3304	3733	}
	3734	+ tc->thin_md = dm_table_get_md(ti->table);
3305	3735	spin_lock_init(&tc->lock);
	3736	+ INIT_LIST_HEAD(&tc->deferred_cells);
3306	3737	bio_list_init(&tc->deferred_bio_list);
3307	3738	bio_list_init(&tc->retry_on_resume_list);
3308	3739	tc->sort_bio_list = RB_ROOT;
@@ -3347,18 +3778,18 @@ static int thin_ctr(struct dm_target ti, unsigned argc, char *argv) · 38 unmodified lines
3347	3778	if (get_pool_mode(tc->pool) == PM_FAIL) {
3348	3779	ti->error = "Couldn't open thin device, Pool is in fail mode";
3349	3780	r = -EINVAL;
3350		− goto bad_thin_open;
	3781	+ goto bad_pool;
3351	3782	}
3352	3783
3353	3784	r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td);
3354	3785	if (r) {
3355	3786	ti->error = "Couldn't open thin internal device";
3356		− goto bad_thin_open;
	3787	+ goto bad_pool;
3357	3788	}
3358	3789
3359	3790	r = dm_set_target_max_io_len(ti, tc->pool->sectors_per_block);
3360	3791	if (r)
3361		− goto bad_target_max_io_len;
	3792	+ goto bad;
3362	3793
3363	3794	ti->num_flush_bios = 1;
3364	3795	ti->flush_supported = true;
@@ -3373,14 +3804,16 @@ static int thin_ctr(struct dm_target ti, unsigned argc, char *argv) · 8 unmodified lines
3373	3804	ti->split_discard_bios = true;
3374	3805	}
3375	3806
3376		− dm_put(pool_md);
3377		−
3378	3807	mutex_unlock(&dm_thin_pool_table.mutex);
3379	3808
3380		− atomic_set(&tc->refcount, 1);
3381		− init_completion(&tc->can_destroy);
3382		−
3383	3809	spin_lock_irqsave(&tc->pool->lock, flags);
	3810	+ if (tc->pool->suspended) {
	3811	+ spin_unlock_irqrestore(&tc->pool->lock, flags);
	3812	+ mutex_lock(&dm_thin_pool_table.mutex); /* reacquire for __pool_dec */
	3813	+ ti->error = "Unable to activate thin device while pool is suspended";
	3814	+ r = -EINVAL;
	3815	+ goto bad;
	3816	+ }
3384	3817	list_add_tail_rcu(&tc->list, &tc->pool->active_thins);
3385	3818	spin_unlock_irqrestore(&tc->pool->lock, flags);
3386	3819	/*
@@ -3391,11 +3824,16 @@ static int thin_ctr(struct dm_target ti, unsigned argc, char *argv) · 4 unmodified lines
3391	3824	*/
3392	3825	synchronize_rcu();
3393	3826
	3827	+ dm_put(pool_md);
	3828	+
	3829	+ atomic_set(&tc->refcount, 1);
	3830	+ init_completion(&tc->can_destroy);
	3831	+
3394	3832	return 0;
3395	3833
3396		−bad_target_max_io_len:
	3834	+bad:
3397	3835	dm_pool_close_thin_device(tc->td);
3398		−bad_thin_open:
	3836	+bad_pool:
3399	3837	__pool_dec(tc->pool);
3400	3838	bad_pool_lookup:
3401	3839	dm_put(pool_md);
@@ -3541,6 +3979,21 @@ err: · 139 unmodified lines
3541	3979	DMEMIT("Error");
3542	3980	}
3543	3981
	3982	+static int thin_merge(struct dm_target ti, struct bvec_merge_data bvm,
	3983	+ struct bio_vec *biovec, int max_size)
	3984	+{
	3985	+ struct thin_c *tc = ti->private;
	3986	+ struct request_queue *q = bdev_get_queue(tc->pool_dev->bdev);
	3987	+
	3988	+ if (!q->merge_bvec_fn)
	3989	+ return max_size;
	3990	+
	3991	+ bvm->bi_bdev = tc->pool_dev->bdev;
	3992	+ bvm->bi_sector = dm_target_offset(ti, bvm->bi_sector);
	3993	+
	3994	+ return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
	3995	+}
	3996	+
3544	3997	static int thin_iterate_devices(struct dm_target *ti,
3545	3998	iterate_devices_callout_fn fn, void *data)
3546	3999	{
@@ -3565,7 +4018,7 @@ static int thin_iterate_devices(struct dm_target *ti, · 18 unmodified lines
3565	4018
3566	4019	static struct target_type thin_target = {
3567	4020	.name = "thin",
3568		− .version = {1, 13, 0},
	4021	+ .version = {1, 14, 0},
3569	4022	.module = THIS_MODULE,
3570	4023	.ctr = thin_ctr,
3571	4024	.dtr = thin_dtr,
@@ -3575,6 +4028,7 @@ static struct target_type thin_target = { · 3 unmodified lines
3575	4028	.presuspend = thin_presuspend,
3576	4029	.postsuspend = thin_postsuspend,
3577	4030	.status = thin_status,
	4031	+ .merge = thin_merge,
3578	4032	.iterate_devices = thin_iterate_devices,
3579	4033	};
3580	4034

drivers/md/dm.c+200−73

@@ -19,6 +19,7 @@ · 18 unmodified lines
19	19	#include <linux/idr.h>
20	20	#include <linux/hdreg.h>
21	21	#include <linux/delay.h>
	22	+#include <linux/wait.h>
22	23
23	24	#include <trace/events/block.h>
24	25
@@ -117,6 +118,7 @@ EXPORT_SYMBOL_GPL(dm_get_rq_mapinfo); · 92 unmodified lines
117	118	#define DMF_NOFLUSH_SUSPENDING 5
118	119	#define DMF_MERGE_IS_OPTIONAL 6
119	120	#define DMF_DEFERRED_REMOVE 7
	121	+#define DMF_SUSPENDED_INTERNALLY 8
120	122
121	123	/*
122	124	* A dummy definition to make RCU happy.
@@ -140,7 +142,7 @@ struct mapped_device { · 17 unmodified lines
140	142	* Use dm_get_live_table{_fast} or take suspend_lock for
141	143	* dereference.
142	144	*/
143		− struct dm_table *map;
	145	+ struct dm_table __rcu *map;
144	146
145	147	struct list_head table_devices;
146	148	struct mutex table_devices_lock;
@@ -525,14 +527,15 @@ retry: · 378 unmodified lines
525	527	goto out;
526	528
527	529	tgt = dm_table_get_target(map, 0);
	530	+ if (!tgt->type->ioctl)
	531	+ goto out;
528	532
529	533	if (dm_suspended_md(md)) {
530	534	r = -EAGAIN;
531	535	goto out;
532	536	}
533	537
534		− if (tgt->type->ioctl)
535		− r = tgt->type->ioctl(tgt, cmd, arg);
	538	+ r = tgt->type->ioctl(tgt, cmd, arg);
536	539
537	540	out:
538	541	dm_put_live_table(md, srcu_idx);
@@ -1607,9 +1610,9 @@ static int dm_merge_bvec(struct request_queue *q, · 1068 unmodified lines
1607	1610	* Find maximum amount of I/O that won't need splitting
1608	1611	*/
1609	1612	max_sectors = min(max_io_len(bvm->bi_sector, ti),
1610		− (sector_t) BIO_MAX_SECTORS);
	1613	+ (sector_t) queue_max_sectors(q));
1611	1614	max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
1612		− if (max_size < 0)
	1615	+ if (unlikely(max_size < 0)) /* this shouldn't _ever_ happen */
1613	1616	max_size = 0;
1614	1617
1615	1618	/*
@@ -1621,10 +1624,10 @@ static int dm_merge_bvec(struct request_queue *q, · 5 unmodified lines
1621	1624	max_size = ti->type->merge(ti, bvm, biovec, max_size);
1622	1625	/*
1623	1626	* If the target doesn't support merge method and some of the devices
1624		− * provided their merge_bvec method (we know this by looking at
1625		− * queue_max_hw_sectors), then we can't allow bios with multiple vector
1626		− * entries. So always set max_size to 0, and the code below allows
1627		− * just one page.
	1627	+ * provided their merge_bvec method (we know this by looking for the
	1628	+ * max_hw_sectors that dm_set_device_limits may set), then we can't
	1629	+ * allow bios with multiple vector entries. So always set max_size
	1630	+ * to 0, and the code below allows just one page.
1628	1631	*/
1629	1632	else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9)
1630	1633	max_size = 0;
@@ -2332,7 +2335,7 @@ static struct dm_table __bind(struct mapped_device md, struct dm_table *t, · 701 unmodified lines
2332	2335
2333	2336	merge_is_optional = dm_table_merge_is_optional(t);
2334	2337
2335		− old_map = md->map;
	2338	+ old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2336	2339	rcu_assign_pointer(md->map, t);
2337	2340	md->immutable_target_type = dm_table_get_immutable_target_type(t);
2338	2341
@@ -2341,7 +2344,8 @@ static struct dm_table __bind(struct mapped_device md, struct dm_table *t, · 2 unmodified lines
2341	2344	set_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
2342	2345	else
2343	2346	clear_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
2344		− dm_sync_table(md);
	2347	+ if (old_map)
	2348	+ dm_sync_table(md);
2345	2349
2346	2350	return old_map;
2347	2351	}
@@ -2351,7 +2355,7 @@ static struct dm_table __bind(struct mapped_device md, struct dm_table *t, · 3 unmodified lines
2351	2355	*/
2352	2356	static struct dm_table __unbind(struct mapped_device md)
2353	2357	{
2354		− struct dm_table *map = md->map;
	2358	+ struct dm_table *map = rcu_dereference_protected(md->map, 1);
2355	2359
2356	2360	if (!map)
2357	2361	return NULL;
@@ -2716,36 +2720,18 @@ static void unlock_fs(struct mapped_device *md) · 358 unmodified lines
2716	2720	}
2717	2721
2718	2722	/*
2719		− * We need to be able to change a mapping table under a mounted
2720		− * filesystem. For example we might want to move some data in
2721		− * the background. Before the table can be swapped with
2722		− * dm_bind_table, dm_suspend must be called to flush any in
2723		− * flight bios and ensure that any further io gets deferred.
2724		− */
2725		−/*
2726		− * Suspend mechanism in request-based dm.
	2723	+ * If __dm_suspend returns 0, the device is completely quiescent
	2724	+ * now. There is no request-processing activity. All new requests
	2725	+ * are being added to md->deferred list.
2727	2726	*
2728		− * 1. Flush all I/Os by lock_fs() if needed.
2729		− * 2. Stop dispatching any I/O by stopping the request_queue.
2730		− * 3. Wait for all in-flight I/Os to be completed or requeued.
2731		− *
2732		− * To abort suspend, start the request_queue.
	2727	+ * Caller must hold md->suspend_lock
2733	2728	*/
2734		−int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
	2729	+static int __dm_suspend(struct mapped_device md, struct dm_table map,
	2730	+ unsigned suspend_flags, int interruptible)
2735	2731	{
2736		− struct dm_table *map = NULL;
2737		− int r = 0;
2738		− int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
2739		− int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
2740		−
2741		− mutex_lock(&md->suspend_lock);
2742		−
2743		− if (dm_suspended_md(md)) {
2744		− r = -EINVAL;
2745		− goto out_unlock;
2746		− }
2747		−
2748		− map = md->map;
	2732	+ bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
	2733	+ bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
	2734	+ int r;
2749	2735
2750	2736	/*
2751	2737	* DMF_NOFLUSH_SUSPENDING must be set before presuspend.
@@ -2754,7 +2740,10 @@ int dm_suspend(struct mapped_device *md, unsigned suspend_flags) · 2 unmodified lines
2754	2740	if (noflush)
2755	2741	set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2756	2742
2757		− /* This does not get reverted if there's an error later. */
	2743	+ /*
	2744	+ * This gets reverted if there's an error later and the targets
	2745	+ * provide the .presuspend_undo hook.
	2746	+ */
2758	2747	dm_table_presuspend_targets(map);
2759	2748
2760	2749	/*
@@ -2765,8 +2754,10 @@ int dm_suspend(struct mapped_device *md, unsigned suspend_flags) · 4 unmodified lines
2765	2754	*/
2766	2755	if (!noflush && do_lockfs) {
2767	2756	r = lock_fs(md);
2768		− if (r)
2769		− goto out_unlock;
	2757	+ if (r) {
	2758	+ dm_table_presuspend_undo_targets(map);
	2759	+ return r;
	2760	+ }
2770	2761	}
2771	2762
2772	2763	/*
@@ -2782,7 +2773,8 @@ int dm_suspend(struct mapped_device *md, unsigned suspend_flags) · 9 unmodified lines
2782	2773	* flush_workqueue(md->wq).
2783	2774	*/
2784	2775	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2785		− synchronize_srcu(&md->io_barrier);
	2776	+ if (map)
	2777	+ synchronize_srcu(&md->io_barrier);
2786	2778
2787	2779	/*
2788	2780	* Stop md->queue before flushing md->wq in case request-based
@@ -2798,11 +2790,12 @@ int dm_suspend(struct mapped_device *md, unsigned suspend_flags) · 9 unmodified lines
2798	2790	* We call dm_wait_for_completion to wait for all existing requests
2799	2791	* to finish.
2800	2792	*/
2801		− r = dm_wait_for_completion(md, TASK_INTERRUPTIBLE);
	2793	+ r = dm_wait_for_completion(md, interruptible);
2802	2794
2803	2795	if (noflush)
2804	2796	clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2805		− synchronize_srcu(&md->io_barrier);
	2797	+ if (map)
	2798	+ synchronize_srcu(&md->io_barrier);
2806	2799
2807	2800	/* were we interrupted ? */
2808	2801	if (r < 0) {
@@ -2812,14 +2805,56 @@ int dm_suspend(struct mapped_device *md, unsigned suspend_flags) · 3 unmodified lines
2812	2805	start_queue(md->queue);
2813	2806
2814	2807	unlock_fs(md);
2815		− goto out_unlock; /* pushback list is already flushed, so skip flush */
	2808	+ dm_table_presuspend_undo_targets(map);
	2809	+ /* pushback list is already flushed, so skip flush */
2816	2810	}
2817	2811
2818		− /*
2819		− * If dm_wait_for_completion returned 0, the device is completely
2820		− * quiescent now. There is no request-processing activity. All new
2821		− * requests are being added to md->deferred list.
2822		− */
	2812	+ return r;
	2813	+}
	2814	+
	2815	+/*
	2816	+ * We need to be able to change a mapping table under a mounted
	2817	+ * filesystem. For example we might want to move some data in
	2818	+ * the background. Before the table can be swapped with
	2819	+ * dm_bind_table, dm_suspend must be called to flush any in
	2820	+ * flight bios and ensure that any further io gets deferred.
	2821	+ */
	2822	+/*
	2823	+ * Suspend mechanism in request-based dm.
	2824	+ *
	2825	+ * 1. Flush all I/Os by lock_fs() if needed.
	2826	+ * 2. Stop dispatching any I/O by stopping the request_queue.
	2827	+ * 3. Wait for all in-flight I/Os to be completed or requeued.
	2828	+ *
	2829	+ * To abort suspend, start the request_queue.
	2830	+ */
	2831	+int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
	2832	+{
	2833	+ struct dm_table *map = NULL;
	2834	+ int r = 0;
	2835	+
	2836	+retry:
	2837	+ mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
	2838	+
	2839	+ if (dm_suspended_md(md)) {
	2840	+ r = -EINVAL;
	2841	+ goto out_unlock;
	2842	+ }
	2843	+
	2844	+ if (dm_suspended_internally_md(md)) {
	2845	+ /* already internally suspended, wait for internal resume */
	2846	+ mutex_unlock(&md->suspend_lock);
	2847	+ r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
	2848	+ if (r)
	2849	+ return r;
	2850	+ goto retry;
	2851	+ }
	2852	+
	2853	+ map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
	2854	+
	2855	+ r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE);
	2856	+ if (r)
	2857	+ goto out_unlock;
2823	2858
2824	2859	set_bit(DMF_SUSPENDED, &md->flags);
2825	2860
@@ -2830,22 +2865,13 @@ out_unlock: · 4 unmodified lines
2830	2865	return r;
2831	2866	}
2832	2867
2833		−int dm_resume(struct mapped_device *md)
	2868	+static int __dm_resume(struct mapped_device md, struct dm_table map)
2834	2869	{
2835		− int r = -EINVAL;
2836		− struct dm_table *map = NULL;
2837		−
2838		− mutex_lock(&md->suspend_lock);
2839		− if (!dm_suspended_md(md))
2840		− goto out;
2841		−
2842		− map = md->map;
2843		− if (!map \|\| !dm_table_get_size(map))
2844		− goto out;
2845		−
2846		− r = dm_table_resume_targets(map);
2847		− if (r)
2848		− goto out;
	2870	+ if (map) {
	2871	+ int r = dm_table_resume_targets(map);
	2872	+ if (r)
	2873	+ return r;
	2874	+ }
2849	2875
2850	2876	dm_queue_flush(md);
2851	2877
@@ -2859,6 +2885,37 @@ int dm_resume(struct mapped_device *md) · 7 unmodified lines
2859	2885
2860	2886	unlock_fs(md);
2861	2887
	2888	+ return 0;
	2889	+}
	2890	+
	2891	+int dm_resume(struct mapped_device *md)
	2892	+{
	2893	+ int r = -EINVAL;
	2894	+ struct dm_table *map = NULL;
	2895	+
	2896	+retry:
	2897	+ mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
	2898	+
	2899	+ if (!dm_suspended_md(md))
	2900	+ goto out;
	2901	+
	2902	+ if (dm_suspended_internally_md(md)) {
	2903	+ /* already internally suspended, wait for internal resume */
	2904	+ mutex_unlock(&md->suspend_lock);
	2905	+ r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
	2906	+ if (r)
	2907	+ return r;
	2908	+ goto retry;
	2909	+ }
	2910	+
	2911	+ map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
	2912	+ if (!map \|\| !dm_table_get_size(map))
	2913	+ goto out;
	2914	+
	2915	+ r = __dm_resume(md, map);
	2916	+ if (r)
	2917	+ goto out;
	2918	+
2862	2919	clear_bit(DMF_SUSPENDED, &md->flags);
2863	2920
2864	2921	r = 0;
@@ -2872,15 +2929,80 @@ out: · 7 unmodified lines
2872	2929	* Internal suspend/resume works like userspace-driven suspend. It waits
2873	2930	* until all bios finish and prevents issuing new bios to the target drivers.
2874	2931	* It may be used only from the kernel.
2875		− *
2876		− * Internal suspend holds md->suspend_lock, which prevents interaction with
2877		− * userspace-driven suspend.
2878	2932	*/
2879	2933
2880		−void dm_internal_suspend(struct mapped_device *md)
	2934	+static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)
2881	2935	{
2882		− mutex_lock(&md->suspend_lock);
	2936	+ struct dm_table *map = NULL;
	2937	+
	2938	+ if (dm_suspended_internally_md(md))
	2939	+ return; /* nested internal suspend */
	2940	+
	2941	+ if (dm_suspended_md(md)) {
	2942	+ set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
	2943	+ return; /* nest suspend */
	2944	+ }
	2945	+
	2946	+ map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
	2947	+
	2948	+ /*
	2949	+ * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
	2950	+ * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend
	2951	+ * would require changing .presuspend to return an error -- avoid this
	2952	+ * until there is a need for more elaborate variants of internal suspend.
	2953	+ */
	2954	+ (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE);
	2955	+
	2956	+ set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
	2957	+
	2958	+ dm_table_postsuspend_targets(map);
	2959	+}
	2960	+
	2961	+static void __dm_internal_resume(struct mapped_device *md)
	2962	+{
	2963	+ if (!dm_suspended_internally_md(md))
	2964	+ return; /* resume from nested internal suspend */
	2965	+
2883	2966	if (dm_suspended_md(md))
	2967	+ goto done; /* resume from nested suspend */
	2968	+
	2969	+ /*
	2970	+ * NOTE: existing callers don't need to call dm_table_resume_targets
	2971	+ * (which may fail -- so best to avoid it for now by passing NULL map)
	2972	+ */
	2973	+ (void) __dm_resume(md, NULL);
	2974	+
	2975	+done:
	2976	+ clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
	2977	+ smp_mb__after_atomic();
	2978	+ wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
	2979	+}
	2980	+
	2981	+void dm_internal_suspend_noflush(struct mapped_device *md)
	2982	+{
	2983	+ mutex_lock(&md->suspend_lock);
	2984	+ __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
	2985	+ mutex_unlock(&md->suspend_lock);
	2986	+}
	2987	+EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
	2988	+
	2989	+void dm_internal_resume(struct mapped_device *md)
	2990	+{
	2991	+ mutex_lock(&md->suspend_lock);
	2992	+ __dm_internal_resume(md);
	2993	+ mutex_unlock(&md->suspend_lock);
	2994	+}
	2995	+EXPORT_SYMBOL_GPL(dm_internal_resume);
	2996	+
	2997	+/*
	2998	+ * Fast variants of internal suspend/resume hold md->suspend_lock,
	2999	+ * which prevents interaction with userspace-driven suspend.
	3000	+ */
	3001	+
	3002	+void dm_internal_suspend_fast(struct mapped_device *md)
	3003	+{
	3004	+ mutex_lock(&md->suspend_lock);
	3005	+ if (dm_suspended_md(md) \|\| dm_suspended_internally_md(md))
2884	3006	return;
2885	3007
2886	3008	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
@@ -2889,9 +3011,9 @@ void dm_internal_suspend(struct mapped_device *md) · 2 unmodified lines
2889	3011	dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2890	3012	}
2891	3013
2892		−void dm_internal_resume(struct mapped_device *md)
	3014	+void dm_internal_resume_fast(struct mapped_device *md)
2893	3015	{
2894		− if (dm_suspended_md(md))
	3016	+ if (dm_suspended_md(md) \|\| dm_suspended_internally_md(md))
2895	3017	goto done;
2896	3018
2897	3019	dm_queue_flush(md);
@@ -2977,6 +3099,11 @@ int dm_suspended_md(struct mapped_device *md) · 79 unmodified lines
2977	3099	return test_bit(DMF_SUSPENDED, &md->flags);
2978	3100	}
2979	3101
	3102	+int dm_suspended_internally_md(struct mapped_device *md)
	3103	+{
	3104	+ return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
	3105	+}
	3106	+
2980	3107	int dm_test_deferred_remove_flag(struct mapped_device *md)
2981	3108	{
2982	3109	return test_bit(DMF_DEFERRED_REMOVE, &md->flags);

drivers/md/dm.h+10−0

@@ -65,6 +65,7 @@ void dm_table_set_restrictions(struct dm_table t, struct request_queue q, · 64 unmodified lines
65	65	struct queue_limits *limits);
66	66	struct list_head dm_table_get_devices(struct dm_table t);
67	67	void dm_table_presuspend_targets(struct dm_table *t);
	68	+void dm_table_presuspend_undo_targets(struct dm_table *t);
68	69	void dm_table_postsuspend_targets(struct dm_table *t);
69	70	int dm_table_resume_targets(struct dm_table *t);
70	71	int dm_table_any_congested(struct dm_table *t, int bdi_bits);
@@ -128,6 +129,15 @@ int dm_deleting_md(struct mapped_device *md); · 57 unmodified lines
128	129	*/
129	130	int dm_suspended_md(struct mapped_device *md);
130	131
	132	+/*
	133	+ * Internal suspend and resume methods.
	134	+ */
	135	+int dm_suspended_internally_md(struct mapped_device *md);
	136	+void dm_internal_suspend_fast(struct mapped_device *md);
	137	+void dm_internal_resume_fast(struct mapped_device *md);
	138	+void dm_internal_suspend_noflush(struct mapped_device *md);
	139	+void dm_internal_resume(struct mapped_device *md);
	140	+
131	141	/*
132	142	* Test if the device is scheduled for deferred remove.
133	143	*/

drivers/md/persistent-data/dm-array.c+3−1

@@ -645,8 +645,10 @@ static int array_resize(struct dm_array_info *info, dm_block_t root, · 644 unmodified lines
645	645	int r;
646	646	struct resize resize;
647	647
648		− if (old_size == new_size)
	648	+ if (old_size == new_size) {
	649	+ *new_root = root;
649	650	return 0;
	651	+ }
650	652
651	653	resize.info = info;
652	654	resize.root = root;

drivers/md/persistent-data/dm-space-map-metadata.c+6−2

@@ -564,7 +564,9 @@ static int sm_bootstrap_get_nr_blocks(struct dm_space_map sm, dm_block_t count · 563 unmodified lines
564	564	{
565	565	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);
566	566
567		− return smm->ll.nr_blocks;
	567	+ *count = smm->ll.nr_blocks;
	568	+
	569	+ return 0;
568	570	}
569	571
570	572	static int sm_bootstrap_get_nr_free(struct dm_space_map sm, dm_block_t count)
@@ -581,7 +583,9 @@ static int sm_bootstrap_get_count(struct dm_space_map *sm, dm_block_t b, · 10 unmodified lines
581	583	{
582	584	struct sm_metadata *smm = container_of(sm, struct sm_metadata, sm);
583	585
584		− return b < smm->begin ? 1 : 0;
	586	+ *result = (b < smm->begin) ? 1 : 0;
	587	+
	588	+ return 0;
585	589	}
586	590
587	591	static int sm_bootstrap_count_is_more_than_one(struct dm_space_map *sm,

drivers/md/persistent-data/dm-transaction-manager.c+75−2

@@ -10,6 +10,8 @@ · 9 unmodified lines
10	10	#include "dm-persistent-data-internal.h"
11	11
12	12	#include <linux/export.h>
	13	+#include <linux/mutex.h>
	14	+#include <linux/hash.h>
13	15	#include <linux/slab.h>
14	16	#include <linux/device-mapper.h>
15	17
@@ -17,6 +19,61 @@ · 1 unmodified line
17	19
18	20	/----------------------------------------------------------------/
19	21
	22	+#define PREFETCH_SIZE 128
	23	+#define PREFETCH_BITS 7
	24	+#define PREFETCH_SENTINEL ((dm_block_t) -1ULL)
	25	+
	26	+struct prefetch_set {
	27	+ struct mutex lock;
	28	+ dm_block_t blocks[PREFETCH_SIZE];
	29	+};
	30	+
	31	+static unsigned prefetch_hash(dm_block_t b)
	32	+{
	33	+ return hash_64(b, PREFETCH_BITS);
	34	+}
	35	+
	36	+static void prefetch_wipe(struct prefetch_set *p)
	37	+{
	38	+ unsigned i;
	39	+ for (i = 0; i < PREFETCH_SIZE; i++)
	40	+ p->blocks[i] = PREFETCH_SENTINEL;
	41	+}
	42	+
	43	+static void prefetch_init(struct prefetch_set *p)
	44	+{
	45	+ mutex_init(&p->lock);
	46	+ prefetch_wipe(p);
	47	+}
	48	+
	49	+static void prefetch_add(struct prefetch_set *p, dm_block_t b)
	50	+{
	51	+ unsigned h = prefetch_hash(b);
	52	+
	53	+ mutex_lock(&p->lock);
	54	+ if (p->blocks[h] == PREFETCH_SENTINEL)
	55	+ p->blocks[h] = b;
	56	+
	57	+ mutex_unlock(&p->lock);
	58	+}
	59	+
	60	+static void prefetch_issue(struct prefetch_set p, struct dm_block_manager bm)
	61	+{
	62	+ unsigned i;
	63	+
	64	+ mutex_lock(&p->lock);
	65	+
	66	+ for (i = 0; i < PREFETCH_SIZE; i++)
	67	+ if (p->blocks[i] != PREFETCH_SENTINEL) {
	68	+ dm_bm_prefetch(bm, p->blocks[i]);
	69	+ p->blocks[i] = PREFETCH_SENTINEL;
	70	+ }
	71	+
	72	+ mutex_unlock(&p->lock);
	73	+}
	74	+
	75	+/----------------------------------------------------------------/
	76	+
20	77	struct shadow_info {
21	78	struct hlist_node hlist;
22	79	dm_block_t where;
@@ -37,6 +94,8 @@ struct dm_transaction_manager { · 14 unmodified lines
37	94
38	95	spinlock_t lock;
39	96	struct hlist_head buckets[DM_HASH_SIZE];
	97	+
	98	+ struct prefetch_set prefetches;
40	99	};
41	100
42	101	/----------------------------------------------------------------/
@@ -117,6 +176,8 @@ static struct dm_transaction_manager dm_tm_create(struct dm_block_manager bm, · 74 unmodified lines
117	176	for (i = 0; i < DM_HASH_SIZE; i++)
118	177	INIT_HLIST_HEAD(tm->buckets + i);
119	178
	179	+ prefetch_init(&tm->prefetches);
	180	+
120	181	return tm;
121	182	}
122	183
@@ -268,8 +329,14 @@ int dm_tm_read_lock(struct dm_transaction_manager *tm, dm_block_t b, · 145 unmodified lines
268	329	struct dm_block_validator *v,
269	330	struct dm_block **blk)
270	331	{
271		− if (tm->is_clone)
272		− return dm_bm_read_try_lock(tm->real->bm, b, v, blk);
	332	+ if (tm->is_clone) {
	333	+ int r = dm_bm_read_try_lock(tm->real->bm, b, v, blk);
	334	+
	335	+ if (r == -EWOULDBLOCK)
	336	+ prefetch_add(&tm->real->prefetches, b);
	337	+
	338	+ return r;
	339	+ }
273	340
274	341	return dm_bm_read_lock(tm->bm, b, v, blk);
275	342	}
@@ -317,6 +384,12 @@ struct dm_block_manager dm_tm_get_bm(struct dm_transaction_manager tm) · 41 unmodified lines
317	384	return tm->bm;
318	385	}
319	386
	387	+void dm_tm_issue_prefetches(struct dm_transaction_manager *tm)
	388	+{
	389	+ prefetch_issue(&tm->prefetches, tm->bm);
	390	+}
	391	+EXPORT_SYMBOL_GPL(dm_tm_issue_prefetches);
	392	+
320	393	/----------------------------------------------------------------/
321	394
322	395	static int dm_tm_create_internal(struct dm_block_manager *bm,

drivers/md/persistent-data/dm-transaction-manager.h+7−0

@@ -108,6 +108,13 @@ int dm_tm_ref(struct dm_transaction_manager *tm, dm_block_t b, · 107 unmodified lines
108	108
109	109	struct dm_block_manager dm_tm_get_bm(struct dm_transaction_manager tm);
110	110
	111	+/*
	112	+ * If you're using a non-blocking clone the tm will build up a list of
	113	+ * requested blocks that weren't in core. This call will request those
	114	+ * blocks to be prefetched.
	115	+ */
	116	+void dm_tm_issue_prefetches(struct dm_transaction_manager *tm);
	117	+
111	118	/*
112	119	* A little utility that ties the knot by producing a transaction manager
113	120	* that has a space map managed by the transaction manager...

include/linux/device-mapper.h+2−0

@@ -64,6 +64,7 @@ typedef int (dm_request_endio_fn) (struct dm_target ti, · 63 unmodified lines
64	64	union map_info *map_context);
65	65
66	66	typedef void (dm_presuspend_fn) (struct dm_target ti);
	67	+typedef void (dm_presuspend_undo_fn) (struct dm_target ti);
67	68	typedef void (dm_postsuspend_fn) (struct dm_target ti);
68	69	typedef int (dm_preresume_fn) (struct dm_target ti);
69	70	typedef void (dm_resume_fn) (struct dm_target ti);
@@ -145,6 +146,7 @@ struct target_type { · 75 unmodified lines
145	146	dm_endio_fn end_io;
146	147	dm_request_endio_fn rq_end_io;
147	148	dm_presuspend_fn presuspend;
	149	+ dm_presuspend_undo_fn presuspend_undo;
148	150	dm_postsuspend_fn postsuspend;
149	151	dm_preresume_fn preresume;
150	152	dm_resume_fn resume;

include/uapi/linux/dm-ioctl.h+7−2

@@ -267,9 +267,9 @@ enum { · 266 unmodified lines
267	267	#define DM_DEV_SET_GEOMETRY _IOWR(DM_IOCTL, DM_DEV_SET_GEOMETRY_CMD, struct dm_ioctl)
268	268
269	269	#define DM_VERSION_MAJOR 4
270		−#define DM_VERSION_MINOR 28
	270	+#define DM_VERSION_MINOR 29
271	271	#define DM_VERSION_PATCHLEVEL 0
272		−#define DM_VERSION_EXTRA "-ioctl (2014-09-17)"
	272	+#define DM_VERSION_EXTRA "-ioctl (2014-10-28)"
273	273
274	274	/* Status bits */
275	275	#define DM_READONLY_FLAG (1 << 0) /* In/Out */
@@ -352,4 +352,9 @@ enum { · 76 unmodified lines
352	352	*/
353	353	#define DM_DEFERRED_REMOVE (1 << 17) /* In/Out */
354	354
	355	+/*
	356	+ * If set, the device is suspended internally.
	357	+ */
	358	+#define DM_INTERNAL_SUSPEND_FLAG (1 << 18) /* Out */
	359	+
355	360	#endif /* _LINUX_DM_IOCTL_H */

Merge tag 'dm-3.19-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code