Merge tag 'for-5.6/drivers-2020-01-27' of git://git.kernel.dk/linux-block

Pull block driver updates from Jens Axboe: "Like the core side, not a lot of changes here, just two main items: - Series of patches (via Coly) with fixes for bcache (Coly, Christoph) - MD pull request from Song" * tag 'for-5.6/drivers-2020-01-27' of git://git.kernel.dk/linux-block: (31 commits) bcache: reap from tail of c->btree_cache in bch_mca_scan() bcache: reap c->btree_cache_freeable from the tail in bch_mca_scan() bcache: remove member accessed from struct btree bcache: print written and keys in trace_bcache_btree_write bcache: avoid unnecessary btree nodes flushing in btree_flush_write() bcache: add code comments for state->pool in __btree_sort() lib: crc64: include <linux/crc64.h> for 'crc64_be' bcache: use read_cache_page_gfp to read the superblock bcache: store a pointer to the on-disk sb in the cache and cached_dev structures bcache: return a pointer to the on-disk sb from read_super bcache: transfer the sb_page reference to register_{bdev,cache} bcache: fix use-after-free in register_bcache() bcache: properly initialize 'path' and 'err' in register_bcache() bcache: rework error unwinding in register_bcache bcache: use a separate data structure for the on-disk super block bcache: cached_dev_free needs to put the sb page md/raid1: introduce wait_for_serialization md/raid1: use bucket based mechanism for IO serialization md: introduce a new struct for IO serialization md: don't destroy serial_info_pool if serialize_policy is true ...

Linus Torvalds committed 6 years ago

Commit 22a8f392 parents 48b4b4f e3de04417 files · +571 −262Browse files

Unified · Split

17 files

drivers/md/bcache/bcache.h +2 −0
drivers/md/bcache/bset.c +5 −0
drivers/md/bcache/btree.c +10 −14
drivers/md/bcache/btree.h +0 −2
drivers/md/bcache/journal.c +75 −5
drivers/md/bcache/super.c +71 −65
drivers/md/md-bitmap.c +12 −13
drivers/md/md.c +199 −55
drivers/md/md.h +27 −18
drivers/md/raid1.c +61 −50
drivers/md/raid5.c +7 −14
include/linux/raid/pq.h +6 −1
include/trace/events/bcache.h +2 −1
include/uapi/linux/bcache.h +52 −0
lib/crc64.c +1 −0
lib/raid6/algos.c +40 −23
lib/raid6/mktables.c +1 −1

drivers/md/bcache/bcache.h+2−0

@@ -301,6 +301,7 @@ struct cached_dev { · 300 unmodified lines
301	301	struct block_device *bdev;
302	302
303	303	struct cache_sb sb;
	304	+ struct cache_sb_disk *sb_disk;
304	305	struct bio sb_bio;
305	306	struct bio_vec sb_bv[1];
306	307	struct closure sb_write;
@@ -403,6 +404,7 @@ enum alloc_reserve { · 96 unmodified lines
403	404	struct cache {
404	405	struct cache_set *set;
405	406	struct cache_sb sb;
	407	+ struct cache_sb_disk *sb_disk;
406	408	struct bio sb_bio;
407	409	struct bio_vec sb_bv[1];
408	410

drivers/md/bcache/bset.c+5−0

@@ -1257,6 +1257,11 @@ static void __btree_sort(struct btree_keys b, struct btree_iter iter, · 1256 unmodified lines
1257	1257	* Our temporary buffer is the same size as the btree node's
1258	1258	* buffer, we can just swap buffers instead of doing a big
1259	1259	* memcpy()
	1260	+ *
	1261	+ * Don't worry event 'out' is allocated from mempool, it can
	1262	+ * still be swapped here. Because state->pool is a page mempool
	1263	+ * creaated by by mempool_init_page_pool(), which allocates
	1264	+ * pages by alloc_pages() indeed.
1260	1265	*/
1261	1266
1262	1267	out->magic = b->set->data->magic;

drivers/md/bcache/btree.c+10−14

@@ -734,34 +734,32 @@ static unsigned long bch_mca_scan(struct shrinker *shrink, · 733 unmodified lines
734	734
735	735	i = 0;
736	736	btree_cache_used = c->btree_cache_used;
737		− list_for_each_entry_safe(b, t, &c->btree_cache_freeable, list) {
	737	+ list_for_each_entry_safe_reverse(b, t, &c->btree_cache_freeable, list) {
738	738	if (nr <= 0)
739	739	goto out;
740	740
741		− if (++i > 3 &&
742		− !mca_reap(b, 0, false)) {
	741	+ if (!mca_reap(b, 0, false)) {
743	742	mca_data_free(b);
744	743	rw_unlock(true, b);
745	744	freed++;
746	745	}
747	746	nr--;
	747	+ i++;
748	748	}
749	749
750		− for (; (nr--) && i < btree_cache_used; i++) {
751		− if (list_empty(&c->btree_cache))
	750	+ list_for_each_entry_safe_reverse(b, t, &c->btree_cache, list) {
	751	+ if (nr <= 0 \|\| i >= btree_cache_used)
752	752	goto out;
753	753
754		− b = list_first_entry(&c->btree_cache, struct btree, list);
755		− list_rotate_left(&c->btree_cache);
756		−
757		− if (!b->accessed &&
758		− !mca_reap(b, 0, false)) {
	754	+ if (!mca_reap(b, 0, false)) {
759	755	mca_bucket_free(b);
760	756	mca_data_free(b);
761	757	rw_unlock(true, b);
762	758	freed++;
763		− } else
764		− b->accessed = 0;
	759	+ }
	760	+
	761	+ nr--;
	762	+ i++;
765	763	}
766	764	out:
767	765	mutex_unlock(&c->bucket_lock);
@@ -1069,7 +1067,6 @@ retry: · 301 unmodified lines
1069	1067	BUG_ON(!b->written);
1070	1068
1071	1069	b->parent = parent;
1072		− b->accessed = 1;
1073	1070
1074	1071	for (; i <= b->keys.nsets && b->keys.set[i].size; i++) {
1075	1072	prefetch(b->keys.set[i].tree);
@@ -1160,7 +1157,6 @@ retry: · 84 unmodified lines
1160	1157	goto retry;
1161	1158	}
1162	1159
1163		− b->accessed = 1;
1164	1160	b->parent = parent;
1165	1161	bch_bset_init_next(&b->keys, b->keys.set->data, bset_magic(&b->c->sb));
1166	1162

drivers/md/bcache/btree.h+0−2

@@ -121,8 +121,6 @@ struct btree { · 120 unmodified lines
121	121	/* Key/pointer for this btree node */
122	122	BKEY_PADDED(key);
123	123
124		− /* Single bit - set when accessed, cleared by shrinker */
125		− unsigned long accessed;
126	124	unsigned long seq;
127	125	struct rw_semaphore lock;
128	126	struct cache_set *c;

drivers/md/bcache/journal.c+75−5

@@ -417,10 +417,14 @@ err: · 416 unmodified lines
417	417
418	418	/* Journalling */
419	419
	420	+#define nr_to_fifo_front(p, front_p, mask) (((p) - (front_p)) & (mask))
	421	+
420	422	static void btree_flush_write(struct cache_set *c)
421	423	{
422	424	struct btree b, t, *btree_nodes[BTREE_FLUSH_NR];
423		− unsigned int i, n;
	425	+ unsigned int i, nr, ref_nr;
	426	+ atomic_t fifo_front_p, now_fifo_front_p;
	427	+ size_t mask;
424	428
425	429	if (c->journal.btree_flushing)
426	430	return;
@@ -433,12 +437,50 @@ static void btree_flush_write(struct cache_set *c) · 6 unmodified lines
433	437	c->journal.btree_flushing = true;
434	438	spin_unlock(&c->journal.flush_write_lock);
435	439
	440	+ /* get the oldest journal entry and check its refcount */
	441	+ spin_lock(&c->journal.lock);
	442	+ fifo_front_p = &fifo_front(&c->journal.pin);
	443	+ ref_nr = atomic_read(fifo_front_p);
	444	+ if (ref_nr <= 0) {
	445	+ /*
	446	+ * do nothing if no btree node references
	447	+ * the oldest journal entry
	448	+ */
	449	+ spin_unlock(&c->journal.lock);
	450	+ goto out;
	451	+ }
	452	+ spin_unlock(&c->journal.lock);
	453	+
	454	+ mask = c->journal.pin.mask;
	455	+ nr = 0;
436	456	atomic_long_inc(&c->flush_write);
437	457	memset(btree_nodes, 0, sizeof(btree_nodes));
438		− n = 0;
439	458
440	459	mutex_lock(&c->bucket_lock);
441	460	list_for_each_entry_safe_reverse(b, t, &c->btree_cache, list) {
	461	+ /*
	462	+ * It is safe to get now_fifo_front_p without holding
	463	+ * c->journal.lock here, because we don't need to know
	464	+ * the exactly accurate value, just check whether the
	465	+ * front pointer of c->journal.pin is changed.
	466	+ */
	467	+ now_fifo_front_p = &fifo_front(&c->journal.pin);
	468	+ /*
	469	+ * If the oldest journal entry is reclaimed and front
	470	+ * pointer of c->journal.pin changes, it is unnecessary
	471	+ * to scan c->btree_cache anymore, just quit the loop and
	472	+ * flush out what we have already.
	473	+ */
	474	+ if (now_fifo_front_p != fifo_front_p)
	475	+ break;
	476	+ /*
	477	+ * quit this loop if all matching btree nodes are
	478	+ * scanned and record in btree_nodes[] already.
	479	+ */
	480	+ ref_nr = atomic_read(fifo_front_p);
	481	+ if (nr >= ref_nr)
	482	+ break;
	483	+
442	484	if (btree_node_journal_flush(b))
443	485	pr_err("BUG: flush_write bit should not be set here!");
444	486
@@ -454,17 +496,44 @@ static void btree_flush_write(struct cache_set *c) · 9 unmodified lines
454	496	continue;
455	497	}
456	498
	499	+ /*
	500	+ * Only select the btree node which exactly references
	501	+ * the oldest journal entry.
	502	+ *
	503	+ * If the journal entry pointed by fifo_front_p is
	504	+ * reclaimed in parallel, don't worry:
	505	+ * - the list_for_each_xxx loop will quit when checking
	506	+ * next now_fifo_front_p.
	507	+ * - If there are matched nodes recorded in btree_nodes[],
	508	+ * they are clean now (this is why and how the oldest
	509	+ * journal entry can be reclaimed). These selected nodes
	510	+ * will be ignored and skipped in the folowing for-loop.
	511	+ */
	512	+ if (nr_to_fifo_front(btree_current_write(b)->journal,
	513	+ fifo_front_p,
	514	+ mask) != 0) {
	515	+ mutex_unlock(&b->write_lock);
	516	+ continue;
	517	+ }
	518	+
457	519	set_btree_node_journal_flush(b);
458	520
459	521	mutex_unlock(&b->write_lock);
460	522
461		− btree_nodes[n++] = b;
462		− if (n == BTREE_FLUSH_NR)
	523	+ btree_nodes[nr++] = b;
	524	+ /*
	525	+ * To avoid holding c->bucket_lock too long time,
	526	+ * only scan for BTREE_FLUSH_NR matched btree nodes
	527	+ * at most. If there are more btree nodes reference
	528	+ * the oldest journal entry, try to flush them next
	529	+ * time when btree_flush_write() is called.
	530	+ */
	531	+ if (nr == BTREE_FLUSH_NR)
463	532	break;
464	533	}
465	534	mutex_unlock(&c->bucket_lock);
466	535
467		− for (i = 0; i < n; i++) {
	536	+ for (i = 0; i < nr; i++) {
468	537	b = btree_nodes[i];
469	538	if (!b) {
470	539	pr_err("BUG: btree_nodes[%d] is NULL", i);
@@ -497,6 +566,7 @@ static void btree_flush_write(struct cache_set *c) · 26 unmodified lines
497	566	mutex_unlock(&b->write_lock);
498	567	}
499	568
	569	+out:
500	570	spin_lock(&c->journal.flush_write_lock);
501	571	c->journal.btree_flushing = false;
502	572	spin_unlock(&c->journal.flush_write_lock);

drivers/md/bcache/super.c+71−65

@@ -15,7 +15,6 @@ · 14 unmodified lines
15	15	#include "writeback.h"
16	16
17	17	#include <linux/blkdev.h>
18		−#include <linux/buffer_head.h>
19	18	#include <linux/debugfs.h>
20	19	#include <linux/genhd.h>
21	20	#include <linux/idr.h>
@@ -60,17 +59,18 @@ struct workqueue_struct *bch_journal_wq; · 38 unmodified lines
60	59	/* Superblock */
61	60
62	61	static const char read_super(struct cache_sb sb, struct block_device *bdev,
63		− struct page **res)
	62	+ struct cache_sb_disk **res)
64	63	{
65	64	const char *err;
66		− struct cache_sb *s;
67		− struct buffer_head *bh = __bread(bdev, 1, SB_SIZE);
	65	+ struct cache_sb_disk *s;
	66	+ struct page *page;
68	67	unsigned int i;
69	68
70		− if (!bh)
	69	+ page = read_cache_page_gfp(bdev->bd_inode->i_mapping,
	70	+ SB_OFFSET >> PAGE_SHIFT, GFP_KERNEL);
	71	+ if (IS_ERR(page))
71	72	return "IO error";
72		−
73		− s = (struct cache_sb *) bh->b_data;
	73	+ s = page_address(page) + offset_in_page(SB_OFFSET);
74	74
75	75	sb->offset = le64_to_cpu(s->offset);
76	76	sb->version = le64_to_cpu(s->version);
@@ -188,12 +188,10 @@ static const char read_super(struct cache_sb sb, struct block_device *bdev, · 111 unmodified lines
188	188	}
189	189
190	190	sb->last_mount = (u32)ktime_get_real_seconds();
191		− err = NULL;
192		−
193		− get_page(bh->b_page);
194		− *res = bh->b_page;
	191	+ *res = s;
	192	+ return NULL;
195	193	err:
196		− put_bh(bh);
	194	+ put_page(page);
197	195	return err;
198	196	}
199	197
@@ -207,15 +205,15 @@ static void write_bdev_super_endio(struct bio *bio) · 7 unmodified lines
207	205	closure_put(&dc->sb_write);
208	206	}
209	207
210		−static void __write_super(struct cache_sb sb, struct bio bio)
	208	+static void __write_super(struct cache_sb sb, struct cache_sb_disk out,
	209	+ struct bio *bio)
211	210	{
212		− struct cache_sb *out = page_address(bio_first_page_all(bio));
213	211	unsigned int i;
214	212
	213	+ bio->bi_opf = REQ_OP_WRITE \| REQ_SYNC \| REQ_META;
215	214	bio->bi_iter.bi_sector = SB_SECTOR;
216		− bio->bi_iter.bi_size = SB_SIZE;
217		− bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_SYNC\|REQ_META);
218		− bch_bio_map(bio, NULL);
	215	+ __bio_add_page(bio, virt_to_page(out), SB_SIZE,
	216	+ offset_in_page(out));
219	217
220	218	out->offset = cpu_to_le64(sb->offset);
221	219	out->version = cpu_to_le64(sb->version);
@@ -257,14 +255,14 @@ void bch_write_bdev_super(struct cached_dev dc, struct closure parent) · 35 unmodified lines
257	255	down(&dc->sb_write_mutex);
258	256	closure_init(cl, parent);
259	257
260		− bio_reset(bio);
	258	+ bio_init(bio, dc->sb_bv, 1);
261	259	bio_set_dev(bio, dc->bdev);
262	260	bio->bi_end_io = write_bdev_super_endio;
263	261	bio->bi_private = dc;
264	262
265	263	closure_get(cl);
266	264	/* I/O request sent to backing device */
267		− __write_super(&dc->sb, bio);
	265	+ __write_super(&dc->sb, dc->sb_disk, bio);
268	266
269	267	closure_return_with_destructor(cl, bch_write_bdev_super_unlock);
270	268	}
@@ -306,13 +304,13 @@ void bcache_write_super(struct cache_set *c) · 35 unmodified lines
306	304
307	305	SET_CACHE_SYNC(&ca->sb, CACHE_SYNC(&c->sb));
308	306
309		− bio_reset(bio);
	307	+ bio_init(bio, ca->sb_bv, 1);
310	308	bio_set_dev(bio, ca->bdev);
311	309	bio->bi_end_io = write_super_endio;
312	310	bio->bi_private = ca;
313	311
314	312	closure_get(cl);
315		− __write_super(&ca->sb, bio);
	313	+ __write_super(&ca->sb, ca->sb_disk, bio);
316	314	}
317	315
318	316	closure_return_with_destructor(cl, bcache_write_super_unlock);
@@ -1275,6 +1273,9 @@ static void cached_dev_free(struct closure *cl) · 956 unmodified lines
1275	1273
1276	1274	mutex_unlock(&bch_register_lock);
1277	1275
	1276	+ if (dc->sb_disk)
	1277	+ put_page(virt_to_page(dc->sb_disk));
	1278	+
1278	1279	if (!IS_ERR_OR_NULL(dc->bdev))
1279	1280	blkdev_put(dc->bdev, FMODE_READ\|FMODE_WRITE\|FMODE_EXCL);
1280	1281
@@ -1350,7 +1351,7 @@ static int cached_dev_init(struct cached_dev *dc, unsigned int block_size) · 69 unmodified lines
1350	1351
1351	1352	/* Cached device - bcache superblock */
1352	1353
1353		−static int register_bdev(struct cache_sb sb, struct page sb_page,
	1354	+static int register_bdev(struct cache_sb sb, struct cache_sb_disk sb_disk,
1354	1355	struct block_device *bdev,
1355	1356	struct cached_dev *dc)
1356	1357	{
@@ -1362,11 +1363,7 @@ static int register_bdev(struct cache_sb sb, struct page sb_page, · 5 unmodified lines
1362	1363	memcpy(&dc->sb, sb, sizeof(struct cache_sb));
1363	1364	dc->bdev = bdev;
1364	1365	dc->bdev->bd_holder = dc;
1365		−
1366		− bio_init(&dc->sb_bio, dc->sb_bio.bi_inline_vecs, 1);
1367		− bio_first_bvec_all(&dc->sb_bio)->bv_page = sb_page;
1368		− get_page(sb_page);
1369		−
	1366	+ dc->sb_disk = sb_disk;
1370	1367
1371	1368	if (cached_dev_init(dc, sb->block_size << 9))
1372	1369	goto err;
@@ -2136,8 +2133,8 @@ void bch_cache_release(struct kobject *kobj) · 763 unmodified lines
2136	2133	for (i = 0; i < RESERVE_NR; i++)
2137	2134	free_fifo(&ca->free[i]);
2138	2135
2139		− if (ca->sb_bio.bi_inline_vecs[0].bv_page)
2140		− put_page(bio_first_page_all(&ca->sb_bio));
	2136	+ if (ca->sb_disk)
	2137	+ put_page(virt_to_page(ca->sb_disk));
2141	2138
2142	2139	if (!IS_ERR_OR_NULL(ca->bdev))
2143	2140	blkdev_put(ca->bdev, FMODE_READ\|FMODE_WRITE\|FMODE_EXCL);
@@ -2259,7 +2256,7 @@ err_free: · 115 unmodified lines
2259	2256	return ret;
2260	2257	}
2261	2258
2262		−static int register_cache(struct cache_sb sb, struct page sb_page,
	2259	+static int register_cache(struct cache_sb sb, struct cache_sb_disk sb_disk,
2263	2260	struct block_device bdev, struct cache ca)
2264	2261	{
2265	2262	const char err = NULL; / must be set for any error case */
@@ -2269,10 +2266,7 @@ static int register_cache(struct cache_sb sb, struct page sb_page, · 3 unmodified lines
2269	2266	memcpy(&ca->sb, sb, sizeof(struct cache_sb));
2270	2267	ca->bdev = bdev;
2271	2268	ca->bdev->bd_holder = ca;
2272		−
2273		− bio_init(&ca->sb_bio, ca->sb_bio.bi_inline_vecs, 1);
2274		− bio_first_bvec_all(&ca->sb_bio)->bv_page = sb_page;
2275		− get_page(sb_page);
	2269	+ ca->sb_disk = sb_disk;
2276	2270
2277	2271	if (blk_queue_discard(bdev_get_queue(bdev)))
2278	2272	ca->discard = CACHE_DISCARD(&ca->sb);
@@ -2372,29 +2366,35 @@ static bool bch_is_open(struct block_device *bdev) · 93 unmodified lines
2372	2366	static ssize_t register_bcache(struct kobject k, struct kobj_attribute attr,
2373	2367	const char *buffer, size_t size)
2374	2368	{
2375		− ssize_t ret = -EINVAL;
2376		− const char *err = "cannot allocate memory";
	2369	+ const char *err;
2377	2370	char *path = NULL;
2378		− struct cache_sb *sb = NULL;
2379		− struct block_device *bdev = NULL;
2380		− struct page *sb_page = NULL;
	2371	+ struct cache_sb *sb;
	2372	+ struct cache_sb_disk *sb_disk;
	2373	+ struct block_device *bdev;
	2374	+ ssize_t ret;
2381	2375
	2376	+ ret = -EBUSY;
	2377	+ err = "failed to reference bcache module";
2382	2378	if (!try_module_get(THIS_MODULE))
2383		− return -EBUSY;
	2379	+ goto out;
2384	2380
2385	2381	/* For latest state of bcache_is_reboot */
2386	2382	smp_mb();
	2383	+ err = "bcache is in reboot";
2387	2384	if (bcache_is_reboot)
2388		− return -EBUSY;
	2385	+ goto out_module_put;
2389	2386
	2387	+ ret = -ENOMEM;
	2388	+ err = "cannot allocate memory";
2390	2389	path = kstrndup(buffer, size, GFP_KERNEL);
2391	2390	if (!path)
2392		− goto err;
	2391	+ goto out_module_put;
2393	2392
2394	2393	sb = kmalloc(sizeof(struct cache_sb), GFP_KERNEL);
2395	2394	if (!sb)
2396		− goto err;
	2395	+ goto out_free_path;
2397	2396
	2397	+ ret = -EINVAL;
2398	2398	err = "failed to open device";
2399	2399	bdev = blkdev_get_by_path(strim(path),
2400	2400	FMODE_READ\|FMODE_WRITE\|FMODE_EXCL,
@@ -2411,57 +2411,63 @@ static ssize_t register_bcache(struct kobject k, struct kobj_attribute attr, · 10 unmodified lines
2411	2411	if (!IS_ERR(bdev))
2412	2412	bdput(bdev);
2413	2413	if (attr == &ksysfs_register_quiet)
2414		− goto quiet_out;
	2414	+ goto done;
2415	2415	}
2416		− goto err;
	2416	+ goto out_free_sb;
2417	2417	}
2418	2418
2419	2419	err = "failed to set blocksize";
2420	2420	if (set_blocksize(bdev, 4096))
2421		− goto err_close;
	2421	+ goto out_blkdev_put;
2422	2422
2423		− err = read_super(sb, bdev, &sb_page);
	2423	+ err = read_super(sb, bdev, &sb_disk);
2424	2424	if (err)
2425		− goto err_close;
	2425	+ goto out_blkdev_put;
2426	2426
2427	2427	err = "failed to register device";
2428	2428	if (SB_IS_BDEV(sb)) {
2429	2429	struct cached_dev dc = kzalloc(sizeof(dc), GFP_KERNEL);
2430	2430
2431	2431	if (!dc)
2432		− goto err_close;
	2432	+ goto out_put_sb_page;
2433	2433
2434	2434	mutex_lock(&bch_register_lock);
2435		− ret = register_bdev(sb, sb_page, bdev, dc);
	2435	+ ret = register_bdev(sb, sb_disk, bdev, dc);
2436	2436	mutex_unlock(&bch_register_lock);
2437	2437	/* blkdev_put() will be called in cached_dev_free() */
2438	2438	if (ret < 0)
2439		− goto err;
	2439	+ goto out_free_sb;
2440	2440	} else {
2441	2441	struct cache ca = kzalloc(sizeof(ca), GFP_KERNEL);
2442	2442
2443	2443	if (!ca)
2444		− goto err_close;
	2444	+ goto out_put_sb_page;
2445	2445
2446	2446	/* blkdev_put() will be called in bch_cache_release() */
2447		− if (register_cache(sb, sb_page, bdev, ca) != 0)
2448		− goto err;
	2447	+ if (register_cache(sb, sb_disk, bdev, ca) != 0)
	2448	+ goto out_free_sb;
2449	2449	}
2450		−quiet_out:
2451		− ret = size;
2452		−out:
2453		− if (sb_page)
2454		− put_page(sb_page);
	2450	+
	2451	+done:
2455	2452	kfree(sb);
2456	2453	kfree(path);
2457	2454	module_put(THIS_MODULE);
2458		− return ret;
	2455	+ return size;
2459	2456
2460		−err_close:
2461		− blkdev_put(bdev, FMODE_READ\|FMODE_WRITE\|FMODE_EXCL);
2462		−err:
2463		− pr_info("error %s: %s", path, err);
2464		− goto out;
	2457	+out_put_sb_page:
	2458	+ put_page(virt_to_page(sb_disk));
	2459	+out_blkdev_put:
	2460	+ blkdev_put(bdev, FMODE_READ \| FMODE_WRITE \| FMODE_EXCL);
	2461	+out_free_sb:
	2462	+ kfree(sb);
	2463	+out_free_path:
	2464	+ kfree(path);
	2465	+ path = NULL;
	2466	+out_module_put:
	2467	+ module_put(THIS_MODULE);
	2468	+out:
	2469	+ pr_info("error %s: %s", path?path:"", err);
	2470	+ return ret;
2465	2471	}
2466	2472
2467	2473

drivers/md/md-bitmap.c+12−13

@@ -1019,8 +1019,6 @@ void md_bitmap_unplug(struct bitmap *bitmap) · 1018 unmodified lines
1019	1019	/* look at each page to see if there are any set bits that need to be
1020	1020	* flushed out to disk */
1021	1021	for (i = 0; i < bitmap->storage.file_pages; i++) {
1022		− if (!bitmap->storage.filemap)
1023		− return;
1024	1022	dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
1025	1023	need_write = test_and_clear_page_attr(bitmap, i,
1026	1024	BITMAP_PAGE_NEEDWRITE);
@@ -1338,7 +1336,8 @@ void md_bitmap_daemon_work(struct mddev *mddev) · 311 unmodified lines
1338	1336	BITMAP_PAGE_DIRTY))
1339	1337	/* bitmap_unplug will handle the rest */
1340	1338	break;
1341		− if (test_and_clear_page_attr(bitmap, j,
	1339	+ if (bitmap->storage.filemap &&
	1340	+ test_and_clear_page_attr(bitmap, j,
1342	1341	BITMAP_PAGE_NEEDWRITE)) {
1343	1342	write_page(bitmap, bitmap->storage.filemap[j], 0);
1344	1343	}
@@ -1790,8 +1789,8 @@ void md_bitmap_destroy(struct mddev *mddev) · 445 unmodified lines
1790	1789	return;
1791	1790
1792	1791	md_bitmap_wait_behind_writes(mddev);
1793		− mempool_destroy(mddev->wb_info_pool);
1794		− mddev->wb_info_pool = NULL;
	1792	+ if (!mddev->serialize_policy)
	1793	+ mddev_destroy_serial_pool(mddev, NULL, true);
1795	1794
1796	1795	mutex_lock(&mddev->bitmap_info.mutex);
1797	1796	spin_lock(&mddev->lock);
@@ -1908,7 +1907,7 @@ int md_bitmap_load(struct mddev *mddev) · 110 unmodified lines
1908	1907	goto out;
1909	1908
1910	1909	rdev_for_each(rdev, mddev)
1911		− mddev_create_wb_pool(mddev, rdev, true);
	1910	+ mddev_create_serial_pool(mddev, rdev, true);
1912	1911
1913	1912	if (mddev_is_clustered(mddev))
1914	1913	md_cluster_ops->load_bitmaps(mddev, mddev->bitmap_info.nodes);
@@ -2475,16 +2474,16 @@ backlog_store(struct mddev mddev, const char buf, size_t len) · 560 unmodified lines
2475	2474	if (backlog > COUNTER_MAX)
2476	2475	return -EINVAL;
2477	2476	mddev->bitmap_info.max_write_behind = backlog;
2478		− if (!backlog && mddev->wb_info_pool) {
2479		− /* wb_info_pool is not needed if backlog is zero */
2480		− mempool_destroy(mddev->wb_info_pool);
2481		− mddev->wb_info_pool = NULL;
2482		− } else if (backlog && !mddev->wb_info_pool) {
2483		− /* wb_info_pool is needed since backlog is not zero */
	2477	+ if (!backlog && mddev->serial_info_pool) {
	2478	+ /* serial_info_pool is not needed if backlog is zero */
	2479	+ if (!mddev->serialize_policy)
	2480	+ mddev_destroy_serial_pool(mddev, NULL, false);
	2481	+ } else if (backlog && !mddev->serial_info_pool) {
	2482	+ /* serial_info_pool is needed since backlog is not zero */
2484	2483	struct md_rdev *rdev;
2485	2484
2486	2485	rdev_for_each(rdev, mddev)
2487		− mddev_create_wb_pool(mddev, rdev, false);
	2486	+ mddev_create_serial_pool(mddev, rdev, false);
2488	2487	}
2489	2488	if (old_mwb != backlog)
2490	2489	md_bitmap_update_sb(mddev->bitmap);

drivers/md/md.c+199−55

@@ -125,74 +125,165 @@ static inline int speed_max(struct mddev *mddev) · 124 unmodified lines
125	125	mddev->sync_speed_max : sysctl_speed_limit_max;
126	126	}
127	127
128		−static int rdev_init_wb(struct md_rdev *rdev)
	128	+static void rdev_uninit_serial(struct md_rdev *rdev)
129	129	{
130		− if (rdev->bdev->bd_queue->nr_hw_queues == 1)
	130	+ if (!test_and_clear_bit(CollisionCheck, &rdev->flags))
	131	+ return;
	132	+
	133	+ kvfree(rdev->serial);
	134	+ rdev->serial = NULL;
	135	+}
	136	+
	137	+static void rdevs_uninit_serial(struct mddev *mddev)
	138	+{
	139	+ struct md_rdev *rdev;
	140	+
	141	+ rdev_for_each(rdev, mddev)
	142	+ rdev_uninit_serial(rdev);
	143	+}
	144	+
	145	+static int rdev_init_serial(struct md_rdev *rdev)
	146	+{
	147	+ /* serial_nums equals with BARRIER_BUCKETS_NR */
	148	+ int i, serial_nums = 1 << ((PAGE_SHIFT - ilog2(sizeof(atomic_t))));
	149	+ struct serial_in_rdev *serial = NULL;
	150	+
	151	+ if (test_bit(CollisionCheck, &rdev->flags))
131	152	return 0;
132	153
133		− spin_lock_init(&rdev->wb_list_lock);
134		− INIT_LIST_HEAD(&rdev->wb_list);
135		− init_waitqueue_head(&rdev->wb_io_wait);
136		− set_bit(WBCollisionCheck, &rdev->flags);
	154	+ serial = kvmalloc(sizeof(struct serial_in_rdev) * serial_nums,
	155	+ GFP_KERNEL);
	156	+ if (!serial)
	157	+ return -ENOMEM;
137	158
138		− return 1;
	159	+ for (i = 0; i < serial_nums; i++) {
	160	+ struct serial_in_rdev *serial_tmp = &serial[i];
	161	+
	162	+ spin_lock_init(&serial_tmp->serial_lock);
	163	+ serial_tmp->serial_rb = RB_ROOT_CACHED;
	164	+ init_waitqueue_head(&serial_tmp->serial_io_wait);
	165	+ }
	166	+
	167	+ rdev->serial = serial;
	168	+ set_bit(CollisionCheck, &rdev->flags);
	169	+
	170	+ return 0;
	171	+}
	172	+
	173	+static int rdevs_init_serial(struct mddev *mddev)
	174	+{
	175	+ struct md_rdev *rdev;
	176	+ int ret = 0;
	177	+
	178	+ rdev_for_each(rdev, mddev) {
	179	+ ret = rdev_init_serial(rdev);
	180	+ if (ret)
	181	+ break;
	182	+ }
	183	+
	184	+ /* Free all resources if pool is not existed */
	185	+ if (ret && !mddev->serial_info_pool)
	186	+ rdevs_uninit_serial(mddev);
	187	+
	188	+ return ret;
139	189	}
140	190
141	191	/*
142		− * Create wb_info_pool if rdev is the first multi-queue device flaged
143		− * with writemostly, also write-behind mode is enabled.
	192	+ * rdev needs to enable serial stuffs if it meets the conditions:
	193	+ * 1. it is multi-queue device flaged with writemostly.
	194	+ * 2. the write-behind mode is enabled.
144	195	*/
145		−void mddev_create_wb_pool(struct mddev mddev, struct md_rdev rdev,
146		− bool is_suspend)
	196	+static int rdev_need_serial(struct md_rdev *rdev)
147	197	{
148		− if (mddev->bitmap_info.max_write_behind == 0)
149		− return;
	198	+ return (rdev && rdev->mddev->bitmap_info.max_write_behind > 0 &&
	199	+ rdev->bdev->bd_queue->nr_hw_queues != 1 &&
	200	+ test_bit(WriteMostly, &rdev->flags));
	201	+}
	202	+
	203	+/*
	204	+ * Init resource for rdev(s), then create serial_info_pool if:
	205	+ * 1. rdev is the first device which return true from rdev_enable_serial.
	206	+ * 2. rdev is NULL, means we want to enable serialization for all rdevs.
	207	+ */
	208	+void mddev_create_serial_pool(struct mddev mddev, struct md_rdev rdev,
	209	+ bool is_suspend)
	210	+{
	211	+ int ret = 0;
150	212
151		− if (!test_bit(WriteMostly, &rdev->flags) \|\| !rdev_init_wb(rdev))
	213	+ if (rdev && !rdev_need_serial(rdev) &&
	214	+ !test_bit(CollisionCheck, &rdev->flags))
152	215	return;
153	216
154		− if (mddev->wb_info_pool == NULL) {
	217	+ if (!is_suspend)
	218	+ mddev_suspend(mddev);
	219	+
	220	+ if (!rdev)
	221	+ ret = rdevs_init_serial(mddev);
	222	+ else
	223	+ ret = rdev_init_serial(rdev);
	224	+ if (ret)
	225	+ goto abort;
	226	+
	227	+ if (mddev->serial_info_pool == NULL) {
155	228	unsigned int noio_flag;
156	229
157		− if (!is_suspend)
158		− mddev_suspend(mddev);
159	230	noio_flag = memalloc_noio_save();
160		− mddev->wb_info_pool = mempool_create_kmalloc_pool(NR_WB_INFOS,
161		− sizeof(struct wb_info));
	231	+ mddev->serial_info_pool =
	232	+ mempool_create_kmalloc_pool(NR_SERIAL_INFOS,
	233	+ sizeof(struct serial_info));
162	234	memalloc_noio_restore(noio_flag);
163		− if (!mddev->wb_info_pool)
164		− pr_err("can't alloc memory pool for writemostly\n");
165		− if (!is_suspend)
166		− mddev_resume(mddev);
	235	+ if (!mddev->serial_info_pool) {
	236	+ rdevs_uninit_serial(mddev);
	237	+ pr_err("can't alloc memory pool for serialization\n");
	238	+ }
167	239	}
	240	+
	241	+abort:
	242	+ if (!is_suspend)
	243	+ mddev_resume(mddev);
168	244	}
169		−EXPORT_SYMBOL_GPL(mddev_create_wb_pool);
170	245
171	246	/*
172		− * destroy wb_info_pool if rdev is the last device flaged with WBCollisionCheck.
	247	+ * Free resource from rdev(s), and destroy serial_info_pool under conditions:
	248	+ * 1. rdev is the last device flaged with CollisionCheck.
	249	+ * 2. when bitmap is destroyed while policy is not enabled.
	250	+ * 3. for disable policy, the pool is destroyed only when no rdev needs it.
173	251	*/
174		−static void mddev_destroy_wb_pool(struct mddev mddev, struct md_rdev rdev)
	252	+void mddev_destroy_serial_pool(struct mddev mddev, struct md_rdev rdev,
	253	+ bool is_suspend)
175	254	{
176		− if (!test_and_clear_bit(WBCollisionCheck, &rdev->flags))
	255	+ if (rdev && !test_bit(CollisionCheck, &rdev->flags))
177	256	return;
178	257
179		− if (mddev->wb_info_pool) {
	258	+ if (mddev->serial_info_pool) {
180	259	struct md_rdev *temp;
181		− int num = 0;
	260	+ int num = 0; /* used to track if other rdevs need the pool */
182	261
183		− /*
184		− * Check if other rdevs need wb_info_pool.
185		− */
186		− rdev_for_each(temp, mddev)
187		− if (temp != rdev &&
188		− test_bit(WBCollisionCheck, &temp->flags))
	262	+ if (!is_suspend)
	263	+ mddev_suspend(mddev);
	264	+ rdev_for_each(temp, mddev) {
	265	+ if (!rdev) {
	266	+ if (!mddev->serialize_policy \|\|
	267	+ !rdev_need_serial(temp))
	268	+ rdev_uninit_serial(temp);
	269	+ else
	270	+ num++;
	271	+ } else if (temp != rdev &&
	272	+ test_bit(CollisionCheck, &temp->flags))
189	273	num++;
190		− if (!num) {
191		− mddev_suspend(rdev->mddev);
192		− mempool_destroy(mddev->wb_info_pool);
193		− mddev->wb_info_pool = NULL;
194		− mddev_resume(rdev->mddev);
195	274	}
	275	+
	276	+ if (rdev)
	277	+ rdev_uninit_serial(rdev);
	278	+
	279	+ if (num)
	280	+ pr_info("The mempool could be used by other devices\n");
	281	+ else {
	282	+ mempool_destroy(mddev->serial_info_pool);
	283	+ mddev->serial_info_pool = NULL;
	284	+ }
	285	+ if (!is_suspend)
	286	+ mddev_resume(mddev);
196	287	}
197	288	}
198	289
@@ -2337,7 +2428,7 @@ static int bind_rdev_to_array(struct md_rdev rdev, struct mddev mddev) · 2138 unmodified lines
2337	2428	pr_debug("md: bind<%s>\n", b);
2338	2429
2339	2430	if (mddev->raid_disks)
2340		− mddev_create_wb_pool(mddev, rdev, false);
	2431	+ mddev_create_serial_pool(mddev, rdev, false);
2341	2432
2342	2433	if ((err = kobject_add(&rdev->kobj, &mddev->kobj, "dev-%s", b)))
2343	2434	goto fail;
@@ -2375,7 +2466,7 @@ static void unbind_rdev_from_array(struct md_rdev *rdev) · 31 unmodified lines
2375	2466	bd_unlink_disk_holder(rdev->bdev, rdev->mddev->gendisk);
2376	2467	list_del_rcu(&rdev->same_set);
2377	2468	pr_debug("md: unbind<%s>\n", bdevname(rdev->bdev,b));
2378		− mddev_destroy_wb_pool(rdev->mddev, rdev);
	2469	+ mddev_destroy_serial_pool(rdev->mddev, rdev, false);
2379	2470	rdev->mddev = NULL;
2380	2471	sysfs_remove_link(&rdev->kobj, "block");
2381	2472	sysfs_put(rdev->sysfs_state);
@@ -2888,10 +2979,10 @@ state_store(struct md_rdev rdev, const char buf, size_t len) · 506 unmodified lines
2888	2979	}
2889	2980	} else if (cmd_match(buf, "writemostly")) {
2890	2981	set_bit(WriteMostly, &rdev->flags);
2891		− mddev_create_wb_pool(rdev->mddev, rdev, false);
	2982	+ mddev_create_serial_pool(rdev->mddev, rdev, false);
2892	2983	err = 0;
2893	2984	} else if (cmd_match(buf, "-writemostly")) {
2894		− mddev_destroy_wb_pool(rdev->mddev, rdev);
	2985	+ mddev_destroy_serial_pool(rdev->mddev, rdev, false);
2895	2986	clear_bit(WriteMostly, &rdev->flags);
2896	2987	err = 0;
2897	2988	} else if (cmd_match(buf, "blocked")) {
@@ -5277,6 +5368,57 @@ static struct md_sysfs_entry md_fail_last_dev = · 2379 unmodified lines
5277	5368	__ATTR(fail_last_dev, S_IRUGO \| S_IWUSR, fail_last_dev_show,
5278	5369	fail_last_dev_store);
5279	5370
	5371	+static ssize_t serialize_policy_show(struct mddev mddev, char page)
	5372	+{
	5373	+ if (mddev->pers == NULL \|\| (mddev->pers->level != 1))
	5374	+ return sprintf(page, "n/a\n");
	5375	+ else
	5376	+ return sprintf(page, "%d\n", mddev->serialize_policy);
	5377	+}
	5378	+
	5379	+/*
	5380	+ * Setting serialize_policy to true to enforce write IO is not reordered
	5381	+ * for raid1.
	5382	+ */
	5383	+static ssize_t
	5384	+serialize_policy_store(struct mddev mddev, const char buf, size_t len)
	5385	+{
	5386	+ int err;
	5387	+ bool value;
	5388	+
	5389	+ err = kstrtobool(buf, &value);
	5390	+ if (err)
	5391	+ return err;
	5392	+
	5393	+ if (value == mddev->serialize_policy)
	5394	+ return len;
	5395	+
	5396	+ err = mddev_lock(mddev);
	5397	+ if (err)
	5398	+ return err;
	5399	+ if (mddev->pers == NULL \|\| (mddev->pers->level != 1)) {
	5400	+ pr_err("md: serialize_policy is only effective for raid1\n");
	5401	+ err = -EINVAL;
	5402	+ goto unlock;
	5403	+ }
	5404	+
	5405	+ mddev_suspend(mddev);
	5406	+ if (value)
	5407	+ mddev_create_serial_pool(mddev, NULL, true);
	5408	+ else
	5409	+ mddev_destroy_serial_pool(mddev, NULL, true);
	5410	+ mddev->serialize_policy = value;
	5411	+ mddev_resume(mddev);
	5412	+unlock:
	5413	+ mddev_unlock(mddev);
	5414	+ return err ?: len;
	5415	+}
	5416	+
	5417	+static struct md_sysfs_entry md_serialize_policy =
	5418	+__ATTR(serialize_policy, S_IRUGO \| S_IWUSR, serialize_policy_show,
	5419	+ serialize_policy_store);
	5420	+
	5421	+
5280	5422	static struct attribute *md_default_attrs[] = {
5281	5423	&md_level.attr,
5282	5424	&md_layout.attr,
@@ -5294,6 +5436,7 @@ static struct attribute *md_default_attrs[] = { · 11 unmodified lines
5294	5436	&max_corr_read_errors.attr,
5295	5437	&md_consistency_policy.attr,
5296	5438	&md_fail_last_dev.attr,
	5439	+ &md_serialize_policy.attr,
5297	5440	NULL,
5298	5441	};
5299	5442
@@ -5769,18 +5912,18 @@ int md_run(struct mddev *mddev) · 469 unmodified lines
5769	5912	goto bitmap_abort;
5770	5913
5771	5914	if (mddev->bitmap_info.max_write_behind > 0) {
5772		− bool creat_pool = false;
	5915	+ bool create_pool = false;
5773	5916
5774	5917	rdev_for_each(rdev, mddev) {
5775	5918	if (test_bit(WriteMostly, &rdev->flags) &&
5776		− rdev_init_wb(rdev))
5777		− creat_pool = true;
5778		− }
5779		− if (creat_pool && mddev->wb_info_pool == NULL) {
5780		− mddev->wb_info_pool =
5781		− mempool_create_kmalloc_pool(NR_WB_INFOS,
5782		− sizeof(struct wb_info));
5783		− if (!mddev->wb_info_pool) {
	5919	+ rdev_init_serial(rdev))
	5920	+ create_pool = true;
	5921	+ }
	5922	+ if (create_pool && mddev->serial_info_pool == NULL) {
	5923	+ mddev->serial_info_pool =
	5924	+ mempool_create_kmalloc_pool(NR_SERIAL_INFOS,
	5925	+ sizeof(struct serial_info));
	5926	+ if (!mddev->serial_info_pool) {
5784	5927	err = -ENOMEM;
5785	5928	goto bitmap_abort;
5786	5929	}
@@ -6025,8 +6168,9 @@ static void __md_stop_writes(struct mddev *mddev) · 238 unmodified lines
6025	6168	mddev->in_sync = 1;
6026	6169	md_update_sb(mddev, 1);
6027	6170	}
6028		− mempool_destroy(mddev->wb_info_pool);
6029		− mddev->wb_info_pool = NULL;
	6171	+ /* disable policy to guarantee rdevs free resources for serialization */
	6172	+ mddev->serialize_policy = 0;
	6173	+ mddev_destroy_serial_pool(mddev, NULL, true);
6030	6174	}
6031	6175
6032	6176	void md_stop_writes(struct mddev *mddev)

drivers/md/md.h+27−18

@@ -32,6 +32,16 @@ · 31 unmodified lines
32	32	* be retried.
33	33	*/
34	34	#define MD_FAILFAST (REQ_FAILFAST_DEV \| REQ_FAILFAST_TRANSPORT)
	35	+
	36	+/*
	37	+ * The struct embedded in rdev is used to serialize IO.
	38	+ */
	39	+struct serial_in_rdev {
	40	+ struct rb_root_cached serial_rb;
	41	+ spinlock_t serial_lock;
	42	+ wait_queue_head_t serial_io_wait;
	43	+};
	44	+
35	45	/*
36	46	* MD's 'extended' device
37	47	*/
@@ -110,12 +120,7 @@ struct md_rdev { · 72 unmodified lines
110	120	* in superblock.
111	121	*/
112	122
113		− /*
114		− * The members for check collision of write behind IOs.
115		− */
116		− struct list_head wb_list;
117		− spinlock_t wb_list_lock;
118		− wait_queue_head_t wb_io_wait;
	123	+ struct serial_in_rdev serial; / used for raid1 io serialization */
119	124
120	125	struct work_struct del_work; /* used for delayed sysfs removal */
121	126
@@ -201,9 +206,9 @@ enum flag_bits { · 79 unmodified lines
201	206	* it didn't fail, so don't use FailFast
202	207	* any more for metadata
203	208	*/
204		− WBCollisionCheck, /*
205		− * multiqueue device should check if there
206		− * is collision between write behind bios.
	209	+ CollisionCheck, /*
	210	+ * check if there is collision between raid1
	211	+ * serial bios.
207	212	*/
208	213	};
209	214
@@ -263,12 +268,13 @@ enum mddev_sb_flags { · 53 unmodified lines
263	268	MD_SB_NEED_REWRITE, /* metadata write needs to be repeated */
264	269	};
265	270
266		−#define NR_WB_INFOS 8
267		−/* record current range of write behind IOs */
268		−struct wb_info {
269		− sector_t lo;
270		− sector_t hi;
271		− struct list_head list;
	271	+#define NR_SERIAL_INFOS 8
	272	+/* record current range of serialize IOs */
	273	+struct serial_info {
	274	+ struct rb_node node;
	275	+ sector_t start; /* start sector of rb node */
	276	+ sector_t last; /* end sector of rb node */
	277	+ sector_t _subtree_last; /* highest sector in subtree of rb node */
272	278	};
273	279
274	280	struct mddev {
@@ -487,13 +493,14 @@ struct mddev { · 212 unmodified lines
487	493	*/
488	494	struct work_struct flush_work;
489	495	struct work_struct event_work; /* used by dm to report failure event */
490		− mempool_t *wb_info_pool;
	496	+ mempool_t *serial_info_pool;
491	497	void (sync_super)(struct mddev mddev, struct md_rdev *rdev);
492	498	struct md_cluster_info *cluster_info;
493	499	unsigned int good_device_nr; /* good device num within cluster raid */
494	500
495	501	bool has_superblocks:1;
496	502	bool fail_last_dev:1;
	503	+ bool serialize_policy:1;
497	504	};
498	505
499	506	enum recovery_flags {
@@ -737,8 +744,10 @@ extern struct bio *bio_alloc_mddev(gfp_t gfp_mask, int nr_iovecs, · 237 unmodified lines
737	744	extern void md_reload_sb(struct mddev *mddev, int raid_disk);
738	745	extern void md_update_sb(struct mddev *mddev, int force);
739	746	extern void md_kick_rdev_from_array(struct md_rdev * rdev);
740		−extern void mddev_create_wb_pool(struct mddev mddev, struct md_rdev rdev,
741		− bool is_suspend);
	747	+extern void mddev_create_serial_pool(struct mddev mddev, struct md_rdev rdev,
	748	+ bool is_suspend);
	749	+extern void mddev_destroy_serial_pool(struct mddev mddev, struct md_rdev rdev,
	750	+ bool is_suspend);
742	751	struct md_rdev md_find_rdev_nr_rcu(struct mddev mddev, int nr);
743	752	struct md_rdev md_find_rdev_rcu(struct mddev mddev, dev_t dev);
744	753

drivers/md/raid1.c+61−50

@@ -29,6 +29,7 @@ · 28 unmodified lines
29	29	#include <linux/module.h>
30	30	#include <linux/seq_file.h>
31	31	#include <linux/ratelimit.h>
	32	+#include <linux/interval_tree_generic.h>
32	33
33	34	#include <trace/events/block.h>
34	35
@@ -50,55 +51,71 @@ static void lower_barrier(struct r1conf *conf, sector_t sector_nr); · 15 unmodified lines
50	51
51	52	#include "raid1-10.c"
52	53
53		−static int check_and_add_wb(struct md_rdev *rdev, sector_t lo, sector_t hi)
	54	+#define START(node) ((node)->start)
	55	+#define LAST(node) ((node)->last)
	56	+INTERVAL_TREE_DEFINE(struct serial_info, node, sector_t, _subtree_last,
	57	+ START, LAST, static inline, raid1_rb);
	58	+
	59	+static int check_and_add_serial(struct md_rdev rdev, struct r1bio r1_bio,
	60	+ struct serial_info *si, int idx)
54	61	{
55		− struct wb_info wi, temp_wi;
56	62	unsigned long flags;
57	63	int ret = 0;
58		− struct mddev *mddev = rdev->mddev;
59		−
60		− wi = mempool_alloc(mddev->wb_info_pool, GFP_NOIO);
61		−
62		− spin_lock_irqsave(&rdev->wb_list_lock, flags);
63		− list_for_each_entry(temp_wi, &rdev->wb_list, list) {
64		− /* collision happened */
65		− if (hi > temp_wi->lo && lo < temp_wi->hi) {
66		− ret = -EBUSY;
67		− break;
68		− }
	64	+ sector_t lo = r1_bio->sector;
	65	+ sector_t hi = lo + r1_bio->sectors;
	66	+ struct serial_in_rdev *serial = &rdev->serial[idx];
	67	+
	68	+ spin_lock_irqsave(&serial->serial_lock, flags);
	69	+ /* collision happened */
	70	+ if (raid1_rb_iter_first(&serial->serial_rb, lo, hi))
	71	+ ret = -EBUSY;
	72	+ else {
	73	+ si->start = lo;
	74	+ si->last = hi;
	75	+ raid1_rb_insert(si, &serial->serial_rb);
69	76	}
70		−
71		− if (!ret) {
72		− wi->lo = lo;
73		− wi->hi = hi;
74		− list_add(&wi->list, &rdev->wb_list);
75		− } else
76		− mempool_free(wi, mddev->wb_info_pool);
77		− spin_unlock_irqrestore(&rdev->wb_list_lock, flags);
	77	+ spin_unlock_irqrestore(&serial->serial_lock, flags);
78	78
79	79	return ret;
80	80	}
81	81
82		−static void remove_wb(struct md_rdev *rdev, sector_t lo, sector_t hi)
	82	+static void wait_for_serialization(struct md_rdev rdev, struct r1bio r1_bio)
	83	+{
	84	+ struct mddev *mddev = rdev->mddev;
	85	+ struct serial_info *si;
	86	+ int idx = sector_to_idx(r1_bio->sector);
	87	+ struct serial_in_rdev *serial = &rdev->serial[idx];
	88	+
	89	+ if (WARN_ON(!mddev->serial_info_pool))
	90	+ return;
	91	+ si = mempool_alloc(mddev->serial_info_pool, GFP_NOIO);
	92	+ wait_event(serial->serial_io_wait,
	93	+ check_and_add_serial(rdev, r1_bio, si, idx) == 0);
	94	+}
	95	+
	96	+static void remove_serial(struct md_rdev *rdev, sector_t lo, sector_t hi)
83	97	{
84		− struct wb_info *wi;
	98	+ struct serial_info *si;
85	99	unsigned long flags;
86	100	int found = 0;
87	101	struct mddev *mddev = rdev->mddev;
88		−
89		− spin_lock_irqsave(&rdev->wb_list_lock, flags);
90		− list_for_each_entry(wi, &rdev->wb_list, list)
91		− if (hi == wi->hi && lo == wi->lo) {
92		− list_del(&wi->list);
93		− mempool_free(wi, mddev->wb_info_pool);
	102	+ int idx = sector_to_idx(lo);
	103	+ struct serial_in_rdev *serial = &rdev->serial[idx];
	104	+
	105	+ spin_lock_irqsave(&serial->serial_lock, flags);
	106	+ for (si = raid1_rb_iter_first(&serial->serial_rb, lo, hi);
	107	+ si; si = raid1_rb_iter_next(si, lo, hi)) {
	108	+ if (si->start == lo && si->last == hi) {
	109	+ raid1_rb_remove(si, &serial->serial_rb);
	110	+ mempool_free(si, mddev->serial_info_pool);
94	111	found = 1;
95	112	break;
96	113	}
97		−
	114	+ }
98	115	if (!found)
99		− WARN(1, "The write behind IO is not recorded\n");
100		− spin_unlock_irqrestore(&rdev->wb_list_lock, flags);
101		− wake_up(&rdev->wb_io_wait);
	116	+ WARN(1, "The write IO is not recorded for serialization\n");
	117	+ spin_unlock_irqrestore(&serial->serial_lock, flags);
	118	+ wake_up(&serial->serial_io_wait);
102	119	}
103	120
104	121	/*
@@ -430,6 +447,8 @@ static void raid1_end_write_request(struct bio *bio) · 325 unmodified lines
430	447	int mirror = find_bio_disk(r1_bio, bio);
431	448	struct md_rdev *rdev = conf->mirrors[mirror].rdev;
432	449	bool discard_error;
	450	+ sector_t lo = r1_bio->sector;
	451	+ sector_t hi = r1_bio->sector + r1_bio->sectors;
433	452
434	453	discard_error = bio->bi_status && bio_op(bio) == REQ_OP_DISCARD;
435	454
@@ -499,12 +518,8 @@ static void raid1_end_write_request(struct bio *bio) · 63 unmodified lines
499	518	}
500	519
501	520	if (behind) {
502		− if (test_bit(WBCollisionCheck, &rdev->flags)) {
503		− sector_t lo = r1_bio->sector;
504		− sector_t hi = r1_bio->sector + r1_bio->sectors;
505		−
506		− remove_wb(rdev, lo, hi);
507		− }
	521	+ if (test_bit(CollisionCheck, &rdev->flags))
	522	+ remove_serial(rdev, lo, hi);
508	523	if (test_bit(WriteMostly, &rdev->flags))
509	524	atomic_dec(&r1_bio->behind_remaining);
510	525
@@ -527,7 +542,8 @@ static void raid1_end_write_request(struct bio *bio) · 16 unmodified lines
527	542	call_bio_endio(r1_bio);
528	543	}
529	544	}
530		− }
	545	+ } else if (rdev->mddev->serialize_policy)
	546	+ remove_serial(rdev, lo, hi);
531	547	if (r1_bio->bios[mirror] == NULL)
532	548	rdev_dec_pending(rdev, conf->mddev);
533	549
@@ -1479,6 +1495,7 @@ static void raid1_write_request(struct mddev mddev, struct bio bio, · 945 unmodified lines
1479	1495
1480	1496	for (i = 0; i < disks; i++) {
1481	1497	struct bio *mbio = NULL;
	1498	+ struct md_rdev *rdev = conf->mirrors[i].rdev;
1482	1499	if (!r1_bio->bios[i])
1483	1500	continue;
1484	1501
@@ -1506,18 +1523,12 @@ static void raid1_write_request(struct mddev mddev, struct bio bio, · 21 unmodified lines
1506	1523	mbio = bio_clone_fast(bio, GFP_NOIO, &mddev->bio_set);
1507	1524
1508	1525	if (r1_bio->behind_master_bio) {
1509		− struct md_rdev *rdev = conf->mirrors[i].rdev;
1510		−
1511		− if (test_bit(WBCollisionCheck, &rdev->flags)) {
1512		− sector_t lo = r1_bio->sector;
1513		− sector_t hi = r1_bio->sector + r1_bio->sectors;
1514		−
1515		− wait_event(rdev->wb_io_wait,
1516		− check_and_add_wb(rdev, lo, hi) == 0);
1517		− }
	1526	+ if (test_bit(CollisionCheck, &rdev->flags))
	1527	+ wait_for_serialization(rdev, r1_bio);
1518	1528	if (test_bit(WriteMostly, &rdev->flags))
1519	1529	atomic_inc(&r1_bio->behind_remaining);
1520		− }
	1530	+ } else if (mddev->serialize_policy)
	1531	+ wait_for_serialization(rdev, r1_bio);
1521	1532
1522	1533	r1_bio->bios[i] = mbio;
1523	1534

drivers/md/raid5.c+7−14

@@ -6598,7 +6598,6 @@ raid5_show_group_thread_cnt(struct mddev mddev, char page) · 6597 unmodified lines
6598	6598
6599	6599	static int alloc_thread_groups(struct r5conf *conf, int cnt,
6600	6600	int *group_cnt,
6601		− int *worker_cnt_per_group,
6602	6601	struct r5worker_group **worker_groups);
6603	6602	static ssize_t
6604	6603	raid5_store_group_thread_cnt(struct mddev mddev, const char page, size_t len)
@@ -6607,7 +6606,7 @@ raid5_store_group_thread_cnt(struct mddev mddev, const char page, size_t len) · 2 unmodified lines
6607	6606	unsigned int new;
6608	6607	int err;
6609	6608	struct r5worker_group new_groups, old_groups;
6610		− int group_cnt, worker_cnt_per_group;
	6609	+ int group_cnt;
6611	6610
6612	6611	if (len >= PAGE_SIZE)
6613	6612	return -EINVAL;
@@ -6630,13 +6629,11 @@ raid5_store_group_thread_cnt(struct mddev mddev, const char page, size_t len) · 16 unmodified lines
6630	6629	if (old_groups)
6631	6630	flush_workqueue(raid5_wq);
6632	6631
6633		− err = alloc_thread_groups(conf, new,
6634		− &group_cnt, &worker_cnt_per_group,
6635		− &new_groups);
	6632	+ err = alloc_thread_groups(conf, new, &group_cnt, &new_groups);
6636	6633	if (!err) {
6637	6634	spin_lock_irq(&conf->device_lock);
6638	6635	conf->group_cnt = group_cnt;
6639		− conf->worker_cnt_per_group = worker_cnt_per_group;
	6636	+ conf->worker_cnt_per_group = new;
6640	6637	conf->worker_groups = new_groups;
6641	6638	spin_unlock_irq(&conf->device_lock);
6642	6639
@@ -6672,16 +6669,13 @@ static struct attribute_group raid5_attrs_group = { · 29 unmodified lines
6672	6669	.attrs = raid5_attrs,
6673	6670	};
6674	6671
6675		−static int alloc_thread_groups(struct r5conf *conf, int cnt,
6676		− int *group_cnt,
6677		− int *worker_cnt_per_group,
	6672	+static int alloc_thread_groups(struct r5conf conf, int cnt, int group_cnt,
6678	6673	struct r5worker_group **worker_groups)
6679	6674	{
6680	6675	int i, j, k;
6681	6676	ssize_t size;
6682	6677	struct r5worker *workers;
6683	6678
6684		− *worker_cnt_per_group = cnt;
6685	6679	if (cnt == 0) {
6686	6680	*group_cnt = 0;
6687	6681	*worker_groups = NULL;
@@ -6882,7 +6876,7 @@ static struct r5conf setup_conf(struct mddev mddev) · 194 unmodified lines
6882	6876	struct disk_info *disk;
6883	6877	char pers_name[6];
6884	6878	int i;
6885		− int group_cnt, worker_cnt_per_group;
	6879	+ int group_cnt;
6886	6880	struct r5worker_group *new_group;
6887	6881	int ret;
6888	6882
@@ -6928,10 +6922,9 @@ static struct r5conf setup_conf(struct mddev mddev) · 39 unmodified lines
6928	6922	for (i = 0; i < PENDING_IO_MAX; i++)
6929	6923	list_add(&conf->pending_data[i].sibling, &conf->free_list);
6930	6924	/* Don't enable multi-threading by default*/
6931		− if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group,
6932		− &new_group)) {
	6925	+ if (!alloc_thread_groups(conf, 0, &group_cnt, &new_group)) {
6933	6926	conf->group_cnt = group_cnt;
6934		− conf->worker_cnt_per_group = worker_cnt_per_group;
	6927	+ conf->worker_cnt_per_group = 0;
6935	6928	conf->worker_groups = new_group;
6936	6929	} else
6937	6930	goto abort;

include/linux/raid/pq.h+6−1

@@ -27,8 +27,8 @@ extern const char raid6_empty_zero_page[PAGE_SIZE]; · 26 unmodified lines
27	27
28	28	#include <errno.h>
29	29	#include <inttypes.h>
30		−#include <limits.h>
31	30	#include <stddef.h>
	31	+#include <string.h>
32	32	#include <sys/mman.h>
33	33	#include <sys/time.h>
34	34	#include <sys/types.h>
@@ -44,6 +44,9 @@ typedef uint64_t u64; · 9 unmodified lines
44	44	#ifndef PAGE_SIZE
45	45	# define PAGE_SIZE 4096
46	46	#endif
	47	+#ifndef PAGE_SHIFT
	48	+# define PAGE_SHIFT 12
	49	+#endif
47	50	extern const char raid6_empty_zero_page[PAGE_SIZE];
48	51
49	52	#define __init
@@ -59,7 +62,9 @@ extern const char raid6_empty_zero_page[PAGE_SIZE]; · 9 unmodified lines
59	62	#define enable_kernel_altivec()
60	63	#define disable_kernel_altivec()
61	64
	65	+#undef EXPORT_SYMBOL
62	66	#define EXPORT_SYMBOL(sym)
	67	+#undef EXPORT_SYMBOL_GPL
63	68	#define EXPORT_SYMBOL_GPL(sym)
64	69	#define MODULE_LICENSE(licence)
65	70	#define MODULE_DESCRIPTION(desc)

include/trace/events/bcache.h+2−1

@@ -275,7 +275,8 @@ TRACE_EVENT(bcache_btree_write, · 274 unmodified lines
275	275	__entry->keys = b->keys.set[b->keys.nsets].data->keys;
276	276	),
277	277
278		− TP_printk("bucket %zu", __entry->bucket)
	278	+ TP_printk("bucket %zu written block %u + %u",
	279	+ __entry->bucket, __entry->block, __entry->keys)
279	280	);
280	281
281	282	DEFINE_EVENT(btree_node, bcache_btree_node_alloc,

include/uapi/linux/bcache.h+52−0

@@ -148,6 +148,7 @@ static inline struct bkey bkey_idx(const struct bkey k, unsigned int nr_keys) · 147 unmodified lines
148	148	#define BCACHE_SB_MAX_VERSION 4
149	149
150	150	#define SB_SECTOR 8
	151	+#define SB_OFFSET (SB_SECTOR << SECTOR_SHIFT)
151	152	#define SB_SIZE 4096
152	153	#define SB_LABEL_SIZE 32
153	154	#define SB_JOURNAL_BUCKETS 256U
@@ -156,6 +157,57 @@ static inline struct bkey bkey_idx(const struct bkey k, unsigned int nr_keys) · 2 unmodified lines
156	157
157	158	#define BDEV_DATA_START_DEFAULT 16 /* sectors */
158	159
	160	+struct cache_sb_disk {
	161	+ __le64 csum;
	162	+ __le64 offset; /* sector where this sb was written */
	163	+ __le64 version;
	164	+
	165	+ __u8 magic[16];
	166	+
	167	+ __u8 uuid[16];
	168	+ union {
	169	+ __u8 set_uuid[16];
	170	+ __le64 set_magic;
	171	+ };
	172	+ __u8 label[SB_LABEL_SIZE];
	173	+
	174	+ __le64 flags;
	175	+ __le64 seq;
	176	+ __le64 pad[8];
	177	+
	178	+ union {
	179	+ struct {
	180	+ /* Cache devices */
	181	+ __le64 nbuckets; /* device size */
	182	+
	183	+ __le16 block_size; /* sectors */
	184	+ __le16 bucket_size; /* sectors */
	185	+
	186	+ __le16 nr_in_set;
	187	+ __le16 nr_this_dev;
	188	+ };
	189	+ struct {
	190	+ /* Backing devices */
	191	+ __le64 data_offset;
	192	+
	193	+ /*
	194	+ * block_size from the cache device section is still used by
	195	+ * backing devices, so don't add anything here until we fix
	196	+ * things to not need it for backing devices anymore
	197	+ */
	198	+ };
	199	+ };
	200	+
	201	+ __le32 last_mount; /* time overflow in y2106 */
	202	+
	203	+ __le16 first_bucket;
	204	+ union {
	205	+ __le16 njournal_buckets;
	206	+ __le16 keys;
	207	+ };
	208	+ __le64 d[SB_JOURNAL_BUCKETS]; /* journal buckets */
	209	+};
	210	+
159	211	struct cache_sb {
160	212	__u64 csum;
161	213	__u64 offset; /* sector where this sb was written */

lib/crc64.c+1−0

@@ -28,6 +28,7 @@ · 27 unmodified lines
28	28
29	29	#include <linux/module.h>
30	30	#include <linux/types.h>
	31	+#include <linux/crc64.h>
31	32	#include "crc64table.h"
32	33
33	34	MODULE_DESCRIPTION("CRC64 calculations");

lib/raid6/algos.c+40−23

@@ -124,6 +124,9 @@ const struct raid6_recov_calls *const raid6_recov_algos[] = { · 123 unmodified lines
124	124	#define time_before(x, y) ((x) < (y))
125	125	#endif
126	126
	127	+#define RAID6_TEST_DISKS 8
	128	+#define RAID6_TEST_DISKS_ORDER 3
	129	+
127	130	static inline const struct raid6_recov_calls *raid6_choose_recov(void)
128	131	{
129	132	const struct raid6_recov_calls const algo;
@@ -146,7 +149,7 @@ static inline const struct raid6_recov_calls *raid6_choose_recov(void) · 16 unmodified lines
146	149	}
147	150
148	151	static inline const struct raid6_calls *raid6_choose_gen(
149		− void (const dptrs)[(65536/PAGE_SIZE)+2], const int disks)
	152	+ void (const dptrs)[RAID6_TEST_DISKS], const int disks)
150	153	{
151	154	unsigned long perf, bestgenperf, bestxorperf, j0, j1;
152	155	int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */
@@ -181,7 +184,8 @@ static inline const struct raid6_calls *raid6_choose_gen( · 28 unmodified lines
181	184	best = *algo;
182	185	}
183	186	pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name,
184		− (perf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2));
	187	+ (perf * HZ * (disks-2)) >>
	188	+ (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2));
185	189
186	190	if (!(*algo)->xor_syndrome)
187	191	continue;
@@ -204,17 +208,24 @@ static inline const struct raid6_calls *raid6_choose_gen( · 16 unmodified lines
204	208	bestxorperf = perf;
205	209
206	210	pr_info("raid6: %-8s xor() %5ld MB/s\n", (*algo)->name,
207		− (perf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2+1));
	211	+ (perf * HZ * (disks-2)) >>
	212	+ (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1));
208	213	}
209	214	}
210	215
211	216	if (best) {
212		− pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
213		− best->name,
214		− (bestgenperf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2));
215		− if (best->xor_syndrome)
216		− pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n",
217		− (bestxorperf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2+1));
	217	+ if (IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
	218	+ pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
	219	+ best->name,
	220	+ (bestgenperf * HZ * (disks-2)) >>
	221	+ (20 - PAGE_SHIFT+RAID6_TIME_JIFFIES_LG2));
	222	+ if (best->xor_syndrome)
	223	+ pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n",
	224	+ (bestxorperf * HZ * (disks-2)) >>
	225	+ (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1));
	226	+ } else
	227	+ pr_info("raid6: skip pq benchmark and using algorithm %s\n",
	228	+ best->name);
218	229	raid6_call = *best;
219	230	} else
220	231	pr_err("raid6: Yikes! No algorithm found!\n");
@@ -228,27 +239,33 @@ static inline const struct raid6_calls *raid6_choose_gen( · 7 unmodified lines
228	239
229	240	int __init raid6_select_algo(void)
230	241	{
231		− const int disks = (65536/PAGE_SIZE)+2;
	242	+ const int disks = RAID6_TEST_DISKS;
232	243
233	244	const struct raid6_calls *gen_best;
234	245	const struct raid6_recov_calls *rec_best;
235		− char *syndromes;
236		− void *dptrs[(65536/PAGE_SIZE)+2];
237		− int i;
238		−
239		− for (i = 0; i < disks-2; i++)
240		− dptrs[i] = ((char )raid6_gfmul) + PAGE_SIZEi;
241		−
242		− /* Normal code - use a 2-page allocation to avoid D$ conflict */
243		− syndromes = (void *) __get_free_pages(GFP_KERNEL, 1);
	246	+ char disk_ptr, p;
	247	+ void *dptrs[RAID6_TEST_DISKS];
	248	+ int i, cycle;
244	249
245		− if (!syndromes) {
	250	+ /* prepare the buffer and fill it circularly with gfmul table */
	251	+ disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER);
	252	+ if (!disk_ptr) {
246	253	pr_err("raid6: Yikes! No memory available.\n");
247	254	return -ENOMEM;
248	255	}
249	256
250		− dptrs[disks-2] = syndromes;
251		− dptrs[disks-1] = syndromes + PAGE_SIZE;
	257	+ p = disk_ptr;
	258	+ for (i = 0; i < disks; i++)
	259	+ dptrs[i] = p + PAGE_SIZE * i;
	260	+
	261	+ cycle = ((disks - 2) * PAGE_SIZE) / 65536;
	262	+ for (i = 0; i < cycle; i++) {
	263	+ memcpy(p, raid6_gfmul, 65536);
	264	+ p += 65536;
	265	+ }
	266	+
	267	+ if ((disks - 2) * PAGE_SIZE % 65536)
	268	+ memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536);
252	269
253	270	/* select raid gen_syndrome function */
254	271	gen_best = raid6_choose_gen(&dptrs, disks);
@@ -256,7 +273,7 @@ int __init raid6_select_algo(void) · 1 unmodified line
256	273	/* select raid recover functions */
257	274	rec_best = raid6_choose_recov();
258	275
259		− free_pages((unsigned long)syndromes, 1);
	276	+ free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER);
260	277
261	278	return gen_best && rec_best ? 0 : -EINVAL;
262	279	}

lib/raid6/mktables.c+1−1

@@ -56,8 +56,8 @@ int main(int argc, char *argv[]) · 55 unmodified lines
56	56	uint8_t v;
57	57	uint8_t exptbl[256], invtbl[256];
58	58
59		− printf("#include <linux/raid/pq.h>\n");
60	59	printf("#include <linux/export.h>\n");
	60	+ printf("#include <linux/raid/pq.h>\n");
61	61
62	62	/* Compute multiplication table */
63	63	printf("\nconst u8 __attribute__((aligned(256)))\n"

Merge tag 'for-5.6/drivers-2020-01-27' of git://git.kernel.dk/linux-block

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code