Merge branch 'md-next' into md-linus

Shaohua Li committed 9 years ago

Commit 20737732 parents b78b499 295307916 files · +3429 −1262Browse files

Unified · Split

16 files

drivers/md/bitmap.c +98 −68
drivers/md/dm-raid.c +2 −2
drivers/md/linear.c +17 −14
drivers/md/md.c +364 −337
drivers/md/md.h +67 −41
drivers/md/multipath.c +34 −58
drivers/md/raid0.c +59 −48
drivers/md/raid1.c +165 −82
drivers/md/raid1.h +11 −8
drivers/md/raid10.c +189 −106
drivers/md/raid10.h +2 −0
drivers/md/raid5-cache.c +1645 −240
drivers/md/raid5.c +381 −242
drivers/md/raid5.h +160 −12
include/uapi/linux/raid/md_p.h +6 −1
lib/raid6/avx2.c +229 −3

@@ -27,6 +27,7 @@ · 26 unmodified lines
27	27	#include <linux/mount.h>
28	28	#include <linux/buffer_head.h>
29	29	#include <linux/seq_file.h>
	30	+#include <trace/events/block.h>
30	31	#include "md.h"
31	32	#include "bitmap.h"
32	33
@@ -208,11 +209,13 @@ static struct md_rdev next_active_rdev(struct md_rdev rdev, struct mddev *mdde · 175 unmodified lines
208	209
209	210	static int write_sb_page(struct bitmap bitmap, struct page page, int wait)
210	211	{
211		− struct md_rdev *rdev = NULL;
	212	+ struct md_rdev *rdev;
212	213	struct block_device *bdev;
213	214	struct mddev *mddev = bitmap->mddev;
214	215	struct bitmap_storage *store = &bitmap->storage;
215	216
	217	+restart:
	218	+ rdev = NULL;
216	219	while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
217	220	int size = PAGE_SIZE;
218	221	loff_t offset = mddev->bitmap_info.offset;
@@ -268,8 +271,8 @@ static int write_sb_page(struct bitmap bitmap, struct page page, int wait) · 49 unmodified lines
268	271	page);
269	272	}
270	273
271		− if (wait)
272		− md_super_wait(mddev);
	274	+ if (wait && md_super_wait(mddev) < 0)
	275	+ goto restart;
273	276	return 0;
274	277
275	278	bad_alignment:
@@ -405,10 +408,10 @@ static int read_page(struct file *file, unsigned long index, · 129 unmodified lines
405	408	ret = -EIO;
406	409	out:
407	410	if (ret)
408		− printk(KERN_ALERT "md: bitmap read error: (%dB @ %llu): %d\n",
409		− (int)PAGE_SIZE,
410		− (unsigned long long)index << PAGE_SHIFT,
411		− ret);
	411	+ pr_err("md: bitmap read error: (%dB @ %llu): %d\n",
	412	+ (int)PAGE_SIZE,
	413	+ (unsigned long long)index << PAGE_SHIFT,
	414	+ ret);
412	415	return ret;
413	416	}
414	417
@@ -416,6 +419,28 @@ out: · 1 unmodified line
416	419	* bitmap file superblock operations
417	420	*/
418	421
	422	+/*
	423	+ * bitmap_wait_writes() should be called before writing any bitmap
	424	+ * blocks, to ensure previous writes, particularly from
	425	+ * bitmap_daemon_work(), have completed.
	426	+ */
	427	+static void bitmap_wait_writes(struct bitmap *bitmap)
	428	+{
	429	+ if (bitmap->storage.file)
	430	+ wait_event(bitmap->write_wait,
	431	+ atomic_read(&bitmap->pending_writes)==0);
	432	+ else
	433	+ /* Note that we ignore the return value. The writes
	434	+ * might have failed, but that would just mean that
	435	+ * some bits which should be cleared haven't been,
	436	+ * which is safe. The relevant bitmap blocks will
	437	+ * probably get written again, but there is no great
	438	+ * loss if they aren't.
	439	+ */
	440	+ md_super_wait(bitmap->mddev);
	441	+}
	442	+
	443	+
419	444	/* update the event counter and sync the superblock to disk */
420	445	void bitmap_update_sb(struct bitmap *bitmap)
421	446	{
@@ -455,24 +480,24 @@ void bitmap_print_sb(struct bitmap *bitmap) · 33 unmodified lines
455	480	if (!bitmap \|\| !bitmap->storage.sb_page)
456	481	return;
457	482	sb = kmap_atomic(bitmap->storage.sb_page);
458		− printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap));
459		− printk(KERN_DEBUG " magic: %08x\n", le32_to_cpu(sb->magic));
460		− printk(KERN_DEBUG " version: %d\n", le32_to_cpu(sb->version));
461		− printk(KERN_DEBUG " uuid: %08x.%08x.%08x.%08x\n",
462		− (__u32 )(sb->uuid+0),
463		− (__u32 )(sb->uuid+4),
464		− (__u32 )(sb->uuid+8),
465		− (__u32 )(sb->uuid+12));
466		− printk(KERN_DEBUG " events: %llu\n",
467		− (unsigned long long) le64_to_cpu(sb->events));
468		− printk(KERN_DEBUG "events cleared: %llu\n",
469		− (unsigned long long) le64_to_cpu(sb->events_cleared));
470		− printk(KERN_DEBUG " state: %08x\n", le32_to_cpu(sb->state));
471		− printk(KERN_DEBUG " chunksize: %d B\n", le32_to_cpu(sb->chunksize));
472		− printk(KERN_DEBUG " daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
473		− printk(KERN_DEBUG " sync size: %llu KB\n",
474		− (unsigned long long)le64_to_cpu(sb->sync_size)/2);
475		− printk(KERN_DEBUG "max write behind: %d\n", le32_to_cpu(sb->write_behind));
	483	+ pr_debug("%s: bitmap file superblock:\n", bmname(bitmap));
	484	+ pr_debug(" magic: %08x\n", le32_to_cpu(sb->magic));
	485	+ pr_debug(" version: %d\n", le32_to_cpu(sb->version));
	486	+ pr_debug(" uuid: %08x.%08x.%08x.%08x\n",
	487	+ (__u32 )(sb->uuid+0),
	488	+ (__u32 )(sb->uuid+4),
	489	+ (__u32 )(sb->uuid+8),
	490	+ (__u32 )(sb->uuid+12));
	491	+ pr_debug(" events: %llu\n",
	492	+ (unsigned long long) le64_to_cpu(sb->events));
	493	+ pr_debug("events cleared: %llu\n",
	494	+ (unsigned long long) le64_to_cpu(sb->events_cleared));
	495	+ pr_debug(" state: %08x\n", le32_to_cpu(sb->state));
	496	+ pr_debug(" chunksize: %d B\n", le32_to_cpu(sb->chunksize));
	497	+ pr_debug(" daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
	498	+ pr_debug(" sync size: %llu KB\n",
	499	+ (unsigned long long)le64_to_cpu(sb->sync_size)/2);
	500	+ pr_debug("max write behind: %d\n", le32_to_cpu(sb->write_behind));
476	501	kunmap_atomic(sb);
477	502	}
478	503
@@ -506,14 +531,14 @@ static int bitmap_new_disk_sb(struct bitmap *bitmap) · 27 unmodified lines
506	531	BUG_ON(!chunksize);
507	532	if (!is_power_of_2(chunksize)) {
508	533	kunmap_atomic(sb);
509		− printk(KERN_ERR "bitmap chunksize not a power of 2\n");
	534	+ pr_warn("bitmap chunksize not a power of 2\n");
510	535	return -EINVAL;
511	536	}
512	537	sb->chunksize = cpu_to_le32(chunksize);
513	538
514	539	daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
515	540	if (!daemon_sleep \|\| (daemon_sleep > MAX_SCHEDULE_TIMEOUT)) {
516		− printk(KERN_INFO "Choosing daemon_sleep default (5 sec)\n");
	541	+ pr_debug("Choosing daemon_sleep default (5 sec)\n");
517	542	daemon_sleep = 5 * HZ;
518	543	}
519	544	sb->daemon_sleep = cpu_to_le32(daemon_sleep);
@@ -584,7 +609,7 @@ re_read: · 64 unmodified lines
584	609	/* to 4k blocks */
585	610	bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks, 4096);
586	611	offset = bitmap->mddev->bitmap_info.offset + (bitmap->cluster_slot * (bm_blocks << 3));
587		− pr_info("%s:%d bm slot: %d offset: %llu\n", __func__, __LINE__,
	612	+ pr_debug("%s:%d bm slot: %d offset: %llu\n", __func__, __LINE__,
588	613	bitmap->cluster_slot, offset);
589	614	}
590	615
@@ -634,7 +659,7 @@ re_read: · 43 unmodified lines
634	659	else if (write_behind > COUNTER_MAX)
635	660	reason = "write-behind limit out of range (0 - 16383)";
636	661	if (reason) {
637		− printk(KERN_INFO "%s: invalid bitmap file superblock: %s\n",
	662	+ pr_warn("%s: invalid bitmap file superblock: %s\n",
638	663	bmname(bitmap), reason);
639	664	goto out;
640	665	}
@@ -648,18 +673,15 @@ re_read: · 7 unmodified lines
648	673	* bitmap's UUID and event counter to the mddev's
649	674	*/
650	675	if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
651		− printk(KERN_INFO
652		− "%s: bitmap superblock UUID mismatch\n",
653		− bmname(bitmap));
	676	+ pr_warn("%s: bitmap superblock UUID mismatch\n",
	677	+ bmname(bitmap));
654	678	goto out;
655	679	}
656	680	events = le64_to_cpu(sb->events);
657	681	if (!nodes && (events < bitmap->mddev->events)) {
658		− printk(KERN_INFO
659		− "%s: bitmap file is out of date (%llu < %llu) "
660		− "-- forcing full recovery\n",
661		− bmname(bitmap), events,
662		− (unsigned long long) bitmap->mddev->events);
	682	+ pr_warn("%s: bitmap file is out of date (%llu < %llu) -- forcing full recovery\n",
	683	+ bmname(bitmap), events,
	684	+ (unsigned long long) bitmap->mddev->events);
663	685	set_bit(BITMAP_STALE, &bitmap->flags);
664	686	}
665	687	}
@@ -679,8 +701,8 @@ out: · 13 unmodified lines
679	701	if (err == 0 && nodes && (bitmap->cluster_slot < 0)) {
680	702	err = md_setup_cluster(bitmap->mddev, nodes);
681	703	if (err) {
682		− pr_err("%s: Could not setup cluster service (%d)\n",
683		− bmname(bitmap), err);
	704	+ pr_warn("%s: Could not setup cluster service (%d)\n",
	705	+ bmname(bitmap), err);
684	706	goto out_no_sb;
685	707	}
686	708	bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev);
@@ -847,15 +869,13 @@ static void bitmap_file_kick(struct bitmap *bitmap) · 160 unmodified lines
847	869	ptr = file_path(bitmap->storage.file,
848	870	path, PAGE_SIZE);
849	871
850		− printk(KERN_ALERT
851		− "%s: kicking failed bitmap file %s from array!\n",
852		− bmname(bitmap), IS_ERR(ptr) ? "" : ptr);
	872	+ pr_warn("%s: kicking failed bitmap file %s from array!\n",
	873	+ bmname(bitmap), IS_ERR(ptr) ? "" : ptr);
853	874
854	875	kfree(path);
855	876	} else
856		− printk(KERN_ALERT
857		− "%s: disabling internal bitmap due to errors\n",
858		− bmname(bitmap));
	877	+ pr_warn("%s: disabling internal bitmap due to errors\n",
	878	+ bmname(bitmap));
859	879	}
860	880	}
861	881
@@ -983,6 +1003,7 @@ void bitmap_unplug(struct bitmap *bitmap) · 121 unmodified lines
983	1003	{
984	1004	unsigned long i;
985	1005	int dirty, need_write;
	1006	+ int writing = 0;
986	1007
987	1008	if (!bitmap \|\| !bitmap->storage.filemap \|\|
988	1009	test_bit(BITMAP_STALE, &bitmap->flags))
@@ -997,15 +1018,19 @@ void bitmap_unplug(struct bitmap *bitmap) · 8 unmodified lines
997	1018	need_write = test_and_clear_page_attr(bitmap, i,
998	1019	BITMAP_PAGE_NEEDWRITE);
999	1020	if (dirty \|\| need_write) {
	1021	+ if (!writing) {
	1022	+ bitmap_wait_writes(bitmap);
	1023	+ if (bitmap->mddev->queue)
	1024	+ blk_add_trace_msg(bitmap->mddev->queue,
	1025	+ "md bitmap_unplug");
	1026	+ }
1000	1027	clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
1001	1028	write_page(bitmap, bitmap->storage.filemap[i], 0);
	1029	+ writing = 1;
1002	1030	}
1003	1031	}
1004		− if (bitmap->storage.file)
1005		− wait_event(bitmap->write_wait,
1006		− atomic_read(&bitmap->pending_writes)==0);
1007		− else
1008		− md_super_wait(bitmap->mddev);
	1032	+ if (writing)
	1033	+ bitmap_wait_writes(bitmap);
1009	1034
1010	1035	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1011	1036	bitmap_file_kick(bitmap);
@@ -1056,14 +1081,13 @@ static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start) · 44 unmodified lines
1056	1081
1057	1082	outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
1058	1083	if (outofdate)
1059		− printk(KERN_INFO "%s: bitmap file is out of date, doing full "
1060		− "recovery\n", bmname(bitmap));
	1084	+ pr_warn("%s: bitmap file is out of date, doing full recovery\n", bmname(bitmap));
1061	1085
1062	1086	if (file && i_size_read(file->f_mapping->host) < store->bytes) {
1063		− printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n",
1064		− bmname(bitmap),
1065		− (unsigned long) i_size_read(file->f_mapping->host),
1066		− store->bytes);
	1087	+ pr_warn("%s: bitmap file too short %lu < %lu\n",
	1088	+ bmname(bitmap),
	1089	+ (unsigned long) i_size_read(file->f_mapping->host),
	1090	+ store->bytes);
1067	1091	goto err;
1068	1092	}
1069	1093
@@ -1137,16 +1161,15 @@ static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start) · 67 unmodified lines
1137	1161	offset = 0;
1138	1162	}
1139	1163
1140		− printk(KERN_INFO "%s: bitmap initialized from disk: "
1141		− "read %lu pages, set %lu of %lu bits\n",
1142		− bmname(bitmap), store->file_pages,
1143		− bit_cnt, chunks);
	1164	+ pr_debug("%s: bitmap initialized from disk: read %lu pages, set %lu of %lu bits\n",
	1165	+ bmname(bitmap), store->file_pages,
	1166	+ bit_cnt, chunks);
1144	1167
1145	1168	return 0;
1146	1169
1147	1170	err:
1148		− printk(KERN_INFO "%s: bitmap initialisation failed: %d\n",
1149		− bmname(bitmap), ret);
	1171	+ pr_warn("%s: bitmap initialisation failed: %d\n",
	1172	+ bmname(bitmap), ret);
1150	1173	return ret;
1151	1174	}
1152	1175
@@ -1225,6 +1248,10 @@ void bitmap_daemon_work(struct mddev *mddev) · 72 unmodified lines
1225	1248	}
1226	1249	bitmap->allclean = 1;
1227	1250
	1251	+ if (bitmap->mddev->queue)
	1252	+ blk_add_trace_msg(bitmap->mddev->queue,
	1253	+ "md bitmap_daemon_work");
	1254	+
1228	1255	/* Any file-page which is PENDING now needs to be written.
1229	1256	* So set NEEDWRITE now, then after we make any last-minute changes
1230	1257	* we will write it.
@@ -1289,6 +1316,7 @@ void bitmap_daemon_work(struct mddev *mddev) · 58 unmodified lines
1289	1316	}
1290	1317	spin_unlock_irq(&counts->lock);
1291	1318
	1319	+ bitmap_wait_writes(bitmap);
1292	1320	/* Now start writeout on any page in NEEDWRITE that isn't DIRTY.
1293	1321	* DIRTY pages need to be written by bitmap_unplug so it can wait
1294	1322	* for them.
@@ -1595,7 +1623,7 @@ void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector, bool force) · 300 unmodified lines
1595	1623	atomic_read(&bitmap->mddev->recovery_active) == 0);
1596	1624
1597	1625	bitmap->mddev->curr_resync_completed = sector;
1598		− set_bit(MD_CHANGE_CLEAN, &bitmap->mddev->flags);
	1626	+ set_bit(MD_SB_CHANGE_CLEAN, &bitmap->mddev->sb_flags);
1599	1627	sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
1600	1628	s = 0;
1601	1629	while (s < sector && s < bitmap->mddev->resync_max_sectors) {
@@ -1825,8 +1853,8 @@ struct bitmap bitmap_create(struct mddev mddev, int slot) · 223 unmodified lines
1825	1853	if (err)
1826	1854	goto error;
1827	1855
1828		− printk(KERN_INFO "created bitmap (%lu pages) for device %s\n",
1829		− bitmap->counts.pages, bmname(bitmap));
	1856	+ pr_debug("created bitmap (%lu pages) for device %s\n",
	1857	+ bitmap->counts.pages, bmname(bitmap));
1830	1858
1831	1859	err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
1832	1860	if (err)
@@ -2029,8 +2057,10 @@ int bitmap_resize(struct bitmap *bitmap, sector_t blocks, · 196 unmodified lines
2029	2057	!bitmap->mddev->bitmap_info.external,
2030	2058	mddev_is_clustered(bitmap->mddev)
2031	2059	? bitmap->cluster_slot : 0);
2032		− if (ret)
	2060	+ if (ret) {
	2061	+ bitmap_file_unmap(&store);
2033	2062	goto err;
	2063	+ }
2034	2064
2035	2065	pages = DIV_ROUND_UP(chunks, PAGE_COUNTER_RATIO);
2036	2066
@@ -2089,7 +2119,7 @@ int bitmap_resize(struct bitmap *bitmap, sector_t blocks, · 52 unmodified lines
2089	2119	bitmap->mddev->bitmap_info.chunksize = 1 << (old_counts.chunkshift +
2090	2120	BITMAP_BLOCK_SHIFT);
2091	2121	blocks = old_counts.chunks << old_counts.chunkshift;
2092		− pr_err("Could not pre-allocate in-memory bitmap for cluster raid\n");
	2122	+ pr_warn("Could not pre-allocate in-memory bitmap for cluster raid\n");
2093	2123	break;
2094	2124	} else
2095	2125	bitmap->counts.bp[page].count += 1;
@@ -2266,7 +2296,7 @@ location_store(struct mddev mddev, const char buf, size_t len) · 170 unmodified lines
2266	2296	/* Ensure new bitmap info is stored in
2267	2297	* metadata promptly.
2268	2298	*/
2269		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
	2299	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
2270	2300	md_wakeup_thread(mddev->thread);
2271	2301	}
2272	2302	rv = 0;

drivers/md/dm-raid.c+2−2

@@ -2011,7 +2011,7 @@ static int super_load(struct md_rdev rdev, struct md_rdev refdev) · 2010 unmodified lines
2011	2011	sb->compat_features = cpu_to_le32(FEATURE_FLAG_SUPPORTS_V190);
2012	2012
2013	2013	/* Force writing of superblocks to disk */
2014		− set_bit(MD_CHANGE_DEVS, &rdev->mddev->flags);
	2014	+ set_bit(MD_SB_CHANGE_DEVS, &rdev->mddev->sb_flags);
2015	2015
2016	2016	/* Any superblock is better than none, choose that if given */
2017	2017	return refdev ? 0 : 1;
@@ -3497,7 +3497,7 @@ static void rs_update_sbs(struct raid_set *rs) · 1479 unmodified lines
3497	3497	struct mddev *mddev = &rs->md;
3498	3498	int ro = mddev->ro;
3499	3499
3500		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
	3500	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
3501	3501	mddev->ro = 0;
3502	3502	md_update_sb(mddev, 1);
3503	3503	mddev->ro = ro;

drivers/md/linear.c+17−14

@@ -21,6 +21,7 @@ · 20 unmodified lines
21	21	#include <linux/seq_file.h>
22	22	#include <linux/module.h>
23	23	#include <linux/slab.h>
	24	+#include <trace/events/block.h>
24	25	#include "md.h"
25	26	#include "linear.h"
26	27
@@ -101,8 +102,8 @@ static struct linear_conf linear_conf(struct mddev mddev, int raid_disks) · 74 unmodified lines
101	102	sector_t sectors;
102	103
103	104	if (j < 0 \|\| j >= raid_disks \|\| disk->rdev) {
104		− printk(KERN_ERR "md/linear:%s: disk numbering problem. Aborting!\n",
105		− mdname(mddev));
	105	+ pr_warn("md/linear:%s: disk numbering problem. Aborting!\n",
	106	+ mdname(mddev));
106	107	goto out;
107	108	}
108	109
@@ -123,8 +124,8 @@ static struct linear_conf linear_conf(struct mddev mddev, int raid_disks) · 14 unmodified lines
123	124	discard_supported = true;
124	125	}
125	126	if (cnt != raid_disks) {
126		− printk(KERN_ERR "md/linear:%s: not enough drives present. Aborting!\n",
127		− mdname(mddev));
	127	+ pr_warn("md/linear:%s: not enough drives present. Aborting!\n",
	128	+ mdname(mddev));
128	129	goto out;
129	130	}
130	131
@@ -227,22 +228,22 @@ static void linear_make_request(struct mddev mddev, struct bio bio) · 96 unmodified lines
227	228	}
228	229
229	230	do {
230		− tmp_dev = which_dev(mddev, bio->bi_iter.bi_sector);
	231	+ sector_t bio_sector = bio->bi_iter.bi_sector;
	232	+ tmp_dev = which_dev(mddev, bio_sector);
231	233	start_sector = tmp_dev->end_sector - tmp_dev->rdev->sectors;
232	234	end_sector = tmp_dev->end_sector;
233	235	data_offset = tmp_dev->rdev->data_offset;
234	236	bio->bi_bdev = tmp_dev->rdev->bdev;
235	237
236		− if (unlikely(bio->bi_iter.bi_sector >= end_sector \|\|
237		− bio->bi_iter.bi_sector < start_sector))
	238	+ if (unlikely(bio_sector >= end_sector \|\|
	239	+ bio_sector < start_sector))
238	240	goto out_of_bounds;
239	241
240	242	if (unlikely(bio_end_sector(bio) > end_sector)) {
241	243	/* This bio crosses a device boundary, so we have to
242	244	* split it.
243	245	*/
244		− split = bio_split(bio, end_sector -
245		− bio->bi_iter.bi_sector,
	246	+ split = bio_split(bio, end_sector - bio_sector,
246	247	GFP_NOIO, fs_bio_set);
247	248	bio_chain(split, bio);
248	249	} else {
@@ -256,15 +257,18 @@ static void linear_make_request(struct mddev mddev, struct bio bio) · 7 unmodified lines
256	257	!blk_queue_discard(bdev_get_queue(split->bi_bdev)))) {
257	258	/* Just ignore it */
258	259	bio_endio(split);
259		− } else
	260	+ } else {
	261	+ if (mddev->gendisk)
	262	+ trace_block_bio_remap(bdev_get_queue(split->bi_bdev),
	263	+ split, disk_devt(mddev->gendisk),
	264	+ bio_sector);
260	265	generic_make_request(split);
	266	+ }
261	267	} while (split != bio);
262	268	return;
263	269
264	270	out_of_bounds:
265		− printk(KERN_ERR
266		− "md/linear:%s: make_request: Sector %llu out of bounds on "
267		− "dev %s: %llu sectors, offset %llu\n",
	271	+ pr_err("md/linear:%s: make_request: Sector %llu out of bounds on dev %s: %llu sectors, offset %llu\n",
268	272	mdname(mddev),
269	273	(unsigned long long)bio->bi_iter.bi_sector,
270	274	bdevname(tmp_dev->rdev->bdev, b),
@@ -275,7 +279,6 @@ out_of_bounds: · 4 unmodified lines
275	279
276	280	static void linear_status (struct seq_file seq, struct mddev mddev)
277	281	{
278		−
279	282	seq_printf(seq, " %dk rounding", mddev->chunk_sectors / 2);
280	283	}
281	284

drivers/md/md.c+364−337

@@ -30,6 +30,18 @@ · 29 unmodified lines
30	30	You should have received a copy of the GNU General Public License
31	31	(for example /usr/src/linux/COPYING); if not, write to the Free
32	32	Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
	33	+
	34	+ Errors, Warnings, etc.
	35	+ Please use:
	36	+ pr_crit() for error conditions that risk data loss
	37	+ pr_err() for error conditions that are unexpected, like an IO error
	38	+ or internal inconsistency
	39	+ pr_warn() for error conditions that could have been predicated, like
	40	+ adding a device to an array when it has incompatible metadata
	41	+ pr_info() for every interesting, very rare events, like an array starting
	42	+ or stopping, or resync starting or stopping
	43	+ pr_debug() for everything else.
	44	+
33	45	*/
34	46
35	47	#include <linux/kthread.h>
@@ -52,6 +64,7 @@ · 16 unmodified lines
52	64	#include <linux/raid/md_p.h>
53	65	#include <linux/raid/md_u.h>
54	66	#include <linux/slab.h>
	67	+#include <trace/events/block.h>
55	68	#include "md.h"
56	69	#include "bitmap.h"
57	70	#include "md-cluster.h"
@@ -684,11 +697,8 @@ static inline sector_t calc_dev_sboffset(struct md_rdev *rdev) · 626 unmodified lines
684	697	static int alloc_disk_sb(struct md_rdev *rdev)
685	698	{
686	699	rdev->sb_page = alloc_page(GFP_KERNEL);
687		− if (!rdev->sb_page) {
688		− printk(KERN_ALERT "md: out of memory.\n");
	700	+ if (!rdev->sb_page)
689	701	return -ENOMEM;
690		− }
691		−
692	702	return 0;
693	703	}
694	704
@@ -715,9 +725,15 @@ static void super_written(struct bio *bio) · 20 unmodified lines
715	725	struct mddev *mddev = rdev->mddev;
716	726
717	727	if (bio->bi_error) {
718		− printk("md: super_written gets error=%d\n", bio->bi_error);
	728	+ pr_err("md: super_written gets error=%d\n", bio->bi_error);
719	729	md_error(mddev, rdev);
720		− }
	730	+ if (!test_bit(Faulty, &rdev->flags)
	731	+ && (bio->bi_opf & MD_FAILFAST)) {
	732	+ set_bit(MD_SB_NEED_REWRITE, &mddev->sb_flags);
	733	+ set_bit(LastDev, &rdev->flags);
	734	+ }
	735	+ } else
	736	+ clear_bit(LastDev, &rdev->flags);
721	737
722	738	if (atomic_dec_and_test(&mddev->pending_writes))
723	739	wake_up(&mddev->sb_wait);
@@ -734,7 +750,13 @@ void md_super_write(struct mddev mddev, struct md_rdev rdev, · 10 unmodified lines
734	750	* if zero is reached.
735	751	* If an error occurred, call md_error
736	752	*/
737		− struct bio *bio = bio_alloc_mddev(GFP_NOIO, 1, mddev);
	753	+ struct bio *bio;
	754	+ int ff = 0;
	755	+
	756	+ if (test_bit(Faulty, &rdev->flags))
	757	+ return;
	758	+
	759	+ bio = bio_alloc_mddev(GFP_NOIO, 1, mddev);
738	760
739	761	atomic_inc(&rdev->nr_pending);
740	762
@@ -743,16 +765,24 @@ void md_super_write(struct mddev mddev, struct md_rdev rdev, · 2 unmodified lines
743	765	bio_add_page(bio, page, size, 0);
744	766	bio->bi_private = rdev;
745	767	bio->bi_end_io = super_written;
746		− bio->bi_opf = REQ_OP_WRITE \| REQ_PREFLUSH \| REQ_FUA;
	768	+
	769	+ if (test_bit(MD_FAILFAST_SUPPORTED, &mddev->flags) &&
	770	+ test_bit(FailFast, &rdev->flags) &&
	771	+ !test_bit(LastDev, &rdev->flags))
	772	+ ff = MD_FAILFAST;
	773	+ bio->bi_opf = REQ_OP_WRITE \| REQ_PREFLUSH \| REQ_FUA \| ff;
747	774
748	775	atomic_inc(&mddev->pending_writes);
749	776	submit_bio(bio);
750	777	}
751	778
752		−void md_super_wait(struct mddev *mddev)
	779	+int md_super_wait(struct mddev *mddev)
753	780	{
754	781	/* wait for all superblock writes that were scheduled to complete */
755	782	wait_event(mddev->sb_wait, atomic_read(&mddev->pending_writes)==0);
	783	+ if (test_and_clear_bit(MD_SB_NEED_REWRITE, &mddev->sb_flags))
	784	+ return -EAGAIN;
	785	+ return 0;
756	786	}
757	787
758	788	int sync_page_io(struct md_rdev *rdev, sector_t sector, int size,
@@ -795,8 +825,8 @@ static int read_disk_sb(struct md_rdev *rdev, int size) · 36 unmodified lines
795	825	return 0;
796	826
797	827	fail:
798		− printk(KERN_WARNING "md: disabled device %s, could not read superblock.\n",
799		− bdevname(rdev->bdev,b));
	828	+ pr_err("md: disabled device %s, could not read superblock.\n",
	829	+ bdevname(rdev->bdev,b));
800	830	return -EINVAL;
801	831	}
802	832
@@ -818,7 +848,6 @@ static int sb_equal(mdp_super_t sb1, mdp_super_t sb2) · 15 unmodified lines
818	848
819	849	if (!tmp1 \|\| !tmp2) {
820	850	ret = 0;
821		− printk(KERN_INFO "md.c sb_equal(): failed to allocate memory!\n");
822	851	goto abort;
823	852	}
824	853
@@ -932,7 +961,7 @@ int md_check_no_bitmap(struct mddev *mddev) · 107 unmodified lines
932	961	{
933	962	if (!mddev->bitmap_info.file && !mddev->bitmap_info.offset)
934	963	return 0;
935		− printk(KERN_ERR "%s: bitmaps are not supported for %s\n",
	964	+ pr_warn("%s: bitmaps are not supported for %s\n",
936	965	mdname(mddev), mddev->pers->name);
937	966	return 1;
938	967	}
@@ -956,7 +985,8 @@ static int super_90_load(struct md_rdev rdev, struct md_rdev refdev, int minor · 17 unmodified lines
956	985	rdev->sb_start = calc_dev_sboffset(rdev);
957	986
958	987	ret = read_disk_sb(rdev, MD_SB_BYTES);
959		− if (ret) return ret;
	988	+ if (ret)
	989	+ return ret;
960	990
961	991	ret = -EINVAL;
962	992
@@ -964,17 +994,15 @@ static int super_90_load(struct md_rdev rdev, struct md_rdev refdev, int minor · 1 unmodified line
964	994	sb = page_address(rdev->sb_page);
965	995
966	996	if (sb->md_magic != MD_SB_MAGIC) {
967		− printk(KERN_ERR "md: invalid raid superblock magic on %s\n",
968		− b);
	997	+ pr_warn("md: invalid raid superblock magic on %s\n", b);
969	998	goto abort;
970	999	}
971	1000
972	1001	if (sb->major_version != 0 \|\|
973	1002	sb->minor_version < 90 \|\|
974	1003	sb->minor_version > 91) {
975		− printk(KERN_WARNING "Bad version number %d.%d on %s\n",
976		− sb->major_version, sb->minor_version,
977		− b);
	1004	+ pr_warn("Bad version number %d.%d on %s\n",
	1005	+ sb->major_version, sb->minor_version, b);
978	1006	goto abort;
979	1007	}
980	1008
@@ -982,8 +1010,7 @@ static int super_90_load(struct md_rdev rdev, struct md_rdev refdev, int minor · 1 unmodified line
982	1010	goto abort;
983	1011
984	1012	if (md_csum_fold(calc_sb_csum(sb)) != md_csum_fold(sb->sb_csum)) {
985		− printk(KERN_WARNING "md: invalid superblock checksum on %s\n",
986		− b);
	1013	+ pr_warn("md: invalid superblock checksum on %s\n", b);
987	1014	goto abort;
988	1015	}
989	1016
@@ -1004,14 +1031,13 @@ static int super_90_load(struct md_rdev rdev, struct md_rdev refdev, int minor · 14 unmodified lines
1004	1031	__u64 ev1, ev2;
1005	1032	mdp_super_t *refsb = page_address(refdev->sb_page);
1006	1033	if (!uuid_equal(refsb, sb)) {
1007		− printk(KERN_WARNING "md: %s has different UUID to %s\n",
	1034	+ pr_warn("md: %s has different UUID to %s\n",
1008	1035	b, bdevname(refdev->bdev,b2));
1009	1036	goto abort;
1010	1037	}
1011	1038	if (!sb_equal(refsb, sb)) {
1012		− printk(KERN_WARNING "md: %s has same UUID"
1013		− " but different superblock to %s\n",
1014		− b, bdevname(refdev->bdev, b2));
	1039	+ pr_warn("md: %s has same UUID but different superblock to %s\n",
	1040	+ b, bdevname(refdev->bdev, b2));
1015	1041	goto abort;
1016	1042	}
1017	1043	ev1 = md_event(sb);
@@ -1158,6 +1184,8 @@ static int super_90_validate(struct mddev mddev, struct md_rdev rdev) · 140 unmodified lines
1158	1184	}
1159	1185	if (desc->state & (1<<MD_DISK_WRITEMOSTLY))
1160	1186	set_bit(WriteMostly, &rdev->flags);
	1187	+ if (desc->state & (1<<MD_DISK_FAILFAST))
	1188	+ set_bit(FailFast, &rdev->flags);
1161	1189	} else /* MULTIPATH are always insync */
1162	1190	set_bit(In_sync, &rdev->flags);
1163	1191	return 0;
@@ -1283,6 +1311,8 @@ static void super_90_sync(struct mddev mddev, struct md_rdev rdev) · 119 unmodified lines
1283	1311	}
1284	1312	if (test_bit(WriteMostly, &rdev2->flags))
1285	1313	d->state \|= (1<<MD_DISK_WRITEMOSTLY);
	1314	+ if (test_bit(FailFast, &rdev2->flags))
	1315	+ d->state \|= (1<<MD_DISK_FAILFAST);
1286	1316	}
1287	1317	/* now set the "removed" and "faulty" bits on any missing devices */
1288	1318	for (i=0 ; i < mddev->raid_disks ; i++) {
@@ -1324,9 +1354,10 @@ super_90_rdev_size_change(struct md_rdev *rdev, sector_t num_sectors) · 35 unmodified lines
1324	1354	if (IS_ENABLED(CONFIG_LBDAF) && (u64)num_sectors >= (2ULL << 32) &&
1325	1355	rdev->mddev->level >= 1)
1326	1356	num_sectors = (sector_t)(2ULL << 32) - 2;
1327		− md_super_write(rdev->mddev, rdev, rdev->sb_start, rdev->sb_size,
	1357	+ do {
	1358	+ md_super_write(rdev->mddev, rdev, rdev->sb_start, rdev->sb_size,
1328	1359	rdev->sb_page);
1329		− md_super_wait(rdev->mddev);
	1360	+ } while (md_super_wait(rdev->mddev) < 0);
1330	1361	return num_sectors;
1331	1362	}
1332	1363
@@ -1413,13 +1444,13 @@ static int super_1_load(struct md_rdev rdev, struct md_rdev refdev, int minor_ · 80 unmodified lines
1413	1444	return -EINVAL;
1414	1445
1415	1446	if (calc_sb_1_csum(sb) != sb->sb_csum) {
1416		− printk("md: invalid superblock checksum on %s\n",
	1447	+ pr_warn("md: invalid superblock checksum on %s\n",
1417	1448	bdevname(rdev->bdev,b));
1418	1449	return -EINVAL;
1419	1450	}
1420	1451	if (le64_to_cpu(sb->data_size) < 10) {
1421		− printk("md: data_size too small on %s\n",
1422		− bdevname(rdev->bdev,b));
	1452	+ pr_warn("md: data_size too small on %s\n",
	1453	+ bdevname(rdev->bdev,b));
1423	1454	return -EINVAL;
1424	1455	}
1425	1456	if (sb->pad0 \|\|
@@ -1503,8 +1534,7 @@ static int super_1_load(struct md_rdev rdev, struct md_rdev refdev, int minor_ · 77 unmodified lines
1503	1534	sb->level != refsb->level \|\|
1504	1535	sb->layout != refsb->layout \|\|
1505	1536	sb->chunksize != refsb->chunksize) {
1506		− printk(KERN_WARNING "md: %s has strangely different"
1507		− " superblock to %s\n",
	1537	+ pr_warn("md: %s has strangely different superblock to %s\n",
1508	1538	bdevname(rdev->bdev,b),
1509	1539	bdevname(refdev->bdev,b2));
1510	1540	return -EINVAL;
@@ -1646,8 +1676,7 @@ static int super_1_validate(struct mddev mddev, struct md_rdev rdev) · 135 unmodified lines
1646	1676	case MD_DISK_ROLE_JOURNAL: /* journal device */
1647	1677	if (!(le32_to_cpu(sb->feature_map) & MD_FEATURE_JOURNAL)) {
1648	1678	/* journal device without journal feature */
1649		− printk(KERN_WARNING
1650		− "md: journal device provided without journal feature, ignoring the device\n");
	1679	+ pr_warn("md: journal device provided without journal feature, ignoring the device\n");
1651	1680	return -EINVAL;
1652	1681	}
1653	1682	set_bit(Journal, &rdev->flags);
@@ -1669,6 +1698,8 @@ static int super_1_validate(struct mddev mddev, struct md_rdev rdev) · 15 unmodified lines
1669	1698	}
1670	1699	if (sb->devflags & WriteMostly1)
1671	1700	set_bit(WriteMostly, &rdev->flags);
	1701	+ if (sb->devflags & FailFast1)
	1702	+ set_bit(FailFast, &rdev->flags);
1672	1703	if (le32_to_cpu(sb->feature_map) & MD_FEATURE_REPLACEMENT)
1673	1704	set_bit(Replacement, &rdev->flags);
1674	1705	} else /* MULTIPATH are always insync */
@@ -1707,6 +1738,10 @@ static void super_1_sync(struct mddev mddev, struct md_rdev rdev) · 32 unmodified lines
1707	1738	sb->chunksize = cpu_to_le32(mddev->chunk_sectors);
1708	1739	sb->level = cpu_to_le32(mddev->level);
1709	1740	sb->layout = cpu_to_le32(mddev->layout);
	1741	+ if (test_bit(FailFast, &rdev->flags))
	1742	+ sb->devflags \|= FailFast1;
	1743	+ else
	1744	+ sb->devflags &= ~FailFast1;
1710	1745
1711	1746	if (test_bit(WriteMostly, &rdev->flags))
1712	1747	sb->devflags \|= WriteMostly1;
@@ -1863,9 +1898,10 @@ super_1_rdev_size_change(struct md_rdev *rdev, sector_t num_sectors) · 150 unmodified lines
1863	1898	sb->data_size = cpu_to_le64(num_sectors);
1864	1899	sb->super_offset = rdev->sb_start;
1865	1900	sb->sb_csum = calc_sb_1_csum(sb);
1866		− md_super_write(rdev->mddev, rdev, rdev->sb_start, rdev->sb_size,
1867		− rdev->sb_page);
1868		− md_super_wait(rdev->mddev);
	1901	+ do {
	1902	+ md_super_write(rdev->mddev, rdev, rdev->sb_start, rdev->sb_size,
	1903	+ rdev->sb_page);
	1904	+ } while (md_super_wait(rdev->mddev) < 0);
1869	1905	return num_sectors;
1870	1906
1871	1907	}
@@ -2004,9 +2040,9 @@ int md_integrity_register(struct mddev *mddev) · 132 unmodified lines
2004	2040	blk_integrity_register(mddev->gendisk,
2005	2041	bdev_get_integrity(reference->bdev));
2006	2042
2007		− printk(KERN_NOTICE "md: data integrity enabled on %s\n", mdname(mddev));
	2043	+ pr_debug("md: data integrity enabled on %s\n", mdname(mddev));
2008	2044	if (bioset_integrity_create(mddev->bio_set, BIO_POOL_SIZE)) {
2009		− printk(KERN_ERR "md: failed to create integrity pool for %s\n",
	2045	+ pr_err("md: failed to create integrity pool for %s\n",
2010	2046	mdname(mddev));
2011	2047	return -EINVAL;
2012	2048	}
@@ -2034,8 +2070,8 @@ int md_integrity_add_rdev(struct md_rdev rdev, struct mddev mddev) · 21 unmodified lines
2034	2070	return 0;
2035	2071
2036	2072	if (blk_integrity_compare(mddev->gendisk, rdev->bdev->bd_disk) != 0) {
2037		− printk(KERN_NOTICE "%s: incompatible integrity profile for %s\n",
2038		− mdname(mddev), bdevname(rdev->bdev, name));
	2073	+ pr_err("%s: incompatible integrity profile for %s\n",
	2074	+ mdname(mddev), bdevname(rdev->bdev, name));
2039	2075	return -ENXIO;
2040	2076	}
2041	2077
@@ -2089,15 +2125,15 @@ static int bind_rdev_to_array(struct md_rdev rdev, struct mddev mddev) · 47 unmodified lines
2089	2125	rcu_read_unlock();
2090	2126	if (!test_bit(Journal, &rdev->flags) &&
2091	2127	mddev->max_disks && rdev->desc_nr >= mddev->max_disks) {
2092		− printk(KERN_WARNING "md: %s: array is limited to %d devices\n",
2093		− mdname(mddev), mddev->max_disks);
	2128	+ pr_warn("md: %s: array is limited to %d devices\n",
	2129	+ mdname(mddev), mddev->max_disks);
2094	2130	return -EBUSY;
2095	2131	}
2096	2132	bdevname(rdev->bdev,b);
2097	2133	strreplace(b, '/', '!');
2098	2134
2099	2135	rdev->mddev = mddev;
2100		− printk(KERN_INFO "md: bind<%s>\n", b);
	2136	+ pr_debug("md: bind<%s>\n", b);
2101	2137
2102	2138	if ((err = kobject_add(&rdev->kobj, &mddev->kobj, "dev-%s", b)))
2103	2139	goto fail;
@@ -2116,8 +2152,8 @@ static int bind_rdev_to_array(struct md_rdev rdev, struct mddev mddev) · 12 unmodified lines
2116	2152	return 0;
2117	2153
2118	2154	fail:
2119		− printk(KERN_WARNING "md: failed to register dev-%s for %s\n",
2120		− b, mdname(mddev));
	2155	+ pr_warn("md: failed to register dev-%s for %s\n",
	2156	+ b, mdname(mddev));
2121	2157	return err;
2122	2158	}
2123	2159
@@ -2134,7 +2170,7 @@ static void unbind_rdev_from_array(struct md_rdev *rdev) · 10 unmodified lines
2134	2170
2135	2171	bd_unlink_disk_holder(rdev->bdev, rdev->mddev->gendisk);
2136	2172	list_del_rcu(&rdev->same_set);
2137		− printk(KERN_INFO "md: unbind<%s>\n", bdevname(rdev->bdev,b));
	2173	+ pr_debug("md: unbind<%s>\n", bdevname(rdev->bdev,b));
2138	2174	rdev->mddev = NULL;
2139	2175	sysfs_remove_link(&rdev->kobj, "block");
2140	2176	sysfs_put(rdev->sysfs_state);
@@ -2164,8 +2200,7 @@ static int lock_rdev(struct md_rdev *rdev, dev_t dev, int shared) · 23 unmodified lines
2164	2200	bdev = blkdev_get_by_dev(dev, FMODE_READ\|FMODE_WRITE\|FMODE_EXCL,
2165	2201	shared ? (struct md_rdev *)lock_rdev : rdev);
2166	2202	if (IS_ERR(bdev)) {
2167		− printk(KERN_ERR "md: could not open %s.\n",
2168		− __bdevname(dev, b));
	2203	+ pr_warn("md: could not open %s.\n", __bdevname(dev, b));
2169	2204	return PTR_ERR(bdev);
2170	2205	}
2171	2206	rdev->bdev = bdev;
@@ -2185,8 +2220,7 @@ static void export_rdev(struct md_rdev *rdev) · 13 unmodified lines
2185	2220	{
2186	2221	char b[BDEVNAME_SIZE];
2187	2222
2188		− printk(KERN_INFO "md: export_rdev(%s)\n",
2189		− bdevname(rdev->bdev,b));
	2223	+ pr_debug("md: export_rdev(%s)\n", bdevname(rdev->bdev,b));
2190	2224	md_rdev_clear(rdev);
2191	2225	#ifndef MODULE
2192	2226	if (test_bit(AutoDetected, &rdev->flags))
@@ -2288,24 +2322,24 @@ void md_update_sb(struct mddev *mddev, int force_change) · 95 unmodified lines
2288	2322
2289	2323	if (mddev->ro) {
2290	2324	if (force_change)
2291		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
	2325	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
2292	2326	return;
2293	2327	}
2294	2328
2295	2329	repeat:
2296	2330	if (mddev_is_clustered(mddev)) {
2297		− if (test_and_clear_bit(MD_CHANGE_DEVS, &mddev->flags))
	2331	+ if (test_and_clear_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags))
2298	2332	force_change = 1;
2299		− if (test_and_clear_bit(MD_CHANGE_CLEAN, &mddev->flags))
	2333	+ if (test_and_clear_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags))
2300	2334	nospares = 1;
2301	2335	ret = md_cluster_ops->metadata_update_start(mddev);
2302	2336	/* Has someone else has updated the sb */
2303	2337	if (!does_sb_need_changing(mddev)) {
2304	2338	if (ret == 0)
2305	2339	md_cluster_ops->metadata_update_cancel(mddev);
2306		− bit_clear_unless(&mddev->flags, BIT(MD_CHANGE_PENDING),
2307		− BIT(MD_CHANGE_DEVS) \|
2308		− BIT(MD_CHANGE_CLEAN));
	2340	+ bit_clear_unless(&mddev->sb_flags, BIT(MD_SB_CHANGE_PENDING),
	2341	+ BIT(MD_SB_CHANGE_DEVS) \|
	2342	+ BIT(MD_SB_CHANGE_CLEAN));
2309	2343	return;
2310	2344	}
2311	2345	}
@@ -2321,10 +2355,10 @@ repeat: · 9 unmodified lines
2321	2355
2322	2356	}
2323	2357	if (!mddev->persistent) {
2324		− clear_bit(MD_CHANGE_CLEAN, &mddev->flags);
2325		− clear_bit(MD_CHANGE_DEVS, &mddev->flags);
	2358	+ clear_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
	2359	+ clear_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
2326	2360	if (!mddev->external) {
2327		− clear_bit(MD_CHANGE_PENDING, &mddev->flags);
	2361	+ clear_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
2328	2362	rdev_for_each(rdev, mddev) {
2329	2363	if (rdev->badblocks.changed) {
2330	2364	rdev->badblocks.changed = 0;
@@ -2344,9 +2378,9 @@ repeat: · 13 unmodified lines
2344	2378
2345	2379	mddev->utime = ktime_get_real_seconds();
2346	2380
2347		− if (test_and_clear_bit(MD_CHANGE_DEVS, &mddev->flags))
	2381	+ if (test_and_clear_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags))
2348	2382	force_change = 1;
2349		− if (test_and_clear_bit(MD_CHANGE_CLEAN, &mddev->flags))
	2383	+ if (test_and_clear_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags))
2350	2384	/* just a clean<-> dirty transition, possibly leave spares alone,
2351	2385	* though if events isn't the right even/odd, we will have to do
2352	2386	* spares after all
@@ -2402,6 +2436,9 @@ repeat: · 49 unmodified lines
2402	2436	pr_debug("md: updating %s RAID superblock on device (in sync %d)\n",
2403	2437	mdname(mddev), mddev->in_sync);
2404	2438
	2439	+ if (mddev->queue)
	2440	+ blk_add_trace_msg(mddev->queue, "md md_update_sb");
	2441	+rewrite:
2405	2442	bitmap_update_sb(mddev->bitmap);
2406	2443	rdev_for_each(rdev, mddev) {
2407	2444	char b[BDEVNAME_SIZE];
@@ -2433,15 +2470,16 @@ repeat: · 25 unmodified lines
2433	2470	/* only need to write one superblock... */
2434	2471	break;
2435	2472	}
2436		− md_super_wait(mddev);
2437		− /* if there was a failure, MD_CHANGE_DEVS was set, and we re-write super */
	2473	+ if (md_super_wait(mddev) < 0)
	2474	+ goto rewrite;
	2475	+ /* if there was a failure, MD_SB_CHANGE_DEVS was set, and we re-write super */
2438	2476
2439	2477	if (mddev_is_clustered(mddev) && ret == 0)
2440	2478	md_cluster_ops->metadata_update_finish(mddev);
2441	2479
2442	2480	if (mddev->in_sync != sync_req \|\|
2443		− !bit_clear_unless(&mddev->flags, BIT(MD_CHANGE_PENDING),
2444		− BIT(MD_CHANGE_DEVS) \| BIT(MD_CHANGE_CLEAN)))
	2481	+ !bit_clear_unless(&mddev->sb_flags, BIT(MD_SB_CHANGE_PENDING),
	2482	+ BIT(MD_SB_CHANGE_DEVS) \| BIT(MD_SB_CHANGE_CLEAN)))
2445	2483	/* have to write it out again */
2446	2484	goto repeat;
2447	2485	wake_up(&mddev->sb_wait);
@@ -2485,7 +2523,7 @@ static int add_bound_rdev(struct md_rdev *rdev) · 37 unmodified lines
2485	2523	}
2486	2524	sysfs_notify_dirent_safe(rdev->sysfs_state);
2487	2525
2488		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
	2526	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
2489	2527	if (mddev->degraded)
2490	2528	set_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
2491	2529	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
@@ -2523,51 +2561,41 @@ struct rdev_sysfs_entry { · 31 unmodified lines
2523	2561	static ssize_t
2524	2562	state_show(struct md_rdev rdev, char page)
2525	2563	{
2526		− char *sep = "";
	2564	+ char *sep = ",";
2527	2565	size_t len = 0;
2528	2566	unsigned long flags = ACCESS_ONCE(rdev->flags);
2529	2567
2530	2568	if (test_bit(Faulty, &flags) \|\|
2531		− rdev->badblocks.unacked_exist) {
2532		− len+= sprintf(page+len, "%sfaulty",sep);
2533		− sep = ",";
2534		− }
2535		− if (test_bit(In_sync, &flags)) {
2536		− len += sprintf(page+len, "%sin_sync",sep);
2537		− sep = ",";
2538		− }
2539		− if (test_bit(Journal, &flags)) {
2540		− len += sprintf(page+len, "%sjournal",sep);
2541		− sep = ",";
2542		− }
2543		− if (test_bit(WriteMostly, &flags)) {
2544		− len += sprintf(page+len, "%swrite_mostly",sep);
2545		− sep = ",";
2546		− }
	2569	+ (!test_bit(ExternalBbl, &flags) &&
	2570	+ rdev->badblocks.unacked_exist))
	2571	+ len += sprintf(page+len, "faulty%s", sep);
	2572	+ if (test_bit(In_sync, &flags))
	2573	+ len += sprintf(page+len, "in_sync%s", sep);
	2574	+ if (test_bit(Journal, &flags))
	2575	+ len += sprintf(page+len, "journal%s", sep);
	2576	+ if (test_bit(WriteMostly, &flags))
	2577	+ len += sprintf(page+len, "write_mostly%s", sep);
2547	2578	if (test_bit(Blocked, &flags) \|\|
2548	2579	(rdev->badblocks.unacked_exist
2549		− && !test_bit(Faulty, &flags))) {
2550		− len += sprintf(page+len, "%sblocked", sep);
2551		− sep = ",";
2552		− }
	2580	+ && !test_bit(Faulty, &flags)))
	2581	+ len += sprintf(page+len, "blocked%s", sep);
2553	2582	if (!test_bit(Faulty, &flags) &&
2554	2583	!test_bit(Journal, &flags) &&
2555		− !test_bit(In_sync, &flags)) {
2556		− len += sprintf(page+len, "%sspare", sep);
2557		− sep = ",";
2558		− }
2559		− if (test_bit(WriteErrorSeen, &flags)) {
2560		− len += sprintf(page+len, "%swrite_error", sep);
2561		− sep = ",";
2562		− }
2563		− if (test_bit(WantReplacement, &flags)) {
2564		− len += sprintf(page+len, "%swant_replacement", sep);
2565		− sep = ",";
2566		− }
2567		− if (test_bit(Replacement, &flags)) {
2568		− len += sprintf(page+len, "%sreplacement", sep);
2569		− sep = ",";
2570		− }
	2584	+ !test_bit(In_sync, &flags))
	2585	+ len += sprintf(page+len, "spare%s", sep);
	2586	+ if (test_bit(WriteErrorSeen, &flags))
	2587	+ len += sprintf(page+len, "write_error%s", sep);
	2588	+ if (test_bit(WantReplacement, &flags))
	2589	+ len += sprintf(page+len, "want_replacement%s", sep);
	2590	+ if (test_bit(Replacement, &flags))
	2591	+ len += sprintf(page+len, "replacement%s", sep);
	2592	+ if (test_bit(ExternalBbl, &flags))
	2593	+ len += sprintf(page+len, "external_bbl%s", sep);
	2594	+ if (test_bit(FailFast, &flags))
	2595	+ len += sprintf(page+len, "failfast%s", sep);
	2596	+
	2597	+ if (len)
	2598	+ len -= strlen(sep);
2571	2599
2572	2600	return len+sprintf(page+len, "\n");
2573	2601	}
@@ -2587,6 +2615,7 @@ state_store(struct md_rdev rdev, const char buf, size_t len) · 13 unmodified lines
2587	2615	* so that it gets rebuilt based on bitmap
2588	2616	* write_error - sets WriteErrorSeen
2589	2617	* -write_error - clears WriteErrorSeen
	2618	+ * {,-}failfast - set/clear FailFast
2590	2619	*/
2591	2620	int err = -EINVAL;
2592	2621	if (cmd_match(buf, "faulty") && rdev->mddev->pers) {
@@ -2610,8 +2639,10 @@ state_store(struct md_rdev rdev, const char buf, size_t len) · 17 unmodified lines
2610	2639
2611	2640	if (err == 0) {
2612	2641	md_kick_rdev_from_array(rdev);
2613		− if (mddev->pers)
2614		− md_update_sb(mddev, 1);
	2642	+ if (mddev->pers) {
	2643	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
	2644	+ md_wakeup_thread(mddev->thread);
	2645	+ }
2615	2646	md_new_event(mddev);
2616	2647	}
2617	2648	}
@@ -2626,6 +2657,7 @@ state_store(struct md_rdev rdev, const char buf, size_t len) · 8 unmodified lines
2626	2657	err = 0;
2627	2658	} else if (cmd_match(buf, "-blocked")) {
2628	2659	if (!test_bit(Faulty, &rdev->flags) &&
	2660	+ !test_bit(ExternalBbl, &rdev->flags) &&
2629	2661	rdev->badblocks.unacked_exist) {
2630	2662	/* metadata handler doesn't understand badblocks,
2631	2663	* so we need to fail the device
@@ -2642,6 +2674,12 @@ state_store(struct md_rdev rdev, const char buf, size_t len) · 10 unmodified lines
2642	2674	} else if (cmd_match(buf, "insync") && rdev->raid_disk == -1) {
2643	2675	set_bit(In_sync, &rdev->flags);
2644	2676	err = 0;
	2677	+ } else if (cmd_match(buf, "failfast")) {
	2678	+ set_bit(FailFast, &rdev->flags);
	2679	+ err = 0;
	2680	+ } else if (cmd_match(buf, "-failfast")) {
	2681	+ clear_bit(FailFast, &rdev->flags);
	2682	+ err = 0;
2645	2683	} else if (cmd_match(buf, "-insync") && rdev->raid_disk >= 0 &&
2646	2684	!test_bit(Journal, &rdev->flags)) {
2647	2685	if (rdev->mddev->pers == NULL) {
@@ -2708,6 +2746,13 @@ state_store(struct md_rdev rdev, const char buf, size_t len) · 60 unmodified lines
2708	2746	}
2709	2747	} else
2710	2748	err = -EBUSY;
	2749	+ } else if (cmd_match(buf, "external_bbl") && (rdev->mddev->external)) {
	2750	+ set_bit(ExternalBbl, &rdev->flags);
	2751	+ rdev->badblocks.shift = 0;
	2752	+ err = 0;
	2753	+ } else if (cmd_match(buf, "-external_bbl") && (rdev->mddev->external)) {
	2754	+ clear_bit(ExternalBbl, &rdev->flags);
	2755	+ err = 0;
2711	2756	}
2712	2757	if (!err)
2713	2758	sysfs_notify_dirent_safe(rdev->sysfs_state);
@@ -3211,10 +3256,8 @@ static struct md_rdev *md_import_device(dev_t newdev, int super_format, int supe · 497 unmodified lines
3211	3256	sector_t size;
3212	3257
3213	3258	rdev = kzalloc(sizeof(*rdev), GFP_KERNEL);
3214		− if (!rdev) {
3215		− printk(KERN_ERR "md: could not alloc mem for new device!\n");
	3259	+ if (!rdev)
3216	3260	return ERR_PTR(-ENOMEM);
3217		− }
3218	3261
3219	3262	err = md_rdev_init(rdev);
3220	3263	if (err)
@@ -3231,8 +3274,7 @@ static struct md_rdev *md_import_device(dev_t newdev, int super_format, int supe · 10 unmodified lines
3231	3274
3232	3275	size = i_size_read(rdev->bdev->bd_inode) >> BLOCK_SIZE_BITS;
3233	3276	if (!size) {
3234		− printk(KERN_WARNING
3235		− "md: %s has zero or unknown size, marking faulty!\n",
	3277	+ pr_warn("md: %s has zero or unknown size, marking faulty!\n",
3236	3278	bdevname(rdev->bdev,b));
3237	3279	err = -EINVAL;
3238	3280	goto abort_free;
@@ -3242,16 +3284,13 @@ static struct md_rdev *md_import_device(dev_t newdev, int super_format, int supe · 3 unmodified lines
3242	3284	err = super_types[super_format].
3243	3285	load_super(rdev, NULL, super_minor);
3244	3286	if (err == -EINVAL) {
3245		− printk(KERN_WARNING
3246		− "md: %s does not have a valid v%d.%d "
3247		− "superblock, not importing!\n",
	3287	+ pr_warn("md: %s does not have a valid v%d.%d superblock, not importing!\n",
3248	3288	bdevname(rdev->bdev,b),
3249		− super_format, super_minor);
	3289	+ super_format, super_minor);
3250	3290	goto abort_free;
3251	3291	}
3252	3292	if (err < 0) {
3253		− printk(KERN_WARNING
3254		− "md: could not read %s's sb, not importing!\n",
	3293	+ pr_warn("md: could not read %s's sb, not importing!\n",
3255	3294	bdevname(rdev->bdev,b));
3256	3295	goto abort_free;
3257	3296	}
@@ -3287,9 +3326,7 @@ static void analyze_sbs(struct mddev *mddev) · 29 unmodified lines
3287	3326	case 0:
3288	3327	break;
3289	3328	default:
3290		− printk( KERN_ERR \
3291		− "md: fatal superblock inconsistency in %s"
3292		− " -- removing from array\n",
	3329	+ pr_warn("md: fatal superblock inconsistency in %s -- removing from array\n",
3293	3330	bdevname(rdev->bdev,b));
3294	3331	md_kick_rdev_from_array(rdev);
3295	3332	}
@@ -3302,18 +3339,16 @@ static void analyze_sbs(struct mddev *mddev) · 6 unmodified lines
3302	3339	if (mddev->max_disks &&
3303	3340	(rdev->desc_nr >= mddev->max_disks \|\|
3304	3341	i > mddev->max_disks)) {
3305		− printk(KERN_WARNING
3306		− "md: %s: %s: only %d devices permitted\n",
3307		− mdname(mddev), bdevname(rdev->bdev, b),
3308		− mddev->max_disks);
	3342	+ pr_warn("md: %s: %s: only %d devices permitted\n",
	3343	+ mdname(mddev), bdevname(rdev->bdev, b),
	3344	+ mddev->max_disks);
3309	3345	md_kick_rdev_from_array(rdev);
3310	3346	continue;
3311	3347	}
3312	3348	if (rdev != freshest) {
3313	3349	if (super_types[mddev->major_version].
3314	3350	validate_super(mddev, rdev)) {
3315		− printk(KERN_WARNING "md: kicking non-fresh %s"
3316		− " from array!\n",
	3351	+ pr_warn("md: kicking non-fresh %s from array!\n",
3317	3352	bdevname(rdev->bdev,b));
3318	3353	md_kick_rdev_from_array(rdev);
3319	3354	continue;
@@ -3384,7 +3419,7 @@ safe_delay_store(struct mddev mddev, const char cbuf, size_t len) · 64 unmodified lines
3384	3419	unsigned long msec;
3385	3420
3386	3421	if (mddev_is_clustered(mddev)) {
3387		− pr_info("md: Safemode is disabled for clustered mode\n");
	3422	+ pr_warn("md: Safemode is disabled for clustered mode\n");
3388	3423	return -EINVAL;
3389	3424	}
3390	3425
@@ -3472,8 +3507,8 @@ level_store(struct mddev mddev, const char buf, size_t len) · 81 unmodified lines
3472	3507
3473	3508	rv = -EINVAL;
3474	3509	if (!mddev->pers->quiesce) {
3475		− printk(KERN_WARNING "md: %s: %s does not support online personality change\n",
3476		− mdname(mddev), mddev->pers->name);
	3510	+ pr_warn("md: %s: %s does not support online personality change\n",
	3511	+ mdname(mddev), mddev->pers->name);
3477	3512	goto out_unlock;
3478	3513	}
3479	3514
@@ -3491,7 +3526,7 @@ level_store(struct mddev mddev, const char buf, size_t len) · 11 unmodified lines
3491	3526	pers = find_pers(level, clevel);
3492	3527	if (!pers \|\| !try_module_get(pers->owner)) {
3493	3528	spin_unlock(&pers_lock);
3494		− printk(KERN_WARNING "md: personality %s not loaded\n", clevel);
	3529	+ pr_warn("md: personality %s not loaded\n", clevel);
3495	3530	rv = -EINVAL;
3496	3531	goto out_unlock;
3497	3532	}
@@ -3505,8 +3540,8 @@ level_store(struct mddev mddev, const char buf, size_t len) · 7 unmodified lines
3505	3540	}
3506	3541	if (!pers->takeover) {
3507	3542	module_put(pers->owner);
3508		− printk(KERN_WARNING "md: %s: %s does not support personality takeover\n",
3509		− mdname(mddev), clevel);
	3543	+ pr_warn("md: %s: %s does not support personality takeover\n",
	3544	+ mdname(mddev), clevel);
3510	3545	rv = -EINVAL;
3511	3546	goto out_unlock;
3512	3547	}
@@ -3526,8 +3561,8 @@ level_store(struct mddev mddev, const char buf, size_t len) · 13 unmodified lines
3526	3561	mddev->delta_disks = 0;
3527	3562	mddev->reshape_backwards = 0;
3528	3563	module_put(pers->owner);
3529		− printk(KERN_WARNING "md: %s: %s would not accept array\n",
3530		− mdname(mddev), clevel);
	3564	+ pr_warn("md: %s: %s would not accept array\n",
	3565	+ mdname(mddev), clevel);
3531	3566	rv = PTR_ERR(priv);
3532	3567	goto out_unlock;
3533	3568	}
@@ -3570,9 +3605,8 @@ level_store(struct mddev mddev, const char buf, size_t len) · 36 unmodified lines
3570	3605	pers->sync_request != NULL) {
3571	3606	/* need to add the md_redundancy_group */
3572	3607	if (sysfs_create_group(&mddev->kobj, &md_redundancy_group))
3573		− printk(KERN_WARNING
3574		− "md: cannot register extra attributes for %s\n",
3575		− mdname(mddev));
	3608	+ pr_warn("md: cannot register extra attributes for %s\n",
	3609	+ mdname(mddev));
3576	3610	mddev->sysfs_action = sysfs_get_dirent(mddev->kobj.sd, "sync_action");
3577	3611	}
3578	3612	if (oldpers->sync_request != NULL &&
@@ -3603,9 +3637,8 @@ level_store(struct mddev mddev, const char buf, size_t len) · 24 unmodified lines
3603	3637	clear_bit(In_sync, &rdev->flags);
3604	3638	else {
3605	3639	if (sysfs_link_rdev(mddev, rdev))
3606		− printk(KERN_WARNING "md: cannot register rd%d"
3607		− " for %s after level change\n",
3608		− rdev->raid_disk, mdname(mddev));
	3640	+ pr_warn("md: cannot register rd%d for %s after level change\n",
	3641	+ rdev->raid_disk, mdname(mddev));
3609	3642	}
3610	3643	}
3611	3644
@@ -3618,7 +3651,7 @@ level_store(struct mddev mddev, const char buf, size_t len) · 6 unmodified lines
3618	3651	}
3619	3652	blk_set_stacking_limits(&mddev->queue->limits);
3620	3653	pers->run(mddev);
3621		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
	3654	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
3622	3655	mddev_resume(mddev);
3623	3656	if (!mddev->thread)
3624	3657	md_update_sb(mddev, 1);
@@ -3813,7 +3846,7 @@ resync_start_store(struct mddev mddev, const char buf, size_t len) · 188 unmodified lines
3813	3846	if (!err) {
3814	3847	mddev->recovery_cp = n;
3815	3848	if (mddev->pers)
3816		− set_bit(MD_CHANGE_CLEAN, &mddev->flags);
	3849	+ set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
3817	3850	}
3818	3851	mddev_unlock(mddev);
3819	3852	return err ?: len;
@@ -3887,7 +3920,7 @@ array_state_show(struct mddev mddev, char page) · 67 unmodified lines
3887	3920	st = read_auto;
3888	3921	break;
3889	3922	case 0:
3890		− if (test_bit(MD_CHANGE_PENDING, &mddev->flags))
	3923	+ if (test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags))
3891	3924	st = write_pending;
3892	3925	else if (mddev->in_sync)
3893	3926	st = clean;
@@ -3925,7 +3958,8 @@ array_state_store(struct mddev mddev, const char buf, size_t len) · 31 unmodified lines
3925	3958	spin_lock(&mddev->lock);
3926	3959	if (st == active) {
3927	3960	restart_array(mddev);
3928		− clear_bit(MD_CHANGE_PENDING, &mddev->flags);
	3961	+ clear_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
	3962	+ md_wakeup_thread(mddev->thread);
3929	3963	wake_up(&mddev->sb_wait);
3930	3964	err = 0;
3931	3965	} else /* st == clean */ {
@@ -3935,7 +3969,7 @@ array_state_store(struct mddev mddev, const char buf, size_t len) · 3 unmodified lines
3935	3969	mddev->in_sync = 1;
3936	3970	if (mddev->safemode == 1)
3937	3971	mddev->safemode = 0;
3938		− set_bit(MD_CHANGE_CLEAN, &mddev->flags);
	3972	+ set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
3939	3973	}
3940	3974	err = 0;
3941	3975	} else
@@ -4001,7 +4035,7 @@ array_state_store(struct mddev mddev, const char buf, size_t len) · 59 unmodified lines
4001	4035	mddev->in_sync = 1;
4002	4036	if (mddev->safemode == 1)
4003	4037	mddev->safemode = 0;
4004		− set_bit(MD_CHANGE_CLEAN, &mddev->flags);
	4038	+ set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
4005	4039	}
4006	4040	err = 0;
4007	4041	} else
@@ -4015,7 +4049,7 @@ array_state_store(struct mddev mddev, const char buf, size_t len) · 7 unmodified lines
4015	4049	err = restart_array(mddev);
4016	4050	if (err)
4017	4051	break;
4018		− clear_bit(MD_CHANGE_PENDING, &mddev->flags);
	4052	+ clear_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
4019	4053	wake_up(&mddev->sb_wait);
4020	4054	err = 0;
4021	4055	} else {
@@ -5071,13 +5105,13 @@ static int md_alloc(dev_t dev, char *name) · 1049 unmodified lines
5071	5105	/* This isn't possible, but as kobject_init_and_add is marked
5072	5106	* __must_check, we must do something with the result
5073	5107	*/
5074		− printk(KERN_WARNING "md: cannot register %s/md - name in use\n",
5075		− disk->disk_name);
	5108	+ pr_debug("md: cannot register %s/md - name in use\n",
	5109	+ disk->disk_name);
5076	5110	error = 0;
5077	5111	}
5078	5112	if (mddev->kobj.sd &&
5079	5113	sysfs_create_group(&mddev->kobj, &md_bitmap_group))
5080		− printk(KERN_DEBUG "pointless warning\n");
	5114	+ pr_debug("pointless warning\n");
5081	5115	mutex_unlock(&mddev->open_mutex);
5082	5116	abort:
5083	5117	mutex_unlock(&disks_mutex);
@@ -5179,15 +5213,15 @@ int md_run(struct mddev *mddev) · 95 unmodified lines
5179	5213	if (mddev->dev_sectors &&
5180	5214	rdev->data_offset + mddev->dev_sectors
5181	5215	> rdev->sb_start) {
5182		− printk("md: %s: data overlaps metadata\n",
5183		− mdname(mddev));
	5216	+ pr_warn("md: %s: data overlaps metadata\n",
	5217	+ mdname(mddev));
5184	5218	return -EINVAL;
5185	5219	}
5186	5220	} else {
5187	5221	if (rdev->sb_start + rdev->sb_size/512
5188	5222	> rdev->data_offset) {
5189		− printk("md: %s: metadata overlaps data\n",
5190		− mdname(mddev));
	5223	+ pr_warn("md: %s: metadata overlaps data\n",
	5224	+ mdname(mddev));
5191	5225	return -EINVAL;
5192	5226	}
5193	5227	}
@@ -5202,11 +5236,11 @@ int md_run(struct mddev *mddev) · 8 unmodified lines
5202	5236	if (!pers \|\| !try_module_get(pers->owner)) {
5203	5237	spin_unlock(&pers_lock);
5204	5238	if (mddev->level != LEVEL_NONE)
5205		− printk(KERN_WARNING "md: personality for level %d is not loaded!\n",
5206		− mddev->level);
	5239	+ pr_warn("md: personality for level %d is not loaded!\n",
	5240	+ mddev->level);
5207	5241	else
5208		− printk(KERN_WARNING "md: personality for level %s is not loaded!\n",
5209		− mddev->clevel);
	5242	+ pr_warn("md: personality for level %s is not loaded!\n",
	5243	+ mddev->clevel);
5210	5244	return -EINVAL;
5211	5245	}
5212	5246	spin_unlock(&pers_lock);
@@ -5236,21 +5270,16 @@ int md_run(struct mddev *mddev) · 23 unmodified lines
5236	5270	if (rdev < rdev2 &&
5237	5271	rdev->bdev->bd_contains ==
5238	5272	rdev2->bdev->bd_contains) {
5239		− printk(KERN_WARNING
5240		− "%s: WARNING: %s appears to be"
5241		− " on the same physical disk as"
5242		− " %s.\n",
5243		− mdname(mddev),
5244		− bdevname(rdev->bdev,b),
5245		− bdevname(rdev2->bdev,b2));
	5273	+ pr_warn("%s: WARNING: %s appears to be on the same physical disk as %s.\n",
	5274	+ mdname(mddev),
	5275	+ bdevname(rdev->bdev,b),
	5276	+ bdevname(rdev2->bdev,b2));
5246	5277	warned = 1;
5247	5278	}
5248	5279	}
5249	5280
5250	5281	if (warned)
5251		− printk(KERN_WARNING
5252		− "True protection against single-disk"
5253		− " failure might be compromised.\n");
	5282	+ pr_warn("True protection against single-disk failure might be compromised.\n");
5254	5283	}
5255	5284
5256	5285	mddev->recovery = 0;
@@ -5264,14 +5293,14 @@ int md_run(struct mddev *mddev) · 7 unmodified lines
5264	5293
5265	5294	err = pers->run(mddev);
5266	5295	if (err)
5267		− printk(KERN_ERR "md: pers->run() failed ...\n");
	5296	+ pr_warn("md: pers->run() failed ...\n");
5268	5297	else if (pers->size(mddev, 0, 0) < mddev->array_sectors) {
5269		− WARN_ONCE(!mddev->external_size, "%s: default size too small,"
5270		− " but 'external_size' not in effect?\n", __func__);
5271		− printk(KERN_ERR
5272		− "md: invalid array_size %llu > default size %llu\n",
5273		− (unsigned long long)mddev->array_sectors / 2,
5274		− (unsigned long long)pers->size(mddev, 0, 0) / 2);
	5298	+ WARN_ONCE(!mddev->external_size,
	5299	+ "%s: default size too small, but 'external_size' not in effect?\n",
	5300	+ __func__);
	5301	+ pr_warn("md: invalid array_size %llu > default size %llu\n",
	5302	+ (unsigned long long)mddev->array_sectors / 2,
	5303	+ (unsigned long long)pers->size(mddev, 0, 0) / 2);
5275	5304	err = -EINVAL;
5276	5305	}
5277	5306	if (err == 0 && pers->sync_request &&
@@ -5281,8 +5310,8 @@ int md_run(struct mddev *mddev) · 3 unmodified lines
5281	5310	bitmap = bitmap_create(mddev, -1);
5282	5311	if (IS_ERR(bitmap)) {
5283	5312	err = PTR_ERR(bitmap);
5284		− printk(KERN_ERR "%s: failed to create bitmap (%d)\n",
5285		− mdname(mddev), err);
	5313	+ pr_warn("%s: failed to create bitmap (%d)\n",
	5314	+ mdname(mddev), err);
5286	5315	} else
5287	5316	mddev->bitmap = bitmap;
5288	5317
@@ -5318,9 +5347,8 @@ int md_run(struct mddev *mddev) · 29 unmodified lines
5318	5347	if (pers->sync_request) {
5319	5348	if (mddev->kobj.sd &&
5320	5349	sysfs_create_group(&mddev->kobj, &md_redundancy_group))
5321		− printk(KERN_WARNING
5322		− "md: cannot register extra attributes for %s\n",
5323		− mdname(mddev));
	5350	+ pr_warn("md: cannot register extra attributes for %s\n",
	5351	+ mdname(mddev));
5324	5352	mddev->sysfs_action = sysfs_get_dirent_safe(mddev->kobj.sd, "sync_action");
5325	5353	} else if (mddev->ro == 2) /* auto-readonly not meaningful */
5326	5354	mddev->ro = 0;
@@ -5350,7 +5378,7 @@ int md_run(struct mddev *mddev) · 23 unmodified lines
5350	5378	set_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
5351	5379	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
5352	5380
5353		− if (mddev->flags & MD_UPDATE_SB_FLAGS)
	5381	+ if (mddev->sb_flags)
5354	5382	md_update_sb(mddev, 0);
5355	5383
5356	5384	md_new_event(mddev);
@@ -5421,8 +5449,7 @@ static int restart_array(struct mddev *mddev) · 64 unmodified lines
5421	5449	mddev->safemode = 0;
5422	5450	mddev->ro = 0;
5423	5451	set_disk_ro(disk, 0);
5424		− printk(KERN_INFO "md: %s switched to read-write mode.\n",
5425		− mdname(mddev));
	5452	+ pr_debug("md: %s switched to read-write mode.\n", mdname(mddev));
5426	5453	/* Kick recovery or resync if necessary */
5427	5454	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
5428	5455	md_wakeup_thread(mddev->thread);
@@ -5446,6 +5473,7 @@ static void md_clean(struct mddev *mddev) · 17 unmodified lines
5446	5473	mddev->level = LEVEL_NONE;
5447	5474	mddev->clevel[0] = 0;
5448	5475	mddev->flags = 0;
	5476	+ mddev->sb_flags = 0;
5449	5477	mddev->ro = 0;
5450	5478	mddev->metadata_type[0] = 0;
5451	5479	mddev->chunk_sectors = 0;
@@ -5490,12 +5518,15 @@ static void __md_stop_writes(struct mddev *mddev) · 38 unmodified lines
5490	5518
5491	5519	del_timer_sync(&mddev->safemode_timer);
5492	5520
	5521	+ if (mddev->pers && mddev->pers->quiesce) {
	5522	+ mddev->pers->quiesce(mddev, 1);
	5523	+ mddev->pers->quiesce(mddev, 0);
	5524	+ }
5493	5525	bitmap_flush(mddev);
5494		− md_super_wait(mddev);
5495	5526
5496	5527	if (mddev->ro == 0 &&
5497	5528	((!mddev->in_sync && !mddev_is_clustered(mddev)) \|\|
5498		− (mddev->flags & MD_UPDATE_SB_FLAGS))) {
	5529	+ mddev->sb_flags)) {
5499	5530	/* mark array as shutdown cleanly */
5500	5531	if (!mddev_is_clustered(mddev))
5501	5532	mddev->in_sync = 1;
@@ -5516,8 +5547,8 @@ static void mddev_detach(struct mddev *mddev) · 14 unmodified lines
5516	5547	struct bitmap *bitmap = mddev->bitmap;
5517	5548	/* wait for behind writes to complete */
5518	5549	if (bitmap && atomic_read(&bitmap->behind_writes) > 0) {
5519		− printk(KERN_INFO "md:%s: behind writes in progress - waiting to stop.\n",
5520		− mdname(mddev));
	5550	+ pr_debug("md:%s: behind writes in progress - waiting to stop.\n",
	5551	+ mdname(mddev));
5521	5552	/* need to kick something here to make sure I/O goes? */
5522	5553	wait_event(bitmap->behind_wait,
5523	5554	atomic_read(&bitmap->behind_writes) == 0);
@@ -5578,20 +5609,20 @@ static int md_set_readonly(struct mddev mddev, struct block_device bdev) · 54 unmodified lines
5578	5609	* which will now never happen */
5579	5610	wake_up_process(mddev->sync_thread->tsk);
5580	5611
5581		− if (mddev->external && test_bit(MD_CHANGE_PENDING, &mddev->flags))
	5612	+ if (mddev->external && test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags))
5582	5613	return -EBUSY;
5583	5614	mddev_unlock(mddev);
5584	5615	wait_event(resync_wait, !test_bit(MD_RECOVERY_RUNNING,
5585	5616	&mddev->recovery));
5586	5617	wait_event(mddev->sb_wait,
5587		− !test_bit(MD_CHANGE_PENDING, &mddev->flags));
	5618	+ !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
5588	5619	mddev_lock_nointr(mddev);
5589	5620
5590	5621	mutex_lock(&mddev->open_mutex);
5591	5622	if ((mddev->pers && atomic_read(&mddev->openers) > !!bdev) \|\|
5592	5623	mddev->sync_thread \|\|
5593	5624	test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) {
5594		− printk("md: %s still in use.\n",mdname(mddev));
	5625	+ pr_warn("md: %s still in use.\n",mdname(mddev));
5595	5626	if (did_freeze) {
5596	5627	clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
5597	5628	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
@@ -5653,7 +5684,7 @@ static int do_md_stop(struct mddev *mddev, int mode, · 55 unmodified lines
5653	5684	mddev->sysfs_active \|\|
5654	5685	mddev->sync_thread \|\|
5655	5686	test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) {
5656		− printk("md: %s still in use.\n",mdname(mddev));
	5687	+ pr_warn("md: %s still in use.\n",mdname(mddev));
5657	5688	mutex_unlock(&mddev->open_mutex);
5658	5689	if (did_freeze) {
5659	5690	clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
@@ -5690,7 +5721,7 @@ static int do_md_stop(struct mddev *mddev, int mode, · 30 unmodified lines
5690	5721	* Free resources if final stop
5691	5722	*/
5692	5723	if (mode == 0) {
5693		− printk(KERN_INFO "md: %s stopped.\n", mdname(mddev));
	5724	+ pr_info("md: %s stopped.\n", mdname(mddev));
5694	5725
5695	5726	bitmap_destroy(mddev);
5696	5727	if (mddev->bitmap_info.file) {
@@ -5722,17 +5753,17 @@ static void autorun_array(struct mddev *mddev) · 25 unmodified lines
5722	5753	if (list_empty(&mddev->disks))
5723	5754	return;
5724	5755
5725		− printk(KERN_INFO "md: running: ");
	5756	+ pr_info("md: running: ");
5726	5757
5727	5758	rdev_for_each(rdev, mddev) {
5728	5759	char b[BDEVNAME_SIZE];
5729		− printk("<%s>", bdevname(rdev->bdev,b));
	5760	+ pr_cont("<%s>", bdevname(rdev->bdev,b));
5730	5761	}
5731		− printk("\n");
	5762	+ pr_cont("\n");
5732	5763
5733	5764	err = do_md_run(mddev);
5734	5765	if (err) {
5735		− printk(KERN_WARNING "md: do_md_run() returned %d\n", err);
	5766	+ pr_warn("md: do_md_run() returned %d\n", err);
5736	5767	do_md_stop(mddev, 0, NULL);
5737	5768	}
5738	5769	}
@@ -5755,7 +5786,7 @@ static void autorun_devices(int part) · 16 unmodified lines
5755	5786	struct mddev *mddev;
5756	5787	char b[BDEVNAME_SIZE];
5757	5788
5758		− printk(KERN_INFO "md: autorun ...\n");
	5789	+ pr_info("md: autorun ...\n");
5759	5790	while (!list_empty(&pending_raid_disks)) {
5760	5791	int unit;
5761	5792	dev_t dev;
@@ -5763,13 +5794,12 @@ static void autorun_devices(int part) · 1 unmodified line
5763	5794	rdev0 = list_entry(pending_raid_disks.next,
5764	5795	struct md_rdev, same_set);
5765	5796
5766		− printk(KERN_INFO "md: considering %s ...\n",
5767		− bdevname(rdev0->bdev,b));
	5797	+ pr_debug("md: considering %s ...\n", bdevname(rdev0->bdev,b));
5768	5798	INIT_LIST_HEAD(&candidates);
5769	5799	rdev_for_each_list(rdev, tmp, &pending_raid_disks)
5770	5800	if (super_90_load(rdev, rdev0, 0) >= 0) {
5771		− printk(KERN_INFO "md: adding %s ...\n",
5772		− bdevname(rdev->bdev,b));
	5801	+ pr_debug("md: adding %s ...\n",
	5802	+ bdevname(rdev->bdev,b));
5773	5803	list_move(&rdev->same_set, &candidates);
5774	5804	}
5775	5805	/*
@@ -5786,8 +5816,8 @@ static void autorun_devices(int part) · 10 unmodified lines
5786	5816	unit = MINOR(dev);
5787	5817	}
5788	5818	if (rdev0->preferred_minor != unit) {
5789		− printk(KERN_INFO "md: unit number in %s is bad: %d\n",
5790		− bdevname(rdev0->bdev, b), rdev0->preferred_minor);
	5819	+ pr_warn("md: unit number in %s is bad: %d\n",
	5820	+ bdevname(rdev0->bdev, b), rdev0->preferred_minor);
5791	5821	break;
5792	5822	}
5793	5823
@@ -5796,21 +5826,17 @@ static void autorun_devices(int part) · 2 unmodified lines
5796	5826	if (!mddev \|\| !mddev->gendisk) {
5797	5827	if (mddev)
5798	5828	mddev_put(mddev);
5799		− printk(KERN_ERR
5800		− "md: cannot allocate memory for md drive.\n");
5801	5829	break;
5802	5830	}
5803	5831	if (mddev_lock(mddev))
5804		− printk(KERN_WARNING "md: %s locked, cannot run\n",
5805		− mdname(mddev));
	5832	+ pr_warn("md: %s locked, cannot run\n", mdname(mddev));
5806	5833	else if (mddev->raid_disks \|\| mddev->major_version
5807	5834	\|\| !list_empty(&mddev->disks)) {
5808		− printk(KERN_WARNING
5809		− "md: %s already running, cannot run %s\n",
	5835	+ pr_warn("md: %s already running, cannot run %s\n",
5810	5836	mdname(mddev), bdevname(rdev0->bdev,b));
5811	5837	mddev_unlock(mddev);
5812	5838	} else {
5813		− printk(KERN_INFO "md: created %s\n", mdname(mddev));
	5839	+ pr_debug("md: created %s\n", mdname(mddev));
5814	5840	mddev->persistent = 1;
5815	5841	rdev_for_each_list(rdev, tmp, &candidates) {
5816	5842	list_del_init(&rdev->same_set);
@@ -5829,7 +5855,7 @@ static void autorun_devices(int part) · 12 unmodified lines
5829	5855	}
5830	5856	mddev_put(mddev);
5831	5857	}
5832		− printk(KERN_INFO "md: ... autorun DONE.\n");
	5858	+ pr_info("md: ... autorun DONE.\n");
5833	5859	}
5834	5860	#endif /* !MODULE */
5835	5861
@@ -5964,6 +5990,8 @@ static int get_disk_info(struct mddev mddev, void __user arg) · 128 unmodified lines
5964	5990	info.state \|= (1<<MD_DISK_JOURNAL);
5965	5991	if (test_bit(WriteMostly, &rdev->flags))
5966	5992	info.state \|= (1<<MD_DISK_WRITEMOSTLY);
	5993	+ if (test_bit(FailFast, &rdev->flags))
	5994	+ info.state \|= (1<<MD_DISK_FAILFAST);
5967	5995	} else {
5968	5996	info.major = info.minor = 0;
5969	5997	info.raid_disk = -1;
@@ -5985,8 +6013,8 @@ static int add_new_disk(struct mddev mddev, mdu_disk_info_t info) · 15 unmodified lines
5985	6013
5986	6014	if (mddev_is_clustered(mddev) &&
5987	6015	!(info->state & ((1 << MD_DISK_CLUSTER_ADD) \| (1 << MD_DISK_CANDIDATE)))) {
5988		− pr_err("%s: Cannot add to clustered mddev.\n",
5989		− mdname(mddev));
	6016	+ pr_warn("%s: Cannot add to clustered mddev.\n",
	6017	+ mdname(mddev));
5990	6018	return -EINVAL;
5991	6019	}
5992	6020
@@ -5998,8 +6026,7 @@ static int add_new_disk(struct mddev mddev, mdu_disk_info_t info) · 5 unmodified lines
5998	6026	/* expecting a device which has a superblock */
5999	6027	rdev = md_import_device(dev, mddev->major_version, mddev->minor_version);
6000	6028	if (IS_ERR(rdev)) {
6001		− printk(KERN_WARNING
6002		− "md: md_import_device returned %ld\n",
	6029	+ pr_warn("md: md_import_device returned %ld\n",
6003	6030	PTR_ERR(rdev));
6004	6031	return PTR_ERR(rdev);
6005	6032	}
@@ -6010,8 +6037,7 @@ static int add_new_disk(struct mddev mddev, mdu_disk_info_t info) · 4 unmodified lines
6010	6037	err = super_types[mddev->major_version]
6011	6038	.load_super(rdev, rdev0, mddev->minor_version);
6012	6039	if (err < 0) {
6013		− printk(KERN_WARNING
6014		− "md: %s has different UUID to %s\n",
	6040	+ pr_warn("md: %s has different UUID to %s\n",
6015	6041	bdevname(rdev->bdev,b),
6016	6042	bdevname(rdev0->bdev,b2));
6017	6043	export_rdev(rdev);
@@ -6032,9 +6058,8 @@ static int add_new_disk(struct mddev mddev, mdu_disk_info_t info) · 14 unmodified lines
6032	6058	if (mddev->pers) {
6033	6059	int err;
6034	6060	if (!mddev->pers->hot_add_disk) {
6035		− printk(KERN_WARNING
6036		− "%s: personality does not support diskops!\n",
6037		− mdname(mddev));
	6061	+ pr_warn("%s: personality does not support diskops!\n",
	6062	+ mdname(mddev));
6038	6063	return -EINVAL;
6039	6064	}
6040	6065	if (mddev->persistent)
@@ -6043,8 +6068,7 @@ static int add_new_disk(struct mddev mddev, mdu_disk_info_t info) · 2 unmodified lines
6043	6068	else
6044	6069	rdev = md_import_device(dev, -1, -1);
6045	6070	if (IS_ERR(rdev)) {
6046		− printk(KERN_WARNING
6047		− "md: md_import_device returned %ld\n",
	6071	+ pr_warn("md: md_import_device returned %ld\n",
6048	6072	PTR_ERR(rdev));
6049	6073	return PTR_ERR(rdev);
6050	6074	}
@@ -6075,6 +6099,10 @@ static int add_new_disk(struct mddev mddev, mdu_disk_info_t info) · 24 unmodified lines
6075	6099	set_bit(WriteMostly, &rdev->flags);
6076	6100	else
6077	6101	clear_bit(WriteMostly, &rdev->flags);
	6102	+ if (info->state & (1<<MD_DISK_FAILFAST))
	6103	+ set_bit(FailFast, &rdev->flags);
	6104	+ else
	6105	+ clear_bit(FailFast, &rdev->flags);
6078	6106
6079	6107	if (info->state & (1<<MD_DISK_JOURNAL)) {
6080	6108	struct md_rdev *rdev2;
@@ -6140,8 +6168,7 @@ static int add_new_disk(struct mddev mddev, mdu_disk_info_t info) · 59 unmodified lines
6140	6168	* for major_version==0 superblocks
6141	6169	*/
6142	6170	if (mddev->major_version != 0) {
6143		− printk(KERN_WARNING "%s: ADD_NEW_DISK not supported\n",
6144		− mdname(mddev));
	6171	+ pr_warn("%s: ADD_NEW_DISK not supported\n", mdname(mddev));
6145	6172	return -EINVAL;
6146	6173	}
6147	6174
@@ -6149,8 +6176,7 @@ static int add_new_disk(struct mddev mddev, mdu_disk_info_t info) · 1 unmodified line
6149	6176	int err;
6150	6177	rdev = md_import_device(dev, -1, 0);
6151	6178	if (IS_ERR(rdev)) {
6152		− printk(KERN_WARNING
6153		− "md: error, md_import_device() returned %ld\n",
	6179	+ pr_warn("md: error, md_import_device() returned %ld\n",
6154	6180	PTR_ERR(rdev));
6155	6181	return PTR_ERR(rdev);
6156	6182	}
@@ -6166,9 +6192,11 @@ static int add_new_disk(struct mddev mddev, mdu_disk_info_t info) · 9 unmodified lines
6166	6192
6167	6193	if (info->state & (1<<MD_DISK_WRITEMOSTLY))
6168	6194	set_bit(WriteMostly, &rdev->flags);
	6195	+ if (info->state & (1<<MD_DISK_FAILFAST))
	6196	+ set_bit(FailFast, &rdev->flags);
6169	6197
6170	6198	if (!mddev->persistent) {
6171		− printk(KERN_INFO "md: nonpersistent superblock ...\n");
	6199	+ pr_debug("md: nonpersistent superblock ...\n");
6172	6200	rdev->sb_start = i_size_read(rdev->bdev->bd_inode) / 512;
6173	6201	} else
6174	6202	rdev->sb_start = calc_dev_sboffset(rdev);
@@ -6207,13 +6235,17 @@ kick_rdev: · 32 unmodified lines
6207	6235	md_cluster_ops->remove_disk(mddev, rdev);
6208	6236
6209	6237	md_kick_rdev_from_array(rdev);
6210		− md_update_sb(mddev, 1);
	6238	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
	6239	+ if (mddev->thread)
	6240	+ md_wakeup_thread(mddev->thread);
	6241	+ else
	6242	+ md_update_sb(mddev, 1);
6211	6243	md_new_event(mddev);
6212	6244
6213	6245	return 0;
6214	6246	busy:
6215		− printk(KERN_WARNING "md: cannot remove active disk %s from %s ...\n",
6216		− bdevname(rdev->bdev,b), mdname(mddev));
	6247	+ pr_debug("md: cannot remove active disk %s from %s ...\n",
	6248	+ bdevname(rdev->bdev,b), mdname(mddev));
6217	6249	return -EBUSY;
6218	6250	}
6219	6251
@@ -6227,22 +6259,19 @@ static int hot_add_disk(struct mddev *mddev, dev_t dev) · 7 unmodified lines
6227	6259	return -ENODEV;
6228	6260
6229	6261	if (mddev->major_version != 0) {
6230		− printk(KERN_WARNING "%s: HOT_ADD may only be used with"
6231		− " version-0 superblocks.\n",
	6262	+ pr_warn("%s: HOT_ADD may only be used with version-0 superblocks.\n",
6232	6263	mdname(mddev));
6233	6264	return -EINVAL;
6234	6265	}
6235	6266	if (!mddev->pers->hot_add_disk) {
6236		− printk(KERN_WARNING
6237		− "%s: personality does not support diskops!\n",
	6267	+ pr_warn("%s: personality does not support diskops!\n",
6238	6268	mdname(mddev));
6239	6269	return -EINVAL;
6240	6270	}
6241	6271
6242	6272	rdev = md_import_device(dev, -1, 0);
6243	6273	if (IS_ERR(rdev)) {
6244		− printk(KERN_WARNING
6245		− "md: error, md_import_device() returned %ld\n",
	6274	+ pr_warn("md: error, md_import_device() returned %ld\n",
6246	6275	PTR_ERR(rdev));
6247	6276	return -EINVAL;
6248	6277	}
@@ -6255,8 +6284,7 @@ static int hot_add_disk(struct mddev *mddev, dev_t dev) · 6 unmodified lines
6255	6284	rdev->sectors = rdev->sb_start;
6256	6285
6257	6286	if (test_bit(Faulty, &rdev->flags)) {
6258		− printk(KERN_WARNING
6259		− "md: can not hot-add faulty %s disk to %s!\n",
	6287	+ pr_warn("md: can not hot-add faulty %s disk to %s!\n",
6260	6288	bdevname(rdev->bdev,b), mdname(mddev));
6261	6289	err = -EINVAL;
6262	6290	goto abort_export;
@@ -6276,7 +6304,9 @@ static int hot_add_disk(struct mddev *mddev, dev_t dev) · 13 unmodified lines
6276	6304
6277	6305	rdev->raid_disk = -1;
6278	6306
6279		− md_update_sb(mddev, 1);
	6307	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
	6308	+ if (!mddev->thread)
	6309	+ md_update_sb(mddev, 1);
6280	6310	/*
6281	6311	* Kick recovery, maybe this spare has to be added to the
6282	6312	* array immediately.
@@ -6312,23 +6342,23 @@ static int set_bitmap_file(struct mddev *mddev, int fd) · 29 unmodified lines
6312	6342	f = fget(fd);
6313	6343
6314	6344	if (f == NULL) {
6315		− printk(KERN_ERR "%s: error: failed to get bitmap file\n",
6316		− mdname(mddev));
	6345	+ pr_warn("%s: error: failed to get bitmap file\n",
	6346	+ mdname(mddev));
6317	6347	return -EBADF;
6318	6348	}
6319	6349
6320	6350	inode = f->f_mapping->host;
6321	6351	if (!S_ISREG(inode->i_mode)) {
6322		− printk(KERN_ERR "%s: error: bitmap file must be a regular file\n",
6323		− mdname(mddev));
	6352	+ pr_warn("%s: error: bitmap file must be a regular file\n",
	6353	+ mdname(mddev));
6324	6354	err = -EBADF;
6325	6355	} else if (!(f->f_mode & FMODE_WRITE)) {
6326		− printk(KERN_ERR "%s: error: bitmap file must open for write\n",
6327		− mdname(mddev));
	6356	+ pr_warn("%s: error: bitmap file must open for write\n",
	6357	+ mdname(mddev));
6328	6358	err = -EBADF;
6329	6359	} else if (atomic_read(&inode->i_writecount) != 1) {
6330		− printk(KERN_ERR "%s: error: bitmap file is already in use\n",
6331		− mdname(mddev));
	6360	+ pr_warn("%s: error: bitmap file is already in use\n",
	6361	+ mdname(mddev));
6332	6362	err = -EBUSY;
6333	6363	}
6334	6364	if (err) {
@@ -6393,8 +6423,7 @@ static int set_array_info(struct mddev mddev, mdu_array_info_t info) · 58 unmodified lines
6393	6423	info->major_version >= ARRAY_SIZE(super_types) \|\|
6394	6424	super_types[info->major_version].name == NULL) {
6395	6425	/* maybe try to auto-load a module? */
6396		− printk(KERN_INFO
6397		− "md: superblock version %d not known\n",
	6426	+ pr_warn("md: superblock version %d not known\n",
6398	6427	info->major_version);
6399	6428	return -EINVAL;
6400	6429	}
@@ -6432,9 +6461,11 @@ static int set_array_info(struct mddev mddev, mdu_array_info_t info) · 31 unmodified lines
6432	6461
6433	6462	mddev->max_disks = MD_SB_DISKS;
6434	6463
6435		− if (mddev->persistent)
	6464	+ if (mddev->persistent) {
6436	6465	mddev->flags = 0;
6437		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
	6466	+ mddev->sb_flags = 0;
	6467	+ }
	6468	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
6438	6469
6439	6470	mddev->bitmap_info.default_offset = MD_SB_BYTES >> 9;
6440	6471	mddev->bitmap_info.default_space = 64*2 - (MD_SB_BYTES >> 9);
@@ -6660,8 +6691,7 @@ static int update_array_info(struct mddev mddev, mdu_array_info_t info) · 219 unmodified lines
6660	6691	if (mddev->bitmap_info.nodes) {
6661	6692	/* hold PW on all the bitmap lock */
6662	6693	if (md_cluster_ops->lock_all_bitmaps(mddev) <= 0) {
6663		− printk("md: can't change bitmap to none since the"
6664		− " array is in use by more than one node\n");
	6694	+ pr_warn("md: can't change bitmap to none since the array is in use by more than one node\n");
6665	6695	rv = -EPERM;
6666	6696	md_cluster_ops->unlock_all_bitmaps(mddev);
6667	6697	goto err;
@@ -6829,7 +6859,7 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode, · 161 unmodified lines
6829	6859	/* need to ensure recovery thread has run */
6830	6860	wait_event_interruptible_timeout(mddev->sb_wait,
6831	6861	!test_bit(MD_RECOVERY_NEEDED,
6832		− &mddev->flags),
	6862	+ &mddev->recovery),
6833	6863	msecs_to_jiffies(5000));
6834	6864	if (cmd == STOP_ARRAY \|\| cmd == STOP_ARRAY_RO) {
6835	6865	/* Need to flush page cache, and ensure no-one else opens
@@ -6847,9 +6877,8 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode, · 11 unmodified lines
6847	6877	}
6848	6878	err = mddev_lock(mddev);
6849	6879	if (err) {
6850		− printk(KERN_INFO
6851		− "md: ioctl lock interrupted, reason %d, cmd %d\n",
6852		− err, cmd);
	6880	+ pr_debug("md: ioctl lock interrupted, reason %d, cmd %d\n",
	6881	+ err, cmd);
6853	6882	goto out;
6854	6883	}
6855	6884
@@ -6864,30 +6893,24 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode, · 8 unmodified lines
6864	6893	if (mddev->pers) {
6865	6894	err = update_array_info(mddev, &info);
6866	6895	if (err) {
6867		− printk(KERN_WARNING "md: couldn't update"
6868		− " array info. %d\n", err);
	6896	+ pr_warn("md: couldn't update array info. %d\n", err);
6869	6897	goto unlock;
6870	6898	}
6871	6899	goto unlock;
6872	6900	}
6873	6901	if (!list_empty(&mddev->disks)) {
6874		− printk(KERN_WARNING
6875		− "md: array %s already has disks!\n",
6876		− mdname(mddev));
	6902	+ pr_warn("md: array %s already has disks!\n", mdname(mddev));
6877	6903	err = -EBUSY;
6878	6904	goto unlock;
6879	6905	}
6880	6906	if (mddev->raid_disks) {
6881		− printk(KERN_WARNING
6882		− "md: array %s already initialised!\n",
6883		− mdname(mddev));
	6907	+ pr_warn("md: array %s already initialised!\n", mdname(mddev));
6884	6908	err = -EBUSY;
6885	6909	goto unlock;
6886	6910	}
6887	6911	err = set_array_info(mddev, &info);
6888	6912	if (err) {
6889		− printk(KERN_WARNING "md: couldn't set"
6890		− " array info. %d\n", err);
	6913	+ pr_warn("md: couldn't set array info. %d\n", err);
6891	6914	goto unlock;
6892	6915	}
6893	6916	goto unlock;
@@ -6987,11 +7010,11 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode, · 93 unmodified lines
6987	7010	/* If a device failed while we were read-only, we
6988	7011	* need to make sure the metadata is updated now.
6989	7012	*/
6990		− if (test_bit(MD_CHANGE_DEVS, &mddev->flags)) {
	7013	+ if (test_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags)) {
6991	7014	mddev_unlock(mddev);
6992	7015	wait_event(mddev->sb_wait,
6993		− !test_bit(MD_CHANGE_DEVS, &mddev->flags) &&
6994		− !test_bit(MD_CHANGE_PENDING, &mddev->flags));
	7016	+ !test_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags) &&
	7017	+ !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
6995	7018	mddev_lock_nointr(mddev);
6996	7019	}
6997	7020	} else {
@@ -7092,7 +7115,8 @@ static int md_open(struct block_device *bdev, fmode_t mode) · 94 unmodified lines
7092	7115
7093	7116	if (test_bit(MD_CLOSING, &mddev->flags)) {
7094	7117	mutex_unlock(&mddev->open_mutex);
7095		− return -ENODEV;
	7118	+ err = -ENODEV;
	7119	+ goto out;
7096	7120	}
7097	7121
7098	7122	err = 0;
@@ -7101,6 +7125,8 @@ static int md_open(struct block_device *bdev, fmode_t mode) · 2 unmodified lines
7101	7125
7102	7126	check_disk_change(bdev);
7103	7127	out:
	7128	+ if (err)
	7129	+ mddev_put(mddev);
7104	7130	return err;
7105	7131	}
7106	7132
@@ -7171,10 +7197,12 @@ static int md_thread(void *arg) · 64 unmodified lines
7171	7197	wait_event_interruptible_timeout
7172	7198	(thread->wqueue,
7173	7199	test_bit(THREAD_WAKEUP, &thread->flags)
7174		− \|\| kthread_should_stop(),
	7200	+ \|\| kthread_should_stop() \|\| kthread_should_park(),
7175	7201	thread->timeout);
7176	7202
7177	7203	clear_bit(THREAD_WAKEUP, &thread->flags);
	7204	+ if (kthread_should_park())
	7205	+ kthread_parkme();
7178	7206	if (!kthread_should_stop())
7179	7207	thread->run(thread);
7180	7208	}
@@ -7588,8 +7616,8 @@ static const struct file_operations md_seq_fops = { · 407 unmodified lines
7588	7616
7589	7617	int register_md_personality(struct md_personality *p)
7590	7618	{
7591		− printk(KERN_INFO "md: %s personality registered for level %d\n",
7592		− p->name, p->level);
	7619	+ pr_debug("md: %s personality registered for level %d\n",
	7620	+ p->name, p->level);
7593	7621	spin_lock(&pers_lock);
7594	7622	list_add_tail(&p->list, &pers_list);
7595	7623	spin_unlock(&pers_lock);
@@ -7599,7 +7627,7 @@ EXPORT_SYMBOL(register_md_personality); · 3 unmodified lines
7599	7627
7600	7628	int unregister_md_personality(struct md_personality *p)
7601	7629	{
7602		− printk(KERN_INFO "md: %s personality unregistered\n", p->name);
	7630	+ pr_debug("md: %s personality unregistered\n", p->name);
7603	7631	spin_lock(&pers_lock);
7604	7632	list_del_init(&p->list);
7605	7633	spin_unlock(&pers_lock);
@@ -7639,7 +7667,7 @@ int md_setup_cluster(struct mddev *mddev, int nodes) · 33 unmodified lines
7639	7667	spin_lock(&pers_lock);
7640	7668	/* ensure module won't be unloaded */
7641	7669	if (!md_cluster_ops \|\| !try_module_get(md_cluster_mod)) {
7642		− pr_err("can't find md-cluster module or get it's reference.\n");
	7670	+ pr_warn("can't find md-cluster module or get it's reference.\n");
7643	7671	spin_unlock(&pers_lock);
7644	7672	return -ENOENT;
7645	7673	}
@@ -7741,8 +7769,8 @@ void md_write_start(struct mddev mddev, struct bio bi) · 95 unmodified lines
7741	7769	spin_lock(&mddev->lock);
7742	7770	if (mddev->in_sync) {
7743	7771	mddev->in_sync = 0;
7744		− set_bit(MD_CHANGE_CLEAN, &mddev->flags);
7745		− set_bit(MD_CHANGE_PENDING, &mddev->flags);
	7772	+ set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
	7773	+ set_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
7746	7774	md_wakeup_thread(mddev->thread);
7747	7775	did_change = 1;
7748	7776	}
@@ -7751,7 +7779,7 @@ void md_write_start(struct mddev mddev, struct bio bi) · 2 unmodified lines
7751	7779	if (did_change)
7752	7780	sysfs_notify_dirent_safe(mddev->sysfs_state);
7753	7781	wait_event(mddev->sb_wait,
7754		− !test_bit(MD_CHANGE_PENDING, &mddev->flags));
	7782	+ !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
7755	7783	}
7756	7784	EXPORT_SYMBOL(md_write_start);
7757	7785
@@ -7772,7 +7800,7 @@ EXPORT_SYMBOL(md_write_end); · 14 unmodified lines
7772	7800	* attempting a GFP_KERNEL allocation while holding the mddev lock.
7773	7801	* Must be called with mddev_lock held.
7774	7802	*
7775		− * In the ->external case MD_CHANGE_PENDING can not be cleared until mddev->lock
	7803	+ * In the ->external case MD_SB_CHANGE_PENDING can not be cleared until mddev->lock
7776	7804	* is dropped, so return -EAGAIN after notifying userspace.
7777	7805	*/
7778	7806	int md_allow_write(struct mddev *mddev)
@@ -7787,8 +7815,8 @@ int md_allow_write(struct mddev *mddev) · 8 unmodified lines
7787	7815	spin_lock(&mddev->lock);
7788	7816	if (mddev->in_sync) {
7789	7817	mddev->in_sync = 0;
7790		− set_bit(MD_CHANGE_CLEAN, &mddev->flags);
7791		− set_bit(MD_CHANGE_PENDING, &mddev->flags);
	7818	+ set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
	7819	+ set_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
7792	7820	if (mddev->safemode_delay &&
7793	7821	mddev->safemode == 0)
7794	7822	mddev->safemode = 1;
@@ -7798,7 +7826,7 @@ int md_allow_write(struct mddev *mddev) · 3 unmodified lines
7798	7826	} else
7799	7827	spin_unlock(&mddev->lock);
7800	7828
7801		− if (test_bit(MD_CHANGE_PENDING, &mddev->flags))
	7829	+ if (test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags))
7802	7830	return -EAGAIN;
7803	7831	else
7804	7832	return 0;
@@ -7914,11 +7942,9 @@ void md_do_sync(struct md_thread *thread) · 109 unmodified lines
7914	7942	mddev2->curr_resync >= mddev->curr_resync) {
7915	7943	if (mddev2_minor != mddev2->md_minor) {
7916	7944	mddev2_minor = mddev2->md_minor;
7917		− printk(KERN_INFO "md: delaying %s of %s"
7918		− " until %s has finished (they"
7919		− " share one or more physical units)\n",
7920		− desc, mdname(mddev),
7921		− mdname(mddev2));
	7945	+ pr_info("md: delaying %s of %s until %s has finished (they share one or more physical units)\n",
	7946	+ desc, mdname(mddev),
	7947	+ mdname(mddev2));
7922	7948	}
7923	7949	mddev_put(mddev2);
7924	7950	if (signal_pending(current))
@@ -7975,12 +8001,10 @@ void md_do_sync(struct md_thread *thread) · 50 unmodified lines
7975	8001	}
7976	8002	}
7977	8003
7978		− printk(KERN_INFO "md: %s of RAID array %s\n", desc, mdname(mddev));
7979		− printk(KERN_INFO "md: minimum _guaranteed_ speed:"
7980		− " %d KB/sec/disk.\n", speed_min(mddev));
7981		− printk(KERN_INFO "md: using maximum available idle IO bandwidth "
7982		− "(but not more than %d KB/sec) for %s.\n",
7983		− speed_max(mddev), desc);
	8004	+ pr_info("md: %s of RAID array %s\n", desc, mdname(mddev));
	8005	+ pr_debug("md: minimum _guaranteed_ speed: %d KB/sec/disk.\n", speed_min(mddev));
	8006	+ pr_debug("md: using maximum available idle IO bandwidth (but not more than %d KB/sec) for %s.\n",
	8007	+ speed_max(mddev), desc);
7984	8008
7985	8009	is_mddev_idle(mddev, 1); /* this initializes IO event counters */
7986	8010
@@ -7997,16 +8021,15 @@ void md_do_sync(struct md_thread *thread) · 10 unmodified lines
7997	8021	* Tune reconstruction:
7998	8022	*/
7999	8023	window = 32*(PAGE_SIZE/512);
8000		− printk(KERN_INFO "md: using %dk window, over a total of %lluk.\n",
8001		− window/2, (unsigned long long)max_sectors/2);
	8024	+ pr_debug("md: using %dk window, over a total of %lluk.\n",
	8025	+ window/2, (unsigned long long)max_sectors/2);
8002	8026
8003	8027	atomic_set(&mddev->recovery_active, 0);
8004	8028	last_check = 0;
8005	8029
8006	8030	if (j>2) {
8007		− printk(KERN_INFO
8008		− "md: resuming %s of %s from checkpoint.\n",
8009		− desc, mdname(mddev));
	8031	+ pr_debug("md: resuming %s of %s from checkpoint.\n",
	8032	+ desc, mdname(mddev));
8010	8033	mddev->curr_resync = j;
8011	8034	} else
8012	8035	mddev->curr_resync = 3; /* no longer delayed */
@@ -8038,7 +8061,7 @@ void md_do_sync(struct md_thread *thread) · 25 unmodified lines
8038	8061	j > mddev->recovery_cp)
8039	8062	mddev->recovery_cp = j;
8040	8063	update_time = jiffies;
8041		− set_bit(MD_CHANGE_CLEAN, &mddev->flags);
	8064	+ set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
8042	8065	sysfs_notify(&mddev->kobj, NULL, "sync_completed");
8043	8066	}
8044	8067
@@ -8133,9 +8156,9 @@ void md_do_sync(struct md_thread *thread) · 88 unmodified lines
8133	8156	}
8134	8157	}
8135	8158	}
8136		− printk(KERN_INFO "md: %s: %s %s.\n",mdname(mddev), desc,
8137		− test_bit(MD_RECOVERY_INTR, &mddev->recovery)
8138		− ? "interrupted" : "done");
	8159	+ pr_info("md: %s: %s %s.\n",mdname(mddev), desc,
	8160	+ test_bit(MD_RECOVERY_INTR, &mddev->recovery)
	8161	+ ? "interrupted" : "done");
8139	8162	/*
8140	8163	* this also signals 'finished resyncing' to md_stop
8141	8164	*/
@@ -8155,9 +8178,8 @@ void md_do_sync(struct md_thread *thread) · 13 unmodified lines
8155	8178	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
8156	8179	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
8157	8180	if (mddev->curr_resync >= mddev->recovery_cp) {
8158		− printk(KERN_INFO
8159		− "md: checkpointing %s of %s.\n",
8160		− desc, mdname(mddev));
	8181	+ pr_debug("md: checkpointing %s of %s.\n",
	8182	+ desc, mdname(mddev));
8161	8183	if (test_bit(MD_RECOVERY_ERROR,
8162	8184	&mddev->recovery))
8163	8185	mddev->recovery_cp =
@@ -8187,8 +8209,8 @@ void md_do_sync(struct md_thread *thread) · 23 unmodified lines
8187	8209	/* set CHANGE_PENDING here since maybe another update is needed,
8188	8210	* so other nodes are informed. It should be harmless for normal
8189	8211	* raid */
8190		− set_mask_bits(&mddev->flags, 0,
8191		− BIT(MD_CHANGE_PENDING) \| BIT(MD_CHANGE_DEVS));
	8212	+ set_mask_bits(&mddev->sb_flags, 0,
	8213	+ BIT(MD_SB_CHANGE_PENDING) \| BIT(MD_SB_CHANGE_DEVS));
8192	8214
8193	8215	spin_lock(&mddev->lock);
8194	8216	if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
@@ -8288,12 +8310,12 @@ static int remove_and_add_spares(struct mddev *mddev, · 93 unmodified lines
8288	8310	if (!test_bit(Journal, &rdev->flags))
8289	8311	spares++;
8290	8312	md_new_event(mddev);
8291		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
	8313	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
8292	8314	}
8293	8315	}
8294	8316	no_add:
8295	8317	if (removed)
8296		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
	8318	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
8297	8319	return spares;
8298	8320	}
8299	8321
@@ -8305,8 +8327,8 @@ static void md_start_sync(struct work_struct *ws) · 5 unmodified lines
8305	8327	mddev,
8306	8328	"resync");
8307	8329	if (!mddev->sync_thread) {
8308		− printk(KERN_ERR "%s: could not start resync thread...\n",
8309		− mdname(mddev));
	8330	+ pr_warn("%s: could not start resync thread...\n",
	8331	+ mdname(mddev));
8310	8332	/* leave the spares where they are, it shouldn't hurt */
8311	8333	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
8312	8334	clear_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
@@ -8356,8 +8378,8 @@ void md_check_recovery(struct mddev *mddev) · 43 unmodified lines
8356	8378
8357	8379	if (signal_pending(current)) {
8358	8380	if (mddev->pers->sync_request && !mddev->external) {
8359		− printk(KERN_INFO "md: %s in immediate safe mode\n",
8360		− mdname(mddev));
	8381	+ pr_debug("md: %s in immediate safe mode\n",
	8382	+ mdname(mddev));
8361	8383	mddev->safemode = 2;
8362	8384	}
8363	8385	flush_signals(current);
@@ -8366,7 +8388,7 @@ void md_check_recovery(struct mddev *mddev) · 2 unmodified lines
8366	8388	if (mddev->ro && !test_bit(MD_RECOVERY_NEEDED, &mddev->recovery))
8367	8389	return;
8368	8390	if ( ! (
8369		− (mddev->flags & MD_UPDATE_SB_FLAGS & ~ (1<<MD_CHANGE_PENDING)) \|\|
	8391	+ (mddev->sb_flags & ~ (1<<MD_SB_CHANGE_PENDING)) \|\|
8370	8392	test_bit(MD_RECOVERY_NEEDED, &mddev->recovery) \|\|
8371	8393	test_bit(MD_RECOVERY_DONE, &mddev->recovery) \|\|
8372	8394	test_bit(MD_RELOAD_SB, &mddev->flags) \|\|
@@ -8404,7 +8426,7 @@ void md_check_recovery(struct mddev *mddev) · 31 unmodified lines
8404	8426	md_reap_sync_thread(mddev);
8405	8427	clear_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
8406	8428	clear_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
8407		− clear_bit(MD_CHANGE_PENDING, &mddev->flags);
	8429	+ clear_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
8408	8430	goto unlock;
8409	8431	}
8410	8432
@@ -8432,7 +8454,7 @@ void md_check_recovery(struct mddev *mddev) · 21 unmodified lines
8432	8454	mddev->recovery_cp == MaxSector) {
8433	8455	mddev->in_sync = 1;
8434	8456	did_change = 1;
8435		− set_bit(MD_CHANGE_CLEAN, &mddev->flags);
	8457	+ set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
8436	8458	}
8437	8459	if (mddev->safemode == 1)
8438	8460	mddev->safemode = 0;
@@ -8441,7 +8463,7 @@ void md_check_recovery(struct mddev *mddev) · 2 unmodified lines
8441	8463	sysfs_notify_dirent_safe(mddev->sysfs_state);
8442	8464	}
8443	8465
8444		− if (mddev->flags & MD_UPDATE_SB_FLAGS)
	8466	+ if (mddev->sb_flags)
8445	8467	md_update_sb(mddev, 0);
8446	8468
8447	8469	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) &&
@@ -8537,7 +8559,7 @@ void md_reap_sync_thread(struct mddev *mddev) · 89 unmodified lines
8537	8559	if (mddev->pers->spare_active(mddev)) {
8538	8560	sysfs_notify(&mddev->kobj, NULL,
8539	8561	"degraded");
8540		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
	8562	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
8541	8563	}
8542	8564	}
8543	8565	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
@@ -8552,7 +8574,7 @@ void md_reap_sync_thread(struct mddev *mddev) · 8 unmodified lines
8552	8574	rdev->saved_raid_disk = -1;
8553	8575
8554	8576	md_update_sb(mddev, 1);
8555		− /* MD_CHANGE_PENDING should be cleared by md_update_sb, so we can
	8577	+ /* MD_SB_CHANGE_PENDING should be cleared by md_update_sb, so we can
8556	8578	* call resync_finish here if MD_CLUSTER_RESYNC_LOCKED is set by
8557	8579	* clustered raid */
8558	8580	if (test_and_clear_bit(MD_CLUSTER_RESYNC_LOCKED, &mddev->flags))
@@ -8614,9 +8636,12 @@ int rdev_set_badblocks(struct md_rdev *rdev, sector_t s, int sectors, · 55 unmodified lines
8614	8636	rv = badblocks_set(&rdev->badblocks, s, sectors, 0);
8615	8637	if (rv == 0) {
8616	8638	/* Make sure they get written out promptly */
	8639	+ if (test_bit(ExternalBbl, &rdev->flags))
	8640	+ sysfs_notify(&rdev->kobj, NULL,
	8641	+ "unacknowledged_bad_blocks");
8617	8642	sysfs_notify_dirent_safe(rdev->sysfs_state);
8618		− set_mask_bits(&mddev->flags, 0,
8619		− BIT(MD_CHANGE_CLEAN) \| BIT(MD_CHANGE_PENDING));
	8643	+ set_mask_bits(&mddev->sb_flags, 0,
	8644	+ BIT(MD_SB_CHANGE_CLEAN) \| BIT(MD_SB_CHANGE_PENDING));
8620	8645	md_wakeup_thread(rdev->mddev->thread);
8621	8646	return 1;
8622	8647	} else
@@ -8627,12 +8652,15 @@ EXPORT_SYMBOL_GPL(rdev_set_badblocks); · 4 unmodified lines
8627	8652	int rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, int sectors,
8628	8653	int is_new)
8629	8654	{
	8655	+ int rv;
8630	8656	if (is_new)
8631	8657	s += rdev->new_data_offset;
8632	8658	else
8633	8659	s += rdev->data_offset;
8634		− return badblocks_clear(&rdev->badblocks,
8635		− s, sectors);
	8660	+ rv = badblocks_clear(&rdev->badblocks, s, sectors);
	8661	+ if ((rv == 0) && test_bit(ExternalBbl, &rdev->flags))
	8662	+ sysfs_notify(&rdev->kobj, NULL, "bad_blocks");
	8663	+ return rv;
8636	8664	}
8637	8665	EXPORT_SYMBOL_GPL(rdev_clear_badblocks);
8638	8666
@@ -8749,7 +8777,7 @@ static void check_sb_changes(struct mddev mddev, struct md_rdev rdev) · 110 unmodified lines
8749	8777	rdev2->saved_raid_disk = role;
8750	8778	ret = remove_and_add_spares(mddev, rdev2);
8751	8779	pr_info("Activated spare: %s\n",
8752		− bdevname(rdev2->bdev,b));
	8780	+ bdevname(rdev2->bdev,b));
8753	8781	/* wakeup mddev->thread here, so array could
8754	8782	* perform resync with the new activated disk */
8755	8783	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
@@ -8785,15 +8813,18 @@ static int read_rdev(struct mddev mddev, struct md_rdev rdev) · 29 unmodified lines
8785	8813	* variable in case we err in the future
8786	8814	*/
8787	8815	rdev->sb_page = NULL;
8788		− alloc_disk_sb(rdev);
8789		− ClearPageUptodate(rdev->sb_page);
8790		− rdev->sb_loaded = 0;
8791		− err = super_types[mddev->major_version].load_super(rdev, NULL, mddev->minor_version);
8792		−
	8816	+ err = alloc_disk_sb(rdev);
	8817	+ if (err == 0) {
	8818	+ ClearPageUptodate(rdev->sb_page);
	8819	+ rdev->sb_loaded = 0;
	8820	+ err = super_types[mddev->major_version].
	8821	+ load_super(rdev, NULL, mddev->minor_version);
	8822	+ }
8793	8823	if (err < 0) {
8794	8824	pr_warn("%s: %d Could not reload rdev(%d) err: %d. Restoring old values\n",
8795	8825	__func__, __LINE__, rdev->desc_nr, err);
8796		− put_page(rdev->sb_page);
	8826	+ if (rdev->sb_page)
	8827	+ put_page(rdev->sb_page);
8797	8828	rdev->sb_page = swapout;
8798	8829	rdev->sb_loaded = 1;
8799	8830	return err;
@@ -8871,9 +8902,6 @@ void md_autodetect_dev(dev_t dev) · 71 unmodified lines
8871	8902	mutex_lock(&detected_devices_mutex);
8872	8903	list_add_tail(&node_detected_dev->list, &all_detected_devices);
8873	8904	mutex_unlock(&detected_devices_mutex);
8874		− } else {
8875		− printk(KERN_CRIT "md: md_autodetect_dev: kzalloc failed"
8876		− ", skipping dev(%d,%d)\n", MAJOR(dev), MINOR(dev));
8877	8905	}
8878	8906	}
8879	8907
@@ -8887,7 +8915,7 @@ static void autostart_arrays(int part) · 7 unmodified lines
8887	8915	i_scanned = 0;
8888	8916	i_passed = 0;
8889	8917
8890		− printk(KERN_INFO "md: Autodetecting RAID arrays.\n");
	8918	+ pr_info("md: Autodetecting RAID arrays.\n");
8891	8919
8892	8920	mutex_lock(&detected_devices_mutex);
8893	8921	while (!list_empty(&all_detected_devices) && i_scanned < INT_MAX) {
@@ -8912,8 +8940,7 @@ static void autostart_arrays(int part) · 18 unmodified lines
8912	8940	}
8913	8941	mutex_unlock(&detected_devices_mutex);
8914	8942
8915		− printk(KERN_INFO "md: Scanned %d and added %d devices.\n",
8916		− i_scanned, i_passed);
	8943	+ pr_debug("md: Scanned %d and added %d devices.\n", i_scanned, i_passed);
8917	8944
8918	8945	autorun_devices(part);
8919	8946	}

drivers/md/md.h+67−41

@@ -29,6 +29,16 @@ · 28 unmodified lines
29	29
30	30	#define MaxSector (~(sector_t)0)
31	31
	32	+/*
	33	+ * These flags should really be called "NO_RETRY" rather than
	34	+ * "FAILFAST" because they don't make any promise about time lapse,
	35	+ * only about the number of retries, which will be zero.
	36	+ * REQ_FAILFAST_DRIVER is not included because
	37	+ * Commit: 4a27446f3e39 ("[SCSI] modify scsi to handle new fail fast flags.")
	38	+ * seems to suggest that the errors it avoids retrying should usually
	39	+ * be retried.
	40	+ */
	41	+#define MD_FAILFAST (REQ_FAILFAST_DEV \| REQ_FAILFAST_TRANSPORT)
32	42	/*
33	43	* MD's 'extended' device
34	44	*/
@@ -168,6 +178,19 @@ enum flag_bits { · 133 unmodified lines
168	178	* so it is safe to remove without
169	179	* another synchronize_rcu() call.
170	180	*/
	181	+ ExternalBbl, /* External metadata provides bad
	182	+ * block management for a disk
	183	+ */
	184	+ FailFast, /* Minimal retries should be attempted on
	185	+ * this device, so use REQ_FAILFAST_DEV.
	186	+ * Also don't try to repair failed reads.
	187	+ * It is expects that no bad block log
	188	+ * is present.
	189	+ */
	190	+ LastDev, /* Seems to be the last working dev as
	191	+ * it didn't fail, so don't use FailFast
	192	+ * any more for metadata
	193	+ */
171	194	};
172	195
173	196	static inline int is_badblock(struct md_rdev *rdev, sector_t s, int sectors,
@@ -189,6 +212,31 @@ extern int rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, int sectors, · 15 unmodified lines
189	212	int is_new);
190	213	struct md_cluster_info;
191	214
	215	+enum mddev_flags {
	216	+ MD_ARRAY_FIRST_USE, /* First use of array, needs initialization */
	217	+ MD_CLOSING, /* If set, we are closing the array, do not open
	218	+ * it then */
	219	+ MD_JOURNAL_CLEAN, /* A raid with journal is already clean */
	220	+ MD_HAS_JOURNAL, /* The raid array has journal feature set */
	221	+ MD_RELOAD_SB, /* Reload the superblock because another node
	222	+ * updated it.
	223	+ */
	224	+ MD_CLUSTER_RESYNC_LOCKED, /* cluster raid only, which means node
	225	+ * already took resync lock, need to
	226	+ * release the lock */
	227	+ MD_FAILFAST_SUPPORTED, /* Using MD_FAILFAST on metadata writes is
	228	+ * supported as calls to md_error() will
	229	+ * never cause the array to become failed.
	230	+ */
	231	+};
	232	+
	233	+enum mddev_sb_flags {
	234	+ MD_SB_CHANGE_DEVS, /* Some device status has changed */
	235	+ MD_SB_CHANGE_CLEAN, /* transition to or from 'clean' */
	236	+ MD_SB_CHANGE_PENDING, /* switch from 'clean' to 'active' in progress */
	237	+ MD_SB_NEED_REWRITE, /* metadata write needs to be repeated */
	238	+};
	239	+
192	240	struct mddev {
193	241	void *private;
194	242	struct md_personality *pers;
@@ -196,21 +244,7 @@ struct mddev { · 1 unmodified line
196	244	int md_minor;
197	245	struct list_head disks;
198	246	unsigned long flags;
199		−#define MD_CHANGE_DEVS 0 /* Some device status has changed */
200		−#define MD_CHANGE_CLEAN 1 /* transition to or from 'clean' */
201		−#define MD_CHANGE_PENDING 2 /* switch from 'clean' to 'active' in progress */
202		−#define MD_UPDATE_SB_FLAGS (1 \| 2 \| 4) /* If these are set, md_update_sb needed */
203		−#define MD_ARRAY_FIRST_USE 3 /* First use of array, needs initialization */
204		−#define MD_CLOSING 4 /* If set, we are closing the array, do not open
205		− * it then */
206		−#define MD_JOURNAL_CLEAN 5 /* A raid with journal is already clean */
207		−#define MD_HAS_JOURNAL 6 /* The raid array has journal feature set */
208		−#define MD_RELOAD_SB 7 /* Reload the superblock because another node
209		− * updated it.
210		− */
211		−#define MD_CLUSTER_RESYNC_LOCKED 8 /* cluster raid only, which means node
212		− * already took resync lock, need to
213		− * release the lock */
	247	+ unsigned long sb_flags;
214	248
215	249	int suspended;
216	250	atomic_t active_io;
@@ -304,31 +338,6 @@ struct mddev { · 87 unmodified lines
304	338	int parallel_resync;
305	339
306	340	int ok_start_degraded;
307		− /* recovery/resync flags
308		− * NEEDED: we might need to start a resync/recover
309		− * RUNNING: a thread is running, or about to be started
310		− * SYNC: actually doing a resync, not a recovery
311		− * RECOVER: doing recovery, or need to try it.
312		− * INTR: resync needs to be aborted for some reason
313		− * DONE: thread is done and is waiting to be reaped
314		− * REQUEST: user-space has requested a sync (used with SYNC)
315		− * CHECK: user-space request for check-only, no repair
316		− * RESHAPE: A reshape is happening
317		− * ERROR: sync-action interrupted because io-error
318		− *
319		− * If neither SYNC or RESHAPE are set, then it is a recovery.
320		− */
321		−#define MD_RECOVERY_RUNNING 0
322		−#define MD_RECOVERY_SYNC 1
323		−#define MD_RECOVERY_RECOVER 2
324		−#define MD_RECOVERY_INTR 3
325		−#define MD_RECOVERY_DONE 4
326		−#define MD_RECOVERY_NEEDED 5
327		−#define MD_RECOVERY_REQUESTED 6
328		−#define MD_RECOVERY_CHECK 7
329		−#define MD_RECOVERY_RESHAPE 8
330		−#define MD_RECOVERY_FROZEN 9
331		−#define MD_RECOVERY_ERROR 10
332	341
333	342	unsigned long recovery;
334	343	/* If a RAID personality determines that recovery (of a particular
@@ -442,6 +451,23 @@ struct mddev { · 107 unmodified lines
442	451	unsigned int good_device_nr; /* good device num within cluster raid */
443	452	};
444	453
	454	+enum recovery_flags {
	455	+ /*
	456	+ * If neither SYNC or RESHAPE are set, then it is a recovery.
	457	+ */
	458	+ MD_RECOVERY_RUNNING, /* a thread is running, or about to be started */
	459	+ MD_RECOVERY_SYNC, /* actually doing a resync, not a recovery */
	460	+ MD_RECOVERY_RECOVER, /* doing recovery, or need to try it. */
	461	+ MD_RECOVERY_INTR, /* resync needs to be aborted for some reason */
	462	+ MD_RECOVERY_DONE, /* thread is done and is waiting to be reaped */
	463	+ MD_RECOVERY_NEEDED, /* we might need to start a resync/recover */
	464	+ MD_RECOVERY_REQUESTED, /* user-space has requested a sync (used with SYNC) */
	465	+ MD_RECOVERY_CHECK, /* user-space request for check-only, no repair */
	466	+ MD_RECOVERY_RESHAPE, /* A reshape is happening */
	467	+ MD_RECOVERY_FROZEN, /* User request to abort, and not restart, any action */
	468	+ MD_RECOVERY_ERROR, /* sync-action interrupted because io-error */
	469	+};
	470	+
445	471	static inline int __must_check mddev_lock(struct mddev *mddev)
446	472	{
447	473	return mutex_lock_interruptible(&mddev->reconfig_mutex);
@@ -623,7 +649,7 @@ extern int mddev_congested(struct mddev *mddev, int bits); · 175 unmodified lines
623	649	extern void md_flush_request(struct mddev mddev, struct bio bio);
624	650	extern void md_super_write(struct mddev mddev, struct md_rdev rdev,
625	651	sector_t sector, int size, struct page *page);
626		−extern void md_super_wait(struct mddev *mddev);
	652	+extern int md_super_wait(struct mddev *mddev);
627	653	extern int sync_page_io(struct md_rdev *rdev, sector_t sector, int size,
628	654	struct page *page, int op, int op_flags,
629	655	bool metadata_op);

drivers/md/multipath.c+34−58

@@ -52,7 +52,7 @@ static int multipath_map (struct mpconf *conf) · 51 unmodified lines
52	52	}
53	53	rcu_read_unlock();
54	54
55		− printk(KERN_ERR "multipath_map(): no more operational IO paths?\n");
	55	+ pr_crit_ratelimited("multipath_map(): no more operational IO paths?\n");
56	56	return (-1);
57	57	}
58	58
@@ -97,9 +97,9 @@ static void multipath_end_request(struct bio *bio) · 38 unmodified lines
97	97	*/
98	98	char b[BDEVNAME_SIZE];
99	99	md_error (mp_bh->mddev, rdev);
100		− printk(KERN_ERR "multipath: %s: rescheduling sector %llu\n",
101		− bdevname(rdev->bdev,b),
102		− (unsigned long long)bio->bi_iter.bi_sector);
	100	+ pr_info("multipath: %s: rescheduling sector %llu\n",
	101	+ bdevname(rdev->bdev,b),
	102	+ (unsigned long long)bio->bi_iter.bi_sector);
103	103	multipath_reschedule_retry(mp_bh);
104	104	} else
105	105	multipath_end_bh_io(mp_bh, bio->bi_error);
@@ -194,8 +194,7 @@ static void multipath_error (struct mddev mddev, struct md_rdev rdev) · 88 unmodified lines
194	194	* first check if this is a queued request for a device
195	195	* which has just failed.
196	196	*/
197		− printk(KERN_ALERT
198		− "multipath: only one IO path left and IO error.\n");
	197	+ pr_warn("multipath: only one IO path left and IO error.\n");
199	198	/* leave it active... it's all we have */
200	199	return;
201	200	}
@@ -209,11 +208,9 @@ static void multipath_error (struct mddev mddev, struct md_rdev rdev) · 7 unmodified lines
209	208	spin_unlock_irqrestore(&conf->device_lock, flags);
210	209	}
211	210	set_bit(Faulty, &rdev->flags);
212		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
213		− printk(KERN_ALERT "multipath: IO failure on %s,"
214		− " disabling IO path.\n"
215		− "multipath: Operation continuing"
216		− " on %d IO paths.\n",
	211	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
	212	+ pr_err("multipath: IO failure on %s, disabling IO path.\n"
	213	+ "multipath: Operation continuing on %d IO paths.\n",
217	214	bdevname(rdev->bdev, b),
218	215	conf->raid_disks - mddev->degraded);
219	216	}
@@ -223,21 +220,21 @@ static void print_multipath_conf (struct mpconf *conf) · 3 unmodified lines
223	220	int i;
224	221	struct multipath_info *tmp;
225	222
226		− printk("MULTIPATH conf printout:\n");
	223	+ pr_debug("MULTIPATH conf printout:\n");
227	224	if (!conf) {
228		− printk("(conf==NULL)\n");
	225	+ pr_debug("(conf==NULL)\n");
229	226	return;
230	227	}
231		− printk(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded,
232		− conf->raid_disks);
	228	+ pr_debug(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded,
	229	+ conf->raid_disks);
233	230
234	231	for (i = 0; i < conf->raid_disks; i++) {
235	232	char b[BDEVNAME_SIZE];
236	233	tmp = conf->multipaths + i;
237	234	if (tmp->rdev)
238		− printk(" disk%d, o:%d, dev:%s\n",
239		− i,!test_bit(Faulty, &tmp->rdev->flags),
240		− bdevname(tmp->rdev->bdev,b));
	235	+ pr_debug(" disk%d, o:%d, dev:%s\n",
	236	+ i,!test_bit(Faulty, &tmp->rdev->flags),
	237	+ bdevname(tmp->rdev->bdev,b));
241	238	}
242	239	}
243	240
@@ -292,8 +289,7 @@ static int multipath_remove_disk(struct mddev mddev, struct md_rdev rdev) · 48 unmodified lines
292	289	if (rdev == p->rdev) {
293	290	if (test_bit(In_sync, &rdev->flags) \|\|
294	291	atomic_read(&rdev->nr_pending)) {
295		− printk(KERN_ERR "hot-remove-disk, slot %d is identified"
296		− " but is still operational!\n", number);
	292	+ pr_warn("hot-remove-disk, slot %d is identified but is still operational!\n", number);
297	293	err = -EBUSY;
298	294	goto abort;
299	295	}
@@ -346,16 +342,14 @@ static void multipathd(struct md_thread *thread) · 46 unmodified lines
346	342	bio->bi_iter.bi_sector = mp_bh->master_bio->bi_iter.bi_sector;
347	343
348	344	if ((mp_bh->path = multipath_map (conf))<0) {
349		− printk(KERN_ALERT "multipath: %s: unrecoverable IO read"
350		− " error for block %llu\n",
351		− bdevname(bio->bi_bdev,b),
352		− (unsigned long long)bio->bi_iter.bi_sector);
	345	+ pr_err("multipath: %s: unrecoverable IO read error for block %llu\n",
	346	+ bdevname(bio->bi_bdev,b),
	347	+ (unsigned long long)bio->bi_iter.bi_sector);
353	348	multipath_end_bh_io(mp_bh, -EIO);
354	349	} else {
355		− printk(KERN_ERR "multipath: %s: redirecting sector %llu"
356		− " to another IO path\n",
357		− bdevname(bio->bi_bdev,b),
358		− (unsigned long long)bio->bi_iter.bi_sector);
	350	+ pr_err("multipath: %s: redirecting sector %llu to another IO path\n",
	351	+ bdevname(bio->bi_bdev,b),
	352	+ (unsigned long long)bio->bi_iter.bi_sector);
359	353	bio = (mp_bh->master_bio);
360	354	bio->bi_iter.bi_sector +=
361	355	conf->multipaths[mp_bh->path].rdev->data_offset;
@@ -389,8 +383,8 @@ static int multipath_run (struct mddev *mddev) · 27 unmodified lines
389	383	return -EINVAL;
390	384
391	385	if (mddev->level != LEVEL_MULTIPATH) {
392		− printk("multipath: %s: raid level not set to multipath IO (%d)\n",
393		− mdname(mddev), mddev->level);
	386	+ pr_warn("multipath: %s: raid level not set to multipath IO (%d)\n",
	387	+ mdname(mddev), mddev->level);
394	388	goto out;
395	389	}
396	390	/*
@@ -401,21 +395,13 @@ static int multipath_run (struct mddev *mddev) · 4 unmodified lines
401	395
402	396	conf = kzalloc(sizeof(struct mpconf), GFP_KERNEL);
403	397	mddev->private = conf;
404		− if (!conf) {
405		− printk(KERN_ERR
406		− "multipath: couldn't allocate memory for %s\n",
407		− mdname(mddev));
	398	+ if (!conf)
408	399	goto out;
409		− }
410	400
411	401	conf->multipaths = kzalloc(sizeof(struct multipath_info)*mddev->raid_disks,
412	402	GFP_KERNEL);
413		− if (!conf->multipaths) {
414		− printk(KERN_ERR
415		− "multipath: couldn't allocate memory for %s\n",
416		− mdname(mddev));
	403	+ if (!conf->multipaths)
417	404	goto out_free_conf;
418		− }
419	405
420	406	working_disks = 0;
421	407	rdev_for_each(rdev, mddev) {
@@ -439,7 +425,7 @@ static int multipath_run (struct mddev *mddev) · 17 unmodified lines
439	425	INIT_LIST_HEAD(&conf->retry_list);
440	426
441	427	if (!working_disks) {
442		− printk(KERN_ERR "multipath: no operational IO paths for %s\n",
	428	+ pr_warn("multipath: no operational IO paths for %s\n",
443	429	mdname(mddev));
444	430	goto out_free_conf;
445	431	}
@@ -447,27 +433,17 @@ static int multipath_run (struct mddev *mddev) · 1 unmodified line
447	433
448	434	conf->pool = mempool_create_kmalloc_pool(NR_RESERVED_BUFS,
449	435	sizeof(struct multipath_bh));
450		− if (conf->pool == NULL) {
451		− printk(KERN_ERR
452		− "multipath: couldn't allocate memory for %s\n",
453		− mdname(mddev));
	436	+ if (conf->pool == NULL)
454	437	goto out_free_conf;
455		− }
456	438
457		− {
458		− mddev->thread = md_register_thread(multipathd, mddev,
459		− "multipath");
460		− if (!mddev->thread) {
461		− printk(KERN_ERR "multipath: couldn't allocate thread"
462		− " for %s\n", mdname(mddev));
463		− goto out_free_conf;
464		− }
465		− }
	439	+ mddev->thread = md_register_thread(multipathd, mddev,
	440	+ "multipath");
	441	+ if (!mddev->thread)
	442	+ goto out_free_conf;
466	443
467		− printk(KERN_INFO
468		− "multipath: array %s active with %d out of %d IO paths\n",
	444	+ pr_info("multipath: array %s active with %d out of %d IO paths\n",
469	445	mdname(mddev), conf->raid_disks - mddev->degraded,
470		− mddev->raid_disks);
	446	+ mddev->raid_disks);
471	447	/*
472	448	* Ok, everything is just fine now
473	449	*/

drivers/md/raid0.c+59−48

@@ -21,6 +21,7 @@ · 20 unmodified lines
21	21	#include <linux/seq_file.h>
22	22	#include <linux/module.h>
23	23	#include <linux/slab.h>
	24	+#include <trace/events/block.h>
24	25	#include "md.h"
25	26	#include "raid0.h"
26	27	#include "raid5.h"
@@ -51,20 +52,21 @@ static void dump_zones(struct mddev *mddev) · 24 unmodified lines
51	52	char b[BDEVNAME_SIZE];
52	53	struct r0conf *conf = mddev->private;
53	54	int raid_disks = conf->strip_zone[0].nb_dev;
54		− printk(KERN_INFO "md: RAID0 configuration for %s - %d zone%s\n",
55		− mdname(mddev),
56		− conf->nr_strip_zones, conf->nr_strip_zones==1?"":"s");
	55	+ pr_debug("md: RAID0 configuration for %s - %d zone%s\n",
	56	+ mdname(mddev),
	57	+ conf->nr_strip_zones, conf->nr_strip_zones==1?"":"s");
57	58	for (j = 0; j < conf->nr_strip_zones; j++) {
58		− printk(KERN_INFO "md: zone%d=[", j);
	59	+ char line[200];
	60	+ int len = 0;
	61	+
59	62	for (k = 0; k < conf->strip_zone[j].nb_dev; k++)
60		− printk(KERN_CONT "%s%s", k?"/":"",
61		− bdevname(conf->devlist[j*raid_disks
62		− + k]->bdev, b));
63		− printk(KERN_CONT "]\n");
	63	+ len += snprintf(line+len, 200-len, "%s%s", k?"/":"",
	64	+ bdevname(conf->devlist[j*raid_disks
	65	+ + k]->bdev, b));
	66	+ pr_debug("md: zone%d=[%s]\n", j, line);
64	67
65	68	zone_size = conf->strip_zone[j].zone_end - zone_start;
66		− printk(KERN_INFO " zone-offset=%10lluKB, "
67		− "device-offset=%10lluKB, size=%10lluKB\n",
	69	+ pr_debug(" zone-offset=%10lluKB, device-offset=%10lluKB, size=%10lluKB\n",
68	70	(unsigned long long)zone_start>>1,
69	71	(unsigned long long)conf->strip_zone[j].dev_start>>1,
70	72	(unsigned long long)zone_size>>1);
@@ -142,9 +144,9 @@ static int create_strip_zones(struct mddev mddev, struct r0conf *private_conf) · 71 unmodified lines
142	144	* chunk size is a multiple of that sector size
143	145	*/
144	146	if ((mddev->chunk_sectors << 9) % blksize) {
145		− printk(KERN_ERR "md/raid0:%s: chunk_size of %d not multiple of block size %d\n",
146		− mdname(mddev),
147		− mddev->chunk_sectors << 9, blksize);
	147	+ pr_warn("md/raid0:%s: chunk_size of %d not multiple of block size %d\n",
	148	+ mdname(mddev),
	149	+ mddev->chunk_sectors << 9, blksize);
148	150	err = -EINVAL;
149	151	goto abort;
150	152	}
@@ -186,19 +188,18 @@ static int create_strip_zones(struct mddev mddev, struct r0conf *private_conf) · 35 unmodified lines
186	188	}
187	189
188	190	if (j < 0) {
189		− printk(KERN_ERR
190		− "md/raid0:%s: remove inactive devices before converting to RAID0\n",
191		− mdname(mddev));
	191	+ pr_warn("md/raid0:%s: remove inactive devices before converting to RAID0\n",
	192	+ mdname(mddev));
192	193	goto abort;
193	194	}
194	195	if (j >= mddev->raid_disks) {
195		− printk(KERN_ERR "md/raid0:%s: bad disk number %d - "
196		− "aborting!\n", mdname(mddev), j);
	196	+ pr_warn("md/raid0:%s: bad disk number %d - aborting!\n",
	197	+ mdname(mddev), j);
197	198	goto abort;
198	199	}
199	200	if (dev[j]) {
200		− printk(KERN_ERR "md/raid0:%s: multiple devices for %d - "
201		− "aborting!\n", mdname(mddev), j);
	201	+ pr_warn("md/raid0:%s: multiple devices for %d - aborting!\n",
	202	+ mdname(mddev), j);
202	203	goto abort;
203	204	}
204	205	dev[j] = rdev1;
@@ -208,8 +209,8 @@ static int create_strip_zones(struct mddev mddev, struct r0conf *private_conf) · 3 unmodified lines
208	209	cnt++;
209	210	}
210	211	if (cnt != mddev->raid_disks) {
211		− printk(KERN_ERR "md/raid0:%s: too few disks (%d of %d) - "
212		− "aborting!\n", mdname(mddev), cnt, mddev->raid_disks);
	212	+ pr_warn("md/raid0:%s: too few disks (%d of %d) - aborting!\n",
	213	+ mdname(mddev), cnt, mddev->raid_disks);
213	214	goto abort;
214	215	}
215	216	zone->nb_dev = cnt;
@@ -357,8 +358,7 @@ static int raid0_run(struct mddev *mddev) · 141 unmodified lines
357	358	int ret;
358	359
359	360	if (mddev->chunk_sectors == 0) {
360		− printk(KERN_ERR "md/raid0:%s: chunk size must be set.\n",
361		− mdname(mddev));
	361	+ pr_warn("md/raid0:%s: chunk size must be set.\n", mdname(mddev));
362	362	return -EINVAL;
363	363	}
364	364	if (md_check_no_bitmap(mddev))
@@ -399,9 +399,9 @@ static int raid0_run(struct mddev *mddev) · 34 unmodified lines
399	399	/* calculate array device size */
400	400	md_set_array_sectors(mddev, raid0_size(mddev, 0, 0));
401	401
402		− printk(KERN_INFO "md/raid0:%s: md_size is %llu sectors.\n",
403		− mdname(mddev),
404		− (unsigned long long)mddev->array_sectors);
	402	+ pr_debug("md/raid0:%s: md_size is %llu sectors.\n",
	403	+ mdname(mddev),
	404	+ (unsigned long long)mddev->array_sectors);
405	405
406	406	if (mddev->queue) {
407	407	/* calculate the max read-ahead size.
@@ -464,7 +464,8 @@ static void raid0_make_request(struct mddev mddev, struct bio bio) · 56 unmodified lines
464	464	}
465	465
466	466	do {
467		− sector_t sector = bio->bi_iter.bi_sector;
	467	+ sector_t bio_sector = bio->bi_iter.bi_sector;
	468	+ sector_t sector = bio_sector;
468	469	unsigned chunk_sects = mddev->chunk_sectors;
469	470
470	471	unsigned sectors = chunk_sects -
@@ -473,7 +474,7 @@ static void raid0_make_request(struct mddev mddev, struct bio bio) · 2 unmodified lines
473	474	: sector_div(sector, chunk_sects));
474	475
475	476	/* Restore due to sector_div */
476		− sector = bio->bi_iter.bi_sector;
	477	+ sector = bio_sector;
477	478
478	479	if (sectors < bio_sectors(bio)) {
479	480	split = bio_split(bio, sectors, GFP_NOIO, fs_bio_set);
@@ -492,8 +493,13 @@ static void raid0_make_request(struct mddev mddev, struct bio bio) · 12 unmodified lines
492	493	!blk_queue_discard(bdev_get_queue(split->bi_bdev)))) {
493	494	/* Just ignore it */
494	495	bio_endio(split);
495		− } else
	496	+ } else {
	497	+ if (mddev->gendisk)
	498	+ trace_block_bio_remap(bdev_get_queue(split->bi_bdev),
	499	+ split, disk_devt(mddev->gendisk),
	500	+ bio_sector);
496	501	generic_make_request(split);
	502	+ }
497	503	} while (split != bio);
498	504	}
499	505
@@ -509,17 +515,17 @@ static void raid0_takeover_raid45(struct mddev mddev) · 9 unmodified lines
509	515	struct r0conf *priv_conf;
510	516
511	517	if (mddev->degraded != 1) {
512		− printk(KERN_ERR "md/raid0:%s: raid5 must be degraded! Degraded disks: %d\n",
513		− mdname(mddev),
514		− mddev->degraded);
	518	+ pr_warn("md/raid0:%s: raid5 must be degraded! Degraded disks: %d\n",
	519	+ mdname(mddev),
	520	+ mddev->degraded);
515	521	return ERR_PTR(-EINVAL);
516	522	}
517	523
518	524	rdev_for_each(rdev, mddev) {
519	525	/* check slot number for a disk */
520	526	if (rdev->raid_disk == mddev->raid_disks-1) {
521		− printk(KERN_ERR "md/raid0:%s: raid5 must have missing parity disk!\n",
522		− mdname(mddev));
	527	+ pr_warn("md/raid0:%s: raid5 must have missing parity disk!\n",
	528	+ mdname(mddev));
523	529	return ERR_PTR(-EINVAL);
524	530	}
525	531	rdev->sectors = mddev->dev_sectors;
@@ -533,8 +539,11 @@ static void raid0_takeover_raid45(struct mddev mddev) · 7 unmodified lines
533	539	mddev->delta_disks = -1;
534	540	/* make sure it will be not marked as dirty */
535	541	mddev->recovery_cp = MaxSector;
	542	+ clear_bit(MD_HAS_JOURNAL, &mddev->flags);
	543	+ clear_bit(MD_JOURNAL_CLEAN, &mddev->flags);
536	544
537	545	create_strip_zones(mddev, &priv_conf);
	546	+
538	547	return priv_conf;
539	548	}
540	549
@@ -549,19 +558,19 @@ static void raid0_takeover_raid10(struct mddev mddev) · 8 unmodified lines
549	558	* - all mirrors must be already degraded
550	559	*/
551	560	if (mddev->layout != ((1 << 8) + 2)) {
552		− printk(KERN_ERR "md/raid0:%s:: Raid0 cannot takeover layout: 0x%x\n",
553		− mdname(mddev),
554		− mddev->layout);
	561	+ pr_warn("md/raid0:%s:: Raid0 cannot takeover layout: 0x%x\n",
	562	+ mdname(mddev),
	563	+ mddev->layout);
555	564	return ERR_PTR(-EINVAL);
556	565	}
557	566	if (mddev->raid_disks & 1) {
558		− printk(KERN_ERR "md/raid0:%s: Raid0 cannot takeover Raid10 with odd disk number.\n",
559		− mdname(mddev));
	567	+ pr_warn("md/raid0:%s: Raid0 cannot takeover Raid10 with odd disk number.\n",
	568	+ mdname(mddev));
560	569	return ERR_PTR(-EINVAL);
561	570	}
562	571	if (mddev->degraded != (mddev->raid_disks>>1)) {
563		− printk(KERN_ERR "md/raid0:%s: All mirrors must be already degraded!\n",
564		− mdname(mddev));
	572	+ pr_warn("md/raid0:%s: All mirrors must be already degraded!\n",
	573	+ mdname(mddev));
565	574	return ERR_PTR(-EINVAL);
566	575	}
567	576
@@ -574,6 +583,7 @@ static void raid0_takeover_raid10(struct mddev mddev) · 6 unmodified lines
574	583	mddev->degraded = 0;
575	584	/* make sure it will be not marked as dirty */
576	585	mddev->recovery_cp = MaxSector;
	586	+ clear_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
577	587
578	588	create_strip_zones(mddev, &priv_conf);
579	589	return priv_conf;
@@ -588,7 +598,7 @@ static void raid0_takeover_raid1(struct mddev mddev) · 8 unmodified lines
588	598	* - (N - 1) mirror drives must be already faulty
589	599	*/
590	600	if ((mddev->raid_disks - 1) != mddev->degraded) {
591		− printk(KERN_ERR "md/raid0:%s: (N - 1) mirrors drives must be already faulty!\n",
	601	+ pr_err("md/raid0:%s: (N - 1) mirrors drives must be already faulty!\n",
592	602	mdname(mddev));
593	603	return ERR_PTR(-EINVAL);
594	604	}
@@ -616,6 +626,7 @@ static void raid0_takeover_raid1(struct mddev mddev) · 21 unmodified lines
616	626	mddev->raid_disks = 1;
617	627	/* make sure it will be not marked as dirty */
618	628	mddev->recovery_cp = MaxSector;
	629	+ clear_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
619	630
620	631	create_strip_zones(mddev, &priv_conf);
621	632	return priv_conf;
@@ -631,8 +642,8 @@ static void raid0_takeover(struct mddev mddev) · 9 unmodified lines
631	642	*/
632	643
633	644	if (mddev->bitmap) {
634		− printk(KERN_ERR "md/raid0: %s: cannot takeover array with bitmap\n",
635		− mdname(mddev));
	645	+ pr_warn("md/raid0: %s: cannot takeover array with bitmap\n",
	646	+ mdname(mddev));
636	647	return ERR_PTR(-EBUSY);
637	648	}
638	649	if (mddev->level == 4)
@@ -642,8 +653,8 @@ static void raid0_takeover(struct mddev mddev) · 3 unmodified lines
642	653	if (mddev->layout == ALGORITHM_PARITY_N)
643	654	return raid0_takeover_raid45(mddev);
644	655
645		− printk(KERN_ERR "md/raid0:%s: Raid can only takeover Raid5 with layout: %d\n",
646		− mdname(mddev), ALGORITHM_PARITY_N);
	656	+ pr_warn("md/raid0:%s: Raid can only takeover Raid5 with layout: %d\n",
	657	+ mdname(mddev), ALGORITHM_PARITY_N);
647	658	}
648	659
649	660	if (mddev->level == 10)
@@ -652,7 +663,7 @@ static void raid0_takeover(struct mddev mddev) · 2 unmodified lines
652	663	if (mddev->level == 1)
653	664	return raid0_takeover_raid1(mddev);
654	665
655		− printk(KERN_ERR "Takeover from raid%i to raid0 not supported\n",
	666	+ pr_warn("Takeover from raid%i to raid0 not supported\n",
656	667	mddev->level);
657	668
658	669	return ERR_PTR(-EINVAL);

drivers/md/raid1.c+165−82

@@ -37,6 +37,7 @@ · 36 unmodified lines
37	37	#include <linux/module.h>
38	38	#include <linux/seq_file.h>
39	39	#include <linux/ratelimit.h>
	40	+#include <trace/events/block.h>
40	41	#include "md.h"
41	42	#include "raid1.h"
42	43	#include "bitmap.h"
@@ -70,6 +71,9 @@ static void allow_barrier(struct r1conf *conf, sector_t start_next_window, · 27 unmodified lines
70	71	sector_t bi_sector);
71	72	static void lower_barrier(struct r1conf *conf);
72	73
	74	+#define raid1_log(md, fmt, args...) \
	75	+ do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid1 " fmt, ##args); } while (0)
	76	+
73	77	static void * r1bio_pool_alloc(gfp_t gfp_flags, void *data)
74	78	{
75	79	struct pool_info *pi = data;
@@ -325,6 +329,11 @@ static void raid1_end_read_request(struct bio *bio) · 249 unmodified lines
325	329
326	330	if (uptodate)
327	331	set_bit(R1BIO_Uptodate, &r1_bio->state);
	332	+ else if (test_bit(FailFast, &rdev->flags) &&
	333	+ test_bit(R1BIO_FailFast, &r1_bio->state))
	334	+ /* This was a fail-fast read so we definitely
	335	+ * want to retry */
	336	+ ;
328	337	else {
329	338	/* If all other devices have failed, we want to return
330	339	* the error upwards rather than fail the last device.
@@ -347,13 +356,10 @@ static void raid1_end_read_request(struct bio *bio) · 16 unmodified lines
347	356	* oops, read error:
348	357	*/
349	358	char b[BDEVNAME_SIZE];
350		− printk_ratelimited(
351		− KERN_ERR "md/raid1:%s: %s: "
352		− "rescheduling sector %llu\n",
353		− mdname(conf->mddev),
354		− bdevname(rdev->bdev,
355		− b),
356		− (unsigned long long)r1_bio->sector);
	359	+ pr_err_ratelimited("md/raid1:%s: %s: rescheduling sector %llu\n",
	360	+ mdname(conf->mddev),
	361	+ bdevname(rdev->bdev, b),
	362	+ (unsigned long long)r1_bio->sector);
357	363	set_bit(R1BIO_ReadError, &r1_bio->state);
358	364	reschedule_retry(r1_bio);
359	365	/* don't drop the reference on read_disk yet */
@@ -416,7 +422,24 @@ static void raid1_end_write_request(struct bio *bio) · 56 unmodified lines
416	422	set_bit(MD_RECOVERY_NEEDED, &
417	423	conf->mddev->recovery);
418	424
419		− set_bit(R1BIO_WriteError, &r1_bio->state);
	425	+ if (test_bit(FailFast, &rdev->flags) &&
	426	+ (bio->bi_opf & MD_FAILFAST) &&
	427	+ /* We never try FailFast to WriteMostly devices */
	428	+ !test_bit(WriteMostly, &rdev->flags)) {
	429	+ md_error(r1_bio->mddev, rdev);
	430	+ if (!test_bit(Faulty, &rdev->flags))
	431	+ /* This is the only remaining device,
	432	+ * We need to retry the write without
	433	+ * FailFast
	434	+ */
	435	+ set_bit(R1BIO_WriteError, &r1_bio->state);
	436	+ else {
	437	+ /* Finished with this branch */
	438	+ r1_bio->bios[mirror] = NULL;
	439	+ to_put = bio;
	440	+ }
	441	+ } else
	442	+ set_bit(R1BIO_WriteError, &r1_bio->state);
420	443	} else {
421	444	/*
422	445	* Set R1BIO_Uptodate in our master bio, so that we
@@ -534,6 +557,7 @@ static int read_balance(struct r1conf conf, struct r1bio r1_bio, int *max_sect · 111 unmodified lines
534	557	best_good_sectors = 0;
535	558	has_nonrot_disk = 0;
536	559	choose_next_idle = 0;
	560	+ clear_bit(R1BIO_FailFast, &r1_bio->state);
537	561
538	562	if ((conf->mddev->recovery_cp < this_sector + sectors) \|\|
539	563	(mddev_is_clustered(conf->mddev) &&
@@ -607,6 +631,10 @@ static int read_balance(struct r1conf conf, struct r1bio r1_bio, int *max_sect · 67 unmodified lines
607	631	} else
608	632	best_good_sectors = sectors;
609	633
	634	+ if (best_disk >= 0)
	635	+ /* At least two disks to choose from so failfast is OK */
	636	+ set_bit(R1BIO_FailFast, &r1_bio->state);
	637	+
610	638	nonrot = blk_queue_nonrot(bdev_get_queue(rdev->bdev));
611	639	has_nonrot_disk \|= nonrot;
612	640	pending = atomic_read(&rdev->nr_pending);
@@ -645,11 +673,6 @@ static int read_balance(struct r1conf conf, struct r1bio r1_bio, int *max_sect · 32 unmodified lines
645	673	}
646	674	break;
647	675	}
648		− /* If device is idle, use it */
649		− if (pending == 0) {
650		− best_disk = disk;
651		− break;
652		− }
653	676
654	677	if (choose_next_idle)
655	678	continue;
@@ -672,7 +695,7 @@ static int read_balance(struct r1conf conf, struct r1bio r1_bio, int *max_sect · 16 unmodified lines
672	695	* mixed ratation/non-rotational disks depending on workload.
673	696	*/
674	697	if (best_disk == -1) {
675		− if (has_nonrot_disk)
	698	+ if (has_nonrot_disk \|\| min_pending == 0)
676	699	best_disk = best_pending_disk;
677	700	else
678	701	best_disk = best_dist_disk;
@@ -745,9 +768,14 @@ static void flush_pending_writes(struct r1conf *conf) · 66 unmodified lines
745	768
746	769	while (bio) { /* submit pending writes */
747	770	struct bio *next = bio->bi_next;
	771	+ struct md_rdev rdev = (void)bio->bi_bdev;
748	772	bio->bi_next = NULL;
749		− if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
750		− !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
	773	+ bio->bi_bdev = rdev->bdev;
	774	+ if (test_bit(Faulty, &rdev->flags)) {
	775	+ bio->bi_error = -EIO;
	776	+ bio_endio(bio);
	777	+ } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
	778	+ !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
751	779	/* Just ignore it */
752	780	bio_endio(bio);
753	781	else
@@ -832,7 +860,7 @@ static bool need_to_wait_for_sync(struct r1conf conf, struct bio bio) · 78 unmodified lines
832	860	else if (conf->barrier && bio_data_dir(bio) == WRITE) {
833	861	if ((conf->mddev->curr_resync_completed
834	862	>= bio_end_sector(bio)) \|\|
835		− (conf->next_resync + NEXT_NORMALIO_DISTANCE
	863	+ (conf->start_next_window + NEXT_NORMALIO_DISTANCE
836	864	<= bio->bi_iter.bi_sector))
837	865	wait = false;
838	866	else
@@ -858,6 +886,7 @@ static sector_t wait_barrier(struct r1conf conf, struct bio bio) · 19 unmodified lines
858	886	* that queue to allow conf->start_next_window
859	887	* to increase.
860	888	*/
	889	+ raid1_log(conf->mddev, "wait barrier");
861	890	wait_event_lock_irq(conf->wait_barrier,
862	891	!conf->array_frozen &&
863	892	(!conf->barrier \|\|
@@ -937,6 +966,7 @@ static void freeze_array(struct r1conf *conf, int extra) · 73 unmodified lines
937	966	*/
938	967	spin_lock_irq(&conf->resync_lock);
939	968	conf->array_frozen = 1;
	969	+ raid1_log(conf->mddev, "wait freeze");
940	970	wait_event_lock_irq_cmd(conf->wait_barrier,
941	971	conf->nr_pending == conf->nr_queued+extra,
942	972	conf->resync_lock,
@@ -1019,9 +1049,14 @@ static void raid1_unplug(struct blk_plug_cb *cb, bool from_schedule) · 76 unmodified lines
1019	1049
1020	1050	while (bio) { /* submit pending writes */
1021	1051	struct bio *next = bio->bi_next;
	1052	+ struct md_rdev rdev = (void)bio->bi_bdev;
1022	1053	bio->bi_next = NULL;
1023		− if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
1024		− !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
	1054	+ bio->bi_bdev = rdev->bdev;
	1055	+ if (test_bit(Faulty, &rdev->flags)) {
	1056	+ bio->bi_error = -EIO;
	1057	+ bio_endio(bio);
	1058	+ } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
	1059	+ !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
1025	1060	/* Just ignore it */
1026	1061	bio_endio(bio);
1027	1062	else
@@ -1136,6 +1171,7 @@ read_again: · 108 unmodified lines
1136	1171	* take care not to over-take any writes
1137	1172	* that are 'behind'
1138	1173	*/
	1174	+ raid1_log(mddev, "wait behind writes");
1139	1175	wait_event(bitmap->behind_wait,
1140	1176	atomic_read(&bitmap->behind_writes) == 0);
1141	1177	}
@@ -1153,8 +1189,16 @@ read_again: · 11 unmodified lines
1153	1189	read_bio->bi_bdev = mirror->rdev->bdev;
1154	1190	read_bio->bi_end_io = raid1_end_read_request;
1155	1191	bio_set_op_attrs(read_bio, op, do_sync);
	1192	+ if (test_bit(FailFast, &mirror->rdev->flags) &&
	1193	+ test_bit(R1BIO_FailFast, &r1_bio->state))
	1194	+ read_bio->bi_opf \|= MD_FAILFAST;
1156	1195	read_bio->bi_private = r1_bio;
1157	1196
	1197	+ if (mddev->gendisk)
	1198	+ trace_block_bio_remap(bdev_get_queue(read_bio->bi_bdev),
	1199	+ read_bio, disk_devt(mddev->gendisk),
	1200	+ r1_bio->sector);
	1201	+
1158	1202	if (max_sectors < r1_bio->sectors) {
1159	1203	/* could not read all from this device, so we will
1160	1204	* need another r1_bio.
@@ -1195,6 +1239,7 @@ read_again: · 34 unmodified lines
1195	1239	*/
1196	1240	if (conf->pending_count >= max_queued_requests) {
1197	1241	md_wakeup_thread(mddev->thread);
	1242	+ raid1_log(mddev, "wait queued");
1198	1243	wait_event(conf->wait_barrier,
1199	1244	conf->pending_count < max_queued_requests);
1200	1245	}
@@ -1286,6 +1331,7 @@ read_again: · 85 unmodified lines
1286	1331	rdev_dec_pending(conf->mirrors[j].rdev, mddev);
1287	1332	r1_bio->state = 0;
1288	1333	allow_barrier(conf, start_next_window, bio->bi_iter.bi_sector);
	1334	+ raid1_log(mddev, "wait rdev %d blocked", blocked_rdev->raid_disk);
1289	1335	md_wait_for_blocked_rdev(blocked_rdev, mddev);
1290	1336	start_next_window = wait_barrier(conf, bio);
1291	1337	/*
@@ -1363,10 +1409,21 @@ read_again: · 71 unmodified lines
1363	1409	mbio->bi_bdev = conf->mirrors[i].rdev->bdev;
1364	1410	mbio->bi_end_io = raid1_end_write_request;
1365	1411	bio_set_op_attrs(mbio, op, do_flush_fua \| do_sync);
	1412	+ if (test_bit(FailFast, &conf->mirrors[i].rdev->flags) &&
	1413	+ !test_bit(WriteMostly, &conf->mirrors[i].rdev->flags) &&
	1414	+ conf->raid_disks - mddev->degraded > 1)
	1415	+ mbio->bi_opf \|= MD_FAILFAST;
1366	1416	mbio->bi_private = r1_bio;
1367	1417
1368	1418	atomic_inc(&r1_bio->remaining);
1369	1419
	1420	+ if (mddev->gendisk)
	1421	+ trace_block_bio_remap(bdev_get_queue(mbio->bi_bdev),
	1422	+ mbio, disk_devt(mddev->gendisk),
	1423	+ r1_bio->sector);
	1424	+ /* flush_pending_writes() needs access to the rdev so...*/
	1425	+ mbio->bi_bdev = (void*)conf->mirrors[i].rdev;
	1426	+
1370	1427	cb = blk_check_plugged(raid1_unplug, mddev, sizeof(*plug));
1371	1428	if (cb)
1372	1429	plug = container_of(cb, struct raid1_plug_cb, cb);
@@ -1436,6 +1493,7 @@ static void raid1_error(struct mddev mddev, struct md_rdev rdev) · 63 unmodified lines
1436	1493	* next level up know.
1437	1494	* else mark the drive as failed
1438	1495	*/
	1496	+ spin_lock_irqsave(&conf->device_lock, flags);
1439	1497	if (test_bit(In_sync, &rdev->flags)
1440	1498	&& (conf->raid_disks - mddev->degraded) == 1) {
1441	1499	/*
@@ -1445,10 +1503,10 @@ static void raid1_error(struct mddev mddev, struct md_rdev rdev) · 3 unmodified lines
1445	1503	* it is very likely to fail.
1446	1504	*/
1447	1505	conf->recovery_disabled = mddev->recovery_disabled;
	1506	+ spin_unlock_irqrestore(&conf->device_lock, flags);
1448	1507	return;
1449	1508	}
1450	1509	set_bit(Blocked, &rdev->flags);
1451		− spin_lock_irqsave(&conf->device_lock, flags);
1452	1510	if (test_and_clear_bit(In_sync, &rdev->flags)) {
1453	1511	mddev->degraded++;
1454	1512	set_bit(Faulty, &rdev->flags);
@@ -1459,36 +1517,35 @@ static void raid1_error(struct mddev mddev, struct md_rdev rdev) · 4 unmodified lines
1459	1517	* if recovery is running, make sure it aborts.
1460	1518	*/
1461	1519	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1462		− set_mask_bits(&mddev->flags, 0,
1463		− BIT(MD_CHANGE_DEVS) \| BIT(MD_CHANGE_PENDING));
1464		− printk(KERN_ALERT
1465		− "md/raid1:%s: Disk failure on %s, disabling device.\n"
1466		− "md/raid1:%s: Operation continuing on %d devices.\n",
1467		− mdname(mddev), bdevname(rdev->bdev, b),
1468		− mdname(mddev), conf->raid_disks - mddev->degraded);
	1520	+ set_mask_bits(&mddev->sb_flags, 0,
	1521	+ BIT(MD_SB_CHANGE_DEVS) \| BIT(MD_SB_CHANGE_PENDING));
	1522	+ pr_crit("md/raid1:%s: Disk failure on %s, disabling device.\n"
	1523	+ "md/raid1:%s: Operation continuing on %d devices.\n",
	1524	+ mdname(mddev), bdevname(rdev->bdev, b),
	1525	+ mdname(mddev), conf->raid_disks - mddev->degraded);
1469	1526	}
1470	1527
1471	1528	static void print_conf(struct r1conf *conf)
1472	1529	{
1473	1530	int i;
1474	1531
1475		− printk(KERN_DEBUG "RAID1 conf printout:\n");
	1532	+ pr_debug("RAID1 conf printout:\n");
1476	1533	if (!conf) {
1477		− printk(KERN_DEBUG "(!conf)\n");
	1534	+ pr_debug("(!conf)\n");
1478	1535	return;
1479	1536	}
1480		− printk(KERN_DEBUG " --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded,
1481		− conf->raid_disks);
	1537	+ pr_debug(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded,
	1538	+ conf->raid_disks);
1482	1539
1483	1540	rcu_read_lock();
1484	1541	for (i = 0; i < conf->raid_disks; i++) {
1485	1542	char b[BDEVNAME_SIZE];
1486	1543	struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
1487	1544	if (rdev)
1488		− printk(KERN_DEBUG " disk %d, wo:%d, o:%d, dev:%s\n",
1489		− i, !test_bit(In_sync, &rdev->flags),
1490		− !test_bit(Faulty, &rdev->flags),
1491		− bdevname(rdev->bdev,b));
	1545	+ pr_debug(" disk %d, wo:%d, o:%d, dev:%s\n",
	1546	+ i, !test_bit(In_sync, &rdev->flags),
	1547	+ !test_bit(Faulty, &rdev->flags),
	1548	+ bdevname(rdev->bdev,b));
1492	1549	}
1493	1550	rcu_read_unlock();
1494	1551	}
@@ -1788,12 +1845,24 @@ static int fix_sync_read_error(struct r1bio *r1_bio) · 293 unmodified lines
1788	1845	sector_t sect = r1_bio->sector;
1789	1846	int sectors = r1_bio->sectors;
1790	1847	int idx = 0;
	1848	+ struct md_rdev *rdev;
	1849	+
	1850	+ rdev = conf->mirrors[r1_bio->read_disk].rdev;
	1851	+ if (test_bit(FailFast, &rdev->flags)) {
	1852	+ /* Don't try recovering from here - just fail it
	1853	+ * ... unless it is the last working device of course */
	1854	+ md_error(mddev, rdev);
	1855	+ if (test_bit(Faulty, &rdev->flags))
	1856	+ /* Don't try to read from here, but make sure
	1857	+ * put_buf does it's thing
	1858	+ */
	1859	+ bio->bi_end_io = end_sync_write;
	1860	+ }
1791	1861
1792	1862	while(sectors) {
1793	1863	int s = sectors;
1794	1864	int d = r1_bio->read_disk;
1795	1865	int success = 0;
1796		− struct md_rdev *rdev;
1797	1866	int start;
1798	1867
1799	1868	if (s > (PAGE_SIZE>>9))
@@ -1825,11 +1894,10 @@ static int fix_sync_read_error(struct r1bio *r1_bio) · 25 unmodified lines
1825	1894	* work just disable and interrupt the recovery.
1826	1895	* Don't fail devices as that won't really help.
1827	1896	*/
1828		− printk(KERN_ALERT "md/raid1:%s: %s: unrecoverable I/O read error"
1829		− " for block %llu\n",
1830		− mdname(mddev),
1831		− bdevname(bio->bi_bdev, b),
1832		− (unsigned long long)r1_bio->sector);
	1897	+ pr_crit_ratelimited("md/raid1:%s: %s: unrecoverable I/O read error for block %llu\n",
	1898	+ mdname(mddev),
	1899	+ bdevname(bio->bi_bdev, b),
	1900	+ (unsigned long long)r1_bio->sector);
1833	1901	for (d = 0; d < conf->raid_disks * 2; d++) {
1834	1902	rdev = conf->mirrors[d].rdev;
1835	1903	if (!rdev \|\| test_bit(Faulty, &rdev->flags))
@@ -2013,6 +2081,9 @@ static void sync_request_write(struct mddev mddev, struct r1bio r1_bio) · 177 unmodified lines
2013	2081	continue;
2014	2082
2015	2083	bio_set_op_attrs(wbio, REQ_OP_WRITE, 0);
	2084	+ if (test_bit(FailFast, &conf->mirrors[i].rdev->flags))
	2085	+ wbio->bi_opf \|= MD_FAILFAST;
	2086	+
2016	2087	wbio->bi_end_io = end_sync_write;
2017	2088	atomic_inc(&r1_bio->remaining);
2018	2089	md_sync_acct(conf->mirrors[i].rdev->bdev, bio_sectors(wbio));
@@ -2122,13 +2193,11 @@ static void fix_read_error(struct r1conf *conf, int read_disk, · 103 unmodified lines
2122	2193	if (r1_sync_page_io(rdev, sect, s,
2123	2194	conf->tmppage, READ)) {
2124	2195	atomic_add(s, &rdev->corrected_errors);
2125		− printk(KERN_INFO
2126		− "md/raid1:%s: read error corrected "
2127		− "(%d sectors at %llu on %s)\n",
2128		− mdname(mddev), s,
2129		− (unsigned long long)(sect +
2130		− rdev->data_offset),
2131		− bdevname(rdev->bdev, b));
	2196	+ pr_info("md/raid1:%s: read error corrected (%d sectors at %llu on %s)\n",
	2197	+ mdname(mddev), s,
	2198	+ (unsigned long long)(sect +
	2199	+ rdev->data_offset),
	2200	+ bdevname(rdev->bdev, b));
2132	2201	}
2133	2202	rdev_dec_pending(rdev, mddev);
2134	2203	} else
@@ -2287,6 +2356,8 @@ static void handle_read_error(struct r1conf conf, struct r1bio r1_bio) · 152 unmodified lines
2287	2356	struct bio *bio;
2288	2357	char b[BDEVNAME_SIZE];
2289	2358	struct md_rdev *rdev;
	2359	+ dev_t bio_dev;
	2360	+ sector_t bio_sector;
2290	2361
2291	2362	clear_bit(R1BIO_ReadError, &r1_bio->state);
2292	2363	/* we got a read error. Maybe the drive is bad. Maybe just
@@ -2300,10 +2371,14 @@ static void handle_read_error(struct r1conf conf, struct r1bio r1_bio) · 7 unmodified lines
2300	2371
2301	2372	bio = r1_bio->bios[r1_bio->read_disk];
2302	2373	bdevname(bio->bi_bdev, b);
	2374	+ bio_dev = bio->bi_bdev->bd_dev;
	2375	+ bio_sector = conf->mirrors[r1_bio->read_disk].rdev->data_offset + r1_bio->sector;
2303	2376	bio_put(bio);
2304	2377	r1_bio->bios[r1_bio->read_disk] = NULL;
2305	2378
2306		− if (mddev->ro == 0) {
	2379	+ rdev = conf->mirrors[r1_bio->read_disk].rdev;
	2380	+ if (mddev->ro == 0
	2381	+ && !test_bit(FailFast, &rdev->flags)) {
2307	2382	freeze_array(conf, 1);
2308	2383	fix_read_error(conf, r1_bio->read_disk,
2309	2384	r1_bio->sector, r1_bio->sectors);
@@ -2312,14 +2387,13 @@ static void handle_read_error(struct r1conf conf, struct r1bio r1_bio) · 2 unmodified lines
2312	2387	r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED;
2313	2388	}
2314	2389
2315		− rdev_dec_pending(conf->mirrors[r1_bio->read_disk].rdev, conf->mddev);
	2390	+ rdev_dec_pending(rdev, conf->mddev);
2316	2391
2317	2392	read_more:
2318	2393	disk = read_balance(conf, r1_bio, &max_sectors);
2319	2394	if (disk == -1) {
2320		− printk(KERN_ALERT "md/raid1:%s: %s: unrecoverable I/O"
2321		− " read error for block %llu\n",
2322		− mdname(mddev), b, (unsigned long long)r1_bio->sector);
	2395	+ pr_crit_ratelimited("md/raid1:%s: %s: unrecoverable I/O read error for block %llu\n",
	2396	+ mdname(mddev), b, (unsigned long long)r1_bio->sector);
2323	2397	raid_end_bio_io(r1_bio);
2324	2398	} else {
2325	2399	const unsigned long do_sync
@@ -2330,16 +2404,17 @@ read_more: · 4 unmodified lines
2330	2404	max_sectors);
2331	2405	r1_bio->bios[r1_bio->read_disk] = bio;
2332	2406	rdev = conf->mirrors[disk].rdev;
2333		− printk_ratelimited(KERN_ERR
2334		− "md/raid1:%s: redirecting sector %llu"
2335		− " to other mirror: %s\n",
2336		− mdname(mddev),
2337		− (unsigned long long)r1_bio->sector,
2338		− bdevname(rdev->bdev, b));
	2407	+ pr_info_ratelimited("md/raid1:%s: redirecting sector %llu to other mirror: %s\n",
	2408	+ mdname(mddev),
	2409	+ (unsigned long long)r1_bio->sector,
	2410	+ bdevname(rdev->bdev, b));
2339	2411	bio->bi_iter.bi_sector = r1_bio->sector + rdev->data_offset;
2340	2412	bio->bi_bdev = rdev->bdev;
2341	2413	bio->bi_end_io = raid1_end_read_request;
2342	2414	bio_set_op_attrs(bio, REQ_OP_READ, do_sync);
	2415	+ if (test_bit(FailFast, &rdev->flags) &&
	2416	+ test_bit(R1BIO_FailFast, &r1_bio->state))
	2417	+ bio->bi_opf \|= MD_FAILFAST;
2343	2418	bio->bi_private = r1_bio;
2344	2419	if (max_sectors < r1_bio->sectors) {
2345	2420	/* Drat - have to split this up more */
@@ -2353,6 +2428,8 @@ read_more: · 7 unmodified lines
2353	2428	else
2354	2429	mbio->bi_phys_segments++;
2355	2430	spin_unlock_irq(&conf->device_lock);
	2431	+ trace_block_bio_remap(bdev_get_queue(bio->bi_bdev),
	2432	+ bio, bio_dev, bio_sector);
2356	2433	generic_make_request(bio);
2357	2434	bio = NULL;
2358	2435
@@ -2367,8 +2444,11 @@ read_more: · 8 unmodified lines
2367	2444	sectors_handled;
2368	2445
2369	2446	goto read_more;
2370		− } else
	2447	+ } else {
	2448	+ trace_block_bio_remap(bdev_get_queue(bio->bi_bdev),
	2449	+ bio, bio_dev, bio_sector);
2371	2450	generic_make_request(bio);
	2451	+ }
2372	2452	}
2373	2453	}
2374	2454
@@ -2384,10 +2464,10 @@ static void raid1d(struct md_thread *thread) · 9 unmodified lines
2384	2464	md_check_recovery(mddev);
2385	2465
2386	2466	if (!list_empty_careful(&conf->bio_end_io_list) &&
2387		− !test_bit(MD_CHANGE_PENDING, &mddev->flags)) {
	2467	+ !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2388	2468	LIST_HEAD(tmp);
2389	2469	spin_lock_irqsave(&conf->device_lock, flags);
2390		− if (!test_bit(MD_CHANGE_PENDING, &mddev->flags)) {
	2470	+ if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2391	2471	while (!list_empty(&conf->bio_end_io_list)) {
2392	2472	list_move(conf->bio_end_io_list.prev, &tmp);
2393	2473	conf->nr_queued--;
@@ -2441,7 +2521,7 @@ static void raid1d(struct md_thread *thread) · 47 unmodified lines
2441	2521	generic_make_request(r1_bio->bios[r1_bio->read_disk]);
2442	2522
2443	2523	cond_resched();
2444		− if (mddev->flags & ~(1<<MD_CHANGE_PENDING))
	2524	+ if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))
2445	2525	md_check_recovery(mddev);
2446	2526	}
2447	2527	blk_finish_plug(&plug);
@@ -2623,6 +2703,8 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr, · 175 unmodified lines
2623	2703	bio->bi_iter.bi_sector = sector_nr + rdev->data_offset;
2624	2704	bio->bi_bdev = rdev->bdev;
2625	2705	bio->bi_private = r1_bio;
	2706	+ if (test_bit(FailFast, &rdev->flags))
	2707	+ bio->bi_opf \|= MD_FAILFAST;
2626	2708	}
2627	2709	}
2628	2710	rcu_read_unlock();
@@ -2642,7 +2724,7 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr, · 13 unmodified lines
2642	2724	min_bad, 0
2643	2725	) && ok;
2644	2726	}
2645		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
	2727	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
2646	2728	*skipped = 1;
2647	2729	put_buf(r1_bio);
2648	2730
@@ -2753,6 +2835,8 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr, · 104 unmodified lines
2753	2835	if (bio->bi_end_io == end_sync_read) {
2754	2836	read_targets--;
2755	2837	md_sync_acct(bio->bi_bdev, nr_sectors);
	2838	+ if (read_targets == 1)
	2839	+ bio->bi_opf &= ~MD_FAILFAST;
2756	2840	generic_make_request(bio);
2757	2841	}
2758	2842	}
@@ -2760,6 +2844,8 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr, · 1 unmodified line
2760	2844	atomic_set(&r1_bio->remaining, 1);
2761	2845	bio = r1_bio->bios[r1_bio->read_disk];
2762	2846	md_sync_acct(bio->bi_bdev, nr_sectors);
	2847	+ if (read_targets == 1)
	2848	+ bio->bi_opf &= ~MD_FAILFAST;
2763	2849	generic_make_request(bio);
2764	2850
2765	2851	}
@@ -2875,12 +2961,8 @@ static struct r1conf setup_conf(struct mddev mddev) · 109 unmodified lines
2875	2961
2876	2962	err = -ENOMEM;
2877	2963	conf->thread = md_register_thread(raid1d, mddev, "raid1");
2878		− if (!conf->thread) {
2879		− printk(KERN_ERR
2880		− "md/raid1:%s: couldn't allocate thread\n",
2881		− mdname(mddev));
	2964	+ if (!conf->thread)
2882	2965	goto abort;
2883		− }
2884	2966
2885	2967	return conf;
2886	2968
@@ -2905,13 +2987,13 @@ static int raid1_run(struct mddev *mddev) · 18 unmodified lines
2905	2987	bool discard_supported = false;
2906	2988
2907	2989	if (mddev->level != 1) {
2908		− printk(KERN_ERR "md/raid1:%s: raid level not set to mirroring (%d)\n",
2909		− mdname(mddev), mddev->level);
	2990	+ pr_warn("md/raid1:%s: raid level not set to mirroring (%d)\n",
	2991	+ mdname(mddev), mddev->level);
2910	2992	return -EIO;
2911	2993	}
2912	2994	if (mddev->reshape_position != MaxSector) {
2913		− printk(KERN_ERR "md/raid1:%s: reshape_position set but not supported\n",
2914		− mdname(mddev));
	2995	+ pr_warn("md/raid1:%s: reshape_position set but not supported\n",
	2996	+ mdname(mddev));
2915	2997	return -EIO;
2916	2998	}
2917	2999	/*
@@ -2950,11 +3032,9 @@ static int raid1_run(struct mddev *mddev) · 32 unmodified lines
2950	3032	mddev->recovery_cp = MaxSector;
2951	3033
2952	3034	if (mddev->recovery_cp != MaxSector)
2953		− printk(KERN_NOTICE "md/raid1:%s: not clean"
2954		− " -- starting background reconstruction\n",
2955		− mdname(mddev));
2956		− printk(KERN_INFO
2957		− "md/raid1:%s: active with %d out of %d mirrors\n",
	3035	+ pr_info("md/raid1:%s: not clean -- starting background reconstruction\n",
	3036	+ mdname(mddev));
	3037	+ pr_info("md/raid1:%s: active with %d out of %d mirrors\n",
2958	3038	mdname(mddev), mddev->raid_disks - mddev->degraded,
2959	3039	mddev->raid_disks);
2960	3040
@@ -2964,6 +3044,7 @@ static int raid1_run(struct mddev *mddev) · 3 unmodified lines
2964	3044	mddev->thread = conf->thread;
2965	3045	conf->thread = NULL;
2966	3046	mddev->private = conf;
	3047	+ set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
2967	3048
2968	3049	md_set_array_sectors(mddev, raid1_size(mddev, 0, 0));
2969	3050
@@ -3107,9 +3188,8 @@ static int raid1_reshape(struct mddev *mddev) · 137 unmodified lines
3107	3188	rdev->raid_disk = d2;
3108	3189	sysfs_unlink_rdev(mddev, rdev);
3109	3190	if (sysfs_link_rdev(mddev, rdev))
3110		− printk(KERN_WARNING
3111		− "md/raid1:%s: cannot register rd%d\n",
3112		− mdname(mddev), rdev->raid_disk);
	3191	+ pr_warn("md/raid1:%s: cannot register rd%d\n",
	3192	+ mdname(mddev), rdev->raid_disk);
3113	3193	}
3114	3194	if (rdev)
3115	3195	newmirrors[d2++].rdev = rdev;
@@ -3163,9 +3243,12 @@ static void raid1_takeover(struct mddev mddev) · 47 unmodified lines
3163	3243	mddev->new_layout = 0;
3164	3244	mddev->new_chunk_sectors = 0;
3165	3245	conf = setup_conf(mddev);
3166		− if (!IS_ERR(conf))
	3246	+ if (!IS_ERR(conf)) {
3167	3247	/* Array must appear to be quiesced */
3168	3248	conf->array_frozen = 1;
	3249	+ clear_bit(MD_HAS_JOURNAL, &mddev->flags);
	3250	+ clear_bit(MD_JOURNAL_CLEAN, &mddev->flags);
	3251	+ }
3169	3252	return conf;
3170	3253	}
3171	3254	return ERR_PTR(-EINVAL);

drivers/md/raid1.h+11−8

@@ -161,14 +161,15 @@ struct r1bio { · 160 unmodified lines
161	161	};
162	162
163	163	/* bits for r1bio.state */
164		−#define R1BIO_Uptodate 0
165		−#define R1BIO_IsSync 1
166		−#define R1BIO_Degraded 2
167		−#define R1BIO_BehindIO 3
	164	+enum r1bio_state {
	165	+ R1BIO_Uptodate,
	166	+ R1BIO_IsSync,
	167	+ R1BIO_Degraded,
	168	+ R1BIO_BehindIO,
168	169	/* Set ReadError on bios that experience a readerror so that
169	170	* raid1d knows what to do with them.
170	171	*/
171		−#define R1BIO_ReadError 4
	172	+ R1BIO_ReadError,
172	173	/* For write-behind requests, we call bi_end_io when
173	174	* the last non-write-behind device completes, providing
174	175	* any write was successful. Otherwise we call when
@@ -176,10 +177,12 @@ struct r1bio { · 1 unmodified line
176	177	* with failure when last write completes (and all failed).
177	178	* Record that bi_end_io was called with this flag...
178	179	*/
179		−#define R1BIO_Returned 6
	180	+ R1BIO_Returned,
180	181	/* If a write for this request means we can clear some
181	182	* known-bad-block records, we set this flag
182	183	*/
183		−#define R1BIO_MadeGood 7
184		−#define R1BIO_WriteError 8
	184	+ R1BIO_MadeGood,
	185	+ R1BIO_WriteError,
	186	+ R1BIO_FailFast,
	187	+};
185	188	#endif

drivers/md/raid10.c+189−106

@@ -25,6 +25,7 @@ · 24 unmodified lines
25	25	#include <linux/seq_file.h>
26	26	#include <linux/ratelimit.h>
27	27	#include <linux/kthread.h>
	28	+#include <trace/events/block.h>
28	29	#include "md.h"
29	30	#include "raid10.h"
30	31	#include "raid0.h"
@@ -99,12 +100,16 @@ static int max_queued_requests = 1024; · 68 unmodified lines
99	100	static void allow_barrier(struct r10conf *conf);
100	101	static void lower_barrier(struct r10conf *conf);
101	102	static int _enough(struct r10conf *conf, int previous, int ignore);
	103	+static int enough(struct r10conf *conf, int ignore);
102	104	static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
103	105	int *skipped);
104	106	static void reshape_request_write(struct mddev mddev, struct r10bio r10_bio);
105	107	static void end_reshape_write(struct bio *bio);
106	108	static void end_reshape(struct r10conf *conf);
107	109
	110	+#define raid10_log(md, fmt, args...) \
	111	+ do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid10 " fmt, ##args); } while (0)
	112	+
108	113	static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
109	114	{
110	115	struct r10conf *conf = data;
@@ -404,8 +409,7 @@ static void raid10_end_read_request(struct bio *bio) · 293 unmodified lines
404	409	* oops, read error - keep the refcount on the rdev
405	410	*/
406	411	char b[BDEVNAME_SIZE];
407		− printk_ratelimited(KERN_ERR
408		− "md/raid10:%s: %s: rescheduling sector %llu\n",
	412	+ pr_err_ratelimited("md/raid10:%s: %s: rescheduling sector %llu\n",
409	413	mdname(conf->mddev),
410	414	bdevname(rdev->bdev, b),
411	415	(unsigned long long)r10_bio->sector);
@@ -447,6 +451,7 @@ static void raid10_end_write_request(struct bio *bio) · 35 unmodified lines
447	451	struct r10conf *conf = r10_bio->mddev->private;
448	452	int slot, repl;
449	453	struct md_rdev *rdev = NULL;
	454	+ struct bio *to_put = NULL;
450	455	bool discard_error;
451	456
452	457	discard_error = bio->bi_error && bio_op(bio) == REQ_OP_DISCARD;
@@ -474,8 +479,24 @@ static void raid10_end_write_request(struct bio *bio) · 21 unmodified lines
474	479	if (!test_and_set_bit(WantReplacement, &rdev->flags))
475	480	set_bit(MD_RECOVERY_NEEDED,
476	481	&rdev->mddev->recovery);
477		− set_bit(R10BIO_WriteError, &r10_bio->state);
	482	+
478	483	dec_rdev = 0;
	484	+ if (test_bit(FailFast, &rdev->flags) &&
	485	+ (bio->bi_opf & MD_FAILFAST)) {
	486	+ md_error(rdev->mddev, rdev);
	487	+ if (!test_bit(Faulty, &rdev->flags))
	488	+ /* This is the only remaining device,
	489	+ * We need to retry the write without
	490	+ * FailFast
	491	+ */
	492	+ set_bit(R10BIO_WriteError, &r10_bio->state);
	493	+ else {
	494	+ r10_bio->devs[slot].bio = NULL;
	495	+ to_put = bio;
	496	+ dec_rdev = 1;
	497	+ }
	498	+ } else
	499	+ set_bit(R10BIO_WriteError, &r10_bio->state);
479	500	}
480	501	} else {
481	502	/*
@@ -525,6 +546,8 @@ static void raid10_end_write_request(struct bio *bio) · 43 unmodified lines
525	546	one_write_done(r10_bio);
526	547	if (dec_rdev)
527	548	rdev_dec_pending(rdev, conf->mddev);
	549	+ if (to_put)
	550	+ bio_put(to_put);
528	551	}
529	552
530	553	/*
@@ -716,6 +739,7 @@ static struct md_rdev read_balance(struct r10conf conf, · 185 unmodified lines
716	739	best_dist = MaxSector;
717	740	best_good_sectors = 0;
718	741	do_balance = 1;
	742	+ clear_bit(R10BIO_FailFast, &r10_bio->state);
719	743	/*
720	744	* Check if we can balance. We can balance on the whole
721	745	* device if no resync is going on (recovery is ok), or below
@@ -780,15 +804,18 @@ static struct md_rdev read_balance(struct r10conf conf, · 58 unmodified lines
780	804	if (!do_balance)
781	805	break;
782	806
	807	+ if (best_slot >= 0)
	808	+ /* At least 2 disks to choose from so failfast is OK */
	809	+ set_bit(R10BIO_FailFast, &r10_bio->state);
783	810	/* This optimisation is debatable, and completely destroys
784	811	* sequential read speed for 'far copies' arrays. So only
785	812	* keep it for 'near' arrays, and review those later.
786	813	*/
787	814	if (geo->near_copies > 1 && !atomic_read(&rdev->nr_pending))
788		− break;
	815	+ new_distance = 0;
789	816
790	817	/* for far > 1 always use the lowest address */
791		− if (geo->far_copies > 1)
	818	+ else if (geo->far_copies > 1)
792	819	new_distance = r10_bio->devs[slot].addr;
793	820	else
794	821	new_distance = abs(r10_bio->devs[slot].addr -
@@ -859,9 +886,14 @@ static void flush_pending_writes(struct r10conf *conf) · 64 unmodified lines
859	886
860	887	while (bio) { /* submit pending writes */
861	888	struct bio *next = bio->bi_next;
	889	+ struct md_rdev rdev = (void)bio->bi_bdev;
862	890	bio->bi_next = NULL;
863		− if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
864		− !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
	891	+ bio->bi_bdev = rdev->bdev;
	892	+ if (test_bit(Faulty, &rdev->flags)) {
	893	+ bio->bi_error = -EIO;
	894	+ bio_endio(bio);
	895	+ } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
	896	+ !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
865	897	/* Just ignore it */
866	898	bio_endio(bio);
867	899	else
@@ -937,6 +969,7 @@ static void wait_barrier(struct r10conf *conf) · 69 unmodified lines
937	969	* that queue to get the nr_pending
938	970	* count down.
939	971	*/
	972	+ raid10_log(conf->mddev, "wait barrier");
940	973	wait_event_lock_irq(conf->wait_barrier,
941	974	!conf->barrier \|\|
942	975	(atomic_read(&conf->nr_pending) &&
@@ -1037,9 +1070,14 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) · 94 unmodified lines
1037	1070
1038	1071	while (bio) { /* submit pending writes */
1039	1072	struct bio *next = bio->bi_next;
	1073	+ struct md_rdev rdev = (void)bio->bi_bdev;
1040	1074	bio->bi_next = NULL;
1041		− if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
1042		− !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
	1075	+ bio->bi_bdev = rdev->bdev;
	1076	+ if (test_bit(Faulty, &rdev->flags)) {
	1077	+ bio->bi_error = -EIO;
	1078	+ bio_endio(bio);
	1079	+ } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
	1080	+ !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
1043	1081	/* Just ignore it */
1044	1082	bio_endio(bio);
1045	1083	else
@@ -1083,6 +1121,7 @@ static void __make_request(struct mddev mddev, struct bio bio) · 37 unmodified lines
1083	1121	/* IO spans the reshape position. Need to wait for
1084	1122	* reshape to pass
1085	1123	*/
	1124	+ raid10_log(conf->mddev, "wait reshape");
1086	1125	allow_barrier(conf);
1087	1126	wait_event(conf->wait_barrier,
1088	1127	conf->reshape_progress <= bio->bi_iter.bi_sector \|\|
@@ -1099,11 +1138,12 @@ static void __make_request(struct mddev mddev, struct bio bio) · 10 unmodified lines
1099	1138	bio->bi_iter.bi_sector < conf->reshape_progress))) {
1100	1139	/* Need to update reshape_position in metadata */
1101	1140	mddev->reshape_position = conf->reshape_progress;
1102		− set_mask_bits(&mddev->flags, 0,
1103		− BIT(MD_CHANGE_DEVS) \| BIT(MD_CHANGE_PENDING));
	1141	+ set_mask_bits(&mddev->sb_flags, 0,
	1142	+ BIT(MD_SB_CHANGE_DEVS) \| BIT(MD_SB_CHANGE_PENDING));
1104	1143	md_wakeup_thread(mddev->thread);
	1144	+ raid10_log(conf->mddev, "wait reshape metadata");
1105	1145	wait_event(mddev->sb_wait,
1106		− !test_bit(MD_CHANGE_PENDING, &mddev->flags));
	1146	+ !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
1107	1147
1108	1148	conf->reshape_safe = mddev->reshape_position;
1109	1149	}
@@ -1154,8 +1194,15 @@ read_again: · 44 unmodified lines
1154	1194	read_bio->bi_bdev = rdev->bdev;
1155	1195	read_bio->bi_end_io = raid10_end_read_request;
1156	1196	bio_set_op_attrs(read_bio, op, do_sync);
	1197	+ if (test_bit(FailFast, &rdev->flags) &&
	1198	+ test_bit(R10BIO_FailFast, &r10_bio->state))
	1199	+ read_bio->bi_opf \|= MD_FAILFAST;
1157	1200	read_bio->bi_private = r10_bio;
1158	1201
	1202	+ if (mddev->gendisk)
	1203	+ trace_block_bio_remap(bdev_get_queue(read_bio->bi_bdev),
	1204	+ read_bio, disk_devt(mddev->gendisk),
	1205	+ r10_bio->sector);
1159	1206	if (max_sectors < r10_bio->sectors) {
1160	1207	/* Could not read all from this device, so we will
1161	1208	* need another r10_bio.
@@ -1195,6 +1242,7 @@ read_again: · 33 unmodified lines
1195	1242	*/
1196	1243	if (conf->pending_count >= max_queued_requests) {
1197	1244	md_wakeup_thread(mddev->thread);
	1245	+ raid10_log(mddev, "wait queued");
1198	1246	wait_event(conf->wait_barrier,
1199	1247	conf->pending_count < max_queued_requests);
1200	1248	}
@@ -1322,6 +1370,7 @@ retry_write: · 121 unmodified lines
1322	1370	}
1323	1371	}
1324	1372	allow_barrier(conf);
	1373	+ raid10_log(conf->mddev, "wait rdev %d blocked", blocked_rdev->raid_disk);
1325	1374	md_wait_for_blocked_rdev(blocked_rdev, mddev);
1326	1375	wait_barrier(conf);
1327	1376	goto retry_write;
@@ -1361,8 +1410,18 @@ retry_write: · 33 unmodified lines
1361	1410	mbio->bi_bdev = rdev->bdev;
1362	1411	mbio->bi_end_io = raid10_end_write_request;
1363	1412	bio_set_op_attrs(mbio, op, do_sync \| do_fua);
	1413	+ if (test_bit(FailFast, &conf->mirrors[d].rdev->flags) &&
	1414	+ enough(conf, d))
	1415	+ mbio->bi_opf \|= MD_FAILFAST;
1364	1416	mbio->bi_private = r10_bio;
1365	1417
	1418	+ if (conf->mddev->gendisk)
	1419	+ trace_block_bio_remap(bdev_get_queue(mbio->bi_bdev),
	1420	+ mbio, disk_devt(conf->mddev->gendisk),
	1421	+ r10_bio->sector);
	1422	+ /* flush_pending_writes() needs access to the rdev so...*/
	1423	+ mbio->bi_bdev = (void*)rdev;
	1424	+
1366	1425	atomic_inc(&r10_bio->remaining);
1367	1426
1368	1427	cb = blk_check_plugged(raid10_unplug, mddev,
@@ -1405,6 +1464,13 @@ retry_write: · 36 unmodified lines
1405	1464	bio_set_op_attrs(mbio, op, do_sync \| do_fua);
1406	1465	mbio->bi_private = r10_bio;
1407	1466
	1467	+ if (conf->mddev->gendisk)
	1468	+ trace_block_bio_remap(bdev_get_queue(mbio->bi_bdev),
	1469	+ mbio, disk_devt(conf->mddev->gendisk),
	1470	+ r10_bio->sector);
	1471	+ /* flush_pending_writes() needs access to the rdev so...*/
	1472	+ mbio->bi_bdev = (void*)rdev;
	1473	+
1408	1474	atomic_inc(&r10_bio->remaining);
1409	1475	spin_lock_irqsave(&conf->device_lock, flags);
1410	1476	bio_list_add(&conf->pending_bio_list, mbio);
@@ -1586,14 +1652,13 @@ static void raid10_error(struct mddev mddev, struct md_rdev rdev) · 175 unmodified lines
1586	1652	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1587	1653	set_bit(Blocked, &rdev->flags);
1588	1654	set_bit(Faulty, &rdev->flags);
1589		− set_mask_bits(&mddev->flags, 0,
1590		− BIT(MD_CHANGE_DEVS) \| BIT(MD_CHANGE_PENDING));
	1655	+ set_mask_bits(&mddev->sb_flags, 0,
	1656	+ BIT(MD_SB_CHANGE_DEVS) \| BIT(MD_SB_CHANGE_PENDING));
1591	1657	spin_unlock_irqrestore(&conf->device_lock, flags);
1592		− printk(KERN_ALERT
1593		− "md/raid10:%s: Disk failure on %s, disabling device.\n"
1594		− "md/raid10:%s: Operation continuing on %d devices.\n",
1595		− mdname(mddev), bdevname(rdev->bdev, b),
1596		− mdname(mddev), conf->geo.raid_disks - mddev->degraded);
	1658	+ pr_crit("md/raid10:%s: Disk failure on %s, disabling device.\n"
	1659	+ "md/raid10:%s: Operation continuing on %d devices.\n",
	1660	+ mdname(mddev), bdevname(rdev->bdev, b),
	1661	+ mdname(mddev), conf->geo.raid_disks - mddev->degraded);
1597	1662	}
1598	1663
1599	1664	static void print_conf(struct r10conf *conf)
@@ -1601,13 +1666,13 @@ static void print_conf(struct r10conf *conf) · 1 unmodified line
1601	1666	int i;
1602	1667	struct md_rdev *rdev;
1603	1668
1604		− printk(KERN_DEBUG "RAID10 conf printout:\n");
	1669	+ pr_debug("RAID10 conf printout:\n");
1605	1670	if (!conf) {
1606		− printk(KERN_DEBUG "(!conf)\n");
	1671	+ pr_debug("(!conf)\n");
1607	1672	return;
1608	1673	}
1609		− printk(KERN_DEBUG " --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
1610		− conf->geo.raid_disks);
	1674	+ pr_debug(" --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
	1675	+ conf->geo.raid_disks);
1611	1676
1612	1677	/* This is only called with ->reconfix_mutex held, so
1613	1678	* rcu protection of rdev is not needed */
@@ -1615,10 +1680,10 @@ static void print_conf(struct r10conf *conf) · 1 unmodified line
1615	1680	char b[BDEVNAME_SIZE];
1616	1681	rdev = conf->mirrors[i].rdev;
1617	1682	if (rdev)
1618		− printk(KERN_DEBUG " disk %d, wo:%d, o:%d, dev:%s\n",
1619		− i, !test_bit(In_sync, &rdev->flags),
1620		− !test_bit(Faulty, &rdev->flags),
1621		− bdevname(rdev->bdev,b));
	1683	+ pr_debug(" disk %d, wo:%d, o:%d, dev:%s\n",
	1684	+ i, !test_bit(In_sync, &rdev->flags),
	1685	+ !test_bit(Faulty, &rdev->flags),
	1686	+ bdevname(rdev->bdev,b));
1622	1687	}
1623	1688	}
1624	1689
@@ -1953,6 +2018,7 @@ static void sync_request_write(struct mddev mddev, struct r10bio r10_bio) · 328 unmodified lines
1953	2018	/* now find blocks with errors */
1954	2019	for (i=0 ; i < conf->copies ; i++) {
1955	2020	int j, d;
	2021	+ struct md_rdev *rdev;
1956	2022
1957	2023	tbio = r10_bio->devs[i].bio;
1958	2024
@@ -1960,6 +2026,8 @@ static void sync_request_write(struct mddev mddev, struct r10bio r10_bio) · 1 unmodified line
1960	2026	continue;
1961	2027	if (i == first)
1962	2028	continue;
	2029	+ d = r10_bio->devs[i].devnum;
	2030	+ rdev = conf->mirrors[d].rdev;
1963	2031	if (!r10_bio->devs[i].bio->bi_error) {
1964	2032	/* We know that the bi_io_vec layout is the same for
1965	2033	* both 'first' and 'i', so we just compare them.
@@ -1982,6 +2050,10 @@ static void sync_request_write(struct mddev mddev, struct r10bio r10_bio) · 16 unmodified lines
1982	2050	if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
1983	2051	/* Don't fix anything. */
1984	2052	continue;
	2053	+ } else if (test_bit(FailFast, &rdev->flags)) {
	2054	+ /* Just give up on this device */
	2055	+ md_error(rdev->mddev, rdev);
	2056	+ continue;
1985	2057	}
1986	2058	/* Ok, we need to write this bio, either to correct an
1987	2059	* inconsistency or to correct an unreadable block.
@@ -1999,11 +2071,12 @@ static void sync_request_write(struct mddev mddev, struct r10bio r10_bio) · 11 unmodified lines
1999	2071
2000	2072	bio_copy_data(tbio, fbio);
2001	2073
2002		− d = r10_bio->devs[i].devnum;
2003	2074	atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2004	2075	atomic_inc(&r10_bio->remaining);
2005	2076	md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(tbio));
2006	2077
	2078	+ if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
	2079	+ tbio->bi_opf \|= MD_FAILFAST;
2007	2080	tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset;
2008	2081	tbio->bi_bdev = conf->mirrors[d].rdev->bdev;
2009	2082	generic_make_request(tbio);
@@ -2109,10 +2182,8 @@ static void fix_recovery_read_error(struct r10bio *r10_bio) · 99 unmodified lines
2109	2182	ok = rdev_set_badblocks(rdev2, addr, s, 0);
2110	2183	if (!ok) {
2111	2184	/* just abort the recovery */
2112		− printk(KERN_NOTICE
2113		− "md/raid10:%s: recovery aborted"
2114		− " due to read error\n",
2115		− mdname(mddev));
	2185	+ pr_notice("md/raid10:%s: recovery aborted due to read error\n",
	2186	+ mdname(mddev));
2116	2187
2117	2188	conf->mirrors[dw].recovery_disabled
2118	2189	= mddev->recovery_disabled;
@@ -2259,14 +2330,11 @@ static void fix_read_error(struct r10conf conf, struct mddev mddev, struct r10 · 140 unmodified lines
2259	2330	char b[BDEVNAME_SIZE];
2260	2331	bdevname(rdev->bdev, b);
2261	2332
2262		− printk(KERN_NOTICE
2263		− "md/raid10:%s: %s: Raid device exceeded "
2264		− "read_error threshold [cur %d:max %d]\n",
2265		− mdname(mddev), b,
2266		− atomic_read(&rdev->read_errors), max_read_errors);
2267		− printk(KERN_NOTICE
2268		− "md/raid10:%s: %s: Failing raid device\n",
2269		− mdname(mddev), b);
	2333	+ pr_notice("md/raid10:%s: %s: Raid device exceeded read_error threshold [cur %d:max %d]\n",
	2334	+ mdname(mddev), b,
	2335	+ atomic_read(&rdev->read_errors), max_read_errors);
	2336	+ pr_notice("md/raid10:%s: %s: Failing raid device\n",
	2337	+ mdname(mddev), b);
2270	2338	md_error(mddev, rdev);
2271	2339	r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED;
2272	2340	return;
@@ -2356,20 +2424,16 @@ static void fix_read_error(struct r10conf conf, struct mddev mddev, struct r10 · 83 unmodified lines
2356	2424	s, conf->tmppage, WRITE)
2357	2425	== 0) {
2358	2426	/* Well, this device is dead */
2359		− printk(KERN_NOTICE
2360		− "md/raid10:%s: read correction "
2361		− "write failed"
2362		− " (%d sectors at %llu on %s)\n",
2363		− mdname(mddev), s,
2364		− (unsigned long long)(
2365		− sect +
2366		− choose_data_offset(r10_bio,
2367		− rdev)),
2368		− bdevname(rdev->bdev, b));
2369		− printk(KERN_NOTICE "md/raid10:%s: %s: failing "
2370		− "drive\n",
2371		− mdname(mddev),
2372		− bdevname(rdev->bdev, b));
	2427	+ pr_notice("md/raid10:%s: read correction write failed (%d sectors at %llu on %s)\n",
	2428	+ mdname(mddev), s,
	2429	+ (unsigned long long)(
	2430	+ sect +
	2431	+ choose_data_offset(r10_bio,
	2432	+ rdev)),
	2433	+ bdevname(rdev->bdev, b));
	2434	+ pr_notice("md/raid10:%s: %s: failing drive\n",
	2435	+ mdname(mddev),
	2436	+ bdevname(rdev->bdev, b));
2373	2437	}
2374	2438	rdev_dec_pending(rdev, mddev);
2375	2439	rcu_read_lock();
@@ -2397,24 +2461,18 @@ static void fix_read_error(struct r10conf conf, struct mddev mddev, struct r10 · 21 unmodified lines
2397	2461	READ)) {
2398	2462	case 0:
2399	2463	/* Well, this device is dead */
2400		− printk(KERN_NOTICE
2401		− "md/raid10:%s: unable to read back "
2402		− "corrected sectors"
2403		− " (%d sectors at %llu on %s)\n",
	2464	+ pr_notice("md/raid10:%s: unable to read back corrected sectors (%d sectors at %llu on %s)\n",
2404	2465	mdname(mddev), s,
2405	2466	(unsigned long long)(
2406	2467	sect +
2407	2468	choose_data_offset(r10_bio, rdev)),
2408	2469	bdevname(rdev->bdev, b));
2409		− printk(KERN_NOTICE "md/raid10:%s: %s: failing "
2410		− "drive\n",
	2470	+ pr_notice("md/raid10:%s: %s: failing drive\n",
2411	2471	mdname(mddev),
2412	2472	bdevname(rdev->bdev, b));
2413	2473	break;
2414	2474	case 1:
2415		− printk(KERN_INFO
2416		− "md/raid10:%s: read error corrected"
2417		− " (%d sectors at %llu on %s)\n",
	2475	+ pr_info("md/raid10:%s: read error corrected (%d sectors at %llu on %s)\n",
2418	2476	mdname(mddev), s,
2419	2477	(unsigned long long)(
2420	2478	sect +
@@ -2503,6 +2561,8 @@ static void handle_read_error(struct mddev mddev, struct r10bio r10_bio) · 82 unmodified lines
2503	2561	char b[BDEVNAME_SIZE];
2504	2562	unsigned long do_sync;
2505	2563	int max_sectors;
	2564	+ dev_t bio_dev;
	2565	+ sector_t bio_last_sector;
2506	2566
2507	2567	/* we got a read error. Maybe the drive is bad. Maybe just
2508	2568	* the block and we can fix it.
@@ -2514,38 +2574,38 @@ static void handle_read_error(struct mddev mddev, struct r10bio r10_bio) · 5 unmodified lines
2514	2574	*/
2515	2575	bio = r10_bio->devs[slot].bio;
2516	2576	bdevname(bio->bi_bdev, b);
	2577	+ bio_dev = bio->bi_bdev->bd_dev;
	2578	+ bio_last_sector = r10_bio->devs[slot].addr + rdev->data_offset + r10_bio->sectors;
2517	2579	bio_put(bio);
2518	2580	r10_bio->devs[slot].bio = NULL;
2519	2581
2520		− if (mddev->ro == 0) {
	2582	+ if (mddev->ro)
	2583	+ r10_bio->devs[slot].bio = IO_BLOCKED;
	2584	+ else if (!test_bit(FailFast, &rdev->flags)) {
2521	2585	freeze_array(conf, 1);
2522	2586	fix_read_error(conf, mddev, r10_bio);
2523	2587	unfreeze_array(conf);
2524	2588	} else
2525		− r10_bio->devs[slot].bio = IO_BLOCKED;
	2589	+ md_error(mddev, rdev);
2526	2590
2527	2591	rdev_dec_pending(rdev, mddev);
2528	2592
2529	2593	read_more:
2530	2594	rdev = read_balance(conf, r10_bio, &max_sectors);
2531	2595	if (rdev == NULL) {
2532		− printk(KERN_ALERT "md/raid10:%s: %s: unrecoverable I/O"
2533		− " read error for block %llu\n",
2534		− mdname(mddev), b,
2535		− (unsigned long long)r10_bio->sector);
	2596	+ pr_crit_ratelimited("md/raid10:%s: %s: unrecoverable I/O read error for block %llu\n",
	2597	+ mdname(mddev), b,
	2598	+ (unsigned long long)r10_bio->sector);
2536	2599	raid_end_bio_io(r10_bio);
2537	2600	return;
2538	2601	}
2539	2602
2540	2603	do_sync = (r10_bio->master_bio->bi_opf & REQ_SYNC);
2541	2604	slot = r10_bio->read_slot;
2542		− printk_ratelimited(
2543		− KERN_ERR
2544		− "md/raid10:%s: %s: redirecting "
2545		− "sector %llu to another mirror\n",
2546		− mdname(mddev),
2547		− bdevname(rdev->bdev, b),
2548		− (unsigned long long)r10_bio->sector);
	2605	+ pr_err_ratelimited("md/raid10:%s: %s: redirecting sector %llu to another mirror\n",
	2606	+ mdname(mddev),
	2607	+ bdevname(rdev->bdev, b),
	2608	+ (unsigned long long)r10_bio->sector);
2549	2609	bio = bio_clone_mddev(r10_bio->master_bio,
2550	2610	GFP_NOIO, mddev);
2551	2611	bio_trim(bio, r10_bio->sector - bio->bi_iter.bi_sector, max_sectors);
@@ -2555,8 +2615,15 @@ read_more: · 3 unmodified lines
2555	2615	+ choose_data_offset(r10_bio, rdev);
2556	2616	bio->bi_bdev = rdev->bdev;
2557	2617	bio_set_op_attrs(bio, REQ_OP_READ, do_sync);
	2618	+ if (test_bit(FailFast, &rdev->flags) &&
	2619	+ test_bit(R10BIO_FailFast, &r10_bio->state))
	2620	+ bio->bi_opf \|= MD_FAILFAST;
2558	2621	bio->bi_private = r10_bio;
2559	2622	bio->bi_end_io = raid10_end_read_request;
	2623	+ trace_block_bio_remap(bdev_get_queue(bio->bi_bdev),
	2624	+ bio, bio_dev,
	2625	+ bio_last_sector - r10_bio->sectors);
	2626	+
2560	2627	if (max_sectors < r10_bio->sectors) {
2561	2628	/* Drat - have to split this up more */
2562	2629	struct bio *mbio = r10_bio->master_bio;
@@ -2694,10 +2761,10 @@ static void raid10d(struct md_thread *thread) · 131 unmodified lines
2694	2761	md_check_recovery(mddev);
2695	2762
2696	2763	if (!list_empty_careful(&conf->bio_end_io_list) &&
2697		− !test_bit(MD_CHANGE_PENDING, &mddev->flags)) {
	2764	+ !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2698	2765	LIST_HEAD(tmp);
2699	2766	spin_lock_irqsave(&conf->device_lock, flags);
2700		− if (!test_bit(MD_CHANGE_PENDING, &mddev->flags)) {
	2767	+ if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2701	2768	while (!list_empty(&conf->bio_end_io_list)) {
2702	2769	list_move(conf->bio_end_io_list.prev, &tmp);
2703	2770	conf->nr_queued--;
@@ -2755,7 +2822,7 @@ static void raid10d(struct md_thread *thread) · 51 unmodified lines
2755	2822	}
2756	2823
2757	2824	cond_resched();
2758		− if (mddev->flags & ~(1<<MD_CHANGE_PENDING))
	2825	+ if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))
2759	2826	md_check_recovery(mddev);
2760	2827	}
2761	2828	blk_finish_plug(&plug);
@@ -3072,6 +3139,8 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, · 310 unmodified lines
3072	3139	bio->bi_private = r10_bio;
3073	3140	bio->bi_end_io = end_sync_read;
3074	3141	bio_set_op_attrs(bio, REQ_OP_READ, 0);
	3142	+ if (test_bit(FailFast, &rdev->flags))
	3143	+ bio->bi_opf \|= MD_FAILFAST;
3075	3144	from_addr = r10_bio->devs[j].addr;
3076	3145	bio->bi_iter.bi_sector = from_addr +
3077	3146	rdev->data_offset;
@@ -3160,8 +3229,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, · 82 unmodified lines
3160	3229	if (!any_working) {
3161	3230	if (!test_and_set_bit(MD_RECOVERY_INTR,
3162	3231	&mddev->recovery))
3163		− printk(KERN_INFO "md/raid10:%s: insufficient "
3164		− "working devices for recovery.\n",
	3232	+ pr_warn("md/raid10:%s: insufficient working devices for recovery.\n",
3165	3233	mdname(mddev));
3166	3234	mirror->recovery_disabled
3167	3235	= mddev->recovery_disabled;
@@ -3178,6 +3246,23 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, · 10 unmodified lines
3178	3246	rdev_dec_pending(mrdev, mddev);
3179	3247	if (mreplace)
3180	3248	rdev_dec_pending(mreplace, mddev);
	3249	+ if (r10_bio->devs[0].bio->bi_opf & MD_FAILFAST) {
	3250	+ /* Only want this if there is elsewhere to
	3251	+ * read from. 'j' is currently the first
	3252	+ * readable copy.
	3253	+ */
	3254	+ int targets = 1;
	3255	+ for (; j < conf->copies; j++) {
	3256	+ int d = r10_bio->devs[j].devnum;
	3257	+ if (conf->mirrors[d].rdev &&
	3258	+ test_bit(In_sync,
	3259	+ &conf->mirrors[d].rdev->flags))
	3260	+ targets++;
	3261	+ }
	3262	+ if (targets == 1)
	3263	+ r10_bio->devs[0].bio->bi_opf
	3264	+ &= ~MD_FAILFAST;
	3265	+ }
3181	3266	}
3182	3267	if (biolist == NULL) {
3183	3268	while (r10_bio) {
@@ -3256,6 +3341,8 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, · 72 unmodified lines
3256	3341	bio->bi_private = r10_bio;
3257	3342	bio->bi_end_io = end_sync_read;
3258	3343	bio_set_op_attrs(bio, REQ_OP_READ, 0);
	3344	+ if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
	3345	+ bio->bi_opf \|= MD_FAILFAST;
3259	3346	bio->bi_iter.bi_sector = sector + rdev->data_offset;
3260	3347	bio->bi_bdev = rdev->bdev;
3261	3348	count++;
@@ -3279,6 +3366,8 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, · 17 unmodified lines
3279	3366	bio->bi_private = r10_bio;
3280	3367	bio->bi_end_io = end_sync_write;
3281	3368	bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
	3369	+ if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
	3370	+ bio->bi_opf \|= MD_FAILFAST;
3282	3371	bio->bi_iter.bi_sector = sector + rdev->data_offset;
3283	3372	bio->bi_bdev = rdev->bdev;
3284	3373	count++;
@@ -3489,15 +3578,14 @@ static struct r10conf setup_conf(struct mddev mddev) · 204 unmodified lines
3489	3578	copies = setup_geo(&geo, mddev, geo_new);
3490	3579
3491	3580	if (copies == -2) {
3492		− printk(KERN_ERR "md/raid10:%s: chunk size must be "
3493		− "at least PAGE_SIZE(%ld) and be a power of 2.\n",
3494		− mdname(mddev), PAGE_SIZE);
	3581	+ pr_warn("md/raid10:%s: chunk size must be at least PAGE_SIZE(%ld) and be a power of 2.\n",
	3582	+ mdname(mddev), PAGE_SIZE);
3495	3583	goto out;
3496	3584	}
3497	3585
3498	3586	if (copies < 2 \|\| copies > mddev->raid_disks) {
3499		− printk(KERN_ERR "md/raid10:%s: unsupported raid10 layout: 0x%8x\n",
3500		− mdname(mddev), mddev->new_layout);
	3587	+ pr_warn("md/raid10:%s: unsupported raid10 layout: 0x%8x\n",
	3588	+ mdname(mddev), mddev->new_layout);
3501	3589	goto out;
3502	3590	}
3503	3591
@@ -3557,9 +3645,6 @@ static struct r10conf setup_conf(struct mddev mddev) · 53 unmodified lines
3557	3645	return conf;
3558	3646
3559	3647	out:
3560		− if (err == -ENOMEM)
3561		− printk(KERN_ERR "md/raid10:%s: couldn't allocate memory.\n",
3562		− mdname(mddev));
3563	3648	if (conf) {
3564	3649	mempool_destroy(conf->r10bio_pool);
3565	3650	kfree(conf->mirrors);
@@ -3656,7 +3741,7 @@ static int raid10_run(struct mddev *mddev) · 90 unmodified lines
3656	3741	}
3657	3742	/* need to check that every block has at least one working mirror */
3658	3743	if (!enough(conf, -1)) {
3659		− printk(KERN_ERR "md/raid10:%s: not enough operational mirrors.\n",
	3744	+ pr_err("md/raid10:%s: not enough operational mirrors.\n",
3660	3745	mdname(mddev));
3661	3746	goto out_free_conf;
3662	3747	}
@@ -3698,11 +3783,9 @@ static int raid10_run(struct mddev *mddev) · 35 unmodified lines
3698	3783	}
3699	3784
3700	3785	if (mddev->recovery_cp != MaxSector)
3701		− printk(KERN_NOTICE "md/raid10:%s: not clean"
3702		− " -- starting background reconstruction\n",
3703		− mdname(mddev));
3704		− printk(KERN_INFO
3705		− "md/raid10:%s: active with %d out of %d devices\n",
	3786	+ pr_notice("md/raid10:%s: not clean -- starting background reconstruction\n",
	3787	+ mdname(mddev));
	3788	+ pr_info("md/raid10:%s: active with %d out of %d devices\n",
3706	3789	mdname(mddev), conf->geo.raid_disks - mddev->degraded,
3707	3790	conf->geo.raid_disks);
3708	3791	/*
@@ -3712,6 +3795,7 @@ static int raid10_run(struct mddev *mddev) · 3 unmodified lines
3712	3795	size = raid10_size(mddev, 0, 0);
3713	3796	md_set_array_sectors(mddev, size);
3714	3797	mddev->resync_max_sectors = size;
	3798	+ set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
3715	3799
3716	3800	if (mddev->queue) {
3717	3801	int stripe = conf->geo.raid_disks *
@@ -3739,7 +3823,7 @@ static int raid10_run(struct mddev *mddev) · 21 unmodified lines
3739	3823
3740	3824	if (max(before_length, after_length) > min_offset_diff) {
3741	3825	/* This cannot work */
3742		− printk("md/raid10: offset difference not enough to continue reshape\n");
	3826	+ pr_warn("md/raid10: offset difference not enough to continue reshape\n");
3743	3827	goto out_free_conf;
3744	3828	}
3745	3829	conf->offset_diff = min_offset_diff;
@@ -3846,8 +3930,8 @@ static void raid10_takeover_raid0(struct mddev mddev, sector_t size, int devs) · 100 unmodified lines
3846	3930	struct r10conf *conf;
3847	3931
3848	3932	if (mddev->degraded > 0) {
3849		− printk(KERN_ERR "md/raid10:%s: Error: degraded raid0!\n",
3850		− mdname(mddev));
	3933	+ pr_warn("md/raid10:%s: Error: degraded raid0!\n",
	3934	+ mdname(mddev));
3851	3935	return ERR_PTR(-EINVAL);
3852	3936	}
3853	3937	sector_div(size, devs);
@@ -3887,9 +3971,8 @@ static void raid10_takeover(struct mddev mddev) · 33 unmodified lines
3887	3971	/* for raid0 takeover only one zone is supported */
3888	3972	raid0_conf = mddev->private;
3889	3973	if (raid0_conf->nr_strip_zones > 1) {
3890		− printk(KERN_ERR "md/raid10:%s: cannot takeover raid 0"
3891		− " with more than one zone.\n",
3892		− mdname(mddev));
	3974	+ pr_warn("md/raid10:%s: cannot takeover raid 0 with more than one zone.\n",
	3975	+ mdname(mddev));
3893	3976	return ERR_PTR(-EINVAL);
3894	3977	}
3895	3978	return raid10_takeover_raid0(mddev,
@@ -4078,8 +4161,8 @@ static int raid10_start_reshape(struct mddev *mddev) · 182 unmodified lines
4078	4161	sector_t size = raid10_size(mddev, 0, 0);
4079	4162	if (size < mddev->array_sectors) {
4080	4163	spin_unlock_irq(&conf->device_lock);
4081		− printk(KERN_ERR "md/raid10:%s: array size must be reduce before number of disks\n",
4082		− mdname(mddev));
	4164	+ pr_warn("md/raid10:%s: array size must be reduce before number of disks\n",
	4165	+ mdname(mddev));
4083	4166	return -EINVAL;
4084	4167	}
4085	4168	mddev->resync_max_sectors = size;
@@ -4126,7 +4209,7 @@ static int raid10_start_reshape(struct mddev *mddev) · 40 unmodified lines
4126	4209	spin_unlock_irq(&conf->device_lock);
4127	4210	mddev->raid_disks = conf->geo.raid_disks;
4128	4211	mddev->reshape_position = conf->reshape_progress;
4129		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
	4212	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
4130	4213
4131	4214	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
4132	4215	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
@@ -4321,9 +4404,9 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, · 188 unmodified lines
4321	4404	else
4322	4405	mddev->curr_resync_completed = conf->reshape_progress;
4323	4406	conf->reshape_checkpoint = jiffies;
4324		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
	4407	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
4325	4408	md_wakeup_thread(mddev->thread);
4326		− wait_event(mddev->sb_wait, mddev->flags == 0 \|\|
	4409	+ wait_event(mddev->sb_wait, mddev->sb_flags == 0 \|\|
4327	4410	test_bit(MD_RECOVERY_INTR, &mddev->recovery));
4328	4411	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
4329	4412	allow_barrier(conf);

drivers/md/raid10.h+2−0

@@ -156,5 +156,7 @@ enum r10bio_state { · 155 unmodified lines
156	156	* flag is set
157	157	*/
158	158	R10BIO_Previous,
	159	+/* failfast devices did receive failfast requests. */
	160	+ R10BIO_FailFast,
159	161	};
160	162	#endif

drivers/md/raid5-cache.c+1645−240

@@ -1,5 +1,6 @@
1	1	/*
2	2	* Copyright (C) 2015 Shaohua Li <shli@fb.com>
	3	+ * Copyright (C) 2016 Song Liu <songliubraving@fb.com>
3	4	*
4	5	* This program is free software; you can redistribute it and/or modify it
5	6	* under the terms and conditions of the GNU General Public License,
@@ -18,8 +19,10 @@ · 12 unmodified lines
18	19	#include <linux/raid/md_p.h>
19	20	#include <linux/crc32c.h>
20	21	#include <linux/random.h>
	22	+#include <linux/kthread.h>
21	23	#include "md.h"
22	24	#include "raid5.h"
	25	+#include "bitmap.h"
23	26
24	27	/*
25	28	* metadata/data stored in disk with 4k size unit (a block) regardless
@@ -28,18 +31,70 @@ · 2 unmodified lines
28	31	#define BLOCK_SECTORS (8)
29	32
30	33	/*
31		− * reclaim runs every 1/4 disk size or 10G reclaimable space. This can prevent
32		− * recovery scans a very long log
	34	+ * log->max_free_space is min(1/4 disk size, 10G reclaimable space).
	35	+ *
	36	+ * In write through mode, the reclaim runs every log->max_free_space.
	37	+ * This can prevent the recovery scans for too long
33	38	*/
34	39	#define RECLAIM_MAX_FREE_SPACE (10 * 1024 * 1024 * 2) /* sector */
35	40	#define RECLAIM_MAX_FREE_SPACE_SHIFT (2)
36	41
	42	+/* wake up reclaim thread periodically */
	43	+#define R5C_RECLAIM_WAKEUP_INTERVAL (30 * HZ)
	44	+/* start flush with these full stripes */
	45	+#define R5C_FULL_STRIPE_FLUSH_BATCH 256
	46	+/* reclaim stripes in groups */
	47	+#define R5C_RECLAIM_STRIPE_GROUP (NR_STRIPE_HASH_LOCKS * 2)
	48	+
37	49	/*
38	50	* We only need 2 bios per I/O unit to make progress, but ensure we
39	51	* have a few more available to not get too tight.
40	52	*/
41	53	#define R5L_POOL_SIZE 4
42	54
	55	+/*
	56	+ * r5c journal modes of the array: write-back or write-through.
	57	+ * write-through mode has identical behavior as existing log only
	58	+ * implementation.
	59	+ */
	60	+enum r5c_journal_mode {
	61	+ R5C_JOURNAL_MODE_WRITE_THROUGH = 0,
	62	+ R5C_JOURNAL_MODE_WRITE_BACK = 1,
	63	+};
	64	+
	65	+static char *r5c_journal_mode_str[] = {"write-through",
	66	+ "write-back"};
	67	+/*
	68	+ * raid5 cache state machine
	69	+ *
	70	+ * With the RAID cache, each stripe works in two phases:
	71	+ * - caching phase
	72	+ * - writing-out phase
	73	+ *
	74	+ * These two phases are controlled by bit STRIPE_R5C_CACHING:
	75	+ * if STRIPE_R5C_CACHING == 0, the stripe is in writing-out phase
	76	+ * if STRIPE_R5C_CACHING == 1, the stripe is in caching phase
	77	+ *
	78	+ * When there is no journal, or the journal is in write-through mode,
	79	+ * the stripe is always in writing-out phase.
	80	+ *
	81	+ * For write-back journal, the stripe is sent to caching phase on write
	82	+ * (r5c_try_caching_write). r5c_make_stripe_write_out() kicks off
	83	+ * the write-out phase by clearing STRIPE_R5C_CACHING.
	84	+ *
	85	+ * Stripes in caching phase do not write the raid disks. Instead, all
	86	+ * writes are committed from the log device. Therefore, a stripe in
	87	+ * caching phase handles writes as:
	88	+ * - write to log device
	89	+ * - return IO
	90	+ *
	91	+ * Stripes in writing-out phase handle writes as:
	92	+ * - calculate parity
	93	+ * - write pending data and parity to journal
	94	+ * - write data and parity to raid disks
	95	+ * - return IO for pending writes
	96	+ */
	97	+
43	98	struct r5l_log {
44	99	struct md_rdev *rdev;
45	100
@@ -58,7 +113,6 @@ struct r5l_log { · 12 unmodified lines
58	113	u64 seq; /* log head sequence */
59	114
60	115	sector_t next_checkpoint;
61		− u64 next_cp_seq;
62	116
63	117	struct mutex io_mutex;
64	118	struct r5l_io_unit current_io; / current io_unit accepting new data */
@@ -96,6 +150,18 @@ struct r5l_log { · 31 unmodified lines
96	150	spinlock_t no_space_stripes_lock;
97	151
98	152	bool need_cache_flush;
	153	+
	154	+ /* for r5c_cache */
	155	+ enum r5c_journal_mode r5c_journal_mode;
	156	+
	157	+ /* all stripes in r5cache, in the order of seq at sh->log_start */
	158	+ struct list_head stripe_in_journal_list;
	159	+
	160	+ spinlock_t stripe_in_journal_lock;
	161	+ atomic_t stripe_in_journal_count;
	162	+
	163	+ /* to submit async io_units, to fulfill ordering of flush */
	164	+ struct work_struct deferred_io_work;
99	165	};
100	166
101	167	/*
@@ -122,6 +188,18 @@ struct r5l_io_unit { · 20 unmodified lines
122	188
123	189	int state;
124	190	bool need_split_bio;
	191	+ struct bio *split_bio;
	192	+
	193	+ unsigned int has_flush:1; /* include flush request */
	194	+ unsigned int has_fua:1; /* include fua request */
	195	+ unsigned int has_null_flush:1; /* include empty flush request */
	196	+ /*
	197	+ * io isn't sent yet, flush/fua request can only be submitted till it's
	198	+ * the first IO in running_ios list
	199	+ */
	200	+ unsigned int io_deferred:1;
	201	+
	202	+ struct bio_list flush_barriers; /* size == 0 flush bios */
125	203	};
126	204
127	205	/* r5l_io_unit state */
@@ -133,6 +211,12 @@ enum r5l_io_unit_state { · 5 unmodified lines
133	211	IO_UNIT_STRIPE_END = 3, /* stripes data finished writing to raid */
134	212	};
135	213
	214	+bool r5c_is_writeback(struct r5l_log *log)
	215	+{
	216	+ return (log != NULL &&
	217	+ log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK);
	218	+}
	219	+
136	220	static sector_t r5l_ring_add(struct r5l_log *log, sector_t start, sector_t inc)
137	221	{
138	222	start += inc;
@@ -168,12 +252,235 @@ static void __r5l_set_io_unit_state(struct r5l_io_unit *io, · 29 unmodified lines
168	252	io->state = state;
169	253	}
170	254
	255	+static void
	256	+r5c_return_dev_pending_writes(struct r5conf conf, struct r5dev dev,
	257	+ struct bio_list *return_bi)
	258	+{
	259	+ struct bio wbi, wbi2;
	260	+
	261	+ wbi = dev->written;
	262	+ dev->written = NULL;
	263	+ while (wbi && wbi->bi_iter.bi_sector <
	264	+ dev->sector + STRIPE_SECTORS) {
	265	+ wbi2 = r5_next_bio(wbi, dev->sector);
	266	+ if (!raid5_dec_bi_active_stripes(wbi)) {
	267	+ md_write_end(conf->mddev);
	268	+ bio_list_add(return_bi, wbi);
	269	+ }
	270	+ wbi = wbi2;
	271	+ }
	272	+}
	273	+
	274	+void r5c_handle_cached_data_endio(struct r5conf *conf,
	275	+ struct stripe_head sh, int disks, struct bio_list return_bi)
	276	+{
	277	+ int i;
	278	+
	279	+ for (i = sh->disks; i--; ) {
	280	+ if (sh->dev[i].written) {
	281	+ set_bit(R5_UPTODATE, &sh->dev[i].flags);
	282	+ r5c_return_dev_pending_writes(conf, &sh->dev[i],
	283	+ return_bi);
	284	+ bitmap_endwrite(conf->mddev->bitmap, sh->sector,
	285	+ STRIPE_SECTORS,
	286	+ !test_bit(STRIPE_DEGRADED, &sh->state),
	287	+ 0);
	288	+ }
	289	+ }
	290	+}
	291	+
	292	+/* Check whether we should flush some stripes to free up stripe cache */
	293	+void r5c_check_stripe_cache_usage(struct r5conf *conf)
	294	+{
	295	+ int total_cached;
	296	+
	297	+ if (!r5c_is_writeback(conf->log))
	298	+ return;
	299	+
	300	+ total_cached = atomic_read(&conf->r5c_cached_partial_stripes) +
	301	+ atomic_read(&conf->r5c_cached_full_stripes);
	302	+
	303	+ /*
	304	+ * The following condition is true for either of the following:
	305	+ * - stripe cache pressure high:
	306	+ * total_cached > 3/4 min_nr_stripes \|\|
	307	+ * empty_inactive_list_nr > 0
	308	+ * - stripe cache pressure moderate:
	309	+ * total_cached > 1/2 min_nr_stripes
	310	+ */
	311	+ if (total_cached > conf->min_nr_stripes * 1 / 2 \|\|
	312	+ atomic_read(&conf->empty_inactive_list_nr) > 0)
	313	+ r5l_wake_reclaim(conf->log, 0);
	314	+}
	315	+
	316	+/*
	317	+ * flush cache when there are R5C_FULL_STRIPE_FLUSH_BATCH or more full
	318	+ * stripes in the cache
	319	+ */
	320	+void r5c_check_cached_full_stripe(struct r5conf *conf)
	321	+{
	322	+ if (!r5c_is_writeback(conf->log))
	323	+ return;
	324	+
	325	+ /*
	326	+ * wake up reclaim for R5C_FULL_STRIPE_FLUSH_BATCH cached stripes
	327	+ * or a full stripe (chunk size / 4k stripes).
	328	+ */
	329	+ if (atomic_read(&conf->r5c_cached_full_stripes) >=
	330	+ min(R5C_FULL_STRIPE_FLUSH_BATCH,
	331	+ conf->chunk_sectors >> STRIPE_SHIFT))
	332	+ r5l_wake_reclaim(conf->log, 0);
	333	+}
	334	+
	335	+/*
	336	+ * Total log space (in sectors) needed to flush all data in cache
	337	+ *
	338	+ * Currently, writing-out phase automatically includes all pending writes
	339	+ * to the same sector. So the reclaim of each stripe takes up to
	340	+ * (conf->raid_disks + 1) pages of log space.
	341	+ *
	342	+ * To totally avoid deadlock due to log space, the code reserves
	343	+ * (conf->raid_disks + 1) pages for each stripe in cache, which is not
	344	+ * necessary in most cases.
	345	+ *
	346	+ * To improve this, we will need writing-out phase to be able to NOT include
	347	+ * pending writes, which will reduce the requirement to
	348	+ * (conf->max_degraded + 1) pages per stripe in cache.
	349	+ */
	350	+static sector_t r5c_log_required_to_flush_cache(struct r5conf *conf)
	351	+{
	352	+ struct r5l_log *log = conf->log;
	353	+
	354	+ if (!r5c_is_writeback(log))
	355	+ return 0;
	356	+
	357	+ return BLOCK_SECTORS * (conf->raid_disks + 1) *
	358	+ atomic_read(&log->stripe_in_journal_count);
	359	+}
	360	+
	361	+/*
	362	+ * evaluate log space usage and update R5C_LOG_TIGHT and R5C_LOG_CRITICAL
	363	+ *
	364	+ * R5C_LOG_TIGHT is set when free space on the log device is less than 3x of
	365	+ * reclaim_required_space. R5C_LOG_CRITICAL is set when free space on the log
	366	+ * device is less than 2x of reclaim_required_space.
	367	+ */
	368	+static inline void r5c_update_log_state(struct r5l_log *log)
	369	+{
	370	+ struct r5conf *conf = log->rdev->mddev->private;
	371	+ sector_t free_space;
	372	+ sector_t reclaim_space;
	373	+ bool wake_reclaim = false;
	374	+
	375	+ if (!r5c_is_writeback(log))
	376	+ return;
	377	+
	378	+ free_space = r5l_ring_distance(log, log->log_start,
	379	+ log->last_checkpoint);
	380	+ reclaim_space = r5c_log_required_to_flush_cache(conf);
	381	+ if (free_space < 2 * reclaim_space)
	382	+ set_bit(R5C_LOG_CRITICAL, &conf->cache_state);
	383	+ else {
	384	+ if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state))
	385	+ wake_reclaim = true;
	386	+ clear_bit(R5C_LOG_CRITICAL, &conf->cache_state);
	387	+ }
	388	+ if (free_space < 3 * reclaim_space)
	389	+ set_bit(R5C_LOG_TIGHT, &conf->cache_state);
	390	+ else
	391	+ clear_bit(R5C_LOG_TIGHT, &conf->cache_state);
	392	+
	393	+ if (wake_reclaim)
	394	+ r5l_wake_reclaim(log, 0);
	395	+}
	396	+
	397	+/*
	398	+ * Put the stripe into writing-out phase by clearing STRIPE_R5C_CACHING.
	399	+ * This function should only be called in write-back mode.
	400	+ */
	401	+void r5c_make_stripe_write_out(struct stripe_head *sh)
	402	+{
	403	+ struct r5conf *conf = sh->raid_conf;
	404	+ struct r5l_log *log = conf->log;
	405	+
	406	+ BUG_ON(!r5c_is_writeback(log));
	407	+
	408	+ WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
	409	+ clear_bit(STRIPE_R5C_CACHING, &sh->state);
	410	+
	411	+ if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
	412	+ atomic_inc(&conf->preread_active_stripes);
	413	+
	414	+ if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) {
	415	+ BUG_ON(atomic_read(&conf->r5c_cached_partial_stripes) == 0);
	416	+ atomic_dec(&conf->r5c_cached_partial_stripes);
	417	+ }
	418	+
	419	+ if (test_and_clear_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) {
	420	+ BUG_ON(atomic_read(&conf->r5c_cached_full_stripes) == 0);
	421	+ atomic_dec(&conf->r5c_cached_full_stripes);
	422	+ }
	423	+}
	424	+
	425	+static void r5c_handle_data_cached(struct stripe_head *sh)
	426	+{
	427	+ int i;
	428	+
	429	+ for (i = sh->disks; i--; )
	430	+ if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
	431	+ set_bit(R5_InJournal, &sh->dev[i].flags);
	432	+ clear_bit(R5_LOCKED, &sh->dev[i].flags);
	433	+ }
	434	+ clear_bit(STRIPE_LOG_TRAPPED, &sh->state);
	435	+}
	436	+
	437	+/*
	438	+ * this journal write must contain full parity,
	439	+ * it may also contain some data pages
	440	+ */
	441	+static void r5c_handle_parity_cached(struct stripe_head *sh)
	442	+{
	443	+ int i;
	444	+
	445	+ for (i = sh->disks; i--; )
	446	+ if (test_bit(R5_InJournal, &sh->dev[i].flags))
	447	+ set_bit(R5_Wantwrite, &sh->dev[i].flags);
	448	+}
	449	+
	450	+/*
	451	+ * Setting proper flags after writing (or flushing) data and/or parity to the
	452	+ * log device. This is called from r5l_log_endio() or r5l_log_flush_endio().
	453	+ */
	454	+static void r5c_finish_cache_stripe(struct stripe_head *sh)
	455	+{
	456	+ struct r5l_log *log = sh->raid_conf->log;
	457	+
	458	+ if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
	459	+ BUG_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
	460	+ /*
	461	+ * Set R5_InJournal for parity dev[pd_idx]. This means
	462	+ * all data AND parity in the journal. For RAID 6, it is
	463	+ * NOT necessary to set the flag for dev[qd_idx], as the
	464	+ * two parities are written out together.
	465	+ */
	466	+ set_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
	467	+ } else if (test_bit(STRIPE_R5C_CACHING, &sh->state)) {
	468	+ r5c_handle_data_cached(sh);
	469	+ } else {
	470	+ r5c_handle_parity_cached(sh);
	471	+ set_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
	472	+ }
	473	+}
	474	+
171	475	static void r5l_io_run_stripes(struct r5l_io_unit *io)
172	476	{
173	477	struct stripe_head sh, next;
174	478
175	479	list_for_each_entry_safe(sh, next, &io->stripe_list, log_list) {
176	480	list_del_init(&sh->log_list);
	481	+
	482	+ r5c_finish_cache_stripe(sh);
	483	+
177	484	set_bit(STRIPE_HANDLE, &sh->state);
178	485	raid5_release_stripe(sh);
179	486	}
@@ -209,9 +516,11 @@ static void r5l_move_to_end_ios(struct r5l_log *log) · 29 unmodified lines
209	516	}
210	517	}
211	518
	519	+static void __r5l_stripe_write_finished(struct r5l_io_unit *io);
212	520	static void r5l_log_endio(struct bio *bio)
213	521	{
214	522	struct r5l_io_unit *io = bio->bi_private;
	523	+ struct r5l_io_unit *io_deferred;
215	524	struct r5l_log *log = io->log;
216	525	unsigned long flags;
217	526
@@ -227,18 +536,89 @@ static void r5l_log_endio(struct bio *bio) · 9 unmodified lines
227	536	r5l_move_to_end_ios(log);
228	537	else
229	538	r5l_log_run_stripes(log);
	539	+ if (!list_empty(&log->running_ios)) {
	540	+ /*
	541	+ * FLUSH/FUA io_unit is deferred because of ordering, now we
	542	+ * can dispatch it
	543	+ */
	544	+ io_deferred = list_first_entry(&log->running_ios,
	545	+ struct r5l_io_unit, log_sibling);
	546	+ if (io_deferred->io_deferred)
	547	+ schedule_work(&log->deferred_io_work);
	548	+ }
	549	+
230	550	spin_unlock_irqrestore(&log->io_list_lock, flags);
231	551
232	552	if (log->need_cache_flush)
233	553	md_wakeup_thread(log->rdev->mddev->thread);
	554	+
	555	+ if (io->has_null_flush) {
	556	+ struct bio *bi;
	557	+
	558	+ WARN_ON(bio_list_empty(&io->flush_barriers));
	559	+ while ((bi = bio_list_pop(&io->flush_barriers)) != NULL) {
	560	+ bio_endio(bi);
	561	+ atomic_dec(&io->pending_stripe);
	562	+ }
	563	+ if (atomic_read(&io->pending_stripe) == 0)
	564	+ __r5l_stripe_write_finished(io);
	565	+ }
	566	+}
	567	+
	568	+static void r5l_do_submit_io(struct r5l_log log, struct r5l_io_unit io)
	569	+{
	570	+ unsigned long flags;
	571	+
	572	+ spin_lock_irqsave(&log->io_list_lock, flags);
	573	+ __r5l_set_io_unit_state(io, IO_UNIT_IO_START);
	574	+ spin_unlock_irqrestore(&log->io_list_lock, flags);
	575	+
	576	+ if (io->has_flush)
	577	+ io->current_bio->bi_opf \|= REQ_PREFLUSH;
	578	+ if (io->has_fua)
	579	+ io->current_bio->bi_opf \|= REQ_FUA;
	580	+ submit_bio(io->current_bio);
	581	+
	582	+ if (!io->split_bio)
	583	+ return;
	584	+
	585	+ if (io->has_flush)
	586	+ io->split_bio->bi_opf \|= REQ_PREFLUSH;
	587	+ if (io->has_fua)
	588	+ io->split_bio->bi_opf \|= REQ_FUA;
	589	+ submit_bio(io->split_bio);
	590	+}
	591	+
	592	+/* deferred io_unit will be dispatched here */
	593	+static void r5l_submit_io_async(struct work_struct *work)
	594	+{
	595	+ struct r5l_log *log = container_of(work, struct r5l_log,
	596	+ deferred_io_work);
	597	+ struct r5l_io_unit *io = NULL;
	598	+ unsigned long flags;
	599	+
	600	+ spin_lock_irqsave(&log->io_list_lock, flags);
	601	+ if (!list_empty(&log->running_ios)) {
	602	+ io = list_first_entry(&log->running_ios, struct r5l_io_unit,
	603	+ log_sibling);
	604	+ if (!io->io_deferred)
	605	+ io = NULL;
	606	+ else
	607	+ io->io_deferred = 0;
	608	+ }
	609	+ spin_unlock_irqrestore(&log->io_list_lock, flags);
	610	+ if (io)
	611	+ r5l_do_submit_io(log, io);
234	612	}
235	613
236	614	static void r5l_submit_current_io(struct r5l_log *log)
237	615	{
238	616	struct r5l_io_unit *io = log->current_io;
	617	+ struct bio *bio;
239	618	struct r5l_meta_block *block;
240	619	unsigned long flags;
241	620	u32 crc;
	621	+ bool do_submit = true;
242	622
243	623	if (!io)
244	624	return;
@@ -247,13 +627,20 @@ static void r5l_submit_current_io(struct r5l_log *log) · 2 unmodified lines
247	627	block->meta_size = cpu_to_le32(io->meta_offset);
248	628	crc = crc32c_le(log->uuid_checksum, block, PAGE_SIZE);
249	629	block->checksum = cpu_to_le32(crc);
	630	+ bio = io->current_bio;
250	631
251	632	log->current_io = NULL;
252	633	spin_lock_irqsave(&log->io_list_lock, flags);
253		− __r5l_set_io_unit_state(io, IO_UNIT_IO_START);
	634	+ if (io->has_flush \|\| io->has_fua) {
	635	+ if (io != list_first_entry(&log->running_ios,
	636	+ struct r5l_io_unit, log_sibling)) {
	637	+ io->io_deferred = 1;
	638	+ do_submit = false;
	639	+ }
	640	+ }
254	641	spin_unlock_irqrestore(&log->io_list_lock, flags);
255		−
256		− submit_bio(io->current_bio);
	642	+ if (do_submit)
	643	+ r5l_do_submit_io(log, io);
257	644	}
258	645
259	646	static struct bio r5l_bio_alloc(struct r5l_log log)
@@ -271,6 +658,7 @@ static void r5_reserve_log_entry(struct r5l_log log, struct r5l_io_unit io) · 11 unmodified lines
271	658	{
272	659	log->log_start = r5l_ring_add(log, log->log_start, BLOCK_SECTORS);
273	660
	661	+ r5c_update_log_state(log);
274	662	/*
275	663	* If we filled up the log device start from the beginning again,
276	664	* which will require a new bio.
@@ -297,6 +685,7 @@ static struct r5l_io_unit r5l_new_meta(struct r5l_log log) · 20 unmodified lines
297	685	io->log = log;
298	686	INIT_LIST_HEAD(&io->log_sibling);
299	687	INIT_LIST_HEAD(&io->stripe_list);
	688	+ bio_list_init(&io->flush_barriers);
300	689	io->state = IO_UNIT_RUNNING;
301	690
302	691	io->meta_page = mempool_alloc(log->meta_pool, GFP_NOIO);
@@ -367,12 +756,11 @@ static void r5l_append_payload_page(struct r5l_log log, struct page page) · 64 unmodified lines
367	756	struct r5l_io_unit *io = log->current_io;
368	757
369	758	if (io->need_split_bio) {
370		− struct bio *prev = io->current_bio;
371		−
	759	+ BUG_ON(io->split_bio);
	760	+ io->split_bio = io->current_bio;
372	761	io->current_bio = r5l_bio_alloc(log);
373		− bio_chain(io->current_bio, prev);
374		−
375		− submit_bio(prev);
	762	+ bio_chain(io->current_bio, io->split_bio);
	763	+ io->need_split_bio = false;
376	764	}
377	765
378	766	if (!bio_add_page(io->current_bio, page, PAGE_SIZE, 0))
@@ -401,50 +789,85 @@ static int r5l_log_stripe(struct r5l_log log, struct stripe_head sh, · 22 unmodified lines
401	789
402	790	io = log->current_io;
403	791
	792	+ if (test_and_clear_bit(STRIPE_R5C_PREFLUSH, &sh->state))
	793	+ io->has_flush = 1;
	794	+
404	795	for (i = 0; i < sh->disks; i++) {
405		− if (!test_bit(R5_Wantwrite, &sh->dev[i].flags))
	796	+ if (!test_bit(R5_Wantwrite, &sh->dev[i].flags) \|\|
	797	+ test_bit(R5_InJournal, &sh->dev[i].flags))
406	798	continue;
407	799	if (i == sh->pd_idx \|\| i == sh->qd_idx)
408	800	continue;
	801	+ if (test_bit(R5_WantFUA, &sh->dev[i].flags) &&
	802	+ log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK) {
	803	+ io->has_fua = 1;
	804	+ /*
	805	+ * we need to flush journal to make sure recovery can
	806	+ * reach the data with fua flag
	807	+ */
	808	+ io->has_flush = 1;
	809	+ }
409	810	r5l_append_payload_meta(log, R5LOG_PAYLOAD_DATA,
410	811	raid5_compute_blocknr(sh, i, 0),
411	812	sh->dev[i].log_checksum, 0, false);
412	813	r5l_append_payload_page(log, sh->dev[i].page);
413	814	}
414	815
415		− if (sh->qd_idx >= 0) {
	816	+ if (parity_pages == 2) {
416	817	r5l_append_payload_meta(log, R5LOG_PAYLOAD_PARITY,
417	818	sh->sector, sh->dev[sh->pd_idx].log_checksum,
418	819	sh->dev[sh->qd_idx].log_checksum, true);
419	820	r5l_append_payload_page(log, sh->dev[sh->pd_idx].page);
420	821	r5l_append_payload_page(log, sh->dev[sh->qd_idx].page);
421		− } else {
	822	+ } else if (parity_pages == 1) {
422	823	r5l_append_payload_meta(log, R5LOG_PAYLOAD_PARITY,
423	824	sh->sector, sh->dev[sh->pd_idx].log_checksum,
424	825	0, false);
425	826	r5l_append_payload_page(log, sh->dev[sh->pd_idx].page);
426		− }
	827	+ } else /* Just writing data, not parity, in caching phase */
	828	+ BUG_ON(parity_pages != 0);
427	829
428	830	list_add_tail(&sh->log_list, &io->stripe_list);
429	831	atomic_inc(&io->pending_stripe);
430	832	sh->log_io = io;
431	833
	834	+ if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
	835	+ return 0;
	836	+
	837	+ if (sh->log_start == MaxSector) {
	838	+ BUG_ON(!list_empty(&sh->r5c));
	839	+ sh->log_start = io->log_start;
	840	+ spin_lock_irq(&log->stripe_in_journal_lock);
	841	+ list_add_tail(&sh->r5c,
	842	+ &log->stripe_in_journal_list);
	843	+ spin_unlock_irq(&log->stripe_in_journal_lock);
	844	+ atomic_inc(&log->stripe_in_journal_count);
	845	+ }
432	846	return 0;
433	847	}
434	848
435		−static void r5l_wake_reclaim(struct r5l_log *log, sector_t space);
	849	+/* add stripe to no_space_stripes, and then wake up reclaim */
	850	+static inline void r5l_add_no_space_stripe(struct r5l_log *log,
	851	+ struct stripe_head *sh)
	852	+{
	853	+ spin_lock(&log->no_space_stripes_lock);
	854	+ list_add_tail(&sh->log_list, &log->no_space_stripes);
	855	+ spin_unlock(&log->no_space_stripes_lock);
	856	+}
	857	+
436	858	/*
437	859	* running in raid5d, where reclaim could wait for raid5d too (when it flushes
438	860	* data from log to raid disks), so we shouldn't wait for reclaim here
439	861	*/
440	862	int r5l_write_stripe(struct r5l_log log, struct stripe_head sh)
441	863	{
	864	+ struct r5conf *conf = sh->raid_conf;
442	865	int write_disks = 0;
443	866	int data_pages, parity_pages;
444		− int meta_size;
445	867	int reserve;
446	868	int i;
447	869	int ret = 0;
	870	+ bool wake_reclaim = false;
448	871
449	872	if (!log)
450	873	return -EAGAIN;
@@ -456,11 +879,15 @@ int r5l_write_stripe(struct r5l_log log, struct stripe_head sh) · 5 unmodified lines
456	879	return -EAGAIN;
457	880	}
458	881
	882	+ WARN_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
	883	+
459	884	for (i = 0; i < sh->disks; i++) {
460	885	void *addr;
461	886
462		− if (!test_bit(R5_Wantwrite, &sh->dev[i].flags))
	887	+ if (!test_bit(R5_Wantwrite, &sh->dev[i].flags) \|\|
	888	+ test_bit(R5_InJournal, &sh->dev[i].flags))
463	889	continue;
	890	+
464	891	write_disks++;
465	892	/* checksum is already calculated in last run */
466	893	if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))
@@ -473,15 +900,6 @@ int r5l_write_stripe(struct r5l_log log, struct stripe_head sh) · 6 unmodified lines
473	900	parity_pages = 1 + !!(sh->qd_idx >= 0);
474	901	data_pages = write_disks - parity_pages;
475	902
476		− meta_size =
477		− ((sizeof(struct r5l_payload_data_parity) + sizeof(__le32))
478		− * data_pages) +
479		− sizeof(struct r5l_payload_data_parity) +
480		− sizeof(__le32) * parity_pages;
481		− /* Doesn't work with very big raid array */
482		− if (meta_size + sizeof(struct r5l_meta_block) > PAGE_SIZE)
483		− return -EINVAL;
484		−
485	903	set_bit(STRIPE_LOG_TRAPPED, &sh->state);
486	904	/*
487	905	* The stripe must enter state machine again to finish the write, so
@@ -493,22 +911,49 @@ int r5l_write_stripe(struct r5l_log log, struct stripe_head sh) · 5 unmodified lines
493	911	mutex_lock(&log->io_mutex);
494	912	/* meta + data */
495	913	reserve = (1 + write_disks) << (PAGE_SHIFT - 9);
496		− if (!r5l_has_free_space(log, reserve)) {
497		− spin_lock(&log->no_space_stripes_lock);
498		− list_add_tail(&sh->log_list, &log->no_space_stripes);
499		− spin_unlock(&log->no_space_stripes_lock);
500	914
501		− r5l_wake_reclaim(log, reserve);
502		− } else {
503		− ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
504		− if (ret) {
505		− spin_lock_irq(&log->io_list_lock);
506		− list_add_tail(&sh->log_list, &log->no_mem_stripes);
507		− spin_unlock_irq(&log->io_list_lock);
	915	+ if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
	916	+ if (!r5l_has_free_space(log, reserve)) {
	917	+ r5l_add_no_space_stripe(log, sh);
	918	+ wake_reclaim = true;
	919	+ } else {
	920	+ ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
	921	+ if (ret) {
	922	+ spin_lock_irq(&log->io_list_lock);
	923	+ list_add_tail(&sh->log_list,
	924	+ &log->no_mem_stripes);
	925	+ spin_unlock_irq(&log->io_list_lock);
	926	+ }
	927	+ }
	928	+ } else { /* R5C_JOURNAL_MODE_WRITE_BACK */
	929	+ /*
	930	+ * log space critical, do not process stripes that are
	931	+ * not in cache yet (sh->log_start == MaxSector).
	932	+ */
	933	+ if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&
	934	+ sh->log_start == MaxSector) {
	935	+ r5l_add_no_space_stripe(log, sh);
	936	+ wake_reclaim = true;
	937	+ reserve = 0;
	938	+ } else if (!r5l_has_free_space(log, reserve)) {
	939	+ if (sh->log_start == log->last_checkpoint)
	940	+ BUG();
	941	+ else
	942	+ r5l_add_no_space_stripe(log, sh);
	943	+ } else {
	944	+ ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
	945	+ if (ret) {
	946	+ spin_lock_irq(&log->io_list_lock);
	947	+ list_add_tail(&sh->log_list,
	948	+ &log->no_mem_stripes);
	949	+ spin_unlock_irq(&log->io_list_lock);
	950	+ }
508	951	}
509	952	}
510	953
511	954	mutex_unlock(&log->io_mutex);
	955	+ if (wake_reclaim)
	956	+ r5l_wake_reclaim(log, reserve);
512	957	return 0;
513	958	}
514	959
@@ -525,17 +970,34 @@ int r5l_handle_flush_request(struct r5l_log log, struct bio bio) · 10 unmodified lines
525	970	{
526	971	if (!log)
527	972	return -ENODEV;
528		− /*
529		− * we flush log disk cache first, then write stripe data to raid disks.
530		− * So if bio is finished, the log disk cache is flushed already. The
531		− * recovery guarantees we can recovery the bio from log disk, so we
532		− * don't need to flush again
533		− */
534		− if (bio->bi_iter.bi_size == 0) {
535		− bio_endio(bio);
536		− return 0;
	973	+
	974	+ if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
	975	+ /*
	976	+ * in write through (journal only)
	977	+ * we flush log disk cache first, then write stripe data to
	978	+ * raid disks. So if bio is finished, the log disk cache is
	979	+ * flushed already. The recovery guarantees we can recovery
	980	+ * the bio from log disk, so we don't need to flush again
	981	+ */
	982	+ if (bio->bi_iter.bi_size == 0) {
	983	+ bio_endio(bio);
	984	+ return 0;
	985	+ }
	986	+ bio->bi_opf &= ~REQ_PREFLUSH;
	987	+ } else {
	988	+ /* write back (with cache) */
	989	+ if (bio->bi_iter.bi_size == 0) {
	990	+ mutex_lock(&log->io_mutex);
	991	+ r5l_get_meta(log, 0);
	992	+ bio_list_add(&log->current_io->flush_barriers, bio);
	993	+ log->current_io->has_flush = 1;
	994	+ log->current_io->has_null_flush = 1;
	995	+ atomic_inc(&log->current_io->pending_stripe);
	996	+ r5l_submit_current_io(log);
	997	+ mutex_unlock(&log->io_mutex);
	998	+ return 0;
	999	+ }
537	1000	}
538		− bio->bi_opf &= ~REQ_PREFLUSH;
539	1001	return -EAGAIN;
540	1002	}
541	1003
@@ -555,10 +1017,40 @@ static void r5l_run_no_space_stripes(struct r5l_log *log) · 13 unmodified lines
555	1017	spin_unlock(&log->no_space_stripes_lock);
556	1018	}
557	1019
	1020	+/*
	1021	+ * calculate new last_checkpoint
	1022	+ * for write through mode, returns log->next_checkpoint
	1023	+ * for write back, returns log_start of first sh in stripe_in_journal_list
	1024	+ */
	1025	+static sector_t r5c_calculate_new_cp(struct r5conf *conf)
	1026	+{
	1027	+ struct stripe_head *sh;
	1028	+ struct r5l_log *log = conf->log;
	1029	+ sector_t new_cp;
	1030	+ unsigned long flags;
	1031	+
	1032	+ if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
	1033	+ return log->next_checkpoint;
	1034	+
	1035	+ spin_lock_irqsave(&log->stripe_in_journal_lock, flags);
	1036	+ if (list_empty(&conf->log->stripe_in_journal_list)) {
	1037	+ /* all stripes flushed */
	1038	+ spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
	1039	+ return log->next_checkpoint;
	1040	+ }
	1041	+ sh = list_first_entry(&conf->log->stripe_in_journal_list,
	1042	+ struct stripe_head, r5c);
	1043	+ new_cp = sh->log_start;
	1044	+ spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
	1045	+ return new_cp;
	1046	+}
	1047	+
558	1048	static sector_t r5l_reclaimable_space(struct r5l_log *log)
559	1049	{
	1050	+ struct r5conf *conf = log->rdev->mddev->private;
	1051	+
560	1052	return r5l_ring_distance(log, log->last_checkpoint,
561		− log->next_checkpoint);
	1053	+ r5c_calculate_new_cp(conf));
562	1054	}
563	1055
564	1056	static void r5l_run_no_mem_stripe(struct r5l_log *log)
@@ -589,7 +1081,6 @@ static bool r5l_complete_finished_ios(struct r5l_log *log) · 24 unmodified lines
589	1081	break;
590	1082
591	1083	log->next_checkpoint = io->log_start;
592		− log->next_cp_seq = io->seq;
593	1084
594	1085	list_del(&io->log_sibling);
595	1086	mempool_free(io, log->io_pool);
@@ -604,6 +1095,7 @@ static bool r5l_complete_finished_ios(struct r5l_log *log) · 8 unmodified lines
604	1095	static void __r5l_stripe_write_finished(struct r5l_io_unit *io)
605	1096	{
606	1097	struct r5l_log *log = io->log;
	1098	+ struct r5conf *conf = log->rdev->mddev->private;
607	1099	unsigned long flags;
608	1100
609	1101	spin_lock_irqsave(&log->io_list_lock, flags);
@@ -614,7 +1106,8 @@ static void __r5l_stripe_write_finished(struct r5l_io_unit *io) · 4 unmodified lines
614	1106	return;
615	1107	}
616	1108
617		− if (r5l_reclaimable_space(log) > log->max_free_space)
	1109	+ if (r5l_reclaimable_space(log) > log->max_free_space \|\|
	1110	+ test_bit(R5C_LOG_TIGHT, &conf->cache_state))
618	1111	r5l_wake_reclaim(log, 0);
619	1112
620	1113	spin_unlock_irqrestore(&log->io_list_lock, flags);
@@ -713,8 +1206,8 @@ static void r5l_write_super_and_discard_space(struct r5l_log *log, · 92 unmodified lines
713	1206	* there is a deadlock. We workaround this issue with a trylock.
714	1207	* FIXME: we could miss discard if we can't take reconfig mutex
715	1208	*/
716		− set_mask_bits(&mddev->flags, 0,
717		− BIT(MD_CHANGE_DEVS) \| BIT(MD_CHANGE_PENDING));
	1209	+ set_mask_bits(&mddev->sb_flags, 0,
	1210	+ BIT(MD_SB_CHANGE_DEVS) \| BIT(MD_SB_CHANGE_PENDING));
718	1211	if (!mddev_trylock(mddev))
719	1212	return;
720	1213	md_update_sb(mddev, 1);
@@ -735,45 +1228,178 @@ static void r5l_write_super_and_discard_space(struct r5l_log *log, · 14 unmodified lines
735	1228	}
736	1229	}
737	1230
738		−
739		−static void r5l_do_reclaim(struct r5l_log *log)
	1231	+/*
	1232	+ * r5c_flush_stripe moves stripe from cached list to handle_list. When called,
	1233	+ * the stripe must be on r5c_cached_full_stripes or r5c_cached_partial_stripes.
	1234	+ *
	1235	+ * must hold conf->device_lock
	1236	+ */
	1237	+static void r5c_flush_stripe(struct r5conf conf, struct stripe_head sh)
740	1238	{
741		− sector_t reclaim_target = xchg(&log->reclaim_target, 0);
742		− sector_t reclaimable;
743		− sector_t next_checkpoint;
744		− u64 next_cp_seq;
	1239	+ BUG_ON(list_empty(&sh->lru));
	1240	+ BUG_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
	1241	+ BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
745	1242
746		− spin_lock_irq(&log->io_list_lock);
747	1243	/*
748		− * move proper io_unit to reclaim list. We should not change the order.
749		− * reclaimable/unreclaimable io_unit can be mixed in the list, we
750		− * shouldn't reuse space of an unreclaimable io_unit
	1244	+ * The stripe is not ON_RELEASE_LIST, so it is safe to call
	1245	+ * raid5_release_stripe() while holding conf->device_lock
751	1246	*/
752		− while (1) {
753		− reclaimable = r5l_reclaimable_space(log);
754		− if (reclaimable >= reclaim_target \|\|
755		− (list_empty(&log->running_ios) &&
756		− list_empty(&log->io_end_ios) &&
757		− list_empty(&log->flushing_ios) &&
758		− list_empty(&log->finished_ios)))
759		− break;
	1247	+ BUG_ON(test_bit(STRIPE_ON_RELEASE_LIST, &sh->state));
	1248	+ assert_spin_locked(&conf->device_lock);
760	1249
761		− md_wakeup_thread(log->rdev->mddev->thread);
762		− wait_event_lock_irq(log->iounit_wait,
763		− r5l_reclaimable_space(log) > reclaimable,
764		− log->io_list_lock);
765		− }
	1250	+ list_del_init(&sh->lru);
	1251	+ atomic_inc(&sh->count);
766	1252
767		− next_checkpoint = log->next_checkpoint;
768		− next_cp_seq = log->next_cp_seq;
769		− spin_unlock_irq(&log->io_list_lock);
	1253	+ set_bit(STRIPE_HANDLE, &sh->state);
	1254	+ atomic_inc(&conf->active_stripes);
	1255	+ r5c_make_stripe_write_out(sh);
770	1256
771		− BUG_ON(reclaimable < 0);
772		− if (reclaimable == 0)
	1257	+ raid5_release_stripe(sh);
	1258	+}
	1259	+
	1260	+/*
	1261	+ * if num == 0, flush all full stripes
	1262	+ * if num > 0, flush all full stripes. If less than num full stripes are
	1263	+ * flushed, flush some partial stripes until totally num stripes are
	1264	+ * flushed or there is no more cached stripes.
	1265	+ */
	1266	+void r5c_flush_cache(struct r5conf *conf, int num)
	1267	+{
	1268	+ int count;
	1269	+ struct stripe_head sh, next;
	1270	+
	1271	+ assert_spin_locked(&conf->device_lock);
	1272	+ if (!conf->log)
773	1273	return;
774	1274
775		− /*
776		− * write_super will flush cache of each raid disk. We must write super
	1275	+ count = 0;
	1276	+ list_for_each_entry_safe(sh, next, &conf->r5c_full_stripe_list, lru) {
	1277	+ r5c_flush_stripe(conf, sh);
	1278	+ count++;
	1279	+ }
	1280	+
	1281	+ if (count >= num)
	1282	+ return;
	1283	+ list_for_each_entry_safe(sh, next,
	1284	+ &conf->r5c_partial_stripe_list, lru) {
	1285	+ r5c_flush_stripe(conf, sh);
	1286	+ if (++count >= num)
	1287	+ break;
	1288	+ }
	1289	+}
	1290	+
	1291	+static void r5c_do_reclaim(struct r5conf *conf)
	1292	+{
	1293	+ struct r5l_log *log = conf->log;
	1294	+ struct stripe_head *sh;
	1295	+ int count = 0;
	1296	+ unsigned long flags;
	1297	+ int total_cached;
	1298	+ int stripes_to_flush;
	1299	+
	1300	+ if (!r5c_is_writeback(log))
	1301	+ return;
	1302	+
	1303	+ total_cached = atomic_read(&conf->r5c_cached_partial_stripes) +
	1304	+ atomic_read(&conf->r5c_cached_full_stripes);
	1305	+
	1306	+ if (total_cached > conf->min_nr_stripes * 3 / 4 \|\|
	1307	+ atomic_read(&conf->empty_inactive_list_nr) > 0)
	1308	+ /*
	1309	+ * if stripe cache pressure high, flush all full stripes and
	1310	+ * some partial stripes
	1311	+ */
	1312	+ stripes_to_flush = R5C_RECLAIM_STRIPE_GROUP;
	1313	+ else if (total_cached > conf->min_nr_stripes * 1 / 2 \|\|
	1314	+ atomic_read(&conf->r5c_cached_full_stripes) >
	1315	+ R5C_FULL_STRIPE_FLUSH_BATCH)
	1316	+ /*
	1317	+ * if stripe cache pressure moderate, or if there is many full
	1318	+ * stripes,flush all full stripes
	1319	+ */
	1320	+ stripes_to_flush = 0;
	1321	+ else
	1322	+ /* no need to flush */
	1323	+ stripes_to_flush = -1;
	1324	+
	1325	+ if (stripes_to_flush >= 0) {
	1326	+ spin_lock_irqsave(&conf->device_lock, flags);
	1327	+ r5c_flush_cache(conf, stripes_to_flush);
	1328	+ spin_unlock_irqrestore(&conf->device_lock, flags);
	1329	+ }
	1330	+
	1331	+ /* if log space is tight, flush stripes on stripe_in_journal_list */
	1332	+ if (test_bit(R5C_LOG_TIGHT, &conf->cache_state)) {
	1333	+ spin_lock_irqsave(&log->stripe_in_journal_lock, flags);
	1334	+ spin_lock(&conf->device_lock);
	1335	+ list_for_each_entry(sh, &log->stripe_in_journal_list, r5c) {
	1336	+ /*
	1337	+ * stripes on stripe_in_journal_list could be in any
	1338	+ * state of the stripe_cache state machine. In this
	1339	+ * case, we only want to flush stripe on
	1340	+ * r5c_cached_full/partial_stripes. The following
	1341	+ * condition makes sure the stripe is on one of the
	1342	+ * two lists.
	1343	+ */
	1344	+ if (!list_empty(&sh->lru) &&
	1345	+ !test_bit(STRIPE_HANDLE, &sh->state) &&
	1346	+ atomic_read(&sh->count) == 0) {
	1347	+ r5c_flush_stripe(conf, sh);
	1348	+ }
	1349	+ if (count++ >= R5C_RECLAIM_STRIPE_GROUP)
	1350	+ break;
	1351	+ }
	1352	+ spin_unlock(&conf->device_lock);
	1353	+ spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
	1354	+ }
	1355	+
	1356	+ if (!test_bit(R5C_LOG_CRITICAL, &conf->cache_state))
	1357	+ r5l_run_no_space_stripes(log);
	1358	+
	1359	+ md_wakeup_thread(conf->mddev->thread);
	1360	+}
	1361	+
	1362	+static void r5l_do_reclaim(struct r5l_log *log)
	1363	+{
	1364	+ struct r5conf *conf = log->rdev->mddev->private;
	1365	+ sector_t reclaim_target = xchg(&log->reclaim_target, 0);
	1366	+ sector_t reclaimable;
	1367	+ sector_t next_checkpoint;
	1368	+ bool write_super;
	1369	+
	1370	+ spin_lock_irq(&log->io_list_lock);
	1371	+ write_super = r5l_reclaimable_space(log) > log->max_free_space \|\|
	1372	+ reclaim_target != 0 \|\| !list_empty(&log->no_space_stripes);
	1373	+ /*
	1374	+ * move proper io_unit to reclaim list. We should not change the order.
	1375	+ * reclaimable/unreclaimable io_unit can be mixed in the list, we
	1376	+ * shouldn't reuse space of an unreclaimable io_unit
	1377	+ */
	1378	+ while (1) {
	1379	+ reclaimable = r5l_reclaimable_space(log);
	1380	+ if (reclaimable >= reclaim_target \|\|
	1381	+ (list_empty(&log->running_ios) &&
	1382	+ list_empty(&log->io_end_ios) &&
	1383	+ list_empty(&log->flushing_ios) &&
	1384	+ list_empty(&log->finished_ios)))
	1385	+ break;
	1386	+
	1387	+ md_wakeup_thread(log->rdev->mddev->thread);
	1388	+ wait_event_lock_irq(log->iounit_wait,
	1389	+ r5l_reclaimable_space(log) > reclaimable,
	1390	+ log->io_list_lock);
	1391	+ }
	1392	+
	1393	+ next_checkpoint = r5c_calculate_new_cp(conf);
	1394	+ spin_unlock_irq(&log->io_list_lock);
	1395	+
	1396	+ BUG_ON(reclaimable < 0);
	1397	+
	1398	+ if (reclaimable == 0 \|\| !write_super)
	1399	+ return;
	1400	+
	1401	+ /*
	1402	+ * write_super will flush cache of each raid disk. We must write super
777	1403	* here, because the log area might be reused soon and we don't want to
778	1404	* confuse recovery
779	1405	*/
@@ -781,7 +1407,7 @@ static void r5l_do_reclaim(struct r5l_log *log) · 1 unmodified line
781	1407
782	1408	mutex_lock(&log->io_mutex);
783	1409	log->last_checkpoint = next_checkpoint;
784		− log->last_cp_seq = next_cp_seq;
	1410	+ r5c_update_log_state(log);
785	1411	mutex_unlock(&log->io_mutex);
786	1412
787	1413	r5l_run_no_space_stripes(log);
@@ -795,14 +1421,17 @@ static void r5l_reclaim_thread(struct md_thread *thread) · 7 unmodified lines
795	1421
796	1422	if (!log)
797	1423	return;
	1424	+ r5c_do_reclaim(conf);
798	1425	r5l_do_reclaim(log);
799	1426	}
800	1427
801		−static void r5l_wake_reclaim(struct r5l_log *log, sector_t space)
	1428	+void r5l_wake_reclaim(struct r5l_log *log, sector_t space)
802	1429	{
803	1430	unsigned long target;
804	1431	unsigned long new = (unsigned long)space; /* overflow in theory */
805	1432
	1433	+ if (!log)
	1434	+ return;
806	1435	do {
807	1436	target = log->reclaim_target;
808	1437	if (new < target)
@@ -816,22 +1445,14 @@ void r5l_quiesce(struct r5l_log *log, int state) · 7 unmodified lines
816	1445	struct mddev *mddev;
817	1446	if (!log \|\| state == 2)
818	1447	return;
819		− if (state == 0) {
820		− /*
821		− * This is a special case for hotadd. In suspend, the array has
822		− * no journal. In resume, journal is initialized as well as the
823		− * reclaim thread.
824		− */
825		− if (log->reclaim_thread)
826		− return;
827		− log->reclaim_thread = md_register_thread(r5l_reclaim_thread,
828		− log->rdev->mddev, "reclaim");
829		− } else if (state == 1) {
	1448	+ if (state == 0)
	1449	+ kthread_unpark(log->reclaim_thread->tsk);
	1450	+ else if (state == 1) {
830	1451	/* make sure r5l_write_super_and_discard_space exits */
831	1452	mddev = log->rdev->mddev;
832	1453	wake_up(&mddev->sb_wait);
833		− r5l_wake_reclaim(log, -1L);
834		− md_unregister_thread(&log->reclaim_thread);
	1454	+ kthread_park(log->reclaim_thread->tsk);
	1455	+ r5l_wake_reclaim(log, MaxSector);
835	1456	r5l_do_reclaim(log);
836	1457	}
837	1458	}
@@ -857,10 +1478,13 @@ struct r5l_recovery_ctx { · 19 unmodified lines
857	1478	sector_t meta_total_blocks; /* total size of current meta and data */
858	1479	sector_t pos; /* recovery position */
859	1480	u64 seq; /* recovery position seq */
	1481	+ int data_parity_stripes; /* number of data_parity stripes */
	1482	+ int data_only_stripes; /* number of data_only stripes */
	1483	+ struct list_head cached_list;
860	1484	};
861	1485
862		−static int r5l_read_meta_block(struct r5l_log *log,
863		− struct r5l_recovery_ctx *ctx)
	1486	+static int r5l_recovery_read_meta_block(struct r5l_log *log,
	1487	+ struct r5l_recovery_ctx *ctx)
864	1488	{
865	1489	struct page *page = ctx->meta_page;
866	1490	struct r5l_meta_block *mb;
@@ -892,170 +1516,618 @@ static int r5l_read_meta_block(struct r5l_log *log, · 25 unmodified lines
892	1516	return 0;
893	1517	}
894	1518
895		−static int r5l_recovery_flush_one_stripe(struct r5l_log *log,
896		− struct r5l_recovery_ctx *ctx,
897		− sector_t stripe_sect,
898		− int offset, sector_t log_offset)
	1519	+static void
	1520	+r5l_recovery_create_empty_meta_block(struct r5l_log *log,
	1521	+ struct page *page,
	1522	+ sector_t pos, u64 seq)
899	1523	{
900		− struct r5conf *conf = log->rdev->mddev->private;
901		− struct stripe_head *sh;
902		− struct r5l_payload_data_parity *payload;
903		− int disk_index;
	1524	+ struct r5l_meta_block *mb;
904	1525
905		− sh = raid5_get_active_stripe(conf, stripe_sect, 0, 0, 0);
906		− while (1) {
907		− payload = page_address(ctx->meta_page) + *offset;
	1526	+ mb = page_address(page);
	1527	+ clear_page(mb);
	1528	+ mb->magic = cpu_to_le32(R5LOG_MAGIC);
	1529	+ mb->version = R5LOG_VERSION;
	1530	+ mb->meta_size = cpu_to_le32(sizeof(struct r5l_meta_block));
	1531	+ mb->seq = cpu_to_le64(seq);
	1532	+ mb->position = cpu_to_le64(pos);
	1533	+}
908	1534
909		− if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) {
910		− raid5_compute_sector(conf,
911		− le64_to_cpu(payload->location), 0,
912		− &disk_index, sh);
	1535	+static int r5l_log_write_empty_meta_block(struct r5l_log *log, sector_t pos,
	1536	+ u64 seq)
	1537	+{
	1538	+ struct page *page;
	1539	+ struct r5l_meta_block *mb;
913	1540
914		− sync_page_io(log->rdev, *log_offset, PAGE_SIZE,
915		− sh->dev[disk_index].page, REQ_OP_READ, 0,
916		− false);
917		− sh->dev[disk_index].log_checksum =
918		− le32_to_cpu(payload->checksum[0]);
919		− set_bit(R5_Wantwrite, &sh->dev[disk_index].flags);
920		− ctx->meta_total_blocks += BLOCK_SECTORS;
921		− } else {
922		− disk_index = sh->pd_idx;
923		− sync_page_io(log->rdev, *log_offset, PAGE_SIZE,
924		− sh->dev[disk_index].page, REQ_OP_READ, 0,
925		− false);
926		− sh->dev[disk_index].log_checksum =
927		− le32_to_cpu(payload->checksum[0]);
928		− set_bit(R5_Wantwrite, &sh->dev[disk_index].flags);
929		−
930		− if (sh->qd_idx >= 0) {
931		− disk_index = sh->qd_idx;
932		− sync_page_io(log->rdev,
933		− r5l_ring_add(log, *log_offset, BLOCK_SECTORS),
934		− PAGE_SIZE, sh->dev[disk_index].page,
935		− REQ_OP_READ, 0, false);
936		− sh->dev[disk_index].log_checksum =
937		− le32_to_cpu(payload->checksum[1]);
938		− set_bit(R5_Wantwrite,
939		− &sh->dev[disk_index].flags);
940		− }
941		− ctx->meta_total_blocks += BLOCK_SECTORS * conf->max_degraded;
942		− }
	1541	+ page = alloc_page(GFP_KERNEL);
	1542	+ if (!page)
	1543	+ return -ENOMEM;
	1544	+ r5l_recovery_create_empty_meta_block(log, page, pos, seq);
	1545	+ mb = page_address(page);
	1546	+ mb->checksum = cpu_to_le32(crc32c_le(log->uuid_checksum,
	1547	+ mb, PAGE_SIZE));
	1548	+ if (!sync_page_io(log->rdev, pos, PAGE_SIZE, page, REQ_OP_WRITE,
	1549	+ REQ_FUA, false)) {
	1550	+ __free_page(page);
	1551	+ return -EIO;
	1552	+ }
	1553	+ __free_page(page);
	1554	+ return 0;
	1555	+}
943	1556
944		− log_offset = r5l_ring_add(log, log_offset,
945		− le32_to_cpu(payload->size));
946		− *offset += sizeof(struct r5l_payload_data_parity) +
947		− sizeof(__le32) *
948		− (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
949		− if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_PARITY)
950		− break;
	1557	+/*
	1558	+ * r5l_recovery_load_data and r5l_recovery_load_parity uses flag R5_Wantwrite
	1559	+ * to mark valid (potentially not flushed) data in the journal.
	1560	+ *
	1561	+ * We already verified checksum in r5l_recovery_verify_data_checksum_for_mb,
	1562	+ * so there should not be any mismatch here.
	1563	+ */
	1564	+static void r5l_recovery_load_data(struct r5l_log *log,
	1565	+ struct stripe_head *sh,
	1566	+ struct r5l_recovery_ctx *ctx,
	1567	+ struct r5l_payload_data_parity *payload,
	1568	+ sector_t log_offset)
	1569	+{
	1570	+ struct mddev *mddev = log->rdev->mddev;
	1571	+ struct r5conf *conf = mddev->private;
	1572	+ int dd_idx;
	1573	+
	1574	+ raid5_compute_sector(conf,
	1575	+ le64_to_cpu(payload->location), 0,
	1576	+ &dd_idx, sh);
	1577	+ sync_page_io(log->rdev, log_offset, PAGE_SIZE,
	1578	+ sh->dev[dd_idx].page, REQ_OP_READ, 0, false);
	1579	+ sh->dev[dd_idx].log_checksum =
	1580	+ le32_to_cpu(payload->checksum[0]);
	1581	+ ctx->meta_total_blocks += BLOCK_SECTORS;
	1582	+
	1583	+ set_bit(R5_Wantwrite, &sh->dev[dd_idx].flags);
	1584	+ set_bit(STRIPE_R5C_CACHING, &sh->state);
	1585	+}
	1586	+
	1587	+static void r5l_recovery_load_parity(struct r5l_log *log,
	1588	+ struct stripe_head *sh,
	1589	+ struct r5l_recovery_ctx *ctx,
	1590	+ struct r5l_payload_data_parity *payload,
	1591	+ sector_t log_offset)
	1592	+{
	1593	+ struct mddev *mddev = log->rdev->mddev;
	1594	+ struct r5conf *conf = mddev->private;
	1595	+
	1596	+ ctx->meta_total_blocks += BLOCK_SECTORS * conf->max_degraded;
	1597	+ sync_page_io(log->rdev, log_offset, PAGE_SIZE,
	1598	+ sh->dev[sh->pd_idx].page, REQ_OP_READ, 0, false);
	1599	+ sh->dev[sh->pd_idx].log_checksum =
	1600	+ le32_to_cpu(payload->checksum[0]);
	1601	+ set_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags);
	1602	+
	1603	+ if (sh->qd_idx >= 0) {
	1604	+ sync_page_io(log->rdev,
	1605	+ r5l_ring_add(log, log_offset, BLOCK_SECTORS),
	1606	+ PAGE_SIZE, sh->dev[sh->qd_idx].page,
	1607	+ REQ_OP_READ, 0, false);
	1608	+ sh->dev[sh->qd_idx].log_checksum =
	1609	+ le32_to_cpu(payload->checksum[1]);
	1610	+ set_bit(R5_Wantwrite, &sh->dev[sh->qd_idx].flags);
951	1611	}
	1612	+ clear_bit(STRIPE_R5C_CACHING, &sh->state);
	1613	+}
952	1614
953		− for (disk_index = 0; disk_index < sh->disks; disk_index++) {
954		− void *addr;
955		− u32 checksum;
	1615	+static void r5l_recovery_reset_stripe(struct stripe_head *sh)
	1616	+{
	1617	+ int i;
	1618	+
	1619	+ sh->state = 0;
	1620	+ sh->log_start = MaxSector;
	1621	+ for (i = sh->disks; i--; )
	1622	+ sh->dev[i].flags = 0;
	1623	+}
	1624	+
	1625	+static void
	1626	+r5l_recovery_replay_one_stripe(struct r5conf *conf,
	1627	+ struct stripe_head *sh,
	1628	+ struct r5l_recovery_ctx *ctx)
	1629	+{
	1630	+ struct md_rdev rdev, rrdev;
	1631	+ int disk_index;
	1632	+ int data_count = 0;
956	1633
	1634	+ for (disk_index = 0; disk_index < sh->disks; disk_index++) {
957	1635	if (!test_bit(R5_Wantwrite, &sh->dev[disk_index].flags))
958	1636	continue;
959		− addr = kmap_atomic(sh->dev[disk_index].page);
960		− checksum = crc32c_le(log->uuid_checksum, addr, PAGE_SIZE);
961		− kunmap_atomic(addr);
962		− if (checksum != sh->dev[disk_index].log_checksum)
963		− goto error;
	1637	+ if (disk_index == sh->qd_idx \|\| disk_index == sh->pd_idx)
	1638	+ continue;
	1639	+ data_count++;
964	1640	}
965	1641
966		− for (disk_index = 0; disk_index < sh->disks; disk_index++) {
967		− struct md_rdev rdev, rrdev;
	1642	+ /*
	1643	+ * stripes that only have parity must have been flushed
	1644	+ * before the crash that we are now recovering from, so
	1645	+ * there is nothing more to recovery.
	1646	+ */
	1647	+ if (data_count == 0)
	1648	+ goto out;
968	1649
969		− if (!test_and_clear_bit(R5_Wantwrite,
970		− &sh->dev[disk_index].flags))
	1650	+ for (disk_index = 0; disk_index < sh->disks; disk_index++) {
	1651	+ if (!test_bit(R5_Wantwrite, &sh->dev[disk_index].flags))
971	1652	continue;
972	1653
973	1654	/* in case device is broken */
	1655	+ rcu_read_lock();
974	1656	rdev = rcu_dereference(conf->disks[disk_index].rdev);
975		− if (rdev)
976		− sync_page_io(rdev, stripe_sect, PAGE_SIZE,
	1657	+ if (rdev) {
	1658	+ atomic_inc(&rdev->nr_pending);
	1659	+ rcu_read_unlock();
	1660	+ sync_page_io(rdev, sh->sector, PAGE_SIZE,
977	1661	sh->dev[disk_index].page, REQ_OP_WRITE, 0,
978	1662	false);
	1663	+ rdev_dec_pending(rdev, rdev->mddev);
	1664	+ rcu_read_lock();
	1665	+ }
979	1666	rrdev = rcu_dereference(conf->disks[disk_index].replacement);
980		− if (rrdev)
981		− sync_page_io(rrdev, stripe_sect, PAGE_SIZE,
	1667	+ if (rrdev) {
	1668	+ atomic_inc(&rrdev->nr_pending);
	1669	+ rcu_read_unlock();
	1670	+ sync_page_io(rrdev, sh->sector, PAGE_SIZE,
982	1671	sh->dev[disk_index].page, REQ_OP_WRITE, 0,
983	1672	false);
	1673	+ rdev_dec_pending(rrdev, rrdev->mddev);
	1674	+ rcu_read_lock();
	1675	+ }
	1676	+ rcu_read_unlock();
984	1677	}
985		− raid5_release_stripe(sh);
	1678	+ ctx->data_parity_stripes++;
	1679	+out:
	1680	+ r5l_recovery_reset_stripe(sh);
	1681	+}
	1682	+
	1683	+static struct stripe_head *
	1684	+r5c_recovery_alloc_stripe(struct r5conf *conf,
	1685	+ sector_t stripe_sect,
	1686	+ sector_t log_start)
	1687	+{
	1688	+ struct stripe_head *sh;
	1689	+
	1690	+ sh = raid5_get_active_stripe(conf, stripe_sect, 0, 1, 0);
	1691	+ if (!sh)
	1692	+ return NULL; /* no more stripe available */
	1693	+
	1694	+ r5l_recovery_reset_stripe(sh);
	1695	+ sh->log_start = log_start;
	1696	+
	1697	+ return sh;
	1698	+}
	1699	+
	1700	+static struct stripe_head *
	1701	+r5c_recovery_lookup_stripe(struct list_head *list, sector_t sect)
	1702	+{
	1703	+ struct stripe_head *sh;
	1704	+
	1705	+ list_for_each_entry(sh, list, lru)
	1706	+ if (sh->sector == sect)
	1707	+ return sh;
	1708	+ return NULL;
	1709	+}
	1710	+
	1711	+static void
	1712	+r5c_recovery_drop_stripes(struct list_head *cached_stripe_list,
	1713	+ struct r5l_recovery_ctx *ctx)
	1714	+{
	1715	+ struct stripe_head sh, next;
	1716	+
	1717	+ list_for_each_entry_safe(sh, next, cached_stripe_list, lru) {
	1718	+ r5l_recovery_reset_stripe(sh);
	1719	+ list_del_init(&sh->lru);
	1720	+ raid5_release_stripe(sh);
	1721	+ }
	1722	+}
	1723	+
	1724	+static void
	1725	+r5c_recovery_replay_stripes(struct list_head *cached_stripe_list,
	1726	+ struct r5l_recovery_ctx *ctx)
	1727	+{
	1728	+ struct stripe_head sh, next;
	1729	+
	1730	+ list_for_each_entry_safe(sh, next, cached_stripe_list, lru)
	1731	+ if (!test_bit(STRIPE_R5C_CACHING, &sh->state)) {
	1732	+ r5l_recovery_replay_one_stripe(sh->raid_conf, sh, ctx);
	1733	+ list_del_init(&sh->lru);
	1734	+ raid5_release_stripe(sh);
	1735	+ }
	1736	+}
	1737	+
	1738	+/* if matches return 0; otherwise return -EINVAL */
	1739	+static int
	1740	+r5l_recovery_verify_data_checksum(struct r5l_log log, struct page page,
	1741	+ sector_t log_offset, __le32 log_checksum)
	1742	+{
	1743	+ void *addr;
	1744	+ u32 checksum;
	1745	+
	1746	+ sync_page_io(log->rdev, log_offset, PAGE_SIZE,
	1747	+ page, REQ_OP_READ, 0, false);
	1748	+ addr = kmap_atomic(page);
	1749	+ checksum = crc32c_le(log->uuid_checksum, addr, PAGE_SIZE);
	1750	+ kunmap_atomic(addr);
	1751	+ return (le32_to_cpu(log_checksum) == checksum) ? 0 : -EINVAL;
	1752	+}
	1753	+
	1754	+/*
	1755	+ * before loading data to stripe cache, we need verify checksum for all data,
	1756	+ * if there is mismatch for any data page, we drop all data in the mata block
	1757	+ */
	1758	+static int
	1759	+r5l_recovery_verify_data_checksum_for_mb(struct r5l_log *log,
	1760	+ struct r5l_recovery_ctx *ctx)
	1761	+{
	1762	+ struct mddev *mddev = log->rdev->mddev;
	1763	+ struct r5conf *conf = mddev->private;
	1764	+ struct r5l_meta_block *mb = page_address(ctx->meta_page);
	1765	+ sector_t mb_offset = sizeof(struct r5l_meta_block);
	1766	+ sector_t log_offset = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
	1767	+ struct page *page;
	1768	+ struct r5l_payload_data_parity *payload;
	1769	+
	1770	+ page = alloc_page(GFP_KERNEL);
	1771	+ if (!page)
	1772	+ return -ENOMEM;
	1773	+
	1774	+ while (mb_offset < le32_to_cpu(mb->meta_size)) {
	1775	+ payload = (void *)mb + mb_offset;
	1776	+
	1777	+ if (payload->header.type == R5LOG_PAYLOAD_DATA) {
	1778	+ if (r5l_recovery_verify_data_checksum(
	1779	+ log, page, log_offset,
	1780	+ payload->checksum[0]) < 0)
	1781	+ goto mismatch;
	1782	+ } else if (payload->header.type == R5LOG_PAYLOAD_PARITY) {
	1783	+ if (r5l_recovery_verify_data_checksum(
	1784	+ log, page, log_offset,
	1785	+ payload->checksum[0]) < 0)
	1786	+ goto mismatch;
	1787	+ if (conf->max_degraded == 2 && /* q for RAID 6 */
	1788	+ r5l_recovery_verify_data_checksum(
	1789	+ log, page,
	1790	+ r5l_ring_add(log, log_offset,
	1791	+ BLOCK_SECTORS),
	1792	+ payload->checksum[1]) < 0)
	1793	+ goto mismatch;
	1794	+ } else /* not R5LOG_PAYLOAD_DATA or R5LOG_PAYLOAD_PARITY */
	1795	+ goto mismatch;
	1796	+
	1797	+ log_offset = r5l_ring_add(log, log_offset,
	1798	+ le32_to_cpu(payload->size));
	1799	+
	1800	+ mb_offset += sizeof(struct r5l_payload_data_parity) +
	1801	+ sizeof(__le32) *
	1802	+ (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
	1803	+ }
	1804	+
	1805	+ put_page(page);
986	1806	return 0;
987	1807
988		−error:
989		− for (disk_index = 0; disk_index < sh->disks; disk_index++)
990		− sh->dev[disk_index].flags = 0;
991		− raid5_release_stripe(sh);
	1808	+mismatch:
	1809	+ put_page(page);
992	1810	return -EINVAL;
993	1811	}
994	1812
995		−static int r5l_recovery_flush_one_meta(struct r5l_log *log,
996		− struct r5l_recovery_ctx *ctx)
	1813	+/*
	1814	+ * Analyze all data/parity pages in one meta block
	1815	+ * Returns:
	1816	+ * 0 for success
	1817	+ * -EINVAL for unknown playload type
	1818	+ * -EAGAIN for checksum mismatch of data page
	1819	+ * -ENOMEM for run out of memory (alloc_page failed or run out of stripes)
	1820	+ */
	1821	+static int
	1822	+r5c_recovery_analyze_meta_block(struct r5l_log *log,
	1823	+ struct r5l_recovery_ctx *ctx,
	1824	+ struct list_head *cached_stripe_list)
997	1825	{
998		− struct r5conf *conf = log->rdev->mddev->private;
999		− struct r5l_payload_data_parity *payload;
	1826	+ struct mddev *mddev = log->rdev->mddev;
	1827	+ struct r5conf *conf = mddev->private;
1000	1828	struct r5l_meta_block *mb;
1001		− int offset;
	1829	+ struct r5l_payload_data_parity *payload;
	1830	+ int mb_offset;
1002	1831	sector_t log_offset;
1003		− sector_t stripe_sector;
	1832	+ sector_t stripe_sect;
	1833	+ struct stripe_head *sh;
	1834	+ int ret;
	1835	+
	1836	+ /*
	1837	+ * for mismatch in data blocks, we will drop all data in this mb, but
	1838	+ * we will still read next mb for other data with FLUSH flag, as
	1839	+ * io_unit could finish out of order.
	1840	+ */
	1841	+ ret = r5l_recovery_verify_data_checksum_for_mb(log, ctx);
	1842	+ if (ret == -EINVAL)
	1843	+ return -EAGAIN;
	1844	+ else if (ret)
	1845	+ return ret; /* -ENOMEM duo to alloc_page() failed */
1004	1846
1005	1847	mb = page_address(ctx->meta_page);
1006		− offset = sizeof(struct r5l_meta_block);
	1848	+ mb_offset = sizeof(struct r5l_meta_block);
1007	1849	log_offset = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
1008	1850
1009		− while (offset < le32_to_cpu(mb->meta_size)) {
	1851	+ while (mb_offset < le32_to_cpu(mb->meta_size)) {
1010	1852	int dd;
1011	1853
1012		− payload = (void *)mb + offset;
1013		− stripe_sector = raid5_compute_sector(conf,
1014		− le64_to_cpu(payload->location), 0, &dd, NULL);
1015		− if (r5l_recovery_flush_one_stripe(log, ctx, stripe_sector,
1016		− &offset, &log_offset))
	1854	+ payload = (void *)mb + mb_offset;
	1855	+ stripe_sect = (payload->header.type == R5LOG_PAYLOAD_DATA) ?
	1856	+ raid5_compute_sector(
	1857	+ conf, le64_to_cpu(payload->location), 0, &dd,
	1858	+ NULL)
	1859	+ : le64_to_cpu(payload->location);
	1860	+
	1861	+ sh = r5c_recovery_lookup_stripe(cached_stripe_list,
	1862	+ stripe_sect);
	1863	+
	1864	+ if (!sh) {
	1865	+ sh = r5c_recovery_alloc_stripe(conf, stripe_sect, ctx->pos);
	1866	+ /*
	1867	+ * cannot get stripe from raid5_get_active_stripe
	1868	+ * try replay some stripes
	1869	+ */
	1870	+ if (!sh) {
	1871	+ r5c_recovery_replay_stripes(
	1872	+ cached_stripe_list, ctx);
	1873	+ sh = r5c_recovery_alloc_stripe(
	1874	+ conf, stripe_sect, ctx->pos);
	1875	+ }
	1876	+ if (!sh) {
	1877	+ pr_debug("md/raid:%s: Increasing stripe cache size to %d to recovery data on journal.\n",
	1878	+ mdname(mddev),
	1879	+ conf->min_nr_stripes * 2);
	1880	+ raid5_set_cache_size(mddev,
	1881	+ conf->min_nr_stripes * 2);
	1882	+ sh = r5c_recovery_alloc_stripe(
	1883	+ conf, stripe_sect, ctx->pos);
	1884	+ }
	1885	+ if (!sh) {
	1886	+ pr_err("md/raid:%s: Cannot get enough stripes due to memory pressure. Recovery failed.\n",
	1887	+ mdname(mddev));
	1888	+ return -ENOMEM;
	1889	+ }
	1890	+ list_add_tail(&sh->lru, cached_stripe_list);
	1891	+ }
	1892	+
	1893	+ if (payload->header.type == R5LOG_PAYLOAD_DATA) {
	1894	+ if (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&
	1895	+ test_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags)) {
	1896	+ r5l_recovery_replay_one_stripe(conf, sh, ctx);
	1897	+ sh->log_start = ctx->pos;
	1898	+ list_move_tail(&sh->lru, cached_stripe_list);
	1899	+ }
	1900	+ r5l_recovery_load_data(log, sh, ctx, payload,
	1901	+ log_offset);
	1902	+ } else if (payload->header.type == R5LOG_PAYLOAD_PARITY)
	1903	+ r5l_recovery_load_parity(log, sh, ctx, payload,
	1904	+ log_offset);
	1905	+ else
1017	1906	return -EINVAL;
	1907	+
	1908	+ log_offset = r5l_ring_add(log, log_offset,
	1909	+ le32_to_cpu(payload->size));
	1910	+
	1911	+ mb_offset += sizeof(struct r5l_payload_data_parity) +
	1912	+ sizeof(__le32) *
	1913	+ (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
1018	1914	}
	1915	+
1019	1916	return 0;
1020	1917	}
1021	1918
1022		−/* copy data/parity from log to raid disks */
1023		−static void r5l_recovery_flush_log(struct r5l_log *log,
1024		− struct r5l_recovery_ctx *ctx)
	1919	+/*
	1920	+ * Load the stripe into cache. The stripe will be written out later by
	1921	+ * the stripe cache state machine.
	1922	+ */
	1923	+static void r5c_recovery_load_one_stripe(struct r5l_log *log,
	1924	+ struct stripe_head *sh)
1025	1925	{
	1926	+ struct r5dev *dev;
	1927	+ int i;
	1928	+
	1929	+ for (i = sh->disks; i--; ) {
	1930	+ dev = sh->dev + i;
	1931	+ if (test_and_clear_bit(R5_Wantwrite, &dev->flags)) {
	1932	+ set_bit(R5_InJournal, &dev->flags);
	1933	+ set_bit(R5_UPTODATE, &dev->flags);
	1934	+ }
	1935	+ }
	1936	+ list_add_tail(&sh->r5c, &log->stripe_in_journal_list);
	1937	+ atomic_inc(&log->stripe_in_journal_count);
	1938	+}
	1939	+
	1940	+/*
	1941	+ * Scan through the log for all to-be-flushed data
	1942	+ *
	1943	+ * For stripes with data and parity, namely Data-Parity stripe
	1944	+ * (STRIPE_R5C_CACHING == 0), we simply replay all the writes.
	1945	+ *
	1946	+ * For stripes with only data, namely Data-Only stripe
	1947	+ * (STRIPE_R5C_CACHING == 1), we load them to stripe cache state machine.
	1948	+ *
	1949	+ * For a stripe, if we see data after parity, we should discard all previous
	1950	+ * data and parity for this stripe, as these data are already flushed to
	1951	+ * the array.
	1952	+ *
	1953	+ * At the end of the scan, we return the new journal_tail, which points to
	1954	+ * first data-only stripe on the journal device, or next invalid meta block.
	1955	+ */
	1956	+static int r5c_recovery_flush_log(struct r5l_log *log,
	1957	+ struct r5l_recovery_ctx *ctx)
	1958	+{
	1959	+ struct stripe_head *sh;
	1960	+ int ret = 0;
	1961	+
	1962	+ /* scan through the log */
1026	1963	while (1) {
1027		− if (r5l_read_meta_block(log, ctx))
1028		− return;
1029		− if (r5l_recovery_flush_one_meta(log, ctx))
1030		− return;
	1964	+ if (r5l_recovery_read_meta_block(log, ctx))
	1965	+ break;
	1966	+
	1967	+ ret = r5c_recovery_analyze_meta_block(log, ctx,
	1968	+ &ctx->cached_list);
	1969	+ /*
	1970	+ * -EAGAIN means mismatch in data block, in this case, we still
	1971	+ * try scan the next metablock
	1972	+ */
	1973	+ if (ret && ret != -EAGAIN)
	1974	+ break; /* ret == -EINVAL or -ENOMEM */
1031	1975	ctx->seq++;
1032	1976	ctx->pos = r5l_ring_add(log, ctx->pos, ctx->meta_total_blocks);
1033	1977	}
	1978	+
	1979	+ if (ret == -ENOMEM) {
	1980	+ r5c_recovery_drop_stripes(&ctx->cached_list, ctx);
	1981	+ return ret;
	1982	+ }
	1983	+
	1984	+ /* replay data-parity stripes */
	1985	+ r5c_recovery_replay_stripes(&ctx->cached_list, ctx);
	1986	+
	1987	+ /* load data-only stripes to stripe cache */
	1988	+ list_for_each_entry(sh, &ctx->cached_list, lru) {
	1989	+ WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
	1990	+ r5c_recovery_load_one_stripe(log, sh);
	1991	+ ctx->data_only_stripes++;
	1992	+ }
	1993	+
	1994	+ return 0;
1034	1995	}
1035	1996
1036		−static int r5l_log_write_empty_meta_block(struct r5l_log *log, sector_t pos,
1037		− u64 seq)
	1997	+/*
	1998	+ * we did a recovery. Now ctx.pos points to an invalid meta block. New
	1999	+ * log will start here. but we can't let superblock point to last valid
	2000	+ * meta block. The log might looks like:
	2001	+ * \| meta 1\| meta 2\| meta 3\|
	2002	+ * meta 1 is valid, meta 2 is invalid. meta 3 could be valid. If
	2003	+ * superblock points to meta 1, we write a new valid meta 2n. if crash
	2004	+ * happens again, new recovery will start from meta 1. Since meta 2n is
	2005	+ * valid now, recovery will think meta 3 is valid, which is wrong.
	2006	+ * The solution is we create a new meta in meta2 with its seq == meta
	2007	+ * 1's seq + 10000 and let superblock points to meta2. The same recovery
	2008	+ * will not think meta 3 is a valid meta, because its seq doesn't match
	2009	+ */
	2010	+
	2011	+/*
	2012	+ * Before recovery, the log looks like the following
	2013	+ *
	2014	+ * ---------------------------------------------
	2015	+ * \| valid log \| invalid log \|
	2016	+ * ---------------------------------------------
	2017	+ * ^
	2018	+ * \|- log->last_checkpoint
	2019	+ * \|- log->last_cp_seq
	2020	+ *
	2021	+ * Now we scan through the log until we see invalid entry
	2022	+ *
	2023	+ * ---------------------------------------------
	2024	+ * \| valid log \| invalid log \|
	2025	+ * ---------------------------------------------
	2026	+ * ^ ^
	2027	+ * \|- log->last_checkpoint \|- ctx->pos
	2028	+ * \|- log->last_cp_seq \|- ctx->seq
	2029	+ *
	2030	+ * From this point, we need to increase seq number by 10 to avoid
	2031	+ * confusing next recovery.
	2032	+ *
	2033	+ * ---------------------------------------------
	2034	+ * \| valid log \| invalid log \|
	2035	+ * ---------------------------------------------
	2036	+ * ^ ^
	2037	+ * \|- log->last_checkpoint \|- ctx->pos+1
	2038	+ * \|- log->last_cp_seq \|- ctx->seq+10001
	2039	+ *
	2040	+ * However, it is not safe to start the state machine yet, because data only
	2041	+ * parities are not yet secured in RAID. To save these data only parities, we
	2042	+ * rewrite them from seq+11.
	2043	+ *
	2044	+ * -----------------------------------------------------------------
	2045	+ * \| valid log \| data only stripes \| invalid log \|
	2046	+ * -----------------------------------------------------------------
	2047	+ * ^ ^
	2048	+ * \|- log->last_checkpoint \|- ctx->pos+n
	2049	+ * \|- log->last_cp_seq \|- ctx->seq+10000+n
	2050	+ *
	2051	+ * If failure happens again during this process, the recovery can safe start
	2052	+ * again from log->last_checkpoint.
	2053	+ *
	2054	+ * Once data only stripes are rewritten to journal, we move log_tail
	2055	+ *
	2056	+ * -----------------------------------------------------------------
	2057	+ * \| old log \| data only stripes \| invalid log \|
	2058	+ * -----------------------------------------------------------------
	2059	+ * ^ ^
	2060	+ * \|- log->last_checkpoint \|- ctx->pos+n
	2061	+ * \|- log->last_cp_seq \|- ctx->seq+10000+n
	2062	+ *
	2063	+ * Then we can safely start the state machine. If failure happens from this
	2064	+ * point on, the recovery will start from new log->last_checkpoint.
	2065	+ */
	2066	+static int
	2067	+r5c_recovery_rewrite_data_only_stripes(struct r5l_log *log,
	2068	+ struct r5l_recovery_ctx *ctx)
1038	2069	{
	2070	+ struct stripe_head sh, next;
	2071	+ struct mddev *mddev = log->rdev->mddev;
1039	2072	struct page *page;
1040		− struct r5l_meta_block *mb;
1041		− u32 crc;
1042	2073
1043		− page = alloc_page(GFP_KERNEL \| __GFP_ZERO);
1044		− if (!page)
	2074	+ page = alloc_page(GFP_KERNEL);
	2075	+ if (!page) {
	2076	+ pr_err("md/raid:%s: cannot allocate memory to rewrite data only stripes\n",
	2077	+ mdname(mddev));
1045	2078	return -ENOMEM;
1046		− mb = page_address(page);
1047		− mb->magic = cpu_to_le32(R5LOG_MAGIC);
1048		− mb->version = R5LOG_VERSION;
1049		− mb->meta_size = cpu_to_le32(sizeof(struct r5l_meta_block));
1050		− mb->seq = cpu_to_le64(seq);
1051		− mb->position = cpu_to_le64(pos);
1052		− crc = crc32c_le(log->uuid_checksum, mb, PAGE_SIZE);
1053		− mb->checksum = cpu_to_le32(crc);
	2079	+ }
1054	2080
1055		− if (!sync_page_io(log->rdev, pos, PAGE_SIZE, page, REQ_OP_WRITE,
1056		− REQ_FUA, false)) {
1057		− __free_page(page);
1058		− return -EIO;
	2081	+ list_for_each_entry_safe(sh, next, &ctx->cached_list, lru) {
	2082	+ struct r5l_meta_block *mb;
	2083	+ int i;
	2084	+ int offset;
	2085	+ sector_t write_pos;
	2086	+
	2087	+ WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
	2088	+ r5l_recovery_create_empty_meta_block(log, page,
	2089	+ ctx->pos, ctx->seq);
	2090	+ mb = page_address(page);
	2091	+ offset = le32_to_cpu(mb->meta_size);
	2092	+ write_pos = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
	2093	+
	2094	+ for (i = sh->disks; i--; ) {
	2095	+ struct r5dev *dev = &sh->dev[i];
	2096	+ struct r5l_payload_data_parity *payload;
	2097	+ void *addr;
	2098	+
	2099	+ if (test_bit(R5_InJournal, &dev->flags)) {
	2100	+ payload = (void *)mb + offset;
	2101	+ payload->header.type = cpu_to_le16(
	2102	+ R5LOG_PAYLOAD_DATA);
	2103	+ payload->size = BLOCK_SECTORS;
	2104	+ payload->location = cpu_to_le64(
	2105	+ raid5_compute_blocknr(sh, i, 0));
	2106	+ addr = kmap_atomic(dev->page);
	2107	+ payload->checksum[0] = cpu_to_le32(
	2108	+ crc32c_le(log->uuid_checksum, addr,
	2109	+ PAGE_SIZE));
	2110	+ kunmap_atomic(addr);
	2111	+ sync_page_io(log->rdev, write_pos, PAGE_SIZE,
	2112	+ dev->page, REQ_OP_WRITE, 0, false);
	2113	+ write_pos = r5l_ring_add(log, write_pos,
	2114	+ BLOCK_SECTORS);
	2115	+ offset += sizeof(__le32) +
	2116	+ sizeof(struct r5l_payload_data_parity);
	2117	+
	2118	+ }
	2119	+ }
	2120	+ mb->meta_size = cpu_to_le32(offset);
	2121	+ mb->checksum = cpu_to_le32(crc32c_le(log->uuid_checksum,
	2122	+ mb, PAGE_SIZE));
	2123	+ sync_page_io(log->rdev, ctx->pos, PAGE_SIZE, page,
	2124	+ REQ_OP_WRITE, REQ_FUA, false);
	2125	+ sh->log_start = ctx->pos;
	2126	+ ctx->pos = write_pos;
	2127	+ ctx->seq += 1;
	2128	+
	2129	+ list_del_init(&sh->lru);
	2130	+ raid5_release_stripe(sh);
1059	2131	}
1060	2132	__free_page(page);
1061	2133	return 0;
@@ -1063,45 +2135,60 @@ static int r5l_log_write_empty_meta_block(struct r5l_log *log, sector_t pos, · 1 unmodified line
1063	2135
1064	2136	static int r5l_recovery_log(struct r5l_log *log)
1065	2137	{
	2138	+ struct mddev *mddev = log->rdev->mddev;
1066	2139	struct r5l_recovery_ctx ctx;
	2140	+ int ret;
	2141	+ sector_t pos;
	2142	+ struct stripe_head *sh;
1067	2143
1068	2144	ctx.pos = log->last_checkpoint;
1069	2145	ctx.seq = log->last_cp_seq;
1070	2146	ctx.meta_page = alloc_page(GFP_KERNEL);
	2147	+ ctx.data_only_stripes = 0;
	2148	+ ctx.data_parity_stripes = 0;
	2149	+ INIT_LIST_HEAD(&ctx.cached_list);
	2150	+
1071	2151	if (!ctx.meta_page)
1072	2152	return -ENOMEM;
1073	2153
1074		− r5l_recovery_flush_log(log, &ctx);
	2154	+ ret = r5c_recovery_flush_log(log, &ctx);
1075	2155	__free_page(ctx.meta_page);
1076	2156
1077		− /*
1078		− * we did a recovery. Now ctx.pos points to an invalid meta block. New
1079		− * log will start here. but we can't let superblock point to last valid
1080		− * meta block. The log might looks like:
1081		− * \| meta 1\| meta 2\| meta 3\|
1082		− * meta 1 is valid, meta 2 is invalid. meta 3 could be valid. If
1083		− * superblock points to meta 1, we write a new valid meta 2n. if crash
1084		− * happens again, new recovery will start from meta 1. Since meta 2n is
1085		− * valid now, recovery will think meta 3 is valid, which is wrong.
1086		− * The solution is we create a new meta in meta2 with its seq == meta
1087		− * 1's seq + 10 and let superblock points to meta2. The same recovery will
1088		− * not think meta 3 is a valid meta, because its seq doesn't match
1089		− */
1090		− if (ctx.seq > log->last_cp_seq) {
1091		− int ret;
1092		−
1093		− ret = r5l_log_write_empty_meta_block(log, ctx.pos, ctx.seq + 10);
1094		− if (ret)
1095		− return ret;
1096		− log->seq = ctx.seq + 11;
1097		− log->log_start = r5l_ring_add(log, ctx.pos, BLOCK_SECTORS);
1098		− r5l_write_super(log, ctx.pos);
1099		− log->last_checkpoint = ctx.pos;
	2157	+ if (ret)
	2158	+ return ret;
	2159	+
	2160	+ pos = ctx.pos;
	2161	+ ctx.seq += 10000;
	2162	+
	2163	+ if (ctx.data_only_stripes == 0) {
1100	2164	log->next_checkpoint = ctx.pos;
	2165	+ r5l_log_write_empty_meta_block(log, ctx.pos, ctx.seq++);
	2166	+ ctx.pos = r5l_ring_add(log, ctx.pos, BLOCK_SECTORS);
1101	2167	} else {
1102		− log->log_start = ctx.pos;
1103		− log->seq = ctx.seq;
	2168	+ sh = list_last_entry(&ctx.cached_list, struct stripe_head, lru);
	2169	+ log->next_checkpoint = sh->log_start;
1104	2170	}
	2171	+
	2172	+ if ((ctx.data_only_stripes == 0) && (ctx.data_parity_stripes == 0))
	2173	+ pr_debug("md/raid:%s: starting from clean shutdown\n",
	2174	+ mdname(mddev));
	2175	+ else {
	2176	+ pr_debug("md/raid:%s: recoverying %d data-only stripes and %d data-parity stripes\n",
	2177	+ mdname(mddev), ctx.data_only_stripes,
	2178	+ ctx.data_parity_stripes);
	2179	+
	2180	+ if (ctx.data_only_stripes > 0)
	2181	+ if (r5c_recovery_rewrite_data_only_stripes(log, &ctx)) {
	2182	+ pr_err("md/raid:%s: failed to rewrite stripes to journal\n",
	2183	+ mdname(mddev));
	2184	+ return -EIO;
	2185	+ }
	2186	+ }
	2187	+
	2188	+ log->log_start = ctx.pos;
	2189	+ log->seq = ctx.seq;
	2190	+ log->last_checkpoint = pos;
	2191	+ r5l_write_super(log, pos);
1105	2192	return 0;
1106	2193	}
1107	2194
@@ -1110,7 +2197,293 @@ static void r5l_write_super(struct r5l_log *log, sector_t cp) · 2 unmodified lines
1110	2197	struct mddev *mddev = log->rdev->mddev;
1111	2198
1112	2199	log->rdev->journal_tail = cp;
1113		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
	2200	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
	2201	+}
	2202	+
	2203	+static ssize_t r5c_journal_mode_show(struct mddev mddev, char page)
	2204	+{
	2205	+ struct r5conf *conf = mddev->private;
	2206	+ int ret;
	2207	+
	2208	+ if (!conf->log)
	2209	+ return 0;
	2210	+
	2211	+ switch (conf->log->r5c_journal_mode) {
	2212	+ case R5C_JOURNAL_MODE_WRITE_THROUGH:
	2213	+ ret = snprintf(
	2214	+ page, PAGE_SIZE, "[%s] %s\n",
	2215	+ r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_THROUGH],
	2216	+ r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_BACK]);
	2217	+ break;
	2218	+ case R5C_JOURNAL_MODE_WRITE_BACK:
	2219	+ ret = snprintf(
	2220	+ page, PAGE_SIZE, "%s [%s]\n",
	2221	+ r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_THROUGH],
	2222	+ r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_BACK]);
	2223	+ break;
	2224	+ default:
	2225	+ ret = 0;
	2226	+ }
	2227	+ return ret;
	2228	+}
	2229	+
	2230	+static ssize_t r5c_journal_mode_store(struct mddev *mddev,
	2231	+ const char *page, size_t length)
	2232	+{
	2233	+ struct r5conf *conf = mddev->private;
	2234	+ struct r5l_log *log = conf->log;
	2235	+ int val = -1, i;
	2236	+ int len = length;
	2237	+
	2238	+ if (!log)
	2239	+ return -ENODEV;
	2240	+
	2241	+ if (len && page[len - 1] == '\n')
	2242	+ len -= 1;
	2243	+ for (i = 0; i < ARRAY_SIZE(r5c_journal_mode_str); i++)
	2244	+ if (strlen(r5c_journal_mode_str[i]) == len &&
	2245	+ strncmp(page, r5c_journal_mode_str[i], len) == 0) {
	2246	+ val = i;
	2247	+ break;
	2248	+ }
	2249	+ if (val < R5C_JOURNAL_MODE_WRITE_THROUGH \|\|
	2250	+ val > R5C_JOURNAL_MODE_WRITE_BACK)
	2251	+ return -EINVAL;
	2252	+
	2253	+ mddev_suspend(mddev);
	2254	+ conf->log->r5c_journal_mode = val;
	2255	+ mddev_resume(mddev);
	2256	+
	2257	+ pr_debug("md/raid:%s: setting r5c cache mode to %d: %s\n",
	2258	+ mdname(mddev), val, r5c_journal_mode_str[val]);
	2259	+ return length;
	2260	+}
	2261	+
	2262	+struct md_sysfs_entry
	2263	+r5c_journal_mode = __ATTR(journal_mode, 0644,
	2264	+ r5c_journal_mode_show, r5c_journal_mode_store);
	2265	+
	2266	+/*
	2267	+ * Try handle write operation in caching phase. This function should only
	2268	+ * be called in write-back mode.
	2269	+ *
	2270	+ * If all outstanding writes can be handled in caching phase, returns 0
	2271	+ * If writes requires write-out phase, call r5c_make_stripe_write_out()
	2272	+ * and returns -EAGAIN
	2273	+ */
	2274	+int r5c_try_caching_write(struct r5conf *conf,
	2275	+ struct stripe_head *sh,
	2276	+ struct stripe_head_state *s,
	2277	+ int disks)
	2278	+{
	2279	+ struct r5l_log *log = conf->log;
	2280	+ int i;
	2281	+ struct r5dev *dev;
	2282	+ int to_cache = 0;
	2283	+
	2284	+ BUG_ON(!r5c_is_writeback(log));
	2285	+
	2286	+ if (!test_bit(STRIPE_R5C_CACHING, &sh->state)) {
	2287	+ /*
	2288	+ * There are two different scenarios here:
	2289	+ * 1. The stripe has some data cached, and it is sent to
	2290	+ * write-out phase for reclaim
	2291	+ * 2. The stripe is clean, and this is the first write
	2292	+ *
	2293	+ * For 1, return -EAGAIN, so we continue with
	2294	+ * handle_stripe_dirtying().
	2295	+ *
	2296	+ * For 2, set STRIPE_R5C_CACHING and continue with caching
	2297	+ * write.
	2298	+ */
	2299	+
	2300	+ /* case 1: anything injournal or anything in written */
	2301	+ if (s->injournal > 0 \|\| s->written > 0)
	2302	+ return -EAGAIN;
	2303	+ /* case 2 */
	2304	+ set_bit(STRIPE_R5C_CACHING, &sh->state);
	2305	+ }
	2306	+
	2307	+ for (i = disks; i--; ) {
	2308	+ dev = &sh->dev[i];
	2309	+ /* if non-overwrite, use writing-out phase */
	2310	+ if (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags) &&
	2311	+ !test_bit(R5_InJournal, &dev->flags)) {
	2312	+ r5c_make_stripe_write_out(sh);
	2313	+ return -EAGAIN;
	2314	+ }
	2315	+ }
	2316	+
	2317	+ for (i = disks; i--; ) {
	2318	+ dev = &sh->dev[i];
	2319	+ if (dev->towrite) {
	2320	+ set_bit(R5_Wantwrite, &dev->flags);
	2321	+ set_bit(R5_Wantdrain, &dev->flags);
	2322	+ set_bit(R5_LOCKED, &dev->flags);
	2323	+ to_cache++;
	2324	+ }
	2325	+ }
	2326	+
	2327	+ if (to_cache) {
	2328	+ set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
	2329	+ /*
	2330	+ * set STRIPE_LOG_TRAPPED, which triggers r5c_cache_data()
	2331	+ * in ops_run_io(). STRIPE_LOG_TRAPPED will be cleared in
	2332	+ * r5c_handle_data_cached()
	2333	+ */
	2334	+ set_bit(STRIPE_LOG_TRAPPED, &sh->state);
	2335	+ }
	2336	+
	2337	+ return 0;
	2338	+}
	2339	+
	2340	+/*
	2341	+ * free extra pages (orig_page) we allocated for prexor
	2342	+ */
	2343	+void r5c_release_extra_page(struct stripe_head *sh)
	2344	+{
	2345	+ struct r5conf *conf = sh->raid_conf;
	2346	+ int i;
	2347	+ bool using_disk_info_extra_page;
	2348	+
	2349	+ using_disk_info_extra_page =
	2350	+ sh->dev[0].orig_page == conf->disks[0].extra_page;
	2351	+
	2352	+ for (i = sh->disks; i--; )
	2353	+ if (sh->dev[i].page != sh->dev[i].orig_page) {
	2354	+ struct page *p = sh->dev[i].orig_page;
	2355	+
	2356	+ sh->dev[i].orig_page = sh->dev[i].page;
	2357	+ if (!using_disk_info_extra_page)
	2358	+ put_page(p);
	2359	+ }
	2360	+
	2361	+ if (using_disk_info_extra_page) {
	2362	+ clear_bit(R5C_EXTRA_PAGE_IN_USE, &conf->cache_state);
	2363	+ md_wakeup_thread(conf->mddev->thread);
	2364	+ }
	2365	+}
	2366	+
	2367	+void r5c_use_extra_page(struct stripe_head *sh)
	2368	+{
	2369	+ struct r5conf *conf = sh->raid_conf;
	2370	+ int i;
	2371	+ struct r5dev *dev;
	2372	+
	2373	+ for (i = sh->disks; i--; ) {
	2374	+ dev = &sh->dev[i];
	2375	+ if (dev->orig_page != dev->page)
	2376	+ put_page(dev->orig_page);
	2377	+ dev->orig_page = conf->disks[i].extra_page;
	2378	+ }
	2379	+}
	2380	+
	2381	+/*
	2382	+ * clean up the stripe (clear R5_InJournal for dev[pd_idx] etc.) after the
	2383	+ * stripe is committed to RAID disks.
	2384	+ */
	2385	+void r5c_finish_stripe_write_out(struct r5conf *conf,
	2386	+ struct stripe_head *sh,
	2387	+ struct stripe_head_state *s)
	2388	+{
	2389	+ int i;
	2390	+ int do_wakeup = 0;
	2391	+
	2392	+ if (!conf->log \|\|
	2393	+ !test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags))
	2394	+ return;
	2395	+
	2396	+ WARN_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
	2397	+ clear_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
	2398	+
	2399	+ if (conf->log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
	2400	+ return;
	2401	+
	2402	+ for (i = sh->disks; i--; ) {
	2403	+ clear_bit(R5_InJournal, &sh->dev[i].flags);
	2404	+ if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
	2405	+ do_wakeup = 1;
	2406	+ }
	2407	+
	2408	+ /*
	2409	+ * analyse_stripe() runs before r5c_finish_stripe_write_out(),
	2410	+ * We updated R5_InJournal, so we also update s->injournal.
	2411	+ */
	2412	+ s->injournal = 0;
	2413	+
	2414	+ if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
	2415	+ if (atomic_dec_and_test(&conf->pending_full_writes))
	2416	+ md_wakeup_thread(conf->mddev->thread);
	2417	+
	2418	+ if (do_wakeup)
	2419	+ wake_up(&conf->wait_for_overlap);
	2420	+
	2421	+ if (conf->log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
	2422	+ return;
	2423	+
	2424	+ spin_lock_irq(&conf->log->stripe_in_journal_lock);
	2425	+ list_del_init(&sh->r5c);
	2426	+ spin_unlock_irq(&conf->log->stripe_in_journal_lock);
	2427	+ sh->log_start = MaxSector;
	2428	+ atomic_dec(&conf->log->stripe_in_journal_count);
	2429	+ r5c_update_log_state(conf->log);
	2430	+}
	2431	+
	2432	+int
	2433	+r5c_cache_data(struct r5l_log log, struct stripe_head sh,
	2434	+ struct stripe_head_state *s)
	2435	+{
	2436	+ struct r5conf *conf = sh->raid_conf;
	2437	+ int pages = 0;
	2438	+ int reserve;
	2439	+ int i;
	2440	+ int ret = 0;
	2441	+
	2442	+ BUG_ON(!log);
	2443	+
	2444	+ for (i = 0; i < sh->disks; i++) {
	2445	+ void *addr;
	2446	+
	2447	+ if (!test_bit(R5_Wantwrite, &sh->dev[i].flags))
	2448	+ continue;
	2449	+ addr = kmap_atomic(sh->dev[i].page);
	2450	+ sh->dev[i].log_checksum = crc32c_le(log->uuid_checksum,
	2451	+ addr, PAGE_SIZE);
	2452	+ kunmap_atomic(addr);
	2453	+ pages++;
	2454	+ }
	2455	+ WARN_ON(pages == 0);
	2456	+
	2457	+ /*
	2458	+ * The stripe must enter state machine again to call endio, so
	2459	+ * don't delay.
	2460	+ */
	2461	+ clear_bit(STRIPE_DELAYED, &sh->state);
	2462	+ atomic_inc(&sh->count);
	2463	+
	2464	+ mutex_lock(&log->io_mutex);
	2465	+ /* meta + data */
	2466	+ reserve = (1 + pages) << (PAGE_SHIFT - 9);
	2467	+
	2468	+ if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&
	2469	+ sh->log_start == MaxSector)
	2470	+ r5l_add_no_space_stripe(log, sh);
	2471	+ else if (!r5l_has_free_space(log, reserve)) {
	2472	+ if (sh->log_start == log->last_checkpoint)
	2473	+ BUG();
	2474	+ else
	2475	+ r5l_add_no_space_stripe(log, sh);
	2476	+ } else {
	2477	+ ret = r5l_log_stripe(log, sh, pages, 0);
	2478	+ if (ret) {
	2479	+ spin_lock_irq(&log->io_list_lock);
	2480	+ list_add_tail(&sh->log_list, &log->no_mem_stripes);
	2481	+ spin_unlock_irq(&log->io_list_lock);
	2482	+ }
	2483	+ }
	2484	+
	2485	+ mutex_unlock(&log->io_mutex);
	2486	+ return 0;
1114	2487	}
1115	2488
1116	2489	static int r5l_load_log(struct r5l_log *log)
@@ -1121,7 +2494,7 @@ static int r5l_load_log(struct r5l_log *log) · 4 unmodified lines
1121	2494	sector_t cp = log->rdev->journal_tail;
1122	2495	u32 stored_crc, expected_crc;
1123	2496	bool create_super = false;
1124		− int ret;
	2497	+ int ret = 0;
1125	2498
1126	2499	/* Make sure it's valid */
1127	2500	if (cp >= rdev->sectors \|\| round_down(cp, BLOCK_SECTORS) != cp)
@@ -1171,11 +2544,18 @@ create: · 43 unmodified lines
1171	2544	if (log->max_free_space > RECLAIM_MAX_FREE_SPACE)
1172	2545	log->max_free_space = RECLAIM_MAX_FREE_SPACE;
1173	2546	log->last_checkpoint = cp;
1174		− log->next_checkpoint = cp;
1175	2547
1176	2548	__free_page(page);
1177	2549
1178		− return r5l_recovery_log(log);
	2550	+ if (create_super) {
	2551	+ log->log_start = r5l_ring_add(log, cp, BLOCK_SECTORS);
	2552	+ log->seq = log->last_cp_seq + 1;
	2553	+ log->next_checkpoint = cp;
	2554	+ } else
	2555	+ ret = r5l_recovery_log(log);
	2556	+
	2557	+ r5c_update_log_state(log);
	2558	+ return ret;
1179	2559	ioerr:
1180	2560	__free_page(page);
1181	2561	return ret;
@@ -1188,6 +2568,22 @@ int r5l_init_log(struct r5conf conf, struct md_rdev rdev) · 6 unmodified lines
1188	2568
1189	2569	if (PAGE_SIZE != 4096)
1190	2570	return -EINVAL;
	2571	+
	2572	+ /*
	2573	+ * The PAGE_SIZE must be big enough to hold 1 r5l_meta_block and
	2574	+ * raid_disks r5l_payload_data_parity.
	2575	+ *
	2576	+ * Write journal and cache does not work for very big array
	2577	+ * (raid_disks > 203)
	2578	+ */
	2579	+ if (sizeof(struct r5l_meta_block) +
	2580	+ ((sizeof(struct r5l_payload_data_parity) + sizeof(__le32)) *
	2581	+ conf->raid_disks) > PAGE_SIZE) {
	2582	+ pr_err("md/raid:%s: write journal/cache doesn't work for array with %d disks\n",
	2583	+ mdname(conf->mddev), conf->raid_disks);
	2584	+ return -EINVAL;
	2585	+ }
	2586	+
1191	2587	log = kzalloc(sizeof(*log), GFP_KERNEL);
1192	2588	if (!log)
1193	2589	return -ENOMEM;
@@ -1227,6 +2623,8 @@ int r5l_init_log(struct r5conf conf, struct md_rdev rdev) · 33 unmodified lines
1227	2623	log->rdev->mddev, "reclaim");
1228	2624	if (!log->reclaim_thread)
1229	2625	goto reclaim_thread;
	2626	+ log->reclaim_thread->timeout = R5C_RECLAIM_WAKEUP_INTERVAL;
	2627	+
1230	2628	init_waitqueue_head(&log->iounit_wait);
1231	2629
1232	2630	INIT_LIST_HEAD(&log->no_mem_stripes);
@@ -1234,6 +2632,13 @@ int r5l_init_log(struct r5conf conf, struct md_rdev rdev) · 1 unmodified line
1234	2632	INIT_LIST_HEAD(&log->no_space_stripes);
1235	2633	spin_lock_init(&log->no_space_stripes_lock);
1236	2634
	2635	+ INIT_WORK(&log->deferred_io_work, r5l_submit_io_async);
	2636	+
	2637	+ log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_THROUGH;
	2638	+ INIT_LIST_HEAD(&log->stripe_in_journal_list);
	2639	+ spin_lock_init(&log->stripe_in_journal_lock);
	2640	+ atomic_set(&log->stripe_in_journal_count, 0);
	2641	+
1237	2642	if (r5l_load_log(log))
1238	2643	goto error;
1239	2644

drivers/md/raid5.c+381−242

@@ -70,19 +70,6 @@ module_param(devices_handle_discard_safely, bool, 0644); · 69 unmodified lines
70	70	MODULE_PARM_DESC(devices_handle_discard_safely,
71	71	"Set to Y if all devices in each array reliably return zeroes on reads from discarded regions");
72	72	static struct workqueue_struct *raid5_wq;
73		−/*
74		− * Stripe cache
75		− */
76		−
77		−#define NR_STRIPES 256
78		−#define STRIPE_SIZE PAGE_SIZE
79		−#define STRIPE_SHIFT (PAGE_SHIFT - 9)
80		−#define STRIPE_SECTORS (STRIPE_SIZE>>9)
81		−#define IO_THRESHOLD 1
82		−#define BYPASS_THRESHOLD 1
83		−#define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head))
84		−#define HASH_MASK (NR_HASH - 1)
85		−#define MAX_STRIPE_BATCH 8
86	73
87	74	static inline struct hlist_head stripe_hash(struct r5conf conf, sector_t sect)
88	75	{
@@ -126,64 +113,6 @@ static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf) · 37 unmodified lines
126	113	local_irq_enable();
127	114	}
128	115
129		−/* bio's attached to a stripe+device for I/O are linked together in bi_sector
130		− * order without overlap. There may be several bio's per stripe+device, and
131		− * a bio could span several devices.
132		− * When walking this list for a particular stripe+device, we must never proceed
133		− * beyond a bio that extends past this device, as the next bio might no longer
134		− * be valid.
135		− * This function is used to determine the 'next' bio in the list, given the sector
136		− * of the current stripe+device
137		− */
138		−static inline struct bio r5_next_bio(struct bio bio, sector_t sector)
139		−{
140		− int sectors = bio_sectors(bio);
141		− if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS)
142		− return bio->bi_next;
143		− else
144		− return NULL;
145		−}
146		−
147		−/*
148		− * We maintain a biased count of active stripes in the bottom 16 bits of
149		− * bi_phys_segments, and a count of processed stripes in the upper 16 bits
150		− */
151		−static inline int raid5_bi_processed_stripes(struct bio *bio)
152		−{
153		− atomic_t segments = (atomic_t )&bio->bi_phys_segments;
154		− return (atomic_read(segments) >> 16) & 0xffff;
155		−}
156		−
157		−static inline int raid5_dec_bi_active_stripes(struct bio *bio)
158		−{
159		− atomic_t segments = (atomic_t )&bio->bi_phys_segments;
160		− return atomic_sub_return(1, segments) & 0xffff;
161		−}
162		−
163		−static inline void raid5_inc_bi_active_stripes(struct bio *bio)
164		−{
165		− atomic_t segments = (atomic_t )&bio->bi_phys_segments;
166		− atomic_inc(segments);
167		−}
168		−
169		−static inline void raid5_set_bi_processed_stripes(struct bio *bio,
170		− unsigned int cnt)
171		−{
172		− atomic_t segments = (atomic_t )&bio->bi_phys_segments;
173		− int old, new;
174		−
175		− do {
176		− old = atomic_read(segments);
177		− new = (old & 0xffff) \| (cnt << 16);
178		− } while (atomic_cmpxchg(segments, old, new) != old);
179		−}
180		−
181		−static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt)
182		−{
183		− atomic_t segments = (atomic_t )&bio->bi_phys_segments;
184		− atomic_set(segments, cnt);
185		−}
186		−
187	116	/* Find first data disk in a raid6 stripe */
188	117	static inline int raid6_d0(struct stripe_head *sh)
189	118	{
@@ -289,8 +218,27 @@ static void raid5_wakeup_stripe_thread(struct stripe_head *sh) · 99 unmodified lines
289	218	static void do_release_stripe(struct r5conf conf, struct stripe_head sh,
290	219	struct list_head *temp_inactive_list)
291	220	{
	221	+ int i;
	222	+ int injournal = 0; /* number of date pages with R5_InJournal */
	223	+
292	224	BUG_ON(!list_empty(&sh->lru));
293	225	BUG_ON(atomic_read(&conf->active_stripes)==0);
	226	+
	227	+ if (r5c_is_writeback(conf->log))
	228	+ for (i = sh->disks; i--; )
	229	+ if (test_bit(R5_InJournal, &sh->dev[i].flags))
	230	+ injournal++;
	231	+ /*
	232	+ * When quiesce in r5c write back, set STRIPE_HANDLE for stripes with
	233	+ * data in journal, so they are not released to cached lists
	234	+ */
	235	+ if (conf->quiesce && r5c_is_writeback(conf->log) &&
	236	+ !test_bit(STRIPE_HANDLE, &sh->state) && injournal != 0) {
	237	+ if (test_bit(STRIPE_R5C_CACHING, &sh->state))
	238	+ r5c_make_stripe_write_out(sh);
	239	+ set_bit(STRIPE_HANDLE, &sh->state);
	240	+ }
	241	+
294	242	if (test_bit(STRIPE_HANDLE, &sh->state)) {
295	243	if (test_bit(STRIPE_DELAYED, &sh->state) &&
296	244	!test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
@@ -316,8 +264,30 @@ static void do_release_stripe(struct r5conf conf, struct stripe_head sh, · 19 unmodified lines
316	264	< IO_THRESHOLD)
317	265	md_wakeup_thread(conf->mddev->thread);
318	266	atomic_dec(&conf->active_stripes);
319		− if (!test_bit(STRIPE_EXPANDING, &sh->state))
320		− list_add_tail(&sh->lru, temp_inactive_list);
	267	+ if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
	268	+ if (!r5c_is_writeback(conf->log))
	269	+ list_add_tail(&sh->lru, temp_inactive_list);
	270	+ else {
	271	+ WARN_ON(test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags));
	272	+ if (injournal == 0)
	273	+ list_add_tail(&sh->lru, temp_inactive_list);
	274	+ else if (injournal == conf->raid_disks - conf->max_degraded) {
	275	+ /* full stripe */
	276	+ if (!test_and_set_bit(STRIPE_R5C_FULL_STRIPE, &sh->state))
	277	+ atomic_inc(&conf->r5c_cached_full_stripes);
	278	+ if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state))
	279	+ atomic_dec(&conf->r5c_cached_partial_stripes);
	280	+ list_add_tail(&sh->lru, &conf->r5c_full_stripe_list);
	281	+ r5c_check_cached_full_stripe(conf);
	282	+ } else {
	283	+ /* partial stripe */
	284	+ if (!test_and_set_bit(STRIPE_R5C_PARTIAL_STRIPE,
	285	+ &sh->state))
	286	+ atomic_inc(&conf->r5c_cached_partial_stripes);
	287	+ list_add_tail(&sh->lru, &conf->r5c_partial_stripe_list);
	288	+ }
	289	+ }
	290	+ }
321	291	}
322	292	}
323	293
@@ -541,7 +511,7 @@ retry: · 217 unmodified lines
541	511
542	512	if (dev->toread \|\| dev->read \|\| dev->towrite \|\| dev->written \|\|
543	513	test_bit(R5_LOCKED, &dev->flags)) {
544		− printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n",
	514	+ pr_err("sector=%llx i=%d %p %p %p %p %d\n",
545	515	(unsigned long long)sh->sector, i, dev->toread,
546	516	dev->read, dev->towrite, dev->written,
547	517	test_bit(R5_LOCKED, &dev->flags));
@@ -680,9 +650,12 @@ raid5_get_active_stripe(struct r5conf *conf, sector_t sector, · 132 unmodified lines
680	650	}
681	651	if (noblock && sh == NULL)
682	652	break;
	653	+
	654	+ r5c_check_stripe_cache_usage(conf);
683	655	if (!sh) {
684	656	set_bit(R5_INACTIVE_BLOCKED,
685	657	&conf->cache_state);
	658	+ r5l_wake_reclaim(conf->log, 0);
686	659	wait_event_lock_irq(
687	660	conf->wait_for_stripe,
688	661	!list_empty(conf->inactive_list + hash) &&
@@ -901,8 +874,19 @@ static void ops_run_io(struct stripe_head sh, struct stripe_head_state s) · 212 unmodified lines
901	874
902	875	might_sleep();
903	876
904		− if (r5l_write_stripe(conf->log, sh) == 0)
905		− return;
	877	+ if (!test_bit(STRIPE_R5C_CACHING, &sh->state)) {
	878	+ /* writing out phase */
	879	+ if (s->waiting_extra_page)
	880	+ return;
	881	+ if (r5l_write_stripe(conf->log, sh) == 0)
	882	+ return;
	883	+ } else { /* caching phase */
	884	+ if (test_bit(STRIPE_LOG_TRAPPED, &sh->state)) {
	885	+ r5c_cache_data(conf->log, sh, s);
	886	+ return;
	887	+ }
	888	+ }
	889	+
906	890	for (i = disks; i--; ) {
907	891	int op, op_flags = 0;
908	892	int replace_only = 0;
@@ -977,7 +961,7 @@ again: · 68 unmodified lines
977	961	if (bad < 0) {
978	962	set_bit(BlockedBadBlocks, &rdev->flags);
979	963	if (!conf->mddev->external &&
980		− conf->mddev->flags) {
	964	+ conf->mddev->sb_flags) {
981	965	/* It is very unlikely, but we might
982	966	* still need to write out the
983	967	* bad block log - better give it
@@ -1115,7 +1099,7 @@ again: · 131 unmodified lines
1115	1099	static struct dma_async_tx_descriptor *
1116	1100	async_copy_data(int frombio, struct bio bio, struct page *page,
1117	1101	sector_t sector, struct dma_async_tx_descriptor *tx,
1118		− struct stripe_head *sh)
	1102	+ struct stripe_head *sh, int no_skipcopy)
1119	1103	{
1120	1104	struct bio_vec bvl;
1121	1105	struct bvec_iter iter;
@@ -1155,7 +1139,8 @@ async_copy_data(int frombio, struct bio bio, struct page *page, · 33 unmodified lines
1155	1139	if (frombio) {
1156	1140	if (sh->raid_conf->skip_copy &&
1157	1141	b_offset == 0 && page_offset == 0 &&
1158		− clen == STRIPE_SIZE)
	1142	+ clen == STRIPE_SIZE &&
	1143	+ !no_skipcopy)
1159	1144	*page = bio_page;
1160	1145	else
1161	1146	tx = async_memcpy(*page, bio_page, page_offset,
@@ -1237,7 +1222,7 @@ static void ops_run_biofill(struct stripe_head *sh) · 75 unmodified lines
1237	1222	while (rbi && rbi->bi_iter.bi_sector <
1238	1223	dev->sector + STRIPE_SECTORS) {
1239	1224	tx = async_copy_data(0, rbi, &dev->page,
1240		− dev->sector, tx, sh);
	1225	+ dev->sector, tx, sh, 0);
1241	1226	rbi = r5_next_bio(rbi, dev->sector);
1242	1227	}
1243	1228	}
@@ -1364,10 +1349,15 @@ static int set_syndrome_sources(struct page **srcs, · 120 unmodified lines
1364	1349	if (i == sh->qd_idx \|\| i == sh->pd_idx \|\|
1365	1350	(srctype == SYNDROME_SRC_ALL) \|\|
1366	1351	(srctype == SYNDROME_SRC_WANT_DRAIN &&
1367		− test_bit(R5_Wantdrain, &dev->flags)) \|\|
	1352	+ (test_bit(R5_Wantdrain, &dev->flags) \|\|
	1353	+ test_bit(R5_InJournal, &dev->flags))) \|\|
1368	1354	(srctype == SYNDROME_SRC_WRITTEN &&
1369		− dev->written))
1370		− srcs[slot] = sh->dev[i].page;
	1355	+ dev->written)) {
	1356	+ if (test_bit(R5_InJournal, &dev->flags))
	1357	+ srcs[slot] = sh->dev[i].orig_page;
	1358	+ else
	1359	+ srcs[slot] = sh->dev[i].page;
	1360	+ }
1371	1361	i = raid6_next_disk(i, disks);
1372	1362	} while (i != d0_idx);
1373	1363
@@ -1546,6 +1536,13 @@ static void ops_complete_prexor(void *stripe_head_ref) · 172 unmodified lines
1546	1536
1547	1537	pr_debug("%s: stripe %llu\n", __func__,
1548	1538	(unsigned long long)sh->sector);
	1539	+
	1540	+ if (r5c_is_writeback(sh->raid_conf->log))
	1541	+ /*
	1542	+ * raid5-cache write back uses orig_page during prexor.
	1543	+ * After prexor, it is time to free orig_page
	1544	+ */
	1545	+ r5c_release_extra_page(sh);
1549	1546	}
1550	1547
1551	1548	static struct dma_async_tx_descriptor *
@@ -1567,7 +1564,9 @@ ops_run_prexor5(struct stripe_head sh, struct raid5_percpu percpu, · 15 unmodified lines
1567	1564	for (i = disks; i--; ) {
1568	1565	struct r5dev *dev = &sh->dev[i];
1569	1566	/* Only process blocks that are known to be uptodate */
1570		− if (test_bit(R5_Wantdrain, &dev->flags))
	1567	+ if (test_bit(R5_InJournal, &dev->flags))
	1568	+ xor_srcs[count++] = dev->orig_page;
	1569	+ else if (test_bit(R5_Wantdrain, &dev->flags))
1571	1570	xor_srcs[count++] = dev->page;
1572	1571	}
1573	1572
@@ -1601,6 +1600,7 @@ ops_run_prexor6(struct stripe_head sh, struct raid5_percpu percpu, · 27 unmodified lines
1601	1600	static struct dma_async_tx_descriptor *
1602	1601	ops_run_biodrain(struct stripe_head sh, struct dma_async_tx_descriptor tx)
1603	1602	{
	1603	+ struct r5conf *conf = sh->raid_conf;
1604	1604	int disks = sh->disks;
1605	1605	int i;
1606	1606	struct stripe_head *head_sh = sh;
@@ -1618,6 +1618,11 @@ ops_run_biodrain(struct stripe_head sh, struct dma_async_tx_descriptor tx) · 11 unmodified lines
1618	1618
1619	1619	again:
1620	1620	dev = &sh->dev[i];
	1621	+ /*
	1622	+ * clear R5_InJournal, so when rewriting a page in
	1623	+ * journal, it is not skipped by r5l_log_stripe()
	1624	+ */
	1625	+ clear_bit(R5_InJournal, &dev->flags);
1621	1626	spin_lock_irq(&sh->stripe_lock);
1622	1627	chosen = dev->towrite;
1623	1628	dev->towrite = NULL;
@@ -1637,8 +1642,10 @@ again: · 13 unmodified lines
1637	1642	set_bit(R5_Discard, &dev->flags);
1638	1643	else {
1639	1644	tx = async_copy_data(1, wbi, &dev->page,
1640		− dev->sector, tx, sh);
1641		− if (dev->page != dev->orig_page) {
	1645	+ dev->sector, tx, sh,
	1646	+ r5c_is_writeback(conf->log));
	1647	+ if (dev->page != dev->orig_page &&
	1648	+ !r5c_is_writeback(conf->log)) {
1642	1649	set_bit(R5_SkipCopy, &dev->flags);
1643	1650	clear_bit(R5_UPTODATE, &dev->flags);
1644	1651	clear_bit(R5_OVERWRITE, &dev->flags);
@@ -1746,7 +1753,8 @@ again: · 101 unmodified lines
1746	1753	xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1747	1754	for (i = disks; i--; ) {
1748	1755	struct r5dev *dev = &sh->dev[i];
1749		− if (head_sh->dev[i].written)
	1756	+ if (head_sh->dev[i].written \|\|
	1757	+ test_bit(R5_InJournal, &head_sh->dev[i].flags))
1750	1758	xor_srcs[count++] = dev->page;
1751	1759	}
1752	1760	} else {
@@ -2000,7 +2008,10 @@ static struct stripe_head alloc_stripe(struct kmem_cache sc, gfp_t gfp, · 247 unmodified lines
2000	2008	spin_lock_init(&sh->batch_lock);
2001	2009	INIT_LIST_HEAD(&sh->batch_list);
2002	2010	INIT_LIST_HEAD(&sh->lru);
	2011	+ INIT_LIST_HEAD(&sh->r5c);
	2012	+ INIT_LIST_HEAD(&sh->log_list);
2003	2013	atomic_set(&sh->count, 1);
	2014	+ sh->log_start = MaxSector;
2004	2015	for (i = 0; i < disks; i++) {
2005	2016	struct r5dev *dev = &sh->dev[i];
2006	2017
@@ -2240,10 +2251,24 @@ static int resize_stripes(struct r5conf *conf, int newsize) · 233 unmodified lines
2240	2251	*/
2241	2252	ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
2242	2253	if (ndisks) {
2243		− for (i=0; i<conf->raid_disks; i++)
	2254	+ for (i = 0; i < conf->pool_size; i++)
2244	2255	ndisks[i] = conf->disks[i];
2245		− kfree(conf->disks);
2246		− conf->disks = ndisks;
	2256	+
	2257	+ for (i = conf->pool_size; i < newsize; i++) {
	2258	+ ndisks[i].extra_page = alloc_page(GFP_NOIO);
	2259	+ if (!ndisks[i].extra_page)
	2260	+ err = -ENOMEM;
	2261	+ }
	2262	+
	2263	+ if (err) {
	2264	+ for (i = conf->pool_size; i < newsize; i++)
	2265	+ if (ndisks[i].extra_page)
	2266	+ put_page(ndisks[i].extra_page);
	2267	+ kfree(ndisks);
	2268	+ } else {
	2269	+ kfree(conf->disks);
	2270	+ conf->disks = ndisks;
	2271	+ }
2247	2272	} else
2248	2273	err = -ENOMEM;
2249	2274
@@ -2342,10 +2367,8 @@ static void raid5_end_read_request(struct bio * bi) · 92 unmodified lines
2342	2367	* replacement device. We just fail those on
2343	2368	* any error
2344	2369	*/
2345		− printk_ratelimited(
2346		− KERN_INFO
2347		− "md/raid:%s: read error corrected"
2348		− " (%lu sectors at %llu on %s)\n",
	2370	+ pr_info_ratelimited(
	2371	+ "md/raid:%s: read error corrected (%lu sectors at %llu on %s)\n",
2349	2372	mdname(conf->mddev), STRIPE_SECTORS,
2350	2373	(unsigned long long)s,
2351	2374	bdevname(rdev->bdev, b));
@@ -2365,36 +2388,29 @@ static void raid5_end_read_request(struct bio * bi) · 13 unmodified lines
2365	2388	clear_bit(R5_UPTODATE, &sh->dev[i].flags);
2366	2389	atomic_inc(&rdev->read_errors);
2367	2390	if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
2368		− printk_ratelimited(
2369		− KERN_WARNING
2370		− "md/raid:%s: read error on replacement device "
2371		− "(sector %llu on %s).\n",
	2391	+ pr_warn_ratelimited(
	2392	+ "md/raid:%s: read error on replacement device (sector %llu on %s).\n",
2372	2393	mdname(conf->mddev),
2373	2394	(unsigned long long)s,
2374	2395	bdn);
2375	2396	else if (conf->mddev->degraded >= conf->max_degraded) {
2376	2397	set_bad = 1;
2377		− printk_ratelimited(
2378		− KERN_WARNING
2379		− "md/raid:%s: read error not correctable "
2380		− "(sector %llu on %s).\n",
	2398	+ pr_warn_ratelimited(
	2399	+ "md/raid:%s: read error not correctable (sector %llu on %s).\n",
2381	2400	mdname(conf->mddev),
2382	2401	(unsigned long long)s,
2383	2402	bdn);
2384	2403	} else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {
2385	2404	/* Oh, no!!! */
2386	2405	set_bad = 1;
2387		− printk_ratelimited(
2388		− KERN_WARNING
2389		− "md/raid:%s: read error NOT corrected!! "
2390		− "(sector %llu on %s).\n",
	2406	+ pr_warn_ratelimited(
	2407	+ "md/raid:%s: read error NOT corrected!! (sector %llu on %s).\n",
2391	2408	mdname(conf->mddev),
2392	2409	(unsigned long long)s,
2393	2410	bdn);
2394	2411	} else if (atomic_read(&rdev->read_errors)
2395	2412	> conf->max_nr_stripes)
2396		− printk(KERN_WARNING
2397		− "md/raid:%s: Too many read errors, failing device %s.\n",
	2413	+ pr_warn("md/raid:%s: Too many read errors, failing device %s.\n",
2398	2414	mdname(conf->mddev), bdn);
2399	2415	else
2400	2416	retry = 1;
@@ -2526,15 +2542,14 @@ static void raid5_error(struct mddev mddev, struct md_rdev rdev) · 125 unmodified lines
2526	2542
2527	2543	set_bit(Blocked, &rdev->flags);
2528	2544	set_bit(Faulty, &rdev->flags);
2529		− set_mask_bits(&mddev->flags, 0,
2530		− BIT(MD_CHANGE_DEVS) \| BIT(MD_CHANGE_PENDING));
2531		− printk(KERN_ALERT
2532		− "md/raid:%s: Disk failure on %s, disabling device.\n"
2533		− "md/raid:%s: Operation continuing on %d devices.\n",
2534		− mdname(mddev),
2535		− bdevname(rdev->bdev, b),
2536		− mdname(mddev),
2537		− conf->raid_disks - mddev->degraded);
	2545	+ set_mask_bits(&mddev->sb_flags, 0,
	2546	+ BIT(MD_SB_CHANGE_DEVS) \| BIT(MD_SB_CHANGE_PENDING));
	2547	+ pr_crit("md/raid:%s: Disk failure on %s, disabling device.\n"
	2548	+ "md/raid:%s: Operation continuing on %d devices.\n",
	2549	+ mdname(mddev),
	2550	+ bdevname(rdev->bdev, b),
	2551	+ mdname(mddev),
	2552	+ conf->raid_disks - mddev->degraded);
2538	2553	}
2539	2554
2540	2555	/*
@@ -2856,8 +2871,8 @@ sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous) · 315 unmodified lines
2856	2871	previous, &dummy1, &sh2);
2857	2872	if (check != sh->sector \|\| dummy1 != dd_idx \|\| sh2.pd_idx != sh->pd_idx
2858	2873	\|\| sh2.qd_idx != sh->qd_idx) {
2859		− printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n",
2860		− mdname(conf->mddev));
	2874	+ pr_warn("md/raid:%s: compute_blocknr: map not correct\n",
	2875	+ mdname(conf->mddev));
2861	2876	return 0;
2862	2877	}
2863	2878	return r_sector;
@@ -2872,6 +2887,13 @@ schedule_reconstruction(struct stripe_head sh, struct stripe_head_state s, · 8 unmodified lines
2872	2887	int level = conf->level;
2873	2888
2874	2889	if (rcw) {
	2890	+ /*
	2891	+ * In some cases, handle_stripe_dirtying initially decided to
	2892	+ * run rmw and allocates extra page for prexor. However, rcw is
	2893	+ * cheaper later on. We need to free the extra page now,
	2894	+ * because we won't be able to do that in ops_complete_prexor().
	2895	+ */
	2896	+ r5c_release_extra_page(sh);
2875	2897
2876	2898	for (i = disks; i--; ) {
2877	2899	struct r5dev *dev = &sh->dev[i];
@@ -2882,6 +2904,9 @@ schedule_reconstruction(struct stripe_head sh, struct stripe_head_state s, · 4 unmodified lines
2882	2904	if (!expand)
2883	2905	clear_bit(R5_UPTODATE, &dev->flags);
2884	2906	s->locked++;
	2907	+ } else if (test_bit(R5_InJournal, &dev->flags)) {
	2908	+ set_bit(R5_LOCKED, &dev->flags);
	2909	+ s->locked++;
2885	2910	}
2886	2911	}
2887	2912	/* if we are not expanding this is a proper write request, and
@@ -2921,6 +2946,9 @@ schedule_reconstruction(struct stripe_head sh, struct stripe_head_state s, · 33 unmodified lines
2921	2946	set_bit(R5_LOCKED, &dev->flags);
2922	2947	clear_bit(R5_UPTODATE, &dev->flags);
2923	2948	s->locked++;
	2949	+ } else if (test_bit(R5_InJournal, &dev->flags)) {
	2950	+ set_bit(R5_LOCKED, &dev->flags);
	2951	+ s->locked++;
2924	2952	}
2925	2953	}
2926	2954	if (!s->locked)
@@ -3564,10 +3592,10 @@ unhash: · 637 unmodified lines
3564	3592	break_stripe_batch_list(head_sh, STRIPE_EXPAND_SYNC_FLAGS);
3565	3593	}
3566	3594
3567		−static void handle_stripe_dirtying(struct r5conf *conf,
3568		− struct stripe_head *sh,
3569		− struct stripe_head_state *s,
3570		− int disks)
	3595	+static int handle_stripe_dirtying(struct r5conf *conf,
	3596	+ struct stripe_head *sh,
	3597	+ struct stripe_head_state *s,
	3598	+ int disks)
3571	3599	{
3572	3600	int rmw = 0, rcw = 0, i;
3573	3601	sector_t recovery_cp = conf->mddev->recovery_cp;
@@ -3592,9 +3620,12 @@ static void handle_stripe_dirtying(struct r5conf *conf, · 18 unmodified lines
3592	3620	} else for (i = disks; i--; ) {
3593	3621	/* would I have to read this buffer for read_modify_write */
3594	3622	struct r5dev *dev = &sh->dev[i];
3595		− if ((dev->towrite \|\| i == sh->pd_idx \|\| i == sh->qd_idx) &&
	3623	+ if ((dev->towrite \|\| i == sh->pd_idx \|\| i == sh->qd_idx \|\|
	3624	+ test_bit(R5_InJournal, &dev->flags)) &&
3596	3625	!test_bit(R5_LOCKED, &dev->flags) &&
3597		− !(test_bit(R5_UPTODATE, &dev->flags) \|\|
	3626	+ !((test_bit(R5_UPTODATE, &dev->flags) &&
	3627	+ (!test_bit(R5_InJournal, &dev->flags) \|\|
	3628	+ dev->page != dev->orig_page)) \|\|
3598	3629	test_bit(R5_Wantcompute, &dev->flags))) {
3599	3630	if (test_bit(R5_Insync, &dev->flags))
3600	3631	rmw++;
@@ -3606,13 +3637,15 @@ static void handle_stripe_dirtying(struct r5conf *conf, · 5 unmodified lines
3606	3637	i != sh->pd_idx && i != sh->qd_idx &&
3607	3638	!test_bit(R5_LOCKED, &dev->flags) &&
3608	3639	!(test_bit(R5_UPTODATE, &dev->flags) \|\|
3609		− test_bit(R5_Wantcompute, &dev->flags))) {
	3640	+ test_bit(R5_InJournal, &dev->flags) \|\|
	3641	+ test_bit(R5_Wantcompute, &dev->flags))) {
3610	3642	if (test_bit(R5_Insync, &dev->flags))
3611	3643	rcw++;
3612	3644	else
3613	3645	rcw += 2*disks;
3614	3646	}
3615	3647	}
	3648	+
3616	3649	pr_debug("for sector %llu, rmw=%d rcw=%d\n",
3617	3650	(unsigned long long)sh->sector, rmw, rcw);
3618	3651	set_bit(STRIPE_HANDLE, &sh->state);
@@ -3624,10 +3657,44 @@ static void handle_stripe_dirtying(struct r5conf *conf, · 5 unmodified lines
3624	3657	(unsigned long long)sh->sector, rmw);
3625	3658	for (i = disks; i--; ) {
3626	3659	struct r5dev *dev = &sh->dev[i];
3627		− if ((dev->towrite \|\| i == sh->pd_idx \|\| i == sh->qd_idx) &&
	3660	+ if (test_bit(R5_InJournal, &dev->flags) &&
	3661	+ dev->page == dev->orig_page &&
	3662	+ !test_bit(R5_LOCKED, &sh->dev[sh->pd_idx].flags)) {
	3663	+ /* alloc page for prexor */
	3664	+ struct page *p = alloc_page(GFP_NOIO);
	3665	+
	3666	+ if (p) {
	3667	+ dev->orig_page = p;
	3668	+ continue;
	3669	+ }
	3670	+
	3671	+ /*
	3672	+ * alloc_page() failed, try use
	3673	+ * disk_info->extra_page
	3674	+ */
	3675	+ if (!test_and_set_bit(R5C_EXTRA_PAGE_IN_USE,
	3676	+ &conf->cache_state)) {
	3677	+ r5c_use_extra_page(sh);
	3678	+ break;
	3679	+ }
	3680	+
	3681	+ /* extra_page in use, add to delayed_list */
	3682	+ set_bit(STRIPE_DELAYED, &sh->state);
	3683	+ s->waiting_extra_page = 1;
	3684	+ return -EAGAIN;
	3685	+ }
	3686	+ }
	3687	+
	3688	+ for (i = disks; i--; ) {
	3689	+ struct r5dev *dev = &sh->dev[i];
	3690	+ if ((dev->towrite \|\|
	3691	+ i == sh->pd_idx \|\| i == sh->qd_idx \|\|
	3692	+ test_bit(R5_InJournal, &dev->flags)) &&
3628	3693	!test_bit(R5_LOCKED, &dev->flags) &&
3629		− !(test_bit(R5_UPTODATE, &dev->flags) \|\|
3630		− test_bit(R5_Wantcompute, &dev->flags)) &&
	3694	+ !((test_bit(R5_UPTODATE, &dev->flags) &&
	3695	+ (!test_bit(R5_InJournal, &dev->flags) \|\|
	3696	+ dev->page != dev->orig_page)) \|\|
	3697	+ test_bit(R5_Wantcompute, &dev->flags)) &&
3631	3698	test_bit(R5_Insync, &dev->flags)) {
3632	3699	if (test_bit(STRIPE_PREREAD_ACTIVE,
3633	3700	&sh->state)) {
@@ -3653,6 +3720,7 @@ static void handle_stripe_dirtying(struct r5conf *conf, · 19 unmodified lines
3653	3720	i != sh->pd_idx && i != sh->qd_idx &&
3654	3721	!test_bit(R5_LOCKED, &dev->flags) &&
3655	3722	!(test_bit(R5_UPTODATE, &dev->flags) \|\|
	3723	+ test_bit(R5_InJournal, &dev->flags) \|\|
3656	3724	test_bit(R5_Wantcompute, &dev->flags))) {
3657	3725	rcw++;
3658	3726	if (test_bit(R5_Insync, &dev->flags) &&
@@ -3692,8 +3760,9 @@ static void handle_stripe_dirtying(struct r5conf *conf, · 33 unmodified lines
3692	3760	*/
3693	3761	if ((s->req_compute \|\| !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
3694	3762	(s->locked == 0 && (rcw == 0 \|\| rmw == 0) &&
3695		− !test_bit(STRIPE_BIT_DELAY, &sh->state)))
	3763	+ !test_bit(STRIPE_BIT_DELAY, &sh->state)))
3696	3764	schedule_reconstruction(sh, s, rcw == 0, 0);
	3765	+ return 0;
3697	3766	}
3698	3767
3699	3768	static void handle_parity_checks5(struct r5conf conf, struct stripe_head sh,
@@ -3777,7 +3846,7 @@ static void handle_parity_checks5(struct r5conf conf, struct stripe_head sh, · 77 unmodified lines
3777	3846	case check_state_compute_run:
3778	3847	break;
3779	3848	default:
3780		− printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
	3849	+ pr_err("%s: unknown check_state: %d sector: %llu\n",
3781	3850	__func__, sh->check_state,
3782	3851	(unsigned long long) sh->sector);
3783	3852	BUG();
@@ -3941,9 +4010,9 @@ static void handle_parity_checks6(struct r5conf conf, struct stripe_head sh, · 157 unmodified lines
3941	4010	case check_state_compute_run:
3942	4011	break;
3943	4012	default:
3944		− printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
3945		− __func__, sh->check_state,
3946		− (unsigned long long) sh->sector);
	4013	+ pr_warn("%s: unknown check_state: %d sector: %llu\n",
	4014	+ __func__, sh->check_state,
	4015	+ (unsigned long long) sh->sector);
3947	4016	BUG();
3948	4017	}
3949	4018	}
@@ -4183,6 +4252,11 @@ static void analyse_stripe(struct stripe_head sh, struct stripe_head_state s) · 233 unmodified lines
4183	4252	if (rdev && !test_bit(Faulty, &rdev->flags))
4184	4253	do_recovery = 1;
4185	4254	}
	4255	+
	4256	+ if (test_bit(R5_InJournal, &dev->flags))
	4257	+ s->injournal++;
	4258	+ if (test_bit(R5_InJournal, &dev->flags) && dev->written)
	4259	+ s->just_cached++;
4186	4260	}
4187	4261	if (test_bit(STRIPE_SYNCING, &sh->state)) {
4188	4262	/* If there is a failed device being replaced,
@@ -4411,7 +4485,8 @@ static void handle_stripe(struct stripe_head *sh) · 222 unmodified lines
4411	4485	struct r5dev *dev = &sh->dev[i];
4412	4486	if (test_bit(R5_LOCKED, &dev->flags) &&
4413	4487	(i == sh->pd_idx \|\| i == sh->qd_idx \|\|
4414		− dev->written)) {
	4488	+ dev->written \|\| test_bit(R5_InJournal,
	4489	+ &dev->flags))) {
4415	4490	pr_debug("Writing block %d\n", i);
4416	4491	set_bit(R5_Wantwrite, &dev->flags);
4417	4492	if (prexor)
@@ -4451,6 +4526,10 @@ static void handle_stripe(struct stripe_head *sh) · 33 unmodified lines
4451	4526	test_bit(R5_Discard, &qdev->flags))))))
4452	4527	handle_stripe_clean_event(conf, sh, disks, &s.return_bi);
4453	4528
	4529	+ if (s.just_cached)
	4530	+ r5c_handle_cached_data_endio(conf, sh, disks, &s.return_bi);
	4531	+ r5l_stripe_write_finished(sh);
	4532	+
4454	4533	/* Now we might consider reading some blocks, either to check/generate
4455	4534	* parity, or to satisfy requests
4456	4535	* or to load a block that is being partially written.
@@ -4462,14 +4541,51 @@ static void handle_stripe(struct stripe_head *sh) · 5 unmodified lines
4462	4541	\|\| s.expanding)
4463	4542	handle_stripe_fill(sh, &s, disks);
4464	4543
4465		− /* Now to consider new write requests and what else, if anything
4466		− * should be read. We do not handle new writes when:
	4544	+ /*
	4545	+ * When the stripe finishes full journal write cycle (write to journal
	4546	+ * and raid disk), this is the clean up procedure so it is ready for
	4547	+ * next operation.
	4548	+ */
	4549	+ r5c_finish_stripe_write_out(conf, sh, &s);
	4550	+
	4551	+ /*
	4552	+ * Now to consider new write requests, cache write back and what else,
	4553	+ * if anything should be read. We do not handle new writes when:
4467	4554	* 1/ A 'write' operation (copy+xor) is already in flight.
4468	4555	* 2/ A 'check' operation is in flight, as it may clobber the parity
4469	4556	* block.
	4557	+ * 3/ A r5c cache log write is in flight.
4470	4558	*/
4471		− if (s.to_write && !sh->reconstruct_state && !sh->check_state)
4472		− handle_stripe_dirtying(conf, sh, &s, disks);
	4559	+
	4560	+ if (!sh->reconstruct_state && !sh->check_state && !sh->log_io) {
	4561	+ if (!r5c_is_writeback(conf->log)) {
	4562	+ if (s.to_write)
	4563	+ handle_stripe_dirtying(conf, sh, &s, disks);
	4564	+ } else { /* write back cache */
	4565	+ int ret = 0;
	4566	+
	4567	+ /* First, try handle writes in caching phase */
	4568	+ if (s.to_write)
	4569	+ ret = r5c_try_caching_write(conf, sh, &s,
	4570	+ disks);
	4571	+ /*
	4572	+ * If caching phase failed: ret == -EAGAIN
	4573	+ * OR
	4574	+ * stripe under reclaim: !caching && injournal
	4575	+ *
	4576	+ * fall back to handle_stripe_dirtying()
	4577	+ */
	4578	+ if (ret == -EAGAIN \|\|
	4579	+ /* stripe under reclaim: !caching && injournal */
	4580	+ (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&
	4581	+ s.injournal > 0)) {
	4582	+ ret = handle_stripe_dirtying(conf, sh, &s,
	4583	+ disks);
	4584	+ if (ret == -EAGAIN)
	4585	+ goto finish;
	4586	+ }
	4587	+ }
	4588	+ }
4473	4589
4474	4590	/* maybe we need to check and possibly fix the parity for this stripe
4475	4591	* Any reads will already have been scheduled, so we just see if enough
@@ -4640,9 +4756,7 @@ finish: · 164 unmodified lines
4640	4756	}
4641	4757
4642	4758	if (!bio_list_empty(&s.return_bi)) {
4643		− if (test_bit(MD_CHANGE_PENDING, &conf->mddev->flags) &&
4644		− (s.failed <= conf->max_degraded \|\|
4645		− conf->mddev->external == 0)) {
	4759	+ if (test_bit(MD_SB_CHANGE_PENDING, &conf->mddev->sb_flags)) {
4646	4760	spin_lock_irq(&conf->device_lock);
4647	4761	bio_list_merge(&conf->return_bi, &s.return_bi);
4648	4762	spin_unlock_irq(&conf->device_lock);
@@ -4698,6 +4812,10 @@ static int raid5_congested(struct mddev *mddev, int bits) · 49 unmodified lines
4698	4812
4699	4813	if (test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state))
4700	4814	return 1;
	4815	+
	4816	+ /* Also checks whether there is pressure on r5cache log space */
	4817	+ if (test_bit(R5C_LOG_TIGHT, &conf->cache_state))
	4818	+ return 1;
4701	4819	if (conf->quiesce)
4702	4820	return 1;
4703	4821	if (atomic_read(&conf->empty_inactive_list_nr))
@@ -5167,6 +5285,7 @@ static void raid5_make_request(struct mddev mddev, struct bio bi) · 463 unmodified lines
5167	5285	int remaining;
5168	5286	DEFINE_WAIT(w);
5169	5287	bool do_prepare;
	5288	+ bool do_flush = false;
5170	5289
5171	5290	if (unlikely(bi->bi_opf & REQ_PREFLUSH)) {
5172	5291	int ret = r5l_handle_flush_request(conf->log, bi);
@@ -5178,6 +5297,11 @@ static void raid5_make_request(struct mddev mddev, struct bio bi) · 5 unmodified lines
5178	5297	return;
5179	5298	}
5180	5299	/* ret == -EAGAIN, fallback */
	5300	+ /*
	5301	+ * if r5l_handle_flush_request() didn't clear REQ_PREFLUSH,
	5302	+ * we need to flush journal device
	5303	+ */
	5304	+ do_flush = bi->bi_opf & REQ_PREFLUSH;
5181	5305	}
5182	5306
5183	5307	md_write_start(mddev, bi);
@@ -5188,6 +5312,7 @@ static void raid5_make_request(struct mddev mddev, struct bio bi) · 4 unmodified lines
5188	5312	* data on failed drives.
5189	5313	*/
5190	5314	if (rw == READ && mddev->degraded == 0 &&
	5315	+ !r5c_is_writeback(conf->log) &&
5191	5316	mddev->reshape_position == MaxSector) {
5192	5317	bi = chunk_aligned_read(mddev, bi);
5193	5318	if (!bi)
@@ -5316,6 +5441,12 @@ static void raid5_make_request(struct mddev mddev, struct bio bi) · 122 unmodified lines
5316	5441	do_prepare = true;
5317	5442	goto retry;
5318	5443	}
	5444	+ if (do_flush) {
	5445	+ set_bit(STRIPE_R5C_PREFLUSH, &sh->state);
	5446	+ /* we only need flush for one stripe */
	5447	+ do_flush = false;
	5448	+ }
	5449	+
5319	5450	set_bit(STRIPE_HANDLE, &sh->state);
5320	5451	clear_bit(STRIPE_DELAYED, &sh->state);
5321	5452	if ((!sh->batch_head \|\| sh == sh->batch_head) &&
@@ -5481,9 +5612,9 @@ static sector_t reshape_request(struct mddev mddev, sector_t sector_nr, int sk · 159 unmodified lines
5481	5612	mddev->reshape_position = conf->reshape_progress;
5482	5613	mddev->curr_resync_completed = sector_nr;
5483	5614	conf->reshape_checkpoint = jiffies;
5484		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
	5615	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
5485	5616	md_wakeup_thread(mddev->thread);
5486		− wait_event(mddev->sb_wait, mddev->flags == 0 \|\|
	5617	+ wait_event(mddev->sb_wait, mddev->sb_flags == 0 \|\|
5487	5618	test_bit(MD_RECOVERY_INTR, &mddev->recovery));
5488	5619	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
5489	5620	return 0;
@@ -5579,10 +5710,10 @@ finish: · 89 unmodified lines
5579	5710	mddev->reshape_position = conf->reshape_progress;
5580	5711	mddev->curr_resync_completed = sector_nr;
5581	5712	conf->reshape_checkpoint = jiffies;
5582		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
	5713	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
5583	5714	md_wakeup_thread(mddev->thread);
5584	5715	wait_event(mddev->sb_wait,
5585		− !test_bit(MD_CHANGE_DEVS, &mddev->flags)
	5716	+ !test_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags)
5586	5717	\|\| test_bit(MD_RECOVERY_INTR, &mddev->recovery));
5587	5718	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
5588	5719	goto ret;
@@ -5857,10 +5988,10 @@ static void raid5d(struct md_thread *thread) · 268 unmodified lines
5857	5988	md_check_recovery(mddev);
5858	5989
5859	5990	if (!bio_list_empty(&conf->return_bi) &&
5860		− !test_bit(MD_CHANGE_PENDING, &mddev->flags)) {
	5991	+ !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
5861	5992	struct bio_list tmp = BIO_EMPTY_LIST;
5862	5993	spin_lock_irq(&conf->device_lock);
5863		− if (!test_bit(MD_CHANGE_PENDING, &mddev->flags)) {
	5994	+ if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
5864	5995	bio_list_merge(&tmp, &conf->return_bi);
5865	5996	bio_list_init(&conf->return_bi);
5866	5997	}
@@ -5907,7 +6038,7 @@ static void raid5d(struct md_thread *thread) · 40 unmodified lines
5907	6038	break;
5908	6039	handled += batch_size;
5909	6040
5910		− if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) {
	6041	+ if (mddev->sb_flags & ~(1 << MD_SB_CHANGE_PENDING)) {
5911	6042	spin_unlock_irq(&conf->device_lock);
5912	6043	md_check_recovery(mddev);
5913	6044	spin_lock_irq(&conf->device_lock);
@@ -6237,6 +6368,7 @@ static struct attribute *raid5_attrs[] = { · 323 unmodified lines
6237	6368	&raid5_group_thread_cnt.attr,
6238	6369	&raid5_skip_copy.attr,
6239	6370	&raid5_rmw_level.attr,
	6371	+ &r5c_journal_mode.attr,
6240	6372	NULL,
6241	6373	};
6242	6374	static struct attribute_group raid5_attrs_group = {
@@ -6363,6 +6495,8 @@ static void raid5_free_percpu(struct r5conf *conf) · 120 unmodified lines
6363	6495
6364	6496	static void free_conf(struct r5conf *conf)
6365	6497	{
	6498	+ int i;
	6499	+
6366	6500	if (conf->log)
6367	6501	r5l_exit_log(conf->log);
6368	6502	if (conf->shrinker.nr_deferred)
@@ -6371,6 +6505,9 @@ static void free_conf(struct r5conf *conf) · 2 unmodified lines
6371	6505	free_thread_groups(conf);
6372	6506	shrink_stripes(conf);
6373	6507	raid5_free_percpu(conf);
	6508	+ for (i = 0; i < conf->pool_size; i++)
	6509	+ if (conf->disks[i].extra_page)
	6510	+ put_page(conf->disks[i].extra_page);
6374	6511	kfree(conf->disks);
6375	6512	kfree(conf->stripe_hashtbl);
6376	6513	kfree(conf);
@@ -6382,8 +6519,8 @@ static int raid456_cpu_up_prepare(unsigned int cpu, struct hlist_node *node) · 5 unmodified lines
6382	6519	struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);
6383	6520
6384	6521	if (alloc_scratch_buffer(conf, percpu)) {
6385		− pr_err("%s: failed memory allocation for cpu%u\n",
6386		− __func__, cpu);
	6522	+ pr_warn("%s: failed memory allocation for cpu%u\n",
	6523	+ __func__, cpu);
6387	6524	return -ENOMEM;
6388	6525	}
6389	6526	return 0;
@@ -6453,29 +6590,29 @@ static struct r5conf setup_conf(struct mddev mddev) · 63 unmodified lines
6453	6590	if (mddev->new_level != 5
6454	6591	&& mddev->new_level != 4
6455	6592	&& mddev->new_level != 6) {
6456		− printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n",
6457		− mdname(mddev), mddev->new_level);
	6593	+ pr_warn("md/raid:%s: raid level not set to 4/5/6 (%d)\n",
	6594	+ mdname(mddev), mddev->new_level);
6458	6595	return ERR_PTR(-EIO);
6459	6596	}
6460	6597	if ((mddev->new_level == 5
6461	6598	&& !algorithm_valid_raid5(mddev->new_layout)) \|\|
6462	6599	(mddev->new_level == 6
6463	6600	&& !algorithm_valid_raid6(mddev->new_layout))) {
6464		− printk(KERN_ERR "md/raid:%s: layout %d not supported\n",
6465		− mdname(mddev), mddev->new_layout);
	6601	+ pr_warn("md/raid:%s: layout %d not supported\n",
	6602	+ mdname(mddev), mddev->new_layout);
6466	6603	return ERR_PTR(-EIO);
6467	6604	}
6468	6605	if (mddev->new_level == 6 && mddev->raid_disks < 4) {
6469		− printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n",
6470		− mdname(mddev), mddev->raid_disks);
	6606	+ pr_warn("md/raid:%s: not enough configured devices (%d, minimum 4)\n",
	6607	+ mdname(mddev), mddev->raid_disks);
6471	6608	return ERR_PTR(-EINVAL);
6472	6609	}
6473	6610
6474	6611	if (!mddev->new_chunk_sectors \|\|
6475	6612	(mddev->new_chunk_sectors << 9) % PAGE_SIZE \|\|
6476	6613	!is_power_of_2(mddev->new_chunk_sectors)) {
6477		− printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n",
6478		− mdname(mddev), mddev->new_chunk_sectors << 9);
	6614	+ pr_warn("md/raid:%s: invalid chunk size %d\n",
	6615	+ mdname(mddev), mddev->new_chunk_sectors << 9);
6479	6616	return ERR_PTR(-EINVAL);
6480	6617	}
6481	6618
@@ -6517,9 +6654,16 @@ static struct r5conf setup_conf(struct mddev mddev) · 35 unmodified lines
6517	6654
6518	6655	conf->disks = kzalloc(max_disks * sizeof(struct disk_info),
6519	6656	GFP_KERNEL);
	6657	+
6520	6658	if (!conf->disks)
6521	6659	goto abort;
6522	6660
	6661	+ for (i = 0; i < max_disks; i++) {
	6662	+ conf->disks[i].extra_page = alloc_page(GFP_KERNEL);
	6663	+ if (!conf->disks[i].extra_page)
	6664	+ goto abort;
	6665	+ }
	6666	+
6523	6667	conf->mddev = mddev;
6524	6668
6525	6669	if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
@@ -6540,6 +6684,11 @@ static struct r5conf setup_conf(struct mddev mddev) · 14 unmodified lines
6540	6684	for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
6541	6685	INIT_LIST_HEAD(conf->temp_inactive_list + i);
6542	6686
	6687	+ atomic_set(&conf->r5c_cached_full_stripes, 0);
	6688	+ INIT_LIST_HEAD(&conf->r5c_full_stripe_list);
	6689	+ atomic_set(&conf->r5c_cached_partial_stripes, 0);
	6690	+ INIT_LIST_HEAD(&conf->r5c_partial_stripe_list);
	6691	+
6543	6692	conf->level = mddev->new_level;
6544	6693	conf->chunk_sectors = mddev->new_chunk_sectors;
6545	6694	if (raid5_alloc_percpu(conf) != 0)
@@ -6566,9 +6715,8 @@ static struct r5conf setup_conf(struct mddev mddev) · 20 unmodified lines
6566	6715
6567	6716	if (test_bit(In_sync, &rdev->flags)) {
6568	6717	char b[BDEVNAME_SIZE];
6569		− printk(KERN_INFO "md/raid:%s: device %s operational as raid"
6570		− " disk %d\n",
6571		− mdname(mddev), bdevname(rdev->bdev, b), raid_disk);
	6718	+ pr_info("md/raid:%s: device %s operational as raid disk %d\n",
	6719	+ mdname(mddev), bdevname(rdev->bdev, b), raid_disk);
6572	6720	} else if (rdev->saved_raid_disk != raid_disk)
6573	6721	/* Cannot rely on bitmap to complete recovery */
6574	6722	conf->fullsync = 1;
@@ -6602,21 +6750,18 @@ static struct r5conf setup_conf(struct mddev mddev) · 27 unmodified lines
6602	6750	((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4);
6603	6751	conf->min_nr_stripes = max(NR_STRIPES, stripes);
6604	6752	if (conf->min_nr_stripes != NR_STRIPES)
6605		− printk(KERN_INFO
6606		− "md/raid:%s: force stripe size %d for reshape\n",
	6753	+ pr_info("md/raid:%s: force stripe size %d for reshape\n",
6607	6754	mdname(mddev), conf->min_nr_stripes);
6608	6755	}
6609	6756	memory = conf->min_nr_stripes * (sizeof(struct stripe_head) +
6610	6757	max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
6611	6758	atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS);
6612	6759	if (grow_stripes(conf, conf->min_nr_stripes)) {
6613		− printk(KERN_ERR
6614		− "md/raid:%s: couldn't allocate %dkB for buffers\n",
6615		− mdname(mddev), memory);
	6760	+ pr_warn("md/raid:%s: couldn't allocate %dkB for buffers\n",
	6761	+ mdname(mddev), memory);
6616	6762	goto abort;
6617	6763	} else
6618		− printk(KERN_INFO "md/raid:%s: allocated %dkB\n",
6619		− mdname(mddev), memory);
	6764	+ pr_debug("md/raid:%s: allocated %dkB\n", mdname(mddev), memory);
6620	6765	/*
6621	6766	* Losing a stripe head costs more than the time to refill it,
6622	6767	* it reduces the queue depth and so can hurt throughput.
@@ -6628,18 +6773,16 @@ static struct r5conf setup_conf(struct mddev mddev) · 5 unmodified lines
6628	6773	conf->shrinker.batch = 128;
6629	6774	conf->shrinker.flags = 0;
6630	6775	if (register_shrinker(&conf->shrinker)) {
6631		− printk(KERN_ERR
6632		− "md/raid:%s: couldn't register shrinker.\n",
6633		− mdname(mddev));
	6776	+ pr_warn("md/raid:%s: couldn't register shrinker.\n",
	6777	+ mdname(mddev));
6634	6778	goto abort;
6635	6779	}
6636	6780
6637	6781	sprintf(pers_name, "raid%d", mddev->new_level);
6638	6782	conf->thread = md_register_thread(raid5d, mddev, pers_name);
6639	6783	if (!conf->thread) {
6640		− printk(KERN_ERR
6641		− "md/raid:%s: couldn't allocate thread.\n",
6642		− mdname(mddev));
	6784	+ pr_warn("md/raid:%s: couldn't allocate thread.\n",
	6785	+ mdname(mddev));
6643	6786	goto abort;
6644	6787	}
6645	6788
@@ -6692,9 +6835,8 @@ static int raid5_run(struct mddev *mddev) · 46 unmodified lines
6692	6835	int first = 1;
6693	6836
6694	6837	if (mddev->recovery_cp != MaxSector)
6695		− printk(KERN_NOTICE "md/raid:%s: not clean"
6696		− " -- starting background reconstruction\n",
6697		− mdname(mddev));
	6838	+ pr_notice("md/raid:%s: not clean -- starting background reconstruction\n",
	6839	+ mdname(mddev));
6698	6840
6699	6841	rdev_for_each(rdev, mddev) {
6700	6842	long long diff;
@@ -6737,15 +6879,14 @@ static int raid5_run(struct mddev *mddev) · 36 unmodified lines
6737	6879	int new_data_disks;
6738	6880
6739	6881	if (journal_dev) {
6740		− printk(KERN_ERR "md/raid:%s: don't support reshape with journal - aborting.\n",
6741		− mdname(mddev));
	6882	+ pr_warn("md/raid:%s: don't support reshape with journal - aborting.\n",
	6883	+ mdname(mddev));
6742	6884	return -EINVAL;
6743	6885	}
6744	6886
6745	6887	if (mddev->new_level != mddev->level) {
6746		− printk(KERN_ERR "md/raid:%s: unsupported reshape "
6747		− "required - aborting.\n",
6748		− mdname(mddev));
	6888	+ pr_warn("md/raid:%s: unsupported reshape required - aborting.\n",
	6889	+ mdname(mddev));
6749	6890	return -EINVAL;
6750	6891	}
6751	6892	old_disks = mddev->raid_disks - mddev->delta_disks;
@@ -6760,8 +6901,8 @@ static int raid5_run(struct mddev *mddev) · 8 unmodified lines
6760	6901	chunk_sectors = max(mddev->chunk_sectors, mddev->new_chunk_sectors);
6761	6902	new_data_disks = mddev->raid_disks - max_degraded;
6762	6903	if (sector_div(here_new, chunk_sectors * new_data_disks)) {
6763		− printk(KERN_ERR "md/raid:%s: reshape_position not "
6764		− "on a stripe boundary\n", mdname(mddev));
	6904	+ pr_warn("md/raid:%s: reshape_position not on a stripe boundary\n",
	6905	+ mdname(mddev));
6765	6906	return -EINVAL;
6766	6907	}
6767	6908	reshape_offset = here_new * chunk_sectors;
@@ -6782,10 +6923,8 @@ static int raid5_run(struct mddev *mddev) · 14 unmodified lines
6782	6923	abs(min_offset_diff) >= mddev->new_chunk_sectors)
6783	6924	/* not really in-place - so OK */;
6784	6925	else if (mddev->ro == 0) {
6785		− printk(KERN_ERR "md/raid:%s: in-place reshape "
6786		− "must be started in read-only mode "
6787		− "- aborting\n",
6788		− mdname(mddev));
	6926	+ pr_warn("md/raid:%s: in-place reshape must be started in read-only mode - aborting\n",
	6927	+ mdname(mddev));
6789	6928	return -EINVAL;
6790	6929	}
6791	6930	} else if (mddev->reshape_backwards
@@ -6794,13 +6933,11 @@ static int raid5_run(struct mddev *mddev) · 2 unmodified lines
6794	6933	: (here_new * chunk_sectors >=
6795	6934	here_old * chunk_sectors + (-min_offset_diff))) {
6796	6935	/* Reading from the same stripe as writing to - bad */
6797		− printk(KERN_ERR "md/raid:%s: reshape_position too early for "
6798		− "auto-recovery - aborting.\n",
6799		− mdname(mddev));
	6936	+ pr_warn("md/raid:%s: reshape_position too early for auto-recovery - aborting.\n",
	6937	+ mdname(mddev));
6800	6938	return -EINVAL;
6801	6939	}
6802		− printk(KERN_INFO "md/raid:%s: reshape will continue\n",
6803		− mdname(mddev));
	6940	+ pr_debug("md/raid:%s: reshape will continue\n", mdname(mddev));
6804	6941	/* OK, we should be able to continue; */
6805	6942	} else {
6806	6943	BUG_ON(mddev->level != mddev->new_level);
@@ -6819,8 +6956,8 @@ static int raid5_run(struct mddev *mddev) · 12 unmodified lines
6819	6956
6820	6957	if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
6821	6958	if (!journal_dev) {
6822		− pr_err("md/raid:%s: journal disk is missing, force array readonly\n",
6823		− mdname(mddev));
	6959	+ pr_warn("md/raid:%s: journal disk is missing, force array readonly\n",
	6960	+ mdname(mddev));
6824	6961	mddev->ro = 1;
6825	6962	set_disk_ro(mddev->gendisk, 1);
6826	6963	} else if (mddev->recovery_cp == MaxSector)
@@ -6847,8 +6984,7 @@ static int raid5_run(struct mddev *mddev) · 20 unmodified lines
6847	6984	if (conf->disks[i].replacement &&
6848	6985	conf->reshape_progress != MaxSector) {
6849	6986	/* replacements and reshape simply do not mix. */
6850		− printk(KERN_ERR "md: cannot handle concurrent "
6851		− "replacement and reshape.\n");
	6987	+ pr_warn("md: cannot handle concurrent replacement and reshape.\n");
6852	6988	goto abort;
6853	6989	}
6854	6990	if (test_bit(In_sync, &rdev->flags)) {
@@ -6890,8 +7026,7 @@ static int raid5_run(struct mddev *mddev) · 35 unmodified lines
6890	7026	mddev->degraded = calc_degraded(conf);
6891	7027
6892	7028	if (has_failed(conf)) {
6893		− printk(KERN_ERR "md/raid:%s: not enough operational devices"
6894		− " (%d/%d failed)\n",
	7029	+ pr_crit("md/raid:%s: not enough operational devices (%d/%d failed)\n",
6895	7030	mdname(mddev), mddev->degraded, conf->raid_disks);
6896	7031	goto abort;
6897	7032	}
@@ -6903,29 +7038,19 @@ static int raid5_run(struct mddev *mddev) · 5 unmodified lines
6903	7038	if (mddev->degraded > dirty_parity_disks &&
6904	7039	mddev->recovery_cp != MaxSector) {
6905	7040	if (mddev->ok_start_degraded)
6906		− printk(KERN_WARNING
6907		− "md/raid:%s: starting dirty degraded array"
6908		− " - data corruption possible.\n",
6909		− mdname(mddev));
	7041	+ pr_crit("md/raid:%s: starting dirty degraded array - data corruption possible.\n",
	7042	+ mdname(mddev));
6910	7043	else {
6911		− printk(KERN_ERR
6912		− "md/raid:%s: cannot start dirty degraded array.\n",
6913		− mdname(mddev));
	7044	+ pr_crit("md/raid:%s: cannot start dirty degraded array.\n",
	7045	+ mdname(mddev));
6914	7046	goto abort;
6915	7047	}
6916	7048	}
6917	7049
6918		− if (mddev->degraded == 0)
6919		− printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d"
6920		− " devices, algorithm %d\n", mdname(mddev), conf->level,
6921		− mddev->raid_disks-mddev->degraded, mddev->raid_disks,
6922		− mddev->new_layout);
6923		− else
6924		− printk(KERN_ALERT "md/raid:%s: raid level %d active with %d"
6925		− " out of %d devices, algorithm %d\n",
6926		− mdname(mddev), conf->level,
6927		− mddev->raid_disks - mddev->degraded,
6928		− mddev->raid_disks, mddev->new_layout);
	7050	+ pr_info("md/raid:%s: raid level %d active with %d out of %d devices, algorithm %d\n",
	7051	+ mdname(mddev), conf->level,
	7052	+ mddev->raid_disks-mddev->degraded, mddev->raid_disks,
	7053	+ mddev->new_layout);
6929	7054
6930	7055	print_raid5_conf(conf);
6931	7056
@@ -6945,9 +7070,8 @@ static int raid5_run(struct mddev *mddev) · 13 unmodified lines
6945	7070	mddev->to_remove = NULL;
6946	7071	else if (mddev->kobj.sd &&
6947	7072	sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
6948		− printk(KERN_WARNING
6949		− "raid5: failed to create sysfs attributes for %s\n",
6950		− mdname(mddev));
	7073	+ pr_warn("raid5: failed to create sysfs attributes for %s\n",
	7074	+ mdname(mddev));
6951	7075	md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
6952	7076
6953	7077	if (mddev->queue) {
@@ -6979,6 +7103,15 @@ static int raid5_run(struct mddev *mddev) · 25 unmodified lines
6979	7103	stripe = (stripe \| (stripe-1)) + 1;
6980	7104	mddev->queue->limits.discard_alignment = stripe;
6981	7105	mddev->queue->limits.discard_granularity = stripe;
	7106	+
	7107	+ /*
	7108	+ * We use 16-bit counter of active stripes in bi_phys_segments
	7109	+ * (minus one for over-loaded initialization)
	7110	+ */
	7111	+ blk_queue_max_hw_sectors(mddev->queue, 0xfffe * STRIPE_SECTORS);
	7112	+ blk_queue_max_discard_sectors(mddev->queue,
	7113	+ 0xfffe * STRIPE_SECTORS);
	7114	+
6982	7115	/*
6983	7116	* unaligned part of discard request will be ignored, so can't
6984	7117	* guarantee discard_zeroes_data
@@ -7035,9 +7168,10 @@ static int raid5_run(struct mddev *mddev) · 50 unmodified lines
7035	7168	if (journal_dev) {
7036	7169	char b[BDEVNAME_SIZE];
7037	7170
7038		− printk(KERN_INFO"md/raid:%s: using device %s as journal\n",
7039		− mdname(mddev), bdevname(journal_dev->bdev, b));
7040		− r5l_init_log(conf, journal_dev);
	7171	+ pr_debug("md/raid:%s: using device %s as journal\n",
	7172	+ mdname(mddev), bdevname(journal_dev->bdev, b));
	7173	+ if (r5l_init_log(conf, journal_dev))
	7174	+ goto abort;
7041	7175	}
7042	7176
7043	7177	return 0;
@@ -7046,7 +7180,7 @@ abort: · 2 unmodified lines
7046	7180	print_raid5_conf(conf);
7047	7181	free_conf(conf);
7048	7182	mddev->private = NULL;
7049		− printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev));
	7183	+ pr_warn("md/raid:%s: failed to run raid set.\n", mdname(mddev));
7050	7184	return -EIO;
7051	7185	}
7052	7186
@@ -7080,12 +7214,12 @@ static void print_raid5_conf (struct r5conf *conf) · 27 unmodified lines
7080	7214	int i;
7081	7215	struct disk_info *tmp;
7082	7216
7083		− printk(KERN_DEBUG "RAID conf printout:\n");
	7217	+ pr_debug("RAID conf printout:\n");
7084	7218	if (!conf) {
7085		− printk("(conf==NULL)\n");
	7219	+ pr_debug("(conf==NULL)\n");
7086	7220	return;
7087	7221	}
7088		− printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level,
	7222	+ pr_debug(" --- level:%d rd:%d wd:%d\n", conf->level,
7089	7223	conf->raid_disks,
7090	7224	conf->raid_disks - conf->mddev->degraded);
7091	7225
@@ -7093,7 +7227,7 @@ static void print_raid5_conf (struct r5conf *conf) · 1 unmodified line
7093	7227	char b[BDEVNAME_SIZE];
7094	7228	tmp = conf->disks + i;
7095	7229	if (tmp->rdev)
7096		− printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n",
	7230	+ pr_debug(" disk %d, o:%d, dev:%s\n",
7097	7231	i, !test_bit(Faulty, &tmp->rdev->flags),
7098	7232	bdevname(tmp->rdev->bdev, b));
7099	7233	}
@@ -7241,8 +7375,8 @@ static int raid5_add_disk(struct mddev mddev, struct md_rdev rdev) · 141 unmodified lines
7241	7375	* write requests running. We should be safe
7242	7376	*/
7243	7377	r5l_init_log(conf, rdev);
7244		− printk(KERN_INFO"md/raid:%s: using device %s as journal\n",
7245		− mdname(mddev), bdevname(rdev->bdev, b));
	7378	+ pr_debug("md/raid:%s: using device %s as journal\n",
	7379	+ mdname(mddev), bdevname(rdev->bdev, b));
7246	7380	return 0;
7247	7381	}
7248	7382	if (mddev->recovery_disabled == conf->recovery_disabled)
@@ -7346,10 +7480,10 @@ static int check_stripe_cache(struct mddev *mddev) · 97 unmodified lines
7346	7480	> conf->min_nr_stripes \|\|
7347	7481	((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
7348	7482	> conf->min_nr_stripes) {
7349		− printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes. Needed %lu\n",
7350		− mdname(mddev),
7351		− ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
7352		− / STRIPE_SIZE)*4);
	7483	+ pr_warn("md/raid:%s: reshape: not enough stripes. Needed %lu\n",
	7484	+ mdname(mddev),
	7485	+ ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
	7486	+ / STRIPE_SIZE)*4);
7353	7487	return 0;
7354	7488	}
7355	7489	return 1;
@@ -7430,8 +7564,8 @@ static int raid5_start_reshape(struct mddev *mddev) · 74 unmodified lines
7430	7564	*/
7431	7565	if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
7432	7566	< mddev->array_sectors) {
7433		− printk(KERN_ERR "md/raid:%s: array size must be reduced "
7434		− "before number of disks\n", mdname(mddev));
	7567	+ pr_warn("md/raid:%s: array size must be reduced before number of disks\n",
	7568	+ mdname(mddev));
7435	7569	return -EINVAL;
7436	7570	}
7437	7571
@@ -7501,7 +7635,7 @@ static int raid5_start_reshape(struct mddev *mddev) · 63 unmodified lines
7501	7635	}
7502	7636	mddev->raid_disks = conf->raid_disks;
7503	7637	mddev->reshape_position = conf->reshape_progress;
7504		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
	7638	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
7505	7639
7506	7640	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
7507	7641	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
@@ -7619,6 +7753,7 @@ static void raid5_quiesce(struct mddev *mddev, int state) · 111 unmodified lines
7619	7753	/* '2' tells resync/reshape to pause so that all
7620	7754	* active stripes can drain
7621	7755	*/
	7756	+ r5c_flush_cache(conf, INT_MAX);
7622	7757	conf->quiesce = 2;
7623	7758	wait_event_cmd(conf->wait_for_quiescent,
7624	7759	atomic_read(&conf->active_stripes) == 0 &&
@@ -7649,8 +7784,8 @@ static void raid45_takeover_raid0(struct mddev mddev, int level) · 24 unmodified lines
7649	7784
7650	7785	/* for raid0 takeover only one zone is supported */
7651	7786	if (raid0_conf->nr_strip_zones > 1) {
7652		− printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n",
7653		− mdname(mddev));
	7787	+ pr_warn("md/raid:%s: cannot takeover raid0 with more than one zone.\n",
	7788	+ mdname(mddev));
7654	7789	return ERR_PTR(-EINVAL);
7655	7790	}
7656	7791
@@ -7671,6 +7806,7 @@ static void raid45_takeover_raid0(struct mddev mddev, int level) · 14 unmodified lines
7671	7806	static void raid5_takeover_raid1(struct mddev mddev)
7672	7807	{
7673	7808	int chunksect;
	7809	+ void *ret;
7674	7810
7675	7811	if (mddev->raid_disks != 2 \|\|
7676	7812	mddev->degraded > 1)
@@ -7692,7 +7828,10 @@ static void raid5_takeover_raid1(struct mddev mddev) · 15 unmodified lines
7692	7828	mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
7693	7829	mddev->new_chunk_sectors = chunksect;
7694	7830
7695		− return setup_conf(mddev);
	7831	+ ret = setup_conf(mddev);
	7832	+ if (!IS_ERR_VALUE(ret))
	7833	+ clear_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
	7834	+ return ret;
7696	7835	}
7697	7836
7698	7837	static void raid5_takeover_raid6(struct mddev mddev)
@@ -7762,7 +7901,7 @@ static int raid5_check_reshape(struct mddev *mddev) · 63 unmodified lines
7762	7901	conf->chunk_sectors = new_chunk ;
7763	7902	mddev->chunk_sectors = new_chunk;
7764	7903	}
7765		− set_bit(MD_CHANGE_DEVS, &mddev->flags);
	7904	+ set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
7766	7905	md_wakeup_thread(mddev->thread);
7767	7906	}
7768	7907	return check_reshape(mddev);

drivers/md/raid5.h+160−12

@@ -226,6 +226,8 @@ struct stripe_head { · 225 unmodified lines
226	226
227	227	struct r5l_io_unit *log_io;
228	228	struct list_head log_list;
	229	+ sector_t log_start; /* first meta block on the journal */
	230	+ struct list_head r5c; /* for r5c_cache->stripe_in_journal */
229	231	/**
230	232	* struct stripe_operations
231	233	* @target - STRIPE_OP_COMPUTE_BLK target
@@ -264,6 +266,7 @@ struct stripe_head_state { · 32 unmodified lines
264	266	int syncing, expanding, expanded, replacing;
265	267	int locked, uptodate, to_read, to_write, failed, written;
266	268	int to_fill, compute, req_compute, non_overwrite;
	269	+ int injournal, just_cached;
267	270	int failed_num[2];
268	271	int p_failed, q_failed;
269	272	int dec_preread_active;
@@ -273,6 +276,7 @@ struct stripe_head_state { · 3 unmodified lines
273	276	struct md_rdev *blocked_rdev;
274	277	int handle_bad_blocks;
275	278	int log_failed;
	279	+ int waiting_extra_page;
276	280	};
277	281
278	282	/* Flags for struct r5dev.flags */
@@ -313,6 +317,11 @@ enum r5dev_flags { · 34 unmodified lines
313	317	*/
314	318	R5_Discard, /* Discard the stripe */
315	319	R5_SkipCopy, /* Don't copy data from bio to stripe cache */
	320	+ R5_InJournal, /* data being written is in the journal device.
	321	+ * if R5_InJournal is set for parity pd_idx, all the
	322	+ * data and parity being written are in the journal
	323	+ * device
	324	+ */
316	325	};
317	326
318	327	/*
@@ -345,7 +354,30 @@ enum { · 26 unmodified lines
345	354	STRIPE_BITMAP_PENDING, /* Being added to bitmap, don't add
346	355	* to batch yet.
347	356	*/
348		− STRIPE_LOG_TRAPPED, /* trapped into log */
	357	+ STRIPE_LOG_TRAPPED, /* trapped into log (see raid5-cache.c)
	358	+ * this bit is used in two scenarios:
	359	+ *
	360	+ * 1. write-out phase
	361	+ * set in first entry of r5l_write_stripe
	362	+ * clear in second entry of r5l_write_stripe
	363	+ * used to bypass logic in handle_stripe
	364	+ *
	365	+ * 2. caching phase
	366	+ * set in r5c_try_caching_write()
	367	+ * clear when journal write is done
	368	+ * used to initiate r5c_cache_data()
	369	+ * also used to bypass logic in handle_stripe
	370	+ */
	371	+ STRIPE_R5C_CACHING, /* the stripe is in caching phase
	372	+ * see more detail in the raid5-cache.c
	373	+ */
	374	+ STRIPE_R5C_PARTIAL_STRIPE, /* in r5c cache (to-be/being handled or
	375	+ * in conf->r5c_partial_stripe_list)
	376	+ */
	377	+ STRIPE_R5C_FULL_STRIPE, /* in r5c cache (to-be/being handled or
	378	+ * in conf->r5c_full_stripe_list)
	379	+ */
	380	+ STRIPE_R5C_PREFLUSH, /* need to flush journal device */
349	381	};
350	382
351	383	#define STRIPE_EXPAND_SYNC_FLAGS \
@@ -408,8 +440,86 @@ enum { · 56 unmodified lines
408	440
409	441	struct disk_info {
410	442	struct md_rdev rdev, replacement;
	443	+ struct page extra_page; / extra page to use in prexor */
411	444	};
412	445
	446	+/*
	447	+ * Stripe cache
	448	+ */
	449	+
	450	+#define NR_STRIPES 256
	451	+#define STRIPE_SIZE PAGE_SIZE
	452	+#define STRIPE_SHIFT (PAGE_SHIFT - 9)
	453	+#define STRIPE_SECTORS (STRIPE_SIZE>>9)
	454	+#define IO_THRESHOLD 1
	455	+#define BYPASS_THRESHOLD 1
	456	+#define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head))
	457	+#define HASH_MASK (NR_HASH - 1)
	458	+#define MAX_STRIPE_BATCH 8
	459	+
	460	+/* bio's attached to a stripe+device for I/O are linked together in bi_sector
	461	+ * order without overlap. There may be several bio's per stripe+device, and
	462	+ * a bio could span several devices.
	463	+ * When walking this list for a particular stripe+device, we must never proceed
	464	+ * beyond a bio that extends past this device, as the next bio might no longer
	465	+ * be valid.
	466	+ * This function is used to determine the 'next' bio in the list, given the
	467	+ * sector of the current stripe+device
	468	+ */
	469	+static inline struct bio r5_next_bio(struct bio bio, sector_t sector)
	470	+{
	471	+ int sectors = bio_sectors(bio);
	472	+
	473	+ if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS)
	474	+ return bio->bi_next;
	475	+ else
	476	+ return NULL;
	477	+}
	478	+
	479	+/*
	480	+ * We maintain a biased count of active stripes in the bottom 16 bits of
	481	+ * bi_phys_segments, and a count of processed stripes in the upper 16 bits
	482	+ */
	483	+static inline int raid5_bi_processed_stripes(struct bio *bio)
	484	+{
	485	+ atomic_t segments = (atomic_t )&bio->bi_phys_segments;
	486	+
	487	+ return (atomic_read(segments) >> 16) & 0xffff;
	488	+}
	489	+
	490	+static inline int raid5_dec_bi_active_stripes(struct bio *bio)
	491	+{
	492	+ atomic_t segments = (atomic_t )&bio->bi_phys_segments;
	493	+
	494	+ return atomic_sub_return(1, segments) & 0xffff;
	495	+}
	496	+
	497	+static inline void raid5_inc_bi_active_stripes(struct bio *bio)
	498	+{
	499	+ atomic_t segments = (atomic_t )&bio->bi_phys_segments;
	500	+
	501	+ atomic_inc(segments);
	502	+}
	503	+
	504	+static inline void raid5_set_bi_processed_stripes(struct bio *bio,
	505	+ unsigned int cnt)
	506	+{
	507	+ atomic_t segments = (atomic_t )&bio->bi_phys_segments;
	508	+ int old, new;
	509	+
	510	+ do {
	511	+ old = atomic_read(segments);
	512	+ new = (old & 0xffff) \| (cnt << 16);
	513	+ } while (atomic_cmpxchg(segments, old, new) != old);
	514	+}
	515	+
	516	+static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt)
	517	+{
	518	+ atomic_t segments = (atomic_t )&bio->bi_phys_segments;
	519	+
	520	+ atomic_set(segments, cnt);
	521	+}
	522	+
413	523	/* NOTE NR_STRIPE_HASH_LOCKS must remain below 64.
414	524	* This is because we sometimes take all the spinlocks
415	525	* and creating that much locking depth can cause
@@ -432,6 +542,30 @@ struct r5worker_group { · 16 unmodified lines
432	542	int stripes_cnt;
433	543	};
434	544
	545	+enum r5_cache_state {
	546	+ R5_INACTIVE_BLOCKED, /* release of inactive stripes blocked,
	547	+ * waiting for 25% to be free
	548	+ */
	549	+ R5_ALLOC_MORE, /* It might help to allocate another
	550	+ * stripe.
	551	+ */
	552	+ R5_DID_ALLOC, /* A stripe was allocated, don't allocate
	553	+ * more until at least one has been
	554	+ * released. This avoids flooding
	555	+ * the cache.
	556	+ */
	557	+ R5C_LOG_TIGHT, /* log device space tight, need to
	558	+ * prioritize stripes at last_checkpoint
	559	+ */
	560	+ R5C_LOG_CRITICAL, /* log device is running out of space,
	561	+ * only process stripes that are already
	562	+ * occupying the log
	563	+ */
	564	+ R5C_EXTRA_PAGE_IN_USE, /* a stripe is using disk_info.extra_page
	565	+ * for prexor
	566	+ */
	567	+};
	568	+
435	569	struct r5conf {
436	570	struct hlist_head *stripe_hashtbl;
437	571	/* only protect corresponding hash list and inactive_list */
@@ -519,23 +653,18 @@ struct r5conf { · 81 unmodified lines
519	653	*/
520	654	atomic_t active_stripes;
521	655	struct list_head inactive_list[NR_STRIPE_HASH_LOCKS];
	656	+
	657	+ atomic_t r5c_cached_full_stripes;
	658	+ struct list_head r5c_full_stripe_list;
	659	+ atomic_t r5c_cached_partial_stripes;
	660	+ struct list_head r5c_partial_stripe_list;
	661	+
522	662	atomic_t empty_inactive_list_nr;
523	663	struct llist_head released_stripes;
524	664	wait_queue_head_t wait_for_quiescent;
525	665	wait_queue_head_t wait_for_stripe;
526	666	wait_queue_head_t wait_for_overlap;
527	667	unsigned long cache_state;
528		−#define R5_INACTIVE_BLOCKED 1 /* release of inactive stripes blocked,
529		− * waiting for 25% to be free
530		− */
531		−#define R5_ALLOC_MORE 2 /* It might help to allocate another
532		− * stripe.
533		− */
534		−#define R5_DID_ALLOC 4 /* A stripe was allocated, don't allocate
535		− * more until at least one has been
536		− * released. This avoids flooding
537		− * the cache.
538		− */
539	668	struct shrinker shrinker;
540	669	int pool_size; /* number of disks in stripeheads in pool */
541	670	spinlock_t device_lock;
@@ -633,4 +762,23 @@ extern void r5l_stripe_write_finished(struct stripe_head *sh); · 91 unmodified lines
633	762	extern int r5l_handle_flush_request(struct r5l_log log, struct bio bio);
634	763	extern void r5l_quiesce(struct r5l_log *log, int state);
635	764	extern bool r5l_log_disk_error(struct r5conf *conf);
	765	+extern bool r5c_is_writeback(struct r5l_log *log);
	766	+extern int
	767	+r5c_try_caching_write(struct r5conf conf, struct stripe_head sh,
	768	+ struct stripe_head_state *s, int disks);
	769	+extern void
	770	+r5c_finish_stripe_write_out(struct r5conf conf, struct stripe_head sh,
	771	+ struct stripe_head_state *s);
	772	+extern void r5c_release_extra_page(struct stripe_head *sh);
	773	+extern void r5c_use_extra_page(struct stripe_head *sh);
	774	+extern void r5l_wake_reclaim(struct r5l_log *log, sector_t space);
	775	+extern void r5c_handle_cached_data_endio(struct r5conf *conf,
	776	+ struct stripe_head sh, int disks, struct bio_list return_bi);
	777	+extern int r5c_cache_data(struct r5l_log log, struct stripe_head sh,
	778	+ struct stripe_head_state *s);
	779	+extern void r5c_make_stripe_write_out(struct stripe_head *sh);
	780	+extern void r5c_flush_cache(struct r5conf *conf, int num);
	781	+extern void r5c_check_stripe_cache_usage(struct r5conf *conf);
	782	+extern void r5c_check_cached_full_stripe(struct r5conf *conf);
	783	+extern struct md_sysfs_entry r5c_journal_mode;
636	784	#endif

include/uapi/linux/raid/md_p.h+6−1

@@ -84,6 +84,10 @@ · 83 unmodified lines
84	84	#define MD_DISK_CANDIDATE 5 /* disk is added as spare (local) until confirmed
85	85	* For clustered enviroments only.
86	86	*/
	87	+#define MD_DISK_FAILFAST 10 /* Send REQ_FAILFAST if there are multiple
	88	+ * devices available - and don't try to
	89	+ * correct read errors.
	90	+ */
87	91
88	92	#define MD_DISK_WRITEMOSTLY 9 /* disk is "write-mostly" is RAID1 config.
89	93	* read requests will only be sent here in
@@ -265,8 +269,9 @@ struct mdp_superblock_1 { · 175 unmodified lines
265	269	__le32 dev_number; /* permanent identifier of this device - not role in raid */
266	270	__le32 cnt_corrected_read; /* number of read errors that were corrected by re-writing */
267	271	__u8 device_uuid[16]; /* user-space setable, ignored by kernel */
268		− __u8 devflags; /* per-device flags. Only one defined...*/
	272	+ __u8 devflags; /* per-device flags. Only two defined...*/
269	273	#define WriteMostly1 1 /* mask for writemostly flag in above */
	274	+#define FailFast1 2 /* Should avoid retries and fixups and just fail */
270	275	/* Bad block log. If there are any bad blocks the feature flag is set.
271	276	* If offset and size are non-zero, that space is reserved and available
272	277	*/

lib/raid6/avx2.c+229−3

@@ -87,9 +87,57 @@ static void raid6_avx21_gen_syndrome(int disks, size_t bytes, void **ptrs) · 86 unmodified lines
87	87	kernel_fpu_end();
88	88	}
89	89
	90	+static void raid6_avx21_xor_syndrome(int disks, int start, int stop,
	91	+ size_t bytes, void **ptrs)
	92	+{
	93	+ u8 dptr = (u8 )ptrs;
	94	+ u8 p, q;
	95	+ int d, z, z0;
	96	+
	97	+ z0 = stop; /* P/Q right side optimization */
	98	+ p = dptr[disks-2]; /* XOR parity */
	99	+ q = dptr[disks-1]; /* RS syndrome */
	100	+
	101	+ kernel_fpu_begin();
	102	+
	103	+ asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));
	104	+
	105	+ for (d = 0 ; d < bytes ; d += 32) {
	106	+ asm volatile("vmovdqa %0,%%ymm4" :: "m" (dptr[z0][d]));
	107	+ asm volatile("vmovdqa %0,%%ymm2" : : "m" (p[d]));
	108	+ asm volatile("vpxor %ymm4,%ymm2,%ymm2");
	109	+ /* P/Q data pages */
	110	+ for (z = z0-1 ; z >= start ; z--) {
	111	+ asm volatile("vpxor %ymm5,%ymm5,%ymm5");
	112	+ asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
	113	+ asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
	114	+ asm volatile("vpand %ymm0,%ymm5,%ymm5");
	115	+ asm volatile("vpxor %ymm5,%ymm4,%ymm4");
	116	+ asm volatile("vmovdqa %0,%%ymm5" :: "m" (dptr[z][d]));
	117	+ asm volatile("vpxor %ymm5,%ymm2,%ymm2");
	118	+ asm volatile("vpxor %ymm5,%ymm4,%ymm4");
	119	+ }
	120	+ /* P/Q left side optimization */
	121	+ for (z = start-1 ; z >= 0 ; z--) {
	122	+ asm volatile("vpxor %ymm5,%ymm5,%ymm5");
	123	+ asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
	124	+ asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
	125	+ asm volatile("vpand %ymm0,%ymm5,%ymm5");
	126	+ asm volatile("vpxor %ymm5,%ymm4,%ymm4");
	127	+ }
	128	+ asm volatile("vpxor %0,%%ymm4,%%ymm4" : : "m" (q[d]));
	129	+ /* Don't use movntdq for r/w memory area < cache line */
	130	+ asm volatile("vmovdqa %%ymm4,%0" : "=m" (q[d]));
	131	+ asm volatile("vmovdqa %%ymm2,%0" : "=m" (p[d]));
	132	+ }
	133	+
	134	+ asm volatile("sfence" : : : "memory");
	135	+ kernel_fpu_end();
	136	+}
	137	+
90	138	const struct raid6_calls raid6_avx2x1 = {
91	139	raid6_avx21_gen_syndrome,
92		− NULL, /* XOR not yet implemented */
	140	+ raid6_avx21_xor_syndrome,
93	141	raid6_have_avx2,
94	142	"avx2x1",
95	143	1 /* Has cache hints */
@@ -149,9 +197,77 @@ static void raid6_avx22_gen_syndrome(int disks, size_t bytes, void **ptrs) · 53 unmodified lines
149	197	kernel_fpu_end();
150	198	}
151	199
	200	+static void raid6_avx22_xor_syndrome(int disks, int start, int stop,
	201	+ size_t bytes, void **ptrs)
	202	+{
	203	+ u8 dptr = (u8 )ptrs;
	204	+ u8 p, q;
	205	+ int d, z, z0;
	206	+
	207	+ z0 = stop; /* P/Q right side optimization */
	208	+ p = dptr[disks-2]; /* XOR parity */
	209	+ q = dptr[disks-1]; /* RS syndrome */
	210	+
	211	+ kernel_fpu_begin();
	212	+
	213	+ asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));
	214	+
	215	+ for (d = 0 ; d < bytes ; d += 64) {
	216	+ asm volatile("vmovdqa %0,%%ymm4" :: "m" (dptr[z0][d]));
	217	+ asm volatile("vmovdqa %0,%%ymm6" :: "m" (dptr[z0][d+32]));
	218	+ asm volatile("vmovdqa %0,%%ymm2" : : "m" (p[d]));
	219	+ asm volatile("vmovdqa %0,%%ymm3" : : "m" (p[d+32]));
	220	+ asm volatile("vpxor %ymm4,%ymm2,%ymm2");
	221	+ asm volatile("vpxor %ymm6,%ymm3,%ymm3");
	222	+ /* P/Q data pages */
	223	+ for (z = z0-1 ; z >= start ; z--) {
	224	+ asm volatile("vpxor %ymm5,%ymm5,%ymm5");
	225	+ asm volatile("vpxor %ymm7,%ymm7,%ymm7");
	226	+ asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
	227	+ asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");
	228	+ asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
	229	+ asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
	230	+ asm volatile("vpand %ymm0,%ymm5,%ymm5");
	231	+ asm volatile("vpand %ymm0,%ymm7,%ymm7");
	232	+ asm volatile("vpxor %ymm5,%ymm4,%ymm4");
	233	+ asm volatile("vpxor %ymm7,%ymm6,%ymm6");
	234	+ asm volatile("vmovdqa %0,%%ymm5" :: "m" (dptr[z][d]));
	235	+ asm volatile("vmovdqa %0,%%ymm7"
	236	+ :: "m" (dptr[z][d+32]));
	237	+ asm volatile("vpxor %ymm5,%ymm2,%ymm2");
	238	+ asm volatile("vpxor %ymm7,%ymm3,%ymm3");
	239	+ asm volatile("vpxor %ymm5,%ymm4,%ymm4");
	240	+ asm volatile("vpxor %ymm7,%ymm6,%ymm6");
	241	+ }
	242	+ /* P/Q left side optimization */
	243	+ for (z = start-1 ; z >= 0 ; z--) {
	244	+ asm volatile("vpxor %ymm5,%ymm5,%ymm5");
	245	+ asm volatile("vpxor %ymm7,%ymm7,%ymm7");
	246	+ asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
	247	+ asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");
	248	+ asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
	249	+ asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
	250	+ asm volatile("vpand %ymm0,%ymm5,%ymm5");
	251	+ asm volatile("vpand %ymm0,%ymm7,%ymm7");
	252	+ asm volatile("vpxor %ymm5,%ymm4,%ymm4");
	253	+ asm volatile("vpxor %ymm7,%ymm6,%ymm6");
	254	+ }
	255	+ asm volatile("vpxor %0,%%ymm4,%%ymm4" : : "m" (q[d]));
	256	+ asm volatile("vpxor %0,%%ymm6,%%ymm6" : : "m" (q[d+32]));
	257	+ /* Don't use movntdq for r/w memory area < cache line */
	258	+ asm volatile("vmovdqa %%ymm4,%0" : "=m" (q[d]));
	259	+ asm volatile("vmovdqa %%ymm6,%0" : "=m" (q[d+32]));
	260	+ asm volatile("vmovdqa %%ymm2,%0" : "=m" (p[d]));
	261	+ asm volatile("vmovdqa %%ymm3,%0" : "=m" (p[d+32]));
	262	+ }
	263	+
	264	+ asm volatile("sfence" : : : "memory");
	265	+ kernel_fpu_end();
	266	+}
	267	+
152	268	const struct raid6_calls raid6_avx2x2 = {
153	269	raid6_avx22_gen_syndrome,
154		− NULL, /* XOR not yet implemented */
	270	+ raid6_avx22_xor_syndrome,
155	271	raid6_have_avx2,
156	272	"avx2x2",
157	273	1 /* Has cache hints */
@@ -242,9 +358,119 @@ static void raid6_avx24_gen_syndrome(int disks, size_t bytes, void **ptrs) · 84 unmodified lines
242	358	kernel_fpu_end();
243	359	}
244	360
	361	+static void raid6_avx24_xor_syndrome(int disks, int start, int stop,
	362	+ size_t bytes, void **ptrs)
	363	+{
	364	+ u8 dptr = (u8 )ptrs;
	365	+ u8 p, q;
	366	+ int d, z, z0;
	367	+
	368	+ z0 = stop; /* P/Q right side optimization */
	369	+ p = dptr[disks-2]; /* XOR parity */
	370	+ q = dptr[disks-1]; /* RS syndrome */
	371	+
	372	+ kernel_fpu_begin();
	373	+
	374	+ asm volatile("vmovdqa %0,%%ymm0" :: "m" (raid6_avx2_constants.x1d[0]));
	375	+
	376	+ for (d = 0 ; d < bytes ; d += 128) {
	377	+ asm volatile("vmovdqa %0,%%ymm4" :: "m" (dptr[z0][d]));
	378	+ asm volatile("vmovdqa %0,%%ymm6" :: "m" (dptr[z0][d+32]));
	379	+ asm volatile("vmovdqa %0,%%ymm12" :: "m" (dptr[z0][d+64]));
	380	+ asm volatile("vmovdqa %0,%%ymm14" :: "m" (dptr[z0][d+96]));
	381	+ asm volatile("vmovdqa %0,%%ymm2" : : "m" (p[d]));
	382	+ asm volatile("vmovdqa %0,%%ymm3" : : "m" (p[d+32]));
	383	+ asm volatile("vmovdqa %0,%%ymm10" : : "m" (p[d+64]));
	384	+ asm volatile("vmovdqa %0,%%ymm11" : : "m" (p[d+96]));
	385	+ asm volatile("vpxor %ymm4,%ymm2,%ymm2");
	386	+ asm volatile("vpxor %ymm6,%ymm3,%ymm3");
	387	+ asm volatile("vpxor %ymm12,%ymm10,%ymm10");
	388	+ asm volatile("vpxor %ymm14,%ymm11,%ymm11");
	389	+ /* P/Q data pages */
	390	+ for (z = z0-1 ; z >= start ; z--) {
	391	+ asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
	392	+ asm volatile("prefetchnta %0" :: "m" (dptr[z][d+64]));
	393	+ asm volatile("vpxor %ymm5,%ymm5,%ymm5");
	394	+ asm volatile("vpxor %ymm7,%ymm7,%ymm7");
	395	+ asm volatile("vpxor %ymm13,%ymm13,%ymm13");
	396	+ asm volatile("vpxor %ymm15,%ymm15,%ymm15");
	397	+ asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
	398	+ asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");
	399	+ asm volatile("vpcmpgtb %ymm12,%ymm13,%ymm13");
	400	+ asm volatile("vpcmpgtb %ymm14,%ymm15,%ymm15");
	401	+ asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
	402	+ asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
	403	+ asm volatile("vpaddb %ymm12,%ymm12,%ymm12");
	404	+ asm volatile("vpaddb %ymm14,%ymm14,%ymm14");
	405	+ asm volatile("vpand %ymm0,%ymm5,%ymm5");
	406	+ asm volatile("vpand %ymm0,%ymm7,%ymm7");
	407	+ asm volatile("vpand %ymm0,%ymm13,%ymm13");
	408	+ asm volatile("vpand %ymm0,%ymm15,%ymm15");
	409	+ asm volatile("vpxor %ymm5,%ymm4,%ymm4");
	410	+ asm volatile("vpxor %ymm7,%ymm6,%ymm6");
	411	+ asm volatile("vpxor %ymm13,%ymm12,%ymm12");
	412	+ asm volatile("vpxor %ymm15,%ymm14,%ymm14");
	413	+ asm volatile("vmovdqa %0,%%ymm5" :: "m" (dptr[z][d]));
	414	+ asm volatile("vmovdqa %0,%%ymm7"
	415	+ :: "m" (dptr[z][d+32]));
	416	+ asm volatile("vmovdqa %0,%%ymm13"
	417	+ :: "m" (dptr[z][d+64]));
	418	+ asm volatile("vmovdqa %0,%%ymm15"
	419	+ :: "m" (dptr[z][d+96]));
	420	+ asm volatile("vpxor %ymm5,%ymm2,%ymm2");
	421	+ asm volatile("vpxor %ymm7,%ymm3,%ymm3");
	422	+ asm volatile("vpxor %ymm13,%ymm10,%ymm10");
	423	+ asm volatile("vpxor %ymm15,%ymm11,%ymm11");
	424	+ asm volatile("vpxor %ymm5,%ymm4,%ymm4");
	425	+ asm volatile("vpxor %ymm7,%ymm6,%ymm6");
	426	+ asm volatile("vpxor %ymm13,%ymm12,%ymm12");
	427	+ asm volatile("vpxor %ymm15,%ymm14,%ymm14");
	428	+ }
	429	+ asm volatile("prefetchnta %0" :: "m" (q[d]));
	430	+ asm volatile("prefetchnta %0" :: "m" (q[d+64]));
	431	+ /* P/Q left side optimization */
	432	+ for (z = start-1 ; z >= 0 ; z--) {
	433	+ asm volatile("vpxor %ymm5,%ymm5,%ymm5");
	434	+ asm volatile("vpxor %ymm7,%ymm7,%ymm7");
	435	+ asm volatile("vpxor %ymm13,%ymm13,%ymm13");
	436	+ asm volatile("vpxor %ymm15,%ymm15,%ymm15");
	437	+ asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
	438	+ asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");
	439	+ asm volatile("vpcmpgtb %ymm12,%ymm13,%ymm13");
	440	+ asm volatile("vpcmpgtb %ymm14,%ymm15,%ymm15");
	441	+ asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
	442	+ asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
	443	+ asm volatile("vpaddb %ymm12,%ymm12,%ymm12");
	444	+ asm volatile("vpaddb %ymm14,%ymm14,%ymm14");
	445	+ asm volatile("vpand %ymm0,%ymm5,%ymm5");
	446	+ asm volatile("vpand %ymm0,%ymm7,%ymm7");
	447	+ asm volatile("vpand %ymm0,%ymm13,%ymm13");
	448	+ asm volatile("vpand %ymm0,%ymm15,%ymm15");
	449	+ asm volatile("vpxor %ymm5,%ymm4,%ymm4");
	450	+ asm volatile("vpxor %ymm7,%ymm6,%ymm6");
	451	+ asm volatile("vpxor %ymm13,%ymm12,%ymm12");
	452	+ asm volatile("vpxor %ymm15,%ymm14,%ymm14");
	453	+ }
	454	+ asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));
	455	+ asm volatile("vmovntdq %%ymm3,%0" : "=m" (p[d+32]));
	456	+ asm volatile("vmovntdq %%ymm10,%0" : "=m" (p[d+64]));
	457	+ asm volatile("vmovntdq %%ymm11,%0" : "=m" (p[d+96]));
	458	+ asm volatile("vpxor %0,%%ymm4,%%ymm4" : : "m" (q[d]));
	459	+ asm volatile("vpxor %0,%%ymm6,%%ymm6" : : "m" (q[d+32]));
	460	+ asm volatile("vpxor %0,%%ymm12,%%ymm12" : : "m" (q[d+64]));
	461	+ asm volatile("vpxor %0,%%ymm14,%%ymm14" : : "m" (q[d+96]));
	462	+ asm volatile("vmovntdq %%ymm4,%0" : "=m" (q[d]));
	463	+ asm volatile("vmovntdq %%ymm6,%0" : "=m" (q[d+32]));
	464	+ asm volatile("vmovntdq %%ymm12,%0" : "=m" (q[d+64]));
	465	+ asm volatile("vmovntdq %%ymm14,%0" : "=m" (q[d+96]));
	466	+ }
	467	+ asm volatile("sfence" : : : "memory");
	468	+ kernel_fpu_end();
	469	+}
	470	+
245	471	const struct raid6_calls raid6_avx2x4 = {
246	472	raid6_avx24_gen_syndrome,
247		− NULL, /* XOR not yet implemented */
	473	+ raid6_avx24_xor_syndrome,
248	474	raid6_have_avx2,
249	475	"avx2x4",
250	476	1 /* Has cache hints */