Merge branch 'md-next' into md-linus

Shaohua Li committed 9 years ago

Commit e265eb32 parents 85724ed b50633526 files · +3582 −1483Browse files

Unified · Split

26 files

Documentation/admin-guide/md.rst +29 −3
Documentation/md/md-cluster.txt +1 −1
Documentation/md/raid5-ppl.txt +44 −0
block/bio.c +13 −48
drivers/md/Makefile +1 −1
drivers/md/bitmap.c +48 −11
drivers/md/bitmap.h +3 −0
drivers/md/linear.c +35 −40
drivers/md/md-cluster.c +207 −16
drivers/md/md-cluster.h +1 −0
drivers/md/md.c +320 −94
drivers/md/md.h +67 −4
drivers/md/raid0.c +40 −38
drivers/md/raid1.c +289 −390
drivers/md/raid1.h +10 −3
drivers/md/raid10.c +306 −430
drivers/md/raid10.h +1 −0
drivers/md/raid5-cache.c +285 −77
drivers/md/raid5-log.h +115 −0
drivers/md/raid5-ppl.c +1271 −0
drivers/md/raid5.c +409 −234
drivers/md/raid5.h +26 −80
include/linux/bio.h +2 −9
include/linux/percpu-refcount.h +1 −0
include/uapi/linux/raid/md_p.h +41 −4
lib/percpu-refcount.c +17 −0

@@ -276,14 +276,14 @@ All md devices contain: · 275 unmodified lines
276	276	array creation it will default to 0, though starting the array as
277	277	``clean`` will set it much larger.
278	278
279		− new_dev
	279	+ new_dev
280	280	This file can be written but not read. The value written should
281	281	be a block device number as major:minor. e.g. 8:0
282	282	This will cause that device to be attached to the array, if it is
283	283	available. It will then appear at md/dev-XXX (depending on the
284	284	name of the device) and further configuration is then possible.
285	285
286		− safe_mode_delay
	286	+ safe_mode_delay
287	287	When an md array has seen no write requests for a certain period
288	288	of time, it will be marked as ``clean``. When another write
289	289	request arrives, the array is marked as ``dirty`` before the write
@@ -292,7 +292,7 @@ All md devices contain: · 2 unmodified lines
292	292	period as a number of seconds. The default is 200msec (0.200).
293	293	Writing a value of 0 disables safemode.
294	294
295		− array_state
	295	+ array_state
296	296	This file contains a single word which describes the current
297	297	state of the array. In many cases, the state can be set by
298	298	writing the word for the desired state, however some states
@@ -401,7 +401,30 @@ All md devices contain: · 102 unmodified lines
401	401	once the array becomes non-degraded, and this fact has been
402	402	recorded in the metadata.
403	403
	404	+ consistency_policy
	405	+ This indicates how the array maintains consistency in case of unexpected
	406	+ shutdown. It can be:
404	407
	408	+ none
	409	+ Array has no redundancy information, e.g. raid0, linear.
	410	+
	411	+ resync
	412	+ Full resync is performed and all redundancy is regenerated when the
	413	+ array is started after unclean shutdown.
	414	+
	415	+ bitmap
	416	+ Resync assisted by a write-intent bitmap.
	417	+
	418	+ journal
	419	+ For raid4/5/6, journal device is used to log transactions and replay
	420	+ after unclean shutdown.
	421	+
	422	+ ppl
	423	+ For raid5 only, Partial Parity Log is used to close the write hole and
	424	+ eliminate resync.
	425	+
	426	+ The accepted values when writing to this file are ``ppl`` and ``resync``,
	427	+ used to enable and disable PPL.
405	428
406	429
407	430	As component devices are added to an md array, they appear in the ``md``
@@ -563,6 +586,9 @@ Each directory contains: · 155 unmodified lines
563	586	adds bad blocks without acknowledging them. This is largely
564	587	for testing.
565	588
	589	+ ppl_sector, ppl_size
	590	+ Location and size (in sectors) of the space used for Partial Parity Log
	591	+ on this device.
566	592
567	593
568	594	An active md device will also contain an entry for each active device

Documentation/md/md-cluster.txt+1−1

@@ -321,4 +321,4 @@ The algorithm is: · 320 unmodified lines
321	321
322	322	There are somethings which are not supported by cluster MD yet.
323	323
324		−- update size and change array_sectors.
	324	+- change array_sectors.

Documentation/md/raid5-ppl.txt+44−0

@@ -0,0 +1,44 @@
	1	+Partial Parity Log
	2	+
	3	+Partial Parity Log (PPL) is a feature available for RAID5 arrays. The issue
	4	+addressed by PPL is that after a dirty shutdown, parity of a particular stripe
	5	+may become inconsistent with data on other member disks. If the array is also
	6	+in degraded state, there is no way to recalculate parity, because one of the
	7	+disks is missing. This can lead to silent data corruption when rebuilding the
	8	+array or using it is as degraded - data calculated from parity for array blocks
	9	+that have not been touched by a write request during the unclean shutdown can
	10	+be incorrect. Such condition is known as the RAID5 Write Hole. Because of
	11	+this, md by default does not allow starting a dirty degraded array.
	12	+
	13	+Partial parity for a write operation is the XOR of stripe data chunks not
	14	+modified by this write. It is just enough data needed for recovering from the
	15	+write hole. XORing partial parity with the modified chunks produces parity for
	16	+the stripe, consistent with its state before the write operation, regardless of
	17	+which chunk writes have completed. If one of the not modified data disks of
	18	+this stripe is missing, this updated parity can be used to recover its
	19	+contents. PPL recovery is also performed when starting an array after an
	20	+unclean shutdown and all disks are available, eliminating the need to resync
	21	+the array. Because of this, using write-intent bitmap and PPL together is not
	22	+supported.
	23	+
	24	+When handling a write request PPL writes partial parity before new data and
	25	+parity are dispatched to disks. PPL is a distributed log - it is stored on
	26	+array member drives in the metadata area, on the parity drive of a particular
	27	+stripe. It does not require a dedicated journaling drive. Write performance is
	28	+reduced by up to 30%-40% but it scales with the number of drives in the array
	29	+and the journaling drive does not become a bottleneck or a single point of
	30	+failure.
	31	+
	32	+Unlike raid5-cache, the other solution in md for closing the write hole, PPL is
	33	+not a true journal. It does not protect from losing in-flight data, only from
	34	+silent data corruption. If a dirty disk of a stripe is lost, no PPL recovery is
	35	+performed for this stripe (parity is not updated). So it is possible to have
	36	+arbitrary data in the written part of a stripe if that disk is lost. In such
	37	+case the behavior is the same as in plain raid5.
	38	+
	39	+PPL is available for md version-1 metadata and external (specifically IMSM)
	40	+metadata arrays. It can be enabled using mdadm option --consistency-policy=ppl.
	41	+
	42	+Currently, volatile write-back cache should be disabled on all member drives
	43	+when using PPL. Otherwise it cannot guarantee consistency in case of power
	44	+failure.

block/bio.c+13−48

@@ -633,20 +633,21 @@ struct bio bio_clone_fast(struct bio bio, gfp_t gfp_mask, struct bio_set *bs) · 632 unmodified lines
633	633	}
634	634	EXPORT_SYMBOL(bio_clone_fast);
635	635
636		−static struct bio __bio_clone_bioset(struct bio bio_src, gfp_t gfp_mask,
637		− struct bio_set *bs, int offset,
638		− int size)
	636	+/**
	637	+ * bio_clone_bioset - clone a bio
	638	+ * @bio_src: bio to clone
	639	+ * @gfp_mask: allocation priority
	640	+ * @bs: bio_set to allocate from
	641	+ *
	642	+ * Clone bio. Caller will own the returned bio, but not the actual data it
	643	+ * points to. Reference count of returned bio will be one.
	644	+ */
	645	+struct bio bio_clone_bioset(struct bio bio_src, gfp_t gfp_mask,
	646	+ struct bio_set *bs)
639	647	{
640	648	struct bvec_iter iter;
641	649	struct bio_vec bv;
642	650	struct bio *bio;
643		− struct bvec_iter iter_src = bio_src->bi_iter;
644		−
645		− /* for supporting partial clone */
646		− if (offset \|\| size != bio_src->bi_iter.bi_size) {
647		− bio_advance_iter(bio_src, &iter_src, offset);
648		− iter_src.bi_size = size;
649		− }
650	651
651	652	/*
652	653	* Pre immutable biovecs, __bio_clone() used to just do a memcpy from
@@ -670,8 +671,7 @@ static struct bio __bio_clone_bioset(struct bio bio_src, gfp_t gfp_mask, · 17 unmodified lines
670	671	* __bio_clone_fast() anyways.
671	672	*/
672	673
673		− bio = bio_alloc_bioset(gfp_mask, __bio_segments(bio_src,
674		− &iter_src), bs);
	674	+ bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs);
675	675	if (!bio)
676	676	return NULL;
677	677	bio->bi_bdev = bio_src->bi_bdev;
@@ -688,7 +688,7 @@ static struct bio __bio_clone_bioset(struct bio bio_src, gfp_t gfp_mask, · 10 unmodified lines
688	688	bio->bi_io_vec[bio->bi_vcnt++] = bio_src->bi_io_vec[0];
689	689	break;
690	690	default:
691		− __bio_for_each_segment(bv, bio_src, iter, iter_src)
	691	+ bio_for_each_segment(bv, bio_src, iter)
692	692	bio->bi_io_vec[bio->bi_vcnt++] = bv;
693	693	break;
694	694	}
@@ -707,43 +707,8 @@ static struct bio __bio_clone_bioset(struct bio bio_src, gfp_t gfp_mask, · 12 unmodified lines
707	707
708	708	return bio;
709	709	}
710		−
711		−/**
712		− * bio_clone_bioset - clone a bio
713		− * @bio_src: bio to clone
714		− * @gfp_mask: allocation priority
715		− * @bs: bio_set to allocate from
716		− *
717		− * Clone bio. Caller will own the returned bio, but not the actual data it
718		− * points to. Reference count of returned bio will be one.
719		− */
720		−struct bio bio_clone_bioset(struct bio bio_src, gfp_t gfp_mask,
721		− struct bio_set *bs)
722		−{
723		− return __bio_clone_bioset(bio_src, gfp_mask, bs, 0,
724		− bio_src->bi_iter.bi_size);
725		−}
726	710	EXPORT_SYMBOL(bio_clone_bioset);
727	711
728		−/**
729		− * bio_clone_bioset_partial - clone a partial bio
730		− * @bio_src: bio to clone
731		− * @gfp_mask: allocation priority
732		− * @bs: bio_set to allocate from
733		− * @offset: cloned starting from the offset
734		− * @size: size for the cloned bio
735		− *
736		− * Clone bio. Caller will own the returned bio, but not the actual data it
737		− * points to. Reference count of returned bio will be one.
738		− */
739		−struct bio bio_clone_bioset_partial(struct bio bio_src, gfp_t gfp_mask,
740		− struct bio_set *bs, int offset,
741		− int size)
742		−{
743		− return __bio_clone_bioset(bio_src, gfp_mask, bs, offset, size);
744		−}
745		−EXPORT_SYMBOL(bio_clone_bioset_partial);
746		−
747	712	/**
748	713	* bio_add_pc_page - attempt to add page to bio
749	714	* @q: the target queue

drivers/md/Makefile+1−1

@@ -18,7 +18,7 @@ dm-cache-cleaner-y += dm-cache-policy-cleaner.o · 17 unmodified lines
18	18	dm-era-y += dm-era-target.o
19	19	dm-verity-y += dm-verity-target.o
20	20	md-mod-y += md.o bitmap.o
21		−raid456-y += raid5.o raid5-cache.o
	21	+raid456-y += raid5.o raid5-cache.o raid5-ppl.o
22	22
23	23	# Note: link order is important. All raid personalities
24	24	# and must come before md.o, as they each initialise

drivers/md/bitmap.c+48−11

@@ -471,6 +471,7 @@ void bitmap_update_sb(struct bitmap *bitmap) · 470 unmodified lines
471	471	kunmap_atomic(sb);
472	472	write_page(bitmap, bitmap->storage.sb_page, 1);
473	473	}
	474	+EXPORT_SYMBOL(bitmap_update_sb);
474	475
475	476	/* print out the bitmap file superblock */
476	477	void bitmap_print_sb(struct bitmap *bitmap)
@@ -696,7 +697,7 @@ re_read: · 219 unmodified lines
696	697
697	698	out:
698	699	kunmap_atomic(sb);
699		− /* Assiging chunksize is required for "re_read" */
	700	+ /* Assigning chunksize is required for "re_read" */
700	701	bitmap->mddev->bitmap_info.chunksize = chunksize;
701	702	if (err == 0 && nodes && (bitmap->cluster_slot < 0)) {
702	703	err = md_setup_cluster(bitmap->mddev, nodes);
@@ -1727,7 +1728,7 @@ void bitmap_flush(struct mddev *mddev) · 1024 unmodified lines
1727	1728	/*
1728	1729	* free memory that was allocated
1729	1730	*/
1730		−static void bitmap_free(struct bitmap *bitmap)
	1731	+void bitmap_free(struct bitmap *bitmap)
1731	1732	{
1732	1733	unsigned long k, pages;
1733	1734	struct bitmap_page *bp;
@@ -1761,6 +1762,21 @@ static void bitmap_free(struct bitmap *bitmap) · 27 unmodified lines
1761	1762	kfree(bp);
1762	1763	kfree(bitmap);
1763	1764	}
	1765	+EXPORT_SYMBOL(bitmap_free);
	1766	+
	1767	+void bitmap_wait_behind_writes(struct mddev *mddev)
	1768	+{
	1769	+ struct bitmap *bitmap = mddev->bitmap;
	1770	+
	1771	+ /* wait for behind writes to complete */
	1772	+ if (bitmap && atomic_read(&bitmap->behind_writes) > 0) {
	1773	+ pr_debug("md:%s: behind writes in progress - waiting to stop.\n",
	1774	+ mdname(mddev));
	1775	+ /* need to kick something here to make sure I/O goes? */
	1776	+ wait_event(bitmap->behind_wait,
	1777	+ atomic_read(&bitmap->behind_writes) == 0);
	1778	+ }
	1779	+}
1764	1780
1765	1781	void bitmap_destroy(struct mddev *mddev)
1766	1782	{
@@ -1769,6 +1785,8 @@ void bitmap_destroy(struct mddev *mddev) · 2 unmodified lines
1769	1785	if (!bitmap) /* there was no bitmap */
1770	1786	return;
1771	1787
	1788	+ bitmap_wait_behind_writes(mddev);
	1789	+
1772	1790	mutex_lock(&mddev->bitmap_info.mutex);
1773	1791	spin_lock(&mddev->lock);
1774	1792	mddev->bitmap = NULL; /* disconnect from the md device */
@@ -1920,6 +1938,27 @@ out: · 145 unmodified lines
1920	1938	}
1921	1939	EXPORT_SYMBOL_GPL(bitmap_load);
1922	1940
	1941	+struct bitmap get_bitmap_from_slot(struct mddev mddev, int slot)
	1942	+{
	1943	+ int rv = 0;
	1944	+ struct bitmap *bitmap;
	1945	+
	1946	+ bitmap = bitmap_create(mddev, slot);
	1947	+ if (IS_ERR(bitmap)) {
	1948	+ rv = PTR_ERR(bitmap);
	1949	+ return ERR_PTR(rv);
	1950	+ }
	1951	+
	1952	+ rv = bitmap_init_from_disk(bitmap, 0);
	1953	+ if (rv) {
	1954	+ bitmap_free(bitmap);
	1955	+ return ERR_PTR(rv);
	1956	+ }
	1957	+
	1958	+ return bitmap;
	1959	+}
	1960	+EXPORT_SYMBOL(get_bitmap_from_slot);
	1961	+
1923	1962	/* Loads the bitmap associated with slot and copies the resync information
1924	1963	* to our bitmap
1925	1964	*/
@@ -1929,14 +1968,13 @@ int bitmap_copy_from_slot(struct mddev *mddev, int slot, · 3 unmodified lines
1929	1968	int rv = 0, i, j;
1930	1969	sector_t block, lo = 0, hi = 0;
1931	1970	struct bitmap_counts *counts;
1932		− struct bitmap *bitmap = bitmap_create(mddev, slot);
1933		−
1934		− if (IS_ERR(bitmap))
1935		− return PTR_ERR(bitmap);
	1971	+ struct bitmap *bitmap;
1936	1972
1937		− rv = bitmap_init_from_disk(bitmap, 0);
1938		− if (rv)
1939		− goto err;
	1973	+ bitmap = get_bitmap_from_slot(mddev, slot);
	1974	+ if (IS_ERR(bitmap)) {
	1975	+ pr_err("%s can't get bitmap from slot %d\n", __func__, slot);
	1976	+ return -1;
	1977	+ }
1940	1978
1941	1979	counts = &bitmap->counts;
1942	1980	for (j = 0; j < counts->chunks; j++) {
@@ -1963,8 +2001,7 @@ int bitmap_copy_from_slot(struct mddev *mddev, int slot, · 20 unmodified lines
1963	2001	bitmap_unplug(mddev->bitmap);
1964	2002	*low = lo;
1965	2003	*high = hi;
1966		−err:
1967		− bitmap_free(bitmap);
	2004	+
1968	2005	return rv;
1969	2006	}
1970	2007	EXPORT_SYMBOL_GPL(bitmap_copy_from_slot);

drivers/md/bitmap.h+3−0

@@ -267,8 +267,11 @@ void bitmap_daemon_work(struct mddev *mddev); · 266 unmodified lines
267	267
268	268	int bitmap_resize(struct bitmap *bitmap, sector_t blocks,
269	269	int chunksize, int init);
	270	+struct bitmap get_bitmap_from_slot(struct mddev mddev, int slot);
270	271	int bitmap_copy_from_slot(struct mddev *mddev, int slot,
271	272	sector_t lo, sector_t hi, bool clear_bits);
	273	+void bitmap_free(struct bitmap *bitmap);
	274	+void bitmap_wait_behind_writes(struct mddev *mddev);
272	275	#endif
273	276
274	277	#endif

drivers/md/linear.c+35−40

@@ -249,54 +249,49 @@ static void linear_make_request(struct mddev mddev, struct bio bio) · 248 unmodified lines
249	249	{
250	250	char b[BDEVNAME_SIZE];
251	251	struct dev_info *tmp_dev;
252		− struct bio *split;
253	252	sector_t start_sector, end_sector, data_offset;
	253	+ sector_t bio_sector = bio->bi_iter.bi_sector;
254	254
255	255	if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
256	256	md_flush_request(mddev, bio);
257	257	return;
258	258	}
259	259
260		− do {
261		− sector_t bio_sector = bio->bi_iter.bi_sector;
262		− tmp_dev = which_dev(mddev, bio_sector);
263		− start_sector = tmp_dev->end_sector - tmp_dev->rdev->sectors;
264		− end_sector = tmp_dev->end_sector;
265		− data_offset = tmp_dev->rdev->data_offset;
266		− bio->bi_bdev = tmp_dev->rdev->bdev;
267		−
268		− if (unlikely(bio_sector >= end_sector \|\|
269		− bio_sector < start_sector))
270		− goto out_of_bounds;
271		−
272		− if (unlikely(bio_end_sector(bio) > end_sector)) {
273		− /* This bio crosses a device boundary, so we have to
274		− * split it.
275		− */
276		− split = bio_split(bio, end_sector - bio_sector,
277		− GFP_NOIO, fs_bio_set);
278		− bio_chain(split, bio);
279		− } else {
280		− split = bio;
281		− }
	260	+ tmp_dev = which_dev(mddev, bio_sector);
	261	+ start_sector = tmp_dev->end_sector - tmp_dev->rdev->sectors;
	262	+ end_sector = tmp_dev->end_sector;
	263	+ data_offset = tmp_dev->rdev->data_offset;
	264	+
	265	+ if (unlikely(bio_sector >= end_sector \|\|
	266	+ bio_sector < start_sector))
	267	+ goto out_of_bounds;
	268	+
	269	+ if (unlikely(bio_end_sector(bio) > end_sector)) {
	270	+ /* This bio crosses a device boundary, so we have to split it */
	271	+ struct bio *split = bio_split(bio, end_sector - bio_sector,
	272	+ GFP_NOIO, mddev->bio_set);
	273	+ bio_chain(split, bio);
	274	+ generic_make_request(bio);
	275	+ bio = split;
	276	+ }
282	277
283		− split->bi_iter.bi_sector = split->bi_iter.bi_sector -
284		− start_sector + data_offset;
285		−
286		− if (unlikely((bio_op(split) == REQ_OP_DISCARD) &&
287		− !blk_queue_discard(bdev_get_queue(split->bi_bdev)))) {
288		− /* Just ignore it */
289		− bio_endio(split);
290		− } else {
291		− if (mddev->gendisk)
292		− trace_block_bio_remap(bdev_get_queue(split->bi_bdev),
293		− split, disk_devt(mddev->gendisk),
294		− bio_sector);
295		− mddev_check_writesame(mddev, split);
296		− mddev_check_write_zeroes(mddev, split);
297		− generic_make_request(split);
298		− }
299		− } while (split != bio);
	278	+ bio->bi_bdev = tmp_dev->rdev->bdev;
	279	+ bio->bi_iter.bi_sector = bio->bi_iter.bi_sector -
	280	+ start_sector + data_offset;
	281	+
	282	+ if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
	283	+ !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) {
	284	+ /* Just ignore it */
	285	+ bio_endio(bio);
	286	+ } else {
	287	+ if (mddev->gendisk)
	288	+ trace_block_bio_remap(bdev_get_queue(bio->bi_bdev),
	289	+ bio, disk_devt(mddev->gendisk),
	290	+ bio_sector);
	291	+ mddev_check_writesame(mddev, bio);
	292	+ mddev_check_write_zeroes(mddev, bio);
	293	+ generic_make_request(bio);
	294	+ }
300	295	return;
301	296
302	297	out_of_bounds:

drivers/md/md-cluster.c+207−16

@@ -67,9 +67,10 @@ struct resync_info { · 66 unmodified lines
67	67	* set up all the related infos such as bitmap and personality */
68	68	#define MD_CLUSTER_ALREADY_IN_CLUSTER 6
69	69	#define MD_CLUSTER_PENDING_RECV_EVENT 7
70		−
	70	+#define MD_CLUSTER_HOLDING_MUTEX_FOR_RECVD 8
71	71
72	72	struct md_cluster_info {
	73	+ struct mddev mddev; / the md device which md_cluster_info belongs to */
73	74	/* dlm lock space and resources for clustered raid. */
74	75	dlm_lockspace_t *lockspace;
75	76	int slot_number;
@@ -103,6 +104,7 @@ enum msg_type { · 27 unmodified lines
103	104	REMOVE,
104	105	RE_ADD,
105	106	BITMAP_NEEDS_SYNC,
	107	+ CHANGE_CAPACITY,
106	108	};
107	109
108	110	struct cluster_msg {
@@ -523,11 +525,17 @@ static void process_add_new_disk(struct mddev mddev, struct cluster_msg cmsg) · 414 unmodified lines
523	525
524	526	static void process_metadata_update(struct mddev mddev, struct cluster_msg msg)
525	527	{
	528	+ int got_lock = 0;
526	529	struct md_cluster_info *cinfo = mddev->cluster_info;
527	530	mddev->good_device_nr = le32_to_cpu(msg->raid_slot);
528		− set_bit(MD_RELOAD_SB, &mddev->flags);
	531	+
529	532	dlm_lock_sync(cinfo->no_new_dev_lockres, DLM_LOCK_CR);
530		− md_wakeup_thread(mddev->thread);
	533	+ wait_event(mddev->thread->wqueue,
	534	+ (got_lock = mddev_trylock(mddev)) \|\|
	535	+ test_bit(MD_CLUSTER_HOLDING_MUTEX_FOR_RECVD, &cinfo->state));
	536	+ md_reload_sb(mddev, mddev->good_device_nr);
	537	+ if (got_lock)
	538	+ mddev_unlock(mddev);
531	539	}
532	540
533	541	static void process_remove_disk(struct mddev mddev, struct cluster_msg msg)
@@ -572,6 +580,10 @@ static int process_recvd_msg(struct mddev mddev, struct cluster_msg msg) · 38 unmodified lines
572	580	case METADATA_UPDATED:
573	581	process_metadata_update(mddev, msg);
574	582	break;
	583	+ case CHANGE_CAPACITY:
	584	+ set_capacity(mddev->gendisk, mddev->array_sectors);
	585	+ revalidate_disk(mddev->gendisk);
	586	+ break;
575	587	case RESYNCING:
576	588	process_suspend_info(mddev, le32_to_cpu(msg->slot),
577	589	le64_to_cpu(msg->low),
@@ -646,11 +658,29 @@ out: · 68 unmodified lines
646	658	* Takes the lock on the TOKEN lock resource so no other
647	659	* node can communicate while the operation is underway.
648	660	*/
649		−static int lock_token(struct md_cluster_info *cinfo)
	661	+static int lock_token(struct md_cluster_info *cinfo, bool mddev_locked)
650	662	{
651		− int error;
	663	+ int error, set_bit = 0;
	664	+ struct mddev *mddev = cinfo->mddev;
652	665
	666	+ /*
	667	+ * If resync thread run after raid1d thread, then process_metadata_update
	668	+ * could not continue if raid1d held reconfig_mutex (and raid1d is blocked
	669	+ * since another node already got EX on Token and waitting the EX of Ack),
	670	+ * so let resync wake up thread in case flag is set.
	671	+ */
	672	+ if (mddev_locked && !test_bit(MD_CLUSTER_HOLDING_MUTEX_FOR_RECVD,
	673	+ &cinfo->state)) {
	674	+ error = test_and_set_bit_lock(MD_CLUSTER_HOLDING_MUTEX_FOR_RECVD,
	675	+ &cinfo->state);
	676	+ WARN_ON_ONCE(error);
	677	+ md_wakeup_thread(mddev->thread);
	678	+ set_bit = 1;
	679	+ }
653	680	error = dlm_lock_sync(cinfo->token_lockres, DLM_LOCK_EX);
	681	+ if (set_bit)
	682	+ clear_bit_unlock(MD_CLUSTER_HOLDING_MUTEX_FOR_RECVD, &cinfo->state);
	683	+
654	684	if (error)
655	685	pr_err("md-cluster(%s:%d): failed to get EX on TOKEN (%d)\n",
656	686	__func__, __LINE__, error);
@@ -663,12 +693,12 @@ static int lock_token(struct md_cluster_info *cinfo) · 6 unmodified lines
663	693	/* lock_comm()
664	694	* Sets the MD_CLUSTER_SEND_LOCK bit to lock the send channel.
665	695	*/
666		−static int lock_comm(struct md_cluster_info *cinfo)
	696	+static int lock_comm(struct md_cluster_info *cinfo, bool mddev_locked)
667	697	{
668	698	wait_event(cinfo->wait,
669	699	!test_and_set_bit(MD_CLUSTER_SEND_LOCK, &cinfo->state));
670	700
671		− return lock_token(cinfo);
	701	+ return lock_token(cinfo, mddev_locked);
672	702	}
673	703
674	704	static void unlock_comm(struct md_cluster_info *cinfo)
@@ -743,11 +773,12 @@ failed_message: · 68 unmodified lines
743	773	return error;
744	774	}
745	775
746		−static int sendmsg(struct md_cluster_info cinfo, struct cluster_msg cmsg)
	776	+static int sendmsg(struct md_cluster_info cinfo, struct cluster_msg cmsg,
	777	+ bool mddev_locked)
747	778	{
748	779	int ret;
749	780
750		− lock_comm(cinfo);
	781	+ lock_comm(cinfo, mddev_locked);
751	782	ret = __sendmsg(cinfo, cmsg);
752	783	unlock_comm(cinfo);
753	784	return ret;
@@ -834,6 +865,7 @@ static int join(struct mddev *mddev, int nodes) · 80 unmodified lines
834	865	mutex_init(&cinfo->recv_mutex);
835	866
836	867	mddev->cluster_info = cinfo;
	868	+ cinfo->mddev = mddev;
837	869
838	870	memset(str, 0, 64);
839	871	sprintf(str, "%pU", mddev->uuid);
@@ -908,6 +940,7 @@ static int join(struct mddev *mddev, int nodes) · 68 unmodified lines
908	940
909	941	return 0;
910	942	err:
	943	+ set_bit(MD_CLUSTER_HOLDING_MUTEX_FOR_RECVD, &cinfo->state);
911	944	md_unregister_thread(&cinfo->recovery_thread);
912	945	md_unregister_thread(&cinfo->recv_thread);
913	946	lockres_free(cinfo->message_lockres);
@@ -943,7 +976,7 @@ static void resync_bitmap(struct mddev *mddev) · 29 unmodified lines
943	976	int err;
944	977
945	978	cmsg.type = cpu_to_le32(BITMAP_NEEDS_SYNC);
946		− err = sendmsg(cinfo, &cmsg);
	979	+ err = sendmsg(cinfo, &cmsg, 1);
947	980	if (err)
948	981	pr_err("%s:%d: failed to send BITMAP_NEEDS_SYNC message (%d)\n",
949	982	__func__, __LINE__, err);
@@ -963,6 +996,7 @@ static int leave(struct mddev *mddev) · 13 unmodified lines
963	996	if (cinfo->slot_number > 0 && mddev->recovery_cp != MaxSector)
964	997	resync_bitmap(mddev);
965	998
	999	+ set_bit(MD_CLUSTER_HOLDING_MUTEX_FOR_RECVD, &cinfo->state);
966	1000	md_unregister_thread(&cinfo->recovery_thread);
967	1001	md_unregister_thread(&cinfo->recv_thread);
968	1002	lockres_free(cinfo->message_lockres);
@@ -997,16 +1031,30 @@ static int slot_number(struct mddev *mddev) · 28 unmodified lines
997	1031	static int metadata_update_start(struct mddev *mddev)
998	1032	{
999	1033	struct md_cluster_info *cinfo = mddev->cluster_info;
	1034	+ int ret;
	1035	+
	1036	+ /*
	1037	+ * metadata_update_start is always called with the protection of
	1038	+ * reconfig_mutex, so set WAITING_FOR_TOKEN here.
	1039	+ */
	1040	+ ret = test_and_set_bit_lock(MD_CLUSTER_HOLDING_MUTEX_FOR_RECVD,
	1041	+ &cinfo->state);
	1042	+ WARN_ON_ONCE(ret);
	1043	+ md_wakeup_thread(mddev->thread);
1000	1044
1001	1045	wait_event(cinfo->wait,
1002	1046	!test_and_set_bit(MD_CLUSTER_SEND_LOCK, &cinfo->state) \|\|
1003	1047	test_and_clear_bit(MD_CLUSTER_SEND_LOCKED_ALREADY, &cinfo->state));
1004	1048
1005	1049	/* If token is already locked, return 0 */
1006		− if (cinfo->token_lockres->mode == DLM_LOCK_EX)
	1050	+ if (cinfo->token_lockres->mode == DLM_LOCK_EX) {
	1051	+ clear_bit_unlock(MD_CLUSTER_HOLDING_MUTEX_FOR_RECVD, &cinfo->state);
1007	1052	return 0;
	1053	+ }
1008	1054
1009		− return lock_token(cinfo);
	1055	+ ret = lock_token(cinfo, 1);
	1056	+ clear_bit_unlock(MD_CLUSTER_HOLDING_MUTEX_FOR_RECVD, &cinfo->state);
	1057	+ return ret;
1010	1058	}
1011	1059
1012	1060	static int metadata_update_finish(struct mddev *mddev)
@@ -1043,6 +1091,141 @@ static void metadata_update_cancel(struct mddev *mddev) · 30 unmodified lines
1043	1091	unlock_comm(cinfo);
1044	1092	}
1045	1093
	1094	+/*
	1095	+ * return 0 if all the bitmaps have the same sync_size
	1096	+ */
	1097	+int cluster_check_sync_size(struct mddev *mddev)
	1098	+{
	1099	+ int i, rv;
	1100	+ bitmap_super_t *sb;
	1101	+ unsigned long my_sync_size, sync_size = 0;
	1102	+ int node_num = mddev->bitmap_info.nodes;
	1103	+ int current_slot = md_cluster_ops->slot_number(mddev);
	1104	+ struct bitmap *bitmap = mddev->bitmap;
	1105	+ char str[64];
	1106	+ struct dlm_lock_resource *bm_lockres;
	1107	+
	1108	+ sb = kmap_atomic(bitmap->storage.sb_page);
	1109	+ my_sync_size = sb->sync_size;
	1110	+ kunmap_atomic(sb);
	1111	+
	1112	+ for (i = 0; i < node_num; i++) {
	1113	+ if (i == current_slot)
	1114	+ continue;
	1115	+
	1116	+ bitmap = get_bitmap_from_slot(mddev, i);
	1117	+ if (IS_ERR(bitmap)) {
	1118	+ pr_err("can't get bitmap from slot %d\n", i);
	1119	+ return -1;
	1120	+ }
	1121	+
	1122	+ /*
	1123	+ * If we can hold the bitmap lock of one node then
	1124	+ * the slot is not occupied, update the sb.
	1125	+ */
	1126	+ snprintf(str, 64, "bitmap%04d", i);
	1127	+ bm_lockres = lockres_init(mddev, str, NULL, 1);
	1128	+ if (!bm_lockres) {
	1129	+ pr_err("md-cluster: Cannot initialize %s\n", str);
	1130	+ bitmap_free(bitmap);
	1131	+ return -1;
	1132	+ }
	1133	+ bm_lockres->flags \|= DLM_LKF_NOQUEUE;
	1134	+ rv = dlm_lock_sync(bm_lockres, DLM_LOCK_PW);
	1135	+ if (!rv)
	1136	+ bitmap_update_sb(bitmap);
	1137	+ lockres_free(bm_lockres);
	1138	+
	1139	+ sb = kmap_atomic(bitmap->storage.sb_page);
	1140	+ if (sync_size == 0)
	1141	+ sync_size = sb->sync_size;
	1142	+ else if (sync_size != sb->sync_size) {
	1143	+ kunmap_atomic(sb);
	1144	+ bitmap_free(bitmap);
	1145	+ return -1;
	1146	+ }
	1147	+ kunmap_atomic(sb);
	1148	+ bitmap_free(bitmap);
	1149	+ }
	1150	+
	1151	+ return (my_sync_size == sync_size) ? 0 : -1;
	1152	+}
	1153	+
	1154	+/*
	1155	+ * Update the size for cluster raid is a little more complex, we perform it
	1156	+ * by the steps:
	1157	+ * 1. hold token lock and update superblock in initiator node.
	1158	+ * 2. send METADATA_UPDATED msg to other nodes.
	1159	+ * 3. The initiator node continues to check each bitmap's sync_size, if all
	1160	+ * bitmaps have the same value of sync_size, then we can set capacity and
	1161	+ * let other nodes to perform it. If one node can't update sync_size
	1162	+ * accordingly, we need to revert to previous value.
	1163	+ */
	1164	+static void update_size(struct mddev *mddev, sector_t old_dev_sectors)
	1165	+{
	1166	+ struct md_cluster_info *cinfo = mddev->cluster_info;
	1167	+ struct cluster_msg cmsg;
	1168	+ struct md_rdev *rdev;
	1169	+ int ret = 0;
	1170	+ int raid_slot = -1;
	1171	+
	1172	+ md_update_sb(mddev, 1);
	1173	+ lock_comm(cinfo, 1);
	1174	+
	1175	+ memset(&cmsg, 0, sizeof(cmsg));
	1176	+ cmsg.type = cpu_to_le32(METADATA_UPDATED);
	1177	+ rdev_for_each(rdev, mddev)
	1178	+ if (rdev->raid_disk >= 0 && !test_bit(Faulty, &rdev->flags)) {
	1179	+ raid_slot = rdev->desc_nr;
	1180	+ break;
	1181	+ }
	1182	+ if (raid_slot >= 0) {
	1183	+ cmsg.raid_slot = cpu_to_le32(raid_slot);
	1184	+ /*
	1185	+ * We can only change capiticy after all the nodes can do it,
	1186	+ * so need to wait after other nodes already received the msg
	1187	+ * and handled the change
	1188	+ */
	1189	+ ret = __sendmsg(cinfo, &cmsg);
	1190	+ if (ret) {
	1191	+ pr_err("%s:%d: failed to send METADATA_UPDATED msg\n",
	1192	+ __func__, __LINE__);
	1193	+ unlock_comm(cinfo);
	1194	+ return;
	1195	+ }
	1196	+ } else {
	1197	+ pr_err("md-cluster: No good device id found to send\n");
	1198	+ unlock_comm(cinfo);
	1199	+ return;
	1200	+ }
	1201	+
	1202	+ /*
	1203	+ * check the sync_size from other node's bitmap, if sync_size
	1204	+ * have already updated in other nodes as expected, send an
	1205	+ * empty metadata msg to permit the change of capacity
	1206	+ */
	1207	+ if (cluster_check_sync_size(mddev) == 0) {
	1208	+ memset(&cmsg, 0, sizeof(cmsg));
	1209	+ cmsg.type = cpu_to_le32(CHANGE_CAPACITY);
	1210	+ ret = __sendmsg(cinfo, &cmsg);
	1211	+ if (ret)
	1212	+ pr_err("%s:%d: failed to send CHANGE_CAPACITY msg\n",
	1213	+ __func__, __LINE__);
	1214	+ set_capacity(mddev->gendisk, mddev->array_sectors);
	1215	+ revalidate_disk(mddev->gendisk);
	1216	+ } else {
	1217	+ /* revert to previous sectors */
	1218	+ ret = mddev->pers->resize(mddev, old_dev_sectors);
	1219	+ if (!ret)
	1220	+ revalidate_disk(mddev->gendisk);
	1221	+ ret = __sendmsg(cinfo, &cmsg);
	1222	+ if (ret)
	1223	+ pr_err("%s:%d: failed to send METADATA_UPDATED msg\n",
	1224	+ __func__, __LINE__);
	1225	+ }
	1226	+ unlock_comm(cinfo);
	1227	+}
	1228	+
1046	1229	static int resync_start(struct mddev *mddev)
1047	1230	{
1048	1231	struct md_cluster_info *cinfo = mddev->cluster_info;
@@ -1069,7 +1252,14 @@ static int resync_info_update(struct mddev *mddev, sector_t lo, sector_t hi) · 20 unmodified lines
1069	1252	cmsg.low = cpu_to_le64(lo);
1070	1253	cmsg.high = cpu_to_le64(hi);
1071	1254
1072		− return sendmsg(cinfo, &cmsg);
	1255	+ /*
	1256	+ * mddev_lock is held if resync_info_update is called from
	1257	+ * resync_finish (md_reap_sync_thread -> resync_finish)
	1258	+ */
	1259	+ if (lo == 0 && hi == 0)
	1260	+ return sendmsg(cinfo, &cmsg, 1);
	1261	+ else
	1262	+ return sendmsg(cinfo, &cmsg, 0);
1073	1263	}
1074	1264
1075	1265	static int resync_finish(struct mddev *mddev)
@@ -1119,7 +1309,7 @@ static int add_new_disk(struct mddev mddev, struct md_rdev rdev) · 43 unmodified lines
1119	1309	cmsg.type = cpu_to_le32(NEWDISK);
1120	1310	memcpy(cmsg.uuid, uuid, 16);
1121	1311	cmsg.raid_slot = cpu_to_le32(rdev->desc_nr);
1122		− lock_comm(cinfo);
	1312	+ lock_comm(cinfo, 1);
1123	1313	ret = __sendmsg(cinfo, &cmsg);
1124	1314	if (ret)
1125	1315	return ret;
@@ -1179,7 +1369,7 @@ static int remove_disk(struct mddev mddev, struct md_rdev rdev) · 53 unmodified lines
1179	1369	struct md_cluster_info *cinfo = mddev->cluster_info;
1180	1370	cmsg.type = cpu_to_le32(REMOVE);
1181	1371	cmsg.raid_slot = cpu_to_le32(rdev->desc_nr);
1182		− return sendmsg(cinfo, &cmsg);
	1372	+ return sendmsg(cinfo, &cmsg, 1);
1183	1373	}
1184	1374
1185	1375	static int lock_all_bitmaps(struct mddev *mddev)
@@ -1243,7 +1433,7 @@ static int gather_bitmaps(struct md_rdev *rdev) · 57 unmodified lines
1243	1433
1244	1434	cmsg.type = cpu_to_le32(RE_ADD);
1245	1435	cmsg.raid_slot = cpu_to_le32(rdev->desc_nr);
1246		− err = sendmsg(cinfo, &cmsg);
	1436	+ err = sendmsg(cinfo, &cmsg, 1);
1247	1437	if (err)
1248	1438	goto out;
1249	1439
@@ -1281,6 +1471,7 @@ static struct md_cluster_operations cluster_ops = { · 31 unmodified lines
1281	1471	.gather_bitmaps = gather_bitmaps,
1282	1472	.lock_all_bitmaps = lock_all_bitmaps,
1283	1473	.unlock_all_bitmaps = unlock_all_bitmaps,
	1474	+ .update_size = update_size,
1284	1475	};
1285	1476
1286	1477	static int __init cluster_init(void)

drivers/md/md-cluster.h+1−0

@@ -27,6 +27,7 @@ struct md_cluster_operations { · 26 unmodified lines
27	27	int (gather_bitmaps)(struct md_rdev rdev);
28	28	int (lock_all_bitmaps)(struct mddev mddev);
29	29	void (unlock_all_bitmaps)(struct mddev mddev);
	30	+ void (update_size)(struct mddev mddev, sector_t old_dev_sectors);
30	31	};
31	32
32	33	#endif /* _MD_CLUSTER_H */

drivers/md/md.c+320−94

@@ -65,6 +65,8 @@ · 64 unmodified lines
65	65	#include <linux/raid/md_p.h>
66	66	#include <linux/raid/md_u.h>
67	67	#include <linux/slab.h>
	68	+#include <linux/percpu-refcount.h>
	69	+
68	70	#include <trace/events/block.h>
69	71	#include "md.h"
70	72	#include "bitmap.h"
@@ -172,6 +174,16 @@ static const struct block_device_operations md_fops; · 101 unmodified lines
172	174
173	175	static int start_readonly;
174	176
	177	+/*
	178	+ * The original mechanism for creating an md device is to create
	179	+ * a device node in /dev and to open it. This causes races with device-close.
	180	+ * The preferred method is to write to the "new_array" module parameter.
	181	+ * This can avoid races.
	182	+ * Setting create_on_open to false disables the original mechanism
	183	+ * so all the races disappear.
	184	+ */
	185	+static bool create_on_open = true;
	186	+
175	187	/* bio_clone_mddev
176	188	* like bio_clone, but with a local bio set
177	189	*/
@@ -1507,6 +1519,12 @@ static int super_1_load(struct md_rdev rdev, struct md_rdev refdev, int minor_ · 1329 unmodified lines
1507	1519	} else if (sb->bblog_offset != 0)
1508	1520	rdev->badblocks.shift = 0;
1509	1521
	1522	+ if (le32_to_cpu(sb->feature_map) & MD_FEATURE_PPL) {
	1523	+ rdev->ppl.offset = (__s16)le16_to_cpu(sb->ppl.offset);
	1524	+ rdev->ppl.size = le16_to_cpu(sb->ppl.size);
	1525	+ rdev->ppl.sector = rdev->sb_start + rdev->ppl.offset;
	1526	+ }
	1527	+
1510	1528	if (!refdev) {
1511	1529	ret = 1;
1512	1530	} else {
@@ -1619,6 +1637,13 @@ static int super_1_validate(struct mddev mddev, struct md_rdev rdev) · 106 unmodified lines
1619	1637
1620	1638	if (le32_to_cpu(sb->feature_map) & MD_FEATURE_JOURNAL)
1621	1639	set_bit(MD_HAS_JOURNAL, &mddev->flags);
	1640	+
	1641	+ if (le32_to_cpu(sb->feature_map) & MD_FEATURE_PPL) {
	1642	+ if (le32_to_cpu(sb->feature_map) &
	1643	+ (MD_FEATURE_BITMAP_OFFSET \| MD_FEATURE_JOURNAL))
	1644	+ return -EINVAL;
	1645	+ set_bit(MD_HAS_PPL, &mddev->flags);
	1646	+ }
1622	1647	} else if (mddev->pers == NULL) {
1623	1648	/* Insist of good event counter while assembling, except for
1624	1649	* spares (which don't need an event count) */
@@ -1832,6 +1857,12 @@ retry: · 207 unmodified lines
1832	1857	if (test_bit(MD_HAS_JOURNAL, &mddev->flags))
1833	1858	sb->feature_map \|= cpu_to_le32(MD_FEATURE_JOURNAL);
1834	1859
	1860	+ if (test_bit(MD_HAS_PPL, &mddev->flags)) {
	1861	+ sb->feature_map \|= cpu_to_le32(MD_FEATURE_PPL);
	1862	+ sb->ppl.offset = cpu_to_le16(rdev->ppl.offset);
	1863	+ sb->ppl.size = cpu_to_le16(rdev->ppl.size);
	1864	+ }
	1865	+
1835	1866	rdev_for_each(rdev2, mddev) {
1836	1867	i = rdev2->desc_nr;
1837	1868	if (test_bit(Faulty, &rdev2->flags))
@@ -2072,6 +2103,10 @@ static int bind_rdev_to_array(struct md_rdev rdev, struct mddev mddev) · 234 unmodified lines
2072	2103	if (find_rdev(mddev, rdev->bdev->bd_dev))
2073	2104	return -EEXIST;
2074	2105
	2106	+ if ((bdev_read_only(rdev->bdev) \|\| bdev_read_only(rdev->meta_bdev)) &&
	2107	+ mddev->pers)
	2108	+ return -EROFS;
	2109	+
2075	2110	/* make sure rdev->sectors exceeds mddev->dev_sectors */
2076	2111	if (!test_bit(Journal, &rdev->flags) &&
2077	2112	rdev->sectors &&
@@ -2233,6 +2268,33 @@ static void export_array(struct mddev *mddev) · 155 unmodified lines
2233	2268	mddev->major_version = 0;
2234	2269	}
2235	2270
	2271	+static bool set_in_sync(struct mddev *mddev)
	2272	+{
	2273	+ WARN_ON_ONCE(!spin_is_locked(&mddev->lock));
	2274	+ if (!mddev->in_sync) {
	2275	+ mddev->sync_checkers++;
	2276	+ spin_unlock(&mddev->lock);
	2277	+ percpu_ref_switch_to_atomic_sync(&mddev->writes_pending);
	2278	+ spin_lock(&mddev->lock);
	2279	+ if (!mddev->in_sync &&
	2280	+ percpu_ref_is_zero(&mddev->writes_pending)) {
	2281	+ mddev->in_sync = 1;
	2282	+ /*
	2283	+ * Ensure ->in_sync is visible before we clear
	2284	+ * ->sync_checkers.
	2285	+ */
	2286	+ smp_mb();
	2287	+ set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
	2288	+ sysfs_notify_dirent_safe(mddev->sysfs_state);
	2289	+ }
	2290	+ if (--mddev->sync_checkers == 0)
	2291	+ percpu_ref_switch_to_percpu(&mddev->writes_pending);
	2292	+ }
	2293	+ if (mddev->safemode == 1)
	2294	+ mddev->safemode = 0;
	2295	+ return mddev->in_sync;
	2296	+}
	2297	+
2236	2298	static void sync_sbs(struct mddev *mddev, int nospares)
2237	2299	{
2238	2300	/* Update each superblock (in-memory image), but
@@ -3131,6 +3193,78 @@ static ssize_t ubb_store(struct md_rdev rdev, const char page, size_t len) · 892 unmodified lines
3131	3193	static struct rdev_sysfs_entry rdev_unack_bad_blocks =
3132	3194	__ATTR(unacknowledged_bad_blocks, S_IRUGO\|S_IWUSR, ubb_show, ubb_store);
3133	3195
	3196	+static ssize_t
	3197	+ppl_sector_show(struct md_rdev rdev, char page)
	3198	+{
	3199	+ return sprintf(page, "%llu\n", (unsigned long long)rdev->ppl.sector);
	3200	+}
	3201	+
	3202	+static ssize_t
	3203	+ppl_sector_store(struct md_rdev rdev, const char buf, size_t len)
	3204	+{
	3205	+ unsigned long long sector;
	3206	+
	3207	+ if (kstrtoull(buf, 10, &sector) < 0)
	3208	+ return -EINVAL;
	3209	+ if (sector != (sector_t)sector)
	3210	+ return -EINVAL;
	3211	+
	3212	+ if (rdev->mddev->pers && test_bit(MD_HAS_PPL, &rdev->mddev->flags) &&
	3213	+ rdev->raid_disk >= 0)
	3214	+ return -EBUSY;
	3215	+
	3216	+ if (rdev->mddev->persistent) {
	3217	+ if (rdev->mddev->major_version == 0)
	3218	+ return -EINVAL;
	3219	+ if ((sector > rdev->sb_start &&
	3220	+ sector - rdev->sb_start > S16_MAX) \|\|
	3221	+ (sector < rdev->sb_start &&
	3222	+ rdev->sb_start - sector > -S16_MIN))
	3223	+ return -EINVAL;
	3224	+ rdev->ppl.offset = sector - rdev->sb_start;
	3225	+ } else if (!rdev->mddev->external) {
	3226	+ return -EBUSY;
	3227	+ }
	3228	+ rdev->ppl.sector = sector;
	3229	+ return len;
	3230	+}
	3231	+
	3232	+static struct rdev_sysfs_entry rdev_ppl_sector =
	3233	+__ATTR(ppl_sector, S_IRUGO\|S_IWUSR, ppl_sector_show, ppl_sector_store);
	3234	+
	3235	+static ssize_t
	3236	+ppl_size_show(struct md_rdev rdev, char page)
	3237	+{
	3238	+ return sprintf(page, "%u\n", rdev->ppl.size);
	3239	+}
	3240	+
	3241	+static ssize_t
	3242	+ppl_size_store(struct md_rdev rdev, const char buf, size_t len)
	3243	+{
	3244	+ unsigned int size;
	3245	+
	3246	+ if (kstrtouint(buf, 10, &size) < 0)
	3247	+ return -EINVAL;
	3248	+
	3249	+ if (rdev->mddev->pers && test_bit(MD_HAS_PPL, &rdev->mddev->flags) &&
	3250	+ rdev->raid_disk >= 0)
	3251	+ return -EBUSY;
	3252	+
	3253	+ if (rdev->mddev->persistent) {
	3254	+ if (rdev->mddev->major_version == 0)
	3255	+ return -EINVAL;
	3256	+ if (size > U16_MAX)
	3257	+ return -EINVAL;
	3258	+ } else if (!rdev->mddev->external) {
	3259	+ return -EBUSY;
	3260	+ }
	3261	+ rdev->ppl.size = size;
	3262	+ return len;
	3263	+}
	3264	+
	3265	+static struct rdev_sysfs_entry rdev_ppl_size =
	3266	+__ATTR(ppl_size, S_IRUGO\|S_IWUSR, ppl_size_show, ppl_size_store);
	3267	+
3134	3268	static struct attribute *rdev_default_attrs[] = {
3135	3269	&rdev_state.attr,
3136	3270	&rdev_errors.attr,
@@ -3141,6 +3275,8 @@ static struct attribute *rdev_default_attrs[] = { · 4 unmodified lines
3141	3275	&rdev_recovery_start.attr,
3142	3276	&rdev_bad_blocks.attr,
3143	3277	&rdev_unack_bad_blocks.attr,
	3278	+ &rdev_ppl_sector.attr,
	3279	+ &rdev_ppl_size.attr,
3144	3280	NULL,
3145	3281	};
3146	3282	static ssize_t
@@ -3903,6 +4039,7 @@ array_state_show(struct mddev mddev, char page) · 756 unmodified lines
3903	4039	st = read_auto;
3904	4040	break;
3905	4041	case 0:
	4042	+ spin_lock(&mddev->lock);
3906	4043	if (test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags))
3907	4044	st = write_pending;
3908	4045	else if (mddev->in_sync)
@@ -3911,6 +4048,7 @@ array_state_show(struct mddev mddev, char page) · 2 unmodified lines
3911	4048	st = active_idle;
3912	4049	else
3913	4050	st = active;
	4051	+ spin_unlock(&mddev->lock);
3914	4052	}
3915	4053	else {
3916	4054	if (list_empty(&mddev->disks) &&
@@ -3931,7 +4069,7 @@ static int restart_array(struct mddev *mddev); · 14 unmodified lines
3931	4069	static ssize_t
3932	4070	array_state_store(struct mddev mddev, const char buf, size_t len)
3933	4071	{
3934		− int err;
	4072	+ int err = 0;
3935	4073	enum array_state st = match_word(buf, array_states);
3936	4074
3937	4075	if (mddev->pers && (st == active \|\| st == clean) && mddev->ro != 1) {
@@ -3944,18 +4082,9 @@ array_state_store(struct mddev mddev, const char buf, size_t len) · 6 unmodified lines
3944	4082	clear_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
3945	4083	md_wakeup_thread(mddev->thread);
3946	4084	wake_up(&mddev->sb_wait);
3947		− err = 0;
3948	4085	} else /* st == clean */ {
3949	4086	restart_array(mddev);
3950		− if (atomic_read(&mddev->writes_pending) == 0) {
3951		− if (mddev->in_sync == 0) {
3952		− mddev->in_sync = 1;
3953		− if (mddev->safemode == 1)
3954		− mddev->safemode = 0;
3955		− set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
3956		− }
3957		− err = 0;
3958		− } else
	4087	+ if (!set_in_sync(mddev))
3959	4088	err = -EBUSY;
3960	4089	}
3961	4090	if (!err)
@@ -4013,15 +4142,7 @@ array_state_store(struct mddev mddev, const char buf, size_t len) · 51 unmodified lines
4013	4142	if (err)
4014	4143	break;
4015	4144	spin_lock(&mddev->lock);
4016		− if (atomic_read(&mddev->writes_pending) == 0) {
4017		− if (mddev->in_sync == 0) {
4018		− mddev->in_sync = 1;
4019		− if (mddev->safemode == 1)
4020		− mddev->safemode = 0;
4021		− set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
4022		− }
4023		− err = 0;
4024		− } else
	4145	+ if (!set_in_sync(mddev))
4025	4146	err = -EBUSY;
4026	4147	spin_unlock(&mddev->lock);
4027	4148	} else
@@ -4843,8 +4964,10 @@ array_size_store(struct mddev mddev, const char buf, size_t len) · 815 unmodified lines
4843	4964	return err;
4844	4965
4845	4966	/* cluster raid doesn't support change array_sectors */
4846		− if (mddev_is_clustered(mddev))
	4967	+ if (mddev_is_clustered(mddev)) {
	4968	+ mddev_unlock(mddev);
4847	4969	return -EINVAL;
	4970	+ }
4848	4971
4849	4972	if (strncmp(buf, "default", 7) == 0) {
4850	4973	if (mddev->pers)
@@ -4877,6 +5000,52 @@ static struct md_sysfs_entry md_array_size = · 26 unmodified lines
4877	5000	__ATTR(array_size, S_IRUGO\|S_IWUSR, array_size_show,
4878	5001	array_size_store);
4879	5002
	5003	+static ssize_t
	5004	+consistency_policy_show(struct mddev mddev, char page)
	5005	+{
	5006	+ int ret;
	5007	+
	5008	+ if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
	5009	+ ret = sprintf(page, "journal\n");
	5010	+ } else if (test_bit(MD_HAS_PPL, &mddev->flags)) {
	5011	+ ret = sprintf(page, "ppl\n");
	5012	+ } else if (mddev->bitmap) {
	5013	+ ret = sprintf(page, "bitmap\n");
	5014	+ } else if (mddev->pers) {
	5015	+ if (mddev->pers->sync_request)
	5016	+ ret = sprintf(page, "resync\n");
	5017	+ else
	5018	+ ret = sprintf(page, "none\n");
	5019	+ } else {
	5020	+ ret = sprintf(page, "unknown\n");
	5021	+ }
	5022	+
	5023	+ return ret;
	5024	+}
	5025	+
	5026	+static ssize_t
	5027	+consistency_policy_store(struct mddev mddev, const char buf, size_t len)
	5028	+{
	5029	+ int err = 0;
	5030	+
	5031	+ if (mddev->pers) {
	5032	+ if (mddev->pers->change_consistency_policy)
	5033	+ err = mddev->pers->change_consistency_policy(mddev, buf);
	5034	+ else
	5035	+ err = -EBUSY;
	5036	+ } else if (mddev->external && strncmp(buf, "ppl", 3) == 0) {
	5037	+ set_bit(MD_HAS_PPL, &mddev->flags);
	5038	+ } else {
	5039	+ err = -EINVAL;
	5040	+ }
	5041	+
	5042	+ return err ? err : len;
	5043	+}
	5044	+
	5045	+static struct md_sysfs_entry md_consistency_policy =
	5046	+__ATTR(consistency_policy, S_IRUGO \| S_IWUSR, consistency_policy_show,
	5047	+ consistency_policy_store);
	5048	+
4880	5049	static struct attribute *md_default_attrs[] = {
4881	5050	&md_level.attr,
4882	5051	&md_layout.attr,
@@ -4892,6 +5061,7 @@ static struct attribute *md_default_attrs[] = { · 9 unmodified lines
4892	5061	&md_reshape_direction.attr,
4893	5062	&md_array_size.attr,
4894	5063	&max_corr_read_errors.attr,
	5064	+ &md_consistency_policy.attr,
4895	5065	NULL,
4896	5066	};
4897	5067
@@ -4976,6 +5146,7 @@ static void md_free(struct kobject *ko) · 78 unmodified lines
4976	5146	del_gendisk(mddev->gendisk);
4977	5147	put_disk(mddev->gendisk);
4978	5148	}
	5149	+ percpu_ref_exit(&mddev->writes_pending);
4979	5150
4980	5151	kfree(mddev);
4981	5152	}
@@ -5001,8 +5172,19 @@ static void mddev_delayed_delete(struct work_struct *ws) · 19 unmodified lines
5001	5172	kobject_put(&mddev->kobj);
5002	5173	}
5003	5174
	5175	+static void no_op(struct percpu_ref *r) {}
	5176	+
5004	5177	static int md_alloc(dev_t dev, char *name)
5005	5178	{
	5179	+ /*
	5180	+ * If dev is zero, name is the name of a device to allocate with
	5181	+ * an arbitrary minor number. It will be "md_???"
	5182	+ * If dev is non-zero it must be a device number with a MAJOR of
	5183	+ * MD_MAJOR or mdp_major. In this case, if "name" is NULL, then
	5184	+ * the device is being created by opening a node in /dev.
	5185	+ * If "name" is not NULL, the device is being created by
	5186	+ * writing to /sys/module/md_mod/parameters/new_array.
	5187	+ */
5006	5188	static DEFINE_MUTEX(disks_mutex);
5007	5189	struct mddev *mddev = mddev_find(dev);
5008	5190	struct gendisk *disk;
@@ -5028,7 +5210,7 @@ static int md_alloc(dev_t dev, char *name) · 19 unmodified lines
5028	5210	if (mddev->gendisk)
5029	5211	goto abort;
5030	5212
5031		− if (name) {
	5213	+ if (name && !dev) {
5032	5214	/* Need to ensure that 'name' is not a duplicate.
5033	5215	*/
5034	5216	struct mddev *mddev2;
@@ -5042,6 +5224,11 @@ static int md_alloc(dev_t dev, char *name) · 7 unmodified lines
5042	5224	}
5043	5225	spin_unlock(&all_mddevs_lock);
5044	5226	}
	5227	+ if (name && dev)
	5228	+ /*
	5229	+ * Creating /dev/mdNNN via "newarray", so adjust hold_active.
	5230	+ */
	5231	+ mddev->hold_active = UNTIL_STOP;
5045	5232
5046	5233	error = -ENOMEM;
5047	5234	mddev->queue = blk_alloc_queue(GFP_KERNEL);
@@ -5052,6 +5239,10 @@ static int md_alloc(dev_t dev, char *name) · 4 unmodified lines
5052	5239	blk_queue_make_request(mddev->queue, md_make_request);
5053	5240	blk_set_stacking_limits(&mddev->queue->limits);
5054	5241
	5242	+ if (percpu_ref_init(&mddev->writes_pending, no_op, 0, GFP_KERNEL) < 0)
	5243	+ goto abort;
	5244	+ /* We want to start with the refcount at zero */
	5245	+ percpu_ref_put(&mddev->writes_pending);
5055	5246	disk = alloc_disk(1 << shift);
5056	5247	if (!disk) {
5057	5248	blk_cleanup_queue(mddev->queue);
@@ -5108,38 +5299,48 @@ static int md_alloc(dev_t dev, char *name) · 50 unmodified lines
5108	5299
5109	5300	static struct kobject md_probe(dev_t dev, int part, void *data)
5110	5301	{
5111		− md_alloc(dev, NULL);
	5302	+ if (create_on_open)
	5303	+ md_alloc(dev, NULL);
5112	5304	return NULL;
5113	5305	}
5114	5306
5115	5307	static int add_named_array(const char val, struct kernel_param kp)
5116	5308	{
5117		− /* val must be "md_" where is not all digits.
5118		− * We allocate an array with a large free minor number, and
	5309	+ /*
	5310	+ * val must be "md_*" or "mdNNN".
	5311	+ * For "md_*" we allocate an array with a large free minor number, and
5119	5312	* set the name to val. val must not already be an active name.
	5313	+ * For "mdNNN" we allocate an array with the minor number NNN
	5314	+ * which must not already be in use.
5120	5315	*/
5121	5316	int len = strlen(val);
5122	5317	char buf[DISK_NAME_LEN];
	5318	+ unsigned long devnum;
5123	5319
5124	5320	while (len && val[len-1] == '\n')
5125	5321	len--;
5126	5322	if (len >= DISK_NAME_LEN)
5127	5323	return -E2BIG;
5128	5324	strlcpy(buf, val, len+1);
5129		− if (strncmp(buf, "md_", 3) != 0)
5130		− return -EINVAL;
5131		− return md_alloc(0, buf);
	5325	+ if (strncmp(buf, "md_", 3) == 0)
	5326	+ return md_alloc(0, buf);
	5327	+ if (strncmp(buf, "md", 2) == 0 &&
	5328	+ isdigit(buf[2]) &&
	5329	+ kstrtoul(buf+2, 10, &devnum) == 0 &&
	5330	+ devnum <= MINORMASK)
	5331	+ return md_alloc(MKDEV(MD_MAJOR, devnum), NULL);
	5332	+
	5333	+ return -EINVAL;
5132	5334	}
5133	5335
5134	5336	static void md_safemode_timeout(unsigned long data)
5135	5337	{
5136	5338	struct mddev mddev = (struct mddev ) data;
5137	5339
5138		− if (!atomic_read(&mddev->writes_pending)) {
5139		− mddev->safemode = 1;
5140		− if (mddev->external)
5141		− sysfs_notify_dirent_safe(mddev->sysfs_state);
5142		− }
	5340	+ mddev->safemode = 1;
	5341	+ if (mddev->external)
	5342	+ sysfs_notify_dirent_safe(mddev->sysfs_state);
	5343	+
5143	5344	md_wakeup_thread(mddev->thread);
5144	5345	}
5145	5346
@@ -5185,6 +5386,13 @@ int md_run(struct mddev *mddev) · 39 unmodified lines
5185	5386	continue;
5186	5387	sync_blockdev(rdev->bdev);
5187	5388	invalidate_bdev(rdev->bdev);
	5389	+ if (mddev->ro != 1 &&
	5390	+ (bdev_read_only(rdev->bdev) \|\|
	5391	+ bdev_read_only(rdev->meta_bdev))) {
	5392	+ mddev->ro = 1;
	5393	+ if (mddev->gendisk)
	5394	+ set_disk_ro(mddev->gendisk, 1);
	5395	+ }
5188	5396
5189	5397	/* perform some consistency tests on the device.
5190	5398	* We don't want the data to overlap the metadata,
@@ -5344,7 +5552,6 @@ int md_run(struct mddev *mddev) · 153 unmodified lines
5344	5552	} else if (mddev->ro == 2) /* auto-readonly not meaningful */
5345	5553	mddev->ro = 0;
5346	5554
5347		− atomic_set(&mddev->writes_pending,0);
5348	5555	atomic_set(&mddev->max_corr_read_errors,
5349	5556	MD_DEFAULT_MAX_CORRECTED_READ_ERRORS);
5350	5557	mddev->safemode = 0;
@@ -5410,6 +5617,9 @@ out: · 59 unmodified lines
5410	5617	static int restart_array(struct mddev *mddev)
5411	5618	{
5412	5619	struct gendisk *disk = mddev->gendisk;
	5620	+ struct md_rdev *rdev;
	5621	+ bool has_journal = false;
	5622	+ bool has_readonly = false;
5413	5623
5414	5624	/* Complain if it has no devices */
5415	5625	if (list_empty(&mddev->disks))
@@ -5418,24 +5628,21 @@ static int restart_array(struct mddev *mddev) · 2 unmodified lines
5418	5628	return -EINVAL;
5419	5629	if (!mddev->ro)
5420	5630	return -EBUSY;
5421		− if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
5422		− struct md_rdev *rdev;
5423		− bool has_journal = false;
5424		−
5425		− rcu_read_lock();
5426		− rdev_for_each_rcu(rdev, mddev) {
5427		− if (test_bit(Journal, &rdev->flags) &&
5428		− !test_bit(Faulty, &rdev->flags)) {
5429		− has_journal = true;
5430		− break;
5431		− }
5432		− }
5433		− rcu_read_unlock();
5434	5631
	5632	+ rcu_read_lock();
	5633	+ rdev_for_each_rcu(rdev, mddev) {
	5634	+ if (test_bit(Journal, &rdev->flags) &&
	5635	+ !test_bit(Faulty, &rdev->flags))
	5636	+ has_journal = true;
	5637	+ if (bdev_read_only(rdev->bdev))
	5638	+ has_readonly = true;
	5639	+ }
	5640	+ rcu_read_unlock();
	5641	+ if (test_bit(MD_HAS_JOURNAL, &mddev->flags) && !has_journal)
5435	5642	/* Don't restart rw with journal missing/faulty */
5436		− if (!has_journal)
5437	5643	return -EINVAL;
5438		− }
	5644	+ if (has_readonly)
	5645	+ return -EROFS;
5439	5646
5440	5647	mddev->safemode = 0;
5441	5648	mddev->ro = 0;
@@ -5535,15 +5742,7 @@ EXPORT_SYMBOL_GPL(md_stop_writes); · 93 unmodified lines
5535	5742
5536	5743	static void mddev_detach(struct mddev *mddev)
5537	5744	{
5538		− struct bitmap *bitmap = mddev->bitmap;
5539		− /* wait for behind writes to complete */
5540		− if (bitmap && atomic_read(&bitmap->behind_writes) > 0) {
5541		− pr_debug("md:%s: behind writes in progress - waiting to stop.\n",
5542		− mdname(mddev));
5543		− /* need to kick something here to make sure I/O goes? */
5544		− wait_event(bitmap->behind_wait,
5545		− atomic_read(&bitmap->behind_writes) == 0);
5546		− }
	5745	+ bitmap_wait_behind_writes(mddev);
5547	5746	if (mddev->pers && mddev->pers->quiesce) {
5548	5747	mddev->pers->quiesce(mddev, 1);
5549	5748	mddev->pers->quiesce(mddev, 0);
@@ -5556,6 +5755,7 @@ static void mddev_detach(struct mddev *mddev) · 6 unmodified lines
5556	5755	static void __md_stop(struct mddev *mddev)
5557	5756	{
5558	5757	struct md_personality *pers = mddev->pers;
	5758	+ bitmap_destroy(mddev);
5559	5759	mddev_detach(mddev);
5560	5760	/* Ensure ->event_work is done */
5561	5761	flush_workqueue(md_misc_wq);
@@ -5576,7 +5776,6 @@ void md_stop(struct mddev *mddev) · 14 unmodified lines
5576	5776	* This is called from dm-raid
5577	5777	*/
5578	5778	__md_stop(mddev);
5579		− bitmap_destroy(mddev);
5580	5779	if (mddev->bio_set)
5581	5780	bioset_free(mddev->bio_set);
5582	5781	}
@@ -5714,7 +5913,6 @@ static int do_md_stop(struct mddev *mddev, int mode, · 131 unmodified lines
5714	5913	if (mode == 0) {
5715	5914	pr_info("md: %s stopped.\n", mdname(mddev));
5716	5915
5717		− bitmap_destroy(mddev);
5718	5916	if (mddev->bitmap_info.file) {
5719	5917	struct file *f = mddev->bitmap_info.file;
5720	5918	spin_lock(&mddev->lock);
@@ -6493,10 +6691,7 @@ static int update_size(struct mddev *mddev, sector_t num_sectors) · 772 unmodified lines
6493	6691	struct md_rdev *rdev;
6494	6692	int rv;
6495	6693	int fit = (num_sectors == 0);
6496		−
6497		− /* cluster raid doesn't support update size */
6498		− if (mddev_is_clustered(mddev))
6499		− return -EINVAL;
	6694	+ sector_t old_dev_sectors = mddev->dev_sectors;
6500	6695
6501	6696	if (mddev->pers->resize == NULL)
6502	6697	return -EINVAL;
@@ -6525,7 +6720,9 @@ static int update_size(struct mddev *mddev, sector_t num_sectors) · 22 unmodified lines
6525	6720	}
6526	6721	rv = mddev->pers->resize(mddev, num_sectors);
6527	6722	if (!rv) {
6528		− if (mddev->queue) {
	6723	+ if (mddev_is_clustered(mddev))
	6724	+ md_cluster_ops->update_size(mddev, old_dev_sectors);
	6725	+ else if (mddev->queue) {
6529	6726	set_capacity(mddev->gendisk, mddev->array_sectors);
6530	6727	revalidate_disk(mddev->gendisk);
6531	6728	}
@@ -6776,6 +6973,7 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode, · 244 unmodified lines
6776	6973	void __user argp = (void __user )arg;
6777	6974	struct mddev *mddev = NULL;
6778	6975	int ro;
	6976	+ bool did_set_md_closing = false;
6779	6977
6780	6978	if (!md_ioctl_valid(cmd))
6781	6979	return -ENOTTY;
@@ -6865,7 +7063,9 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode, · 83 unmodified lines
6865	7063	err = -EBUSY;
6866	7064	goto out;
6867	7065	}
	7066	+ WARN_ON_ONCE(test_bit(MD_CLOSING, &mddev->flags));
6868	7067	set_bit(MD_CLOSING, &mddev->flags);
	7068	+ did_set_md_closing = true;
6869	7069	mutex_unlock(&mddev->open_mutex);
6870	7070	sync_blockdev(bdev);
6871	7071	}
@@ -7058,6 +7258,8 @@ unlock: · 186 unmodified lines
7058	7258	mddev->hold_active = 0;
7059	7259	mddev_unlock(mddev);
7060	7260	out:
	7261	+ if(did_set_md_closing)
	7262	+ clear_bit(MD_CLOSING, &mddev->flags);
7061	7263	return err;
7062	7264	}
7063	7265	#ifdef CONFIG_COMPAT
@@ -7208,8 +7410,8 @@ void md_wakeup_thread(struct md_thread *thread) · 144 unmodified lines
7208	7410	{
7209	7411	if (thread) {
7210	7412	pr_debug("md: waking up MD thread %s.\n", thread->tsk->comm);
7211		− set_bit(THREAD_WAKEUP, &thread->flags);
7212		− wake_up(&thread->wqueue);
	7413	+ if (!test_and_set_bit(THREAD_WAKEUP, &thread->flags))
	7414	+ wake_up(&thread->wqueue);
7213	7415	}
7214	7416	}
7215	7417	EXPORT_SYMBOL(md_wakeup_thread);
@@ -7756,10 +7958,13 @@ void md_write_start(struct mddev mddev, struct bio bi) · 540 unmodified lines
7756	7958	md_wakeup_thread(mddev->sync_thread);
7757	7959	did_change = 1;
7758	7960	}
7759		− atomic_inc(&mddev->writes_pending);
	7961	+ rcu_read_lock();
	7962	+ percpu_ref_get(&mddev->writes_pending);
	7963	+ smp_mb(); /* Match smp_mb in set_in_sync() */
7760	7964	if (mddev->safemode == 1)
7761	7965	mddev->safemode = 0;
7762		− if (mddev->in_sync) {
	7966	+ /* sync_checkers is always 0 when writes_pending is in per-cpu mode */
	7967	+ if (mddev->in_sync \|\| !mddev->sync_checkers) {
7763	7968	spin_lock(&mddev->lock);
7764	7969	if (mddev->in_sync) {
7765	7970	mddev->in_sync = 0;
@@ -7770,6 +7975,7 @@ void md_write_start(struct mddev mddev, struct bio bi) · 4 unmodified lines
7770	7975	}
7771	7976	spin_unlock(&mddev->lock);
7772	7977	}
	7978	+ rcu_read_unlock();
7773	7979	if (did_change)
7774	7980	sysfs_notify_dirent_safe(mddev->sysfs_state);
7775	7981	wait_event(mddev->sb_wait,
@@ -7777,15 +7983,38 @@ void md_write_start(struct mddev mddev, struct bio bi) · 1 unmodified line
7777	7983	}
7778	7984	EXPORT_SYMBOL(md_write_start);
7779	7985
	7986	+/* md_write_inc can only be called when md_write_start() has
	7987	+ * already been called at least once of the current request.
	7988	+ * It increments the counter and is useful when a single request
	7989	+ * is split into several parts. Each part causes an increment and
	7990	+ * so needs a matching md_write_end().
	7991	+ * Unlike md_write_start(), it is safe to call md_write_inc() inside
	7992	+ * a spinlocked region.
	7993	+ */
	7994	+void md_write_inc(struct mddev mddev, struct bio bi)
	7995	+{
	7996	+ if (bio_data_dir(bi) != WRITE)
	7997	+ return;
	7998	+ WARN_ON_ONCE(mddev->in_sync \|\| mddev->ro);
	7999	+ percpu_ref_get(&mddev->writes_pending);
	8000	+}
	8001	+EXPORT_SYMBOL(md_write_inc);
	8002	+
7780	8003	void md_write_end(struct mddev *mddev)
7781	8004	{
7782		− if (atomic_dec_and_test(&mddev->writes_pending)) {
7783		− if (mddev->safemode == 2)
7784		− md_wakeup_thread(mddev->thread);
7785		− else if (mddev->safemode_delay)
7786		− mod_timer(&mddev->safemode_timer, jiffies + mddev->safemode_delay);
7787		− }
	8005	+ percpu_ref_put(&mddev->writes_pending);
	8006	+
	8007	+ if (mddev->safemode == 2)
	8008	+ md_wakeup_thread(mddev->thread);
	8009	+ else if (mddev->safemode_delay)
	8010	+ /* The roundup() ensures this only performs locking once
	8011	+ * every ->safemode_delay jiffies
	8012	+ */
	8013	+ mod_timer(&mddev->safemode_timer,
	8014	+ roundup(jiffies, mddev->safemode_delay) +
	8015	+ mddev->safemode_delay);
7788	8016	}
	8017	+
7789	8018	EXPORT_SYMBOL(md_write_end);
7790	8019
7791	8020	/* md_allow_write(mddev)
@@ -8385,9 +8614,8 @@ void md_check_recovery(struct mddev *mddev) · 593 unmodified lines
8385	8614	(mddev->sb_flags & ~ (1<<MD_SB_CHANGE_PENDING)) \|\|
8386	8615	test_bit(MD_RECOVERY_NEEDED, &mddev->recovery) \|\|
8387	8616	test_bit(MD_RECOVERY_DONE, &mddev->recovery) \|\|
8388		− test_bit(MD_RELOAD_SB, &mddev->flags) \|\|
8389	8617	(mddev->external == 0 && mddev->safemode == 1) \|\|
8390		− (mddev->safemode == 2 && ! atomic_read(&mddev->writes_pending)
	8618	+ (mddev->safemode == 2
8391	8619	&& !mddev->in_sync && mddev->recovery_cp == MaxSector)
8392	8620	))
8393	8621	return;
@@ -8434,27 +8662,12 @@ void md_check_recovery(struct mddev *mddev) · 40 unmodified lines
8434	8662	rdev->raid_disk < 0)
8435	8663	md_kick_rdev_from_array(rdev);
8436	8664	}
8437		−
8438		− if (test_and_clear_bit(MD_RELOAD_SB, &mddev->flags))
8439		− md_reload_sb(mddev, mddev->good_device_nr);
8440	8665	}
8441	8666
8442		− if (!mddev->external) {
8443		− int did_change = 0;
	8667	+ if (!mddev->external && !mddev->in_sync) {
8444	8668	spin_lock(&mddev->lock);
8445		− if (mddev->safemode &&
8446		− !atomic_read(&mddev->writes_pending) &&
8447		− !mddev->in_sync &&
8448		− mddev->recovery_cp == MaxSector) {
8449		− mddev->in_sync = 1;
8450		− did_change = 1;
8451		− set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
8452		− }
8453		− if (mddev->safemode == 1)
8454		− mddev->safemode = 0;
	8669	+ set_in_sync(mddev);
8455	8670	spin_unlock(&mddev->lock);
8456		− if (did_change)
8457		− sysfs_notify_dirent_safe(mddev->sysfs_state);
8458	8671	}
8459	8672
8460	8673	if (mddev->sb_flags)
@@ -8747,6 +8960,18 @@ static void check_sb_changes(struct mddev mddev, struct md_rdev rdev) · 286 unmodified lines
8747	8960	int role, ret;
8748	8961	char b[BDEVNAME_SIZE];
8749	8962
	8963	+ /*
	8964	+ * If size is changed in another node then we need to
	8965	+ * do resize as well.
	8966	+ */
	8967	+ if (mddev->dev_sectors != le64_to_cpu(sb->size)) {
	8968	+ ret = mddev->pers->resize(mddev, le64_to_cpu(sb->size));
	8969	+ if (ret)
	8970	+ pr_info("md-cluster: resize failed\n");
	8971	+ else
	8972	+ bitmap_update_sb(mddev->bitmap);
	8973	+ }
	8974	+
8750	8975	/* Check for change of roles in the active devices */
8751	8976	rdev_for_each(rdev2, mddev) {
8752	8977	if (test_bit(Faulty, &rdev2->flags))
@@ -8997,6 +9222,7 @@ static int set_ro(const char val, struct kernel_param kp) · 244 unmodified lines
8997	9222	module_param_call(start_ro, set_ro, get_ro, NULL, S_IRUSR\|S_IWUSR);
8998	9223	module_param(start_dirty_degraded, int, S_IRUGO\|S_IWUSR);
8999	9224	module_param_call(new_array, add_named_array, NULL, NULL, S_IWUSR);
	9225	+module_param(create_on_open, bool, S_IRUSR\|S_IWUSR);
9000	9226
9001	9227	MODULE_LICENSE("GPL");
9002	9228	MODULE_DESCRIPTION("MD RAID framework");

drivers/md/md.h+67−4

@@ -122,6 +122,13 @@ struct md_rdev { · 121 unmodified lines
122	122	* sysfs entry */
123	123
124	124	struct badblocks badblocks;
	125	+
	126	+ struct {
	127	+ short offset; /* Offset from superblock to start of PPL.
	128	+ * Not used by external metadata. */
	129	+ unsigned int size; /* Size in sectors of the PPL space */
	130	+ sector_t sector; /* First sector of the PPL space */
	131	+ } ppl;
125	132	};
126	133	enum flag_bits {
127	134	Faulty, /* device is known to have a fault */
@@ -219,9 +226,6 @@ enum mddev_flags { · 91 unmodified lines
219	226	* it then */
220	227	MD_JOURNAL_CLEAN, /* A raid with journal is already clean */
221	228	MD_HAS_JOURNAL, /* The raid array has journal feature set */
222		− MD_RELOAD_SB, /* Reload the superblock because another node
223		− * updated it.
224		− */
225	229	MD_CLUSTER_RESYNC_LOCKED, /* cluster raid only, which means node
226	230	* already took resync lock, need to
227	231	* release the lock */
@@ -229,6 +233,7 @@ enum mddev_flags { · 1 unmodified line
229	233	* supported as calls to md_error() will
230	234	* never cause the array to become failed.
231	235	*/
	236	+ MD_HAS_PPL, /* The raid array has PPL feature set */
232	237	};
233	238
234	239	enum mddev_sb_flags {
@@ -404,7 +409,8 @@ struct mddev { · 169 unmodified lines
404	409	*/
405	410	unsigned int safemode_delay;
406	411	struct timer_list safemode_timer;
407		− atomic_t writes_pending;
	412	+ struct percpu_ref writes_pending;
	413	+ int sync_checkers; /* # of threads checking writes_pending */
408	414	struct request_queue queue; / for plugging ... */
409	415
410	416	struct bitmap bitmap; / the bitmap for the device */
@@ -540,6 +546,8 @@ struct md_personality · 129 unmodified lines
540	546	/* congested implements bdi.congested_fn().
541	547	* Will not be called while array is 'suspended' */
542	548	int (congested)(struct mddev mddev, int bits);
	549	+ /* Changes the consistency policy of an active array. */
	550	+ int (change_consistency_policy)(struct mddev mddev, const char *buf);
543	551	};
544	552
545	553	struct md_sysfs_entry {
@@ -641,6 +649,7 @@ extern void md_wakeup_thread(struct md_thread *thread); · 95 unmodified lines
641	649	extern void md_check_recovery(struct mddev *mddev);
642	650	extern void md_reap_sync_thread(struct mddev *mddev);
643	651	extern void md_write_start(struct mddev mddev, struct bio bi);
	652	+extern void md_write_inc(struct mddev mddev, struct bio bi);
644	653	extern void md_write_end(struct mddev *mddev);
645	654	extern void md_done_sync(struct mddev *mddev, int blocks, int ok);
646	655	extern void md_error(struct mddev mddev, struct md_rdev rdev);
@@ -716,4 +725,58 @@ static inline void mddev_check_write_zeroes(struct mddev mddev, struct bio bio · 69 unmodified lines
716	725	!bdev_get_queue(bio->bi_bdev)->limits.max_write_zeroes_sectors)
717	726	mddev->queue->limits.max_write_zeroes_sectors = 0;
718	727	}
	728	+
	729	+/* Maximum size of each resync request */
	730	+#define RESYNC_BLOCK_SIZE (64*1024)
	731	+#define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
	732	+
	733	+/* for managing resync I/O pages */
	734	+struct resync_pages {
	735	+ unsigned idx; /* for get/put page from the pool */
	736	+ void *raid_bio;
	737	+ struct page *pages[RESYNC_PAGES];
	738	+};
	739	+
	740	+static inline int resync_alloc_pages(struct resync_pages *rp,
	741	+ gfp_t gfp_flags)
	742	+{
	743	+ int i;
	744	+
	745	+ for (i = 0; i < RESYNC_PAGES; i++) {
	746	+ rp->pages[i] = alloc_page(gfp_flags);
	747	+ if (!rp->pages[i])
	748	+ goto out_free;
	749	+ }
	750	+
	751	+ return 0;
	752	+
	753	+out_free:
	754	+ while (--i >= 0)
	755	+ put_page(rp->pages[i]);
	756	+ return -ENOMEM;
	757	+}
	758	+
	759	+static inline void resync_free_pages(struct resync_pages *rp)
	760	+{
	761	+ int i;
	762	+
	763	+ for (i = 0; i < RESYNC_PAGES; i++)
	764	+ put_page(rp->pages[i]);
	765	+}
	766	+
	767	+static inline void resync_get_all_pages(struct resync_pages *rp)
	768	+{
	769	+ int i;
	770	+
	771	+ for (i = 0; i < RESYNC_PAGES; i++)
	772	+ get_page(rp->pages[i]);
	773	+}
	774	+
	775	+static inline struct page resync_fetch_page(struct resync_pages rp,
	776	+ unsigned idx)
	777	+{
	778	+ if (WARN_ON_ONCE(idx >= RESYNC_PAGES))
	779	+ return NULL;
	780	+ return rp->pages[idx];
	781	+}
719	782	#endif /* _MD_MD_H */

drivers/md/raid0.c+40−38

@@ -29,7 +29,8 @@ · 28 unmodified lines
29	29	#define UNSUPPORTED_MDDEV_FLAGS \
30	30	((1L << MD_HAS_JOURNAL) \| \
31	31	(1L << MD_JOURNAL_CLEAN) \| \
32		− (1L << MD_FAILFAST_SUPPORTED))
	32	+ (1L << MD_FAILFAST_SUPPORTED) \|\
	33	+ (1L << MD_HAS_PPL))
33	34
34	35	static int raid0_congested(struct mddev *mddev, int bits)
35	36	{
@@ -462,53 +463,54 @@ static void raid0_make_request(struct mddev mddev, struct bio bio) · 426 unmodified lines
462	463	{
463	464	struct strip_zone *zone;
464	465	struct md_rdev *tmp_dev;
465		− struct bio *split;
	466	+ sector_t bio_sector;
	467	+ sector_t sector;
	468	+ unsigned chunk_sects;
	469	+ unsigned sectors;
466	470
467	471	if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
468	472	md_flush_request(mddev, bio);
469	473	return;
470	474	}
471	475
472		− do {
473		− sector_t bio_sector = bio->bi_iter.bi_sector;
474		− sector_t sector = bio_sector;
475		− unsigned chunk_sects = mddev->chunk_sectors;
	476	+ bio_sector = bio->bi_iter.bi_sector;
	477	+ sector = bio_sector;
	478	+ chunk_sects = mddev->chunk_sectors;
476	479
477		− unsigned sectors = chunk_sects -
478		− (likely(is_power_of_2(chunk_sects))
479		− ? (sector & (chunk_sects-1))
480		− : sector_div(sector, chunk_sects));
	480	+ sectors = chunk_sects -
	481	+ (likely(is_power_of_2(chunk_sects))
	482	+ ? (sector & (chunk_sects-1))
	483	+ : sector_div(sector, chunk_sects));
481	484
482		− /* Restore due to sector_div */
483		− sector = bio_sector;
	485	+ /* Restore due to sector_div */
	486	+ sector = bio_sector;
484	487
485		− if (sectors < bio_sectors(bio)) {
486		− split = bio_split(bio, sectors, GFP_NOIO, fs_bio_set);
487		− bio_chain(split, bio);
488		− } else {
489		− split = bio;
490		− }
	488	+ if (sectors < bio_sectors(bio)) {
	489	+ struct bio *split = bio_split(bio, sectors, GFP_NOIO, mddev->bio_set);
	490	+ bio_chain(split, bio);
	491	+ generic_make_request(bio);
	492	+ bio = split;
	493	+ }
491	494
492		− zone = find_zone(mddev->private, &sector);
493		− tmp_dev = map_sector(mddev, zone, sector, &sector);
494		− split->bi_bdev = tmp_dev->bdev;
495		− split->bi_iter.bi_sector = sector + zone->dev_start +
496		− tmp_dev->data_offset;
497		−
498		− if (unlikely((bio_op(split) == REQ_OP_DISCARD) &&
499		− !blk_queue_discard(bdev_get_queue(split->bi_bdev)))) {
500		− /* Just ignore it */
501		− bio_endio(split);
502		− } else {
503		− if (mddev->gendisk)
504		− trace_block_bio_remap(bdev_get_queue(split->bi_bdev),
505		− split, disk_devt(mddev->gendisk),
506		− bio_sector);
507		− mddev_check_writesame(mddev, split);
508		− mddev_check_write_zeroes(mddev, split);
509		− generic_make_request(split);
510		− }
511		− } while (split != bio);
	495	+ zone = find_zone(mddev->private, &sector);
	496	+ tmp_dev = map_sector(mddev, zone, sector, &sector);
	497	+ bio->bi_bdev = tmp_dev->bdev;
	498	+ bio->bi_iter.bi_sector = sector + zone->dev_start +
	499	+ tmp_dev->data_offset;
	500	+
	501	+ if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
	502	+ !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) {
	503	+ /* Just ignore it */
	504	+ bio_endio(bio);
	505	+ } else {
	506	+ if (mddev->gendisk)
	507	+ trace_block_bio_remap(bdev_get_queue(bio->bi_bdev),
	508	+ bio, disk_devt(mddev->gendisk),
	509	+ bio_sector);
	510	+ mddev_check_writesame(mddev, bio);
	511	+ mddev_check_write_zeroes(mddev, bio);
	512	+ generic_make_request(bio);
	513	+ }
512	514	}
513	515
514	516	static void raid0_status(struct seq_file seq, struct mddev mddev)

drivers/md/raid1.c+289−390

@@ -47,7 +47,8 @@ · 46 unmodified lines
47	47
48	48	#define UNSUPPORTED_MDDEV_FLAGS \
49	49	((1L << MD_HAS_JOURNAL) \| \
50		− (1L << MD_JOURNAL_CLEAN))
	50	+ (1L << MD_JOURNAL_CLEAN) \| \
	51	+ (1L << MD_HAS_PPL))
51	52
52	53	/*
53	54	* Number of guaranteed r1bios in case of extreme VM load:
@@ -80,6 +81,24 @@ static void lower_barrier(struct r1conf *conf, sector_t sector_nr); · 26 unmodified lines
80	81	#define raid1_log(md, fmt, args...) \
81	82	do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid1 " fmt, ##args); } while (0)
82	83
	84	+/*
	85	+ * 'strct resync_pages' stores actual pages used for doing the resync
	86	+ * IO, and it is per-bio, so make .bi_private points to it.
	87	+ */
	88	+static inline struct resync_pages get_resync_pages(struct bio bio)
	89	+{
	90	+ return bio->bi_private;
	91	+}
	92	+
	93	+/*
	94	+ * for resync bio, r1bio pointer can be retrieved from the per-bio
	95	+ * 'struct resync_pages'.
	96	+ */
	97	+static inline struct r1bio get_resync_r1bio(struct bio bio)
	98	+{
	99	+ return get_resync_pages(bio)->raid_bio;
	100	+}
	101	+
83	102	static void * r1bio_pool_alloc(gfp_t gfp_flags, void *data)
84	103	{
85	104	struct pool_info *pi = data;
@@ -94,10 +113,8 @@ static void r1bio_pool_free(void r1_bio, void data) · 8 unmodified lines
94	113	kfree(r1_bio);
95	114	}
96	115
97		−#define RESYNC_BLOCK_SIZE (64*1024)
98	116	#define RESYNC_DEPTH 32
99	117	#define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
100		−#define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
101	118	#define RESYNC_WINDOW (RESYNC_BLOCK_SIZE * RESYNC_DEPTH)
102	119	#define RESYNC_WINDOW_SECTORS (RESYNC_WINDOW >> 9)
103	120	#define CLUSTER_RESYNC_WINDOW (16 * RESYNC_WINDOW)
@@ -109,12 +126,18 @@ static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data) · 5 unmodified lines
109	126	struct r1bio *r1_bio;
110	127	struct bio *bio;
111	128	int need_pages;
112		− int i, j;
	129	+ int j;
	130	+ struct resync_pages *rps;
113	131
114	132	r1_bio = r1bio_pool_alloc(gfp_flags, pi);
115	133	if (!r1_bio)
116	134	return NULL;
117	135
	136	+ rps = kmalloc(sizeof(struct resync_pages) * pi->raid_disks,
	137	+ gfp_flags);
	138	+ if (!rps)
	139	+ goto out_free_r1bio;
	140	+
118	141	/*
119	142	* Allocate bios : 1 for reading, n-1 for writing
120	143	*/
@@ -134,19 +157,22 @@ static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data) · 13 unmodified lines
134	157	need_pages = pi->raid_disks;
135	158	else
136	159	need_pages = 1;
137		− for (j = 0; j < need_pages; j++) {
	160	+ for (j = 0; j < pi->raid_disks; j++) {
	161	+ struct resync_pages *rp = &rps[j];
	162	+
138	163	bio = r1_bio->bios[j];
139		− bio->bi_vcnt = RESYNC_PAGES;
140	164
141		− if (bio_alloc_pages(bio, gfp_flags))
142		− goto out_free_pages;
143		− }
144		− /* If not user-requests, copy the page pointers to all bios */
145		− if (!test_bit(MD_RECOVERY_REQUESTED, &pi->mddev->recovery)) {
146		− for (i=0; i<RESYNC_PAGES ; i++)
147		− for (j=1; j<pi->raid_disks; j++)
148		− r1_bio->bios[j]->bi_io_vec[i].bv_page =
149		− r1_bio->bios[0]->bi_io_vec[i].bv_page;
	165	+ if (j < need_pages) {
	166	+ if (resync_alloc_pages(rp, gfp_flags))
	167	+ goto out_free_pages;
	168	+ } else {
	169	+ memcpy(rp, &rps[0], sizeof(*rp));
	170	+ resync_get_all_pages(rp);
	171	+ }
	172	+
	173	+ rp->idx = 0;
	174	+ rp->raid_bio = r1_bio;
	175	+ bio->bi_private = rp;
150	176	}
151	177
152	178	r1_bio->master_bio = NULL;
@@ -155,11 +181,14 @@ static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data) · 2 unmodified lines
155	181
156	182	out_free_pages:
157	183	while (--j >= 0)
158		− bio_free_pages(r1_bio->bios[j]);
	184	+ resync_free_pages(&rps[j]);
159	185
160	186	out_free_bio:
161	187	while (++j < pi->raid_disks)
162	188	bio_put(r1_bio->bios[j]);
	189	+ kfree(rps);
	190	+
	191	+out_free_r1bio:
163	192	r1bio_pool_free(r1_bio, data);
164	193	return NULL;
165	194	}
@@ -167,18 +196,18 @@ out_free_bio: · 1 unmodified line
167	196	static void r1buf_pool_free(void __r1_bio, void data)
168	197	{
169	198	struct pool_info *pi = data;
170		− int i,j;
	199	+ int i;
171	200	struct r1bio *r1bio = __r1_bio;
	201	+ struct resync_pages *rp = NULL;
172	202
173		− for (i = 0; i < RESYNC_PAGES; i++)
174		− for (j = pi->raid_disks; j-- ;) {
175		− if (j == 0 \|\|
176		− r1bio->bios[j]->bi_io_vec[i].bv_page !=
177		− r1bio->bios[0]->bi_io_vec[i].bv_page)
178		− safe_put_page(r1bio->bios[j]->bi_io_vec[i].bv_page);
179		− }
180		− for (i=0 ; i < pi->raid_disks; i++)
	203	+ for (i = pi->raid_disks; i--; ) {
	204	+ rp = get_resync_pages(r1bio->bios[i]);
	205	+ resync_free_pages(rp);
181	206	bio_put(r1bio->bios[i]);
	207	+ }
	208	+
	209	+ /* resync pages array stored in the 1st bio's .bi_private */
	210	+ kfree(rp);
182	211
183	212	r1bio_pool_free(r1bio, data);
184	213	}
@@ -245,35 +274,17 @@ static void reschedule_retry(struct r1bio *r1_bio) · 60 unmodified lines
245	274	static void call_bio_endio(struct r1bio *r1_bio)
246	275	{
247	276	struct bio *bio = r1_bio->master_bio;
248		− int done;
249	277	struct r1conf *conf = r1_bio->mddev->private;
250		− sector_t bi_sector = bio->bi_iter.bi_sector;
251		−
252		− if (bio->bi_phys_segments) {
253		− unsigned long flags;
254		− spin_lock_irqsave(&conf->device_lock, flags);
255		− bio->bi_phys_segments--;
256		− done = (bio->bi_phys_segments == 0);
257		− spin_unlock_irqrestore(&conf->device_lock, flags);
258		− /*
259		− * make_request() might be waiting for
260		− * bi_phys_segments to decrease
261		− */
262		− wake_up(&conf->wait_barrier);
263		− } else
264		− done = 1;
265	278
266	279	if (!test_bit(R1BIO_Uptodate, &r1_bio->state))
267	280	bio->bi_error = -EIO;
268	281
269		− if (done) {
270		− bio_endio(bio);
271		− /*
272		− * Wake up any possible resync thread that waits for the device
273		− * to go idle.
274		− */
275		− allow_barrier(conf, bi_sector);
276		− }
	282	+ bio_endio(bio);
	283	+ /*
	284	+ * Wake up any possible resync thread that waits for the device
	285	+ * to go idle.
	286	+ */
	287	+ allow_barrier(conf, r1_bio->sector);
277	288	}
278	289
279	290	static void raid_end_bio_io(struct r1bio *r1_bio)
@@ -377,12 +388,9 @@ static void close_write(struct r1bio *r1_bio) · 97 unmodified lines
377	388	{
378	389	/* it really is the end of this request */
379	390	if (test_bit(R1BIO_BehindIO, &r1_bio->state)) {
380		− /* free extra copy of the data pages */
381		− int i = r1_bio->behind_page_count;
382		− while (i--)
383		− safe_put_page(r1_bio->behind_bvecs[i].bv_page);
384		− kfree(r1_bio->behind_bvecs);
385		− r1_bio->behind_bvecs = NULL;
	391	+ bio_free_pages(r1_bio->behind_master_bio);
	392	+ bio_put(r1_bio->behind_master_bio);
	393	+ r1_bio->behind_master_bio = NULL;
386	394	}
387	395	/* clear the bitmap if all writes complete successfully */
388	396	bitmap_endwrite(r1_bio->mddev->bitmap, r1_bio->sector,
@@ -484,6 +492,10 @@ static void raid1_end_write_request(struct bio *bio) · 95 unmodified lines
484	492	}
485	493
486	494	if (behind) {
	495	+ /* we release behind master bio when all write are done */
	496	+ if (r1_bio->behind_master_bio == bio)
	497	+ to_put = NULL;
	498	+
487	499	if (test_bit(WriteMostly, &rdev->flags))
488	500	atomic_dec(&r1_bio->behind_remaining);
489	501
@@ -775,6 +787,30 @@ static int raid1_congested(struct mddev *mddev, int bits) · 285 unmodified lines
775	787	return ret;
776	788	}
777	789
	790	+static void flush_bio_list(struct r1conf conf, struct bio bio)
	791	+{
	792	+ /* flush any pending bitmap writes to disk before proceeding w/ I/O */
	793	+ bitmap_unplug(conf->mddev->bitmap);
	794	+ wake_up(&conf->wait_barrier);
	795	+
	796	+ while (bio) { /* submit pending writes */
	797	+ struct bio *next = bio->bi_next;
	798	+ struct md_rdev rdev = (void)bio->bi_bdev;
	799	+ bio->bi_next = NULL;
	800	+ bio->bi_bdev = rdev->bdev;
	801	+ if (test_bit(Faulty, &rdev->flags)) {
	802	+ bio->bi_error = -EIO;
	803	+ bio_endio(bio);
	804	+ } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
	805	+ !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
	806	+ /* Just ignore it */
	807	+ bio_endio(bio);
	808	+ else
	809	+ generic_make_request(bio);
	810	+ bio = next;
	811	+ }
	812	+}
	813	+
778	814	static void flush_pending_writes(struct r1conf *conf)
779	815	{
780	816	/* Any writes that have been queued but are awaiting
@@ -787,27 +823,7 @@ static void flush_pending_writes(struct r1conf *conf) · 6 unmodified lines
787	823	bio = bio_list_get(&conf->pending_bio_list);
788	824	conf->pending_count = 0;
789	825	spin_unlock_irq(&conf->device_lock);
790		− /* flush any pending bitmap writes to
791		− * disk before proceeding w/ I/O */
792		− bitmap_unplug(conf->mddev->bitmap);
793		− wake_up(&conf->wait_barrier);
794		−
795		− while (bio) { /* submit pending writes */
796		− struct bio *next = bio->bi_next;
797		− struct md_rdev rdev = (void)bio->bi_bdev;
798		− bio->bi_next = NULL;
799		− bio->bi_bdev = rdev->bdev;
800		− if (test_bit(Faulty, &rdev->flags)) {
801		− bio->bi_error = -EIO;
802		− bio_endio(bio);
803		− } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
804		− !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
805		− /* Just ignore it */
806		− bio_endio(bio);
807		− else
808		− generic_make_request(bio);
809		− bio = next;
810		− }
	826	+ flush_bio_list(conf, bio);
811	827	} else
812	828	spin_unlock_irq(&conf->device_lock);
813	829	}
@@ -869,7 +885,7 @@ static void raise_barrier(struct r1conf *conf, sector_t sector_nr) · 55 unmodified lines
869	885	atomic_read(&conf->barrier[idx]) < RESYNC_DEPTH,
870	886	conf->resync_lock);
871	887
872		− atomic_inc(&conf->nr_pending[idx]);
	888	+ atomic_inc(&conf->nr_sync_pending);
873	889	spin_unlock_irq(&conf->resync_lock);
874	890	}
875	891
@@ -880,7 +896,7 @@ static void lower_barrier(struct r1conf *conf, sector_t sector_nr) · 4 unmodified lines
880	896	BUG_ON(atomic_read(&conf->barrier[idx]) <= 0);
881	897
882	898	atomic_dec(&conf->barrier[idx]);
883		− atomic_dec(&conf->nr_pending[idx]);
	899	+ atomic_dec(&conf->nr_sync_pending);
884	900	wake_up(&conf->wait_barrier);
885	901	}
886	902
@@ -1017,7 +1033,8 @@ static int get_unqueued_pending(struct r1conf *conf) · 130 unmodified lines
1017	1033	{
1018	1034	int idx, ret;
1019	1035
1020		− for (ret = 0, idx = 0; idx < BARRIER_BUCKETS_NR; idx++)
	1036	+ ret = atomic_read(&conf->nr_sync_pending);
	1037	+ for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++)
1021	1038	ret += atomic_read(&conf->nr_pending[idx]) -
1022	1039	atomic_read(&conf->nr_queued[idx]);
1023	1040
@@ -1068,39 +1085,49 @@ static void unfreeze_array(struct r1conf *conf) · 44 unmodified lines
1068	1085	wake_up(&conf->wait_barrier);
1069	1086	}
1070	1087
1071		−/* duplicate the data pages for behind I/O
1072		− */
1073		−static void alloc_behind_pages(struct bio bio, struct r1bio r1_bio)
	1088	+static struct bio alloc_behind_master_bio(struct r1bio r1_bio,
	1089	+ struct bio *bio)
1074	1090	{
1075		− int i;
1076		− struct bio_vec *bvec;
1077		− struct bio_vec bvecs = kzalloc(bio->bi_vcnt sizeof(struct bio_vec),
1078		− GFP_NOIO);
1079		− if (unlikely(!bvecs))
1080		− return;
	1091	+ int size = bio->bi_iter.bi_size;
	1092	+ unsigned vcnt = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
	1093	+ int i = 0;
	1094	+ struct bio *behind_bio = NULL;
	1095	+
	1096	+ behind_bio = bio_alloc_mddev(GFP_NOIO, vcnt, r1_bio->mddev);
	1097	+ if (!behind_bio)
	1098	+ goto fail;
1081	1099
1082		− bio_for_each_segment_all(bvec, bio, i) {
1083		− bvecs[i] = *bvec;
1084		− bvecs[i].bv_page = alloc_page(GFP_NOIO);
1085		− if (unlikely(!bvecs[i].bv_page))
1086		− goto do_sync_io;
1087		− memcpy(kmap(bvecs[i].bv_page) + bvec->bv_offset,
1088		− kmap(bvec->bv_page) + bvec->bv_offset, bvec->bv_len);
1089		− kunmap(bvecs[i].bv_page);
1090		− kunmap(bvec->bv_page);
1091		− }
1092		− r1_bio->behind_bvecs = bvecs;
1093		− r1_bio->behind_page_count = bio->bi_vcnt;
	1100	+ /* discard op, we don't support writezero/writesame yet */
	1101	+ if (!bio_has_data(bio))
	1102	+ goto skip_copy;
	1103	+
	1104	+ while (i < vcnt && size) {
	1105	+ struct page *page;
	1106	+ int len = min_t(int, PAGE_SIZE, size);
	1107	+
	1108	+ page = alloc_page(GFP_NOIO);
	1109	+ if (unlikely(!page))
	1110	+ goto free_pages;
	1111	+
	1112	+ bio_add_page(behind_bio, page, len, 0);
	1113	+
	1114	+ size -= len;
	1115	+ i++;
	1116	+ }
	1117	+
	1118	+ bio_copy_data(behind_bio, bio);
	1119	+skip_copy:
	1120	+ r1_bio->behind_master_bio = behind_bio;;
1094	1121	set_bit(R1BIO_BehindIO, &r1_bio->state);
1095		− return;
1096	1122
1097		−do_sync_io:
1098		− for (i = 0; i < bio->bi_vcnt; i++)
1099		− if (bvecs[i].bv_page)
1100		− put_page(bvecs[i].bv_page);
1101		− kfree(bvecs);
	1123	+ return behind_bio;
	1124	+
	1125	+free_pages:
1102	1126	pr_debug("%dB behind alloc failed, doing sync I/O\n",
1103	1127	bio->bi_iter.bi_size);
	1128	+ bio_free_pages(behind_bio);
	1129	+fail:
	1130	+ return behind_bio;
1104	1131	}
1105	1132
1106	1133	struct raid1_plug_cb {
@@ -1130,91 +1157,102 @@ static void raid1_unplug(struct blk_plug_cb *cb, bool from_schedule) · 23 unmodified lines
1130	1157
1131	1158	/* we aren't scheduling, so we can do the write-out directly. */
1132	1159	bio = bio_list_get(&plug->pending);
1133		− bitmap_unplug(mddev->bitmap);
1134		− wake_up(&conf->wait_barrier);
1135		−
1136		− while (bio) { /* submit pending writes */
1137		− struct bio *next = bio->bi_next;
1138		− struct md_rdev rdev = (void)bio->bi_bdev;
1139		− bio->bi_next = NULL;
1140		− bio->bi_bdev = rdev->bdev;
1141		− if (test_bit(Faulty, &rdev->flags)) {
1142		− bio->bi_error = -EIO;
1143		− bio_endio(bio);
1144		− } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
1145		− !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
1146		− /* Just ignore it */
1147		− bio_endio(bio);
1148		− else
1149		− generic_make_request(bio);
1150		− bio = next;
1151		− }
	1160	+ flush_bio_list(conf, bio);
1152	1161	kfree(plug);
1153	1162	}
1154	1163
	1164	+static void init_r1bio(struct r1bio r1_bio, struct mddev mddev, struct bio *bio)
	1165	+{
	1166	+ r1_bio->master_bio = bio;
	1167	+ r1_bio->sectors = bio_sectors(bio);
	1168	+ r1_bio->state = 0;
	1169	+ r1_bio->mddev = mddev;
	1170	+ r1_bio->sector = bio->bi_iter.bi_sector;
	1171	+}
	1172	+
1155	1173	static inline struct r1bio *
1156		−alloc_r1bio(struct mddev mddev, struct bio bio, sector_t sectors_handled)
	1174	+alloc_r1bio(struct mddev mddev, struct bio bio)
1157	1175	{
1158	1176	struct r1conf *conf = mddev->private;
1159	1177	struct r1bio *r1_bio;
1160	1178
1161	1179	r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO);
1162		−
1163		− r1_bio->master_bio = bio;
1164		− r1_bio->sectors = bio_sectors(bio) - sectors_handled;
1165		− r1_bio->state = 0;
1166		− r1_bio->mddev = mddev;
1167		− r1_bio->sector = bio->bi_iter.bi_sector + sectors_handled;
1168		−
	1180	+ /* Ensure no bio records IO_BLOCKED */
	1181	+ memset(r1_bio->bios, 0, conf->raid_disks * sizeof(r1_bio->bios[0]));
	1182	+ init_r1bio(r1_bio, mddev, bio);
1169	1183	return r1_bio;
1170	1184	}
1171	1185
1172		−static void raid1_read_request(struct mddev mddev, struct bio bio)
	1186	+static void raid1_read_request(struct mddev mddev, struct bio bio,
	1187	+ int max_read_sectors, struct r1bio *r1_bio)
1173	1188	{
1174	1189	struct r1conf *conf = mddev->private;
1175	1190	struct raid1_info *mirror;
1176		− struct r1bio *r1_bio;
1177	1191	struct bio *read_bio;
1178	1192	struct bitmap *bitmap = mddev->bitmap;
1179	1193	const int op = bio_op(bio);
1180	1194	const unsigned long do_sync = (bio->bi_opf & REQ_SYNC);
1181		− int sectors_handled;
1182	1195	int max_sectors;
1183	1196	int rdisk;
	1197	+ bool print_msg = !!r1_bio;
	1198	+ char b[BDEVNAME_SIZE];
1184	1199
1185	1200	/*
1186		− * Still need barrier for READ in case that whole
1187		− * array is frozen.
	1201	+ * If r1_bio is set, we are blocking the raid1d thread
	1202	+ * so there is a tiny risk of deadlock. So ask for
	1203	+ * emergency memory if needed.
1188	1204	*/
1189		− wait_read_barrier(conf, bio->bi_iter.bi_sector);
	1205	+ gfp_t gfp = r1_bio ? (GFP_NOIO \| __GFP_HIGH) : GFP_NOIO;
1190	1206
1191		− r1_bio = alloc_r1bio(mddev, bio, 0);
	1207	+ if (print_msg) {
	1208	+ /* Need to get the block device name carefully */
	1209	+ struct md_rdev *rdev;
	1210	+ rcu_read_lock();
	1211	+ rdev = rcu_dereference(conf->mirrors[r1_bio->read_disk].rdev);
	1212	+ if (rdev)
	1213	+ bdevname(rdev->bdev, b);
	1214	+ else
	1215	+ strcpy(b, "???");
	1216	+ rcu_read_unlock();
	1217	+ }
1192	1218
1193	1219	/*
1194		− * We might need to issue multiple reads to different
1195		− * devices if there are bad blocks around, so we keep
1196		− * track of the number of reads in bio->bi_phys_segments.
1197		− * If this is 0, there is only one r1_bio and no locking
1198		− * will be needed when requests complete. If it is
1199		− * non-zero, then it is the number of not-completed requests.
	1220	+ * Still need barrier for READ in case that whole
	1221	+ * array is frozen.
1200	1222	*/
1201		− bio->bi_phys_segments = 0;
1202		− bio_clear_flag(bio, BIO_SEG_VALID);
	1223	+ wait_read_barrier(conf, bio->bi_iter.bi_sector);
	1224	+
	1225	+ if (!r1_bio)
	1226	+ r1_bio = alloc_r1bio(mddev, bio);
	1227	+ else
	1228	+ init_r1bio(r1_bio, mddev, bio);
	1229	+ r1_bio->sectors = max_read_sectors;
1203	1230
1204	1231	/*
1205	1232	* make_request() can abort the operation when read-ahead is being
1206	1233	* used and no empty request is available.
1207	1234	*/
1208		−read_again:
1209	1235	rdisk = read_balance(conf, r1_bio, &max_sectors);
1210	1236
1211	1237	if (rdisk < 0) {
1212	1238	/* couldn't find anywhere to read from */
	1239	+ if (print_msg) {
	1240	+ pr_crit_ratelimited("md/raid1:%s: %s: unrecoverable I/O read error for block %llu\n",
	1241	+ mdname(mddev),
	1242	+ b,
	1243	+ (unsigned long long)r1_bio->sector);
	1244	+ }
1213	1245	raid_end_bio_io(r1_bio);
1214	1246	return;
1215	1247	}
1216	1248	mirror = conf->mirrors + rdisk;
1217	1249
	1250	+ if (print_msg)
	1251	+ pr_info_ratelimited("md/raid1:%s: redirecting sector %llu to other mirror: %s\n",
	1252	+ mdname(mddev),
	1253	+ (unsigned long long)r1_bio->sector,
	1254	+ bdevname(mirror->rdev->bdev, b));
	1255	+
1218	1256	if (test_bit(WriteMostly, &mirror->rdev->flags) &&
1219	1257	bitmap) {
1220	1258	/*
@@ -1225,11 +1263,20 @@ read_again: · 4 unmodified lines
1225	1263	wait_event(bitmap->behind_wait,
1226	1264	atomic_read(&bitmap->behind_writes) == 0);
1227	1265	}
	1266	+
	1267	+ if (max_sectors < bio_sectors(bio)) {
	1268	+ struct bio *split = bio_split(bio, max_sectors,
	1269	+ gfp, conf->bio_split);
	1270	+ bio_chain(split, bio);
	1271	+ generic_make_request(bio);
	1272	+ bio = split;
	1273	+ r1_bio->master_bio = bio;
	1274	+ r1_bio->sectors = max_sectors;
	1275	+ }
	1276	+
1228	1277	r1_bio->read_disk = rdisk;
1229	1278
1230		− read_bio = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
1231		− bio_trim(read_bio, r1_bio->sector - bio->bi_iter.bi_sector,
1232		− max_sectors);
	1279	+ read_bio = bio_clone_fast(bio, gfp, mddev->bio_set);
1233	1280
1234	1281	r1_bio->bios[rdisk] = read_bio;
1235	1282
@@ -1248,35 +1295,11 @@ read_again: · 12 unmodified lines
1248	1295	read_bio, disk_devt(mddev->gendisk),
1249	1296	r1_bio->sector);
1250	1297
1251		− if (max_sectors < r1_bio->sectors) {
1252		− /*
1253		− * could not read all from this device, so we will need another
1254		− * r1_bio.
1255		− */
1256		− sectors_handled = (r1_bio->sector + max_sectors
1257		− - bio->bi_iter.bi_sector);
1258		− r1_bio->sectors = max_sectors;
1259		− spin_lock_irq(&conf->device_lock);
1260		− if (bio->bi_phys_segments == 0)
1261		− bio->bi_phys_segments = 2;
1262		− else
1263		− bio->bi_phys_segments++;
1264		− spin_unlock_irq(&conf->device_lock);
1265		−
1266		− /*
1267		− * Cannot call generic_make_request directly as that will be
1268		− * queued in __make_request and subsequent mempool_alloc might
1269		− * block waiting for it. So hand bio over to raid1d.
1270		− */
1271		− reschedule_retry(r1_bio);
1272		−
1273		− r1_bio = alloc_r1bio(mddev, bio, sectors_handled);
1274		− goto read_again;
1275		− } else
1276		− generic_make_request(read_bio);
	1298	+ generic_make_request(read_bio);
1277	1299	}
1278	1300
1279		−static void raid1_write_request(struct mddev mddev, struct bio bio)
	1301	+static void raid1_write_request(struct mddev mddev, struct bio bio,
	1302	+ int max_write_sectors)
1280	1303	{
1281	1304	struct r1conf *conf = mddev->private;
1282	1305	struct r1bio *r1_bio;
@@ -1287,7 +1310,6 @@ static void raid1_write_request(struct mddev mddev, struct bio bio) · 4 unmodified lines
1287	1310	struct blk_plug_cb *cb;
1288	1311	struct raid1_plug_cb *plug = NULL;
1289	1312	int first_clone;
1290		− int sectors_handled;
1291	1313	int max_sectors;
1292	1314
1293	1315	/*
@@ -1326,17 +1348,8 @@ static void raid1_write_request(struct mddev mddev, struct bio bio) · 32 unmodified lines
1326	1348	}
1327	1349	wait_barrier(conf, bio->bi_iter.bi_sector);
1328	1350
1329		− r1_bio = alloc_r1bio(mddev, bio, 0);
1330		−
1331		− /* We might need to issue multiple writes to different
1332		− * devices if there are bad blocks around, so we keep
1333		− * track of the number of writes in bio->bi_phys_segments.
1334		− * If this is 0, there is only one r1_bio and no locking
1335		− * will be needed when requests complete. If it is
1336		− * non-zero, then it is the number of not-completed requests.
1337		− */
1338		− bio->bi_phys_segments = 0;
1339		− bio_clear_flag(bio, BIO_SEG_VALID);
	1351	+ r1_bio = alloc_r1bio(mddev, bio);
	1352	+ r1_bio->sectors = max_write_sectors;
1340	1353
1341	1354	if (conf->pending_count >= max_queued_requests) {
1342	1355	md_wakeup_thread(mddev->thread);
@@ -1435,31 +1448,26 @@ static void raid1_write_request(struct mddev mddev, struct bio bio) · 92 unmodified lines
1435	1448	goto retry_write;
1436	1449	}
1437	1450
1438		− if (max_sectors < r1_bio->sectors) {
1439		− /* We are splitting this write into multiple parts, so
1440		− * we need to prepare for allocating another r1_bio.
1441		− */
	1451	+ if (max_sectors < bio_sectors(bio)) {
	1452	+ struct bio *split = bio_split(bio, max_sectors,
	1453	+ GFP_NOIO, conf->bio_split);
	1454	+ bio_chain(split, bio);
	1455	+ generic_make_request(bio);
	1456	+ bio = split;
	1457	+ r1_bio->master_bio = bio;
1442	1458	r1_bio->sectors = max_sectors;
1443		− spin_lock_irq(&conf->device_lock);
1444		− if (bio->bi_phys_segments == 0)
1445		− bio->bi_phys_segments = 2;
1446		− else
1447		− bio->bi_phys_segments++;
1448		− spin_unlock_irq(&conf->device_lock);
1449	1459	}
1450		− sectors_handled = r1_bio->sector + max_sectors - bio->bi_iter.bi_sector;
1451	1460
1452	1461	atomic_set(&r1_bio->remaining, 1);
1453	1462	atomic_set(&r1_bio->behind_remaining, 0);
1454	1463
1455	1464	first_clone = 1;
	1465	+
1456	1466	for (i = 0; i < disks; i++) {
1457	1467	struct bio *mbio = NULL;
1458		− sector_t offset;
1459	1468	if (!r1_bio->bios[i])
1460	1469	continue;
1461	1470
1462		− offset = r1_bio->sector - bio->bi_iter.bi_sector;
1463	1471
1464	1472	if (first_clone) {
1465	1473	/* do behind I/O ?
@@ -1470,11 +1478,7 @@ static void raid1_write_request(struct mddev mddev, struct bio bio) · 4 unmodified lines
1470	1478	(atomic_read(&bitmap->behind_writes)
1471	1479	< mddev->bitmap_info.max_write_behind) &&
1472	1480	!waitqueue_active(&bitmap->behind_wait)) {
1473		− mbio = bio_clone_bioset_partial(bio, GFP_NOIO,
1474		− mddev->bio_set,
1475		− offset << 9,
1476		− max_sectors << 9);
1477		− alloc_behind_pages(mbio, r1_bio);
	1481	+ mbio = alloc_behind_master_bio(r1_bio, bio);
1478	1482	}
1479	1483
1480	1484	bitmap_startwrite(bitmap, r1_bio->sector,
@@ -1485,26 +1489,15 @@ static void raid1_write_request(struct mddev mddev, struct bio bio) · 4 unmodified lines
1485	1489	}
1486	1490
1487	1491	if (!mbio) {
1488		− if (r1_bio->behind_bvecs)
1489		− mbio = bio_clone_bioset_partial(bio, GFP_NOIO,
1490		− mddev->bio_set,
1491		− offset << 9,
1492		− max_sectors << 9);
1493		− else {
	1492	+ if (r1_bio->behind_master_bio)
	1493	+ mbio = bio_clone_fast(r1_bio->behind_master_bio,
	1494	+ GFP_NOIO,
	1495	+ mddev->bio_set);
	1496	+ else
1494	1497	mbio = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
1495		− bio_trim(mbio, offset, max_sectors);
1496		− }
1497	1498	}
1498	1499
1499		− if (r1_bio->behind_bvecs) {
1500		− struct bio_vec *bvec;
1501		− int j;
1502		−
1503		− /*
1504		− * We trimmed the bio, so _all is legit
1505		− */
1506		− bio_for_each_segment_all(bvec, mbio, j)
1507		− bvec->bv_page = r1_bio->behind_bvecs[j].bv_page;
	1500	+ if (r1_bio->behind_master_bio) {
1508	1501	if (test_bit(WriteMostly, &conf->mirrors[i].rdev->flags))
1509	1502	atomic_inc(&r1_bio->behind_remaining);
1510	1503	}
@@ -1548,17 +1541,6 @@ static void raid1_write_request(struct mddev mddev, struct bio bio) · 37 unmodified lines
1548	1541	if (!plug)
1549	1542	md_wakeup_thread(mddev->thread);
1550	1543	}
1551		− /* Mustn't call r1_bio_write_done before this next test,
1552		− * as it could result in the bio being freed.
1553		− */
1554		− if (sectors_handled < bio_sectors(bio)) {
1555		− r1_bio_write_done(r1_bio);
1556		− /* We need another r1_bio. It has already been counted
1557		− * in bio->bi_phys_segments
1558		− */
1559		− r1_bio = alloc_r1bio(mddev, bio, sectors_handled);
1560		− goto retry_write;
1561		− }
1562	1544
1563	1545	r1_bio_write_done(r1_bio);
1564	1546
@@ -1568,7 +1550,6 @@ static void raid1_write_request(struct mddev mddev, struct bio bio) · 3 unmodified lines
1568	1550
1569	1551	static void raid1_make_request(struct mddev mddev, struct bio bio)
1570	1552	{
1571		− struct bio *split;
1572	1553	sector_t sectors;
1573	1554
1574	1555	if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
@@ -1576,43 +1557,20 @@ static void raid1_make_request(struct mddev mddev, struct bio bio) · 1 unmodified line
1576	1557	return;
1577	1558	}
1578	1559
1579		− /* if bio exceeds barrier unit boundary, split it */
1580		− do {
1581		− sectors = align_to_barrier_unit_end(
1582		− bio->bi_iter.bi_sector, bio_sectors(bio));
1583		− if (sectors < bio_sectors(bio)) {
1584		− split = bio_split(bio, sectors, GFP_NOIO, fs_bio_set);
1585		− bio_chain(split, bio);
1586		− } else {
1587		− split = bio;
1588		− }
1589		−
1590		− if (bio_data_dir(split) == READ) {
1591		− raid1_read_request(mddev, split);
	1560	+ /*
	1561	+ * There is a limit to the maximum size, but
	1562	+ * the read/write handler might find a lower limit
	1563	+ * due to bad blocks. To avoid multiple splits,
	1564	+ * we pass the maximum number of sectors down
	1565	+ * and let the lower level perform the split.
	1566	+ */
	1567	+ sectors = align_to_barrier_unit_end(
	1568	+ bio->bi_iter.bi_sector, bio_sectors(bio));
1592	1569
1593		− /*
1594		− * If a bio is splitted, the first part of bio will
1595		− * pass barrier but the bio is queued in
1596		− * current->bio_list (see generic_make_request). If
1597		− * there is a raise_barrier() called here, the second
1598		− * part of bio can't pass barrier. But since the first
1599		− * part bio isn't dispatched to underlaying disks yet,
1600		− * the barrier is never released, hence raise_barrier
1601		− * will alays wait. We have a deadlock.
1602		− * Note, this only happens in read path. For write
1603		− * path, the first part of bio is dispatched in a
1604		− * schedule() call (because of blk plug) or offloaded
1605		− * to raid10d.
1606		− * Quitting from the function immediately can change
1607		− * the bio order queued in bio_list and avoid the deadlock.
1608		− */
1609		− if (split != bio) {
1610		− generic_make_request(bio);
1611		− break;
1612		− }
1613		− } else
1614		− raid1_write_request(mddev, split);
1615		− } while (split != bio);
	1570	+ if (bio_data_dir(bio) == READ)
	1571	+ raid1_read_request(mddev, bio, sectors, NULL);
	1572	+ else
	1573	+ raid1_write_request(mddev, bio, sectors);
1616	1574	}
1617	1575
1618	1576	static void raid1_status(struct seq_file seq, struct mddev mddev)
@@ -1874,9 +1832,9 @@ static int raid1_remove_disk(struct mddev mddev, struct md_rdev rdev) · 255 unmodified lines
1874	1832	p->rdev = repl;
1875	1833	conf->mirrors[conf->raid_disks + number].rdev = NULL;
1876	1834	unfreeze_array(conf);
1877		− clear_bit(WantReplacement, &rdev->flags);
1878		− } else
1879		− clear_bit(WantReplacement, &rdev->flags);
	1835	+ }
	1836	+
	1837	+ clear_bit(WantReplacement, &rdev->flags);
1880	1838	err = md_integrity_register(mddev);
1881	1839	}
1882	1840	abort:
@@ -1887,7 +1845,7 @@ abort: · 4 unmodified lines
1887	1845
1888	1846	static void end_sync_read(struct bio *bio)
1889	1847	{
1890		− struct r1bio *r1_bio = bio->bi_private;
	1848	+ struct r1bio *r1_bio = get_resync_r1bio(bio);
1891	1849
1892	1850	update_head_pos(r1_bio->read_disk, r1_bio);
1893	1851
@@ -1906,7 +1864,7 @@ static void end_sync_read(struct bio *bio) · 12 unmodified lines
1906	1864	static void end_sync_write(struct bio *bio)
1907	1865	{
1908	1866	int uptodate = !bio->bi_error;
1909		− struct r1bio *r1_bio = bio->bi_private;
	1867	+ struct r1bio *r1_bio = get_resync_r1bio(bio);
1910	1868	struct mddev *mddev = r1_bio->mddev;
1911	1869	struct r1conf *conf = mddev->private;
1912	1870	sector_t first_bad;
@@ -1985,6 +1943,7 @@ static int fix_sync_read_error(struct r1bio *r1_bio) · 72 unmodified lines
1985	1943	struct mddev *mddev = r1_bio->mddev;
1986	1944	struct r1conf *conf = mddev->private;
1987	1945	struct bio *bio = r1_bio->bios[r1_bio->read_disk];
	1946	+ struct page **pages = get_resync_pages(bio)->pages;
1988	1947	sector_t sect = r1_bio->sector;
1989	1948	int sectors = r1_bio->sectors;
1990	1949	int idx = 0;
@@ -2018,7 +1977,7 @@ static int fix_sync_read_error(struct r1bio *r1_bio) · 27 unmodified lines
2018	1977	*/
2019	1978	rdev = conf->mirrors[d].rdev;
2020	1979	if (sync_page_io(rdev, sect, s<<9,
2021		− bio->bi_io_vec[idx].bv_page,
	1980	+ pages[idx],
2022	1981	REQ_OP_READ, 0, false)) {
2023	1982	success = 1;
2024	1983	break;
@@ -2073,7 +2032,7 @@ static int fix_sync_read_error(struct r1bio *r1_bio) · 48 unmodified lines
2073	2032	continue;
2074	2033	rdev = conf->mirrors[d].rdev;
2075	2034	if (r1_sync_page_io(rdev, sect, s,
2076		− bio->bi_io_vec[idx].bv_page,
	2035	+ pages[idx],
2077	2036	WRITE) == 0) {
2078	2037	r1_bio->bios[d]->bi_end_io = NULL;
2079	2038	rdev_dec_pending(rdev, mddev);
@@ -2088,7 +2047,7 @@ static int fix_sync_read_error(struct r1bio *r1_bio) · 8 unmodified lines
2088	2047	continue;
2089	2048	rdev = conf->mirrors[d].rdev;
2090	2049	if (r1_sync_page_io(rdev, sect, s,
2091		− bio->bi_io_vec[idx].bv_page,
	2050	+ pages[idx],
2092	2051	READ) != 0)
2093	2052	atomic_add(s, &rdev->corrected_errors);
2094	2053	}
@@ -2122,7 +2081,9 @@ static void process_checks(struct r1bio *r1_bio) · 27 unmodified lines
2122	2081	int j;
2123	2082	int size;
2124	2083	int error;
	2084	+ struct bio_vec *bi;
2125	2085	struct bio *b = r1_bio->bios[i];
	2086	+ struct resync_pages *rp = get_resync_pages(b);
2126	2087	if (b->bi_end_io != end_sync_read)
2127	2088	continue;
2128	2089	/* fixup the bio for reuse, but preserve errno */
@@ -2135,12 +2096,11 @@ static void process_checks(struct r1bio *r1_bio) · 6 unmodified lines
2135	2096	conf->mirrors[i].rdev->data_offset;
2136	2097	b->bi_bdev = conf->mirrors[i].rdev->bdev;
2137	2098	b->bi_end_io = end_sync_read;
2138		− b->bi_private = r1_bio;
	2099	+ rp->raid_bio = r1_bio;
	2100	+ b->bi_private = rp;
2139	2101
2140	2102	size = b->bi_iter.bi_size;
2141		− for (j = 0; j < vcnt ; j++) {
2142		− struct bio_vec *bi;
2143		− bi = &b->bi_io_vec[j];
	2103	+ bio_for_each_segment_all(bi, b, j) {
2144	2104	bi->bv_offset = 0;
2145	2105	if (size > PAGE_SIZE)
2146	2106	bi->bv_len = PAGE_SIZE;
@@ -2162,20 +2122,24 @@ static void process_checks(struct r1bio *r1_bio) · 15 unmodified lines
2162	2122	struct bio *pbio = r1_bio->bios[primary];
2163	2123	struct bio *sbio = r1_bio->bios[i];
2164	2124	int error = sbio->bi_error;
	2125	+ struct page **ppages = get_resync_pages(pbio)->pages;
	2126	+ struct page **spages = get_resync_pages(sbio)->pages;
	2127	+ struct bio_vec *bi;
	2128	+ int page_len[RESYNC_PAGES] = { 0 };
2165	2129
2166	2130	if (sbio->bi_end_io != end_sync_read)
2167	2131	continue;
2168	2132	/* Now we can 'fixup' the error value */
2169	2133	sbio->bi_error = 0;
2170	2134
	2135	+ bio_for_each_segment_all(bi, sbio, j)
	2136	+ page_len[j] = bi->bv_len;
	2137	+
2171	2138	if (!error) {
2172	2139	for (j = vcnt; j-- ; ) {
2173		− struct page p, s;
2174		− p = pbio->bi_io_vec[j].bv_page;
2175		− s = sbio->bi_io_vec[j].bv_page;
2176		− if (memcmp(page_address(p),
2177		− page_address(s),
2178		− sbio->bi_io_vec[j].bv_len))
	2140	+ if (memcmp(page_address(ppages[j]),
	2141	+ page_address(spages[j]),
	2142	+ page_len[j]))
2179	2143	break;
2180	2144	}
2181	2145	} else
@@ -2222,6 +2186,8 @@ static void sync_request_write(struct mddev mddev, struct r1bio r1_bio) · 40 unmodified lines
2222	2186	(i == r1_bio->read_disk \|\|
2223	2187	!test_bit(MD_RECOVERY_SYNC, &mddev->recovery))))
2224	2188	continue;
	2189	+ if (test_bit(Faulty, &conf->mirrors[i].rdev->flags))
	2190	+ continue;
2225	2191
2226	2192	bio_set_op_attrs(wbio, REQ_OP_WRITE, 0);
2227	2193	if (test_bit(FailFast, &conf->mirrors[i].rdev->flags))
@@ -2391,18 +2357,11 @@ static int narrow_write_error(struct r1bio *r1_bio, int i) · 163 unmodified lines
2391	2357	/* Write at 'sector' for 'sectors'*/
2392	2358
2393	2359	if (test_bit(R1BIO_BehindIO, &r1_bio->state)) {
2394		− unsigned vcnt = r1_bio->behind_page_count;
2395		− struct bio_vec *vec = r1_bio->behind_bvecs;
2396		−
2397		− while (!vec->bv_page) {
2398		− vec++;
2399		− vcnt--;
2400		− }
2401		−
2402		− wbio = bio_alloc_mddev(GFP_NOIO, vcnt, mddev);
2403		− memcpy(wbio->bi_io_vec, vec, vcnt * sizeof(struct bio_vec));
2404		−
2405		− wbio->bi_vcnt = vcnt;
	2360	+ wbio = bio_clone_fast(r1_bio->behind_master_bio,
	2361	+ GFP_NOIO,
	2362	+ mddev->bio_set);
	2363	+ /* We really need a _all clone */
	2364	+ wbio->bi_iter = (struct bvec_iter){ 0 };
2406	2365	} else {
2407	2366	wbio = bio_clone_fast(r1_bio->master_bio, GFP_NOIO,
2408	2367	mddev->bio_set);
@@ -2501,11 +2460,8 @@ static void handle_write_finished(struct r1conf conf, struct r1bio r1_bio) · 92 unmodified lines
2501	2460
2502	2461	static void handle_read_error(struct r1conf conf, struct r1bio r1_bio)
2503	2462	{
2504		− int disk;
2505		− int max_sectors;
2506	2463	struct mddev *mddev = conf->mddev;
2507	2464	struct bio *bio;
2508		− char b[BDEVNAME_SIZE];
2509	2465	struct md_rdev *rdev;
2510	2466	dev_t bio_dev;
2511	2467	sector_t bio_sector;
@@ -2521,7 +2477,6 @@ static void handle_read_error(struct r1conf conf, struct r1bio r1_bio) · 9 unmodified lines
2521	2477	*/
2522	2478
2523	2479	bio = r1_bio->bios[r1_bio->read_disk];
2524		− bdevname(bio->bi_bdev, b);
2525	2480	bio_dev = bio->bi_bdev->bd_dev;
2526	2481	bio_sector = conf->mirrors[r1_bio->read_disk].rdev->data_offset + r1_bio->sector;
2527	2482	bio_put(bio);
@@ -2539,62 +2494,12 @@ static void handle_read_error(struct r1conf conf, struct r1bio r1_bio) · 11 unmodified lines
2539	2494	}
2540	2495
2541	2496	rdev_dec_pending(rdev, conf->mddev);
	2497	+ allow_barrier(conf, r1_bio->sector);
	2498	+ bio = r1_bio->master_bio;
2542	2499
2543		−read_more:
2544		− disk = read_balance(conf, r1_bio, &max_sectors);
2545		− if (disk == -1) {
2546		− pr_crit_ratelimited("md/raid1:%s: %s: unrecoverable I/O read error for block %llu\n",
2547		− mdname(mddev), b, (unsigned long long)r1_bio->sector);
2548		− raid_end_bio_io(r1_bio);
2549		− } else {
2550		− const unsigned long do_sync
2551		− = r1_bio->master_bio->bi_opf & REQ_SYNC;
2552		− r1_bio->read_disk = disk;
2553		− bio = bio_clone_fast(r1_bio->master_bio, GFP_NOIO,
2554		− mddev->bio_set);
2555		− bio_trim(bio, r1_bio->sector - bio->bi_iter.bi_sector,
2556		− max_sectors);
2557		− r1_bio->bios[r1_bio->read_disk] = bio;
2558		− rdev = conf->mirrors[disk].rdev;
2559		− pr_info_ratelimited("md/raid1:%s: redirecting sector %llu to other mirror: %s\n",
2560		− mdname(mddev),
2561		− (unsigned long long)r1_bio->sector,
2562		− bdevname(rdev->bdev, b));
2563		− bio->bi_iter.bi_sector = r1_bio->sector + rdev->data_offset;
2564		− bio->bi_bdev = rdev->bdev;
2565		− bio->bi_end_io = raid1_end_read_request;
2566		− bio_set_op_attrs(bio, REQ_OP_READ, do_sync);
2567		− if (test_bit(FailFast, &rdev->flags) &&
2568		− test_bit(R1BIO_FailFast, &r1_bio->state))
2569		− bio->bi_opf \|= MD_FAILFAST;
2570		− bio->bi_private = r1_bio;
2571		− if (max_sectors < r1_bio->sectors) {
2572		− /* Drat - have to split this up more */
2573		− struct bio *mbio = r1_bio->master_bio;
2574		− int sectors_handled = (r1_bio->sector + max_sectors
2575		− - mbio->bi_iter.bi_sector);
2576		− r1_bio->sectors = max_sectors;
2577		− spin_lock_irq(&conf->device_lock);
2578		− if (mbio->bi_phys_segments == 0)
2579		− mbio->bi_phys_segments = 2;
2580		− else
2581		− mbio->bi_phys_segments++;
2582		− spin_unlock_irq(&conf->device_lock);
2583		− trace_block_bio_remap(bdev_get_queue(bio->bi_bdev),
2584		− bio, bio_dev, bio_sector);
2585		− generic_make_request(bio);
2586		− bio = NULL;
2587		−
2588		− r1_bio = alloc_r1bio(mddev, mbio, sectors_handled);
2589		− set_bit(R1BIO_ReadError, &r1_bio->state);
2590		−
2591		− goto read_more;
2592		− } else {
2593		− trace_block_bio_remap(bdev_get_queue(bio->bi_bdev),
2594		− bio, bio_dev, bio_sector);
2595		− generic_make_request(bio);
2596		− }
2597		− }
	2500	+ /* Reuse the old r1_bio so that the IO_BLOCKED settings are preserved */
	2501	+ r1_bio->state = 0;
	2502	+ raid1_read_request(mddev, bio, r1_bio->sectors, r1_bio);
2598	2503	}
2599	2504
2600	2505	static void raid1d(struct md_thread *thread)
@@ -2660,10 +2565,7 @@ static void raid1d(struct md_thread *thread) · 59 unmodified lines
2660	2565	else if (test_bit(R1BIO_ReadError, &r1_bio->state))
2661	2566	handle_read_error(conf, r1_bio);
2662	2567	else
2663		− /* just a partial read to be scheduled from separate
2664		− * context
2665		− */
2666		− generic_make_request(r1_bio->bios[r1_bio->read_disk]);
	2568	+ WARN_ON_ONCE(1);
2667	2569
2668	2570	cond_resched();
2669	2571	if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))
@@ -2793,7 +2695,6 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr, · 123 unmodified lines
2793	2695	for (i = 0; i < conf->raid_disks * 2; i++) {
2794	2696	struct md_rdev *rdev;
2795	2697	bio = r1_bio->bios[i];
2796		− bio_reset(bio);
2797	2698
2798	2699	rdev = rcu_dereference(conf->mirrors[i].rdev);
2799	2700	if (rdev == NULL \|\|
@@ -2849,7 +2750,6 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr, · 49 unmodified lines
2849	2750	atomic_inc(&rdev->nr_pending);
2850	2751	bio->bi_iter.bi_sector = sector_nr + rdev->data_offset;
2851	2752	bio->bi_bdev = rdev->bdev;
2852		− bio->bi_private = r1_bio;
2853	2753	if (test_bit(FailFast, &rdev->flags))
2854	2754	bio->bi_opf \|= MD_FAILFAST;
2855	2755	}
@@ -2935,31 +2835,25 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr, · 79 unmodified lines
2935	2835	}
2936	2836
2937	2837	for (i = 0 ; i < conf->raid_disks * 2; i++) {
	2838	+ struct resync_pages *rp;
	2839	+
2938	2840	bio = r1_bio->bios[i];
	2841	+ rp = get_resync_pages(bio);
2939	2842	if (bio->bi_end_io) {
2940		− page = bio->bi_io_vec[bio->bi_vcnt].bv_page;
2941		− if (bio_add_page(bio, page, len, 0) == 0) {
2942		− /* stop here */
2943		− bio->bi_io_vec[bio->bi_vcnt].bv_page = page;
2944		− while (i > 0) {
2945		− i--;
2946		− bio = r1_bio->bios[i];
2947		− if (bio->bi_end_io==NULL)
2948		− continue;
2949		− /* remove last page from this bio */
2950		− bio->bi_vcnt--;
2951		− bio->bi_iter.bi_size -= len;
2952		− bio_clear_flag(bio, BIO_SEG_VALID);
2953		− }
2954		− goto bio_full;
2955		− }
	2843	+ page = resync_fetch_page(rp, rp->idx++);
	2844	+
	2845	+ /*
	2846	+ * won't fail because the vec table is big
	2847	+ * enough to hold all these pages
	2848	+ */
	2849	+ bio_add_page(bio, page, len, 0);
2956	2850	}
2957	2851	}
2958	2852	nr_sectors += len>>9;
2959	2853	sector_nr += len>>9;
2960	2854	sync_blocks -= (len>>9);
2961		− } while (r1_bio->bios[disk]->bi_vcnt < RESYNC_PAGES);
2962		− bio_full:
	2855	+ } while (get_resync_pages(r1_bio->bios[disk]->bi_private)->idx < RESYNC_PAGES);
	2856	+
2963	2857	r1_bio->sectors = nr_sectors;
2964	2858
2965	2859	if (mddev_is_clustered(mddev) &&
@@ -3059,12 +2953,15 @@ static struct r1conf setup_conf(struct mddev mddev) · 93 unmodified lines
3059	2953	if (!conf->r1bio_pool)
3060	2954	goto abort;
3061	2955
	2956	+ conf->bio_split = bioset_create(BIO_POOL_SIZE, 0);
	2957	+ if (!conf->bio_split)
	2958	+ goto abort;
	2959	+
3062	2960	conf->poolinfo->mddev = mddev;
3063	2961
3064	2962	err = -EINVAL;
3065	2963	spin_lock_init(&conf->device_lock);
3066	2964	rdev_for_each(rdev, mddev) {
3067		− struct request_queue *q;
3068	2965	int disk_idx = rdev->raid_disk;
3069	2966	if (disk_idx >= mddev->raid_disks
3070	2967	\|\| disk_idx < 0)
@@ -3077,8 +2974,6 @@ static struct r1conf setup_conf(struct mddev mddev) · 6 unmodified lines
3077	2974	if (disk->rdev)
3078	2975	goto abort;
3079	2976	disk->rdev = rdev;
3080		− q = bdev_get_queue(rdev->bdev);
3081		−
3082	2977	disk->head_position = 0;
3083	2978	disk->seq_start = MaxSector;
3084	2979	}
@@ -3140,6 +3035,8 @@ static struct r1conf setup_conf(struct mddev mddev) · 55 unmodified lines
3140	3035	kfree(conf->nr_waiting);
3141	3036	kfree(conf->nr_queued);
3142	3037	kfree(conf->barrier);
	3038	+ if (conf->bio_split)
	3039	+ bioset_free(conf->bio_split);
3143	3040	kfree(conf);
3144	3041	}
3145	3042	return ERR_PTR(err);
@@ -3247,6 +3144,8 @@ static void raid1_free(struct mddev mddev, void priv) · 101 unmodified lines
3247	3144	kfree(conf->nr_waiting);
3248	3145	kfree(conf->nr_queued);
3249	3146	kfree(conf->barrier);
	3147	+ if (conf->bio_split)
	3148	+ bioset_free(conf->bio_split);
3250	3149	kfree(conf);
3251	3150	}
3252	3151

drivers/md/raid1.h+10−3

@@ -84,6 +84,7 @@ struct r1conf { · 83 unmodified lines
84	84	*/
85	85	wait_queue_head_t wait_barrier;
86	86	spinlock_t resync_lock;
	87	+ atomic_t nr_sync_pending;
87	88	atomic_t *nr_pending;
88	89	atomic_t *nr_waiting;
89	90	atomic_t *nr_queued;
@@ -107,6 +108,8 @@ struct r1conf { · 17 unmodified lines
107	108	mempool_t *r1bio_pool;
108	109	mempool_t *r1buf_pool;
109	110
	111	+ struct bio_set *bio_split;
	112	+
110	113	/* temporary buffer to synchronous IO when attempting to repair
111	114	* a read error.
112	115	*/
@@ -153,9 +156,13 @@ struct r1bio { · 40 unmodified lines
153	156	int read_disk;
154	157
155	158	struct list_head retry_list;
156		− /* Next two are only valid when R1BIO_BehindIO is set */
157		− struct bio_vec *behind_bvecs;
158		− int behind_page_count;
	159	+
	160	+ /*
	161	+ * When R1BIO_BehindIO is set, we store pages for write behind
	162	+ * in behind_master_bio.
	163	+ */
	164	+ struct bio *behind_master_bio;
	165	+
159	166	/*
160	167	* if the IO is in WRITE direction, then multiple bios are used.
161	168	* We choose the number when they are allocated.

drivers/md/raid10.c+306−430

@@ -110,6 +110,24 @@ static void end_reshape(struct r10conf *conf); · 109 unmodified lines
110	110	#define raid10_log(md, fmt, args...) \
111	111	do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid10 " fmt, ##args); } while (0)
112	112
	113	+/*
	114	+ * 'strct resync_pages' stores actual pages used for doing the resync
	115	+ * IO, and it is per-bio, so make .bi_private points to it.
	116	+ */
	117	+static inline struct resync_pages get_resync_pages(struct bio bio)
	118	+{
	119	+ return bio->bi_private;
	120	+}
	121	+
	122	+/*
	123	+ * for resync bio, r10bio pointer can be retrieved from the per-bio
	124	+ * 'struct resync_pages'.
	125	+ */
	126	+static inline struct r10bio get_resync_r10bio(struct bio bio)
	127	+{
	128	+ return get_resync_pages(bio)->raid_bio;
	129	+}
	130	+
113	131	static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
114	132	{
115	133	struct r10conf *conf = data;
@@ -125,9 +143,6 @@ static void r10bio_pool_free(void r10_bio, void data) · 9 unmodified lines
125	143	kfree(r10_bio);
126	144	}
127	145
128		−/* Maximum size of each resync request */
129		−#define RESYNC_BLOCK_SIZE (64*1024)
130		−#define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
131	146	/* amount of memory to reserve for resync requests */
132	147	#define RESYNC_WINDOW (1024*1024)
133	148	/* maximum number of concurrent requests, memory permitting */
@@ -143,11 +158,11 @@ static void r10bio_pool_free(void r10_bio, void data) · 9 unmodified lines
143	158	static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
144	159	{
145	160	struct r10conf *conf = data;
146		− struct page *page;
147	161	struct r10bio *r10_bio;
148	162	struct bio *bio;
149		− int i, j;
150		− int nalloc;
	163	+ int j;
	164	+ int nalloc, nalloc_rp;
	165	+ struct resync_pages *rps;
151	166
152	167	r10_bio = r10bio_pool_alloc(gfp_flags, conf);
153	168	if (!r10_bio)
@@ -159,6 +174,15 @@ static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data) · 5 unmodified lines
159	174	else
160	175	nalloc = 2; /* recovery */
161	176
	177	+ /* allocate once for all bios */
	178	+ if (!conf->have_replacement)
	179	+ nalloc_rp = nalloc;
	180	+ else
	181	+ nalloc_rp = nalloc * 2;
	182	+ rps = kmalloc(sizeof(struct resync_pages) * nalloc_rp, gfp_flags);
	183	+ if (!rps)
	184	+ goto out_free_r10bio;
	185	+
162	186	/*
163	187	* Allocate bios.
164	188	*/
@@ -178,36 +202,40 @@ static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data) · 13 unmodified lines
178	202	* Allocate RESYNC_PAGES data pages and attach them
179	203	* where needed.
180	204	*/
181		− for (j = 0 ; j < nalloc; j++) {
	205	+ for (j = 0; j < nalloc; j++) {
182	206	struct bio *rbio = r10_bio->devs[j].repl_bio;
	207	+ struct resync_pages rp, rp_repl;
	208	+
	209	+ rp = &rps[j];
	210	+ if (rbio)
	211	+ rp_repl = &rps[nalloc + j];
	212	+
183	213	bio = r10_bio->devs[j].bio;
184		− for (i = 0; i < RESYNC_PAGES; i++) {
185		− if (j > 0 && !test_bit(MD_RECOVERY_SYNC,
186		− &conf->mddev->recovery)) {
187		− /* we can share bv_page's during recovery
188		− * and reshape */
189		− struct bio *rbio = r10_bio->devs[0].bio;
190		− page = rbio->bi_io_vec[i].bv_page;
191		− get_page(page);
192		− } else
193		− page = alloc_page(gfp_flags);
194		− if (unlikely(!page))
	214	+
	215	+ if (!j \|\| test_bit(MD_RECOVERY_SYNC,
	216	+ &conf->mddev->recovery)) {
	217	+ if (resync_alloc_pages(rp, gfp_flags))
195	218	goto out_free_pages;
	219	+ } else {
	220	+ memcpy(rp, &rps[0], sizeof(*rp));
	221	+ resync_get_all_pages(rp);
	222	+ }
196	223
197		− bio->bi_io_vec[i].bv_page = page;
198		− if (rbio)
199		− rbio->bi_io_vec[i].bv_page = page;
	224	+ rp->idx = 0;
	225	+ rp->raid_bio = r10_bio;
	226	+ bio->bi_private = rp;
	227	+ if (rbio) {
	228	+ memcpy(rp_repl, rp, sizeof(*rp));
	229	+ rbio->bi_private = rp_repl;
200	230	}
201	231	}
202	232
203	233	return r10_bio;
204	234
205	235	out_free_pages:
206		− for ( ; i > 0 ; i--)
207		− safe_put_page(bio->bi_io_vec[i-1].bv_page);
208		− while (j--)
209		− for (i = 0; i < RESYNC_PAGES ; i++)
210		− safe_put_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page);
	236	+ while (--j >= 0)
	237	+ resync_free_pages(&rps[j * 2]);
	238	+
211	239	j = 0;
212	240	out_free_bio:
213	241	for ( ; j < nalloc; j++) {
@@ -216,30 +244,34 @@ out_free_bio: · 2 unmodified lines
216	244	if (r10_bio->devs[j].repl_bio)
217	245	bio_put(r10_bio->devs[j].repl_bio);
218	246	}
	247	+ kfree(rps);
	248	+out_free_r10bio:
219	249	r10bio_pool_free(r10_bio, conf);
220	250	return NULL;
221	251	}
222	252
223	253	static void r10buf_pool_free(void __r10_bio, void data)
224	254	{
225		− int i;
226	255	struct r10conf *conf = data;
227	256	struct r10bio *r10bio = __r10_bio;
228	257	int j;
	258	+ struct resync_pages *rp = NULL;
229	259
230		− for (j=0; j < conf->copies; j++) {
	260	+ for (j = conf->copies; j--; ) {
231	261	struct bio *bio = r10bio->devs[j].bio;
232		− if (bio) {
233		− for (i = 0; i < RESYNC_PAGES; i++) {
234		− safe_put_page(bio->bi_io_vec[i].bv_page);
235		− bio->bi_io_vec[i].bv_page = NULL;
236		− }
237		− bio_put(bio);
238		− }
	262	+
	263	+ rp = get_resync_pages(bio);
	264	+ resync_free_pages(rp);
	265	+ bio_put(bio);
	266	+
239	267	bio = r10bio->devs[j].repl_bio;
240	268	if (bio)
241	269	bio_put(bio);
242	270	}
	271	+
	272	+ /* resync pages array stored in the 1st bio's .bi_private */
	273	+ kfree(rp);
	274	+
243	275	r10bio_pool_free(r10bio, conf);
244	276	}
245	277
@@ -301,27 +333,18 @@ static void reschedule_retry(struct r10bio *r10_bio) · 55 unmodified lines
301	333	static void raid_end_bio_io(struct r10bio *r10_bio)
302	334	{
303	335	struct bio *bio = r10_bio->master_bio;
304		− int done;
305	336	struct r10conf *conf = r10_bio->mddev->private;
306	337
307		− if (bio->bi_phys_segments) {
308		− unsigned long flags;
309		− spin_lock_irqsave(&conf->device_lock, flags);
310		− bio->bi_phys_segments--;
311		− done = (bio->bi_phys_segments == 0);
312		− spin_unlock_irqrestore(&conf->device_lock, flags);
313		− } else
314		− done = 1;
315	338	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
316	339	bio->bi_error = -EIO;
317		− if (done) {
318		− bio_endio(bio);
319		− /*
320		− * Wake up any possible resync thread that waits for the device
321		− * to go idle.
322		− */
323		− allow_barrier(conf);
324		− }
	340	+
	341	+ bio_endio(bio);
	342	+ /*
	343	+ * Wake up any possible resync thread that waits for the device
	344	+ * to go idle.
	345	+ */
	346	+ allow_barrier(conf);
	347	+
325	348	free_r10bio(r10_bio);
326	349	}
327	350
@@ -1095,12 +1118,41 @@ static void raid10_read_request(struct mddev mddev, struct bio bio, · 767 unmodified lines
1095	1118	struct bio *read_bio;
1096	1119	const int op = bio_op(bio);
1097	1120	const unsigned long do_sync = (bio->bi_opf & REQ_SYNC);
1098		− int sectors_handled;
1099	1121	int max_sectors;
1100	1122	sector_t sectors;
1101	1123	struct md_rdev *rdev;
1102		− int slot;
	1124	+ char b[BDEVNAME_SIZE];
	1125	+ int slot = r10_bio->read_slot;
	1126	+ struct md_rdev *err_rdev = NULL;
	1127	+ gfp_t gfp = GFP_NOIO;
1103	1128
	1129	+ if (r10_bio->devs[slot].rdev) {
	1130	+ /*
	1131	+ * This is an error retry, but we cannot
	1132	+ * safely dereference the rdev in the r10_bio,
	1133	+ * we must use the one in conf.
	1134	+ * If it has already been disconnected (unlikely)
	1135	+ * we lose the device name in error messages.
	1136	+ */
	1137	+ int disk;
	1138	+ /*
	1139	+ * As we are blocking raid10, it is a little safer to
	1140	+ * use __GFP_HIGH.
	1141	+ */
	1142	+ gfp = GFP_NOIO \| __GFP_HIGH;
	1143	+
	1144	+ rcu_read_lock();
	1145	+ disk = r10_bio->devs[slot].devnum;
	1146	+ err_rdev = rcu_dereference(conf->mirrors[disk].rdev);
	1147	+ if (err_rdev)
	1148	+ bdevname(err_rdev->bdev, b);
	1149	+ else {
	1150	+ strcpy(b, "???");
	1151	+ /* This never gets dereferenced */
	1152	+ err_rdev = r10_bio->devs[slot].rdev;
	1153	+ }
	1154	+ rcu_read_unlock();
	1155	+ }
1104	1156	/*
1105	1157	* Register the new request and wait if the reconstruction
1106	1158	* thread has put up a bar for new requests.
@@ -1108,7 +1160,7 @@ static void raid10_read_request(struct mddev mddev, struct bio bio, · 1 unmodified line
1108	1160	*/
1109	1161	wait_barrier(conf);
1110	1162
1111		− sectors = bio_sectors(bio);
	1163	+ sectors = r10_bio->sectors;
1112	1164	while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1113	1165	bio->bi_iter.bi_sector < conf->reshape_progress &&
1114	1166	bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {
@@ -1125,17 +1177,33 @@ static void raid10_read_request(struct mddev mddev, struct bio bio, · 10 unmodified lines
1125	1177	wait_barrier(conf);
1126	1178	}
1127	1179
1128		−read_again:
1129	1180	rdev = read_balance(conf, r10_bio, &max_sectors);
1130	1181	if (!rdev) {
	1182	+ if (err_rdev) {
	1183	+ pr_crit_ratelimited("md/raid10:%s: %s: unrecoverable I/O read error for block %llu\n",
	1184	+ mdname(mddev), b,
	1185	+ (unsigned long long)r10_bio->sector);
	1186	+ }
1131	1187	raid_end_bio_io(r10_bio);
1132	1188	return;
1133	1189	}
	1190	+ if (err_rdev)
	1191	+ pr_err_ratelimited("md/raid10:%s: %s: redirecting sector %llu to another mirror\n",
	1192	+ mdname(mddev),
	1193	+ bdevname(rdev->bdev, b),
	1194	+ (unsigned long long)r10_bio->sector);
	1195	+ if (max_sectors < bio_sectors(bio)) {
	1196	+ struct bio *split = bio_split(bio, max_sectors,
	1197	+ gfp, conf->bio_split);
	1198	+ bio_chain(split, bio);
	1199	+ generic_make_request(bio);
	1200	+ bio = split;
	1201	+ r10_bio->master_bio = bio;
	1202	+ r10_bio->sectors = max_sectors;
	1203	+ }
1134	1204	slot = r10_bio->read_slot;
1135	1205
1136		− read_bio = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
1137		− bio_trim(read_bio, r10_bio->sector - bio->bi_iter.bi_sector,
1138		− max_sectors);
	1206	+ read_bio = bio_clone_fast(bio, gfp, mddev->bio_set);
1139	1207
1140	1208	r10_bio->devs[slot].bio = read_bio;
1141	1209	r10_bio->devs[slot].rdev = rdev;
@@ -1154,55 +1222,86 @@ read_again: · 12 unmodified lines
1154	1222	trace_block_bio_remap(bdev_get_queue(read_bio->bi_bdev),
1155	1223	read_bio, disk_devt(mddev->gendisk),
1156	1224	r10_bio->sector);
1157		− if (max_sectors < r10_bio->sectors) {
1158		− /*
1159		− * Could not read all from this device, so we will need another
1160		− * r10_bio.
1161		− */
1162		− sectors_handled = (r10_bio->sector + max_sectors
1163		− - bio->bi_iter.bi_sector);
1164		− r10_bio->sectors = max_sectors;
1165		− spin_lock_irq(&conf->device_lock);
1166		− if (bio->bi_phys_segments == 0)
1167		− bio->bi_phys_segments = 2;
1168		− else
1169		− bio->bi_phys_segments++;
1170		− spin_unlock_irq(&conf->device_lock);
1171		− /*
1172		− * Cannot call generic_make_request directly as that will be
1173		− * queued in __generic_make_request and subsequent
1174		− * mempool_alloc might block waiting for it. so hand bio over
1175		− * to raid10d.
1176		− */
1177		− reschedule_retry(r10_bio);
1178		−
1179		− r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
1180		−
1181		− r10_bio->master_bio = bio;
1182		− r10_bio->sectors = bio_sectors(bio) - sectors_handled;
1183		− r10_bio->state = 0;
1184		− r10_bio->mddev = mddev;
1185		− r10_bio->sector = bio->bi_iter.bi_sector + sectors_handled;
1186		− goto read_again;
1187		− } else
1188		− generic_make_request(read_bio);
	1225	+ generic_make_request(read_bio);
1189	1226	return;
1190	1227	}
1191	1228
1192		−static void raid10_write_request(struct mddev mddev, struct bio bio,
1193		− struct r10bio *r10_bio)
	1229	+static void raid10_write_one_disk(struct mddev mddev, struct r10bio r10_bio,
	1230	+ struct bio *bio, bool replacement,
	1231	+ int n_copy)
1194	1232	{
1195		− struct r10conf *conf = mddev->private;
1196		− int i;
1197	1233	const int op = bio_op(bio);
1198	1234	const unsigned long do_sync = (bio->bi_opf & REQ_SYNC);
1199	1235	const unsigned long do_fua = (bio->bi_opf & REQ_FUA);
1200	1236	unsigned long flags;
1201		− struct md_rdev *blocked_rdev;
1202	1237	struct blk_plug_cb *cb;
1203	1238	struct raid10_plug_cb *plug = NULL;
	1239	+ struct r10conf *conf = mddev->private;
	1240	+ struct md_rdev *rdev;
	1241	+ int devnum = r10_bio->devs[n_copy].devnum;
	1242	+ struct bio *mbio;
	1243	+
	1244	+ if (replacement) {
	1245	+ rdev = conf->mirrors[devnum].replacement;
	1246	+ if (rdev == NULL) {
	1247	+ /* Replacement just got moved to main 'rdev' */
	1248	+ smp_mb();
	1249	+ rdev = conf->mirrors[devnum].rdev;
	1250	+ }
	1251	+ } else
	1252	+ rdev = conf->mirrors[devnum].rdev;
	1253	+
	1254	+ mbio = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
	1255	+ if (replacement)
	1256	+ r10_bio->devs[n_copy].repl_bio = mbio;
	1257	+ else
	1258	+ r10_bio->devs[n_copy].bio = mbio;
	1259	+
	1260	+ mbio->bi_iter.bi_sector = (r10_bio->devs[n_copy].addr +
	1261	+ choose_data_offset(r10_bio, rdev));
	1262	+ mbio->bi_bdev = rdev->bdev;
	1263	+ mbio->bi_end_io = raid10_end_write_request;
	1264	+ bio_set_op_attrs(mbio, op, do_sync \| do_fua);
	1265	+ if (!replacement && test_bit(FailFast,
	1266	+ &conf->mirrors[devnum].rdev->flags)
	1267	+ && enough(conf, devnum))
	1268	+ mbio->bi_opf \|= MD_FAILFAST;
	1269	+ mbio->bi_private = r10_bio;
	1270	+
	1271	+ if (conf->mddev->gendisk)
	1272	+ trace_block_bio_remap(bdev_get_queue(mbio->bi_bdev),
	1273	+ mbio, disk_devt(conf->mddev->gendisk),
	1274	+ r10_bio->sector);
	1275	+ /* flush_pending_writes() needs access to the rdev so...*/
	1276	+ mbio->bi_bdev = (void *)rdev;
	1277	+
	1278	+ atomic_inc(&r10_bio->remaining);
	1279	+
	1280	+ cb = blk_check_plugged(raid10_unplug, mddev, sizeof(*plug));
	1281	+ if (cb)
	1282	+ plug = container_of(cb, struct raid10_plug_cb, cb);
	1283	+ else
	1284	+ plug = NULL;
	1285	+ spin_lock_irqsave(&conf->device_lock, flags);
	1286	+ if (plug) {
	1287	+ bio_list_add(&plug->pending, mbio);
	1288	+ plug->pending_cnt++;
	1289	+ } else {
	1290	+ bio_list_add(&conf->pending_bio_list, mbio);
	1291	+ conf->pending_count++;
	1292	+ }
	1293	+ spin_unlock_irqrestore(&conf->device_lock, flags);
	1294	+ if (!plug)
	1295	+ md_wakeup_thread(mddev->thread);
	1296	+}
	1297	+
	1298	+static void raid10_write_request(struct mddev mddev, struct bio bio,
	1299	+ struct r10bio *r10_bio)
	1300	+{
	1301	+ struct r10conf *conf = mddev->private;
	1302	+ int i;
	1303	+ struct md_rdev *blocked_rdev;
1204	1304	sector_t sectors;
1205		− int sectors_handled;
1206	1305	int max_sectors;
1207	1306
1208	1307	md_write_start(mddev, bio);
@@ -1214,7 +1313,7 @@ static void raid10_write_request(struct mddev mddev, struct bio bio, · 5 unmodified lines
1214	1313	*/
1215	1314	wait_barrier(conf);
1216	1315
1217		− sectors = bio_sectors(bio);
	1316	+ sectors = r10_bio->sectors;
1218	1317	while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1219	1318	bio->bi_iter.bi_sector < conf->reshape_progress &&
1220	1319	bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {
@@ -1262,9 +1361,7 @@ static void raid10_write_request(struct mddev mddev, struct bio bio, · 41 unmodified lines
1262	1361	* on which we have seen a write error, we want to avoid
1263	1362	* writing to those blocks. This potentially requires several
1264	1363	* writes to write around the bad blocks. Each set of writes
1265		− * gets its own r10_bio with a set of bios attached. The number
1266		− * of r10_bios is recored in bio->bi_phys_segments just as with
1267		− * the read case.
	1364	+ * gets its own r10_bio with a set of bios attached.
1268	1365	*/
1269	1366
1270	1367	r10_bio->read_slot = -1; /* make sure repl_bio gets freed */
@@ -1384,145 +1481,31 @@ retry_write: · 113 unmodified lines
1384	1481	goto retry_write;
1385	1482	}
1386	1483
1387		− if (max_sectors < r10_bio->sectors) {
1388		− /* We are splitting this into multiple parts, so
1389		− * we need to prepare for allocating another r10_bio.
1390		− */
	1484	+ if (max_sectors < r10_bio->sectors)
1391	1485	r10_bio->sectors = max_sectors;
1392		− spin_lock_irq(&conf->device_lock);
1393		− if (bio->bi_phys_segments == 0)
1394		− bio->bi_phys_segments = 2;
1395		− else
1396		− bio->bi_phys_segments++;
1397		− spin_unlock_irq(&conf->device_lock);
	1486	+
	1487	+ if (r10_bio->sectors < bio_sectors(bio)) {
	1488	+ struct bio *split = bio_split(bio, r10_bio->sectors,
	1489	+ GFP_NOIO, conf->bio_split);
	1490	+ bio_chain(split, bio);
	1491	+ generic_make_request(bio);
	1492	+ bio = split;
	1493	+ r10_bio->master_bio = bio;
1398	1494	}
1399		− sectors_handled = r10_bio->sector + max_sectors -
1400		− bio->bi_iter.bi_sector;
1401	1495
1402	1496	atomic_set(&r10_bio->remaining, 1);
1403	1497	bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0);
1404	1498
1405	1499	for (i = 0; i < conf->copies; i++) {
1406		− struct bio *mbio;
1407		− int d = r10_bio->devs[i].devnum;
1408		− if (r10_bio->devs[i].bio) {
1409		− struct md_rdev *rdev = conf->mirrors[d].rdev;
1410		− mbio = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
1411		− bio_trim(mbio, r10_bio->sector - bio->bi_iter.bi_sector,
1412		− max_sectors);
1413		− r10_bio->devs[i].bio = mbio;
1414		−
1415		− mbio->bi_iter.bi_sector = (r10_bio->devs[i].addr+
1416		− choose_data_offset(r10_bio, rdev));
1417		− mbio->bi_bdev = rdev->bdev;
1418		− mbio->bi_end_io = raid10_end_write_request;
1419		− bio_set_op_attrs(mbio, op, do_sync \| do_fua);
1420		− if (test_bit(FailFast, &conf->mirrors[d].rdev->flags) &&
1421		− enough(conf, d))
1422		− mbio->bi_opf \|= MD_FAILFAST;
1423		− mbio->bi_private = r10_bio;
1424		−
1425		− if (conf->mddev->gendisk)
1426		− trace_block_bio_remap(bdev_get_queue(mbio->bi_bdev),
1427		− mbio, disk_devt(conf->mddev->gendisk),
1428		− r10_bio->sector);
1429		− /* flush_pending_writes() needs access to the rdev so...*/
1430		− mbio->bi_bdev = (void*)rdev;
1431		−
1432		− atomic_inc(&r10_bio->remaining);
1433		−
1434		− cb = blk_check_plugged(raid10_unplug, mddev,
1435		− sizeof(*plug));
1436		− if (cb)
1437		− plug = container_of(cb, struct raid10_plug_cb,
1438		− cb);
1439		− else
1440		− plug = NULL;
1441		− spin_lock_irqsave(&conf->device_lock, flags);
1442		− if (plug) {
1443		− bio_list_add(&plug->pending, mbio);
1444		− plug->pending_cnt++;
1445		− } else {
1446		− bio_list_add(&conf->pending_bio_list, mbio);
1447		− conf->pending_count++;
1448		− }
1449		− spin_unlock_irqrestore(&conf->device_lock, flags);
1450		− if (!plug)
1451		− md_wakeup_thread(mddev->thread);
1452		− }
1453		−
1454		− if (r10_bio->devs[i].repl_bio) {
1455		− struct md_rdev *rdev = conf->mirrors[d].replacement;
1456		− if (rdev == NULL) {
1457		− /* Replacement just got moved to main 'rdev' */
1458		− smp_mb();
1459		− rdev = conf->mirrors[d].rdev;
1460		− }
1461		− mbio = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
1462		− bio_trim(mbio, r10_bio->sector - bio->bi_iter.bi_sector,
1463		− max_sectors);
1464		− r10_bio->devs[i].repl_bio = mbio;
1465		−
1466		− mbio->bi_iter.bi_sector = (r10_bio->devs[i].addr +
1467		− choose_data_offset(r10_bio, rdev));
1468		− mbio->bi_bdev = rdev->bdev;
1469		− mbio->bi_end_io = raid10_end_write_request;
1470		− bio_set_op_attrs(mbio, op, do_sync \| do_fua);
1471		− mbio->bi_private = r10_bio;
1472		−
1473		− if (conf->mddev->gendisk)
1474		− trace_block_bio_remap(bdev_get_queue(mbio->bi_bdev),
1475		− mbio, disk_devt(conf->mddev->gendisk),
1476		− r10_bio->sector);
1477		− /* flush_pending_writes() needs access to the rdev so...*/
1478		− mbio->bi_bdev = (void*)rdev;
1479		−
1480		− atomic_inc(&r10_bio->remaining);
1481		−
1482		− cb = blk_check_plugged(raid10_unplug, mddev,
1483		− sizeof(*plug));
1484		− if (cb)
1485		− plug = container_of(cb, struct raid10_plug_cb,
1486		− cb);
1487		− else
1488		− plug = NULL;
1489		− spin_lock_irqsave(&conf->device_lock, flags);
1490		− if (plug) {
1491		− bio_list_add(&plug->pending, mbio);
1492		− plug->pending_cnt++;
1493		− } else {
1494		− bio_list_add(&conf->pending_bio_list, mbio);
1495		− conf->pending_count++;
1496		− }
1497		− spin_unlock_irqrestore(&conf->device_lock, flags);
1498		− if (!plug)
1499		− md_wakeup_thread(mddev->thread);
1500		− }
1501		− }
1502		−
1503		− /* Don't remove the bias on 'remaining' (one_write_done) until
1504		− * after checking if we need to go around again.
1505		− */
1506		−
1507		− if (sectors_handled < bio_sectors(bio)) {
1508		− one_write_done(r10_bio);
1509		− /* We need another r10_bio. It has already been counted
1510		− * in bio->bi_phys_segments.
1511		− */
1512		− r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
1513		−
1514		− r10_bio->master_bio = bio;
1515		− r10_bio->sectors = bio_sectors(bio) - sectors_handled;
1516		−
1517		− r10_bio->mddev = mddev;
1518		− r10_bio->sector = bio->bi_iter.bi_sector + sectors_handled;
1519		− r10_bio->state = 0;
1520		− goto retry_write;
	1500	+ if (r10_bio->devs[i].bio)
	1501	+ raid10_write_one_disk(mddev, r10_bio, bio, false, i);
	1502	+ if (r10_bio->devs[i].repl_bio)
	1503	+ raid10_write_one_disk(mddev, r10_bio, bio, true, i);
1521	1504	}
1522	1505	one_write_done(r10_bio);
1523	1506	}
1524	1507
1525		−static void __make_request(struct mddev mddev, struct bio bio)
	1508	+static void __make_request(struct mddev mddev, struct bio bio, int sectors)
1526	1509	{
1527	1510	struct r10conf *conf = mddev->private;
1528	1511	struct r10bio *r10_bio;
@@ -1530,21 +1513,12 @@ static void __make_request(struct mddev mddev, struct bio bio) · 1 unmodified line
1530	1513	r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
1531	1514
1532	1515	r10_bio->master_bio = bio;
1533		− r10_bio->sectors = bio_sectors(bio);
	1516	+ r10_bio->sectors = sectors;
1534	1517
1535	1518	r10_bio->mddev = mddev;
1536	1519	r10_bio->sector = bio->bi_iter.bi_sector;
1537	1520	r10_bio->state = 0;
1538		−
1539		− /*
1540		− * We might need to issue multiple reads to different devices if there
1541		− * are bad blocks around, so we keep track of the number of reads in
1542		− * bio->bi_phys_segments. If this is 0, there is only one r10_bio and
1543		− * no locking will be needed when the request completes. If it is
1544		− * non-zero, then it is the number of not-completed requests.
1545		− */
1546		− bio->bi_phys_segments = 0;
1547		− bio_clear_flag(bio, BIO_SEG_VALID);
	1521	+ memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * conf->copies);
1548	1522
1549	1523	if (bio_data_dir(bio) == READ)
1550	1524	raid10_read_request(mddev, bio, r10_bio);
@@ -1557,54 +1531,26 @@ static void raid10_make_request(struct mddev mddev, struct bio bio) · 6 unmodified lines
1557	1531	struct r10conf *conf = mddev->private;
1558	1532	sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
1559	1533	int chunk_sects = chunk_mask + 1;
1560		−
1561		− struct bio *split;
	1534	+ int sectors = bio_sectors(bio);
1562	1535
1563	1536	if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
1564	1537	md_flush_request(mddev, bio);
1565	1538	return;
1566	1539	}
1567	1540
1568		− do {
1569		−
1570		− /*
1571		− * If this request crosses a chunk boundary, we need to split
1572		− * it.
1573		− */
1574		− if (unlikely((bio->bi_iter.bi_sector & chunk_mask) +
1575		− bio_sectors(bio) > chunk_sects
1576		− && (conf->geo.near_copies < conf->geo.raid_disks
1577		− \|\| conf->prev.near_copies <
1578		− conf->prev.raid_disks))) {
1579		− split = bio_split(bio, chunk_sects -
1580		− (bio->bi_iter.bi_sector &
1581		− (chunk_sects - 1)),
1582		− GFP_NOIO, fs_bio_set);
1583		− bio_chain(split, bio);
1584		− } else {
1585		− split = bio;
1586		− }
1587		−
1588		− /*
1589		− * If a bio is splitted, the first part of bio will pass
1590		− * barrier but the bio is queued in current->bio_list (see
1591		− * generic_make_request). If there is a raise_barrier() called
1592		− * here, the second part of bio can't pass barrier. But since
1593		− * the first part bio isn't dispatched to underlaying disks
1594		− * yet, the barrier is never released, hence raise_barrier will
1595		− * alays wait. We have a deadlock.
1596		− * Note, this only happens in read path. For write path, the
1597		− * first part of bio is dispatched in a schedule() call
1598		− * (because of blk plug) or offloaded to raid10d.
1599		− * Quitting from the function immediately can change the bio
1600		− * order queued in bio_list and avoid the deadlock.
1601		− */
1602		− __make_request(mddev, split);
1603		− if (split != bio && bio_data_dir(bio) == READ) {
1604		− generic_make_request(bio);
1605		− break;
1606		− }
1607		− } while (split != bio);
	1541	+ /*
	1542	+ * If this request crosses a chunk boundary, we need to split
	1543	+ * it.
	1544	+ */
	1545	+ if (unlikely((bio->bi_iter.bi_sector & chunk_mask) +
	1546	+ sectors > chunk_sects
	1547	+ && (conf->geo.near_copies < conf->geo.raid_disks
	1548	+ \|\| conf->prev.near_copies <
	1549	+ conf->prev.raid_disks)))
	1550	+ sectors = chunk_sects -
	1551	+ (bio->bi_iter.bi_sector &
	1552	+ (chunk_sects - 1));
	1553	+ __make_request(mddev, bio, sectors);
1608	1554
1609	1555	/* In case raid10d snuck in to freeze_array */
1610	1556	wake_up(&conf->wait_barrier);
@@ -1928,13 +1874,9 @@ static int raid10_remove_disk(struct mddev mddev, struct md_rdev rdev) · 317 unmodified lines
1928	1874	* but will never see neither -- if they are careful.
1929	1875	*/
1930	1876	p->replacement = NULL;
1931		− clear_bit(WantReplacement, &rdev->flags);
1932		− } else
1933		− /* We might have just remove the Replacement as faulty
1934		− * Clear the flag just in case
1935		− */
1936		− clear_bit(WantReplacement, &rdev->flags);
	1877	+ }
1937	1878
	1879	+ clear_bit(WantReplacement, &rdev->flags);
1938	1880	err = md_integrity_register(mddev);
1939	1881
1940	1882	abort:
@@ -1943,17 +1885,9 @@ abort: · 2 unmodified lines
1943	1885	return err;
1944	1886	}
1945	1887
1946		−static void end_sync_read(struct bio *bio)
	1888	+static void __end_sync_read(struct r10bio r10_bio, struct bio bio, int d)
1947	1889	{
1948		− struct r10bio *r10_bio = bio->bi_private;
1949	1890	struct r10conf *conf = r10_bio->mddev->private;
1950		− int d;
1951		−
1952		− if (bio == r10_bio->master_bio) {
1953		− /* this is a reshape read */
1954		− d = r10_bio->read_slot; /* really the read dev */
1955		− } else
1956		− d = find_bio_disk(conf, r10_bio, bio, NULL, NULL);
1957	1891
1958	1892	if (!bio->bi_error)
1959	1893	set_bit(R10BIO_Uptodate, &r10_bio->state);
@@ -1977,6 +1911,23 @@ static void end_sync_read(struct bio *bio) · 17 unmodified lines
1977	1911	}
1978	1912	}
1979	1913
	1914	+static void end_sync_read(struct bio *bio)
	1915	+{
	1916	+ struct r10bio *r10_bio = get_resync_r10bio(bio);
	1917	+ struct r10conf *conf = r10_bio->mddev->private;
	1918	+ int d = find_bio_disk(conf, r10_bio, bio, NULL, NULL);
	1919	+
	1920	+ __end_sync_read(r10_bio, bio, d);
	1921	+}
	1922	+
	1923	+static void end_reshape_read(struct bio *bio)
	1924	+{
	1925	+ /* reshape read bio isn't allocated from r10buf_pool */
	1926	+ struct r10bio *r10_bio = bio->bi_private;
	1927	+
	1928	+ __end_sync_read(r10_bio, bio, r10_bio->read_slot);
	1929	+}
	1930	+
1980	1931	static void end_sync_request(struct r10bio *r10_bio)
1981	1932	{
1982	1933	struct mddev *mddev = r10_bio->mddev;
@@ -2006,7 +1957,7 @@ static void end_sync_request(struct r10bio *r10_bio) · 23 unmodified lines
2006	1957
2007	1958	static void end_sync_write(struct bio *bio)
2008	1959	{
2009		− struct r10bio *r10_bio = bio->bi_private;
	1960	+ struct r10bio *r10_bio = get_resync_r10bio(bio);
2010	1961	struct mddev *mddev = r10_bio->mddev;
2011	1962	struct r10conf *conf = mddev->private;
2012	1963	int d;
@@ -2065,6 +2016,7 @@ static void sync_request_write(struct mddev mddev, struct r10bio r10_bio) · 52 unmodified lines
2065	2016	int i, first;
2066	2017	struct bio tbio, fbio;
2067	2018	int vcnt;
	2019	+ struct page tpages, fpages;
2068	2020
2069	2021	atomic_set(&r10_bio->remaining, 1);
2070	2022
@@ -2080,12 +2032,14 @@ static void sync_request_write(struct mddev mddev, struct r10bio r10_bio) · 9 unmodified lines
2080	2032	fbio = r10_bio->devs[i].bio;
2081	2033	fbio->bi_iter.bi_size = r10_bio->sectors << 9;
2082	2034	fbio->bi_iter.bi_idx = 0;
	2035	+ fpages = get_resync_pages(fbio)->pages;
2083	2036
2084	2037	vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9);
2085	2038	/* now find blocks with errors */
2086	2039	for (i=0 ; i < conf->copies ; i++) {
2087	2040	int j, d;
2088	2041	struct md_rdev *rdev;
	2042	+ struct resync_pages *rp;
2089	2043
2090	2044	tbio = r10_bio->devs[i].bio;
2091	2045
@@ -2093,6 +2047,8 @@ static void sync_request_write(struct mddev mddev, struct r10bio r10_bio) · 1 unmodified line
2093	2047	continue;
2094	2048	if (i == first)
2095	2049	continue;
	2050	+
	2051	+ tpages = get_resync_pages(tbio)->pages;
2096	2052	d = r10_bio->devs[i].devnum;
2097	2053	rdev = conf->mirrors[d].rdev;
2098	2054	if (!r10_bio->devs[i].bio->bi_error) {
@@ -2105,8 +2061,8 @@ static void sync_request_write(struct mddev mddev, struct r10bio r10_bio) · 6 unmodified lines
2105	2061	int len = PAGE_SIZE;
2106	2062	if (sectors < (len / 512))
2107	2063	len = sectors * 512;
2108		− if (memcmp(page_address(fbio->bi_io_vec[j].bv_page),
2109		− page_address(tbio->bi_io_vec[j].bv_page),
	2064	+ if (memcmp(page_address(fpages[j]),
	2065	+ page_address(tpages[j]),
2110	2066	len))
2111	2067	break;
2112	2068	sectors -= len/512;
@@ -2127,11 +2083,13 @@ static void sync_request_write(struct mddev mddev, struct r10bio r10_bio) · 14 unmodified lines
2127	2083	* First we need to fixup bv_offset, bv_len and
2128	2084	* bi_vecs, as the read request might have corrupted these
2129	2085	*/
	2086	+ rp = get_resync_pages(tbio);
2130	2087	bio_reset(tbio);
2131	2088
2132	2089	tbio->bi_vcnt = vcnt;
2133	2090	tbio->bi_iter.bi_size = fbio->bi_iter.bi_size;
2134		− tbio->bi_private = r10_bio;
	2091	+ rp->raid_bio = r10_bio;
	2092	+ tbio->bi_private = rp;
2135	2093	tbio->bi_iter.bi_sector = r10_bio->devs[i].addr;
2136	2094	tbio->bi_end_io = end_sync_write;
2137	2095	bio_set_op_attrs(tbio, REQ_OP_WRITE, 0);
@@ -2202,6 +2160,7 @@ static void fix_recovery_read_error(struct r10bio *r10_bio) · 64 unmodified lines
2202	2160	int idx = 0;
2203	2161	int dr = r10_bio->devs[0].devnum;
2204	2162	int dw = r10_bio->devs[1].devnum;
	2163	+ struct page **pages = get_resync_pages(bio)->pages;
2205	2164
2206	2165	while (sectors) {
2207	2166	int s = sectors;
@@ -2217,7 +2176,7 @@ static void fix_recovery_read_error(struct r10bio *r10_bio) · 9 unmodified lines
2217	2176	ok = sync_page_io(rdev,
2218	2177	addr,
2219	2178	s << 9,
2220		− bio->bi_io_vec[idx].bv_page,
	2179	+ pages[idx],
2221	2180	REQ_OP_READ, 0, false);
2222	2181	if (ok) {
2223	2182	rdev = conf->mirrors[dw].rdev;
@@ -2225,7 +2184,7 @@ static void fix_recovery_read_error(struct r10bio *r10_bio) · 1 unmodified line
2225	2184	ok = sync_page_io(rdev,
2226	2185	addr,
2227	2186	s << 9,
2228		− bio->bi_io_vec[idx].bv_page,
	2187	+ pages[idx],
2229	2188	REQ_OP_WRITE, 0, false);
2230	2189	if (!ok) {
2231	2190	set_bit(WriteErrorSeen, &rdev->flags);
@@ -2625,9 +2584,6 @@ static void handle_read_error(struct mddev mddev, struct r10bio r10_bio) · 393 unmodified lines
2625	2584	struct bio *bio;
2626	2585	struct r10conf *conf = mddev->private;
2627	2586	struct md_rdev *rdev = r10_bio->devs[slot].rdev;
2628		− char b[BDEVNAME_SIZE];
2629		− unsigned long do_sync;
2630		− int max_sectors;
2631	2587	dev_t bio_dev;
2632	2588	sector_t bio_last_sector;
2633	2589
@@ -2640,7 +2596,6 @@ static void handle_read_error(struct mddev mddev, struct r10bio r10_bio) · 6 unmodified lines
2640	2596	* frozen.
2641	2597	*/
2642	2598	bio = r10_bio->devs[slot].bio;
2643		− bdevname(bio->bi_bdev, b);
2644	2599	bio_dev = bio->bi_bdev->bd_dev;
2645	2600	bio_last_sector = r10_bio->devs[slot].addr + rdev->data_offset + r10_bio->sectors;
2646	2601	bio_put(bio);
@@ -2656,69 +2611,9 @@ static void handle_read_error(struct mddev mddev, struct r10bio r10_bio) · 9 unmodified lines
2656	2611	md_error(mddev, rdev);
2657	2612
2658	2613	rdev_dec_pending(rdev, mddev);
2659		−
2660		−read_more:
2661		− rdev = read_balance(conf, r10_bio, &max_sectors);
2662		− if (rdev == NULL) {
2663		− pr_crit_ratelimited("md/raid10:%s: %s: unrecoverable I/O read error for block %llu\n",
2664		− mdname(mddev), b,
2665		− (unsigned long long)r10_bio->sector);
2666		− raid_end_bio_io(r10_bio);
2667		− return;
2668		− }
2669		−
2670		− do_sync = (r10_bio->master_bio->bi_opf & REQ_SYNC);
2671		− slot = r10_bio->read_slot;
2672		− pr_err_ratelimited("md/raid10:%s: %s: redirecting sector %llu to another mirror\n",
2673		− mdname(mddev),
2674		− bdevname(rdev->bdev, b),
2675		− (unsigned long long)r10_bio->sector);
2676		− bio = bio_clone_fast(r10_bio->master_bio, GFP_NOIO, mddev->bio_set);
2677		− bio_trim(bio, r10_bio->sector - bio->bi_iter.bi_sector, max_sectors);
2678		− r10_bio->devs[slot].bio = bio;
2679		− r10_bio->devs[slot].rdev = rdev;
2680		− bio->bi_iter.bi_sector = r10_bio->devs[slot].addr
2681		− + choose_data_offset(r10_bio, rdev);
2682		− bio->bi_bdev = rdev->bdev;
2683		− bio_set_op_attrs(bio, REQ_OP_READ, do_sync);
2684		− if (test_bit(FailFast, &rdev->flags) &&
2685		− test_bit(R10BIO_FailFast, &r10_bio->state))
2686		− bio->bi_opf \|= MD_FAILFAST;
2687		− bio->bi_private = r10_bio;
2688		− bio->bi_end_io = raid10_end_read_request;
2689		− trace_block_bio_remap(bdev_get_queue(bio->bi_bdev),
2690		− bio, bio_dev,
2691		− bio_last_sector - r10_bio->sectors);
2692		−
2693		− if (max_sectors < r10_bio->sectors) {
2694		− /* Drat - have to split this up more */
2695		− struct bio *mbio = r10_bio->master_bio;
2696		− int sectors_handled =
2697		− r10_bio->sector + max_sectors
2698		− - mbio->bi_iter.bi_sector;
2699		− r10_bio->sectors = max_sectors;
2700		− spin_lock_irq(&conf->device_lock);
2701		− if (mbio->bi_phys_segments == 0)
2702		− mbio->bi_phys_segments = 2;
2703		− else
2704		− mbio->bi_phys_segments++;
2705		− spin_unlock_irq(&conf->device_lock);
2706		− generic_make_request(bio);
2707		−
2708		− r10_bio = mempool_alloc(conf->r10bio_pool,
2709		− GFP_NOIO);
2710		− r10_bio->master_bio = mbio;
2711		− r10_bio->sectors = bio_sectors(mbio) - sectors_handled;
2712		− r10_bio->state = 0;
2713		− set_bit(R10BIO_ReadError,
2714		− &r10_bio->state);
2715		− r10_bio->mddev = mddev;
2716		− r10_bio->sector = mbio->bi_iter.bi_sector
2717		− + sectors_handled;
2718		−
2719		− goto read_more;
2720		− } else
2721		− generic_make_request(bio);
	2614	+ allow_barrier(conf);
	2615	+ r10_bio->state = 0;
	2616	+ raid10_read_request(mddev, r10_bio->master_bio, r10_bio);
2722	2617	}
2723	2618
2724	2619	static void handle_write_completed(struct r10conf conf, struct r10bio r10_bio)
@@ -2805,6 +2700,11 @@ static void handle_write_completed(struct r10conf conf, struct r10bio r10_bio) · 80 unmodified lines
2805	2700	list_add(&r10_bio->retry_list, &conf->bio_end_io_list);
2806	2701	conf->nr_queued++;
2807	2702	spin_unlock_irq(&conf->device_lock);
	2703	+ /*
	2704	+ * In case freeze_array() is waiting for condition
	2705	+ * nr_pending == nr_queued + extra to be true.
	2706	+ */
	2707	+ wake_up(&conf->wait_barrier);
2808	2708	md_wakeup_thread(conf->mddev->thread);
2809	2709	} else {
2810	2710	if (test_bit(R10BIO_WriteError,
@@ -2879,13 +2779,8 @@ static void raid10d(struct md_thread *thread) · 68 unmodified lines
2879	2779	recovery_request_write(mddev, r10_bio);
2880	2780	else if (test_bit(R10BIO_ReadError, &r10_bio->state))
2881	2781	handle_read_error(mddev, r10_bio);
2882		− else {
2883		− /* just a partial read to be scheduled from a
2884		− * separate context
2885		− */
2886		− int slot = r10_bio->read_slot;
2887		− generic_make_request(r10_bio->devs[slot].bio);
2888		− }
	2782	+ else
	2783	+ WARN_ON_ONCE(1);
2889	2784
2890	2785	cond_resched();
2891	2786	if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))
@@ -3199,10 +3094,8 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, · 307 unmodified lines
3199	3094	}
3200	3095	}
3201	3096	bio = r10_bio->devs[0].bio;
3202		− bio_reset(bio);
3203	3097	bio->bi_next = biolist;
3204	3098	biolist = bio;
3205		− bio->bi_private = r10_bio;
3206	3099	bio->bi_end_io = end_sync_read;
3207	3100	bio_set_op_attrs(bio, REQ_OP_READ, 0);
3208	3101	if (test_bit(FailFast, &rdev->flags))
@@ -3226,10 +3119,8 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, · 17 unmodified lines
3226	3119
3227	3120	if (!test_bit(In_sync, &mrdev->flags)) {
3228	3121	bio = r10_bio->devs[1].bio;
3229		− bio_reset(bio);
3230	3122	bio->bi_next = biolist;
3231	3123	biolist = bio;
3232		− bio->bi_private = r10_bio;
3233	3124	bio->bi_end_io = end_sync_write;
3234	3125	bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
3235	3126	bio->bi_iter.bi_sector = to_addr
@@ -3254,10 +3145,8 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, · 18 unmodified lines
3254	3145	if (mreplace == NULL \|\| bio == NULL \|\|
3255	3146	test_bit(Faulty, &mreplace->flags))
3256	3147	break;
3257		− bio_reset(bio);
3258	3148	bio->bi_next = biolist;
3259	3149	biolist = bio;
3260		− bio->bi_private = r10_bio;
3261	3150	bio->bi_end_io = end_sync_write;
3262	3151	bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
3263	3152	bio->bi_iter.bi_sector = to_addr +
@@ -3379,7 +3268,6 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, · 115 unmodified lines
3379	3268	r10_bio->devs[i].repl_bio->bi_end_io = NULL;
3380	3269
3381	3270	bio = r10_bio->devs[i].bio;
3382		− bio_reset(bio);
3383	3271	bio->bi_error = -EIO;
3384	3272	rcu_read_lock();
3385	3273	rdev = rcu_dereference(conf->mirrors[d].rdev);
@@ -3404,7 +3292,6 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, · 18 unmodified lines
3404	3292	atomic_inc(&r10_bio->remaining);
3405	3293	bio->bi_next = biolist;
3406	3294	biolist = bio;
3407		− bio->bi_private = r10_bio;
3408	3295	bio->bi_end_io = end_sync_read;
3409	3296	bio_set_op_attrs(bio, REQ_OP_READ, 0);
3410	3297	if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
@@ -3423,13 +3310,11 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, · 12 unmodified lines
3423	3310
3424	3311	/* Need to set up for writing to the replacement */
3425	3312	bio = r10_bio->devs[i].repl_bio;
3426		− bio_reset(bio);
3427	3313	bio->bi_error = -EIO;
3428	3314
3429	3315	sector = r10_bio->devs[i].addr;
3430	3316	bio->bi_next = biolist;
3431	3317	biolist = bio;
3432		− bio->bi_private = r10_bio;
3433	3318	bio->bi_end_io = end_sync_write;
3434	3319	bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
3435	3320	if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
@@ -3468,27 +3353,17 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, · 32 unmodified lines
3468	3353	if (len == 0)
3469	3354	break;
3470	3355	for (bio= biolist ; bio ; bio=bio->bi_next) {
3471		− struct bio *bio2;
3472		− page = bio->bi_io_vec[bio->bi_vcnt].bv_page;
3473		− if (bio_add_page(bio, page, len, 0))
3474		− continue;
3475		−
3476		− /* stop here */
3477		− bio->bi_io_vec[bio->bi_vcnt].bv_page = page;
3478		− for (bio2 = biolist;
3479		− bio2 && bio2 != bio;
3480		− bio2 = bio2->bi_next) {
3481		− /* remove last page from this bio */
3482		− bio2->bi_vcnt--;
3483		− bio2->bi_iter.bi_size -= len;
3484		− bio_clear_flag(bio2, BIO_SEG_VALID);
3485		− }
3486		− goto bio_full;
	3356	+ struct resync_pages *rp = get_resync_pages(bio);
	3357	+ page = resync_fetch_page(rp, rp->idx++);
	3358	+ /*
	3359	+ * won't fail because the vec table is big enough
	3360	+ * to hold all these pages
	3361	+ */
	3362	+ bio_add_page(bio, page, len, 0);
3487	3363	}
3488	3364	nr_sectors += len>>9;
3489	3365	sector_nr += len>>9;
3490		− } while (biolist->bi_vcnt < RESYNC_PAGES);
3491		− bio_full:
	3366	+ } while (get_resync_pages(biolist)->idx < RESYNC_PAGES);
3492	3367	r10_bio->sectors = nr_sectors;
3493	3368
3494	3369	while (biolist) {
@@ -3496,7 +3371,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, · 1 unmodified line
3496	3371	biolist = biolist->bi_next;
3497	3372
3498	3373	bio->bi_next = NULL;
3499		− r10_bio = bio->bi_private;
	3374	+ r10_bio = get_resync_r10bio(bio);
3500	3375	r10_bio->sectors = nr_sectors;
3501	3376
3502	3377	if (bio->bi_end_io == end_sync_read) {
@@ -3678,6 +3553,10 @@ static struct r10conf setup_conf(struct mddev mddev) · 175 unmodified lines
3678	3553	if (!conf->r10bio_pool)
3679	3554	goto out;
3680	3555
	3556	+ conf->bio_split = bioset_create(BIO_POOL_SIZE, 0);
	3557	+ if (!conf->bio_split)
	3558	+ goto out;
	3559	+
3681	3560	calc_sectors(conf, mddev->dev_sectors);
3682	3561	if (mddev->reshape_position == MaxSector) {
3683	3562	conf->prev = conf->geo;
@@ -3715,6 +3594,8 @@ static struct r10conf setup_conf(struct mddev mddev) · 31 unmodified lines
3715	3594	mempool_destroy(conf->r10bio_pool);
3716	3595	kfree(conf->mirrors);
3717	3596	safe_put_page(conf->tmppage);
	3597	+ if (conf->bio_split)
	3598	+ bioset_free(conf->bio_split);
3718	3599	kfree(conf);
3719	3600	}
3720	3601	return ERR_PTR(err);
@@ -3760,7 +3641,6 @@ static int raid10_run(struct mddev *mddev) · 39 unmodified lines
3760	3641
3761	3642	rdev_for_each(rdev, mddev) {
3762	3643	long long diff;
3763		− struct request_queue *q;
3764	3644
3765	3645	disk_idx = rdev->raid_disk;
3766	3646	if (disk_idx < 0)
@@ -3779,7 +3659,6 @@ static int raid10_run(struct mddev *mddev) · 12 unmodified lines
3779	3659	goto out_free_conf;
3780	3660	disk->rdev = rdev;
3781	3661	}
3782		− q = bdev_get_queue(rdev->bdev);
3783	3662	diff = (rdev->new_data_offset - rdev->data_offset);
3784	3663	if (!mddev->reshape_backwards)
3785	3664	diff = -diff;
@@ -3796,6 +3675,7 @@ static int raid10_run(struct mddev *mddev) · 10 unmodified lines
3796	3675
3797	3676	if (blk_queue_discard(bdev_get_queue(rdev->bdev)))
3798	3677	discard_supported = true;
	3678	+ first = 0;
3799	3679	}
3800	3680
3801	3681	if (mddev->queue) {
@@ -3925,6 +3805,8 @@ static void raid10_free(struct mddev mddev, void priv) · 123 unmodified lines
3925	3805	kfree(conf->mirrors);
3926	3806	kfree(conf->mirrors_old);
3927	3807	kfree(conf->mirrors_new);
	3808	+ if (conf->bio_split)
	3809	+ bioset_free(conf->bio_split);
3928	3810	kfree(conf);
3929	3811	}
3930	3812
@@ -4198,6 +4080,7 @@ static int raid10_start_reshape(struct mddev *mddev) · 267 unmodified lines
4198	4080	diff = 0;
4199	4081	if (first \|\| diff < min_offset_diff)
4200	4082	min_offset_diff = diff;
	4083	+ first = 0;
4201	4084	}
4202	4085	}
4203	4086
@@ -4388,6 +4271,7 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, · 184 unmodified lines
4388	4271	struct bio *blist;
4389	4272	struct bio bio, read_bio;
4390	4273	int sectors_done = 0;
	4274	+ struct page **pages;
4391	4275
4392	4276	if (sector_nr == 0) {
4393	4277	/* If restarting in the middle, skip the initial sectors */
@@ -4508,7 +4392,7 @@ read_more: · 114 unmodified lines
4508	4392	read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr
4509	4393	+ rdev->data_offset);
4510	4394	read_bio->bi_private = r10_bio;
4511		− read_bio->bi_end_io = end_sync_read;
	4395	+ read_bio->bi_end_io = end_reshape_read;
4512	4396	bio_set_op_attrs(read_bio, REQ_OP_READ, 0);
4513	4397	read_bio->bi_flags &= (~0UL << BIO_RESET_BITS);
4514	4398	read_bio->bi_error = 0;
@@ -4538,11 +4422,9 @@ read_more: · 23 unmodified lines
4538	4422	if (!rdev2 \|\| test_bit(Faulty, &rdev2->flags))
4539	4423	continue;
4540	4424
4541		− bio_reset(b);
4542	4425	b->bi_bdev = rdev2->bdev;
4543	4426	b->bi_iter.bi_sector = r10_bio->devs[s/2].addr +
4544	4427	rdev2->new_data_offset;
4545		− b->bi_private = r10_bio;
4546	4428	b->bi_end_io = end_reshape_write;
4547	4429	bio_set_op_attrs(b, REQ_OP_WRITE, 0);
4548	4430	b->bi_next = blist;
@@ -4552,31 +4434,22 @@ read_more: · 3 unmodified lines
4552	4434	/* Now add as many pages as possible to all of these bios. */
4553	4435
4554	4436	nr_sectors = 0;
	4437	+ pages = get_resync_pages(r10_bio->devs[0].bio)->pages;
4555	4438	for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) {
4556		− struct page *page = r10_bio->devs[0].bio->bi_io_vec[s/(PAGE_SIZE>>9)].bv_page;
	4439	+ struct page *page = pages[s / (PAGE_SIZE >> 9)];
4557	4440	int len = (max_sectors - s) << 9;
4558	4441	if (len > PAGE_SIZE)
4559	4442	len = PAGE_SIZE;
4560	4443	for (bio = blist; bio ; bio = bio->bi_next) {
4561		− struct bio *bio2;
4562		− if (bio_add_page(bio, page, len, 0))
4563		− continue;
4564		−
4565		− /* Didn't fit, must stop */
4566		− for (bio2 = blist;
4567		− bio2 && bio2 != bio;
4568		− bio2 = bio2->bi_next) {
4569		− /* Remove last page from this bio */
4570		− bio2->bi_vcnt--;
4571		− bio2->bi_iter.bi_size -= len;
4572		− bio_clear_flag(bio2, BIO_SEG_VALID);
4573		− }
4574		− goto bio_full;
	4444	+ /*
	4445	+ * won't fail because the vec table is big enough
	4446	+ * to hold all these pages
	4447	+ */
	4448	+ bio_add_page(bio, page, len, 0);
4575	4449	}
4576	4450	sector_nr += len >> 9;
4577	4451	nr_sectors += len >> 9;
4578	4452	}
4579		−bio_full:
4580	4453	rcu_read_unlock();
4581	4454	r10_bio->sectors = nr_sectors;
4582	4455
@@ -4690,7 +4563,10 @@ static int handle_reshape_read_error(struct mddev *mddev, · 107 unmodified lines
4690	4563	struct r10bio *r10b = &on_stack.r10_bio;
4691	4564	int slot = 0;
4692	4565	int idx = 0;
4693		− struct bio_vec *bvec = r10_bio->master_bio->bi_io_vec;
	4566	+ struct page **pages;
	4567	+
	4568	+ /* reshape IOs share pages from .devs[0].bio */
	4569	+ pages = get_resync_pages(r10_bio->devs[0].bio)->pages;
4694	4570
4695	4571	r10b->sector = r10_bio->sector;
4696	4572	__raid10_find_phys(&conf->prev, r10b);
@@ -4719,7 +4595,7 @@ static int handle_reshape_read_error(struct mddev *mddev, · 22 unmodified lines
4719	4595	success = sync_page_io(rdev,
4720	4596	addr,
4721	4597	s << 9,
4722		− bvec[idx].bv_page,
	4598	+ pages[idx],
4723	4599	REQ_OP_READ, 0, false);
4724	4600	rdev_dec_pending(rdev, mddev);
4725	4601	rcu_read_lock();
@@ -4747,7 +4623,7 @@ static int handle_reshape_read_error(struct mddev *mddev, · 21 unmodified lines
4747	4623
4748	4624	static void end_reshape_write(struct bio *bio)
4749	4625	{
4750		− struct r10bio *r10_bio = bio->bi_private;
	4626	+ struct r10bio *r10_bio = get_resync_r10bio(bio);
4751	4627	struct mddev *mddev = r10_bio->mddev;
4752	4628	struct r10conf *conf = mddev->private;
4753	4629	int d;

drivers/md/raid10.h+1−0

@@ -82,6 +82,7 @@ struct r10conf { · 81 unmodified lines
82	82	mempool_t *r10bio_pool;
83	83	mempool_t *r10buf_pool;
84	84	struct page *tmppage;
	85	+ struct bio_set *bio_split;
85	86
86	87	/* When taking over an array from a different personality, we store
87	88	* the new thread here until we fully activate the array.

drivers/md/raid5-cache.c+285−77

@@ -30,6 +30,7 @@ · 29 unmodified lines
30	30	* underneath hardware sector size. only works with PAGE_SIZE == 4096
31	31	*/
32	32	#define BLOCK_SECTORS (8)
	33	+#define BLOCK_SECTOR_SHIFT (3)
33	34
34	35	/*
35	36	* log->max_free_space is min(1/4 disk size, 10G reclaimable space).
@@ -43,7 +44,7 @@ · 7 unmodified lines
43	44	/* wake up reclaim thread periodically */
44	45	#define R5C_RECLAIM_WAKEUP_INTERVAL (30 * HZ)
45	46	/* start flush with these full stripes */
46		−#define R5C_FULL_STRIPE_FLUSH_BATCH 256
	47	+#define R5C_FULL_STRIPE_FLUSH_BATCH(conf) (conf->max_nr_stripes / 4)
47	48	/* reclaim stripes in groups */
48	49	#define R5C_RECLAIM_STRIPE_GROUP (NR_STRIPE_HASH_LOCKS * 2)
49	50
@@ -307,8 +308,7 @@ static void __r5l_set_io_unit_state(struct r5l_io_unit *io, · 257 unmodified lines
307	308	}
308	309
309	310	static void
310		−r5c_return_dev_pending_writes(struct r5conf conf, struct r5dev dev,
311		− struct bio_list *return_bi)
	311	+r5c_return_dev_pending_writes(struct r5conf conf, struct r5dev dev)
312	312	{
313	313	struct bio wbi, wbi2;
314	314
@@ -317,24 +317,21 @@ r5c_return_dev_pending_writes(struct r5conf conf, struct r5dev dev, · 2 unmodified lines
317	317	while (wbi && wbi->bi_iter.bi_sector <
318	318	dev->sector + STRIPE_SECTORS) {
319	319	wbi2 = r5_next_bio(wbi, dev->sector);
320		− if (!raid5_dec_bi_active_stripes(wbi)) {
321		− md_write_end(conf->mddev);
322		− bio_list_add(return_bi, wbi);
323		− }
	320	+ md_write_end(conf->mddev);
	321	+ bio_endio(wbi);
324	322	wbi = wbi2;
325	323	}
326	324	}
327	325
328	326	void r5c_handle_cached_data_endio(struct r5conf *conf,
329		− struct stripe_head sh, int disks, struct bio_list return_bi)
	327	+ struct stripe_head *sh, int disks)
330	328	{
331	329	int i;
332	330
333	331	for (i = sh->disks; i--; ) {
334	332	if (sh->dev[i].written) {
335	333	set_bit(R5_UPTODATE, &sh->dev[i].flags);
336		− r5c_return_dev_pending_writes(conf, &sh->dev[i],
337		− return_bi);
	334	+ r5c_return_dev_pending_writes(conf, &sh->dev[i]);
338	335	bitmap_endwrite(conf->mddev->bitmap, sh->sector,
339	336	STRIPE_SECTORS,
340	337	!test_bit(STRIPE_DEGRADED, &sh->state),
@@ -343,6 +340,8 @@ void r5c_handle_cached_data_endio(struct r5conf *conf, · 2 unmodified lines
343	340	}
344	341	}
345	342
	343	+void r5l_wake_reclaim(struct r5l_log *log, sector_t space);
	344	+
346	345	/* Check whether we should flush some stripes to free up stripe cache */
347	346	void r5c_check_stripe_cache_usage(struct r5conf *conf)
348	347	{
@@ -381,7 +380,7 @@ void r5c_check_cached_full_stripe(struct r5conf *conf) · 32 unmodified lines
381	380	* or a full stripe (chunk size / 4k stripes).
382	381	*/
383	382	if (atomic_read(&conf->r5c_cached_full_stripes) >=
384		− min(R5C_FULL_STRIPE_FLUSH_BATCH,
	383	+ min(R5C_FULL_STRIPE_FLUSH_BATCH(conf),
385	384	conf->chunk_sectors >> STRIPE_SHIFT))
386	385	r5l_wake_reclaim(conf->log, 0);
387	386	}
@@ -590,7 +589,7 @@ static void r5l_log_endio(struct bio *bio) · 202 unmodified lines
590	589
591	590	spin_lock_irqsave(&log->io_list_lock, flags);
592	591	__r5l_set_io_unit_state(io, IO_UNIT_IO_END);
593		− if (log->need_cache_flush)
	592	+ if (log->need_cache_flush && !list_empty(&io->stripe_list))
594	593	r5l_move_to_end_ios(log);
595	594	else
596	595	r5l_log_run_stripes(log);
@@ -618,9 +617,11 @@ static void r5l_log_endio(struct bio *bio) · 21 unmodified lines
618	617	bio_endio(bi);
619	618	atomic_dec(&io->pending_stripe);
620	619	}
621		− if (atomic_read(&io->pending_stripe) == 0)
622		− __r5l_stripe_write_finished(io);
623	620	}
	621	+
	622	+ /* finish flush only io_unit and PAYLOAD_FLUSH only io_unit */
	623	+ if (atomic_read(&io->pending_stripe) == 0)
	624	+ __r5l_stripe_write_finished(io);
624	625	}
625	626
626	627	static void r5l_do_submit_io(struct r5l_log log, struct r5l_io_unit io)
@@ -842,6 +843,41 @@ static void r5l_append_payload_page(struct r5l_log log, struct page page) · 215 unmodified lines
842	843	r5_reserve_log_entry(log, io);
843	844	}
844	845
	846	+static void r5l_append_flush_payload(struct r5l_log *log, sector_t sect)
	847	+{
	848	+ struct mddev *mddev = log->rdev->mddev;
	849	+ struct r5conf *conf = mddev->private;
	850	+ struct r5l_io_unit *io;
	851	+ struct r5l_payload_flush *payload;
	852	+ int meta_size;
	853	+
	854	+ /*
	855	+ * payload_flush requires extra writes to the journal.
	856	+ * To avoid handling the extra IO in quiesce, just skip
	857	+ * flush_payload
	858	+ */
	859	+ if (conf->quiesce)
	860	+ return;
	861	+
	862	+ mutex_lock(&log->io_mutex);
	863	+ meta_size = sizeof(struct r5l_payload_flush) + sizeof(__le64);
	864	+
	865	+ if (r5l_get_meta(log, meta_size)) {
	866	+ mutex_unlock(&log->io_mutex);
	867	+ return;
	868	+ }
	869	+
	870	+ /* current implementation is one stripe per flush payload */
	871	+ io = log->current_io;
	872	+ payload = page_address(io->meta_page) + io->meta_offset;
	873	+ payload->header.type = cpu_to_le16(R5LOG_PAYLOAD_FLUSH);
	874	+ payload->header.flags = cpu_to_le16(0);
	875	+ payload->size = cpu_to_le32(sizeof(__le64));
	876	+ payload->flush_stripes[0] = cpu_to_le64(sect);
	877	+ io->meta_offset += meta_size;
	878	+ mutex_unlock(&log->io_mutex);
	879	+}
	880	+
845	881	static int r5l_log_stripe(struct r5l_log log, struct stripe_head sh,
846	882	int data_pages, int parity_pages)
847	883	{
@@ -1393,7 +1429,7 @@ static void r5c_do_reclaim(struct r5conf *conf) · 545 unmodified lines
1393	1429	stripes_to_flush = R5C_RECLAIM_STRIPE_GROUP;
1394	1430	else if (total_cached > conf->min_nr_stripes * 1 / 2 \|\|
1395	1431	atomic_read(&conf->r5c_cached_full_stripes) - flushing_full >
1396		− R5C_FULL_STRIPE_FLUSH_BATCH)
	1432	+ R5C_FULL_STRIPE_FLUSH_BATCH(conf))
1397	1433	/*
1398	1434	* if stripe cache pressure moderate, or if there is many full
1399	1435	* stripes,flush all full stripes
@@ -1552,6 +1588,8 @@ bool r5l_log_disk_error(struct r5conf *conf) · 152 unmodified lines
1552	1588	return ret;
1553	1589	}
1554	1590
	1591	+#define R5L_RECOVERY_PAGE_POOL_SIZE 256
	1592	+
1555	1593	struct r5l_recovery_ctx {
1556	1594	struct page meta_page; / current meta */
1557	1595	sector_t meta_total_blocks; /* total size of current meta and data */
@@ -1560,18 +1598,131 @@ struct r5l_recovery_ctx { · 2 unmodified lines
1560	1598	int data_parity_stripes; /* number of data_parity stripes */
1561	1599	int data_only_stripes; /* number of data_only stripes */
1562	1600	struct list_head cached_list;
	1601	+
	1602	+ /*
	1603	+ * read ahead page pool (ra_pool)
	1604	+ * in recovery, log is read sequentially. It is not efficient to
	1605	+ * read every page with sync_page_io(). The read ahead page pool
	1606	+ * reads multiple pages with one IO, so further log read can
	1607	+ * just copy data from the pool.
	1608	+ */
	1609	+ struct page *ra_pool[R5L_RECOVERY_PAGE_POOL_SIZE];
	1610	+ sector_t pool_offset; /* offset of first page in the pool */
	1611	+ int total_pages; /* total allocated pages */
	1612	+ int valid_pages; /* pages with valid data */
	1613	+ struct bio ra_bio; / bio to do the read ahead */
1563	1614	};
1564	1615
	1616	+static int r5l_recovery_allocate_ra_pool(struct r5l_log *log,
	1617	+ struct r5l_recovery_ctx *ctx)
	1618	+{
	1619	+ struct page *page;
	1620	+
	1621	+ ctx->ra_bio = bio_alloc_bioset(GFP_KERNEL, BIO_MAX_PAGES, log->bs);
	1622	+ if (!ctx->ra_bio)
	1623	+ return -ENOMEM;
	1624	+
	1625	+ ctx->valid_pages = 0;
	1626	+ ctx->total_pages = 0;
	1627	+ while (ctx->total_pages < R5L_RECOVERY_PAGE_POOL_SIZE) {
	1628	+ page = alloc_page(GFP_KERNEL);
	1629	+
	1630	+ if (!page)
	1631	+ break;
	1632	+ ctx->ra_pool[ctx->total_pages] = page;
	1633	+ ctx->total_pages += 1;
	1634	+ }
	1635	+
	1636	+ if (ctx->total_pages == 0) {
	1637	+ bio_put(ctx->ra_bio);
	1638	+ return -ENOMEM;
	1639	+ }
	1640	+
	1641	+ ctx->pool_offset = 0;
	1642	+ return 0;
	1643	+}
	1644	+
	1645	+static void r5l_recovery_free_ra_pool(struct r5l_log *log,
	1646	+ struct r5l_recovery_ctx *ctx)
	1647	+{
	1648	+ int i;
	1649	+
	1650	+ for (i = 0; i < ctx->total_pages; ++i)
	1651	+ put_page(ctx->ra_pool[i]);
	1652	+ bio_put(ctx->ra_bio);
	1653	+}
	1654	+
	1655	+/*
	1656	+ * fetch ctx->valid_pages pages from offset
	1657	+ * In normal cases, ctx->valid_pages == ctx->total_pages after the call.
	1658	+ * However, if the offset is close to the end of the journal device,
	1659	+ * ctx->valid_pages could be smaller than ctx->total_pages
	1660	+ */
	1661	+static int r5l_recovery_fetch_ra_pool(struct r5l_log *log,
	1662	+ struct r5l_recovery_ctx *ctx,
	1663	+ sector_t offset)
	1664	+{
	1665	+ bio_reset(ctx->ra_bio);
	1666	+ ctx->ra_bio->bi_bdev = log->rdev->bdev;
	1667	+ bio_set_op_attrs(ctx->ra_bio, REQ_OP_READ, 0);
	1668	+ ctx->ra_bio->bi_iter.bi_sector = log->rdev->data_offset + offset;
	1669	+
	1670	+ ctx->valid_pages = 0;
	1671	+ ctx->pool_offset = offset;
	1672	+
	1673	+ while (ctx->valid_pages < ctx->total_pages) {
	1674	+ bio_add_page(ctx->ra_bio,
	1675	+ ctx->ra_pool[ctx->valid_pages], PAGE_SIZE, 0);
	1676	+ ctx->valid_pages += 1;
	1677	+
	1678	+ offset = r5l_ring_add(log, offset, BLOCK_SECTORS);
	1679	+
	1680	+ if (offset == 0) /* reached end of the device */
	1681	+ break;
	1682	+ }
	1683	+
	1684	+ return submit_bio_wait(ctx->ra_bio);
	1685	+}
	1686	+
	1687	+/*
	1688	+ * try read a page from the read ahead page pool, if the page is not in the
	1689	+ * pool, call r5l_recovery_fetch_ra_pool
	1690	+ */
	1691	+static int r5l_recovery_read_page(struct r5l_log *log,
	1692	+ struct r5l_recovery_ctx *ctx,
	1693	+ struct page *page,
	1694	+ sector_t offset)
	1695	+{
	1696	+ int ret;
	1697	+
	1698	+ if (offset < ctx->pool_offset \|\|
	1699	+ offset >= ctx->pool_offset + ctx->valid_pages * BLOCK_SECTORS) {
	1700	+ ret = r5l_recovery_fetch_ra_pool(log, ctx, offset);
	1701	+ if (ret)
	1702	+ return ret;
	1703	+ }
	1704	+
	1705	+ BUG_ON(offset < ctx->pool_offset \|\|
	1706	+ offset >= ctx->pool_offset + ctx->valid_pages * BLOCK_SECTORS);
	1707	+
	1708	+ memcpy(page_address(page),
	1709	+ page_address(ctx->ra_pool[(offset - ctx->pool_offset) >>
	1710	+ BLOCK_SECTOR_SHIFT]),
	1711	+ PAGE_SIZE);
	1712	+ return 0;
	1713	+}
	1714	+
1565	1715	static int r5l_recovery_read_meta_block(struct r5l_log *log,
1566	1716	struct r5l_recovery_ctx *ctx)
1567	1717	{
1568	1718	struct page *page = ctx->meta_page;
1569	1719	struct r5l_meta_block *mb;
1570	1720	u32 crc, stored_crc;
	1721	+ int ret;
1571	1722
1572		− if (!sync_page_io(log->rdev, ctx->pos, PAGE_SIZE, page, REQ_OP_READ, 0,
1573		− false))
1574		− return -EIO;
	1723	+ ret = r5l_recovery_read_page(log, ctx, page, ctx->pos);
	1724	+ if (ret != 0)
	1725	+ return ret;
1575	1726
1576	1727	mb = page_address(page);
1577	1728	stored_crc = le32_to_cpu(mb->checksum);
@@ -1653,8 +1804,7 @@ static void r5l_recovery_load_data(struct r5l_log *log, · 75 unmodified lines
1653	1804	raid5_compute_sector(conf,
1654	1805	le64_to_cpu(payload->location), 0,
1655	1806	&dd_idx, sh);
1656		− sync_page_io(log->rdev, log_offset, PAGE_SIZE,
1657		− sh->dev[dd_idx].page, REQ_OP_READ, 0, false);
	1807	+ r5l_recovery_read_page(log, ctx, sh->dev[dd_idx].page, log_offset);
1658	1808	sh->dev[dd_idx].log_checksum =
1659	1809	le32_to_cpu(payload->checksum[0]);
1660	1810	ctx->meta_total_blocks += BLOCK_SECTORS;
@@ -1673,17 +1823,15 @@ static void r5l_recovery_load_parity(struct r5l_log *log, · 12 unmodified lines
1673	1823	struct r5conf *conf = mddev->private;
1674	1824
1675	1825	ctx->meta_total_blocks += BLOCK_SECTORS * conf->max_degraded;
1676		− sync_page_io(log->rdev, log_offset, PAGE_SIZE,
1677		− sh->dev[sh->pd_idx].page, REQ_OP_READ, 0, false);
	1826	+ r5l_recovery_read_page(log, ctx, sh->dev[sh->pd_idx].page, log_offset);
1678	1827	sh->dev[sh->pd_idx].log_checksum =
1679	1828	le32_to_cpu(payload->checksum[0]);
1680	1829	set_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags);
1681	1830
1682	1831	if (sh->qd_idx >= 0) {
1683		− sync_page_io(log->rdev,
1684		− r5l_ring_add(log, log_offset, BLOCK_SECTORS),
1685		− PAGE_SIZE, sh->dev[sh->qd_idx].page,
1686		− REQ_OP_READ, 0, false);
	1832	+ r5l_recovery_read_page(
	1833	+ log, ctx, sh->dev[sh->qd_idx].page,
	1834	+ r5l_ring_add(log, log_offset, BLOCK_SECTORS));
1687	1835	sh->dev[sh->qd_idx].log_checksum =
1688	1836	le32_to_cpu(payload->checksum[1]);
1689	1837	set_bit(R5_Wantwrite, &sh->dev[sh->qd_idx].flags);
@@ -1814,14 +1962,15 @@ r5c_recovery_replay_stripes(struct list_head *cached_stripe_list, · 124 unmodified lines
1814	1962
1815	1963	/* if matches return 0; otherwise return -EINVAL */
1816	1964	static int
1817		−r5l_recovery_verify_data_checksum(struct r5l_log log, struct page page,
	1965	+r5l_recovery_verify_data_checksum(struct r5l_log *log,
	1966	+ struct r5l_recovery_ctx *ctx,
	1967	+ struct page *page,
1818	1968	sector_t log_offset, __le32 log_checksum)
1819	1969	{
1820	1970	void *addr;
1821	1971	u32 checksum;
1822	1972
1823		− sync_page_io(log->rdev, log_offset, PAGE_SIZE,
1824		− page, REQ_OP_READ, 0, false);
	1973	+ r5l_recovery_read_page(log, ctx, page, log_offset);
1825	1974	addr = kmap_atomic(page);
1826	1975	checksum = crc32c_le(log->uuid_checksum, addr, PAGE_SIZE);
1827	1976	kunmap_atomic(addr);
@@ -1843,6 +1992,7 @@ r5l_recovery_verify_data_checksum_for_mb(struct r5l_log *log, · 15 unmodified lines
1843	1992	sector_t log_offset = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
1844	1993	struct page *page;
1845	1994	struct r5l_payload_data_parity *payload;
	1995	+ struct r5l_payload_flush *payload_flush;
1846	1996
1847	1997	page = alloc_page(GFP_KERNEL);
1848	1998	if (!page)
@@ -1850,33 +2000,42 @@ r5l_recovery_verify_data_checksum_for_mb(struct r5l_log *log, · 1 unmodified line
1850	2000
1851	2001	while (mb_offset < le32_to_cpu(mb->meta_size)) {
1852	2002	payload = (void *)mb + mb_offset;
	2003	+ payload_flush = (void *)mb + mb_offset;
1853	2004
1854		− if (payload->header.type == R5LOG_PAYLOAD_DATA) {
	2005	+ if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) {
1855	2006	if (r5l_recovery_verify_data_checksum(
1856		− log, page, log_offset,
	2007	+ log, ctx, page, log_offset,
1857	2008	payload->checksum[0]) < 0)
1858	2009	goto mismatch;
1859		− } else if (payload->header.type == R5LOG_PAYLOAD_PARITY) {
	2010	+ } else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_PARITY) {
1860	2011	if (r5l_recovery_verify_data_checksum(
1861		− log, page, log_offset,
	2012	+ log, ctx, page, log_offset,
1862	2013	payload->checksum[0]) < 0)
1863	2014	goto mismatch;
1864	2015	if (conf->max_degraded == 2 && /* q for RAID 6 */
1865	2016	r5l_recovery_verify_data_checksum(
1866		− log, page,
	2017	+ log, ctx, page,
1867	2018	r5l_ring_add(log, log_offset,
1868	2019	BLOCK_SECTORS),
1869	2020	payload->checksum[1]) < 0)
1870	2021	goto mismatch;
1871		− } else /* not R5LOG_PAYLOAD_DATA or R5LOG_PAYLOAD_PARITY */
	2022	+ } else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) {
	2023	+ /* nothing to do for R5LOG_PAYLOAD_FLUSH here */
	2024	+ } else /* not R5LOG_PAYLOAD_DATA/PARITY/FLUSH */
1872	2025	goto mismatch;
1873	2026
1874		− log_offset = r5l_ring_add(log, log_offset,
1875		− le32_to_cpu(payload->size));
	2027	+ if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) {
	2028	+ mb_offset += sizeof(struct r5l_payload_flush) +
	2029	+ le32_to_cpu(payload_flush->size);
	2030	+ } else {
	2031	+ /* DATA or PARITY payload */
	2032	+ log_offset = r5l_ring_add(log, log_offset,
	2033	+ le32_to_cpu(payload->size));
	2034	+ mb_offset += sizeof(struct r5l_payload_data_parity) +
	2035	+ sizeof(__le32) *
	2036	+ (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
	2037	+ }
1876	2038
1877		− mb_offset += sizeof(struct r5l_payload_data_parity) +
1878		− sizeof(__le32) *
1879		− (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
1880	2039	}
1881	2040
1882	2041	put_page(page);
@@ -1904,6 +2063,7 @@ r5c_recovery_analyze_meta_block(struct r5l_log *log, · 21 unmodified lines
1904	2063	struct r5conf *conf = mddev->private;
1905	2064	struct r5l_meta_block *mb;
1906	2065	struct r5l_payload_data_parity *payload;
	2066	+ struct r5l_payload_flush *payload_flush;
1907	2067	int mb_offset;
1908	2068	sector_t log_offset;
1909	2069	sector_t stripe_sect;
@@ -1929,7 +2089,31 @@ r5c_recovery_analyze_meta_block(struct r5l_log *log, · 19 unmodified lines
1929	2089	int dd;
1930	2090
1931	2091	payload = (void *)mb + mb_offset;
1932		− stripe_sect = (payload->header.type == R5LOG_PAYLOAD_DATA) ?
	2092	+ payload_flush = (void *)mb + mb_offset;
	2093	+
	2094	+ if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) {
	2095	+ int i, count;
	2096	+
	2097	+ count = le32_to_cpu(payload_flush->size) / sizeof(__le64);
	2098	+ for (i = 0; i < count; ++i) {
	2099	+ stripe_sect = le64_to_cpu(payload_flush->flush_stripes[i]);
	2100	+ sh = r5c_recovery_lookup_stripe(cached_stripe_list,
	2101	+ stripe_sect);
	2102	+ if (sh) {
	2103	+ WARN_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
	2104	+ r5l_recovery_reset_stripe(sh);
	2105	+ list_del_init(&sh->lru);
	2106	+ raid5_release_stripe(sh);
	2107	+ }
	2108	+ }
	2109	+
	2110	+ mb_offset += sizeof(struct r5l_payload_flush) +
	2111	+ le32_to_cpu(payload_flush->size);
	2112	+ continue;
	2113	+ }
	2114	+
	2115	+ /* DATA or PARITY payload */
	2116	+ stripe_sect = (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) ?
1933	2117	raid5_compute_sector(
1934	2118	conf, le64_to_cpu(payload->location), 0, &dd,
1935	2119	NULL)
@@ -1967,7 +2151,7 @@ r5c_recovery_analyze_meta_block(struct r5l_log *log, · 31 unmodified lines
1967	2151	list_add_tail(&sh->lru, cached_stripe_list);
1968	2152	}
1969	2153
1970		− if (payload->header.type == R5LOG_PAYLOAD_DATA) {
	2154	+ if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) {
1971	2155	if (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&
1972	2156	test_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags)) {
1973	2157	r5l_recovery_replay_one_stripe(conf, sh, ctx);
@@ -1975,7 +2159,7 @@ r5c_recovery_analyze_meta_block(struct r5l_log *log, · 1 unmodified line
1975	2159	}
1976	2160	r5l_recovery_load_data(log, sh, ctx, payload,
1977	2161	log_offset);
1978		− } else if (payload->header.type == R5LOG_PAYLOAD_PARITY)
	2162	+ } else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_PARITY)
1979	2163	r5l_recovery_load_parity(log, sh, ctx, payload,
1980	2164	log_offset);
1981	2165	else
@@ -2177,7 +2361,7 @@ r5c_recovery_rewrite_data_only_stripes(struct r5l_log *log, · 195 unmodified lines
2177	2361	payload = (void *)mb + offset;
2178	2362	payload->header.type = cpu_to_le16(
2179	2363	R5LOG_PAYLOAD_DATA);
2180		− payload->size = BLOCK_SECTORS;
	2364	+ payload->size = cpu_to_le32(BLOCK_SECTORS);
2181	2365	payload->location = cpu_to_le64(
2182	2366	raid5_compute_blocknr(sh, i, 0));
2183	2367	addr = kmap_atomic(dev->page);
@@ -2241,55 +2425,70 @@ static void r5c_recovery_flush_data_only_stripes(struct r5l_log *log, · 57 unmodified lines
2241	2425	static int r5l_recovery_log(struct r5l_log *log)
2242	2426	{
2243	2427	struct mddev *mddev = log->rdev->mddev;
2244		− struct r5l_recovery_ctx ctx;
	2428	+ struct r5l_recovery_ctx *ctx;
2245	2429	int ret;
2246	2430	sector_t pos;
2247	2431
2248		− ctx.pos = log->last_checkpoint;
2249		− ctx.seq = log->last_cp_seq;
2250		− ctx.meta_page = alloc_page(GFP_KERNEL);
2251		− ctx.data_only_stripes = 0;
2252		− ctx.data_parity_stripes = 0;
2253		− INIT_LIST_HEAD(&ctx.cached_list);
2254		−
2255		− if (!ctx.meta_page)
	2432	+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	2433	+ if (!ctx)
2256	2434	return -ENOMEM;
2257	2435
2258		− ret = r5c_recovery_flush_log(log, &ctx);
2259		− __free_page(ctx.meta_page);
	2436	+ ctx->pos = log->last_checkpoint;
	2437	+ ctx->seq = log->last_cp_seq;
	2438	+ INIT_LIST_HEAD(&ctx->cached_list);
	2439	+ ctx->meta_page = alloc_page(GFP_KERNEL);
2260	2440
2261		− if (ret)
2262		− return ret;
	2441	+ if (!ctx->meta_page) {
	2442	+ ret = -ENOMEM;
	2443	+ goto meta_page;
	2444	+ }
2263	2445
2264		− pos = ctx.pos;
2265		− ctx.seq += 10000;
	2446	+ if (r5l_recovery_allocate_ra_pool(log, ctx) != 0) {
	2447	+ ret = -ENOMEM;
	2448	+ goto ra_pool;
	2449	+ }
2266	2450
	2451	+ ret = r5c_recovery_flush_log(log, ctx);
2267	2452
2268		− if ((ctx.data_only_stripes == 0) && (ctx.data_parity_stripes == 0))
	2453	+ if (ret)
	2454	+ goto error;
	2455	+
	2456	+ pos = ctx->pos;
	2457	+ ctx->seq += 10000;
	2458	+
	2459	+ if ((ctx->data_only_stripes == 0) && (ctx->data_parity_stripes == 0))
2269	2460	pr_debug("md/raid:%s: starting from clean shutdown\n",
2270	2461	mdname(mddev));
2271	2462	else
2272	2463	pr_debug("md/raid:%s: recovering %d data-only stripes and %d data-parity stripes\n",
2273		− mdname(mddev), ctx.data_only_stripes,
2274		− ctx.data_parity_stripes);
2275		−
2276		− if (ctx.data_only_stripes == 0) {
2277		− log->next_checkpoint = ctx.pos;
2278		− r5l_log_write_empty_meta_block(log, ctx.pos, ctx.seq++);
2279		− ctx.pos = r5l_ring_add(log, ctx.pos, BLOCK_SECTORS);
2280		− } else if (r5c_recovery_rewrite_data_only_stripes(log, &ctx)) {
	2464	+ mdname(mddev), ctx->data_only_stripes,
	2465	+ ctx->data_parity_stripes);
	2466	+
	2467	+ if (ctx->data_only_stripes == 0) {
	2468	+ log->next_checkpoint = ctx->pos;
	2469	+ r5l_log_write_empty_meta_block(log, ctx->pos, ctx->seq++);
	2470	+ ctx->pos = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
	2471	+ } else if (r5c_recovery_rewrite_data_only_stripes(log, ctx)) {
2281	2472	pr_err("md/raid:%s: failed to rewrite stripes to journal\n",
2282	2473	mdname(mddev));
2283		− return -EIO;
	2474	+ ret = -EIO;
	2475	+ goto error;
2284	2476	}
2285	2477
2286		− log->log_start = ctx.pos;
2287		− log->seq = ctx.seq;
	2478	+ log->log_start = ctx->pos;
	2479	+ log->seq = ctx->seq;
2288	2480	log->last_checkpoint = pos;
2289	2481	r5l_write_super(log, pos);
2290	2482
2291		− r5c_recovery_flush_data_only_stripes(log, &ctx);
2292		− return 0;
	2483	+ r5c_recovery_flush_data_only_stripes(log, ctx);
	2484	+ ret = 0;
	2485	+error:
	2486	+ r5l_recovery_free_ra_pool(log, ctx);
	2487	+ra_pool:
	2488	+ __free_page(ctx->meta_page);
	2489	+meta_page:
	2490	+ kfree(ctx);
	2491	+ return ret;
2293	2492	}
2294	2493
2295	2494	static void r5l_write_super(struct r5l_log *log, sector_t cp)
@@ -2618,11 +2817,11 @@ void r5c_finish_stripe_write_out(struct r5conf *conf, · 322 unmodified lines
2618	2817	atomic_dec(&conf->r5c_flushing_full_stripes);
2619	2818	atomic_dec(&conf->r5c_cached_full_stripes);
2620	2819	}
	2820	+
	2821	+ r5l_append_flush_payload(log, sh->sector);
2621	2822	}
2622	2823
2623		−int
2624		−r5c_cache_data(struct r5l_log log, struct stripe_head sh,
2625		− struct stripe_head_state *s)
	2824	+int r5c_cache_data(struct r5l_log log, struct stripe_head sh)
2626	2825	{
2627	2826	struct r5conf *conf = sh->raid_conf;
2628	2827	int pages = 0;
@@ -2785,6 +2984,10 @@ int r5l_init_log(struct r5conf conf, struct md_rdev rdev) · 156 unmodified lines
2785	2984	{
2786	2985	struct request_queue *q = bdev_get_queue(rdev->bdev);
2787	2986	struct r5l_log *log;
	2987	+ char b[BDEVNAME_SIZE];
	2988	+
	2989	+ pr_debug("md/raid:%s: using device %s as journal\n",
	2990	+ mdname(conf->mddev), bdevname(rdev->bdev, b));
2788	2991
2789	2992	if (PAGE_SIZE != 4096)
2790	2993	return -EINVAL;
@@ -2887,8 +3090,13 @@ io_kc: · 96 unmodified lines
2887	3090	return -EINVAL;
2888	3091	}
2889	3092
2890		−void r5l_exit_log(struct r5l_log *log)
	3093	+void r5l_exit_log(struct r5conf *conf)
2891	3094	{
	3095	+ struct r5l_log *log = conf->log;
	3096	+
	3097	+ conf->log = NULL;
	3098	+ synchronize_rcu();
	3099	+
2892	3100	flush_work(&log->disable_writeback_work);
2893	3101	md_unregister_thread(&log->reclaim_thread);
2894	3102	mempool_destroy(log->meta_pool);

drivers/md/raid5-log.h+115−0

@@ -0,0 +1,115 @@
	1	+#ifndef _RAID5_LOG_H
	2	+#define _RAID5_LOG_H
	3	+
	4	+extern int r5l_init_log(struct r5conf conf, struct md_rdev rdev);
	5	+extern void r5l_exit_log(struct r5conf *conf);
	6	+extern int r5l_write_stripe(struct r5l_log log, struct stripe_head head_sh);
	7	+extern void r5l_write_stripe_run(struct r5l_log *log);
	8	+extern void r5l_flush_stripe_to_raid(struct r5l_log *log);
	9	+extern void r5l_stripe_write_finished(struct stripe_head *sh);
	10	+extern int r5l_handle_flush_request(struct r5l_log log, struct bio bio);
	11	+extern void r5l_quiesce(struct r5l_log *log, int state);
	12	+extern bool r5l_log_disk_error(struct r5conf *conf);
	13	+extern bool r5c_is_writeback(struct r5l_log *log);
	14	+extern int
	15	+r5c_try_caching_write(struct r5conf conf, struct stripe_head sh,
	16	+ struct stripe_head_state *s, int disks);
	17	+extern void
	18	+r5c_finish_stripe_write_out(struct r5conf conf, struct stripe_head sh,
	19	+ struct stripe_head_state *s);
	20	+extern void r5c_release_extra_page(struct stripe_head *sh);
	21	+extern void r5c_use_extra_page(struct stripe_head *sh);
	22	+extern void r5l_wake_reclaim(struct r5l_log *log, sector_t space);
	23	+extern void r5c_handle_cached_data_endio(struct r5conf *conf,
	24	+ struct stripe_head *sh, int disks);
	25	+extern int r5c_cache_data(struct r5l_log log, struct stripe_head sh);
	26	+extern void r5c_make_stripe_write_out(struct stripe_head *sh);
	27	+extern void r5c_flush_cache(struct r5conf *conf, int num);
	28	+extern void r5c_check_stripe_cache_usage(struct r5conf *conf);
	29	+extern void r5c_check_cached_full_stripe(struct r5conf *conf);
	30	+extern struct md_sysfs_entry r5c_journal_mode;
	31	+extern void r5c_update_on_rdev_error(struct mddev *mddev);
	32	+extern bool r5c_big_stripe_cached(struct r5conf *conf, sector_t sect);
	33	+
	34	+extern struct dma_async_tx_descriptor *
	35	+ops_run_partial_parity(struct stripe_head sh, struct raid5_percpu percpu,
	36	+ struct dma_async_tx_descriptor *tx);
	37	+extern int ppl_init_log(struct r5conf *conf);
	38	+extern void ppl_exit_log(struct r5conf *conf);
	39	+extern int ppl_write_stripe(struct r5conf conf, struct stripe_head sh);
	40	+extern void ppl_write_stripe_run(struct r5conf *conf);
	41	+extern void ppl_stripe_write_finished(struct stripe_head *sh);
	42	+extern int ppl_modify_log(struct r5conf conf, struct md_rdev rdev, bool add);
	43	+
	44	+static inline bool raid5_has_ppl(struct r5conf *conf)
	45	+{
	46	+ return test_bit(MD_HAS_PPL, &conf->mddev->flags);
	47	+}
	48	+
	49	+static inline int log_stripe(struct stripe_head sh, struct stripe_head_state s)
	50	+{
	51	+ struct r5conf *conf = sh->raid_conf;
	52	+
	53	+ if (conf->log) {
	54	+ if (!test_bit(STRIPE_R5C_CACHING, &sh->state)) {
	55	+ /* writing out phase */
	56	+ if (s->waiting_extra_page)
	57	+ return 0;
	58	+ return r5l_write_stripe(conf->log, sh);
	59	+ } else if (test_bit(STRIPE_LOG_TRAPPED, &sh->state)) {
	60	+ /* caching phase */
	61	+ return r5c_cache_data(conf->log, sh);
	62	+ }
	63	+ } else if (raid5_has_ppl(conf)) {
	64	+ return ppl_write_stripe(conf, sh);
	65	+ }
	66	+
	67	+ return -EAGAIN;
	68	+}
	69	+
	70	+static inline void log_stripe_write_finished(struct stripe_head *sh)
	71	+{
	72	+ struct r5conf *conf = sh->raid_conf;
	73	+
	74	+ if (conf->log)
	75	+ r5l_stripe_write_finished(sh);
	76	+ else if (raid5_has_ppl(conf))
	77	+ ppl_stripe_write_finished(sh);
	78	+}
	79	+
	80	+static inline void log_write_stripe_run(struct r5conf *conf)
	81	+{
	82	+ if (conf->log)
	83	+ r5l_write_stripe_run(conf->log);
	84	+ else if (raid5_has_ppl(conf))
	85	+ ppl_write_stripe_run(conf);
	86	+}
	87	+
	88	+static inline void log_exit(struct r5conf *conf)
	89	+{
	90	+ if (conf->log)
	91	+ r5l_exit_log(conf);
	92	+ else if (raid5_has_ppl(conf))
	93	+ ppl_exit_log(conf);
	94	+}
	95	+
	96	+static inline int log_init(struct r5conf conf, struct md_rdev journal_dev,
	97	+ bool ppl)
	98	+{
	99	+ if (journal_dev)
	100	+ return r5l_init_log(conf, journal_dev);
	101	+ else if (ppl)
	102	+ return ppl_init_log(conf);
	103	+
	104	+ return 0;
	105	+}
	106	+
	107	+static inline int log_modify(struct r5conf conf, struct md_rdev rdev, bool add)
	108	+{
	109	+ if (raid5_has_ppl(conf))
	110	+ return ppl_modify_log(conf, rdev, add);
	111	+
	112	+ return 0;
	113	+}
	114	+
	115	+#endif

drivers/md/raid5-ppl.c+1271−0

@@ -0,0 +1,1271 @@
	1	+/*
	2	+ * Partial Parity Log for closing the RAID5 write hole
	3	+ * Copyright (c) 2017, Intel Corporation.
	4	+ *
	5	+ * This program is free software; you can redistribute it and/or modify it
	6	+ * under the terms and conditions of the GNU General Public License,
	7	+ * version 2, as published by the Free Software Foundation.
	8	+ *
	9	+ * This program is distributed in the hope it will be useful, but WITHOUT
	10	+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	11	+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	12	+ * more details.
	13	+ */
	14	+
	15	+#include <linux/kernel.h>
	16	+#include <linux/blkdev.h>
	17	+#include <linux/slab.h>
	18	+#include <linux/crc32c.h>
	19	+#include <linux/flex_array.h>
	20	+#include <linux/async_tx.h>
	21	+#include <linux/raid/md_p.h>
	22	+#include "md.h"
	23	+#include "raid5.h"
	24	+
	25	+/*
	26	+ * PPL consists of a 4KB header (struct ppl_header) and at least 128KB for
	27	+ * partial parity data. The header contains an array of entries
	28	+ * (struct ppl_header_entry) which describe the logged write requests.
	29	+ * Partial parity for the entries comes after the header, written in the same
	30	+ * sequence as the entries:
	31	+ *
	32	+ * Header
	33	+ * entry0
	34	+ * ...
	35	+ * entryN
	36	+ * PP data
	37	+ * PP for entry0
	38	+ * ...
	39	+ * PP for entryN
	40	+ *
	41	+ * An entry describes one or more consecutive stripe_heads, up to a full
	42	+ * stripe. The modifed raid data chunks form an m-by-n matrix, where m is the
	43	+ * number of stripe_heads in the entry and n is the number of modified data
	44	+ * disks. Every stripe_head in the entry must write to the same data disks.
	45	+ * An example of a valid case described by a single entry (writes to the first
	46	+ * stripe of a 4 disk array, 16k chunk size):
	47	+ *
	48	+ * sh->sector dd0 dd1 dd2 ppl
	49	+ * +-----+-----+-----+
	50	+ * 0 \| --- \| --- \| --- \| +----+
	51	+ * 8 \| -W- \| -W- \| --- \| \| pp \| data_sector = 8
	52	+ * 16 \| -W- \| -W- \| --- \| \| pp \| data_size = 3 * 2 * 4k
	53	+ * 24 \| -W- \| -W- \| --- \| \| pp \| pp_size = 3 * 4k
	54	+ * +-----+-----+-----+ +----+
	55	+ *
	56	+ * data_sector is the first raid sector of the modified data, data_size is the
	57	+ * total size of modified data and pp_size is the size of partial parity for
	58	+ * this entry. Entries for full stripe writes contain no partial parity
	59	+ * (pp_size = 0), they only mark the stripes for which parity should be
	60	+ * recalculated after an unclean shutdown. Every entry holds a checksum of its
	61	+ * partial parity, the header also has a checksum of the header itself.
	62	+ *
	63	+ * A write request is always logged to the PPL instance stored on the parity
	64	+ * disk of the corresponding stripe. For each member disk there is one ppl_log
	65	+ * used to handle logging for this disk, independently from others. They are
	66	+ * grouped in child_logs array in struct ppl_conf, which is assigned to
	67	+ * r5conf->log_private.
	68	+ *
	69	+ * ppl_io_unit represents a full PPL write, header_page contains the ppl_header.
	70	+ * PPL entries for logged stripes are added in ppl_log_stripe(). A stripe_head
	71	+ * can be appended to the last entry if it meets the conditions for a valid
	72	+ * entry described above, otherwise a new entry is added. Checksums of entries
	73	+ * are calculated incrementally as stripes containing partial parity are being
	74	+ * added. ppl_submit_iounit() calculates the checksum of the header and submits
	75	+ * a bio containing the header page and partial parity pages (sh->ppl_page) for
	76	+ * all stripes of the io_unit. When the PPL write completes, the stripes
	77	+ * associated with the io_unit are released and raid5d starts writing their data
	78	+ * and parity. When all stripes are written, the io_unit is freed and the next
	79	+ * can be submitted.
	80	+ *
	81	+ * An io_unit is used to gather stripes until it is submitted or becomes full
	82	+ * (if the maximum number of entries or size of PPL is reached). Another io_unit
	83	+ * can't be submitted until the previous has completed (PPL and stripe
	84	+ * data+parity is written). The log->io_list tracks all io_units of a log
	85	+ * (for a single member disk). New io_units are added to the end of the list
	86	+ * and the first io_unit is submitted, if it is not submitted already.
	87	+ * The current io_unit accepting new stripes is always at the end of the list.
	88	+ */
	89	+
	90	+struct ppl_conf {
	91	+ struct mddev *mddev;
	92	+
	93	+ /* array of child logs, one for each raid disk */
	94	+ struct ppl_log *child_logs;
	95	+ int count;
	96	+
	97	+ int block_size; /* the logical block size used for data_sector
	98	+ * in ppl_header_entry */
	99	+ u32 signature; /* raid array identifier */
	100	+ atomic64_t seq; /* current log write sequence number */
	101	+
	102	+ struct kmem_cache *io_kc;
	103	+ mempool_t *io_pool;
	104	+ struct bio_set *bs;
	105	+
	106	+ /* used only for recovery */
	107	+ int recovered_entries;
	108	+ int mismatch_count;
	109	+
	110	+ /* stripes to retry if failed to allocate io_unit */
	111	+ struct list_head no_mem_stripes;
	112	+ spinlock_t no_mem_stripes_lock;
	113	+};
	114	+
	115	+struct ppl_log {
	116	+ struct ppl_conf ppl_conf; / shared between all log instances */
	117	+
	118	+ struct md_rdev rdev; / array member disk associated with
	119	+ * this log instance */
	120	+ struct mutex io_mutex;
	121	+ struct ppl_io_unit current_io; / current io_unit accepting new data
	122	+ * always at the end of io_list */
	123	+ spinlock_t io_list_lock;
	124	+ struct list_head io_list; /* all io_units of this log */
	125	+};
	126	+
	127	+#define PPL_IO_INLINE_BVECS 32
	128	+
	129	+struct ppl_io_unit {
	130	+ struct ppl_log *log;
	131	+
	132	+ struct page header_page; / for ppl_header */
	133	+
	134	+ unsigned int entries_count; /* number of entries in ppl_header */
	135	+ unsigned int pp_size; /* total size current of partial parity */
	136	+
	137	+ u64 seq; /* sequence number of this log write */
	138	+ struct list_head log_sibling; /* log->io_list */
	139	+
	140	+ struct list_head stripe_list; /* stripes added to the io_unit */
	141	+ atomic_t pending_stripes; /* how many stripes not written to raid */
	142	+
	143	+ bool submitted; /* true if write to log started */
	144	+
	145	+ /* inline bio and its biovec for submitting the iounit */
	146	+ struct bio bio;
	147	+ struct bio_vec biovec[PPL_IO_INLINE_BVECS];
	148	+};
	149	+
	150	+struct dma_async_tx_descriptor *
	151	+ops_run_partial_parity(struct stripe_head sh, struct raid5_percpu percpu,
	152	+ struct dma_async_tx_descriptor *tx)
	153	+{
	154	+ int disks = sh->disks;
	155	+ struct page **srcs = flex_array_get(percpu->scribble, 0);
	156	+ int count = 0, pd_idx = sh->pd_idx, i;
	157	+ struct async_submit_ctl submit;
	158	+
	159	+ pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
	160	+
	161	+ /*
	162	+ * Partial parity is the XOR of stripe data chunks that are not changed
	163	+ * during the write request. Depending on available data
	164	+ * (read-modify-write vs. reconstruct-write case) we calculate it
	165	+ * differently.
	166	+ */
	167	+ if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
	168	+ /*
	169	+ * rmw: xor old data and parity from updated disks
	170	+ * This is calculated earlier by ops_run_prexor5() so just copy
	171	+ * the parity dev page.
	172	+ */
	173	+ srcs[count++] = sh->dev[pd_idx].page;
	174	+ } else if (sh->reconstruct_state == reconstruct_state_drain_run) {
	175	+ /* rcw: xor data from all not updated disks */
	176	+ for (i = disks; i--;) {
	177	+ struct r5dev *dev = &sh->dev[i];
	178	+ if (test_bit(R5_UPTODATE, &dev->flags))
	179	+ srcs[count++] = dev->page;
	180	+ }
	181	+ } else {
	182	+ return tx;
	183	+ }
	184	+
	185	+ init_async_submit(&submit, ASYNC_TX_FENCE\|ASYNC_TX_XOR_ZERO_DST, tx,
	186	+ NULL, sh, flex_array_get(percpu->scribble, 0)
	187	+ + sizeof(struct page ) (sh->disks + 2));
	188	+
	189	+ if (count == 1)
	190	+ tx = async_memcpy(sh->ppl_page, srcs[0], 0, 0, PAGE_SIZE,
	191	+ &submit);
	192	+ else
	193	+ tx = async_xor(sh->ppl_page, srcs, 0, count, PAGE_SIZE,
	194	+ &submit);
	195	+
	196	+ return tx;
	197	+}
	198	+
	199	+static void ppl_io_pool_alloc(gfp_t gfp_mask, void pool_data)
	200	+{
	201	+ struct kmem_cache *kc = pool_data;
	202	+ struct ppl_io_unit *io;
	203	+
	204	+ io = kmem_cache_alloc(kc, gfp_mask);
	205	+ if (!io)
	206	+ return NULL;
	207	+
	208	+ io->header_page = alloc_page(gfp_mask);
	209	+ if (!io->header_page) {
	210	+ kmem_cache_free(kc, io);
	211	+ return NULL;
	212	+ }
	213	+
	214	+ return io;
	215	+}
	216	+
	217	+static void ppl_io_pool_free(void element, void pool_data)
	218	+{
	219	+ struct kmem_cache *kc = pool_data;
	220	+ struct ppl_io_unit *io = element;
	221	+
	222	+ __free_page(io->header_page);
	223	+ kmem_cache_free(kc, io);
	224	+}
	225	+
	226	+static struct ppl_io_unit ppl_new_iounit(struct ppl_log log,
	227	+ struct stripe_head *sh)
	228	+{
	229	+ struct ppl_conf *ppl_conf = log->ppl_conf;
	230	+ struct ppl_io_unit *io;
	231	+ struct ppl_header *pplhdr;
	232	+ struct page *header_page;
	233	+
	234	+ io = mempool_alloc(ppl_conf->io_pool, GFP_NOWAIT);
	235	+ if (!io)
	236	+ return NULL;
	237	+
	238	+ header_page = io->header_page;
	239	+ memset(io, 0, sizeof(*io));
	240	+ io->header_page = header_page;
	241	+
	242	+ io->log = log;
	243	+ INIT_LIST_HEAD(&io->log_sibling);
	244	+ INIT_LIST_HEAD(&io->stripe_list);
	245	+ atomic_set(&io->pending_stripes, 0);
	246	+ bio_init(&io->bio, io->biovec, PPL_IO_INLINE_BVECS);
	247	+
	248	+ pplhdr = page_address(io->header_page);
	249	+ clear_page(pplhdr);
	250	+ memset(pplhdr->reserved, 0xff, PPL_HDR_RESERVED);
	251	+ pplhdr->signature = cpu_to_le32(ppl_conf->signature);
	252	+
	253	+ io->seq = atomic64_add_return(1, &ppl_conf->seq);
	254	+ pplhdr->generation = cpu_to_le64(io->seq);
	255	+
	256	+ return io;
	257	+}
	258	+
	259	+static int ppl_log_stripe(struct ppl_log log, struct stripe_head sh)
	260	+{
	261	+ struct ppl_io_unit *io = log->current_io;
	262	+ struct ppl_header_entry *e = NULL;
	263	+ struct ppl_header *pplhdr;
	264	+ int i;
	265	+ sector_t data_sector = 0;
	266	+ int data_disks = 0;
	267	+ unsigned int entry_space = (log->rdev->ppl.size << 9) - PPL_HEADER_SIZE;
	268	+ struct r5conf *conf = sh->raid_conf;
	269	+
	270	+ pr_debug("%s: stripe: %llu\n", __func__, (unsigned long long)sh->sector);
	271	+
	272	+ /* check if current io_unit is full */
	273	+ if (io && (io->pp_size == entry_space \|\|
	274	+ io->entries_count == PPL_HDR_MAX_ENTRIES)) {
	275	+ pr_debug("%s: add io_unit blocked by seq: %llu\n",
	276	+ __func__, io->seq);
	277	+ io = NULL;
	278	+ }
	279	+
	280	+ /* add a new unit if there is none or the current is full */
	281	+ if (!io) {
	282	+ io = ppl_new_iounit(log, sh);
	283	+ if (!io)
	284	+ return -ENOMEM;
	285	+ spin_lock_irq(&log->io_list_lock);
	286	+ list_add_tail(&io->log_sibling, &log->io_list);
	287	+ spin_unlock_irq(&log->io_list_lock);
	288	+
	289	+ log->current_io = io;
	290	+ }
	291	+
	292	+ for (i = 0; i < sh->disks; i++) {
	293	+ struct r5dev *dev = &sh->dev[i];
	294	+
	295	+ if (i != sh->pd_idx && test_bit(R5_Wantwrite, &dev->flags)) {
	296	+ if (!data_disks \|\| dev->sector < data_sector)
	297	+ data_sector = dev->sector;
	298	+ data_disks++;
	299	+ }
	300	+ }
	301	+ BUG_ON(!data_disks);
	302	+
	303	+ pr_debug("%s: seq: %llu data_sector: %llu data_disks: %d\n", __func__,
	304	+ io->seq, (unsigned long long)data_sector, data_disks);
	305	+
	306	+ pplhdr = page_address(io->header_page);
	307	+
	308	+ if (io->entries_count > 0) {
	309	+ struct ppl_header_entry *last =
	310	+ &pplhdr->entries[io->entries_count - 1];
	311	+ struct stripe_head *sh_last = list_last_entry(
	312	+ &io->stripe_list, struct stripe_head, log_list);
	313	+ u64 data_sector_last = le64_to_cpu(last->data_sector);
	314	+ u32 data_size_last = le32_to_cpu(last->data_size);
	315	+
	316	+ /*
	317	+ * Check if we can append the stripe to the last entry. It must
	318	+ * be just after the last logged stripe and write to the same
	319	+ * disks. Use bit shift and logarithm to avoid 64-bit division.
	320	+ */
	321	+ if ((sh->sector == sh_last->sector + STRIPE_SECTORS) &&
	322	+ (data_sector >> ilog2(conf->chunk_sectors) ==
	323	+ data_sector_last >> ilog2(conf->chunk_sectors)) &&
	324	+ ((data_sector - data_sector_last) * data_disks ==
	325	+ data_size_last >> 9))
	326	+ e = last;
	327	+ }
	328	+
	329	+ if (!e) {
	330	+ e = &pplhdr->entries[io->entries_count++];
	331	+ e->data_sector = cpu_to_le64(data_sector);
	332	+ e->parity_disk = cpu_to_le32(sh->pd_idx);
	333	+ e->checksum = cpu_to_le32(~0);
	334	+ }
	335	+
	336	+ le32_add_cpu(&e->data_size, data_disks << PAGE_SHIFT);
	337	+
	338	+ /* don't write any PP if full stripe write */
	339	+ if (!test_bit(STRIPE_FULL_WRITE, &sh->state)) {
	340	+ le32_add_cpu(&e->pp_size, PAGE_SIZE);
	341	+ io->pp_size += PAGE_SIZE;
	342	+ e->checksum = cpu_to_le32(crc32c_le(le32_to_cpu(e->checksum),
	343	+ page_address(sh->ppl_page),
	344	+ PAGE_SIZE));
	345	+ }
	346	+
	347	+ list_add_tail(&sh->log_list, &io->stripe_list);
	348	+ atomic_inc(&io->pending_stripes);
	349	+ sh->ppl_io = io;
	350	+
	351	+ return 0;
	352	+}
	353	+
	354	+int ppl_write_stripe(struct r5conf conf, struct stripe_head sh)
	355	+{
	356	+ struct ppl_conf *ppl_conf = conf->log_private;
	357	+ struct ppl_io_unit *io = sh->ppl_io;
	358	+ struct ppl_log *log;
	359	+
	360	+ if (io \|\| test_bit(STRIPE_SYNCING, &sh->state) \|\| !sh->ppl_page \|\|
	361	+ !test_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags) \|\|
	362	+ !test_bit(R5_Insync, &sh->dev[sh->pd_idx].flags)) {
	363	+ clear_bit(STRIPE_LOG_TRAPPED, &sh->state);
	364	+ return -EAGAIN;
	365	+ }
	366	+
	367	+ log = &ppl_conf->child_logs[sh->pd_idx];
	368	+
	369	+ mutex_lock(&log->io_mutex);
	370	+
	371	+ if (!log->rdev \|\| test_bit(Faulty, &log->rdev->flags)) {
	372	+ mutex_unlock(&log->io_mutex);
	373	+ return -EAGAIN;
	374	+ }
	375	+
	376	+ set_bit(STRIPE_LOG_TRAPPED, &sh->state);
	377	+ clear_bit(STRIPE_DELAYED, &sh->state);
	378	+ atomic_inc(&sh->count);
	379	+
	380	+ if (ppl_log_stripe(log, sh)) {
	381	+ spin_lock_irq(&ppl_conf->no_mem_stripes_lock);
	382	+ list_add_tail(&sh->log_list, &ppl_conf->no_mem_stripes);
	383	+ spin_unlock_irq(&ppl_conf->no_mem_stripes_lock);
	384	+ }
	385	+
	386	+ mutex_unlock(&log->io_mutex);
	387	+
	388	+ return 0;
	389	+}
	390	+
	391	+static void ppl_log_endio(struct bio *bio)
	392	+{
	393	+ struct ppl_io_unit *io = bio->bi_private;
	394	+ struct ppl_log *log = io->log;
	395	+ struct ppl_conf *ppl_conf = log->ppl_conf;
	396	+ struct stripe_head sh, next;
	397	+
	398	+ pr_debug("%s: seq: %llu\n", __func__, io->seq);
	399	+
	400	+ if (bio->bi_error)
	401	+ md_error(ppl_conf->mddev, log->rdev);
	402	+
	403	+ list_for_each_entry_safe(sh, next, &io->stripe_list, log_list) {
	404	+ list_del_init(&sh->log_list);
	405	+
	406	+ set_bit(STRIPE_HANDLE, &sh->state);
	407	+ raid5_release_stripe(sh);
	408	+ }
	409	+}
	410	+
	411	+static void ppl_submit_iounit_bio(struct ppl_io_unit io, struct bio bio)
	412	+{
	413	+ char b[BDEVNAME_SIZE];
	414	+
	415	+ pr_debug("%s: seq: %llu size: %u sector: %llu dev: %s\n",
	416	+ __func__, io->seq, bio->bi_iter.bi_size,
	417	+ (unsigned long long)bio->bi_iter.bi_sector,
	418	+ bdevname(bio->bi_bdev, b));
	419	+
	420	+ submit_bio(bio);
	421	+}
	422	+
	423	+static void ppl_submit_iounit(struct ppl_io_unit *io)
	424	+{
	425	+ struct ppl_log *log = io->log;
	426	+ struct ppl_conf *ppl_conf = log->ppl_conf;
	427	+ struct ppl_header *pplhdr = page_address(io->header_page);
	428	+ struct bio *bio = &io->bio;
	429	+ struct stripe_head *sh;
	430	+ int i;
	431	+
	432	+ bio->bi_private = io;
	433	+
	434	+ if (!log->rdev \|\| test_bit(Faulty, &log->rdev->flags)) {
	435	+ ppl_log_endio(bio);
	436	+ return;
	437	+ }
	438	+
	439	+ for (i = 0; i < io->entries_count; i++) {
	440	+ struct ppl_header_entry *e = &pplhdr->entries[i];
	441	+
	442	+ pr_debug("%s: seq: %llu entry: %d data_sector: %llu pp_size: %u data_size: %u\n",
	443	+ __func__, io->seq, i, le64_to_cpu(e->data_sector),
	444	+ le32_to_cpu(e->pp_size), le32_to_cpu(e->data_size));
	445	+
	446	+ e->data_sector = cpu_to_le64(le64_to_cpu(e->data_sector) >>
	447	+ ilog2(ppl_conf->block_size >> 9));
	448	+ e->checksum = cpu_to_le32(~le32_to_cpu(e->checksum));
	449	+ }
	450	+
	451	+ pplhdr->entries_count = cpu_to_le32(io->entries_count);
	452	+ pplhdr->checksum = cpu_to_le32(~crc32c_le(~0, pplhdr, PPL_HEADER_SIZE));
	453	+
	454	+ bio->bi_end_io = ppl_log_endio;
	455	+ bio->bi_opf = REQ_OP_WRITE \| REQ_FUA;
	456	+ bio->bi_bdev = log->rdev->bdev;
	457	+ bio->bi_iter.bi_sector = log->rdev->ppl.sector;
	458	+ bio_add_page(bio, io->header_page, PAGE_SIZE, 0);
	459	+
	460	+ list_for_each_entry(sh, &io->stripe_list, log_list) {
	461	+ /* entries for full stripe writes have no partial parity */
	462	+ if (test_bit(STRIPE_FULL_WRITE, &sh->state))
	463	+ continue;
	464	+
	465	+ if (!bio_add_page(bio, sh->ppl_page, PAGE_SIZE, 0)) {
	466	+ struct bio *prev = bio;
	467	+
	468	+ bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_PAGES,
	469	+ ppl_conf->bs);
	470	+ bio->bi_opf = prev->bi_opf;
	471	+ bio->bi_bdev = prev->bi_bdev;
	472	+ bio->bi_iter.bi_sector = bio_end_sector(prev);
	473	+ bio_add_page(bio, sh->ppl_page, PAGE_SIZE, 0);
	474	+
	475	+ bio_chain(bio, prev);
	476	+ ppl_submit_iounit_bio(io, prev);
	477	+ }
	478	+ }
	479	+
	480	+ ppl_submit_iounit_bio(io, bio);
	481	+}
	482	+
	483	+static void ppl_submit_current_io(struct ppl_log *log)
	484	+{
	485	+ struct ppl_io_unit *io;
	486	+
	487	+ spin_lock_irq(&log->io_list_lock);
	488	+
	489	+ io = list_first_entry_or_null(&log->io_list, struct ppl_io_unit,
	490	+ log_sibling);
	491	+ if (io && io->submitted)
	492	+ io = NULL;
	493	+
	494	+ spin_unlock_irq(&log->io_list_lock);
	495	+
	496	+ if (io) {
	497	+ io->submitted = true;
	498	+
	499	+ if (io == log->current_io)
	500	+ log->current_io = NULL;
	501	+
	502	+ ppl_submit_iounit(io);
	503	+ }
	504	+}
	505	+
	506	+void ppl_write_stripe_run(struct r5conf *conf)
	507	+{
	508	+ struct ppl_conf *ppl_conf = conf->log_private;
	509	+ struct ppl_log *log;
	510	+ int i;
	511	+
	512	+ for (i = 0; i < ppl_conf->count; i++) {
	513	+ log = &ppl_conf->child_logs[i];
	514	+
	515	+ mutex_lock(&log->io_mutex);
	516	+ ppl_submit_current_io(log);
	517	+ mutex_unlock(&log->io_mutex);
	518	+ }
	519	+}
	520	+
	521	+static void ppl_io_unit_finished(struct ppl_io_unit *io)
	522	+{
	523	+ struct ppl_log *log = io->log;
	524	+ struct ppl_conf *ppl_conf = log->ppl_conf;
	525	+ unsigned long flags;
	526	+
	527	+ pr_debug("%s: seq: %llu\n", __func__, io->seq);
	528	+
	529	+ local_irq_save(flags);
	530	+
	531	+ spin_lock(&log->io_list_lock);
	532	+ list_del(&io->log_sibling);
	533	+ spin_unlock(&log->io_list_lock);
	534	+
	535	+ mempool_free(io, ppl_conf->io_pool);
	536	+
	537	+ spin_lock(&ppl_conf->no_mem_stripes_lock);
	538	+ if (!list_empty(&ppl_conf->no_mem_stripes)) {
	539	+ struct stripe_head *sh;
	540	+
	541	+ sh = list_first_entry(&ppl_conf->no_mem_stripes,
	542	+ struct stripe_head, log_list);
	543	+ list_del_init(&sh->log_list);
	544	+ set_bit(STRIPE_HANDLE, &sh->state);
	545	+ raid5_release_stripe(sh);
	546	+ }
	547	+ spin_unlock(&ppl_conf->no_mem_stripes_lock);
	548	+
	549	+ local_irq_restore(flags);
	550	+}
	551	+
	552	+void ppl_stripe_write_finished(struct stripe_head *sh)
	553	+{
	554	+ struct ppl_io_unit *io;
	555	+
	556	+ io = sh->ppl_io;
	557	+ sh->ppl_io = NULL;
	558	+
	559	+ if (io && atomic_dec_and_test(&io->pending_stripes))
	560	+ ppl_io_unit_finished(io);
	561	+}
	562	+
	563	+static void ppl_xor(int size, struct page page1, struct page page2)
	564	+{
	565	+ struct async_submit_ctl submit;
	566	+ struct dma_async_tx_descriptor *tx;
	567	+ struct page *xor_srcs[] = { page1, page2 };
	568	+
	569	+ init_async_submit(&submit, ASYNC_TX_ACK\|ASYNC_TX_XOR_DROP_DST,
	570	+ NULL, NULL, NULL, NULL);
	571	+ tx = async_xor(page1, xor_srcs, 0, 2, size, &submit);
	572	+
	573	+ async_tx_quiesce(&tx);
	574	+}
	575	+
	576	+/*
	577	+ * PPL recovery strategy: xor partial parity and data from all modified data
	578	+ * disks within a stripe and write the result as the new stripe parity. If all
	579	+ * stripe data disks are modified (full stripe write), no partial parity is
	580	+ * available, so just xor the data disks.
	581	+ *
	582	+ * Recovery of a PPL entry shall occur only if all modified data disks are
	583	+ * available and read from all of them succeeds.
	584	+ *
	585	+ * A PPL entry applies to a stripe, partial parity size for an entry is at most
	586	+ * the size of the chunk. Examples of possible cases for a single entry:
	587	+ *
	588	+ * case 0: single data disk write:
	589	+ * data0 data1 data2 ppl parity
	590	+ * +--------+--------+--------+ +--------------------+
	591	+ * \| ------ \| ------ \| ------ \| +----+ \| (no change) \|
	592	+ * \| ------ \| -data- \| ------ \| \| pp \| -> \| data1 ^ pp \|
	593	+ * \| ------ \| -data- \| ------ \| \| pp \| -> \| data1 ^ pp \|
	594	+ * \| ------ \| ------ \| ------ \| +----+ \| (no change) \|
	595	+ * +--------+--------+--------+ +--------------------+
	596	+ * pp_size = data_size
	597	+ *
	598	+ * case 1: more than one data disk write:
	599	+ * data0 data1 data2 ppl parity
	600	+ * +--------+--------+--------+ +--------------------+
	601	+ * \| ------ \| ------ \| ------ \| +----+ \| (no change) \|
	602	+ * \| -data- \| -data- \| ------ \| \| pp \| -> \| data0 ^ data1 ^ pp \|
	603	+ * \| -data- \| -data- \| ------ \| \| pp \| -> \| data0 ^ data1 ^ pp \|
	604	+ * \| ------ \| ------ \| ------ \| +----+ \| (no change) \|
	605	+ * +--------+--------+--------+ +--------------------+
	606	+ * pp_size = data_size / modified_data_disks
	607	+ *
	608	+ * case 2: write to all data disks (also full stripe write):
	609	+ * data0 data1 data2 parity
	610	+ * +--------+--------+--------+ +--------------------+
	611	+ * \| ------ \| ------ \| ------ \| \| (no change) \|
	612	+ * \| -data- \| -data- \| -data- \| --------> \| xor all data \|
	613	+ * \| ------ \| ------ \| ------ \| --------> \| (no change) \|
	614	+ * \| ------ \| ------ \| ------ \| \| (no change) \|
	615	+ * +--------+--------+--------+ +--------------------+
	616	+ * pp_size = 0
	617	+ *
	618	+ * The following cases are possible only in other implementations. The recovery
	619	+ * code can handle them, but they are not generated at runtime because they can
	620	+ * be reduced to cases 0, 1 and 2:
	621	+ *
	622	+ * case 3:
	623	+ * data0 data1 data2 ppl parity
	624	+ * +--------+--------+--------+ +----+ +--------------------+
	625	+ * \| ------ \| -data- \| -data- \| \| pp \| \| data1 ^ data2 ^ pp \|
	626	+ * \| ------ \| -data- \| -data- \| \| pp \| -> \| data1 ^ data2 ^ pp \|
	627	+ * \| -data- \| -data- \| -data- \| \| -- \| -> \| xor all data \|
	628	+ * \| -data- \| -data- \| ------ \| \| pp \| \| data0 ^ data1 ^ pp \|
	629	+ * +--------+--------+--------+ +----+ +--------------------+
	630	+ * pp_size = chunk_size
	631	+ *
	632	+ * case 4:
	633	+ * data0 data1 data2 ppl parity
	634	+ * +--------+--------+--------+ +----+ +--------------------+
	635	+ * \| ------ \| -data- \| ------ \| \| pp \| \| data1 ^ pp \|
	636	+ * \| ------ \| ------ \| ------ \| \| -- \| -> \| (no change) \|
	637	+ * \| ------ \| ------ \| ------ \| \| -- \| -> \| (no change) \|
	638	+ * \| -data- \| ------ \| ------ \| \| pp \| \| data0 ^ pp \|
	639	+ * +--------+--------+--------+ +----+ +--------------------+
	640	+ * pp_size = chunk_size
	641	+ */
	642	+static int ppl_recover_entry(struct ppl_log log, struct ppl_header_entry e,
	643	+ sector_t ppl_sector)
	644	+{
	645	+ struct ppl_conf *ppl_conf = log->ppl_conf;
	646	+ struct mddev *mddev = ppl_conf->mddev;
	647	+ struct r5conf *conf = mddev->private;
	648	+ int block_size = ppl_conf->block_size;
	649	+ struct page *page1;
	650	+ struct page *page2;
	651	+ sector_t r_sector_first;
	652	+ sector_t r_sector_last;
	653	+ int strip_sectors;
	654	+ int data_disks;
	655	+ int i;
	656	+ int ret = 0;
	657	+ char b[BDEVNAME_SIZE];
	658	+ unsigned int pp_size = le32_to_cpu(e->pp_size);
	659	+ unsigned int data_size = le32_to_cpu(e->data_size);
	660	+
	661	+ page1 = alloc_page(GFP_KERNEL);
	662	+ page2 = alloc_page(GFP_KERNEL);
	663	+
	664	+ if (!page1 \|\| !page2) {
	665	+ ret = -ENOMEM;
	666	+ goto out;
	667	+ }
	668	+
	669	+ r_sector_first = le64_to_cpu(e->data_sector) * (block_size >> 9);
	670	+
	671	+ if ((pp_size >> 9) < conf->chunk_sectors) {
	672	+ if (pp_size > 0) {
	673	+ data_disks = data_size / pp_size;
	674	+ strip_sectors = pp_size >> 9;
	675	+ } else {
	676	+ data_disks = conf->raid_disks - conf->max_degraded;
	677	+ strip_sectors = (data_size >> 9) / data_disks;
	678	+ }
	679	+ r_sector_last = r_sector_first +
	680	+ (data_disks - 1) * conf->chunk_sectors +
	681	+ strip_sectors;
	682	+ } else {
	683	+ data_disks = conf->raid_disks - conf->max_degraded;
	684	+ strip_sectors = conf->chunk_sectors;
	685	+ r_sector_last = r_sector_first + (data_size >> 9);
	686	+ }
	687	+
	688	+ pr_debug("%s: array sector first: %llu last: %llu\n", __func__,
	689	+ (unsigned long long)r_sector_first,
	690	+ (unsigned long long)r_sector_last);
	691	+
	692	+ /* if start and end is 4k aligned, use a 4k block */
	693	+ if (block_size == 512 &&
	694	+ (r_sector_first & (STRIPE_SECTORS - 1)) == 0 &&
	695	+ (r_sector_last & (STRIPE_SECTORS - 1)) == 0)
	696	+ block_size = STRIPE_SIZE;
	697	+
	698	+ /* iterate through blocks in strip */
	699	+ for (i = 0; i < strip_sectors; i += (block_size >> 9)) {
	700	+ bool update_parity = false;
	701	+ sector_t parity_sector;
	702	+ struct md_rdev *parity_rdev;
	703	+ struct stripe_head sh;
	704	+ int disk;
	705	+ int indent = 0;
	706	+
	707	+ pr_debug("%s:%*s iter %d start\n", __func__, indent, "", i);
	708	+ indent += 2;
	709	+
	710	+ memset(page_address(page1), 0, PAGE_SIZE);
	711	+
	712	+ /* iterate through data member disks */
	713	+ for (disk = 0; disk < data_disks; disk++) {
	714	+ int dd_idx;
	715	+ struct md_rdev *rdev;
	716	+ sector_t sector;
	717	+ sector_t r_sector = r_sector_first + i +
	718	+ (disk * conf->chunk_sectors);
	719	+
	720	+ pr_debug("%s:%*s data member disk %d start\n",
	721	+ __func__, indent, "", disk);
	722	+ indent += 2;
	723	+
	724	+ if (r_sector >= r_sector_last) {
	725	+ pr_debug("%s:%*s array sector %llu doesn't need parity update\n",
	726	+ __func__, indent, "",
	727	+ (unsigned long long)r_sector);
	728	+ indent -= 2;
	729	+ continue;
	730	+ }
	731	+
	732	+ update_parity = true;
	733	+
	734	+ /* map raid sector to member disk */
	735	+ sector = raid5_compute_sector(conf, r_sector, 0,
	736	+ &dd_idx, NULL);
	737	+ pr_debug("%s:%*s processing array sector %llu => data member disk %d, sector %llu\n",
	738	+ __func__, indent, "",
	739	+ (unsigned long long)r_sector, dd_idx,
	740	+ (unsigned long long)sector);
	741	+
	742	+ rdev = conf->disks[dd_idx].rdev;
	743	+ if (!rdev) {
	744	+ pr_debug("%s:%*s data member disk %d missing\n",
	745	+ __func__, indent, "", dd_idx);
	746	+ update_parity = false;
	747	+ break;
	748	+ }
	749	+
	750	+ pr_debug("%s:%*s reading data member disk %s sector %llu\n",
	751	+ __func__, indent, "", bdevname(rdev->bdev, b),
	752	+ (unsigned long long)sector);
	753	+ if (!sync_page_io(rdev, sector, block_size, page2,
	754	+ REQ_OP_READ, 0, false)) {
	755	+ md_error(mddev, rdev);
	756	+ pr_debug("%s:%*s read failed!\n", __func__,
	757	+ indent, "");
	758	+ ret = -EIO;
	759	+ goto out;
	760	+ }
	761	+
	762	+ ppl_xor(block_size, page1, page2);
	763	+
	764	+ indent -= 2;
	765	+ }
	766	+
	767	+ if (!update_parity)
	768	+ continue;
	769	+
	770	+ if (pp_size > 0) {
	771	+ pr_debug("%s:%*s reading pp disk sector %llu\n",
	772	+ __func__, indent, "",
	773	+ (unsigned long long)(ppl_sector + i));
	774	+ if (!sync_page_io(log->rdev,
	775	+ ppl_sector - log->rdev->data_offset + i,
	776	+ block_size, page2, REQ_OP_READ, 0,
	777	+ false)) {
	778	+ pr_debug("%s:%*s read failed!\n", __func__,
	779	+ indent, "");
	780	+ md_error(mddev, log->rdev);
	781	+ ret = -EIO;
	782	+ goto out;
	783	+ }
	784	+
	785	+ ppl_xor(block_size, page1, page2);
	786	+ }
	787	+
	788	+ /* map raid sector to parity disk */
	789	+ parity_sector = raid5_compute_sector(conf, r_sector_first + i,
	790	+ 0, &disk, &sh);
	791	+ BUG_ON(sh.pd_idx != le32_to_cpu(e->parity_disk));
	792	+ parity_rdev = conf->disks[sh.pd_idx].rdev;
	793	+
	794	+ BUG_ON(parity_rdev->bdev->bd_dev != log->rdev->bdev->bd_dev);
	795	+ pr_debug("%s:%*s write parity at sector %llu, disk %s\n",
	796	+ __func__, indent, "",
	797	+ (unsigned long long)parity_sector,
	798	+ bdevname(parity_rdev->bdev, b));
	799	+ if (!sync_page_io(parity_rdev, parity_sector, block_size,
	800	+ page1, REQ_OP_WRITE, 0, false)) {
	801	+ pr_debug("%s:%*s parity write error!\n", __func__,
	802	+ indent, "");
	803	+ md_error(mddev, parity_rdev);
	804	+ ret = -EIO;
	805	+ goto out;
	806	+ }
	807	+ }
	808	+out:
	809	+ if (page1)
	810	+ __free_page(page1);
	811	+ if (page2)
	812	+ __free_page(page2);
	813	+ return ret;
	814	+}
	815	+
	816	+static int ppl_recover(struct ppl_log log, struct ppl_header pplhdr)
	817	+{
	818	+ struct ppl_conf *ppl_conf = log->ppl_conf;
	819	+ struct md_rdev *rdev = log->rdev;
	820	+ struct mddev *mddev = rdev->mddev;
	821	+ sector_t ppl_sector = rdev->ppl.sector + (PPL_HEADER_SIZE >> 9);
	822	+ struct page *page;
	823	+ int i;
	824	+ int ret = 0;
	825	+
	826	+ page = alloc_page(GFP_KERNEL);
	827	+ if (!page)
	828	+ return -ENOMEM;
	829	+
	830	+ /* iterate through all PPL entries saved */
	831	+ for (i = 0; i < le32_to_cpu(pplhdr->entries_count); i++) {
	832	+ struct ppl_header_entry *e = &pplhdr->entries[i];
	833	+ u32 pp_size = le32_to_cpu(e->pp_size);
	834	+ sector_t sector = ppl_sector;
	835	+ int ppl_entry_sectors = pp_size >> 9;
	836	+ u32 crc, crc_stored;
	837	+
	838	+ pr_debug("%s: disk: %d entry: %d ppl_sector: %llu pp_size: %u\n",
	839	+ __func__, rdev->raid_disk, i,
	840	+ (unsigned long long)ppl_sector, pp_size);
	841	+
	842	+ crc = ~0;
	843	+ crc_stored = le32_to_cpu(e->checksum);
	844	+
	845	+ /* read parial parity for this entry and calculate its checksum */
	846	+ while (pp_size) {
	847	+ int s = pp_size > PAGE_SIZE ? PAGE_SIZE : pp_size;
	848	+
	849	+ if (!sync_page_io(rdev, sector - rdev->data_offset,
	850	+ s, page, REQ_OP_READ, 0, false)) {
	851	+ md_error(mddev, rdev);
	852	+ ret = -EIO;
	853	+ goto out;
	854	+ }
	855	+
	856	+ crc = crc32c_le(crc, page_address(page), s);
	857	+
	858	+ pp_size -= s;
	859	+ sector += s >> 9;
	860	+ }
	861	+
	862	+ crc = ~crc;
	863	+
	864	+ if (crc != crc_stored) {
	865	+ /*
	866	+ * Don't recover this entry if the checksum does not
	867	+ * match, but keep going and try to recover other
	868	+ * entries.
	869	+ */
	870	+ pr_debug("%s: ppl entry crc does not match: stored: 0x%x calculated: 0x%x\n",
	871	+ __func__, crc_stored, crc);
	872	+ ppl_conf->mismatch_count++;
	873	+ } else {
	874	+ ret = ppl_recover_entry(log, e, ppl_sector);
	875	+ if (ret)
	876	+ goto out;
	877	+ ppl_conf->recovered_entries++;
	878	+ }
	879	+
	880	+ ppl_sector += ppl_entry_sectors;
	881	+ }
	882	+
	883	+ /* flush the disk cache after recovery if necessary */
	884	+ ret = blkdev_issue_flush(rdev->bdev, GFP_KERNEL, NULL);
	885	+out:
	886	+ __free_page(page);
	887	+ return ret;
	888	+}
	889	+
	890	+static int ppl_write_empty_header(struct ppl_log *log)
	891	+{
	892	+ struct page *page;
	893	+ struct ppl_header *pplhdr;
	894	+ struct md_rdev *rdev = log->rdev;
	895	+ int ret = 0;
	896	+
	897	+ pr_debug("%s: disk: %d ppl_sector: %llu\n", __func__,
	898	+ rdev->raid_disk, (unsigned long long)rdev->ppl.sector);
	899	+
	900	+ page = alloc_page(GFP_NOIO \| __GFP_ZERO);
	901	+ if (!page)
	902	+ return -ENOMEM;
	903	+
	904	+ pplhdr = page_address(page);
	905	+ memset(pplhdr->reserved, 0xff, PPL_HDR_RESERVED);
	906	+ pplhdr->signature = cpu_to_le32(log->ppl_conf->signature);
	907	+ pplhdr->checksum = cpu_to_le32(~crc32c_le(~0, pplhdr, PAGE_SIZE));
	908	+
	909	+ if (!sync_page_io(rdev, rdev->ppl.sector - rdev->data_offset,
	910	+ PPL_HEADER_SIZE, page, REQ_OP_WRITE \| REQ_FUA, 0,
	911	+ false)) {
	912	+ md_error(rdev->mddev, rdev);
	913	+ ret = -EIO;
	914	+ }
	915	+
	916	+ __free_page(page);
	917	+ return ret;
	918	+}
	919	+
	920	+static int ppl_load_distributed(struct ppl_log *log)
	921	+{
	922	+ struct ppl_conf *ppl_conf = log->ppl_conf;
	923	+ struct md_rdev *rdev = log->rdev;
	924	+ struct mddev *mddev = rdev->mddev;
	925	+ struct page *page;
	926	+ struct ppl_header *pplhdr;
	927	+ u32 crc, crc_stored;
	928	+ u32 signature;
	929	+ int ret = 0;
	930	+
	931	+ pr_debug("%s: disk: %d\n", __func__, rdev->raid_disk);
	932	+
	933	+ /* read PPL header */
	934	+ page = alloc_page(GFP_KERNEL);
	935	+ if (!page)
	936	+ return -ENOMEM;
	937	+
	938	+ if (!sync_page_io(rdev, rdev->ppl.sector - rdev->data_offset,
	939	+ PAGE_SIZE, page, REQ_OP_READ, 0, false)) {
	940	+ md_error(mddev, rdev);
	941	+ ret = -EIO;
	942	+ goto out;
	943	+ }
	944	+ pplhdr = page_address(page);
	945	+
	946	+ /* check header validity */
	947	+ crc_stored = le32_to_cpu(pplhdr->checksum);
	948	+ pplhdr->checksum = 0;
	949	+ crc = ~crc32c_le(~0, pplhdr, PAGE_SIZE);
	950	+
	951	+ if (crc_stored != crc) {
	952	+ pr_debug("%s: ppl header crc does not match: stored: 0x%x calculated: 0x%x\n",
	953	+ __func__, crc_stored, crc);
	954	+ ppl_conf->mismatch_count++;
	955	+ goto out;
	956	+ }
	957	+
	958	+ signature = le32_to_cpu(pplhdr->signature);
	959	+
	960	+ if (mddev->external) {
	961	+ /*
	962	+ * For external metadata the header signature is set and
	963	+ * validated in userspace.
	964	+ */
	965	+ ppl_conf->signature = signature;
	966	+ } else if (ppl_conf->signature != signature) {
	967	+ pr_debug("%s: ppl header signature does not match: stored: 0x%x configured: 0x%x\n",
	968	+ __func__, signature, ppl_conf->signature);
	969	+ ppl_conf->mismatch_count++;
	970	+ goto out;
	971	+ }
	972	+
	973	+ /* attempt to recover from log if we are starting a dirty array */
	974	+ if (!mddev->pers && mddev->recovery_cp != MaxSector)
	975	+ ret = ppl_recover(log, pplhdr);
	976	+out:
	977	+ /* write empty header if we are starting the array */
	978	+ if (!ret && !mddev->pers)
	979	+ ret = ppl_write_empty_header(log);
	980	+
	981	+ __free_page(page);
	982	+
	983	+ pr_debug("%s: return: %d mismatch_count: %d recovered_entries: %d\n",
	984	+ __func__, ret, ppl_conf->mismatch_count,
	985	+ ppl_conf->recovered_entries);
	986	+ return ret;
	987	+}
	988	+
	989	+static int ppl_load(struct ppl_conf *ppl_conf)
	990	+{
	991	+ int ret = 0;
	992	+ u32 signature = 0;
	993	+ bool signature_set = false;
	994	+ int i;
	995	+
	996	+ for (i = 0; i < ppl_conf->count; i++) {
	997	+ struct ppl_log *log = &ppl_conf->child_logs[i];
	998	+
	999	+ /* skip missing drive */
	1000	+ if (!log->rdev)
	1001	+ continue;
	1002	+
	1003	+ ret = ppl_load_distributed(log);
	1004	+ if (ret)
	1005	+ break;
	1006	+
	1007	+ /*
	1008	+ * For external metadata we can't check if the signature is
	1009	+ * correct on a single drive, but we can check if it is the same
	1010	+ * on all drives.
	1011	+ */
	1012	+ if (ppl_conf->mddev->external) {
	1013	+ if (!signature_set) {
	1014	+ signature = ppl_conf->signature;
	1015	+ signature_set = true;
	1016	+ } else if (signature != ppl_conf->signature) {
	1017	+ pr_warn("md/raid:%s: PPL header signature does not match on all member drives\n",
	1018	+ mdname(ppl_conf->mddev));
	1019	+ ret = -EINVAL;
	1020	+ break;
	1021	+ }
	1022	+ }
	1023	+ }
	1024	+
	1025	+ pr_debug("%s: return: %d mismatch_count: %d recovered_entries: %d\n",
	1026	+ __func__, ret, ppl_conf->mismatch_count,
	1027	+ ppl_conf->recovered_entries);
	1028	+ return ret;
	1029	+}
	1030	+
	1031	+static void __ppl_exit_log(struct ppl_conf *ppl_conf)
	1032	+{
	1033	+ clear_bit(MD_HAS_PPL, &ppl_conf->mddev->flags);
	1034	+
	1035	+ kfree(ppl_conf->child_logs);
	1036	+
	1037	+ if (ppl_conf->bs)
	1038	+ bioset_free(ppl_conf->bs);
	1039	+ mempool_destroy(ppl_conf->io_pool);
	1040	+ kmem_cache_destroy(ppl_conf->io_kc);
	1041	+
	1042	+ kfree(ppl_conf);
	1043	+}
	1044	+
	1045	+void ppl_exit_log(struct r5conf *conf)
	1046	+{
	1047	+ struct ppl_conf *ppl_conf = conf->log_private;
	1048	+
	1049	+ if (ppl_conf) {
	1050	+ __ppl_exit_log(ppl_conf);
	1051	+ conf->log_private = NULL;
	1052	+ }
	1053	+}
	1054	+
	1055	+static int ppl_validate_rdev(struct md_rdev *rdev)
	1056	+{
	1057	+ char b[BDEVNAME_SIZE];
	1058	+ int ppl_data_sectors;
	1059	+ int ppl_size_new;
	1060	+
	1061	+ /*
	1062	+ * The configured PPL size must be enough to store
	1063	+ * the header and (at the very least) partial parity
	1064	+ * for one stripe. Round it down to ensure the data
	1065	+ * space is cleanly divisible by stripe size.
	1066	+ */
	1067	+ ppl_data_sectors = rdev->ppl.size - (PPL_HEADER_SIZE >> 9);
	1068	+
	1069	+ if (ppl_data_sectors > 0)
	1070	+ ppl_data_sectors = rounddown(ppl_data_sectors, STRIPE_SECTORS);
	1071	+
	1072	+ if (ppl_data_sectors <= 0) {
	1073	+ pr_warn("md/raid:%s: PPL space too small on %s\n",
	1074	+ mdname(rdev->mddev), bdevname(rdev->bdev, b));
	1075	+ return -ENOSPC;
	1076	+ }
	1077	+
	1078	+ ppl_size_new = ppl_data_sectors + (PPL_HEADER_SIZE >> 9);
	1079	+
	1080	+ if ((rdev->ppl.sector < rdev->data_offset &&
	1081	+ rdev->ppl.sector + ppl_size_new > rdev->data_offset) \|\|
	1082	+ (rdev->ppl.sector >= rdev->data_offset &&
	1083	+ rdev->data_offset + rdev->sectors > rdev->ppl.sector)) {
	1084	+ pr_warn("md/raid:%s: PPL space overlaps with data on %s\n",
	1085	+ mdname(rdev->mddev), bdevname(rdev->bdev, b));
	1086	+ return -EINVAL;
	1087	+ }
	1088	+
	1089	+ if (!rdev->mddev->external &&
	1090	+ ((rdev->ppl.offset > 0 && rdev->ppl.offset < (rdev->sb_size >> 9)) \|\|
	1091	+ (rdev->ppl.offset <= 0 && rdev->ppl.offset + ppl_size_new > 0))) {
	1092	+ pr_warn("md/raid:%s: PPL space overlaps with superblock on %s\n",
	1093	+ mdname(rdev->mddev), bdevname(rdev->bdev, b));
	1094	+ return -EINVAL;
	1095	+ }
	1096	+
	1097	+ rdev->ppl.size = ppl_size_new;
	1098	+
	1099	+ return 0;
	1100	+}
	1101	+
	1102	+int ppl_init_log(struct r5conf *conf)
	1103	+{
	1104	+ struct ppl_conf *ppl_conf;
	1105	+ struct mddev *mddev = conf->mddev;
	1106	+ int ret = 0;
	1107	+ int i;
	1108	+ bool need_cache_flush = false;
	1109	+
	1110	+ pr_debug("md/raid:%s: enabling distributed Partial Parity Log\n",
	1111	+ mdname(conf->mddev));
	1112	+
	1113	+ if (PAGE_SIZE != 4096)
	1114	+ return -EINVAL;
	1115	+
	1116	+ if (mddev->level != 5) {
	1117	+ pr_warn("md/raid:%s PPL is not compatible with raid level %d\n",
	1118	+ mdname(mddev), mddev->level);
	1119	+ return -EINVAL;
	1120	+ }
	1121	+
	1122	+ if (mddev->bitmap_info.file \|\| mddev->bitmap_info.offset) {
	1123	+ pr_warn("md/raid:%s PPL is not compatible with bitmap\n",
	1124	+ mdname(mddev));
	1125	+ return -EINVAL;
	1126	+ }
	1127	+
	1128	+ if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
	1129	+ pr_warn("md/raid:%s PPL is not compatible with journal\n",
	1130	+ mdname(mddev));
	1131	+ return -EINVAL;
	1132	+ }
	1133	+
	1134	+ ppl_conf = kzalloc(sizeof(struct ppl_conf), GFP_KERNEL);
	1135	+ if (!ppl_conf)
	1136	+ return -ENOMEM;
	1137	+
	1138	+ ppl_conf->mddev = mddev;
	1139	+
	1140	+ ppl_conf->io_kc = KMEM_CACHE(ppl_io_unit, 0);
	1141	+ if (!ppl_conf->io_kc) {
	1142	+ ret = -ENOMEM;
	1143	+ goto err;
	1144	+ }
	1145	+
	1146	+ ppl_conf->io_pool = mempool_create(conf->raid_disks, ppl_io_pool_alloc,
	1147	+ ppl_io_pool_free, ppl_conf->io_kc);
	1148	+ if (!ppl_conf->io_pool) {
	1149	+ ret = -ENOMEM;
	1150	+ goto err;
	1151	+ }
	1152	+
	1153	+ ppl_conf->bs = bioset_create(conf->raid_disks, 0);
	1154	+ if (!ppl_conf->bs) {
	1155	+ ret = -ENOMEM;
	1156	+ goto err;
	1157	+ }
	1158	+
	1159	+ ppl_conf->count = conf->raid_disks;
	1160	+ ppl_conf->child_logs = kcalloc(ppl_conf->count, sizeof(struct ppl_log),
	1161	+ GFP_KERNEL);
	1162	+ if (!ppl_conf->child_logs) {
	1163	+ ret = -ENOMEM;
	1164	+ goto err;
	1165	+ }
	1166	+
	1167	+ atomic64_set(&ppl_conf->seq, 0);
	1168	+ INIT_LIST_HEAD(&ppl_conf->no_mem_stripes);
	1169	+ spin_lock_init(&ppl_conf->no_mem_stripes_lock);
	1170	+
	1171	+ if (!mddev->external) {
	1172	+ ppl_conf->signature = ~crc32c_le(~0, mddev->uuid, sizeof(mddev->uuid));
	1173	+ ppl_conf->block_size = 512;
	1174	+ } else {
	1175	+ ppl_conf->block_size = queue_logical_block_size(mddev->queue);
	1176	+ }
	1177	+
	1178	+ for (i = 0; i < ppl_conf->count; i++) {
	1179	+ struct ppl_log *log = &ppl_conf->child_logs[i];
	1180	+ struct md_rdev *rdev = conf->disks[i].rdev;
	1181	+
	1182	+ mutex_init(&log->io_mutex);
	1183	+ spin_lock_init(&log->io_list_lock);
	1184	+ INIT_LIST_HEAD(&log->io_list);
	1185	+
	1186	+ log->ppl_conf = ppl_conf;
	1187	+ log->rdev = rdev;
	1188	+
	1189	+ if (rdev) {
	1190	+ struct request_queue *q;
	1191	+
	1192	+ ret = ppl_validate_rdev(rdev);
	1193	+ if (ret)
	1194	+ goto err;
	1195	+
	1196	+ q = bdev_get_queue(rdev->bdev);
	1197	+ if (test_bit(QUEUE_FLAG_WC, &q->queue_flags))
	1198	+ need_cache_flush = true;
	1199	+ }
	1200	+ }
	1201	+
	1202	+ if (need_cache_flush)
	1203	+ pr_warn("md/raid:%s: Volatile write-back cache should be disabled on all member drives when using PPL!\n",
	1204	+ mdname(mddev));
	1205	+
	1206	+ /* load and possibly recover the logs from the member disks */
	1207	+ ret = ppl_load(ppl_conf);
	1208	+
	1209	+ if (ret) {
	1210	+ goto err;
	1211	+ } else if (!mddev->pers &&
	1212	+ mddev->recovery_cp == 0 && !mddev->degraded &&
	1213	+ ppl_conf->recovered_entries > 0 &&
	1214	+ ppl_conf->mismatch_count == 0) {
	1215	+ /*
	1216	+ * If we are starting a dirty array and the recovery succeeds
	1217	+ * without any issues, set the array as clean.
	1218	+ */
	1219	+ mddev->recovery_cp = MaxSector;
	1220	+ set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
	1221	+ } else if (mddev->pers && ppl_conf->mismatch_count > 0) {
	1222	+ /* no mismatch allowed when enabling PPL for a running array */
	1223	+ ret = -EINVAL;
	1224	+ goto err;
	1225	+ }
	1226	+
	1227	+ conf->log_private = ppl_conf;
	1228	+ set_bit(MD_HAS_PPL, &ppl_conf->mddev->flags);
	1229	+
	1230	+ return 0;
	1231	+err:
	1232	+ __ppl_exit_log(ppl_conf);
	1233	+ return ret;
	1234	+}
	1235	+
	1236	+int ppl_modify_log(struct r5conf conf, struct md_rdev rdev, bool add)
	1237	+{
	1238	+ struct ppl_conf *ppl_conf = conf->log_private;
	1239	+ struct ppl_log *log;
	1240	+ int ret = 0;
	1241	+ char b[BDEVNAME_SIZE];
	1242	+
	1243	+ if (!rdev)
	1244	+ return -EINVAL;
	1245	+
	1246	+ pr_debug("%s: disk: %d operation: %s dev: %s\n",
	1247	+ __func__, rdev->raid_disk, add ? "add" : "remove",
	1248	+ bdevname(rdev->bdev, b));
	1249	+
	1250	+ if (rdev->raid_disk < 0)
	1251	+ return 0;
	1252	+
	1253	+ if (rdev->raid_disk >= ppl_conf->count)
	1254	+ return -ENODEV;
	1255	+
	1256	+ log = &ppl_conf->child_logs[rdev->raid_disk];
	1257	+
	1258	+ mutex_lock(&log->io_mutex);
	1259	+ if (add) {
	1260	+ ret = ppl_validate_rdev(rdev);
	1261	+ if (!ret) {
	1262	+ log->rdev = rdev;
	1263	+ ret = ppl_write_empty_header(log);
	1264	+ }
	1265	+ } else {
	1266	+ log->rdev = NULL;
	1267	+ }
	1268	+ mutex_unlock(&log->io_mutex);
	1269	+
	1270	+ return ret;
	1271	+}

drivers/md/raid5.c+409−234

@@ -58,11 +58,13 @@ · 57 unmodified lines
58	58	#include <linux/sched/signal.h>
59	59
60	60	#include <trace/events/block.h>
	61	+#include <linux/list_sort.h>
61	62
62	63	#include "md.h"
63	64	#include "raid5.h"
64	65	#include "raid0.h"
65	66	#include "bitmap.h"
	67	+#include "raid5-log.h"
66	68
67	69	#define UNSUPPORTED_MDDEV_FLAGS (1L << MD_FAILFAST_SUPPORTED)
68	70
@@ -156,17 +158,6 @@ static int raid6_idx_to_slot(int idx, struct stripe_head *sh, · 87 unmodified lines
156	158	return slot;
157	159	}
158	160
159		−static void return_io(struct bio_list *return_bi)
160		−{
161		− struct bio *bi;
162		− while ((bi = bio_list_pop(return_bi)) != NULL) {
163		− bi->bi_iter.bi_size = 0;
164		− trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
165		− bi, 0);
166		− bio_endio(bi);
167		− }
168		−}
169		−
170	161	static void print_raid5_conf (struct r5conf *conf);
171	162
172	163	static int stripe_operations_active(struct stripe_head *sh)
@@ -176,6 +167,13 @@ static int stripe_operations_active(struct stripe_head *sh) · 3 unmodified lines
176	167	test_bit(STRIPE_COMPUTE_RUN, &sh->state);
177	168	}
178	169
	170	+static bool stripe_is_lowprio(struct stripe_head *sh)
	171	+{
	172	+ return (test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) \|\|
	173	+ test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) &&
	174	+ !test_bit(STRIPE_R5C_CACHING, &sh->state);
	175	+}
	176	+
179	177	static void raid5_wakeup_stripe_thread(struct stripe_head *sh)
180	178	{
181	179	struct r5conf *conf = sh->raid_conf;
@@ -191,7 +189,10 @@ static void raid5_wakeup_stripe_thread(struct stripe_head *sh) · 9 unmodified lines
191	189	if (list_empty(&sh->lru)) {
192	190	struct r5worker_group *group;
193	191	group = conf->worker_groups + cpu_to_group(cpu);
194		− list_add_tail(&sh->lru, &group->handle_list);
	192	+ if (stripe_is_lowprio(sh))
	193	+ list_add_tail(&sh->lru, &group->loprio_list);
	194	+ else
	195	+ list_add_tail(&sh->lru, &group->handle_list);
195	196	group->stripes_cnt++;
196	197	sh->group = group;
197	198	}
@@ -254,7 +255,12 @@ static void do_release_stripe(struct r5conf conf, struct stripe_head sh, · 56 unmodified lines
254	255	clear_bit(STRIPE_DELAYED, &sh->state);
255	256	clear_bit(STRIPE_BIT_DELAY, &sh->state);
256	257	if (conf->worker_cnt_per_group == 0) {
257		− list_add_tail(&sh->lru, &conf->handle_list);
	258	+ if (stripe_is_lowprio(sh))
	259	+ list_add_tail(&sh->lru,
	260	+ &conf->loprio_list);
	261	+ else
	262	+ list_add_tail(&sh->lru,
	263	+ &conf->handle_list);
258	264	} else {
259	265	raid5_wakeup_stripe_thread(sh);
260	266	return;
@@ -481,6 +487,7 @@ static int grow_buffers(struct stripe_head *sh, gfp_t gfp) · 220 unmodified lines
481	487	sh->dev[i].page = page;
482	488	sh->dev[i].orig_page = page;
483	489	}
	490	+
484	491	return 0;
485	492	}
486	493
@@ -729,7 +736,7 @@ static bool stripe_can_batch(struct stripe_head *sh) · 242 unmodified lines
729	736	{
730	737	struct r5conf *conf = sh->raid_conf;
731	738
732		− if (conf->log)
	739	+ if (conf->log \|\| raid5_has_ppl(conf))
733	740	return false;
734	741	return test_bit(STRIPE_BATCH_READY, &sh->state) &&
735	742	!test_bit(STRIPE_BITMAP_PENDING, &sh->state) &&
@@ -863,41 +870,107 @@ static int use_new_offset(struct r5conf conf, struct stripe_head sh) · 127 unmodified lines
863	870	return 1;
864	871	}
865	872
866		−static void flush_deferred_bios(struct r5conf *conf)
	873	+static void dispatch_bio_list(struct bio_list *tmp)
867	874	{
868		− struct bio_list tmp;
869	875	struct bio *bio;
870	876
871		− if (!conf->batch_bio_dispatch \|\| !conf->group_cnt)
	877	+ while ((bio = bio_list_pop(tmp)))
	878	+ generic_make_request(bio);
	879	+}
	880	+
	881	+static int cmp_stripe(void priv, struct list_head a, struct list_head *b)
	882	+{
	883	+ const struct r5pending_data *da = list_entry(a,
	884	+ struct r5pending_data, sibling);
	885	+ const struct r5pending_data *db = list_entry(b,
	886	+ struct r5pending_data, sibling);
	887	+ if (da->sector > db->sector)
	888	+ return 1;
	889	+ if (da->sector < db->sector)
	890	+ return -1;
	891	+ return 0;
	892	+}
	893	+
	894	+static void dispatch_defer_bios(struct r5conf *conf, int target,
	895	+ struct bio_list *list)
	896	+{
	897	+ struct r5pending_data *data;
	898	+ struct list_head first, next = NULL;
	899	+ int cnt = 0;
	900	+
	901	+ if (conf->pending_data_cnt == 0)
	902	+ return;
	903	+
	904	+ list_sort(NULL, &conf->pending_list, cmp_stripe);
	905	+
	906	+ first = conf->pending_list.next;
	907	+
	908	+ /* temporarily move the head */
	909	+ if (conf->next_pending_data)
	910	+ list_move_tail(&conf->pending_list,
	911	+ &conf->next_pending_data->sibling);
	912	+
	913	+ while (!list_empty(&conf->pending_list)) {
	914	+ data = list_first_entry(&conf->pending_list,
	915	+ struct r5pending_data, sibling);
	916	+ if (&data->sibling == first)
	917	+ first = data->sibling.next;
	918	+ next = data->sibling.next;
	919	+
	920	+ bio_list_merge(list, &data->bios);
	921	+ list_move(&data->sibling, &conf->free_list);
	922	+ cnt++;
	923	+ if (cnt >= target)
	924	+ break;
	925	+ }
	926	+ conf->pending_data_cnt -= cnt;
	927	+ BUG_ON(conf->pending_data_cnt < 0 \|\| cnt < target);
	928	+
	929	+ if (next != &conf->pending_list)
	930	+ conf->next_pending_data = list_entry(next,
	931	+ struct r5pending_data, sibling);
	932	+ else
	933	+ conf->next_pending_data = NULL;
	934	+ /* list isn't empty */
	935	+ if (first != &conf->pending_list)
	936	+ list_move_tail(&conf->pending_list, first);
	937	+}
	938	+
	939	+static void flush_deferred_bios(struct r5conf *conf)
	940	+{
	941	+ struct bio_list tmp = BIO_EMPTY_LIST;
	942	+
	943	+ if (conf->pending_data_cnt == 0)
872	944	return;
873	945
874		− bio_list_init(&tmp);
875	946	spin_lock(&conf->pending_bios_lock);
876		− bio_list_merge(&tmp, &conf->pending_bios);
877		− bio_list_init(&conf->pending_bios);
	947	+ dispatch_defer_bios(conf, conf->pending_data_cnt, &tmp);
	948	+ BUG_ON(conf->pending_data_cnt != 0);
878	949	spin_unlock(&conf->pending_bios_lock);
879	950
880		− while ((bio = bio_list_pop(&tmp)))
881		− generic_make_request(bio);
	951	+ dispatch_bio_list(&tmp);
882	952	}
883	953
884		−static void defer_bio_issue(struct r5conf conf, struct bio bio)
	954	+static void defer_issue_bios(struct r5conf *conf, sector_t sector,
	955	+ struct bio_list *bios)
885	956	{
886		− /*
887		− * change group_cnt will drain all bios, so this is safe
888		− *
889		− * A read generally means a read-modify-write, which usually means a
890		− * randwrite, so we don't delay it
891		− */
892		− if (!conf->batch_bio_dispatch \|\| !conf->group_cnt \|\|
893		− bio_op(bio) == REQ_OP_READ) {
894		− generic_make_request(bio);
895		− return;
896		− }
	957	+ struct bio_list tmp = BIO_EMPTY_LIST;
	958	+ struct r5pending_data *ent;
	959	+
897	960	spin_lock(&conf->pending_bios_lock);
898		− bio_list_add(&conf->pending_bios, bio);
	961	+ ent = list_first_entry(&conf->free_list, struct r5pending_data,
	962	+ sibling);
	963	+ list_move_tail(&ent->sibling, &conf->pending_list);
	964	+ ent->sector = sector;
	965	+ bio_list_init(&ent->bios);
	966	+ bio_list_merge(&ent->bios, bios);
	967	+ conf->pending_data_cnt++;
	968	+ if (conf->pending_data_cnt >= PENDING_IO_MAX)
	969	+ dispatch_defer_bios(conf, PENDING_IO_ONE_FLUSH, &tmp);
	970	+
899	971	spin_unlock(&conf->pending_bios_lock);
900		− md_wakeup_thread(conf->mddev->thread);
	972	+
	973	+ dispatch_bio_list(&tmp);
901	974	}
902	975
903	976	static void
@@ -910,21 +983,15 @@ static void ops_run_io(struct stripe_head sh, struct stripe_head_state s) · 6 unmodified lines
910	983	struct r5conf *conf = sh->raid_conf;
911	984	int i, disks = sh->disks;
912	985	struct stripe_head *head_sh = sh;
	986	+ struct bio_list pending_bios = BIO_EMPTY_LIST;
	987	+ bool should_defer;
913	988
914	989	might_sleep();
915	990
916		− if (!test_bit(STRIPE_R5C_CACHING, &sh->state)) {
917		− /* writing out phase */
918		− if (s->waiting_extra_page)
919		− return;
920		− if (r5l_write_stripe(conf->log, sh) == 0)
921		− return;
922		− } else { /* caching phase */
923		− if (test_bit(STRIPE_LOG_TRAPPED, &sh->state)) {
924		− r5c_cache_data(conf->log, sh, s);
925		− return;
926		− }
927		− }
	991	+ if (log_stripe(sh, s) == 0)
	992	+ return;
	993	+
	994	+ should_defer = conf->batch_bio_dispatch && conf->group_cnt;
928	995
929	996	for (i = disks; i--; ) {
930	997	int op, op_flags = 0;
@@ -1080,7 +1147,10 @@ again: · 149 unmodified lines
1080	1147	trace_block_bio_remap(bdev_get_queue(bi->bi_bdev),
1081	1148	bi, disk_devt(conf->mddev->gendisk),
1082	1149	sh->dev[i].sector);
1083		− defer_bio_issue(conf, bi);
	1150	+ if (should_defer && op_is_write(op))
	1151	+ bio_list_add(&pending_bios, bi);
	1152	+ else
	1153	+ generic_make_request(bi);
1084	1154	}
1085	1155	if (rrdev) {
1086	1156	if (s->syncing \|\| s->expanding \|\| s->expanded
@@ -1125,7 +1195,10 @@ again: · 38 unmodified lines
1125	1195	trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev),
1126	1196	rbi, disk_devt(conf->mddev->gendisk),
1127	1197	sh->dev[i].sector);
1128		− defer_bio_issue(conf, rbi);
	1198	+ if (should_defer && op_is_write(op))
	1199	+ bio_list_add(&pending_bios, rbi);
	1200	+ else
	1201	+ generic_make_request(rbi);
1129	1202	}
1130	1203	if (!rdev && !rrdev) {
1131	1204	if (op_is_write(op))
@@ -1143,6 +1216,9 @@ again: · 11 unmodified lines
1143	1216	if (sh != head_sh)
1144	1217	goto again;
1145	1218	}
	1219	+
	1220	+ if (should_defer && !bio_list_empty(&pending_bios))
	1221	+ defer_issue_bios(conf, head_sh->sector, &pending_bios);
1146	1222	}
1147	1223
1148	1224	static struct dma_async_tx_descriptor *
@@ -1212,7 +1288,6 @@ async_copy_data(int frombio, struct bio bio, struct page *page, · 63 unmodified lines
1212	1288	static void ops_complete_biofill(void *stripe_head_ref)
1213	1289	{
1214	1290	struct stripe_head *sh = stripe_head_ref;
1215		− struct bio_list return_bi = BIO_EMPTY_LIST;
1216	1291	int i;
1217	1292
1218	1293	pr_debug("%s: stripe %llu\n", __func__,
@@ -1236,16 +1311,13 @@ static void ops_complete_biofill(void *stripe_head_ref) · 17 unmodified lines
1236	1311	while (rbi && rbi->bi_iter.bi_sector <
1237	1312	dev->sector + STRIPE_SECTORS) {
1238	1313	rbi2 = r5_next_bio(rbi, dev->sector);
1239		− if (!raid5_dec_bi_active_stripes(rbi))
1240		− bio_list_add(&return_bi, rbi);
	1314	+ bio_endio(rbi);
1241	1315	rbi = rbi2;
1242	1316	}
1243	1317	}
1244	1318	}
1245	1319	clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
1246	1320
1247		− return_io(&return_bi);
1248		−
1249	1321	set_bit(STRIPE_HANDLE, &sh->state);
1250	1322	raid5_release_stripe(sh);
1251	1323	}
@@ -2014,6 +2086,9 @@ static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) · 762 unmodified lines
2014	2086	tx = ops_run_prexor6(sh, percpu, tx);
2015	2087	}
2016	2088
	2089	+ if (test_bit(STRIPE_OP_PARTIAL_PARITY, &ops_request))
	2090	+ tx = ops_run_partial_parity(sh, percpu, tx);
	2091	+
2017	2092	if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
2018	2093	tx = ops_run_biodrain(sh, tx);
2019	2094	overlap_clear++;
@@ -2046,8 +2121,15 @@ static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) · 26 unmodified lines
2046	2121	put_cpu();
2047	2122	}
2048	2123
	2124	+static void free_stripe(struct kmem_cache sc, struct stripe_head sh)
	2125	+{
	2126	+ if (sh->ppl_page)
	2127	+ __free_page(sh->ppl_page);
	2128	+ kmem_cache_free(sc, sh);
	2129	+}
	2130	+
2049	2131	static struct stripe_head alloc_stripe(struct kmem_cache sc, gfp_t gfp,
2050		− int disks)
	2132	+ int disks, struct r5conf *conf)
2051	2133	{
2052	2134	struct stripe_head *sh;
2053	2135	int i;
@@ -2061,6 +2143,7 @@ static struct stripe_head alloc_stripe(struct kmem_cache sc, gfp_t gfp, · 7 unmodified lines
2061	2143	INIT_LIST_HEAD(&sh->r5c);
2062	2144	INIT_LIST_HEAD(&sh->log_list);
2063	2145	atomic_set(&sh->count, 1);
	2146	+ sh->raid_conf = conf;
2064	2147	sh->log_start = MaxSector;
2065	2148	for (i = 0; i < disks; i++) {
2066	2149	struct r5dev *dev = &sh->dev[i];
@@ -2068,6 +2151,14 @@ static struct stripe_head alloc_stripe(struct kmem_cache sc, gfp_t gfp, · 1 unmodified line
2068	2151	bio_init(&dev->req, &dev->vec, 1);
2069	2152	bio_init(&dev->rreq, &dev->rvec, 1);
2070	2153	}
	2154	+
	2155	+ if (raid5_has_ppl(conf)) {
	2156	+ sh->ppl_page = alloc_page(gfp);
	2157	+ if (!sh->ppl_page) {
	2158	+ free_stripe(sc, sh);
	2159	+ sh = NULL;
	2160	+ }
	2161	+ }
2071	2162	}
2072	2163	return sh;
2073	2164	}
@@ -2075,15 +2166,13 @@ static int grow_one_stripe(struct r5conf *conf, gfp_t gfp) · 1 unmodified line
2075	2166	{
2076	2167	struct stripe_head *sh;
2077	2168
2078		− sh = alloc_stripe(conf->slab_cache, gfp, conf->pool_size);
	2169	+ sh = alloc_stripe(conf->slab_cache, gfp, conf->pool_size, conf);
2079	2170	if (!sh)
2080	2171	return 0;
2081	2172
2082		− sh->raid_conf = conf;
2083		−
2084	2173	if (grow_buffers(sh, gfp)) {
2085	2174	shrink_buffers(sh);
2086		− kmem_cache_free(conf->slab_cache, sh);
	2175	+ free_stripe(conf->slab_cache, sh);
2087	2176	return 0;
2088	2177	}
2089	2178	sh->hash_lock_index =
@@ -2210,7 +2299,7 @@ static int resize_stripes(struct r5conf *conf, int newsize) · 120 unmodified lines
2210	2299	* pages have been transferred over, and the old kmem_cache is
2211	2300	* freed when all stripes are done.
2212	2301	* 3/ reallocate conf->disks to be suitable bigger. If this fails,
2213		− * we simple return a failre status - no need to clean anything up.
	2302	+ * we simple return a failure status - no need to clean anything up.
2214	2303	* 4/ allocate new pages for the new slots in the new stripe_heads.
2215	2304	* If this fails, we don't bother trying the shrink the
2216	2305	* stripe_heads down again, we just leave them as they are.
@@ -2228,9 +2317,6 @@ static int resize_stripes(struct r5conf *conf, int newsize) · 11 unmodified lines
2228	2317	int i;
2229	2318	int hash, cnt;
2230	2319
2231		− if (newsize <= conf->pool_size)
2232		− return 0; /* never bother to shrink */
2233		−
2234	2320	err = md_allow_write(conf->mddev);
2235	2321	if (err)
2236	2322	return err;
@@ -2246,11 +2332,10 @@ static int resize_stripes(struct r5conf *conf, int newsize) · 9 unmodified lines
2246	2332	mutex_lock(&conf->cache_size_mutex);
2247	2333
2248	2334	for (i = conf->max_nr_stripes; i; i--) {
2249		− nsh = alloc_stripe(sc, GFP_KERNEL, newsize);
	2335	+ nsh = alloc_stripe(sc, GFP_KERNEL, newsize, conf);
2250	2336	if (!nsh)
2251	2337	break;
2252	2338
2253		− nsh->raid_conf = conf;
2254	2339	list_add(&nsh->lru, &newstripes);
2255	2340	}
2256	2341	if (i) {
@@ -2258,7 +2343,7 @@ static int resize_stripes(struct r5conf *conf, int newsize) · 1 unmodified line
2258	2343	while (!list_empty(&newstripes)) {
2259	2344	nsh = list_entry(newstripes.next, struct stripe_head, lru);
2260	2345	list_del(&nsh->lru);
2261		− kmem_cache_free(sc, nsh);
	2346	+ free_stripe(sc, nsh);
2262	2347	}
2263	2348	kmem_cache_destroy(sc);
2264	2349	mutex_unlock(&conf->cache_size_mutex);
@@ -2284,7 +2369,7 @@ static int resize_stripes(struct r5conf *conf, int newsize) · 19 unmodified lines
2284	2369	nsh->dev[i].orig_page = osh->dev[i].page;
2285	2370	}
2286	2371	nsh->hash_lock_index = hash;
2287		− kmem_cache_free(conf->slab_cache, osh);
	2372	+ free_stripe(conf->slab_cache, osh);
2288	2373	cnt++;
2289	2374	if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS +
2290	2375	!!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) {
@@ -2323,6 +2408,10 @@ static int resize_stripes(struct r5conf *conf, int newsize) · 32 unmodified lines
2323	2408	err = -ENOMEM;
2324	2409
2325	2410	mutex_unlock(&conf->cache_size_mutex);
	2411	+
	2412	+ conf->slab_cache = sc;
	2413	+ conf->active_name = 1-conf->active_name;
	2414	+
2326	2415	/* Step 4, return new stripes to service */
2327	2416	while(!list_empty(&newstripes)) {
2328	2417	nsh = list_entry(newstripes.next, struct stripe_head, lru);
@@ -2340,8 +2429,6 @@ static int resize_stripes(struct r5conf *conf, int newsize) · 11 unmodified lines
2340	2429	}
2341	2430	/* critical section pass, GFP_NOIO no longer needed */
2342	2431
2343		− conf->slab_cache = sc;
2344		− conf->active_name = 1-conf->active_name;
2345	2432	if (!err)
2346	2433	conf->pool_size = newsize;
2347	2434	return err;
@@ -2359,7 +2446,7 @@ static int drop_one_stripe(struct r5conf *conf) · 11 unmodified lines
2359	2446	return 0;
2360	2447	BUG_ON(atomic_read(&sh->count));
2361	2448	shrink_buffers(sh);
2362		− kmem_cache_free(conf->slab_cache, sh);
	2449	+ free_stripe(conf->slab_cache, sh);
2363	2450	atomic_dec(&conf->active_stripes);
2364	2451	conf->max_nr_stripes--;
2365	2452	return 1;
@@ -3082,6 +3169,12 @@ schedule_reconstruction(struct stripe_head sh, struct stripe_head_state s, · 716 unmodified lines
3082	3169	s->locked++;
3083	3170	}
3084	3171
	3172	+ if (raid5_has_ppl(sh->raid_conf) && sh->ppl_page &&
	3173	+ test_bit(STRIPE_OP_BIODRAIN, &s->ops_request) &&
	3174	+ !test_bit(STRIPE_FULL_WRITE, &sh->state) &&
	3175	+ test_bit(R5_Insync, &sh->dev[pd_idx].flags))
	3176	+ set_bit(STRIPE_OP_PARTIAL_PARITY, &s->ops_request);
	3177	+
3085	3178	pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
3086	3179	__func__, (unsigned long long)sh->sector,
3087	3180	s->locked, s->ops_request);
@@ -3103,14 +3196,6 @@ static int add_stripe_bio(struct stripe_head sh, struct bio bi, int dd_idx, · 15 unmodified lines
3103	3196	(unsigned long long)bi->bi_iter.bi_sector,
3104	3197	(unsigned long long)sh->sector);
3105	3198
3106		− /*
3107		− * If several bio share a stripe. The bio bi_phys_segments acts as a
3108		− * reference count to avoid race. The reference count should already be
3109		− * increased before this function is called (for example, in
3110		− * raid5_make_request()), so other bio sharing this stripe will not free the
3111		− * stripe. If a stripe is owned by one stripe, the stripe lock will
3112		− * protect it.
3113		− */
3114	3199	spin_lock_irq(&sh->stripe_lock);
3115	3200	/* Don't allow new IO added to stripes in batch list */
3116	3201	if (sh->batch_head)
@@ -3129,6 +3214,36 @@ static int add_stripe_bio(struct stripe_head sh, struct bio bi, int dd_idx, · 12 unmodified lines
3129	3214	if (bip && (bip)->bi_iter.bi_sector < bio_end_sector(bi))
3130	3215	goto overlap;
3131	3216
	3217	+ if (forwrite && raid5_has_ppl(conf)) {
	3218	+ /*
	3219	+ * With PPL only writes to consecutive data chunks within a
	3220	+ * stripe are allowed because for a single stripe_head we can
	3221	+ * only have one PPL entry at a time, which describes one data
	3222	+ * range. Not really an overlap, but wait_for_overlap can be
	3223	+ * used to handle this.
	3224	+ */
	3225	+ sector_t sector;
	3226	+ sector_t first = 0;
	3227	+ sector_t last = 0;
	3228	+ int count = 0;
	3229	+ int i;
	3230	+
	3231	+ for (i = 0; i < sh->disks; i++) {
	3232	+ if (i != sh->pd_idx &&
	3233	+ (i == dd_idx \|\| sh->dev[i].towrite)) {
	3234	+ sector = sh->dev[i].sector;
	3235	+ if (count == 0 \|\| sector < first)
	3236	+ first = sector;
	3237	+ if (sector > last)
	3238	+ last = sector;
	3239	+ count++;
	3240	+ }
	3241	+ }
	3242	+
	3243	+ if (first + conf->chunk_sectors * (count - 1) != last)
	3244	+ goto overlap;
	3245	+ }
	3246	+
3132	3247	if (!forwrite \|\| previous)
3133	3248	clear_bit(STRIPE_BATCH_READY, &sh->state);
3134	3249
@@ -3136,7 +3251,8 @@ static int add_stripe_bio(struct stripe_head sh, struct bio bi, int dd_idx, · 1 unmodified line
3136	3251	if (*bip)
3137	3252	bi->bi_next = *bip;
3138	3253	*bip = bi;
3139		− raid5_inc_bi_active_stripes(bi);
	3254	+ bio_inc_remaining(bi);
	3255	+ md_write_inc(conf->mddev, bi);
3140	3256
3141	3257	if (forwrite) {
3142	3258	/* check if page is covered */
@@ -3213,8 +3329,7 @@ static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, · 70 unmodified lines
3213	3329
3214	3330	static void
3215	3331	handle_failed_stripe(struct r5conf conf, struct stripe_head sh,
3216		− struct stripe_head_state *s, int disks,
3217		− struct bio_list *return_bi)
	3332	+ struct stripe_head_state *s, int disks)
3218	3333	{
3219	3334	int i;
3220	3335	BUG_ON(sh->batch_head);
@@ -3250,7 +3365,7 @@ handle_failed_stripe(struct r5conf conf, struct stripe_head sh, · 29 unmodified lines
3250	3365	if (bi)
3251	3366	bitmap_end = 1;
3252	3367
3253		− r5l_stripe_write_finished(sh);
	3368	+ log_stripe_write_finished(sh);
3254	3369
3255	3370	if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
3256	3371	wake_up(&conf->wait_for_overlap);
@@ -3260,10 +3375,8 @@ handle_failed_stripe(struct r5conf conf, struct stripe_head sh, · 3 unmodified lines
3260	3375	struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
3261	3376
3262	3377	bi->bi_error = -EIO;
3263		− if (!raid5_dec_bi_active_stripes(bi)) {
3264		− md_write_end(conf->mddev);
3265		− bio_list_add(return_bi, bi);
3266		− }
	3378	+ md_write_end(conf->mddev);
	3379	+ bio_endio(bi);
3267	3380	bi = nextbi;
3268	3381	}
3269	3382	if (bitmap_end)
@@ -3284,10 +3397,8 @@ handle_failed_stripe(struct r5conf conf, struct stripe_head sh, · 14 unmodified lines
3284	3397	struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
3285	3398
3286	3399	bi->bi_error = -EIO;
3287		− if (!raid5_dec_bi_active_stripes(bi)) {
3288		− md_write_end(conf->mddev);
3289		− bio_list_add(return_bi, bi);
3290		− }
	3400	+ md_write_end(conf->mddev);
	3401	+ bio_endio(bi);
3291	3402	bi = bi2;
3292	3403	}
3293	3404
@@ -3312,8 +3423,7 @@ handle_failed_stripe(struct r5conf conf, struct stripe_head sh, · 18 unmodified lines
3312	3423	r5_next_bio(bi, sh->dev[i].sector);
3313	3424
3314	3425	bi->bi_error = -EIO;
3315		− if (!raid5_dec_bi_active_stripes(bi))
3316		− bio_list_add(return_bi, bi);
	3426	+ bio_endio(bi);
3317	3427	bi = nextbi;
3318	3428	}
3319	3429	}
@@ -3449,7 +3559,7 @@ static int need_this_block(struct stripe_head sh, struct stripe_head_state s, · 129 unmodified lines
3449	3559	!test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
3450	3560	/* Pre-reads at not permitted until after short delay
3451	3561	* to gather multiple requests. However if this
3452		− * device is no Insync, the block could only be be computed
	3562	+ * device is no Insync, the block could only be computed
3453	3563	* and there is no need to delay that.
3454	3564	*/
3455	3565	return 0;
@@ -3468,7 +3578,7 @@ static int need_this_block(struct stripe_head sh, struct stripe_head_state s, · 12 unmodified lines
3468	3578
3469	3579	/* If we are forced to do a reconstruct-write, either because
3470	3580	* the current RAID6 implementation only supports that, or
3471		− * or because parity cannot be trusted and we are currently
	3581	+ * because parity cannot be trusted and we are currently
3472	3582	* recovering it, there is extra need to be careful.
3473	3583	* If one of the devices that we would need to read, because
3474	3584	* it is not being overwritten (and maybe not written at all)
@@ -3508,9 +3618,20 @@ static int fetch_block(struct stripe_head sh, struct stripe_head_state s, · 33 unmodified lines
3508	3618	BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
3509	3619	BUG_ON(test_bit(R5_Wantread, &dev->flags));
3510	3620	BUG_ON(sh->batch_head);
	3621	+
	3622	+ /*
	3623	+ * In the raid6 case if the only non-uptodate disk is P
	3624	+ * then we already trusted P to compute the other failed
	3625	+ * drives. It is safe to compute rather than re-read P.
	3626	+ * In other cases we only compute blocks from failed
	3627	+ * devices, otherwise check/repair might fail to detect
	3628	+ * a real inconsistency.
	3629	+ */
	3630	+
3511	3631	if ((s->uptodate == disks - 1) &&
	3632	+ ((sh->qd_idx >= 0 && sh->pd_idx == disk_idx) \|\|
3512	3633	(s->failed && (disk_idx == s->failed_num[0] \|\|
3513		− disk_idx == s->failed_num[1]))) {
	3634	+ disk_idx == s->failed_num[1])))) {
3514	3635	/* have disk failed, and we're requested to fetch it;
3515	3636	* do compute it
3516	3637	*/
@@ -3612,7 +3733,7 @@ static void break_stripe_batch_list(struct stripe_head *head_sh, · 95 unmodified lines
3612	3733	* never LOCKED, so we don't need to test 'failed' directly.
3613	3734	*/
3614	3735	static void handle_stripe_clean_event(struct r5conf *conf,
3615		− struct stripe_head sh, int disks, struct bio_list return_bi)
	3736	+ struct stripe_head *sh, int disks)
3616	3737	{
3617	3738	int i;
3618	3739	struct r5dev *dev;
@@ -3644,10 +3765,8 @@ returnbi: · 25 unmodified lines
3644	3765	while (wbi && wbi->bi_iter.bi_sector <
3645	3766	dev->sector + STRIPE_SECTORS) {
3646	3767	wbi2 = r5_next_bio(wbi, dev->sector);
3647		− if (!raid5_dec_bi_active_stripes(wbi)) {
3648		− md_write_end(conf->mddev);
3649		− bio_list_add(return_bi, wbi);
3650		− }
	3768	+ md_write_end(conf->mddev);
	3769	+ bio_endio(wbi);
3651	3770	wbi = wbi2;
3652	3771	}
3653	3772	bitmap_endwrite(conf->mddev->bitmap, sh->sector,
@@ -3669,7 +3788,7 @@ returnbi: · 15 unmodified lines
3669	3788	discard_pending = 1;
3670	3789	}
3671	3790
3672		− r5l_stripe_write_finished(sh);
	3791	+ log_stripe_write_finished(sh);
3673	3792
3674	3793	if (!discard_pending &&
3675	3794	test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {
@@ -4556,7 +4675,8 @@ static void handle_stripe(struct stripe_head *sh) · 880 unmodified lines
4556	4675	if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))
4557	4676	goto finish;
4558	4677
4559		− if (s.handle_bad_blocks) {
	4678	+ if (s.handle_bad_blocks \|\|
	4679	+ test_bit(MD_SB_CHANGE_PENDING, &conf->mddev->sb_flags)) {
4560	4680	set_bit(STRIPE_HANDLE, &sh->state);
4561	4681	goto finish;
4562	4682	}
@@ -4589,7 +4709,7 @@ static void handle_stripe(struct stripe_head *sh) · 26 unmodified lines
4589	4709	sh->reconstruct_state = 0;
4590	4710	break_stripe_batch_list(sh, 0);
4591	4711	if (s.to_read+s.to_write+s.written)
4592		− handle_failed_stripe(conf, sh, &s, disks, &s.return_bi);
	4712	+ handle_failed_stripe(conf, sh, &s, disks);
4593	4713	if (s.syncing + s.replacing)
4594	4714	handle_failed_sync(conf, sh, &s);
4595	4715	}
@@ -4655,11 +4775,11 @@ static void handle_stripe(struct stripe_head *sh) · 59 unmodified lines
4655	4775	&& !test_bit(R5_LOCKED, &qdev->flags)
4656	4776	&& (test_bit(R5_UPTODATE, &qdev->flags) \|\|
4657	4777	test_bit(R5_Discard, &qdev->flags))))))
4658		− handle_stripe_clean_event(conf, sh, disks, &s.return_bi);
	4778	+ handle_stripe_clean_event(conf, sh, disks);
4659	4779
4660	4780	if (s.just_cached)
4661		− r5c_handle_cached_data_endio(conf, sh, disks, &s.return_bi);
4662		− r5l_stripe_write_finished(sh);
	4781	+ r5c_handle_cached_data_endio(conf, sh, disks);
	4782	+ log_stripe_write_finished(sh);
4663	4783
4664	4784	/* Now we might consider reading some blocks, either to check/generate
4665	4785	* parity, or to satisfy requests
@@ -4886,16 +5006,6 @@ finish: · 220 unmodified lines
4886	5006	md_wakeup_thread(conf->mddev->thread);
4887	5007	}
4888	5008
4889		− if (!bio_list_empty(&s.return_bi)) {
4890		− if (test_bit(MD_SB_CHANGE_PENDING, &conf->mddev->sb_flags)) {
4891		− spin_lock_irq(&conf->device_lock);
4892		− bio_list_merge(&conf->return_bi, &s.return_bi);
4893		− spin_unlock_irq(&conf->device_lock);
4894		− md_wakeup_thread(conf->mddev->thread);
4895		− } else
4896		− return_io(&s.return_bi);
4897		− }
4898		−
4899	5009	clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
4900	5010	}
4901	5011
@@ -4984,12 +5094,14 @@ static void add_bio_to_retry(struct bio bi,struct r5conf conf) · 82 unmodified lines
4984	5094	md_wakeup_thread(conf->mddev->thread);
4985	5095	}
4986	5096
4987		−static struct bio remove_bio_from_retry(struct r5conf conf)
	5097	+static struct bio remove_bio_from_retry(struct r5conf conf,
	5098	+ unsigned int *offset)
4988	5099	{
4989	5100	struct bio *bi;
4990	5101
4991	5102	bi = conf->retry_read_aligned;
4992	5103	if (bi) {
	5104	+ *offset = conf->retry_read_offset;
4993	5105	conf->retry_read_aligned = NULL;
4994	5106	return bi;
4995	5107	}
@@ -4997,11 +5109,7 @@ static struct bio remove_bio_from_retry(struct r5conf conf) · 1 unmodified line
4997	5109	if(bi) {
4998	5110	conf->retry_read_aligned_list = bi->bi_next;
4999	5111	bi->bi_next = NULL;
5000		− /*
5001		− * this sets the active strip count to 1 and the processed
5002		− * strip count to zero (upper 8 bits)
5003		− */
5004		− raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */
	5112	+ *offset = 0;
5005	5113	}
5006	5114
5007	5115	return bi;
@@ -5136,24 +5244,20 @@ static int raid5_read_one_chunk(struct mddev mddev, struct bio raid_bio) · 128 unmodified lines
5136	5244	static struct bio chunk_aligned_read(struct mddev mddev, struct bio *raid_bio)
5137	5245	{
5138	5246	struct bio *split;
	5247	+ sector_t sector = raid_bio->bi_iter.bi_sector;
	5248	+ unsigned chunk_sects = mddev->chunk_sectors;
	5249	+ unsigned sectors = chunk_sects - (sector & (chunk_sects-1));
5139	5250
5140		− do {
5141		− sector_t sector = raid_bio->bi_iter.bi_sector;
5142		− unsigned chunk_sects = mddev->chunk_sectors;
5143		− unsigned sectors = chunk_sects - (sector & (chunk_sects-1));
5144		−
5145		− if (sectors < bio_sectors(raid_bio)) {
5146		− split = bio_split(raid_bio, sectors, GFP_NOIO, fs_bio_set);
5147		− bio_chain(split, raid_bio);
5148		− } else
5149		− split = raid_bio;
	5251	+ if (sectors < bio_sectors(raid_bio)) {
	5252	+ struct r5conf *conf = mddev->private;
	5253	+ split = bio_split(raid_bio, sectors, GFP_NOIO, conf->bio_split);
	5254	+ bio_chain(split, raid_bio);
	5255	+ generic_make_request(raid_bio);
	5256	+ raid_bio = split;
	5257	+ }
5150	5258
5151		− if (!raid5_read_one_chunk(mddev, split)) {
5152		− if (split != raid_bio)
5153		− generic_make_request(raid_bio);
5154		− return split;
5155		− }
5156		− } while (split != raid_bio);
	5259	+ if (!raid5_read_one_chunk(mddev, raid_bio))
	5260	+ return raid_bio;
5157	5261
5158	5262	return NULL;
5159	5263	}
@@ -5170,19 +5274,27 @@ static struct bio chunk_aligned_read(struct mddev mddev, struct bio *raid_bio) · 10 unmodified lines
5170	5274	*/
5171	5275	static struct stripe_head __get_priority_stripe(struct r5conf conf, int group)
5172	5276	{
5173		− struct stripe_head sh = NULL, tmp;
	5277	+ struct stripe_head sh, tmp;
5174	5278	struct list_head *handle_list = NULL;
5175		− struct r5worker_group *wg = NULL;
	5279	+ struct r5worker_group *wg;
	5280	+ bool second_try = !r5c_is_writeback(conf->log);
	5281	+ bool try_loprio = test_bit(R5C_LOG_TIGHT, &conf->cache_state);
5176	5282
	5283	+again:
	5284	+ wg = NULL;
	5285	+ sh = NULL;
5177	5286	if (conf->worker_cnt_per_group == 0) {
5178		− handle_list = &conf->handle_list;
	5287	+ handle_list = try_loprio ? &conf->loprio_list :
	5288	+ &conf->handle_list;
5179	5289	} else if (group != ANY_GROUP) {
5180		− handle_list = &conf->worker_groups[group].handle_list;
	5290	+ handle_list = try_loprio ? &conf->worker_groups[group].loprio_list :
	5291	+ &conf->worker_groups[group].handle_list;
5181	5292	wg = &conf->worker_groups[group];
5182	5293	} else {
5183	5294	int i;
5184	5295	for (i = 0; i < conf->group_cnt; i++) {
5185		− handle_list = &conf->worker_groups[i].handle_list;
	5296	+ handle_list = try_loprio ? &conf->worker_groups[i].loprio_list :
	5297	+ &conf->worker_groups[i].handle_list;
5186	5298	wg = &conf->worker_groups[i];
5187	5299	if (!list_empty(handle_list))
5188	5300	break;
@@ -5233,8 +5345,13 @@ static struct stripe_head __get_priority_stripe(struct r5conf conf, int group) · 44 unmodified lines
5233	5345	wg = NULL;
5234	5346	}
5235	5347
5236		− if (!sh)
5237		− return NULL;
	5348	+ if (!sh) {
	5349	+ if (second_try)
	5350	+ return NULL;
	5351	+ second_try = true;
	5352	+ try_loprio = !try_loprio;
	5353	+ goto again;
	5354	+ }
5238	5355
5239	5356	if (wg) {
5240	5357	wg->stripes_cnt--;
@@ -5323,7 +5440,6 @@ static void make_discard_request(struct mddev mddev, struct bio bi) · 82 unmodified lines
5323	5440	struct r5conf *conf = mddev->private;
5324	5441	sector_t logical_sector, last_sector;
5325	5442	struct stripe_head *sh;
5326		− int remaining;
5327	5443	int stripe_sectors;
5328	5444
5329	5445	if (mddev->reshape_position != MaxSector)
@@ -5334,7 +5450,7 @@ static void make_discard_request(struct mddev mddev, struct bio bi) · 4 unmodified lines
5334	5450	last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9);
5335	5451
5336	5452	bi->bi_next = NULL;
5337		− bi->bi_phys_segments = 1; /* over-loaded to count active stripes */
	5453	+ md_write_start(mddev, bi);
5338	5454
5339	5455	stripe_sectors = conf->chunk_sectors *
5340	5456	(conf->raid_disks - conf->max_degraded);
@@ -5380,7 +5496,8 @@ static void make_discard_request(struct mddev mddev, struct bio bi) · 39 unmodified lines
5380	5496	continue;
5381	5497	sh->dev[d].towrite = bi;
5382	5498	set_bit(R5_OVERWRITE, &sh->dev[d].flags);
5383		− raid5_inc_bi_active_stripes(bi);
	5499	+ bio_inc_remaining(bi);
	5500	+ md_write_inc(mddev, bi);
5384	5501	sh->overwrite_disks++;
5385	5502	}
5386	5503	spin_unlock_irq(&sh->stripe_lock);
@@ -5403,11 +5520,8 @@ static void make_discard_request(struct mddev mddev, struct bio bi) · 16 unmodified lines
5403	5520	release_stripe_plug(mddev, sh);
5404	5521	}
5405	5522
5406		− remaining = raid5_dec_bi_active_stripes(bi);
5407		− if (remaining == 0) {
5408		− md_write_end(mddev);
5409		− bio_endio(bi);
5410		− }
	5523	+ md_write_end(mddev);
	5524	+ bio_endio(bi);
5411	5525	}
5412	5526
5413	5527	static void raid5_make_request(struct mddev mddev, struct bio bi)
@@ -5418,7 +5532,6 @@ static void raid5_make_request(struct mddev mddev, struct bio bi) · 4 unmodified lines
5418	5532	sector_t logical_sector, last_sector;
5419	5533	struct stripe_head *sh;
5420	5534	const int rw = bio_data_dir(bi);
5421		− int remaining;
5422	5535	DEFINE_WAIT(w);
5423	5536	bool do_prepare;
5424	5537	bool do_flush = false;
@@ -5440,8 +5553,6 @@ static void raid5_make_request(struct mddev mddev, struct bio bi) · 15 unmodified lines
5440	5553	do_flush = bi->bi_opf & REQ_PREFLUSH;
5441	5554	}
5442	5555
5443		− md_write_start(mddev, bi);
5444		−
5445	5556	/*
5446	5557	* If array is degraded, better not do chunk aligned read because
5447	5558	* later we might have to read it again in order to reconstruct
@@ -5462,7 +5573,7 @@ static void raid5_make_request(struct mddev mddev, struct bio bi) · 14 unmodified lines
5462	5573	logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
5463	5574	last_sector = bio_end_sector(bi);
5464	5575	bi->bi_next = NULL;
5465		− bi->bi_phys_segments = 1; /* over-loaded to count active stripes */
	5576	+ md_write_start(mddev, bi);
5466	5577
5467	5578	prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
5468	5579	for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
@@ -5597,16 +5708,9 @@ static void raid5_make_request(struct mddev mddev, struct bio bi) · 128 unmodified lines
5597	5708	}
5598	5709	finish_wait(&conf->wait_for_overlap, &w);
5599	5710
5600		− remaining = raid5_dec_bi_active_stripes(bi);
5601		− if (remaining == 0) {
5602		−
5603		− if ( rw == WRITE )
5604		− md_write_end(mddev);
5605		−
5606		− trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
5607		− bi, 0);
5608		− bio_endio(bi);
5609		− }
	5711	+ if (rw == WRITE)
	5712	+ md_write_end(mddev);
	5713	+ bio_endio(bi);
5610	5714	}
5611	5715
5612	5716	static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks);
@@ -5955,7 +6059,8 @@ static inline sector_t raid5_sync_request(struct mddev *mddev, sector_t sector_n · 342 unmodified lines
5955	6059	return STRIPE_SECTORS;
5956	6060	}
5957	6061
5958		−static int retry_aligned_read(struct r5conf conf, struct bio raid_bio)
	6062	+static int retry_aligned_read(struct r5conf conf, struct bio raid_bio,
	6063	+ unsigned int offset)
5959	6064	{
5960	6065	/* We may not be able to submit a whole bio at once as there
5961	6066	* may not be enough stripe_heads available.
@@ -5971,7 +6076,6 @@ static int retry_aligned_read(struct r5conf conf, struct bio raid_bio) · 9 unmodified lines
5971	6076	int dd_idx;
5972	6077	sector_t sector, logical_sector, last_sector;
5973	6078	int scnt = 0;
5974		− int remaining;
5975	6079	int handled = 0;
5976	6080
5977	6081	logical_sector = raid_bio->bi_iter.bi_sector &
@@ -5985,7 +6089,7 @@ static int retry_aligned_read(struct r5conf conf, struct bio raid_bio) · 7 unmodified lines
5985	6089	sector += STRIPE_SECTORS,
5986	6090	scnt++) {
5987	6091
5988		− if (scnt < raid5_bi_processed_stripes(raid_bio))
	6092	+ if (scnt < offset)
5989	6093	/* already done this stripe */
5990	6094	continue;
5991	6095
@@ -5993,15 +6097,15 @@ static int retry_aligned_read(struct r5conf conf, struct bio raid_bio) · 1 unmodified line
5993	6097
5994	6098	if (!sh) {
5995	6099	/* failed to get a stripe - must wait */
5996		− raid5_set_bi_processed_stripes(raid_bio, scnt);
5997	6100	conf->retry_read_aligned = raid_bio;
	6101	+ conf->retry_read_offset = scnt;
5998	6102	return handled;
5999	6103	}
6000	6104
6001	6105	if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) {
6002	6106	raid5_release_stripe(sh);
6003		− raid5_set_bi_processed_stripes(raid_bio, scnt);
6004	6107	conf->retry_read_aligned = raid_bio;
	6108	+ conf->retry_read_offset = scnt;
6005	6109	return handled;
6006	6110	}
6007	6111
@@ -6010,12 +6114,9 @@ static int retry_aligned_read(struct r5conf conf, struct bio raid_bio) · 2 unmodified lines
6010	6114	raid5_release_stripe(sh);
6011	6115	handled++;
6012	6116	}
6013		− remaining = raid5_dec_bi_active_stripes(raid_bio);
6014		− if (remaining == 0) {
6015		− trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev),
6016		− raid_bio, 0);
6017		− bio_endio(raid_bio);
6018		− }
	6117	+
	6118	+ bio_endio(raid_bio);
	6119	+
6019	6120	if (atomic_dec_and_test(&conf->active_aligned_reads))
6020	6121	wake_up(&conf->wait_for_quiescent);
6021	6122	return handled;
@@ -6058,7 +6159,7 @@ static int handle_active_stripes(struct r5conf *conf, int group, · 36 unmodified lines
6058	6159
6059	6160	for (i = 0; i < batch_size; i++)
6060	6161	handle_stripe(batch[i]);
6061		− r5l_write_stripe_run(conf->log);
	6162	+ log_write_stripe_run(conf);
6062	6163
6063	6164	cond_resched();
6064	6165
@@ -6075,6 +6176,7 @@ static void raid5_do_work(struct work_struct *work) · 10 unmodified lines
6075	6176	struct r5worker *worker = container_of(work, struct r5worker, work);
6076	6177	struct r5worker_group *group = worker->group;
6077	6178	struct r5conf *conf = group->conf;
	6179	+ struct mddev *mddev = conf->mddev;
6078	6180	int group_id = group - conf->worker_groups;
6079	6181	int handled;
6080	6182	struct blk_plug plug;
@@ -6095,6 +6197,9 @@ static void raid5_do_work(struct work_struct *work) · 14 unmodified lines
6095	6197	if (!batch_size && !released)
6096	6198	break;
6097	6199	handled += batch_size;
	6200	+ wait_event_lock_irq(mddev->sb_wait,
	6201	+ !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags),
	6202	+ conf->device_lock);
6098	6203	}
6099	6204	pr_debug("%d stripes handled\n", handled);
6100	6205
@@ -6122,24 +6227,13 @@ static void raid5d(struct md_thread *thread) · 21 unmodified lines
6122	6227
6123	6228	md_check_recovery(mddev);
6124	6229
6125		− if (!bio_list_empty(&conf->return_bi) &&
6126		− !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
6127		− struct bio_list tmp = BIO_EMPTY_LIST;
6128		− spin_lock_irq(&conf->device_lock);
6129		− if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
6130		− bio_list_merge(&tmp, &conf->return_bi);
6131		− bio_list_init(&conf->return_bi);
6132		− }
6133		− spin_unlock_irq(&conf->device_lock);
6134		− return_io(&tmp);
6135		− }
6136		−
6137	6230	blk_start_plug(&plug);
6138	6231	handled = 0;
6139	6232	spin_lock_irq(&conf->device_lock);
6140	6233	while (1) {
6141	6234	struct bio *bio;
6142	6235	int batch_size, released;
	6236	+ unsigned int offset;
6143	6237
6144	6238	released = release_stripe_list(conf, conf->temp_inactive_list);
6145	6239	if (released)
@@ -6157,10 +6251,10 @@ static void raid5d(struct md_thread *thread) · 11 unmodified lines
6157	6251	}
6158	6252	raid5_activate_delayed(conf);
6159	6253
6160		− while ((bio = remove_bio_from_retry(conf))) {
	6254	+ while ((bio = remove_bio_from_retry(conf, &offset))) {
6161	6255	int ok;
6162	6256	spin_unlock_irq(&conf->device_lock);
6163		− ok = retry_aligned_read(conf, bio);
	6257	+ ok = retry_aligned_read(conf, bio, offset);
6164	6258	spin_lock_irq(&conf->device_lock);
6165	6259	if (!ok)
6166	6260	break;
@@ -6544,6 +6638,7 @@ static int alloc_thread_groups(struct r5conf *conf, int cnt, · 377 unmodified lines
6544	6638
6545	6639	group = &(*worker_groups)[i];
6546	6640	INIT_LIST_HEAD(&group->handle_list);
	6641	+ INIT_LIST_HEAD(&group->loprio_list);
6547	6642	group->conf = conf;
6548	6643	group->workers = workers + i * cnt;
6549	6644
@@ -6634,8 +6729,8 @@ static void free_conf(struct r5conf *conf) · 84 unmodified lines
6634	6729	{
6635	6730	int i;
6636	6731
6637		− if (conf->log)
6638		− r5l_exit_log(conf->log);
	6732	+ log_exit(conf);
	6733	+
6639	6734	if (conf->shrinker.nr_deferred)
6640	6735	unregister_shrinker(&conf->shrinker);
6641	6736
@@ -6646,7 +6741,10 @@ static void free_conf(struct r5conf *conf) · 4 unmodified lines
6646	6741	if (conf->disks[i].extra_page)
6647	6742	put_page(conf->disks[i].extra_page);
6648	6743	kfree(conf->disks);
	6744	+ if (conf->bio_split)
	6745	+ bioset_free(conf->bio_split);
6649	6746	kfree(conf->stripe_hashtbl);
	6747	+ kfree(conf->pending_data);
6650	6748	kfree(conf);
6651	6749	}
6652	6750
@@ -6756,6 +6854,14 @@ static struct r5conf setup_conf(struct mddev mddev) · 103 unmodified lines
6756	6854	conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);
6757	6855	if (conf == NULL)
6758	6856	goto abort;
	6857	+ INIT_LIST_HEAD(&conf->free_list);
	6858	+ INIT_LIST_HEAD(&conf->pending_list);
	6859	+ conf->pending_data = kzalloc(sizeof(struct r5pending_data) *
	6860	+ PENDING_IO_MAX, GFP_KERNEL);
	6861	+ if (!conf->pending_data)
	6862	+ goto abort;
	6863	+ for (i = 0; i < PENDING_IO_MAX; i++)
	6864	+ list_add(&conf->pending_data[i].sibling, &conf->free_list);
6759	6865	/* Don't enable multi-threading by default*/
6760	6866	if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group,
6761	6867	&new_group)) {
@@ -6771,15 +6877,14 @@ static struct r5conf setup_conf(struct mddev mddev) · 9 unmodified lines
6771	6877	init_waitqueue_head(&conf->wait_for_stripe);
6772	6878	init_waitqueue_head(&conf->wait_for_overlap);
6773	6879	INIT_LIST_HEAD(&conf->handle_list);
	6880	+ INIT_LIST_HEAD(&conf->loprio_list);
6774	6881	INIT_LIST_HEAD(&conf->hold_list);
6775	6882	INIT_LIST_HEAD(&conf->delayed_list);
6776	6883	INIT_LIST_HEAD(&conf->bitmap_list);
6777		− bio_list_init(&conf->return_bi);
6778	6884	init_llist_head(&conf->released_stripes);
6779	6885	atomic_set(&conf->active_stripes, 0);
6780	6886	atomic_set(&conf->preread_active_stripes, 0);
6781	6887	atomic_set(&conf->active_aligned_reads, 0);
6782		− bio_list_init(&conf->pending_bios);
6783	6888	spin_lock_init(&conf->pending_bios_lock);
6784	6889	conf->batch_bio_dispatch = true;
6785	6890	rdev_for_each(rdev, mddev) {
@@ -6813,6 +6918,9 @@ static struct r5conf setup_conf(struct mddev mddev) · 27 unmodified lines
6813	6918	goto abort;
6814	6919	}
6815	6920
	6921	+ conf->bio_split = bioset_create(BIO_POOL_SIZE, 0);
	6922	+ if (!conf->bio_split)
	6923	+ goto abort;
6816	6924	conf->mddev = mddev;
6817	6925
6818	6926	if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
@@ -7097,6 +7205,13 @@ static int raid5_run(struct mddev *mddev) · 278 unmodified lines
7097	7205	BUG_ON(mddev->delta_disks != 0);
7098	7206	}
7099	7207
	7208	+ if (test_bit(MD_HAS_JOURNAL, &mddev->flags) &&
	7209	+ test_bit(MD_HAS_PPL, &mddev->flags)) {
	7210	+ pr_warn("md/raid:%s: using journal device and PPL not allowed - disabling PPL\n",
	7211	+ mdname(mddev));
	7212	+ clear_bit(MD_HAS_PPL, &mddev->flags);
	7213	+ }
	7214	+
7100	7215	if (mddev->private == NULL)
7101	7216	conf = setup_conf(mddev);
7102	7217	else
@@ -7188,7 +7303,10 @@ static int raid5_run(struct mddev *mddev) · 85 unmodified lines
7188	7303
7189	7304	if (mddev->degraded > dirty_parity_disks &&
7190	7305	mddev->recovery_cp != MaxSector) {
7191		− if (mddev->ok_start_degraded)
	7306	+ if (test_bit(MD_HAS_PPL, &mddev->flags))
	7307	+ pr_crit("md/raid:%s: starting dirty degraded array with PPL.\n",
	7308	+ mdname(mddev));
	7309	+ else if (mddev->ok_start_degraded)
7192	7310	pr_crit("md/raid:%s: starting dirty degraded array - data corruption possible.\n",
7193	7311	mdname(mddev));
7194	7312	else {
@@ -7254,14 +7372,6 @@ static int raid5_run(struct mddev *mddev) · 59 unmodified lines
7254	7372	mddev->queue->limits.discard_alignment = stripe;
7255	7373	mddev->queue->limits.discard_granularity = stripe;
7256	7374
7257		− /*
7258		− * We use 16-bit counter of active stripes in bi_phys_segments
7259		− * (minus one for over-loaded initialization)
7260		− */
7261		− blk_queue_max_hw_sectors(mddev->queue, 0xfffe * STRIPE_SECTORS);
7262		− blk_queue_max_discard_sectors(mddev->queue,
7263		− 0xfffe * STRIPE_SECTORS);
7264		−
7265	7375	blk_queue_max_write_same_sectors(mddev->queue, 0);
7266	7376	blk_queue_max_write_zeroes_sectors(mddev->queue, 0);
7267	7377
@@ -7299,14 +7409,8 @@ static int raid5_run(struct mddev *mddev) · 31 unmodified lines
7299	7409	blk_queue_max_hw_sectors(mddev->queue, UINT_MAX);
7300	7410	}
7301	7411
7302		− if (journal_dev) {
7303		− char b[BDEVNAME_SIZE];
7304		−
7305		− pr_debug("md/raid:%s: using device %s as journal\n",
7306		− mdname(mddev), bdevname(journal_dev->bdev, b));
7307		− if (r5l_init_log(conf, journal_dev))
7308		− goto abort;
7309		− }
	7412	+ if (log_init(conf, journal_dev, raid5_has_ppl(conf)))
	7413	+ goto abort;
7310	7414
7311	7415	return 0;
7312	7416	abort:
@@ -7420,17 +7524,16 @@ static int raid5_remove_disk(struct mddev mddev, struct md_rdev rdev) · 107 unmodified lines
7420	7524
7421	7525	print_raid5_conf(conf);
7422	7526	if (test_bit(Journal, &rdev->flags) && conf->log) {
7423		− struct r5l_log *log;
7424	7527	/*
7425	7528	* we can't wait pending write here, as this is called in
7426	7529	* raid5d, wait will deadlock.
	7530	+ * neilb: there is no locking about new writes here,
	7531	+ * so this cannot be safe.
7427	7532	*/
7428		− if (atomic_read(&mddev->writes_pending))
	7533	+ if (atomic_read(&conf->active_stripes)) {
7429	7534	return -EBUSY;
7430		− log = conf->log;
7431		− conf->log = NULL;
7432		− synchronize_rcu();
7433		− r5l_exit_log(log);
	7535	+ }
	7536	+ log_exit(conf);
7434	7537	return 0;
7435	7538	}
7436	7539	if (rdev == p->rdev)
@@ -7469,6 +7572,11 @@ static int raid5_remove_disk(struct mddev mddev, struct md_rdev rdev) · 32 unmodified lines
7469	7572	*rdevp = rdev;
7470	7573	}
7471	7574	}
	7575	+ if (!err) {
	7576	+ err = log_modify(conf, rdev, false);
	7577	+ if (err)
	7578	+ goto abort;
	7579	+ }
7472	7580	if (p->replacement) {
7473	7581	/* We must have just cleared 'rdev' */
7474	7582	p->rdev = p->replacement;
@@ -7477,12 +7585,12 @@ static int raid5_remove_disk(struct mddev mddev, struct md_rdev rdev) · 2 unmodified lines
7477	7585	* but will never see neither - if they are careful
7478	7586	*/
7479	7587	p->replacement = NULL;
7480		− clear_bit(WantReplacement, &rdev->flags);
7481		− } else
7482		− /* We might have just removed the Replacement as faulty-
7483		− * clear the bit just in case
7484		− */
7485		− clear_bit(WantReplacement, &rdev->flags);
	7588	+
	7589	+ if (!err)
	7590	+ err = log_modify(conf, p->rdev, true);
	7591	+ }
	7592	+
	7593	+ clear_bit(WantReplacement, &rdev->flags);
7486	7594	abort:
7487	7595
7488	7596	print_raid5_conf(conf);
@@ -7499,7 +7607,6 @@ static int raid5_add_disk(struct mddev mddev, struct md_rdev rdev) · 10 unmodified lines
7499	7607	int last = conf->raid_disks - 1;
7500	7608
7501	7609	if (test_bit(Journal, &rdev->flags)) {
7502		− char b[BDEVNAME_SIZE];
7503	7610	if (conf->log)
7504	7611	return -EBUSY;
7505	7612
@@ -7508,9 +7615,7 @@ static int raid5_add_disk(struct mddev mddev, struct md_rdev rdev) · 2 unmodified lines
7508	7615	* The array is in readonly mode if journal is missing, so no
7509	7616	* write requests running. We should be safe
7510	7617	*/
7511		− r5l_init_log(conf, rdev);
7512		− pr_debug("md/raid:%s: using device %s as journal\n",
7513		− mdname(mddev), bdevname(rdev->bdev, b));
	7618	+ log_init(conf, rdev, false);
7514	7619	return 0;
7515	7620	}
7516	7621	if (mddev->recovery_disabled == conf->recovery_disabled)
@@ -7537,10 +7642,12 @@ static int raid5_add_disk(struct mddev mddev, struct md_rdev rdev) · 20 unmodified lines
7537	7642	if (p->rdev == NULL) {
7538	7643	clear_bit(In_sync, &rdev->flags);
7539	7644	rdev->raid_disk = disk;
7540		− err = 0;
7541	7645	if (rdev->saved_raid_disk != disk)
7542	7646	conf->fullsync = 1;
7543	7647	rcu_assign_pointer(p->rdev, rdev);
	7648	+
	7649	+ err = log_modify(conf, rdev, true);
	7650	+
7544	7651	goto out;
7545	7652	}
7546	7653	}
@@ -7574,7 +7681,7 @@ static int raid5_resize(struct mddev *mddev, sector_t sectors) · 27 unmodified lines
7574	7681	sector_t newsize;
7575	7682	struct r5conf *conf = mddev->private;
7576	7683
7577		− if (conf->log)
	7684	+ if (conf->log \|\| raid5_has_ppl(conf))
7578	7685	return -EINVAL;
7579	7686	sectors &= ~((sector_t)conf->chunk_sectors - 1);
7580	7687	newsize = raid5_size(mddev, sectors, mddev->raid_disks);
@@ -7625,7 +7732,7 @@ static int check_reshape(struct mddev *mddev) · 44 unmodified lines
7625	7732	{
7626	7733	struct r5conf *conf = mddev->private;
7627	7734
7628		− if (conf->log)
	7735	+ if (conf->log \|\| raid5_has_ppl(conf))
7629	7736	return -EINVAL;
7630	7737	if (mddev->delta_disks == 0 &&
7631	7738	mddev->new_layout == mddev->layout &&
@@ -7658,6 +7765,9 @@ static int check_reshape(struct mddev *mddev) · 26 unmodified lines
7658	7765	mddev->chunk_sectors)
7659	7766	) < 0)
7660	7767	return -ENOMEM;
	7768	+
	7769	+ if (conf->previous_raid_disks + mddev->delta_disks <= conf->pool_size)
	7770	+ return 0; /* never bother to shrink */
7661	7771	return resize_stripes(conf, (conf->previous_raid_disks
7662	7772	+ mddev->delta_disks));
7663	7773	}
@@ -8148,6 +8258,68 @@ static void raid6_takeover(struct mddev mddev) · 484 unmodified lines
8148	8258	return setup_conf(mddev);
8149	8259	}
8150	8260
	8261	+static int raid5_change_consistency_policy(struct mddev mddev, const char buf)
	8262	+{
	8263	+ struct r5conf *conf;
	8264	+ int err;
	8265	+
	8266	+ err = mddev_lock(mddev);
	8267	+ if (err)
	8268	+ return err;
	8269	+ conf = mddev->private;
	8270	+ if (!conf) {
	8271	+ mddev_unlock(mddev);
	8272	+ return -ENODEV;
	8273	+ }
	8274	+
	8275	+ if (strncmp(buf, "ppl", 3) == 0) {
	8276	+ /* ppl only works with RAID 5 */
	8277	+ if (!raid5_has_ppl(conf) && conf->level == 5) {
	8278	+ err = log_init(conf, NULL, true);
	8279	+ if (!err) {
	8280	+ err = resize_stripes(conf, conf->pool_size);
	8281	+ if (err)
	8282	+ log_exit(conf);
	8283	+ }
	8284	+ } else
	8285	+ err = -EINVAL;
	8286	+ } else if (strncmp(buf, "resync", 6) == 0) {
	8287	+ if (raid5_has_ppl(conf)) {
	8288	+ mddev_suspend(mddev);
	8289	+ log_exit(conf);
	8290	+ mddev_resume(mddev);
	8291	+ err = resize_stripes(conf, conf->pool_size);
	8292	+ } else if (test_bit(MD_HAS_JOURNAL, &conf->mddev->flags) &&
	8293	+ r5l_log_disk_error(conf)) {
	8294	+ bool journal_dev_exists = false;
	8295	+ struct md_rdev *rdev;
	8296	+
	8297	+ rdev_for_each(rdev, mddev)
	8298	+ if (test_bit(Journal, &rdev->flags)) {
	8299	+ journal_dev_exists = true;
	8300	+ break;
	8301	+ }
	8302	+
	8303	+ if (!journal_dev_exists) {
	8304	+ mddev_suspend(mddev);
	8305	+ clear_bit(MD_HAS_JOURNAL, &mddev->flags);
	8306	+ mddev_resume(mddev);
	8307	+ } else /* need remove journal device first */
	8308	+ err = -EBUSY;
	8309	+ } else
	8310	+ err = -EINVAL;
	8311	+ } else {
	8312	+ err = -EINVAL;
	8313	+ }
	8314	+
	8315	+ if (!err)
	8316	+ md_update_sb(mddev, 1);
	8317	+
	8318	+ mddev_unlock(mddev);
	8319	+
	8320	+ return err;
	8321	+}
	8322	+
8151	8323	static struct md_personality raid6_personality =
8152	8324	{
8153	8325	.name = "raid6",
@@ -8170,6 +8342,7 @@ static struct md_personality raid6_personality = · 16 unmodified lines
8170	8342	.quiesce = raid5_quiesce,
8171	8343	.takeover = raid6_takeover,
8172	8344	.congested = raid5_congested,
	8345	+ .change_consistency_policy = raid5_change_consistency_policy,
8173	8346	};
8174	8347	static struct md_personality raid5_personality =
8175	8348	{
@@ -8193,6 +8366,7 @@ static struct md_personality raid5_personality = · 17 unmodified lines
8193	8366	.quiesce = raid5_quiesce,
8194	8367	.takeover = raid5_takeover,
8195	8368	.congested = raid5_congested,
	8369	+ .change_consistency_policy = raid5_change_consistency_policy,
8196	8370	};
8197	8371
8198	8372	static struct md_personality raid4_personality =
@@ -8217,6 +8391,7 @@ static struct md_personality raid4_personality = · 18 unmodified lines
8217	8391	.quiesce = raid5_quiesce,
8218	8392	.takeover = raid4_takeover,
8219	8393	.congested = raid5_congested,
	8394	+ .change_consistency_policy = raid5_change_consistency_policy,
8220	8395	};
8221	8396
8222	8397	static int __init raid5_init(void)

drivers/md/raid5.h+26−80

@@ -224,10 +224,16 @@ struct stripe_head { · 223 unmodified lines
224	224	spinlock_t batch_lock; /* only header's lock is useful */
225	225	struct list_head batch_list; /* protected by head's batch lock*/
226	226
227		− struct r5l_io_unit *log_io;
	227	+ union {
	228	+ struct r5l_io_unit *log_io;
	229	+ struct ppl_io_unit *ppl_io;
	230	+ };
	231	+
228	232	struct list_head log_list;
229	233	sector_t log_start; /* first meta block on the journal */
230	234	struct list_head r5c; /* for r5c_cache->stripe_in_journal */
	235	+
	236	+ struct page ppl_page; / partial parity of this stripe */
231	237	/**
232	238	* struct stripe_operations
233	239	* @target - STRIPE_OP_COMPUTE_BLK target
@@ -272,7 +278,6 @@ struct stripe_head_state { · 38 unmodified lines
272	278	int dec_preread_active;
273	279	unsigned long ops_request;
274	280
275		− struct bio_list return_bi;
276	281	struct md_rdev *blocked_rdev;
277	282	int handle_bad_blocks;
278	283	int log_failed;
@@ -400,6 +405,7 @@ enum { · 121 unmodified lines
400	405	STRIPE_OP_BIODRAIN,
401	406	STRIPE_OP_RECONSTRUCT,
402	407	STRIPE_OP_CHECK,
	408	+ STRIPE_OP_PARTIAL_PARITY,
403	409	};
404	410
405	411	/*
@@ -481,50 +487,6 @@ static inline struct bio r5_next_bio(struct bio bio, sector_t sector) · 75 unmodified lines
481	487	return NULL;
482	488	}
483	489
484		−/*
485		− * We maintain a biased count of active stripes in the bottom 16 bits of
486		− * bi_phys_segments, and a count of processed stripes in the upper 16 bits
487		− */
488		−static inline int raid5_bi_processed_stripes(struct bio *bio)
489		−{
490		− atomic_t segments = (atomic_t )&bio->bi_phys_segments;
491		−
492		− return (atomic_read(segments) >> 16) & 0xffff;
493		−}
494		−
495		−static inline int raid5_dec_bi_active_stripes(struct bio *bio)
496		−{
497		− atomic_t segments = (atomic_t )&bio->bi_phys_segments;
498		−
499		− return atomic_sub_return(1, segments) & 0xffff;
500		−}
501		−
502		−static inline void raid5_inc_bi_active_stripes(struct bio *bio)
503		−{
504		− atomic_t segments = (atomic_t )&bio->bi_phys_segments;
505		−
506		− atomic_inc(segments);
507		−}
508		−
509		−static inline void raid5_set_bi_processed_stripes(struct bio *bio,
510		− unsigned int cnt)
511		−{
512		− atomic_t segments = (atomic_t )&bio->bi_phys_segments;
513		− int old, new;
514		−
515		− do {
516		− old = atomic_read(segments);
517		− new = (old & 0xffff) \| (cnt << 16);
518		− } while (atomic_cmpxchg(segments, old, new) != old);
519		−}
520		−
521		−static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt)
522		−{
523		− atomic_t segments = (atomic_t )&bio->bi_phys_segments;
524		−
525		− atomic_set(segments, cnt);
526		−}
527		−
528	490	/* NOTE NR_STRIPE_HASH_LOCKS must remain below 64.
529	491	* This is because we sometimes take all the spinlocks
530	492	* and creating that much locking depth can cause
@@ -542,6 +504,7 @@ struct r5worker { · 11 unmodified lines
542	504
543	505	struct r5worker_group {
544	506	struct list_head handle_list;
	507	+ struct list_head loprio_list;
545	508	struct r5conf *conf;
546	509	struct r5worker *workers;
547	510	int stripes_cnt;
@@ -571,6 +534,14 @@ enum r5_cache_state { · 23 unmodified lines
571	534	*/
572	535	};
573	536
	537	+#define PENDING_IO_MAX 512
	538	+#define PENDING_IO_ONE_FLUSH 128
	539	+struct r5pending_data {
	540	+ struct list_head sibling;
	541	+ sector_t sector; /* stripe sector */
	542	+ struct bio_list bios;
	543	+};
	544	+
574	545	struct r5conf {
575	546	struct hlist_head *stripe_hashtbl;
576	547	/* only protect corresponding hash list and inactive_list */
@@ -608,10 +579,12 @@ struct r5conf { · 31 unmodified lines
608	579	*/
609	580
610	581	struct list_head handle_list; /* stripes needing handling */
	582	+ struct list_head loprio_list; /* low priority stripes */
611	583	struct list_head hold_list; /* preread ready stripes */
612	584	struct list_head delayed_list; /* stripes that have plugged requests */
613	585	struct list_head bitmap_list; /* stripes delaying awaiting bitmap update */
614	586	struct bio retry_read_aligned; / currently retrying aligned bios */
	587	+ unsigned int retry_read_offset; /* sector offset into retry_read_aligned */
615	588	struct bio retry_read_aligned_list; / aligned bios retry list */
616	589	atomic_t preread_active_stripes; /* stripes with scheduled io */
617	590	atomic_t active_aligned_reads;
@@ -621,9 +594,6 @@ struct r5conf { · 3 unmodified lines
621	594	int skip_copy; /* Don't copy data from bio to stripe cache */
622	595	struct list_head last_hold; / detect hold_list promotions */
623	596
624		− /* bios to have bi_end_io called after metadata is synced */
625		− struct bio_list return_bi;
626		−
627	597	atomic_t reshape_stripes; /* stripes with pending writes for reshape */
628	598	/* unfortunately we need two cache names as we temporarily have
629	599	* two caches.
@@ -676,6 +646,7 @@ struct r5conf { · 46 unmodified lines
676	646	int pool_size; /* number of disks in stripeheads in pool */
677	647	spinlock_t device_lock;
678	648	struct disk_info *disks;
	649	+ struct bio_set *bio_split;
679	650
680	651	/* When taking over an array from a different personality, we store
681	652	* the new thread here until we fully activate the array.
@@ -686,10 +657,15 @@ struct r5conf { · 4 unmodified lines
686	657	int group_cnt;
687	658	int worker_cnt_per_group;
688	659	struct r5l_log *log;
	660	+ void *log_private;
689	661
690		− struct bio_list pending_bios;
691	662	spinlock_t pending_bios_lock;
692	663	bool batch_bio_dispatch;
	664	+ struct r5pending_data *pending_data;
	665	+ struct list_head free_list;
	666	+ struct list_head pending_list;
	667	+ int pending_data_cnt;
	668	+ struct r5pending_data *next_pending_data;
693	669	};
694	670
695	671
@@ -765,34 +741,4 @@ extern struct stripe_head * · 69 unmodified lines
765	741	raid5_get_active_stripe(struct r5conf *conf, sector_t sector,
766	742	int previous, int noblock, int noquiesce);
767	743	extern int raid5_calc_degraded(struct r5conf *conf);
768		−extern int r5l_init_log(struct r5conf conf, struct md_rdev rdev);
769		−extern void r5l_exit_log(struct r5l_log *log);
770		−extern int r5l_write_stripe(struct r5l_log log, struct stripe_head head_sh);
771		−extern void r5l_write_stripe_run(struct r5l_log *log);
772		−extern void r5l_flush_stripe_to_raid(struct r5l_log *log);
773		−extern void r5l_stripe_write_finished(struct stripe_head *sh);
774		−extern int r5l_handle_flush_request(struct r5l_log log, struct bio bio);
775		−extern void r5l_quiesce(struct r5l_log *log, int state);
776		−extern bool r5l_log_disk_error(struct r5conf *conf);
777		−extern bool r5c_is_writeback(struct r5l_log *log);
778		−extern int
779		−r5c_try_caching_write(struct r5conf conf, struct stripe_head sh,
780		− struct stripe_head_state *s, int disks);
781		−extern void
782		−r5c_finish_stripe_write_out(struct r5conf conf, struct stripe_head sh,
783		− struct stripe_head_state *s);
784		−extern void r5c_release_extra_page(struct stripe_head *sh);
785		−extern void r5c_use_extra_page(struct stripe_head *sh);
786		−extern void r5l_wake_reclaim(struct r5l_log *log, sector_t space);
787		−extern void r5c_handle_cached_data_endio(struct r5conf *conf,
788		− struct stripe_head sh, int disks, struct bio_list return_bi);
789		−extern int r5c_cache_data(struct r5l_log log, struct stripe_head sh,
790		− struct stripe_head_state *s);
791		−extern void r5c_make_stripe_write_out(struct stripe_head *sh);
792		−extern void r5c_flush_cache(struct r5conf *conf, int num);
793		−extern void r5c_check_stripe_cache_usage(struct r5conf *conf);
794		−extern void r5c_check_cached_full_stripe(struct r5conf *conf);
795		−extern struct md_sysfs_entry r5c_journal_mode;
796		−extern void r5c_update_on_rdev_error(struct mddev *mddev);
797		−extern bool r5c_big_stripe_cached(struct r5conf *conf, sector_t sect);
798	744	#endif

include/linux/bio.h+2−9

@@ -183,7 +183,7 @@ static inline void bio_advance_iter(struct bio bio, struct bvec_iter iter, · 182 unmodified lines
183	183
184	184	#define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len)
185	185
186		−static inline unsigned __bio_segments(struct bio bio, struct bvec_iter bvec)
	186	+static inline unsigned bio_segments(struct bio *bio)
187	187	{
188	188	unsigned segs = 0;
189	189	struct bio_vec bv;
@@ -205,17 +205,12 @@ static inline unsigned __bio_segments(struct bio bio, struct bvec_iter bvec) · 15 unmodified lines
205	205	break;
206	206	}
207	207
208		− __bio_for_each_segment(bv, bio, iter, *bvec)
	208	+ bio_for_each_segment(bv, bio, iter)
209	209	segs++;
210	210
211	211	return segs;
212	212	}
213	213
214		−static inline unsigned bio_segments(struct bio *bio)
215		−{
216		− return __bio_segments(bio, &bio->bi_iter);
217		−}
218		−
219	214	/*
220	215	* get a reference to a bio, so it won't disappear. the intended use is
221	216	* something like:
@@ -389,8 +384,6 @@ extern void bio_put(struct bio *); · 167 unmodified lines
389	384	extern void __bio_clone_fast(struct bio , struct bio );
390	385	extern struct bio bio_clone_fast(struct bio , gfp_t, struct bio_set *);
391	386	extern struct bio bio_clone_bioset(struct bio , gfp_t, struct bio_set *bs);
392		−extern struct bio bio_clone_bioset_partial(struct bio , gfp_t,
393		− struct bio_set *, int, int);
394	387
395	388	extern struct bio_set *fs_bio_set;
396	389

include/linux/percpu-refcount.h+1−0

@@ -99,6 +99,7 @@ int __must_check percpu_ref_init(struct percpu_ref *ref, · 98 unmodified lines
99	99	void percpu_ref_exit(struct percpu_ref *ref);
100	100	void percpu_ref_switch_to_atomic(struct percpu_ref *ref,
101	101	percpu_ref_func_t *confirm_switch);
	102	+void percpu_ref_switch_to_atomic_sync(struct percpu_ref *ref);
102	103	void percpu_ref_switch_to_percpu(struct percpu_ref *ref);
103	104	void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
104	105	percpu_ref_func_t *confirm_kill);

include/uapi/linux/raid/md_p.h+41−4

@@ -242,10 +242,18 @@ struct mdp_superblock_1 { · 241 unmodified lines
242	242
243	243	__le32 chunksize; /* in 512byte sectors */
244	244	__le32 raid_disks;
245		− __le32 bitmap_offset; /* sectors after start of superblock that bitmap starts
246		− * NOTE: signed, so bitmap can be before superblock
247		− * only meaningful of feature_map[0] is set.
248		− */
	245	+ union {
	246	+ __le32 bitmap_offset; /* sectors after start of superblock that bitmap starts
	247	+ * NOTE: signed, so bitmap can be before superblock
	248	+ * only meaningful of feature_map[0] is set.
	249	+ */
	250	+
	251	+ /* only meaningful when feature_map[MD_FEATURE_PPL] is set */
	252	+ struct {
	253	+ __le16 offset; /* sectors from start of superblock that ppl starts (signed) */
	254	+ __le16 size; /* ppl size in sectors */
	255	+ } ppl;
	256	+ };
249	257
250	258	/* These are only valid with feature bit '4' */
251	259	__le32 new_level; /* new level we are reshaping to */
@@ -318,6 +326,7 @@ struct mdp_superblock_1 { · 66 unmodified lines
318	326	*/
319	327	#define MD_FEATURE_CLUSTERED 256 /* clustered MD */
320	328	#define MD_FEATURE_JOURNAL 512 /* support write cache */
	329	+#define MD_FEATURE_PPL 1024 /* support PPL */
321	330	#define MD_FEATURE_ALL (MD_FEATURE_BITMAP_OFFSET \
322	331	\|MD_FEATURE_RECOVERY_OFFSET \
323	332	\|MD_FEATURE_RESHAPE_ACTIVE \
@@ -328,6 +337,7 @@ struct mdp_superblock_1 { · 4 unmodified lines
328	337	\|MD_FEATURE_RECOVERY_BITMAP \
329	338	\|MD_FEATURE_CLUSTERED \
330	339	\|MD_FEATURE_JOURNAL \
	340	+ \|MD_FEATURE_PPL \
331	341	)
332	342
333	343	struct r5l_payload_header {
@@ -388,4 +398,31 @@ struct r5l_meta_block { · 54 unmodified lines
388	398
389	399	#define R5LOG_VERSION 0x1
390	400	#define R5LOG_MAGIC 0x6433c509
	401	+
	402	+struct ppl_header_entry {
	403	+ __le64 data_sector; /* raid sector of the new data */
	404	+ __le32 pp_size; /* length of partial parity */
	405	+ __le32 data_size; /* length of data */
	406	+ __le32 parity_disk; /* member disk containing parity */
	407	+ __le32 checksum; /* checksum of partial parity data for this
	408	+ * entry (~crc32c) */
	409	+} __attribute__ ((__packed__));
	410	+
	411	+#define PPL_HEADER_SIZE 4096
	412	+#define PPL_HDR_RESERVED 512
	413	+#define PPL_HDR_ENTRY_SPACE \
	414	+ (PPL_HEADER_SIZE - PPL_HDR_RESERVED - 4 * sizeof(__le32) - sizeof(__le64))
	415	+#define PPL_HDR_MAX_ENTRIES \
	416	+ (PPL_HDR_ENTRY_SPACE / sizeof(struct ppl_header_entry))
	417	+
	418	+struct ppl_header {
	419	+ __u8 reserved[PPL_HDR_RESERVED];/* reserved space, fill with 0xff */
	420	+ __le32 signature; /* signature (family number of volume) */
	421	+ __le32 padding; /* zero pad */
	422	+ __le64 generation; /* generation number of the header */
	423	+ __le32 entries_count; /* number of entries in entry array */
	424	+ __le32 checksum; /* checksum of the header (~crc32c) */
	425	+ struct ppl_header_entry entries[PPL_HDR_MAX_ENTRIES];
	426	+} __attribute__ ((__packed__));
	427	+
391	428	#endif

lib/percpu-refcount.c+17−0

@@ -260,6 +260,22 @@ void percpu_ref_switch_to_atomic(struct percpu_ref *ref, · 259 unmodified lines
260	260
261	261	spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
262	262	}
	263	+EXPORT_SYMBOL_GPL(percpu_ref_switch_to_atomic);
	264	+
	265	+/**
	266	+ * percpu_ref_switch_to_atomic_sync - switch a percpu_ref to atomic mode
	267	+ * @ref: percpu_ref to switch to atomic mode
	268	+ *
	269	+ * Schedule switching the ref to atomic mode, and wait for the
	270	+ * switch to complete. Caller must ensure that no other thread
	271	+ * will switch back to percpu mode.
	272	+ */
	273	+void percpu_ref_switch_to_atomic_sync(struct percpu_ref *ref)
	274	+{
	275	+ percpu_ref_switch_to_atomic(ref, NULL);
	276	+ wait_event(percpu_ref_switch_waitq, !ref->confirm_switch);
	277	+}
	278	+EXPORT_SYMBOL_GPL(percpu_ref_switch_to_atomic_sync);
263	279
264	280	/**
265	281	* percpu_ref_switch_to_percpu - switch a percpu_ref to percpu mode
@@ -290,6 +306,7 @@ void percpu_ref_switch_to_percpu(struct percpu_ref *ref) · 24 unmodified lines
290	306
291	307	spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
292	308	}
	309	+EXPORT_SYMBOL_GPL(percpu_ref_switch_to_percpu);
293	310
294	311	/**
295	312	* percpu_ref_kill_and_confirm - drop the initial ref and schedule confirmation