History

Merge tag 'for-5.6/drivers-2020-01-27' of git://git.kernel.dk/linux-block

Pull block driver updates from Jens Axboe: "Like the core side, not a lot of changes here, just two main items: - Series of patches (via Coly) with fixes for bcache (Coly, Christoph) - MD pull request from Song" * tag 'for-5.6/drivers-2020-01-27' of git://git.kernel.dk/linux-block: (31 commits) bcache: reap from tail of c->btree_cache in bch_mca_scan() bcache: reap c->btree_cache_freeable from the tail in bch_mca_scan() bcache: remove member accessed from struct btree bcache: print written and keys in trace_bcache_btree_write bcache: avoid unnecessary btree nodes flushing in btree_flush_write() bcache: add code comments for state->pool in __btree_sort() lib: crc64: include <linux/crc64.h> for 'crc64_be' bcache: use read_cache_page_gfp to read the superblock bcache: store a pointer to the on-disk sb in the cache and cached_dev structures bcache: return a pointer to the on-disk sb from read_super bcache: transfer the sb_page reference to register_{bdev,cache} bcache: fix use-after-free in register_bcache() bcache: properly initialize 'path' and 'err' in register_bcache() bcache: rework error unwinding in register_bcache bcache: use a separate data structure for the on-disk super block bcache: cached_dev_free needs to put the sb page md/raid1: introduce wait_for_serialization md/raid1: use bucket based mechanism for IO serialization md: introduce a new struct for IO serialization md: don't destroy serial_info_pool if serialize_policy is true ...

Linus Torvalds committed 6 years ago

Commit 22a8f392 parents 48b4b4f e3de04417 files · +571 −262Browse files

Unified · Split

17 files
  1. drivers/md/bcache/bcache.h +2 −0
  2. drivers/md/bcache/bset.c +5 −0
  3. drivers/md/bcache/btree.c +10 −14
  4. drivers/md/bcache/btree.h +0 −2
  5. drivers/md/bcache/journal.c +75 −5
  6. drivers/md/bcache/super.c +71 −65
  7. drivers/md/md-bitmap.c +12 −13
  8. drivers/md/md.c +199 −55
  9. drivers/md/md.h +27 −18
  10. drivers/md/raid1.c +61 −50
  11. drivers/md/raid5.c +7 −14
  12. include/linux/raid/pq.h +6 −1
  13. include/trace/events/bcache.h +2 −1
  14. include/uapi/linux/bcache.h +52 −0
  15. lib/crc64.c +1 −0
  16. lib/raid6/algos.c +40 −23
  17. lib/raid6/mktables.c +1 −1
drivers/md/bcache/bcache.h+2−0
@@ -301,6 +301,7 @@ struct cached_dev { · 300 unmodified lines
301 struct block_device *bdev;301 struct block_device *bdev;
302 302
303 struct cache_sb sb;303 struct cache_sb sb;
304 struct cache_sb_disk *sb_disk;
304 struct bio sb_bio;305 struct bio sb_bio;
305 struct bio_vec sb_bv[1];306 struct bio_vec sb_bv[1];
306 struct closure sb_write;307 struct closure sb_write;
@@ -403,6 +404,7 @@ enum alloc_reserve { · 96 unmodified lines
403struct cache {404struct cache {
404 struct cache_set *set;405 struct cache_set *set;
405 struct cache_sb sb;406 struct cache_sb sb;
407 struct cache_sb_disk *sb_disk;
406 struct bio sb_bio;408 struct bio sb_bio;
407 struct bio_vec sb_bv[1];409 struct bio_vec sb_bv[1];
408 410
drivers/md/bcache/bset.c+5−0
@@ -1257,6 +1257,11 @@ static void __btree_sort(struct btree_keys *b, struct btree_iter *iter, · 1256 unmodified lines
1257 * Our temporary buffer is the same size as the btree node's1257 * Our temporary buffer is the same size as the btree node's
1258 * buffer, we can just swap buffers instead of doing a big1258 * buffer, we can just swap buffers instead of doing a big
1259 * memcpy()1259 * memcpy()
1260 *
1261 * Don't worry event 'out' is allocated from mempool, it can
1262 * still be swapped here. Because state->pool is a page mempool
1263 * creaated by by mempool_init_page_pool(), which allocates
1264 * pages by alloc_pages() indeed.
1260 */1265 */
1261 1266
1262 out->magic = b->set->data->magic;1267 out->magic = b->set->data->magic;
drivers/md/bcache/btree.c+10−14
@@ -734,34 +734,32 @@ static unsigned long bch_mca_scan(struct shrinker *shrink, · 733 unmodified lines
734 734
735 i = 0;735 i = 0;
736 btree_cache_used = c->btree_cache_used;736 btree_cache_used = c->btree_cache_used;
737 list_for_each_entry_safe(b, t, &c->btree_cache_freeable, list) {737 list_for_each_entry_safe_reverse(b, t, &c->btree_cache_freeable, list) {
738 if (nr <= 0)738 if (nr <= 0)
739 goto out;739 goto out;
740 740
741 if (++i > 3 &&741 if (!mca_reap(b, 0, false)) {
742 !mca_reap(b, 0, false)) {
743 mca_data_free(b);742 mca_data_free(b);
744 rw_unlock(true, b);743 rw_unlock(true, b);
745 freed++;744 freed++;
746 }745 }
747 nr--;746 nr--;
747 i++;
748 }748 }
749 749
750 for (; (nr--) && i < btree_cache_used; i++) {750 list_for_each_entry_safe_reverse(b, t, &c->btree_cache, list) {
751 if (list_empty(&c->btree_cache))751 if (nr <= 0 || i >= btree_cache_used)
752 goto out;752 goto out;
753 753
754 b = list_first_entry(&c->btree_cache, struct btree, list);754 if (!mca_reap(b, 0, false)) {
755 list_rotate_left(&c->btree_cache);
756
757 if (!b->accessed &&
758 !mca_reap(b, 0, false)) {
759 mca_bucket_free(b);755 mca_bucket_free(b);
760 mca_data_free(b);756 mca_data_free(b);
761 rw_unlock(true, b);757 rw_unlock(true, b);
762 freed++;758 freed++;
763 } else759 }
764 b->accessed = 0;760
761 nr--;
762 i++;
765 }763 }
766out:764out:
767 mutex_unlock(&c->bucket_lock);765 mutex_unlock(&c->bucket_lock);
@@ -1069,7 +1067,6 @@ retry: · 301 unmodified lines
1069 BUG_ON(!b->written);1067 BUG_ON(!b->written);
1070 1068
1071 b->parent = parent;1069 b->parent = parent;
1072 b->accessed = 1;
1073 1070
1074 for (; i <= b->keys.nsets && b->keys.set[i].size; i++) {1071 for (; i <= b->keys.nsets && b->keys.set[i].size; i++) {
1075 prefetch(b->keys.set[i].tree);1072 prefetch(b->keys.set[i].tree);
@@ -1160,7 +1157,6 @@ retry: · 84 unmodified lines
1160 goto retry;1157 goto retry;
1161 }1158 }
1162 1159
1163 b->accessed = 1;
1164 b->parent = parent;1160 b->parent = parent;
1165 bch_bset_init_next(&b->keys, b->keys.set->data, bset_magic(&b->c->sb));1161 bch_bset_init_next(&b->keys, b->keys.set->data, bset_magic(&b->c->sb));
1166 1162
drivers/md/bcache/btree.h+0−2
@@ -121,8 +121,6 @@ struct btree { · 120 unmodified lines
121 /* Key/pointer for this btree node */121 /* Key/pointer for this btree node */
122 BKEY_PADDED(key);122 BKEY_PADDED(key);
123 123
124 /* Single bit - set when accessed, cleared by shrinker */
125 unsigned long accessed;
126 unsigned long seq;124 unsigned long seq;
127 struct rw_semaphore lock;125 struct rw_semaphore lock;
128 struct cache_set *c;126 struct cache_set *c;
drivers/md/bcache/journal.c+75−5
@@ -417,10 +417,14 @@ err: · 416 unmodified lines
417 417
418/* Journalling */418/* Journalling */
419 419
420#define nr_to_fifo_front(p, front_p, mask) (((p) - (front_p)) & (mask))
421
420static void btree_flush_write(struct cache_set *c)422static void btree_flush_write(struct cache_set *c)
421{423{
422 struct btree *b, *t, *btree_nodes[BTREE_FLUSH_NR];424 struct btree *b, *t, *btree_nodes[BTREE_FLUSH_NR];
423 unsigned int i, n;425 unsigned int i, nr, ref_nr;
426 atomic_t *fifo_front_p, *now_fifo_front_p;
427 size_t mask;
424 428
425 if (c->journal.btree_flushing)429 if (c->journal.btree_flushing)
426 return;430 return;
@@ -433,12 +437,50 @@ static void btree_flush_write(struct cache_set *c) · 6 unmodified lines
433 c->journal.btree_flushing = true;437 c->journal.btree_flushing = true;
434 spin_unlock(&c->journal.flush_write_lock);438 spin_unlock(&c->journal.flush_write_lock);
435 439
440 /* get the oldest journal entry and check its refcount */
441 spin_lock(&c->journal.lock);
442 fifo_front_p = &fifo_front(&c->journal.pin);
443 ref_nr = atomic_read(fifo_front_p);
444 if (ref_nr <= 0) {
445 /*
446 * do nothing if no btree node references
447 * the oldest journal entry
448 */
449 spin_unlock(&c->journal.lock);
450 goto out;
451 }
452 spin_unlock(&c->journal.lock);
453
454 mask = c->journal.pin.mask;
455 nr = 0;
436 atomic_long_inc(&c->flush_write);456 atomic_long_inc(&c->flush_write);
437 memset(btree_nodes, 0, sizeof(btree_nodes));457 memset(btree_nodes, 0, sizeof(btree_nodes));
438 n = 0;
439 458
440 mutex_lock(&c->bucket_lock);459 mutex_lock(&c->bucket_lock);
441 list_for_each_entry_safe_reverse(b, t, &c->btree_cache, list) {460 list_for_each_entry_safe_reverse(b, t, &c->btree_cache, list) {
461 /*
462 * It is safe to get now_fifo_front_p without holding
463 * c->journal.lock here, because we don't need to know
464 * the exactly accurate value, just check whether the
465 * front pointer of c->journal.pin is changed.
466 */
467 now_fifo_front_p = &fifo_front(&c->journal.pin);
468 /*
469 * If the oldest journal entry is reclaimed and front
470 * pointer of c->journal.pin changes, it is unnecessary
471 * to scan c->btree_cache anymore, just quit the loop and
472 * flush out what we have already.
473 */
474 if (now_fifo_front_p != fifo_front_p)
475 break;
476 /*
477 * quit this loop if all matching btree nodes are
478 * scanned and record in btree_nodes[] already.
479 */
480 ref_nr = atomic_read(fifo_front_p);
481 if (nr >= ref_nr)
482 break;
483
442 if (btree_node_journal_flush(b))484 if (btree_node_journal_flush(b))
443 pr_err("BUG: flush_write bit should not be set here!");485 pr_err("BUG: flush_write bit should not be set here!");
444 486
@@ -454,17 +496,44 @@ static void btree_flush_write(struct cache_set *c) · 9 unmodified lines
454 continue;496 continue;
455 }497 }
456 498
499 /*
500 * Only select the btree node which exactly references
501 * the oldest journal entry.
502 *
503 * If the journal entry pointed by fifo_front_p is
504 * reclaimed in parallel, don't worry:
505 * - the list_for_each_xxx loop will quit when checking
506 * next now_fifo_front_p.
507 * - If there are matched nodes recorded in btree_nodes[],
508 * they are clean now (this is why and how the oldest
509 * journal entry can be reclaimed). These selected nodes
510 * will be ignored and skipped in the folowing for-loop.
511 */
512 if (nr_to_fifo_front(btree_current_write(b)->journal,
513 fifo_front_p,
514 mask) != 0) {
515 mutex_unlock(&b->write_lock);
516 continue;
517 }
518
457 set_btree_node_journal_flush(b);519 set_btree_node_journal_flush(b);
458 520
459 mutex_unlock(&b->write_lock);521 mutex_unlock(&b->write_lock);
460 522
461 btree_nodes[n++] = b;523 btree_nodes[nr++] = b;
462 if (n == BTREE_FLUSH_NR)524 /*
525 * To avoid holding c->bucket_lock too long time,
526 * only scan for BTREE_FLUSH_NR matched btree nodes
527 * at most. If there are more btree nodes reference
528 * the oldest journal entry, try to flush them next
529 * time when btree_flush_write() is called.
530 */
531 if (nr == BTREE_FLUSH_NR)
463 break;532 break;
464 }533 }
465 mutex_unlock(&c->bucket_lock);534 mutex_unlock(&c->bucket_lock);
466 535
467 for (i = 0; i < n; i++) {536 for (i = 0; i < nr; i++) {
468 b = btree_nodes[i];537 b = btree_nodes[i];
469 if (!b) {538 if (!b) {
470 pr_err("BUG: btree_nodes[%d] is NULL", i);539 pr_err("BUG: btree_nodes[%d] is NULL", i);
@@ -497,6 +566,7 @@ static void btree_flush_write(struct cache_set *c) · 26 unmodified lines
497 mutex_unlock(&b->write_lock);566 mutex_unlock(&b->write_lock);
498 }567 }
499 568
569out:
500 spin_lock(&c->journal.flush_write_lock);570 spin_lock(&c->journal.flush_write_lock);
501 c->journal.btree_flushing = false;571 c->journal.btree_flushing = false;
502 spin_unlock(&c->journal.flush_write_lock);572 spin_unlock(&c->journal.flush_write_lock);
drivers/md/bcache/super.c+71−65
@@ -15,7 +15,6 @@ · 14 unmodified lines
15#include "writeback.h"15#include "writeback.h"
16 16
17#include <linux/blkdev.h>17#include <linux/blkdev.h>
18#include <linux/buffer_head.h>
19#include <linux/debugfs.h>18#include <linux/debugfs.h>
20#include <linux/genhd.h>19#include <linux/genhd.h>
21#include <linux/idr.h>20#include <linux/idr.h>
@@ -60,17 +59,18 @@ struct workqueue_struct *bch_journal_wq; · 38 unmodified lines
60/* Superblock */59/* Superblock */
61 60
62static const char *read_super(struct cache_sb *sb, struct block_device *bdev,61static const char *read_super(struct cache_sb *sb, struct block_device *bdev,
63 struct page **res)62 struct cache_sb_disk **res)
64{63{
65 const char *err;64 const char *err;
66 struct cache_sb *s;65 struct cache_sb_disk *s;
67 struct buffer_head *bh = __bread(bdev, 1, SB_SIZE);66 struct page *page;
68 unsigned int i;67 unsigned int i;
69 68
70 if (!bh)69 page = read_cache_page_gfp(bdev->bd_inode->i_mapping,
70 SB_OFFSET >> PAGE_SHIFT, GFP_KERNEL);
71 if (IS_ERR(page))
71 return "IO error";72 return "IO error";
72 73 s = page_address(page) + offset_in_page(SB_OFFSET);
73 s = (struct cache_sb *) bh->b_data;
74 74
75 sb->offset = le64_to_cpu(s->offset);75 sb->offset = le64_to_cpu(s->offset);
76 sb->version = le64_to_cpu(s->version);76 sb->version = le64_to_cpu(s->version);
@@ -188,12 +188,10 @@ static const char *read_super(struct cache_sb *sb, struct block_device *bdev, · 111 unmodified lines
188 }188 }
189 189
190 sb->last_mount = (u32)ktime_get_real_seconds();190 sb->last_mount = (u32)ktime_get_real_seconds();
191 err = NULL;191 *res = s;
192 192 return NULL;
193 get_page(bh->b_page);
194 *res = bh->b_page;
195err:193err:
196 put_bh(bh);194 put_page(page);
197 return err;195 return err;
198}196}
199 197
@@ -207,15 +205,15 @@ static void write_bdev_super_endio(struct bio *bio) · 7 unmodified lines
207 closure_put(&dc->sb_write);205 closure_put(&dc->sb_write);
208}206}
209 207
210static void __write_super(struct cache_sb *sb, struct bio *bio)208static void __write_super(struct cache_sb *sb, struct cache_sb_disk *out,
209 struct bio *bio)
211{210{
212 struct cache_sb *out = page_address(bio_first_page_all(bio));
213 unsigned int i;211 unsigned int i;
214 212
213 bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_META;
215 bio->bi_iter.bi_sector = SB_SECTOR;214 bio->bi_iter.bi_sector = SB_SECTOR;
216 bio->bi_iter.bi_size = SB_SIZE;215 __bio_add_page(bio, virt_to_page(out), SB_SIZE,
217 bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_SYNC|REQ_META);216 offset_in_page(out));
218 bch_bio_map(bio, NULL);
219 217
220 out->offset = cpu_to_le64(sb->offset);218 out->offset = cpu_to_le64(sb->offset);
221 out->version = cpu_to_le64(sb->version);219 out->version = cpu_to_le64(sb->version);
@@ -257,14 +255,14 @@ void bch_write_bdev_super(struct cached_dev *dc, struct closure *parent) · 35 unmodified lines
257 down(&dc->sb_write_mutex);255 down(&dc->sb_write_mutex);
258 closure_init(cl, parent);256 closure_init(cl, parent);
259 257
260 bio_reset(bio);258 bio_init(bio, dc->sb_bv, 1);
261 bio_set_dev(bio, dc->bdev);259 bio_set_dev(bio, dc->bdev);
262 bio->bi_end_io = write_bdev_super_endio;260 bio->bi_end_io = write_bdev_super_endio;
263 bio->bi_private = dc;261 bio->bi_private = dc;
264 262
265 closure_get(cl);263 closure_get(cl);
266 /* I/O request sent to backing device */264 /* I/O request sent to backing device */
267 __write_super(&dc->sb, bio);265 __write_super(&dc->sb, dc->sb_disk, bio);
268 266
269 closure_return_with_destructor(cl, bch_write_bdev_super_unlock);267 closure_return_with_destructor(cl, bch_write_bdev_super_unlock);
270}268}
@@ -306,13 +304,13 @@ void bcache_write_super(struct cache_set *c) · 35 unmodified lines
306 304
307 SET_CACHE_SYNC(&ca->sb, CACHE_SYNC(&c->sb));305 SET_CACHE_SYNC(&ca->sb, CACHE_SYNC(&c->sb));
308 306
309 bio_reset(bio);307 bio_init(bio, ca->sb_bv, 1);
310 bio_set_dev(bio, ca->bdev);308 bio_set_dev(bio, ca->bdev);
311 bio->bi_end_io = write_super_endio;309 bio->bi_end_io = write_super_endio;
312 bio->bi_private = ca;310 bio->bi_private = ca;
313 311
314 closure_get(cl);312 closure_get(cl);
315 __write_super(&ca->sb, bio);313 __write_super(&ca->sb, ca->sb_disk, bio);
316 }314 }
317 315
318 closure_return_with_destructor(cl, bcache_write_super_unlock);316 closure_return_with_destructor(cl, bcache_write_super_unlock);
@@ -1275,6 +1273,9 @@ static void cached_dev_free(struct closure *cl) · 956 unmodified lines
1275 1273
1276 mutex_unlock(&bch_register_lock);1274 mutex_unlock(&bch_register_lock);
1277 1275
1276 if (dc->sb_disk)
1277 put_page(virt_to_page(dc->sb_disk));
1278
1278 if (!IS_ERR_OR_NULL(dc->bdev))1279 if (!IS_ERR_OR_NULL(dc->bdev))
1279 blkdev_put(dc->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);1280 blkdev_put(dc->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
1280 1281
@@ -1350,7 +1351,7 @@ static int cached_dev_init(struct cached_dev *dc, unsigned int block_size) · 69 unmodified lines
1350 1351
1351/* Cached device - bcache superblock */1352/* Cached device - bcache superblock */
1352 1353
1353static int register_bdev(struct cache_sb *sb, struct page *sb_page,1354static int register_bdev(struct cache_sb *sb, struct cache_sb_disk *sb_disk,
1354 struct block_device *bdev,1355 struct block_device *bdev,
1355 struct cached_dev *dc)1356 struct cached_dev *dc)
1356{1357{
@@ -1362,11 +1363,7 @@ static int register_bdev(struct cache_sb *sb, struct page *sb_page, · 5 unmodified lines
1362 memcpy(&dc->sb, sb, sizeof(struct cache_sb));1363 memcpy(&dc->sb, sb, sizeof(struct cache_sb));
1363 dc->bdev = bdev;1364 dc->bdev = bdev;
1364 dc->bdev->bd_holder = dc;1365 dc->bdev->bd_holder = dc;
1365 1366 dc->sb_disk = sb_disk;
1366 bio_init(&dc->sb_bio, dc->sb_bio.bi_inline_vecs, 1);
1367 bio_first_bvec_all(&dc->sb_bio)->bv_page = sb_page;
1368 get_page(sb_page);
1369
1370 1367
1371 if (cached_dev_init(dc, sb->block_size << 9))1368 if (cached_dev_init(dc, sb->block_size << 9))
1372 goto err;1369 goto err;
@@ -2136,8 +2133,8 @@ void bch_cache_release(struct kobject *kobj) · 763 unmodified lines
2136 for (i = 0; i < RESERVE_NR; i++)2133 for (i = 0; i < RESERVE_NR; i++)
2137 free_fifo(&ca->free[i]);2134 free_fifo(&ca->free[i]);
2138 2135
2139 if (ca->sb_bio.bi_inline_vecs[0].bv_page)2136 if (ca->sb_disk)
2140 put_page(bio_first_page_all(&ca->sb_bio));2137 put_page(virt_to_page(ca->sb_disk));
2141 2138
2142 if (!IS_ERR_OR_NULL(ca->bdev))2139 if (!IS_ERR_OR_NULL(ca->bdev))
2143 blkdev_put(ca->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);2140 blkdev_put(ca->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
@@ -2259,7 +2256,7 @@ err_free: · 115 unmodified lines
2259 return ret;2256 return ret;
2260}2257}
2261 2258
2262static int register_cache(struct cache_sb *sb, struct page *sb_page,2259static int register_cache(struct cache_sb *sb, struct cache_sb_disk *sb_disk,
2263 struct block_device *bdev, struct cache *ca)2260 struct block_device *bdev, struct cache *ca)
2264{2261{
2265 const char *err = NULL; /* must be set for any error case */2262 const char *err = NULL; /* must be set for any error case */
@@ -2269,10 +2266,7 @@ static int register_cache(struct cache_sb *sb, struct page *sb_page, · 3 unmodified lines
2269 memcpy(&ca->sb, sb, sizeof(struct cache_sb));2266 memcpy(&ca->sb, sb, sizeof(struct cache_sb));
2270 ca->bdev = bdev;2267 ca->bdev = bdev;
2271 ca->bdev->bd_holder = ca;2268 ca->bdev->bd_holder = ca;
2272 2269 ca->sb_disk = sb_disk;
2273 bio_init(&ca->sb_bio, ca->sb_bio.bi_inline_vecs, 1);
2274 bio_first_bvec_all(&ca->sb_bio)->bv_page = sb_page;
2275 get_page(sb_page);
2276 2270
2277 if (blk_queue_discard(bdev_get_queue(bdev)))2271 if (blk_queue_discard(bdev_get_queue(bdev)))
2278 ca->discard = CACHE_DISCARD(&ca->sb);2272 ca->discard = CACHE_DISCARD(&ca->sb);
@@ -2372,29 +2366,35 @@ static bool bch_is_open(struct block_device *bdev) · 93 unmodified lines
2372static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr,2366static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr,
2373 const char *buffer, size_t size)2367 const char *buffer, size_t size)
2374{2368{
2375 ssize_t ret = -EINVAL;2369 const char *err;
2376 const char *err = "cannot allocate memory";
2377 char *path = NULL;2370 char *path = NULL;
2378 struct cache_sb *sb = NULL;2371 struct cache_sb *sb;
2379 struct block_device *bdev = NULL;2372 struct cache_sb_disk *sb_disk;
2380 struct page *sb_page = NULL;2373 struct block_device *bdev;
2374 ssize_t ret;
2381 2375
2376 ret = -EBUSY;
2377 err = "failed to reference bcache module";
2382 if (!try_module_get(THIS_MODULE))2378 if (!try_module_get(THIS_MODULE))
2383 return -EBUSY;2379 goto out;
2384 2380
2385 /* For latest state of bcache_is_reboot */2381 /* For latest state of bcache_is_reboot */
2386 smp_mb();2382 smp_mb();
2383 err = "bcache is in reboot";
2387 if (bcache_is_reboot)2384 if (bcache_is_reboot)
2388 return -EBUSY;2385 goto out_module_put;
2389 2386
2387 ret = -ENOMEM;
2388 err = "cannot allocate memory";
2390 path = kstrndup(buffer, size, GFP_KERNEL);2389 path = kstrndup(buffer, size, GFP_KERNEL);
2391 if (!path)2390 if (!path)
2392 goto err;2391 goto out_module_put;
2393 2392
2394 sb = kmalloc(sizeof(struct cache_sb), GFP_KERNEL);2393 sb = kmalloc(sizeof(struct cache_sb), GFP_KERNEL);
2395 if (!sb)2394 if (!sb)
2396 goto err;2395 goto out_free_path;
2397 2396
2397 ret = -EINVAL;
2398 err = "failed to open device";2398 err = "failed to open device";
2399 bdev = blkdev_get_by_path(strim(path),2399 bdev = blkdev_get_by_path(strim(path),
2400 FMODE_READ|FMODE_WRITE|FMODE_EXCL,2400 FMODE_READ|FMODE_WRITE|FMODE_EXCL,
@@ -2411,57 +2411,63 @@ static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr, · 10 unmodified lines
2411 if (!IS_ERR(bdev))2411 if (!IS_ERR(bdev))
2412 bdput(bdev);2412 bdput(bdev);
2413 if (attr == &ksysfs_register_quiet)2413 if (attr == &ksysfs_register_quiet)
2414 goto quiet_out;2414 goto done;
2415 }2415 }
2416 goto err;2416 goto out_free_sb;
2417 }2417 }
2418 2418
2419 err = "failed to set blocksize";2419 err = "failed to set blocksize";
2420 if (set_blocksize(bdev, 4096))2420 if (set_blocksize(bdev, 4096))
2421 goto err_close;2421 goto out_blkdev_put;
2422 2422
2423 err = read_super(sb, bdev, &sb_page);2423 err = read_super(sb, bdev, &sb_disk);
2424 if (err)2424 if (err)
2425 goto err_close;2425 goto out_blkdev_put;
2426 2426
2427 err = "failed to register device";2427 err = "failed to register device";
2428 if (SB_IS_BDEV(sb)) {2428 if (SB_IS_BDEV(sb)) {
2429 struct cached_dev *dc = kzalloc(sizeof(*dc), GFP_KERNEL);2429 struct cached_dev *dc = kzalloc(sizeof(*dc), GFP_KERNEL);
2430 2430
2431 if (!dc)2431 if (!dc)
2432 goto err_close;2432 goto out_put_sb_page;
2433 2433
2434 mutex_lock(&bch_register_lock);2434 mutex_lock(&bch_register_lock);
2435 ret = register_bdev(sb, sb_page, bdev, dc);2435 ret = register_bdev(sb, sb_disk, bdev, dc);
2436 mutex_unlock(&bch_register_lock);2436 mutex_unlock(&bch_register_lock);
2437 /* blkdev_put() will be called in cached_dev_free() */2437 /* blkdev_put() will be called in cached_dev_free() */
2438 if (ret < 0)2438 if (ret < 0)
2439 goto err;2439 goto out_free_sb;
2440 } else {2440 } else {
2441 struct cache *ca = kzalloc(sizeof(*ca), GFP_KERNEL);2441 struct cache *ca = kzalloc(sizeof(*ca), GFP_KERNEL);
2442 2442
2443 if (!ca)2443 if (!ca)
2444 goto err_close;2444 goto out_put_sb_page;
2445 2445
2446 /* blkdev_put() will be called in bch_cache_release() */2446 /* blkdev_put() will be called in bch_cache_release() */
2447 if (register_cache(sb, sb_page, bdev, ca) != 0)2447 if (register_cache(sb, sb_disk, bdev, ca) != 0)
2448 goto err;2448 goto out_free_sb;
2449 }2449 }
2450quiet_out:2450
2451 ret = size;2451done:
2452out:
2453 if (sb_page)
2454 put_page(sb_page);
2455 kfree(sb);2452 kfree(sb);
2456 kfree(path);2453 kfree(path);
2457 module_put(THIS_MODULE);2454 module_put(THIS_MODULE);
2458 return ret;2455 return size;
2459 2456
2460err_close:2457out_put_sb_page:
2461 blkdev_put(bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);2458 put_page(virt_to_page(sb_disk));
2462err:2459out_blkdev_put:
2463 pr_info("error %s: %s", path, err);2460 blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
2464 goto out;2461out_free_sb:
2462 kfree(sb);
2463out_free_path:
2464 kfree(path);
2465 path = NULL;
2466out_module_put:
2467 module_put(THIS_MODULE);
2468out:
2469 pr_info("error %s: %s", path?path:"", err);
2470 return ret;
2465}2471}
2466 2472
2467 2473
drivers/md/md-bitmap.c+12−13
@@ -1019,8 +1019,6 @@ void md_bitmap_unplug(struct bitmap *bitmap) · 1018 unmodified lines
1019 /* look at each page to see if there are any set bits that need to be1019 /* look at each page to see if there are any set bits that need to be
1020 * flushed out to disk */1020 * flushed out to disk */
1021 for (i = 0; i < bitmap->storage.file_pages; i++) {1021 for (i = 0; i < bitmap->storage.file_pages; i++) {
1022 if (!bitmap->storage.filemap)
1023 return;
1024 dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);1022 dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
1025 need_write = test_and_clear_page_attr(bitmap, i,1023 need_write = test_and_clear_page_attr(bitmap, i,
1026 BITMAP_PAGE_NEEDWRITE);1024 BITMAP_PAGE_NEEDWRITE);
@@ -1338,7 +1336,8 @@ void md_bitmap_daemon_work(struct mddev *mddev) · 311 unmodified lines
1338 BITMAP_PAGE_DIRTY))1336 BITMAP_PAGE_DIRTY))
1339 /* bitmap_unplug will handle the rest */1337 /* bitmap_unplug will handle the rest */
1340 break;1338 break;
1341 if (test_and_clear_page_attr(bitmap, j,1339 if (bitmap->storage.filemap &&
1340 test_and_clear_page_attr(bitmap, j,
1342 BITMAP_PAGE_NEEDWRITE)) {1341 BITMAP_PAGE_NEEDWRITE)) {
1343 write_page(bitmap, bitmap->storage.filemap[j], 0);1342 write_page(bitmap, bitmap->storage.filemap[j], 0);
1344 }1343 }
@@ -1790,8 +1789,8 @@ void md_bitmap_destroy(struct mddev *mddev) · 445 unmodified lines
1790 return;1789 return;
1791 1790
1792 md_bitmap_wait_behind_writes(mddev);1791 md_bitmap_wait_behind_writes(mddev);
1793 mempool_destroy(mddev->wb_info_pool);1792 if (!mddev->serialize_policy)
1794 mddev->wb_info_pool = NULL;1793 mddev_destroy_serial_pool(mddev, NULL, true);
1795 1794
1796 mutex_lock(&mddev->bitmap_info.mutex);1795 mutex_lock(&mddev->bitmap_info.mutex);
1797 spin_lock(&mddev->lock);1796 spin_lock(&mddev->lock);
@@ -1908,7 +1907,7 @@ int md_bitmap_load(struct mddev *mddev) · 110 unmodified lines
1908 goto out;1907 goto out;
1909 1908
1910 rdev_for_each(rdev, mddev)1909 rdev_for_each(rdev, mddev)
1911 mddev_create_wb_pool(mddev, rdev, true);1910 mddev_create_serial_pool(mddev, rdev, true);
1912 1911
1913 if (mddev_is_clustered(mddev))1912 if (mddev_is_clustered(mddev))
1914 md_cluster_ops->load_bitmaps(mddev, mddev->bitmap_info.nodes);1913 md_cluster_ops->load_bitmaps(mddev, mddev->bitmap_info.nodes);
@@ -2475,16 +2474,16 @@ backlog_store(struct mddev *mddev, const char *buf, size_t len) · 560 unmodified lines
2475 if (backlog > COUNTER_MAX)2474 if (backlog > COUNTER_MAX)
2476 return -EINVAL;2475 return -EINVAL;
2477 mddev->bitmap_info.max_write_behind = backlog;2476 mddev->bitmap_info.max_write_behind = backlog;
2478 if (!backlog && mddev->wb_info_pool) {2477 if (!backlog && mddev->serial_info_pool) {
2479 /* wb_info_pool is not needed if backlog is zero */2478 /* serial_info_pool is not needed if backlog is zero */
2480 mempool_destroy(mddev->wb_info_pool);2479 if (!mddev->serialize_policy)
2481 mddev->wb_info_pool = NULL;2480 mddev_destroy_serial_pool(mddev, NULL, false);
2482 } else if (backlog && !mddev->wb_info_pool) {2481 } else if (backlog && !mddev->serial_info_pool) {
2483 /* wb_info_pool is needed since backlog is not zero */2482 /* serial_info_pool is needed since backlog is not zero */
2484 struct md_rdev *rdev;2483 struct md_rdev *rdev;
2485 2484
2486 rdev_for_each(rdev, mddev)2485 rdev_for_each(rdev, mddev)
2487 mddev_create_wb_pool(mddev, rdev, false);2486 mddev_create_serial_pool(mddev, rdev, false);
2488 }2487 }
2489 if (old_mwb != backlog)2488 if (old_mwb != backlog)
2490 md_bitmap_update_sb(mddev->bitmap);2489 md_bitmap_update_sb(mddev->bitmap);
drivers/md/md.c+199−55
@@ -125,74 +125,165 @@ static inline int speed_max(struct mddev *mddev) · 124 unmodified lines
125 mddev->sync_speed_max : sysctl_speed_limit_max;125 mddev->sync_speed_max : sysctl_speed_limit_max;
126}126}
127 127
128static int rdev_init_wb(struct md_rdev *rdev)128static void rdev_uninit_serial(struct md_rdev *rdev)
129{129{
130 if (rdev->bdev->bd_queue->nr_hw_queues == 1)130 if (!test_and_clear_bit(CollisionCheck, &rdev->flags))
131 return;
132
133 kvfree(rdev->serial);
134 rdev->serial = NULL;
135}
136
137static void rdevs_uninit_serial(struct mddev *mddev)
138{
139 struct md_rdev *rdev;
140
141 rdev_for_each(rdev, mddev)
142 rdev_uninit_serial(rdev);
143}
144
145static int rdev_init_serial(struct md_rdev *rdev)
146{
147 /* serial_nums equals with BARRIER_BUCKETS_NR */
148 int i, serial_nums = 1 << ((PAGE_SHIFT - ilog2(sizeof(atomic_t))));
149 struct serial_in_rdev *serial = NULL;
150
151 if (test_bit(CollisionCheck, &rdev->flags))
131 return 0;152 return 0;
132 153
133 spin_lock_init(&rdev->wb_list_lock);154 serial = kvmalloc(sizeof(struct serial_in_rdev) * serial_nums,
134 INIT_LIST_HEAD(&rdev->wb_list);155 GFP_KERNEL);
135 init_waitqueue_head(&rdev->wb_io_wait);156 if (!serial)
136 set_bit(WBCollisionCheck, &rdev->flags);157 return -ENOMEM;
137 158
138 return 1;159 for (i = 0; i < serial_nums; i++) {
160 struct serial_in_rdev *serial_tmp = &serial[i];
161
162 spin_lock_init(&serial_tmp->serial_lock);
163 serial_tmp->serial_rb = RB_ROOT_CACHED;
164 init_waitqueue_head(&serial_tmp->serial_io_wait);
165 }
166
167 rdev->serial = serial;
168 set_bit(CollisionCheck, &rdev->flags);
169
170 return 0;
171}
172
173static int rdevs_init_serial(struct mddev *mddev)
174{
175 struct md_rdev *rdev;
176 int ret = 0;
177
178 rdev_for_each(rdev, mddev) {
179 ret = rdev_init_serial(rdev);
180 if (ret)
181 break;
182 }
183
184 /* Free all resources if pool is not existed */
185 if (ret && !mddev->serial_info_pool)
186 rdevs_uninit_serial(mddev);
187
188 return ret;
139}189}
140 190
141/*191/*
142 * Create wb_info_pool if rdev is the first multi-queue device flaged192 * rdev needs to enable serial stuffs if it meets the conditions:
143 * with writemostly, also write-behind mode is enabled.193 * 1. it is multi-queue device flaged with writemostly.
194 * 2. the write-behind mode is enabled.
144 */195 */
145void mddev_create_wb_pool(struct mddev *mddev, struct md_rdev *rdev,196static int rdev_need_serial(struct md_rdev *rdev)
146 bool is_suspend)
147{197{
148 if (mddev->bitmap_info.max_write_behind == 0)198 return (rdev && rdev->mddev->bitmap_info.max_write_behind > 0 &&
149 return;199 rdev->bdev->bd_queue->nr_hw_queues != 1 &&
200 test_bit(WriteMostly, &rdev->flags));
201}
202
203/*
204 * Init resource for rdev(s), then create serial_info_pool if:
205 * 1. rdev is the first device which return true from rdev_enable_serial.
206 * 2. rdev is NULL, means we want to enable serialization for all rdevs.
207 */
208void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev,
209 bool is_suspend)
210{
211 int ret = 0;
150 212
151 if (!test_bit(WriteMostly, &rdev->flags) || !rdev_init_wb(rdev))213 if (rdev && !rdev_need_serial(rdev) &&
214 !test_bit(CollisionCheck, &rdev->flags))
152 return;215 return;
153 216
154 if (mddev->wb_info_pool == NULL) {217 if (!is_suspend)
218 mddev_suspend(mddev);
219
220 if (!rdev)
221 ret = rdevs_init_serial(mddev);
222 else
223 ret = rdev_init_serial(rdev);
224 if (ret)
225 goto abort;
226
227 if (mddev->serial_info_pool == NULL) {
155 unsigned int noio_flag;228 unsigned int noio_flag;
156 229
157 if (!is_suspend)
158 mddev_suspend(mddev);
159 noio_flag = memalloc_noio_save();230 noio_flag = memalloc_noio_save();
160 mddev->wb_info_pool = mempool_create_kmalloc_pool(NR_WB_INFOS,231 mddev->serial_info_pool =
161 sizeof(struct wb_info));232 mempool_create_kmalloc_pool(NR_SERIAL_INFOS,
233 sizeof(struct serial_info));
162 memalloc_noio_restore(noio_flag);234 memalloc_noio_restore(noio_flag);
163 if (!mddev->wb_info_pool)235 if (!mddev->serial_info_pool) {
164 pr_err("can't alloc memory pool for writemostly\n");236 rdevs_uninit_serial(mddev);
165 if (!is_suspend)237 pr_err("can't alloc memory pool for serialization\n");
166 mddev_resume(mddev);238 }
167 }239 }
240
241abort:
242 if (!is_suspend)
243 mddev_resume(mddev);
168}244}
169EXPORT_SYMBOL_GPL(mddev_create_wb_pool);
170 245
171/*246/*
172 * destroy wb_info_pool if rdev is the last device flaged with WBCollisionCheck.247 * Free resource from rdev(s), and destroy serial_info_pool under conditions:
248 * 1. rdev is the last device flaged with CollisionCheck.
249 * 2. when bitmap is destroyed while policy is not enabled.
250 * 3. for disable policy, the pool is destroyed only when no rdev needs it.
173 */251 */
174static void mddev_destroy_wb_pool(struct mddev *mddev, struct md_rdev *rdev)252void mddev_destroy_serial_pool(struct mddev *mddev, struct md_rdev *rdev,
253 bool is_suspend)
175{254{
176 if (!test_and_clear_bit(WBCollisionCheck, &rdev->flags))255 if (rdev && !test_bit(CollisionCheck, &rdev->flags))
177 return;256 return;
178 257
179 if (mddev->wb_info_pool) {258 if (mddev->serial_info_pool) {
180 struct md_rdev *temp;259 struct md_rdev *temp;
181 int num = 0;260 int num = 0; /* used to track if other rdevs need the pool */
182 261
183 /*262 if (!is_suspend)
184 * Check if other rdevs need wb_info_pool.263 mddev_suspend(mddev);
185 */264 rdev_for_each(temp, mddev) {
186 rdev_for_each(temp, mddev)265 if (!rdev) {
187 if (temp != rdev &&266 if (!mddev->serialize_policy ||
188 test_bit(WBCollisionCheck, &temp->flags))267 !rdev_need_serial(temp))
268 rdev_uninit_serial(temp);
269 else
270 num++;
271 } else if (temp != rdev &&
272 test_bit(CollisionCheck, &temp->flags))
189 num++;273 num++;
190 if (!num) {
191 mddev_suspend(rdev->mddev);
192 mempool_destroy(mddev->wb_info_pool);
193 mddev->wb_info_pool = NULL;
194 mddev_resume(rdev->mddev);
195 }274 }
275
276 if (rdev)
277 rdev_uninit_serial(rdev);
278
279 if (num)
280 pr_info("The mempool could be used by other devices\n");
281 else {
282 mempool_destroy(mddev->serial_info_pool);
283 mddev->serial_info_pool = NULL;
284 }
285 if (!is_suspend)
286 mddev_resume(mddev);
196 }287 }
197}288}
198 289
@@ -2337,7 +2428,7 @@ static int bind_rdev_to_array(struct md_rdev *rdev, struct mddev *mddev) · 2138 unmodified lines
2337 pr_debug("md: bind<%s>\n", b);2428 pr_debug("md: bind<%s>\n", b);
2338 2429
2339 if (mddev->raid_disks)2430 if (mddev->raid_disks)
2340 mddev_create_wb_pool(mddev, rdev, false);2431 mddev_create_serial_pool(mddev, rdev, false);
2341 2432
2342 if ((err = kobject_add(&rdev->kobj, &mddev->kobj, "dev-%s", b)))2433 if ((err = kobject_add(&rdev->kobj, &mddev->kobj, "dev-%s", b)))
2343 goto fail;2434 goto fail;
@@ -2375,7 +2466,7 @@ static void unbind_rdev_from_array(struct md_rdev *rdev) · 31 unmodified lines
2375 bd_unlink_disk_holder(rdev->bdev, rdev->mddev->gendisk);2466 bd_unlink_disk_holder(rdev->bdev, rdev->mddev->gendisk);
2376 list_del_rcu(&rdev->same_set);2467 list_del_rcu(&rdev->same_set);
2377 pr_debug("md: unbind<%s>\n", bdevname(rdev->bdev,b));2468 pr_debug("md: unbind<%s>\n", bdevname(rdev->bdev,b));
2378 mddev_destroy_wb_pool(rdev->mddev, rdev);2469 mddev_destroy_serial_pool(rdev->mddev, rdev, false);
2379 rdev->mddev = NULL;2470 rdev->mddev = NULL;
2380 sysfs_remove_link(&rdev->kobj, "block");2471 sysfs_remove_link(&rdev->kobj, "block");
2381 sysfs_put(rdev->sysfs_state);2472 sysfs_put(rdev->sysfs_state);
@@ -2888,10 +2979,10 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len) · 506 unmodified lines
2888 }2979 }
2889 } else if (cmd_match(buf, "writemostly")) {2980 } else if (cmd_match(buf, "writemostly")) {
2890 set_bit(WriteMostly, &rdev->flags);2981 set_bit(WriteMostly, &rdev->flags);
2891 mddev_create_wb_pool(rdev->mddev, rdev, false);2982 mddev_create_serial_pool(rdev->mddev, rdev, false);
2892 err = 0;2983 err = 0;
2893 } else if (cmd_match(buf, "-writemostly")) {2984 } else if (cmd_match(buf, "-writemostly")) {
2894 mddev_destroy_wb_pool(rdev->mddev, rdev);2985 mddev_destroy_serial_pool(rdev->mddev, rdev, false);
2895 clear_bit(WriteMostly, &rdev->flags);2986 clear_bit(WriteMostly, &rdev->flags);
2896 err = 0;2987 err = 0;
2897 } else if (cmd_match(buf, "blocked")) {2988 } else if (cmd_match(buf, "blocked")) {
@@ -5277,6 +5368,57 @@ static struct md_sysfs_entry md_fail_last_dev = · 2379 unmodified lines
5277__ATTR(fail_last_dev, S_IRUGO | S_IWUSR, fail_last_dev_show,5368__ATTR(fail_last_dev, S_IRUGO | S_IWUSR, fail_last_dev_show,
5278 fail_last_dev_store);5369 fail_last_dev_store);
5279 5370
5371static ssize_t serialize_policy_show(struct mddev *mddev, char *page)
5372{
5373 if (mddev->pers == NULL || (mddev->pers->level != 1))
5374 return sprintf(page, "n/a\n");
5375 else
5376 return sprintf(page, "%d\n", mddev->serialize_policy);
5377}
5378
5379/*
5380 * Setting serialize_policy to true to enforce write IO is not reordered
5381 * for raid1.
5382 */
5383static ssize_t
5384serialize_policy_store(struct mddev *mddev, const char *buf, size_t len)
5385{
5386 int err;
5387 bool value;
5388
5389 err = kstrtobool(buf, &value);
5390 if (err)
5391 return err;
5392
5393 if (value == mddev->serialize_policy)
5394 return len;
5395
5396 err = mddev_lock(mddev);
5397 if (err)
5398 return err;
5399 if (mddev->pers == NULL || (mddev->pers->level != 1)) {
5400 pr_err("md: serialize_policy is only effective for raid1\n");
5401 err = -EINVAL;
5402 goto unlock;
5403 }
5404
5405 mddev_suspend(mddev);
5406 if (value)
5407 mddev_create_serial_pool(mddev, NULL, true);
5408 else
5409 mddev_destroy_serial_pool(mddev, NULL, true);
5410 mddev->serialize_policy = value;
5411 mddev_resume(mddev);
5412unlock:
5413 mddev_unlock(mddev);
5414 return err ?: len;
5415}
5416
5417static struct md_sysfs_entry md_serialize_policy =
5418__ATTR(serialize_policy, S_IRUGO | S_IWUSR, serialize_policy_show,
5419 serialize_policy_store);
5420
5421
5280static struct attribute *md_default_attrs[] = {5422static struct attribute *md_default_attrs[] = {
5281 &md_level.attr,5423 &md_level.attr,
5282 &md_layout.attr,5424 &md_layout.attr,
@@ -5294,6 +5436,7 @@ static struct attribute *md_default_attrs[] = { · 11 unmodified lines
5294 &max_corr_read_errors.attr,5436 &max_corr_read_errors.attr,
5295 &md_consistency_policy.attr,5437 &md_consistency_policy.attr,
5296 &md_fail_last_dev.attr,5438 &md_fail_last_dev.attr,
5439 &md_serialize_policy.attr,
5297 NULL,5440 NULL,
5298};5441};
5299 5442
@@ -5769,18 +5912,18 @@ int md_run(struct mddev *mddev) · 469 unmodified lines
5769 goto bitmap_abort;5912 goto bitmap_abort;
5770 5913
5771 if (mddev->bitmap_info.max_write_behind > 0) {5914 if (mddev->bitmap_info.max_write_behind > 0) {
5772 bool creat_pool = false;5915 bool create_pool = false;
5773 5916
5774 rdev_for_each(rdev, mddev) {5917 rdev_for_each(rdev, mddev) {
5775 if (test_bit(WriteMostly, &rdev->flags) &&5918 if (test_bit(WriteMostly, &rdev->flags) &&
5776 rdev_init_wb(rdev))5919 rdev_init_serial(rdev))
5777 creat_pool = true;5920 create_pool = true;
5778 }5921 }
5779 if (creat_pool && mddev->wb_info_pool == NULL) {5922 if (create_pool && mddev->serial_info_pool == NULL) {
5780 mddev->wb_info_pool =5923 mddev->serial_info_pool =
5781 mempool_create_kmalloc_pool(NR_WB_INFOS,5924 mempool_create_kmalloc_pool(NR_SERIAL_INFOS,
5782 sizeof(struct wb_info));5925 sizeof(struct serial_info));
5783 if (!mddev->wb_info_pool) {5926 if (!mddev->serial_info_pool) {
5784 err = -ENOMEM;5927 err = -ENOMEM;
5785 goto bitmap_abort;5928 goto bitmap_abort;
5786 }5929 }
@@ -6025,8 +6168,9 @@ static void __md_stop_writes(struct mddev *mddev) · 238 unmodified lines
6025 mddev->in_sync = 1;6168 mddev->in_sync = 1;
6026 md_update_sb(mddev, 1);6169 md_update_sb(mddev, 1);
6027 }6170 }
6028 mempool_destroy(mddev->wb_info_pool);6171 /* disable policy to guarantee rdevs free resources for serialization */
6029 mddev->wb_info_pool = NULL;6172 mddev->serialize_policy = 0;
6173 mddev_destroy_serial_pool(mddev, NULL, true);
6030}6174}
6031 6175
6032void md_stop_writes(struct mddev *mddev)6176void md_stop_writes(struct mddev *mddev)
drivers/md/md.h+27−18
@@ -32,6 +32,16 @@ · 31 unmodified lines
32 * be retried.32 * be retried.
33 */33 */
34#define MD_FAILFAST (REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT)34#define MD_FAILFAST (REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT)
35
36/*
37 * The struct embedded in rdev is used to serialize IO.
38 */
39struct serial_in_rdev {
40 struct rb_root_cached serial_rb;
41 spinlock_t serial_lock;
42 wait_queue_head_t serial_io_wait;
43};
44
35/*45/*
36 * MD's 'extended' device46 * MD's 'extended' device
37 */47 */
@@ -110,12 +120,7 @@ struct md_rdev { · 72 unmodified lines
110 * in superblock.120 * in superblock.
111 */121 */
112 122
113 /*123 struct serial_in_rdev *serial; /* used for raid1 io serialization */
114 * The members for check collision of write behind IOs.
115 */
116 struct list_head wb_list;
117 spinlock_t wb_list_lock;
118 wait_queue_head_t wb_io_wait;
119 124
120 struct work_struct del_work; /* used for delayed sysfs removal */125 struct work_struct del_work; /* used for delayed sysfs removal */
121 126
@@ -201,9 +206,9 @@ enum flag_bits { · 79 unmodified lines
201 * it didn't fail, so don't use FailFast206 * it didn't fail, so don't use FailFast
202 * any more for metadata207 * any more for metadata
203 */208 */
204 WBCollisionCheck, /*209 CollisionCheck, /*
205 * multiqueue device should check if there210 * check if there is collision between raid1
206 * is collision between write behind bios.211 * serial bios.
207 */212 */
208};213};
209 214
@@ -263,12 +268,13 @@ enum mddev_sb_flags { · 53 unmodified lines
263 MD_SB_NEED_REWRITE, /* metadata write needs to be repeated */268 MD_SB_NEED_REWRITE, /* metadata write needs to be repeated */
264};269};
265 270
266#define NR_WB_INFOS 8271#define NR_SERIAL_INFOS 8
267/* record current range of write behind IOs */272/* record current range of serialize IOs */
268struct wb_info {273struct serial_info {
269 sector_t lo;274 struct rb_node node;
270 sector_t hi;275 sector_t start; /* start sector of rb node */
271 struct list_head list;276 sector_t last; /* end sector of rb node */
277 sector_t _subtree_last; /* highest sector in subtree of rb node */
272};278};
273 279
274struct mddev {280struct mddev {
@@ -487,13 +493,14 @@ struct mddev { · 212 unmodified lines
487 */493 */
488 struct work_struct flush_work;494 struct work_struct flush_work;
489 struct work_struct event_work; /* used by dm to report failure event */495 struct work_struct event_work; /* used by dm to report failure event */
490 mempool_t *wb_info_pool;496 mempool_t *serial_info_pool;
491 void (*sync_super)(struct mddev *mddev, struct md_rdev *rdev);497 void (*sync_super)(struct mddev *mddev, struct md_rdev *rdev);
492 struct md_cluster_info *cluster_info;498 struct md_cluster_info *cluster_info;
493 unsigned int good_device_nr; /* good device num within cluster raid */499 unsigned int good_device_nr; /* good device num within cluster raid */
494 500
495 bool has_superblocks:1;501 bool has_superblocks:1;
496 bool fail_last_dev:1;502 bool fail_last_dev:1;
503 bool serialize_policy:1;
497};504};
498 505
499enum recovery_flags {506enum recovery_flags {
@@ -737,8 +744,10 @@ extern struct bio *bio_alloc_mddev(gfp_t gfp_mask, int nr_iovecs, · 237 unmodified lines
737extern void md_reload_sb(struct mddev *mddev, int raid_disk);744extern void md_reload_sb(struct mddev *mddev, int raid_disk);
738extern void md_update_sb(struct mddev *mddev, int force);745extern void md_update_sb(struct mddev *mddev, int force);
739extern void md_kick_rdev_from_array(struct md_rdev * rdev);746extern void md_kick_rdev_from_array(struct md_rdev * rdev);
740extern void mddev_create_wb_pool(struct mddev *mddev, struct md_rdev *rdev,747extern void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev,
741 bool is_suspend);748 bool is_suspend);
749extern void mddev_destroy_serial_pool(struct mddev *mddev, struct md_rdev *rdev,
750 bool is_suspend);
742struct md_rdev *md_find_rdev_nr_rcu(struct mddev *mddev, int nr);751struct md_rdev *md_find_rdev_nr_rcu(struct mddev *mddev, int nr);
743struct md_rdev *md_find_rdev_rcu(struct mddev *mddev, dev_t dev);752struct md_rdev *md_find_rdev_rcu(struct mddev *mddev, dev_t dev);
744 753
drivers/md/raid1.c+61−50
@@ -29,6 +29,7 @@ · 28 unmodified lines
29#include <linux/module.h>29#include <linux/module.h>
30#include <linux/seq_file.h>30#include <linux/seq_file.h>
31#include <linux/ratelimit.h>31#include <linux/ratelimit.h>
32#include <linux/interval_tree_generic.h>
32 33
33#include <trace/events/block.h>34#include <trace/events/block.h>
34 35
@@ -50,55 +51,71 @@ static void lower_barrier(struct r1conf *conf, sector_t sector_nr); · 15 unmodified lines
50 51
51#include "raid1-10.c"52#include "raid1-10.c"
52 53
53static int check_and_add_wb(struct md_rdev *rdev, sector_t lo, sector_t hi)54#define START(node) ((node)->start)
55#define LAST(node) ((node)->last)
56INTERVAL_TREE_DEFINE(struct serial_info, node, sector_t, _subtree_last,
57 START, LAST, static inline, raid1_rb);
58
59static int check_and_add_serial(struct md_rdev *rdev, struct r1bio *r1_bio,
60 struct serial_info *si, int idx)
54{61{
55 struct wb_info *wi, *temp_wi;
56 unsigned long flags;62 unsigned long flags;
57 int ret = 0;63 int ret = 0;
58 struct mddev *mddev = rdev->mddev;64 sector_t lo = r1_bio->sector;
59 65 sector_t hi = lo + r1_bio->sectors;
60 wi = mempool_alloc(mddev->wb_info_pool, GFP_NOIO);66 struct serial_in_rdev *serial = &rdev->serial[idx];
61 67
62 spin_lock_irqsave(&rdev->wb_list_lock, flags);68 spin_lock_irqsave(&serial->serial_lock, flags);
63 list_for_each_entry(temp_wi, &rdev->wb_list, list) {69 /* collision happened */
64 /* collision happened */70 if (raid1_rb_iter_first(&serial->serial_rb, lo, hi))
65 if (hi > temp_wi->lo && lo < temp_wi->hi) {71 ret = -EBUSY;
66 ret = -EBUSY;72 else {
67 break;73 si->start = lo;
68 }74 si->last = hi;
75 raid1_rb_insert(si, &serial->serial_rb);
69 }76 }
70 77 spin_unlock_irqrestore(&serial->serial_lock, flags);
71 if (!ret) {
72 wi->lo = lo;
73 wi->hi = hi;
74 list_add(&wi->list, &rdev->wb_list);
75 } else
76 mempool_free(wi, mddev->wb_info_pool);
77 spin_unlock_irqrestore(&rdev->wb_list_lock, flags);
78 78
79 return ret;79 return ret;
80}80}
81 81
82static void remove_wb(struct md_rdev *rdev, sector_t lo, sector_t hi)82static void wait_for_serialization(struct md_rdev *rdev, struct r1bio *r1_bio)
83{
84 struct mddev *mddev = rdev->mddev;
85 struct serial_info *si;
86 int idx = sector_to_idx(r1_bio->sector);
87 struct serial_in_rdev *serial = &rdev->serial[idx];
88
89 if (WARN_ON(!mddev->serial_info_pool))
90 return;
91 si = mempool_alloc(mddev->serial_info_pool, GFP_NOIO);
92 wait_event(serial->serial_io_wait,
93 check_and_add_serial(rdev, r1_bio, si, idx) == 0);
94}
95
96static void remove_serial(struct md_rdev *rdev, sector_t lo, sector_t hi)
83{97{
84 struct wb_info *wi;98 struct serial_info *si;
85 unsigned long flags;99 unsigned long flags;
86 int found = 0;100 int found = 0;
87 struct mddev *mddev = rdev->mddev;101 struct mddev *mddev = rdev->mddev;
88 102 int idx = sector_to_idx(lo);
89 spin_lock_irqsave(&rdev->wb_list_lock, flags);103 struct serial_in_rdev *serial = &rdev->serial[idx];
90 list_for_each_entry(wi, &rdev->wb_list, list)104
91 if (hi == wi->hi && lo == wi->lo) {105 spin_lock_irqsave(&serial->serial_lock, flags);
92 list_del(&wi->list);106 for (si = raid1_rb_iter_first(&serial->serial_rb, lo, hi);
93 mempool_free(wi, mddev->wb_info_pool);107 si; si = raid1_rb_iter_next(si, lo, hi)) {
108 if (si->start == lo && si->last == hi) {
109 raid1_rb_remove(si, &serial->serial_rb);
110 mempool_free(si, mddev->serial_info_pool);
94 found = 1;111 found = 1;
95 break;112 break;
96 }113 }
97 114 }
98 if (!found)115 if (!found)
99 WARN(1, "The write behind IO is not recorded\n");116 WARN(1, "The write IO is not recorded for serialization\n");
100 spin_unlock_irqrestore(&rdev->wb_list_lock, flags);117 spin_unlock_irqrestore(&serial->serial_lock, flags);
101 wake_up(&rdev->wb_io_wait);118 wake_up(&serial->serial_io_wait);
102}119}
103 120
104/*121/*
@@ -430,6 +447,8 @@ static void raid1_end_write_request(struct bio *bio) · 325 unmodified lines
430 int mirror = find_bio_disk(r1_bio, bio);447 int mirror = find_bio_disk(r1_bio, bio);
431 struct md_rdev *rdev = conf->mirrors[mirror].rdev;448 struct md_rdev *rdev = conf->mirrors[mirror].rdev;
432 bool discard_error;449 bool discard_error;
450 sector_t lo = r1_bio->sector;
451 sector_t hi = r1_bio->sector + r1_bio->sectors;
433 452
434 discard_error = bio->bi_status && bio_op(bio) == REQ_OP_DISCARD;453 discard_error = bio->bi_status && bio_op(bio) == REQ_OP_DISCARD;
435 454
@@ -499,12 +518,8 @@ static void raid1_end_write_request(struct bio *bio) · 63 unmodified lines
499 }518 }
500 519
501 if (behind) {520 if (behind) {
502 if (test_bit(WBCollisionCheck, &rdev->flags)) {521 if (test_bit(CollisionCheck, &rdev->flags))
503 sector_t lo = r1_bio->sector;522 remove_serial(rdev, lo, hi);
504 sector_t hi = r1_bio->sector + r1_bio->sectors;
505
506 remove_wb(rdev, lo, hi);
507 }
508 if (test_bit(WriteMostly, &rdev->flags))523 if (test_bit(WriteMostly, &rdev->flags))
509 atomic_dec(&r1_bio->behind_remaining);524 atomic_dec(&r1_bio->behind_remaining);
510 525
@@ -527,7 +542,8 @@ static void raid1_end_write_request(struct bio *bio) · 16 unmodified lines
527 call_bio_endio(r1_bio);542 call_bio_endio(r1_bio);
528 }543 }
529 }544 }
530 }545 } else if (rdev->mddev->serialize_policy)
546 remove_serial(rdev, lo, hi);
531 if (r1_bio->bios[mirror] == NULL)547 if (r1_bio->bios[mirror] == NULL)
532 rdev_dec_pending(rdev, conf->mddev);548 rdev_dec_pending(rdev, conf->mddev);
533 549
@@ -1479,6 +1495,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, · 945 unmodified lines
1479 1495
1480 for (i = 0; i < disks; i++) {1496 for (i = 0; i < disks; i++) {
1481 struct bio *mbio = NULL;1497 struct bio *mbio = NULL;
1498 struct md_rdev *rdev = conf->mirrors[i].rdev;
1482 if (!r1_bio->bios[i])1499 if (!r1_bio->bios[i])
1483 continue;1500 continue;
1484 1501
@@ -1506,18 +1523,12 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, · 21 unmodified lines
1506 mbio = bio_clone_fast(bio, GFP_NOIO, &mddev->bio_set);1523 mbio = bio_clone_fast(bio, GFP_NOIO, &mddev->bio_set);
1507 1524
1508 if (r1_bio->behind_master_bio) {1525 if (r1_bio->behind_master_bio) {
1509 struct md_rdev *rdev = conf->mirrors[i].rdev;1526 if (test_bit(CollisionCheck, &rdev->flags))
1510 1527 wait_for_serialization(rdev, r1_bio);
1511 if (test_bit(WBCollisionCheck, &rdev->flags)) {
1512 sector_t lo = r1_bio->sector;
1513 sector_t hi = r1_bio->sector + r1_bio->sectors;
1514
1515 wait_event(rdev->wb_io_wait,
1516 check_and_add_wb(rdev, lo, hi) == 0);
1517 }
1518 if (test_bit(WriteMostly, &rdev->flags))1528 if (test_bit(WriteMostly, &rdev->flags))
1519 atomic_inc(&r1_bio->behind_remaining);1529 atomic_inc(&r1_bio->behind_remaining);
1520 }1530 } else if (mddev->serialize_policy)
1531 wait_for_serialization(rdev, r1_bio);
1521 1532
1522 r1_bio->bios[i] = mbio;1533 r1_bio->bios[i] = mbio;
1523 1534
drivers/md/raid5.c+7−14
@@ -6598,7 +6598,6 @@ raid5_show_group_thread_cnt(struct mddev *mddev, char *page) · 6597 unmodified lines
6598 6598
6599static int alloc_thread_groups(struct r5conf *conf, int cnt,6599static int alloc_thread_groups(struct r5conf *conf, int cnt,
6600 int *group_cnt,6600 int *group_cnt,
6601 int *worker_cnt_per_group,
6602 struct r5worker_group **worker_groups);6601 struct r5worker_group **worker_groups);
6603static ssize_t6602static ssize_t
6604raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len)6603raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len)
@@ -6607,7 +6606,7 @@ raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len) · 2 unmodified lines
6607 unsigned int new;6606 unsigned int new;
6608 int err;6607 int err;
6609 struct r5worker_group *new_groups, *old_groups;6608 struct r5worker_group *new_groups, *old_groups;
6610 int group_cnt, worker_cnt_per_group;6609 int group_cnt;
6611 6610
6612 if (len >= PAGE_SIZE)6611 if (len >= PAGE_SIZE)
6613 return -EINVAL;6612 return -EINVAL;
@@ -6630,13 +6629,11 @@ raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len) · 16 unmodified lines
6630 if (old_groups)6629 if (old_groups)
6631 flush_workqueue(raid5_wq);6630 flush_workqueue(raid5_wq);
6632 6631
6633 err = alloc_thread_groups(conf, new,6632 err = alloc_thread_groups(conf, new, &group_cnt, &new_groups);
6634 &group_cnt, &worker_cnt_per_group,
6635 &new_groups);
6636 if (!err) {6633 if (!err) {
6637 spin_lock_irq(&conf->device_lock);6634 spin_lock_irq(&conf->device_lock);
6638 conf->group_cnt = group_cnt;6635 conf->group_cnt = group_cnt;
6639 conf->worker_cnt_per_group = worker_cnt_per_group;6636 conf->worker_cnt_per_group = new;
6640 conf->worker_groups = new_groups;6637 conf->worker_groups = new_groups;
6641 spin_unlock_irq(&conf->device_lock);6638 spin_unlock_irq(&conf->device_lock);
6642 6639
@@ -6672,16 +6669,13 @@ static struct attribute_group raid5_attrs_group = { · 29 unmodified lines
6672 .attrs = raid5_attrs,6669 .attrs = raid5_attrs,
6673};6670};
6674 6671
6675static int alloc_thread_groups(struct r5conf *conf, int cnt,6672static int alloc_thread_groups(struct r5conf *conf, int cnt, int *group_cnt,
6676 int *group_cnt,
6677 int *worker_cnt_per_group,
6678 struct r5worker_group **worker_groups)6673 struct r5worker_group **worker_groups)
6679{6674{
6680 int i, j, k;6675 int i, j, k;
6681 ssize_t size;6676 ssize_t size;
6682 struct r5worker *workers;6677 struct r5worker *workers;
6683 6678
6684 *worker_cnt_per_group = cnt;
6685 if (cnt == 0) {6679 if (cnt == 0) {
6686 *group_cnt = 0;6680 *group_cnt = 0;
6687 *worker_groups = NULL;6681 *worker_groups = NULL;
@@ -6882,7 +6876,7 @@ static struct r5conf *setup_conf(struct mddev *mddev) · 194 unmodified lines
6882 struct disk_info *disk;6876 struct disk_info *disk;
6883 char pers_name[6];6877 char pers_name[6];
6884 int i;6878 int i;
6885 int group_cnt, worker_cnt_per_group;6879 int group_cnt;
6886 struct r5worker_group *new_group;6880 struct r5worker_group *new_group;
6887 int ret;6881 int ret;
6888 6882
@@ -6928,10 +6922,9 @@ static struct r5conf *setup_conf(struct mddev *mddev) · 39 unmodified lines
6928 for (i = 0; i < PENDING_IO_MAX; i++)6922 for (i = 0; i < PENDING_IO_MAX; i++)
6929 list_add(&conf->pending_data[i].sibling, &conf->free_list);6923 list_add(&conf->pending_data[i].sibling, &conf->free_list);
6930 /* Don't enable multi-threading by default*/6924 /* Don't enable multi-threading by default*/
6931 if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group,6925 if (!alloc_thread_groups(conf, 0, &group_cnt, &new_group)) {
6932 &new_group)) {
6933 conf->group_cnt = group_cnt;6926 conf->group_cnt = group_cnt;
6934 conf->worker_cnt_per_group = worker_cnt_per_group;6927 conf->worker_cnt_per_group = 0;
6935 conf->worker_groups = new_group;6928 conf->worker_groups = new_group;
6936 } else6929 } else
6937 goto abort;6930 goto abort;
include/linux/raid/pq.h+6−1
@@ -27,8 +27,8 @@ extern const char raid6_empty_zero_page[PAGE_SIZE]; · 26 unmodified lines
27 27
28#include <errno.h>28#include <errno.h>
29#include <inttypes.h>29#include <inttypes.h>
30#include <limits.h>
31#include <stddef.h>30#include <stddef.h>
31#include <string.h>
32#include <sys/mman.h>32#include <sys/mman.h>
33#include <sys/time.h>33#include <sys/time.h>
34#include <sys/types.h>34#include <sys/types.h>
@@ -44,6 +44,9 @@ typedef uint64_t u64; · 9 unmodified lines
44#ifndef PAGE_SIZE44#ifndef PAGE_SIZE
45# define PAGE_SIZE 409645# define PAGE_SIZE 4096
46#endif46#endif
47#ifndef PAGE_SHIFT
48# define PAGE_SHIFT 12
49#endif
47extern const char raid6_empty_zero_page[PAGE_SIZE];50extern const char raid6_empty_zero_page[PAGE_SIZE];
48 51
49#define __init52#define __init
@@ -59,7 +62,9 @@ extern const char raid6_empty_zero_page[PAGE_SIZE]; · 9 unmodified lines
59#define enable_kernel_altivec()62#define enable_kernel_altivec()
60#define disable_kernel_altivec()63#define disable_kernel_altivec()
61 64
65#undef EXPORT_SYMBOL
62#define EXPORT_SYMBOL(sym)66#define EXPORT_SYMBOL(sym)
67#undef EXPORT_SYMBOL_GPL
63#define EXPORT_SYMBOL_GPL(sym)68#define EXPORT_SYMBOL_GPL(sym)
64#define MODULE_LICENSE(licence)69#define MODULE_LICENSE(licence)
65#define MODULE_DESCRIPTION(desc)70#define MODULE_DESCRIPTION(desc)
include/trace/events/bcache.h+2−1
@@ -275,7 +275,8 @@ TRACE_EVENT(bcache_btree_write, · 274 unmodified lines
275 __entry->keys = b->keys.set[b->keys.nsets].data->keys;275 __entry->keys = b->keys.set[b->keys.nsets].data->keys;
276 ),276 ),
277 277
278 TP_printk("bucket %zu", __entry->bucket)278 TP_printk("bucket %zu written block %u + %u",
279 __entry->bucket, __entry->block, __entry->keys)
279);280);
280 281
281DEFINE_EVENT(btree_node, bcache_btree_node_alloc,282DEFINE_EVENT(btree_node, bcache_btree_node_alloc,
include/uapi/linux/bcache.h+52−0
@@ -148,6 +148,7 @@ static inline struct bkey *bkey_idx(const struct bkey *k, unsigned int nr_keys) · 147 unmodified lines
148#define BCACHE_SB_MAX_VERSION 4148#define BCACHE_SB_MAX_VERSION 4
149 149
150#define SB_SECTOR 8150#define SB_SECTOR 8
151#define SB_OFFSET (SB_SECTOR << SECTOR_SHIFT)
151#define SB_SIZE 4096152#define SB_SIZE 4096
152#define SB_LABEL_SIZE 32153#define SB_LABEL_SIZE 32
153#define SB_JOURNAL_BUCKETS 256U154#define SB_JOURNAL_BUCKETS 256U
@@ -156,6 +157,57 @@ static inline struct bkey *bkey_idx(const struct bkey *k, unsigned int nr_keys) · 2 unmodified lines
156 157
157#define BDEV_DATA_START_DEFAULT 16 /* sectors */158#define BDEV_DATA_START_DEFAULT 16 /* sectors */
158 159
160struct cache_sb_disk {
161 __le64 csum;
162 __le64 offset; /* sector where this sb was written */
163 __le64 version;
164
165 __u8 magic[16];
166
167 __u8 uuid[16];
168 union {
169 __u8 set_uuid[16];
170 __le64 set_magic;
171 };
172 __u8 label[SB_LABEL_SIZE];
173
174 __le64 flags;
175 __le64 seq;
176 __le64 pad[8];
177
178 union {
179 struct {
180 /* Cache devices */
181 __le64 nbuckets; /* device size */
182
183 __le16 block_size; /* sectors */
184 __le16 bucket_size; /* sectors */
185
186 __le16 nr_in_set;
187 __le16 nr_this_dev;
188 };
189 struct {
190 /* Backing devices */
191 __le64 data_offset;
192
193 /*
194 * block_size from the cache device section is still used by
195 * backing devices, so don't add anything here until we fix
196 * things to not need it for backing devices anymore
197 */
198 };
199 };
200
201 __le32 last_mount; /* time overflow in y2106 */
202
203 __le16 first_bucket;
204 union {
205 __le16 njournal_buckets;
206 __le16 keys;
207 };
208 __le64 d[SB_JOURNAL_BUCKETS]; /* journal buckets */
209};
210
159struct cache_sb {211struct cache_sb {
160 __u64 csum;212 __u64 csum;
161 __u64 offset; /* sector where this sb was written */213 __u64 offset; /* sector where this sb was written */
lib/crc64.c+1−0
@@ -28,6 +28,7 @@ · 27 unmodified lines
28 28
29#include <linux/module.h>29#include <linux/module.h>
30#include <linux/types.h>30#include <linux/types.h>
31#include <linux/crc64.h>
31#include "crc64table.h"32#include "crc64table.h"
32 33
33MODULE_DESCRIPTION("CRC64 calculations");34MODULE_DESCRIPTION("CRC64 calculations");
lib/raid6/algos.c+40−23
@@ -124,6 +124,9 @@ const struct raid6_recov_calls *const raid6_recov_algos[] = { · 123 unmodified lines
124#define time_before(x, y) ((x) < (y))124#define time_before(x, y) ((x) < (y))
125#endif125#endif
126 126
127#define RAID6_TEST_DISKS 8
128#define RAID6_TEST_DISKS_ORDER 3
129
127static inline const struct raid6_recov_calls *raid6_choose_recov(void)130static inline const struct raid6_recov_calls *raid6_choose_recov(void)
128{131{
129 const struct raid6_recov_calls *const *algo;132 const struct raid6_recov_calls *const *algo;
@@ -146,7 +149,7 @@ static inline const struct raid6_recov_calls *raid6_choose_recov(void) · 16 unmodified lines
146}149}
147 150
148static inline const struct raid6_calls *raid6_choose_gen(151static inline const struct raid6_calls *raid6_choose_gen(
149 void *(*const dptrs)[(65536/PAGE_SIZE)+2], const int disks)152 void *(*const dptrs)[RAID6_TEST_DISKS], const int disks)
150{153{
151 unsigned long perf, bestgenperf, bestxorperf, j0, j1;154 unsigned long perf, bestgenperf, bestxorperf, j0, j1;
152 int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */155 int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */
@@ -181,7 +184,8 @@ static inline const struct raid6_calls *raid6_choose_gen( · 28 unmodified lines
181 best = *algo;184 best = *algo;
182 }185 }
183 pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name,186 pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name,
184 (perf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2));187 (perf * HZ * (disks-2)) >>
188 (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2));
185 189
186 if (!(*algo)->xor_syndrome)190 if (!(*algo)->xor_syndrome)
187 continue;191 continue;
@@ -204,17 +208,24 @@ static inline const struct raid6_calls *raid6_choose_gen( · 16 unmodified lines
204 bestxorperf = perf;208 bestxorperf = perf;
205 209
206 pr_info("raid6: %-8s xor() %5ld MB/s\n", (*algo)->name,210 pr_info("raid6: %-8s xor() %5ld MB/s\n", (*algo)->name,
207 (perf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2+1));211 (perf * HZ * (disks-2)) >>
212 (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1));
208 }213 }
209 }214 }
210 215
211 if (best) {216 if (best) {
212 pr_info("raid6: using algorithm %s gen() %ld MB/s\n",217 if (IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
213 best->name,218 pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
214 (bestgenperf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2));219 best->name,
215 if (best->xor_syndrome)220 (bestgenperf * HZ * (disks-2)) >>
216 pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n",221 (20 - PAGE_SHIFT+RAID6_TIME_JIFFIES_LG2));
217 (bestxorperf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2+1));222 if (best->xor_syndrome)
223 pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n",
224 (bestxorperf * HZ * (disks-2)) >>
225 (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1));
226 } else
227 pr_info("raid6: skip pq benchmark and using algorithm %s\n",
228 best->name);
218 raid6_call = *best;229 raid6_call = *best;
219 } else230 } else
220 pr_err("raid6: Yikes! No algorithm found!\n");231 pr_err("raid6: Yikes! No algorithm found!\n");
@@ -228,27 +239,33 @@ static inline const struct raid6_calls *raid6_choose_gen( · 7 unmodified lines
228 239
229int __init raid6_select_algo(void)240int __init raid6_select_algo(void)
230{241{
231 const int disks = (65536/PAGE_SIZE)+2;242 const int disks = RAID6_TEST_DISKS;
232 243
233 const struct raid6_calls *gen_best;244 const struct raid6_calls *gen_best;
234 const struct raid6_recov_calls *rec_best;245 const struct raid6_recov_calls *rec_best;
235 char *syndromes;246 char *disk_ptr, *p;
236 void *dptrs[(65536/PAGE_SIZE)+2];247 void *dptrs[RAID6_TEST_DISKS];
237 int i;248 int i, cycle;
238
239 for (i = 0; i < disks-2; i++)
240 dptrs[i] = ((char *)raid6_gfmul) + PAGE_SIZE*i;
241
242 /* Normal code - use a 2-page allocation to avoid D$ conflict */
243 syndromes = (void *) __get_free_pages(GFP_KERNEL, 1);
244 249
245 if (!syndromes) {250 /* prepare the buffer and fill it circularly with gfmul table */
251 disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER);
252 if (!disk_ptr) {
246 pr_err("raid6: Yikes! No memory available.\n");253 pr_err("raid6: Yikes! No memory available.\n");
247 return -ENOMEM;254 return -ENOMEM;
248 }255 }
249 256
250 dptrs[disks-2] = syndromes;257 p = disk_ptr;
251 dptrs[disks-1] = syndromes + PAGE_SIZE;258 for (i = 0; i < disks; i++)
259 dptrs[i] = p + PAGE_SIZE * i;
260
261 cycle = ((disks - 2) * PAGE_SIZE) / 65536;
262 for (i = 0; i < cycle; i++) {
263 memcpy(p, raid6_gfmul, 65536);
264 p += 65536;
265 }
266
267 if ((disks - 2) * PAGE_SIZE % 65536)
268 memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536);
252 269
253 /* select raid gen_syndrome function */270 /* select raid gen_syndrome function */
254 gen_best = raid6_choose_gen(&dptrs, disks);271 gen_best = raid6_choose_gen(&dptrs, disks);
@@ -256,7 +273,7 @@ int __init raid6_select_algo(void) · 1 unmodified line
256 /* select raid recover functions */273 /* select raid recover functions */
257 rec_best = raid6_choose_recov();274 rec_best = raid6_choose_recov();
258 275
259 free_pages((unsigned long)syndromes, 1);276 free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER);
260 277
261 return gen_best && rec_best ? 0 : -EINVAL;278 return gen_best && rec_best ? 0 : -EINVAL;
262}279}
lib/raid6/mktables.c+1−1
@@ -56,8 +56,8 @@ int main(int argc, char *argv[]) · 55 unmodified lines
56 uint8_t v;56 uint8_t v;
57 uint8_t exptbl[256], invtbl[256];57 uint8_t exptbl[256], invtbl[256];
58 58
59 printf("#include <linux/raid/pq.h>\n");
60 printf("#include <linux/export.h>\n");59 printf("#include <linux/export.h>\n");
60 printf("#include <linux/raid/pq.h>\n");
61 61
62 /* Compute multiplication table */62 /* Compute multiplication table */
63 printf("\nconst u8 __attribute__((aligned(256)))\n"63 printf("\nconst u8 __attribute__((aligned(256)))\n"

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t