History

md/r5cache: improve recovery with read ahead page pool

In r5cache recovery, the journal device is scanned page by page. Currently, we use sync_page_io() to read journal device. This is not efficient when we have to recovery many stripes from the journal. To improve the speed of recovery, this patch introduces a read ahead page pool (ra_pool) to recovery_ctx. With ra_pool, multiple consecutive pages are read in one IO. Then the recovery code read the journal from ra_pool. With ra_pool, r5l_recovery_ctx has become much bigger. Therefore, r5l_recovery_log() is refactored so r5l_recovery_ctx is not using stack space.

Song Liu committed 9 years ago

Commit effe6ee1 parent aaf9f121 file · +175 −46Browse files

Unified · Split

drivers/md/raid5-cache.c+175−46
@@ -30,6 +30,7 @@ · 29 unmodified lines
3030 * underneath hardware sector size. only works with PAGE_SIZE == 4096
3131 */
3232 #define BLOCK_SECTORS (8)
33+#define BLOCK_SECTOR_SHIFT (3)
3334
3435 /*
3536 * log->max_free_space is min(1/4 disk size, 10G reclaimable space).
@@ -1552,6 +1553,8 @@ bool r5l_log_disk_error(struct r5conf *conf) · 1516 unmodified lines
15521553 return ret;
15531554 }
15541555
1556+#define R5L_RECOVERY_PAGE_POOL_SIZE 256
1557+
15551558 struct r5l_recovery_ctx {
15561559 struct page *meta_page; /* current meta */
15571560 sector_t meta_total_blocks; /* total size of current meta and data */
@@ -1560,18 +1563,131 @@ struct r5l_recovery_ctx { · 2 unmodified lines
15601563 int data_parity_stripes; /* number of data_parity stripes */
15611564 int data_only_stripes; /* number of data_only stripes */
15621565 struct list_head cached_list;
1566+
1567+ /*
1568+ * read ahead page pool (ra_pool)
1569+ * in recovery, log is read sequentially. It is not efficient to
1570+ * read every page with sync_page_io(). The read ahead page pool
1571+ * reads multiple pages with one IO, so further log read can
1572+ * just copy data from the pool.
1573+ */
1574+ struct page *ra_pool[R5L_RECOVERY_PAGE_POOL_SIZE];
1575+ sector_t pool_offset; /* offset of first page in the pool */
1576+ int total_pages; /* total allocated pages */
1577+ int valid_pages; /* pages with valid data */
1578+ struct bio *ra_bio; /* bio to do the read ahead */
15631579 };
15641580
1581+static int r5l_recovery_allocate_ra_pool(struct r5l_log *log,
1582+ struct r5l_recovery_ctx *ctx)
1583+{
1584+ struct page *page;
1585+
1586+ ctx->ra_bio = bio_alloc_bioset(GFP_KERNEL, BIO_MAX_PAGES, log->bs);
1587+ if (!ctx->ra_bio)
1588+ return -ENOMEM;
1589+
1590+ ctx->valid_pages = 0;
1591+ ctx->total_pages = 0;
1592+ while (ctx->total_pages < R5L_RECOVERY_PAGE_POOL_SIZE) {
1593+ page = alloc_page(GFP_KERNEL);
1594+
1595+ if (!page)
1596+ break;
1597+ ctx->ra_pool[ctx->total_pages] = page;
1598+ ctx->total_pages += 1;
1599+ }
1600+
1601+ if (ctx->total_pages == 0) {
1602+ bio_put(ctx->ra_bio);
1603+ return -ENOMEM;
1604+ }
1605+
1606+ ctx->pool_offset = 0;
1607+ return 0;
1608+}
1609+
1610+static void r5l_recovery_free_ra_pool(struct r5l_log *log,
1611+ struct r5l_recovery_ctx *ctx)
1612+{
1613+ int i;
1614+
1615+ for (i = 0; i < ctx->total_pages; ++i)
1616+ put_page(ctx->ra_pool[i]);
1617+ bio_put(ctx->ra_bio);
1618+}
1619+
1620+/*
1621+ * fetch ctx->valid_pages pages from offset
1622+ * In normal cases, ctx->valid_pages == ctx->total_pages after the call.
1623+ * However, if the offset is close to the end of the journal device,
1624+ * ctx->valid_pages could be smaller than ctx->total_pages
1625+ */
1626+static int r5l_recovery_fetch_ra_pool(struct r5l_log *log,
1627+ struct r5l_recovery_ctx *ctx,
1628+ sector_t offset)
1629+{
1630+ bio_reset(ctx->ra_bio);
1631+ ctx->ra_bio->bi_bdev = log->rdev->bdev;
1632+ bio_set_op_attrs(ctx->ra_bio, REQ_OP_READ, 0);
1633+ ctx->ra_bio->bi_iter.bi_sector = log->rdev->data_offset + offset;
1634+
1635+ ctx->valid_pages = 0;
1636+ ctx->pool_offset = offset;
1637+
1638+ while (ctx->valid_pages < ctx->total_pages) {
1639+ bio_add_page(ctx->ra_bio,
1640+ ctx->ra_pool[ctx->valid_pages], PAGE_SIZE, 0);
1641+ ctx->valid_pages += 1;
1642+
1643+ offset = r5l_ring_add(log, offset, BLOCK_SECTORS);
1644+
1645+ if (offset == 0) /* reached end of the device */
1646+ break;
1647+ }
1648+
1649+ return submit_bio_wait(ctx->ra_bio);
1650+}
1651+
1652+/*
1653+ * try read a page from the read ahead page pool, if the page is not in the
1654+ * pool, call r5l_recovery_fetch_ra_pool
1655+ */
1656+static int r5l_recovery_read_page(struct r5l_log *log,
1657+ struct r5l_recovery_ctx *ctx,
1658+ struct page *page,
1659+ sector_t offset)
1660+{
1661+ int ret;
1662+
1663+ if (offset < ctx->pool_offset ||
1664+ offset >= ctx->pool_offset + ctx->valid_pages * BLOCK_SECTORS) {
1665+ ret = r5l_recovery_fetch_ra_pool(log, ctx, offset);
1666+ if (ret)
1667+ return ret;
1668+ }
1669+
1670+ BUG_ON(offset < ctx->pool_offset ||
1671+ offset >= ctx->pool_offset + ctx->valid_pages * BLOCK_SECTORS);
1672+
1673+ memcpy(page_address(page),
1674+ page_address(ctx->ra_pool[(offset - ctx->pool_offset) >>
1675+ BLOCK_SECTOR_SHIFT]),
1676+ PAGE_SIZE);
1677+ return 0;
1678+}
1679+
15651680 static int r5l_recovery_read_meta_block(struct r5l_log *log,
15661681 struct r5l_recovery_ctx *ctx)
15671682 {
15681683 struct page *page = ctx->meta_page;
15691684 struct r5l_meta_block *mb;
15701685 u32 crc, stored_crc;
1686+ int ret;
15711687
1572− if (!sync_page_io(log->rdev, ctx->pos, PAGE_SIZE, page, REQ_OP_READ, 0,
1573− false))
1574− return -EIO;
1688+ ret = r5l_recovery_read_page(log, ctx, page, ctx->pos);
1689+ if (ret != 0)
1690+ return ret;
15751691
15761692 mb = page_address(page);
15771693 stored_crc = le32_to_cpu(mb->checksum);
@@ -1653,8 +1769,7 @@ static void r5l_recovery_load_data(struct r5l_log *log, · 75 unmodified lines
16531769 raid5_compute_sector(conf,
16541770 le64_to_cpu(payload->location), 0,
16551771 &dd_idx, sh);
1656− sync_page_io(log->rdev, log_offset, PAGE_SIZE,
1657− sh->dev[dd_idx].page, REQ_OP_READ, 0, false);
1772+ r5l_recovery_read_page(log, ctx, sh->dev[dd_idx].page, log_offset);
16581773 sh->dev[dd_idx].log_checksum =
16591774 le32_to_cpu(payload->checksum[0]);
16601775 ctx->meta_total_blocks += BLOCK_SECTORS;
@@ -1673,17 +1788,15 @@ static void r5l_recovery_load_parity(struct r5l_log *log, · 12 unmodified lines
16731788 struct r5conf *conf = mddev->private;
16741789
16751790 ctx->meta_total_blocks += BLOCK_SECTORS * conf->max_degraded;
1676− sync_page_io(log->rdev, log_offset, PAGE_SIZE,
1677− sh->dev[sh->pd_idx].page, REQ_OP_READ, 0, false);
1791+ r5l_recovery_read_page(log, ctx, sh->dev[sh->pd_idx].page, log_offset);
16781792 sh->dev[sh->pd_idx].log_checksum =
16791793 le32_to_cpu(payload->checksum[0]);
16801794 set_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags);
16811795
16821796 if (sh->qd_idx >= 0) {
1683− sync_page_io(log->rdev,
1684− r5l_ring_add(log, log_offset, BLOCK_SECTORS),
1685− PAGE_SIZE, sh->dev[sh->qd_idx].page,
1686− REQ_OP_READ, 0, false);
1797+ r5l_recovery_read_page(
1798+ log, ctx, sh->dev[sh->qd_idx].page,
1799+ r5l_ring_add(log, log_offset, BLOCK_SECTORS));
16871800 sh->dev[sh->qd_idx].log_checksum =
16881801 le32_to_cpu(payload->checksum[1]);
16891802 set_bit(R5_Wantwrite, &sh->dev[sh->qd_idx].flags);
@@ -1814,14 +1927,15 @@ r5c_recovery_replay_stripes(struct list_head *cached_stripe_list, · 124 unmodified lines
18141927
18151928 /* if matches return 0; otherwise return -EINVAL */
18161929 static int
1817−r5l_recovery_verify_data_checksum(struct r5l_log *log, struct page *page,
1930+r5l_recovery_verify_data_checksum(struct r5l_log *log,
1931+ struct r5l_recovery_ctx *ctx,
1932+ struct page *page,
18181933 sector_t log_offset, __le32 log_checksum)
18191934 {
18201935 void *addr;
18211936 u32 checksum;
18221937
1823− sync_page_io(log->rdev, log_offset, PAGE_SIZE,
1824− page, REQ_OP_READ, 0, false);
1938+ r5l_recovery_read_page(log, ctx, page, log_offset);
18251939 addr = kmap_atomic(page);
18261940 checksum = crc32c_le(log->uuid_checksum, addr, PAGE_SIZE);
18271941 kunmap_atomic(addr);
@@ -1853,17 +1967,17 @@ r5l_recovery_verify_data_checksum_for_mb(struct r5l_log *log, · 25 unmodified lines
18531967
18541968 if (payload->header.type == R5LOG_PAYLOAD_DATA) {
18551969 if (r5l_recovery_verify_data_checksum(
1856− log, page, log_offset,
1970+ log, ctx, page, log_offset,
18571971 payload->checksum[0]) < 0)
18581972 goto mismatch;
18591973 } else if (payload->header.type == R5LOG_PAYLOAD_PARITY) {
18601974 if (r5l_recovery_verify_data_checksum(
1861− log, page, log_offset,
1975+ log, ctx, page, log_offset,
18621976 payload->checksum[0]) < 0)
18631977 goto mismatch;
18641978 if (conf->max_degraded == 2 && /* q for RAID 6 */
18651979 r5l_recovery_verify_data_checksum(
1866− log, page,
1980+ log, ctx, page,
18671981 r5l_ring_add(log, log_offset,
18681982 BLOCK_SECTORS),
18691983 payload->checksum[1]) < 0)
@@ -2241,55 +2355,70 @@ static void r5c_recovery_flush_data_only_stripes(struct r5l_log *log, · 371 unmodified lines
22412355 static int r5l_recovery_log(struct r5l_log *log)
22422356 {
22432357 struct mddev *mddev = log->rdev->mddev;
2244− struct r5l_recovery_ctx ctx;
2358+ struct r5l_recovery_ctx *ctx;
22452359 int ret;
22462360 sector_t pos;
22472361
2248− ctx.pos = log->last_checkpoint;
2249− ctx.seq = log->last_cp_seq;
2250− ctx.meta_page = alloc_page(GFP_KERNEL);
2251− ctx.data_only_stripes = 0;
2252− ctx.data_parity_stripes = 0;
2253− INIT_LIST_HEAD(&ctx.cached_list);
2254−
2255− if (!ctx.meta_page)
2362+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
2363+ if (!ctx)
22562364 return -ENOMEM;
22572365
2258− ret = r5c_recovery_flush_log(log, &ctx);
2259− __free_page(ctx.meta_page);
2366+ ctx->pos = log->last_checkpoint;
2367+ ctx->seq = log->last_cp_seq;
2368+ INIT_LIST_HEAD(&ctx->cached_list);
2369+ ctx->meta_page = alloc_page(GFP_KERNEL);
22602370
2261− if (ret)
2262− return ret;
2371+ if (!ctx->meta_page) {
2372+ ret = -ENOMEM;
2373+ goto meta_page;
2374+ }
2375+
2376+ if (r5l_recovery_allocate_ra_pool(log, ctx) != 0) {
2377+ ret = -ENOMEM;
2378+ goto ra_pool;
2379+ }
22632380
2264− pos = ctx.pos;
2265− ctx.seq += 10000;
2381+ ret = r5c_recovery_flush_log(log, ctx);
22662382
2383+ if (ret)
2384+ goto error;
22672385
2268− if ((ctx.data_only_stripes == 0) && (ctx.data_parity_stripes == 0))
2386+ pos = ctx->pos;
2387+ ctx->seq += 10000;
2388+
2389+ if ((ctx->data_only_stripes == 0) && (ctx->data_parity_stripes == 0))
22692390 pr_debug("md/raid:%s: starting from clean shutdown\n",
22702391 mdname(mddev));
22712392 else
22722393 pr_debug("md/raid:%s: recovering %d data-only stripes and %d data-parity stripes\n",
2273− mdname(mddev), ctx.data_only_stripes,
2274− ctx.data_parity_stripes);
2275−
2276− if (ctx.data_only_stripes == 0) {
2277− log->next_checkpoint = ctx.pos;
2278− r5l_log_write_empty_meta_block(log, ctx.pos, ctx.seq++);
2279− ctx.pos = r5l_ring_add(log, ctx.pos, BLOCK_SECTORS);
2280− } else if (r5c_recovery_rewrite_data_only_stripes(log, &ctx)) {
2394+ mdname(mddev), ctx->data_only_stripes,
2395+ ctx->data_parity_stripes);
2396+
2397+ if (ctx->data_only_stripes == 0) {
2398+ log->next_checkpoint = ctx->pos;
2399+ r5l_log_write_empty_meta_block(log, ctx->pos, ctx->seq++);
2400+ ctx->pos = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
2401+ } else if (r5c_recovery_rewrite_data_only_stripes(log, ctx)) {
22812402 pr_err("md/raid:%s: failed to rewrite stripes to journal\n",
22822403 mdname(mddev));
2283− return -EIO;
2404+ ret = -EIO;
2405+ goto error;
22842406 }
22852407
2286− log->log_start = ctx.pos;
2287− log->seq = ctx.seq;
2408+ log->log_start = ctx->pos;
2409+ log->seq = ctx->seq;
22882410 log->last_checkpoint = pos;
22892411 r5l_write_super(log, pos);
22902412
2291− r5c_recovery_flush_data_only_stripes(log, &ctx);
2292− return 0;
2413+ r5c_recovery_flush_data_only_stripes(log, ctx);
2414+ ret = 0;
2415+error:
2416+ r5l_recovery_free_ra_pool(log, ctx);
2417+ra_pool:
2418+ __free_page(ctx->meta_page);
2419+meta_page:
2420+ kfree(ctx);
2421+ return ret;
22932422 }
22942423
22952424 static void r5l_write_super(struct r5l_log *log, sector_t cp)