// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (C) 2015 Shaohua Li <shli@fb.com> * Copyright (C) 2016 Song Liu <songliubraving@fb.com> */#include <linux/kernel.h>#include <linux/wait.h>#include <linux/blkdev.h>#include <linux/slab.h>#include <linux/raid/md_p.h>#include <linux/crc32c.h>#include <linux/random.h>#include <linux/kthread.h>#include <linux/types.h>#include "md.h"#include "raid5.h"#include "md-bitmap.h"#include "raid5-log.h" /* * metadata/data stored in disk with 4k size unit (a block) regardless * underneath hardware sector size. only works with PAGE_SIZE == 4096 */#define BLOCK_SECTORS (8)#define BLOCK_SECTOR_SHIFT (3) /* * log->max_free_space is min(1/4 disk size, 10G reclaimable space). * * In write through mode, the reclaim runs every log->max_free_space. * This can prevent the recovery scans for too long */#define RECLAIM_MAX_FREE_SPACE (10 * 1024 * 1024 * 2) /* sector */#define RECLAIM_MAX_FREE_SPACE_SHIFT (2) /* wake up reclaim thread periodically */#define R5C_RECLAIM_WAKEUP_INTERVAL (30 * HZ)/* start flush with these full stripes */#define R5C_FULL_STRIPE_FLUSH_BATCH(conf) (conf->max_nr_stripes / 4)/* reclaim stripes in groups */#define R5C_RECLAIM_STRIPE_GROUP (NR_STRIPE_HASH_LOCKS * 2) /* * We only need 2 bios per I/O unit to make progress, but ensure we * have a few more available to not get too tight. */#define R5L_POOL_SIZE 4 static char *r5c_journal_mode_str[] = {"write-through", "write-back"};/* * raid5 cache state machine * * With the RAID cache, each stripe works in two phases: * - caching phase * - writing-out phase * * These two phases are controlled by bit STRIPE_R5C_CACHING: * if STRIPE_R5C_CACHING == 0, the stripe is in writing-out phase * if STRIPE_R5C_CACHING == 1, the stripe is in caching phase * * When there is no journal, or the journal is in write-through mode, * the stripe is always in writing-out phase. * * For write-back journal, the stripe is sent to caching phase on write * (r5c_try_caching_write). r5c_make_stripe_write_out() kicks off * the write-out phase by clearing STRIPE_R5C_CACHING. * * Stripes in caching phase do not write the raid disks. Instead, all * writes are committed from the log device. Therefore, a stripe in * caching phase handles writes as: * - write to log device * - return IO * * Stripes in writing-out phase handle writes as: * - calculate parity * - write pending data and parity to journal * - write data and parity to raid disks * - return IO for pending writes */ struct r5l_log { struct md_rdev *rdev; u32 uuid_checksum; sector_t device_size; /* log device size, round to * BLOCK_SECTORS */ sector_t max_free_space; /* reclaim run if free space is at * this size */ sector_t last_checkpoint; /* log tail. where recovery scan * starts from */ u64 last_cp_seq; /* log tail sequence */ sector_t log_start; /* log head. where new data appends */ u64 seq; /* log head sequence */ sector_t next_checkpoint; struct mutex io_mutex; struct r5l_io_unit *current_io; /* current io_unit accepting new data */ spinlock_t io_list_lock; struct list_head running_ios; /* io_units which are still running, * and have not yet been completely * written to the log */ struct list_head io_end_ios; /* io_units which have been completely * written to the log but not yet written * to the RAID */ struct list_head flushing_ios; /* io_units which are waiting for log * cache flush */ struct list_head finished_ios; /* io_units which settle down in log disk */ struct bio flush_bio; struct list_head no_mem_stripes; /* pending stripes, -ENOMEM */ struct kmem_cache *io_kc; mempool_t io_pool; struct bio_set bs; mempool_t meta_pool; struct md_thread __rcu *reclaim_thread; unsigned long reclaim_target; /* number of space that need to be * reclaimed. if it's 0, reclaim spaces * used by io_units which are in * IO_UNIT_STRIPE_END state (eg, reclaim * doesn't wait for specific io_unit * switching to IO_UNIT_STRIPE_END * state) */ wait_queue_head_t iounit_wait; struct list_head no_space_stripes; /* pending stripes, log has no space */ spinlock_t no_space_stripes_lock; bool need_cache_flush; /* for r5c_cache */ enum r5c_journal_mode r5c_journal_mode; /* all stripes in r5cache, in the order of seq at sh->log_start */ struct list_head stripe_in_journal_list; spinlock_t stripe_in_journal_lock; atomic_t stripe_in_journal_count; /* to submit async io_units, to fulfill ordering of flush */ struct work_struct deferred_io_work; /* to disable write back during in degraded mode */ struct work_struct disable_writeback_work; /* to for chunk_aligned_read in writeback mode, details below */ spinlock_t tree_lock; struct radix_tree_root big_stripe_tree;}; /* * Enable chunk_aligned_read() with write back cache. * * Each chunk may contain more than one stripe (for example, a 256kB * chunk contains 64 4kB-page, so this chunk contain 64 stripes). For * chunk_aligned_read, these stripes are grouped into one "big_stripe". * For each big_stripe, we count how many stripes of this big_stripe * are in the write back cache. These data are tracked in a radix tree * (big_stripe_tree). We use radix_tree item pointer as the counter. * r5c_tree_index() is used to calculate keys for the radix tree. * * chunk_aligned_read() calls r5c_big_stripe_cached() to look up * big_stripe of each chunk in the tree. If this big_stripe is in the * tree, chunk_aligned_read() aborts. This look up is protected by * rcu_read_lock(). * * It is necessary to remember whether a stripe is counted in * big_stripe_tree. Instead of adding new flag, we reuses existing flags: * STRIPE_R5C_PARTIAL_STRIPE and STRIPE_R5C_FULL_STRIPE. If either of these * two flags are set, the stripe is counted in big_stripe_tree. This * requires moving set_bit(STRIPE_R5C_PARTIAL_STRIPE) to * r5c_try_caching_write(); and moving clear_bit of * STRIPE_R5C_PARTIAL_STRIPE and STRIPE_R5C_FULL_STRIPE to * r5c_finish_stripe_write_out(). */ /* * radix tree requests lowest 2 bits of data pointer to be 2b'00. * So it is necessary to left shift the counter by 2 bits before using it * as data pointer of the tree. */#define R5C_RADIX_COUNT_SHIFT 2 /* * calculate key for big_stripe_tree * * sect: align_bi->bi_iter.bi_sector or sh->sector */static inline sector_t r5c_tree_index(struct r5conf *conf, sector_t sect){ sector_div(sect, conf->chunk_sectors); return sect;} /* * an IO range starts from a meta data block and end at the next meta data * block. The io unit's the meta data block tracks data/parity followed it. io * unit is written to log disk with normal write, as we always flush log disk * first and then start move data to raid disks, there is no requirement to * write io unit with FLUSH/FUA */struct r5l_io_unit { struct r5l_log *log; struct page *meta_page; /* store meta block */ int meta_offset; /* current offset in meta_page */ struct bio *current_bio;/* current_bio accepting new data */ atomic_t pending_stripe;/* how many stripes not flushed to raid */ u64 seq; /* seq number of the metablock */ sector_t log_start; /* where the io_unit starts */ sector_t log_end; /* where the io_unit ends */ struct list_head log_sibling; /* log->running_ios */ struct list_head stripe_list; /* stripes added to the io_unit */ int state; bool need_split_bio; struct bio *split_bio; unsigned int has_flush:1; /* include flush request */ unsigned int has_fua:1; /* include fua request */ unsigned int has_null_flush:1; /* include null flush request */ unsigned int has_flush_payload:1; /* include flush payload */ /* * io isn't sent yet, flush/fua request can only be submitted till it's * the first IO in running_ios list */ unsigned int io_deferred:1; struct bio_list flush_barriers; /* size == 0 flush bios */}; /* r5l_io_unit state */enum r5l_io_unit_state { IO_UNIT_RUNNING = 0, /* accepting new IO */ IO_UNIT_IO_START = 1, /* io_unit bio start writing to log, * don't accepting new bio */ IO_UNIT_IO_END = 2, /* io_unit bio finish writing to log */ IO_UNIT_STRIPE_END = 3, /* stripes data finished writing to raid */}; bool r5c_is_writeback(struct r5l_log *log){ return (log != NULL && log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK);} static sector_t r5l_ring_add(struct r5l_log *log, sector_t start, sector_t inc){ start += inc; if (start >= log->device_size) start = start - log->device_size; return start;} static sector_t r5l_ring_distance(struct r5l_log *log, sector_t start, sector_t end){ if (end >= start) return end - start; else return end + log->device_size - start;} static bool r5l_has_free_space(struct r5l_log *log, sector_t size){ sector_t used_size; used_size = r5l_ring_distance(log, log->last_checkpoint, log->log_start); return log->device_size > used_size + size;} static void __r5l_set_io_unit_state(struct r5l_io_unit *io, enum r5l_io_unit_state state){ if (WARN_ON(io->state >= state)) return; io->state = state;} static voidr5c_return_dev_pending_writes(struct r5conf *conf, struct r5dev *dev){ struct bio *wbi, *wbi2; wbi = dev->written; dev->written = NULL; while (wbi && wbi->bi_iter.bi_sector < dev->sector + RAID5_STRIPE_SECTORS(conf)) { wbi2 = r5_next_bio(conf, wbi, dev->sector); md_write_end(conf->mddev); bio_endio(wbi); wbi = wbi2; }} void r5c_handle_cached_data_endio(struct r5conf *conf, struct stripe_head *sh, int disks){ int i; for (i = sh->disks; i--; ) { if (sh->dev[i].written) { set_bit(R5_UPTODATE, &sh->dev[i].flags); r5c_return_dev_pending_writes(conf, &sh->dev[i]); } }} void r5l_wake_reclaim(struct r5l_log *log, sector_t space); /* Check whether we should flush some stripes to free up stripe cache */void r5c_check_stripe_cache_usage(struct r5conf *conf){ int total_cached; struct r5l_log *log = READ_ONCE(conf->log); if (!r5c_is_writeback(log)) return; total_cached = atomic_read(&conf->r5c_cached_partial_stripes) + atomic_read(&conf->r5c_cached_full_stripes); /* * The following condition is true for either of the following: * - stripe cache pressure high: * total_cached > 3/4 min_nr_stripes || * empty_inactive_list_nr > 0 * - stripe cache pressure moderate: * total_cached > 1/2 min_nr_stripes */ if (total_cached > conf->min_nr_stripes * 1 / 2 || atomic_read(&conf->empty_inactive_list_nr) > 0) r5l_wake_reclaim(log, 0);} /* * flush cache when there are R5C_FULL_STRIPE_FLUSH_BATCH or more full * stripes in the cache */void r5c_check_cached_full_stripe(struct r5conf *conf){ struct r5l_log *log = READ_ONCE(conf->log); if (!r5c_is_writeback(log)) return; /* * wake up reclaim for R5C_FULL_STRIPE_FLUSH_BATCH cached stripes * or a full stripe (chunk size / 4k stripes). */ if (atomic_read(&conf->r5c_cached_full_stripes) >= min(R5C_FULL_STRIPE_FLUSH_BATCH(conf), conf->chunk_sectors >> RAID5_STRIPE_SHIFT(conf))) r5l_wake_reclaim(log, 0);} /* * Total log space (in sectors) needed to flush all data in cache * * To avoid deadlock due to log space, it is necessary to reserve log * space to flush critical stripes (stripes that occupying log space near * last_checkpoint). This function helps check how much log space is * required to flush all cached stripes. * * To reduce log space requirements, two mechanisms are used to give cache * flush higher priorities: * 1. In handle_stripe_dirtying() and schedule_reconstruction(), * stripes ALREADY in journal can be flushed w/o pending writes; * 2. In r5l_write_stripe() and r5c_cache_data(), stripes NOT in journal * can be delayed (r5l_add_no_space_stripe). * * In cache flush, the stripe goes through 1 and then 2. For a stripe that * already passed 1, flushing it requires at most (conf->max_degraded + 1) * pages of journal space. For stripes that has not passed 1, flushing it * requires (conf->raid_disks + 1) pages of journal space. There are at * most (conf->group_cnt + 1) stripe that passed 1. So total journal space * required to flush all cached stripes (in pages) is: * * (stripe_in_journal_count - group_cnt - 1) * (max_degraded + 1) + * (group_cnt + 1) * (raid_disks + 1) * or * (stripe_in_journal_count) * (max_degraded + 1) + * (group_cnt + 1) * (raid_disks - max_degraded) */static sector_t r5c_log_required_to_flush_cache(struct r5conf *conf){ struct r5l_log *log = READ_ONCE(conf->log); if (!r5c_is_writeback(log)) return 0; return BLOCK_SECTORS * ((conf->max_degraded + 1) * atomic_read(&log->stripe_in_journal_count) + (conf->raid_disks - conf->max_degraded) * (conf->group_cnt + 1));} /* * evaluate log space usage and update R5C_LOG_TIGHT and R5C_LOG_CRITICAL * * R5C_LOG_TIGHT is set when free space on the log device is less than 3x of * reclaim_required_space. R5C_LOG_CRITICAL is set when free space on the log * device is less than 2x of reclaim_required_space. */static inline void r5c_update_log_state(struct r5l_log *log){ struct r5conf *conf = log->rdev->mddev->private; sector_t free_space; sector_t reclaim_space; bool wake_reclaim = false; if (!r5c_is_writeback(log)) return; free_space = r5l_ring_distance(log, log->log_start, log->last_checkpoint); reclaim_space = r5c_log_required_to_flush_cache(conf); if (free_space < 2 * reclaim_space) set_bit(R5C_LOG_CRITICAL, &conf->cache_state); else { if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state)) wake_reclaim = true; clear_bit(R5C_LOG_CRITICAL, &conf->cache_state); } if (free_space < 3 * reclaim_space) set_bit(R5C_LOG_TIGHT, &conf->cache_state); else clear_bit(R5C_LOG_TIGHT, &conf->cache_state); if (wake_reclaim) r5l_wake_reclaim(log, 0);} /* * Put the stripe into writing-out phase by clearing STRIPE_R5C_CACHING. * This function should only be called in write-back mode. */void r5c_make_stripe_write_out(struct stripe_head *sh){ struct r5conf *conf = sh->raid_conf; struct r5l_log *log = READ_ONCE(conf->log); BUG_ON(!r5c_is_writeback(log)); WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state)); clear_bit(STRIPE_R5C_CACHING, &sh->state); if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) atomic_inc(&conf->preread_active_stripes);} static void r5c_handle_data_cached(struct stripe_head *sh){ int i; for (i = sh->disks; i--; ) if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) { set_bit(R5_InJournal, &sh->dev[i].flags); clear_bit(R5_LOCKED, &sh->dev[i].flags); } clear_bit(STRIPE_LOG_TRAPPED, &sh->state);} /* * this journal write must contain full parity, * it may also contain some data pages */static void r5c_handle_parity_cached(struct stripe_head *sh){ int i; for (i = sh->disks; i--; ) if (test_bit(R5_InJournal, &sh->dev[i].flags)) set_bit(R5_Wantwrite, &sh->dev[i].flags);} /* * Setting proper flags after writing (or flushing) data and/or parity to the * log device. This is called from r5l_log_endio() or r5l_log_flush_endio(). */static void r5c_finish_cache_stripe(struct stripe_head *sh){ struct r5l_log *log = READ_ONCE(sh->raid_conf->log); if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) { BUG_ON(test_bit(STRIPE_R5C_CACHING, &sh->state)); /* * Set R5_InJournal for parity dev[pd_idx]. This means * all data AND parity in the journal. For RAID 6, it is * NOT necessary to set the flag for dev[qd_idx], as theLines 1–500 of 3187. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2015 Shaohua Li <shli@fb.com> * Copyright (C) 2016 Song Liu <songliubraving@fb.com> */ #include <linux/kernel.h> #include <linux/wait.h> #include <linux/blkdev.h> #include <linux/slab.h> #include <linux/raid/md_p.h> #include <linux/crc32c.h> #include <linux/random.h> #include <linux/kthread.h> #include <linux/types.h> #include "md.h" #include "raid5.h" #include "md-bitmap.h" #include "raid5-log.h" /* * metadata/data stored in disk with 4k size unit (a block) regardless * underneath hardware sector size. only works with PAGE_SIZE == 4096 */ #define BLOCK_SECTORS (8) #define BLOCK_SECTOR_SHIFT (3) /* * log->max_free_space is min(1/4 disk size, 10G reclaimable space). * * In write through mode, the reclaim runs every log->max_free_space. * This can prevent the recovery scans for too long */ #define RECLAIM_MAX_FREE_SPACE (10 * 1024 * 1024 * 2) /* sector */ #define RECLAIM_MAX_FREE_SPACE_SHIFT (2) /* wake up reclaim thread periodically */ #define R5C_RECLAIM_WAKEUP_INTERVAL (30 * HZ) /* start flush with these full stripes */ #define R5C_FULL_STRIPE_FLUSH_BATCH(conf) (conf->max_nr_stripes / 4) /* reclaim stripes in groups */ #define R5C_RECLAIM_STRIPE_GROUP (NR_STRIPE_HASH_LOCKS * 2) /* * We only need 2 bios per I/O unit to make progress, but ensure we * have a few more available to not get too tight. */ #define R5L_POOL_SIZE 4 static char *r5c_journal_mode_str[] = {"write-through", "write-back"}; /* * raid5 cache state machine * * With the RAID cache, each stripe works in two phases: * - caching phase * - writing-out phase * * These two phases are controlled by bit STRIPE_R5C_CACHING: * if STRIPE_R5C_CACHING == 0, the stripe is in writing-out phase * if STRIPE_R5C_CACHING == 1, the stripe is in caching phase * * When there is no journal, or the journal is in write-through mode, * the stripe is always in writing-out phase. * * For write-back journal, the stripe is sent to caching phase on write * (r5c_try_caching_write). r5c_make_stripe_write_out() kicks off * the write-out phase by clearing STRIPE_R5C_CACHING. * * Stripes in caching phase do not write the raid disks. Instead, all * writes are committed from the log device. Therefore, a stripe in * caching phase handles writes as: * - write to log device * - return IO * * Stripes in writing-out phase handle writes as: * - calculate parity * - write pending data and parity to journal * - write data and parity to raid disks * - return IO for pending writes */ struct r5l_log { struct md_rdev *rdev; u32 uuid_checksum; sector_t device_size; /* log device size, round to * BLOCK_SECTORS */ sector_t max_free_space; /* reclaim run if free space is at * this size */ sector_t last_checkpoint; /* log tail. where recovery scan * starts from */ u64 last_cp_seq; /* log tail sequence */ sector_t log_start; /* log head. where new data appends */ u64 seq; /* log head sequence */ sector_t next_checkpoint; struct mutex io_mutex; struct r5l_io_unit *current_io; /* current io_unit accepting new data */ spinlock_t io_list_lock; struct list_head running_ios; /* io_units which are still running, * and have not yet been completely * written to the log */ struct list_head io_end_ios; /* io_units which have been completely * written to the log but not yet written * to the RAID */ struct list_head flushing_ios; /* io_units which are waiting for log * cache flush */ struct list_head finished_ios; /* io_units which settle down in log disk */ struct bio flush_bio; struct list_head no_mem_stripes; /* pending stripes, -ENOMEM */ struct kmem_cache *io_kc; mempool_t io_pool; struct bio_set bs; mempool_t meta_pool; struct md_thread __rcu *reclaim_thread; unsigned long reclaim_target; /* number of space that need to be * reclaimed. if it's 0, reclaim spaces * used by io_units which are in * IO_UNIT_STRIPE_END state (eg, reclaim * doesn't wait for specific io_unit * switching to IO_UNIT_STRIPE_END * state) */ wait_queue_head_t iounit_wait; struct list_head no_space_stripes; /* pending stripes, log has no space */ spinlock_t no_space_stripes_lock; bool need_cache_flush; /* for r5c_cache */ enum r5c_journal_mode r5c_journal_mode; /* all stripes in r5cache, in the order of seq at sh->log_start */ struct list_head stripe_in_journal_list; spinlock_t stripe_in_journal_lock; atomic_t stripe_in_journal_count; /* to submit async io_units, to fulfill ordering of flush */ struct work_struct deferred_io_work; /* to disable write back during in degraded mode */ struct work_struct disable_writeback_work; /* to for chunk_aligned_read in writeback mode, details below */ spinlock_t tree_lock; struct radix_tree_root big_stripe_tree; }; /* * Enable chunk_aligned_read() with write back cache. * * Each chunk may contain more than one stripe (for example, a 256kB * chunk contains 64 4kB-page, so this chunk contain 64 stripes). For * chunk_aligned_read, these stripes are grouped into one "big_stripe". * For each big_stripe, we count how many stripes of this big_stripe * are in the write back cache. These data are tracked in a radix tree * (big_stripe_tree). We use radix_tree item pointer as the counter. * r5c_tree_index() is used to calculate keys for the radix tree. * * chunk_aligned_read() calls r5c_big_stripe_cached() to look up * big_stripe of each chunk in the tree. If this big_stripe is in the * tree, chunk_aligned_read() aborts. This look up is protected by * rcu_read_lock(). * * It is necessary to remember whether a stripe is counted in * big_stripe_tree. Instead of adding new flag, we reuses existing flags: * STRIPE_R5C_PARTIAL_STRIPE and STRIPE_R5C_FULL_STRIPE. If either of these * two flags are set, the stripe is counted in big_stripe_tree. This * requires moving set_bit(STRIPE_R5C_PARTIAL_STRIPE) to * r5c_try_caching_write(); and moving clear_bit of * STRIPE_R5C_PARTIAL_STRIPE and STRIPE_R5C_FULL_STRIPE to * r5c_finish_stripe_write_out(). */ /* * radix tree requests lowest 2 bits of data pointer to be 2b'00. * So it is necessary to left shift the counter by 2 bits before using it * as data pointer of the tree. */ #define R5C_RADIX_COUNT_SHIFT 2 /* * calculate key for big_stripe_tree * * sect: align_bi->bi_iter.bi_sector or sh->sector */ static inline sector_t r5c_tree_index(struct r5conf *conf, sector_t sect) { sector_div(sect, conf->chunk_sectors); return sect; } /* * an IO range starts from a meta data block and end at the next meta data * block. The io unit's the meta data block tracks data/parity followed it. io * unit is written to log disk with normal write, as we always flush log disk * first and then start move data to raid disks, there is no requirement to * write io unit with FLUSH/FUA */ struct r5l_io_unit { struct r5l_log *log; struct page *meta_page; /* store meta block */ int meta_offset; /* current offset in meta_page */ struct bio *current_bio;/* current_bio accepting new data */ atomic_t pending_stripe;/* how many stripes not flushed to raid */ u64 seq; /* seq number of the metablock */ sector_t log_start; /* where the io_unit starts */ sector_t log_end; /* where the io_unit ends */ struct list_head log_sibling; /* log->running_ios */ struct list_head stripe_list; /* stripes added to the io_unit */ int state; bool need_split_bio; struct bio *split_bio; unsigned int has_flush:1; /* include flush request */ unsigned int has_fua:1; /* include fua request */ unsigned int has_null_flush:1; /* include null flush request */ unsigned int has_flush_payload:1; /* include flush payload */ /* * io isn't sent yet, flush/fua request can only be submitted till it's * the first IO in running_ios list */ unsigned int io_deferred:1; struct bio_list flush_barriers; /* size == 0 flush bios */ }; /* r5l_io_unit state */ enum r5l_io_unit_state { IO_UNIT_RUNNING = 0, /* accepting new IO */ IO_UNIT_IO_START = 1, /* io_unit bio start writing to log, * don't accepting new bio */ IO_UNIT_IO_END = 2, /* io_unit bio finish writing to log */ IO_UNIT_STRIPE_END = 3, /* stripes data finished writing to raid */ }; bool r5c_is_writeback(struct r5l_log *log) { return (log != NULL && log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK); } static sector_t r5l_ring_add(struct r5l_log *log, sector_t start, sector_t inc) { start += inc; if (start >= log->device_size) start = start - log->device_size; return start; } static sector_t r5l_ring_distance(struct r5l_log *log, sector_t start, sector_t end) { if (end >= start) return end - start; else return end + log->device_size - start; } static bool r5l_has_free_space(struct r5l_log *log, sector_t size) { sector_t used_size; used_size = r5l_ring_distance(log, log->last_checkpoint, log->log_start); return log->device_size > used_size + size; } static void __r5l_set_io_unit_state(struct r5l_io_unit *io, enum r5l_io_unit_state state) { if (WARN_ON(io->state >= state)) return; io->state = state; } static void r5c_return_dev_pending_writes(struct r5conf *conf, struct r5dev *dev) { struct bio *wbi, *wbi2; wbi = dev->written; dev->written = NULL; while (wbi && wbi->bi_iter.bi_sector < dev->sector + RAID5_STRIPE_SECTORS(conf)) { wbi2 = r5_next_bio(conf, wbi, dev->sector); md_write_end(conf->mddev); bio_endio(wbi); wbi = wbi2; } } void r5c_handle_cached_data_endio(struct r5conf *conf, struct stripe_head *sh, int disks) { int i; for (i = sh->disks; i--; ) { if (sh->dev[i].written) { set_bit(R5_UPTODATE, &sh->dev[i].flags); r5c_return_dev_pending_writes(conf, &sh->dev[i]); } } } void r5l_wake_reclaim(struct r5l_log *log, sector_t space); /* Check whether we should flush some stripes to free up stripe cache */ void r5c_check_stripe_cache_usage(struct r5conf *conf) { int total_cached; struct r5l_log *log = READ_ONCE(conf->log); if (!r5c_is_writeback(log)) return; total_cached = atomic_read(&conf->r5c_cached_partial_stripes) + atomic_read(&conf->r5c_cached_full_stripes); /* * The following condition is true for either of the following: * - stripe cache pressure high: * total_cached > 3/4 min_nr_stripes || * empty_inactive_list_nr > 0 * - stripe cache pressure moderate: * total_cached > 1/2 min_nr_stripes */ if (total_cached > conf->min_nr_stripes * 1 / 2 || atomic_read(&conf->empty_inactive_list_nr) > 0) r5l_wake_reclaim(log, 0); } /* * flush cache when there are R5C_FULL_STRIPE_FLUSH_BATCH or more full * stripes in the cache */ void r5c_check_cached_full_stripe(struct r5conf *conf) { struct r5l_log *log = READ_ONCE(conf->log); if (!r5c_is_writeback(log)) return; /* * wake up reclaim for R5C_FULL_STRIPE_FLUSH_BATCH cached stripes * or a full stripe (chunk size / 4k stripes). */ if (atomic_read(&conf->r5c_cached_full_stripes) >= min(R5C_FULL_STRIPE_FLUSH_BATCH(conf), conf->chunk_sectors >> RAID5_STRIPE_SHIFT(conf))) r5l_wake_reclaim(log, 0); } /* * Total log space (in sectors) needed to flush all data in cache * * To avoid deadlock due to log space, it is necessary to reserve log * space to flush critical stripes (stripes that occupying log space near * last_checkpoint). This function helps check how much log space is * required to flush all cached stripes. * * To reduce log space requirements, two mechanisms are used to give cache * flush higher priorities: * 1. In handle_stripe_dirtying() and schedule_reconstruction(), * stripes ALREADY in journal can be flushed w/o pending writes; * 2. In r5l_write_stripe() and r5c_cache_data(), stripes NOT in journal * can be delayed (r5l_add_no_space_stripe). * * In cache flush, the stripe goes through 1 and then 2. For a stripe that * already passed 1, flushing it requires at most (conf->max_degraded + 1) * pages of journal space. For stripes that has not passed 1, flushing it * requires (conf->raid_disks + 1) pages of journal space. There are at * most (conf->group_cnt + 1) stripe that passed 1. So total journal space * required to flush all cached stripes (in pages) is: * * (stripe_in_journal_count - group_cnt - 1) * (max_degraded + 1) + * (group_cnt + 1) * (raid_disks + 1) * or * (stripe_in_journal_count) * (max_degraded + 1) + * (group_cnt + 1) * (raid_disks - max_degraded) */ static sector_t r5c_log_required_to_flush_cache(struct r5conf *conf) { struct r5l_log *log = READ_ONCE(conf->log); if (!r5c_is_writeback(log)) return 0; return BLOCK_SECTORS * ((conf->max_degraded + 1) * atomic_read(&log->stripe_in_journal_count) + (conf->raid_disks - conf->max_degraded) * (conf->group_cnt + 1)); } /* * evaluate log space usage and update R5C_LOG_TIGHT and R5C_LOG_CRITICAL * * R5C_LOG_TIGHT is set when free space on the log device is less than 3x of * reclaim_required_space. R5C_LOG_CRITICAL is set when free space on the log * device is less than 2x of reclaim_required_space. */ static inline void r5c_update_log_state(struct r5l_log *log) { struct r5conf *conf = log->rdev->mddev->private; sector_t free_space; sector_t reclaim_space; bool wake_reclaim = false; if (!r5c_is_writeback(log)) return; free_space = r5l_ring_distance(log, log->log_start, log->last_checkpoint); reclaim_space = r5c_log_required_to_flush_cache(conf); if (free_space < 2 * reclaim_space) set_bit(R5C_LOG_CRITICAL, &conf->cache_state); else { if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state)) wake_reclaim = true; clear_bit(R5C_LOG_CRITICAL, &conf->cache_state); } if (free_space < 3 * reclaim_space) set_bit(R5C_LOG_TIGHT, &conf->cache_state); else clear_bit(R5C_LOG_TIGHT, &conf->cache_state); if (wake_reclaim) r5l_wake_reclaim(log, 0); } /* * Put the stripe into writing-out phase by clearing STRIPE_R5C_CACHING. * This function should only be called in write-back mode. */ void r5c_make_stripe_write_out(struct stripe_head *sh) { struct r5conf *conf = sh->raid_conf; struct r5l_log *log = READ_ONCE(conf->log); BUG_ON(!r5c_is_writeback(log)); WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state)); clear_bit(STRIPE_R5C_CACHING, &sh->state); if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) atomic_inc(&conf->preread_active_stripes); } static void r5c_handle_data_cached(struct stripe_head *sh) { int i; for (i = sh->disks; i--; ) if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) { set_bit(R5_InJournal, &sh->dev[i].flags); clear_bit(R5_LOCKED, &sh->dev[i].flags); } clear_bit(STRIPE_LOG_TRAPPED, &sh->state); } /* * this journal write must contain full parity, * it may also contain some data pages */ static void r5c_handle_parity_cached(struct stripe_head *sh) { int i; for (i = sh->disks; i--; ) if (test_bit(R5_InJournal, &sh->dev[i].flags)) set_bit(R5_Wantwrite, &sh->dev[i].flags); } /* * Setting proper flags after writing (or flushing) data and/or parity to the * log device. This is called from r5l_log_endio() or r5l_log_flush_endio(). */ static void r5c_finish_cache_stripe(struct stripe_head *sh) { struct r5l_log *log = READ_ONCE(sh->raid_conf->log); if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) { BUG_ON(test_bit(STRIPE_R5C_CACHING, &sh->state)); /* * Set R5_InJournal for parity dev[pd_idx]. This means * all data AND parity in the journal. For RAID 6, it is * NOT necessary to set the flag for dev[qd_idx], as the