// SPDX-License-Identifier: GPL-2.0/* * background writeback - scan btree for dirty data and write it to the backing * device * * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com> * Copyright 2012 Google, Inc. */ #include "bcache.h"#include "btree.h"#include "debug.h"#include "writeback.h" #include <linux/delay.h>#include <linux/kthread.h>#include <linux/sched/clock.h>#include <trace/events/bcache.h> static void update_gc_after_writeback(struct cache_set *c){ if (c->gc_after_writeback != (BCH_ENABLE_AUTO_GC) || c->gc_stats.in_use < BCH_AUTO_GC_DIRTY_THRESHOLD) return; c->gc_after_writeback |= BCH_DO_AUTO_GC;} /* Rate limiting */static uint64_t __calc_target_rate(struct cached_dev *dc){ struct cache_set *c = dc->disk.c; /* * This is the size of the cache, minus the amount used for * flash-only devices */ uint64_t cache_sectors = c->nbuckets * c->cache->sb.bucket_size - atomic_long_read(&c->flash_dev_dirty_sectors); /* * Unfortunately there is no control of global dirty data. If the * user states that they want 10% dirty data in the cache, and has, * e.g., 5 backing volumes of equal size, we try and ensure each * backing volume uses about 2% of the cache for dirty data. */ uint32_t bdev_share = div64_u64(bdev_sectors(dc->bdev) << WRITEBACK_SHARE_SHIFT, c->cached_dev_sectors); uint64_t cache_dirty_target = div_u64(cache_sectors * dc->writeback_percent, 100); /* Ensure each backing dev gets at least one dirty share */ if (bdev_share < 1) bdev_share = 1; return (cache_dirty_target * bdev_share) >> WRITEBACK_SHARE_SHIFT;} static void __update_writeback_rate(struct cached_dev *dc){ /* * PI controller: * Figures out the amount that should be written per second. * * First, the error (number of sectors that are dirty beyond our * target) is calculated. The error is accumulated (numerically * integrated). * * Then, the proportional value and integral value are scaled * based on configured values. These are stored as inverses to * avoid fixed point math and to make configuration easy-- e.g. * the default value of 40 for writeback_rate_p_term_inverse * attempts to write at a rate that would retire all the dirty * blocks in 40 seconds. * * The writeback_rate_i_inverse value of 10000 means that 1/10000th * of the error is accumulated in the integral term per second. * This acts as a slow, long-term average that is not subject to * variations in usage like the p term. */ int64_t target = __calc_target_rate(dc); int64_t dirty = bcache_dev_sectors_dirty(&dc->disk); int64_t error = dirty - target; int64_t proportional_scaled = div_s64(error, dc->writeback_rate_p_term_inverse); int64_t integral_scaled; uint32_t new_rate; /* * We need to consider the number of dirty buckets as well * when calculating the proportional_scaled, Otherwise we might * have an unreasonable small writeback rate at a highly fragmented situation * when very few dirty sectors consumed a lot dirty buckets, the * worst case is when dirty buckets reached cutoff_writeback_sync and * dirty data is still not even reached to writeback percent, so the rate * still will be at the minimum value, which will cause the write * stuck at a non-writeback mode. */ struct cache_set *c = dc->disk.c; int64_t dirty_buckets = c->nbuckets - c->avail_nbuckets; if (dc->writeback_consider_fragment && c->gc_stats.in_use > BCH_WRITEBACK_FRAGMENT_THRESHOLD_LOW && dirty > 0) { int64_t fragment = div_s64((dirty_buckets * c->cache->sb.bucket_size), dirty); int64_t fp_term; int64_t fps; if (c->gc_stats.in_use <= BCH_WRITEBACK_FRAGMENT_THRESHOLD_MID) { fp_term = (int64_t)dc->writeback_rate_fp_term_low * (c->gc_stats.in_use - BCH_WRITEBACK_FRAGMENT_THRESHOLD_LOW); } else if (c->gc_stats.in_use <= BCH_WRITEBACK_FRAGMENT_THRESHOLD_HIGH) { fp_term = (int64_t)dc->writeback_rate_fp_term_mid * (c->gc_stats.in_use - BCH_WRITEBACK_FRAGMENT_THRESHOLD_MID); } else { fp_term = (int64_t)dc->writeback_rate_fp_term_high * (c->gc_stats.in_use - BCH_WRITEBACK_FRAGMENT_THRESHOLD_HIGH); } fps = div_s64(dirty, dirty_buckets) * fp_term; if (fragment > 3 && fps > proportional_scaled) { /* Only overrite the p when fragment > 3 */ proportional_scaled = fps; } } if ((error < 0 && dc->writeback_rate_integral > 0) || (error > 0 && time_before64(local_clock(), dc->writeback_rate.next + NSEC_PER_MSEC))) { /* * Only decrease the integral term if it's more than * zero. Only increase the integral term if the device * is keeping up. (Don't wind up the integral * ineffectively in either case). * * It's necessary to scale this by * writeback_rate_update_seconds to keep the integral * term dimensioned properly. */ dc->writeback_rate_integral += error * dc->writeback_rate_update_seconds; } integral_scaled = div_s64(dc->writeback_rate_integral, dc->writeback_rate_i_term_inverse); new_rate = clamp_t(int32_t, (proportional_scaled + integral_scaled), dc->writeback_rate_minimum, NSEC_PER_SEC); dc->writeback_rate_proportional = proportional_scaled; dc->writeback_rate_integral_scaled = integral_scaled; dc->writeback_rate_change = new_rate - atomic_long_read(&dc->writeback_rate.rate); atomic_long_set(&dc->writeback_rate.rate, new_rate); dc->writeback_rate_target = target;} static bool set_at_max_writeback_rate(struct cache_set *c, struct cached_dev *dc){ /* Don't sst max writeback rate if it is disabled */ if (!c->idle_max_writeback_rate_enabled) return false; /* Don't set max writeback rate if gc is running */ if (!c->gc_mark_valid) return false; /* * Idle_counter is increased everytime when update_writeback_rate() is * called. If all backing devices attached to the same cache set have * identical dc->writeback_rate_update_seconds values, it is about 6 * rounds of update_writeback_rate() on each backing device before * c->at_max_writeback_rate is set to 1, and then max wrteback rate set * to each dc->writeback_rate.rate. * In order to avoid extra locking cost for counting exact dirty cached * devices number, c->attached_dev_nr is used to calculate the idle * throushold. It might be bigger if not all cached device are in write- * back mode, but it still works well with limited extra rounds of * update_writeback_rate(). */ if (atomic_inc_return(&c->idle_counter) < atomic_read(&c->attached_dev_nr) * 6) return false; if (atomic_read(&c->at_max_writeback_rate) != 1) atomic_set(&c->at_max_writeback_rate, 1); atomic_long_set(&dc->writeback_rate.rate, INT_MAX); /* keep writeback_rate_target as existing value */ dc->writeback_rate_proportional = 0; dc->writeback_rate_integral_scaled = 0; dc->writeback_rate_change = 0; /* * Check c->idle_counter and c->at_max_writeback_rate agagain in case * new I/O arrives during before set_at_max_writeback_rate() returns. * Then the writeback rate is set to 1, and its new value should be * decided via __update_writeback_rate(). */ if ((atomic_read(&c->idle_counter) < atomic_read(&c->attached_dev_nr) * 6) || !atomic_read(&c->at_max_writeback_rate)) return false; return true;} static void update_writeback_rate(struct work_struct *work){ struct cached_dev *dc = container_of(to_delayed_work(work), struct cached_dev, writeback_rate_update); struct cache_set *c = dc->disk.c; /* * should check BCACHE_DEV_RATE_DW_RUNNING before calling * cancel_delayed_work_sync(). */ set_bit(BCACHE_DEV_RATE_DW_RUNNING, &dc->disk.flags); /* paired with where BCACHE_DEV_RATE_DW_RUNNING is tested */ smp_mb__after_atomic(); /* * CACHE_SET_IO_DISABLE might be set via sysfs interface, * check it here too. */ if (!test_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags) || test_bit(CACHE_SET_IO_DISABLE, &c->flags)) { clear_bit(BCACHE_DEV_RATE_DW_RUNNING, &dc->disk.flags); /* paired with where BCACHE_DEV_RATE_DW_RUNNING is tested */ smp_mb__after_atomic(); return; } if (atomic_read(&dc->has_dirty) && dc->writeback_percent) { /* * If the whole cache set is idle, set_at_max_writeback_rate() * will set writeback rate to a max number. Then it is * unncessary to update writeback rate for an idle cache set * in maximum writeback rate number(s). */ if (!set_at_max_writeback_rate(c, dc)) { down_read(&dc->writeback_lock); __update_writeback_rate(dc); update_gc_after_writeback(c); up_read(&dc->writeback_lock); } } /* * CACHE_SET_IO_DISABLE might be set via sysfs interface, * check it here too. */ if (test_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags) && !test_bit(CACHE_SET_IO_DISABLE, &c->flags)) { schedule_delayed_work(&dc->writeback_rate_update, dc->writeback_rate_update_seconds * HZ); } /* * should check BCACHE_DEV_RATE_DW_RUNNING before calling * cancel_delayed_work_sync(). */ clear_bit(BCACHE_DEV_RATE_DW_RUNNING, &dc->disk.flags); /* paired with where BCACHE_DEV_RATE_DW_RUNNING is tested */ smp_mb__after_atomic();} static unsigned int writeback_delay(struct cached_dev *dc, unsigned int sectors){ if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) || !dc->writeback_percent) return 0; return bch_next_delay(&dc->writeback_rate, sectors);} struct dirty_io { struct closure cl; struct cached_dev *dc; uint16_t sequence; struct bio bio;}; static void dirty_init(struct keybuf_key *w){ struct dirty_io *io = w->private; struct bio *bio = &io->bio; bio_init(bio, bio->bi_inline_vecs, DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS)); if (!io->dc->writeback_percent) bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0)); bio->bi_iter.bi_size = KEY_SIZE(&w->key) << 9; bio->bi_private = w; bch_bio_map(bio, NULL);} static void dirty_io_destructor(struct closure *cl){ struct dirty_io *io = container_of(cl, struct dirty_io, cl); kfree(io);} static void write_dirty_finish(struct closure *cl){ struct dirty_io *io = container_of(cl, struct dirty_io, cl); struct keybuf_key *w = io->bio.bi_private; struct cached_dev *dc = io->dc; bio_free_pages(&io->bio); /* This is kind of a dumb way of signalling errors. */ if (KEY_DIRTY(&w->key)) { int ret; unsigned int i; struct keylist keys; bch_keylist_init(&keys); bkey_copy(keys.top, &w->key); SET_KEY_DIRTY(keys.top, false); bch_keylist_push(&keys); for (i = 0; i < KEY_PTRS(&w->key); i++) atomic_inc(&PTR_BUCKET(dc->disk.c, &w->key, i)->pin); ret = bch_btree_insert(dc->disk.c, &keys, NULL, &w->key); if (ret) trace_bcache_writeback_collision(&w->key); atomic_long_inc(ret ? &dc->disk.c->writeback_keys_failed : &dc->disk.c->writeback_keys_done); } bch_keybuf_del(&dc->writeback_keys, w); up(&dc->in_flight); closure_return_with_destructor(cl, dirty_io_destructor);} static void dirty_endio(struct bio *bio){ struct keybuf_key *w = bio->bi_private; struct dirty_io *io = w->private; if (bio->bi_status) { SET_KEY_DIRTY(&w->key, false); bch_count_backing_io_errors(io->dc, bio); } closure_put(&io->cl);} static void write_dirty(struct closure *cl){ struct dirty_io *io = container_of(cl, struct dirty_io, cl); struct keybuf_key *w = io->bio.bi_private; struct cached_dev *dc = io->dc; uint16_t next_sequence; if (atomic_read(&dc->writeback_sequence_next) != io->sequence) { /* Not our turn to write; wait for a write to complete */ closure_wait(&dc->writeback_ordering_wait, cl); if (atomic_read(&dc->writeback_sequence_next) == io->sequence) { /* * Edge case-- it happened in indeterminate order * relative to when we were added to wait list.. */ closure_wake_up(&dc->writeback_ordering_wait); } continue_at(cl, write_dirty, io->dc->writeback_write_wq); return; } next_sequence = io->sequence + 1; /* * IO errors are signalled using the dirty bit on the key. * If we failed to read, we should not attempt to write to the * backing device. Instead, immediately go to write_dirty_finish * to clean up. */ if (KEY_DIRTY(&w->key)) { dirty_init(w); bio_set_op_attrs(&io->bio, REQ_OP_WRITE, 0); io->bio.bi_iter.bi_sector = KEY_START(&w->key); bio_set_dev(&io->bio, io->dc->bdev); io->bio.bi_end_io = dirty_endio; /* I/O request sent to backing device */ closure_bio_submit(io->dc->disk.c, &io->bio, cl); } atomic_set(&dc->writeback_sequence_next, next_sequence); closure_wake_up(&dc->writeback_ordering_wait); continue_at(cl, write_dirty_finish, io->dc->writeback_write_wq);} static void read_dirty_endio(struct bio *bio){ struct keybuf_key *w = bio->bi_private; struct dirty_io *io = w->private; /* is_read = 1 */ bch_count_io_errors(io->dc->disk.c->cache, bio->bi_status, 1, "reading dirty data from cache"); dirty_endio(bio);} static void read_dirty_submit(struct closure *cl){ struct dirty_io *io = container_of(cl, struct dirty_io, cl); closure_bio_submit(io->dc->disk.c, &io->bio, cl); continue_at(cl, write_dirty, io->dc->writeback_write_wq);} static void read_dirty(struct cached_dev *dc){ unsigned int delay = 0; struct keybuf_key *next, *keys[MAX_WRITEBACKS_IN_PASS], *w; size_t size; int nk, i; struct dirty_io *io; struct closure cl; uint16_t sequence = 0; BUG_ON(!llist_empty(&dc->writeback_ordering_wait.list)); atomic_set(&dc->writeback_sequence_next, sequence); closure_init_stack(&cl); /* * XXX: if we error, background writeback just spins. Should use some * mempools. */ next = bch_keybuf_next(&dc->writeback_keys); while (!kthread_should_stop() && !test_bit(CACHE_SET_IO_DISABLE, &dc->disk.c->flags) && next) { size = 0; nk = 0; do { BUG_ON(ptr_stale(dc->disk.c, &next->key, 0)); /* * Don't combine too many operations, even if they * are all small. */ if (nk >= MAX_WRITEBACKS_IN_PASS) break; /* * If the current operation is very large, don't * further combine operations. */ if (size >= MAX_WRITESIZE_IN_PASS) break; /* * Operations are only eligible to be combined * if they are contiguous. * * TODO: add a heuristic willing to fire a * certain amount of non-contiguous IO per pass, * so that we can benefit from backing device * command queueing. */ if ((nk != 0) && bkey_cmp(&keys[nk-1]->key, &START_KEY(&next->key))) break; size += KEY_SIZE(&next->key); keys[nk++] = next; } while ((next = bch_keybuf_next(&dc->writeback_keys))); /* Now we have gathered a set of 1..5 keys to write back. */ for (i = 0; i < nk; i++) { w = keys[i]; io = kzalloc(struct_size(io, bio.bi_inline_vecs,Lines 1–500 of 1058. Load more
// SPDX-License-Identifier: GPL-2.0 /* * background writeback - scan btree for dirty data and write it to the backing * device * * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com> * Copyright 2012 Google, Inc. */ #include "bcache.h" #include "btree.h" #include "debug.h" #include "writeback.h" #include <linux/delay.h> #include <linux/kthread.h> #include <linux/sched/clock.h> #include <trace/events/bcache.h> static void update_gc_after_writeback(struct cache_set *c) { if (c->gc_after_writeback != (BCH_ENABLE_AUTO_GC) || c->gc_stats.in_use < BCH_AUTO_GC_DIRTY_THRESHOLD) return; c->gc_after_writeback |= BCH_DO_AUTO_GC; } /* Rate limiting */ static uint64_t __calc_target_rate(struct cached_dev *dc) { struct cache_set *c = dc->disk.c; /* * This is the size of the cache, minus the amount used for * flash-only devices */ uint64_t cache_sectors = c->nbuckets * c->cache->sb.bucket_size - atomic_long_read(&c->flash_dev_dirty_sectors); /* * Unfortunately there is no control of global dirty data. If the * user states that they want 10% dirty data in the cache, and has, * e.g., 5 backing volumes of equal size, we try and ensure each * backing volume uses about 2% of the cache for dirty data. */ uint32_t bdev_share = div64_u64(bdev_sectors(dc->bdev) << WRITEBACK_SHARE_SHIFT, c->cached_dev_sectors); uint64_t cache_dirty_target = div_u64(cache_sectors * dc->writeback_percent, 100); /* Ensure each backing dev gets at least one dirty share */ if (bdev_share < 1) bdev_share = 1; return (cache_dirty_target * bdev_share) >> WRITEBACK_SHARE_SHIFT; } static void __update_writeback_rate(struct cached_dev *dc) { /* * PI controller: * Figures out the amount that should be written per second. * * First, the error (number of sectors that are dirty beyond our * target) is calculated. The error is accumulated (numerically * integrated). * * Then, the proportional value and integral value are scaled * based on configured values. These are stored as inverses to * avoid fixed point math and to make configuration easy-- e.g. * the default value of 40 for writeback_rate_p_term_inverse * attempts to write at a rate that would retire all the dirty * blocks in 40 seconds. * * The writeback_rate_i_inverse value of 10000 means that 1/10000th * of the error is accumulated in the integral term per second. * This acts as a slow, long-term average that is not subject to * variations in usage like the p term. */ int64_t target = __calc_target_rate(dc); int64_t dirty = bcache_dev_sectors_dirty(&dc->disk); int64_t error = dirty - target; int64_t proportional_scaled = div_s64(error, dc->writeback_rate_p_term_inverse); int64_t integral_scaled; uint32_t new_rate; /* * We need to consider the number of dirty buckets as well * when calculating the proportional_scaled, Otherwise we might * have an unreasonable small writeback rate at a highly fragmented situation * when very few dirty sectors consumed a lot dirty buckets, the * worst case is when dirty buckets reached cutoff_writeback_sync and * dirty data is still not even reached to writeback percent, so the rate * still will be at the minimum value, which will cause the write * stuck at a non-writeback mode. */ struct cache_set *c = dc->disk.c; int64_t dirty_buckets = c->nbuckets - c->avail_nbuckets; if (dc->writeback_consider_fragment && c->gc_stats.in_use > BCH_WRITEBACK_FRAGMENT_THRESHOLD_LOW && dirty > 0) { int64_t fragment = div_s64((dirty_buckets * c->cache->sb.bucket_size), dirty); int64_t fp_term; int64_t fps; if (c->gc_stats.in_use <= BCH_WRITEBACK_FRAGMENT_THRESHOLD_MID) { fp_term = (int64_t)dc->writeback_rate_fp_term_low * (c->gc_stats.in_use - BCH_WRITEBACK_FRAGMENT_THRESHOLD_LOW); } else if (c->gc_stats.in_use <= BCH_WRITEBACK_FRAGMENT_THRESHOLD_HIGH) { fp_term = (int64_t)dc->writeback_rate_fp_term_mid * (c->gc_stats.in_use - BCH_WRITEBACK_FRAGMENT_THRESHOLD_MID); } else { fp_term = (int64_t)dc->writeback_rate_fp_term_high * (c->gc_stats.in_use - BCH_WRITEBACK_FRAGMENT_THRESHOLD_HIGH); } fps = div_s64(dirty, dirty_buckets) * fp_term; if (fragment > 3 && fps > proportional_scaled) { /* Only overrite the p when fragment > 3 */ proportional_scaled = fps; } } if ((error < 0 && dc->writeback_rate_integral > 0) || (error > 0 && time_before64(local_clock(), dc->writeback_rate.next + NSEC_PER_MSEC))) { /* * Only decrease the integral term if it's more than * zero. Only increase the integral term if the device * is keeping up. (Don't wind up the integral * ineffectively in either case). * * It's necessary to scale this by * writeback_rate_update_seconds to keep the integral * term dimensioned properly. */ dc->writeback_rate_integral += error * dc->writeback_rate_update_seconds; } integral_scaled = div_s64(dc->writeback_rate_integral, dc->writeback_rate_i_term_inverse); new_rate = clamp_t(int32_t, (proportional_scaled + integral_scaled), dc->writeback_rate_minimum, NSEC_PER_SEC); dc->writeback_rate_proportional = proportional_scaled; dc->writeback_rate_integral_scaled = integral_scaled; dc->writeback_rate_change = new_rate - atomic_long_read(&dc->writeback_rate.rate); atomic_long_set(&dc->writeback_rate.rate, new_rate); dc->writeback_rate_target = target; } static bool set_at_max_writeback_rate(struct cache_set *c, struct cached_dev *dc) { /* Don't sst max writeback rate if it is disabled */ if (!c->idle_max_writeback_rate_enabled) return false; /* Don't set max writeback rate if gc is running */ if (!c->gc_mark_valid) return false; /* * Idle_counter is increased everytime when update_writeback_rate() is * called. If all backing devices attached to the same cache set have * identical dc->writeback_rate_update_seconds values, it is about 6 * rounds of update_writeback_rate() on each backing device before * c->at_max_writeback_rate is set to 1, and then max wrteback rate set * to each dc->writeback_rate.rate. * In order to avoid extra locking cost for counting exact dirty cached * devices number, c->attached_dev_nr is used to calculate the idle * throushold. It might be bigger if not all cached device are in write- * back mode, but it still works well with limited extra rounds of * update_writeback_rate(). */ if (atomic_inc_return(&c->idle_counter) < atomic_read(&c->attached_dev_nr) * 6) return false; if (atomic_read(&c->at_max_writeback_rate) != 1) atomic_set(&c->at_max_writeback_rate, 1); atomic_long_set(&dc->writeback_rate.rate, INT_MAX); /* keep writeback_rate_target as existing value */ dc->writeback_rate_proportional = 0; dc->writeback_rate_integral_scaled = 0; dc->writeback_rate_change = 0; /* * Check c->idle_counter and c->at_max_writeback_rate agagain in case * new I/O arrives during before set_at_max_writeback_rate() returns. * Then the writeback rate is set to 1, and its new value should be * decided via __update_writeback_rate(). */ if ((atomic_read(&c->idle_counter) < atomic_read(&c->attached_dev_nr) * 6) || !atomic_read(&c->at_max_writeback_rate)) return false; return true; } static void update_writeback_rate(struct work_struct *work) { struct cached_dev *dc = container_of(to_delayed_work(work), struct cached_dev, writeback_rate_update); struct cache_set *c = dc->disk.c; /* * should check BCACHE_DEV_RATE_DW_RUNNING before calling * cancel_delayed_work_sync(). */ set_bit(BCACHE_DEV_RATE_DW_RUNNING, &dc->disk.flags); /* paired with where BCACHE_DEV_RATE_DW_RUNNING is tested */ smp_mb__after_atomic(); /* * CACHE_SET_IO_DISABLE might be set via sysfs interface, * check it here too. */ if (!test_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags) || test_bit(CACHE_SET_IO_DISABLE, &c->flags)) { clear_bit(BCACHE_DEV_RATE_DW_RUNNING, &dc->disk.flags); /* paired with where BCACHE_DEV_RATE_DW_RUNNING is tested */ smp_mb__after_atomic(); return; } if (atomic_read(&dc->has_dirty) && dc->writeback_percent) { /* * If the whole cache set is idle, set_at_max_writeback_rate() * will set writeback rate to a max number. Then it is * unncessary to update writeback rate for an idle cache set * in maximum writeback rate number(s). */ if (!set_at_max_writeback_rate(c, dc)) { down_read(&dc->writeback_lock); __update_writeback_rate(dc); update_gc_after_writeback(c); up_read(&dc->writeback_lock); } } /* * CACHE_SET_IO_DISABLE might be set via sysfs interface, * check it here too. */ if (test_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags) && !test_bit(CACHE_SET_IO_DISABLE, &c->flags)) { schedule_delayed_work(&dc->writeback_rate_update, dc->writeback_rate_update_seconds * HZ); } /* * should check BCACHE_DEV_RATE_DW_RUNNING before calling * cancel_delayed_work_sync(). */ clear_bit(BCACHE_DEV_RATE_DW_RUNNING, &dc->disk.flags); /* paired with where BCACHE_DEV_RATE_DW_RUNNING is tested */ smp_mb__after_atomic(); } static unsigned int writeback_delay(struct cached_dev *dc, unsigned int sectors) { if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) || !dc->writeback_percent) return 0; return bch_next_delay(&dc->writeback_rate, sectors); } struct dirty_io { struct closure cl; struct cached_dev *dc; uint16_t sequence; struct bio bio; }; static void dirty_init(struct keybuf_key *w) { struct dirty_io *io = w->private; struct bio *bio = &io->bio; bio_init(bio, bio->bi_inline_vecs, DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS)); if (!io->dc->writeback_percent) bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0)); bio->bi_iter.bi_size = KEY_SIZE(&w->key) << 9; bio->bi_private = w; bch_bio_map(bio, NULL); } static void dirty_io_destructor(struct closure *cl) { struct dirty_io *io = container_of(cl, struct dirty_io, cl); kfree(io); } static void write_dirty_finish(struct closure *cl) { struct dirty_io *io = container_of(cl, struct dirty_io, cl); struct keybuf_key *w = io->bio.bi_private; struct cached_dev *dc = io->dc; bio_free_pages(&io->bio); /* This is kind of a dumb way of signalling errors. */ if (KEY_DIRTY(&w->key)) { int ret; unsigned int i; struct keylist keys; bch_keylist_init(&keys); bkey_copy(keys.top, &w->key); SET_KEY_DIRTY(keys.top, false); bch_keylist_push(&keys); for (i = 0; i < KEY_PTRS(&w->key); i++) atomic_inc(&PTR_BUCKET(dc->disk.c, &w->key, i)->pin); ret = bch_btree_insert(dc->disk.c, &keys, NULL, &w->key); if (ret) trace_bcache_writeback_collision(&w->key); atomic_long_inc(ret ? &dc->disk.c->writeback_keys_failed : &dc->disk.c->writeback_keys_done); } bch_keybuf_del(&dc->writeback_keys, w); up(&dc->in_flight); closure_return_with_destructor(cl, dirty_io_destructor); } static void dirty_endio(struct bio *bio) { struct keybuf_key *w = bio->bi_private; struct dirty_io *io = w->private; if (bio->bi_status) { SET_KEY_DIRTY(&w->key, false); bch_count_backing_io_errors(io->dc, bio); } closure_put(&io->cl); } static void write_dirty(struct closure *cl) { struct dirty_io *io = container_of(cl, struct dirty_io, cl); struct keybuf_key *w = io->bio.bi_private; struct cached_dev *dc = io->dc; uint16_t next_sequence; if (atomic_read(&dc->writeback_sequence_next) != io->sequence) { /* Not our turn to write; wait for a write to complete */ closure_wait(&dc->writeback_ordering_wait, cl); if (atomic_read(&dc->writeback_sequence_next) == io->sequence) { /* * Edge case-- it happened in indeterminate order * relative to when we were added to wait list.. */ closure_wake_up(&dc->writeback_ordering_wait); } continue_at(cl, write_dirty, io->dc->writeback_write_wq); return; } next_sequence = io->sequence + 1; /* * IO errors are signalled using the dirty bit on the key. * If we failed to read, we should not attempt to write to the * backing device. Instead, immediately go to write_dirty_finish * to clean up. */ if (KEY_DIRTY(&w->key)) { dirty_init(w); bio_set_op_attrs(&io->bio, REQ_OP_WRITE, 0); io->bio.bi_iter.bi_sector = KEY_START(&w->key); bio_set_dev(&io->bio, io->dc->bdev); io->bio.bi_end_io = dirty_endio; /* I/O request sent to backing device */ closure_bio_submit(io->dc->disk.c, &io->bio, cl); } atomic_set(&dc->writeback_sequence_next, next_sequence); closure_wake_up(&dc->writeback_ordering_wait); continue_at(cl, write_dirty_finish, io->dc->writeback_write_wq); } static void read_dirty_endio(struct bio *bio) { struct keybuf_key *w = bio->bi_private; struct dirty_io *io = w->private; /* is_read = 1 */ bch_count_io_errors(io->dc->disk.c->cache, bio->bi_status, 1, "reading dirty data from cache"); dirty_endio(bio); } static void read_dirty_submit(struct closure *cl) { struct dirty_io *io = container_of(cl, struct dirty_io, cl); closure_bio_submit(io->dc->disk.c, &io->bio, cl); continue_at(cl, write_dirty, io->dc->writeback_write_wq); } static void read_dirty(struct cached_dev *dc) { unsigned int delay = 0; struct keybuf_key *next, *keys[MAX_WRITEBACKS_IN_PASS], *w; size_t size; int nk, i; struct dirty_io *io; struct closure cl; uint16_t sequence = 0; BUG_ON(!llist_empty(&dc->writeback_ordering_wait.list)); atomic_set(&dc->writeback_sequence_next, sequence); closure_init_stack(&cl); /* * XXX: if we error, background writeback just spins. Should use some * mempools. */ next = bch_keybuf_next(&dc->writeback_keys); while (!kthread_should_stop() && !test_bit(CACHE_SET_IO_DISABLE, &dc->disk.c->flags) && next) { size = 0; nk = 0; do { BUG_ON(ptr_stale(dc->disk.c, &next->key, 0)); /* * Don't combine too many operations, even if they * are all small. */ if (nk >= MAX_WRITEBACKS_IN_PASS) break; /* * If the current operation is very large, don't * further combine operations. */ if (size >= MAX_WRITESIZE_IN_PASS) break; /* * Operations are only eligible to be combined * if they are contiguous. * * TODO: add a heuristic willing to fire a * certain amount of non-contiguous IO per pass, * so that we can benefit from backing device * command queueing. */ if ((nk != 0) && bkey_cmp(&keys[nk-1]->key, &START_KEY(&next->key))) break; size += KEY_SIZE(&next->key); keys[nk++] = next; } while ((next = bch_keybuf_next(&dc->writeback_keys))); /* Now we have gathered a set of 1..5 keys to write back. */ for (i = 0; i < nk; i++) { w = keys[i]; io = kzalloc(struct_size(io, bio.bi_inline_vecs,