// SPDX-License-Identifier: GPL-2.0/* * Main bcache entry point - handle a read or a write request and decide what to * do with it; the make_request functions are called by the block layer. * * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com> * Copyright 2012 Google, Inc. */ #include "bcache.h"#include "btree.h"#include "debug.h"#include "request.h"#include "writeback.h" #include <linux/module.h>#include <linux/hash.h>#include <linux/random.h>#include <linux/backing-dev.h> #include <trace/events/bcache.h> #define CUTOFF_CACHE_ADD 95#define CUTOFF_CACHE_READA 90 struct kmem_cache *bch_search_cache; static CLOSURE_CALLBACK(bch_data_insert_start); static unsigned int cache_mode(struct cached_dev *dc){ return BDEV_CACHE_MODE(&dc->sb);} static bool verify(struct cached_dev *dc){ return dc->verify;} static void bio_csum(struct bio *bio, struct bkey *k){ struct bio_vec bv; struct bvec_iter iter; uint64_t csum = 0; bio_for_each_segment(bv, bio, iter) { void *d = bvec_kmap_local(&bv); csum = crc64_be(csum, d, bv.bv_len); kunmap_local(d); } k->ptr[KEY_PTRS(k)] = csum & (~0ULL >> 1);} /* Insert data into cache */ static CLOSURE_CALLBACK(bch_data_insert_keys){ closure_type(op, struct data_insert_op, cl); atomic_t *journal_ref = NULL; struct bkey *replace_key = op->replace ? &op->replace_key : NULL; int ret; if (!op->replace) journal_ref = bch_journal(op->c, &op->insert_keys, op->flush_journal ? cl : NULL); ret = bch_btree_insert(op->c, &op->insert_keys, journal_ref, replace_key); if (ret == -ESRCH) { op->replace_collision = true; } else if (ret) { op->status = BLK_STS_RESOURCE; op->insert_data_done = true; } if (journal_ref) atomic_dec_bug(journal_ref); if (!op->insert_data_done) { continue_at(cl, bch_data_insert_start, op->wq); return; } bch_keylist_free(&op->insert_keys); closure_return(cl);} static int bch_keylist_realloc(struct keylist *l, unsigned int u64s, struct cache_set *c){ size_t oldsize = bch_keylist_nkeys(l); size_t newsize = oldsize + u64s; /* * The journalling code doesn't handle the case where the keys to insert * is bigger than an empty write: If we just return -ENOMEM here, * bch_data_insert_keys() will insert the keys created so far * and finish the rest when the keylist is empty. */ if (newsize * sizeof(uint64_t) > block_bytes(c->cache) - sizeof(struct jset)) return -ENOMEM; return __bch_keylist_realloc(l, u64s);} static void bch_data_invalidate(struct closure *cl){ struct data_insert_op *op = container_of(cl, struct data_insert_op, cl); struct bio *bio = op->bio; pr_debug("invalidating %i sectors from %llu\n", bio_sectors(bio), (uint64_t) bio->bi_iter.bi_sector); while (bio_sectors(bio)) { unsigned int sectors = min(bio_sectors(bio), 1U << (KEY_SIZE_BITS - 1)); if (bch_keylist_realloc(&op->insert_keys, 2, op->c)) goto out; bio->bi_iter.bi_sector += sectors; bio->bi_iter.bi_size -= sectors << 9; bch_keylist_add(&op->insert_keys, &KEY(op->inode, bio->bi_iter.bi_sector, sectors)); } op->insert_data_done = true; /* get in bch_data_insert() */ bio_put(bio);out: continue_at(cl, bch_data_insert_keys, op->wq);} static CLOSURE_CALLBACK(bch_data_insert_error){ closure_type(op, struct data_insert_op, cl); /* * Our data write just errored, which means we've got a bunch of keys to * insert that point to data that wasn't successfully written. * * We don't have to insert those keys but we still have to invalidate * that region of the cache - so, if we just strip off all the pointers * from the keys we'll accomplish just that. */ struct bkey *src = op->insert_keys.keys, *dst = op->insert_keys.keys; while (src != op->insert_keys.top) { struct bkey *n = bkey_next(src); SET_KEY_PTRS(src, 0); memmove(dst, src, bkey_bytes(src)); dst = bkey_next(dst); src = n; } op->insert_keys.top = dst; bch_data_insert_keys(&cl->work);} static void bch_data_insert_endio(struct bio *bio){ struct closure *cl = bio->bi_private; struct data_insert_op *op = container_of(cl, struct data_insert_op, cl); if (bio->bi_status) { /* TODO: We could try to recover from this. */ if (op->writeback) op->status = bio->bi_status; else if (!op->replace) set_closure_fn(cl, bch_data_insert_error, op->wq); else set_closure_fn(cl, NULL, NULL); } bch_bbio_endio(op->c, bio, bio->bi_status, "writing data to cache");} static CLOSURE_CALLBACK(bch_data_insert_start){ closure_type(op, struct data_insert_op, cl); struct bio *bio = op->bio, *n; if (op->bypass) return bch_data_invalidate(cl); if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0) wake_up_gc(op->c); /* * Journal writes are marked REQ_PREFLUSH; if the original write was a * flush, it'll wait on the journal write. */ bio->bi_opf &= ~(REQ_PREFLUSH|REQ_FUA); do { unsigned int i; struct bkey *k; struct bio_set *split = &op->c->bio_split; /* 1 for the device pointer and 1 for the chksum */ if (bch_keylist_realloc(&op->insert_keys, 3 + (op->csum ? 1 : 0), op->c)) { continue_at(cl, bch_data_insert_keys, op->wq); return; } k = op->insert_keys.top; bkey_init(k); SET_KEY_INODE(k, op->inode); SET_KEY_OFFSET(k, bio->bi_iter.bi_sector); if (!bch_alloc_sectors(op->c, k, bio_sectors(bio), op->write_point, op->write_prio, op->writeback)) goto err; n = bio_next_split(bio, KEY_SIZE(k), GFP_NOIO, split); n->bi_end_io = bch_data_insert_endio; n->bi_private = cl; if (op->writeback) { SET_KEY_DIRTY(k, true); for (i = 0; i < KEY_PTRS(k); i++) SET_GC_MARK(PTR_BUCKET(op->c, k, i), GC_MARK_DIRTY); } SET_KEY_CSUM(k, op->csum); if (KEY_CSUM(k)) bio_csum(n, k); trace_bcache_cache_insert(k); bch_keylist_push(&op->insert_keys); n->bi_opf = REQ_OP_WRITE; bch_submit_bbio(n, op->c, k, 0); } while (n != bio); op->insert_data_done = true; continue_at(cl, bch_data_insert_keys, op->wq); return;err: /* bch_alloc_sectors() blocks if s->writeback = true */ BUG_ON(op->writeback); /* * But if it's not a writeback write we'd rather just bail out if * there aren't any buckets ready to write to - it might take awhile and * we might be starving btree writes for gc or something. */ if (!op->replace) { /* * Writethrough write: We can't complete the write until we've * updated the index. But we don't want to delay the write while * we wait for buckets to be freed up, so just invalidate the * rest of the write. */ op->bypass = true; return bch_data_invalidate(cl); } else { /* * From a cache miss, we can just insert the keys for the data * we have written or bail out if we didn't do anything. */ op->insert_data_done = true; bio_put(bio); if (!bch_keylist_empty(&op->insert_keys)) continue_at(cl, bch_data_insert_keys, op->wq); else closure_return(cl); }} /** * bch_data_insert - stick some data in the cache * @cl: closure pointer. * * This is the starting point for any data to end up in a cache device; it could * be from a normal write, or a writeback write, or a write to a flash only * volume - it's also used by the moving garbage collector to compact data in * mostly empty buckets. * * It first writes the data to the cache, creating a list of keys to be inserted * (if the data had to be fragmented there will be multiple keys); after the * data is written it calls bch_journal, and after the keys have been added to * the next journal write they're inserted into the btree. * * It inserts the data in op->bio; bi_sector is used for the key offset, * and op->inode is used for the key inode. * * If op->bypass is true, instead of inserting the data it invalidates the * region of the cache represented by op->bio and op->inode. */CLOSURE_CALLBACK(bch_data_insert){ closure_type(op, struct data_insert_op, cl); trace_bcache_write(op->c, op->inode, op->bio, op->writeback, op->bypass); bch_keylist_init(&op->insert_keys); bio_get(op->bio); bch_data_insert_start(&cl->work);} /* * Congested? Return 0 (not congested) or the limit (in sectors) * beyond which we should bypass the cache due to congestion. */unsigned int bch_get_congested(const struct cache_set *c){ int i; if (!c->congested_read_threshold_us && !c->congested_write_threshold_us) return 0; i = (local_clock_us() - c->congested_last_us) / 1024; if (i < 0) return 0; i += atomic_read(&c->congested); if (i >= 0) return 0; i += CONGESTED_MAX; if (i > 0) i = fract_exp_two(i, 6); i -= hweight32(get_random_u32()); return i > 0 ? i : 1;} static void add_sequential(struct task_struct *t){ ewma_add(t->sequential_io_avg, t->sequential_io, 8, 0); t->sequential_io = 0;} static struct hlist_head *iohash(struct cached_dev *dc, uint64_t k){ return &dc->io_hash[hash_64(k, RECENT_IO_BITS)];} static bool check_should_bypass(struct cached_dev *dc, struct bio *bio){ struct cache_set *c = dc->disk.c; unsigned int mode = cache_mode(dc); unsigned int sectors, congested; struct task_struct *task = current; struct io *i; if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) || (bio_op(bio) == REQ_OP_DISCARD)) goto skip; if (c->gc_stats.in_use > CUTOFF_CACHE_ADD) { /* * If cached buckets are all clean now, 'true' will be * returned and all requests will bypass the cache device. * Then c->sectors_to_gc has no chance to be negative, and * gc thread won't wake up and caching won't work forever. * Here call force_wake_up_gc() to avoid such aftermath. */ if (BDEV_STATE(&dc->sb) == BDEV_STATE_CLEAN && c->gc_mark_valid) force_wake_up_gc(c); goto skip; } if (mode == CACHE_MODE_NONE || (mode == CACHE_MODE_WRITEAROUND && op_is_write(bio_op(bio)))) goto skip; /* * If the bio is for read-ahead or background IO, bypass it or * not depends on the following situations, * - If the IO is for meta data, always cache it and no bypass * - If the IO is not meta data, check dc->cache_reada_policy, * BCH_CACHE_READA_ALL: cache it and not bypass * BCH_CACHE_READA_META_ONLY: not cache it and bypass * That is, read-ahead request for metadata always get cached * (eg, for gfs2 or xfs). */ if ((bio->bi_opf & (REQ_RAHEAD|REQ_BACKGROUND))) { if (!(bio->bi_opf & (REQ_META|REQ_PRIO)) && (dc->cache_readahead_policy != BCH_CACHE_READA_ALL)) goto skip; } if (bio->bi_iter.bi_sector & (c->cache->sb.block_size - 1) || bio_sectors(bio) & (c->cache->sb.block_size - 1)) { pr_debug("skipping unaligned io\n"); goto skip; } if (bypass_torture_test(dc)) { if (get_random_u32_below(4) == 3) goto skip; else goto rescale; } congested = bch_get_congested(c); if (!congested && !dc->sequential_cutoff) goto rescale; spin_lock(&dc->io_lock); hlist_for_each_entry(i, iohash(dc, bio->bi_iter.bi_sector), hash) if (i->last == bio->bi_iter.bi_sector && time_before(jiffies, i->jiffies)) goto found; i = list_first_entry(&dc->io_lru, struct io, lru); add_sequential(task); i->sequential = 0;found: if (i->sequential + bio->bi_iter.bi_size > i->sequential) i->sequential += bio->bi_iter.bi_size; i->last = bio_end_sector(bio); i->jiffies = jiffies + msecs_to_jiffies(5000); task->sequential_io = i->sequential; hlist_del(&i->hash); hlist_add_head(&i->hash, iohash(dc, i->last)); list_move_tail(&i->lru, &dc->io_lru); spin_unlock(&dc->io_lock); sectors = max(task->sequential_io, task->sequential_io_avg) >> 9; if (dc->sequential_cutoff && sectors >= dc->sequential_cutoff >> 9) { trace_bcache_bypass_sequential(bio); goto skip; } if (congested && sectors >= congested) { trace_bcache_bypass_congested(bio); goto skip; } rescale: bch_rescale_priorities(c, bio_sectors(bio)); return false;skip: bch_mark_sectors_bypassed(c, dc, bio_sectors(bio)); return true;} /* Cache lookup */ struct search { /* Stack frame for bio_complete */ struct closure cl; struct bbio bio; struct bio *orig_bio; struct bio *cache_miss; struct bcache_device *d; unsigned int insert_bio_sectors; unsigned int recoverable:1; unsigned int write:1; unsigned int read_dirty_data:1; unsigned int cache_missed:1; struct block_device *orig_bdev; unsigned long start_time; struct btree_op op; struct data_insert_op iop;}; static void bch_cache_read_endio(struct bio *bio){Lines 1–500 of 1346. Load more
// SPDX-License-Identifier: GPL-2.0 /* * Main bcache entry point - handle a read or a write request and decide what to * do with it; the make_request functions are called by the block layer. * * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com> * Copyright 2012 Google, Inc. */ #include "bcache.h" #include "btree.h" #include "debug.h" #include "request.h" #include "writeback.h" #include <linux/module.h> #include <linux/hash.h> #include <linux/random.h> #include <linux/backing-dev.h> #include <trace/events/bcache.h> #define CUTOFF_CACHE_ADD 95 #define CUTOFF_CACHE_READA 90 struct kmem_cache *bch_search_cache; static CLOSURE_CALLBACK(bch_data_insert_start); static unsigned int cache_mode(struct cached_dev *dc) { return BDEV_CACHE_MODE(&dc->sb); } static bool verify(struct cached_dev *dc) { return dc->verify; } static void bio_csum(struct bio *bio, struct bkey *k) { struct bio_vec bv; struct bvec_iter iter; uint64_t csum = 0; bio_for_each_segment(bv, bio, iter) { void *d = bvec_kmap_local(&bv); csum = crc64_be(csum, d, bv.bv_len); kunmap_local(d); } k->ptr[KEY_PTRS(k)] = csum & (~0ULL >> 1); } /* Insert data into cache */ static CLOSURE_CALLBACK(bch_data_insert_keys) { closure_type(op, struct data_insert_op, cl); atomic_t *journal_ref = NULL; struct bkey *replace_key = op->replace ? &op->replace_key : NULL; int ret; if (!op->replace) journal_ref = bch_journal(op->c, &op->insert_keys, op->flush_journal ? cl : NULL); ret = bch_btree_insert(op->c, &op->insert_keys, journal_ref, replace_key); if (ret == -ESRCH) { op->replace_collision = true; } else if (ret) { op->status = BLK_STS_RESOURCE; op->insert_data_done = true; } if (journal_ref) atomic_dec_bug(journal_ref); if (!op->insert_data_done) { continue_at(cl, bch_data_insert_start, op->wq); return; } bch_keylist_free(&op->insert_keys); closure_return(cl); } static int bch_keylist_realloc(struct keylist *l, unsigned int u64s, struct cache_set *c) { size_t oldsize = bch_keylist_nkeys(l); size_t newsize = oldsize + u64s; /* * The journalling code doesn't handle the case where the keys to insert * is bigger than an empty write: If we just return -ENOMEM here, * bch_data_insert_keys() will insert the keys created so far * and finish the rest when the keylist is empty. */ if (newsize * sizeof(uint64_t) > block_bytes(c->cache) - sizeof(struct jset)) return -ENOMEM; return __bch_keylist_realloc(l, u64s); } static void bch_data_invalidate(struct closure *cl) { struct data_insert_op *op = container_of(cl, struct data_insert_op, cl); struct bio *bio = op->bio; pr_debug("invalidating %i sectors from %llu\n", bio_sectors(bio), (uint64_t) bio->bi_iter.bi_sector); while (bio_sectors(bio)) { unsigned int sectors = min(bio_sectors(bio), 1U << (KEY_SIZE_BITS - 1)); if (bch_keylist_realloc(&op->insert_keys, 2, op->c)) goto out; bio->bi_iter.bi_sector += sectors; bio->bi_iter.bi_size -= sectors << 9; bch_keylist_add(&op->insert_keys, &KEY(op->inode, bio->bi_iter.bi_sector, sectors)); } op->insert_data_done = true; /* get in bch_data_insert() */ bio_put(bio); out: continue_at(cl, bch_data_insert_keys, op->wq); } static CLOSURE_CALLBACK(bch_data_insert_error) { closure_type(op, struct data_insert_op, cl); /* * Our data write just errored, which means we've got a bunch of keys to * insert that point to data that wasn't successfully written. * * We don't have to insert those keys but we still have to invalidate * that region of the cache - so, if we just strip off all the pointers * from the keys we'll accomplish just that. */ struct bkey *src = op->insert_keys.keys, *dst = op->insert_keys.keys; while (src != op->insert_keys.top) { struct bkey *n = bkey_next(src); SET_KEY_PTRS(src, 0); memmove(dst, src, bkey_bytes(src)); dst = bkey_next(dst); src = n; } op->insert_keys.top = dst; bch_data_insert_keys(&cl->work); } static void bch_data_insert_endio(struct bio *bio) { struct closure *cl = bio->bi_private; struct data_insert_op *op = container_of(cl, struct data_insert_op, cl); if (bio->bi_status) { /* TODO: We could try to recover from this. */ if (op->writeback) op->status = bio->bi_status; else if (!op->replace) set_closure_fn(cl, bch_data_insert_error, op->wq); else set_closure_fn(cl, NULL, NULL); } bch_bbio_endio(op->c, bio, bio->bi_status, "writing data to cache"); } static CLOSURE_CALLBACK(bch_data_insert_start) { closure_type(op, struct data_insert_op, cl); struct bio *bio = op->bio, *n; if (op->bypass) return bch_data_invalidate(cl); if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0) wake_up_gc(op->c); /* * Journal writes are marked REQ_PREFLUSH; if the original write was a * flush, it'll wait on the journal write. */ bio->bi_opf &= ~(REQ_PREFLUSH|REQ_FUA); do { unsigned int i; struct bkey *k; struct bio_set *split = &op->c->bio_split; /* 1 for the device pointer and 1 for the chksum */ if (bch_keylist_realloc(&op->insert_keys, 3 + (op->csum ? 1 : 0), op->c)) { continue_at(cl, bch_data_insert_keys, op->wq); return; } k = op->insert_keys.top; bkey_init(k); SET_KEY_INODE(k, op->inode); SET_KEY_OFFSET(k, bio->bi_iter.bi_sector); if (!bch_alloc_sectors(op->c, k, bio_sectors(bio), op->write_point, op->write_prio, op->writeback)) goto err; n = bio_next_split(bio, KEY_SIZE(k), GFP_NOIO, split); n->bi_end_io = bch_data_insert_endio; n->bi_private = cl; if (op->writeback) { SET_KEY_DIRTY(k, true); for (i = 0; i < KEY_PTRS(k); i++) SET_GC_MARK(PTR_BUCKET(op->c, k, i), GC_MARK_DIRTY); } SET_KEY_CSUM(k, op->csum); if (KEY_CSUM(k)) bio_csum(n, k); trace_bcache_cache_insert(k); bch_keylist_push(&op->insert_keys); n->bi_opf = REQ_OP_WRITE; bch_submit_bbio(n, op->c, k, 0); } while (n != bio); op->insert_data_done = true; continue_at(cl, bch_data_insert_keys, op->wq); return; err: /* bch_alloc_sectors() blocks if s->writeback = true */ BUG_ON(op->writeback); /* * But if it's not a writeback write we'd rather just bail out if * there aren't any buckets ready to write to - it might take awhile and * we might be starving btree writes for gc or something. */ if (!op->replace) { /* * Writethrough write: We can't complete the write until we've * updated the index. But we don't want to delay the write while * we wait for buckets to be freed up, so just invalidate the * rest of the write. */ op->bypass = true; return bch_data_invalidate(cl); } else { /* * From a cache miss, we can just insert the keys for the data * we have written or bail out if we didn't do anything. */ op->insert_data_done = true; bio_put(bio); if (!bch_keylist_empty(&op->insert_keys)) continue_at(cl, bch_data_insert_keys, op->wq); else closure_return(cl); } } /** * bch_data_insert - stick some data in the cache * @cl: closure pointer. * * This is the starting point for any data to end up in a cache device; it could * be from a normal write, or a writeback write, or a write to a flash only * volume - it's also used by the moving garbage collector to compact data in * mostly empty buckets. * * It first writes the data to the cache, creating a list of keys to be inserted * (if the data had to be fragmented there will be multiple keys); after the * data is written it calls bch_journal, and after the keys have been added to * the next journal write they're inserted into the btree. * * It inserts the data in op->bio; bi_sector is used for the key offset, * and op->inode is used for the key inode. * * If op->bypass is true, instead of inserting the data it invalidates the * region of the cache represented by op->bio and op->inode. */ CLOSURE_CALLBACK(bch_data_insert) { closure_type(op, struct data_insert_op, cl); trace_bcache_write(op->c, op->inode, op->bio, op->writeback, op->bypass); bch_keylist_init(&op->insert_keys); bio_get(op->bio); bch_data_insert_start(&cl->work); } /* * Congested? Return 0 (not congested) or the limit (in sectors) * beyond which we should bypass the cache due to congestion. */ unsigned int bch_get_congested(const struct cache_set *c) { int i; if (!c->congested_read_threshold_us && !c->congested_write_threshold_us) return 0; i = (local_clock_us() - c->congested_last_us) / 1024; if (i < 0) return 0; i += atomic_read(&c->congested); if (i >= 0) return 0; i += CONGESTED_MAX; if (i > 0) i = fract_exp_two(i, 6); i -= hweight32(get_random_u32()); return i > 0 ? i : 1; } static void add_sequential(struct task_struct *t) { ewma_add(t->sequential_io_avg, t->sequential_io, 8, 0); t->sequential_io = 0; } static struct hlist_head *iohash(struct cached_dev *dc, uint64_t k) { return &dc->io_hash[hash_64(k, RECENT_IO_BITS)]; } static bool check_should_bypass(struct cached_dev *dc, struct bio *bio) { struct cache_set *c = dc->disk.c; unsigned int mode = cache_mode(dc); unsigned int sectors, congested; struct task_struct *task = current; struct io *i; if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) || (bio_op(bio) == REQ_OP_DISCARD)) goto skip; if (c->gc_stats.in_use > CUTOFF_CACHE_ADD) { /* * If cached buckets are all clean now, 'true' will be * returned and all requests will bypass the cache device. * Then c->sectors_to_gc has no chance to be negative, and * gc thread won't wake up and caching won't work forever. * Here call force_wake_up_gc() to avoid such aftermath. */ if (BDEV_STATE(&dc->sb) == BDEV_STATE_CLEAN && c->gc_mark_valid) force_wake_up_gc(c); goto skip; } if (mode == CACHE_MODE_NONE || (mode == CACHE_MODE_WRITEAROUND && op_is_write(bio_op(bio)))) goto skip; /* * If the bio is for read-ahead or background IO, bypass it or * not depends on the following situations, * - If the IO is for meta data, always cache it and no bypass * - If the IO is not meta data, check dc->cache_reada_policy, * BCH_CACHE_READA_ALL: cache it and not bypass * BCH_CACHE_READA_META_ONLY: not cache it and bypass * That is, read-ahead request for metadata always get cached * (eg, for gfs2 or xfs). */ if ((bio->bi_opf & (REQ_RAHEAD|REQ_BACKGROUND))) { if (!(bio->bi_opf & (REQ_META|REQ_PRIO)) && (dc->cache_readahead_policy != BCH_CACHE_READA_ALL)) goto skip; } if (bio->bi_iter.bi_sector & (c->cache->sb.block_size - 1) || bio_sectors(bio) & (c->cache->sb.block_size - 1)) { pr_debug("skipping unaligned io\n"); goto skip; } if (bypass_torture_test(dc)) { if (get_random_u32_below(4) == 3) goto skip; else goto rescale; } congested = bch_get_congested(c); if (!congested && !dc->sequential_cutoff) goto rescale; spin_lock(&dc->io_lock); hlist_for_each_entry(i, iohash(dc, bio->bi_iter.bi_sector), hash) if (i->last == bio->bi_iter.bi_sector && time_before(jiffies, i->jiffies)) goto found; i = list_first_entry(&dc->io_lru, struct io, lru); add_sequential(task); i->sequential = 0; found: if (i->sequential + bio->bi_iter.bi_size > i->sequential) i->sequential += bio->bi_iter.bi_size; i->last = bio_end_sector(bio); i->jiffies = jiffies + msecs_to_jiffies(5000); task->sequential_io = i->sequential; hlist_del(&i->hash); hlist_add_head(&i->hash, iohash(dc, i->last)); list_move_tail(&i->lru, &dc->io_lru); spin_unlock(&dc->io_lock); sectors = max(task->sequential_io, task->sequential_io_avg) >> 9; if (dc->sequential_cutoff && sectors >= dc->sequential_cutoff >> 9) { trace_bcache_bypass_sequential(bio); goto skip; } if (congested && sectors >= congested) { trace_bcache_bypass_congested(bio); goto skip; } rescale: bch_rescale_priorities(c, bio_sectors(bio)); return false; skip: bch_mark_sectors_bypassed(c, dc, bio_sectors(bio)); return true; } /* Cache lookup */ struct search { /* Stack frame for bio_complete */ struct closure cl; struct bbio bio; struct bio *orig_bio; struct bio *cache_miss; struct bcache_device *d; unsigned int insert_bio_sectors; unsigned int recoverable:1; unsigned int write:1; unsigned int read_dirty_data:1; unsigned int cache_missed:1; struct block_device *orig_bdev; unsigned long start_time; struct btree_op op; struct data_insert_op iop; }; static void bch_cache_read_endio(struct bio *bio) {