/* * Main bcache entry point - handle a read or a write request and decide what to * do with it; the make_request functions are called by the block layer. * * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com> * Copyright 2012 Google, Inc. */ #include "bcache.h"#include "btree.h"#include "debug.h"#include "request.h"#include "writeback.h" #include <linux/module.h>#include <linux/hash.h>#include <linux/random.h>#include <linux/backing-dev.h> #include <trace/events/bcache.h> #define CUTOFF_CACHE_ADD	95#define CUTOFF_CACHE_READA	90 struct kmem_cache *bch_search_cache; static void bch_data_insert_start(struct closure *); static unsigned cache_mode(struct cached_dev *dc, struct bio *bio){	return BDEV_CACHE_MODE(&dc->sb);} static bool verify(struct cached_dev *dc, struct bio *bio){	return dc->verify;} static void bio_csum(struct bio *bio, struct bkey *k){	struct bio_vec bv;	struct bvec_iter iter;	uint64_t csum = 0; 	bio_for_each_segment(bv, bio, iter) {		void *d = kmap(bv.bv_page) + bv.bv_offset;		csum = bch_crc64_update(csum, d, bv.bv_len);		kunmap(bv.bv_page);	} 	k->ptr[KEY_PTRS(k)] = csum & (~0ULL >> 1);} /* Insert data into cache */ static void bch_data_insert_keys(struct closure *cl){	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);	atomic_t *journal_ref = NULL;	struct bkey *replace_key = op->replace ? &op->replace_key : NULL;	int ret; 	/*	 * If we're looping, might already be waiting on	 * another journal write - can't wait on more than one journal write at	 * a time	 *	 * XXX: this looks wrong	 */#if 0	while (atomic_read(&s->cl.remaining) & CLOSURE_WAITING)		closure_sync(&s->cl);#endif 	if (!op->replace)		journal_ref = bch_journal(op->c, &op->insert_keys,					  op->flush_journal ? cl : NULL); 	ret = bch_btree_insert(op->c, &op->insert_keys,			       journal_ref, replace_key);	if (ret == -ESRCH) {		op->replace_collision = true;	} else if (ret) {		op->status		= BLK_STS_RESOURCE;		op->insert_data_done	= true;	} 	if (journal_ref)		atomic_dec_bug(journal_ref); 	if (!op->insert_data_done) {		continue_at(cl, bch_data_insert_start, op->wq);		return;	} 	bch_keylist_free(&op->insert_keys);	closure_return(cl);} static int bch_keylist_realloc(struct keylist *l, unsigned u64s,			       struct cache_set *c){	size_t oldsize = bch_keylist_nkeys(l);	size_t newsize = oldsize + u64s; 	/*	 * The journalling code doesn't handle the case where the keys to insert	 * is bigger than an empty write: If we just return -ENOMEM here,	 * bio_insert() and bio_invalidate() will insert the keys created so far	 * and finish the rest when the keylist is empty.	 */	if (newsize * sizeof(uint64_t) > block_bytes(c) - sizeof(struct jset))		return -ENOMEM; 	return __bch_keylist_realloc(l, u64s);} static void bch_data_invalidate(struct closure *cl){	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);	struct bio *bio = op->bio; 	pr_debug("invalidating %i sectors from %llu",		 bio_sectors(bio), (uint64_t) bio->bi_iter.bi_sector); 	while (bio_sectors(bio)) {		unsigned sectors = min(bio_sectors(bio),				       1U << (KEY_SIZE_BITS - 1)); 		if (bch_keylist_realloc(&op->insert_keys, 2, op->c))			goto out; 		bio->bi_iter.bi_sector	+= sectors;		bio->bi_iter.bi_size	-= sectors << 9; 		bch_keylist_add(&op->insert_keys,				&KEY(op->inode, bio->bi_iter.bi_sector, sectors));	} 	op->insert_data_done = true;	bio_put(bio);out:	continue_at(cl, bch_data_insert_keys, op->wq);} static void bch_data_insert_error(struct closure *cl){	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl); 	/*	 * Our data write just errored, which means we've got a bunch of keys to	 * insert that point to data that wasn't succesfully written.	 *	 * We don't have to insert those keys but we still have to invalidate	 * that region of the cache - so, if we just strip off all the pointers	 * from the keys we'll accomplish just that.	 */ 	struct bkey *src = op->insert_keys.keys, *dst = op->insert_keys.keys; 	while (src != op->insert_keys.top) {		struct bkey *n = bkey_next(src); 		SET_KEY_PTRS(src, 0);		memmove(dst, src, bkey_bytes(src)); 		dst = bkey_next(dst);		src = n;	} 	op->insert_keys.top = dst; 	bch_data_insert_keys(cl);} static void bch_data_insert_endio(struct bio *bio){	struct closure *cl = bio->bi_private;	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl); 	if (bio->bi_status) {		/* TODO: We could try to recover from this. */		if (op->writeback)			op->status = bio->bi_status;		else if (!op->replace)			set_closure_fn(cl, bch_data_insert_error, op->wq);		else			set_closure_fn(cl, NULL, NULL);	} 	bch_bbio_endio(op->c, bio, bio->bi_status, "writing data to cache");} static void bch_data_insert_start(struct closure *cl){	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);	struct bio *bio = op->bio, *n; 	if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0)		wake_up_gc(op->c); 	if (op->bypass)		return bch_data_invalidate(cl); 	/*	 * Journal writes are marked REQ_PREFLUSH; if the original write was a	 * flush, it'll wait on the journal write.	 */	bio->bi_opf &= ~(REQ_PREFLUSH|REQ_FUA); 	do {		unsigned i;		struct bkey *k;		struct bio_set *split = op->c->bio_split; 		/* 1 for the device pointer and 1 for the chksum */		if (bch_keylist_realloc(&op->insert_keys,					3 + (op->csum ? 1 : 0),					op->c)) {			continue_at(cl, bch_data_insert_keys, op->wq);			return;		} 		k = op->insert_keys.top;		bkey_init(k);		SET_KEY_INODE(k, op->inode);		SET_KEY_OFFSET(k, bio->bi_iter.bi_sector); 		if (!bch_alloc_sectors(op->c, k, bio_sectors(bio),				       op->write_point, op->write_prio,				       op->writeback))			goto err; 		n = bio_next_split(bio, KEY_SIZE(k), GFP_NOIO, split); 		n->bi_end_io	= bch_data_insert_endio;		n->bi_private	= cl; 		if (op->writeback) {			SET_KEY_DIRTY(k, true); 			for (i = 0; i < KEY_PTRS(k); i++)				SET_GC_MARK(PTR_BUCKET(op->c, k, i),					    GC_MARK_DIRTY);		} 		SET_KEY_CSUM(k, op->csum);		if (KEY_CSUM(k))			bio_csum(n, k); 		trace_bcache_cache_insert(k);		bch_keylist_push(&op->insert_keys); 		bio_set_op_attrs(n, REQ_OP_WRITE, 0);		bch_submit_bbio(n, op->c, k, 0);	} while (n != bio); 	op->insert_data_done = true;	continue_at(cl, bch_data_insert_keys, op->wq);	return;err:	/* bch_alloc_sectors() blocks if s->writeback = true */	BUG_ON(op->writeback); 	/*	 * But if it's not a writeback write we'd rather just bail out if	 * there aren't any buckets ready to write to - it might take awhile and	 * we might be starving btree writes for gc or something.	 */ 	if (!op->replace) {		/*		 * Writethrough write: We can't complete the write until we've		 * updated the index. But we don't want to delay the write while		 * we wait for buckets to be freed up, so just invalidate the		 * rest of the write.		 */		op->bypass = true;		return bch_data_invalidate(cl);	} else {		/*		 * From a cache miss, we can just insert the keys for the data		 * we have written or bail out if we didn't do anything.		 */		op->insert_data_done = true;		bio_put(bio); 		if (!bch_keylist_empty(&op->insert_keys))			continue_at(cl, bch_data_insert_keys, op->wq);		else			closure_return(cl);	}} /** * bch_data_insert - stick some data in the cache * * This is the starting point for any data to end up in a cache device; it could * be from a normal write, or a writeback write, or a write to a flash only * volume - it's also used by the moving garbage collector to compact data in * mostly empty buckets. * * It first writes the data to the cache, creating a list of keys to be inserted * (if the data had to be fragmented there will be multiple keys); after the * data is written it calls bch_journal, and after the keys have been added to * the next journal write they're inserted into the btree. * * It inserts the data in s->cache_bio; bi_sector is used for the key offset, * and op->inode is used for the key inode. * * If s->bypass is true, instead of inserting the data it invalidates the * region of the cache represented by s->cache_bio and op->inode. */void bch_data_insert(struct closure *cl){	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl); 	trace_bcache_write(op->c, op->inode, op->bio,			   op->writeback, op->bypass); 	bch_keylist_init(&op->insert_keys);	bio_get(op->bio);	bch_data_insert_start(cl);} /* Congested? */ unsigned bch_get_congested(struct cache_set *c){	int i;	long rand; 	if (!c->congested_read_threshold_us &&	    !c->congested_write_threshold_us)		return 0; 	i = (local_clock_us() - c->congested_last_us) / 1024;	if (i < 0)		return 0; 	i += atomic_read(&c->congested);	if (i >= 0)		return 0; 	i += CONGESTED_MAX; 	if (i > 0)		i = fract_exp_two(i, 6); 	rand = get_random_int();	i -= bitmap_weight(&rand, BITS_PER_LONG); 	return i > 0 ? i : 1;} static void add_sequential(struct task_struct *t){	ewma_add(t->sequential_io_avg,		 t->sequential_io, 8, 0); 	t->sequential_io = 0;} static struct hlist_head *iohash(struct cached_dev *dc, uint64_t k){	return &dc->io_hash[hash_64(k, RECENT_IO_BITS)];} static bool check_should_bypass(struct cached_dev *dc, struct bio *bio){	struct cache_set *c = dc->disk.c;	unsigned mode = cache_mode(dc, bio);	unsigned sectors, congested = bch_get_congested(c);	struct task_struct *task = current;	struct io *i; 	if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) ||	    c->gc_stats.in_use > CUTOFF_CACHE_ADD ||	    (bio_op(bio) == REQ_OP_DISCARD))		goto skip; 	if (mode == CACHE_MODE_NONE ||	    (mode == CACHE_MODE_WRITEAROUND &&	     op_is_write(bio_op(bio))))		goto skip; 	if (bio->bi_iter.bi_sector & (c->sb.block_size - 1) ||	    bio_sectors(bio) & (c->sb.block_size - 1)) {		pr_debug("skipping unaligned io");		goto skip;	} 	if (bypass_torture_test(dc)) {		if ((get_random_int() & 3) == 3)			goto skip;		else			goto rescale;	} 	if (!congested && !dc->sequential_cutoff)		goto rescale; 	if (!congested &&	    mode == CACHE_MODE_WRITEBACK &&	    op_is_write(bio->bi_opf) &&	    op_is_sync(bio->bi_opf))		goto rescale; 	spin_lock(&dc->io_lock); 	hlist_for_each_entry(i, iohash(dc, bio->bi_iter.bi_sector), hash)		if (i->last == bio->bi_iter.bi_sector &&		    time_before(jiffies, i->jiffies))			goto found; 	i = list_first_entry(&dc->io_lru, struct io, lru); 	add_sequential(task);	i->sequential = 0;found:	if (i->sequential + bio->bi_iter.bi_size > i->sequential)		i->sequential	+= bio->bi_iter.bi_size; 	i->last			 = bio_end_sector(bio);	i->jiffies		 = jiffies + msecs_to_jiffies(5000);	task->sequential_io	 = i->sequential; 	hlist_del(&i->hash);	hlist_add_head(&i->hash, iohash(dc, i->last));	list_move_tail(&i->lru, &dc->io_lru); 	spin_unlock(&dc->io_lock); 	sectors = max(task->sequential_io,		      task->sequential_io_avg) >> 9; 	if (dc->sequential_cutoff &&	    sectors >= dc->sequential_cutoff >> 9) {		trace_bcache_bypass_sequential(bio);		goto skip;	} 	if (congested && sectors >= congested) {		trace_bcache_bypass_congested(bio);		goto skip;	} rescale:	bch_rescale_priorities(c, bio_sectors(bio));	return false;skip:	bch_mark_sectors_bypassed(c, dc, bio_sectors(bio));	return true;} /* Cache lookup */ struct search {	/* Stack frame for bio_complete */	struct closure		cl; 	struct bbio		bio;	struct bio		*orig_bio;	struct bio		*cache_miss;	struct bcache_device	*d; 	unsigned		insert_bio_sectors;	unsigned		recoverable:1;	unsigned		write:1;	unsigned		read_dirty_data:1; 	unsigned long		start_time; 	struct btree_op		op;	struct data_insert_op	iop;}; static void bch_cache_read_endio(struct bio *bio){	struct bbio *b = container_of(bio, struct bbio, bio);	struct closure *cl = bio->bi_private;	struct search *s = container_of(cl, struct search, cl); 	/*	 * If the bucket was reused while our bio was in flight, we might have	 * read the wrong data. Set s->error but not error so it doesn't get	 * counted against the cache device, but we'll still reread the data	 * from the backing device.	 */ 	if (bio->bi_status)		s->iop.status = bio->bi_status;	else if (!KEY_DIRTY(&b->key) &&		 ptr_stale(s->iop.c, &b->key, 0)) {		atomic_long_inc(&s->iop.c->cache_read_races);		s->iop.status = BLK_STS_IOERR;	} 	bch_bbio_endio(s->iop.c, bio, bio->bi_status, "reading from cache");}

Lines 1–500 of 1157. Load more