/* * Main bcache entry point - handle a read or a write request and decide what to * do with it; the make_request functions are called by the block layer. * * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com> * Copyright 2012 Google, Inc. */ #include "bcache.h"#include "btree.h"#include "debug.h"#include "request.h"#include "writeback.h" #include <linux/cgroup.h>#include <linux/module.h>#include <linux/hash.h>#include <linux/random.h>#include "blk-cgroup.h" #include <trace/events/bcache.h> #define CUTOFF_CACHE_ADD	95#define CUTOFF_CACHE_READA	90 struct kmem_cache *bch_search_cache; static void bch_data_insert_start(struct closure *); /* Cgroup interface */ #ifdef CONFIG_CGROUP_BCACHEstatic struct bch_cgroup bcache_default_cgroup = { .cache_mode = -1 }; static struct bch_cgroup *cgroup_to_bcache(struct cgroup *cgroup){	struct cgroup_subsys_state *css;	return cgroup &&		(css = cgroup_subsys_state(cgroup, bcache_subsys_id))		? container_of(css, struct bch_cgroup, css)		: &bcache_default_cgroup;} struct bch_cgroup *bch_bio_to_cgroup(struct bio *bio){	struct cgroup_subsys_state *css = bio->bi_css		? cgroup_subsys_state(bio->bi_css->cgroup, bcache_subsys_id)		: task_subsys_state(current, bcache_subsys_id); 	return css		? container_of(css, struct bch_cgroup, css)		: &bcache_default_cgroup;} static ssize_t cache_mode_read(struct cgroup *cgrp, struct cftype *cft,			struct file *file,			char __user *buf, size_t nbytes, loff_t *ppos){	char tmp[1024];	int len = bch_snprint_string_list(tmp, PAGE_SIZE, bch_cache_modes,					  cgroup_to_bcache(cgrp)->cache_mode + 1); 	if (len < 0)		return len; 	return simple_read_from_buffer(buf, nbytes, ppos, tmp, len);} static int cache_mode_write(struct cgroup *cgrp, struct cftype *cft,			    const char *buf){	int v = bch_read_string_list(buf, bch_cache_modes);	if (v < 0)		return v; 	cgroup_to_bcache(cgrp)->cache_mode = v - 1;	return 0;} static u64 bch_verify_read(struct cgroup *cgrp, struct cftype *cft){	return cgroup_to_bcache(cgrp)->verify;} static int bch_verify_write(struct cgroup *cgrp, struct cftype *cft, u64 val){	cgroup_to_bcache(cgrp)->verify = val;	return 0;} static u64 bch_cache_hits_read(struct cgroup *cgrp, struct cftype *cft){	struct bch_cgroup *bcachecg = cgroup_to_bcache(cgrp);	return atomic_read(&bcachecg->stats.cache_hits);} static u64 bch_cache_misses_read(struct cgroup *cgrp, struct cftype *cft){	struct bch_cgroup *bcachecg = cgroup_to_bcache(cgrp);	return atomic_read(&bcachecg->stats.cache_misses);} static u64 bch_cache_bypass_hits_read(struct cgroup *cgrp,					 struct cftype *cft){	struct bch_cgroup *bcachecg = cgroup_to_bcache(cgrp);	return atomic_read(&bcachecg->stats.cache_bypass_hits);} static u64 bch_cache_bypass_misses_read(struct cgroup *cgrp,					   struct cftype *cft){	struct bch_cgroup *bcachecg = cgroup_to_bcache(cgrp);	return atomic_read(&bcachecg->stats.cache_bypass_misses);} static struct cftype bch_files[] = {	{		.name		= "cache_mode",		.read		= cache_mode_read,		.write_string	= cache_mode_write,	},	{		.name		= "verify",		.read_u64	= bch_verify_read,		.write_u64	= bch_verify_write,	},	{		.name		= "cache_hits",		.read_u64	= bch_cache_hits_read,	},	{		.name		= "cache_misses",		.read_u64	= bch_cache_misses_read,	},	{		.name		= "cache_bypass_hits",		.read_u64	= bch_cache_bypass_hits_read,	},	{		.name		= "cache_bypass_misses",		.read_u64	= bch_cache_bypass_misses_read,	},	{ }	/* terminate */}; static void init_bch_cgroup(struct bch_cgroup *cg){	cg->cache_mode = -1;} static struct cgroup_subsys_state *bcachecg_create(struct cgroup *cgroup){	struct bch_cgroup *cg; 	cg = kzalloc(sizeof(*cg), GFP_KERNEL);	if (!cg)		return ERR_PTR(-ENOMEM);	init_bch_cgroup(cg);	return &cg->css;} static void bcachecg_destroy(struct cgroup *cgroup){	struct bch_cgroup *cg = cgroup_to_bcache(cgroup);	free_css_id(&bcache_subsys, &cg->css);	kfree(cg);} struct cgroup_subsys bcache_subsys = {	.create		= bcachecg_create,	.destroy	= bcachecg_destroy,	.subsys_id	= bcache_subsys_id,	.name		= "bcache",	.module		= THIS_MODULE,};EXPORT_SYMBOL_GPL(bcache_subsys);#endif static unsigned cache_mode(struct cached_dev *dc, struct bio *bio){#ifdef CONFIG_CGROUP_BCACHE	int r = bch_bio_to_cgroup(bio)->cache_mode;	if (r >= 0)		return r;#endif	return BDEV_CACHE_MODE(&dc->sb);} static bool verify(struct cached_dev *dc, struct bio *bio){#ifdef CONFIG_CGROUP_BCACHE	if (bch_bio_to_cgroup(bio)->verify)		return true;#endif	return dc->verify;} static void bio_csum(struct bio *bio, struct bkey *k){	struct bio_vec *bv;	uint64_t csum = 0;	int i; 	bio_for_each_segment(bv, bio, i) {		void *d = kmap(bv->bv_page) + bv->bv_offset;		csum = bch_crc64_update(csum, d, bv->bv_len);		kunmap(bv->bv_page);	} 	k->ptr[KEY_PTRS(k)] = csum & (~0ULL >> 1);} /* Insert data into cache */ static void bch_data_insert_keys(struct closure *cl){	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);	atomic_t *journal_ref = NULL;	struct bkey *replace_key = op->replace ? &op->replace_key : NULL;	int ret; 	/*	 * If we're looping, might already be waiting on	 * another journal write - can't wait on more than one journal write at	 * a time	 *	 * XXX: this looks wrong	 */#if 0	while (atomic_read(&s->cl.remaining) & CLOSURE_WAITING)		closure_sync(&s->cl);#endif 	if (!op->replace)		journal_ref = bch_journal(op->c, &op->insert_keys,					  op->flush_journal ? cl : NULL); 	ret = bch_btree_insert(op->c, &op->insert_keys,			       journal_ref, replace_key);	if (ret == -ESRCH) {		op->replace_collision = true;	} else if (ret) {		op->error		= -ENOMEM;		op->insert_data_done	= true;	} 	if (journal_ref)		atomic_dec_bug(journal_ref); 	if (!op->insert_data_done)		continue_at(cl, bch_data_insert_start, bcache_wq); 	bch_keylist_free(&op->insert_keys);	closure_return(cl);} static void bch_data_invalidate(struct closure *cl){	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);	struct bio *bio = op->bio; 	pr_debug("invalidating %i sectors from %llu",		 bio_sectors(bio), (uint64_t) bio->bi_sector); 	while (bio_sectors(bio)) {		unsigned sectors = min(bio_sectors(bio),				       1U << (KEY_SIZE_BITS - 1)); 		if (bch_keylist_realloc(&op->insert_keys, 0, op->c))			goto out; 		bio->bi_sector	+= sectors;		bio->bi_size	-= sectors << 9; 		bch_keylist_add(&op->insert_keys,				&KEY(op->inode, bio->bi_sector, sectors));	} 	op->insert_data_done = true;	bio_put(bio);out:	continue_at(cl, bch_data_insert_keys, bcache_wq);} static void bch_data_insert_error(struct closure *cl){	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl); 	/*	 * Our data write just errored, which means we've got a bunch of keys to	 * insert that point to data that wasn't succesfully written.	 *	 * We don't have to insert those keys but we still have to invalidate	 * that region of the cache - so, if we just strip off all the pointers	 * from the keys we'll accomplish just that.	 */ 	struct bkey *src = op->insert_keys.keys, *dst = op->insert_keys.keys; 	while (src != op->insert_keys.top) {		struct bkey *n = bkey_next(src); 		SET_KEY_PTRS(src, 0);		memmove(dst, src, bkey_bytes(src)); 		dst = bkey_next(dst);		src = n;	} 	op->insert_keys.top = dst; 	bch_data_insert_keys(cl);} static void bch_data_insert_endio(struct bio *bio, int error){	struct closure *cl = bio->bi_private;	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl); 	if (error) {		/* TODO: We could try to recover from this. */		if (op->writeback)			op->error = error;		else if (!op->replace)			set_closure_fn(cl, bch_data_insert_error, bcache_wq);		else			set_closure_fn(cl, NULL, NULL);	} 	bch_bbio_endio(op->c, bio, error, "writing data to cache");} static void bch_data_insert_start(struct closure *cl){	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);	struct bio *bio = op->bio, *n; 	if (op->bypass)		return bch_data_invalidate(cl); 	if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0) {		set_gc_sectors(op->c);		wake_up_gc(op->c);	} 	/*	 * Journal writes are marked REQ_FLUSH; if the original write was a	 * flush, it'll wait on the journal write.	 */	bio->bi_rw &= ~(REQ_FLUSH|REQ_FUA); 	do {		unsigned i;		struct bkey *k;		struct bio_set *split = op->c->bio_split; 		/* 1 for the device pointer and 1 for the chksum */		if (bch_keylist_realloc(&op->insert_keys,					1 + (op->csum ? 1 : 0),					op->c))			continue_at(cl, bch_data_insert_keys, bcache_wq); 		k = op->insert_keys.top;		bkey_init(k);		SET_KEY_INODE(k, op->inode);		SET_KEY_OFFSET(k, bio->bi_sector); 		if (!bch_alloc_sectors(op->c, k, bio_sectors(bio),				       op->write_point, op->write_prio,				       op->writeback))			goto err; 		n = bch_bio_split(bio, KEY_SIZE(k), GFP_NOIO, split); 		n->bi_end_io	= bch_data_insert_endio;		n->bi_private	= cl; 		if (op->writeback) {			SET_KEY_DIRTY(k, true); 			for (i = 0; i < KEY_PTRS(k); i++)				SET_GC_MARK(PTR_BUCKET(op->c, k, i),					    GC_MARK_DIRTY);		} 		SET_KEY_CSUM(k, op->csum);		if (KEY_CSUM(k))			bio_csum(n, k); 		trace_bcache_cache_insert(k);		bch_keylist_push(&op->insert_keys); 		n->bi_rw |= REQ_WRITE;		bch_submit_bbio(n, op->c, k, 0);	} while (n != bio); 	op->insert_data_done = true;	continue_at(cl, bch_data_insert_keys, bcache_wq);err:	/* bch_alloc_sectors() blocks if s->writeback = true */	BUG_ON(op->writeback); 	/*	 * But if it's not a writeback write we'd rather just bail out if	 * there aren't any buckets ready to write to - it might take awhile and	 * we might be starving btree writes for gc or something.	 */ 	if (!op->replace) {		/*		 * Writethrough write: We can't complete the write until we've		 * updated the index. But we don't want to delay the write while		 * we wait for buckets to be freed up, so just invalidate the		 * rest of the write.		 */		op->bypass = true;		return bch_data_invalidate(cl);	} else {		/*		 * From a cache miss, we can just insert the keys for the data		 * we have written or bail out if we didn't do anything.		 */		op->insert_data_done = true;		bio_put(bio); 		if (!bch_keylist_empty(&op->insert_keys))			continue_at(cl, bch_data_insert_keys, bcache_wq);		else			closure_return(cl);	}} /** * bch_data_insert - stick some data in the cache * * This is the starting point for any data to end up in a cache device; it could * be from a normal write, or a writeback write, or a write to a flash only * volume - it's also used by the moving garbage collector to compact data in * mostly empty buckets. * * It first writes the data to the cache, creating a list of keys to be inserted * (if the data had to be fragmented there will be multiple keys); after the * data is written it calls bch_journal, and after the keys have been added to * the next journal write they're inserted into the btree. * * It inserts the data in s->cache_bio; bi_sector is used for the key offset, * and op->inode is used for the key inode. * * If s->bypass is true, instead of inserting the data it invalidates the * region of the cache represented by s->cache_bio and op->inode. */void bch_data_insert(struct closure *cl){	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl); 	trace_bcache_write(op->bio, op->writeback, op->bypass); 	bch_keylist_init(&op->insert_keys);	bio_get(op->bio);	bch_data_insert_start(cl);} /* Congested? */ unsigned bch_get_congested(struct cache_set *c){	int i;	long rand; 	if (!c->congested_read_threshold_us &&	    !c->congested_write_threshold_us)		return 0; 	i = (local_clock_us() - c->congested_last_us) / 1024;	if (i < 0)		return 0; 	i += atomic_read(&c->congested);	if (i >= 0)		return 0; 	i += CONGESTED_MAX; 	if (i > 0)		i = fract_exp_two(i, 6); 	rand = get_random_int();	i -= bitmap_weight(&rand, BITS_PER_LONG); 	return i > 0 ? i : 1;} static void add_sequential(struct task_struct *t){	ewma_add(t->sequential_io_avg,		 t->sequential_io, 8, 0); 	t->sequential_io = 0;}

Lines 1–500 of 1338. Load more