// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (C) 2012 Red Hat. All rights reserved. * * This file is released under the GPL. */ #include "dm.h"#include "dm-bio-prison-v2.h"#include "dm-bio-record.h"#include "dm-cache-metadata.h"#include "dm-io-tracker.h"#include "dm-cache-background-tracker.h" #include <linux/dm-io.h>#include <linux/dm-kcopyd.h>#include <linux/jiffies.h>#include <linux/init.h>#include <linux/kstrtox.h>#include <linux/mempool.h>#include <linux/module.h>#include <linux/rwsem.h>#include <linux/slab.h>#include <linux/vmalloc.h> #define DM_MSG_PREFIX "cache" DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(cache_copy_throttle,	"A percentage of time allocated for copying to and/or from cache"); /*----------------------------------------------------------------*/ /* * Glossary: * * oblock: index of an origin block * cblock: index of a cache block * promotion: movement of a block from origin to cache * demotion: movement of a block from cache to origin * migration: movement of a block between the origin and cache device, *	      either direction */ /*----------------------------------------------------------------*/ /* * Represents a chunk of future work.  'input' allows continuations to pass * values between themselves, typically error values. */struct continuation {	struct work_struct ws;	blk_status_t input;}; static inline void init_continuation(struct continuation *k,				     void (*fn)(struct work_struct *)){	INIT_WORK(&k->ws, fn);	k->input = 0;} static inline void queue_continuation(struct workqueue_struct *wq,				      struct continuation *k){	queue_work(wq, &k->ws);} /*----------------------------------------------------------------*/ /* * The batcher collects together pieces of work that need a particular * operation to occur before they can proceed (typically a commit). */struct batcher {	/*	 * The operation that everyone is waiting for.	 */	blk_status_t (*commit_op)(void *context);	void *commit_context; 	/*	 * This is how bios should be issued once the commit op is complete	 * (accounted_request).	 */	void (*issue_op)(struct bio *bio, void *context);	void *issue_context; 	/*	 * Queued work gets put on here after commit.	 */	struct workqueue_struct *wq; 	spinlock_t lock;	struct list_head work_items;	struct bio_list bios;	struct work_struct commit_work; 	bool commit_scheduled;}; static void __commit(struct work_struct *_ws){	struct batcher *b = container_of(_ws, struct batcher, commit_work);	blk_status_t r;	struct list_head work_items;	struct work_struct *ws, *tmp;	struct continuation *k;	struct bio *bio;	struct bio_list bios; 	INIT_LIST_HEAD(&work_items);	bio_list_init(&bios); 	/*	 * We have to grab these before the commit_op to avoid a race	 * condition.	 */	spin_lock_irq(&b->lock);	list_splice_init(&b->work_items, &work_items);	bio_list_merge_init(&bios, &b->bios);	b->commit_scheduled = false;	spin_unlock_irq(&b->lock); 	r = b->commit_op(b->commit_context); 	list_for_each_entry_safe(ws, tmp, &work_items, entry) {		k = container_of(ws, struct continuation, ws);		k->input = r;		INIT_LIST_HEAD(&ws->entry); /* to avoid a WARN_ON */		queue_work(b->wq, ws);	} 	while ((bio = bio_list_pop(&bios))) {		if (r) {			bio->bi_status = r;			bio_endio(bio);		} else			b->issue_op(bio, b->issue_context);	}} static void batcher_init(struct batcher *b,			 blk_status_t (*commit_op)(void *),			 void *commit_context,			 void (*issue_op)(struct bio *bio, void *),			 void *issue_context,			 struct workqueue_struct *wq){	b->commit_op = commit_op;	b->commit_context = commit_context;	b->issue_op = issue_op;	b->issue_context = issue_context;	b->wq = wq; 	spin_lock_init(&b->lock);	INIT_LIST_HEAD(&b->work_items);	bio_list_init(&b->bios);	INIT_WORK(&b->commit_work, __commit);	b->commit_scheduled = false;} static void async_commit(struct batcher *b){	queue_work(b->wq, &b->commit_work);} static void continue_after_commit(struct batcher *b, struct continuation *k){	bool commit_scheduled; 	spin_lock_irq(&b->lock);	commit_scheduled = b->commit_scheduled;	list_add_tail(&k->ws.entry, &b->work_items);	spin_unlock_irq(&b->lock); 	if (commit_scheduled)		async_commit(b);} /* * Bios are errored if commit failed. */static void issue_after_commit(struct batcher *b, struct bio *bio){	bool commit_scheduled; 	spin_lock_irq(&b->lock);	commit_scheduled = b->commit_scheduled;	bio_list_add(&b->bios, bio);	spin_unlock_irq(&b->lock); 	if (commit_scheduled)		async_commit(b);} /* * Call this if some urgent work is waiting for the commit to complete. */static void schedule_commit(struct batcher *b){	bool immediate; 	spin_lock_irq(&b->lock);	immediate = !list_empty(&b->work_items) || !bio_list_empty(&b->bios);	b->commit_scheduled = true;	spin_unlock_irq(&b->lock); 	if (immediate)		async_commit(b);} /* * There are a couple of places where we let a bio run, but want to do some * work before calling its endio function.  We do this by temporarily * changing the endio fn. */struct dm_hook_info {	bio_end_io_t *bi_end_io;}; static void dm_hook_bio(struct dm_hook_info *h, struct bio *bio,			bio_end_io_t *bi_end_io, void *bi_private){	h->bi_end_io = bio->bi_end_io; 	bio->bi_end_io = bi_end_io;	bio->bi_private = bi_private;} static void dm_unhook_bio(struct dm_hook_info *h, struct bio *bio){	bio->bi_end_io = h->bi_end_io;} /*----------------------------------------------------------------*/ #define MIGRATION_POOL_SIZE 128#define COMMIT_PERIOD HZ#define MIGRATION_COUNT_WINDOW 10 /* * The block size of the device holding cache data must be * between 32KB and 1GB. */#define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (32 * 1024 >> SECTOR_SHIFT)#define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT) enum cache_metadata_mode {	CM_WRITE,		/* metadata may be changed */	CM_READ_ONLY,		/* metadata may not be changed */	CM_FAIL}; enum cache_io_mode {	/*	 * Data is written to cached blocks only.  These blocks are marked	 * dirty.  If you lose the cache device you will lose data.	 * Potential performance increase for both reads and writes.	 */	CM_IO_WRITEBACK, 	/*	 * Data is written to both cache and origin.  Blocks are never	 * dirty.  Potential performance benfit for reads only.	 */	CM_IO_WRITETHROUGH, 	/*	 * A degraded mode useful for various cache coherency situations	 * (eg, rolling back snapshots).  Reads and writes always go to the	 * origin.  If a write goes to a cached oblock, then the cache	 * block is invalidated.	 */	CM_IO_PASSTHROUGH}; struct cache_features {	enum cache_metadata_mode mode;	enum cache_io_mode io_mode;	unsigned int metadata_version;	bool discard_passdown:1;}; struct cache_stats {	atomic_t read_hit;	atomic_t read_miss;	atomic_t write_hit;	atomic_t write_miss;	atomic_t demotion;	atomic_t promotion;	atomic_t writeback;	atomic_t copies_avoided;	atomic_t cache_cell_clash;	atomic_t commit_count;	atomic_t discard_count;}; struct cache {	struct dm_target *ti;	spinlock_t lock; 	/*	 * Fields for converting from sectors to blocks.	 */	int sectors_per_block_shift;	sector_t sectors_per_block; 	struct dm_cache_metadata *cmd; 	/*	 * Metadata is written to this device.	 */	struct dm_dev *metadata_dev; 	/*	 * The slower of the two data devices.  Typically a spindle.	 */	struct dm_dev *origin_dev; 	/*	 * The faster of the two data devices.  Typically an SSD.	 */	struct dm_dev *cache_dev; 	/*	 * Size of the origin device in _complete_ blocks and native sectors.	 */	dm_oblock_t origin_blocks;	sector_t origin_sectors; 	/*	 * Size of the cache device in blocks.	 */	dm_cblock_t cache_size; 	/*	 * Invalidation fields.	 */	spinlock_t invalidation_lock;	struct list_head invalidation_requests; 	sector_t migration_threshold; 	/*	 * The number of in flight migrations that are performing	 * background io. eg, promotion, writeback.	 */	atomic_t nr_io_migrations; 	struct bio_list deferred_bios; 	struct rw_semaphore quiesce_lock; 	/*	 * origin_blocks entries, discarded if set.	 */	dm_dblock_t discard_nr_blocks;	unsigned long *discard_bitset;	uint32_t discard_block_size; /* a power of 2 times sectors per block */ 	/*	 * Rather than reconstructing the table line for the status we just	 * save it and regurgitate.	 */	unsigned int nr_ctr_args;	const char **ctr_args; 	struct dm_kcopyd_client *copier;	struct work_struct deferred_bio_worker;	struct work_struct migration_worker;	struct workqueue_struct *wq;	struct delayed_work waker;	struct dm_bio_prison_v2 *prison; 	/*	 * cache_size entries, dirty if set	 */	unsigned long *dirty_bitset;	atomic_t nr_dirty; 	unsigned int policy_nr_args;	struct dm_cache_policy *policy; 	/*	 * Cache features such as write-through.	 */	struct cache_features features; 	struct cache_stats stats; 	bool need_tick_bio:1;	bool sized:1;	bool invalidate:1;	bool commit_requested:1;	bool loaded_mappings:1;	bool loaded_discards:1; 	/* background work management */	bool background_work_allowed;	unsigned background_work_nr;	spinlock_t background_work_lock;	wait_queue_head_t background_work_wait; 	struct batcher committer;	struct work_struct commit_ws; 	struct dm_io_tracker tracker; 	mempool_t migration_pool; 	struct bio_set bs; 	/*	 * Cache_size entries. Set bits indicate blocks mapped beyond the	 * target length, which are marked for invalidation.	 */	unsigned long *invalid_bitset;}; struct per_bio_data {	bool tick:1;	unsigned int req_nr:2;	struct dm_bio_prison_cell_v2 *cell;	struct dm_hook_info hook_info;	sector_t len;}; struct dm_cache_migration {	struct continuation k;	struct cache *cache; 	struct policy_work *op;	struct bio *overwrite_bio;	struct dm_bio_prison_cell_v2 *cell; 	dm_cblock_t invalidate_cblock;	dm_oblock_t invalidate_oblock;}; /*----------------------------------------------------------------*/ static bool writethrough_mode(struct cache *cache){	return cache->features.io_mode == CM_IO_WRITETHROUGH;} static bool writeback_mode(struct cache *cache){	return cache->features.io_mode == CM_IO_WRITEBACK;} static inline bool passthrough_mode(struct cache *cache){	return unlikely(cache->features.io_mode == CM_IO_PASSTHROUGH);} /*----------------------------------------------------------------*/ static void wake_deferred_bio_worker(struct cache *cache){	queue_work(cache->wq, &cache->deferred_bio_worker);} static void wake_migration_worker(struct cache *cache){	if (passthrough_mode(cache))		return; 	queue_work(cache->wq, &cache->migration_worker);} /*----------------------------------------------------------------*/ static struct dm_bio_prison_cell_v2 *alloc_prison_cell(struct cache *cache){	return dm_bio_prison_alloc_cell_v2(cache->prison, GFP_NOIO);} static void free_prison_cell(struct cache *cache, struct dm_bio_prison_cell_v2 *cell){	dm_bio_prison_free_cell_v2(cache->prison, cell);} static struct dm_cache_migration *alloc_migration(struct cache *cache){	struct dm_cache_migration *mg; 	mg = mempool_alloc(&cache->migration_pool, GFP_NOIO); 	memset(mg, 0, sizeof(*mg)); 	mg->cache = cache; 	return mg;} static void free_migration(struct dm_cache_migration *mg){	mempool_free(mg, &mg->cache->migration_pool);}

Lines 1–500 of 3615. Load more