// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (C) 2018 Red Hat. All rights reserved. * * This file is released under the GPL. */ #include <linux/device-mapper.h>#include <linux/module.h>#include <linux/init.h>#include <linux/vmalloc.h>#include <linux/kthread.h>#include <linux/dm-io.h>#include <linux/dm-kcopyd.h>#include <linux/dax.h>#include <linux/libnvdimm.h>#include <linux/delay.h>#include "dm-io-tracker.h" #define DM_MSG_PREFIX "writecache" #define HIGH_WATERMARK			50#define LOW_WATERMARK			45#define MAX_WRITEBACK_JOBS		min(0x10000000 / PAGE_SIZE, totalram_pages() / 16)#define ENDIO_LATENCY			16#define WRITEBACK_LATENCY		64#define AUTOCOMMIT_BLOCKS_SSD		65536#define AUTOCOMMIT_BLOCKS_PMEM		64#define AUTOCOMMIT_MSEC			1000#define MAX_AGE_DIV			16#define MAX_AGE_UNSPECIFIED		-1UL#define PAUSE_WRITEBACK			(HZ * 3) #define BITMAP_GRANULARITY	65536#if BITMAP_GRANULARITY < PAGE_SIZE#undef BITMAP_GRANULARITY#define BITMAP_GRANULARITY	PAGE_SIZE#endif #if IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API) && IS_ENABLED(CONFIG_FS_DAX)#define DM_WRITECACHE_HAS_PMEM#endif #ifdef DM_WRITECACHE_HAS_PMEM#define pmem_assign(dest, src)					\do {								\	typeof(dest) uniq = (src);				\	memcpy_flushcache(&(dest), &uniq, sizeof(dest));	\} while (0)#else#define pmem_assign(dest, src)	((dest) = (src))#endif #if IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC) && defined(DM_WRITECACHE_HAS_PMEM)#define DM_WRITECACHE_HANDLE_HARDWARE_ERRORS#endif #define MEMORY_SUPERBLOCK_MAGIC		0x23489321#define MEMORY_SUPERBLOCK_VERSION	1 struct wc_memory_entry {	__le64 original_sector;	__le64 seq_count;}; struct wc_memory_superblock {	union {		struct {			__le32 magic;			__le32 version;			__le32 block_size;			__le32 pad;			__le64 n_blocks;			__le64 seq_count;		};		__le64 padding[8];	};	struct wc_memory_entry entries[];}; struct wc_entry {	struct rb_node rb_node;	struct list_head lru;	unsigned short wc_list_contiguous;#if BITS_PER_LONG == 64	bool write_in_progress : 1;	unsigned long index : 47;#else	bool write_in_progress;	unsigned long index;#endif	unsigned long age;#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS	uint64_t original_sector;	uint64_t seq_count;#endif}; #ifdef DM_WRITECACHE_HAS_PMEM#define WC_MODE_PMEM(wc)			((wc)->pmem_mode)#define WC_MODE_FUA(wc)				((wc)->writeback_fua)#else#define WC_MODE_PMEM(wc)			false#define WC_MODE_FUA(wc)				false#endif#define WC_MODE_SORT_FREELIST(wc)		(!WC_MODE_PMEM(wc)) struct dm_writecache {	struct mutex lock;	struct list_head lru;	union {		struct list_head freelist;		struct {			struct rb_root freetree;			struct wc_entry *current_free;		};	};	struct rb_root tree; 	size_t freelist_size;	size_t writeback_size;	size_t freelist_high_watermark;	size_t freelist_low_watermark;	unsigned long max_age;	unsigned long pause; 	unsigned int uncommitted_blocks;	unsigned int autocommit_blocks;	unsigned int max_writeback_jobs; 	int error; 	unsigned long autocommit_jiffies;	struct timer_list autocommit_timer;	struct wait_queue_head freelist_wait; 	struct timer_list max_age_timer; 	atomic_t bio_in_progress[2];	struct wait_queue_head bio_in_progress_wait[2]; 	struct dm_target *ti;	struct dm_dev *dev;	struct dm_dev *ssd_dev;	sector_t start_sector;	void *memory_map;	uint64_t memory_map_size;	size_t metadata_sectors;	size_t n_blocks;	uint64_t seq_count;	sector_t data_device_sectors;	void *block_start;	struct wc_entry *entries;	unsigned int block_size;	unsigned char block_size_bits; 	bool pmem_mode:1;	bool writeback_fua:1; 	bool overwrote_committed:1;	bool memory_vmapped:1; 	bool start_sector_set:1;	bool high_wm_percent_set:1;	bool low_wm_percent_set:1;	bool max_writeback_jobs_set:1;	bool autocommit_blocks_set:1;	bool autocommit_time_set:1;	bool max_age_set:1;	bool writeback_fua_set:1;	bool flush_on_suspend:1;	bool cleaner:1;	bool cleaner_set:1;	bool metadata_only:1;	bool pause_set:1; 	unsigned int high_wm_percent_value;	unsigned int low_wm_percent_value;	unsigned int autocommit_time_value;	unsigned int max_age_value;	unsigned int pause_value; 	unsigned int writeback_all;	struct workqueue_struct *writeback_wq;	struct work_struct writeback_work;	struct work_struct flush_work; 	struct dm_io_tracker iot; 	struct dm_io_client *dm_io; 	raw_spinlock_t endio_list_lock;	struct list_head endio_list;	struct task_struct *endio_thread; 	struct task_struct *flush_thread;	struct bio_list flush_list; 	struct dm_kcopyd_client *dm_kcopyd;	unsigned long *dirty_bitmap;	unsigned int dirty_bitmap_size; 	struct bio_set bio_set;	mempool_t copy_pool; 	struct {		unsigned long long reads;		unsigned long long read_hits;		unsigned long long writes;		unsigned long long write_hits_uncommitted;		unsigned long long write_hits_committed;		unsigned long long writes_around;		unsigned long long writes_allocate;		unsigned long long writes_blocked_on_freelist;		unsigned long long flushes;		unsigned long long discards;	} stats;}; #define WB_LIST_INLINE		16 struct writeback_struct {	struct list_head endio_entry;	struct dm_writecache *wc;	struct wc_entry **wc_list;	unsigned int wc_list_n;	struct wc_entry *wc_list_inline[WB_LIST_INLINE];	struct bio bio;}; struct copy_struct {	struct list_head endio_entry;	struct dm_writecache *wc;	struct wc_entry *e;	unsigned int n_entries;	int error;}; DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(dm_writecache_throttle,					    "A percentage of time allocated for data copying"); static void wc_lock(struct dm_writecache *wc){	mutex_lock(&wc->lock);} static void wc_unlock(struct dm_writecache *wc){	mutex_unlock(&wc->lock);} #ifdef DM_WRITECACHE_HAS_PMEMstatic int persistent_memory_claim(struct dm_writecache *wc){	int r;	loff_t s;	long p, da;	unsigned long pfn;	int id;	struct page **pages;	sector_t offset; 	wc->memory_vmapped = false; 	s = wc->memory_map_size;	p = s >> PAGE_SHIFT;	if (!p) {		r = -EINVAL;		goto err1;	}	if (p != s >> PAGE_SHIFT) {		r = -EOVERFLOW;		goto err1;	} 	offset = get_start_sect(wc->ssd_dev->bdev);	if (offset & (PAGE_SIZE / 512 - 1)) {		r = -EINVAL;		goto err1;	}	offset >>= PAGE_SHIFT - 9; 	id = dax_read_lock(); 	da = dax_direct_access(wc->ssd_dev->dax_dev, offset, p, DAX_ACCESS,			&wc->memory_map, &pfn);	if (da < 0) {		wc->memory_map = NULL;		r = da;		goto err2;	}	if (!pfn_valid(pfn)) {		wc->memory_map = NULL;		r = -EOPNOTSUPP;		goto err2;	}	if (da != p) {		long i; 		wc->memory_map = NULL;		pages = vmalloc_array(p, sizeof(struct page *));		if (!pages) {			r = -ENOMEM;			goto err2;		}		i = 0;		do {			long daa; 			daa = dax_direct_access(wc->ssd_dev->dax_dev, offset + i,					p - i, DAX_ACCESS, NULL, &pfn);			if (daa <= 0) {				r = daa ? daa : -EINVAL;				goto err3;			}			if (!pfn_valid(pfn)) {				r = -EOPNOTSUPP;				goto err3;			}			while (daa-- && i < p) {				pages[i++] = pfn_to_page(pfn);				pfn++;				if (!(i & 15))					cond_resched();			}		} while (i < p);		wc->memory_map = vmap(pages, p, VM_MAP, PAGE_KERNEL);		if (!wc->memory_map) {			r = -ENOMEM;			goto err3;		}		vfree(pages);		wc->memory_vmapped = true;	} 	dax_read_unlock(id); 	wc->memory_map += (size_t)wc->start_sector << SECTOR_SHIFT;	wc->memory_map_size -= (size_t)wc->start_sector << SECTOR_SHIFT; 	return 0;err3:	vfree(pages);err2:	dax_read_unlock(id);err1:	return r;}#elsestatic int persistent_memory_claim(struct dm_writecache *wc){	return -EOPNOTSUPP;}#endif static void persistent_memory_release(struct dm_writecache *wc){	if (wc->memory_vmapped)		vunmap(wc->memory_map - ((size_t)wc->start_sector << SECTOR_SHIFT));} static struct page *persistent_memory_page(void *addr){	if (is_vmalloc_addr(addr))		return vmalloc_to_page(addr);	else		return virt_to_page(addr);} static unsigned int persistent_memory_page_offset(void *addr){	return (unsigned long)addr & (PAGE_SIZE - 1);} static void persistent_memory_flush_cache(void *ptr, size_t size){	if (is_vmalloc_addr(ptr))		flush_kernel_vmap_range(ptr, size);} static void persistent_memory_invalidate_cache(void *ptr, size_t size){	if (is_vmalloc_addr(ptr))		invalidate_kernel_vmap_range(ptr, size);} static struct wc_memory_superblock *sb(struct dm_writecache *wc){	return wc->memory_map;} static struct wc_memory_entry *memory_entry(struct dm_writecache *wc, struct wc_entry *e){	return &sb(wc)->entries[e->index];} static void *memory_data(struct dm_writecache *wc, struct wc_entry *e){	return (char *)wc->block_start + (e->index << wc->block_size_bits);} static sector_t cache_sector(struct dm_writecache *wc, struct wc_entry *e){	return wc->start_sector + wc->metadata_sectors +		((sector_t)e->index << (wc->block_size_bits - SECTOR_SHIFT));} static uint64_t read_original_sector(struct dm_writecache *wc, struct wc_entry *e){#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS	return e->original_sector;#else	return le64_to_cpu(memory_entry(wc, e)->original_sector);#endif} static uint64_t read_seq_count(struct dm_writecache *wc, struct wc_entry *e){#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS	return e->seq_count;#else	return le64_to_cpu(memory_entry(wc, e)->seq_count);#endif} static void clear_seq_count(struct dm_writecache *wc, struct wc_entry *e){#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS	e->seq_count = -1;#endif	pmem_assign(memory_entry(wc, e)->seq_count, cpu_to_le64(-1));} static void write_original_sector_seq_count(struct dm_writecache *wc, struct wc_entry *e,					    uint64_t original_sector, uint64_t seq_count){	struct wc_memory_entry me;#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS	e->original_sector = original_sector;	e->seq_count = seq_count;#endif	me.original_sector = cpu_to_le64(original_sector);	me.seq_count = cpu_to_le64(seq_count);	pmem_assign(*memory_entry(wc, e), me);} #define writecache_error(wc, err, msg, arg...)				\do {									\	if (!cmpxchg(&(wc)->error, 0, err))				\		DMERR(msg, ##arg);					\	wake_up(&(wc)->freelist_wait);					\} while (0) #define writecache_has_error(wc)	(unlikely(READ_ONCE((wc)->error))) static void writecache_flush_all_metadata(struct dm_writecache *wc){	if (!WC_MODE_PMEM(wc))		memset(wc->dirty_bitmap, -1, wc->dirty_bitmap_size);} static void writecache_flush_region(struct dm_writecache *wc, void *ptr, size_t size){	if (!WC_MODE_PMEM(wc))		__set_bit(((char *)ptr - (char *)wc->memory_map) / BITMAP_GRANULARITY,			  wc->dirty_bitmap);} static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev); struct io_notify {	struct dm_writecache *wc;	struct completion c;	atomic_t count;}; static void writecache_notify_io(unsigned long error, unsigned long unsup, void *context){	struct io_notify *endio = context; 	if (unlikely(error != 0))		writecache_error(endio->wc, -EIO, "error writing metadata");	else if (unlikely(unsup != 0))		writecache_error(endio->wc, -EOPNOTSUPP, "error writing metadata");	BUG_ON(atomic_read(&endio->count) <= 0);	if (atomic_dec_and_test(&endio->count))		complete(&endio->c);} static void writecache_wait_for_ios(struct dm_writecache *wc, int direction){	wait_event(wc->bio_in_progress_wait[direction],		   !atomic_read(&wc->bio_in_progress[direction]));} static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios){	struct dm_io_region region;	struct dm_io_request req;	struct io_notify endio = {

Lines 1–500 of 2773. Load more