// SPDX-License-Identifier: GPL-2.0/* * Copyright (C) 2018 Red Hat. All rights reserved. * * This file is released under the GPL. */ #include <linux/device-mapper.h>#include <linux/module.h>#include <linux/init.h>#include <linux/vmalloc.h>#include <linux/kthread.h>#include <linux/dm-io.h>#include <linux/dm-kcopyd.h>#include <linux/dax.h>#include <linux/pfn_t.h>#include <linux/libnvdimm.h> #define DM_MSG_PREFIX "writecache" #define HIGH_WATERMARK			50#define LOW_WATERMARK			45#define MAX_WRITEBACK_JOBS		0#define ENDIO_LATENCY			16#define WRITEBACK_LATENCY		64#define AUTOCOMMIT_BLOCKS_SSD		65536#define AUTOCOMMIT_BLOCKS_PMEM		64#define AUTOCOMMIT_MSEC			1000#define MAX_AGE_DIV			16#define MAX_AGE_UNSPECIFIED		-1UL #define BITMAP_GRANULARITY	65536#if BITMAP_GRANULARITY < PAGE_SIZE#undef BITMAP_GRANULARITY#define BITMAP_GRANULARITY	PAGE_SIZE#endif #if IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API) && IS_ENABLED(CONFIG_DAX_DRIVER)#define DM_WRITECACHE_HAS_PMEM#endif #ifdef DM_WRITECACHE_HAS_PMEM#define pmem_assign(dest, src)					\do {								\	typeof(dest) uniq = (src);				\	memcpy_flushcache(&(dest), &uniq, sizeof(dest));	\} while (0)#else#define pmem_assign(dest, src)	((dest) = (src))#endif #if defined(__HAVE_ARCH_MEMCPY_MCSAFE) && defined(DM_WRITECACHE_HAS_PMEM)#define DM_WRITECACHE_HANDLE_HARDWARE_ERRORS#endif #define MEMORY_SUPERBLOCK_MAGIC		0x23489321#define MEMORY_SUPERBLOCK_VERSION	1 struct wc_memory_entry {	__le64 original_sector;	__le64 seq_count;}; struct wc_memory_superblock {	union {		struct {			__le32 magic;			__le32 version;			__le32 block_size;			__le32 pad;			__le64 n_blocks;			__le64 seq_count;		};		__le64 padding[8];	};	struct wc_memory_entry entries[0];}; struct wc_entry {	struct rb_node rb_node;	struct list_head lru;	unsigned short wc_list_contiguous;	bool write_in_progress#if BITS_PER_LONG == 64		:1#endif	;	unsigned long index#if BITS_PER_LONG == 64		:47#endif	;	unsigned long age;#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS	uint64_t original_sector;	uint64_t seq_count;#endif}; #ifdef DM_WRITECACHE_HAS_PMEM#define WC_MODE_PMEM(wc)			((wc)->pmem_mode)#define WC_MODE_FUA(wc)				((wc)->writeback_fua)#else#define WC_MODE_PMEM(wc)			false#define WC_MODE_FUA(wc)				false#endif#define WC_MODE_SORT_FREELIST(wc)		(!WC_MODE_PMEM(wc)) struct dm_writecache {	struct mutex lock;	struct list_head lru;	union {		struct list_head freelist;		struct {			struct rb_root freetree;			struct wc_entry *current_free;		};	};	struct rb_root tree; 	size_t freelist_size;	size_t writeback_size;	size_t freelist_high_watermark;	size_t freelist_low_watermark;	unsigned long max_age; 	unsigned uncommitted_blocks;	unsigned autocommit_blocks;	unsigned max_writeback_jobs; 	int error; 	unsigned long autocommit_jiffies;	struct timer_list autocommit_timer;	struct wait_queue_head freelist_wait; 	struct timer_list max_age_timer; 	atomic_t bio_in_progress[2];	struct wait_queue_head bio_in_progress_wait[2]; 	struct dm_target *ti;	struct dm_dev *dev;	struct dm_dev *ssd_dev;	sector_t start_sector;	void *memory_map;	uint64_t memory_map_size;	size_t metadata_sectors;	size_t n_blocks;	uint64_t seq_count;	void *block_start;	struct wc_entry *entries;	unsigned block_size;	unsigned char block_size_bits; 	bool pmem_mode:1;	bool writeback_fua:1; 	bool overwrote_committed:1;	bool memory_vmapped:1; 	bool high_wm_percent_set:1;	bool low_wm_percent_set:1;	bool max_writeback_jobs_set:1;	bool autocommit_blocks_set:1;	bool autocommit_time_set:1;	bool writeback_fua_set:1;	bool flush_on_suspend:1;	bool cleaner:1; 	unsigned writeback_all;	struct workqueue_struct *writeback_wq;	struct work_struct writeback_work;	struct work_struct flush_work; 	struct dm_io_client *dm_io; 	raw_spinlock_t endio_list_lock;	struct list_head endio_list;	struct task_struct *endio_thread; 	struct task_struct *flush_thread;	struct bio_list flush_list; 	struct dm_kcopyd_client *dm_kcopyd;	unsigned long *dirty_bitmap;	unsigned dirty_bitmap_size; 	struct bio_set bio_set;	mempool_t copy_pool;}; #define WB_LIST_INLINE		16 struct writeback_struct {	struct list_head endio_entry;	struct dm_writecache *wc;	struct wc_entry **wc_list;	unsigned wc_list_n;	struct wc_entry *wc_list_inline[WB_LIST_INLINE];	struct bio bio;}; struct copy_struct {	struct list_head endio_entry;	struct dm_writecache *wc;	struct wc_entry *e;	unsigned n_entries;	int error;}; DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(dm_writecache_throttle,					    "A percentage of time allocated for data copying"); static void wc_lock(struct dm_writecache *wc){	mutex_lock(&wc->lock);} static void wc_unlock(struct dm_writecache *wc){	mutex_unlock(&wc->lock);} #ifdef DM_WRITECACHE_HAS_PMEMstatic int persistent_memory_claim(struct dm_writecache *wc){	int r;	loff_t s;	long p, da;	pfn_t pfn;	int id;	struct page **pages; 	wc->memory_vmapped = false; 	s = wc->memory_map_size;	p = s >> PAGE_SHIFT;	if (!p) {		r = -EINVAL;		goto err1;	}	if (p != s >> PAGE_SHIFT) {		r = -EOVERFLOW;		goto err1;	} 	id = dax_read_lock(); 	da = dax_direct_access(wc->ssd_dev->dax_dev, 0, p, &wc->memory_map, &pfn);	if (da < 0) {		wc->memory_map = NULL;		r = da;		goto err2;	}	if (!pfn_t_has_page(pfn)) {		wc->memory_map = NULL;		r = -EOPNOTSUPP;		goto err2;	}	if (da != p) {		long i;		wc->memory_map = NULL;		pages = kvmalloc_array(p, sizeof(struct page *), GFP_KERNEL);		if (!pages) {			r = -ENOMEM;			goto err2;		}		i = 0;		do {			long daa;			daa = dax_direct_access(wc->ssd_dev->dax_dev, i, p - i,						NULL, &pfn);			if (daa <= 0) {				r = daa ? daa : -EINVAL;				goto err3;			}			if (!pfn_t_has_page(pfn)) {				r = -EOPNOTSUPP;				goto err3;			}			while (daa-- && i < p) {				pages[i++] = pfn_t_to_page(pfn);				pfn.val++;				if (!(i & 15))					cond_resched();			}		} while (i < p);		wc->memory_map = vmap(pages, p, VM_MAP, PAGE_KERNEL);		if (!wc->memory_map) {			r = -ENOMEM;			goto err3;		}		kvfree(pages);		wc->memory_vmapped = true;	} 	dax_read_unlock(id); 	wc->memory_map += (size_t)wc->start_sector << SECTOR_SHIFT;	wc->memory_map_size -= (size_t)wc->start_sector << SECTOR_SHIFT; 	return 0;err3:	kvfree(pages);err2:	dax_read_unlock(id);err1:	return r;}#elsestatic int persistent_memory_claim(struct dm_writecache *wc){	BUG();}#endif static void persistent_memory_release(struct dm_writecache *wc){	if (wc->memory_vmapped)		vunmap(wc->memory_map - ((size_t)wc->start_sector << SECTOR_SHIFT));} static struct page *persistent_memory_page(void *addr){	if (is_vmalloc_addr(addr))		return vmalloc_to_page(addr);	else		return virt_to_page(addr);} static unsigned persistent_memory_page_offset(void *addr){	return (unsigned long)addr & (PAGE_SIZE - 1);} static void persistent_memory_flush_cache(void *ptr, size_t size){	if (is_vmalloc_addr(ptr))		flush_kernel_vmap_range(ptr, size);} static void persistent_memory_invalidate_cache(void *ptr, size_t size){	if (is_vmalloc_addr(ptr))		invalidate_kernel_vmap_range(ptr, size);} static struct wc_memory_superblock *sb(struct dm_writecache *wc){	return wc->memory_map;} static struct wc_memory_entry *memory_entry(struct dm_writecache *wc, struct wc_entry *e){	return &sb(wc)->entries[e->index];} static void *memory_data(struct dm_writecache *wc, struct wc_entry *e){	return (char *)wc->block_start + (e->index << wc->block_size_bits);} static sector_t cache_sector(struct dm_writecache *wc, struct wc_entry *e){	return wc->start_sector + wc->metadata_sectors +		((sector_t)e->index << (wc->block_size_bits - SECTOR_SHIFT));} static uint64_t read_original_sector(struct dm_writecache *wc, struct wc_entry *e){#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS	return e->original_sector;#else	return le64_to_cpu(memory_entry(wc, e)->original_sector);#endif} static uint64_t read_seq_count(struct dm_writecache *wc, struct wc_entry *e){#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS	return e->seq_count;#else	return le64_to_cpu(memory_entry(wc, e)->seq_count);#endif} static void clear_seq_count(struct dm_writecache *wc, struct wc_entry *e){#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS	e->seq_count = -1;#endif	pmem_assign(memory_entry(wc, e)->seq_count, cpu_to_le64(-1));} static void write_original_sector_seq_count(struct dm_writecache *wc, struct wc_entry *e,					    uint64_t original_sector, uint64_t seq_count){	struct wc_memory_entry me;#ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS	e->original_sector = original_sector;	e->seq_count = seq_count;#endif	me.original_sector = cpu_to_le64(original_sector);	me.seq_count = cpu_to_le64(seq_count);	pmem_assign(*memory_entry(wc, e), me);} #define writecache_error(wc, err, msg, arg...)				\do {									\	if (!cmpxchg(&(wc)->error, 0, err))				\		DMERR(msg, ##arg);					\	wake_up(&(wc)->freelist_wait);					\} while (0) #define writecache_has_error(wc)	(unlikely(READ_ONCE((wc)->error))) static void writecache_flush_all_metadata(struct dm_writecache *wc){	if (!WC_MODE_PMEM(wc))		memset(wc->dirty_bitmap, -1, wc->dirty_bitmap_size);} static void writecache_flush_region(struct dm_writecache *wc, void *ptr, size_t size){	if (!WC_MODE_PMEM(wc))		__set_bit(((char *)ptr - (char *)wc->memory_map) / BITMAP_GRANULARITY,			  wc->dirty_bitmap);} static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev); struct io_notify {	struct dm_writecache *wc;	struct completion c;	atomic_t count;}; static void writecache_notify_io(unsigned long error, void *context){	struct io_notify *endio = context; 	if (unlikely(error != 0))		writecache_error(endio->wc, -EIO, "error writing metadata");	BUG_ON(atomic_read(&endio->count) <= 0);	if (atomic_dec_and_test(&endio->count))		complete(&endio->c);} static void writecache_wait_for_ios(struct dm_writecache *wc, int direction){	wait_event(wc->bio_in_progress_wait[direction],		   !atomic_read(&wc->bio_in_progress[direction]));} static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios){	struct dm_io_region region;	struct dm_io_request req;	struct io_notify endio = {		wc,		COMPLETION_INITIALIZER_ONSTACK(endio.c),		ATOMIC_INIT(1),	};	unsigned bitmap_bits = wc->dirty_bitmap_size * 8;	unsigned i = 0; 	while (1) {		unsigned j;		i = find_next_bit(wc->dirty_bitmap, bitmap_bits, i);		if (unlikely(i == bitmap_bits))			break;		j = find_next_zero_bit(wc->dirty_bitmap, bitmap_bits, i); 		region.bdev = wc->ssd_dev->bdev;		region.sector = (sector_t)i * (BITMAP_GRANULARITY >> SECTOR_SHIFT);		region.count = (sector_t)(j - i) * (BITMAP_GRANULARITY >> SECTOR_SHIFT); 		if (unlikely(region.sector >= wc->metadata_sectors))			break;		if (unlikely(region.sector + region.count > wc->metadata_sectors))			region.count = wc->metadata_sectors - region.sector; 		region.sector += wc->start_sector;		atomic_inc(&endio.count);		req.bi_op = REQ_OP_WRITE;		req.bi_op_flags = REQ_SYNC;		req.mem.type = DM_IO_VMA;		req.mem.ptr.vma = (char *)wc->memory_map + (size_t)i * BITMAP_GRANULARITY;		req.client = wc->dm_io;		req.notify.fn = writecache_notify_io;		req.notify.context = &endio; 		/* writing via async dm-io (implied by notify.fn above) won't return an error */	        (void) dm_io(&req, 1, &region, NULL);		i = j;	} 	writecache_notify_io(0, &endio);	wait_for_completion_io(&endio.c);

Lines 1–500 of 2546. Load more