// SPDX-License-Identifier: GPL-2.0-only/* * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003 * * bitmap_create  - sets up the bitmap structure * bitmap_destroy - destroys the bitmap structure * * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.: * - added disk storage for bitmap * - changes to allow various bitmap chunk sizes */ /* * Still to do: * * flush after percent set rather than just time based. (maybe both). */ #include <linux/blkdev.h>#include <linux/module.h>#include <linux/errno.h>#include <linux/slab.h>#include <linux/init.h>#include <linux/timer.h>#include <linux/sched.h>#include <linux/list.h>#include <linux/file.h>#include <linux/mount.h>#include <linux/buffer_head.h>#include <linux/seq_file.h>#include <trace/events/block.h> #include "md.h"#include "md-bitmap.h"#include "md-cluster.h" /* * in-memory bitmap: * * Use 16 bit block counters to track pending writes to each "chunk". * The 2 high order bits are special-purpose, the first is a flag indicating * whether a resync is needed.  The second is a flag indicating whether a * resync is active. * This means that the counter is actually 14 bits: * * +--------+--------+------------------------------------------------+ * | resync | resync |               counter                          | * | needed | active |                                                | * |  (0-1) |  (0-1) |              (0-16383)                         | * +--------+--------+------------------------------------------------+ * * The "resync needed" bit is set when: *    a '1' bit is read from storage at startup. *    a write request fails on some drives *    a resync is aborted on a chunk with 'resync active' set * It is cleared (and resync-active set) when a resync starts across all drives * of the chunk. * * * The "resync active" bit is set when: *    a resync is started on all drives, and resync_needed is set. *       resync_needed will be cleared (as long as resync_active wasn't already set). * It is cleared when a resync completes. * * The counter counts pending write requests, plus the on-disk bit. * When the counter is '1' and the resync bits are clear, the on-disk * bit can be cleared as well, thus setting the counter to 0. * When we set a bit, or in the counter (to start a write), if the fields is * 0, we first set the disk bit and set the counter to 1. * * If the counter is 0, the on-disk bit is clear and the stripe is clean * Anything that dirties the stripe pushes the counter to 2 (at least) * and sets the on-disk bit (lazily). * If a periodic sweep find the counter at 2, it is decremented to 1. * If the sweep find the counter at 1, the on-disk bit is cleared and the * counter goes to zero. * * Also, we'll hijack the "map" pointer itself and use it as two 16 bit block * counters as a fallback when "page" memory cannot be allocated: * * Normal case (page memory allocated): * *     page pointer (32-bit) * *     [ ] ------+ *               | *               +-------> [   ][   ]..[   ] (4096 byte page == 2048 counters) *                          c1   c2    c2048 * * Hijacked case (page memory allocation failed): * *     hijacked page pointer (32-bit) * *     [		  ][		  ] (no page memory allocated) *      counter #1 (16-bit) counter #2 (16-bit) * */ typedef __u16 bitmap_counter_t; #define PAGE_BITS (PAGE_SIZE << 3)#define PAGE_BIT_SHIFT (PAGE_SHIFT + 3) #define COUNTER_BITS 16#define COUNTER_BIT_SHIFT 4#define COUNTER_BYTE_SHIFT (COUNTER_BIT_SHIFT - 3) #define NEEDED_MASK ((bitmap_counter_t) (1 << (COUNTER_BITS - 1)))#define RESYNC_MASK ((bitmap_counter_t) (1 << (COUNTER_BITS - 2)))#define COUNTER_MAX ((bitmap_counter_t) RESYNC_MASK - 1) #define NEEDED(x) (((bitmap_counter_t) x) & NEEDED_MASK)#define RESYNC(x) (((bitmap_counter_t) x) & RESYNC_MASK)#define COUNTER(x) (((bitmap_counter_t) x) & COUNTER_MAX) /* how many counters per page? */#define PAGE_COUNTER_RATIO (PAGE_BITS / COUNTER_BITS)/* same, except a shift value for more efficient bitops */#define PAGE_COUNTER_SHIFT (PAGE_BIT_SHIFT - COUNTER_BIT_SHIFT)/* same, except a mask value for more efficient bitops */#define PAGE_COUNTER_MASK  (PAGE_COUNTER_RATIO - 1) #define BITMAP_BLOCK_SHIFT 9 /* * bitmap structures: */ /* the in-memory bitmap is represented by bitmap_pages */struct bitmap_page {	/*	 * map points to the actual memory page	 */	char *map;	/*	 * in emergencies (when map cannot be alloced), hijack the map	 * pointer and use it as two counters itself	 */	unsigned int hijacked:1;	/*	 * If any counter in this page is '1' or '2' - and so could be	 * cleared then that page is marked as 'pending'	 */	unsigned int pending:1;	/*	 * count of dirty bits on the page	 */	unsigned int  count:30;}; /* the main bitmap structure - one per mddev */struct bitmap { 	struct bitmap_counts {		spinlock_t lock;		struct bitmap_page *bp;		/* total number of pages in the bitmap */		unsigned long pages;		/* number of pages not yet allocated */		unsigned long missing_pages;		/* chunksize = 2^chunkshift (for bitops) */		unsigned long chunkshift;		/* total number of data chunks for the array */		unsigned long chunks;	} counts; 	struct mddev *mddev; /* the md device that the bitmap is for */ 	__u64	events_cleared;	int need_sync; 	struct bitmap_storage {		/* backing disk file */		struct file *file;		/* cached copy of the bitmap file superblock */		struct page *sb_page;		unsigned long sb_index;		/* list of cache pages for the file */		struct page **filemap;		/* attributes associated filemap pages */		unsigned long *filemap_attr;		/* number of pages in the file */		unsigned long file_pages;		/* total bytes in the bitmap */		unsigned long bytes;	} storage; 	unsigned long flags; 	int allclean; 	atomic_t behind_writes;	/* highest actual value at runtime */	unsigned long behind_writes_used; 	/*	 * the bitmap daemon - periodically wakes up and sweeps the bitmap	 * file, cleaning up bits and flushing out pages to disk as necessary	 */	unsigned long daemon_lastrun; /* jiffies of last run */	/*	 * when we lasted called end_sync to update bitmap with resync	 * progress.	 */	unsigned long last_end_sync; 	/* pending writes to the bitmap file */	atomic_t pending_writes;	wait_queue_head_t write_wait;	wait_queue_head_t overflow_wait;	wait_queue_head_t behind_wait; 	struct kernfs_node *sysfs_can_clear;	/* slot offset for clustered env */	int cluster_slot;}; static struct workqueue_struct *md_bitmap_wq;static struct attribute_group md_bitmap_internal_group; static int __bitmap_resize(struct bitmap *bitmap, sector_t blocks,			   int chunksize, bool init); static inline char *bmname(struct bitmap *bitmap){	return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";} static bool bitmap_enabled(void *data, bool flush){	struct bitmap *bitmap = data; 	if (!flush)		return true; 	/*	 * If caller want to flush bitmap pages to underlying disks, check if	 * there are cached pages in filemap.	 */	return !test_bit(BITMAP_STALE, &bitmap->flags) &&	       bitmap->storage.filemap != NULL;} /* * check a page and, if necessary, allocate it (or hijack it if the alloc fails) * * 1) check to see if this page is allocated, if it's not then try to alloc * 2) if the alloc fails, set the page's hijacked flag so we'll use the *    page pointer directly as a counter * * if we find our page, we increment the page's refcount so that it stays * allocated while we're using it */static int md_bitmap_checkpage(struct bitmap_counts *bitmap,			       unsigned long page, int create, int no_hijack)__releases(bitmap->lock)__acquires(bitmap->lock){	unsigned char *mappage; 	WARN_ON_ONCE(page >= bitmap->pages);	if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */		return 0; 	if (bitmap->bp[page].map) /* page is already allocated, just return */		return 0; 	if (!create)		return -ENOENT; 	/* this page has not been allocated yet */ 	spin_unlock_irq(&bitmap->lock);	/* It is possible that this is being called inside a	 * prepare_to_wait/finish_wait loop from raid5c:make_request().	 * In general it is not permitted to sleep in that context as it	 * can cause the loop to spin freely.	 * That doesn't apply here as we can only reach this point	 * once with any loop.	 * When this function completes, either bp[page].map or	 * bp[page].hijacked.  In either case, this function will	 * abort before getting to this point again.  So there is	 * no risk of a free-spin, and so it is safe to assert	 * that sleeping here is allowed.	 */	sched_annotate_sleep();	mappage = kzalloc(PAGE_SIZE, GFP_NOIO);	spin_lock_irq(&bitmap->lock); 	if (mappage == NULL) {		pr_debug("md/bitmap: map page allocation failed, hijacking\n");		/* We don't support hijack for cluster raid */		if (no_hijack)			return -ENOMEM;		/* failed - set the hijacked flag so that we can use the		 * pointer as a counter */		if (!bitmap->bp[page].map)			bitmap->bp[page].hijacked = 1;	} else if (bitmap->bp[page].map ||		   bitmap->bp[page].hijacked) {		/* somebody beat us to getting the page */		kfree(mappage);	} else { 		/* no page was in place and we have one, so install it */ 		bitmap->bp[page].map = mappage;		bitmap->missing_pages--;	}	return 0;} /* if page is completely empty, put it back on the free list, or dealloc it *//* if page was hijacked, unmark the flag so it might get alloced next time *//* Note: lock should be held when calling this */static void md_bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page){	char *ptr; 	if (bitmap->bp[page].count) /* page is still busy */		return; 	/* page is no longer in use, it can be released */ 	if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */		bitmap->bp[page].hijacked = 0;		bitmap->bp[page].map = NULL;	} else {		/* normal case, free the page */		ptr = bitmap->bp[page].map;		bitmap->bp[page].map = NULL;		bitmap->missing_pages++;		kfree(ptr);	}} /* * bitmap file handling - read and write the bitmap file and its superblock */ /* * basic page I/O operations */ /* IO operations when bitmap is stored near all superblocks */ /* choose a good rdev and read the page from there */static int read_sb_page(struct mddev *mddev, loff_t offset,		struct page *page, unsigned long index, int size){ 	sector_t sector = mddev->bitmap_info.offset + offset +		index * (PAGE_SIZE / SECTOR_SIZE);	struct md_rdev *rdev; 	rdev_for_each(rdev, mddev) {		u32 iosize = roundup(size, bdev_logical_block_size(rdev->bdev)); 		if (!test_bit(In_sync, &rdev->flags) ||		    test_bit(Faulty, &rdev->flags) ||		    test_bit(Bitmap_sync, &rdev->flags))			continue; 		if (sync_page_io(rdev, sector, iosize, page, REQ_OP_READ, true))			return 0;	}	return -EIO;} static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mddev){	/* Iterate the disks of an mddev, using rcu to protect access to the	 * linked list, and raising the refcount of devices we return to ensure	 * they don't disappear while in use.	 * As devices are only added or removed when raid_disk is < 0 and	 * nr_pending is 0 and In_sync is clear, the entries we return will	 * still be in the same position on the list when we re-enter	 * list_for_each_entry_continue_rcu.	 *	 * Note that if entered with 'rdev == NULL' to start at the	 * beginning, we temporarily assign 'rdev' to an address which	 * isn't really an rdev, but which can be used by	 * list_for_each_entry_continue_rcu() to find the first entry.	 */	rcu_read_lock();	if (rdev == NULL)		/* start at the beginning */		rdev = list_entry(&mddev->disks, struct md_rdev, same_set);	else {		/* release the previous rdev and start from there. */		rdev_dec_pending(rdev, mddev);	}	list_for_each_entry_continue_rcu(rdev, &mddev->disks, same_set) {		if (rdev->raid_disk >= 0 &&		    !test_bit(Faulty, &rdev->flags)) {			/* this is a usable devices */			atomic_inc(&rdev->nr_pending);			rcu_read_unlock();			return rdev;		}	}	rcu_read_unlock();	return NULL;} static unsigned int optimal_io_size(struct block_device *bdev,				    unsigned int last_page_size,				    unsigned int io_size){	if (bdev_io_opt(bdev) > bdev_logical_block_size(bdev))		return roundup(last_page_size, bdev_io_opt(bdev));	return io_size;} static unsigned int bitmap_io_size(unsigned int io_size, unsigned int opt_size,				   loff_t start, loff_t boundary){	if (io_size != opt_size &&	    start + opt_size / SECTOR_SIZE <= boundary)		return opt_size;	if (start + io_size / SECTOR_SIZE <= boundary)		return io_size; 	/* Overflows boundary */	return 0;} static int __write_sb_page(struct md_rdev *rdev, struct bitmap *bitmap,			   unsigned long pg_index, struct page *page){	struct block_device *bdev;	struct mddev *mddev = bitmap->mddev;	struct bitmap_storage *store = &bitmap->storage;	unsigned long num_pages = bitmap->storage.file_pages;	unsigned int bitmap_limit = (num_pages - pg_index % num_pages) << PAGE_SHIFT;	loff_t sboff, offset = mddev->bitmap_info.offset;	sector_t ps = pg_index * PAGE_SIZE / SECTOR_SIZE;	unsigned int size = PAGE_SIZE;	unsigned int opt_size = PAGE_SIZE;	sector_t doff; 	bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;	/* we compare length (page numbers), not page offset. */	if ((pg_index - store->sb_index) == num_pages - 1) {		unsigned int last_page_size = store->bytes & (PAGE_SIZE - 1); 		if (last_page_size == 0)			last_page_size = PAGE_SIZE;		size = roundup(last_page_size, bdev_logical_block_size(bdev));		opt_size = optimal_io_size(bdev, last_page_size, size);	} 	sboff = rdev->sb_start + offset;	doff = rdev->data_offset; 	/* Just make sure we aren't corrupting data or metadata */	if (mddev->external) {		/* Bitmap could be anywhere. */		if (sboff + ps > doff &&		    sboff < (doff + mddev->dev_sectors + PAGE_SIZE / SECTOR_SIZE))			return -EINVAL;	} else if (offset < 0) {		/* DATA  BITMAP METADATA  */		size = bitmap_io_size(size, opt_size, offset + ps, 0);		if (size == 0)			/* bitmap runs in to metadata */			return -EINVAL; 		if (doff + mddev->dev_sectors > sboff)			/* data runs in to bitmap */			return -EINVAL;	} else if (rdev->sb_start < rdev->data_offset) {		/* METADATA BITMAP DATA */		size = bitmap_io_size(size, opt_size, sboff + ps, doff);		if (size == 0)			/* bitmap runs in to data */			return -EINVAL;	} 	md_write_metadata(mddev, rdev, sboff + ps, (int)min(size, bitmap_limit),			  page, 0);	return 0;} static void write_sb_page(struct bitmap *bitmap, unsigned long pg_index,			  struct page *page, bool wait){	struct mddev *mddev = bitmap->mddev; 	do {		struct md_rdev *rdev = NULL; 		while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {			if (__write_sb_page(rdev, bitmap, pg_index, page) < 0) {				set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);				return;			}		}	} while (wait && md_super_wait(mddev) < 0);}

Lines 1–500 of 3152. Load more