// SPDX-License-Identifier: GPL-2.0-only/*   drbd_bitmap.c    This file is part of DRBD by Philipp Reisner and Lars Ellenberg.    Copyright (C) 2004-2008, LINBIT Information Technologies GmbH.   Copyright (C) 2004-2008, Philipp Reisner <philipp.reisner@linbit.com>.   Copyright (C) 2004-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.  */ #define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt #include <linux/bitmap.h>#include <linux/vmalloc.h>#include <linux/string.h>#include <linux/drbd.h>#include <linux/slab.h>#include <linux/highmem.h> #include "drbd_int.h"  /* OPAQUE outside this file! * interface defined in drbd_int.h  * convention: * function name drbd_bm_... => used elsewhere, "public". * function name      bm_... => internal to implementation, "private". */  /* * LIMITATIONS: * We want to support >= peta byte of backend storage, while for now still using * a granularity of one bit per 4KiB of storage. * 1 << 50		bytes backend storage (1 PiB) * 1 << (50 - 12)	bits needed *	38 --> we need u64 to index and count bits * 1 << (38 - 3)	bitmap bytes needed *	35 --> we still need u64 to index and count bytes *			(that's 32 GiB of bitmap for 1 PiB storage) * 1 << (35 - 2)	32bit longs needed *	33 --> we'd even need u64 to index and count 32bit long words. * 1 << (35 - 3)	64bit longs needed *	32 --> we could get away with a 32bit unsigned int to index and count *	64bit long words, but I rather stay with unsigned long for now. *	We probably should neither count nor point to bytes or long words *	directly, but either by bitnumber, or by page index and offset. * 1 << (35 - 12) *	22 --> we need that much 4KiB pages of bitmap. *	1 << (22 + 3) --> on a 64bit arch, *	we need 32 MiB to store the array of page pointers. * * Because I'm lazy, and because the resulting patch was too large, too ugly * and still incomplete, on 32bit we still "only" support 16 TiB (minus some), * (1 << 32) bits * 4k storage. *  * bitmap storage and IO: *	Bitmap is stored little endian on disk, and is kept little endian in *	core memory. Currently we still hold the full bitmap in core as long *	as we are "attached" to a local disk, which at 32 GiB for 1PiB storage *	seems excessive. * *	We plan to reduce the amount of in-core bitmap pages by paging them in *	and out against their on-disk location as necessary, but need to make *	sure we don't cause too much meta data IO, and must not deadlock in *	tight memory situations. This needs some more work. */ /* * NOTE *  Access to the *bm_pages is protected by bm_lock. *  It is safe to read the other members within the lock. * *  drbd_bm_set_bits is called from bio_endio callbacks, *  We may be called with irq already disabled, *  so we need spin_lock_irqsave(). *  And we need the kmap_atomic. */struct drbd_bitmap {	struct page **bm_pages;	spinlock_t bm_lock; 	/* exclusively to be used by __al_write_transaction(),	 * drbd_bm_mark_for_writeout() and	 * and drbd_bm_write_hinted() -> bm_rw() called from there.	 */	unsigned int n_bitmap_hints;	unsigned int al_bitmap_hints[AL_UPDATES_PER_TRANSACTION]; 	/* see LIMITATIONS: above */ 	unsigned long bm_set;       /* nr of set bits; THINK maybe atomic_t? */	unsigned long bm_bits;	size_t   bm_words;	size_t   bm_number_of_pages;	sector_t bm_dev_capacity;	struct mutex bm_change; /* serializes resize operations */ 	wait_queue_head_t bm_io_wait; /* used to serialize IO of single pages */ 	enum bm_flag bm_flags; 	/* debugging aid, in case we are still racy somewhere */	char          *bm_why;	struct task_struct *bm_task;}; #define bm_print_lock_info(m) __bm_print_lock_info(m, __func__)static void __bm_print_lock_info(struct drbd_device *device, const char *func){	struct drbd_bitmap *b = device->bitmap;	if (!drbd_ratelimit())		return;	drbd_err(device, "FIXME %s[%d] in %s, bitmap locked for '%s' by %s[%d]\n",		 current->comm, task_pid_nr(current),		 func, b->bm_why ?: "?",		 b->bm_task->comm, task_pid_nr(b->bm_task));} void drbd_bm_lock(struct drbd_device *device, char *why, enum bm_flag flags){	struct drbd_bitmap *b = device->bitmap;	int trylock_failed; 	if (!b) {		drbd_err(device, "FIXME no bitmap in drbd_bm_lock!?\n");		return;	} 	trylock_failed = !mutex_trylock(&b->bm_change); 	if (trylock_failed) {		drbd_warn(device, "%s[%d] going to '%s' but bitmap already locked for '%s' by %s[%d]\n",			  current->comm, task_pid_nr(current),			  why, b->bm_why ?: "?",			  b->bm_task->comm, task_pid_nr(b->bm_task));		mutex_lock(&b->bm_change);	}	if (BM_LOCKED_MASK & b->bm_flags)		drbd_err(device, "FIXME bitmap already locked in bm_lock\n");	b->bm_flags |= flags & BM_LOCKED_MASK; 	b->bm_why  = why;	b->bm_task = current;} void drbd_bm_unlock(struct drbd_device *device){	struct drbd_bitmap *b = device->bitmap;	if (!b) {		drbd_err(device, "FIXME no bitmap in drbd_bm_unlock!?\n");		return;	} 	if (!(BM_LOCKED_MASK & device->bitmap->bm_flags))		drbd_err(device, "FIXME bitmap not locked in bm_unlock\n"); 	b->bm_flags &= ~BM_LOCKED_MASK;	b->bm_why  = NULL;	b->bm_task = NULL;	mutex_unlock(&b->bm_change);} /* we store some "meta" info about our pages in page->private *//* at a granularity of 4k storage per bitmap bit: * one peta byte storage: 1<<50 byte, 1<<38 * 4k storage blocks *  1<<38 bits, *  1<<23 4k bitmap pages. * Use 24 bits as page index, covers 2 peta byte storage * at a granularity of 4k per bit. * Used to report the failed page idx on io error from the endio handlers. */#define BM_PAGE_IDX_MASK	((1UL<<24)-1)/* this page is currently read in, or written back */#define BM_PAGE_IO_LOCK		31/* if there has been an IO error for this page */#define BM_PAGE_IO_ERROR	30/* this is to be able to intelligently skip disk IO, * set if bits have been set since last IO. */#define BM_PAGE_NEED_WRITEOUT	29/* to mark for lazy writeout once syncer cleared all clearable bits, * we if bits have been cleared since last IO. */#define BM_PAGE_LAZY_WRITEOUT	28/* pages marked with this "HINT" will be considered for writeout * on activity log transactions */#define BM_PAGE_HINT_WRITEOUT	27 /* store_page_idx uses non-atomic assignment. It is only used directly after * allocating the page.  All other bm_set_page_* and bm_clear_page_* need to * use atomic bit manipulation, as set_out_of_sync (and therefore bitmap * changes) may happen from various contexts, and wait_on_bit/wake_up_bit * requires it all to be atomic as well. */static void bm_store_page_idx(struct page *page, unsigned long idx){	BUG_ON(0 != (idx & ~BM_PAGE_IDX_MASK));	set_page_private(page, idx);} static unsigned long bm_page_to_idx(struct page *page){	return page_private(page) & BM_PAGE_IDX_MASK;} /* As is very unlikely that the same page is under IO from more than one * context, we can get away with a bit per page and one wait queue per bitmap. */static void bm_page_lock_io(struct drbd_device *device, int page_nr){	struct drbd_bitmap *b = device->bitmap;	void *addr = &page_private(b->bm_pages[page_nr]);	wait_event(b->bm_io_wait, !test_and_set_bit(BM_PAGE_IO_LOCK, addr));} static void bm_page_unlock_io(struct drbd_device *device, int page_nr){	struct drbd_bitmap *b = device->bitmap;	void *addr = &page_private(b->bm_pages[page_nr]);	clear_bit_unlock(BM_PAGE_IO_LOCK, addr);	wake_up(&device->bitmap->bm_io_wait);} /* set _before_ submit_io, so it may be reset due to being changed * while this page is in flight... will get submitted later again */static void bm_set_page_unchanged(struct page *page){	/* use cmpxchg? */	clear_bit(BM_PAGE_NEED_WRITEOUT, &page_private(page));	clear_bit(BM_PAGE_LAZY_WRITEOUT, &page_private(page));} static void bm_set_page_need_writeout(struct page *page){	set_bit(BM_PAGE_NEED_WRITEOUT, &page_private(page));} void drbd_bm_reset_al_hints(struct drbd_device *device){	device->bitmap->n_bitmap_hints = 0;} /** * drbd_bm_mark_for_writeout() - mark a page with a "hint" to be considered for writeout * @device:	DRBD device. * @page_nr:	the bitmap page to mark with the "hint" flag * * From within an activity log transaction, we mark a few pages with these * hints, then call drbd_bm_write_hinted(), which will only write out changed * pages which are flagged with this mark. */void drbd_bm_mark_for_writeout(struct drbd_device *device, int page_nr){	struct drbd_bitmap *b = device->bitmap;	struct page *page;	if (page_nr >= device->bitmap->bm_number_of_pages) {		drbd_warn(device, "BAD: page_nr: %u, number_of_pages: %u\n",			 page_nr, (int)device->bitmap->bm_number_of_pages);		return;	}	page = device->bitmap->bm_pages[page_nr];	BUG_ON(b->n_bitmap_hints >= ARRAY_SIZE(b->al_bitmap_hints));	if (!test_and_set_bit(BM_PAGE_HINT_WRITEOUT, &page_private(page)))		b->al_bitmap_hints[b->n_bitmap_hints++] = page_nr;} static int bm_test_page_unchanged(struct page *page){	volatile const unsigned long *addr = &page_private(page);	return (*addr & ((1UL<<BM_PAGE_NEED_WRITEOUT)|(1UL<<BM_PAGE_LAZY_WRITEOUT))) == 0;} static void bm_set_page_io_err(struct page *page){	set_bit(BM_PAGE_IO_ERROR, &page_private(page));} static void bm_clear_page_io_err(struct page *page){	clear_bit(BM_PAGE_IO_ERROR, &page_private(page));} static void bm_set_page_lazy_writeout(struct page *page){	set_bit(BM_PAGE_LAZY_WRITEOUT, &page_private(page));} static int bm_test_page_lazy_writeout(struct page *page){	return test_bit(BM_PAGE_LAZY_WRITEOUT, &page_private(page));} /* on a 32bit box, this would allow for exactly (2<<38) bits. */static unsigned int bm_word_to_page_idx(struct drbd_bitmap *b, unsigned long long_nr){	/* page_nr = (word*sizeof(long)) >> PAGE_SHIFT; */	unsigned int page_nr = long_nr >> (PAGE_SHIFT - LN2_BPL + 3);	BUG_ON(page_nr >= b->bm_number_of_pages);	return page_nr;} static unsigned int bm_bit_to_page_idx(struct drbd_bitmap *b, u64 bitnr){	/* page_nr = (bitnr/8) >> PAGE_SHIFT; */	unsigned int page_nr = bitnr >> (PAGE_SHIFT + 3);	BUG_ON(page_nr >= b->bm_number_of_pages);	return page_nr;} static unsigned long *__bm_map_pidx(struct drbd_bitmap *b, unsigned int idx){	struct page *page = b->bm_pages[idx];	return (unsigned long *) kmap_atomic(page);} static unsigned long *bm_map_pidx(struct drbd_bitmap *b, unsigned int idx){	return __bm_map_pidx(b, idx);} static void __bm_unmap(unsigned long *p_addr){	kunmap_atomic(p_addr);}; static void bm_unmap(unsigned long *p_addr){	return __bm_unmap(p_addr);} /* long word offset of _bitmap_ sector */#define S2W(s)	((s)<<(BM_EXT_SHIFT-BM_BLOCK_SHIFT-LN2_BPL))/* word offset from start of bitmap to word number _in_page_ * modulo longs per page#define MLPP(X) ((X) % (PAGE_SIZE/sizeof(long)) hm, well, Philipp thinks gcc might not optimize the % into & (... - 1) so do it explicitly: */#define MLPP(X) ((X) & ((PAGE_SIZE/sizeof(long))-1)) /* Long words per page */#define LWPP (PAGE_SIZE/sizeof(long)) /* * actually most functions herein should take a struct drbd_bitmap*, not a * struct drbd_device*, but for the debug macros I like to have the device around * to be able to report device specific. */  static void bm_free_pages(struct page **pages, unsigned long number){	unsigned long i;	if (!pages)		return; 	for (i = 0; i < number; i++) {		if (!pages[i]) {			pr_alert("bm_free_pages tried to free a NULL pointer; i=%lu n=%lu\n",				 i, number);			continue;		}		__free_page(pages[i]);		pages[i] = NULL;	}} static inline void bm_vk_free(void *ptr){	kvfree(ptr);} /* * "have" and "want" are NUMBER OF PAGES. */static struct page **bm_realloc_pages(struct drbd_bitmap *b, unsigned long want){	struct page **old_pages = b->bm_pages;	struct page **new_pages, *page;	unsigned int i, bytes;	unsigned long have = b->bm_number_of_pages; 	BUG_ON(have == 0 && old_pages != NULL);	BUG_ON(have != 0 && old_pages == NULL); 	if (have == want)		return old_pages; 	/* Trying kmalloc first, falling back to vmalloc.	 * GFP_NOIO, as this is called while drbd IO is "suspended",	 * and during resize or attach on diskless Primary,	 * we must not block on IO to ourselves.	 * Context is receiver thread or dmsetup. */	bytes = sizeof(struct page *)*want;	new_pages = kzalloc(bytes, GFP_NOIO | __GFP_NOWARN);	if (!new_pages) {		new_pages = __vmalloc(bytes, GFP_NOIO | __GFP_ZERO);		if (!new_pages)			return NULL;	} 	if (want >= have) {		for (i = 0; i < have; i++)			new_pages[i] = old_pages[i];		for (; i < want; i++) {			page = alloc_page(GFP_NOIO | __GFP_HIGHMEM);			if (!page) {				bm_free_pages(new_pages + have, i - have);				bm_vk_free(new_pages);				return NULL;			}			/* we want to know which page it is			 * from the endio handlers */			bm_store_page_idx(page, i);			new_pages[i] = page;		}	} else {		for (i = 0; i < want; i++)			new_pages[i] = old_pages[i];		/* NOT HERE, we are outside the spinlock!		bm_free_pages(old_pages + want, have - want);		*/	} 	return new_pages;} /* * allocates the drbd_bitmap and stores it in device->bitmap. */int drbd_bm_init(struct drbd_device *device){	struct drbd_bitmap *b = device->bitmap;	WARN_ON(b != NULL);	b = kzalloc_obj(struct drbd_bitmap);	if (!b)		return -ENOMEM;	spin_lock_init(&b->bm_lock);	mutex_init(&b->bm_change);	init_waitqueue_head(&b->bm_io_wait); 	device->bitmap = b; 	return 0;} sector_t drbd_bm_capacity(struct drbd_device *device){	if (!expect(device, device->bitmap))		return 0;	return device->bitmap->bm_dev_capacity;} /* called on driver unload. TODO: call when a device is destroyed. */void drbd_bm_cleanup(struct drbd_device *device){	if (!expect(device, device->bitmap))		return;	bm_free_pages(device->bitmap->bm_pages, device->bitmap->bm_number_of_pages);	bm_vk_free(device->bitmap->bm_pages);	kfree(device->bitmap);	device->bitmap = NULL;} /* * since (b->bm_bits % BITS_PER_LONG) != 0, * this masks out the remaining bits. * Returns the number of bits cleared. */#ifndef BITS_PER_PAGE#define BITS_PER_PAGE		(1UL << (PAGE_SHIFT + 3))#define BITS_PER_PAGE_MASK	(BITS_PER_PAGE - 1)#else# if BITS_PER_PAGE != (1UL << (PAGE_SHIFT + 3))#  error "ambiguous BITS_PER_PAGE"# endif#endif#define BITS_PER_LONG_MASK	(BITS_PER_LONG - 1)static int bm_clear_surplus(struct drbd_bitmap *b){	unsigned long mask;	unsigned long *p_addr, *bm;	int tmp;	int cleared = 0; 	/* number of bits modulo bits per page */	tmp = (b->bm_bits & BITS_PER_PAGE_MASK);	/* mask the used bits of the word containing the last bit */	mask = (1UL << (tmp & BITS_PER_LONG_MASK)) -1;	/* bitmap is always stored little endian,	 * on disk and in core memory alike */	mask = cpu_to_lel(mask); 	p_addr = bm_map_pidx(b, b->bm_number_of_pages - 1);	bm = p_addr + (tmp/BITS_PER_LONG);	if (mask) {		/* If mask != 0, we are not exactly aligned, so bm now points

Lines 1–500 of 1697. Load more