// SPDX-License-Identifier: GPL-2.0-only/* * linux/kernel/power/snapshot.c * * This file provides system snapshot/restore functionality for swsusp. * * Copyright (C) 1998-2005 Pavel Machek <pavel@ucw.cz> * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl> */ #define pr_fmt(fmt) "PM: hibernation: " fmt #include <linux/version.h>#include <linux/module.h>#include <linux/mm.h>#include <linux/suspend.h>#include <linux/delay.h>#include <linux/bitops.h>#include <linux/spinlock.h>#include <linux/kernel.h>#include <linux/pm.h>#include <linux/device.h>#include <linux/init.h>#include <linux/memblock.h>#include <linux/nmi.h>#include <linux/syscalls.h>#include <linux/console.h>#include <linux/highmem.h>#include <linux/list.h>#include <linux/slab.h>#include <linux/compiler.h>#include <linux/ktime.h>#include <linux/set_memory.h> #include <linux/uaccess.h>#include <asm/mmu_context.h>#include <asm/tlbflush.h>#include <asm/io.h> #include "power.h" #if defined(CONFIG_STRICT_KERNEL_RWX) && defined(CONFIG_ARCH_HAS_SET_MEMORY)static bool hibernate_restore_protection;static bool hibernate_restore_protection_active; void enable_restore_image_protection(void){	hibernate_restore_protection = true;} static inline void hibernate_restore_protection_begin(void){	hibernate_restore_protection_active = hibernate_restore_protection;} static inline void hibernate_restore_protection_end(void){	hibernate_restore_protection_active = false;} static inline int __must_check hibernate_restore_protect_page(void *page_address){	if (hibernate_restore_protection_active)		return set_memory_ro((unsigned long)page_address, 1);	return 0;} static inline int hibernate_restore_unprotect_page(void *page_address){	if (hibernate_restore_protection_active)		return set_memory_rw((unsigned long)page_address, 1);	return 0;}#elsestatic inline void hibernate_restore_protection_begin(void) {}static inline void hibernate_restore_protection_end(void) {}static inline int __must_check hibernate_restore_protect_page(void *page_address) {return 0; }static inline int hibernate_restore_unprotect_page(void *page_address) {return 0; }#endif /* CONFIG_STRICT_KERNEL_RWX  && CONFIG_ARCH_HAS_SET_MEMORY */  /* * The calls to set_direct_map_*() should not fail because remapping a page * here means that we only update protection bits in an existing PTE. * It is still worth to have a warning here if something changes and this * will no longer be the case. */static inline void hibernate_map_page(struct page *page){	if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) {		int ret = set_direct_map_default_noflush(page); 		if (ret)			pr_warn_once("Failed to remap page\n");	} else {		debug_pagealloc_map_pages(page, 1);	}} static inline void hibernate_unmap_page(struct page *page){	if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) {		unsigned long addr = (unsigned long)page_address(page);		int ret  = set_direct_map_invalid_noflush(page); 		if (ret)			pr_warn_once("Failed to remap page\n"); 		flush_tlb_kernel_range(addr, addr + PAGE_SIZE);	} else {		debug_pagealloc_unmap_pages(page, 1);	}} static int swsusp_page_is_free(struct page *);static void swsusp_set_page_forbidden(struct page *);static void swsusp_unset_page_forbidden(struct page *); /* * Number of bytes to reserve for memory allocations made by device drivers * from their ->freeze() and ->freeze_noirq() callbacks so that they don't * cause image creation to fail (tunable via /sys/power/reserved_size). */unsigned long reserved_size; void __init hibernate_reserved_size_init(void){	reserved_size = SPARE_PAGES * PAGE_SIZE;} /* * Preferred image size in bytes (tunable via /sys/power/image_size). * When it is set to N, swsusp will do its best to ensure the image * size will not exceed N bytes, but if that is impossible, it will * try to create the smallest image possible. */unsigned long image_size; void __init hibernate_image_size_init(void){	image_size = ((totalram_pages() * 2) / 5) * PAGE_SIZE;} /* * List of PBEs needed for restoring the pages that were allocated before * the suspend and included in the suspend image, but have also been * allocated by the "resume" kernel, so their contents cannot be written * directly to their "original" page frames. */struct pbe *restore_pblist; /* struct linked_page is used to build chains of pages */ #define LINKED_PAGE_DATA_SIZE	(PAGE_SIZE - sizeof(void *)) struct linked_page {	struct linked_page *next;	char data[LINKED_PAGE_DATA_SIZE];} __packed; /* * List of "safe" pages (ie. pages that were not used by the image kernel * before hibernation) that may be used as temporary storage for image kernel * memory contents. */static struct linked_page *safe_pages_list; /* Pointer to an auxiliary buffer (1 page) */static void *buffer; #define PG_ANY		0#define PG_SAFE		1#define PG_UNSAFE_CLEAR	1#define PG_UNSAFE_KEEP	0 static unsigned int allocated_unsafe_pages; /** * get_image_page - Allocate a page for a hibernation image. * @gfp_mask: GFP mask for the allocation. * @safe_needed: Get pages that were not used before hibernation (restore only) * * During image restoration, for storing the PBE list and the image data, we can * only use memory pages that do not conflict with the pages used before * hibernation.  The "unsafe" pages have PageNosaveFree set and we count them * using allocated_unsafe_pages. * * Each allocated image page is marked as PageNosave and PageNosaveFree so that * swsusp_free() can release it. */static void *get_image_page(gfp_t gfp_mask, int safe_needed){	void *res; 	res = (void *)get_zeroed_page(gfp_mask);	if (safe_needed)		while (res && swsusp_page_is_free(virt_to_page(res))) {			/* The page is unsafe, mark it for swsusp_free() */			swsusp_set_page_forbidden(virt_to_page(res));			allocated_unsafe_pages++;			res = (void *)get_zeroed_page(gfp_mask);		}	if (res) {		swsusp_set_page_forbidden(virt_to_page(res));		swsusp_set_page_free(virt_to_page(res));	}	return res;} static void *__get_safe_page(gfp_t gfp_mask){	if (safe_pages_list) {		void *ret = safe_pages_list; 		safe_pages_list = safe_pages_list->next;		memset(ret, 0, PAGE_SIZE);		return ret;	}	return get_image_page(gfp_mask, PG_SAFE);} unsigned long get_safe_page(gfp_t gfp_mask){	return (unsigned long)__get_safe_page(gfp_mask);} static struct page *alloc_image_page(gfp_t gfp_mask){	struct page *page; 	page = alloc_page(gfp_mask);	if (page) {		swsusp_set_page_forbidden(page);		swsusp_set_page_free(page);	}	return page;} static void recycle_safe_page(void *page_address){	struct linked_page *lp = page_address; 	lp->next = safe_pages_list;	safe_pages_list = lp;} /** * free_image_page - Free a page allocated for hibernation image. * @addr: Address of the page to free. * @clear_nosave_free: If set, clear the PageNosaveFree bit for the page. * * The page to free should have been allocated by get_image_page() (page flags * set by it are affected). */static inline void free_image_page(void *addr, int clear_nosave_free){	struct page *page; 	BUG_ON(!virt_addr_valid(addr)); 	page = virt_to_page(addr); 	swsusp_unset_page_forbidden(page);	if (clear_nosave_free)		swsusp_unset_page_free(page); 	__free_page(page);} static inline void free_list_of_pages(struct linked_page *list,				      int clear_page_nosave){	while (list) {		struct linked_page *lp = list->next; 		free_image_page(list, clear_page_nosave);		list = lp;	}} /* * struct chain_allocator is used for allocating small objects out of * a linked list of pages called 'the chain'. * * The chain grows each time when there is no room for a new object in * the current page.  The allocated objects cannot be freed individually. * It is only possible to free them all at once, by freeing the entire * chain. * * NOTE: The chain allocator may be inefficient if the allocated objects * are not much smaller than PAGE_SIZE. */struct chain_allocator {	struct linked_page *chain;	/* the chain */	unsigned int used_space;	/* total size of objects allocated out					   of the current page */	gfp_t gfp_mask;		/* mask for allocating pages */	int safe_needed;	/* if set, only "safe" pages are allocated */}; static void chain_init(struct chain_allocator *ca, gfp_t gfp_mask,		       int safe_needed){	ca->chain = NULL;	ca->used_space = LINKED_PAGE_DATA_SIZE;	ca->gfp_mask = gfp_mask;	ca->safe_needed = safe_needed;} static void *chain_alloc(struct chain_allocator *ca, unsigned int size){	void *ret; 	if (LINKED_PAGE_DATA_SIZE - ca->used_space < size) {		struct linked_page *lp; 		lp = ca->safe_needed ? __get_safe_page(ca->gfp_mask) :					get_image_page(ca->gfp_mask, PG_ANY);		if (!lp)			return NULL; 		lp->next = ca->chain;		ca->chain = lp;		ca->used_space = 0;	}	ret = ca->chain->data + ca->used_space;	ca->used_space += size;	return ret;} /* * Data types related to memory bitmaps. * * Memory bitmap is a structure consisting of many linked lists of * objects.  The main list's elements are of type struct zone_bitmap * and each of them corresponds to one zone.  For each zone bitmap * object there is a list of objects of type struct bm_block that * represent each blocks of bitmap in which information is stored. * * struct memory_bitmap contains a pointer to the main list of zone * bitmap objects, a struct bm_position used for browsing the bitmap, * and a pointer to the list of pages used for allocating all of the * zone bitmap objects and bitmap block objects. * * NOTE: It has to be possible to lay out the bitmap in memory * using only allocations of order 0.  Additionally, the bitmap is * designed to work with arbitrary number of zones (this is over the * top for now, but let's avoid making unnecessary assumptions ;-). * * struct zone_bitmap contains a pointer to a list of bitmap block * objects and a pointer to the bitmap block object that has been * most recently used for setting bits.  Additionally, it contains the * PFNs that correspond to the start and end of the represented zone. * * struct bm_block contains a pointer to the memory page in which * information is stored (in the form of a block of bitmap) * It also contains the pfns that correspond to the start and end of * the represented memory area. * * The memory bitmap is organized as a radix tree to guarantee fast random * access to the bits. There is one radix tree for each zone (as returned * from create_mem_extents). * * One radix tree is represented by one struct mem_zone_bm_rtree. There are * two linked lists for the nodes of the tree, one for the inner nodes and * one for the leaf nodes. The linked leaf nodes are used for fast linear * access of the memory bitmap. * * The struct rtree_node represents one node of the radix tree. */ #define BM_END_OF_MAP	(~0UL) #define BM_BITS_PER_BLOCK	(PAGE_SIZE * BITS_PER_BYTE)#define BM_BLOCK_SHIFT		(PAGE_SHIFT + 3)#define BM_BLOCK_MASK		((1UL << BM_BLOCK_SHIFT) - 1) /* * struct rtree_node is a wrapper struct to link the nodes * of the rtree together for easy linear iteration over * bits and easy freeing */struct rtree_node {	struct list_head list;	unsigned long *data;}; /* * struct mem_zone_bm_rtree represents a bitmap used for one * populated memory zone. */struct mem_zone_bm_rtree {	struct list_head list;		/* Link Zones together         */	struct list_head nodes;		/* Radix Tree inner nodes      */	struct list_head leaves;	/* Radix Tree leaves           */	unsigned long start_pfn;	/* Zone start page frame       */	unsigned long end_pfn;		/* Zone end page frame + 1     */	struct rtree_node *rtree;	/* Radix Tree Root             */	int levels;			/* Number of Radix Tree Levels */	unsigned int blocks;		/* Number of Bitmap Blocks     */}; /* struct bm_position is used for browsing memory bitmaps */ struct bm_position {	struct mem_zone_bm_rtree *zone;	struct rtree_node *node;	unsigned long node_pfn;	unsigned long cur_pfn;	int node_bit;}; struct memory_bitmap {	struct list_head zones;	struct linked_page *p_list;	/* list of pages used to store zone					   bitmap objects and bitmap block					   objects */	struct bm_position cur;	/* most recently used bit position */}; /* Functions that operate on memory bitmaps */ #define BM_ENTRIES_PER_LEVEL	(PAGE_SIZE / sizeof(unsigned long))#if BITS_PER_LONG == 32#define BM_RTREE_LEVEL_SHIFT	(PAGE_SHIFT - 2)#else#define BM_RTREE_LEVEL_SHIFT	(PAGE_SHIFT - 3)#endif#define BM_RTREE_LEVEL_MASK	((1UL << BM_RTREE_LEVEL_SHIFT) - 1) /** * alloc_rtree_node - Allocate a new node and add it to the radix tree. * @gfp_mask: GFP mask for the allocation. * @safe_needed: Get pages not used before hibernation (restore only) * @ca: Pointer to a linked list of pages ("a chain") to allocate from * @list: Radix Tree node to add. * * This function is used to allocate inner nodes as well as the * leave nodes of the radix tree. It also adds the node to the * corresponding linked list passed in by the *list parameter. */static struct rtree_node *alloc_rtree_node(gfp_t gfp_mask, int safe_needed,					   struct chain_allocator *ca,					   struct list_head *list){	struct rtree_node *node; 	node = chain_alloc(ca, sizeof(struct rtree_node));	if (!node)		return NULL; 	node->data = get_image_page(gfp_mask, safe_needed);	if (!node->data)		return NULL; 	list_add_tail(&node->list, list); 	return node;} /* * add_rtree_block - Add a new leave node to the radix tree. * * The leave nodes need to be allocated in order to keep the leaves * linked list in order. This is guaranteed by the zone->blocks * counter. */static int add_rtree_block(struct mem_zone_bm_rtree *zone, gfp_t gfp_mask,			   int safe_needed, struct chain_allocator *ca){	struct rtree_node *node, *block, **dst;	unsigned int levels_needed, block_nr;	int i; 	block_nr = zone->blocks;	levels_needed = 0; 	/* How many levels do we need for this block nr? */	while (block_nr) {		levels_needed += 1;		block_nr >>= BM_RTREE_LEVEL_SHIFT;	} 	/* Make sure the rtree has enough levels */	for (i = zone->levels; i < levels_needed; i++) {		node = alloc_rtree_node(gfp_mask, safe_needed, ca,					&zone->nodes);		if (!node)			return -ENOMEM; 		node->data[0] = (unsigned long)zone->rtree;		zone->rtree = node;		zone->levels += 1;	} 	/* Allocate new block */	block = alloc_rtree_node(gfp_mask, safe_needed, ca, &zone->leaves);	if (!block)		return -ENOMEM; 

Lines 1–500 of 2931. Load more