// SPDX-License-Identifier: GPL-2.0-only/* * mm_init.c - Memory initialisation verification and debugging * * Copyright 2008 IBM Corporation, 2008 * Author Mel Gorman <mel@csn.ul.ie> * */#include <linux/kernel.h>#include <linux/init.h>#include <linux/kobject.h>#include <linux/export.h>#include <linux/memory.h>#include <linux/notifier.h>#include <linux/sched.h>#include <linux/mman.h>#include <linux/memblock.h>#include <linux/page-isolation.h>#include <linux/padata.h>#include <linux/nmi.h>#include <linux/buffer_head.h>#include <linux/kmemleak.h>#include <linux/kfence.h>#include <linux/page_ext.h>#include <linux/pti.h>#include <linux/pgtable.h>#include <linux/stackdepot.h>#include <linux/swap.h>#include <linux/cma.h>#include <linux/crash_dump.h>#include <linux/execmem.h>#include <linux/vmstat.h>#include <linux/kexec_handover.h>#include <linux/hugetlb.h>#include "internal.h"#include "mm_init.h"#include "page_alloc.h"#include "sparse.h"#include "slab.h"#include "shuffle.h"#include "vmalloc.h" #include <asm/setup.h> #ifndef CONFIG_NUMAunsigned long max_mapnr;EXPORT_SYMBOL(max_mapnr); struct page *mem_map;EXPORT_SYMBOL(mem_map);#endif /* * high_memory defines the upper bound on direct map memory, then end * of ZONE_NORMAL. */void *high_memory;EXPORT_SYMBOL(high_memory); unsigned long zero_page_pfn __ro_after_init;EXPORT_SYMBOL(zero_page_pfn); #ifndef __HAVE_COLOR_ZERO_PAGEconst uint8_t empty_zero_page[PAGE_SIZE] __aligned(PAGE_SIZE);EXPORT_SYMBOL(empty_zero_page); struct page *__zero_page __ro_after_init;EXPORT_SYMBOL(__zero_page);#endif /* __HAVE_COLOR_ZERO_PAGE */ #ifdef CONFIG_DEBUG_MEMORY_INITint __meminitdata mminit_loglevel; /* The zonelists are simply reported, validation is manual. */void __init mminit_verify_zonelist(void){	int nid; 	if (mminit_loglevel < MMINIT_VERIFY)		return; 	for_each_online_node(nid) {		pg_data_t *pgdat = NODE_DATA(nid);		struct zone *zone;		struct zoneref *z;		struct zonelist *zonelist;		int i, listid, zoneid; 		for (i = 0; i < MAX_ZONELISTS * MAX_NR_ZONES; i++) { 			/* Identify the zone and nodelist */			zoneid = i % MAX_NR_ZONES;			listid = i / MAX_NR_ZONES;			zonelist = &pgdat->node_zonelists[listid];			zone = &pgdat->node_zones[zoneid];			if (!populated_zone(zone))				continue; 			/* Print information about the zonelist */			printk(KERN_DEBUG "mminit::zonelist %s %d:%s = ",				listid > 0 ? "thisnode" : "general", nid,				zone->name); 			/* Iterate the zonelist */			for_each_zone_zonelist(zone, z, zonelist, zoneid)				pr_cont("%d:%s ", zone_to_nid(zone), zone->name);			pr_cont("\n");		}	}} void __init mminit_verify_pageflags_layout(void){	int shift, width;	unsigned long or_mask, add_mask; 	shift = BITS_PER_LONG;	width = shift - NR_NON_PAGEFLAG_BITS;	mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths",		"Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n",		SECTIONS_WIDTH,		NODES_WIDTH,		ZONES_WIDTH,		LAST_CPUPID_WIDTH,		KASAN_TAG_WIDTH,		LRU_GEN_WIDTH,		LRU_REFS_WIDTH,		NR_PAGEFLAGS);	mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts",		"Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n",		SECTIONS_SHIFT,		NODES_SHIFT,		ZONES_SHIFT,		LAST_CPUPID_SHIFT,		KASAN_TAG_WIDTH);	mminit_dprintk(MMINIT_TRACE, "pageflags_layout_pgshifts",		"Section %lu Node %lu Zone %lu Lastcpupid %lu Kasantag %lu\n",		(unsigned long)SECTIONS_PGSHIFT,		(unsigned long)NODES_PGSHIFT,		(unsigned long)ZONES_PGSHIFT,		(unsigned long)LAST_CPUPID_PGSHIFT,		(unsigned long)KASAN_TAG_PGSHIFT);	mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodezoneid",		"Node/Zone ID: %lu -> %lu\n",		(unsigned long)(ZONEID_PGOFF + ZONEID_SHIFT),		(unsigned long)ZONEID_PGOFF);	mminit_dprintk(MMINIT_TRACE, "pageflags_layout_usage",		"location: %d -> %d layout %d -> %d unused %d -> %d page-flags\n",		shift, width, width, NR_PAGEFLAGS, NR_PAGEFLAGS, 0);#ifdef NODE_NOT_IN_PAGE_FLAGS	mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags",		"Node not in page flags");#endif#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS	mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags",		"Last cpupid not in page flags");#endif 	if (SECTIONS_WIDTH) {		shift -= SECTIONS_WIDTH;		BUG_ON(shift != SECTIONS_PGSHIFT);	}	if (NODES_WIDTH) {		shift -= NODES_WIDTH;		BUG_ON(shift != NODES_PGSHIFT);	}	if (ZONES_WIDTH) {		shift -= ZONES_WIDTH;		BUG_ON(shift != ZONES_PGSHIFT);	} 	/* Check for bitmask overlaps */	or_mask = (ZONES_MASK << ZONES_PGSHIFT) |			(NODES_MASK << NODES_PGSHIFT) |			(SECTIONS_MASK << SECTIONS_PGSHIFT);	add_mask = (ZONES_MASK << ZONES_PGSHIFT) +			(NODES_MASK << NODES_PGSHIFT) +			(SECTIONS_MASK << SECTIONS_PGSHIFT);	BUG_ON(or_mask != add_mask);} static __init int set_mminit_loglevel(char *str){	get_option(&str, &mminit_loglevel);	return 0;}early_param("mminit_loglevel", set_mminit_loglevel);#endif /* CONFIG_DEBUG_MEMORY_INIT */ struct kobject *mm_kobj; #ifdef CONFIG_SMPs32 vm_committed_as_batch = 32; void mm_compute_batch(int overcommit_policy){	u64 memsized_batch;	s32 nr = num_present_cpus();	s32 batch = max_t(s32, nr*2, 32);	unsigned long ram_pages = totalram_pages(); 	/*	 * For policy OVERCOMMIT_NEVER, set batch size to 0.4% of	 * (total memory/#cpus), and lift it to 25% for other policies	 * to ease the possible lock contention for percpu_counter	 * vm_committed_as, while the max limit is INT_MAX	 */	if (overcommit_policy == OVERCOMMIT_NEVER)		memsized_batch = min_t(u64, ram_pages/nr/256, INT_MAX);	else		memsized_batch = min_t(u64, ram_pages/nr/4, INT_MAX); 	vm_committed_as_batch = max_t(s32, memsized_batch, batch);} static int __meminit mm_compute_batch_notifier(struct notifier_block *self,					unsigned long action, void *arg){	switch (action) {	case MEM_ONLINE:	case MEM_OFFLINE:		mm_compute_batch(sysctl_overcommit_memory);		break;	default:		break;	}	return NOTIFY_OK;} static int __init mm_compute_batch_init(void){	mm_compute_batch(sysctl_overcommit_memory);	hotplug_memory_notifier(mm_compute_batch_notifier, MM_COMPUTE_BATCH_PRI);	return 0;} __initcall(mm_compute_batch_init); #endif static int __init mm_sysfs_init(void){	mm_kobj = kobject_create_and_add("mm", kernel_kobj);	if (!mm_kobj)		return -ENOMEM; 	return 0;}postcore_initcall(mm_sysfs_init); static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata;static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata;static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; static unsigned long required_kernelcore __initdata;static unsigned long required_kernelcore_percent __initdata;static unsigned long required_movablecore __initdata;static unsigned long required_movablecore_percent __initdata; static unsigned long nr_kernel_pages __initdata;static unsigned long nr_all_pages __initdata; static bool deferred_struct_pages __meminitdata; static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); static int __init cmdline_parse_core(char *p, unsigned long *core,				     unsigned long *percent){	unsigned long long coremem;	char *endptr; 	if (!p)		return -EINVAL; 	/* Value may be a percentage of total memory, otherwise bytes */	coremem = simple_strtoull(p, &endptr, 0);	if (*endptr == '%') {		/* Paranoid check for percent values greater than 100 */		WARN_ON(coremem > 100); 		*percent = coremem;	} else {		coremem = memparse(p, &p);		/* Paranoid check that UL is enough for the coremem value */		WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 		*core = coremem >> PAGE_SHIFT;		*percent = 0UL;	}	return 0;} bool mirrored_kernelcore __initdata_memblock; /* * kernelcore=size sets the amount of memory for use for allocations that * cannot be reclaimed or migrated. */static int __init cmdline_parse_kernelcore(char *p){	/* parse kernelcore=mirror */	if (parse_option_str(p, "mirror")) {		mirrored_kernelcore = true;		return 0;	} 	return cmdline_parse_core(p, &required_kernelcore,				  &required_kernelcore_percent);}early_param("kernelcore", cmdline_parse_kernelcore); /* * movablecore=size sets the amount of memory for use for allocations that * can be reclaimed or migrated. */static int __init cmdline_parse_movablecore(char *p){	return cmdline_parse_core(p, &required_movablecore,				  &required_movablecore_percent);}early_param("movablecore", cmdline_parse_movablecore); /* * early_calculate_totalpages() * Sum pages in active regions for movable zone. * Populate N_MEMORY for calculating usable_nodes. */static unsigned long __init early_calculate_totalpages(void){	unsigned long totalpages = 0;	unsigned long start_pfn, end_pfn;	int i, nid; 	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {		unsigned long pages = end_pfn - start_pfn; 		totalpages += pages;		node_set_state(nid, N_MEMORY);	}	return totalpages;} /* * This finds a zone that can be used for ZONE_MOVABLE pages. The * assumption is made that zones within a node are ordered in monotonic * increasing memory addresses so that the "highest" populated zone is used */static void __init find_usable_zone_for_movable(void){	int zone_index;	for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {		if (zone_index == ZONE_MOVABLE)			continue; 		if (arch_zone_highest_possible_pfn[zone_index] >				arch_zone_lowest_possible_pfn[zone_index])			break;	} 	VM_BUG_ON(zone_index == -1);	movable_zone = zone_index;} /* * Find the PFN the Movable zone begins in each node. Kernel memory * is spread evenly between nodes as long as the nodes have enough * memory. When they don't, some nodes will have more kernelcore than * others */static void __init find_zone_movable_pfns_for_nodes(void){	int i, nid;	unsigned long usable_startpfn;	unsigned long kernelcore_node, kernelcore_remaining;	/* save the state before borrow the nodemask */	nodemask_t saved_node_state = node_states[N_MEMORY];	unsigned long totalpages = early_calculate_totalpages();	int usable_nodes = nodes_weight(node_states[N_MEMORY]);	struct memblock_region *r; 	/* Need to find movable_zone earlier when movable_node is specified. */	find_usable_zone_for_movable(); 	/*	 * If movable_node is specified, ignore kernelcore and movablecore	 * options.	 */	if (movable_node_is_enabled()) {		for_each_mem_region(r) {			if (!memblock_is_hotpluggable(r))				continue; 			nid = memblock_get_region_node(r); 			usable_startpfn = memblock_region_memory_base_pfn(r);			zone_movable_pfn[nid] = zone_movable_pfn[nid] ?				min(usable_startpfn, zone_movable_pfn[nid]) :				usable_startpfn;		} 		goto out2;	} 	/*	 * If kernelcore=mirror is specified, ignore movablecore option	 */	if (mirrored_kernelcore) {		bool mem_below_4gb_not_mirrored = false; 		if (!memblock_has_mirror()) {			pr_warn("The system has no mirror memory, ignore kernelcore=mirror.\n");			goto out;		} 		if (is_kdump_kernel()) {			pr_warn("The system is under kdump, ignore kernelcore=mirror.\n");			goto out;		} 		for_each_mem_region(r) {			if (memblock_is_mirror(r))				continue; 			nid = memblock_get_region_node(r); 			usable_startpfn = memblock_region_memory_base_pfn(r); 			if (usable_startpfn < PHYS_PFN(SZ_4G)) {				mem_below_4gb_not_mirrored = true;				continue;			} 			zone_movable_pfn[nid] = zone_movable_pfn[nid] ?				min(usable_startpfn, zone_movable_pfn[nid]) :				usable_startpfn;		} 		if (mem_below_4gb_not_mirrored)			pr_warn("This configuration results in unmirrored kernel memory.\n"); 		goto out2;	} 	/*	 * If kernelcore=nn% or movablecore=nn% was specified, calculate the	 * amount of necessary memory.	 */	if (required_kernelcore_percent)		required_kernelcore = (totalpages * 100 * required_kernelcore_percent) /				       10000UL;	if (required_movablecore_percent)		required_movablecore = (totalpages * 100 * required_movablecore_percent) /					10000UL; 	/*	 * If movablecore= was specified, calculate what size of	 * kernelcore that corresponds so that memory usable for	 * any allocation type is evenly spread. If both kernelcore	 * and movablecore are specified, then the value of kernelcore	 * will be used for required_kernelcore if it's greater than	 * what movablecore would have allowed.	 */	if (required_movablecore) {		unsigned long corepages; 		/*		 * Round-up so that ZONE_MOVABLE is at least as large as what		 * was requested by the user		 */		required_movablecore =			round_up(required_movablecore, MAX_ORDER_NR_PAGES);		required_movablecore = min(totalpages, required_movablecore);		corepages = totalpages - required_movablecore; 		required_kernelcore = max(required_kernelcore, corepages);	} 	/*	 * If kernelcore was not specified or kernelcore size is larger	 * than totalpages, there is no ZONE_MOVABLE.	 */	if (!required_kernelcore || required_kernelcore >= totalpages)		goto out; 	/* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */	usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; restart:	/* Spread kernelcore memory as evenly as possible throughout nodes */	kernelcore_node = required_kernelcore / usable_nodes;	for_each_node_state(nid, N_MEMORY) {		unsigned long start_pfn, end_pfn; 		/*		 * Recalculate kernelcore_node if the division per node		 * now exceeds what is necessary to satisfy the requested		 * amount of memory for the kernel		 */		if (required_kernelcore < kernelcore_node)

Lines 1–500 of 2693. Load more