// SPDX-License-Identifier: GPL-2.0/* * sparse memory mappings. */#include <linux/mm.h>#include <linux/slab.h>#include <linux/mmzone.h>#include <linux/memblock.h>#include <linux/compiler.h>#include <linux/highmem.h>#include <linux/export.h>#include <linux/spinlock.h>#include <linux/vmalloc.h>#include <linux/swap.h>#include <linux/swapops.h>#include <linux/bootmem_info.h>#include <linux/vmstat.h>#include "internal.h"#include <asm/dma.h> /* * Permanent SPARSEMEM data: * * 1) mem_section	- memory sections, mem_map's for valid memory */#ifdef CONFIG_SPARSEMEM_EXTREMEstruct mem_section **mem_section;#elsestruct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]	____cacheline_internodealigned_in_smp;#endifEXPORT_SYMBOL(mem_section); #ifdef NODE_NOT_IN_PAGE_FLAGS/* * If we did not store the node number in the page then we have to * do a lookup in the section_to_node_table in order to find which * node the page belongs to. */#if MAX_NUMNODES <= 256static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;#elsestatic u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;#endif int memdesc_nid(memdesc_flags_t mdf){	return section_to_node_table[memdesc_section(mdf)];}EXPORT_SYMBOL(memdesc_nid); static void set_section_nid(unsigned long section_nr, int nid){	section_to_node_table[section_nr] = nid;}#else /* !NODE_NOT_IN_PAGE_FLAGS */static inline void set_section_nid(unsigned long section_nr, int nid){}#endif #ifdef CONFIG_SPARSEMEM_EXTREMEstatic noinline struct mem_section __ref *sparse_index_alloc(int nid){	struct mem_section *section = NULL;	unsigned long array_size = SECTIONS_PER_ROOT *				   sizeof(struct mem_section); 	if (slab_is_available()) {		section = kzalloc_node(array_size, GFP_KERNEL, nid);	} else {		section = memblock_alloc_node(array_size, SMP_CACHE_BYTES,					      nid);		if (!section)			panic("%s: Failed to allocate %lu bytes nid=%d\n",			      __func__, array_size, nid);	} 	return section;} int __meminit sparse_index_init(unsigned long section_nr, int nid){	unsigned long root = SECTION_NR_TO_ROOT(section_nr);	struct mem_section *section; 	/*	 * An existing section is possible in the sub-section hotplug	 * case. First hot-add instantiates, follow-on hot-add reuses	 * the existing section.	 *	 * The mem_hotplug_lock resolves the apparent race below.	 */	if (mem_section[root])		return 0; 	section = sparse_index_alloc(nid);	if (!section)		return -ENOMEM; 	mem_section[root] = section; 	return 0;}#else /* !SPARSEMEM_EXTREME */int sparse_index_init(unsigned long section_nr, int nid){	return 0;}#endif /* * During early boot, before section_mem_map is used for an actual * mem_map, we use section_mem_map to store the section's NUMA * node.  This keeps us from having to use another data structure.  The * node information is cleared just before we store the real mem_map. */static inline unsigned long sparse_encode_early_nid(int nid){	return ((unsigned long)nid << SECTION_NID_SHIFT);} static inline int sparse_early_nid(struct mem_section *section){	return (section->section_mem_map >> SECTION_NID_SHIFT);} /* Validate the physical addressing limitations of the model */static void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn,						unsigned long *end_pfn){	unsigned long max_sparsemem_pfn = (DIRECT_MAP_PHYSMEM_END + 1) >> PAGE_SHIFT; 	/*	 * Sanity checks - do not allow an architecture to pass	 * in larger pfns than the maximum scope of sparsemem:	 */	if (*start_pfn > max_sparsemem_pfn) {		mminit_dprintk(MMINIT_WARNING, "pfnvalidation",			"Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",			*start_pfn, *end_pfn, max_sparsemem_pfn);		WARN_ON_ONCE(1);		*start_pfn = max_sparsemem_pfn;		*end_pfn = max_sparsemem_pfn;	} else if (*end_pfn > max_sparsemem_pfn) {		mminit_dprintk(MMINIT_WARNING, "pfnvalidation",			"End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",			*start_pfn, *end_pfn, max_sparsemem_pfn);		WARN_ON_ONCE(1);		*end_pfn = max_sparsemem_pfn;	}} /* * There are a number of times that we loop over NR_MEM_SECTIONS, * looking for section_present() on each.  But, when we have very * large physical address spaces, NR_MEM_SECTIONS can also be * very large which makes the loops quite long. * * Keeping track of this gives us an easy way to break out of * those loops early. */unsigned long __highest_present_section_nr; static inline unsigned long first_present_section_nr(void){	return next_present_section_nr(-1);} /* Record a memory area against a node. */static void __init memory_present(int nid, unsigned long start, unsigned long end){	unsigned long pfn; 	start &= PAGE_SECTION_MASK;	mminit_validate_memmodel_limits(&start, &end);	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) {		unsigned long section_nr = pfn_to_section_nr(pfn);		struct mem_section *ms; 		sparse_index_init(section_nr, nid);		set_section_nid(section_nr, nid); 		ms = __nr_to_section(section_nr);		if (!ms->section_mem_map) {			ms->section_mem_map = sparse_encode_early_nid(nid) |							SECTION_IS_ONLINE;			__section_mark_present(ms, section_nr);		}	}} /* * Mark all memblocks as present using memory_present(). * This is a convenience function that is useful to mark all of the systems * memory as present during initialization. */static void __init memblocks_present(void){	unsigned long start, end;	int i, nid; #ifdef CONFIG_SPARSEMEM_EXTREME	unsigned long size, align; 	size = sizeof(struct mem_section *) * NR_SECTION_ROOTS;	align = 1 << (INTERNODE_CACHE_SHIFT);	mem_section = memblock_alloc_or_panic(size, align);#endif 	for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid)		memory_present(nid, start, end);} static unsigned long usemap_size(void){	return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS) * sizeof(unsigned long);} size_t mem_section_usage_size(void){	return sizeof(struct mem_section_usage) + usemap_size();} #ifdef CONFIG_SPARSEMEM_VMEMMAPunsigned long __init section_map_size(void){	return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE);} #elseunsigned long __init section_map_size(void){	return PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION);} struct page __init *__populate_section_memmap(unsigned long pfn,		unsigned long nr_pages, int nid, struct vmem_altmap *altmap,		struct dev_pagemap *pgmap){	unsigned long size = section_map_size();	struct page *map;	phys_addr_t addr = __pa(MAX_DMA_ADDRESS); 	map = memmap_alloc(size, size, addr, nid, false);	if (!map)		panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n",		      __func__, size, PAGE_SIZE, nid, &addr); 	return map;}#endif /* !CONFIG_SPARSEMEM_VMEMMAP */ void __weak __meminit vmemmap_populate_print_last(void){} static void *sparse_usagebuf __meminitdata;static void *sparse_usagebuf_end __meminitdata; /* * Helper function that is used for generic section initialization, and * can also be used by any hooks added above. */void __init sparse_init_early_section(int nid, struct page *map,				      unsigned long pnum, unsigned long flags){	BUG_ON(!sparse_usagebuf || sparse_usagebuf >= sparse_usagebuf_end);	sparse_init_one_section(__nr_to_section(pnum), pnum, map,			sparse_usagebuf, SECTION_IS_EARLY | flags);	sparse_usagebuf = (void *)sparse_usagebuf + mem_section_usage_size();} static int __init sparse_usage_init(int nid, unsigned long map_count){	unsigned long size; 	size = mem_section_usage_size() * map_count;	sparse_usagebuf = memblock_alloc_node(size, SMP_CACHE_BYTES, nid);	if (!sparse_usagebuf) {		sparse_usagebuf_end = NULL;		return -ENOMEM;	} 	sparse_usagebuf_end = sparse_usagebuf + size;	return 0;} static void __init sparse_usage_fini(void){	sparse_usagebuf = sparse_usagebuf_end = NULL;} /* * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end) * And number of present sections in this node is map_count. */static void __init sparse_init_nid(int nid, unsigned long pnum_begin,				   unsigned long pnum_end,				   unsigned long map_count){	unsigned long pnum;	struct page *map;	struct mem_section *ms; 	if (sparse_usage_init(nid, map_count)) {		pr_err("%s: node[%d] usemap allocation failed", __func__, nid);		goto failed;	} 	sparse_vmemmap_init_nid_early(nid); 	for_each_present_section_nr(pnum_begin, pnum) {		unsigned long pfn = section_nr_to_pfn(pnum); 		if (pnum >= pnum_end)			break; 		ms = __nr_to_section(pnum);		if (!preinited_vmemmap_section(ms)) {			map = __populate_section_memmap(pfn, PAGES_PER_SECTION,					nid, NULL, NULL);			if (!map) {				pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.",				       __func__, nid);				pnum_begin = pnum;				sparse_usage_fini();				goto failed;			}			memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page),							   PAGE_SIZE));			sparse_init_early_section(nid, map, pnum, 0);		}	}	sparse_usage_fini();	return;failed:	/*	 * We failed to allocate, mark all the following pnums as not present,	 * except the ones already initialized earlier.	 */	for_each_present_section_nr(pnum_begin, pnum) {		if (pnum >= pnum_end)			break;		ms = __nr_to_section(pnum);		if (!preinited_vmemmap_section(ms))			ms->section_mem_map = 0;	}} /* * Allocate the accumulated non-linear sections, allocate a mem_map * for each and record the physical to section mapping. */void __init sparse_init(void){	unsigned long pnum_end, pnum_begin, map_count = 1;	int nid_begin; 	/* see include/linux/mmzone.h 'struct mem_section' definition */	BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section)));	memblocks_present(); 	if (compound_info_has_mask()) {		VM_WARN_ON_ONCE(!IS_ALIGNED((unsigned long) pfn_to_page(0),				    MAX_FOLIO_VMEMMAP_ALIGN));	} 	pnum_begin = first_present_section_nr();	nid_begin = sparse_early_nid(__nr_to_section(pnum_begin)); 	/* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */	set_pageblock_order(); 	for_each_present_section_nr(pnum_begin + 1, pnum_end) {		int nid = sparse_early_nid(__nr_to_section(pnum_end)); 		if (nid == nid_begin) {			map_count++;			continue;		}		/* Init node with sections in range [pnum_begin, pnum_end) */		sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count);		nid_begin = nid;		pnum_begin = pnum_end;		map_count = 1;	}	/* cover the last node */	sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count);	vmemmap_populate_print_last();}