// SPDX-License-Identifier: GPL-2.0/* * sparse memory mappings. */#include <linux/mm.h>#include <linux/slab.h>#include <linux/mmzone.h>#include <linux/memblock.h>#include <linux/compiler.h>#include <linux/highmem.h>#include <linux/export.h>#include <linux/spinlock.h>#include <linux/vmalloc.h>#include <linux/swap.h>#include <linux/swapops.h> #include "internal.h"#include <asm/dma.h> /* * Permanent SPARSEMEM data: * * 1) mem_section	- memory sections, mem_map's for valid memory */#ifdef CONFIG_SPARSEMEM_EXTREMEstruct mem_section **mem_section;#elsestruct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]	____cacheline_internodealigned_in_smp;#endifEXPORT_SYMBOL(mem_section); #ifdef NODE_NOT_IN_PAGE_FLAGS/* * If we did not store the node number in the page then we have to * do a lookup in the section_to_node_table in order to find which * node the page belongs to. */#if MAX_NUMNODES <= 256static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;#elsestatic u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;#endif int page_to_nid(const struct page *page){	return section_to_node_table[page_to_section(page)];}EXPORT_SYMBOL(page_to_nid); static void set_section_nid(unsigned long section_nr, int nid){	section_to_node_table[section_nr] = nid;}#else /* !NODE_NOT_IN_PAGE_FLAGS */static inline void set_section_nid(unsigned long section_nr, int nid){}#endif #ifdef CONFIG_SPARSEMEM_EXTREMEstatic noinline struct mem_section __ref *sparse_index_alloc(int nid){	struct mem_section *section = NULL;	unsigned long array_size = SECTIONS_PER_ROOT *				   sizeof(struct mem_section); 	if (slab_is_available()) {		section = kzalloc_node(array_size, GFP_KERNEL, nid);	} else {		section = memblock_alloc_node(array_size, SMP_CACHE_BYTES,					      nid);		if (!section)			panic("%s: Failed to allocate %lu bytes nid=%d\n",			      __func__, array_size, nid);	} 	return section;} static int __meminit sparse_index_init(unsigned long section_nr, int nid){	unsigned long root = SECTION_NR_TO_ROOT(section_nr);	struct mem_section *section; 	/*	 * An existing section is possible in the sub-section hotplug	 * case. First hot-add instantiates, follow-on hot-add reuses	 * the existing section.	 *	 * The mem_hotplug_lock resolves the apparent race below.	 */	if (mem_section[root])		return 0; 	section = sparse_index_alloc(nid);	if (!section)		return -ENOMEM; 	mem_section[root] = section; 	return 0;}#else /* !SPARSEMEM_EXTREME */static inline int sparse_index_init(unsigned long section_nr, int nid){	return 0;}#endif #ifdef CONFIG_SPARSEMEM_EXTREMEunsigned long __section_nr(struct mem_section *ms){	unsigned long root_nr;	struct mem_section *root = NULL; 	for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) {		root = __nr_to_section(root_nr * SECTIONS_PER_ROOT);		if (!root)			continue; 		if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT)))		     break;	} 	VM_BUG_ON(!root); 	return (root_nr * SECTIONS_PER_ROOT) + (ms - root);}#elseunsigned long __section_nr(struct mem_section *ms){	return (unsigned long)(ms - mem_section[0]);}#endif /* * During early boot, before section_mem_map is used for an actual * mem_map, we use section_mem_map to store the section's NUMA * node.  This keeps us from having to use another data structure.  The * node information is cleared just before we store the real mem_map. */static inline unsigned long sparse_encode_early_nid(int nid){	return (nid << SECTION_NID_SHIFT);} static inline int sparse_early_nid(struct mem_section *section){	return (section->section_mem_map >> SECTION_NID_SHIFT);} /* Validate the physical addressing limitations of the model */void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn,						unsigned long *end_pfn){	unsigned long max_sparsemem_pfn = 1UL << (MAX_PHYSMEM_BITS-PAGE_SHIFT); 	/*	 * Sanity checks - do not allow an architecture to pass	 * in larger pfns than the maximum scope of sparsemem:	 */	if (*start_pfn > max_sparsemem_pfn) {		mminit_dprintk(MMINIT_WARNING, "pfnvalidation",			"Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",			*start_pfn, *end_pfn, max_sparsemem_pfn);		WARN_ON_ONCE(1);		*start_pfn = max_sparsemem_pfn;		*end_pfn = max_sparsemem_pfn;	} else if (*end_pfn > max_sparsemem_pfn) {		mminit_dprintk(MMINIT_WARNING, "pfnvalidation",			"End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",			*start_pfn, *end_pfn, max_sparsemem_pfn);		WARN_ON_ONCE(1);		*end_pfn = max_sparsemem_pfn;	}} /* * There are a number of times that we loop over NR_MEM_SECTIONS, * looking for section_present() on each.  But, when we have very * large physical address spaces, NR_MEM_SECTIONS can also be * very large which makes the loops quite long. * * Keeping track of this gives us an easy way to break out of * those loops early. */unsigned long __highest_present_section_nr;static void section_mark_present(struct mem_section *ms){	unsigned long section_nr = __section_nr(ms); 	if (section_nr > __highest_present_section_nr)		__highest_present_section_nr = section_nr; 	ms->section_mem_map |= SECTION_MARKED_PRESENT;} #define for_each_present_section_nr(start, section_nr)		\	for (section_nr = next_present_section_nr(start-1);	\	     ((section_nr != -1) &&				\	      (section_nr <= __highest_present_section_nr));	\	     section_nr = next_present_section_nr(section_nr)) static inline unsigned long first_present_section_nr(void){	return next_present_section_nr(-1);} #ifdef CONFIG_SPARSEMEM_VMEMMAPstatic void subsection_mask_set(unsigned long *map, unsigned long pfn,		unsigned long nr_pages){	int idx = subsection_map_index(pfn);	int end = subsection_map_index(pfn + nr_pages - 1); 	bitmap_set(map, idx, end - idx + 1);} void __init subsection_map_init(unsigned long pfn, unsigned long nr_pages){	int end_sec = pfn_to_section_nr(pfn + nr_pages - 1);	unsigned long nr, start_sec = pfn_to_section_nr(pfn); 	if (!nr_pages)		return; 	for (nr = start_sec; nr <= end_sec; nr++) {		struct mem_section *ms;		unsigned long pfns; 		pfns = min(nr_pages, PAGES_PER_SECTION				- (pfn & ~PAGE_SECTION_MASK));		ms = __nr_to_section(nr);		subsection_mask_set(ms->usage->subsection_map, pfn, pfns); 		pr_debug("%s: sec: %lu pfns: %lu set(%d, %d)\n", __func__, nr,				pfns, subsection_map_index(pfn),				subsection_map_index(pfn + pfns - 1)); 		pfn += pfns;		nr_pages -= pfns;	}}#elsevoid __init subsection_map_init(unsigned long pfn, unsigned long nr_pages){}#endif /* Record a memory area against a node. */static void __init memory_present(int nid, unsigned long start, unsigned long end){	unsigned long pfn; #ifdef CONFIG_SPARSEMEM_EXTREME	if (unlikely(!mem_section)) {		unsigned long size, align; 		size = sizeof(struct mem_section *) * NR_SECTION_ROOTS;		align = 1 << (INTERNODE_CACHE_SHIFT);		mem_section = memblock_alloc(size, align);		if (!mem_section)			panic("%s: Failed to allocate %lu bytes align=0x%lx\n",			      __func__, size, align);	}#endif 	start &= PAGE_SECTION_MASK;	mminit_validate_memmodel_limits(&start, &end);	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) {		unsigned long section = pfn_to_section_nr(pfn);		struct mem_section *ms; 		sparse_index_init(section, nid);		set_section_nid(section, nid); 		ms = __nr_to_section(section);		if (!ms->section_mem_map) {			ms->section_mem_map = sparse_encode_early_nid(nid) |							SECTION_IS_ONLINE;			section_mark_present(ms);		}	}} /* * Mark all memblocks as present using memory_present(). * This is a convenience function that is useful to mark all of the systems * memory as present during initialization. */static void __init memblocks_present(void){	unsigned long start, end;	int i, nid; 	for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid)		memory_present(nid, start, end);} /* * Subtle, we encode the real pfn into the mem_map such that * the identity pfn - section_mem_map will return the actual * physical page frame number. */static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum){	unsigned long coded_mem_map =		(unsigned long)(mem_map - (section_nr_to_pfn(pnum)));	BUILD_BUG_ON(SECTION_MAP_LAST_BIT > (1UL<<PFN_SECTION_SHIFT));	BUG_ON(coded_mem_map & ~SECTION_MAP_MASK);	return coded_mem_map;} #ifdef CONFIG_MEMORY_HOTPLUG/* * Decode mem_map from the coded memmap */struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum){	/* mask off the extra low bits of information */	coded_mem_map &= SECTION_MAP_MASK;	return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum);}#endif /* CONFIG_MEMORY_HOTPLUG */ static void __meminit sparse_init_one_section(struct mem_section *ms,		unsigned long pnum, struct page *mem_map,		struct mem_section_usage *usage, unsigned long flags){	ms->section_mem_map &= ~SECTION_MAP_MASK;	ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum)		| SECTION_HAS_MEM_MAP | flags;	ms->usage = usage;} static unsigned long usemap_size(void){	return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS) * sizeof(unsigned long);} size_t mem_section_usage_size(void){	return sizeof(struct mem_section_usage) + usemap_size();} static inline phys_addr_t pgdat_to_phys(struct pglist_data *pgdat){#ifndef CONFIG_NEED_MULTIPLE_NODES	return __pa_symbol(pgdat);#else	return __pa(pgdat);#endif} #ifdef CONFIG_MEMORY_HOTREMOVEstatic struct mem_section_usage * __initsparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,					 unsigned long size){	struct mem_section_usage *usage;	unsigned long goal, limit;	int nid;	/*	 * A page may contain usemaps for other sections preventing the	 * page being freed and making a section unremovable while	 * other sections referencing the usemap remain active. Similarly,	 * a pgdat can prevent a section being removed. If section A	 * contains a pgdat and section B contains the usemap, both	 * sections become inter-dependent. This allocates usemaps	 * from the same section as the pgdat where possible to avoid	 * this problem.	 */	goal = pgdat_to_phys(pgdat) & (PAGE_SECTION_MASK << PAGE_SHIFT);	limit = goal + (1UL << PA_SECTION_SHIFT);	nid = early_pfn_to_nid(goal >> PAGE_SHIFT);again:	usage = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, goal, limit, nid);	if (!usage && limit) {		limit = 0;		goto again;	}	return usage;} static void __init check_usemap_section_nr(int nid,		struct mem_section_usage *usage){	unsigned long usemap_snr, pgdat_snr;	static unsigned long old_usemap_snr;	static unsigned long old_pgdat_snr;	struct pglist_data *pgdat = NODE_DATA(nid);	int usemap_nid; 	/* First call */	if (!old_usemap_snr) {		old_usemap_snr = NR_MEM_SECTIONS;		old_pgdat_snr = NR_MEM_SECTIONS;	} 	usemap_snr = pfn_to_section_nr(__pa(usage) >> PAGE_SHIFT);	pgdat_snr = pfn_to_section_nr(pgdat_to_phys(pgdat) >> PAGE_SHIFT);	if (usemap_snr == pgdat_snr)		return; 	if (old_usemap_snr == usemap_snr && old_pgdat_snr == pgdat_snr)		/* skip redundant message */		return; 	old_usemap_snr = usemap_snr;	old_pgdat_snr = pgdat_snr; 	usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr));	if (usemap_nid != nid) {		pr_info("node %d must be removed before remove section %ld\n",			nid, usemap_snr);		return;	}	/*	 * There is a circular dependency.	 * Some platforms allow un-removable section because they will just	 * gather other removable sections for dynamic partitioning.	 * Just notify un-removable section's number here.	 */	pr_info("Section %ld and %ld (node %d) have a circular dependency on usemap and pgdat allocations\n",		usemap_snr, pgdat_snr, nid);}#elsestatic struct mem_section_usage * __initsparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,					 unsigned long size){	return memblock_alloc_node(size, SMP_CACHE_BYTES, pgdat->node_id);} static void __init check_usemap_section_nr(int nid,		struct mem_section_usage *usage){}#endif /* CONFIG_MEMORY_HOTREMOVE */ #ifdef CONFIG_SPARSEMEM_VMEMMAPstatic unsigned long __init section_map_size(void){	return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE);} #elsestatic unsigned long __init section_map_size(void){	return PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION);} struct page __init *__populate_section_memmap(unsigned long pfn,		unsigned long nr_pages, int nid, struct vmem_altmap *altmap){	unsigned long size = section_map_size();	struct page *map = sparse_buffer_alloc(size);	phys_addr_t addr = __pa(MAX_DMA_ADDRESS); 	if (map)		return map; 	map = memblock_alloc_try_nid_raw(size, size, addr,					  MEMBLOCK_ALLOC_ACCESSIBLE, nid);	if (!map)		panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n",		      __func__, size, PAGE_SIZE, nid, &addr); 	return map;}#endif /* !CONFIG_SPARSEMEM_VMEMMAP */ static void *sparsemap_buf __meminitdata;static void *sparsemap_buf_end __meminitdata; static inline void __meminit sparse_buffer_free(unsigned long size){	WARN_ON(!sparsemap_buf || size == 0);	memblock_free_early(__pa(sparsemap_buf), size);} static void __init sparse_buffer_init(unsigned long size, int nid){	phys_addr_t addr = __pa(MAX_DMA_ADDRESS);	WARN_ON(sparsemap_buf);	/* forgot to call sparse_buffer_fini()? */	/*	 * Pre-allocated buffer is mainly used by __populate_section_memmap	 * and we want it to be properly aligned to the section size - this is	 * especially the case for VMEMMAP which maps memmap to PMDs	 */	sparsemap_buf = memblock_alloc_exact_nid_raw(size, section_map_size(),					addr, MEMBLOCK_ALLOC_ACCESSIBLE, nid);	sparsemap_buf_end = sparsemap_buf + size;} static void __init sparse_buffer_fini(void){	unsigned long size = sparsemap_buf_end - sparsemap_buf; 

Lines 1–500 of 981. Load more