/* * sparse memory mappings. */#include <linux/mm.h>#include <linux/slab.h>#include <linux/mmzone.h>#include <linux/bootmem.h>#include <linux/compiler.h>#include <linux/highmem.h>#include <linux/export.h>#include <linux/spinlock.h>#include <linux/vmalloc.h> #include "internal.h"#include <asm/dma.h>#include <asm/pgalloc.h>#include <asm/pgtable.h> /* * Permanent SPARSEMEM data: * * 1) mem_section	- memory sections, mem_map's for valid memory */#ifdef CONFIG_SPARSEMEM_EXTREMEstruct mem_section *mem_section[NR_SECTION_ROOTS]	____cacheline_internodealigned_in_smp;#elsestruct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]	____cacheline_internodealigned_in_smp;#endifEXPORT_SYMBOL(mem_section); #ifdef NODE_NOT_IN_PAGE_FLAGS/* * If we did not store the node number in the page then we have to * do a lookup in the section_to_node_table in order to find which * node the page belongs to. */#if MAX_NUMNODES <= 256static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;#elsestatic u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;#endif int page_to_nid(const struct page *page){	return section_to_node_table[page_to_section(page)];}EXPORT_SYMBOL(page_to_nid); static void set_section_nid(unsigned long section_nr, int nid){	section_to_node_table[section_nr] = nid;}#else /* !NODE_NOT_IN_PAGE_FLAGS */static inline void set_section_nid(unsigned long section_nr, int nid){}#endif #ifdef CONFIG_SPARSEMEM_EXTREMEstatic noinline struct mem_section __ref *sparse_index_alloc(int nid){	struct mem_section *section = NULL;	unsigned long array_size = SECTIONS_PER_ROOT *				   sizeof(struct mem_section); 	if (slab_is_available()) {		if (node_state(nid, N_HIGH_MEMORY))			section = kzalloc_node(array_size, GFP_KERNEL, nid);		else			section = kzalloc(array_size, GFP_KERNEL);	} else {		section = memblock_virt_alloc_node(array_size, nid);	} 	return section;} static int __meminit sparse_index_init(unsigned long section_nr, int nid){	unsigned long root = SECTION_NR_TO_ROOT(section_nr);	struct mem_section *section; 	if (mem_section[root])		return -EEXIST; 	section = sparse_index_alloc(nid);	if (!section)		return -ENOMEM; 	mem_section[root] = section; 	return 0;}#else /* !SPARSEMEM_EXTREME */static inline int sparse_index_init(unsigned long section_nr, int nid){	return 0;}#endif #ifdef CONFIG_SPARSEMEM_EXTREMEint __section_nr(struct mem_section* ms){	unsigned long root_nr;	struct mem_section* root; 	for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) {		root = __nr_to_section(root_nr * SECTIONS_PER_ROOT);		if (!root)			continue; 		if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT)))		     break;	} 	VM_BUG_ON(root_nr == NR_SECTION_ROOTS); 	return (root_nr * SECTIONS_PER_ROOT) + (ms - root);}#elseint __section_nr(struct mem_section* ms){	return (int)(ms - mem_section[0]);}#endif /* * During early boot, before section_mem_map is used for an actual * mem_map, we use section_mem_map to store the section's NUMA * node.  This keeps us from having to use another data structure.  The * node information is cleared just before we store the real mem_map. */static inline unsigned long sparse_encode_early_nid(int nid){	return (nid << SECTION_NID_SHIFT);} static inline int sparse_early_nid(struct mem_section *section){	return (section->section_mem_map >> SECTION_NID_SHIFT);} /* Validate the physical addressing limitations of the model */void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn,						unsigned long *end_pfn){	unsigned long max_sparsemem_pfn = 1UL << (MAX_PHYSMEM_BITS-PAGE_SHIFT); 	/*	 * Sanity checks - do not allow an architecture to pass	 * in larger pfns than the maximum scope of sparsemem:	 */	if (*start_pfn > max_sparsemem_pfn) {		mminit_dprintk(MMINIT_WARNING, "pfnvalidation",			"Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",			*start_pfn, *end_pfn, max_sparsemem_pfn);		WARN_ON_ONCE(1);		*start_pfn = max_sparsemem_pfn;		*end_pfn = max_sparsemem_pfn;	} else if (*end_pfn > max_sparsemem_pfn) {		mminit_dprintk(MMINIT_WARNING, "pfnvalidation",			"End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",			*start_pfn, *end_pfn, max_sparsemem_pfn);		WARN_ON_ONCE(1);		*end_pfn = max_sparsemem_pfn;	}} /* * There are a number of times that we loop over NR_MEM_SECTIONS, * looking for section_present() on each.  But, when we have very * large physical address spaces, NR_MEM_SECTIONS can also be * very large which makes the loops quite long. * * Keeping track of this gives us an easy way to break out of * those loops early. */int __highest_present_section_nr;static void section_mark_present(struct mem_section *ms){	int section_nr = __section_nr(ms); 	if (section_nr > __highest_present_section_nr)		__highest_present_section_nr = section_nr; 	ms->section_mem_map |= SECTION_MARKED_PRESENT;} static inline int next_present_section_nr(int section_nr){	do {		section_nr++;		if (present_section_nr(section_nr))			return section_nr;	} while ((section_nr < NR_MEM_SECTIONS) &&		 (section_nr <= __highest_present_section_nr)); 	return -1;}#define for_each_present_section_nr(start, section_nr)		\	for (section_nr = next_present_section_nr(start-1);	\	     ((section_nr >= 0) &&				\	      (section_nr < NR_MEM_SECTIONS) &&			\	      (section_nr <= __highest_present_section_nr));	\	     section_nr = next_present_section_nr(section_nr)) /* Record a memory area against a node. */void __init memory_present(int nid, unsigned long start, unsigned long end){	unsigned long pfn; 	start &= PAGE_SECTION_MASK;	mminit_validate_memmodel_limits(&start, &end);	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) {		unsigned long section = pfn_to_section_nr(pfn);		struct mem_section *ms; 		sparse_index_init(section, nid);		set_section_nid(section, nid); 		ms = __nr_to_section(section);		if (!ms->section_mem_map) {			ms->section_mem_map = sparse_encode_early_nid(nid) |							SECTION_IS_ONLINE;			section_mark_present(ms);		}	}} /* * Only used by the i386 NUMA architecures, but relatively * generic code. */unsigned long __init node_memmap_size_bytes(int nid, unsigned long start_pfn,						     unsigned long end_pfn){	unsigned long pfn;	unsigned long nr_pages = 0; 	mminit_validate_memmodel_limits(&start_pfn, &end_pfn);	for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {		if (nid != early_pfn_to_nid(pfn))			continue; 		if (pfn_present(pfn))			nr_pages += PAGES_PER_SECTION;	} 	return nr_pages * sizeof(struct page);} /* * Subtle, we encode the real pfn into the mem_map such that * the identity pfn - section_mem_map will return the actual * physical page frame number. */static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum){	return (unsigned long)(mem_map - (section_nr_to_pfn(pnum)));} /* * Decode mem_map from the coded memmap */struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum){	/* mask off the extra low bits of information */	coded_mem_map &= SECTION_MAP_MASK;	return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum);} static int __meminit sparse_init_one_section(struct mem_section *ms,		unsigned long pnum, struct page *mem_map,		unsigned long *pageblock_bitmap){	if (!present_section(ms))		return -EINVAL; 	ms->section_mem_map &= ~SECTION_MAP_MASK;	ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) |							SECTION_HAS_MEM_MAP; 	ms->pageblock_flags = pageblock_bitmap; 	return 1;} unsigned long usemap_size(void){	return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS) * sizeof(unsigned long);} #ifdef CONFIG_MEMORY_HOTPLUGstatic unsigned long *__kmalloc_section_usemap(void){	return kmalloc(usemap_size(), GFP_KERNEL);}#endif /* CONFIG_MEMORY_HOTPLUG */ #ifdef CONFIG_MEMORY_HOTREMOVEstatic unsigned long * __initsparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,					 unsigned long size){	unsigned long goal, limit;	unsigned long *p;	int nid;	/*	 * A page may contain usemaps for other sections preventing the	 * page being freed and making a section unremovable while	 * other sections referencing the usemap remain active. Similarly,	 * a pgdat can prevent a section being removed. If section A	 * contains a pgdat and section B contains the usemap, both	 * sections become inter-dependent. This allocates usemaps	 * from the same section as the pgdat where possible to avoid	 * this problem.	 */	goal = __pa(pgdat) & (PAGE_SECTION_MASK << PAGE_SHIFT);	limit = goal + (1UL << PA_SECTION_SHIFT);	nid = early_pfn_to_nid(goal >> PAGE_SHIFT);again:	p = memblock_virt_alloc_try_nid_nopanic(size,						SMP_CACHE_BYTES, goal, limit,						nid);	if (!p && limit) {		limit = 0;		goto again;	}	return p;} static void __init check_usemap_section_nr(int nid, unsigned long *usemap){	unsigned long usemap_snr, pgdat_snr;	static unsigned long old_usemap_snr = NR_MEM_SECTIONS;	static unsigned long old_pgdat_snr = NR_MEM_SECTIONS;	struct pglist_data *pgdat = NODE_DATA(nid);	int usemap_nid; 	usemap_snr = pfn_to_section_nr(__pa(usemap) >> PAGE_SHIFT);	pgdat_snr = pfn_to_section_nr(__pa(pgdat) >> PAGE_SHIFT);	if (usemap_snr == pgdat_snr)		return; 	if (old_usemap_snr == usemap_snr && old_pgdat_snr == pgdat_snr)		/* skip redundant message */		return; 	old_usemap_snr = usemap_snr;	old_pgdat_snr = pgdat_snr; 	usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr));	if (usemap_nid != nid) {		pr_info("node %d must be removed before remove section %ld\n",			nid, usemap_snr);		return;	}	/*	 * There is a circular dependency.	 * Some platforms allow un-removable section because they will just	 * gather other removable sections for dynamic partitioning.	 * Just notify un-removable section's number here.	 */	pr_info("Section %ld and %ld (node %d) have a circular dependency on usemap and pgdat allocations\n",		usemap_snr, pgdat_snr, nid);}#elsestatic unsigned long * __initsparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,					 unsigned long size){	return memblock_virt_alloc_node_nopanic(size, pgdat->node_id);} static void __init check_usemap_section_nr(int nid, unsigned long *usemap){}#endif /* CONFIG_MEMORY_HOTREMOVE */ static void __init sparse_early_usemaps_alloc_node(void *data,				 unsigned long pnum_begin,				 unsigned long pnum_end,				 unsigned long usemap_count, int nodeid){	void *usemap;	unsigned long pnum;	unsigned long **usemap_map = (unsigned long **)data;	int size = usemap_size(); 	usemap = sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid),							  size * usemap_count);	if (!usemap) {		pr_warn("%s: allocation failed\n", __func__);		return;	} 	for (pnum = pnum_begin; pnum < pnum_end; pnum++) {		if (!present_section_nr(pnum))			continue;		usemap_map[pnum] = usemap;		usemap += size;		check_usemap_section_nr(nodeid, usemap_map[pnum]);	}} #ifndef CONFIG_SPARSEMEM_VMEMMAPstruct page __init *sparse_mem_map_populate(unsigned long pnum, int nid){	struct page *map;	unsigned long size; 	map = alloc_remap(nid, sizeof(struct page) * PAGES_PER_SECTION);	if (map)		return map; 	size = PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION);	map = memblock_virt_alloc_try_nid(size,					  PAGE_SIZE, __pa(MAX_DMA_ADDRESS),					  BOOTMEM_ALLOC_ACCESSIBLE, nid);	return map;}void __init sparse_mem_maps_populate_node(struct page **map_map,					  unsigned long pnum_begin,					  unsigned long pnum_end,					  unsigned long map_count, int nodeid){	void *map;	unsigned long pnum;	unsigned long size = sizeof(struct page) * PAGES_PER_SECTION; 	map = alloc_remap(nodeid, size * map_count);	if (map) {		for (pnum = pnum_begin; pnum < pnum_end; pnum++) {			if (!present_section_nr(pnum))				continue;			map_map[pnum] = map;			map += size;		}		return;	} 	size = PAGE_ALIGN(size);	map = memblock_virt_alloc_try_nid(size * map_count,					  PAGE_SIZE, __pa(MAX_DMA_ADDRESS),					  BOOTMEM_ALLOC_ACCESSIBLE, nodeid);	if (map) {		for (pnum = pnum_begin; pnum < pnum_end; pnum++) {			if (!present_section_nr(pnum))				continue;			map_map[pnum] = map;			map += size;		}		return;	} 	/* fallback */	for (pnum = pnum_begin; pnum < pnum_end; pnum++) {		struct mem_section *ms; 		if (!present_section_nr(pnum))			continue;		map_map[pnum] = sparse_mem_map_populate(pnum, nodeid);		if (map_map[pnum])			continue;		ms = __nr_to_section(pnum);		pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",		       __func__);		ms->section_mem_map = 0;	}}#endif /* !CONFIG_SPARSEMEM_VMEMMAP */ #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHERstatic void __init sparse_early_mem_maps_alloc_node(void *data,				 unsigned long pnum_begin,				 unsigned long pnum_end,				 unsigned long map_count, int nodeid){	struct page **map_map = (struct page **)data;	sparse_mem_maps_populate_node(map_map, pnum_begin, pnum_end,					 map_count, nodeid);}#elsestatic struct page __init *sparse_early_mem_map_alloc(unsigned long pnum){	struct page *map;	struct mem_section *ms = __nr_to_section(pnum);	int nid = sparse_early_nid(ms); 	map = sparse_mem_map_populate(pnum, nid);	if (map)		return map; 	pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",	       __func__);	ms->section_mem_map = 0;	return NULL;}#endif

Lines 1–500 of 883. Load more