// SPDX-License-Identifier: GPL-2.0-only/* * linux/mm/page_alloc.c * * Manages the free list, the system allocates free pages here. * Note that kmalloc() lives in slab.c * * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds * Swap reorganised 29.12.95, Stephen Tweedie * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) */ #include <linux/stddef.h>#include <linux/mm.h>#include <linux/highmem.h>#include <linux/swap.h>#include <linux/interrupt.h>#include <linux/pagemap.h>#include <linux/jiffies.h>#include <linux/memblock.h>#include <linux/compiler.h>#include <linux/kernel.h>#include <linux/kasan.h>#include <linux/module.h>#include <linux/suspend.h>#include <linux/pagevec.h>#include <linux/blkdev.h>#include <linux/slab.h>#include <linux/ratelimit.h>#include <linux/oom.h>#include <linux/topology.h>#include <linux/sysctl.h>#include <linux/cpu.h>#include <linux/cpuset.h>#include <linux/memory_hotplug.h>#include <linux/nodemask.h>#include <linux/vmalloc.h>#include <linux/vmstat.h>#include <linux/mempolicy.h>#include <linux/memremap.h>#include <linux/stop_machine.h>#include <linux/random.h>#include <linux/sort.h>#include <linux/pfn.h>#include <linux/backing-dev.h>#include <linux/fault-inject.h>#include <linux/page-isolation.h>#include <linux/debugobjects.h>#include <linux/kmemleak.h>#include <linux/compaction.h>#include <trace/events/kmem.h>#include <trace/events/oom.h>#include <linux/prefetch.h>#include <linux/mm_inline.h>#include <linux/mmu_notifier.h>#include <linux/migrate.h>#include <linux/hugetlb.h>#include <linux/sched/rt.h>#include <linux/sched/mm.h>#include <linux/page_owner.h>#include <linux/kthread.h>#include <linux/memcontrol.h>#include <linux/ftrace.h>#include <linux/lockdep.h>#include <linux/nmi.h>#include <linux/psi.h>#include <linux/padata.h>#include <linux/khugepaged.h>#include <linux/buffer_head.h>#include <asm/sections.h>#include <asm/tlbflush.h>#include <asm/div64.h>#include "internal.h"#include "shuffle.h"#include "page_reporting.h" /* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */typedef int __bitwise fpi_t; /* No special request */#define FPI_NONE ((__force fpi_t)0) /* * Skip free page reporting notification for the (possibly merged) page. * This does not hinder free page reporting from grabbing the page, * reporting it and marking it "reported" - it only skips notifying * the free page reporting infrastructure about a newly freed page. For * example, used when temporarily pulling a page from a freelist and * putting it back unmodified. */#define FPI_SKIP_REPORT_NOTIFY ((__force fpi_t)BIT(0)) /* * Place the (possibly merged) page to the tail of the freelist. Will ignore * page shuffling (relevant code - e.g., memory onlining - is expected to * shuffle the whole zone). * * Note: No code should rely on this flag for correctness - it's purely * to allow for optimizations when handing back either fresh pages * (memory onlining) or untouched pages (page isolation, free page * reporting). */#define FPI_TO_TAIL ((__force fpi_t)BIT(1)) /* * Don't poison memory with KASAN (only for the tag-based modes). * During boot, all non-reserved memblock memory is exposed to page_alloc. * Poisoning all that memory lengthens boot time, especially on systems with * large amount of RAM. This flag is used to skip that poisoning. * This is only done for the tag-based KASAN modes, as those are able to * detect memory corruptions with the memory tags assigned by default. * All memory allocated normally after boot gets poisoned as usual. */#define FPI_SKIP_KASAN_POISON ((__force fpi_t)BIT(2)) /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */static DEFINE_MUTEX(pcp_batch_high_lock);#define MIN_PERCPU_PAGELIST_FRACTION (8) #ifdef CONFIG_USE_PERCPU_NUMA_NODE_IDDEFINE_PER_CPU(int, numa_node);EXPORT_PER_CPU_SYMBOL(numa_node);#endif DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); #ifdef CONFIG_HAVE_MEMORYLESS_NODES/* * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() * defined in <linux/topology.h>. */DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */EXPORT_PER_CPU_SYMBOL(_numa_mem_);#endif /* work_structs for global per-cpu drains */struct pcpu_drain { struct zone *zone; struct work_struct work;};static DEFINE_MUTEX(pcpu_drain_mutex);static DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain); #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPYvolatile unsigned long latent_entropy __latent_entropy;EXPORT_SYMBOL(latent_entropy);#endif /* * Array of node states. */nodemask_t node_states[NR_NODE_STATES] __read_mostly = { [N_POSSIBLE] = NODE_MASK_ALL, [N_ONLINE] = { { [0] = 1UL } },#ifndef CONFIG_NUMA [N_NORMAL_MEMORY] = { { [0] = 1UL } },#ifdef CONFIG_HIGHMEM [N_HIGH_MEMORY] = { { [0] = 1UL } },#endif [N_MEMORY] = { { [0] = 1UL } }, [N_CPU] = { { [0] = 1UL } },#endif /* NUMA */};EXPORT_SYMBOL(node_states); atomic_long_t _totalram_pages __read_mostly;EXPORT_SYMBOL(_totalram_pages);unsigned long totalreserve_pages __read_mostly;unsigned long totalcma_pages __read_mostly; int percpu_pagelist_fraction;gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc);EXPORT_SYMBOL(init_on_alloc); DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free);EXPORT_SYMBOL(init_on_free); static bool _init_on_alloc_enabled_early __read_mostly = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON);static int __init early_init_on_alloc(char *buf){ return kstrtobool(buf, &_init_on_alloc_enabled_early);}early_param("init_on_alloc", early_init_on_alloc); static bool _init_on_free_enabled_early __read_mostly = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON);static int __init early_init_on_free(char *buf){ return kstrtobool(buf, &_init_on_free_enabled_early);}early_param("init_on_free", early_init_on_free); /* * A cached value of the page's pageblock's migratetype, used when the page is * put on a pcplist. Used to avoid the pageblock migratetype lookup when * freeing from pcplists in most cases, at the cost of possibly becoming stale. * Also the migratetype set in the page does not necessarily match the pcplist * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any * other index - this ensures that it will be put on the correct CMA freelist. */static inline int get_pcppage_migratetype(struct page *page){ return page->index;} static inline void set_pcppage_migratetype(struct page *page, int migratetype){ page->index = migratetype;} #ifdef CONFIG_PM_SLEEP/* * The following functions are used by the suspend/hibernate code to temporarily * change gfp_allowed_mask in order to avoid using I/O during memory allocations * while devices are suspended. To avoid races with the suspend/hibernate code, * they should always be called with system_transition_mutex held * (gfp_allowed_mask also should only be modified with system_transition_mutex * held, unless the suspend/hibernate code is guaranteed not to run in parallel * with that modification). */ static gfp_t saved_gfp_mask; void pm_restore_gfp_mask(void){ WARN_ON(!mutex_is_locked(&system_transition_mutex)); if (saved_gfp_mask) { gfp_allowed_mask = saved_gfp_mask; saved_gfp_mask = 0; }} void pm_restrict_gfp_mask(void){ WARN_ON(!mutex_is_locked(&system_transition_mutex)); WARN_ON(saved_gfp_mask); saved_gfp_mask = gfp_allowed_mask; gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);} bool pm_suspended_storage(void){ if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) return false; return true;}#endif /* CONFIG_PM_SLEEP */ #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLEunsigned int pageblock_order __read_mostly;#endif static void __free_pages_ok(struct page *page, unsigned int order, fpi_t fpi_flags); /* * results with 256, 32 in the lowmem_reserve sysctl: * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) * 1G machine -> (16M dma, 784M normal, 224M high) * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA * * TBD: should special case ZONE_DMA32 machines here - in those we normally * don't need any ZONE_NORMAL reservation */int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = {#ifdef CONFIG_ZONE_DMA [ZONE_DMA] = 256,#endif#ifdef CONFIG_ZONE_DMA32 [ZONE_DMA32] = 256,#endif [ZONE_NORMAL] = 32,#ifdef CONFIG_HIGHMEM [ZONE_HIGHMEM] = 0,#endif [ZONE_MOVABLE] = 0,}; static char * const zone_names[MAX_NR_ZONES] = {#ifdef CONFIG_ZONE_DMA "DMA",#endif#ifdef CONFIG_ZONE_DMA32 "DMA32",#endif "Normal",#ifdef CONFIG_HIGHMEM "HighMem",#endif "Movable",#ifdef CONFIG_ZONE_DEVICE "Device",#endif}; const char * const migratetype_names[MIGRATE_TYPES] = { "Unmovable", "Movable", "Reclaimable", "HighAtomic",#ifdef CONFIG_CMA "CMA",#endif#ifdef CONFIG_MEMORY_ISOLATION "Isolate",#endif}; compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = { [NULL_COMPOUND_DTOR] = NULL, [COMPOUND_PAGE_DTOR] = free_compound_page,#ifdef CONFIG_HUGETLB_PAGE [HUGETLB_PAGE_DTOR] = free_huge_page,#endif#ifdef CONFIG_TRANSPARENT_HUGEPAGE [TRANSHUGE_PAGE_DTOR] = free_transhuge_page,#endif}; int min_free_kbytes = 1024;int user_min_free_kbytes = -1;#ifdef CONFIG_DISCONTIGMEM/* * DiscontigMem defines memory ranges as separate pg_data_t even if the ranges * are not on separate NUMA nodes. Functionally this works but with * watermark_boost_factor, it can reclaim prematurely as the ranges can be * quite small. By default, do not boost watermarks on discontigmem as in * many cases very high-order allocations like THP are likely to be * unsupported and the premature reclaim offsets the advantage of long-term * fragmentation avoidance. */int watermark_boost_factor __read_mostly;#elseint watermark_boost_factor __read_mostly = 15000;#endifint watermark_scale_factor = 10; static unsigned long nr_kernel_pages __initdata;static unsigned long nr_all_pages __initdata;static unsigned long dma_reserve __initdata; static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata;static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata;static unsigned long required_kernelcore __initdata;static unsigned long required_kernelcore_percent __initdata;static unsigned long required_movablecore __initdata;static unsigned long required_movablecore_percent __initdata;static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata;static bool mirrored_kernelcore __meminitdata; /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */int movable_zone;EXPORT_SYMBOL(movable_zone); #if MAX_NUMNODES > 1unsigned int nr_node_ids __read_mostly = MAX_NUMNODES;unsigned int nr_online_nodes __read_mostly = 1;EXPORT_SYMBOL(nr_node_ids);EXPORT_SYMBOL(nr_online_nodes);#endif int page_group_by_mobility_disabled __read_mostly; #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT/* * During boot we initialize deferred pages on-demand, as needed, but once * page_alloc_init_late() has finished, the deferred pages are all initialized, * and we can permanently disable that path. */static DEFINE_STATIC_KEY_TRUE(deferred_pages); /* * Calling kasan_free_pages() only after deferred memory initialization * has completed. Poisoning pages during deferred memory init will greatly * lengthen the process and cause problem in large memory systems as the * deferred pages initialization is done with interrupt disabled. * * Assuming that there will be no reference to those newly initialized * pages before they are ever allocated, this should have no effect on * KASAN memory tracking as the poison will be properly inserted at page * allocation time. The only corner case is when pages are allocated by * on-demand allocation and then freed again before the deferred pages * initialization is done, but this is not likely to happen. */static inline void kasan_free_nondeferred_pages(struct page *page, int order, bool init, fpi_t fpi_flags){ if (static_branch_unlikely(&deferred_pages)) return; if (!IS_ENABLED(CONFIG_KASAN_GENERIC) && (fpi_flags & FPI_SKIP_KASAN_POISON)) return; kasan_free_pages(page, order, init);} /* Returns true if the struct page for the pfn is uninitialised */static inline bool __meminit early_page_uninitialised(unsigned long pfn){ int nid = early_pfn_to_nid(pfn); if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) return true; return false;} /* * Returns true when the remaining initialisation should be deferred until * later in the boot cycle when it can be parallelised. */static bool __meminitdefer_init(int nid, unsigned long pfn, unsigned long end_pfn){ static unsigned long prev_end_pfn, nr_initialised; /* * prev_end_pfn static that contains the end of previous zone * No need to protect because called very early in boot before smp_init. */ if (prev_end_pfn != end_pfn) { prev_end_pfn = end_pfn; nr_initialised = 0; } /* Always populate low zones for address-constrained allocations */ if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) return false; if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX) return true; /* * We start only with one section of pages, more pages are added as * needed until the rest of deferred pages are initialized. */ nr_initialised++; if ((nr_initialised > PAGES_PER_SECTION) && (pfn & (PAGES_PER_SECTION - 1)) == 0) { NODE_DATA(nid)->first_deferred_pfn = pfn; return true; } return false;}#elsestatic inline void kasan_free_nondeferred_pages(struct page *page, int order, bool init, fpi_t fpi_flags){ if (!IS_ENABLED(CONFIG_KASAN_GENERIC) && (fpi_flags & FPI_SKIP_KASAN_POISON)) return; kasan_free_pages(page, order, init);} static inline bool early_page_uninitialised(unsigned long pfn){ return false;} static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn){ return false;}#endif /* Return a pointer to the bitmap storing bits affecting a block of pages */static inline unsigned long *get_pageblock_bitmap(struct page *page, unsigned long pfn){#ifdef CONFIG_SPARSEMEM return section_to_usemap(__pfn_to_section(pfn));#else return page_zone(page)->pageblock_flags;#endif /* CONFIG_SPARSEMEM */} static inline int pfn_to_bitidx(struct page *page, unsigned long pfn){#ifdef CONFIG_SPARSEMEM pfn &= (PAGES_PER_SECTION-1);#else pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);#endif /* CONFIG_SPARSEMEM */ return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;} static __always_inlineunsigned long __get_pfnblock_flags_mask(struct page *page, unsigned long pfn, unsigned long mask)Lines 1–500 of 9176. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * linux/mm/page_alloc.c * * Manages the free list, the system allocates free pages here. * Note that kmalloc() lives in slab.c * * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds * Swap reorganised 29.12.95, Stephen Tweedie * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) */ #include <linux/stddef.h> #include <linux/mm.h> #include <linux/highmem.h> #include <linux/swap.h> #include <linux/interrupt.h> #include <linux/pagemap.h> #include <linux/jiffies.h> #include <linux/memblock.h> #include <linux/compiler.h> #include <linux/kernel.h> #include <linux/kasan.h> #include <linux/module.h> #include <linux/suspend.h> #include <linux/pagevec.h> #include <linux/blkdev.h> #include <linux/slab.h> #include <linux/ratelimit.h> #include <linux/oom.h> #include <linux/topology.h> #include <linux/sysctl.h> #include <linux/cpu.h> #include <linux/cpuset.h> #include <linux/memory_hotplug.h> #include <linux/nodemask.h> #include <linux/vmalloc.h> #include <linux/vmstat.h> #include <linux/mempolicy.h> #include <linux/memremap.h> #include <linux/stop_machine.h> #include <linux/random.h> #include <linux/sort.h> #include <linux/pfn.h> #include <linux/backing-dev.h> #include <linux/fault-inject.h> #include <linux/page-isolation.h> #include <linux/debugobjects.h> #include <linux/kmemleak.h> #include <linux/compaction.h> #include <trace/events/kmem.h> #include <trace/events/oom.h> #include <linux/prefetch.h> #include <linux/mm_inline.h> #include <linux/mmu_notifier.h> #include <linux/migrate.h> #include <linux/hugetlb.h> #include <linux/sched/rt.h> #include <linux/sched/mm.h> #include <linux/page_owner.h> #include <linux/kthread.h> #include <linux/memcontrol.h> #include <linux/ftrace.h> #include <linux/lockdep.h> #include <linux/nmi.h> #include <linux/psi.h> #include <linux/padata.h> #include <linux/khugepaged.h> #include <linux/buffer_head.h> #include <asm/sections.h> #include <asm/tlbflush.h> #include <asm/div64.h> #include "internal.h" #include "shuffle.h" #include "page_reporting.h" /* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */ typedef int __bitwise fpi_t; /* No special request */ #define FPI_NONE ((__force fpi_t)0) /* * Skip free page reporting notification for the (possibly merged) page. * This does not hinder free page reporting from grabbing the page, * reporting it and marking it "reported" - it only skips notifying * the free page reporting infrastructure about a newly freed page. For * example, used when temporarily pulling a page from a freelist and * putting it back unmodified. */ #define FPI_SKIP_REPORT_NOTIFY ((__force fpi_t)BIT(0)) /* * Place the (possibly merged) page to the tail of the freelist. Will ignore * page shuffling (relevant code - e.g., memory onlining - is expected to * shuffle the whole zone). * * Note: No code should rely on this flag for correctness - it's purely * to allow for optimizations when handing back either fresh pages * (memory onlining) or untouched pages (page isolation, free page * reporting). */ #define FPI_TO_TAIL ((__force fpi_t)BIT(1)) /* * Don't poison memory with KASAN (only for the tag-based modes). * During boot, all non-reserved memblock memory is exposed to page_alloc. * Poisoning all that memory lengthens boot time, especially on systems with * large amount of RAM. This flag is used to skip that poisoning. * This is only done for the tag-based KASAN modes, as those are able to * detect memory corruptions with the memory tags assigned by default. * All memory allocated normally after boot gets poisoned as usual. */ #define FPI_SKIP_KASAN_POISON ((__force fpi_t)BIT(2)) /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ static DEFINE_MUTEX(pcp_batch_high_lock); #define MIN_PERCPU_PAGELIST_FRACTION (8) #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID DEFINE_PER_CPU(int, numa_node); EXPORT_PER_CPU_SYMBOL(numa_node); #endif DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); #ifdef CONFIG_HAVE_MEMORYLESS_NODES /* * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() * defined in <linux/topology.h>. */ DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ EXPORT_PER_CPU_SYMBOL(_numa_mem_); #endif /* work_structs for global per-cpu drains */ struct pcpu_drain { struct zone *zone; struct work_struct work; }; static DEFINE_MUTEX(pcpu_drain_mutex); static DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain); #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY volatile unsigned long latent_entropy __latent_entropy; EXPORT_SYMBOL(latent_entropy); #endif /* * Array of node states. */ nodemask_t node_states[NR_NODE_STATES] __read_mostly = { [N_POSSIBLE] = NODE_MASK_ALL, [N_ONLINE] = { { [0] = 1UL } }, #ifndef CONFIG_NUMA [N_NORMAL_MEMORY] = { { [0] = 1UL } }, #ifdef CONFIG_HIGHMEM [N_HIGH_MEMORY] = { { [0] = 1UL } }, #endif [N_MEMORY] = { { [0] = 1UL } }, [N_CPU] = { { [0] = 1UL } }, #endif /* NUMA */ }; EXPORT_SYMBOL(node_states); atomic_long_t _totalram_pages __read_mostly; EXPORT_SYMBOL(_totalram_pages); unsigned long totalreserve_pages __read_mostly; unsigned long totalcma_pages __read_mostly; int percpu_pagelist_fraction; gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc); EXPORT_SYMBOL(init_on_alloc); DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); EXPORT_SYMBOL(init_on_free); static bool _init_on_alloc_enabled_early __read_mostly = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON); static int __init early_init_on_alloc(char *buf) { return kstrtobool(buf, &_init_on_alloc_enabled_early); } early_param("init_on_alloc", early_init_on_alloc); static bool _init_on_free_enabled_early __read_mostly = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON); static int __init early_init_on_free(char *buf) { return kstrtobool(buf, &_init_on_free_enabled_early); } early_param("init_on_free", early_init_on_free); /* * A cached value of the page's pageblock's migratetype, used when the page is * put on a pcplist. Used to avoid the pageblock migratetype lookup when * freeing from pcplists in most cases, at the cost of possibly becoming stale. * Also the migratetype set in the page does not necessarily match the pcplist * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any * other index - this ensures that it will be put on the correct CMA freelist. */ static inline int get_pcppage_migratetype(struct page *page) { return page->index; } static inline void set_pcppage_migratetype(struct page *page, int migratetype) { page->index = migratetype; } #ifdef CONFIG_PM_SLEEP /* * The following functions are used by the suspend/hibernate code to temporarily * change gfp_allowed_mask in order to avoid using I/O during memory allocations * while devices are suspended. To avoid races with the suspend/hibernate code, * they should always be called with system_transition_mutex held * (gfp_allowed_mask also should only be modified with system_transition_mutex * held, unless the suspend/hibernate code is guaranteed not to run in parallel * with that modification). */ static gfp_t saved_gfp_mask; void pm_restore_gfp_mask(void) { WARN_ON(!mutex_is_locked(&system_transition_mutex)); if (saved_gfp_mask) { gfp_allowed_mask = saved_gfp_mask; saved_gfp_mask = 0; } } void pm_restrict_gfp_mask(void) { WARN_ON(!mutex_is_locked(&system_transition_mutex)); WARN_ON(saved_gfp_mask); saved_gfp_mask = gfp_allowed_mask; gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); } bool pm_suspended_storage(void) { if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) return false; return true; } #endif /* CONFIG_PM_SLEEP */ #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE unsigned int pageblock_order __read_mostly; #endif static void __free_pages_ok(struct page *page, unsigned int order, fpi_t fpi_flags); /* * results with 256, 32 in the lowmem_reserve sysctl: * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) * 1G machine -> (16M dma, 784M normal, 224M high) * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA * * TBD: should special case ZONE_DMA32 machines here - in those we normally * don't need any ZONE_NORMAL reservation */ int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = { #ifdef CONFIG_ZONE_DMA [ZONE_DMA] = 256, #endif #ifdef CONFIG_ZONE_DMA32 [ZONE_DMA32] = 256, #endif [ZONE_NORMAL] = 32, #ifdef CONFIG_HIGHMEM [ZONE_HIGHMEM] = 0, #endif [ZONE_MOVABLE] = 0, }; static char * const zone_names[MAX_NR_ZONES] = { #ifdef CONFIG_ZONE_DMA "DMA", #endif #ifdef CONFIG_ZONE_DMA32 "DMA32", #endif "Normal", #ifdef CONFIG_HIGHMEM "HighMem", #endif "Movable", #ifdef CONFIG_ZONE_DEVICE "Device", #endif }; const char * const migratetype_names[MIGRATE_TYPES] = { "Unmovable", "Movable", "Reclaimable", "HighAtomic", #ifdef CONFIG_CMA "CMA", #endif #ifdef CONFIG_MEMORY_ISOLATION "Isolate", #endif }; compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = { [NULL_COMPOUND_DTOR] = NULL, [COMPOUND_PAGE_DTOR] = free_compound_page, #ifdef CONFIG_HUGETLB_PAGE [HUGETLB_PAGE_DTOR] = free_huge_page, #endif #ifdef CONFIG_TRANSPARENT_HUGEPAGE [TRANSHUGE_PAGE_DTOR] = free_transhuge_page, #endif }; int min_free_kbytes = 1024; int user_min_free_kbytes = -1; #ifdef CONFIG_DISCONTIGMEM /* * DiscontigMem defines memory ranges as separate pg_data_t even if the ranges * are not on separate NUMA nodes. Functionally this works but with * watermark_boost_factor, it can reclaim prematurely as the ranges can be * quite small. By default, do not boost watermarks on discontigmem as in * many cases very high-order allocations like THP are likely to be * unsupported and the premature reclaim offsets the advantage of long-term * fragmentation avoidance. */ int watermark_boost_factor __read_mostly; #else int watermark_boost_factor __read_mostly = 15000; #endif int watermark_scale_factor = 10; static unsigned long nr_kernel_pages __initdata; static unsigned long nr_all_pages __initdata; static unsigned long dma_reserve __initdata; static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; static unsigned long required_kernelcore __initdata; static unsigned long required_kernelcore_percent __initdata; static unsigned long required_movablecore __initdata; static unsigned long required_movablecore_percent __initdata; static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; static bool mirrored_kernelcore __meminitdata; /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ int movable_zone; EXPORT_SYMBOL(movable_zone); #if MAX_NUMNODES > 1 unsigned int nr_node_ids __read_mostly = MAX_NUMNODES; unsigned int nr_online_nodes __read_mostly = 1; EXPORT_SYMBOL(nr_node_ids); EXPORT_SYMBOL(nr_online_nodes); #endif int page_group_by_mobility_disabled __read_mostly; #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT /* * During boot we initialize deferred pages on-demand, as needed, but once * page_alloc_init_late() has finished, the deferred pages are all initialized, * and we can permanently disable that path. */ static DEFINE_STATIC_KEY_TRUE(deferred_pages); /* * Calling kasan_free_pages() only after deferred memory initialization * has completed. Poisoning pages during deferred memory init will greatly * lengthen the process and cause problem in large memory systems as the * deferred pages initialization is done with interrupt disabled. * * Assuming that there will be no reference to those newly initialized * pages before they are ever allocated, this should have no effect on * KASAN memory tracking as the poison will be properly inserted at page * allocation time. The only corner case is when pages are allocated by * on-demand allocation and then freed again before the deferred pages * initialization is done, but this is not likely to happen. */ static inline void kasan_free_nondeferred_pages(struct page *page, int order, bool init, fpi_t fpi_flags) { if (static_branch_unlikely(&deferred_pages)) return; if (!IS_ENABLED(CONFIG_KASAN_GENERIC) && (fpi_flags & FPI_SKIP_KASAN_POISON)) return; kasan_free_pages(page, order, init); } /* Returns true if the struct page for the pfn is uninitialised */ static inline bool __meminit early_page_uninitialised(unsigned long pfn) { int nid = early_pfn_to_nid(pfn); if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) return true; return false; } /* * Returns true when the remaining initialisation should be deferred until * later in the boot cycle when it can be parallelised. */ static bool __meminit defer_init(int nid, unsigned long pfn, unsigned long end_pfn) { static unsigned long prev_end_pfn, nr_initialised; /* * prev_end_pfn static that contains the end of previous zone * No need to protect because called very early in boot before smp_init. */ if (prev_end_pfn != end_pfn) { prev_end_pfn = end_pfn; nr_initialised = 0; } /* Always populate low zones for address-constrained allocations */ if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) return false; if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX) return true; /* * We start only with one section of pages, more pages are added as * needed until the rest of deferred pages are initialized. */ nr_initialised++; if ((nr_initialised > PAGES_PER_SECTION) && (pfn & (PAGES_PER_SECTION - 1)) == 0) { NODE_DATA(nid)->first_deferred_pfn = pfn; return true; } return false; } #else static inline void kasan_free_nondeferred_pages(struct page *page, int order, bool init, fpi_t fpi_flags) { if (!IS_ENABLED(CONFIG_KASAN_GENERIC) && (fpi_flags & FPI_SKIP_KASAN_POISON)) return; kasan_free_pages(page, order, init); } static inline bool early_page_uninitialised(unsigned long pfn) { return false; } static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) { return false; } #endif /* Return a pointer to the bitmap storing bits affecting a block of pages */ static inline unsigned long *get_pageblock_bitmap(struct page *page, unsigned long pfn) { #ifdef CONFIG_SPARSEMEM return section_to_usemap(__pfn_to_section(pfn)); #else return page_zone(page)->pageblock_flags; #endif /* CONFIG_SPARSEMEM */ } static inline int pfn_to_bitidx(struct page *page, unsigned long pfn) { #ifdef CONFIG_SPARSEMEM pfn &= (PAGES_PER_SECTION-1); #else pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); #endif /* CONFIG_SPARSEMEM */ return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; } static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page, unsigned long pfn, unsigned long mask)