// SPDX-License-Identifier: GPL-2.0-only/* * * Manages the free list, the system allocates free pages here. * Note that kmalloc() lives in slab.c * * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds * Swap reorganised 29.12.95, Stephen Tweedie * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) */ #include <linux/stddef.h>#include <linux/mm.h>#include <linux/highmem.h>#include <linux/interrupt.h>#include <linux/jiffies.h>#include <linux/compiler.h>#include <linux/kernel.h>#include <linux/kasan.h>#include <linux/kmsan.h>#include <linux/module.h>#include <linux/suspend.h>#include <linux/ratelimit.h>#include <linux/oom.h>#include <linux/topology.h>#include <linux/sysctl.h>#include <linux/cpu.h>#include <linux/cpuset.h>#include <linux/folio_batch.h>#include <linux/memory_hotplug.h>#include <linux/nodemask.h>#include <linux/vmstat.h>#include <linux/fault-inject.h>#include <linux/compaction.h>#include <trace/events/kmem.h>#include <trace/events/oom.h>#include <linux/prefetch.h>#include <linux/mm_inline.h>#include <linux/mmu_notifier.h>#include <linux/migrate.h>#include <linux/sched/mm.h>#include <linux/page_owner.h>#include <linux/page_table_check.h>#include <linux/memcontrol.h>#include <linux/ftrace.h>#include <linux/lockdep.h>#include <linux/psi.h>#include <linux/khugepaged.h>#include <linux/delayacct.h>#include <linux/cacheinfo.h>#include <linux/pgalloc_tag.h>#include <asm/div64.h>#include "internal.h"#include "mm_init.h"#include "page_alloc.h"#include "shuffle.h"#include "page_reporting.h" /* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */typedef int __bitwise fpi_t; /* No special request */#define FPI_NONE ((__force fpi_t)0) /* * Skip free page reporting notification for the (possibly merged) page. * This does not hinder free page reporting from grabbing the page, * reporting it and marking it "reported" - it only skips notifying * the free page reporting infrastructure about a newly freed page. For * example, used when temporarily pulling a page from a freelist and * putting it back unmodified. */#define FPI_SKIP_REPORT_NOTIFY ((__force fpi_t)BIT(0)) /* * Place the (possibly merged) page to the tail of the freelist. Will ignore * page shuffling (relevant code - e.g., memory onlining - is expected to * shuffle the whole zone). * * Note: No code should rely on this flag for correctness - it's purely * to allow for optimizations when handing back either fresh pages * (memory onlining) or untouched pages (page isolation, free page * reporting). */#define FPI_TO_TAIL ((__force fpi_t)BIT(1)) /* Free the page without taking locks. Rely on trylock only. */#define FPI_NOLOCK ((__force fpi_t)BIT(2)) /* free_pages_prepare() has already been called for page(s) being freed. */#define FPI_PREPARED ((__force fpi_t)BIT(3)) /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */static DEFINE_MUTEX(pcp_batch_high_lock);#define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8) /* * Locking a pcp requires a PCP lookup followed by a spinlock. To avoid * a migration causing the wrong PCP to be locked and remote memory being * potentially allocated, pin the task to the CPU for the lookup+lock. * preempt_disable is used on !RT because it is faster than migrate_disable. * migrate_disable is used on RT because otherwise RT spinlock usage is * interfered with and a high priority task cannot preempt the allocator. */#ifndef CONFIG_PREEMPT_RT#define pcpu_task_pin() preempt_disable()#define pcpu_task_unpin() preempt_enable()#else#define pcpu_task_pin() migrate_disable()#define pcpu_task_unpin() migrate_enable()#endif /* * A helper to lookup and trylock pcp with embedded spinlock. * The return value should be used with the unlock helper. * NULL return value means the trylock failed. */#ifdef CONFIG_SMP#define pcp_spin_trylock(ptr) \({ \ struct per_cpu_pages *_ret; \ pcpu_task_pin(); \ _ret = this_cpu_ptr(ptr); \ if (!spin_trylock(&_ret->lock)) { \ pcpu_task_unpin(); \ _ret = NULL; \ } \ _ret; \}) #define pcp_spin_unlock(ptr) \({ \ spin_unlock(&ptr->lock); \ pcpu_task_unpin(); \}) /* * On CONFIG_SMP=n the UP implementation of spin_trylock() never fails and thus * is not compatible with our locking scheme. However we do not need pcp for * scalability in the first place, so just make all the trylocks fail and take * the slow path unconditionally. */#else#define pcp_spin_trylock(ptr) \ NULL #define pcp_spin_unlock(ptr) \ BUG_ON(1)#endif /* * In some cases we do not need to pin the task to the CPU because we are * already given a specific cpu's pcp pointer. */#define pcp_spin_lock_nopin(ptr) \ spin_lock(&(ptr)->lock)#define pcp_spin_unlock_nopin(ptr) \ spin_unlock(&(ptr)->lock) #ifdef CONFIG_USE_PERCPU_NUMA_NODE_IDDEFINE_PER_CPU(int, numa_node);EXPORT_PER_CPU_SYMBOL(numa_node);#endif #ifdef CONFIG_NUMADEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);#endif #ifdef CONFIG_HAVE_MEMORYLESS_NODES/* * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() * defined in <linux/topology.h>. */DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */EXPORT_PER_CPU_SYMBOL(_numa_mem_);#endif static DEFINE_MUTEX(pcpu_drain_mutex); #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPYvolatile unsigned long latent_entropy __latent_entropy;EXPORT_SYMBOL(latent_entropy);#endif /* * Array of node states. */nodemask_t node_states[NR_NODE_STATES] __read_mostly = { [N_POSSIBLE] = NODE_MASK_ALL, [N_ONLINE] = { { [0] = 1UL } },#ifndef CONFIG_NUMA [N_NORMAL_MEMORY] = { { [0] = 1UL } },#ifdef CONFIG_HIGHMEM [N_HIGH_MEMORY] = { { [0] = 1UL } },#endif [N_MEMORY] = { { [0] = 1UL } }, [N_CPU] = { { [0] = 1UL } },#endif /* NUMA */};EXPORT_SYMBOL(node_states); gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLEunsigned int pageblock_order __read_mostly;#endif static void __free_pages_ok(struct page *page, unsigned int order, fpi_t fpi_flags);static void reserve_highatomic_pageblock(struct page *page, int order, struct zone *zone); /* * results with 256, 32 in the lowmem_reserve sysctl: * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) * 1G machine -> (16M dma, 784M normal, 224M high) * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA * * TBD: should special case ZONE_DMA32 machines here - in those we normally * don't need any ZONE_NORMAL reservation */static int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = {#ifdef CONFIG_ZONE_DMA [ZONE_DMA] = 256,#endif#ifdef CONFIG_ZONE_DMA32 [ZONE_DMA32] = 256,#endif [ZONE_NORMAL] = 32,#ifdef CONFIG_HIGHMEM [ZONE_HIGHMEM] = 0,#endif [ZONE_MOVABLE] = 0,}; char * const zone_names[MAX_NR_ZONES] = {#ifdef CONFIG_ZONE_DMA "DMA",#endif#ifdef CONFIG_ZONE_DMA32 "DMA32",#endif "Normal",#ifdef CONFIG_HIGHMEM "HighMem",#endif "Movable",#ifdef CONFIG_ZONE_DEVICE "Device",#endif}; const char * const migratetype_names[MIGRATE_TYPES] = { "Unmovable", "Movable", "Reclaimable", "HighAtomic",#ifdef CONFIG_CMA "CMA",#endif#ifdef CONFIG_MEMORY_ISOLATION "Isolate",#endif}; int min_free_kbytes = 1024;int user_min_free_kbytes = -1;static int watermark_boost_factor __read_mostly = 15000;static int watermark_scale_factor = 10;int defrag_mode; /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */int movable_zone;EXPORT_SYMBOL(movable_zone); #if MAX_NUMNODES > 1unsigned int nr_node_ids __read_mostly = MAX_NUMNODES;unsigned int nr_online_nodes __read_mostly = 1;EXPORT_SYMBOL(nr_node_ids);EXPORT_SYMBOL(nr_online_nodes);#endif /* * When page allocations stall for longer than a threshold, * ALLOC_STALL_WARN_MSECS, leave a warning in the kernel log. Only one warning * will be printed during this duration for the entire system. */#define ALLOC_STALL_WARN_MSECS (10 * 1000UL)static unsigned long alloc_stall_warn_jiffies = INITIAL_JIFFIES; static bool page_contains_unaccepted(struct page *page, unsigned int order);static bool cond_accept_memory(struct zone *zone, unsigned int order, int alloc_flags);static bool __free_unaccepted(struct page *page); int page_group_by_mobility_disabled __read_mostly; #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT/* * During boot we initialize deferred pages on-demand, as needed, but once * page_alloc_init_late() has finished, the deferred pages are all initialized, * and we can permanently disable that path. */DEFINE_STATIC_KEY_TRUE(deferred_pages); /* * deferred_grow_zone() is __init, but it is called from * get_page_from_freelist() during early boot until deferred_pages permanently * disables this call. This is why we have refdata wrapper to avoid warning, * and to ensure that the function body gets unloaded. */static bool __ref_deferred_grow_zone(struct zone *zone, unsigned int order){ return deferred_grow_zone(zone, order);}#elsestatic inline bool _deferred_grow_zone(struct zone *zone, unsigned int order){ return false;}#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ /* Return a pointer to the bitmap storing bits affecting a block of pages */static inline unsigned long *get_pageblock_bitmap(const struct page *page, unsigned long pfn){#ifdef CONFIG_SPARSEMEM return section_to_usemap(__pfn_to_section(pfn));#else return page_zone(page)->pageblock_flags;#endif /* CONFIG_SPARSEMEM */} static inline int pfn_to_bitidx(const struct page *page, unsigned long pfn){#ifdef CONFIG_SPARSEMEM pfn &= (PAGES_PER_SECTION-1);#else pfn = pfn - pageblock_start_pfn(page_zone(page)->zone_start_pfn);#endif /* CONFIG_SPARSEMEM */ return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;} static __always_inline bool is_standalone_pb_bit(enum pageblock_bits pb_bit){ return pb_bit >= PB_compact_skip && pb_bit < __NR_PAGEBLOCK_BITS;} static __always_inline voidget_pfnblock_bitmap_bitidx(const struct page *page, unsigned long pfn, unsigned long **bitmap_word, unsigned long *bitidx){ unsigned long *bitmap; unsigned long word_bitidx; #ifdef CONFIG_MEMORY_ISOLATION BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 8);#else BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);#endif BUILD_BUG_ON(__MIGRATE_TYPE_END > PAGEBLOCK_MIGRATETYPE_MASK); VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); bitmap = get_pageblock_bitmap(page, pfn); *bitidx = pfn_to_bitidx(page, pfn); word_bitidx = *bitidx / BITS_PER_LONG; *bitidx &= (BITS_PER_LONG - 1); *bitmap_word = &bitmap[word_bitidx];} /** * __get_pfnblock_flags_mask - Return the requested group of flags for * a pageblock_nr_pages block of pages * @page: The page within the block of interest * @pfn: The target page frame number * @mask: mask of bits that the caller is interested in * * Return: pageblock_bits flags */static unsigned long __get_pfnblock_flags_mask(const struct page *page, unsigned long pfn, unsigned long mask){ unsigned long *bitmap_word; unsigned long bitidx; unsigned long word; get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx); /* * This races, without locks, with set_pfnblock_migratetype(). Ensure * a consistent read of the memory array, so that results, even though * racy, are not corrupted. */ word = READ_ONCE(*bitmap_word); return (word >> bitidx) & mask;} /** * get_pfnblock_bit - Check if a standalone bit of a pageblock is set * @page: The page within the block of interest * @pfn: The target page frame number * @pb_bit: pageblock bit to check * * Return: true if the bit is set, otherwise false */bool get_pfnblock_bit(const struct page *page, unsigned long pfn, enum pageblock_bits pb_bit){ unsigned long *bitmap_word; unsigned long bitidx; if (WARN_ON_ONCE(!is_standalone_pb_bit(pb_bit))) return false; get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx); return test_bit(bitidx + pb_bit, bitmap_word);} /** * get_pfnblock_migratetype - Return the migratetype of a pageblock * @page: The page within the block of interest * @pfn: The target page frame number * * Return: The migratetype of the pageblock * * Use get_pfnblock_migratetype() if caller already has both @page and @pfn * to save a call to page_to_pfn(). */enum migratetypeget_pfnblock_migratetype(const struct page *page, unsigned long pfn){ unsigned long mask = PAGEBLOCK_MIGRATETYPE_MASK | PAGEBLOCK_ISO_MASK; unsigned long flags; flags = __get_pfnblock_flags_mask(page, pfn, mask); #ifdef CONFIG_MEMORY_ISOLATION if (flags & BIT(PB_migrate_isolate)) return MIGRATE_ISOLATE;#endif return flags & PAGEBLOCK_MIGRATETYPE_MASK;} /** * __set_pfnblock_flags_mask - Set the requested group of flags for * a pageblock_nr_pages block of pages * @page: The page within the block of interest * @pfn: The target page frame number * @flags: The flags to set * @mask: mask of bits that the caller is interested in */static void __set_pfnblock_flags_mask(struct page *page, unsigned long pfn, unsigned long flags, unsigned long mask){ unsigned long *bitmap_word; unsigned long bitidx; unsigned long word; get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx); mask <<= bitidx; flags <<= bitidx; word = READ_ONCE(*bitmap_word); do { } while (!try_cmpxchg(bitmap_word, &word, (word & ~mask) | flags));} /** * set_pfnblock_bit - Set a standalone bit of a pageblock * @page: The page within the block of interest * @pfn: The target page frame number * @pb_bit: pageblock bit to set */void set_pfnblock_bit(const struct page *page, unsigned long pfn, enum pageblock_bits pb_bit){ unsigned long *bitmap_word; unsigned long bitidx; if (WARN_ON_ONCE(!is_standalone_pb_bit(pb_bit))) return; get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx); set_bit(bitidx + pb_bit, bitmap_word);} Lines 1–500 of 8074. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * * Manages the free list, the system allocates free pages here. * Note that kmalloc() lives in slab.c * * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds * Swap reorganised 29.12.95, Stephen Tweedie * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 * (lots of bits borrowed from Ingo Molnar & Andrew Morton) */ #include <linux/stddef.h> #include <linux/mm.h> #include <linux/highmem.h> #include <linux/interrupt.h> #include <linux/jiffies.h> #include <linux/compiler.h> #include <linux/kernel.h> #include <linux/kasan.h> #include <linux/kmsan.h> #include <linux/module.h> #include <linux/suspend.h> #include <linux/ratelimit.h> #include <linux/oom.h> #include <linux/topology.h> #include <linux/sysctl.h> #include <linux/cpu.h> #include <linux/cpuset.h> #include <linux/folio_batch.h> #include <linux/memory_hotplug.h> #include <linux/nodemask.h> #include <linux/vmstat.h> #include <linux/fault-inject.h> #include <linux/compaction.h> #include <trace/events/kmem.h> #include <trace/events/oom.h> #include <linux/prefetch.h> #include <linux/mm_inline.h> #include <linux/mmu_notifier.h> #include <linux/migrate.h> #include <linux/sched/mm.h> #include <linux/page_owner.h> #include <linux/page_table_check.h> #include <linux/memcontrol.h> #include <linux/ftrace.h> #include <linux/lockdep.h> #include <linux/psi.h> #include <linux/khugepaged.h> #include <linux/delayacct.h> #include <linux/cacheinfo.h> #include <linux/pgalloc_tag.h> #include <asm/div64.h> #include "internal.h" #include "mm_init.h" #include "page_alloc.h" #include "shuffle.h" #include "page_reporting.h" /* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */ typedef int __bitwise fpi_t; /* No special request */ #define FPI_NONE ((__force fpi_t)0) /* * Skip free page reporting notification for the (possibly merged) page. * This does not hinder free page reporting from grabbing the page, * reporting it and marking it "reported" - it only skips notifying * the free page reporting infrastructure about a newly freed page. For * example, used when temporarily pulling a page from a freelist and * putting it back unmodified. */ #define FPI_SKIP_REPORT_NOTIFY ((__force fpi_t)BIT(0)) /* * Place the (possibly merged) page to the tail of the freelist. Will ignore * page shuffling (relevant code - e.g., memory onlining - is expected to * shuffle the whole zone). * * Note: No code should rely on this flag for correctness - it's purely * to allow for optimizations when handing back either fresh pages * (memory onlining) or untouched pages (page isolation, free page * reporting). */ #define FPI_TO_TAIL ((__force fpi_t)BIT(1)) /* Free the page without taking locks. Rely on trylock only. */ #define FPI_NOLOCK ((__force fpi_t)BIT(2)) /* free_pages_prepare() has already been called for page(s) being freed. */ #define FPI_PREPARED ((__force fpi_t)BIT(3)) /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ static DEFINE_MUTEX(pcp_batch_high_lock); #define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8) /* * Locking a pcp requires a PCP lookup followed by a spinlock. To avoid * a migration causing the wrong PCP to be locked and remote memory being * potentially allocated, pin the task to the CPU for the lookup+lock. * preempt_disable is used on !RT because it is faster than migrate_disable. * migrate_disable is used on RT because otherwise RT spinlock usage is * interfered with and a high priority task cannot preempt the allocator. */ #ifndef CONFIG_PREEMPT_RT #define pcpu_task_pin() preempt_disable() #define pcpu_task_unpin() preempt_enable() #else #define pcpu_task_pin() migrate_disable() #define pcpu_task_unpin() migrate_enable() #endif /* * A helper to lookup and trylock pcp with embedded spinlock. * The return value should be used with the unlock helper. * NULL return value means the trylock failed. */ #ifdef CONFIG_SMP #define pcp_spin_trylock(ptr) \ ({ \ struct per_cpu_pages *_ret; \ pcpu_task_pin(); \ _ret = this_cpu_ptr(ptr); \ if (!spin_trylock(&_ret->lock)) { \ pcpu_task_unpin(); \ _ret = NULL; \ } \ _ret; \ }) #define pcp_spin_unlock(ptr) \ ({ \ spin_unlock(&ptr->lock); \ pcpu_task_unpin(); \ }) /* * On CONFIG_SMP=n the UP implementation of spin_trylock() never fails and thus * is not compatible with our locking scheme. However we do not need pcp for * scalability in the first place, so just make all the trylocks fail and take * the slow path unconditionally. */ #else #define pcp_spin_trylock(ptr) \ NULL #define pcp_spin_unlock(ptr) \ BUG_ON(1) #endif /* * In some cases we do not need to pin the task to the CPU because we are * already given a specific cpu's pcp pointer. */ #define pcp_spin_lock_nopin(ptr) \ spin_lock(&(ptr)->lock) #define pcp_spin_unlock_nopin(ptr) \ spin_unlock(&(ptr)->lock) #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID DEFINE_PER_CPU(int, numa_node); EXPORT_PER_CPU_SYMBOL(numa_node); #endif #ifdef CONFIG_NUMA DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); #endif #ifdef CONFIG_HAVE_MEMORYLESS_NODES /* * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() * defined in <linux/topology.h>. */ DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ EXPORT_PER_CPU_SYMBOL(_numa_mem_); #endif static DEFINE_MUTEX(pcpu_drain_mutex); #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY volatile unsigned long latent_entropy __latent_entropy; EXPORT_SYMBOL(latent_entropy); #endif /* * Array of node states. */ nodemask_t node_states[NR_NODE_STATES] __read_mostly = { [N_POSSIBLE] = NODE_MASK_ALL, [N_ONLINE] = { { [0] = 1UL } }, #ifndef CONFIG_NUMA [N_NORMAL_MEMORY] = { { [0] = 1UL } }, #ifdef CONFIG_HIGHMEM [N_HIGH_MEMORY] = { { [0] = 1UL } }, #endif [N_MEMORY] = { { [0] = 1UL } }, [N_CPU] = { { [0] = 1UL } }, #endif /* NUMA */ }; EXPORT_SYMBOL(node_states); gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE unsigned int pageblock_order __read_mostly; #endif static void __free_pages_ok(struct page *page, unsigned int order, fpi_t fpi_flags); static void reserve_highatomic_pageblock(struct page *page, int order, struct zone *zone); /* * results with 256, 32 in the lowmem_reserve sysctl: * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) * 1G machine -> (16M dma, 784M normal, 224M high) * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA * * TBD: should special case ZONE_DMA32 machines here - in those we normally * don't need any ZONE_NORMAL reservation */ static int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = { #ifdef CONFIG_ZONE_DMA [ZONE_DMA] = 256, #endif #ifdef CONFIG_ZONE_DMA32 [ZONE_DMA32] = 256, #endif [ZONE_NORMAL] = 32, #ifdef CONFIG_HIGHMEM [ZONE_HIGHMEM] = 0, #endif [ZONE_MOVABLE] = 0, }; char * const zone_names[MAX_NR_ZONES] = { #ifdef CONFIG_ZONE_DMA "DMA", #endif #ifdef CONFIG_ZONE_DMA32 "DMA32", #endif "Normal", #ifdef CONFIG_HIGHMEM "HighMem", #endif "Movable", #ifdef CONFIG_ZONE_DEVICE "Device", #endif }; const char * const migratetype_names[MIGRATE_TYPES] = { "Unmovable", "Movable", "Reclaimable", "HighAtomic", #ifdef CONFIG_CMA "CMA", #endif #ifdef CONFIG_MEMORY_ISOLATION "Isolate", #endif }; int min_free_kbytes = 1024; int user_min_free_kbytes = -1; static int watermark_boost_factor __read_mostly = 15000; static int watermark_scale_factor = 10; int defrag_mode; /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ int movable_zone; EXPORT_SYMBOL(movable_zone); #if MAX_NUMNODES > 1 unsigned int nr_node_ids __read_mostly = MAX_NUMNODES; unsigned int nr_online_nodes __read_mostly = 1; EXPORT_SYMBOL(nr_node_ids); EXPORT_SYMBOL(nr_online_nodes); #endif /* * When page allocations stall for longer than a threshold, * ALLOC_STALL_WARN_MSECS, leave a warning in the kernel log. Only one warning * will be printed during this duration for the entire system. */ #define ALLOC_STALL_WARN_MSECS (10 * 1000UL) static unsigned long alloc_stall_warn_jiffies = INITIAL_JIFFIES; static bool page_contains_unaccepted(struct page *page, unsigned int order); static bool cond_accept_memory(struct zone *zone, unsigned int order, int alloc_flags); static bool __free_unaccepted(struct page *page); int page_group_by_mobility_disabled __read_mostly; #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT /* * During boot we initialize deferred pages on-demand, as needed, but once * page_alloc_init_late() has finished, the deferred pages are all initialized, * and we can permanently disable that path. */ DEFINE_STATIC_KEY_TRUE(deferred_pages); /* * deferred_grow_zone() is __init, but it is called from * get_page_from_freelist() during early boot until deferred_pages permanently * disables this call. This is why we have refdata wrapper to avoid warning, * and to ensure that the function body gets unloaded. */ static bool __ref _deferred_grow_zone(struct zone *zone, unsigned int order) { return deferred_grow_zone(zone, order); } #else static inline bool _deferred_grow_zone(struct zone *zone, unsigned int order) { return false; } #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ /* Return a pointer to the bitmap storing bits affecting a block of pages */ static inline unsigned long *get_pageblock_bitmap(const struct page *page, unsigned long pfn) { #ifdef CONFIG_SPARSEMEM return section_to_usemap(__pfn_to_section(pfn)); #else return page_zone(page)->pageblock_flags; #endif /* CONFIG_SPARSEMEM */ } static inline int pfn_to_bitidx(const struct page *page, unsigned long pfn) { #ifdef CONFIG_SPARSEMEM pfn &= (PAGES_PER_SECTION-1); #else pfn = pfn - pageblock_start_pfn(page_zone(page)->zone_start_pfn); #endif /* CONFIG_SPARSEMEM */ return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; } static __always_inline bool is_standalone_pb_bit(enum pageblock_bits pb_bit) { return pb_bit >= PB_compact_skip && pb_bit < __NR_PAGEBLOCK_BITS; } static __always_inline void get_pfnblock_bitmap_bitidx(const struct page *page, unsigned long pfn, unsigned long **bitmap_word, unsigned long *bitidx) { unsigned long *bitmap; unsigned long word_bitidx; #ifdef CONFIG_MEMORY_ISOLATION BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 8); #else BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); #endif BUILD_BUG_ON(__MIGRATE_TYPE_END > PAGEBLOCK_MIGRATETYPE_MASK); VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); bitmap = get_pageblock_bitmap(page, pfn); *bitidx = pfn_to_bitidx(page, pfn); word_bitidx = *bitidx / BITS_PER_LONG; *bitidx &= (BITS_PER_LONG - 1); *bitmap_word = &bitmap[word_bitidx]; } /** * __get_pfnblock_flags_mask - Return the requested group of flags for * a pageblock_nr_pages block of pages * @page: The page within the block of interest * @pfn: The target page frame number * @mask: mask of bits that the caller is interested in * * Return: pageblock_bits flags */ static unsigned long __get_pfnblock_flags_mask(const struct page *page, unsigned long pfn, unsigned long mask) { unsigned long *bitmap_word; unsigned long bitidx; unsigned long word; get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx); /* * This races, without locks, with set_pfnblock_migratetype(). Ensure * a consistent read of the memory array, so that results, even though * racy, are not corrupted. */ word = READ_ONCE(*bitmap_word); return (word >> bitidx) & mask; } /** * get_pfnblock_bit - Check if a standalone bit of a pageblock is set * @page: The page within the block of interest * @pfn: The target page frame number * @pb_bit: pageblock bit to check * * Return: true if the bit is set, otherwise false */ bool get_pfnblock_bit(const struct page *page, unsigned long pfn, enum pageblock_bits pb_bit) { unsigned long *bitmap_word; unsigned long bitidx; if (WARN_ON_ONCE(!is_standalone_pb_bit(pb_bit))) return false; get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx); return test_bit(bitidx + pb_bit, bitmap_word); } /** * get_pfnblock_migratetype - Return the migratetype of a pageblock * @page: The page within the block of interest * @pfn: The target page frame number * * Return: The migratetype of the pageblock * * Use get_pfnblock_migratetype() if caller already has both @page and @pfn * to save a call to page_to_pfn(). */ enum migratetype get_pfnblock_migratetype(const struct page *page, unsigned long pfn) { unsigned long mask = PAGEBLOCK_MIGRATETYPE_MASK | PAGEBLOCK_ISO_MASK; unsigned long flags; flags = __get_pfnblock_flags_mask(page, pfn, mask); #ifdef CONFIG_MEMORY_ISOLATION if (flags & BIT(PB_migrate_isolate)) return MIGRATE_ISOLATE; #endif return flags & PAGEBLOCK_MIGRATETYPE_MASK; } /** * __set_pfnblock_flags_mask - Set the requested group of flags for * a pageblock_nr_pages block of pages * @page: The page within the block of interest * @pfn: The target page frame number * @flags: The flags to set * @mask: mask of bits that the caller is interested in */ static void __set_pfnblock_flags_mask(struct page *page, unsigned long pfn, unsigned long flags, unsigned long mask) { unsigned long *bitmap_word; unsigned long bitidx; unsigned long word; get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx); mask <<= bitidx; flags <<= bitidx; word = READ_ONCE(*bitmap_word); do { } while (!try_cmpxchg(bitmap_word, &word, (word & ~mask) | flags)); } /** * set_pfnblock_bit - Set a standalone bit of a pageblock * @page: The page within the block of interest * @pfn: The target page frame number * @pb_bit: pageblock bit to set */ void set_pfnblock_bit(const struct page *page, unsigned long pfn, enum pageblock_bits pb_bit) { unsigned long *bitmap_word; unsigned long bitidx; if (WARN_ON_ONCE(!is_standalone_pb_bit(pb_bit))) return; get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx); set_bit(bitidx + pb_bit, bitmap_word); }