// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (C) 2009 Red Hat, Inc. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/mm.h>#include <linux/sched.h>#include <linux/sched/mm.h>#include <linux/sched/numa_balancing.h>#include <linux/highmem.h>#include <linux/hugetlb.h>#include <linux/mmu_notifier.h>#include <linux/rmap.h>#include <linux/swap.h>#include <linux/list_lru.h>#include <linux/shrinker.h>#include <linux/mm_inline.h>#include <linux/swapops.h>#include <linux/backing-dev.h>#include <linux/dax.h>#include <linux/mm_types.h>#include <linux/khugepaged.h>#include <linux/freezer.h>#include <linux/mman.h>#include <linux/memremap.h>#include <linux/pagemap.h>#include <linux/debugfs.h>#include <linux/migrate.h>#include <linux/hashtable.h>#include <linux/userfaultfd_k.h>#include <linux/page_idle.h>#include <linux/shmem_fs.h>#include <linux/oom.h>#include <linux/numa.h>#include <linux/page_owner.h>#include <linux/sched/sysctl.h>#include <linux/memory-tiers.h>#include <linux/compat.h>#include <linux/pgalloc.h>#include <linux/pgalloc_tag.h>#include <linux/pagewalk.h>#include <linux/cleanup.h> #include <asm/tlb.h>#include "internal.h"#include "swap.h" #define CREATE_TRACE_POINTS#include <trace/events/thp.h> /* * By default, transparent hugepage support is disabled in order to avoid * risking an increased memory footprint for applications that are not * guaranteed to benefit from it. When transparent hugepage support is * enabled, it is for all mappings, and khugepaged scans all mappings. * Defrag is invoked by khugepaged hugepage allocations and by page faults * for all hugepage allocations. */unsigned long transparent_hugepage_flags __read_mostly =#ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS (1<<TRANSPARENT_HUGEPAGE_FLAG)|#endif#ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|#endif (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)| (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); static struct lock_class_key deferred_split_key;static struct list_lru deferred_split_lru;static struct shrinker *deferred_split_shrinker;static unsigned long deferred_split_count(struct shrinker *shrink, struct shrink_control *sc);static unsigned long deferred_split_scan(struct shrinker *shrink, struct shrink_control *sc);static bool split_underused_thp = true; #define HUGE_ZERO_UNSET_PFN (~0UL)struct folio *huge_zero_folio __read_mostly;unsigned long huge_zero_pfn __read_mostly = HUGE_ZERO_UNSET_PFN;#ifndef CONFIG_PERSISTENT_HUGE_ZERO_FOLIOstatic atomic_t huge_zero_refcount;static DEFINE_SPINLOCK(huge_zero_lock);static struct shrinker *huge_zero_folio_shrinker;#endif unsigned long huge_anon_orders_always __read_mostly;unsigned long huge_anon_orders_madvise __read_mostly;unsigned long huge_anon_orders_inherit __read_mostly;static bool anon_orders_configured __initdata; static inline bool file_thp_enabled(struct vm_area_struct *vma){ struct inode *inode; if (!vma->vm_file) return false; inode = file_inode(vma->vm_file); if (IS_ANON_FILE(inode)) return false; if (!mapping_pmd_folio_support(vma->vm_file->f_mapping)) return false; return S_ISREG(inode->i_mode);} /* If returns true, we are unable to access the VMA's folios. */static bool vma_is_special_huge(const struct vm_area_struct *vma){ if (vma_is_dax(vma)) return false; return vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT);} unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, vm_flags_t vm_flags, enum tva_type type, unsigned long orders){ const bool smaps = type == TVA_SMAPS; const bool in_pf = type == TVA_PAGEFAULT; const bool forced_collapse = type == TVA_FORCED_COLLAPSE; unsigned long supported_orders; /* Check the intersection of requested and supported orders. */ if (vma_is_anonymous(vma)) supported_orders = THP_ORDERS_ALL_ANON; else if (vma_is_dax(vma) || vma_is_special_huge(vma)) supported_orders = THP_ORDERS_ALL_SPECIAL_DAX; else supported_orders = THP_ORDERS_ALL_FILE_DEFAULT; orders &= supported_orders; if (!orders) return 0; if (!vma->vm_mm) /* vdso */ return 0; if (thp_disabled_by_hw() || vma_thp_disabled(vma, vm_flags, forced_collapse)) return 0; /* khugepaged doesn't collapse DAX vma, but page fault is fine. */ if (vma_is_dax(vma)) return in_pf ? orders : 0; /* * khugepaged special VMA and hugetlb VMA. * Must be checked after dax since some dax mappings may have * VM_MIXEDMAP set. */ if (!in_pf && !smaps && (vm_flags & VM_NO_KHUGEPAGED)) return 0; /* * Check alignment for file vma and size for both file and anon vma by * filtering out the unsuitable orders. * * Skip the check for page fault. Huge fault does the check in fault * handlers. */ if (!in_pf) { int order = highest_order(orders); unsigned long addr; while (orders) { addr = vma->vm_end - (PAGE_SIZE << order); if (thp_vma_suitable_order(vma, addr, order)) break; order = next_order(&orders, order); } if (!orders) return 0; } /* * Enabled via shmem mount options or sysfs settings. * Must be done before hugepage flags check since shmem has its * own flags. */ if (!in_pf && shmem_file(vma->vm_file)) return orders & shmem_allowable_huge_orders(file_inode(vma->vm_file), vma, vma_start_pgoff(vma), 0, forced_collapse); if (!vma_is_anonymous(vma)) { /* * Enforce THP collapse requirements as necessary. Anonymous vmas * were already handled in thp_vma_allowable_orders(). */ if (!forced_collapse && (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) && !hugepage_global_always()))) return 0; /* * Trust that ->huge_fault() handlers know what they are doing * in fault path. */ if (((in_pf || smaps)) && vma->vm_ops->huge_fault) return orders; /* Only regular file is valid in collapse path */ if (((!in_pf || smaps)) && file_thp_enabled(vma)) return orders; return 0; } if (vma_is_temporary_stack(vma)) return 0; /* * THPeligible bit of smaps should show 1 for proper VMAs even * though anon_vma is not initialized yet. * * Allow page fault since anon_vma may be not initialized until * the first page fault. */ if (!vma->anon_vma) return (smaps || in_pf) ? orders : 0; return orders;} static struct folio *alloc_huge_zero_folio(void){ struct folio *zero_folio; zero_folio = folio_alloc((GFP_TRANSHUGE | __GFP_ZERO | __GFP_ZEROTAGS) & ~__GFP_MOVABLE, HPAGE_PMD_ORDER); if (!zero_folio) { count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); return NULL; } folio_clear_large_rmappable(zero_folio); /* Explicitly not rmappable. */ return zero_folio;} #ifdef CONFIG_PERSISTENT_HUGE_ZERO_FOLIOstatic int __init huge_zero_init(void){ huge_zero_folio = alloc_huge_zero_folio(); if (!huge_zero_folio) { pr_warn("Allocating persistent huge zero folio failed\n"); } else { huge_zero_pfn = folio_pfn(huge_zero_folio); count_vm_event(THP_ZERO_PAGE_ALLOC); } return 0;} static void __init huge_zero_shrinker_exit(void){} struct folio *mm_get_huge_zero_folio(struct mm_struct *mm){ return huge_zero_folio;} void mm_put_huge_zero_folio(struct mm_struct *mm){}#elsestatic bool get_huge_zero_folio(void){ struct folio *zero_folio; /* Paired with atomic_set_release(). */ if (likely(atomic_inc_not_zero(&huge_zero_refcount))) return true; zero_folio = alloc_huge_zero_folio(); if (unlikely(!zero_folio)) return false; /* Paired with critical section in shrink_huge_zero_folio_scan(). */ spin_lock(&huge_zero_lock); if (huge_zero_folio) { /* Somebody else already installed it. */ atomic_inc(&huge_zero_refcount); spin_unlock(&huge_zero_lock); folio_put(zero_folio); return true; } WRITE_ONCE(huge_zero_folio, zero_folio); WRITE_ONCE(huge_zero_pfn, folio_pfn(zero_folio)); /* Paired with atomic_inc_not_zero(). +1 for shrinker pin. */ atomic_set_release(&huge_zero_refcount, 2); spin_unlock(&huge_zero_lock); count_vm_event(THP_ZERO_PAGE_ALLOC); return true;} static void put_huge_zero_folio(void){ /* * Counter should never go to zero here. Only shrinker can put * last reference. */ WARN_ON_ONCE(atomic_dec_and_test(&huge_zero_refcount));} static unsigned long shrink_huge_zero_folio_count(struct shrinker *shrink, struct shrink_control *sc){ /* we can free zero page only if last reference remains */ return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;} static unsigned long shrink_huge_zero_folio_scan(struct shrinker *shrink, struct shrink_control *sc){ struct folio *zero_folio; /* Paired with critical section in get_huge_zero_folio(). */ scoped_guard(spinlock, &huge_zero_lock) { /* Paired with atomic_inc_not_zero() in get_huge_zero_folio(). */ if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) != 1) return 0; zero_folio = huge_zero_folio; VM_WARN_ON_ONCE(!zero_folio); WRITE_ONCE(huge_zero_folio, NULL); WRITE_ONCE(huge_zero_pfn, HUGE_ZERO_UNSET_PFN); } folio_put(zero_folio); return HPAGE_PMD_NR;} static int __init huge_zero_init(void){ huge_zero_folio_shrinker = shrinker_alloc(0, "thp-zero"); if (!huge_zero_folio_shrinker) { shrinker_free(deferred_split_shrinker); list_lru_destroy(&deferred_split_lru); return -ENOMEM; } huge_zero_folio_shrinker->count_objects = shrink_huge_zero_folio_count; huge_zero_folio_shrinker->scan_objects = shrink_huge_zero_folio_scan; shrinker_register(huge_zero_folio_shrinker); return 0;} static void __init huge_zero_shrinker_exit(void){ shrinker_free(huge_zero_folio_shrinker);} struct folio *mm_get_huge_zero_folio(struct mm_struct *mm){ if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm)) return READ_ONCE(huge_zero_folio); if (!get_huge_zero_folio()) return NULL; if (mm_flags_test_and_set(MMF_HUGE_ZERO_FOLIO, mm)) put_huge_zero_folio(); return READ_ONCE(huge_zero_folio);} void mm_put_huge_zero_folio(struct mm_struct *mm){ if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm)) put_huge_zero_folio();}#endif /* CONFIG_PERSISTENT_HUGE_ZERO_FOLIO */ #ifdef CONFIG_SYSFSstatic ssize_t enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf){ const char *output; if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags)) output = "[always] madvise never"; else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags)) output = "always [madvise] never"; else output = "always madvise [never]"; return sysfs_emit(buf, "%s\n", output);} enum anon_enabled_mode { ANON_ENABLED_ALWAYS = 0, ANON_ENABLED_INHERIT = 1, ANON_ENABLED_MADVISE = 2, ANON_ENABLED_NEVER = 3,}; static const char * const anon_enabled_mode_strings[] = { [ANON_ENABLED_ALWAYS] = "always", [ANON_ENABLED_INHERIT] = "inherit", [ANON_ENABLED_MADVISE] = "madvise", [ANON_ENABLED_NEVER] = "never",}; enum global_enabled_mode { GLOBAL_ENABLED_ALWAYS = 0, GLOBAL_ENABLED_MADVISE = 1, GLOBAL_ENABLED_NEVER = 2,}; static const char * const global_enabled_mode_strings[] = { [GLOBAL_ENABLED_ALWAYS] = "always", [GLOBAL_ENABLED_MADVISE] = "madvise", [GLOBAL_ENABLED_NEVER] = "never",}; static bool set_global_enabled_mode(enum global_enabled_mode mode){ static const unsigned long thp_flags[] = { TRANSPARENT_HUGEPAGE_FLAG, TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, }; enum global_enabled_mode m; bool changed = false; for (m = 0; m < ARRAY_SIZE(thp_flags); m++) { if (m == mode) changed |= !test_and_set_bit(thp_flags[m], &transparent_hugepage_flags); else changed |= test_and_clear_bit(thp_flags[m], &transparent_hugepage_flags); } return changed;} static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count){ int mode; mode = sysfs_match_string(global_enabled_mode_strings, buf); if (mode < 0) return -EINVAL; if (set_global_enabled_mode(mode)) { int err = start_stop_khugepaged(); if (err) return err; } else { /* * Recalculate watermarks even when the mode didn't * change, as the previous code always called * start_stop_khugepaged() which does this internally. */ set_recommended_min_free_kbytes(); } return count;} static struct kobj_attribute enabled_attr = __ATTR_RW(enabled); ssize_t single_hugepage_flag_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf, enum transparent_hugepage_flag flag){ return sysfs_emit(buf, "%d\n", !!test_bit(flag, &transparent_hugepage_flags));} ssize_t single_hugepage_flag_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count, enum transparent_hugepage_flag flag){ unsigned long value; int ret; ret = kstrtoul(buf, 10, &value); if (ret < 0) return ret; if (value > 1) return -EINVAL; if (value) set_bit(flag, &transparent_hugepage_flags); else clear_bit(flag, &transparent_hugepage_flags); return count;Lines 1–500 of 5160. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2009 Red Hat, Inc. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/mm.h> #include <linux/sched.h> #include <linux/sched/mm.h> #include <linux/sched/numa_balancing.h> #include <linux/highmem.h> #include <linux/hugetlb.h> #include <linux/mmu_notifier.h> #include <linux/rmap.h> #include <linux/swap.h> #include <linux/list_lru.h> #include <linux/shrinker.h> #include <linux/mm_inline.h> #include <linux/swapops.h> #include <linux/backing-dev.h> #include <linux/dax.h> #include <linux/mm_types.h> #include <linux/khugepaged.h> #include <linux/freezer.h> #include <linux/mman.h> #include <linux/memremap.h> #include <linux/pagemap.h> #include <linux/debugfs.h> #include <linux/migrate.h> #include <linux/hashtable.h> #include <linux/userfaultfd_k.h> #include <linux/page_idle.h> #include <linux/shmem_fs.h> #include <linux/oom.h> #include <linux/numa.h> #include <linux/page_owner.h> #include <linux/sched/sysctl.h> #include <linux/memory-tiers.h> #include <linux/compat.h> #include <linux/pgalloc.h> #include <linux/pgalloc_tag.h> #include <linux/pagewalk.h> #include <linux/cleanup.h> #include <asm/tlb.h> #include "internal.h" #include "swap.h" #define CREATE_TRACE_POINTS #include <trace/events/thp.h> /* * By default, transparent hugepage support is disabled in order to avoid * risking an increased memory footprint for applications that are not * guaranteed to benefit from it. When transparent hugepage support is * enabled, it is for all mappings, and khugepaged scans all mappings. * Defrag is invoked by khugepaged hugepage allocations and by page faults * for all hugepage allocations. */ unsigned long transparent_hugepage_flags __read_mostly = #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS (1<<TRANSPARENT_HUGEPAGE_FLAG)| #endif #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)| #endif (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)| (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); static struct lock_class_key deferred_split_key; static struct list_lru deferred_split_lru; static struct shrinker *deferred_split_shrinker; static unsigned long deferred_split_count(struct shrinker *shrink, struct shrink_control *sc); static unsigned long deferred_split_scan(struct shrinker *shrink, struct shrink_control *sc); static bool split_underused_thp = true; #define HUGE_ZERO_UNSET_PFN (~0UL) struct folio *huge_zero_folio __read_mostly; unsigned long huge_zero_pfn __read_mostly = HUGE_ZERO_UNSET_PFN; #ifndef CONFIG_PERSISTENT_HUGE_ZERO_FOLIO static atomic_t huge_zero_refcount; static DEFINE_SPINLOCK(huge_zero_lock); static struct shrinker *huge_zero_folio_shrinker; #endif unsigned long huge_anon_orders_always __read_mostly; unsigned long huge_anon_orders_madvise __read_mostly; unsigned long huge_anon_orders_inherit __read_mostly; static bool anon_orders_configured __initdata; static inline bool file_thp_enabled(struct vm_area_struct *vma) { struct inode *inode; if (!vma->vm_file) return false; inode = file_inode(vma->vm_file); if (IS_ANON_FILE(inode)) return false; if (!mapping_pmd_folio_support(vma->vm_file->f_mapping)) return false; return S_ISREG(inode->i_mode); } /* If returns true, we are unable to access the VMA's folios. */ static bool vma_is_special_huge(const struct vm_area_struct *vma) { if (vma_is_dax(vma)) return false; return vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT); } unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, vm_flags_t vm_flags, enum tva_type type, unsigned long orders) { const bool smaps = type == TVA_SMAPS; const bool in_pf = type == TVA_PAGEFAULT; const bool forced_collapse = type == TVA_FORCED_COLLAPSE; unsigned long supported_orders; /* Check the intersection of requested and supported orders. */ if (vma_is_anonymous(vma)) supported_orders = THP_ORDERS_ALL_ANON; else if (vma_is_dax(vma) || vma_is_special_huge(vma)) supported_orders = THP_ORDERS_ALL_SPECIAL_DAX; else supported_orders = THP_ORDERS_ALL_FILE_DEFAULT; orders &= supported_orders; if (!orders) return 0; if (!vma->vm_mm) /* vdso */ return 0; if (thp_disabled_by_hw() || vma_thp_disabled(vma, vm_flags, forced_collapse)) return 0; /* khugepaged doesn't collapse DAX vma, but page fault is fine. */ if (vma_is_dax(vma)) return in_pf ? orders : 0; /* * khugepaged special VMA and hugetlb VMA. * Must be checked after dax since some dax mappings may have * VM_MIXEDMAP set. */ if (!in_pf && !smaps && (vm_flags & VM_NO_KHUGEPAGED)) return 0; /* * Check alignment for file vma and size for both file and anon vma by * filtering out the unsuitable orders. * * Skip the check for page fault. Huge fault does the check in fault * handlers. */ if (!in_pf) { int order = highest_order(orders); unsigned long addr; while (orders) { addr = vma->vm_end - (PAGE_SIZE << order); if (thp_vma_suitable_order(vma, addr, order)) break; order = next_order(&orders, order); } if (!orders) return 0; } /* * Enabled via shmem mount options or sysfs settings. * Must be done before hugepage flags check since shmem has its * own flags. */ if (!in_pf && shmem_file(vma->vm_file)) return orders & shmem_allowable_huge_orders(file_inode(vma->vm_file), vma, vma_start_pgoff(vma), 0, forced_collapse); if (!vma_is_anonymous(vma)) { /* * Enforce THP collapse requirements as necessary. Anonymous vmas * were already handled in thp_vma_allowable_orders(). */ if (!forced_collapse && (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) && !hugepage_global_always()))) return 0; /* * Trust that ->huge_fault() handlers know what they are doing * in fault path. */ if (((in_pf || smaps)) && vma->vm_ops->huge_fault) return orders; /* Only regular file is valid in collapse path */ if (((!in_pf || smaps)) && file_thp_enabled(vma)) return orders; return 0; } if (vma_is_temporary_stack(vma)) return 0; /* * THPeligible bit of smaps should show 1 for proper VMAs even * though anon_vma is not initialized yet. * * Allow page fault since anon_vma may be not initialized until * the first page fault. */ if (!vma->anon_vma) return (smaps || in_pf) ? orders : 0; return orders; } static struct folio *alloc_huge_zero_folio(void) { struct folio *zero_folio; zero_folio = folio_alloc((GFP_TRANSHUGE | __GFP_ZERO | __GFP_ZEROTAGS) & ~__GFP_MOVABLE, HPAGE_PMD_ORDER); if (!zero_folio) { count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); return NULL; } folio_clear_large_rmappable(zero_folio); /* Explicitly not rmappable. */ return zero_folio; } #ifdef CONFIG_PERSISTENT_HUGE_ZERO_FOLIO static int __init huge_zero_init(void) { huge_zero_folio = alloc_huge_zero_folio(); if (!huge_zero_folio) { pr_warn("Allocating persistent huge zero folio failed\n"); } else { huge_zero_pfn = folio_pfn(huge_zero_folio); count_vm_event(THP_ZERO_PAGE_ALLOC); } return 0; } static void __init huge_zero_shrinker_exit(void) { } struct folio *mm_get_huge_zero_folio(struct mm_struct *mm) { return huge_zero_folio; } void mm_put_huge_zero_folio(struct mm_struct *mm) { } #else static bool get_huge_zero_folio(void) { struct folio *zero_folio; /* Paired with atomic_set_release(). */ if (likely(atomic_inc_not_zero(&huge_zero_refcount))) return true; zero_folio = alloc_huge_zero_folio(); if (unlikely(!zero_folio)) return false; /* Paired with critical section in shrink_huge_zero_folio_scan(). */ spin_lock(&huge_zero_lock); if (huge_zero_folio) { /* Somebody else already installed it. */ atomic_inc(&huge_zero_refcount); spin_unlock(&huge_zero_lock); folio_put(zero_folio); return true; } WRITE_ONCE(huge_zero_folio, zero_folio); WRITE_ONCE(huge_zero_pfn, folio_pfn(zero_folio)); /* Paired with atomic_inc_not_zero(). +1 for shrinker pin. */ atomic_set_release(&huge_zero_refcount, 2); spin_unlock(&huge_zero_lock); count_vm_event(THP_ZERO_PAGE_ALLOC); return true; } static void put_huge_zero_folio(void) { /* * Counter should never go to zero here. Only shrinker can put * last reference. */ WARN_ON_ONCE(atomic_dec_and_test(&huge_zero_refcount)); } static unsigned long shrink_huge_zero_folio_count(struct shrinker *shrink, struct shrink_control *sc) { /* we can free zero page only if last reference remains */ return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; } static unsigned long shrink_huge_zero_folio_scan(struct shrinker *shrink, struct shrink_control *sc) { struct folio *zero_folio; /* Paired with critical section in get_huge_zero_folio(). */ scoped_guard(spinlock, &huge_zero_lock) { /* Paired with atomic_inc_not_zero() in get_huge_zero_folio(). */ if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) != 1) return 0; zero_folio = huge_zero_folio; VM_WARN_ON_ONCE(!zero_folio); WRITE_ONCE(huge_zero_folio, NULL); WRITE_ONCE(huge_zero_pfn, HUGE_ZERO_UNSET_PFN); } folio_put(zero_folio); return HPAGE_PMD_NR; } static int __init huge_zero_init(void) { huge_zero_folio_shrinker = shrinker_alloc(0, "thp-zero"); if (!huge_zero_folio_shrinker) { shrinker_free(deferred_split_shrinker); list_lru_destroy(&deferred_split_lru); return -ENOMEM; } huge_zero_folio_shrinker->count_objects = shrink_huge_zero_folio_count; huge_zero_folio_shrinker->scan_objects = shrink_huge_zero_folio_scan; shrinker_register(huge_zero_folio_shrinker); return 0; } static void __init huge_zero_shrinker_exit(void) { shrinker_free(huge_zero_folio_shrinker); } struct folio *mm_get_huge_zero_folio(struct mm_struct *mm) { if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm)) return READ_ONCE(huge_zero_folio); if (!get_huge_zero_folio()) return NULL; if (mm_flags_test_and_set(MMF_HUGE_ZERO_FOLIO, mm)) put_huge_zero_folio(); return READ_ONCE(huge_zero_folio); } void mm_put_huge_zero_folio(struct mm_struct *mm) { if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm)) put_huge_zero_folio(); } #endif /* CONFIG_PERSISTENT_HUGE_ZERO_FOLIO */ #ifdef CONFIG_SYSFS static ssize_t enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { const char *output; if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags)) output = "[always] madvise never"; else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags)) output = "always [madvise] never"; else output = "always madvise [never]"; return sysfs_emit(buf, "%s\n", output); } enum anon_enabled_mode { ANON_ENABLED_ALWAYS = 0, ANON_ENABLED_INHERIT = 1, ANON_ENABLED_MADVISE = 2, ANON_ENABLED_NEVER = 3, }; static const char * const anon_enabled_mode_strings[] = { [ANON_ENABLED_ALWAYS] = "always", [ANON_ENABLED_INHERIT] = "inherit", [ANON_ENABLED_MADVISE] = "madvise", [ANON_ENABLED_NEVER] = "never", }; enum global_enabled_mode { GLOBAL_ENABLED_ALWAYS = 0, GLOBAL_ENABLED_MADVISE = 1, GLOBAL_ENABLED_NEVER = 2, }; static const char * const global_enabled_mode_strings[] = { [GLOBAL_ENABLED_ALWAYS] = "always", [GLOBAL_ENABLED_MADVISE] = "madvise", [GLOBAL_ENABLED_NEVER] = "never", }; static bool set_global_enabled_mode(enum global_enabled_mode mode) { static const unsigned long thp_flags[] = { TRANSPARENT_HUGEPAGE_FLAG, TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, }; enum global_enabled_mode m; bool changed = false; for (m = 0; m < ARRAY_SIZE(thp_flags); m++) { if (m == mode) changed |= !test_and_set_bit(thp_flags[m], &transparent_hugepage_flags); else changed |= test_and_clear_bit(thp_flags[m], &transparent_hugepage_flags); } return changed; } static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { int mode; mode = sysfs_match_string(global_enabled_mode_strings, buf); if (mode < 0) return -EINVAL; if (set_global_enabled_mode(mode)) { int err = start_stop_khugepaged(); if (err) return err; } else { /* * Recalculate watermarks even when the mode didn't * change, as the previous code always called * start_stop_khugepaged() which does this internally. */ set_recommended_min_free_kbytes(); } return count; } static struct kobj_attribute enabled_attr = __ATTR_RW(enabled); ssize_t single_hugepage_flag_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf, enum transparent_hugepage_flag flag) { return sysfs_emit(buf, "%d\n", !!test_bit(flag, &transparent_hugepage_flags)); } ssize_t single_hugepage_flag_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count, enum transparent_hugepage_flag flag) { unsigned long value; int ret; ret = kstrtoul(buf, 10, &value); if (ret < 0) return ret; if (value > 1) return -EINVAL; if (value) set_bit(flag, &transparent_hugepage_flags); else clear_bit(flag, &transparent_hugepage_flags); return count;