// SPDX-License-Identifier: GPL-2.0-or-later/* memcontrol.c - Memory Controller * * Copyright IBM Corporation, 2007 * Author Balbir Singh <balbir@linux.vnet.ibm.com> * * Copyright 2007 OpenVZ SWsoft Inc * Author: Pavel Emelianov <xemul@openvz.org> * * Memory thresholds * Copyright (C) 2009 Nokia Corporation * Author: Kirill A. Shutemov * * Kernel Memory Controller * Copyright (C) 2012 Parallels Inc. and Google Inc. * Authors: Glauber Costa and Suleiman Souhlal * * Native page reclaim * Charge lifetime sanitation * Lockless page tracking & accounting * Unified hierarchy configuration model * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner * * Per memcg lru locking * Copyright (C) 2020 Alibaba, Inc, Alex Shi */ #include <linux/cgroup-defs.h>#include <linux/page_counter.h>#include <linux/memcontrol.h>#include <linux/cgroup.h>#include <linux/cpuset.h>#include <linux/sched/mm.h>#include <linux/shmem_fs.h>#include <linux/hugetlb.h>#include <linux/pagemap.h>#include <linux/folio_batch.h>#include <linux/vm_event_item.h>#include <linux/smp.h>#include <linux/page-flags.h>#include <linux/backing-dev.h>#include <linux/bit_spinlock.h>#include <linux/rcupdate.h>#include <linux/limits.h>#include <linux/export.h>#include <linux/list.h>#include <linux/mutex.h>#include <linux/rbtree.h>#include <linux/slab.h>#include <linux/swapops.h>#include <linux/spinlock.h>#include <linux/fs.h>#include <linux/seq_file.h>#include <linux/vmpressure.h>#include <linux/memremap.h>#include <linux/mm_inline.h>#include <linux/cpu.h>#include <linux/oom.h>#include <linux/lockdep.h>#include <linux/resume_user_mode.h>#include <linux/psi.h>#include <linux/seq_buf.h>#include <linux/sched/isolation.h>#include <linux/kmemleak.h>#include "internal.h"#include "swap.h"#include "swap_table.h"#include <net/sock.h>#include <net/ip.h>#include "slab.h"#include "memcontrol-v1.h" #include <linux/uaccess.h> #define CREATE_TRACE_POINTS#include <trace/events/memcg.h>#undef CREATE_TRACE_POINTS #include <trace/events/vmscan.h> struct cgroup_subsys memory_cgrp_subsys __read_mostly;EXPORT_SYMBOL(memory_cgrp_subsys); struct mem_cgroup *root_mem_cgroup __read_mostly;EXPORT_SYMBOL(root_mem_cgroup); /* Active memory cgroup to use from an interrupt context */DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg);EXPORT_PER_CPU_SYMBOL_GPL(int_active_memcg); /* Socket memory accounting disabled? */static bool cgroup_memory_nosocket __ro_after_init; /* Kernel memory accounting disabled? */static bool cgroup_memory_nokmem __ro_after_init; /* BPF memory accounting disabled? */static bool cgroup_memory_nobpf __ro_after_init; static struct workqueue_struct *memcg_wq __ro_after_init; static struct kmem_cache *memcg_cachep;static struct kmem_cache *memcg_pn_cachep; #ifdef CONFIG_CGROUP_WRITEBACKstatic DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);#endif static inline bool task_is_dying(void){ return tsk_is_oom_victim(current) || fatal_signal_pending(current) || (current->flags & PF_EXITING);} /* Some nice accessors for the vmpressure. */struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg){ if (!memcg) memcg = root_mem_cgroup; return &memcg->vmpressure;} struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr){ return container_of(vmpr, struct mem_cgroup, vmpressure);} #define SEQ_BUF_SIZE SZ_4K#define CURRENT_OBJCG_UPDATE_BIT 0#define CURRENT_OBJCG_UPDATE_FLAG (1UL << CURRENT_OBJCG_UPDATE_BIT) static DEFINE_SPINLOCK(objcg_lock); bool mem_cgroup_kmem_disabled(void){ return cgroup_memory_nokmem;} static void memcg_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages); static void memcg_uncharge_kmem(struct mem_cgroup *memcg, unsigned int nr_pages){ mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); memcg1_account_kmem(memcg, -nr_pages); if (!mem_cgroup_is_root(memcg)) memcg_uncharge(memcg, nr_pages);} static void obj_cgroup_release(struct percpu_ref *ref){ struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt); unsigned int nr_bytes; unsigned int nr_pages; unsigned long flags; /* * At this point all allocated objects are freed, and * objcg->nr_charged_bytes can't have an arbitrary byte value. * However, it can be PAGE_SIZE or (x * PAGE_SIZE). * * The following sequence can lead to it: * 1) CPU0: objcg cached in one of stock->cached[i] * 2) CPU1: we do a small allocation (e.g. 92 bytes), * PAGE_SIZE bytes are charged * 3) CPU1: a process from another memcg is allocating something, * the stock if flushed, * objcg->nr_charged_bytes = PAGE_SIZE - 92 * 4) CPU0: we do release this object, * 92 bytes are added to stock->nr_bytes[i] * 5) CPU0: stock is flushed, * 92 bytes are added to objcg->nr_charged_bytes * * In the result, nr_charged_bytes == PAGE_SIZE. * This page will be uncharged in obj_cgroup_release(). */ nr_bytes = atomic_read(&objcg->nr_charged_bytes); WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1)); nr_pages = nr_bytes >> PAGE_SHIFT; if (nr_pages) { struct mem_cgroup *memcg; memcg = get_mem_cgroup_from_objcg(objcg); memcg_uncharge_kmem(memcg, nr_pages); mem_cgroup_put(memcg); } spin_lock_irqsave(&objcg_lock, flags); list_del(&objcg->list); spin_unlock_irqrestore(&objcg_lock, flags); percpu_ref_exit(ref); kfree_rcu(objcg, rcu);} static struct obj_cgroup *obj_cgroup_alloc(void){ struct obj_cgroup *objcg; int ret; objcg = kzalloc_obj(struct obj_cgroup); if (!objcg) return NULL; ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0, GFP_KERNEL); if (ret) { kfree(objcg); return NULL; } INIT_LIST_HEAD(&objcg->list); return objcg;} static inline struct obj_cgroup *__memcg_reparent_objcgs(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid){ struct obj_cgroup *objcg, *iter; struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid]; struct mem_cgroup_per_node *parent_pn = parent->nodeinfo[nid]; objcg = rcu_replace_pointer(pn->objcg, NULL, true); /* 1) Ready to reparent active objcg. */ list_add(&objcg->list, &pn->objcg_list); /* 2) Reparent active objcg and already reparented objcgs to parent. */ list_for_each_entry(iter, &pn->objcg_list, list) WRITE_ONCE(iter->memcg, parent); /* 3) Move already reparented objcgs to the parent's list */ list_splice(&pn->objcg_list, &parent_pn->objcg_list); return objcg;} #ifdef CONFIG_MEMCG_V1static void __mem_cgroup_flush_stats(struct mem_cgroup *memcg, bool force); static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent){ if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) return; /* * Reparent stats exposed non-hierarchically. Flush @memcg's stats first * to read its stats accurately , and conservatively flush @parent's * stats after reparenting to avoid hiding a potentially large stat * update (e.g. from callers of mem_cgroup_flush_stats_ratelimited()). */ __mem_cgroup_flush_stats(memcg, true); /* The following counts are all non-hierarchical and need to be reparented. */ reparent_memcg1_state_local(memcg, parent); reparent_memcg1_lruvec_state_local(memcg, parent); __mem_cgroup_flush_stats(parent, true);}#elsestatic inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent){}#endif static inline void reparent_locks(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid){ spin_lock_irq(&objcg_lock); spin_lock_nested(&mem_cgroup_lruvec(memcg, NODE_DATA(nid))->lru_lock, 1); spin_lock_nested(&mem_cgroup_lruvec(parent, NODE_DATA(nid))->lru_lock, 2);} static inline void reparent_unlocks(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid){ spin_unlock(&mem_cgroup_lruvec(parent, NODE_DATA(nid))->lru_lock); spin_unlock(&mem_cgroup_lruvec(memcg, NODE_DATA(nid))->lru_lock); spin_unlock_irq(&objcg_lock);} static void memcg_reparent_objcgs(struct mem_cgroup *memcg){ struct obj_cgroup *objcg; struct mem_cgroup *parent = parent_mem_cgroup(memcg); int nid; for_each_node(nid) {retry: if (lru_gen_enabled()) max_lru_gen_memcg(parent, nid); reparent_locks(memcg, parent, nid); if (lru_gen_enabled()) { if (!recheck_lru_gen_max_memcg(parent, nid)) { reparent_unlocks(memcg, parent, nid); cond_resched(); goto retry; } lru_gen_reparent_memcg(memcg, parent, nid); } else { lru_reparent_memcg(memcg, parent, nid); } objcg = __memcg_reparent_objcgs(memcg, parent, nid); reparent_unlocks(memcg, parent, nid); percpu_ref_kill(&objcg->refcnt); } reparent_state_local(memcg, parent);} /* * A lot of the calls to the cache allocation functions are expected to be * inlined by the compiler. Since the calls to memcg_slab_post_alloc_hook() are * conditional to this static branch, we'll have to allow modules that does * kmem_cache_alloc and the such to see this symbol as well */DEFINE_STATIC_KEY_FALSE(memcg_kmem_online_key);EXPORT_SYMBOL(memcg_kmem_online_key); DEFINE_STATIC_KEY_FALSE(memcg_bpf_enabled_key);EXPORT_SYMBOL(memcg_bpf_enabled_key); /** * get_mem_cgroup_css_from_folio - acquire a css of the memcg associated with a folio * @folio: folio of interest * * If memcg is bound to the default hierarchy, css of the memcg associated * with @folio is returned. The returned css remains associated with @folio * until it is released. * * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup * is returned. */struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(struct folio *folio){ struct mem_cgroup *memcg; if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) return &root_mem_cgroup->css; memcg = get_mem_cgroup_from_folio(folio); return memcg ? &memcg->css : &root_mem_cgroup->css;} /** * page_cgroup_ino - return inode number of the memcg a page is charged to * @page: the page * * Look up the closest online ancestor of the memory cgroup @page is charged to * and return its inode number or 0 if @page is not charged to any cgroup. It * is safe to call this function without holding a reference to @page. * * Note, this function is inherently racy, because there is nothing to prevent * the cgroup inode from getting torn down and potentially reallocated a moment * after page_cgroup_ino() returns, so it only should be used by callers that * do not care (such as procfs interfaces). */ino_t page_cgroup_ino(struct page *page){ struct mem_cgroup *memcg; unsigned long ino = 0; rcu_read_lock(); /* page_folio() is racy here, but the entire function is racy anyway */ memcg = folio_memcg_check(page_folio(page)); while (memcg && !css_is_online(&memcg->css)) memcg = parent_mem_cgroup(memcg); if (memcg) ino = cgroup_ino(memcg->css.cgroup); rcu_read_unlock(); return ino;}EXPORT_SYMBOL_GPL(page_cgroup_ino); /* Subset of node_stat_item for memcg stats */static const unsigned int memcg_node_stat_items[] = { NR_INACTIVE_ANON, NR_ACTIVE_ANON, NR_INACTIVE_FILE, NR_ACTIVE_FILE, NR_UNEVICTABLE, NR_SLAB_RECLAIMABLE_B, NR_SLAB_UNRECLAIMABLE_B, WORKINGSET_REFAULT_ANON, WORKINGSET_REFAULT_FILE, WORKINGSET_ACTIVATE_ANON, WORKINGSET_ACTIVATE_FILE, WORKINGSET_RESTORE_ANON, WORKINGSET_RESTORE_FILE, WORKINGSET_NODERECLAIM, NR_ANON_MAPPED, NR_FILE_MAPPED, NR_FILE_PAGES, NR_FILE_DIRTY, NR_WRITEBACK, NR_SHMEM, NR_SHMEM_THPS, NR_FILE_THPS, NR_ANON_THPS, NR_VMSCAN_WRITE, NR_VMALLOC, NR_KERNEL_STACK_KB, NR_PAGETABLE, NR_SECONDARY_PAGETABLE,#ifdef CONFIG_SWAP NR_SWAPCACHE,#endif#ifdef CONFIG_NUMA_BALANCING PGPROMOTE_SUCCESS,#endif PGDEMOTE_KSWAPD, PGDEMOTE_DIRECT, PGDEMOTE_KHUGEPAGED, PGDEMOTE_PROACTIVE, PGSTEAL_KSWAPD, PGSTEAL_DIRECT, PGSTEAL_KHUGEPAGED, PGSTEAL_PROACTIVE, PGSTEAL_ANON, PGSTEAL_FILE, PGSCAN_KSWAPD, PGSCAN_DIRECT, PGSCAN_KHUGEPAGED, PGSCAN_PROACTIVE, PGSCAN_ANON, PGSCAN_FILE, PGROTATE_ANON, PGROTATE_FILE, PGREFILL,#ifdef CONFIG_HUGETLB_PAGE NR_HUGETLB,#endif}; static const unsigned int memcg_stat_items[] = { MEMCG_SWAP, MEMCG_SOCK, MEMCG_PERCPU_B, MEMCG_KMEM, MEMCG_ZSWAP_B, MEMCG_ZSWAPPED, MEMCG_ZSWAP_INCOMP,}; #define NR_MEMCG_NODE_STAT_ITEMS ARRAY_SIZE(memcg_node_stat_items)#define MEMCG_VMSTAT_SIZE (NR_MEMCG_NODE_STAT_ITEMS + \ ARRAY_SIZE(memcg_stat_items))#define BAD_STAT_IDX(index) ((u32)(index) >= U8_MAX)static u8 mem_cgroup_stats_index[MEMCG_NR_STAT] __read_mostly; static void init_memcg_stats(void){ u8 i, j = 0; BUILD_BUG_ON(MEMCG_NR_STAT >= U8_MAX); memset(mem_cgroup_stats_index, U8_MAX, sizeof(mem_cgroup_stats_index)); for (i = 0; i < NR_MEMCG_NODE_STAT_ITEMS; ++i, ++j) mem_cgroup_stats_index[memcg_node_stat_items[i]] = j; for (i = 0; i < ARRAY_SIZE(memcg_stat_items); ++i, ++j) mem_cgroup_stats_index[memcg_stat_items[i]] = j;} static inline int memcg_stats_index(int idx){ return mem_cgroup_stats_index[idx];} struct lruvec_stats_percpu { /* Local (CPU and cgroup) state */ long state[NR_MEMCG_NODE_STAT_ITEMS]; /* Delta calculation for lockless upward propagation */ long state_prev[NR_MEMCG_NODE_STAT_ITEMS];}; struct lruvec_stats { /* Aggregated (CPU and subtree) state */ long state[NR_MEMCG_NODE_STAT_ITEMS]; /* Non-hierarchical (CPU aggregated) state */ long state_local[NR_MEMCG_NODE_STAT_ITEMS]; /* Pending child counts during tree propagation */ long state_pending[NR_MEMCG_NODE_STAT_ITEMS];}; unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx){ struct mem_cgroup_per_node *pn; long x; int i; if (mem_cgroup_disabled()) return node_page_state(lruvec_pgdat(lruvec), idx); Lines 1–500 of 6218. Load more
// SPDX-License-Identifier: GPL-2.0-or-later /* memcontrol.c - Memory Controller * * Copyright IBM Corporation, 2007 * Author Balbir Singh <balbir@linux.vnet.ibm.com> * * Copyright 2007 OpenVZ SWsoft Inc * Author: Pavel Emelianov <xemul@openvz.org> * * Memory thresholds * Copyright (C) 2009 Nokia Corporation * Author: Kirill A. Shutemov * * Kernel Memory Controller * Copyright (C) 2012 Parallels Inc. and Google Inc. * Authors: Glauber Costa and Suleiman Souhlal * * Native page reclaim * Charge lifetime sanitation * Lockless page tracking & accounting * Unified hierarchy configuration model * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner * * Per memcg lru locking * Copyright (C) 2020 Alibaba, Inc, Alex Shi */ #include <linux/cgroup-defs.h> #include <linux/page_counter.h> #include <linux/memcontrol.h> #include <linux/cgroup.h> #include <linux/cpuset.h> #include <linux/sched/mm.h> #include <linux/shmem_fs.h> #include <linux/hugetlb.h> #include <linux/pagemap.h> #include <linux/folio_batch.h> #include <linux/vm_event_item.h> #include <linux/smp.h> #include <linux/page-flags.h> #include <linux/backing-dev.h> #include <linux/bit_spinlock.h> #include <linux/rcupdate.h> #include <linux/limits.h> #include <linux/export.h> #include <linux/list.h> #include <linux/mutex.h> #include <linux/rbtree.h> #include <linux/slab.h> #include <linux/swapops.h> #include <linux/spinlock.h> #include <linux/fs.h> #include <linux/seq_file.h> #include <linux/vmpressure.h> #include <linux/memremap.h> #include <linux/mm_inline.h> #include <linux/cpu.h> #include <linux/oom.h> #include <linux/lockdep.h> #include <linux/resume_user_mode.h> #include <linux/psi.h> #include <linux/seq_buf.h> #include <linux/sched/isolation.h> #include <linux/kmemleak.h> #include "internal.h" #include "swap.h" #include "swap_table.h" #include <net/sock.h> #include <net/ip.h> #include "slab.h" #include "memcontrol-v1.h" #include <linux/uaccess.h> #define CREATE_TRACE_POINTS #include <trace/events/memcg.h> #undef CREATE_TRACE_POINTS #include <trace/events/vmscan.h> struct cgroup_subsys memory_cgrp_subsys __read_mostly; EXPORT_SYMBOL(memory_cgrp_subsys); struct mem_cgroup *root_mem_cgroup __read_mostly; EXPORT_SYMBOL(root_mem_cgroup); /* Active memory cgroup to use from an interrupt context */ DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg); EXPORT_PER_CPU_SYMBOL_GPL(int_active_memcg); /* Socket memory accounting disabled? */ static bool cgroup_memory_nosocket __ro_after_init; /* Kernel memory accounting disabled? */ static bool cgroup_memory_nokmem __ro_after_init; /* BPF memory accounting disabled? */ static bool cgroup_memory_nobpf __ro_after_init; static struct workqueue_struct *memcg_wq __ro_after_init; static struct kmem_cache *memcg_cachep; static struct kmem_cache *memcg_pn_cachep; #ifdef CONFIG_CGROUP_WRITEBACK static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq); #endif static inline bool task_is_dying(void) { return tsk_is_oom_victim(current) || fatal_signal_pending(current) || (current->flags & PF_EXITING); } /* Some nice accessors for the vmpressure. */ struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg) { if (!memcg) memcg = root_mem_cgroup; return &memcg->vmpressure; } struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr) { return container_of(vmpr, struct mem_cgroup, vmpressure); } #define SEQ_BUF_SIZE SZ_4K #define CURRENT_OBJCG_UPDATE_BIT 0 #define CURRENT_OBJCG_UPDATE_FLAG (1UL << CURRENT_OBJCG_UPDATE_BIT) static DEFINE_SPINLOCK(objcg_lock); bool mem_cgroup_kmem_disabled(void) { return cgroup_memory_nokmem; } static void memcg_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages); static void memcg_uncharge_kmem(struct mem_cgroup *memcg, unsigned int nr_pages) { mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages); memcg1_account_kmem(memcg, -nr_pages); if (!mem_cgroup_is_root(memcg)) memcg_uncharge(memcg, nr_pages); } static void obj_cgroup_release(struct percpu_ref *ref) { struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt); unsigned int nr_bytes; unsigned int nr_pages; unsigned long flags; /* * At this point all allocated objects are freed, and * objcg->nr_charged_bytes can't have an arbitrary byte value. * However, it can be PAGE_SIZE or (x * PAGE_SIZE). * * The following sequence can lead to it: * 1) CPU0: objcg cached in one of stock->cached[i] * 2) CPU1: we do a small allocation (e.g. 92 bytes), * PAGE_SIZE bytes are charged * 3) CPU1: a process from another memcg is allocating something, * the stock if flushed, * objcg->nr_charged_bytes = PAGE_SIZE - 92 * 4) CPU0: we do release this object, * 92 bytes are added to stock->nr_bytes[i] * 5) CPU0: stock is flushed, * 92 bytes are added to objcg->nr_charged_bytes * * In the result, nr_charged_bytes == PAGE_SIZE. * This page will be uncharged in obj_cgroup_release(). */ nr_bytes = atomic_read(&objcg->nr_charged_bytes); WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1)); nr_pages = nr_bytes >> PAGE_SHIFT; if (nr_pages) { struct mem_cgroup *memcg; memcg = get_mem_cgroup_from_objcg(objcg); memcg_uncharge_kmem(memcg, nr_pages); mem_cgroup_put(memcg); } spin_lock_irqsave(&objcg_lock, flags); list_del(&objcg->list); spin_unlock_irqrestore(&objcg_lock, flags); percpu_ref_exit(ref); kfree_rcu(objcg, rcu); } static struct obj_cgroup *obj_cgroup_alloc(void) { struct obj_cgroup *objcg; int ret; objcg = kzalloc_obj(struct obj_cgroup); if (!objcg) return NULL; ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0, GFP_KERNEL); if (ret) { kfree(objcg); return NULL; } INIT_LIST_HEAD(&objcg->list); return objcg; } static inline struct obj_cgroup *__memcg_reparent_objcgs(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid) { struct obj_cgroup *objcg, *iter; struct mem_cgroup_per_node *pn = memcg->nodeinfo[nid]; struct mem_cgroup_per_node *parent_pn = parent->nodeinfo[nid]; objcg = rcu_replace_pointer(pn->objcg, NULL, true); /* 1) Ready to reparent active objcg. */ list_add(&objcg->list, &pn->objcg_list); /* 2) Reparent active objcg and already reparented objcgs to parent. */ list_for_each_entry(iter, &pn->objcg_list, list) WRITE_ONCE(iter->memcg, parent); /* 3) Move already reparented objcgs to the parent's list */ list_splice(&pn->objcg_list, &parent_pn->objcg_list); return objcg; } #ifdef CONFIG_MEMCG_V1 static void __mem_cgroup_flush_stats(struct mem_cgroup *memcg, bool force); static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent) { if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) return; /* * Reparent stats exposed non-hierarchically. Flush @memcg's stats first * to read its stats accurately , and conservatively flush @parent's * stats after reparenting to avoid hiding a potentially large stat * update (e.g. from callers of mem_cgroup_flush_stats_ratelimited()). */ __mem_cgroup_flush_stats(memcg, true); /* The following counts are all non-hierarchical and need to be reparented. */ reparent_memcg1_state_local(memcg, parent); reparent_memcg1_lruvec_state_local(memcg, parent); __mem_cgroup_flush_stats(parent, true); } #else static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent) { } #endif static inline void reparent_locks(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid) { spin_lock_irq(&objcg_lock); spin_lock_nested(&mem_cgroup_lruvec(memcg, NODE_DATA(nid))->lru_lock, 1); spin_lock_nested(&mem_cgroup_lruvec(parent, NODE_DATA(nid))->lru_lock, 2); } static inline void reparent_unlocks(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid) { spin_unlock(&mem_cgroup_lruvec(parent, NODE_DATA(nid))->lru_lock); spin_unlock(&mem_cgroup_lruvec(memcg, NODE_DATA(nid))->lru_lock); spin_unlock_irq(&objcg_lock); } static void memcg_reparent_objcgs(struct mem_cgroup *memcg) { struct obj_cgroup *objcg; struct mem_cgroup *parent = parent_mem_cgroup(memcg); int nid; for_each_node(nid) { retry: if (lru_gen_enabled()) max_lru_gen_memcg(parent, nid); reparent_locks(memcg, parent, nid); if (lru_gen_enabled()) { if (!recheck_lru_gen_max_memcg(parent, nid)) { reparent_unlocks(memcg, parent, nid); cond_resched(); goto retry; } lru_gen_reparent_memcg(memcg, parent, nid); } else { lru_reparent_memcg(memcg, parent, nid); } objcg = __memcg_reparent_objcgs(memcg, parent, nid); reparent_unlocks(memcg, parent, nid); percpu_ref_kill(&objcg->refcnt); } reparent_state_local(memcg, parent); } /* * A lot of the calls to the cache allocation functions are expected to be * inlined by the compiler. Since the calls to memcg_slab_post_alloc_hook() are * conditional to this static branch, we'll have to allow modules that does * kmem_cache_alloc and the such to see this symbol as well */ DEFINE_STATIC_KEY_FALSE(memcg_kmem_online_key); EXPORT_SYMBOL(memcg_kmem_online_key); DEFINE_STATIC_KEY_FALSE(memcg_bpf_enabled_key); EXPORT_SYMBOL(memcg_bpf_enabled_key); /** * get_mem_cgroup_css_from_folio - acquire a css of the memcg associated with a folio * @folio: folio of interest * * If memcg is bound to the default hierarchy, css of the memcg associated * with @folio is returned. The returned css remains associated with @folio * until it is released. * * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup * is returned. */ struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(struct folio *folio) { struct mem_cgroup *memcg; if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) return &root_mem_cgroup->css; memcg = get_mem_cgroup_from_folio(folio); return memcg ? &memcg->css : &root_mem_cgroup->css; } /** * page_cgroup_ino - return inode number of the memcg a page is charged to * @page: the page * * Look up the closest online ancestor of the memory cgroup @page is charged to * and return its inode number or 0 if @page is not charged to any cgroup. It * is safe to call this function without holding a reference to @page. * * Note, this function is inherently racy, because there is nothing to prevent * the cgroup inode from getting torn down and potentially reallocated a moment * after page_cgroup_ino() returns, so it only should be used by callers that * do not care (such as procfs interfaces). */ ino_t page_cgroup_ino(struct page *page) { struct mem_cgroup *memcg; unsigned long ino = 0; rcu_read_lock(); /* page_folio() is racy here, but the entire function is racy anyway */ memcg = folio_memcg_check(page_folio(page)); while (memcg && !css_is_online(&memcg->css)) memcg = parent_mem_cgroup(memcg); if (memcg) ino = cgroup_ino(memcg->css.cgroup); rcu_read_unlock(); return ino; } EXPORT_SYMBOL_GPL(page_cgroup_ino); /* Subset of node_stat_item for memcg stats */ static const unsigned int memcg_node_stat_items[] = { NR_INACTIVE_ANON, NR_ACTIVE_ANON, NR_INACTIVE_FILE, NR_ACTIVE_FILE, NR_UNEVICTABLE, NR_SLAB_RECLAIMABLE_B, NR_SLAB_UNRECLAIMABLE_B, WORKINGSET_REFAULT_ANON, WORKINGSET_REFAULT_FILE, WORKINGSET_ACTIVATE_ANON, WORKINGSET_ACTIVATE_FILE, WORKINGSET_RESTORE_ANON, WORKINGSET_RESTORE_FILE, WORKINGSET_NODERECLAIM, NR_ANON_MAPPED, NR_FILE_MAPPED, NR_FILE_PAGES, NR_FILE_DIRTY, NR_WRITEBACK, NR_SHMEM, NR_SHMEM_THPS, NR_FILE_THPS, NR_ANON_THPS, NR_VMSCAN_WRITE, NR_VMALLOC, NR_KERNEL_STACK_KB, NR_PAGETABLE, NR_SECONDARY_PAGETABLE, #ifdef CONFIG_SWAP NR_SWAPCACHE, #endif #ifdef CONFIG_NUMA_BALANCING PGPROMOTE_SUCCESS, #endif PGDEMOTE_KSWAPD, PGDEMOTE_DIRECT, PGDEMOTE_KHUGEPAGED, PGDEMOTE_PROACTIVE, PGSTEAL_KSWAPD, PGSTEAL_DIRECT, PGSTEAL_KHUGEPAGED, PGSTEAL_PROACTIVE, PGSTEAL_ANON, PGSTEAL_FILE, PGSCAN_KSWAPD, PGSCAN_DIRECT, PGSCAN_KHUGEPAGED, PGSCAN_PROACTIVE, PGSCAN_ANON, PGSCAN_FILE, PGROTATE_ANON, PGROTATE_FILE, PGREFILL, #ifdef CONFIG_HUGETLB_PAGE NR_HUGETLB, #endif }; static const unsigned int memcg_stat_items[] = { MEMCG_SWAP, MEMCG_SOCK, MEMCG_PERCPU_B, MEMCG_KMEM, MEMCG_ZSWAP_B, MEMCG_ZSWAPPED, MEMCG_ZSWAP_INCOMP, }; #define NR_MEMCG_NODE_STAT_ITEMS ARRAY_SIZE(memcg_node_stat_items) #define MEMCG_VMSTAT_SIZE (NR_MEMCG_NODE_STAT_ITEMS + \ ARRAY_SIZE(memcg_stat_items)) #define BAD_STAT_IDX(index) ((u32)(index) >= U8_MAX) static u8 mem_cgroup_stats_index[MEMCG_NR_STAT] __read_mostly; static void init_memcg_stats(void) { u8 i, j = 0; BUILD_BUG_ON(MEMCG_NR_STAT >= U8_MAX); memset(mem_cgroup_stats_index, U8_MAX, sizeof(mem_cgroup_stats_index)); for (i = 0; i < NR_MEMCG_NODE_STAT_ITEMS; ++i, ++j) mem_cgroup_stats_index[memcg_node_stat_items[i]] = j; for (i = 0; i < ARRAY_SIZE(memcg_stat_items); ++i, ++j) mem_cgroup_stats_index[memcg_stat_items[i]] = j; } static inline int memcg_stats_index(int idx) { return mem_cgroup_stats_index[idx]; } struct lruvec_stats_percpu { /* Local (CPU and cgroup) state */ long state[NR_MEMCG_NODE_STAT_ITEMS]; /* Delta calculation for lockless upward propagation */ long state_prev[NR_MEMCG_NODE_STAT_ITEMS]; }; struct lruvec_stats { /* Aggregated (CPU and subtree) state */ long state[NR_MEMCG_NODE_STAT_ITEMS]; /* Non-hierarchical (CPU aggregated) state */ long state_local[NR_MEMCG_NODE_STAT_ITEMS]; /* Pending child counts during tree propagation */ long state_pending[NR_MEMCG_NODE_STAT_ITEMS]; }; unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx) { struct mem_cgroup_per_node *pn; long x; int i; if (mem_cgroup_disabled()) return node_page_state(lruvec_pgdat(lruvec), idx);