// SPDX-License-Identifier: GPL-2.0-or-later/* memcontrol.c - Memory Controller * * Copyright IBM Corporation, 2007 * Author Balbir Singh <balbir@linux.vnet.ibm.com> * * Copyright 2007 OpenVZ SWsoft Inc * Author: Pavel Emelianov <xemul@openvz.org> * * Memory thresholds * Copyright (C) 2009 Nokia Corporation * Author: Kirill A. Shutemov * * Kernel Memory Controller * Copyright (C) 2012 Parallels Inc. and Google Inc. * Authors: Glauber Costa and Suleiman Souhlal * * Native page reclaim * Charge lifetime sanitation * Lockless page tracking & accounting * Unified hierarchy configuration model * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner */ #include <linux/page_counter.h>#include <linux/memcontrol.h>#include <linux/cgroup.h>#include <linux/pagewalk.h>#include <linux/sched/mm.h>#include <linux/shmem_fs.h>#include <linux/hugetlb.h>#include <linux/pagemap.h>#include <linux/vm_event_item.h>#include <linux/smp.h>#include <linux/page-flags.h>#include <linux/backing-dev.h>#include <linux/bit_spinlock.h>#include <linux/rcupdate.h>#include <linux/limits.h>#include <linux/export.h>#include <linux/mutex.h>#include <linux/rbtree.h>#include <linux/slab.h>#include <linux/swap.h>#include <linux/swapops.h>#include <linux/spinlock.h>#include <linux/eventfd.h>#include <linux/poll.h>#include <linux/sort.h>#include <linux/fs.h>#include <linux/seq_file.h>#include <linux/vmpressure.h>#include <linux/mm_inline.h>#include <linux/swap_cgroup.h>#include <linux/cpu.h>#include <linux/oom.h>#include <linux/lockdep.h>#include <linux/file.h>#include <linux/tracehook.h>#include <linux/psi.h>#include <linux/seq_buf.h>#include "internal.h"#include <net/sock.h>#include <net/ip.h>#include "slab.h" #include <linux/uaccess.h> #include <trace/events/vmscan.h> struct cgroup_subsys memory_cgrp_subsys __read_mostly;EXPORT_SYMBOL(memory_cgrp_subsys); struct mem_cgroup *root_mem_cgroup __read_mostly; /* Socket memory accounting disabled? */static bool cgroup_memory_nosocket; /* Kernel memory accounting disabled? */static bool cgroup_memory_nokmem; /* Whether the swap controller is active */#ifdef CONFIG_MEMCG_SWAPbool cgroup_memory_noswap __read_mostly;#else#define cgroup_memory_noswap 1#endif #ifdef CONFIG_CGROUP_WRITEBACKstatic DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);#endif /* Whether legacy memory+swap accounting is active */static bool do_memsw_account(void){ return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_noswap;} #define THRESHOLDS_EVENTS_TARGET 128#define SOFTLIMIT_EVENTS_TARGET 1024 /* * Cgroups above their limits are maintained in a RB-Tree, independent of * their hierarchy representation */ struct mem_cgroup_tree_per_node { struct rb_root rb_root; struct rb_node *rb_rightmost; spinlock_t lock;}; struct mem_cgroup_tree { struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];}; static struct mem_cgroup_tree soft_limit_tree __read_mostly; /* for OOM */struct mem_cgroup_eventfd_list { struct list_head list; struct eventfd_ctx *eventfd;}; /* * cgroup_event represents events which userspace want to receive. */struct mem_cgroup_event { /* * memcg which the event belongs to. */ struct mem_cgroup *memcg; /* * eventfd to signal userspace about the event. */ struct eventfd_ctx *eventfd; /* * Each of these stored in a list by the cgroup. */ struct list_head list; /* * register_event() callback will be used to add new userspace * waiter for changes related to this event. Use eventfd_signal() * on eventfd to send notification to userspace. */ int (*register_event)(struct mem_cgroup *memcg, struct eventfd_ctx *eventfd, const char *args); /* * unregister_event() callback will be called when userspace closes * the eventfd or on cgroup removing. This callback must be set, * if you want provide notification functionality. */ void (*unregister_event)(struct mem_cgroup *memcg, struct eventfd_ctx *eventfd); /* * All fields below needed to unregister event when * userspace closes eventfd. */ poll_table pt; wait_queue_head_t *wqh; wait_queue_entry_t wait; struct work_struct remove;}; static void mem_cgroup_threshold(struct mem_cgroup *memcg);static void mem_cgroup_oom_notify(struct mem_cgroup *memcg); /* Stuffs for move charges at task migration. *//* * Types of charges to be moved. */#define MOVE_ANON 0x1U#define MOVE_FILE 0x2U#define MOVE_MASK (MOVE_ANON | MOVE_FILE) /* "mc" and its members are protected by cgroup_mutex */static struct move_charge_struct { spinlock_t lock; /* for from, to */ struct mm_struct *mm; struct mem_cgroup *from; struct mem_cgroup *to; unsigned long flags; unsigned long precharge; unsigned long moved_charge; unsigned long moved_swap; struct task_struct *moving_task; /* a task moving charges */ wait_queue_head_t waitq; /* a waitq for other context */} mc = { .lock = __SPIN_LOCK_UNLOCKED(mc.lock), .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),}; /* * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft * limit reclaim to prevent infinite loops, if they ever occur. */#define MEM_CGROUP_MAX_RECLAIM_LOOPS 100#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2 enum charge_type { MEM_CGROUP_CHARGE_TYPE_CACHE = 0, MEM_CGROUP_CHARGE_TYPE_ANON, MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */ MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */ NR_CHARGE_TYPE,}; /* for encoding cft->private value on file */enum res_type { _MEM, _MEMSWAP, _OOM_TYPE, _KMEM, _TCP,}; #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))#define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)#define MEMFILE_ATTR(val) ((val) & 0xffff)/* Used for OOM nofiier */#define OOM_CONTROL (0) /* * Iteration constructs for visiting all cgroups (under a tree). If * loops are exited prematurely (break), mem_cgroup_iter_break() must * be used for reference counting. */#define for_each_mem_cgroup_tree(iter, root) \ for (iter = mem_cgroup_iter(root, NULL, NULL); \ iter != NULL; \ iter = mem_cgroup_iter(root, iter, NULL)) #define for_each_mem_cgroup(iter) \ for (iter = mem_cgroup_iter(NULL, NULL, NULL); \ iter != NULL; \ iter = mem_cgroup_iter(NULL, iter, NULL)) static inline bool should_force_charge(void){ return tsk_is_oom_victim(current) || fatal_signal_pending(current) || (current->flags & PF_EXITING);} /* Some nice accessors for the vmpressure. */struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg){ if (!memcg) memcg = root_mem_cgroup; return &memcg->vmpressure;} struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr){ return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;} #ifdef CONFIG_MEMCG_KMEMextern spinlock_t css_set_lock; static void obj_cgroup_release(struct percpu_ref *ref){ struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt); struct mem_cgroup *memcg; unsigned int nr_bytes; unsigned int nr_pages; unsigned long flags; /* * At this point all allocated objects are freed, and * objcg->nr_charged_bytes can't have an arbitrary byte value. * However, it can be PAGE_SIZE or (x * PAGE_SIZE). * * The following sequence can lead to it: * 1) CPU0: objcg == stock->cached_objcg * 2) CPU1: we do a small allocation (e.g. 92 bytes), * PAGE_SIZE bytes are charged * 3) CPU1: a process from another memcg is allocating something, * the stock if flushed, * objcg->nr_charged_bytes = PAGE_SIZE - 92 * 5) CPU0: we do release this object, * 92 bytes are added to stock->nr_bytes * 6) CPU0: stock is flushed, * 92 bytes are added to objcg->nr_charged_bytes * * In the result, nr_charged_bytes == PAGE_SIZE. * This page will be uncharged in obj_cgroup_release(). */ nr_bytes = atomic_read(&objcg->nr_charged_bytes); WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1)); nr_pages = nr_bytes >> PAGE_SHIFT; spin_lock_irqsave(&css_set_lock, flags); memcg = obj_cgroup_memcg(objcg); if (nr_pages) __memcg_kmem_uncharge(memcg, nr_pages); list_del(&objcg->list); mem_cgroup_put(memcg); spin_unlock_irqrestore(&css_set_lock, flags); percpu_ref_exit(ref); kfree_rcu(objcg, rcu);} static struct obj_cgroup *obj_cgroup_alloc(void){ struct obj_cgroup *objcg; int ret; objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL); if (!objcg) return NULL; ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0, GFP_KERNEL); if (ret) { kfree(objcg); return NULL; } INIT_LIST_HEAD(&objcg->list); return objcg;} static void memcg_reparent_objcgs(struct mem_cgroup *memcg, struct mem_cgroup *parent){ struct obj_cgroup *objcg, *iter; objcg = rcu_replace_pointer(memcg->objcg, NULL, true); spin_lock_irq(&css_set_lock); /* Move active objcg to the parent's list */ xchg(&objcg->memcg, parent); css_get(&parent->css); list_add(&objcg->list, &parent->objcg_list); /* Move already reparented objcgs to the parent's list */ list_for_each_entry(iter, &memcg->objcg_list, list) { css_get(&parent->css); xchg(&iter->memcg, parent); css_put(&memcg->css); } list_splice(&memcg->objcg_list, &parent->objcg_list); spin_unlock_irq(&css_set_lock); percpu_ref_kill(&objcg->refcnt);} /* * This will be used as a shrinker list's index. * The main reason for not using cgroup id for this: * this works better in sparse environments, where we have a lot of memcgs, * but only a few kmem-limited. Or also, if we have, for instance, 200 * memcgs, and none but the 200th is kmem-limited, we'd have to have a * 200 entry array for that. * * The current size of the caches array is stored in memcg_nr_cache_ids. It * will double each time we have to increase it. */static DEFINE_IDA(memcg_cache_ida);int memcg_nr_cache_ids; /* Protects memcg_nr_cache_ids */static DECLARE_RWSEM(memcg_cache_ids_sem); void memcg_get_cache_ids(void){ down_read(&memcg_cache_ids_sem);} void memcg_put_cache_ids(void){ up_read(&memcg_cache_ids_sem);} /* * MIN_SIZE is different than 1, because we would like to avoid going through * the alloc/free process all the time. In a small machine, 4 kmem-limited * cgroups is a reasonable guess. In the future, it could be a parameter or * tunable, but that is strictly not necessary. * * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get * this constant directly from cgroup, but it is understandable that this is * better kept as an internal representation in cgroup.c. In any case, the * cgrp_id space is not getting any smaller, and we don't have to necessarily * increase ours as well if it increases. */#define MEMCG_CACHES_MIN_SIZE 4#define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX /* * A lot of the calls to the cache allocation functions are expected to be * inlined by the compiler. Since the calls to memcg_slab_pre_alloc_hook() are * conditional to this static branch, we'll have to allow modules that does * kmem_cache_alloc and the such to see this symbol as well */DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);EXPORT_SYMBOL(memcg_kmem_enabled_key);#endif static int memcg_shrinker_map_size;static DEFINE_MUTEX(memcg_shrinker_map_mutex); static void memcg_free_shrinker_map_rcu(struct rcu_head *head){ kvfree(container_of(head, struct memcg_shrinker_map, rcu));} static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg, int size, int old_size){ struct memcg_shrinker_map *new, *old; int nid; lockdep_assert_held(&memcg_shrinker_map_mutex); for_each_node(nid) { old = rcu_dereference_protected( mem_cgroup_nodeinfo(memcg, nid)->shrinker_map, true); /* Not yet online memcg */ if (!old) return 0; new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid); if (!new) return -ENOMEM; /* Set all old bits, clear all new bits */ memset(new->map, (int)0xff, old_size); memset((void *)new->map + old_size, 0, size - old_size); rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, new); call_rcu(&old->rcu, memcg_free_shrinker_map_rcu); } return 0;} static void memcg_free_shrinker_maps(struct mem_cgroup *memcg){ struct mem_cgroup_per_node *pn; struct memcg_shrinker_map *map; int nid; if (mem_cgroup_is_root(memcg)) return; for_each_node(nid) { pn = mem_cgroup_nodeinfo(memcg, nid); map = rcu_dereference_protected(pn->shrinker_map, true); if (map) kvfree(map); rcu_assign_pointer(pn->shrinker_map, NULL); }} static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg){ struct memcg_shrinker_map *map; int nid, size, ret = 0; if (mem_cgroup_is_root(memcg)) return 0; mutex_lock(&memcg_shrinker_map_mutex); size = memcg_shrinker_map_size; for_each_node(nid) { map = kvzalloc_node(sizeof(*map) + size, GFP_KERNEL, nid); if (!map) { memcg_free_shrinker_maps(memcg); ret = -ENOMEM; break; } rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map); } mutex_unlock(&memcg_shrinker_map_mutex); return ret;} int memcg_expand_shrinker_maps(int new_id){ int size, old_size, ret = 0; struct mem_cgroup *memcg; size = DIV_ROUND_UP(new_id + 1, BITS_PER_LONG) * sizeof(unsigned long); old_size = memcg_shrinker_map_size; if (size <= old_size) return 0; mutex_lock(&memcg_shrinker_map_mutex); if (!root_mem_cgroup) goto unlock; for_each_mem_cgroup(memcg) { if (mem_cgroup_is_root(memcg)) continue; ret = memcg_expand_one_shrinker_map(memcg, size, old_size); if (ret) {Lines 1–500 of 7420. Load more
// SPDX-License-Identifier: GPL-2.0-or-later /* memcontrol.c - Memory Controller * * Copyright IBM Corporation, 2007 * Author Balbir Singh <balbir@linux.vnet.ibm.com> * * Copyright 2007 OpenVZ SWsoft Inc * Author: Pavel Emelianov <xemul@openvz.org> * * Memory thresholds * Copyright (C) 2009 Nokia Corporation * Author: Kirill A. Shutemov * * Kernel Memory Controller * Copyright (C) 2012 Parallels Inc. and Google Inc. * Authors: Glauber Costa and Suleiman Souhlal * * Native page reclaim * Charge lifetime sanitation * Lockless page tracking & accounting * Unified hierarchy configuration model * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner */ #include <linux/page_counter.h> #include <linux/memcontrol.h> #include <linux/cgroup.h> #include <linux/pagewalk.h> #include <linux/sched/mm.h> #include <linux/shmem_fs.h> #include <linux/hugetlb.h> #include <linux/pagemap.h> #include <linux/vm_event_item.h> #include <linux/smp.h> #include <linux/page-flags.h> #include <linux/backing-dev.h> #include <linux/bit_spinlock.h> #include <linux/rcupdate.h> #include <linux/limits.h> #include <linux/export.h> #include <linux/mutex.h> #include <linux/rbtree.h> #include <linux/slab.h> #include <linux/swap.h> #include <linux/swapops.h> #include <linux/spinlock.h> #include <linux/eventfd.h> #include <linux/poll.h> #include <linux/sort.h> #include <linux/fs.h> #include <linux/seq_file.h> #include <linux/vmpressure.h> #include <linux/mm_inline.h> #include <linux/swap_cgroup.h> #include <linux/cpu.h> #include <linux/oom.h> #include <linux/lockdep.h> #include <linux/file.h> #include <linux/tracehook.h> #include <linux/psi.h> #include <linux/seq_buf.h> #include "internal.h" #include <net/sock.h> #include <net/ip.h> #include "slab.h" #include <linux/uaccess.h> #include <trace/events/vmscan.h> struct cgroup_subsys memory_cgrp_subsys __read_mostly; EXPORT_SYMBOL(memory_cgrp_subsys); struct mem_cgroup *root_mem_cgroup __read_mostly; /* Socket memory accounting disabled? */ static bool cgroup_memory_nosocket; /* Kernel memory accounting disabled? */ static bool cgroup_memory_nokmem; /* Whether the swap controller is active */ #ifdef CONFIG_MEMCG_SWAP bool cgroup_memory_noswap __read_mostly; #else #define cgroup_memory_noswap 1 #endif #ifdef CONFIG_CGROUP_WRITEBACK static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq); #endif /* Whether legacy memory+swap accounting is active */ static bool do_memsw_account(void) { return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_noswap; } #define THRESHOLDS_EVENTS_TARGET 128 #define SOFTLIMIT_EVENTS_TARGET 1024 /* * Cgroups above their limits are maintained in a RB-Tree, independent of * their hierarchy representation */ struct mem_cgroup_tree_per_node { struct rb_root rb_root; struct rb_node *rb_rightmost; spinlock_t lock; }; struct mem_cgroup_tree { struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES]; }; static struct mem_cgroup_tree soft_limit_tree __read_mostly; /* for OOM */ struct mem_cgroup_eventfd_list { struct list_head list; struct eventfd_ctx *eventfd; }; /* * cgroup_event represents events which userspace want to receive. */ struct mem_cgroup_event { /* * memcg which the event belongs to. */ struct mem_cgroup *memcg; /* * eventfd to signal userspace about the event. */ struct eventfd_ctx *eventfd; /* * Each of these stored in a list by the cgroup. */ struct list_head list; /* * register_event() callback will be used to add new userspace * waiter for changes related to this event. Use eventfd_signal() * on eventfd to send notification to userspace. */ int (*register_event)(struct mem_cgroup *memcg, struct eventfd_ctx *eventfd, const char *args); /* * unregister_event() callback will be called when userspace closes * the eventfd or on cgroup removing. This callback must be set, * if you want provide notification functionality. */ void (*unregister_event)(struct mem_cgroup *memcg, struct eventfd_ctx *eventfd); /* * All fields below needed to unregister event when * userspace closes eventfd. */ poll_table pt; wait_queue_head_t *wqh; wait_queue_entry_t wait; struct work_struct remove; }; static void mem_cgroup_threshold(struct mem_cgroup *memcg); static void mem_cgroup_oom_notify(struct mem_cgroup *memcg); /* Stuffs for move charges at task migration. */ /* * Types of charges to be moved. */ #define MOVE_ANON 0x1U #define MOVE_FILE 0x2U #define MOVE_MASK (MOVE_ANON | MOVE_FILE) /* "mc" and its members are protected by cgroup_mutex */ static struct move_charge_struct { spinlock_t lock; /* for from, to */ struct mm_struct *mm; struct mem_cgroup *from; struct mem_cgroup *to; unsigned long flags; unsigned long precharge; unsigned long moved_charge; unsigned long moved_swap; struct task_struct *moving_task; /* a task moving charges */ wait_queue_head_t waitq; /* a waitq for other context */ } mc = { .lock = __SPIN_LOCK_UNLOCKED(mc.lock), .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq), }; /* * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft * limit reclaim to prevent infinite loops, if they ever occur. */ #define MEM_CGROUP_MAX_RECLAIM_LOOPS 100 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2 enum charge_type { MEM_CGROUP_CHARGE_TYPE_CACHE = 0, MEM_CGROUP_CHARGE_TYPE_ANON, MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */ MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */ NR_CHARGE_TYPE, }; /* for encoding cft->private value on file */ enum res_type { _MEM, _MEMSWAP, _OOM_TYPE, _KMEM, _TCP, }; #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val)) #define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff) #define MEMFILE_ATTR(val) ((val) & 0xffff) /* Used for OOM nofiier */ #define OOM_CONTROL (0) /* * Iteration constructs for visiting all cgroups (under a tree). If * loops are exited prematurely (break), mem_cgroup_iter_break() must * be used for reference counting. */ #define for_each_mem_cgroup_tree(iter, root) \ for (iter = mem_cgroup_iter(root, NULL, NULL); \ iter != NULL; \ iter = mem_cgroup_iter(root, iter, NULL)) #define for_each_mem_cgroup(iter) \ for (iter = mem_cgroup_iter(NULL, NULL, NULL); \ iter != NULL; \ iter = mem_cgroup_iter(NULL, iter, NULL)) static inline bool should_force_charge(void) { return tsk_is_oom_victim(current) || fatal_signal_pending(current) || (current->flags & PF_EXITING); } /* Some nice accessors for the vmpressure. */ struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg) { if (!memcg) memcg = root_mem_cgroup; return &memcg->vmpressure; } struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr) { return &container_of(vmpr, struct mem_cgroup, vmpressure)->css; } #ifdef CONFIG_MEMCG_KMEM extern spinlock_t css_set_lock; static void obj_cgroup_release(struct percpu_ref *ref) { struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt); struct mem_cgroup *memcg; unsigned int nr_bytes; unsigned int nr_pages; unsigned long flags; /* * At this point all allocated objects are freed, and * objcg->nr_charged_bytes can't have an arbitrary byte value. * However, it can be PAGE_SIZE or (x * PAGE_SIZE). * * The following sequence can lead to it: * 1) CPU0: objcg == stock->cached_objcg * 2) CPU1: we do a small allocation (e.g. 92 bytes), * PAGE_SIZE bytes are charged * 3) CPU1: a process from another memcg is allocating something, * the stock if flushed, * objcg->nr_charged_bytes = PAGE_SIZE - 92 * 5) CPU0: we do release this object, * 92 bytes are added to stock->nr_bytes * 6) CPU0: stock is flushed, * 92 bytes are added to objcg->nr_charged_bytes * * In the result, nr_charged_bytes == PAGE_SIZE. * This page will be uncharged in obj_cgroup_release(). */ nr_bytes = atomic_read(&objcg->nr_charged_bytes); WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1)); nr_pages = nr_bytes >> PAGE_SHIFT; spin_lock_irqsave(&css_set_lock, flags); memcg = obj_cgroup_memcg(objcg); if (nr_pages) __memcg_kmem_uncharge(memcg, nr_pages); list_del(&objcg->list); mem_cgroup_put(memcg); spin_unlock_irqrestore(&css_set_lock, flags); percpu_ref_exit(ref); kfree_rcu(objcg, rcu); } static struct obj_cgroup *obj_cgroup_alloc(void) { struct obj_cgroup *objcg; int ret; objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL); if (!objcg) return NULL; ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0, GFP_KERNEL); if (ret) { kfree(objcg); return NULL; } INIT_LIST_HEAD(&objcg->list); return objcg; } static void memcg_reparent_objcgs(struct mem_cgroup *memcg, struct mem_cgroup *parent) { struct obj_cgroup *objcg, *iter; objcg = rcu_replace_pointer(memcg->objcg, NULL, true); spin_lock_irq(&css_set_lock); /* Move active objcg to the parent's list */ xchg(&objcg->memcg, parent); css_get(&parent->css); list_add(&objcg->list, &parent->objcg_list); /* Move already reparented objcgs to the parent's list */ list_for_each_entry(iter, &memcg->objcg_list, list) { css_get(&parent->css); xchg(&iter->memcg, parent); css_put(&memcg->css); } list_splice(&memcg->objcg_list, &parent->objcg_list); spin_unlock_irq(&css_set_lock); percpu_ref_kill(&objcg->refcnt); } /* * This will be used as a shrinker list's index. * The main reason for not using cgroup id for this: * this works better in sparse environments, where we have a lot of memcgs, * but only a few kmem-limited. Or also, if we have, for instance, 200 * memcgs, and none but the 200th is kmem-limited, we'd have to have a * 200 entry array for that. * * The current size of the caches array is stored in memcg_nr_cache_ids. It * will double each time we have to increase it. */ static DEFINE_IDA(memcg_cache_ida); int memcg_nr_cache_ids; /* Protects memcg_nr_cache_ids */ static DECLARE_RWSEM(memcg_cache_ids_sem); void memcg_get_cache_ids(void) { down_read(&memcg_cache_ids_sem); } void memcg_put_cache_ids(void) { up_read(&memcg_cache_ids_sem); } /* * MIN_SIZE is different than 1, because we would like to avoid going through * the alloc/free process all the time. In a small machine, 4 kmem-limited * cgroups is a reasonable guess. In the future, it could be a parameter or * tunable, but that is strictly not necessary. * * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get * this constant directly from cgroup, but it is understandable that this is * better kept as an internal representation in cgroup.c. In any case, the * cgrp_id space is not getting any smaller, and we don't have to necessarily * increase ours as well if it increases. */ #define MEMCG_CACHES_MIN_SIZE 4 #define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX /* * A lot of the calls to the cache allocation functions are expected to be * inlined by the compiler. Since the calls to memcg_slab_pre_alloc_hook() are * conditional to this static branch, we'll have to allow modules that does * kmem_cache_alloc and the such to see this symbol as well */ DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key); EXPORT_SYMBOL(memcg_kmem_enabled_key); #endif static int memcg_shrinker_map_size; static DEFINE_MUTEX(memcg_shrinker_map_mutex); static void memcg_free_shrinker_map_rcu(struct rcu_head *head) { kvfree(container_of(head, struct memcg_shrinker_map, rcu)); } static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg, int size, int old_size) { struct memcg_shrinker_map *new, *old; int nid; lockdep_assert_held(&memcg_shrinker_map_mutex); for_each_node(nid) { old = rcu_dereference_protected( mem_cgroup_nodeinfo(memcg, nid)->shrinker_map, true); /* Not yet online memcg */ if (!old) return 0; new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid); if (!new) return -ENOMEM; /* Set all old bits, clear all new bits */ memset(new->map, (int)0xff, old_size); memset((void *)new->map + old_size, 0, size - old_size); rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, new); call_rcu(&old->rcu, memcg_free_shrinker_map_rcu); } return 0; } static void memcg_free_shrinker_maps(struct mem_cgroup *memcg) { struct mem_cgroup_per_node *pn; struct memcg_shrinker_map *map; int nid; if (mem_cgroup_is_root(memcg)) return; for_each_node(nid) { pn = mem_cgroup_nodeinfo(memcg, nid); map = rcu_dereference_protected(pn->shrinker_map, true); if (map) kvfree(map); rcu_assign_pointer(pn->shrinker_map, NULL); } } static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg) { struct memcg_shrinker_map *map; int nid, size, ret = 0; if (mem_cgroup_is_root(memcg)) return 0; mutex_lock(&memcg_shrinker_map_mutex); size = memcg_shrinker_map_size; for_each_node(nid) { map = kvzalloc_node(sizeof(*map) + size, GFP_KERNEL, nid); if (!map) { memcg_free_shrinker_maps(memcg); ret = -ENOMEM; break; } rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map); } mutex_unlock(&memcg_shrinker_map_mutex); return ret; } int memcg_expand_shrinker_maps(int new_id) { int size, old_size, ret = 0; struct mem_cgroup *memcg; size = DIV_ROUND_UP(new_id + 1, BITS_PER_LONG) * sizeof(unsigned long); old_size = memcg_shrinker_map_size; if (size <= old_size) return 0; mutex_lock(&memcg_shrinker_map_mutex); if (!root_mem_cgroup) goto unlock; for_each_mem_cgroup(memcg) { if (mem_cgroup_is_root(memcg)) continue; ret = memcg_expand_one_shrinker_map(memcg, size, old_size); if (ret) {