/* memcontrol.c - Memory Controller * * Copyright IBM Corporation, 2007 * Author Balbir Singh <balbir@linux.vnet.ibm.com> * * Copyright 2007 OpenVZ SWsoft Inc * Author: Pavel Emelianov <xemul@openvz.org> * * Memory thresholds * Copyright (C) 2009 Nokia Corporation * Author: Kirill A. Shutemov * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. */ #include <linux/res_counter.h>#include <linux/memcontrol.h>#include <linux/cgroup.h>#include <linux/mm.h>#include <linux/hugetlb.h>#include <linux/pagemap.h>#include <linux/smp.h>#include <linux/page-flags.h>#include <linux/backing-dev.h>#include <linux/bit_spinlock.h>#include <linux/rcupdate.h>#include <linux/limits.h>#include <linux/export.h>#include <linux/mutex.h>#include <linux/rbtree.h>#include <linux/slab.h>#include <linux/swap.h>#include <linux/swapops.h>#include <linux/spinlock.h>#include <linux/eventfd.h>#include <linux/sort.h>#include <linux/fs.h>#include <linux/seq_file.h>#include <linux/vmalloc.h>#include <linux/mm_inline.h>#include <linux/page_cgroup.h>#include <linux/cpu.h>#include <linux/oom.h>#include "internal.h"#include <net/sock.h>#include <net/tcp_memcontrol.h> #include <asm/uaccess.h> #include <trace/events/vmscan.h> struct cgroup_subsys mem_cgroup_subsys __read_mostly;#define MEM_CGROUP_RECLAIM_RETRIES 5static struct mem_cgroup *root_mem_cgroup __read_mostly; #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */int do_swap_account __read_mostly; /* for remember boot option*/#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP_ENABLEDstatic int really_do_swap_account __initdata = 1;#elsestatic int really_do_swap_account __initdata = 0;#endif #else#define do_swap_account 0#endif /* * Statistics for memory cgroup. */enum mem_cgroup_stat_index { /* * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss. */ MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */ MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */ MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */ MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */ MEM_CGROUP_STAT_NSTATS,}; static const char * const mem_cgroup_stat_names[] = { "cache", "rss", "mapped_file", "swap",}; enum mem_cgroup_events_index { MEM_CGROUP_EVENTS_PGPGIN, /* # of pages paged in */ MEM_CGROUP_EVENTS_PGPGOUT, /* # of pages paged out */ MEM_CGROUP_EVENTS_PGFAULT, /* # of page-faults */ MEM_CGROUP_EVENTS_PGMAJFAULT, /* # of major page-faults */ MEM_CGROUP_EVENTS_NSTATS,}; static const char * const mem_cgroup_events_names[] = { "pgpgin", "pgpgout", "pgfault", "pgmajfault",}; /* * Per memcg event counter is incremented at every pagein/pageout. With THP, * it will be incremated by the number of pages. This counter is used for * for trigger some periodic events. This is straightforward and better * than using jiffies etc. to handle periodic memcg event. */enum mem_cgroup_events_target { MEM_CGROUP_TARGET_THRESH, MEM_CGROUP_TARGET_SOFTLIMIT, MEM_CGROUP_TARGET_NUMAINFO, MEM_CGROUP_NTARGETS,};#define THRESHOLDS_EVENTS_TARGET 128#define SOFTLIMIT_EVENTS_TARGET 1024#define NUMAINFO_EVENTS_TARGET 1024 struct mem_cgroup_stat_cpu { long count[MEM_CGROUP_STAT_NSTATS]; unsigned long events[MEM_CGROUP_EVENTS_NSTATS]; unsigned long nr_page_events; unsigned long targets[MEM_CGROUP_NTARGETS];}; struct mem_cgroup_reclaim_iter { /* css_id of the last scanned hierarchy member */ int position; /* scan generation, increased every round-trip */ unsigned int generation;}; /* * per-zone information in memory controller. */struct mem_cgroup_per_zone { struct lruvec lruvec; unsigned long lru_size[NR_LRU_LISTS]; struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1]; struct rb_node tree_node; /* RB tree node */ unsigned long long usage_in_excess;/* Set to the value by which */ /* the soft limit is exceeded*/ bool on_tree; struct mem_cgroup *memcg; /* Back pointer, we cannot */ /* use container_of */}; struct mem_cgroup_per_node { struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];}; struct mem_cgroup_lru_info { struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];}; /* * Cgroups above their limits are maintained in a RB-Tree, independent of * their hierarchy representation */ struct mem_cgroup_tree_per_zone { struct rb_root rb_root; spinlock_t lock;}; struct mem_cgroup_tree_per_node { struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];}; struct mem_cgroup_tree { struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];}; static struct mem_cgroup_tree soft_limit_tree __read_mostly; struct mem_cgroup_threshold { struct eventfd_ctx *eventfd; u64 threshold;}; /* For threshold */struct mem_cgroup_threshold_ary { /* An array index points to threshold just below or equal to usage. */ int current_threshold; /* Size of entries[] */ unsigned int size; /* Array of thresholds */ struct mem_cgroup_threshold entries[0];}; struct mem_cgroup_thresholds { /* Primary thresholds array */ struct mem_cgroup_threshold_ary *primary; /* * Spare threshold array. * This is needed to make mem_cgroup_unregister_event() "never fail". * It must be able to store at least primary->size - 1 entries. */ struct mem_cgroup_threshold_ary *spare;}; /* for OOM */struct mem_cgroup_eventfd_list { struct list_head list; struct eventfd_ctx *eventfd;}; static void mem_cgroup_threshold(struct mem_cgroup *memcg);static void mem_cgroup_oom_notify(struct mem_cgroup *memcg); /* * The memory controller data structure. The memory controller controls both * page cache and RSS per cgroup. We would eventually like to provide * statistics based on the statistics developed by Rik Van Riel for clock-pro, * to help the administrator determine what knobs to tune. * * TODO: Add a water mark for the memory controller. Reclaim will begin when * we hit the water mark. May be even add a low water mark, such that * no reclaim occurs from a cgroup at it's low water mark, this is * a feature that will be implemented much later in the future. */struct mem_cgroup { struct cgroup_subsys_state css; /* * the counter to account for memory usage */ struct res_counter res; union { /* * the counter to account for mem+swap usage. */ struct res_counter memsw; /* * rcu_freeing is used only when freeing struct mem_cgroup, * so put it into a union to avoid wasting more memory. * It must be disjoint from the css field. It could be * in a union with the res field, but res plays a much * larger part in mem_cgroup life than memsw, and might * be of interest, even at time of free, when debugging. * So share rcu_head with the less interesting memsw. */ struct rcu_head rcu_freeing; /* * We also need some space for a worker in deferred freeing. * By the time we call it, rcu_freeing is no longer in use. */ struct work_struct work_freeing; }; /* * Per cgroup active and inactive list, similar to the * per zone LRU lists. */ struct mem_cgroup_lru_info info; int last_scanned_node;#if MAX_NUMNODES > 1 nodemask_t scan_nodes; atomic_t numainfo_events; atomic_t numainfo_updating;#endif /* * Should the accounting and control be hierarchical, per subtree? */ bool use_hierarchy; bool oom_lock; atomic_t under_oom; atomic_t refcnt; int swappiness; /* OOM-Killer disable */ int oom_kill_disable; /* set when res.limit == memsw.limit */ bool memsw_is_minimum; /* protect arrays of thresholds */ struct mutex thresholds_lock; /* thresholds for memory usage. RCU-protected */ struct mem_cgroup_thresholds thresholds; /* thresholds for mem+swap usage. RCU-protected */ struct mem_cgroup_thresholds memsw_thresholds; /* For oom notifier event fd */ struct list_head oom_notify; /* * Should we move charges of a task when a task is moved into this * mem_cgroup ? And what type of charges should we move ? */ unsigned long move_charge_at_immigrate; /* * set > 0 if pages under this cgroup are moving to other cgroup. */ atomic_t moving_account; /* taken only while moving_account > 0 */ spinlock_t move_lock; /* * percpu counter. */ struct mem_cgroup_stat_cpu __percpu *stat; /* * used when a cpu is offlined or other synchronizations * See mem_cgroup_read_stat(). */ struct mem_cgroup_stat_cpu nocpu_base; spinlock_t pcp_counter_lock; #ifdef CONFIG_INET struct tcp_memcontrol tcp_mem;#endif}; /* Stuffs for move charges at task migration. *//* * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a * left-shifted bitmap of these types. */enum move_type { MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */ MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */ NR_MOVE_TYPE,}; /* "mc" and its members are protected by cgroup_mutex */static struct move_charge_struct { spinlock_t lock; /* for from, to */ struct mem_cgroup *from; struct mem_cgroup *to; unsigned long precharge; unsigned long moved_charge; unsigned long moved_swap; struct task_struct *moving_task; /* a task moving charges */ wait_queue_head_t waitq; /* a waitq for other context */} mc = { .lock = __SPIN_LOCK_UNLOCKED(mc.lock), .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),}; static bool move_anon(void){ return test_bit(MOVE_CHARGE_TYPE_ANON, &mc.to->move_charge_at_immigrate);} static bool move_file(void){ return test_bit(MOVE_CHARGE_TYPE_FILE, &mc.to->move_charge_at_immigrate);} /* * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft * limit reclaim to prevent infinite loops, if they ever occur. */#define MEM_CGROUP_MAX_RECLAIM_LOOPS 100#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2 enum charge_type { MEM_CGROUP_CHARGE_TYPE_CACHE = 0, MEM_CGROUP_CHARGE_TYPE_MAPPED, MEM_CGROUP_CHARGE_TYPE_SHMEM, /* used by page migration of shmem */ MEM_CGROUP_CHARGE_TYPE_FORCE, /* used by force_empty */ MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */ MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */ NR_CHARGE_TYPE,}; /* for encoding cft->private value on file */#define _MEM (0)#define _MEMSWAP (1)#define _OOM_TYPE (2)#define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))#define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)#define MEMFILE_ATTR(val) ((val) & 0xffff)/* Used for OOM nofiier */#define OOM_CONTROL (0) /* * Reclaim flags for mem_cgroup_hierarchical_reclaim */#define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0#define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)#define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1#define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT) static void mem_cgroup_get(struct mem_cgroup *memcg);static void mem_cgroup_put(struct mem_cgroup *memcg); /* Writing them here to avoid exposing memcg's inner layout */#ifdef CONFIG_CGROUP_MEM_RES_CTLR_KMEM#include <net/sock.h>#include <net/ip.h> static bool mem_cgroup_is_root(struct mem_cgroup *memcg);void sock_update_memcg(struct sock *sk){ if (mem_cgroup_sockets_enabled) { struct mem_cgroup *memcg; struct cg_proto *cg_proto; BUG_ON(!sk->sk_prot->proto_cgroup); /* Socket cloning can throw us here with sk_cgrp already * filled. It won't however, necessarily happen from * process context. So the test for root memcg given * the current task's memcg won't help us in this case. * * Respecting the original socket's memcg is a better * decision in this case. */ if (sk->sk_cgrp) { BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg)); mem_cgroup_get(sk->sk_cgrp->memcg); return; } rcu_read_lock(); memcg = mem_cgroup_from_task(current); cg_proto = sk->sk_prot->proto_cgroup(memcg); if (!mem_cgroup_is_root(memcg) && memcg_proto_active(cg_proto)) { mem_cgroup_get(memcg); sk->sk_cgrp = cg_proto; } rcu_read_unlock(); }}EXPORT_SYMBOL(sock_update_memcg); void sock_release_memcg(struct sock *sk){ if (mem_cgroup_sockets_enabled && sk->sk_cgrp) { struct mem_cgroup *memcg; WARN_ON(!sk->sk_cgrp->memcg); memcg = sk->sk_cgrp->memcg; mem_cgroup_put(memcg); }} #ifdef CONFIG_INETstruct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg){ if (!memcg || mem_cgroup_is_root(memcg)) return NULL; return &memcg->tcp_mem.cg_proto;}EXPORT_SYMBOL(tcp_proto_cgroup);#endif /* CONFIG_INET */#endif /* CONFIG_CGROUP_MEM_RES_CTLR_KMEM */ #if defined(CONFIG_INET) && defined(CONFIG_CGROUP_MEM_RES_CTLR_KMEM)static void disarm_sock_keys(struct mem_cgroup *memcg){ if (!memcg_proto_activated(&memcg->tcp_mem.cg_proto)) return; static_key_slow_dec(&memcg_socket_limit_enabled);}#elsestatic void disarm_sock_keys(struct mem_cgroup *memcg){}#endif static void drain_all_stock_async(struct mem_cgroup *memcg); static struct mem_cgroup_per_zone *mem_cgroup_zoneinfo(struct mem_cgroup *memcg, int nid, int zid){ return &memcg->info.nodeinfo[nid]->zoneinfo[zid];} struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg){ return &memcg->css;} static struct mem_cgroup_per_zone *page_cgroup_zoneinfo(struct mem_cgroup *memcg, struct page *page){Lines 1–500 of 5556. Load more
/* memcontrol.c - Memory Controller * * Copyright IBM Corporation, 2007 * Author Balbir Singh <balbir@linux.vnet.ibm.com> * * Copyright 2007 OpenVZ SWsoft Inc * Author: Pavel Emelianov <xemul@openvz.org> * * Memory thresholds * Copyright (C) 2009 Nokia Corporation * Author: Kirill A. Shutemov * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. */ #include <linux/res_counter.h> #include <linux/memcontrol.h> #include <linux/cgroup.h> #include <linux/mm.h> #include <linux/hugetlb.h> #include <linux/pagemap.h> #include <linux/smp.h> #include <linux/page-flags.h> #include <linux/backing-dev.h> #include <linux/bit_spinlock.h> #include <linux/rcupdate.h> #include <linux/limits.h> #include <linux/export.h> #include <linux/mutex.h> #include <linux/rbtree.h> #include <linux/slab.h> #include <linux/swap.h> #include <linux/swapops.h> #include <linux/spinlock.h> #include <linux/eventfd.h> #include <linux/sort.h> #include <linux/fs.h> #include <linux/seq_file.h> #include <linux/vmalloc.h> #include <linux/mm_inline.h> #include <linux/page_cgroup.h> #include <linux/cpu.h> #include <linux/oom.h> #include "internal.h" #include <net/sock.h> #include <net/tcp_memcontrol.h> #include <asm/uaccess.h> #include <trace/events/vmscan.h> struct cgroup_subsys mem_cgroup_subsys __read_mostly; #define MEM_CGROUP_RECLAIM_RETRIES 5 static struct mem_cgroup *root_mem_cgroup __read_mostly; #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP /* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */ int do_swap_account __read_mostly; /* for remember boot option*/ #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP_ENABLED static int really_do_swap_account __initdata = 1; #else static int really_do_swap_account __initdata = 0; #endif #else #define do_swap_account 0 #endif /* * Statistics for memory cgroup. */ enum mem_cgroup_stat_index { /* * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss. */ MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */ MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */ MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */ MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */ MEM_CGROUP_STAT_NSTATS, }; static const char * const mem_cgroup_stat_names[] = { "cache", "rss", "mapped_file", "swap", }; enum mem_cgroup_events_index { MEM_CGROUP_EVENTS_PGPGIN, /* # of pages paged in */ MEM_CGROUP_EVENTS_PGPGOUT, /* # of pages paged out */ MEM_CGROUP_EVENTS_PGFAULT, /* # of page-faults */ MEM_CGROUP_EVENTS_PGMAJFAULT, /* # of major page-faults */ MEM_CGROUP_EVENTS_NSTATS, }; static const char * const mem_cgroup_events_names[] = { "pgpgin", "pgpgout", "pgfault", "pgmajfault", }; /* * Per memcg event counter is incremented at every pagein/pageout. With THP, * it will be incremated by the number of pages. This counter is used for * for trigger some periodic events. This is straightforward and better * than using jiffies etc. to handle periodic memcg event. */ enum mem_cgroup_events_target { MEM_CGROUP_TARGET_THRESH, MEM_CGROUP_TARGET_SOFTLIMIT, MEM_CGROUP_TARGET_NUMAINFO, MEM_CGROUP_NTARGETS, }; #define THRESHOLDS_EVENTS_TARGET 128 #define SOFTLIMIT_EVENTS_TARGET 1024 #define NUMAINFO_EVENTS_TARGET 1024 struct mem_cgroup_stat_cpu { long count[MEM_CGROUP_STAT_NSTATS]; unsigned long events[MEM_CGROUP_EVENTS_NSTATS]; unsigned long nr_page_events; unsigned long targets[MEM_CGROUP_NTARGETS]; }; struct mem_cgroup_reclaim_iter { /* css_id of the last scanned hierarchy member */ int position; /* scan generation, increased every round-trip */ unsigned int generation; }; /* * per-zone information in memory controller. */ struct mem_cgroup_per_zone { struct lruvec lruvec; unsigned long lru_size[NR_LRU_LISTS]; struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1]; struct rb_node tree_node; /* RB tree node */ unsigned long long usage_in_excess;/* Set to the value by which */ /* the soft limit is exceeded*/ bool on_tree; struct mem_cgroup *memcg; /* Back pointer, we cannot */ /* use container_of */ }; struct mem_cgroup_per_node { struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES]; }; struct mem_cgroup_lru_info { struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES]; }; /* * Cgroups above their limits are maintained in a RB-Tree, independent of * their hierarchy representation */ struct mem_cgroup_tree_per_zone { struct rb_root rb_root; spinlock_t lock; }; struct mem_cgroup_tree_per_node { struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES]; }; struct mem_cgroup_tree { struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES]; }; static struct mem_cgroup_tree soft_limit_tree __read_mostly; struct mem_cgroup_threshold { struct eventfd_ctx *eventfd; u64 threshold; }; /* For threshold */ struct mem_cgroup_threshold_ary { /* An array index points to threshold just below or equal to usage. */ int current_threshold; /* Size of entries[] */ unsigned int size; /* Array of thresholds */ struct mem_cgroup_threshold entries[0]; }; struct mem_cgroup_thresholds { /* Primary thresholds array */ struct mem_cgroup_threshold_ary *primary; /* * Spare threshold array. * This is needed to make mem_cgroup_unregister_event() "never fail". * It must be able to store at least primary->size - 1 entries. */ struct mem_cgroup_threshold_ary *spare; }; /* for OOM */ struct mem_cgroup_eventfd_list { struct list_head list; struct eventfd_ctx *eventfd; }; static void mem_cgroup_threshold(struct mem_cgroup *memcg); static void mem_cgroup_oom_notify(struct mem_cgroup *memcg); /* * The memory controller data structure. The memory controller controls both * page cache and RSS per cgroup. We would eventually like to provide * statistics based on the statistics developed by Rik Van Riel for clock-pro, * to help the administrator determine what knobs to tune. * * TODO: Add a water mark for the memory controller. Reclaim will begin when * we hit the water mark. May be even add a low water mark, such that * no reclaim occurs from a cgroup at it's low water mark, this is * a feature that will be implemented much later in the future. */ struct mem_cgroup { struct cgroup_subsys_state css; /* * the counter to account for memory usage */ struct res_counter res; union { /* * the counter to account for mem+swap usage. */ struct res_counter memsw; /* * rcu_freeing is used only when freeing struct mem_cgroup, * so put it into a union to avoid wasting more memory. * It must be disjoint from the css field. It could be * in a union with the res field, but res plays a much * larger part in mem_cgroup life than memsw, and might * be of interest, even at time of free, when debugging. * So share rcu_head with the less interesting memsw. */ struct rcu_head rcu_freeing; /* * We also need some space for a worker in deferred freeing. * By the time we call it, rcu_freeing is no longer in use. */ struct work_struct work_freeing; }; /* * Per cgroup active and inactive list, similar to the * per zone LRU lists. */ struct mem_cgroup_lru_info info; int last_scanned_node; #if MAX_NUMNODES > 1 nodemask_t scan_nodes; atomic_t numainfo_events; atomic_t numainfo_updating; #endif /* * Should the accounting and control be hierarchical, per subtree? */ bool use_hierarchy; bool oom_lock; atomic_t under_oom; atomic_t refcnt; int swappiness; /* OOM-Killer disable */ int oom_kill_disable; /* set when res.limit == memsw.limit */ bool memsw_is_minimum; /* protect arrays of thresholds */ struct mutex thresholds_lock; /* thresholds for memory usage. RCU-protected */ struct mem_cgroup_thresholds thresholds; /* thresholds for mem+swap usage. RCU-protected */ struct mem_cgroup_thresholds memsw_thresholds; /* For oom notifier event fd */ struct list_head oom_notify; /* * Should we move charges of a task when a task is moved into this * mem_cgroup ? And what type of charges should we move ? */ unsigned long move_charge_at_immigrate; /* * set > 0 if pages under this cgroup are moving to other cgroup. */ atomic_t moving_account; /* taken only while moving_account > 0 */ spinlock_t move_lock; /* * percpu counter. */ struct mem_cgroup_stat_cpu __percpu *stat; /* * used when a cpu is offlined or other synchronizations * See mem_cgroup_read_stat(). */ struct mem_cgroup_stat_cpu nocpu_base; spinlock_t pcp_counter_lock; #ifdef CONFIG_INET struct tcp_memcontrol tcp_mem; #endif }; /* Stuffs for move charges at task migration. */ /* * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a * left-shifted bitmap of these types. */ enum move_type { MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */ MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */ NR_MOVE_TYPE, }; /* "mc" and its members are protected by cgroup_mutex */ static struct move_charge_struct { spinlock_t lock; /* for from, to */ struct mem_cgroup *from; struct mem_cgroup *to; unsigned long precharge; unsigned long moved_charge; unsigned long moved_swap; struct task_struct *moving_task; /* a task moving charges */ wait_queue_head_t waitq; /* a waitq for other context */ } mc = { .lock = __SPIN_LOCK_UNLOCKED(mc.lock), .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq), }; static bool move_anon(void) { return test_bit(MOVE_CHARGE_TYPE_ANON, &mc.to->move_charge_at_immigrate); } static bool move_file(void) { return test_bit(MOVE_CHARGE_TYPE_FILE, &mc.to->move_charge_at_immigrate); } /* * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft * limit reclaim to prevent infinite loops, if they ever occur. */ #define MEM_CGROUP_MAX_RECLAIM_LOOPS 100 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2 enum charge_type { MEM_CGROUP_CHARGE_TYPE_CACHE = 0, MEM_CGROUP_CHARGE_TYPE_MAPPED, MEM_CGROUP_CHARGE_TYPE_SHMEM, /* used by page migration of shmem */ MEM_CGROUP_CHARGE_TYPE_FORCE, /* used by force_empty */ MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */ MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */ NR_CHARGE_TYPE, }; /* for encoding cft->private value on file */ #define _MEM (0) #define _MEMSWAP (1) #define _OOM_TYPE (2) #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val)) #define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff) #define MEMFILE_ATTR(val) ((val) & 0xffff) /* Used for OOM nofiier */ #define OOM_CONTROL (0) /* * Reclaim flags for mem_cgroup_hierarchical_reclaim */ #define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0 #define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT) #define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1 #define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT) static void mem_cgroup_get(struct mem_cgroup *memcg); static void mem_cgroup_put(struct mem_cgroup *memcg); /* Writing them here to avoid exposing memcg's inner layout */ #ifdef CONFIG_CGROUP_MEM_RES_CTLR_KMEM #include <net/sock.h> #include <net/ip.h> static bool mem_cgroup_is_root(struct mem_cgroup *memcg); void sock_update_memcg(struct sock *sk) { if (mem_cgroup_sockets_enabled) { struct mem_cgroup *memcg; struct cg_proto *cg_proto; BUG_ON(!sk->sk_prot->proto_cgroup); /* Socket cloning can throw us here with sk_cgrp already * filled. It won't however, necessarily happen from * process context. So the test for root memcg given * the current task's memcg won't help us in this case. * * Respecting the original socket's memcg is a better * decision in this case. */ if (sk->sk_cgrp) { BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg)); mem_cgroup_get(sk->sk_cgrp->memcg); return; } rcu_read_lock(); memcg = mem_cgroup_from_task(current); cg_proto = sk->sk_prot->proto_cgroup(memcg); if (!mem_cgroup_is_root(memcg) && memcg_proto_active(cg_proto)) { mem_cgroup_get(memcg); sk->sk_cgrp = cg_proto; } rcu_read_unlock(); } } EXPORT_SYMBOL(sock_update_memcg); void sock_release_memcg(struct sock *sk) { if (mem_cgroup_sockets_enabled && sk->sk_cgrp) { struct mem_cgroup *memcg; WARN_ON(!sk->sk_cgrp->memcg); memcg = sk->sk_cgrp->memcg; mem_cgroup_put(memcg); } } #ifdef CONFIG_INET struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg) { if (!memcg || mem_cgroup_is_root(memcg)) return NULL; return &memcg->tcp_mem.cg_proto; } EXPORT_SYMBOL(tcp_proto_cgroup); #endif /* CONFIG_INET */ #endif /* CONFIG_CGROUP_MEM_RES_CTLR_KMEM */ #if defined(CONFIG_INET) && defined(CONFIG_CGROUP_MEM_RES_CTLR_KMEM) static void disarm_sock_keys(struct mem_cgroup *memcg) { if (!memcg_proto_activated(&memcg->tcp_mem.cg_proto)) return; static_key_slow_dec(&memcg_socket_limit_enabled); } #else static void disarm_sock_keys(struct mem_cgroup *memcg) { } #endif static void drain_all_stock_async(struct mem_cgroup *memcg); static struct mem_cgroup_per_zone * mem_cgroup_zoneinfo(struct mem_cgroup *memcg, int nid, int zid) { return &memcg->info.nodeinfo[nid]->zoneinfo[zid]; } struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg) { return &memcg->css; } static struct mem_cgroup_per_zone * page_cgroup_zoneinfo(struct mem_cgroup *memcg, struct page *page) {