7064 lines · 187.6 KB · C
RawDownload
/* memcontrol.c - Memory Controller * * Copyright IBM Corporation, 2007 * Author Balbir Singh <balbir@linux.vnet.ibm.com> * * Copyright 2007 OpenVZ SWsoft Inc * Author: Pavel Emelianov <xemul@openvz.org> * * Memory thresholds * Copyright (C) 2009 Nokia Corporation * Author: Kirill A. Shutemov * * Kernel Memory Controller * Copyright (C) 2012 Parallels Inc. and Google Inc. * Authors: Glauber Costa and Suleiman Souhlal * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the * GNU General Public License for more details. */ #include <linux/res_counter.h>#include <linux/memcontrol.h>#include <linux/cgroup.h>#include <linux/mm.h>#include <linux/hugetlb.h>#include <linux/pagemap.h>#include <linux/smp.h>#include <linux/page-flags.h>#include <linux/backing-dev.h>#include <linux/bit_spinlock.h>#include <linux/rcupdate.h>#include <linux/limits.h>#include <linux/export.h>#include <linux/mutex.h>#include <linux/rbtree.h>#include <linux/slab.h>#include <linux/swap.h>#include <linux/swapops.h>#include <linux/spinlock.h>#include <linux/eventfd.h>#include <linux/sort.h>#include <linux/fs.h>#include <linux/seq_file.h>#include <linux/vmalloc.h>#include <linux/vmpressure.h>#include <linux/mm_inline.h>#include <linux/page_cgroup.h>#include <linux/cpu.h>#include <linux/oom.h>#include <linux/lockdep.h>#include "internal.h"#include <net/sock.h>#include <net/ip.h>#include <net/tcp_memcontrol.h>#include "slab.h" #include <asm/uaccess.h> #include <trace/events/vmscan.h> struct cgroup_subsys mem_cgroup_subsys __read_mostly;EXPORT_SYMBOL(mem_cgroup_subsys); #define MEM_CGROUP_RECLAIM_RETRIES	5static struct mem_cgroup *root_mem_cgroup __read_mostly; #ifdef CONFIG_MEMCG_SWAP/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */int do_swap_account __read_mostly; /* for remember boot option*/#ifdef CONFIG_MEMCG_SWAP_ENABLEDstatic int really_do_swap_account __initdata = 1;#elsestatic int really_do_swap_account __initdata = 0;#endif #else#define do_swap_account		0#endif  static const char * const mem_cgroup_stat_names[] = {	"cache",	"rss",	"rss_huge",	"mapped_file",	"writeback",	"swap",}; enum mem_cgroup_events_index {	MEM_CGROUP_EVENTS_PGPGIN,	/* # of pages paged in */	MEM_CGROUP_EVENTS_PGPGOUT,	/* # of pages paged out */	MEM_CGROUP_EVENTS_PGFAULT,	/* # of page-faults */	MEM_CGROUP_EVENTS_PGMAJFAULT,	/* # of major page-faults */	MEM_CGROUP_EVENTS_NSTATS,}; static const char * const mem_cgroup_events_names[] = {	"pgpgin",	"pgpgout",	"pgfault",	"pgmajfault",}; static const char * const mem_cgroup_lru_names[] = {	"inactive_anon",	"active_anon",	"inactive_file",	"active_file",	"unevictable",}; /* * Per memcg event counter is incremented at every pagein/pageout. With THP, * it will be incremated by the number of pages. This counter is used for * for trigger some periodic events. This is straightforward and better * than using jiffies etc. to handle periodic memcg event. */enum mem_cgroup_events_target {	MEM_CGROUP_TARGET_THRESH,	MEM_CGROUP_TARGET_SOFTLIMIT,	MEM_CGROUP_TARGET_NUMAINFO,	MEM_CGROUP_NTARGETS,};#define THRESHOLDS_EVENTS_TARGET 128#define SOFTLIMIT_EVENTS_TARGET 1024#define NUMAINFO_EVENTS_TARGET	1024 struct mem_cgroup_stat_cpu {	long count[MEM_CGROUP_STAT_NSTATS];	unsigned long events[MEM_CGROUP_EVENTS_NSTATS];	unsigned long nr_page_events;	unsigned long targets[MEM_CGROUP_NTARGETS];}; struct mem_cgroup_reclaim_iter {	/*	 * last scanned hierarchy member. Valid only if last_dead_count	 * matches memcg->dead_count of the hierarchy root group.	 */	struct mem_cgroup *last_visited;	unsigned long last_dead_count; 	/* scan generation, increased every round-trip */	unsigned int generation;}; /* * per-zone information in memory controller. */struct mem_cgroup_per_zone {	struct lruvec		lruvec;	unsigned long		lru_size[NR_LRU_LISTS]; 	struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1]; 	struct rb_node		tree_node;	/* RB tree node */	unsigned long long	usage_in_excess;/* Set to the value by which */						/* the soft limit is exceeded*/	bool			on_tree;	struct mem_cgroup	*memcg;		/* Back pointer, we cannot */						/* use container_of	   */}; struct mem_cgroup_per_node {	struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];}; /* * Cgroups above their limits are maintained in a RB-Tree, independent of * their hierarchy representation */ struct mem_cgroup_tree_per_zone {	struct rb_root rb_root;	spinlock_t lock;}; struct mem_cgroup_tree_per_node {	struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];}; struct mem_cgroup_tree {	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];}; static struct mem_cgroup_tree soft_limit_tree __read_mostly; struct mem_cgroup_threshold {	struct eventfd_ctx *eventfd;	u64 threshold;}; /* For threshold */struct mem_cgroup_threshold_ary {	/* An array index points to threshold just below or equal to usage. */	int current_threshold;	/* Size of entries[] */	unsigned int size;	/* Array of thresholds */	struct mem_cgroup_threshold entries[0];}; struct mem_cgroup_thresholds {	/* Primary thresholds array */	struct mem_cgroup_threshold_ary *primary;	/*	 * Spare threshold array.	 * This is needed to make mem_cgroup_unregister_event() "never fail".	 * It must be able to store at least primary->size - 1 entries.	 */	struct mem_cgroup_threshold_ary *spare;}; /* for OOM */struct mem_cgroup_eventfd_list {	struct list_head list;	struct eventfd_ctx *eventfd;}; static void mem_cgroup_threshold(struct mem_cgroup *memcg);static void mem_cgroup_oom_notify(struct mem_cgroup *memcg); /* * The memory controller data structure. The memory controller controls both * page cache and RSS per cgroup. We would eventually like to provide * statistics based on the statistics developed by Rik Van Riel for clock-pro, * to help the administrator determine what knobs to tune. * * TODO: Add a water mark for the memory controller. Reclaim will begin when * we hit the water mark. May be even add a low water mark, such that * no reclaim occurs from a cgroup at it's low water mark, this is * a feature that will be implemented much later in the future. */struct mem_cgroup {	struct cgroup_subsys_state css;	/*	 * the counter to account for memory usage	 */	struct res_counter res; 	/* vmpressure notifications */	struct vmpressure vmpressure; 	/*	 * the counter to account for mem+swap usage.	 */	struct res_counter memsw; 	/*	 * the counter to account for kernel memory usage.	 */	struct res_counter kmem;	/*	 * Should the accounting and control be hierarchical, per subtree?	 */	bool use_hierarchy;	unsigned long kmem_account_flags; /* See KMEM_ACCOUNTED_*, below */ 	bool		oom_lock;	atomic_t	under_oom;	atomic_t	oom_wakeups; 	int	swappiness;	/* OOM-Killer disable */	int		oom_kill_disable; 	/* set when res.limit == memsw.limit */	bool		memsw_is_minimum; 	/* protect arrays of thresholds */	struct mutex thresholds_lock; 	/* thresholds for memory usage. RCU-protected */	struct mem_cgroup_thresholds thresholds; 	/* thresholds for mem+swap usage. RCU-protected */	struct mem_cgroup_thresholds memsw_thresholds; 	/* For oom notifier event fd */	struct list_head oom_notify; 	/*	 * Should we move charges of a task when a task is moved into this	 * mem_cgroup ? And what type of charges should we move ?	 */	unsigned long move_charge_at_immigrate;	/*	 * set > 0 if pages under this cgroup are moving to other cgroup.	 */	atomic_t	moving_account;	/* taken only while moving_account > 0 */	spinlock_t	move_lock;	/*	 * percpu counter.	 */	struct mem_cgroup_stat_cpu __percpu *stat;	/*	 * used when a cpu is offlined or other synchronizations	 * See mem_cgroup_read_stat().	 */	struct mem_cgroup_stat_cpu nocpu_base;	spinlock_t pcp_counter_lock; 	atomic_t	dead_count;#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)	struct cg_proto tcp_mem;#endif#if defined(CONFIG_MEMCG_KMEM)	/* analogous to slab_common's slab_caches list. per-memcg */	struct list_head memcg_slab_caches;	/* Not a spinlock, we can take a lot of time walking the list */	struct mutex slab_caches_mutex;        /* Index in the kmem_cache->memcg_params->memcg_caches array */	int kmemcg_id;#endif 	int last_scanned_node;#if MAX_NUMNODES > 1	nodemask_t	scan_nodes;	atomic_t	numainfo_events;	atomic_t	numainfo_updating;#endif 	struct mem_cgroup_per_node *nodeinfo[0];	/* WARNING: nodeinfo must be the last member here */}; static size_t memcg_size(void){	return sizeof(struct mem_cgroup) +		nr_node_ids * sizeof(struct mem_cgroup_per_node *);} /* internal only representation about the status of kmem accounting. */enum {	KMEM_ACCOUNTED_ACTIVE = 0, /* accounted by this cgroup itself */	KMEM_ACCOUNTED_ACTIVATED, /* static key enabled. */	KMEM_ACCOUNTED_DEAD, /* dead memcg with pending kmem charges */}; /* We account when limit is on, but only after call sites are patched */#define KMEM_ACCOUNTED_MASK \		((1 << KMEM_ACCOUNTED_ACTIVE) | (1 << KMEM_ACCOUNTED_ACTIVATED)) #ifdef CONFIG_MEMCG_KMEMstatic inline void memcg_kmem_set_active(struct mem_cgroup *memcg){	set_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);} static bool memcg_kmem_is_active(struct mem_cgroup *memcg){	return test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);} static void memcg_kmem_set_activated(struct mem_cgroup *memcg){	set_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);} static void memcg_kmem_clear_activated(struct mem_cgroup *memcg){	clear_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);} static void memcg_kmem_mark_dead(struct mem_cgroup *memcg){	/*	 * Our caller must use css_get() first, because memcg_uncharge_kmem()	 * will call css_put() if it sees the memcg is dead.	 */	smp_wmb();	if (test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags))		set_bit(KMEM_ACCOUNTED_DEAD, &memcg->kmem_account_flags);} static bool memcg_kmem_test_and_clear_dead(struct mem_cgroup *memcg){	return test_and_clear_bit(KMEM_ACCOUNTED_DEAD,				  &memcg->kmem_account_flags);}#endif /* Stuffs for move charges at task migration. *//* * Types of charges to be moved. "move_charge_at_immitgrate" and * "immigrate_flags" are treated as a left-shifted bitmap of these types. */enum move_type {	MOVE_CHARGE_TYPE_ANON,	/* private anonymous page and swap of it */	MOVE_CHARGE_TYPE_FILE,	/* file page(including tmpfs) and swap of it */	NR_MOVE_TYPE,}; /* "mc" and its members are protected by cgroup_mutex */static struct move_charge_struct {	spinlock_t	  lock; /* for from, to */	struct mem_cgroup *from;	struct mem_cgroup *to;	unsigned long immigrate_flags;	unsigned long precharge;	unsigned long moved_charge;	unsigned long moved_swap;	struct task_struct *moving_task;	/* a task moving charges */	wait_queue_head_t waitq;		/* a waitq for other context */} mc = {	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),}; static bool move_anon(void){	return test_bit(MOVE_CHARGE_TYPE_ANON, &mc.immigrate_flags);} static bool move_file(void){	return test_bit(MOVE_CHARGE_TYPE_FILE, &mc.immigrate_flags);} /* * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft * limit reclaim to prevent infinite loops, if they ever occur. */#define	MEM_CGROUP_MAX_RECLAIM_LOOPS		100#define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	2 enum charge_type {	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,	MEM_CGROUP_CHARGE_TYPE_ANON,	MEM_CGROUP_CHARGE_TYPE_SWAPOUT,	/* for accounting swapcache */	MEM_CGROUP_CHARGE_TYPE_DROP,	/* a page was unused swap cache */	NR_CHARGE_TYPE,}; /* for encoding cft->private value on file */enum res_type {	_MEM,	_MEMSWAP,	_OOM_TYPE,	_KMEM,}; #define MEMFILE_PRIVATE(x, val)	((x) << 16 | (val))#define MEMFILE_TYPE(val)	((val) >> 16 & 0xffff)#define MEMFILE_ATTR(val)	((val) & 0xffff)/* Used for OOM nofiier */#define OOM_CONTROL		(0) /* * Reclaim flags for mem_cgroup_hierarchical_reclaim */#define MEM_CGROUP_RECLAIM_NOSWAP_BIT	0x0#define MEM_CGROUP_RECLAIM_NOSWAP	(1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)#define MEM_CGROUP_RECLAIM_SHRINK_BIT	0x1#define MEM_CGROUP_RECLAIM_SHRINK	(1 << MEM_CGROUP_RECLAIM_SHRINK_BIT) /* * The memcg_create_mutex will be held whenever a new cgroup is created. * As a consequence, any change that needs to protect against new child cgroups * appearing has to hold it as well. */static DEFINE_MUTEX(memcg_create_mutex); struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *s){	return s ? container_of(s, struct mem_cgroup, css) : NULL;} /* Some nice accessors for the vmpressure. */struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg){	if (!memcg)		memcg = root_mem_cgroup;	return &memcg->vmpressure;} struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr){	return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;} struct vmpressure *css_to_vmpressure(struct cgroup_subsys_state *css){	return &mem_cgroup_from_css(css)->vmpressure;} static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg){	return (memcg == root_mem_cgroup);

Lines 1–500 of 7064.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t