/* *  kernel/cpuset.c * *  Processor and Memory placement constraints for sets of tasks. * *  Copyright (C) 2003 BULL SA. *  Copyright (C) 2004-2007 Silicon Graphics, Inc. *  Copyright (C) 2006 Google, Inc * *  Portions derived from Patrick Mochel's sysfs code. *  sysfs is Copyright (c) 2001-3 Patrick Mochel * *  2003-10-10 Written by Simon Derr. *  2003-10-22 Updates by Stephen Hemminger. *  2004 May-July Rework by Paul Jackson. *  2006 Rework by Paul Menage to use generic cgroups *  2008 Rework of the scheduler domains and CPU hotplug handling *       by Max Krasnyansky * *  This file is subject to the terms and conditions of the GNU General Public *  License.  See the file COPYING in the main directory of the Linux *  distribution for more details. */ #include <linux/cpu.h>#include <linux/cpumask.h>#include <linux/cpuset.h>#include <linux/err.h>#include <linux/errno.h>#include <linux/file.h>#include <linux/fs.h>#include <linux/init.h>#include <linux/interrupt.h>#include <linux/kernel.h>#include <linux/kmod.h>#include <linux/list.h>#include <linux/mempolicy.h>#include <linux/mm.h>#include <linux/memory.h>#include <linux/export.h>#include <linux/mount.h>#include <linux/namei.h>#include <linux/pagemap.h>#include <linux/proc_fs.h>#include <linux/rcupdate.h>#include <linux/sched.h>#include <linux/seq_file.h>#include <linux/security.h>#include <linux/slab.h>#include <linux/spinlock.h>#include <linux/stat.h>#include <linux/string.h>#include <linux/time.h>#include <linux/backing-dev.h>#include <linux/sort.h> #include <asm/uaccess.h>#include <linux/atomic.h>#include <linux/mutex.h>#include <linux/workqueue.h>#include <linux/cgroup.h> /* * Workqueue for cpuset related tasks. * * Using kevent workqueue may cause deadlock when memory_migrate * is set. So we create a separate workqueue thread for cpuset. */static struct workqueue_struct *cpuset_wq; /* * Tracks how many cpusets are currently defined in system. * When there is only one cpuset (the root cpuset) we can * short circuit some hooks. */int number_of_cpusets __read_mostly; /* Forward declare cgroup structures */struct cgroup_subsys cpuset_subsys;struct cpuset; /* See "Frequency meter" comments, below. */ struct fmeter {	int cnt;		/* unprocessed events count */	int val;		/* most recent output value */	time_t time;		/* clock (secs) when val computed */	spinlock_t lock;	/* guards read or write of above */}; struct cpuset {	struct cgroup_subsys_state css; 	unsigned long flags;		/* "unsigned long" so bitops work */	cpumask_var_t cpus_allowed;	/* CPUs allowed to tasks in cpuset */	nodemask_t mems_allowed;	/* Memory Nodes allowed to tasks */ 	struct cpuset *parent;		/* my parent */ 	struct fmeter fmeter;		/* memory_pressure filter */ 	/* partition number for rebuild_sched_domains() */	int pn; 	/* for custom sched domain */	int relax_domain_level; 	/* used for walking a cpuset hierarchy */	struct list_head stack_list;}; /* Retrieve the cpuset for a cgroup */static inline struct cpuset *cgroup_cs(struct cgroup *cont){	return container_of(cgroup_subsys_state(cont, cpuset_subsys_id),			    struct cpuset, css);} /* Retrieve the cpuset for a task */static inline struct cpuset *task_cs(struct task_struct *task){	return container_of(task_subsys_state(task, cpuset_subsys_id),			    struct cpuset, css);} #ifdef CONFIG_NUMAstatic inline bool task_has_mempolicy(struct task_struct *task){	return task->mempolicy;}#elsestatic inline bool task_has_mempolicy(struct task_struct *task){	return false;}#endif  /* bits in struct cpuset flags field */typedef enum {	CS_CPU_EXCLUSIVE,	CS_MEM_EXCLUSIVE,	CS_MEM_HARDWALL,	CS_MEMORY_MIGRATE,	CS_SCHED_LOAD_BALANCE,	CS_SPREAD_PAGE,	CS_SPREAD_SLAB,} cpuset_flagbits_t; /* the type of hotplug event */enum hotplug_event {	CPUSET_CPU_OFFLINE,	CPUSET_MEM_OFFLINE,}; /* convenient tests for these bits */static inline int is_cpu_exclusive(const struct cpuset *cs){	return test_bit(CS_CPU_EXCLUSIVE, &cs->flags);} static inline int is_mem_exclusive(const struct cpuset *cs){	return test_bit(CS_MEM_EXCLUSIVE, &cs->flags);} static inline int is_mem_hardwall(const struct cpuset *cs){	return test_bit(CS_MEM_HARDWALL, &cs->flags);} static inline int is_sched_load_balance(const struct cpuset *cs){	return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);} static inline int is_memory_migrate(const struct cpuset *cs){	return test_bit(CS_MEMORY_MIGRATE, &cs->flags);} static inline int is_spread_page(const struct cpuset *cs){	return test_bit(CS_SPREAD_PAGE, &cs->flags);} static inline int is_spread_slab(const struct cpuset *cs){	return test_bit(CS_SPREAD_SLAB, &cs->flags);} static struct cpuset top_cpuset = {	.flags = ((1 << CS_CPU_EXCLUSIVE) | (1 << CS_MEM_EXCLUSIVE)),}; /* * There are two global mutexes guarding cpuset structures.  The first * is the main control groups cgroup_mutex, accessed via * cgroup_lock()/cgroup_unlock().  The second is the cpuset-specific * callback_mutex, below. They can nest.  It is ok to first take * cgroup_mutex, then nest callback_mutex.  We also require taking * task_lock() when dereferencing a task's cpuset pointer.  See "The * task_lock() exception", at the end of this comment. * * A task must hold both mutexes to modify cpusets.  If a task * holds cgroup_mutex, then it blocks others wanting that mutex, * ensuring that it is the only task able to also acquire callback_mutex * and be able to modify cpusets.  It can perform various checks on * the cpuset structure first, knowing nothing will change.  It can * also allocate memory while just holding cgroup_mutex.  While it is * performing these checks, various callback routines can briefly * acquire callback_mutex to query cpusets.  Once it is ready to make * the changes, it takes callback_mutex, blocking everyone else. * * Calls to the kernel memory allocator can not be made while holding * callback_mutex, as that would risk double tripping on callback_mutex * from one of the callbacks into the cpuset code from within * __alloc_pages(). * * If a task is only holding callback_mutex, then it has read-only * access to cpusets. * * Now, the task_struct fields mems_allowed and mempolicy may be changed * by other task, we use alloc_lock in the task_struct fields to protect * them. * * The cpuset_common_file_read() handlers only hold callback_mutex across * small pieces of code, such as when reading out possibly multi-word * cpumasks and nodemasks. * * Accessing a task's cpuset should be done in accordance with the * guidelines for accessing subsystem state in kernel/cgroup.c */ static DEFINE_MUTEX(callback_mutex); /* * cpuset_buffer_lock protects both the cpuset_name and cpuset_nodelist * buffers.  They are statically allocated to prevent using excess stack * when calling cpuset_print_task_mems_allowed(). */#define CPUSET_NAME_LEN		(128)#define	CPUSET_NODELIST_LEN	(256)static char cpuset_name[CPUSET_NAME_LEN];static char cpuset_nodelist[CPUSET_NODELIST_LEN];static DEFINE_SPINLOCK(cpuset_buffer_lock); /* * This is ugly, but preserves the userspace API for existing cpuset * users. If someone tries to mount the "cpuset" filesystem, we * silently switch it to mount "cgroup" instead */static struct dentry *cpuset_mount(struct file_system_type *fs_type,			 int flags, const char *unused_dev_name, void *data){	struct file_system_type *cgroup_fs = get_fs_type("cgroup");	struct dentry *ret = ERR_PTR(-ENODEV);	if (cgroup_fs) {		char mountopts[] =			"cpuset,noprefix,"			"release_agent=/sbin/cpuset_release_agent";		ret = cgroup_fs->mount(cgroup_fs, flags,					   unused_dev_name, mountopts);		put_filesystem(cgroup_fs);	}	return ret;} static struct file_system_type cpuset_fs_type = {	.name = "cpuset",	.mount = cpuset_mount,}; /* * Return in pmask the portion of a cpusets's cpus_allowed that * are online.  If none are online, walk up the cpuset hierarchy * until we find one that does have some online cpus.  If we get * all the way to the top and still haven't found any online cpus, * return cpu_online_mask.  Or if passed a NULL cs from an exit'ing * task, return cpu_online_mask. * * One way or another, we guarantee to return some non-empty subset * of cpu_online_mask. * * Call with callback_mutex held. */ static void guarantee_online_cpus(const struct cpuset *cs,				  struct cpumask *pmask){	while (cs && !cpumask_intersects(cs->cpus_allowed, cpu_online_mask))		cs = cs->parent;	if (cs)		cpumask_and(pmask, cs->cpus_allowed, cpu_online_mask);	else		cpumask_copy(pmask, cpu_online_mask);	BUG_ON(!cpumask_intersects(pmask, cpu_online_mask));} /* * Return in *pmask the portion of a cpusets's mems_allowed that * are online, with memory.  If none are online with memory, walk * up the cpuset hierarchy until we find one that does have some * online mems.  If we get all the way to the top and still haven't * found any online mems, return node_states[N_HIGH_MEMORY]. * * One way or another, we guarantee to return some non-empty subset * of node_states[N_HIGH_MEMORY]. * * Call with callback_mutex held. */ static void guarantee_online_mems(const struct cpuset *cs, nodemask_t *pmask){	while (cs && !nodes_intersects(cs->mems_allowed,					node_states[N_HIGH_MEMORY]))		cs = cs->parent;	if (cs)		nodes_and(*pmask, cs->mems_allowed,					node_states[N_HIGH_MEMORY]);	else		*pmask = node_states[N_HIGH_MEMORY];	BUG_ON(!nodes_intersects(*pmask, node_states[N_HIGH_MEMORY]));} /* * update task's spread flag if cpuset's page/slab spread flag is set * * Called with callback_mutex/cgroup_mutex held */static void cpuset_update_task_spread_flag(struct cpuset *cs,					struct task_struct *tsk){	if (is_spread_page(cs))		tsk->flags |= PF_SPREAD_PAGE;	else		tsk->flags &= ~PF_SPREAD_PAGE;	if (is_spread_slab(cs))		tsk->flags |= PF_SPREAD_SLAB;	else		tsk->flags &= ~PF_SPREAD_SLAB;} /* * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q? * * One cpuset is a subset of another if all its allowed CPUs and * Memory Nodes are a subset of the other, and its exclusive flags * are only set if the other's are set.  Call holding cgroup_mutex. */ static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q){	return	cpumask_subset(p->cpus_allowed, q->cpus_allowed) &&		nodes_subset(p->mems_allowed, q->mems_allowed) &&		is_cpu_exclusive(p) <= is_cpu_exclusive(q) &&		is_mem_exclusive(p) <= is_mem_exclusive(q);} /** * alloc_trial_cpuset - allocate a trial cpuset * @cs: the cpuset that the trial cpuset duplicates */static struct cpuset *alloc_trial_cpuset(const struct cpuset *cs){	struct cpuset *trial; 	trial = kmemdup(cs, sizeof(*cs), GFP_KERNEL);	if (!trial)		return NULL; 	if (!alloc_cpumask_var(&trial->cpus_allowed, GFP_KERNEL)) {		kfree(trial);		return NULL;	}	cpumask_copy(trial->cpus_allowed, cs->cpus_allowed); 	return trial;} /** * free_trial_cpuset - free the trial cpuset * @trial: the trial cpuset to be freed */static void free_trial_cpuset(struct cpuset *trial){	free_cpumask_var(trial->cpus_allowed);	kfree(trial);} /* * validate_change() - Used to validate that any proposed cpuset change *		       follows the structural rules for cpusets. * * If we replaced the flag and mask values of the current cpuset * (cur) with those values in the trial cpuset (trial), would * our various subset and exclusive rules still be valid?  Presumes * cgroup_mutex held. * * 'cur' is the address of an actual, in-use cpuset.  Operations * such as list traversal that depend on the actual address of the * cpuset in the list must use cur below, not trial. * * 'trial' is the address of bulk structure copy of cur, with * perhaps one or more of the fields cpus_allowed, mems_allowed, * or flags changed to new, trial values. * * Return 0 if valid, -errno if not. */ static int validate_change(const struct cpuset *cur, const struct cpuset *trial){	struct cgroup *cont;	struct cpuset *c, *par; 	/* Each of our child cpusets must be a subset of us */	list_for_each_entry(cont, &cur->css.cgroup->children, sibling) {		if (!is_cpuset_subset(cgroup_cs(cont), trial))			return -EBUSY;	} 	/* Remaining checks don't apply to root cpuset */	if (cur == &top_cpuset)		return 0; 	par = cur->parent; 	/* We must be a subset of our parent cpuset */	if (!is_cpuset_subset(trial, par))		return -EACCES; 	/*	 * If either I or some sibling (!= me) is exclusive, we can't	 * overlap	 */	list_for_each_entry(cont, &par->css.cgroup->children, sibling) {		c = cgroup_cs(cont);		if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) &&		    c != cur &&		    cpumask_intersects(trial->cpus_allowed, c->cpus_allowed))			return -EINVAL;		if ((is_mem_exclusive(trial) || is_mem_exclusive(c)) &&		    c != cur &&		    nodes_intersects(trial->mems_allowed, c->mems_allowed))			return -EINVAL;	} 	/* Cpusets with tasks can't have empty cpus_allowed or mems_allowed */	if (cgroup_task_count(cur->css.cgroup)) {		if (cpumask_empty(trial->cpus_allowed) ||		    nodes_empty(trial->mems_allowed)) {			return -ENOSPC;		}	} 	return 0;} #ifdef CONFIG_SMP/* * Helper routine for generate_sched_domains(). * Do cpusets a, b have overlapping cpus_allowed masks? */static int cpusets_overlap(struct cpuset *a, struct cpuset *b){	return cpumask_intersects(a->cpus_allowed, b->cpus_allowed);} static voidupdate_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c){	if (dattr->relax_domain_level < c->relax_domain_level)		dattr->relax_domain_level = c->relax_domain_level;	return;} static voidupdate_domain_attr_tree(struct sched_domain_attr *dattr, struct cpuset *c){	LIST_HEAD(q); 	list_add(&c->stack_list, &q);	while (!list_empty(&q)) {		struct cpuset *cp;		struct cgroup *cont;		struct cpuset *child; 		cp = list_first_entry(&q, struct cpuset, stack_list);		list_del(q.next); 		if (cpumask_empty(cp->cpus_allowed))			continue; 		if (is_sched_load_balance(cp))			update_domain_attr(dattr, cp); 		list_for_each_entry(cont, &cp->css.cgroup->children, sibling) {			child = cgroup_cs(cont);			list_add_tail(&child->stack_list, &q);		}

Lines 1–500 of 2633. Load more