/* *  kernel/cpuset.c * *  Processor and Memory placement constraints for sets of tasks. * *  Copyright (C) 2003 BULL SA. *  Copyright (C) 2004-2007 Silicon Graphics, Inc. *  Copyright (C) 2006 Google, Inc * *  Portions derived from Patrick Mochel's sysfs code. *  sysfs is Copyright (c) 2001-3 Patrick Mochel * *  2003-10-10 Written by Simon Derr. *  2003-10-22 Updates by Stephen Hemminger. *  2004 May-July Rework by Paul Jackson. *  2006 Rework by Paul Menage to use generic cgroups *  2008 Rework of the scheduler domains and CPU hotplug handling *       by Max Krasnyansky * *  This file is subject to the terms and conditions of the GNU General Public *  License.  See the file COPYING in the main directory of the Linux *  distribution for more details. */ #include <linux/cpu.h>#include <linux/cpumask.h>#include <linux/cpuset.h>#include <linux/err.h>#include <linux/errno.h>#include <linux/file.h>#include <linux/fs.h>#include <linux/init.h>#include <linux/interrupt.h>#include <linux/kernel.h>#include <linux/kmod.h>#include <linux/list.h>#include <linux/mempolicy.h>#include <linux/mm.h>#include <linux/memory.h>#include <linux/export.h>#include <linux/mount.h>#include <linux/fs_context.h>#include <linux/namei.h>#include <linux/pagemap.h>#include <linux/proc_fs.h>#include <linux/rcupdate.h>#include <linux/sched.h>#include <linux/sched/deadline.h>#include <linux/sched/mm.h>#include <linux/sched/task.h>#include <linux/seq_file.h>#include <linux/security.h>#include <linux/slab.h>#include <linux/spinlock.h>#include <linux/stat.h>#include <linux/string.h>#include <linux/time.h>#include <linux/time64.h>#include <linux/backing-dev.h>#include <linux/sort.h>#include <linux/oom.h>#include <linux/sched/isolation.h>#include <linux/uaccess.h>#include <linux/atomic.h>#include <linux/mutex.h>#include <linux/cgroup.h>#include <linux/wait.h> DEFINE_STATIC_KEY_FALSE(cpusets_pre_enable_key);DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key); /* See "Frequency meter" comments, below. */ struct fmeter {	int cnt;		/* unprocessed events count */	int val;		/* most recent output value */	time64_t time;		/* clock (secs) when val computed */	spinlock_t lock;	/* guards read or write of above */}; struct cpuset {	struct cgroup_subsys_state css; 	unsigned long flags;		/* "unsigned long" so bitops work */ 	/*	 * On default hierarchy:	 *	 * The user-configured masks can only be changed by writing to	 * cpuset.cpus and cpuset.mems, and won't be limited by the	 * parent masks.	 *	 * The effective masks is the real masks that apply to the tasks	 * in the cpuset. They may be changed if the configured masks are	 * changed or hotplug happens.	 *	 * effective_mask == configured_mask & parent's effective_mask,	 * and if it ends up empty, it will inherit the parent's mask.	 *	 *	 * On legacy hierachy:	 *	 * The user-configured masks are always the same with effective masks.	 */ 	/* user-configured CPUs and Memory Nodes allow to tasks */	cpumask_var_t cpus_allowed;	nodemask_t mems_allowed; 	/* effective CPUs and Memory Nodes allow to tasks */	cpumask_var_t effective_cpus;	nodemask_t effective_mems; 	/*	 * CPUs allocated to child sub-partitions (default hierarchy only)	 * - CPUs granted by the parent = effective_cpus U subparts_cpus	 * - effective_cpus and subparts_cpus are mutually exclusive.	 *	 * effective_cpus contains only onlined CPUs, but subparts_cpus	 * may have offlined ones.	 */	cpumask_var_t subparts_cpus; 	/*	 * This is old Memory Nodes tasks took on.	 *	 * - top_cpuset.old_mems_allowed is initialized to mems_allowed.	 * - A new cpuset's old_mems_allowed is initialized when some	 *   task is moved into it.	 * - old_mems_allowed is used in cpuset_migrate_mm() when we change	 *   cpuset.mems_allowed and have tasks' nodemask updated, and	 *   then old_mems_allowed is updated to mems_allowed.	 */	nodemask_t old_mems_allowed; 	struct fmeter fmeter;		/* memory_pressure filter */ 	/*	 * Tasks are being attached to this cpuset.  Used to prevent	 * zeroing cpus/mems_allowed between ->can_attach() and ->attach().	 */	int attach_in_progress; 	/* partition number for rebuild_sched_domains() */	int pn; 	/* for custom sched domain */	int relax_domain_level; 	/* number of CPUs in subparts_cpus */	int nr_subparts_cpus; 	/* partition root state */	int partition_root_state; 	/*	 * Default hierarchy only:	 * use_parent_ecpus - set if using parent's effective_cpus	 * child_ecpus_count - # of children with use_parent_ecpus set	 */	int use_parent_ecpus;	int child_ecpus_count;}; /* * Partition root states: * *   0 - not a partition root * *   1 - partition root * *  -1 - invalid partition root *       None of the cpus in cpus_allowed can be put into the parent's *       subparts_cpus. In this case, the cpuset is not a real partition *       root anymore.  However, the CPU_EXCLUSIVE bit will still be set *       and the cpuset can be restored back to a partition root if the *       parent cpuset can give more CPUs back to this child cpuset. */#define PRS_DISABLED		0#define PRS_ENABLED		1#define PRS_ERROR		-1 /* * Temporary cpumasks for working with partitions that are passed among * functions to avoid memory allocation in inner functions. */struct tmpmasks {	cpumask_var_t addmask, delmask;	/* For partition root */	cpumask_var_t new_cpus;		/* For update_cpumasks_hier() */}; static inline struct cpuset *css_cs(struct cgroup_subsys_state *css){	return css ? container_of(css, struct cpuset, css) : NULL;} /* Retrieve the cpuset for a task */static inline struct cpuset *task_cs(struct task_struct *task){	return css_cs(task_css(task, cpuset_cgrp_id));} static inline struct cpuset *parent_cs(struct cpuset *cs){	return css_cs(cs->css.parent);} /* bits in struct cpuset flags field */typedef enum {	CS_ONLINE,	CS_CPU_EXCLUSIVE,	CS_MEM_EXCLUSIVE,	CS_MEM_HARDWALL,	CS_MEMORY_MIGRATE,	CS_SCHED_LOAD_BALANCE,	CS_SPREAD_PAGE,	CS_SPREAD_SLAB,} cpuset_flagbits_t; /* convenient tests for these bits */static inline bool is_cpuset_online(struct cpuset *cs){	return test_bit(CS_ONLINE, &cs->flags) && !css_is_dying(&cs->css);} static inline int is_cpu_exclusive(const struct cpuset *cs){	return test_bit(CS_CPU_EXCLUSIVE, &cs->flags);} static inline int is_mem_exclusive(const struct cpuset *cs){	return test_bit(CS_MEM_EXCLUSIVE, &cs->flags);} static inline int is_mem_hardwall(const struct cpuset *cs){	return test_bit(CS_MEM_HARDWALL, &cs->flags);} static inline int is_sched_load_balance(const struct cpuset *cs){	return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);} static inline int is_memory_migrate(const struct cpuset *cs){	return test_bit(CS_MEMORY_MIGRATE, &cs->flags);} static inline int is_spread_page(const struct cpuset *cs){	return test_bit(CS_SPREAD_PAGE, &cs->flags);} static inline int is_spread_slab(const struct cpuset *cs){	return test_bit(CS_SPREAD_SLAB, &cs->flags);} static inline int is_partition_root(const struct cpuset *cs){	return cs->partition_root_state > 0;} static struct cpuset top_cpuset = {	.flags = ((1 << CS_ONLINE) | (1 << CS_CPU_EXCLUSIVE) |		  (1 << CS_MEM_EXCLUSIVE)),	.partition_root_state = PRS_ENABLED,}; /** * cpuset_for_each_child - traverse online children of a cpuset * @child_cs: loop cursor pointing to the current child * @pos_css: used for iteration * @parent_cs: target cpuset to walk children of * * Walk @child_cs through the online children of @parent_cs.  Must be used * with RCU read locked. */#define cpuset_for_each_child(child_cs, pos_css, parent_cs)		\	css_for_each_child((pos_css), &(parent_cs)->css)		\		if (is_cpuset_online(((child_cs) = css_cs((pos_css))))) /** * cpuset_for_each_descendant_pre - pre-order walk of a cpuset's descendants * @des_cs: loop cursor pointing to the current descendant * @pos_css: used for iteration * @root_cs: target cpuset to walk ancestor of * * Walk @des_cs through the online descendants of @root_cs.  Must be used * with RCU read locked.  The caller may modify @pos_css by calling * css_rightmost_descendant() to skip subtree.  @root_cs is included in the * iteration and the first node to be visited. */#define cpuset_for_each_descendant_pre(des_cs, pos_css, root_cs)	\	css_for_each_descendant_pre((pos_css), &(root_cs)->css)		\		if (is_cpuset_online(((des_cs) = css_cs((pos_css))))) /* * There are two global locks guarding cpuset structures - cpuset_mutex and * callback_lock. We also require taking task_lock() when dereferencing a * task's cpuset pointer. See "The task_lock() exception", at the end of this * comment. * * A task must hold both locks to modify cpusets.  If a task holds * cpuset_mutex, then it blocks others wanting that mutex, ensuring that it * is the only task able to also acquire callback_lock and be able to * modify cpusets.  It can perform various checks on the cpuset structure * first, knowing nothing will change.  It can also allocate memory while * just holding cpuset_mutex.  While it is performing these checks, various * callback routines can briefly acquire callback_lock to query cpusets. * Once it is ready to make the changes, it takes callback_lock, blocking * everyone else. * * Calls to the kernel memory allocator can not be made while holding * callback_lock, as that would risk double tripping on callback_lock * from one of the callbacks into the cpuset code from within * __alloc_pages(). * * If a task is only holding callback_lock, then it has read-only * access to cpusets. * * Now, the task_struct fields mems_allowed and mempolicy may be changed * by other task, we use alloc_lock in the task_struct fields to protect * them. * * The cpuset_common_file_read() handlers only hold callback_lock across * small pieces of code, such as when reading out possibly multi-word * cpumasks and nodemasks. * * Accessing a task's cpuset should be done in accordance with the * guidelines for accessing subsystem state in kernel/cgroup.c */ DEFINE_STATIC_PERCPU_RWSEM(cpuset_rwsem); void cpuset_read_lock(void){	percpu_down_read(&cpuset_rwsem);} void cpuset_read_unlock(void){	percpu_up_read(&cpuset_rwsem);} static DEFINE_SPINLOCK(callback_lock); static struct workqueue_struct *cpuset_migrate_mm_wq; /* * CPU / memory hotplug is handled asynchronously. */static void cpuset_hotplug_workfn(struct work_struct *work);static DECLARE_WORK(cpuset_hotplug_work, cpuset_hotplug_workfn); static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq); /* * Cgroup v2 behavior is used on the "cpus" and "mems" control files when * on default hierarchy or when the cpuset_v2_mode flag is set by mounting * the v1 cpuset cgroup filesystem with the "cpuset_v2_mode" mount option. * With v2 behavior, "cpus" and "mems" are always what the users have * requested and won't be changed by hotplug events. Only the effective * cpus or mems will be affected. */static inline bool is_in_v2_mode(void){	return cgroup_subsys_on_dfl(cpuset_cgrp_subsys) ||	      (cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE);} /* * Return in pmask the portion of a cpusets's cpus_allowed that * are online.  If none are online, walk up the cpuset hierarchy * until we find one that does have some online cpus. * * One way or another, we guarantee to return some non-empty subset * of cpu_online_mask. * * Call with callback_lock or cpuset_mutex held. */static void guarantee_online_cpus(struct cpuset *cs, struct cpumask *pmask){	while (!cpumask_intersects(cs->effective_cpus, cpu_online_mask)) {		cs = parent_cs(cs);		if (unlikely(!cs)) {			/*			 * The top cpuset doesn't have any online cpu as a			 * consequence of a race between cpuset_hotplug_work			 * and cpu hotplug notifier.  But we know the top			 * cpuset's effective_cpus is on its way to to be			 * identical to cpu_online_mask.			 */			cpumask_copy(pmask, cpu_online_mask);			return;		}	}	cpumask_and(pmask, cs->effective_cpus, cpu_online_mask);} /* * Return in *pmask the portion of a cpusets's mems_allowed that * are online, with memory.  If none are online with memory, walk * up the cpuset hierarchy until we find one that does have some * online mems.  The top cpuset always has some mems online. * * One way or another, we guarantee to return some non-empty subset * of node_states[N_MEMORY]. * * Call with callback_lock or cpuset_mutex held. */static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask){	while (!nodes_intersects(cs->effective_mems, node_states[N_MEMORY]))		cs = parent_cs(cs);	nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY]);} /* * update task's spread flag if cpuset's page/slab spread flag is set * * Call with callback_lock or cpuset_mutex held. */static void cpuset_update_task_spread_flag(struct cpuset *cs,					struct task_struct *tsk){	if (is_spread_page(cs))		task_set_spread_page(tsk);	else		task_clear_spread_page(tsk); 	if (is_spread_slab(cs))		task_set_spread_slab(tsk);	else		task_clear_spread_slab(tsk);} /* * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q? * * One cpuset is a subset of another if all its allowed CPUs and * Memory Nodes are a subset of the other, and its exclusive flags * are only set if the other's are set.  Call holding cpuset_mutex. */ static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q){	return	cpumask_subset(p->cpus_allowed, q->cpus_allowed) &&		nodes_subset(p->mems_allowed, q->mems_allowed) &&		is_cpu_exclusive(p) <= is_cpu_exclusive(q) &&		is_mem_exclusive(p) <= is_mem_exclusive(q);} /** * alloc_cpumasks - allocate three cpumasks for cpuset * @cs:  the cpuset that have cpumasks to be allocated. * @tmp: the tmpmasks structure pointer * Return: 0 if successful, -ENOMEM otherwise. * * Only one of the two input arguments should be non-NULL. */static inline int alloc_cpumasks(struct cpuset *cs, struct tmpmasks *tmp){	cpumask_var_t *pmask1, *pmask2, *pmask3; 	if (cs) {		pmask1 = &cs->cpus_allowed;		pmask2 = &cs->effective_cpus;		pmask3 = &cs->subparts_cpus;	} else {		pmask1 = &tmp->new_cpus;		pmask2 = &tmp->addmask;		pmask3 = &tmp->delmask;	} 	if (!zalloc_cpumask_var(pmask1, GFP_KERNEL))		return -ENOMEM; 	if (!zalloc_cpumask_var(pmask2, GFP_KERNEL))		goto free_one; 	if (!zalloc_cpumask_var(pmask3, GFP_KERNEL))		goto free_two; 	return 0; free_two:	free_cpumask_var(*pmask2);free_one:	free_cpumask_var(*pmask1);	return -ENOMEM;} /** * free_cpumasks - free cpumasks in a tmpmasks structure * @cs:  the cpuset that have cpumasks to be free. * @tmp: the tmpmasks structure pointer */

Lines 1–500 of 3623. Load more