// SPDX-License-Identifier: GPL-2.0/* *  kernel/cpuset.c * *  Processor and Memory placement constraints for sets of tasks. * *  Copyright (C) 2003 BULL SA. *  Copyright (C) 2004-2007 Silicon Graphics, Inc. *  Copyright (C) 2006 Google, Inc * *  Portions derived from Patrick Mochel's sysfs code. *  sysfs is Copyright (c) 2001-3 Patrick Mochel * *  2003-10-10 Written by Simon Derr. *  2003-10-22 Updates by Stephen Hemminger. *  2004 May-July Rework by Paul Jackson. *  2006 Rework by Paul Menage to use generic cgroups *  2008 Rework of the scheduler domains and CPU hotplug handling *       by Max Krasnyansky */#include "cpuset-internal.h" #include <linux/init.h>#include <linux/interrupt.h>#include <linux/kernel.h>#include <linux/mempolicy.h>#include <linux/mm.h>#include <linux/memory.h>#include <linux/rcupdate.h>#include <linux/sched.h>#include <linux/sched/deadline.h>#include <linux/sched/mm.h>#include <linux/sched/task.h>#include <linux/security.h>#include <linux/oom.h>#include <linux/sched/isolation.h>#include <linux/wait.h>#include <linux/workqueue.h>#include <linux/task_work.h>#include <linux/llist.h> DEFINE_STATIC_KEY_FALSE(cpusets_pre_enable_key);DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key); /* * There could be abnormal cpuset configurations for cpu or memory * node binding, add this key to provide a quick low-cost judgment * of the situation. */DEFINE_STATIC_KEY_FALSE(cpusets_insane_config_key); static const char * const perr_strings[] = {	[PERR_INVCPUS]   = "Invalid cpu list in cpuset.cpus.exclusive",	[PERR_INVPARENT] = "Parent is an invalid partition root",	[PERR_NOTPART]   = "Parent is not a partition root",	[PERR_NOTEXCL]   = "Cpu list in cpuset.cpus not exclusive",	[PERR_NOCPUS]    = "Parent unable to distribute cpu downstream",	[PERR_HOTPLUG]   = "No cpu available due to hotplug",	[PERR_CPUSEMPTY] = "cpuset.cpus and cpuset.cpus.exclusive are empty",	[PERR_HKEEPING]  = "partition config conflicts with housekeeping setup",	[PERR_ACCESS]    = "Enable partition not permitted",	[PERR_REMOTE]    = "Have remote partition underneath",}; /* * CPUSET Locking Convention * ------------------------- * * Below are the four global/local locks guarding cpuset structures in lock * acquisition order: *  - cpuset_top_mutex *  - cpu_hotplug_lock (cpus_read_lock/cpus_write_lock) *  - cpuset_mutex *  - callback_lock (raw spinlock) * * As cpuset will now indirectly flush a number of different workqueues in * housekeeping_update() to update housekeeping cpumasks when the set of * isolated CPUs is going to be changed, it may be vulnerable to deadlock * if we hold cpus_read_lock while calling into housekeeping_update(). * * The first cpuset_top_mutex will be held except when calling into * cpuset_handle_hotplug() from the CPU hotplug code where cpus_write_lock * and cpuset_mutex will be held instead. The main purpose of this mutex * is to prevent regular cpuset control file write actions from interfering * with the call to housekeeping_update(), though CPU hotplug operation can * still happen in parallel. This mutex also provides protection for some * internal variables. * * A task must hold all the remaining three locks to modify externally visible * or used fields of cpusets, though some of the internally used cpuset fields * and internal variables can be modified without holding callback_lock. If only * reliable read access of the externally used fields are needed, a task can * hold either cpuset_mutex or callback_lock which are exposed to other * external subsystems. * * If a task holds cpu_hotplug_lock and cpuset_mutex, it blocks others, * ensuring that it is the only task able to also acquire callback_lock and * be able to modify cpusets.  It can perform various checks on the cpuset * structure first, knowing nothing will change. It can also allocate memory * without holding callback_lock. While it is performing these checks, various * callback routines can briefly acquire callback_lock to query cpusets.  Once * it is ready to make the changes, it takes callback_lock, blocking everyone * else. * * Calls to the kernel memory allocator cannot be made while holding * callback_lock which is a spinlock, as the memory allocator may sleep or * call back into cpuset code and acquire callback_lock. * * Now, the task_struct fields mems_allowed and mempolicy may be changed * by other task, we use alloc_lock in the task_struct fields to protect * them. * * The cpuset_common_seq_show() handlers only hold callback_lock across * small pieces of code, such as when reading out possibly multi-word * cpumasks and nodemasks. */ static DEFINE_MUTEX(cpuset_top_mutex);static DEFINE_MUTEX(cpuset_mutex); /* * File level internal variables below follow one of the following exclusion * rules. * * RWCS: Read/write-able by holding either cpus_write_lock (and optionally *	 cpuset_mutex) or both cpus_read_lock and cpuset_mutex. * * CSCB: Readable by holding either cpuset_mutex or callback_lock. Writable *	 by holding both cpuset_mutex and callback_lock. * * T:	 Read/write-able by holding the cpuset_top_mutex. */ /* * For local partitions, update to subpartitions_cpus & isolated_cpus is done * in update_parent_effective_cpumask(). For remote partitions, it is done in * the remote_partition_*() and remote_cpus_update() helpers. *//* * Exclusive CPUs distributed out to local or remote sub-partitions of * top_cpuset */static cpumask_var_t	subpartitions_cpus;	/* RWCS */ /* * Exclusive CPUs in isolated partitions (shown in cpuset.cpus.isolated) */static cpumask_var_t	isolated_cpus;		/* CSCB */ /* * Set if housekeeping cpumasks are to be updated. */static bool		update_housekeeping;	/* RWCS */ /* * Copy of isolated_cpus to be passed to housekeeping_update() */static cpumask_var_t	isolated_hk_cpus;	/* T */ /* * A flag to force sched domain rebuild at the end of an operation. * It can be set in *  - update_partition_sd_lb() *  - update_cpumasks_hier() *  - cpuset_update_flag() *  - cpuset_hotplug_update_tasks() *  - cpuset_handle_hotplug() * * Protected by cpuset_mutex (with cpus_read_lock held) or cpus_write_lock. * * Note that update_relax_domain_level() in cpuset-v1.c can still call * rebuild_sched_domains_locked() directly without using this flag. */static bool force_sd_rebuild;			/* RWCS */ /* * Partition root states: * *   0 - member (not a partition root) *   1 - partition root *   2 - partition root without load balancing (isolated) *  -1 - invalid partition root *  -2 - invalid isolated partition root * *  There are 2 types of partitions - local or remote. Local partitions are *  those whose parents are partition root themselves. Setting of *  cpuset.cpus.exclusive are optional in setting up local partitions. *  Remote partitions are those whose parents are not partition roots. Passing *  down exclusive CPUs by setting cpuset.cpus.exclusive along its ancestor *  nodes are mandatory in creating a remote partition. * *  For simplicity, a local partition can be created under a local or remote *  partition but a remote partition cannot have any partition root in its *  ancestor chain except the cgroup root. * *  A valid partition can be formed by setting exclusive_cpus or cpus_allowed *  if exclusive_cpus is not set. In the case of partition with empty *  exclusive_cpus, all the conflicting exclusive CPUs specified in the *  following cpumasks of sibling cpusets will be removed from its *  cpus_allowed in determining its effective_xcpus. *  - effective_xcpus *  - exclusive_cpus * *  The "cpuset.cpus.exclusive" control file should be used for setting up *  partition if the users want to get as many CPUs as possible. */#define PRS_MEMBER		0#define PRS_ROOT		1#define PRS_ISOLATED		2#define PRS_INVALID_ROOT	-1#define PRS_INVALID_ISOLATED	-2 /* * Temporary cpumasks for working with partitions that are passed among * functions to avoid memory allocation in inner functions. */struct tmpmasks {	cpumask_var_t addmask, delmask;	/* For partition root */	cpumask_var_t new_cpus;		/* For update_cpumasks_hier() */}; void inc_dl_tasks_cs(struct task_struct *p){	struct cpuset *cs = task_cs(p); 	atomic_inc(&cs->nr_deadline_tasks);} void dec_dl_tasks_cs(struct task_struct *p){	struct cpuset *cs = task_cs(p); 	atomic_dec(&cs->nr_deadline_tasks);} static inline bool is_partition_valid(const struct cpuset *cs){	return cs->partition_root_state > 0;} static inline bool is_partition_invalid(const struct cpuset *cs){	return cs->partition_root_state < 0;} static inline bool cs_is_member(const struct cpuset *cs){	return cs->partition_root_state == PRS_MEMBER;} /* * Callers should hold callback_lock to modify partition_root_state. */static inline void make_partition_invalid(struct cpuset *cs){	if (cs->partition_root_state > 0)		cs->partition_root_state = -cs->partition_root_state;} /* * Send notification event of whenever partition_root_state changes. */static inline void notify_partition_change(struct cpuset *cs, int old_prs){	if (old_prs == cs->partition_root_state)		return;	cgroup_file_notify(&cs->partition_file); 	/* Reset prs_err if not invalid */	if (is_partition_valid(cs))		WRITE_ONCE(cs->prs_err, PERR_NONE);} /* * The top_cpuset is always synchronized to cpu_active_mask and we should avoid * using cpu_online_mask as much as possible. An active CPU is always an online * CPU, but not vice versa. cpu_active_mask and cpu_online_mask can differ * during hotplug operations. A CPU is marked active at the last stage of CPU * bringup (CPUHP_AP_ACTIVE). It is also the stage where cpuset hotplug code * will be called to update the sched domains so that the scheduler can move * a normal task to a newly active CPU or remove tasks away from a newly * inactivated CPU. The online bit is set much earlier in the CPU bringup * process and cleared much later in CPU teardown. * * If cpu_online_mask is used while a hotunplug operation is happening in * parallel, we may leave an offline CPU in cpu_allowed or some other masks. */struct cpuset top_cpuset = {	.flags = BIT(CS_CPU_EXCLUSIVE) |		 BIT(CS_MEM_EXCLUSIVE) | BIT(CS_SCHED_LOAD_BALANCE),	.partition_root_state = PRS_ROOT,	.dl_bw_cpu = -1,}; /** * cpuset_lock - Acquire the global cpuset mutex * * This locks the global cpuset mutex to prevent modifications to cpuset * hierarchy and configurations. This helper is not enough to make modification. */void cpuset_lock(void){	mutex_lock(&cpuset_mutex);} void cpuset_unlock(void){	mutex_unlock(&cpuset_mutex);} void lockdep_assert_cpuset_lock_held(void){	lockdep_assert_held(&cpuset_mutex);} /** * cpuset_full_lock - Acquire full protection for cpuset modification * * Takes both CPU hotplug read lock (cpus_read_lock()) and cpuset mutex * to safely modify cpuset data. */void cpuset_full_lock(void){	mutex_lock(&cpuset_top_mutex);	cpus_read_lock();	mutex_lock(&cpuset_mutex);} void cpuset_full_unlock(void){	mutex_unlock(&cpuset_mutex);	cpus_read_unlock();	mutex_unlock(&cpuset_top_mutex);} #ifdef CONFIG_LOCKDEPbool lockdep_is_cpuset_held(void){	return lockdep_is_held(&cpuset_mutex) ||	       lockdep_is_held(&cpuset_top_mutex);}#endif static DEFINE_SPINLOCK(callback_lock); void cpuset_callback_lock_irq(void){	spin_lock_irq(&callback_lock);} void cpuset_callback_unlock_irq(void){	spin_unlock_irq(&callback_lock);} static struct workqueue_struct *cpuset_migrate_mm_wq; static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq); /* * Cpuset task attach context * Protected by cpuset_mutex */static struct {	int in_progress;	bool cpus_updated;	bool mems_updated;	bool task_work_queued;	bool many_dest_cs;	/* Have many destination cpusets */	struct cpuset *old_cs;	/* Source cpuset */	nodemask_t nodemask_to;} attach_ctx;static LLIST_HEAD(src_cs_head);static LLIST_HEAD(dst_cs_head); /* * Wait if task attach is in progress until it is done and then acquire * cpuset_mutex before returning. */static void wait_attach_done_lock(void)	    __acquires(&cpuset_mutex){	for (;;) {		mutex_lock(&cpuset_mutex);		if (!attach_ctx.in_progress)			return; 		mutex_unlock(&cpuset_mutex); 		/* Wait until attach operation is done to prevent racing */		wait_event(cpuset_attach_wq, attach_ctx.in_progress == 0);	}} static inline void check_insane_mems_config(nodemask_t *nodes){	if (!cpusets_insane_config() &&		movable_only_nodes(nodes)) {		static_branch_enable_cpuslocked(&cpusets_insane_config_key);		pr_info("Unsupported (movable nodes only) cpuset configuration detected (nmask=%*pbl)!\n"			"Cpuset allocations might fail even with a lot of memory available.\n",			nodemask_pr_args(nodes));	}} /* * decrease attach_ctx.in_progress. * wake_up cpuset_attach_wq if attach_ctx.in_progress==0. */static inline void dec_attach_in_progress_locked(void){	lockdep_assert_cpuset_lock_held(); 	attach_ctx.in_progress--;	if (!attach_ctx.in_progress)		wake_up(&cpuset_attach_wq);} static inline void dec_attach_in_progress(void){	mutex_lock(&cpuset_mutex);	dec_attach_in_progress_locked();	mutex_unlock(&cpuset_mutex);} static inline bool cpuset_v2(void){	return !IS_ENABLED(CONFIG_CPUSETS_V1) ||		cgroup_subsys_on_dfl(cpuset_cgrp_subsys);} /* * Cgroup v2 behavior is used on the "cpus" and "mems" control files when * on default hierarchy or when the cpuset_v2_mode flag is set by mounting * the v1 cpuset cgroup filesystem with the "cpuset_v2_mode" mount option. * With v2 behavior, "cpus" and "mems" are always what the users have * requested and won't be changed by hotplug events. Only the effective * cpus or mems will be affected. */static inline bool is_in_v2_mode(void){	return cpuset_v2() ||	      (cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE);} /** * partition_is_populated - check if partition has tasks * @cs: partition root to be checked * @excluded_child: a child cpuset to be excluded in task checking * Return: true if there are tasks, false otherwise * * @cs should be a valid partition root or going to become a partition root. * @excluded_child should be non-NULL when this cpuset is going to become a * partition itself. * * Note that a remote partition is not allowed underneath a valid local * or remote partition. So if a non-partition root child is populated, * the whole partition is considered populated. */static inline bool partition_is_populated(struct cpuset *cs,					  struct cpuset *excluded_child){	struct cpuset *cp;	struct cgroup_subsys_state *pos_css; 	/*	 * We cannot call cs_is_populated(cs) directly, as	 * nr_populated_domain_children may include populated	 * csets from descendants that are partitions.	 */	if (cgroup_has_tasks(cs->css.cgroup))		return true; 	rcu_read_lock();	cpuset_for_each_descendant_pre(cp, pos_css, cs) {		if (cp == cs || cp == excluded_child)			continue; 		if (is_partition_valid(cp)) {			pos_css = css_rightmost_descendant(pos_css);			continue;		} 		if (cpuset_is_populated(cp)) {			rcu_read_unlock();			return true;		}	}	rcu_read_unlock();	return false;} /* * Return in pmask the portion of a task's cpusets's cpus_allowed that * are online and are capable of running the task.  If none are found, * walk up the cpuset hierarchy until we find one that does have some * appropriate cpus. * * One way or another, we guarantee to return some non-empty subset * of cpu_active_mask.

Lines 1–500 of 4587. Load more