/* CPU control. * (C) 2001, 2002, 2003, 2004 Rusty Russell * * This code is licenced under the GPL. */#include <linux/sched/mm.h>#include <linux/proc_fs.h>#include <linux/smp.h>#include <linux/init.h>#include <linux/notifier.h>#include <linux/sched/signal.h>#include <linux/sched/hotplug.h>#include <linux/sched/isolation.h>#include <linux/sched/task.h>#include <linux/sched/smt.h>#include <linux/unistd.h>#include <linux/cpu.h>#include <linux/oom.h>#include <linux/rcupdate.h>#include <linux/export.h>#include <linux/bug.h>#include <linux/kthread.h>#include <linux/stop_machine.h>#include <linux/mutex.h>#include <linux/gfp.h>#include <linux/suspend.h>#include <linux/lockdep.h>#include <linux/tick.h>#include <linux/irq.h>#include <linux/nmi.h>#include <linux/smpboot.h>#include <linux/relay.h>#include <linux/slab.h>#include <linux/percpu-rwsem.h> #include <trace/events/power.h>#define CREATE_TRACE_POINTS#include <trace/events/cpuhp.h> #include "smpboot.h" /** * cpuhp_cpu_state - Per cpu hotplug state storage * @state:	The current cpu state * @target:	The target state * @thread:	Pointer to the hotplug thread * @should_run:	Thread should execute * @rollback:	Perform a rollback * @single:	Single callback invocation * @bringup:	Single callback bringup or teardown selector * @cb_state:	The state for a single callback (install/uninstall) * @result:	Result of the operation * @done_up:	Signal completion to the issuer of the task for cpu-up * @done_down:	Signal completion to the issuer of the task for cpu-down */struct cpuhp_cpu_state {	enum cpuhp_state	state;	enum cpuhp_state	target;	enum cpuhp_state	fail;#ifdef CONFIG_SMP	struct task_struct	*thread;	bool			should_run;	bool			rollback;	bool			single;	bool			bringup;	struct hlist_node	*node;	struct hlist_node	*last;	enum cpuhp_state	cb_state;	int			result;	struct completion	done_up;	struct completion	done_down;#endif}; static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {	.fail = CPUHP_INVALID,}; #ifdef CONFIG_SMPcpumask_t cpus_booted_once_mask;#endif #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)static struct lockdep_map cpuhp_state_up_map =	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);static struct lockdep_map cpuhp_state_down_map =	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);  static inline void cpuhp_lock_acquire(bool bringup){	lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);} static inline void cpuhp_lock_release(bool bringup){	lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);}#else static inline void cpuhp_lock_acquire(bool bringup) { }static inline void cpuhp_lock_release(bool bringup) { } #endif /** * cpuhp_step - Hotplug state machine step * @name:	Name of the step * @startup:	Startup function of the step * @teardown:	Teardown function of the step * @cant_stop:	Bringup/teardown can't be stopped at this step */struct cpuhp_step {	const char		*name;	union {		int		(*single)(unsigned int cpu);		int		(*multi)(unsigned int cpu,					 struct hlist_node *node);	} startup;	union {		int		(*single)(unsigned int cpu);		int		(*multi)(unsigned int cpu,					 struct hlist_node *node);	} teardown;	struct hlist_head	list;	bool			cant_stop;	bool			multi_instance;}; static DEFINE_MUTEX(cpuhp_state_mutex);static struct cpuhp_step cpuhp_hp_states[]; static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state){	return cpuhp_hp_states + state;} /** * cpuhp_invoke_callback _ Invoke the callbacks for a given state * @cpu:	The cpu for which the callback should be invoked * @state:	The state to do callbacks for * @bringup:	True if the bringup callback should be invoked * @node:	For multi-instance, do a single entry callback for install/remove * @lastp:	For multi-instance rollback, remember how far we got * * Called from cpu hotplug and from the state register machinery. */static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,				 bool bringup, struct hlist_node *node,				 struct hlist_node **lastp){	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);	struct cpuhp_step *step = cpuhp_get_step(state);	int (*cbm)(unsigned int cpu, struct hlist_node *node);	int (*cb)(unsigned int cpu);	int ret, cnt; 	if (st->fail == state) {		st->fail = CPUHP_INVALID; 		if (!(bringup ? step->startup.single : step->teardown.single))			return 0; 		return -EAGAIN;	} 	if (!step->multi_instance) {		WARN_ON_ONCE(lastp && *lastp);		cb = bringup ? step->startup.single : step->teardown.single;		if (!cb)			return 0;		trace_cpuhp_enter(cpu, st->target, state, cb);		ret = cb(cpu);		trace_cpuhp_exit(cpu, st->state, state, ret);		return ret;	}	cbm = bringup ? step->startup.multi : step->teardown.multi;	if (!cbm)		return 0; 	/* Single invocation for instance add/remove */	if (node) {		WARN_ON_ONCE(lastp && *lastp);		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);		ret = cbm(cpu, node);		trace_cpuhp_exit(cpu, st->state, state, ret);		return ret;	} 	/* State transition. Invoke on all instances */	cnt = 0;	hlist_for_each(node, &step->list) {		if (lastp && node == *lastp)			break; 		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);		ret = cbm(cpu, node);		trace_cpuhp_exit(cpu, st->state, state, ret);		if (ret) {			if (!lastp)				goto err; 			*lastp = node;			return ret;		}		cnt++;	}	if (lastp)		*lastp = NULL;	return 0;err:	/* Rollback the instances if one failed */	cbm = !bringup ? step->startup.multi : step->teardown.multi;	if (!cbm)		return ret; 	hlist_for_each(node, &step->list) {		if (!cnt--)			break; 		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);		ret = cbm(cpu, node);		trace_cpuhp_exit(cpu, st->state, state, ret);		/*		 * Rollback must not fail,		 */		WARN_ON_ONCE(ret);	}	return ret;} #ifdef CONFIG_SMPstatic bool cpuhp_is_ap_state(enum cpuhp_state state){	/*	 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation	 * purposes as that state is handled explicitly in cpu_down.	 */	return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;} static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup){	struct completion *done = bringup ? &st->done_up : &st->done_down;	wait_for_completion(done);} static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup){	struct completion *done = bringup ? &st->done_up : &st->done_down;	complete(done);} /* * The former STARTING/DYING states, ran with IRQs disabled and must not fail. */static bool cpuhp_is_atomic_state(enum cpuhp_state state){	return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;} /* Serializes the updates to cpu_online_mask, cpu_present_mask */static DEFINE_MUTEX(cpu_add_remove_lock);bool cpuhp_tasks_frozen;EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen); /* * The following two APIs (cpu_maps_update_begin/done) must be used when * attempting to serialize the updates to cpu_online_mask & cpu_present_mask. */void cpu_maps_update_begin(void){	mutex_lock(&cpu_add_remove_lock);} void cpu_maps_update_done(void){	mutex_unlock(&cpu_add_remove_lock);} /* * If set, cpu_up and cpu_down will return -EBUSY and do nothing. * Should always be manipulated under cpu_add_remove_lock */static int cpu_hotplug_disabled; #ifdef CONFIG_HOTPLUG_CPU DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock); void cpus_read_lock(void){	percpu_down_read(&cpu_hotplug_lock);}EXPORT_SYMBOL_GPL(cpus_read_lock); int cpus_read_trylock(void){	return percpu_down_read_trylock(&cpu_hotplug_lock);}EXPORT_SYMBOL_GPL(cpus_read_trylock); void cpus_read_unlock(void){	percpu_up_read(&cpu_hotplug_lock);}EXPORT_SYMBOL_GPL(cpus_read_unlock); void cpus_write_lock(void){	percpu_down_write(&cpu_hotplug_lock);} void cpus_write_unlock(void){	percpu_up_write(&cpu_hotplug_lock);} void lockdep_assert_cpus_held(void){	/*	 * We can't have hotplug operations before userspace starts running,	 * and some init codepaths will knowingly not take the hotplug lock.	 * This is all valid, so mute lockdep until it makes sense to report	 * unheld locks.	 */	if (system_state < SYSTEM_RUNNING)		return; 	percpu_rwsem_assert_held(&cpu_hotplug_lock);} static void lockdep_acquire_cpus_lock(void){	rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_);} static void lockdep_release_cpus_lock(void){	rwsem_release(&cpu_hotplug_lock.dep_map, _THIS_IP_);} /* * Wait for currently running CPU hotplug operations to complete (if any) and * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the * hotplug path before performing hotplug operations. So acquiring that lock * guarantees mutual exclusion from any currently running hotplug operations. */void cpu_hotplug_disable(void){	cpu_maps_update_begin();	cpu_hotplug_disabled++;	cpu_maps_update_done();}EXPORT_SYMBOL_GPL(cpu_hotplug_disable); static void __cpu_hotplug_enable(void){	if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))		return;	cpu_hotplug_disabled--;} void cpu_hotplug_enable(void){	cpu_maps_update_begin();	__cpu_hotplug_enable();	cpu_maps_update_done();}EXPORT_SYMBOL_GPL(cpu_hotplug_enable); #else static void lockdep_acquire_cpus_lock(void){} static void lockdep_release_cpus_lock(void){} #endif	/* CONFIG_HOTPLUG_CPU */ /* * Architectures that need SMT-specific errata handling during SMT hotplug * should override this. */void __weak arch_smt_update(void) { } #ifdef CONFIG_HOTPLUG_SMTenum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED; void __init cpu_smt_disable(bool force){	if (!cpu_smt_possible())		return; 	if (force) {		pr_info("SMT: Force disabled\n");		cpu_smt_control = CPU_SMT_FORCE_DISABLED;	} else {		pr_info("SMT: disabled\n");		cpu_smt_control = CPU_SMT_DISABLED;	}} /* * The decision whether SMT is supported can only be done after the full * CPU identification. Called from architecture code. */void __init cpu_smt_check_topology(void){	if (!topology_smt_supported())		cpu_smt_control = CPU_SMT_NOT_SUPPORTED;} static int __init smt_cmdline_disable(char *str){	cpu_smt_disable(str && !strcmp(str, "force"));	return 0;}early_param("nosmt", smt_cmdline_disable); static inline bool cpu_smt_allowed(unsigned int cpu){	if (cpu_smt_control == CPU_SMT_ENABLED)		return true; 	if (topology_is_primary_thread(cpu))		return true; 	/*	 * On x86 it's required to boot all logical CPUs at least once so	 * that the init code can get a chance to set CR4.MCE on each	 * CPU. Otherwise, a broadcasted MCE observing CR4.MCE=0b on any	 * core will shutdown the machine.	 */	return !cpumask_test_cpu(cpu, &cpus_booted_once_mask);} /* Returns true if SMT is not supported of forcefully (irreversibly) disabled */bool cpu_smt_possible(void){	return cpu_smt_control != CPU_SMT_FORCE_DISABLED &&		cpu_smt_control != CPU_SMT_NOT_SUPPORTED;}EXPORT_SYMBOL_GPL(cpu_smt_possible);#elsestatic inline bool cpu_smt_allowed(unsigned int cpu) { return true; }#endif static inline enum cpuhp_statecpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target){	enum cpuhp_state prev_state = st->state; 	st->rollback = false;	st->last = NULL; 	st->target = target;	st->single = false;	st->bringup = st->state < target; 	return prev_state;} static inline voidcpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state){	st->rollback = true; 	/*	 * If we have st->last we need to undo partial multi_instance of this	 * state first. Otherwise start undo at the previous state.	 */	if (!st->last) {		if (st->bringup)			st->state--;		else			st->state++;	} 	st->target = prev_state;	st->bringup = !st->bringup;} /* Regular hotplug invocation of the AP hotplug thread */static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st){	if (!st->single && st->state == st->target)		return; 	st->result = 0;	/*	 * Make sure the above stores are visible before should_run becomes	 * true. Paired with the mb() above in cpuhp_thread_fun()	 */	smp_mb();	st->should_run = true;

Lines 1–500 of 2565. Load more