{	percpu_up_read(&cpu_hotplug_lock);}EXPORT_SYMBOL_GPL(cpus_read_unlock); void cpus_write_lock(void){	percpu_down_write(&cpu_hotplug_lock);} void cpus_write_unlock(void){	percpu_up_write(&cpu_hotplug_lock);} void lockdep_assert_cpus_held(void){	/*	 * We can't have hotplug operations before userspace starts running,	 * and some init codepaths will knowingly not take the hotplug lock.	 * This is all valid, so mute lockdep until it makes sense to report	 * unheld locks.	 */	if (system_state < SYSTEM_RUNNING)		return; 	percpu_rwsem_assert_held(&cpu_hotplug_lock);}EXPORT_SYMBOL_GPL(lockdep_assert_cpus_held); #ifdef CONFIG_LOCKDEPint lockdep_is_cpus_held(void){	return percpu_rwsem_is_held(&cpu_hotplug_lock);} int lockdep_is_cpus_write_held(void){	return percpu_rwsem_is_write_held(&cpu_hotplug_lock);}#endif static void lockdep_acquire_cpus_lock(void){	rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_);} static void lockdep_release_cpus_lock(void){	rwsem_release(&cpu_hotplug_lock.dep_map, _THIS_IP_);} /* Declare CPU offlining not supported */void cpu_hotplug_disable_offlining(void){	cpu_maps_update_begin();	cpu_hotplug_offline_disabled = true;	cpu_maps_update_done();} /* * Wait for currently running CPU hotplug operations to complete (if any) and * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the * hotplug path before performing hotplug operations. So acquiring that lock * guarantees mutual exclusion from any currently running hotplug operations. */void cpu_hotplug_disable(void){	cpu_maps_update_begin();	cpu_hotplug_disabled++;	cpu_maps_update_done();}EXPORT_SYMBOL_GPL(cpu_hotplug_disable); static void __cpu_hotplug_enable(void){	if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))		return;	cpu_hotplug_disabled--;} void cpu_hotplug_enable(void){	cpu_maps_update_begin();	__cpu_hotplug_enable();	cpu_maps_update_done();}EXPORT_SYMBOL_GPL(cpu_hotplug_enable); #else static void lockdep_acquire_cpus_lock(void){} static void lockdep_release_cpus_lock(void){} #endif	/* CONFIG_HOTPLUG_CPU */ /* * Architectures that need SMT-specific errata handling during SMT hotplug * should override this. */void __weak arch_smt_update(void) { } #ifdef CONFIG_HOTPLUG_SMT enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;static unsigned int cpu_smt_max_threads __ro_after_init;unsigned int cpu_smt_num_threads __read_mostly = UINT_MAX; void __init cpu_smt_disable(bool force){	if (!cpu_smt_possible())		return; 	if (force) {		pr_info("SMT: Force disabled\n");		cpu_smt_control = CPU_SMT_FORCE_DISABLED;	} else {		pr_info("SMT: disabled\n");		cpu_smt_control = CPU_SMT_DISABLED;	}	cpu_smt_num_threads = 1;} /* * The decision whether SMT is supported can only be done after the full * CPU identification. Called from architecture code. */void __init cpu_smt_set_num_threads(unsigned int num_threads,				    unsigned int max_threads){	WARN_ON(!num_threads || (num_threads > max_threads)); 	if (max_threads == 1)		cpu_smt_control = CPU_SMT_NOT_SUPPORTED; 	cpu_smt_max_threads = max_threads; 	/*	 * If SMT has been disabled via the kernel command line or SMT is	 * not supported, set cpu_smt_num_threads to 1 for consistency.	 * If enabled, take the architecture requested number of threads	 * to bring up into account.	 */	if (cpu_smt_control != CPU_SMT_ENABLED)		cpu_smt_num_threads = 1;	else if (num_threads < cpu_smt_num_threads)		cpu_smt_num_threads = num_threads;} static int __init smt_cmdline_disable(char *str){	cpu_smt_disable(str && !strcmp(str, "force"));	return 0;}early_param("nosmt", smt_cmdline_disable); /* * For Archicture supporting partial SMT states check if the thread is allowed. * Otherwise this has already been checked through cpu_smt_max_threads when * setting the SMT level. */static inline bool cpu_smt_thread_allowed(unsigned int cpu){#ifdef CONFIG_SMT_NUM_THREADS_DYNAMIC	return topology_smt_thread_allowed(cpu);#else	return true;#endif} static inline bool cpu_bootable(unsigned int cpu){	if (cpu_smt_control == CPU_SMT_ENABLED && cpu_smt_thread_allowed(cpu))		return true; 	/* All CPUs are bootable if controls are not configured */	if (cpu_smt_control == CPU_SMT_NOT_IMPLEMENTED)		return true; 	/* All CPUs are bootable if CPU is not SMT capable */	if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)		return true; 	if (topology_is_primary_thread(cpu))		return true; 	/*	 * On x86 it's required to boot all logical CPUs at least once so	 * that the init code can get a chance to set CR4.MCE on each	 * CPU. Otherwise, a broadcasted MCE observing CR4.MCE=0b on any	 * core will shutdown the machine.	 */	return !cpumask_test_cpu(cpu, &cpus_booted_once_mask);} /* Returns true if SMT is supported and not forcefully (irreversibly) disabled */bool cpu_smt_possible(void){	return cpu_smt_control != CPU_SMT_FORCE_DISABLED &&		cpu_smt_control != CPU_SMT_NOT_SUPPORTED;}EXPORT_SYMBOL_GPL(cpu_smt_possible); #elsestatic inline bool cpu_bootable(unsigned int cpu) { return true; }#endif static inline enum cpuhp_statecpuhp_set_state(int cpu, struct cpuhp_cpu_state *st, enum cpuhp_state target){	enum cpuhp_state prev_state = st->state;	bool bringup = st->state < target; 	st->rollback = false;	st->last = NULL; 	st->target = target;	st->single = false;	st->bringup = bringup;	if (cpu_dying(cpu) != !bringup)		set_cpu_dying(cpu, !bringup); 	return prev_state;} static inline voidcpuhp_reset_state(int cpu, struct cpuhp_cpu_state *st,		  enum cpuhp_state prev_state){	bool bringup = !st->bringup; 	st->target = prev_state; 	/*	 * Already rolling back. No need invert the bringup value or to change	 * the current state.	 */	if (st->rollback)		return; 	st->rollback = true; 	/*	 * If we have st->last we need to undo partial multi_instance of this	 * state first. Otherwise start undo at the previous state.	 */	if (!st->last) {		if (st->bringup)			st->state--;		else			st->state++;	} 	st->bringup = bringup;	if (cpu_dying(cpu) != !bringup)		set_cpu_dying(cpu, !bringup);} /* Regular hotplug invocation of the AP hotplug thread */static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st){	if (!st->single && st->state == st->target)		return; 	st->result = 0;	/*	 * Make sure the above stores are visible before should_run becomes	 * true. Paired with the mb() above in cpuhp_thread_fun()	 */	smp_mb();	st->should_run = true;	wake_up_process(st->thread);	wait_for_ap_thread(st, st->bringup);} static int cpuhp_kick_ap(int cpu, struct cpuhp_cpu_state *st,			 enum cpuhp_state target){	enum cpuhp_state prev_state;	int ret; 	prev_state = cpuhp_set_state(cpu, st, target);	__cpuhp_kick_ap(st);	if ((ret = st->result)) {		cpuhp_reset_state(cpu, st, prev_state);		__cpuhp_kick_ap(st);	} 	return ret;} static int bringup_wait_for_ap_online(unsigned int cpu){	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 	/* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */	wait_for_ap_thread(st, true);	if (WARN_ON_ONCE((!cpu_online(cpu))))		return -ECANCELED; 	/* Unpark the hotplug thread of the target cpu */	kthread_unpark(st->thread); 	/*	 * SMT soft disabling on X86 requires to bring the CPU out of the	 * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit.  The	 * CPU marked itself as booted_once in notify_cpu_starting() so the	 * cpu_bootable() check will now return false if this is not the	 * primary sibling.	 */	if (!cpu_bootable(cpu))		return -ECANCELED;	return 0;} #ifdef CONFIG_HOTPLUG_SPLIT_STARTUPstatic int cpuhp_kick_ap_alive(unsigned int cpu){	if (!cpuhp_can_boot_ap(cpu))		return -EAGAIN; 	return arch_cpuhp_kick_ap_alive(cpu, idle_thread_get(cpu));} static int cpuhp_bringup_ap(unsigned int cpu){	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);	int ret; 	/*	 * Some architectures have to walk the irq descriptors to	 * setup the vector space for the cpu which comes online.	 * Prevent irq alloc/free across the bringup.	 */	irq_lock_sparse(); 	ret = cpuhp_bp_sync_alive(cpu);	if (ret)		goto out_unlock; 	ret = bringup_wait_for_ap_online(cpu);	if (ret)		goto out_unlock; 	irq_unlock_sparse(); 	if (st->target <= CPUHP_AP_ONLINE_IDLE)		return 0; 	return cpuhp_kick_ap(cpu, st, st->target); out_unlock:	irq_unlock_sparse();	return ret;}#elsestatic int bringup_cpu(unsigned int cpu){	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);	struct task_struct *idle = idle_thread_get(cpu);	int ret; 	if (!cpuhp_can_boot_ap(cpu))		return -EAGAIN; 	/*	 * Some architectures have to walk the irq descriptors to	 * setup the vector space for the cpu which comes online.	 *	 * Prevent irq alloc/free across the bringup by acquiring the	 * sparse irq lock. Hold it until the upcoming CPU completes the	 * startup in cpuhp_online_idle() which allows to avoid	 * intermediate synchronization points in the architecture code.	 */	irq_lock_sparse(); 	ret = __cpu_up(cpu, idle);	if (ret)		goto out_unlock; 	ret = cpuhp_bp_sync_alive(cpu);	if (ret)		goto out_unlock; 	ret = bringup_wait_for_ap_online(cpu);	if (ret)		goto out_unlock; 	irq_unlock_sparse(); 	if (st->target <= CPUHP_AP_ONLINE_IDLE)		return 0; 	return cpuhp_kick_ap(cpu, st, st->target); out_unlock:	irq_unlock_sparse();	return ret;}#endif static int finish_cpu(unsigned int cpu){	struct task_struct *idle = idle_thread_get(cpu);	struct mm_struct *mm = idle->active_mm; 	/*	 * sched_force_init_mm() ensured the use of &init_mm,	 * drop that refcount now that the CPU has stopped.	 */	WARN_ON(mm != &init_mm);	idle->active_mm = NULL;	mmdrop_lazy_tlb(mm); 	return 0;} /* * Hotplug state machine related functions */ /* * Get the next state to run. Empty ones will be skipped. Returns true if a * state must be run. * * st->state will be modified ahead of time, to match state_to_run, as if it * has already ran. */static bool cpuhp_next_state(bool bringup,			     enum cpuhp_state *state_to_run,			     struct cpuhp_cpu_state *st,			     enum cpuhp_state target){	do {		if (bringup) {			if (st->state >= target)				return false; 			*state_to_run = ++st->state;		} else {			if (st->state <= target)				return false; 			*state_to_run = st->state--;		} 		if (!cpuhp_step_empty(bringup, cpuhp_get_step(*state_to_run)))			break;	} while (true); 	return true;} static int __cpuhp_invoke_callback_range(bool bringup,					 unsigned int cpu,					 struct cpuhp_cpu_state *st,					 enum cpuhp_state target,					 bool nofail){	enum cpuhp_state state;	int ret = 0; 	while (cpuhp_next_state(bringup, &state, st, target)) {		int err; 		err = cpuhp_invoke_callback(cpu, state, bringup, NULL, NULL);		if (!err)			continue; 		if (nofail) {			pr_warn("CPU %u %s state %s (%d) failed (%d)\n",				cpu, bringup ? "UP" : "DOWN",				cpuhp_get_step(st->state)->name,				st->state, err);			ret = -1;		} else {			ret = err;			break;		}	} 	return ret;} static inline int cpuhp_invoke_callback_range(bool bringup,					      unsigned int cpu,					      struct cpuhp_cpu_state *st,					      enum cpuhp_state target){	return __cpuhp_invoke_callback_range(bringup, cpu, st, target, false);} static inline void cpuhp_invoke_callback_range_nofail(bool bringup,						      unsigned int cpu,

Lines 501–1000 of 3339. Previous lines Load more