2554 lines · 60.8 KB · C
RawDownload
	if (!cpu_present(cpu))		return -EINVAL; 	cpus_write_lock(); 	cpuhp_tasks_frozen = tasks_frozen; 	prev_state = cpuhp_set_state(st, target);	/*	 * If the current CPU state is in the range of the AP hotplug thread,	 * then we need to kick the thread.	 */	if (st->state > CPUHP_TEARDOWN_CPU) {		st->target = max((int)target, CPUHP_TEARDOWN_CPU);		ret = cpuhp_kick_ap_work(cpu);		/*		 * The AP side has done the error rollback already. Just		 * return the error code..		 */		if (ret)			goto out; 		/*		 * We might have stopped still in the range of the AP hotplug		 * thread. Nothing to do anymore.		 */		if (st->state > CPUHP_TEARDOWN_CPU)			goto out; 		st->target = target;	}	/*	 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need	 * to do the further cleanups.	 */	ret = cpuhp_down_callbacks(cpu, st, target);	if (ret && st->state == CPUHP_TEARDOWN_CPU && st->state < prev_state) {		cpuhp_reset_state(st, prev_state);		__cpuhp_kick_ap(st);	} out:	cpus_write_unlock();	/*	 * Do post unplug cleanup. This is still protected against	 * concurrent CPU hotplug via cpu_add_remove_lock.	 */	lockup_detector_cleanup();	arch_smt_update();	return ret;} static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target){	if (cpu_hotplug_disabled)		return -EBUSY;	return _cpu_down(cpu, 0, target);} static int cpu_down(unsigned int cpu, enum cpuhp_state target){	int err; 	cpu_maps_update_begin();	err = cpu_down_maps_locked(cpu, target);	cpu_maps_update_done();	return err;} /** * cpu_device_down - Bring down a cpu device * @dev: Pointer to the cpu device to offline * * This function is meant to be used by device core cpu subsystem only. * * Other subsystems should use remove_cpu() instead. */int cpu_device_down(struct device *dev){	return cpu_down(dev->id, CPUHP_OFFLINE);} int remove_cpu(unsigned int cpu){	int ret; 	lock_device_hotplug();	ret = device_offline(get_cpu_device(cpu));	unlock_device_hotplug(); 	return ret;}EXPORT_SYMBOL_GPL(remove_cpu); void smp_shutdown_nonboot_cpus(unsigned int primary_cpu){	unsigned int cpu;	int error; 	cpu_maps_update_begin(); 	/*	 * Make certain the cpu I'm about to reboot on is online.	 *	 * This is inline to what migrate_to_reboot_cpu() already do.	 */	if (!cpu_online(primary_cpu))		primary_cpu = cpumask_first(cpu_online_mask); 	for_each_online_cpu(cpu) {		if (cpu == primary_cpu)			continue; 		error = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);		if (error) {			pr_err("Failed to offline CPU%d - error=%d",				cpu, error);			break;		}	} 	/*	 * Ensure all but the reboot CPU are offline.	 */	BUG_ON(num_online_cpus() > 1); 	/*	 * Make sure the CPUs won't be enabled by someone else after this	 * point. Kexec will reboot to a new kernel shortly resetting	 * everything along the way.	 */	cpu_hotplug_disabled++; 	cpu_maps_update_done();} #else#define takedown_cpu		NULL#endif /*CONFIG_HOTPLUG_CPU*/ /** * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU * @cpu: cpu that just started * * It must be called by the arch code on the new cpu, before the new cpu * enables interrupts and before the "boot" cpu returns from __cpu_up(). */void notify_cpu_starting(unsigned int cpu){	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);	enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);	int ret; 	rcu_cpu_starting(cpu);	/* Enables RCU usage on this CPU. */	cpumask_set_cpu(cpu, &cpus_booted_once_mask);	while (st->state < target) {		st->state++;		ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);		/*		 * STARTING must not fail!		 */		WARN_ON_ONCE(ret);	}} /* * Called from the idle task. Wake up the controlling task which brings the * hotplug thread of the upcoming CPU up and then delegates the rest of the * online bringup to the hotplug thread. */void cpuhp_online_idle(enum cpuhp_state state){	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 	/* Happens for the boot cpu */	if (state != CPUHP_AP_ONLINE_IDLE)		return; 	/*	 * Unpart the stopper thread before we start the idle loop (and start	 * scheduling); this ensures the stopper task is always available.	 */	stop_machine_unpark(smp_processor_id()); 	st->state = CPUHP_AP_ONLINE_IDLE;	complete_ap_thread(st, true);} /* Requires cpu_add_remove_lock to be held */static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target){	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);	struct task_struct *idle;	int ret = 0; 	cpus_write_lock(); 	if (!cpu_present(cpu)) {		ret = -EINVAL;		goto out;	} 	/*	 * The caller of cpu_up() might have raced with another	 * caller. Nothing to do.	 */	if (st->state >= target)		goto out; 	if (st->state == CPUHP_OFFLINE) {		/* Let it fail before we try to bring the cpu up */		idle = idle_thread_get(cpu);		if (IS_ERR(idle)) {			ret = PTR_ERR(idle);			goto out;		}	} 	cpuhp_tasks_frozen = tasks_frozen; 	cpuhp_set_state(st, target);	/*	 * If the current CPU state is in the range of the AP hotplug thread,	 * then we need to kick the thread once more.	 */	if (st->state > CPUHP_BRINGUP_CPU) {		ret = cpuhp_kick_ap_work(cpu);		/*		 * The AP side has done the error rollback already. Just		 * return the error code..		 */		if (ret)			goto out;	} 	/*	 * Try to reach the target state. We max out on the BP at	 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is	 * responsible for bringing it up to the target state.	 */	target = min((int)target, CPUHP_BRINGUP_CPU);	ret = cpuhp_up_callbacks(cpu, st, target);out:	cpus_write_unlock();	arch_smt_update();	return ret;} static int cpu_up(unsigned int cpu, enum cpuhp_state target){	int err = 0; 	if (!cpu_possible(cpu)) {		pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",		       cpu);#if defined(CONFIG_IA64)		pr_err("please check additional_cpus= boot parameter\n");#endif		return -EINVAL;	} 	err = try_online_node(cpu_to_node(cpu));	if (err)		return err; 	cpu_maps_update_begin(); 	if (cpu_hotplug_disabled) {		err = -EBUSY;		goto out;	}	if (!cpu_smt_allowed(cpu)) {		err = -EPERM;		goto out;	} 	err = _cpu_up(cpu, 0, target);out:	cpu_maps_update_done();	return err;} /** * cpu_device_up - Bring up a cpu device * @dev: Pointer to the cpu device to online * * This function is meant to be used by device core cpu subsystem only. * * Other subsystems should use add_cpu() instead. */int cpu_device_up(struct device *dev){	return cpu_up(dev->id, CPUHP_ONLINE);} int add_cpu(unsigned int cpu){	int ret; 	lock_device_hotplug();	ret = device_online(get_cpu_device(cpu));	unlock_device_hotplug(); 	return ret;}EXPORT_SYMBOL_GPL(add_cpu); /** * bringup_hibernate_cpu - Bring up the CPU that we hibernated on * @sleep_cpu: The cpu we hibernated on and should be brought up. * * On some architectures like arm64, we can hibernate on any CPU, but on * wake up the CPU we hibernated on might be offline as a side effect of * using maxcpus= for example. */int bringup_hibernate_cpu(unsigned int sleep_cpu){	int ret; 	if (!cpu_online(sleep_cpu)) {		pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n");		ret = cpu_up(sleep_cpu, CPUHP_ONLINE);		if (ret) {			pr_err("Failed to bring hibernate-CPU up!\n");			return ret;		}	}	return 0;} void bringup_nonboot_cpus(unsigned int setup_max_cpus){	unsigned int cpu; 	for_each_present_cpu(cpu) {		if (num_online_cpus() >= setup_max_cpus)			break;		if (!cpu_online(cpu))			cpu_up(cpu, CPUHP_ONLINE);	}} #ifdef CONFIG_PM_SLEEP_SMPstatic cpumask_var_t frozen_cpus; int freeze_secondary_cpus(int primary){	int cpu, error = 0; 	cpu_maps_update_begin();	if (primary == -1) {		primary = cpumask_first(cpu_online_mask);		if (!housekeeping_cpu(primary, HK_FLAG_TIMER))			primary = housekeeping_any_cpu(HK_FLAG_TIMER);	} else {		if (!cpu_online(primary))			primary = cpumask_first(cpu_online_mask);	} 	/*	 * We take down all of the non-boot CPUs in one shot to avoid races	 * with the userspace trying to use the CPU hotplug at the same time	 */	cpumask_clear(frozen_cpus); 	pr_info("Disabling non-boot CPUs ...\n");	for_each_online_cpu(cpu) {		if (cpu == primary)			continue; 		if (pm_wakeup_pending()) {			pr_info("Wakeup pending. Abort CPU freeze\n");			error = -EBUSY;			break;		} 		trace_suspend_resume(TPS("CPU_OFF"), cpu, true);		error = _cpu_down(cpu, 1, CPUHP_OFFLINE);		trace_suspend_resume(TPS("CPU_OFF"), cpu, false);		if (!error)			cpumask_set_cpu(cpu, frozen_cpus);		else {			pr_err("Error taking CPU%d down: %d\n", cpu, error);			break;		}	} 	if (!error)		BUG_ON(num_online_cpus() > 1);	else		pr_err("Non-boot CPUs are not disabled\n"); 	/*	 * Make sure the CPUs won't be enabled by someone else. We need to do	 * this even in case of failure as all freeze_secondary_cpus() users are	 * supposed to do thaw_secondary_cpus() on the failure path.	 */	cpu_hotplug_disabled++; 	cpu_maps_update_done();	return error;} void __weak arch_thaw_secondary_cpus_begin(void){} void __weak arch_thaw_secondary_cpus_end(void){} void thaw_secondary_cpus(void){	int cpu, error; 	/* Allow everyone to use the CPU hotplug again */	cpu_maps_update_begin();	__cpu_hotplug_enable();	if (cpumask_empty(frozen_cpus))		goto out; 	pr_info("Enabling non-boot CPUs ...\n"); 	arch_thaw_secondary_cpus_begin(); 	for_each_cpu(cpu, frozen_cpus) {		trace_suspend_resume(TPS("CPU_ON"), cpu, true);		error = _cpu_up(cpu, 1, CPUHP_ONLINE);		trace_suspend_resume(TPS("CPU_ON"), cpu, false);		if (!error) {			pr_info("CPU%d is up\n", cpu);			continue;		}		pr_warn("Error taking CPU%d up: %d\n", cpu, error);	} 	arch_thaw_secondary_cpus_end(); 	cpumask_clear(frozen_cpus);out:	cpu_maps_update_done();} static int __init alloc_frozen_cpus(void){	if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))		return -ENOMEM;	return 0;}core_initcall(alloc_frozen_cpus); /* * When callbacks for CPU hotplug notifications are being executed, we must * ensure that the state of the system with respect to the tasks being frozen * or not, as reported by the notification, remains unchanged *throughout the * duration* of the execution of the callbacks. * Hence we need to prevent the freezer from racing with regular CPU hotplug. * * This synchronization is implemented by mutually excluding regular CPU * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/ * Hibernate notifications. */static intcpu_hotplug_pm_callback(struct notifier_block *nb,			unsigned long action, void *ptr){	switch (action) { 	case PM_SUSPEND_PREPARE:	case PM_HIBERNATION_PREPARE:		cpu_hotplug_disable();		break; 	case PM_POST_SUSPEND:	case PM_POST_HIBERNATION:		cpu_hotplug_enable();		break; 	default:		return NOTIFY_DONE;	} 	return NOTIFY_OK;}  static int __init cpu_hotplug_pm_sync_init(void){	/*	 * cpu_hotplug_pm_callback has higher priority than x86	 * bsp_pm_callback which depends on cpu_hotplug_pm_callback	 * to disable cpu hotplug to avoid cpu hotplug race.	 */	pm_notifier(cpu_hotplug_pm_callback, 0);	return 0;}core_initcall(cpu_hotplug_pm_sync_init); #endif /* CONFIG_PM_SLEEP_SMP */ 

Lines 1001–1500 of 2554. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t