struct cpuhp_cpu_state *st,						      enum cpuhp_state target){	__cpuhp_invoke_callback_range(bringup, cpu, st, target, true);} static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st){	if (IS_ENABLED(CONFIG_HOTPLUG_CPU))		return true;	/*	 * When CPU hotplug is disabled, then taking the CPU down is not	 * possible because takedown_cpu() and the architecture and	 * subsystem specific mechanisms are not available. So the CPU	 * which would be completely unplugged again needs to stay around	 * in the current state.	 */	return st->state <= CPUHP_BRINGUP_CPU;} static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,			      enum cpuhp_state target){	enum cpuhp_state prev_state = st->state;	int ret = 0; 	ret = cpuhp_invoke_callback_range(true, cpu, st, target);	if (ret) {		pr_debug("CPU UP failed (%d) CPU %u state %s (%d)\n",			 ret, cpu, cpuhp_get_step(st->state)->name,			 st->state); 		cpuhp_reset_state(cpu, st, prev_state);		if (can_rollback_cpu(st))			WARN_ON(cpuhp_invoke_callback_range(false, cpu, st,							    prev_state));	}	return ret;} /* * The cpu hotplug threads manage the bringup and teardown of the cpus */static int cpuhp_should_run(unsigned int cpu){	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 	return st->should_run;} /* * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke * callbacks when a state gets [un]installed at runtime. * * Each invocation of this function by the smpboot thread does a single AP * state callback. * * It has 3 modes of operation: *  - single: runs st->cb_state *  - up:     runs ++st->state, while st->state < st->target *  - down:   runs st->state--, while st->state > st->target * * When complete or on error, should_run is cleared and the completion is fired. */static void cpuhp_thread_fun(unsigned int cpu){	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);	bool bringup = st->bringup;	enum cpuhp_state state; 	if (WARN_ON_ONCE(!st->should_run))		return; 	/*	 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures	 * that if we see ->should_run we also see the rest of the state.	 */	smp_mb(); 	/*	 * The BP holds the hotplug lock, but we're now running on the AP,	 * ensure that anybody asserting the lock is held, will actually find	 * it so.	 */	lockdep_acquire_cpus_lock();	cpuhp_lock_acquire(bringup); 	if (st->single) {		state = st->cb_state;		st->should_run = false;	} else {		st->should_run = cpuhp_next_state(bringup, &state, st, st->target);		if (!st->should_run)			goto end;	} 	WARN_ON_ONCE(!cpuhp_is_ap_state(state)); 	if (cpuhp_is_atomic_state(state)) {		local_irq_disable();		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);		local_irq_enable(); 		/*		 * STARTING/DYING must not fail!		 */		WARN_ON_ONCE(st->result);	} else {		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);	} 	if (st->result) {		/*		 * If we fail on a rollback, we're up a creek without no		 * paddle, no way forward, no way back. We loose, thanks for		 * playing.		 */		WARN_ON_ONCE(st->rollback);		st->should_run = false;	} end:	cpuhp_lock_release(bringup);	lockdep_release_cpus_lock(); 	if (!st->should_run)		complete_ap_thread(st, bringup);} /* Invoke a single callback on a remote cpu */static intcpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,			 struct hlist_node *node){	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);	int ret; 	if (!cpu_online(cpu))		return 0; 	cpuhp_lock_acquire(false);	cpuhp_lock_release(false); 	cpuhp_lock_acquire(true);	cpuhp_lock_release(true); 	/*	 * If we are up and running, use the hotplug thread. For early calls	 * we invoke the thread function directly.	 */	if (!st->thread)		return cpuhp_invoke_callback(cpu, state, bringup, node, NULL); 	st->rollback = false;	st->last = NULL; 	st->node = node;	st->bringup = bringup;	st->cb_state = state;	st->single = true; 	__cpuhp_kick_ap(st); 	/*	 * If we failed and did a partial, do a rollback.	 */	if ((ret = st->result) && st->last) {		st->rollback = true;		st->bringup = !bringup; 		__cpuhp_kick_ap(st);	} 	/*	 * Clean up the leftovers so the next hotplug operation wont use stale	 * data.	 */	st->node = st->last = NULL;	return ret;} static int cpuhp_kick_ap_work(unsigned int cpu){	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);	enum cpuhp_state prev_state = st->state;	int ret; 	cpuhp_lock_acquire(false);	cpuhp_lock_release(false); 	cpuhp_lock_acquire(true);	cpuhp_lock_release(true); 	trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);	ret = cpuhp_kick_ap(cpu, st, st->target);	trace_cpuhp_exit(cpu, st->state, prev_state, ret); 	return ret;} static struct smp_hotplug_thread cpuhp_threads = {	.store			= &cpuhp_state.thread,	.thread_should_run	= cpuhp_should_run,	.thread_fn		= cpuhp_thread_fun,	.thread_comm		= "cpuhp/%u",	.selfparking		= true,}; static __init void cpuhp_init_state(void){	struct cpuhp_cpu_state *st;	int cpu; 	for_each_possible_cpu(cpu) {		st = per_cpu_ptr(&cpuhp_state, cpu);		init_completion(&st->done_up);		init_completion(&st->done_down);	}} void __init cpuhp_threads_init(void){	cpuhp_init_state();	BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));	kthread_unpark(this_cpu_read(cpuhp_state.thread));} #ifdef CONFIG_HOTPLUG_CPU#ifndef arch_clear_mm_cpumask_cpu#define arch_clear_mm_cpumask_cpu(cpu, mm) cpumask_clear_cpu(cpu, mm_cpumask(mm))#endif /** * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU * @cpu: a CPU id * * This function walks all processes, finds a valid mm struct for each one and * then clears a corresponding bit in mm's cpumask.  While this all sounds * trivial, there are various non-obvious corner cases, which this function * tries to solve in a safe manner. * * Also note that the function uses a somewhat relaxed locking scheme, so it may * be called only for an already offlined CPU. */void clear_tasks_mm_cpumask(int cpu){	struct task_struct *p; 	/*	 * This function is called after the cpu is taken down and marked	 * offline, so its not like new tasks will ever get this cpu set in	 * their mm mask. -- Peter Zijlstra	 * Thus, we may use rcu_read_lock() here, instead of grabbing	 * full-fledged tasklist_lock.	 */	WARN_ON(cpu_online(cpu));	rcu_read_lock();	for_each_process(p) {		struct task_struct *t; 		/*		 * Main thread might exit, but other threads may still have		 * a valid mm. Find one.		 */		t = find_lock_task_mm(p);		if (!t)			continue;		arch_clear_mm_cpumask_cpu(cpu, t->mm);		task_unlock(t);	}	rcu_read_unlock();} /* Take this CPU down. */static int take_cpu_down(void *_param){	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);	enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);	int err, cpu = smp_processor_id(); 	/* Ensure this CPU doesn't handle any more interrupts. */	err = __cpu_disable();	if (err < 0)		return err; 	/*	 * Must be called from CPUHP_TEARDOWN_CPU, which means, as we are going	 * down, that the current state is CPUHP_TEARDOWN_CPU - 1.	 */	WARN_ON(st->state != (CPUHP_TEARDOWN_CPU - 1)); 	/*	 * Invoke the former CPU_DYING callbacks. DYING must not fail!	 */	cpuhp_invoke_callback_range_nofail(false, cpu, st, target); 	/* Park the stopper thread */	stop_machine_park(cpu);	return 0;} static int takedown_cpu(unsigned int cpu){	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);	int err; 	/* Park the smpboot threads */	kthread_park(st->thread); 	/*	 * Prevent irq alloc/free while the dying cpu reorganizes the	 * interrupt affinities.	 */	irq_lock_sparse(); 	err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));	if (err) {		/* CPU refused to die */		irq_unlock_sparse();		/* Unpark the hotplug thread so we can rollback there */		kthread_unpark(st->thread);		return err;	}	BUG_ON(cpu_online(cpu)); 	/*	 * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed	 * all runnable tasks from the CPU, there's only the idle task left now	 * that the migration thread is done doing the stop_machine thing.	 *	 * Wait for the stop thread to go away.	 */	wait_for_ap_thread(st, false);	BUG_ON(st->state != CPUHP_AP_IDLE_DEAD); 	/* Interrupts are moved away from the dying cpu, reenable alloc/free */	irq_unlock_sparse(); 	hotplug_cpu__broadcast_tick_pull(cpu);	/* This actually kills the CPU. */	__cpu_die(cpu); 	cpuhp_bp_sync_dead(cpu); 	lockdep_cleanup_dead_cpu(cpu, idle_thread_get(cpu)); 	/*	 * Callbacks must be re-integrated right away to the RCU state machine.	 * Otherwise an RCU callback could block a further teardown function	 * waiting for its completion.	 */	rcutree_migrate_callbacks(cpu); 	return 0;} static void cpuhp_complete_idle_dead(void *arg){	struct cpuhp_cpu_state *st = arg; 	complete_ap_thread(st, false);} void cpuhp_report_idle_dead(void){	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 	BUG_ON(st->state != CPUHP_AP_OFFLINE);	tick_assert_timekeeping_handover();	rcutree_report_cpu_dead();	st->state = CPUHP_AP_IDLE_DEAD;	/*	 * We cannot call complete after rcutree_report_cpu_dead() so we delegate it	 * to an online cpu.	 */	smp_call_function_single(cpumask_first(cpu_online_mask),				 cpuhp_complete_idle_dead, st, 0);} static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,				enum cpuhp_state target){	enum cpuhp_state prev_state = st->state;	int ret = 0; 	ret = cpuhp_invoke_callback_range(false, cpu, st, target);	if (ret) {		pr_debug("CPU DOWN failed (%d) CPU %u state %s (%d)\n",			 ret, cpu, cpuhp_get_step(st->state)->name,			 st->state); 		cpuhp_reset_state(cpu, st, prev_state); 		if (st->state < prev_state)			WARN_ON(cpuhp_invoke_callback_range(true, cpu, st,							    prev_state));	} 	return ret;} /* Requires cpu_add_remove_lock to be held */static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,			   enum cpuhp_state target){	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);	int prev_state, ret = 0; 	if (num_online_cpus() == 1)		return -EBUSY; 	if (!cpu_present(cpu))		return -EINVAL; 	cpus_write_lock(); 	/*	 * Keep at least one housekeeping cpu onlined to avoid generating	 * an empty sched_domain span.	 */	if (cpumask_any_and(cpu_online_mask,			    housekeeping_cpumask(HK_TYPE_DOMAIN)) >= nr_cpu_ids) {		ret = -EBUSY;		goto out;	} 	cpuhp_tasks_frozen = tasks_frozen; 	prev_state = cpuhp_set_state(cpu, st, target);	/*	 * If the current CPU state is in the range of the AP hotplug thread,	 * then we need to kick the thread.	 */	if (st->state > CPUHP_TEARDOWN_CPU) {		st->target = max((int)target, CPUHP_TEARDOWN_CPU);		ret = cpuhp_kick_ap_work(cpu);		/*		 * The AP side has done the error rollback already. Just		 * return the error code..		 */		if (ret)			goto out; 		/*		 * We might have stopped still in the range of the AP hotplug		 * thread. Nothing to do anymore.		 */		if (st->state > CPUHP_TEARDOWN_CPU)			goto out; 		st->target = target;	}	/*	 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need	 * to do the further cleanups.	 */	ret = cpuhp_down_callbacks(cpu, st, target);	if (ret && st->state < prev_state) {		if (st->state == CPUHP_TEARDOWN_CPU) {			cpuhp_reset_state(cpu, st, prev_state);			__cpuhp_kick_ap(st);		} else {			WARN(1, "DEAD callback error for CPU%d", cpu);		}	} out:	cpus_write_unlock();	arch_smt_update();	return ret;} static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target){	/*	 * If the platform does not support hotplug, report it explicitly to	 * differentiate it from a transient offlining failure.	 */	if (cpu_hotplug_offline_disabled)		return -EOPNOTSUPP;	if (cpu_hotplug_disabled)		return -EBUSY;	return _cpu_down(cpu, 0, target);} static int cpu_down(unsigned int cpu, enum cpuhp_state target){	int err; 	cpu_maps_update_begin();	err = cpu_down_maps_locked(cpu, target);	cpu_maps_update_done();	return err;} /** * cpu_device_down - Bring down a cpu device * @dev: Pointer to the cpu device to offline * * This function is meant to be used by device core cpu subsystem only.

Lines 1001–1500 of 3339. Previous lines Load more