2554 lines · 60.8 KB · C
RawDownload
	wake_up_process(st->thread);	wait_for_ap_thread(st, st->bringup);} static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target){	enum cpuhp_state prev_state;	int ret; 	prev_state = cpuhp_set_state(st, target);	__cpuhp_kick_ap(st);	if ((ret = st->result)) {		cpuhp_reset_state(st, prev_state);		__cpuhp_kick_ap(st);	} 	return ret;} static int bringup_wait_for_ap(unsigned int cpu){	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 	/* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */	wait_for_ap_thread(st, true);	if (WARN_ON_ONCE((!cpu_online(cpu))))		return -ECANCELED; 	/* Unpark the hotplug thread of the target cpu */	kthread_unpark(st->thread); 	/*	 * SMT soft disabling on X86 requires to bring the CPU out of the	 * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit.  The	 * CPU marked itself as booted_once in notify_cpu_starting() so the	 * cpu_smt_allowed() check will now return false if this is not the	 * primary sibling.	 */	if (!cpu_smt_allowed(cpu))		return -ECANCELED; 	if (st->target <= CPUHP_AP_ONLINE_IDLE)		return 0; 	return cpuhp_kick_ap(st, st->target);} static int bringup_cpu(unsigned int cpu){	struct task_struct *idle = idle_thread_get(cpu);	int ret; 	/*	 * Some architectures have to walk the irq descriptors to	 * setup the vector space for the cpu which comes online.	 * Prevent irq alloc/free across the bringup.	 */	irq_lock_sparse(); 	/* Arch-specific enabling code. */	ret = __cpu_up(cpu, idle);	irq_unlock_sparse();	if (ret)		return ret;	return bringup_wait_for_ap(cpu);} static int finish_cpu(unsigned int cpu){	struct task_struct *idle = idle_thread_get(cpu);	struct mm_struct *mm = idle->active_mm; 	/*	 * idle_task_exit() will have switched to &init_mm, now	 * clean up any remaining active_mm state.	 */	if (mm != &init_mm)		idle->active_mm = &init_mm;	mmdrop(mm);	return 0;} /* * Hotplug state machine related functions */ static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st){	for (st->state--; st->state > st->target; st->state--)		cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);} static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st){	if (IS_ENABLED(CONFIG_HOTPLUG_CPU))		return true;	/*	 * When CPU hotplug is disabled, then taking the CPU down is not	 * possible because takedown_cpu() and the architecture and	 * subsystem specific mechanisms are not available. So the CPU	 * which would be completely unplugged again needs to stay around	 * in the current state.	 */	return st->state <= CPUHP_BRINGUP_CPU;} static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,			      enum cpuhp_state target){	enum cpuhp_state prev_state = st->state;	int ret = 0; 	while (st->state < target) {		st->state++;		ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);		if (ret) {			if (can_rollback_cpu(st)) {				st->target = prev_state;				undo_cpu_up(cpu, st);			}			break;		}	}	return ret;} /* * The cpu hotplug threads manage the bringup and teardown of the cpus */static void cpuhp_create(unsigned int cpu){	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 	init_completion(&st->done_up);	init_completion(&st->done_down);} static int cpuhp_should_run(unsigned int cpu){	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 	return st->should_run;} /* * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke * callbacks when a state gets [un]installed at runtime. * * Each invocation of this function by the smpboot thread does a single AP * state callback. * * It has 3 modes of operation: *  - single: runs st->cb_state *  - up:     runs ++st->state, while st->state < st->target *  - down:   runs st->state--, while st->state > st->target * * When complete or on error, should_run is cleared and the completion is fired. */static void cpuhp_thread_fun(unsigned int cpu){	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);	bool bringup = st->bringup;	enum cpuhp_state state; 	if (WARN_ON_ONCE(!st->should_run))		return; 	/*	 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures	 * that if we see ->should_run we also see the rest of the state.	 */	smp_mb(); 	/*	 * The BP holds the hotplug lock, but we're now running on the AP,	 * ensure that anybody asserting the lock is held, will actually find	 * it so.	 */	lockdep_acquire_cpus_lock();	cpuhp_lock_acquire(bringup); 	if (st->single) {		state = st->cb_state;		st->should_run = false;	} else {		if (bringup) {			st->state++;			state = st->state;			st->should_run = (st->state < st->target);			WARN_ON_ONCE(st->state > st->target);		} else {			state = st->state;			st->state--;			st->should_run = (st->state > st->target);			WARN_ON_ONCE(st->state < st->target);		}	} 	WARN_ON_ONCE(!cpuhp_is_ap_state(state)); 	if (cpuhp_is_atomic_state(state)) {		local_irq_disable();		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);		local_irq_enable(); 		/*		 * STARTING/DYING must not fail!		 */		WARN_ON_ONCE(st->result);	} else {		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);	} 	if (st->result) {		/*		 * If we fail on a rollback, we're up a creek without no		 * paddle, no way forward, no way back. We loose, thanks for		 * playing.		 */		WARN_ON_ONCE(st->rollback);		st->should_run = false;	} 	cpuhp_lock_release(bringup);	lockdep_release_cpus_lock(); 	if (!st->should_run)		complete_ap_thread(st, bringup);} /* Invoke a single callback on a remote cpu */static intcpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,			 struct hlist_node *node){	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);	int ret; 	if (!cpu_online(cpu))		return 0; 	cpuhp_lock_acquire(false);	cpuhp_lock_release(false); 	cpuhp_lock_acquire(true);	cpuhp_lock_release(true); 	/*	 * If we are up and running, use the hotplug thread. For early calls	 * we invoke the thread function directly.	 */	if (!st->thread)		return cpuhp_invoke_callback(cpu, state, bringup, node, NULL); 	st->rollback = false;	st->last = NULL; 	st->node = node;	st->bringup = bringup;	st->cb_state = state;	st->single = true; 	__cpuhp_kick_ap(st); 	/*	 * If we failed and did a partial, do a rollback.	 */	if ((ret = st->result) && st->last) {		st->rollback = true;		st->bringup = !bringup; 		__cpuhp_kick_ap(st);	} 	/*	 * Clean up the leftovers so the next hotplug operation wont use stale	 * data.	 */	st->node = st->last = NULL;	return ret;} static int cpuhp_kick_ap_work(unsigned int cpu){	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);	enum cpuhp_state prev_state = st->state;	int ret; 	cpuhp_lock_acquire(false);	cpuhp_lock_release(false); 	cpuhp_lock_acquire(true);	cpuhp_lock_release(true); 	trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);	ret = cpuhp_kick_ap(st, st->target);	trace_cpuhp_exit(cpu, st->state, prev_state, ret); 	return ret;} static struct smp_hotplug_thread cpuhp_threads = {	.store			= &cpuhp_state.thread,	.create			= &cpuhp_create,	.thread_should_run	= cpuhp_should_run,	.thread_fn		= cpuhp_thread_fun,	.thread_comm		= "cpuhp/%u",	.selfparking		= true,}; void __init cpuhp_threads_init(void){	BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));	kthread_unpark(this_cpu_read(cpuhp_state.thread));} #ifdef CONFIG_HOTPLUG_CPU/** * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU * @cpu: a CPU id * * This function walks all processes, finds a valid mm struct for each one and * then clears a corresponding bit in mm's cpumask.  While this all sounds * trivial, there are various non-obvious corner cases, which this function * tries to solve in a safe manner. * * Also note that the function uses a somewhat relaxed locking scheme, so it may * be called only for an already offlined CPU. */void clear_tasks_mm_cpumask(int cpu){	struct task_struct *p; 	/*	 * This function is called after the cpu is taken down and marked	 * offline, so its not like new tasks will ever get this cpu set in	 * their mm mask. -- Peter Zijlstra	 * Thus, we may use rcu_read_lock() here, instead of grabbing	 * full-fledged tasklist_lock.	 */	WARN_ON(cpu_online(cpu));	rcu_read_lock();	for_each_process(p) {		struct task_struct *t; 		/*		 * Main thread might exit, but other threads may still have		 * a valid mm. Find one.		 */		t = find_lock_task_mm(p);		if (!t)			continue;		cpumask_clear_cpu(cpu, mm_cpumask(t->mm));		task_unlock(t);	}	rcu_read_unlock();} /* Take this CPU down. */static int take_cpu_down(void *_param){	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);	enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);	int err, cpu = smp_processor_id();	int ret; 	/* Ensure this CPU doesn't handle any more interrupts. */	err = __cpu_disable();	if (err < 0)		return err; 	/*	 * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not	 * do this step again.	 */	WARN_ON(st->state != CPUHP_TEARDOWN_CPU);	st->state--;	/* Invoke the former CPU_DYING callbacks */	for (; st->state > target; st->state--) {		ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);		/*		 * DYING must not fail!		 */		WARN_ON_ONCE(ret);	} 	/* Give up timekeeping duties */	tick_handover_do_timer();	/* Remove CPU from timer broadcasting */	tick_offline_cpu(cpu);	/* Park the stopper thread */	stop_machine_park(cpu);	return 0;} static int takedown_cpu(unsigned int cpu){	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);	int err; 	/* Park the smpboot threads */	kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread); 	/*	 * Prevent irq alloc/free while the dying cpu reorganizes the	 * interrupt affinities.	 */	irq_lock_sparse(); 	/*	 * So now all preempt/rcu users must observe !cpu_active().	 */	err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));	if (err) {		/* CPU refused to die */		irq_unlock_sparse();		/* Unpark the hotplug thread so we can rollback there */		kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);		return err;	}	BUG_ON(cpu_online(cpu)); 	/*	 * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed	 * all runnable tasks from the CPU, there's only the idle task left now	 * that the migration thread is done doing the stop_machine thing.	 *	 * Wait for the stop thread to go away.	 */	wait_for_ap_thread(st, false);	BUG_ON(st->state != CPUHP_AP_IDLE_DEAD); 	/* Interrupts are moved away from the dying cpu, reenable alloc/free */	irq_unlock_sparse(); 	hotplug_cpu__broadcast_tick_pull(cpu);	/* This actually kills the CPU. */	__cpu_die(cpu); 	tick_cleanup_dead_cpu(cpu);	rcutree_migrate_callbacks(cpu);	return 0;} static void cpuhp_complete_idle_dead(void *arg){	struct cpuhp_cpu_state *st = arg; 	complete_ap_thread(st, false);} void cpuhp_report_idle_dead(void){	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 	BUG_ON(st->state != CPUHP_AP_OFFLINE);	rcu_report_dead(smp_processor_id());	st->state = CPUHP_AP_IDLE_DEAD;	/*	 * We cannot call complete after rcu_report_dead() so we delegate it	 * to an online cpu.	 */	smp_call_function_single(cpumask_first(cpu_online_mask),				 cpuhp_complete_idle_dead, st, 0);} static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st){	for (st->state++; st->state < st->target; st->state++)		cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);} static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,				enum cpuhp_state target){	enum cpuhp_state prev_state = st->state;	int ret = 0; 	for (; st->state > target; st->state--) {		ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);		if (ret) {			st->target = prev_state;			if (st->state < prev_state)				undo_cpu_down(cpu, st);			break;		}	}	return ret;} /* Requires cpu_add_remove_lock to be held */static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,			   enum cpuhp_state target){	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);	int prev_state, ret = 0; 	if (num_online_cpus() == 1)		return -EBUSY; 

Lines 501–1000 of 2554. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t