/* CPU control. * (C) 2001, 2002, 2003, 2004 Rusty Russell * * This code is licenced under the GPL. */#include <linux/proc_fs.h>#include <linux/smp.h>#include <linux/init.h>#include <linux/notifier.h>#include <linux/sched.h>#include <linux/unistd.h>#include <linux/cpu.h>#include <linux/oom.h>#include <linux/rcupdate.h>#include <linux/export.h>#include <linux/bug.h>#include <linux/kthread.h>#include <linux/stop_machine.h>#include <linux/mutex.h>#include <linux/gfp.h>#include <linux/suspend.h>#include <linux/lockdep.h>#include <linux/tick.h>#include <linux/irq.h>#include <trace/events/power.h> #include "smpboot.h" #ifdef CONFIG_SMP/* Serializes the updates to cpu_online_mask, cpu_present_mask */static DEFINE_MUTEX(cpu_add_remove_lock); /* * The following two APIs (cpu_maps_update_begin/done) must be used when * attempting to serialize the updates to cpu_online_mask & cpu_present_mask. * The APIs cpu_notifier_register_begin/done() must be used to protect CPU * hotplug callback (un)registration performed using __register_cpu_notifier() * or __unregister_cpu_notifier(). */void cpu_maps_update_begin(void){ mutex_lock(&cpu_add_remove_lock);}EXPORT_SYMBOL(cpu_notifier_register_begin); void cpu_maps_update_done(void){ mutex_unlock(&cpu_add_remove_lock);}EXPORT_SYMBOL(cpu_notifier_register_done); static RAW_NOTIFIER_HEAD(cpu_chain); /* If set, cpu_up and cpu_down will return -EBUSY and do nothing. * Should always be manipulated under cpu_add_remove_lock */static int cpu_hotplug_disabled; #ifdef CONFIG_HOTPLUG_CPU static struct { struct task_struct *active_writer; /* wait queue to wake up the active_writer */ wait_queue_head_t wq; /* verifies that no writer will get active while readers are active */ struct mutex lock; /* * Also blocks the new readers during * an ongoing cpu hotplug operation. */ atomic_t refcount; #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map;#endif} cpu_hotplug = { .active_writer = NULL, .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq), .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),#ifdef CONFIG_DEBUG_LOCK_ALLOC .dep_map = {.name = "cpu_hotplug.lock" },#endif}; /* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */#define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map)#define cpuhp_lock_acquire_tryread() \ lock_map_acquire_tryread(&cpu_hotplug.dep_map)#define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map)#define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map) void get_online_cpus(void){ might_sleep(); if (cpu_hotplug.active_writer == current) return; cpuhp_lock_acquire_read(); mutex_lock(&cpu_hotplug.lock); atomic_inc(&cpu_hotplug.refcount); mutex_unlock(&cpu_hotplug.lock);}EXPORT_SYMBOL_GPL(get_online_cpus); void put_online_cpus(void){ int refcount; if (cpu_hotplug.active_writer == current) return; refcount = atomic_dec_return(&cpu_hotplug.refcount); if (WARN_ON(refcount < 0)) /* try to fix things up */ atomic_inc(&cpu_hotplug.refcount); if (refcount <= 0 && waitqueue_active(&cpu_hotplug.wq)) wake_up(&cpu_hotplug.wq); cpuhp_lock_release(); }EXPORT_SYMBOL_GPL(put_online_cpus); /* * This ensures that the hotplug operation can begin only when the * refcount goes to zero. * * Note that during a cpu-hotplug operation, the new readers, if any, * will be blocked by the cpu_hotplug.lock * * Since cpu_hotplug_begin() is always called after invoking * cpu_maps_update_begin(), we can be sure that only one writer is active. * * Note that theoretically, there is a possibility of a livelock: * - Refcount goes to zero, last reader wakes up the sleeping * writer. * - Last reader unlocks the cpu_hotplug.lock. * - A new reader arrives at this moment, bumps up the refcount. * - The writer acquires the cpu_hotplug.lock finds the refcount * non zero and goes to sleep again. * * However, this is very difficult to achieve in practice since * get_online_cpus() not an api which is called all that often. * */void cpu_hotplug_begin(void){ DEFINE_WAIT(wait); cpu_hotplug.active_writer = current; cpuhp_lock_acquire(); for (;;) { mutex_lock(&cpu_hotplug.lock); prepare_to_wait(&cpu_hotplug.wq, &wait, TASK_UNINTERRUPTIBLE); if (likely(!atomic_read(&cpu_hotplug.refcount))) break; mutex_unlock(&cpu_hotplug.lock); schedule(); } finish_wait(&cpu_hotplug.wq, &wait);} void cpu_hotplug_done(void){ cpu_hotplug.active_writer = NULL; mutex_unlock(&cpu_hotplug.lock); cpuhp_lock_release();} /* * Wait for currently running CPU hotplug operations to complete (if any) and * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the * hotplug path before performing hotplug operations. So acquiring that lock * guarantees mutual exclusion from any currently running hotplug operations. */void cpu_hotplug_disable(void){ cpu_maps_update_begin(); cpu_hotplug_disabled++; cpu_maps_update_done();}EXPORT_SYMBOL_GPL(cpu_hotplug_disable); void cpu_hotplug_enable(void){ cpu_maps_update_begin(); WARN_ON(--cpu_hotplug_disabled < 0); cpu_maps_update_done();}EXPORT_SYMBOL_GPL(cpu_hotplug_enable);#endif /* CONFIG_HOTPLUG_CPU */ /* Need to know about CPUs going up/down? */int register_cpu_notifier(struct notifier_block *nb){ int ret; cpu_maps_update_begin(); ret = raw_notifier_chain_register(&cpu_chain, nb); cpu_maps_update_done(); return ret;} int __register_cpu_notifier(struct notifier_block *nb){ return raw_notifier_chain_register(&cpu_chain, nb);} static int __cpu_notify(unsigned long val, void *v, int nr_to_call, int *nr_calls){ int ret; ret = __raw_notifier_call_chain(&cpu_chain, val, v, nr_to_call, nr_calls); return notifier_to_errno(ret);} static int cpu_notify(unsigned long val, void *v){ return __cpu_notify(val, v, -1, NULL);} #ifdef CONFIG_HOTPLUG_CPU static void cpu_notify_nofail(unsigned long val, void *v){ BUG_ON(cpu_notify(val, v));}EXPORT_SYMBOL(register_cpu_notifier);EXPORT_SYMBOL(__register_cpu_notifier); void unregister_cpu_notifier(struct notifier_block *nb){ cpu_maps_update_begin(); raw_notifier_chain_unregister(&cpu_chain, nb); cpu_maps_update_done();}EXPORT_SYMBOL(unregister_cpu_notifier); void __unregister_cpu_notifier(struct notifier_block *nb){ raw_notifier_chain_unregister(&cpu_chain, nb);}EXPORT_SYMBOL(__unregister_cpu_notifier); /** * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU * @cpu: a CPU id * * This function walks all processes, finds a valid mm struct for each one and * then clears a corresponding bit in mm's cpumask. While this all sounds * trivial, there are various non-obvious corner cases, which this function * tries to solve in a safe manner. * * Also note that the function uses a somewhat relaxed locking scheme, so it may * be called only for an already offlined CPU. */void clear_tasks_mm_cpumask(int cpu){ struct task_struct *p; /* * This function is called after the cpu is taken down and marked * offline, so its not like new tasks will ever get this cpu set in * their mm mask. -- Peter Zijlstra * Thus, we may use rcu_read_lock() here, instead of grabbing * full-fledged tasklist_lock. */ WARN_ON(cpu_online(cpu)); rcu_read_lock(); for_each_process(p) { struct task_struct *t; /* * Main thread might exit, but other threads may still have * a valid mm. Find one. */ t = find_lock_task_mm(p); if (!t) continue; cpumask_clear_cpu(cpu, mm_cpumask(t->mm)); task_unlock(t); } rcu_read_unlock();} static inline void check_for_tasks(int dead_cpu){ struct task_struct *g, *p; read_lock(&tasklist_lock); for_each_process_thread(g, p) { if (!p->on_rq) continue; /* * We do the check with unlocked task_rq(p)->lock. * Order the reading to do not warn about a task, * which was running on this cpu in the past, and * it's just been woken on another cpu. */ rmb(); if (task_cpu(p) != dead_cpu) continue; pr_warn("Task %s (pid=%d) is on cpu %d (state=%ld, flags=%x)\n", p->comm, task_pid_nr(p), dead_cpu, p->state, p->flags); } read_unlock(&tasklist_lock);} struct take_cpu_down_param { unsigned long mod; void *hcpu;}; /* Take this CPU down. */static int take_cpu_down(void *_param){ struct take_cpu_down_param *param = _param; int err; /* Ensure this CPU doesn't handle any more interrupts. */ err = __cpu_disable(); if (err < 0) return err; cpu_notify(CPU_DYING | param->mod, param->hcpu); /* Give up timekeeping duties */ tick_handover_do_timer(); /* Park the stopper thread */ stop_machine_park((long)param->hcpu); return 0;} /* Requires cpu_add_remove_lock to be held */static int _cpu_down(unsigned int cpu, int tasks_frozen){ int err, nr_calls = 0; void *hcpu = (void *)(long)cpu; unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0; struct take_cpu_down_param tcd_param = { .mod = mod, .hcpu = hcpu, }; if (num_online_cpus() == 1) return -EBUSY; if (!cpu_online(cpu)) return -EINVAL; cpu_hotplug_begin(); err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls); if (err) { nr_calls--; __cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL); pr_warn("%s: attempt to take down CPU %u failed\n", __func__, cpu); goto out_release; } /* * By now we've cleared cpu_active_mask, wait for all preempt-disabled * and RCU users of this state to go away such that all new such users * will observe it. * * For CONFIG_PREEMPT we have preemptible RCU and its sync_rcu() might * not imply sync_sched(), so wait for both. * * Do sync before park smpboot threads to take care the rcu boost case. */ if (IS_ENABLED(CONFIG_PREEMPT)) synchronize_rcu_mult(call_rcu, call_rcu_sched); else synchronize_rcu(); smpboot_park_threads(cpu); /* * Prevent irq alloc/free while the dying cpu reorganizes the * interrupt affinities. */ irq_lock_sparse(); /* * So now all preempt/rcu users must observe !cpu_active(). */ err = stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu)); if (err) { /* CPU didn't die: tell everyone. Can't complain. */ cpu_notify_nofail(CPU_DOWN_FAILED | mod, hcpu); irq_unlock_sparse(); goto out_release; } BUG_ON(cpu_online(cpu)); /* * The migration_call() CPU_DYING callback will have removed all * runnable tasks from the cpu, there's only the idle task left now * that the migration thread is done doing the stop_machine thing. * * Wait for the stop thread to go away. */ while (!per_cpu(cpu_dead_idle, cpu)) cpu_relax(); smp_mb(); /* Read from cpu_dead_idle before __cpu_die(). */ per_cpu(cpu_dead_idle, cpu) = false; /* Interrupts are moved away from the dying cpu, reenable alloc/free */ irq_unlock_sparse(); hotplug_cpu__broadcast_tick_pull(cpu); /* This actually kills the CPU. */ __cpu_die(cpu); /* CPU is completely dead: tell everyone. Too late to complain. */ tick_cleanup_dead_cpu(cpu); cpu_notify_nofail(CPU_DEAD | mod, hcpu); check_for_tasks(cpu); out_release: cpu_hotplug_done(); if (!err) cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu); return err;} int cpu_down(unsigned int cpu){ int err; cpu_maps_update_begin(); if (cpu_hotplug_disabled) { err = -EBUSY; goto out; } err = _cpu_down(cpu, 0); out: cpu_maps_update_done(); return err;}EXPORT_SYMBOL(cpu_down);#endif /*CONFIG_HOTPLUG_CPU*/ /* * Unpark per-CPU smpboot kthreads at CPU-online time. */static int smpboot_thread_call(struct notifier_block *nfb, unsigned long action, void *hcpu){ int cpu = (long)hcpu; switch (action & ~CPU_TASKS_FROZEN) { case CPU_DOWN_FAILED: case CPU_ONLINE: smpboot_unpark_threads(cpu); break; default: break; } return NOTIFY_OK;} static struct notifier_block smpboot_thread_notifier = { .notifier_call = smpboot_thread_call, .priority = CPU_PRI_SMPBOOT,}; void smpboot_thread_init(void){ register_cpu_notifier(&smpboot_thread_notifier);} /* Requires cpu_add_remove_lock to be held */static int _cpu_up(unsigned int cpu, int tasks_frozen){ int ret, nr_calls = 0; void *hcpu = (void *)(long)cpu; unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0; struct task_struct *idle; cpu_hotplug_begin(); if (cpu_online(cpu) || !cpu_present(cpu)) { ret = -EINVAL; goto out; } idle = idle_thread_get(cpu);Lines 1–500 of 791. Load more
/* CPU control. * (C) 2001, 2002, 2003, 2004 Rusty Russell * * This code is licenced under the GPL. */ #include <linux/proc_fs.h> #include <linux/smp.h> #include <linux/init.h> #include <linux/notifier.h> #include <linux/sched.h> #include <linux/unistd.h> #include <linux/cpu.h> #include <linux/oom.h> #include <linux/rcupdate.h> #include <linux/export.h> #include <linux/bug.h> #include <linux/kthread.h> #include <linux/stop_machine.h> #include <linux/mutex.h> #include <linux/gfp.h> #include <linux/suspend.h> #include <linux/lockdep.h> #include <linux/tick.h> #include <linux/irq.h> #include <trace/events/power.h> #include "smpboot.h" #ifdef CONFIG_SMP /* Serializes the updates to cpu_online_mask, cpu_present_mask */ static DEFINE_MUTEX(cpu_add_remove_lock); /* * The following two APIs (cpu_maps_update_begin/done) must be used when * attempting to serialize the updates to cpu_online_mask & cpu_present_mask. * The APIs cpu_notifier_register_begin/done() must be used to protect CPU * hotplug callback (un)registration performed using __register_cpu_notifier() * or __unregister_cpu_notifier(). */ void cpu_maps_update_begin(void) { mutex_lock(&cpu_add_remove_lock); } EXPORT_SYMBOL(cpu_notifier_register_begin); void cpu_maps_update_done(void) { mutex_unlock(&cpu_add_remove_lock); } EXPORT_SYMBOL(cpu_notifier_register_done); static RAW_NOTIFIER_HEAD(cpu_chain); /* If set, cpu_up and cpu_down will return -EBUSY and do nothing. * Should always be manipulated under cpu_add_remove_lock */ static int cpu_hotplug_disabled; #ifdef CONFIG_HOTPLUG_CPU static struct { struct task_struct *active_writer; /* wait queue to wake up the active_writer */ wait_queue_head_t wq; /* verifies that no writer will get active while readers are active */ struct mutex lock; /* * Also blocks the new readers during * an ongoing cpu hotplug operation. */ atomic_t refcount; #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; #endif } cpu_hotplug = { .active_writer = NULL, .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq), .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock), #ifdef CONFIG_DEBUG_LOCK_ALLOC .dep_map = {.name = "cpu_hotplug.lock" }, #endif }; /* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */ #define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map) #define cpuhp_lock_acquire_tryread() \ lock_map_acquire_tryread(&cpu_hotplug.dep_map) #define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map) #define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map) void get_online_cpus(void) { might_sleep(); if (cpu_hotplug.active_writer == current) return; cpuhp_lock_acquire_read(); mutex_lock(&cpu_hotplug.lock); atomic_inc(&cpu_hotplug.refcount); mutex_unlock(&cpu_hotplug.lock); } EXPORT_SYMBOL_GPL(get_online_cpus); void put_online_cpus(void) { int refcount; if (cpu_hotplug.active_writer == current) return; refcount = atomic_dec_return(&cpu_hotplug.refcount); if (WARN_ON(refcount < 0)) /* try to fix things up */ atomic_inc(&cpu_hotplug.refcount); if (refcount <= 0 && waitqueue_active(&cpu_hotplug.wq)) wake_up(&cpu_hotplug.wq); cpuhp_lock_release(); } EXPORT_SYMBOL_GPL(put_online_cpus); /* * This ensures that the hotplug operation can begin only when the * refcount goes to zero. * * Note that during a cpu-hotplug operation, the new readers, if any, * will be blocked by the cpu_hotplug.lock * * Since cpu_hotplug_begin() is always called after invoking * cpu_maps_update_begin(), we can be sure that only one writer is active. * * Note that theoretically, there is a possibility of a livelock: * - Refcount goes to zero, last reader wakes up the sleeping * writer. * - Last reader unlocks the cpu_hotplug.lock. * - A new reader arrives at this moment, bumps up the refcount. * - The writer acquires the cpu_hotplug.lock finds the refcount * non zero and goes to sleep again. * * However, this is very difficult to achieve in practice since * get_online_cpus() not an api which is called all that often. * */ void cpu_hotplug_begin(void) { DEFINE_WAIT(wait); cpu_hotplug.active_writer = current; cpuhp_lock_acquire(); for (;;) { mutex_lock(&cpu_hotplug.lock); prepare_to_wait(&cpu_hotplug.wq, &wait, TASK_UNINTERRUPTIBLE); if (likely(!atomic_read(&cpu_hotplug.refcount))) break; mutex_unlock(&cpu_hotplug.lock); schedule(); } finish_wait(&cpu_hotplug.wq, &wait); } void cpu_hotplug_done(void) { cpu_hotplug.active_writer = NULL; mutex_unlock(&cpu_hotplug.lock); cpuhp_lock_release(); } /* * Wait for currently running CPU hotplug operations to complete (if any) and * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the * hotplug path before performing hotplug operations. So acquiring that lock * guarantees mutual exclusion from any currently running hotplug operations. */ void cpu_hotplug_disable(void) { cpu_maps_update_begin(); cpu_hotplug_disabled++; cpu_maps_update_done(); } EXPORT_SYMBOL_GPL(cpu_hotplug_disable); void cpu_hotplug_enable(void) { cpu_maps_update_begin(); WARN_ON(--cpu_hotplug_disabled < 0); cpu_maps_update_done(); } EXPORT_SYMBOL_GPL(cpu_hotplug_enable); #endif /* CONFIG_HOTPLUG_CPU */ /* Need to know about CPUs going up/down? */ int register_cpu_notifier(struct notifier_block *nb) { int ret; cpu_maps_update_begin(); ret = raw_notifier_chain_register(&cpu_chain, nb); cpu_maps_update_done(); return ret; } int __register_cpu_notifier(struct notifier_block *nb) { return raw_notifier_chain_register(&cpu_chain, nb); } static int __cpu_notify(unsigned long val, void *v, int nr_to_call, int *nr_calls) { int ret; ret = __raw_notifier_call_chain(&cpu_chain, val, v, nr_to_call, nr_calls); return notifier_to_errno(ret); } static int cpu_notify(unsigned long val, void *v) { return __cpu_notify(val, v, -1, NULL); } #ifdef CONFIG_HOTPLUG_CPU static void cpu_notify_nofail(unsigned long val, void *v) { BUG_ON(cpu_notify(val, v)); } EXPORT_SYMBOL(register_cpu_notifier); EXPORT_SYMBOL(__register_cpu_notifier); void unregister_cpu_notifier(struct notifier_block *nb) { cpu_maps_update_begin(); raw_notifier_chain_unregister(&cpu_chain, nb); cpu_maps_update_done(); } EXPORT_SYMBOL(unregister_cpu_notifier); void __unregister_cpu_notifier(struct notifier_block *nb) { raw_notifier_chain_unregister(&cpu_chain, nb); } EXPORT_SYMBOL(__unregister_cpu_notifier); /** * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU * @cpu: a CPU id * * This function walks all processes, finds a valid mm struct for each one and * then clears a corresponding bit in mm's cpumask. While this all sounds * trivial, there are various non-obvious corner cases, which this function * tries to solve in a safe manner. * * Also note that the function uses a somewhat relaxed locking scheme, so it may * be called only for an already offlined CPU. */ void clear_tasks_mm_cpumask(int cpu) { struct task_struct *p; /* * This function is called after the cpu is taken down and marked * offline, so its not like new tasks will ever get this cpu set in * their mm mask. -- Peter Zijlstra * Thus, we may use rcu_read_lock() here, instead of grabbing * full-fledged tasklist_lock. */ WARN_ON(cpu_online(cpu)); rcu_read_lock(); for_each_process(p) { struct task_struct *t; /* * Main thread might exit, but other threads may still have * a valid mm. Find one. */ t = find_lock_task_mm(p); if (!t) continue; cpumask_clear_cpu(cpu, mm_cpumask(t->mm)); task_unlock(t); } rcu_read_unlock(); } static inline void check_for_tasks(int dead_cpu) { struct task_struct *g, *p; read_lock(&tasklist_lock); for_each_process_thread(g, p) { if (!p->on_rq) continue; /* * We do the check with unlocked task_rq(p)->lock. * Order the reading to do not warn about a task, * which was running on this cpu in the past, and * it's just been woken on another cpu. */ rmb(); if (task_cpu(p) != dead_cpu) continue; pr_warn("Task %s (pid=%d) is on cpu %d (state=%ld, flags=%x)\n", p->comm, task_pid_nr(p), dead_cpu, p->state, p->flags); } read_unlock(&tasklist_lock); } struct take_cpu_down_param { unsigned long mod; void *hcpu; }; /* Take this CPU down. */ static int take_cpu_down(void *_param) { struct take_cpu_down_param *param = _param; int err; /* Ensure this CPU doesn't handle any more interrupts. */ err = __cpu_disable(); if (err < 0) return err; cpu_notify(CPU_DYING | param->mod, param->hcpu); /* Give up timekeeping duties */ tick_handover_do_timer(); /* Park the stopper thread */ stop_machine_park((long)param->hcpu); return 0; } /* Requires cpu_add_remove_lock to be held */ static int _cpu_down(unsigned int cpu, int tasks_frozen) { int err, nr_calls = 0; void *hcpu = (void *)(long)cpu; unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0; struct take_cpu_down_param tcd_param = { .mod = mod, .hcpu = hcpu, }; if (num_online_cpus() == 1) return -EBUSY; if (!cpu_online(cpu)) return -EINVAL; cpu_hotplug_begin(); err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls); if (err) { nr_calls--; __cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL); pr_warn("%s: attempt to take down CPU %u failed\n", __func__, cpu); goto out_release; } /* * By now we've cleared cpu_active_mask, wait for all preempt-disabled * and RCU users of this state to go away such that all new such users * will observe it. * * For CONFIG_PREEMPT we have preemptible RCU and its sync_rcu() might * not imply sync_sched(), so wait for both. * * Do sync before park smpboot threads to take care the rcu boost case. */ if (IS_ENABLED(CONFIG_PREEMPT)) synchronize_rcu_mult(call_rcu, call_rcu_sched); else synchronize_rcu(); smpboot_park_threads(cpu); /* * Prevent irq alloc/free while the dying cpu reorganizes the * interrupt affinities. */ irq_lock_sparse(); /* * So now all preempt/rcu users must observe !cpu_active(). */ err = stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu)); if (err) { /* CPU didn't die: tell everyone. Can't complain. */ cpu_notify_nofail(CPU_DOWN_FAILED | mod, hcpu); irq_unlock_sparse(); goto out_release; } BUG_ON(cpu_online(cpu)); /* * The migration_call() CPU_DYING callback will have removed all * runnable tasks from the cpu, there's only the idle task left now * that the migration thread is done doing the stop_machine thing. * * Wait for the stop thread to go away. */ while (!per_cpu(cpu_dead_idle, cpu)) cpu_relax(); smp_mb(); /* Read from cpu_dead_idle before __cpu_die(). */ per_cpu(cpu_dead_idle, cpu) = false; /* Interrupts are moved away from the dying cpu, reenable alloc/free */ irq_unlock_sparse(); hotplug_cpu__broadcast_tick_pull(cpu); /* This actually kills the CPU. */ __cpu_die(cpu); /* CPU is completely dead: tell everyone. Too late to complain. */ tick_cleanup_dead_cpu(cpu); cpu_notify_nofail(CPU_DEAD | mod, hcpu); check_for_tasks(cpu); out_release: cpu_hotplug_done(); if (!err) cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu); return err; } int cpu_down(unsigned int cpu) { int err; cpu_maps_update_begin(); if (cpu_hotplug_disabled) { err = -EBUSY; goto out; } err = _cpu_down(cpu, 0); out: cpu_maps_update_done(); return err; } EXPORT_SYMBOL(cpu_down); #endif /*CONFIG_HOTPLUG_CPU*/ /* * Unpark per-CPU smpboot kthreads at CPU-online time. */ static int smpboot_thread_call(struct notifier_block *nfb, unsigned long action, void *hcpu) { int cpu = (long)hcpu; switch (action & ~CPU_TASKS_FROZEN) { case CPU_DOWN_FAILED: case CPU_ONLINE: smpboot_unpark_threads(cpu); break; default: break; } return NOTIFY_OK; } static struct notifier_block smpboot_thread_notifier = { .notifier_call = smpboot_thread_call, .priority = CPU_PRI_SMPBOOT, }; void smpboot_thread_init(void) { register_cpu_notifier(&smpboot_thread_notifier); } /* Requires cpu_add_remove_lock to be held */ static int _cpu_up(unsigned int cpu, int tasks_frozen) { int ret, nr_calls = 0; void *hcpu = (void *)(long)cpu; unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0; struct task_struct *idle; cpu_hotplug_begin(); if (cpu_online(cpu) || !cpu_present(cpu)) { ret = -EINVAL; goto out; } idle = idle_thread_get(cpu);