/* CPU control. * (C) 2001, 2002, 2003, 2004 Rusty Russell * * This code is licenced under the GPL. */#include <linux/proc_fs.h>#include <linux/smp.h>#include <linux/init.h>#include <linux/notifier.h>#include <linux/sched/signal.h>#include <linux/sched/hotplug.h>#include <linux/sched/task.h>#include <linux/unistd.h>#include <linux/cpu.h>#include <linux/oom.h>#include <linux/rcupdate.h>#include <linux/export.h>#include <linux/bug.h>#include <linux/kthread.h>#include <linux/stop_machine.h>#include <linux/mutex.h>#include <linux/gfp.h>#include <linux/suspend.h>#include <linux/lockdep.h>#include <linux/tick.h>#include <linux/irq.h>#include <linux/nmi.h>#include <linux/smpboot.h>#include <linux/relay.h>#include <linux/slab.h>#include <linux/percpu-rwsem.h> #include <trace/events/power.h>#define CREATE_TRACE_POINTS#include <trace/events/cpuhp.h> #include "smpboot.h" /** * cpuhp_cpu_state - Per cpu hotplug state storage * @state: The current cpu state * @target: The target state * @thread: Pointer to the hotplug thread * @should_run: Thread should execute * @rollback: Perform a rollback * @single: Single callback invocation * @bringup: Single callback bringup or teardown selector * @cb_state: The state for a single callback (install/uninstall) * @result: Result of the operation * @done_up: Signal completion to the issuer of the task for cpu-up * @done_down: Signal completion to the issuer of the task for cpu-down */struct cpuhp_cpu_state { enum cpuhp_state state; enum cpuhp_state target; enum cpuhp_state fail;#ifdef CONFIG_SMP struct task_struct *thread; bool should_run; bool rollback; bool single; bool bringup; bool booted_once; struct hlist_node *node; struct hlist_node *last; enum cpuhp_state cb_state; int result; struct completion done_up; struct completion done_down;#endif}; static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = { .fail = CPUHP_INVALID,}; #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)static struct lockdep_map cpuhp_state_up_map = STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);static struct lockdep_map cpuhp_state_down_map = STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map); static inline void cpuhp_lock_acquire(bool bringup){ lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);} static inline void cpuhp_lock_release(bool bringup){ lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);}#else static inline void cpuhp_lock_acquire(bool bringup) { }static inline void cpuhp_lock_release(bool bringup) { } #endif /** * cpuhp_step - Hotplug state machine step * @name: Name of the step * @startup: Startup function of the step * @teardown: Teardown function of the step * @skip_onerr: Do not invoke the functions on error rollback * Will go away once the notifiers are gone * @cant_stop: Bringup/teardown can't be stopped at this step */struct cpuhp_step { const char *name; union { int (*single)(unsigned int cpu); int (*multi)(unsigned int cpu, struct hlist_node *node); } startup; union { int (*single)(unsigned int cpu); int (*multi)(unsigned int cpu, struct hlist_node *node); } teardown; struct hlist_head list; bool skip_onerr; bool cant_stop; bool multi_instance;}; static DEFINE_MUTEX(cpuhp_state_mutex);static struct cpuhp_step cpuhp_hp_states[]; static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state){ return cpuhp_hp_states + state;} /** * cpuhp_invoke_callback _ Invoke the callbacks for a given state * @cpu: The cpu for which the callback should be invoked * @state: The state to do callbacks for * @bringup: True if the bringup callback should be invoked * @node: For multi-instance, do a single entry callback for install/remove * @lastp: For multi-instance rollback, remember how far we got * * Called from cpu hotplug and from the state register machinery. */static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state, bool bringup, struct hlist_node *node, struct hlist_node **lastp){ struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); struct cpuhp_step *step = cpuhp_get_step(state); int (*cbm)(unsigned int cpu, struct hlist_node *node); int (*cb)(unsigned int cpu); int ret, cnt; if (st->fail == state) { st->fail = CPUHP_INVALID; if (!(bringup ? step->startup.single : step->teardown.single)) return 0; return -EAGAIN; } if (!step->multi_instance) { WARN_ON_ONCE(lastp && *lastp); cb = bringup ? step->startup.single : step->teardown.single; if (!cb) return 0; trace_cpuhp_enter(cpu, st->target, state, cb); ret = cb(cpu); trace_cpuhp_exit(cpu, st->state, state, ret); return ret; } cbm = bringup ? step->startup.multi : step->teardown.multi; if (!cbm) return 0; /* Single invocation for instance add/remove */ if (node) { WARN_ON_ONCE(lastp && *lastp); trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); ret = cbm(cpu, node); trace_cpuhp_exit(cpu, st->state, state, ret); return ret; } /* State transition. Invoke on all instances */ cnt = 0; hlist_for_each(node, &step->list) { if (lastp && node == *lastp) break; trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); ret = cbm(cpu, node); trace_cpuhp_exit(cpu, st->state, state, ret); if (ret) { if (!lastp) goto err; *lastp = node; return ret; } cnt++; } if (lastp) *lastp = NULL; return 0;err: /* Rollback the instances if one failed */ cbm = !bringup ? step->startup.multi : step->teardown.multi; if (!cbm) return ret; hlist_for_each(node, &step->list) { if (!cnt--) break; trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); ret = cbm(cpu, node); trace_cpuhp_exit(cpu, st->state, state, ret); /* * Rollback must not fail, */ WARN_ON_ONCE(ret); } return ret;} #ifdef CONFIG_SMPstatic bool cpuhp_is_ap_state(enum cpuhp_state state){ /* * The extra check for CPUHP_TEARDOWN_CPU is only for documentation * purposes as that state is handled explicitly in cpu_down. */ return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;} static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup){ struct completion *done = bringup ? &st->done_up : &st->done_down; wait_for_completion(done);} static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup){ struct completion *done = bringup ? &st->done_up : &st->done_down; complete(done);} /* * The former STARTING/DYING states, ran with IRQs disabled and must not fail. */static bool cpuhp_is_atomic_state(enum cpuhp_state state){ return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;} /* Serializes the updates to cpu_online_mask, cpu_present_mask */static DEFINE_MUTEX(cpu_add_remove_lock);bool cpuhp_tasks_frozen;EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen); /* * The following two APIs (cpu_maps_update_begin/done) must be used when * attempting to serialize the updates to cpu_online_mask & cpu_present_mask. */void cpu_maps_update_begin(void){ mutex_lock(&cpu_add_remove_lock);} void cpu_maps_update_done(void){ mutex_unlock(&cpu_add_remove_lock);} /* * If set, cpu_up and cpu_down will return -EBUSY and do nothing. * Should always be manipulated under cpu_add_remove_lock */static int cpu_hotplug_disabled; #ifdef CONFIG_HOTPLUG_CPU DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock); void cpus_read_lock(void){ percpu_down_read(&cpu_hotplug_lock);}EXPORT_SYMBOL_GPL(cpus_read_lock); int cpus_read_trylock(void){ return percpu_down_read_trylock(&cpu_hotplug_lock);}EXPORT_SYMBOL_GPL(cpus_read_trylock); void cpus_read_unlock(void){ percpu_up_read(&cpu_hotplug_lock);}EXPORT_SYMBOL_GPL(cpus_read_unlock); void cpus_write_lock(void){ percpu_down_write(&cpu_hotplug_lock);} void cpus_write_unlock(void){ percpu_up_write(&cpu_hotplug_lock);} void lockdep_assert_cpus_held(void){ percpu_rwsem_assert_held(&cpu_hotplug_lock);} /* * Wait for currently running CPU hotplug operations to complete (if any) and * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the * hotplug path before performing hotplug operations. So acquiring that lock * guarantees mutual exclusion from any currently running hotplug operations. */void cpu_hotplug_disable(void){ cpu_maps_update_begin(); cpu_hotplug_disabled++; cpu_maps_update_done();}EXPORT_SYMBOL_GPL(cpu_hotplug_disable); static void __cpu_hotplug_enable(void){ if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n")) return; cpu_hotplug_disabled--;} void cpu_hotplug_enable(void){ cpu_maps_update_begin(); __cpu_hotplug_enable(); cpu_maps_update_done();}EXPORT_SYMBOL_GPL(cpu_hotplug_enable);#endif /* CONFIG_HOTPLUG_CPU */ #ifdef CONFIG_HOTPLUG_SMTenum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;EXPORT_SYMBOL_GPL(cpu_smt_control); static bool cpu_smt_available __read_mostly; void __init cpu_smt_disable(bool force){ if (cpu_smt_control == CPU_SMT_FORCE_DISABLED || cpu_smt_control == CPU_SMT_NOT_SUPPORTED) return; if (force) { pr_info("SMT: Force disabled\n"); cpu_smt_control = CPU_SMT_FORCE_DISABLED; } else { cpu_smt_control = CPU_SMT_DISABLED; }} /* * The decision whether SMT is supported can only be done after the full * CPU identification. Called from architecture code before non boot CPUs * are brought up. */void __init cpu_smt_check_topology_early(void){ if (!topology_smt_supported()) cpu_smt_control = CPU_SMT_NOT_SUPPORTED;} /* * If SMT was disabled by BIOS, detect it here, after the CPUs have been * brought online. This ensures the smt/l1tf sysfs entries are consistent * with reality. cpu_smt_available is set to true during the bringup of non * boot CPUs when a SMT sibling is detected. Note, this may overwrite * cpu_smt_control's previous setting. */void __init cpu_smt_check_topology(void){ if (!cpu_smt_available) cpu_smt_control = CPU_SMT_NOT_SUPPORTED;} static int __init smt_cmdline_disable(char *str){ cpu_smt_disable(str && !strcmp(str, "force")); return 0;}early_param("nosmt", smt_cmdline_disable); static inline bool cpu_smt_allowed(unsigned int cpu){ if (topology_is_primary_thread(cpu)) return true; /* * If the CPU is not a 'primary' thread and the booted_once bit is * set then the processor has SMT support. Store this information * for the late check of SMT support in cpu_smt_check_topology(). */ if (per_cpu(cpuhp_state, cpu).booted_once) cpu_smt_available = true; if (cpu_smt_control == CPU_SMT_ENABLED) return true; /* * On x86 it's required to boot all logical CPUs at least once so * that the init code can get a chance to set CR4.MCE on each * CPU. Otherwise, a broadacasted MCE observing CR4.MCE=0b on any * core will shutdown the machine. */ return !per_cpu(cpuhp_state, cpu).booted_once;}#elsestatic inline bool cpu_smt_allowed(unsigned int cpu) { return true; }#endif static inline enum cpuhp_statecpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target){ enum cpuhp_state prev_state = st->state; st->rollback = false; st->last = NULL; st->target = target; st->single = false; st->bringup = st->state < target; return prev_state;} static inline voidcpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state){ st->rollback = true; /* * If we have st->last we need to undo partial multi_instance of this * state first. Otherwise start undo at the previous state. */ if (!st->last) { if (st->bringup) st->state--; else st->state++; } st->target = prev_state; st->bringup = !st->bringup;} /* Regular hotplug invocation of the AP hotplug thread */static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st){ if (!st->single && st->state == st->target) return; st->result = 0; /* * Make sure the above stores are visible before should_run becomes * true. Paired with the mb() above in cpuhp_thread_fun() */ smp_mb(); st->should_run = true; wake_up_process(st->thread); wait_for_ap_thread(st, st->bringup);} static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target){ enum cpuhp_state prev_state; int ret; prev_state = cpuhp_set_state(st, target); __cpuhp_kick_ap(st); if ((ret = st->result)) { cpuhp_reset_state(st, prev_state); __cpuhp_kick_ap(st); } return ret;} static int bringup_wait_for_ap(unsigned int cpu){ struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);Lines 1–500 of 2285. Load more
/* CPU control. * (C) 2001, 2002, 2003, 2004 Rusty Russell * * This code is licenced under the GPL. */ #include <linux/proc_fs.h> #include <linux/smp.h> #include <linux/init.h> #include <linux/notifier.h> #include <linux/sched/signal.h> #include <linux/sched/hotplug.h> #include <linux/sched/task.h> #include <linux/unistd.h> #include <linux/cpu.h> #include <linux/oom.h> #include <linux/rcupdate.h> #include <linux/export.h> #include <linux/bug.h> #include <linux/kthread.h> #include <linux/stop_machine.h> #include <linux/mutex.h> #include <linux/gfp.h> #include <linux/suspend.h> #include <linux/lockdep.h> #include <linux/tick.h> #include <linux/irq.h> #include <linux/nmi.h> #include <linux/smpboot.h> #include <linux/relay.h> #include <linux/slab.h> #include <linux/percpu-rwsem.h> #include <trace/events/power.h> #define CREATE_TRACE_POINTS #include <trace/events/cpuhp.h> #include "smpboot.h" /** * cpuhp_cpu_state - Per cpu hotplug state storage * @state: The current cpu state * @target: The target state * @thread: Pointer to the hotplug thread * @should_run: Thread should execute * @rollback: Perform a rollback * @single: Single callback invocation * @bringup: Single callback bringup or teardown selector * @cb_state: The state for a single callback (install/uninstall) * @result: Result of the operation * @done_up: Signal completion to the issuer of the task for cpu-up * @done_down: Signal completion to the issuer of the task for cpu-down */ struct cpuhp_cpu_state { enum cpuhp_state state; enum cpuhp_state target; enum cpuhp_state fail; #ifdef CONFIG_SMP struct task_struct *thread; bool should_run; bool rollback; bool single; bool bringup; bool booted_once; struct hlist_node *node; struct hlist_node *last; enum cpuhp_state cb_state; int result; struct completion done_up; struct completion done_down; #endif }; static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = { .fail = CPUHP_INVALID, }; #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP) static struct lockdep_map cpuhp_state_up_map = STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map); static struct lockdep_map cpuhp_state_down_map = STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map); static inline void cpuhp_lock_acquire(bool bringup) { lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map); } static inline void cpuhp_lock_release(bool bringup) { lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map); } #else static inline void cpuhp_lock_acquire(bool bringup) { } static inline void cpuhp_lock_release(bool bringup) { } #endif /** * cpuhp_step - Hotplug state machine step * @name: Name of the step * @startup: Startup function of the step * @teardown: Teardown function of the step * @skip_onerr: Do not invoke the functions on error rollback * Will go away once the notifiers are gone * @cant_stop: Bringup/teardown can't be stopped at this step */ struct cpuhp_step { const char *name; union { int (*single)(unsigned int cpu); int (*multi)(unsigned int cpu, struct hlist_node *node); } startup; union { int (*single)(unsigned int cpu); int (*multi)(unsigned int cpu, struct hlist_node *node); } teardown; struct hlist_head list; bool skip_onerr; bool cant_stop; bool multi_instance; }; static DEFINE_MUTEX(cpuhp_state_mutex); static struct cpuhp_step cpuhp_hp_states[]; static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state) { return cpuhp_hp_states + state; } /** * cpuhp_invoke_callback _ Invoke the callbacks for a given state * @cpu: The cpu for which the callback should be invoked * @state: The state to do callbacks for * @bringup: True if the bringup callback should be invoked * @node: For multi-instance, do a single entry callback for install/remove * @lastp: For multi-instance rollback, remember how far we got * * Called from cpu hotplug and from the state register machinery. */ static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state, bool bringup, struct hlist_node *node, struct hlist_node **lastp) { struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); struct cpuhp_step *step = cpuhp_get_step(state); int (*cbm)(unsigned int cpu, struct hlist_node *node); int (*cb)(unsigned int cpu); int ret, cnt; if (st->fail == state) { st->fail = CPUHP_INVALID; if (!(bringup ? step->startup.single : step->teardown.single)) return 0; return -EAGAIN; } if (!step->multi_instance) { WARN_ON_ONCE(lastp && *lastp); cb = bringup ? step->startup.single : step->teardown.single; if (!cb) return 0; trace_cpuhp_enter(cpu, st->target, state, cb); ret = cb(cpu); trace_cpuhp_exit(cpu, st->state, state, ret); return ret; } cbm = bringup ? step->startup.multi : step->teardown.multi; if (!cbm) return 0; /* Single invocation for instance add/remove */ if (node) { WARN_ON_ONCE(lastp && *lastp); trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); ret = cbm(cpu, node); trace_cpuhp_exit(cpu, st->state, state, ret); return ret; } /* State transition. Invoke on all instances */ cnt = 0; hlist_for_each(node, &step->list) { if (lastp && node == *lastp) break; trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); ret = cbm(cpu, node); trace_cpuhp_exit(cpu, st->state, state, ret); if (ret) { if (!lastp) goto err; *lastp = node; return ret; } cnt++; } if (lastp) *lastp = NULL; return 0; err: /* Rollback the instances if one failed */ cbm = !bringup ? step->startup.multi : step->teardown.multi; if (!cbm) return ret; hlist_for_each(node, &step->list) { if (!cnt--) break; trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); ret = cbm(cpu, node); trace_cpuhp_exit(cpu, st->state, state, ret); /* * Rollback must not fail, */ WARN_ON_ONCE(ret); } return ret; } #ifdef CONFIG_SMP static bool cpuhp_is_ap_state(enum cpuhp_state state) { /* * The extra check for CPUHP_TEARDOWN_CPU is only for documentation * purposes as that state is handled explicitly in cpu_down. */ return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU; } static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup) { struct completion *done = bringup ? &st->done_up : &st->done_down; wait_for_completion(done); } static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup) { struct completion *done = bringup ? &st->done_up : &st->done_down; complete(done); } /* * The former STARTING/DYING states, ran with IRQs disabled and must not fail. */ static bool cpuhp_is_atomic_state(enum cpuhp_state state) { return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE; } /* Serializes the updates to cpu_online_mask, cpu_present_mask */ static DEFINE_MUTEX(cpu_add_remove_lock); bool cpuhp_tasks_frozen; EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen); /* * The following two APIs (cpu_maps_update_begin/done) must be used when * attempting to serialize the updates to cpu_online_mask & cpu_present_mask. */ void cpu_maps_update_begin(void) { mutex_lock(&cpu_add_remove_lock); } void cpu_maps_update_done(void) { mutex_unlock(&cpu_add_remove_lock); } /* * If set, cpu_up and cpu_down will return -EBUSY and do nothing. * Should always be manipulated under cpu_add_remove_lock */ static int cpu_hotplug_disabled; #ifdef CONFIG_HOTPLUG_CPU DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock); void cpus_read_lock(void) { percpu_down_read(&cpu_hotplug_lock); } EXPORT_SYMBOL_GPL(cpus_read_lock); int cpus_read_trylock(void) { return percpu_down_read_trylock(&cpu_hotplug_lock); } EXPORT_SYMBOL_GPL(cpus_read_trylock); void cpus_read_unlock(void) { percpu_up_read(&cpu_hotplug_lock); } EXPORT_SYMBOL_GPL(cpus_read_unlock); void cpus_write_lock(void) { percpu_down_write(&cpu_hotplug_lock); } void cpus_write_unlock(void) { percpu_up_write(&cpu_hotplug_lock); } void lockdep_assert_cpus_held(void) { percpu_rwsem_assert_held(&cpu_hotplug_lock); } /* * Wait for currently running CPU hotplug operations to complete (if any) and * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the * hotplug path before performing hotplug operations. So acquiring that lock * guarantees mutual exclusion from any currently running hotplug operations. */ void cpu_hotplug_disable(void) { cpu_maps_update_begin(); cpu_hotplug_disabled++; cpu_maps_update_done(); } EXPORT_SYMBOL_GPL(cpu_hotplug_disable); static void __cpu_hotplug_enable(void) { if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n")) return; cpu_hotplug_disabled--; } void cpu_hotplug_enable(void) { cpu_maps_update_begin(); __cpu_hotplug_enable(); cpu_maps_update_done(); } EXPORT_SYMBOL_GPL(cpu_hotplug_enable); #endif /* CONFIG_HOTPLUG_CPU */ #ifdef CONFIG_HOTPLUG_SMT enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED; EXPORT_SYMBOL_GPL(cpu_smt_control); static bool cpu_smt_available __read_mostly; void __init cpu_smt_disable(bool force) { if (cpu_smt_control == CPU_SMT_FORCE_DISABLED || cpu_smt_control == CPU_SMT_NOT_SUPPORTED) return; if (force) { pr_info("SMT: Force disabled\n"); cpu_smt_control = CPU_SMT_FORCE_DISABLED; } else { cpu_smt_control = CPU_SMT_DISABLED; } } /* * The decision whether SMT is supported can only be done after the full * CPU identification. Called from architecture code before non boot CPUs * are brought up. */ void __init cpu_smt_check_topology_early(void) { if (!topology_smt_supported()) cpu_smt_control = CPU_SMT_NOT_SUPPORTED; } /* * If SMT was disabled by BIOS, detect it here, after the CPUs have been * brought online. This ensures the smt/l1tf sysfs entries are consistent * with reality. cpu_smt_available is set to true during the bringup of non * boot CPUs when a SMT sibling is detected. Note, this may overwrite * cpu_smt_control's previous setting. */ void __init cpu_smt_check_topology(void) { if (!cpu_smt_available) cpu_smt_control = CPU_SMT_NOT_SUPPORTED; } static int __init smt_cmdline_disable(char *str) { cpu_smt_disable(str && !strcmp(str, "force")); return 0; } early_param("nosmt", smt_cmdline_disable); static inline bool cpu_smt_allowed(unsigned int cpu) { if (topology_is_primary_thread(cpu)) return true; /* * If the CPU is not a 'primary' thread and the booted_once bit is * set then the processor has SMT support. Store this information * for the late check of SMT support in cpu_smt_check_topology(). */ if (per_cpu(cpuhp_state, cpu).booted_once) cpu_smt_available = true; if (cpu_smt_control == CPU_SMT_ENABLED) return true; /* * On x86 it's required to boot all logical CPUs at least once so * that the init code can get a chance to set CR4.MCE on each * CPU. Otherwise, a broadacasted MCE observing CR4.MCE=0b on any * core will shutdown the machine. */ return !per_cpu(cpuhp_state, cpu).booted_once; } #else static inline bool cpu_smt_allowed(unsigned int cpu) { return true; } #endif static inline enum cpuhp_state cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target) { enum cpuhp_state prev_state = st->state; st->rollback = false; st->last = NULL; st->target = target; st->single = false; st->bringup = st->state < target; return prev_state; } static inline void cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state) { st->rollback = true; /* * If we have st->last we need to undo partial multi_instance of this * state first. Otherwise start undo at the previous state. */ if (!st->last) { if (st->bringup) st->state--; else st->state++; } st->target = prev_state; st->bringup = !st->bringup; } /* Regular hotplug invocation of the AP hotplug thread */ static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st) { if (!st->single && st->state == st->target) return; st->result = 0; /* * Make sure the above stores are visible before should_run becomes * true. Paired with the mb() above in cpuhp_thread_fun() */ smp_mb(); st->should_run = true; wake_up_process(st->thread); wait_for_ap_thread(st, st->bringup); } static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target) { enum cpuhp_state prev_state; int ret; prev_state = cpuhp_set_state(st, target); __cpuhp_kick_ap(st); if ((ret = st->result)) { cpuhp_reset_state(st, prev_state); __cpuhp_kick_ap(st); } return ret; } static int bringup_wait_for_ap(unsigned int cpu) { struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);