// SPDX-License-Identifier: GPL-2.0-or-later /* *	x86 SMP booting functions * *	(c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk> *	(c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com> *	Copyright 2001 Andi Kleen, SuSE Labs. * *	Much of the core SMP work is based on previous work by Thomas Radke, to *	whom a great many thanks are extended. * *	Thanks to Intel for making available several different Pentium, *	Pentium Pro and Pentium-II/Xeon MP machines. *	Original development of Linux SMP code supported by Caldera. * *	Fixes *		Felix Koop	:	NR_CPUS used properly *		Jose Renau	:	Handle single CPU case. *		Alan Cox	:	By repeated request 8) - Total BogoMIPS report. *		Greg Wright	:	Fix for kernel stacks panic. *		Erich Boleyn	:	MP v1.4 and additional changes. *	Matthias Sattler	:	Changes for 2.1 kernel map. *	Michel Lespinasse	:	Changes for 2.1 kernel map. *	Michael Chastain	:	Change trampoline.S to gnu as. *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine *		Ingo Molnar	:	Added APIC timers, based on code *					from Jose Renau *		Ingo Molnar	:	various cleanups and rewrites *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug. *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs *	Andi Kleen		:	Changed for SMP boot into long mode. *		Martin J. Bligh	: 	Added support for multi-quad systems *		Dave Jones	:	Report invalid combinations of Athlon CPUs. *		Rusty Russell	:	Hacked into shape for new "hotplug" boot process. *      Andi Kleen              :       Converted to new state machine. *	Ashok Raj		: 	CPU hotplug support *	Glauber Costa		:	i386 and x86_64 integration */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/init.h>#include <linux/smp.h>#include <linux/export.h>#include <linux/sched.h>#include <linux/sched/topology.h>#include <linux/sched/hotplug.h>#include <linux/sched/task_stack.h>#include <linux/percpu.h>#include <linux/memblock.h>#include <linux/err.h>#include <linux/nmi.h>#include <linux/tboot.h>#include <linux/gfp.h>#include <linux/cpuidle.h>#include <linux/kexec.h>#include <linux/numa.h>#include <linux/pgtable.h>#include <linux/overflow.h>#include <linux/stackprotector.h>#include <linux/cpuhotplug.h>#include <linux/mc146818rtc.h>#include <linux/acpi.h> #include <asm/acpi.h>#include <asm/cacheinfo.h>#include <asm/cpuid/api.h>#include <asm/desc.h>#include <asm/nmi.h>#include <asm/irq.h>#include <asm/realmode.h>#include <asm/cpu.h>#include <asm/numa.h>#include <asm/tlbflush.h>#include <asm/mtrr.h>#include <asm/mwait.h>#include <asm/apic.h>#include <asm/io_apic.h>#include <asm/fpu/api.h>#include <asm/setup.h>#include <asm/uv/uv.h>#include <asm/microcode.h>#include <asm/i8259.h>#include <asm/misc.h>#include <asm/qspinlock.h>#include <asm/intel-family.h>#include <asm/cpu_device_id.h>#include <asm/spec-ctrl.h>#include <asm/hw_irq.h>#include <asm/stackprotector.h>#include <asm/sev.h>#include <asm/spec-ctrl.h> /* representing HT siblings of each logical CPU */DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);EXPORT_PER_CPU_SYMBOL(cpu_sibling_map); /* representing HT and core siblings of each logical CPU */DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);EXPORT_PER_CPU_SYMBOL(cpu_core_map); /* representing HT, core, and die siblings of each logical CPU */DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_die_map);EXPORT_PER_CPU_SYMBOL(cpu_die_map); /* Representing CPUs for which sibling maps can be computed */static cpumask_var_t cpu_sibling_setup_mask; struct mwait_cpu_dead {	unsigned int	control;	unsigned int	status;}; #define CPUDEAD_MWAIT_WAIT	0xDEADBEEF#define CPUDEAD_MWAIT_KEXEC_HLT	0x4A17DEAD /* * Cache line aligned data for mwait_play_dead(). Separate on purpose so * that it's unlikely to be touched by other CPUs. */static DEFINE_PER_CPU_ALIGNED(struct mwait_cpu_dead, mwait_cpu_dead); /* Maximum number of SMT threads on any online core */int __read_mostly __max_smt_threads = 1; /* Flag to indicate if a complete sched domain rebuild is required */bool x86_topology_update; int arch_update_cpu_topology(void){	int retval = x86_topology_update; 	x86_topology_update = false;	return retval;} static unsigned int smpboot_warm_reset_vector_count; static inline void smpboot_setup_warm_reset_vector(unsigned long start_eip){	unsigned long flags; 	spin_lock_irqsave(&rtc_lock, flags);	if (!smpboot_warm_reset_vector_count++) {		CMOS_WRITE(0xa, 0xf);		*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_HIGH)) = start_eip >> 4;		*((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) = start_eip & 0xf;	}	spin_unlock_irqrestore(&rtc_lock, flags);} static inline void smpboot_restore_warm_reset_vector(void){	unsigned long flags; 	/*	 * Paranoid:  Set warm reset code and vector here back	 * to default values.	 */	spin_lock_irqsave(&rtc_lock, flags);	if (!--smpboot_warm_reset_vector_count) {		CMOS_WRITE(0, 0xf);		*((volatile u32 *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) = 0;	}	spin_unlock_irqrestore(&rtc_lock, flags); } /* Run the next set of setup steps for the upcoming CPU */static void ap_starting(void){	int cpuid = smp_processor_id(); 	/* Mop up eventual mwait_play_dead() wreckage */	this_cpu_write(mwait_cpu_dead.status, 0);	this_cpu_write(mwait_cpu_dead.control, 0); 	/*	 * If woken up by an INIT in an 82489DX configuration the alive	 * synchronization guarantees that the CPU does not reach this	 * point before an INIT_deassert IPI reaches the local APIC, so it	 * is now safe to touch the local APIC.	 *	 * Set up this CPU, first the APIC, which is probably redundant on	 * most boards.	 */	apic_ap_setup(); 	/* Save the processor parameters. */	identify_secondary_cpu(cpuid); 	/*	 * The topology information must be up to date before	 * notify_cpu_starting().	 */	set_cpu_sibling_map(cpuid); 	ap_init_aperfmperf(); 	pr_debug("Stack at about %p\n", &cpuid); 	wmb(); 	/*	 * This runs the AP through all the cpuhp states to its target	 * state CPUHP_ONLINE.	 */	notify_cpu_starting(cpuid);} static void ap_calibrate_delay(void){	/*	 * Calibrate the delay loop and update loops_per_jiffy in cpu_data.	 * identify_secondary_cpu() stored a value that is close but not as	 * accurate as the value just calculated.	 *	 * As this is invoked after the TSC synchronization check,	 * calibrate_delay_is_known() will skip the calibration routine	 * when TSC is synchronized across sockets.	 */	calibrate_delay();	cpu_data(smp_processor_id()).loops_per_jiffy = loops_per_jiffy;} /* * Activate a secondary processor. */static void notrace __noendbr start_secondary(void *unused){	/*	 * Don't put *anything* except direct CPU state initialization	 * before cpu_init(), SMP booting is too fragile that we want to	 * limit the things done here to the most necessary things.	 */	cr4_init(); 	/*	 * 32-bit specific. 64-bit reaches this code with the correct page	 * table established. Yet another historical divergence.	 */	if (IS_ENABLED(CONFIG_X86_32)) {		/* switch away from the initial page table */		load_cr3(swapper_pg_dir);		__flush_tlb_all();	} 	cpu_init_exception_handling(false); 	/*	 * Load the microcode before reaching the AP alive synchronization	 * point below so it is not part of the full per CPU serialized	 * bringup part when "parallel" bringup is enabled.	 *	 * That's even safe when hyperthreading is enabled in the CPU as	 * the core code starts the primary threads first and leaves the	 * secondary threads waiting for SIPI. Loading microcode on	 * physical cores concurrently is a safe operation.	 *	 * This covers both the Intel specific issue that concurrent	 * microcode loading on SMT siblings must be prohibited and the	 * vendor independent issue`that microcode loading which changes	 * CPUID, MSRs etc. must be strictly serialized to maintain	 * software state correctness.	 */	load_ucode_ap(); 	/*	 * Synchronization point with the hotplug core. Sets this CPUs	 * synchronization state to ALIVE and spin-waits for the control CPU to	 * release this CPU for further bringup.	 */	cpuhp_ap_sync_alive(); 	cpu_init();	fpu__init_cpu();	rcutree_report_cpu_starting(raw_smp_processor_id());	x86_cpuinit.early_percpu_clock_init(); 	ap_starting(); 	/* Check TSC synchronization with the control CPU. */	check_tsc_sync_target(); 	/*	 * Calibrate the delay loop after the TSC synchronization check.	 * This allows to skip the calibration when TSC is synchronized	 * across sockets.	 */	ap_calibrate_delay(); 	speculative_store_bypass_ht_init(); 	/*	 * Lock vector_lock, set CPU online and bring the vector	 * allocator online. Online must be set with vector_lock held	 * to prevent a concurrent irq setup/teardown from seeing a	 * half valid vector space.	 */	lock_vector_lock();	set_cpu_online(smp_processor_id(), true);	lapic_online();	unlock_vector_lock();	x86_platform.nmi_init(); 	/* enable local interrupts */	local_irq_enable(); 	x86_cpuinit.setup_percpu_clockev(); 	wmb();	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);}ANNOTATE_NOENDBR_SYM(start_secondary); static booltopology_same_node(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o){	int cpu1 = c->cpu_index, cpu2 = o->cpu_index; 	return (cpu_to_node(cpu1) == cpu_to_node(cpu2));} static booltopology_sane(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o, const char *name){	int cpu1 = c->cpu_index, cpu2 = o->cpu_index; 	return !WARN_ONCE(!topology_same_node(c, o),		"sched: CPU #%d's %s-sibling CPU #%d is not on the same node! "		"[node: %d != %d]. Ignoring dependency.\n",		cpu1, name, cpu2, cpu_to_node(cpu1), cpu_to_node(cpu2));} #define link_mask(mfunc, c1, c2)					\do {									\	cpumask_set_cpu((c1), mfunc(c2));				\	cpumask_set_cpu((c2), mfunc(c1));				\} while (0) static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o){	if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {		int cpu1 = c->cpu_index, cpu2 = o->cpu_index; 		if (c->topo.pkg_id == o->topo.pkg_id &&		    c->topo.die_id == o->topo.die_id &&		    c->topo.amd_node_id == o->topo.amd_node_id &&		    per_cpu_llc_id(cpu1) == per_cpu_llc_id(cpu2)) {			if (c->topo.core_id == o->topo.core_id)				return topology_sane(c, o, "smt"); 			if ((c->topo.cu_id != 0xff) &&			    (o->topo.cu_id != 0xff) &&			    (c->topo.cu_id == o->topo.cu_id))				return topology_sane(c, o, "smt");		} 	} else if (c->topo.pkg_id == o->topo.pkg_id &&		   c->topo.die_id == o->topo.die_id &&		   c->topo.core_id == o->topo.core_id) {		return topology_sane(c, o, "smt");	} 	return false;} static bool match_die(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o){	if (c->topo.pkg_id != o->topo.pkg_id || c->topo.die_id != o->topo.die_id)		return false; 	if (cpu_feature_enabled(X86_FEATURE_TOPOEXT) && topology_amd_nodes_per_pkg() > 1)		return c->topo.amd_node_id == o->topo.amd_node_id; 	return true;} static bool match_l2c(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o){	int cpu1 = c->cpu_index, cpu2 = o->cpu_index; 	/* If the arch didn't set up l2c_id, fall back to SMT */	if (per_cpu_l2c_id(cpu1) == BAD_APICID)		return match_smt(c, o); 	/* Do not match if L2 cache id does not match: */	if (per_cpu_l2c_id(cpu1) != per_cpu_l2c_id(cpu2))		return false; 	return topology_sane(c, o, "l2c");} /* * Unlike the other levels, we do not enforce keeping a * multicore group inside a NUMA node.  If this happens, we will * discard the MC level of the topology later. */static bool match_pkg(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o){	if (c->topo.pkg_id == o->topo.pkg_id)		return true;	return false;} /* * Define intel_cod_cpu[] for Intel COD (Cluster-on-Die) CPUs. * * Any Intel CPU that has multiple nodes per package and does not * match intel_cod_cpu[] has the SNC (Sub-NUMA Cluster) topology. * * When in SNC mode, these CPUs enumerate an LLC that is shared * by multiple NUMA nodes. The LLC is shared for off-package data * access but private to the NUMA node (half of the package) for * on-package access. CPUID (the source of the information about * the LLC) can only enumerate the cache as shared or unshared, * but not this particular configuration. */ static const struct x86_cpu_id intel_cod_cpu[] = {	X86_MATCH_VFM(INTEL_HASWELL_X,	 0),	/* COD */	X86_MATCH_VFM(INTEL_BROADWELL_X, 0),	/* COD */	X86_MATCH_VFM(INTEL_ANY,	 1),	/* SNC */	{}}; /* * Allows splitting the LLC by matching 'core_id % split_llc'. * * This is mostly a debug hack to emulate systems with multiple LLCs per node * on systems that do not naturally have this. */static unsigned int split_llc = 0; static int __init split_llc_setup(char *str){	get_option(&str, &split_llc);	return 0;}early_param("split_llc", split_llc_setup); static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o){	const struct x86_cpu_id *id = x86_match_cpu(intel_cod_cpu);	int cpu1 = c->cpu_index, cpu2 = o->cpu_index;	bool intel_snc = id && id->driver_data; 	/* Do not match if we do not have a valid APICID for cpu: */	if (per_cpu_llc_id(cpu1) == BAD_APICID)		return false; 	/* Do not match if LLC id does not match: */	if (per_cpu_llc_id(cpu1) != per_cpu_llc_id(cpu2))		return false; 	if (split_llc &&	    (per_cpu_core_id(cpu1) % split_llc) !=	    (per_cpu_core_id(cpu2) % split_llc))		return false; 	/*	 * Allow the SNC topology without warning. Return of false	 * means 'c' does not share the LLC of 'o'. This will be	 * reflected to userspace.	 */	if (match_pkg(c, o) && !topology_same_node(c, o) && intel_snc)		return false; 	return topology_sane(c, o, "llc");}  static inline int x86_sched_itmt_flags(void){	return sysctl_sched_itmt_enabled ? SD_ASYM_PACKING : 0;} #ifdef CONFIG_SCHED_MCstatic int x86_core_flags(void){	return cpu_core_flags() | x86_sched_itmt_flags();}#endif#ifdef CONFIG_SCHED_CLUSTERstatic int x86_cluster_flags(void){	return cpu_cluster_flags() | x86_sched_itmt_flags();}#endif static struct sched_domain_topology_level x86_topology[] = {	SDTL_INIT(tl_smt_mask, cpu_smt_flags, SMT),#ifdef CONFIG_SCHED_CLUSTER	SDTL_INIT(tl_cls_mask, x86_cluster_flags, CLS),#endif#ifdef CONFIG_SCHED_MC	SDTL_INIT(tl_mc_mask, x86_core_flags, MC),#endif	SDTL_INIT(tl_pkg_mask, x86_sched_itmt_flags, PKG),	{ NULL },

Lines 1–500 of 1528. Load more