// SPDX-License-Identifier: GPL-2.0-only/* cpu_feature_enabled() cannot be used this early */#define USE_EARLY_PGTABLE_L5 #include <linux/memblock.h>#include <linux/linkage.h>#include <linux/bitops.h>#include <linux/kernel.h>#include <linux/export.h>#include <linux/kvm_types.h>#include <linux/percpu.h>#include <linux/string.h>#include <linux/ctype.h>#include <linux/delay.h>#include <linux/sched/mm.h>#include <linux/sched/clock.h>#include <linux/sched/task.h>#include <linux/sched/smt.h>#include <linux/init.h>#include <linux/kprobes.h>#include <linux/kgdb.h>#include <linux/mem_encrypt.h>#include <linux/smp.h>#include <linux/cpu.h>#include <linux/io.h>#include <linux/syscore_ops.h>#include <linux/pgtable.h>#include <linux/stackprotector.h>#include <linux/utsname.h>#include <linux/efi.h> #include <asm/alternative.h>#include <asm/cmdline.h>#include <asm/cpuid/api.h>#include <asm/perf_event.h>#include <asm/mmu_context.h>#include <asm/doublefault.h>#include <asm/archrandom.h>#include <asm/hypervisor.h>#include <asm/processor.h>#include <asm/tlbflush.h>#include <asm/debugreg.h>#include <asm/sections.h>#include <asm/vsyscall.h>#include <linux/topology.h>#include <linux/cpumask.h>#include <linux/atomic.h>#include <asm/proto.h>#include <asm/setup.h>#include <asm/apic.h>#include <asm/desc.h>#include <asm/fpu/api.h>#include <asm/mtrr.h>#include <asm/hwcap2.h>#include <linux/numa.h>#include <asm/numa.h>#include <asm/asm.h>#include <asm/bugs.h>#include <asm/cpu.h>#include <asm/mce.h>#include <asm/msr.h>#include <asm/cacheinfo.h>#include <asm/memtype.h>#include <asm/microcode.h>#include <asm/intel-family.h>#include <asm/cpu_device_id.h>#include <asm/fred.h>#include <asm/uv/uv.h>#include <asm/ia32.h>#include <asm/set_memory.h>#include <asm/traps.h>#include <asm/sev.h>#include <asm/tdx.h>#include <asm/virt.h>#include <asm/posted_intr.h>#include <asm/runtime-const.h> #include "cpu.h" DEFINE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);EXPORT_PER_CPU_SYMBOL(cpu_info); /* Used for modules: built-in code uses runtime constants */unsigned long USER_PTR_MAX;EXPORT_SYMBOL(USER_PTR_MAX); u32 elf_hwcap2 __read_mostly; /* Number of siblings per CPU package */unsigned int __max_threads_per_core __ro_after_init = 1;EXPORT_SYMBOL(__max_threads_per_core); unsigned int __max_dies_per_package __ro_after_init = 1;EXPORT_SYMBOL(__max_dies_per_package); unsigned int __max_logical_packages __ro_after_init = 1;EXPORT_SYMBOL(__max_logical_packages); unsigned int __num_nodes_per_package __ro_after_init = 1;EXPORT_SYMBOL(__num_nodes_per_package); unsigned int __num_cores_per_package __ro_after_init = 1;EXPORT_SYMBOL(__num_cores_per_package); unsigned int __num_threads_per_package __ro_after_init = 1;EXPORT_SYMBOL(__num_threads_per_package); static struct ppin_info {	int	feature;	int	msr_ppin_ctl;	int	msr_ppin;} ppin_info[] = {	[X86_VENDOR_INTEL] = {		.feature = X86_FEATURE_INTEL_PPIN,		.msr_ppin_ctl = MSR_PPIN_CTL,		.msr_ppin = MSR_PPIN	},	[X86_VENDOR_AMD] = {		.feature = X86_FEATURE_AMD_PPIN,		.msr_ppin_ctl = MSR_AMD_PPIN_CTL,		.msr_ppin = MSR_AMD_PPIN	},}; static const struct x86_cpu_id ppin_cpuids[] = {	X86_MATCH_FEATURE(X86_FEATURE_AMD_PPIN, &ppin_info[X86_VENDOR_AMD]),	X86_MATCH_FEATURE(X86_FEATURE_INTEL_PPIN, &ppin_info[X86_VENDOR_INTEL]), 	/* Legacy models without CPUID enumeration */	X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &ppin_info[X86_VENDOR_INTEL]),	X86_MATCH_VFM(INTEL_HASWELL_X, &ppin_info[X86_VENDOR_INTEL]),	X86_MATCH_VFM(INTEL_BROADWELL_D, &ppin_info[X86_VENDOR_INTEL]),	X86_MATCH_VFM(INTEL_BROADWELL_X, &ppin_info[X86_VENDOR_INTEL]),	X86_MATCH_VFM(INTEL_SKYLAKE_X, &ppin_info[X86_VENDOR_INTEL]),	X86_MATCH_VFM(INTEL_ICELAKE_X, &ppin_info[X86_VENDOR_INTEL]),	X86_MATCH_VFM(INTEL_ICELAKE_D, &ppin_info[X86_VENDOR_INTEL]),	X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),	X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &ppin_info[X86_VENDOR_INTEL]),	X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &ppin_info[X86_VENDOR_INTEL]),	X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &ppin_info[X86_VENDOR_INTEL]), 	{}}; static void ppin_init(struct cpuinfo_x86 *c){	const struct x86_cpu_id *id;	unsigned long long val;	struct ppin_info *info; 	id = x86_match_cpu(ppin_cpuids);	if (!id)		return; 	/*	 * Testing the presence of the MSR is not enough. Need to check	 * that the PPIN_CTL allows reading of the PPIN.	 */	info = (struct ppin_info *)id->driver_data; 	if (rdmsrq_safe(info->msr_ppin_ctl, &val))		goto clear_ppin; 	if ((val & 3UL) == 1UL) {		/* PPIN locked in disabled mode */		goto clear_ppin;	} 	/* If PPIN is disabled, try to enable */	if (!(val & 2UL)) {		wrmsrq_safe(info->msr_ppin_ctl,  val | 2UL);		rdmsrq_safe(info->msr_ppin_ctl, &val);	} 	/* Is the enable bit set? */	if (val & 2UL) {		c->ppin = native_rdmsrq(info->msr_ppin);		set_cpu_cap(c, info->feature);		return;	} clear_ppin:	setup_clear_cpu_cap(info->feature);} static void default_init(struct cpuinfo_x86 *c){#ifdef CONFIG_X86_64	cpu_detect_cache_sizes(c);#else	/* Not much we can do here... */	/* Check if at least it has cpuid */	if (c->cpuid_level == -1) {		/* No cpuid. It must be an ancient CPU */		if (c->x86 == 4)			strcpy(c->x86_model_id, "486");		else if (c->x86 == 3)			strcpy(c->x86_model_id, "386");	}#endif} static const struct cpu_dev default_cpu = {	.c_init		= default_init,	.c_vendor	= "Unknown",	.c_x86_vendor	= X86_VENDOR_UNKNOWN,}; static const struct cpu_dev *this_cpu = &default_cpu; DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {#ifdef CONFIG_X86_64	/*	 * We need valid kernel segments for data and code in long mode too	 * IRET will check the segment types  kkeil 2000/10/28	 * Also sysret mandates a special GDT layout	 *	 * TLS descriptors are currently at a different place compared to i386.	 * Hopefully nobody expects them at a fixed place (Wine?)	 */	[GDT_ENTRY_KERNEL32_CS]		= GDT_ENTRY_INIT(DESC_CODE32, 0, 0xfffff),	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(DESC_CODE64, 0, 0xfffff),	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(DESC_DATA64, 0, 0xfffff),	[GDT_ENTRY_DEFAULT_USER32_CS]	= GDT_ENTRY_INIT(DESC_CODE32 | DESC_USER, 0, 0xfffff),	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(DESC_DATA64 | DESC_USER, 0, 0xfffff),	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(DESC_CODE64 | DESC_USER, 0, 0xfffff),#else	[GDT_ENTRY_KERNEL_CS]		= GDT_ENTRY_INIT(DESC_CODE32, 0, 0xfffff),	[GDT_ENTRY_KERNEL_DS]		= GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),	[GDT_ENTRY_DEFAULT_USER_CS]	= GDT_ENTRY_INIT(DESC_CODE32 | DESC_USER, 0, 0xfffff),	[GDT_ENTRY_DEFAULT_USER_DS]	= GDT_ENTRY_INIT(DESC_DATA32 | DESC_USER, 0, 0xfffff),	/*	 * Segments used for calling PnP BIOS have byte granularity.	 * They code segments and data segments have fixed 64k limits,	 * the transfer segment sizes are set at run time.	 */	[GDT_ENTRY_PNPBIOS_CS32]	= GDT_ENTRY_INIT(DESC_CODE32_BIOS, 0, 0xffff),	[GDT_ENTRY_PNPBIOS_CS16]	= GDT_ENTRY_INIT(DESC_CODE16, 0, 0xffff),	[GDT_ENTRY_PNPBIOS_DS]		= GDT_ENTRY_INIT(DESC_DATA16, 0, 0xffff),	[GDT_ENTRY_PNPBIOS_TS1]		= GDT_ENTRY_INIT(DESC_DATA16, 0, 0),	[GDT_ENTRY_PNPBIOS_TS2]		= GDT_ENTRY_INIT(DESC_DATA16, 0, 0),	/*	 * The APM segments have byte granularity and their bases	 * are set at run time.  All have 64k limits.	 */	[GDT_ENTRY_APMBIOS_BASE]	= GDT_ENTRY_INIT(DESC_CODE32_BIOS, 0, 0xffff),	[GDT_ENTRY_APMBIOS_BASE+1]	= GDT_ENTRY_INIT(DESC_CODE16, 0, 0xffff),	[GDT_ENTRY_APMBIOS_BASE+2]	= GDT_ENTRY_INIT(DESC_DATA32_BIOS, 0, 0xffff), 	[GDT_ENTRY_ESPFIX_SS]		= GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),	[GDT_ENTRY_PERCPU]		= GDT_ENTRY_INIT(DESC_DATA32, 0, 0xfffff),#endif} };EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);SYM_PIC_ALIAS(gdt_page); #ifdef CONFIG_X86_64static int __init x86_nopcid_setup(char *s){	/* nopcid doesn't accept parameters */	if (s)		return -EINVAL; 	/* do not emit a message if the feature is not present */	if (!boot_cpu_has(X86_FEATURE_PCID))		return 0; 	setup_clear_cpu_cap(X86_FEATURE_PCID);	pr_info("nopcid: PCID feature disabled\n");	return 0;}early_param("nopcid", x86_nopcid_setup);#endif static int __init x86_noinvpcid_setup(char *s){	/* noinvpcid doesn't accept parameters */	if (s)		return -EINVAL; 	/* do not emit a message if the feature is not present */	if (!boot_cpu_has(X86_FEATURE_INVPCID))		return 0; 	setup_clear_cpu_cap(X86_FEATURE_INVPCID);	pr_info("noinvpcid: INVPCID feature disabled\n");	return 0;}early_param("noinvpcid", x86_noinvpcid_setup); /* Standard macro to see if a specific flag is changeable */static inline bool flag_is_changeable_p(unsigned long flag){	unsigned long f1, f2; 	if (!IS_ENABLED(CONFIG_X86_32))		return true; 	/*	 * Cyrix and IDT cpus allow disabling of CPUID	 * so the code below may return different results	 * when it is executed before and after enabling	 * the CPUID. Add "volatile" to not allow gcc to	 * optimize the subsequent calls to this function.	 */	asm volatile ("pushfl		\n\t"		      "pushfl		\n\t"		      "popl %0		\n\t"		      "movl %0, %1	\n\t"		      "xorl %2, %0	\n\t"		      "pushl %0		\n\t"		      "popfl		\n\t"		      "pushfl		\n\t"		      "popl %0		\n\t"		      "popfl		\n\t" 		      : "=&r" (f1), "=&r" (f2)		      : "ir" (flag)); 	return (f1 ^ f2) & flag;} #ifdef CONFIG_X86_32static int cachesize_override = -1;static int disable_x86_serial_nr = 1; static int __init cachesize_setup(char *str){	get_option(&str, &cachesize_override);	return 1;}__setup("cachesize=", cachesize_setup); /* Probe for the CPUID instruction */bool cpuid_feature(void){	return flag_is_changeable_p(X86_EFLAGS_ID);} static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c){	struct msr val; 	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)		return; 	/* Disable processor serial number: */ 	rdmsrq(MSR_IA32_BBL_CR_CTL, val.q);	val.l |= 0x200000;	wrmsrq(MSR_IA32_BBL_CR_CTL, val.q); 	pr_notice("CPU serial number disabled.\n");	clear_cpu_cap(c, X86_FEATURE_PN); 	/* Disabling the serial number may affect the cpuid level */	c->cpuid_level = cpuid_eax(0);} static int __init x86_serial_nr_setup(char *s){	disable_x86_serial_nr = 0;	return 1;}__setup("serialnumber", x86_serial_nr_setup);#elsestatic inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c){}#endif static __always_inline void setup_smep(struct cpuinfo_x86 *c){	if (cpu_has(c, X86_FEATURE_SMEP))		cr4_set_bits(X86_CR4_SMEP);} static __always_inline void setup_smap(struct cpuinfo_x86 *c){	unsigned long eflags = native_save_fl(); 	/* This should have been cleared long ago */	BUG_ON(eflags & X86_EFLAGS_AC); 	if (cpu_has(c, X86_FEATURE_SMAP))		cr4_set_bits(X86_CR4_SMAP);} static __always_inline void setup_umip(struct cpuinfo_x86 *c){	/* Check the boot processor, plus build option for UMIP. */	if (!cpu_feature_enabled(X86_FEATURE_UMIP))		goto out; 	/* Check the current processor's cpuid bits. */	if (!cpu_has(c, X86_FEATURE_UMIP))		goto out; 	cr4_set_bits(X86_CR4_UMIP); 	pr_info_once("x86/cpu: User Mode Instruction Prevention (UMIP) activated\n"); 	return; out:	/*	 * Make sure UMIP is disabled in case it was enabled in a	 * previous boot (e.g., via kexec).	 */	cr4_clear_bits(X86_CR4_UMIP);} static int enable_lass(unsigned int cpu){	cr4_set_bits(X86_CR4_LASS); 	return 0;} /* * Finalize features that need to be enabled just before entering * userspace. Note that this only runs on a single CPU. Use appropriate * callbacks if all the CPUs need to reflect the same change. */static int cpu_finalize_pre_userspace(void){	if (!cpu_feature_enabled(X86_FEATURE_LASS))		return 0; 	/* Runs on all online CPUs and future CPUs that come online. */	cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/lass:enable", enable_lass, NULL); 	return 0;}late_initcall(cpu_finalize_pre_userspace); /* These bits should not change their value after CPU init is finished. */static const unsigned long cr4_pinned_mask = X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP |					     X86_CR4_FSGSBASE | X86_CR4_CET; /* * The CR pinning protects against ROP on the 'mov %reg, %CRn' instruction(s). * Since you can ROP directly to these instructions (barring shadow stack), * any protection must follow immediately and unconditionally after that. * * Specifically, the CR[04] write functions below will have the value * validation controlled by the @cr_pinning static_branch which is * __ro_after_init, just like the cr4_pinned_bits value. * * Once set, an attacker will have to defeat page-tables to get around these * restrictions. Which is a much bigger ask than 'simple' ROP. */static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);static unsigned long cr4_pinned_bits __ro_after_init; void native_write_cr0(unsigned long val){	unsigned long bits_missing = 0; set_register:	asm volatile("mov %0,%%cr0": "+r" (val) : : "memory"); 	if (static_branch_likely(&cr_pinning)) {		if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {			bits_missing = X86_CR0_WP;			val |= bits_missing;			goto set_register;		}		/* Warn after we've set the missing bits. */		WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n");	}}EXPORT_SYMBOL(native_write_cr0); void __no_profile native_write_cr4(unsigned long val){	unsigned long bits_changed = 0; set_register:	asm volatile("mov %0,%%cr4": "+r" (val) : : "memory"); 	if (static_branch_likely(&cr_pinning)) {		if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) {			bits_changed = (val & cr4_pinned_mask) ^ cr4_pinned_bits;			val = (val & ~cr4_pinned_mask) | cr4_pinned_bits;			goto set_register;		}		/* Warn after we've corrected the changed bits. */		WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n",			  bits_changed);	}}#if IS_MODULE(CONFIG_LKDTM)EXPORT_SYMBOL_GPL(native_write_cr4);#endif void cr4_update_irqsoff(unsigned long set, unsigned long clear){	unsigned long newval, cr4 = this_cpu_read(cpu_tlbstate.cr4); 

Lines 1–500 of 2688. Load more