Merge tag 'x86_urgent_for_v5.16_rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 fixes from Borislav Petkov: - Fix a couple of SWAPGS fencing issues in the x86 entry code - Use the proper operand types in __{get,put}_user() to prevent truncation in SEV-ES string io - Make sure the kernel mappings are present in trampoline_pgd in order to prevent any potential accesses to unmapped memory after switching to it - Fix a trivial list corruption in objtool's pv_ops validation - Disable the clocksource watchdog for TSC on platforms which claim that the TSC is constant, doesn't stop in sleep states, CPU has TSC adjust and the number of sockets of the platform are max 2, to prevent erroneous markings of the TSC as unstable. - Make sure TSC adjust is always checked not only when going idle - Prevent a stack leak by initializing struct _fpx_sw_bytes properly in the FPU code - Fix INTEL_FAM6_RAPTORLAKE define naming to adhere to the convention * tag 'x86_urgent_for_v5.16_rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: x86/xen: Add xenpv_restore_regs_and_return_to_usermode() x86/entry: Use the correct fence macro after swapgs in kernel CR3 x86/entry: Add a fence for kernel entry SWAPGS in paranoid_entry() x86/sev: Fix SEV-ES INS/OUTS instructions for word, dword, and qword x86/64/mm: Map all kernel memory into trampoline_pgd objtool: Fix pv_ops noinstr validation x86/tsc: Disable clocksource watchdog for TSC on qualified platorms x86/tsc: Add a timer to make sure TSC_adjust is always checked x86/fpu/signal: Initialize sw_bytes in save_xstate_epilog() x86/cpu: Drop spurious underscore from RAPTOR_LAKE #define

Linus Torvalds committed 4 years ago

Commit f5d54a42 parents 90bf8d9 5c8f6a210 files · +159 −43Browse files

Unified · Split

10 files

arch/x86/entry/entry_64.S +17 −18
arch/x86/include/asm/intel-family.h +1 −1
arch/x86/kernel/fpu/signal.c +1 −1
arch/x86/kernel/sev.c +39 −18
arch/x86/kernel/tsc.c +24 −4
arch/x86/kernel/tsc_sync.c +41 −0
arch/x86/realmode/init.c +11 −1
arch/x86/xen/xen-asm.S +20 −0
tools/objtool/elf.c +1 −0
tools/objtool/objtool.c +4 −0

arch/x86/entry/entry_64.S+17−18

@@ -574,6 +574,10 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) · 573 unmodified lines
574	ud2	574	ud2
575	1:	575	1:
576	#endif	576	#endif
		577	#ifdef CONFIG_XEN_PV
		578	ALTERNATIVE "", "jmp xenpv_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV
		579	#endif
		580
577	POP_REGS pop_rdi=0	581	POP_REGS pop_rdi=0
578		582
579	/*	583	/*
@@ -890,6 +894,7 @@ SYM_CODE_START_LOCAL(paranoid_entry) · 310 unmodified lines
890	.Lparanoid_entry_checkgs:	894	.Lparanoid_entry_checkgs:
891	/* EBX = 1 -> kernel GSBASE active, no restore required */	895	/* EBX = 1 -> kernel GSBASE active, no restore required */
892	movl $1, %ebx	896	movl $1, %ebx
		897
893	/*	898	/*
894	* The kernel-enforced convention is a negative GSBASE indicates	899	* The kernel-enforced convention is a negative GSBASE indicates
895	* a kernel value. No SWAPGS needed on entry and exit.	900	* a kernel value. No SWAPGS needed on entry and exit.
@@ -897,21 +902,14 @@ SYM_CODE_START_LOCAL(paranoid_entry) · 1 unmodified line
897	movl $MSR_GS_BASE, %ecx	902	movl $MSR_GS_BASE, %ecx
898	rdmsr	903	rdmsr
899	testl %edx, %edx	904	testl %edx, %edx
900	jns .Lparanoid_entry_swapgs	905	js .Lparanoid_kernel_gsbase
901	ret
902		906
903	.Lparanoid_entry_swapgs:	907	/* EBX = 0 -> SWAPGS required on exit */
		908	xorl %ebx, %ebx
904	swapgs	909	swapgs
		910	.Lparanoid_kernel_gsbase:
905		911
906	/*
907	* The above SAVE_AND_SWITCH_TO_KERNEL_CR3 macro doesn't do an
908	* unconditional CR3 write, even in the PTI case. So do an lfence
909	* to prevent GS speculation, regardless of whether PTI is enabled.
910	*/
911	FENCE_SWAPGS_KERNEL_ENTRY	912	FENCE_SWAPGS_KERNEL_ENTRY
912
913	/* EBX = 0 -> SWAPGS required on exit */
914	xorl %ebx, %ebx
915	ret	913	ret
916	SYM_CODE_END(paranoid_entry)	914	SYM_CODE_END(paranoid_entry)
917		915
@@ -993,11 +991,6 @@ SYM_CODE_START_LOCAL(error_entry) · 75 unmodified lines
993	pushq %r12	991	pushq %r12
994	ret	992	ret
995		993
996	.Lerror_entry_done_lfence:
997	FENCE_SWAPGS_KERNEL_ENTRY
998	.Lerror_entry_done:
999	ret
1000
1001	/*	994	/*
1002	* There are two places in the kernel that can potentially fault with	995	* There are two places in the kernel that can potentially fault with
1003	* usergs. Handle them here. B stepping K8s sometimes report a	996	* usergs. Handle them here. B stepping K8s sometimes report a
@@ -1020,8 +1013,14 @@ SYM_CODE_START_LOCAL(error_entry) · 16 unmodified lines
1020	* .Lgs_change's error handler with kernel gsbase.	1013	* .Lgs_change's error handler with kernel gsbase.
1021	*/	1014	*/
1022	SWAPGS	1015	SWAPGS
1023	FENCE_SWAPGS_USER_ENTRY	1016
1024	jmp .Lerror_entry_done	1017	/*
		1018	* Issue an LFENCE to prevent GS speculation, regardless of whether it is a
		1019	* kernel or user gsbase.
		1020	*/
		1021	.Lerror_entry_done_lfence:
		1022	FENCE_SWAPGS_KERNEL_ENTRY
		1023	ret
1025		1024
1026	.Lbstep_iret:	1025	.Lbstep_iret:
1027	/* Fix truncated RIP */	1026	/* Fix truncated RIP */

arch/x86/include/asm/intel-family.h+1−1

@@ -108,7 +108,7 @@ · 107 unmodified lines
108	#define INTEL_FAM6_ALDERLAKE 0x97 /* Golden Cove / Gracemont */	108	#define INTEL_FAM6_ALDERLAKE 0x97 /* Golden Cove / Gracemont */
109	#define INTEL_FAM6_ALDERLAKE_L 0x9A /* Golden Cove / Gracemont */	109	#define INTEL_FAM6_ALDERLAKE_L 0x9A /* Golden Cove / Gracemont */
110		110
111	#define INTEL_FAM6_RAPTOR_LAKE 0xB7	111	#define INTEL_FAM6_RAPTORLAKE 0xB7
112		112
113	/* "Small Core" Processors (Atom) */	113	/* "Small Core" Processors (Atom) */
114		114

arch/x86/kernel/fpu/signal.c+1−1

@@ -118,7 +118,7 @@ static inline bool save_xstate_epilog(void __user *buf, int ia32_frame, · 117 unmodified lines
118	struct fpstate *fpstate)	118	struct fpstate *fpstate)
119	{	119	{
120	struct xregs_state __user *x = buf;	120	struct xregs_state __user *x = buf;
121	struct _fpx_sw_bytes sw_bytes;	121	struct _fpx_sw_bytes sw_bytes = {};
122	u32 xfeatures;	122	u32 xfeatures;
123	int err;	123	int err;
124		124

arch/x86/kernel/sev.c+39−18

 				   char *dst, char *buf, size_t size)
 {
 	unsigned long error_code = X86_PF_PROT | X86_PF_WRITE;
-	char __user *target = (char __user *)dst;
-	u64 d8;
-	u32 d4;
-	u16 d2;
-	u8  d1;
 	/*
 	 * This function uses __put_user() independent of whether kernel or user
 	 * instructions here would cause infinite nesting.
 	 */
 	switch (size) {
-	case 1:
+	case 1: {
+		u8 d1;
+		u8 __user *target = (u8 __user *)dst;
 		memcpy(&d1, buf, 1);
 		if (__put_user(d1, target))
 			goto fault;
 		break;
-	case 2:
+	}
+	case 2: {
+		u16 d2;
+		u16 __user *target = (u16 __user *)dst;
 		memcpy(&d2, buf, 2);
 		if (__put_user(d2, target))
 			goto fault;
 		break;
-	case 4:
+	}
+	case 4: {
+		u32 d4;
+		u32 __user *target = (u32 __user *)dst;
 		memcpy(&d4, buf, 4);
 		if (__put_user(d4, target))
 			goto fault;
 		break;
-	case 8:
+	}
+	case 8: {
+		u64 d8;
+		u64 __user *target = (u64 __user *)dst;
 		memcpy(&d8, buf, 8);
 		if (__put_user(d8, target))
 			goto fault;
 		break;
+	}
 	default:
 		WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
 		return ES_UNSUPPORTED;
 				  char *src, char *buf, size_t size)
 {
 	unsigned long error_code = X86_PF_PROT;
-	char __user *s = (char __user *)src;
-	u64 d8;
-	u32 d4;
-	u16 d2;
-	u8  d1;
 	/*
 	 * This function uses __get_user() independent of whether kernel or user
 	 * instructions here would cause infinite nesting.
 	 */
 	switch (size) {
-	case 1:
+	case 1: {
+		u8 d1;
+		u8 __user *s = (u8 __user *)src;
 		if (__get_user(d1, s))
 			goto fault;
 		memcpy(buf, &d1, 1);
 		break;
-	case 2:
+	}
+	case 2: {
+		u16 d2;
+		u16 __user *s = (u16 __user *)src;
 		if (__get_user(d2, s))
 			goto fault;
 		memcpy(buf, &d2, 2);
 		break;
-	case 4:
+	}
+	case 4: {
+		u32 d4;
+		u32 __user *s = (u32 __user *)src;
 		if (__get_user(d4, s))
 			goto fault;
 		memcpy(buf, &d4, 4);
 		break;
-	case 8:
+	}
+	case 8: {
+		u64 d8;
+		u64 __user *s = (u64 __user *)src;
 		if (__get_user(d8, s))
 			goto fault;
 		memcpy(buf, &d8, 8);
 		break;
+	}
 	default:
 		WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
 		return ES_UNSUPPORTED;

arch/x86/kernel/tsc.c+24−4

 EXPORT_SYMBOL_GPL(mark_tsc_unstable);
+static void __init tsc_disable_clocksource_watchdog(void)
+{
+	clocksource_tsc_early.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
+	clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
+}
 static void __init check_system_tsc_reliable(void)
 {
 #if defined(CONFIG_MGEODEGX1) || defined(CONFIG_MGEODE_LX) || defined(CONFIG_X86_GENERIC)
 #endif
 	if (boot_cpu_has(X86_FEATURE_TSC_RELIABLE))
 		tsc_clocksource_reliable = 1;
+	/*
+	 * Disable the clocksource watchdog when the system has:
+	 *  - TSC running at constant frequency
+	 *  - TSC which does not stop in C-States
+	 *  - the TSC_ADJUST register which allows to detect even minimal
+	 *    modifications
+	 *  - not more than two sockets. As the number of sockets cannot be
+	 *    evaluated at the early boot stage where this has to be
+	 *    invoked, check the number of online memory nodes as a
+	 *    fallback solution which is an reasonable estimate.
+	 */
+	if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC) &&
+	    boot_cpu_has(X86_FEATURE_NONSTOP_TSC) &&
+	    boot_cpu_has(X86_FEATURE_TSC_ADJUST) &&
+	    nr_online_nodes <= 2)
+		tsc_disable_clocksource_watchdog();
 }
 /*
 	if (tsc_unstable)
 		goto unreg;
-	if (tsc_clocksource_reliable || no_tsc_watchdog)
-		clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
 	if (boot_cpu_has(X86_FEATURE_NONSTOP_TSC_S3))
 		clocksource_tsc.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
 	}
 	if (tsc_clocksource_reliable || no_tsc_watchdog)
-		clocksource_tsc_early.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
+		tsc_disable_clocksource_watchdog();
 	clocksource_register_khz(&clocksource_tsc_early, tsc_khz);
 	detect_art();

arch/x86/kernel/tsc_sync.c+41−0

@@ -30,6 +30,7 @@ struct tsc_adjust { · 29 unmodified lines
30	};	30	};
31		31
32	static DEFINE_PER_CPU(struct tsc_adjust, tsc_adjust);	32	static DEFINE_PER_CPU(struct tsc_adjust, tsc_adjust);
		33	static struct timer_list tsc_sync_check_timer;
33		34
34	/*	35	/*
35	* TSC's on different sockets may be reset asynchronously.	36	* TSC's on different sockets may be reset asynchronously.
@@ -77,6 +78,46 @@ void tsc_verify_tsc_adjust(bool resume) · 41 unmodified lines
77	}	78	}
78	}	79	}
79		80
		81	/*
		82	* Normally the tsc_sync will be checked every time system enters idle
		83	* state, but there is still caveat that a system won't enter idle,
		84	* either because it's too busy or configured purposely to not enter
		85	* idle.
		86	*
		87	* So setup a periodic timer (every 10 minutes) to make sure the check
		88	* is always on.
		89	*/
		90
		91	#define SYNC_CHECK_INTERVAL (HZ * 600)
		92
		93	static void tsc_sync_check_timer_fn(struct timer_list *unused)
		94	{
		95	int next_cpu;
		96
		97	tsc_verify_tsc_adjust(false);
		98
		99	/* Run the check for all onlined CPUs in turn */
		100	next_cpu = cpumask_next(raw_smp_processor_id(), cpu_online_mask);
		101	if (next_cpu >= nr_cpu_ids)
		102	next_cpu = cpumask_first(cpu_online_mask);
		103
		104	tsc_sync_check_timer.expires += SYNC_CHECK_INTERVAL;
		105	add_timer_on(&tsc_sync_check_timer, next_cpu);
		106	}
		107
		108	static int __init start_sync_check_timer(void)
		109	{
		110	if (!cpu_feature_enabled(X86_FEATURE_TSC_ADJUST) \|\| tsc_clocksource_reliable)
		111	return 0;
		112
		113	timer_setup(&tsc_sync_check_timer, tsc_sync_check_timer_fn, 0);
		114	tsc_sync_check_timer.expires = jiffies + SYNC_CHECK_INTERVAL;
		115	add_timer(&tsc_sync_check_timer);
		116
		117	return 0;
		118	}
		119	late_initcall(start_sync_check_timer);
		120
80	static void tsc_sanitize_first_cpu(struct tsc_adjust *cur, s64 bootval,	121	static void tsc_sanitize_first_cpu(struct tsc_adjust *cur, s64 bootval,
81	unsigned int cpu, bool bootcpu)	122	unsigned int cpu, bool bootcpu)
82	{	123	{

arch/x86/realmode/init.c+11−1

@@ -72,6 +72,7 @@ static void __init setup_real_mode(void) · 71 unmodified lines
72	#ifdef CONFIG_X86_64	72	#ifdef CONFIG_X86_64
73	u64 *trampoline_pgd;	73	u64 *trampoline_pgd;
74	u64 efer;	74	u64 efer;
		75	int i;
75	#endif	76	#endif
76		77
77	base = (unsigned char *)real_mode_header;	78	base = (unsigned char *)real_mode_header;
@@ -128,8 +129,17 @@ static void __init setup_real_mode(void) · 50 unmodified lines
128	trampoline_header->flags = 0;	129	trampoline_header->flags = 0;
129		130
130	trampoline_pgd = (u64 *) __va(real_mode_header->trampoline_pgd);	131	trampoline_pgd = (u64 *) __va(real_mode_header->trampoline_pgd);
		132
		133	/* Map the real mode stub as virtual == physical */
131	trampoline_pgd[0] = trampoline_pgd_entry.pgd;	134	trampoline_pgd[0] = trampoline_pgd_entry.pgd;
132	trampoline_pgd[511] = init_top_pgt[511].pgd;	135
		136	/*
		137	* Include the entirety of the kernel mapping into the trampoline
		138	* PGD. This way, all mappings present in the normal kernel page
		139	* tables are usable while running on trampoline_pgd.
		140	*/
		141	for (i = pgd_index(__PAGE_OFFSET); i < PTRS_PER_PGD; i++)
		142	trampoline_pgd[i] = init_top_pgt[i].pgd;
133	#endif	143	#endif
134		144
135	sme_sev_setup_real_mode(trampoline_header);	145	sme_sev_setup_real_mode(trampoline_header);

arch/x86/xen/xen-asm.S+20−0

@@ -20,6 +20,7 @@ · 19 unmodified lines
20		20
21	#include <linux/init.h>	21	#include <linux/init.h>
22	#include <linux/linkage.h>	22	#include <linux/linkage.h>
		23	#include <../entry/calling.h>
23		24
24	.pushsection .noinstr.text, "ax"	25	.pushsection .noinstr.text, "ax"
25	/*	26	/*
@@ -192,6 +193,25 @@ SYM_CODE_START(xen_iret) · 166 unmodified lines
192	jmp hypercall_iret	193	jmp hypercall_iret
193	SYM_CODE_END(xen_iret)	194	SYM_CODE_END(xen_iret)
194		195
		196	/*
		197	* XEN pv doesn't use trampoline stack, PER_CPU_VAR(cpu_tss_rw + TSS_sp0) is
		198	* also the kernel stack. Reusing swapgs_restore_regs_and_return_to_usermode()
		199	* in XEN pv would cause %rsp to move up to the top of the kernel stack and
		200	* leave the IRET frame below %rsp, which is dangerous to be corrupted if #NMI
		201	* interrupts. And swapgs_restore_regs_and_return_to_usermode() pushing the IRET
		202	* frame at the same address is useless.
		203	*/
		204	SYM_CODE_START(xenpv_restore_regs_and_return_to_usermode)
		205	UNWIND_HINT_REGS
		206	POP_REGS
		207
		208	/* stackleak_erase() can work safely on the kernel stack. */
		209	STACKLEAK_ERASE_NOCLOBBER
		210
		211	addq $8, %rsp /* skip regs->orig_ax */
		212	jmp xen_iret
		213	SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode)
		214
195	/*	215	/*
196	* Xen handles syscall callbacks much like ordinary exceptions, which	216	* Xen handles syscall callbacks much like ordinary exceptions, which
197	* means we have:	217	* means we have:

tools/objtool/elf.c+1−0

@@ -375,6 +375,7 @@ static int read_symbols(struct elf *elf) · 374 unmodified lines
375	return -1;	375	return -1;
376	}	376	}
377	memset(sym, 0, sizeof(*sym));	377	memset(sym, 0, sizeof(*sym));
		378	INIT_LIST_HEAD(&sym->pv_target);
378	sym->alias = sym;	379	sym->alias = sym;
379		380
380	sym->idx = i;	381	sym->idx = i;

tools/objtool/objtool.c+4−0

 	    !strcmp(func->name, "_paravirt_ident_64"))
 		return;
+	/* already added this function */
+	if (!list_empty(&func->pv_target))
+		return;
 	list_add(&func->pv_target, &f->pv_ops[idx].targets);
 	f->pv_ops[idx].clean = false;
 }

Merge tag 'x86_urgent_for_v5.16_rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code