x86: Add magic AMD return-thunk

Note: needs to be in a section distinct from Retpolines such that the Retpoline RET substitution cannot possibly use immediate jumps. ORC unwinding for zen_untrain_ret() and __x86_return_thunk() is a little tricky but works due to the fact that zen_untrain_ret() doesn't have any stack ops and as such will emit a single ORC entry at the start (+0x3f). Meanwhile, unwinding an IP, including the __x86_return_thunk() one (+0x40) will search for the largest ORC entry smaller or equal to the IP, these will find the one ORC entry (+0x3f) and all works. [ Alexandre: SVM part. ] [ bp: Build fix, massages. ]

Peter Zijlstra committed 4 years ago

Commit a1491801 parent 951ddec9 files · +128 −9Browse files

Unified · Split

9 files

arch/x86/entry/entry_64.S +6 −0
arch/x86/entry/entry_64_compat.S +4 −0
arch/x86/include/asm/cpufeatures.h +1 −0
arch/x86/include/asm/disabled-features.h +2 −1
arch/x86/include/asm/nospec-branch.h +17 −0
arch/x86/kernel/vmlinux.lds.S +1 −1
arch/x86/kvm/svm/vmenter.S +18 −0
arch/x86/lib/retpoline.S +60 −4
tools/objtool/check.c +19 −3

arch/x86/entry/entry_64.S+6−0

 SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL)
 	ANNOTATE_NOENDBR
+	UNTRAIN_RET
 	/* Construct struct pt_regs on stack */
 	pushq	$__USER_DS				/* pt_regs->ss */
 	pushq	%rdi				/* Stash user RDI */
 	swapgs					/* to kernel GS */
 	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi	/* to kernel CR3 */
+	UNTRAIN_RET
 	movq	PER_CPU_VAR(espfix_waddr), %rdi
 	movq	%rax, (0*8)(%rdi)		/* user RAX */
 	 * be retrieved from a kernel internal table.
 	 */
 	SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
+	UNTRAIN_RET
 	/*
 	 * Handling GSBASE depends on the availability of FSGSBASE.
 	FENCE_SWAPGS_USER_ENTRY
 	/* We have user CR3.  Change to kernel CR3. */
 	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
+	UNTRAIN_RET
 	leaq	8(%rsp), %rdi			/* arg0 = pt_regs pointer */
 .Lerror_entry_from_usermode_after_swapgs:
 	swapgs
 	FENCE_SWAPGS_USER_ENTRY
 	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
+	UNTRAIN_RET
 	/*
 	 * Pretend that the exception came from user mode: set up pt_regs
 	movq	%rsp, %rdx
 	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
 	UNWIND_HINT_IRET_REGS base=%rdx offset=8
+	UNTRAIN_RET
 	pushq	5*8(%rdx)	/* pt_regs->ss */
 	pushq	4*8(%rdx)	/* pt_regs->rsp */
 	pushq	3*8(%rdx)	/* pt_regs->flags */

arch/x86/entry/entry_64_compat.S+4−0

 #include <asm/irqflags.h>
 #include <asm/asm.h>
 #include <asm/smap.h>
+#include <asm/nospec-branch.h>
 #include <linux/linkage.h>
 #include <linux/err.h>
 	pushq	$__USER32_CS		/* pt_regs->cs */
 	pushq	$0			/* pt_regs->ip = 0 (placeholder) */
 SYM_INNER_LABEL(entry_SYSENTER_compat_after_hwframe, SYM_L_GLOBAL)
+	UNTRAIN_RET
 	/*
 	 * User tracing code (ptrace or signal handlers) might assume that
 SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL)
 	ANNOTATE_NOENDBR
+	UNTRAIN_RET
 	/* Construct struct pt_regs on stack */
 	pushq	$__USER32_DS		/* pt_regs->ss */
 	pushq	0*8(%rax)		/* regs->orig_ax */
 .Lint80_keep_stack:
+	UNTRAIN_RET
 	PUSH_AND_CLEAR_REGS rax=$-ENOSYS
 	UNWIND_HINT_REGS

arch/x86/include/asm/cpufeatures.h+1−0

@@ -301,6 +301,7 @@ · 300 unmodified lines
301	#define X86_FEATURE_RETPOLINE (1132+12) / "" Generic Retpoline mitigation for Spectre variant 2 */	301	#define X86_FEATURE_RETPOLINE (1132+12) / "" Generic Retpoline mitigation for Spectre variant 2 */
302	#define X86_FEATURE_RETPOLINE_LFENCE (1132+13) / "" Use LFENCE for Spectre variant 2 */	302	#define X86_FEATURE_RETPOLINE_LFENCE (1132+13) / "" Use LFENCE for Spectre variant 2 */
303	#define X86_FEATURE_RETHUNK (1132+14) / "" Use REturn THUNK */	303	#define X86_FEATURE_RETHUNK (1132+14) / "" Use REturn THUNK */
		304	#define X86_FEATURE_UNRET (1132+15) / "" AMD BTB untrain return */
304		305
305	/* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */	306	/* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */
306	#define X86_FEATURE_AVX_VNNI (1232+ 4) / AVX VNNI instructions */	307	#define X86_FEATURE_AVX_VNNI (1232+ 4) / AVX VNNI instructions */

arch/x86/include/asm/disabled-features.h+2−1

@@ -55,7 +55,8 @@ · 54 unmodified lines
55	#else	55	#else
56	# define DISABLE_RETPOLINE ((1 << (X86_FEATURE_RETPOLINE & 31)) \| \	56	# define DISABLE_RETPOLINE ((1 << (X86_FEATURE_RETPOLINE & 31)) \| \
57	(1 << (X86_FEATURE_RETPOLINE_LFENCE & 31)) \| \	57	(1 << (X86_FEATURE_RETPOLINE_LFENCE & 31)) \| \
58	(1 << (X86_FEATURE_RETHUNK & 31)))	58	(1 << (X86_FEATURE_RETHUNK & 31)) \| \
		59	(1 << (X86_FEATURE_UNRET & 31)))
59	#endif	60	#endif
60		61
61	#ifdef CONFIG_INTEL_IOMMU_SVM	62	#ifdef CONFIG_INTEL_IOMMU_SVM

arch/x86/include/asm/nospec-branch.h+17−0

@@ -112,6 +112,22 @@ · 111 unmodified lines
112	#endif	112	#endif
113	.endm	113	.endm
114		114
		115	/*
		116	* Mitigate RETBleed for AMD/Hygon Zen uarch. Requires KERNEL CR3 because the
		117	* return thunk isn't mapped into the userspace tables (then again, AMD
		118	* typically has NO_MELTDOWN).
		119	*
		120	* Doesn't clobber any registers but does require a stable stack.
		121	*
		122	* As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point
		123	* where we have a stack but before any RET instruction.
		124	*/
		125	.macro UNTRAIN_RET
		126	#ifdef CONFIG_RETPOLINE
		127	ALTERNATIVE "", "call zen_untrain_ret", X86_FEATURE_UNRET
		128	#endif
		129	.endm
		130
115	#else /* __ASSEMBLY__ */	131	#else /* __ASSEMBLY__ */
116		132
117	#define ANNOTATE_RETPOLINE_SAFE \	133	#define ANNOTATE_RETPOLINE_SAFE \
@@ -124,6 +140,7 @@ typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE]; · 6 unmodified lines
124	extern retpoline_thunk_t __x86_indirect_thunk_array[];	140	extern retpoline_thunk_t __x86_indirect_thunk_array[];
125		141
126	extern void __x86_return_thunk(void);	142	extern void __x86_return_thunk(void);
		143	extern void zen_untrain_ret(void);
127		144
128	#ifdef CONFIG_RETPOLINE	145	#ifdef CONFIG_RETPOLINE
129		146

arch/x86/kernel/vmlinux.lds.S+1−1

@@ -141,7 +141,7 @@ SECTIONS · 140 unmodified lines
141		141
142	#ifdef CONFIG_RETPOLINE	142	#ifdef CONFIG_RETPOLINE
143	__indirect_thunk_start = .;	143	__indirect_thunk_start = .;
144	*(.text.__x86.indirect_thunk)	144	(.text.__x86.)
145	__indirect_thunk_end = .;	145	__indirect_thunk_end = .;
146	#endif	146	#endif
147	} :text =0xcccc	147	} :text =0xcccc

arch/x86/kvm/svm/vmenter.S+18−0

@@ -110,6 +110,15 @@ SYM_FUNC_START(__svm_vcpu_run) · 109 unmodified lines
110	mov %r15, VCPU_R15(%_ASM_AX)	110	mov %r15, VCPU_R15(%_ASM_AX)
111	#endif	111	#endif
112		112
		113	/*
		114	* Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be
		115	* untrained as soon as we exit the VM and are back to the
		116	* kernel. This should be done before re-enabling interrupts
		117	* because interrupt handlers won't sanitize 'ret' if the return is
		118	* from the kernel.
		119	*/
		120	UNTRAIN_RET
		121
113	/*	122	/*
114	* Clear all general purpose registers except RSP and RAX to prevent	123	* Clear all general purpose registers except RSP and RAX to prevent
115	* speculative use of the guest's values, even those that are reloaded	124	* speculative use of the guest's values, even those that are reloaded
@@ -190,6 +199,15 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) · 74 unmodified lines
190	FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE	199	FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE
191	#endif	200	#endif
192		201
		202	/*
		203	* Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be
		204	* untrained as soon as we exit the VM and are back to the
		205	* kernel. This should be done before re-enabling interrupts
		206	* because interrupt handlers won't sanitize RET if the return is
		207	* from the kernel.
		208	*/
		209	UNTRAIN_RET
		210
193	pop %_ASM_BX	211	pop %_ASM_BX
194		212
195	#ifdef CONFIG_X86_64	213	#ifdef CONFIG_X86_64

arch/x86/lib/retpoline.S+60−4

@@ -72,11 +72,67 @@ SYM_CODE_END(__x86_indirect_thunk_array) · 71 unmodified lines
72	* This function name is magical and is used by -mfunction-return=thunk-extern	72	* This function name is magical and is used by -mfunction-return=thunk-extern
73	* for the compiler to generate JMPs to it.	73	* for the compiler to generate JMPs to it.
74	*/	74	*/
75	SYM_CODE_START(__x86_return_thunk)	75	.section .text.__x86.return_thunk
76	UNWIND_HINT_EMPTY	76
77	ANNOTATE_NOENDBR	77	/*
		78	* Safety details here pertain to the AMD Zen{1,2} microarchitecture:
		79	* 1) The RET at __x86_return_thunk must be on a 64 byte boundary, for
		80	* alignment within the BTB.
		81	* 2) The instruction at zen_untrain_ret must contain, and not
		82	* end with, the 0xc3 byte of the RET.
		83	* 3) STIBP must be enabled, or SMT disabled, to prevent the sibling thread
		84	* from re-poisioning the BTB prediction.
		85	*/
		86	.align 64
		87	.skip 63, 0xcc
		88	SYM_FUNC_START_NOALIGN(zen_untrain_ret);
		89
		90	/*
		91	* As executed from zen_untrain_ret, this is:
		92	*
		93	* TEST $0xcc, %bl
		94	* LFENCE
		95	* JMP __x86_return_thunk
		96	*
		97	* Executing the TEST instruction has a side effect of evicting any BTB
		98	* prediction (potentially attacker controlled) attached to the RET, as
		99	* __x86_return_thunk + 1 isn't an instruction boundary at the moment.
		100	*/
		101	.byte 0xf6
		102
		103	/*
		104	* As executed from __x86_return_thunk, this is a plain RET.
		105	*
		106	* As part of the TEST above, RET is the ModRM byte, and INT3 the imm8.
		107	*
		108	* We subsequently jump backwards and architecturally execute the RET.
		109	* This creates a correct BTB prediction (type=ret), but in the
		110	* meantime we suffer Straight Line Speculation (because the type was
		111	* no branch) which is halted by the INT3.
		112	*
		113	* With SMT enabled and STIBP active, a sibling thread cannot poison
		114	* RET's prediction to a type of its choice, but can evict the
		115	* prediction due to competitive sharing. If the prediction is
		116	* evicted, __x86_return_thunk will suffer Straight Line Speculation
		117	* which will be contained safely by the INT3.
		118	*/
		119	SYM_INNER_LABEL(__x86_return_thunk, SYM_L_GLOBAL)
78	ret	120	ret
79	int3	121	int3
80	SYM_CODE_END(__x86_return_thunk)	122	SYM_CODE_END(__x86_return_thunk)
81		123
82	__EXPORT_THUNK(__x86_return_thunk)	124	/*
		125	* Ensure the TEST decoding / BTB invalidation is complete.
		126	*/
		127	lfence
		128
		129	/*
		130	* Jump back and execute the RET in the middle of the TEST instruction.
		131	* INT3 is for SLS protection.
		132	*/
		133	jmp __x86_return_thunk
		134	int3
		135	SYM_FUNC_END(zen_untrain_ret)
		136	__EXPORT_THUNK(zen_untrain_ret)
		137
		138	EXPORT_SYMBOL(__x86_return_thunk)

tools/objtool/check.c+19−3

 	annotate_call_site(file, insn, false);
 }
-static void add_return_call(struct objtool_file *file, struct instruction *insn)
+static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
 {
 	/*
 	 * Return thunk tail calls are really just returns in disguise,
 	insn->type = INSN_RETURN;
 	insn->retpoline_safe = true;
-	list_add_tail(&insn->call_node, &file->return_thunk_list);
+	if (add)
+		list_add_tail(&insn->call_node, &file->return_thunk_list);
 }
 static bool same_function(struct instruction *insn1, struct instruction *insn2)
 			add_retpoline_call(file, insn);
 			continue;
 		} else if (reloc->sym->return_thunk) {
-			add_return_call(file, insn);
+			add_return_call(file, insn, true);
 			continue;
 		} else if (insn->func) {
 			/*
 		jump_dest = find_insn(file, dest_sec, dest_off);
 		if (!jump_dest) {
+			struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
+			/*
+			 * This is a special case for zen_untrain_ret().
+			 * It jumps to __x86_return_thunk(), but objtool
+			 * can't find the thunk's starting RET
+			 * instruction, because the RET is also in the
+			 * middle of another instruction.  Objtool only
+			 * knows about the outer instruction.
+			 */
+			if (sym && sym->return_thunk) {
+				add_return_call(file, insn, false);
+				continue;
+			}
 			WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
 				  insn->sec, insn->offset, dest_sec->name,
 				  dest_off);