History

x86: Add magic AMD return-thunk

Note: needs to be in a section distinct from Retpolines such that the Retpoline RET substitution cannot possibly use immediate jumps. ORC unwinding for zen_untrain_ret() and __x86_return_thunk() is a little tricky but works due to the fact that zen_untrain_ret() doesn't have any stack ops and as such will emit a single ORC entry at the start (+0x3f). Meanwhile, unwinding an IP, including the __x86_return_thunk() one (+0x40) will search for the largest ORC entry smaller or equal to the IP, these will find the one ORC entry (+0x3f) and all works. [ Alexandre: SVM part. ] [ bp: Build fix, massages. ]

Peter Zijlstra committed 4 years ago

Commit a1491801 parent 951ddec9 files · +128 −9Browse files

Unified · Split

9 files
  1. arch/x86/entry/entry_64.S +6 −0
  2. arch/x86/entry/entry_64_compat.S +4 −0
  3. arch/x86/include/asm/cpufeatures.h +1 −0
  4. arch/x86/include/asm/disabled-features.h +2 −1
  5. arch/x86/include/asm/nospec-branch.h +17 −0
  6. arch/x86/kernel/vmlinux.lds.S +1 −1
  7. arch/x86/kvm/svm/vmenter.S +18 −0
  8. arch/x86/lib/retpoline.S +60 −4
  9. tools/objtool/check.c +19 −3
arch/x86/entry/entry_64.S+6−0
@@ -96,6 +96,7 @@ SYM_CODE_START(entry_SYSCALL_64) · 95 unmodified lines
9696
9797 SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL)
9898 ANNOTATE_NOENDBR
99+ UNTRAIN_RET
99100
100101 /* Construct struct pt_regs on stack */
101102 pushq $__USER_DS /* pt_regs->ss */
@@ -717,6 +718,7 @@ native_irq_return_ldt: · 615 unmodified lines
717718 pushq %rdi /* Stash user RDI */
718719 swapgs /* to kernel GS */
719720 SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi /* to kernel CR3 */
721+ UNTRAIN_RET
720722
721723 movq PER_CPU_VAR(espfix_waddr), %rdi
722724 movq %rax, (0*8)(%rdi) /* user RAX */
@@ -911,6 +913,7 @@ SYM_CODE_START_LOCAL(paranoid_entry) · 188 unmodified lines
911913 * be retrieved from a kernel internal table.
912914 */
913915 SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
916+ UNTRAIN_RET
914917
915918 /*
916919 * Handling GSBASE depends on the availability of FSGSBASE.
@@ -1020,6 +1023,7 @@ SYM_CODE_START_LOCAL(error_entry) · 103 unmodified lines
10201023 FENCE_SWAPGS_USER_ENTRY
10211024 /* We have user CR3. Change to kernel CR3. */
10221025 SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1026+ UNTRAIN_RET
10231027
10241028 leaq 8(%rsp), %rdi /* arg0 = pt_regs pointer */
10251029 .Lerror_entry_from_usermode_after_swapgs:
@@ -1072,6 +1076,7 @@ SYM_CODE_START_LOCAL(error_entry) · 46 unmodified lines
10721076 swapgs
10731077 FENCE_SWAPGS_USER_ENTRY
10741078 SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1079+ UNTRAIN_RET
10751080
10761081 /*
10771082 * Pretend that the exception came from user mode: set up pt_regs
@@ -1167,6 +1172,7 @@ SYM_CODE_START(asm_exc_nmi) · 89 unmodified lines
11671172 movq %rsp, %rdx
11681173 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
11691174 UNWIND_HINT_IRET_REGS base=%rdx offset=8
1175+ UNTRAIN_RET
11701176 pushq 5*8(%rdx) /* pt_regs->ss */
11711177 pushq 4*8(%rdx) /* pt_regs->rsp */
11721178 pushq 3*8(%rdx) /* pt_regs->flags */
arch/x86/entry/entry_64_compat.S+4−0
@@ -14,6 +14,7 @@ · 13 unmodified lines
1414 #include <asm/irqflags.h>
1515 #include <asm/asm.h>
1616 #include <asm/smap.h>
17+#include <asm/nospec-branch.h>
1718 #include <linux/linkage.h>
1819 #include <linux/err.h>
1920
@@ -72,6 +73,7 @@ SYM_CODE_START(entry_SYSENTER_compat) · 52 unmodified lines
7273 pushq $__USER32_CS /* pt_regs->cs */
7374 pushq $0 /* pt_regs->ip = 0 (placeholder) */
7475 SYM_INNER_LABEL(entry_SYSENTER_compat_after_hwframe, SYM_L_GLOBAL)
76+ UNTRAIN_RET
7577
7678 /*
7779 * User tracing code (ptrace or signal handlers) might assume that
@@ -190,6 +192,7 @@ SYM_CODE_START(entry_SYSCALL_compat) · 112 unmodified lines
190192
191193 SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL)
192194 ANNOTATE_NOENDBR
195+ UNTRAIN_RET
193196
194197 /* Construct struct pt_regs on stack */
195198 pushq $__USER32_DS /* pt_regs->ss */
@@ -332,6 +335,7 @@ SYM_CODE_START(entry_INT80_compat) · 136 unmodified lines
332335 pushq 0*8(%rax) /* regs->orig_ax */
333336 .Lint80_keep_stack:
334337
338+ UNTRAIN_RET
335339 PUSH_AND_CLEAR_REGS rax=$-ENOSYS
336340 UNWIND_HINT_REGS
337341
arch/x86/include/asm/cpufeatures.h+1−0
@@ -301,6 +301,7 @@ · 300 unmodified lines
301301 #define X86_FEATURE_RETPOLINE (11*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */
302302 #define X86_FEATURE_RETPOLINE_LFENCE (11*32+13) /* "" Use LFENCE for Spectre variant 2 */
303303 #define X86_FEATURE_RETHUNK (11*32+14) /* "" Use REturn THUNK */
304+#define X86_FEATURE_UNRET (11*32+15) /* "" AMD BTB untrain return */
304305
305306 /* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */
306307 #define X86_FEATURE_AVX_VNNI (12*32+ 4) /* AVX VNNI instructions */
arch/x86/include/asm/disabled-features.h+2−1
@@ -55,7 +55,8 @@ · 54 unmodified lines
5555 #else
5656 # define DISABLE_RETPOLINE ((1 << (X86_FEATURE_RETPOLINE & 31)) | \
5757 (1 << (X86_FEATURE_RETPOLINE_LFENCE & 31)) | \
58− (1 << (X86_FEATURE_RETHUNK & 31)))
58+ (1 << (X86_FEATURE_RETHUNK & 31)) | \
59+ (1 << (X86_FEATURE_UNRET & 31)))
5960 #endif
6061
6162 #ifdef CONFIG_INTEL_IOMMU_SVM
arch/x86/include/asm/nospec-branch.h+17−0
@@ -112,6 +112,22 @@ · 111 unmodified lines
112112 #endif
113113 .endm
114114
115+/*
116+ * Mitigate RETBleed for AMD/Hygon Zen uarch. Requires KERNEL CR3 because the
117+ * return thunk isn't mapped into the userspace tables (then again, AMD
118+ * typically has NO_MELTDOWN).
119+ *
120+ * Doesn't clobber any registers but does require a stable stack.
121+ *
122+ * As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point
123+ * where we have a stack but before any RET instruction.
124+ */
125+.macro UNTRAIN_RET
126+#ifdef CONFIG_RETPOLINE
127+ ALTERNATIVE "", "call zen_untrain_ret", X86_FEATURE_UNRET
128+#endif
129+.endm
130+
115131 #else /* __ASSEMBLY__ */
116132
117133 #define ANNOTATE_RETPOLINE_SAFE \
@@ -124,6 +140,7 @@ typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE]; · 6 unmodified lines
124140 extern retpoline_thunk_t __x86_indirect_thunk_array[];
125141
126142 extern void __x86_return_thunk(void);
143+extern void zen_untrain_ret(void);
127144
128145 #ifdef CONFIG_RETPOLINE
129146
arch/x86/kernel/vmlinux.lds.S+1−1
@@ -141,7 +141,7 @@ SECTIONS · 140 unmodified lines
141141
142142 #ifdef CONFIG_RETPOLINE
143143 __indirect_thunk_start = .;
144− *(.text.__x86.indirect_thunk)
144+ *(.text.__x86.*)
145145 __indirect_thunk_end = .;
146146 #endif
147147 } :text =0xcccc
arch/x86/kvm/svm/vmenter.S+18−0
@@ -110,6 +110,15 @@ SYM_FUNC_START(__svm_vcpu_run) · 109 unmodified lines
110110 mov %r15, VCPU_R15(%_ASM_AX)
111111 #endif
112112
113+ /*
114+ * Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be
115+ * untrained as soon as we exit the VM and are back to the
116+ * kernel. This should be done before re-enabling interrupts
117+ * because interrupt handlers won't sanitize 'ret' if the return is
118+ * from the kernel.
119+ */
120+ UNTRAIN_RET
121+
113122 /*
114123 * Clear all general purpose registers except RSP and RAX to prevent
115124 * speculative use of the guest's values, even those that are reloaded
@@ -190,6 +199,15 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) · 74 unmodified lines
190199 FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE
191200 #endif
192201
202+ /*
203+ * Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be
204+ * untrained as soon as we exit the VM and are back to the
205+ * kernel. This should be done before re-enabling interrupts
206+ * because interrupt handlers won't sanitize RET if the return is
207+ * from the kernel.
208+ */
209+ UNTRAIN_RET
210+
193211 pop %_ASM_BX
194212
195213 #ifdef CONFIG_X86_64
arch/x86/lib/retpoline.S+60−4
@@ -72,11 +72,67 @@ SYM_CODE_END(__x86_indirect_thunk_array) · 71 unmodified lines
7272 * This function name is magical and is used by -mfunction-return=thunk-extern
7373 * for the compiler to generate JMPs to it.
7474 */
75−SYM_CODE_START(__x86_return_thunk)
76− UNWIND_HINT_EMPTY
77− ANNOTATE_NOENDBR
75+ .section .text.__x86.return_thunk
76+
77+/*
78+ * Safety details here pertain to the AMD Zen{1,2} microarchitecture:
79+ * 1) The RET at __x86_return_thunk must be on a 64 byte boundary, for
80+ * alignment within the BTB.
81+ * 2) The instruction at zen_untrain_ret must contain, and not
82+ * end with, the 0xc3 byte of the RET.
83+ * 3) STIBP must be enabled, or SMT disabled, to prevent the sibling thread
84+ * from re-poisioning the BTB prediction.
85+ */
86+ .align 64
87+ .skip 63, 0xcc
88+SYM_FUNC_START_NOALIGN(zen_untrain_ret);
89+
90+ /*
91+ * As executed from zen_untrain_ret, this is:
92+ *
93+ * TEST $0xcc, %bl
94+ * LFENCE
95+ * JMP __x86_return_thunk
96+ *
97+ * Executing the TEST instruction has a side effect of evicting any BTB
98+ * prediction (potentially attacker controlled) attached to the RET, as
99+ * __x86_return_thunk + 1 isn't an instruction boundary at the moment.
100+ */
101+ .byte 0xf6
102+
103+ /*
104+ * As executed from __x86_return_thunk, this is a plain RET.
105+ *
106+ * As part of the TEST above, RET is the ModRM byte, and INT3 the imm8.
107+ *
108+ * We subsequently jump backwards and architecturally execute the RET.
109+ * This creates a correct BTB prediction (type=ret), but in the
110+ * meantime we suffer Straight Line Speculation (because the type was
111+ * no branch) which is halted by the INT3.
112+ *
113+ * With SMT enabled and STIBP active, a sibling thread cannot poison
114+ * RET's prediction to a type of its choice, but can evict the
115+ * prediction due to competitive sharing. If the prediction is
116+ * evicted, __x86_return_thunk will suffer Straight Line Speculation
117+ * which will be contained safely by the INT3.
118+ */
119+SYM_INNER_LABEL(__x86_return_thunk, SYM_L_GLOBAL)
78120 ret
79121 int3
80122 SYM_CODE_END(__x86_return_thunk)
81123
82−__EXPORT_THUNK(__x86_return_thunk)
124+ /*
125+ * Ensure the TEST decoding / BTB invalidation is complete.
126+ */
127+ lfence
128+
129+ /*
130+ * Jump back and execute the RET in the middle of the TEST instruction.
131+ * INT3 is for SLS protection.
132+ */
133+ jmp __x86_return_thunk
134+ int3
135+SYM_FUNC_END(zen_untrain_ret)
136+__EXPORT_THUNK(zen_untrain_ret)
137+
138+EXPORT_SYMBOL(__x86_return_thunk)
tools/objtool/check.c+19−3
@@ -1302,7 +1302,7 @@ static void add_retpoline_call(struct objtool_file *file, struct instruction *in · 1301 unmodified lines
13021302 annotate_call_site(file, insn, false);
13031303 }
13041304
1305−static void add_return_call(struct objtool_file *file, struct instruction *insn)
1305+static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
13061306 {
13071307 /*
13081308 * Return thunk tail calls are really just returns in disguise,
@@ -1311,7 +1311,8 @@ static void add_return_call(struct objtool_file *file, struct instruction *insn) · 2 unmodified lines
13111311 insn->type = INSN_RETURN;
13121312 insn->retpoline_safe = true;
13131313
1314− list_add_tail(&insn->call_node, &file->return_thunk_list);
1314+ if (add)
1315+ list_add_tail(&insn->call_node, &file->return_thunk_list);
13151316 }
13161317
13171318 static bool same_function(struct instruction *insn1, struct instruction *insn2)
@@ -1367,7 +1368,7 @@ static int add_jump_destinations(struct objtool_file *file) · 49 unmodified lines
13671368 add_retpoline_call(file, insn);
13681369 continue;
13691370 } else if (reloc->sym->return_thunk) {
1370− add_return_call(file, insn);
1371+ add_return_call(file, insn, true);
13711372 continue;
13721373 } else if (insn->func) {
13731374 /*
@@ -1387,6 +1388,21 @@ static int add_jump_destinations(struct objtool_file *file) · 13 unmodified lines
13871388
13881389 jump_dest = find_insn(file, dest_sec, dest_off);
13891390 if (!jump_dest) {
1391+ struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1392+
1393+ /*
1394+ * This is a special case for zen_untrain_ret().
1395+ * It jumps to __x86_return_thunk(), but objtool
1396+ * can't find the thunk's starting RET
1397+ * instruction, because the RET is also in the
1398+ * middle of another instruction. Objtool only
1399+ * knows about the outer instruction.
1400+ */
1401+ if (sym && sym->return_thunk) {
1402+ add_return_call(file, insn, false);
1403+ continue;
1404+ }
1405+
13901406 WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
13911407 insn->sec, insn->offset, dest_sec->name,
13921408 dest_off);