History

Merge tag 'x86_urgent_for_v6.5_rc7' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 fixes from Borislav Petkov: "Extraordinary embargoed times call for extraordinary measures. That's why this week's x86/urgent branch is larger than usual, containing all the known fallout fixes after the SRSO mitigation got merged. I know, it is a bit late in the game but everyone who has reported a bug stemming from the SRSO pile, has tested that branch and has confirmed that it fixes their bug. Also, I've run it on every possible hardware I have and it is looking good. It is running on this very machine while I'm typing, for 2 days now without an issue. Famous last words... - Use LEA ...%rsp instead of ADD %rsp in the Zen1/2 SRSO return sequence as latter clobbers flags which interferes with fastop emulation in KVM, leading to guests freezing during boot - A fix for the DIV(0) quotient data leak on Zen1 to clear the divider buffers at the right time - Disable the SRSO mitigation on unaffected configurations as it got enabled there unnecessarily - Change .text section name to fix CONFIG_LTO_CLANG builds - Improve the optprobe indirect jmp check so that certain configurations can still be able to use optprobes at all - A serious and good scrubbing of the untraining routines by PeterZ: - Add proper speculation stopping traps so that objtool is happy - Adjust objtool to handle the new thunks - Make the thunk pointer assignable to the different untraining sequences at runtime, thus avoiding the alternative at the return thunk. It simplifies the code a bit too. - Add a entry_untrain_ret() main entry point which selects the respective untraining sequence - Rename things so that they're more clear - Fix stack validation with FRAME_POINTER=y builds - Fix static call patching to handle when a JMP to the return thunk is the last insn on the very last module memory page - Add more documentation about what each untraining routine does and why" * tag 'x86_urgent_for_v6.5_rc7' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: x86/srso: Correct the mitigation status when SMT is disabled x86/static_call: Fix __static_call_fixup() objtool/x86: Fixup frame-pointer vs rethunk x86/srso: Explain the untraining sequences a bit more x86/cpu/kvm: Provide UNTRAIN_RET_VM x86/cpu: Cleanup the untrain mess x86/cpu: Rename srso_(.*)_alias to srso_alias_\1 x86/cpu: Rename original retbleed methods x86/cpu: Clean up SRSO return thunk mess x86/alternative: Make custom return thunk unconditional objtool/x86: Fix SRSO mess x86/cpu: Fix up srso_safe_ret() and __x86_return_thunk() x86/cpu: Fix __x86_return_thunk symbol type x86/retpoline,kprobes: Skip optprobe check for indirect jumps with retpolines and IBT x86/retpoline,kprobes: Fix position of thunk sections with CONFIG_LTO_CLANG x86/srso: Disable the mitigation on unaffected configurations x86/CPU/AMD: Fix the DIV(0) initial fix attempt x86/retpoline: Don't clobber RFLAGS during srso_safe_ret()

Linus Torvalds committed 3 years ago

Commit bf98bae2 parents 4e7ffde 6405b7218 files · +236 −133Browse files

Unified · Split

18 files
  1. Documentation/admin-guide/hw-vuln/srso.rst +2 −2
  2. arch/x86/include/asm/entry-common.h +1 −0
  3. arch/x86/include/asm/nospec-branch.h +27 −22
  4. arch/x86/kernel/alternative.c +0 −4
  5. arch/x86/kernel/cpu/amd.c +1 −0
  6. arch/x86/kernel/cpu/bugs.c +23 −4
  7. arch/x86/kernel/kprobes/opt.c +16 −24
  8. arch/x86/kernel/static_call.c +13 −0
  9. arch/x86/kernel/traps.c +0 −2
  10. arch/x86/kernel/vmlinux.lds.S +9 −11
  11. arch/x86/kvm/svm/svm.c +2 −0
  12. arch/x86/kvm/svm/vmenter.S +2 −5
  13. arch/x86/lib/retpoline.S +95 −42
  14. tools/objtool/arch/x86/decode.c +7 −4
  15. tools/objtool/check.c +35 −10
  16. tools/objtool/include/objtool/arch.h +1 −0
  17. tools/objtool/include/objtool/elf.h +1 −0
  18. tools/perf/util/thread-stack.c +1 −3
Documentation/admin-guide/hw-vuln/srso.rst+2−2
@@ -141,8 +141,8 @@ sequence. · 140 unmodified lines
141141 To ensure the safety of this mitigation, the kernel must ensure that the
142142 safe return sequence is itself free from attacker interference. In Zen3
143143 and Zen4, this is accomplished by creating a BTB alias between the
144−untraining function srso_untrain_ret_alias() and the safe return
145−function srso_safe_ret_alias() which results in evicting a potentially
144+untraining function srso_alias_untrain_ret() and the safe return
145+function srso_alias_safe_ret() which results in evicting a potentially
146146 poisoned BTB entry and using that safe one for all function returns.
147147
148148 In older Zen1 and Zen2, this is accomplished using a reinterpretation
arch/x86/include/asm/entry-common.h+1−0
@@ -92,6 +92,7 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, · 91 unmodified lines
9292 static __always_inline void arch_exit_to_user_mode(void)
9393 {
9494 mds_user_clear_cpu_buffers();
95+ amd_clear_divider();
9596 }
9697 #define arch_exit_to_user_mode arch_exit_to_user_mode
9798
arch/x86/include/asm/nospec-branch.h+27−22
@@ -272,9 +272,9 @@ · 271 unmodified lines
272272 .endm
273273
274274 #ifdef CONFIG_CPU_UNRET_ENTRY
275−#define CALL_ZEN_UNTRAIN_RET "call zen_untrain_ret"
275+#define CALL_UNTRAIN_RET "call entry_untrain_ret"
276276 #else
277−#define CALL_ZEN_UNTRAIN_RET ""
277+#define CALL_UNTRAIN_RET ""
278278 #endif
279279
280280 /*
@@ -282,7 +282,7 @@ · 1 unmodified line
282282 * return thunk isn't mapped into the userspace tables (then again, AMD
283283 * typically has NO_MELTDOWN).
284284 *
285− * While zen_untrain_ret() doesn't clobber anything but requires stack,
285+ * While retbleed_untrain_ret() doesn't clobber anything but requires stack,
286286 * entry_ibpb() will clobber AX, CX, DX.
287287 *
288288 * As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point
@@ -293,14 +293,20 @@ · 4 unmodified lines
293293 defined(CONFIG_CALL_DEPTH_TRACKING) || defined(CONFIG_CPU_SRSO)
294294 VALIDATE_UNRET_END
295295 ALTERNATIVE_3 "", \
296− CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \
296+ CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \
297297 "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \
298298 __stringify(RESET_CALL_DEPTH), X86_FEATURE_CALL_DEPTH
299299 #endif
300+.endm
300301
301−#ifdef CONFIG_CPU_SRSO
302− ALTERNATIVE_2 "", "call srso_untrain_ret", X86_FEATURE_SRSO, \
303− "call srso_untrain_ret_alias", X86_FEATURE_SRSO_ALIAS
302+.macro UNTRAIN_RET_VM
303+#if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \
304+ defined(CONFIG_CALL_DEPTH_TRACKING) || defined(CONFIG_CPU_SRSO)
305+ VALIDATE_UNRET_END
306+ ALTERNATIVE_3 "", \
307+ CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \
308+ "call entry_ibpb", X86_FEATURE_IBPB_ON_VMEXIT, \
309+ __stringify(RESET_CALL_DEPTH), X86_FEATURE_CALL_DEPTH
304310 #endif
305311 .endm
306312
@@ -309,15 +315,10 @@ · 2 unmodified lines
309315 defined(CONFIG_CALL_DEPTH_TRACKING)
310316 VALIDATE_UNRET_END
311317 ALTERNATIVE_3 "", \
312− CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \
318+ CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \
313319 "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \
314320 __stringify(RESET_CALL_DEPTH_FROM_CALL), X86_FEATURE_CALL_DEPTH
315321 #endif
316−
317−#ifdef CONFIG_CPU_SRSO
318− ALTERNATIVE_2 "", "call srso_untrain_ret", X86_FEATURE_SRSO, \
319− "call srso_untrain_ret_alias", X86_FEATURE_SRSO_ALIAS
320−#endif
321322 .endm
322323
323324
@@ -341,17 +342,24 @@ extern retpoline_thunk_t __x86_indirect_thunk_array[]; · 17 unmodified lines
341342 extern retpoline_thunk_t __x86_indirect_call_thunk_array[];
342343 extern retpoline_thunk_t __x86_indirect_jump_thunk_array[];
343344
345+#ifdef CONFIG_RETHUNK
344346 extern void __x86_return_thunk(void);
345−extern void zen_untrain_ret(void);
347+#else
348+static inline void __x86_return_thunk(void) {}
349+#endif
350+
351+extern void retbleed_return_thunk(void);
352+extern void srso_return_thunk(void);
353+extern void srso_alias_return_thunk(void);
354+
355+extern void retbleed_untrain_ret(void);
346356 extern void srso_untrain_ret(void);
347−extern void srso_untrain_ret_alias(void);
357+extern void srso_alias_untrain_ret(void);
358+
359+extern void entry_untrain_ret(void);
348360 extern void entry_ibpb(void);
349361
350−#ifdef CONFIG_CALL_THUNKS
351362 extern void (*x86_return_thunk)(void);
352−#else
353−#define x86_return_thunk (&__x86_return_thunk)
354−#endif
355363
356364 #ifdef CONFIG_CALL_DEPTH_TRACKING
357365 extern void __x86_return_skl(void);
@@ -478,9 +486,6 @@ enum ssb_mitigation { · 120 unmodified lines
478486 SPEC_STORE_BYPASS_SECCOMP,
479487 };
480488
481−extern char __indirect_thunk_start[];
482−extern char __indirect_thunk_end[];
483−
484489 static __always_inline
485490 void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature)
486491 {
arch/x86/kernel/alternative.c+0−4
@@ -687,10 +687,6 @@ void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) · 686 unmodified lines
687687
688688 #ifdef CONFIG_RETHUNK
689689
690−#ifdef CONFIG_CALL_THUNKS
691−void (*x86_return_thunk)(void) __ro_after_init = &__x86_return_thunk;
692−#endif
693−
694690 /*
695691 * Rewrite the compiler generated return thunk tail-calls.
696692 *
arch/x86/kernel/cpu/amd.c+1−0
@@ -1329,3 +1329,4 @@ void noinstr amd_clear_divider(void) · 1328 unmodified lines
13291329 asm volatile(ALTERNATIVE("", "div %2\n\t", X86_BUG_DIV0)
13301330 :: "a" (0), "d" (0), "r" (1));
13311331 }
1332+EXPORT_SYMBOL_GPL(amd_clear_divider);
arch/x86/kernel/cpu/bugs.c+23−4
@@ -63,6 +63,8 @@ EXPORT_SYMBOL_GPL(x86_pred_cmd); · 62 unmodified lines
6363
6464 static DEFINE_MUTEX(spec_ctrl_mutex);
6565
66+void (*x86_return_thunk)(void) __ro_after_init = &__x86_return_thunk;
67+
6668 /* Update SPEC_CTRL MSR and its cached copy unconditionally */
6769 static void update_spec_ctrl(u64 val)
6870 {
@@ -165,6 +167,11 @@ void __init cpu_select_mitigations(void) · 96 unmodified lines
165167 md_clear_select_mitigation();
166168 srbds_select_mitigation();
167169 l1d_flush_select_mitigation();
170+
171+ /*
172+ * srso_select_mitigation() depends and must run after
173+ * retbleed_select_mitigation().
174+ */
168175 srso_select_mitigation();
169176 gds_select_mitigation();
170177 }
@@ -1035,6 +1042,9 @@ do_cmd_auto: · 864 unmodified lines
10351042 setup_force_cpu_cap(X86_FEATURE_RETHUNK);
10361043 setup_force_cpu_cap(X86_FEATURE_UNRET);
10371044
1045+ if (IS_ENABLED(CONFIG_RETHUNK))
1046+ x86_return_thunk = retbleed_return_thunk;
1047+
10381048 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
10391049 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
10401050 pr_err(RETBLEED_UNTRAIN_MSG);
@@ -1044,6 +1054,7 @@ do_cmd_auto: · 3 unmodified lines
10441054
10451055 case RETBLEED_MITIGATION_IBPB:
10461056 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
1057+ setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
10471058 mitigate_smt = true;
10481059 break;
10491060
@@ -2417,9 +2428,10 @@ static void __init srso_select_mitigation(void) · 1367 unmodified lines
24172428 * Zen1/2 with SMT off aren't vulnerable after the right
24182429 * IBPB microcode has been applied.
24192430 */
2420− if ((boot_cpu_data.x86 < 0x19) &&
2421− (!cpu_smt_possible() || (cpu_smt_control == CPU_SMT_DISABLED)))
2431+ if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) {
24222432 setup_force_cpu_cap(X86_FEATURE_SRSO_NO);
2433+ return;
2434+ }
24232435 }
24242436
24252437 if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
@@ -2448,11 +2460,15 @@ static void __init srso_select_mitigation(void) · 22 unmodified lines
24482460 * like ftrace, static_call, etc.
24492461 */
24502462 setup_force_cpu_cap(X86_FEATURE_RETHUNK);
2463+ setup_force_cpu_cap(X86_FEATURE_UNRET);
24512464
2452− if (boot_cpu_data.x86 == 0x19)
2465+ if (boot_cpu_data.x86 == 0x19) {
24532466 setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS);
2454− else
2467+ x86_return_thunk = srso_alias_return_thunk;
2468+ } else {
24552469 setup_force_cpu_cap(X86_FEATURE_SRSO);
2470+ x86_return_thunk = srso_return_thunk;
2471+ }
24562472 srso_mitigation = SRSO_MITIGATION_SAFE_RET;
24572473 } else {
24582474 pr_err("WARNING: kernel not compiled with CPU_SRSO.\n");
@@ -2696,6 +2712,9 @@ static ssize_t retbleed_show_state(char *buf) · 237 unmodified lines
26962712
26972713 static ssize_t srso_show_state(char *buf)
26982714 {
2715+ if (boot_cpu_has(X86_FEATURE_SRSO_NO))
2716+ return sysfs_emit(buf, "Mitigation: SMT disabled\n");
2717+
26992718 return sysfs_emit(buf, "%s%s\n",
27002719 srso_strings[srso_mitigation],
27012720 (cpu_has_ibpb_brtype_microcode() ? "" : ", no microcode"));
arch/x86/kernel/kprobes/opt.c+16−24
@@ -226,7 +226,7 @@ static int copy_optimized_instructions(u8 *dest, u8 *src, u8 *real) · 225 unmodified lines
226226 }
227227
228228 /* Check whether insn is indirect jump */
229−static int __insn_is_indirect_jump(struct insn *insn)
229+static int insn_is_indirect_jump(struct insn *insn)
230230 {
231231 return ((insn->opcode.bytes[0] == 0xff &&
232232 (X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */
@@ -260,26 +260,6 @@ static int insn_jump_into_range(struct insn *insn, unsigned long start, int len) · 27 unmodified lines
260260 return (start <= target && target <= start + len);
261261 }
262262
263−static int insn_is_indirect_jump(struct insn *insn)
264−{
265− int ret = __insn_is_indirect_jump(insn);
266−
267−#ifdef CONFIG_RETPOLINE
268− /*
269− * Jump to x86_indirect_thunk_* is treated as an indirect jump.
270− * Note that even with CONFIG_RETPOLINE=y, the kernel compiled with
271− * older gcc may use indirect jump. So we add this check instead of
272− * replace indirect-jump check.
273− */
274− if (!ret)
275− ret = insn_jump_into_range(insn,
276− (unsigned long)__indirect_thunk_start,
277− (unsigned long)__indirect_thunk_end -
278− (unsigned long)__indirect_thunk_start);
279−#endif
280− return ret;
281−}
282−
283263 /* Decode whole function to ensure any instructions don't jump into target */
284264 static int can_optimize(unsigned long paddr)
285265 {
@@ -334,9 +314,21 @@ static int can_optimize(unsigned long paddr) · 48 unmodified lines
334314 /* Recover address */
335315 insn.kaddr = (void *)addr;
336316 insn.next_byte = (void *)(addr + insn.length);
337− /* Check any instructions don't jump into target */
338− if (insn_is_indirect_jump(&insn) ||
339− insn_jump_into_range(&insn, paddr + INT3_INSN_SIZE,
317+ /*
318+ * Check any instructions don't jump into target, indirectly or
319+ * directly.
320+ *
321+ * The indirect case is present to handle a code with jump
322+ * tables. When the kernel uses retpolines, the check should in
323+ * theory additionally look for jumps to indirect thunks.
324+ * However, the kernel built with retpolines or IBT has jump
325+ * tables disabled so the check can be skipped altogether.
326+ */
327+ if (!IS_ENABLED(CONFIG_RETPOLINE) &&
328+ !IS_ENABLED(CONFIG_X86_KERNEL_IBT) &&
329+ insn_is_indirect_jump(&insn))
330+ return 0;
331+ if (insn_jump_into_range(&insn, paddr + INT3_INSN_SIZE,
340332 DISP32_SIZE))
341333 return 0;
342334 addr += insn.length;
arch/x86/kernel/static_call.c+13−0
@@ -186,6 +186,19 @@ EXPORT_SYMBOL_GPL(arch_static_call_transform); · 185 unmodified lines
186186 */
187187 bool __static_call_fixup(void *tramp, u8 op, void *dest)
188188 {
189+ unsigned long addr = (unsigned long)tramp;
190+ /*
191+ * Not all .return_sites are a static_call trampoline (most are not).
192+ * Check if the 3 bytes after the return are still kernel text, if not,
193+ * then this definitely is not a trampoline and we need not worry
194+ * further.
195+ *
196+ * This avoids the memcmp() below tripping over pagefaults etc..
197+ */
198+ if (((addr >> PAGE_SHIFT) != ((addr + 7) >> PAGE_SHIFT)) &&
199+ !kernel_text_address(addr + 7))
200+ return false;
201+
189202 if (memcmp(tramp+5, tramp_ud, 3)) {
190203 /* Not a trampoline site, not our problem. */
191204 return false;
arch/x86/kernel/traps.c+0−2
@@ -206,8 +206,6 @@ DEFINE_IDTENTRY(exc_divide_error) · 205 unmodified lines
206206 {
207207 do_error_trap(regs, 0, "divide error", X86_TRAP_DE, SIGFPE,
208208 FPE_INTDIV, error_get_trap_addr(regs));
209−
210− amd_clear_divider();
211209 }
212210
213211 DEFINE_IDTENTRY(exc_overflow)
arch/x86/kernel/vmlinux.lds.S+9−11
@@ -133,27 +133,25 @@ SECTIONS · 132 unmodified lines
133133 KPROBES_TEXT
134134 SOFTIRQENTRY_TEXT
135135 #ifdef CONFIG_RETPOLINE
136− __indirect_thunk_start = .;
137− *(.text.__x86.indirect_thunk)
138− *(.text.__x86.return_thunk)
139− __indirect_thunk_end = .;
136+ *(.text..__x86.indirect_thunk)
137+ *(.text..__x86.return_thunk)
140138 #endif
141139 STATIC_CALL_TEXT
142140
143141 ALIGN_ENTRY_TEXT_BEGIN
144142 #ifdef CONFIG_CPU_SRSO
145− *(.text.__x86.rethunk_untrain)
143+ *(.text..__x86.rethunk_untrain)
146144 #endif
147145
148146 ENTRY_TEXT
149147
150148 #ifdef CONFIG_CPU_SRSO
151149 /*
152− * See the comment above srso_untrain_ret_alias()'s
150+ * See the comment above srso_alias_untrain_ret()'s
153151 * definition.
154152 */
155− . = srso_untrain_ret_alias | (1 << 2) | (1 << 8) | (1 << 14) | (1 << 20);
156− *(.text.__x86.rethunk_safe)
153+ . = srso_alias_untrain_ret | (1 << 2) | (1 << 8) | (1 << 14) | (1 << 20);
154+ *(.text..__x86.rethunk_safe)
157155 #endif
158156 ALIGN_ENTRY_TEXT_END
159157 *(.gnu.warning)
@@ -523,7 +521,7 @@ INIT_PER_CPU(irq_stack_backing_store); · 363 unmodified lines
523521 #endif
524522
525523 #ifdef CONFIG_RETHUNK
526−. = ASSERT((__ret & 0x3f) == 0, "__ret not cacheline-aligned");
524+. = ASSERT((retbleed_return_thunk & 0x3f) == 0, "retbleed_return_thunk not cacheline-aligned");
527525 . = ASSERT((srso_safe_ret & 0x3f) == 0, "srso_safe_ret not cacheline-aligned");
528526 #endif
529527
@@ -538,8 +536,8 @@ INIT_PER_CPU(irq_stack_backing_store); · 8 unmodified lines
538536 * Instead do: (A | B) - (A & B) in order to compute the XOR
539537 * of the two function addresses:
540538 */
541−. = ASSERT(((ABSOLUTE(srso_untrain_ret_alias) | srso_safe_ret_alias) -
542− (ABSOLUTE(srso_untrain_ret_alias) & srso_safe_ret_alias)) == ((1 << 2) | (1 << 8) | (1 << 14) | (1 << 20)),
539+. = ASSERT(((ABSOLUTE(srso_alias_untrain_ret) | srso_alias_safe_ret) -
540+ (ABSOLUTE(srso_alias_untrain_ret) & srso_alias_safe_ret)) == ((1 << 2) | (1 << 8) | (1 << 14) | (1 << 20)),
543541 "SRSO function pair won't alias");
544542 #endif
545543
arch/x86/kvm/svm/svm.c+2−0
@@ -4006,6 +4006,8 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_in · 4005 unmodified lines
40064006
40074007 guest_state_enter_irqoff();
40084008
4009+ amd_clear_divider();
4010+
40094011 if (sev_es_guest(vcpu->kvm))
40104012 __svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted);
40114013 else
arch/x86/kvm/svm/vmenter.S+2−5
@@ -222,10 +222,7 @@ SYM_FUNC_START(__svm_vcpu_run) · 221 unmodified lines
222222 * because interrupt handlers won't sanitize 'ret' if the return is
223223 * from the kernel.
224224 */
225− UNTRAIN_RET
226−
227− /* SRSO */
228− ALTERNATIVE "", "call entry_ibpb", X86_FEATURE_IBPB_ON_VMEXIT
225+ UNTRAIN_RET_VM
229226
230227 /*
231228 * Clear all general purpose registers except RSP and RAX to prevent
@@ -362,7 +359,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) · 130 unmodified lines
362359 * because interrupt handlers won't sanitize RET if the return is
363360 * from the kernel.
364361 */
365− UNTRAIN_RET
362+ UNTRAIN_RET_VM
366363
367364 /* "Pop" @spec_ctrl_intercepted. */
368365 pop %_ASM_BX
arch/x86/lib/retpoline.S+95−42
@@ -13,7 +13,7 @@ · 12 unmodified lines
1313 #include <asm/frame.h>
1414 #include <asm/nops.h>
1515
16− .section .text.__x86.indirect_thunk
16+ .section .text..__x86.indirect_thunk
1717
1818
1919 .macro POLINE reg
@@ -133,75 +133,106 @@ SYM_CODE_END(__x86_indirect_jump_thunk_array) · 113 unmodified lines
133133 #ifdef CONFIG_RETHUNK
134134
135135 /*
136− * srso_untrain_ret_alias() and srso_safe_ret_alias() are placed at
136+ * srso_alias_untrain_ret() and srso_alias_safe_ret() are placed at
137137 * special addresses:
138138 *
139− * - srso_untrain_ret_alias() is 2M aligned
140− * - srso_safe_ret_alias() is also in the same 2M page but bits 2, 8, 14
139+ * - srso_alias_untrain_ret() is 2M aligned
140+ * - srso_alias_safe_ret() is also in the same 2M page but bits 2, 8, 14
141141 * and 20 in its virtual address are set (while those bits in the
142− * srso_untrain_ret_alias() function are cleared).
142+ * srso_alias_untrain_ret() function are cleared).
143143 *
144144 * This guarantees that those two addresses will alias in the branch
145145 * target buffer of Zen3/4 generations, leading to any potential
146146 * poisoned entries at that BTB slot to get evicted.
147147 *
148− * As a result, srso_safe_ret_alias() becomes a safe return.
148+ * As a result, srso_alias_safe_ret() becomes a safe return.
149149 */
150150 #ifdef CONFIG_CPU_SRSO
151− .section .text.__x86.rethunk_untrain
151+ .section .text..__x86.rethunk_untrain
152152
153−SYM_START(srso_untrain_ret_alias, SYM_L_GLOBAL, SYM_A_NONE)
153+SYM_START(srso_alias_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE)
154+ UNWIND_HINT_FUNC
154155 ANNOTATE_NOENDBR
155156 ASM_NOP2
156157 lfence
157− jmp __x86_return_thunk
158−SYM_FUNC_END(srso_untrain_ret_alias)
159−__EXPORT_THUNK(srso_untrain_ret_alias)
160−
161− .section .text.__x86.rethunk_safe
158+ jmp srso_alias_return_thunk
159+SYM_FUNC_END(srso_alias_untrain_ret)
160+__EXPORT_THUNK(srso_alias_untrain_ret)
161+
162+ .section .text..__x86.rethunk_safe
163+#else
164+/* dummy definition for alternatives */
165+SYM_START(srso_alias_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE)
166+ ANNOTATE_UNRET_SAFE
167+ ret
168+ int3
169+SYM_FUNC_END(srso_alias_untrain_ret)
162170 #endif
163171
164−/* Needs a definition for the __x86_return_thunk alternative below. */
165−SYM_START(srso_safe_ret_alias, SYM_L_GLOBAL, SYM_A_NONE)
166−#ifdef CONFIG_CPU_SRSO
167− add $8, %_ASM_SP
172+SYM_START(srso_alias_safe_ret, SYM_L_GLOBAL, SYM_A_NONE)
173+ lea 8(%_ASM_SP), %_ASM_SP
168174 UNWIND_HINT_FUNC
169−#endif
170175 ANNOTATE_UNRET_SAFE
171176 ret
172177 int3
173−SYM_FUNC_END(srso_safe_ret_alias)
178+SYM_FUNC_END(srso_alias_safe_ret)
174179
175− .section .text.__x86.return_thunk
180+ .section .text..__x86.return_thunk
181+
182+SYM_CODE_START(srso_alias_return_thunk)
183+ UNWIND_HINT_FUNC
184+ ANNOTATE_NOENDBR
185+ call srso_alias_safe_ret
186+ ud2
187+SYM_CODE_END(srso_alias_return_thunk)
188+
189+/*
190+ * Some generic notes on the untraining sequences:
191+ *
192+ * They are interchangeable when it comes to flushing potentially wrong
193+ * RET predictions from the BTB.
194+ *
195+ * The SRSO Zen1/2 (MOVABS) untraining sequence is longer than the
196+ * Retbleed sequence because the return sequence done there
197+ * (srso_safe_ret()) is longer and the return sequence must fully nest
198+ * (end before) the untraining sequence. Therefore, the untraining
199+ * sequence must fully overlap the return sequence.
200+ *
201+ * Regarding alignment - the instructions which need to be untrained,
202+ * must all start at a cacheline boundary for Zen1/2 generations. That
203+ * is, instruction sequences starting at srso_safe_ret() and
204+ * the respective instruction sequences at retbleed_return_thunk()
205+ * must start at a cacheline boundary.
206+ */
176207
177208 /*
178209 * Safety details here pertain to the AMD Zen{1,2} microarchitecture:
179− * 1) The RET at __x86_return_thunk must be on a 64 byte boundary, for
210+ * 1) The RET at retbleed_return_thunk must be on a 64 byte boundary, for
180211 * alignment within the BTB.
181− * 2) The instruction at zen_untrain_ret must contain, and not
212+ * 2) The instruction at retbleed_untrain_ret must contain, and not
182213 * end with, the 0xc3 byte of the RET.
183214 * 3) STIBP must be enabled, or SMT disabled, to prevent the sibling thread
184215 * from re-poisioning the BTB prediction.
185216 */
186217 .align 64
187− .skip 64 - (__ret - zen_untrain_ret), 0xcc
188−SYM_START(zen_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE)
218+ .skip 64 - (retbleed_return_thunk - retbleed_untrain_ret), 0xcc
219+SYM_START(retbleed_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE)
189220 ANNOTATE_NOENDBR
190221 /*
191− * As executed from zen_untrain_ret, this is:
222+ * As executed from retbleed_untrain_ret, this is:
192223 *
193224 * TEST $0xcc, %bl
194225 * LFENCE
195− * JMP __x86_return_thunk
226+ * JMP retbleed_return_thunk
196227 *
197228 * Executing the TEST instruction has a side effect of evicting any BTB
198229 * prediction (potentially attacker controlled) attached to the RET, as
199− * __x86_return_thunk + 1 isn't an instruction boundary at the moment.
230+ * retbleed_return_thunk + 1 isn't an instruction boundary at the moment.
200231 */
201232 .byte 0xf6
202233
203234 /*
204− * As executed from __x86_return_thunk, this is a plain RET.
235+ * As executed from retbleed_return_thunk, this is a plain RET.
205236 *
206237 * As part of the TEST above, RET is the ModRM byte, and INT3 the imm8.
207238 *
@@ -213,13 +244,13 @@ SYM_START(zen_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) · 5 unmodified lines
213244 * With SMT enabled and STIBP active, a sibling thread cannot poison
214245 * RET's prediction to a type of its choice, but can evict the
215246 * prediction due to competitive sharing. If the prediction is
216− * evicted, __x86_return_thunk will suffer Straight Line Speculation
247+ * evicted, retbleed_return_thunk will suffer Straight Line Speculation
217248 * which will be contained safely by the INT3.
218249 */
219−SYM_INNER_LABEL(__ret, SYM_L_GLOBAL)
250+SYM_INNER_LABEL(retbleed_return_thunk, SYM_L_GLOBAL)
220251 ret
221252 int3
222−SYM_CODE_END(__ret)
253+SYM_CODE_END(retbleed_return_thunk)
223254
224255 /*
225256 * Ensure the TEST decoding / BTB invalidation is complete.
@@ -230,16 +261,16 @@ SYM_CODE_END(__ret) · 4 unmodified lines
230261 * Jump back and execute the RET in the middle of the TEST instruction.
231262 * INT3 is for SLS protection.
232263 */
233− jmp __ret
264+ jmp retbleed_return_thunk
234265 int3
235−SYM_FUNC_END(zen_untrain_ret)
236−__EXPORT_THUNK(zen_untrain_ret)
266+SYM_FUNC_END(retbleed_untrain_ret)
267+__EXPORT_THUNK(retbleed_untrain_ret)
237268
238269 /*
239− * SRSO untraining sequence for Zen1/2, similar to zen_untrain_ret()
270+ * SRSO untraining sequence for Zen1/2, similar to retbleed_untrain_ret()
240271 * above. On kernel entry, srso_untrain_ret() is executed which is a
241272 *
242− * movabs $0xccccccc308c48348,%rax
273+ * movabs $0xccccc30824648d48,%rax
243274 *
244275 * and when the return thunk executes the inner label srso_safe_ret()
245276 * later, it is a stack manipulation and a RET which is mispredicted and
@@ -251,22 +282,44 @@ SYM_START(srso_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) · 5 unmodified lines
251282 ANNOTATE_NOENDBR
252283 .byte 0x48, 0xb8
253284
285+/*
286+ * This forces the function return instruction to speculate into a trap
287+ * (UD2 in srso_return_thunk() below). This RET will then mispredict
288+ * and execution will continue at the return site read from the top of
289+ * the stack.
290+ */
254291 SYM_INNER_LABEL(srso_safe_ret, SYM_L_GLOBAL)
255− add $8, %_ASM_SP
292+ lea 8(%_ASM_SP), %_ASM_SP
256293 ret
257294 int3
258295 int3
259− int3
296+ /* end of movabs */
260297 lfence
261298 call srso_safe_ret
262− int3
299+ ud2
263300 SYM_CODE_END(srso_safe_ret)
264301 SYM_FUNC_END(srso_untrain_ret)
265302 __EXPORT_THUNK(srso_untrain_ret)
266303
267−SYM_FUNC_START(__x86_return_thunk)
268− ALTERNATIVE_2 "jmp __ret", "call srso_safe_ret", X86_FEATURE_SRSO, \
269− "call srso_safe_ret_alias", X86_FEATURE_SRSO_ALIAS
304+SYM_CODE_START(srso_return_thunk)
305+ UNWIND_HINT_FUNC
306+ ANNOTATE_NOENDBR
307+ call srso_safe_ret
308+ ud2
309+SYM_CODE_END(srso_return_thunk)
310+
311+SYM_FUNC_START(entry_untrain_ret)
312+ ALTERNATIVE_2 "jmp retbleed_untrain_ret", \
313+ "jmp srso_untrain_ret", X86_FEATURE_SRSO, \
314+ "jmp srso_alias_untrain_ret", X86_FEATURE_SRSO_ALIAS
315+SYM_FUNC_END(entry_untrain_ret)
316+__EXPORT_THUNK(entry_untrain_ret)
317+
318+SYM_CODE_START(__x86_return_thunk)
319+ UNWIND_HINT_FUNC
320+ ANNOTATE_NOENDBR
321+ ANNOTATE_UNRET_SAFE
322+ ret
270323 int3
271324 SYM_CODE_END(__x86_return_thunk)
272325 EXPORT_SYMBOL(__x86_return_thunk)
tools/objtool/arch/x86/decode.c+7−4
@@ -824,8 +824,11 @@ bool arch_is_retpoline(struct symbol *sym) · 823 unmodified lines
824824
825825 bool arch_is_rethunk(struct symbol *sym)
826826 {
827− return !strcmp(sym->name, "__x86_return_thunk") ||
828− !strcmp(sym->name, "srso_untrain_ret") ||
829− !strcmp(sym->name, "srso_safe_ret") ||
830− !strcmp(sym->name, "__ret");
827+ return !strcmp(sym->name, "__x86_return_thunk");
828+}
829+
830+bool arch_is_embedded_insn(struct symbol *sym)
831+{
832+ return !strcmp(sym->name, "retbleed_return_thunk") ||
833+ !strcmp(sym->name, "srso_safe_ret");
831834 }
tools/objtool/check.c+35−10
@@ -389,7 +389,7 @@ static int decode_instructions(struct objtool_file *file) · 388 unmodified lines
389389 if (!strcmp(sec->name, ".noinstr.text") ||
390390 !strcmp(sec->name, ".entry.text") ||
391391 !strcmp(sec->name, ".cpuidle.text") ||
392− !strncmp(sec->name, ".text.__x86.", 12))
392+ !strncmp(sec->name, ".text..__x86.", 13))
393393 sec->noinstr = true;
394394
395395 /*
@@ -455,7 +455,7 @@ static int decode_instructions(struct objtool_file *file) · 59 unmodified lines
455455 return -1;
456456 }
457457
458− if (func->return_thunk || func->alias != func)
458+ if (func->embedded_insn || func->alias != func)
459459 continue;
460460
461461 if (!find_insn(file, sec, func->offset)) {
@@ -1288,16 +1288,33 @@ static int add_ignore_alternatives(struct objtool_file *file) · 826 unmodified lines
12881288 return 0;
12891289 }
12901290
1291+/*
1292+ * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1293+ * will be added to the .retpoline_sites section.
1294+ */
12911295 __weak bool arch_is_retpoline(struct symbol *sym)
12921296 {
12931297 return false;
12941298 }
12951299
1300+/*
1301+ * Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1302+ * will be added to the .return_sites section.
1303+ */
12961304 __weak bool arch_is_rethunk(struct symbol *sym)
12971305 {
12981306 return false;
12991307 }
13001308
1309+/*
1310+ * Symbols that are embedded inside other instructions, because sometimes crazy
1311+ * code exists. These are mostly ignored for validation purposes.
1312+ */
1313+__weak bool arch_is_embedded_insn(struct symbol *sym)
1314+{
1315+ return false;
1316+}
1317+
13011318 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
13021319 {
13031320 struct reloc *reloc;
@@ -1576,14 +1593,14 @@ static int add_jump_destinations(struct objtool_file *file) · 272 unmodified lines
15761593 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
15771594
15781595 /*
1579− * This is a special case for zen_untrain_ret().
1596+ * This is a special case for retbleed_untrain_ret().
15801597 * It jumps to __x86_return_thunk(), but objtool
15811598 * can't find the thunk's starting RET
15821599 * instruction, because the RET is also in the
15831600 * middle of another instruction. Objtool only
15841601 * knows about the outer instruction.
15851602 */
1586− if (sym && sym->return_thunk) {
1603+ if (sym && sym->embedded_insn) {
15871604 add_return_call(file, insn, false);
15881605 continue;
15891606 }
@@ -2502,6 +2519,9 @@ static int classify_symbols(struct objtool_file *file) · 912 unmodified lines
25022519 if (arch_is_rethunk(func))
25032520 func->return_thunk = true;
25042521
2522+ if (arch_is_embedded_insn(func))
2523+ func->embedded_insn = true;
2524+
25052525 if (arch_ftrace_match(func->name))
25062526 func->fentry = true;
25072527
@@ -2630,12 +2650,17 @@ static int decode_sections(struct objtool_file *file) · 122 unmodified lines
26302650 return 0;
26312651 }
26322652
2633−static bool is_fentry_call(struct instruction *insn)
2653+static bool is_special_call(struct instruction *insn)
26342654 {
2635− if (insn->type == INSN_CALL &&
2636− insn_call_dest(insn) &&
2637− insn_call_dest(insn)->fentry)
2638− return true;
2655+ if (insn->type == INSN_CALL) {
2656+ struct symbol *dest = insn_call_dest(insn);
2657+
2658+ if (!dest)
2659+ return false;
2660+
2661+ if (dest->fentry || dest->embedded_insn)
2662+ return true;
2663+ }
26392664
26402665 return false;
26412666 }
@@ -3636,7 +3661,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, · 994 unmodified lines
36363661 if (ret)
36373662 return ret;
36383663
3639− if (opts.stackval && func && !is_fentry_call(insn) &&
3664+ if (opts.stackval && func && !is_special_call(insn) &&
36403665 !has_valid_stack_frame(&state)) {
36413666 WARN_INSN(insn, "call without frame pointer save/setup");
36423667 return 1;
tools/objtool/include/objtool/arch.h+1−0
@@ -90,6 +90,7 @@ int arch_decode_hint_reg(u8 sp_reg, int *base); · 89 unmodified lines
9090
9191 bool arch_is_retpoline(struct symbol *sym);
9292 bool arch_is_rethunk(struct symbol *sym);
93+bool arch_is_embedded_insn(struct symbol *sym);
9394
9495 int arch_rewrite_retpolines(struct objtool_file *file);
9596
tools/objtool/include/objtool/elf.h+1−0
@@ -66,6 +66,7 @@ struct symbol { · 65 unmodified lines
6666 u8 fentry : 1;
6767 u8 profiling_func : 1;
6868 u8 warned : 1;
69+ u8 embedded_insn : 1;
6970 struct list_head pv_target;
7071 struct reloc *relocs;
7172 };
tools/perf/util/thread-stack.c+1−3
@@ -1038,9 +1038,7 @@ static int thread_stack__trace_end(struct thread_stack *ts, · 1037 unmodified lines
10381038
10391039 static bool is_x86_retpoline(const char *name)
10401040 {
1041− const char *p = strstr(name, "__x86_indirect_thunk_");
1042−
1043− return p == name || !strcmp(name, "__indirect_thunk_start");
1041+ return strstr(name, "__x86_indirect_thunk_") == name;
10441042 }
10451043
10461044 /*

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t