Pull x86 fixes from Borislav Petkov: "Extraordinary embargoed times call for extraordinary measures. That's why this week's x86/urgent branch is larger than usual, containing all the known fallout fixes after the SRSO mitigation got merged. I know, it is a bit late in the game but everyone who has reported a bug stemming from the SRSO pile, has tested that branch and has confirmed that it fixes their bug. Also, I've run it on every possible hardware I have and it is looking good. It is running on this very machine while I'm typing, for 2 days now without an issue. Famous last words... - Use LEA ...%rsp instead of ADD %rsp in the Zen1/2 SRSO return sequence as latter clobbers flags which interferes with fastop emulation in KVM, leading to guests freezing during boot - A fix for the DIV(0) quotient data leak on Zen1 to clear the divider buffers at the right time - Disable the SRSO mitigation on unaffected configurations as it got enabled there unnecessarily - Change .text section name to fix CONFIG_LTO_CLANG builds - Improve the optprobe indirect jmp check so that certain configurations can still be able to use optprobes at all - A serious and good scrubbing of the untraining routines by PeterZ: - Add proper speculation stopping traps so that objtool is happy - Adjust objtool to handle the new thunks - Make the thunk pointer assignable to the different untraining sequences at runtime, thus avoiding the alternative at the return thunk. It simplifies the code a bit too. - Add a entry_untrain_ret() main entry point which selects the respective untraining sequence - Rename things so that they're more clear - Fix stack validation with FRAME_POINTER=y builds - Fix static call patching to handle when a JMP to the return thunk is the last insn on the very last module memory page - Add more documentation about what each untraining routine does and why" * tag 'x86_urgent_for_v6.5_rc7' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: x86/srso: Correct the mitigation status when SMT is disabled x86/static_call: Fix __static_call_fixup() objtool/x86: Fixup frame-pointer vs rethunk x86/srso: Explain the untraining sequences a bit more x86/cpu/kvm: Provide UNTRAIN_RET_VM x86/cpu: Cleanup the untrain mess x86/cpu: Rename srso_(.*)_alias to srso_alias_\1 x86/cpu: Rename original retbleed methods x86/cpu: Clean up SRSO return thunk mess x86/alternative: Make custom return thunk unconditional objtool/x86: Fix SRSO mess x86/cpu: Fix up srso_safe_ret() and __x86_return_thunk() x86/cpu: Fix __x86_return_thunk symbol type x86/retpoline,kprobes: Skip optprobe check for indirect jumps with retpolines and IBT x86/retpoline,kprobes: Fix position of thunk sections with CONFIG_LTO_CLANG x86/srso: Disable the mitigation on unaffected configurations x86/CPU/AMD: Fix the DIV(0) initial fix attempt x86/retpoline: Don't clobber RFLAGS during srso_safe_ret()
Unified · Split
| @@ -141,8 +141,8 @@ sequence. · 140 unmodified lines | ||
| 141 | 141 | To ensure the safety of this mitigation, the kernel must ensure that the |
| 142 | 142 | safe return sequence is itself free from attacker interference. In Zen3 |
| 143 | 143 | and Zen4, this is accomplished by creating a BTB alias between the |
| 144 | −untraining function srso_untrain_ret_alias() and the safe return | |
| 145 | −function srso_safe_ret_alias() which results in evicting a potentially | |
| 144 | +untraining function srso_alias_untrain_ret() and the safe return | |
| 145 | +function srso_alias_safe_ret() which results in evicting a potentially | |
| 146 | 146 | poisoned BTB entry and using that safe one for all function returns. |
| 147 | 147 | |
| 148 | 148 | In older Zen1 and Zen2, this is accomplished using a reinterpretation |
| @@ -92,6 +92,7 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, · 91 unmodified lines | ||
| 92 | 92 | static __always_inline void arch_exit_to_user_mode(void) |
| 93 | 93 | { |
| 94 | 94 | mds_user_clear_cpu_buffers(); |
| 95 | + amd_clear_divider(); | |
| 95 | 96 | } |
| 96 | 97 | #define arch_exit_to_user_mode arch_exit_to_user_mode |
| 97 | 98 | |
| @@ -272,9 +272,9 @@ · 271 unmodified lines | ||
| 272 | 272 | .endm |
| 273 | 273 | |
| 274 | 274 | #ifdef CONFIG_CPU_UNRET_ENTRY |
| 275 | −#define CALL_ZEN_UNTRAIN_RET "call zen_untrain_ret" | |
| 275 | +#define CALL_UNTRAIN_RET "call entry_untrain_ret" | |
| 276 | 276 | #else |
| 277 | −#define CALL_ZEN_UNTRAIN_RET "" | |
| 277 | +#define CALL_UNTRAIN_RET "" | |
| 278 | 278 | #endif |
| 279 | 279 | |
| 280 | 280 | /* |
| @@ -282,7 +282,7 @@ · 1 unmodified line | ||
| 282 | 282 | * return thunk isn't mapped into the userspace tables (then again, AMD |
| 283 | 283 | * typically has NO_MELTDOWN). |
| 284 | 284 | * |
| 285 | − * While zen_untrain_ret() doesn't clobber anything but requires stack, | |
| 285 | + * While retbleed_untrain_ret() doesn't clobber anything but requires stack, | |
| 286 | 286 | * entry_ibpb() will clobber AX, CX, DX. |
| 287 | 287 | * |
| 288 | 288 | * As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point |
| @@ -293,14 +293,20 @@ · 4 unmodified lines | ||
| 293 | 293 | defined(CONFIG_CALL_DEPTH_TRACKING) || defined(CONFIG_CPU_SRSO) |
| 294 | 294 | VALIDATE_UNRET_END |
| 295 | 295 | ALTERNATIVE_3 "", \ |
| 296 | − CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \ | |
| 296 | + CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \ | |
| 297 | 297 | "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \ |
| 298 | 298 | __stringify(RESET_CALL_DEPTH), X86_FEATURE_CALL_DEPTH |
| 299 | 299 | #endif |
| 300 | +.endm | |
| 300 | 301 | |
| 301 | −#ifdef CONFIG_CPU_SRSO | |
| 302 | − ALTERNATIVE_2 "", "call srso_untrain_ret", X86_FEATURE_SRSO, \ | |
| 303 | − "call srso_untrain_ret_alias", X86_FEATURE_SRSO_ALIAS | |
| 302 | +.macro UNTRAIN_RET_VM | |
| 303 | +#if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \ | |
| 304 | + defined(CONFIG_CALL_DEPTH_TRACKING) || defined(CONFIG_CPU_SRSO) | |
| 305 | + VALIDATE_UNRET_END | |
| 306 | + ALTERNATIVE_3 "", \ | |
| 307 | + CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \ | |
| 308 | + "call entry_ibpb", X86_FEATURE_IBPB_ON_VMEXIT, \ | |
| 309 | + __stringify(RESET_CALL_DEPTH), X86_FEATURE_CALL_DEPTH | |
| 304 | 310 | #endif |
| 305 | 311 | .endm |
| 306 | 312 | |
| @@ -309,15 +315,10 @@ · 2 unmodified lines | ||
| 309 | 315 | defined(CONFIG_CALL_DEPTH_TRACKING) |
| 310 | 316 | VALIDATE_UNRET_END |
| 311 | 317 | ALTERNATIVE_3 "", \ |
| 312 | − CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \ | |
| 318 | + CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \ | |
| 313 | 319 | "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \ |
| 314 | 320 | __stringify(RESET_CALL_DEPTH_FROM_CALL), X86_FEATURE_CALL_DEPTH |
| 315 | 321 | #endif |
| 316 | − | |
| 317 | −#ifdef CONFIG_CPU_SRSO | |
| 318 | − ALTERNATIVE_2 "", "call srso_untrain_ret", X86_FEATURE_SRSO, \ | |
| 319 | − "call srso_untrain_ret_alias", X86_FEATURE_SRSO_ALIAS | |
| 320 | −#endif | |
| 321 | 322 | .endm |
| 322 | 323 | |
| 323 | 324 | |
| @@ -341,17 +342,24 @@ extern retpoline_thunk_t __x86_indirect_thunk_array[]; · 17 unmodified lines | ||
| 341 | 342 | extern retpoline_thunk_t __x86_indirect_call_thunk_array[]; |
| 342 | 343 | extern retpoline_thunk_t __x86_indirect_jump_thunk_array[]; |
| 343 | 344 | |
| 345 | +#ifdef CONFIG_RETHUNK | |
| 344 | 346 | extern void __x86_return_thunk(void); |
| 345 | −extern void zen_untrain_ret(void); | |
| 347 | +#else | |
| 348 | +static inline void __x86_return_thunk(void) {} | |
| 349 | +#endif | |
| 350 | + | |
| 351 | +extern void retbleed_return_thunk(void); | |
| 352 | +extern void srso_return_thunk(void); | |
| 353 | +extern void srso_alias_return_thunk(void); | |
| 354 | + | |
| 355 | +extern void retbleed_untrain_ret(void); | |
| 346 | 356 | extern void srso_untrain_ret(void); |
| 347 | −extern void srso_untrain_ret_alias(void); | |
| 357 | +extern void srso_alias_untrain_ret(void); | |
| 358 | + | |
| 359 | +extern void entry_untrain_ret(void); | |
| 348 | 360 | extern void entry_ibpb(void); |
| 349 | 361 | |
| 350 | −#ifdef CONFIG_CALL_THUNKS | |
| 351 | 362 | extern void (*x86_return_thunk)(void); |
| 352 | −#else | |
| 353 | −#define x86_return_thunk (&__x86_return_thunk) | |
| 354 | −#endif | |
| 355 | 363 | |
| 356 | 364 | #ifdef CONFIG_CALL_DEPTH_TRACKING |
| 357 | 365 | extern void __x86_return_skl(void); |
| @@ -478,9 +486,6 @@ enum ssb_mitigation { · 120 unmodified lines | ||
| 478 | 486 | SPEC_STORE_BYPASS_SECCOMP, |
| 479 | 487 | }; |
| 480 | 488 | |
| 481 | −extern char __indirect_thunk_start[]; | |
| 482 | −extern char __indirect_thunk_end[]; | |
| 483 | − | |
| 484 | 489 | static __always_inline |
| 485 | 490 | void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature) |
| 486 | 491 | { |
| @@ -687,10 +687,6 @@ void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) · 686 unmodified lines | ||
| 687 | 687 | |
| 688 | 688 | #ifdef CONFIG_RETHUNK |
| 689 | 689 | |
| 690 | −#ifdef CONFIG_CALL_THUNKS | |
| 691 | −void (*x86_return_thunk)(void) __ro_after_init = &__x86_return_thunk; | |
| 692 | −#endif | |
| 693 | − | |
| 694 | 690 | /* |
| 695 | 691 | * Rewrite the compiler generated return thunk tail-calls. |
| 696 | 692 | * |
| @@ -1329,3 +1329,4 @@ void noinstr amd_clear_divider(void) · 1328 unmodified lines | ||
| 1329 | 1329 | asm volatile(ALTERNATIVE("", "div %2\n\t", X86_BUG_DIV0) |
| 1330 | 1330 | :: "a" (0), "d" (0), "r" (1)); |
| 1331 | 1331 | } |
| 1332 | +EXPORT_SYMBOL_GPL(amd_clear_divider); | |
| @@ -63,6 +63,8 @@ EXPORT_SYMBOL_GPL(x86_pred_cmd); · 62 unmodified lines | ||
| 63 | 63 | |
| 64 | 64 | static DEFINE_MUTEX(spec_ctrl_mutex); |
| 65 | 65 | |
| 66 | +void (*x86_return_thunk)(void) __ro_after_init = &__x86_return_thunk; | |
| 67 | + | |
| 66 | 68 | /* Update SPEC_CTRL MSR and its cached copy unconditionally */ |
| 67 | 69 | static void update_spec_ctrl(u64 val) |
| 68 | 70 | { |
| @@ -165,6 +167,11 @@ void __init cpu_select_mitigations(void) · 96 unmodified lines | ||
| 165 | 167 | md_clear_select_mitigation(); |
| 166 | 168 | srbds_select_mitigation(); |
| 167 | 169 | l1d_flush_select_mitigation(); |
| 170 | + | |
| 171 | + /* | |
| 172 | + * srso_select_mitigation() depends and must run after | |
| 173 | + * retbleed_select_mitigation(). | |
| 174 | + */ | |
| 168 | 175 | srso_select_mitigation(); |
| 169 | 176 | gds_select_mitigation(); |
| 170 | 177 | } |
| @@ -1035,6 +1042,9 @@ do_cmd_auto: · 864 unmodified lines | ||
| 1035 | 1042 | setup_force_cpu_cap(X86_FEATURE_RETHUNK); |
| 1036 | 1043 | setup_force_cpu_cap(X86_FEATURE_UNRET); |
| 1037 | 1044 | |
| 1045 | + if (IS_ENABLED(CONFIG_RETHUNK)) | |
| 1046 | + x86_return_thunk = retbleed_return_thunk; | |
| 1047 | + | |
| 1038 | 1048 | if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && |
| 1039 | 1049 | boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) |
| 1040 | 1050 | pr_err(RETBLEED_UNTRAIN_MSG); |
| @@ -1044,6 +1054,7 @@ do_cmd_auto: · 3 unmodified lines | ||
| 1044 | 1054 | |
| 1045 | 1055 | case RETBLEED_MITIGATION_IBPB: |
| 1046 | 1056 | setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); |
| 1057 | + setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); | |
| 1047 | 1058 | mitigate_smt = true; |
| 1048 | 1059 | break; |
| 1049 | 1060 | |
| @@ -2417,9 +2428,10 @@ static void __init srso_select_mitigation(void) · 1367 unmodified lines | ||
| 2417 | 2428 | * Zen1/2 with SMT off aren't vulnerable after the right |
| 2418 | 2429 | * IBPB microcode has been applied. |
| 2419 | 2430 | */ |
| 2420 | − if ((boot_cpu_data.x86 < 0x19) && | |
| 2421 | − (!cpu_smt_possible() || (cpu_smt_control == CPU_SMT_DISABLED))) | |
| 2431 | + if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) { | |
| 2422 | 2432 | setup_force_cpu_cap(X86_FEATURE_SRSO_NO); |
| 2433 | + return; | |
| 2434 | + } | |
| 2423 | 2435 | } |
| 2424 | 2436 | |
| 2425 | 2437 | if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { |
| @@ -2448,11 +2460,15 @@ static void __init srso_select_mitigation(void) · 22 unmodified lines | ||
| 2448 | 2460 | * like ftrace, static_call, etc. |
| 2449 | 2461 | */ |
| 2450 | 2462 | setup_force_cpu_cap(X86_FEATURE_RETHUNK); |
| 2463 | + setup_force_cpu_cap(X86_FEATURE_UNRET); | |
| 2451 | 2464 | |
| 2452 | − if (boot_cpu_data.x86 == 0x19) | |
| 2465 | + if (boot_cpu_data.x86 == 0x19) { | |
| 2453 | 2466 | setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS); |
| 2454 | − else | |
| 2467 | + x86_return_thunk = srso_alias_return_thunk; | |
| 2468 | + } else { | |
| 2455 | 2469 | setup_force_cpu_cap(X86_FEATURE_SRSO); |
| 2470 | + x86_return_thunk = srso_return_thunk; | |
| 2471 | + } | |
| 2456 | 2472 | srso_mitigation = SRSO_MITIGATION_SAFE_RET; |
| 2457 | 2473 | } else { |
| 2458 | 2474 | pr_err("WARNING: kernel not compiled with CPU_SRSO.\n"); |
| @@ -2696,6 +2712,9 @@ static ssize_t retbleed_show_state(char *buf) · 237 unmodified lines | ||
| 2696 | 2712 | |
| 2697 | 2713 | static ssize_t srso_show_state(char *buf) |
| 2698 | 2714 | { |
| 2715 | + if (boot_cpu_has(X86_FEATURE_SRSO_NO)) | |
| 2716 | + return sysfs_emit(buf, "Mitigation: SMT disabled\n"); | |
| 2717 | + | |
| 2699 | 2718 | return sysfs_emit(buf, "%s%s\n", |
| 2700 | 2719 | srso_strings[srso_mitigation], |
| 2701 | 2720 | (cpu_has_ibpb_brtype_microcode() ? "" : ", no microcode")); |
| @@ -226,7 +226,7 @@ static int copy_optimized_instructions(u8 *dest, u8 *src, u8 *real) · 225 unmodified lines | ||
| 226 | 226 | } |
| 227 | 227 | |
| 228 | 228 | /* Check whether insn is indirect jump */ |
| 229 | −static int __insn_is_indirect_jump(struct insn *insn) | |
| 229 | +static int insn_is_indirect_jump(struct insn *insn) | |
| 230 | 230 | { |
| 231 | 231 | return ((insn->opcode.bytes[0] == 0xff && |
| 232 | 232 | (X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */ |
| @@ -260,26 +260,6 @@ static int insn_jump_into_range(struct insn *insn, unsigned long start, int len) · 27 unmodified lines | ||
| 260 | 260 | return (start <= target && target <= start + len); |
| 261 | 261 | } |
| 262 | 262 | |
| 263 | −static int insn_is_indirect_jump(struct insn *insn) | |
| 264 | −{ | |
| 265 | − int ret = __insn_is_indirect_jump(insn); | |
| 266 | − | |
| 267 | −#ifdef CONFIG_RETPOLINE | |
| 268 | − /* | |
| 269 | − * Jump to x86_indirect_thunk_* is treated as an indirect jump. | |
| 270 | − * Note that even with CONFIG_RETPOLINE=y, the kernel compiled with | |
| 271 | − * older gcc may use indirect jump. So we add this check instead of | |
| 272 | − * replace indirect-jump check. | |
| 273 | − */ | |
| 274 | − if (!ret) | |
| 275 | − ret = insn_jump_into_range(insn, | |
| 276 | − (unsigned long)__indirect_thunk_start, | |
| 277 | − (unsigned long)__indirect_thunk_end - | |
| 278 | − (unsigned long)__indirect_thunk_start); | |
| 279 | −#endif | |
| 280 | − return ret; | |
| 281 | −} | |
| 282 | − | |
| 283 | 263 | /* Decode whole function to ensure any instructions don't jump into target */ |
| 284 | 264 | static int can_optimize(unsigned long paddr) |
| 285 | 265 | { |
| @@ -334,9 +314,21 @@ static int can_optimize(unsigned long paddr) · 48 unmodified lines | ||
| 334 | 314 | /* Recover address */ |
| 335 | 315 | insn.kaddr = (void *)addr; |
| 336 | 316 | insn.next_byte = (void *)(addr + insn.length); |
| 337 | − /* Check any instructions don't jump into target */ | |
| 338 | − if (insn_is_indirect_jump(&insn) || | |
| 339 | − insn_jump_into_range(&insn, paddr + INT3_INSN_SIZE, | |
| 317 | + /* | |
| 318 | + * Check any instructions don't jump into target, indirectly or | |
| 319 | + * directly. | |
| 320 | + * | |
| 321 | + * The indirect case is present to handle a code with jump | |
| 322 | + * tables. When the kernel uses retpolines, the check should in | |
| 323 | + * theory additionally look for jumps to indirect thunks. | |
| 324 | + * However, the kernel built with retpolines or IBT has jump | |
| 325 | + * tables disabled so the check can be skipped altogether. | |
| 326 | + */ | |
| 327 | + if (!IS_ENABLED(CONFIG_RETPOLINE) && | |
| 328 | + !IS_ENABLED(CONFIG_X86_KERNEL_IBT) && | |
| 329 | + insn_is_indirect_jump(&insn)) | |
| 330 | + return 0; | |
| 331 | + if (insn_jump_into_range(&insn, paddr + INT3_INSN_SIZE, | |
| 340 | 332 | DISP32_SIZE)) |
| 341 | 333 | return 0; |
| 342 | 334 | addr += insn.length; |
| @@ -186,6 +186,19 @@ EXPORT_SYMBOL_GPL(arch_static_call_transform); · 185 unmodified lines | ||
| 186 | 186 | */ |
| 187 | 187 | bool __static_call_fixup(void *tramp, u8 op, void *dest) |
| 188 | 188 | { |
| 189 | + unsigned long addr = (unsigned long)tramp; | |
| 190 | + /* | |
| 191 | + * Not all .return_sites are a static_call trampoline (most are not). | |
| 192 | + * Check if the 3 bytes after the return are still kernel text, if not, | |
| 193 | + * then this definitely is not a trampoline and we need not worry | |
| 194 | + * further. | |
| 195 | + * | |
| 196 | + * This avoids the memcmp() below tripping over pagefaults etc.. | |
| 197 | + */ | |
| 198 | + if (((addr >> PAGE_SHIFT) != ((addr + 7) >> PAGE_SHIFT)) && | |
| 199 | + !kernel_text_address(addr + 7)) | |
| 200 | + return false; | |
| 201 | + | |
| 189 | 202 | if (memcmp(tramp+5, tramp_ud, 3)) { |
| 190 | 203 | /* Not a trampoline site, not our problem. */ |
| 191 | 204 | return false; |
| @@ -206,8 +206,6 @@ DEFINE_IDTENTRY(exc_divide_error) · 205 unmodified lines | ||
| 206 | 206 | { |
| 207 | 207 | do_error_trap(regs, 0, "divide error", X86_TRAP_DE, SIGFPE, |
| 208 | 208 | FPE_INTDIV, error_get_trap_addr(regs)); |
| 209 | − | |
| 210 | − amd_clear_divider(); | |
| 211 | 209 | } |
| 212 | 210 | |
| 213 | 211 | DEFINE_IDTENTRY(exc_overflow) |
| @@ -133,27 +133,25 @@ SECTIONS · 132 unmodified lines | ||
| 133 | 133 | KPROBES_TEXT |
| 134 | 134 | SOFTIRQENTRY_TEXT |
| 135 | 135 | #ifdef CONFIG_RETPOLINE |
| 136 | − __indirect_thunk_start = .; | |
| 137 | − *(.text.__x86.indirect_thunk) | |
| 138 | − *(.text.__x86.return_thunk) | |
| 139 | − __indirect_thunk_end = .; | |
| 136 | + *(.text..__x86.indirect_thunk) | |
| 137 | + *(.text..__x86.return_thunk) | |
| 140 | 138 | #endif |
| 141 | 139 | STATIC_CALL_TEXT |
| 142 | 140 | |
| 143 | 141 | ALIGN_ENTRY_TEXT_BEGIN |
| 144 | 142 | #ifdef CONFIG_CPU_SRSO |
| 145 | − *(.text.__x86.rethunk_untrain) | |
| 143 | + *(.text..__x86.rethunk_untrain) | |
| 146 | 144 | #endif |
| 147 | 145 | |
| 148 | 146 | ENTRY_TEXT |
| 149 | 147 | |
| 150 | 148 | #ifdef CONFIG_CPU_SRSO |
| 151 | 149 | /* |
| 152 | − * See the comment above srso_untrain_ret_alias()'s | |
| 150 | + * See the comment above srso_alias_untrain_ret()'s | |
| 153 | 151 | * definition. |
| 154 | 152 | */ |
| 155 | − . = srso_untrain_ret_alias | (1 << 2) | (1 << 8) | (1 << 14) | (1 << 20); | |
| 156 | − *(.text.__x86.rethunk_safe) | |
| 153 | + . = srso_alias_untrain_ret | (1 << 2) | (1 << 8) | (1 << 14) | (1 << 20); | |
| 154 | + *(.text..__x86.rethunk_safe) | |
| 157 | 155 | #endif |
| 158 | 156 | ALIGN_ENTRY_TEXT_END |
| 159 | 157 | *(.gnu.warning) |
| @@ -523,7 +521,7 @@ INIT_PER_CPU(irq_stack_backing_store); · 363 unmodified lines | ||
| 523 | 521 | #endif |
| 524 | 522 | |
| 525 | 523 | #ifdef CONFIG_RETHUNK |
| 526 | −. = ASSERT((__ret & 0x3f) == 0, "__ret not cacheline-aligned"); | |
| 524 | +. = ASSERT((retbleed_return_thunk & 0x3f) == 0, "retbleed_return_thunk not cacheline-aligned"); | |
| 527 | 525 | . = ASSERT((srso_safe_ret & 0x3f) == 0, "srso_safe_ret not cacheline-aligned"); |
| 528 | 526 | #endif |
| 529 | 527 | |
| @@ -538,8 +536,8 @@ INIT_PER_CPU(irq_stack_backing_store); · 8 unmodified lines | ||
| 538 | 536 | * Instead do: (A | B) - (A & B) in order to compute the XOR |
| 539 | 537 | * of the two function addresses: |
| 540 | 538 | */ |
| 541 | −. = ASSERT(((ABSOLUTE(srso_untrain_ret_alias) | srso_safe_ret_alias) - | |
| 542 | − (ABSOLUTE(srso_untrain_ret_alias) & srso_safe_ret_alias)) == ((1 << 2) | (1 << 8) | (1 << 14) | (1 << 20)), | |
| 539 | +. = ASSERT(((ABSOLUTE(srso_alias_untrain_ret) | srso_alias_safe_ret) - | |
| 540 | + (ABSOLUTE(srso_alias_untrain_ret) & srso_alias_safe_ret)) == ((1 << 2) | (1 << 8) | (1 << 14) | (1 << 20)), | |
| 543 | 541 | "SRSO function pair won't alias"); |
| 544 | 542 | #endif |
| 545 | 543 | |
| @@ -4006,6 +4006,8 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_in · 4005 unmodified lines | ||
| 4006 | 4006 | |
| 4007 | 4007 | guest_state_enter_irqoff(); |
| 4008 | 4008 | |
| 4009 | + amd_clear_divider(); | |
| 4010 | + | |
| 4009 | 4011 | if (sev_es_guest(vcpu->kvm)) |
| 4010 | 4012 | __svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted); |
| 4011 | 4013 | else |
| @@ -222,10 +222,7 @@ SYM_FUNC_START(__svm_vcpu_run) · 221 unmodified lines | ||
| 222 | 222 | * because interrupt handlers won't sanitize 'ret' if the return is |
| 223 | 223 | * from the kernel. |
| 224 | 224 | */ |
| 225 | − UNTRAIN_RET | |
| 226 | − | |
| 227 | − /* SRSO */ | |
| 228 | − ALTERNATIVE "", "call entry_ibpb", X86_FEATURE_IBPB_ON_VMEXIT | |
| 225 | + UNTRAIN_RET_VM | |
| 229 | 226 | |
| 230 | 227 | /* |
| 231 | 228 | * Clear all general purpose registers except RSP and RAX to prevent |
| @@ -362,7 +359,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) · 130 unmodified lines | ||
| 362 | 359 | * because interrupt handlers won't sanitize RET if the return is |
| 363 | 360 | * from the kernel. |
| 364 | 361 | */ |
| 365 | − UNTRAIN_RET | |
| 362 | + UNTRAIN_RET_VM | |
| 366 | 363 | |
| 367 | 364 | /* "Pop" @spec_ctrl_intercepted. */ |
| 368 | 365 | pop %_ASM_BX |
| @@ -13,7 +13,7 @@ · 12 unmodified lines | ||
| 13 | 13 | #include <asm/frame.h> |
| 14 | 14 | #include <asm/nops.h> |
| 15 | 15 | |
| 16 | − .section .text.__x86.indirect_thunk | |
| 16 | + .section .text..__x86.indirect_thunk | |
| 17 | 17 | |
| 18 | 18 | |
| 19 | 19 | .macro POLINE reg |
| @@ -133,75 +133,106 @@ SYM_CODE_END(__x86_indirect_jump_thunk_array) · 113 unmodified lines | ||
| 133 | 133 | #ifdef CONFIG_RETHUNK |
| 134 | 134 | |
| 135 | 135 | /* |
| 136 | − * srso_untrain_ret_alias() and srso_safe_ret_alias() are placed at | |
| 136 | + * srso_alias_untrain_ret() and srso_alias_safe_ret() are placed at | |
| 137 | 137 | * special addresses: |
| 138 | 138 | * |
| 139 | − * - srso_untrain_ret_alias() is 2M aligned | |
| 140 | − * - srso_safe_ret_alias() is also in the same 2M page but bits 2, 8, 14 | |
| 139 | + * - srso_alias_untrain_ret() is 2M aligned | |
| 140 | + * - srso_alias_safe_ret() is also in the same 2M page but bits 2, 8, 14 | |
| 141 | 141 | * and 20 in its virtual address are set (while those bits in the |
| 142 | − * srso_untrain_ret_alias() function are cleared). | |
| 142 | + * srso_alias_untrain_ret() function are cleared). | |
| 143 | 143 | * |
| 144 | 144 | * This guarantees that those two addresses will alias in the branch |
| 145 | 145 | * target buffer of Zen3/4 generations, leading to any potential |
| 146 | 146 | * poisoned entries at that BTB slot to get evicted. |
| 147 | 147 | * |
| 148 | − * As a result, srso_safe_ret_alias() becomes a safe return. | |
| 148 | + * As a result, srso_alias_safe_ret() becomes a safe return. | |
| 149 | 149 | */ |
| 150 | 150 | #ifdef CONFIG_CPU_SRSO |
| 151 | − .section .text.__x86.rethunk_untrain | |
| 151 | + .section .text..__x86.rethunk_untrain | |
| 152 | 152 | |
| 153 | −SYM_START(srso_untrain_ret_alias, SYM_L_GLOBAL, SYM_A_NONE) | |
| 153 | +SYM_START(srso_alias_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) | |
| 154 | + UNWIND_HINT_FUNC | |
| 154 | 155 | ANNOTATE_NOENDBR |
| 155 | 156 | ASM_NOP2 |
| 156 | 157 | lfence |
| 157 | − jmp __x86_return_thunk | |
| 158 | −SYM_FUNC_END(srso_untrain_ret_alias) | |
| 159 | −__EXPORT_THUNK(srso_untrain_ret_alias) | |
| 160 | − | |
| 161 | − .section .text.__x86.rethunk_safe | |
| 158 | + jmp srso_alias_return_thunk | |
| 159 | +SYM_FUNC_END(srso_alias_untrain_ret) | |
| 160 | +__EXPORT_THUNK(srso_alias_untrain_ret) | |
| 161 | + | |
| 162 | + .section .text..__x86.rethunk_safe | |
| 163 | +#else | |
| 164 | +/* dummy definition for alternatives */ | |
| 165 | +SYM_START(srso_alias_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) | |
| 166 | + ANNOTATE_UNRET_SAFE | |
| 167 | + ret | |
| 168 | + int3 | |
| 169 | +SYM_FUNC_END(srso_alias_untrain_ret) | |
| 162 | 170 | #endif |
| 163 | 171 | |
| 164 | −/* Needs a definition for the __x86_return_thunk alternative below. */ | |
| 165 | −SYM_START(srso_safe_ret_alias, SYM_L_GLOBAL, SYM_A_NONE) | |
| 166 | −#ifdef CONFIG_CPU_SRSO | |
| 167 | − add $8, %_ASM_SP | |
| 172 | +SYM_START(srso_alias_safe_ret, SYM_L_GLOBAL, SYM_A_NONE) | |
| 173 | + lea 8(%_ASM_SP), %_ASM_SP | |
| 168 | 174 | UNWIND_HINT_FUNC |
| 169 | −#endif | |
| 170 | 175 | ANNOTATE_UNRET_SAFE |
| 171 | 176 | ret |
| 172 | 177 | int3 |
| 173 | −SYM_FUNC_END(srso_safe_ret_alias) | |
| 178 | +SYM_FUNC_END(srso_alias_safe_ret) | |
| 174 | 179 | |
| 175 | − .section .text.__x86.return_thunk | |
| 180 | + .section .text..__x86.return_thunk | |
| 181 | + | |
| 182 | +SYM_CODE_START(srso_alias_return_thunk) | |
| 183 | + UNWIND_HINT_FUNC | |
| 184 | + ANNOTATE_NOENDBR | |
| 185 | + call srso_alias_safe_ret | |
| 186 | + ud2 | |
| 187 | +SYM_CODE_END(srso_alias_return_thunk) | |
| 188 | + | |
| 189 | +/* | |
| 190 | + * Some generic notes on the untraining sequences: | |
| 191 | + * | |
| 192 | + * They are interchangeable when it comes to flushing potentially wrong | |
| 193 | + * RET predictions from the BTB. | |
| 194 | + * | |
| 195 | + * The SRSO Zen1/2 (MOVABS) untraining sequence is longer than the | |
| 196 | + * Retbleed sequence because the return sequence done there | |
| 197 | + * (srso_safe_ret()) is longer and the return sequence must fully nest | |
| 198 | + * (end before) the untraining sequence. Therefore, the untraining | |
| 199 | + * sequence must fully overlap the return sequence. | |
| 200 | + * | |
| 201 | + * Regarding alignment - the instructions which need to be untrained, | |
| 202 | + * must all start at a cacheline boundary for Zen1/2 generations. That | |
| 203 | + * is, instruction sequences starting at srso_safe_ret() and | |
| 204 | + * the respective instruction sequences at retbleed_return_thunk() | |
| 205 | + * must start at a cacheline boundary. | |
| 206 | + */ | |
| 176 | 207 | |
| 177 | 208 | /* |
| 178 | 209 | * Safety details here pertain to the AMD Zen{1,2} microarchitecture: |
| 179 | − * 1) The RET at __x86_return_thunk must be on a 64 byte boundary, for | |
| 210 | + * 1) The RET at retbleed_return_thunk must be on a 64 byte boundary, for | |
| 180 | 211 | * alignment within the BTB. |
| 181 | − * 2) The instruction at zen_untrain_ret must contain, and not | |
| 212 | + * 2) The instruction at retbleed_untrain_ret must contain, and not | |
| 182 | 213 | * end with, the 0xc3 byte of the RET. |
| 183 | 214 | * 3) STIBP must be enabled, or SMT disabled, to prevent the sibling thread |
| 184 | 215 | * from re-poisioning the BTB prediction. |
| 185 | 216 | */ |
| 186 | 217 | .align 64 |
| 187 | − .skip 64 - (__ret - zen_untrain_ret), 0xcc | |
| 188 | −SYM_START(zen_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) | |
| 218 | + .skip 64 - (retbleed_return_thunk - retbleed_untrain_ret), 0xcc | |
| 219 | +SYM_START(retbleed_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) | |
| 189 | 220 | ANNOTATE_NOENDBR |
| 190 | 221 | /* |
| 191 | − * As executed from zen_untrain_ret, this is: | |
| 222 | + * As executed from retbleed_untrain_ret, this is: | |
| 192 | 223 | * |
| 193 | 224 | * TEST $0xcc, %bl |
| 194 | 225 | * LFENCE |
| 195 | − * JMP __x86_return_thunk | |
| 226 | + * JMP retbleed_return_thunk | |
| 196 | 227 | * |
| 197 | 228 | * Executing the TEST instruction has a side effect of evicting any BTB |
| 198 | 229 | * prediction (potentially attacker controlled) attached to the RET, as |
| 199 | − * __x86_return_thunk + 1 isn't an instruction boundary at the moment. | |
| 230 | + * retbleed_return_thunk + 1 isn't an instruction boundary at the moment. | |
| 200 | 231 | */ |
| 201 | 232 | .byte 0xf6 |
| 202 | 233 | |
| 203 | 234 | /* |
| 204 | − * As executed from __x86_return_thunk, this is a plain RET. | |
| 235 | + * As executed from retbleed_return_thunk, this is a plain RET. | |
| 205 | 236 | * |
| 206 | 237 | * As part of the TEST above, RET is the ModRM byte, and INT3 the imm8. |
| 207 | 238 | * |
| @@ -213,13 +244,13 @@ SYM_START(zen_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) · 5 unmodified lines | ||
| 213 | 244 | * With SMT enabled and STIBP active, a sibling thread cannot poison |
| 214 | 245 | * RET's prediction to a type of its choice, but can evict the |
| 215 | 246 | * prediction due to competitive sharing. If the prediction is |
| 216 | − * evicted, __x86_return_thunk will suffer Straight Line Speculation | |
| 247 | + * evicted, retbleed_return_thunk will suffer Straight Line Speculation | |
| 217 | 248 | * which will be contained safely by the INT3. |
| 218 | 249 | */ |
| 219 | −SYM_INNER_LABEL(__ret, SYM_L_GLOBAL) | |
| 250 | +SYM_INNER_LABEL(retbleed_return_thunk, SYM_L_GLOBAL) | |
| 220 | 251 | ret |
| 221 | 252 | int3 |
| 222 | −SYM_CODE_END(__ret) | |
| 253 | +SYM_CODE_END(retbleed_return_thunk) | |
| 223 | 254 | |
| 224 | 255 | /* |
| 225 | 256 | * Ensure the TEST decoding / BTB invalidation is complete. |
| @@ -230,16 +261,16 @@ SYM_CODE_END(__ret) · 4 unmodified lines | ||
| 230 | 261 | * Jump back and execute the RET in the middle of the TEST instruction. |
| 231 | 262 | * INT3 is for SLS protection. |
| 232 | 263 | */ |
| 233 | − jmp __ret | |
| 264 | + jmp retbleed_return_thunk | |
| 234 | 265 | int3 |
| 235 | −SYM_FUNC_END(zen_untrain_ret) | |
| 236 | −__EXPORT_THUNK(zen_untrain_ret) | |
| 266 | +SYM_FUNC_END(retbleed_untrain_ret) | |
| 267 | +__EXPORT_THUNK(retbleed_untrain_ret) | |
| 237 | 268 | |
| 238 | 269 | /* |
| 239 | − * SRSO untraining sequence for Zen1/2, similar to zen_untrain_ret() | |
| 270 | + * SRSO untraining sequence for Zen1/2, similar to retbleed_untrain_ret() | |
| 240 | 271 | * above. On kernel entry, srso_untrain_ret() is executed which is a |
| 241 | 272 | * |
| 242 | − * movabs $0xccccccc308c48348,%rax | |
| 273 | + * movabs $0xccccc30824648d48,%rax | |
| 243 | 274 | * |
| 244 | 275 | * and when the return thunk executes the inner label srso_safe_ret() |
| 245 | 276 | * later, it is a stack manipulation and a RET which is mispredicted and |
| @@ -251,22 +282,44 @@ SYM_START(srso_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) · 5 unmodified lines | ||
| 251 | 282 | ANNOTATE_NOENDBR |
| 252 | 283 | .byte 0x48, 0xb8 |
| 253 | 284 | |
| 285 | +/* | |
| 286 | + * This forces the function return instruction to speculate into a trap | |
| 287 | + * (UD2 in srso_return_thunk() below). This RET will then mispredict | |
| 288 | + * and execution will continue at the return site read from the top of | |
| 289 | + * the stack. | |
| 290 | + */ | |
| 254 | 291 | SYM_INNER_LABEL(srso_safe_ret, SYM_L_GLOBAL) |
| 255 | − add $8, %_ASM_SP | |
| 292 | + lea 8(%_ASM_SP), %_ASM_SP | |
| 256 | 293 | ret |
| 257 | 294 | int3 |
| 258 | 295 | int3 |
| 259 | − int3 | |
| 296 | + /* end of movabs */ | |
| 260 | 297 | lfence |
| 261 | 298 | call srso_safe_ret |
| 262 | − int3 | |
| 299 | + ud2 | |
| 263 | 300 | SYM_CODE_END(srso_safe_ret) |
| 264 | 301 | SYM_FUNC_END(srso_untrain_ret) |
| 265 | 302 | __EXPORT_THUNK(srso_untrain_ret) |
| 266 | 303 | |
| 267 | −SYM_FUNC_START(__x86_return_thunk) | |
| 268 | − ALTERNATIVE_2 "jmp __ret", "call srso_safe_ret", X86_FEATURE_SRSO, \ | |
| 269 | − "call srso_safe_ret_alias", X86_FEATURE_SRSO_ALIAS | |
| 304 | +SYM_CODE_START(srso_return_thunk) | |
| 305 | + UNWIND_HINT_FUNC | |
| 306 | + ANNOTATE_NOENDBR | |
| 307 | + call srso_safe_ret | |
| 308 | + ud2 | |
| 309 | +SYM_CODE_END(srso_return_thunk) | |
| 310 | + | |
| 311 | +SYM_FUNC_START(entry_untrain_ret) | |
| 312 | + ALTERNATIVE_2 "jmp retbleed_untrain_ret", \ | |
| 313 | + "jmp srso_untrain_ret", X86_FEATURE_SRSO, \ | |
| 314 | + "jmp srso_alias_untrain_ret", X86_FEATURE_SRSO_ALIAS | |
| 315 | +SYM_FUNC_END(entry_untrain_ret) | |
| 316 | +__EXPORT_THUNK(entry_untrain_ret) | |
| 317 | + | |
| 318 | +SYM_CODE_START(__x86_return_thunk) | |
| 319 | + UNWIND_HINT_FUNC | |
| 320 | + ANNOTATE_NOENDBR | |
| 321 | + ANNOTATE_UNRET_SAFE | |
| 322 | + ret | |
| 270 | 323 | int3 |
| 271 | 324 | SYM_CODE_END(__x86_return_thunk) |
| 272 | 325 | EXPORT_SYMBOL(__x86_return_thunk) |
| @@ -824,8 +824,11 @@ bool arch_is_retpoline(struct symbol *sym) · 823 unmodified lines | ||
| 824 | 824 | |
| 825 | 825 | bool arch_is_rethunk(struct symbol *sym) |
| 826 | 826 | { |
| 827 | − return !strcmp(sym->name, "__x86_return_thunk") || | |
| 828 | − !strcmp(sym->name, "srso_untrain_ret") || | |
| 829 | − !strcmp(sym->name, "srso_safe_ret") || | |
| 830 | − !strcmp(sym->name, "__ret"); | |
| 827 | + return !strcmp(sym->name, "__x86_return_thunk"); | |
| 828 | +} | |
| 829 | + | |
| 830 | +bool arch_is_embedded_insn(struct symbol *sym) | |
| 831 | +{ | |
| 832 | + return !strcmp(sym->name, "retbleed_return_thunk") || | |
| 833 | + !strcmp(sym->name, "srso_safe_ret"); | |
| 831 | 834 | } |
| @@ -389,7 +389,7 @@ static int decode_instructions(struct objtool_file *file) · 388 unmodified lines | ||
| 389 | 389 | if (!strcmp(sec->name, ".noinstr.text") || |
| 390 | 390 | !strcmp(sec->name, ".entry.text") || |
| 391 | 391 | !strcmp(sec->name, ".cpuidle.text") || |
| 392 | − !strncmp(sec->name, ".text.__x86.", 12)) | |
| 392 | + !strncmp(sec->name, ".text..__x86.", 13)) | |
| 393 | 393 | sec->noinstr = true; |
| 394 | 394 | |
| 395 | 395 | /* |
| @@ -455,7 +455,7 @@ static int decode_instructions(struct objtool_file *file) · 59 unmodified lines | ||
| 455 | 455 | return -1; |
| 456 | 456 | } |
| 457 | 457 | |
| 458 | − if (func->return_thunk || func->alias != func) | |
| 458 | + if (func->embedded_insn || func->alias != func) | |
| 459 | 459 | continue; |
| 460 | 460 | |
| 461 | 461 | if (!find_insn(file, sec, func->offset)) { |
| @@ -1288,16 +1288,33 @@ static int add_ignore_alternatives(struct objtool_file *file) · 826 unmodified lines | ||
| 1288 | 1288 | return 0; |
| 1289 | 1289 | } |
| 1290 | 1290 | |
| 1291 | +/* | |
| 1292 | + * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol | |
| 1293 | + * will be added to the .retpoline_sites section. | |
| 1294 | + */ | |
| 1291 | 1295 | __weak bool arch_is_retpoline(struct symbol *sym) |
| 1292 | 1296 | { |
| 1293 | 1297 | return false; |
| 1294 | 1298 | } |
| 1295 | 1299 | |
| 1300 | +/* | |
| 1301 | + * Symbols that replace INSN_RETURN, every (tail) call to such a symbol | |
| 1302 | + * will be added to the .return_sites section. | |
| 1303 | + */ | |
| 1296 | 1304 | __weak bool arch_is_rethunk(struct symbol *sym) |
| 1297 | 1305 | { |
| 1298 | 1306 | return false; |
| 1299 | 1307 | } |
| 1300 | 1308 | |
| 1309 | +/* | |
| 1310 | + * Symbols that are embedded inside other instructions, because sometimes crazy | |
| 1311 | + * code exists. These are mostly ignored for validation purposes. | |
| 1312 | + */ | |
| 1313 | +__weak bool arch_is_embedded_insn(struct symbol *sym) | |
| 1314 | +{ | |
| 1315 | + return false; | |
| 1316 | +} | |
| 1317 | + | |
| 1301 | 1318 | static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn) |
| 1302 | 1319 | { |
| 1303 | 1320 | struct reloc *reloc; |
| @@ -1576,14 +1593,14 @@ static int add_jump_destinations(struct objtool_file *file) · 272 unmodified lines | ||
| 1576 | 1593 | struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off); |
| 1577 | 1594 | |
| 1578 | 1595 | /* |
| 1579 | − * This is a special case for zen_untrain_ret(). | |
| 1596 | + * This is a special case for retbleed_untrain_ret(). | |
| 1580 | 1597 | * It jumps to __x86_return_thunk(), but objtool |
| 1581 | 1598 | * can't find the thunk's starting RET |
| 1582 | 1599 | * instruction, because the RET is also in the |
| 1583 | 1600 | * middle of another instruction. Objtool only |
| 1584 | 1601 | * knows about the outer instruction. |
| 1585 | 1602 | */ |
| 1586 | − if (sym && sym->return_thunk) { | |
| 1603 | + if (sym && sym->embedded_insn) { | |
| 1587 | 1604 | add_return_call(file, insn, false); |
| 1588 | 1605 | continue; |
| 1589 | 1606 | } |
| @@ -2502,6 +2519,9 @@ static int classify_symbols(struct objtool_file *file) · 912 unmodified lines | ||
| 2502 | 2519 | if (arch_is_rethunk(func)) |
| 2503 | 2520 | func->return_thunk = true; |
| 2504 | 2521 | |
| 2522 | + if (arch_is_embedded_insn(func)) | |
| 2523 | + func->embedded_insn = true; | |
| 2524 | + | |
| 2505 | 2525 | if (arch_ftrace_match(func->name)) |
| 2506 | 2526 | func->fentry = true; |
| 2507 | 2527 | |
| @@ -2630,12 +2650,17 @@ static int decode_sections(struct objtool_file *file) · 122 unmodified lines | ||
| 2630 | 2650 | return 0; |
| 2631 | 2651 | } |
| 2632 | 2652 | |
| 2633 | −static bool is_fentry_call(struct instruction *insn) | |
| 2653 | +static bool is_special_call(struct instruction *insn) | |
| 2634 | 2654 | { |
| 2635 | − if (insn->type == INSN_CALL && | |
| 2636 | − insn_call_dest(insn) && | |
| 2637 | − insn_call_dest(insn)->fentry) | |
| 2638 | − return true; | |
| 2655 | + if (insn->type == INSN_CALL) { | |
| 2656 | + struct symbol *dest = insn_call_dest(insn); | |
| 2657 | + | |
| 2658 | + if (!dest) | |
| 2659 | + return false; | |
| 2660 | + | |
| 2661 | + if (dest->fentry || dest->embedded_insn) | |
| 2662 | + return true; | |
| 2663 | + } | |
| 2639 | 2664 | |
| 2640 | 2665 | return false; |
| 2641 | 2666 | } |
| @@ -3636,7 +3661,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, · 994 unmodified lines | ||
| 3636 | 3661 | if (ret) |
| 3637 | 3662 | return ret; |
| 3638 | 3663 | |
| 3639 | − if (opts.stackval && func && !is_fentry_call(insn) && | |
| 3664 | + if (opts.stackval && func && !is_special_call(insn) && | |
| 3640 | 3665 | !has_valid_stack_frame(&state)) { |
| 3641 | 3666 | WARN_INSN(insn, "call without frame pointer save/setup"); |
| 3642 | 3667 | return 1; |
| @@ -90,6 +90,7 @@ int arch_decode_hint_reg(u8 sp_reg, int *base); · 89 unmodified lines | ||
| 90 | 90 | |
| 91 | 91 | bool arch_is_retpoline(struct symbol *sym); |
| 92 | 92 | bool arch_is_rethunk(struct symbol *sym); |
| 93 | +bool arch_is_embedded_insn(struct symbol *sym); | |
| 93 | 94 | |
| 94 | 95 | int arch_rewrite_retpolines(struct objtool_file *file); |
| 95 | 96 | |
| @@ -66,6 +66,7 @@ struct symbol { · 65 unmodified lines | ||
| 66 | 66 | u8 fentry : 1; |
| 67 | 67 | u8 profiling_func : 1; |
| 68 | 68 | u8 warned : 1; |
| 69 | + u8 embedded_insn : 1; | |
| 69 | 70 | struct list_head pv_target; |
| 70 | 71 | struct reloc *relocs; |
| 71 | 72 | }; |
| @@ -1038,9 +1038,7 @@ static int thread_stack__trace_end(struct thread_stack *ts, · 1037 unmodified lines | ||
| 1038 | 1038 | |
| 1039 | 1039 | static bool is_x86_retpoline(const char *name) |
| 1040 | 1040 | { |
| 1041 | − const char *p = strstr(name, "__x86_indirect_thunk_"); | |
| 1042 | − | |
| 1043 | − return p == name || !strcmp(name, "__indirect_thunk_start"); | |
| 1041 | + return strstr(name, "__x86_indirect_thunk_") == name; | |
| 1044 | 1042 | } |
| 1045 | 1043 | |
| 1046 | 1044 | /* |