Pull x86 fixes from Borislav Petkov: "Extraordinary embargoed times call for extraordinary measures. That's why this week's x86/urgent branch is larger than usual, containing all the known fallout fixes after the SRSO mitigation got merged. I know, it is a bit late in the game but everyone who has reported a bug stemming from the SRSO pile, has tested that branch and has confirmed that it fixes their bug. Also, I've run it on every possible hardware I have and it is looking good. It is running on this very machine while I'm typing, for 2 days now without an issue. Famous last words... - Use LEA ...%rsp instead of ADD %rsp in the Zen1/2 SRSO return sequence as latter clobbers flags which interferes with fastop emulation in KVM, leading to guests freezing during boot - A fix for the DIV(0) quotient data leak on Zen1 to clear the divider buffers at the right time - Disable the SRSO mitigation on unaffected configurations as it got enabled there unnecessarily - Change .text section name to fix CONFIG_LTO_CLANG builds - Improve the optprobe indirect jmp check so that certain configurations can still be able to use optprobes at all - A serious and good scrubbing of the untraining routines by PeterZ: - Add proper speculation stopping traps so that objtool is happy - Adjust objtool to handle the new thunks - Make the thunk pointer assignable to the different untraining sequences at runtime, thus avoiding the alternative at the return thunk. It simplifies the code a bit too. - Add a entry_untrain_ret() main entry point which selects the respective untraining sequence - Rename things so that they're more clear - Fix stack validation with FRAME_POINTER=y builds - Fix static call patching to handle when a JMP to the return thunk is the last insn on the very last module memory page - Add more documentation about what each untraining routine does and why" * tag 'x86_urgent_for_v6.5_rc7' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: x86/srso: Correct the mitigation status when SMT is disabled x86/static_call: Fix __static_call_fixup() objtool/x86: Fixup frame-pointer vs rethunk x86/srso: Explain the untraining sequences a bit more x86/cpu/kvm: Provide UNTRAIN_RET_VM x86/cpu: Cleanup the untrain mess x86/cpu: Rename srso_(.*)_alias to srso_alias_\1 x86/cpu: Rename original retbleed methods x86/cpu: Clean up SRSO return thunk mess x86/alternative: Make custom return thunk unconditional objtool/x86: Fix SRSO mess x86/cpu: Fix up srso_safe_ret() and __x86_return_thunk() x86/cpu: Fix __x86_return_thunk symbol type x86/retpoline,kprobes: Skip optprobe check for indirect jumps with retpolines and IBT x86/retpoline,kprobes: Fix position of thunk sections with CONFIG_LTO_CLANG x86/srso: Disable the mitigation on unaffected configurations x86/CPU/AMD: Fix the DIV(0) initial fix attempt x86/retpoline: Don't clobber RFLAGS during srso_safe_ret()
Unified · Split
| @@ -141,8 +141,8 @@ sequence. · 140 unmodified lines | |||
| 141 | To ensure the safety of this mitigation, the kernel must ensure that the | 141 | To ensure the safety of this mitigation, the kernel must ensure that the |
| 142 | safe return sequence is itself free from attacker interference. In Zen3 | 142 | safe return sequence is itself free from attacker interference. In Zen3 |
| 143 | and Zen4, this is accomplished by creating a BTB alias between the | 143 | and Zen4, this is accomplished by creating a BTB alias between the |
| 144 | untraining function srso_untrain_ret_alias() and the safe return | 144 | untraining function srso_alias_untrain_ret() and the safe return |
| 145 | function srso_safe_ret_alias() which results in evicting a potentially | 145 | function srso_alias_safe_ret() which results in evicting a potentially |
| 146 | poisoned BTB entry and using that safe one for all function returns. | 146 | poisoned BTB entry and using that safe one for all function returns. |
| 147 | 147 | ||
| 148 | In older Zen1 and Zen2, this is accomplished using a reinterpretation | 148 | In older Zen1 and Zen2, this is accomplished using a reinterpretation |
| @@ -92,6 +92,7 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, · 91 unmodified lines | |||
| 92 | static __always_inline void arch_exit_to_user_mode(void) | 92 | static __always_inline void arch_exit_to_user_mode(void) |
| 93 | { | 93 | { |
| 94 | mds_user_clear_cpu_buffers(); | 94 | mds_user_clear_cpu_buffers(); |
| 95 | amd_clear_divider(); | ||
| 95 | } | 96 | } |
| 96 | #define arch_exit_to_user_mode arch_exit_to_user_mode | 97 | #define arch_exit_to_user_mode arch_exit_to_user_mode |
| 97 | 98 | ||
| @@ -272,9 +272,9 @@ · 271 unmodified lines | |||
| 272 | .endm | 272 | .endm |
| 273 | 273 | ||
| 274 | #ifdef CONFIG_CPU_UNRET_ENTRY | 274 | #ifdef CONFIG_CPU_UNRET_ENTRY |
| 275 | #define CALL_ZEN_UNTRAIN_RET "call zen_untrain_ret" | 275 | #define CALL_UNTRAIN_RET "call entry_untrain_ret" |
| 276 | #else | 276 | #else |
| 277 | #define CALL_ZEN_UNTRAIN_RET "" | 277 | #define CALL_UNTRAIN_RET "" |
| 278 | #endif | 278 | #endif |
| 279 | 279 | ||
| 280 | /* | 280 | /* |
| @@ -282,7 +282,7 @@ · 1 unmodified line | |||
| 282 | * return thunk isn't mapped into the userspace tables (then again, AMD | 282 | * return thunk isn't mapped into the userspace tables (then again, AMD |
| 283 | * typically has NO_MELTDOWN). | 283 | * typically has NO_MELTDOWN). |
| 284 | * | 284 | * |
| 285 | * While zen_untrain_ret() doesn't clobber anything but requires stack, | 285 | * While retbleed_untrain_ret() doesn't clobber anything but requires stack, |
| 286 | * entry_ibpb() will clobber AX, CX, DX. | 286 | * entry_ibpb() will clobber AX, CX, DX. |
| 287 | * | 287 | * |
| 288 | * As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point | 288 | * As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point |
| @@ -293,14 +293,20 @@ · 4 unmodified lines | |||
| 293 | defined(CONFIG_CALL_DEPTH_TRACKING) || defined(CONFIG_CPU_SRSO) | 293 | defined(CONFIG_CALL_DEPTH_TRACKING) || defined(CONFIG_CPU_SRSO) |
| 294 | VALIDATE_UNRET_END | 294 | VALIDATE_UNRET_END |
| 295 | ALTERNATIVE_3 "", \ | 295 | ALTERNATIVE_3 "", \ |
| 296 | CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \ | 296 | CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \ |
| 297 | "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \ | 297 | "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \ |
| 298 | __stringify(RESET_CALL_DEPTH), X86_FEATURE_CALL_DEPTH | 298 | __stringify(RESET_CALL_DEPTH), X86_FEATURE_CALL_DEPTH |
| 299 | #endif | 299 | #endif |
| 300 | .endm | ||
| 300 | 301 | ||
| 301 | #ifdef CONFIG_CPU_SRSO | 302 | .macro UNTRAIN_RET_VM |
| 302 | ALTERNATIVE_2 "", "call srso_untrain_ret", X86_FEATURE_SRSO, \ | 303 | #if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \ |
| 303 | "call srso_untrain_ret_alias", X86_FEATURE_SRSO_ALIAS | 304 | defined(CONFIG_CALL_DEPTH_TRACKING) || defined(CONFIG_CPU_SRSO) |
| 305 | VALIDATE_UNRET_END | ||
| 306 | ALTERNATIVE_3 "", \ | ||
| 307 | CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \ | ||
| 308 | "call entry_ibpb", X86_FEATURE_IBPB_ON_VMEXIT, \ | ||
| 309 | __stringify(RESET_CALL_DEPTH), X86_FEATURE_CALL_DEPTH | ||
| 304 | #endif | 310 | #endif |
| 305 | .endm | 311 | .endm |
| 306 | 312 | ||
| @@ -309,15 +315,10 @@ · 2 unmodified lines | |||
| 309 | defined(CONFIG_CALL_DEPTH_TRACKING) | 315 | defined(CONFIG_CALL_DEPTH_TRACKING) |
| 310 | VALIDATE_UNRET_END | 316 | VALIDATE_UNRET_END |
| 311 | ALTERNATIVE_3 "", \ | 317 | ALTERNATIVE_3 "", \ |
| 312 | CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \ | 318 | CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \ |
| 313 | "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \ | 319 | "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \ |
| 314 | __stringify(RESET_CALL_DEPTH_FROM_CALL), X86_FEATURE_CALL_DEPTH | 320 | __stringify(RESET_CALL_DEPTH_FROM_CALL), X86_FEATURE_CALL_DEPTH |
| 315 | #endif | 321 | #endif |
| 316 | |||
| 317 | #ifdef CONFIG_CPU_SRSO | ||
| 318 | ALTERNATIVE_2 "", "call srso_untrain_ret", X86_FEATURE_SRSO, \ | ||
| 319 | "call srso_untrain_ret_alias", X86_FEATURE_SRSO_ALIAS | ||
| 320 | #endif | ||
| 321 | .endm | 322 | .endm |
| 322 | 323 | ||
| 323 | 324 | ||
| @@ -341,17 +342,24 @@ extern retpoline_thunk_t __x86_indirect_thunk_array[]; · 17 unmodified lines | |||
| 341 | extern retpoline_thunk_t __x86_indirect_call_thunk_array[]; | 342 | extern retpoline_thunk_t __x86_indirect_call_thunk_array[]; |
| 342 | extern retpoline_thunk_t __x86_indirect_jump_thunk_array[]; | 343 | extern retpoline_thunk_t __x86_indirect_jump_thunk_array[]; |
| 343 | 344 | ||
| 345 | #ifdef CONFIG_RETHUNK | ||
| 344 | extern void __x86_return_thunk(void); | 346 | extern void __x86_return_thunk(void); |
| 345 | extern void zen_untrain_ret(void); | 347 | #else |
| 348 | static inline void __x86_return_thunk(void) {} | ||
| 349 | #endif | ||
| 350 | |||
| 351 | extern void retbleed_return_thunk(void); | ||
| 352 | extern void srso_return_thunk(void); | ||
| 353 | extern void srso_alias_return_thunk(void); | ||
| 354 | |||
| 355 | extern void retbleed_untrain_ret(void); | ||
| 346 | extern void srso_untrain_ret(void); | 356 | extern void srso_untrain_ret(void); |
| 347 | extern void srso_untrain_ret_alias(void); | 357 | extern void srso_alias_untrain_ret(void); |
| 358 | |||
| 359 | extern void entry_untrain_ret(void); | ||
| 348 | extern void entry_ibpb(void); | 360 | extern void entry_ibpb(void); |
| 349 | 361 | ||
| 350 | #ifdef CONFIG_CALL_THUNKS | ||
| 351 | extern void (*x86_return_thunk)(void); | 362 | extern void (*x86_return_thunk)(void); |
| 352 | #else | ||
| 353 | #define x86_return_thunk (&__x86_return_thunk) | ||
| 354 | #endif | ||
| 355 | 363 | ||
| 356 | #ifdef CONFIG_CALL_DEPTH_TRACKING | 364 | #ifdef CONFIG_CALL_DEPTH_TRACKING |
| 357 | extern void __x86_return_skl(void); | 365 | extern void __x86_return_skl(void); |
| @@ -478,9 +486,6 @@ enum ssb_mitigation { · 120 unmodified lines | |||
| 478 | SPEC_STORE_BYPASS_SECCOMP, | 486 | SPEC_STORE_BYPASS_SECCOMP, |
| 479 | }; | 487 | }; |
| 480 | 488 | ||
| 481 | extern char __indirect_thunk_start[]; | ||
| 482 | extern char __indirect_thunk_end[]; | ||
| 483 | |||
| 484 | static __always_inline | 489 | static __always_inline |
| 485 | void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature) | 490 | void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature) |
| 486 | { | 491 | { |
| @@ -687,10 +687,6 @@ void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) · 686 unmodified lines | |||
| 687 | 687 | ||
| 688 | #ifdef CONFIG_RETHUNK | 688 | #ifdef CONFIG_RETHUNK |
| 689 | 689 | ||
| 690 | #ifdef CONFIG_CALL_THUNKS | ||
| 691 | void (*x86_return_thunk)(void) __ro_after_init = &__x86_return_thunk; | ||
| 692 | #endif | ||
| 693 | |||
| 694 | /* | 690 | /* |
| 695 | * Rewrite the compiler generated return thunk tail-calls. | 691 | * Rewrite the compiler generated return thunk tail-calls. |
| 696 | * | 692 | * |
| @@ -1329,3 +1329,4 @@ void noinstr amd_clear_divider(void) · 1328 unmodified lines | |||
| 1329 | asm volatile(ALTERNATIVE("", "div %2\n\t", X86_BUG_DIV0) | 1329 | asm volatile(ALTERNATIVE("", "div %2\n\t", X86_BUG_DIV0) |
| 1330 | :: "a" (0), "d" (0), "r" (1)); | 1330 | :: "a" (0), "d" (0), "r" (1)); |
| 1331 | } | 1331 | } |
| 1332 | EXPORT_SYMBOL_GPL(amd_clear_divider); | ||
| @@ -63,6 +63,8 @@ EXPORT_SYMBOL_GPL(x86_pred_cmd); · 62 unmodified lines | |||
| 63 | 63 | ||
| 64 | static DEFINE_MUTEX(spec_ctrl_mutex); | 64 | static DEFINE_MUTEX(spec_ctrl_mutex); |
| 65 | 65 | ||
| 66 | void (*x86_return_thunk)(void) __ro_after_init = &__x86_return_thunk; | ||
| 67 | |||
| 66 | /* Update SPEC_CTRL MSR and its cached copy unconditionally */ | 68 | /* Update SPEC_CTRL MSR and its cached copy unconditionally */ |
| 67 | static void update_spec_ctrl(u64 val) | 69 | static void update_spec_ctrl(u64 val) |
| 68 | { | 70 | { |
| @@ -165,6 +167,11 @@ void __init cpu_select_mitigations(void) · 96 unmodified lines | |||
| 165 | md_clear_select_mitigation(); | 167 | md_clear_select_mitigation(); |
| 166 | srbds_select_mitigation(); | 168 | srbds_select_mitigation(); |
| 167 | l1d_flush_select_mitigation(); | 169 | l1d_flush_select_mitigation(); |
| 170 | |||
| 171 | /* | ||
| 172 | * srso_select_mitigation() depends and must run after | ||
| 173 | * retbleed_select_mitigation(). | ||
| 174 | */ | ||
| 168 | srso_select_mitigation(); | 175 | srso_select_mitigation(); |
| 169 | gds_select_mitigation(); | 176 | gds_select_mitigation(); |
| 170 | } | 177 | } |
| @@ -1035,6 +1042,9 @@ do_cmd_auto: · 864 unmodified lines | |||
| 1035 | setup_force_cpu_cap(X86_FEATURE_RETHUNK); | 1042 | setup_force_cpu_cap(X86_FEATURE_RETHUNK); |
| 1036 | setup_force_cpu_cap(X86_FEATURE_UNRET); | 1043 | setup_force_cpu_cap(X86_FEATURE_UNRET); |
| 1037 | 1044 | ||
| 1045 | if (IS_ENABLED(CONFIG_RETHUNK)) | ||
| 1046 | x86_return_thunk = retbleed_return_thunk; | ||
| 1047 | |||
| 1038 | if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && | 1048 | if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && |
| 1039 | boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) | 1049 | boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) |
| 1040 | pr_err(RETBLEED_UNTRAIN_MSG); | 1050 | pr_err(RETBLEED_UNTRAIN_MSG); |
| @@ -1044,6 +1054,7 @@ do_cmd_auto: · 3 unmodified lines | |||
| 1044 | 1054 | ||
| 1045 | case RETBLEED_MITIGATION_IBPB: | 1055 | case RETBLEED_MITIGATION_IBPB: |
| 1046 | setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); | 1056 | setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); |
| 1057 | setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT); | ||
| 1047 | mitigate_smt = true; | 1058 | mitigate_smt = true; |
| 1048 | break; | 1059 | break; |
| 1049 | 1060 | ||
| @@ -2417,9 +2428,10 @@ static void __init srso_select_mitigation(void) · 1367 unmodified lines | |||
| 2417 | * Zen1/2 with SMT off aren't vulnerable after the right | 2428 | * Zen1/2 with SMT off aren't vulnerable after the right |
| 2418 | * IBPB microcode has been applied. | 2429 | * IBPB microcode has been applied. |
| 2419 | */ | 2430 | */ |
| 2420 | if ((boot_cpu_data.x86 < 0x19) && | 2431 | if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) { |
| 2421 | (!cpu_smt_possible() || (cpu_smt_control == CPU_SMT_DISABLED))) | ||
| 2422 | setup_force_cpu_cap(X86_FEATURE_SRSO_NO); | 2432 | setup_force_cpu_cap(X86_FEATURE_SRSO_NO); |
| 2433 | return; | ||
| 2434 | } | ||
| 2423 | } | 2435 | } |
| 2424 | 2436 | ||
| 2425 | if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { | 2437 | if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { |
| @@ -2448,11 +2460,15 @@ static void __init srso_select_mitigation(void) · 22 unmodified lines | |||
| 2448 | * like ftrace, static_call, etc. | 2460 | * like ftrace, static_call, etc. |
| 2449 | */ | 2461 | */ |
| 2450 | setup_force_cpu_cap(X86_FEATURE_RETHUNK); | 2462 | setup_force_cpu_cap(X86_FEATURE_RETHUNK); |
| 2463 | setup_force_cpu_cap(X86_FEATURE_UNRET); | ||
| 2451 | 2464 | ||
| 2452 | if (boot_cpu_data.x86 == 0x19) | 2465 | if (boot_cpu_data.x86 == 0x19) { |
| 2453 | setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS); | 2466 | setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS); |
| 2454 | else | 2467 | x86_return_thunk = srso_alias_return_thunk; |
| 2468 | } else { | ||
| 2455 | setup_force_cpu_cap(X86_FEATURE_SRSO); | 2469 | setup_force_cpu_cap(X86_FEATURE_SRSO); |
| 2470 | x86_return_thunk = srso_return_thunk; | ||
| 2471 | } | ||
| 2456 | srso_mitigation = SRSO_MITIGATION_SAFE_RET; | 2472 | srso_mitigation = SRSO_MITIGATION_SAFE_RET; |
| 2457 | } else { | 2473 | } else { |
| 2458 | pr_err("WARNING: kernel not compiled with CPU_SRSO.\n"); | 2474 | pr_err("WARNING: kernel not compiled with CPU_SRSO.\n"); |
| @@ -2696,6 +2712,9 @@ static ssize_t retbleed_show_state(char *buf) · 237 unmodified lines | |||
| 2696 | 2712 | ||
| 2697 | static ssize_t srso_show_state(char *buf) | 2713 | static ssize_t srso_show_state(char *buf) |
| 2698 | { | 2714 | { |
| 2715 | if (boot_cpu_has(X86_FEATURE_SRSO_NO)) | ||
| 2716 | return sysfs_emit(buf, "Mitigation: SMT disabled\n"); | ||
| 2717 | |||
| 2699 | return sysfs_emit(buf, "%s%s\n", | 2718 | return sysfs_emit(buf, "%s%s\n", |
| 2700 | srso_strings[srso_mitigation], | 2719 | srso_strings[srso_mitigation], |
| 2701 | (cpu_has_ibpb_brtype_microcode() ? "" : ", no microcode")); | 2720 | (cpu_has_ibpb_brtype_microcode() ? "" : ", no microcode")); |
| @@ -226,7 +226,7 @@ static int copy_optimized_instructions(u8 *dest, u8 *src, u8 *real) · 225 unmodified lines | |||
| 226 | } | 226 | } |
| 227 | 227 | ||
| 228 | /* Check whether insn is indirect jump */ | 228 | /* Check whether insn is indirect jump */ |
| 229 | static int __insn_is_indirect_jump(struct insn *insn) | 229 | static int insn_is_indirect_jump(struct insn *insn) |
| 230 | { | 230 | { |
| 231 | return ((insn->opcode.bytes[0] == 0xff && | 231 | return ((insn->opcode.bytes[0] == 0xff && |
| 232 | (X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */ | 232 | (X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */ |
| @@ -260,26 +260,6 @@ static int insn_jump_into_range(struct insn *insn, unsigned long start, int len) · 27 unmodified lines | |||
| 260 | return (start <= target && target <= start + len); | 260 | return (start <= target && target <= start + len); |
| 261 | } | 261 | } |
| 262 | 262 | ||
| 263 | static int insn_is_indirect_jump(struct insn *insn) | ||
| 264 | { | ||
| 265 | int ret = __insn_is_indirect_jump(insn); | ||
| 266 | |||
| 267 | #ifdef CONFIG_RETPOLINE | ||
| 268 | /* | ||
| 269 | * Jump to x86_indirect_thunk_* is treated as an indirect jump. | ||
| 270 | * Note that even with CONFIG_RETPOLINE=y, the kernel compiled with | ||
| 271 | * older gcc may use indirect jump. So we add this check instead of | ||
| 272 | * replace indirect-jump check. | ||
| 273 | */ | ||
| 274 | if (!ret) | ||
| 275 | ret = insn_jump_into_range(insn, | ||
| 276 | (unsigned long)__indirect_thunk_start, | ||
| 277 | (unsigned long)__indirect_thunk_end - | ||
| 278 | (unsigned long)__indirect_thunk_start); | ||
| 279 | #endif | ||
| 280 | return ret; | ||
| 281 | } | ||
| 282 | |||
| 283 | /* Decode whole function to ensure any instructions don't jump into target */ | 263 | /* Decode whole function to ensure any instructions don't jump into target */ |
| 284 | static int can_optimize(unsigned long paddr) | 264 | static int can_optimize(unsigned long paddr) |
| 285 | { | 265 | { |
| @@ -334,9 +314,21 @@ static int can_optimize(unsigned long paddr) · 48 unmodified lines | |||
| 334 | /* Recover address */ | 314 | /* Recover address */ |
| 335 | insn.kaddr = (void *)addr; | 315 | insn.kaddr = (void *)addr; |
| 336 | insn.next_byte = (void *)(addr + insn.length); | 316 | insn.next_byte = (void *)(addr + insn.length); |
| 337 | /* Check any instructions don't jump into target */ | 317 | /* |
| 338 | if (insn_is_indirect_jump(&insn) || | 318 | * Check any instructions don't jump into target, indirectly or |
| 339 | insn_jump_into_range(&insn, paddr + INT3_INSN_SIZE, | 319 | * directly. |
| 320 | * | ||
| 321 | * The indirect case is present to handle a code with jump | ||
| 322 | * tables. When the kernel uses retpolines, the check should in | ||
| 323 | * theory additionally look for jumps to indirect thunks. | ||
| 324 | * However, the kernel built with retpolines or IBT has jump | ||
| 325 | * tables disabled so the check can be skipped altogether. | ||
| 326 | */ | ||
| 327 | if (!IS_ENABLED(CONFIG_RETPOLINE) && | ||
| 328 | !IS_ENABLED(CONFIG_X86_KERNEL_IBT) && | ||
| 329 | insn_is_indirect_jump(&insn)) | ||
| 330 | return 0; | ||
| 331 | if (insn_jump_into_range(&insn, paddr + INT3_INSN_SIZE, | ||
| 340 | DISP32_SIZE)) | 332 | DISP32_SIZE)) |
| 341 | return 0; | 333 | return 0; |
| 342 | addr += insn.length; | 334 | addr += insn.length; |
| @@ -186,6 +186,19 @@ EXPORT_SYMBOL_GPL(arch_static_call_transform); · 185 unmodified lines | |||
| 186 | */ | 186 | */ |
| 187 | bool __static_call_fixup(void *tramp, u8 op, void *dest) | 187 | bool __static_call_fixup(void *tramp, u8 op, void *dest) |
| 188 | { | 188 | { |
| 189 | unsigned long addr = (unsigned long)tramp; | ||
| 190 | /* | ||
| 191 | * Not all .return_sites are a static_call trampoline (most are not). | ||
| 192 | * Check if the 3 bytes after the return are still kernel text, if not, | ||
| 193 | * then this definitely is not a trampoline and we need not worry | ||
| 194 | * further. | ||
| 195 | * | ||
| 196 | * This avoids the memcmp() below tripping over pagefaults etc.. | ||
| 197 | */ | ||
| 198 | if (((addr >> PAGE_SHIFT) != ((addr + 7) >> PAGE_SHIFT)) && | ||
| 199 | !kernel_text_address(addr + 7)) | ||
| 200 | return false; | ||
| 201 | |||
| 189 | if (memcmp(tramp+5, tramp_ud, 3)) { | 202 | if (memcmp(tramp+5, tramp_ud, 3)) { |
| 190 | /* Not a trampoline site, not our problem. */ | 203 | /* Not a trampoline site, not our problem. */ |
| 191 | return false; | 204 | return false; |
| @@ -206,8 +206,6 @@ DEFINE_IDTENTRY(exc_divide_error) · 205 unmodified lines | |||
| 206 | { | 206 | { |
| 207 | do_error_trap(regs, 0, "divide error", X86_TRAP_DE, SIGFPE, | 207 | do_error_trap(regs, 0, "divide error", X86_TRAP_DE, SIGFPE, |
| 208 | FPE_INTDIV, error_get_trap_addr(regs)); | 208 | FPE_INTDIV, error_get_trap_addr(regs)); |
| 209 | |||
| 210 | amd_clear_divider(); | ||
| 211 | } | 209 | } |
| 212 | 210 | ||
| 213 | DEFINE_IDTENTRY(exc_overflow) | 211 | DEFINE_IDTENTRY(exc_overflow) |
| @@ -133,27 +133,25 @@ SECTIONS · 132 unmodified lines | |||
| 133 | KPROBES_TEXT | 133 | KPROBES_TEXT |
| 134 | SOFTIRQENTRY_TEXT | 134 | SOFTIRQENTRY_TEXT |
| 135 | #ifdef CONFIG_RETPOLINE | 135 | #ifdef CONFIG_RETPOLINE |
| 136 | __indirect_thunk_start = .; | 136 | *(.text..__x86.indirect_thunk) |
| 137 | *(.text.__x86.indirect_thunk) | 137 | *(.text..__x86.return_thunk) |
| 138 | *(.text.__x86.return_thunk) | ||
| 139 | __indirect_thunk_end = .; | ||
| 140 | #endif | 138 | #endif |
| 141 | STATIC_CALL_TEXT | 139 | STATIC_CALL_TEXT |
| 142 | 140 | ||
| 143 | ALIGN_ENTRY_TEXT_BEGIN | 141 | ALIGN_ENTRY_TEXT_BEGIN |
| 144 | #ifdef CONFIG_CPU_SRSO | 142 | #ifdef CONFIG_CPU_SRSO |
| 145 | *(.text.__x86.rethunk_untrain) | 143 | *(.text..__x86.rethunk_untrain) |
| 146 | #endif | 144 | #endif |
| 147 | 145 | ||
| 148 | ENTRY_TEXT | 146 | ENTRY_TEXT |
| 149 | 147 | ||
| 150 | #ifdef CONFIG_CPU_SRSO | 148 | #ifdef CONFIG_CPU_SRSO |
| 151 | /* | 149 | /* |
| 152 | * See the comment above srso_untrain_ret_alias()'s | 150 | * See the comment above srso_alias_untrain_ret()'s |
| 153 | * definition. | 151 | * definition. |
| 154 | */ | 152 | */ |
| 155 | . = srso_untrain_ret_alias | (1 << 2) | (1 << 8) | (1 << 14) | (1 << 20); | 153 | . = srso_alias_untrain_ret | (1 << 2) | (1 << 8) | (1 << 14) | (1 << 20); |
| 156 | *(.text.__x86.rethunk_safe) | 154 | *(.text..__x86.rethunk_safe) |
| 157 | #endif | 155 | #endif |
| 158 | ALIGN_ENTRY_TEXT_END | 156 | ALIGN_ENTRY_TEXT_END |
| 159 | *(.gnu.warning) | 157 | *(.gnu.warning) |
| @@ -523,7 +521,7 @@ INIT_PER_CPU(irq_stack_backing_store); · 363 unmodified lines | |||
| 523 | #endif | 521 | #endif |
| 524 | 522 | ||
| 525 | #ifdef CONFIG_RETHUNK | 523 | #ifdef CONFIG_RETHUNK |
| 526 | . = ASSERT((__ret & 0x3f) == 0, "__ret not cacheline-aligned"); | 524 | . = ASSERT((retbleed_return_thunk & 0x3f) == 0, "retbleed_return_thunk not cacheline-aligned"); |
| 527 | . = ASSERT((srso_safe_ret & 0x3f) == 0, "srso_safe_ret not cacheline-aligned"); | 525 | . = ASSERT((srso_safe_ret & 0x3f) == 0, "srso_safe_ret not cacheline-aligned"); |
| 528 | #endif | 526 | #endif |
| 529 | 527 | ||
| @@ -538,8 +536,8 @@ INIT_PER_CPU(irq_stack_backing_store); · 8 unmodified lines | |||
| 538 | * Instead do: (A | B) - (A & B) in order to compute the XOR | 536 | * Instead do: (A | B) - (A & B) in order to compute the XOR |
| 539 | * of the two function addresses: | 537 | * of the two function addresses: |
| 540 | */ | 538 | */ |
| 541 | . = ASSERT(((ABSOLUTE(srso_untrain_ret_alias) | srso_safe_ret_alias) - | 539 | . = ASSERT(((ABSOLUTE(srso_alias_untrain_ret) | srso_alias_safe_ret) - |
| 542 | (ABSOLUTE(srso_untrain_ret_alias) & srso_safe_ret_alias)) == ((1 << 2) | (1 << 8) | (1 << 14) | (1 << 20)), | 540 | (ABSOLUTE(srso_alias_untrain_ret) & srso_alias_safe_ret)) == ((1 << 2) | (1 << 8) | (1 << 14) | (1 << 20)), |
| 543 | "SRSO function pair won't alias"); | 541 | "SRSO function pair won't alias"); |
| 544 | #endif | 542 | #endif |
| 545 | 543 | ||
| @@ -4006,6 +4006,8 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_in · 4005 unmodified lines | |||
| 4006 | 4006 | ||
| 4007 | guest_state_enter_irqoff(); | 4007 | guest_state_enter_irqoff(); |
| 4008 | 4008 | ||
| 4009 | amd_clear_divider(); | ||
| 4010 | |||
| 4009 | if (sev_es_guest(vcpu->kvm)) | 4011 | if (sev_es_guest(vcpu->kvm)) |
| 4010 | __svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted); | 4012 | __svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted); |
| 4011 | else | 4013 | else |
| @@ -222,10 +222,7 @@ SYM_FUNC_START(__svm_vcpu_run) · 221 unmodified lines | |||
| 222 | * because interrupt handlers won't sanitize 'ret' if the return is | 222 | * because interrupt handlers won't sanitize 'ret' if the return is |
| 223 | * from the kernel. | 223 | * from the kernel. |
| 224 | */ | 224 | */ |
| 225 | UNTRAIN_RET | 225 | UNTRAIN_RET_VM |
| 226 | |||
| 227 | /* SRSO */ | ||
| 228 | ALTERNATIVE "", "call entry_ibpb", X86_FEATURE_IBPB_ON_VMEXIT | ||
| 229 | 226 | ||
| 230 | /* | 227 | /* |
| 231 | * Clear all general purpose registers except RSP and RAX to prevent | 228 | * Clear all general purpose registers except RSP and RAX to prevent |
| @@ -362,7 +359,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) · 130 unmodified lines | |||
| 362 | * because interrupt handlers won't sanitize RET if the return is | 359 | * because interrupt handlers won't sanitize RET if the return is |
| 363 | * from the kernel. | 360 | * from the kernel. |
| 364 | */ | 361 | */ |
| 365 | UNTRAIN_RET | 362 | UNTRAIN_RET_VM |
| 366 | 363 | ||
| 367 | /* "Pop" @spec_ctrl_intercepted. */ | 364 | /* "Pop" @spec_ctrl_intercepted. */ |
| 368 | pop %_ASM_BX | 365 | pop %_ASM_BX |
| @@ -13,7 +13,7 @@ · 12 unmodified lines | |||
| 13 | #include <asm/frame.h> | 13 | #include <asm/frame.h> |
| 14 | #include <asm/nops.h> | 14 | #include <asm/nops.h> |
| 15 | 15 | ||
| 16 | .section .text.__x86.indirect_thunk | 16 | .section .text..__x86.indirect_thunk |
| 17 | 17 | ||
| 18 | 18 | ||
| 19 | .macro POLINE reg | 19 | .macro POLINE reg |
| @@ -133,75 +133,106 @@ SYM_CODE_END(__x86_indirect_jump_thunk_array) · 113 unmodified lines | |||
| 133 | #ifdef CONFIG_RETHUNK | 133 | #ifdef CONFIG_RETHUNK |
| 134 | 134 | ||
| 135 | /* | 135 | /* |
| 136 | * srso_untrain_ret_alias() and srso_safe_ret_alias() are placed at | 136 | * srso_alias_untrain_ret() and srso_alias_safe_ret() are placed at |
| 137 | * special addresses: | 137 | * special addresses: |
| 138 | * | 138 | * |
| 139 | * - srso_untrain_ret_alias() is 2M aligned | 139 | * - srso_alias_untrain_ret() is 2M aligned |
| 140 | * - srso_safe_ret_alias() is also in the same 2M page but bits 2, 8, 14 | 140 | * - srso_alias_safe_ret() is also in the same 2M page but bits 2, 8, 14 |
| 141 | * and 20 in its virtual address are set (while those bits in the | 141 | * and 20 in its virtual address are set (while those bits in the |
| 142 | * srso_untrain_ret_alias() function are cleared). | 142 | * srso_alias_untrain_ret() function are cleared). |
| 143 | * | 143 | * |
| 144 | * This guarantees that those two addresses will alias in the branch | 144 | * This guarantees that those two addresses will alias in the branch |
| 145 | * target buffer of Zen3/4 generations, leading to any potential | 145 | * target buffer of Zen3/4 generations, leading to any potential |
| 146 | * poisoned entries at that BTB slot to get evicted. | 146 | * poisoned entries at that BTB slot to get evicted. |
| 147 | * | 147 | * |
| 148 | * As a result, srso_safe_ret_alias() becomes a safe return. | 148 | * As a result, srso_alias_safe_ret() becomes a safe return. |
| 149 | */ | 149 | */ |
| 150 | #ifdef CONFIG_CPU_SRSO | 150 | #ifdef CONFIG_CPU_SRSO |
| 151 | .section .text.__x86.rethunk_untrain | 151 | .section .text..__x86.rethunk_untrain |
| 152 | 152 | ||
| 153 | SYM_START(srso_untrain_ret_alias, SYM_L_GLOBAL, SYM_A_NONE) | 153 | SYM_START(srso_alias_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) |
| 154 | UNWIND_HINT_FUNC | ||
| 154 | ANNOTATE_NOENDBR | 155 | ANNOTATE_NOENDBR |
| 155 | ASM_NOP2 | 156 | ASM_NOP2 |
| 156 | lfence | 157 | lfence |
| 157 | jmp __x86_return_thunk | 158 | jmp srso_alias_return_thunk |
| 158 | SYM_FUNC_END(srso_untrain_ret_alias) | 159 | SYM_FUNC_END(srso_alias_untrain_ret) |
| 159 | __EXPORT_THUNK(srso_untrain_ret_alias) | 160 | __EXPORT_THUNK(srso_alias_untrain_ret) |
| 160 | 161 | ||
| 161 | .section .text.__x86.rethunk_safe | 162 | .section .text..__x86.rethunk_safe |
| 163 | #else | ||
| 164 | /* dummy definition for alternatives */ | ||
| 165 | SYM_START(srso_alias_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) | ||
| 166 | ANNOTATE_UNRET_SAFE | ||
| 167 | ret | ||
| 168 | int3 | ||
| 169 | SYM_FUNC_END(srso_alias_untrain_ret) | ||
| 162 | #endif | 170 | #endif |
| 163 | 171 | ||
| 164 | /* Needs a definition for the __x86_return_thunk alternative below. */ | 172 | SYM_START(srso_alias_safe_ret, SYM_L_GLOBAL, SYM_A_NONE) |
| 165 | SYM_START(srso_safe_ret_alias, SYM_L_GLOBAL, SYM_A_NONE) | 173 | lea 8(%_ASM_SP), %_ASM_SP |
| 166 | #ifdef CONFIG_CPU_SRSO | ||
| 167 | add $8, %_ASM_SP | ||
| 168 | UNWIND_HINT_FUNC | 174 | UNWIND_HINT_FUNC |
| 169 | #endif | ||
| 170 | ANNOTATE_UNRET_SAFE | 175 | ANNOTATE_UNRET_SAFE |
| 171 | ret | 176 | ret |
| 172 | int3 | 177 | int3 |
| 173 | SYM_FUNC_END(srso_safe_ret_alias) | 178 | SYM_FUNC_END(srso_alias_safe_ret) |
| 174 | 179 | ||
| 175 | .section .text.__x86.return_thunk | 180 | .section .text..__x86.return_thunk |
| 181 | |||
| 182 | SYM_CODE_START(srso_alias_return_thunk) | ||
| 183 | UNWIND_HINT_FUNC | ||
| 184 | ANNOTATE_NOENDBR | ||
| 185 | call srso_alias_safe_ret | ||
| 186 | ud2 | ||
| 187 | SYM_CODE_END(srso_alias_return_thunk) | ||
| 188 | |||
| 189 | /* | ||
| 190 | * Some generic notes on the untraining sequences: | ||
| 191 | * | ||
| 192 | * They are interchangeable when it comes to flushing potentially wrong | ||
| 193 | * RET predictions from the BTB. | ||
| 194 | * | ||
| 195 | * The SRSO Zen1/2 (MOVABS) untraining sequence is longer than the | ||
| 196 | * Retbleed sequence because the return sequence done there | ||
| 197 | * (srso_safe_ret()) is longer and the return sequence must fully nest | ||
| 198 | * (end before) the untraining sequence. Therefore, the untraining | ||
| 199 | * sequence must fully overlap the return sequence. | ||
| 200 | * | ||
| 201 | * Regarding alignment - the instructions which need to be untrained, | ||
| 202 | * must all start at a cacheline boundary for Zen1/2 generations. That | ||
| 203 | * is, instruction sequences starting at srso_safe_ret() and | ||
| 204 | * the respective instruction sequences at retbleed_return_thunk() | ||
| 205 | * must start at a cacheline boundary. | ||
| 206 | */ | ||
| 176 | 207 | ||
| 177 | /* | 208 | /* |
| 178 | * Safety details here pertain to the AMD Zen{1,2} microarchitecture: | 209 | * Safety details here pertain to the AMD Zen{1,2} microarchitecture: |
| 179 | * 1) The RET at __x86_return_thunk must be on a 64 byte boundary, for | 210 | * 1) The RET at retbleed_return_thunk must be on a 64 byte boundary, for |
| 180 | * alignment within the BTB. | 211 | * alignment within the BTB. |
| 181 | * 2) The instruction at zen_untrain_ret must contain, and not | 212 | * 2) The instruction at retbleed_untrain_ret must contain, and not |
| 182 | * end with, the 0xc3 byte of the RET. | 213 | * end with, the 0xc3 byte of the RET. |
| 183 | * 3) STIBP must be enabled, or SMT disabled, to prevent the sibling thread | 214 | * 3) STIBP must be enabled, or SMT disabled, to prevent the sibling thread |
| 184 | * from re-poisioning the BTB prediction. | 215 | * from re-poisioning the BTB prediction. |
| 185 | */ | 216 | */ |
| 186 | .align 64 | 217 | .align 64 |
| 187 | .skip 64 - (__ret - zen_untrain_ret), 0xcc | 218 | .skip 64 - (retbleed_return_thunk - retbleed_untrain_ret), 0xcc |
| 188 | SYM_START(zen_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) | 219 | SYM_START(retbleed_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) |
| 189 | ANNOTATE_NOENDBR | 220 | ANNOTATE_NOENDBR |
| 190 | /* | 221 | /* |
| 191 | * As executed from zen_untrain_ret, this is: | 222 | * As executed from retbleed_untrain_ret, this is: |
| 192 | * | 223 | * |
| 193 | * TEST $0xcc, %bl | 224 | * TEST $0xcc, %bl |
| 194 | * LFENCE | 225 | * LFENCE |
| 195 | * JMP __x86_return_thunk | 226 | * JMP retbleed_return_thunk |
| 196 | * | 227 | * |
| 197 | * Executing the TEST instruction has a side effect of evicting any BTB | 228 | * Executing the TEST instruction has a side effect of evicting any BTB |
| 198 | * prediction (potentially attacker controlled) attached to the RET, as | 229 | * prediction (potentially attacker controlled) attached to the RET, as |
| 199 | * __x86_return_thunk + 1 isn't an instruction boundary at the moment. | 230 | * retbleed_return_thunk + 1 isn't an instruction boundary at the moment. |
| 200 | */ | 231 | */ |
| 201 | .byte 0xf6 | 232 | .byte 0xf6 |
| 202 | 233 | ||
| 203 | /* | 234 | /* |
| 204 | * As executed from __x86_return_thunk, this is a plain RET. | 235 | * As executed from retbleed_return_thunk, this is a plain RET. |
| 205 | * | 236 | * |
| 206 | * As part of the TEST above, RET is the ModRM byte, and INT3 the imm8. | 237 | * As part of the TEST above, RET is the ModRM byte, and INT3 the imm8. |
| 207 | * | 238 | * |
| @@ -213,13 +244,13 @@ SYM_START(zen_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) · 5 unmodified lines | |||
| 213 | * With SMT enabled and STIBP active, a sibling thread cannot poison | 244 | * With SMT enabled and STIBP active, a sibling thread cannot poison |
| 214 | * RET's prediction to a type of its choice, but can evict the | 245 | * RET's prediction to a type of its choice, but can evict the |
| 215 | * prediction due to competitive sharing. If the prediction is | 246 | * prediction due to competitive sharing. If the prediction is |
| 216 | * evicted, __x86_return_thunk will suffer Straight Line Speculation | 247 | * evicted, retbleed_return_thunk will suffer Straight Line Speculation |
| 217 | * which will be contained safely by the INT3. | 248 | * which will be contained safely by the INT3. |
| 218 | */ | 249 | */ |
| 219 | SYM_INNER_LABEL(__ret, SYM_L_GLOBAL) | 250 | SYM_INNER_LABEL(retbleed_return_thunk, SYM_L_GLOBAL) |
| 220 | ret | 251 | ret |
| 221 | int3 | 252 | int3 |
| 222 | SYM_CODE_END(__ret) | 253 | SYM_CODE_END(retbleed_return_thunk) |
| 223 | 254 | ||
| 224 | /* | 255 | /* |
| 225 | * Ensure the TEST decoding / BTB invalidation is complete. | 256 | * Ensure the TEST decoding / BTB invalidation is complete. |
| @@ -230,16 +261,16 @@ SYM_CODE_END(__ret) · 4 unmodified lines | |||
| 230 | * Jump back and execute the RET in the middle of the TEST instruction. | 261 | * Jump back and execute the RET in the middle of the TEST instruction. |
| 231 | * INT3 is for SLS protection. | 262 | * INT3 is for SLS protection. |
| 232 | */ | 263 | */ |
| 233 | jmp __ret | 264 | jmp retbleed_return_thunk |
| 234 | int3 | 265 | int3 |
| 235 | SYM_FUNC_END(zen_untrain_ret) | 266 | SYM_FUNC_END(retbleed_untrain_ret) |
| 236 | __EXPORT_THUNK(zen_untrain_ret) | 267 | __EXPORT_THUNK(retbleed_untrain_ret) |
| 237 | 268 | ||
| 238 | /* | 269 | /* |
| 239 | * SRSO untraining sequence for Zen1/2, similar to zen_untrain_ret() | 270 | * SRSO untraining sequence for Zen1/2, similar to retbleed_untrain_ret() |
| 240 | * above. On kernel entry, srso_untrain_ret() is executed which is a | 271 | * above. On kernel entry, srso_untrain_ret() is executed which is a |
| 241 | * | 272 | * |
| 242 | * movabs $0xccccccc308c48348,%rax | 273 | * movabs $0xccccc30824648d48,%rax |
| 243 | * | 274 | * |
| 244 | * and when the return thunk executes the inner label srso_safe_ret() | 275 | * and when the return thunk executes the inner label srso_safe_ret() |
| 245 | * later, it is a stack manipulation and a RET which is mispredicted and | 276 | * later, it is a stack manipulation and a RET which is mispredicted and |
| @@ -251,22 +282,44 @@ SYM_START(srso_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) · 5 unmodified lines | |||
| 251 | ANNOTATE_NOENDBR | 282 | ANNOTATE_NOENDBR |
| 252 | .byte 0x48, 0xb8 | 283 | .byte 0x48, 0xb8 |
| 253 | 284 | ||
| 285 | /* | ||
| 286 | * This forces the function return instruction to speculate into a trap | ||
| 287 | * (UD2 in srso_return_thunk() below). This RET will then mispredict | ||
| 288 | * and execution will continue at the return site read from the top of | ||
| 289 | * the stack. | ||
| 290 | */ | ||
| 254 | SYM_INNER_LABEL(srso_safe_ret, SYM_L_GLOBAL) | 291 | SYM_INNER_LABEL(srso_safe_ret, SYM_L_GLOBAL) |
| 255 | add $8, %_ASM_SP | 292 | lea 8(%_ASM_SP), %_ASM_SP |
| 256 | ret | 293 | ret |
| 257 | int3 | 294 | int3 |
| 258 | int3 | 295 | int3 |
| 259 | int3 | 296 | /* end of movabs */ |
| 260 | lfence | 297 | lfence |
| 261 | call srso_safe_ret | 298 | call srso_safe_ret |
| 262 | int3 | 299 | ud2 |
| 263 | SYM_CODE_END(srso_safe_ret) | 300 | SYM_CODE_END(srso_safe_ret) |
| 264 | SYM_FUNC_END(srso_untrain_ret) | 301 | SYM_FUNC_END(srso_untrain_ret) |
| 265 | __EXPORT_THUNK(srso_untrain_ret) | 302 | __EXPORT_THUNK(srso_untrain_ret) |
| 266 | 303 | ||
| 267 | SYM_FUNC_START(__x86_return_thunk) | 304 | SYM_CODE_START(srso_return_thunk) |
| 268 | ALTERNATIVE_2 "jmp __ret", "call srso_safe_ret", X86_FEATURE_SRSO, \ | 305 | UNWIND_HINT_FUNC |
| 269 | "call srso_safe_ret_alias", X86_FEATURE_SRSO_ALIAS | 306 | ANNOTATE_NOENDBR |
| 307 | call srso_safe_ret | ||
| 308 | ud2 | ||
| 309 | SYM_CODE_END(srso_return_thunk) | ||
| 310 | |||
| 311 | SYM_FUNC_START(entry_untrain_ret) | ||
| 312 | ALTERNATIVE_2 "jmp retbleed_untrain_ret", \ | ||
| 313 | "jmp srso_untrain_ret", X86_FEATURE_SRSO, \ | ||
| 314 | "jmp srso_alias_untrain_ret", X86_FEATURE_SRSO_ALIAS | ||
| 315 | SYM_FUNC_END(entry_untrain_ret) | ||
| 316 | __EXPORT_THUNK(entry_untrain_ret) | ||
| 317 | |||
| 318 | SYM_CODE_START(__x86_return_thunk) | ||
| 319 | UNWIND_HINT_FUNC | ||
| 320 | ANNOTATE_NOENDBR | ||
| 321 | ANNOTATE_UNRET_SAFE | ||
| 322 | ret | ||
| 270 | int3 | 323 | int3 |
| 271 | SYM_CODE_END(__x86_return_thunk) | 324 | SYM_CODE_END(__x86_return_thunk) |
| 272 | EXPORT_SYMBOL(__x86_return_thunk) | 325 | EXPORT_SYMBOL(__x86_return_thunk) |
| @@ -824,8 +824,11 @@ bool arch_is_retpoline(struct symbol *sym) · 823 unmodified lines | |||
| 824 | 824 | ||
| 825 | bool arch_is_rethunk(struct symbol *sym) | 825 | bool arch_is_rethunk(struct symbol *sym) |
| 826 | { | 826 | { |
| 827 | return !strcmp(sym->name, "__x86_return_thunk") || | 827 | return !strcmp(sym->name, "__x86_return_thunk"); |
| 828 | !strcmp(sym->name, "srso_untrain_ret") || | 828 | } |
| 829 | !strcmp(sym->name, "srso_safe_ret") || | 829 | |
| 830 | !strcmp(sym->name, "__ret"); | 830 | bool arch_is_embedded_insn(struct symbol *sym) |
| 831 | { | ||
| 832 | return !strcmp(sym->name, "retbleed_return_thunk") || | ||
| 833 | !strcmp(sym->name, "srso_safe_ret"); | ||
| 831 | } | 834 | } |
| @@ -389,7 +389,7 @@ static int decode_instructions(struct objtool_file *file) · 388 unmodified lines | |||
| 389 | if (!strcmp(sec->name, ".noinstr.text") || | 389 | if (!strcmp(sec->name, ".noinstr.text") || |
| 390 | !strcmp(sec->name, ".entry.text") || | 390 | !strcmp(sec->name, ".entry.text") || |
| 391 | !strcmp(sec->name, ".cpuidle.text") || | 391 | !strcmp(sec->name, ".cpuidle.text") || |
| 392 | !strncmp(sec->name, ".text.__x86.", 12)) | 392 | !strncmp(sec->name, ".text..__x86.", 13)) |
| 393 | sec->noinstr = true; | 393 | sec->noinstr = true; |
| 394 | 394 | ||
| 395 | /* | 395 | /* |
| @@ -455,7 +455,7 @@ static int decode_instructions(struct objtool_file *file) · 59 unmodified lines | |||
| 455 | return -1; | 455 | return -1; |
| 456 | } | 456 | } |
| 457 | 457 | ||
| 458 | if (func->return_thunk || func->alias != func) | 458 | if (func->embedded_insn || func->alias != func) |
| 459 | continue; | 459 | continue; |
| 460 | 460 | ||
| 461 | if (!find_insn(file, sec, func->offset)) { | 461 | if (!find_insn(file, sec, func->offset)) { |
| @@ -1288,16 +1288,33 @@ static int add_ignore_alternatives(struct objtool_file *file) · 826 unmodified lines | |||
| 1288 | return 0; | 1288 | return 0; |
| 1289 | } | 1289 | } |
| 1290 | 1290 | ||
| 1291 | /* | ||
| 1292 | * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol | ||
| 1293 | * will be added to the .retpoline_sites section. | ||
| 1294 | */ | ||
| 1291 | __weak bool arch_is_retpoline(struct symbol *sym) | 1295 | __weak bool arch_is_retpoline(struct symbol *sym) |
| 1292 | { | 1296 | { |
| 1293 | return false; | 1297 | return false; |
| 1294 | } | 1298 | } |
| 1295 | 1299 | ||
| 1300 | /* | ||
| 1301 | * Symbols that replace INSN_RETURN, every (tail) call to such a symbol | ||
| 1302 | * will be added to the .return_sites section. | ||
| 1303 | */ | ||
| 1296 | __weak bool arch_is_rethunk(struct symbol *sym) | 1304 | __weak bool arch_is_rethunk(struct symbol *sym) |
| 1297 | { | 1305 | { |
| 1298 | return false; | 1306 | return false; |
| 1299 | } | 1307 | } |
| 1300 | 1308 | ||
| 1309 | /* | ||
| 1310 | * Symbols that are embedded inside other instructions, because sometimes crazy | ||
| 1311 | * code exists. These are mostly ignored for validation purposes. | ||
| 1312 | */ | ||
| 1313 | __weak bool arch_is_embedded_insn(struct symbol *sym) | ||
| 1314 | { | ||
| 1315 | return false; | ||
| 1316 | } | ||
| 1317 | |||
| 1301 | static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn) | 1318 | static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn) |
| 1302 | { | 1319 | { |
| 1303 | struct reloc *reloc; | 1320 | struct reloc *reloc; |
| @@ -1576,14 +1593,14 @@ static int add_jump_destinations(struct objtool_file *file) · 272 unmodified lines | |||
| 1576 | struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off); | 1593 | struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off); |
| 1577 | 1594 | ||
| 1578 | /* | 1595 | /* |
| 1579 | * This is a special case for zen_untrain_ret(). | 1596 | * This is a special case for retbleed_untrain_ret(). |
| 1580 | * It jumps to __x86_return_thunk(), but objtool | 1597 | * It jumps to __x86_return_thunk(), but objtool |
| 1581 | * can't find the thunk's starting RET | 1598 | * can't find the thunk's starting RET |
| 1582 | * instruction, because the RET is also in the | 1599 | * instruction, because the RET is also in the |
| 1583 | * middle of another instruction. Objtool only | 1600 | * middle of another instruction. Objtool only |
| 1584 | * knows about the outer instruction. | 1601 | * knows about the outer instruction. |
| 1585 | */ | 1602 | */ |
| 1586 | if (sym && sym->return_thunk) { | 1603 | if (sym && sym->embedded_insn) { |
| 1587 | add_return_call(file, insn, false); | 1604 | add_return_call(file, insn, false); |
| 1588 | continue; | 1605 | continue; |
| 1589 | } | 1606 | } |
| @@ -2502,6 +2519,9 @@ static int classify_symbols(struct objtool_file *file) · 912 unmodified lines | |||
| 2502 | if (arch_is_rethunk(func)) | 2519 | if (arch_is_rethunk(func)) |
| 2503 | func->return_thunk = true; | 2520 | func->return_thunk = true; |
| 2504 | 2521 | ||
| 2522 | if (arch_is_embedded_insn(func)) | ||
| 2523 | func->embedded_insn = true; | ||
| 2524 | |||
| 2505 | if (arch_ftrace_match(func->name)) | 2525 | if (arch_ftrace_match(func->name)) |
| 2506 | func->fentry = true; | 2526 | func->fentry = true; |
| 2507 | 2527 | ||
| @@ -2630,12 +2650,17 @@ static int decode_sections(struct objtool_file *file) · 122 unmodified lines | |||
| 2630 | return 0; | 2650 | return 0; |
| 2631 | } | 2651 | } |
| 2632 | 2652 | ||
| 2633 | static bool is_fentry_call(struct instruction *insn) | 2653 | static bool is_special_call(struct instruction *insn) |
| 2634 | { | 2654 | { |
| 2635 | if (insn->type == INSN_CALL && | 2655 | if (insn->type == INSN_CALL) { |
| 2636 | insn_call_dest(insn) && | 2656 | struct symbol *dest = insn_call_dest(insn); |
| 2637 | insn_call_dest(insn)->fentry) | 2657 | |
| 2638 | return true; | 2658 | if (!dest) |
| 2659 | return false; | ||
| 2660 | |||
| 2661 | if (dest->fentry || dest->embedded_insn) | ||
| 2662 | return true; | ||
| 2663 | } | ||
| 2639 | 2664 | ||
| 2640 | return false; | 2665 | return false; |
| 2641 | } | 2666 | } |
| @@ -3636,7 +3661,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, · 994 unmodified lines | |||
| 3636 | if (ret) | 3661 | if (ret) |
| 3637 | return ret; | 3662 | return ret; |
| 3638 | 3663 | ||
| 3639 | if (opts.stackval && func && !is_fentry_call(insn) && | 3664 | if (opts.stackval && func && !is_special_call(insn) && |
| 3640 | !has_valid_stack_frame(&state)) { | 3665 | !has_valid_stack_frame(&state)) { |
| 3641 | WARN_INSN(insn, "call without frame pointer save/setup"); | 3666 | WARN_INSN(insn, "call without frame pointer save/setup"); |
| 3642 | return 1; | 3667 | return 1; |
| @@ -90,6 +90,7 @@ int arch_decode_hint_reg(u8 sp_reg, int *base); · 89 unmodified lines | |||
| 90 | 90 | ||
| 91 | bool arch_is_retpoline(struct symbol *sym); | 91 | bool arch_is_retpoline(struct symbol *sym); |
| 92 | bool arch_is_rethunk(struct symbol *sym); | 92 | bool arch_is_rethunk(struct symbol *sym); |
| 93 | bool arch_is_embedded_insn(struct symbol *sym); | ||
| 93 | 94 | ||
| 94 | int arch_rewrite_retpolines(struct objtool_file *file); | 95 | int arch_rewrite_retpolines(struct objtool_file *file); |
| 95 | 96 | ||
| @@ -66,6 +66,7 @@ struct symbol { · 65 unmodified lines | |||
| 66 | u8 fentry : 1; | 66 | u8 fentry : 1; |
| 67 | u8 profiling_func : 1; | 67 | u8 profiling_func : 1; |
| 68 | u8 warned : 1; | 68 | u8 warned : 1; |
| 69 | u8 embedded_insn : 1; | ||
| 69 | struct list_head pv_target; | 70 | struct list_head pv_target; |
| 70 | struct reloc *relocs; | 71 | struct reloc *relocs; |
| 71 | }; | 72 | }; |
| @@ -1038,9 +1038,7 @@ static int thread_stack__trace_end(struct thread_stack *ts, · 1037 unmodified lines | |||
| 1038 | 1038 | ||
| 1039 | static bool is_x86_retpoline(const char *name) | 1039 | static bool is_x86_retpoline(const char *name) |
| 1040 | { | 1040 | { |
| 1041 | const char *p = strstr(name, "__x86_indirect_thunk_"); | 1041 | return strstr(name, "__x86_indirect_thunk_") == name; |
| 1042 | |||
| 1043 | return p == name || !strcmp(name, "__indirect_thunk_start"); | ||
| 1044 | } | 1042 | } |
| 1045 | 1043 | ||
| 1046 | /* | 1044 | /* |