History

x86: Add straight-line-speculation mitigation

Make use of an upcoming GCC feature to mitigate straight-line-speculation for x86: https://gcc.gnu.org/g:53a643f8568067d7700a9f2facc8ba39974973d3 https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102952 https://bugs.llvm.org/show_bug.cgi?id=52323 It's built tested on x86_64-allyesconfig using GCC-12 and GCC-11. Maintenance overhead of this should be fairly low due to objtool validation. Size overhead of all these additional int3 instructions comes to: text data bss dec hex filename 22267751 6933356 2011368 31212475 1dc43bb defconfig-build/vmlinux 22804126 6933356 1470696 31208178 1dc32f2 defconfig-build/vmlinux.sls Or roughly 2.4% additional text.

Peter Zijlstra committed 4 years ago

Commit e463a091 parent 26c44b710 files · +38 −7Browse files

Unified · Split

10 files
  1. arch/x86/Kconfig +12 −0
  2. arch/x86/Makefile +4 −0
  3. arch/x86/include/asm/linkage.h +10 −0
  4. arch/x86/include/asm/static_call.h +1 −1
  5. arch/x86/kernel/ftrace.c +1 −1
  6. arch/x86/kernel/static_call.c +3 −2
  7. arch/x86/lib/memmove_64.S +1 −1
  8. arch/x86/lib/retpoline.S +1 −1
  9. scripts/Makefile.build +2 −1
  10. scripts/link-vmlinux.sh +3 −0
arch/x86/Kconfig+12−0
@@ -472,6 +472,18 @@ config RETPOLINE · 471 unmodified lines
472 branches. Requires a compiler with -mindirect-branch=thunk-extern472 branches. Requires a compiler with -mindirect-branch=thunk-extern
473 support for full protection. The kernel may run slower.473 support for full protection. The kernel may run slower.
474 474
475config CC_HAS_SLS
476 def_bool $(cc-option,-mharden-sls=all)
477
478config SLS
479 bool "Mitigate Straight-Line-Speculation"
480 depends on CC_HAS_SLS && X86_64
481 default n
482 help
483 Compile the kernel with straight-line-speculation options to guard
484 against straight line speculation. The kernel image might be slightly
485 larger.
486
475config X86_CPU_RESCTRL487config X86_CPU_RESCTRL
476 bool "x86 CPU resource control support"488 bool "x86 CPU resource control support"
477 depends on X86 && (CPU_SUP_INTEL || CPU_SUP_AMD)489 depends on X86 && (CPU_SUP_INTEL || CPU_SUP_AMD)
arch/x86/Makefile+4−0
@@ -191,6 +191,10 @@ ifdef CONFIG_RETPOLINE · 190 unmodified lines
191 endif191 endif
192endif192endif
193 193
194ifdef CONFIG_SLS
195 KBUILD_CFLAGS += -mharden-sls=all
196endif
197
194KBUILD_LDFLAGS += -m elf_$(UTS_MACHINE)198KBUILD_LDFLAGS += -m elf_$(UTS_MACHINE)
195 199
196ifdef CONFIG_LTO_CLANG200ifdef CONFIG_LTO_CLANG
arch/x86/include/asm/linkage.h+10−0
@@ -18,9 +18,19 @@ · 17 unmodified lines
18#define __ALIGN_STR __stringify(__ALIGN)18#define __ALIGN_STR __stringify(__ALIGN)
19#endif19#endif
20 20
21#ifdef CONFIG_SLS
22#define RET ret; int3
23#else
24#define RET ret
25#endif
26
21#else /* __ASSEMBLY__ */27#else /* __ASSEMBLY__ */
22 28
29#ifdef CONFIG_SLS
30#define ASM_RET "ret; int3\n\t"
31#else
23#define ASM_RET "ret\n\t"32#define ASM_RET "ret\n\t"
33#endif
24 34
25#endif /* __ASSEMBLY__ */35#endif /* __ASSEMBLY__ */
26 36
arch/x86/include/asm/static_call.h+1−1
@@ -36,7 +36,7 @@ · 35 unmodified lines
36 __ARCH_DEFINE_STATIC_CALL_TRAMP(name, ".byte 0xe9; .long " #func " - (. + 4)")36 __ARCH_DEFINE_STATIC_CALL_TRAMP(name, ".byte 0xe9; .long " #func " - (. + 4)")
37 37
38#define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) \38#define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) \
39 __ARCH_DEFINE_STATIC_CALL_TRAMP(name, "ret; nop; nop; nop; nop")39 __ARCH_DEFINE_STATIC_CALL_TRAMP(name, "ret; int3; nop; nop; nop")
40 40
41 41
42#define ARCH_ADD_TRAMP_KEY(name) \42#define ARCH_ADD_TRAMP_KEY(name) \
arch/x86/kernel/ftrace.c+1−1
@@ -303,7 +303,7 @@ union ftrace_op_code_union { · 302 unmodified lines
303 } __attribute__((packed));303 } __attribute__((packed));
304};304};
305 305
306#define RET_SIZE 1306#define RET_SIZE 1 + IS_ENABLED(CONFIG_SLS)
307 307
308static unsigned long308static unsigned long
309create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)309create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
arch/x86/kernel/static_call.c+3−2
@@ -17,6 +17,8 @@ enum insn_type { · 16 unmodified lines
17 */17 */
18static const u8 xor5rax[] = { 0x66, 0x66, 0x48, 0x31, 0xc0 };18static const u8 xor5rax[] = { 0x66, 0x66, 0x48, 0x31, 0xc0 };
19 19
20static const u8 retinsn[] = { RET_INSN_OPCODE, 0xcc, 0xcc, 0xcc, 0xcc };
21
20static void __ref __static_call_transform(void *insn, enum insn_type type, void *func)22static void __ref __static_call_transform(void *insn, enum insn_type type, void *func)
21{23{
22 const void *emulate = NULL;24 const void *emulate = NULL;
@@ -42,8 +44,7 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, void · 19 unmodified lines
42 break;44 break;
43 45
44 case RET:46 case RET:
45 code = text_gen_insn(RET_INSN_OPCODE, insn, func);47 code = &retinsn;
46 size = RET_INSN_SIZE;
47 break;48 break;
48 }49 }
49 50
arch/x86/lib/memmove_64.S+1−1
@@ -40,7 +40,7 @@ SYM_FUNC_START(__memmove) · 39 unmodified lines
40 /* FSRM implies ERMS => no length checks, do the copy directly */40 /* FSRM implies ERMS => no length checks, do the copy directly */
41.Lmemmove_begin_forward:41.Lmemmove_begin_forward:
42 ALTERNATIVE "cmp $0x20, %rdx; jb 1f", "", X86_FEATURE_FSRM42 ALTERNATIVE "cmp $0x20, %rdx; jb 1f", "", X86_FEATURE_FSRM
43 ALTERNATIVE "", "movq %rdx, %rcx; rep movsb; RET", X86_FEATURE_ERMS43 ALTERNATIVE "", __stringify(movq %rdx, %rcx; rep movsb; RET), X86_FEATURE_ERMS
44 44
45 /*45 /*
46 * movsq instruction have many startup latency46 * movsq instruction have many startup latency
arch/x86/lib/retpoline.S+1−1
@@ -34,7 +34,7 @@ SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) · 33 unmodified lines
34 34
35 ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \35 ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \
36 __stringify(RETPOLINE \reg), X86_FEATURE_RETPOLINE, \36 __stringify(RETPOLINE \reg), X86_FEATURE_RETPOLINE, \
37 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_AMD37 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg; int3), X86_FEATURE_RETPOLINE_AMD
38 38
39.endm39.endm
40 40
scripts/Makefile.build+2−1
@@ -234,7 +234,8 @@ objtool_args = \ · 233 unmodified lines
234 $(if $(CONFIG_GCOV_KERNEL)$(CONFIG_LTO_CLANG), --no-unreachable)\234 $(if $(CONFIG_GCOV_KERNEL)$(CONFIG_LTO_CLANG), --no-unreachable)\
235 $(if $(CONFIG_RETPOLINE), --retpoline) \235 $(if $(CONFIG_RETPOLINE), --retpoline) \
236 $(if $(CONFIG_X86_SMAP), --uaccess) \236 $(if $(CONFIG_X86_SMAP), --uaccess) \
237 $(if $(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL), --mcount)237 $(if $(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL), --mcount) \
238 $(if $(CONFIG_SLS), --sls)
238 239
239cmd_objtool = $(if $(objtool-enabled), ; $(objtool) $(objtool_args) $@)240cmd_objtool = $(if $(objtool-enabled), ; $(objtool) $(objtool_args) $@)
240cmd_gen_objtooldep = $(if $(objtool-enabled), { echo ; echo '$@: $$(wildcard $(objtool))' ; } >> $(dot-target).cmd)241cmd_gen_objtooldep = $(if $(objtool-enabled), { echo ; echo '$@: $$(wildcard $(objtool))' ; } >> $(dot-target).cmd)
scripts/link-vmlinux.sh+3−0
@@ -139,6 +139,9 @@ objtool_link() · 138 unmodified lines
139 if [ -n "${CONFIG_X86_SMAP}" ]; then139 if [ -n "${CONFIG_X86_SMAP}" ]; then
140 objtoolopt="${objtoolopt} --uaccess"140 objtoolopt="${objtoolopt} --uaccess"
141 fi141 fi
142 if [ -n "${CONFIG_SLS}" ]; then
143 objtoolopt="${objtoolopt} --sls"
144 fi
142 info OBJTOOL ${1}145 info OBJTOOL ${1}
143 tools/objtool/objtool ${objtoolcmd} ${objtoolopt} ${1}146 tools/objtool/objtool ${objtoolcmd} ${objtoolopt} ${1}
144 fi147 fi