History

Merge branch 'x86-pti-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 pti bits and fixes from Thomas Gleixner: "This last update contains: - An objtool fix to prevent a segfault with the gold linker by changing the invocation order. That's not just for gold, it's a general robustness improvement. - An improved error message for objtool which spares tearing hairs. - Make KASAN fail loudly if there is not enough memory instead of oopsing at some random place later - RSB fill on context switch to prevent RSB underflow and speculation through other units. - Make the retpoline/RSB functionality work reliably for both Intel and AMD - Add retpoline to the module version magic so mismatch can be detected - A small (non-fix) update for cpufeatures which prevents cpu feature clashing for the upcoming extra mitigation bits to ease backporting" * 'x86-pti-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: module: Add retpoline tag to VERMAGIC x86/cpufeature: Move processor tracing out of scattered features objtool: Improve error message for bad file argument objtool: Fix seg fault with gold linker x86/retpoline: Add LFENCE to the retpoline/RSB filling RSB macros x86/retpoline: Fill RSB on context switch for affected CPUs x86/kasan: Panic if there is not enough memory to boot

Linus Torvalds committed 8 years ago

Commit 88dc7fc2 parents dd43f34 6cfb52110 files · +99 −19Browse files

Unified · Split

10 files
  1. arch/x86/entry/entry_32.S +11 −0
  2. arch/x86/entry/entry_64.S +11 −0
  3. arch/x86/include/asm/cpufeatures.h +2 −1
  4. arch/x86/include/asm/nospec-branch.h +5 −1
  5. arch/x86/kernel/cpu/bugs.c +36 −0
  6. arch/x86/kernel/cpu/scattered.c +0 −1
  7. arch/x86/mm/kasan_init_64.c +14 −10
  8. include/linux/vermagic.h +7 −1
  9. scripts/Makefile.build +10 −4
  10. tools/objtool/elf.c +3 −1
arch/x86/entry/entry_32.S+11−0
@@ -244,6 +244,17 @@ ENTRY(__switch_to_asm) · 243 unmodified lines
244244 movl %ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset
245245 #endif
246246
247+#ifdef CONFIG_RETPOLINE
248+ /*
249+ * When switching from a shallower to a deeper call stack
250+ * the RSB may either underflow or use entries populated
251+ * with userspace addresses. On CPUs where those concerns
252+ * exist, overwrite the RSB with entries which capture
253+ * speculative execution to prevent attack.
254+ */
255+ FILL_RETURN_BUFFER %ebx, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
256+#endif
257+
247258 /* restore callee-saved registers */
248259 popl %esi
249260 popl %edi
arch/x86/entry/entry_64.S+11−0
@@ -491,6 +491,17 @@ ENTRY(__switch_to_asm) · 490 unmodified lines
491491 movq %rbx, PER_CPU_VAR(irq_stack_union)+stack_canary_offset
492492 #endif
493493
494+#ifdef CONFIG_RETPOLINE
495+ /*
496+ * When switching from a shallower to a deeper call stack
497+ * the RSB may either underflow or use entries populated
498+ * with userspace addresses. On CPUs where those concerns
499+ * exist, overwrite the RSB with entries which capture
500+ * speculative execution to prevent attack.
501+ */
502+ FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
503+#endif
504+
494505 /* restore callee-saved registers */
495506 popq %r15
496507 popq %r14
arch/x86/include/asm/cpufeatures.h+2−1
@@ -206,11 +206,11 @@ · 205 unmodified lines
206206 #define X86_FEATURE_RETPOLINE ( 7*32+12) /* Generic Retpoline mitigation for Spectre variant 2 */
207207 #define X86_FEATURE_RETPOLINE_AMD ( 7*32+13) /* AMD Retpoline mitigation for Spectre variant 2 */
208208 #define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory Number */
209−#define X86_FEATURE_INTEL_PT ( 7*32+15) /* Intel Processor Trace */
210209 #define X86_FEATURE_AVX512_4VNNIW ( 7*32+16) /* AVX-512 Neural Network Instructions */
211210 #define X86_FEATURE_AVX512_4FMAPS ( 7*32+17) /* AVX-512 Multiply Accumulation Single precision */
212211
213212 #define X86_FEATURE_MBA ( 7*32+18) /* Memory Bandwidth Allocation */
213+#define X86_FEATURE_RSB_CTXSW ( 7*32+19) /* Fill RSB on context switches */
214214
215215 /* Virtualization flags: Linux defined, word 8 */
216216 #define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */
@@ -245,6 +245,7 @@ · 28 unmodified lines
245245 #define X86_FEATURE_AVX512IFMA ( 9*32+21) /* AVX-512 Integer Fused Multiply-Add instructions */
246246 #define X86_FEATURE_CLFLUSHOPT ( 9*32+23) /* CLFLUSHOPT instruction */
247247 #define X86_FEATURE_CLWB ( 9*32+24) /* CLWB instruction */
248+#define X86_FEATURE_INTEL_PT ( 9*32+25) /* Intel Processor Trace */
248249 #define X86_FEATURE_AVX512PF ( 9*32+26) /* AVX-512 Prefetch */
249250 #define X86_FEATURE_AVX512ER ( 9*32+27) /* AVX-512 Exponential and Reciprocal */
250251 #define X86_FEATURE_AVX512CD ( 9*32+28) /* AVX-512 Conflict Detection */
arch/x86/include/asm/nospec-branch.h+5−1
@@ -11,7 +11,7 @@ · 10 unmodified lines
1111 * Fill the CPU return stack buffer.
1212 *
1313 * Each entry in the RSB, if used for a speculative 'ret', contains an
14− * infinite 'pause; jmp' loop to capture speculative execution.
14+ * infinite 'pause; lfence; jmp' loop to capture speculative execution.
1515 *
1616 * This is required in various cases for retpoline and IBRS-based
1717 * mitigations for the Spectre variant 2 vulnerability. Sometimes to
@@ -38,11 +38,13 @@ · 20 unmodified lines
3838 call 772f; \
3939 773: /* speculation trap */ \
4040 pause; \
41+ lfence; \
4142 jmp 773b; \
4243 772: \
4344 call 774f; \
4445 775: /* speculation trap */ \
4546 pause; \
47+ lfence; \
4648 jmp 775b; \
4749 774: \
4850 dec reg; \
@@ -73,6 +75,7 @@ · 24 unmodified lines
7375 call .Ldo_rop_\@
7476 .Lspec_trap_\@:
7577 pause
78+ lfence
7679 jmp .Lspec_trap_\@
7780 .Ldo_rop_\@:
7881 mov \reg, (%_ASM_SP)
@@ -165,6 +168,7 @@ · 86 unmodified lines
165168 " .align 16\n" \
166169 "901: call 903f;\n" \
167170 "902: pause;\n" \
171+ " lfence;\n" \
168172 " jmp 902b;\n" \
169173 " .align 16\n" \
170174 "903: addl $4, %%esp;\n" \
arch/x86/kernel/cpu/bugs.c+36−0
@@ -23,6 +23,7 @@ · 22 unmodified lines
2323 #include <asm/alternative.h>
2424 #include <asm/pgtable.h>
2525 #include <asm/set_memory.h>
26+#include <asm/intel-family.h>
2627
2728 static void __init spectre_v2_select_mitigation(void);
2829
@@ -155,6 +156,23 @@ disable: · 126 unmodified lines
155156 return SPECTRE_V2_CMD_NONE;
156157 }
157158
159+/* Check for Skylake-like CPUs (for RSB handling) */
160+static bool __init is_skylake_era(void)
161+{
162+ if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
163+ boot_cpu_data.x86 == 6) {
164+ switch (boot_cpu_data.x86_model) {
165+ case INTEL_FAM6_SKYLAKE_MOBILE:
166+ case INTEL_FAM6_SKYLAKE_DESKTOP:
167+ case INTEL_FAM6_SKYLAKE_X:
168+ case INTEL_FAM6_KABYLAKE_MOBILE:
169+ case INTEL_FAM6_KABYLAKE_DESKTOP:
170+ return true;
171+ }
172+ }
173+ return false;
174+}
175+
158176 static void __init spectre_v2_select_mitigation(void)
159177 {
160178 enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
@@ -213,6 +231,24 @@ retpoline_auto: · 52 unmodified lines
213231
214232 spectre_v2_enabled = mode;
215233 pr_info("%s\n", spectre_v2_strings[mode]);
234+
235+ /*
236+ * If neither SMEP or KPTI are available, there is a risk of
237+ * hitting userspace addresses in the RSB after a context switch
238+ * from a shallow call stack to a deeper one. To prevent this fill
239+ * the entire RSB, even when using IBRS.
240+ *
241+ * Skylake era CPUs have a separate issue with *underflow* of the
242+ * RSB, when they will predict 'ret' targets from the generic BTB.
243+ * The proper mitigation for this is IBRS. If IBRS is not supported
244+ * or deactivated in favour of retpolines the RSB fill on context
245+ * switch is required.
246+ */
247+ if ((!boot_cpu_has(X86_FEATURE_PTI) &&
248+ !boot_cpu_has(X86_FEATURE_SMEP)) || is_skylake_era()) {
249+ setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
250+ pr_info("Filling RSB on context switch\n");
251+ }
216252 }
217253
218254 #undef pr_fmt
arch/x86/kernel/cpu/scattered.c+0−1
@@ -21,7 +21,6 @@ struct cpuid_bit { · 20 unmodified lines
2121 static const struct cpuid_bit cpuid_bits[] = {
2222 { X86_FEATURE_APERFMPERF, CPUID_ECX, 0, 0x00000006, 0 },
2323 { X86_FEATURE_EPB, CPUID_ECX, 3, 0x00000006, 0 },
24− { X86_FEATURE_INTEL_PT, CPUID_EBX, 25, 0x00000007, 0 },
2524 { X86_FEATURE_AVX512_4VNNIW, CPUID_EDX, 2, 0x00000007, 0 },
2625 { X86_FEATURE_AVX512_4FMAPS, CPUID_EDX, 3, 0x00000007, 0 },
2726 { X86_FEATURE_CAT_L3, CPUID_EBX, 1, 0x00000010, 0 },
arch/x86/mm/kasan_init_64.c+14−10
@@ -21,10 +21,14 @@ extern struct range pfn_mapped[E820_MAX_ENTRIES]; · 20 unmodified lines
2121
2222 static p4d_t tmp_p4d_table[PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE);
2323
24−static __init void *early_alloc(size_t size, int nid)
24+static __init void *early_alloc(size_t size, int nid, bool panic)
2525 {
26− return memblock_virt_alloc_try_nid_nopanic(size, size,
27− __pa(MAX_DMA_ADDRESS), BOOTMEM_ALLOC_ACCESSIBLE, nid);
26+ if (panic)
27+ return memblock_virt_alloc_try_nid(size, size,
28+ __pa(MAX_DMA_ADDRESS), BOOTMEM_ALLOC_ACCESSIBLE, nid);
29+ else
30+ return memblock_virt_alloc_try_nid_nopanic(size, size,
31+ __pa(MAX_DMA_ADDRESS), BOOTMEM_ALLOC_ACCESSIBLE, nid);
2832 }
2933
3034 static void __init kasan_populate_pmd(pmd_t *pmd, unsigned long addr,
@@ -38,14 +42,14 @@ static void __init kasan_populate_pmd(pmd_t *pmd, unsigned long addr, · 7 unmodified lines
3842 if (boot_cpu_has(X86_FEATURE_PSE) &&
3943 ((end - addr) == PMD_SIZE) &&
4044 IS_ALIGNED(addr, PMD_SIZE)) {
41− p = early_alloc(PMD_SIZE, nid);
45+ p = early_alloc(PMD_SIZE, nid, false);
4246 if (p && pmd_set_huge(pmd, __pa(p), PAGE_KERNEL))
4347 return;
4448 else if (p)
4549 memblock_free(__pa(p), PMD_SIZE);
4650 }
4751
48− p = early_alloc(PAGE_SIZE, nid);
52+ p = early_alloc(PAGE_SIZE, nid, true);
4953 pmd_populate_kernel(&init_mm, pmd, p);
5054 }
5155
@@ -57,7 +61,7 @@ static void __init kasan_populate_pmd(pmd_t *pmd, unsigned long addr, · 5 unmodified lines
5761 if (!pte_none(*pte))
5862 continue;
5963
60− p = early_alloc(PAGE_SIZE, nid);
64+ p = early_alloc(PAGE_SIZE, nid, true);
6165 entry = pfn_pte(PFN_DOWN(__pa(p)), PAGE_KERNEL);
6266 set_pte_at(&init_mm, addr, pte, entry);
6367 } while (pte++, addr += PAGE_SIZE, addr != end);
@@ -75,14 +79,14 @@ static void __init kasan_populate_pud(pud_t *pud, unsigned long addr, · 11 unmodified lines
7579 if (boot_cpu_has(X86_FEATURE_GBPAGES) &&
7680 ((end - addr) == PUD_SIZE) &&
7781 IS_ALIGNED(addr, PUD_SIZE)) {
78− p = early_alloc(PUD_SIZE, nid);
82+ p = early_alloc(PUD_SIZE, nid, false);
7983 if (p && pud_set_huge(pud, __pa(p), PAGE_KERNEL))
8084 return;
8185 else if (p)
8286 memblock_free(__pa(p), PUD_SIZE);
8387 }
8488
85− p = early_alloc(PAGE_SIZE, nid);
89+ p = early_alloc(PAGE_SIZE, nid, true);
8690 pud_populate(&init_mm, pud, p);
8791 }
8892
@@ -101,7 +105,7 @@ static void __init kasan_populate_p4d(p4d_t *p4d, unsigned long addr, · 12 unmodified lines
101105 unsigned long next;
102106
103107 if (p4d_none(*p4d)) {
104− void *p = early_alloc(PAGE_SIZE, nid);
108+ void *p = early_alloc(PAGE_SIZE, nid, true);
105109
106110 p4d_populate(&init_mm, p4d, p);
107111 }
@@ -122,7 +126,7 @@ static void __init kasan_populate_pgd(pgd_t *pgd, unsigned long addr, · 14 unmodified lines
122126 unsigned long next;
123127
124128 if (pgd_none(*pgd)) {
125− p = early_alloc(PAGE_SIZE, nid);
129+ p = early_alloc(PAGE_SIZE, nid, true);
126130 pgd_populate(&init_mm, pgd, p);
127131 }
128132
include/linux/vermagic.h+7−1
@@ -31,11 +31,17 @@ · 30 unmodified lines
3131 #else
3232 #define MODULE_RANDSTRUCT_PLUGIN
3333 #endif
34+#ifdef RETPOLINE
35+#define MODULE_VERMAGIC_RETPOLINE "retpoline "
36+#else
37+#define MODULE_VERMAGIC_RETPOLINE ""
38+#endif
3439
3540 #define VERMAGIC_STRING \
3641 UTS_RELEASE " " \
3742 MODULE_VERMAGIC_SMP MODULE_VERMAGIC_PREEMPT \
3843 MODULE_VERMAGIC_MODULE_UNLOAD MODULE_VERMAGIC_MODVERSIONS \
3944 MODULE_ARCH_VERMAGIC \
40− MODULE_RANDSTRUCT_PLUGIN
45+ MODULE_RANDSTRUCT_PLUGIN \
46+ MODULE_VERMAGIC_RETPOLINE
4147
scripts/Makefile.build+10−4
@@ -265,12 +265,18 @@ else · 264 unmodified lines
265265 objtool_args += $(call cc-ifversion, -lt, 0405, --no-unreachable)
266266 endif
267267
268+ifdef CONFIG_MODVERSIONS
269+objtool_o = $(@D)/.tmp_$(@F)
270+else
271+objtool_o = $(@)
272+endif
273+
268274 # 'OBJECT_FILES_NON_STANDARD := y': skip objtool checking for a directory
269275 # 'OBJECT_FILES_NON_STANDARD_foo.o := 'y': skip objtool checking for a file
270276 # 'OBJECT_FILES_NON_STANDARD_foo.o := 'n': override directory skip for a file
271277 cmd_objtool = $(if $(patsubst y%,, \
272278 $(OBJECT_FILES_NON_STANDARD_$(basetarget).o)$(OBJECT_FILES_NON_STANDARD)n), \
273− $(__objtool_obj) $(objtool_args) "$(@)";)
279+ $(__objtool_obj) $(objtool_args) "$(objtool_o)";)
274280 objtool_obj = $(if $(patsubst y%,, \
275281 $(OBJECT_FILES_NON_STANDARD_$(basetarget).o)$(OBJECT_FILES_NON_STANDARD)n), \
276282 $(__objtool_obj))
@@ -286,16 +292,16 @@ objtool_dep = $(objtool_obj) \ · 9 unmodified lines
286292 define rule_cc_o_c
287293 $(call echo-cmd,checksrc) $(cmd_checksrc) \
288294 $(call cmd_and_fixdep,cc_o_c) \
289− $(cmd_modversions_c) \
290295 $(cmd_checkdoc) \
291296 $(call echo-cmd,objtool) $(cmd_objtool) \
297+ $(cmd_modversions_c) \
292298 $(call echo-cmd,record_mcount) $(cmd_record_mcount)
293299 endef
294300
295301 define rule_as_o_S
296302 $(call cmd_and_fixdep,as_o_S) \
297− $(cmd_modversions_S) \
298− $(call echo-cmd,objtool) $(cmd_objtool)
303+ $(call echo-cmd,objtool) $(cmd_objtool) \
304+ $(cmd_modversions_S)
299305 endef
300306
301307 # List module undefined symbols (or empty line if not enabled)
tools/objtool/elf.c+3−1
@@ -26,6 +26,7 @@ · 25 unmodified lines
2626 #include <stdlib.h>
2727 #include <string.h>
2828 #include <unistd.h>
29+#include <errno.h>
2930
3031 #include "elf.h"
3132 #include "warn.h"
@@ -358,7 +359,8 @@ struct elf *elf_open(const char *name, int flags) · 326 unmodified lines
358359
359360 elf->fd = open(name, flags);
360361 if (elf->fd == -1) {
361− perror("open");
362+ fprintf(stderr, "objtool: Can't open '%s': %s\n",
363+ name, strerror(errno));
362364 goto err;
363365 }
364366