Merge branch 'x86-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 fixes from Ingo Molnar: "Various fixes: - Fix the PAT performance regression that downgraded write-combining device memory regions to uncached. - There's been a number of bugs in 32-bit double fault handling - hopefully all fixed now. - Fix an LDT crash - Fix an FPU over-optimization that broke with GCC9 code optimizations. - Misc cleanups" * 'x86-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: x86/mm/pat: Fix off-by-one bugs in interval tree search x86/ioperm: Save an indentation level in tss_update_io_bitmap() x86/fpu: Don't cache access to fpu_fpregs_owner_ctx x86/entry/32: Remove unused 'restore_all_notrace' local label x86/ptrace: Document FSBASE and GSBASE ABI oddities x86/ptrace: Remove set_segment_reg() implementations for current x86/traps: die() instead of panicking on a double fault x86/doublefault/32: Rewrite the x86_32 #DF handler and unify with 64-bit x86/doublefault/32: Move #DF stack and TSS to cpu_entry_area x86/doublefault/32: Rename doublefault.c to doublefault_32.c x86/traps: Disentangle the 32-bit and 64-bit doublefault code lkdtm: Add a DOUBLE_FAULT crash type on x86 selftests/x86/single_step_syscall: Check SYSENTER directly x86/mm/32: Sync only to VMALLOC_END in vmalloc_sync_all()

Linus Torvalds committed 6 years ago

Commit e5b3fc12 parents b7fcf31 91298f123 files · +467 −171Browse files

Unified · Split

23 files

arch/x86/Kconfig.debug +1 −1
arch/x86/entry/entry_32.S +42 −1
arch/x86/include/asm/cpu_entry_area.h +12 −0
arch/x86/include/asm/doublefault.h +13 −0
arch/x86/include/asm/fpu/internal.h +1 −1
arch/x86/include/asm/pgtable_32_types.h +4 −3
arch/x86/include/asm/processor.h +0 −2
arch/x86/include/asm/traps.h +3 −0
arch/x86/kernel/Makefile +3 −1
arch/x86/kernel/cpu/common.c +2 −10
arch/x86/kernel/doublefault.c +0 −86
arch/x86/kernel/doublefault_32.c +136 −0
arch/x86/kernel/dumpstack_32.c +30 −0
arch/x86/kernel/process.c +26 −26
arch/x86/kernel/ptrace.c +24 −12
arch/x86/kernel/traps.c +20 −11
arch/x86/mm/cpu_entry_area.c +13 −1
arch/x86/mm/fault.c +1 −1
arch/x86/mm/pat_interval.c +6 −6
drivers/misc/lkdtm/bugs.c +39 −0
drivers/misc/lkdtm/core.c +3 −0
drivers/misc/lkdtm/lkdtm.h +3 −0
tools/testing/selftests/x86/single_step_syscall.c +85 −9

arch/x86/Kconfig.debug+1−1

@@ -117,7 +117,7 @@ config DEBUG_WX · 116 unmodified lines
117	117
118	118	config DOUBLEFAULT
119	119	default y
120		− bool "Enable doublefault exception handler" if EXPERT
	120	+ bool "Enable doublefault exception handler" if EXPERT && X86_32
121	121	---help---
122	122	This option allows trapping of rare doublefault exceptions that
123	123	would otherwise cause a system to silently reboot. Disabling this

arch/x86/entry/entry_32.S+42−1

@@ -1090,7 +1090,6 @@ SYM_FUNC_START(entry_INT80_32) · 1089 unmodified lines
1090	1090	restore_all:
1091	1091	TRACE_IRQS_IRET
1092	1092	SWITCH_TO_ENTRY_STACK
1093		−.Lrestore_all_notrace:
1094	1093	CHECK_AND_APPLY_ESPFIX
1095	1094	.Lrestore_nocheck:
1096	1095	/* Switch back to user CR3 */
@@ -1537,6 +1536,48 @@ SYM_CODE_START(debug) · 440 unmodified lines
1537	1536	jmp common_exception
1538	1537	SYM_CODE_END(debug)
1539	1538
	1539	+#ifdef CONFIG_DOUBLEFAULT
	1540	+SYM_CODE_START(double_fault)
	1541	+1:
	1542	+ /*
	1543	+ * This is a task gate handler, not an interrupt gate handler.
	1544	+ * The error code is on the stack, but the stack is otherwise
	1545	+ * empty. Interrupts are off. Our state is sane with the following
	1546	+ * exceptions:
	1547	+ *
	1548	+ * - CR0.TS is set. "TS" literally means "task switched".
	1549	+ * - EFLAGS.NT is set because we're a "nested task".
	1550	+ * - The doublefault TSS has back_link set and has been marked busy.
	1551	+ * - TR points to the doublefault TSS and the normal TSS is busy.
	1552	+ * - CR3 is the normal kernel PGD. This would be delightful, except
	1553	+ * that the CPU didn't bother to save the old CR3 anywhere. This
	1554	+ * would make it very awkward to return back to the context we came
	1555	+ * from.
	1556	+ *
	1557	+ * The rest of EFLAGS is sanitized for us, so we don't need to
	1558	+ * worry about AC or DF.
	1559	+ *
	1560	+ * Don't even bother popping the error code. It's always zero,
	1561	+ * and ignoring it makes us a bit more robust against buggy
	1562	+ * hypervisor task gate implementations.
	1563	+ *
	1564	+ * We will manually undo the task switch instead of doing a
	1565	+ * task-switching IRET.
	1566	+ */
	1567	+
	1568	+ clts /* clear CR0.TS */
	1569	+ pushl $X86_EFLAGS_FIXED
	1570	+ popfl /* clear EFLAGS.NT */
	1571	+
	1572	+ call doublefault_shim
	1573	+
	1574	+ /* We don't support returning, so we have no IRET here. */
	1575	+1:
	1576	+ hlt
	1577	+ jmp 1b
	1578	+SYM_CODE_END(double_fault)
	1579	+#endif
	1580	+
1540	1581	/*
1541	1582	* NMI is doubly nasty. It can happen on the first instruction of
1542	1583	* entry_SYSENTER_32 (just like #DB), but it can also interrupt the beginning

arch/x86/include/asm/cpu_entry_area.h+12−0

@@ -65,6 +65,13 @@ enum exception_stack_ordering { · 64 unmodified lines
65	65
66	66	#endif
67	67
	68	+#ifdef CONFIG_X86_32
	69	+struct doublefault_stack {
	70	+ unsigned long stack[(PAGE_SIZE - sizeof(struct x86_hw_tss)) / sizeof(unsigned long)];
	71	+ struct x86_hw_tss tss;
	72	+} __aligned(PAGE_SIZE);
	73	+#endif
	74	+
68	75	/*
69	76	* cpu_entry_area is a percpu region that contains things needed by the CPU
70	77	* and early entry/exit code. Real types aren't used for all fields here
@@ -86,6 +93,11 @@ struct cpu_entry_area { · 15 unmodified lines
86	93	#endif
87	94	struct entry_stack_page entry_stack_page;
88	95
	96	+#ifdef CONFIG_X86_32
	97	+ char guard_doublefault_stack[PAGE_SIZE];
	98	+ struct doublefault_stack doublefault_stack;
	99	+#endif
	100	+
89	101	/*
90	102	* On x86_64, the TSS is mapped RO. On x86_32, it's mapped RW because
91	103	* we need task switches to work, and task switches write to the TSS.

arch/x86/include/asm/doublefault.h+13−0

@@ -0,0 +1,13 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+#ifndef _ASM_X86_DOUBLEFAULT_H
	3	+#define _ASM_X86_DOUBLEFAULT_H
	4	+
	5	+#if defined(CONFIG_X86_32) && defined(CONFIG_DOUBLEFAULT)
	6	+extern void doublefault_init_cpu_tss(void);
	7	+#else
	8	+static inline void doublefault_init_cpu_tss(void)
	9	+{
	10	+}
	11	+#endif
	12	+
	13	+#endif /* _ASM_X86_DOUBLEFAULT_H */

arch/x86/include/asm/fpu/internal.h+1−1

@@ -509,7 +509,7 @@ static inline void __fpu_invalidate_fpregs_state(struct fpu *fpu) · 508 unmodified lines
509	509
510	510	static inline int fpregs_state_valid(struct fpu *fpu, unsigned int cpu)
511	511	{
512		− return fpu == this_cpu_read_stable(fpu_fpregs_owner_ctx) && cpu == fpu->last_cpu;
	512	+ return fpu == this_cpu_read(fpu_fpregs_owner_ctx) && cpu == fpu->last_cpu;
513	513	}
514	514
515	515	/*

arch/x86/include/asm/pgtable_32_types.h+4−3

@@ -41,10 +41,11 @@ extern bool __vmalloc_start_set; /* set once high_memory is set */ · 40 unmodified lines
41	41	#endif
42	42
43	43	/*
44		− * Define this here and validate with BUILD_BUG_ON() in pgtable_32.c
45		− * to avoid include recursion hell
	44	+ * This is an upper bound on sizeof(struct cpu_entry_area) / PAGE_SIZE.
	45	+ * Define this here and validate with BUILD_BUG_ON() in cpu_entry_area.c
	46	+ * to avoid include recursion hell.
46	47	*/
47		−#define CPU_ENTRY_AREA_PAGES (NR_CPUS * 41)
	48	+#define CPU_ENTRY_AREA_PAGES (NR_CPUS * 43)
48	49
49	50	/* The +1 is for the readonly IDT page: */
50	51	#define CPU_ENTRY_AREA_BASE \

arch/x86/include/asm/processor.h+0−2

@@ -166,7 +166,6 @@ enum cpuid_regs_idx { · 165 unmodified lines
166	166	extern struct cpuinfo_x86 boot_cpu_data;
167	167	extern struct cpuinfo_x86 new_cpu_data;
168	168
169		−extern struct x86_hw_tss doublefault_tss;
170	169	extern __u32 cpu_caps_cleared[NCAPINTS + NBUGINTS];
171	170	extern __u32 cpu_caps_set[NCAPINTS + NBUGINTS];
172	171
@@ -997,7 +996,6 @@ bool xen_set_default_idle(void); · 824 unmodified lines
997	996	#endif
998	997
999	998	void stop_this_cpu(void *dummy);
1000		−void df_debug(struct pt_regs *regs, long error_code);
1001	999	void microcode_check(void);
1002	1000
1003	1001	enum l1tf_mitigations {

arch/x86/include/asm/traps.h+3−0

@@ -69,6 +69,9 @@ dotraplinkage void do_overflow(struct pt_regs *regs, long error_code); · 68 unmodified lines
69	69	dotraplinkage void do_bounds(struct pt_regs *regs, long error_code);
70	70	dotraplinkage void do_invalid_op(struct pt_regs *regs, long error_code);
71	71	dotraplinkage void do_device_not_available(struct pt_regs *regs, long error_code);
	72	+#if defined(CONFIG_X86_64) \|\| defined(CONFIG_DOUBLEFAULT)
	73	+dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsigned long cr2);
	74	+#endif
72	75	dotraplinkage void do_coprocessor_segment_overrun(struct pt_regs *regs, long error_code);
73	76	dotraplinkage void do_invalid_TSS(struct pt_regs *regs, long error_code);
74	77	dotraplinkage void do_segment_not_present(struct pt_regs *regs, long error_code);

arch/x86/kernel/Makefile+3−1

@@ -100,7 +100,9 @@ obj-$(CONFIG_KEXEC_FILE) += kexec-bzimage64.o · 99 unmodified lines
100	100	obj-$(CONFIG_CRASH_DUMP) += crash_dump_$(BITS).o
101	101	obj-y += kprobes/
102	102	obj-$(CONFIG_MODULES) += module.o
103		−obj-$(CONFIG_DOUBLEFAULT) += doublefault.o
	103	+ifeq ($(CONFIG_X86_32),y)
	104	+obj-$(CONFIG_DOUBLEFAULT) += doublefault_32.o
	105	+endif
104	106	obj-$(CONFIG_KGDB) += kgdb.o
105	107	obj-$(CONFIG_VM86) += vm86_32.o
106	108	obj-$(CONFIG_EARLY_PRINTK) += early_printk.o

arch/x86/kernel/cpu/common.c+2−10

@@ -24,6 +24,7 @@ · 23 unmodified lines
24	24	#include <asm/stackprotector.h>
25	25	#include <asm/perf_event.h>
26	26	#include <asm/mmu_context.h>
	27	+#include <asm/doublefault.h>
27	28	#include <asm/archrandom.h>
28	29	#include <asm/hypervisor.h>
29	30	#include <asm/processor.h>
@@ -1814,8 +1815,6 @@ static inline void tss_setup_ist(struct tss_struct *tss) · 1784 unmodified lines
1814	1815	tss->x86_tss.ist[IST_INDEX_MCE] = __this_cpu_ist_top_va(MCE);
1815	1816	}
1816	1817
1817		−static inline void gdt_setup_doublefault_tss(int cpu) { }
1818		−
1819	1818	#else /* CONFIG_X86_64 */
1820	1819
1821	1820	static inline void setup_getcpu(int cpu) { }
@@ -1827,13 +1826,6 @@ static inline void ucode_cpu_init(int cpu) · 5 unmodified lines
1827	1826
1828	1827	static inline void tss_setup_ist(struct tss_struct *tss) { }
1829	1828
1830		−static inline void gdt_setup_doublefault_tss(int cpu)
1831		−{
1832		−#ifdef CONFIG_DOUBLEFAULT
1833		− /* Set up the doublefault TSS pointer in the GDT */
1834		− __set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
1835		−#endif
1836		−}
1837	1829	#endif /* !CONFIG_X86_64 */
1838	1830
1839	1831	static inline void tss_setup_io_bitmap(struct tss_struct *tss)
@@ -1923,7 +1915,7 @@ void cpu_init(void) · 83 unmodified lines
1923	1915	clear_all_debug_regs();
1924	1916	dbg_restore_debug_regs();
1925	1917
1926		− gdt_setup_doublefault_tss(cpu);
	1918	+ doublefault_init_cpu_tss();
1927	1919
1928	1920	fpu__init_cpu();
1929	1921

arch/x86/kernel/doublefault.c+0−86

@@ -1,86 +0,0 @@
1		−// SPDX-License-Identifier: GPL-2.0
2		−#include <linux/mm.h>
3		−#include <linux/sched.h>
4		−#include <linux/sched/debug.h>
5		−#include <linux/init_task.h>
6		−#include <linux/fs.h>
7		−
8		−#include <linux/uaccess.h>
9		−#include <asm/pgtable.h>
10		−#include <asm/processor.h>
11		−#include <asm/desc.h>
12		−
13		−#ifdef CONFIG_X86_32
14		−
15		−#define DOUBLEFAULT_STACKSIZE (1024)
16		−static unsigned long doublefault_stack[DOUBLEFAULT_STACKSIZE];
17		−#define STACK_START (unsigned long)(doublefault_stack+DOUBLEFAULT_STACKSIZE)
18		−
19		−#define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + MAXMEM)
20		−
21		−static void doublefault_fn(void)
22		−{
23		− struct desc_ptr gdt_desc = {0, 0};
24		− unsigned long gdt, tss;
25		−
26		− native_store_gdt(&gdt_desc);
27		− gdt = gdt_desc.address;
28		−
29		− printk(KERN_EMERG "PANIC: double fault, gdt at %08lx [%d bytes]\n", gdt, gdt_desc.size);
30		−
31		− if (ptr_ok(gdt)) {
32		− gdt += GDT_ENTRY_TSS << 3;
33		− tss = get_desc_base((struct desc_struct *)gdt);
34		− printk(KERN_EMERG "double fault, tss at %08lx\n", tss);
35		−
36		− if (ptr_ok(tss)) {
37		− struct x86_hw_tss t = (struct x86_hw_tss )tss;
38		−
39		− printk(KERN_EMERG "eip = %08lx, esp = %08lx\n",
40		− t->ip, t->sp);
41		−
42		− printk(KERN_EMERG "eax = %08lx, ebx = %08lx, ecx = %08lx, edx = %08lx\n",
43		− t->ax, t->bx, t->cx, t->dx);
44		− printk(KERN_EMERG "esi = %08lx, edi = %08lx\n",
45		− t->si, t->di);
46		− }
47		− }
48		−
49		− for (;;)
50		− cpu_relax();
51		−}
52		−
53		−struct x86_hw_tss doublefault_tss __cacheline_aligned = {
54		− .sp0 = STACK_START,
55		− .ss0 = __KERNEL_DS,
56		− .ldt = 0,
57		− .io_bitmap_base = IO_BITMAP_OFFSET_INVALID,
58		−
59		− .ip = (unsigned long) doublefault_fn,
60		− /* 0x2 bit is always set */
61		− .flags = X86_EFLAGS_SF \| 0x2,
62		− .sp = STACK_START,
63		− .es = __USER_DS,
64		− .cs = __KERNEL_CS,
65		− .ss = __KERNEL_DS,
66		− .ds = __USER_DS,
67		− .fs = __KERNEL_PERCPU,
68		−#ifndef CONFIG_X86_32_LAZY_GS
69		− .gs = __KERNEL_STACK_CANARY,
70		−#endif
71		−
72		− .__cr3 = __pa_nodebug(swapper_pg_dir),
73		−};
74		−
75		−/* dummy for do_double_fault() call */
76		−void df_debug(struct pt_regs *regs, long error_code) {}
77		−
78		−#else /* !CONFIG_X86_32 */
79		−
80		−void df_debug(struct pt_regs *regs, long error_code)
81		−{
82		− pr_emerg("PANIC: double fault, error_code: 0x%lx\n", error_code);
83		− show_regs(regs);
84		− panic("Machine halted.");
85		−}
86		−#endif

arch/x86/kernel/doublefault_32.c+136−0

@@ -0,0 +1,136 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+#include <linux/mm.h>
	3	+#include <linux/sched.h>
	4	+#include <linux/sched/debug.h>
	5	+#include <linux/init_task.h>
	6	+#include <linux/fs.h>
	7	+
	8	+#include <linux/uaccess.h>
	9	+#include <asm/pgtable.h>
	10	+#include <asm/processor.h>
	11	+#include <asm/desc.h>
	12	+#include <asm/traps.h>
	13	+
	14	+extern void double_fault(void);
	15	+#define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + MAXMEM)
	16	+
	17	+#define TSS(x) this_cpu_read(cpu_tss_rw.x86_tss.x)
	18	+
	19	+static void set_df_gdt_entry(unsigned int cpu);
	20	+
	21	+/*
	22	+ * Called by double_fault with CR0.TS and EFLAGS.NT cleared. The CPU thinks
	23	+ * we're running the doublefault task. Cannot return.
	24	+ */
	25	+asmlinkage notrace void __noreturn doublefault_shim(void)
	26	+{
	27	+ unsigned long cr2;
	28	+ struct pt_regs regs;
	29	+
	30	+ BUILD_BUG_ON(sizeof(struct doublefault_stack) != PAGE_SIZE);
	31	+
	32	+ cr2 = native_read_cr2();
	33	+
	34	+ /* Reset back to the normal kernel task. */
	35	+ force_reload_TR();
	36	+ set_df_gdt_entry(smp_processor_id());
	37	+
	38	+ trace_hardirqs_off();
	39	+
	40	+ /*
	41	+ * Fill in pt_regs. A downside of doing this in C is that the unwinder
	42	+ * won't see it (no ENCODE_FRAME_POINTER), so a nested stack dump
	43	+ * won't successfully unwind to the source of the double fault.
	44	+ * The main dump from do_double_fault() is fine, though, since it
	45	+ * uses these regs directly.
	46	+ *
	47	+ * If anyone ever cares, this could be moved to asm.
	48	+ */
	49	+ regs.ss = TSS(ss);
	50	+ regs.__ssh = 0;
	51	+ regs.sp = TSS(sp);
	52	+ regs.flags = TSS(flags);
	53	+ regs.cs = TSS(cs);
	54	+ /* We won't go through the entry asm, so we can leave __csh as 0. */
	55	+ regs.__csh = 0;
	56	+ regs.ip = TSS(ip);
	57	+ regs.orig_ax = 0;
	58	+ regs.gs = TSS(gs);
	59	+ regs.__gsh = 0;
	60	+ regs.fs = TSS(fs);
	61	+ regs.__fsh = 0;
	62	+ regs.es = TSS(es);
	63	+ regs.__esh = 0;
	64	+ regs.ds = TSS(ds);
	65	+ regs.__dsh = 0;
	66	+ regs.ax = TSS(ax);
	67	+ regs.bp = TSS(bp);
	68	+ regs.di = TSS(di);
	69	+ regs.si = TSS(si);
	70	+ regs.dx = TSS(dx);
	71	+ regs.cx = TSS(cx);
	72	+ regs.bx = TSS(bx);
	73	+
	74	+ do_double_fault(&regs, 0, cr2);
	75	+
	76	+ /*
	77	+ * x86_32 does not save the original CR3 anywhere on a task switch.
	78	+ * This means that, even if we wanted to return, we would need to find
	79	+ * some way to reconstruct CR3. We could make a credible guess based
	80	+ * on cpu_tlbstate, but that would be racy and would not account for
	81	+ * PTI.
	82	+ *
	83	+ * Instead, don't bother. We can return through
	84	+ * rewind_stack_do_exit() instead.
	85	+ */
	86	+ panic("cannot return from double fault\n");
	87	+}
	88	+NOKPROBE_SYMBOL(doublefault_shim);
	89	+
	90	+DEFINE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack) = {
	91	+ .tss = {
	92	+ /*
	93	+ * No sp0 or ss0 -- we never run CPL != 0 with this TSS
	94	+ * active. sp is filled in later.
	95	+ */
	96	+ .ldt = 0,
	97	+ .io_bitmap_base = IO_BITMAP_OFFSET_INVALID,
	98	+
	99	+ .ip = (unsigned long) double_fault,
	100	+ .flags = X86_EFLAGS_FIXED,
	101	+ .es = __USER_DS,
	102	+ .cs = __KERNEL_CS,
	103	+ .ss = __KERNEL_DS,
	104	+ .ds = __USER_DS,
	105	+ .fs = __KERNEL_PERCPU,
	106	+#ifndef CONFIG_X86_32_LAZY_GS
	107	+ .gs = __KERNEL_STACK_CANARY,
	108	+#endif
	109	+
	110	+ .__cr3 = __pa_nodebug(swapper_pg_dir),
	111	+ },
	112	+};
	113	+
	114	+static void set_df_gdt_entry(unsigned int cpu)
	115	+{
	116	+ /* Set up doublefault TSS pointer in the GDT */
	117	+ __set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS,
	118	+ &get_cpu_entry_area(cpu)->doublefault_stack.tss);
	119	+
	120	+}
	121	+
	122	+void doublefault_init_cpu_tss(void)
	123	+{
	124	+ unsigned int cpu = smp_processor_id();
	125	+ struct cpu_entry_area *cea = get_cpu_entry_area(cpu);
	126	+
	127	+ /*
	128	+ * The linker isn't smart enough to initialize percpu variables that
	129	+ * point to other places in percpu space.
	130	+ */
	131	+ this_cpu_write(doublefault_stack.tss.sp,
	132	+ (unsigned long)&cea->doublefault_stack.stack +
	133	+ sizeof(doublefault_stack.stack));
	134	+
	135	+ set_df_gdt_entry(cpu);
	136	+}

arch/x86/kernel/dumpstack_32.c+30−0

@@ -29,6 +29,9 @@ const char *stack_type_name(enum stack_type type) · 28 unmodified lines
29	29	if (type == STACK_TYPE_ENTRY)
30	30	return "ENTRY_TRAMPOLINE";
31	31
	32	+ if (type == STACK_TYPE_EXCEPTION)
	33	+ return "#DF";
	34	+
32	35	return NULL;
33	36	}
34	37
@@ -82,6 +85,30 @@ static bool in_softirq_stack(unsigned long stack, struct stack_info info) · 47 unmodified lines
82	85	return true;
83	86	}
84	87
	88	+static bool in_doublefault_stack(unsigned long stack, struct stack_info info)
	89	+{
	90	+#ifdef CONFIG_DOUBLEFAULT
	91	+ struct cpu_entry_area *cea = get_cpu_entry_area(raw_smp_processor_id());
	92	+ struct doublefault_stack *ss = &cea->doublefault_stack;
	93	+
	94	+ void *begin = ss->stack;
	95	+ void *end = begin + sizeof(ss->stack);
	96	+
	97	+ if ((void )stack < begin \|\| (void )stack >= end)
	98	+ return false;
	99	+
	100	+ info->type = STACK_TYPE_EXCEPTION;
	101	+ info->begin = begin;
	102	+ info->end = end;
	103	+ info->next_sp = (unsigned long *)this_cpu_read(cpu_tss_rw.x86_tss.sp);
	104	+
	105	+ return true;
	106	+#else
	107	+ return false;
	108	+#endif
	109	+}
	110	+
	111	+
85	112	int get_stack_info(unsigned long stack, struct task_struct task,
86	113	struct stack_info info, unsigned long visit_mask)
87	114	{
@@ -105,6 +132,9 @@ int get_stack_info(unsigned long stack, struct task_struct task, · 17 unmodified lines
105	132	if (in_softirq_stack(stack, info))
106	133	goto recursion_check;
107	134
	135	+ if (in_doublefault_stack(stack, info))
	136	+ goto recursion_check;
	137	+
108	138	goto unknown;
109	139
110	140	recursion_check:

arch/x86/kernel/process.c+26−26

@@ -377,37 +377,37 @@ static void tss_copy_io_bitmap(struct tss_struct tss, struct io_bitmap iobm) · 376 unmodified lines
377	377	void tss_update_io_bitmap(void)
378	378	{
379	379	struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
	380	+ struct thread_struct *t = &current->thread;
380	381	u16 *base = &tss->x86_tss.io_bitmap_base;
381	382
382		− if (test_thread_flag(TIF_IO_BITMAP)) {
383		− struct thread_struct *t = &current->thread;
384		−
385		− if (IS_ENABLED(CONFIG_X86_IOPL_IOPERM) && t->iopl_emul == 3) {
386		− *base = IO_BITMAP_OFFSET_VALID_ALL;
387		− } else {
388		− struct io_bitmap *iobm = t->io_bitmap;
389		− /*
390		− * Only copy bitmap data when the sequence number
391		− * differs. The update time is accounted to the
392		− * incoming task.
393		− */
394		− if (tss->io_bitmap.prev_sequence != iobm->sequence)
395		− tss_copy_io_bitmap(tss, iobm);
396		−
397		− /* Enable the bitmap */
398		− *base = IO_BITMAP_OFFSET_VALID_MAP;
399		− }
	383	+ if (!test_thread_flag(TIF_IO_BITMAP)) {
	384	+ tss_invalidate_io_bitmap(tss);
	385	+ return;
	386	+ }
	387	+
	388	+ if (IS_ENABLED(CONFIG_X86_IOPL_IOPERM) && t->iopl_emul == 3) {
	389	+ *base = IO_BITMAP_OFFSET_VALID_ALL;
	390	+ } else {
	391	+ struct io_bitmap *iobm = t->io_bitmap;
	392	+
400	393	/*
401		− * Make sure that the TSS limit is covering the io bitmap.
402		− * It might have been cut down by a VMEXIT to 0x67 which
403		− * would cause a subsequent I/O access from user space to
404		− * trigger a #GP because tbe bitmap is outside the TSS
405		− * limit.
	394	+ * Only copy bitmap data when the sequence number differs. The
	395	+ * update time is accounted to the incoming task.
406	396	*/
407		− refresh_tss_limit();
408		− } else {
409		− tss_invalidate_io_bitmap(tss);
	397	+ if (tss->io_bitmap.prev_sequence != iobm->sequence)
	398	+ tss_copy_io_bitmap(tss, iobm);
	399	+
	400	+ /* Enable the bitmap */
	401	+ *base = IO_BITMAP_OFFSET_VALID_MAP;
410	402	}
	403	+
	404	+ /*
	405	+ * Make sure that the TSS limit is covering the IO bitmap. It might have
	406	+ * been cut down by a VMEXIT to 0x67 which would cause a subsequent I/O
	407	+ * access from user space to trigger a #GP because tbe bitmap is outside
	408	+ * the TSS limit.
	409	+ */
	410	+ refresh_tss_limit();
411	411	}
412	412	#else /* CONFIG_X86_IOPL_IOPERM */
413	413	static inline void switch_to_bitmap(unsigned long tifp) { }

arch/x86/kernel/ptrace.c+24−12

@@ -182,6 +182,9 @@ static u16 get_segment_reg(struct task_struct *task, unsigned long offset) · 181 unmodified lines
182	182	static int set_segment_reg(struct task_struct *task,
183	183	unsigned long offset, u16 value)
184	184	{
	185	+ if (WARN_ON_ONCE(task == current))
	186	+ return -EIO;
	187	+
185	188	/*
186	189	* The value argument was already truncated to 16 bits.
187	190	*/
@@ -209,10 +212,7 @@ static int set_segment_reg(struct task_struct *task, · 21 unmodified lines
209	212	break;
210	213
211	214	case offsetof(struct user_regs_struct, gs):
212		− if (task == current)
213		− set_user_gs(task_pt_regs(task), value);
214		− else
215		− task_user_gs(task) = value;
	215	+ task_user_gs(task) = value;
216	216	}
217	217
218	218	return 0;
@@ -272,32 +272,41 @@ static u16 get_segment_reg(struct task_struct *task, unsigned long offset) · 53 unmodified lines
272	272	static int set_segment_reg(struct task_struct *task,
273	273	unsigned long offset, u16 value)
274	274	{
	275	+ if (WARN_ON_ONCE(task == current))
	276	+ return -EIO;
	277	+
275	278	/*
276	279	* The value argument was already truncated to 16 bits.
277	280	*/
278	281	if (invalid_selector(value))
279	282	return -EIO;
280	283
	284	+ /*
	285	+ * This function has some ABI oddities.
	286	+ *
	287	+ * A 32-bit ptracer probably expects that writing FS or GS will change
	288	+ * FSBASE or GSBASE respectively. In the absence of FSGSBASE support,
	289	+ * this code indeed has that effect. When FSGSBASE is added, this
	290	+ * will require a special case.
	291	+ *
	292	+ * For existing 64-bit ptracers, writing FS or GS also currently
	293	+ * changes the base if the selector is nonzero the next time the task
	294	+ * is run. This behavior may not be needed, and trying to preserve it
	295	+ * when FSGSBASE is added would be complicated at best.
	296	+ */
	297	+
281	298	switch (offset) {
282	299	case offsetof(struct user_regs_struct,fs):
283	300	task->thread.fsindex = value;
284		− if (task == current)
285		− loadsegment(fs, task->thread.fsindex);
286	301	break;
287	302	case offsetof(struct user_regs_struct,gs):
288	303	task->thread.gsindex = value;
289		− if (task == current)
290		− load_gs_index(task->thread.gsindex);
291	304	break;
292	305	case offsetof(struct user_regs_struct,ds):
293	306	task->thread.ds = value;
294		− if (task == current)
295		− loadsegment(ds, task->thread.ds);
296	307	break;
297	308	case offsetof(struct user_regs_struct,es):
298	309	task->thread.es = value;
299		− if (task == current)
300		− loadsegment(es, task->thread.es);
301	310	break;
302	311
303	312	/*
@@ -375,6 +384,9 @@ static int putreg(struct task_struct *child, · 71 unmodified lines
375	384	* When changing the FS base, use do_arch_prctl_64()
376	385	* to set the index to zero and to set the base
377	386	* as requested.
	387	+ *
	388	+ * NB: This behavior is nonsensical and likely needs to
	389	+ * change when FSGSBASE support is added.
378	390	*/
379	391	if (child->thread.fsbase != value)
380	392	return do_arch_prctl_64(child, ARCH_SET_FS, value);

arch/x86/kernel/traps.c+20−11

@@ -306,8 +306,23 @@ __visible void __noreturn handle_stack_overflow(const char *message, · 305 unmodified lines
306	306	}
307	307	#endif
308	308
309		−#ifdef CONFIG_X86_64
310		−/* Runs on IST stack */
	309	+#if defined(CONFIG_X86_64) \|\| defined(CONFIG_DOUBLEFAULT)
	310	+/*
	311	+ * Runs on an IST stack for x86_64 and on a special task stack for x86_32.
	312	+ *
	313	+ * On x86_64, this is more or less a normal kernel entry. Notwithstanding the
	314	+ * SDM's warnings about double faults being unrecoverable, returning works as
	315	+ * expected. Presumably what the SDM actually means is that the CPU may get
	316	+ * the register state wrong on entry, so returning could be a bad idea.
	317	+ *
	318	+ * Various CPU engineers have promised that double faults due to an IRET fault
	319	+ * while the stack is read-only are, in fact, recoverable.
	320	+ *
	321	+ * On x86_32, this is entered through a task gate, and regs are synthesized
	322	+ * from the TSS. Returning is, in principle, okay, but changes to regs will
	323	+ * be lost. If, for some reason, we need to return to a context with modified
	324	+ * regs, the shim code could be adjusted to synchronize the registers.
	325	+ */
311	326	dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsigned long cr2)
312	327	{
313	328	static const char str[] = "double fault";
@@ -411,15 +426,9 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsign · 97 unmodified lines
411	426	handle_stack_overflow("kernel stack overflow (double-fault)", regs, cr2);
412	427	#endif
413	428
414		−#ifdef CONFIG_DOUBLEFAULT
415		− df_debug(regs, error_code);
416		−#endif
417		− /*
418		− * This is always a kernel trap and never fixable (and thus must
419		− * never return).
420		− */
421		− for (;;)
422		− die(str, regs, error_code);
	429	+ pr_emerg("PANIC: double fault, error_code: 0x%lx\n", error_code);
	430	+ die("double fault", regs, error_code);
	431	+ panic("Machine halted.");
423	432	}
424	433	#endif
425	434

arch/x86/mm/cpu_entry_area.c+13−1

@@ -17,6 +17,10 @@ static DEFINE_PER_CPU_PAGE_ALIGNED(struct exception_stacks, exception_stacks); · 16 unmodified lines
17	17	DEFINE_PER_CPU(struct cea_exception_stacks*, cea_exception_stacks);
18	18	#endif
19	19
	20	+#if defined(CONFIG_X86_32) && defined(CONFIG_DOUBLEFAULT)
	21	+DECLARE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack);
	22	+#endif
	23	+
20	24	struct cpu_entry_area *get_cpu_entry_area(int cpu)
21	25	{
22	26	unsigned long va = CPU_ENTRY_AREA_PER_CPU + cpu * CPU_ENTRY_AREA_SIZE;
@@ -108,7 +112,15 @@ static void __init percpu_setup_exception_stacks(unsigned int cpu) · 85 unmodified lines
108	112	cea_map_stack(MCE);
109	113	}
110	114	#else
111		−static inline void percpu_setup_exception_stacks(unsigned int cpu) {}
	115	+static inline void percpu_setup_exception_stacks(unsigned int cpu)
	116	+{
	117	+#ifdef CONFIG_DOUBLEFAULT
	118	+ struct cpu_entry_area *cea = get_cpu_entry_area(cpu);
	119	+
	120	+ cea_map_percpu_pages(&cea->doublefault_stack,
	121	+ &per_cpu(doublefault_stack, cpu), 1, PAGE_KERNEL);
	122	+#endif
	123	+}
112	124	#endif
113	125
114	126	/* Setup the fixmap mappings only once per-processor */

arch/x86/mm/fault.c+1−1

@@ -197,7 +197,7 @@ void vmalloc_sync_all(void) · 196 unmodified lines
197	197	return;
198	198
199	199	for (address = VMALLOC_START & PMD_MASK;
200		− address >= TASK_SIZE_MAX && address < FIXADDR_TOP;
	200	+ address >= TASK_SIZE_MAX && address < VMALLOC_END;
201	201	address += PMD_SIZE) {
202	202	struct page *page;
203	203

arch/x86/mm/pat_interval.c+6−6

@@ -56,7 +56,7 @@ static struct memtype *memtype_match(u64 start, u64 end, int match_type) · 55 unmodified lines
56	56	{
57	57	struct memtype *match;
58	58
59		− match = memtype_interval_iter_first(&memtype_rbroot, start, end);
	59	+ match = memtype_interval_iter_first(&memtype_rbroot, start, end-1);
60	60	while (match != NULL && match->start < end) {
61	61	if ((match_type == MEMTYPE_EXACT_MATCH) &&
62	62	(match->start == start) && (match->end == end))
@@ -66,7 +66,7 @@ static struct memtype *memtype_match(u64 start, u64 end, int match_type) · 3 unmodified lines
66	66	(match->start < start) && (match->end == end))
67	67	return match;
68	68
69		− match = memtype_interval_iter_next(match, start, end);
	69	+ match = memtype_interval_iter_next(match, start, end-1);
70	70	}
71	71
72	72	return NULL; /* Returns NULL if there is no match */
@@ -79,7 +79,7 @@ static int memtype_check_conflict(u64 start, u64 end, · 6 unmodified lines
79	79	struct memtype *match;
80	80	enum page_cache_mode found_type = reqtype;
81	81
82		− match = memtype_interval_iter_first(&memtype_rbroot, start, end);
	82	+ match = memtype_interval_iter_first(&memtype_rbroot, start, end-1);
83	83	if (match == NULL)
84	84	goto success;
85	85
@@ -89,12 +89,12 @@ static int memtype_check_conflict(u64 start, u64 end, · 3 unmodified lines
89	89	dprintk("Overlap at 0x%Lx-0x%Lx\n", match->start, match->end);
90	90	found_type = match->type;
91	91
92		− match = memtype_interval_iter_next(match, start, end);
	92	+ match = memtype_interval_iter_next(match, start, end-1);
93	93	while (match) {
94	94	if (match->type != found_type)
95	95	goto failure;
96	96
97		− match = memtype_interval_iter_next(match, start, end);
	97	+ match = memtype_interval_iter_next(match, start, end-1);
98	98	}
99	99	success:
100	100	if (newtype)
@@ -160,7 +160,7 @@ struct memtype *memtype_erase(u64 start, u64 end) · 59 unmodified lines
160	160	struct memtype *memtype_lookup(u64 addr)
161	161	{
162	162	return memtype_interval_iter_first(&memtype_rbroot, addr,
163		− addr + PAGE_SIZE);
	163	+ addr + PAGE_SIZE-1);
164	164	}
165	165
166	166	#if defined(CONFIG_DEBUG_FS)

drivers/misc/lkdtm/bugs.c+39−0

@@ -12,6 +12,10 @@ · 11 unmodified lines
12	12	#include <linux/sched/task_stack.h>
13	13	#include <linux/uaccess.h>
14	14
	15	+#ifdef CONFIG_X86_32
	16	+#include <asm/desc.h>
	17	+#endif
	18	+
15	19	struct lkdtm_list {
16	20	struct list_head node;
17	21	};
@@ -337,3 +341,38 @@ void lkdtm_UNSET_SMEP(void) · 319 unmodified lines
337	341	pr_err("FAIL: this test is x86_64-only\n");
338	342	#endif
339	343	}
	344	+
	345	+#ifdef CONFIG_X86_32
	346	+void lkdtm_DOUBLE_FAULT(void)
	347	+{
	348	+ /*
	349	+ * Trigger #DF by setting the stack limit to zero. This clobbers
	350	+ * a GDT TLS slot, which is okay because the current task will die
	351	+ * anyway due to the double fault.
	352	+ */
	353	+ struct desc_struct d = {
	354	+ .type = 3, /* expand-up, writable, accessed data */
	355	+ .p = 1, /* present */
	356	+ .d = 1, /* 32-bit */
	357	+ .g = 0, /* limit in bytes */
	358	+ .s = 1, /* not system */
	359	+ };
	360	+
	361	+ local_irq_disable();
	362	+ write_gdt_entry(get_cpu_gdt_rw(smp_processor_id()),
	363	+ GDT_ENTRY_TLS_MIN, &d, DESCTYPE_S);
	364	+
	365	+ /*
	366	+ * Put our zero-limit segment in SS and then trigger a fault. The
	367	+ * 4-byte access to (%esp) will fault with #SS, and the attempt to
	368	+ * deliver the fault will recursively cause #SS and result in #DF.
	369	+ * This whole process happens while NMIs and MCEs are blocked by the
	370	+ * MOV SS window. This is nice because an NMI with an invalid SS
	371	+ * would also double-fault, resulting in the NMI or MCE being lost.
	372	+ */
	373	+ asm volatile ("movw %0, %%ss; addl $0, (%%esp)" ::
	374	+ "r" ((unsigned short)(GDT_ENTRY_TLS_MIN << 3)));
	375	+
	376	+ panic("tried to double fault but didn't die\n");
	377	+}
	378	+#endif

drivers/misc/lkdtm/core.c+3−0

@@ -171,6 +171,9 @@ static const struct crashtype crashtypes[] = { · 170 unmodified lines
171	171	CRASHTYPE(USERCOPY_KERNEL_DS),
172	172	CRASHTYPE(STACKLEAK_ERASING),
173	173	CRASHTYPE(CFI_FORWARD_PROTO),
	174	+#ifdef CONFIG_X86_32
	175	+ CRASHTYPE(DOUBLE_FAULT),
	176	+#endif
174	177	};
175	178
176	179

drivers/misc/lkdtm/lkdtm.h+3−0

@@ -28,6 +28,9 @@ void lkdtm_CORRUPT_USER_DS(void); · 27 unmodified lines
28	28	void lkdtm_STACK_GUARD_PAGE_LEADING(void);
29	29	void lkdtm_STACK_GUARD_PAGE_TRAILING(void);
30	30	void lkdtm_UNSET_SMEP(void);
	31	+#ifdef CONFIG_X86_32
	32	+void lkdtm_DOUBLE_FAULT(void);
	33	+#endif
31	34
32	35	/* lkdtm_heap.c */
33	36	void __init lkdtm_heap_init(void);

tools/testing/selftests/x86/single_step_syscall.c+85−9

@@ -43,7 +43,19 @@ static void sethandler(int sig, void (handler)(int, siginfo_t , void *), · 42 unmodified lines
43	43	err(1, "sigaction");
44	44	}
45	45
46		−static volatile sig_atomic_t sig_traps;
	46	+static void clearhandler(int sig)
	47	+{
	48	+ struct sigaction sa;
	49	+ memset(&sa, 0, sizeof(sa));
	50	+ sa.sa_handler = SIG_DFL;
	51	+ sigemptyset(&sa.sa_mask);
	52	+ if (sigaction(sig, &sa, 0))
	53	+ err(1, "sigaction");
	54	+}
	55	+
	56	+static volatile sig_atomic_t sig_traps, sig_eflags;
	57	+sigjmp_buf jmpbuf;
	58	+static unsigned char altstack_data[SIGSTKSZ];
47	59
48	60	#ifdef __x86_64__
49	61	# define REG_IP REG_RIP
@@ -90,6 +102,25 @@ static void sigtrap(int sig, siginfo_t info, void ctx_void) · 40 unmodified lines
90	102	}
91	103	}
92	104
	105	+static char const * const signames[] = {
	106	+ [SIGSEGV] = "SIGSEGV",
	107	+ [SIGBUS] = "SIBGUS",
	108	+ [SIGTRAP] = "SIGTRAP",
	109	+ [SIGILL] = "SIGILL",
	110	+};
	111	+
	112	+static void print_and_longjmp(int sig, siginfo_t si, void ctx_void)
	113	+{
	114	+ ucontext_t *ctx = ctx_void;
	115	+
	116	+ printf("\tGot %s with RIP=%lx, TF=%ld\n", signames[sig],
	117	+ (unsigned long)ctx->uc_mcontext.gregs[REG_IP],
	118	+ (unsigned long)ctx->uc_mcontext.gregs[REG_EFL] & X86_EFLAGS_TF);
	119	+
	120	+ sig_eflags = (unsigned long)ctx->uc_mcontext.gregs[REG_EFL];
	121	+ siglongjmp(jmpbuf, 1);
	122	+}
	123	+
93	124	static void check_result(void)
94	125	{
95	126	unsigned long new_eflags = get_eflags();
@@ -109,6 +140,22 @@ static void check_result(void) · 13 unmodified lines
109	140	sig_traps = 0;
110	141	}
111	142
	143	+static void fast_syscall_no_tf(void)
	144	+{
	145	+ sig_traps = 0;
	146	+ printf("[RUN]\tFast syscall with TF cleared\n");
	147	+ fflush(stdout); /* Force a syscall */
	148	+ if (get_eflags() & X86_EFLAGS_TF) {
	149	+ printf("[FAIL]\tTF is now set\n");
	150	+ exit(1);
	151	+ }
	152	+ if (sig_traps) {
	153	+ printf("[FAIL]\tGot SIGTRAP\n");
	154	+ exit(1);
	155	+ }
	156	+ printf("[OK]\tNothing unexpected happened\n");
	157	+}
	158	+
112	159	int main()
113	160	{
114	161	#ifdef CAN_BUILD_32
@@ -163,17 +210,46 @@ int main() · 48 unmodified lines
163	210	check_result();
164	211
165	212	/* Now make sure that another fast syscall doesn't set TF again. */
166		− printf("[RUN]\tFast syscall with TF cleared\n");
167		− fflush(stdout); /* Force a syscall */
168		− if (get_eflags() & X86_EFLAGS_TF) {
169		− printf("[FAIL]\tTF is now set\n");
170		− exit(1);
	213	+ fast_syscall_no_tf();
	214	+
	215	+ /*
	216	+ * And do a forced SYSENTER to make sure that this works even if
	217	+ * fast syscalls don't use SYSENTER.
	218	+ *
	219	+ * Invoking SYSENTER directly breaks all the rules. Just handle
	220	+ * the SIGSEGV.
	221	+ */
	222	+ if (sigsetjmp(jmpbuf, 1) == 0) {
	223	+ unsigned long nr = SYS_getpid;
	224	+ printf("[RUN]\tSet TF and check SYSENTER\n");
	225	+ stack_t stack = {
	226	+ .ss_sp = altstack_data,
	227	+ .ss_size = SIGSTKSZ,
	228	+ };
	229	+ if (sigaltstack(&stack, NULL) != 0)
	230	+ err(1, "sigaltstack");
	231	+ sethandler(SIGSEGV, print_and_longjmp,
	232	+ SA_RESETHAND \| SA_ONSTACK);
	233	+ sethandler(SIGILL, print_and_longjmp, SA_RESETHAND);
	234	+ set_eflags(get_eflags() \| X86_EFLAGS_TF);
	235	+ /* Clear EBP first to make sure we segfault cleanly. */
	236	+ asm volatile ("xorl %%ebp, %%ebp; SYSENTER" : "+a" (nr) :: "flags", "rcx"
	237	+#ifdef __x86_64__
	238	+ , "r11"
	239	+#endif
	240	+ );
	241	+
	242	+ /* We're unreachable here. SYSENTER forgets RIP. */
171	243	}
172		− if (sig_traps) {
173		− printf("[FAIL]\tGot SIGTRAP\n");
	244	+ clearhandler(SIGSEGV);
	245	+ clearhandler(SIGILL);
	246	+ if (!(sig_eflags & X86_EFLAGS_TF)) {
	247	+ printf("[FAIL]\tTF was cleared\n");
174	248	exit(1);
175	249	}
176		− printf("[OK]\tNothing unexpected happened\n");
	250	+
	251	+ /* Now make sure that another fast syscall doesn't set TF again. */
	252	+ fast_syscall_no_tf();
177	253
178	254	return 0;
179	255	}

Merge branch 'x86-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code