History

Merge tag 'x86-urgent-2020-09-06' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 fixes from Ingo Molnar: - more generic entry code ABI fallout - debug register handling bugfixes - fix vmalloc mappings on 32-bit kernels - kprobes instrumentation output fix on 32-bit kernels - fix over-eager WARN_ON_ONCE() on !SMAP hardware - NUMA debugging fix - fix Clang related crash on !RETPOLINE kernels * tag 'x86-urgent-2020-09-06' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: x86/entry: Unbreak 32bit fast syscall x86/debug: Allow a single level of #DB recursion x86/entry: Fix AC assertion tracing/kprobes, x86/ptrace: Fix regs argument order for i386 x86, fakenuma: Fix invalid starting node ID x86/mm/32: Bring back vmalloc faulting on x86_32 x86/cmdline: Disable jump tables for cmdline.c

Linus Torvalds committed 6 years ago

Commit 015b3152 parents 68beef5 4facb959 files · +213 −63Browse files

Unified · Split

9 files
  1. arch/x86/entry/common.c +20 −9
  2. arch/x86/include/asm/entry-common.h +10 −2
  3. arch/x86/include/asm/ptrace.h +1 −1
  4. arch/x86/kernel/traps.c +31 −34
  5. arch/x86/lib/Makefile +1 −1
  6. arch/x86/mm/fault.c +78 −0
  7. arch/x86/mm/numa_emulation.c +1 −1
  8. include/linux/entry-common.h +42 −9
  9. kernel/entry/common.c +29 −6
arch/x86/entry/common.c+20−9
@@ -60,16 +60,10 @@ __visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs) · 59 unmodified lines
60#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)60#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
61static __always_inline unsigned int syscall_32_enter(struct pt_regs *regs)61static __always_inline unsigned int syscall_32_enter(struct pt_regs *regs)
62{62{
63 unsigned int nr = (unsigned int)regs->orig_ax;
64
65 if (IS_ENABLED(CONFIG_IA32_EMULATION))63 if (IS_ENABLED(CONFIG_IA32_EMULATION))
66 current_thread_info()->status |= TS_COMPAT;64 current_thread_info()->status |= TS_COMPAT;
67 /*65
68 * Subtlety here: if ptrace pokes something larger than 2^32-1 into66 return (unsigned int)regs->orig_ax;
69 * orig_ax, the unsigned int return value truncates it. This may
70 * or may not be necessary, but it matches the old asm behavior.
71 */
72 return (unsigned int)syscall_enter_from_user_mode(regs, nr);
73}67}
74 68
75/*69/*
@@ -91,15 +85,29 @@ __visible noinstr void do_int80_syscall_32(struct pt_regs *regs) · 15 unmodified lines
91{85{
92 unsigned int nr = syscall_32_enter(regs);86 unsigned int nr = syscall_32_enter(regs);
93 87
88 /*
89 * Subtlety here: if ptrace pokes something larger than 2^32-1 into
90 * orig_ax, the unsigned int return value truncates it. This may
91 * or may not be necessary, but it matches the old asm behavior.
92 */
93 nr = (unsigned int)syscall_enter_from_user_mode(regs, nr);
94
94 do_syscall_32_irqs_on(regs, nr);95 do_syscall_32_irqs_on(regs, nr);
95 syscall_exit_to_user_mode(regs);96 syscall_exit_to_user_mode(regs);
96}97}
97 98
98static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)99static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
99{100{
100 unsigned int nr = syscall_32_enter(regs);101 unsigned int nr = syscall_32_enter(regs);
101 int res;102 int res;
102 103
104 /*
105 * This cannot use syscall_enter_from_user_mode() as it has to
106 * fetch EBP before invoking any of the syscall entry work
107 * functions.
108 */
109 syscall_enter_from_user_mode_prepare(regs);
110
103 instrumentation_begin();111 instrumentation_begin();
104 /* Fetch EBP from where the vDSO stashed it. */112 /* Fetch EBP from where the vDSO stashed it. */
105 if (IS_ENABLED(CONFIG_X86_64)) {113 if (IS_ENABLED(CONFIG_X86_64)) {
@@ -122,6 +130,9 @@ static noinstr bool __do_fast_syscall_32(struct pt_regs *regs) · 16 unmodified lines
122 return false;130 return false;
123 }131 }
124 132
133 /* The case truncates any ptrace induced syscall nr > 2^32 -1 */
134 nr = (unsigned int)syscall_enter_from_user_mode_work(regs, nr);
135
125 /* Now this is just like a normal syscall. */136 /* Now this is just like a normal syscall. */
126 do_syscall_32_irqs_on(regs, nr);137 do_syscall_32_irqs_on(regs, nr);
127 syscall_exit_to_user_mode(regs);138 syscall_exit_to_user_mode(regs);
arch/x86/include/asm/entry-common.h+10−2
@@ -18,8 +18,16 @@ static __always_inline void arch_check_user_regs(struct pt_regs *regs) · 17 unmodified lines
18 * state, not the interrupt state as imagined by Xen.18 * state, not the interrupt state as imagined by Xen.
19 */19 */
20 unsigned long flags = native_save_fl();20 unsigned long flags = native_save_fl();
21 WARN_ON_ONCE(flags & (X86_EFLAGS_AC | X86_EFLAGS_DF |21 unsigned long mask = X86_EFLAGS_DF | X86_EFLAGS_NT;
22 X86_EFLAGS_NT));22
23 /*
24 * For !SMAP hardware we patch out CLAC on entry.
25 */
26 if (boot_cpu_has(X86_FEATURE_SMAP) ||
27 (IS_ENABLED(CONFIG_64_BIT) && boot_cpu_has(X86_FEATURE_XENPV)))
28 mask |= X86_EFLAGS_AC;
29
30 WARN_ON_ONCE(flags & mask);
23 31
24 /* We think we came from user mode. Make sure pt_regs agrees. */32 /* We think we came from user mode. Make sure pt_regs agrees. */
25 WARN_ON_ONCE(!user_mode(regs));33 WARN_ON_ONCE(!user_mode(regs));
arch/x86/include/asm/ptrace.h+1−1
@@ -327,8 +327,8 @@ static inline unsigned long regs_get_kernel_argument(struct pt_regs *regs, · 326 unmodified lines
327 static const unsigned int argument_offs[] = {327 static const unsigned int argument_offs[] = {
328#ifdef __i386__328#ifdef __i386__
329 offsetof(struct pt_regs, ax),329 offsetof(struct pt_regs, ax),
330 offsetof(struct pt_regs, cx),
331 offsetof(struct pt_regs, dx),330 offsetof(struct pt_regs, dx),
331 offsetof(struct pt_regs, cx),
332#define NR_REG_ARGUMENTS 3332#define NR_REG_ARGUMENTS 3
333#else333#else
334 offsetof(struct pt_regs, di),334 offsetof(struct pt_regs, di),
arch/x86/kernel/traps.c+31−34
@@ -729,20 +729,9 @@ static bool is_sysenter_singlestep(struct pt_regs *regs) · 728 unmodified lines
729#endif729#endif
730}730}
731 731
732static __always_inline void debug_enter(unsigned long *dr6, unsigned long *dr7)732static __always_inline unsigned long debug_read_clear_dr6(void)
733{733{
734 /*734 unsigned long dr6;
735 * Disable breakpoints during exception handling; recursive exceptions
736 * are exceedingly 'fun'.
737 *
738 * Since this function is NOKPROBE, and that also applies to
739 * HW_BREAKPOINT_X, we can't hit a breakpoint before this (XXX except a
740 * HW_BREAKPOINT_W on our stack)
741 *
742 * Entry text is excluded for HW_BP_X and cpu_entry_area, which
743 * includes the entry stack is excluded for everything.
744 */
745 *dr7 = local_db_save();
746 735
747 /*736 /*
748 * The Intel SDM says:737 * The Intel SDM says:
@@ -755,15 +744,12 @@ static __always_inline void debug_enter(unsigned long *dr6, unsigned long *dr7) · 6 unmodified lines
755 *744 *
756 * Keep it simple: clear DR6 immediately.745 * Keep it simple: clear DR6 immediately.
757 */746 */
758 get_debugreg(*dr6, 6);747 get_debugreg(dr6, 6);
759 set_debugreg(0, 6);748 set_debugreg(0, 6);
760 /* Filter out all the reserved bits which are preset to 1 */749 /* Filter out all the reserved bits which are preset to 1 */
761 *dr6 &= ~DR6_RESERVED;750 dr6 &= ~DR6_RESERVED;
762}
763 751
764static __always_inline void debug_exit(unsigned long dr7)752 return dr6;
765{
766 local_db_restore(dr7);
767}753}
768 754
769/*755/*
@@ -863,6 +849,18 @@ out: · 93 unmodified lines
863static __always_inline void exc_debug_kernel(struct pt_regs *regs,849static __always_inline void exc_debug_kernel(struct pt_regs *regs,
864 unsigned long dr6)850 unsigned long dr6)
865{851{
852 /*
853 * Disable breakpoints during exception handling; recursive exceptions
854 * are exceedingly 'fun'.
855 *
856 * Since this function is NOKPROBE, and that also applies to
857 * HW_BREAKPOINT_X, we can't hit a breakpoint before this (XXX except a
858 * HW_BREAKPOINT_W on our stack)
859 *
860 * Entry text is excluded for HW_BP_X and cpu_entry_area, which
861 * includes the entry stack is excluded for everything.
862 */
863 unsigned long dr7 = local_db_save();
866 bool irq_state = idtentry_enter_nmi(regs);864 bool irq_state = idtentry_enter_nmi(regs);
867 instrumentation_begin();865 instrumentation_begin();
868 866
@@ -883,6 +881,8 @@ static __always_inline void exc_debug_kernel(struct pt_regs *regs, · 14 unmodified lines
883 881
884 instrumentation_end();882 instrumentation_end();
885 idtentry_exit_nmi(regs, irq_state);883 idtentry_exit_nmi(regs, irq_state);
884
885 local_db_restore(dr7);
886}886}
887 887
888static __always_inline void exc_debug_user(struct pt_regs *regs,888static __always_inline void exc_debug_user(struct pt_regs *regs,
@@ -894,6 +894,15 @@ static __always_inline void exc_debug_user(struct pt_regs *regs, · 5 unmodified lines
894 */894 */
895 WARN_ON_ONCE(!user_mode(regs));895 WARN_ON_ONCE(!user_mode(regs));
896 896
897 /*
898 * NB: We can't easily clear DR7 here because
899 * idtentry_exit_to_usermode() can invoke ptrace, schedule, access
900 * user memory, etc. This means that a recursive #DB is possible. If
901 * this happens, that #DB will hit exc_debug_kernel() and clear DR7.
902 * Since we're not on the IST stack right now, everything will be
903 * fine.
904 */
905
897 irqentry_enter_from_user_mode(regs);906 irqentry_enter_from_user_mode(regs);
898 instrumentation_begin();907 instrumentation_begin();
899 908
@@ -907,36 +916,24 @@ static __always_inline void exc_debug_user(struct pt_regs *regs, · 7 unmodified lines
907/* IST stack entry */916/* IST stack entry */
908DEFINE_IDTENTRY_DEBUG(exc_debug)917DEFINE_IDTENTRY_DEBUG(exc_debug)
909{918{
910 unsigned long dr6, dr7;919 exc_debug_kernel(regs, debug_read_clear_dr6());
911
912 debug_enter(&dr6, &dr7);
913 exc_debug_kernel(regs, dr6);
914 debug_exit(dr7);
915}920}
916 921
917/* User entry, runs on regular task stack */922/* User entry, runs on regular task stack */
918DEFINE_IDTENTRY_DEBUG_USER(exc_debug)923DEFINE_IDTENTRY_DEBUG_USER(exc_debug)
919{924{
920 unsigned long dr6, dr7;925 exc_debug_user(regs, debug_read_clear_dr6());
921
922 debug_enter(&dr6, &dr7);
923 exc_debug_user(regs, dr6);
924 debug_exit(dr7);
925}926}
926#else927#else
927/* 32 bit does not have separate entry points. */928/* 32 bit does not have separate entry points. */
928DEFINE_IDTENTRY_RAW(exc_debug)929DEFINE_IDTENTRY_RAW(exc_debug)
929{930{
930 unsigned long dr6, dr7;931 unsigned long dr6 = debug_read_clear_dr6();
931
932 debug_enter(&dr6, &dr7);
933 932
934 if (user_mode(regs))933 if (user_mode(regs))
935 exc_debug_user(regs, dr6);934 exc_debug_user(regs, dr6);
936 else935 else
937 exc_debug_kernel(regs, dr6);936 exc_debug_kernel(regs, dr6);
938
939 debug_exit(dr7);
940}937}
941#endif938#endif
942 939
arch/x86/lib/Makefile+1−1
@@ -24,7 +24,7 @@ ifdef CONFIG_FUNCTION_TRACER · 23 unmodified lines
24CFLAGS_REMOVE_cmdline.o = -pg24CFLAGS_REMOVE_cmdline.o = -pg
25endif25endif
26 26
27CFLAGS_cmdline.o := -fno-stack-protector27CFLAGS_cmdline.o := -fno-stack-protector -fno-jump-tables
28endif28endif
29 29
30inat_tables_script = $(srctree)/arch/x86/tools/gen-insn-attr-x86.awk30inat_tables_script = $(srctree)/arch/x86/tools/gen-insn-attr-x86.awk
arch/x86/mm/fault.c+78−0
@@ -190,6 +190,53 @@ static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address) · 189 unmodified lines
190 return pmd_k;190 return pmd_k;
191}191}
192 192
193/*
194 * Handle a fault on the vmalloc or module mapping area
195 *
196 * This is needed because there is a race condition between the time
197 * when the vmalloc mapping code updates the PMD to the point in time
198 * where it synchronizes this update with the other page-tables in the
199 * system.
200 *
201 * In this race window another thread/CPU can map an area on the same
202 * PMD, finds it already present and does not synchronize it with the
203 * rest of the system yet. As a result v[mz]alloc might return areas
204 * which are not mapped in every page-table in the system, causing an
205 * unhandled page-fault when they are accessed.
206 */
207static noinline int vmalloc_fault(unsigned long address)
208{
209 unsigned long pgd_paddr;
210 pmd_t *pmd_k;
211 pte_t *pte_k;
212
213 /* Make sure we are in vmalloc area: */
214 if (!(address >= VMALLOC_START && address < VMALLOC_END))
215 return -1;
216
217 /*
218 * Synchronize this task's top level page-table
219 * with the 'reference' page table.
220 *
221 * Do _not_ use "current" here. We might be inside
222 * an interrupt in the middle of a task switch..
223 */
224 pgd_paddr = read_cr3_pa();
225 pmd_k = vmalloc_sync_one(__va(pgd_paddr), address);
226 if (!pmd_k)
227 return -1;
228
229 if (pmd_large(*pmd_k))
230 return 0;
231
232 pte_k = pte_offset_kernel(pmd_k, address);
233 if (!pte_present(*pte_k))
234 return -1;
235
236 return 0;
237}
238NOKPROBE_SYMBOL(vmalloc_fault);
239
193void arch_sync_kernel_mappings(unsigned long start, unsigned long end)240void arch_sync_kernel_mappings(unsigned long start, unsigned long end)
194{241{
195 unsigned long addr;242 unsigned long addr;
@@ -1110,6 +1157,37 @@ do_kern_addr_fault(struct pt_regs *regs, unsigned long hw_error_code, · 914 unmodified lines
1110 */1157 */
1111 WARN_ON_ONCE(hw_error_code & X86_PF_PK);1158 WARN_ON_ONCE(hw_error_code & X86_PF_PK);
1112 1159
1160#ifdef CONFIG_X86_32
1161 /*
1162 * We can fault-in kernel-space virtual memory on-demand. The
1163 * 'reference' page table is init_mm.pgd.
1164 *
1165 * NOTE! We MUST NOT take any locks for this case. We may
1166 * be in an interrupt or a critical region, and should
1167 * only copy the information from the master page table,
1168 * nothing more.
1169 *
1170 * Before doing this on-demand faulting, ensure that the
1171 * fault is not any of the following:
1172 * 1. A fault on a PTE with a reserved bit set.
1173 * 2. A fault caused by a user-mode access. (Do not demand-
1174 * fault kernel memory due to user-mode accesses).
1175 * 3. A fault caused by a page-level protection violation.
1176 * (A demand fault would be on a non-present page which
1177 * would have X86_PF_PROT==0).
1178 *
1179 * This is only needed to close a race condition on x86-32 in
1180 * the vmalloc mapping/unmapping code. See the comment above
1181 * vmalloc_fault() for details. On x86-64 the race does not
1182 * exist as the vmalloc mappings don't need to be synchronized
1183 * there.
1184 */
1185 if (!(hw_error_code & (X86_PF_RSVD | X86_PF_USER | X86_PF_PROT))) {
1186 if (vmalloc_fault(address) >= 0)
1187 return;
1188 }
1189#endif
1190
1113 /* Was the fault spurious, caused by lazy TLB invalidation? */1191 /* Was the fault spurious, caused by lazy TLB invalidation? */
1114 if (spurious_kernel_fault(hw_error_code, address))1192 if (spurious_kernel_fault(hw_error_code, address))
1115 return;1193 return;
arch/x86/mm/numa_emulation.c+1−1
@@ -321,7 +321,7 @@ static int __init split_nodes_size_interleave(struct numa_meminfo *ei, · 320 unmodified lines
321 u64 addr, u64 max_addr, u64 size)321 u64 addr, u64 max_addr, u64 size)
322{322{
323 return split_nodes_size_interleave_uniform(ei, pi, addr, max_addr, size,323 return split_nodes_size_interleave_uniform(ei, pi, addr, max_addr, size,
324 0, NULL, NUMA_NO_NODE);324 0, NULL, 0);
325}325}
326 326
327static int __init setup_emu2phys_nid(int *dfl_phys_nid)327static int __init setup_emu2phys_nid(int *dfl_phys_nid)
include/linux/entry-common.h+42−9
@@ -110,15 +110,30 @@ static inline __must_check int arch_syscall_enter_tracehook(struct pt_regs *regs · 109 unmodified lines
110#endif110#endif
111 111
112/**112/**
113 * syscall_enter_from_user_mode - Check and handle work before invoking113 * syscall_enter_from_user_mode_prepare - Establish state and enable interrupts
114 * a syscall
115 * @regs: Pointer to currents pt_regs114 * @regs: Pointer to currents pt_regs
116 * @syscall: The syscall number
117 *115 *
118 * Invoked from architecture specific syscall entry code with interrupts116 * Invoked from architecture specific syscall entry code with interrupts
119 * disabled. The calling code has to be non-instrumentable. When the117 * disabled. The calling code has to be non-instrumentable. When the
120 * function returns all state is correct and the subsequent functions can be118 * function returns all state is correct, interrupts are enabled and the
121 * instrumented.119 * subsequent functions can be instrumented.
120 *
121 * This handles lockdep, RCU (context tracking) and tracing state.
122 *
123 * This is invoked when there is extra architecture specific functionality
124 * to be done between establishing state and handling user mode entry work.
125 */
126void syscall_enter_from_user_mode_prepare(struct pt_regs *regs);
127
128/**
129 * syscall_enter_from_user_mode_work - Check and handle work before invoking
130 * a syscall
131 * @regs: Pointer to currents pt_regs
132 * @syscall: The syscall number
133 *
134 * Invoked from architecture specific syscall entry code with interrupts
135 * enabled after invoking syscall_enter_from_user_mode_prepare() and extra
136 * architecture specific work.
122 *137 *
123 * Returns: The original or a modified syscall number138 * Returns: The original or a modified syscall number
124 *139 *
@@ -127,12 +142,30 @@ static inline __must_check int arch_syscall_enter_tracehook(struct pt_regs *regs · 2 unmodified lines
127 * syscall_set_return_value() first. If neither of those are called and -1142 * syscall_set_return_value() first. If neither of those are called and -1
128 * is returned, then the syscall will fail with ENOSYS.143 * is returned, then the syscall will fail with ENOSYS.
129 *144 *
130 * The following functionality is handled here:145 * It handles the following work items:
131 *146 *
132 * 1) Establish state (lockdep, RCU (context tracking), tracing)147 * 1) TIF flag dependent invocations of arch_syscall_enter_tracehook(),
133 * 2) TIF flag dependent invocations of arch_syscall_enter_tracehook(),
134 * __secure_computing(), trace_sys_enter()148 * __secure_computing(), trace_sys_enter()
135 * 3) Invocation of audit_syscall_entry()149 * 2) Invocation of audit_syscall_entry()
150 */
151long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall);
152
153/**
154 * syscall_enter_from_user_mode - Establish state and check and handle work
155 * before invoking a syscall
156 * @regs: Pointer to currents pt_regs
157 * @syscall: The syscall number
158 *
159 * Invoked from architecture specific syscall entry code with interrupts
160 * disabled. The calling code has to be non-instrumentable. When the
161 * function returns all state is correct, interrupts are enabled and the
162 * subsequent functions can be instrumented.
163 *
164 * This is combination of syscall_enter_from_user_mode_prepare() and
165 * syscall_enter_from_user_mode_work().
166 *
167 * Returns: The original or a modified syscall number. See
168 * syscall_enter_from_user_mode_work() for further explanation.
136 */169 */
137long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall);170long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall);
138 171
kernel/entry/common.c+29−6
@@ -69,22 +69,45 @@ static long syscall_trace_enter(struct pt_regs *regs, long syscall, · 68 unmodified lines
69 return ret ? : syscall_get_nr(current, regs);69 return ret ? : syscall_get_nr(current, regs);
70}70}
71 71
72noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall)72static __always_inline long
73__syscall_enter_from_user_work(struct pt_regs *regs, long syscall)
73{74{
74 unsigned long ti_work;75 unsigned long ti_work;
75 76
76 enter_from_user_mode(regs);
77 instrumentation_begin();
78
79 local_irq_enable();
80 ti_work = READ_ONCE(current_thread_info()->flags);77 ti_work = READ_ONCE(current_thread_info()->flags);
81 if (ti_work & SYSCALL_ENTER_WORK)78 if (ti_work & SYSCALL_ENTER_WORK)
82 syscall = syscall_trace_enter(regs, syscall, ti_work);79 syscall = syscall_trace_enter(regs, syscall, ti_work);
83 instrumentation_end();
84 80
85 return syscall;81 return syscall;
86}82}
87 83
84long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall)
85{
86 return __syscall_enter_from_user_work(regs, syscall);
87}
88
89noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall)
90{
91 long ret;
92
93 enter_from_user_mode(regs);
94
95 instrumentation_begin();
96 local_irq_enable();
97 ret = __syscall_enter_from_user_work(regs, syscall);
98 instrumentation_end();
99
100 return ret;
101}
102
103noinstr void syscall_enter_from_user_mode_prepare(struct pt_regs *regs)
104{
105 enter_from_user_mode(regs);
106 instrumentation_begin();
107 local_irq_enable();
108 instrumentation_end();
109}
110
88/**111/**
89 * exit_to_user_mode - Fixup state when exiting to user mode112 * exit_to_user_mode - Fixup state when exiting to user mode
90 *113 *