History

Merge tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 fixes from Catalin Marinas: - Fix kernel build with the integrated LLVM assembler which doesn't see the -Wa,-march option. - Fix "make vdso_install" when COMPAT_VDSO is disabled. - Make KVM more robust if the AT S1E1R instruction triggers an exception (architecture corner cases). * tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: KVM: arm64: Set HCR_EL2.PTW to prevent AT taking synchronous exception KVM: arm64: Survive synchronous exceptions caused by AT instructions KVM: arm64: Add kvm_extable for vaxorcism code arm64: vdso32: make vdso32 install conditional arm64: use a common .arch preamble for inline assembly

Linus Torvalds committed 6 years ago

Commit 96d454c2 parents ef91bb1 71a7f8c12 files · +172 −41Browse files

Unified · Split

12 files
  1. arch/arm64/Makefile +10 −4
  2. arch/arm64/include/asm/compiler.h +6 −0
  3. arch/arm64/include/asm/kvm_arm.h +2 −1
  4. arch/arm64/include/asm/kvm_asm.h +43 −0
  5. arch/arm64/include/asm/tlbflush.h +4 −2
  6. arch/arm64/kernel/image-vars.h +4 −0
  7. arch/arm64/kernel/vmlinux.lds.S +8 −0
  8. arch/arm64/kvm/hyp/entry.S +9 −6
  9. arch/arm64/kvm/hyp/hyp-entry.S +41 −24
  10. arch/arm64/kvm/hyp/include/hyp/switch.h +35 −4
  11. arch/arm64/kvm/hyp/nvhe/switch.c +5 −0
  12. arch/arm64/kvm/hyp/vhe/switch.c +5 −0
arch/arm64/Makefile+10−4
@@ -82,8 +82,8 @@ endif · 81 unmodified lines
8282 # compiler to generate them and consequently to break the single image contract
8383 # we pass it only to the assembler. This option is utilized only in case of non
8484 # integrated assemblers.
85−ifneq ($(CONFIG_AS_HAS_ARMV8_4), y)
86−branch-prot-flags-$(CONFIG_AS_HAS_PAC) += -Wa,-march=armv8.3-a
85+ifeq ($(CONFIG_AS_HAS_PAC), y)
86+asm-arch := armv8.3-a
8787 endif
8888 endif
8989
@@ -91,7 +91,12 @@ KBUILD_CFLAGS += $(branch-prot-flags-y) · 1 unmodified line
9191
9292 ifeq ($(CONFIG_AS_HAS_ARMV8_4), y)
9393 # make sure to pass the newest target architecture to -march.
94−KBUILD_CFLAGS += -Wa,-march=armv8.4-a
94+asm-arch := armv8.4-a
95+endif
96+
97+ifdef asm-arch
98+KBUILD_CFLAGS += -Wa,-march=$(asm-arch) \
99+ -DARM64_ASM_ARCH='"$(asm-arch)"'
95100 endif
96101
97102 ifeq ($(CONFIG_SHADOW_CALL_STACK), y)
@@ -165,7 +170,8 @@ zinstall install: · 67 unmodified lines
165170 PHONY += vdso_install
166171 vdso_install:
167172 $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso $@
168− $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso32 $@
173+ $(if $(CONFIG_COMPAT_VDSO), \
174+ $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso32 $@)
169175
170176 # We use MRPROPER_FILES and CLEAN_FILES now
171177 archclean:
arch/arm64/include/asm/compiler.h+6−0
@@ -2,6 +2,12 @@ · 1 unmodified line
22 #ifndef __ASM_COMPILER_H
33 #define __ASM_COMPILER_H
44
5+#ifdef ARM64_ASM_ARCH
6+#define ARM64_ASM_PREAMBLE ".arch " ARM64_ASM_ARCH "\n"
7+#else
8+#define ARM64_ASM_PREAMBLE
9+#endif
10+
511 /*
612 * The EL0/EL1 pointer bits used by a pointer authentication code.
713 * This is dependent on TBI0/TBI1 being enabled, or bits 63:56 would also apply.
arch/arm64/include/asm/kvm_arm.h+2−1
@@ -71,11 +71,12 @@ · 70 unmodified lines
7171 * IMO: Override CPSR.I and enable signaling with VI
7272 * FMO: Override CPSR.F and enable signaling with VF
7373 * SWIO: Turn set/way invalidates into set/way clean+invalidate
74+ * PTW: Take a stage2 fault if a stage1 walk steps in device memory
7475 */
7576 #define HCR_GUEST_FLAGS (HCR_TSC | HCR_TSW | HCR_TWE | HCR_TWI | HCR_VM | \
7677 HCR_BSU_IS | HCR_FB | HCR_TAC | \
7778 HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \
78− HCR_FMO | HCR_IMO)
79+ HCR_FMO | HCR_IMO | HCR_PTW )
7980 #define HCR_VIRT_EXCP_MASK (HCR_VSE | HCR_VI | HCR_VF)
8081 #define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK)
8182 #define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H)
arch/arm64/include/asm/kvm_asm.h+43−0
@@ -169,6 +169,34 @@ extern char __smccc_workaround_1_smc[__SMCCC_WORKAROUND_1_SMC_SZ]; · 168 unmodified lines
169169 *__hyp_this_cpu_ptr(sym); \
170170 })
171171
172+#define __KVM_EXTABLE(from, to) \
173+ " .pushsection __kvm_ex_table, \"a\"\n" \
174+ " .align 3\n" \
175+ " .long (" #from " - .), (" #to " - .)\n" \
176+ " .popsection\n"
177+
178+
179+#define __kvm_at(at_op, addr) \
180+( { \
181+ int __kvm_at_err = 0; \
182+ u64 spsr, elr; \
183+ asm volatile( \
184+ " mrs %1, spsr_el2\n" \
185+ " mrs %2, elr_el2\n" \
186+ "1: at "at_op", %3\n" \
187+ " isb\n" \
188+ " b 9f\n" \
189+ "2: msr spsr_el2, %1\n" \
190+ " msr elr_el2, %2\n" \
191+ " mov %w0, %4\n" \
192+ "9:\n" \
193+ __KVM_EXTABLE(1b, 2b) \
194+ : "+r" (__kvm_at_err), "=&r" (spsr), "=&r" (elr) \
195+ : "r" (addr), "i" (-EFAULT)); \
196+ __kvm_at_err; \
197+} )
198+
199+
172200 #else /* __ASSEMBLY__ */
173201
174202 .macro hyp_adr_this_cpu reg, sym, tmp
@@ -193,6 +221,21 @@ extern char __smccc_workaround_1_smc[__SMCCC_WORKAROUND_1_SMC_SZ]; · 18 unmodified lines
193221 ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
194222 .endm
195223
224+/*
225+ * KVM extable for unexpected exceptions.
226+ * In the same format _asm_extable, but output to a different section so that
227+ * it can be mapped to EL2. The KVM version is not sorted. The caller must
228+ * ensure:
229+ * x18 has the hypervisor value to allow any Shadow-Call-Stack instrumented
230+ * code to write to it, and that SPSR_EL2 and ELR_EL2 are restored by the fixup.
231+ */
232+.macro _kvm_extable, from, to
233+ .pushsection __kvm_ex_table, "a"
234+ .align 3
235+ .long (\from - .), (\to - .)
236+ .popsection
237+.endm
238+
196239 #endif
197240
198241 #endif /* __ARM_KVM_ASM_H__ */
arch/arm64/include/asm/tlbflush.h+4−2
@@ -28,14 +28,16 @@ · 27 unmodified lines
2828 * not. The macros handles invoking the asm with or without the
2929 * register argument as appropriate.
3030 */
31−#define __TLBI_0(op, arg) asm ("tlbi " #op "\n" \
31+#define __TLBI_0(op, arg) asm (ARM64_ASM_PREAMBLE \
32+ "tlbi " #op "\n" \
3233 ALTERNATIVE("nop\n nop", \
3334 "dsb ish\n tlbi " #op, \
3435 ARM64_WORKAROUND_REPEAT_TLBI, \
3536 CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \
3637 : : )
3738
38−#define __TLBI_1(op, arg) asm ("tlbi " #op ", %0\n" \
39+#define __TLBI_1(op, arg) asm (ARM64_ASM_PREAMBLE \
40+ "tlbi " #op ", %0\n" \
3941 ALTERNATIVE("nop\n nop", \
4042 "dsb ish\n tlbi " #op ", %0", \
4143 ARM64_WORKAROUND_REPEAT_TLBI, \
arch/arm64/kernel/image-vars.h+4−0
@@ -103,6 +103,10 @@ KVM_NVHE_ALIAS(vgic_v3_cpuif_trap); · 102 unmodified lines
103103 KVM_NVHE_ALIAS(gic_pmr_sync);
104104 #endif
105105
106+/* EL2 exception handling */
107+KVM_NVHE_ALIAS(__start___kvm_ex_table);
108+KVM_NVHE_ALIAS(__stop___kvm_ex_table);
109+
106110 #endif /* CONFIG_KVM */
107111
108112 #endif /* __ARM64_KERNEL_IMAGE_VARS_H */
arch/arm64/kernel/vmlinux.lds.S+8−0
@@ -20,6 +20,13 @@ ENTRY(_text) · 19 unmodified lines
2020
2121 jiffies = jiffies_64;
2222
23+
24+#define HYPERVISOR_EXTABLE \
25+ . = ALIGN(SZ_8); \
26+ __start___kvm_ex_table = .; \
27+ *(__kvm_ex_table) \
28+ __stop___kvm_ex_table = .;
29+
2330 #define HYPERVISOR_TEXT \
2431 /* \
2532 * Align to 4 KB so that \
@@ -35,6 +42,7 @@ jiffies = jiffies_64; · 9 unmodified lines
3542 __hyp_idmap_text_end = .; \
3643 __hyp_text_start = .; \
3744 *(.hyp.text) \
45+ HYPERVISOR_EXTABLE \
3846 __hyp_text_end = .;
3947
4048 #define IDMAP_TEXT \
arch/arm64/kvm/hyp/entry.S+9−6
@@ -196,20 +196,23 @@ alternative_endif · 195 unmodified lines
196196 // This is our single instruction exception window. A pending
197197 // SError is guaranteed to occur at the earliest when we unmask
198198 // it, and at the latest just after the ISB.
199− .global abort_guest_exit_start
200199 abort_guest_exit_start:
201200
202201 isb
203202
204− .global abort_guest_exit_end
205203 abort_guest_exit_end:
206204
207205 msr daifset, #4 // Mask aborts
206+ ret
207+
208+ _kvm_extable abort_guest_exit_start, 9997f
209+ _kvm_extable abort_guest_exit_end, 9997f
210+9997:
211+ msr daifset, #4 // Mask aborts
212+ mov x0, #(1 << ARM_EXIT_WITH_SERROR_BIT)
208213
209− // If the exception took place, restore the EL1 exception
210− // context so that we can report some information.
211− // Merge the exception code with the SError pending bit.
212− tbz x0, #ARM_EXIT_WITH_SERROR_BIT, 1f
214+ // restore the EL1 exception context so that we can report some
215+ // information. Merge the exception code with the SError pending bit.
213216 msr elr_el2, x2
214217 msr esr_el2, x3
215218 msr spsr_el2, x4
arch/arm64/kvm/hyp/hyp-entry.S+41−24
@@ -15,6 +15,30 @@ · 14 unmodified lines
1515 #include <asm/kvm_mmu.h>
1616 #include <asm/mmu.h>
1717
18+.macro save_caller_saved_regs_vect
19+ /* x0 and x1 were saved in the vector entry */
20+ stp x2, x3, [sp, #-16]!
21+ stp x4, x5, [sp, #-16]!
22+ stp x6, x7, [sp, #-16]!
23+ stp x8, x9, [sp, #-16]!
24+ stp x10, x11, [sp, #-16]!
25+ stp x12, x13, [sp, #-16]!
26+ stp x14, x15, [sp, #-16]!
27+ stp x16, x17, [sp, #-16]!
28+.endm
29+
30+.macro restore_caller_saved_regs_vect
31+ ldp x16, x17, [sp], #16
32+ ldp x14, x15, [sp], #16
33+ ldp x12, x13, [sp], #16
34+ ldp x10, x11, [sp], #16
35+ ldp x8, x9, [sp], #16
36+ ldp x6, x7, [sp], #16
37+ ldp x4, x5, [sp], #16
38+ ldp x2, x3, [sp], #16
39+ ldp x0, x1, [sp], #16
40+.endm
41+
1842 .text
1943
2044 .macro do_el2_call
@@ -143,13 +167,19 @@ el1_error: · 122 unmodified lines
143167 b __guest_exit
144168
145169 el2_sync:
146− /* Check for illegal exception return, otherwise panic */
170+ /* Check for illegal exception return */
147171 mrs x0, spsr_el2
172+ tbnz x0, #20, 1f
148173
149− /* if this was something else, then panic! */
150− tst x0, #PSR_IL_BIT
151− b.eq __hyp_panic
174+ save_caller_saved_regs_vect
175+ stp x29, x30, [sp, #-16]!
176+ bl kvm_unexpected_el2_exception
177+ ldp x29, x30, [sp], #16
178+ restore_caller_saved_regs_vect
152179
180+ eret
181+
182+1:
153183 /* Let's attempt a recovery from the illegal exception return */
154184 get_vcpu_ptr x1, x0
155185 mov x0, #ARM_EXCEPTION_IL
@@ -157,27 +187,14 @@ el2_sync: · 1 unmodified line
157187
158188
159189 el2_error:
160− ldp x0, x1, [sp], #16
190+ save_caller_saved_regs_vect
191+ stp x29, x30, [sp, #-16]!
192+
193+ bl kvm_unexpected_el2_exception
194+
195+ ldp x29, x30, [sp], #16
196+ restore_caller_saved_regs_vect
161197
162− /*
163− * Only two possibilities:
164− * 1) Either we come from the exit path, having just unmasked
165− * PSTATE.A: change the return code to an EL2 fault, and
166− * carry on, as we're already in a sane state to handle it.
167− * 2) Or we come from anywhere else, and that's a bug: we panic.
168− *
169− * For (1), x0 contains the original return code and x1 doesn't
170− * contain anything meaningful at that stage. We can reuse them
171− * as temp registers.
172− * For (2), who cares?
173− */
174− mrs x0, elr_el2
175− adr x1, abort_guest_exit_start
176− cmp x0, x1
177− adr x1, abort_guest_exit_end
178− ccmp x0, x1, #4, ne
179− b.ne __hyp_panic
180− mov x0, #(1 << ARM_EXIT_WITH_SERROR_BIT)
181198 eret
182199 sb
183200
arch/arm64/kvm/hyp/include/hyp/switch.h+35−4
@@ -17,6 +17,7 @@ · 16 unmodified lines
1717
1818 #include <asm/barrier.h>
1919 #include <asm/cpufeature.h>
20+#include <asm/extable.h>
2021 #include <asm/kprobes.h>
2122 #include <asm/kvm_asm.h>
2223 #include <asm/kvm_emulate.h>
@@ -29,6 +30,9 @@ · 6 unmodified lines
2930
3031 extern const char __hyp_panic_string[];
3132
33+extern struct exception_table_entry __start___kvm_ex_table;
34+extern struct exception_table_entry __stop___kvm_ex_table;
35+
3236 /* Check whether the FP regs were dirtied while in the host-side run loop: */
3337 static inline bool update_fp_enabled(struct kvm_vcpu *vcpu)
3438 {
@@ -142,10 +146,10 @@ static inline bool __translate_far_to_hpfar(u64 far, u64 *hpfar) · 107 unmodified lines
142146 * saved the guest context yet, and we may return early...
143147 */
144148 par = read_sysreg(par_el1);
145− asm volatile("at s1e1r, %0" : : "r" (far));
146− isb();
147−
148− tmp = read_sysreg(par_el1);
149+ if (!__kvm_at("s1e1r", far))
150+ tmp = read_sysreg(par_el1);
151+ else
152+ tmp = SYS_PAR_EL1_F; /* back to the guest */
149153 write_sysreg(par, par_el1);
150154
151155 if (unlikely(tmp & SYS_PAR_EL1_F))
@@ -508,4 +512,31 @@ static inline void __set_host_arch_workaround_state(struct kvm_vcpu *vcpu) · 356 unmodified lines
508512 #endif
509513 }
510514
515+static inline void __kvm_unexpected_el2_exception(void)
516+{
517+ unsigned long addr, fixup;
518+ struct kvm_cpu_context *host_ctxt;
519+ struct exception_table_entry *entry, *end;
520+ unsigned long elr_el2 = read_sysreg(elr_el2);
521+
522+ entry = hyp_symbol_addr(__start___kvm_ex_table);
523+ end = hyp_symbol_addr(__stop___kvm_ex_table);
524+ host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt;
525+
526+ while (entry < end) {
527+ addr = (unsigned long)&entry->insn + entry->insn;
528+ fixup = (unsigned long)&entry->fixup + entry->fixup;
529+
530+ if (addr != elr_el2) {
531+ entry++;
532+ continue;
533+ }
534+
535+ write_sysreg(fixup, elr_el2);
536+ return;
537+ }
538+
539+ hyp_panic(host_ctxt);
540+}
541+
511542 #endif /* __ARM64_KVM_HYP_SWITCH_H__ */
arch/arm64/kvm/hyp/nvhe/switch.c+5−0
@@ -270,3 +270,8 @@ void __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt) · 269 unmodified lines
270270 read_sysreg(hpfar_el2), par, vcpu);
271271 unreachable();
272272 }
273+
274+asmlinkage void kvm_unexpected_el2_exception(void)
275+{
276+ return __kvm_unexpected_el2_exception();
277+}
arch/arm64/kvm/hyp/vhe/switch.c+5−0
@@ -217,3 +217,8 @@ void __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt) · 216 unmodified lines
217217 __hyp_call_panic(spsr, elr, par, host_ctxt);
218218 unreachable();
219219 }
220+
221+asmlinkage void kvm_unexpected_el2_exception(void)
222+{
223+ return __kvm_unexpected_el2_exception();
224+}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t