History

Merge tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 fixes from Catalin Marinas: - Fix kernel build with the integrated LLVM assembler which doesn't see the -Wa,-march option. - Fix "make vdso_install" when COMPAT_VDSO is disabled. - Make KVM more robust if the AT S1E1R instruction triggers an exception (architecture corner cases). * tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: KVM: arm64: Set HCR_EL2.PTW to prevent AT taking synchronous exception KVM: arm64: Survive synchronous exceptions caused by AT instructions KVM: arm64: Add kvm_extable for vaxorcism code arm64: vdso32: make vdso32 install conditional arm64: use a common .arch preamble for inline assembly

Linus Torvalds committed 6 years ago

Commit 96d454c2 parents ef91bb1 71a7f8c12 files · +172 −41Browse files

Unified · Split

12 files
  1. arch/arm64/Makefile +10 −4
  2. arch/arm64/include/asm/compiler.h +6 −0
  3. arch/arm64/include/asm/kvm_arm.h +2 −1
  4. arch/arm64/include/asm/kvm_asm.h +43 −0
  5. arch/arm64/include/asm/tlbflush.h +4 −2
  6. arch/arm64/kernel/image-vars.h +4 −0
  7. arch/arm64/kernel/vmlinux.lds.S +8 −0
  8. arch/arm64/kvm/hyp/entry.S +9 −6
  9. arch/arm64/kvm/hyp/hyp-entry.S +41 −24
  10. arch/arm64/kvm/hyp/include/hyp/switch.h +35 −4
  11. arch/arm64/kvm/hyp/nvhe/switch.c +5 −0
  12. arch/arm64/kvm/hyp/vhe/switch.c +5 −0
arch/arm64/Makefile+10−4
@@ -82,8 +82,8 @@ endif · 81 unmodified lines
82# compiler to generate them and consequently to break the single image contract82# compiler to generate them and consequently to break the single image contract
83# we pass it only to the assembler. This option is utilized only in case of non83# we pass it only to the assembler. This option is utilized only in case of non
84# integrated assemblers.84# integrated assemblers.
85ifneq ($(CONFIG_AS_HAS_ARMV8_4), y)85ifeq ($(CONFIG_AS_HAS_PAC), y)
86branch-prot-flags-$(CONFIG_AS_HAS_PAC) += -Wa,-march=armv8.3-a86asm-arch := armv8.3-a
87endif87endif
88endif88endif
89 89
@@ -91,7 +91,12 @@ KBUILD_CFLAGS += $(branch-prot-flags-y) · 1 unmodified line
91 91
92ifeq ($(CONFIG_AS_HAS_ARMV8_4), y)92ifeq ($(CONFIG_AS_HAS_ARMV8_4), y)
93# make sure to pass the newest target architecture to -march.93# make sure to pass the newest target architecture to -march.
94KBUILD_CFLAGS += -Wa,-march=armv8.4-a94asm-arch := armv8.4-a
95endif
96
97ifdef asm-arch
98KBUILD_CFLAGS += -Wa,-march=$(asm-arch) \
99 -DARM64_ASM_ARCH='"$(asm-arch)"'
95endif100endif
96 101
97ifeq ($(CONFIG_SHADOW_CALL_STACK), y)102ifeq ($(CONFIG_SHADOW_CALL_STACK), y)
@@ -165,7 +170,8 @@ zinstall install: · 67 unmodified lines
165PHONY += vdso_install170PHONY += vdso_install
166vdso_install:171vdso_install:
167 $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso $@172 $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso $@
168 $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso32 $@173 $(if $(CONFIG_COMPAT_VDSO), \
174 $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso32 $@)
169 175
170# We use MRPROPER_FILES and CLEAN_FILES now176# We use MRPROPER_FILES and CLEAN_FILES now
171archclean:177archclean:
arch/arm64/include/asm/compiler.h+6−0
@@ -2,6 +2,12 @@ · 1 unmodified line
2#ifndef __ASM_COMPILER_H2#ifndef __ASM_COMPILER_H
3#define __ASM_COMPILER_H3#define __ASM_COMPILER_H
4 4
5#ifdef ARM64_ASM_ARCH
6#define ARM64_ASM_PREAMBLE ".arch " ARM64_ASM_ARCH "\n"
7#else
8#define ARM64_ASM_PREAMBLE
9#endif
10
5/*11/*
6 * The EL0/EL1 pointer bits used by a pointer authentication code.12 * The EL0/EL1 pointer bits used by a pointer authentication code.
7 * This is dependent on TBI0/TBI1 being enabled, or bits 63:56 would also apply.13 * This is dependent on TBI0/TBI1 being enabled, or bits 63:56 would also apply.
arch/arm64/include/asm/kvm_arm.h+2−1
@@ -71,11 +71,12 @@ · 70 unmodified lines
71 * IMO: Override CPSR.I and enable signaling with VI71 * IMO: Override CPSR.I and enable signaling with VI
72 * FMO: Override CPSR.F and enable signaling with VF72 * FMO: Override CPSR.F and enable signaling with VF
73 * SWIO: Turn set/way invalidates into set/way clean+invalidate73 * SWIO: Turn set/way invalidates into set/way clean+invalidate
74 * PTW: Take a stage2 fault if a stage1 walk steps in device memory
74 */75 */
75#define HCR_GUEST_FLAGS (HCR_TSC | HCR_TSW | HCR_TWE | HCR_TWI | HCR_VM | \76#define HCR_GUEST_FLAGS (HCR_TSC | HCR_TSW | HCR_TWE | HCR_TWI | HCR_VM | \
76 HCR_BSU_IS | HCR_FB | HCR_TAC | \77 HCR_BSU_IS | HCR_FB | HCR_TAC | \
77 HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \78 HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \
78 HCR_FMO | HCR_IMO)79 HCR_FMO | HCR_IMO | HCR_PTW )
79#define HCR_VIRT_EXCP_MASK (HCR_VSE | HCR_VI | HCR_VF)80#define HCR_VIRT_EXCP_MASK (HCR_VSE | HCR_VI | HCR_VF)
80#define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK)81#define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK)
81#define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H)82#define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H)
arch/arm64/include/asm/kvm_asm.h+43−0
@@ -169,6 +169,34 @@ extern char __smccc_workaround_1_smc[__SMCCC_WORKAROUND_1_SMC_SZ]; · 168 unmodified lines
169 *__hyp_this_cpu_ptr(sym); \169 *__hyp_this_cpu_ptr(sym); \
170 })170 })
171 171
172#define __KVM_EXTABLE(from, to) \
173 " .pushsection __kvm_ex_table, \"a\"\n" \
174 " .align 3\n" \
175 " .long (" #from " - .), (" #to " - .)\n" \
176 " .popsection\n"
177
178
179#define __kvm_at(at_op, addr) \
180( { \
181 int __kvm_at_err = 0; \
182 u64 spsr, elr; \
183 asm volatile( \
184 " mrs %1, spsr_el2\n" \
185 " mrs %2, elr_el2\n" \
186 "1: at "at_op", %3\n" \
187 " isb\n" \
188 " b 9f\n" \
189 "2: msr spsr_el2, %1\n" \
190 " msr elr_el2, %2\n" \
191 " mov %w0, %4\n" \
192 "9:\n" \
193 __KVM_EXTABLE(1b, 2b) \
194 : "+r" (__kvm_at_err), "=&r" (spsr), "=&r" (elr) \
195 : "r" (addr), "i" (-EFAULT)); \
196 __kvm_at_err; \
197} )
198
199
172#else /* __ASSEMBLY__ */200#else /* __ASSEMBLY__ */
173 201
174.macro hyp_adr_this_cpu reg, sym, tmp202.macro hyp_adr_this_cpu reg, sym, tmp
@@ -193,6 +221,21 @@ extern char __smccc_workaround_1_smc[__SMCCC_WORKAROUND_1_SMC_SZ]; · 18 unmodified lines
193 ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]221 ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU]
194.endm222.endm
195 223
224/*
225 * KVM extable for unexpected exceptions.
226 * In the same format _asm_extable, but output to a different section so that
227 * it can be mapped to EL2. The KVM version is not sorted. The caller must
228 * ensure:
229 * x18 has the hypervisor value to allow any Shadow-Call-Stack instrumented
230 * code to write to it, and that SPSR_EL2 and ELR_EL2 are restored by the fixup.
231 */
232.macro _kvm_extable, from, to
233 .pushsection __kvm_ex_table, "a"
234 .align 3
235 .long (\from - .), (\to - .)
236 .popsection
237.endm
238
196#endif239#endif
197 240
198#endif /* __ARM_KVM_ASM_H__ */241#endif /* __ARM_KVM_ASM_H__ */
arch/arm64/include/asm/tlbflush.h+4−2
@@ -28,14 +28,16 @@ · 27 unmodified lines
28 * not. The macros handles invoking the asm with or without the28 * not. The macros handles invoking the asm with or without the
29 * register argument as appropriate.29 * register argument as appropriate.
30 */30 */
31#define __TLBI_0(op, arg) asm ("tlbi " #op "\n" \31#define __TLBI_0(op, arg) asm (ARM64_ASM_PREAMBLE \
32 "tlbi " #op "\n" \
32 ALTERNATIVE("nop\n nop", \33 ALTERNATIVE("nop\n nop", \
33 "dsb ish\n tlbi " #op, \34 "dsb ish\n tlbi " #op, \
34 ARM64_WORKAROUND_REPEAT_TLBI, \35 ARM64_WORKAROUND_REPEAT_TLBI, \
35 CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \36 CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \
36 : : )37 : : )
37 38
38#define __TLBI_1(op, arg) asm ("tlbi " #op ", %0\n" \39#define __TLBI_1(op, arg) asm (ARM64_ASM_PREAMBLE \
40 "tlbi " #op ", %0\n" \
39 ALTERNATIVE("nop\n nop", \41 ALTERNATIVE("nop\n nop", \
40 "dsb ish\n tlbi " #op ", %0", \42 "dsb ish\n tlbi " #op ", %0", \
41 ARM64_WORKAROUND_REPEAT_TLBI, \43 ARM64_WORKAROUND_REPEAT_TLBI, \
arch/arm64/kernel/image-vars.h+4−0
@@ -103,6 +103,10 @@ KVM_NVHE_ALIAS(vgic_v3_cpuif_trap); · 102 unmodified lines
103KVM_NVHE_ALIAS(gic_pmr_sync);103KVM_NVHE_ALIAS(gic_pmr_sync);
104#endif104#endif
105 105
106/* EL2 exception handling */
107KVM_NVHE_ALIAS(__start___kvm_ex_table);
108KVM_NVHE_ALIAS(__stop___kvm_ex_table);
109
106#endif /* CONFIG_KVM */110#endif /* CONFIG_KVM */
107 111
108#endif /* __ARM64_KERNEL_IMAGE_VARS_H */112#endif /* __ARM64_KERNEL_IMAGE_VARS_H */
arch/arm64/kernel/vmlinux.lds.S+8−0
@@ -20,6 +20,13 @@ ENTRY(_text) · 19 unmodified lines
20 20
21jiffies = jiffies_64;21jiffies = jiffies_64;
22 22
23
24#define HYPERVISOR_EXTABLE \
25 . = ALIGN(SZ_8); \
26 __start___kvm_ex_table = .; \
27 *(__kvm_ex_table) \
28 __stop___kvm_ex_table = .;
29
23#define HYPERVISOR_TEXT \30#define HYPERVISOR_TEXT \
24 /* \31 /* \
25 * Align to 4 KB so that \32 * Align to 4 KB so that \
@@ -35,6 +42,7 @@ jiffies = jiffies_64; · 9 unmodified lines
35 __hyp_idmap_text_end = .; \42 __hyp_idmap_text_end = .; \
36 __hyp_text_start = .; \43 __hyp_text_start = .; \
37 *(.hyp.text) \44 *(.hyp.text) \
45 HYPERVISOR_EXTABLE \
38 __hyp_text_end = .;46 __hyp_text_end = .;
39 47
40#define IDMAP_TEXT \48#define IDMAP_TEXT \
arch/arm64/kvm/hyp/entry.S+9−6
@@ -196,20 +196,23 @@ alternative_endif · 195 unmodified lines
196 // This is our single instruction exception window. A pending196 // This is our single instruction exception window. A pending
197 // SError is guaranteed to occur at the earliest when we unmask197 // SError is guaranteed to occur at the earliest when we unmask
198 // it, and at the latest just after the ISB.198 // it, and at the latest just after the ISB.
199 .global abort_guest_exit_start
200abort_guest_exit_start:199abort_guest_exit_start:
201 200
202 isb201 isb
203 202
204 .global abort_guest_exit_end
205abort_guest_exit_end:203abort_guest_exit_end:
206 204
207 msr daifset, #4 // Mask aborts205 msr daifset, #4 // Mask aborts
206 ret
207
208 _kvm_extable abort_guest_exit_start, 9997f
209 _kvm_extable abort_guest_exit_end, 9997f
2109997:
211 msr daifset, #4 // Mask aborts
212 mov x0, #(1 << ARM_EXIT_WITH_SERROR_BIT)
208 213
209 // If the exception took place, restore the EL1 exception214 // restore the EL1 exception context so that we can report some
210 // context so that we can report some information.215 // information. Merge the exception code with the SError pending bit.
211 // Merge the exception code with the SError pending bit.
212 tbz x0, #ARM_EXIT_WITH_SERROR_BIT, 1f
213 msr elr_el2, x2216 msr elr_el2, x2
214 msr esr_el2, x3217 msr esr_el2, x3
215 msr spsr_el2, x4218 msr spsr_el2, x4
arch/arm64/kvm/hyp/hyp-entry.S+41−24
@@ -15,6 +15,30 @@ · 14 unmodified lines
15#include <asm/kvm_mmu.h>15#include <asm/kvm_mmu.h>
16#include <asm/mmu.h>16#include <asm/mmu.h>
17 17
18.macro save_caller_saved_regs_vect
19 /* x0 and x1 were saved in the vector entry */
20 stp x2, x3, [sp, #-16]!
21 stp x4, x5, [sp, #-16]!
22 stp x6, x7, [sp, #-16]!
23 stp x8, x9, [sp, #-16]!
24 stp x10, x11, [sp, #-16]!
25 stp x12, x13, [sp, #-16]!
26 stp x14, x15, [sp, #-16]!
27 stp x16, x17, [sp, #-16]!
28.endm
29
30.macro restore_caller_saved_regs_vect
31 ldp x16, x17, [sp], #16
32 ldp x14, x15, [sp], #16
33 ldp x12, x13, [sp], #16
34 ldp x10, x11, [sp], #16
35 ldp x8, x9, [sp], #16
36 ldp x6, x7, [sp], #16
37 ldp x4, x5, [sp], #16
38 ldp x2, x3, [sp], #16
39 ldp x0, x1, [sp], #16
40.endm
41
18 .text42 .text
19 43
20.macro do_el2_call44.macro do_el2_call
@@ -143,13 +167,19 @@ el1_error: · 122 unmodified lines
143 b __guest_exit167 b __guest_exit
144 168
145el2_sync:169el2_sync:
146 /* Check for illegal exception return, otherwise panic */170 /* Check for illegal exception return */
147 mrs x0, spsr_el2171 mrs x0, spsr_el2
172 tbnz x0, #20, 1f
148 173
149 /* if this was something else, then panic! */174 save_caller_saved_regs_vect
150 tst x0, #PSR_IL_BIT175 stp x29, x30, [sp, #-16]!
151 b.eq __hyp_panic176 bl kvm_unexpected_el2_exception
177 ldp x29, x30, [sp], #16
178 restore_caller_saved_regs_vect
152 179
180 eret
181
1821:
153 /* Let's attempt a recovery from the illegal exception return */183 /* Let's attempt a recovery from the illegal exception return */
154 get_vcpu_ptr x1, x0184 get_vcpu_ptr x1, x0
155 mov x0, #ARM_EXCEPTION_IL185 mov x0, #ARM_EXCEPTION_IL
@@ -157,27 +187,14 @@ el2_sync: · 1 unmodified line
157 187
158 188
159el2_error:189el2_error:
160 ldp x0, x1, [sp], #16190 save_caller_saved_regs_vect
191 stp x29, x30, [sp, #-16]!
192
193 bl kvm_unexpected_el2_exception
194
195 ldp x29, x30, [sp], #16
196 restore_caller_saved_regs_vect
161 197
162 /*
163 * Only two possibilities:
164 * 1) Either we come from the exit path, having just unmasked
165 * PSTATE.A: change the return code to an EL2 fault, and
166 * carry on, as we're already in a sane state to handle it.
167 * 2) Or we come from anywhere else, and that's a bug: we panic.
168 *
169 * For (1), x0 contains the original return code and x1 doesn't
170 * contain anything meaningful at that stage. We can reuse them
171 * as temp registers.
172 * For (2), who cares?
173 */
174 mrs x0, elr_el2
175 adr x1, abort_guest_exit_start
176 cmp x0, x1
177 adr x1, abort_guest_exit_end
178 ccmp x0, x1, #4, ne
179 b.ne __hyp_panic
180 mov x0, #(1 << ARM_EXIT_WITH_SERROR_BIT)
181 eret198 eret
182 sb199 sb
183 200
arch/arm64/kvm/hyp/include/hyp/switch.h+35−4
@@ -17,6 +17,7 @@ · 16 unmodified lines
17 17
18#include <asm/barrier.h>18#include <asm/barrier.h>
19#include <asm/cpufeature.h>19#include <asm/cpufeature.h>
20#include <asm/extable.h>
20#include <asm/kprobes.h>21#include <asm/kprobes.h>
21#include <asm/kvm_asm.h>22#include <asm/kvm_asm.h>
22#include <asm/kvm_emulate.h>23#include <asm/kvm_emulate.h>
@@ -29,6 +30,9 @@ · 6 unmodified lines
29 30
30extern const char __hyp_panic_string[];31extern const char __hyp_panic_string[];
31 32
33extern struct exception_table_entry __start___kvm_ex_table;
34extern struct exception_table_entry __stop___kvm_ex_table;
35
32/* Check whether the FP regs were dirtied while in the host-side run loop: */36/* Check whether the FP regs were dirtied while in the host-side run loop: */
33static inline bool update_fp_enabled(struct kvm_vcpu *vcpu)37static inline bool update_fp_enabled(struct kvm_vcpu *vcpu)
34{38{
@@ -142,10 +146,10 @@ static inline bool __translate_far_to_hpfar(u64 far, u64 *hpfar) · 107 unmodified lines
142 * saved the guest context yet, and we may return early...146 * saved the guest context yet, and we may return early...
143 */147 */
144 par = read_sysreg(par_el1);148 par = read_sysreg(par_el1);
145 asm volatile("at s1e1r, %0" : : "r" (far));149 if (!__kvm_at("s1e1r", far))
146 isb();150 tmp = read_sysreg(par_el1);
147 151 else
148 tmp = read_sysreg(par_el1);152 tmp = SYS_PAR_EL1_F; /* back to the guest */
149 write_sysreg(par, par_el1);153 write_sysreg(par, par_el1);
150 154
151 if (unlikely(tmp & SYS_PAR_EL1_F))155 if (unlikely(tmp & SYS_PAR_EL1_F))
@@ -508,4 +512,31 @@ static inline void __set_host_arch_workaround_state(struct kvm_vcpu *vcpu) · 356 unmodified lines
508#endif512#endif
509}513}
510 514
515static inline void __kvm_unexpected_el2_exception(void)
516{
517 unsigned long addr, fixup;
518 struct kvm_cpu_context *host_ctxt;
519 struct exception_table_entry *entry, *end;
520 unsigned long elr_el2 = read_sysreg(elr_el2);
521
522 entry = hyp_symbol_addr(__start___kvm_ex_table);
523 end = hyp_symbol_addr(__stop___kvm_ex_table);
524 host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt;
525
526 while (entry < end) {
527 addr = (unsigned long)&entry->insn + entry->insn;
528 fixup = (unsigned long)&entry->fixup + entry->fixup;
529
530 if (addr != elr_el2) {
531 entry++;
532 continue;
533 }
534
535 write_sysreg(fixup, elr_el2);
536 return;
537 }
538
539 hyp_panic(host_ctxt);
540}
541
511#endif /* __ARM64_KVM_HYP_SWITCH_H__ */542#endif /* __ARM64_KVM_HYP_SWITCH_H__ */
arch/arm64/kvm/hyp/nvhe/switch.c+5−0
@@ -270,3 +270,8 @@ void __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt) · 269 unmodified lines
270 read_sysreg(hpfar_el2), par, vcpu);270 read_sysreg(hpfar_el2), par, vcpu);
271 unreachable();271 unreachable();
272}272}
273
274asmlinkage void kvm_unexpected_el2_exception(void)
275{
276 return __kvm_unexpected_el2_exception();
277}
arch/arm64/kvm/hyp/vhe/switch.c+5−0
@@ -217,3 +217,8 @@ void __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt) · 216 unmodified lines
217 __hyp_call_panic(spsr, elr, par, host_ctxt);217 __hyp_call_panic(spsr, elr, par, host_ctxt);
218 unreachable();218 unreachable();
219}219}
220
221asmlinkage void kvm_unexpected_el2_exception(void)
222{
223 return __kvm_unexpected_el2_exception();
224}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t