Pull arm64 fixes from Catalin Marinas: - Fix kernel build with the integrated LLVM assembler which doesn't see the -Wa,-march option. - Fix "make vdso_install" when COMPAT_VDSO is disabled. - Make KVM more robust if the AT S1E1R instruction triggers an exception (architecture corner cases). * tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: KVM: arm64: Set HCR_EL2.PTW to prevent AT taking synchronous exception KVM: arm64: Survive synchronous exceptions caused by AT instructions KVM: arm64: Add kvm_extable for vaxorcism code arm64: vdso32: make vdso32 install conditional arm64: use a common .arch preamble for inline assembly
Unified · Split
| @@ -82,8 +82,8 @@ endif · 81 unmodified lines | ||
| 82 | 82 | # compiler to generate them and consequently to break the single image contract |
| 83 | 83 | # we pass it only to the assembler. This option is utilized only in case of non |
| 84 | 84 | # integrated assemblers. |
| 85 | −ifneq ($(CONFIG_AS_HAS_ARMV8_4), y) | |
| 86 | −branch-prot-flags-$(CONFIG_AS_HAS_PAC) += -Wa,-march=armv8.3-a | |
| 85 | +ifeq ($(CONFIG_AS_HAS_PAC), y) | |
| 86 | +asm-arch := armv8.3-a | |
| 87 | 87 | endif |
| 88 | 88 | endif |
| 89 | 89 | |
| @@ -91,7 +91,12 @@ KBUILD_CFLAGS += $(branch-prot-flags-y) · 1 unmodified line | ||
| 91 | 91 | |
| 92 | 92 | ifeq ($(CONFIG_AS_HAS_ARMV8_4), y) |
| 93 | 93 | # make sure to pass the newest target architecture to -march. |
| 94 | −KBUILD_CFLAGS += -Wa,-march=armv8.4-a | |
| 94 | +asm-arch := armv8.4-a | |
| 95 | +endif | |
| 96 | + | |
| 97 | +ifdef asm-arch | |
| 98 | +KBUILD_CFLAGS += -Wa,-march=$(asm-arch) \ | |
| 99 | + -DARM64_ASM_ARCH='"$(asm-arch)"' | |
| 95 | 100 | endif |
| 96 | 101 | |
| 97 | 102 | ifeq ($(CONFIG_SHADOW_CALL_STACK), y) |
| @@ -165,7 +170,8 @@ zinstall install: · 67 unmodified lines | ||
| 165 | 170 | PHONY += vdso_install |
| 166 | 171 | vdso_install: |
| 167 | 172 | $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso $@ |
| 168 | − $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso32 $@ | |
| 173 | + $(if $(CONFIG_COMPAT_VDSO), \ | |
| 174 | + $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso32 $@) | |
| 169 | 175 | |
| 170 | 176 | # We use MRPROPER_FILES and CLEAN_FILES now |
| 171 | 177 | archclean: |
| @@ -2,6 +2,12 @@ · 1 unmodified line | ||
| 2 | 2 | #ifndef __ASM_COMPILER_H |
| 3 | 3 | #define __ASM_COMPILER_H |
| 4 | 4 | |
| 5 | +#ifdef ARM64_ASM_ARCH | |
| 6 | +#define ARM64_ASM_PREAMBLE ".arch " ARM64_ASM_ARCH "\n" | |
| 7 | +#else | |
| 8 | +#define ARM64_ASM_PREAMBLE | |
| 9 | +#endif | |
| 10 | + | |
| 5 | 11 | /* |
| 6 | 12 | * The EL0/EL1 pointer bits used by a pointer authentication code. |
| 7 | 13 | * This is dependent on TBI0/TBI1 being enabled, or bits 63:56 would also apply. |
| @@ -71,11 +71,12 @@ · 70 unmodified lines | ||
| 71 | 71 | * IMO: Override CPSR.I and enable signaling with VI |
| 72 | 72 | * FMO: Override CPSR.F and enable signaling with VF |
| 73 | 73 | * SWIO: Turn set/way invalidates into set/way clean+invalidate |
| 74 | + * PTW: Take a stage2 fault if a stage1 walk steps in device memory | |
| 74 | 75 | */ |
| 75 | 76 | #define HCR_GUEST_FLAGS (HCR_TSC | HCR_TSW | HCR_TWE | HCR_TWI | HCR_VM | \ |
| 76 | 77 | HCR_BSU_IS | HCR_FB | HCR_TAC | \ |
| 77 | 78 | HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \ |
| 78 | − HCR_FMO | HCR_IMO) | |
| 79 | + HCR_FMO | HCR_IMO | HCR_PTW ) | |
| 79 | 80 | #define HCR_VIRT_EXCP_MASK (HCR_VSE | HCR_VI | HCR_VF) |
| 80 | 81 | #define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK) |
| 81 | 82 | #define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H) |
| @@ -169,6 +169,34 @@ extern char __smccc_workaround_1_smc[__SMCCC_WORKAROUND_1_SMC_SZ]; · 168 unmodified lines | ||
| 169 | 169 | *__hyp_this_cpu_ptr(sym); \ |
| 170 | 170 | }) |
| 171 | 171 | |
| 172 | +#define __KVM_EXTABLE(from, to) \ | |
| 173 | + " .pushsection __kvm_ex_table, \"a\"\n" \ | |
| 174 | + " .align 3\n" \ | |
| 175 | + " .long (" #from " - .), (" #to " - .)\n" \ | |
| 176 | + " .popsection\n" | |
| 177 | + | |
| 178 | + | |
| 179 | +#define __kvm_at(at_op, addr) \ | |
| 180 | +( { \ | |
| 181 | + int __kvm_at_err = 0; \ | |
| 182 | + u64 spsr, elr; \ | |
| 183 | + asm volatile( \ | |
| 184 | + " mrs %1, spsr_el2\n" \ | |
| 185 | + " mrs %2, elr_el2\n" \ | |
| 186 | + "1: at "at_op", %3\n" \ | |
| 187 | + " isb\n" \ | |
| 188 | + " b 9f\n" \ | |
| 189 | + "2: msr spsr_el2, %1\n" \ | |
| 190 | + " msr elr_el2, %2\n" \ | |
| 191 | + " mov %w0, %4\n" \ | |
| 192 | + "9:\n" \ | |
| 193 | + __KVM_EXTABLE(1b, 2b) \ | |
| 194 | + : "+r" (__kvm_at_err), "=&r" (spsr), "=&r" (elr) \ | |
| 195 | + : "r" (addr), "i" (-EFAULT)); \ | |
| 196 | + __kvm_at_err; \ | |
| 197 | +} ) | |
| 198 | + | |
| 199 | + | |
| 172 | 200 | #else /* __ASSEMBLY__ */ |
| 173 | 201 | |
| 174 | 202 | .macro hyp_adr_this_cpu reg, sym, tmp |
| @@ -193,6 +221,21 @@ extern char __smccc_workaround_1_smc[__SMCCC_WORKAROUND_1_SMC_SZ]; · 18 unmodified lines | ||
| 193 | 221 | ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU] |
| 194 | 222 | .endm |
| 195 | 223 | |
| 224 | +/* | |
| 225 | + * KVM extable for unexpected exceptions. | |
| 226 | + * In the same format _asm_extable, but output to a different section so that | |
| 227 | + * it can be mapped to EL2. The KVM version is not sorted. The caller must | |
| 228 | + * ensure: | |
| 229 | + * x18 has the hypervisor value to allow any Shadow-Call-Stack instrumented | |
| 230 | + * code to write to it, and that SPSR_EL2 and ELR_EL2 are restored by the fixup. | |
| 231 | + */ | |
| 232 | +.macro _kvm_extable, from, to | |
| 233 | + .pushsection __kvm_ex_table, "a" | |
| 234 | + .align 3 | |
| 235 | + .long (\from - .), (\to - .) | |
| 236 | + .popsection | |
| 237 | +.endm | |
| 238 | + | |
| 196 | 239 | #endif |
| 197 | 240 | |
| 198 | 241 | #endif /* __ARM_KVM_ASM_H__ */ |
| @@ -28,14 +28,16 @@ · 27 unmodified lines | ||
| 28 | 28 | * not. The macros handles invoking the asm with or without the |
| 29 | 29 | * register argument as appropriate. |
| 30 | 30 | */ |
| 31 | −#define __TLBI_0(op, arg) asm ("tlbi " #op "\n" \ | |
| 31 | +#define __TLBI_0(op, arg) asm (ARM64_ASM_PREAMBLE \ | |
| 32 | + "tlbi " #op "\n" \ | |
| 32 | 33 | ALTERNATIVE("nop\n nop", \ |
| 33 | 34 | "dsb ish\n tlbi " #op, \ |
| 34 | 35 | ARM64_WORKAROUND_REPEAT_TLBI, \ |
| 35 | 36 | CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \ |
| 36 | 37 | : : ) |
| 37 | 38 | |
| 38 | −#define __TLBI_1(op, arg) asm ("tlbi " #op ", %0\n" \ | |
| 39 | +#define __TLBI_1(op, arg) asm (ARM64_ASM_PREAMBLE \ | |
| 40 | + "tlbi " #op ", %0\n" \ | |
| 39 | 41 | ALTERNATIVE("nop\n nop", \ |
| 40 | 42 | "dsb ish\n tlbi " #op ", %0", \ |
| 41 | 43 | ARM64_WORKAROUND_REPEAT_TLBI, \ |
| @@ -103,6 +103,10 @@ KVM_NVHE_ALIAS(vgic_v3_cpuif_trap); · 102 unmodified lines | ||
| 103 | 103 | KVM_NVHE_ALIAS(gic_pmr_sync); |
| 104 | 104 | #endif |
| 105 | 105 | |
| 106 | +/* EL2 exception handling */ | |
| 107 | +KVM_NVHE_ALIAS(__start___kvm_ex_table); | |
| 108 | +KVM_NVHE_ALIAS(__stop___kvm_ex_table); | |
| 109 | + | |
| 106 | 110 | #endif /* CONFIG_KVM */ |
| 107 | 111 | |
| 108 | 112 | #endif /* __ARM64_KERNEL_IMAGE_VARS_H */ |
| @@ -20,6 +20,13 @@ ENTRY(_text) · 19 unmodified lines | ||
| 20 | 20 | |
| 21 | 21 | jiffies = jiffies_64; |
| 22 | 22 | |
| 23 | + | |
| 24 | +#define HYPERVISOR_EXTABLE \ | |
| 25 | + . = ALIGN(SZ_8); \ | |
| 26 | + __start___kvm_ex_table = .; \ | |
| 27 | + *(__kvm_ex_table) \ | |
| 28 | + __stop___kvm_ex_table = .; | |
| 29 | + | |
| 23 | 30 | #define HYPERVISOR_TEXT \ |
| 24 | 31 | /* \ |
| 25 | 32 | * Align to 4 KB so that \ |
| @@ -35,6 +42,7 @@ jiffies = jiffies_64; · 9 unmodified lines | ||
| 35 | 42 | __hyp_idmap_text_end = .; \ |
| 36 | 43 | __hyp_text_start = .; \ |
| 37 | 44 | *(.hyp.text) \ |
| 45 | + HYPERVISOR_EXTABLE \ | |
| 38 | 46 | __hyp_text_end = .; |
| 39 | 47 | |
| 40 | 48 | #define IDMAP_TEXT \ |
| @@ -196,20 +196,23 @@ alternative_endif · 195 unmodified lines | ||
| 196 | 196 | // This is our single instruction exception window. A pending |
| 197 | 197 | // SError is guaranteed to occur at the earliest when we unmask |
| 198 | 198 | // it, and at the latest just after the ISB. |
| 199 | − .global abort_guest_exit_start | |
| 200 | 199 | abort_guest_exit_start: |
| 201 | 200 | |
| 202 | 201 | isb |
| 203 | 202 | |
| 204 | − .global abort_guest_exit_end | |
| 205 | 203 | abort_guest_exit_end: |
| 206 | 204 | |
| 207 | 205 | msr daifset, #4 // Mask aborts |
| 206 | + ret | |
| 207 | + | |
| 208 | + _kvm_extable abort_guest_exit_start, 9997f | |
| 209 | + _kvm_extable abort_guest_exit_end, 9997f | |
| 210 | +9997: | |
| 211 | + msr daifset, #4 // Mask aborts | |
| 212 | + mov x0, #(1 << ARM_EXIT_WITH_SERROR_BIT) | |
| 208 | 213 | |
| 209 | − // If the exception took place, restore the EL1 exception | |
| 210 | − // context so that we can report some information. | |
| 211 | − // Merge the exception code with the SError pending bit. | |
| 212 | − tbz x0, #ARM_EXIT_WITH_SERROR_BIT, 1f | |
| 214 | + // restore the EL1 exception context so that we can report some | |
| 215 | + // information. Merge the exception code with the SError pending bit. | |
| 213 | 216 | msr elr_el2, x2 |
| 214 | 217 | msr esr_el2, x3 |
| 215 | 218 | msr spsr_el2, x4 |
| @@ -15,6 +15,30 @@ · 14 unmodified lines | ||
| 15 | 15 | #include <asm/kvm_mmu.h> |
| 16 | 16 | #include <asm/mmu.h> |
| 17 | 17 | |
| 18 | +.macro save_caller_saved_regs_vect | |
| 19 | + /* x0 and x1 were saved in the vector entry */ | |
| 20 | + stp x2, x3, [sp, #-16]! | |
| 21 | + stp x4, x5, [sp, #-16]! | |
| 22 | + stp x6, x7, [sp, #-16]! | |
| 23 | + stp x8, x9, [sp, #-16]! | |
| 24 | + stp x10, x11, [sp, #-16]! | |
| 25 | + stp x12, x13, [sp, #-16]! | |
| 26 | + stp x14, x15, [sp, #-16]! | |
| 27 | + stp x16, x17, [sp, #-16]! | |
| 28 | +.endm | |
| 29 | + | |
| 30 | +.macro restore_caller_saved_regs_vect | |
| 31 | + ldp x16, x17, [sp], #16 | |
| 32 | + ldp x14, x15, [sp], #16 | |
| 33 | + ldp x12, x13, [sp], #16 | |
| 34 | + ldp x10, x11, [sp], #16 | |
| 35 | + ldp x8, x9, [sp], #16 | |
| 36 | + ldp x6, x7, [sp], #16 | |
| 37 | + ldp x4, x5, [sp], #16 | |
| 38 | + ldp x2, x3, [sp], #16 | |
| 39 | + ldp x0, x1, [sp], #16 | |
| 40 | +.endm | |
| 41 | + | |
| 18 | 42 | .text |
| 19 | 43 | |
| 20 | 44 | .macro do_el2_call |
| @@ -143,13 +167,19 @@ el1_error: · 122 unmodified lines | ||
| 143 | 167 | b __guest_exit |
| 144 | 168 | |
| 145 | 169 | el2_sync: |
| 146 | − /* Check for illegal exception return, otherwise panic */ | |
| 170 | + /* Check for illegal exception return */ | |
| 147 | 171 | mrs x0, spsr_el2 |
| 172 | + tbnz x0, #20, 1f | |
| 148 | 173 | |
| 149 | − /* if this was something else, then panic! */ | |
| 150 | − tst x0, #PSR_IL_BIT | |
| 151 | − b.eq __hyp_panic | |
| 174 | + save_caller_saved_regs_vect | |
| 175 | + stp x29, x30, [sp, #-16]! | |
| 176 | + bl kvm_unexpected_el2_exception | |
| 177 | + ldp x29, x30, [sp], #16 | |
| 178 | + restore_caller_saved_regs_vect | |
| 152 | 179 | |
| 180 | + eret | |
| 181 | + | |
| 182 | +1: | |
| 153 | 183 | /* Let's attempt a recovery from the illegal exception return */ |
| 154 | 184 | get_vcpu_ptr x1, x0 |
| 155 | 185 | mov x0, #ARM_EXCEPTION_IL |
| @@ -157,27 +187,14 @@ el2_sync: · 1 unmodified line | ||
| 157 | 187 | |
| 158 | 188 | |
| 159 | 189 | el2_error: |
| 160 | − ldp x0, x1, [sp], #16 | |
| 190 | + save_caller_saved_regs_vect | |
| 191 | + stp x29, x30, [sp, #-16]! | |
| 192 | + | |
| 193 | + bl kvm_unexpected_el2_exception | |
| 194 | + | |
| 195 | + ldp x29, x30, [sp], #16 | |
| 196 | + restore_caller_saved_regs_vect | |
| 161 | 197 | |
| 162 | − /* | |
| 163 | − * Only two possibilities: | |
| 164 | − * 1) Either we come from the exit path, having just unmasked | |
| 165 | − * PSTATE.A: change the return code to an EL2 fault, and | |
| 166 | − * carry on, as we're already in a sane state to handle it. | |
| 167 | − * 2) Or we come from anywhere else, and that's a bug: we panic. | |
| 168 | − * | |
| 169 | − * For (1), x0 contains the original return code and x1 doesn't | |
| 170 | − * contain anything meaningful at that stage. We can reuse them | |
| 171 | − * as temp registers. | |
| 172 | − * For (2), who cares? | |
| 173 | − */ | |
| 174 | − mrs x0, elr_el2 | |
| 175 | − adr x1, abort_guest_exit_start | |
| 176 | − cmp x0, x1 | |
| 177 | − adr x1, abort_guest_exit_end | |
| 178 | − ccmp x0, x1, #4, ne | |
| 179 | − b.ne __hyp_panic | |
| 180 | − mov x0, #(1 << ARM_EXIT_WITH_SERROR_BIT) | |
| 181 | 198 | eret |
| 182 | 199 | sb |
| 183 | 200 | |
| @@ -17,6 +17,7 @@ · 16 unmodified lines | ||
| 17 | 17 | |
| 18 | 18 | #include <asm/barrier.h> |
| 19 | 19 | #include <asm/cpufeature.h> |
| 20 | +#include <asm/extable.h> | |
| 20 | 21 | #include <asm/kprobes.h> |
| 21 | 22 | #include <asm/kvm_asm.h> |
| 22 | 23 | #include <asm/kvm_emulate.h> |
| @@ -29,6 +30,9 @@ · 6 unmodified lines | ||
| 29 | 30 | |
| 30 | 31 | extern const char __hyp_panic_string[]; |
| 31 | 32 | |
| 33 | +extern struct exception_table_entry __start___kvm_ex_table; | |
| 34 | +extern struct exception_table_entry __stop___kvm_ex_table; | |
| 35 | + | |
| 32 | 36 | /* Check whether the FP regs were dirtied while in the host-side run loop: */ |
| 33 | 37 | static inline bool update_fp_enabled(struct kvm_vcpu *vcpu) |
| 34 | 38 | { |
| @@ -142,10 +146,10 @@ static inline bool __translate_far_to_hpfar(u64 far, u64 *hpfar) · 107 unmodified lines | ||
| 142 | 146 | * saved the guest context yet, and we may return early... |
| 143 | 147 | */ |
| 144 | 148 | par = read_sysreg(par_el1); |
| 145 | − asm volatile("at s1e1r, %0" : : "r" (far)); | |
| 146 | − isb(); | |
| 147 | − | |
| 148 | − tmp = read_sysreg(par_el1); | |
| 149 | + if (!__kvm_at("s1e1r", far)) | |
| 150 | + tmp = read_sysreg(par_el1); | |
| 151 | + else | |
| 152 | + tmp = SYS_PAR_EL1_F; /* back to the guest */ | |
| 149 | 153 | write_sysreg(par, par_el1); |
| 150 | 154 | |
| 151 | 155 | if (unlikely(tmp & SYS_PAR_EL1_F)) |
| @@ -508,4 +512,31 @@ static inline void __set_host_arch_workaround_state(struct kvm_vcpu *vcpu) · 356 unmodified lines | ||
| 508 | 512 | #endif |
| 509 | 513 | } |
| 510 | 514 | |
| 515 | +static inline void __kvm_unexpected_el2_exception(void) | |
| 516 | +{ | |
| 517 | + unsigned long addr, fixup; | |
| 518 | + struct kvm_cpu_context *host_ctxt; | |
| 519 | + struct exception_table_entry *entry, *end; | |
| 520 | + unsigned long elr_el2 = read_sysreg(elr_el2); | |
| 521 | + | |
| 522 | + entry = hyp_symbol_addr(__start___kvm_ex_table); | |
| 523 | + end = hyp_symbol_addr(__stop___kvm_ex_table); | |
| 524 | + host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt; | |
| 525 | + | |
| 526 | + while (entry < end) { | |
| 527 | + addr = (unsigned long)&entry->insn + entry->insn; | |
| 528 | + fixup = (unsigned long)&entry->fixup + entry->fixup; | |
| 529 | + | |
| 530 | + if (addr != elr_el2) { | |
| 531 | + entry++; | |
| 532 | + continue; | |
| 533 | + } | |
| 534 | + | |
| 535 | + write_sysreg(fixup, elr_el2); | |
| 536 | + return; | |
| 537 | + } | |
| 538 | + | |
| 539 | + hyp_panic(host_ctxt); | |
| 540 | +} | |
| 541 | + | |
| 511 | 542 | #endif /* __ARM64_KVM_HYP_SWITCH_H__ */ |
| @@ -270,3 +270,8 @@ void __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt) · 269 unmodified lines | ||
| 270 | 270 | read_sysreg(hpfar_el2), par, vcpu); |
| 271 | 271 | unreachable(); |
| 272 | 272 | } |
| 273 | + | |
| 274 | +asmlinkage void kvm_unexpected_el2_exception(void) | |
| 275 | +{ | |
| 276 | + return __kvm_unexpected_el2_exception(); | |
| 277 | +} | |
| @@ -217,3 +217,8 @@ void __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt) · 216 unmodified lines | ||
| 217 | 217 | __hyp_call_panic(spsr, elr, par, host_ctxt); |
| 218 | 218 | unreachable(); |
| 219 | 219 | } |
| 220 | + | |
| 221 | +asmlinkage void kvm_unexpected_el2_exception(void) | |
| 222 | +{ | |
| 223 | + return __kvm_unexpected_el2_exception(); | |
| 224 | +} | |