Pull arm64 fixes from Catalin Marinas: - Fix kernel build with the integrated LLVM assembler which doesn't see the -Wa,-march option. - Fix "make vdso_install" when COMPAT_VDSO is disabled. - Make KVM more robust if the AT S1E1R instruction triggers an exception (architecture corner cases). * tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: KVM: arm64: Set HCR_EL2.PTW to prevent AT taking synchronous exception KVM: arm64: Survive synchronous exceptions caused by AT instructions KVM: arm64: Add kvm_extable for vaxorcism code arm64: vdso32: make vdso32 install conditional arm64: use a common .arch preamble for inline assembly
Unified · Split
| @@ -82,8 +82,8 @@ endif · 81 unmodified lines | |||
| 82 | # compiler to generate them and consequently to break the single image contract | 82 | # compiler to generate them and consequently to break the single image contract |
| 83 | # we pass it only to the assembler. This option is utilized only in case of non | 83 | # we pass it only to the assembler. This option is utilized only in case of non |
| 84 | # integrated assemblers. | 84 | # integrated assemblers. |
| 85 | ifneq ($(CONFIG_AS_HAS_ARMV8_4), y) | 85 | ifeq ($(CONFIG_AS_HAS_PAC), y) |
| 86 | branch-prot-flags-$(CONFIG_AS_HAS_PAC) += -Wa,-march=armv8.3-a | 86 | asm-arch := armv8.3-a |
| 87 | endif | 87 | endif |
| 88 | endif | 88 | endif |
| 89 | 89 | ||
| @@ -91,7 +91,12 @@ KBUILD_CFLAGS += $(branch-prot-flags-y) · 1 unmodified line | |||
| 91 | 91 | ||
| 92 | ifeq ($(CONFIG_AS_HAS_ARMV8_4), y) | 92 | ifeq ($(CONFIG_AS_HAS_ARMV8_4), y) |
| 93 | # make sure to pass the newest target architecture to -march. | 93 | # make sure to pass the newest target architecture to -march. |
| 94 | KBUILD_CFLAGS += -Wa,-march=armv8.4-a | 94 | asm-arch := armv8.4-a |
| 95 | endif | ||
| 96 | |||
| 97 | ifdef asm-arch | ||
| 98 | KBUILD_CFLAGS += -Wa,-march=$(asm-arch) \ | ||
| 99 | -DARM64_ASM_ARCH='"$(asm-arch)"' | ||
| 95 | endif | 100 | endif |
| 96 | 101 | ||
| 97 | ifeq ($(CONFIG_SHADOW_CALL_STACK), y) | 102 | ifeq ($(CONFIG_SHADOW_CALL_STACK), y) |
| @@ -165,7 +170,8 @@ zinstall install: · 67 unmodified lines | |||
| 165 | PHONY += vdso_install | 170 | PHONY += vdso_install |
| 166 | vdso_install: | 171 | vdso_install: |
| 167 | $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso $@ | 172 | $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso $@ |
| 168 | $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso32 $@ | 173 | $(if $(CONFIG_COMPAT_VDSO), \ |
| 174 | $(Q)$(MAKE) $(build)=arch/arm64/kernel/vdso32 $@) | ||
| 169 | 175 | ||
| 170 | # We use MRPROPER_FILES and CLEAN_FILES now | 176 | # We use MRPROPER_FILES and CLEAN_FILES now |
| 171 | archclean: | 177 | archclean: |
| @@ -2,6 +2,12 @@ · 1 unmodified line | |||
| 2 | #ifndef __ASM_COMPILER_H | 2 | #ifndef __ASM_COMPILER_H |
| 3 | #define __ASM_COMPILER_H | 3 | #define __ASM_COMPILER_H |
| 4 | 4 | ||
| 5 | #ifdef ARM64_ASM_ARCH | ||
| 6 | #define ARM64_ASM_PREAMBLE ".arch " ARM64_ASM_ARCH "\n" | ||
| 7 | #else | ||
| 8 | #define ARM64_ASM_PREAMBLE | ||
| 9 | #endif | ||
| 10 | |||
| 5 | /* | 11 | /* |
| 6 | * The EL0/EL1 pointer bits used by a pointer authentication code. | 12 | * The EL0/EL1 pointer bits used by a pointer authentication code. |
| 7 | * This is dependent on TBI0/TBI1 being enabled, or bits 63:56 would also apply. | 13 | * This is dependent on TBI0/TBI1 being enabled, or bits 63:56 would also apply. |
| @@ -71,11 +71,12 @@ · 70 unmodified lines | |||
| 71 | * IMO: Override CPSR.I and enable signaling with VI | 71 | * IMO: Override CPSR.I and enable signaling with VI |
| 72 | * FMO: Override CPSR.F and enable signaling with VF | 72 | * FMO: Override CPSR.F and enable signaling with VF |
| 73 | * SWIO: Turn set/way invalidates into set/way clean+invalidate | 73 | * SWIO: Turn set/way invalidates into set/way clean+invalidate |
| 74 | * PTW: Take a stage2 fault if a stage1 walk steps in device memory | ||
| 74 | */ | 75 | */ |
| 75 | #define HCR_GUEST_FLAGS (HCR_TSC | HCR_TSW | HCR_TWE | HCR_TWI | HCR_VM | \ | 76 | #define HCR_GUEST_FLAGS (HCR_TSC | HCR_TSW | HCR_TWE | HCR_TWI | HCR_VM | \ |
| 76 | HCR_BSU_IS | HCR_FB | HCR_TAC | \ | 77 | HCR_BSU_IS | HCR_FB | HCR_TAC | \ |
| 77 | HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \ | 78 | HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \ |
| 78 | HCR_FMO | HCR_IMO) | 79 | HCR_FMO | HCR_IMO | HCR_PTW ) |
| 79 | #define HCR_VIRT_EXCP_MASK (HCR_VSE | HCR_VI | HCR_VF) | 80 | #define HCR_VIRT_EXCP_MASK (HCR_VSE | HCR_VI | HCR_VF) |
| 80 | #define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK) | 81 | #define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK) |
| 81 | #define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H) | 82 | #define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H) |
| @@ -169,6 +169,34 @@ extern char __smccc_workaround_1_smc[__SMCCC_WORKAROUND_1_SMC_SZ]; · 168 unmodified lines | |||
| 169 | *__hyp_this_cpu_ptr(sym); \ | 169 | *__hyp_this_cpu_ptr(sym); \ |
| 170 | }) | 170 | }) |
| 171 | 171 | ||
| 172 | #define __KVM_EXTABLE(from, to) \ | ||
| 173 | " .pushsection __kvm_ex_table, \"a\"\n" \ | ||
| 174 | " .align 3\n" \ | ||
| 175 | " .long (" #from " - .), (" #to " - .)\n" \ | ||
| 176 | " .popsection\n" | ||
| 177 | |||
| 178 | |||
| 179 | #define __kvm_at(at_op, addr) \ | ||
| 180 | ( { \ | ||
| 181 | int __kvm_at_err = 0; \ | ||
| 182 | u64 spsr, elr; \ | ||
| 183 | asm volatile( \ | ||
| 184 | " mrs %1, spsr_el2\n" \ | ||
| 185 | " mrs %2, elr_el2\n" \ | ||
| 186 | "1: at "at_op", %3\n" \ | ||
| 187 | " isb\n" \ | ||
| 188 | " b 9f\n" \ | ||
| 189 | "2: msr spsr_el2, %1\n" \ | ||
| 190 | " msr elr_el2, %2\n" \ | ||
| 191 | " mov %w0, %4\n" \ | ||
| 192 | "9:\n" \ | ||
| 193 | __KVM_EXTABLE(1b, 2b) \ | ||
| 194 | : "+r" (__kvm_at_err), "=&r" (spsr), "=&r" (elr) \ | ||
| 195 | : "r" (addr), "i" (-EFAULT)); \ | ||
| 196 | __kvm_at_err; \ | ||
| 197 | } ) | ||
| 198 | |||
| 199 | |||
| 172 | #else /* __ASSEMBLY__ */ | 200 | #else /* __ASSEMBLY__ */ |
| 173 | 201 | ||
| 174 | .macro hyp_adr_this_cpu reg, sym, tmp | 202 | .macro hyp_adr_this_cpu reg, sym, tmp |
| @@ -193,6 +221,21 @@ extern char __smccc_workaround_1_smc[__SMCCC_WORKAROUND_1_SMC_SZ]; · 18 unmodified lines | |||
| 193 | ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU] | 221 | ldr \vcpu, [\ctxt, #HOST_CONTEXT_VCPU] |
| 194 | .endm | 222 | .endm |
| 195 | 223 | ||
| 224 | /* | ||
| 225 | * KVM extable for unexpected exceptions. | ||
| 226 | * In the same format _asm_extable, but output to a different section so that | ||
| 227 | * it can be mapped to EL2. The KVM version is not sorted. The caller must | ||
| 228 | * ensure: | ||
| 229 | * x18 has the hypervisor value to allow any Shadow-Call-Stack instrumented | ||
| 230 | * code to write to it, and that SPSR_EL2 and ELR_EL2 are restored by the fixup. | ||
| 231 | */ | ||
| 232 | .macro _kvm_extable, from, to | ||
| 233 | .pushsection __kvm_ex_table, "a" | ||
| 234 | .align 3 | ||
| 235 | .long (\from - .), (\to - .) | ||
| 236 | .popsection | ||
| 237 | .endm | ||
| 238 | |||
| 196 | #endif | 239 | #endif |
| 197 | 240 | ||
| 198 | #endif /* __ARM_KVM_ASM_H__ */ | 241 | #endif /* __ARM_KVM_ASM_H__ */ |
| @@ -28,14 +28,16 @@ · 27 unmodified lines | |||
| 28 | * not. The macros handles invoking the asm with or without the | 28 | * not. The macros handles invoking the asm with or without the |
| 29 | * register argument as appropriate. | 29 | * register argument as appropriate. |
| 30 | */ | 30 | */ |
| 31 | #define __TLBI_0(op, arg) asm ("tlbi " #op "\n" \ | 31 | #define __TLBI_0(op, arg) asm (ARM64_ASM_PREAMBLE \ |
| 32 | "tlbi " #op "\n" \ | ||
| 32 | ALTERNATIVE("nop\n nop", \ | 33 | ALTERNATIVE("nop\n nop", \ |
| 33 | "dsb ish\n tlbi " #op, \ | 34 | "dsb ish\n tlbi " #op, \ |
| 34 | ARM64_WORKAROUND_REPEAT_TLBI, \ | 35 | ARM64_WORKAROUND_REPEAT_TLBI, \ |
| 35 | CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \ | 36 | CONFIG_ARM64_WORKAROUND_REPEAT_TLBI) \ |
| 36 | : : ) | 37 | : : ) |
| 37 | 38 | ||
| 38 | #define __TLBI_1(op, arg) asm ("tlbi " #op ", %0\n" \ | 39 | #define __TLBI_1(op, arg) asm (ARM64_ASM_PREAMBLE \ |
| 40 | "tlbi " #op ", %0\n" \ | ||
| 39 | ALTERNATIVE("nop\n nop", \ | 41 | ALTERNATIVE("nop\n nop", \ |
| 40 | "dsb ish\n tlbi " #op ", %0", \ | 42 | "dsb ish\n tlbi " #op ", %0", \ |
| 41 | ARM64_WORKAROUND_REPEAT_TLBI, \ | 43 | ARM64_WORKAROUND_REPEAT_TLBI, \ |
| @@ -103,6 +103,10 @@ KVM_NVHE_ALIAS(vgic_v3_cpuif_trap); · 102 unmodified lines | |||
| 103 | KVM_NVHE_ALIAS(gic_pmr_sync); | 103 | KVM_NVHE_ALIAS(gic_pmr_sync); |
| 104 | #endif | 104 | #endif |
| 105 | 105 | ||
| 106 | /* EL2 exception handling */ | ||
| 107 | KVM_NVHE_ALIAS(__start___kvm_ex_table); | ||
| 108 | KVM_NVHE_ALIAS(__stop___kvm_ex_table); | ||
| 109 | |||
| 106 | #endif /* CONFIG_KVM */ | 110 | #endif /* CONFIG_KVM */ |
| 107 | 111 | ||
| 108 | #endif /* __ARM64_KERNEL_IMAGE_VARS_H */ | 112 | #endif /* __ARM64_KERNEL_IMAGE_VARS_H */ |
| @@ -20,6 +20,13 @@ ENTRY(_text) · 19 unmodified lines | |||
| 20 | 20 | ||
| 21 | jiffies = jiffies_64; | 21 | jiffies = jiffies_64; |
| 22 | 22 | ||
| 23 | |||
| 24 | #define HYPERVISOR_EXTABLE \ | ||
| 25 | . = ALIGN(SZ_8); \ | ||
| 26 | __start___kvm_ex_table = .; \ | ||
| 27 | *(__kvm_ex_table) \ | ||
| 28 | __stop___kvm_ex_table = .; | ||
| 29 | |||
| 23 | #define HYPERVISOR_TEXT \ | 30 | #define HYPERVISOR_TEXT \ |
| 24 | /* \ | 31 | /* \ |
| 25 | * Align to 4 KB so that \ | 32 | * Align to 4 KB so that \ |
| @@ -35,6 +42,7 @@ jiffies = jiffies_64; · 9 unmodified lines | |||
| 35 | __hyp_idmap_text_end = .; \ | 42 | __hyp_idmap_text_end = .; \ |
| 36 | __hyp_text_start = .; \ | 43 | __hyp_text_start = .; \ |
| 37 | *(.hyp.text) \ | 44 | *(.hyp.text) \ |
| 45 | HYPERVISOR_EXTABLE \ | ||
| 38 | __hyp_text_end = .; | 46 | __hyp_text_end = .; |
| 39 | 47 | ||
| 40 | #define IDMAP_TEXT \ | 48 | #define IDMAP_TEXT \ |
| @@ -196,20 +196,23 @@ alternative_endif · 195 unmodified lines | |||
| 196 | // This is our single instruction exception window. A pending | 196 | // This is our single instruction exception window. A pending |
| 197 | // SError is guaranteed to occur at the earliest when we unmask | 197 | // SError is guaranteed to occur at the earliest when we unmask |
| 198 | // it, and at the latest just after the ISB. | 198 | // it, and at the latest just after the ISB. |
| 199 | .global abort_guest_exit_start | ||
| 200 | abort_guest_exit_start: | 199 | abort_guest_exit_start: |
| 201 | 200 | ||
| 202 | isb | 201 | isb |
| 203 | 202 | ||
| 204 | .global abort_guest_exit_end | ||
| 205 | abort_guest_exit_end: | 203 | abort_guest_exit_end: |
| 206 | 204 | ||
| 207 | msr daifset, #4 // Mask aborts | 205 | msr daifset, #4 // Mask aborts |
| 206 | ret | ||
| 207 | |||
| 208 | _kvm_extable abort_guest_exit_start, 9997f | ||
| 209 | _kvm_extable abort_guest_exit_end, 9997f | ||
| 210 | 9997: | ||
| 211 | msr daifset, #4 // Mask aborts | ||
| 212 | mov x0, #(1 << ARM_EXIT_WITH_SERROR_BIT) | ||
| 208 | 213 | ||
| 209 | // If the exception took place, restore the EL1 exception | 214 | // restore the EL1 exception context so that we can report some |
| 210 | // context so that we can report some information. | 215 | // information. Merge the exception code with the SError pending bit. |
| 211 | // Merge the exception code with the SError pending bit. | ||
| 212 | tbz x0, #ARM_EXIT_WITH_SERROR_BIT, 1f | ||
| 213 | msr elr_el2, x2 | 216 | msr elr_el2, x2 |
| 214 | msr esr_el2, x3 | 217 | msr esr_el2, x3 |
| 215 | msr spsr_el2, x4 | 218 | msr spsr_el2, x4 |
| @@ -15,6 +15,30 @@ · 14 unmodified lines | |||
| 15 | #include <asm/kvm_mmu.h> | 15 | #include <asm/kvm_mmu.h> |
| 16 | #include <asm/mmu.h> | 16 | #include <asm/mmu.h> |
| 17 | 17 | ||
| 18 | .macro save_caller_saved_regs_vect | ||
| 19 | /* x0 and x1 were saved in the vector entry */ | ||
| 20 | stp x2, x3, [sp, #-16]! | ||
| 21 | stp x4, x5, [sp, #-16]! | ||
| 22 | stp x6, x7, [sp, #-16]! | ||
| 23 | stp x8, x9, [sp, #-16]! | ||
| 24 | stp x10, x11, [sp, #-16]! | ||
| 25 | stp x12, x13, [sp, #-16]! | ||
| 26 | stp x14, x15, [sp, #-16]! | ||
| 27 | stp x16, x17, [sp, #-16]! | ||
| 28 | .endm | ||
| 29 | |||
| 30 | .macro restore_caller_saved_regs_vect | ||
| 31 | ldp x16, x17, [sp], #16 | ||
| 32 | ldp x14, x15, [sp], #16 | ||
| 33 | ldp x12, x13, [sp], #16 | ||
| 34 | ldp x10, x11, [sp], #16 | ||
| 35 | ldp x8, x9, [sp], #16 | ||
| 36 | ldp x6, x7, [sp], #16 | ||
| 37 | ldp x4, x5, [sp], #16 | ||
| 38 | ldp x2, x3, [sp], #16 | ||
| 39 | ldp x0, x1, [sp], #16 | ||
| 40 | .endm | ||
| 41 | |||
| 18 | .text | 42 | .text |
| 19 | 43 | ||
| 20 | .macro do_el2_call | 44 | .macro do_el2_call |
| @@ -143,13 +167,19 @@ el1_error: · 122 unmodified lines | |||
| 143 | b __guest_exit | 167 | b __guest_exit |
| 144 | 168 | ||
| 145 | el2_sync: | 169 | el2_sync: |
| 146 | /* Check for illegal exception return, otherwise panic */ | 170 | /* Check for illegal exception return */ |
| 147 | mrs x0, spsr_el2 | 171 | mrs x0, spsr_el2 |
| 172 | tbnz x0, #20, 1f | ||
| 148 | 173 | ||
| 149 | /* if this was something else, then panic! */ | 174 | save_caller_saved_regs_vect |
| 150 | tst x0, #PSR_IL_BIT | 175 | stp x29, x30, [sp, #-16]! |
| 151 | b.eq __hyp_panic | 176 | bl kvm_unexpected_el2_exception |
| 177 | ldp x29, x30, [sp], #16 | ||
| 178 | restore_caller_saved_regs_vect | ||
| 152 | 179 | ||
| 180 | eret | ||
| 181 | |||
| 182 | 1: | ||
| 153 | /* Let's attempt a recovery from the illegal exception return */ | 183 | /* Let's attempt a recovery from the illegal exception return */ |
| 154 | get_vcpu_ptr x1, x0 | 184 | get_vcpu_ptr x1, x0 |
| 155 | mov x0, #ARM_EXCEPTION_IL | 185 | mov x0, #ARM_EXCEPTION_IL |
| @@ -157,27 +187,14 @@ el2_sync: · 1 unmodified line | |||
| 157 | 187 | ||
| 158 | 188 | ||
| 159 | el2_error: | 189 | el2_error: |
| 160 | ldp x0, x1, [sp], #16 | 190 | save_caller_saved_regs_vect |
| 191 | stp x29, x30, [sp, #-16]! | ||
| 192 | |||
| 193 | bl kvm_unexpected_el2_exception | ||
| 194 | |||
| 195 | ldp x29, x30, [sp], #16 | ||
| 196 | restore_caller_saved_regs_vect | ||
| 161 | 197 | ||
| 162 | /* | ||
| 163 | * Only two possibilities: | ||
| 164 | * 1) Either we come from the exit path, having just unmasked | ||
| 165 | * PSTATE.A: change the return code to an EL2 fault, and | ||
| 166 | * carry on, as we're already in a sane state to handle it. | ||
| 167 | * 2) Or we come from anywhere else, and that's a bug: we panic. | ||
| 168 | * | ||
| 169 | * For (1), x0 contains the original return code and x1 doesn't | ||
| 170 | * contain anything meaningful at that stage. We can reuse them | ||
| 171 | * as temp registers. | ||
| 172 | * For (2), who cares? | ||
| 173 | */ | ||
| 174 | mrs x0, elr_el2 | ||
| 175 | adr x1, abort_guest_exit_start | ||
| 176 | cmp x0, x1 | ||
| 177 | adr x1, abort_guest_exit_end | ||
| 178 | ccmp x0, x1, #4, ne | ||
| 179 | b.ne __hyp_panic | ||
| 180 | mov x0, #(1 << ARM_EXIT_WITH_SERROR_BIT) | ||
| 181 | eret | 198 | eret |
| 182 | sb | 199 | sb |
| 183 | 200 | ||
| @@ -17,6 +17,7 @@ · 16 unmodified lines | |||
| 17 | 17 | ||
| 18 | #include <asm/barrier.h> | 18 | #include <asm/barrier.h> |
| 19 | #include <asm/cpufeature.h> | 19 | #include <asm/cpufeature.h> |
| 20 | #include <asm/extable.h> | ||
| 20 | #include <asm/kprobes.h> | 21 | #include <asm/kprobes.h> |
| 21 | #include <asm/kvm_asm.h> | 22 | #include <asm/kvm_asm.h> |
| 22 | #include <asm/kvm_emulate.h> | 23 | #include <asm/kvm_emulate.h> |
| @@ -29,6 +30,9 @@ · 6 unmodified lines | |||
| 29 | 30 | ||
| 30 | extern const char __hyp_panic_string[]; | 31 | extern const char __hyp_panic_string[]; |
| 31 | 32 | ||
| 33 | extern struct exception_table_entry __start___kvm_ex_table; | ||
| 34 | extern struct exception_table_entry __stop___kvm_ex_table; | ||
| 35 | |||
| 32 | /* Check whether the FP regs were dirtied while in the host-side run loop: */ | 36 | /* Check whether the FP regs were dirtied while in the host-side run loop: */ |
| 33 | static inline bool update_fp_enabled(struct kvm_vcpu *vcpu) | 37 | static inline bool update_fp_enabled(struct kvm_vcpu *vcpu) |
| 34 | { | 38 | { |
| @@ -142,10 +146,10 @@ static inline bool __translate_far_to_hpfar(u64 far, u64 *hpfar) · 107 unmodified lines | |||
| 142 | * saved the guest context yet, and we may return early... | 146 | * saved the guest context yet, and we may return early... |
| 143 | */ | 147 | */ |
| 144 | par = read_sysreg(par_el1); | 148 | par = read_sysreg(par_el1); |
| 145 | asm volatile("at s1e1r, %0" : : "r" (far)); | 149 | if (!__kvm_at("s1e1r", far)) |
| 146 | isb(); | 150 | tmp = read_sysreg(par_el1); |
| 147 | 151 | else | |
| 148 | tmp = read_sysreg(par_el1); | 152 | tmp = SYS_PAR_EL1_F; /* back to the guest */ |
| 149 | write_sysreg(par, par_el1); | 153 | write_sysreg(par, par_el1); |
| 150 | 154 | ||
| 151 | if (unlikely(tmp & SYS_PAR_EL1_F)) | 155 | if (unlikely(tmp & SYS_PAR_EL1_F)) |
| @@ -508,4 +512,31 @@ static inline void __set_host_arch_workaround_state(struct kvm_vcpu *vcpu) · 356 unmodified lines | |||
| 508 | #endif | 512 | #endif |
| 509 | } | 513 | } |
| 510 | 514 | ||
| 515 | static inline void __kvm_unexpected_el2_exception(void) | ||
| 516 | { | ||
| 517 | unsigned long addr, fixup; | ||
| 518 | struct kvm_cpu_context *host_ctxt; | ||
| 519 | struct exception_table_entry *entry, *end; | ||
| 520 | unsigned long elr_el2 = read_sysreg(elr_el2); | ||
| 521 | |||
| 522 | entry = hyp_symbol_addr(__start___kvm_ex_table); | ||
| 523 | end = hyp_symbol_addr(__stop___kvm_ex_table); | ||
| 524 | host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt; | ||
| 525 | |||
| 526 | while (entry < end) { | ||
| 527 | addr = (unsigned long)&entry->insn + entry->insn; | ||
| 528 | fixup = (unsigned long)&entry->fixup + entry->fixup; | ||
| 529 | |||
| 530 | if (addr != elr_el2) { | ||
| 531 | entry++; | ||
| 532 | continue; | ||
| 533 | } | ||
| 534 | |||
| 535 | write_sysreg(fixup, elr_el2); | ||
| 536 | return; | ||
| 537 | } | ||
| 538 | |||
| 539 | hyp_panic(host_ctxt); | ||
| 540 | } | ||
| 541 | |||
| 511 | #endif /* __ARM64_KVM_HYP_SWITCH_H__ */ | 542 | #endif /* __ARM64_KVM_HYP_SWITCH_H__ */ |
| @@ -270,3 +270,8 @@ void __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt) · 269 unmodified lines | |||
| 270 | read_sysreg(hpfar_el2), par, vcpu); | 270 | read_sysreg(hpfar_el2), par, vcpu); |
| 271 | unreachable(); | 271 | unreachable(); |
| 272 | } | 272 | } |
| 273 | |||
| 274 | asmlinkage void kvm_unexpected_el2_exception(void) | ||
| 275 | { | ||
| 276 | return __kvm_unexpected_el2_exception(); | ||
| 277 | } | ||
| @@ -217,3 +217,8 @@ void __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt) · 216 unmodified lines | |||
| 217 | __hyp_call_panic(spsr, elr, par, host_ctxt); | 217 | __hyp_call_panic(spsr, elr, par, host_ctxt); |
| 218 | unreachable(); | 218 | unreachable(); |
| 219 | } | 219 | } |
| 220 | |||
| 221 | asmlinkage void kvm_unexpected_el2_exception(void) | ||
| 222 | { | ||
| 223 | return __kvm_unexpected_el2_exception(); | ||
| 224 | } | ||