* kvm-arm64/nv-nv: : . : Flick the switch on the NV support by adding the missing piece : in the form of the VNCR page management. From the cover letter: : : "This is probably the most interesting bit of the whole NV adventure. : So far, everything else has been a walk in the park, but this one is : where the real fun takes place. : : With FEAT_NV2, most of the NV support revolves around tricking a guest : into accessing memory while it tries to access system registers. The : hypervisor's job is to handle the context switch of the actual : registers with the state in memory as needed." : . KVM: arm64: nv: Release faulted-in VNCR page from mmu_lock critical section KVM: arm64: nv: Handle TLBI S1E2 for VNCR invalidation with mmu_lock held KVM: arm64: nv: Hold mmu_lock when invalidating VNCR SW-TLB before translating KVM: arm64: Document NV caps and vcpu flags KVM: arm64: Allow userspace to request KVM_ARM_VCPU_EL2* KVM: arm64: nv: Remove dead code from ERET handling KVM: arm64: nv: Plumb TLBI S1E2 into system instruction dispatch KVM: arm64: nv: Add S1 TLB invalidation primitive for VNCR_EL2 KVM: arm64: nv: Program host's VNCR_EL2 to the fixmap address KVM: arm64: nv: Handle VNCR_EL2 invalidation from MMU notifiers KVM: arm64: nv: Handle mapping of VNCR_EL2 at EL2 KVM: arm64: nv: Handle VNCR_EL2-triggered faults KVM: arm64: nv: Add userspace and guest handling of VNCR_EL2 KVM: arm64: nv: Add pseudo-TLB backing VNCR_EL2 KVM: arm64: nv: Don't adjust PSTATE.M when L2 is nesting KVM: arm64: nv: Move TLBI range decoding to a helper KVM: arm64: nv: Snapshot S1 ASID tagging information during walk KVM: arm64: nv: Extract translation helper from the AT code KVM: arm64: nv: Allocate VNCR page when required arm64: sysreg: Add layout for VNCR_EL2
Unified · Split
| @@ -3460,7 +3460,8 @@ The initial values are defined as: · 3459 unmodified lines | |||
| 3460 | - FPSIMD/NEON registers: set to 0 | 3460 | - FPSIMD/NEON registers: set to 0 |
| 3461 | - SVE registers: set to 0 | 3461 | - SVE registers: set to 0 |
| 3462 | - System registers: Reset to their architecturally defined | 3462 | - System registers: Reset to their architecturally defined |
| 3463 | values as for a warm reset to EL1 (resp. SVC) | 3463 | values as for a warm reset to EL1 (resp. SVC) or EL2 (in the |
| 3464 | case of EL2 being enabled). | ||
| 3464 | 3465 | ||
| 3465 | Note that because some registers reflect machine topology, all vcpus | 3466 | Note that because some registers reflect machine topology, all vcpus |
| 3466 | should be created before this ioctl is invoked. | 3467 | should be created before this ioctl is invoked. |
| @@ -3527,6 +3528,17 @@ Possible features: · 60 unmodified lines | |||
| 3527 | - the KVM_REG_ARM64_SVE_VLS pseudo-register is immutable, and can | 3528 | - the KVM_REG_ARM64_SVE_VLS pseudo-register is immutable, and can |
| 3528 | no longer be written using KVM_SET_ONE_REG. | 3529 | no longer be written using KVM_SET_ONE_REG. |
| 3529 | 3530 | ||
| 3531 | - KVM_ARM_VCPU_HAS_EL2: Enable Nested Virtualisation support, | ||
| 3532 | booting the guest from EL2 instead of EL1. | ||
| 3533 | Depends on KVM_CAP_ARM_EL2. | ||
| 3534 | The VM is running with HCR_EL2.E2H being RES1 (VHE) unless | ||
| 3535 | KVM_ARM_VCPU_HAS_EL2_E2H0 is also set. | ||
| 3536 | |||
| 3537 | - KVM_ARM_VCPU_HAS_EL2_E2H0: Restrict Nested Virtualisation | ||
| 3538 | support to HCR_EL2.E2H being RES0 (non-VHE). | ||
| 3539 | Depends on KVM_CAP_ARM_EL2_E2H0. | ||
| 3540 | KVM_ARM_VCPU_HAS_EL2 must also be set. | ||
| 3541 | |||
| 3530 | 4.83 KVM_ARM_PREFERRED_TARGET | 3542 | 4.83 KVM_ARM_PREFERRED_TARGET |
| 3531 | ----------------------------- | 3543 | ----------------------------- |
| 3532 | 3544 | ||
| @@ -100,6 +100,8 @@ · 99 unmodified lines | |||
| 100 | #define ESR_ELx_AET_CE (UL(6) << ESR_ELx_AET_SHIFT) | 100 | #define ESR_ELx_AET_CE (UL(6) << ESR_ELx_AET_SHIFT) |
| 101 | 101 | ||
| 102 | /* Shared ISS field definitions for Data/Instruction aborts */ | 102 | /* Shared ISS field definitions for Data/Instruction aborts */ |
| 103 | #define ESR_ELx_VNCR_SHIFT (13) | ||
| 104 | #define ESR_ELx_VNCR (UL(1) << ESR_ELx_VNCR_SHIFT) | ||
| 103 | #define ESR_ELx_SET_SHIFT (11) | 105 | #define ESR_ELx_SET_SHIFT (11) |
| 104 | #define ESR_ELx_SET_MASK (UL(3) << ESR_ELx_SET_SHIFT) | 106 | #define ESR_ELx_SET_MASK (UL(3) << ESR_ELx_SET_SHIFT) |
| 105 | #define ESR_ELx_FnV_SHIFT (10) | 107 | #define ESR_ELx_FnV_SHIFT (10) |
| @@ -48,6 +48,12 @@ enum fixed_addresses { · 47 unmodified lines | |||
| 48 | FIX_EARLYCON_MEM_BASE, | 48 | FIX_EARLYCON_MEM_BASE, |
| 49 | FIX_TEXT_POKE0, | 49 | FIX_TEXT_POKE0, |
| 50 | 50 | ||
| 51 | #ifdef CONFIG_KVM | ||
| 52 | /* One slot per CPU, mapping the guest's VNCR page at EL2. */ | ||
| 53 | FIX_VNCR_END, | ||
| 54 | FIX_VNCR = FIX_VNCR_END + NR_CPUS, | ||
| 55 | #endif | ||
| 56 | |||
| 51 | #ifdef CONFIG_ACPI_APEI_GHES | 57 | #ifdef CONFIG_ACPI_APEI_GHES |
| 52 | /* Used for GHES mapping from assorted contexts */ | 58 | /* Used for GHES mapping from assorted contexts */ |
| 53 | FIX_APEI_GHES_IRQ, | 59 | FIX_APEI_GHES_IRQ, |
| @@ -39,7 +39,7 @@ · 38 unmodified lines | |||
| 39 | 39 | ||
| 40 | #define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS | 40 | #define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS |
| 41 | 41 | ||
| 42 | #define KVM_VCPU_MAX_FEATURES 7 | 42 | #define KVM_VCPU_MAX_FEATURES 9 |
| 43 | #define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1) | 43 | #define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1) |
| 44 | 44 | ||
| 45 | #define KVM_REQ_SLEEP \ | 45 | #define KVM_REQ_SLEEP \ |
| @@ -53,6 +53,7 @@ · 7 unmodified lines | |||
| 53 | #define KVM_REQ_RESYNC_PMU_EL0 KVM_ARCH_REQ(7) | 53 | #define KVM_REQ_RESYNC_PMU_EL0 KVM_ARCH_REQ(7) |
| 54 | #define KVM_REQ_NESTED_S2_UNMAP KVM_ARCH_REQ(8) | 54 | #define KVM_REQ_NESTED_S2_UNMAP KVM_ARCH_REQ(8) |
| 55 | #define KVM_REQ_GUEST_HYP_IRQ_PENDING KVM_ARCH_REQ(9) | 55 | #define KVM_REQ_GUEST_HYP_IRQ_PENDING KVM_ARCH_REQ(9) |
| 56 | #define KVM_REQ_MAP_L1_VNCR_EL2 KVM_ARCH_REQ(10) | ||
| 56 | 57 | ||
| 57 | #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ | 58 | #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ |
| 58 | KVM_DIRTY_LOG_INITIALLY_SET) | 59 | KVM_DIRTY_LOG_INITIALLY_SET) |
| @@ -395,6 +396,9 @@ struct kvm_arch { · 336 unmodified lines | |||
| 395 | /* Masks for VNCR-backed and general EL2 sysregs */ | 396 | /* Masks for VNCR-backed and general EL2 sysregs */ |
| 396 | struct kvm_sysreg_masks *sysreg_masks; | 397 | struct kvm_sysreg_masks *sysreg_masks; |
| 397 | 398 | ||
| 399 | /* Count the number of VNCR_EL2 currently mapped */ | ||
| 400 | atomic_t vncr_map_count; | ||
| 401 | |||
| 398 | /* | 402 | /* |
| 399 | * For an untrusted host VM, 'pkvm.handle' is used to lookup | 403 | * For an untrusted host VM, 'pkvm.handle' is used to lookup |
| 400 | * the associated pKVM instance in the hypervisor. | 404 | * the associated pKVM instance in the hypervisor. |
| @@ -573,6 +577,8 @@ enum vcpu_sysreg { · 172 unmodified lines | |||
| 573 | VNCR(HDFGRTR2_EL2), | 577 | VNCR(HDFGRTR2_EL2), |
| 574 | VNCR(HDFGWTR2_EL2), | 578 | VNCR(HDFGWTR2_EL2), |
| 575 | 579 | ||
| 580 | VNCR(VNCR_EL2), | ||
| 581 | |||
| 576 | VNCR(CNTVOFF_EL2), | 582 | VNCR(CNTVOFF_EL2), |
| 577 | VNCR(CNTV_CVAL_EL0), | 583 | VNCR(CNTV_CVAL_EL0), |
| 578 | VNCR(CNTV_CTL_EL0), | 584 | VNCR(CNTV_CTL_EL0), |
| @@ -696,6 +702,8 @@ struct kvm_host_data { · 117 unmodified lines | |||
| 696 | #define KVM_HOST_DATA_FLAG_HAS_TRBE 1 | 702 | #define KVM_HOST_DATA_FLAG_HAS_TRBE 1 |
| 697 | #define KVM_HOST_DATA_FLAG_TRBE_ENABLED 4 | 703 | #define KVM_HOST_DATA_FLAG_TRBE_ENABLED 4 |
| 698 | #define KVM_HOST_DATA_FLAG_EL1_TRACING_CONFIGURED 5 | 704 | #define KVM_HOST_DATA_FLAG_EL1_TRACING_CONFIGURED 5 |
| 705 | #define KVM_HOST_DATA_FLAG_VCPU_IN_HYP_CONTEXT 6 | ||
| 706 | #define KVM_HOST_DATA_FLAG_L1_VNCR_MAPPED 7 | ||
| 699 | unsigned long flags; | 707 | unsigned long flags; |
| 700 | 708 | ||
| 701 | struct kvm_cpu_context host_ctxt; | 709 | struct kvm_cpu_context host_ctxt; |
| @@ -772,6 +780,8 @@ struct vcpu_reset_state { · 70 unmodified lines | |||
| 772 | bool reset; | 780 | bool reset; |
| 773 | }; | 781 | }; |
| 774 | 782 | ||
| 783 | struct vncr_tlb; | ||
| 784 | |||
| 775 | struct kvm_vcpu_arch { | 785 | struct kvm_vcpu_arch { |
| 776 | struct kvm_cpu_context ctxt; | 786 | struct kvm_cpu_context ctxt; |
| 777 | 787 | ||
| @@ -866,6 +876,9 @@ struct kvm_vcpu_arch { · 88 unmodified lines | |||
| 866 | 876 | ||
| 867 | /* Per-vcpu CCSIDR override or NULL */ | 877 | /* Per-vcpu CCSIDR override or NULL */ |
| 868 | u32 *ccsidr; | 878 | u32 *ccsidr; |
| 879 | |||
| 880 | /* Per-vcpu TLB for VNCR_EL2 -- NULL when !NV */ | ||
| 881 | struct vncr_tlb *vncr_tlb; | ||
| 869 | }; | 882 | }; |
| 870 | 883 | ||
| 871 | /* | 884 | /* |
| @@ -231,6 +231,38 @@ static inline u64 kvm_encode_nested_level(struct kvm_s2_trans *trans) · 230 unmodified lines | |||
| 231 | shift; \ | 231 | shift; \ |
| 232 | }) | 232 | }) |
| 233 | 233 | ||
| 234 | static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid) | ||
| 235 | { | ||
| 236 | u64 base, tg, num, scale; | ||
| 237 | int shift; | ||
| 238 | |||
| 239 | tg = FIELD_GET(GENMASK(47, 46), val); | ||
| 240 | |||
| 241 | switch(tg) { | ||
| 242 | case 1: | ||
| 243 | shift = 12; | ||
| 244 | break; | ||
| 245 | case 2: | ||
| 246 | shift = 14; | ||
| 247 | break; | ||
| 248 | case 3: | ||
| 249 | default: /* IMPDEF: handle tg==0 as 64k */ | ||
| 250 | shift = 16; | ||
| 251 | break; | ||
| 252 | } | ||
| 253 | |||
| 254 | base = (val & GENMASK(36, 0)) << shift; | ||
| 255 | |||
| 256 | if (asid) | ||
| 257 | *asid = FIELD_GET(TLBIR_ASID_MASK, val); | ||
| 258 | |||
| 259 | scale = FIELD_GET(GENMASK(45, 44), val); | ||
| 260 | num = FIELD_GET(GENMASK(43, 39), val); | ||
| 261 | *range = __TLBI_RANGE_PAGES(num, scale) << shift; | ||
| 262 | |||
| 263 | return base; | ||
| 264 | } | ||
| 265 | |||
| 234 | static inline unsigned int ps_to_output_size(unsigned int ps) | 266 | static inline unsigned int ps_to_output_size(unsigned int ps) |
| 235 | { | 267 | { |
| 236 | switch (ps) { | 268 | switch (ps) { |
| @@ -245,4 +277,72 @@ static inline unsigned int ps_to_output_size(unsigned int ps) · 8 unmodified lines | |||
| 245 | } | 277 | } |
| 246 | } | 278 | } |
| 247 | 279 | ||
| 280 | enum trans_regime { | ||
| 281 | TR_EL10, | ||
| 282 | TR_EL20, | ||
| 283 | TR_EL2, | ||
| 284 | }; | ||
| 285 | |||
| 286 | struct s1_walk_info { | ||
| 287 | u64 baddr; | ||
| 288 | enum trans_regime regime; | ||
| 289 | unsigned int max_oa_bits; | ||
| 290 | unsigned int pgshift; | ||
| 291 | unsigned int txsz; | ||
| 292 | int sl; | ||
| 293 | bool as_el0; | ||
| 294 | bool hpd; | ||
| 295 | bool e0poe; | ||
| 296 | bool poe; | ||
| 297 | bool pan; | ||
| 298 | bool be; | ||
| 299 | bool s2; | ||
| 300 | }; | ||
| 301 | |||
| 302 | struct s1_walk_result { | ||
| 303 | union { | ||
| 304 | struct { | ||
| 305 | u64 desc; | ||
| 306 | u64 pa; | ||
| 307 | s8 level; | ||
| 308 | u8 APTable; | ||
| 309 | bool nG; | ||
| 310 | u16 asid; | ||
| 311 | bool UXNTable; | ||
| 312 | bool PXNTable; | ||
| 313 | bool uwxn; | ||
| 314 | bool uov; | ||
| 315 | bool ur; | ||
| 316 | bool uw; | ||
| 317 | bool ux; | ||
| 318 | bool pwxn; | ||
| 319 | bool pov; | ||
| 320 | bool pr; | ||
| 321 | bool pw; | ||
| 322 | bool px; | ||
| 323 | }; | ||
| 324 | struct { | ||
| 325 | u8 fst; | ||
| 326 | bool ptw; | ||
| 327 | bool s2; | ||
| 328 | }; | ||
| 329 | }; | ||
| 330 | bool failed; | ||
| 331 | }; | ||
| 332 | |||
| 333 | int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, | ||
| 334 | struct s1_walk_result *wr, u64 va); | ||
| 335 | |||
| 336 | /* VNCR management */ | ||
| 337 | int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu); | ||
| 338 | int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu); | ||
| 339 | void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val); | ||
| 340 | |||
| 341 | #define vncr_fixmap(c) \ | ||
| 342 | ({ \ | ||
| 343 | u32 __c = (c); \ | ||
| 344 | BUG_ON(__c >= NR_CPUS); \ | ||
| 345 | (FIX_VNCR - __c); \ | ||
| 346 | }) | ||
| 347 | |||
| 248 | #endif /* __ARM64_KVM_NESTED_H */ | 348 | #endif /* __ARM64_KVM_NESTED_H */ |
| @@ -534,7 +534,7 @@ · 533 unmodified lines | |||
| 534 | #define SYS_VTTBR_EL2 sys_reg(3, 4, 2, 1, 0) | 534 | #define SYS_VTTBR_EL2 sys_reg(3, 4, 2, 1, 0) |
| 535 | #define SYS_VTCR_EL2 sys_reg(3, 4, 2, 1, 2) | 535 | #define SYS_VTCR_EL2 sys_reg(3, 4, 2, 1, 2) |
| 536 | 536 | ||
| 537 | #define SYS_VNCR_EL2 sys_reg(3, 4, 2, 2, 0) | 537 | #define SYS_HAFGRTR_EL2 sys_reg(3, 4, 3, 1, 6) |
| 538 | #define SYS_SPSR_EL2 sys_reg(3, 4, 4, 0, 0) | 538 | #define SYS_SPSR_EL2 sys_reg(3, 4, 4, 0, 0) |
| 539 | #define SYS_ELR_EL2 sys_reg(3, 4, 4, 0, 1) | 539 | #define SYS_ELR_EL2 sys_reg(3, 4, 4, 0, 1) |
| 540 | #define SYS_SP_EL1 sys_reg(3, 4, 4, 1, 0) | 540 | #define SYS_SP_EL1 sys_reg(3, 4, 4, 1, 0) |
| @@ -368,6 +368,12 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 367 unmodified lines | |||
| 368 | case KVM_CAP_ARM_EL1_32BIT: | 368 | case KVM_CAP_ARM_EL1_32BIT: |
| 369 | r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1); | 369 | r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1); |
| 370 | break; | 370 | break; |
| 371 | case KVM_CAP_ARM_EL2: | ||
| 372 | r = cpus_have_final_cap(ARM64_HAS_NESTED_VIRT); | ||
| 373 | break; | ||
| 374 | case KVM_CAP_ARM_EL2_E2H0: | ||
| 375 | r = cpus_have_final_cap(ARM64_HAS_HCR_NV1); | ||
| 376 | break; | ||
| 371 | case KVM_CAP_GUEST_DEBUG_HW_BPS: | 377 | case KVM_CAP_GUEST_DEBUG_HW_BPS: |
| 372 | r = get_num_brps(); | 378 | r = get_num_brps(); |
| 373 | break; | 379 | break; |
| @@ -843,6 +849,10 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) · 469 unmodified lines | |||
| 843 | return ret; | 849 | return ret; |
| 844 | 850 | ||
| 845 | if (vcpu_has_nv(vcpu)) { | 851 | if (vcpu_has_nv(vcpu)) { |
| 852 | ret = kvm_vcpu_allocate_vncr_tlb(vcpu); | ||
| 853 | if (ret) | ||
| 854 | return ret; | ||
| 855 | |||
| 846 | ret = kvm_vgic_vcpu_nv_init(vcpu); | 856 | ret = kvm_vgic_vcpu_nv_init(vcpu); |
| 847 | if (ret) | 857 | if (ret) |
| 848 | return ret; | 858 | return ret; |
| @@ -10,56 +10,6 @@ · 9 unmodified lines | |||
| 10 | #include <asm/kvm_hyp.h> | 10 | #include <asm/kvm_hyp.h> |
| 11 | #include <asm/kvm_mmu.h> | 11 | #include <asm/kvm_mmu.h> |
| 12 | 12 | ||
| 13 | enum trans_regime { | ||
| 14 | TR_EL10, | ||
| 15 | TR_EL20, | ||
| 16 | TR_EL2, | ||
| 17 | }; | ||
| 18 | |||
| 19 | struct s1_walk_info { | ||
| 20 | u64 baddr; | ||
| 21 | enum trans_regime regime; | ||
| 22 | unsigned int max_oa_bits; | ||
| 23 | unsigned int pgshift; | ||
| 24 | unsigned int txsz; | ||
| 25 | int sl; | ||
| 26 | bool hpd; | ||
| 27 | bool e0poe; | ||
| 28 | bool poe; | ||
| 29 | bool pan; | ||
| 30 | bool be; | ||
| 31 | bool s2; | ||
| 32 | }; | ||
| 33 | |||
| 34 | struct s1_walk_result { | ||
| 35 | union { | ||
| 36 | struct { | ||
| 37 | u64 desc; | ||
| 38 | u64 pa; | ||
| 39 | s8 level; | ||
| 40 | u8 APTable; | ||
| 41 | bool UXNTable; | ||
| 42 | bool PXNTable; | ||
| 43 | bool uwxn; | ||
| 44 | bool uov; | ||
| 45 | bool ur; | ||
| 46 | bool uw; | ||
| 47 | bool ux; | ||
| 48 | bool pwxn; | ||
| 49 | bool pov; | ||
| 50 | bool pr; | ||
| 51 | bool pw; | ||
| 52 | bool px; | ||
| 53 | }; | ||
| 54 | struct { | ||
| 55 | u8 fst; | ||
| 56 | bool ptw; | ||
| 57 | bool s2; | ||
| 58 | }; | ||
| 59 | }; | ||
| 60 | bool failed; | ||
| 61 | }; | ||
| 62 | |||
| 63 | static void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) | 13 | static void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) |
| 64 | { | 14 | { |
| 65 | wr->fst = fst; | 15 | wr->fst = fst; |
| @@ -145,20 +95,15 @@ static void compute_s1poe(struct kvm_vcpu *vcpu, struct s1_walk_info *wi) · 79 unmodified lines | |||
| 145 | } | 95 | } |
| 146 | } | 96 | } |
| 147 | 97 | ||
| 148 | static int setup_s1_walk(struct kvm_vcpu *vcpu, u32 op, struct s1_walk_info *wi, | 98 | static int setup_s1_walk(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, |
| 149 | struct s1_walk_result *wr, u64 va) | 99 | struct s1_walk_result *wr, u64 va) |
| 150 | { | 100 | { |
| 151 | u64 hcr, sctlr, tcr, tg, ps, ia_bits, ttbr; | 101 | u64 hcr, sctlr, tcr, tg, ps, ia_bits, ttbr; |
| 152 | unsigned int stride, x; | 102 | unsigned int stride, x; |
| 153 | bool va55, tbi, lva, as_el0; | 103 | bool va55, tbi, lva; |
| 154 | 104 | ||
| 155 | hcr = __vcpu_sys_reg(vcpu, HCR_EL2); | 105 | hcr = __vcpu_sys_reg(vcpu, HCR_EL2); |
| 156 | 106 | ||
| 157 | wi->regime = compute_translation_regime(vcpu, op); | ||
| 158 | as_el0 = (op == OP_AT_S1E0R || op == OP_AT_S1E0W); | ||
| 159 | wi->pan = (op == OP_AT_S1E1RP || op == OP_AT_S1E1WP) && | ||
| 160 | (*vcpu_cpsr(vcpu) & PSR_PAN_BIT); | ||
| 161 | |||
| 162 | va55 = va & BIT(55); | 107 | va55 = va & BIT(55); |
| 163 | 108 | ||
| 164 | if (wi->regime == TR_EL2 && va55) | 109 | if (wi->regime == TR_EL2 && va55) |
| @@ -319,7 +264,7 @@ static int setup_s1_walk(struct kvm_vcpu *vcpu, u32 op, struct s1_walk_info *wi, · 154 unmodified lines | |||
| 319 | 264 | ||
| 320 | /* R_BNDVG and following statements */ | 265 | /* R_BNDVG and following statements */ |
| 321 | if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, E0PD, IMP) && | 266 | if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, E0PD, IMP) && |
| 322 | as_el0 && (tcr & (va55 ? TCR_E0PD1 : TCR_E0PD0))) | 267 | wi->as_el0 && (tcr & (va55 ? TCR_E0PD1 : TCR_E0PD0))) |
| 323 | goto transfault_l0; | 268 | goto transfault_l0; |
| 324 | 269 | ||
| 325 | /* AArch64.S1StartLevel() */ | 270 | /* AArch64.S1StartLevel() */ |
| @@ -469,6 +414,33 @@ static int walk_s1(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, · 143 unmodified lines | |||
| 469 | wr->pa = desc & GENMASK(47, va_bottom); | 414 | wr->pa = desc & GENMASK(47, va_bottom); |
| 470 | wr->pa |= va & GENMASK_ULL(va_bottom - 1, 0); | 415 | wr->pa |= va & GENMASK_ULL(va_bottom - 1, 0); |
| 471 | 416 | ||
| 417 | wr->nG = (wi->regime != TR_EL2) && (desc & PTE_NG); | ||
| 418 | if (wr->nG) { | ||
| 419 | u64 asid_ttbr, tcr; | ||
| 420 | |||
| 421 | switch (wi->regime) { | ||
| 422 | case TR_EL10: | ||
| 423 | tcr = vcpu_read_sys_reg(vcpu, TCR_EL1); | ||
| 424 | asid_ttbr = ((tcr & TCR_A1) ? | ||
| 425 | vcpu_read_sys_reg(vcpu, TTBR1_EL1) : | ||
| 426 | vcpu_read_sys_reg(vcpu, TTBR0_EL1)); | ||
| 427 | break; | ||
| 428 | case TR_EL20: | ||
| 429 | tcr = vcpu_read_sys_reg(vcpu, TCR_EL2); | ||
| 430 | asid_ttbr = ((tcr & TCR_A1) ? | ||
| 431 | vcpu_read_sys_reg(vcpu, TTBR1_EL2) : | ||
| 432 | vcpu_read_sys_reg(vcpu, TTBR0_EL2)); | ||
| 433 | break; | ||
| 434 | default: | ||
| 435 | BUG(); | ||
| 436 | } | ||
| 437 | |||
| 438 | wr->asid = FIELD_GET(TTBR_ASID_MASK, asid_ttbr); | ||
| 439 | if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) || | ||
| 440 | !(tcr & TCR_ASID16)) | ||
| 441 | wr->asid &= GENMASK(7, 0); | ||
| 442 | } | ||
| 443 | |||
| 472 | return 0; | 444 | return 0; |
| 473 | 445 | ||
| 474 | addrsz: | 446 | addrsz: |
| @@ -1155,7 +1127,12 @@ static u64 handle_at_slow(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) · 680 unmodified lines | |||
| 1155 | bool perm_fail = false; | 1127 | bool perm_fail = false; |
| 1156 | int ret, idx; | 1128 | int ret, idx; |
| 1157 | 1129 | ||
| 1158 | ret = setup_s1_walk(vcpu, op, &wi, &wr, vaddr); | 1130 | wi.regime = compute_translation_regime(vcpu, op); |
| 1131 | wi.as_el0 = (op == OP_AT_S1E0R || op == OP_AT_S1E0W); | ||
| 1132 | wi.pan = (op == OP_AT_S1E1RP || op == OP_AT_S1E1WP) && | ||
| 1133 | (*vcpu_cpsr(vcpu) & PSR_PAN_BIT); | ||
| 1134 | |||
| 1135 | ret = setup_s1_walk(vcpu, &wi, &wr, vaddr); | ||
| 1159 | if (ret) | 1136 | if (ret) |
| 1160 | goto compute_par; | 1137 | goto compute_par; |
| 1161 | 1138 | ||
| @@ -1457,3 +1434,31 @@ void __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) · 295 unmodified lines | |||
| 1457 | par = compute_par_s12(vcpu, par, &out); | 1434 | par = compute_par_s12(vcpu, par, &out); |
| 1458 | vcpu_write_sys_reg(vcpu, par, PAR_EL1); | 1435 | vcpu_write_sys_reg(vcpu, par, PAR_EL1); |
| 1459 | } | 1436 | } |
| 1437 | |||
| 1438 | /* | ||
| 1439 | * Translate a VA for a given EL in a given translation regime, with | ||
| 1440 | * or without PAN. This requires wi->{regime, as_el0, pan} to be | ||
| 1441 | * set. The rest of the wi and wr should be 0-initialised. | ||
| 1442 | */ | ||
| 1443 | int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, | ||
| 1444 | struct s1_walk_result *wr, u64 va) | ||
| 1445 | { | ||
| 1446 | int ret; | ||
| 1447 | |||
| 1448 | ret = setup_s1_walk(vcpu, wi, wr, va); | ||
| 1449 | if (ret) | ||
| 1450 | return ret; | ||
| 1451 | |||
| 1452 | if (wr->level == S1_MMU_DISABLED) { | ||
| 1453 | wr->ur = wr->uw = wr->ux = true; | ||
| 1454 | wr->pr = wr->pw = wr->px = true; | ||
| 1455 | } else { | ||
| 1456 | ret = walk_s1(vcpu, wi, wr, va); | ||
| 1457 | if (ret) | ||
| 1458 | return ret; | ||
| 1459 | |||
| 1460 | compute_s1_permissions(vcpu, wi, wr); | ||
| 1461 | } | ||
| 1462 | |||
| 1463 | return 0; | ||
| 1464 | } | ||
| @@ -2662,13 +2662,6 @@ void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu) · 2661 unmodified lines | |||
| 2662 | { | 2662 | { |
| 2663 | u64 spsr, elr, esr; | 2663 | u64 spsr, elr, esr; |
| 2664 | 2664 | ||
| 2665 | /* | ||
| 2666 | * Forward this trap to the virtual EL2 if the virtual | ||
| 2667 | * HCR_EL2.NV bit is set and this is coming from !EL2. | ||
| 2668 | */ | ||
| 2669 | if (forward_hcr_traps(vcpu, HCR_NV)) | ||
| 2670 | return; | ||
| 2671 | |||
| 2672 | spsr = vcpu_read_sys_reg(vcpu, SPSR_EL2); | 2665 | spsr = vcpu_read_sys_reg(vcpu, SPSR_EL2); |
| 2673 | spsr = kvm_check_illegal_exception_return(vcpu, spsr); | 2666 | spsr = kvm_check_illegal_exception_return(vcpu, spsr); |
| 2674 | 2667 | ||
| @@ -394,6 +394,7 @@ static exit_handle_fn arm_exit_handlers[] = { · 393 unmodified lines | |||
| 394 | [ESR_ELx_EC_ERET] = kvm_handle_eret, | 394 | [ESR_ELx_EC_ERET] = kvm_handle_eret, |
| 395 | [ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort, | 395 | [ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort, |
| 396 | [ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort, | 396 | [ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort, |
| 397 | [ESR_ELx_EC_DABT_CUR] = kvm_handle_vncr_abort, | ||
| 397 | [ESR_ELx_EC_SOFTSTP_LOW]= kvm_handle_guest_debug, | 398 | [ESR_ELx_EC_SOFTSTP_LOW]= kvm_handle_guest_debug, |
| 398 | [ESR_ELx_EC_WATCHPT_LOW]= kvm_handle_guest_debug, | 399 | [ESR_ELx_EC_WATCHPT_LOW]= kvm_handle_guest_debug, |
| 399 | [ESR_ELx_EC_BREAKPT_LOW]= kvm_handle_guest_debug, | 400 | [ESR_ELx_EC_BREAKPT_LOW]= kvm_handle_guest_debug, |
| @@ -48,21 +48,46 @@ DEFINE_PER_CPU(unsigned long, kvm_hyp_vector); · 47 unmodified lines | |||
| 48 | 48 | ||
| 49 | static u64 __compute_hcr(struct kvm_vcpu *vcpu) | 49 | static u64 __compute_hcr(struct kvm_vcpu *vcpu) |
| 50 | { | 50 | { |
| 51 | u64 guest_hcr = __vcpu_sys_reg(vcpu, HCR_EL2); | ||
| 51 | u64 hcr = vcpu->arch.hcr_el2; | 52 | u64 hcr = vcpu->arch.hcr_el2; |
| 52 | 53 | ||
| 53 | if (!vcpu_has_nv(vcpu)) | 54 | if (!vcpu_has_nv(vcpu)) |
| 54 | return hcr; | 55 | return hcr; |
| 55 | 56 | ||
| 57 | /* | ||
| 58 | * We rely on the invariant that a vcpu entered from HYP | ||
| 59 | * context must also exit in the same context, as only an ERET | ||
| 60 | * instruction can kick us out of it, and we obviously trap | ||
| 61 | * that sucker. PSTATE.M will get fixed-up on exit. | ||
| 62 | */ | ||
| 56 | if (is_hyp_ctxt(vcpu)) { | 63 | if (is_hyp_ctxt(vcpu)) { |
| 64 | host_data_set_flag(VCPU_IN_HYP_CONTEXT); | ||
| 65 | |||
| 57 | hcr |= HCR_NV | HCR_NV2 | HCR_AT | HCR_TTLB; | 66 | hcr |= HCR_NV | HCR_NV2 | HCR_AT | HCR_TTLB; |
| 58 | 67 | ||
| 59 | if (!vcpu_el2_e2h_is_set(vcpu)) | 68 | if (!vcpu_el2_e2h_is_set(vcpu)) |
| 60 | hcr |= HCR_NV1; | 69 | hcr |= HCR_NV1; |
| 61 | 70 | ||
| 62 | write_sysreg_s(vcpu->arch.ctxt.vncr_array, SYS_VNCR_EL2); | 71 | write_sysreg_s(vcpu->arch.ctxt.vncr_array, SYS_VNCR_EL2); |
| 72 | } else { | ||
| 73 | host_data_clear_flag(VCPU_IN_HYP_CONTEXT); | ||
| 74 | |||
| 75 | if (guest_hcr & HCR_NV) { | ||
| 76 | u64 va = __fix_to_virt(vncr_fixmap(smp_processor_id())); | ||
| 77 | |||
| 78 | /* Inherit the low bits from the actual register */ | ||
| 79 | va |= __vcpu_sys_reg(vcpu, VNCR_EL2) & GENMASK(PAGE_SHIFT - 1, 0); | ||
| 80 | write_sysreg_s(va, SYS_VNCR_EL2); | ||
| 81 | |||
| 82 | /* Force NV2 in case the guest is forgetful... */ | ||
| 83 | guest_hcr |= HCR_NV2; | ||
| 84 | } | ||
| 63 | } | 85 | } |
| 64 | 86 | ||
| 65 | return hcr | (__vcpu_sys_reg(vcpu, HCR_EL2) & ~NV_HCR_GUEST_EXCLUDE); | 87 | BUG_ON(host_data_test_flag(VCPU_IN_HYP_CONTEXT) && |
| 88 | host_data_test_flag(L1_VNCR_MAPPED)); | ||
| 89 | |||
| 90 | return hcr | (guest_hcr & ~NV_HCR_GUEST_EXCLUDE); | ||
| 66 | } | 91 | } |
| 67 | 92 | ||
| 68 | static void __activate_cptr_traps(struct kvm_vcpu *vcpu) | 93 | static void __activate_cptr_traps(struct kvm_vcpu *vcpu) |
| @@ -459,6 +484,14 @@ static bool kvm_hyp_handle_tlbi_el2(struct kvm_vcpu *vcpu, u64 *exit_code) · 390 unmodified lines | |||
| 459 | if (ret) | 484 | if (ret) |
| 460 | return false; | 485 | return false; |
| 461 | 486 | ||
| 487 | /* | ||
| 488 | * If we have to check for any VNCR mapping being invalidated, | ||
| 489 | * go back to the slow path for further processing. | ||
| 490 | */ | ||
| 491 | if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu) && | ||
| 492 | atomic_read(&vcpu->kvm->arch.vncr_map_count)) | ||
| 493 | return false; | ||
| 494 | |||
| 462 | __kvm_skip_instr(vcpu); | 495 | __kvm_skip_instr(vcpu); |
| 463 | 496 | ||
| 464 | return true; | 497 | return true; |
| @@ -568,9 +601,12 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) · 103 unmodified lines | |||
| 568 | 601 | ||
| 569 | /* | 602 | /* |
| 570 | * If we were in HYP context on entry, adjust the PSTATE view | 603 | * If we were in HYP context on entry, adjust the PSTATE view |
| 571 | * so that the usual helpers work correctly. | 604 | * so that the usual helpers work correctly. This enforces our |
| 605 | * invariant that the guest's HYP context status is preserved | ||
| 606 | * across a run. | ||
| 572 | */ | 607 | */ |
| 573 | if (vcpu_has_nv(vcpu) && (read_sysreg(hcr_el2) & HCR_NV)) { | 608 | if (vcpu_has_nv(vcpu) && |
| 609 | unlikely(host_data_test_flag(VCPU_IN_HYP_CONTEXT))) { | ||
| 574 | u64 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT); | 610 | u64 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT); |
| 575 | 611 | ||
| 576 | switch (mode) { | 612 | switch (mode) { |
| @@ -586,6 +622,10 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) · 9 unmodified lines | |||
| 586 | *vcpu_cpsr(vcpu) |= mode; | 622 | *vcpu_cpsr(vcpu) |= mode; |
| 587 | } | 623 | } |
| 588 | 624 | ||
| 625 | /* Apply extreme paranoia! */ | ||
| 626 | BUG_ON(vcpu_has_nv(vcpu) && | ||
| 627 | !!host_data_test_flag(VCPU_IN_HYP_CONTEXT) != is_hyp_ctxt(vcpu)); | ||
| 628 | |||
| 589 | return __fixup_guest_exit(vcpu, exit_code, hyp_exit_handlers); | 629 | return __fixup_guest_exit(vcpu, exit_code, hyp_exit_handlers); |
| 590 | } | 630 | } |
| 591 | 631 | ||
| @@ -8,6 +8,7 @@ · 7 unmodified lines | |||
| 8 | #include <linux/kvm.h> | 8 | #include <linux/kvm.h> |
| 9 | #include <linux/kvm_host.h> | 9 | #include <linux/kvm_host.h> |
| 10 | 10 | ||
| 11 | #include <asm/fixmap.h> | ||
| 11 | #include <asm/kvm_arm.h> | 12 | #include <asm/kvm_arm.h> |
| 12 | #include <asm/kvm_emulate.h> | 13 | #include <asm/kvm_emulate.h> |
| 13 | #include <asm/kvm_mmu.h> | 14 | #include <asm/kvm_mmu.h> |
| @@ -16,6 +17,24 @@ · 2 unmodified lines | |||
| 16 | 17 | ||
| 17 | #include "sys_regs.h" | 18 | #include "sys_regs.h" |
| 18 | 19 | ||
| 20 | struct vncr_tlb { | ||
| 21 | /* The guest's VNCR_EL2 */ | ||
| 22 | u64 gva; | ||
| 23 | struct s1_walk_info wi; | ||
| 24 | struct s1_walk_result wr; | ||
| 25 | |||
| 26 | u64 hpa; | ||
| 27 | |||
| 28 | /* -1 when not mapped on a CPU */ | ||
| 29 | int cpu; | ||
| 30 | |||
| 31 | /* | ||
| 32 | * true if the TLB is valid. Can only be changed with the | ||
| 33 | * mmu_lock held. | ||
| 34 | */ | ||
| 35 | bool valid; | ||
| 36 | }; | ||
| 37 | |||
| 19 | /* | 38 | /* |
| 20 | * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between | 39 | * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between |
| 21 | * memory usage and potential number of different sets of S2 PTs in | 40 | * memory usage and potential number of different sets of S2 PTs in |
| @@ -28,6 +47,7 @@ void kvm_init_nested(struct kvm *kvm) · 6 unmodified lines | |||
| 28 | { | 47 | { |
| 29 | kvm->arch.nested_mmus = NULL; | 48 | kvm->arch.nested_mmus = NULL; |
| 30 | kvm->arch.nested_mmus_size = 0; | 49 | kvm->arch.nested_mmus_size = 0; |
| 50 | atomic_set(&kvm->arch.vncr_map_count, 0); | ||
| 31 | } | 51 | } |
| 32 | 52 | ||
| 33 | static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) | 53 | static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) |
| @@ -55,6 +75,13 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) · 21 unmodified lines | |||
| 55 | !cpus_have_final_cap(ARM64_HAS_HCR_NV1)) | 75 | !cpus_have_final_cap(ARM64_HAS_HCR_NV1)) |
| 56 | return -EINVAL; | 76 | return -EINVAL; |
| 57 | 77 | ||
| 78 | if (!vcpu->arch.ctxt.vncr_array) | ||
| 79 | vcpu->arch.ctxt.vncr_array = (u64 *)__get_free_page(GFP_KERNEL_ACCOUNT | | ||
| 80 | __GFP_ZERO); | ||
| 81 | |||
| 82 | if (!vcpu->arch.ctxt.vncr_array) | ||
| 83 | return -ENOMEM; | ||
| 84 | |||
| 58 | /* | 85 | /* |
| 59 | * Let's treat memory allocation failures as benign: If we fail to | 86 | * Let's treat memory allocation failures as benign: If we fail to |
| 60 | * allocate anything, return an error and keep the allocated array | 87 | * allocate anything, return an error and keep the allocated array |
| @@ -85,6 +112,9 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) · 24 unmodified lines | |||
| 85 | for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++) | 112 | for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++) |
| 86 | kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]); | 113 | kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]); |
| 87 | 114 | ||
| 115 | free_page((unsigned long)vcpu->arch.ctxt.vncr_array); | ||
| 116 | vcpu->arch.ctxt.vncr_array = NULL; | ||
| 117 | |||
| 88 | return ret; | 118 | return ret; |
| 89 | } | 119 | } |
| 90 | 120 | ||
| @@ -405,6 +435,30 @@ static unsigned int ttl_to_size(u8 ttl) · 314 unmodified lines | |||
| 405 | return max_size; | 435 | return max_size; |
| 406 | } | 436 | } |
| 407 | 437 | ||
| 438 | static u8 pgshift_level_to_ttl(u16 shift, u8 level) | ||
| 439 | { | ||
| 440 | u8 ttl; | ||
| 441 | |||
| 442 | switch(shift) { | ||
| 443 | case 12: | ||
| 444 | ttl = TLBI_TTL_TG_4K; | ||
| 445 | break; | ||
| 446 | case 14: | ||
| 447 | ttl = TLBI_TTL_TG_16K; | ||
| 448 | break; | ||
| 449 | case 16: | ||
| 450 | ttl = TLBI_TTL_TG_64K; | ||
| 451 | break; | ||
| 452 | default: | ||
| 453 | BUG(); | ||
| 454 | } | ||
| 455 | |||
| 456 | ttl <<= 2; | ||
| 457 | ttl |= level & 3; | ||
| 458 | |||
| 459 | return ttl; | ||
| 460 | } | ||
| 461 | |||
| 408 | /* | 462 | /* |
| 409 | * Compute the equivalent of the TTL field by parsing the shadow PT. The | 463 | * Compute the equivalent of the TTL field by parsing the shadow PT. The |
| 410 | * granule size is extracted from the cached VTCR_EL2.TG0 while the level is | 464 | * granule size is extracted from the cached VTCR_EL2.TG0 while the level is |
| @@ -676,23 +730,36 @@ void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu) · 265 unmodified lines | |||
| 676 | void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu) | 730 | void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu) |
| 677 | { | 731 | { |
| 678 | /* | 732 | /* |
| 679 | * The vCPU kept its reference on the MMU after the last put, keep | 733 | * If the vCPU kept its reference on the MMU after the last put, |
| 680 | * rolling with it. | 734 | * keep rolling with it. |
| 681 | */ | 735 | */ |
| 682 | if (vcpu->arch.hw_mmu) | ||
| 683 | return; | ||
| 684 | |||
| 685 | if (is_hyp_ctxt(vcpu)) { | 736 | if (is_hyp_ctxt(vcpu)) { |
| 686 | vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; | 737 | if (!vcpu->arch.hw_mmu) |
| 738 | vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; | ||
| 687 | } else { | 739 | } else { |
| 688 | write_lock(&vcpu->kvm->mmu_lock); | 740 | if (!vcpu->arch.hw_mmu) { |
| 689 | vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu); | 741 | scoped_guard(write_lock, &vcpu->kvm->mmu_lock) |
| 690 | write_unlock(&vcpu->kvm->mmu_lock); | 742 | vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu); |
| 743 | } | ||
| 744 | |||
| 745 | if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV) | ||
| 746 | kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); | ||
| 691 | } | 747 | } |
| 692 | } | 748 | } |
| 693 | 749 | ||
| 694 | void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu) | 750 | void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu) |
| 695 | { | 751 | { |
| 752 | /* Unconditionally drop the VNCR mapping if we have one */ | ||
| 753 | if (host_data_test_flag(L1_VNCR_MAPPED)) { | ||
| 754 | BUG_ON(vcpu->arch.vncr_tlb->cpu != smp_processor_id()); | ||
| 755 | BUG_ON(is_hyp_ctxt(vcpu)); | ||
| 756 | |||
| 757 | clear_fixmap(vncr_fixmap(vcpu->arch.vncr_tlb->cpu)); | ||
| 758 | vcpu->arch.vncr_tlb->cpu = -1; | ||
| 759 | host_data_clear_flag(L1_VNCR_MAPPED); | ||
| 760 | atomic_dec(&vcpu->kvm->arch.vncr_map_count); | ||
| 761 | } | ||
| 762 | |||
| 696 | /* | 763 | /* |
| 697 | * Keep a reference on the associated stage-2 MMU if the vCPU is | 764 | * Keep a reference on the associated stage-2 MMU if the vCPU is |
| 698 | * scheduling out and not in WFI emulation, suggesting it is likely to | 765 | * scheduling out and not in WFI emulation, suggesting it is likely to |
| @@ -743,6 +810,245 @@ int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2) · 44 unmodified lines | |||
| 743 | return kvm_inject_nested_sync(vcpu, esr_el2); | 810 | return kvm_inject_nested_sync(vcpu, esr_el2); |
| 744 | } | 811 | } |
| 745 | 812 | ||
| 813 | static void invalidate_vncr(struct vncr_tlb *vt) | ||
| 814 | { | ||
| 815 | vt->valid = false; | ||
| 816 | if (vt->cpu != -1) | ||
| 817 | clear_fixmap(vncr_fixmap(vt->cpu)); | ||
| 818 | } | ||
| 819 | |||
| 820 | static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) | ||
| 821 | { | ||
| 822 | struct kvm_vcpu *vcpu; | ||
| 823 | unsigned long i; | ||
| 824 | |||
| 825 | lockdep_assert_held_write(&kvm->mmu_lock); | ||
| 826 | |||
| 827 | if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) | ||
| 828 | return; | ||
| 829 | |||
| 830 | kvm_for_each_vcpu(i, vcpu, kvm) { | ||
| 831 | struct vncr_tlb *vt = vcpu->arch.vncr_tlb; | ||
| 832 | u64 ipa_start, ipa_end, ipa_size; | ||
| 833 | |||
| 834 | /* | ||
| 835 | * Careful here: We end-up here from an MMU notifier, | ||
| 836 | * and this can race against a vcpu not being onlined | ||
| 837 | * yet, without the pseudo-TLB being allocated. | ||
| 838 | * | ||
| 839 | * Skip those, as they obviously don't participate in | ||
| 840 | * the invalidation at this stage. | ||
| 841 | */ | ||
| 842 | if (!vt) | ||
| 843 | continue; | ||
| 844 | |||
| 845 | if (!vt->valid) | ||
| 846 | continue; | ||
| 847 | |||
| 848 | ipa_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, | ||
| 849 | vt->wr.level)); | ||
| 850 | ipa_start = vt->wr.pa & (ipa_size - 1); | ||
| 851 | ipa_end = ipa_start + ipa_size; | ||
| 852 | |||
| 853 | if (ipa_end <= start || ipa_start >= end) | ||
| 854 | continue; | ||
| 855 | |||
| 856 | invalidate_vncr(vt); | ||
| 857 | } | ||
| 858 | } | ||
| 859 | |||
| 860 | struct s1e2_tlbi_scope { | ||
| 861 | enum { | ||
| 862 | TLBI_ALL, | ||
| 863 | TLBI_VA, | ||
| 864 | TLBI_VAA, | ||
| 865 | TLBI_ASID, | ||
| 866 | } type; | ||
| 867 | |||
| 868 | u16 asid; | ||
| 869 | u64 va; | ||
| 870 | u64 size; | ||
| 871 | }; | ||
| 872 | |||
| 873 | static void invalidate_vncr_va(struct kvm *kvm, | ||
| 874 | struct s1e2_tlbi_scope *scope) | ||
| 875 | { | ||
| 876 | struct kvm_vcpu *vcpu; | ||
| 877 | unsigned long i; | ||
| 878 | |||
| 879 | lockdep_assert_held_write(&kvm->mmu_lock); | ||
| 880 | |||
| 881 | kvm_for_each_vcpu(i, vcpu, kvm) { | ||
| 882 | struct vncr_tlb *vt = vcpu->arch.vncr_tlb; | ||
| 883 | u64 va_start, va_end, va_size; | ||
| 884 | |||
| 885 | if (!vt->valid) | ||
| 886 | continue; | ||
| 887 | |||
| 888 | va_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, | ||
| 889 | vt->wr.level)); | ||
| 890 | va_start = vt->gva & (va_size - 1); | ||
| 891 | va_end = va_start + va_size; | ||
| 892 | |||
| 893 | switch (scope->type) { | ||
| 894 | case TLBI_ALL: | ||
| 895 | break; | ||
| 896 | |||
| 897 | case TLBI_VA: | ||
| 898 | if (va_end <= scope->va || | ||
| 899 | va_start >= (scope->va + scope->size)) | ||
| 900 | continue; | ||
| 901 | if (vt->wr.nG && vt->wr.asid != scope->asid) | ||
| 902 | continue; | ||
| 903 | break; | ||
| 904 | |||
| 905 | case TLBI_VAA: | ||
| 906 | if (va_end <= scope->va || | ||
| 907 | va_start >= (scope->va + scope->size)) | ||
| 908 | continue; | ||
| 909 | break; | ||
| 910 | |||
| 911 | case TLBI_ASID: | ||
| 912 | if (!vt->wr.nG || vt->wr.asid != scope->asid) | ||
| 913 | continue; | ||
| 914 | break; | ||
| 915 | } | ||
| 916 | |||
| 917 | invalidate_vncr(vt); | ||
| 918 | } | ||
| 919 | } | ||
| 920 | |||
| 921 | static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val, | ||
| 922 | struct s1e2_tlbi_scope *scope) | ||
| 923 | { | ||
| 924 | switch (inst) { | ||
| 925 | case OP_TLBI_ALLE2: | ||
| 926 | case OP_TLBI_ALLE2IS: | ||
| 927 | case OP_TLBI_ALLE2OS: | ||
| 928 | case OP_TLBI_VMALLE1: | ||
| 929 | case OP_TLBI_VMALLE1IS: | ||
| 930 | case OP_TLBI_VMALLE1OS: | ||
| 931 | case OP_TLBI_ALLE2NXS: | ||
| 932 | case OP_TLBI_ALLE2ISNXS: | ||
| 933 | case OP_TLBI_ALLE2OSNXS: | ||
| 934 | case OP_TLBI_VMALLE1NXS: | ||
| 935 | case OP_TLBI_VMALLE1ISNXS: | ||
| 936 | case OP_TLBI_VMALLE1OSNXS: | ||
| 937 | scope->type = TLBI_ALL; | ||
| 938 | break; | ||
| 939 | case OP_TLBI_VAE2: | ||
| 940 | case OP_TLBI_VAE2IS: | ||
| 941 | case OP_TLBI_VAE2OS: | ||
| 942 | case OP_TLBI_VAE1: | ||
| 943 | case OP_TLBI_VAE1IS: | ||
| 944 | case OP_TLBI_VAE1OS: | ||
| 945 | case OP_TLBI_VAE2NXS: | ||
| 946 | case OP_TLBI_VAE2ISNXS: | ||
| 947 | case OP_TLBI_VAE2OSNXS: | ||
| 948 | case OP_TLBI_VAE1NXS: | ||
| 949 | case OP_TLBI_VAE1ISNXS: | ||
| 950 | case OP_TLBI_VAE1OSNXS: | ||
| 951 | case OP_TLBI_VALE2: | ||
| 952 | case OP_TLBI_VALE2IS: | ||
| 953 | case OP_TLBI_VALE2OS: | ||
| 954 | case OP_TLBI_VALE1: | ||
| 955 | case OP_TLBI_VALE1IS: | ||
| 956 | case OP_TLBI_VALE1OS: | ||
| 957 | case OP_TLBI_VALE2NXS: | ||
| 958 | case OP_TLBI_VALE2ISNXS: | ||
| 959 | case OP_TLBI_VALE2OSNXS: | ||
| 960 | case OP_TLBI_VALE1NXS: | ||
| 961 | case OP_TLBI_VALE1ISNXS: | ||
| 962 | case OP_TLBI_VALE1OSNXS: | ||
| 963 | scope->type = TLBI_VA; | ||
| 964 | scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); | ||
| 965 | if (!scope->size) | ||
| 966 | scope->size = SZ_1G; | ||
| 967 | scope->va = (val << 12) & ~(scope->size - 1); | ||
| 968 | scope->asid = FIELD_GET(TLBIR_ASID_MASK, val); | ||
| 969 | break; | ||
| 970 | case OP_TLBI_ASIDE1: | ||
| 971 | case OP_TLBI_ASIDE1IS: | ||
| 972 | case OP_TLBI_ASIDE1OS: | ||
| 973 | case OP_TLBI_ASIDE1NXS: | ||
| 974 | case OP_TLBI_ASIDE1ISNXS: | ||
| 975 | case OP_TLBI_ASIDE1OSNXS: | ||
| 976 | scope->type = TLBI_ASID; | ||
| 977 | scope->asid = FIELD_GET(TLBIR_ASID_MASK, val); | ||
| 978 | break; | ||
| 979 | case OP_TLBI_VAAE1: | ||
| 980 | case OP_TLBI_VAAE1IS: | ||
| 981 | case OP_TLBI_VAAE1OS: | ||
| 982 | case OP_TLBI_VAAE1NXS: | ||
| 983 | case OP_TLBI_VAAE1ISNXS: | ||
| 984 | case OP_TLBI_VAAE1OSNXS: | ||
| 985 | case OP_TLBI_VAALE1: | ||
| 986 | case OP_TLBI_VAALE1IS: | ||
| 987 | case OP_TLBI_VAALE1OS: | ||
| 988 | case OP_TLBI_VAALE1NXS: | ||
| 989 | case OP_TLBI_VAALE1ISNXS: | ||
| 990 | case OP_TLBI_VAALE1OSNXS: | ||
| 991 | scope->type = TLBI_VAA; | ||
| 992 | scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); | ||
| 993 | if (!scope->size) | ||
| 994 | scope->size = SZ_1G; | ||
| 995 | scope->va = (val << 12) & ~(scope->size - 1); | ||
| 996 | break; | ||
| 997 | case OP_TLBI_RVAE2: | ||
| 998 | case OP_TLBI_RVAE2IS: | ||
| 999 | case OP_TLBI_RVAE2OS: | ||
| 1000 | case OP_TLBI_RVAE1: | ||
| 1001 | case OP_TLBI_RVAE1IS: | ||
| 1002 | case OP_TLBI_RVAE1OS: | ||
| 1003 | case OP_TLBI_RVAE2NXS: | ||
| 1004 | case OP_TLBI_RVAE2ISNXS: | ||
| 1005 | case OP_TLBI_RVAE2OSNXS: | ||
| 1006 | case OP_TLBI_RVAE1NXS: | ||
| 1007 | case OP_TLBI_RVAE1ISNXS: | ||
| 1008 | case OP_TLBI_RVAE1OSNXS: | ||
| 1009 | case OP_TLBI_RVALE2: | ||
| 1010 | case OP_TLBI_RVALE2IS: | ||
| 1011 | case OP_TLBI_RVALE2OS: | ||
| 1012 | case OP_TLBI_RVALE1: | ||
| 1013 | case OP_TLBI_RVALE1IS: | ||
| 1014 | case OP_TLBI_RVALE1OS: | ||
| 1015 | case OP_TLBI_RVALE2NXS: | ||
| 1016 | case OP_TLBI_RVALE2ISNXS: | ||
| 1017 | case OP_TLBI_RVALE2OSNXS: | ||
| 1018 | case OP_TLBI_RVALE1NXS: | ||
| 1019 | case OP_TLBI_RVALE1ISNXS: | ||
| 1020 | case OP_TLBI_RVALE1OSNXS: | ||
| 1021 | scope->type = TLBI_VA; | ||
| 1022 | scope->va = decode_range_tlbi(val, &scope->size, &scope->asid); | ||
| 1023 | break; | ||
| 1024 | case OP_TLBI_RVAAE1: | ||
| 1025 | case OP_TLBI_RVAAE1IS: | ||
| 1026 | case OP_TLBI_RVAAE1OS: | ||
| 1027 | case OP_TLBI_RVAAE1NXS: | ||
| 1028 | case OP_TLBI_RVAAE1ISNXS: | ||
| 1029 | case OP_TLBI_RVAAE1OSNXS: | ||
| 1030 | case OP_TLBI_RVAALE1: | ||
| 1031 | case OP_TLBI_RVAALE1IS: | ||
| 1032 | case OP_TLBI_RVAALE1OS: | ||
| 1033 | case OP_TLBI_RVAALE1NXS: | ||
| 1034 | case OP_TLBI_RVAALE1ISNXS: | ||
| 1035 | case OP_TLBI_RVAALE1OSNXS: | ||
| 1036 | scope->type = TLBI_VAA; | ||
| 1037 | scope->va = decode_range_tlbi(val, &scope->size, NULL); | ||
| 1038 | break; | ||
| 1039 | } | ||
| 1040 | } | ||
| 1041 | |||
| 1042 | void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val) | ||
| 1043 | { | ||
| 1044 | struct s1e2_tlbi_scope scope = {}; | ||
| 1045 | |||
| 1046 | compute_s1_tlbi_range(vcpu, inst, val, &scope); | ||
| 1047 | |||
| 1048 | guard(write_lock)(&vcpu->kvm->mmu_lock); | ||
| 1049 | invalidate_vncr_va(vcpu->kvm, &scope); | ||
| 1050 | } | ||
| 1051 | |||
| 746 | void kvm_nested_s2_wp(struct kvm *kvm) | 1052 | void kvm_nested_s2_wp(struct kvm *kvm) |
| 747 | { | 1053 | { |
| 748 | int i; | 1054 | int i; |
| @@ -755,6 +1061,8 @@ void kvm_nested_s2_wp(struct kvm *kvm) · 6 unmodified lines | |||
| 755 | if (kvm_s2_mmu_valid(mmu)) | 1061 | if (kvm_s2_mmu_valid(mmu)) |
| 756 | kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu)); | 1062 | kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu)); |
| 757 | } | 1063 | } |
| 1064 | |||
| 1065 | kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits)); | ||
| 758 | } | 1066 | } |
| 759 | 1067 | ||
| 760 | void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) | 1068 | void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) |
| @@ -769,6 +1077,8 @@ void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) · 8 unmodified lines | |||
| 769 | if (kvm_s2_mmu_valid(mmu)) | 1077 | if (kvm_s2_mmu_valid(mmu)) |
| 770 | kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block); | 1078 | kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block); |
| 771 | } | 1079 | } |
| 1080 | |||
| 1081 | kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits)); | ||
| 772 | } | 1082 | } |
| 773 | 1083 | ||
| 774 | void kvm_nested_s2_flush(struct kvm *kvm) | 1084 | void kvm_nested_s2_flush(struct kvm *kvm) |
| @@ -801,6 +1111,295 @@ void kvm_arch_flush_shadow_all(struct kvm *kvm) · 26 unmodified lines | |||
| 801 | kvm_uninit_stage2_mmu(kvm); | 1111 | kvm_uninit_stage2_mmu(kvm); |
| 802 | } | 1112 | } |
| 803 | 1113 | ||
| 1114 | /* | ||
| 1115 | * Dealing with VNCR_EL2 exposed by the *guest* is a complicated matter: | ||
| 1116 | * | ||
| 1117 | * - We introduce an internal representation of a vcpu-private TLB, | ||
| 1118 | * representing the mapping between the guest VA contained in VNCR_EL2, | ||
| 1119 | * the IPA the guest's EL2 PTs point to, and the actual PA this lives at. | ||
| 1120 | * | ||
| 1121 | * - On translation fault from a nested VNCR access, we create such a TLB. | ||
| 1122 | * If there is no mapping to describe, the guest inherits the fault. | ||
| 1123 | * Crucially, no actual mapping is done at this stage. | ||
| 1124 | * | ||
| 1125 | * - On vcpu_load() in a non-HYP context with HCR_EL2.NV==1, if the above | ||
| 1126 | * TLB exists, we map it in the fixmap for this CPU, and run with it. We | ||
| 1127 | * have to respect the permissions dictated by the guest, but not the | ||
| 1128 | * memory type (FWB is a must). | ||
| 1129 | * | ||
| 1130 | * - Note that we usually don't do a vcpu_load() on the back of a fault | ||
| 1131 | * (unless we are preempted), so the resolution of a translation fault | ||
| 1132 | * must go via a request that will map the VNCR page in the fixmap. | ||
| 1133 | * vcpu_load() might as well use the same mechanism. | ||
| 1134 | * | ||
| 1135 | * - On vcpu_put() in a non-HYP context with HCR_EL2.NV==1, if the TLB was | ||
| 1136 | * mapped, we unmap it. Yes it is that simple. The TLB still exists | ||
| 1137 | * though, and may be reused at a later load. | ||
| 1138 | * | ||
| 1139 | * - On permission fault, we simply forward the fault to the guest's EL2. | ||
| 1140 | * Get out of my way. | ||
| 1141 | * | ||
| 1142 | * - On any TLBI for the EL2&0 translation regime, we must find any TLB that | ||
| 1143 | * intersects with the TLBI request, invalidate it, and unmap the page | ||
| 1144 | * from the fixmap. Because we need to look at all the vcpu-private TLBs, | ||
| 1145 | * this requires some wide-ranging locking to ensure that nothing races | ||
| 1146 | * against it. This may require some refcounting to avoid the search when | ||
| 1147 | * no such TLB is present. | ||
| 1148 | * | ||
| 1149 | * - On MMU notifiers, we must invalidate our TLB in a similar way, but | ||
| 1150 | * looking at the IPA instead. The funny part is that there may not be a | ||
| 1151 | * stage-2 mapping for this page if L1 hasn't accessed it using LD/ST | ||
| 1152 | * instructions. | ||
| 1153 | */ | ||
| 1154 | |||
| 1155 | int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu) | ||
| 1156 | { | ||
| 1157 | if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) | ||
| 1158 | return 0; | ||
| 1159 | |||
| 1160 | vcpu->arch.vncr_tlb = kzalloc(sizeof(*vcpu->arch.vncr_tlb), | ||
| 1161 | GFP_KERNEL_ACCOUNT); | ||
| 1162 | if (!vcpu->arch.vncr_tlb) | ||
| 1163 | return -ENOMEM; | ||
| 1164 | |||
| 1165 | return 0; | ||
| 1166 | } | ||
| 1167 | |||
| 1168 | static u64 read_vncr_el2(struct kvm_vcpu *vcpu) | ||
| 1169 | { | ||
| 1170 | return (u64)sign_extend64(__vcpu_sys_reg(vcpu, VNCR_EL2), 48); | ||
| 1171 | } | ||
| 1172 | |||
| 1173 | static int kvm_translate_vncr(struct kvm_vcpu *vcpu) | ||
| 1174 | { | ||
| 1175 | bool write_fault, writable; | ||
| 1176 | unsigned long mmu_seq; | ||
| 1177 | struct vncr_tlb *vt; | ||
| 1178 | struct page *page; | ||
| 1179 | u64 va, pfn, gfn; | ||
| 1180 | int ret; | ||
| 1181 | |||
| 1182 | vt = vcpu->arch.vncr_tlb; | ||
| 1183 | |||
| 1184 | /* | ||
| 1185 | * If we're about to walk the EL2 S1 PTs, we must invalidate the | ||
| 1186 | * current TLB, as it could be sampled from another vcpu doing a | ||
| 1187 | * TLBI *IS. A real CPU wouldn't do that, but we only keep a single | ||
| 1188 | * translation, so not much of a choice. | ||
| 1189 | * | ||
| 1190 | * We also prepare the next walk wilst we're at it. | ||
| 1191 | */ | ||
| 1192 | scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { | ||
| 1193 | invalidate_vncr(vt); | ||
| 1194 | |||
| 1195 | vt->wi = (struct s1_walk_info) { | ||
| 1196 | .regime = TR_EL20, | ||
| 1197 | .as_el0 = false, | ||
| 1198 | .pan = false, | ||
| 1199 | }; | ||
| 1200 | vt->wr = (struct s1_walk_result){}; | ||
| 1201 | } | ||
| 1202 | |||
| 1203 | guard(srcu)(&vcpu->kvm->srcu); | ||
| 1204 | |||
| 1205 | va = read_vncr_el2(vcpu); | ||
| 1206 | |||
| 1207 | ret = __kvm_translate_va(vcpu, &vt->wi, &vt->wr, va); | ||
| 1208 | if (ret) | ||
| 1209 | return ret; | ||
| 1210 | |||
| 1211 | write_fault = kvm_is_write_fault(vcpu); | ||
| 1212 | |||
| 1213 | mmu_seq = vcpu->kvm->mmu_invalidate_seq; | ||
| 1214 | smp_rmb(); | ||
| 1215 | |||
| 1216 | gfn = vt->wr.pa >> PAGE_SHIFT; | ||
| 1217 | pfn = kvm_faultin_pfn(vcpu, gfn, write_fault, &writable, &page); | ||
| 1218 | if (is_error_noslot_pfn(pfn) || (write_fault && !writable)) | ||
| 1219 | return -EFAULT; | ||
| 1220 | |||
| 1221 | scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { | ||
| 1222 | if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) | ||
| 1223 | return -EAGAIN; | ||
| 1224 | |||
| 1225 | vt->gva = va; | ||
| 1226 | vt->hpa = pfn << PAGE_SHIFT; | ||
| 1227 | vt->valid = true; | ||
| 1228 | vt->cpu = -1; | ||
| 1229 | |||
| 1230 | kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); | ||
| 1231 | kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw); | ||
| 1232 | } | ||
| 1233 | |||
| 1234 | if (vt->wr.pw) | ||
| 1235 | mark_page_dirty(vcpu->kvm, gfn); | ||
| 1236 | |||
| 1237 | return 0; | ||
| 1238 | } | ||
| 1239 | |||
| 1240 | static void inject_vncr_perm(struct kvm_vcpu *vcpu) | ||
| 1241 | { | ||
| 1242 | struct vncr_tlb *vt = vcpu->arch.vncr_tlb; | ||
| 1243 | u64 esr = kvm_vcpu_get_esr(vcpu); | ||
| 1244 | |||
| 1245 | /* Adjust the fault level to reflect that of the guest's */ | ||
| 1246 | esr &= ~ESR_ELx_FSC; | ||
| 1247 | esr |= FIELD_PREP(ESR_ELx_FSC, | ||
| 1248 | ESR_ELx_FSC_PERM_L(vt->wr.level)); | ||
| 1249 | |||
| 1250 | kvm_inject_nested_sync(vcpu, esr); | ||
| 1251 | } | ||
| 1252 | |||
| 1253 | static bool kvm_vncr_tlb_lookup(struct kvm_vcpu *vcpu) | ||
| 1254 | { | ||
| 1255 | struct vncr_tlb *vt = vcpu->arch.vncr_tlb; | ||
| 1256 | |||
| 1257 | lockdep_assert_held_read(&vcpu->kvm->mmu_lock); | ||
| 1258 | |||
| 1259 | if (!vt->valid) | ||
| 1260 | return false; | ||
| 1261 | |||
| 1262 | if (read_vncr_el2(vcpu) != vt->gva) | ||
| 1263 | return false; | ||
| 1264 | |||
| 1265 | if (vt->wr.nG) { | ||
| 1266 | u64 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2); | ||
| 1267 | u64 ttbr = ((tcr & TCR_A1) ? | ||
| 1268 | vcpu_read_sys_reg(vcpu, TTBR1_EL2) : | ||
| 1269 | vcpu_read_sys_reg(vcpu, TTBR0_EL2)); | ||
| 1270 | u16 asid; | ||
| 1271 | |||
| 1272 | asid = FIELD_GET(TTBR_ASID_MASK, ttbr); | ||
| 1273 | if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) || | ||
| 1274 | !(tcr & TCR_ASID16)) | ||
| 1275 | asid &= GENMASK(7, 0); | ||
| 1276 | |||
| 1277 | return asid != vt->wr.asid; | ||
| 1278 | } | ||
| 1279 | |||
| 1280 | return true; | ||
| 1281 | } | ||
| 1282 | |||
| 1283 | int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu) | ||
| 1284 | { | ||
| 1285 | struct vncr_tlb *vt = vcpu->arch.vncr_tlb; | ||
| 1286 | u64 esr = kvm_vcpu_get_esr(vcpu); | ||
| 1287 | |||
| 1288 | BUG_ON(!(esr & ESR_ELx_VNCR_SHIFT)); | ||
| 1289 | |||
| 1290 | if (esr_fsc_is_permission_fault(esr)) { | ||
| 1291 | inject_vncr_perm(vcpu); | ||
| 1292 | } else if (esr_fsc_is_translation_fault(esr)) { | ||
| 1293 | bool valid; | ||
| 1294 | int ret; | ||
| 1295 | |||
| 1296 | scoped_guard(read_lock, &vcpu->kvm->mmu_lock) | ||
| 1297 | valid = kvm_vncr_tlb_lookup(vcpu); | ||
| 1298 | |||
| 1299 | if (!valid) | ||
| 1300 | ret = kvm_translate_vncr(vcpu); | ||
| 1301 | else | ||
| 1302 | ret = -EPERM; | ||
| 1303 | |||
| 1304 | switch (ret) { | ||
| 1305 | case -EAGAIN: | ||
| 1306 | case -ENOMEM: | ||
| 1307 | /* Let's try again... */ | ||
| 1308 | break; | ||
| 1309 | case -EFAULT: | ||
| 1310 | case -EINVAL: | ||
| 1311 | case -ENOENT: | ||
| 1312 | case -EACCES: | ||
| 1313 | /* | ||
| 1314 | * Translation failed, inject the corresponding | ||
| 1315 | * exception back to EL2. | ||
| 1316 | */ | ||
| 1317 | BUG_ON(!vt->wr.failed); | ||
| 1318 | |||
| 1319 | esr &= ~ESR_ELx_FSC; | ||
| 1320 | esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst); | ||
| 1321 | |||
| 1322 | kvm_inject_nested_sync(vcpu, esr); | ||
| 1323 | break; | ||
| 1324 | case -EPERM: | ||
| 1325 | /* Hack to deal with POE until we get kernel support */ | ||
| 1326 | inject_vncr_perm(vcpu); | ||
| 1327 | break; | ||
| 1328 | case 0: | ||
| 1329 | break; | ||
| 1330 | } | ||
| 1331 | } else { | ||
| 1332 | WARN_ONCE(1, "Unhandled VNCR abort, ESR=%llx\n", esr); | ||
| 1333 | } | ||
| 1334 | |||
| 1335 | return 1; | ||
| 1336 | } | ||
| 1337 | |||
| 1338 | static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) | ||
| 1339 | { | ||
| 1340 | struct vncr_tlb *vt = vcpu->arch.vncr_tlb; | ||
| 1341 | pgprot_t prot; | ||
| 1342 | |||
| 1343 | guard(preempt)(); | ||
| 1344 | guard(read_lock)(&vcpu->kvm->mmu_lock); | ||
| 1345 | |||
| 1346 | /* | ||
| 1347 | * The request to map VNCR may have raced against some other | ||
| 1348 | * event, such as an interrupt, and may not be valid anymore. | ||
| 1349 | */ | ||
| 1350 | if (is_hyp_ctxt(vcpu)) | ||
| 1351 | return; | ||
| 1352 | |||
| 1353 | /* | ||
| 1354 | * Check that the pseudo-TLB is valid and that VNCR_EL2 still | ||
| 1355 | * contains the expected value. If it doesn't, we simply bail out | ||
| 1356 | * without a mapping -- a transformed MSR/MRS will generate the | ||
| 1357 | * fault and allows us to populate the pseudo-TLB. | ||
| 1358 | */ | ||
| 1359 | if (!vt->valid) | ||
| 1360 | return; | ||
| 1361 | |||
| 1362 | if (read_vncr_el2(vcpu) != vt->gva) | ||
| 1363 | return; | ||
| 1364 | |||
| 1365 | if (vt->wr.nG) { | ||
| 1366 | u64 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2); | ||
| 1367 | u64 ttbr = ((tcr & TCR_A1) ? | ||
| 1368 | vcpu_read_sys_reg(vcpu, TTBR1_EL2) : | ||
| 1369 | vcpu_read_sys_reg(vcpu, TTBR0_EL2)); | ||
| 1370 | u16 asid; | ||
| 1371 | |||
| 1372 | asid = FIELD_GET(TTBR_ASID_MASK, ttbr); | ||
| 1373 | if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) || | ||
| 1374 | !(tcr & TCR_ASID16)) | ||
| 1375 | asid &= GENMASK(7, 0); | ||
| 1376 | |||
| 1377 | if (asid != vt->wr.asid) | ||
| 1378 | return; | ||
| 1379 | } | ||
| 1380 | |||
| 1381 | vt->cpu = smp_processor_id(); | ||
| 1382 | |||
| 1383 | if (vt->wr.pw && vt->wr.pr) | ||
| 1384 | prot = PAGE_KERNEL; | ||
| 1385 | else if (vt->wr.pr) | ||
| 1386 | prot = PAGE_KERNEL_RO; | ||
| 1387 | else | ||
| 1388 | prot = PAGE_NONE; | ||
| 1389 | |||
| 1390 | /* | ||
| 1391 | * We can't map write-only (or no permission at all) in the kernel, | ||
| 1392 | * but the guest can do it if using POE, so we'll have to turn a | ||
| 1393 | * translation fault into a permission fault at runtime. | ||
| 1394 | * FIXME: WO doesn't work at all, need POE support in the kernel. | ||
| 1395 | */ | ||
| 1396 | if (pgprot_val(prot) != pgprot_val(PAGE_NONE)) { | ||
| 1397 | __set_fixmap(vncr_fixmap(vt->cpu), vt->hpa, prot); | ||
| 1398 | host_data_set_flag(L1_VNCR_MAPPED); | ||
| 1399 | atomic_inc(&vcpu->kvm->arch.vncr_map_count); | ||
| 1400 | } | ||
| 1401 | } | ||
| 1402 | |||
| 804 | /* | 1403 | /* |
| 805 | * Our emulated CPU doesn't support all the possible features. For the | 1404 | * Our emulated CPU doesn't support all the possible features. For the |
| 806 | * sake of simplicity (and probably mental sanity), wipe out a number | 1405 | * sake of simplicity (and probably mental sanity), wipe out a number |
| @@ -1151,6 +1750,9 @@ int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu) · 344 unmodified lines | |||
| 1151 | res0 |= ICH_HCR_EL2_DVIM | ICH_HCR_EL2_vSGIEOICount; | 1750 | res0 |= ICH_HCR_EL2_DVIM | ICH_HCR_EL2_vSGIEOICount; |
| 1152 | set_sysreg_masks(kvm, ICH_HCR_EL2, res0, res1); | 1751 | set_sysreg_masks(kvm, ICH_HCR_EL2, res0, res1); |
| 1153 | 1752 | ||
| 1753 | /* VNCR_EL2 */ | ||
| 1754 | set_sysreg_masks(kvm, VNCR_EL2, VNCR_EL2_RES0, VNCR_EL2_RES1); | ||
| 1755 | |||
| 1154 | out: | 1756 | out: |
| 1155 | for (enum vcpu_sysreg sr = __SANITISED_REG_START__; sr < NR_SYS_REGS; sr++) | 1757 | for (enum vcpu_sysreg sr = __SANITISED_REG_START__; sr < NR_SYS_REGS; sr++) |
| 1156 | (void)__vcpu_sys_reg(vcpu, sr); | 1758 | (void)__vcpu_sys_reg(vcpu, sr); |
| @@ -1171,6 +1773,9 @@ void check_nested_vcpu_requests(struct kvm_vcpu *vcpu) · 14 unmodified lines | |||
| 1171 | write_unlock(&vcpu->kvm->mmu_lock); | 1773 | write_unlock(&vcpu->kvm->mmu_lock); |
| 1172 | } | 1774 | } |
| 1173 | 1775 | ||
| 1776 | if (kvm_check_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu)) | ||
| 1777 | kvm_map_l1_vncr(vcpu); | ||
| 1778 | |||
| 1174 | /* Must be last, as may switch context! */ | 1779 | /* Must be last, as may switch context! */ |
| 1175 | if (kvm_check_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu)) | 1780 | if (kvm_check_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu)) |
| 1176 | kvm_inject_nested_irq(vcpu); | 1781 | kvm_inject_nested_irq(vcpu); |
| @@ -158,6 +158,8 @@ void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu) · 157 unmodified lines | |||
| 158 | if (sve_state) | 158 | if (sve_state) |
| 159 | kvm_unshare_hyp(sve_state, sve_state + vcpu_sve_state_size(vcpu)); | 159 | kvm_unshare_hyp(sve_state, sve_state + vcpu_sve_state_size(vcpu)); |
| 160 | kfree(sve_state); | 160 | kfree(sve_state); |
| 161 | free_page((unsigned long)vcpu->arch.ctxt.vncr_array); | ||
| 162 | kfree(vcpu->arch.vncr_tlb); | ||
| 161 | kfree(vcpu->arch.ccsidr); | 163 | kfree(vcpu->arch.ccsidr); |
| 162 | } | 164 | } |
| 163 | 165 | ||
| @@ -2306,15 +2306,6 @@ static bool bad_redir_trap(struct kvm_vcpu *vcpu, · 2305 unmodified lines | |||
| 2306 | "trap of EL2 register redirected to EL1"); | 2306 | "trap of EL2 register redirected to EL1"); |
| 2307 | } | 2307 | } |
| 2308 | 2308 | ||
| 2309 | #define EL2_REG(name, acc, rst, v) { \ | ||
| 2310 | SYS_DESC(SYS_##name), \ | ||
| 2311 | .access = acc, \ | ||
| 2312 | .reset = rst, \ | ||
| 2313 | .reg = name, \ | ||
| 2314 | .visibility = el2_visibility, \ | ||
| 2315 | .val = v, \ | ||
| 2316 | } | ||
| 2317 | |||
| 2318 | #define EL2_REG_FILTERED(name, acc, rst, v, filter) { \ | 2309 | #define EL2_REG_FILTERED(name, acc, rst, v, filter) { \ |
| 2319 | SYS_DESC(SYS_##name), \ | 2310 | SYS_DESC(SYS_##name), \ |
| 2320 | .access = acc, \ | 2311 | .access = acc, \ |
| @@ -2324,6 +2315,9 @@ static bool bad_redir_trap(struct kvm_vcpu *vcpu, · 3 unmodified lines | |||
| 2324 | .val = v, \ | 2315 | .val = v, \ |
| 2325 | } | 2316 | } |
| 2326 | 2317 | ||
| 2318 | #define EL2_REG(name, acc, rst, v) \ | ||
| 2319 | EL2_REG_FILTERED(name, acc, rst, v, el2_visibility) | ||
| 2320 | |||
| 2327 | #define EL2_REG_VNCR(name, rst, v) EL2_REG(name, bad_vncr_trap, rst, v) | 2321 | #define EL2_REG_VNCR(name, rst, v) EL2_REG(name, bad_vncr_trap, rst, v) |
| 2328 | #define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v) | 2322 | #define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v) |
| 2329 | 2323 | ||
| @@ -2471,6 +2465,16 @@ static unsigned int sve_el2_visibility(const struct kvm_vcpu *vcpu, · 141 unmodified lines | |||
| 2471 | return __el2_visibility(vcpu, rd, sve_visibility); | 2465 | return __el2_visibility(vcpu, rd, sve_visibility); |
| 2472 | } | 2466 | } |
| 2473 | 2467 | ||
| 2468 | static unsigned int vncr_el2_visibility(const struct kvm_vcpu *vcpu, | ||
| 2469 | const struct sys_reg_desc *rd) | ||
| 2470 | { | ||
| 2471 | if (el2_visibility(vcpu, rd) == 0 && | ||
| 2472 | kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) | ||
| 2473 | return 0; | ||
| 2474 | |||
| 2475 | return REG_HIDDEN; | ||
| 2476 | } | ||
| 2477 | |||
| 2474 | static bool access_zcr_el2(struct kvm_vcpu *vcpu, | 2478 | static bool access_zcr_el2(struct kvm_vcpu *vcpu, |
| 2475 | struct sys_reg_params *p, | 2479 | struct sys_reg_params *p, |
| 2476 | const struct sys_reg_desc *r) | 2480 | const struct sys_reg_desc *r) |
| @@ -3311,6 +3315,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 834 unmodified lines | |||
| 3311 | tcr2_el2_visibility), | 3315 | tcr2_el2_visibility), |
| 3312 | EL2_REG_VNCR(VTTBR_EL2, reset_val, 0), | 3316 | EL2_REG_VNCR(VTTBR_EL2, reset_val, 0), |
| 3313 | EL2_REG_VNCR(VTCR_EL2, reset_val, 0), | 3317 | EL2_REG_VNCR(VTCR_EL2, reset_val, 0), |
| 3318 | EL2_REG_FILTERED(VNCR_EL2, bad_vncr_trap, reset_val, 0, | ||
| 3319 | vncr_el2_visibility), | ||
| 3314 | 3320 | ||
| 3315 | { SYS_DESC(SYS_DACR32_EL2), undef_access, reset_unknown, DACR32_EL2 }, | 3321 | { SYS_DESC(SYS_DACR32_EL2), undef_access, reset_unknown, DACR32_EL2 }, |
| 3316 | EL2_REG_VNCR(HDFGRTR_EL2, reset_val, 0), | 3322 | EL2_REG_VNCR(HDFGRTR_EL2, reset_val, 0), |
| @@ -3594,8 +3600,7 @@ static bool handle_ripas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 277 unmodified lines | |||
| 3594 | { | 3600 | { |
| 3595 | u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2); | 3601 | u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2); |
| 3596 | u64 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); | 3602 | u64 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); |
| 3597 | u64 base, range, tg, num, scale; | 3603 | u64 base, range; |
| 3598 | int shift; | ||
| 3599 | 3604 | ||
| 3600 | if (!kvm_supported_tlbi_ipas2_op(vcpu, sys_encoding)) | 3605 | if (!kvm_supported_tlbi_ipas2_op(vcpu, sys_encoding)) |
| 3601 | return undef_access(vcpu, p, r); | 3606 | return undef_access(vcpu, p, r); |
| @@ -3605,26 +3610,7 @@ static bool handle_ripas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 3 unmodified lines | |||
| 3605 | * of the guest's S2 (different base granule size, for example), we | 3610 | * of the guest's S2 (different base granule size, for example), we |
| 3606 | * decide to ignore TTL and only use the described range. | 3611 | * decide to ignore TTL and only use the described range. |
| 3607 | */ | 3612 | */ |
| 3608 | tg = FIELD_GET(GENMASK(47, 46), p->regval); | 3613 | base = decode_range_tlbi(p->regval, &range, NULL); |
| 3609 | scale = FIELD_GET(GENMASK(45, 44), p->regval); | ||
| 3610 | num = FIELD_GET(GENMASK(43, 39), p->regval); | ||
| 3611 | base = p->regval & GENMASK(36, 0); | ||
| 3612 | |||
| 3613 | switch(tg) { | ||
| 3614 | case 1: | ||
| 3615 | shift = 12; | ||
| 3616 | break; | ||
| 3617 | case 2: | ||
| 3618 | shift = 14; | ||
| 3619 | break; | ||
| 3620 | case 3: | ||
| 3621 | default: /* IMPDEF: handle tg==0 as 64k */ | ||
| 3622 | shift = 16; | ||
| 3623 | break; | ||
| 3624 | } | ||
| 3625 | |||
| 3626 | base <<= shift; | ||
| 3627 | range = __TLBI_RANGE_PAGES(num, scale) << shift; | ||
| 3628 | 3614 | ||
| 3629 | kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, get_vmid(vttbr), | 3615 | kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, get_vmid(vttbr), |
| 3630 | &(union tlbi_info) { | 3616 | &(union tlbi_info) { |
| @@ -3690,11 +3676,22 @@ static void s2_mmu_tlbi_s1e1(struct kvm_s2_mmu *mmu, · 59 unmodified lines | |||
| 3690 | WARN_ON(__kvm_tlbi_s1e2(mmu, info->va.addr, info->va.encoding)); | 3676 | WARN_ON(__kvm_tlbi_s1e2(mmu, info->va.addr, info->va.encoding)); |
| 3691 | } | 3677 | } |
| 3692 | 3678 | ||
| 3679 | static bool handle_tlbi_el2(struct kvm_vcpu *vcpu, struct sys_reg_params *p, | ||
| 3680 | const struct sys_reg_desc *r) | ||
| 3681 | { | ||
| 3682 | u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2); | ||
| 3683 | |||
| 3684 | if (!kvm_supported_tlbi_s1e2_op(vcpu, sys_encoding)) | ||
| 3685 | return undef_access(vcpu, p, r); | ||
| 3686 | |||
| 3687 | kvm_handle_s1e2_tlbi(vcpu, sys_encoding, p->regval); | ||
| 3688 | return true; | ||
| 3689 | } | ||
| 3690 | |||
| 3693 | static bool handle_tlbi_el1(struct kvm_vcpu *vcpu, struct sys_reg_params *p, | 3691 | static bool handle_tlbi_el1(struct kvm_vcpu *vcpu, struct sys_reg_params *p, |
| 3694 | const struct sys_reg_desc *r) | 3692 | const struct sys_reg_desc *r) |
| 3695 | { | 3693 | { |
| 3696 | u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2); | 3694 | u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2); |
| 3697 | u64 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); | ||
| 3698 | 3695 | ||
| 3699 | /* | 3696 | /* |
| 3700 | * If we're here, this is because we've trapped on a EL1 TLBI | 3697 | * If we're here, this is because we've trapped on a EL1 TLBI |
| @@ -3705,6 +3702,13 @@ static bool handle_tlbi_el1(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 4 unmodified lines | |||
| 3705 | * - HCR_EL2.E2H == 0 : a non-VHE guest | 3702 | * - HCR_EL2.E2H == 0 : a non-VHE guest |
| 3706 | * - HCR_EL2.{E2H,TGE} == { 1, 0 } : a VHE guest in guest mode | 3703 | * - HCR_EL2.{E2H,TGE} == { 1, 0 } : a VHE guest in guest mode |
| 3707 | * | 3704 | * |
| 3705 | * Another possibility is that we are invalidating the EL2 context | ||
| 3706 | * using EL1 instructions, but that we landed here because we need | ||
| 3707 | * additional invalidation for structures that are not held in the | ||
| 3708 | * CPU TLBs (such as the VNCR pseudo-TLB and its EL2 mapping). In | ||
| 3709 | * that case, we are guaranteed that HCR_EL2.{E2H,TGE} == { 1, 1 } | ||
| 3710 | * as we don't allow an NV-capable L1 in a nVHE configuration. | ||
| 3711 | * | ||
| 3708 | * We don't expect these helpers to ever be called when running | 3712 | * We don't expect these helpers to ever be called when running |
| 3709 | * in a vEL1 context. | 3713 | * in a vEL1 context. |
| 3710 | */ | 3714 | */ |
| @@ -3714,7 +3718,13 @@ static bool handle_tlbi_el1(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 3 unmodified lines | |||
| 3714 | if (!kvm_supported_tlbi_s1e1_op(vcpu, sys_encoding)) | 3718 | if (!kvm_supported_tlbi_s1e1_op(vcpu, sys_encoding)) |
| 3715 | return undef_access(vcpu, p, r); | 3719 | return undef_access(vcpu, p, r); |
| 3716 | 3720 | ||
| 3717 | kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, get_vmid(vttbr), | 3721 | if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu)) { |
| 3722 | kvm_handle_s1e2_tlbi(vcpu, sys_encoding, p->regval); | ||
| 3723 | return true; | ||
| 3724 | } | ||
| 3725 | |||
| 3726 | kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, | ||
| 3727 | get_vmid(__vcpu_sys_reg(vcpu, VTTBR_EL2)), | ||
| 3718 | &(union tlbi_info) { | 3728 | &(union tlbi_info) { |
| 3719 | .va = { | 3729 | .va = { |
| 3720 | .addr = p->regval, | 3730 | .addr = p->regval, |
| @@ -3836,16 +3846,21 @@ static struct sys_reg_desc sys_insn_descs[] = { · 115 unmodified lines | |||
| 3836 | SYS_INSN(TLBI_IPAS2LE1IS, handle_ipas2e1is), | 3846 | SYS_INSN(TLBI_IPAS2LE1IS, handle_ipas2e1is), |
| 3837 | SYS_INSN(TLBI_RIPAS2LE1IS, handle_ripas2e1is), | 3847 | SYS_INSN(TLBI_RIPAS2LE1IS, handle_ripas2e1is), |
| 3838 | 3848 | ||
| 3839 | SYS_INSN(TLBI_ALLE2OS, undef_access), | 3849 | SYS_INSN(TLBI_ALLE2OS, handle_tlbi_el2), |
| 3840 | SYS_INSN(TLBI_VAE2OS, undef_access), | 3850 | SYS_INSN(TLBI_VAE2OS, handle_tlbi_el2), |
| 3841 | SYS_INSN(TLBI_ALLE1OS, handle_alle1is), | 3851 | SYS_INSN(TLBI_ALLE1OS, handle_alle1is), |
| 3842 | SYS_INSN(TLBI_VALE2OS, undef_access), | 3852 | SYS_INSN(TLBI_VALE2OS, handle_tlbi_el2), |
| 3843 | SYS_INSN(TLBI_VMALLS12E1OS, handle_vmalls12e1is), | 3853 | SYS_INSN(TLBI_VMALLS12E1OS, handle_vmalls12e1is), |
| 3844 | 3854 | ||
| 3845 | SYS_INSN(TLBI_RVAE2IS, undef_access), | 3855 | SYS_INSN(TLBI_RVAE2IS, handle_tlbi_el2), |
| 3846 | SYS_INSN(TLBI_RVALE2IS, undef_access), | 3856 | SYS_INSN(TLBI_RVALE2IS, handle_tlbi_el2), |
| 3857 | SYS_INSN(TLBI_ALLE2IS, handle_tlbi_el2), | ||
| 3858 | SYS_INSN(TLBI_VAE2IS, handle_tlbi_el2), | ||
| 3847 | 3859 | ||
| 3848 | SYS_INSN(TLBI_ALLE1IS, handle_alle1is), | 3860 | SYS_INSN(TLBI_ALLE1IS, handle_alle1is), |
| 3861 | |||
| 3862 | SYS_INSN(TLBI_VALE2IS, handle_tlbi_el2), | ||
| 3863 | |||
| 3849 | SYS_INSN(TLBI_VMALLS12E1IS, handle_vmalls12e1is), | 3864 | SYS_INSN(TLBI_VMALLS12E1IS, handle_vmalls12e1is), |
| 3850 | SYS_INSN(TLBI_IPAS2E1OS, handle_ipas2e1is), | 3865 | SYS_INSN(TLBI_IPAS2E1OS, handle_ipas2e1is), |
| 3851 | SYS_INSN(TLBI_IPAS2E1, handle_ipas2e1is), | 3866 | SYS_INSN(TLBI_IPAS2E1, handle_ipas2e1is), |
| @@ -3855,11 +3870,17 @@ static struct sys_reg_desc sys_insn_descs[] = { · 3 unmodified lines | |||
| 3855 | SYS_INSN(TLBI_IPAS2LE1, handle_ipas2e1is), | 3870 | SYS_INSN(TLBI_IPAS2LE1, handle_ipas2e1is), |
| 3856 | SYS_INSN(TLBI_RIPAS2LE1, handle_ripas2e1is), | 3871 | SYS_INSN(TLBI_RIPAS2LE1, handle_ripas2e1is), |
| 3857 | SYS_INSN(TLBI_RIPAS2LE1OS, handle_ripas2e1is), | 3872 | SYS_INSN(TLBI_RIPAS2LE1OS, handle_ripas2e1is), |
| 3858 | SYS_INSN(TLBI_RVAE2OS, undef_access), | 3873 | SYS_INSN(TLBI_RVAE2OS, handle_tlbi_el2), |
| 3859 | SYS_INSN(TLBI_RVALE2OS, undef_access), | 3874 | SYS_INSN(TLBI_RVALE2OS, handle_tlbi_el2), |
| 3860 | SYS_INSN(TLBI_RVAE2, undef_access), | 3875 | SYS_INSN(TLBI_RVAE2, handle_tlbi_el2), |
| 3861 | SYS_INSN(TLBI_RVALE2, undef_access), | 3876 | SYS_INSN(TLBI_RVALE2, handle_tlbi_el2), |
| 3877 | SYS_INSN(TLBI_ALLE2, handle_tlbi_el2), | ||
| 3878 | SYS_INSN(TLBI_VAE2, handle_tlbi_el2), | ||
| 3879 | |||
| 3862 | SYS_INSN(TLBI_ALLE1, handle_alle1is), | 3880 | SYS_INSN(TLBI_ALLE1, handle_alle1is), |
| 3881 | |||
| 3882 | SYS_INSN(TLBI_VALE2, handle_tlbi_el2), | ||
| 3883 | |||
| 3863 | SYS_INSN(TLBI_VMALLS12E1, handle_vmalls12e1is), | 3884 | SYS_INSN(TLBI_VMALLS12E1, handle_vmalls12e1is), |
| 3864 | 3885 | ||
| 3865 | SYS_INSN(TLBI_IPAS2E1ISNXS, handle_ipas2e1is), | 3886 | SYS_INSN(TLBI_IPAS2E1ISNXS, handle_ipas2e1is), |
| @@ -3867,19 +3888,19 @@ static struct sys_reg_desc sys_insn_descs[] = { · 1 unmodified line | |||
| 3867 | SYS_INSN(TLBI_IPAS2LE1ISNXS, handle_ipas2e1is), | 3888 | SYS_INSN(TLBI_IPAS2LE1ISNXS, handle_ipas2e1is), |
| 3868 | SYS_INSN(TLBI_RIPAS2LE1ISNXS, handle_ripas2e1is), | 3889 | SYS_INSN(TLBI_RIPAS2LE1ISNXS, handle_ripas2e1is), |
| 3869 | 3890 | ||
| 3870 | SYS_INSN(TLBI_ALLE2OSNXS, undef_access), | 3891 | SYS_INSN(TLBI_ALLE2OSNXS, handle_tlbi_el2), |
| 3871 | SYS_INSN(TLBI_VAE2OSNXS, undef_access), | 3892 | SYS_INSN(TLBI_VAE2OSNXS, handle_tlbi_el2), |
| 3872 | SYS_INSN(TLBI_ALLE1OSNXS, handle_alle1is), | 3893 | SYS_INSN(TLBI_ALLE1OSNXS, handle_alle1is), |
| 3873 | SYS_INSN(TLBI_VALE2OSNXS, undef_access), | 3894 | SYS_INSN(TLBI_VALE2OSNXS, handle_tlbi_el2), |
| 3874 | SYS_INSN(TLBI_VMALLS12E1OSNXS, handle_vmalls12e1is), | 3895 | SYS_INSN(TLBI_VMALLS12E1OSNXS, handle_vmalls12e1is), |
| 3875 | 3896 | ||
| 3876 | SYS_INSN(TLBI_RVAE2ISNXS, undef_access), | 3897 | SYS_INSN(TLBI_RVAE2ISNXS, handle_tlbi_el2), |
| 3877 | SYS_INSN(TLBI_RVALE2ISNXS, undef_access), | 3898 | SYS_INSN(TLBI_RVALE2ISNXS, handle_tlbi_el2), |
| 3878 | SYS_INSN(TLBI_ALLE2ISNXS, undef_access), | 3899 | SYS_INSN(TLBI_ALLE2ISNXS, handle_tlbi_el2), |
| 3879 | SYS_INSN(TLBI_VAE2ISNXS, undef_access), | 3900 | SYS_INSN(TLBI_VAE2ISNXS, handle_tlbi_el2), |
| 3880 | 3901 | ||
| 3881 | SYS_INSN(TLBI_ALLE1ISNXS, handle_alle1is), | 3902 | SYS_INSN(TLBI_ALLE1ISNXS, handle_alle1is), |
| 3882 | SYS_INSN(TLBI_VALE2ISNXS, undef_access), | 3903 | SYS_INSN(TLBI_VALE2ISNXS, handle_tlbi_el2), |
| 3883 | SYS_INSN(TLBI_VMALLS12E1ISNXS, handle_vmalls12e1is), | 3904 | SYS_INSN(TLBI_VMALLS12E1ISNXS, handle_vmalls12e1is), |
| 3884 | SYS_INSN(TLBI_IPAS2E1OSNXS, handle_ipas2e1is), | 3905 | SYS_INSN(TLBI_IPAS2E1OSNXS, handle_ipas2e1is), |
| 3885 | SYS_INSN(TLBI_IPAS2E1NXS, handle_ipas2e1is), | 3906 | SYS_INSN(TLBI_IPAS2E1NXS, handle_ipas2e1is), |
| @@ -3889,14 +3910,14 @@ static struct sys_reg_desc sys_insn_descs[] = { · 3 unmodified lines | |||
| 3889 | SYS_INSN(TLBI_IPAS2LE1NXS, handle_ipas2e1is), | 3910 | SYS_INSN(TLBI_IPAS2LE1NXS, handle_ipas2e1is), |
| 3890 | SYS_INSN(TLBI_RIPAS2LE1NXS, handle_ripas2e1is), | 3911 | SYS_INSN(TLBI_RIPAS2LE1NXS, handle_ripas2e1is), |
| 3891 | SYS_INSN(TLBI_RIPAS2LE1OSNXS, handle_ripas2e1is), | 3912 | SYS_INSN(TLBI_RIPAS2LE1OSNXS, handle_ripas2e1is), |
| 3892 | SYS_INSN(TLBI_RVAE2OSNXS, undef_access), | 3913 | SYS_INSN(TLBI_RVAE2OSNXS, handle_tlbi_el2), |
| 3893 | SYS_INSN(TLBI_RVALE2OSNXS, undef_access), | 3914 | SYS_INSN(TLBI_RVALE2OSNXS, handle_tlbi_el2), |
| 3894 | SYS_INSN(TLBI_RVAE2NXS, undef_access), | 3915 | SYS_INSN(TLBI_RVAE2NXS, handle_tlbi_el2), |
| 3895 | SYS_INSN(TLBI_RVALE2NXS, undef_access), | 3916 | SYS_INSN(TLBI_RVALE2NXS, handle_tlbi_el2), |
| 3896 | SYS_INSN(TLBI_ALLE2NXS, undef_access), | 3917 | SYS_INSN(TLBI_ALLE2NXS, handle_tlbi_el2), |
| 3897 | SYS_INSN(TLBI_VAE2NXS, undef_access), | 3918 | SYS_INSN(TLBI_VAE2NXS, handle_tlbi_el2), |
| 3898 | SYS_INSN(TLBI_ALLE1NXS, handle_alle1is), | 3919 | SYS_INSN(TLBI_ALLE1NXS, handle_alle1is), |
| 3899 | SYS_INSN(TLBI_VALE2NXS, undef_access), | 3920 | SYS_INSN(TLBI_VALE2NXS, handle_tlbi_el2), |
| 3900 | SYS_INSN(TLBI_VMALLS12E1NXS, handle_vmalls12e1is), | 3921 | SYS_INSN(TLBI_VMALLS12E1NXS, handle_vmalls12e1is), |
| 3901 | }; | 3922 | }; |
| 3902 | 3923 | ||
| @@ -3816,6 +3816,12 @@ Sysreg SMCR_EL2 3 4 1 2 6 · 3815 unmodified lines | |||
| 3816 | Fields SMCR_ELx | 3816 | Fields SMCR_ELx |
| 3817 | EndSysreg | 3817 | EndSysreg |
| 3818 | 3818 | ||
| 3819 | Sysreg VNCR_EL2 3 4 2 2 0 | ||
| 3820 | Field 63:57 RESS | ||
| 3821 | Field 56:12 BADDR | ||
| 3822 | Res0 11:0 | ||
| 3823 | EndSysreg | ||
| 3824 | |||
| 3819 | Sysreg GCSCR_EL2 3 4 2 5 0 | 3825 | Sysreg GCSCR_EL2 3 4 2 5 0 |
| 3820 | Fields GCSCR_ELx | 3826 | Fields GCSCR_ELx |
| 3821 | EndSysreg | 3827 | EndSysreg |
| @@ -930,6 +930,8 @@ struct kvm_enable_cap { · 929 unmodified lines | |||
| 930 | #define KVM_CAP_X86_APIC_BUS_CYCLES_NS 237 | 930 | #define KVM_CAP_X86_APIC_BUS_CYCLES_NS 237 |
| 931 | #define KVM_CAP_X86_GUEST_MODE 238 | 931 | #define KVM_CAP_X86_GUEST_MODE 238 |
| 932 | #define KVM_CAP_ARM_WRITABLE_IMP_ID_REGS 239 | 932 | #define KVM_CAP_ARM_WRITABLE_IMP_ID_REGS 239 |
| 933 | #define KVM_CAP_ARM_EL2 240 | ||
| 934 | #define KVM_CAP_ARM_EL2_E2H0 241 | ||
| 933 | 935 | ||
| 934 | struct kvm_irq_routing_irqchip { | 936 | struct kvm_irq_routing_irqchip { |
| 935 | __u32 irqchip; | 937 | __u32 irqchip; |