* kvm-arm64/nv-nv: : . : Flick the switch on the NV support by adding the missing piece : in the form of the VNCR page management. From the cover letter: : : "This is probably the most interesting bit of the whole NV adventure. : So far, everything else has been a walk in the park, but this one is : where the real fun takes place. : : With FEAT_NV2, most of the NV support revolves around tricking a guest : into accessing memory while it tries to access system registers. The : hypervisor's job is to handle the context switch of the actual : registers with the state in memory as needed." : . KVM: arm64: nv: Release faulted-in VNCR page from mmu_lock critical section KVM: arm64: nv: Handle TLBI S1E2 for VNCR invalidation with mmu_lock held KVM: arm64: nv: Hold mmu_lock when invalidating VNCR SW-TLB before translating KVM: arm64: Document NV caps and vcpu flags KVM: arm64: Allow userspace to request KVM_ARM_VCPU_EL2* KVM: arm64: nv: Remove dead code from ERET handling KVM: arm64: nv: Plumb TLBI S1E2 into system instruction dispatch KVM: arm64: nv: Add S1 TLB invalidation primitive for VNCR_EL2 KVM: arm64: nv: Program host's VNCR_EL2 to the fixmap address KVM: arm64: nv: Handle VNCR_EL2 invalidation from MMU notifiers KVM: arm64: nv: Handle mapping of VNCR_EL2 at EL2 KVM: arm64: nv: Handle VNCR_EL2-triggered faults KVM: arm64: nv: Add userspace and guest handling of VNCR_EL2 KVM: arm64: nv: Add pseudo-TLB backing VNCR_EL2 KVM: arm64: nv: Don't adjust PSTATE.M when L2 is nesting KVM: arm64: nv: Move TLBI range decoding to a helper KVM: arm64: nv: Snapshot S1 ASID tagging information during walk KVM: arm64: nv: Extract translation helper from the AT code KVM: arm64: nv: Allocate VNCR page when required arm64: sysreg: Add layout for VNCR_EL2
Unified · Split
| @@ -3460,7 +3460,8 @@ The initial values are defined as: · 3459 unmodified lines | ||
| 3460 | 3460 | - FPSIMD/NEON registers: set to 0 |
| 3461 | 3461 | - SVE registers: set to 0 |
| 3462 | 3462 | - System registers: Reset to their architecturally defined |
| 3463 | − values as for a warm reset to EL1 (resp. SVC) | |
| 3463 | + values as for a warm reset to EL1 (resp. SVC) or EL2 (in the | |
| 3464 | + case of EL2 being enabled). | |
| 3464 | 3465 | |
| 3465 | 3466 | Note that because some registers reflect machine topology, all vcpus |
| 3466 | 3467 | should be created before this ioctl is invoked. |
| @@ -3527,6 +3528,17 @@ Possible features: · 60 unmodified lines | ||
| 3527 | 3528 | - the KVM_REG_ARM64_SVE_VLS pseudo-register is immutable, and can |
| 3528 | 3529 | no longer be written using KVM_SET_ONE_REG. |
| 3529 | 3530 | |
| 3531 | + - KVM_ARM_VCPU_HAS_EL2: Enable Nested Virtualisation support, | |
| 3532 | + booting the guest from EL2 instead of EL1. | |
| 3533 | + Depends on KVM_CAP_ARM_EL2. | |
| 3534 | + The VM is running with HCR_EL2.E2H being RES1 (VHE) unless | |
| 3535 | + KVM_ARM_VCPU_HAS_EL2_E2H0 is also set. | |
| 3536 | + | |
| 3537 | + - KVM_ARM_VCPU_HAS_EL2_E2H0: Restrict Nested Virtualisation | |
| 3538 | + support to HCR_EL2.E2H being RES0 (non-VHE). | |
| 3539 | + Depends on KVM_CAP_ARM_EL2_E2H0. | |
| 3540 | + KVM_ARM_VCPU_HAS_EL2 must also be set. | |
| 3541 | + | |
| 3530 | 3542 | 4.83 KVM_ARM_PREFERRED_TARGET |
| 3531 | 3543 | ----------------------------- |
| 3532 | 3544 | |
| @@ -100,6 +100,8 @@ · 99 unmodified lines | ||
| 100 | 100 | #define ESR_ELx_AET_CE (UL(6) << ESR_ELx_AET_SHIFT) |
| 101 | 101 | |
| 102 | 102 | /* Shared ISS field definitions for Data/Instruction aborts */ |
| 103 | +#define ESR_ELx_VNCR_SHIFT (13) | |
| 104 | +#define ESR_ELx_VNCR (UL(1) << ESR_ELx_VNCR_SHIFT) | |
| 103 | 105 | #define ESR_ELx_SET_SHIFT (11) |
| 104 | 106 | #define ESR_ELx_SET_MASK (UL(3) << ESR_ELx_SET_SHIFT) |
| 105 | 107 | #define ESR_ELx_FnV_SHIFT (10) |
| @@ -48,6 +48,12 @@ enum fixed_addresses { · 47 unmodified lines | ||
| 48 | 48 | FIX_EARLYCON_MEM_BASE, |
| 49 | 49 | FIX_TEXT_POKE0, |
| 50 | 50 | |
| 51 | +#ifdef CONFIG_KVM | |
| 52 | + /* One slot per CPU, mapping the guest's VNCR page at EL2. */ | |
| 53 | + FIX_VNCR_END, | |
| 54 | + FIX_VNCR = FIX_VNCR_END + NR_CPUS, | |
| 55 | +#endif | |
| 56 | + | |
| 51 | 57 | #ifdef CONFIG_ACPI_APEI_GHES |
| 52 | 58 | /* Used for GHES mapping from assorted contexts */ |
| 53 | 59 | FIX_APEI_GHES_IRQ, |
| @@ -39,7 +39,7 @@ · 38 unmodified lines | ||
| 39 | 39 | |
| 40 | 40 | #define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS |
| 41 | 41 | |
| 42 | −#define KVM_VCPU_MAX_FEATURES 7 | |
| 42 | +#define KVM_VCPU_MAX_FEATURES 9 | |
| 43 | 43 | #define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1) |
| 44 | 44 | |
| 45 | 45 | #define KVM_REQ_SLEEP \ |
| @@ -53,6 +53,7 @@ · 7 unmodified lines | ||
| 53 | 53 | #define KVM_REQ_RESYNC_PMU_EL0 KVM_ARCH_REQ(7) |
| 54 | 54 | #define KVM_REQ_NESTED_S2_UNMAP KVM_ARCH_REQ(8) |
| 55 | 55 | #define KVM_REQ_GUEST_HYP_IRQ_PENDING KVM_ARCH_REQ(9) |
| 56 | +#define KVM_REQ_MAP_L1_VNCR_EL2 KVM_ARCH_REQ(10) | |
| 56 | 57 | |
| 57 | 58 | #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ |
| 58 | 59 | KVM_DIRTY_LOG_INITIALLY_SET) |
| @@ -395,6 +396,9 @@ struct kvm_arch { · 336 unmodified lines | ||
| 395 | 396 | /* Masks for VNCR-backed and general EL2 sysregs */ |
| 396 | 397 | struct kvm_sysreg_masks *sysreg_masks; |
| 397 | 398 | |
| 399 | + /* Count the number of VNCR_EL2 currently mapped */ | |
| 400 | + atomic_t vncr_map_count; | |
| 401 | + | |
| 398 | 402 | /* |
| 399 | 403 | * For an untrusted host VM, 'pkvm.handle' is used to lookup |
| 400 | 404 | * the associated pKVM instance in the hypervisor. |
| @@ -573,6 +577,8 @@ enum vcpu_sysreg { · 172 unmodified lines | ||
| 573 | 577 | VNCR(HDFGRTR2_EL2), |
| 574 | 578 | VNCR(HDFGWTR2_EL2), |
| 575 | 579 | |
| 580 | + VNCR(VNCR_EL2), | |
| 581 | + | |
| 576 | 582 | VNCR(CNTVOFF_EL2), |
| 577 | 583 | VNCR(CNTV_CVAL_EL0), |
| 578 | 584 | VNCR(CNTV_CTL_EL0), |
| @@ -696,6 +702,8 @@ struct kvm_host_data { · 117 unmodified lines | ||
| 696 | 702 | #define KVM_HOST_DATA_FLAG_HAS_TRBE 1 |
| 697 | 703 | #define KVM_HOST_DATA_FLAG_TRBE_ENABLED 4 |
| 698 | 704 | #define KVM_HOST_DATA_FLAG_EL1_TRACING_CONFIGURED 5 |
| 705 | +#define KVM_HOST_DATA_FLAG_VCPU_IN_HYP_CONTEXT 6 | |
| 706 | +#define KVM_HOST_DATA_FLAG_L1_VNCR_MAPPED 7 | |
| 699 | 707 | unsigned long flags; |
| 700 | 708 | |
| 701 | 709 | struct kvm_cpu_context host_ctxt; |
| @@ -772,6 +780,8 @@ struct vcpu_reset_state { · 70 unmodified lines | ||
| 772 | 780 | bool reset; |
| 773 | 781 | }; |
| 774 | 782 | |
| 783 | +struct vncr_tlb; | |
| 784 | + | |
| 775 | 785 | struct kvm_vcpu_arch { |
| 776 | 786 | struct kvm_cpu_context ctxt; |
| 777 | 787 | |
| @@ -866,6 +876,9 @@ struct kvm_vcpu_arch { · 88 unmodified lines | ||
| 866 | 876 | |
| 867 | 877 | /* Per-vcpu CCSIDR override or NULL */ |
| 868 | 878 | u32 *ccsidr; |
| 879 | + | |
| 880 | + /* Per-vcpu TLB for VNCR_EL2 -- NULL when !NV */ | |
| 881 | + struct vncr_tlb *vncr_tlb; | |
| 869 | 882 | }; |
| 870 | 883 | |
| 871 | 884 | /* |
| @@ -231,6 +231,38 @@ static inline u64 kvm_encode_nested_level(struct kvm_s2_trans *trans) · 230 unmodified lines | ||
| 231 | 231 | shift; \ |
| 232 | 232 | }) |
| 233 | 233 | |
| 234 | +static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid) | |
| 235 | +{ | |
| 236 | + u64 base, tg, num, scale; | |
| 237 | + int shift; | |
| 238 | + | |
| 239 | + tg = FIELD_GET(GENMASK(47, 46), val); | |
| 240 | + | |
| 241 | + switch(tg) { | |
| 242 | + case 1: | |
| 243 | + shift = 12; | |
| 244 | + break; | |
| 245 | + case 2: | |
| 246 | + shift = 14; | |
| 247 | + break; | |
| 248 | + case 3: | |
| 249 | + default: /* IMPDEF: handle tg==0 as 64k */ | |
| 250 | + shift = 16; | |
| 251 | + break; | |
| 252 | + } | |
| 253 | + | |
| 254 | + base = (val & GENMASK(36, 0)) << shift; | |
| 255 | + | |
| 256 | + if (asid) | |
| 257 | + *asid = FIELD_GET(TLBIR_ASID_MASK, val); | |
| 258 | + | |
| 259 | + scale = FIELD_GET(GENMASK(45, 44), val); | |
| 260 | + num = FIELD_GET(GENMASK(43, 39), val); | |
| 261 | + *range = __TLBI_RANGE_PAGES(num, scale) << shift; | |
| 262 | + | |
| 263 | + return base; | |
| 264 | +} | |
| 265 | + | |
| 234 | 266 | static inline unsigned int ps_to_output_size(unsigned int ps) |
| 235 | 267 | { |
| 236 | 268 | switch (ps) { |
| @@ -245,4 +277,72 @@ static inline unsigned int ps_to_output_size(unsigned int ps) · 8 unmodified lines | ||
| 245 | 277 | } |
| 246 | 278 | } |
| 247 | 279 | |
| 280 | +enum trans_regime { | |
| 281 | + TR_EL10, | |
| 282 | + TR_EL20, | |
| 283 | + TR_EL2, | |
| 284 | +}; | |
| 285 | + | |
| 286 | +struct s1_walk_info { | |
| 287 | + u64 baddr; | |
| 288 | + enum trans_regime regime; | |
| 289 | + unsigned int max_oa_bits; | |
| 290 | + unsigned int pgshift; | |
| 291 | + unsigned int txsz; | |
| 292 | + int sl; | |
| 293 | + bool as_el0; | |
| 294 | + bool hpd; | |
| 295 | + bool e0poe; | |
| 296 | + bool poe; | |
| 297 | + bool pan; | |
| 298 | + bool be; | |
| 299 | + bool s2; | |
| 300 | +}; | |
| 301 | + | |
| 302 | +struct s1_walk_result { | |
| 303 | + union { | |
| 304 | + struct { | |
| 305 | + u64 desc; | |
| 306 | + u64 pa; | |
| 307 | + s8 level; | |
| 308 | + u8 APTable; | |
| 309 | + bool nG; | |
| 310 | + u16 asid; | |
| 311 | + bool UXNTable; | |
| 312 | + bool PXNTable; | |
| 313 | + bool uwxn; | |
| 314 | + bool uov; | |
| 315 | + bool ur; | |
| 316 | + bool uw; | |
| 317 | + bool ux; | |
| 318 | + bool pwxn; | |
| 319 | + bool pov; | |
| 320 | + bool pr; | |
| 321 | + bool pw; | |
| 322 | + bool px; | |
| 323 | + }; | |
| 324 | + struct { | |
| 325 | + u8 fst; | |
| 326 | + bool ptw; | |
| 327 | + bool s2; | |
| 328 | + }; | |
| 329 | + }; | |
| 330 | + bool failed; | |
| 331 | +}; | |
| 332 | + | |
| 333 | +int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, | |
| 334 | + struct s1_walk_result *wr, u64 va); | |
| 335 | + | |
| 336 | +/* VNCR management */ | |
| 337 | +int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu); | |
| 338 | +int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu); | |
| 339 | +void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val); | |
| 340 | + | |
| 341 | +#define vncr_fixmap(c) \ | |
| 342 | + ({ \ | |
| 343 | + u32 __c = (c); \ | |
| 344 | + BUG_ON(__c >= NR_CPUS); \ | |
| 345 | + (FIX_VNCR - __c); \ | |
| 346 | + }) | |
| 347 | + | |
| 248 | 348 | #endif /* __ARM64_KVM_NESTED_H */ |
| @@ -534,7 +534,7 @@ · 533 unmodified lines | ||
| 534 | 534 | #define SYS_VTTBR_EL2 sys_reg(3, 4, 2, 1, 0) |
| 535 | 535 | #define SYS_VTCR_EL2 sys_reg(3, 4, 2, 1, 2) |
| 536 | 536 | |
| 537 | −#define SYS_VNCR_EL2 sys_reg(3, 4, 2, 2, 0) | |
| 537 | +#define SYS_HAFGRTR_EL2 sys_reg(3, 4, 3, 1, 6) | |
| 538 | 538 | #define SYS_SPSR_EL2 sys_reg(3, 4, 4, 0, 0) |
| 539 | 539 | #define SYS_ELR_EL2 sys_reg(3, 4, 4, 0, 1) |
| 540 | 540 | #define SYS_SP_EL1 sys_reg(3, 4, 4, 1, 0) |
| @@ -368,6 +368,12 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 367 unmodified lines | ||
| 368 | 368 | case KVM_CAP_ARM_EL1_32BIT: |
| 369 | 369 | r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1); |
| 370 | 370 | break; |
| 371 | + case KVM_CAP_ARM_EL2: | |
| 372 | + r = cpus_have_final_cap(ARM64_HAS_NESTED_VIRT); | |
| 373 | + break; | |
| 374 | + case KVM_CAP_ARM_EL2_E2H0: | |
| 375 | + r = cpus_have_final_cap(ARM64_HAS_HCR_NV1); | |
| 376 | + break; | |
| 371 | 377 | case KVM_CAP_GUEST_DEBUG_HW_BPS: |
| 372 | 378 | r = get_num_brps(); |
| 373 | 379 | break; |
| @@ -843,6 +849,10 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) · 469 unmodified lines | ||
| 843 | 849 | return ret; |
| 844 | 850 | |
| 845 | 851 | if (vcpu_has_nv(vcpu)) { |
| 852 | + ret = kvm_vcpu_allocate_vncr_tlb(vcpu); | |
| 853 | + if (ret) | |
| 854 | + return ret; | |
| 855 | + | |
| 846 | 856 | ret = kvm_vgic_vcpu_nv_init(vcpu); |
| 847 | 857 | if (ret) |
| 848 | 858 | return ret; |
| @@ -10,56 +10,6 @@ · 9 unmodified lines | ||
| 10 | 10 | #include <asm/kvm_hyp.h> |
| 11 | 11 | #include <asm/kvm_mmu.h> |
| 12 | 12 | |
| 13 | −enum trans_regime { | |
| 14 | − TR_EL10, | |
| 15 | − TR_EL20, | |
| 16 | − TR_EL2, | |
| 17 | −}; | |
| 18 | − | |
| 19 | −struct s1_walk_info { | |
| 20 | − u64 baddr; | |
| 21 | − enum trans_regime regime; | |
| 22 | − unsigned int max_oa_bits; | |
| 23 | − unsigned int pgshift; | |
| 24 | − unsigned int txsz; | |
| 25 | − int sl; | |
| 26 | − bool hpd; | |
| 27 | − bool e0poe; | |
| 28 | − bool poe; | |
| 29 | − bool pan; | |
| 30 | − bool be; | |
| 31 | − bool s2; | |
| 32 | −}; | |
| 33 | − | |
| 34 | −struct s1_walk_result { | |
| 35 | − union { | |
| 36 | − struct { | |
| 37 | − u64 desc; | |
| 38 | − u64 pa; | |
| 39 | − s8 level; | |
| 40 | − u8 APTable; | |
| 41 | − bool UXNTable; | |
| 42 | − bool PXNTable; | |
| 43 | − bool uwxn; | |
| 44 | − bool uov; | |
| 45 | − bool ur; | |
| 46 | − bool uw; | |
| 47 | − bool ux; | |
| 48 | − bool pwxn; | |
| 49 | − bool pov; | |
| 50 | − bool pr; | |
| 51 | − bool pw; | |
| 52 | − bool px; | |
| 53 | − }; | |
| 54 | − struct { | |
| 55 | − u8 fst; | |
| 56 | − bool ptw; | |
| 57 | − bool s2; | |
| 58 | − }; | |
| 59 | − }; | |
| 60 | − bool failed; | |
| 61 | −}; | |
| 62 | − | |
| 63 | 13 | static void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) |
| 64 | 14 | { |
| 65 | 15 | wr->fst = fst; |
| @@ -145,20 +95,15 @@ static void compute_s1poe(struct kvm_vcpu *vcpu, struct s1_walk_info *wi) · 79 unmodified lines | ||
| 145 | 95 | } |
| 146 | 96 | } |
| 147 | 97 | |
| 148 | −static int setup_s1_walk(struct kvm_vcpu *vcpu, u32 op, struct s1_walk_info *wi, | |
| 98 | +static int setup_s1_walk(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, | |
| 149 | 99 | struct s1_walk_result *wr, u64 va) |
| 150 | 100 | { |
| 151 | 101 | u64 hcr, sctlr, tcr, tg, ps, ia_bits, ttbr; |
| 152 | 102 | unsigned int stride, x; |
| 153 | − bool va55, tbi, lva, as_el0; | |
| 103 | + bool va55, tbi, lva; | |
| 154 | 104 | |
| 155 | 105 | hcr = __vcpu_sys_reg(vcpu, HCR_EL2); |
| 156 | 106 | |
| 157 | − wi->regime = compute_translation_regime(vcpu, op); | |
| 158 | − as_el0 = (op == OP_AT_S1E0R || op == OP_AT_S1E0W); | |
| 159 | − wi->pan = (op == OP_AT_S1E1RP || op == OP_AT_S1E1WP) && | |
| 160 | − (*vcpu_cpsr(vcpu) & PSR_PAN_BIT); | |
| 161 | − | |
| 162 | 107 | va55 = va & BIT(55); |
| 163 | 108 | |
| 164 | 109 | if (wi->regime == TR_EL2 && va55) |
| @@ -319,7 +264,7 @@ static int setup_s1_walk(struct kvm_vcpu *vcpu, u32 op, struct s1_walk_info *wi, · 154 unmodified lines | ||
| 319 | 264 | |
| 320 | 265 | /* R_BNDVG and following statements */ |
| 321 | 266 | if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, E0PD, IMP) && |
| 322 | − as_el0 && (tcr & (va55 ? TCR_E0PD1 : TCR_E0PD0))) | |
| 267 | + wi->as_el0 && (tcr & (va55 ? TCR_E0PD1 : TCR_E0PD0))) | |
| 323 | 268 | goto transfault_l0; |
| 324 | 269 | |
| 325 | 270 | /* AArch64.S1StartLevel() */ |
| @@ -469,6 +414,33 @@ static int walk_s1(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, · 143 unmodified lines | ||
| 469 | 414 | wr->pa = desc & GENMASK(47, va_bottom); |
| 470 | 415 | wr->pa |= va & GENMASK_ULL(va_bottom - 1, 0); |
| 471 | 416 | |
| 417 | + wr->nG = (wi->regime != TR_EL2) && (desc & PTE_NG); | |
| 418 | + if (wr->nG) { | |
| 419 | + u64 asid_ttbr, tcr; | |
| 420 | + | |
| 421 | + switch (wi->regime) { | |
| 422 | + case TR_EL10: | |
| 423 | + tcr = vcpu_read_sys_reg(vcpu, TCR_EL1); | |
| 424 | + asid_ttbr = ((tcr & TCR_A1) ? | |
| 425 | + vcpu_read_sys_reg(vcpu, TTBR1_EL1) : | |
| 426 | + vcpu_read_sys_reg(vcpu, TTBR0_EL1)); | |
| 427 | + break; | |
| 428 | + case TR_EL20: | |
| 429 | + tcr = vcpu_read_sys_reg(vcpu, TCR_EL2); | |
| 430 | + asid_ttbr = ((tcr & TCR_A1) ? | |
| 431 | + vcpu_read_sys_reg(vcpu, TTBR1_EL2) : | |
| 432 | + vcpu_read_sys_reg(vcpu, TTBR0_EL2)); | |
| 433 | + break; | |
| 434 | + default: | |
| 435 | + BUG(); | |
| 436 | + } | |
| 437 | + | |
| 438 | + wr->asid = FIELD_GET(TTBR_ASID_MASK, asid_ttbr); | |
| 439 | + if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) || | |
| 440 | + !(tcr & TCR_ASID16)) | |
| 441 | + wr->asid &= GENMASK(7, 0); | |
| 442 | + } | |
| 443 | + | |
| 472 | 444 | return 0; |
| 473 | 445 | |
| 474 | 446 | addrsz: |
| @@ -1155,7 +1127,12 @@ static u64 handle_at_slow(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) · 680 unmodified lines | ||
| 1155 | 1127 | bool perm_fail = false; |
| 1156 | 1128 | int ret, idx; |
| 1157 | 1129 | |
| 1158 | − ret = setup_s1_walk(vcpu, op, &wi, &wr, vaddr); | |
| 1130 | + wi.regime = compute_translation_regime(vcpu, op); | |
| 1131 | + wi.as_el0 = (op == OP_AT_S1E0R || op == OP_AT_S1E0W); | |
| 1132 | + wi.pan = (op == OP_AT_S1E1RP || op == OP_AT_S1E1WP) && | |
| 1133 | + (*vcpu_cpsr(vcpu) & PSR_PAN_BIT); | |
| 1134 | + | |
| 1135 | + ret = setup_s1_walk(vcpu, &wi, &wr, vaddr); | |
| 1159 | 1136 | if (ret) |
| 1160 | 1137 | goto compute_par; |
| 1161 | 1138 | |
| @@ -1457,3 +1434,31 @@ void __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) · 295 unmodified lines | ||
| 1457 | 1434 | par = compute_par_s12(vcpu, par, &out); |
| 1458 | 1435 | vcpu_write_sys_reg(vcpu, par, PAR_EL1); |
| 1459 | 1436 | } |
| 1437 | + | |
| 1438 | +/* | |
| 1439 | + * Translate a VA for a given EL in a given translation regime, with | |
| 1440 | + * or without PAN. This requires wi->{regime, as_el0, pan} to be | |
| 1441 | + * set. The rest of the wi and wr should be 0-initialised. | |
| 1442 | + */ | |
| 1443 | +int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, | |
| 1444 | + struct s1_walk_result *wr, u64 va) | |
| 1445 | +{ | |
| 1446 | + int ret; | |
| 1447 | + | |
| 1448 | + ret = setup_s1_walk(vcpu, wi, wr, va); | |
| 1449 | + if (ret) | |
| 1450 | + return ret; | |
| 1451 | + | |
| 1452 | + if (wr->level == S1_MMU_DISABLED) { | |
| 1453 | + wr->ur = wr->uw = wr->ux = true; | |
| 1454 | + wr->pr = wr->pw = wr->px = true; | |
| 1455 | + } else { | |
| 1456 | + ret = walk_s1(vcpu, wi, wr, va); | |
| 1457 | + if (ret) | |
| 1458 | + return ret; | |
| 1459 | + | |
| 1460 | + compute_s1_permissions(vcpu, wi, wr); | |
| 1461 | + } | |
| 1462 | + | |
| 1463 | + return 0; | |
| 1464 | +} | |
| @@ -2662,13 +2662,6 @@ void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu) · 2661 unmodified lines | ||
| 2662 | 2662 | { |
| 2663 | 2663 | u64 spsr, elr, esr; |
| 2664 | 2664 | |
| 2665 | − /* | |
| 2666 | − * Forward this trap to the virtual EL2 if the virtual | |
| 2667 | − * HCR_EL2.NV bit is set and this is coming from !EL2. | |
| 2668 | − */ | |
| 2669 | − if (forward_hcr_traps(vcpu, HCR_NV)) | |
| 2670 | − return; | |
| 2671 | − | |
| 2672 | 2665 | spsr = vcpu_read_sys_reg(vcpu, SPSR_EL2); |
| 2673 | 2666 | spsr = kvm_check_illegal_exception_return(vcpu, spsr); |
| 2674 | 2667 | |
| @@ -394,6 +394,7 @@ static exit_handle_fn arm_exit_handlers[] = { · 393 unmodified lines | ||
| 394 | 394 | [ESR_ELx_EC_ERET] = kvm_handle_eret, |
| 395 | 395 | [ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort, |
| 396 | 396 | [ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort, |
| 397 | + [ESR_ELx_EC_DABT_CUR] = kvm_handle_vncr_abort, | |
| 397 | 398 | [ESR_ELx_EC_SOFTSTP_LOW]= kvm_handle_guest_debug, |
| 398 | 399 | [ESR_ELx_EC_WATCHPT_LOW]= kvm_handle_guest_debug, |
| 399 | 400 | [ESR_ELx_EC_BREAKPT_LOW]= kvm_handle_guest_debug, |
| @@ -48,21 +48,46 @@ DEFINE_PER_CPU(unsigned long, kvm_hyp_vector); · 47 unmodified lines | ||
| 48 | 48 | |
| 49 | 49 | static u64 __compute_hcr(struct kvm_vcpu *vcpu) |
| 50 | 50 | { |
| 51 | + u64 guest_hcr = __vcpu_sys_reg(vcpu, HCR_EL2); | |
| 51 | 52 | u64 hcr = vcpu->arch.hcr_el2; |
| 52 | 53 | |
| 53 | 54 | if (!vcpu_has_nv(vcpu)) |
| 54 | 55 | return hcr; |
| 55 | 56 | |
| 57 | + /* | |
| 58 | + * We rely on the invariant that a vcpu entered from HYP | |
| 59 | + * context must also exit in the same context, as only an ERET | |
| 60 | + * instruction can kick us out of it, and we obviously trap | |
| 61 | + * that sucker. PSTATE.M will get fixed-up on exit. | |
| 62 | + */ | |
| 56 | 63 | if (is_hyp_ctxt(vcpu)) { |
| 64 | + host_data_set_flag(VCPU_IN_HYP_CONTEXT); | |
| 65 | + | |
| 57 | 66 | hcr |= HCR_NV | HCR_NV2 | HCR_AT | HCR_TTLB; |
| 58 | 67 | |
| 59 | 68 | if (!vcpu_el2_e2h_is_set(vcpu)) |
| 60 | 69 | hcr |= HCR_NV1; |
| 61 | 70 | |
| 62 | 71 | write_sysreg_s(vcpu->arch.ctxt.vncr_array, SYS_VNCR_EL2); |
| 72 | + } else { | |
| 73 | + host_data_clear_flag(VCPU_IN_HYP_CONTEXT); | |
| 74 | + | |
| 75 | + if (guest_hcr & HCR_NV) { | |
| 76 | + u64 va = __fix_to_virt(vncr_fixmap(smp_processor_id())); | |
| 77 | + | |
| 78 | + /* Inherit the low bits from the actual register */ | |
| 79 | + va |= __vcpu_sys_reg(vcpu, VNCR_EL2) & GENMASK(PAGE_SHIFT - 1, 0); | |
| 80 | + write_sysreg_s(va, SYS_VNCR_EL2); | |
| 81 | + | |
| 82 | + /* Force NV2 in case the guest is forgetful... */ | |
| 83 | + guest_hcr |= HCR_NV2; | |
| 84 | + } | |
| 63 | 85 | } |
| 64 | 86 | |
| 65 | − return hcr | (__vcpu_sys_reg(vcpu, HCR_EL2) & ~NV_HCR_GUEST_EXCLUDE); | |
| 87 | + BUG_ON(host_data_test_flag(VCPU_IN_HYP_CONTEXT) && | |
| 88 | + host_data_test_flag(L1_VNCR_MAPPED)); | |
| 89 | + | |
| 90 | + return hcr | (guest_hcr & ~NV_HCR_GUEST_EXCLUDE); | |
| 66 | 91 | } |
| 67 | 92 | |
| 68 | 93 | static void __activate_cptr_traps(struct kvm_vcpu *vcpu) |
| @@ -459,6 +484,14 @@ static bool kvm_hyp_handle_tlbi_el2(struct kvm_vcpu *vcpu, u64 *exit_code) · 390 unmodified lines | ||
| 459 | 484 | if (ret) |
| 460 | 485 | return false; |
| 461 | 486 | |
| 487 | + /* | |
| 488 | + * If we have to check for any VNCR mapping being invalidated, | |
| 489 | + * go back to the slow path for further processing. | |
| 490 | + */ | |
| 491 | + if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu) && | |
| 492 | + atomic_read(&vcpu->kvm->arch.vncr_map_count)) | |
| 493 | + return false; | |
| 494 | + | |
| 462 | 495 | __kvm_skip_instr(vcpu); |
| 463 | 496 | |
| 464 | 497 | return true; |
| @@ -568,9 +601,12 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) · 103 unmodified lines | ||
| 568 | 601 | |
| 569 | 602 | /* |
| 570 | 603 | * If we were in HYP context on entry, adjust the PSTATE view |
| 571 | − * so that the usual helpers work correctly. | |
| 604 | + * so that the usual helpers work correctly. This enforces our | |
| 605 | + * invariant that the guest's HYP context status is preserved | |
| 606 | + * across a run. | |
| 572 | 607 | */ |
| 573 | − if (vcpu_has_nv(vcpu) && (read_sysreg(hcr_el2) & HCR_NV)) { | |
| 608 | + if (vcpu_has_nv(vcpu) && | |
| 609 | + unlikely(host_data_test_flag(VCPU_IN_HYP_CONTEXT))) { | |
| 574 | 610 | u64 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT); |
| 575 | 611 | |
| 576 | 612 | switch (mode) { |
| @@ -586,6 +622,10 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) · 9 unmodified lines | ||
| 586 | 622 | *vcpu_cpsr(vcpu) |= mode; |
| 587 | 623 | } |
| 588 | 624 | |
| 625 | + /* Apply extreme paranoia! */ | |
| 626 | + BUG_ON(vcpu_has_nv(vcpu) && | |
| 627 | + !!host_data_test_flag(VCPU_IN_HYP_CONTEXT) != is_hyp_ctxt(vcpu)); | |
| 628 | + | |
| 589 | 629 | return __fixup_guest_exit(vcpu, exit_code, hyp_exit_handlers); |
| 590 | 630 | } |
| 591 | 631 | |
| @@ -8,6 +8,7 @@ · 7 unmodified lines | ||
| 8 | 8 | #include <linux/kvm.h> |
| 9 | 9 | #include <linux/kvm_host.h> |
| 10 | 10 | |
| 11 | +#include <asm/fixmap.h> | |
| 11 | 12 | #include <asm/kvm_arm.h> |
| 12 | 13 | #include <asm/kvm_emulate.h> |
| 13 | 14 | #include <asm/kvm_mmu.h> |
| @@ -16,6 +17,24 @@ · 2 unmodified lines | ||
| 16 | 17 | |
| 17 | 18 | #include "sys_regs.h" |
| 18 | 19 | |
| 20 | +struct vncr_tlb { | |
| 21 | + /* The guest's VNCR_EL2 */ | |
| 22 | + u64 gva; | |
| 23 | + struct s1_walk_info wi; | |
| 24 | + struct s1_walk_result wr; | |
| 25 | + | |
| 26 | + u64 hpa; | |
| 27 | + | |
| 28 | + /* -1 when not mapped on a CPU */ | |
| 29 | + int cpu; | |
| 30 | + | |
| 31 | + /* | |
| 32 | + * true if the TLB is valid. Can only be changed with the | |
| 33 | + * mmu_lock held. | |
| 34 | + */ | |
| 35 | + bool valid; | |
| 36 | +}; | |
| 37 | + | |
| 19 | 38 | /* |
| 20 | 39 | * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between |
| 21 | 40 | * memory usage and potential number of different sets of S2 PTs in |
| @@ -28,6 +47,7 @@ void kvm_init_nested(struct kvm *kvm) · 6 unmodified lines | ||
| 28 | 47 | { |
| 29 | 48 | kvm->arch.nested_mmus = NULL; |
| 30 | 49 | kvm->arch.nested_mmus_size = 0; |
| 50 | + atomic_set(&kvm->arch.vncr_map_count, 0); | |
| 31 | 51 | } |
| 32 | 52 | |
| 33 | 53 | static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) |
| @@ -55,6 +75,13 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) · 21 unmodified lines | ||
| 55 | 75 | !cpus_have_final_cap(ARM64_HAS_HCR_NV1)) |
| 56 | 76 | return -EINVAL; |
| 57 | 77 | |
| 78 | + if (!vcpu->arch.ctxt.vncr_array) | |
| 79 | + vcpu->arch.ctxt.vncr_array = (u64 *)__get_free_page(GFP_KERNEL_ACCOUNT | | |
| 80 | + __GFP_ZERO); | |
| 81 | + | |
| 82 | + if (!vcpu->arch.ctxt.vncr_array) | |
| 83 | + return -ENOMEM; | |
| 84 | + | |
| 58 | 85 | /* |
| 59 | 86 | * Let's treat memory allocation failures as benign: If we fail to |
| 60 | 87 | * allocate anything, return an error and keep the allocated array |
| @@ -85,6 +112,9 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) · 24 unmodified lines | ||
| 85 | 112 | for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++) |
| 86 | 113 | kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]); |
| 87 | 114 | |
| 115 | + free_page((unsigned long)vcpu->arch.ctxt.vncr_array); | |
| 116 | + vcpu->arch.ctxt.vncr_array = NULL; | |
| 117 | + | |
| 88 | 118 | return ret; |
| 89 | 119 | } |
| 90 | 120 | |
| @@ -405,6 +435,30 @@ static unsigned int ttl_to_size(u8 ttl) · 314 unmodified lines | ||
| 405 | 435 | return max_size; |
| 406 | 436 | } |
| 407 | 437 | |
| 438 | +static u8 pgshift_level_to_ttl(u16 shift, u8 level) | |
| 439 | +{ | |
| 440 | + u8 ttl; | |
| 441 | + | |
| 442 | + switch(shift) { | |
| 443 | + case 12: | |
| 444 | + ttl = TLBI_TTL_TG_4K; | |
| 445 | + break; | |
| 446 | + case 14: | |
| 447 | + ttl = TLBI_TTL_TG_16K; | |
| 448 | + break; | |
| 449 | + case 16: | |
| 450 | + ttl = TLBI_TTL_TG_64K; | |
| 451 | + break; | |
| 452 | + default: | |
| 453 | + BUG(); | |
| 454 | + } | |
| 455 | + | |
| 456 | + ttl <<= 2; | |
| 457 | + ttl |= level & 3; | |
| 458 | + | |
| 459 | + return ttl; | |
| 460 | +} | |
| 461 | + | |
| 408 | 462 | /* |
| 409 | 463 | * Compute the equivalent of the TTL field by parsing the shadow PT. The |
| 410 | 464 | * granule size is extracted from the cached VTCR_EL2.TG0 while the level is |
| @@ -676,23 +730,36 @@ void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu) · 265 unmodified lines | ||
| 676 | 730 | void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu) |
| 677 | 731 | { |
| 678 | 732 | /* |
| 679 | − * The vCPU kept its reference on the MMU after the last put, keep | |
| 680 | − * rolling with it. | |
| 733 | + * If the vCPU kept its reference on the MMU after the last put, | |
| 734 | + * keep rolling with it. | |
| 681 | 735 | */ |
| 682 | − if (vcpu->arch.hw_mmu) | |
| 683 | − return; | |
| 684 | − | |
| 685 | 736 | if (is_hyp_ctxt(vcpu)) { |
| 686 | − vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; | |
| 737 | + if (!vcpu->arch.hw_mmu) | |
| 738 | + vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu; | |
| 687 | 739 | } else { |
| 688 | − write_lock(&vcpu->kvm->mmu_lock); | |
| 689 | − vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu); | |
| 690 | − write_unlock(&vcpu->kvm->mmu_lock); | |
| 740 | + if (!vcpu->arch.hw_mmu) { | |
| 741 | + scoped_guard(write_lock, &vcpu->kvm->mmu_lock) | |
| 742 | + vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu); | |
| 743 | + } | |
| 744 | + | |
| 745 | + if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV) | |
| 746 | + kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); | |
| 691 | 747 | } |
| 692 | 748 | } |
| 693 | 749 | |
| 694 | 750 | void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu) |
| 695 | 751 | { |
| 752 | + /* Unconditionally drop the VNCR mapping if we have one */ | |
| 753 | + if (host_data_test_flag(L1_VNCR_MAPPED)) { | |
| 754 | + BUG_ON(vcpu->arch.vncr_tlb->cpu != smp_processor_id()); | |
| 755 | + BUG_ON(is_hyp_ctxt(vcpu)); | |
| 756 | + | |
| 757 | + clear_fixmap(vncr_fixmap(vcpu->arch.vncr_tlb->cpu)); | |
| 758 | + vcpu->arch.vncr_tlb->cpu = -1; | |
| 759 | + host_data_clear_flag(L1_VNCR_MAPPED); | |
| 760 | + atomic_dec(&vcpu->kvm->arch.vncr_map_count); | |
| 761 | + } | |
| 762 | + | |
| 696 | 763 | /* |
| 697 | 764 | * Keep a reference on the associated stage-2 MMU if the vCPU is |
| 698 | 765 | * scheduling out and not in WFI emulation, suggesting it is likely to |
| @@ -743,6 +810,245 @@ int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2) · 44 unmodified lines | ||
| 743 | 810 | return kvm_inject_nested_sync(vcpu, esr_el2); |
| 744 | 811 | } |
| 745 | 812 | |
| 813 | +static void invalidate_vncr(struct vncr_tlb *vt) | |
| 814 | +{ | |
| 815 | + vt->valid = false; | |
| 816 | + if (vt->cpu != -1) | |
| 817 | + clear_fixmap(vncr_fixmap(vt->cpu)); | |
| 818 | +} | |
| 819 | + | |
| 820 | +static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) | |
| 821 | +{ | |
| 822 | + struct kvm_vcpu *vcpu; | |
| 823 | + unsigned long i; | |
| 824 | + | |
| 825 | + lockdep_assert_held_write(&kvm->mmu_lock); | |
| 826 | + | |
| 827 | + if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) | |
| 828 | + return; | |
| 829 | + | |
| 830 | + kvm_for_each_vcpu(i, vcpu, kvm) { | |
| 831 | + struct vncr_tlb *vt = vcpu->arch.vncr_tlb; | |
| 832 | + u64 ipa_start, ipa_end, ipa_size; | |
| 833 | + | |
| 834 | + /* | |
| 835 | + * Careful here: We end-up here from an MMU notifier, | |
| 836 | + * and this can race against a vcpu not being onlined | |
| 837 | + * yet, without the pseudo-TLB being allocated. | |
| 838 | + * | |
| 839 | + * Skip those, as they obviously don't participate in | |
| 840 | + * the invalidation at this stage. | |
| 841 | + */ | |
| 842 | + if (!vt) | |
| 843 | + continue; | |
| 844 | + | |
| 845 | + if (!vt->valid) | |
| 846 | + continue; | |
| 847 | + | |
| 848 | + ipa_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, | |
| 849 | + vt->wr.level)); | |
| 850 | + ipa_start = vt->wr.pa & (ipa_size - 1); | |
| 851 | + ipa_end = ipa_start + ipa_size; | |
| 852 | + | |
| 853 | + if (ipa_end <= start || ipa_start >= end) | |
| 854 | + continue; | |
| 855 | + | |
| 856 | + invalidate_vncr(vt); | |
| 857 | + } | |
| 858 | +} | |
| 859 | + | |
| 860 | +struct s1e2_tlbi_scope { | |
| 861 | + enum { | |
| 862 | + TLBI_ALL, | |
| 863 | + TLBI_VA, | |
| 864 | + TLBI_VAA, | |
| 865 | + TLBI_ASID, | |
| 866 | + } type; | |
| 867 | + | |
| 868 | + u16 asid; | |
| 869 | + u64 va; | |
| 870 | + u64 size; | |
| 871 | +}; | |
| 872 | + | |
| 873 | +static void invalidate_vncr_va(struct kvm *kvm, | |
| 874 | + struct s1e2_tlbi_scope *scope) | |
| 875 | +{ | |
| 876 | + struct kvm_vcpu *vcpu; | |
| 877 | + unsigned long i; | |
| 878 | + | |
| 879 | + lockdep_assert_held_write(&kvm->mmu_lock); | |
| 880 | + | |
| 881 | + kvm_for_each_vcpu(i, vcpu, kvm) { | |
| 882 | + struct vncr_tlb *vt = vcpu->arch.vncr_tlb; | |
| 883 | + u64 va_start, va_end, va_size; | |
| 884 | + | |
| 885 | + if (!vt->valid) | |
| 886 | + continue; | |
| 887 | + | |
| 888 | + va_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, | |
| 889 | + vt->wr.level)); | |
| 890 | + va_start = vt->gva & (va_size - 1); | |
| 891 | + va_end = va_start + va_size; | |
| 892 | + | |
| 893 | + switch (scope->type) { | |
| 894 | + case TLBI_ALL: | |
| 895 | + break; | |
| 896 | + | |
| 897 | + case TLBI_VA: | |
| 898 | + if (va_end <= scope->va || | |
| 899 | + va_start >= (scope->va + scope->size)) | |
| 900 | + continue; | |
| 901 | + if (vt->wr.nG && vt->wr.asid != scope->asid) | |
| 902 | + continue; | |
| 903 | + break; | |
| 904 | + | |
| 905 | + case TLBI_VAA: | |
| 906 | + if (va_end <= scope->va || | |
| 907 | + va_start >= (scope->va + scope->size)) | |
| 908 | + continue; | |
| 909 | + break; | |
| 910 | + | |
| 911 | + case TLBI_ASID: | |
| 912 | + if (!vt->wr.nG || vt->wr.asid != scope->asid) | |
| 913 | + continue; | |
| 914 | + break; | |
| 915 | + } | |
| 916 | + | |
| 917 | + invalidate_vncr(vt); | |
| 918 | + } | |
| 919 | +} | |
| 920 | + | |
| 921 | +static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val, | |
| 922 | + struct s1e2_tlbi_scope *scope) | |
| 923 | +{ | |
| 924 | + switch (inst) { | |
| 925 | + case OP_TLBI_ALLE2: | |
| 926 | + case OP_TLBI_ALLE2IS: | |
| 927 | + case OP_TLBI_ALLE2OS: | |
| 928 | + case OP_TLBI_VMALLE1: | |
| 929 | + case OP_TLBI_VMALLE1IS: | |
| 930 | + case OP_TLBI_VMALLE1OS: | |
| 931 | + case OP_TLBI_ALLE2NXS: | |
| 932 | + case OP_TLBI_ALLE2ISNXS: | |
| 933 | + case OP_TLBI_ALLE2OSNXS: | |
| 934 | + case OP_TLBI_VMALLE1NXS: | |
| 935 | + case OP_TLBI_VMALLE1ISNXS: | |
| 936 | + case OP_TLBI_VMALLE1OSNXS: | |
| 937 | + scope->type = TLBI_ALL; | |
| 938 | + break; | |
| 939 | + case OP_TLBI_VAE2: | |
| 940 | + case OP_TLBI_VAE2IS: | |
| 941 | + case OP_TLBI_VAE2OS: | |
| 942 | + case OP_TLBI_VAE1: | |
| 943 | + case OP_TLBI_VAE1IS: | |
| 944 | + case OP_TLBI_VAE1OS: | |
| 945 | + case OP_TLBI_VAE2NXS: | |
| 946 | + case OP_TLBI_VAE2ISNXS: | |
| 947 | + case OP_TLBI_VAE2OSNXS: | |
| 948 | + case OP_TLBI_VAE1NXS: | |
| 949 | + case OP_TLBI_VAE1ISNXS: | |
| 950 | + case OP_TLBI_VAE1OSNXS: | |
| 951 | + case OP_TLBI_VALE2: | |
| 952 | + case OP_TLBI_VALE2IS: | |
| 953 | + case OP_TLBI_VALE2OS: | |
| 954 | + case OP_TLBI_VALE1: | |
| 955 | + case OP_TLBI_VALE1IS: | |
| 956 | + case OP_TLBI_VALE1OS: | |
| 957 | + case OP_TLBI_VALE2NXS: | |
| 958 | + case OP_TLBI_VALE2ISNXS: | |
| 959 | + case OP_TLBI_VALE2OSNXS: | |
| 960 | + case OP_TLBI_VALE1NXS: | |
| 961 | + case OP_TLBI_VALE1ISNXS: | |
| 962 | + case OP_TLBI_VALE1OSNXS: | |
| 963 | + scope->type = TLBI_VA; | |
| 964 | + scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); | |
| 965 | + if (!scope->size) | |
| 966 | + scope->size = SZ_1G; | |
| 967 | + scope->va = (val << 12) & ~(scope->size - 1); | |
| 968 | + scope->asid = FIELD_GET(TLBIR_ASID_MASK, val); | |
| 969 | + break; | |
| 970 | + case OP_TLBI_ASIDE1: | |
| 971 | + case OP_TLBI_ASIDE1IS: | |
| 972 | + case OP_TLBI_ASIDE1OS: | |
| 973 | + case OP_TLBI_ASIDE1NXS: | |
| 974 | + case OP_TLBI_ASIDE1ISNXS: | |
| 975 | + case OP_TLBI_ASIDE1OSNXS: | |
| 976 | + scope->type = TLBI_ASID; | |
| 977 | + scope->asid = FIELD_GET(TLBIR_ASID_MASK, val); | |
| 978 | + break; | |
| 979 | + case OP_TLBI_VAAE1: | |
| 980 | + case OP_TLBI_VAAE1IS: | |
| 981 | + case OP_TLBI_VAAE1OS: | |
| 982 | + case OP_TLBI_VAAE1NXS: | |
| 983 | + case OP_TLBI_VAAE1ISNXS: | |
| 984 | + case OP_TLBI_VAAE1OSNXS: | |
| 985 | + case OP_TLBI_VAALE1: | |
| 986 | + case OP_TLBI_VAALE1IS: | |
| 987 | + case OP_TLBI_VAALE1OS: | |
| 988 | + case OP_TLBI_VAALE1NXS: | |
| 989 | + case OP_TLBI_VAALE1ISNXS: | |
| 990 | + case OP_TLBI_VAALE1OSNXS: | |
| 991 | + scope->type = TLBI_VAA; | |
| 992 | + scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); | |
| 993 | + if (!scope->size) | |
| 994 | + scope->size = SZ_1G; | |
| 995 | + scope->va = (val << 12) & ~(scope->size - 1); | |
| 996 | + break; | |
| 997 | + case OP_TLBI_RVAE2: | |
| 998 | + case OP_TLBI_RVAE2IS: | |
| 999 | + case OP_TLBI_RVAE2OS: | |
| 1000 | + case OP_TLBI_RVAE1: | |
| 1001 | + case OP_TLBI_RVAE1IS: | |
| 1002 | + case OP_TLBI_RVAE1OS: | |
| 1003 | + case OP_TLBI_RVAE2NXS: | |
| 1004 | + case OP_TLBI_RVAE2ISNXS: | |
| 1005 | + case OP_TLBI_RVAE2OSNXS: | |
| 1006 | + case OP_TLBI_RVAE1NXS: | |
| 1007 | + case OP_TLBI_RVAE1ISNXS: | |
| 1008 | + case OP_TLBI_RVAE1OSNXS: | |
| 1009 | + case OP_TLBI_RVALE2: | |
| 1010 | + case OP_TLBI_RVALE2IS: | |
| 1011 | + case OP_TLBI_RVALE2OS: | |
| 1012 | + case OP_TLBI_RVALE1: | |
| 1013 | + case OP_TLBI_RVALE1IS: | |
| 1014 | + case OP_TLBI_RVALE1OS: | |
| 1015 | + case OP_TLBI_RVALE2NXS: | |
| 1016 | + case OP_TLBI_RVALE2ISNXS: | |
| 1017 | + case OP_TLBI_RVALE2OSNXS: | |
| 1018 | + case OP_TLBI_RVALE1NXS: | |
| 1019 | + case OP_TLBI_RVALE1ISNXS: | |
| 1020 | + case OP_TLBI_RVALE1OSNXS: | |
| 1021 | + scope->type = TLBI_VA; | |
| 1022 | + scope->va = decode_range_tlbi(val, &scope->size, &scope->asid); | |
| 1023 | + break; | |
| 1024 | + case OP_TLBI_RVAAE1: | |
| 1025 | + case OP_TLBI_RVAAE1IS: | |
| 1026 | + case OP_TLBI_RVAAE1OS: | |
| 1027 | + case OP_TLBI_RVAAE1NXS: | |
| 1028 | + case OP_TLBI_RVAAE1ISNXS: | |
| 1029 | + case OP_TLBI_RVAAE1OSNXS: | |
| 1030 | + case OP_TLBI_RVAALE1: | |
| 1031 | + case OP_TLBI_RVAALE1IS: | |
| 1032 | + case OP_TLBI_RVAALE1OS: | |
| 1033 | + case OP_TLBI_RVAALE1NXS: | |
| 1034 | + case OP_TLBI_RVAALE1ISNXS: | |
| 1035 | + case OP_TLBI_RVAALE1OSNXS: | |
| 1036 | + scope->type = TLBI_VAA; | |
| 1037 | + scope->va = decode_range_tlbi(val, &scope->size, NULL); | |
| 1038 | + break; | |
| 1039 | + } | |
| 1040 | +} | |
| 1041 | + | |
| 1042 | +void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val) | |
| 1043 | +{ | |
| 1044 | + struct s1e2_tlbi_scope scope = {}; | |
| 1045 | + | |
| 1046 | + compute_s1_tlbi_range(vcpu, inst, val, &scope); | |
| 1047 | + | |
| 1048 | + guard(write_lock)(&vcpu->kvm->mmu_lock); | |
| 1049 | + invalidate_vncr_va(vcpu->kvm, &scope); | |
| 1050 | +} | |
| 1051 | + | |
| 746 | 1052 | void kvm_nested_s2_wp(struct kvm *kvm) |
| 747 | 1053 | { |
| 748 | 1054 | int i; |
| @@ -755,6 +1061,8 @@ void kvm_nested_s2_wp(struct kvm *kvm) · 6 unmodified lines | ||
| 755 | 1061 | if (kvm_s2_mmu_valid(mmu)) |
| 756 | 1062 | kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu)); |
| 757 | 1063 | } |
| 1064 | + | |
| 1065 | + kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits)); | |
| 758 | 1066 | } |
| 759 | 1067 | |
| 760 | 1068 | void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) |
| @@ -769,6 +1077,8 @@ void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) · 8 unmodified lines | ||
| 769 | 1077 | if (kvm_s2_mmu_valid(mmu)) |
| 770 | 1078 | kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block); |
| 771 | 1079 | } |
| 1080 | + | |
| 1081 | + kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits)); | |
| 772 | 1082 | } |
| 773 | 1083 | |
| 774 | 1084 | void kvm_nested_s2_flush(struct kvm *kvm) |
| @@ -801,6 +1111,295 @@ void kvm_arch_flush_shadow_all(struct kvm *kvm) · 26 unmodified lines | ||
| 801 | 1111 | kvm_uninit_stage2_mmu(kvm); |
| 802 | 1112 | } |
| 803 | 1113 | |
| 1114 | +/* | |
| 1115 | + * Dealing with VNCR_EL2 exposed by the *guest* is a complicated matter: | |
| 1116 | + * | |
| 1117 | + * - We introduce an internal representation of a vcpu-private TLB, | |
| 1118 | + * representing the mapping between the guest VA contained in VNCR_EL2, | |
| 1119 | + * the IPA the guest's EL2 PTs point to, and the actual PA this lives at. | |
| 1120 | + * | |
| 1121 | + * - On translation fault from a nested VNCR access, we create such a TLB. | |
| 1122 | + * If there is no mapping to describe, the guest inherits the fault. | |
| 1123 | + * Crucially, no actual mapping is done at this stage. | |
| 1124 | + * | |
| 1125 | + * - On vcpu_load() in a non-HYP context with HCR_EL2.NV==1, if the above | |
| 1126 | + * TLB exists, we map it in the fixmap for this CPU, and run with it. We | |
| 1127 | + * have to respect the permissions dictated by the guest, but not the | |
| 1128 | + * memory type (FWB is a must). | |
| 1129 | + * | |
| 1130 | + * - Note that we usually don't do a vcpu_load() on the back of a fault | |
| 1131 | + * (unless we are preempted), so the resolution of a translation fault | |
| 1132 | + * must go via a request that will map the VNCR page in the fixmap. | |
| 1133 | + * vcpu_load() might as well use the same mechanism. | |
| 1134 | + * | |
| 1135 | + * - On vcpu_put() in a non-HYP context with HCR_EL2.NV==1, if the TLB was | |
| 1136 | + * mapped, we unmap it. Yes it is that simple. The TLB still exists | |
| 1137 | + * though, and may be reused at a later load. | |
| 1138 | + * | |
| 1139 | + * - On permission fault, we simply forward the fault to the guest's EL2. | |
| 1140 | + * Get out of my way. | |
| 1141 | + * | |
| 1142 | + * - On any TLBI for the EL2&0 translation regime, we must find any TLB that | |
| 1143 | + * intersects with the TLBI request, invalidate it, and unmap the page | |
| 1144 | + * from the fixmap. Because we need to look at all the vcpu-private TLBs, | |
| 1145 | + * this requires some wide-ranging locking to ensure that nothing races | |
| 1146 | + * against it. This may require some refcounting to avoid the search when | |
| 1147 | + * no such TLB is present. | |
| 1148 | + * | |
| 1149 | + * - On MMU notifiers, we must invalidate our TLB in a similar way, but | |
| 1150 | + * looking at the IPA instead. The funny part is that there may not be a | |
| 1151 | + * stage-2 mapping for this page if L1 hasn't accessed it using LD/ST | |
| 1152 | + * instructions. | |
| 1153 | + */ | |
| 1154 | + | |
| 1155 | +int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu) | |
| 1156 | +{ | |
| 1157 | + if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) | |
| 1158 | + return 0; | |
| 1159 | + | |
| 1160 | + vcpu->arch.vncr_tlb = kzalloc(sizeof(*vcpu->arch.vncr_tlb), | |
| 1161 | + GFP_KERNEL_ACCOUNT); | |
| 1162 | + if (!vcpu->arch.vncr_tlb) | |
| 1163 | + return -ENOMEM; | |
| 1164 | + | |
| 1165 | + return 0; | |
| 1166 | +} | |
| 1167 | + | |
| 1168 | +static u64 read_vncr_el2(struct kvm_vcpu *vcpu) | |
| 1169 | +{ | |
| 1170 | + return (u64)sign_extend64(__vcpu_sys_reg(vcpu, VNCR_EL2), 48); | |
| 1171 | +} | |
| 1172 | + | |
| 1173 | +static int kvm_translate_vncr(struct kvm_vcpu *vcpu) | |
| 1174 | +{ | |
| 1175 | + bool write_fault, writable; | |
| 1176 | + unsigned long mmu_seq; | |
| 1177 | + struct vncr_tlb *vt; | |
| 1178 | + struct page *page; | |
| 1179 | + u64 va, pfn, gfn; | |
| 1180 | + int ret; | |
| 1181 | + | |
| 1182 | + vt = vcpu->arch.vncr_tlb; | |
| 1183 | + | |
| 1184 | + /* | |
| 1185 | + * If we're about to walk the EL2 S1 PTs, we must invalidate the | |
| 1186 | + * current TLB, as it could be sampled from another vcpu doing a | |
| 1187 | + * TLBI *IS. A real CPU wouldn't do that, but we only keep a single | |
| 1188 | + * translation, so not much of a choice. | |
| 1189 | + * | |
| 1190 | + * We also prepare the next walk wilst we're at it. | |
| 1191 | + */ | |
| 1192 | + scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { | |
| 1193 | + invalidate_vncr(vt); | |
| 1194 | + | |
| 1195 | + vt->wi = (struct s1_walk_info) { | |
| 1196 | + .regime = TR_EL20, | |
| 1197 | + .as_el0 = false, | |
| 1198 | + .pan = false, | |
| 1199 | + }; | |
| 1200 | + vt->wr = (struct s1_walk_result){}; | |
| 1201 | + } | |
| 1202 | + | |
| 1203 | + guard(srcu)(&vcpu->kvm->srcu); | |
| 1204 | + | |
| 1205 | + va = read_vncr_el2(vcpu); | |
| 1206 | + | |
| 1207 | + ret = __kvm_translate_va(vcpu, &vt->wi, &vt->wr, va); | |
| 1208 | + if (ret) | |
| 1209 | + return ret; | |
| 1210 | + | |
| 1211 | + write_fault = kvm_is_write_fault(vcpu); | |
| 1212 | + | |
| 1213 | + mmu_seq = vcpu->kvm->mmu_invalidate_seq; | |
| 1214 | + smp_rmb(); | |
| 1215 | + | |
| 1216 | + gfn = vt->wr.pa >> PAGE_SHIFT; | |
| 1217 | + pfn = kvm_faultin_pfn(vcpu, gfn, write_fault, &writable, &page); | |
| 1218 | + if (is_error_noslot_pfn(pfn) || (write_fault && !writable)) | |
| 1219 | + return -EFAULT; | |
| 1220 | + | |
| 1221 | + scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { | |
| 1222 | + if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) | |
| 1223 | + return -EAGAIN; | |
| 1224 | + | |
| 1225 | + vt->gva = va; | |
| 1226 | + vt->hpa = pfn << PAGE_SHIFT; | |
| 1227 | + vt->valid = true; | |
| 1228 | + vt->cpu = -1; | |
| 1229 | + | |
| 1230 | + kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); | |
| 1231 | + kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw); | |
| 1232 | + } | |
| 1233 | + | |
| 1234 | + if (vt->wr.pw) | |
| 1235 | + mark_page_dirty(vcpu->kvm, gfn); | |
| 1236 | + | |
| 1237 | + return 0; | |
| 1238 | +} | |
| 1239 | + | |
| 1240 | +static void inject_vncr_perm(struct kvm_vcpu *vcpu) | |
| 1241 | +{ | |
| 1242 | + struct vncr_tlb *vt = vcpu->arch.vncr_tlb; | |
| 1243 | + u64 esr = kvm_vcpu_get_esr(vcpu); | |
| 1244 | + | |
| 1245 | + /* Adjust the fault level to reflect that of the guest's */ | |
| 1246 | + esr &= ~ESR_ELx_FSC; | |
| 1247 | + esr |= FIELD_PREP(ESR_ELx_FSC, | |
| 1248 | + ESR_ELx_FSC_PERM_L(vt->wr.level)); | |
| 1249 | + | |
| 1250 | + kvm_inject_nested_sync(vcpu, esr); | |
| 1251 | +} | |
| 1252 | + | |
| 1253 | +static bool kvm_vncr_tlb_lookup(struct kvm_vcpu *vcpu) | |
| 1254 | +{ | |
| 1255 | + struct vncr_tlb *vt = vcpu->arch.vncr_tlb; | |
| 1256 | + | |
| 1257 | + lockdep_assert_held_read(&vcpu->kvm->mmu_lock); | |
| 1258 | + | |
| 1259 | + if (!vt->valid) | |
| 1260 | + return false; | |
| 1261 | + | |
| 1262 | + if (read_vncr_el2(vcpu) != vt->gva) | |
| 1263 | + return false; | |
| 1264 | + | |
| 1265 | + if (vt->wr.nG) { | |
| 1266 | + u64 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2); | |
| 1267 | + u64 ttbr = ((tcr & TCR_A1) ? | |
| 1268 | + vcpu_read_sys_reg(vcpu, TTBR1_EL2) : | |
| 1269 | + vcpu_read_sys_reg(vcpu, TTBR0_EL2)); | |
| 1270 | + u16 asid; | |
| 1271 | + | |
| 1272 | + asid = FIELD_GET(TTBR_ASID_MASK, ttbr); | |
| 1273 | + if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) || | |
| 1274 | + !(tcr & TCR_ASID16)) | |
| 1275 | + asid &= GENMASK(7, 0); | |
| 1276 | + | |
| 1277 | + return asid != vt->wr.asid; | |
| 1278 | + } | |
| 1279 | + | |
| 1280 | + return true; | |
| 1281 | +} | |
| 1282 | + | |
| 1283 | +int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu) | |
| 1284 | +{ | |
| 1285 | + struct vncr_tlb *vt = vcpu->arch.vncr_tlb; | |
| 1286 | + u64 esr = kvm_vcpu_get_esr(vcpu); | |
| 1287 | + | |
| 1288 | + BUG_ON(!(esr & ESR_ELx_VNCR_SHIFT)); | |
| 1289 | + | |
| 1290 | + if (esr_fsc_is_permission_fault(esr)) { | |
| 1291 | + inject_vncr_perm(vcpu); | |
| 1292 | + } else if (esr_fsc_is_translation_fault(esr)) { | |
| 1293 | + bool valid; | |
| 1294 | + int ret; | |
| 1295 | + | |
| 1296 | + scoped_guard(read_lock, &vcpu->kvm->mmu_lock) | |
| 1297 | + valid = kvm_vncr_tlb_lookup(vcpu); | |
| 1298 | + | |
| 1299 | + if (!valid) | |
| 1300 | + ret = kvm_translate_vncr(vcpu); | |
| 1301 | + else | |
| 1302 | + ret = -EPERM; | |
| 1303 | + | |
| 1304 | + switch (ret) { | |
| 1305 | + case -EAGAIN: | |
| 1306 | + case -ENOMEM: | |
| 1307 | + /* Let's try again... */ | |
| 1308 | + break; | |
| 1309 | + case -EFAULT: | |
| 1310 | + case -EINVAL: | |
| 1311 | + case -ENOENT: | |
| 1312 | + case -EACCES: | |
| 1313 | + /* | |
| 1314 | + * Translation failed, inject the corresponding | |
| 1315 | + * exception back to EL2. | |
| 1316 | + */ | |
| 1317 | + BUG_ON(!vt->wr.failed); | |
| 1318 | + | |
| 1319 | + esr &= ~ESR_ELx_FSC; | |
| 1320 | + esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst); | |
| 1321 | + | |
| 1322 | + kvm_inject_nested_sync(vcpu, esr); | |
| 1323 | + break; | |
| 1324 | + case -EPERM: | |
| 1325 | + /* Hack to deal with POE until we get kernel support */ | |
| 1326 | + inject_vncr_perm(vcpu); | |
| 1327 | + break; | |
| 1328 | + case 0: | |
| 1329 | + break; | |
| 1330 | + } | |
| 1331 | + } else { | |
| 1332 | + WARN_ONCE(1, "Unhandled VNCR abort, ESR=%llx\n", esr); | |
| 1333 | + } | |
| 1334 | + | |
| 1335 | + return 1; | |
| 1336 | +} | |
| 1337 | + | |
| 1338 | +static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) | |
| 1339 | +{ | |
| 1340 | + struct vncr_tlb *vt = vcpu->arch.vncr_tlb; | |
| 1341 | + pgprot_t prot; | |
| 1342 | + | |
| 1343 | + guard(preempt)(); | |
| 1344 | + guard(read_lock)(&vcpu->kvm->mmu_lock); | |
| 1345 | + | |
| 1346 | + /* | |
| 1347 | + * The request to map VNCR may have raced against some other | |
| 1348 | + * event, such as an interrupt, and may not be valid anymore. | |
| 1349 | + */ | |
| 1350 | + if (is_hyp_ctxt(vcpu)) | |
| 1351 | + return; | |
| 1352 | + | |
| 1353 | + /* | |
| 1354 | + * Check that the pseudo-TLB is valid and that VNCR_EL2 still | |
| 1355 | + * contains the expected value. If it doesn't, we simply bail out | |
| 1356 | + * without a mapping -- a transformed MSR/MRS will generate the | |
| 1357 | + * fault and allows us to populate the pseudo-TLB. | |
| 1358 | + */ | |
| 1359 | + if (!vt->valid) | |
| 1360 | + return; | |
| 1361 | + | |
| 1362 | + if (read_vncr_el2(vcpu) != vt->gva) | |
| 1363 | + return; | |
| 1364 | + | |
| 1365 | + if (vt->wr.nG) { | |
| 1366 | + u64 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2); | |
| 1367 | + u64 ttbr = ((tcr & TCR_A1) ? | |
| 1368 | + vcpu_read_sys_reg(vcpu, TTBR1_EL2) : | |
| 1369 | + vcpu_read_sys_reg(vcpu, TTBR0_EL2)); | |
| 1370 | + u16 asid; | |
| 1371 | + | |
| 1372 | + asid = FIELD_GET(TTBR_ASID_MASK, ttbr); | |
| 1373 | + if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) || | |
| 1374 | + !(tcr & TCR_ASID16)) | |
| 1375 | + asid &= GENMASK(7, 0); | |
| 1376 | + | |
| 1377 | + if (asid != vt->wr.asid) | |
| 1378 | + return; | |
| 1379 | + } | |
| 1380 | + | |
| 1381 | + vt->cpu = smp_processor_id(); | |
| 1382 | + | |
| 1383 | + if (vt->wr.pw && vt->wr.pr) | |
| 1384 | + prot = PAGE_KERNEL; | |
| 1385 | + else if (vt->wr.pr) | |
| 1386 | + prot = PAGE_KERNEL_RO; | |
| 1387 | + else | |
| 1388 | + prot = PAGE_NONE; | |
| 1389 | + | |
| 1390 | + /* | |
| 1391 | + * We can't map write-only (or no permission at all) in the kernel, | |
| 1392 | + * but the guest can do it if using POE, so we'll have to turn a | |
| 1393 | + * translation fault into a permission fault at runtime. | |
| 1394 | + * FIXME: WO doesn't work at all, need POE support in the kernel. | |
| 1395 | + */ | |
| 1396 | + if (pgprot_val(prot) != pgprot_val(PAGE_NONE)) { | |
| 1397 | + __set_fixmap(vncr_fixmap(vt->cpu), vt->hpa, prot); | |
| 1398 | + host_data_set_flag(L1_VNCR_MAPPED); | |
| 1399 | + atomic_inc(&vcpu->kvm->arch.vncr_map_count); | |
| 1400 | + } | |
| 1401 | +} | |
| 1402 | + | |
| 804 | 1403 | /* |
| 805 | 1404 | * Our emulated CPU doesn't support all the possible features. For the |
| 806 | 1405 | * sake of simplicity (and probably mental sanity), wipe out a number |
| @@ -1151,6 +1750,9 @@ int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu) · 344 unmodified lines | ||
| 1151 | 1750 | res0 |= ICH_HCR_EL2_DVIM | ICH_HCR_EL2_vSGIEOICount; |
| 1152 | 1751 | set_sysreg_masks(kvm, ICH_HCR_EL2, res0, res1); |
| 1153 | 1752 | |
| 1753 | + /* VNCR_EL2 */ | |
| 1754 | + set_sysreg_masks(kvm, VNCR_EL2, VNCR_EL2_RES0, VNCR_EL2_RES1); | |
| 1755 | + | |
| 1154 | 1756 | out: |
| 1155 | 1757 | for (enum vcpu_sysreg sr = __SANITISED_REG_START__; sr < NR_SYS_REGS; sr++) |
| 1156 | 1758 | (void)__vcpu_sys_reg(vcpu, sr); |
| @@ -1171,6 +1773,9 @@ void check_nested_vcpu_requests(struct kvm_vcpu *vcpu) · 14 unmodified lines | ||
| 1171 | 1773 | write_unlock(&vcpu->kvm->mmu_lock); |
| 1172 | 1774 | } |
| 1173 | 1775 | |
| 1776 | + if (kvm_check_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu)) | |
| 1777 | + kvm_map_l1_vncr(vcpu); | |
| 1778 | + | |
| 1174 | 1779 | /* Must be last, as may switch context! */ |
| 1175 | 1780 | if (kvm_check_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu)) |
| 1176 | 1781 | kvm_inject_nested_irq(vcpu); |
| @@ -158,6 +158,8 @@ void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu) · 157 unmodified lines | ||
| 158 | 158 | if (sve_state) |
| 159 | 159 | kvm_unshare_hyp(sve_state, sve_state + vcpu_sve_state_size(vcpu)); |
| 160 | 160 | kfree(sve_state); |
| 161 | + free_page((unsigned long)vcpu->arch.ctxt.vncr_array); | |
| 162 | + kfree(vcpu->arch.vncr_tlb); | |
| 161 | 163 | kfree(vcpu->arch.ccsidr); |
| 162 | 164 | } |
| 163 | 165 | |
| @@ -2306,15 +2306,6 @@ static bool bad_redir_trap(struct kvm_vcpu *vcpu, · 2305 unmodified lines | ||
| 2306 | 2306 | "trap of EL2 register redirected to EL1"); |
| 2307 | 2307 | } |
| 2308 | 2308 | |
| 2309 | −#define EL2_REG(name, acc, rst, v) { \ | |
| 2310 | − SYS_DESC(SYS_##name), \ | |
| 2311 | − .access = acc, \ | |
| 2312 | − .reset = rst, \ | |
| 2313 | − .reg = name, \ | |
| 2314 | − .visibility = el2_visibility, \ | |
| 2315 | − .val = v, \ | |
| 2316 | −} | |
| 2317 | − | |
| 2318 | 2309 | #define EL2_REG_FILTERED(name, acc, rst, v, filter) { \ |
| 2319 | 2310 | SYS_DESC(SYS_##name), \ |
| 2320 | 2311 | .access = acc, \ |
| @@ -2324,6 +2315,9 @@ static bool bad_redir_trap(struct kvm_vcpu *vcpu, · 3 unmodified lines | ||
| 2324 | 2315 | .val = v, \ |
| 2325 | 2316 | } |
| 2326 | 2317 | |
| 2318 | +#define EL2_REG(name, acc, rst, v) \ | |
| 2319 | + EL2_REG_FILTERED(name, acc, rst, v, el2_visibility) | |
| 2320 | + | |
| 2327 | 2321 | #define EL2_REG_VNCR(name, rst, v) EL2_REG(name, bad_vncr_trap, rst, v) |
| 2328 | 2322 | #define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v) |
| 2329 | 2323 | |
| @@ -2471,6 +2465,16 @@ static unsigned int sve_el2_visibility(const struct kvm_vcpu *vcpu, · 141 unmodified lines | ||
| 2471 | 2465 | return __el2_visibility(vcpu, rd, sve_visibility); |
| 2472 | 2466 | } |
| 2473 | 2467 | |
| 2468 | +static unsigned int vncr_el2_visibility(const struct kvm_vcpu *vcpu, | |
| 2469 | + const struct sys_reg_desc *rd) | |
| 2470 | +{ | |
| 2471 | + if (el2_visibility(vcpu, rd) == 0 && | |
| 2472 | + kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) | |
| 2473 | + return 0; | |
| 2474 | + | |
| 2475 | + return REG_HIDDEN; | |
| 2476 | +} | |
| 2477 | + | |
| 2474 | 2478 | static bool access_zcr_el2(struct kvm_vcpu *vcpu, |
| 2475 | 2479 | struct sys_reg_params *p, |
| 2476 | 2480 | const struct sys_reg_desc *r) |
| @@ -3311,6 +3315,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 834 unmodified lines | ||
| 3311 | 3315 | tcr2_el2_visibility), |
| 3312 | 3316 | EL2_REG_VNCR(VTTBR_EL2, reset_val, 0), |
| 3313 | 3317 | EL2_REG_VNCR(VTCR_EL2, reset_val, 0), |
| 3318 | + EL2_REG_FILTERED(VNCR_EL2, bad_vncr_trap, reset_val, 0, | |
| 3319 | + vncr_el2_visibility), | |
| 3314 | 3320 | |
| 3315 | 3321 | { SYS_DESC(SYS_DACR32_EL2), undef_access, reset_unknown, DACR32_EL2 }, |
| 3316 | 3322 | EL2_REG_VNCR(HDFGRTR_EL2, reset_val, 0), |
| @@ -3594,8 +3600,7 @@ static bool handle_ripas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 277 unmodified lines | ||
| 3594 | 3600 | { |
| 3595 | 3601 | u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2); |
| 3596 | 3602 | u64 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); |
| 3597 | − u64 base, range, tg, num, scale; | |
| 3598 | − int shift; | |
| 3603 | + u64 base, range; | |
| 3599 | 3604 | |
| 3600 | 3605 | if (!kvm_supported_tlbi_ipas2_op(vcpu, sys_encoding)) |
| 3601 | 3606 | return undef_access(vcpu, p, r); |
| @@ -3605,26 +3610,7 @@ static bool handle_ripas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 3 unmodified lines | ||
| 3605 | 3610 | * of the guest's S2 (different base granule size, for example), we |
| 3606 | 3611 | * decide to ignore TTL and only use the described range. |
| 3607 | 3612 | */ |
| 3608 | − tg = FIELD_GET(GENMASK(47, 46), p->regval); | |
| 3609 | − scale = FIELD_GET(GENMASK(45, 44), p->regval); | |
| 3610 | − num = FIELD_GET(GENMASK(43, 39), p->regval); | |
| 3611 | − base = p->regval & GENMASK(36, 0); | |
| 3612 | − | |
| 3613 | − switch(tg) { | |
| 3614 | − case 1: | |
| 3615 | − shift = 12; | |
| 3616 | − break; | |
| 3617 | − case 2: | |
| 3618 | − shift = 14; | |
| 3619 | − break; | |
| 3620 | − case 3: | |
| 3621 | − default: /* IMPDEF: handle tg==0 as 64k */ | |
| 3622 | − shift = 16; | |
| 3623 | − break; | |
| 3624 | − } | |
| 3625 | − | |
| 3626 | − base <<= shift; | |
| 3627 | − range = __TLBI_RANGE_PAGES(num, scale) << shift; | |
| 3613 | + base = decode_range_tlbi(p->regval, &range, NULL); | |
| 3628 | 3614 | |
| 3629 | 3615 | kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, get_vmid(vttbr), |
| 3630 | 3616 | &(union tlbi_info) { |
| @@ -3690,11 +3676,22 @@ static void s2_mmu_tlbi_s1e1(struct kvm_s2_mmu *mmu, · 59 unmodified lines | ||
| 3690 | 3676 | WARN_ON(__kvm_tlbi_s1e2(mmu, info->va.addr, info->va.encoding)); |
| 3691 | 3677 | } |
| 3692 | 3678 | |
| 3679 | +static bool handle_tlbi_el2(struct kvm_vcpu *vcpu, struct sys_reg_params *p, | |
| 3680 | + const struct sys_reg_desc *r) | |
| 3681 | +{ | |
| 3682 | + u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2); | |
| 3683 | + | |
| 3684 | + if (!kvm_supported_tlbi_s1e2_op(vcpu, sys_encoding)) | |
| 3685 | + return undef_access(vcpu, p, r); | |
| 3686 | + | |
| 3687 | + kvm_handle_s1e2_tlbi(vcpu, sys_encoding, p->regval); | |
| 3688 | + return true; | |
| 3689 | +} | |
| 3690 | + | |
| 3693 | 3691 | static bool handle_tlbi_el1(struct kvm_vcpu *vcpu, struct sys_reg_params *p, |
| 3694 | 3692 | const struct sys_reg_desc *r) |
| 3695 | 3693 | { |
| 3696 | 3694 | u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2); |
| 3697 | − u64 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); | |
| 3698 | 3695 | |
| 3699 | 3696 | /* |
| 3700 | 3697 | * If we're here, this is because we've trapped on a EL1 TLBI |
| @@ -3705,6 +3702,13 @@ static bool handle_tlbi_el1(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 4 unmodified lines | ||
| 3705 | 3702 | * - HCR_EL2.E2H == 0 : a non-VHE guest |
| 3706 | 3703 | * - HCR_EL2.{E2H,TGE} == { 1, 0 } : a VHE guest in guest mode |
| 3707 | 3704 | * |
| 3705 | + * Another possibility is that we are invalidating the EL2 context | |
| 3706 | + * using EL1 instructions, but that we landed here because we need | |
| 3707 | + * additional invalidation for structures that are not held in the | |
| 3708 | + * CPU TLBs (such as the VNCR pseudo-TLB and its EL2 mapping). In | |
| 3709 | + * that case, we are guaranteed that HCR_EL2.{E2H,TGE} == { 1, 1 } | |
| 3710 | + * as we don't allow an NV-capable L1 in a nVHE configuration. | |
| 3711 | + * | |
| 3708 | 3712 | * We don't expect these helpers to ever be called when running |
| 3709 | 3713 | * in a vEL1 context. |
| 3710 | 3714 | */ |
| @@ -3714,7 +3718,13 @@ static bool handle_tlbi_el1(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 3 unmodified lines | ||
| 3714 | 3718 | if (!kvm_supported_tlbi_s1e1_op(vcpu, sys_encoding)) |
| 3715 | 3719 | return undef_access(vcpu, p, r); |
| 3716 | 3720 | |
| 3717 | − kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, get_vmid(vttbr), | |
| 3721 | + if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu)) { | |
| 3722 | + kvm_handle_s1e2_tlbi(vcpu, sys_encoding, p->regval); | |
| 3723 | + return true; | |
| 3724 | + } | |
| 3725 | + | |
| 3726 | + kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, | |
| 3727 | + get_vmid(__vcpu_sys_reg(vcpu, VTTBR_EL2)), | |
| 3718 | 3728 | &(union tlbi_info) { |
| 3719 | 3729 | .va = { |
| 3720 | 3730 | .addr = p->regval, |
| @@ -3836,16 +3846,21 @@ static struct sys_reg_desc sys_insn_descs[] = { · 115 unmodified lines | ||
| 3836 | 3846 | SYS_INSN(TLBI_IPAS2LE1IS, handle_ipas2e1is), |
| 3837 | 3847 | SYS_INSN(TLBI_RIPAS2LE1IS, handle_ripas2e1is), |
| 3838 | 3848 | |
| 3839 | − SYS_INSN(TLBI_ALLE2OS, undef_access), | |
| 3840 | − SYS_INSN(TLBI_VAE2OS, undef_access), | |
| 3849 | + SYS_INSN(TLBI_ALLE2OS, handle_tlbi_el2), | |
| 3850 | + SYS_INSN(TLBI_VAE2OS, handle_tlbi_el2), | |
| 3841 | 3851 | SYS_INSN(TLBI_ALLE1OS, handle_alle1is), |
| 3842 | − SYS_INSN(TLBI_VALE2OS, undef_access), | |
| 3852 | + SYS_INSN(TLBI_VALE2OS, handle_tlbi_el2), | |
| 3843 | 3853 | SYS_INSN(TLBI_VMALLS12E1OS, handle_vmalls12e1is), |
| 3844 | 3854 | |
| 3845 | − SYS_INSN(TLBI_RVAE2IS, undef_access), | |
| 3846 | − SYS_INSN(TLBI_RVALE2IS, undef_access), | |
| 3855 | + SYS_INSN(TLBI_RVAE2IS, handle_tlbi_el2), | |
| 3856 | + SYS_INSN(TLBI_RVALE2IS, handle_tlbi_el2), | |
| 3857 | + SYS_INSN(TLBI_ALLE2IS, handle_tlbi_el2), | |
| 3858 | + SYS_INSN(TLBI_VAE2IS, handle_tlbi_el2), | |
| 3847 | 3859 | |
| 3848 | 3860 | SYS_INSN(TLBI_ALLE1IS, handle_alle1is), |
| 3861 | + | |
| 3862 | + SYS_INSN(TLBI_VALE2IS, handle_tlbi_el2), | |
| 3863 | + | |
| 3849 | 3864 | SYS_INSN(TLBI_VMALLS12E1IS, handle_vmalls12e1is), |
| 3850 | 3865 | SYS_INSN(TLBI_IPAS2E1OS, handle_ipas2e1is), |
| 3851 | 3866 | SYS_INSN(TLBI_IPAS2E1, handle_ipas2e1is), |
| @@ -3855,11 +3870,17 @@ static struct sys_reg_desc sys_insn_descs[] = { · 3 unmodified lines | ||
| 3855 | 3870 | SYS_INSN(TLBI_IPAS2LE1, handle_ipas2e1is), |
| 3856 | 3871 | SYS_INSN(TLBI_RIPAS2LE1, handle_ripas2e1is), |
| 3857 | 3872 | SYS_INSN(TLBI_RIPAS2LE1OS, handle_ripas2e1is), |
| 3858 | − SYS_INSN(TLBI_RVAE2OS, undef_access), | |
| 3859 | − SYS_INSN(TLBI_RVALE2OS, undef_access), | |
| 3860 | − SYS_INSN(TLBI_RVAE2, undef_access), | |
| 3861 | − SYS_INSN(TLBI_RVALE2, undef_access), | |
| 3873 | + SYS_INSN(TLBI_RVAE2OS, handle_tlbi_el2), | |
| 3874 | + SYS_INSN(TLBI_RVALE2OS, handle_tlbi_el2), | |
| 3875 | + SYS_INSN(TLBI_RVAE2, handle_tlbi_el2), | |
| 3876 | + SYS_INSN(TLBI_RVALE2, handle_tlbi_el2), | |
| 3877 | + SYS_INSN(TLBI_ALLE2, handle_tlbi_el2), | |
| 3878 | + SYS_INSN(TLBI_VAE2, handle_tlbi_el2), | |
| 3879 | + | |
| 3862 | 3880 | SYS_INSN(TLBI_ALLE1, handle_alle1is), |
| 3881 | + | |
| 3882 | + SYS_INSN(TLBI_VALE2, handle_tlbi_el2), | |
| 3883 | + | |
| 3863 | 3884 | SYS_INSN(TLBI_VMALLS12E1, handle_vmalls12e1is), |
| 3864 | 3885 | |
| 3865 | 3886 | SYS_INSN(TLBI_IPAS2E1ISNXS, handle_ipas2e1is), |
| @@ -3867,19 +3888,19 @@ static struct sys_reg_desc sys_insn_descs[] = { · 1 unmodified line | ||
| 3867 | 3888 | SYS_INSN(TLBI_IPAS2LE1ISNXS, handle_ipas2e1is), |
| 3868 | 3889 | SYS_INSN(TLBI_RIPAS2LE1ISNXS, handle_ripas2e1is), |
| 3869 | 3890 | |
| 3870 | − SYS_INSN(TLBI_ALLE2OSNXS, undef_access), | |
| 3871 | − SYS_INSN(TLBI_VAE2OSNXS, undef_access), | |
| 3891 | + SYS_INSN(TLBI_ALLE2OSNXS, handle_tlbi_el2), | |
| 3892 | + SYS_INSN(TLBI_VAE2OSNXS, handle_tlbi_el2), | |
| 3872 | 3893 | SYS_INSN(TLBI_ALLE1OSNXS, handle_alle1is), |
| 3873 | − SYS_INSN(TLBI_VALE2OSNXS, undef_access), | |
| 3894 | + SYS_INSN(TLBI_VALE2OSNXS, handle_tlbi_el2), | |
| 3874 | 3895 | SYS_INSN(TLBI_VMALLS12E1OSNXS, handle_vmalls12e1is), |
| 3875 | 3896 | |
| 3876 | − SYS_INSN(TLBI_RVAE2ISNXS, undef_access), | |
| 3877 | − SYS_INSN(TLBI_RVALE2ISNXS, undef_access), | |
| 3878 | − SYS_INSN(TLBI_ALLE2ISNXS, undef_access), | |
| 3879 | − SYS_INSN(TLBI_VAE2ISNXS, undef_access), | |
| 3897 | + SYS_INSN(TLBI_RVAE2ISNXS, handle_tlbi_el2), | |
| 3898 | + SYS_INSN(TLBI_RVALE2ISNXS, handle_tlbi_el2), | |
| 3899 | + SYS_INSN(TLBI_ALLE2ISNXS, handle_tlbi_el2), | |
| 3900 | + SYS_INSN(TLBI_VAE2ISNXS, handle_tlbi_el2), | |
| 3880 | 3901 | |
| 3881 | 3902 | SYS_INSN(TLBI_ALLE1ISNXS, handle_alle1is), |
| 3882 | − SYS_INSN(TLBI_VALE2ISNXS, undef_access), | |
| 3903 | + SYS_INSN(TLBI_VALE2ISNXS, handle_tlbi_el2), | |
| 3883 | 3904 | SYS_INSN(TLBI_VMALLS12E1ISNXS, handle_vmalls12e1is), |
| 3884 | 3905 | SYS_INSN(TLBI_IPAS2E1OSNXS, handle_ipas2e1is), |
| 3885 | 3906 | SYS_INSN(TLBI_IPAS2E1NXS, handle_ipas2e1is), |
| @@ -3889,14 +3910,14 @@ static struct sys_reg_desc sys_insn_descs[] = { · 3 unmodified lines | ||
| 3889 | 3910 | SYS_INSN(TLBI_IPAS2LE1NXS, handle_ipas2e1is), |
| 3890 | 3911 | SYS_INSN(TLBI_RIPAS2LE1NXS, handle_ripas2e1is), |
| 3891 | 3912 | SYS_INSN(TLBI_RIPAS2LE1OSNXS, handle_ripas2e1is), |
| 3892 | − SYS_INSN(TLBI_RVAE2OSNXS, undef_access), | |
| 3893 | − SYS_INSN(TLBI_RVALE2OSNXS, undef_access), | |
| 3894 | − SYS_INSN(TLBI_RVAE2NXS, undef_access), | |
| 3895 | − SYS_INSN(TLBI_RVALE2NXS, undef_access), | |
| 3896 | − SYS_INSN(TLBI_ALLE2NXS, undef_access), | |
| 3897 | − SYS_INSN(TLBI_VAE2NXS, undef_access), | |
| 3913 | + SYS_INSN(TLBI_RVAE2OSNXS, handle_tlbi_el2), | |
| 3914 | + SYS_INSN(TLBI_RVALE2OSNXS, handle_tlbi_el2), | |
| 3915 | + SYS_INSN(TLBI_RVAE2NXS, handle_tlbi_el2), | |
| 3916 | + SYS_INSN(TLBI_RVALE2NXS, handle_tlbi_el2), | |
| 3917 | + SYS_INSN(TLBI_ALLE2NXS, handle_tlbi_el2), | |
| 3918 | + SYS_INSN(TLBI_VAE2NXS, handle_tlbi_el2), | |
| 3898 | 3919 | SYS_INSN(TLBI_ALLE1NXS, handle_alle1is), |
| 3899 | − SYS_INSN(TLBI_VALE2NXS, undef_access), | |
| 3920 | + SYS_INSN(TLBI_VALE2NXS, handle_tlbi_el2), | |
| 3900 | 3921 | SYS_INSN(TLBI_VMALLS12E1NXS, handle_vmalls12e1is), |
| 3901 | 3922 | }; |
| 3902 | 3923 | |
| @@ -3816,6 +3816,12 @@ Sysreg SMCR_EL2 3 4 1 2 6 · 3815 unmodified lines | ||
| 3816 | 3816 | Fields SMCR_ELx |
| 3817 | 3817 | EndSysreg |
| 3818 | 3818 | |
| 3819 | +Sysreg VNCR_EL2 3 4 2 2 0 | |
| 3820 | +Field 63:57 RESS | |
| 3821 | +Field 56:12 BADDR | |
| 3822 | +Res0 11:0 | |
| 3823 | +EndSysreg | |
| 3824 | + | |
| 3819 | 3825 | Sysreg GCSCR_EL2 3 4 2 5 0 |
| 3820 | 3826 | Fields GCSCR_ELx |
| 3821 | 3827 | EndSysreg |
| @@ -930,6 +930,8 @@ struct kvm_enable_cap { · 929 unmodified lines | ||
| 930 | 930 | #define KVM_CAP_X86_APIC_BUS_CYCLES_NS 237 |
| 931 | 931 | #define KVM_CAP_X86_GUEST_MODE 238 |
| 932 | 932 | #define KVM_CAP_ARM_WRITABLE_IMP_ID_REGS 239 |
| 933 | +#define KVM_CAP_ARM_EL2 240 | |
| 934 | +#define KVM_CAP_ARM_EL2_E2H0 241 | |
| 933 | 935 | |
| 934 | 936 | struct kvm_irq_routing_irqchip { |
| 935 | 937 | __u32 irqchip; |