History

Merge branch kvm-arm64/nv-nv into kvmarm-master/next

* kvm-arm64/nv-nv: : . : Flick the switch on the NV support by adding the missing piece : in the form of the VNCR page management. From the cover letter: : : "This is probably the most interesting bit of the whole NV adventure. : So far, everything else has been a walk in the park, but this one is : where the real fun takes place. : : With FEAT_NV2, most of the NV support revolves around tricking a guest : into accessing memory while it tries to access system registers. The : hypervisor's job is to handle the context switch of the actual : registers with the state in memory as needed." : . KVM: arm64: nv: Release faulted-in VNCR page from mmu_lock critical section KVM: arm64: nv: Handle TLBI S1E2 for VNCR invalidation with mmu_lock held KVM: arm64: nv: Hold mmu_lock when invalidating VNCR SW-TLB before translating KVM: arm64: Document NV caps and vcpu flags KVM: arm64: Allow userspace to request KVM_ARM_VCPU_EL2* KVM: arm64: nv: Remove dead code from ERET handling KVM: arm64: nv: Plumb TLBI S1E2 into system instruction dispatch KVM: arm64: nv: Add S1 TLB invalidation primitive for VNCR_EL2 KVM: arm64: nv: Program host's VNCR_EL2 to the fixmap address KVM: arm64: nv: Handle VNCR_EL2 invalidation from MMU notifiers KVM: arm64: nv: Handle mapping of VNCR_EL2 at EL2 KVM: arm64: nv: Handle VNCR_EL2-triggered faults KVM: arm64: nv: Add userspace and guest handling of VNCR_EL2 KVM: arm64: nv: Add pseudo-TLB backing VNCR_EL2 KVM: arm64: nv: Don't adjust PSTATE.M when L2 is nesting KVM: arm64: nv: Move TLBI range decoding to a helper KVM: arm64: nv: Snapshot S1 ASID tagging information during walk KVM: arm64: nv: Extract translation helper from the AT code KVM: arm64: nv: Allocate VNCR page when required arm64: sysreg: Add layout for VNCR_EL2

Marc Zyngier committed 1 year ago

Commit 7f3225f2 parents 35e4d7f 538fbac16 files · +956 −138Browse files

Unified · Split

16 files
  1. Documentation/virt/kvm/api.rst +13 −1
  2. arch/arm64/include/asm/esr.h +2 −0
  3. arch/arm64/include/asm/fixmap.h +6 −0
  4. arch/arm64/include/asm/kvm_host.h +14 −1
  5. arch/arm64/include/asm/kvm_nested.h +100 −0
  6. arch/arm64/include/asm/sysreg.h +1 −1
  7. arch/arm64/kvm/arm.c +10 −0
  8. arch/arm64/kvm/at.c +64 −59
  9. arch/arm64/kvm/emulate-nested.c +0 −7
  10. arch/arm64/kvm/handle_exit.c +1 −0
  11. arch/arm64/kvm/hyp/vhe/switch.c +43 −3
  12. arch/arm64/kvm/nested.c +614 −9
  13. arch/arm64/kvm/reset.c +2 −0
  14. arch/arm64/kvm/sys_regs.c +78 −57
  15. arch/arm64/tools/sysreg +6 −0
  16. include/uapi/linux/kvm.h +2 −0
Documentation/virt/kvm/api.rst+13−1
@@ -3460,7 +3460,8 @@ The initial values are defined as: · 3459 unmodified lines
3460 - FPSIMD/NEON registers: set to 03460 - FPSIMD/NEON registers: set to 0
3461 - SVE registers: set to 03461 - SVE registers: set to 0
3462 - System registers: Reset to their architecturally defined3462 - System registers: Reset to their architecturally defined
3463 values as for a warm reset to EL1 (resp. SVC)3463 values as for a warm reset to EL1 (resp. SVC) or EL2 (in the
3464 case of EL2 being enabled).
3464 3465
3465Note that because some registers reflect machine topology, all vcpus3466Note that because some registers reflect machine topology, all vcpus
3466should be created before this ioctl is invoked.3467should be created before this ioctl is invoked.
@@ -3527,6 +3528,17 @@ Possible features: · 60 unmodified lines
3527 - the KVM_REG_ARM64_SVE_VLS pseudo-register is immutable, and can3528 - the KVM_REG_ARM64_SVE_VLS pseudo-register is immutable, and can
3528 no longer be written using KVM_SET_ONE_REG.3529 no longer be written using KVM_SET_ONE_REG.
3529 3530
3531 - KVM_ARM_VCPU_HAS_EL2: Enable Nested Virtualisation support,
3532 booting the guest from EL2 instead of EL1.
3533 Depends on KVM_CAP_ARM_EL2.
3534 The VM is running with HCR_EL2.E2H being RES1 (VHE) unless
3535 KVM_ARM_VCPU_HAS_EL2_E2H0 is also set.
3536
3537 - KVM_ARM_VCPU_HAS_EL2_E2H0: Restrict Nested Virtualisation
3538 support to HCR_EL2.E2H being RES0 (non-VHE).
3539 Depends on KVM_CAP_ARM_EL2_E2H0.
3540 KVM_ARM_VCPU_HAS_EL2 must also be set.
3541
35304.83 KVM_ARM_PREFERRED_TARGET35424.83 KVM_ARM_PREFERRED_TARGET
3531-----------------------------3543-----------------------------
3532 3544
arch/arm64/include/asm/esr.h+2−0
@@ -100,6 +100,8 @@ · 99 unmodified lines
100#define ESR_ELx_AET_CE (UL(6) << ESR_ELx_AET_SHIFT)100#define ESR_ELx_AET_CE (UL(6) << ESR_ELx_AET_SHIFT)
101 101
102/* Shared ISS field definitions for Data/Instruction aborts */102/* Shared ISS field definitions for Data/Instruction aborts */
103#define ESR_ELx_VNCR_SHIFT (13)
104#define ESR_ELx_VNCR (UL(1) << ESR_ELx_VNCR_SHIFT)
103#define ESR_ELx_SET_SHIFT (11)105#define ESR_ELx_SET_SHIFT (11)
104#define ESR_ELx_SET_MASK (UL(3) << ESR_ELx_SET_SHIFT)106#define ESR_ELx_SET_MASK (UL(3) << ESR_ELx_SET_SHIFT)
105#define ESR_ELx_FnV_SHIFT (10)107#define ESR_ELx_FnV_SHIFT (10)
arch/arm64/include/asm/fixmap.h+6−0
@@ -48,6 +48,12 @@ enum fixed_addresses { · 47 unmodified lines
48 FIX_EARLYCON_MEM_BASE,48 FIX_EARLYCON_MEM_BASE,
49 FIX_TEXT_POKE0,49 FIX_TEXT_POKE0,
50 50
51#ifdef CONFIG_KVM
52 /* One slot per CPU, mapping the guest's VNCR page at EL2. */
53 FIX_VNCR_END,
54 FIX_VNCR = FIX_VNCR_END + NR_CPUS,
55#endif
56
51#ifdef CONFIG_ACPI_APEI_GHES57#ifdef CONFIG_ACPI_APEI_GHES
52 /* Used for GHES mapping from assorted contexts */58 /* Used for GHES mapping from assorted contexts */
53 FIX_APEI_GHES_IRQ,59 FIX_APEI_GHES_IRQ,
arch/arm64/include/asm/kvm_host.h+14−1
@@ -39,7 +39,7 @@ · 38 unmodified lines
39 39
40#define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS40#define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS
41 41
42#define KVM_VCPU_MAX_FEATURES 742#define KVM_VCPU_MAX_FEATURES 9
43#define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1)43#define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1)
44 44
45#define KVM_REQ_SLEEP \45#define KVM_REQ_SLEEP \
@@ -53,6 +53,7 @@ · 7 unmodified lines
53#define KVM_REQ_RESYNC_PMU_EL0 KVM_ARCH_REQ(7)53#define KVM_REQ_RESYNC_PMU_EL0 KVM_ARCH_REQ(7)
54#define KVM_REQ_NESTED_S2_UNMAP KVM_ARCH_REQ(8)54#define KVM_REQ_NESTED_S2_UNMAP KVM_ARCH_REQ(8)
55#define KVM_REQ_GUEST_HYP_IRQ_PENDING KVM_ARCH_REQ(9)55#define KVM_REQ_GUEST_HYP_IRQ_PENDING KVM_ARCH_REQ(9)
56#define KVM_REQ_MAP_L1_VNCR_EL2 KVM_ARCH_REQ(10)
56 57
57#define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \58#define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \
58 KVM_DIRTY_LOG_INITIALLY_SET)59 KVM_DIRTY_LOG_INITIALLY_SET)
@@ -395,6 +396,9 @@ struct kvm_arch { · 336 unmodified lines
395 /* Masks for VNCR-backed and general EL2 sysregs */396 /* Masks for VNCR-backed and general EL2 sysregs */
396 struct kvm_sysreg_masks *sysreg_masks;397 struct kvm_sysreg_masks *sysreg_masks;
397 398
399 /* Count the number of VNCR_EL2 currently mapped */
400 atomic_t vncr_map_count;
401
398 /*402 /*
399 * For an untrusted host VM, 'pkvm.handle' is used to lookup403 * For an untrusted host VM, 'pkvm.handle' is used to lookup
400 * the associated pKVM instance in the hypervisor.404 * the associated pKVM instance in the hypervisor.
@@ -573,6 +577,8 @@ enum vcpu_sysreg { · 172 unmodified lines
573 VNCR(HDFGRTR2_EL2),577 VNCR(HDFGRTR2_EL2),
574 VNCR(HDFGWTR2_EL2),578 VNCR(HDFGWTR2_EL2),
575 579
580 VNCR(VNCR_EL2),
581
576 VNCR(CNTVOFF_EL2),582 VNCR(CNTVOFF_EL2),
577 VNCR(CNTV_CVAL_EL0),583 VNCR(CNTV_CVAL_EL0),
578 VNCR(CNTV_CTL_EL0),584 VNCR(CNTV_CTL_EL0),
@@ -696,6 +702,8 @@ struct kvm_host_data { · 117 unmodified lines
696#define KVM_HOST_DATA_FLAG_HAS_TRBE 1702#define KVM_HOST_DATA_FLAG_HAS_TRBE 1
697#define KVM_HOST_DATA_FLAG_TRBE_ENABLED 4703#define KVM_HOST_DATA_FLAG_TRBE_ENABLED 4
698#define KVM_HOST_DATA_FLAG_EL1_TRACING_CONFIGURED 5704#define KVM_HOST_DATA_FLAG_EL1_TRACING_CONFIGURED 5
705#define KVM_HOST_DATA_FLAG_VCPU_IN_HYP_CONTEXT 6
706#define KVM_HOST_DATA_FLAG_L1_VNCR_MAPPED 7
699 unsigned long flags;707 unsigned long flags;
700 708
701 struct kvm_cpu_context host_ctxt;709 struct kvm_cpu_context host_ctxt;
@@ -772,6 +780,8 @@ struct vcpu_reset_state { · 70 unmodified lines
772 bool reset;780 bool reset;
773};781};
774 782
783struct vncr_tlb;
784
775struct kvm_vcpu_arch {785struct kvm_vcpu_arch {
776 struct kvm_cpu_context ctxt;786 struct kvm_cpu_context ctxt;
777 787
@@ -866,6 +876,9 @@ struct kvm_vcpu_arch { · 88 unmodified lines
866 876
867 /* Per-vcpu CCSIDR override or NULL */877 /* Per-vcpu CCSIDR override or NULL */
868 u32 *ccsidr;878 u32 *ccsidr;
879
880 /* Per-vcpu TLB for VNCR_EL2 -- NULL when !NV */
881 struct vncr_tlb *vncr_tlb;
869};882};
870 883
871/*884/*
arch/arm64/include/asm/kvm_nested.h+100−0
@@ -231,6 +231,38 @@ static inline u64 kvm_encode_nested_level(struct kvm_s2_trans *trans) · 230 unmodified lines
231 shift; \231 shift; \
232 })232 })
233 233
234static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid)
235{
236 u64 base, tg, num, scale;
237 int shift;
238
239 tg = FIELD_GET(GENMASK(47, 46), val);
240
241 switch(tg) {
242 case 1:
243 shift = 12;
244 break;
245 case 2:
246 shift = 14;
247 break;
248 case 3:
249 default: /* IMPDEF: handle tg==0 as 64k */
250 shift = 16;
251 break;
252 }
253
254 base = (val & GENMASK(36, 0)) << shift;
255
256 if (asid)
257 *asid = FIELD_GET(TLBIR_ASID_MASK, val);
258
259 scale = FIELD_GET(GENMASK(45, 44), val);
260 num = FIELD_GET(GENMASK(43, 39), val);
261 *range = __TLBI_RANGE_PAGES(num, scale) << shift;
262
263 return base;
264}
265
234static inline unsigned int ps_to_output_size(unsigned int ps)266static inline unsigned int ps_to_output_size(unsigned int ps)
235{267{
236 switch (ps) {268 switch (ps) {
@@ -245,4 +277,72 @@ static inline unsigned int ps_to_output_size(unsigned int ps) · 8 unmodified lines
245 }277 }
246}278}
247 279
280enum trans_regime {
281 TR_EL10,
282 TR_EL20,
283 TR_EL2,
284};
285
286struct s1_walk_info {
287 u64 baddr;
288 enum trans_regime regime;
289 unsigned int max_oa_bits;
290 unsigned int pgshift;
291 unsigned int txsz;
292 int sl;
293 bool as_el0;
294 bool hpd;
295 bool e0poe;
296 bool poe;
297 bool pan;
298 bool be;
299 bool s2;
300};
301
302struct s1_walk_result {
303 union {
304 struct {
305 u64 desc;
306 u64 pa;
307 s8 level;
308 u8 APTable;
309 bool nG;
310 u16 asid;
311 bool UXNTable;
312 bool PXNTable;
313 bool uwxn;
314 bool uov;
315 bool ur;
316 bool uw;
317 bool ux;
318 bool pwxn;
319 bool pov;
320 bool pr;
321 bool pw;
322 bool px;
323 };
324 struct {
325 u8 fst;
326 bool ptw;
327 bool s2;
328 };
329 };
330 bool failed;
331};
332
333int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi,
334 struct s1_walk_result *wr, u64 va);
335
336/* VNCR management */
337int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu);
338int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu);
339void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val);
340
341#define vncr_fixmap(c) \
342 ({ \
343 u32 __c = (c); \
344 BUG_ON(__c >= NR_CPUS); \
345 (FIX_VNCR - __c); \
346 })
347
248#endif /* __ARM64_KVM_NESTED_H */348#endif /* __ARM64_KVM_NESTED_H */
arch/arm64/include/asm/sysreg.h+1−1
@@ -534,7 +534,7 @@ · 533 unmodified lines
534#define SYS_VTTBR_EL2 sys_reg(3, 4, 2, 1, 0)534#define SYS_VTTBR_EL2 sys_reg(3, 4, 2, 1, 0)
535#define SYS_VTCR_EL2 sys_reg(3, 4, 2, 1, 2)535#define SYS_VTCR_EL2 sys_reg(3, 4, 2, 1, 2)
536 536
537#define SYS_VNCR_EL2 sys_reg(3, 4, 2, 2, 0)537#define SYS_HAFGRTR_EL2 sys_reg(3, 4, 3, 1, 6)
538#define SYS_SPSR_EL2 sys_reg(3, 4, 4, 0, 0)538#define SYS_SPSR_EL2 sys_reg(3, 4, 4, 0, 0)
539#define SYS_ELR_EL2 sys_reg(3, 4, 4, 0, 1)539#define SYS_ELR_EL2 sys_reg(3, 4, 4, 0, 1)
540#define SYS_SP_EL1 sys_reg(3, 4, 4, 1, 0)540#define SYS_SP_EL1 sys_reg(3, 4, 4, 1, 0)
arch/arm64/kvm/arm.c+10−0
@@ -368,6 +368,12 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 367 unmodified lines
368 case KVM_CAP_ARM_EL1_32BIT:368 case KVM_CAP_ARM_EL1_32BIT:
369 r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1);369 r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1);
370 break;370 break;
371 case KVM_CAP_ARM_EL2:
372 r = cpus_have_final_cap(ARM64_HAS_NESTED_VIRT);
373 break;
374 case KVM_CAP_ARM_EL2_E2H0:
375 r = cpus_have_final_cap(ARM64_HAS_HCR_NV1);
376 break;
371 case KVM_CAP_GUEST_DEBUG_HW_BPS:377 case KVM_CAP_GUEST_DEBUG_HW_BPS:
372 r = get_num_brps();378 r = get_num_brps();
373 break;379 break;
@@ -843,6 +849,10 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) · 469 unmodified lines
843 return ret;849 return ret;
844 850
845 if (vcpu_has_nv(vcpu)) {851 if (vcpu_has_nv(vcpu)) {
852 ret = kvm_vcpu_allocate_vncr_tlb(vcpu);
853 if (ret)
854 return ret;
855
846 ret = kvm_vgic_vcpu_nv_init(vcpu);856 ret = kvm_vgic_vcpu_nv_init(vcpu);
847 if (ret)857 if (ret)
848 return ret;858 return ret;
arch/arm64/kvm/at.c+64−59
@@ -10,56 +10,6 @@ · 9 unmodified lines
10#include <asm/kvm_hyp.h>10#include <asm/kvm_hyp.h>
11#include <asm/kvm_mmu.h>11#include <asm/kvm_mmu.h>
12 12
13enum trans_regime {
14 TR_EL10,
15 TR_EL20,
16 TR_EL2,
17};
18
19struct s1_walk_info {
20 u64 baddr;
21 enum trans_regime regime;
22 unsigned int max_oa_bits;
23 unsigned int pgshift;
24 unsigned int txsz;
25 int sl;
26 bool hpd;
27 bool e0poe;
28 bool poe;
29 bool pan;
30 bool be;
31 bool s2;
32};
33
34struct s1_walk_result {
35 union {
36 struct {
37 u64 desc;
38 u64 pa;
39 s8 level;
40 u8 APTable;
41 bool UXNTable;
42 bool PXNTable;
43 bool uwxn;
44 bool uov;
45 bool ur;
46 bool uw;
47 bool ux;
48 bool pwxn;
49 bool pov;
50 bool pr;
51 bool pw;
52 bool px;
53 };
54 struct {
55 u8 fst;
56 bool ptw;
57 bool s2;
58 };
59 };
60 bool failed;
61};
62
63static void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw)13static void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw)
64{14{
65 wr->fst = fst;15 wr->fst = fst;
@@ -145,20 +95,15 @@ static void compute_s1poe(struct kvm_vcpu *vcpu, struct s1_walk_info *wi) · 79 unmodified lines
145 }95 }
146}96}
147 97
148static int setup_s1_walk(struct kvm_vcpu *vcpu, u32 op, struct s1_walk_info *wi,98static int setup_s1_walk(struct kvm_vcpu *vcpu, struct s1_walk_info *wi,
149 struct s1_walk_result *wr, u64 va)99 struct s1_walk_result *wr, u64 va)
150{100{
151 u64 hcr, sctlr, tcr, tg, ps, ia_bits, ttbr;101 u64 hcr, sctlr, tcr, tg, ps, ia_bits, ttbr;
152 unsigned int stride, x;102 unsigned int stride, x;
153 bool va55, tbi, lva, as_el0;103 bool va55, tbi, lva;
154 104
155 hcr = __vcpu_sys_reg(vcpu, HCR_EL2);105 hcr = __vcpu_sys_reg(vcpu, HCR_EL2);
156 106
157 wi->regime = compute_translation_regime(vcpu, op);
158 as_el0 = (op == OP_AT_S1E0R || op == OP_AT_S1E0W);
159 wi->pan = (op == OP_AT_S1E1RP || op == OP_AT_S1E1WP) &&
160 (*vcpu_cpsr(vcpu) & PSR_PAN_BIT);
161
162 va55 = va & BIT(55);107 va55 = va & BIT(55);
163 108
164 if (wi->regime == TR_EL2 && va55)109 if (wi->regime == TR_EL2 && va55)
@@ -319,7 +264,7 @@ static int setup_s1_walk(struct kvm_vcpu *vcpu, u32 op, struct s1_walk_info *wi, · 154 unmodified lines
319 264
320 /* R_BNDVG and following statements */265 /* R_BNDVG and following statements */
321 if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, E0PD, IMP) &&266 if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, E0PD, IMP) &&
322 as_el0 && (tcr & (va55 ? TCR_E0PD1 : TCR_E0PD0)))267 wi->as_el0 && (tcr & (va55 ? TCR_E0PD1 : TCR_E0PD0)))
323 goto transfault_l0;268 goto transfault_l0;
324 269
325 /* AArch64.S1StartLevel() */270 /* AArch64.S1StartLevel() */
@@ -469,6 +414,33 @@ static int walk_s1(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, · 143 unmodified lines
469 wr->pa = desc & GENMASK(47, va_bottom);414 wr->pa = desc & GENMASK(47, va_bottom);
470 wr->pa |= va & GENMASK_ULL(va_bottom - 1, 0);415 wr->pa |= va & GENMASK_ULL(va_bottom - 1, 0);
471 416
417 wr->nG = (wi->regime != TR_EL2) && (desc & PTE_NG);
418 if (wr->nG) {
419 u64 asid_ttbr, tcr;
420
421 switch (wi->regime) {
422 case TR_EL10:
423 tcr = vcpu_read_sys_reg(vcpu, TCR_EL1);
424 asid_ttbr = ((tcr & TCR_A1) ?
425 vcpu_read_sys_reg(vcpu, TTBR1_EL1) :
426 vcpu_read_sys_reg(vcpu, TTBR0_EL1));
427 break;
428 case TR_EL20:
429 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2);
430 asid_ttbr = ((tcr & TCR_A1) ?
431 vcpu_read_sys_reg(vcpu, TTBR1_EL2) :
432 vcpu_read_sys_reg(vcpu, TTBR0_EL2));
433 break;
434 default:
435 BUG();
436 }
437
438 wr->asid = FIELD_GET(TTBR_ASID_MASK, asid_ttbr);
439 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) ||
440 !(tcr & TCR_ASID16))
441 wr->asid &= GENMASK(7, 0);
442 }
443
472 return 0;444 return 0;
473 445
474addrsz:446addrsz:
@@ -1155,7 +1127,12 @@ static u64 handle_at_slow(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) · 680 unmodified lines
1155 bool perm_fail = false;1127 bool perm_fail = false;
1156 int ret, idx;1128 int ret, idx;
1157 1129
1158 ret = setup_s1_walk(vcpu, op, &wi, &wr, vaddr);1130 wi.regime = compute_translation_regime(vcpu, op);
1131 wi.as_el0 = (op == OP_AT_S1E0R || op == OP_AT_S1E0W);
1132 wi.pan = (op == OP_AT_S1E1RP || op == OP_AT_S1E1WP) &&
1133 (*vcpu_cpsr(vcpu) & PSR_PAN_BIT);
1134
1135 ret = setup_s1_walk(vcpu, &wi, &wr, vaddr);
1159 if (ret)1136 if (ret)
1160 goto compute_par;1137 goto compute_par;
1161 1138
@@ -1457,3 +1434,31 @@ void __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) · 295 unmodified lines
1457 par = compute_par_s12(vcpu, par, &out);1434 par = compute_par_s12(vcpu, par, &out);
1458 vcpu_write_sys_reg(vcpu, par, PAR_EL1);1435 vcpu_write_sys_reg(vcpu, par, PAR_EL1);
1459}1436}
1437
1438/*
1439 * Translate a VA for a given EL in a given translation regime, with
1440 * or without PAN. This requires wi->{regime, as_el0, pan} to be
1441 * set. The rest of the wi and wr should be 0-initialised.
1442 */
1443int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi,
1444 struct s1_walk_result *wr, u64 va)
1445{
1446 int ret;
1447
1448 ret = setup_s1_walk(vcpu, wi, wr, va);
1449 if (ret)
1450 return ret;
1451
1452 if (wr->level == S1_MMU_DISABLED) {
1453 wr->ur = wr->uw = wr->ux = true;
1454 wr->pr = wr->pw = wr->px = true;
1455 } else {
1456 ret = walk_s1(vcpu, wi, wr, va);
1457 if (ret)
1458 return ret;
1459
1460 compute_s1_permissions(vcpu, wi, wr);
1461 }
1462
1463 return 0;
1464}
arch/arm64/kvm/emulate-nested.c+0−7
@@ -2662,13 +2662,6 @@ void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu) · 2661 unmodified lines
2662{2662{
2663 u64 spsr, elr, esr;2663 u64 spsr, elr, esr;
2664 2664
2665 /*
2666 * Forward this trap to the virtual EL2 if the virtual
2667 * HCR_EL2.NV bit is set and this is coming from !EL2.
2668 */
2669 if (forward_hcr_traps(vcpu, HCR_NV))
2670 return;
2671
2672 spsr = vcpu_read_sys_reg(vcpu, SPSR_EL2);2665 spsr = vcpu_read_sys_reg(vcpu, SPSR_EL2);
2673 spsr = kvm_check_illegal_exception_return(vcpu, spsr);2666 spsr = kvm_check_illegal_exception_return(vcpu, spsr);
2674 2667
arch/arm64/kvm/handle_exit.c+1−0
@@ -394,6 +394,7 @@ static exit_handle_fn arm_exit_handlers[] = { · 393 unmodified lines
394 [ESR_ELx_EC_ERET] = kvm_handle_eret,394 [ESR_ELx_EC_ERET] = kvm_handle_eret,
395 [ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort,395 [ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort,
396 [ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort,396 [ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort,
397 [ESR_ELx_EC_DABT_CUR] = kvm_handle_vncr_abort,
397 [ESR_ELx_EC_SOFTSTP_LOW]= kvm_handle_guest_debug,398 [ESR_ELx_EC_SOFTSTP_LOW]= kvm_handle_guest_debug,
398 [ESR_ELx_EC_WATCHPT_LOW]= kvm_handle_guest_debug,399 [ESR_ELx_EC_WATCHPT_LOW]= kvm_handle_guest_debug,
399 [ESR_ELx_EC_BREAKPT_LOW]= kvm_handle_guest_debug,400 [ESR_ELx_EC_BREAKPT_LOW]= kvm_handle_guest_debug,
arch/arm64/kvm/hyp/vhe/switch.c+43−3
@@ -48,21 +48,46 @@ DEFINE_PER_CPU(unsigned long, kvm_hyp_vector); · 47 unmodified lines
48 48
49static u64 __compute_hcr(struct kvm_vcpu *vcpu)49static u64 __compute_hcr(struct kvm_vcpu *vcpu)
50{50{
51 u64 guest_hcr = __vcpu_sys_reg(vcpu, HCR_EL2);
51 u64 hcr = vcpu->arch.hcr_el2;52 u64 hcr = vcpu->arch.hcr_el2;
52 53
53 if (!vcpu_has_nv(vcpu))54 if (!vcpu_has_nv(vcpu))
54 return hcr;55 return hcr;
55 56
57 /*
58 * We rely on the invariant that a vcpu entered from HYP
59 * context must also exit in the same context, as only an ERET
60 * instruction can kick us out of it, and we obviously trap
61 * that sucker. PSTATE.M will get fixed-up on exit.
62 */
56 if (is_hyp_ctxt(vcpu)) {63 if (is_hyp_ctxt(vcpu)) {
64 host_data_set_flag(VCPU_IN_HYP_CONTEXT);
65
57 hcr |= HCR_NV | HCR_NV2 | HCR_AT | HCR_TTLB;66 hcr |= HCR_NV | HCR_NV2 | HCR_AT | HCR_TTLB;
58 67
59 if (!vcpu_el2_e2h_is_set(vcpu))68 if (!vcpu_el2_e2h_is_set(vcpu))
60 hcr |= HCR_NV1;69 hcr |= HCR_NV1;
61 70
62 write_sysreg_s(vcpu->arch.ctxt.vncr_array, SYS_VNCR_EL2);71 write_sysreg_s(vcpu->arch.ctxt.vncr_array, SYS_VNCR_EL2);
72 } else {
73 host_data_clear_flag(VCPU_IN_HYP_CONTEXT);
74
75 if (guest_hcr & HCR_NV) {
76 u64 va = __fix_to_virt(vncr_fixmap(smp_processor_id()));
77
78 /* Inherit the low bits from the actual register */
79 va |= __vcpu_sys_reg(vcpu, VNCR_EL2) & GENMASK(PAGE_SHIFT - 1, 0);
80 write_sysreg_s(va, SYS_VNCR_EL2);
81
82 /* Force NV2 in case the guest is forgetful... */
83 guest_hcr |= HCR_NV2;
84 }
63 }85 }
64 86
65 return hcr | (__vcpu_sys_reg(vcpu, HCR_EL2) & ~NV_HCR_GUEST_EXCLUDE);87 BUG_ON(host_data_test_flag(VCPU_IN_HYP_CONTEXT) &&
88 host_data_test_flag(L1_VNCR_MAPPED));
89
90 return hcr | (guest_hcr & ~NV_HCR_GUEST_EXCLUDE);
66}91}
67 92
68static void __activate_cptr_traps(struct kvm_vcpu *vcpu)93static void __activate_cptr_traps(struct kvm_vcpu *vcpu)
@@ -459,6 +484,14 @@ static bool kvm_hyp_handle_tlbi_el2(struct kvm_vcpu *vcpu, u64 *exit_code) · 390 unmodified lines
459 if (ret)484 if (ret)
460 return false;485 return false;
461 486
487 /*
488 * If we have to check for any VNCR mapping being invalidated,
489 * go back to the slow path for further processing.
490 */
491 if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu) &&
492 atomic_read(&vcpu->kvm->arch.vncr_map_count))
493 return false;
494
462 __kvm_skip_instr(vcpu);495 __kvm_skip_instr(vcpu);
463 496
464 return true;497 return true;
@@ -568,9 +601,12 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) · 103 unmodified lines
568 601
569 /*602 /*
570 * If we were in HYP context on entry, adjust the PSTATE view603 * If we were in HYP context on entry, adjust the PSTATE view
571 * so that the usual helpers work correctly.604 * so that the usual helpers work correctly. This enforces our
605 * invariant that the guest's HYP context status is preserved
606 * across a run.
572 */607 */
573 if (vcpu_has_nv(vcpu) && (read_sysreg(hcr_el2) & HCR_NV)) {608 if (vcpu_has_nv(vcpu) &&
609 unlikely(host_data_test_flag(VCPU_IN_HYP_CONTEXT))) {
574 u64 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT);610 u64 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT);
575 611
576 switch (mode) {612 switch (mode) {
@@ -586,6 +622,10 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) · 9 unmodified lines
586 *vcpu_cpsr(vcpu) |= mode;622 *vcpu_cpsr(vcpu) |= mode;
587 }623 }
588 624
625 /* Apply extreme paranoia! */
626 BUG_ON(vcpu_has_nv(vcpu) &&
627 !!host_data_test_flag(VCPU_IN_HYP_CONTEXT) != is_hyp_ctxt(vcpu));
628
589 return __fixup_guest_exit(vcpu, exit_code, hyp_exit_handlers);629 return __fixup_guest_exit(vcpu, exit_code, hyp_exit_handlers);
590}630}
591 631
arch/arm64/kvm/nested.c+614−9
@@ -8,6 +8,7 @@ · 7 unmodified lines
8#include <linux/kvm.h>8#include <linux/kvm.h>
9#include <linux/kvm_host.h>9#include <linux/kvm_host.h>
10 10
11#include <asm/fixmap.h>
11#include <asm/kvm_arm.h>12#include <asm/kvm_arm.h>
12#include <asm/kvm_emulate.h>13#include <asm/kvm_emulate.h>
13#include <asm/kvm_mmu.h>14#include <asm/kvm_mmu.h>
@@ -16,6 +17,24 @@ · 2 unmodified lines
16 17
17#include "sys_regs.h"18#include "sys_regs.h"
18 19
20struct vncr_tlb {
21 /* The guest's VNCR_EL2 */
22 u64 gva;
23 struct s1_walk_info wi;
24 struct s1_walk_result wr;
25
26 u64 hpa;
27
28 /* -1 when not mapped on a CPU */
29 int cpu;
30
31 /*
32 * true if the TLB is valid. Can only be changed with the
33 * mmu_lock held.
34 */
35 bool valid;
36};
37
19/*38/*
20 * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between39 * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between
21 * memory usage and potential number of different sets of S2 PTs in40 * memory usage and potential number of different sets of S2 PTs in
@@ -28,6 +47,7 @@ void kvm_init_nested(struct kvm *kvm) · 6 unmodified lines
28{47{
29 kvm->arch.nested_mmus = NULL;48 kvm->arch.nested_mmus = NULL;
30 kvm->arch.nested_mmus_size = 0;49 kvm->arch.nested_mmus_size = 0;
50 atomic_set(&kvm->arch.vncr_map_count, 0);
31}51}
32 52
33static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)53static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)
@@ -55,6 +75,13 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) · 21 unmodified lines
55 !cpus_have_final_cap(ARM64_HAS_HCR_NV1))75 !cpus_have_final_cap(ARM64_HAS_HCR_NV1))
56 return -EINVAL;76 return -EINVAL;
57 77
78 if (!vcpu->arch.ctxt.vncr_array)
79 vcpu->arch.ctxt.vncr_array = (u64 *)__get_free_page(GFP_KERNEL_ACCOUNT |
80 __GFP_ZERO);
81
82 if (!vcpu->arch.ctxt.vncr_array)
83 return -ENOMEM;
84
58 /*85 /*
59 * Let's treat memory allocation failures as benign: If we fail to86 * Let's treat memory allocation failures as benign: If we fail to
60 * allocate anything, return an error and keep the allocated array87 * allocate anything, return an error and keep the allocated array
@@ -85,6 +112,9 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) · 24 unmodified lines
85 for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++)112 for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++)
86 kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]);113 kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]);
87 114
115 free_page((unsigned long)vcpu->arch.ctxt.vncr_array);
116 vcpu->arch.ctxt.vncr_array = NULL;
117
88 return ret;118 return ret;
89 }119 }
90 120
@@ -405,6 +435,30 @@ static unsigned int ttl_to_size(u8 ttl) · 314 unmodified lines
405 return max_size;435 return max_size;
406}436}
407 437
438static u8 pgshift_level_to_ttl(u16 shift, u8 level)
439{
440 u8 ttl;
441
442 switch(shift) {
443 case 12:
444 ttl = TLBI_TTL_TG_4K;
445 break;
446 case 14:
447 ttl = TLBI_TTL_TG_16K;
448 break;
449 case 16:
450 ttl = TLBI_TTL_TG_64K;
451 break;
452 default:
453 BUG();
454 }
455
456 ttl <<= 2;
457 ttl |= level & 3;
458
459 return ttl;
460}
461
408/*462/*
409 * Compute the equivalent of the TTL field by parsing the shadow PT. The463 * Compute the equivalent of the TTL field by parsing the shadow PT. The
410 * granule size is extracted from the cached VTCR_EL2.TG0 while the level is464 * granule size is extracted from the cached VTCR_EL2.TG0 while the level is
@@ -676,23 +730,36 @@ void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu) · 265 unmodified lines
676void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu)730void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu)
677{731{
678 /*732 /*
679 * The vCPU kept its reference on the MMU after the last put, keep733 * If the vCPU kept its reference on the MMU after the last put,
680 * rolling with it.734 * keep rolling with it.
681 */735 */
682 if (vcpu->arch.hw_mmu)
683 return;
684
685 if (is_hyp_ctxt(vcpu)) {736 if (is_hyp_ctxt(vcpu)) {
686 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;737 if (!vcpu->arch.hw_mmu)
738 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
687 } else {739 } else {
688 write_lock(&vcpu->kvm->mmu_lock);740 if (!vcpu->arch.hw_mmu) {
689 vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu);741 scoped_guard(write_lock, &vcpu->kvm->mmu_lock)
690 write_unlock(&vcpu->kvm->mmu_lock);742 vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu);
743 }
744
745 if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV)
746 kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu);
691 }747 }
692}748}
693 749
694void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu)750void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu)
695{751{
752 /* Unconditionally drop the VNCR mapping if we have one */
753 if (host_data_test_flag(L1_VNCR_MAPPED)) {
754 BUG_ON(vcpu->arch.vncr_tlb->cpu != smp_processor_id());
755 BUG_ON(is_hyp_ctxt(vcpu));
756
757 clear_fixmap(vncr_fixmap(vcpu->arch.vncr_tlb->cpu));
758 vcpu->arch.vncr_tlb->cpu = -1;
759 host_data_clear_flag(L1_VNCR_MAPPED);
760 atomic_dec(&vcpu->kvm->arch.vncr_map_count);
761 }
762
696 /*763 /*
697 * Keep a reference on the associated stage-2 MMU if the vCPU is764 * Keep a reference on the associated stage-2 MMU if the vCPU is
698 * scheduling out and not in WFI emulation, suggesting it is likely to765 * scheduling out and not in WFI emulation, suggesting it is likely to
@@ -743,6 +810,245 @@ int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2) · 44 unmodified lines
743 return kvm_inject_nested_sync(vcpu, esr_el2);810 return kvm_inject_nested_sync(vcpu, esr_el2);
744}811}
745 812
813static void invalidate_vncr(struct vncr_tlb *vt)
814{
815 vt->valid = false;
816 if (vt->cpu != -1)
817 clear_fixmap(vncr_fixmap(vt->cpu));
818}
819
820static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end)
821{
822 struct kvm_vcpu *vcpu;
823 unsigned long i;
824
825 lockdep_assert_held_write(&kvm->mmu_lock);
826
827 if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY))
828 return;
829
830 kvm_for_each_vcpu(i, vcpu, kvm) {
831 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
832 u64 ipa_start, ipa_end, ipa_size;
833
834 /*
835 * Careful here: We end-up here from an MMU notifier,
836 * and this can race against a vcpu not being onlined
837 * yet, without the pseudo-TLB being allocated.
838 *
839 * Skip those, as they obviously don't participate in
840 * the invalidation at this stage.
841 */
842 if (!vt)
843 continue;
844
845 if (!vt->valid)
846 continue;
847
848 ipa_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift,
849 vt->wr.level));
850 ipa_start = vt->wr.pa & (ipa_size - 1);
851 ipa_end = ipa_start + ipa_size;
852
853 if (ipa_end <= start || ipa_start >= end)
854 continue;
855
856 invalidate_vncr(vt);
857 }
858}
859
860struct s1e2_tlbi_scope {
861 enum {
862 TLBI_ALL,
863 TLBI_VA,
864 TLBI_VAA,
865 TLBI_ASID,
866 } type;
867
868 u16 asid;
869 u64 va;
870 u64 size;
871};
872
873static void invalidate_vncr_va(struct kvm *kvm,
874 struct s1e2_tlbi_scope *scope)
875{
876 struct kvm_vcpu *vcpu;
877 unsigned long i;
878
879 lockdep_assert_held_write(&kvm->mmu_lock);
880
881 kvm_for_each_vcpu(i, vcpu, kvm) {
882 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
883 u64 va_start, va_end, va_size;
884
885 if (!vt->valid)
886 continue;
887
888 va_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift,
889 vt->wr.level));
890 va_start = vt->gva & (va_size - 1);
891 va_end = va_start + va_size;
892
893 switch (scope->type) {
894 case TLBI_ALL:
895 break;
896
897 case TLBI_VA:
898 if (va_end <= scope->va ||
899 va_start >= (scope->va + scope->size))
900 continue;
901 if (vt->wr.nG && vt->wr.asid != scope->asid)
902 continue;
903 break;
904
905 case TLBI_VAA:
906 if (va_end <= scope->va ||
907 va_start >= (scope->va + scope->size))
908 continue;
909 break;
910
911 case TLBI_ASID:
912 if (!vt->wr.nG || vt->wr.asid != scope->asid)
913 continue;
914 break;
915 }
916
917 invalidate_vncr(vt);
918 }
919}
920
921static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val,
922 struct s1e2_tlbi_scope *scope)
923{
924 switch (inst) {
925 case OP_TLBI_ALLE2:
926 case OP_TLBI_ALLE2IS:
927 case OP_TLBI_ALLE2OS:
928 case OP_TLBI_VMALLE1:
929 case OP_TLBI_VMALLE1IS:
930 case OP_TLBI_VMALLE1OS:
931 case OP_TLBI_ALLE2NXS:
932 case OP_TLBI_ALLE2ISNXS:
933 case OP_TLBI_ALLE2OSNXS:
934 case OP_TLBI_VMALLE1NXS:
935 case OP_TLBI_VMALLE1ISNXS:
936 case OP_TLBI_VMALLE1OSNXS:
937 scope->type = TLBI_ALL;
938 break;
939 case OP_TLBI_VAE2:
940 case OP_TLBI_VAE2IS:
941 case OP_TLBI_VAE2OS:
942 case OP_TLBI_VAE1:
943 case OP_TLBI_VAE1IS:
944 case OP_TLBI_VAE1OS:
945 case OP_TLBI_VAE2NXS:
946 case OP_TLBI_VAE2ISNXS:
947 case OP_TLBI_VAE2OSNXS:
948 case OP_TLBI_VAE1NXS:
949 case OP_TLBI_VAE1ISNXS:
950 case OP_TLBI_VAE1OSNXS:
951 case OP_TLBI_VALE2:
952 case OP_TLBI_VALE2IS:
953 case OP_TLBI_VALE2OS:
954 case OP_TLBI_VALE1:
955 case OP_TLBI_VALE1IS:
956 case OP_TLBI_VALE1OS:
957 case OP_TLBI_VALE2NXS:
958 case OP_TLBI_VALE2ISNXS:
959 case OP_TLBI_VALE2OSNXS:
960 case OP_TLBI_VALE1NXS:
961 case OP_TLBI_VALE1ISNXS:
962 case OP_TLBI_VALE1OSNXS:
963 scope->type = TLBI_VA;
964 scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val));
965 if (!scope->size)
966 scope->size = SZ_1G;
967 scope->va = (val << 12) & ~(scope->size - 1);
968 scope->asid = FIELD_GET(TLBIR_ASID_MASK, val);
969 break;
970 case OP_TLBI_ASIDE1:
971 case OP_TLBI_ASIDE1IS:
972 case OP_TLBI_ASIDE1OS:
973 case OP_TLBI_ASIDE1NXS:
974 case OP_TLBI_ASIDE1ISNXS:
975 case OP_TLBI_ASIDE1OSNXS:
976 scope->type = TLBI_ASID;
977 scope->asid = FIELD_GET(TLBIR_ASID_MASK, val);
978 break;
979 case OP_TLBI_VAAE1:
980 case OP_TLBI_VAAE1IS:
981 case OP_TLBI_VAAE1OS:
982 case OP_TLBI_VAAE1NXS:
983 case OP_TLBI_VAAE1ISNXS:
984 case OP_TLBI_VAAE1OSNXS:
985 case OP_TLBI_VAALE1:
986 case OP_TLBI_VAALE1IS:
987 case OP_TLBI_VAALE1OS:
988 case OP_TLBI_VAALE1NXS:
989 case OP_TLBI_VAALE1ISNXS:
990 case OP_TLBI_VAALE1OSNXS:
991 scope->type = TLBI_VAA;
992 scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val));
993 if (!scope->size)
994 scope->size = SZ_1G;
995 scope->va = (val << 12) & ~(scope->size - 1);
996 break;
997 case OP_TLBI_RVAE2:
998 case OP_TLBI_RVAE2IS:
999 case OP_TLBI_RVAE2OS:
1000 case OP_TLBI_RVAE1:
1001 case OP_TLBI_RVAE1IS:
1002 case OP_TLBI_RVAE1OS:
1003 case OP_TLBI_RVAE2NXS:
1004 case OP_TLBI_RVAE2ISNXS:
1005 case OP_TLBI_RVAE2OSNXS:
1006 case OP_TLBI_RVAE1NXS:
1007 case OP_TLBI_RVAE1ISNXS:
1008 case OP_TLBI_RVAE1OSNXS:
1009 case OP_TLBI_RVALE2:
1010 case OP_TLBI_RVALE2IS:
1011 case OP_TLBI_RVALE2OS:
1012 case OP_TLBI_RVALE1:
1013 case OP_TLBI_RVALE1IS:
1014 case OP_TLBI_RVALE1OS:
1015 case OP_TLBI_RVALE2NXS:
1016 case OP_TLBI_RVALE2ISNXS:
1017 case OP_TLBI_RVALE2OSNXS:
1018 case OP_TLBI_RVALE1NXS:
1019 case OP_TLBI_RVALE1ISNXS:
1020 case OP_TLBI_RVALE1OSNXS:
1021 scope->type = TLBI_VA;
1022 scope->va = decode_range_tlbi(val, &scope->size, &scope->asid);
1023 break;
1024 case OP_TLBI_RVAAE1:
1025 case OP_TLBI_RVAAE1IS:
1026 case OP_TLBI_RVAAE1OS:
1027 case OP_TLBI_RVAAE1NXS:
1028 case OP_TLBI_RVAAE1ISNXS:
1029 case OP_TLBI_RVAAE1OSNXS:
1030 case OP_TLBI_RVAALE1:
1031 case OP_TLBI_RVAALE1IS:
1032 case OP_TLBI_RVAALE1OS:
1033 case OP_TLBI_RVAALE1NXS:
1034 case OP_TLBI_RVAALE1ISNXS:
1035 case OP_TLBI_RVAALE1OSNXS:
1036 scope->type = TLBI_VAA;
1037 scope->va = decode_range_tlbi(val, &scope->size, NULL);
1038 break;
1039 }
1040}
1041
1042void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val)
1043{
1044 struct s1e2_tlbi_scope scope = {};
1045
1046 compute_s1_tlbi_range(vcpu, inst, val, &scope);
1047
1048 guard(write_lock)(&vcpu->kvm->mmu_lock);
1049 invalidate_vncr_va(vcpu->kvm, &scope);
1050}
1051
746void kvm_nested_s2_wp(struct kvm *kvm)1052void kvm_nested_s2_wp(struct kvm *kvm)
747{1053{
748 int i;1054 int i;
@@ -755,6 +1061,8 @@ void kvm_nested_s2_wp(struct kvm *kvm) · 6 unmodified lines
755 if (kvm_s2_mmu_valid(mmu))1061 if (kvm_s2_mmu_valid(mmu))
756 kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu));1062 kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu));
757 }1063 }
1064
1065 kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));
758}1066}
759 1067
760void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block)1068void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block)
@@ -769,6 +1077,8 @@ void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) · 8 unmodified lines
769 if (kvm_s2_mmu_valid(mmu))1077 if (kvm_s2_mmu_valid(mmu))
770 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block);1078 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block);
771 }1079 }
1080
1081 kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));
772}1082}
773 1083
774void kvm_nested_s2_flush(struct kvm *kvm)1084void kvm_nested_s2_flush(struct kvm *kvm)
@@ -801,6 +1111,295 @@ void kvm_arch_flush_shadow_all(struct kvm *kvm) · 26 unmodified lines
801 kvm_uninit_stage2_mmu(kvm);1111 kvm_uninit_stage2_mmu(kvm);
802}1112}
803 1113
1114/*
1115 * Dealing with VNCR_EL2 exposed by the *guest* is a complicated matter:
1116 *
1117 * - We introduce an internal representation of a vcpu-private TLB,
1118 * representing the mapping between the guest VA contained in VNCR_EL2,
1119 * the IPA the guest's EL2 PTs point to, and the actual PA this lives at.
1120 *
1121 * - On translation fault from a nested VNCR access, we create such a TLB.
1122 * If there is no mapping to describe, the guest inherits the fault.
1123 * Crucially, no actual mapping is done at this stage.
1124 *
1125 * - On vcpu_load() in a non-HYP context with HCR_EL2.NV==1, if the above
1126 * TLB exists, we map it in the fixmap for this CPU, and run with it. We
1127 * have to respect the permissions dictated by the guest, but not the
1128 * memory type (FWB is a must).
1129 *
1130 * - Note that we usually don't do a vcpu_load() on the back of a fault
1131 * (unless we are preempted), so the resolution of a translation fault
1132 * must go via a request that will map the VNCR page in the fixmap.
1133 * vcpu_load() might as well use the same mechanism.
1134 *
1135 * - On vcpu_put() in a non-HYP context with HCR_EL2.NV==1, if the TLB was
1136 * mapped, we unmap it. Yes it is that simple. The TLB still exists
1137 * though, and may be reused at a later load.
1138 *
1139 * - On permission fault, we simply forward the fault to the guest's EL2.
1140 * Get out of my way.
1141 *
1142 * - On any TLBI for the EL2&0 translation regime, we must find any TLB that
1143 * intersects with the TLBI request, invalidate it, and unmap the page
1144 * from the fixmap. Because we need to look at all the vcpu-private TLBs,
1145 * this requires some wide-ranging locking to ensure that nothing races
1146 * against it. This may require some refcounting to avoid the search when
1147 * no such TLB is present.
1148 *
1149 * - On MMU notifiers, we must invalidate our TLB in a similar way, but
1150 * looking at the IPA instead. The funny part is that there may not be a
1151 * stage-2 mapping for this page if L1 hasn't accessed it using LD/ST
1152 * instructions.
1153 */
1154
1155int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu)
1156{
1157 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY))
1158 return 0;
1159
1160 vcpu->arch.vncr_tlb = kzalloc(sizeof(*vcpu->arch.vncr_tlb),
1161 GFP_KERNEL_ACCOUNT);
1162 if (!vcpu->arch.vncr_tlb)
1163 return -ENOMEM;
1164
1165 return 0;
1166}
1167
1168static u64 read_vncr_el2(struct kvm_vcpu *vcpu)
1169{
1170 return (u64)sign_extend64(__vcpu_sys_reg(vcpu, VNCR_EL2), 48);
1171}
1172
1173static int kvm_translate_vncr(struct kvm_vcpu *vcpu)
1174{
1175 bool write_fault, writable;
1176 unsigned long mmu_seq;
1177 struct vncr_tlb *vt;
1178 struct page *page;
1179 u64 va, pfn, gfn;
1180 int ret;
1181
1182 vt = vcpu->arch.vncr_tlb;
1183
1184 /*
1185 * If we're about to walk the EL2 S1 PTs, we must invalidate the
1186 * current TLB, as it could be sampled from another vcpu doing a
1187 * TLBI *IS. A real CPU wouldn't do that, but we only keep a single
1188 * translation, so not much of a choice.
1189 *
1190 * We also prepare the next walk wilst we're at it.
1191 */
1192 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) {
1193 invalidate_vncr(vt);
1194
1195 vt->wi = (struct s1_walk_info) {
1196 .regime = TR_EL20,
1197 .as_el0 = false,
1198 .pan = false,
1199 };
1200 vt->wr = (struct s1_walk_result){};
1201 }
1202
1203 guard(srcu)(&vcpu->kvm->srcu);
1204
1205 va = read_vncr_el2(vcpu);
1206
1207 ret = __kvm_translate_va(vcpu, &vt->wi, &vt->wr, va);
1208 if (ret)
1209 return ret;
1210
1211 write_fault = kvm_is_write_fault(vcpu);
1212
1213 mmu_seq = vcpu->kvm->mmu_invalidate_seq;
1214 smp_rmb();
1215
1216 gfn = vt->wr.pa >> PAGE_SHIFT;
1217 pfn = kvm_faultin_pfn(vcpu, gfn, write_fault, &writable, &page);
1218 if (is_error_noslot_pfn(pfn) || (write_fault && !writable))
1219 return -EFAULT;
1220
1221 scoped_guard(write_lock, &vcpu->kvm->mmu_lock) {
1222 if (mmu_invalidate_retry(vcpu->kvm, mmu_seq))
1223 return -EAGAIN;
1224
1225 vt->gva = va;
1226 vt->hpa = pfn << PAGE_SHIFT;
1227 vt->valid = true;
1228 vt->cpu = -1;
1229
1230 kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu);
1231 kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw);
1232 }
1233
1234 if (vt->wr.pw)
1235 mark_page_dirty(vcpu->kvm, gfn);
1236
1237 return 0;
1238}
1239
1240static void inject_vncr_perm(struct kvm_vcpu *vcpu)
1241{
1242 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1243 u64 esr = kvm_vcpu_get_esr(vcpu);
1244
1245 /* Adjust the fault level to reflect that of the guest's */
1246 esr &= ~ESR_ELx_FSC;
1247 esr |= FIELD_PREP(ESR_ELx_FSC,
1248 ESR_ELx_FSC_PERM_L(vt->wr.level));
1249
1250 kvm_inject_nested_sync(vcpu, esr);
1251}
1252
1253static bool kvm_vncr_tlb_lookup(struct kvm_vcpu *vcpu)
1254{
1255 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1256
1257 lockdep_assert_held_read(&vcpu->kvm->mmu_lock);
1258
1259 if (!vt->valid)
1260 return false;
1261
1262 if (read_vncr_el2(vcpu) != vt->gva)
1263 return false;
1264
1265 if (vt->wr.nG) {
1266 u64 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2);
1267 u64 ttbr = ((tcr & TCR_A1) ?
1268 vcpu_read_sys_reg(vcpu, TTBR1_EL2) :
1269 vcpu_read_sys_reg(vcpu, TTBR0_EL2));
1270 u16 asid;
1271
1272 asid = FIELD_GET(TTBR_ASID_MASK, ttbr);
1273 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) ||
1274 !(tcr & TCR_ASID16))
1275 asid &= GENMASK(7, 0);
1276
1277 return asid != vt->wr.asid;
1278 }
1279
1280 return true;
1281}
1282
1283int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu)
1284{
1285 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1286 u64 esr = kvm_vcpu_get_esr(vcpu);
1287
1288 BUG_ON(!(esr & ESR_ELx_VNCR_SHIFT));
1289
1290 if (esr_fsc_is_permission_fault(esr)) {
1291 inject_vncr_perm(vcpu);
1292 } else if (esr_fsc_is_translation_fault(esr)) {
1293 bool valid;
1294 int ret;
1295
1296 scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
1297 valid = kvm_vncr_tlb_lookup(vcpu);
1298
1299 if (!valid)
1300 ret = kvm_translate_vncr(vcpu);
1301 else
1302 ret = -EPERM;
1303
1304 switch (ret) {
1305 case -EAGAIN:
1306 case -ENOMEM:
1307 /* Let's try again... */
1308 break;
1309 case -EFAULT:
1310 case -EINVAL:
1311 case -ENOENT:
1312 case -EACCES:
1313 /*
1314 * Translation failed, inject the corresponding
1315 * exception back to EL2.
1316 */
1317 BUG_ON(!vt->wr.failed);
1318
1319 esr &= ~ESR_ELx_FSC;
1320 esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst);
1321
1322 kvm_inject_nested_sync(vcpu, esr);
1323 break;
1324 case -EPERM:
1325 /* Hack to deal with POE until we get kernel support */
1326 inject_vncr_perm(vcpu);
1327 break;
1328 case 0:
1329 break;
1330 }
1331 } else {
1332 WARN_ONCE(1, "Unhandled VNCR abort, ESR=%llx\n", esr);
1333 }
1334
1335 return 1;
1336}
1337
1338static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu)
1339{
1340 struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1341 pgprot_t prot;
1342
1343 guard(preempt)();
1344 guard(read_lock)(&vcpu->kvm->mmu_lock);
1345
1346 /*
1347 * The request to map VNCR may have raced against some other
1348 * event, such as an interrupt, and may not be valid anymore.
1349 */
1350 if (is_hyp_ctxt(vcpu))
1351 return;
1352
1353 /*
1354 * Check that the pseudo-TLB is valid and that VNCR_EL2 still
1355 * contains the expected value. If it doesn't, we simply bail out
1356 * without a mapping -- a transformed MSR/MRS will generate the
1357 * fault and allows us to populate the pseudo-TLB.
1358 */
1359 if (!vt->valid)
1360 return;
1361
1362 if (read_vncr_el2(vcpu) != vt->gva)
1363 return;
1364
1365 if (vt->wr.nG) {
1366 u64 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2);
1367 u64 ttbr = ((tcr & TCR_A1) ?
1368 vcpu_read_sys_reg(vcpu, TTBR1_EL2) :
1369 vcpu_read_sys_reg(vcpu, TTBR0_EL2));
1370 u16 asid;
1371
1372 asid = FIELD_GET(TTBR_ASID_MASK, ttbr);
1373 if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) ||
1374 !(tcr & TCR_ASID16))
1375 asid &= GENMASK(7, 0);
1376
1377 if (asid != vt->wr.asid)
1378 return;
1379 }
1380
1381 vt->cpu = smp_processor_id();
1382
1383 if (vt->wr.pw && vt->wr.pr)
1384 prot = PAGE_KERNEL;
1385 else if (vt->wr.pr)
1386 prot = PAGE_KERNEL_RO;
1387 else
1388 prot = PAGE_NONE;
1389
1390 /*
1391 * We can't map write-only (or no permission at all) in the kernel,
1392 * but the guest can do it if using POE, so we'll have to turn a
1393 * translation fault into a permission fault at runtime.
1394 * FIXME: WO doesn't work at all, need POE support in the kernel.
1395 */
1396 if (pgprot_val(prot) != pgprot_val(PAGE_NONE)) {
1397 __set_fixmap(vncr_fixmap(vt->cpu), vt->hpa, prot);
1398 host_data_set_flag(L1_VNCR_MAPPED);
1399 atomic_inc(&vcpu->kvm->arch.vncr_map_count);
1400 }
1401}
1402
804/*1403/*
805 * Our emulated CPU doesn't support all the possible features. For the1404 * Our emulated CPU doesn't support all the possible features. For the
806 * sake of simplicity (and probably mental sanity), wipe out a number1405 * sake of simplicity (and probably mental sanity), wipe out a number
@@ -1151,6 +1750,9 @@ int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu) · 344 unmodified lines
1151 res0 |= ICH_HCR_EL2_DVIM | ICH_HCR_EL2_vSGIEOICount;1750 res0 |= ICH_HCR_EL2_DVIM | ICH_HCR_EL2_vSGIEOICount;
1152 set_sysreg_masks(kvm, ICH_HCR_EL2, res0, res1);1751 set_sysreg_masks(kvm, ICH_HCR_EL2, res0, res1);
1153 1752
1753 /* VNCR_EL2 */
1754 set_sysreg_masks(kvm, VNCR_EL2, VNCR_EL2_RES0, VNCR_EL2_RES1);
1755
1154out:1756out:
1155 for (enum vcpu_sysreg sr = __SANITISED_REG_START__; sr < NR_SYS_REGS; sr++)1757 for (enum vcpu_sysreg sr = __SANITISED_REG_START__; sr < NR_SYS_REGS; sr++)
1156 (void)__vcpu_sys_reg(vcpu, sr);1758 (void)__vcpu_sys_reg(vcpu, sr);
@@ -1171,6 +1773,9 @@ void check_nested_vcpu_requests(struct kvm_vcpu *vcpu) · 14 unmodified lines
1171 write_unlock(&vcpu->kvm->mmu_lock);1773 write_unlock(&vcpu->kvm->mmu_lock);
1172 }1774 }
1173 1775
1776 if (kvm_check_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu))
1777 kvm_map_l1_vncr(vcpu);
1778
1174 /* Must be last, as may switch context! */1779 /* Must be last, as may switch context! */
1175 if (kvm_check_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu))1780 if (kvm_check_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu))
1176 kvm_inject_nested_irq(vcpu);1781 kvm_inject_nested_irq(vcpu);
arch/arm64/kvm/reset.c+2−0
@@ -158,6 +158,8 @@ void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu) · 157 unmodified lines
158 if (sve_state)158 if (sve_state)
159 kvm_unshare_hyp(sve_state, sve_state + vcpu_sve_state_size(vcpu));159 kvm_unshare_hyp(sve_state, sve_state + vcpu_sve_state_size(vcpu));
160 kfree(sve_state);160 kfree(sve_state);
161 free_page((unsigned long)vcpu->arch.ctxt.vncr_array);
162 kfree(vcpu->arch.vncr_tlb);
161 kfree(vcpu->arch.ccsidr);163 kfree(vcpu->arch.ccsidr);
162}164}
163 165
arch/arm64/kvm/sys_regs.c+78−57
@@ -2306,15 +2306,6 @@ static bool bad_redir_trap(struct kvm_vcpu *vcpu, · 2305 unmodified lines
2306 "trap of EL2 register redirected to EL1");2306 "trap of EL2 register redirected to EL1");
2307}2307}
2308 2308
2309#define EL2_REG(name, acc, rst, v) { \
2310 SYS_DESC(SYS_##name), \
2311 .access = acc, \
2312 .reset = rst, \
2313 .reg = name, \
2314 .visibility = el2_visibility, \
2315 .val = v, \
2316}
2317
2318#define EL2_REG_FILTERED(name, acc, rst, v, filter) { \2309#define EL2_REG_FILTERED(name, acc, rst, v, filter) { \
2319 SYS_DESC(SYS_##name), \2310 SYS_DESC(SYS_##name), \
2320 .access = acc, \2311 .access = acc, \
@@ -2324,6 +2315,9 @@ static bool bad_redir_trap(struct kvm_vcpu *vcpu, · 3 unmodified lines
2324 .val = v, \2315 .val = v, \
2325}2316}
2326 2317
2318#define EL2_REG(name, acc, rst, v) \
2319 EL2_REG_FILTERED(name, acc, rst, v, el2_visibility)
2320
2327#define EL2_REG_VNCR(name, rst, v) EL2_REG(name, bad_vncr_trap, rst, v)2321#define EL2_REG_VNCR(name, rst, v) EL2_REG(name, bad_vncr_trap, rst, v)
2328#define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v)2322#define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v)
2329 2323
@@ -2471,6 +2465,16 @@ static unsigned int sve_el2_visibility(const struct kvm_vcpu *vcpu, · 141 unmodified lines
2471 return __el2_visibility(vcpu, rd, sve_visibility);2465 return __el2_visibility(vcpu, rd, sve_visibility);
2472}2466}
2473 2467
2468static unsigned int vncr_el2_visibility(const struct kvm_vcpu *vcpu,
2469 const struct sys_reg_desc *rd)
2470{
2471 if (el2_visibility(vcpu, rd) == 0 &&
2472 kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY))
2473 return 0;
2474
2475 return REG_HIDDEN;
2476}
2477
2474static bool access_zcr_el2(struct kvm_vcpu *vcpu,2478static bool access_zcr_el2(struct kvm_vcpu *vcpu,
2475 struct sys_reg_params *p,2479 struct sys_reg_params *p,
2476 const struct sys_reg_desc *r)2480 const struct sys_reg_desc *r)
@@ -3311,6 +3315,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 834 unmodified lines
3311 tcr2_el2_visibility),3315 tcr2_el2_visibility),
3312 EL2_REG_VNCR(VTTBR_EL2, reset_val, 0),3316 EL2_REG_VNCR(VTTBR_EL2, reset_val, 0),
3313 EL2_REG_VNCR(VTCR_EL2, reset_val, 0),3317 EL2_REG_VNCR(VTCR_EL2, reset_val, 0),
3318 EL2_REG_FILTERED(VNCR_EL2, bad_vncr_trap, reset_val, 0,
3319 vncr_el2_visibility),
3314 3320
3315 { SYS_DESC(SYS_DACR32_EL2), undef_access, reset_unknown, DACR32_EL2 },3321 { SYS_DESC(SYS_DACR32_EL2), undef_access, reset_unknown, DACR32_EL2 },
3316 EL2_REG_VNCR(HDFGRTR_EL2, reset_val, 0),3322 EL2_REG_VNCR(HDFGRTR_EL2, reset_val, 0),
@@ -3594,8 +3600,7 @@ static bool handle_ripas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 277 unmodified lines
3594{3600{
3595 u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2);3601 u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2);
3596 u64 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);3602 u64 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
3597 u64 base, range, tg, num, scale;3603 u64 base, range;
3598 int shift;
3599 3604
3600 if (!kvm_supported_tlbi_ipas2_op(vcpu, sys_encoding))3605 if (!kvm_supported_tlbi_ipas2_op(vcpu, sys_encoding))
3601 return undef_access(vcpu, p, r);3606 return undef_access(vcpu, p, r);
@@ -3605,26 +3610,7 @@ static bool handle_ripas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 3 unmodified lines
3605 * of the guest's S2 (different base granule size, for example), we3610 * of the guest's S2 (different base granule size, for example), we
3606 * decide to ignore TTL and only use the described range.3611 * decide to ignore TTL and only use the described range.
3607 */3612 */
3608 tg = FIELD_GET(GENMASK(47, 46), p->regval);3613 base = decode_range_tlbi(p->regval, &range, NULL);
3609 scale = FIELD_GET(GENMASK(45, 44), p->regval);
3610 num = FIELD_GET(GENMASK(43, 39), p->regval);
3611 base = p->regval & GENMASK(36, 0);
3612
3613 switch(tg) {
3614 case 1:
3615 shift = 12;
3616 break;
3617 case 2:
3618 shift = 14;
3619 break;
3620 case 3:
3621 default: /* IMPDEF: handle tg==0 as 64k */
3622 shift = 16;
3623 break;
3624 }
3625
3626 base <<= shift;
3627 range = __TLBI_RANGE_PAGES(num, scale) << shift;
3628 3614
3629 kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, get_vmid(vttbr),3615 kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, get_vmid(vttbr),
3630 &(union tlbi_info) {3616 &(union tlbi_info) {
@@ -3690,11 +3676,22 @@ static void s2_mmu_tlbi_s1e1(struct kvm_s2_mmu *mmu, · 59 unmodified lines
3690 WARN_ON(__kvm_tlbi_s1e2(mmu, info->va.addr, info->va.encoding));3676 WARN_ON(__kvm_tlbi_s1e2(mmu, info->va.addr, info->va.encoding));
3691}3677}
3692 3678
3679static bool handle_tlbi_el2(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
3680 const struct sys_reg_desc *r)
3681{
3682 u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2);
3683
3684 if (!kvm_supported_tlbi_s1e2_op(vcpu, sys_encoding))
3685 return undef_access(vcpu, p, r);
3686
3687 kvm_handle_s1e2_tlbi(vcpu, sys_encoding, p->regval);
3688 return true;
3689}
3690
3693static bool handle_tlbi_el1(struct kvm_vcpu *vcpu, struct sys_reg_params *p,3691static bool handle_tlbi_el1(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
3694 const struct sys_reg_desc *r)3692 const struct sys_reg_desc *r)
3695{3693{
3696 u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2);3694 u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2);
3697 u64 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
3698 3695
3699 /*3696 /*
3700 * If we're here, this is because we've trapped on a EL1 TLBI3697 * If we're here, this is because we've trapped on a EL1 TLBI
@@ -3705,6 +3702,13 @@ static bool handle_tlbi_el1(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 4 unmodified lines
3705 * - HCR_EL2.E2H == 0 : a non-VHE guest3702 * - HCR_EL2.E2H == 0 : a non-VHE guest
3706 * - HCR_EL2.{E2H,TGE} == { 1, 0 } : a VHE guest in guest mode3703 * - HCR_EL2.{E2H,TGE} == { 1, 0 } : a VHE guest in guest mode
3707 *3704 *
3705 * Another possibility is that we are invalidating the EL2 context
3706 * using EL1 instructions, but that we landed here because we need
3707 * additional invalidation for structures that are not held in the
3708 * CPU TLBs (such as the VNCR pseudo-TLB and its EL2 mapping). In
3709 * that case, we are guaranteed that HCR_EL2.{E2H,TGE} == { 1, 1 }
3710 * as we don't allow an NV-capable L1 in a nVHE configuration.
3711 *
3708 * We don't expect these helpers to ever be called when running3712 * We don't expect these helpers to ever be called when running
3709 * in a vEL1 context.3713 * in a vEL1 context.
3710 */3714 */
@@ -3714,7 +3718,13 @@ static bool handle_tlbi_el1(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 3 unmodified lines
3714 if (!kvm_supported_tlbi_s1e1_op(vcpu, sys_encoding))3718 if (!kvm_supported_tlbi_s1e1_op(vcpu, sys_encoding))
3715 return undef_access(vcpu, p, r);3719 return undef_access(vcpu, p, r);
3716 3720
3717 kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, get_vmid(vttbr),3721 if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu)) {
3722 kvm_handle_s1e2_tlbi(vcpu, sys_encoding, p->regval);
3723 return true;
3724 }
3725
3726 kvm_s2_mmu_iterate_by_vmid(vcpu->kvm,
3727 get_vmid(__vcpu_sys_reg(vcpu, VTTBR_EL2)),
3718 &(union tlbi_info) {3728 &(union tlbi_info) {
3719 .va = {3729 .va = {
3720 .addr = p->regval,3730 .addr = p->regval,
@@ -3836,16 +3846,21 @@ static struct sys_reg_desc sys_insn_descs[] = { · 115 unmodified lines
3836 SYS_INSN(TLBI_IPAS2LE1IS, handle_ipas2e1is),3846 SYS_INSN(TLBI_IPAS2LE1IS, handle_ipas2e1is),
3837 SYS_INSN(TLBI_RIPAS2LE1IS, handle_ripas2e1is),3847 SYS_INSN(TLBI_RIPAS2LE1IS, handle_ripas2e1is),
3838 3848
3839 SYS_INSN(TLBI_ALLE2OS, undef_access),3849 SYS_INSN(TLBI_ALLE2OS, handle_tlbi_el2),
3840 SYS_INSN(TLBI_VAE2OS, undef_access),3850 SYS_INSN(TLBI_VAE2OS, handle_tlbi_el2),
3841 SYS_INSN(TLBI_ALLE1OS, handle_alle1is),3851 SYS_INSN(TLBI_ALLE1OS, handle_alle1is),
3842 SYS_INSN(TLBI_VALE2OS, undef_access),3852 SYS_INSN(TLBI_VALE2OS, handle_tlbi_el2),
3843 SYS_INSN(TLBI_VMALLS12E1OS, handle_vmalls12e1is),3853 SYS_INSN(TLBI_VMALLS12E1OS, handle_vmalls12e1is),
3844 3854
3845 SYS_INSN(TLBI_RVAE2IS, undef_access),3855 SYS_INSN(TLBI_RVAE2IS, handle_tlbi_el2),
3846 SYS_INSN(TLBI_RVALE2IS, undef_access),3856 SYS_INSN(TLBI_RVALE2IS, handle_tlbi_el2),
3857 SYS_INSN(TLBI_ALLE2IS, handle_tlbi_el2),
3858 SYS_INSN(TLBI_VAE2IS, handle_tlbi_el2),
3847 3859
3848 SYS_INSN(TLBI_ALLE1IS, handle_alle1is),3860 SYS_INSN(TLBI_ALLE1IS, handle_alle1is),
3861
3862 SYS_INSN(TLBI_VALE2IS, handle_tlbi_el2),
3863
3849 SYS_INSN(TLBI_VMALLS12E1IS, handle_vmalls12e1is),3864 SYS_INSN(TLBI_VMALLS12E1IS, handle_vmalls12e1is),
3850 SYS_INSN(TLBI_IPAS2E1OS, handle_ipas2e1is),3865 SYS_INSN(TLBI_IPAS2E1OS, handle_ipas2e1is),
3851 SYS_INSN(TLBI_IPAS2E1, handle_ipas2e1is),3866 SYS_INSN(TLBI_IPAS2E1, handle_ipas2e1is),
@@ -3855,11 +3870,17 @@ static struct sys_reg_desc sys_insn_descs[] = { · 3 unmodified lines
3855 SYS_INSN(TLBI_IPAS2LE1, handle_ipas2e1is),3870 SYS_INSN(TLBI_IPAS2LE1, handle_ipas2e1is),
3856 SYS_INSN(TLBI_RIPAS2LE1, handle_ripas2e1is),3871 SYS_INSN(TLBI_RIPAS2LE1, handle_ripas2e1is),
3857 SYS_INSN(TLBI_RIPAS2LE1OS, handle_ripas2e1is),3872 SYS_INSN(TLBI_RIPAS2LE1OS, handle_ripas2e1is),
3858 SYS_INSN(TLBI_RVAE2OS, undef_access),3873 SYS_INSN(TLBI_RVAE2OS, handle_tlbi_el2),
3859 SYS_INSN(TLBI_RVALE2OS, undef_access),3874 SYS_INSN(TLBI_RVALE2OS, handle_tlbi_el2),
3860 SYS_INSN(TLBI_RVAE2, undef_access),3875 SYS_INSN(TLBI_RVAE2, handle_tlbi_el2),
3861 SYS_INSN(TLBI_RVALE2, undef_access),3876 SYS_INSN(TLBI_RVALE2, handle_tlbi_el2),
3877 SYS_INSN(TLBI_ALLE2, handle_tlbi_el2),
3878 SYS_INSN(TLBI_VAE2, handle_tlbi_el2),
3879
3862 SYS_INSN(TLBI_ALLE1, handle_alle1is),3880 SYS_INSN(TLBI_ALLE1, handle_alle1is),
3881
3882 SYS_INSN(TLBI_VALE2, handle_tlbi_el2),
3883
3863 SYS_INSN(TLBI_VMALLS12E1, handle_vmalls12e1is),3884 SYS_INSN(TLBI_VMALLS12E1, handle_vmalls12e1is),
3864 3885
3865 SYS_INSN(TLBI_IPAS2E1ISNXS, handle_ipas2e1is),3886 SYS_INSN(TLBI_IPAS2E1ISNXS, handle_ipas2e1is),
@@ -3867,19 +3888,19 @@ static struct sys_reg_desc sys_insn_descs[] = { · 1 unmodified line
3867 SYS_INSN(TLBI_IPAS2LE1ISNXS, handle_ipas2e1is),3888 SYS_INSN(TLBI_IPAS2LE1ISNXS, handle_ipas2e1is),
3868 SYS_INSN(TLBI_RIPAS2LE1ISNXS, handle_ripas2e1is),3889 SYS_INSN(TLBI_RIPAS2LE1ISNXS, handle_ripas2e1is),
3869 3890
3870 SYS_INSN(TLBI_ALLE2OSNXS, undef_access),3891 SYS_INSN(TLBI_ALLE2OSNXS, handle_tlbi_el2),
3871 SYS_INSN(TLBI_VAE2OSNXS, undef_access),3892 SYS_INSN(TLBI_VAE2OSNXS, handle_tlbi_el2),
3872 SYS_INSN(TLBI_ALLE1OSNXS, handle_alle1is),3893 SYS_INSN(TLBI_ALLE1OSNXS, handle_alle1is),
3873 SYS_INSN(TLBI_VALE2OSNXS, undef_access),3894 SYS_INSN(TLBI_VALE2OSNXS, handle_tlbi_el2),
3874 SYS_INSN(TLBI_VMALLS12E1OSNXS, handle_vmalls12e1is),3895 SYS_INSN(TLBI_VMALLS12E1OSNXS, handle_vmalls12e1is),
3875 3896
3876 SYS_INSN(TLBI_RVAE2ISNXS, undef_access),3897 SYS_INSN(TLBI_RVAE2ISNXS, handle_tlbi_el2),
3877 SYS_INSN(TLBI_RVALE2ISNXS, undef_access),3898 SYS_INSN(TLBI_RVALE2ISNXS, handle_tlbi_el2),
3878 SYS_INSN(TLBI_ALLE2ISNXS, undef_access),3899 SYS_INSN(TLBI_ALLE2ISNXS, handle_tlbi_el2),
3879 SYS_INSN(TLBI_VAE2ISNXS, undef_access),3900 SYS_INSN(TLBI_VAE2ISNXS, handle_tlbi_el2),
3880 3901
3881 SYS_INSN(TLBI_ALLE1ISNXS, handle_alle1is),3902 SYS_INSN(TLBI_ALLE1ISNXS, handle_alle1is),
3882 SYS_INSN(TLBI_VALE2ISNXS, undef_access),3903 SYS_INSN(TLBI_VALE2ISNXS, handle_tlbi_el2),
3883 SYS_INSN(TLBI_VMALLS12E1ISNXS, handle_vmalls12e1is),3904 SYS_INSN(TLBI_VMALLS12E1ISNXS, handle_vmalls12e1is),
3884 SYS_INSN(TLBI_IPAS2E1OSNXS, handle_ipas2e1is),3905 SYS_INSN(TLBI_IPAS2E1OSNXS, handle_ipas2e1is),
3885 SYS_INSN(TLBI_IPAS2E1NXS, handle_ipas2e1is),3906 SYS_INSN(TLBI_IPAS2E1NXS, handle_ipas2e1is),
@@ -3889,14 +3910,14 @@ static struct sys_reg_desc sys_insn_descs[] = { · 3 unmodified lines
3889 SYS_INSN(TLBI_IPAS2LE1NXS, handle_ipas2e1is),3910 SYS_INSN(TLBI_IPAS2LE1NXS, handle_ipas2e1is),
3890 SYS_INSN(TLBI_RIPAS2LE1NXS, handle_ripas2e1is),3911 SYS_INSN(TLBI_RIPAS2LE1NXS, handle_ripas2e1is),
3891 SYS_INSN(TLBI_RIPAS2LE1OSNXS, handle_ripas2e1is),3912 SYS_INSN(TLBI_RIPAS2LE1OSNXS, handle_ripas2e1is),
3892 SYS_INSN(TLBI_RVAE2OSNXS, undef_access),3913 SYS_INSN(TLBI_RVAE2OSNXS, handle_tlbi_el2),
3893 SYS_INSN(TLBI_RVALE2OSNXS, undef_access),3914 SYS_INSN(TLBI_RVALE2OSNXS, handle_tlbi_el2),
3894 SYS_INSN(TLBI_RVAE2NXS, undef_access),3915 SYS_INSN(TLBI_RVAE2NXS, handle_tlbi_el2),
3895 SYS_INSN(TLBI_RVALE2NXS, undef_access),3916 SYS_INSN(TLBI_RVALE2NXS, handle_tlbi_el2),
3896 SYS_INSN(TLBI_ALLE2NXS, undef_access),3917 SYS_INSN(TLBI_ALLE2NXS, handle_tlbi_el2),
3897 SYS_INSN(TLBI_VAE2NXS, undef_access),3918 SYS_INSN(TLBI_VAE2NXS, handle_tlbi_el2),
3898 SYS_INSN(TLBI_ALLE1NXS, handle_alle1is),3919 SYS_INSN(TLBI_ALLE1NXS, handle_alle1is),
3899 SYS_INSN(TLBI_VALE2NXS, undef_access),3920 SYS_INSN(TLBI_VALE2NXS, handle_tlbi_el2),
3900 SYS_INSN(TLBI_VMALLS12E1NXS, handle_vmalls12e1is),3921 SYS_INSN(TLBI_VMALLS12E1NXS, handle_vmalls12e1is),
3901};3922};
3902 3923
arch/arm64/tools/sysreg+6−0
@@ -3816,6 +3816,12 @@ Sysreg SMCR_EL2 3 4 1 2 6 · 3815 unmodified lines
3816Fields SMCR_ELx3816Fields SMCR_ELx
3817EndSysreg3817EndSysreg
3818 3818
3819Sysreg VNCR_EL2 3 4 2 2 0
3820Field 63:57 RESS
3821Field 56:12 BADDR
3822Res0 11:0
3823EndSysreg
3824
3819Sysreg GCSCR_EL2 3 4 2 5 03825Sysreg GCSCR_EL2 3 4 2 5 0
3820Fields GCSCR_ELx3826Fields GCSCR_ELx
3821EndSysreg3827EndSysreg
include/uapi/linux/kvm.h+2−0
@@ -930,6 +930,8 @@ struct kvm_enable_cap { · 929 unmodified lines
930#define KVM_CAP_X86_APIC_BUS_CYCLES_NS 237930#define KVM_CAP_X86_APIC_BUS_CYCLES_NS 237
931#define KVM_CAP_X86_GUEST_MODE 238931#define KVM_CAP_X86_GUEST_MODE 238
932#define KVM_CAP_ARM_WRITABLE_IMP_ID_REGS 239932#define KVM_CAP_ARM_WRITABLE_IMP_ID_REGS 239
933#define KVM_CAP_ARM_EL2 240
934#define KVM_CAP_ARM_EL2_E2H0 241
933 935
934struct kvm_irq_routing_irqchip {936struct kvm_irq_routing_irqchip {
935 __u32 irqchip;937 __u32 irqchip;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t