History

Merge branch kvm-arm64/nv-nv into kvmarm-master/next

* kvm-arm64/nv-nv: : . : Flick the switch on the NV support by adding the missing piece : in the form of the VNCR page management. From the cover letter: : : "This is probably the most interesting bit of the whole NV adventure. : So far, everything else has been a walk in the park, but this one is : where the real fun takes place. : : With FEAT_NV2, most of the NV support revolves around tricking a guest : into accessing memory while it tries to access system registers. The : hypervisor's job is to handle the context switch of the actual : registers with the state in memory as needed." : . KVM: arm64: nv: Release faulted-in VNCR page from mmu_lock critical section KVM: arm64: nv: Handle TLBI S1E2 for VNCR invalidation with mmu_lock held KVM: arm64: nv: Hold mmu_lock when invalidating VNCR SW-TLB before translating KVM: arm64: Document NV caps and vcpu flags KVM: arm64: Allow userspace to request KVM_ARM_VCPU_EL2* KVM: arm64: nv: Remove dead code from ERET handling KVM: arm64: nv: Plumb TLBI S1E2 into system instruction dispatch KVM: arm64: nv: Add S1 TLB invalidation primitive for VNCR_EL2 KVM: arm64: nv: Program host's VNCR_EL2 to the fixmap address KVM: arm64: nv: Handle VNCR_EL2 invalidation from MMU notifiers KVM: arm64: nv: Handle mapping of VNCR_EL2 at EL2 KVM: arm64: nv: Handle VNCR_EL2-triggered faults KVM: arm64: nv: Add userspace and guest handling of VNCR_EL2 KVM: arm64: nv: Add pseudo-TLB backing VNCR_EL2 KVM: arm64: nv: Don't adjust PSTATE.M when L2 is nesting KVM: arm64: nv: Move TLBI range decoding to a helper KVM: arm64: nv: Snapshot S1 ASID tagging information during walk KVM: arm64: nv: Extract translation helper from the AT code KVM: arm64: nv: Allocate VNCR page when required arm64: sysreg: Add layout for VNCR_EL2

Marc Zyngier committed 1 year ago

Commit 7f3225f2 parents 35e4d7f 538fbac16 files · +956 −138Browse files

Unified · Split

16 files
  1. Documentation/virt/kvm/api.rst +13 −1
  2. arch/arm64/include/asm/esr.h +2 −0
  3. arch/arm64/include/asm/fixmap.h +6 −0
  4. arch/arm64/include/asm/kvm_host.h +14 −1
  5. arch/arm64/include/asm/kvm_nested.h +100 −0
  6. arch/arm64/include/asm/sysreg.h +1 −1
  7. arch/arm64/kvm/arm.c +10 −0
  8. arch/arm64/kvm/at.c +64 −59
  9. arch/arm64/kvm/emulate-nested.c +0 −7
  10. arch/arm64/kvm/handle_exit.c +1 −0
  11. arch/arm64/kvm/hyp/vhe/switch.c +43 −3
  12. arch/arm64/kvm/nested.c +614 −9
  13. arch/arm64/kvm/reset.c +2 −0
  14. arch/arm64/kvm/sys_regs.c +78 −57
  15. arch/arm64/tools/sysreg +6 −0
  16. include/uapi/linux/kvm.h +2 −0
Documentation/virt/kvm/api.rst+13−1
@@ -3460,7 +3460,8 @@ The initial values are defined as: · 3459 unmodified lines
34603460 - FPSIMD/NEON registers: set to 0
34613461 - SVE registers: set to 0
34623462 - System registers: Reset to their architecturally defined
3463− values as for a warm reset to EL1 (resp. SVC)
3463+ values as for a warm reset to EL1 (resp. SVC) or EL2 (in the
3464+ case of EL2 being enabled).
34643465
34653466 Note that because some registers reflect machine topology, all vcpus
34663467 should be created before this ioctl is invoked.
@@ -3527,6 +3528,17 @@ Possible features: · 60 unmodified lines
35273528 - the KVM_REG_ARM64_SVE_VLS pseudo-register is immutable, and can
35283529 no longer be written using KVM_SET_ONE_REG.
35293530
3531+ - KVM_ARM_VCPU_HAS_EL2: Enable Nested Virtualisation support,
3532+ booting the guest from EL2 instead of EL1.
3533+ Depends on KVM_CAP_ARM_EL2.
3534+ The VM is running with HCR_EL2.E2H being RES1 (VHE) unless
3535+ KVM_ARM_VCPU_HAS_EL2_E2H0 is also set.
3536+
3537+ - KVM_ARM_VCPU_HAS_EL2_E2H0: Restrict Nested Virtualisation
3538+ support to HCR_EL2.E2H being RES0 (non-VHE).
3539+ Depends on KVM_CAP_ARM_EL2_E2H0.
3540+ KVM_ARM_VCPU_HAS_EL2 must also be set.
3541+
35303542 4.83 KVM_ARM_PREFERRED_TARGET
35313543 -----------------------------
35323544
arch/arm64/include/asm/esr.h+2−0
@@ -100,6 +100,8 @@ · 99 unmodified lines
100100 #define ESR_ELx_AET_CE (UL(6) << ESR_ELx_AET_SHIFT)
101101
102102 /* Shared ISS field definitions for Data/Instruction aborts */
103+#define ESR_ELx_VNCR_SHIFT (13)
104+#define ESR_ELx_VNCR (UL(1) << ESR_ELx_VNCR_SHIFT)
103105 #define ESR_ELx_SET_SHIFT (11)
104106 #define ESR_ELx_SET_MASK (UL(3) << ESR_ELx_SET_SHIFT)
105107 #define ESR_ELx_FnV_SHIFT (10)
arch/arm64/include/asm/fixmap.h+6−0
@@ -48,6 +48,12 @@ enum fixed_addresses { · 47 unmodified lines
4848 FIX_EARLYCON_MEM_BASE,
4949 FIX_TEXT_POKE0,
5050
51+#ifdef CONFIG_KVM
52+ /* One slot per CPU, mapping the guest's VNCR page at EL2. */
53+ FIX_VNCR_END,
54+ FIX_VNCR = FIX_VNCR_END + NR_CPUS,
55+#endif
56+
5157 #ifdef CONFIG_ACPI_APEI_GHES
5258 /* Used for GHES mapping from assorted contexts */
5359 FIX_APEI_GHES_IRQ,
arch/arm64/include/asm/kvm_host.h+14−1
@@ -39,7 +39,7 @@ · 38 unmodified lines
3939
4040 #define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS
4141
42−#define KVM_VCPU_MAX_FEATURES 7
42+#define KVM_VCPU_MAX_FEATURES 9
4343 #define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1)
4444
4545 #define KVM_REQ_SLEEP \
@@ -53,6 +53,7 @@ · 7 unmodified lines
5353 #define KVM_REQ_RESYNC_PMU_EL0 KVM_ARCH_REQ(7)
5454 #define KVM_REQ_NESTED_S2_UNMAP KVM_ARCH_REQ(8)
5555 #define KVM_REQ_GUEST_HYP_IRQ_PENDING KVM_ARCH_REQ(9)
56+#define KVM_REQ_MAP_L1_VNCR_EL2 KVM_ARCH_REQ(10)
5657
5758 #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \
5859 KVM_DIRTY_LOG_INITIALLY_SET)
@@ -395,6 +396,9 @@ struct kvm_arch { · 336 unmodified lines
395396 /* Masks for VNCR-backed and general EL2 sysregs */
396397 struct kvm_sysreg_masks *sysreg_masks;
397398
399+ /* Count the number of VNCR_EL2 currently mapped */
400+ atomic_t vncr_map_count;
401+
398402 /*
399403 * For an untrusted host VM, 'pkvm.handle' is used to lookup
400404 * the associated pKVM instance in the hypervisor.
@@ -573,6 +577,8 @@ enum vcpu_sysreg { · 172 unmodified lines
573577 VNCR(HDFGRTR2_EL2),
574578 VNCR(HDFGWTR2_EL2),
575579
580+ VNCR(VNCR_EL2),
581+
576582 VNCR(CNTVOFF_EL2),
577583 VNCR(CNTV_CVAL_EL0),
578584 VNCR(CNTV_CTL_EL0),
@@ -696,6 +702,8 @@ struct kvm_host_data { · 117 unmodified lines
696702 #define KVM_HOST_DATA_FLAG_HAS_TRBE 1
697703 #define KVM_HOST_DATA_FLAG_TRBE_ENABLED 4
698704 #define KVM_HOST_DATA_FLAG_EL1_TRACING_CONFIGURED 5
705+#define KVM_HOST_DATA_FLAG_VCPU_IN_HYP_CONTEXT 6
706+#define KVM_HOST_DATA_FLAG_L1_VNCR_MAPPED 7
699707 unsigned long flags;
700708
701709 struct kvm_cpu_context host_ctxt;
@@ -772,6 +780,8 @@ struct vcpu_reset_state { · 70 unmodified lines
772780 bool reset;
773781 };
774782
783+struct vncr_tlb;
784+
775785 struct kvm_vcpu_arch {
776786 struct kvm_cpu_context ctxt;
777787
@@ -866,6 +876,9 @@ struct kvm_vcpu_arch { · 88 unmodified lines
866876
867877 /* Per-vcpu CCSIDR override or NULL */
868878 u32 *ccsidr;
879+
880+ /* Per-vcpu TLB for VNCR_EL2 -- NULL when !NV */
881+ struct vncr_tlb *vncr_tlb;
869882 };
870883
871884 /*
arch/arm64/include/asm/kvm_nested.h+100−0
@@ -231,6 +231,38 @@ static inline u64 kvm_encode_nested_level(struct kvm_s2_trans *trans) · 230 unmodified lines
231231 shift; \
232232 })
233233
234+static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid)
235+{
236+ u64 base, tg, num, scale;
237+ int shift;
238+
239+ tg = FIELD_GET(GENMASK(47, 46), val);
240+
241+ switch(tg) {
242+ case 1:
243+ shift = 12;
244+ break;
245+ case 2:
246+ shift = 14;
247+ break;
248+ case 3:
249+ default: /* IMPDEF: handle tg==0 as 64k */
250+ shift = 16;
251+ break;
252+ }
253+
254+ base = (val & GENMASK(36, 0)) << shift;
255+
256+ if (asid)
257+ *asid = FIELD_GET(TLBIR_ASID_MASK, val);
258+
259+ scale = FIELD_GET(GENMASK(45, 44), val);
260+ num = FIELD_GET(GENMASK(43, 39), val);
261+ *range = __TLBI_RANGE_PAGES(num, scale) << shift;
262+
263+ return base;
264+}
265+
234266 static inline unsigned int ps_to_output_size(unsigned int ps)
235267 {
236268 switch (ps) {
@@ -245,4 +277,72 @@ static inline unsigned int ps_to_output_size(unsigned int ps) · 8 unmodified lines
245277 }
246278 }
247279
280+enum trans_regime {
281+ TR_EL10,
282+ TR_EL20,
283+ TR_EL2,
284+};
285+
286+struct s1_walk_info {
287+ u64 baddr;
288+ enum trans_regime regime;
289+ unsigned int max_oa_bits;
290+ unsigned int pgshift;
291+ unsigned int txsz;
292+ int sl;
293+ bool as_el0;
294+ bool hpd;
295+ bool e0poe;
296+ bool poe;
297+ bool pan;
298+ bool be;
299+ bool s2;
300+};
301+
302+struct s1_walk_result {
303+ union {
304+ struct {
305+ u64 desc;
306+ u64 pa;
307+ s8 level;
308+ u8 APTable;
309+ bool nG;
310+ u16 asid;
311+ bool UXNTable;
312+ bool PXNTable;
313+ bool uwxn;
314+ bool uov;
315+ bool ur;
316+ bool uw;
317+ bool ux;
318+ bool pwxn;
319+ bool pov;
320+ bool pr;
321+ bool pw;
322+ bool px;
323+ };
324+ struct {
325+ u8 fst;
326+ bool ptw;
327+ bool s2;
328+ };
329+ };
330+ bool failed;
331+};
332+
333+int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi,
334+ struct s1_walk_result *wr, u64 va);
335+
336+/* VNCR management */
337+int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu);
338+int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu);
339+void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val);
340+
341+#define vncr_fixmap(c) \
342+ ({ \
343+ u32 __c = (c); \
344+ BUG_ON(__c >= NR_CPUS); \
345+ (FIX_VNCR - __c); \
346+ })
347+
248348 #endif /* __ARM64_KVM_NESTED_H */
arch/arm64/include/asm/sysreg.h+1−1
@@ -534,7 +534,7 @@ · 533 unmodified lines
534534 #define SYS_VTTBR_EL2 sys_reg(3, 4, 2, 1, 0)
535535 #define SYS_VTCR_EL2 sys_reg(3, 4, 2, 1, 2)
536536
537−#define SYS_VNCR_EL2 sys_reg(3, 4, 2, 2, 0)
537+#define SYS_HAFGRTR_EL2 sys_reg(3, 4, 3, 1, 6)
538538 #define SYS_SPSR_EL2 sys_reg(3, 4, 4, 0, 0)
539539 #define SYS_ELR_EL2 sys_reg(3, 4, 4, 0, 1)
540540 #define SYS_SP_EL1 sys_reg(3, 4, 4, 1, 0)
arch/arm64/kvm/arm.c+10−0
@@ -368,6 +368,12 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 367 unmodified lines
368368 case KVM_CAP_ARM_EL1_32BIT:
369369 r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1);
370370 break;
371+ case KVM_CAP_ARM_EL2:
372+ r = cpus_have_final_cap(ARM64_HAS_NESTED_VIRT);
373+ break;
374+ case KVM_CAP_ARM_EL2_E2H0:
375+ r = cpus_have_final_cap(ARM64_HAS_HCR_NV1);
376+ break;
371377 case KVM_CAP_GUEST_DEBUG_HW_BPS:
372378 r = get_num_brps();
373379 break;
@@ -843,6 +849,10 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) · 469 unmodified lines
843849 return ret;
844850
845851 if (vcpu_has_nv(vcpu)) {
852+ ret = kvm_vcpu_allocate_vncr_tlb(vcpu);
853+ if (ret)
854+ return ret;
855+
846856 ret = kvm_vgic_vcpu_nv_init(vcpu);
847857 if (ret)
848858 return ret;
arch/arm64/kvm/at.c+64−59
@@ -10,56 +10,6 @@ · 9 unmodified lines
1010 #include <asm/kvm_hyp.h>
1111 #include <asm/kvm_mmu.h>
1212
13−enum trans_regime {
14− TR_EL10,
15− TR_EL20,
16− TR_EL2,
17−};
18−
19−struct s1_walk_info {
20− u64 baddr;
21− enum trans_regime regime;
22− unsigned int max_oa_bits;
23− unsigned int pgshift;
24− unsigned int txsz;
25− int sl;
26− bool hpd;
27− bool e0poe;
28− bool poe;
29− bool pan;
30− bool be;
31− bool s2;
32−};
33−
34−struct s1_walk_result {
35− union {
36− struct {
37− u64 desc;
38− u64 pa;
39− s8 level;
40− u8 APTable;
41− bool UXNTable;
42− bool PXNTable;
43− bool uwxn;
44− bool uov;
45− bool ur;
46− bool uw;
47− bool ux;
48− bool pwxn;
49− bool pov;
50− bool pr;
51− bool pw;
52− bool px;
53− };
54− struct {
55− u8 fst;
56− bool ptw;
57− bool s2;
58− };
59− };
60− bool failed;
61−};
62−
6313 static void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw)
6414 {
6515 wr->fst = fst;
@@ -145,20 +95,15 @@ static void compute_s1poe(struct kvm_vcpu *vcpu, struct s1_walk_info *wi) · 79 unmodified lines
14595 }
14696 }
14797
148−static int setup_s1_walk(struct kvm_vcpu *vcpu, u32 op, struct s1_walk_info *wi,
98+static int setup_s1_walk(struct kvm_vcpu *vcpu, struct s1_walk_info *wi,
14999 struct s1_walk_result *wr, u64 va)
150100 {
151101 u64 hcr, sctlr, tcr, tg, ps, ia_bits, ttbr;
152102 unsigned int stride, x;
153− bool va55, tbi, lva, as_el0;
103+ bool va55, tbi, lva;
154104
155105 hcr = __vcpu_sys_reg(vcpu, HCR_EL2);
156106
157− wi->regime = compute_translation_regime(vcpu, op);
158− as_el0 = (op == OP_AT_S1E0R || op == OP_AT_S1E0W);
159− wi->pan = (op == OP_AT_S1E1RP || op == OP_AT_S1E1WP) &&
160− (*vcpu_cpsr(vcpu) & PSR_PAN_BIT);
161−
162107 va55 = va & BIT(55);
163108
164109 if (wi->regime == TR_EL2 && va55)
@@ -319,7 +264,7 @@ static int setup_s1_walk(struct kvm_vcpu *vcpu, u32 op, struct s1_walk_info *wi, · 154 unmodified lines
319264
320265 /* R_BNDVG and following statements */
321266 if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR2_EL1, E0PD, IMP) &&
322− as_el0 && (tcr & (va55 ? TCR_E0PD1 : TCR_E0PD0)))
267+ wi->as_el0 && (tcr & (va55 ? TCR_E0PD1 : TCR_E0PD0)))
323268 goto transfault_l0;
324269
325270 /* AArch64.S1StartLevel() */
@@ -469,6 +414,33 @@ static int walk_s1(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, · 143 unmodified lines
469414 wr->pa = desc & GENMASK(47, va_bottom);
470415 wr->pa |= va & GENMASK_ULL(va_bottom - 1, 0);
471416
417+ wr->nG = (wi->regime != TR_EL2) && (desc & PTE_NG);
418+ if (wr->nG) {
419+ u64 asid_ttbr, tcr;
420+
421+ switch (wi->regime) {
422+ case TR_EL10:
423+ tcr = vcpu_read_sys_reg(vcpu, TCR_EL1);
424+ asid_ttbr = ((tcr & TCR_A1) ?
425+ vcpu_read_sys_reg(vcpu, TTBR1_EL1) :
426+ vcpu_read_sys_reg(vcpu, TTBR0_EL1));
427+ break;
428+ case TR_EL20:
429+ tcr = vcpu_read_sys_reg(vcpu, TCR_EL2);
430+ asid_ttbr = ((tcr & TCR_A1) ?
431+ vcpu_read_sys_reg(vcpu, TTBR1_EL2) :
432+ vcpu_read_sys_reg(vcpu, TTBR0_EL2));
433+ break;
434+ default:
435+ BUG();
436+ }
437+
438+ wr->asid = FIELD_GET(TTBR_ASID_MASK, asid_ttbr);
439+ if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) ||
440+ !(tcr & TCR_ASID16))
441+ wr->asid &= GENMASK(7, 0);
442+ }
443+
472444 return 0;
473445
474446 addrsz:
@@ -1155,7 +1127,12 @@ static u64 handle_at_slow(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) · 680 unmodified lines
11551127 bool perm_fail = false;
11561128 int ret, idx;
11571129
1158− ret = setup_s1_walk(vcpu, op, &wi, &wr, vaddr);
1130+ wi.regime = compute_translation_regime(vcpu, op);
1131+ wi.as_el0 = (op == OP_AT_S1E0R || op == OP_AT_S1E0W);
1132+ wi.pan = (op == OP_AT_S1E1RP || op == OP_AT_S1E1WP) &&
1133+ (*vcpu_cpsr(vcpu) & PSR_PAN_BIT);
1134+
1135+ ret = setup_s1_walk(vcpu, &wi, &wr, vaddr);
11591136 if (ret)
11601137 goto compute_par;
11611138
@@ -1457,3 +1434,31 @@ void __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) · 295 unmodified lines
14571434 par = compute_par_s12(vcpu, par, &out);
14581435 vcpu_write_sys_reg(vcpu, par, PAR_EL1);
14591436 }
1437+
1438+/*
1439+ * Translate a VA for a given EL in a given translation regime, with
1440+ * or without PAN. This requires wi->{regime, as_el0, pan} to be
1441+ * set. The rest of the wi and wr should be 0-initialised.
1442+ */
1443+int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi,
1444+ struct s1_walk_result *wr, u64 va)
1445+{
1446+ int ret;
1447+
1448+ ret = setup_s1_walk(vcpu, wi, wr, va);
1449+ if (ret)
1450+ return ret;
1451+
1452+ if (wr->level == S1_MMU_DISABLED) {
1453+ wr->ur = wr->uw = wr->ux = true;
1454+ wr->pr = wr->pw = wr->px = true;
1455+ } else {
1456+ ret = walk_s1(vcpu, wi, wr, va);
1457+ if (ret)
1458+ return ret;
1459+
1460+ compute_s1_permissions(vcpu, wi, wr);
1461+ }
1462+
1463+ return 0;
1464+}
arch/arm64/kvm/emulate-nested.c+0−7
@@ -2662,13 +2662,6 @@ void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu) · 2661 unmodified lines
26622662 {
26632663 u64 spsr, elr, esr;
26642664
2665− /*
2666− * Forward this trap to the virtual EL2 if the virtual
2667− * HCR_EL2.NV bit is set and this is coming from !EL2.
2668− */
2669− if (forward_hcr_traps(vcpu, HCR_NV))
2670− return;
2671−
26722665 spsr = vcpu_read_sys_reg(vcpu, SPSR_EL2);
26732666 spsr = kvm_check_illegal_exception_return(vcpu, spsr);
26742667
arch/arm64/kvm/handle_exit.c+1−0
@@ -394,6 +394,7 @@ static exit_handle_fn arm_exit_handlers[] = { · 393 unmodified lines
394394 [ESR_ELx_EC_ERET] = kvm_handle_eret,
395395 [ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort,
396396 [ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort,
397+ [ESR_ELx_EC_DABT_CUR] = kvm_handle_vncr_abort,
397398 [ESR_ELx_EC_SOFTSTP_LOW]= kvm_handle_guest_debug,
398399 [ESR_ELx_EC_WATCHPT_LOW]= kvm_handle_guest_debug,
399400 [ESR_ELx_EC_BREAKPT_LOW]= kvm_handle_guest_debug,
arch/arm64/kvm/hyp/vhe/switch.c+43−3
@@ -48,21 +48,46 @@ DEFINE_PER_CPU(unsigned long, kvm_hyp_vector); · 47 unmodified lines
4848
4949 static u64 __compute_hcr(struct kvm_vcpu *vcpu)
5050 {
51+ u64 guest_hcr = __vcpu_sys_reg(vcpu, HCR_EL2);
5152 u64 hcr = vcpu->arch.hcr_el2;
5253
5354 if (!vcpu_has_nv(vcpu))
5455 return hcr;
5556
57+ /*
58+ * We rely on the invariant that a vcpu entered from HYP
59+ * context must also exit in the same context, as only an ERET
60+ * instruction can kick us out of it, and we obviously trap
61+ * that sucker. PSTATE.M will get fixed-up on exit.
62+ */
5663 if (is_hyp_ctxt(vcpu)) {
64+ host_data_set_flag(VCPU_IN_HYP_CONTEXT);
65+
5766 hcr |= HCR_NV | HCR_NV2 | HCR_AT | HCR_TTLB;
5867
5968 if (!vcpu_el2_e2h_is_set(vcpu))
6069 hcr |= HCR_NV1;
6170
6271 write_sysreg_s(vcpu->arch.ctxt.vncr_array, SYS_VNCR_EL2);
72+ } else {
73+ host_data_clear_flag(VCPU_IN_HYP_CONTEXT);
74+
75+ if (guest_hcr & HCR_NV) {
76+ u64 va = __fix_to_virt(vncr_fixmap(smp_processor_id()));
77+
78+ /* Inherit the low bits from the actual register */
79+ va |= __vcpu_sys_reg(vcpu, VNCR_EL2) & GENMASK(PAGE_SHIFT - 1, 0);
80+ write_sysreg_s(va, SYS_VNCR_EL2);
81+
82+ /* Force NV2 in case the guest is forgetful... */
83+ guest_hcr |= HCR_NV2;
84+ }
6385 }
6486
65− return hcr | (__vcpu_sys_reg(vcpu, HCR_EL2) & ~NV_HCR_GUEST_EXCLUDE);
87+ BUG_ON(host_data_test_flag(VCPU_IN_HYP_CONTEXT) &&
88+ host_data_test_flag(L1_VNCR_MAPPED));
89+
90+ return hcr | (guest_hcr & ~NV_HCR_GUEST_EXCLUDE);
6691 }
6792
6893 static void __activate_cptr_traps(struct kvm_vcpu *vcpu)
@@ -459,6 +484,14 @@ static bool kvm_hyp_handle_tlbi_el2(struct kvm_vcpu *vcpu, u64 *exit_code) · 390 unmodified lines
459484 if (ret)
460485 return false;
461486
487+ /*
488+ * If we have to check for any VNCR mapping being invalidated,
489+ * go back to the slow path for further processing.
490+ */
491+ if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu) &&
492+ atomic_read(&vcpu->kvm->arch.vncr_map_count))
493+ return false;
494+
462495 __kvm_skip_instr(vcpu);
463496
464497 return true;
@@ -568,9 +601,12 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) · 103 unmodified lines
568601
569602 /*
570603 * If we were in HYP context on entry, adjust the PSTATE view
571− * so that the usual helpers work correctly.
604+ * so that the usual helpers work correctly. This enforces our
605+ * invariant that the guest's HYP context status is preserved
606+ * across a run.
572607 */
573− if (vcpu_has_nv(vcpu) && (read_sysreg(hcr_el2) & HCR_NV)) {
608+ if (vcpu_has_nv(vcpu) &&
609+ unlikely(host_data_test_flag(VCPU_IN_HYP_CONTEXT))) {
574610 u64 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT);
575611
576612 switch (mode) {
@@ -586,6 +622,10 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) · 9 unmodified lines
586622 *vcpu_cpsr(vcpu) |= mode;
587623 }
588624
625+ /* Apply extreme paranoia! */
626+ BUG_ON(vcpu_has_nv(vcpu) &&
627+ !!host_data_test_flag(VCPU_IN_HYP_CONTEXT) != is_hyp_ctxt(vcpu));
628+
589629 return __fixup_guest_exit(vcpu, exit_code, hyp_exit_handlers);
590630 }
591631
arch/arm64/kvm/nested.c+614−9
@@ -8,6 +8,7 @@ · 7 unmodified lines
88 #include <linux/kvm.h>
99 #include <linux/kvm_host.h>
1010
11+#include <asm/fixmap.h>
1112 #include <asm/kvm_arm.h>
1213 #include <asm/kvm_emulate.h>
1314 #include <asm/kvm_mmu.h>
@@ -16,6 +17,24 @@ · 2 unmodified lines
1617
1718 #include "sys_regs.h"
1819
20+struct vncr_tlb {
21+ /* The guest's VNCR_EL2 */
22+ u64 gva;
23+ struct s1_walk_info wi;
24+ struct s1_walk_result wr;
25+
26+ u64 hpa;
27+
28+ /* -1 when not mapped on a CPU */
29+ int cpu;
30+
31+ /*
32+ * true if the TLB is valid. Can only be changed with the
33+ * mmu_lock held.
34+ */
35+ bool valid;
36+};
37+
1938 /*
2039 * Ratio of live shadow S2 MMU per vcpu. This is a trade-off between
2140 * memory usage and potential number of different sets of S2 PTs in
@@ -28,6 +47,7 @@ void kvm_init_nested(struct kvm *kvm) · 6 unmodified lines
2847 {
2948 kvm->arch.nested_mmus = NULL;
3049 kvm->arch.nested_mmus_size = 0;
50+ atomic_set(&kvm->arch.vncr_map_count, 0);
3151 }
3252
3353 static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)
@@ -55,6 +75,13 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) · 21 unmodified lines
5575 !cpus_have_final_cap(ARM64_HAS_HCR_NV1))
5676 return -EINVAL;
5777
78+ if (!vcpu->arch.ctxt.vncr_array)
79+ vcpu->arch.ctxt.vncr_array = (u64 *)__get_free_page(GFP_KERNEL_ACCOUNT |
80+ __GFP_ZERO);
81+
82+ if (!vcpu->arch.ctxt.vncr_array)
83+ return -ENOMEM;
84+
5885 /*
5986 * Let's treat memory allocation failures as benign: If we fail to
6087 * allocate anything, return an error and keep the allocated array
@@ -85,6 +112,9 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) · 24 unmodified lines
85112 for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++)
86113 kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]);
87114
115+ free_page((unsigned long)vcpu->arch.ctxt.vncr_array);
116+ vcpu->arch.ctxt.vncr_array = NULL;
117+
88118 return ret;
89119 }
90120
@@ -405,6 +435,30 @@ static unsigned int ttl_to_size(u8 ttl) · 314 unmodified lines
405435 return max_size;
406436 }
407437
438+static u8 pgshift_level_to_ttl(u16 shift, u8 level)
439+{
440+ u8 ttl;
441+
442+ switch(shift) {
443+ case 12:
444+ ttl = TLBI_TTL_TG_4K;
445+ break;
446+ case 14:
447+ ttl = TLBI_TTL_TG_16K;
448+ break;
449+ case 16:
450+ ttl = TLBI_TTL_TG_64K;
451+ break;
452+ default:
453+ BUG();
454+ }
455+
456+ ttl <<= 2;
457+ ttl |= level & 3;
458+
459+ return ttl;
460+}
461+
408462 /*
409463 * Compute the equivalent of the TTL field by parsing the shadow PT. The
410464 * granule size is extracted from the cached VTCR_EL2.TG0 while the level is
@@ -676,23 +730,36 @@ void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu) · 265 unmodified lines
676730 void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu)
677731 {
678732 /*
679− * The vCPU kept its reference on the MMU after the last put, keep
680− * rolling with it.
733+ * If the vCPU kept its reference on the MMU after the last put,
734+ * keep rolling with it.
681735 */
682− if (vcpu->arch.hw_mmu)
683− return;
684−
685736 if (is_hyp_ctxt(vcpu)) {
686− vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
737+ if (!vcpu->arch.hw_mmu)
738+ vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
687739 } else {
688− write_lock(&vcpu->kvm->mmu_lock);
689− vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu);
690− write_unlock(&vcpu->kvm->mmu_lock);
740+ if (!vcpu->arch.hw_mmu) {
741+ scoped_guard(write_lock, &vcpu->kvm->mmu_lock)
742+ vcpu->arch.hw_mmu = get_s2_mmu_nested(vcpu);
743+ }
744+
745+ if (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV)
746+ kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu);
691747 }
692748 }
693749
694750 void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu)
695751 {
752+ /* Unconditionally drop the VNCR mapping if we have one */
753+ if (host_data_test_flag(L1_VNCR_MAPPED)) {
754+ BUG_ON(vcpu->arch.vncr_tlb->cpu != smp_processor_id());
755+ BUG_ON(is_hyp_ctxt(vcpu));
756+
757+ clear_fixmap(vncr_fixmap(vcpu->arch.vncr_tlb->cpu));
758+ vcpu->arch.vncr_tlb->cpu = -1;
759+ host_data_clear_flag(L1_VNCR_MAPPED);
760+ atomic_dec(&vcpu->kvm->arch.vncr_map_count);
761+ }
762+
696763 /*
697764 * Keep a reference on the associated stage-2 MMU if the vCPU is
698765 * scheduling out and not in WFI emulation, suggesting it is likely to
@@ -743,6 +810,245 @@ int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2) · 44 unmodified lines
743810 return kvm_inject_nested_sync(vcpu, esr_el2);
744811 }
745812
813+static void invalidate_vncr(struct vncr_tlb *vt)
814+{
815+ vt->valid = false;
816+ if (vt->cpu != -1)
817+ clear_fixmap(vncr_fixmap(vt->cpu));
818+}
819+
820+static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end)
821+{
822+ struct kvm_vcpu *vcpu;
823+ unsigned long i;
824+
825+ lockdep_assert_held_write(&kvm->mmu_lock);
826+
827+ if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY))
828+ return;
829+
830+ kvm_for_each_vcpu(i, vcpu, kvm) {
831+ struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
832+ u64 ipa_start, ipa_end, ipa_size;
833+
834+ /*
835+ * Careful here: We end-up here from an MMU notifier,
836+ * and this can race against a vcpu not being onlined
837+ * yet, without the pseudo-TLB being allocated.
838+ *
839+ * Skip those, as they obviously don't participate in
840+ * the invalidation at this stage.
841+ */
842+ if (!vt)
843+ continue;
844+
845+ if (!vt->valid)
846+ continue;
847+
848+ ipa_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift,
849+ vt->wr.level));
850+ ipa_start = vt->wr.pa & (ipa_size - 1);
851+ ipa_end = ipa_start + ipa_size;
852+
853+ if (ipa_end <= start || ipa_start >= end)
854+ continue;
855+
856+ invalidate_vncr(vt);
857+ }
858+}
859+
860+struct s1e2_tlbi_scope {
861+ enum {
862+ TLBI_ALL,
863+ TLBI_VA,
864+ TLBI_VAA,
865+ TLBI_ASID,
866+ } type;
867+
868+ u16 asid;
869+ u64 va;
870+ u64 size;
871+};
872+
873+static void invalidate_vncr_va(struct kvm *kvm,
874+ struct s1e2_tlbi_scope *scope)
875+{
876+ struct kvm_vcpu *vcpu;
877+ unsigned long i;
878+
879+ lockdep_assert_held_write(&kvm->mmu_lock);
880+
881+ kvm_for_each_vcpu(i, vcpu, kvm) {
882+ struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
883+ u64 va_start, va_end, va_size;
884+
885+ if (!vt->valid)
886+ continue;
887+
888+ va_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift,
889+ vt->wr.level));
890+ va_start = vt->gva & (va_size - 1);
891+ va_end = va_start + va_size;
892+
893+ switch (scope->type) {
894+ case TLBI_ALL:
895+ break;
896+
897+ case TLBI_VA:
898+ if (va_end <= scope->va ||
899+ va_start >= (scope->va + scope->size))
900+ continue;
901+ if (vt->wr.nG && vt->wr.asid != scope->asid)
902+ continue;
903+ break;
904+
905+ case TLBI_VAA:
906+ if (va_end <= scope->va ||
907+ va_start >= (scope->va + scope->size))
908+ continue;
909+ break;
910+
911+ case TLBI_ASID:
912+ if (!vt->wr.nG || vt->wr.asid != scope->asid)
913+ continue;
914+ break;
915+ }
916+
917+ invalidate_vncr(vt);
918+ }
919+}
920+
921+static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val,
922+ struct s1e2_tlbi_scope *scope)
923+{
924+ switch (inst) {
925+ case OP_TLBI_ALLE2:
926+ case OP_TLBI_ALLE2IS:
927+ case OP_TLBI_ALLE2OS:
928+ case OP_TLBI_VMALLE1:
929+ case OP_TLBI_VMALLE1IS:
930+ case OP_TLBI_VMALLE1OS:
931+ case OP_TLBI_ALLE2NXS:
932+ case OP_TLBI_ALLE2ISNXS:
933+ case OP_TLBI_ALLE2OSNXS:
934+ case OP_TLBI_VMALLE1NXS:
935+ case OP_TLBI_VMALLE1ISNXS:
936+ case OP_TLBI_VMALLE1OSNXS:
937+ scope->type = TLBI_ALL;
938+ break;
939+ case OP_TLBI_VAE2:
940+ case OP_TLBI_VAE2IS:
941+ case OP_TLBI_VAE2OS:
942+ case OP_TLBI_VAE1:
943+ case OP_TLBI_VAE1IS:
944+ case OP_TLBI_VAE1OS:
945+ case OP_TLBI_VAE2NXS:
946+ case OP_TLBI_VAE2ISNXS:
947+ case OP_TLBI_VAE2OSNXS:
948+ case OP_TLBI_VAE1NXS:
949+ case OP_TLBI_VAE1ISNXS:
950+ case OP_TLBI_VAE1OSNXS:
951+ case OP_TLBI_VALE2:
952+ case OP_TLBI_VALE2IS:
953+ case OP_TLBI_VALE2OS:
954+ case OP_TLBI_VALE1:
955+ case OP_TLBI_VALE1IS:
956+ case OP_TLBI_VALE1OS:
957+ case OP_TLBI_VALE2NXS:
958+ case OP_TLBI_VALE2ISNXS:
959+ case OP_TLBI_VALE2OSNXS:
960+ case OP_TLBI_VALE1NXS:
961+ case OP_TLBI_VALE1ISNXS:
962+ case OP_TLBI_VALE1OSNXS:
963+ scope->type = TLBI_VA;
964+ scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val));
965+ if (!scope->size)
966+ scope->size = SZ_1G;
967+ scope->va = (val << 12) & ~(scope->size - 1);
968+ scope->asid = FIELD_GET(TLBIR_ASID_MASK, val);
969+ break;
970+ case OP_TLBI_ASIDE1:
971+ case OP_TLBI_ASIDE1IS:
972+ case OP_TLBI_ASIDE1OS:
973+ case OP_TLBI_ASIDE1NXS:
974+ case OP_TLBI_ASIDE1ISNXS:
975+ case OP_TLBI_ASIDE1OSNXS:
976+ scope->type = TLBI_ASID;
977+ scope->asid = FIELD_GET(TLBIR_ASID_MASK, val);
978+ break;
979+ case OP_TLBI_VAAE1:
980+ case OP_TLBI_VAAE1IS:
981+ case OP_TLBI_VAAE1OS:
982+ case OP_TLBI_VAAE1NXS:
983+ case OP_TLBI_VAAE1ISNXS:
984+ case OP_TLBI_VAAE1OSNXS:
985+ case OP_TLBI_VAALE1:
986+ case OP_TLBI_VAALE1IS:
987+ case OP_TLBI_VAALE1OS:
988+ case OP_TLBI_VAALE1NXS:
989+ case OP_TLBI_VAALE1ISNXS:
990+ case OP_TLBI_VAALE1OSNXS:
991+ scope->type = TLBI_VAA;
992+ scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val));
993+ if (!scope->size)
994+ scope->size = SZ_1G;
995+ scope->va = (val << 12) & ~(scope->size - 1);
996+ break;
997+ case OP_TLBI_RVAE2:
998+ case OP_TLBI_RVAE2IS:
999+ case OP_TLBI_RVAE2OS:
1000+ case OP_TLBI_RVAE1:
1001+ case OP_TLBI_RVAE1IS:
1002+ case OP_TLBI_RVAE1OS:
1003+ case OP_TLBI_RVAE2NXS:
1004+ case OP_TLBI_RVAE2ISNXS:
1005+ case OP_TLBI_RVAE2OSNXS:
1006+ case OP_TLBI_RVAE1NXS:
1007+ case OP_TLBI_RVAE1ISNXS:
1008+ case OP_TLBI_RVAE1OSNXS:
1009+ case OP_TLBI_RVALE2:
1010+ case OP_TLBI_RVALE2IS:
1011+ case OP_TLBI_RVALE2OS:
1012+ case OP_TLBI_RVALE1:
1013+ case OP_TLBI_RVALE1IS:
1014+ case OP_TLBI_RVALE1OS:
1015+ case OP_TLBI_RVALE2NXS:
1016+ case OP_TLBI_RVALE2ISNXS:
1017+ case OP_TLBI_RVALE2OSNXS:
1018+ case OP_TLBI_RVALE1NXS:
1019+ case OP_TLBI_RVALE1ISNXS:
1020+ case OP_TLBI_RVALE1OSNXS:
1021+ scope->type = TLBI_VA;
1022+ scope->va = decode_range_tlbi(val, &scope->size, &scope->asid);
1023+ break;
1024+ case OP_TLBI_RVAAE1:
1025+ case OP_TLBI_RVAAE1IS:
1026+ case OP_TLBI_RVAAE1OS:
1027+ case OP_TLBI_RVAAE1NXS:
1028+ case OP_TLBI_RVAAE1ISNXS:
1029+ case OP_TLBI_RVAAE1OSNXS:
1030+ case OP_TLBI_RVAALE1:
1031+ case OP_TLBI_RVAALE1IS:
1032+ case OP_TLBI_RVAALE1OS:
1033+ case OP_TLBI_RVAALE1NXS:
1034+ case OP_TLBI_RVAALE1ISNXS:
1035+ case OP_TLBI_RVAALE1OSNXS:
1036+ scope->type = TLBI_VAA;
1037+ scope->va = decode_range_tlbi(val, &scope->size, NULL);
1038+ break;
1039+ }
1040+}
1041+
1042+void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val)
1043+{
1044+ struct s1e2_tlbi_scope scope = {};
1045+
1046+ compute_s1_tlbi_range(vcpu, inst, val, &scope);
1047+
1048+ guard(write_lock)(&vcpu->kvm->mmu_lock);
1049+ invalidate_vncr_va(vcpu->kvm, &scope);
1050+}
1051+
7461052 void kvm_nested_s2_wp(struct kvm *kvm)
7471053 {
7481054 int i;
@@ -755,6 +1061,8 @@ void kvm_nested_s2_wp(struct kvm *kvm) · 6 unmodified lines
7551061 if (kvm_s2_mmu_valid(mmu))
7561062 kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu));
7571063 }
1064+
1065+ kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));
7581066 }
7591067
7601068 void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block)
@@ -769,6 +1077,8 @@ void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) · 8 unmodified lines
7691077 if (kvm_s2_mmu_valid(mmu))
7701078 kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block);
7711079 }
1080+
1081+ kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));
7721082 }
7731083
7741084 void kvm_nested_s2_flush(struct kvm *kvm)
@@ -801,6 +1111,295 @@ void kvm_arch_flush_shadow_all(struct kvm *kvm) · 26 unmodified lines
8011111 kvm_uninit_stage2_mmu(kvm);
8021112 }
8031113
1114+/*
1115+ * Dealing with VNCR_EL2 exposed by the *guest* is a complicated matter:
1116+ *
1117+ * - We introduce an internal representation of a vcpu-private TLB,
1118+ * representing the mapping between the guest VA contained in VNCR_EL2,
1119+ * the IPA the guest's EL2 PTs point to, and the actual PA this lives at.
1120+ *
1121+ * - On translation fault from a nested VNCR access, we create such a TLB.
1122+ * If there is no mapping to describe, the guest inherits the fault.
1123+ * Crucially, no actual mapping is done at this stage.
1124+ *
1125+ * - On vcpu_load() in a non-HYP context with HCR_EL2.NV==1, if the above
1126+ * TLB exists, we map it in the fixmap for this CPU, and run with it. We
1127+ * have to respect the permissions dictated by the guest, but not the
1128+ * memory type (FWB is a must).
1129+ *
1130+ * - Note that we usually don't do a vcpu_load() on the back of a fault
1131+ * (unless we are preempted), so the resolution of a translation fault
1132+ * must go via a request that will map the VNCR page in the fixmap.
1133+ * vcpu_load() might as well use the same mechanism.
1134+ *
1135+ * - On vcpu_put() in a non-HYP context with HCR_EL2.NV==1, if the TLB was
1136+ * mapped, we unmap it. Yes it is that simple. The TLB still exists
1137+ * though, and may be reused at a later load.
1138+ *
1139+ * - On permission fault, we simply forward the fault to the guest's EL2.
1140+ * Get out of my way.
1141+ *
1142+ * - On any TLBI for the EL2&0 translation regime, we must find any TLB that
1143+ * intersects with the TLBI request, invalidate it, and unmap the page
1144+ * from the fixmap. Because we need to look at all the vcpu-private TLBs,
1145+ * this requires some wide-ranging locking to ensure that nothing races
1146+ * against it. This may require some refcounting to avoid the search when
1147+ * no such TLB is present.
1148+ *
1149+ * - On MMU notifiers, we must invalidate our TLB in a similar way, but
1150+ * looking at the IPA instead. The funny part is that there may not be a
1151+ * stage-2 mapping for this page if L1 hasn't accessed it using LD/ST
1152+ * instructions.
1153+ */
1154+
1155+int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu)
1156+{
1157+ if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY))
1158+ return 0;
1159+
1160+ vcpu->arch.vncr_tlb = kzalloc(sizeof(*vcpu->arch.vncr_tlb),
1161+ GFP_KERNEL_ACCOUNT);
1162+ if (!vcpu->arch.vncr_tlb)
1163+ return -ENOMEM;
1164+
1165+ return 0;
1166+}
1167+
1168+static u64 read_vncr_el2(struct kvm_vcpu *vcpu)
1169+{
1170+ return (u64)sign_extend64(__vcpu_sys_reg(vcpu, VNCR_EL2), 48);
1171+}
1172+
1173+static int kvm_translate_vncr(struct kvm_vcpu *vcpu)
1174+{
1175+ bool write_fault, writable;
1176+ unsigned long mmu_seq;
1177+ struct vncr_tlb *vt;
1178+ struct page *page;
1179+ u64 va, pfn, gfn;
1180+ int ret;
1181+
1182+ vt = vcpu->arch.vncr_tlb;
1183+
1184+ /*
1185+ * If we're about to walk the EL2 S1 PTs, we must invalidate the
1186+ * current TLB, as it could be sampled from another vcpu doing a
1187+ * TLBI *IS. A real CPU wouldn't do that, but we only keep a single
1188+ * translation, so not much of a choice.
1189+ *
1190+ * We also prepare the next walk wilst we're at it.
1191+ */
1192+ scoped_guard(write_lock, &vcpu->kvm->mmu_lock) {
1193+ invalidate_vncr(vt);
1194+
1195+ vt->wi = (struct s1_walk_info) {
1196+ .regime = TR_EL20,
1197+ .as_el0 = false,
1198+ .pan = false,
1199+ };
1200+ vt->wr = (struct s1_walk_result){};
1201+ }
1202+
1203+ guard(srcu)(&vcpu->kvm->srcu);
1204+
1205+ va = read_vncr_el2(vcpu);
1206+
1207+ ret = __kvm_translate_va(vcpu, &vt->wi, &vt->wr, va);
1208+ if (ret)
1209+ return ret;
1210+
1211+ write_fault = kvm_is_write_fault(vcpu);
1212+
1213+ mmu_seq = vcpu->kvm->mmu_invalidate_seq;
1214+ smp_rmb();
1215+
1216+ gfn = vt->wr.pa >> PAGE_SHIFT;
1217+ pfn = kvm_faultin_pfn(vcpu, gfn, write_fault, &writable, &page);
1218+ if (is_error_noslot_pfn(pfn) || (write_fault && !writable))
1219+ return -EFAULT;
1220+
1221+ scoped_guard(write_lock, &vcpu->kvm->mmu_lock) {
1222+ if (mmu_invalidate_retry(vcpu->kvm, mmu_seq))
1223+ return -EAGAIN;
1224+
1225+ vt->gva = va;
1226+ vt->hpa = pfn << PAGE_SHIFT;
1227+ vt->valid = true;
1228+ vt->cpu = -1;
1229+
1230+ kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu);
1231+ kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw);
1232+ }
1233+
1234+ if (vt->wr.pw)
1235+ mark_page_dirty(vcpu->kvm, gfn);
1236+
1237+ return 0;
1238+}
1239+
1240+static void inject_vncr_perm(struct kvm_vcpu *vcpu)
1241+{
1242+ struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1243+ u64 esr = kvm_vcpu_get_esr(vcpu);
1244+
1245+ /* Adjust the fault level to reflect that of the guest's */
1246+ esr &= ~ESR_ELx_FSC;
1247+ esr |= FIELD_PREP(ESR_ELx_FSC,
1248+ ESR_ELx_FSC_PERM_L(vt->wr.level));
1249+
1250+ kvm_inject_nested_sync(vcpu, esr);
1251+}
1252+
1253+static bool kvm_vncr_tlb_lookup(struct kvm_vcpu *vcpu)
1254+{
1255+ struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1256+
1257+ lockdep_assert_held_read(&vcpu->kvm->mmu_lock);
1258+
1259+ if (!vt->valid)
1260+ return false;
1261+
1262+ if (read_vncr_el2(vcpu) != vt->gva)
1263+ return false;
1264+
1265+ if (vt->wr.nG) {
1266+ u64 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2);
1267+ u64 ttbr = ((tcr & TCR_A1) ?
1268+ vcpu_read_sys_reg(vcpu, TTBR1_EL2) :
1269+ vcpu_read_sys_reg(vcpu, TTBR0_EL2));
1270+ u16 asid;
1271+
1272+ asid = FIELD_GET(TTBR_ASID_MASK, ttbr);
1273+ if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) ||
1274+ !(tcr & TCR_ASID16))
1275+ asid &= GENMASK(7, 0);
1276+
1277+ return asid != vt->wr.asid;
1278+ }
1279+
1280+ return true;
1281+}
1282+
1283+int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu)
1284+{
1285+ struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1286+ u64 esr = kvm_vcpu_get_esr(vcpu);
1287+
1288+ BUG_ON(!(esr & ESR_ELx_VNCR_SHIFT));
1289+
1290+ if (esr_fsc_is_permission_fault(esr)) {
1291+ inject_vncr_perm(vcpu);
1292+ } else if (esr_fsc_is_translation_fault(esr)) {
1293+ bool valid;
1294+ int ret;
1295+
1296+ scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
1297+ valid = kvm_vncr_tlb_lookup(vcpu);
1298+
1299+ if (!valid)
1300+ ret = kvm_translate_vncr(vcpu);
1301+ else
1302+ ret = -EPERM;
1303+
1304+ switch (ret) {
1305+ case -EAGAIN:
1306+ case -ENOMEM:
1307+ /* Let's try again... */
1308+ break;
1309+ case -EFAULT:
1310+ case -EINVAL:
1311+ case -ENOENT:
1312+ case -EACCES:
1313+ /*
1314+ * Translation failed, inject the corresponding
1315+ * exception back to EL2.
1316+ */
1317+ BUG_ON(!vt->wr.failed);
1318+
1319+ esr &= ~ESR_ELx_FSC;
1320+ esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst);
1321+
1322+ kvm_inject_nested_sync(vcpu, esr);
1323+ break;
1324+ case -EPERM:
1325+ /* Hack to deal with POE until we get kernel support */
1326+ inject_vncr_perm(vcpu);
1327+ break;
1328+ case 0:
1329+ break;
1330+ }
1331+ } else {
1332+ WARN_ONCE(1, "Unhandled VNCR abort, ESR=%llx\n", esr);
1333+ }
1334+
1335+ return 1;
1336+}
1337+
1338+static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu)
1339+{
1340+ struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
1341+ pgprot_t prot;
1342+
1343+ guard(preempt)();
1344+ guard(read_lock)(&vcpu->kvm->mmu_lock);
1345+
1346+ /*
1347+ * The request to map VNCR may have raced against some other
1348+ * event, such as an interrupt, and may not be valid anymore.
1349+ */
1350+ if (is_hyp_ctxt(vcpu))
1351+ return;
1352+
1353+ /*
1354+ * Check that the pseudo-TLB is valid and that VNCR_EL2 still
1355+ * contains the expected value. If it doesn't, we simply bail out
1356+ * without a mapping -- a transformed MSR/MRS will generate the
1357+ * fault and allows us to populate the pseudo-TLB.
1358+ */
1359+ if (!vt->valid)
1360+ return;
1361+
1362+ if (read_vncr_el2(vcpu) != vt->gva)
1363+ return;
1364+
1365+ if (vt->wr.nG) {
1366+ u64 tcr = vcpu_read_sys_reg(vcpu, TCR_EL2);
1367+ u64 ttbr = ((tcr & TCR_A1) ?
1368+ vcpu_read_sys_reg(vcpu, TTBR1_EL2) :
1369+ vcpu_read_sys_reg(vcpu, TTBR0_EL2));
1370+ u16 asid;
1371+
1372+ asid = FIELD_GET(TTBR_ASID_MASK, ttbr);
1373+ if (!kvm_has_feat_enum(vcpu->kvm, ID_AA64MMFR0_EL1, ASIDBITS, 16) ||
1374+ !(tcr & TCR_ASID16))
1375+ asid &= GENMASK(7, 0);
1376+
1377+ if (asid != vt->wr.asid)
1378+ return;
1379+ }
1380+
1381+ vt->cpu = smp_processor_id();
1382+
1383+ if (vt->wr.pw && vt->wr.pr)
1384+ prot = PAGE_KERNEL;
1385+ else if (vt->wr.pr)
1386+ prot = PAGE_KERNEL_RO;
1387+ else
1388+ prot = PAGE_NONE;
1389+
1390+ /*
1391+ * We can't map write-only (or no permission at all) in the kernel,
1392+ * but the guest can do it if using POE, so we'll have to turn a
1393+ * translation fault into a permission fault at runtime.
1394+ * FIXME: WO doesn't work at all, need POE support in the kernel.
1395+ */
1396+ if (pgprot_val(prot) != pgprot_val(PAGE_NONE)) {
1397+ __set_fixmap(vncr_fixmap(vt->cpu), vt->hpa, prot);
1398+ host_data_set_flag(L1_VNCR_MAPPED);
1399+ atomic_inc(&vcpu->kvm->arch.vncr_map_count);
1400+ }
1401+}
1402+
8041403 /*
8051404 * Our emulated CPU doesn't support all the possible features. For the
8061405 * sake of simplicity (and probably mental sanity), wipe out a number
@@ -1151,6 +1750,9 @@ int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu) · 344 unmodified lines
11511750 res0 |= ICH_HCR_EL2_DVIM | ICH_HCR_EL2_vSGIEOICount;
11521751 set_sysreg_masks(kvm, ICH_HCR_EL2, res0, res1);
11531752
1753+ /* VNCR_EL2 */
1754+ set_sysreg_masks(kvm, VNCR_EL2, VNCR_EL2_RES0, VNCR_EL2_RES1);
1755+
11541756 out:
11551757 for (enum vcpu_sysreg sr = __SANITISED_REG_START__; sr < NR_SYS_REGS; sr++)
11561758 (void)__vcpu_sys_reg(vcpu, sr);
@@ -1171,6 +1773,9 @@ void check_nested_vcpu_requests(struct kvm_vcpu *vcpu) · 14 unmodified lines
11711773 write_unlock(&vcpu->kvm->mmu_lock);
11721774 }
11731775
1776+ if (kvm_check_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu))
1777+ kvm_map_l1_vncr(vcpu);
1778+
11741779 /* Must be last, as may switch context! */
11751780 if (kvm_check_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu))
11761781 kvm_inject_nested_irq(vcpu);
arch/arm64/kvm/reset.c+2−0
@@ -158,6 +158,8 @@ void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu) · 157 unmodified lines
158158 if (sve_state)
159159 kvm_unshare_hyp(sve_state, sve_state + vcpu_sve_state_size(vcpu));
160160 kfree(sve_state);
161+ free_page((unsigned long)vcpu->arch.ctxt.vncr_array);
162+ kfree(vcpu->arch.vncr_tlb);
161163 kfree(vcpu->arch.ccsidr);
162164 }
163165
arch/arm64/kvm/sys_regs.c+78−57
@@ -2306,15 +2306,6 @@ static bool bad_redir_trap(struct kvm_vcpu *vcpu, · 2305 unmodified lines
23062306 "trap of EL2 register redirected to EL1");
23072307 }
23082308
2309−#define EL2_REG(name, acc, rst, v) { \
2310− SYS_DESC(SYS_##name), \
2311− .access = acc, \
2312− .reset = rst, \
2313− .reg = name, \
2314− .visibility = el2_visibility, \
2315− .val = v, \
2316−}
2317−
23182309 #define EL2_REG_FILTERED(name, acc, rst, v, filter) { \
23192310 SYS_DESC(SYS_##name), \
23202311 .access = acc, \
@@ -2324,6 +2315,9 @@ static bool bad_redir_trap(struct kvm_vcpu *vcpu, · 3 unmodified lines
23242315 .val = v, \
23252316 }
23262317
2318+#define EL2_REG(name, acc, rst, v) \
2319+ EL2_REG_FILTERED(name, acc, rst, v, el2_visibility)
2320+
23272321 #define EL2_REG_VNCR(name, rst, v) EL2_REG(name, bad_vncr_trap, rst, v)
23282322 #define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v)
23292323
@@ -2471,6 +2465,16 @@ static unsigned int sve_el2_visibility(const struct kvm_vcpu *vcpu, · 141 unmodified lines
24712465 return __el2_visibility(vcpu, rd, sve_visibility);
24722466 }
24732467
2468+static unsigned int vncr_el2_visibility(const struct kvm_vcpu *vcpu,
2469+ const struct sys_reg_desc *rd)
2470+{
2471+ if (el2_visibility(vcpu, rd) == 0 &&
2472+ kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY))
2473+ return 0;
2474+
2475+ return REG_HIDDEN;
2476+}
2477+
24742478 static bool access_zcr_el2(struct kvm_vcpu *vcpu,
24752479 struct sys_reg_params *p,
24762480 const struct sys_reg_desc *r)
@@ -3311,6 +3315,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 834 unmodified lines
33113315 tcr2_el2_visibility),
33123316 EL2_REG_VNCR(VTTBR_EL2, reset_val, 0),
33133317 EL2_REG_VNCR(VTCR_EL2, reset_val, 0),
3318+ EL2_REG_FILTERED(VNCR_EL2, bad_vncr_trap, reset_val, 0,
3319+ vncr_el2_visibility),
33143320
33153321 { SYS_DESC(SYS_DACR32_EL2), undef_access, reset_unknown, DACR32_EL2 },
33163322 EL2_REG_VNCR(HDFGRTR_EL2, reset_val, 0),
@@ -3594,8 +3600,7 @@ static bool handle_ripas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 277 unmodified lines
35943600 {
35953601 u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2);
35963602 u64 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
3597− u64 base, range, tg, num, scale;
3598− int shift;
3603+ u64 base, range;
35993604
36003605 if (!kvm_supported_tlbi_ipas2_op(vcpu, sys_encoding))
36013606 return undef_access(vcpu, p, r);
@@ -3605,26 +3610,7 @@ static bool handle_ripas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 3 unmodified lines
36053610 * of the guest's S2 (different base granule size, for example), we
36063611 * decide to ignore TTL and only use the described range.
36073612 */
3608− tg = FIELD_GET(GENMASK(47, 46), p->regval);
3609− scale = FIELD_GET(GENMASK(45, 44), p->regval);
3610− num = FIELD_GET(GENMASK(43, 39), p->regval);
3611− base = p->regval & GENMASK(36, 0);
3612−
3613− switch(tg) {
3614− case 1:
3615− shift = 12;
3616− break;
3617− case 2:
3618− shift = 14;
3619− break;
3620− case 3:
3621− default: /* IMPDEF: handle tg==0 as 64k */
3622− shift = 16;
3623− break;
3624− }
3625−
3626− base <<= shift;
3627− range = __TLBI_RANGE_PAGES(num, scale) << shift;
3613+ base = decode_range_tlbi(p->regval, &range, NULL);
36283614
36293615 kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, get_vmid(vttbr),
36303616 &(union tlbi_info) {
@@ -3690,11 +3676,22 @@ static void s2_mmu_tlbi_s1e1(struct kvm_s2_mmu *mmu, · 59 unmodified lines
36903676 WARN_ON(__kvm_tlbi_s1e2(mmu, info->va.addr, info->va.encoding));
36913677 }
36923678
3679+static bool handle_tlbi_el2(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
3680+ const struct sys_reg_desc *r)
3681+{
3682+ u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2);
3683+
3684+ if (!kvm_supported_tlbi_s1e2_op(vcpu, sys_encoding))
3685+ return undef_access(vcpu, p, r);
3686+
3687+ kvm_handle_s1e2_tlbi(vcpu, sys_encoding, p->regval);
3688+ return true;
3689+}
3690+
36933691 static bool handle_tlbi_el1(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
36943692 const struct sys_reg_desc *r)
36953693 {
36963694 u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2);
3697− u64 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
36983695
36993696 /*
37003697 * If we're here, this is because we've trapped on a EL1 TLBI
@@ -3705,6 +3702,13 @@ static bool handle_tlbi_el1(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 4 unmodified lines
37053702 * - HCR_EL2.E2H == 0 : a non-VHE guest
37063703 * - HCR_EL2.{E2H,TGE} == { 1, 0 } : a VHE guest in guest mode
37073704 *
3705+ * Another possibility is that we are invalidating the EL2 context
3706+ * using EL1 instructions, but that we landed here because we need
3707+ * additional invalidation for structures that are not held in the
3708+ * CPU TLBs (such as the VNCR pseudo-TLB and its EL2 mapping). In
3709+ * that case, we are guaranteed that HCR_EL2.{E2H,TGE} == { 1, 1 }
3710+ * as we don't allow an NV-capable L1 in a nVHE configuration.
3711+ *
37083712 * We don't expect these helpers to ever be called when running
37093713 * in a vEL1 context.
37103714 */
@@ -3714,7 +3718,13 @@ static bool handle_tlbi_el1(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 3 unmodified lines
37143718 if (!kvm_supported_tlbi_s1e1_op(vcpu, sys_encoding))
37153719 return undef_access(vcpu, p, r);
37163720
3717− kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, get_vmid(vttbr),
3721+ if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu)) {
3722+ kvm_handle_s1e2_tlbi(vcpu, sys_encoding, p->regval);
3723+ return true;
3724+ }
3725+
3726+ kvm_s2_mmu_iterate_by_vmid(vcpu->kvm,
3727+ get_vmid(__vcpu_sys_reg(vcpu, VTTBR_EL2)),
37183728 &(union tlbi_info) {
37193729 .va = {
37203730 .addr = p->regval,
@@ -3836,16 +3846,21 @@ static struct sys_reg_desc sys_insn_descs[] = { · 115 unmodified lines
38363846 SYS_INSN(TLBI_IPAS2LE1IS, handle_ipas2e1is),
38373847 SYS_INSN(TLBI_RIPAS2LE1IS, handle_ripas2e1is),
38383848
3839− SYS_INSN(TLBI_ALLE2OS, undef_access),
3840− SYS_INSN(TLBI_VAE2OS, undef_access),
3849+ SYS_INSN(TLBI_ALLE2OS, handle_tlbi_el2),
3850+ SYS_INSN(TLBI_VAE2OS, handle_tlbi_el2),
38413851 SYS_INSN(TLBI_ALLE1OS, handle_alle1is),
3842− SYS_INSN(TLBI_VALE2OS, undef_access),
3852+ SYS_INSN(TLBI_VALE2OS, handle_tlbi_el2),
38433853 SYS_INSN(TLBI_VMALLS12E1OS, handle_vmalls12e1is),
38443854
3845− SYS_INSN(TLBI_RVAE2IS, undef_access),
3846− SYS_INSN(TLBI_RVALE2IS, undef_access),
3855+ SYS_INSN(TLBI_RVAE2IS, handle_tlbi_el2),
3856+ SYS_INSN(TLBI_RVALE2IS, handle_tlbi_el2),
3857+ SYS_INSN(TLBI_ALLE2IS, handle_tlbi_el2),
3858+ SYS_INSN(TLBI_VAE2IS, handle_tlbi_el2),
38473859
38483860 SYS_INSN(TLBI_ALLE1IS, handle_alle1is),
3861+
3862+ SYS_INSN(TLBI_VALE2IS, handle_tlbi_el2),
3863+
38493864 SYS_INSN(TLBI_VMALLS12E1IS, handle_vmalls12e1is),
38503865 SYS_INSN(TLBI_IPAS2E1OS, handle_ipas2e1is),
38513866 SYS_INSN(TLBI_IPAS2E1, handle_ipas2e1is),
@@ -3855,11 +3870,17 @@ static struct sys_reg_desc sys_insn_descs[] = { · 3 unmodified lines
38553870 SYS_INSN(TLBI_IPAS2LE1, handle_ipas2e1is),
38563871 SYS_INSN(TLBI_RIPAS2LE1, handle_ripas2e1is),
38573872 SYS_INSN(TLBI_RIPAS2LE1OS, handle_ripas2e1is),
3858− SYS_INSN(TLBI_RVAE2OS, undef_access),
3859− SYS_INSN(TLBI_RVALE2OS, undef_access),
3860− SYS_INSN(TLBI_RVAE2, undef_access),
3861− SYS_INSN(TLBI_RVALE2, undef_access),
3873+ SYS_INSN(TLBI_RVAE2OS, handle_tlbi_el2),
3874+ SYS_INSN(TLBI_RVALE2OS, handle_tlbi_el2),
3875+ SYS_INSN(TLBI_RVAE2, handle_tlbi_el2),
3876+ SYS_INSN(TLBI_RVALE2, handle_tlbi_el2),
3877+ SYS_INSN(TLBI_ALLE2, handle_tlbi_el2),
3878+ SYS_INSN(TLBI_VAE2, handle_tlbi_el2),
3879+
38623880 SYS_INSN(TLBI_ALLE1, handle_alle1is),
3881+
3882+ SYS_INSN(TLBI_VALE2, handle_tlbi_el2),
3883+
38633884 SYS_INSN(TLBI_VMALLS12E1, handle_vmalls12e1is),
38643885
38653886 SYS_INSN(TLBI_IPAS2E1ISNXS, handle_ipas2e1is),
@@ -3867,19 +3888,19 @@ static struct sys_reg_desc sys_insn_descs[] = { · 1 unmodified line
38673888 SYS_INSN(TLBI_IPAS2LE1ISNXS, handle_ipas2e1is),
38683889 SYS_INSN(TLBI_RIPAS2LE1ISNXS, handle_ripas2e1is),
38693890
3870− SYS_INSN(TLBI_ALLE2OSNXS, undef_access),
3871− SYS_INSN(TLBI_VAE2OSNXS, undef_access),
3891+ SYS_INSN(TLBI_ALLE2OSNXS, handle_tlbi_el2),
3892+ SYS_INSN(TLBI_VAE2OSNXS, handle_tlbi_el2),
38723893 SYS_INSN(TLBI_ALLE1OSNXS, handle_alle1is),
3873− SYS_INSN(TLBI_VALE2OSNXS, undef_access),
3894+ SYS_INSN(TLBI_VALE2OSNXS, handle_tlbi_el2),
38743895 SYS_INSN(TLBI_VMALLS12E1OSNXS, handle_vmalls12e1is),
38753896
3876− SYS_INSN(TLBI_RVAE2ISNXS, undef_access),
3877− SYS_INSN(TLBI_RVALE2ISNXS, undef_access),
3878− SYS_INSN(TLBI_ALLE2ISNXS, undef_access),
3879− SYS_INSN(TLBI_VAE2ISNXS, undef_access),
3897+ SYS_INSN(TLBI_RVAE2ISNXS, handle_tlbi_el2),
3898+ SYS_INSN(TLBI_RVALE2ISNXS, handle_tlbi_el2),
3899+ SYS_INSN(TLBI_ALLE2ISNXS, handle_tlbi_el2),
3900+ SYS_INSN(TLBI_VAE2ISNXS, handle_tlbi_el2),
38803901
38813902 SYS_INSN(TLBI_ALLE1ISNXS, handle_alle1is),
3882− SYS_INSN(TLBI_VALE2ISNXS, undef_access),
3903+ SYS_INSN(TLBI_VALE2ISNXS, handle_tlbi_el2),
38833904 SYS_INSN(TLBI_VMALLS12E1ISNXS, handle_vmalls12e1is),
38843905 SYS_INSN(TLBI_IPAS2E1OSNXS, handle_ipas2e1is),
38853906 SYS_INSN(TLBI_IPAS2E1NXS, handle_ipas2e1is),
@@ -3889,14 +3910,14 @@ static struct sys_reg_desc sys_insn_descs[] = { · 3 unmodified lines
38893910 SYS_INSN(TLBI_IPAS2LE1NXS, handle_ipas2e1is),
38903911 SYS_INSN(TLBI_RIPAS2LE1NXS, handle_ripas2e1is),
38913912 SYS_INSN(TLBI_RIPAS2LE1OSNXS, handle_ripas2e1is),
3892− SYS_INSN(TLBI_RVAE2OSNXS, undef_access),
3893− SYS_INSN(TLBI_RVALE2OSNXS, undef_access),
3894− SYS_INSN(TLBI_RVAE2NXS, undef_access),
3895− SYS_INSN(TLBI_RVALE2NXS, undef_access),
3896− SYS_INSN(TLBI_ALLE2NXS, undef_access),
3897− SYS_INSN(TLBI_VAE2NXS, undef_access),
3913+ SYS_INSN(TLBI_RVAE2OSNXS, handle_tlbi_el2),
3914+ SYS_INSN(TLBI_RVALE2OSNXS, handle_tlbi_el2),
3915+ SYS_INSN(TLBI_RVAE2NXS, handle_tlbi_el2),
3916+ SYS_INSN(TLBI_RVALE2NXS, handle_tlbi_el2),
3917+ SYS_INSN(TLBI_ALLE2NXS, handle_tlbi_el2),
3918+ SYS_INSN(TLBI_VAE2NXS, handle_tlbi_el2),
38983919 SYS_INSN(TLBI_ALLE1NXS, handle_alle1is),
3899− SYS_INSN(TLBI_VALE2NXS, undef_access),
3920+ SYS_INSN(TLBI_VALE2NXS, handle_tlbi_el2),
39003921 SYS_INSN(TLBI_VMALLS12E1NXS, handle_vmalls12e1is),
39013922 };
39023923
arch/arm64/tools/sysreg+6−0
@@ -3816,6 +3816,12 @@ Sysreg SMCR_EL2 3 4 1 2 6 · 3815 unmodified lines
38163816 Fields SMCR_ELx
38173817 EndSysreg
38183818
3819+Sysreg VNCR_EL2 3 4 2 2 0
3820+Field 63:57 RESS
3821+Field 56:12 BADDR
3822+Res0 11:0
3823+EndSysreg
3824+
38193825 Sysreg GCSCR_EL2 3 4 2 5 0
38203826 Fields GCSCR_ELx
38213827 EndSysreg
include/uapi/linux/kvm.h+2−0
@@ -930,6 +930,8 @@ struct kvm_enable_cap { · 929 unmodified lines
930930 #define KVM_CAP_X86_APIC_BUS_CYCLES_NS 237
931931 #define KVM_CAP_X86_GUEST_MODE 238
932932 #define KVM_CAP_ARM_WRITABLE_IMP_ID_REGS 239
933+#define KVM_CAP_ARM_EL2 240
934+#define KVM_CAP_ARM_EL2_E2H0 241
933935
934936 struct kvm_irq_routing_irqchip {
935937 __u32 irqchip;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t