History

Merge tag 'kvmarm-fixes-6.18-1' of git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into HEAD

KVM/arm64 fixes for 6.18, take #1 Improvements and bug fixes: - Fix the handling of ZCR_EL2 in NV VMs (20250926194108.84093-1-oliver.upton@linux.dev) - Pick the correct translation regime when doing a PTW on the back of a SEA (20250926224246.731748-1-oliver.upton@linux.dev) - Prevent userspace from injecting an event into a vcpu that isn't initialised yet (20250930085237.108326-1-oliver.upton@linux.dev) - Move timer save/restore to the sysreg handling code, fixing EL2 timer access in the process (20250929160458.3351788-1-maz@kernel.org) - Add FGT-based trapping of MDSCR_EL1 to reduce the overhead of debug (20250924235150.617451-1-oliver.upton@linux.dev) - Fix trapping configuration when the host isn't GICv3 (20251007160704.1673584-1-sascha.bischoff@arm.com) - Improve the detection of HCR_EL2.E2H being RES1 (20251009121239.29370-1-maz@kernel.org) - Drop a spurious 'break' statement in the S1 PTW (20250930135621.162050-1-osama.abdelkader@gmail.com) - Don't try to access SPE when owned by EL3 (20251010174707.1684200-1-mukesh.ojha@oss.qualcomm.com) Documentation updates: - Document the failure modes of event injection (20250930233620.124607-1-oliver.upton@linux.dev) - Document that a GICv3 guest can be created on a GICv5 host with FEAT_GCIE_LEGACY (20251007154848.1640444-1-sascha.bischoff@arm.com) Selftest improvements: - Add a selftest for the effective value of HCR_EL2.AMO (20250926224454.734066-1-oliver.upton@linux.dev) - Address build warning in the timer selftest when building with clang (20250926155838.2612205-1-seanjc@google.com) - Teach irq_fd selftests about non-x86 architectures (20250930193301.119859-1-oliver.upton@linux.dev) - Add missing sysregs to the set_id_regs selftest (20251012154352.61133-1-zenghui.yu@linux.dev) - Fix vcpu allocation in the vgic_lpi_stress selftest (20251008154520.54801-1-zenghui.yu@linux.dev) - Correctly enable interrupts in the vgic_lpi_stress selftest (20251007195254.260539-1-oliver.upton@linux.dev)

Paolo Bonzini committed 1 year ago

Commit 5d26eaa2 parents 3a86608 ca88ecd30 files · +565 −354Browse files

Unified · Split

30 files
  1. Documentation/virt/kvm/api.rst +5 −0
  2. Documentation/virt/kvm/devices/arm-vgic-v3.rst +2 −1
  3. arch/arm64/include/asm/el2_setup.h +32 −6
  4. arch/arm64/include/asm/kvm_host.h +50 −0
  5. arch/arm64/kvm/arch_timer.c +14 −91
  6. arch/arm64/kvm/arm.c +7 −0
  7. arch/arm64/kvm/at.c +5 −2
  8. arch/arm64/kvm/config.c +90 −0
  9. arch/arm64/kvm/debug.c +10 −5
  10. arch/arm64/kvm/guest.c +0 −70
  11. arch/arm64/kvm/handle_exit.c +6 −1
  12. arch/arm64/kvm/hyp/include/hyp/switch.h +17 −131
  13. arch/arm64/kvm/hyp/nvhe/pkvm.c +1 −0
  14. arch/arm64/kvm/nested.c +6 −3
  15. arch/arm64/kvm/sys_regs.c +105 −26
  16. arch/arm64/kvm/sys_regs.h +6 −0
  17. arch/arm64/kvm/vgic/vgic-v3.c +4 −1
  18. include/kvm/arm_arch_timer.h +16 −8
  19. tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c +1 −1
  20. tools/testing/selftests/kvm/arm64/external_aborts.c +43 −0
  21. tools/testing/selftests/kvm/arm64/get-reg-list.c +96 −3
  22. tools/testing/selftests/kvm/arm64/set_id_regs.c +3 −0
  23. tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c +2 −1
  24. tools/testing/selftests/kvm/include/arm64/processor.h +11 −1
  25. tools/testing/selftests/kvm/include/kvm_util.h +2 −0
  26. tools/testing/selftests/kvm/irqfd_test.c +11 −3
  27. tools/testing/selftests/kvm/lib/arm64/processor.c +5 −0
  28. tools/testing/selftests/kvm/lib/kvm_util.c +5 −0
  29. tools/testing/selftests/kvm/lib/s390/processor.c +5 −0
  30. tools/testing/selftests/kvm/lib/x86/processor.c +5 −0
Documentation/virt/kvm/api.rst+5−0
@@ -1229,6 +1229,9 @@ It is not possible to read back a pending external abort (injected via · 1228 unmodified lines
1229KVM_SET_VCPU_EVENTS or otherwise) because such an exception is always delivered1229KVM_SET_VCPU_EVENTS or otherwise) because such an exception is always delivered
1230directly to the virtual CPU).1230directly to the virtual CPU).
1231 1231
1232Calling this ioctl on a vCPU that hasn't been initialized will return
1233-ENOEXEC.
1234
1232::1235::
1233 1236
1234 struct kvm_vcpu_events {1237 struct kvm_vcpu_events {
@@ -1309,6 +1312,8 @@ exceptions by manipulating individual registers using the KVM_SET_ONE_REG API. · 74 unmodified lines
1309 1312
1310See KVM_GET_VCPU_EVENTS for the data structure.1313See KVM_GET_VCPU_EVENTS for the data structure.
1311 1314
1315Calling this ioctl on a vCPU that hasn't been initialized will return
1316-ENOEXEC.
1312 1317
13134.33 KVM_GET_DEBUGREGS13184.33 KVM_GET_DEBUGREGS
1314----------------------1319----------------------
Documentation/virt/kvm/devices/arm-vgic-v3.rst+2−1
@@ -13,7 +13,8 @@ will act as the VM interrupt controller, requiring emulated user-space devices · 12 unmodified lines
13to inject interrupts to the VGIC instead of directly to CPUs. It is not13to inject interrupts to the VGIC instead of directly to CPUs. It is not
14possible to create both a GICv3 and GICv2 on the same VM.14possible to create both a GICv3 and GICv2 on the same VM.
15 15
16Creating a guest GICv3 device requires a host GICv3 as well.16Creating a guest GICv3 device requires a host GICv3 host, or a GICv5 host with
17support for FEAT_GCIE_LEGACY.
17 18
18 19
19Groups:20Groups:
arch/arm64/include/asm/el2_setup.h+32−6
@@ -24,22 +24,48 @@ · 23 unmodified lines
24 * ID_AA64MMFR4_EL1.E2H0 < 0. On such CPUs HCR_EL2.E2H is RES1, but it24 * ID_AA64MMFR4_EL1.E2H0 < 0. On such CPUs HCR_EL2.E2H is RES1, but it
25 * can reset into an UNKNOWN state and might not read as 1 until it has25 * can reset into an UNKNOWN state and might not read as 1 until it has
26 * been initialized explicitly.26 * been initialized explicitly.
27 *
28 * Fruity CPUs seem to have HCR_EL2.E2H set to RAO/WI, but
29 * don't advertise it (they predate this relaxation).
30 *
31 * Initalize HCR_EL2.E2H so that later code can rely upon HCR_EL2.E2H27 * Initalize HCR_EL2.E2H so that later code can rely upon HCR_EL2.E2H
32 * indicating whether the CPU is running in E2H mode.28 * indicating whether the CPU is running in E2H mode.
33 */29 */
34 mrs_s x1, SYS_ID_AA64MMFR4_EL130 mrs_s x1, SYS_ID_AA64MMFR4_EL1
35 sbfx x1, x1, #ID_AA64MMFR4_EL1_E2H0_SHIFT, #ID_AA64MMFR4_EL1_E2H0_WIDTH31 sbfx x1, x1, #ID_AA64MMFR4_EL1_E2H0_SHIFT, #ID_AA64MMFR4_EL1_E2H0_WIDTH
36 cmp x1, #032 cmp x1, #0
37 b.ge .LnVHE_\@33 b.lt .LnE2H0_\@
38 34
35 /*
36 * Unfortunately, HCR_EL2.E2H can be RES1 even if not advertised
37 * as such via ID_AA64MMFR4_EL1.E2H0:
38 *
39 * - Fruity CPUs predate the !FEAT_E2H0 relaxation, and seem to
40 * have HCR_EL2.E2H implemented as RAO/WI.
41 *
42 * - On CPUs that lack FEAT_FGT, a hypervisor can't trap guest
43 * reads of ID_AA64MMFR4_EL1 to advertise !FEAT_E2H0. NV
44 * guests on these hosts can write to HCR_EL2.E2H without
45 * trapping to the hypervisor, but these writes have no
46 * functional effect.
47 *
48 * Handle both cases by checking for an essential VHE property
49 * (system register remapping) to decide whether we're
50 * effectively VHE-only or not.
51 */
52 msr_hcr_el2 x0 // Setup HCR_EL2 as nVHE
53 isb
54 mov x1, #1 // Write something to FAR_EL1
55 msr far_el1, x1
56 isb
57 mov x1, #2 // Try to overwrite it via FAR_EL2
58 msr far_el2, x1
59 isb
60 mrs x1, far_el1 // If we see the latest write in FAR_EL1,
61 cmp x1, #2 // we can safely assume we are VHE only.
62 b.ne .LnVHE_\@ // Otherwise, we know that nVHE works.
63
64.LnE2H0_\@:
39 orr x0, x0, #HCR_E2H65 orr x0, x0, #HCR_E2H
40.LnVHE_\@:
41 msr_hcr_el2 x066 msr_hcr_el2 x0
42 isb67 isb
68.LnVHE_\@:
43.endm69.endm
44 70
45.macro __init_el2_sctlr71.macro __init_el2_sctlr
arch/arm64/include/asm/kvm_host.h+50−0
@@ -816,6 +816,11 @@ struct kvm_vcpu_arch { · 815 unmodified lines
816 u64 hcrx_el2;816 u64 hcrx_el2;
817 u64 mdcr_el2;817 u64 mdcr_el2;
818 818
819 struct {
820 u64 r;
821 u64 w;
822 } fgt[__NR_FGT_GROUP_IDS__];
823
819 /* Exception Information */824 /* Exception Information */
820 struct kvm_vcpu_fault_info fault;825 struct kvm_vcpu_fault_info fault;
821 826
@@ -1600,6 +1605,51 @@ static inline bool kvm_arch_has_irq_bypass(void) · 778 unmodified lines
1600void compute_fgu(struct kvm *kvm, enum fgt_group_id fgt);1605void compute_fgu(struct kvm *kvm, enum fgt_group_id fgt);
1601void get_reg_fixed_bits(struct kvm *kvm, enum vcpu_sysreg reg, u64 *res0, u64 *res1);1606void get_reg_fixed_bits(struct kvm *kvm, enum vcpu_sysreg reg, u64 *res0, u64 *res1);
1602void check_feature_map(void);1607void check_feature_map(void);
1608void kvm_vcpu_load_fgt(struct kvm_vcpu *vcpu);
1609
1610static __always_inline enum fgt_group_id __fgt_reg_to_group_id(enum vcpu_sysreg reg)
1611{
1612 switch (reg) {
1613 case HFGRTR_EL2:
1614 case HFGWTR_EL2:
1615 return HFGRTR_GROUP;
1616 case HFGITR_EL2:
1617 return HFGITR_GROUP;
1618 case HDFGRTR_EL2:
1619 case HDFGWTR_EL2:
1620 return HDFGRTR_GROUP;
1621 case HAFGRTR_EL2:
1622 return HAFGRTR_GROUP;
1623 case HFGRTR2_EL2:
1624 case HFGWTR2_EL2:
1625 return HFGRTR2_GROUP;
1626 case HFGITR2_EL2:
1627 return HFGITR2_GROUP;
1628 case HDFGRTR2_EL2:
1629 case HDFGWTR2_EL2:
1630 return HDFGRTR2_GROUP;
1631 default:
1632 BUILD_BUG_ON(1);
1633 }
1634}
1603 1635
1636#define vcpu_fgt(vcpu, reg) \
1637 ({ \
1638 enum fgt_group_id id = __fgt_reg_to_group_id(reg); \
1639 u64 *p; \
1640 switch (reg) { \
1641 case HFGWTR_EL2: \
1642 case HDFGWTR_EL2: \
1643 case HFGWTR2_EL2: \
1644 case HDFGWTR2_EL2: \
1645 p = &(vcpu)->arch.fgt[id].w; \
1646 break; \
1647 default: \
1648 p = &(vcpu)->arch.fgt[id].r; \
1649 break; \
1650 } \
1651 \
1652 p; \
1653 })
1604 1654
1605#endif /* __ARM64_KVM_HOST_H__ */1655#endif /* __ARM64_KVM_HOST_H__ */
arch/arm64/kvm/arch_timer.c+14−91
@@ -66,7 +66,7 @@ static int nr_timers(struct kvm_vcpu *vcpu) · 65 unmodified lines
66 66
67u32 timer_get_ctl(struct arch_timer_context *ctxt)67u32 timer_get_ctl(struct arch_timer_context *ctxt)
68{68{
69 struct kvm_vcpu *vcpu = ctxt->vcpu;69 struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctxt);
70 70
71 switch(arch_timer_ctx_index(ctxt)) {71 switch(arch_timer_ctx_index(ctxt)) {
72 case TIMER_VTIMER:72 case TIMER_VTIMER:
@@ -85,7 +85,7 @@ u32 timer_get_ctl(struct arch_timer_context *ctxt) · 12 unmodified lines
85 85
86u64 timer_get_cval(struct arch_timer_context *ctxt)86u64 timer_get_cval(struct arch_timer_context *ctxt)
87{87{
88 struct kvm_vcpu *vcpu = ctxt->vcpu;88 struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctxt);
89 89
90 switch(arch_timer_ctx_index(ctxt)) {90 switch(arch_timer_ctx_index(ctxt)) {
91 case TIMER_VTIMER:91 case TIMER_VTIMER:
@@ -104,7 +104,7 @@ u64 timer_get_cval(struct arch_timer_context *ctxt) · 12 unmodified lines
104 104
105static void timer_set_ctl(struct arch_timer_context *ctxt, u32 ctl)105static void timer_set_ctl(struct arch_timer_context *ctxt, u32 ctl)
106{106{
107 struct kvm_vcpu *vcpu = ctxt->vcpu;107 struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctxt);
108 108
109 switch(arch_timer_ctx_index(ctxt)) {109 switch(arch_timer_ctx_index(ctxt)) {
110 case TIMER_VTIMER:110 case TIMER_VTIMER:
@@ -126,7 +126,7 @@ static void timer_set_ctl(struct arch_timer_context *ctxt, u32 ctl) · 15 unmodified lines
126 126
127static void timer_set_cval(struct arch_timer_context *ctxt, u64 cval)127static void timer_set_cval(struct arch_timer_context *ctxt, u64 cval)
128{128{
129 struct kvm_vcpu *vcpu = ctxt->vcpu;129 struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctxt);
130 130
131 switch(arch_timer_ctx_index(ctxt)) {131 switch(arch_timer_ctx_index(ctxt)) {
132 case TIMER_VTIMER:132 case TIMER_VTIMER:
@@ -146,16 +146,6 @@ static void timer_set_cval(struct arch_timer_context *ctxt, u64 cval) · 13 unmodified lines
146 }146 }
147}147}
148 148
149static void timer_set_offset(struct arch_timer_context *ctxt, u64 offset)
150{
151 if (!ctxt->offset.vm_offset) {
152 WARN(offset, "timer %ld\n", arch_timer_ctx_index(ctxt));
153 return;
154 }
155
156 WRITE_ONCE(*ctxt->offset.vm_offset, offset);
157}
158
159u64 kvm_phys_timer_read(void)149u64 kvm_phys_timer_read(void)
160{150{
161 return timecounter->cc->read(timecounter->cc);151 return timecounter->cc->read(timecounter->cc);
@@ -343,7 +333,7 @@ static enum hrtimer_restart kvm_hrtimer_expire(struct hrtimer *hrt) · 181 unmodified lines
343 u64 ns;333 u64 ns;
344 334
345 ctx = container_of(hrt, struct arch_timer_context, hrtimer);335 ctx = container_of(hrt, struct arch_timer_context, hrtimer);
346 vcpu = ctx->vcpu;336 vcpu = timer_context_to_vcpu(ctx);
347 337
348 trace_kvm_timer_hrtimer_expire(ctx);338 trace_kvm_timer_hrtimer_expire(ctx);
349 339
@@ -436,8 +426,9 @@ static void kvm_timer_update_status(struct arch_timer_context *ctx, bool level) · 86 unmodified lines
436 *426 *
437 * But hey, it's fast, right?427 * But hey, it's fast, right?
438 */428 */
439 if (is_hyp_ctxt(ctx->vcpu) &&429 struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctx);
440 (ctx == vcpu_vtimer(ctx->vcpu) || ctx == vcpu_ptimer(ctx->vcpu))) {430 if (is_hyp_ctxt(vcpu) &&
431 (ctx == vcpu_vtimer(vcpu) || ctx == vcpu_ptimer(vcpu))) {
441 unsigned long val = timer_get_ctl(ctx);432 unsigned long val = timer_get_ctl(ctx);
442 __assign_bit(__ffs(ARCH_TIMER_CTRL_IT_STAT), &val, level);433 __assign_bit(__ffs(ARCH_TIMER_CTRL_IT_STAT), &val, level);
443 timer_set_ctl(ctx, val);434 timer_set_ctl(ctx, val);
@@ -470,7 +461,7 @@ static void timer_emulate(struct arch_timer_context *ctx) · 26 unmodified lines
470 trace_kvm_timer_emulate(ctx, should_fire);461 trace_kvm_timer_emulate(ctx, should_fire);
471 462
472 if (should_fire != ctx->irq.level)463 if (should_fire != ctx->irq.level)
473 kvm_timer_update_irq(ctx->vcpu, should_fire, ctx);464 kvm_timer_update_irq(timer_context_to_vcpu(ctx), should_fire, ctx);
474 465
475 kvm_timer_update_status(ctx, should_fire);466 kvm_timer_update_status(ctx, should_fire);
476 467
@@ -498,7 +489,7 @@ static void set_cntpoff(u64 cntpoff) · 21 unmodified lines
498 489
499static void timer_save_state(struct arch_timer_context *ctx)490static void timer_save_state(struct arch_timer_context *ctx)
500{491{
501 struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu);492 struct arch_timer_cpu *timer = vcpu_timer(timer_context_to_vcpu(ctx));
502 enum kvm_arch_timers index = arch_timer_ctx_index(ctx);493 enum kvm_arch_timers index = arch_timer_ctx_index(ctx);
503 unsigned long flags;494 unsigned long flags;
504 495
@@ -609,7 +600,7 @@ static void kvm_timer_unblocking(struct kvm_vcpu *vcpu) · 104 unmodified lines
609 600
610static void timer_restore_state(struct arch_timer_context *ctx)601static void timer_restore_state(struct arch_timer_context *ctx)
611{602{
612 struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu);603 struct arch_timer_cpu *timer = vcpu_timer(timer_context_to_vcpu(ctx));
613 enum kvm_arch_timers index = arch_timer_ctx_index(ctx);604 enum kvm_arch_timers index = arch_timer_ctx_index(ctx);
614 unsigned long flags;605 unsigned long flags;
615 606
@@ -668,7 +659,7 @@ static inline void set_timer_irq_phys_active(struct arch_timer_context *ctx, boo · 52 unmodified lines
668 659
669static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx)660static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx)
670{661{
671 struct kvm_vcpu *vcpu = ctx->vcpu;662 struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctx);
672 bool phys_active = false;663 bool phys_active = false;
673 664
674 /*665 /*
@@ -677,7 +668,7 @@ static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx) · 2 unmodified lines
677 * this point and the register restoration, we'll take the668 * this point and the register restoration, we'll take the
678 * interrupt anyway.669 * interrupt anyway.
679 */670 */
680 kvm_timer_update_irq(ctx->vcpu, kvm_timer_should_fire(ctx), ctx);671 kvm_timer_update_irq(vcpu, kvm_timer_should_fire(ctx), ctx);
681 672
682 if (irqchip_in_kernel(vcpu->kvm))673 if (irqchip_in_kernel(vcpu->kvm))
683 phys_active = kvm_vgic_map_is_active(vcpu, timer_irq(ctx));674 phys_active = kvm_vgic_map_is_active(vcpu, timer_irq(ctx));
@@ -1063,7 +1054,7 @@ static void timer_context_init(struct kvm_vcpu *vcpu, int timerid) · 379 unmodified lines
1063 struct arch_timer_context *ctxt = vcpu_get_timer(vcpu, timerid);1054 struct arch_timer_context *ctxt = vcpu_get_timer(vcpu, timerid);
1064 struct kvm *kvm = vcpu->kvm;1055 struct kvm *kvm = vcpu->kvm;
1065 1056
1066 ctxt->vcpu = vcpu;1057 ctxt->timer_id = timerid;
1067 1058
1068 if (timerid == TIMER_VTIMER)1059 if (timerid == TIMER_VTIMER)
1069 ctxt->offset.vm_offset = &kvm->arch.timer_data.voffset;1060 ctxt->offset.vm_offset = &kvm->arch.timer_data.voffset;
@@ -1121,49 +1112,6 @@ void kvm_timer_cpu_down(void) · 51 unmodified lines
1121 disable_percpu_irq(host_ptimer_irq);1112 disable_percpu_irq(host_ptimer_irq);
1122}1113}
1123 1114
1124int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
1125{
1126 struct arch_timer_context *timer;
1127
1128 switch (regid) {
1129 case KVM_REG_ARM_TIMER_CTL:
1130 timer = vcpu_vtimer(vcpu);
1131 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value);
1132 break;
1133 case KVM_REG_ARM_TIMER_CNT:
1134 if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET,
1135 &vcpu->kvm->arch.flags)) {
1136 timer = vcpu_vtimer(vcpu);
1137 timer_set_offset(timer, kvm_phys_timer_read() - value);
1138 }
1139 break;
1140 case KVM_REG_ARM_TIMER_CVAL:
1141 timer = vcpu_vtimer(vcpu);
1142 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value);
1143 break;
1144 case KVM_REG_ARM_PTIMER_CTL:
1145 timer = vcpu_ptimer(vcpu);
1146 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value);
1147 break;
1148 case KVM_REG_ARM_PTIMER_CNT:
1149 if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET,
1150 &vcpu->kvm->arch.flags)) {
1151 timer = vcpu_ptimer(vcpu);
1152 timer_set_offset(timer, kvm_phys_timer_read() - value);
1153 }
1154 break;
1155 case KVM_REG_ARM_PTIMER_CVAL:
1156 timer = vcpu_ptimer(vcpu);
1157 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value);
1158 break;
1159
1160 default:
1161 return -1;
1162 }
1163
1164 return 0;
1165}
1166
1167static u64 read_timer_ctl(struct arch_timer_context *timer)1115static u64 read_timer_ctl(struct arch_timer_context *timer)
1168{1116{
1169 /*1117 /*
@@ -1180,31 +1128,6 @@ static u64 read_timer_ctl(struct arch_timer_context *timer) · 10 unmodified lines
1180 return ctl;1128 return ctl;
1181}1129}
1182 1130
1183u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
1184{
1185 switch (regid) {
1186 case KVM_REG_ARM_TIMER_CTL:
1187 return kvm_arm_timer_read(vcpu,
1188 vcpu_vtimer(vcpu), TIMER_REG_CTL);
1189 case KVM_REG_ARM_TIMER_CNT:
1190 return kvm_arm_timer_read(vcpu,
1191 vcpu_vtimer(vcpu), TIMER_REG_CNT);
1192 case KVM_REG_ARM_TIMER_CVAL:
1193 return kvm_arm_timer_read(vcpu,
1194 vcpu_vtimer(vcpu), TIMER_REG_CVAL);
1195 case KVM_REG_ARM_PTIMER_CTL:
1196 return kvm_arm_timer_read(vcpu,
1197 vcpu_ptimer(vcpu), TIMER_REG_CTL);
1198 case KVM_REG_ARM_PTIMER_CNT:
1199 return kvm_arm_timer_read(vcpu,
1200 vcpu_ptimer(vcpu), TIMER_REG_CNT);
1201 case KVM_REG_ARM_PTIMER_CVAL:
1202 return kvm_arm_timer_read(vcpu,
1203 vcpu_ptimer(vcpu), TIMER_REG_CVAL);
1204 }
1205 return (u64)-1;
1206}
1207
1208static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu,1131static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu,
1209 struct arch_timer_context *timer,1132 struct arch_timer_context *timer,
1210 enum kvm_arch_timer_regs treg)1133 enum kvm_arch_timer_regs treg)
arch/arm64/kvm/arm.c+7−0
@@ -642,6 +642,7 @@ nommu: · 641 unmodified lines
642 vcpu->arch.hcr_el2 |= HCR_TWI;642 vcpu->arch.hcr_el2 |= HCR_TWI;
643 643
644 vcpu_set_pauth_traps(vcpu);644 vcpu_set_pauth_traps(vcpu);
645 kvm_vcpu_load_fgt(vcpu);
645 646
646 if (is_protected_kvm_enabled()) {647 if (is_protected_kvm_enabled()) {
647 kvm_call_hyp_nvhe(__pkvm_vcpu_load,648 kvm_call_hyp_nvhe(__pkvm_vcpu_load,
@@ -1794,6 +1795,9 @@ long kvm_arch_vcpu_ioctl(struct file *filp, · 1146 unmodified lines
1794 case KVM_GET_VCPU_EVENTS: {1795 case KVM_GET_VCPU_EVENTS: {
1795 struct kvm_vcpu_events events;1796 struct kvm_vcpu_events events;
1796 1797
1798 if (!kvm_vcpu_initialized(vcpu))
1799 return -ENOEXEC;
1800
1797 if (kvm_arm_vcpu_get_events(vcpu, &events))1801 if (kvm_arm_vcpu_get_events(vcpu, &events))
1798 return -EINVAL;1802 return -EINVAL;
1799 1803
@@ -1805,6 +1809,9 @@ long kvm_arch_vcpu_ioctl(struct file *filp, · 5 unmodified lines
1805 case KVM_SET_VCPU_EVENTS: {1809 case KVM_SET_VCPU_EVENTS: {
1806 struct kvm_vcpu_events events;1810 struct kvm_vcpu_events events;
1807 1811
1812 if (!kvm_vcpu_initialized(vcpu))
1813 return -ENOEXEC;
1814
1808 if (copy_from_user(&events, argp, sizeof(events)))1815 if (copy_from_user(&events, argp, sizeof(events)))
1809 return -EFAULT;1816 return -EFAULT;
1810 1817
arch/arm64/kvm/at.c+5−2
@@ -91,7 +91,6 @@ static enum trans_regime compute_translation_regime(struct kvm_vcpu *vcpu, u32 o · 90 unmodified lines
91 case OP_AT_S1E2W:91 case OP_AT_S1E2W:
92 case OP_AT_S1E2A:92 case OP_AT_S1E2A:
93 return vcpu_el2_e2h_is_set(vcpu) ? TR_EL20 : TR_EL2;93 return vcpu_el2_e2h_is_set(vcpu) ? TR_EL20 : TR_EL2;
94 break;
95 default:94 default:
96 return (vcpu_el2_e2h_is_set(vcpu) &&95 return (vcpu_el2_e2h_is_set(vcpu) &&
97 vcpu_el2_tge_is_set(vcpu)) ? TR_EL20 : TR_EL10;96 vcpu_el2_tge_is_set(vcpu)) ? TR_EL20 : TR_EL10;
@@ -1602,13 +1601,17 @@ int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa, int *level) · 1504 unmodified lines
1602 .fn = match_s1_desc,1601 .fn = match_s1_desc,
1603 .priv = &dm,1602 .priv = &dm,
1604 },1603 },
1605 .regime = TR_EL10,
1606 .as_el0 = false,1604 .as_el0 = false,
1607 .pan = false,1605 .pan = false,
1608 };1606 };
1609 struct s1_walk_result wr = {};1607 struct s1_walk_result wr = {};
1610 int ret;1608 int ret;
1611 1609
1610 if (is_hyp_ctxt(vcpu))
1611 wi.regime = vcpu_el2_e2h_is_set(vcpu) ? TR_EL20 : TR_EL2;
1612 else
1613 wi.regime = TR_EL10;
1614
1612 ret = setup_s1_walk(vcpu, &wi, &wr, va);1615 ret = setup_s1_walk(vcpu, &wi, &wr, va);
1613 if (ret)1616 if (ret)
1614 return ret;1617 return ret;
arch/arm64/kvm/config.c+90−0
@@ -5,6 +5,8 @@ · 4 unmodified lines
5 */5 */
6 6
7#include <linux/kvm_host.h>7#include <linux/kvm_host.h>
8#include <asm/kvm_emulate.h>
9#include <asm/kvm_nested.h>
8#include <asm/sysreg.h>10#include <asm/sysreg.h>
9 11
10/*12/*
@@ -1428,3 +1430,91 @@ void get_reg_fixed_bits(struct kvm *kvm, enum vcpu_sysreg reg, u64 *res0, u64 *r · 1417 unmodified lines
1428 break;1430 break;
1429 }1431 }
1430}1432}
1433
1434static __always_inline struct fgt_masks *__fgt_reg_to_masks(enum vcpu_sysreg reg)
1435{
1436 switch (reg) {
1437 case HFGRTR_EL2:
1438 return &hfgrtr_masks;
1439 case HFGWTR_EL2:
1440 return &hfgwtr_masks;
1441 case HFGITR_EL2:
1442 return &hfgitr_masks;
1443 case HDFGRTR_EL2:
1444 return &hdfgrtr_masks;
1445 case HDFGWTR_EL2:
1446 return &hdfgwtr_masks;
1447 case HAFGRTR_EL2:
1448 return &hafgrtr_masks;
1449 case HFGRTR2_EL2:
1450 return &hfgrtr2_masks;
1451 case HFGWTR2_EL2:
1452 return &hfgwtr2_masks;
1453 case HFGITR2_EL2:
1454 return &hfgitr2_masks;
1455 case HDFGRTR2_EL2:
1456 return &hdfgrtr2_masks;
1457 case HDFGWTR2_EL2:
1458 return &hdfgwtr2_masks;
1459 default:
1460 BUILD_BUG_ON(1);
1461 }
1462}
1463
1464static __always_inline void __compute_fgt(struct kvm_vcpu *vcpu, enum vcpu_sysreg reg)
1465{
1466 u64 fgu = vcpu->kvm->arch.fgu[__fgt_reg_to_group_id(reg)];
1467 struct fgt_masks *m = __fgt_reg_to_masks(reg);
1468 u64 clear = 0, set = 0, val = m->nmask;
1469
1470 set |= fgu & m->mask;
1471 clear |= fgu & m->nmask;
1472
1473 if (is_nested_ctxt(vcpu)) {
1474 u64 nested = __vcpu_sys_reg(vcpu, reg);
1475 set |= nested & m->mask;
1476 clear |= ~nested & m->nmask;
1477 }
1478
1479 val |= set;
1480 val &= ~clear;
1481 *vcpu_fgt(vcpu, reg) = val;
1482}
1483
1484static void __compute_hfgwtr(struct kvm_vcpu *vcpu)
1485{
1486 __compute_fgt(vcpu, HFGWTR_EL2);
1487
1488 if (cpus_have_final_cap(ARM64_WORKAROUND_AMPERE_AC03_CPU_38))
1489 *vcpu_fgt(vcpu, HFGWTR_EL2) |= HFGWTR_EL2_TCR_EL1;
1490}
1491
1492static void __compute_hdfgwtr(struct kvm_vcpu *vcpu)
1493{
1494 __compute_fgt(vcpu, HDFGWTR_EL2);
1495
1496 if (is_hyp_ctxt(vcpu))
1497 *vcpu_fgt(vcpu, HDFGWTR_EL2) |= HDFGWTR_EL2_MDSCR_EL1;
1498}
1499
1500void kvm_vcpu_load_fgt(struct kvm_vcpu *vcpu)
1501{
1502 if (!cpus_have_final_cap(ARM64_HAS_FGT))
1503 return;
1504
1505 __compute_fgt(vcpu, HFGRTR_EL2);
1506 __compute_hfgwtr(vcpu);
1507 __compute_fgt(vcpu, HFGITR_EL2);
1508 __compute_fgt(vcpu, HDFGRTR_EL2);
1509 __compute_hdfgwtr(vcpu);
1510 __compute_fgt(vcpu, HAFGRTR_EL2);
1511
1512 if (!cpus_have_final_cap(ARM64_HAS_FGT2))
1513 return;
1514
1515 __compute_fgt(vcpu, HFGRTR2_EL2);
1516 __compute_fgt(vcpu, HFGWTR2_EL2);
1517 __compute_fgt(vcpu, HFGITR2_EL2);
1518 __compute_fgt(vcpu, HDFGRTR2_EL2);
1519 __compute_fgt(vcpu, HDFGWTR2_EL2);
1520}
arch/arm64/kvm/debug.c+10−5
@@ -15,6 +15,12 @@ · 14 unmodified lines
15#include <asm/kvm_arm.h>15#include <asm/kvm_arm.h>
16#include <asm/kvm_emulate.h>16#include <asm/kvm_emulate.h>
17 17
18static int cpu_has_spe(u64 dfr0)
19{
20 return cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_PMSVer_SHIFT) &&
21 !(read_sysreg_s(SYS_PMBIDR_EL1) & PMBIDR_EL1_P);
22}
23
18/**24/**
19 * kvm_arm_setup_mdcr_el2 - configure vcpu mdcr_el2 value25 * kvm_arm_setup_mdcr_el2 - configure vcpu mdcr_el2 value
20 *26 *
@@ -77,13 +83,12 @@ void kvm_init_host_debug_data(void) · 56 unmodified lines
77 *host_data_ptr(debug_brps) = SYS_FIELD_GET(ID_AA64DFR0_EL1, BRPs, dfr0);83 *host_data_ptr(debug_brps) = SYS_FIELD_GET(ID_AA64DFR0_EL1, BRPs, dfr0);
78 *host_data_ptr(debug_wrps) = SYS_FIELD_GET(ID_AA64DFR0_EL1, WRPs, dfr0);84 *host_data_ptr(debug_wrps) = SYS_FIELD_GET(ID_AA64DFR0_EL1, WRPs, dfr0);
79 85
86 if (cpu_has_spe(dfr0))
87 host_data_set_flag(HAS_SPE);
88
80 if (has_vhe())89 if (has_vhe())
81 return;90 return;
82 91
83 if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_PMSVer_SHIFT) &&
84 !(read_sysreg_s(SYS_PMBIDR_EL1) & PMBIDR_EL1_P))
85 host_data_set_flag(HAS_SPE);
86
87 /* Check if we have BRBE implemented and available at the host */92 /* Check if we have BRBE implemented and available at the host */
88 if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_BRBE_SHIFT))93 if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_BRBE_SHIFT))
89 host_data_set_flag(HAS_BRBE);94 host_data_set_flag(HAS_BRBE);
@@ -102,7 +107,7 @@ void kvm_init_host_debug_data(void) · 12 unmodified lines
102void kvm_debug_init_vhe(void)107void kvm_debug_init_vhe(void)
103{108{
104 /* Clear PMSCR_EL1.E{0,1}SPE which reset to UNKNOWN values. */109 /* Clear PMSCR_EL1.E{0,1}SPE which reset to UNKNOWN values. */
105 if (SYS_FIELD_GET(ID_AA64DFR0_EL1, PMSVer, read_sysreg(id_aa64dfr0_el1)))110 if (host_data_test_flag(HAS_SPE))
106 write_sysreg_el1(0, SYS_PMSCR);111 write_sysreg_el1(0, SYS_PMSCR);
107}112}
108 113
arch/arm64/kvm/guest.c+0−70
@@ -591,64 +591,6 @@ static unsigned long num_core_regs(const struct kvm_vcpu *vcpu) · 590 unmodified lines
591 return copy_core_reg_indices(vcpu, NULL);591 return copy_core_reg_indices(vcpu, NULL);
592}592}
593 593
594static const u64 timer_reg_list[] = {
595 KVM_REG_ARM_TIMER_CTL,
596 KVM_REG_ARM_TIMER_CNT,
597 KVM_REG_ARM_TIMER_CVAL,
598 KVM_REG_ARM_PTIMER_CTL,
599 KVM_REG_ARM_PTIMER_CNT,
600 KVM_REG_ARM_PTIMER_CVAL,
601};
602
603#define NUM_TIMER_REGS ARRAY_SIZE(timer_reg_list)
604
605static bool is_timer_reg(u64 index)
606{
607 switch (index) {
608 case KVM_REG_ARM_TIMER_CTL:
609 case KVM_REG_ARM_TIMER_CNT:
610 case KVM_REG_ARM_TIMER_CVAL:
611 case KVM_REG_ARM_PTIMER_CTL:
612 case KVM_REG_ARM_PTIMER_CNT:
613 case KVM_REG_ARM_PTIMER_CVAL:
614 return true;
615 }
616 return false;
617}
618
619static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
620{
621 for (int i = 0; i < NUM_TIMER_REGS; i++) {
622 if (put_user(timer_reg_list[i], uindices))
623 return -EFAULT;
624 uindices++;
625 }
626
627 return 0;
628}
629
630static int set_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
631{
632 void __user *uaddr = (void __user *)(long)reg->addr;
633 u64 val;
634 int ret;
635
636 ret = copy_from_user(&val, uaddr, KVM_REG_SIZE(reg->id));
637 if (ret != 0)
638 return -EFAULT;
639
640 return kvm_arm_timer_set_reg(vcpu, reg->id, val);
641}
642
643static int get_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
644{
645 void __user *uaddr = (void __user *)(long)reg->addr;
646 u64 val;
647
648 val = kvm_arm_timer_get_reg(vcpu, reg->id);
649 return copy_to_user(uaddr, &val, KVM_REG_SIZE(reg->id)) ? -EFAULT : 0;
650}
651
652static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu)594static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu)
653{595{
654 const unsigned int slices = vcpu_sve_slices(vcpu);596 const unsigned int slices = vcpu_sve_slices(vcpu);
@@ -724,7 +666,6 @@ unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu) · 69 unmodified lines
724 res += num_sve_regs(vcpu);666 res += num_sve_regs(vcpu);
725 res += kvm_arm_num_sys_reg_descs(vcpu);667 res += kvm_arm_num_sys_reg_descs(vcpu);
726 res += kvm_arm_get_fw_num_regs(vcpu);668 res += kvm_arm_get_fw_num_regs(vcpu);
727 res += NUM_TIMER_REGS;
728 669
729 return res;670 return res;
730}671}
@@ -755,11 +696,6 @@ int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) · 24 unmodified lines
755 return ret;696 return ret;
756 uindices += kvm_arm_get_fw_num_regs(vcpu);697 uindices += kvm_arm_get_fw_num_regs(vcpu);
757 698
758 ret = copy_timer_indices(vcpu, uindices);
759 if (ret < 0)
760 return ret;
761 uindices += NUM_TIMER_REGS;
762
763 return kvm_arm_copy_sys_reg_indices(vcpu, uindices);699 return kvm_arm_copy_sys_reg_indices(vcpu, uindices);
764}700}
765 701
@@ -777,9 +713,6 @@ int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) · 11 unmodified lines
777 case KVM_REG_ARM64_SVE: return get_sve_reg(vcpu, reg);713 case KVM_REG_ARM64_SVE: return get_sve_reg(vcpu, reg);
778 }714 }
779 715
780 if (is_timer_reg(reg->id))
781 return get_timer_reg(vcpu, reg);
782
783 return kvm_arm_sys_reg_get_reg(vcpu, reg);716 return kvm_arm_sys_reg_get_reg(vcpu, reg);
784}717}
785 718
@@ -797,9 +730,6 @@ int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) · 11 unmodified lines
797 case KVM_REG_ARM64_SVE: return set_sve_reg(vcpu, reg);730 case KVM_REG_ARM64_SVE: return set_sve_reg(vcpu, reg);
798 }731 }
799 732
800 if (is_timer_reg(reg->id))
801 return set_timer_reg(vcpu, reg);
802
803 return kvm_arm_sys_reg_set_reg(vcpu, reg);733 return kvm_arm_sys_reg_set_reg(vcpu, reg);
804}734}
805 735
arch/arm64/kvm/handle_exit.c+6−1
@@ -147,7 +147,12 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu) · 146 unmodified lines
147 if (esr & ESR_ELx_WFx_ISS_RV) {147 if (esr & ESR_ELx_WFx_ISS_RV) {
148 u64 val, now;148 u64 val, now;
149 149
150 now = kvm_arm_timer_get_reg(vcpu, KVM_REG_ARM_TIMER_CNT);150 now = kvm_phys_timer_read();
151 if (is_hyp_ctxt(vcpu) && vcpu_el2_e2h_is_set(vcpu))
152 now -= timer_get_offset(vcpu_hvtimer(vcpu));
153 else
154 now -= timer_get_offset(vcpu_vtimer(vcpu));
155
151 val = vcpu_get_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu));156 val = vcpu_get_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu));
152 157
153 if (now >= val)158 if (now >= val)
arch/arm64/kvm/hyp/include/hyp/switch.h+17−131
@@ -195,123 +195,6 @@ static inline void __deactivate_cptr_traps(struct kvm_vcpu *vcpu) · 194 unmodified lines
195 __deactivate_cptr_traps_nvhe(vcpu);195 __deactivate_cptr_traps_nvhe(vcpu);
196}196}
197 197
198#define reg_to_fgt_masks(reg) \
199 ({ \
200 struct fgt_masks *m; \
201 switch(reg) { \
202 case HFGRTR_EL2: \
203 m = &hfgrtr_masks; \
204 break; \
205 case HFGWTR_EL2: \
206 m = &hfgwtr_masks; \
207 break; \
208 case HFGITR_EL2: \
209 m = &hfgitr_masks; \
210 break; \
211 case HDFGRTR_EL2: \
212 m = &hdfgrtr_masks; \
213 break; \
214 case HDFGWTR_EL2: \
215 m = &hdfgwtr_masks; \
216 break; \
217 case HAFGRTR_EL2: \
218 m = &hafgrtr_masks; \
219 break; \
220 case HFGRTR2_EL2: \
221 m = &hfgrtr2_masks; \
222 break; \
223 case HFGWTR2_EL2: \
224 m = &hfgwtr2_masks; \
225 break; \
226 case HFGITR2_EL2: \
227 m = &hfgitr2_masks; \
228 break; \
229 case HDFGRTR2_EL2: \
230 m = &hdfgrtr2_masks; \
231 break; \
232 case HDFGWTR2_EL2: \
233 m = &hdfgwtr2_masks; \
234 break; \
235 default: \
236 BUILD_BUG_ON(1); \
237 } \
238 \
239 m; \
240 })
241
242#define compute_clr_set(vcpu, reg, clr, set) \
243 do { \
244 u64 hfg = __vcpu_sys_reg(vcpu, reg); \
245 struct fgt_masks *m = reg_to_fgt_masks(reg); \
246 set |= hfg & m->mask; \
247 clr |= ~hfg & m->nmask; \
248 } while(0)
249
250#define reg_to_fgt_group_id(reg) \
251 ({ \
252 enum fgt_group_id id; \
253 switch(reg) { \
254 case HFGRTR_EL2: \
255 case HFGWTR_EL2: \
256 id = HFGRTR_GROUP; \
257 break; \
258 case HFGITR_EL2: \
259 id = HFGITR_GROUP; \
260 break; \
261 case HDFGRTR_EL2: \
262 case HDFGWTR_EL2: \
263 id = HDFGRTR_GROUP; \
264 break; \
265 case HAFGRTR_EL2: \
266 id = HAFGRTR_GROUP; \
267 break; \
268 case HFGRTR2_EL2: \
269 case HFGWTR2_EL2: \
270 id = HFGRTR2_GROUP; \
271 break; \
272 case HFGITR2_EL2: \
273 id = HFGITR2_GROUP; \
274 break; \
275 case HDFGRTR2_EL2: \
276 case HDFGWTR2_EL2: \
277 id = HDFGRTR2_GROUP; \
278 break; \
279 default: \
280 BUILD_BUG_ON(1); \
281 } \
282 \
283 id; \
284 })
285
286#define compute_undef_clr_set(vcpu, kvm, reg, clr, set) \
287 do { \
288 u64 hfg = kvm->arch.fgu[reg_to_fgt_group_id(reg)]; \
289 struct fgt_masks *m = reg_to_fgt_masks(reg); \
290 set |= hfg & m->mask; \
291 clr |= hfg & m->nmask; \
292 } while(0)
293
294#define update_fgt_traps_cs(hctxt, vcpu, kvm, reg, clr, set) \
295 do { \
296 struct fgt_masks *m = reg_to_fgt_masks(reg); \
297 u64 c = clr, s = set; \
298 u64 val; \
299 \
300 ctxt_sys_reg(hctxt, reg) = read_sysreg_s(SYS_ ## reg); \
301 if (is_nested_ctxt(vcpu)) \
302 compute_clr_set(vcpu, reg, c, s); \
303 \
304 compute_undef_clr_set(vcpu, kvm, reg, c, s); \
305 \
306 val = m->nmask; \
307 val |= s; \
308 val &= ~c; \
309 write_sysreg_s(val, SYS_ ## reg); \
310 } while(0)
311
312#define update_fgt_traps(hctxt, vcpu, kvm, reg) \
313 update_fgt_traps_cs(hctxt, vcpu, kvm, reg, 0, 0)
314
315static inline bool cpu_has_amu(void)198static inline bool cpu_has_amu(void)
316{199{
317 u64 pfr0 = read_sysreg_s(SYS_ID_AA64PFR0_EL1);200 u64 pfr0 = read_sysreg_s(SYS_ID_AA64PFR0_EL1);
@@ -320,33 +203,36 @@ static inline bool cpu_has_amu(void) · 2 unmodified lines
320 ID_AA64PFR0_EL1_AMU_SHIFT);203 ID_AA64PFR0_EL1_AMU_SHIFT);
321}204}
322 205
206#define __activate_fgt(hctxt, vcpu, reg) \
207 do { \
208 ctxt_sys_reg(hctxt, reg) = read_sysreg_s(SYS_ ## reg); \
209 write_sysreg_s(*vcpu_fgt(vcpu, reg), SYS_ ## reg); \
210 } while (0)
211
323static inline void __activate_traps_hfgxtr(struct kvm_vcpu *vcpu)212static inline void __activate_traps_hfgxtr(struct kvm_vcpu *vcpu)
324{213{
325 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt);214 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt);
326 struct kvm *kvm = kern_hyp_va(vcpu->kvm);
327 215
328 if (!cpus_have_final_cap(ARM64_HAS_FGT))216 if (!cpus_have_final_cap(ARM64_HAS_FGT))
329 return;217 return;
330 218
331 update_fgt_traps(hctxt, vcpu, kvm, HFGRTR_EL2);219 __activate_fgt(hctxt, vcpu, HFGRTR_EL2);
332 update_fgt_traps_cs(hctxt, vcpu, kvm, HFGWTR_EL2, 0,220 __activate_fgt(hctxt, vcpu, HFGWTR_EL2);
333 cpus_have_final_cap(ARM64_WORKAROUND_AMPERE_AC03_CPU_38) ?221 __activate_fgt(hctxt, vcpu, HFGITR_EL2);
334 HFGWTR_EL2_TCR_EL1_MASK : 0);222 __activate_fgt(hctxt, vcpu, HDFGRTR_EL2);
335 update_fgt_traps(hctxt, vcpu, kvm, HFGITR_EL2);223 __activate_fgt(hctxt, vcpu, HDFGWTR_EL2);
336 update_fgt_traps(hctxt, vcpu, kvm, HDFGRTR_EL2);
337 update_fgt_traps(hctxt, vcpu, kvm, HDFGWTR_EL2);
338 224
339 if (cpu_has_amu())225 if (cpu_has_amu())
340 update_fgt_traps(hctxt, vcpu, kvm, HAFGRTR_EL2);226 __activate_fgt(hctxt, vcpu, HAFGRTR_EL2);
341 227
342 if (!cpus_have_final_cap(ARM64_HAS_FGT2))228 if (!cpus_have_final_cap(ARM64_HAS_FGT2))
343 return;229 return;
344 230
345 update_fgt_traps(hctxt, vcpu, kvm, HFGRTR2_EL2);231 __activate_fgt(hctxt, vcpu, HFGRTR2_EL2);
346 update_fgt_traps(hctxt, vcpu, kvm, HFGWTR2_EL2);232 __activate_fgt(hctxt, vcpu, HFGWTR2_EL2);
347 update_fgt_traps(hctxt, vcpu, kvm, HFGITR2_EL2);233 __activate_fgt(hctxt, vcpu, HFGITR2_EL2);
348 update_fgt_traps(hctxt, vcpu, kvm, HDFGRTR2_EL2);234 __activate_fgt(hctxt, vcpu, HDFGRTR2_EL2);
349 update_fgt_traps(hctxt, vcpu, kvm, HDFGWTR2_EL2);235 __activate_fgt(hctxt, vcpu, HDFGWTR2_EL2);
350}236}
351 237
352#define __deactivate_fgt(htcxt, vcpu, reg) \238#define __deactivate_fgt(htcxt, vcpu, reg) \
arch/arm64/kvm/hyp/nvhe/pkvm.c+1−0
@@ -172,6 +172,7 @@ static int pkvm_vcpu_init_traps(struct pkvm_hyp_vcpu *hyp_vcpu) · 171 unmodified lines
172 172
173 /* Trust the host for non-protected vcpu features. */173 /* Trust the host for non-protected vcpu features. */
174 vcpu->arch.hcrx_el2 = host_vcpu->arch.hcrx_el2;174 vcpu->arch.hcrx_el2 = host_vcpu->arch.hcrx_el2;
175 memcpy(vcpu->arch.fgt, host_vcpu->arch.fgt, sizeof(vcpu->arch.fgt));
175 return 0;176 return 0;
176 }177 }
177 178
arch/arm64/kvm/nested.c+6−3
@@ -1859,13 +1859,16 @@ void kvm_nested_setup_mdcr_el2(struct kvm_vcpu *vcpu) · 1858 unmodified lines
1859{1859{
1860 u64 guest_mdcr = __vcpu_sys_reg(vcpu, MDCR_EL2);1860 u64 guest_mdcr = __vcpu_sys_reg(vcpu, MDCR_EL2);
1861 1861
1862 if (is_nested_ctxt(vcpu))
1863 vcpu->arch.mdcr_el2 |= (guest_mdcr & NV_MDCR_GUEST_INCLUDE);
1862 /*1864 /*
1863 * In yet another example where FEAT_NV2 is fscking broken, accesses1865 * In yet another example where FEAT_NV2 is fscking broken, accesses
1864 * to MDSCR_EL1 are redirected to the VNCR despite having an effect1866 * to MDSCR_EL1 are redirected to the VNCR despite having an effect
1865 * at EL2. Use a big hammer to apply sanity.1867 * at EL2. Use a big hammer to apply sanity.
1868 *
1869 * Unless of course we have FEAT_FGT, in which case we can precisely
1870 * trap MDSCR_EL1.
1866 */1871 */
1867 if (is_hyp_ctxt(vcpu))1872 else if (!cpus_have_final_cap(ARM64_HAS_FGT))
1868 vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;1873 vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
1869 else
1870 vcpu->arch.mdcr_el2 |= (guest_mdcr & NV_MDCR_GUEST_INCLUDE);
1871}1874}
arch/arm64/kvm/sys_regs.c+105−26
@@ -203,7 +203,6 @@ static void locate_register(const struct kvm_vcpu *vcpu, enum vcpu_sysreg reg, · 202 unmodified lines
203 MAPPED_EL2_SYSREG(AMAIR_EL2, AMAIR_EL1, NULL );203 MAPPED_EL2_SYSREG(AMAIR_EL2, AMAIR_EL1, NULL );
204 MAPPED_EL2_SYSREG(ELR_EL2, ELR_EL1, NULL );204 MAPPED_EL2_SYSREG(ELR_EL2, ELR_EL1, NULL );
205 MAPPED_EL2_SYSREG(SPSR_EL2, SPSR_EL1, NULL );205 MAPPED_EL2_SYSREG(SPSR_EL2, SPSR_EL1, NULL );
206 MAPPED_EL2_SYSREG(ZCR_EL2, ZCR_EL1, NULL );
207 MAPPED_EL2_SYSREG(CONTEXTIDR_EL2, CONTEXTIDR_EL1, NULL );206 MAPPED_EL2_SYSREG(CONTEXTIDR_EL2, CONTEXTIDR_EL1, NULL );
208 MAPPED_EL2_SYSREG(SCTLR2_EL2, SCTLR2_EL1, NULL );207 MAPPED_EL2_SYSREG(SCTLR2_EL2, SCTLR2_EL1, NULL );
209 case CNTHCTL_EL2:208 case CNTHCTL_EL2:
@@ -1595,14 +1594,47 @@ static bool access_arch_timer(struct kvm_vcpu *vcpu, · 1385 unmodified lines
1595 return true;1594 return true;
1596}1595}
1597 1596
1598static bool access_hv_timer(struct kvm_vcpu *vcpu,1597static int arch_timer_set_user(struct kvm_vcpu *vcpu,
1599 struct sys_reg_params *p,1598 const struct sys_reg_desc *rd,
1600 const struct sys_reg_desc *r)1599 u64 val)
1601{1600{
1602 if (!vcpu_el2_e2h_is_set(vcpu))1601 switch (reg_to_encoding(rd)) {
1603 return undef_access(vcpu, p, r);1602 case SYS_CNTV_CTL_EL0:
1603 case SYS_CNTP_CTL_EL0:
1604 case SYS_CNTHV_CTL_EL2:
1605 case SYS_CNTHP_CTL_EL2:
1606 val &= ~ARCH_TIMER_CTRL_IT_STAT;
1607 break;
1608 case SYS_CNTVCT_EL0:
1609 if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags))
1610 timer_set_offset(vcpu_vtimer(vcpu), kvm_phys_timer_read() - val);
1611 return 0;
1612 case SYS_CNTPCT_EL0:
1613 if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags))
1614 timer_set_offset(vcpu_ptimer(vcpu), kvm_phys_timer_read() - val);
1615 return 0;
1616 }
1604 1617
1605 return access_arch_timer(vcpu, p, r);1618 __vcpu_assign_sys_reg(vcpu, rd->reg, val);
1619 return 0;
1620}
1621
1622static int arch_timer_get_user(struct kvm_vcpu *vcpu,
1623 const struct sys_reg_desc *rd,
1624 u64 *val)
1625{
1626 switch (reg_to_encoding(rd)) {
1627 case SYS_CNTVCT_EL0:
1628 *val = kvm_phys_timer_read() - timer_get_offset(vcpu_vtimer(vcpu));
1629 break;
1630 case SYS_CNTPCT_EL0:
1631 *val = kvm_phys_timer_read() - timer_get_offset(vcpu_ptimer(vcpu));
1632 break;
1633 default:
1634 *val = __vcpu_sys_reg(vcpu, rd->reg);
1635 }
1636
1637 return 0;
1606}1638}
1607 1639
1608static s64 kvm_arm64_ftr_safe_value(u32 id, const struct arm64_ftr_bits *ftrp,1640static s64 kvm_arm64_ftr_safe_value(u32 id, const struct arm64_ftr_bits *ftrp,
@@ -2507,15 +2539,20 @@ static bool bad_redir_trap(struct kvm_vcpu *vcpu, · 898 unmodified lines
2507 "trap of EL2 register redirected to EL1");2539 "trap of EL2 register redirected to EL1");
2508}2540}
2509 2541
2510#define EL2_REG_FILTERED(name, acc, rst, v, filter) { \2542#define SYS_REG_USER_FILTER(name, acc, rst, v, gu, su, filter) { \
2511 SYS_DESC(SYS_##name), \2543 SYS_DESC(SYS_##name), \
2512 .access = acc, \2544 .access = acc, \
2513 .reset = rst, \2545 .reset = rst, \
2514 .reg = name, \2546 .reg = name, \
2547 .get_user = gu, \
2548 .set_user = su, \
2515 .visibility = filter, \2549 .visibility = filter, \
2516 .val = v, \2550 .val = v, \
2517}2551}
2518 2552
2553#define EL2_REG_FILTERED(name, acc, rst, v, filter) \
2554 SYS_REG_USER_FILTER(name, acc, rst, v, NULL, NULL, filter)
2555
2519#define EL2_REG(name, acc, rst, v) \2556#define EL2_REG(name, acc, rst, v) \
2520 EL2_REG_FILTERED(name, acc, rst, v, el2_visibility)2557 EL2_REG_FILTERED(name, acc, rst, v, el2_visibility)
2521 2558
@@ -2526,6 +2563,10 @@ static bool bad_redir_trap(struct kvm_vcpu *vcpu, · 4 unmodified lines
2526 EL2_REG_VNCR_FILT(name, hidden_visibility)2563 EL2_REG_VNCR_FILT(name, hidden_visibility)
2527#define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v)2564#define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v)
2528 2565
2566#define TIMER_REG(name, vis) \
2567 SYS_REG_USER_FILTER(name, access_arch_timer, reset_val, 0, \
2568 arch_timer_get_user, arch_timer_set_user, vis)
2569
2529/*2570/*
2530 * Since reset() callback and field val are not used for idregs, they will be2571 * Since reset() callback and field val are not used for idregs, they will be
2531 * used for specific purposes for idregs.2572 * used for specific purposes for idregs.
@@ -2705,18 +2746,17 @@ static bool access_zcr_el2(struct kvm_vcpu *vcpu, · 173 unmodified lines
2705 2746
2706 if (guest_hyp_sve_traps_enabled(vcpu)) {2747 if (guest_hyp_sve_traps_enabled(vcpu)) {
2707 kvm_inject_nested_sve_trap(vcpu);2748 kvm_inject_nested_sve_trap(vcpu);
2708 return true;2749 return false;
2709 }2750 }
2710 2751
2711 if (!p->is_write) {2752 if (!p->is_write) {
2712 p->regval = vcpu_read_sys_reg(vcpu, ZCR_EL2);2753 p->regval = __vcpu_sys_reg(vcpu, ZCR_EL2);
2713 return true;2754 return true;
2714 }2755 }
2715 2756
2716 vq = SYS_FIELD_GET(ZCR_ELx, LEN, p->regval) + 1;2757 vq = SYS_FIELD_GET(ZCR_ELx, LEN, p->regval) + 1;
2717 vq = min(vq, vcpu_sve_max_vq(vcpu));2758 vq = min(vq, vcpu_sve_max_vq(vcpu));
2718 vcpu_write_sys_reg(vcpu, vq - 1, ZCR_EL2);2759 __vcpu_assign_sys_reg(vcpu, ZCR_EL2, vq - 1);
2719
2720 return true;2760 return true;
2721}2761}
2722 2762
@@ -2833,6 +2873,16 @@ static unsigned int s1pie_el2_visibility(const struct kvm_vcpu *vcpu, · 110 unmodified lines
2833 return __el2_visibility(vcpu, rd, s1pie_visibility);2873 return __el2_visibility(vcpu, rd, s1pie_visibility);
2834}2874}
2835 2875
2876static unsigned int cnthv_visibility(const struct kvm_vcpu *vcpu,
2877 const struct sys_reg_desc *rd)
2878{
2879 if (vcpu_has_nv(vcpu) &&
2880 !vcpu_has_feature(vcpu, KVM_ARM_VCPU_HAS_EL2_E2H0))
2881 return 0;
2882
2883 return REG_HIDDEN;
2884}
2885
2836static bool access_mdcr(struct kvm_vcpu *vcpu,2886static bool access_mdcr(struct kvm_vcpu *vcpu,
2837 struct sys_reg_params *p,2887 struct sys_reg_params *p,
2838 const struct sys_reg_desc *r)2888 const struct sys_reg_desc *r)
@@ -3482,17 +3532,19 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 643 unmodified lines
3482 AMU_AMEVTYPER1_EL0(14),3532 AMU_AMEVTYPER1_EL0(14),
3483 AMU_AMEVTYPER1_EL0(15),3533 AMU_AMEVTYPER1_EL0(15),
3484 3534
3485 { SYS_DESC(SYS_CNTPCT_EL0), access_arch_timer },3535 { SYS_DESC(SYS_CNTPCT_EL0), .access = access_arch_timer,
3486 { SYS_DESC(SYS_CNTVCT_EL0), access_arch_timer },3536 .get_user = arch_timer_get_user, .set_user = arch_timer_set_user },
3537 { SYS_DESC(SYS_CNTVCT_EL0), .access = access_arch_timer,
3538 .get_user = arch_timer_get_user, .set_user = arch_timer_set_user },
3487 { SYS_DESC(SYS_CNTPCTSS_EL0), access_arch_timer },3539 { SYS_DESC(SYS_CNTPCTSS_EL0), access_arch_timer },
3488 { SYS_DESC(SYS_CNTVCTSS_EL0), access_arch_timer },3540 { SYS_DESC(SYS_CNTVCTSS_EL0), access_arch_timer },
3489 { SYS_DESC(SYS_CNTP_TVAL_EL0), access_arch_timer },3541 { SYS_DESC(SYS_CNTP_TVAL_EL0), access_arch_timer },
3490 { SYS_DESC(SYS_CNTP_CTL_EL0), access_arch_timer },3542 TIMER_REG(CNTP_CTL_EL0, NULL),
3491 { SYS_DESC(SYS_CNTP_CVAL_EL0), access_arch_timer },3543 TIMER_REG(CNTP_CVAL_EL0, NULL),
3492 3544
3493 { SYS_DESC(SYS_CNTV_TVAL_EL0), access_arch_timer },3545 { SYS_DESC(SYS_CNTV_TVAL_EL0), access_arch_timer },
3494 { SYS_DESC(SYS_CNTV_CTL_EL0), access_arch_timer },3546 TIMER_REG(CNTV_CTL_EL0, NULL),
3495 { SYS_DESC(SYS_CNTV_CVAL_EL0), access_arch_timer },3547 TIMER_REG(CNTV_CVAL_EL0, NULL),
3496 3548
3497 /* PMEVCNTRn_EL0 */3549 /* PMEVCNTRn_EL0 */
3498 PMU_PMEVCNTR_EL0(0),3550 PMU_PMEVCNTR_EL0(0),
@@ -3690,12 +3742,12 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 191 unmodified lines
3690 EL2_REG_VNCR(CNTVOFF_EL2, reset_val, 0),3742 EL2_REG_VNCR(CNTVOFF_EL2, reset_val, 0),
3691 EL2_REG(CNTHCTL_EL2, access_rw, reset_val, 0),3743 EL2_REG(CNTHCTL_EL2, access_rw, reset_val, 0),
3692 { SYS_DESC(SYS_CNTHP_TVAL_EL2), access_arch_timer },3744 { SYS_DESC(SYS_CNTHP_TVAL_EL2), access_arch_timer },
3693 EL2_REG(CNTHP_CTL_EL2, access_arch_timer, reset_val, 0),3745 TIMER_REG(CNTHP_CTL_EL2, el2_visibility),
3694 EL2_REG(CNTHP_CVAL_EL2, access_arch_timer, reset_val, 0),3746 TIMER_REG(CNTHP_CVAL_EL2, el2_visibility),
3695 3747
3696 { SYS_DESC(SYS_CNTHV_TVAL_EL2), access_hv_timer },3748 { SYS_DESC(SYS_CNTHV_TVAL_EL2), access_arch_timer, .visibility = cnthv_visibility },
3697 EL2_REG(CNTHV_CTL_EL2, access_hv_timer, reset_val, 0),3749 TIMER_REG(CNTHV_CTL_EL2, cnthv_visibility),
3698 EL2_REG(CNTHV_CVAL_EL2, access_hv_timer, reset_val, 0),3750 TIMER_REG(CNTHV_CVAL_EL2, cnthv_visibility),
3699 3751
3700 { SYS_DESC(SYS_CNTKCTL_EL12), access_cntkctl_el12 },3752 { SYS_DESC(SYS_CNTKCTL_EL12), access_cntkctl_el12 },
3701 3753
@@ -5233,15 +5285,28 @@ static int demux_c15_set(struct kvm_vcpu *vcpu, u64 id, void __user *uaddr) · 1531 unmodified lines
5233 }5285 }
5234}5286}
5235 5287
5288static u64 kvm_one_reg_to_id(const struct kvm_one_reg *reg)
5289{
5290 switch(reg->id) {
5291 case KVM_REG_ARM_TIMER_CVAL:
5292 return TO_ARM64_SYS_REG(CNTV_CVAL_EL0);
5293 case KVM_REG_ARM_TIMER_CNT:
5294 return TO_ARM64_SYS_REG(CNTVCT_EL0);
5295 default:
5296 return reg->id;
5297 }
5298}
5299
5236int kvm_sys_reg_get_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg,5300int kvm_sys_reg_get_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg,
5237 const struct sys_reg_desc table[], unsigned int num)5301 const struct sys_reg_desc table[], unsigned int num)
5238{5302{
5239 u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr;5303 u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr;
5240 const struct sys_reg_desc *r;5304 const struct sys_reg_desc *r;
5305 u64 id = kvm_one_reg_to_id(reg);
5241 u64 val;5306 u64 val;
5242 int ret;5307 int ret;
5243 5308
5244 r = id_to_sys_reg_desc(vcpu, reg->id, table, num);5309 r = id_to_sys_reg_desc(vcpu, id, table, num);
5245 if (!r || sysreg_hidden(vcpu, r))5310 if (!r || sysreg_hidden(vcpu, r))
5246 return -ENOENT;5311 return -ENOENT;
5247 5312
@@ -5274,13 +5339,14 @@ int kvm_sys_reg_set_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg, · 26 unmodified lines
5274{5339{
5275 u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr;5340 u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr;
5276 const struct sys_reg_desc *r;5341 const struct sys_reg_desc *r;
5342 u64 id = kvm_one_reg_to_id(reg);
5277 u64 val;5343 u64 val;
5278 int ret;5344 int ret;
5279 5345
5280 if (get_user(val, uaddr))5346 if (get_user(val, uaddr))
5281 return -EFAULT;5347 return -EFAULT;
5282 5348
5283 r = id_to_sys_reg_desc(vcpu, reg->id, table, num);5349 r = id_to_sys_reg_desc(vcpu, id, table, num);
5284 if (!r || sysreg_hidden(vcpu, r))5350 if (!r || sysreg_hidden(vcpu, r))
5285 return -ENOENT;5351 return -ENOENT;
5286 5352
@@ -5340,10 +5406,23 @@ static u64 sys_reg_to_index(const struct sys_reg_desc *reg) · 53 unmodified lines
5340 5406
5341static bool copy_reg_to_user(const struct sys_reg_desc *reg, u64 __user **uind)5407static bool copy_reg_to_user(const struct sys_reg_desc *reg, u64 __user **uind)
5342{5408{
5409 u64 idx;
5410
5343 if (!*uind)5411 if (!*uind)
5344 return true;5412 return true;
5345 5413
5346 if (put_user(sys_reg_to_index(reg), *uind))5414 switch (reg_to_encoding(reg)) {
5415 case SYS_CNTV_CVAL_EL0:
5416 idx = KVM_REG_ARM_TIMER_CVAL;
5417 break;
5418 case SYS_CNTVCT_EL0:
5419 idx = KVM_REG_ARM_TIMER_CNT;
5420 break;
5421 default:
5422 idx = sys_reg_to_index(reg);
5423 }
5424
5425 if (put_user(idx, *uind))
5347 return false;5426 return false;
5348 5427
5349 (*uind)++;5428 (*uind)++;
arch/arm64/kvm/sys_regs.h+6−0
@@ -257,4 +257,10 @@ int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu); · 256 unmodified lines
257 (val); \257 (val); \
258})258})
259 259
260#define TO_ARM64_SYS_REG(r) ARM64_SYS_REG(sys_reg_Op0(SYS_ ## r), \
261 sys_reg_Op1(SYS_ ## r), \
262 sys_reg_CRn(SYS_ ## r), \
263 sys_reg_CRm(SYS_ ## r), \
264 sys_reg_Op2(SYS_ ## r))
265
260#endif /* __ARM64_KVM_SYS_REGS_LOCAL_H__ */266#endif /* __ARM64_KVM_SYS_REGS_LOCAL_H__ */
arch/arm64/kvm/vgic/vgic-v3.c+4−1
@@ -297,8 +297,11 @@ void vcpu_set_ich_hcr(struct kvm_vcpu *vcpu) · 296 unmodified lines
297{297{
298 struct vgic_v3_cpu_if *vgic_v3 = &vcpu->arch.vgic_cpu.vgic_v3;298 struct vgic_v3_cpu_if *vgic_v3 = &vcpu->arch.vgic_cpu.vgic_v3;
299 299
300 if (!vgic_is_v3(vcpu->kvm))
301 return;
302
300 /* Hide GICv3 sysreg if necessary */303 /* Hide GICv3 sysreg if necessary */
301 if (!kvm_has_gicv3(vcpu->kvm)) {304 if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2) {
302 vgic_v3->vgic_hcr |= (ICH_HCR_EL2_TALL0 | ICH_HCR_EL2_TALL1 |305 vgic_v3->vgic_hcr |= (ICH_HCR_EL2_TALL0 | ICH_HCR_EL2_TALL1 |
303 ICH_HCR_EL2_TC);306 ICH_HCR_EL2_TC);
304 return;307 return;
include/kvm/arm_arch_timer.h+16−8
@@ -51,8 +51,6 @@ struct arch_timer_vm_data { · 50 unmodified lines
51};51};
52 52
53struct arch_timer_context {53struct arch_timer_context {
54 struct kvm_vcpu *vcpu;
55
56 /* Emulated Timer (may be unused) */54 /* Emulated Timer (may be unused) */
57 struct hrtimer hrtimer;55 struct hrtimer hrtimer;
58 u64 ns_frac;56 u64 ns_frac;
@@ -71,6 +69,9 @@ struct arch_timer_context { · 12 unmodified lines
71 bool level;69 bool level;
72 } irq;70 } irq;
73 71
72 /* Who am I? */
73 enum kvm_arch_timers timer_id;
74
74 /* Duplicated state from arch_timer.c for convenience */75 /* Duplicated state from arch_timer.c for convenience */
75 u32 host_timer_irq;76 u32 host_timer_irq;
76};77};
@@ -106,9 +107,6 @@ void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu); · 29 unmodified lines
106 107
107void kvm_timer_init_vm(struct kvm *kvm);108void kvm_timer_init_vm(struct kvm *kvm);
108 109
109u64 kvm_arm_timer_get_reg(struct kvm_vcpu *, u64 regid);
110int kvm_arm_timer_set_reg(struct kvm_vcpu *, u64 regid, u64 value);
111
112int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);110int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
113int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);111int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
114int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);112int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
@@ -127,9 +125,9 @@ void kvm_timer_init_vhe(void); · 12 unmodified lines
127#define vcpu_hvtimer(v) (&(v)->arch.timer_cpu.timers[TIMER_HVTIMER])125#define vcpu_hvtimer(v) (&(v)->arch.timer_cpu.timers[TIMER_HVTIMER])
128#define vcpu_hptimer(v) (&(v)->arch.timer_cpu.timers[TIMER_HPTIMER])126#define vcpu_hptimer(v) (&(v)->arch.timer_cpu.timers[TIMER_HPTIMER])
129 127
130#define arch_timer_ctx_index(ctx) ((ctx) - vcpu_timer((ctx)->vcpu)->timers)128#define arch_timer_ctx_index(ctx) ((ctx)->timer_id)
131 129#define timer_context_to_vcpu(ctx) container_of((ctx), struct kvm_vcpu, arch.timer_cpu.timers[(ctx)->timer_id])
132#define timer_vm_data(ctx) (&(ctx)->vcpu->kvm->arch.timer_data)130#define timer_vm_data(ctx) (&(timer_context_to_vcpu(ctx)->kvm->arch.timer_data))
133#define timer_irq(ctx) (timer_vm_data(ctx)->ppi[arch_timer_ctx_index(ctx)])131#define timer_irq(ctx) (timer_vm_data(ctx)->ppi[arch_timer_ctx_index(ctx)])
134 132
135u64 kvm_arm_timer_read_sysreg(struct kvm_vcpu *vcpu,133u64 kvm_arm_timer_read_sysreg(struct kvm_vcpu *vcpu,
@@ -178,4 +176,14 @@ static inline u64 timer_get_offset(struct arch_timer_context *ctxt) · 42 unmodified lines
178 return offset;176 return offset;
179}177}
180 178
179static inline void timer_set_offset(struct arch_timer_context *ctxt, u64 offset)
180{
181 if (!ctxt->offset.vm_offset) {
182 WARN(offset, "timer %d\n", arch_timer_ctx_index(ctxt));
183 return;
184 }
185
186 WRITE_ONCE(*ctxt->offset.vm_offset, offset);
187}
188
181#endif189#endif
tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c+1−1
@@ -1020,7 +1020,7 @@ static void set_counter_defaults(void) · 1019 unmodified lines
1020{1020{
1021 const uint64_t MIN_ROLLOVER_SECS = 40ULL * 365 * 24 * 3600;1021 const uint64_t MIN_ROLLOVER_SECS = 40ULL * 365 * 24 * 3600;
1022 uint64_t freq = read_sysreg(CNTFRQ_EL0);1022 uint64_t freq = read_sysreg(CNTFRQ_EL0);
1023 uint64_t width = ilog2(MIN_ROLLOVER_SECS * freq);1023 int width = ilog2(MIN_ROLLOVER_SECS * freq);
1024 1024
1025 width = clamp(width, 56, 64);1025 width = clamp(width, 56, 64);
1026 CVAL_MAX = GENMASK_ULL(width - 1, 0);1026 CVAL_MAX = GENMASK_ULL(width - 1, 0);
tools/testing/selftests/kvm/arm64/external_aborts.c+43−0
@@ -359,6 +359,44 @@ static void test_mmio_ease(void) · 358 unmodified lines
359 kvm_vm_free(vm);359 kvm_vm_free(vm);
360}360}
361 361
362static void test_serror_amo_guest(void)
363{
364 /*
365 * The ISB is entirely unnecessary (and highlights how FEAT_NV2 is borked)
366 * since the write is redirected to memory. But don't write (intentionally)
367 * broken code!
368 */
369 sysreg_clear_set(hcr_el2, HCR_EL2_AMO | HCR_EL2_TGE, 0);
370 isb();
371
372 GUEST_SYNC(0);
373 GUEST_ASSERT(read_sysreg(isr_el1) & ISR_EL1_A);
374
375 /*
376 * KVM treats the effective value of AMO as 1 when
377 * HCR_EL2.{E2H,TGE} = {1, 0}, meaning the SError will be taken when
378 * unmasked.
379 */
380 local_serror_enable();
381 isb();
382 local_serror_disable();
383
384 GUEST_FAIL("Should've taken pending SError exception");
385}
386
387static void test_serror_amo(void)
388{
389 struct kvm_vcpu *vcpu;
390 struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_serror_amo_guest,
391 unexpected_dabt_handler);
392
393 vm_install_exception_handler(vm, VECTOR_ERROR_CURRENT, expect_serror_handler);
394 vcpu_run_expect_sync(vcpu);
395 vcpu_inject_serror(vcpu);
396 vcpu_run_expect_done(vcpu);
397 kvm_vm_free(vm);
398}
399
362int main(void)400int main(void)
363{401{
364 test_mmio_abort();402 test_mmio_abort();
@@ -369,4 +407,9 @@ int main(void) · 4 unmodified lines
369 test_serror_emulated();407 test_serror_emulated();
370 test_mmio_ease();408 test_mmio_ease();
371 test_s1ptw_abort();409 test_s1ptw_abort();
410
411 if (!test_supports_el2())
412 return 0;
413
414 test_serror_amo();
372}415}
tools/testing/selftests/kvm/arm64/get-reg-list.c+96−3
@@ -65,6 +65,9 @@ static struct feature_id_reg feat_id_regs[] = { · 64 unmodified lines
65 REG_FEAT(SCTLR2_EL1, ID_AA64MMFR3_EL1, SCTLRX, IMP),65 REG_FEAT(SCTLR2_EL1, ID_AA64MMFR3_EL1, SCTLRX, IMP),
66 REG_FEAT(VDISR_EL2, ID_AA64PFR0_EL1, RAS, IMP),66 REG_FEAT(VDISR_EL2, ID_AA64PFR0_EL1, RAS, IMP),
67 REG_FEAT(VSESR_EL2, ID_AA64PFR0_EL1, RAS, IMP),67 REG_FEAT(VSESR_EL2, ID_AA64PFR0_EL1, RAS, IMP),
68 REG_FEAT(VNCR_EL2, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY),
69 REG_FEAT(CNTHV_CTL_EL2, ID_AA64MMFR1_EL1, VH, IMP),
70 REG_FEAT(CNTHV_CVAL_EL2,ID_AA64MMFR1_EL1, VH, IMP),
68};71};
69 72
70bool filter_reg(__u64 reg)73bool filter_reg(__u64 reg)
@@ -345,9 +348,20 @@ static __u64 base_regs[] = { · 274 unmodified lines
345 KVM_REG_ARM_FW_FEAT_BMAP_REG(1), /* KVM_REG_ARM_STD_HYP_BMAP */348 KVM_REG_ARM_FW_FEAT_BMAP_REG(1), /* KVM_REG_ARM_STD_HYP_BMAP */
346 KVM_REG_ARM_FW_FEAT_BMAP_REG(2), /* KVM_REG_ARM_VENDOR_HYP_BMAP */349 KVM_REG_ARM_FW_FEAT_BMAP_REG(2), /* KVM_REG_ARM_VENDOR_HYP_BMAP */
347 KVM_REG_ARM_FW_FEAT_BMAP_REG(3), /* KVM_REG_ARM_VENDOR_HYP_BMAP_2 */350 KVM_REG_ARM_FW_FEAT_BMAP_REG(3), /* KVM_REG_ARM_VENDOR_HYP_BMAP_2 */
348 ARM64_SYS_REG(3, 3, 14, 3, 1), /* CNTV_CTL_EL0 */351
349 ARM64_SYS_REG(3, 3, 14, 3, 2), /* CNTV_CVAL_EL0 */352 /*
350 ARM64_SYS_REG(3, 3, 14, 0, 2),353 * EL0 Virtual Timer Registers
354 *
355 * WARNING:
356 * KVM_REG_ARM_TIMER_CVAL and KVM_REG_ARM_TIMER_CNT are not defined
357 * with the appropriate register encodings. Their values have been
358 * accidentally swapped. As this is set API, the definitions here
359 * must be used, rather than ones derived from the encodings.
360 */
361 KVM_ARM64_SYS_REG(SYS_CNTV_CTL_EL0),
362 KVM_REG_ARM_TIMER_CVAL,
363 KVM_REG_ARM_TIMER_CNT,
364
351 ARM64_SYS_REG(3, 0, 0, 0, 0), /* MIDR_EL1 */365 ARM64_SYS_REG(3, 0, 0, 0, 0), /* MIDR_EL1 */
352 ARM64_SYS_REG(3, 0, 0, 0, 6), /* REVIDR_EL1 */366 ARM64_SYS_REG(3, 0, 0, 0, 6), /* REVIDR_EL1 */
353 ARM64_SYS_REG(3, 1, 0, 0, 1), /* CLIDR_EL1 */367 ARM64_SYS_REG(3, 1, 0, 0, 1), /* CLIDR_EL1 */
@@ -755,6 +769,10 @@ static __u64 el2_regs[] = { · 401 unmodified lines
755 SYS_REG(VSESR_EL2),769 SYS_REG(VSESR_EL2),
756};770};
757 771
772static __u64 el2_e2h0_regs[] = {
773 /* Empty */
774};
775
758#define BASE_SUBLIST \776#define BASE_SUBLIST \
759 { "base", .regs = base_regs, .regs_n = ARRAY_SIZE(base_regs), }777 { "base", .regs = base_regs, .regs_n = ARRAY_SIZE(base_regs), }
760#define VREGS_SUBLIST \778#define VREGS_SUBLIST \
@@ -789,6 +807,15 @@ static __u64 el2_regs[] = { · 28 unmodified lines
789 .regs = el2_regs, \807 .regs = el2_regs, \
790 .regs_n = ARRAY_SIZE(el2_regs), \808 .regs_n = ARRAY_SIZE(el2_regs), \
791 }809 }
810#define EL2_E2H0_SUBLIST \
811 EL2_SUBLIST, \
812 { \
813 .name = "EL2 E2H0", \
814 .capability = KVM_CAP_ARM_EL2_E2H0, \
815 .feature = KVM_ARM_VCPU_HAS_EL2_E2H0, \
816 .regs = el2_e2h0_regs, \
817 .regs_n = ARRAY_SIZE(el2_e2h0_regs), \
818 }
792 819
793static struct vcpu_reg_list vregs_config = {820static struct vcpu_reg_list vregs_config = {
794 .sublists = {821 .sublists = {
@@ -897,6 +924,65 @@ static struct vcpu_reg_list el2_pauth_pmu_config = { · 102 unmodified lines
897 },924 },
898};925};
899 926
927static struct vcpu_reg_list el2_e2h0_vregs_config = {
928 .sublists = {
929 BASE_SUBLIST,
930 EL2_E2H0_SUBLIST,
931 VREGS_SUBLIST,
932 {0},
933 },
934};
935
936static struct vcpu_reg_list el2_e2h0_vregs_pmu_config = {
937 .sublists = {
938 BASE_SUBLIST,
939 EL2_E2H0_SUBLIST,
940 VREGS_SUBLIST,
941 PMU_SUBLIST,
942 {0},
943 },
944};
945
946static struct vcpu_reg_list el2_e2h0_sve_config = {
947 .sublists = {
948 BASE_SUBLIST,
949 EL2_E2H0_SUBLIST,
950 SVE_SUBLIST,
951 {0},
952 },
953};
954
955static struct vcpu_reg_list el2_e2h0_sve_pmu_config = {
956 .sublists = {
957 BASE_SUBLIST,
958 EL2_E2H0_SUBLIST,
959 SVE_SUBLIST,
960 PMU_SUBLIST,
961 {0},
962 },
963};
964
965static struct vcpu_reg_list el2_e2h0_pauth_config = {
966 .sublists = {
967 BASE_SUBLIST,
968 EL2_E2H0_SUBLIST,
969 VREGS_SUBLIST,
970 PAUTH_SUBLIST,
971 {0},
972 },
973};
974
975static struct vcpu_reg_list el2_e2h0_pauth_pmu_config = {
976 .sublists = {
977 BASE_SUBLIST,
978 EL2_E2H0_SUBLIST,
979 VREGS_SUBLIST,
980 PAUTH_SUBLIST,
981 PMU_SUBLIST,
982 {0},
983 },
984};
985
900struct vcpu_reg_list *vcpu_configs[] = {986struct vcpu_reg_list *vcpu_configs[] = {
901 &vregs_config,987 &vregs_config,
902 &vregs_pmu_config,988 &vregs_pmu_config,
@@ -911,5 +997,12 @@ struct vcpu_reg_list *vcpu_configs[] = { · 8 unmodified lines
911 &el2_sve_pmu_config,997 &el2_sve_pmu_config,
912 &el2_pauth_config,998 &el2_pauth_config,
913 &el2_pauth_pmu_config,999 &el2_pauth_pmu_config,
1000
1001 &el2_e2h0_vregs_config,
1002 &el2_e2h0_vregs_pmu_config,
1003 &el2_e2h0_sve_config,
1004 &el2_e2h0_sve_pmu_config,
1005 &el2_e2h0_pauth_config,
1006 &el2_e2h0_pauth_pmu_config,
914};1007};
915int vcpu_configs_n = ARRAY_SIZE(vcpu_configs);1008int vcpu_configs_n = ARRAY_SIZE(vcpu_configs);
tools/testing/selftests/kvm/arm64/set_id_regs.c+3−0
@@ -249,11 +249,14 @@ static void guest_code(void) · 248 unmodified lines
249 GUEST_REG_SYNC(SYS_ID_AA64ISAR2_EL1);249 GUEST_REG_SYNC(SYS_ID_AA64ISAR2_EL1);
250 GUEST_REG_SYNC(SYS_ID_AA64ISAR3_EL1);250 GUEST_REG_SYNC(SYS_ID_AA64ISAR3_EL1);
251 GUEST_REG_SYNC(SYS_ID_AA64PFR0_EL1);251 GUEST_REG_SYNC(SYS_ID_AA64PFR0_EL1);
252 GUEST_REG_SYNC(SYS_ID_AA64PFR1_EL1);
252 GUEST_REG_SYNC(SYS_ID_AA64MMFR0_EL1);253 GUEST_REG_SYNC(SYS_ID_AA64MMFR0_EL1);
253 GUEST_REG_SYNC(SYS_ID_AA64MMFR1_EL1);254 GUEST_REG_SYNC(SYS_ID_AA64MMFR1_EL1);
254 GUEST_REG_SYNC(SYS_ID_AA64MMFR2_EL1);255 GUEST_REG_SYNC(SYS_ID_AA64MMFR2_EL1);
255 GUEST_REG_SYNC(SYS_ID_AA64MMFR3_EL1);256 GUEST_REG_SYNC(SYS_ID_AA64MMFR3_EL1);
256 GUEST_REG_SYNC(SYS_ID_AA64ZFR0_EL1);257 GUEST_REG_SYNC(SYS_ID_AA64ZFR0_EL1);
258 GUEST_REG_SYNC(SYS_MPIDR_EL1);
259 GUEST_REG_SYNC(SYS_CLIDR_EL1);
257 GUEST_REG_SYNC(SYS_CTR_EL0);260 GUEST_REG_SYNC(SYS_CTR_EL0);
258 GUEST_REG_SYNC(SYS_MIDR_EL1);261 GUEST_REG_SYNC(SYS_MIDR_EL1);
259 GUEST_REG_SYNC(SYS_REVIDR_EL1);262 GUEST_REG_SYNC(SYS_REVIDR_EL1);
tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c+2−1
@@ -123,6 +123,7 @@ static void guest_setup_gic(void) · 122 unmodified lines
123static void guest_code(size_t nr_lpis)123static void guest_code(size_t nr_lpis)
124{124{
125 guest_setup_gic();125 guest_setup_gic();
126 local_irq_enable();
126 127
127 GUEST_SYNC(0);128 GUEST_SYNC(0);
128 129
@@ -331,7 +332,7 @@ static void setup_vm(void) · 202 unmodified lines
331{332{
332 int i;333 int i;
333 334
334 vcpus = malloc(test_data.nr_cpus * sizeof(struct kvm_vcpu));335 vcpus = malloc(test_data.nr_cpus * sizeof(struct kvm_vcpu *));
335 TEST_ASSERT(vcpus, "Failed to allocate vCPU array");336 TEST_ASSERT(vcpus, "Failed to allocate vCPU array");
336 337
337 vm = vm_create_with_vcpus(test_data.nr_cpus, guest_code, vcpus);338 vm = vm_create_with_vcpus(test_data.nr_cpus, guest_code, vcpus);
tools/testing/selftests/kvm/include/arm64/processor.h+11−1
@@ -305,7 +305,17 @@ void test_wants_mte(void); · 304 unmodified lines
305void test_disable_default_vgic(void);305void test_disable_default_vgic(void);
306 306
307bool vm_supports_el2(struct kvm_vm *vm);307bool vm_supports_el2(struct kvm_vm *vm);
308static bool vcpu_has_el2(struct kvm_vcpu *vcpu)308
309static inline bool test_supports_el2(void)
310{
311 struct kvm_vm *vm = vm_create(1);
312 bool supported = vm_supports_el2(vm);
313
314 kvm_vm_free(vm);
315 return supported;
316}
317
318static inline bool vcpu_has_el2(struct kvm_vcpu *vcpu)
309{319{
310 return vcpu->init.features[0] & BIT(KVM_ARM_VCPU_HAS_EL2);320 return vcpu->init.features[0] & BIT(KVM_ARM_VCPU_HAS_EL2);
311}321}
tools/testing/selftests/kvm/include/kvm_util.h+2−0
@@ -1273,4 +1273,6 @@ bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr); · 1272 unmodified lines
1273 1273
1274uint32_t guest_get_vcpuid(void);1274uint32_t guest_get_vcpuid(void);
1275 1275
1276bool kvm_arch_has_default_irqchip(void);
1277
1276#endif /* SELFTEST_KVM_UTIL_H */1278#endif /* SELFTEST_KVM_UTIL_H */
tools/testing/selftests/kvm/irqfd_test.c+11−3
@@ -89,11 +89,19 @@ static void juggle_eventfd_primary(struct kvm_vm *vm, int eventfd) · 88 unmodified lines
89int main(int argc, char *argv[])89int main(int argc, char *argv[])
90{90{
91 pthread_t racing_thread;91 pthread_t racing_thread;
92 struct kvm_vcpu *unused;
92 int r, i;93 int r, i;
93 94
94 /* Create "full" VMs, as KVM_IRQFD requires an in-kernel IRQ chip. */95 TEST_REQUIRE(kvm_arch_has_default_irqchip());
95 vm1 = vm_create(1);96
96 vm2 = vm_create(1);97 /*
98 * Create "full" VMs, as KVM_IRQFD requires an in-kernel IRQ chip. Also
99 * create an unused vCPU as certain architectures (like arm64) need to
100 * complete IRQ chip initialization after all possible vCPUs for a VM
101 * have been created.
102 */
103 vm1 = vm_create_with_one_vcpu(&unused, NULL);
104 vm2 = vm_create_with_one_vcpu(&unused, NULL);
97 105
98 WRITE_ONCE(__eventfd, kvm_new_eventfd());106 WRITE_ONCE(__eventfd, kvm_new_eventfd());
99 107
tools/testing/selftests/kvm/lib/arm64/processor.c+5−0
@@ -725,3 +725,8 @@ void kvm_arch_vm_release(struct kvm_vm *vm) · 724 unmodified lines
725 if (vm->arch.has_gic)725 if (vm->arch.has_gic)
726 close(vm->arch.gic_fd);726 close(vm->arch.gic_fd);
727}727}
728
729bool kvm_arch_has_default_irqchip(void)
730{
731 return request_vgic && kvm_supports_vgic_v3();
732}
tools/testing/selftests/kvm/lib/kvm_util.c+5−0
@@ -2344,3 +2344,8 @@ bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr) · 2343 unmodified lines
2344 pg = paddr >> vm->page_shift;2344 pg = paddr >> vm->page_shift;
2345 return sparsebit_is_set(region->protected_phy_pages, pg);2345 return sparsebit_is_set(region->protected_phy_pages, pg);
2346}2346}
2347
2348__weak bool kvm_arch_has_default_irqchip(void)
2349{
2350 return false;
2351}
tools/testing/selftests/kvm/lib/s390/processor.c+5−0
@@ -221,3 +221,8 @@ void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent) · 220 unmodified lines
221void assert_on_unhandled_exception(struct kvm_vcpu *vcpu)221void assert_on_unhandled_exception(struct kvm_vcpu *vcpu)
222{222{
223}223}
224
225bool kvm_arch_has_default_irqchip(void)
226{
227 return true;
228}
tools/testing/selftests/kvm/lib/x86/processor.c+5−0
@@ -1318,3 +1318,8 @@ bool sys_clocksource_is_based_on_tsc(void) · 1317 unmodified lines
1318 1318
1319 return ret;1319 return ret;
1320}1320}
1321
1322bool kvm_arch_has_default_irqchip(void)
1323{
1324 return true;
1325}