Merge tag 'kvmarm-fixes-6.18-1' of git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into HEAD

KVM/arm64 fixes for 6.18, take #1 Improvements and bug fixes: - Fix the handling of ZCR_EL2 in NV VMs (20250926194108.84093-1-oliver.upton@linux.dev) - Pick the correct translation regime when doing a PTW on the back of a SEA (20250926224246.731748-1-oliver.upton@linux.dev) - Prevent userspace from injecting an event into a vcpu that isn't initialised yet (20250930085237.108326-1-oliver.upton@linux.dev) - Move timer save/restore to the sysreg handling code, fixing EL2 timer access in the process (20250929160458.3351788-1-maz@kernel.org) - Add FGT-based trapping of MDSCR_EL1 to reduce the overhead of debug (20250924235150.617451-1-oliver.upton@linux.dev) - Fix trapping configuration when the host isn't GICv3 (20251007160704.1673584-1-sascha.bischoff@arm.com) - Improve the detection of HCR_EL2.E2H being RES1 (20251009121239.29370-1-maz@kernel.org) - Drop a spurious 'break' statement in the S1 PTW (20250930135621.162050-1-osama.abdelkader@gmail.com) - Don't try to access SPE when owned by EL3 (20251010174707.1684200-1-mukesh.ojha@oss.qualcomm.com) Documentation updates: - Document the failure modes of event injection (20250930233620.124607-1-oliver.upton@linux.dev) - Document that a GICv3 guest can be created on a GICv5 host with FEAT_GCIE_LEGACY (20251007154848.1640444-1-sascha.bischoff@arm.com) Selftest improvements: - Add a selftest for the effective value of HCR_EL2.AMO (20250926224454.734066-1-oliver.upton@linux.dev) - Address build warning in the timer selftest when building with clang (20250926155838.2612205-1-seanjc@google.com) - Teach irq_fd selftests about non-x86 architectures (20250930193301.119859-1-oliver.upton@linux.dev) - Add missing sysregs to the set_id_regs selftest (20251012154352.61133-1-zenghui.yu@linux.dev) - Fix vcpu allocation in the vgic_lpi_stress selftest (20251008154520.54801-1-zenghui.yu@linux.dev) - Correctly enable interrupts in the vgic_lpi_stress selftest (20251007195254.260539-1-oliver.upton@linux.dev)

Paolo Bonzini committed 1 year ago

Commit 5d26eaa2 parents 3a86608 ca88ecd30 files · +565 −354Browse files

Unified · Split

30 files

Documentation/virt/kvm/api.rst +5 −0
Documentation/virt/kvm/devices/arm-vgic-v3.rst +2 −1
arch/arm64/include/asm/el2_setup.h +32 −6
arch/arm64/include/asm/kvm_host.h +50 −0
arch/arm64/kvm/arch_timer.c +14 −91
arch/arm64/kvm/arm.c +7 −0
arch/arm64/kvm/at.c +5 −2
arch/arm64/kvm/config.c +90 −0
arch/arm64/kvm/debug.c +10 −5
arch/arm64/kvm/guest.c +0 −70
arch/arm64/kvm/handle_exit.c +6 −1
arch/arm64/kvm/hyp/include/hyp/switch.h +17 −131
arch/arm64/kvm/hyp/nvhe/pkvm.c +1 −0
arch/arm64/kvm/nested.c +6 −3
arch/arm64/kvm/sys_regs.c +105 −26
arch/arm64/kvm/sys_regs.h +6 −0
arch/arm64/kvm/vgic/vgic-v3.c +4 −1
include/kvm/arm_arch_timer.h +16 −8
tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c +1 −1
tools/testing/selftests/kvm/arm64/external_aborts.c +43 −0
tools/testing/selftests/kvm/arm64/get-reg-list.c +96 −3
tools/testing/selftests/kvm/arm64/set_id_regs.c +3 −0
tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c +2 −1
tools/testing/selftests/kvm/include/arm64/processor.h +11 −1
tools/testing/selftests/kvm/include/kvm_util.h +2 −0
tools/testing/selftests/kvm/irqfd_test.c +11 −3
tools/testing/selftests/kvm/lib/arm64/processor.c +5 −0
tools/testing/selftests/kvm/lib/kvm_util.c +5 −0
tools/testing/selftests/kvm/lib/s390/processor.c +5 −0
tools/testing/selftests/kvm/lib/x86/processor.c +5 −0

Documentation/virt/kvm/api.rst+5−0

@@ -1229,6 +1229,9 @@ It is not possible to read back a pending external abort (injected via · 1228 unmodified lines
1229	1229	KVM_SET_VCPU_EVENTS or otherwise) because such an exception is always delivered
1230	1230	directly to the virtual CPU).
1231	1231
	1232	+Calling this ioctl on a vCPU that hasn't been initialized will return
	1233	+-ENOEXEC.
	1234	+
1232	1235	::
1233	1236
1234	1237	struct kvm_vcpu_events {
@@ -1309,6 +1312,8 @@ exceptions by manipulating individual registers using the KVM_SET_ONE_REG API. · 74 unmodified lines
1309	1312
1310	1313	See KVM_GET_VCPU_EVENTS for the data structure.
1311	1314
	1315	+Calling this ioctl on a vCPU that hasn't been initialized will return
	1316	+-ENOEXEC.
1312	1317
1313	1318	4.33 KVM_GET_DEBUGREGS
1314	1319	----------------------

Documentation/virt/kvm/devices/arm-vgic-v3.rst+2−1

@@ -13,7 +13,8 @@ will act as the VM interrupt controller, requiring emulated user-space devices · 12 unmodified lines
13	13	to inject interrupts to the VGIC instead of directly to CPUs. It is not
14	14	possible to create both a GICv3 and GICv2 on the same VM.
15	15
16		−Creating a guest GICv3 device requires a host GICv3 as well.
	16	+Creating a guest GICv3 device requires a host GICv3 host, or a GICv5 host with
	17	+support for FEAT_GCIE_LEGACY.
17	18
18	19
19	20	Groups:

arch/arm64/include/asm/el2_setup.h+32−6

@@ -24,22 +24,48 @@ · 23 unmodified lines
24	24	* ID_AA64MMFR4_EL1.E2H0 < 0. On such CPUs HCR_EL2.E2H is RES1, but it
25	25	* can reset into an UNKNOWN state and might not read as 1 until it has
26	26	* been initialized explicitly.
27		− *
28		− * Fruity CPUs seem to have HCR_EL2.E2H set to RAO/WI, but
29		− * don't advertise it (they predate this relaxation).
30		− *
31	27	* Initalize HCR_EL2.E2H so that later code can rely upon HCR_EL2.E2H
32	28	* indicating whether the CPU is running in E2H mode.
33	29	*/
34	30	mrs_s x1, SYS_ID_AA64MMFR4_EL1
35	31	sbfx x1, x1, #ID_AA64MMFR4_EL1_E2H0_SHIFT, #ID_AA64MMFR4_EL1_E2H0_WIDTH
36	32	cmp x1, #0
37		− b.ge .LnVHE_\@
	33	+ b.lt .LnE2H0_\@
38	34
	35	+ /*
	36	+ * Unfortunately, HCR_EL2.E2H can be RES1 even if not advertised
	37	+ * as such via ID_AA64MMFR4_EL1.E2H0:
	38	+ *
	39	+ * - Fruity CPUs predate the !FEAT_E2H0 relaxation, and seem to
	40	+ * have HCR_EL2.E2H implemented as RAO/WI.
	41	+ *
	42	+ * - On CPUs that lack FEAT_FGT, a hypervisor can't trap guest
	43	+ * reads of ID_AA64MMFR4_EL1 to advertise !FEAT_E2H0. NV
	44	+ * guests on these hosts can write to HCR_EL2.E2H without
	45	+ * trapping to the hypervisor, but these writes have no
	46	+ * functional effect.
	47	+ *
	48	+ * Handle both cases by checking for an essential VHE property
	49	+ * (system register remapping) to decide whether we're
	50	+ * effectively VHE-only or not.
	51	+ */
	52	+ msr_hcr_el2 x0 // Setup HCR_EL2 as nVHE
	53	+ isb
	54	+ mov x1, #1 // Write something to FAR_EL1
	55	+ msr far_el1, x1
	56	+ isb
	57	+ mov x1, #2 // Try to overwrite it via FAR_EL2
	58	+ msr far_el2, x1
	59	+ isb
	60	+ mrs x1, far_el1 // If we see the latest write in FAR_EL1,
	61	+ cmp x1, #2 // we can safely assume we are VHE only.
	62	+ b.ne .LnVHE_\@ // Otherwise, we know that nVHE works.
	63	+
	64	+.LnE2H0_\@:
39	65	orr x0, x0, #HCR_E2H
40		−.LnVHE_\@:
41	66	msr_hcr_el2 x0
42	67	isb
	68	+.LnVHE_\@:
43	69	.endm
44	70
45	71	.macro __init_el2_sctlr

arch/arm64/include/asm/kvm_host.h+50−0

@@ -816,6 +816,11 @@ struct kvm_vcpu_arch { · 815 unmodified lines
816	816	u64 hcrx_el2;
817	817	u64 mdcr_el2;
818	818
	819	+ struct {
	820	+ u64 r;
	821	+ u64 w;
	822	+ } fgt[__NR_FGT_GROUP_IDS__];
	823	+
819	824	/* Exception Information */
820	825	struct kvm_vcpu_fault_info fault;
821	826
@@ -1600,6 +1605,51 @@ static inline bool kvm_arch_has_irq_bypass(void) · 778 unmodified lines
1600	1605	void compute_fgu(struct kvm *kvm, enum fgt_group_id fgt);
1601	1606	void get_reg_fixed_bits(struct kvm kvm, enum vcpu_sysreg reg, u64 res0, u64 *res1);
1602	1607	void check_feature_map(void);
	1608	+void kvm_vcpu_load_fgt(struct kvm_vcpu *vcpu);
	1609	+
	1610	+static __always_inline enum fgt_group_id __fgt_reg_to_group_id(enum vcpu_sysreg reg)
	1611	+{
	1612	+ switch (reg) {
	1613	+ case HFGRTR_EL2:
	1614	+ case HFGWTR_EL2:
	1615	+ return HFGRTR_GROUP;
	1616	+ case HFGITR_EL2:
	1617	+ return HFGITR_GROUP;
	1618	+ case HDFGRTR_EL2:
	1619	+ case HDFGWTR_EL2:
	1620	+ return HDFGRTR_GROUP;
	1621	+ case HAFGRTR_EL2:
	1622	+ return HAFGRTR_GROUP;
	1623	+ case HFGRTR2_EL2:
	1624	+ case HFGWTR2_EL2:
	1625	+ return HFGRTR2_GROUP;
	1626	+ case HFGITR2_EL2:
	1627	+ return HFGITR2_GROUP;
	1628	+ case HDFGRTR2_EL2:
	1629	+ case HDFGWTR2_EL2:
	1630	+ return HDFGRTR2_GROUP;
	1631	+ default:
	1632	+ BUILD_BUG_ON(1);
	1633	+ }
	1634	+}
1603	1635
	1636	+#define vcpu_fgt(vcpu, reg) \
	1637	+ ({ \
	1638	+ enum fgt_group_id id = __fgt_reg_to_group_id(reg); \
	1639	+ u64 *p; \
	1640	+ switch (reg) { \
	1641	+ case HFGWTR_EL2: \
	1642	+ case HDFGWTR_EL2: \
	1643	+ case HFGWTR2_EL2: \
	1644	+ case HDFGWTR2_EL2: \
	1645	+ p = &(vcpu)->arch.fgt[id].w; \
	1646	+ break; \
	1647	+ default: \
	1648	+ p = &(vcpu)->arch.fgt[id].r; \
	1649	+ break; \
	1650	+ } \
	1651	+ \
	1652	+ p; \
	1653	+ })
1604	1654
1605	1655	#endif /* __ARM64_KVM_HOST_H__ */

arch/arm64/kvm/arch_timer.c+14−91

@@ -66,7 +66,7 @@ static int nr_timers(struct kvm_vcpu *vcpu) · 65 unmodified lines
66	66
67	67	u32 timer_get_ctl(struct arch_timer_context *ctxt)
68	68	{
69		− struct kvm_vcpu *vcpu = ctxt->vcpu;
	69	+ struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctxt);
70	70
71	71	switch(arch_timer_ctx_index(ctxt)) {
72	72	case TIMER_VTIMER:
@@ -85,7 +85,7 @@ u32 timer_get_ctl(struct arch_timer_context *ctxt) · 12 unmodified lines
85	85
86	86	u64 timer_get_cval(struct arch_timer_context *ctxt)
87	87	{
88		− struct kvm_vcpu *vcpu = ctxt->vcpu;
	88	+ struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctxt);
89	89
90	90	switch(arch_timer_ctx_index(ctxt)) {
91	91	case TIMER_VTIMER:
@@ -104,7 +104,7 @@ u64 timer_get_cval(struct arch_timer_context *ctxt) · 12 unmodified lines
104	104
105	105	static void timer_set_ctl(struct arch_timer_context *ctxt, u32 ctl)
106	106	{
107		− struct kvm_vcpu *vcpu = ctxt->vcpu;
	107	+ struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctxt);
108	108
109	109	switch(arch_timer_ctx_index(ctxt)) {
110	110	case TIMER_VTIMER:
@@ -126,7 +126,7 @@ static void timer_set_ctl(struct arch_timer_context *ctxt, u32 ctl) · 15 unmodified lines
126	126
127	127	static void timer_set_cval(struct arch_timer_context *ctxt, u64 cval)
128	128	{
129		− struct kvm_vcpu *vcpu = ctxt->vcpu;
	129	+ struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctxt);
130	130
131	131	switch(arch_timer_ctx_index(ctxt)) {
132	132	case TIMER_VTIMER:
@@ -146,16 +146,6 @@ static void timer_set_cval(struct arch_timer_context *ctxt, u64 cval) · 13 unmodified lines
146	146	}
147	147	}
148	148
149		−static void timer_set_offset(struct arch_timer_context *ctxt, u64 offset)
150		−{
151		− if (!ctxt->offset.vm_offset) {
152		− WARN(offset, "timer %ld\n", arch_timer_ctx_index(ctxt));
153		− return;
154		− }
155		−
156		− WRITE_ONCE(*ctxt->offset.vm_offset, offset);
157		−}
158		−
159	149	u64 kvm_phys_timer_read(void)
160	150	{
161	151	return timecounter->cc->read(timecounter->cc);
@@ -343,7 +333,7 @@ static enum hrtimer_restart kvm_hrtimer_expire(struct hrtimer *hrt) · 181 unmodified lines
343	333	u64 ns;
344	334
345	335	ctx = container_of(hrt, struct arch_timer_context, hrtimer);
346		− vcpu = ctx->vcpu;
	336	+ vcpu = timer_context_to_vcpu(ctx);
347	337
348	338	trace_kvm_timer_hrtimer_expire(ctx);
349	339
@@ -436,8 +426,9 @@ static void kvm_timer_update_status(struct arch_timer_context *ctx, bool level) · 86 unmodified lines
436	426	*
437	427	* But hey, it's fast, right?
438	428	*/
439		− if (is_hyp_ctxt(ctx->vcpu) &&
440		− (ctx == vcpu_vtimer(ctx->vcpu) \|\| ctx == vcpu_ptimer(ctx->vcpu))) {
	429	+ struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctx);
	430	+ if (is_hyp_ctxt(vcpu) &&
	431	+ (ctx == vcpu_vtimer(vcpu) \|\| ctx == vcpu_ptimer(vcpu))) {
441	432	unsigned long val = timer_get_ctl(ctx);
442	433	__assign_bit(__ffs(ARCH_TIMER_CTRL_IT_STAT), &val, level);
443	434	timer_set_ctl(ctx, val);
@@ -470,7 +461,7 @@ static void timer_emulate(struct arch_timer_context *ctx) · 26 unmodified lines
470	461	trace_kvm_timer_emulate(ctx, should_fire);
471	462
472	463	if (should_fire != ctx->irq.level)
473		− kvm_timer_update_irq(ctx->vcpu, should_fire, ctx);
	464	+ kvm_timer_update_irq(timer_context_to_vcpu(ctx), should_fire, ctx);
474	465
475	466	kvm_timer_update_status(ctx, should_fire);
476	467
@@ -498,7 +489,7 @@ static void set_cntpoff(u64 cntpoff) · 21 unmodified lines
498	489
499	490	static void timer_save_state(struct arch_timer_context *ctx)
500	491	{
501		− struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu);
	492	+ struct arch_timer_cpu *timer = vcpu_timer(timer_context_to_vcpu(ctx));
502	493	enum kvm_arch_timers index = arch_timer_ctx_index(ctx);
503	494	unsigned long flags;
504	495
@@ -609,7 +600,7 @@ static void kvm_timer_unblocking(struct kvm_vcpu *vcpu) · 104 unmodified lines
609	600
610	601	static void timer_restore_state(struct arch_timer_context *ctx)
611	602	{
612		− struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu);
	603	+ struct arch_timer_cpu *timer = vcpu_timer(timer_context_to_vcpu(ctx));
613	604	enum kvm_arch_timers index = arch_timer_ctx_index(ctx);
614	605	unsigned long flags;
615	606
@@ -668,7 +659,7 @@ static inline void set_timer_irq_phys_active(struct arch_timer_context *ctx, boo · 52 unmodified lines
668	659
669	660	static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx)
670	661	{
671		− struct kvm_vcpu *vcpu = ctx->vcpu;
	662	+ struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctx);
672	663	bool phys_active = false;
673	664
674	665	/*
@@ -677,7 +668,7 @@ static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx) · 2 unmodified lines
677	668	* this point and the register restoration, we'll take the
678	669	* interrupt anyway.
679	670	*/
680		− kvm_timer_update_irq(ctx->vcpu, kvm_timer_should_fire(ctx), ctx);
	671	+ kvm_timer_update_irq(vcpu, kvm_timer_should_fire(ctx), ctx);
681	672
682	673	if (irqchip_in_kernel(vcpu->kvm))
683	674	phys_active = kvm_vgic_map_is_active(vcpu, timer_irq(ctx));
@@ -1063,7 +1054,7 @@ static void timer_context_init(struct kvm_vcpu *vcpu, int timerid) · 379 unmodified lines
1063	1054	struct arch_timer_context *ctxt = vcpu_get_timer(vcpu, timerid);
1064	1055	struct kvm *kvm = vcpu->kvm;
1065	1056
1066		− ctxt->vcpu = vcpu;
	1057	+ ctxt->timer_id = timerid;
1067	1058
1068	1059	if (timerid == TIMER_VTIMER)
1069	1060	ctxt->offset.vm_offset = &kvm->arch.timer_data.voffset;
@@ -1121,49 +1112,6 @@ void kvm_timer_cpu_down(void) · 51 unmodified lines
1121	1112	disable_percpu_irq(host_ptimer_irq);
1122	1113	}
1123	1114
1124		−int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
1125		−{
1126		− struct arch_timer_context *timer;
1127		−
1128		− switch (regid) {
1129		− case KVM_REG_ARM_TIMER_CTL:
1130		− timer = vcpu_vtimer(vcpu);
1131		− kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value);
1132		− break;
1133		− case KVM_REG_ARM_TIMER_CNT:
1134		− if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET,
1135		− &vcpu->kvm->arch.flags)) {
1136		− timer = vcpu_vtimer(vcpu);
1137		− timer_set_offset(timer, kvm_phys_timer_read() - value);
1138		− }
1139		− break;
1140		− case KVM_REG_ARM_TIMER_CVAL:
1141		− timer = vcpu_vtimer(vcpu);
1142		− kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value);
1143		− break;
1144		− case KVM_REG_ARM_PTIMER_CTL:
1145		− timer = vcpu_ptimer(vcpu);
1146		− kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value);
1147		− break;
1148		− case KVM_REG_ARM_PTIMER_CNT:
1149		− if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET,
1150		− &vcpu->kvm->arch.flags)) {
1151		− timer = vcpu_ptimer(vcpu);
1152		− timer_set_offset(timer, kvm_phys_timer_read() - value);
1153		− }
1154		− break;
1155		− case KVM_REG_ARM_PTIMER_CVAL:
1156		− timer = vcpu_ptimer(vcpu);
1157		− kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value);
1158		− break;
1159		−
1160		− default:
1161		− return -1;
1162		− }
1163		−
1164		− return 0;
1165		−}
1166		−
1167	1115	static u64 read_timer_ctl(struct arch_timer_context *timer)
1168	1116	{
1169	1117	/*
@@ -1180,31 +1128,6 @@ static u64 read_timer_ctl(struct arch_timer_context *timer) · 10 unmodified lines
1180	1128	return ctl;
1181	1129	}
1182	1130
1183		−u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
1184		−{
1185		− switch (regid) {
1186		− case KVM_REG_ARM_TIMER_CTL:
1187		− return kvm_arm_timer_read(vcpu,
1188		− vcpu_vtimer(vcpu), TIMER_REG_CTL);
1189		− case KVM_REG_ARM_TIMER_CNT:
1190		− return kvm_arm_timer_read(vcpu,
1191		− vcpu_vtimer(vcpu), TIMER_REG_CNT);
1192		− case KVM_REG_ARM_TIMER_CVAL:
1193		− return kvm_arm_timer_read(vcpu,
1194		− vcpu_vtimer(vcpu), TIMER_REG_CVAL);
1195		− case KVM_REG_ARM_PTIMER_CTL:
1196		− return kvm_arm_timer_read(vcpu,
1197		− vcpu_ptimer(vcpu), TIMER_REG_CTL);
1198		− case KVM_REG_ARM_PTIMER_CNT:
1199		− return kvm_arm_timer_read(vcpu,
1200		− vcpu_ptimer(vcpu), TIMER_REG_CNT);
1201		− case KVM_REG_ARM_PTIMER_CVAL:
1202		− return kvm_arm_timer_read(vcpu,
1203		− vcpu_ptimer(vcpu), TIMER_REG_CVAL);
1204		− }
1205		− return (u64)-1;
1206		−}
1207		−
1208	1131	static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu,
1209	1132	struct arch_timer_context *timer,
1210	1133	enum kvm_arch_timer_regs treg)

arch/arm64/kvm/arm.c+7−0

@@ -642,6 +642,7 @@ nommu: · 641 unmodified lines
642	642	vcpu->arch.hcr_el2 \|= HCR_TWI;
643	643
644	644	vcpu_set_pauth_traps(vcpu);
	645	+ kvm_vcpu_load_fgt(vcpu);
645	646
646	647	if (is_protected_kvm_enabled()) {
647	648	kvm_call_hyp_nvhe(__pkvm_vcpu_load,
@@ -1794,6 +1795,9 @@ long kvm_arch_vcpu_ioctl(struct file *filp, · 1146 unmodified lines
1794	1795	case KVM_GET_VCPU_EVENTS: {
1795	1796	struct kvm_vcpu_events events;
1796	1797
	1798	+ if (!kvm_vcpu_initialized(vcpu))
	1799	+ return -ENOEXEC;
	1800	+
1797	1801	if (kvm_arm_vcpu_get_events(vcpu, &events))
1798	1802	return -EINVAL;
1799	1803
@@ -1805,6 +1809,9 @@ long kvm_arch_vcpu_ioctl(struct file *filp, · 5 unmodified lines
1805	1809	case KVM_SET_VCPU_EVENTS: {
1806	1810	struct kvm_vcpu_events events;
1807	1811
	1812	+ if (!kvm_vcpu_initialized(vcpu))
	1813	+ return -ENOEXEC;
	1814	+
1808	1815	if (copy_from_user(&events, argp, sizeof(events)))
1809	1816	return -EFAULT;
1810	1817

arch/arm64/kvm/at.c+5−2

@@ -91,7 +91,6 @@ static enum trans_regime compute_translation_regime(struct kvm_vcpu *vcpu, u32 o · 90 unmodified lines
91	91	case OP_AT_S1E2W:
92	92	case OP_AT_S1E2A:
93	93	return vcpu_el2_e2h_is_set(vcpu) ? TR_EL20 : TR_EL2;
94		− break;
95	94	default:
96	95	return (vcpu_el2_e2h_is_set(vcpu) &&
97	96	vcpu_el2_tge_is_set(vcpu)) ? TR_EL20 : TR_EL10;
@@ -1602,13 +1601,17 @@ int __kvm_find_s1_desc_level(struct kvm_vcpu vcpu, u64 va, u64 ipa, int level) · 1504 unmodified lines
1602	1601	.fn = match_s1_desc,
1603	1602	.priv = &dm,
1604	1603	},
1605		− .regime = TR_EL10,
1606	1604	.as_el0 = false,
1607	1605	.pan = false,
1608	1606	};
1609	1607	struct s1_walk_result wr = {};
1610	1608	int ret;
1611	1609
	1610	+ if (is_hyp_ctxt(vcpu))
	1611	+ wi.regime = vcpu_el2_e2h_is_set(vcpu) ? TR_EL20 : TR_EL2;
	1612	+ else
	1613	+ wi.regime = TR_EL10;
	1614	+
1612	1615	ret = setup_s1_walk(vcpu, &wi, &wr, va);
1613	1616	if (ret)
1614	1617	return ret;

arch/arm64/kvm/config.c+90−0

@@ -5,6 +5,8 @@ · 4 unmodified lines
5	5	*/
6	6
7	7	#include <linux/kvm_host.h>
	8	+#include <asm/kvm_emulate.h>
	9	+#include <asm/kvm_nested.h>
8	10	#include <asm/sysreg.h>
9	11
10	12	/*
@@ -1428,3 +1430,91 @@ void get_reg_fixed_bits(struct kvm kvm, enum vcpu_sysreg reg, u64 res0, u64 *r · 1417 unmodified lines
1428	1430	break;
1429	1431	}
1430	1432	}
	1433	+
	1434	+static __always_inline struct fgt_masks *__fgt_reg_to_masks(enum vcpu_sysreg reg)
	1435	+{
	1436	+ switch (reg) {
	1437	+ case HFGRTR_EL2:
	1438	+ return &hfgrtr_masks;
	1439	+ case HFGWTR_EL2:
	1440	+ return &hfgwtr_masks;
	1441	+ case HFGITR_EL2:
	1442	+ return &hfgitr_masks;
	1443	+ case HDFGRTR_EL2:
	1444	+ return &hdfgrtr_masks;
	1445	+ case HDFGWTR_EL2:
	1446	+ return &hdfgwtr_masks;
	1447	+ case HAFGRTR_EL2:
	1448	+ return &hafgrtr_masks;
	1449	+ case HFGRTR2_EL2:
	1450	+ return &hfgrtr2_masks;
	1451	+ case HFGWTR2_EL2:
	1452	+ return &hfgwtr2_masks;
	1453	+ case HFGITR2_EL2:
	1454	+ return &hfgitr2_masks;
	1455	+ case HDFGRTR2_EL2:
	1456	+ return &hdfgrtr2_masks;
	1457	+ case HDFGWTR2_EL2:
	1458	+ return &hdfgwtr2_masks;
	1459	+ default:
	1460	+ BUILD_BUG_ON(1);
	1461	+ }
	1462	+}
	1463	+
	1464	+static __always_inline void __compute_fgt(struct kvm_vcpu *vcpu, enum vcpu_sysreg reg)
	1465	+{
	1466	+ u64 fgu = vcpu->kvm->arch.fgu[__fgt_reg_to_group_id(reg)];
	1467	+ struct fgt_masks *m = __fgt_reg_to_masks(reg);
	1468	+ u64 clear = 0, set = 0, val = m->nmask;
	1469	+
	1470	+ set \|= fgu & m->mask;
	1471	+ clear \|= fgu & m->nmask;
	1472	+
	1473	+ if (is_nested_ctxt(vcpu)) {
	1474	+ u64 nested = __vcpu_sys_reg(vcpu, reg);
	1475	+ set \|= nested & m->mask;
	1476	+ clear \|= ~nested & m->nmask;
	1477	+ }
	1478	+
	1479	+ val \|= set;
	1480	+ val &= ~clear;
	1481	+ *vcpu_fgt(vcpu, reg) = val;
	1482	+}
	1483	+
	1484	+static void __compute_hfgwtr(struct kvm_vcpu *vcpu)
	1485	+{
	1486	+ __compute_fgt(vcpu, HFGWTR_EL2);
	1487	+
	1488	+ if (cpus_have_final_cap(ARM64_WORKAROUND_AMPERE_AC03_CPU_38))
	1489	+ *vcpu_fgt(vcpu, HFGWTR_EL2) \|= HFGWTR_EL2_TCR_EL1;
	1490	+}
	1491	+
	1492	+static void __compute_hdfgwtr(struct kvm_vcpu *vcpu)
	1493	+{
	1494	+ __compute_fgt(vcpu, HDFGWTR_EL2);
	1495	+
	1496	+ if (is_hyp_ctxt(vcpu))
	1497	+ *vcpu_fgt(vcpu, HDFGWTR_EL2) \|= HDFGWTR_EL2_MDSCR_EL1;
	1498	+}
	1499	+
	1500	+void kvm_vcpu_load_fgt(struct kvm_vcpu *vcpu)
	1501	+{
	1502	+ if (!cpus_have_final_cap(ARM64_HAS_FGT))
	1503	+ return;
	1504	+
	1505	+ __compute_fgt(vcpu, HFGRTR_EL2);
	1506	+ __compute_hfgwtr(vcpu);
	1507	+ __compute_fgt(vcpu, HFGITR_EL2);
	1508	+ __compute_fgt(vcpu, HDFGRTR_EL2);
	1509	+ __compute_hdfgwtr(vcpu);
	1510	+ __compute_fgt(vcpu, HAFGRTR_EL2);
	1511	+
	1512	+ if (!cpus_have_final_cap(ARM64_HAS_FGT2))
	1513	+ return;
	1514	+
	1515	+ __compute_fgt(vcpu, HFGRTR2_EL2);
	1516	+ __compute_fgt(vcpu, HFGWTR2_EL2);
	1517	+ __compute_fgt(vcpu, HFGITR2_EL2);
	1518	+ __compute_fgt(vcpu, HDFGRTR2_EL2);
	1519	+ __compute_fgt(vcpu, HDFGWTR2_EL2);
	1520	+}

arch/arm64/kvm/debug.c+10−5

@@ -15,6 +15,12 @@ · 14 unmodified lines
15	15	#include <asm/kvm_arm.h>
16	16	#include <asm/kvm_emulate.h>
17	17
	18	+static int cpu_has_spe(u64 dfr0)
	19	+{
	20	+ return cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_PMSVer_SHIFT) &&
	21	+ !(read_sysreg_s(SYS_PMBIDR_EL1) & PMBIDR_EL1_P);
	22	+}
	23	+
18	24	/**
19	25	* kvm_arm_setup_mdcr_el2 - configure vcpu mdcr_el2 value
20	26	*
@@ -77,13 +83,12 @@ void kvm_init_host_debug_data(void) · 56 unmodified lines
77	83	*host_data_ptr(debug_brps) = SYS_FIELD_GET(ID_AA64DFR0_EL1, BRPs, dfr0);
78	84	*host_data_ptr(debug_wrps) = SYS_FIELD_GET(ID_AA64DFR0_EL1, WRPs, dfr0);
79	85
	86	+ if (cpu_has_spe(dfr0))
	87	+ host_data_set_flag(HAS_SPE);
	88	+
80	89	if (has_vhe())
81	90	return;
82	91
83		− if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_PMSVer_SHIFT) &&
84		− !(read_sysreg_s(SYS_PMBIDR_EL1) & PMBIDR_EL1_P))
85		− host_data_set_flag(HAS_SPE);
86		−
87	92	/* Check if we have BRBE implemented and available at the host */
88	93	if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_BRBE_SHIFT))
89	94	host_data_set_flag(HAS_BRBE);
@@ -102,7 +107,7 @@ void kvm_init_host_debug_data(void) · 12 unmodified lines
102	107	void kvm_debug_init_vhe(void)
103	108	{
104	109	/* Clear PMSCR_EL1.E{0,1}SPE which reset to UNKNOWN values. */
105		− if (SYS_FIELD_GET(ID_AA64DFR0_EL1, PMSVer, read_sysreg(id_aa64dfr0_el1)))
	110	+ if (host_data_test_flag(HAS_SPE))
106	111	write_sysreg_el1(0, SYS_PMSCR);
107	112	}
108	113

arch/arm64/kvm/guest.c+0−70

@@ -591,64 +591,6 @@ static unsigned long num_core_regs(const struct kvm_vcpu *vcpu) · 590 unmodified lines
591	591	return copy_core_reg_indices(vcpu, NULL);
592	592	}
593	593
594		−static const u64 timer_reg_list[] = {
595		− KVM_REG_ARM_TIMER_CTL,
596		− KVM_REG_ARM_TIMER_CNT,
597		− KVM_REG_ARM_TIMER_CVAL,
598		− KVM_REG_ARM_PTIMER_CTL,
599		− KVM_REG_ARM_PTIMER_CNT,
600		− KVM_REG_ARM_PTIMER_CVAL,
601		−};
602		−
603		−#define NUM_TIMER_REGS ARRAY_SIZE(timer_reg_list)
604		−
605		−static bool is_timer_reg(u64 index)
606		−{
607		− switch (index) {
608		− case KVM_REG_ARM_TIMER_CTL:
609		− case KVM_REG_ARM_TIMER_CNT:
610		− case KVM_REG_ARM_TIMER_CVAL:
611		− case KVM_REG_ARM_PTIMER_CTL:
612		− case KVM_REG_ARM_PTIMER_CNT:
613		− case KVM_REG_ARM_PTIMER_CVAL:
614		− return true;
615		− }
616		− return false;
617		−}
618		−
619		−static int copy_timer_indices(struct kvm_vcpu vcpu, u64 __user uindices)
620		−{
621		− for (int i = 0; i < NUM_TIMER_REGS; i++) {
622		− if (put_user(timer_reg_list[i], uindices))
623		− return -EFAULT;
624		− uindices++;
625		− }
626		−
627		− return 0;
628		−}
629		−
630		−static int set_timer_reg(struct kvm_vcpu vcpu, const struct kvm_one_reg reg)
631		−{
632		− void __user uaddr = (void __user )(long)reg->addr;
633		− u64 val;
634		− int ret;
635		−
636		− ret = copy_from_user(&val, uaddr, KVM_REG_SIZE(reg->id));
637		− if (ret != 0)
638		− return -EFAULT;
639		−
640		− return kvm_arm_timer_set_reg(vcpu, reg->id, val);
641		−}
642		−
643		−static int get_timer_reg(struct kvm_vcpu vcpu, const struct kvm_one_reg reg)
644		−{
645		− void __user uaddr = (void __user )(long)reg->addr;
646		− u64 val;
647		−
648		− val = kvm_arm_timer_get_reg(vcpu, reg->id);
649		− return copy_to_user(uaddr, &val, KVM_REG_SIZE(reg->id)) ? -EFAULT : 0;
650		−}
651		−
652	594	static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu)
653	595	{
654	596	const unsigned int slices = vcpu_sve_slices(vcpu);
@@ -724,7 +666,6 @@ unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu) · 69 unmodified lines
724	666	res += num_sve_regs(vcpu);
725	667	res += kvm_arm_num_sys_reg_descs(vcpu);
726	668	res += kvm_arm_get_fw_num_regs(vcpu);
727		− res += NUM_TIMER_REGS;
728	669
729	670	return res;
730	671	}
@@ -755,11 +696,6 @@ int kvm_arm_copy_reg_indices(struct kvm_vcpu vcpu, u64 __user uindices) · 24 unmodified lines
755	696	return ret;
756	697	uindices += kvm_arm_get_fw_num_regs(vcpu);
757	698
758		− ret = copy_timer_indices(vcpu, uindices);
759		− if (ret < 0)
760		− return ret;
761		− uindices += NUM_TIMER_REGS;
762		−
763	699	return kvm_arm_copy_sys_reg_indices(vcpu, uindices);
764	700	}
765	701
@@ -777,9 +713,6 @@ int kvm_arm_get_reg(struct kvm_vcpu vcpu, const struct kvm_one_reg reg) · 11 unmodified lines
777	713	case KVM_REG_ARM64_SVE: return get_sve_reg(vcpu, reg);
778	714	}
779	715
780		− if (is_timer_reg(reg->id))
781		− return get_timer_reg(vcpu, reg);
782		−
783	716	return kvm_arm_sys_reg_get_reg(vcpu, reg);
784	717	}
785	718
@@ -797,9 +730,6 @@ int kvm_arm_set_reg(struct kvm_vcpu vcpu, const struct kvm_one_reg reg) · 11 unmodified lines
797	730	case KVM_REG_ARM64_SVE: return set_sve_reg(vcpu, reg);
798	731	}
799	732
800		− if (is_timer_reg(reg->id))
801		− return set_timer_reg(vcpu, reg);
802		−
803	733	return kvm_arm_sys_reg_set_reg(vcpu, reg);
804	734	}
805	735

arch/arm64/kvm/handle_exit.c+6−1

@@ -147,7 +147,12 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu) · 146 unmodified lines
147	147	if (esr & ESR_ELx_WFx_ISS_RV) {
148	148	u64 val, now;
149	149
150		− now = kvm_arm_timer_get_reg(vcpu, KVM_REG_ARM_TIMER_CNT);
	150	+ now = kvm_phys_timer_read();
	151	+ if (is_hyp_ctxt(vcpu) && vcpu_el2_e2h_is_set(vcpu))
	152	+ now -= timer_get_offset(vcpu_hvtimer(vcpu));
	153	+ else
	154	+ now -= timer_get_offset(vcpu_vtimer(vcpu));
	155	+
151	156	val = vcpu_get_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu));
152	157
153	158	if (now >= val)

arch/arm64/kvm/hyp/include/hyp/switch.h+17−131

@@ -195,123 +195,6 @@ static inline void __deactivate_cptr_traps(struct kvm_vcpu *vcpu) · 194 unmodified lines
195	195	__deactivate_cptr_traps_nvhe(vcpu);
196	196	}
197	197
198		−#define reg_to_fgt_masks(reg) \
199		− ({ \
200		− struct fgt_masks *m; \
201		− switch(reg) { \
202		− case HFGRTR_EL2: \
203		− m = &hfgrtr_masks; \
204		− break; \
205		− case HFGWTR_EL2: \
206		− m = &hfgwtr_masks; \
207		− break; \
208		− case HFGITR_EL2: \
209		− m = &hfgitr_masks; \
210		− break; \
211		− case HDFGRTR_EL2: \
212		− m = &hdfgrtr_masks; \
213		− break; \
214		− case HDFGWTR_EL2: \
215		− m = &hdfgwtr_masks; \
216		− break; \
217		− case HAFGRTR_EL2: \
218		− m = &hafgrtr_masks; \
219		− break; \
220		− case HFGRTR2_EL2: \
221		− m = &hfgrtr2_masks; \
222		− break; \
223		− case HFGWTR2_EL2: \
224		− m = &hfgwtr2_masks; \
225		− break; \
226		− case HFGITR2_EL2: \
227		− m = &hfgitr2_masks; \
228		− break; \
229		− case HDFGRTR2_EL2: \
230		− m = &hdfgrtr2_masks; \
231		− break; \
232		− case HDFGWTR2_EL2: \
233		− m = &hdfgwtr2_masks; \
234		− break; \
235		− default: \
236		− BUILD_BUG_ON(1); \
237		− } \
238		− \
239		− m; \
240		− })
241		−
242		−#define compute_clr_set(vcpu, reg, clr, set) \
243		− do { \
244		− u64 hfg = __vcpu_sys_reg(vcpu, reg); \
245		− struct fgt_masks *m = reg_to_fgt_masks(reg); \
246		− set \|= hfg & m->mask; \
247		− clr \|= ~hfg & m->nmask; \
248		− } while(0)
249		−
250		−#define reg_to_fgt_group_id(reg) \
251		− ({ \
252		− enum fgt_group_id id; \
253		− switch(reg) { \
254		− case HFGRTR_EL2: \
255		− case HFGWTR_EL2: \
256		− id = HFGRTR_GROUP; \
257		− break; \
258		− case HFGITR_EL2: \
259		− id = HFGITR_GROUP; \
260		− break; \
261		− case HDFGRTR_EL2: \
262		− case HDFGWTR_EL2: \
263		− id = HDFGRTR_GROUP; \
264		− break; \
265		− case HAFGRTR_EL2: \
266		− id = HAFGRTR_GROUP; \
267		− break; \
268		− case HFGRTR2_EL2: \
269		− case HFGWTR2_EL2: \
270		− id = HFGRTR2_GROUP; \
271		− break; \
272		− case HFGITR2_EL2: \
273		− id = HFGITR2_GROUP; \
274		− break; \
275		− case HDFGRTR2_EL2: \
276		− case HDFGWTR2_EL2: \
277		− id = HDFGRTR2_GROUP; \
278		− break; \
279		− default: \
280		− BUILD_BUG_ON(1); \
281		− } \
282		− \
283		− id; \
284		− })
285		−
286		−#define compute_undef_clr_set(vcpu, kvm, reg, clr, set) \
287		− do { \
288		− u64 hfg = kvm->arch.fgu[reg_to_fgt_group_id(reg)]; \
289		− struct fgt_masks *m = reg_to_fgt_masks(reg); \
290		− set \|= hfg & m->mask; \
291		− clr \|= hfg & m->nmask; \
292		− } while(0)
293		−
294		−#define update_fgt_traps_cs(hctxt, vcpu, kvm, reg, clr, set) \
295		− do { \
296		− struct fgt_masks *m = reg_to_fgt_masks(reg); \
297		− u64 c = clr, s = set; \
298		− u64 val; \
299		− \
300		− ctxt_sys_reg(hctxt, reg) = read_sysreg_s(SYS_ ## reg); \
301		− if (is_nested_ctxt(vcpu)) \
302		− compute_clr_set(vcpu, reg, c, s); \
303		− \
304		− compute_undef_clr_set(vcpu, kvm, reg, c, s); \
305		− \
306		− val = m->nmask; \
307		− val \|= s; \
308		− val &= ~c; \
309		− write_sysreg_s(val, SYS_ ## reg); \
310		− } while(0)
311		−
312		−#define update_fgt_traps(hctxt, vcpu, kvm, reg) \
313		− update_fgt_traps_cs(hctxt, vcpu, kvm, reg, 0, 0)
314		−
315	198	static inline bool cpu_has_amu(void)
316	199	{
317	200	u64 pfr0 = read_sysreg_s(SYS_ID_AA64PFR0_EL1);
@@ -320,33 +203,36 @@ static inline bool cpu_has_amu(void) · 2 unmodified lines
320	203	ID_AA64PFR0_EL1_AMU_SHIFT);
321	204	}
322	205
	206	+#define __activate_fgt(hctxt, vcpu, reg) \
	207	+ do { \
	208	+ ctxt_sys_reg(hctxt, reg) = read_sysreg_s(SYS_ ## reg); \
	209	+ write_sysreg_s(*vcpu_fgt(vcpu, reg), SYS_ ## reg); \
	210	+ } while (0)
	211	+
323	212	static inline void __activate_traps_hfgxtr(struct kvm_vcpu *vcpu)
324	213	{
325	214	struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt);
326		− struct kvm *kvm = kern_hyp_va(vcpu->kvm);
327	215
328	216	if (!cpus_have_final_cap(ARM64_HAS_FGT))
329	217	return;
330	218
331		− update_fgt_traps(hctxt, vcpu, kvm, HFGRTR_EL2);
332		− update_fgt_traps_cs(hctxt, vcpu, kvm, HFGWTR_EL2, 0,
333		− cpus_have_final_cap(ARM64_WORKAROUND_AMPERE_AC03_CPU_38) ?
334		− HFGWTR_EL2_TCR_EL1_MASK : 0);
335		− update_fgt_traps(hctxt, vcpu, kvm, HFGITR_EL2);
336		− update_fgt_traps(hctxt, vcpu, kvm, HDFGRTR_EL2);
337		− update_fgt_traps(hctxt, vcpu, kvm, HDFGWTR_EL2);
	219	+ __activate_fgt(hctxt, vcpu, HFGRTR_EL2);
	220	+ __activate_fgt(hctxt, vcpu, HFGWTR_EL2);
	221	+ __activate_fgt(hctxt, vcpu, HFGITR_EL2);
	222	+ __activate_fgt(hctxt, vcpu, HDFGRTR_EL2);
	223	+ __activate_fgt(hctxt, vcpu, HDFGWTR_EL2);
338	224
339	225	if (cpu_has_amu())
340		− update_fgt_traps(hctxt, vcpu, kvm, HAFGRTR_EL2);
	226	+ __activate_fgt(hctxt, vcpu, HAFGRTR_EL2);
341	227
342	228	if (!cpus_have_final_cap(ARM64_HAS_FGT2))
343	229	return;
344	230
345		− update_fgt_traps(hctxt, vcpu, kvm, HFGRTR2_EL2);
346		− update_fgt_traps(hctxt, vcpu, kvm, HFGWTR2_EL2);
347		− update_fgt_traps(hctxt, vcpu, kvm, HFGITR2_EL2);
348		− update_fgt_traps(hctxt, vcpu, kvm, HDFGRTR2_EL2);
349		− update_fgt_traps(hctxt, vcpu, kvm, HDFGWTR2_EL2);
	231	+ __activate_fgt(hctxt, vcpu, HFGRTR2_EL2);
	232	+ __activate_fgt(hctxt, vcpu, HFGWTR2_EL2);
	233	+ __activate_fgt(hctxt, vcpu, HFGITR2_EL2);
	234	+ __activate_fgt(hctxt, vcpu, HDFGRTR2_EL2);
	235	+ __activate_fgt(hctxt, vcpu, HDFGWTR2_EL2);
350	236	}
351	237
352	238	#define __deactivate_fgt(htcxt, vcpu, reg) \

arch/arm64/kvm/hyp/nvhe/pkvm.c+1−0

@@ -172,6 +172,7 @@ static int pkvm_vcpu_init_traps(struct pkvm_hyp_vcpu *hyp_vcpu) · 171 unmodified lines
172	172
173	173	/* Trust the host for non-protected vcpu features. */
174	174	vcpu->arch.hcrx_el2 = host_vcpu->arch.hcrx_el2;
	175	+ memcpy(vcpu->arch.fgt, host_vcpu->arch.fgt, sizeof(vcpu->arch.fgt));
175	176	return 0;
176	177	}
177	178

arch/arm64/kvm/nested.c+6−3

@@ -1859,13 +1859,16 @@ void kvm_nested_setup_mdcr_el2(struct kvm_vcpu *vcpu) · 1858 unmodified lines
1859	1859	{
1860	1860	u64 guest_mdcr = __vcpu_sys_reg(vcpu, MDCR_EL2);
1861	1861
	1862	+ if (is_nested_ctxt(vcpu))
	1863	+ vcpu->arch.mdcr_el2 \|= (guest_mdcr & NV_MDCR_GUEST_INCLUDE);
1862	1864	/*
1863	1865	* In yet another example where FEAT_NV2 is fscking broken, accesses
1864	1866	* to MDSCR_EL1 are redirected to the VNCR despite having an effect
1865	1867	* at EL2. Use a big hammer to apply sanity.
	1868	+ *
	1869	+ * Unless of course we have FEAT_FGT, in which case we can precisely
	1870	+ * trap MDSCR_EL1.
1866	1871	*/
1867		− if (is_hyp_ctxt(vcpu))
	1872	+ else if (!cpus_have_final_cap(ARM64_HAS_FGT))
1868	1873	vcpu->arch.mdcr_el2 \|= MDCR_EL2_TDA;
1869		− else
1870		− vcpu->arch.mdcr_el2 \|= (guest_mdcr & NV_MDCR_GUEST_INCLUDE);
1871	1874	}

arch/arm64/kvm/sys_regs.c+105−26

@@ -203,7 +203,6 @@ static void locate_register(const struct kvm_vcpu *vcpu, enum vcpu_sysreg reg, · 202 unmodified lines
203	203	MAPPED_EL2_SYSREG(AMAIR_EL2, AMAIR_EL1, NULL );
204	204	MAPPED_EL2_SYSREG(ELR_EL2, ELR_EL1, NULL );
205	205	MAPPED_EL2_SYSREG(SPSR_EL2, SPSR_EL1, NULL );
206		− MAPPED_EL2_SYSREG(ZCR_EL2, ZCR_EL1, NULL );
207	206	MAPPED_EL2_SYSREG(CONTEXTIDR_EL2, CONTEXTIDR_EL1, NULL );
208	207	MAPPED_EL2_SYSREG(SCTLR2_EL2, SCTLR2_EL1, NULL );
209	208	case CNTHCTL_EL2:
@@ -1595,14 +1594,47 @@ static bool access_arch_timer(struct kvm_vcpu *vcpu, · 1385 unmodified lines
1595	1594	return true;
1596	1595	}
1597	1596
1598		−static bool access_hv_timer(struct kvm_vcpu *vcpu,
1599		− struct sys_reg_params *p,
1600		− const struct sys_reg_desc *r)
	1597	+static int arch_timer_set_user(struct kvm_vcpu *vcpu,
	1598	+ const struct sys_reg_desc *rd,
	1599	+ u64 val)
1601	1600	{
1602		− if (!vcpu_el2_e2h_is_set(vcpu))
1603		− return undef_access(vcpu, p, r);
	1601	+ switch (reg_to_encoding(rd)) {
	1602	+ case SYS_CNTV_CTL_EL0:
	1603	+ case SYS_CNTP_CTL_EL0:
	1604	+ case SYS_CNTHV_CTL_EL2:
	1605	+ case SYS_CNTHP_CTL_EL2:
	1606	+ val &= ~ARCH_TIMER_CTRL_IT_STAT;
	1607	+ break;
	1608	+ case SYS_CNTVCT_EL0:
	1609	+ if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags))
	1610	+ timer_set_offset(vcpu_vtimer(vcpu), kvm_phys_timer_read() - val);
	1611	+ return 0;
	1612	+ case SYS_CNTPCT_EL0:
	1613	+ if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags))
	1614	+ timer_set_offset(vcpu_ptimer(vcpu), kvm_phys_timer_read() - val);
	1615	+ return 0;
	1616	+ }
1604	1617
1605		− return access_arch_timer(vcpu, p, r);
	1618	+ __vcpu_assign_sys_reg(vcpu, rd->reg, val);
	1619	+ return 0;
	1620	+}
	1621	+
	1622	+static int arch_timer_get_user(struct kvm_vcpu *vcpu,
	1623	+ const struct sys_reg_desc *rd,
	1624	+ u64 *val)
	1625	+{
	1626	+ switch (reg_to_encoding(rd)) {
	1627	+ case SYS_CNTVCT_EL0:
	1628	+ *val = kvm_phys_timer_read() - timer_get_offset(vcpu_vtimer(vcpu));
	1629	+ break;
	1630	+ case SYS_CNTPCT_EL0:
	1631	+ *val = kvm_phys_timer_read() - timer_get_offset(vcpu_ptimer(vcpu));
	1632	+ break;
	1633	+ default:
	1634	+ *val = __vcpu_sys_reg(vcpu, rd->reg);
	1635	+ }
	1636	+
	1637	+ return 0;
1606	1638	}
1607	1639
1608	1640	static s64 kvm_arm64_ftr_safe_value(u32 id, const struct arm64_ftr_bits *ftrp,
@@ -2507,15 +2539,20 @@ static bool bad_redir_trap(struct kvm_vcpu *vcpu, · 898 unmodified lines
2507	2539	"trap of EL2 register redirected to EL1");
2508	2540	}
2509	2541
2510		−#define EL2_REG_FILTERED(name, acc, rst, v, filter) { \
	2542	+#define SYS_REG_USER_FILTER(name, acc, rst, v, gu, su, filter) { \
2511	2543	SYS_DESC(SYS_##name), \
2512	2544	.access = acc, \
2513	2545	.reset = rst, \
2514	2546	.reg = name, \
	2547	+ .get_user = gu, \
	2548	+ .set_user = su, \
2515	2549	.visibility = filter, \
2516	2550	.val = v, \
2517	2551	}
2518	2552
	2553	+#define EL2_REG_FILTERED(name, acc, rst, v, filter) \
	2554	+ SYS_REG_USER_FILTER(name, acc, rst, v, NULL, NULL, filter)
	2555	+
2519	2556	#define EL2_REG(name, acc, rst, v) \
2520	2557	EL2_REG_FILTERED(name, acc, rst, v, el2_visibility)
2521	2558
@@ -2526,6 +2563,10 @@ static bool bad_redir_trap(struct kvm_vcpu *vcpu, · 4 unmodified lines
2526	2563	EL2_REG_VNCR_FILT(name, hidden_visibility)
2527	2564	#define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v)
2528	2565
	2566	+#define TIMER_REG(name, vis) \
	2567	+ SYS_REG_USER_FILTER(name, access_arch_timer, reset_val, 0, \
	2568	+ arch_timer_get_user, arch_timer_set_user, vis)
	2569	+
2529	2570	/*
2530	2571	* Since reset() callback and field val are not used for idregs, they will be
2531	2572	* used for specific purposes for idregs.
@@ -2705,18 +2746,17 @@ static bool access_zcr_el2(struct kvm_vcpu *vcpu, · 173 unmodified lines
2705	2746
2706	2747	if (guest_hyp_sve_traps_enabled(vcpu)) {
2707	2748	kvm_inject_nested_sve_trap(vcpu);
2708		− return true;
	2749	+ return false;
2709	2750	}
2710	2751
2711	2752	if (!p->is_write) {
2712		− p->regval = vcpu_read_sys_reg(vcpu, ZCR_EL2);
	2753	+ p->regval = __vcpu_sys_reg(vcpu, ZCR_EL2);
2713	2754	return true;
2714	2755	}
2715	2756
2716	2757	vq = SYS_FIELD_GET(ZCR_ELx, LEN, p->regval) + 1;
2717	2758	vq = min(vq, vcpu_sve_max_vq(vcpu));
2718		− vcpu_write_sys_reg(vcpu, vq - 1, ZCR_EL2);
2719		−
	2759	+ __vcpu_assign_sys_reg(vcpu, ZCR_EL2, vq - 1);
2720	2760	return true;
2721	2761	}
2722	2762
@@ -2833,6 +2873,16 @@ static unsigned int s1pie_el2_visibility(const struct kvm_vcpu *vcpu, · 110 unmodified lines
2833	2873	return __el2_visibility(vcpu, rd, s1pie_visibility);
2834	2874	}
2835	2875
	2876	+static unsigned int cnthv_visibility(const struct kvm_vcpu *vcpu,
	2877	+ const struct sys_reg_desc *rd)
	2878	+{
	2879	+ if (vcpu_has_nv(vcpu) &&
	2880	+ !vcpu_has_feature(vcpu, KVM_ARM_VCPU_HAS_EL2_E2H0))
	2881	+ return 0;
	2882	+
	2883	+ return REG_HIDDEN;
	2884	+}
	2885	+
2836	2886	static bool access_mdcr(struct kvm_vcpu *vcpu,
2837	2887	struct sys_reg_params *p,
2838	2888	const struct sys_reg_desc *r)
@@ -3482,17 +3532,19 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 643 unmodified lines
3482	3532	AMU_AMEVTYPER1_EL0(14),
3483	3533	AMU_AMEVTYPER1_EL0(15),
3484	3534
3485		− { SYS_DESC(SYS_CNTPCT_EL0), access_arch_timer },
3486		− { SYS_DESC(SYS_CNTVCT_EL0), access_arch_timer },
	3535	+ { SYS_DESC(SYS_CNTPCT_EL0), .access = access_arch_timer,
	3536	+ .get_user = arch_timer_get_user, .set_user = arch_timer_set_user },
	3537	+ { SYS_DESC(SYS_CNTVCT_EL0), .access = access_arch_timer,
	3538	+ .get_user = arch_timer_get_user, .set_user = arch_timer_set_user },
3487	3539	{ SYS_DESC(SYS_CNTPCTSS_EL0), access_arch_timer },
3488	3540	{ SYS_DESC(SYS_CNTVCTSS_EL0), access_arch_timer },
3489	3541	{ SYS_DESC(SYS_CNTP_TVAL_EL0), access_arch_timer },
3490		− { SYS_DESC(SYS_CNTP_CTL_EL0), access_arch_timer },
3491		− { SYS_DESC(SYS_CNTP_CVAL_EL0), access_arch_timer },
	3542	+ TIMER_REG(CNTP_CTL_EL0, NULL),
	3543	+ TIMER_REG(CNTP_CVAL_EL0, NULL),
3492	3544
3493	3545	{ SYS_DESC(SYS_CNTV_TVAL_EL0), access_arch_timer },
3494		− { SYS_DESC(SYS_CNTV_CTL_EL0), access_arch_timer },
3495		− { SYS_DESC(SYS_CNTV_CVAL_EL0), access_arch_timer },
	3546	+ TIMER_REG(CNTV_CTL_EL0, NULL),
	3547	+ TIMER_REG(CNTV_CVAL_EL0, NULL),
3496	3548
3497	3549	/* PMEVCNTRn_EL0 */
3498	3550	PMU_PMEVCNTR_EL0(0),
@@ -3690,12 +3742,12 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 191 unmodified lines
3690	3742	EL2_REG_VNCR(CNTVOFF_EL2, reset_val, 0),
3691	3743	EL2_REG(CNTHCTL_EL2, access_rw, reset_val, 0),
3692	3744	{ SYS_DESC(SYS_CNTHP_TVAL_EL2), access_arch_timer },
3693		− EL2_REG(CNTHP_CTL_EL2, access_arch_timer, reset_val, 0),
3694		− EL2_REG(CNTHP_CVAL_EL2, access_arch_timer, reset_val, 0),
	3745	+ TIMER_REG(CNTHP_CTL_EL2, el2_visibility),
	3746	+ TIMER_REG(CNTHP_CVAL_EL2, el2_visibility),
3695	3747
3696		− { SYS_DESC(SYS_CNTHV_TVAL_EL2), access_hv_timer },
3697		− EL2_REG(CNTHV_CTL_EL2, access_hv_timer, reset_val, 0),
3698		− EL2_REG(CNTHV_CVAL_EL2, access_hv_timer, reset_val, 0),
	3748	+ { SYS_DESC(SYS_CNTHV_TVAL_EL2), access_arch_timer, .visibility = cnthv_visibility },
	3749	+ TIMER_REG(CNTHV_CTL_EL2, cnthv_visibility),
	3750	+ TIMER_REG(CNTHV_CVAL_EL2, cnthv_visibility),
3699	3751
3700	3752	{ SYS_DESC(SYS_CNTKCTL_EL12), access_cntkctl_el12 },
3701	3753
@@ -5233,15 +5285,28 @@ static int demux_c15_set(struct kvm_vcpu vcpu, u64 id, void __user uaddr) · 1531 unmodified lines
5233	5285	}
5234	5286	}
5235	5287
	5288	+static u64 kvm_one_reg_to_id(const struct kvm_one_reg *reg)
	5289	+{
	5290	+ switch(reg->id) {
	5291	+ case KVM_REG_ARM_TIMER_CVAL:
	5292	+ return TO_ARM64_SYS_REG(CNTV_CVAL_EL0);
	5293	+ case KVM_REG_ARM_TIMER_CNT:
	5294	+ return TO_ARM64_SYS_REG(CNTVCT_EL0);
	5295	+ default:
	5296	+ return reg->id;
	5297	+ }
	5298	+}
	5299	+
5236	5300	int kvm_sys_reg_get_user(struct kvm_vcpu vcpu, const struct kvm_one_reg reg,
5237	5301	const struct sys_reg_desc table[], unsigned int num)
5238	5302	{
5239	5303	u64 __user uaddr = (u64 __user )(unsigned long)reg->addr;
5240	5304	const struct sys_reg_desc *r;
	5305	+ u64 id = kvm_one_reg_to_id(reg);
5241	5306	u64 val;
5242	5307	int ret;
5243	5308
5244		− r = id_to_sys_reg_desc(vcpu, reg->id, table, num);
	5309	+ r = id_to_sys_reg_desc(vcpu, id, table, num);
5245	5310	if (!r \|\| sysreg_hidden(vcpu, r))
5246	5311	return -ENOENT;
5247	5312
@@ -5274,13 +5339,14 @@ int kvm_sys_reg_set_user(struct kvm_vcpu vcpu, const struct kvm_one_reg reg, · 26 unmodified lines
5274	5339	{
5275	5340	u64 __user uaddr = (u64 __user )(unsigned long)reg->addr;
5276	5341	const struct sys_reg_desc *r;
	5342	+ u64 id = kvm_one_reg_to_id(reg);
5277	5343	u64 val;
5278	5344	int ret;
5279	5345
5280	5346	if (get_user(val, uaddr))
5281	5347	return -EFAULT;
5282	5348
5283		− r = id_to_sys_reg_desc(vcpu, reg->id, table, num);
	5349	+ r = id_to_sys_reg_desc(vcpu, id, table, num);
5284	5350	if (!r \|\| sysreg_hidden(vcpu, r))
5285	5351	return -ENOENT;
5286	5352
@@ -5340,10 +5406,23 @@ static u64 sys_reg_to_index(const struct sys_reg_desc *reg) · 53 unmodified lines
5340	5406
5341	5407	static bool copy_reg_to_user(const struct sys_reg_desc reg, u64 __user *uind)
5342	5408	{
	5409	+ u64 idx;
	5410	+
5343	5411	if (!*uind)
5344	5412	return true;
5345	5413
5346		− if (put_user(sys_reg_to_index(reg), *uind))
	5414	+ switch (reg_to_encoding(reg)) {
	5415	+ case SYS_CNTV_CVAL_EL0:
	5416	+ idx = KVM_REG_ARM_TIMER_CVAL;
	5417	+ break;
	5418	+ case SYS_CNTVCT_EL0:
	5419	+ idx = KVM_REG_ARM_TIMER_CNT;
	5420	+ break;
	5421	+ default:
	5422	+ idx = sys_reg_to_index(reg);
	5423	+ }
	5424	+
	5425	+ if (put_user(idx, *uind))
5347	5426	return false;
5348	5427
5349	5428	(*uind)++;

arch/arm64/kvm/sys_regs.h+6−0

@@ -257,4 +257,10 @@ int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu); · 256 unmodified lines
257	257	(val); \
258	258	})
259	259
	260	+#define TO_ARM64_SYS_REG(r) ARM64_SYS_REG(sys_reg_Op0(SYS_ ## r), \
	261	+ sys_reg_Op1(SYS_ ## r), \
	262	+ sys_reg_CRn(SYS_ ## r), \
	263	+ sys_reg_CRm(SYS_ ## r), \
	264	+ sys_reg_Op2(SYS_ ## r))
	265	+
260	266	#endif /* __ARM64_KVM_SYS_REGS_LOCAL_H__ */

arch/arm64/kvm/vgic/vgic-v3.c+4−1

@@ -297,8 +297,11 @@ void vcpu_set_ich_hcr(struct kvm_vcpu *vcpu) · 296 unmodified lines
297	297	{
298	298	struct vgic_v3_cpu_if *vgic_v3 = &vcpu->arch.vgic_cpu.vgic_v3;
299	299
	300	+ if (!vgic_is_v3(vcpu->kvm))
	301	+ return;
	302	+
300	303	/* Hide GICv3 sysreg if necessary */
301		− if (!kvm_has_gicv3(vcpu->kvm)) {
	304	+ if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2) {
302	305	vgic_v3->vgic_hcr \|= (ICH_HCR_EL2_TALL0 \| ICH_HCR_EL2_TALL1 \|
303	306	ICH_HCR_EL2_TC);
304	307	return;

include/kvm/arm_arch_timer.h+16−8

@@ -51,8 +51,6 @@ struct arch_timer_vm_data { · 50 unmodified lines
51	51	};
52	52
53	53	struct arch_timer_context {
54		− struct kvm_vcpu *vcpu;
55		−
56	54	/* Emulated Timer (may be unused) */
57	55	struct hrtimer hrtimer;
58	56	u64 ns_frac;
@@ -71,6 +69,9 @@ struct arch_timer_context { · 12 unmodified lines
71	69	bool level;
72	70	} irq;
73	71
	72	+ /* Who am I? */
	73	+ enum kvm_arch_timers timer_id;
	74	+
74	75	/* Duplicated state from arch_timer.c for convenience */
75	76	u32 host_timer_irq;
76	77	};
@@ -106,9 +107,6 @@ void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu); · 29 unmodified lines
106	107
107	108	void kvm_timer_init_vm(struct kvm *kvm);
108	109
109		−u64 kvm_arm_timer_get_reg(struct kvm_vcpu *, u64 regid);
110		−int kvm_arm_timer_set_reg(struct kvm_vcpu *, u64 regid, u64 value);
111		−
112	110	int kvm_arm_timer_set_attr(struct kvm_vcpu vcpu, struct kvm_device_attr attr);
113	111	int kvm_arm_timer_get_attr(struct kvm_vcpu vcpu, struct kvm_device_attr attr);
114	112	int kvm_arm_timer_has_attr(struct kvm_vcpu vcpu, struct kvm_device_attr attr);
@@ -127,9 +125,9 @@ void kvm_timer_init_vhe(void); · 12 unmodified lines
127	125	#define vcpu_hvtimer(v) (&(v)->arch.timer_cpu.timers[TIMER_HVTIMER])
128	126	#define vcpu_hptimer(v) (&(v)->arch.timer_cpu.timers[TIMER_HPTIMER])
129	127
130		−#define arch_timer_ctx_index(ctx) ((ctx) - vcpu_timer((ctx)->vcpu)->timers)
131		−
132		−#define timer_vm_data(ctx) (&(ctx)->vcpu->kvm->arch.timer_data)
	128	+#define arch_timer_ctx_index(ctx) ((ctx)->timer_id)
	129	+#define timer_context_to_vcpu(ctx) container_of((ctx), struct kvm_vcpu, arch.timer_cpu.timers[(ctx)->timer_id])
	130	+#define timer_vm_data(ctx) (&(timer_context_to_vcpu(ctx)->kvm->arch.timer_data))
133	131	#define timer_irq(ctx) (timer_vm_data(ctx)->ppi[arch_timer_ctx_index(ctx)])
134	132
135	133	u64 kvm_arm_timer_read_sysreg(struct kvm_vcpu *vcpu,
@@ -178,4 +176,14 @@ static inline u64 timer_get_offset(struct arch_timer_context *ctxt) · 42 unmodified lines
178	176	return offset;
179	177	}
180	178
	179	+static inline void timer_set_offset(struct arch_timer_context *ctxt, u64 offset)
	180	+{
	181	+ if (!ctxt->offset.vm_offset) {
	182	+ WARN(offset, "timer %d\n", arch_timer_ctx_index(ctxt));
	183	+ return;
	184	+ }
	185	+
	186	+ WRITE_ONCE(*ctxt->offset.vm_offset, offset);
	187	+}
	188	+
181	189	#endif

tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c+1−1

@@ -1020,7 +1020,7 @@ static void set_counter_defaults(void) · 1019 unmodified lines
1020	1020	{
1021	1021	const uint64_t MIN_ROLLOVER_SECS = 40ULL * 365 * 24 * 3600;
1022	1022	uint64_t freq = read_sysreg(CNTFRQ_EL0);
1023		− uint64_t width = ilog2(MIN_ROLLOVER_SECS * freq);
	1023	+ int width = ilog2(MIN_ROLLOVER_SECS * freq);
1024	1024
1025	1025	width = clamp(width, 56, 64);
1026	1026	CVAL_MAX = GENMASK_ULL(width - 1, 0);

tools/testing/selftests/kvm/arm64/external_aborts.c+43−0

@@ -359,6 +359,44 @@ static void test_mmio_ease(void) · 358 unmodified lines
359	359	kvm_vm_free(vm);
360	360	}
361	361
	362	+static void test_serror_amo_guest(void)
	363	+{
	364	+ /*
	365	+ * The ISB is entirely unnecessary (and highlights how FEAT_NV2 is borked)
	366	+ * since the write is redirected to memory. But don't write (intentionally)
	367	+ * broken code!
	368	+ */
	369	+ sysreg_clear_set(hcr_el2, HCR_EL2_AMO \| HCR_EL2_TGE, 0);
	370	+ isb();
	371	+
	372	+ GUEST_SYNC(0);
	373	+ GUEST_ASSERT(read_sysreg(isr_el1) & ISR_EL1_A);
	374	+
	375	+ /*
	376	+ * KVM treats the effective value of AMO as 1 when
	377	+ * HCR_EL2.{E2H,TGE} = {1, 0}, meaning the SError will be taken when
	378	+ * unmasked.
	379	+ */
	380	+ local_serror_enable();
	381	+ isb();
	382	+ local_serror_disable();
	383	+
	384	+ GUEST_FAIL("Should've taken pending SError exception");
	385	+}
	386	+
	387	+static void test_serror_amo(void)
	388	+{
	389	+ struct kvm_vcpu *vcpu;
	390	+ struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_serror_amo_guest,
	391	+ unexpected_dabt_handler);
	392	+
	393	+ vm_install_exception_handler(vm, VECTOR_ERROR_CURRENT, expect_serror_handler);
	394	+ vcpu_run_expect_sync(vcpu);
	395	+ vcpu_inject_serror(vcpu);
	396	+ vcpu_run_expect_done(vcpu);
	397	+ kvm_vm_free(vm);
	398	+}
	399	+
362	400	int main(void)
363	401	{
364	402	test_mmio_abort();
@@ -369,4 +407,9 @@ int main(void) · 4 unmodified lines
369	407	test_serror_emulated();
370	408	test_mmio_ease();
371	409	test_s1ptw_abort();
	410	+
	411	+ if (!test_supports_el2())
	412	+ return 0;
	413	+
	414	+ test_serror_amo();
372	415	}

tools/testing/selftests/kvm/arm64/get-reg-list.c+96−3

@@ -65,6 +65,9 @@ static struct feature_id_reg feat_id_regs[] = { · 64 unmodified lines
65	65	REG_FEAT(SCTLR2_EL1, ID_AA64MMFR3_EL1, SCTLRX, IMP),
66	66	REG_FEAT(VDISR_EL2, ID_AA64PFR0_EL1, RAS, IMP),
67	67	REG_FEAT(VSESR_EL2, ID_AA64PFR0_EL1, RAS, IMP),
	68	+ REG_FEAT(VNCR_EL2, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY),
	69	+ REG_FEAT(CNTHV_CTL_EL2, ID_AA64MMFR1_EL1, VH, IMP),
	70	+ REG_FEAT(CNTHV_CVAL_EL2,ID_AA64MMFR1_EL1, VH, IMP),
68	71	};
69	72
70	73	bool filter_reg(__u64 reg)
@@ -345,9 +348,20 @@ static __u64 base_regs[] = { · 274 unmodified lines
345	348	KVM_REG_ARM_FW_FEAT_BMAP_REG(1), /* KVM_REG_ARM_STD_HYP_BMAP */
346	349	KVM_REG_ARM_FW_FEAT_BMAP_REG(2), /* KVM_REG_ARM_VENDOR_HYP_BMAP */
347	350	KVM_REG_ARM_FW_FEAT_BMAP_REG(3), /* KVM_REG_ARM_VENDOR_HYP_BMAP_2 */
348		− ARM64_SYS_REG(3, 3, 14, 3, 1), /* CNTV_CTL_EL0 */
349		− ARM64_SYS_REG(3, 3, 14, 3, 2), /* CNTV_CVAL_EL0 */
350		− ARM64_SYS_REG(3, 3, 14, 0, 2),
	351	+
	352	+ /*
	353	+ * EL0 Virtual Timer Registers
	354	+ *
	355	+ * WARNING:
	356	+ * KVM_REG_ARM_TIMER_CVAL and KVM_REG_ARM_TIMER_CNT are not defined
	357	+ * with the appropriate register encodings. Their values have been
	358	+ * accidentally swapped. As this is set API, the definitions here
	359	+ * must be used, rather than ones derived from the encodings.
	360	+ */
	361	+ KVM_ARM64_SYS_REG(SYS_CNTV_CTL_EL0),
	362	+ KVM_REG_ARM_TIMER_CVAL,
	363	+ KVM_REG_ARM_TIMER_CNT,
	364	+
351	365	ARM64_SYS_REG(3, 0, 0, 0, 0), /* MIDR_EL1 */
352	366	ARM64_SYS_REG(3, 0, 0, 0, 6), /* REVIDR_EL1 */
353	367	ARM64_SYS_REG(3, 1, 0, 0, 1), /* CLIDR_EL1 */
@@ -755,6 +769,10 @@ static __u64 el2_regs[] = { · 401 unmodified lines
755	769	SYS_REG(VSESR_EL2),
756	770	};
757	771
	772	+static __u64 el2_e2h0_regs[] = {
	773	+ /* Empty */
	774	+};
	775	+
758	776	#define BASE_SUBLIST \
759	777	{ "base", .regs = base_regs, .regs_n = ARRAY_SIZE(base_regs), }
760	778	#define VREGS_SUBLIST \
@@ -789,6 +807,15 @@ static __u64 el2_regs[] = { · 28 unmodified lines
789	807	.regs = el2_regs, \
790	808	.regs_n = ARRAY_SIZE(el2_regs), \
791	809	}
	810	+#define EL2_E2H0_SUBLIST \
	811	+ EL2_SUBLIST, \
	812	+ { \
	813	+ .name = "EL2 E2H0", \
	814	+ .capability = KVM_CAP_ARM_EL2_E2H0, \
	815	+ .feature = KVM_ARM_VCPU_HAS_EL2_E2H0, \
	816	+ .regs = el2_e2h0_regs, \
	817	+ .regs_n = ARRAY_SIZE(el2_e2h0_regs), \
	818	+ }
792	819
793	820	static struct vcpu_reg_list vregs_config = {
794	821	.sublists = {
@@ -897,6 +924,65 @@ static struct vcpu_reg_list el2_pauth_pmu_config = { · 102 unmodified lines
897	924	},
898	925	};
899	926
	927	+static struct vcpu_reg_list el2_e2h0_vregs_config = {
	928	+ .sublists = {
	929	+ BASE_SUBLIST,
	930	+ EL2_E2H0_SUBLIST,
	931	+ VREGS_SUBLIST,
	932	+ {0},
	933	+ },
	934	+};
	935	+
	936	+static struct vcpu_reg_list el2_e2h0_vregs_pmu_config = {
	937	+ .sublists = {
	938	+ BASE_SUBLIST,
	939	+ EL2_E2H0_SUBLIST,
	940	+ VREGS_SUBLIST,
	941	+ PMU_SUBLIST,
	942	+ {0},
	943	+ },
	944	+};
	945	+
	946	+static struct vcpu_reg_list el2_e2h0_sve_config = {
	947	+ .sublists = {
	948	+ BASE_SUBLIST,
	949	+ EL2_E2H0_SUBLIST,
	950	+ SVE_SUBLIST,
	951	+ {0},
	952	+ },
	953	+};
	954	+
	955	+static struct vcpu_reg_list el2_e2h0_sve_pmu_config = {
	956	+ .sublists = {
	957	+ BASE_SUBLIST,
	958	+ EL2_E2H0_SUBLIST,
	959	+ SVE_SUBLIST,
	960	+ PMU_SUBLIST,
	961	+ {0},
	962	+ },
	963	+};
	964	+
	965	+static struct vcpu_reg_list el2_e2h0_pauth_config = {
	966	+ .sublists = {
	967	+ BASE_SUBLIST,
	968	+ EL2_E2H0_SUBLIST,
	969	+ VREGS_SUBLIST,
	970	+ PAUTH_SUBLIST,
	971	+ {0},
	972	+ },
	973	+};
	974	+
	975	+static struct vcpu_reg_list el2_e2h0_pauth_pmu_config = {
	976	+ .sublists = {
	977	+ BASE_SUBLIST,
	978	+ EL2_E2H0_SUBLIST,
	979	+ VREGS_SUBLIST,
	980	+ PAUTH_SUBLIST,
	981	+ PMU_SUBLIST,
	982	+ {0},
	983	+ },
	984	+};
	985	+
900	986	struct vcpu_reg_list *vcpu_configs[] = {
901	987	&vregs_config,
902	988	&vregs_pmu_config,
@@ -911,5 +997,12 @@ struct vcpu_reg_list *vcpu_configs[] = { · 8 unmodified lines
911	997	&el2_sve_pmu_config,
912	998	&el2_pauth_config,
913	999	&el2_pauth_pmu_config,
	1000	+
	1001	+ &el2_e2h0_vregs_config,
	1002	+ &el2_e2h0_vregs_pmu_config,
	1003	+ &el2_e2h0_sve_config,
	1004	+ &el2_e2h0_sve_pmu_config,
	1005	+ &el2_e2h0_pauth_config,
	1006	+ &el2_e2h0_pauth_pmu_config,
914	1007	};
915	1008	int vcpu_configs_n = ARRAY_SIZE(vcpu_configs);

tools/testing/selftests/kvm/arm64/set_id_regs.c+3−0

@@ -249,11 +249,14 @@ static void guest_code(void) · 248 unmodified lines
249	249	GUEST_REG_SYNC(SYS_ID_AA64ISAR2_EL1);
250	250	GUEST_REG_SYNC(SYS_ID_AA64ISAR3_EL1);
251	251	GUEST_REG_SYNC(SYS_ID_AA64PFR0_EL1);
	252	+ GUEST_REG_SYNC(SYS_ID_AA64PFR1_EL1);
252	253	GUEST_REG_SYNC(SYS_ID_AA64MMFR0_EL1);
253	254	GUEST_REG_SYNC(SYS_ID_AA64MMFR1_EL1);
254	255	GUEST_REG_SYNC(SYS_ID_AA64MMFR2_EL1);
255	256	GUEST_REG_SYNC(SYS_ID_AA64MMFR3_EL1);
256	257	GUEST_REG_SYNC(SYS_ID_AA64ZFR0_EL1);
	258	+ GUEST_REG_SYNC(SYS_MPIDR_EL1);
	259	+ GUEST_REG_SYNC(SYS_CLIDR_EL1);
257	260	GUEST_REG_SYNC(SYS_CTR_EL0);
258	261	GUEST_REG_SYNC(SYS_MIDR_EL1);
259	262	GUEST_REG_SYNC(SYS_REVIDR_EL1);

tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c+2−1

@@ -123,6 +123,7 @@ static void guest_setup_gic(void) · 122 unmodified lines
123	123	static void guest_code(size_t nr_lpis)
124	124	{
125	125	guest_setup_gic();
	126	+ local_irq_enable();
126	127
127	128	GUEST_SYNC(0);
128	129
@@ -331,7 +332,7 @@ static void setup_vm(void) · 202 unmodified lines
331	332	{
332	333	int i;
333	334
334		− vcpus = malloc(test_data.nr_cpus * sizeof(struct kvm_vcpu));
	335	+ vcpus = malloc(test_data.nr_cpus * sizeof(struct kvm_vcpu *));
335	336	TEST_ASSERT(vcpus, "Failed to allocate vCPU array");
336	337
337	338	vm = vm_create_with_vcpus(test_data.nr_cpus, guest_code, vcpus);

tools/testing/selftests/kvm/include/arm64/processor.h+11−1

@@ -305,7 +305,17 @@ void test_wants_mte(void); · 304 unmodified lines
305	305	void test_disable_default_vgic(void);
306	306
307	307	bool vm_supports_el2(struct kvm_vm *vm);
308		−static bool vcpu_has_el2(struct kvm_vcpu *vcpu)
	308	+
	309	+static inline bool test_supports_el2(void)
	310	+{
	311	+ struct kvm_vm *vm = vm_create(1);
	312	+ bool supported = vm_supports_el2(vm);
	313	+
	314	+ kvm_vm_free(vm);
	315	+ return supported;
	316	+}
	317	+
	318	+static inline bool vcpu_has_el2(struct kvm_vcpu *vcpu)
309	319	{
310	320	return vcpu->init.features[0] & BIT(KVM_ARM_VCPU_HAS_EL2);
311	321	}

tools/testing/selftests/kvm/include/kvm_util.h+2−0

@@ -1273,4 +1273,6 @@ bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr); · 1272 unmodified lines
1273	1273
1274	1274	uint32_t guest_get_vcpuid(void);
1275	1275
	1276	+bool kvm_arch_has_default_irqchip(void);
	1277	+
1276	1278	#endif /* SELFTEST_KVM_UTIL_H */

tools/testing/selftests/kvm/irqfd_test.c+11−3

@@ -89,11 +89,19 @@ static void juggle_eventfd_primary(struct kvm_vm *vm, int eventfd) · 88 unmodified lines
89	89	int main(int argc, char *argv[])
90	90	{
91	91	pthread_t racing_thread;
	92	+ struct kvm_vcpu *unused;
92	93	int r, i;
93	94
94		− /* Create "full" VMs, as KVM_IRQFD requires an in-kernel IRQ chip. */
95		− vm1 = vm_create(1);
96		− vm2 = vm_create(1);
	95	+ TEST_REQUIRE(kvm_arch_has_default_irqchip());
	96	+
	97	+ /*
	98	+ * Create "full" VMs, as KVM_IRQFD requires an in-kernel IRQ chip. Also
	99	+ * create an unused vCPU as certain architectures (like arm64) need to
	100	+ * complete IRQ chip initialization after all possible vCPUs for a VM
	101	+ * have been created.
	102	+ */
	103	+ vm1 = vm_create_with_one_vcpu(&unused, NULL);
	104	+ vm2 = vm_create_with_one_vcpu(&unused, NULL);
97	105
98	106	WRITE_ONCE(__eventfd, kvm_new_eventfd());
99	107

tools/testing/selftests/kvm/lib/arm64/processor.c+5−0

@@ -725,3 +725,8 @@ void kvm_arch_vm_release(struct kvm_vm *vm) · 724 unmodified lines
725	725	if (vm->arch.has_gic)
726	726	close(vm->arch.gic_fd);
727	727	}
	728	+
	729	+bool kvm_arch_has_default_irqchip(void)
	730	+{
	731	+ return request_vgic && kvm_supports_vgic_v3();
	732	+}

tools/testing/selftests/kvm/lib/kvm_util.c+5−0

@@ -2344,3 +2344,8 @@ bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr) · 2343 unmodified lines
2344	2344	pg = paddr >> vm->page_shift;
2345	2345	return sparsebit_is_set(region->protected_phy_pages, pg);
2346	2346	}
	2347	+
	2348	+__weak bool kvm_arch_has_default_irqchip(void)
	2349	+{
	2350	+ return false;
	2351	+}

tools/testing/selftests/kvm/lib/s390/processor.c+5−0

@@ -221,3 +221,8 @@ void vcpu_arch_dump(FILE stream, struct kvm_vcpu vcpu, uint8_t indent) · 220 unmodified lines
221	221	void assert_on_unhandled_exception(struct kvm_vcpu *vcpu)
222	222	{
223	223	}
	224	+
	225	+bool kvm_arch_has_default_irqchip(void)
	226	+{
	227	+ return true;
	228	+}

tools/testing/selftests/kvm/lib/x86/processor.c+5−0

@@ -1318,3 +1318,8 @@ bool sys_clocksource_is_based_on_tsc(void) · 1317 unmodified lines
1318	1318
1319	1319	return ret;
1320	1320	}
	1321	+
	1322	+bool kvm_arch_has_default_irqchip(void)
	1323	+{
	1324	+ return true;
	1325	+}