Merge tag 'kvm-arm-for-v4.12' of git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into HEAD

KVM/ARM Changes for v4.12. Changes include: - Using the common sysreg definitions between KVM and arm64 - Improved hyp-stub implementation with support for kexec and kdump on the 32-bit side - Proper PMU exception handling - Performance improvements of our GIC handling - Support for irqchip in userspace with in-kernel arch-timers and PMU support - A fix for a race condition in our PSCI code

Paolo Bonzini committed 9 years ago

Commit c24a7be2 parents 70f3aac 1edb63250 files · +1174 −938Browse files

Unified · Split

50 files

Documentation/virtual/kvm/api.txt +41 −0
Documentation/virtual/kvm/arm/hyp-abi.txt +53 −0
arch/arm/boot/compressed/head.S +11 −1
arch/arm/include/asm/kvm_asm.h +4 −3
arch/arm/include/asm/kvm_host.h +0 −6
arch/arm/include/asm/kvm_mmu.h +0 −1
arch/arm/include/asm/proc-fns.h +2 −2
arch/arm/include/asm/virt.h +13 −1
arch/arm/include/uapi/asm/kvm.h +2 −0
arch/arm/kernel/hyp-stub.S +34 −9
arch/arm/kernel/reboot.c +5 −2
arch/arm/kvm/arm.c +36 −30
arch/arm/kvm/coproc.c +21 −3
arch/arm/kvm/coproc.h +0 −18
arch/arm/kvm/handle_exit.c +8 −0
arch/arm/kvm/hyp/hyp-entry.S +23 −5
arch/arm/kvm/init.S +43 −8
arch/arm/kvm/interrupts.S +0 −4
arch/arm/kvm/mmu.c +13 −23
arch/arm/kvm/psci.c +7 −1
arch/arm/mm/mmu.c +5 −0
arch/arm/mm/proc-v7.S +10 −5
arch/arm64/include/asm/arch_gicv3.h +13 −68
arch/arm64/include/asm/kvm_asm.h +3 −2
arch/arm64/include/asm/kvm_host.h +0 −7
arch/arm64/include/asm/kvm_mmu.h +0 −1
arch/arm64/include/asm/sysreg.h +155 −7
arch/arm64/include/asm/virt.h +22 −9
arch/arm64/include/uapi/asm/kvm.h +2 −0
arch/arm64/kernel/head.S +4 −4
arch/arm64/kernel/hyp-stub.S +14 −24
arch/arm64/kvm/hyp-init.S +36 −10
arch/arm64/kvm/hyp.S +1 −4
arch/arm64/kvm/hyp/hyp-entry.S +21 −22
arch/arm64/kvm/sys_regs.c +180 −316
arch/arm64/kvm/sys_regs.h +5 −18
arch/arm64/kvm/sys_regs_generic_v8.c +1 −3
include/kvm/arm_arch_timer.h +2 −0
include/kvm/arm_pmu.h +7 −0
include/kvm/arm_vgic.h +3 −6
include/uapi/linux/kvm.h +8 −0
virt/kvm/arm/arch_timer.c +98 −26
virt/kvm/arm/hyp/vgic-v2-sr.c +7 −71
virt/kvm/arm/hyp/vgic-v3-sr.c +24 −63
virt/kvm/arm/pmu.c +35 −4
virt/kvm/arm/vgic/vgic-init.c +68 −40
virt/kvm/arm/vgic/vgic-v2.c +39 −51
virt/kvm/arm/vgic/vgic-v3.c +45 −42
virt/kvm/arm/vgic/vgic.c +44 −16
virt/kvm/arm/vgic/vgic.h +6 −2

Documentation/virtual/kvm/api.txt+41−0

@@ -4120,3 +4120,44 @@ This capability indicates that guest using memory monotoring instructions · 4119 unmodified lines
4120	4120	(MWAIT/MWAITX) to stop the virtual CPU will not cause a VM exit. As such time
4121	4121	spent while virtual CPU is halted in this way will then be accounted for as
4122	4122	guest running time on the host (as opposed to e.g. HLT).
	4123	+
	4124	+8.9 KVM_CAP_ARM_USER_IRQ
	4125	+
	4126	+Architectures: arm, arm64
	4127	+This capability, if KVM_CHECK_EXTENSION indicates that it is available, means
	4128	+that if userspace creates a VM without an in-kernel interrupt controller, it
	4129	+will be notified of changes to the output level of in-kernel emulated devices,
	4130	+which can generate virtual interrupts, presented to the VM.
	4131	+For such VMs, on every return to userspace, the kernel
	4132	+updates the vcpu's run->s.regs.device_irq_level field to represent the actual
	4133	+output level of the device.
	4134	+
	4135	+Whenever kvm detects a change in the device output level, kvm guarantees at
	4136	+least one return to userspace before running the VM. This exit could either
	4137	+be a KVM_EXIT_INTR or any other exit event, like KVM_EXIT_MMIO. This way,
	4138	+userspace can always sample the device output level and re-compute the state of
	4139	+the userspace interrupt controller. Userspace should always check the state
	4140	+of run->s.regs.device_irq_level on every kvm exit.
	4141	+The value in run->s.regs.device_irq_level can represent both level and edge
	4142	+triggered interrupt signals, depending on the device. Edge triggered interrupt
	4143	+signals will exit to userspace with the bit in run->s.regs.device_irq_level
	4144	+set exactly once per edge signal.
	4145	+
	4146	+The field run->s.regs.device_irq_level is available independent of
	4147	+run->kvm_valid_regs or run->kvm_dirty_regs bits.
	4148	+
	4149	+If KVM_CAP_ARM_USER_IRQ is supported, the KVM_CHECK_EXTENSION ioctl returns a
	4150	+number larger than 0 indicating the version of this capability is implemented
	4151	+and thereby which bits in in run->s.regs.device_irq_level can signal values.
	4152	+
	4153	+Currently the following bits are defined for the device_irq_level bitmap:
	4154	+
	4155	+ KVM_CAP_ARM_USER_IRQ >= 1:
	4156	+
	4157	+ KVM_ARM_DEV_EL1_VTIMER - EL1 virtual timer
	4158	+ KVM_ARM_DEV_EL1_PTIMER - EL1 physical timer
	4159	+ KVM_ARM_DEV_PMU - ARM PMU overflow interrupt signal
	4160	+
	4161	+Future versions of kvm may implement additional events. These will get
	4162	+indicated by returning a higher number from KVM_CHECK_EXTENSION and will be
	4163	+listed above.

Documentation/virtual/kvm/arm/hyp-abi.txt+53−0

@@ -0,0 +1,53 @@
	1	+* Internal ABI between the kernel and HYP
	2	+
	3	+This file documents the interaction between the Linux kernel and the
	4	+hypervisor layer when running Linux as a hypervisor (for example
	5	+KVM). It doesn't cover the interaction of the kernel with the
	6	+hypervisor when running as a guest (under Xen, KVM or any other
	7	+hypervisor), or any hypervisor-specific interaction when the kernel is
	8	+used as a host.
	9	+
	10	+On arm and arm64 (without VHE), the kernel doesn't run in hypervisor
	11	+mode, but still needs to interact with it, allowing a built-in
	12	+hypervisor to be either installed or torn down.
	13	+
	14	+In order to achieve this, the kernel must be booted at HYP (arm) or
	15	+EL2 (arm64), allowing it to install a set of stubs before dropping to
	16	+SVC/EL1. These stubs are accessible by using a 'hvc #0' instruction,
	17	+and only act on individual CPUs.
	18	+
	19	+Unless specified otherwise, any built-in hypervisor must implement
	20	+these functions (see arch/arm{,64}/include/asm/virt.h):
	21	+
	22	+* r0/x0 = HVC_SET_VECTORS
	23	+ r1/x1 = vectors
	24	+
	25	+ Set HVBAR/VBAR_EL2 to 'vectors' to enable a hypervisor. 'vectors'
	26	+ must be a physical address, and respect the alignment requirements
	27	+ of the architecture. Only implemented by the initial stubs, not by
	28	+ Linux hypervisors.
	29	+
	30	+* r0/x0 = HVC_RESET_VECTORS
	31	+
	32	+ Turn HYP/EL2 MMU off, and reset HVBAR/VBAR_EL2 to the initials
	33	+ stubs' exception vector value. This effectively disables an existing
	34	+ hypervisor.
	35	+
	36	+* r0/x0 = HVC_SOFT_RESTART
	37	+ r1/x1 = restart address
	38	+ x2 = x0's value when entering the next payload (arm64)
	39	+ x3 = x1's value when entering the next payload (arm64)
	40	+ x4 = x2's value when entering the next payload (arm64)
	41	+
	42	+ Mask all exceptions, disable the MMU, move the arguments into place
	43	+ (arm64 only), and jump to the restart address while at HYP/EL2. This
	44	+ hypercall is not expected to return to its caller.
	45	+
	46	+Any other value of r0/x0 triggers a hypervisor-specific handling,
	47	+which is not documented here.
	48	+
	49	+The return value of a stub hypercall is held by r0/x0, and is 0 on
	50	+success, and HVC_STUB_ERR on error. A stub hypercall is allowed to
	51	+clobber any of the caller-saved registers (x0-x18 on arm64, r0-r3 and
	52	+ip on arm). It is thus recommended to use a function call to perform
	53	+the hypercall.

arch/arm/boot/compressed/head.S+11−1

@@ -422,7 +422,17 @@ dtb_check_done: · 421 unmodified lines
422	422	cmp r0, #HYP_MODE
423	423	bne 1f
424	424
425		− bl __hyp_get_vectors
	425	+ /*
	426	+ * Compute the address of the hyp vectors after relocation.
	427	+ * This requires some arithmetic since we cannot directly
	428	+ * reference __hyp_stub_vectors in a PC-relative way.
	429	+ * Call __hyp_set_vectors with the new address so that we
	430	+ * can HVC again after the copy.
	431	+ */
	432	+0: adr r0, 0b
	433	+ movw r1, #:lower16:__hyp_stub_vectors - 0b
	434	+ movt r1, #:upper16:__hyp_stub_vectors - 0b
	435	+ add r0, r0, r1
426	436	sub r0, r0, r5
427	437	add r0, r0, r10
428	438	bl __hyp_set_vectors

arch/arm/include/asm/kvm_asm.h+4−3

@@ -33,7 +33,7 @@ · 32 unmodified lines
33	33	#define ARM_EXCEPTION_IRQ 5
34	34	#define ARM_EXCEPTION_FIQ 6
35	35	#define ARM_EXCEPTION_HVC 7
36		−
	36	+#define ARM_EXCEPTION_HYP_GONE HVC_STUB_ERR
37	37	/*
38	38	* The rr_lo_hi macro swaps a pair of registers depending on
39	39	* current endianness. It is used in conjunction with ldrd and strd
@@ -72,10 +72,11 @@ extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu); · 32 unmodified lines
72	72
73	73	extern void __init_stage2_translation(void);
74	74
75		−extern void __kvm_hyp_reset(unsigned long);
76		−
77	75	extern u64 __vgic_v3_get_ich_vtr_el2(void);
	76	+extern u64 __vgic_v3_read_vmcr(void);
	77	+extern void __vgic_v3_write_vmcr(u32 vmcr);
78	78	extern void __vgic_v3_init_lrs(void);
	79	+
79	80	#endif
80	81
81	82	#endif /* __ARM_KVM_ASM_H__ */

arch/arm/include/asm/kvm_host.h+0−6

@@ -269,12 +269,6 @@ static inline void __cpu_init_stage2(void) · 268 unmodified lines
269	269	kvm_call_hyp(__init_stage2_translation);
270	270	}
271	271
272		−static inline void __cpu_reset_hyp_mode(unsigned long vector_ptr,
273		− phys_addr_t phys_idmap_start)
274		−{
275		− kvm_call_hyp((void *)virt_to_idmap(__kvm_hyp_reset), vector_ptr);
276		−}
277		−
278	272	static inline int kvm_arch_dev_ioctl_check_extension(struct kvm *kvm, long ext)
279	273	{
280	274	return 0;

arch/arm/include/asm/kvm_mmu.h+0−1

@@ -56,7 +56,6 @@ void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu); · 55 unmodified lines
56	56
57	57	phys_addr_t kvm_mmu_get_httbr(void);
58	58	phys_addr_t kvm_get_idmap_vector(void);
59		−phys_addr_t kvm_get_idmap_start(void);
60	59	int kvm_mmu_init(void);
61	60	void kvm_clear_hyp_idmap(void);
62	61

arch/arm/include/asm/proc-fns.h+2−2

@@ -43,7 +43,7 @@ extern struct processor { · 42 unmodified lines
43	43	/*
44	44	* Special stuff for a reset
45	45	*/
46		− void (*reset)(unsigned long addr) __attribute__((noreturn));
	46	+ void (*reset)(unsigned long addr, bool hvc) __attribute__((noreturn));
47	47	/*
48	48	* Idle the processor
49	49	*/
@@ -88,7 +88,7 @@ extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte); · 38 unmodified lines
88	88	#else
89	89	extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext);
90	90	#endif
91		−extern void cpu_reset(unsigned long addr) __attribute__((noreturn));
	91	+extern void cpu_reset(unsigned long addr, bool hvc) __attribute__((noreturn));
92	92
93	93	/* These three are private to arch/arm/kernel/suspend.c */
94	94	extern void cpu_do_suspend(void *);

arch/arm/include/asm/virt.h+13−1

@@ -53,7 +53,7 @@ static inline void sync_boot_mode(void) · 52 unmodified lines
53	53	}
54	54
55	55	void __hyp_set_vectors(unsigned long phys_vector_base);
56		−unsigned long __hyp_get_vectors(void);
	56	+void __hyp_reset_vectors(void);
57	57	#else
58	58	#define __boot_cpu_mode (SVC_MODE)
59	59	#define sync_boot_mode()
@@ -94,6 +94,18 @@ extern char __hyp_text_start[]; · 34 unmodified lines
94	94	extern char __hyp_text_end[];
95	95	#endif
96	96
	97	+#else
	98	+
	99	+/* Only assembly code should need those */
	100	+
	101	+#define HVC_SET_VECTORS 0
	102	+#define HVC_SOFT_RESTART 1
	103	+#define HVC_RESET_VECTORS 2
	104	+
	105	+#define HVC_STUB_HCALL_NR 3
	106	+
97	107	#endif /* __ASSEMBLY__ */
98	108
	109	+#define HVC_STUB_ERR 0xbadca11
	110	+
99	111	#endif /* ! VIRT_H */

arch/arm/include/uapi/asm/kvm.h+2−0

@@ -116,6 +116,8 @@ struct kvm_debug_exit_arch { · 115 unmodified lines
116	116	};
117	117
118	118	struct kvm_sync_regs {
	119	+ /* Used with KVM_CAP_ARM_USER_IRQ */
	120	+ __u64 device_irq_level;
119	121	};
120	122
121	123	struct kvm_arch_memory_slot {

arch/arm/kernel/hyp-stub.S+34−9

@@ -125,7 +125,7 @@ ENTRY(__hyp_stub_install_secondary) · 124 unmodified lines
125	125	* (see safe_svcmode_maskall).
126	126	*/
127	127	@ Now install the hypervisor stub:
128		− adr r7, __hyp_stub_vectors
	128	+ W(adr) r7, __hyp_stub_vectors
129	129	mcr p15, 4, r7, c12, c0, 0 @ set hypervisor vector base (HVBAR)
130	130
131	131	@ Disable all traps, so we don't get any nasty surprise
@@ -202,9 +202,23 @@ ARM_BE8(orr r7, r7, #(1 << 25)) @ HSCTLR.EE · 70 unmodified lines
202	202	ENDPROC(__hyp_stub_install_secondary)
203	203
204	204	__hyp_stub_do_trap:
205		− cmp r0, #-1
206		− mrceq p15, 4, r0, c12, c0, 0 @ get HVBAR
207		− mcrne p15, 4, r0, c12, c0, 0 @ set HVBAR
	205	+ teq r0, #HVC_SET_VECTORS
	206	+ bne 1f
	207	+ mcr p15, 4, r1, c12, c0, 0 @ set HVBAR
	208	+ b __hyp_stub_exit
	209	+
	210	+1: teq r0, #HVC_SOFT_RESTART
	211	+ bne 1f
	212	+ bx r1
	213	+
	214	+1: teq r0, #HVC_RESET_VECTORS
	215	+ beq __hyp_stub_exit
	216	+
	217	+ ldr r0, =HVC_STUB_ERR
	218	+ __ERET
	219	+
	220	+__hyp_stub_exit:
	221	+ mov r0, #0
208	222	__ERET
209	223	ENDPROC(__hyp_stub_do_trap)
210	224
@@ -230,15 +244,26 @@ ENDPROC(__hyp_stub_do_trap) · 19 unmodified lines
230	244	* so you will need to set that to something sensible at the new hypervisor's
231	245	* initialisation entry point.
232	246	*/
233		−ENTRY(__hyp_get_vectors)
234		− mov r0, #-1
235		−ENDPROC(__hyp_get_vectors)
236		− @ fall through
237	247	ENTRY(__hyp_set_vectors)
	248	+ mov r1, r0
	249	+ mov r0, #HVC_SET_VECTORS
238	250	__HVC(0)
239	251	ret lr
240	252	ENDPROC(__hyp_set_vectors)
241	253
	254	+ENTRY(__hyp_soft_restart)
	255	+ mov r1, r0
	256	+ mov r0, #HVC_SOFT_RESTART
	257	+ __HVC(0)
	258	+ ret lr
	259	+ENDPROC(__hyp_soft_restart)
	260	+
	261	+ENTRY(__hyp_reset_vectors)
	262	+ mov r0, #HVC_RESET_VECTORS
	263	+ __HVC(0)
	264	+ ret lr
	265	+ENDPROC(__hyp_reset_vectors)
	266	+
242	267	#ifndef ZIMAGE
243	268	.align 2
244	269	.L__boot_cpu_mode_offset:
@@ -246,7 +271,7 @@ ENDPROC(__hyp_set_vectors) · 1 unmodified line
246	271	#endif
247	272
248	273	.align 5
249		−__hyp_stub_vectors:
	274	+ENTRY(__hyp_stub_vectors)
250	275	__hyp_stub_reset: W(b) .
251	276	__hyp_stub_und: W(b) .
252	277	__hyp_stub_svc: W(b) .

arch/arm/kernel/reboot.c+5−2

@@ -12,10 +12,11 @@ · 11 unmodified lines
12	12
13	13	#include <asm/cacheflush.h>
14	14	#include <asm/idmap.h>
	15	+#include <asm/virt.h>
15	16
16	17	#include "reboot.h"
17	18
18		−typedef void (*phys_reset_t)(unsigned long);
	19	+typedef void (*phys_reset_t)(unsigned long, bool);
19	20
20	21	/*
21	22	* Function pointers to optional machine specific functions
@@ -51,7 +52,9 @@ static void __soft_restart(void *addr) · 29 unmodified lines
51	52
52	53	/* Switch to the identity mapping. */
53	54	phys_reset = (phys_reset_t)virt_to_idmap(cpu_reset);
54		− phys_reset((unsigned long)addr);
	55	+
	56	+ /* original stub should be restored by kvm */
	57	+ phys_reset((unsigned long)addr, is_hyp_mode_available());
55	58
56	59	/* Should never get here. */
57	60	BUG();

arch/arm/kvm/arm.c+36−30

@@ -53,7 +53,6 @@ __asm__(".arch_extension virt"); · 52 unmodified lines
53	53
54	54	static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
55	55	static kvm_cpu_context_t __percpu *kvm_host_cpu_state;
56		−static unsigned long hyp_default_vectors;
57	56
58	57	/* Per-CPU variable containing the currently running vcpu. */
59	58	static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_arm_running_vcpu);
@@ -227,6 +226,13 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 167 unmodified lines
227	226	else
228	227	r = kvm->arch.vgic.msis_require_devid;
229	228	break;
	229	+ case KVM_CAP_ARM_USER_IRQ:
	230	+ /*
	231	+ * 1: EL1_VTIMER, EL1_PTIMER, and PMU.
	232	+ * (bump this number if adding more devices)
	233	+ */
	234	+ r = 1;
	235	+ break;
230	236	default:
231	237	r = kvm_arch_dev_ioctl_check_extension(kvm, ext);
232	238	break;
@@ -348,15 +354,14 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) · 115 unmodified lines
348	354	vcpu->arch.host_cpu_context = this_cpu_ptr(kvm_host_cpu_state);
349	355
350	356	kvm_arm_set_running_vcpu(vcpu);
	357	+
	358	+ kvm_vgic_load(vcpu);
351	359	}
352	360
353	361	void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
354	362	{
355		− /*
356		− * The arch-generic KVM code expects the cpu field of a vcpu to be -1
357		− * if the vcpu is no longer assigned to a cpu. This is used for the
358		− * optimized make_all_cpus_request path.
359		− */
	363	+ kvm_vgic_put(vcpu);
	364	+
360	365	vcpu->cpu = -1;
361	366
362	367	kvm_arm_set_running_vcpu(NULL);
@@ -514,13 +519,7 @@ static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu) · 151 unmodified lines
514	519	return ret;
515	520	}
516	521
517		− /*
518		− * Enable the arch timers only if we have an in-kernel VGIC
519		− * and it has been properly initialized, since we cannot handle
520		− * interrupts from the virtual timer with a userspace gic.
521		− */
522		− if (irqchip_in_kernel(kvm) && vgic_initialized(kvm))
523		− ret = kvm_timer_enable(vcpu);
	522	+ ret = kvm_timer_enable(vcpu);
524	523
525	524	return ret;
526	525	}
@@ -630,16 +629,23 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu vcpu, struct kvm_run run) · 103 unmodified lines
630	629	* non-preemptible context.
631	630	*/
632	631	preempt_disable();
	632	+
633	633	kvm_pmu_flush_hwstate(vcpu);
	634	+
634	635	kvm_timer_flush_hwstate(vcpu);
635	636	kvm_vgic_flush_hwstate(vcpu);
636	637
637	638	local_irq_disable();
638	639
639	640	/*
640		− * Re-check atomic conditions
	641	+ * If we have a singal pending, or need to notify a userspace
	642	+ * irqchip about timer or PMU level changes, then we exit (and
	643	+ * update the timer level state in kvm_timer_update_run
	644	+ * below).
641	645	*/
642		− if (signal_pending(current)) {
	646	+ if (signal_pending(current) \|\|
	647	+ kvm_timer_should_notify_user(vcpu) \|\|
	648	+ kvm_pmu_should_notify_user(vcpu)) {
643	649	ret = -EINTR;
644	650	run->exit_reason = KVM_EXIT_INTR;
645	651	}
@@ -711,6 +717,12 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu vcpu, struct kvm_run run) · 65 unmodified lines
711	717	ret = handle_exit(vcpu, run, ret);
712	718	}
713	719
	720	+ /* Tell userspace about in-kernel device output levels */
	721	+ if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
	722	+ kvm_timer_update_run(vcpu);
	723	+ kvm_pmu_update_run(vcpu);
	724	+ }
	725	+
714	726	if (vcpu->sigset_active)
715	727	sigprocmask(SIG_SETMASK, &sigsaved, NULL);
716	728	return ret;
@@ -1109,8 +1121,16 @@ static void cpu_init_hyp_mode(void *dummy) · 392 unmodified lines
1109	1121	kvm_arm_init_debug();
1110	1122	}
1111	1123
	1124	+static void cpu_hyp_reset(void)
	1125	+{
	1126	+ if (!is_kernel_in_hyp_mode())
	1127	+ __hyp_reset_vectors();
	1128	+}
	1129	+
1112	1130	static void cpu_hyp_reinit(void)
1113	1131	{
	1132	+ cpu_hyp_reset();
	1133	+
1114	1134	if (is_kernel_in_hyp_mode()) {
1115	1135	/*
1116	1136	* __cpu_init_stage2() is safe to call even if the PM
@@ -1118,18 +1138,10 @@ static void cpu_hyp_reinit(void) · 1 unmodified line
1118	1138	*/
1119	1139	__cpu_init_stage2();
1120	1140	} else {
1121		− if (__hyp_get_vectors() == hyp_default_vectors)
1122		− cpu_init_hyp_mode(NULL);
	1141	+ cpu_init_hyp_mode(NULL);
1123	1142	}
1124	1143	}
1125	1144
1126		−static void cpu_hyp_reset(void)
1127		−{
1128		− if (!is_kernel_in_hyp_mode())
1129		− __cpu_reset_hyp_mode(hyp_default_vectors,
1130		− kvm_get_idmap_start());
1131		−}
1132		−
1133	1145	static void _kvm_arch_hardware_enable(void *discard)
1134	1146	{
1135	1147	if (!__this_cpu_read(kvm_arm_hardware_enabled)) {
@@ -1312,12 +1324,6 @@ static int init_hyp_mode(void) · 176 unmodified lines
1312	1324	if (err)
1313	1325	goto out_err;
1314	1326
1315		− /*
1316		− * It is probably enough to obtain the default on one
1317		− * CPU. It's unlikely to be different on the others.
1318		− */
1319		− hyp_default_vectors = __hyp_get_vectors();
1320		−
1321	1327	/*
1322	1328	* Allocate stack pages for Hypervisor-mode
1323	1329	*/

arch/arm/kvm/coproc.c+21−3

@@ -40,6 +40,24 @@ · 39 unmodified lines
40	40	* Co-processor emulation
41	41	*****************************************************************************/
42	42
	43	+static bool write_to_read_only(struct kvm_vcpu *vcpu,
	44	+ const struct coproc_params *params)
	45	+{
	46	+ WARN_ONCE(1, "CP15 write to read-only register\n");
	47	+ print_cp_instr(params);
	48	+ kvm_inject_undefined(vcpu);
	49	+ return false;
	50	+}
	51	+
	52	+static bool read_from_write_only(struct kvm_vcpu *vcpu,
	53	+ const struct coproc_params *params)
	54	+{
	55	+ WARN_ONCE(1, "CP15 read to write-only register\n");
	56	+ print_cp_instr(params);
	57	+ kvm_inject_undefined(vcpu);
	58	+ return false;
	59	+}
	60	+
43	61	/* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */
44	62	static u32 cache_levels;
45	63
@@ -502,15 +520,15 @@ static int emulate_cp15(struct kvm_vcpu *vcpu, · 456 unmodified lines
502	520	if (likely(r->access(vcpu, params, r))) {
503	521	/* Skip instruction, since it was emulated */
504	522	kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
505		− return 1;
506	523	}
507		− /* If access function fails, it should complain. */
508	524	} else {
	525	+ /* If access function fails, it should complain. */
509	526	kvm_err("Unsupported guest CP15 access at: %08lx\n",
510	527	*vcpu_pc(vcpu));
511	528	print_cp_instr(params);
	529	+ kvm_inject_undefined(vcpu);
512	530	}
513		− kvm_inject_undefined(vcpu);
	531	+
514	532	return 1;
515	533	}
516	534

arch/arm/kvm/coproc.h+0−18

@@ -81,24 +81,6 @@ static inline bool read_zero(struct kvm_vcpu *vcpu, · 80 unmodified lines
81	81	return true;
82	82	}
83	83
84		−static inline bool write_to_read_only(struct kvm_vcpu *vcpu,
85		− const struct coproc_params *params)
86		−{
87		− kvm_debug("CP15 write to read-only register at: %08lx\n",
88		− *vcpu_pc(vcpu));
89		− print_cp_instr(params);
90		− return false;
91		−}
92		−
93		−static inline bool read_from_write_only(struct kvm_vcpu *vcpu,
94		− const struct coproc_params *params)
95		−{
96		− kvm_debug("CP15 read to write-only register at: %08lx\n",
97		− *vcpu_pc(vcpu));
98		− print_cp_instr(params);
99		− return false;
100		−}
101		−
102	84	/* Reset functions */
103	85	static inline void reset_unknown(struct kvm_vcpu *vcpu,
104	86	const struct coproc_reg *r)

arch/arm/kvm/handle_exit.c+8−0

@@ -160,6 +160,14 @@ int handle_exit(struct kvm_vcpu vcpu, struct kvm_run run, · 159 unmodified lines
160	160	case ARM_EXCEPTION_DATA_ABORT:
161	161	kvm_inject_vabt(vcpu);
162	162	return 1;
	163	+ case ARM_EXCEPTION_HYP_GONE:
	164	+ /*
	165	+ * HYP has been reset to the hyp-stub. This happens
	166	+ * when a guest is pre-empted by kvm_reboot()'s
	167	+ * shutdown call.
	168	+ */
	169	+ run->exit_reason = KVM_EXIT_FAIL_ENTRY;
	170	+ return 0;
163	171	default:
164	172	kvm_pr_unimpl("Unsupported exception type: %d",
165	173	exception_index);

arch/arm/kvm/hyp/hyp-entry.S+23−5

@@ -126,11 +126,29 @@ hyp_hvc: · 125 unmodified lines
126	126	*/
127	127	pop {r0, r1, r2}
128	128
129		− /* Check for __hyp_get_vectors */
130		− cmp r0, #-1
131		− mrceq p15, 4, r0, c12, c0, 0 @ get HVBAR
132		− beq 1f
	129	+ /*
	130	+ * Check if we have a kernel function, which is guaranteed to be
	131	+ * bigger than the maximum hyp stub hypercall
	132	+ */
	133	+ cmp r0, #HVC_STUB_HCALL_NR
	134	+ bhs 1f
133	135
	136	+ /*
	137	+ * Not a kernel function, treat it as a stub hypercall.
	138	+ * Compute the physical address for __kvm_handle_stub_hvc
	139	+ * (as the code lives in the idmaped page) and branch there.
	140	+ * We hijack ip (r12) as a tmp register.
	141	+ */
	142	+ push {r1}
	143	+ ldr r1, =kimage_voffset
	144	+ ldr r1, [r1]
	145	+ ldr ip, =__kvm_handle_stub_hvc
	146	+ sub ip, ip, r1
	147	+ pop {r1}
	148	+
	149	+ bx ip
	150	+
	151	+1:
134	152	push {lr}
135	153
136	154	mov lr, r0
@@ -142,7 +160,7 @@ THUMB( orr lr, #1) · 5 unmodified lines
142	160	blx lr @ Call the HYP function
143	161
144	162	pop {lr}
145		−1: eret
	163	+ eret
146	164
147	165	guest_trap:
148	166	load_vcpu r0 @ Load VCPU pointer to r0

arch/arm/kvm/init.S+43−8

@@ -23,6 +23,7 @@ · 22 unmodified lines
23	23	#include <asm/kvm_asm.h>
24	24	#include <asm/kvm_arm.h>
25	25	#include <asm/kvm_mmu.h>
	26	+#include <asm/virt.h>
26	27
27	28	/********************************************************************
28	29	* Hypervisor initialization
@@ -39,6 +40,10 @@ · 10 unmodified lines
39	40	* - Setup the page tables
40	41	* - Enable the MMU
41	42	* - Profit! (or eret, if you only care about the code).
	43	+ *
	44	+ * Another possibility is to get a HYP stub hypercall.
	45	+ * We discriminate between the two by checking if r0 contains a value
	46	+ * that is less than HVC_STUB_HCALL_NR.
42	47	*/
43	48
44	49	.text
@@ -58,6 +63,10 @@ __kvm_hyp_init: · 13 unmodified lines
58	63	W(b) .
59	64
60	65	__do_hyp_init:
	66	+ @ Check for a stub hypercall
	67	+ cmp r0, #HVC_STUB_HCALL_NR
	68	+ blo __kvm_handle_stub_hvc
	69	+
61	70	@ Set stack pointer
62	71	mov sp, r0
63	72
@@ -112,20 +121,46 @@ __do_hyp_init: · 48 unmodified lines
112	121
113	122	eret
114	123
115		− @ r0 : stub vectors address
116		−ENTRY(__kvm_hyp_reset)
	124	+ENTRY(__kvm_handle_stub_hvc)
	125	+ cmp r0, #HVC_SOFT_RESTART
	126	+ bne 1f
	127	+
	128	+ /* The target is expected in r1 */
	129	+ msr ELR_hyp, r1
	130	+ mrs r0, cpsr
	131	+ bic r0, r0, #MODE_MASK
	132	+ orr r0, r0, #HYP_MODE
	133	+THUMB( orr r0, r0, #PSR_T_BIT )
	134	+ msr spsr_cxsf, r0
	135	+ b reset
	136	+
	137	+1: cmp r0, #HVC_RESET_VECTORS
	138	+ bne 1f
	139	+
	140	+reset:
117	141	/* We're now in idmap, disable MMU */
118	142	mrc p15, 4, r1, c1, c0, 0 @ HSCTLR
119		− ldr r2, =(HSCTLR_M \| HSCTLR_A \| HSCTLR_C \| HSCTLR_I)
120		− bic r1, r1, r2
	143	+ ldr r0, =(HSCTLR_M \| HSCTLR_A \| HSCTLR_C \| HSCTLR_I)
	144	+ bic r1, r1, r0
121	145	mcr p15, 4, r1, c1, c0, 0 @ HSCTLR
122	146
123		− /* Install stub vectors */
124		− mcr p15, 4, r0, c12, c0, 0 @ HVBAR
125		− isb
	147	+ /*
	148	+ * Install stub vectors, using ardb's VA->PA trick.
	149	+ */
	150	+0: adr r0, 0b @ PA(0)
	151	+ movw r1, #:lower16:__hyp_stub_vectors - 0b @ VA(stub) - VA(0)
	152	+ movt r1, #:upper16:__hyp_stub_vectors - 0b
	153	+ add r1, r1, r0 @ PA(stub)
	154	+ mcr p15, 4, r1, c12, c0, 0 @ HVBAR
	155	+ b exit
	156	+
	157	+1: ldr r0, =HVC_STUB_ERR
	158	+ eret
126	159
	160	+exit:
	161	+ mov r0, #0
127	162	eret
128		−ENDPROC(__kvm_hyp_reset)
	163	+ENDPROC(__kvm_handle_stub_hvc)
129	164
130	165	.ltorg
131	166

arch/arm/kvm/interrupts.S+0−4

@@ -37,10 +37,6 @@ · 36 unmodified lines
37	37	* in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are
38	38	* passed in r0 (strictly 32bit).
39	39	*
40		− * A function pointer with a value of 0xffffffff has a special meaning,
41		− * and is used to implement __hyp_get_vectors in the same way as in
42		− * arch/arm/kernel/hyp_stub.S.
43		− *
44	40	* The calling convention follows the standard AAPCS:
45	41	* r0 - r3: caller save
46	42	* r12: caller save

arch/arm/kvm/mmu.c+13−23

@@ -1512,7 +1512,8 @@ static int handle_hva_to_gpa(struct kvm *kvm, · 1511 unmodified lines
1512	1512	unsigned long start,
1513	1513	unsigned long end,
1514	1514	int (handler)(struct kvm kvm,
1515		− gpa_t gpa, void *data),
	1515	+ gpa_t gpa, u64 size,
	1516	+ void *data),
1516	1517	void *data)
1517	1518	{
1518	1519	struct kvm_memslots *slots;
@@ -1524,7 +1525,7 @@ static int handle_hva_to_gpa(struct kvm *kvm, · 5 unmodified lines
1524	1525	/* we only care about the pages that the guest sees */
1525	1526	kvm_for_each_memslot(memslot, slots) {
1526	1527	unsigned long hva_start, hva_end;
1527		− gfn_t gfn, gfn_end;
	1528	+ gfn_t gpa;
1528	1529
1529	1530	hva_start = max(start, memslot->userspace_addr);
1530	1531	hva_end = min(end, memslot->userspace_addr +
@@ -1532,25 +1533,16 @@ static int handle_hva_to_gpa(struct kvm *kvm, · 1 unmodified line
1532	1533	if (hva_start >= hva_end)
1533	1534	continue;
1534	1535
1535		− /*
1536		− * {gfn(page) \| page intersects with [hva_start, hva_end)} =
1537		− * {gfn_start, gfn_start+1, ..., gfn_end-1}.
1538		− */
1539		− gfn = hva_to_gfn_memslot(hva_start, memslot);
1540		− gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
1541		−
1542		− for (; gfn < gfn_end; ++gfn) {
1543		− gpa_t gpa = gfn << PAGE_SHIFT;
1544		− ret \|= handler(kvm, gpa, data);
1545		− }
	1536	+ gpa = hva_to_gfn_memslot(hva_start, memslot) << PAGE_SHIFT;
	1537	+ ret \|= handler(kvm, gpa, (u64)(hva_end - hva_start), data);
1546	1538	}
1547	1539
1548	1540	return ret;
1549	1541	}
1550	1542
1551		−static int kvm_unmap_hva_handler(struct kvm kvm, gpa_t gpa, void data)
	1543	+static int kvm_unmap_hva_handler(struct kvm kvm, gpa_t gpa, u64 size, void data)
1552	1544	{
1553		− unmap_stage2_range(kvm, gpa, PAGE_SIZE);
	1545	+ unmap_stage2_range(kvm, gpa, size);
1554	1546	return 0;
1555	1547	}
1556	1548
@@ -1577,10 +1569,11 @@ int kvm_unmap_hva_range(struct kvm *kvm, · 20 unmodified lines
1577	1569	return 0;
1578	1570	}
1579	1571
1580		−static int kvm_set_spte_handler(struct kvm kvm, gpa_t gpa, void data)
	1572	+static int kvm_set_spte_handler(struct kvm kvm, gpa_t gpa, u64 size, void data)
1581	1573	{
1582	1574	pte_t pte = (pte_t )data;
1583	1575
	1576	+ WARN_ON(size != PAGE_SIZE);
1584	1577	/*
1585	1578	* We can always call stage2_set_pte with KVM_S2PTE_FLAG_LOGGING_ACTIVE
1586	1579	* flag clear because MMU notifiers will have unmapped a huge PMD before
@@ -1606,11 +1599,12 @@ void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) · 19 unmodified lines
1606	1599	handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte);
1607	1600	}
1608	1601
1609		−static int kvm_age_hva_handler(struct kvm kvm, gpa_t gpa, void data)
	1602	+static int kvm_age_hva_handler(struct kvm kvm, gpa_t gpa, u64 size, void data)
1610	1603	{
1611	1604	pmd_t *pmd;
1612	1605	pte_t *pte;
1613	1606
	1607	+ WARN_ON(size != PAGE_SIZE && size != PMD_SIZE);
1614	1608	pmd = stage2_get_pmd(kvm, NULL, gpa);
1615	1609	if (!pmd \|\| pmd_none(pmd)) / Nothing there */
1616	1610	return 0;
@@ -1625,11 +1619,12 @@ static int kvm_age_hva_handler(struct kvm kvm, gpa_t gpa, void data) · 8 unmodified lines
1625	1619	return stage2_ptep_test_and_clear_young(pte);
1626	1620	}
1627	1621
1628		−static int kvm_test_age_hva_handler(struct kvm kvm, gpa_t gpa, void data)
	1622	+static int kvm_test_age_hva_handler(struct kvm kvm, gpa_t gpa, u64 size, void data)
1629	1623	{
1630	1624	pmd_t *pmd;
1631	1625	pte_t *pte;
1632	1626
	1627	+ WARN_ON(size != PAGE_SIZE && size != PMD_SIZE);
1633	1628	pmd = stage2_get_pmd(kvm, NULL, gpa);
1634	1629	if (!pmd \|\| pmd_none(pmd)) / Nothing there */
1635	1630	return 0;
@@ -1674,11 +1669,6 @@ phys_addr_t kvm_get_idmap_vector(void) · 38 unmodified lines
1674	1669	return hyp_idmap_vector;
1675	1670	}
1676	1671
1677		−phys_addr_t kvm_get_idmap_start(void)
1678		−{
1679		− return hyp_idmap_start;
1680		−}
1681		−
1682	1672	static int kvm_map_idmap_text(pgd_t *pgd)
1683	1673	{
1684	1674	int err;

arch/arm/kvm/psci.c+7−1

@@ -208,9 +208,10 @@ int kvm_psci_version(struct kvm_vcpu *vcpu) · 207 unmodified lines
208	208
209	209	static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu)
210	210	{
211		− int ret = 1;
	211	+ struct kvm *kvm = vcpu->kvm;
212	212	unsigned long psci_fn = vcpu_get_reg(vcpu, 0) & ~((u32) 0);
213	213	unsigned long val;
	214	+ int ret = 1;
214	215
215	216	switch (psci_fn) {
216	217	case PSCI_0_2_FN_PSCI_VERSION:
@@ -230,7 +231,9 @@ static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu) · 13 unmodified lines
230	231	break;
231	232	case PSCI_0_2_FN_CPU_ON:
232	233	case PSCI_0_2_FN64_CPU_ON:
	234	+ mutex_lock(&kvm->lock);
233	235	val = kvm_psci_vcpu_on(vcpu);
	236	+ mutex_unlock(&kvm->lock);
234	237	break;
235	238	case PSCI_0_2_FN_AFFINITY_INFO:
236	239	case PSCI_0_2_FN64_AFFINITY_INFO:
@@ -279,6 +282,7 @@ static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu) · 42 unmodified lines
279	282
280	283	static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu)
281	284	{
	285	+ struct kvm *kvm = vcpu->kvm;
282	286	unsigned long psci_fn = vcpu_get_reg(vcpu, 0) & ~((u32) 0);
283	287	unsigned long val;
284	288
@@ -288,7 +292,9 @@ static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu) · 3 unmodified lines
288	292	val = PSCI_RET_SUCCESS;
289	293	break;
290	294	case KVM_PSCI_FN_CPU_ON:
	295	+ mutex_lock(&kvm->lock);
291	296	val = kvm_psci_vcpu_on(vcpu);
	297	+ mutex_unlock(&kvm->lock);
292	298	break;
293	299	default:
294	300	val = PSCI_RET_NOT_SUPPORTED;

arch/arm/mm/mmu.c+5−0

@@ -87,6 +87,8 @@ struct cachepolicy { · 86 unmodified lines
87	87	#define s2_policy(policy) 0
88	88	#endif
89	89
	90	+unsigned long kimage_voffset __ro_after_init;
	91	+
90	92	static struct cachepolicy cache_policies[] __initdata = {
91	93	{
92	94	.policy = "uncached",
@@ -1635,4 +1637,7 @@ void __init paging_init(const struct machine_desc *mdesc) · 1542 unmodified lines
1635	1637
1636	1638	empty_zero_page = virt_to_page(zero_page);
1637	1639	__flush_dcache_page(NULL, empty_zero_page);
	1640	+
	1641	+ /* Compute the virt/idmap offset, mostly for the sake of KVM */
	1642	+ kimage_voffset = (unsigned long)&kimage_voffset - virt_to_idmap(&kimage_voffset);
1638	1643	}

arch/arm/mm/proc-v7.S+10−5

@@ -39,13 +39,14 @@ ENTRY(cpu_v7_proc_fin) · 38 unmodified lines
39	39	ENDPROC(cpu_v7_proc_fin)
40	40
41	41	/*
42		− * cpu_v7_reset(loc)
	42	+ * cpu_v7_reset(loc, hyp)
43	43	*
44	44	* Perform a soft reset of the system. Put the CPU into the
45	45	* same state as it would be if it had been reset, and branch
46	46	* to what would be the reset vector.
47	47	*
48	48	* - loc - location to jump to for soft reset
	49	+ * - hyp - indicate if restart occurs in HYP mode
49	50	*
50	51	* This code must be executed using a flat identity mapping with
51	52	* caches disabled.
@@ -53,11 +54,15 @@ ENDPROC(cpu_v7_proc_fin) · 1 unmodified line
53	54	.align 5
54	55	.pushsection .idmap.text, "ax"
55	56	ENTRY(cpu_v7_reset)
56		− mrc p15, 0, r1, c1, c0, 0 @ ctrl register
57		− bic r1, r1, #0x1 @ ...............m
58		− THUMB( bic r1, r1, #1 << 30 ) @ SCTLR.TE (Thumb exceptions)
59		− mcr p15, 0, r1, c1, c0, 0 @ disable MMU
	57	+ mrc p15, 0, r2, c1, c0, 0 @ ctrl register
	58	+ bic r2, r2, #0x1 @ ...............m
	59	+ THUMB( bic r2, r2, #1 << 30 ) @ SCTLR.TE (Thumb exceptions)
	60	+ mcr p15, 0, r2, c1, c0, 0 @ disable MMU
60	61	isb
	62	+#ifdef CONFIG_ARM_VIRT_EXT
	63	+ teq r1, #0
	64	+ bne __hyp_soft_restart
	65	+#endif
61	66	bx r0
62	67	ENDPROC(cpu_v7_reset)
63	68	.popsection

arch/arm64/include/asm/arch_gicv3.h+13−68

@@ -20,69 +20,14 @@ · 19 unmodified lines
20	20
21	21	#include <asm/sysreg.h>
22	22
23		−#define ICC_EOIR1_EL1 sys_reg(3, 0, 12, 12, 1)
24		−#define ICC_DIR_EL1 sys_reg(3, 0, 12, 11, 1)
25		−#define ICC_IAR1_EL1 sys_reg(3, 0, 12, 12, 0)
26		−#define ICC_SGI1R_EL1 sys_reg(3, 0, 12, 11, 5)
27		−#define ICC_PMR_EL1 sys_reg(3, 0, 4, 6, 0)
28		−#define ICC_CTLR_EL1 sys_reg(3, 0, 12, 12, 4)
29		−#define ICC_SRE_EL1 sys_reg(3, 0, 12, 12, 5)
30		−#define ICC_GRPEN1_EL1 sys_reg(3, 0, 12, 12, 7)
31		−#define ICC_BPR1_EL1 sys_reg(3, 0, 12, 12, 3)
32		−
33		−#define ICC_SRE_EL2 sys_reg(3, 4, 12, 9, 5)
34		−
35		−/*
36		− * System register definitions
37		− */
38		−#define ICH_VSEIR_EL2 sys_reg(3, 4, 12, 9, 4)
39		−#define ICH_HCR_EL2 sys_reg(3, 4, 12, 11, 0)
40		−#define ICH_VTR_EL2 sys_reg(3, 4, 12, 11, 1)
41		−#define ICH_MISR_EL2 sys_reg(3, 4, 12, 11, 2)
42		−#define ICH_EISR_EL2 sys_reg(3, 4, 12, 11, 3)
43		−#define ICH_ELSR_EL2 sys_reg(3, 4, 12, 11, 5)
44		−#define ICH_VMCR_EL2 sys_reg(3, 4, 12, 11, 7)
45		−
46		−#define __LR0_EL2(x) sys_reg(3, 4, 12, 12, x)
47		−#define __LR8_EL2(x) sys_reg(3, 4, 12, 13, x)
48		−
49		−#define ICH_LR0_EL2 __LR0_EL2(0)
50		−#define ICH_LR1_EL2 __LR0_EL2(1)
51		−#define ICH_LR2_EL2 __LR0_EL2(2)
52		−#define ICH_LR3_EL2 __LR0_EL2(3)
53		−#define ICH_LR4_EL2 __LR0_EL2(4)
54		−#define ICH_LR5_EL2 __LR0_EL2(5)
55		−#define ICH_LR6_EL2 __LR0_EL2(6)
56		−#define ICH_LR7_EL2 __LR0_EL2(7)
57		−#define ICH_LR8_EL2 __LR8_EL2(0)
58		−#define ICH_LR9_EL2 __LR8_EL2(1)
59		−#define ICH_LR10_EL2 __LR8_EL2(2)
60		−#define ICH_LR11_EL2 __LR8_EL2(3)
61		−#define ICH_LR12_EL2 __LR8_EL2(4)
62		−#define ICH_LR13_EL2 __LR8_EL2(5)
63		−#define ICH_LR14_EL2 __LR8_EL2(6)
64		−#define ICH_LR15_EL2 __LR8_EL2(7)
65		−
66		−#define __AP0Rx_EL2(x) sys_reg(3, 4, 12, 8, x)
67		−#define ICH_AP0R0_EL2 __AP0Rx_EL2(0)
68		−#define ICH_AP0R1_EL2 __AP0Rx_EL2(1)
69		−#define ICH_AP0R2_EL2 __AP0Rx_EL2(2)
70		−#define ICH_AP0R3_EL2 __AP0Rx_EL2(3)
71		−
72		−#define __AP1Rx_EL2(x) sys_reg(3, 4, 12, 9, x)
73		−#define ICH_AP1R0_EL2 __AP1Rx_EL2(0)
74		−#define ICH_AP1R1_EL2 __AP1Rx_EL2(1)
75		−#define ICH_AP1R2_EL2 __AP1Rx_EL2(2)
76		−#define ICH_AP1R3_EL2 __AP1Rx_EL2(3)
77		−
78	23	#ifndef __ASSEMBLY__
79	24
80	25	#include <linux/stringify.h>
81	26	#include <asm/barrier.h>
82	27	#include <asm/cacheflush.h>
83	28
84		−#define read_gicreg read_sysreg_s
85		−#define write_gicreg write_sysreg_s
	29	+#define read_gicreg(r) read_sysreg_s(SYS_ ## r)
	30	+#define write_gicreg(v, r) write_sysreg_s(v, SYS_ ## r)
86	31
87	32	/*
88	33	* Low-level accessors
@@ -93,13 +38,13 @@ · 4 unmodified lines
93	38
94	39	static inline void gic_write_eoir(u32 irq)
95	40	{
96		− write_sysreg_s(irq, ICC_EOIR1_EL1);
	41	+ write_sysreg_s(irq, SYS_ICC_EOIR1_EL1);
97	42	isb();
98	43	}
99	44
100	45	static inline void gic_write_dir(u32 irq)
101	46	{
102		− write_sysreg_s(irq, ICC_DIR_EL1);
	47	+ write_sysreg_s(irq, SYS_ICC_DIR_EL1);
103	48	isb();
104	49	}
105	50
@@ -107,7 +52,7 @@ static inline u64 gic_read_iar_common(void) · 1 unmodified line
107	52	{
108	53	u64 irqstat;
109	54
110		− irqstat = read_sysreg_s(ICC_IAR1_EL1);
	55	+ irqstat = read_sysreg_s(SYS_ICC_IAR1_EL1);
111	56	dsb(sy);
112	57	return irqstat;
113	58	}
@@ -124,7 +69,7 @@ static inline u64 gic_read_iar_cavium_thunderx(void) · 10 unmodified lines
124	69	u64 irqstat;
125	70
126	71	nops(8);
127		− irqstat = read_sysreg_s(ICC_IAR1_EL1);
	72	+ irqstat = read_sysreg_s(SYS_ICC_IAR1_EL1);
128	73	nops(4);
129	74	mb();
130	75
@@ -133,40 +78,40 @@ static inline u64 gic_read_iar_cavium_thunderx(void) · 2 unmodified lines
133	78
134	79	static inline void gic_write_pmr(u32 val)
135	80	{
136		− write_sysreg_s(val, ICC_PMR_EL1);
	81	+ write_sysreg_s(val, SYS_ICC_PMR_EL1);
137	82	}
138	83
139	84	static inline void gic_write_ctlr(u32 val)
140	85	{
141		− write_sysreg_s(val, ICC_CTLR_EL1);
	86	+ write_sysreg_s(val, SYS_ICC_CTLR_EL1);
142	87	isb();
143	88	}
144	89
145	90	static inline void gic_write_grpen1(u32 val)
146	91	{
147		− write_sysreg_s(val, ICC_GRPEN1_EL1);
	92	+ write_sysreg_s(val, SYS_ICC_GRPEN1_EL1);
148	93	isb();
149	94	}
150	95
151	96	static inline void gic_write_sgi1r(u64 val)
152	97	{
153		− write_sysreg_s(val, ICC_SGI1R_EL1);
	98	+ write_sysreg_s(val, SYS_ICC_SGI1R_EL1);
154	99	}
155	100
156	101	static inline u32 gic_read_sre(void)
157	102	{
158		− return read_sysreg_s(ICC_SRE_EL1);
	103	+ return read_sysreg_s(SYS_ICC_SRE_EL1);
159	104	}
160	105
161	106	static inline void gic_write_sre(u32 val)
162	107	{
163		− write_sysreg_s(val, ICC_SRE_EL1);
	108	+ write_sysreg_s(val, SYS_ICC_SRE_EL1);
164	109	isb();
165	110	}
166	111
167	112	static inline void gic_write_bpr1(u32 val)
168	113	{
169		− asm volatile("msr_s " __stringify(ICC_BPR1_EL1) ", %0" : : "r" (val));
	114	+ write_sysreg_s(val, SYS_ICC_BPR1_EL1);
170	115	}
171	116
172	117	#define gic_read_typer(c) readq_relaxed(c)

arch/arm64/include/asm/kvm_asm.h+3−2

@@ -28,7 +28,7 @@ · 27 unmodified lines
28	28	#define ARM_EXCEPTION_EL1_SERROR 1
29	29	#define ARM_EXCEPTION_TRAP 2
30	30	/* The hyp-stub will return this for any kvm_call_hyp() call */
31		−#define ARM_EXCEPTION_HYP_GONE 3
	31	+#define ARM_EXCEPTION_HYP_GONE HVC_STUB_ERR
32	32
33	33	#define KVM_ARM64_DEBUG_DIRTY_SHIFT 0
34	34	#define KVM_ARM64_DEBUG_DIRTY (1 << KVM_ARM64_DEBUG_DIRTY_SHIFT)
@@ -47,7 +47,6 @@ struct kvm_vcpu; · 12 unmodified lines
47	47
48	48	extern char __kvm_hyp_init[];
49	49	extern char __kvm_hyp_init_end[];
50		−extern char __kvm_hyp_reset[];
51	50
52	51	extern char __kvm_hyp_vector[];
53	52
@@ -59,6 +58,8 @@ extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu); · 5 unmodified lines
59	58	extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
60	59
61	60	extern u64 __vgic_v3_get_ich_vtr_el2(void);
	61	+extern u64 __vgic_v3_read_vmcr(void);
	62	+extern void __vgic_v3_write_vmcr(u32 vmcr);
62	63	extern void __vgic_v3_init_lrs(void);
63	64
64	65	extern u32 __kvm_get_mdcr_el2(void);

arch/arm64/include/asm/kvm_host.h+0−7

@@ -361,13 +361,6 @@ static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr, · 360 unmodified lines
361	361	__kvm_call_hyp((void *)pgd_ptr, hyp_stack_ptr, vector_ptr);
362	362	}
363	363
364		−void __kvm_hyp_teardown(void);
365		−static inline void __cpu_reset_hyp_mode(unsigned long vector_ptr,
366		− phys_addr_t phys_idmap_start)
367		−{
368		− kvm_call_hyp(__kvm_hyp_teardown, phys_idmap_start);
369		−}
370		−
371	364	static inline void kvm_arch_hardware_unsetup(void) {}
372	365	static inline void kvm_arch_sync_events(struct kvm *kvm) {}
373	366	static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}

arch/arm64/include/asm/kvm_mmu.h+0−1

@@ -155,7 +155,6 @@ void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu); · 154 unmodified lines
155	155
156	156	phys_addr_t kvm_mmu_get_httbr(void);
157	157	phys_addr_t kvm_get_idmap_vector(void);
158		−phys_addr_t kvm_get_idmap_start(void);
159	158	int kvm_mmu_init(void);
160	159	void kvm_clear_hyp_idmap(void);
161	160

arch/arm64/include/asm/sysreg.h+155−7

@@ -48,6 +48,8 @@ · 47 unmodified lines
48	48	((crn) << CRn_shift) \| ((crm) << CRm_shift) \| \
49	49	((op2) << Op2_shift))
50	50
	51	+#define sys_insn sys_reg
	52	+
51	53	#define sys_reg_Op0(id) (((id) >> Op0_shift) & Op0_mask)
52	54	#define sys_reg_Op1(id) (((id) >> Op1_shift) & Op1_mask)
53	55	#define sys_reg_CRn(id) (((id) >> CRn_shift) & CRn_mask)
@@ -81,6 +83,41 @@ · 27 unmodified lines
81	83
82	84	#endif /* CONFIG_BROKEN_GAS_INST */
83	85
	86	+#define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4)
	87	+#define REG_PSTATE_UAO_IMM sys_reg(0, 0, 4, 0, 3)
	88	+
	89	+#define SET_PSTATE_PAN(x) __emit_inst(0xd5000000 \| REG_PSTATE_PAN_IMM \| \
	90	+ (!!x)<<8 \| 0x1f)
	91	+#define SET_PSTATE_UAO(x) __emit_inst(0xd5000000 \| REG_PSTATE_UAO_IMM \| \
	92	+ (!!x)<<8 \| 0x1f)
	93	+
	94	+#define SYS_DC_ISW sys_insn(1, 0, 7, 6, 2)
	95	+#define SYS_DC_CSW sys_insn(1, 0, 7, 10, 2)
	96	+#define SYS_DC_CISW sys_insn(1, 0, 7, 14, 2)
	97	+
	98	+#define SYS_OSDTRRX_EL1 sys_reg(2, 0, 0, 0, 2)
	99	+#define SYS_MDCCINT_EL1 sys_reg(2, 0, 0, 2, 0)
	100	+#define SYS_MDSCR_EL1 sys_reg(2, 0, 0, 2, 2)
	101	+#define SYS_OSDTRTX_EL1 sys_reg(2, 0, 0, 3, 2)
	102	+#define SYS_OSECCR_EL1 sys_reg(2, 0, 0, 6, 2)
	103	+#define SYS_DBGBVRn_EL1(n) sys_reg(2, 0, 0, n, 4)
	104	+#define SYS_DBGBCRn_EL1(n) sys_reg(2, 0, 0, n, 5)
	105	+#define SYS_DBGWVRn_EL1(n) sys_reg(2, 0, 0, n, 6)
	106	+#define SYS_DBGWCRn_EL1(n) sys_reg(2, 0, 0, n, 7)
	107	+#define SYS_MDRAR_EL1 sys_reg(2, 0, 1, 0, 0)
	108	+#define SYS_OSLAR_EL1 sys_reg(2, 0, 1, 0, 4)
	109	+#define SYS_OSLSR_EL1 sys_reg(2, 0, 1, 1, 4)
	110	+#define SYS_OSDLR_EL1 sys_reg(2, 0, 1, 3, 4)
	111	+#define SYS_DBGPRCR_EL1 sys_reg(2, 0, 1, 4, 4)
	112	+#define SYS_DBGCLAIMSET_EL1 sys_reg(2, 0, 7, 8, 6)
	113	+#define SYS_DBGCLAIMCLR_EL1 sys_reg(2, 0, 7, 9, 6)
	114	+#define SYS_DBGAUTHSTATUS_EL1 sys_reg(2, 0, 7, 14, 6)
	115	+#define SYS_MDCCSR_EL0 sys_reg(2, 3, 0, 1, 0)
	116	+#define SYS_DBGDTR_EL0 sys_reg(2, 3, 0, 4, 0)
	117	+#define SYS_DBGDTRRX_EL0 sys_reg(2, 3, 0, 5, 0)
	118	+#define SYS_DBGDTRTX_EL0 sys_reg(2, 3, 0, 5, 0)
	119	+#define SYS_DBGVCR32_EL2 sys_reg(2, 4, 0, 7, 0)
	120	+
84	121	#define SYS_MIDR_EL1 sys_reg(3, 0, 0, 0, 0)
85	122	#define SYS_MPIDR_EL1 sys_reg(3, 0, 0, 0, 5)
86	123	#define SYS_REVIDR_EL1 sys_reg(3, 0, 0, 0, 6)
@@ -88,6 +125,7 @@ · 1 unmodified line
88	125	#define SYS_ID_PFR0_EL1 sys_reg(3, 0, 0, 1, 0)
89	126	#define SYS_ID_PFR1_EL1 sys_reg(3, 0, 0, 1, 1)
90	127	#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
	128	+#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
91	129	#define SYS_ID_MMFR0_EL1 sys_reg(3, 0, 0, 1, 4)
92	130	#define SYS_ID_MMFR1_EL1 sys_reg(3, 0, 0, 1, 5)
93	131	#define SYS_ID_MMFR2_EL1 sys_reg(3, 0, 0, 1, 6)
@@ -118,17 +156,127 @@ · 24 unmodified lines
118	156	#define SYS_ID_AA64MMFR1_EL1 sys_reg(3, 0, 0, 7, 1)
119	157	#define SYS_ID_AA64MMFR2_EL1 sys_reg(3, 0, 0, 7, 2)
120	158
121		−#define SYS_CNTFRQ_EL0 sys_reg(3, 3, 14, 0, 0)
	159	+#define SYS_SCTLR_EL1 sys_reg(3, 0, 1, 0, 0)
	160	+#define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1)
	161	+#define SYS_CPACR_EL1 sys_reg(3, 0, 1, 0, 2)
	162	+
	163	+#define SYS_TTBR0_EL1 sys_reg(3, 0, 2, 0, 0)
	164	+#define SYS_TTBR1_EL1 sys_reg(3, 0, 2, 0, 1)
	165	+#define SYS_TCR_EL1 sys_reg(3, 0, 2, 0, 2)
	166	+
	167	+#define SYS_ICC_PMR_EL1 sys_reg(3, 0, 4, 6, 0)
	168	+
	169	+#define SYS_AFSR0_EL1 sys_reg(3, 0, 5, 1, 0)
	170	+#define SYS_AFSR1_EL1 sys_reg(3, 0, 5, 1, 1)
	171	+#define SYS_ESR_EL1 sys_reg(3, 0, 5, 2, 0)
	172	+#define SYS_FAR_EL1 sys_reg(3, 0, 6, 0, 0)
	173	+#define SYS_PAR_EL1 sys_reg(3, 0, 7, 4, 0)
	174	+
	175	+#define SYS_PMINTENSET_EL1 sys_reg(3, 0, 9, 14, 1)
	176	+#define SYS_PMINTENCLR_EL1 sys_reg(3, 0, 9, 14, 2)
	177	+
	178	+#define SYS_MAIR_EL1 sys_reg(3, 0, 10, 2, 0)
	179	+#define SYS_AMAIR_EL1 sys_reg(3, 0, 10, 3, 0)
	180	+
	181	+#define SYS_VBAR_EL1 sys_reg(3, 0, 12, 0, 0)
	182	+
	183	+#define SYS_ICC_DIR_EL1 sys_reg(3, 0, 12, 11, 1)
	184	+#define SYS_ICC_SGI1R_EL1 sys_reg(3, 0, 12, 11, 5)
	185	+#define SYS_ICC_IAR1_EL1 sys_reg(3, 0, 12, 12, 0)
	186	+#define SYS_ICC_EOIR1_EL1 sys_reg(3, 0, 12, 12, 1)
	187	+#define SYS_ICC_BPR1_EL1 sys_reg(3, 0, 12, 12, 3)
	188	+#define SYS_ICC_CTLR_EL1 sys_reg(3, 0, 12, 12, 4)
	189	+#define SYS_ICC_SRE_EL1 sys_reg(3, 0, 12, 12, 5)
	190	+#define SYS_ICC_GRPEN1_EL1 sys_reg(3, 0, 12, 12, 7)
	191	+
	192	+#define SYS_CONTEXTIDR_EL1 sys_reg(3, 0, 13, 0, 1)
	193	+#define SYS_TPIDR_EL1 sys_reg(3, 0, 13, 0, 4)
	194	+
	195	+#define SYS_CNTKCTL_EL1 sys_reg(3, 0, 14, 1, 0)
	196	+
	197	+#define SYS_CLIDR_EL1 sys_reg(3, 1, 0, 0, 1)
	198	+#define SYS_AIDR_EL1 sys_reg(3, 1, 0, 0, 7)
	199	+
	200	+#define SYS_CSSELR_EL1 sys_reg(3, 2, 0, 0, 0)
	201	+
122	202	#define SYS_CTR_EL0 sys_reg(3, 3, 0, 0, 1)
123	203	#define SYS_DCZID_EL0 sys_reg(3, 3, 0, 0, 7)
124	204
125		−#define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4)
126		−#define REG_PSTATE_UAO_IMM sys_reg(0, 0, 4, 0, 3)
	205	+#define SYS_PMCR_EL0 sys_reg(3, 3, 9, 12, 0)
	206	+#define SYS_PMCNTENSET_EL0 sys_reg(3, 3, 9, 12, 1)
	207	+#define SYS_PMCNTENCLR_EL0 sys_reg(3, 3, 9, 12, 2)
	208	+#define SYS_PMOVSCLR_EL0 sys_reg(3, 3, 9, 12, 3)
	209	+#define SYS_PMSWINC_EL0 sys_reg(3, 3, 9, 12, 4)
	210	+#define SYS_PMSELR_EL0 sys_reg(3, 3, 9, 12, 5)
	211	+#define SYS_PMCEID0_EL0 sys_reg(3, 3, 9, 12, 6)
	212	+#define SYS_PMCEID1_EL0 sys_reg(3, 3, 9, 12, 7)
	213	+#define SYS_PMCCNTR_EL0 sys_reg(3, 3, 9, 13, 0)
	214	+#define SYS_PMXEVTYPER_EL0 sys_reg(3, 3, 9, 13, 1)
	215	+#define SYS_PMXEVCNTR_EL0 sys_reg(3, 3, 9, 13, 2)
	216	+#define SYS_PMUSERENR_EL0 sys_reg(3, 3, 9, 14, 0)
	217	+#define SYS_PMOVSSET_EL0 sys_reg(3, 3, 9, 14, 3)
	218	+
	219	+#define SYS_TPIDR_EL0 sys_reg(3, 3, 13, 0, 2)
	220	+#define SYS_TPIDRRO_EL0 sys_reg(3, 3, 13, 0, 3)
127	221
128		−#define SET_PSTATE_PAN(x) __emit_inst(0xd5000000 \| REG_PSTATE_PAN_IMM \| \
129		− (!!x)<<8 \| 0x1f)
130		−#define SET_PSTATE_UAO(x) __emit_inst(0xd5000000 \| REG_PSTATE_UAO_IMM \| \
131		− (!!x)<<8 \| 0x1f)
	222	+#define SYS_CNTFRQ_EL0 sys_reg(3, 3, 14, 0, 0)
	223	+
	224	+#define SYS_CNTP_TVAL_EL0 sys_reg(3, 3, 14, 2, 0)
	225	+#define SYS_CNTP_CTL_EL0 sys_reg(3, 3, 14, 2, 1)
	226	+#define SYS_CNTP_CVAL_EL0 sys_reg(3, 3, 14, 2, 2)
	227	+
	228	+#define __PMEV_op2(n) ((n) & 0x7)
	229	+#define __CNTR_CRm(n) (0x8 \| (((n) >> 3) & 0x3))
	230	+#define SYS_PMEVCNTRn_EL0(n) sys_reg(3, 3, 14, __CNTR_CRm(n), __PMEV_op2(n))
	231	+#define __TYPER_CRm(n) (0xc \| (((n) >> 3) & 0x3))
	232	+#define SYS_PMEVTYPERn_EL0(n) sys_reg(3, 3, 14, __TYPER_CRm(n), __PMEV_op2(n))
	233	+
	234	+#define SYS_PMCCFILTR_EL0 sys_reg (3, 3, 14, 15, 7)
	235	+
	236	+#define SYS_DACR32_EL2 sys_reg(3, 4, 3, 0, 0)
	237	+#define SYS_IFSR32_EL2 sys_reg(3, 4, 5, 0, 1)
	238	+#define SYS_FPEXC32_EL2 sys_reg(3, 4, 5, 3, 0)
	239	+
	240	+#define __SYS__AP0Rx_EL2(x) sys_reg(3, 4, 12, 8, x)
	241	+#define SYS_ICH_AP0R0_EL2 __SYS__AP0Rx_EL2(0)
	242	+#define SYS_ICH_AP0R1_EL2 __SYS__AP0Rx_EL2(1)
	243	+#define SYS_ICH_AP0R2_EL2 __SYS__AP0Rx_EL2(2)
	244	+#define SYS_ICH_AP0R3_EL2 __SYS__AP0Rx_EL2(3)
	245	+
	246	+#define __SYS__AP1Rx_EL2(x) sys_reg(3, 4, 12, 9, x)
	247	+#define SYS_ICH_AP1R0_EL2 __SYS__AP1Rx_EL2(0)
	248	+#define SYS_ICH_AP1R1_EL2 __SYS__AP1Rx_EL2(1)
	249	+#define SYS_ICH_AP1R2_EL2 __SYS__AP1Rx_EL2(2)
	250	+#define SYS_ICH_AP1R3_EL2 __SYS__AP1Rx_EL2(3)
	251	+
	252	+#define SYS_ICH_VSEIR_EL2 sys_reg(3, 4, 12, 9, 4)
	253	+#define SYS_ICC_SRE_EL2 sys_reg(3, 4, 12, 9, 5)
	254	+#define SYS_ICH_HCR_EL2 sys_reg(3, 4, 12, 11, 0)
	255	+#define SYS_ICH_VTR_EL2 sys_reg(3, 4, 12, 11, 1)
	256	+#define SYS_ICH_MISR_EL2 sys_reg(3, 4, 12, 11, 2)
	257	+#define SYS_ICH_EISR_EL2 sys_reg(3, 4, 12, 11, 3)
	258	+#define SYS_ICH_ELSR_EL2 sys_reg(3, 4, 12, 11, 5)
	259	+#define SYS_ICH_VMCR_EL2 sys_reg(3, 4, 12, 11, 7)
	260	+
	261	+#define __SYS__LR0_EL2(x) sys_reg(3, 4, 12, 12, x)
	262	+#define SYS_ICH_LR0_EL2 __SYS__LR0_EL2(0)
	263	+#define SYS_ICH_LR1_EL2 __SYS__LR0_EL2(1)
	264	+#define SYS_ICH_LR2_EL2 __SYS__LR0_EL2(2)
	265	+#define SYS_ICH_LR3_EL2 __SYS__LR0_EL2(3)
	266	+#define SYS_ICH_LR4_EL2 __SYS__LR0_EL2(4)
	267	+#define SYS_ICH_LR5_EL2 __SYS__LR0_EL2(5)
	268	+#define SYS_ICH_LR6_EL2 __SYS__LR0_EL2(6)
	269	+#define SYS_ICH_LR7_EL2 __SYS__LR0_EL2(7)
	270	+
	271	+#define __SYS__LR8_EL2(x) sys_reg(3, 4, 12, 13, x)
	272	+#define SYS_ICH_LR8_EL2 __SYS__LR8_EL2(0)
	273	+#define SYS_ICH_LR9_EL2 __SYS__LR8_EL2(1)
	274	+#define SYS_ICH_LR10_EL2 __SYS__LR8_EL2(2)
	275	+#define SYS_ICH_LR11_EL2 __SYS__LR8_EL2(3)
	276	+#define SYS_ICH_LR12_EL2 __SYS__LR8_EL2(4)
	277	+#define SYS_ICH_LR13_EL2 __SYS__LR8_EL2(5)
	278	+#define SYS_ICH_LR14_EL2 __SYS__LR8_EL2(6)
	279	+#define SYS_ICH_LR15_EL2 __SYS__LR8_EL2(7)
132	280
133	281	/* Common SCTLR_ELx flags. */
134	282	#define SCTLR_ELx_EE (1 << 25)

arch/arm64/include/asm/virt.h+22−9

@@ -19,25 +19,38 @@ · 18 unmodified lines
19	19	#define __ASM__VIRT_H
20	20
21	21	/*
22		− * The arm64 hcall implementation uses x0 to specify the hcall type. A value
23		− * less than 0xfff indicates a special hcall, such as get/set vector.
24		− * Any other value is used as a pointer to the function to call.
	22	+ * The arm64 hcall implementation uses x0 to specify the hcall
	23	+ * number. A value less than HVC_STUB_HCALL_NR indicates a special
	24	+ * hcall, such as set vector. Any other value is handled in a
	25	+ * hypervisor specific way.
	26	+ *
	27	+ * The hypercall is allowed to clobber any of the caller-saved
	28	+ * registers (x0-x18), so it is advisable to use it through the
	29	+ * indirection of a function call (as implemented in hyp-stub.S).
25	30	*/
26	31
27		−/* HVC_GET_VECTORS - Return the value of the vbar_el2 register. */
28		−#define HVC_GET_VECTORS 0
29		−
30	32	/*
31	33	* HVC_SET_VECTORS - Set the value of the vbar_el2 register.
32	34	*
33	35	* @x1: Physical address of the new vector table.
34	36	*/
35		−#define HVC_SET_VECTORS 1
	37	+#define HVC_SET_VECTORS 0
36	38
37	39	/*
38	40	* HVC_SOFT_RESTART - CPU soft reset, used by the cpu_soft_restart routine.
39	41	*/
40		−#define HVC_SOFT_RESTART 2
	42	+#define HVC_SOFT_RESTART 1
	43	+
	44	+/*
	45	+ * HVC_RESET_VECTORS - Restore the vectors to the original HYP stubs
	46	+ */
	47	+#define HVC_RESET_VECTORS 2
	48	+
	49	+/* Max number of HYP stub hypercalls */
	50	+#define HVC_STUB_HCALL_NR 3
	51	+
	52	+/* Error returned when an invalid stub number is passed into x0 */
	53	+#define HVC_STUB_ERR 0xbadca11
41	54
42	55	#define BOOT_CPU_MODE_EL1 (0xe11)
43	56	#define BOOT_CPU_MODE_EL2 (0xe12)
@@ -61,7 +74,7 @@ · 17 unmodified lines
61	74	extern u32 __boot_cpu_mode[2];
62	75
63	76	void __hyp_set_vectors(phys_addr_t phys_vector_base);
64		−phys_addr_t __hyp_get_vectors(void);
	77	+void __hyp_reset_vectors(void);
65	78
66	79	/* Reports the availability of HYP mode */
67	80	static inline bool is_hyp_mode_available(void)

arch/arm64/include/uapi/asm/kvm.h+2−0

@@ -145,6 +145,8 @@ struct kvm_debug_exit_arch { · 144 unmodified lines
145	145	#define KVM_GUESTDBG_USE_HW (1 << 17)
146	146
147	147	struct kvm_sync_regs {
	148	+ /* Used with KVM_CAP_ARM_USER_IRQ */
	149	+ __u64 device_irq_level;
148	150	};
149	151
150	152	struct kvm_arch_memory_slot {

arch/arm64/kernel/head.S+4−4

@@ -594,14 +594,14 @@ set_hcr: · 593 unmodified lines
594	594	cmp x0, #1
595	595	b.ne 3f
596	596
597		− mrs_s x0, ICC_SRE_EL2
	597	+ mrs_s x0, SYS_ICC_SRE_EL2
598	598	orr x0, x0, #ICC_SRE_EL2_SRE // Set ICC_SRE_EL2.SRE==1
599	599	orr x0, x0, #ICC_SRE_EL2_ENABLE // Set ICC_SRE_EL2.Enable==1
600		− msr_s ICC_SRE_EL2, x0
	600	+ msr_s SYS_ICC_SRE_EL2, x0
601	601	isb // Make sure SRE is now set
602		− mrs_s x0, ICC_SRE_EL2 // Read SRE back,
	602	+ mrs_s x0, SYS_ICC_SRE_EL2 // Read SRE back,
603	603	tbz x0, #0, 3f // and check that it sticks
604		− msr_s ICH_HCR_EL2, xzr // Reset ICC_HCR_EL2 to defaults
	604	+ msr_s SYS_ICH_HCR_EL2, xzr // Reset ICC_HCR_EL2 to defaults
605	605
606	606	3:
607	607	#endif

arch/arm64/kernel/hyp-stub.S+14−24

@@ -55,18 +55,7 @@ ENDPROC(__hyp_stub_vectors) · 54 unmodified lines
55	55	.align 11
56	56
57	57	el1_sync:
58		− mrs x30, esr_el2
59		− lsr x30, x30, #ESR_ELx_EC_SHIFT
60		−
61		− cmp x30, #ESR_ELx_EC_HVC64
62		− b.ne 9f // Not an HVC trap
63		−
64		− cmp x0, #HVC_GET_VECTORS
65		− b.ne 1f
66		− mrs x0, vbar_el2
67		− b 9f
68		−
69		−1: cmp x0, #HVC_SET_VECTORS
	58	+ cmp x0, #HVC_SET_VECTORS
70	59	b.ne 2f
71	60	msr vbar_el2, x1
72	61	b 9f
@@ -79,10 +68,15 @@ el1_sync: · 6 unmodified lines
79	68	mov x1, x3
80	69	br x4 // no return
81	70
	71	+3: cmp x0, #HVC_RESET_VECTORS
	72	+ beq 9f // Nothing to reset!
	73	+
82	74	/* Someone called kvm_call_hyp() against the hyp-stub... */
83		−3: mov x0, #ARM_EXCEPTION_HYP_GONE
	75	+ ldr x0, =HVC_STUB_ERR
	76	+ eret
84	77
85		−9: eret
	78	+9: mov x0, xzr
	79	+ eret
86	80	ENDPROC(el1_sync)
87	81
88	82	.macro invalid_vector label
@@ -121,19 +115,15 @@ ENDPROC(\label) · 32 unmodified lines
121	115	* initialisation entry point.
122	116	*/
123	117
124		−ENTRY(__hyp_get_vectors)
125		− str lr, [sp, #-16]!
126		− mov x0, #HVC_GET_VECTORS
127		− hvc #0
128		− ldr lr, [sp], #16
129		− ret
130		−ENDPROC(__hyp_get_vectors)
131		−
132	118	ENTRY(__hyp_set_vectors)
133		− str lr, [sp, #-16]!
134	119	mov x1, x0
135	120	mov x0, #HVC_SET_VECTORS
136	121	hvc #0
137		− ldr lr, [sp], #16
138	122	ret
139	123	ENDPROC(__hyp_set_vectors)
	124	+
	125	+ENTRY(__hyp_reset_vectors)
	126	+ mov x0, #HVC_RESET_VECTORS
	127	+ hvc #0
	128	+ ret
	129	+ENDPROC(__hyp_reset_vectors)

arch/arm64/kvm/hyp-init.S+36−10

@@ -22,6 +22,7 @@ · 21 unmodified lines
22	22	#include <asm/kvm_mmu.h>
23	23	#include <asm/pgtable-hwdef.h>
24	24	#include <asm/sysreg.h>
	25	+#include <asm/virt.h>
25	26
26	27	.text
27	28	.pushsection .hyp.idmap.text, "ax"
@@ -58,6 +59,9 @@ __invalid: · 30 unmodified lines
58	59	* x2: HYP vectors
59	60	*/
60	61	__do_hyp_init:
	62	+ /* Check for a stub HVC call */
	63	+ cmp x0, #HVC_STUB_HCALL_NR
	64	+ b.lo __kvm_handle_stub_hvc
61	65
62	66	msr ttbr0_el2, x0
63	67
@@ -119,23 +123,45 @@ __do_hyp_init: · 55 unmodified lines
119	123	eret
120	124	ENDPROC(__kvm_hyp_init)
121	125
	126	+ENTRY(__kvm_handle_stub_hvc)
	127	+ cmp x0, #HVC_SOFT_RESTART
	128	+ b.ne 1f
	129	+
	130	+ /* This is where we're about to jump, staying at EL2 */
	131	+ msr elr_el2, x1
	132	+ mov x0, #(PSR_F_BIT \| PSR_I_BIT \| PSR_A_BIT \| PSR_D_BIT \| PSR_MODE_EL2h)
	133	+ msr spsr_el2, x0
	134	+
	135	+ /* Shuffle the arguments, and don't come back */
	136	+ mov x0, x2
	137	+ mov x1, x3
	138	+ mov x2, x4
	139	+ b reset
	140	+
	141	+1: cmp x0, #HVC_RESET_VECTORS
	142	+ b.ne 1f
	143	+reset:
122	144	/*
123		− * Reset kvm back to the hyp stub.
	145	+ * Reset kvm back to the hyp stub. Do not clobber x0-x4 in
	146	+ * case we coming via HVC_SOFT_RESTART.
124	147	*/
125		−ENTRY(__kvm_hyp_reset)
126		− /* We're now in idmap, disable MMU */
127		− mrs x0, sctlr_el2
128		− ldr x1, =SCTLR_ELx_FLAGS
129		− bic x0, x0, x1 // Clear SCTL_M and etc
130		− msr sctlr_el2, x0
	148	+ mrs x5, sctlr_el2
	149	+ ldr x6, =SCTLR_ELx_FLAGS
	150	+ bic x5, x5, x6 // Clear SCTL_M and etc
	151	+ msr sctlr_el2, x5
131	152	isb
132	153
133	154	/* Install stub vectors */
134		− adr_l x0, __hyp_stub_vectors
135		− msr vbar_el2, x0
	155	+ adr_l x5, __hyp_stub_vectors
	156	+ msr vbar_el2, x5
	157	+ mov x0, xzr
	158	+ eret
136	159
	160	+1: /* Bad stub call */
	161	+ ldr x0, =HVC_STUB_ERR
137	162	eret
138		−ENDPROC(__kvm_hyp_reset)
	163	+
	164	+ENDPROC(__kvm_handle_stub_hvc)
139	165
140	166	.ltorg
141	167

arch/arm64/kvm/hyp.S+1−4

@@ -36,15 +36,12 @@ · 35 unmodified lines
36	36	* passed in x0.
37	37	*
38	38	* A function pointer with a value less than 0xfff has a special meaning,
39		− * and is used to implement __hyp_get_vectors in the same way as in
	39	+ * and is used to implement hyp stubs in the same way as in
40	40	* arch/arm64/kernel/hyp_stub.S.
41		− * HVC behaves as a 'bl' call and will clobber lr.
42	41	*/
43	42	ENTRY(__kvm_call_hyp)
44	43	alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
45		− str lr, [sp, #-16]!
46	44	hvc #0
47		− ldr lr, [sp], #16
48	45	ret
49	46	alternative_else_nop_endif
50	47	b __vhe_hyp_call

arch/arm64/kvm/hyp/hyp-entry.S+21−22

@@ -32,17 +32,17 @@ · 31 unmodified lines
32	32	* Shuffle the parameters before calling the function
33	33	* pointed to in x0. Assumes parameters in x[1,2,3].
34	34	*/
	35	+ str lr, [sp, #-16]!
35	36	mov lr, x0
36	37	mov x0, x1
37	38	mov x1, x2
38	39	mov x2, x3
39	40	blr lr
	41	+ ldr lr, [sp], #16
40	42	.endm
41	43
42	44	ENTRY(__vhe_hyp_call)
43		− str lr, [sp, #-16]!
44	45	do_el2_call
45		− ldr lr, [sp], #16
46	46	/*
47	47	* We used to rely on having an exception return to get
48	48	* an implicit isb. In the E2H case, we don't have it anymore.
@@ -53,21 +53,6 @@ ENTRY(__vhe_hyp_call) · 4 unmodified lines
53	53	ret
54	54	ENDPROC(__vhe_hyp_call)
55	55
56		−/*
57		− * Compute the idmap address of __kvm_hyp_reset based on the idmap
58		− * start passed as a parameter, and jump there.
59		− *
60		− * x0: HYP phys_idmap_start
61		− */
62		−ENTRY(__kvm_hyp_teardown)
63		− mov x4, x0
64		− adr_l x3, __kvm_hyp_reset
65		−
66		− /* insert __kvm_hyp_reset()s offset into phys_idmap_start */
67		− bfi x4, x3, #0, #PAGE_SHIFT
68		− br x4
69		−ENDPROC(__kvm_hyp_teardown)
70		−
71	56	el1_sync: // Guest trapped into EL2
72	57	stp x0, x1, [sp, #-16]!
73	58
@@ -87,10 +72,24 @@ alternative_endif · 13 unmodified lines
87	72	/* Here, we're pretty sure the host called HVC. */
88	73	ldp x0, x1, [sp], #16
89	74
90		− cmp x0, #HVC_GET_VECTORS
91		− b.ne 1f
92		− mrs x0, vbar_el2
93		− b 2f
	75	+ /* Check for a stub HVC call */
	76	+ cmp x0, #HVC_STUB_HCALL_NR
	77	+ b.hs 1f
	78	+
	79	+ /*
	80	+ * Compute the idmap address of __kvm_handle_stub_hvc and
	81	+ * jump there. Since we use kimage_voffset, do not use the
	82	+ * HYP VA for __kvm_handle_stub_hvc, but the kernel VA instead
	83	+ * (by loading it from the constant pool).
	84	+ *
	85	+ * Preserve x0-x4, which may contain stub parameters.
	86	+ */
	87	+ ldr x5, =__kvm_handle_stub_hvc
	88	+ ldr_l x6, kimage_voffset
	89	+
	90	+ /* x5 = __pa(x5) */
	91	+ sub x5, x5, x6
	92	+ br x5
94	93
95	94	1:
96	95	/*
@@ -99,7 +98,7 @@ alternative_endif · 2 unmodified lines
99	98	kern_hyp_va x0
100	99	do_el2_call
101	100
102		−2: eret
	101	+ eret
103	102
104	103	el1_trap:
105	104	/*

arch/arm64/kvm/sys_regs.c+180−316

@@ -55,6 +55,15 @@ · 54 unmodified lines
55	55	* 64bit interface.
56	56	*/
57	57
	58	+static bool read_from_write_only(struct kvm_vcpu *vcpu,
	59	+ const struct sys_reg_params *params)
	60	+{
	61	+ WARN_ONCE(1, "Unexpected sys_reg read to write-only register\n");
	62	+ print_sys_reg_instr(params);
	63	+ kvm_inject_undefined(vcpu);
	64	+ return false;
	65	+}
	66	+
58	67	/* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */
59	68	static u32 cache_levels;
60	69
@@ -460,35 +469,35 @@ static void reset_pmcr(struct kvm_vcpu vcpu, const struct sys_reg_desc r) · 399 unmodified lines
460	469	vcpu_sys_reg(vcpu, PMCR_EL0) = val;
461	470	}
462	471
463		−static bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu)
	472	+static bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)
464	473	{
465	474	u64 reg = vcpu_sys_reg(vcpu, PMUSERENR_EL0);
	475	+ bool enabled = (reg & flags) \|\| vcpu_mode_priv(vcpu);
466	476
467		− return !((reg & ARMV8_PMU_USERENR_EN) \|\| vcpu_mode_priv(vcpu));
	477	+ if (!enabled)
	478	+ kvm_inject_undefined(vcpu);
	479	+
	480	+ return !enabled;
468	481	}
469	482
470		−static bool pmu_write_swinc_el0_disabled(struct kvm_vcpu *vcpu)
	483	+static bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu)
471	484	{
472		− u64 reg = vcpu_sys_reg(vcpu, PMUSERENR_EL0);
	485	+ return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_EN);
	486	+}
473	487
474		− return !((reg & (ARMV8_PMU_USERENR_SW \| ARMV8_PMU_USERENR_EN))
475		− \|\| vcpu_mode_priv(vcpu));
	488	+static bool pmu_write_swinc_el0_disabled(struct kvm_vcpu *vcpu)
	489	+{
	490	+ return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_SW \| ARMV8_PMU_USERENR_EN);
476	491	}
477	492
478	493	static bool pmu_access_cycle_counter_el0_disabled(struct kvm_vcpu *vcpu)
479	494	{
480		− u64 reg = vcpu_sys_reg(vcpu, PMUSERENR_EL0);
481		−
482		− return !((reg & (ARMV8_PMU_USERENR_CR \| ARMV8_PMU_USERENR_EN))
483		− \|\| vcpu_mode_priv(vcpu));
	495	+ return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_CR \| ARMV8_PMU_USERENR_EN);
484	496	}
485	497
486	498	static bool pmu_access_event_counter_el0_disabled(struct kvm_vcpu *vcpu)
487	499	{
488		− u64 reg = vcpu_sys_reg(vcpu, PMUSERENR_EL0);
489		−
490		− return !((reg & (ARMV8_PMU_USERENR_ER \| ARMV8_PMU_USERENR_EN))
491		− \|\| vcpu_mode_priv(vcpu));
	500	+ return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_ER \| ARMV8_PMU_USERENR_EN);
492	501	}
493	502
494	503	static bool access_pmcr(struct kvm_vcpu vcpu, struct sys_reg_params p,
@@ -567,8 +576,10 @@ static bool pmu_counter_idx_valid(struct kvm_vcpu *vcpu, u64 idx) · 72 unmodified lines
567	576
568	577	pmcr = vcpu_sys_reg(vcpu, PMCR_EL0);
569	578	val = (pmcr >> ARMV8_PMU_PMCR_N_SHIFT) & ARMV8_PMU_PMCR_N_MASK;
570		− if (idx >= val && idx != ARMV8_PMU_CYCLE_IDX)
	579	+ if (idx >= val && idx != ARMV8_PMU_CYCLE_IDX) {
	580	+ kvm_inject_undefined(vcpu);
571	581	return false;
	582	+ }
572	583
573	584	return true;
574	585	}
@@ -707,8 +718,10 @@ static bool access_pminten(struct kvm_vcpu vcpu, struct sys_reg_params p, · 132 unmodified lines
707	718	if (!kvm_arm_pmu_v3_ready(vcpu))
708	719	return trap_raz_wi(vcpu, p, r);
709	720
710		− if (!vcpu_mode_priv(vcpu))
	721	+ if (!vcpu_mode_priv(vcpu)) {
	722	+ kvm_inject_undefined(vcpu);
711	723	return false;
	724	+ }
712	725
713	726	if (p->is_write) {
714	727	u64 val = p->regval & mask;
@@ -759,16 +772,15 @@ static bool access_pmswinc(struct kvm_vcpu vcpu, struct sys_reg_params p, · 44 unmodified lines
759	772	if (!kvm_arm_pmu_v3_ready(vcpu))
760	773	return trap_raz_wi(vcpu, p, r);
761	774
	775	+ if (!p->is_write)
	776	+ return read_from_write_only(vcpu, p);
	777	+
762	778	if (pmu_write_swinc_el0_disabled(vcpu))
763	779	return false;
764	780
765		− if (p->is_write) {
766		− mask = kvm_pmu_valid_counter_mask(vcpu);
767		− kvm_pmu_software_increment(vcpu, p->regval & mask);
768		− return true;
769		− }
770		−
771		− return false;
	781	+ mask = kvm_pmu_valid_counter_mask(vcpu);
	782	+ kvm_pmu_software_increment(vcpu, p->regval & mask);
	783	+ return true;
772	784	}
773	785
774	786	static bool access_pmuserenr(struct kvm_vcpu vcpu, struct sys_reg_params p,
@@ -778,8 +790,10 @@ static bool access_pmuserenr(struct kvm_vcpu vcpu, struct sys_reg_params p, · 3 unmodified lines
778	790	return trap_raz_wi(vcpu, p, r);
779	791
780	792	if (p->is_write) {
781		− if (!vcpu_mode_priv(vcpu))
	793	+ if (!vcpu_mode_priv(vcpu)) {
	794	+ kvm_inject_undefined(vcpu);
782	795	return false;
	796	+ }
783	797
784	798	vcpu_sys_reg(vcpu, PMUSERENR_EL0) = p->regval
785	799	& ARMV8_PMU_USERENR_MASK;
@@ -793,31 +807,23 @@ static bool access_pmuserenr(struct kvm_vcpu vcpu, struct sys_reg_params p, · 7 unmodified lines
793	807
794	808	/* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
795	809	#define DBG_BCR_BVR_WCR_WVR_EL1(n) \
796		− /* DBGBVRn_EL1 */ \
797		− { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b100), \
	810	+ { SYS_DESC(SYS_DBGBVRn_EL1(n)), \
798	811	trap_bvr, reset_bvr, n, 0, get_bvr, set_bvr }, \
799		− /* DBGBCRn_EL1 */ \
800		− { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b101), \
	812	+ { SYS_DESC(SYS_DBGBCRn_EL1(n)), \
801	813	trap_bcr, reset_bcr, n, 0, get_bcr, set_bcr }, \
802		− /* DBGWVRn_EL1 */ \
803		− { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b110), \
	814	+ { SYS_DESC(SYS_DBGWVRn_EL1(n)), \
804	815	trap_wvr, reset_wvr, n, 0, get_wvr, set_wvr }, \
805		− /* DBGWCRn_EL1 */ \
806		− { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b111), \
	816	+ { SYS_DESC(SYS_DBGWCRn_EL1(n)), \
807	817	trap_wcr, reset_wcr, n, 0, get_wcr, set_wcr }
808	818
809	819	/* Macro to expand the PMEVCNTRn_EL0 register */
810	820	#define PMU_PMEVCNTR_EL0(n) \
811		− /* PMEVCNTRn_EL0 */ \
812		− { Op0(0b11), Op1(0b011), CRn(0b1110), \
813		− CRm((0b1000 \| (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
	821	+ { SYS_DESC(SYS_PMEVCNTRn_EL0(n)), \
814	822	access_pmu_evcntr, reset_unknown, (PMEVCNTR0_EL0 + n), }
815	823
816	824	/* Macro to expand the PMEVTYPERn_EL0 register */
817	825	#define PMU_PMEVTYPER_EL0(n) \
818		− /* PMEVTYPERn_EL0 */ \
819		− { Op0(0b11), Op1(0b011), CRn(0b1110), \
820		− CRm((0b1100 \| (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
	826	+ { SYS_DESC(SYS_PMEVTYPERn_EL0(n)), \
821	827	access_pmu_evtyper, reset_unknown, (PMEVTYPER0_EL0 + n), }
822	828
823	829	static bool access_cntp_tval(struct kvm_vcpu *vcpu,
@@ -887,24 +893,14 @@ static bool access_cntp_cval(struct kvm_vcpu *vcpu, · 63 unmodified lines
887	893	* more demanding guest...
888	894	*/
889	895	static const struct sys_reg_desc sys_reg_descs[] = {
890		− /* DC ISW */
891		− { Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b0110), Op2(0b010),
892		− access_dcsw },
893		− /* DC CSW */
894		− { Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b1010), Op2(0b010),
895		− access_dcsw },
896		− /* DC CISW */
897		− { Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b1110), Op2(0b010),
898		− access_dcsw },
	896	+ { SYS_DESC(SYS_DC_ISW), access_dcsw },
	897	+ { SYS_DESC(SYS_DC_CSW), access_dcsw },
	898	+ { SYS_DESC(SYS_DC_CISW), access_dcsw },
899	899
900	900	DBG_BCR_BVR_WCR_WVR_EL1(0),
901	901	DBG_BCR_BVR_WCR_WVR_EL1(1),
902		− /* MDCCINT_EL1 */
903		− { Op0(0b10), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b000),
904		− trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
905		− /* MDSCR_EL1 */
906		− { Op0(0b10), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b010),
907		− trap_debug_regs, reset_val, MDSCR_EL1, 0 },
	902	+ { SYS_DESC(SYS_MDCCINT_EL1), trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
	903	+ { SYS_DESC(SYS_MDSCR_EL1), trap_debug_regs, reset_val, MDSCR_EL1, 0 },
908	904	DBG_BCR_BVR_WCR_WVR_EL1(2),
909	905	DBG_BCR_BVR_WCR_WVR_EL1(3),
910	906	DBG_BCR_BVR_WCR_WVR_EL1(4),
@@ -920,179 +916,77 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 9 unmodified lines
920	916	DBG_BCR_BVR_WCR_WVR_EL1(14),
921	917	DBG_BCR_BVR_WCR_WVR_EL1(15),
922	918
923		− /* MDRAR_EL1 */
924		− { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000),
925		− trap_raz_wi },
926		− /* OSLAR_EL1 */
927		− { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b100),
928		− trap_raz_wi },
929		− /* OSLSR_EL1 */
930		− { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0001), Op2(0b100),
931		− trap_oslsr_el1 },
932		− /* OSDLR_EL1 */
933		− { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0011), Op2(0b100),
934		− trap_raz_wi },
935		− /* DBGPRCR_EL1 */
936		− { Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0100), Op2(0b100),
937		− trap_raz_wi },
938		− /* DBGCLAIMSET_EL1 */
939		− { Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1000), Op2(0b110),
940		− trap_raz_wi },
941		− /* DBGCLAIMCLR_EL1 */
942		− { Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1001), Op2(0b110),
943		− trap_raz_wi },
944		− /* DBGAUTHSTATUS_EL1 */
945		− { Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1110), Op2(0b110),
946		− trap_dbgauthstatus_el1 },
947		−
948		− /* MDCCSR_EL1 */
949		− { Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0001), Op2(0b000),
950		− trap_raz_wi },
951		− /* DBGDTR_EL0 */
952		− { Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0100), Op2(0b000),
953		− trap_raz_wi },
954		− /* DBGDTR[TR]X_EL0 */
955		− { Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0101), Op2(0b000),
956		− trap_raz_wi },
957		−
958		− /* DBGVCR32_EL2 */
959		− { Op0(0b10), Op1(0b100), CRn(0b0000), CRm(0b0111), Op2(0b000),
960		− NULL, reset_val, DBGVCR32_EL2, 0 },
961		−
962		− /* MPIDR_EL1 */
963		− { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b101),
964		− NULL, reset_mpidr, MPIDR_EL1 },
965		− /* SCTLR_EL1 */
966		− { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000),
967		− access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
968		− /* CPACR_EL1 */
969		− { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b010),
970		− NULL, reset_val, CPACR_EL1, 0 },
971		− /* TTBR0_EL1 */
972		− { Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b000),
973		− access_vm_reg, reset_unknown, TTBR0_EL1 },
974		− /* TTBR1_EL1 */
975		− { Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b001),
976		− access_vm_reg, reset_unknown, TTBR1_EL1 },
977		− /* TCR_EL1 */
978		− { Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b010),
979		− access_vm_reg, reset_val, TCR_EL1, 0 },
980		−
981		− /* AFSR0_EL1 */
982		− { Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0001), Op2(0b000),
983		− access_vm_reg, reset_unknown, AFSR0_EL1 },
984		− /* AFSR1_EL1 */
985		− { Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0001), Op2(0b001),
986		− access_vm_reg, reset_unknown, AFSR1_EL1 },
987		− /* ESR_EL1 */
988		− { Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0010), Op2(0b000),
989		− access_vm_reg, reset_unknown, ESR_EL1 },
990		− /* FAR_EL1 */
991		− { Op0(0b11), Op1(0b000), CRn(0b0110), CRm(0b0000), Op2(0b000),
992		− access_vm_reg, reset_unknown, FAR_EL1 },
993		− /* PAR_EL1 */
994		− { Op0(0b11), Op1(0b000), CRn(0b0111), CRm(0b0100), Op2(0b000),
995		− NULL, reset_unknown, PAR_EL1 },
996		−
997		− /* PMINTENSET_EL1 */
998		− { Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b001),
999		− access_pminten, reset_unknown, PMINTENSET_EL1 },
1000		− /* PMINTENCLR_EL1 */
1001		− { Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b010),
1002		− access_pminten, NULL, PMINTENSET_EL1 },
1003		−
1004		− /* MAIR_EL1 */
1005		− { Op0(0b11), Op1(0b000), CRn(0b1010), CRm(0b0010), Op2(0b000),
1006		− access_vm_reg, reset_unknown, MAIR_EL1 },
1007		− /* AMAIR_EL1 */
1008		− { Op0(0b11), Op1(0b000), CRn(0b1010), CRm(0b0011), Op2(0b000),
1009		− access_vm_reg, reset_amair_el1, AMAIR_EL1 },
1010		−
1011		− /* VBAR_EL1 */
1012		− { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b0000), Op2(0b000),
1013		− NULL, reset_val, VBAR_EL1, 0 },
1014		−
1015		− /* ICC_SGI1R_EL1 */
1016		− { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b1011), Op2(0b101),
1017		− access_gic_sgi },
1018		− /* ICC_SRE_EL1 */
1019		− { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b1100), Op2(0b101),
1020		− access_gic_sre },
1021		−
1022		− /* CONTEXTIDR_EL1 */
1023		− { Op0(0b11), Op1(0b000), CRn(0b1101), CRm(0b0000), Op2(0b001),
1024		− access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
1025		− /* TPIDR_EL1 */
1026		− { Op0(0b11), Op1(0b000), CRn(0b1101), CRm(0b0000), Op2(0b100),
1027		− NULL, reset_unknown, TPIDR_EL1 },
1028		−
1029		− /* CNTKCTL_EL1 */
1030		− { Op0(0b11), Op1(0b000), CRn(0b1110), CRm(0b0001), Op2(0b000),
1031		− NULL, reset_val, CNTKCTL_EL1, 0},
1032		−
1033		− /* CSSELR_EL1 */
1034		− { Op0(0b11), Op1(0b010), CRn(0b0000), CRm(0b0000), Op2(0b000),
1035		− NULL, reset_unknown, CSSELR_EL1 },
1036		−
1037		− /* PMCR_EL0 */
1038		− { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b000),
1039		− access_pmcr, reset_pmcr, },
1040		− /* PMCNTENSET_EL0 */
1041		− { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b001),
1042		− access_pmcnten, reset_unknown, PMCNTENSET_EL0 },
1043		− /* PMCNTENCLR_EL0 */
1044		− { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b010),
1045		− access_pmcnten, NULL, PMCNTENSET_EL0 },
1046		− /* PMOVSCLR_EL0 */
1047		− { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b011),
1048		− access_pmovs, NULL, PMOVSSET_EL0 },
1049		− /* PMSWINC_EL0 */
1050		− { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b100),
1051		− access_pmswinc, reset_unknown, PMSWINC_EL0 },
1052		− /* PMSELR_EL0 */
1053		− { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b101),
1054		− access_pmselr, reset_unknown, PMSELR_EL0 },
1055		− /* PMCEID0_EL0 */
1056		− { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b110),
1057		− access_pmceid },
1058		− /* PMCEID1_EL0 */
1059		− { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b111),
1060		− access_pmceid },
1061		− /* PMCCNTR_EL0 */
1062		− { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b000),
1063		− access_pmu_evcntr, reset_unknown, PMCCNTR_EL0 },
1064		− /* PMXEVTYPER_EL0 */
1065		− { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b001),
1066		− access_pmu_evtyper },
1067		− /* PMXEVCNTR_EL0 */
1068		− { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b010),
1069		− access_pmu_evcntr },
1070		− /* PMUSERENR_EL0
1071		− * This register resets as unknown in 64bit mode while it resets as zero
	919	+ { SYS_DESC(SYS_MDRAR_EL1), trap_raz_wi },
	920	+ { SYS_DESC(SYS_OSLAR_EL1), trap_raz_wi },
	921	+ { SYS_DESC(SYS_OSLSR_EL1), trap_oslsr_el1 },
	922	+ { SYS_DESC(SYS_OSDLR_EL1), trap_raz_wi },
	923	+ { SYS_DESC(SYS_DBGPRCR_EL1), trap_raz_wi },
	924	+ { SYS_DESC(SYS_DBGCLAIMSET_EL1), trap_raz_wi },
	925	+ { SYS_DESC(SYS_DBGCLAIMCLR_EL1), trap_raz_wi },
	926	+ { SYS_DESC(SYS_DBGAUTHSTATUS_EL1), trap_dbgauthstatus_el1 },
	927	+
	928	+ { SYS_DESC(SYS_MDCCSR_EL0), trap_raz_wi },
	929	+ { SYS_DESC(SYS_DBGDTR_EL0), trap_raz_wi },
	930	+ // DBGDTR[TR]X_EL0 share the same encoding
	931	+ { SYS_DESC(SYS_DBGDTRTX_EL0), trap_raz_wi },
	932	+
	933	+ { SYS_DESC(SYS_DBGVCR32_EL2), NULL, reset_val, DBGVCR32_EL2, 0 },
	934	+
	935	+ { SYS_DESC(SYS_MPIDR_EL1), NULL, reset_mpidr, MPIDR_EL1 },
	936	+ { SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
	937	+ { SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
	938	+ { SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 },
	939	+ { SYS_DESC(SYS_TTBR1_EL1), access_vm_reg, reset_unknown, TTBR1_EL1 },
	940	+ { SYS_DESC(SYS_TCR_EL1), access_vm_reg, reset_val, TCR_EL1, 0 },
	941	+
	942	+ { SYS_DESC(SYS_AFSR0_EL1), access_vm_reg, reset_unknown, AFSR0_EL1 },
	943	+ { SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
	944	+ { SYS_DESC(SYS_ESR_EL1), access_vm_reg, reset_unknown, ESR_EL1 },
	945	+ { SYS_DESC(SYS_FAR_EL1), access_vm_reg, reset_unknown, FAR_EL1 },
	946	+ { SYS_DESC(SYS_PAR_EL1), NULL, reset_unknown, PAR_EL1 },
	947	+
	948	+ { SYS_DESC(SYS_PMINTENSET_EL1), access_pminten, reset_unknown, PMINTENSET_EL1 },
	949	+ { SYS_DESC(SYS_PMINTENCLR_EL1), access_pminten, NULL, PMINTENSET_EL1 },
	950	+
	951	+ { SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 },
	952	+ { SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 },
	953	+
	954	+ { SYS_DESC(SYS_VBAR_EL1), NULL, reset_val, VBAR_EL1, 0 },
	955	+
	956	+ { SYS_DESC(SYS_ICC_SGI1R_EL1), access_gic_sgi },
	957	+ { SYS_DESC(SYS_ICC_SRE_EL1), access_gic_sre },
	958	+
	959	+ { SYS_DESC(SYS_CONTEXTIDR_EL1), access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
	960	+ { SYS_DESC(SYS_TPIDR_EL1), NULL, reset_unknown, TPIDR_EL1 },
	961	+
	962	+ { SYS_DESC(SYS_CNTKCTL_EL1), NULL, reset_val, CNTKCTL_EL1, 0},
	963	+
	964	+ { SYS_DESC(SYS_CSSELR_EL1), NULL, reset_unknown, CSSELR_EL1 },
	965	+
	966	+ { SYS_DESC(SYS_PMCR_EL0), access_pmcr, reset_pmcr, },
	967	+ { SYS_DESC(SYS_PMCNTENSET_EL0), access_pmcnten, reset_unknown, PMCNTENSET_EL0 },
	968	+ { SYS_DESC(SYS_PMCNTENCLR_EL0), access_pmcnten, NULL, PMCNTENSET_EL0 },
	969	+ { SYS_DESC(SYS_PMOVSCLR_EL0), access_pmovs, NULL, PMOVSSET_EL0 },
	970	+ { SYS_DESC(SYS_PMSWINC_EL0), access_pmswinc, reset_unknown, PMSWINC_EL0 },
	971	+ { SYS_DESC(SYS_PMSELR_EL0), access_pmselr, reset_unknown, PMSELR_EL0 },
	972	+ { SYS_DESC(SYS_PMCEID0_EL0), access_pmceid },
	973	+ { SYS_DESC(SYS_PMCEID1_EL0), access_pmceid },
	974	+ { SYS_DESC(SYS_PMCCNTR_EL0), access_pmu_evcntr, reset_unknown, PMCCNTR_EL0 },
	975	+ { SYS_DESC(SYS_PMXEVTYPER_EL0), access_pmu_evtyper },
	976	+ { SYS_DESC(SYS_PMXEVCNTR_EL0), access_pmu_evcntr },
	977	+ /*
	978	+ * PMUSERENR_EL0 resets as unknown in 64bit mode while it resets as zero
1072	979	* in 32bit mode. Here we choose to reset it as zero for consistency.
1073	980	*/
1074		− { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b000),
1075		− access_pmuserenr, reset_val, PMUSERENR_EL0, 0 },
1076		− /* PMOVSSET_EL0 */
1077		− { Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b011),
1078		− access_pmovs, reset_unknown, PMOVSSET_EL0 },
1079		−
1080		− /* TPIDR_EL0 */
1081		− { Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b010),
1082		− NULL, reset_unknown, TPIDR_EL0 },
1083		− /* TPIDRRO_EL0 */
1084		− { Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b011),
1085		− NULL, reset_unknown, TPIDRRO_EL0 },
1086		−
1087		− /* CNTP_TVAL_EL0 */
1088		− { Op0(0b11), Op1(0b011), CRn(0b1110), CRm(0b0010), Op2(0b000),
1089		− access_cntp_tval },
1090		− /* CNTP_CTL_EL0 */
1091		− { Op0(0b11), Op1(0b011), CRn(0b1110), CRm(0b0010), Op2(0b001),
1092		− access_cntp_ctl },
1093		− /* CNTP_CVAL_EL0 */
1094		− { Op0(0b11), Op1(0b011), CRn(0b1110), CRm(0b0010), Op2(0b010),
1095		− access_cntp_cval },
	981	+ { SYS_DESC(SYS_PMUSERENR_EL0), access_pmuserenr, reset_val, PMUSERENR_EL0, 0 },
	982	+ { SYS_DESC(SYS_PMOVSSET_EL0), access_pmovs, reset_unknown, PMOVSSET_EL0 },
	983	+
	984	+ { SYS_DESC(SYS_TPIDR_EL0), NULL, reset_unknown, TPIDR_EL0 },
	985	+ { SYS_DESC(SYS_TPIDRRO_EL0), NULL, reset_unknown, TPIDRRO_EL0 },
	986	+
	987	+ { SYS_DESC(SYS_CNTP_TVAL_EL0), access_cntp_tval },
	988	+ { SYS_DESC(SYS_CNTP_CTL_EL0), access_cntp_ctl },
	989	+ { SYS_DESC(SYS_CNTP_CVAL_EL0), access_cntp_cval },
1096	990
1097	991	/* PMEVCNTRn_EL0 */
1098	992	PMU_PMEVCNTR_EL0(0),
@@ -1158,22 +1052,15 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 59 unmodified lines
1158	1052	PMU_PMEVTYPER_EL0(28),
1159	1053	PMU_PMEVTYPER_EL0(29),
1160	1054	PMU_PMEVTYPER_EL0(30),
1161		− /* PMCCFILTR_EL0
1162		− * This register resets as unknown in 64bit mode while it resets as zero
	1055	+ /*
	1056	+ * PMCCFILTR_EL0 resets as unknown in 64bit mode while it resets as zero
1163	1057	* in 32bit mode. Here we choose to reset it as zero for consistency.
1164	1058	*/
1165		− { Op0(0b11), Op1(0b011), CRn(0b1110), CRm(0b1111), Op2(0b111),
1166		− access_pmu_evtyper, reset_val, PMCCFILTR_EL0, 0 },
1167		−
1168		− /* DACR32_EL2 */
1169		− { Op0(0b11), Op1(0b100), CRn(0b0011), CRm(0b0000), Op2(0b000),
1170		− NULL, reset_unknown, DACR32_EL2 },
1171		− /* IFSR32_EL2 */
1172		− { Op0(0b11), Op1(0b100), CRn(0b0101), CRm(0b0000), Op2(0b001),
1173		− NULL, reset_unknown, IFSR32_EL2 },
1174		− /* FPEXC32_EL2 */
1175		− { Op0(0b11), Op1(0b100), CRn(0b0101), CRm(0b0011), Op2(0b000),
1176		− NULL, reset_val, FPEXC32_EL2, 0x70 },
	1059	+ { SYS_DESC(SYS_PMCCFILTR_EL0), access_pmu_evtyper, reset_val, PMCCFILTR_EL0, 0 },
	1060	+
	1061	+ { SYS_DESC(SYS_DACR32_EL2), NULL, reset_unknown, DACR32_EL2 },
	1062	+ { SYS_DESC(SYS_IFSR32_EL2), NULL, reset_unknown, IFSR32_EL2 },
	1063	+ { SYS_DESC(SYS_FPEXC32_EL2), NULL, reset_val, FPEXC32_EL2, 0x70 },
1177	1064	};
1178	1065
1179	1066	static bool trap_dbgidr(struct kvm_vcpu *vcpu,
@@ -1557,6 +1444,22 @@ int kvm_handle_cp14_load_store(struct kvm_vcpu vcpu, struct kvm_run run) · 377 unmodified lines
1557	1444	return 1;
1558	1445	}
1559	1446
	1447	+static void perform_access(struct kvm_vcpu *vcpu,
	1448	+ struct sys_reg_params *params,
	1449	+ const struct sys_reg_desc *r)
	1450	+{
	1451	+ /*
	1452	+ * Not having an accessor means that we have configured a trap
	1453	+ * that we don't know how to handle. This certainly qualifies
	1454	+ * as a gross bug that should be fixed right away.
	1455	+ */
	1456	+ BUG_ON(!r->access);
	1457	+
	1458	+ /* Skip instruction if instructed so */
	1459	+ if (likely(r->access(vcpu, params, r)))
	1460	+ kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
	1461	+}
	1462	+
1560	1463	/*
1561	1464	* emulate_cp -- tries to match a sys_reg access in a handling table, and
1562	1465	* call the corresponding trap handler.
@@ -1580,20 +1483,8 @@ static int emulate_cp(struct kvm_vcpu *vcpu, · 17 unmodified lines
1580	1483	r = find_reg(params, table, num);
1581	1484
1582	1485	if (r) {
1583		− /*
1584		− * Not having an accessor means that we have
1585		− * configured a trap that we don't know how to
1586		− * handle. This certainly qualifies as a gross bug
1587		− * that should be fixed right away.
1588		− */
1589		− BUG_ON(!r->access);
1590		−
1591		− if (likely(r->access(vcpu, params, r))) {
1592		− /* Skip instruction, since it was emulated */
1593		− kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
1594		− /* Handled */
1595		− return 0;
1596		− }
	1486	+ perform_access(vcpu, params, r);
	1487	+ return 0;
1597	1488	}
1598	1489
1599	1490	/* Not handled */
@@ -1660,20 +1551,25 @@ static int kvm_handle_cp_64(struct kvm_vcpu *vcpu, · 60 unmodified lines
1660	1551	params.regval \|= vcpu_get_reg(vcpu, Rt2) << 32;
1661	1552	}
1662	1553
1663		− if (!emulate_cp(vcpu, &params, target_specific, nr_specific))
1664		− goto out;
1665		− if (!emulate_cp(vcpu, &params, global, nr_global))
1666		− goto out;
1667		−
1668		− unhandled_cp_access(vcpu, &params);
	1554	+ /*
	1555	+ * Try to emulate the coprocessor access using the target
	1556	+ * specific table first, and using the global table afterwards.
	1557	+ * If either of the tables contains a handler, handle the
	1558	+ * potential register operation in the case of a read and return
	1559	+ * with success.
	1560	+ */
	1561	+ if (!emulate_cp(vcpu, &params, target_specific, nr_specific) \|\|
	1562	+ !emulate_cp(vcpu, &params, global, nr_global)) {
	1563	+ /* Split up the value between registers for the read side */
	1564	+ if (!params.is_write) {
	1565	+ vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
	1566	+ vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));
	1567	+ }
1669	1568
1670		−out:
1671		− /* Split up the value between registers for the read side */
1672		− if (!params.is_write) {
1673		− vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
1674		− vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));
	1569	+ return 1;
1675	1570	}
1676	1571
	1572	+ unhandled_cp_access(vcpu, &params);
1677	1573	return 1;
1678	1574	}
1679	1575
@@ -1763,26 +1659,13 @@ static int emulate_sys_reg(struct kvm_vcpu *vcpu, · 83 unmodified lines
1763	1659	r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
1764	1660
1765	1661	if (likely(r)) {
1766		− /*
1767		− * Not having an accessor means that we have
1768		− * configured a trap that we don't know how to
1769		− * handle. This certainly qualifies as a gross bug
1770		− * that should be fixed right away.
1771		− */
1772		− BUG_ON(!r->access);
1773		−
1774		− if (likely(r->access(vcpu, params, r))) {
1775		− /* Skip instruction, since it was emulated */
1776		− kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
1777		− return 1;
1778		− }
1779		− /* If access function fails, it should complain. */
	1662	+ perform_access(vcpu, params, r);
1780	1663	} else {
1781	1664	kvm_err("Unsupported guest sys_reg access at: %lx\n",
1782	1665	*vcpu_pc(vcpu));
1783	1666	print_sys_reg_instr(params);
	1667	+ kvm_inject_undefined(vcpu);
1784	1668	}
1785		− kvm_inject_undefined(vcpu);
1786	1669	return 1;
1787	1670	}
1788	1671
@@ -1932,44 +1815,25 @@ FUNCTION_INVARIANT(aidr_el1) · 143 unmodified lines
1932	1815
1933	1816	/* ->val is filled in by kvm_sys_reg_table_init() */
1934	1817	static struct sys_reg_desc invariant_sys_regs[] = {
1935		− { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b000),
1936		− NULL, get_midr_el1 },
1937		− { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b110),
1938		− NULL, get_revidr_el1 },
1939		− { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b000),
1940		− NULL, get_id_pfr0_el1 },
1941		− { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b001),
1942		− NULL, get_id_pfr1_el1 },
1943		− { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b010),
1944		− NULL, get_id_dfr0_el1 },
1945		− { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b011),
1946		− NULL, get_id_afr0_el1 },
1947		− { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b100),
1948		− NULL, get_id_mmfr0_el1 },
1949		− { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b101),
1950		− NULL, get_id_mmfr1_el1 },
1951		− { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b110),
1952		− NULL, get_id_mmfr2_el1 },
1953		− { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b111),
1954		− NULL, get_id_mmfr3_el1 },
1955		− { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b000),
1956		− NULL, get_id_isar0_el1 },
1957		− { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b001),
1958		− NULL, get_id_isar1_el1 },
1959		− { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b010),
1960		− NULL, get_id_isar2_el1 },
1961		− { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b011),
1962		− NULL, get_id_isar3_el1 },
1963		− { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b100),
1964		− NULL, get_id_isar4_el1 },
1965		− { Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b101),
1966		− NULL, get_id_isar5_el1 },
1967		− { Op0(0b11), Op1(0b001), CRn(0b0000), CRm(0b0000), Op2(0b001),
1968		− NULL, get_clidr_el1 },
1969		− { Op0(0b11), Op1(0b001), CRn(0b0000), CRm(0b0000), Op2(0b111),
1970		− NULL, get_aidr_el1 },
1971		− { Op0(0b11), Op1(0b011), CRn(0b0000), CRm(0b0000), Op2(0b001),
1972		− NULL, get_ctr_el0 },
	1818	+ { SYS_DESC(SYS_MIDR_EL1), NULL, get_midr_el1 },
	1819	+ { SYS_DESC(SYS_REVIDR_EL1), NULL, get_revidr_el1 },
	1820	+ { SYS_DESC(SYS_ID_PFR0_EL1), NULL, get_id_pfr0_el1 },
	1821	+ { SYS_DESC(SYS_ID_PFR1_EL1), NULL, get_id_pfr1_el1 },
	1822	+ { SYS_DESC(SYS_ID_DFR0_EL1), NULL, get_id_dfr0_el1 },
	1823	+ { SYS_DESC(SYS_ID_AFR0_EL1), NULL, get_id_afr0_el1 },
	1824	+ { SYS_DESC(SYS_ID_MMFR0_EL1), NULL, get_id_mmfr0_el1 },
	1825	+ { SYS_DESC(SYS_ID_MMFR1_EL1), NULL, get_id_mmfr1_el1 },
	1826	+ { SYS_DESC(SYS_ID_MMFR2_EL1), NULL, get_id_mmfr2_el1 },
	1827	+ { SYS_DESC(SYS_ID_MMFR3_EL1), NULL, get_id_mmfr3_el1 },
	1828	+ { SYS_DESC(SYS_ID_ISAR0_EL1), NULL, get_id_isar0_el1 },
	1829	+ { SYS_DESC(SYS_ID_ISAR1_EL1), NULL, get_id_isar1_el1 },
	1830	+ { SYS_DESC(SYS_ID_ISAR2_EL1), NULL, get_id_isar2_el1 },
	1831	+ { SYS_DESC(SYS_ID_ISAR3_EL1), NULL, get_id_isar3_el1 },
	1832	+ { SYS_DESC(SYS_ID_ISAR4_EL1), NULL, get_id_isar4_el1 },
	1833	+ { SYS_DESC(SYS_ID_ISAR5_EL1), NULL, get_id_isar5_el1 },
	1834	+ { SYS_DESC(SYS_CLIDR_EL1), NULL, get_clidr_el1 },
	1835	+ { SYS_DESC(SYS_AIDR_EL1), NULL, get_aidr_el1 },
	1836	+ { SYS_DESC(SYS_CTR_EL0), NULL, get_ctr_el0 },
1973	1837	};
1974	1838
1975	1839	static int reg_from_user(u64 val, const void __user uaddr, u64 id)

arch/arm64/kvm/sys_regs.h+5−18

@@ -83,24 +83,6 @@ static inline bool read_zero(struct kvm_vcpu *vcpu, · 82 unmodified lines
83	83	return true;
84	84	}
85	85
86		−static inline bool write_to_read_only(struct kvm_vcpu *vcpu,
87		− const struct sys_reg_params *params)
88		−{
89		− kvm_debug("sys_reg write to read-only register at: %lx\n",
90		− *vcpu_pc(vcpu));
91		− print_sys_reg_instr(params);
92		− return false;
93		−}
94		−
95		−static inline bool read_from_write_only(struct kvm_vcpu *vcpu,
96		− const struct sys_reg_params *params)
97		−{
98		− kvm_debug("sys_reg read to write-only register at: %lx\n",
99		− *vcpu_pc(vcpu));
100		− print_sys_reg_instr(params);
101		− return false;
102		−}
103		−
104	86	/* Reset functions */
105	87	static inline void reset_unknown(struct kvm_vcpu *vcpu,
106	88	const struct sys_reg_desc *r)
@@ -147,4 +129,9 @@ const struct sys_reg_desc *find_reg_by_id(u64 id, · 40 unmodified lines
147	129	#define CRm(_x) .CRm = _x
148	130	#define Op2(_x) .Op2 = _x
149	131
	132	+#define SYS_DESC(reg) \
	133	+ Op0(sys_reg_Op0(reg)), Op1(sys_reg_Op1(reg)), \
	134	+ CRn(sys_reg_CRn(reg)), CRm(sys_reg_CRm(reg)), \
	135	+ Op2(sys_reg_Op2(reg))
	136	+
150	137	#endif /* __ARM64_KVM_SYS_REGS_LOCAL_H__ */

arch/arm64/kvm/sys_regs_generic_v8.c+1−3

@@ -52,9 +52,7 @@ static void reset_actlr(struct kvm_vcpu vcpu, const struct sys_reg_desc r) · 51 unmodified lines
52	52	* Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
53	53	*/
54	54	static const struct sys_reg_desc genericv8_sys_regs[] = {
55		− /* ACTLR_EL1 */
56		− { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b001),
57		− access_actlr, reset_actlr, ACTLR_EL1 },
	55	+ { SYS_DESC(SYS_ACTLR_EL1), access_actlr, reset_actlr, ACTLR_EL1 },
58	56	};
59	57
60	58	static const struct sys_reg_desc genericv8_cp15_regs[] = {

include/kvm/arm_arch_timer.h+2−0

@@ -63,6 +63,8 @@ int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu, · 62 unmodified lines
63	63	void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu);
64	64	void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu);
65	65	void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu);
	66	+bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu);
	67	+void kvm_timer_update_run(struct kvm_vcpu *vcpu);
66	68	void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu);
67	69
68	70	u64 kvm_arm_timer_get_reg(struct kvm_vcpu *, u64 regid);

include/kvm/arm_pmu.h+7−0

@@ -50,6 +50,8 @@ void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, u64 val); · 49 unmodified lines
50	50	void kvm_pmu_overflow_set(struct kvm_vcpu *vcpu, u64 val);
51	51	void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu);
52	52	void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu);
	53	+bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu);
	54	+void kvm_pmu_update_run(struct kvm_vcpu *vcpu);
53	55	void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val);
54	56	void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val);
55	57	void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,
@@ -85,6 +87,11 @@ static inline void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, u64 val) {} · 29 unmodified lines
85	87	static inline void kvm_pmu_overflow_set(struct kvm_vcpu *vcpu, u64 val) {}
86	88	static inline void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu) {}
87	89	static inline void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu) {}
	90	+static inline bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu)
	91	+{
	92	+ return false;
	93	+}
	94	+static inline void kvm_pmu_update_run(struct kvm_vcpu *vcpu) {}
88	95	static inline void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val) {}
89	96	static inline void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val) {}
90	97	static inline void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu,

include/kvm/arm_vgic.h+3−6

@@ -225,8 +225,6 @@ struct vgic_dist { · 224 unmodified lines
225	225	struct vgic_v2_cpu_if {
226	226	u32 vgic_hcr;
227	227	u32 vgic_vmcr;
228		− u32 vgic_misr; /* Saved only */
229		− u64 vgic_eisr; /* Saved only */
230	228	u64 vgic_elrsr; /* Saved only */
231	229	u32 vgic_apr;
232	230	u32 vgic_lr[VGIC_V2_MAX_LRS];
@@ -236,8 +234,6 @@ struct vgic_v3_cpu_if { · 3 unmodified lines
236	234	u32 vgic_hcr;
237	235	u32 vgic_vmcr;
238	236	u32 vgic_sre; /* Restored only, change ignored */
239		− u32 vgic_misr; /* Saved only */
240		− u32 vgic_eisr; /* Saved only */
241	237	u32 vgic_elrsr; /* Saved only */
242	238	u32 vgic_ap0r[4];
243	239	u32 vgic_ap1r[4];
@@ -264,8 +260,6 @@ struct vgic_cpu { · 20 unmodified lines
264	260	*/
265	261	struct list_head ap_list_head;
266	262
267		− u64 live_lrs;
268		−
269	263	/*
270	264	* Members below are used with GICv3 emulation only and represent
271	265	* parts of the redistributor.
@@ -306,6 +300,9 @@ bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int virt_irq); · 34 unmodified lines
306	300
307	301	int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu);
308	302
	303	+void kvm_vgic_load(struct kvm_vcpu *vcpu);
	304	+void kvm_vgic_put(struct kvm_vcpu *vcpu);
	305	+
309	306	#define irqchip_in_kernel(k) (!!((k)->arch.vgic.in_kernel))
310	307	#define vgic_initialized(k) ((k)->arch.vgic.initialized)
311	308	#define vgic_ready(k) ((k)->arch.vgic.ready)

include/uapi/linux/kvm.h+8−0

@@ -894,6 +894,7 @@ struct kvm_ppc_resize_hpt { · 893 unmodified lines
894	894	#define KVM_CAP_S390_AIS 141
895	895	#define KVM_CAP_SPAPR_TCE_VFIO 142
896	896	#define KVM_CAP_X86_GUEST_MWAIT 143
	897	+#define KVM_CAP_ARM_USER_IRQ 144
897	898
898	899	#ifdef KVM_CAP_IRQ_ROUTING
899	900
@@ -1371,4 +1372,11 @@ struct kvm_assigned_msix_entry { · 471 unmodified lines
1371	1372	#define KVM_X2APIC_API_USE_32BIT_IDS (1ULL << 0)
1372	1373	#define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK (1ULL << 1)
1373	1374
	1375	+/* Available with KVM_CAP_ARM_USER_IRQ */
	1376	+
	1377	+/* Bits for run->s.regs.device_irq_level */
	1378	+#define KVM_ARM_DEV_EL1_VTIMER (1 << 0)
	1379	+#define KVM_ARM_DEV_EL1_PTIMER (1 << 1)
	1380	+#define KVM_ARM_DEV_PMU (1 << 2)
	1381	+
1374	1382	#endif /* __LINUX_KVM_H */

virt/kvm/arm/arch_timer.c+98−26

@@ -184,28 +184,47 @@ bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx) · 183 unmodified lines
184	184	return cval <= now;
185	185	}
186	186
	187	+/*
	188	+ * Reflect the timer output level into the kvm_run structure
	189	+ */
	190	+void kvm_timer_update_run(struct kvm_vcpu *vcpu)
	191	+{
	192	+ struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
	193	+ struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
	194	+ struct kvm_sync_regs *regs = &vcpu->run->s.regs;
	195	+
	196	+ /* Populate the device bitmap with the timer states */
	197	+ regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER \|
	198	+ KVM_ARM_DEV_EL1_PTIMER);
	199	+ if (vtimer->irq.level)
	200	+ regs->device_irq_level \|= KVM_ARM_DEV_EL1_VTIMER;
	201	+ if (ptimer->irq.level)
	202	+ regs->device_irq_level \|= KVM_ARM_DEV_EL1_PTIMER;
	203	+}
	204	+
187	205	static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
188	206	struct arch_timer_context *timer_ctx)
189	207	{
190	208	int ret;
191	209
192		− BUG_ON(!vgic_initialized(vcpu->kvm));
193		−
194	210	timer_ctx->active_cleared_last = false;
195	211	timer_ctx->irq.level = new_level;
196	212	trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_ctx->irq.irq,
197	213	timer_ctx->irq.level);
198	214
199		− ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id, timer_ctx->irq.irq,
200		− timer_ctx->irq.level);
201		− WARN_ON(ret);
	215	+ if (likely(irqchip_in_kernel(vcpu->kvm))) {
	216	+ ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
	217	+ timer_ctx->irq.irq,
	218	+ timer_ctx->irq.level);
	219	+ WARN_ON(ret);
	220	+ }
202	221	}
203	222
204	223	/*
205	224	* Check if there was a change in the timer state (should we raise or lower
206	225	* the line level to the GIC).
207	226	*/
208		−static int kvm_timer_update_state(struct kvm_vcpu *vcpu)
	227	+static void kvm_timer_update_state(struct kvm_vcpu *vcpu)
209	228	{
210	229	struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
211	230	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
@@ -217,16 +236,14 @@ static int kvm_timer_update_state(struct kvm_vcpu *vcpu) · 5 unmodified lines
217	236	* because the guest would never see the interrupt. Instead wait
218	237	* until we call this function from kvm_timer_flush_hwstate.
219	238	*/
220		− if (!vgic_initialized(vcpu->kvm) \|\| !timer->enabled)
221		− return -ENODEV;
	239	+ if (unlikely(!timer->enabled))
	240	+ return;
222	241
223	242	if (kvm_timer_should_fire(vtimer) != vtimer->irq.level)
224	243	kvm_timer_update_irq(vcpu, !vtimer->irq.level, vtimer);
225	244
226	245	if (kvm_timer_should_fire(ptimer) != ptimer->irq.level)
227	246	kvm_timer_update_irq(vcpu, !ptimer->irq.level, ptimer);
228		−
229		− return 0;
230	247	}
231	248
232	249	/* Schedule the background timer for the emulated timer. */
@@ -286,25 +303,12 @@ void kvm_timer_unschedule(struct kvm_vcpu *vcpu) · 53 unmodified lines
286	303	timer_disarm(timer);
287	304	}
288	305
289		−/**
290		− * kvm_timer_flush_hwstate - prepare to move the virt timer to the cpu
291		− * @vcpu: The vcpu pointer
292		− *
293		− * Check if the virtual timer has expired while we were running in the host,
294		− * and inject an interrupt if that was the case.
295		− */
296		−void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu)
	306	+static void kvm_timer_flush_hwstate_vgic(struct kvm_vcpu *vcpu)
297	307	{
298	308	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
299	309	bool phys_active;
300	310	int ret;
301	311
302		− if (kvm_timer_update_state(vcpu))
303		− return;
304		−
305		− /* Set the background timer for the physical timer emulation. */
306		− kvm_timer_emulate(vcpu, vcpu_ptimer(vcpu));
307		−
308	312	/*
309	313	* If we enter the guest with the virtual input level to the VGIC
310	314	* asserted, then we have already told the VGIC what we need to, and
@@ -356,11 +360,72 @@ void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu) · 45 unmodified lines
356	360	vtimer->active_cleared_last = !phys_active;
357	361	}
358	362
	363	+bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu)
	364	+{
	365	+ struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
	366	+ struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
	367	+ struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
	368	+ bool vlevel, plevel;
	369	+
	370	+ if (likely(irqchip_in_kernel(vcpu->kvm)))
	371	+ return false;
	372	+
	373	+ vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER;
	374	+ plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER;
	375	+
	376	+ return vtimer->irq.level != vlevel \|\|
	377	+ ptimer->irq.level != plevel;
	378	+}
	379	+
	380	+static void kvm_timer_flush_hwstate_user(struct kvm_vcpu *vcpu)
	381	+{
	382	+ struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
	383	+
	384	+ /*
	385	+ * To prevent continuously exiting from the guest, we mask the
	386	+ * physical interrupt such that the guest can make forward progress.
	387	+ * Once we detect the output level being deasserted, we unmask the
	388	+ * interrupt again so that we exit from the guest when the timer
	389	+ * fires.
	390	+ */
	391	+ if (vtimer->irq.level)
	392	+ disable_percpu_irq(host_vtimer_irq);
	393	+ else
	394	+ enable_percpu_irq(host_vtimer_irq, 0);
	395	+}
	396	+
	397	+/**
	398	+ * kvm_timer_flush_hwstate - prepare timers before running the vcpu
	399	+ * @vcpu: The vcpu pointer
	400	+ *
	401	+ * Check if the virtual timer has expired while we were running in the host,
	402	+ * and inject an interrupt if that was the case, making sure the timer is
	403	+ * masked or disabled on the host so that we keep executing. Also schedule a
	404	+ * software timer for the physical timer if it is enabled.
	405	+ */
	406	+void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu)
	407	+{
	408	+ struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
	409	+
	410	+ if (unlikely(!timer->enabled))
	411	+ return;
	412	+
	413	+ kvm_timer_update_state(vcpu);
	414	+
	415	+ /* Set the background timer for the physical timer emulation. */
	416	+ kvm_timer_emulate(vcpu, vcpu_ptimer(vcpu));
	417	+
	418	+ if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
	419	+ kvm_timer_flush_hwstate_user(vcpu);
	420	+ else
	421	+ kvm_timer_flush_hwstate_vgic(vcpu);
	422	+}
	423	+
359	424	/**
360	425	* kvm_timer_sync_hwstate - sync timer state from cpu
361	426	* @vcpu: The vcpu pointer
362	427	*
363		− * Check if the virtual timer has expired while we were running in the guest,
	428	+ * Check if any of the timers have expired while we were running in the guest,
364	429	* and inject an interrupt if that was the case.
365	430	*/
366	431	void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
@@ -560,6 +625,13 @@ int kvm_timer_enable(struct kvm_vcpu *vcpu) · 193 unmodified lines
560	625	if (timer->enabled)
561	626	return 0;
562	627
	628	+ /* Without a VGIC we do not map virtual IRQs to physical IRQs */
	629	+ if (!irqchip_in_kernel(vcpu->kvm))
	630	+ goto no_vgic;
	631	+
	632	+ if (!vgic_initialized(vcpu->kvm))
	633	+ return -ENODEV;
	634	+
563	635	/*
564	636	* Find the physical IRQ number corresponding to the host_vtimer_irq
565	637	*/
@@ -583,8 +655,8 @@ int kvm_timer_enable(struct kvm_vcpu *vcpu) · 17 unmodified lines
583	655	if (ret)
584	656	return ret;
585	657
	658	+no_vgic:
586	659	timer->enabled = 1;
587		−
588	660	return 0;
589	661	}
590	662

virt/kvm/arm/hyp/vgic-v2-sr.c+7−71

@@ -22,49 +22,6 @@ · 21 unmodified lines
22	22	#include <asm/kvm_emulate.h>
23	23	#include <asm/kvm_hyp.h>
24	24
25		−static void __hyp_text save_maint_int_state(struct kvm_vcpu *vcpu,
26		− void __iomem *base)
27		−{
28		− struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
29		− int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
30		− u32 eisr0, eisr1;
31		− int i;
32		− bool expect_mi;
33		−
34		− expect_mi = !!(cpu_if->vgic_hcr & GICH_HCR_UIE);
35		−
36		− for (i = 0; i < nr_lr; i++) {
37		− if (!(vcpu->arch.vgic_cpu.live_lrs & (1UL << i)))
38		− continue;
39		−
40		− expect_mi \|= (!(cpu_if->vgic_lr[i] & GICH_LR_HW) &&
41		− (cpu_if->vgic_lr[i] & GICH_LR_EOI));
42		− }
43		−
44		− if (expect_mi) {
45		− cpu_if->vgic_misr = readl_relaxed(base + GICH_MISR);
46		−
47		− if (cpu_if->vgic_misr & GICH_MISR_EOI) {
48		− eisr0 = readl_relaxed(base + GICH_EISR0);
49		− if (unlikely(nr_lr > 32))
50		− eisr1 = readl_relaxed(base + GICH_EISR1);
51		− else
52		− eisr1 = 0;
53		− } else {
54		− eisr0 = eisr1 = 0;
55		− }
56		− } else {
57		− cpu_if->vgic_misr = 0;
58		− eisr0 = eisr1 = 0;
59		− }
60		−
61		−#ifdef CONFIG_CPU_BIG_ENDIAN
62		− cpu_if->vgic_eisr = ((u64)eisr0 << 32) \| eisr1;
63		−#else
64		− cpu_if->vgic_eisr = ((u64)eisr1 << 32) \| eisr0;
65		−#endif
66		−}
67		−
68	25	static void __hyp_text save_elrsr(struct kvm_vcpu vcpu, void __iomem base)
69	26	{
70	27	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
@@ -87,13 +44,10 @@ static void __hyp_text save_elrsr(struct kvm_vcpu vcpu, void __iomem base) · 16 unmodified lines
87	44	static void __hyp_text save_lrs(struct kvm_vcpu vcpu, void __iomem base)
88	45	{
89	46	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
90		− int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
91	47	int i;
	48	+ u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
92	49
93		− for (i = 0; i < nr_lr; i++) {
94		− if (!(vcpu->arch.vgic_cpu.live_lrs & (1UL << i)))
95		− continue;
96		−
	50	+ for (i = 0; i < used_lrs; i++) {
97	51	if (cpu_if->vgic_elrsr & (1UL << i))
98	52	cpu_if->vgic_lr[i] &= ~GICH_LR_STATE;
99	53	else
@@ -110,26 +64,20 @@ void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu) · 10 unmodified lines
110	64	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
111	65	struct vgic_dist *vgic = &kvm->arch.vgic;
112	66	void __iomem *base = kern_hyp_va(vgic->vctrl_base);
	67	+ u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
113	68
114	69	if (!base)
115	70	return;
116	71
117		− cpu_if->vgic_vmcr = readl_relaxed(base + GICH_VMCR);
118		−
119		− if (vcpu->arch.vgic_cpu.live_lrs) {
	72	+ if (used_lrs) {
120	73	cpu_if->vgic_apr = readl_relaxed(base + GICH_APR);
121	74
122		− save_maint_int_state(vcpu, base);
123	75	save_elrsr(vcpu, base);
124	76	save_lrs(vcpu, base);
125	77
126	78	writel_relaxed(0, base + GICH_HCR);
127		−
128		− vcpu->arch.vgic_cpu.live_lrs = 0;
129	79	} else {
130		− cpu_if->vgic_eisr = 0;
131	80	cpu_if->vgic_elrsr = ~0UL;
132		− cpu_if->vgic_misr = 0;
133	81	cpu_if->vgic_apr = 0;
134	82	}
135	83	}
@@ -141,32 +89,20 @@ void __hyp_text __vgic_v2_restore_state(struct kvm_vcpu *vcpu) · 5 unmodified lines
141	89	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
142	90	struct vgic_dist *vgic = &kvm->arch.vgic;
143	91	void __iomem *base = kern_hyp_va(vgic->vctrl_base);
144		− int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
145	92	int i;
146		− u64 live_lrs = 0;
	93	+ u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
147	94
148	95	if (!base)
149	96	return;
150	97
151		−
152		− for (i = 0; i < nr_lr; i++)
153		− if (cpu_if->vgic_lr[i] & GICH_LR_STATE)
154		− live_lrs \|= 1UL << i;
155		−
156		− if (live_lrs) {
	98	+ if (used_lrs) {
157	99	writel_relaxed(cpu_if->vgic_hcr, base + GICH_HCR);
158	100	writel_relaxed(cpu_if->vgic_apr, base + GICH_APR);
159		− for (i = 0; i < nr_lr; i++) {
160		− if (!(live_lrs & (1UL << i)))
161		− continue;
162		−
	101	+ for (i = 0; i < used_lrs; i++) {
163	102	writel_relaxed(cpu_if->vgic_lr[i],
164	103	base + GICH_LR0 + (i * 4));
165	104	}
166	105	}
167		−
168		− writel_relaxed(cpu_if->vgic_vmcr, base + GICH_VMCR);
169		− vcpu->arch.vgic_cpu.live_lrs = live_lrs;
170	106	}
171	107
172	108	#ifdef CONFIG_ARM64

virt/kvm/arm/hyp/vgic-v3-sr.c+24−63

@@ -118,66 +118,32 @@ static void __hyp_text __gic_v3_set_lr(u64 val, int lr) · 117 unmodified lines
118	118	}
119	119	}
120	120
121		−static void __hyp_text save_maint_int_state(struct kvm_vcpu *vcpu, int nr_lr)
122		−{
123		− struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
124		− int i;
125		− bool expect_mi;
126		−
127		− expect_mi = !!(cpu_if->vgic_hcr & ICH_HCR_UIE);
128		−
129		− for (i = 0; i < nr_lr; i++) {
130		− if (!(vcpu->arch.vgic_cpu.live_lrs & (1UL << i)))
131		− continue;
132		−
133		− expect_mi \|= (!(cpu_if->vgic_lr[i] & ICH_LR_HW) &&
134		− (cpu_if->vgic_lr[i] & ICH_LR_EOI));
135		− }
136		−
137		− if (expect_mi) {
138		− cpu_if->vgic_misr = read_gicreg(ICH_MISR_EL2);
139		−
140		− if (cpu_if->vgic_misr & ICH_MISR_EOI)
141		− cpu_if->vgic_eisr = read_gicreg(ICH_EISR_EL2);
142		− else
143		− cpu_if->vgic_eisr = 0;
144		− } else {
145		− cpu_if->vgic_misr = 0;
146		− cpu_if->vgic_eisr = 0;
147		− }
148		−}
149		−
150	121	void __hyp_text __vgic_v3_save_state(struct kvm_vcpu *vcpu)
151	122	{
152	123	struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
	124	+ u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
153	125	u64 val;
154	126
155	127	/*
156	128	* Make sure stores to the GIC via the memory mapped interface
157	129	* are now visible to the system register interface.
158	130	*/
159		− if (!cpu_if->vgic_sre)
	131	+ if (!cpu_if->vgic_sre) {
160	132	dsb(st);
	133	+ cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
	134	+ }
161	135
162		− cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
163		−
164		− if (vcpu->arch.vgic_cpu.live_lrs) {
	136	+ if (used_lrs) {
165	137	int i;
166		− u32 max_lr_idx, nr_pri_bits;
	138	+ u32 nr_pri_bits;
167	139
168	140	cpu_if->vgic_elrsr = read_gicreg(ICH_ELSR_EL2);
169	141
170	142	write_gicreg(0, ICH_HCR_EL2);
171	143	val = read_gicreg(ICH_VTR_EL2);
172		− max_lr_idx = vtr_to_max_lr_idx(val);
173	144	nr_pri_bits = vtr_to_nr_pri_bits(val);
174	145
175		− save_maint_int_state(vcpu, max_lr_idx + 1);
176		−
177		− for (i = 0; i <= max_lr_idx; i++) {
178		− if (!(vcpu->arch.vgic_cpu.live_lrs & (1UL << i)))
179		− continue;
180		−
	146	+ for (i = 0; i < used_lrs; i++) {
181	147	if (cpu_if->vgic_elrsr & (1 << i))
182	148	cpu_if->vgic_lr[i] &= ~ICH_LR_STATE;
183	149	else
@@ -205,11 +171,7 @@ void __hyp_text __vgic_v3_save_state(struct kvm_vcpu *vcpu) · 21 unmodified lines
205	171	default:
206	172	cpu_if->vgic_ap1r[0] = read_gicreg(ICH_AP1R0_EL2);
207	173	}
208		−
209		− vcpu->arch.vgic_cpu.live_lrs = 0;
210	174	} else {
211		− cpu_if->vgic_misr = 0;
212		− cpu_if->vgic_eisr = 0;
213	175	cpu_if->vgic_elrsr = 0xffff;
214	176	cpu_if->vgic_ap0r[0] = 0;
215	177	cpu_if->vgic_ap0r[1] = 0;
@@ -234,9 +196,9 @@ void __hyp_text __vgic_v3_save_state(struct kvm_vcpu *vcpu) · 18 unmodified lines
234	196	void __hyp_text __vgic_v3_restore_state(struct kvm_vcpu *vcpu)
235	197	{
236	198	struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
	199	+ u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
237	200	u64 val;
238		− u32 max_lr_idx, nr_pri_bits;
239		− u16 live_lrs = 0;
	201	+ u32 nr_pri_bits;
240	202	int i;
241	203
242	204	/*
@@ -245,25 +207,19 @@ void __hyp_text __vgic_v3_restore_state(struct kvm_vcpu *vcpu) · 2 unmodified lines
245	207	* delivered as a FIQ to the guest, with potentially fatal
246	208	* consequences. So we must make sure that ICC_SRE_EL1 has
247	209	* been actually programmed with the value we want before
248		− * starting to mess with the rest of the GIC.
	210	+ * starting to mess with the rest of the GIC, and VMCR_EL2 in
	211	+ * particular.
249	212	*/
250	213	if (!cpu_if->vgic_sre) {
251	214	write_gicreg(0, ICC_SRE_EL1);
252	215	isb();
	216	+ write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
253	217	}
254	218
255	219	val = read_gicreg(ICH_VTR_EL2);
256		− max_lr_idx = vtr_to_max_lr_idx(val);
257	220	nr_pri_bits = vtr_to_nr_pri_bits(val);
258	221
259		− for (i = 0; i <= max_lr_idx; i++) {
260		− if (cpu_if->vgic_lr[i] & ICH_LR_STATE)
261		− live_lrs \|= (1 << i);
262		− }
263		−
264		− write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
265		−
266		− if (live_lrs) {
	222	+ if (used_lrs) {
267	223	write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
268	224
269	225	switch (nr_pri_bits) {
@@ -286,12 +242,8 @@ void __hyp_text __vgic_v3_restore_state(struct kvm_vcpu *vcpu) · 16 unmodified lines
286	242	write_gicreg(cpu_if->vgic_ap1r[0], ICH_AP1R0_EL2);
287	243	}
288	244
289		− for (i = 0; i <= max_lr_idx; i++) {
290		− if (!(live_lrs & (1 << i)))
291		− continue;
292		−
	245	+ for (i = 0; i < used_lrs; i++)
293	246	__gic_v3_set_lr(cpu_if->vgic_lr[i], i);
294		− }
295	247	}
296	248
297	249	/*
@@ -303,7 +255,6 @@ void __hyp_text __vgic_v3_restore_state(struct kvm_vcpu *vcpu) · 5 unmodified lines
303	255	isb();
304	256	dsb(sy);
305	257	}
306		− vcpu->arch.vgic_cpu.live_lrs = live_lrs;
307	258
308	259	/*
309	260	* Prevent the guest from touching the GIC system registers if
@@ -326,3 +277,13 @@ u64 __hyp_text __vgic_v3_get_ich_vtr_el2(void) · 16 unmodified lines
326	277	{
327	278	return read_gicreg(ICH_VTR_EL2);
328	279	}
	280	+
	281	+u64 __hyp_text __vgic_v3_read_vmcr(void)
	282	+{
	283	+ return read_gicreg(ICH_VMCR_EL2);
	284	+}
	285	+
	286	+void __hyp_text __vgic_v3_write_vmcr(u32 vmcr)
	287	+{
	288	+ write_gicreg(vmcr, ICH_VMCR_EL2);
	289	+}

virt/kvm/arm/pmu.c+35−4

@@ -230,13 +230,44 @@ static void kvm_pmu_update_state(struct kvm_vcpu *vcpu) · 229 unmodified lines
230	230	return;
231	231
232	232	overflow = !!kvm_pmu_overflow_status(vcpu);
233		− if (pmu->irq_level != overflow) {
234		− pmu->irq_level = overflow;
235		− kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
236		− pmu->irq_num, overflow);
	233	+ if (pmu->irq_level == overflow)
	234	+ return;
	235	+
	236	+ pmu->irq_level = overflow;
	237	+
	238	+ if (likely(irqchip_in_kernel(vcpu->kvm))) {
	239	+ int ret;
	240	+ ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
	241	+ pmu->irq_num, overflow);
	242	+ WARN_ON(ret);
237	243	}
238	244	}
239	245
	246	+bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu)
	247	+{
	248	+ struct kvm_pmu *pmu = &vcpu->arch.pmu;
	249	+ struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
	250	+ bool run_level = sregs->device_irq_level & KVM_ARM_DEV_PMU;
	251	+
	252	+ if (likely(irqchip_in_kernel(vcpu->kvm)))
	253	+ return false;
	254	+
	255	+ return pmu->irq_level != run_level;
	256	+}
	257	+
	258	+/*
	259	+ * Reflect the PMU overflow interrupt output level into the kvm_run structure
	260	+ */
	261	+void kvm_pmu_update_run(struct kvm_vcpu *vcpu)
	262	+{
	263	+ struct kvm_sync_regs *regs = &vcpu->run->s.regs;
	264	+
	265	+ /* Populate the timer bitmap for user space */
	266	+ regs->device_irq_level &= ~KVM_ARM_DEV_PMU;
	267	+ if (vcpu->arch.pmu.irq_level)
	268	+ regs->device_irq_level \|= KVM_ARM_DEV_PMU;
	269	+}
	270	+
240	271	/**
241	272	* kvm_pmu_flush_hwstate - flush pmu state to cpu
242	273	* @vcpu: The vcpu pointer

virt/kvm/arm/vgic/vgic-init.c+68−40

@@ -24,7 +24,12 @@ · 23 unmodified lines
24	24
25	25	/*
26	26	* Initialization rules: there are multiple stages to the vgic
27		− * initialization, both for the distributor and the CPU interfaces.
	27	+ * initialization, both for the distributor and the CPU interfaces. The basic
	28	+ * idea is that even though the VGIC is not functional or not requested from
	29	+ * user space, the critical path of the run loop can still call VGIC functions
	30	+ * that just won't do anything, without them having to check additional
	31	+ * initialization flags to ensure they don't look at uninitialized data
	32	+ * structures.
28	33	*
29	34	* Distributor:
30	35	*
@@ -39,23 +44,67 @@ · 8 unmodified lines
39	44	*
40	45	* CPU Interface:
41	46	*
42		− * - kvm_vgic_cpu_early_init(): initialization of static data that
	47	+ * - kvm_vgic_vcpu_early_init(): initialization of static data that
43	48	* doesn't depend on any sizing information or emulation type. No
44	49	* allocation is allowed there.
45	50	*/
46	51
47	52	/* EARLY INIT */
48	53
49		−/*
50		− * Those 2 functions should not be needed anymore but they
51		− * still are called from arm.c
	54	+/**
	55	+ * kvm_vgic_early_init() - Initialize static VGIC VCPU data structures
	56	+ * @kvm: The VM whose VGIC districutor should be initialized
	57	+ *
	58	+ * Only do initialization of static structures that don't require any
	59	+ * allocation or sizing information from userspace. vgic_init() called
	60	+ * kvm_vgic_dist_init() which takes care of the rest.
52	61	*/
53	62	void kvm_vgic_early_init(struct kvm *kvm)
54	63	{
	64	+ struct vgic_dist *dist = &kvm->arch.vgic;
	65	+
	66	+ INIT_LIST_HEAD(&dist->lpi_list_head);
	67	+ spin_lock_init(&dist->lpi_list_lock);
55	68	}
56	69
	70	+/**
	71	+ * kvm_vgic_vcpu_early_init() - Initialize static VGIC VCPU data structures
	72	+ * @vcpu: The VCPU whose VGIC data structures whould be initialized
	73	+ *
	74	+ * Only do initialization, but do not actually enable the VGIC CPU interface
	75	+ * yet.
	76	+ */
57	77	void kvm_vgic_vcpu_early_init(struct kvm_vcpu *vcpu)
58	78	{
	79	+ struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
	80	+ int i;
	81	+
	82	+ INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
	83	+ spin_lock_init(&vgic_cpu->ap_list_lock);
	84	+
	85	+ /*
	86	+ * Enable and configure all SGIs to be edge-triggered and
	87	+ * configure all PPIs as level-triggered.
	88	+ */
	89	+ for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
	90	+ struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
	91	+
	92	+ INIT_LIST_HEAD(&irq->ap_list);
	93	+ spin_lock_init(&irq->irq_lock);
	94	+ irq->intid = i;
	95	+ irq->vcpu = NULL;
	96	+ irq->target_vcpu = vcpu;
	97	+ irq->targets = 1U << vcpu->vcpu_id;
	98	+ kref_init(&irq->refcount);
	99	+ if (vgic_irq_is_sgi(i)) {
	100	+ /* SGIs */
	101	+ irq->enabled = 1;
	102	+ irq->config = VGIC_CONFIG_EDGE;
	103	+ } else {
	104	+ /* PPIs */
	105	+ irq->config = VGIC_CONFIG_LEVEL;
	106	+ }
	107	+ }
59	108	}
60	109
61	110	/* CREATION */
@@ -148,9 +197,6 @@ static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis) · 86 unmodified lines
148	197	struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
149	198	int i;
150	199
151		− INIT_LIST_HEAD(&dist->lpi_list_head);
152		− spin_lock_init(&dist->lpi_list_lock);
153		−
154	200	dist->spis = kcalloc(nr_spis, sizeof(struct vgic_irq), GFP_KERNEL);
155	201	if (!dist->spis)
156	202	return -ENOMEM;
@@ -181,41 +227,11 @@ static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis) · 24 unmodified lines
181	227	}
182	228
183	229	/**
184		− * kvm_vgic_vcpu_init: initialize the vcpu data structures and
185		− * enable the VCPU interface
186		− * @vcpu: the VCPU which's VGIC should be initialized
	230	+ * kvm_vgic_vcpu_init() - Enable the VCPU interface
	231	+ * @vcpu: the VCPU which's VGIC should be enabled
187	232	*/
188	233	static void kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
189	234	{
190		− struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
191		− int i;
192		−
193		− INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
194		− spin_lock_init(&vgic_cpu->ap_list_lock);
195		−
196		− /*
197		− * Enable and configure all SGIs to be edge-triggered and
198		− * configure all PPIs as level-triggered.
199		− */
200		− for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
201		− struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
202		−
203		− INIT_LIST_HEAD(&irq->ap_list);
204		− spin_lock_init(&irq->irq_lock);
205		− irq->intid = i;
206		− irq->vcpu = NULL;
207		− irq->target_vcpu = vcpu;
208		− irq->targets = 1U << vcpu->vcpu_id;
209		− kref_init(&irq->refcount);
210		− if (vgic_irq_is_sgi(i)) {
211		− /* SGIs */
212		− irq->enabled = 1;
213		− irq->config = VGIC_CONFIG_EDGE;
214		− } else {
215		− /* PPIs */
216		− irq->config = VGIC_CONFIG_LEVEL;
217		− }
218		− }
219	235	if (kvm_vgic_global_state.type == VGIC_V2)
220	236	vgic_v2_enable(vcpu);
221	237	else
@@ -262,6 +278,18 @@ int vgic_init(struct kvm *kvm) · 40 unmodified lines
262	278	vgic_debug_init(kvm);
263	279
264	280	dist->initialized = true;
	281	+
	282	+ /*
	283	+ * If we're initializing GICv2 on-demand when first running the VCPU
	284	+ * then we need to load the VGIC state onto the CPU. We can detect
	285	+ * this easily by checking if we are in between vcpu_load and vcpu_put
	286	+ * when we just initialized the VGIC.
	287	+ */
	288	+ preempt_disable();
	289	+ vcpu = kvm_arm_get_running_vcpu();
	290	+ if (vcpu)
	291	+ kvm_vgic_load(vcpu);
	292	+ preempt_enable();
265	293	out:
266	294	return ret;
267	295	}

virt/kvm/arm/vgic/vgic-v2.c+39−51

@@ -22,59 +22,17 @@ · 21 unmodified lines
22	22
23	23	#include "vgic.h"
24	24
25		−/*
26		− * Call this function to convert a u64 value to an unsigned long * bitmask
27		− * in a way that works on both 32-bit and 64-bit LE and BE platforms.
28		− *
29		− * Warning: Calling this function may modify *val.
30		− */
31		−static unsigned long u64_to_bitmask(u64 val)
32		−{
33		−#if defined(CONFIG_CPU_BIG_ENDIAN) && BITS_PER_LONG == 32
34		− val = (val >> 32) \| (*val << 32);
35		−#endif
36		− return (unsigned long *)val;
37		−}
38		−
39		−void vgic_v2_process_maintenance(struct kvm_vcpu *vcpu)
	25	+void vgic_v2_set_underflow(struct kvm_vcpu *vcpu)
40	26	{
41	27	struct vgic_v2_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v2;
42	28
43		− if (cpuif->vgic_misr & GICH_MISR_EOI) {
44		− u64 eisr = cpuif->vgic_eisr;
45		− unsigned long *eisr_bmap = u64_to_bitmask(&eisr);
46		− int lr;
47		−
48		− for_each_set_bit(lr, eisr_bmap, kvm_vgic_global_state.nr_lr) {
49		− u32 intid = cpuif->vgic_lr[lr] & GICH_LR_VIRTUALID;
50		−
51		− WARN_ON(cpuif->vgic_lr[lr] & GICH_LR_STATE);
52		−
53		− /* Only SPIs require notification */
54		− if (vgic_valid_spi(vcpu->kvm, intid))
55		− kvm_notify_acked_irq(vcpu->kvm, 0,
56		− intid - VGIC_NR_PRIVATE_IRQS);
57		− }
58		− }
59		−
60		− /* check and disable underflow maintenance IRQ */
61		− cpuif->vgic_hcr &= ~GICH_HCR_UIE;
62		−
63		− /*
64		− * In the next iterations of the vcpu loop, if we sync the
65		− * vgic state after flushing it, but before entering the guest
66		− * (this happens for pending signals and vmid rollovers), then
67		− * make sure we don't pick up any old maintenance interrupts
68		− * here.
69		− */
70		− cpuif->vgic_eisr = 0;
	29	+ cpuif->vgic_hcr \|= GICH_HCR_UIE;
71	30	}
72	31
73		−void vgic_v2_set_underflow(struct kvm_vcpu *vcpu)
	32	+static bool lr_signals_eoi_mi(u32 lr_val)
74	33	{
75		− struct vgic_v2_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v2;
76		−
77		− cpuif->vgic_hcr \|= GICH_HCR_UIE;
	34	+ return !(lr_val & GICH_LR_STATE) && (lr_val & GICH_LR_EOI) &&
	35	+ !(lr_val & GICH_LR_HW);
78	36	}
79	37
80	38	/*
@@ -86,14 +44,22 @@ void vgic_v2_set_underflow(struct kvm_vcpu *vcpu) · 5 unmodified lines
86	44	*/
87	45	void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu)
88	46	{
89		− struct vgic_v2_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v2;
	47	+ struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
	48	+ struct vgic_v2_cpu_if *cpuif = &vgic_cpu->vgic_v2;
90	49	int lr;
91	50
92		− for (lr = 0; lr < vcpu->arch.vgic_cpu.used_lrs; lr++) {
	51	+ cpuif->vgic_hcr &= ~GICH_HCR_UIE;
	52	+
	53	+ for (lr = 0; lr < vgic_cpu->used_lrs; lr++) {
93	54	u32 val = cpuif->vgic_lr[lr];
94	55	u32 intid = val & GICH_LR_VIRTUALID;
95	56	struct vgic_irq *irq;
96	57
	58	+ /* Notify fds when the guest EOI'ed a level-triggered SPI */
	59	+ if (lr_signals_eoi_mi(val) && vgic_valid_spi(vcpu->kvm, intid))
	60	+ kvm_notify_acked_irq(vcpu->kvm, 0,
	61	+ intid - VGIC_NR_PRIVATE_IRQS);
	62	+
97	63	irq = vgic_get_irq(vcpu->kvm, vcpu, intid);
98	64
99	65	spin_lock(&irq->irq_lock);
@@ -126,6 +92,8 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu) · 26 unmodified lines
126	92	spin_unlock(&irq->irq_lock);
127	93	vgic_put_irq(vcpu->kvm, irq);
128	94	}
	95	+
	96	+ vgic_cpu->used_lrs = 0;
129	97	}
130	98
131	99	/*
@@ -184,6 +152,7 @@ void vgic_v2_clear_lr(struct kvm_vcpu *vcpu, int lr) · 52 unmodified lines
184	152
185	153	void vgic_v2_set_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp)
186	154	{
	155	+ struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
187	156	u32 vmcr;
188	157
189	158	vmcr = (vmcrp->ctlr << GICH_VMCR_CTRL_SHIFT) & GICH_VMCR_CTRL_MASK;
@@ -194,12 +163,15 @@ void vgic_v2_set_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp) · 4 unmodified lines
194	163	vmcr \|= (vmcrp->pmr << GICH_VMCR_PRIMASK_SHIFT) &
195	164	GICH_VMCR_PRIMASK_MASK;
196	165
197		− vcpu->arch.vgic_cpu.vgic_v2.vgic_vmcr = vmcr;
	166	+ cpu_if->vgic_vmcr = vmcr;
198	167	}
199	168
200	169	void vgic_v2_get_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp)
201	170	{
202		− u32 vmcr = vcpu->arch.vgic_cpu.vgic_v2.vgic_vmcr;
	171	+ struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
	172	+ u32 vmcr;
	173	+
	174	+ vmcr = cpu_if->vgic_vmcr;
203	175
204	176	vmcrp->ctlr = (vmcr & GICH_VMCR_CTRL_MASK) >>
205	177	GICH_VMCR_CTRL_SHIFT;
@@ -375,3 +347,19 @@ out: · 169 unmodified lines
375	347
376	348	return ret;
377	349	}
	350	+
	351	+void vgic_v2_load(struct kvm_vcpu *vcpu)
	352	+{
	353	+ struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
	354	+ struct vgic_dist *vgic = &vcpu->kvm->arch.vgic;
	355	+
	356	+ writel_relaxed(cpu_if->vgic_vmcr, vgic->vctrl_base + GICH_VMCR);
	357	+}
	358	+
	359	+void vgic_v2_put(struct kvm_vcpu *vcpu)
	360	+{
	361	+ struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
	362	+ struct vgic_dist *vgic = &vcpu->kvm->arch.vgic;
	363	+
	364	+ cpu_if->vgic_vmcr = readl_relaxed(vgic->vctrl_base + GICH_VMCR);
	365	+}

virt/kvm/arm/vgic/vgic-v3.c+45−42

@@ -21,59 +21,29 @@ · 20 unmodified lines
21	21
22	22	#include "vgic.h"
23	23
24		−void vgic_v3_process_maintenance(struct kvm_vcpu *vcpu)
	24	+void vgic_v3_set_underflow(struct kvm_vcpu *vcpu)
25	25	{
26	26	struct vgic_v3_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v3;
27		− u32 model = vcpu->kvm->arch.vgic.vgic_model;
28		−
29		− if (cpuif->vgic_misr & ICH_MISR_EOI) {
30		− unsigned long eisr_bmap = cpuif->vgic_eisr;
31		− int lr;
32		−
33		− for_each_set_bit(lr, &eisr_bmap, kvm_vgic_global_state.nr_lr) {
34		− u32 intid;
35		− u64 val = cpuif->vgic_lr[lr];
36		−
37		− if (model == KVM_DEV_TYPE_ARM_VGIC_V3)
38		− intid = val & ICH_LR_VIRTUAL_ID_MASK;
39		− else
40		− intid = val & GICH_LR_VIRTUALID;
41		−
42		− WARN_ON(cpuif->vgic_lr[lr] & ICH_LR_STATE);
43		−
44		− /* Only SPIs require notification */
45		− if (vgic_valid_spi(vcpu->kvm, intid))
46		− kvm_notify_acked_irq(vcpu->kvm, 0,
47		− intid - VGIC_NR_PRIVATE_IRQS);
48		− }
49		−
50		− /*
51		− * In the next iterations of the vcpu loop, if we sync
52		− * the vgic state after flushing it, but before
53		− * entering the guest (this happens for pending
54		− * signals and vmid rollovers), then make sure we
55		− * don't pick up any old maintenance interrupts here.
56		− */
57		− cpuif->vgic_eisr = 0;
58		− }
59	27
60		− cpuif->vgic_hcr &= ~ICH_HCR_UIE;
	28	+ cpuif->vgic_hcr \|= ICH_HCR_UIE;
61	29	}
62	30
63		−void vgic_v3_set_underflow(struct kvm_vcpu *vcpu)
	31	+static bool lr_signals_eoi_mi(u64 lr_val)
64	32	{
65		− struct vgic_v3_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v3;
66		−
67		− cpuif->vgic_hcr \|= ICH_HCR_UIE;
	33	+ return !(lr_val & ICH_LR_STATE) && (lr_val & ICH_LR_EOI) &&
	34	+ !(lr_val & ICH_LR_HW);
68	35	}
69	36
70	37	void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu)
71	38	{
72		− struct vgic_v3_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v3;
	39	+ struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
	40	+ struct vgic_v3_cpu_if *cpuif = &vgic_cpu->vgic_v3;
73	41	u32 model = vcpu->kvm->arch.vgic.vgic_model;
74	42	int lr;
75	43
76		− for (lr = 0; lr < vcpu->arch.vgic_cpu.used_lrs; lr++) {
	44	+ cpuif->vgic_hcr &= ~ICH_HCR_UIE;
	45	+
	46	+ for (lr = 0; lr < vgic_cpu->used_lrs; lr++) {
77	47	u64 val = cpuif->vgic_lr[lr];
78	48	u32 intid;
79	49	struct vgic_irq *irq;
@@ -82,6 +52,12 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu) · 2 unmodified lines
82	52	intid = val & ICH_LR_VIRTUAL_ID_MASK;
83	53	else
84	54	intid = val & GICH_LR_VIRTUALID;
	55	+
	56	+ /* Notify fds when the guest EOI'ed a level-triggered IRQ */
	57	+ if (lr_signals_eoi_mi(val) && vgic_valid_spi(vcpu->kvm, intid))
	58	+ kvm_notify_acked_irq(vcpu->kvm, 0,
	59	+ intid - VGIC_NR_PRIVATE_IRQS);
	60	+
85	61	irq = vgic_get_irq(vcpu->kvm, vcpu, intid);
86	62	if (!irq) /* An LPI could have been unmapped. */
87	63	continue;
@@ -117,6 +93,8 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu) · 29 unmodified lines
117	93	spin_unlock(&irq->irq_lock);
118	94	vgic_put_irq(vcpu->kvm, irq);
119	95	}
	96	+
	97	+ vgic_cpu->used_lrs = 0;
120	98	}
121	99
122	100	/* Requires the irq to be locked already */
@@ -173,6 +151,7 @@ void vgic_v3_clear_lr(struct kvm_vcpu *vcpu, int lr) · 50 unmodified lines
173	151
174	152	void vgic_v3_set_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp)
175	153	{
	154	+ struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
176	155	u32 vmcr;
177	156
178	157	/*
@@ -188,12 +167,15 @@ void vgic_v3_set_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp) · 9 unmodified lines
188	167	vmcr \|= (vmcrp->grpen0 << ICH_VMCR_ENG0_SHIFT) & ICH_VMCR_ENG0_MASK;
189	168	vmcr \|= (vmcrp->grpen1 << ICH_VMCR_ENG1_SHIFT) & ICH_VMCR_ENG1_MASK;
190	169
191		− vcpu->arch.vgic_cpu.vgic_v3.vgic_vmcr = vmcr;
	170	+ cpu_if->vgic_vmcr = vmcr;
192	171	}
193	172
194	173	void vgic_v3_get_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp)
195	174	{
196		− u32 vmcr = vcpu->arch.vgic_cpu.vgic_v3.vgic_vmcr;
	175	+ struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
	176	+ u32 vmcr;
	177	+
	178	+ vmcr = cpu_if->vgic_vmcr;
197	179
198	180	/*
199	181	* Ignore the FIQen bit, because GIC emulation always implies
@@ -386,3 +368,24 @@ int vgic_v3_probe(const struct gic_kvm_info *info) · 186 unmodified lines
386	368
387	369	return 0;
388	370	}
	371	+
	372	+void vgic_v3_load(struct kvm_vcpu *vcpu)
	373	+{
	374	+ struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
	375	+
	376	+ /*
	377	+ * If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen
	378	+ * is dependent on ICC_SRE_EL1.SRE, and we have to perform the
	379	+ * VMCR_EL2 save/restore in the world switch.
	380	+ */
	381	+ if (likely(cpu_if->vgic_sre))
	382	+ kvm_call_hyp(__vgic_v3_write_vmcr, cpu_if->vgic_vmcr);
	383	+}
	384	+
	385	+void vgic_v3_put(struct kvm_vcpu *vcpu)
	386	+{
	387	+ struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
	388	+
	389	+ if (likely(cpu_if->vgic_sre))
	390	+ cpu_if->vgic_vmcr = kvm_call_hyp(__vgic_v3_read_vmcr);
	391	+}

virt/kvm/arm/vgic/vgic.c+44−16

@@ -527,14 +527,6 @@ retry: · 526 unmodified lines
527	527	spin_unlock(&vgic_cpu->ap_list_lock);
528	528	}
529	529
530		−static inline void vgic_process_maintenance_interrupt(struct kvm_vcpu *vcpu)
531		−{
532		− if (kvm_vgic_global_state.type == VGIC_V2)
533		− vgic_v2_process_maintenance(vcpu);
534		− else
535		− vgic_v3_process_maintenance(vcpu);
536		−}
537		−
538	530	static inline void vgic_fold_lr_state(struct kvm_vcpu *vcpu)
539	531	{
540	532	if (kvm_vgic_global_state.type == VGIC_V2)
@@ -601,10 +593,8 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu) · 60 unmodified lines
601	593
602	594	DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&vgic_cpu->ap_list_lock));
603	595
604		− if (compute_ap_list_depth(vcpu) > kvm_vgic_global_state.nr_lr) {
605		− vgic_set_underflow(vcpu);
	596	+ if (compute_ap_list_depth(vcpu) > kvm_vgic_global_state.nr_lr)
606	597	vgic_sort_ap_list(vcpu);
607		− }
608	598
609	599	list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
610	600	spin_lock(&irq->irq_lock);
@@ -623,8 +613,12 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu) · 12 unmodified lines
623	613	next:
624	614	spin_unlock(&irq->irq_lock);
625	615
626		− if (count == kvm_vgic_global_state.nr_lr)
	616	+ if (count == kvm_vgic_global_state.nr_lr) {
	617	+ if (!list_is_last(&irq->ap_list,
	618	+ &vgic_cpu->ap_list_head))
	619	+ vgic_set_underflow(vcpu);
627	620	break;
	621	+ }
628	622	}
629	623
630	624	vcpu->arch.vgic_cpu.used_lrs = count;
@@ -637,18 +631,30 @@ next: · 6 unmodified lines
637	631	/* Sync back the hardware VGIC state into our emulation after a guest's run. */
638	632	void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
639	633	{
640		− if (unlikely(!vgic_initialized(vcpu->kvm)))
	634	+ struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
	635	+
	636	+ /* An empty ap_list_head implies used_lrs == 0 */
	637	+ if (list_empty(&vcpu->arch.vgic_cpu.ap_list_head))
641	638	return;
642	639
643		− vgic_process_maintenance_interrupt(vcpu);
644		− vgic_fold_lr_state(vcpu);
	640	+ if (vgic_cpu->used_lrs)
	641	+ vgic_fold_lr_state(vcpu);
645	642	vgic_prune_ap_list(vcpu);
646	643	}
647	644
648	645	/* Flush our emulation state into the GIC hardware before entering the guest. */
649	646	void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
650	647	{
651		− if (unlikely(!vgic_initialized(vcpu->kvm)))
	648	+ /*
	649	+ * If there are no virtual interrupts active or pending for this
	650	+ * VCPU, then there is no work to do and we can bail out without
	651	+ * taking any lock. There is a potential race with someone injecting
	652	+ * interrupts to the VCPU, but it is a benign race as the VCPU will
	653	+ * either observe the new interrupt before or after doing this check,
	654	+ * and introducing additional synchronization mechanism doesn't change
	655	+ * this.
	656	+ */
	657	+ if (list_empty(&vcpu->arch.vgic_cpu.ap_list_head))
652	658	return;
653	659
654	660	spin_lock(&vcpu->arch.vgic_cpu.ap_list_lock);
@@ -656,6 +662,28 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu) · 1 unmodified line
656	662	spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock);
657	663	}
658	664
	665	+void kvm_vgic_load(struct kvm_vcpu *vcpu)
	666	+{
	667	+ if (unlikely(!vgic_initialized(vcpu->kvm)))
	668	+ return;
	669	+
	670	+ if (kvm_vgic_global_state.type == VGIC_V2)
	671	+ vgic_v2_load(vcpu);
	672	+ else
	673	+ vgic_v3_load(vcpu);
	674	+}
	675	+
	676	+void kvm_vgic_put(struct kvm_vcpu *vcpu)
	677	+{
	678	+ if (unlikely(!vgic_initialized(vcpu->kvm)))
	679	+ return;
	680	+
	681	+ if (kvm_vgic_global_state.type == VGIC_V2)
	682	+ vgic_v2_put(vcpu);
	683	+ else
	684	+ vgic_v3_put(vcpu);
	685	+}
	686	+
659	687	int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
660	688	{
661	689	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;

virt/kvm/arm/vgic/vgic.h+6−2

@@ -112,7 +112,6 @@ void vgic_kick_vcpus(struct kvm *kvm); · 111 unmodified lines
112	112	int vgic_check_ioaddr(struct kvm kvm, phys_addr_t ioaddr,
113	113	phys_addr_t addr, phys_addr_t alignment);
114	114
115		−void vgic_v2_process_maintenance(struct kvm_vcpu *vcpu);
116	115	void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu);
117	116	void vgic_v2_populate_lr(struct kvm_vcpu vcpu, struct vgic_irq irq, int lr);
118	117	void vgic_v2_clear_lr(struct kvm_vcpu *vcpu, int lr);
@@ -130,6 +129,9 @@ int vgic_v2_map_resources(struct kvm *kvm); · 11 unmodified lines
130	129	int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address,
131	130	enum vgic_type);
132	131
	132	+void vgic_v2_load(struct kvm_vcpu *vcpu);
	133	+void vgic_v2_put(struct kvm_vcpu *vcpu);
	134	+
133	135	static inline void vgic_get_irq_kref(struct vgic_irq *irq)
134	136	{
135	137	if (irq->intid < VGIC_MIN_LPI)
@@ -138,7 +140,6 @@ static inline void vgic_get_irq_kref(struct vgic_irq *irq) · 2 unmodified lines
138	140	kref_get(&irq->refcount);
139	141	}
140	142
141		−void vgic_v3_process_maintenance(struct kvm_vcpu *vcpu);
142	143	void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu);
143	144	void vgic_v3_populate_lr(struct kvm_vcpu vcpu, struct vgic_irq irq, int lr);
144	145	void vgic_v3_clear_lr(struct kvm_vcpu *vcpu, int lr);
@@ -150,6 +151,9 @@ int vgic_v3_probe(const struct gic_kvm_info *info); · 5 unmodified lines
150	151	int vgic_v3_map_resources(struct kvm *kvm);
151	152	int vgic_register_redist_iodevs(struct kvm *kvm, gpa_t dist_base_address);
152	153
	154	+void vgic_v3_load(struct kvm_vcpu *vcpu);
	155	+void vgic_v3_put(struct kvm_vcpu *vcpu);
	156	+
153	157	int vgic_register_its_iodevs(struct kvm *kvm);
154	158	bool vgic_has_its(struct kvm *kvm);
155	159	int kvm_vgic_register_its_device(void);