lael-labs/linux

Documentation/virtual/kvm/api.txt+41−0

@@ -4120,3 +4120,44 @@ This capability indicates that guest using memory monotoring instructions · 4119 unmodified lines
4120	(MWAIT/MWAITX) to stop the virtual CPU will not cause a VM exit. As such time	4120	(MWAIT/MWAITX) to stop the virtual CPU will not cause a VM exit. As such time
4121	spent while virtual CPU is halted in this way will then be accounted for as	4121	spent while virtual CPU is halted in this way will then be accounted for as
4122	guest running time on the host (as opposed to e.g. HLT).	4122	guest running time on the host (as opposed to e.g. HLT).
		4123
		4124	8.9 KVM_CAP_ARM_USER_IRQ
		4125
		4126	Architectures: arm, arm64
		4127	This capability, if KVM_CHECK_EXTENSION indicates that it is available, means
		4128	that if userspace creates a VM without an in-kernel interrupt controller, it
		4129	will be notified of changes to the output level of in-kernel emulated devices,
		4130	which can generate virtual interrupts, presented to the VM.
		4131	For such VMs, on every return to userspace, the kernel
		4132	updates the vcpu's run->s.regs.device_irq_level field to represent the actual
		4133	output level of the device.
		4134
		4135	Whenever kvm detects a change in the device output level, kvm guarantees at
		4136	least one return to userspace before running the VM. This exit could either
		4137	be a KVM_EXIT_INTR or any other exit event, like KVM_EXIT_MMIO. This way,
		4138	userspace can always sample the device output level and re-compute the state of
		4139	the userspace interrupt controller. Userspace should always check the state
		4140	of run->s.regs.device_irq_level on every kvm exit.
		4141	The value in run->s.regs.device_irq_level can represent both level and edge
		4142	triggered interrupt signals, depending on the device. Edge triggered interrupt
		4143	signals will exit to userspace with the bit in run->s.regs.device_irq_level
		4144	set exactly once per edge signal.
		4145
		4146	The field run->s.regs.device_irq_level is available independent of
		4147	run->kvm_valid_regs or run->kvm_dirty_regs bits.
		4148
		4149	If KVM_CAP_ARM_USER_IRQ is supported, the KVM_CHECK_EXTENSION ioctl returns a
		4150	number larger than 0 indicating the version of this capability is implemented
		4151	and thereby which bits in in run->s.regs.device_irq_level can signal values.
		4152
		4153	Currently the following bits are defined for the device_irq_level bitmap:
		4154
		4155	KVM_CAP_ARM_USER_IRQ >= 1:
		4156
		4157	KVM_ARM_DEV_EL1_VTIMER - EL1 virtual timer
		4158	KVM_ARM_DEV_EL1_PTIMER - EL1 physical timer
		4159	KVM_ARM_DEV_PMU - ARM PMU overflow interrupt signal
		4160
		4161	Future versions of kvm may implement additional events. These will get
		4162	indicated by returning a higher number from KVM_CHECK_EXTENSION and will be
		4163	listed above.

Documentation/virtual/kvm/arm/hyp-abi.txt+53−0

@@ -0,0 +1,53 @@
	1	* Internal ABI between the kernel and HYP
	2
	3	This file documents the interaction between the Linux kernel and the
	4	hypervisor layer when running Linux as a hypervisor (for example
	5	KVM). It doesn't cover the interaction of the kernel with the
	6	hypervisor when running as a guest (under Xen, KVM or any other
	7	hypervisor), or any hypervisor-specific interaction when the kernel is
	8	used as a host.
	9
	10	On arm and arm64 (without VHE), the kernel doesn't run in hypervisor
	11	mode, but still needs to interact with it, allowing a built-in
	12	hypervisor to be either installed or torn down.
	13
	14	In order to achieve this, the kernel must be booted at HYP (arm) or
	15	EL2 (arm64), allowing it to install a set of stubs before dropping to
	16	SVC/EL1. These stubs are accessible by using a 'hvc #0' instruction,
	17	and only act on individual CPUs.
	18
	19	Unless specified otherwise, any built-in hypervisor must implement
	20	these functions (see arch/arm{,64}/include/asm/virt.h):
	21
	22	* r0/x0 = HVC_SET_VECTORS
	23	r1/x1 = vectors
	24
	25	Set HVBAR/VBAR_EL2 to 'vectors' to enable a hypervisor. 'vectors'
	26	must be a physical address, and respect the alignment requirements
	27	of the architecture. Only implemented by the initial stubs, not by
	28	Linux hypervisors.
	29
	30	* r0/x0 = HVC_RESET_VECTORS
	31
	32	Turn HYP/EL2 MMU off, and reset HVBAR/VBAR_EL2 to the initials
	33	stubs' exception vector value. This effectively disables an existing
	34	hypervisor.
	35
	36	* r0/x0 = HVC_SOFT_RESTART
	37	r1/x1 = restart address
	38	x2 = x0's value when entering the next payload (arm64)
	39	x3 = x1's value when entering the next payload (arm64)
	40	x4 = x2's value when entering the next payload (arm64)
	41
	42	Mask all exceptions, disable the MMU, move the arguments into place
	43	(arm64 only), and jump to the restart address while at HYP/EL2. This
	44	hypercall is not expected to return to its caller.
	45
	46	Any other value of r0/x0 triggers a hypervisor-specific handling,
	47	which is not documented here.
	48
	49	The return value of a stub hypercall is held by r0/x0, and is 0 on
	50	success, and HVC_STUB_ERR on error. A stub hypercall is allowed to
	51	clobber any of the caller-saved registers (x0-x18 on arm64, r0-r3 and
	52	ip on arm). It is thus recommended to use a function call to perform
	53	the hypercall.

arch/arm/boot/compressed/head.S+11−1

@@ -422,7 +422,17 @@ dtb_check_done: · 421 unmodified lines
422	cmp r0, #HYP_MODE	422	cmp r0, #HYP_MODE
423	bne 1f	423	bne 1f
424		424
425	bl __hyp_get_vectors	425	/*
		426	* Compute the address of the hyp vectors after relocation.
		427	* This requires some arithmetic since we cannot directly
		428	* reference __hyp_stub_vectors in a PC-relative way.
		429	* Call __hyp_set_vectors with the new address so that we
		430	* can HVC again after the copy.
		431	*/
		432	0: adr r0, 0b
		433	movw r1, #:lower16:__hyp_stub_vectors - 0b
		434	movt r1, #:upper16:__hyp_stub_vectors - 0b
		435	add r0, r0, r1
426	sub r0, r0, r5	436	sub r0, r0, r5
427	add r0, r0, r10	437	add r0, r0, r10
428	bl __hyp_set_vectors	438	bl __hyp_set_vectors

arch/arm/include/asm/kvm_asm.h+4−3

@@ -33,7 +33,7 @@ · 32 unmodified lines
33	#define ARM_EXCEPTION_IRQ 5	33	#define ARM_EXCEPTION_IRQ 5
34	#define ARM_EXCEPTION_FIQ 6	34	#define ARM_EXCEPTION_FIQ 6
35	#define ARM_EXCEPTION_HVC 7	35	#define ARM_EXCEPTION_HVC 7
36		36	#define ARM_EXCEPTION_HYP_GONE HVC_STUB_ERR
37	/*	37	/*
38	* The rr_lo_hi macro swaps a pair of registers depending on	38	* The rr_lo_hi macro swaps a pair of registers depending on
39	* current endianness. It is used in conjunction with ldrd and strd	39	* current endianness. It is used in conjunction with ldrd and strd
@@ -72,10 +72,11 @@ extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu); · 32 unmodified lines
72		72
73	extern void __init_stage2_translation(void);	73	extern void __init_stage2_translation(void);
74		74
75	extern void __kvm_hyp_reset(unsigned long);
76
77	extern u64 __vgic_v3_get_ich_vtr_el2(void);	75	extern u64 __vgic_v3_get_ich_vtr_el2(void);
		76	extern u64 __vgic_v3_read_vmcr(void);
		77	extern void __vgic_v3_write_vmcr(u32 vmcr);
78	extern void __vgic_v3_init_lrs(void);	78	extern void __vgic_v3_init_lrs(void);
		79
79	#endif	80	#endif
80		81
81	#endif /* __ARM_KVM_ASM_H__ */	82	#endif /* __ARM_KVM_ASM_H__ */

arch/arm/include/asm/kvm_host.h+0−6

 	kvm_call_hyp(__init_stage2_translation);
 }
-static inline void __cpu_reset_hyp_mode(unsigned long vector_ptr,
-					phys_addr_t phys_idmap_start)
-{
-	kvm_call_hyp((void *)virt_to_idmap(__kvm_hyp_reset), vector_ptr);
-}
 static inline int kvm_arch_dev_ioctl_check_extension(struct kvm *kvm, long ext)
 {
 	return 0;

arch/arm/include/asm/kvm_mmu.h+0−1

@@ -56,7 +56,6 @@ void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu); · 55 unmodified lines
56		56
57	phys_addr_t kvm_mmu_get_httbr(void);	57	phys_addr_t kvm_mmu_get_httbr(void);
58	phys_addr_t kvm_get_idmap_vector(void);	58	phys_addr_t kvm_get_idmap_vector(void);
59	phys_addr_t kvm_get_idmap_start(void);
60	int kvm_mmu_init(void);	59	int kvm_mmu_init(void);
61	void kvm_clear_hyp_idmap(void);	60	void kvm_clear_hyp_idmap(void);
62		61

arch/arm/include/asm/proc-fns.h+2−2

@@ -43,7 +43,7 @@ extern struct processor { · 42 unmodified lines
43	/*	43	/*
44	* Special stuff for a reset	44	* Special stuff for a reset
45	*/	45	*/
46	void (*reset)(unsigned long addr) __attribute__((noreturn));	46	void (*reset)(unsigned long addr, bool hvc) __attribute__((noreturn));
47	/*	47	/*
48	* Idle the processor	48	* Idle the processor
49	*/	49	*/
@@ -88,7 +88,7 @@ extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte); · 38 unmodified lines
88	#else	88	#else
89	extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext);	89	extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext);
90	#endif	90	#endif
91	extern void cpu_reset(unsigned long addr) __attribute__((noreturn));	91	extern void cpu_reset(unsigned long addr, bool hvc) __attribute__((noreturn));
92		92
93	/* These three are private to arch/arm/kernel/suspend.c */	93	/* These three are private to arch/arm/kernel/suspend.c */
94	extern void cpu_do_suspend(void *);	94	extern void cpu_do_suspend(void *);

arch/arm/include/asm/virt.h+13−1

 }
 void __hyp_set_vectors(unsigned long phys_vector_base);
-unsigned long __hyp_get_vectors(void);
+void __hyp_reset_vectors(void);
 #else
 #define __boot_cpu_mode	(SVC_MODE)
 #define sync_boot_mode()
 extern char __hyp_text_end[];
 #endif
+#else
+/* Only assembly code should need those */
+#define HVC_SET_VECTORS 0
+#define HVC_SOFT_RESTART 1
+#define HVC_RESET_VECTORS 2
+#define HVC_STUB_HCALL_NR 3
 #endif /* __ASSEMBLY__ */
+#define HVC_STUB_ERR	0xbadca11
 #endif /* ! VIRT_H */

arch/arm/include/uapi/asm/kvm.h+2−0

@@ -116,6 +116,8 @@ struct kvm_debug_exit_arch { · 115 unmodified lines
116	};	116	};
117		117
118	struct kvm_sync_regs {	118	struct kvm_sync_regs {
		119	/* Used with KVM_CAP_ARM_USER_IRQ */
		120	__u64 device_irq_level;
119	};	121	};
120		122
121	struct kvm_arch_memory_slot {	123	struct kvm_arch_memory_slot {

arch/arm/kernel/hyp-stub.S+34−9

  * (see safe_svcmode_maskall).
  */
 	@ Now install the hypervisor stub:
-	adr	r7, __hyp_stub_vectors
+	W(adr)	r7, __hyp_stub_vectors
 	mcr	p15, 4, r7, c12, c0, 0	@ set hypervisor vector base (HVBAR)
 	@ Disable all traps, so we don't get any nasty surprise
 ENDPROC(__hyp_stub_install_secondary)
 __hyp_stub_do_trap:
-	cmp	r0, #-1
+	teq	r0, #HVC_SET_VECTORS
-	mrceq	p15, 4, r0, c12, c0, 0	@ get HVBAR
+	bne	1f
-	mcrne	p15, 4, r0, c12, c0, 0	@ set HVBAR
+	mcr	p15, 4, r1, c12, c0, 0	@ set HVBAR
+	b	__hyp_stub_exit
+1:	teq	r0, #HVC_SOFT_RESTART
+	bne	1f
+	bx	r1
+1:	teq	r0, #HVC_RESET_VECTORS
+	beq	__hyp_stub_exit
+	ldr	r0, =HVC_STUB_ERR
+	__ERET
+__hyp_stub_exit:
+	mov	r0, #0
 	__ERET
 ENDPROC(__hyp_stub_do_trap)
  * so you will need to set that to something sensible at the new hypervisor's
  * initialisation entry point.
  */
-ENTRY(__hyp_get_vectors)
-	mov	r0, #-1
-ENDPROC(__hyp_get_vectors)
-	@ fall through
 ENTRY(__hyp_set_vectors)
+	mov	r1, r0
+	mov	r0, #HVC_SET_VECTORS
 	__HVC(0)
 	ret	lr
 ENDPROC(__hyp_set_vectors)
+ENTRY(__hyp_soft_restart)
+	mov	r1, r0
+	mov	r0, #HVC_SOFT_RESTART
+	__HVC(0)
+	ret	lr
+ENDPROC(__hyp_soft_restart)
+ENTRY(__hyp_reset_vectors)
+	mov	r0, #HVC_RESET_VECTORS
+	__HVC(0)
+	ret	lr
+ENDPROC(__hyp_reset_vectors)
 #ifndef ZIMAGE
 .align 2
 .L__boot_cpu_mode_offset:
 #endif
 .align 5
-__hyp_stub_vectors:
+ENTRY(__hyp_stub_vectors)
 __hyp_stub_reset:	W(b)	.
 __hyp_stub_und:		W(b)	.
 __hyp_stub_svc:		W(b)	.

arch/arm/kernel/reboot.c+5−2

@@ -12,10 +12,11 @@ · 11 unmodified lines
12		12
13	#include <asm/cacheflush.h>	13	#include <asm/cacheflush.h>
14	#include <asm/idmap.h>	14	#include <asm/idmap.h>
		15	#include <asm/virt.h>
15		16
16	#include "reboot.h"	17	#include "reboot.h"
17		18
18	typedef void (*phys_reset_t)(unsigned long);	19	typedef void (*phys_reset_t)(unsigned long, bool);
19		20
20	/*	21	/*
21	* Function pointers to optional machine specific functions	22	* Function pointers to optional machine specific functions
@@ -51,7 +52,9 @@ static void __soft_restart(void *addr) · 29 unmodified lines
51		52
52	/* Switch to the identity mapping. */	53	/* Switch to the identity mapping. */
53	phys_reset = (phys_reset_t)virt_to_idmap(cpu_reset);	54	phys_reset = (phys_reset_t)virt_to_idmap(cpu_reset);
54	phys_reset((unsigned long)addr);	55
		56	/* original stub should be restored by kvm */
		57	phys_reset((unsigned long)addr, is_hyp_mode_available());
55		58
56	/* Should never get here. */	59	/* Should never get here. */
57	BUG();	60	BUG();

arch/arm/kvm/arm.c+36−30

@@ -53,7 +53,6 @@ __asm__(".arch_extension virt"); · 52 unmodified lines
53		53
54	static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);	54	static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
55	static kvm_cpu_context_t __percpu *kvm_host_cpu_state;	55	static kvm_cpu_context_t __percpu *kvm_host_cpu_state;
56	static unsigned long hyp_default_vectors;
57		56
58	/* Per-CPU variable containing the currently running vcpu. */	57	/* Per-CPU variable containing the currently running vcpu. */
59	static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_arm_running_vcpu);	58	static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_arm_running_vcpu);
@@ -227,6 +226,13 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 167 unmodified lines
227	else	226	else
228	r = kvm->arch.vgic.msis_require_devid;	227	r = kvm->arch.vgic.msis_require_devid;
229	break;	228	break;
		229	case KVM_CAP_ARM_USER_IRQ:
		230	/*
		231	* 1: EL1_VTIMER, EL1_PTIMER, and PMU.
		232	* (bump this number if adding more devices)
		233	*/
		234	r = 1;
		235	break;
230	default:	236	default:
231	r = kvm_arch_dev_ioctl_check_extension(kvm, ext);	237	r = kvm_arch_dev_ioctl_check_extension(kvm, ext);
232	break;	238	break;
@@ -348,15 +354,14 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) · 115 unmodified lines
348	vcpu->arch.host_cpu_context = this_cpu_ptr(kvm_host_cpu_state);	354	vcpu->arch.host_cpu_context = this_cpu_ptr(kvm_host_cpu_state);
349		355
350	kvm_arm_set_running_vcpu(vcpu);	356	kvm_arm_set_running_vcpu(vcpu);
		357
		358	kvm_vgic_load(vcpu);
351	}	359	}
352		360
353	void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)	361	void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
354	{	362	{
355	/*	363	kvm_vgic_put(vcpu);
356	* The arch-generic KVM code expects the cpu field of a vcpu to be -1	364
357	* if the vcpu is no longer assigned to a cpu. This is used for the
358	* optimized make_all_cpus_request path.
359	*/
360	vcpu->cpu = -1;	365	vcpu->cpu = -1;
361		366
362	kvm_arm_set_running_vcpu(NULL);	367	kvm_arm_set_running_vcpu(NULL);
@@ -514,13 +519,7 @@ static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu) · 151 unmodified lines
514	return ret;	519	return ret;
515	}	520	}
516		521
517	/*	522	ret = kvm_timer_enable(vcpu);
518	* Enable the arch timers only if we have an in-kernel VGIC
519	* and it has been properly initialized, since we cannot handle
520	* interrupts from the virtual timer with a userspace gic.
521	*/
522	if (irqchip_in_kernel(kvm) && vgic_initialized(kvm))
523	ret = kvm_timer_enable(vcpu);
524		523
525	return ret;	524	return ret;
526	}	525	}
@@ -630,16 +629,23 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu vcpu, struct kvm_run run) · 103 unmodified lines
630	* non-preemptible context.	629	* non-preemptible context.
631	*/	630	*/
632	preempt_disable();	631	preempt_disable();
		632
633	kvm_pmu_flush_hwstate(vcpu);	633	kvm_pmu_flush_hwstate(vcpu);
		634
634	kvm_timer_flush_hwstate(vcpu);	635	kvm_timer_flush_hwstate(vcpu);
635	kvm_vgic_flush_hwstate(vcpu);	636	kvm_vgic_flush_hwstate(vcpu);
636		637
637	local_irq_disable();	638	local_irq_disable();
638		639
639	/*	640	/*
640	* Re-check atomic conditions	641	* If we have a singal pending, or need to notify a userspace
		642	* irqchip about timer or PMU level changes, then we exit (and
		643	* update the timer level state in kvm_timer_update_run
		644	* below).
641	*/	645	*/
642	if (signal_pending(current)) {	646	if (signal_pending(current) \|\|
		647	kvm_timer_should_notify_user(vcpu) \|\|
		648	kvm_pmu_should_notify_user(vcpu)) {
643	ret = -EINTR;	649	ret = -EINTR;
644	run->exit_reason = KVM_EXIT_INTR;	650	run->exit_reason = KVM_EXIT_INTR;
645	}	651	}
@@ -711,6 +717,12 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu vcpu, struct kvm_run run) · 65 unmodified lines
711	ret = handle_exit(vcpu, run, ret);	717	ret = handle_exit(vcpu, run, ret);
712	}	718	}
713		719
		720	/* Tell userspace about in-kernel device output levels */
		721	if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
		722	kvm_timer_update_run(vcpu);
		723	kvm_pmu_update_run(vcpu);
		724	}
		725
714	if (vcpu->sigset_active)	726	if (vcpu->sigset_active)
715	sigprocmask(SIG_SETMASK, &sigsaved, NULL);	727	sigprocmask(SIG_SETMASK, &sigsaved, NULL);
716	return ret;	728	return ret;
@@ -1109,8 +1121,16 @@ static void cpu_init_hyp_mode(void *dummy) · 392 unmodified lines
1109	kvm_arm_init_debug();	1121	kvm_arm_init_debug();
1110	}	1122	}
1111		1123
		1124	static void cpu_hyp_reset(void)
		1125	{
		1126	if (!is_kernel_in_hyp_mode())
		1127	__hyp_reset_vectors();
		1128	}
		1129
1112	static void cpu_hyp_reinit(void)	1130	static void cpu_hyp_reinit(void)
1113	{	1131	{
		1132	cpu_hyp_reset();
		1133
1114	if (is_kernel_in_hyp_mode()) {	1134	if (is_kernel_in_hyp_mode()) {
1115	/*	1135	/*
1116	* __cpu_init_stage2() is safe to call even if the PM	1136	* __cpu_init_stage2() is safe to call even if the PM
@@ -1118,18 +1138,10 @@ static void cpu_hyp_reinit(void) · 1 unmodified line
1118	*/	1138	*/
1119	__cpu_init_stage2();	1139	__cpu_init_stage2();
1120	} else {	1140	} else {
1121	if (__hyp_get_vectors() == hyp_default_vectors)	1141	cpu_init_hyp_mode(NULL);
1122	cpu_init_hyp_mode(NULL);
1123	}	1142	}
1124	}	1143	}
1125		1144
1126	static void cpu_hyp_reset(void)
1127	{
1128	if (!is_kernel_in_hyp_mode())
1129	__cpu_reset_hyp_mode(hyp_default_vectors,
1130	kvm_get_idmap_start());
1131	}
1132
1133	static void _kvm_arch_hardware_enable(void *discard)	1145	static void _kvm_arch_hardware_enable(void *discard)
1134	{	1146	{
1135	if (!__this_cpu_read(kvm_arm_hardware_enabled)) {	1147	if (!__this_cpu_read(kvm_arm_hardware_enabled)) {
@@ -1312,12 +1324,6 @@ static int init_hyp_mode(void) · 176 unmodified lines
1312	if (err)	1324	if (err)
1313	goto out_err;	1325	goto out_err;
1314		1326
1315	/*
1316	* It is probably enough to obtain the default on one
1317	* CPU. It's unlikely to be different on the others.
1318	*/
1319	hyp_default_vectors = __hyp_get_vectors();
1320
1321	/*	1327	/*
1322	* Allocate stack pages for Hypervisor-mode	1328	* Allocate stack pages for Hypervisor-mode
1323	*/	1329	*/

arch/arm/kvm/coproc.c+21−3

@@ -40,6 +40,24 @@ · 39 unmodified lines
40	* Co-processor emulation	40	* Co-processor emulation
41	*****************************************************************************/	41	*****************************************************************************/
42		42
		43	static bool write_to_read_only(struct kvm_vcpu *vcpu,
		44	const struct coproc_params *params)
		45	{
		46	WARN_ONCE(1, "CP15 write to read-only register\n");
		47	print_cp_instr(params);
		48	kvm_inject_undefined(vcpu);
		49	return false;
		50	}
		51
		52	static bool read_from_write_only(struct kvm_vcpu *vcpu,
		53	const struct coproc_params *params)
		54	{
		55	WARN_ONCE(1, "CP15 read to write-only register\n");
		56	print_cp_instr(params);
		57	kvm_inject_undefined(vcpu);
		58	return false;
		59	}
		60
43	/* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */	61	/* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */
44	static u32 cache_levels;	62	static u32 cache_levels;
45		63
@@ -502,15 +520,15 @@ static int emulate_cp15(struct kvm_vcpu *vcpu, · 456 unmodified lines
502	if (likely(r->access(vcpu, params, r))) {	520	if (likely(r->access(vcpu, params, r))) {
503	/* Skip instruction, since it was emulated */	521	/* Skip instruction, since it was emulated */
504	kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));	522	kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
505	return 1;
506	}	523	}
507	/* If access function fails, it should complain. */
508	} else {	524	} else {
		525	/* If access function fails, it should complain. */
509	kvm_err("Unsupported guest CP15 access at: %08lx\n",	526	kvm_err("Unsupported guest CP15 access at: %08lx\n",
510	*vcpu_pc(vcpu));	527	*vcpu_pc(vcpu));
511	print_cp_instr(params);	528	print_cp_instr(params);
		529	kvm_inject_undefined(vcpu);
512	}	530	}
513	kvm_inject_undefined(vcpu);	531
514	return 1;	532	return 1;
515	}	533	}
516		534

arch/arm/kvm/coproc.h+0−18

@@ -81,24 +81,6 @@ static inline bool read_zero(struct kvm_vcpu *vcpu, · 80 unmodified lines
81	return true;	81	return true;
82	}	82	}
83		83
84	static inline bool write_to_read_only(struct kvm_vcpu *vcpu,
85	const struct coproc_params *params)
86	{
87	kvm_debug("CP15 write to read-only register at: %08lx\n",
88	*vcpu_pc(vcpu));
89	print_cp_instr(params);
90	return false;
91	}
92
93	static inline bool read_from_write_only(struct kvm_vcpu *vcpu,
94	const struct coproc_params *params)
95	{
96	kvm_debug("CP15 read to write-only register at: %08lx\n",
97	*vcpu_pc(vcpu));
98	print_cp_instr(params);
99	return false;
100	}
101
102	/* Reset functions */	84	/* Reset functions */
103	static inline void reset_unknown(struct kvm_vcpu *vcpu,	85	static inline void reset_unknown(struct kvm_vcpu *vcpu,
104	const struct coproc_reg *r)	86	const struct coproc_reg *r)

arch/arm/kvm/handle_exit.c+8−0

@@ -160,6 +160,14 @@ int handle_exit(struct kvm_vcpu vcpu, struct kvm_run run, · 159 unmodified lines
160	case ARM_EXCEPTION_DATA_ABORT:	160	case ARM_EXCEPTION_DATA_ABORT:
161	kvm_inject_vabt(vcpu);	161	kvm_inject_vabt(vcpu);
162	return 1;	162	return 1;
		163	case ARM_EXCEPTION_HYP_GONE:
		164	/*
		165	* HYP has been reset to the hyp-stub. This happens
		166	* when a guest is pre-empted by kvm_reboot()'s
		167	* shutdown call.
		168	*/
		169	run->exit_reason = KVM_EXIT_FAIL_ENTRY;
		170	return 0;
163	default:	171	default:
164	kvm_pr_unimpl("Unsupported exception type: %d",	172	kvm_pr_unimpl("Unsupported exception type: %d",
165	exception_index);	173	exception_index);

arch/arm/kvm/hyp/hyp-entry.S+23−5

 	 */
 	pop	{r0, r1, r2}
-	/* Check for __hyp_get_vectors */
+	/*
-	cmp	r0, #-1
+	 * Check if we have a kernel function, which is guaranteed to be
-	mrceq	p15, 4, r0, c12, c0, 0	@ get HVBAR
+	 * bigger than the maximum hyp stub hypercall
-	beq	1f
+	 */
+	cmp	r0, #HVC_STUB_HCALL_NR
+	bhs	1f
+	/*
+	 * Not a kernel function, treat it as a stub hypercall.
+	 * Compute the physical address for __kvm_handle_stub_hvc
+	 * (as the code lives in the idmaped page) and branch there.
+	 * We hijack ip (r12) as a tmp register.
+	 */
+	push	{r1}
+	ldr	r1, =kimage_voffset
+	ldr	r1, [r1]
+	ldr	ip, =__kvm_handle_stub_hvc
+	sub	ip, ip, r1
+	pop	{r1}
+	bx	ip
+1:
 	push	{lr}
 	mov	lr, r0
 	blx	lr			@ Call the HYP function
 	pop	{lr}
-1:	eret
+	eret
 guest_trap:
 	load_vcpu r0			@ Load VCPU pointer to r0

arch/arm/kvm/init.S+43−8

 #include <asm/kvm_asm.h>
 #include <asm/kvm_arm.h>
 #include <asm/kvm_mmu.h>
+#include <asm/virt.h>
 /********************************************************************
  * Hypervisor initialization
  * - Setup the page tables
  * - Enable the MMU
  * - Profit! (or eret, if you only care about the code).
+ *
+ * Another possibility is to get a HYP stub hypercall.
+ * We discriminate between the two by checking if r0 contains a value
+ * that is less than HVC_STUB_HCALL_NR.
  */
 	.text
 	W(b)	.
 __do_hyp_init:
+	@ Check for a stub hypercall
+	cmp	r0, #HVC_STUB_HCALL_NR
+	blo	__kvm_handle_stub_hvc
 	@ Set stack pointer
 	mov	sp, r0
 	eret
-	@ r0 : stub vectors address
+ENTRY(__kvm_handle_stub_hvc)
-ENTRY(__kvm_hyp_reset)
+	cmp	r0, #HVC_SOFT_RESTART
+	bne	1f
+	/* The target is expected in r1 */
+	msr	ELR_hyp, r1
+	mrs	r0, cpsr
+	bic	r0, r0, #MODE_MASK
+	orr	r0, r0, #HYP_MODE
+THUMB(	orr	r0, r0, #PSR_T_BIT	)
+	msr	spsr_cxsf, r0
+	b	reset
+1:	cmp	r0, #HVC_RESET_VECTORS
+	bne	1f
+reset:
 	/* We're now in idmap, disable MMU */
 	mrc	p15, 4, r1, c1, c0, 0	@ HSCTLR
-	ldr	r2, =(HSCTLR_M | HSCTLR_A | HSCTLR_C | HSCTLR_I)
+	ldr	r0, =(HSCTLR_M | HSCTLR_A | HSCTLR_C | HSCTLR_I)
-	bic	r1, r1, r2
+	bic	r1, r1, r0
 	mcr	p15, 4, r1, c1, c0, 0	@ HSCTLR
-	/* Install stub vectors */
+	/*
-	mcr	p15, 4, r0, c12, c0, 0	@ HVBAR
+	 * Install stub vectors, using ardb's VA->PA trick.
-	isb
+	 */
+0:	adr	r0, 0b					@ PA(0)
+	movw	r1, #:lower16:__hyp_stub_vectors - 0b   @ VA(stub) - VA(0)
+	movt	r1, #:upper16:__hyp_stub_vectors - 0b
+	add	r1, r1, r0				@ PA(stub)
+	mcr	p15, 4, r1, c12, c0, 0	@ HVBAR
+	b	exit
+1:	ldr	r0, =HVC_STUB_ERR
+	eret
+exit:
+	mov	r0, #0
 	eret
-ENDPROC(__kvm_hyp_reset)
+ENDPROC(__kvm_handle_stub_hvc)
 	.ltorg

arch/arm/kvm/interrupts.S+0−4

  * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c).  Return values are
  * passed in r0 (strictly 32bit).
  *
- * A function pointer with a value of 0xffffffff has a special meaning,
- * and is used to implement __hyp_get_vectors in the same way as in
- * arch/arm/kernel/hyp_stub.S.
- *
  * The calling convention follows the standard AAPCS:
  *   r0 - r3: caller save
  *   r12:     caller save

arch/arm/kvm/mmu.c+13−23

 			     unsigned long start,
 			     unsigned long end,
 			     int (*handler)(struct kvm *kvm,
-					    gpa_t gpa, void *data),
+					    gpa_t gpa, u64 size,
+					    void *data),
 			     void *data)
 {
 	struct kvm_memslots *slots;
 	/* we only care about the pages that the guest sees */
 	kvm_for_each_memslot(memslot, slots) {
 		unsigned long hva_start, hva_end;
-		gfn_t gfn, gfn_end;
+		gfn_t gpa;
 		hva_start = max(start, memslot->userspace_addr);
 		hva_end = min(end, memslot->userspace_addr +
 		if (hva_start >= hva_end)
 			continue;
-		/*
+		gpa = hva_to_gfn_memslot(hva_start, memslot) << PAGE_SHIFT;
-		 * {gfn(page) | page intersects with [hva_start, hva_end)} =
+		ret |= handler(kvm, gpa, (u64)(hva_end - hva_start), data);
-		 * {gfn_start, gfn_start+1, ..., gfn_end-1}.
-		 */
-		gfn = hva_to_gfn_memslot(hva_start, memslot);
-		gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
-		for (; gfn < gfn_end; ++gfn) {
-			gpa_t gpa = gfn << PAGE_SHIFT;
-			ret |= handler(kvm, gpa, data);
-		}
 	}
 	return ret;
 }
-static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
+static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data)
 {
-	unmap_stage2_range(kvm, gpa, PAGE_SIZE);
+	unmap_stage2_range(kvm, gpa, size);
 	return 0;
 }
 	return 0;
 }
-static int kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
+static int kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data)
 {
 	pte_t *pte = (pte_t *)data;
+	WARN_ON(size != PAGE_SIZE);
 	/*
 	 * We can always call stage2_set_pte with KVM_S2PTE_FLAG_LOGGING_ACTIVE
 	 * flag clear because MMU notifiers will have unmapped a huge PMD before
 	handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte);
 }
-static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
+static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data)
 {
 	pmd_t *pmd;
 	pte_t *pte;
+	WARN_ON(size != PAGE_SIZE && size != PMD_SIZE);
 	pmd = stage2_get_pmd(kvm, NULL, gpa);
 	if (!pmd || pmd_none(*pmd))	/* Nothing there */
 		return 0;
 	return stage2_ptep_test_and_clear_young(pte);
 }
-static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
+static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data)
 {
 	pmd_t *pmd;
 	pte_t *pte;
+	WARN_ON(size != PAGE_SIZE && size != PMD_SIZE);
 	pmd = stage2_get_pmd(kvm, NULL, gpa);
 	if (!pmd || pmd_none(*pmd))	/* Nothing there */
 		return 0;
 	return hyp_idmap_vector;
 }
-phys_addr_t kvm_get_idmap_start(void)
-{
-	return hyp_idmap_start;
-}
 static int kvm_map_idmap_text(pgd_t *pgd)
 {
 	int err;

arch/arm/kvm/psci.c+7−1

 static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu)
 {
-	int ret = 1;
+	struct kvm *kvm = vcpu->kvm;
 	unsigned long psci_fn = vcpu_get_reg(vcpu, 0) & ~((u32) 0);
 	unsigned long val;
+	int ret = 1;
 	switch (psci_fn) {
 	case PSCI_0_2_FN_PSCI_VERSION:
 		break;
 	case PSCI_0_2_FN_CPU_ON:
 	case PSCI_0_2_FN64_CPU_ON:
+		mutex_lock(&kvm->lock);
 		val = kvm_psci_vcpu_on(vcpu);
+		mutex_unlock(&kvm->lock);
 		break;
 	case PSCI_0_2_FN_AFFINITY_INFO:
 	case PSCI_0_2_FN64_AFFINITY_INFO:
 static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu)
 {
+	struct kvm *kvm = vcpu->kvm;
 	unsigned long psci_fn = vcpu_get_reg(vcpu, 0) & ~((u32) 0);
 	unsigned long val;
 		val = PSCI_RET_SUCCESS;
 		break;
 	case KVM_PSCI_FN_CPU_ON:
+		mutex_lock(&kvm->lock);
 		val = kvm_psci_vcpu_on(vcpu);
+		mutex_unlock(&kvm->lock);
 		break;
 	default:
 		val = PSCI_RET_NOT_SUPPORTED;

arch/arm/mm/mmu.c+5−0

 #define s2_policy(policy)	0
 #endif
+unsigned long kimage_voffset __ro_after_init;
 static struct cachepolicy cache_policies[] __initdata = {
 	{
 		.policy		= "uncached",
 	empty_zero_page = virt_to_page(zero_page);
 	__flush_dcache_page(NULL, empty_zero_page);
+	/* Compute the virt/idmap offset, mostly for the sake of KVM */
+	kimage_voffset = (unsigned long)&kimage_voffset - virt_to_idmap(&kimage_voffset);
 }

arch/arm/mm/proc-v7.S+10−5

 ENDPROC(cpu_v7_proc_fin)
 /*
- *	cpu_v7_reset(loc)
+ *	cpu_v7_reset(loc, hyp)
  *
  *	Perform a soft reset of the system.  Put the CPU into the
  *	same state as it would be if it had been reset, and branch
  *	to what would be the reset vector.
  *
  *	- loc   - location to jump to for soft reset
+ *	- hyp   - indicate if restart occurs in HYP mode
  *
  *	This code must be executed using a flat identity mapping with
  *      caches disabled.
 	.align	5
 	.pushsection	.idmap.text, "ax"
 ENTRY(cpu_v7_reset)
-	mrc	p15, 0, r1, c1, c0, 0		@ ctrl register
+	mrc	p15, 0, r2, c1, c0, 0		@ ctrl register
-	bic	r1, r1, #0x1			@ ...............m
+	bic	r2, r2, #0x1			@ ...............m
- THUMB(	bic	r1, r1, #1 << 30 )		@ SCTLR.TE (Thumb exceptions)
+ THUMB(	bic	r2, r2, #1 << 30 )		@ SCTLR.TE (Thumb exceptions)
-	mcr	p15, 0, r1, c1, c0, 0		@ disable MMU
+	mcr	p15, 0, r2, c1, c0, 0		@ disable MMU
 	isb
+#ifdef CONFIG_ARM_VIRT_EXT
+	teq	r1, #0
+	bne	__hyp_soft_restart
+#endif
 	bx	r0
 ENDPROC(cpu_v7_reset)
 	.popsection

arch/arm64/include/asm/arch_gicv3.h+13−68

 #include <asm/sysreg.h>
-#define ICC_EOIR1_EL1			sys_reg(3, 0, 12, 12, 1)
-#define ICC_DIR_EL1			sys_reg(3, 0, 12, 11, 1)
-#define ICC_IAR1_EL1			sys_reg(3, 0, 12, 12, 0)
-#define ICC_SGI1R_EL1			sys_reg(3, 0, 12, 11, 5)
-#define ICC_PMR_EL1			sys_reg(3, 0, 4, 6, 0)
-#define ICC_CTLR_EL1			sys_reg(3, 0, 12, 12, 4)
-#define ICC_SRE_EL1			sys_reg(3, 0, 12, 12, 5)
-#define ICC_GRPEN1_EL1			sys_reg(3, 0, 12, 12, 7)
-#define ICC_BPR1_EL1			sys_reg(3, 0, 12, 12, 3)
-#define ICC_SRE_EL2			sys_reg(3, 4, 12, 9, 5)
-/*
- * System register definitions
- */
-#define ICH_VSEIR_EL2			sys_reg(3, 4, 12, 9, 4)
-#define ICH_HCR_EL2			sys_reg(3, 4, 12, 11, 0)
-#define ICH_VTR_EL2			sys_reg(3, 4, 12, 11, 1)
-#define ICH_MISR_EL2			sys_reg(3, 4, 12, 11, 2)
-#define ICH_EISR_EL2			sys_reg(3, 4, 12, 11, 3)
-#define ICH_ELSR_EL2			sys_reg(3, 4, 12, 11, 5)
-#define ICH_VMCR_EL2			sys_reg(3, 4, 12, 11, 7)
-#define __LR0_EL2(x)			sys_reg(3, 4, 12, 12, x)
-#define __LR8_EL2(x)			sys_reg(3, 4, 12, 13, x)
-#define ICH_LR0_EL2			__LR0_EL2(0)
-#define ICH_LR1_EL2			__LR0_EL2(1)
-#define ICH_LR2_EL2			__LR0_EL2(2)
-#define ICH_LR3_EL2			__LR0_EL2(3)
-#define ICH_LR4_EL2			__LR0_EL2(4)
-#define ICH_LR5_EL2			__LR0_EL2(5)
-#define ICH_LR6_EL2			__LR0_EL2(6)
-#define ICH_LR7_EL2			__LR0_EL2(7)
-#define ICH_LR8_EL2			__LR8_EL2(0)
-#define ICH_LR9_EL2			__LR8_EL2(1)
-#define ICH_LR10_EL2			__LR8_EL2(2)
-#define ICH_LR11_EL2			__LR8_EL2(3)
-#define ICH_LR12_EL2			__LR8_EL2(4)
-#define ICH_LR13_EL2			__LR8_EL2(5)
-#define ICH_LR14_EL2			__LR8_EL2(6)
-#define ICH_LR15_EL2			__LR8_EL2(7)
-#define __AP0Rx_EL2(x)			sys_reg(3, 4, 12, 8, x)
-#define ICH_AP0R0_EL2			__AP0Rx_EL2(0)
-#define ICH_AP0R1_EL2			__AP0Rx_EL2(1)
-#define ICH_AP0R2_EL2			__AP0Rx_EL2(2)
-#define ICH_AP0R3_EL2			__AP0Rx_EL2(3)
-#define __AP1Rx_EL2(x)			sys_reg(3, 4, 12, 9, x)
-#define ICH_AP1R0_EL2			__AP1Rx_EL2(0)
-#define ICH_AP1R1_EL2			__AP1Rx_EL2(1)
-#define ICH_AP1R2_EL2			__AP1Rx_EL2(2)
-#define ICH_AP1R3_EL2			__AP1Rx_EL2(3)
 #ifndef __ASSEMBLY__
 #include <linux/stringify.h>
 #include <asm/barrier.h>
 #include <asm/cacheflush.h>
-#define read_gicreg			read_sysreg_s
+#define read_gicreg(r)			read_sysreg_s(SYS_ ## r)
-#define write_gicreg			write_sysreg_s
+#define write_gicreg(v, r)		write_sysreg_s(v, SYS_ ## r)
 /*
  * Low-level accessors
 static inline void gic_write_eoir(u32 irq)
 {
-	write_sysreg_s(irq, ICC_EOIR1_EL1);
+	write_sysreg_s(irq, SYS_ICC_EOIR1_EL1);
 	isb();
 }
 static inline void gic_write_dir(u32 irq)
 {
-	write_sysreg_s(irq, ICC_DIR_EL1);
+	write_sysreg_s(irq, SYS_ICC_DIR_EL1);
 	isb();
 }
 {
 	u64 irqstat;
-	irqstat = read_sysreg_s(ICC_IAR1_EL1);
+	irqstat = read_sysreg_s(SYS_ICC_IAR1_EL1);
 	dsb(sy);
 	return irqstat;
 }
 	u64 irqstat;
 	nops(8);
-	irqstat = read_sysreg_s(ICC_IAR1_EL1);
+	irqstat = read_sysreg_s(SYS_ICC_IAR1_EL1);
 	nops(4);
 	mb();
 static inline void gic_write_pmr(u32 val)
 {
-	write_sysreg_s(val, ICC_PMR_EL1);
+	write_sysreg_s(val, SYS_ICC_PMR_EL1);
 }
 static inline void gic_write_ctlr(u32 val)
 {
-	write_sysreg_s(val, ICC_CTLR_EL1);
+	write_sysreg_s(val, SYS_ICC_CTLR_EL1);
 	isb();
 }
 static inline void gic_write_grpen1(u32 val)
 {
-	write_sysreg_s(val, ICC_GRPEN1_EL1);
+	write_sysreg_s(val, SYS_ICC_GRPEN1_EL1);
 	isb();
 }
 static inline void gic_write_sgi1r(u64 val)
 {
-	write_sysreg_s(val, ICC_SGI1R_EL1);
+	write_sysreg_s(val, SYS_ICC_SGI1R_EL1);
 }
 static inline u32 gic_read_sre(void)
 {
-	return read_sysreg_s(ICC_SRE_EL1);
+	return read_sysreg_s(SYS_ICC_SRE_EL1);
 }
 static inline void gic_write_sre(u32 val)
 {
-	write_sysreg_s(val, ICC_SRE_EL1);
+	write_sysreg_s(val, SYS_ICC_SRE_EL1);
 	isb();
 }
 static inline void gic_write_bpr1(u32 val)
 {
-	asm volatile("msr_s " __stringify(ICC_BPR1_EL1) ", %0" : : "r" (val));
+	write_sysreg_s(val, SYS_ICC_BPR1_EL1);
 }
 #define gic_read_typer(c)		readq_relaxed(c)

arch/arm64/include/asm/kvm_asm.h+3−2

 #define ARM_EXCEPTION_EL1_SERROR  1
 #define ARM_EXCEPTION_TRAP	  2
 /* The hyp-stub will return this for any kvm_call_hyp() call */
-#define ARM_EXCEPTION_HYP_GONE	  3
+#define ARM_EXCEPTION_HYP_GONE	  HVC_STUB_ERR
 #define KVM_ARM64_DEBUG_DIRTY_SHIFT	0
 #define KVM_ARM64_DEBUG_DIRTY		(1 << KVM_ARM64_DEBUG_DIRTY_SHIFT)
 extern char __kvm_hyp_init[];
 extern char __kvm_hyp_init_end[];
-extern char __kvm_hyp_reset[];
 extern char __kvm_hyp_vector[];
 extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
 extern u64 __vgic_v3_get_ich_vtr_el2(void);
+extern u64 __vgic_v3_read_vmcr(void);
+extern void __vgic_v3_write_vmcr(u32 vmcr);
 extern void __vgic_v3_init_lrs(void);
 extern u32 __kvm_get_mdcr_el2(void);

arch/arm64/include/asm/kvm_host.h+0−7

@@ -361,13 +361,6 @@ static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr, · 360 unmodified lines
361	__kvm_call_hyp((void *)pgd_ptr, hyp_stack_ptr, vector_ptr);	361	__kvm_call_hyp((void *)pgd_ptr, hyp_stack_ptr, vector_ptr);
362	}	362	}
363		363
364	void __kvm_hyp_teardown(void);
365	static inline void __cpu_reset_hyp_mode(unsigned long vector_ptr,
366	phys_addr_t phys_idmap_start)
367	{
368	kvm_call_hyp(__kvm_hyp_teardown, phys_idmap_start);
369	}
370
371	static inline void kvm_arch_hardware_unsetup(void) {}	364	static inline void kvm_arch_hardware_unsetup(void) {}
372	static inline void kvm_arch_sync_events(struct kvm *kvm) {}	365	static inline void kvm_arch_sync_events(struct kvm *kvm) {}
373	static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}	366	static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}

arch/arm64/include/asm/kvm_mmu.h+0−1

@@ -155,7 +155,6 @@ void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu); · 154 unmodified lines
155		155
156	phys_addr_t kvm_mmu_get_httbr(void);	156	phys_addr_t kvm_mmu_get_httbr(void);
157	phys_addr_t kvm_get_idmap_vector(void);	157	phys_addr_t kvm_get_idmap_vector(void);
158	phys_addr_t kvm_get_idmap_start(void);
159	int kvm_mmu_init(void);	158	int kvm_mmu_init(void);
160	void kvm_clear_hyp_idmap(void);	159	void kvm_clear_hyp_idmap(void);
161		160

arch/arm64/include/asm/sysreg.h+155−7

@@ -48,6 +48,8 @@ · 47 unmodified lines
48	((crn) << CRn_shift) \| ((crm) << CRm_shift) \| \	48	((crn) << CRn_shift) \| ((crm) << CRm_shift) \| \
49	((op2) << Op2_shift))	49	((op2) << Op2_shift))
50		50
		51	#define sys_insn sys_reg
		52
51	#define sys_reg_Op0(id) (((id) >> Op0_shift) & Op0_mask)	53	#define sys_reg_Op0(id) (((id) >> Op0_shift) & Op0_mask)
52	#define sys_reg_Op1(id) (((id) >> Op1_shift) & Op1_mask)	54	#define sys_reg_Op1(id) (((id) >> Op1_shift) & Op1_mask)
53	#define sys_reg_CRn(id) (((id) >> CRn_shift) & CRn_mask)	55	#define sys_reg_CRn(id) (((id) >> CRn_shift) & CRn_mask)
@@ -81,6 +83,41 @@ · 27 unmodified lines
81		83
82	#endif /* CONFIG_BROKEN_GAS_INST */	84	#endif /* CONFIG_BROKEN_GAS_INST */
83		85
		86	#define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4)
		87	#define REG_PSTATE_UAO_IMM sys_reg(0, 0, 4, 0, 3)
		88
		89	#define SET_PSTATE_PAN(x) __emit_inst(0xd5000000 \| REG_PSTATE_PAN_IMM \| \
		90	(!!x)<<8 \| 0x1f)
		91	#define SET_PSTATE_UAO(x) __emit_inst(0xd5000000 \| REG_PSTATE_UAO_IMM \| \
		92	(!!x)<<8 \| 0x1f)
		93
		94	#define SYS_DC_ISW sys_insn(1, 0, 7, 6, 2)
		95	#define SYS_DC_CSW sys_insn(1, 0, 7, 10, 2)
		96	#define SYS_DC_CISW sys_insn(1, 0, 7, 14, 2)
		97
		98	#define SYS_OSDTRRX_EL1 sys_reg(2, 0, 0, 0, 2)
		99	#define SYS_MDCCINT_EL1 sys_reg(2, 0, 0, 2, 0)
		100	#define SYS_MDSCR_EL1 sys_reg(2, 0, 0, 2, 2)
		101	#define SYS_OSDTRTX_EL1 sys_reg(2, 0, 0, 3, 2)
		102	#define SYS_OSECCR_EL1 sys_reg(2, 0, 0, 6, 2)
		103	#define SYS_DBGBVRn_EL1(n) sys_reg(2, 0, 0, n, 4)
		104	#define SYS_DBGBCRn_EL1(n) sys_reg(2, 0, 0, n, 5)
		105	#define SYS_DBGWVRn_EL1(n) sys_reg(2, 0, 0, n, 6)
		106	#define SYS_DBGWCRn_EL1(n) sys_reg(2, 0, 0, n, 7)
		107	#define SYS_MDRAR_EL1 sys_reg(2, 0, 1, 0, 0)
		108	#define SYS_OSLAR_EL1 sys_reg(2, 0, 1, 0, 4)
		109	#define SYS_OSLSR_EL1 sys_reg(2, 0, 1, 1, 4)
		110	#define SYS_OSDLR_EL1 sys_reg(2, 0, 1, 3, 4)
		111	#define SYS_DBGPRCR_EL1 sys_reg(2, 0, 1, 4, 4)
		112	#define SYS_DBGCLAIMSET_EL1 sys_reg(2, 0, 7, 8, 6)
		113	#define SYS_DBGCLAIMCLR_EL1 sys_reg(2, 0, 7, 9, 6)
		114	#define SYS_DBGAUTHSTATUS_EL1 sys_reg(2, 0, 7, 14, 6)
		115	#define SYS_MDCCSR_EL0 sys_reg(2, 3, 0, 1, 0)
		116	#define SYS_DBGDTR_EL0 sys_reg(2, 3, 0, 4, 0)
		117	#define SYS_DBGDTRRX_EL0 sys_reg(2, 3, 0, 5, 0)
		118	#define SYS_DBGDTRTX_EL0 sys_reg(2, 3, 0, 5, 0)
		119	#define SYS_DBGVCR32_EL2 sys_reg(2, 4, 0, 7, 0)
		120
84	#define SYS_MIDR_EL1 sys_reg(3, 0, 0, 0, 0)	121	#define SYS_MIDR_EL1 sys_reg(3, 0, 0, 0, 0)
85	#define SYS_MPIDR_EL1 sys_reg(3, 0, 0, 0, 5)	122	#define SYS_MPIDR_EL1 sys_reg(3, 0, 0, 0, 5)
86	#define SYS_REVIDR_EL1 sys_reg(3, 0, 0, 0, 6)	123	#define SYS_REVIDR_EL1 sys_reg(3, 0, 0, 0, 6)
@@ -88,6 +125,7 @@ · 1 unmodified line
88	#define SYS_ID_PFR0_EL1 sys_reg(3, 0, 0, 1, 0)	125	#define SYS_ID_PFR0_EL1 sys_reg(3, 0, 0, 1, 0)
89	#define SYS_ID_PFR1_EL1 sys_reg(3, 0, 0, 1, 1)	126	#define SYS_ID_PFR1_EL1 sys_reg(3, 0, 0, 1, 1)
90	#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)	127	#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
		128	#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
91	#define SYS_ID_MMFR0_EL1 sys_reg(3, 0, 0, 1, 4)	129	#define SYS_ID_MMFR0_EL1 sys_reg(3, 0, 0, 1, 4)
92	#define SYS_ID_MMFR1_EL1 sys_reg(3, 0, 0, 1, 5)	130	#define SYS_ID_MMFR1_EL1 sys_reg(3, 0, 0, 1, 5)
93	#define SYS_ID_MMFR2_EL1 sys_reg(3, 0, 0, 1, 6)	131	#define SYS_ID_MMFR2_EL1 sys_reg(3, 0, 0, 1, 6)
@@ -118,17 +156,127 @@ · 24 unmodified lines
118	#define SYS_ID_AA64MMFR1_EL1 sys_reg(3, 0, 0, 7, 1)	156	#define SYS_ID_AA64MMFR1_EL1 sys_reg(3, 0, 0, 7, 1)
119	#define SYS_ID_AA64MMFR2_EL1 sys_reg(3, 0, 0, 7, 2)	157	#define SYS_ID_AA64MMFR2_EL1 sys_reg(3, 0, 0, 7, 2)
120		158
121	#define SYS_CNTFRQ_EL0 sys_reg(3, 3, 14, 0, 0)	159	#define SYS_SCTLR_EL1 sys_reg(3, 0, 1, 0, 0)
		160	#define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1)
		161	#define SYS_CPACR_EL1 sys_reg(3, 0, 1, 0, 2)
		162
		163	#define SYS_TTBR0_EL1 sys_reg(3, 0, 2, 0, 0)
		164	#define SYS_TTBR1_EL1 sys_reg(3, 0, 2, 0, 1)
		165	#define SYS_TCR_EL1 sys_reg(3, 0, 2, 0, 2)
		166
		167	#define SYS_ICC_PMR_EL1 sys_reg(3, 0, 4, 6, 0)
		168
		169	#define SYS_AFSR0_EL1 sys_reg(3, 0, 5, 1, 0)
		170	#define SYS_AFSR1_EL1 sys_reg(3, 0, 5, 1, 1)
		171	#define SYS_ESR_EL1 sys_reg(3, 0, 5, 2, 0)
		172	#define SYS_FAR_EL1 sys_reg(3, 0, 6, 0, 0)
		173	#define SYS_PAR_EL1 sys_reg(3, 0, 7, 4, 0)
		174
		175	#define SYS_PMINTENSET_EL1 sys_reg(3, 0, 9, 14, 1)
		176	#define SYS_PMINTENCLR_EL1 sys_reg(3, 0, 9, 14, 2)
		177
		178	#define SYS_MAIR_EL1 sys_reg(3, 0, 10, 2, 0)
		179	#define SYS_AMAIR_EL1 sys_reg(3, 0, 10, 3, 0)
		180
		181	#define SYS_VBAR_EL1 sys_reg(3, 0, 12, 0, 0)
		182
		183	#define SYS_ICC_DIR_EL1 sys_reg(3, 0, 12, 11, 1)
		184	#define SYS_ICC_SGI1R_EL1 sys_reg(3, 0, 12, 11, 5)
		185	#define SYS_ICC_IAR1_EL1 sys_reg(3, 0, 12, 12, 0)
		186	#define SYS_ICC_EOIR1_EL1 sys_reg(3, 0, 12, 12, 1)
		187	#define SYS_ICC_BPR1_EL1 sys_reg(3, 0, 12, 12, 3)
		188	#define SYS_ICC_CTLR_EL1 sys_reg(3, 0, 12, 12, 4)
		189	#define SYS_ICC_SRE_EL1 sys_reg(3, 0, 12, 12, 5)
		190	#define SYS_ICC_GRPEN1_EL1 sys_reg(3, 0, 12, 12, 7)
		191
		192	#define SYS_CONTEXTIDR_EL1 sys_reg(3, 0, 13, 0, 1)
		193	#define SYS_TPIDR_EL1 sys_reg(3, 0, 13, 0, 4)
		194
		195	#define SYS_CNTKCTL_EL1 sys_reg(3, 0, 14, 1, 0)
		196
		197	#define SYS_CLIDR_EL1 sys_reg(3, 1, 0, 0, 1)
		198	#define SYS_AIDR_EL1 sys_reg(3, 1, 0, 0, 7)
		199
		200	#define SYS_CSSELR_EL1 sys_reg(3, 2, 0, 0, 0)
		201
122	#define SYS_CTR_EL0 sys_reg(3, 3, 0, 0, 1)	202	#define SYS_CTR_EL0 sys_reg(3, 3, 0, 0, 1)
123	#define SYS_DCZID_EL0 sys_reg(3, 3, 0, 0, 7)	203	#define SYS_DCZID_EL0 sys_reg(3, 3, 0, 0, 7)
124		204
125	#define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4)	205	#define SYS_PMCR_EL0 sys_reg(3, 3, 9, 12, 0)
126	#define REG_PSTATE_UAO_IMM sys_reg(0, 0, 4, 0, 3)	206	#define SYS_PMCNTENSET_EL0 sys_reg(3, 3, 9, 12, 1)
		207	#define SYS_PMCNTENCLR_EL0 sys_reg(3, 3, 9, 12, 2)
		208	#define SYS_PMOVSCLR_EL0 sys_reg(3, 3, 9, 12, 3)
		209	#define SYS_PMSWINC_EL0 sys_reg(3, 3, 9, 12, 4)
		210	#define SYS_PMSELR_EL0 sys_reg(3, 3, 9, 12, 5)
		211	#define SYS_PMCEID0_EL0 sys_reg(3, 3, 9, 12, 6)
		212	#define SYS_PMCEID1_EL0 sys_reg(3, 3, 9, 12, 7)
		213	#define SYS_PMCCNTR_EL0 sys_reg(3, 3, 9, 13, 0)
		214	#define SYS_PMXEVTYPER_EL0 sys_reg(3, 3, 9, 13, 1)
		215	#define SYS_PMXEVCNTR_EL0 sys_reg(3, 3, 9, 13, 2)
		216	#define SYS_PMUSERENR_EL0 sys_reg(3, 3, 9, 14, 0)
		217	#define SYS_PMOVSSET_EL0 sys_reg(3, 3, 9, 14, 3)
		218
		219	#define SYS_TPIDR_EL0 sys_reg(3, 3, 13, 0, 2)
		220	#define SYS_TPIDRRO_EL0 sys_reg(3, 3, 13, 0, 3)
127		221
128	#define SET_PSTATE_PAN(x) __emit_inst(0xd5000000 \| REG_PSTATE_PAN_IMM \| \	222	#define SYS_CNTFRQ_EL0 sys_reg(3, 3, 14, 0, 0)
129	(!!x)<<8 \| 0x1f)	223
130	#define SET_PSTATE_UAO(x) __emit_inst(0xd5000000 \| REG_PSTATE_UAO_IMM \| \	224	#define SYS_CNTP_TVAL_EL0 sys_reg(3, 3, 14, 2, 0)
131	(!!x)<<8 \| 0x1f)	225	#define SYS_CNTP_CTL_EL0 sys_reg(3, 3, 14, 2, 1)
		226	#define SYS_CNTP_CVAL_EL0 sys_reg(3, 3, 14, 2, 2)
		227
		228	#define __PMEV_op2(n) ((n) & 0x7)
		229	#define __CNTR_CRm(n) (0x8 \| (((n) >> 3) & 0x3))
		230	#define SYS_PMEVCNTRn_EL0(n) sys_reg(3, 3, 14, __CNTR_CRm(n), __PMEV_op2(n))
		231	#define __TYPER_CRm(n) (0xc \| (((n) >> 3) & 0x3))
		232	#define SYS_PMEVTYPERn_EL0(n) sys_reg(3, 3, 14, __TYPER_CRm(n), __PMEV_op2(n))
		233
		234	#define SYS_PMCCFILTR_EL0 sys_reg (3, 3, 14, 15, 7)
		235
		236	#define SYS_DACR32_EL2 sys_reg(3, 4, 3, 0, 0)
		237	#define SYS_IFSR32_EL2 sys_reg(3, 4, 5, 0, 1)
		238	#define SYS_FPEXC32_EL2 sys_reg(3, 4, 5, 3, 0)
		239
		240	#define __SYS__AP0Rx_EL2(x) sys_reg(3, 4, 12, 8, x)
		241	#define SYS_ICH_AP0R0_EL2 __SYS__AP0Rx_EL2(0)
		242	#define SYS_ICH_AP0R1_EL2 __SYS__AP0Rx_EL2(1)
		243	#define SYS_ICH_AP0R2_EL2 __SYS__AP0Rx_EL2(2)
		244	#define SYS_ICH_AP0R3_EL2 __SYS__AP0Rx_EL2(3)
		245
		246	#define __SYS__AP1Rx_EL2(x) sys_reg(3, 4, 12, 9, x)
		247	#define SYS_ICH_AP1R0_EL2 __SYS__AP1Rx_EL2(0)
		248	#define SYS_ICH_AP1R1_EL2 __SYS__AP1Rx_EL2(1)
		249	#define SYS_ICH_AP1R2_EL2 __SYS__AP1Rx_EL2(2)
		250	#define SYS_ICH_AP1R3_EL2 __SYS__AP1Rx_EL2(3)
		251
		252	#define SYS_ICH_VSEIR_EL2 sys_reg(3, 4, 12, 9, 4)
		253	#define SYS_ICC_SRE_EL2 sys_reg(3, 4, 12, 9, 5)
		254	#define SYS_ICH_HCR_EL2 sys_reg(3, 4, 12, 11, 0)
		255	#define SYS_ICH_VTR_EL2 sys_reg(3, 4, 12, 11, 1)
		256	#define SYS_ICH_MISR_EL2 sys_reg(3, 4, 12, 11, 2)
		257	#define SYS_ICH_EISR_EL2 sys_reg(3, 4, 12, 11, 3)
		258	#define SYS_ICH_ELSR_EL2 sys_reg(3, 4, 12, 11, 5)
		259	#define SYS_ICH_VMCR_EL2 sys_reg(3, 4, 12, 11, 7)
		260
		261	#define __SYS__LR0_EL2(x) sys_reg(3, 4, 12, 12, x)
		262	#define SYS_ICH_LR0_EL2 __SYS__LR0_EL2(0)
		263	#define SYS_ICH_LR1_EL2 __SYS__LR0_EL2(1)
		264	#define SYS_ICH_LR2_EL2 __SYS__LR0_EL2(2)
		265	#define SYS_ICH_LR3_EL2 __SYS__LR0_EL2(3)
		266	#define SYS_ICH_LR4_EL2 __SYS__LR0_EL2(4)
		267	#define SYS_ICH_LR5_EL2 __SYS__LR0_EL2(5)
		268	#define SYS_ICH_LR6_EL2 __SYS__LR0_EL2(6)
		269	#define SYS_ICH_LR7_EL2 __SYS__LR0_EL2(7)
		270
		271	#define __SYS__LR8_EL2(x) sys_reg(3, 4, 12, 13, x)
		272	#define SYS_ICH_LR8_EL2 __SYS__LR8_EL2(0)
		273	#define SYS_ICH_LR9_EL2 __SYS__LR8_EL2(1)
		274	#define SYS_ICH_LR10_EL2 __SYS__LR8_EL2(2)
		275	#define SYS_ICH_LR11_EL2 __SYS__LR8_EL2(3)
		276	#define SYS_ICH_LR12_EL2 __SYS__LR8_EL2(4)
		277	#define SYS_ICH_LR13_EL2 __SYS__LR8_EL2(5)
		278	#define SYS_ICH_LR14_EL2 __SYS__LR8_EL2(6)
		279	#define SYS_ICH_LR15_EL2 __SYS__LR8_EL2(7)
132		280
133	/* Common SCTLR_ELx flags. */	281	/* Common SCTLR_ELx flags. */
134	#define SCTLR_ELx_EE (1 << 25)	282	#define SCTLR_ELx_EE (1 << 25)

arch/arm64/include/asm/virt.h+22−9

 #define __ASM__VIRT_H
 /*
- * The arm64 hcall implementation uses x0 to specify the hcall type. A value
+ * The arm64 hcall implementation uses x0 to specify the hcall
- * less than 0xfff indicates a special hcall, such as get/set vector.
+ * number. A value less than HVC_STUB_HCALL_NR indicates a special
- * Any other value is used as a pointer to the function to call.
+ * hcall, such as set vector. Any other value is handled in a
+ * hypervisor specific way.
+ *
+ * The hypercall is allowed to clobber any of the caller-saved
+ * registers (x0-x18), so it is advisable to use it through the
+ * indirection of a function call (as implemented in hyp-stub.S).
  */
-/* HVC_GET_VECTORS - Return the value of the vbar_el2 register. */
-#define HVC_GET_VECTORS 0
 /*
  * HVC_SET_VECTORS - Set the value of the vbar_el2 register.
  *
  * @x1: Physical address of the new vector table.
  */
-#define HVC_SET_VECTORS 1
+#define HVC_SET_VECTORS 0
 /*
  * HVC_SOFT_RESTART - CPU soft reset, used by the cpu_soft_restart routine.
  */
-#define HVC_SOFT_RESTART 2
+#define HVC_SOFT_RESTART 1
+/*
+ * HVC_RESET_VECTORS - Restore the vectors to the original HYP stubs
+ */
+#define HVC_RESET_VECTORS 2
+/* Max number of HYP stub hypercalls */
+#define HVC_STUB_HCALL_NR 3
+/* Error returned when an invalid stub number is passed into x0 */
+#define HVC_STUB_ERR	0xbadca11
 #define BOOT_CPU_MODE_EL1	(0xe11)
 #define BOOT_CPU_MODE_EL2	(0xe12)
 extern u32 __boot_cpu_mode[2];
 void __hyp_set_vectors(phys_addr_t phys_vector_base);
-phys_addr_t __hyp_get_vectors(void);
+void __hyp_reset_vectors(void);
 /* Reports the availability of HYP mode */
 static inline bool is_hyp_mode_available(void)

arch/arm64/include/uapi/asm/kvm.h+2−0

@@ -145,6 +145,8 @@ struct kvm_debug_exit_arch { · 144 unmodified lines
145	#define KVM_GUESTDBG_USE_HW (1 << 17)	145	#define KVM_GUESTDBG_USE_HW (1 << 17)
146		146
147	struct kvm_sync_regs {	147	struct kvm_sync_regs {
		148	/* Used with KVM_CAP_ARM_USER_IRQ */
		149	__u64 device_irq_level;
148	};	150	};
149		151
150	struct kvm_arch_memory_slot {	152	struct kvm_arch_memory_slot {

arch/arm64/kernel/head.S+4−4

@@ -594,14 +594,14 @@ set_hcr: · 593 unmodified lines
594	cmp x0, #1	594	cmp x0, #1
595	b.ne 3f	595	b.ne 3f
596		596
597	mrs_s x0, ICC_SRE_EL2	597	mrs_s x0, SYS_ICC_SRE_EL2
598	orr x0, x0, #ICC_SRE_EL2_SRE // Set ICC_SRE_EL2.SRE==1	598	orr x0, x0, #ICC_SRE_EL2_SRE // Set ICC_SRE_EL2.SRE==1
599	orr x0, x0, #ICC_SRE_EL2_ENABLE // Set ICC_SRE_EL2.Enable==1	599	orr x0, x0, #ICC_SRE_EL2_ENABLE // Set ICC_SRE_EL2.Enable==1
600	msr_s ICC_SRE_EL2, x0	600	msr_s SYS_ICC_SRE_EL2, x0
601	isb // Make sure SRE is now set	601	isb // Make sure SRE is now set
602	mrs_s x0, ICC_SRE_EL2 // Read SRE back,	602	mrs_s x0, SYS_ICC_SRE_EL2 // Read SRE back,
603	tbz x0, #0, 3f // and check that it sticks	603	tbz x0, #0, 3f // and check that it sticks
604	msr_s ICH_HCR_EL2, xzr // Reset ICC_HCR_EL2 to defaults	604	msr_s SYS_ICH_HCR_EL2, xzr // Reset ICC_HCR_EL2 to defaults
605		605
606	3:	606	3:
607	#endif	607	#endif

arch/arm64/kernel/hyp-stub.S+14−24

 	.align 11
 el1_sync:
-	mrs	x30, esr_el2
+	cmp	x0, #HVC_SET_VECTORS
-	lsr	x30, x30, #ESR_ELx_EC_SHIFT
-	cmp	x30, #ESR_ELx_EC_HVC64
-	b.ne	9f				// Not an HVC trap
-	cmp	x0, #HVC_GET_VECTORS
-	b.ne	1f
-	mrs	x0, vbar_el2
-	b	9f
-1:	cmp	x0, #HVC_SET_VECTORS
 	b.ne	2f
 	msr	vbar_el2, x1
 	b	9f
 	mov	x1, x3
 	br	x4				// no return
+3:	cmp	x0, #HVC_RESET_VECTORS
+	beq	9f				// Nothing to reset!
 	/* Someone called kvm_call_hyp() against the hyp-stub... */
-3:	mov	x0, #ARM_EXCEPTION_HYP_GONE
+	ldr	x0, =HVC_STUB_ERR
+	eret
-9:	eret
+9:	mov	x0, xzr
+	eret
 ENDPROC(el1_sync)
 .macro invalid_vector	label
  * initialisation entry point.
  */
-ENTRY(__hyp_get_vectors)
-	str	lr, [sp, #-16]!
-	mov	x0, #HVC_GET_VECTORS
-	hvc	#0
-	ldr	lr, [sp], #16
-	ret
-ENDPROC(__hyp_get_vectors)
 ENTRY(__hyp_set_vectors)
-	str	lr, [sp, #-16]!
 	mov	x1, x0
 	mov	x0, #HVC_SET_VECTORS
 	hvc	#0
-	ldr	lr, [sp], #16
 	ret
 ENDPROC(__hyp_set_vectors)
+ENTRY(__hyp_reset_vectors)
+	mov	x0, #HVC_RESET_VECTORS
+	hvc	#0
+	ret
+ENDPROC(__hyp_reset_vectors)

arch/arm64/kvm/hyp-init.S+36−10

@@ -22,6 +22,7 @@ · 21 unmodified lines
22	#include <asm/kvm_mmu.h>	22	#include <asm/kvm_mmu.h>
23	#include <asm/pgtable-hwdef.h>	23	#include <asm/pgtable-hwdef.h>
24	#include <asm/sysreg.h>	24	#include <asm/sysreg.h>
		25	#include <asm/virt.h>
25		26
26	.text	27	.text
27	.pushsection .hyp.idmap.text, "ax"	28	.pushsection .hyp.idmap.text, "ax"
@@ -58,6 +59,9 @@ __invalid: · 30 unmodified lines
58	* x2: HYP vectors	59	* x2: HYP vectors
59	*/	60	*/
60	__do_hyp_init:	61	__do_hyp_init:
		62	/* Check for a stub HVC call */
		63	cmp x0, #HVC_STUB_HCALL_NR
		64	b.lo __kvm_handle_stub_hvc
61		65
62	msr ttbr0_el2, x0	66	msr ttbr0_el2, x0
63		67
@@ -119,23 +123,45 @@ __do_hyp_init: · 55 unmodified lines
119	eret	123	eret
120	ENDPROC(__kvm_hyp_init)	124	ENDPROC(__kvm_hyp_init)
121		125
		126	ENTRY(__kvm_handle_stub_hvc)
		127	cmp x0, #HVC_SOFT_RESTART
		128	b.ne 1f
		129
		130	/* This is where we're about to jump, staying at EL2 */
		131	msr elr_el2, x1
		132	mov x0, #(PSR_F_BIT \| PSR_I_BIT \| PSR_A_BIT \| PSR_D_BIT \| PSR_MODE_EL2h)
		133	msr spsr_el2, x0
		134
		135	/* Shuffle the arguments, and don't come back */
		136	mov x0, x2
		137	mov x1, x3
		138	mov x2, x4
		139	b reset
		140
		141	1: cmp x0, #HVC_RESET_VECTORS
		142	b.ne 1f
		143	reset:
122	/*	144	/*
123	* Reset kvm back to the hyp stub.	145	* Reset kvm back to the hyp stub. Do not clobber x0-x4 in
		146	* case we coming via HVC_SOFT_RESTART.
124	*/	147	*/
125	ENTRY(__kvm_hyp_reset)	148	mrs x5, sctlr_el2
126	/* We're now in idmap, disable MMU */	149	ldr x6, =SCTLR_ELx_FLAGS
127	mrs x0, sctlr_el2	150	bic x5, x5, x6 // Clear SCTL_M and etc
128	ldr x1, =SCTLR_ELx_FLAGS	151	msr sctlr_el2, x5
129	bic x0, x0, x1 // Clear SCTL_M and etc
130	msr sctlr_el2, x0
131	isb	152	isb
132		153
133	/* Install stub vectors */	154	/* Install stub vectors */
134	adr_l x0, __hyp_stub_vectors	155	adr_l x5, __hyp_stub_vectors
135	msr vbar_el2, x0	156	msr vbar_el2, x5
		157	mov x0, xzr
		158	eret
136		159
		160	1: /* Bad stub call */
		161	ldr x0, =HVC_STUB_ERR
137	eret	162	eret
138	ENDPROC(__kvm_hyp_reset)	163
		164	ENDPROC(__kvm_handle_stub_hvc)
139		165
140	.ltorg	166	.ltorg
141		167

arch/arm64/kvm/hyp.S+1−4

  * passed in x0.
  *
  * A function pointer with a value less than 0xfff has a special meaning,
- * and is used to implement __hyp_get_vectors in the same way as in
+ * and is used to implement hyp stubs in the same way as in
  * arch/arm64/kernel/hyp_stub.S.
- * HVC behaves as a 'bl' call and will clobber lr.
  */
 ENTRY(__kvm_call_hyp)
 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
-	str     lr, [sp, #-16]!
 	hvc	#0
-	ldr     lr, [sp], #16
 	ret
 alternative_else_nop_endif
 	b	__vhe_hyp_call

arch/arm64/kvm/hyp/hyp-entry.S+21−22

@@ -32,17 +32,17 @@ · 31 unmodified lines
32	* Shuffle the parameters before calling the function	32	* Shuffle the parameters before calling the function
33	* pointed to in x0. Assumes parameters in x[1,2,3].	33	* pointed to in x0. Assumes parameters in x[1,2,3].
34	*/	34	*/
		35	str lr, [sp, #-16]!
35	mov lr, x0	36	mov lr, x0
36	mov x0, x1	37	mov x0, x1
37	mov x1, x2	38	mov x1, x2
38	mov x2, x3	39	mov x2, x3
39	blr lr	40	blr lr
		41	ldr lr, [sp], #16
40	.endm	42	.endm
41		43
42	ENTRY(__vhe_hyp_call)	44	ENTRY(__vhe_hyp_call)
43	str lr, [sp, #-16]!
44	do_el2_call	45	do_el2_call
45	ldr lr, [sp], #16
46	/*	46	/*
47	* We used to rely on having an exception return to get	47	* We used to rely on having an exception return to get
48	* an implicit isb. In the E2H case, we don't have it anymore.	48	* an implicit isb. In the E2H case, we don't have it anymore.
@@ -53,21 +53,6 @@ ENTRY(__vhe_hyp_call) · 4 unmodified lines
53	ret	53	ret
54	ENDPROC(__vhe_hyp_call)	54	ENDPROC(__vhe_hyp_call)
55		55
56	/*
57	* Compute the idmap address of __kvm_hyp_reset based on the idmap
58	* start passed as a parameter, and jump there.
59	*
60	* x0: HYP phys_idmap_start
61	*/
62	ENTRY(__kvm_hyp_teardown)
63	mov x4, x0
64	adr_l x3, __kvm_hyp_reset
65
66	/* insert __kvm_hyp_reset()s offset into phys_idmap_start */
67	bfi x4, x3, #0, #PAGE_SHIFT
68	br x4
69	ENDPROC(__kvm_hyp_teardown)
70
71	el1_sync: // Guest trapped into EL2	56	el1_sync: // Guest trapped into EL2
72	stp x0, x1, [sp, #-16]!	57	stp x0, x1, [sp, #-16]!
73		58
@@ -87,10 +72,24 @@ alternative_endif · 13 unmodified lines
87	/* Here, we're pretty sure the host called HVC. */	72	/* Here, we're pretty sure the host called HVC. */
88	ldp x0, x1, [sp], #16	73	ldp x0, x1, [sp], #16
89		74
90	cmp x0, #HVC_GET_VECTORS	75	/* Check for a stub HVC call */
91	b.ne 1f	76	cmp x0, #HVC_STUB_HCALL_NR
92	mrs x0, vbar_el2	77	b.hs 1f
93	b 2f	78
		79	/*
		80	* Compute the idmap address of __kvm_handle_stub_hvc and
		81	* jump there. Since we use kimage_voffset, do not use the
		82	* HYP VA for __kvm_handle_stub_hvc, but the kernel VA instead
		83	* (by loading it from the constant pool).
		84	*
		85	* Preserve x0-x4, which may contain stub parameters.
		86	*/
		87	ldr x5, =__kvm_handle_stub_hvc
		88	ldr_l x6, kimage_voffset
		89
		90	/* x5 = __pa(x5) */
		91	sub x5, x5, x6
		92	br x5
94		93
95	1:	94	1:
96	/*	95	/*
@@ -99,7 +98,7 @@ alternative_endif · 2 unmodified lines
99	kern_hyp_va x0	98	kern_hyp_va x0
100	do_el2_call	99	do_el2_call
101		100
102	2: eret	101	eret
103		102
104	el1_trap:	103	el1_trap:
105	/*	104	/*

arch/arm64/kvm/sys_regs.c+180−316

@@ -55,6 +55,15 @@ · 54 unmodified lines
55	* 64bit interface.	55	* 64bit interface.
56	*/	56	*/
57		57
		58	static bool read_from_write_only(struct kvm_vcpu *vcpu,
		59	const struct sys_reg_params *params)
		60	{
		61	WARN_ONCE(1, "Unexpected sys_reg read to write-only register\n");
		62	print_sys_reg_instr(params);
		63	kvm_inject_undefined(vcpu);
		64	return false;
		65	}
		66
58	/* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */	67	/* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */
59	static u32 cache_levels;	68	static u32 cache_levels;
60		69
@@ -460,35 +469,35 @@ static void reset_pmcr(struct kvm_vcpu vcpu, const struct sys_reg_desc r) · 399 unmodified lines
460	vcpu_sys_reg(vcpu, PMCR_EL0) = val;	469	vcpu_sys_reg(vcpu, PMCR_EL0) = val;
461	}	470	}
462		471
463	static bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu)	472	static bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)
464	{	473	{
465	u64 reg = vcpu_sys_reg(vcpu, PMUSERENR_EL0);	474	u64 reg = vcpu_sys_reg(vcpu, PMUSERENR_EL0);
		475	bool enabled = (reg & flags) \|\| vcpu_mode_priv(vcpu);
466		476
467	return !((reg & ARMV8_PMU_USERENR_EN) \|\| vcpu_mode_priv(vcpu));	477	if (!enabled)
		478	kvm_inject_undefined(vcpu);
		479
		480	return !enabled;
468	}	481	}
469		482
470	static bool pmu_write_swinc_el0_disabled(struct kvm_vcpu *vcpu)	483	static bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu)
471	{	484	{
472	u64 reg = vcpu_sys_reg(vcpu, PMUSERENR_EL0);	485	return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_EN);
		486	}
473		487
474	return !((reg & (ARMV8_PMU_USERENR_SW \| ARMV8_PMU_USERENR_EN))	488	static bool pmu_write_swinc_el0_disabled(struct kvm_vcpu *vcpu)
475	\|\| vcpu_mode_priv(vcpu));	489	{
		490	return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_SW \| ARMV8_PMU_USERENR_EN);
476	}	491	}
477		492
478	static bool pmu_access_cycle_counter_el0_disabled(struct kvm_vcpu *vcpu)	493	static bool pmu_access_cycle_counter_el0_disabled(struct kvm_vcpu *vcpu)
479	{	494	{
480	u64 reg = vcpu_sys_reg(vcpu, PMUSERENR_EL0);	495	return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_CR \| ARMV8_PMU_USERENR_EN);
481
482	return !((reg & (ARMV8_PMU_USERENR_CR \| ARMV8_PMU_USERENR_EN))
483	\|\| vcpu_mode_priv(vcpu));
484	}	496	}
485		497
486	static bool pmu_access_event_counter_el0_disabled(struct kvm_vcpu *vcpu)	498	static bool pmu_access_event_counter_el0_disabled(struct kvm_vcpu *vcpu)
487	{	499	{
488	u64 reg = vcpu_sys_reg(vcpu, PMUSERENR_EL0);	500	return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_ER \| ARMV8_PMU_USERENR_EN);
489
490	return !((reg & (ARMV8_PMU_USERENR_ER \| ARMV8_PMU_USERENR_EN))
491	\|\| vcpu_mode_priv(vcpu));
492	}	501	}
493		502
494	static bool access_pmcr(struct kvm_vcpu vcpu, struct sys_reg_params p,	503	static bool access_pmcr(struct kvm_vcpu vcpu, struct sys_reg_params p,
@@ -567,8 +576,10 @@ static bool pmu_counter_idx_valid(struct kvm_vcpu *vcpu, u64 idx) · 72 unmodified lines
567		576
568	pmcr = vcpu_sys_reg(vcpu, PMCR_EL0);	577	pmcr = vcpu_sys_reg(vcpu, PMCR_EL0);
569	val = (pmcr >> ARMV8_PMU_PMCR_N_SHIFT) & ARMV8_PMU_PMCR_N_MASK;	578	val = (pmcr >> ARMV8_PMU_PMCR_N_SHIFT) & ARMV8_PMU_PMCR_N_MASK;
570	if (idx >= val && idx != ARMV8_PMU_CYCLE_IDX)	579	if (idx >= val && idx != ARMV8_PMU_CYCLE_IDX) {
		580	kvm_inject_undefined(vcpu);
571	return false;	581	return false;
		582	}
572		583
573	return true;	584	return true;
574	}	585	}
@@ -707,8 +718,10 @@ static bool access_pminten(struct kvm_vcpu vcpu, struct sys_reg_params p, · 132 unmodified lines
707	if (!kvm_arm_pmu_v3_ready(vcpu))	718	if (!kvm_arm_pmu_v3_ready(vcpu))
708	return trap_raz_wi(vcpu, p, r);	719	return trap_raz_wi(vcpu, p, r);
709		720
710	if (!vcpu_mode_priv(vcpu))	721	if (!vcpu_mode_priv(vcpu)) {
		722	kvm_inject_undefined(vcpu);
711	return false;	723	return false;
		724	}
712		725
713	if (p->is_write) {	726	if (p->is_write) {
714	u64 val = p->regval & mask;	727	u64 val = p->regval & mask;
@@ -759,16 +772,15 @@ static bool access_pmswinc(struct kvm_vcpu vcpu, struct sys_reg_params p, · 44 unmodified lines
759	if (!kvm_arm_pmu_v3_ready(vcpu))	772	if (!kvm_arm_pmu_v3_ready(vcpu))
760	return trap_raz_wi(vcpu, p, r);	773	return trap_raz_wi(vcpu, p, r);
761		774
		775	if (!p->is_write)
		776	return read_from_write_only(vcpu, p);
		777
762	if (pmu_write_swinc_el0_disabled(vcpu))	778	if (pmu_write_swinc_el0_disabled(vcpu))
763	return false;	779	return false;
764		780
765	if (p->is_write) {	781	mask = kvm_pmu_valid_counter_mask(vcpu);
766	mask = kvm_pmu_valid_counter_mask(vcpu);	782	kvm_pmu_software_increment(vcpu, p->regval & mask);
767	kvm_pmu_software_increment(vcpu, p->regval & mask);	783	return true;
768	return true;
769	}
770
771	return false;
772	}	784	}
773		785
774	static bool access_pmuserenr(struct kvm_vcpu vcpu, struct sys_reg_params p,	786	static bool access_pmuserenr(struct kvm_vcpu vcpu, struct sys_reg_params p,
@@ -778,8 +790,10 @@ static bool access_pmuserenr(struct kvm_vcpu vcpu, struct sys_reg_params p, · 3 unmodified lines
778	return trap_raz_wi(vcpu, p, r);	790	return trap_raz_wi(vcpu, p, r);
779		791
780	if (p->is_write) {	792	if (p->is_write) {
781	if (!vcpu_mode_priv(vcpu))	793	if (!vcpu_mode_priv(vcpu)) {
		794	kvm_inject_undefined(vcpu);
782	return false;	795	return false;
		796	}
783		797
784	vcpu_sys_reg(vcpu, PMUSERENR_EL0) = p->regval	798	vcpu_sys_reg(vcpu, PMUSERENR_EL0) = p->regval
785	& ARMV8_PMU_USERENR_MASK;	799	& ARMV8_PMU_USERENR_MASK;
@@ -793,31 +807,23 @@ static bool access_pmuserenr(struct kvm_vcpu vcpu, struct sys_reg_params p, · 7 unmodified lines
793		807
794	/* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */	808	/* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
795	#define DBG_BCR_BVR_WCR_WVR_EL1(n) \	809	#define DBG_BCR_BVR_WCR_WVR_EL1(n) \
796	/* DBGBVRn_EL1 */ \	810	{ SYS_DESC(SYS_DBGBVRn_EL1(n)), \
797	{ Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b100), \
798	trap_bvr, reset_bvr, n, 0, get_bvr, set_bvr }, \	811	trap_bvr, reset_bvr, n, 0, get_bvr, set_bvr }, \
799	/* DBGBCRn_EL1 */ \	812	{ SYS_DESC(SYS_DBGBCRn_EL1(n)), \
800	{ Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b101), \
801	trap_bcr, reset_bcr, n, 0, get_bcr, set_bcr }, \	813	trap_bcr, reset_bcr, n, 0, get_bcr, set_bcr }, \
802	/* DBGWVRn_EL1 */ \	814	{ SYS_DESC(SYS_DBGWVRn_EL1(n)), \
803	{ Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b110), \
804	trap_wvr, reset_wvr, n, 0, get_wvr, set_wvr }, \	815	trap_wvr, reset_wvr, n, 0, get_wvr, set_wvr }, \
805	/* DBGWCRn_EL1 */ \	816	{ SYS_DESC(SYS_DBGWCRn_EL1(n)), \
806	{ Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b111), \
807	trap_wcr, reset_wcr, n, 0, get_wcr, set_wcr }	817	trap_wcr, reset_wcr, n, 0, get_wcr, set_wcr }
808		818
809	/* Macro to expand the PMEVCNTRn_EL0 register */	819	/* Macro to expand the PMEVCNTRn_EL0 register */
810	#define PMU_PMEVCNTR_EL0(n) \	820	#define PMU_PMEVCNTR_EL0(n) \
811	/* PMEVCNTRn_EL0 */ \	821	{ SYS_DESC(SYS_PMEVCNTRn_EL0(n)), \
812	{ Op0(0b11), Op1(0b011), CRn(0b1110), \
813	CRm((0b1000 \| (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
814	access_pmu_evcntr, reset_unknown, (PMEVCNTR0_EL0 + n), }	822	access_pmu_evcntr, reset_unknown, (PMEVCNTR0_EL0 + n), }
815		823
816	/* Macro to expand the PMEVTYPERn_EL0 register */	824	/* Macro to expand the PMEVTYPERn_EL0 register */
817	#define PMU_PMEVTYPER_EL0(n) \	825	#define PMU_PMEVTYPER_EL0(n) \
818	/* PMEVTYPERn_EL0 */ \	826	{ SYS_DESC(SYS_PMEVTYPERn_EL0(n)), \
819	{ Op0(0b11), Op1(0b011), CRn(0b1110), \
820	CRm((0b1100 \| (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \
821	access_pmu_evtyper, reset_unknown, (PMEVTYPER0_EL0 + n), }	827	access_pmu_evtyper, reset_unknown, (PMEVTYPER0_EL0 + n), }
822		828
823	static bool access_cntp_tval(struct kvm_vcpu *vcpu,	829	static bool access_cntp_tval(struct kvm_vcpu *vcpu,
@@ -887,24 +893,14 @@ static bool access_cntp_cval(struct kvm_vcpu *vcpu, · 63 unmodified lines
887	* more demanding guest...	893	* more demanding guest...
888	*/	894	*/
889	static const struct sys_reg_desc sys_reg_descs[] = {	895	static const struct sys_reg_desc sys_reg_descs[] = {
890	/* DC ISW */	896	{ SYS_DESC(SYS_DC_ISW), access_dcsw },
891	{ Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b0110), Op2(0b010),	897	{ SYS_DESC(SYS_DC_CSW), access_dcsw },
892	access_dcsw },	898	{ SYS_DESC(SYS_DC_CISW), access_dcsw },
893	/* DC CSW */
894	{ Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b1010), Op2(0b010),
895	access_dcsw },
896	/* DC CISW */
897	{ Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b1110), Op2(0b010),
898	access_dcsw },
899		899
900	DBG_BCR_BVR_WCR_WVR_EL1(0),	900	DBG_BCR_BVR_WCR_WVR_EL1(0),
901	DBG_BCR_BVR_WCR_WVR_EL1(1),	901	DBG_BCR_BVR_WCR_WVR_EL1(1),
902	/* MDCCINT_EL1 */	902	{ SYS_DESC(SYS_MDCCINT_EL1), trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
903	{ Op0(0b10), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b000),	903	{ SYS_DESC(SYS_MDSCR_EL1), trap_debug_regs, reset_val, MDSCR_EL1, 0 },
904	trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
905	/* MDSCR_EL1 */
906	{ Op0(0b10), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b010),
907	trap_debug_regs, reset_val, MDSCR_EL1, 0 },
908	DBG_BCR_BVR_WCR_WVR_EL1(2),	904	DBG_BCR_BVR_WCR_WVR_EL1(2),
909	DBG_BCR_BVR_WCR_WVR_EL1(3),	905	DBG_BCR_BVR_WCR_WVR_EL1(3),
910	DBG_BCR_BVR_WCR_WVR_EL1(4),	906	DBG_BCR_BVR_WCR_WVR_EL1(4),
@@ -920,179 +916,77 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 9 unmodified lines
920	DBG_BCR_BVR_WCR_WVR_EL1(14),	916	DBG_BCR_BVR_WCR_WVR_EL1(14),
921	DBG_BCR_BVR_WCR_WVR_EL1(15),	917	DBG_BCR_BVR_WCR_WVR_EL1(15),
922		918
923	/* MDRAR_EL1 */	919	{ SYS_DESC(SYS_MDRAR_EL1), trap_raz_wi },
924	{ Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000),	920	{ SYS_DESC(SYS_OSLAR_EL1), trap_raz_wi },
925	trap_raz_wi },	921	{ SYS_DESC(SYS_OSLSR_EL1), trap_oslsr_el1 },
926	/* OSLAR_EL1 */	922	{ SYS_DESC(SYS_OSDLR_EL1), trap_raz_wi },
927	{ Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b100),	923	{ SYS_DESC(SYS_DBGPRCR_EL1), trap_raz_wi },
928	trap_raz_wi },	924	{ SYS_DESC(SYS_DBGCLAIMSET_EL1), trap_raz_wi },
929	/* OSLSR_EL1 */	925	{ SYS_DESC(SYS_DBGCLAIMCLR_EL1), trap_raz_wi },
930	{ Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0001), Op2(0b100),	926	{ SYS_DESC(SYS_DBGAUTHSTATUS_EL1), trap_dbgauthstatus_el1 },
931	trap_oslsr_el1 },	927
932	/* OSDLR_EL1 */	928	{ SYS_DESC(SYS_MDCCSR_EL0), trap_raz_wi },
933	{ Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0011), Op2(0b100),	929	{ SYS_DESC(SYS_DBGDTR_EL0), trap_raz_wi },
934	trap_raz_wi },	930	// DBGDTR[TR]X_EL0 share the same encoding
935	/* DBGPRCR_EL1 */	931	{ SYS_DESC(SYS_DBGDTRTX_EL0), trap_raz_wi },
936	{ Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0100), Op2(0b100),	932
937	trap_raz_wi },	933	{ SYS_DESC(SYS_DBGVCR32_EL2), NULL, reset_val, DBGVCR32_EL2, 0 },
938	/* DBGCLAIMSET_EL1 */	934
939	{ Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1000), Op2(0b110),	935	{ SYS_DESC(SYS_MPIDR_EL1), NULL, reset_mpidr, MPIDR_EL1 },
940	trap_raz_wi },	936	{ SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
941	/* DBGCLAIMCLR_EL1 */	937	{ SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
942	{ Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1001), Op2(0b110),	938	{ SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 },
943	trap_raz_wi },	939	{ SYS_DESC(SYS_TTBR1_EL1), access_vm_reg, reset_unknown, TTBR1_EL1 },
944	/* DBGAUTHSTATUS_EL1 */	940	{ SYS_DESC(SYS_TCR_EL1), access_vm_reg, reset_val, TCR_EL1, 0 },
945	{ Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1110), Op2(0b110),	941
946	trap_dbgauthstatus_el1 },	942	{ SYS_DESC(SYS_AFSR0_EL1), access_vm_reg, reset_unknown, AFSR0_EL1 },
947		943	{ SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
948	/* MDCCSR_EL1 */	944	{ SYS_DESC(SYS_ESR_EL1), access_vm_reg, reset_unknown, ESR_EL1 },
949	{ Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0001), Op2(0b000),	945	{ SYS_DESC(SYS_FAR_EL1), access_vm_reg, reset_unknown, FAR_EL1 },
950	trap_raz_wi },	946	{ SYS_DESC(SYS_PAR_EL1), NULL, reset_unknown, PAR_EL1 },
951	/* DBGDTR_EL0 */	947
952	{ Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0100), Op2(0b000),	948	{ SYS_DESC(SYS_PMINTENSET_EL1), access_pminten, reset_unknown, PMINTENSET_EL1 },
953	trap_raz_wi },	949	{ SYS_DESC(SYS_PMINTENCLR_EL1), access_pminten, NULL, PMINTENSET_EL1 },
954	/* DBGDTR[TR]X_EL0 */	950
955	{ Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0101), Op2(0b000),	951	{ SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 },
956	trap_raz_wi },	952	{ SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 },
957		953
958	/* DBGVCR32_EL2 */	954	{ SYS_DESC(SYS_VBAR_EL1), NULL, reset_val, VBAR_EL1, 0 },
959	{ Op0(0b10), Op1(0b100), CRn(0b0000), CRm(0b0111), Op2(0b000),	955
960	NULL, reset_val, DBGVCR32_EL2, 0 },	956	{ SYS_DESC(SYS_ICC_SGI1R_EL1), access_gic_sgi },
961		957	{ SYS_DESC(SYS_ICC_SRE_EL1), access_gic_sre },
962	/* MPIDR_EL1 */	958
963	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b101),	959	{ SYS_DESC(SYS_CONTEXTIDR_EL1), access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
964	NULL, reset_mpidr, MPIDR_EL1 },	960	{ SYS_DESC(SYS_TPIDR_EL1), NULL, reset_unknown, TPIDR_EL1 },
965	/* SCTLR_EL1 */	961
966	{ Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000),	962	{ SYS_DESC(SYS_CNTKCTL_EL1), NULL, reset_val, CNTKCTL_EL1, 0},
967	access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },	963
968	/* CPACR_EL1 */	964	{ SYS_DESC(SYS_CSSELR_EL1), NULL, reset_unknown, CSSELR_EL1 },
969	{ Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b010),	965
970	NULL, reset_val, CPACR_EL1, 0 },	966	{ SYS_DESC(SYS_PMCR_EL0), access_pmcr, reset_pmcr, },
971	/* TTBR0_EL1 */	967	{ SYS_DESC(SYS_PMCNTENSET_EL0), access_pmcnten, reset_unknown, PMCNTENSET_EL0 },
972	{ Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b000),	968	{ SYS_DESC(SYS_PMCNTENCLR_EL0), access_pmcnten, NULL, PMCNTENSET_EL0 },
973	access_vm_reg, reset_unknown, TTBR0_EL1 },	969	{ SYS_DESC(SYS_PMOVSCLR_EL0), access_pmovs, NULL, PMOVSSET_EL0 },
974	/* TTBR1_EL1 */	970	{ SYS_DESC(SYS_PMSWINC_EL0), access_pmswinc, reset_unknown, PMSWINC_EL0 },
975	{ Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b001),	971	{ SYS_DESC(SYS_PMSELR_EL0), access_pmselr, reset_unknown, PMSELR_EL0 },
976	access_vm_reg, reset_unknown, TTBR1_EL1 },	972	{ SYS_DESC(SYS_PMCEID0_EL0), access_pmceid },
977	/* TCR_EL1 */	973	{ SYS_DESC(SYS_PMCEID1_EL0), access_pmceid },
978	{ Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b010),	974	{ SYS_DESC(SYS_PMCCNTR_EL0), access_pmu_evcntr, reset_unknown, PMCCNTR_EL0 },
979	access_vm_reg, reset_val, TCR_EL1, 0 },	975	{ SYS_DESC(SYS_PMXEVTYPER_EL0), access_pmu_evtyper },
980		976	{ SYS_DESC(SYS_PMXEVCNTR_EL0), access_pmu_evcntr },
981	/* AFSR0_EL1 */	977	/*
982	{ Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0001), Op2(0b000),	978	* PMUSERENR_EL0 resets as unknown in 64bit mode while it resets as zero
983	access_vm_reg, reset_unknown, AFSR0_EL1 },
984	/* AFSR1_EL1 */
985	{ Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0001), Op2(0b001),
986	access_vm_reg, reset_unknown, AFSR1_EL1 },
987	/* ESR_EL1 */
988	{ Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0010), Op2(0b000),
989	access_vm_reg, reset_unknown, ESR_EL1 },
990	/* FAR_EL1 */
991	{ Op0(0b11), Op1(0b000), CRn(0b0110), CRm(0b0000), Op2(0b000),
992	access_vm_reg, reset_unknown, FAR_EL1 },
993	/* PAR_EL1 */
994	{ Op0(0b11), Op1(0b000), CRn(0b0111), CRm(0b0100), Op2(0b000),
995	NULL, reset_unknown, PAR_EL1 },
996
997	/* PMINTENSET_EL1 */
998	{ Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b001),
999	access_pminten, reset_unknown, PMINTENSET_EL1 },
1000	/* PMINTENCLR_EL1 */
1001	{ Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b010),
1002	access_pminten, NULL, PMINTENSET_EL1 },
1003
1004	/* MAIR_EL1 */
1005	{ Op0(0b11), Op1(0b000), CRn(0b1010), CRm(0b0010), Op2(0b000),
1006	access_vm_reg, reset_unknown, MAIR_EL1 },
1007	/* AMAIR_EL1 */
1008	{ Op0(0b11), Op1(0b000), CRn(0b1010), CRm(0b0011), Op2(0b000),
1009	access_vm_reg, reset_amair_el1, AMAIR_EL1 },
1010
1011	/* VBAR_EL1 */
1012	{ Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b0000), Op2(0b000),
1013	NULL, reset_val, VBAR_EL1, 0 },
1014
1015	/* ICC_SGI1R_EL1 */
1016	{ Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b1011), Op2(0b101),
1017	access_gic_sgi },
1018	/* ICC_SRE_EL1 */
1019	{ Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b1100), Op2(0b101),
1020	access_gic_sre },
1021
1022	/* CONTEXTIDR_EL1 */
1023	{ Op0(0b11), Op1(0b000), CRn(0b1101), CRm(0b0000), Op2(0b001),
1024	access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
1025	/* TPIDR_EL1 */
1026	{ Op0(0b11), Op1(0b000), CRn(0b1101), CRm(0b0000), Op2(0b100),
1027	NULL, reset_unknown, TPIDR_EL1 },
1028
1029	/* CNTKCTL_EL1 */
1030	{ Op0(0b11), Op1(0b000), CRn(0b1110), CRm(0b0001), Op2(0b000),
1031	NULL, reset_val, CNTKCTL_EL1, 0},
1032
1033	/* CSSELR_EL1 */
1034	{ Op0(0b11), Op1(0b010), CRn(0b0000), CRm(0b0000), Op2(0b000),
1035	NULL, reset_unknown, CSSELR_EL1 },
1036
1037	/* PMCR_EL0 */
1038	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b000),
1039	access_pmcr, reset_pmcr, },
1040	/* PMCNTENSET_EL0 */
1041	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b001),
1042	access_pmcnten, reset_unknown, PMCNTENSET_EL0 },
1043	/* PMCNTENCLR_EL0 */
1044	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b010),
1045	access_pmcnten, NULL, PMCNTENSET_EL0 },
1046	/* PMOVSCLR_EL0 */
1047	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b011),
1048	access_pmovs, NULL, PMOVSSET_EL0 },
1049	/* PMSWINC_EL0 */
1050	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b100),
1051	access_pmswinc, reset_unknown, PMSWINC_EL0 },
1052	/* PMSELR_EL0 */
1053	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b101),
1054	access_pmselr, reset_unknown, PMSELR_EL0 },
1055	/* PMCEID0_EL0 */
1056	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b110),
1057	access_pmceid },
1058	/* PMCEID1_EL0 */
1059	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b111),
1060	access_pmceid },
1061	/* PMCCNTR_EL0 */
1062	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b000),
1063	access_pmu_evcntr, reset_unknown, PMCCNTR_EL0 },
1064	/* PMXEVTYPER_EL0 */
1065	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b001),
1066	access_pmu_evtyper },
1067	/* PMXEVCNTR_EL0 */
1068	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b010),
1069	access_pmu_evcntr },
1070	/* PMUSERENR_EL0
1071	* This register resets as unknown in 64bit mode while it resets as zero
1072	* in 32bit mode. Here we choose to reset it as zero for consistency.	979	* in 32bit mode. Here we choose to reset it as zero for consistency.
1073	*/	980	*/
1074	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b000),	981	{ SYS_DESC(SYS_PMUSERENR_EL0), access_pmuserenr, reset_val, PMUSERENR_EL0, 0 },
1075	access_pmuserenr, reset_val, PMUSERENR_EL0, 0 },	982	{ SYS_DESC(SYS_PMOVSSET_EL0), access_pmovs, reset_unknown, PMOVSSET_EL0 },
1076	/* PMOVSSET_EL0 */	983
1077	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b011),	984	{ SYS_DESC(SYS_TPIDR_EL0), NULL, reset_unknown, TPIDR_EL0 },
1078	access_pmovs, reset_unknown, PMOVSSET_EL0 },	985	{ SYS_DESC(SYS_TPIDRRO_EL0), NULL, reset_unknown, TPIDRRO_EL0 },
1079		986
1080	/* TPIDR_EL0 */	987	{ SYS_DESC(SYS_CNTP_TVAL_EL0), access_cntp_tval },
1081	{ Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b010),	988	{ SYS_DESC(SYS_CNTP_CTL_EL0), access_cntp_ctl },
1082	NULL, reset_unknown, TPIDR_EL0 },	989	{ SYS_DESC(SYS_CNTP_CVAL_EL0), access_cntp_cval },
1083	/* TPIDRRO_EL0 */
1084	{ Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b011),
1085	NULL, reset_unknown, TPIDRRO_EL0 },
1086
1087	/* CNTP_TVAL_EL0 */
1088	{ Op0(0b11), Op1(0b011), CRn(0b1110), CRm(0b0010), Op2(0b000),
1089	access_cntp_tval },
1090	/* CNTP_CTL_EL0 */
1091	{ Op0(0b11), Op1(0b011), CRn(0b1110), CRm(0b0010), Op2(0b001),
1092	access_cntp_ctl },
1093	/* CNTP_CVAL_EL0 */
1094	{ Op0(0b11), Op1(0b011), CRn(0b1110), CRm(0b0010), Op2(0b010),
1095	access_cntp_cval },
1096		990
1097	/* PMEVCNTRn_EL0 */	991	/* PMEVCNTRn_EL0 */
1098	PMU_PMEVCNTR_EL0(0),	992	PMU_PMEVCNTR_EL0(0),
@@ -1158,22 +1052,15 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 59 unmodified lines
1158	PMU_PMEVTYPER_EL0(28),	1052	PMU_PMEVTYPER_EL0(28),
1159	PMU_PMEVTYPER_EL0(29),	1053	PMU_PMEVTYPER_EL0(29),
1160	PMU_PMEVTYPER_EL0(30),	1054	PMU_PMEVTYPER_EL0(30),
1161	/* PMCCFILTR_EL0	1055	/*
1162	* This register resets as unknown in 64bit mode while it resets as zero	1056	* PMCCFILTR_EL0 resets as unknown in 64bit mode while it resets as zero
1163	* in 32bit mode. Here we choose to reset it as zero for consistency.	1057	* in 32bit mode. Here we choose to reset it as zero for consistency.
1164	*/	1058	*/
1165	{ Op0(0b11), Op1(0b011), CRn(0b1110), CRm(0b1111), Op2(0b111),	1059	{ SYS_DESC(SYS_PMCCFILTR_EL0), access_pmu_evtyper, reset_val, PMCCFILTR_EL0, 0 },
1166	access_pmu_evtyper, reset_val, PMCCFILTR_EL0, 0 },	1060
1167		1061	{ SYS_DESC(SYS_DACR32_EL2), NULL, reset_unknown, DACR32_EL2 },
1168	/* DACR32_EL2 */	1062	{ SYS_DESC(SYS_IFSR32_EL2), NULL, reset_unknown, IFSR32_EL2 },
1169	{ Op0(0b11), Op1(0b100), CRn(0b0011), CRm(0b0000), Op2(0b000),	1063	{ SYS_DESC(SYS_FPEXC32_EL2), NULL, reset_val, FPEXC32_EL2, 0x70 },
1170	NULL, reset_unknown, DACR32_EL2 },
1171	/* IFSR32_EL2 */
1172	{ Op0(0b11), Op1(0b100), CRn(0b0101), CRm(0b0000), Op2(0b001),
1173	NULL, reset_unknown, IFSR32_EL2 },
1174	/* FPEXC32_EL2 */
1175	{ Op0(0b11), Op1(0b100), CRn(0b0101), CRm(0b0011), Op2(0b000),
1176	NULL, reset_val, FPEXC32_EL2, 0x70 },
1177	};	1064	};
1178		1065
1179	static bool trap_dbgidr(struct kvm_vcpu *vcpu,	1066	static bool trap_dbgidr(struct kvm_vcpu *vcpu,
@@ -1557,6 +1444,22 @@ int kvm_handle_cp14_load_store(struct kvm_vcpu vcpu, struct kvm_run run) · 377 unmodified lines
1557	return 1;	1444	return 1;
1558	}	1445	}
1559		1446
		1447	static void perform_access(struct kvm_vcpu *vcpu,
		1448	struct sys_reg_params *params,
		1449	const struct sys_reg_desc *r)
		1450	{
		1451	/*
		1452	* Not having an accessor means that we have configured a trap
		1453	* that we don't know how to handle. This certainly qualifies
		1454	* as a gross bug that should be fixed right away.
		1455	*/
		1456	BUG_ON(!r->access);
		1457
		1458	/* Skip instruction if instructed so */
		1459	if (likely(r->access(vcpu, params, r)))
		1460	kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
		1461	}
		1462
1560	/*	1463	/*
1561	* emulate_cp -- tries to match a sys_reg access in a handling table, and	1464	* emulate_cp -- tries to match a sys_reg access in a handling table, and
1562	* call the corresponding trap handler.	1465	* call the corresponding trap handler.
@@ -1580,20 +1483,8 @@ static int emulate_cp(struct kvm_vcpu *vcpu, · 17 unmodified lines
1580	r = find_reg(params, table, num);	1483	r = find_reg(params, table, num);
1581		1484
1582	if (r) {	1485	if (r) {
1583	/*	1486	perform_access(vcpu, params, r);
1584	* Not having an accessor means that we have	1487	return 0;
1585	* configured a trap that we don't know how to
1586	* handle. This certainly qualifies as a gross bug
1587	* that should be fixed right away.
1588	*/
1589	BUG_ON(!r->access);
1590
1591	if (likely(r->access(vcpu, params, r))) {
1592	/* Skip instruction, since it was emulated */
1593	kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
1594	/* Handled */
1595	return 0;
1596	}
1597	}	1488	}
1598		1489
1599	/* Not handled */	1490	/* Not handled */
@@ -1660,20 +1551,25 @@ static int kvm_handle_cp_64(struct kvm_vcpu *vcpu, · 60 unmodified lines
1660	params.regval \|= vcpu_get_reg(vcpu, Rt2) << 32;	1551	params.regval \|= vcpu_get_reg(vcpu, Rt2) << 32;
1661	}	1552	}
1662		1553
1663	if (!emulate_cp(vcpu, &params, target_specific, nr_specific))	1554	/*
1664	goto out;	1555	* Try to emulate the coprocessor access using the target
1665	if (!emulate_cp(vcpu, &params, global, nr_global))	1556	* specific table first, and using the global table afterwards.
1666	goto out;	1557	* If either of the tables contains a handler, handle the
1667		1558	* potential register operation in the case of a read and return
1668	unhandled_cp_access(vcpu, &params);	1559	* with success.
		1560	*/
		1561	if (!emulate_cp(vcpu, &params, target_specific, nr_specific) \|\|
		1562	!emulate_cp(vcpu, &params, global, nr_global)) {
		1563	/* Split up the value between registers for the read side */
		1564	if (!params.is_write) {
		1565	vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
		1566	vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));
		1567	}
1669		1568
1670	out:	1569	return 1;
1671	/* Split up the value between registers for the read side */
1672	if (!params.is_write) {
1673	vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
1674	vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));
1675	}	1570	}
1676		1571
		1572	unhandled_cp_access(vcpu, &params);
1677	return 1;	1573	return 1;
1678	}	1574	}
1679		1575
@@ -1763,26 +1659,13 @@ static int emulate_sys_reg(struct kvm_vcpu *vcpu, · 83 unmodified lines
1763	r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));	1659	r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
1764		1660
1765	if (likely(r)) {	1661	if (likely(r)) {
1766	/*	1662	perform_access(vcpu, params, r);
1767	* Not having an accessor means that we have
1768	* configured a trap that we don't know how to
1769	* handle. This certainly qualifies as a gross bug
1770	* that should be fixed right away.
1771	*/
1772	BUG_ON(!r->access);
1773
1774	if (likely(r->access(vcpu, params, r))) {
1775	/* Skip instruction, since it was emulated */
1776	kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
1777	return 1;
1778	}
1779	/* If access function fails, it should complain. */
1780	} else {	1663	} else {
1781	kvm_err("Unsupported guest sys_reg access at: %lx\n",	1664	kvm_err("Unsupported guest sys_reg access at: %lx\n",
1782	*vcpu_pc(vcpu));	1665	*vcpu_pc(vcpu));
1783	print_sys_reg_instr(params);	1666	print_sys_reg_instr(params);
		1667	kvm_inject_undefined(vcpu);
1784	}	1668	}
1785	kvm_inject_undefined(vcpu);
1786	return 1;	1669	return 1;
1787	}	1670	}
1788		1671
@@ -1932,44 +1815,25 @@ FUNCTION_INVARIANT(aidr_el1) · 143 unmodified lines
1932		1815
1933	/* ->val is filled in by kvm_sys_reg_table_init() */	1816	/* ->val is filled in by kvm_sys_reg_table_init() */
1934	static struct sys_reg_desc invariant_sys_regs[] = {	1817	static struct sys_reg_desc invariant_sys_regs[] = {
1935	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b000),	1818	{ SYS_DESC(SYS_MIDR_EL1), NULL, get_midr_el1 },
1936	NULL, get_midr_el1 },	1819	{ SYS_DESC(SYS_REVIDR_EL1), NULL, get_revidr_el1 },
1937	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b110),	1820	{ SYS_DESC(SYS_ID_PFR0_EL1), NULL, get_id_pfr0_el1 },
1938	NULL, get_revidr_el1 },	1821	{ SYS_DESC(SYS_ID_PFR1_EL1), NULL, get_id_pfr1_el1 },
1939	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b000),	1822	{ SYS_DESC(SYS_ID_DFR0_EL1), NULL, get_id_dfr0_el1 },
1940	NULL, get_id_pfr0_el1 },	1823	{ SYS_DESC(SYS_ID_AFR0_EL1), NULL, get_id_afr0_el1 },
1941	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b001),	1824	{ SYS_DESC(SYS_ID_MMFR0_EL1), NULL, get_id_mmfr0_el1 },
1942	NULL, get_id_pfr1_el1 },	1825	{ SYS_DESC(SYS_ID_MMFR1_EL1), NULL, get_id_mmfr1_el1 },
1943	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b010),	1826	{ SYS_DESC(SYS_ID_MMFR2_EL1), NULL, get_id_mmfr2_el1 },
1944	NULL, get_id_dfr0_el1 },	1827	{ SYS_DESC(SYS_ID_MMFR3_EL1), NULL, get_id_mmfr3_el1 },
1945	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b011),	1828	{ SYS_DESC(SYS_ID_ISAR0_EL1), NULL, get_id_isar0_el1 },
1946	NULL, get_id_afr0_el1 },	1829	{ SYS_DESC(SYS_ID_ISAR1_EL1), NULL, get_id_isar1_el1 },
1947	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b100),	1830	{ SYS_DESC(SYS_ID_ISAR2_EL1), NULL, get_id_isar2_el1 },
1948	NULL, get_id_mmfr0_el1 },	1831	{ SYS_DESC(SYS_ID_ISAR3_EL1), NULL, get_id_isar3_el1 },
1949	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b101),	1832	{ SYS_DESC(SYS_ID_ISAR4_EL1), NULL, get_id_isar4_el1 },
1950	NULL, get_id_mmfr1_el1 },	1833	{ SYS_DESC(SYS_ID_ISAR5_EL1), NULL, get_id_isar5_el1 },
1951	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b110),	1834	{ SYS_DESC(SYS_CLIDR_EL1), NULL, get_clidr_el1 },
1952	NULL, get_id_mmfr2_el1 },	1835	{ SYS_DESC(SYS_AIDR_EL1), NULL, get_aidr_el1 },
1953	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b111),	1836	{ SYS_DESC(SYS_CTR_EL0), NULL, get_ctr_el0 },
1954	NULL, get_id_mmfr3_el1 },
1955	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b000),
1956	NULL, get_id_isar0_el1 },
1957	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b001),
1958	NULL, get_id_isar1_el1 },
1959	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b010),
1960	NULL, get_id_isar2_el1 },
1961	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b011),
1962	NULL, get_id_isar3_el1 },
1963	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b100),
1964	NULL, get_id_isar4_el1 },
1965	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b101),
1966	NULL, get_id_isar5_el1 },
1967	{ Op0(0b11), Op1(0b001), CRn(0b0000), CRm(0b0000), Op2(0b001),
1968	NULL, get_clidr_el1 },
1969	{ Op0(0b11), Op1(0b001), CRn(0b0000), CRm(0b0000), Op2(0b111),
1970	NULL, get_aidr_el1 },
1971	{ Op0(0b11), Op1(0b011), CRn(0b0000), CRm(0b0000), Op2(0b001),
1972	NULL, get_ctr_el0 },
1973	};	1837	};
1974		1838
1975	static int reg_from_user(u64 val, const void __user uaddr, u64 id)	1839	static int reg_from_user(u64 val, const void __user uaddr, u64 id)

arch/arm64/kvm/sys_regs.h+5−18

@@ -83,24 +83,6 @@ static inline bool read_zero(struct kvm_vcpu *vcpu, · 82 unmodified lines
83	return true;	83	return true;
84	}	84	}
85		85
86	static inline bool write_to_read_only(struct kvm_vcpu *vcpu,
87	const struct sys_reg_params *params)
88	{
89	kvm_debug("sys_reg write to read-only register at: %lx\n",
90	*vcpu_pc(vcpu));
91	print_sys_reg_instr(params);
92	return false;
93	}
94
95	static inline bool read_from_write_only(struct kvm_vcpu *vcpu,
96	const struct sys_reg_params *params)
97	{
98	kvm_debug("sys_reg read to write-only register at: %lx\n",
99	*vcpu_pc(vcpu));
100	print_sys_reg_instr(params);
101	return false;
102	}
103
104	/* Reset functions */	86	/* Reset functions */
105	static inline void reset_unknown(struct kvm_vcpu *vcpu,	87	static inline void reset_unknown(struct kvm_vcpu *vcpu,
106	const struct sys_reg_desc *r)	88	const struct sys_reg_desc *r)
@@ -147,4 +129,9 @@ const struct sys_reg_desc *find_reg_by_id(u64 id, · 40 unmodified lines
147	#define CRm(_x) .CRm = _x	129	#define CRm(_x) .CRm = _x
148	#define Op2(_x) .Op2 = _x	130	#define Op2(_x) .Op2 = _x
149		131
		132	#define SYS_DESC(reg) \
		133	Op0(sys_reg_Op0(reg)), Op1(sys_reg_Op1(reg)), \
		134	CRn(sys_reg_CRn(reg)), CRm(sys_reg_CRm(reg)), \
		135	Op2(sys_reg_Op2(reg))
		136
150	#endif /* __ARM64_KVM_SYS_REGS_LOCAL_H__ */	137	#endif /* __ARM64_KVM_SYS_REGS_LOCAL_H__ */

arch/arm64/kvm/sys_regs_generic_v8.c+1−3

  * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
  */
 static const struct sys_reg_desc genericv8_sys_regs[] = {
-	/* ACTLR_EL1 */
+	{ SYS_DESC(SYS_ACTLR_EL1), access_actlr, reset_actlr, ACTLR_EL1 },
-	{ Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b001),
-	  access_actlr, reset_actlr, ACTLR_EL1 },
 };
 static const struct sys_reg_desc genericv8_cp15_regs[] = {

include/kvm/arm_arch_timer.h+2−0

@@ -63,6 +63,8 @@ int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu, · 62 unmodified lines
63	void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu);	63	void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu);
64	void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu);	64	void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu);
65	void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu);	65	void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu);
		66	bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu);
		67	void kvm_timer_update_run(struct kvm_vcpu *vcpu);
66	void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu);	68	void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu);
67		69
68	u64 kvm_arm_timer_get_reg(struct kvm_vcpu *, u64 regid);	70	u64 kvm_arm_timer_get_reg(struct kvm_vcpu *, u64 regid);

include/kvm/arm_pmu.h+7−0

 void kvm_pmu_overflow_set(struct kvm_vcpu *vcpu, u64 val);
 void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu);
 void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu);
+bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu);
+void kvm_pmu_update_run(struct kvm_vcpu *vcpu);
 void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val);
 void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val);
 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,
 static inline void kvm_pmu_overflow_set(struct kvm_vcpu *vcpu, u64 val) {}
 static inline void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu) {}
 static inline void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu) {}
+static inline bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu)
+{
+	return false;
+}
+static inline void kvm_pmu_update_run(struct kvm_vcpu *vcpu) {}
 static inline void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val) {}
 static inline void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val) {}
 static inline void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu,

include/kvm/arm_vgic.h+3−6

 struct vgic_v2_cpu_if {
 	u32		vgic_hcr;
 	u32		vgic_vmcr;
-	u32		vgic_misr;	/* Saved only */
-	u64		vgic_eisr;	/* Saved only */
 	u64		vgic_elrsr;	/* Saved only */
 	u32		vgic_apr;
 	u32		vgic_lr[VGIC_V2_MAX_LRS];
 	u32		vgic_hcr;
 	u32		vgic_vmcr;
 	u32		vgic_sre;	/* Restored only, change ignored */
-	u32		vgic_misr;	/* Saved only */
-	u32		vgic_eisr;	/* Saved only */
 	u32		vgic_elrsr;	/* Saved only */
 	u32		vgic_ap0r[4];
 	u32		vgic_ap1r[4];
 	 */
 	struct list_head ap_list_head;
-	u64 live_lrs;
 	/*
 	 * Members below are used with GICv3 emulation only and represent
 	 * parts of the redistributor.
 int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu);
+void kvm_vgic_load(struct kvm_vcpu *vcpu);
+void kvm_vgic_put(struct kvm_vcpu *vcpu);
 #define irqchip_in_kernel(k)	(!!((k)->arch.vgic.in_kernel))
 #define vgic_initialized(k)	((k)->arch.vgic.initialized)
 #define vgic_ready(k)		((k)->arch.vgic.ready)

include/uapi/linux/kvm.h+8−0

 #define KVM_CAP_S390_AIS 141
 #define KVM_CAP_SPAPR_TCE_VFIO 142
 #define KVM_CAP_X86_GUEST_MWAIT 143
+#define KVM_CAP_ARM_USER_IRQ 144
 #ifdef KVM_CAP_IRQ_ROUTING
 #define KVM_X2APIC_API_USE_32BIT_IDS            (1ULL << 0)
 #define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK  (1ULL << 1)
+/* Available with KVM_CAP_ARM_USER_IRQ */
+/* Bits for run->s.regs.device_irq_level */
+#define KVM_ARM_DEV_EL1_VTIMER		(1 << 0)
+#define KVM_ARM_DEV_EL1_PTIMER		(1 << 1)
+#define KVM_ARM_DEV_PMU			(1 << 2)
 #endif /* __LINUX_KVM_H */

virt/kvm/arm/arch_timer.c+98−26

 	return cval <= now;
 }
+/*
+ * Reflect the timer output level into the kvm_run structure
+ */
+void kvm_timer_update_run(struct kvm_vcpu *vcpu)
+{
+	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
+	struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
+	struct kvm_sync_regs *regs = &vcpu->run->s.regs;
+	/* Populate the device bitmap with the timer states */
+	regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER |
+				    KVM_ARM_DEV_EL1_PTIMER);
+	if (vtimer->irq.level)
+		regs->device_irq_level |= KVM_ARM_DEV_EL1_VTIMER;
+	if (ptimer->irq.level)
+		regs->device_irq_level |= KVM_ARM_DEV_EL1_PTIMER;
+}
 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
 				 struct arch_timer_context *timer_ctx)
 {
 	int ret;
-	BUG_ON(!vgic_initialized(vcpu->kvm));
 	timer_ctx->active_cleared_last = false;
 	timer_ctx->irq.level = new_level;
 	trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_ctx->irq.irq,
 				   timer_ctx->irq.level);
-	ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id, timer_ctx->irq.irq,
+	if (likely(irqchip_in_kernel(vcpu->kvm))) {
-				  timer_ctx->irq.level);
+		ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
-	WARN_ON(ret);
+					  timer_ctx->irq.irq,
+					  timer_ctx->irq.level);
+		WARN_ON(ret);
+	}
 }
 /*
  * Check if there was a change in the timer state (should we raise or lower
  * the line level to the GIC).
  */
-static int kvm_timer_update_state(struct kvm_vcpu *vcpu)
+static void kvm_timer_update_state(struct kvm_vcpu *vcpu)
 {
 	struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 	 * because the guest would never see the interrupt.  Instead wait
 	 * until we call this function from kvm_timer_flush_hwstate.
 	 */
-	if (!vgic_initialized(vcpu->kvm) || !timer->enabled)
+	if (unlikely(!timer->enabled))
-		return -ENODEV;
+		return;
 	if (kvm_timer_should_fire(vtimer) != vtimer->irq.level)
 		kvm_timer_update_irq(vcpu, !vtimer->irq.level, vtimer);
 	if (kvm_timer_should_fire(ptimer) != ptimer->irq.level)
 		kvm_timer_update_irq(vcpu, !ptimer->irq.level, ptimer);
-	return 0;
 }
 /* Schedule the background timer for the emulated timer. */
 	timer_disarm(timer);
 }
-/**
+static void kvm_timer_flush_hwstate_vgic(struct kvm_vcpu *vcpu)
- * kvm_timer_flush_hwstate - prepare to move the virt timer to the cpu
- * @vcpu: The vcpu pointer
- *
- * Check if the virtual timer has expired while we were running in the host,
- * and inject an interrupt if that was the case.
- */
-void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu)
 {
 	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 	bool phys_active;
 	int ret;
-	if (kvm_timer_update_state(vcpu))
-		return;
-	/* Set the background timer for the physical timer emulation. */
-	kvm_timer_emulate(vcpu, vcpu_ptimer(vcpu));
 	/*
 	* If we enter the guest with the virtual input level to the VGIC
 	* asserted, then we have already told the VGIC what we need to, and
 	vtimer->active_cleared_last = !phys_active;
 }
+bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu)
+{
+	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
+	struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
+	struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
+	bool vlevel, plevel;
+	if (likely(irqchip_in_kernel(vcpu->kvm)))
+		return false;
+	vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER;
+	plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER;
+	return vtimer->irq.level != vlevel ||
+	       ptimer->irq.level != plevel;
+}
+static void kvm_timer_flush_hwstate_user(struct kvm_vcpu *vcpu)
+{
+	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
+	/*
+	 * To prevent continuously exiting from the guest, we mask the
+	 * physical interrupt such that the guest can make forward progress.
+	 * Once we detect the output level being deasserted, we unmask the
+	 * interrupt again so that we exit from the guest when the timer
+	 * fires.
+	*/
+	if (vtimer->irq.level)
+		disable_percpu_irq(host_vtimer_irq);
+	else
+		enable_percpu_irq(host_vtimer_irq, 0);
+}
+/**
+ * kvm_timer_flush_hwstate - prepare timers before running the vcpu
+ * @vcpu: The vcpu pointer
+ *
+ * Check if the virtual timer has expired while we were running in the host,
+ * and inject an interrupt if that was the case, making sure the timer is
+ * masked or disabled on the host so that we keep executing.  Also schedule a
+ * software timer for the physical timer if it is enabled.
+ */
+void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu)
+{
+	struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
+	if (unlikely(!timer->enabled))
+		return;
+	kvm_timer_update_state(vcpu);
+	/* Set the background timer for the physical timer emulation. */
+	kvm_timer_emulate(vcpu, vcpu_ptimer(vcpu));
+	if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
+		kvm_timer_flush_hwstate_user(vcpu);
+	else
+		kvm_timer_flush_hwstate_vgic(vcpu);
+}
 /**
  * kvm_timer_sync_hwstate - sync timer state from cpu
  * @vcpu: The vcpu pointer
  *
- * Check if the virtual timer has expired while we were running in the guest,
+ * Check if any of the timers have expired while we were running in the guest,
  * and inject an interrupt if that was the case.
  */
 void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
 	if (timer->enabled)
 		return 0;
+	/* Without a VGIC we do not map virtual IRQs to physical IRQs */
+	if (!irqchip_in_kernel(vcpu->kvm))
+		goto no_vgic;
+	if (!vgic_initialized(vcpu->kvm))
+		return -ENODEV;
 	/*
 	 * Find the physical IRQ number corresponding to the host_vtimer_irq
 	 */
 	if (ret)
 		return ret;
+no_vgic:
 	timer->enabled = 1;
 	return 0;
 }

virt/kvm/arm/hyp/vgic-v2-sr.c+7−71

@@ -22,49 +22,6 @@ · 21 unmodified lines
22	#include <asm/kvm_emulate.h>	22	#include <asm/kvm_emulate.h>
23	#include <asm/kvm_hyp.h>	23	#include <asm/kvm_hyp.h>
24		24
25	static void __hyp_text save_maint_int_state(struct kvm_vcpu *vcpu,
26	void __iomem *base)
27	{
28	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
29	int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
30	u32 eisr0, eisr1;
31	int i;
32	bool expect_mi;
33
34	expect_mi = !!(cpu_if->vgic_hcr & GICH_HCR_UIE);
35
36	for (i = 0; i < nr_lr; i++) {
37	if (!(vcpu->arch.vgic_cpu.live_lrs & (1UL << i)))
38	continue;
39
40	expect_mi \|= (!(cpu_if->vgic_lr[i] & GICH_LR_HW) &&
41	(cpu_if->vgic_lr[i] & GICH_LR_EOI));
42	}
43
44	if (expect_mi) {
45	cpu_if->vgic_misr = readl_relaxed(base + GICH_MISR);
46
47	if (cpu_if->vgic_misr & GICH_MISR_EOI) {
48	eisr0 = readl_relaxed(base + GICH_EISR0);
49	if (unlikely(nr_lr > 32))
50	eisr1 = readl_relaxed(base + GICH_EISR1);
51	else
52	eisr1 = 0;
53	} else {
54	eisr0 = eisr1 = 0;
55	}
56	} else {
57	cpu_if->vgic_misr = 0;
58	eisr0 = eisr1 = 0;
59	}
60
61	#ifdef CONFIG_CPU_BIG_ENDIAN
62	cpu_if->vgic_eisr = ((u64)eisr0 << 32) \| eisr1;
63	#else
64	cpu_if->vgic_eisr = ((u64)eisr1 << 32) \| eisr0;
65	#endif
66	}
67
68	static void __hyp_text save_elrsr(struct kvm_vcpu vcpu, void __iomem base)	25	static void __hyp_text save_elrsr(struct kvm_vcpu vcpu, void __iomem base)
69	{	26	{
70	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;	27	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
@@ -87,13 +44,10 @@ static void __hyp_text save_elrsr(struct kvm_vcpu vcpu, void __iomem base) · 16 unmodified lines
87	static void __hyp_text save_lrs(struct kvm_vcpu vcpu, void __iomem base)	44	static void __hyp_text save_lrs(struct kvm_vcpu vcpu, void __iomem base)
88	{	45	{
89	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;	46	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
90	int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
91	int i;	47	int i;
		48	u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
92		49
93	for (i = 0; i < nr_lr; i++) {	50	for (i = 0; i < used_lrs; i++) {
94	if (!(vcpu->arch.vgic_cpu.live_lrs & (1UL << i)))
95	continue;
96
97	if (cpu_if->vgic_elrsr & (1UL << i))	51	if (cpu_if->vgic_elrsr & (1UL << i))
98	cpu_if->vgic_lr[i] &= ~GICH_LR_STATE;	52	cpu_if->vgic_lr[i] &= ~GICH_LR_STATE;
99	else	53	else
@@ -110,26 +64,20 @@ void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu) · 10 unmodified lines
110	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;	64	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
111	struct vgic_dist *vgic = &kvm->arch.vgic;	65	struct vgic_dist *vgic = &kvm->arch.vgic;
112	void __iomem *base = kern_hyp_va(vgic->vctrl_base);	66	void __iomem *base = kern_hyp_va(vgic->vctrl_base);
		67	u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
113		68
114	if (!base)	69	if (!base)
115	return;	70	return;
116		71
117	cpu_if->vgic_vmcr = readl_relaxed(base + GICH_VMCR);	72	if (used_lrs) {
118
119	if (vcpu->arch.vgic_cpu.live_lrs) {
120	cpu_if->vgic_apr = readl_relaxed(base + GICH_APR);	73	cpu_if->vgic_apr = readl_relaxed(base + GICH_APR);
121		74
122	save_maint_int_state(vcpu, base);
123	save_elrsr(vcpu, base);	75	save_elrsr(vcpu, base);
124	save_lrs(vcpu, base);	76	save_lrs(vcpu, base);
125		77
126	writel_relaxed(0, base + GICH_HCR);	78	writel_relaxed(0, base + GICH_HCR);
127
128	vcpu->arch.vgic_cpu.live_lrs = 0;
129	} else {	79	} else {
130	cpu_if->vgic_eisr = 0;
131	cpu_if->vgic_elrsr = ~0UL;	80	cpu_if->vgic_elrsr = ~0UL;
132	cpu_if->vgic_misr = 0;
133	cpu_if->vgic_apr = 0;	81	cpu_if->vgic_apr = 0;
134	}	82	}
135	}	83	}
@@ -141,32 +89,20 @@ void __hyp_text __vgic_v2_restore_state(struct kvm_vcpu *vcpu) · 5 unmodified lines
141	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;	89	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
142	struct vgic_dist *vgic = &kvm->arch.vgic;	90	struct vgic_dist *vgic = &kvm->arch.vgic;
143	void __iomem *base = kern_hyp_va(vgic->vctrl_base);	91	void __iomem *base = kern_hyp_va(vgic->vctrl_base);
144	int nr_lr = (kern_hyp_va(&kvm_vgic_global_state))->nr_lr;
145	int i;	92	int i;
146	u64 live_lrs = 0;	93	u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
147		94
148	if (!base)	95	if (!base)
149	return;	96	return;
150		97
151		98	if (used_lrs) {
152	for (i = 0; i < nr_lr; i++)
153	if (cpu_if->vgic_lr[i] & GICH_LR_STATE)
154	live_lrs \|= 1UL << i;
155
156	if (live_lrs) {
157	writel_relaxed(cpu_if->vgic_hcr, base + GICH_HCR);	99	writel_relaxed(cpu_if->vgic_hcr, base + GICH_HCR);
158	writel_relaxed(cpu_if->vgic_apr, base + GICH_APR);	100	writel_relaxed(cpu_if->vgic_apr, base + GICH_APR);
159	for (i = 0; i < nr_lr; i++) {	101	for (i = 0; i < used_lrs; i++) {
160	if (!(live_lrs & (1UL << i)))
161	continue;
162
163	writel_relaxed(cpu_if->vgic_lr[i],	102	writel_relaxed(cpu_if->vgic_lr[i],
164	base + GICH_LR0 + (i * 4));	103	base + GICH_LR0 + (i * 4));
165	}	104	}
166	}	105	}
167
168	writel_relaxed(cpu_if->vgic_vmcr, base + GICH_VMCR);
169	vcpu->arch.vgic_cpu.live_lrs = live_lrs;
170	}	106	}
171		107
172	#ifdef CONFIG_ARM64	108	#ifdef CONFIG_ARM64

virt/kvm/arm/hyp/vgic-v3-sr.c+24−63

 	}
 }
-static void __hyp_text save_maint_int_state(struct kvm_vcpu *vcpu, int nr_lr)
-{
-	struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
-	int i;
-	bool expect_mi;
-	expect_mi = !!(cpu_if->vgic_hcr & ICH_HCR_UIE);
-	for (i = 0; i < nr_lr; i++) {
-		if (!(vcpu->arch.vgic_cpu.live_lrs & (1UL << i)))
-				continue;
-		expect_mi |= (!(cpu_if->vgic_lr[i] & ICH_LR_HW) &&
-			      (cpu_if->vgic_lr[i] & ICH_LR_EOI));
-	}
-	if (expect_mi) {
-		cpu_if->vgic_misr  = read_gicreg(ICH_MISR_EL2);
-		if (cpu_if->vgic_misr & ICH_MISR_EOI)
-			cpu_if->vgic_eisr = read_gicreg(ICH_EISR_EL2);
-		else
-			cpu_if->vgic_eisr = 0;
-	} else {
-		cpu_if->vgic_misr = 0;
-		cpu_if->vgic_eisr = 0;
-	}
-}
 void __hyp_text __vgic_v3_save_state(struct kvm_vcpu *vcpu)
 {
 	struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
+	u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
 	u64 val;
 	/*
 	 * Make sure stores to the GIC via the memory mapped interface
 	 * are now visible to the system register interface.
 	 */
-	if (!cpu_if->vgic_sre)
+	if (!cpu_if->vgic_sre) {
 		dsb(st);
+		cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
+	}
-	cpu_if->vgic_vmcr  = read_gicreg(ICH_VMCR_EL2);
+	if (used_lrs) {
-	if (vcpu->arch.vgic_cpu.live_lrs) {
 		int i;
-		u32 max_lr_idx, nr_pri_bits;
+		u32 nr_pri_bits;
 		cpu_if->vgic_elrsr = read_gicreg(ICH_ELSR_EL2);
 		write_gicreg(0, ICH_HCR_EL2);
 		val = read_gicreg(ICH_VTR_EL2);
-		max_lr_idx = vtr_to_max_lr_idx(val);
 		nr_pri_bits = vtr_to_nr_pri_bits(val);
-		save_maint_int_state(vcpu, max_lr_idx + 1);
+		for (i = 0; i < used_lrs; i++) {
-		for (i = 0; i <= max_lr_idx; i++) {
-			if (!(vcpu->arch.vgic_cpu.live_lrs & (1UL << i)))
-				continue;
 			if (cpu_if->vgic_elrsr & (1 << i))
 				cpu_if->vgic_lr[i] &= ~ICH_LR_STATE;
 			else
 		default:
 			cpu_if->vgic_ap1r[0] = read_gicreg(ICH_AP1R0_EL2);
 		}
-		vcpu->arch.vgic_cpu.live_lrs = 0;
 	} else {
-		cpu_if->vgic_misr  = 0;
-		cpu_if->vgic_eisr  = 0;
 		cpu_if->vgic_elrsr = 0xffff;
 		cpu_if->vgic_ap0r[0] = 0;
 		cpu_if->vgic_ap0r[1] = 0;
 void __hyp_text __vgic_v3_restore_state(struct kvm_vcpu *vcpu)
 {
 	struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
+	u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
 	u64 val;
-	u32 max_lr_idx, nr_pri_bits;
+	u32 nr_pri_bits;
-	u16 live_lrs = 0;
 	int i;
 	/*
 	 * delivered as a FIQ to the guest, with potentially fatal
 	 * consequences. So we must make sure that ICC_SRE_EL1 has
 	 * been actually programmed with the value we want before
-	 * starting to mess with the rest of the GIC.
+	 * starting to mess with the rest of the GIC, and VMCR_EL2 in
+	 * particular.
 	 */
 	if (!cpu_if->vgic_sre) {
 		write_gicreg(0, ICC_SRE_EL1);
 		isb();
+		write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
 	}
 	val = read_gicreg(ICH_VTR_EL2);
-	max_lr_idx = vtr_to_max_lr_idx(val);
 	nr_pri_bits = vtr_to_nr_pri_bits(val);
-	for (i = 0; i <= max_lr_idx; i++) {
+	if (used_lrs) {
-		if (cpu_if->vgic_lr[i] & ICH_LR_STATE)
-			live_lrs |= (1 << i);
-	}
-	write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
-	if (live_lrs) {
 		write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
 		switch (nr_pri_bits) {
 			write_gicreg(cpu_if->vgic_ap1r[0], ICH_AP1R0_EL2);
 		}
-		for (i = 0; i <= max_lr_idx; i++) {
+		for (i = 0; i < used_lrs; i++)
-			if (!(live_lrs & (1 << i)))
-				continue;
 			__gic_v3_set_lr(cpu_if->vgic_lr[i], i);
-		}
 	}
 	/*
 		isb();
 		dsb(sy);
 	}
-	vcpu->arch.vgic_cpu.live_lrs = live_lrs;
 	/*
 	 * Prevent the guest from touching the GIC system registers if
 {
 	return read_gicreg(ICH_VTR_EL2);
 }
+u64 __hyp_text __vgic_v3_read_vmcr(void)
+{
+	return read_gicreg(ICH_VMCR_EL2);
+}
+void __hyp_text __vgic_v3_write_vmcr(u32 vmcr)
+{
+	write_gicreg(vmcr, ICH_VMCR_EL2);
+}

virt/kvm/arm/pmu.c+35−4

@@ -230,13 +230,44 @@ static void kvm_pmu_update_state(struct kvm_vcpu *vcpu) · 229 unmodified lines
230	return;	230	return;
231		231
232	overflow = !!kvm_pmu_overflow_status(vcpu);	232	overflow = !!kvm_pmu_overflow_status(vcpu);
233	if (pmu->irq_level != overflow) {	233	if (pmu->irq_level == overflow)
234	pmu->irq_level = overflow;	234	return;
235	kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,	235
236	pmu->irq_num, overflow);	236	pmu->irq_level = overflow;
		237
		238	if (likely(irqchip_in_kernel(vcpu->kvm))) {
		239	int ret;
		240	ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
		241	pmu->irq_num, overflow);
		242	WARN_ON(ret);
237	}	243	}
238	}	244	}
239		245
		246	bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu)
		247	{
		248	struct kvm_pmu *pmu = &vcpu->arch.pmu;
		249	struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
		250	bool run_level = sregs->device_irq_level & KVM_ARM_DEV_PMU;
		251
		252	if (likely(irqchip_in_kernel(vcpu->kvm)))
		253	return false;
		254
		255	return pmu->irq_level != run_level;
		256	}
		257
		258	/*
		259	* Reflect the PMU overflow interrupt output level into the kvm_run structure
		260	*/
		261	void kvm_pmu_update_run(struct kvm_vcpu *vcpu)
		262	{
		263	struct kvm_sync_regs *regs = &vcpu->run->s.regs;
		264
		265	/* Populate the timer bitmap for user space */
		266	regs->device_irq_level &= ~KVM_ARM_DEV_PMU;
		267	if (vcpu->arch.pmu.irq_level)
		268	regs->device_irq_level \|= KVM_ARM_DEV_PMU;
		269	}
		270
240	/**	271	/**
241	* kvm_pmu_flush_hwstate - flush pmu state to cpu	272	* kvm_pmu_flush_hwstate - flush pmu state to cpu
242	* @vcpu: The vcpu pointer	273	* @vcpu: The vcpu pointer

virt/kvm/arm/vgic/vgic-init.c+68−40

 /*
  * Initialization rules: there are multiple stages to the vgic
- * initialization, both for the distributor and the CPU interfaces.
+ * initialization, both for the distributor and the CPU interfaces.  The basic
+ * idea is that even though the VGIC is not functional or not requested from
+ * user space, the critical path of the run loop can still call VGIC functions
+ * that just won't do anything, without them having to check additional
+ * initialization flags to ensure they don't look at uninitialized data
+ * structures.
  *
  * Distributor:
  *
  *
  * CPU Interface:
  *
- * - kvm_vgic_cpu_early_init(): initialization of static data that
+ * - kvm_vgic_vcpu_early_init(): initialization of static data that
  *   doesn't depend on any sizing information or emulation type. No
  *   allocation is allowed there.
  */
 /* EARLY INIT */
-/*
+/**
- * Those 2 functions should not be needed anymore but they
+ * kvm_vgic_early_init() - Initialize static VGIC VCPU data structures
- * still are called from arm.c
+ * @kvm: The VM whose VGIC districutor should be initialized
+ *
+ * Only do initialization of static structures that don't require any
+ * allocation or sizing information from userspace.  vgic_init() called
+ * kvm_vgic_dist_init() which takes care of the rest.
  */
 void kvm_vgic_early_init(struct kvm *kvm)
 {
+	struct vgic_dist *dist = &kvm->arch.vgic;
+	INIT_LIST_HEAD(&dist->lpi_list_head);
+	spin_lock_init(&dist->lpi_list_lock);
 }
+/**
+ * kvm_vgic_vcpu_early_init() - Initialize static VGIC VCPU data structures
+ * @vcpu: The VCPU whose VGIC data structures whould be initialized
+ *
+ * Only do initialization, but do not actually enable the VGIC CPU interface
+ * yet.
+ */
 void kvm_vgic_vcpu_early_init(struct kvm_vcpu *vcpu)
 {
+	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
+	int i;
+	INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
+	spin_lock_init(&vgic_cpu->ap_list_lock);
+	/*
+	 * Enable and configure all SGIs to be edge-triggered and
+	 * configure all PPIs as level-triggered.
+	 */
+	for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
+		struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
+		INIT_LIST_HEAD(&irq->ap_list);
+		spin_lock_init(&irq->irq_lock);
+		irq->intid = i;
+		irq->vcpu = NULL;
+		irq->target_vcpu = vcpu;
+		irq->targets = 1U << vcpu->vcpu_id;
+		kref_init(&irq->refcount);
+		if (vgic_irq_is_sgi(i)) {
+			/* SGIs */
+			irq->enabled = 1;
+			irq->config = VGIC_CONFIG_EDGE;
+		} else {
+			/* PPIs */
+			irq->config = VGIC_CONFIG_LEVEL;
+		}
+	}
 }
 /* CREATION */
 	struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
 	int i;
-	INIT_LIST_HEAD(&dist->lpi_list_head);
-	spin_lock_init(&dist->lpi_list_lock);
 	dist->spis = kcalloc(nr_spis, sizeof(struct vgic_irq), GFP_KERNEL);
 	if (!dist->spis)
 		return  -ENOMEM;
 }
 /**
- * kvm_vgic_vcpu_init: initialize the vcpu data structures and
+ * kvm_vgic_vcpu_init() - Enable the VCPU interface
- * enable the VCPU interface
+ * @vcpu: the VCPU which's VGIC should be enabled
- * @vcpu: the VCPU which's VGIC should be initialized
  */
 static void kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
 {
-	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
-	int i;
-	INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
-	spin_lock_init(&vgic_cpu->ap_list_lock);
-	/*
-	 * Enable and configure all SGIs to be edge-triggered and
-	 * configure all PPIs as level-triggered.
-	 */
-	for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
-		struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
-		INIT_LIST_HEAD(&irq->ap_list);
-		spin_lock_init(&irq->irq_lock);
-		irq->intid = i;
-		irq->vcpu = NULL;
-		irq->target_vcpu = vcpu;
-		irq->targets = 1U << vcpu->vcpu_id;
-		kref_init(&irq->refcount);
-		if (vgic_irq_is_sgi(i)) {
-			/* SGIs */
-			irq->enabled = 1;
-			irq->config = VGIC_CONFIG_EDGE;
-		} else {
-			/* PPIs */
-			irq->config = VGIC_CONFIG_LEVEL;
-		}
-	}
 	if (kvm_vgic_global_state.type == VGIC_V2)
 		vgic_v2_enable(vcpu);
 	else
 	vgic_debug_init(kvm);
 	dist->initialized = true;
+	/*
+	 * If we're initializing GICv2 on-demand when first running the VCPU
+	 * then we need to load the VGIC state onto the CPU.  We can detect
+	 * this easily by checking if we are in between vcpu_load and vcpu_put
+	 * when we just initialized the VGIC.
+	 */
+	preempt_disable();
+	vcpu = kvm_arm_get_running_vcpu();
+	if (vcpu)
+		kvm_vgic_load(vcpu);
+	preempt_enable();
 out:
 	return ret;
 }

virt/kvm/arm/vgic/vgic-v2.c+39−51

@@ -22,59 +22,17 @@ · 21 unmodified lines
22		22
23	#include "vgic.h"	23	#include "vgic.h"
24		24
25	/*	25	void vgic_v2_set_underflow(struct kvm_vcpu *vcpu)
26	* Call this function to convert a u64 value to an unsigned long * bitmask
27	* in a way that works on both 32-bit and 64-bit LE and BE platforms.
28	*
29	* Warning: Calling this function may modify *val.
30	*/
31	static unsigned long u64_to_bitmask(u64 val)
32	{
33	#if defined(CONFIG_CPU_BIG_ENDIAN) && BITS_PER_LONG == 32
34	val = (val >> 32) \| (*val << 32);
35	#endif
36	return (unsigned long *)val;
37	}
38
39	void vgic_v2_process_maintenance(struct kvm_vcpu *vcpu)
40	{	26	{
41	struct vgic_v2_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v2;	27	struct vgic_v2_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v2;
42		28
43	if (cpuif->vgic_misr & GICH_MISR_EOI) {	29	cpuif->vgic_hcr \|= GICH_HCR_UIE;
44	u64 eisr = cpuif->vgic_eisr;
45	unsigned long *eisr_bmap = u64_to_bitmask(&eisr);
46	int lr;
47
48	for_each_set_bit(lr, eisr_bmap, kvm_vgic_global_state.nr_lr) {
49	u32 intid = cpuif->vgic_lr[lr] & GICH_LR_VIRTUALID;
50
51	WARN_ON(cpuif->vgic_lr[lr] & GICH_LR_STATE);
52
53	/* Only SPIs require notification */
54	if (vgic_valid_spi(vcpu->kvm, intid))
55	kvm_notify_acked_irq(vcpu->kvm, 0,
56	intid - VGIC_NR_PRIVATE_IRQS);
57	}
58	}
59
60	/* check and disable underflow maintenance IRQ */
61	cpuif->vgic_hcr &= ~GICH_HCR_UIE;
62
63	/*
64	* In the next iterations of the vcpu loop, if we sync the
65	* vgic state after flushing it, but before entering the guest
66	* (this happens for pending signals and vmid rollovers), then
67	* make sure we don't pick up any old maintenance interrupts
68	* here.
69	*/
70	cpuif->vgic_eisr = 0;
71	}	30	}
72		31
73	void vgic_v2_set_underflow(struct kvm_vcpu *vcpu)	32	static bool lr_signals_eoi_mi(u32 lr_val)
74	{	33	{
75	struct vgic_v2_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v2;	34	return !(lr_val & GICH_LR_STATE) && (lr_val & GICH_LR_EOI) &&
76		35	!(lr_val & GICH_LR_HW);
77	cpuif->vgic_hcr \|= GICH_HCR_UIE;
78	}	36	}
79		37
80	/*	38	/*
@@ -86,14 +44,22 @@ void vgic_v2_set_underflow(struct kvm_vcpu *vcpu) · 5 unmodified lines
86	*/	44	*/
87	void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu)	45	void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu)
88	{	46	{
89	struct vgic_v2_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v2;	47	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
		48	struct vgic_v2_cpu_if *cpuif = &vgic_cpu->vgic_v2;
90	int lr;	49	int lr;
91		50
92	for (lr = 0; lr < vcpu->arch.vgic_cpu.used_lrs; lr++) {	51	cpuif->vgic_hcr &= ~GICH_HCR_UIE;
		52
		53	for (lr = 0; lr < vgic_cpu->used_lrs; lr++) {
93	u32 val = cpuif->vgic_lr[lr];	54	u32 val = cpuif->vgic_lr[lr];
94	u32 intid = val & GICH_LR_VIRTUALID;	55	u32 intid = val & GICH_LR_VIRTUALID;
95	struct vgic_irq *irq;	56	struct vgic_irq *irq;
96		57
		58	/* Notify fds when the guest EOI'ed a level-triggered SPI */
		59	if (lr_signals_eoi_mi(val) && vgic_valid_spi(vcpu->kvm, intid))
		60	kvm_notify_acked_irq(vcpu->kvm, 0,
		61	intid - VGIC_NR_PRIVATE_IRQS);
		62
97	irq = vgic_get_irq(vcpu->kvm, vcpu, intid);	63	irq = vgic_get_irq(vcpu->kvm, vcpu, intid);
98		64
99	spin_lock(&irq->irq_lock);	65	spin_lock(&irq->irq_lock);
@@ -126,6 +92,8 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu) · 26 unmodified lines
126	spin_unlock(&irq->irq_lock);	92	spin_unlock(&irq->irq_lock);
127	vgic_put_irq(vcpu->kvm, irq);	93	vgic_put_irq(vcpu->kvm, irq);
128	}	94	}
		95
		96	vgic_cpu->used_lrs = 0;
129	}	97	}
130		98
131	/*	99	/*
@@ -184,6 +152,7 @@ void vgic_v2_clear_lr(struct kvm_vcpu *vcpu, int lr) · 52 unmodified lines
184		152
185	void vgic_v2_set_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp)	153	void vgic_v2_set_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp)
186	{	154	{
		155	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
187	u32 vmcr;	156	u32 vmcr;
188		157
189	vmcr = (vmcrp->ctlr << GICH_VMCR_CTRL_SHIFT) & GICH_VMCR_CTRL_MASK;	158	vmcr = (vmcrp->ctlr << GICH_VMCR_CTRL_SHIFT) & GICH_VMCR_CTRL_MASK;
@@ -194,12 +163,15 @@ void vgic_v2_set_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp) · 4 unmodified lines
194	vmcr \|= (vmcrp->pmr << GICH_VMCR_PRIMASK_SHIFT) &	163	vmcr \|= (vmcrp->pmr << GICH_VMCR_PRIMASK_SHIFT) &
195	GICH_VMCR_PRIMASK_MASK;	164	GICH_VMCR_PRIMASK_MASK;
196		165
197	vcpu->arch.vgic_cpu.vgic_v2.vgic_vmcr = vmcr;	166	cpu_if->vgic_vmcr = vmcr;
198	}	167	}
199		168
200	void vgic_v2_get_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp)	169	void vgic_v2_get_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp)
201	{	170	{
202	u32 vmcr = vcpu->arch.vgic_cpu.vgic_v2.vgic_vmcr;	171	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
		172	u32 vmcr;
		173
		174	vmcr = cpu_if->vgic_vmcr;
203		175
204	vmcrp->ctlr = (vmcr & GICH_VMCR_CTRL_MASK) >>	176	vmcrp->ctlr = (vmcr & GICH_VMCR_CTRL_MASK) >>
205	GICH_VMCR_CTRL_SHIFT;	177	GICH_VMCR_CTRL_SHIFT;
@@ -375,3 +347,19 @@ out: · 169 unmodified lines
375		347
376	return ret;	348	return ret;
377	}	349	}
		350
		351	void vgic_v2_load(struct kvm_vcpu *vcpu)
		352	{
		353	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
		354	struct vgic_dist *vgic = &vcpu->kvm->arch.vgic;
		355
		356	writel_relaxed(cpu_if->vgic_vmcr, vgic->vctrl_base + GICH_VMCR);
		357	}
		358
		359	void vgic_v2_put(struct kvm_vcpu *vcpu)
		360	{
		361	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
		362	struct vgic_dist *vgic = &vcpu->kvm->arch.vgic;
		363
		364	cpu_if->vgic_vmcr = readl_relaxed(vgic->vctrl_base + GICH_VMCR);
		365	}

virt/kvm/arm/vgic/vgic-v3.c+45−42

@@ -21,59 +21,29 @@ · 20 unmodified lines
21		21
22	#include "vgic.h"	22	#include "vgic.h"
23		23
24	void vgic_v3_process_maintenance(struct kvm_vcpu *vcpu)	24	void vgic_v3_set_underflow(struct kvm_vcpu *vcpu)
25	{	25	{
26	struct vgic_v3_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v3;	26	struct vgic_v3_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v3;
27	u32 model = vcpu->kvm->arch.vgic.vgic_model;
28
29	if (cpuif->vgic_misr & ICH_MISR_EOI) {
30	unsigned long eisr_bmap = cpuif->vgic_eisr;
31	int lr;
32
33	for_each_set_bit(lr, &eisr_bmap, kvm_vgic_global_state.nr_lr) {
34	u32 intid;
35	u64 val = cpuif->vgic_lr[lr];
36
37	if (model == KVM_DEV_TYPE_ARM_VGIC_V3)
38	intid = val & ICH_LR_VIRTUAL_ID_MASK;
39	else
40	intid = val & GICH_LR_VIRTUALID;
41
42	WARN_ON(cpuif->vgic_lr[lr] & ICH_LR_STATE);
43
44	/* Only SPIs require notification */
45	if (vgic_valid_spi(vcpu->kvm, intid))
46	kvm_notify_acked_irq(vcpu->kvm, 0,
47	intid - VGIC_NR_PRIVATE_IRQS);
48	}
49
50	/*
51	* In the next iterations of the vcpu loop, if we sync
52	* the vgic state after flushing it, but before
53	* entering the guest (this happens for pending
54	* signals and vmid rollovers), then make sure we
55	* don't pick up any old maintenance interrupts here.
56	*/
57	cpuif->vgic_eisr = 0;
58	}
59		27
60	cpuif->vgic_hcr &= ~ICH_HCR_UIE;	28	cpuif->vgic_hcr \|= ICH_HCR_UIE;
61	}	29	}
62		30
63	void vgic_v3_set_underflow(struct kvm_vcpu *vcpu)	31	static bool lr_signals_eoi_mi(u64 lr_val)
64	{	32	{
65	struct vgic_v3_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v3;	33	return !(lr_val & ICH_LR_STATE) && (lr_val & ICH_LR_EOI) &&
66		34	!(lr_val & ICH_LR_HW);
67	cpuif->vgic_hcr \|= ICH_HCR_UIE;
68	}	35	}
69		36
70	void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu)	37	void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu)
71	{	38	{
72	struct vgic_v3_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v3;	39	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
		40	struct vgic_v3_cpu_if *cpuif = &vgic_cpu->vgic_v3;
73	u32 model = vcpu->kvm->arch.vgic.vgic_model;	41	u32 model = vcpu->kvm->arch.vgic.vgic_model;
74	int lr;	42	int lr;
75		43
76	for (lr = 0; lr < vcpu->arch.vgic_cpu.used_lrs; lr++) {	44	cpuif->vgic_hcr &= ~ICH_HCR_UIE;
		45
		46	for (lr = 0; lr < vgic_cpu->used_lrs; lr++) {
77	u64 val = cpuif->vgic_lr[lr];	47	u64 val = cpuif->vgic_lr[lr];
78	u32 intid;	48	u32 intid;
79	struct vgic_irq *irq;	49	struct vgic_irq *irq;
@@ -82,6 +52,12 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu) · 2 unmodified lines
82	intid = val & ICH_LR_VIRTUAL_ID_MASK;	52	intid = val & ICH_LR_VIRTUAL_ID_MASK;
83	else	53	else
84	intid = val & GICH_LR_VIRTUALID;	54	intid = val & GICH_LR_VIRTUALID;
		55
		56	/* Notify fds when the guest EOI'ed a level-triggered IRQ */
		57	if (lr_signals_eoi_mi(val) && vgic_valid_spi(vcpu->kvm, intid))
		58	kvm_notify_acked_irq(vcpu->kvm, 0,
		59	intid - VGIC_NR_PRIVATE_IRQS);
		60
85	irq = vgic_get_irq(vcpu->kvm, vcpu, intid);	61	irq = vgic_get_irq(vcpu->kvm, vcpu, intid);
86	if (!irq) /* An LPI could have been unmapped. */	62	if (!irq) /* An LPI could have been unmapped. */
87	continue;	63	continue;
@@ -117,6 +93,8 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu) · 29 unmodified lines
117	spin_unlock(&irq->irq_lock);	93	spin_unlock(&irq->irq_lock);
118	vgic_put_irq(vcpu->kvm, irq);	94	vgic_put_irq(vcpu->kvm, irq);
119	}	95	}
		96
		97	vgic_cpu->used_lrs = 0;
120	}	98	}
121		99
122	/* Requires the irq to be locked already */	100	/* Requires the irq to be locked already */
@@ -173,6 +151,7 @@ void vgic_v3_clear_lr(struct kvm_vcpu *vcpu, int lr) · 50 unmodified lines
173		151
174	void vgic_v3_set_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp)	152	void vgic_v3_set_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp)
175	{	153	{
		154	struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
176	u32 vmcr;	155	u32 vmcr;
177		156
178	/*	157	/*
@@ -188,12 +167,15 @@ void vgic_v3_set_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp) · 9 unmodified lines
188	vmcr \|= (vmcrp->grpen0 << ICH_VMCR_ENG0_SHIFT) & ICH_VMCR_ENG0_MASK;	167	vmcr \|= (vmcrp->grpen0 << ICH_VMCR_ENG0_SHIFT) & ICH_VMCR_ENG0_MASK;
189	vmcr \|= (vmcrp->grpen1 << ICH_VMCR_ENG1_SHIFT) & ICH_VMCR_ENG1_MASK;	168	vmcr \|= (vmcrp->grpen1 << ICH_VMCR_ENG1_SHIFT) & ICH_VMCR_ENG1_MASK;
190		169
191	vcpu->arch.vgic_cpu.vgic_v3.vgic_vmcr = vmcr;	170	cpu_if->vgic_vmcr = vmcr;
192	}	171	}
193		172
194	void vgic_v3_get_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp)	173	void vgic_v3_get_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp)
195	{	174	{
196	u32 vmcr = vcpu->arch.vgic_cpu.vgic_v3.vgic_vmcr;	175	struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
		176	u32 vmcr;
		177
		178	vmcr = cpu_if->vgic_vmcr;
197		179
198	/*	180	/*
199	* Ignore the FIQen bit, because GIC emulation always implies	181	* Ignore the FIQen bit, because GIC emulation always implies
@@ -386,3 +368,24 @@ int vgic_v3_probe(const struct gic_kvm_info *info) · 186 unmodified lines
386		368
387	return 0;	369	return 0;
388	}	370	}
		371
		372	void vgic_v3_load(struct kvm_vcpu *vcpu)
		373	{
		374	struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
		375
		376	/*
		377	* If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen
		378	* is dependent on ICC_SRE_EL1.SRE, and we have to perform the
		379	* VMCR_EL2 save/restore in the world switch.
		380	*/
		381	if (likely(cpu_if->vgic_sre))
		382	kvm_call_hyp(__vgic_v3_write_vmcr, cpu_if->vgic_vmcr);
		383	}
		384
		385	void vgic_v3_put(struct kvm_vcpu *vcpu)
		386	{
		387	struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
		388
		389	if (likely(cpu_if->vgic_sre))
		390	cpu_if->vgic_vmcr = kvm_call_hyp(__vgic_v3_read_vmcr);
		391	}

virt/kvm/arm/vgic/vgic.c+44−16

 	spin_unlock(&vgic_cpu->ap_list_lock);
 }
-static inline void vgic_process_maintenance_interrupt(struct kvm_vcpu *vcpu)
-{
-	if (kvm_vgic_global_state.type == VGIC_V2)
-		vgic_v2_process_maintenance(vcpu);
-	else
-		vgic_v3_process_maintenance(vcpu);
-}
 static inline void vgic_fold_lr_state(struct kvm_vcpu *vcpu)
 {
 	if (kvm_vgic_global_state.type == VGIC_V2)
 	DEBUG_SPINLOCK_BUG_ON(!spin_is_locked(&vgic_cpu->ap_list_lock));
-	if (compute_ap_list_depth(vcpu) > kvm_vgic_global_state.nr_lr) {
+	if (compute_ap_list_depth(vcpu) > kvm_vgic_global_state.nr_lr)
-		vgic_set_underflow(vcpu);
 		vgic_sort_ap_list(vcpu);
-	}
 	list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
 		spin_lock(&irq->irq_lock);
 next:
 		spin_unlock(&irq->irq_lock);
-		if (count == kvm_vgic_global_state.nr_lr)
+		if (count == kvm_vgic_global_state.nr_lr) {
+			if (!list_is_last(&irq->ap_list,
+					  &vgic_cpu->ap_list_head))
+				vgic_set_underflow(vcpu);
 			break;
+		}
 	}
 	vcpu->arch.vgic_cpu.used_lrs = count;
 /* Sync back the hardware VGIC state into our emulation after a guest's run. */
 void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
 {
-	if (unlikely(!vgic_initialized(vcpu->kvm)))
+	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
+	/* An empty ap_list_head implies used_lrs == 0 */
+	if (list_empty(&vcpu->arch.vgic_cpu.ap_list_head))
 		return;
-	vgic_process_maintenance_interrupt(vcpu);
+	if (vgic_cpu->used_lrs)
-	vgic_fold_lr_state(vcpu);
+		vgic_fold_lr_state(vcpu);
 	vgic_prune_ap_list(vcpu);
 }
 /* Flush our emulation state into the GIC hardware before entering the guest. */
 void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
 {
-	if (unlikely(!vgic_initialized(vcpu->kvm)))
+	/*
+	 * If there are no virtual interrupts active or pending for this
+	 * VCPU, then there is no work to do and we can bail out without
+	 * taking any lock.  There is a potential race with someone injecting
+	 * interrupts to the VCPU, but it is a benign race as the VCPU will
+	 * either observe the new interrupt before or after doing this check,
+	 * and introducing additional synchronization mechanism doesn't change
+	 * this.
+	 */
+	if (list_empty(&vcpu->arch.vgic_cpu.ap_list_head))
 		return;
 	spin_lock(&vcpu->arch.vgic_cpu.ap_list_lock);
 	spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock);
 }
+void kvm_vgic_load(struct kvm_vcpu *vcpu)
+{
+	if (unlikely(!vgic_initialized(vcpu->kvm)))
+		return;
+	if (kvm_vgic_global_state.type == VGIC_V2)
+		vgic_v2_load(vcpu);
+	else
+		vgic_v3_load(vcpu);
+}
+void kvm_vgic_put(struct kvm_vcpu *vcpu)
+{
+	if (unlikely(!vgic_initialized(vcpu->kvm)))
+		return;
+	if (kvm_vgic_global_state.type == VGIC_V2)
+		vgic_v2_put(vcpu);
+	else
+		vgic_v3_put(vcpu);
+}
 int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
 {
 	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;

virt/kvm/arm/vgic/vgic.h+6−2

 int vgic_check_ioaddr(struct kvm *kvm, phys_addr_t *ioaddr,
 		      phys_addr_t addr, phys_addr_t alignment);
-void vgic_v2_process_maintenance(struct kvm_vcpu *vcpu);
 void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu);
 void vgic_v2_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr);
 void vgic_v2_clear_lr(struct kvm_vcpu *vcpu, int lr);
 int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address,
 			     enum vgic_type);
+void vgic_v2_load(struct kvm_vcpu *vcpu);
+void vgic_v2_put(struct kvm_vcpu *vcpu);
 static inline void vgic_get_irq_kref(struct vgic_irq *irq)
 {
 	if (irq->intid < VGIC_MIN_LPI)
 	kref_get(&irq->refcount);
 }
-void vgic_v3_process_maintenance(struct kvm_vcpu *vcpu);
 void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu);
 void vgic_v3_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr);
 void vgic_v3_clear_lr(struct kvm_vcpu *vcpu, int lr);
 int vgic_v3_map_resources(struct kvm *kvm);
 int vgic_register_redist_iodevs(struct kvm *kvm, gpa_t dist_base_address);
+void vgic_v3_load(struct kvm_vcpu *vcpu);
+void vgic_v3_put(struct kvm_vcpu *vcpu);
 int vgic_register_its_iodevs(struct kvm *kvm);
 bool vgic_has_its(struct kvm *kvm);
 int kvm_vgic_register_its_device(void);