lael-labs/linux

arch/arm64/include/asm/kvm_asm.h+1−0

@@ -63,6 +63,7 @@ enum __kvm_host_smccc_func { · 62 unmodified lines
63		63
64	/* Hypercalls available after pKVM finalisation */	64	/* Hypercalls available after pKVM finalisation */
65	__KVM_HOST_SMCCC_FUNC___pkvm_host_share_hyp,	65	__KVM_HOST_SMCCC_FUNC___pkvm_host_share_hyp,
		66	__KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_hyp,
66	__KVM_HOST_SMCCC_FUNC___kvm_adjust_pc,	67	__KVM_HOST_SMCCC_FUNC___kvm_adjust_pc,
67	__KVM_HOST_SMCCC_FUNC___kvm_vcpu_run,	68	__KVM_HOST_SMCCC_FUNC___kvm_vcpu_run,
68	__KVM_HOST_SMCCC_FUNC___kvm_flush_vm_context,	69	__KVM_HOST_SMCCC_FUNC___kvm_flush_vm_context,

arch/arm64/include/asm/kvm_emulate.h+1−1

@@ -388,7 +388,7 @@ static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu) · 387 unmodified lines
388	*vcpu_cpsr(vcpu) \|= PSR_AA32_E_BIT;	388	*vcpu_cpsr(vcpu) \|= PSR_AA32_E_BIT;
389	} else {	389	} else {
390	u64 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1);	390	u64 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1);
391	sctlr \|= (1 << 25);	391	sctlr \|= SCTLR_ELx_EE;
392	vcpu_write_sys_reg(vcpu, sctlr, SCTLR_EL1);	392	vcpu_write_sys_reg(vcpu, sctlr, SCTLR_EL1);
393	}	393	}
394	}	394	}

arch/arm64/include/asm/kvm_host.h+19−27

@@ -26,7 +26,6 @@ · 25 unmodified lines
26	#include <asm/fpsimd.h>	26	#include <asm/fpsimd.h>
27	#include <asm/kvm.h>	27	#include <asm/kvm.h>
28	#include <asm/kvm_asm.h>	28	#include <asm/kvm_asm.h>
29	#include <asm/thread_info.h>
30		29
31	#define __KVM_HAVE_ARCH_INTC_INITIALIZED	30	#define __KVM_HAVE_ARCH_INTC_INITIALIZED
32		31
@@ -298,9 +297,6 @@ struct kvm_vcpu_arch { · 265 unmodified lines
298	/* Exception Information */	297	/* Exception Information */
299	struct kvm_vcpu_fault_info fault;	298	struct kvm_vcpu_fault_info fault;
300		299
301	/* State of various workarounds, see kvm_asm.h for bit assignment */
302	u64 workaround_flags;
303
304	/* Miscellaneous vcpu state flags */	300	/* Miscellaneous vcpu state flags */
305	u64 flags;	301	u64 flags;
306		302
@@ -321,8 +317,8 @@ struct kvm_vcpu_arch { · 14 unmodified lines
321	struct kvm_guest_debug_arch vcpu_debug_state;	317	struct kvm_guest_debug_arch vcpu_debug_state;
322	struct kvm_guest_debug_arch external_debug_state;	318	struct kvm_guest_debug_arch external_debug_state;
323		319
324	struct thread_info host_thread_info; / hyp VA */
325	struct user_fpsimd_state host_fpsimd_state; / hyp VA */	320	struct user_fpsimd_state host_fpsimd_state; / hyp VA */
		321	struct task_struct *parent_task;
326		322
327	struct {	323	struct {
328	/* {Break,watch}point registers */	324	/* {Break,watch}point registers */
@@ -367,9 +363,6 @@ struct kvm_vcpu_arch { · 38 unmodified lines
367	int target;	363	int target;
368	DECLARE_BITMAP(features, KVM_VCPU_MAX_FEATURES);	364	DECLARE_BITMAP(features, KVM_VCPU_MAX_FEATURES);
369		365
370	/* Detect first run of a vcpu */
371	bool has_run_once;
372
373	/* Virtual SError ESR to restore when HCR_EL2.VSE is set */	366	/* Virtual SError ESR to restore when HCR_EL2.VSE is set */
374	u64 vsesr_el2;	367	u64 vsesr_el2;
375		368
@@ -411,20 +404,17 @@ struct kvm_vcpu_arch { · 35 unmodified lines
411	#define KVM_ARM64_DEBUG_DIRTY (1 << 0)	404	#define KVM_ARM64_DEBUG_DIRTY (1 << 0)
412	#define KVM_ARM64_FP_ENABLED (1 << 1) /* guest FP regs loaded */	405	#define KVM_ARM64_FP_ENABLED (1 << 1) /* guest FP regs loaded */
413	#define KVM_ARM64_FP_HOST (1 << 2) /* host FP regs loaded */	406	#define KVM_ARM64_FP_HOST (1 << 2) /* host FP regs loaded */
414	#define KVM_ARM64_HOST_SVE_IN_USE (1 << 3) /* backup for host TIF_SVE */
415	#define KVM_ARM64_HOST_SVE_ENABLED (1 << 4) /* SVE enabled for EL0 */	407	#define KVM_ARM64_HOST_SVE_ENABLED (1 << 4) /* SVE enabled for EL0 */
416	#define KVM_ARM64_GUEST_HAS_SVE (1 << 5) /* SVE exposed to guest */	408	#define KVM_ARM64_GUEST_HAS_SVE (1 << 5) /* SVE exposed to guest */
417	#define KVM_ARM64_VCPU_SVE_FINALIZED (1 << 6) /* SVE config completed */	409	#define KVM_ARM64_VCPU_SVE_FINALIZED (1 << 6) /* SVE config completed */
418	#define KVM_ARM64_GUEST_HAS_PTRAUTH (1 << 7) /* PTRAUTH exposed to guest */	410	#define KVM_ARM64_GUEST_HAS_PTRAUTH (1 << 7) /* PTRAUTH exposed to guest */
419	#define KVM_ARM64_PENDING_EXCEPTION (1 << 8) /* Exception pending */	411	#define KVM_ARM64_PENDING_EXCEPTION (1 << 8) /* Exception pending */
		412	/*
		413	* Overlaps with KVM_ARM64_EXCEPT_MASK on purpose so that it can't be
		414	* set together with an exception...
		415	*/
		416	#define KVM_ARM64_INCREMENT_PC (1 << 9) /* Increment PC */
420	#define KVM_ARM64_EXCEPT_MASK (7 << 9) /* Target EL/MODE */	417	#define KVM_ARM64_EXCEPT_MASK (7 << 9) /* Target EL/MODE */
421	#define KVM_ARM64_DEBUG_STATE_SAVE_SPE (1 << 12) /* Save SPE context if active */
422	#define KVM_ARM64_DEBUG_STATE_SAVE_TRBE (1 << 13) /* Save TRBE context if active */
423
424	#define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE \| \
425	KVM_GUESTDBG_USE_SW_BP \| \
426	KVM_GUESTDBG_USE_HW \| \
427	KVM_GUESTDBG_SINGLESTEP)
428	/*	418	/*
429	* When KVM_ARM64_PENDING_EXCEPTION is set, KVM_ARM64_EXCEPT_MASK can	419	* When KVM_ARM64_PENDING_EXCEPTION is set, KVM_ARM64_EXCEPT_MASK can
430	* take the following values:	420	* take the following values:
@@ -442,11 +432,14 @@ struct kvm_vcpu_arch { · 11 unmodified lines
442	#define KVM_ARM64_EXCEPT_AA64_EL1 (0 << 11)	432	#define KVM_ARM64_EXCEPT_AA64_EL1 (0 << 11)
443	#define KVM_ARM64_EXCEPT_AA64_EL2 (1 << 11)	433	#define KVM_ARM64_EXCEPT_AA64_EL2 (1 << 11)
444		434
445	/*	435	#define KVM_ARM64_DEBUG_STATE_SAVE_SPE (1 << 12) /* Save SPE context if active */
446	* Overlaps with KVM_ARM64_EXCEPT_MASK on purpose so that it can't be	436	#define KVM_ARM64_DEBUG_STATE_SAVE_TRBE (1 << 13) /* Save TRBE context if active */
447	* set together with an exception...	437	#define KVM_ARM64_FP_FOREIGN_FPSTATE (1 << 14)
448	*/	438
449	#define KVM_ARM64_INCREMENT_PC (1 << 9) /* Increment PC */	439	#define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE \| \
		440	KVM_GUESTDBG_USE_SW_BP \| \
		441	KVM_GUESTDBG_USE_HW \| \
		442	KVM_GUESTDBG_SINGLESTEP)
450		443
451	#define vcpu_has_sve(vcpu) (system_supports_sve() && \	444	#define vcpu_has_sve(vcpu) (system_supports_sve() && \
452	((vcpu)->arch.flags & KVM_ARM64_GUEST_HAS_SVE))	445	((vcpu)->arch.flags & KVM_ARM64_GUEST_HAS_SVE))
@@ -606,6 +599,8 @@ int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, · 153 unmodified lines
606	void kvm_arm_halt_guest(struct kvm *kvm);	599	void kvm_arm_halt_guest(struct kvm *kvm);
607	void kvm_arm_resume_guest(struct kvm *kvm);	600	void kvm_arm_resume_guest(struct kvm *kvm);
608		601
		602	#define vcpu_has_run_once(vcpu) !!rcu_access_pointer((vcpu)->pid)
		603
609	#ifndef __KVM_NVHE_HYPERVISOR__	604	#ifndef __KVM_NVHE_HYPERVISOR__
610	#define kvm_call_hyp_nvhe(f, ...) \	605	#define kvm_call_hyp_nvhe(f, ...) \
611	({ \	606	({ \
@@ -736,8 +731,10 @@ long kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm, · 124 unmodified lines
736	/* Guest/host FPSIMD coordination helpers */	731	/* Guest/host FPSIMD coordination helpers */
737	int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu);	732	int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu);
738	void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu);	733	void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu);
		734	void kvm_arch_vcpu_ctxflush_fp(struct kvm_vcpu *vcpu);
739	void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu);	735	void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu);
740	void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu);	736	void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu);
		737	void kvm_vcpu_unshare_task_fp(struct kvm_vcpu *vcpu);
741		738
742	static inline bool kvm_pmu_counter_deferred(struct perf_event_attr *attr)	739	static inline bool kvm_pmu_counter_deferred(struct perf_event_attr *attr)
743	{	740	{
@@ -748,12 +745,7 @@ static inline bool kvm_pmu_counter_deferred(struct perf_event_attr *attr) · 4 unmodified lines
748	void kvm_arch_vcpu_load_debug_state_flags(struct kvm_vcpu *vcpu);	745	void kvm_arch_vcpu_load_debug_state_flags(struct kvm_vcpu *vcpu);
749	void kvm_arch_vcpu_put_debug_state_flags(struct kvm_vcpu *vcpu);	746	void kvm_arch_vcpu_put_debug_state_flags(struct kvm_vcpu *vcpu);
750		747
751	#ifdef CONFIG_KVM /* Avoid conflicts with core headers if CONFIG_KVM=n */	748	#ifdef CONFIG_KVM
752	static inline int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
753	{
754	return kvm_arch_vcpu_run_map_fp(vcpu);
755	}
756
757	void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr);	749	void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr);
758	void kvm_clr_pmu_events(u32 clr);	750	void kvm_clr_pmu_events(u32 clr);
759		751

arch/arm64/include/asm/kvm_hyp.h+0−1

@@ -90,7 +90,6 @@ void __debug_restore_host_buffers_nvhe(struct kvm_vcpu *vcpu); · 89 unmodified lines
90		90
91	void __fpsimd_save_state(struct user_fpsimd_state *fp_regs);	91	void __fpsimd_save_state(struct user_fpsimd_state *fp_regs);
92	void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs);	92	void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs);
93	void __sve_save_state(void sve_pffr, u32 fpsr);
94	void __sve_restore_state(void sve_pffr, u32 fpsr);	93	void __sve_restore_state(void sve_pffr, u32 fpsr);
95		94
96	#ifndef __KVM_NVHE_HYPERVISOR__	95	#ifndef __KVM_NVHE_HYPERVISOR__

arch/arm64/include/asm/kvm_mmu.h+2−0

@@ -150,6 +150,8 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v) · 149 unmodified lines
150	#include <asm/kvm_pgtable.h>	150	#include <asm/kvm_pgtable.h>
151	#include <asm/stage2_pgtable.h>	151	#include <asm/stage2_pgtable.h>
152		152
		153	int kvm_share_hyp(void from, void to);
		154	void kvm_unshare_hyp(void from, void to);
153	int create_hyp_mappings(void from, void to, enum kvm_pgtable_prot prot);	155	int create_hyp_mappings(void from, void to, enum kvm_pgtable_prot prot);
154	int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size,	156	int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size,
155	void __iomem **kaddr,	157	void __iomem **kaddr,

arch/arm64/include/asm/kvm_pgtable.h+25−5

 int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,
 			enum kvm_pgtable_prot prot);
+/**
+ * kvm_pgtable_hyp_unmap() - Remove a mapping from a hypervisor stage-1 page-table.
+ * @pgt:	Page-table structure initialised by kvm_pgtable_hyp_init().
+ * @addr:	Virtual address from which to remove the mapping.
+ * @size:	Size of the mapping.
+ *
+ * The offset of @addr within a page is ignored, @size is rounded-up to
+ * the next page boundary and @phys is rounded-down to the previous page
+ * boundary.
+ *
+ * TLB invalidation is performed for each page-table entry cleared during the
+ * unmapping operation and the reference count for the page-table page
+ * containing the cleared entry is decremented, with unreferenced pages being
+ * freed. The unmapping operation will stop early if it encounters either an
+ * invalid page-table entry or a valid block mapping which maps beyond the range
+ * being unmapped.
+ *
+ * Return: Number of bytes unmapped, which may be 0.
+ */
+u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size);
 /**
  * kvm_get_vtcr() - Helper to construct VTCR_EL2
  * @mmfr0:	Sanitized value of SYS_ID_AA64MMFR0_EL1 register.
 /**
  * __kvm_pgtable_stage2_init() - Initialise a guest stage-2 page-table.
  * @pgt:	Uninitialised page-table structure to initialise.
- * @arch:	Arch-specific KVM structure representing the guest virtual
+ * @mmu:	S2 MMU context for this S2 translation
- *		machine.
  * @mm_ops:	Memory management callbacks.
  * @flags:	Stage-2 configuration flags.
  * @force_pte_cb: Function that returns true if page level mappings must
  *
  * Return: 0 on success, negative error code on failure.
  */
-int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_arch *arch,
+int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,
 			      struct kvm_pgtable_mm_ops *mm_ops,
 			      enum kvm_pgtable_stage2_flags flags,
 			      kvm_pgtable_force_pte_cb_t force_pte_cb);
-#define kvm_pgtable_stage2_init(pgt, arch, mm_ops) \
+#define kvm_pgtable_stage2_init(pgt, mmu, mm_ops) \
-	__kvm_pgtable_stage2_init(pgt, arch, mm_ops, 0, NULL)
+	__kvm_pgtable_stage2_init(pgt, mmu, mm_ops, 0, NULL)
 /**
  * kvm_pgtable_stage2_destroy() - Destroy an unused guest stage-2 page-table.

arch/arm64/include/asm/kvm_pkvm.h+71−0

@@ -0,0 +1,71 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Copyright (C) 2020 - Google LLC
	4	* Author: Quentin Perret <qperret@google.com>
	5	*/
	6	#ifndef __ARM64_KVM_PKVM_H__
	7	#define __ARM64_KVM_PKVM_H__
	8
	9	#include <linux/memblock.h>
	10	#include <asm/kvm_pgtable.h>
	11
	12	#define HYP_MEMBLOCK_REGIONS 128
	13
	14	extern struct memblock_region kvm_nvhe_sym(hyp_memory)[];
	15	extern unsigned int kvm_nvhe_sym(hyp_memblock_nr);
	16
	17	static inline unsigned long __hyp_pgtable_max_pages(unsigned long nr_pages)
	18	{
	19	unsigned long total = 0, i;
	20
	21	/* Provision the worst case scenario */
	22	for (i = 0; i < KVM_PGTABLE_MAX_LEVELS; i++) {
	23	nr_pages = DIV_ROUND_UP(nr_pages, PTRS_PER_PTE);
	24	total += nr_pages;
	25	}
	26
	27	return total;
	28	}
	29
	30	static inline unsigned long __hyp_pgtable_total_pages(void)
	31	{
	32	unsigned long res = 0, i;
	33
	34	/* Cover all of memory with page-granularity */
	35	for (i = 0; i < kvm_nvhe_sym(hyp_memblock_nr); i++) {
	36	struct memblock_region *reg = &kvm_nvhe_sym(hyp_memory)[i];
	37	res += __hyp_pgtable_max_pages(reg->size >> PAGE_SHIFT);
	38	}
	39
	40	return res;
	41	}
	42
	43	static inline unsigned long hyp_s1_pgtable_pages(void)
	44	{
	45	unsigned long res;
	46
	47	res = __hyp_pgtable_total_pages();
	48
	49	/* Allow 1 GiB for private mappings */
	50	res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT);
	51
	52	return res;
	53	}
	54
	55	static inline unsigned long host_s2_pgtable_pages(void)
	56	{
	57	unsigned long res;
	58
	59	/*
	60	* Include an extra 16 pages to safely upper-bound the worst case of
	61	* concatenated pgds.
	62	*/
	63	res = __hyp_pgtable_total_pages() + 16;
	64
	65	/* Allow 1 GiB for MMIO mappings */
	66	res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT);
	67
	68	return res;
	69	}
	70
	71	#endif /* __ARM64_KVM_PKVM_H__ */

arch/arm64/include/asm/mmu.h+1−0

@@ -15,6 +15,7 @@ · 14 unmodified lines
15	#ifndef __ASSEMBLY__	15	#ifndef __ASSEMBLY__
16		16
17	#include <linux/refcount.h>	17	#include <linux/refcount.h>
		18	#include <asm/cpufeature.h>
18		19
19	typedef struct {	20	typedef struct {
20	atomic64_t id;	21	atomic64_t id;

arch/arm64/include/asm/sysreg.h+1−0

@@ -937,6 +937,7 @@ · 936 unmodified lines
937	#define ID_AA64DFR0_PMUVER_8_1 0x4	937	#define ID_AA64DFR0_PMUVER_8_1 0x4
938	#define ID_AA64DFR0_PMUVER_8_4 0x5	938	#define ID_AA64DFR0_PMUVER_8_4 0x5
939	#define ID_AA64DFR0_PMUVER_8_5 0x6	939	#define ID_AA64DFR0_PMUVER_8_5 0x6
		940	#define ID_AA64DFR0_PMUVER_8_7 0x7
940	#define ID_AA64DFR0_PMUVER_IMP_DEF 0xf	941	#define ID_AA64DFR0_PMUVER_IMP_DEF 0xf
941		942
942	#define ID_AA64DFR0_PMSVER_8_2 0x1	943	#define ID_AA64DFR0_PMSVER_8_2 0x1

arch/arm64/kernel/asm-offsets.c+0−1

@@ -111,7 +111,6 @@ int main(void) · 110 unmodified lines
111	#ifdef CONFIG_KVM	111	#ifdef CONFIG_KVM
112	DEFINE(VCPU_CONTEXT, offsetof(struct kvm_vcpu, arch.ctxt));	112	DEFINE(VCPU_CONTEXT, offsetof(struct kvm_vcpu, arch.ctxt));
113	DEFINE(VCPU_FAULT_DISR, offsetof(struct kvm_vcpu, arch.fault.disr_el1));	113	DEFINE(VCPU_FAULT_DISR, offsetof(struct kvm_vcpu, arch.fault.disr_el1));
114	DEFINE(VCPU_WORKAROUND_FLAGS, offsetof(struct kvm_vcpu, arch.workaround_flags));
115	DEFINE(VCPU_HCR_EL2, offsetof(struct kvm_vcpu, arch.hcr_el2));	114	DEFINE(VCPU_HCR_EL2, offsetof(struct kvm_vcpu, arch.hcr_el2));
116	DEFINE(CPU_USER_PT_REGS, offsetof(struct kvm_cpu_context, regs));	115	DEFINE(CPU_USER_PT_REGS, offsetof(struct kvm_cpu_context, regs));
117	DEFINE(CPU_RGSR_EL1, offsetof(struct kvm_cpu_context, sys_regs[RGSR_EL1]));	116	DEFINE(CPU_RGSR_EL1, offsetof(struct kvm_cpu_context, sys_regs[RGSR_EL1]));

arch/arm64/kernel/fpsimd.c+5−1

  * indicate whether or not the userland FPSIMD state of the current task is
  * present in the registers. The flag is set unless the FPSIMD registers of this
  * CPU currently contain the most recent userland FPSIMD state of the current
- * task.
+ * task. If the task is behaving as a VMM, then this is will be managed by
+ * KVM which will clear it to indicate that the vcpu FPSIMD state is currently
+ * loaded on the CPU, allowing the state to be saved if a FPSIMD-aware
+ * softirq kicks in. Upon vcpu_put(), KVM will save the vcpu FP state and
+ * flag the register state as invalid.
  *
  * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may
  * save the task's FPSIMD context back to task_struct from softirq context.

arch/arm64/kvm/.gitignore+2−0

@@ -0,0 +1,2 @@
	1	# SPDX-License-Identifier: GPL-2.0-only
	2	hyp_constants.h

arch/arm64/kvm/Makefile+17−1

@@ -13,7 +13,7 @@ obj-$(CONFIG_KVM) += hyp/ · 12 unmodified lines
13	kvm-y += arm.o mmu.o mmio.o psci.o perf.o hypercalls.o pvtime.o \	13	kvm-y += arm.o mmu.o mmio.o psci.o perf.o hypercalls.o pvtime.o \
14	inject_fault.o va_layout.o handle_exit.o \	14	inject_fault.o va_layout.o handle_exit.o \
15	guest.o debug.o reset.o sys_regs.o \	15	guest.o debug.o reset.o sys_regs.o \
16	vgic-sys-reg-v3.o fpsimd.o pmu.o \	16	vgic-sys-reg-v3.o fpsimd.o pmu.o pkvm.o \
17	arch_timer.o trng.o\	17	arch_timer.o trng.o\
18	vgic/vgic.o vgic/vgic-init.o \	18	vgic/vgic.o vgic/vgic-init.o \
19	vgic/vgic-irqfd.o vgic/vgic-v2.o \	19	vgic/vgic-irqfd.o vgic/vgic-v2.o \
@@ -23,3 +23,19 @@ kvm-y += arm.o mmu.o mmio.o psci.o perf.o hypercalls.o pvtime.o \ · 3 unmodified lines
23	vgic/vgic-its.o vgic/vgic-debug.o	23	vgic/vgic-its.o vgic/vgic-debug.o
24		24
25	kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o	25	kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o
		26
		27	always-y := hyp_constants.h hyp-constants.s
		28
		29	define rule_gen_hyp_constants
		30	$(call filechk,offsets,__HYP_CONSTANTS_H__)
		31	endef
		32
		33	CFLAGS_hyp-constants.o = -I $(srctree)/$(src)/hyp/include
		34	$(obj)/hyp-constants.s: $(src)/hyp/hyp-constants.c FORCE
		35	$(call if_changed_dep,cc_s_c)
		36
		37	$(obj)/hyp_constants.h: $(obj)/hyp-constants.s FORCE
		38	$(call if_changed_rule,gen_hyp_constants)
		39
		40	obj-kvm := $(addprefix $(obj)/, $(kvm-y))
		41	$(obj-kvm): $(obj)/hyp_constants.h

arch/arm64/kvm/arm.c+37−27

 	if (ret)
 		return ret;
-	ret = create_hyp_mappings(kvm, kvm + 1, PAGE_HYP);
+	ret = kvm_share_hyp(kvm, kvm + 1);
 	if (ret)
 		goto out_free_stage2_pgd;
 	kvm_vgic_destroy(kvm);
 	kvm_destroy_vcpus(kvm);
+	kvm_unshare_hyp(kvm, kvm + 1);
 }
 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 	if (err)
 		return err;
-	return create_hyp_mappings(vcpu, vcpu + 1, PAGE_HYP);
+	return kvm_share_hyp(vcpu, vcpu + 1);
 }
 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 {
-	if (vcpu->arch.has_run_once && unlikely(!irqchip_in_kernel(vcpu->kvm)))
+	if (vcpu_has_run_once(vcpu) && unlikely(!irqchip_in_kernel(vcpu->kvm)))
 		static_branch_dec(&userspace_irqchip_in_use);
 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
 	spin_unlock(&kvm_vmid_lock);
 }
-static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
+static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
+{
+	return vcpu->arch.target >= 0;
+}
+/*
+ * Handle both the initialisation that is being done when the vcpu is
+ * run for the first time, as well as the updates that must be
+ * performed each time we get a new thread dealing with this vcpu.
+ */
+int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
 {
 	struct kvm *kvm = vcpu->kvm;
-	int ret = 0;
+	int ret;
-	if (likely(vcpu->arch.has_run_once))
+	if (!kvm_vcpu_initialized(vcpu))
-		return 0;
+		return -ENOEXEC;
 	if (!kvm_arm_vcpu_is_finalized(vcpu))
 		return -EPERM;
-	vcpu->arch.has_run_once = true;
+	ret = kvm_arch_vcpu_run_map_fp(vcpu);
+	if (ret)
+		return ret;
+	if (likely(vcpu_has_run_once(vcpu)))
+		return 0;
 	kvm_arm_vcpu_init_debug(vcpu);
 		ret = kvm_vgic_map_resources(kvm);
 		if (ret)
 			return ret;
-	} else {
-		/*
-		 * Tell the rest of the code that there are userspace irqchip
-		 * VMs in the wild.
-		 */
-		static_branch_inc(&userspace_irqchip_in_use);
 	}
 	ret = kvm_timer_enable(vcpu);
 		return ret;
 	ret = kvm_arm_pmu_v3_enable(vcpu);
+	if (ret)
+		return ret;
+	if (!irqchip_in_kernel(kvm)) {
+		/*
+		 * Tell the rest of the code that there are userspace irqchip
+		 * VMs in the wild.
+		 */
+		static_branch_inc(&userspace_irqchip_in_use);
+	}
 	/*
 	 * Initialize traps for protected VMs.
 	preempt_enable();
 }
-static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
-{
-	return vcpu->arch.target >= 0;
-}
 static void check_vcpu_requests(struct kvm_vcpu *vcpu)
 {
 	if (kvm_request_pending(vcpu)) {
 	struct kvm_run *run = vcpu->run;
 	int ret;
-	if (unlikely(!kvm_vcpu_initialized(vcpu)))
-		return -ENOEXEC;
-	ret = kvm_vcpu_first_run_init(vcpu);
-	if (ret)
-		return ret;
 	if (run->exit_reason == KVM_EXIT_MMIO) {
 		ret = kvm_handle_mmio_return(vcpu);
 		if (ret)
 		}
 		kvm_arm_setup_debug(vcpu);
+		kvm_arch_vcpu_ctxflush_fp(vcpu);
 		/**************************************************************
 		 * Enter the guest
 	 * need to invalidate the I-cache though, as FWB does *not*
 	 * imply CTR_EL0.DIC.
 	 */
-	if (vcpu->arch.has_run_once) {
+	if (vcpu_has_run_once(vcpu)) {
 		if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
 			stage2_unmap_vm(vcpu->kvm);
 		else

arch/arm64/kvm/fpsimd.c+41−38

  */
 #include <linux/irqflags.h>
 #include <linux/sched.h>
-#include <linux/thread_info.h>
 #include <linux/kvm_host.h>
 #include <asm/fpsimd.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_mmu.h>
 #include <asm/sysreg.h>
+void kvm_vcpu_unshare_task_fp(struct kvm_vcpu *vcpu)
+{
+	struct task_struct *p = vcpu->arch.parent_task;
+	struct user_fpsimd_state *fpsimd;
+	if (!is_protected_kvm_enabled() || !p)
+		return;
+	fpsimd = &p->thread.uw.fpsimd_state;
+	kvm_unshare_hyp(fpsimd, fpsimd + 1);
+	put_task_struct(p);
+}
 /*
  * Called on entry to KVM_RUN unless this vcpu previously ran at least
  * once and the most recent prior KVM_RUN for this vcpu was called from
 {
 	int ret;
-	struct thread_info *ti = &current->thread_info;
 	struct user_fpsimd_state *fpsimd = &current->thread.uw.fpsimd_state;
-	/*
+	kvm_vcpu_unshare_task_fp(vcpu);
-	 * Make sure the host task thread flags and fpsimd state are
-	 * visible to hyp:
-	 */
-	ret = create_hyp_mappings(ti, ti + 1, PAGE_HYP);
-	if (ret)
-		goto error;
-	ret = create_hyp_mappings(fpsimd, fpsimd + 1, PAGE_HYP);
+	/* Make sure the host task fpsimd state is visible to hyp: */
+	ret = kvm_share_hyp(fpsimd, fpsimd + 1);
 	if (ret)
-		goto error;
+		return ret;
-	if (vcpu->arch.sve_state) {
-		void *sve_end;
-		sve_end = vcpu->arch.sve_state + vcpu_sve_state_size(vcpu);
+	vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd);
-		ret = create_hyp_mappings(vcpu->arch.sve_state, sve_end,
+	/*
-					  PAGE_HYP);
+	 * We need to keep current's task_struct pinned until its data has been
-		if (ret)
+	 * unshared with the hypervisor to make sure it is not re-used by the
-			goto error;
+	 * kernel and donated to someone else while already shared -- see
+	 * kvm_vcpu_unshare_task_fp() for the matching put_task_struct().
+	 */
+	if (is_protected_kvm_enabled()) {
+		get_task_struct(current);
+		vcpu->arch.parent_task = current;
 	}
-	vcpu->arch.host_thread_info = kern_hyp_va(ti);
+	return 0;
-	vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd);
-error:
-	return ret;
 }
 /*
  *
  * Here, we just set the correct metadata to indicate that the FPSIMD
  * state in the cpu regs (if any) belongs to current on the host.
- *
- * TIF_SVE is backed up here, since it may get clobbered with guest state.
- * This flag is restored by kvm_arch_vcpu_put_fp(vcpu).
  */
 void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu)
 {
 	BUG_ON(!current->mm);
+	BUG_ON(test_thread_flag(TIF_SVE));
-	vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED |
+	vcpu->arch.flags &= ~KVM_ARM64_FP_ENABLED;
-			      KVM_ARM64_HOST_SVE_IN_USE |
-			      KVM_ARM64_HOST_SVE_ENABLED);
 	vcpu->arch.flags |= KVM_ARM64_FP_HOST;
-	if (test_thread_flag(TIF_SVE))
-		vcpu->arch.flags |= KVM_ARM64_HOST_SVE_IN_USE;
 	if (read_sysreg(cpacr_el1) & CPACR_EL1_ZEN_EL0EN)
 		vcpu->arch.flags |= KVM_ARM64_HOST_SVE_ENABLED;
 }
+void kvm_arch_vcpu_ctxflush_fp(struct kvm_vcpu *vcpu)
+{
+	if (test_thread_flag(TIF_FOREIGN_FPSTATE))
+		vcpu->arch.flags |= KVM_ARM64_FP_FOREIGN_FPSTATE;
+	else
+		vcpu->arch.flags &= ~KVM_ARM64_FP_FOREIGN_FPSTATE;
+}
 /*
  * If the guest FPSIMD state was loaded, update the host's context
  * tracking data mark the CPU FPSIMD regs as dirty and belonging to vcpu
 void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu)
 {
 	unsigned long flags;
-	bool host_has_sve = system_supports_sve();
-	bool guest_has_sve = vcpu_has_sve(vcpu);
 	local_irq_save(flags);
 	if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) {
-		if (guest_has_sve) {
+		if (vcpu_has_sve(vcpu)) {
 			__vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_el1(SYS_ZCR);
 			/* Restore the VL that was saved when bound to the CPU */
 		}
 		fpsimd_save_and_flush_cpu_state();
-	} else if (has_vhe() && host_has_sve) {
+	} else if (has_vhe() && system_supports_sve()) {
 		/*
 		 * The FPSIMD/SVE state in the CPU has not been touched, and we
 		 * have SVE (and VHE): CPACR_EL1 (alias CPTR_EL2) has been
 			sysreg_clear_set(CPACR_EL1, CPACR_EL1_ZEN_EL0EN, 0);
 	}
-	update_thread_flag(TIF_SVE,
+	update_thread_flag(TIF_SVE, 0);
-			   vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE);
 	local_irq_restore(flags);
 }

arch/arm64/kvm/hyp/Makefile+1−1

@@ -10,4 +10,4 @@ subdir-ccflags-y := -I$(incdir) \ · 9 unmodified lines
10	-DDISABLE_BRANCH_PROFILING \	10	-DDISABLE_BRANCH_PROFILING \
11	$(DISABLE_STACKLEAK_PLUGIN)	11	$(DISABLE_STACKLEAK_PLUGIN)
12		12
13	obj-$(CONFIG_KVM) += vhe/ nvhe/ pgtable.o reserved_mem.o	13	obj-$(CONFIG_KVM) += vhe/ nvhe/ pgtable.o

arch/arm64/kvm/hyp/fpsimd.S+0−6

@@ -25,9 +25,3 @@ SYM_FUNC_START(__sve_restore_state) · 24 unmodified lines
25	sve_load 0, x1, x2, 3	25	sve_load 0, x1, x2, 3
26	ret	26	ret
27	SYM_FUNC_END(__sve_restore_state)	27	SYM_FUNC_END(__sve_restore_state)
28
29	SYM_FUNC_START(__sve_save_state)
30	mov x2, #1
31	sve_save 0, x1, x2, 3
32	ret
33	SYM_FUNC_END(__sve_save_state)

arch/arm64/kvm/hyp/hyp-constants.c+10−0

@@ -0,0 +1,10 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2
	3	#include <linux/kbuild.h>
	4	#include <nvhe/memory.h>
	5
	6	int main(void)
	7	{
	8	DEFINE(STRUCT_HYP_PAGE_SIZE, sizeof(struct hyp_page));
	9	return 0;
	10	}

arch/arm64/kvm/hyp/include/hyp/switch.h+4−26

 #include <asm/fpsimd.h>
 #include <asm/debug-monitors.h>
 #include <asm/processor.h>
-#include <asm/thread_info.h>
 struct kvm_exception_table_entry {
 	int insn, fixup;
 	 * trap the accesses.
 	 */
 	if (!system_supports_fpsimd() ||
-	    vcpu->arch.host_thread_info->flags & _TIF_FOREIGN_FPSTATE)
+	    vcpu->arch.flags & KVM_ARM64_FP_FOREIGN_FPSTATE)
 		vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED |
 				      KVM_ARM64_FP_HOST);
 	return __get_fault_info(vcpu->arch.fault.esr_el2, &vcpu->arch.fault);
 }
-static inline void __hyp_sve_save_host(struct kvm_vcpu *vcpu)
-{
-	struct thread_struct *thread;
-	thread = container_of(vcpu->arch.host_fpsimd_state, struct thread_struct,
-			      uw.fpsimd_state);
-	__sve_save_state(sve_pffr(thread), &vcpu->arch.host_fpsimd_state->fpsr);
-}
 static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
 {
 	sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
  */
 static bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)
 {
-	bool sve_guest, sve_host;
+	bool sve_guest;
 	u8 esr_ec;
 	u64 reg;
 	if (!system_supports_fpsimd())
 		return false;
-	if (system_supports_sve()) {
+	sve_guest = vcpu_has_sve(vcpu);
-		sve_guest = vcpu_has_sve(vcpu);
-		sve_host = vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE;
-	} else {
-		sve_guest = false;
-		sve_host = false;
-	}
 	esr_ec = kvm_vcpu_trap_get_class(vcpu);
 	/* Don't handle SVE traps for non-SVE vcpus here: */
 	isb();
 	if (vcpu->arch.flags & KVM_ARM64_FP_HOST) {
-		if (sve_host)
+		__fpsimd_save_state(vcpu->arch.host_fpsimd_state);
-			__hyp_sve_save_host(vcpu);
-		else
-			__fpsimd_save_state(vcpu->arch.host_fpsimd_state);
 		vcpu->arch.flags &= ~KVM_ARM64_FP_HOST;
 	}

arch/arm64/kvm/hyp/include/nvhe/mem_protect.h+6−0

 	PKVM_PAGE_OWNED			= 0ULL,
 	PKVM_PAGE_SHARED_OWNED		= KVM_PGTABLE_PROT_SW0,
 	PKVM_PAGE_SHARED_BORROWED	= KVM_PGTABLE_PROT_SW1,
+	__PKVM_PAGE_RESERVED		= KVM_PGTABLE_PROT_SW0 |
+					  KVM_PGTABLE_PROT_SW1,
+	/* Meta-states which aren't encoded directly in the PTE's SW bits */
+	PKVM_NOPAGE,
 };
 #define PKVM_PAGE_STATE_PROT_MASK	(KVM_PGTABLE_PROT_SW0 | KVM_PGTABLE_PROT_SW1)
 int __pkvm_prot_finalize(void);
 int __pkvm_host_share_hyp(u64 pfn);
+int __pkvm_host_unshare_hyp(u64 pfn);
 bool addr_is_memory(phys_addr_t phys);
 int host_stage2_idmap_locked(phys_addr_t addr, u64 size, enum kvm_pgtable_prot prot);

arch/arm64/kvm/hyp/include/nvhe/mm.h+0−59

@@ -10,13 +10,8 @@ · 9 unmodified lines
10	#include <nvhe/memory.h>	10	#include <nvhe/memory.h>
11	#include <nvhe/spinlock.h>	11	#include <nvhe/spinlock.h>
12		12
13	#define HYP_MEMBLOCK_REGIONS 128
14	extern struct memblock_region kvm_nvhe_sym(hyp_memory)[];
15	extern unsigned int kvm_nvhe_sym(hyp_memblock_nr);
16	extern struct kvm_pgtable pkvm_pgtable;	13	extern struct kvm_pgtable pkvm_pgtable;
17	extern hyp_spinlock_t pkvm_pgd_lock;	14	extern hyp_spinlock_t pkvm_pgd_lock;
18	extern struct hyp_pool hpool;
19	extern u64 __io_map_base;
20		15
21	int hyp_create_idmap(u32 hyp_va_bits);	16	int hyp_create_idmap(u32 hyp_va_bits);
22	int hyp_map_vectors(void);	17	int hyp_map_vectors(void);
@@ -39,58 +34,4 @@ static inline void hyp_vmemmap_range(phys_addr_t phys, unsigned long size, · 16 unmodified lines
39	end = ALIGN(end, PAGE_SIZE);	34	end = ALIGN(end, PAGE_SIZE);
40	}	35	}
41		36
42	static inline unsigned long __hyp_pgtable_max_pages(unsigned long nr_pages)
43	{
44	unsigned long total = 0, i;
45
46	/* Provision the worst case scenario */
47	for (i = 0; i < KVM_PGTABLE_MAX_LEVELS; i++) {
48	nr_pages = DIV_ROUND_UP(nr_pages, PTRS_PER_PTE);
49	total += nr_pages;
50	}
51
52	return total;
53	}
54
55	static inline unsigned long __hyp_pgtable_total_pages(void)
56	{
57	unsigned long res = 0, i;
58
59	/* Cover all of memory with page-granularity */
60	for (i = 0; i < kvm_nvhe_sym(hyp_memblock_nr); i++) {
61	struct memblock_region *reg = &kvm_nvhe_sym(hyp_memory)[i];
62	res += __hyp_pgtable_max_pages(reg->size >> PAGE_SHIFT);
63	}
64
65	return res;
66	}
67
68	static inline unsigned long hyp_s1_pgtable_pages(void)
69	{
70	unsigned long res;
71
72	res = __hyp_pgtable_total_pages();
73
74	/* Allow 1 GiB for private mappings */
75	res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT);
76
77	return res;
78	}
79
80	static inline unsigned long host_s2_pgtable_pages(void)
81	{
82	unsigned long res;
83
84	/*
85	* Include an extra 16 pages to safely upper-bound the worst case of
86	* concatenated pgds.
87	*/
88	res = __hyp_pgtable_total_pages() + 16;
89
90	/* Allow 1 GiB for MMIO mappings */
91	res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT);
92
93	return res;
94	}
95
96	#endif /* __KVM_HYP_MM_H */	37	#endif /* __KVM_HYP_MM_H */

arch/arm64/kvm/hyp/nvhe/early_alloc.c+5−0

 	return hyp_early_alloc_contig(1);
 }
+static void hyp_early_alloc_get_page(void *addr) { }
+static void hyp_early_alloc_put_page(void *addr) { }
 void hyp_early_alloc_init(void *virt, unsigned long size)
 {
 	base = cur = (unsigned long)virt;
 	hyp_early_alloc_mm_ops.zalloc_page = hyp_early_alloc_page;
 	hyp_early_alloc_mm_ops.phys_to_virt = hyp_phys_to_virt;
 	hyp_early_alloc_mm_ops.virt_to_phys = hyp_virt_to_phys;
+	hyp_early_alloc_mm_ops.get_page = hyp_early_alloc_get_page;
+	hyp_early_alloc_mm_ops.put_page = hyp_early_alloc_put_page;
 }

arch/arm64/kvm/hyp/nvhe/hyp-main.c+8−0

 	cpu_reg(host_ctxt, 1) = __pkvm_host_share_hyp(pfn);
 }
+static void handle___pkvm_host_unshare_hyp(struct kvm_cpu_context *host_ctxt)
+{
+	DECLARE_REG(u64, pfn, host_ctxt, 1);
+	cpu_reg(host_ctxt, 1) = __pkvm_host_unshare_hyp(pfn);
+}
 static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ctxt)
 {
 	DECLARE_REG(phys_addr_t, phys, host_ctxt, 1);
 	HANDLE_FUNC(__pkvm_prot_finalize),
 	HANDLE_FUNC(__pkvm_host_share_hyp),
+	HANDLE_FUNC(__pkvm_host_unshare_hyp),
 	HANDLE_FUNC(__kvm_adjust_pc),
 	HANDLE_FUNC(__kvm_vcpu_run),
 	HANDLE_FUNC(__kvm_flush_vm_context),

arch/arm64/kvm/hyp/nvhe/mem_protect.c+428−77

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	#include <asm/kvm_hyp.h>	9	#include <asm/kvm_hyp.h>
10	#include <asm/kvm_mmu.h>	10	#include <asm/kvm_mmu.h>
11	#include <asm/kvm_pgtable.h>	11	#include <asm/kvm_pgtable.h>
		12	#include <asm/kvm_pkvm.h>
12	#include <asm/stage2_pgtable.h>	13	#include <asm/stage2_pgtable.h>
13		14
14	#include <hyp/fault.h>	15	#include <hyp/fault.h>
@@ -27,6 +28,26 @@ static struct hyp_pool host_s2_pool; · 12 unmodified lines
27		28
28	const u8 pkvm_hyp_id = 1;	29	const u8 pkvm_hyp_id = 1;
29		30
		31	static void host_lock_component(void)
		32	{
		33	hyp_spin_lock(&host_kvm.lock);
		34	}
		35
		36	static void host_unlock_component(void)
		37	{
		38	hyp_spin_unlock(&host_kvm.lock);
		39	}
		40
		41	static void hyp_lock_component(void)
		42	{
		43	hyp_spin_lock(&pkvm_pgd_lock);
		44	}
		45
		46	static void hyp_unlock_component(void)
		47	{
		48	hyp_spin_unlock(&pkvm_pgd_lock);
		49	}
		50
30	static void *host_s2_zalloc_pages_exact(size_t size)	51	static void *host_s2_zalloc_pages_exact(size_t size)
31	{	52	{
32	void *addr = hyp_alloc_pages(&host_s2_pool, get_order(size));	53	void *addr = hyp_alloc_pages(&host_s2_pool, get_order(size));
@@ -103,19 +124,19 @@ int kvm_host_prepare_stage2(void *pgt_pool_base) · 70 unmodified lines
103		124
104	prepare_host_vtcr();	125	prepare_host_vtcr();
105	hyp_spin_lock_init(&host_kvm.lock);	126	hyp_spin_lock_init(&host_kvm.lock);
		127	mmu->arch = &host_kvm.arch;
106		128
107	ret = prepare_s2_pool(pgt_pool_base);	129	ret = prepare_s2_pool(pgt_pool_base);
108	if (ret)	130	if (ret)
109	return ret;	131	return ret;
110		132
111	ret = __kvm_pgtable_stage2_init(&host_kvm.pgt, &host_kvm.arch,	133	ret = __kvm_pgtable_stage2_init(&host_kvm.pgt, mmu,
112	&host_kvm.mm_ops, KVM_HOST_S2_FLAGS,	134	&host_kvm.mm_ops, KVM_HOST_S2_FLAGS,
113	host_stage2_force_pte_cb);	135	host_stage2_force_pte_cb);
114	if (ret)	136	if (ret)
115	return ret;	137	return ret;
116		138
117	mmu->pgd_phys = __hyp_pa(host_kvm.pgt.pgd);	139	mmu->pgd_phys = __hyp_pa(host_kvm.pgt.pgd);
118	mmu->arch = &host_kvm.arch;
119	mmu->pgt = &host_kvm.pgt;	140	mmu->pgt = &host_kvm.pgt;
120	WRITE_ONCE(mmu->vmid.vmid_gen, 0);	141	WRITE_ONCE(mmu->vmid.vmid_gen, 0);
121	WRITE_ONCE(mmu->vmid.vmid, 0);	142	WRITE_ONCE(mmu->vmid.vmid, 0);
@@ -338,116 +359,446 @@ static int host_stage2_idmap(u64 addr) · 216 unmodified lines
338		359
339	prot = is_memory ? PKVM_HOST_MEM_PROT : PKVM_HOST_MMIO_PROT;	360	prot = is_memory ? PKVM_HOST_MEM_PROT : PKVM_HOST_MMIO_PROT;
340		361
341	hyp_spin_lock(&host_kvm.lock);	362	host_lock_component();
342	ret = host_stage2_adjust_range(addr, &range);	363	ret = host_stage2_adjust_range(addr, &range);
343	if (ret)	364	if (ret)
344	goto unlock;	365	goto unlock;
345		366
346	ret = host_stage2_idmap_locked(range.start, range.end - range.start, prot);	367	ret = host_stage2_idmap_locked(range.start, range.end - range.start, prot);
347	unlock:	368	unlock:
348	hyp_spin_unlock(&host_kvm.lock);	369	host_unlock_component();
349		370
350	return ret;	371	return ret;
351	}	372	}
352		373
353	static inline bool check_prot(enum kvm_pgtable_prot prot,	374	void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt)
354	enum kvm_pgtable_prot required,
355	enum kvm_pgtable_prot denied)
356	{	375	{
357	return (prot & (required \| denied)) == required;	376	struct kvm_vcpu_fault_info fault;
		377	u64 esr, addr;
		378	int ret = 0;
		379
		380	esr = read_sysreg_el2(SYS_ESR);
		381	BUG_ON(!__get_fault_info(esr, &fault));
		382
		383	addr = (fault.hpfar_el2 & HPFAR_MASK) << 8;
		384	ret = host_stage2_idmap(addr);
		385	BUG_ON(ret && ret != -EAGAIN);
358	}	386	}
359		387
360	int __pkvm_host_share_hyp(u64 pfn)	388	/* This corresponds to locking order */
		389	enum pkvm_component_id {
		390	PKVM_ID_HOST,
		391	PKVM_ID_HYP,
		392	};
		393
		394	struct pkvm_mem_transition {
		395	u64 nr_pages;
		396
		397	struct {
		398	enum pkvm_component_id id;
		399	/* Address in the initiator's address space */
		400	u64 addr;
		401
		402	union {
		403	struct {
		404	/* Address in the completer's address space */
		405	u64 completer_addr;
		406	} host;
		407	};
		408	} initiator;
		409
		410	struct {
		411	enum pkvm_component_id id;
		412	} completer;
		413	};
		414
		415	struct pkvm_mem_share {
		416	const struct pkvm_mem_transition tx;
		417	const enum kvm_pgtable_prot completer_prot;
		418	};
		419
		420	struct check_walk_data {
		421	enum pkvm_page_state desired;
		422	enum pkvm_page_state (*get_page_state)(kvm_pte_t pte);
		423	};
		424
		425	static int __check_page_state_visitor(u64 addr, u64 end, u32 level,
		426	kvm_pte_t *ptep,
		427	enum kvm_pgtable_walk_flags flag,
		428	void * const arg)
361	{	429	{
362	phys_addr_t addr = hyp_pfn_to_phys(pfn);	430	struct check_walk_data *d = arg;
363	enum kvm_pgtable_prot prot, cur;	431	kvm_pte_t pte = *ptep;
364	void *virt = __hyp_va(addr);
365	enum pkvm_page_state state;
366	kvm_pte_t pte;
367	int ret;
368		432
369	if (!addr_is_memory(addr))	433	if (kvm_pte_valid(pte) && !addr_is_memory(kvm_pte_to_phys(pte)))
370	return -EINVAL;	434	return -EINVAL;
371		435
372	hyp_spin_lock(&host_kvm.lock);	436	return d->get_page_state(pte) == d->desired ? 0 : -EPERM;
373	hyp_spin_lock(&pkvm_pgd_lock);	437	}
		438
		439	static int check_page_state_range(struct kvm_pgtable *pgt, u64 addr, u64 size,
		440	struct check_walk_data *data)
		441	{
		442	struct kvm_pgtable_walker walker = {
		443	.cb = __check_page_state_visitor,
		444	.arg = data,
		445	.flags = KVM_PGTABLE_WALK_LEAF,
		446	};
		447
		448	return kvm_pgtable_walk(pgt, addr, size, &walker);
		449	}
		450
		451	static enum pkvm_page_state host_get_page_state(kvm_pte_t pte)
		452	{
		453	if (!kvm_pte_valid(pte) && pte)
		454	return PKVM_NOPAGE;
		455
		456	return pkvm_getstate(kvm_pgtable_stage2_pte_prot(pte));
		457	}
		458
		459	static int __host_check_page_state_range(u64 addr, u64 size,
		460	enum pkvm_page_state state)
		461	{
		462	struct check_walk_data d = {
		463	.desired = state,
		464	.get_page_state = host_get_page_state,
		465	};
		466
		467	hyp_assert_lock_held(&host_kvm.lock);
		468	return check_page_state_range(&host_kvm.pgt, addr, size, &d);
		469	}
		470
		471	static int __host_set_page_state_range(u64 addr, u64 size,
		472	enum pkvm_page_state state)
		473	{
		474	enum kvm_pgtable_prot prot = pkvm_mkstate(PKVM_HOST_MEM_PROT, state);
		475
		476	return host_stage2_idmap_locked(addr, size, prot);
		477	}
		478
		479	static int host_request_owned_transition(u64 *completer_addr,
		480	const struct pkvm_mem_transition *tx)
		481	{
		482	u64 size = tx->nr_pages * PAGE_SIZE;
		483	u64 addr = tx->initiator.addr;
		484
		485	*completer_addr = tx->initiator.host.completer_addr;
		486	return __host_check_page_state_range(addr, size, PKVM_PAGE_OWNED);
		487	}
		488
		489	static int host_request_unshare(u64 *completer_addr,
		490	const struct pkvm_mem_transition *tx)
		491	{
		492	u64 size = tx->nr_pages * PAGE_SIZE;
		493	u64 addr = tx->initiator.addr;
		494
		495	*completer_addr = tx->initiator.host.completer_addr;
		496	return __host_check_page_state_range(addr, size, PKVM_PAGE_SHARED_OWNED);
		497	}
		498
		499	static int host_initiate_share(u64 *completer_addr,
		500	const struct pkvm_mem_transition *tx)
		501	{
		502	u64 size = tx->nr_pages * PAGE_SIZE;
		503	u64 addr = tx->initiator.addr;
		504
		505	*completer_addr = tx->initiator.host.completer_addr;
		506	return __host_set_page_state_range(addr, size, PKVM_PAGE_SHARED_OWNED);
		507	}
		508
		509	static int host_initiate_unshare(u64 *completer_addr,
		510	const struct pkvm_mem_transition *tx)
		511	{
		512	u64 size = tx->nr_pages * PAGE_SIZE;
		513	u64 addr = tx->initiator.addr;
		514
		515	*completer_addr = tx->initiator.host.completer_addr;
		516	return __host_set_page_state_range(addr, size, PKVM_PAGE_OWNED);
		517	}
		518
		519	static enum pkvm_page_state hyp_get_page_state(kvm_pte_t pte)
		520	{
		521	if (!kvm_pte_valid(pte))
		522	return PKVM_NOPAGE;
		523
		524	return pkvm_getstate(kvm_pgtable_stage2_pte_prot(pte));
		525	}
		526
		527	static int __hyp_check_page_state_range(u64 addr, u64 size,
		528	enum pkvm_page_state state)
		529	{
		530	struct check_walk_data d = {
		531	.desired = state,
		532	.get_page_state = hyp_get_page_state,
		533	};
		534
		535	hyp_assert_lock_held(&pkvm_pgd_lock);
		536	return check_page_state_range(&pkvm_pgtable, addr, size, &d);
		537	}
		538
		539	static bool __hyp_ack_skip_pgtable_check(const struct pkvm_mem_transition *tx)
		540	{
		541	return !(IS_ENABLED(CONFIG_NVHE_EL2_DEBUG) \|\|
		542	tx->initiator.id != PKVM_ID_HOST);
		543	}
		544
		545	static int hyp_ack_share(u64 addr, const struct pkvm_mem_transition *tx,
		546	enum kvm_pgtable_prot perms)
		547	{
		548	u64 size = tx->nr_pages * PAGE_SIZE;
		549
		550	if (perms != PAGE_HYP)
		551	return -EPERM;
		552
		553	if (__hyp_ack_skip_pgtable_check(tx))
		554	return 0;
		555
		556	return __hyp_check_page_state_range(addr, size, PKVM_NOPAGE);
		557	}
		558
		559	static int hyp_ack_unshare(u64 addr, const struct pkvm_mem_transition *tx)
		560	{
		561	u64 size = tx->nr_pages * PAGE_SIZE;
		562
		563	if (__hyp_ack_skip_pgtable_check(tx))
		564	return 0;
		565
		566	return __hyp_check_page_state_range(addr, size,
		567	PKVM_PAGE_SHARED_BORROWED);
		568	}
		569
		570	static int hyp_complete_share(u64 addr, const struct pkvm_mem_transition *tx,
		571	enum kvm_pgtable_prot perms)
		572	{
		573	void start = (void )addr, end = start + (tx->nr_pages PAGE_SIZE);
		574	enum kvm_pgtable_prot prot;
		575
		576	prot = pkvm_mkstate(perms, PKVM_PAGE_SHARED_BORROWED);
		577	return pkvm_create_mappings_locked(start, end, prot);
		578	}
		579
		580	static int hyp_complete_unshare(u64 addr, const struct pkvm_mem_transition *tx)
		581	{
		582	u64 size = tx->nr_pages * PAGE_SIZE;
		583	int ret = kvm_pgtable_hyp_unmap(&pkvm_pgtable, addr, size);
		584
		585	return (ret != size) ? -EFAULT : 0;
		586	}
		587
		588	static int check_share(struct pkvm_mem_share *share)
		589	{
		590	const struct pkvm_mem_transition *tx = &share->tx;
		591	u64 completer_addr;
		592	int ret;
		593
		594	switch (tx->initiator.id) {
		595	case PKVM_ID_HOST:
		596	ret = host_request_owned_transition(&completer_addr, tx);
		597	break;
		598	default:
		599	ret = -EINVAL;
		600	}
374		601
375	ret = kvm_pgtable_get_leaf(&host_kvm.pgt, addr, &pte, NULL);
376	if (ret)	602	if (ret)
377	goto unlock;	603	return ret;
378	if (!pte)
379	goto map_shared;
380		604
381	/*	605	switch (tx->completer.id) {
382	* Check attributes in the host stage-2 PTE. We need the page to be:	606	case PKVM_ID_HYP:
383	* - mapped RWX as we're sharing memory;	607	ret = hyp_ack_share(completer_addr, tx, share->completer_prot);
384	* - not borrowed, as that implies absence of ownership.	608	break;
385	* Otherwise, we can't let it got through	609	default:
386	*/	610	ret = -EINVAL;
387	cur = kvm_pgtable_stage2_pte_prot(pte);
388	prot = pkvm_mkstate(0, PKVM_PAGE_SHARED_BORROWED);
389	if (!check_prot(cur, PKVM_HOST_MEM_PROT, prot)) {
390	ret = -EPERM;
391	goto unlock;
392	}	611	}
393		612
394	state = pkvm_getstate(cur);	613	return ret;
395	if (state == PKVM_PAGE_OWNED)	614	}
396	goto map_shared;
397		615
398	/*	616	static int __do_share(struct pkvm_mem_share *share)
399	* Tolerate double-sharing the same page, but this requires	617	{
400	* cross-checking the hypervisor stage-1.	618	const struct pkvm_mem_transition *tx = &share->tx;
401	*/	619	u64 completer_addr;
402	if (state != PKVM_PAGE_SHARED_OWNED) {	620	int ret;
403	ret = -EPERM;	621
404	goto unlock;	622	switch (tx->initiator.id) {
		623	case PKVM_ID_HOST:
		624	ret = host_initiate_share(&completer_addr, tx);
		625	break;
		626	default:
		627	ret = -EINVAL;
405	}	628	}
406		629
407	ret = kvm_pgtable_get_leaf(&pkvm_pgtable, (u64)virt, &pte, NULL);
408	if (ret)	630	if (ret)
409	goto unlock;	631	return ret;
410		632
411	/*	633	switch (tx->completer.id) {
412	* If the page has been shared with the hypervisor, it must be	634	case PKVM_ID_HYP:
413	* already mapped as SHARED_BORROWED in its stage-1.	635	ret = hyp_complete_share(completer_addr, tx, share->completer_prot);
414	*/	636	break;
415	cur = kvm_pgtable_hyp_pte_prot(pte);	637	default:
416	prot = pkvm_mkstate(PAGE_HYP, PKVM_PAGE_SHARED_BORROWED);	638	ret = -EINVAL;
417	if (!check_prot(cur, prot, ~prot))	639	}
418	ret = -EPERM;
419	goto unlock;
420		640
421	map_shared:	641	return ret;
422	/*	642	}
423	* If the page is not yet shared, adjust mappings in both page-tables	643
424	* while both locks are held.	644	/*
425	*/	645	* do_share():
426	prot = pkvm_mkstate(PAGE_HYP, PKVM_PAGE_SHARED_BORROWED);	646	*
427	ret = pkvm_create_mappings_locked(virt, virt + PAGE_SIZE, prot);	647	* The page owner grants access to another component with a given set
428	BUG_ON(ret);	648	* of permissions.
		649	*
		650	* Initiator: OWNED => SHARED_OWNED
		651	* Completer: NOPAGE => SHARED_BORROWED
		652	*/
		653	static int do_share(struct pkvm_mem_share *share)
		654	{
		655	int ret;
429		656
430	prot = pkvm_mkstate(PKVM_HOST_MEM_PROT, PKVM_PAGE_SHARED_OWNED);	657	ret = check_share(share);
431	ret = host_stage2_idmap_locked(addr, PAGE_SIZE, prot);	658	if (ret)
432	BUG_ON(ret);	659	return ret;
433		660
434	unlock:	661	return WARN_ON(__do_share(share));
435	hyp_spin_unlock(&pkvm_pgd_lock);	662	}
436	hyp_spin_unlock(&host_kvm.lock);	663
		664	static int check_unshare(struct pkvm_mem_share *share)
		665	{
		666	const struct pkvm_mem_transition *tx = &share->tx;
		667	u64 completer_addr;
		668	int ret;
		669
		670	switch (tx->initiator.id) {
		671	case PKVM_ID_HOST:
		672	ret = host_request_unshare(&completer_addr, tx);
		673	break;
		674	default:
		675	ret = -EINVAL;
		676	}
		677
		678	if (ret)
		679	return ret;
		680
		681	switch (tx->completer.id) {
		682	case PKVM_ID_HYP:
		683	ret = hyp_ack_unshare(completer_addr, tx);
		684	break;
		685	default:
		686	ret = -EINVAL;
		687	}
437		688
438	return ret;	689	return ret;
439	}	690	}
440		691
441	void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt)	692	static int __do_unshare(struct pkvm_mem_share *share)
442	{	693	{
443	struct kvm_vcpu_fault_info fault;	694	const struct pkvm_mem_transition *tx = &share->tx;
444	u64 esr, addr;	695	u64 completer_addr;
445	int ret = 0;	696	int ret;
446		697
447	esr = read_sysreg_el2(SYS_ESR);	698	switch (tx->initiator.id) {
448	BUG_ON(!__get_fault_info(esr, &fault));	699	case PKVM_ID_HOST:
		700	ret = host_initiate_unshare(&completer_addr, tx);
		701	break;
		702	default:
		703	ret = -EINVAL;
		704	}
449		705
450	addr = (fault.hpfar_el2 & HPFAR_MASK) << 8;	706	if (ret)
451	ret = host_stage2_idmap(addr);	707	return ret;
452	BUG_ON(ret && ret != -EAGAIN);	708
		709	switch (tx->completer.id) {
		710	case PKVM_ID_HYP:
		711	ret = hyp_complete_unshare(completer_addr, tx);
		712	break;
		713	default:
		714	ret = -EINVAL;
		715	}
		716
		717	return ret;
		718	}
		719
		720	/*
		721	* do_unshare():
		722	*
		723	* The page owner revokes access from another component for a range of
		724	* pages which were previously shared using do_share().
		725	*
		726	* Initiator: SHARED_OWNED => OWNED
		727	* Completer: SHARED_BORROWED => NOPAGE
		728	*/
		729	static int do_unshare(struct pkvm_mem_share *share)
		730	{
		731	int ret;
		732
		733	ret = check_unshare(share);
		734	if (ret)
		735	return ret;
		736
		737	return WARN_ON(__do_unshare(share));
		738	}
		739
		740	int __pkvm_host_share_hyp(u64 pfn)
		741	{
		742	int ret;
		743	u64 host_addr = hyp_pfn_to_phys(pfn);
		744	u64 hyp_addr = (u64)__hyp_va(host_addr);
		745	struct pkvm_mem_share share = {
		746	.tx = {
		747	.nr_pages = 1,
		748	.initiator = {
		749	.id = PKVM_ID_HOST,
		750	.addr = host_addr,
		751	.host = {
		752	.completer_addr = hyp_addr,
		753	},
		754	},
		755	.completer = {
		756	.id = PKVM_ID_HYP,
		757	},
		758	},
		759	.completer_prot = PAGE_HYP,
		760	};
		761
		762	host_lock_component();
		763	hyp_lock_component();
		764
		765	ret = do_share(&share);
		766
		767	hyp_unlock_component();
		768	host_unlock_component();
		769
		770	return ret;
		771	}
		772
		773	int __pkvm_host_unshare_hyp(u64 pfn)
		774	{
		775	int ret;
		776	u64 host_addr = hyp_pfn_to_phys(pfn);
		777	u64 hyp_addr = (u64)__hyp_va(host_addr);
		778	struct pkvm_mem_share share = {
		779	.tx = {
		780	.nr_pages = 1,
		781	.initiator = {
		782	.id = PKVM_ID_HOST,
		783	.addr = host_addr,
		784	.host = {
		785	.completer_addr = hyp_addr,
		786	},
		787	},
		788	.completer = {
		789	.id = PKVM_ID_HYP,
		790	},
		791	},
		792	.completer_prot = PAGE_HYP,
		793	};
		794
		795	host_lock_component();
		796	hyp_lock_component();
		797
		798	ret = do_unshare(&share);
		799
		800	hyp_unlock_component();
		801	host_unlock_component();
		802
		803	return ret;
453	}	804	}

arch/arm64/kvm/hyp/nvhe/mm.c+3−1

 #include <asm/kvm_hyp.h>
 #include <asm/kvm_mmu.h>
 #include <asm/kvm_pgtable.h>
+#include <asm/kvm_pkvm.h>
 #include <asm/spectre.h>
 #include <nvhe/early_alloc.h>
 struct kvm_pgtable pkvm_pgtable;
 hyp_spinlock_t pkvm_pgd_lock;
-u64 __io_map_base;
 struct memblock_region hyp_memory[HYP_MEMBLOCK_REGIONS];
 unsigned int hyp_memblock_nr;
+static u64 __io_map_base;
 static int __pkvm_create_mappings(unsigned long start, unsigned long size,
 				  unsigned long phys, enum kvm_pgtable_prot prot)
 {

arch/arm64/kvm/hyp/nvhe/page_alloc.c+1−1

@@ -241,7 +241,7 @@ int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages, · 240 unmodified lines
241	int i;	241	int i;
242		242
243	hyp_spin_lock_init(&pool->lock);	243	hyp_spin_lock_init(&pool->lock);
244	pool->max_order = min(MAX_ORDER, get_order(nr_pages << PAGE_SHIFT));	244	pool->max_order = min(MAX_ORDER, get_order((nr_pages + 1) << PAGE_SHIFT));
245	for (i = 0; i < pool->max_order; i++)	245	for (i = 0; i < pool->max_order; i++)
246	INIT_LIST_HEAD(&pool->free_area[i]);	246	INIT_LIST_HEAD(&pool->free_area[i]);
247	pool->range_start = phys;	247	pool->range_start = phys;

arch/arm64/kvm/hyp/nvhe/setup.c+19−6

 #include <asm/kvm_hyp.h>
 #include <asm/kvm_mmu.h>
 #include <asm/kvm_pgtable.h>
+#include <asm/kvm_pkvm.h>
 #include <nvhe/early_alloc.h>
 #include <nvhe/fixed_config.h>
 #include <nvhe/mm.h>
 #include <nvhe/trap_handler.h>
-struct hyp_pool hpool;
 unsigned long hyp_nr_cpus;
 #define hyp_percpu_size ((unsigned long)__per_cpu_end - \
 static void *hyp_pgt_base;
 static void *host_s2_pgt_base;
 static struct kvm_pgtable_mm_ops pkvm_pgtable_mm_ops;
+static struct hyp_pool hpool;
 static int divide_memory_pool(void *virt, unsigned long size)
 {
 					 enum kvm_pgtable_walk_flags flag,
 					 void * const arg)
 {
+	struct kvm_pgtable_mm_ops *mm_ops = arg;
 	enum kvm_pgtable_prot prot;
 	enum pkvm_page_state state;
 	kvm_pte_t pte = *ptep;
 	if (!kvm_pte_valid(pte))
 		return 0;
+	/*
+	 * Fix-up the refcount for the page-table pages as the early allocator
+	 * was unable to access the hyp_vmemmap and so the buddy allocator has
+	 * initialised the refcount to '1'.
+	 */
+	mm_ops->get_page(ptep);
+	if (flag != KVM_PGTABLE_WALK_LEAF)
+		return 0;
 	if (level != (KVM_PGTABLE_MAX_LEVELS - 1))
 		return -EINVAL;
 {
 	struct kvm_pgtable_walker walker = {
 		.cb	= finalize_host_mappings_walker,
-		.flags	= KVM_PGTABLE_WALK_LEAF,
+		.flags	= KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
+		.arg	= pkvm_pgtable.mm_ops,
 	};
 	int i, ret;
 	if (ret)
 		goto out;
-	ret = finalize_host_mappings();
-	if (ret)
-		goto out;
 	pkvm_pgtable_mm_ops = (struct kvm_pgtable_mm_ops) {
 		.zalloc_page = hyp_zalloc_hyp_page,
 		.phys_to_virt = hyp_phys_to_virt,
 		.virt_to_phys = hyp_virt_to_phys,
 		.get_page = hpool_get_page,
 		.put_page = hpool_put_page,
+		.page_count = hyp_page_count,
 	};
 	pkvm_pgtable.mm_ops = &pkvm_pgtable_mm_ops;
+	ret = finalize_host_mappings();
+	if (ret)
+		goto out;
 out:
 	/*
 	 * We tail-called to here from handle___pkvm_init() and will not return,

arch/arm64/kvm/hyp/nvhe/switch.c+0−1

@@ -25,7 +25,6 @@ · 24 unmodified lines
25	#include <asm/fpsimd.h>	25	#include <asm/fpsimd.h>
26	#include <asm/debug-monitors.h>	26	#include <asm/debug-monitors.h>
27	#include <asm/processor.h>	27	#include <asm/processor.h>
28	#include <asm/thread_info.h>
29		28
30	#include <nvhe/fixed_config.h>	29	#include <nvhe/fixed_config.h>
31	#include <nvhe/mem_protect.h>	30	#include <nvhe/mem_protect.h>

arch/arm64/kvm/hyp/pgtable.c+85−23

 	return prot;
 }
-static bool hyp_pte_needs_update(kvm_pte_t old, kvm_pte_t new)
-{
-	/*
-	 * Tolerate KVM recreating the exact same mapping, or changing software
-	 * bits if the existing mapping was valid.
-	 */
-	if (old == new)
-		return false;
-	if (!kvm_pte_valid(old))
-		return true;
-	return !WARN_ON((old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW);
-}
 static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level,
 				    kvm_pte_t *ptep, struct hyp_map_data *data)
 {
 	if (!kvm_block_mapping_supported(addr, end, phys, level))
 		return false;
+	data->phys += granule;
 	new = kvm_init_valid_leaf_pte(phys, data->attr, level);
-	if (hyp_pte_needs_update(old, new))
+	if (old == new)
-		smp_store_release(ptep, new);
+		return true;
+	if (!kvm_pte_valid(old))
+		data->mm_ops->get_page(ptep);
+	else if (WARN_ON((old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW))
+		return false;
-	data->phys += granule;
+	smp_store_release(ptep, new);
 	return true;
 }
 		return -ENOMEM;
 	kvm_set_table_pte(ptep, childp, mm_ops);
+	mm_ops->get_page(ptep);
 	return 0;
 }
 	return ret;
 }
+struct hyp_unmap_data {
+	u64				unmapped;
+	struct kvm_pgtable_mm_ops	*mm_ops;
+};
+static int hyp_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
+			    enum kvm_pgtable_walk_flags flag, void * const arg)
+{
+	kvm_pte_t pte = *ptep, *childp = NULL;
+	u64 granule = kvm_granule_size(level);
+	struct hyp_unmap_data *data = arg;
+	struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;
+	if (!kvm_pte_valid(pte))
+		return -EINVAL;
+	if (kvm_pte_table(pte, level)) {
+		childp = kvm_pte_follow(pte, mm_ops);
+		if (mm_ops->page_count(childp) != 1)
+			return 0;
+		kvm_clear_pte(ptep);
+		dsb(ishst);
+		__tlbi_level(vae2is, __TLBI_VADDR(addr, 0), level);
+	} else {
+		if (end - addr < granule)
+			return -EINVAL;
+		kvm_clear_pte(ptep);
+		dsb(ishst);
+		__tlbi_level(vale2is, __TLBI_VADDR(addr, 0), level);
+		data->unmapped += granule;
+	}
+	dsb(ish);
+	isb();
+	mm_ops->put_page(ptep);
+	if (childp)
+		mm_ops->put_page(childp);
+	return 0;
+}
+u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)
+{
+	struct hyp_unmap_data unmap_data = {
+		.mm_ops	= pgt->mm_ops,
+	};
+	struct kvm_pgtable_walker walker = {
+		.cb	= hyp_unmap_walker,
+		.arg	= &unmap_data,
+		.flags	= KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
+	};
+	if (!pgt->mm_ops->page_count)
+		return 0;
+	kvm_pgtable_walk(pgt, addr, size, &walker);
+	return unmap_data.unmapped;
+}
 int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,
 			 struct kvm_pgtable_mm_ops *mm_ops)
 {
 			   enum kvm_pgtable_walk_flags flag, void * const arg)
 {
 	struct kvm_pgtable_mm_ops *mm_ops = arg;
+	kvm_pte_t pte = *ptep;
+	if (!kvm_pte_valid(pte))
+		return 0;
+	mm_ops->put_page(ptep);
+	if (kvm_pte_table(pte, level))
+		mm_ops->put_page(kvm_pte_follow(pte, mm_ops));
-	mm_ops->put_page((void *)kvm_pte_follow(*ptep, mm_ops));
 	return 0;
 }
 {
 	struct kvm_pgtable_walker walker = {
 		.cb	= hyp_free_walker,
-		.flags	= KVM_PGTABLE_WALK_TABLE_POST,
+		.flags	= KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
 		.arg	= pgt->mm_ops,
 	};
 }
-int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_arch *arch,
+int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,
 			      struct kvm_pgtable_mm_ops *mm_ops,
 			      enum kvm_pgtable_stage2_flags flags,
 			      kvm_pgtable_force_pte_cb_t force_pte_cb)
 {
 	size_t pgd_sz;
-	u64 vtcr = arch->vtcr;
+	u64 vtcr = mmu->arch->vtcr;
 	u32 ia_bits = VTCR_EL2_IPA(vtcr);
 	u32 sl0 = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);
 	u32 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0;
 	pgt->ia_bits		= ia_bits;
 	pgt->start_level	= start_level;
 	pgt->mm_ops		= mm_ops;
-	pgt->mmu		= &arch->mmu;
+	pgt->mmu		= mmu;
 	pgt->flags		= flags;
 	pgt->force_pte_cb	= force_pte_cb;

arch/arm64/kvm/hyp/vhe/switch.c+0−1

@@ -24,7 +24,6 @@ · 23 unmodified lines
24	#include <asm/fpsimd.h>	24	#include <asm/fpsimd.h>
25	#include <asm/debug-monitors.h>	25	#include <asm/debug-monitors.h>
26	#include <asm/processor.h>	26	#include <asm/processor.h>
27	#include <asm/thread_info.h>
28		27
29	/* VHE specific context */	28	/* VHE specific context */
30	DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data);	29	DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data);

arch/arm64/kvm/mmu.c+137−13

@@ -239,6 +239,9 @@ void free_hyp_pgds(void) · 238 unmodified lines
239		239
240	static bool kvm_host_owns_hyp_mappings(void)	240	static bool kvm_host_owns_hyp_mappings(void)
241	{	241	{
		242	if (is_kernel_in_hyp_mode())
		243	return false;
		244
242	if (static_branch_likely(&kvm_protected_mode_initialized))	245	if (static_branch_likely(&kvm_protected_mode_initialized))
243	return false;	246	return false;
244		247
@@ -281,14 +284,117 @@ static phys_addr_t kvm_kaddr_to_phys(void *kaddr) · 36 unmodified lines
281	}	284	}
282	}	285	}
283		286
284	static int pkvm_share_hyp(phys_addr_t start, phys_addr_t end)	287	struct hyp_shared_pfn {
		288	u64 pfn;
		289	int count;
		290	struct rb_node node;
		291	};
		292
		293	static DEFINE_MUTEX(hyp_shared_pfns_lock);
		294	static struct rb_root hyp_shared_pfns = RB_ROOT;
		295
		296	static struct hyp_shared_pfn find_shared_pfn(u64 pfn, struct rb_node **node,
		297	struct rb_node **parent)
285	{	298	{
286	phys_addr_t addr;	299	struct hyp_shared_pfn *this;
		300
		301	*node = &hyp_shared_pfns.rb_node;
		302	*parent = NULL;
		303	while (**node) {
		304	this = container_of(**node, struct hyp_shared_pfn, node);
		305	parent = *node;
		306	if (this->pfn < pfn)
		307	node = &((*node)->rb_left);
		308	else if (this->pfn > pfn)
		309	node = &((*node)->rb_right);
		310	else
		311	return this;
		312	}
		313
		314	return NULL;
		315	}
		316
		317	static int share_pfn_hyp(u64 pfn)
		318	{
		319	struct rb_node *node, parent;
		320	struct hyp_shared_pfn *this;
		321	int ret = 0;
		322
		323	mutex_lock(&hyp_shared_pfns_lock);
		324	this = find_shared_pfn(pfn, &node, &parent);
		325	if (this) {
		326	this->count++;
		327	goto unlock;
		328	}
		329
		330	this = kzalloc(sizeof(*this), GFP_KERNEL);
		331	if (!this) {
		332	ret = -ENOMEM;
		333	goto unlock;
		334	}
		335
		336	this->pfn = pfn;
		337	this->count = 1;
		338	rb_link_node(&this->node, parent, node);
		339	rb_insert_color(&this->node, &hyp_shared_pfns);
		340	ret = kvm_call_hyp_nvhe(__pkvm_host_share_hyp, pfn, 1);
		341	unlock:
		342	mutex_unlock(&hyp_shared_pfns_lock);
		343
		344	return ret;
		345	}
		346
		347	static int unshare_pfn_hyp(u64 pfn)
		348	{
		349	struct rb_node *node, parent;
		350	struct hyp_shared_pfn *this;
		351	int ret = 0;
		352
		353	mutex_lock(&hyp_shared_pfns_lock);
		354	this = find_shared_pfn(pfn, &node, &parent);
		355	if (WARN_ON(!this)) {
		356	ret = -ENOENT;
		357	goto unlock;
		358	}
		359
		360	this->count--;
		361	if (this->count)
		362	goto unlock;
		363
		364	rb_erase(&this->node, &hyp_shared_pfns);
		365	kfree(this);
		366	ret = kvm_call_hyp_nvhe(__pkvm_host_unshare_hyp, pfn, 1);
		367	unlock:
		368	mutex_unlock(&hyp_shared_pfns_lock);
		369
		370	return ret;
		371	}
		372
		373	int kvm_share_hyp(void from, void to)
		374	{
		375	phys_addr_t start, end, cur;
		376	u64 pfn;
287	int ret;	377	int ret;
288		378
289	for (addr = ALIGN_DOWN(start, PAGE_SIZE); addr < end; addr += PAGE_SIZE) {	379	if (is_kernel_in_hyp_mode())
290	ret = kvm_call_hyp_nvhe(__pkvm_host_share_hyp,	380	return 0;
291	__phys_to_pfn(addr));	381
		382	/*
		383	* The share hcall maps things in the 'fixed-offset' region of the hyp
		384	* VA space, so we can only share physically contiguous data-structures
		385	* for now.
		386	*/
		387	if (is_vmalloc_or_module_addr(from) \|\| is_vmalloc_or_module_addr(to))
		388	return -EINVAL;
		389
		390	if (kvm_host_owns_hyp_mappings())
		391	return create_hyp_mappings(from, to, PAGE_HYP);
		392
		393	start = ALIGN_DOWN(__pa(from), PAGE_SIZE);
		394	end = PAGE_ALIGN(__pa(to));
		395	for (cur = start; cur < end; cur += PAGE_SIZE) {
		396	pfn = __phys_to_pfn(cur);
		397	ret = share_pfn_hyp(pfn);
292	if (ret)	398	if (ret)
293	return ret;	399	return ret;
294	}	400	}
@@ -296,6 +402,22 @@ static int pkvm_share_hyp(phys_addr_t start, phys_addr_t end) · 1 unmodified line
296	return 0;	402	return 0;
297	}	403	}
298		404
		405	void kvm_unshare_hyp(void from, void to)
		406	{
		407	phys_addr_t start, end, cur;
		408	u64 pfn;
		409
		410	if (is_kernel_in_hyp_mode() \|\| kvm_host_owns_hyp_mappings() \|\| !from)
		411	return;
		412
		413	start = ALIGN_DOWN(__pa(from), PAGE_SIZE);
		414	end = PAGE_ALIGN(__pa(to));
		415	for (cur = start; cur < end; cur += PAGE_SIZE) {
		416	pfn = __phys_to_pfn(cur);
		417	WARN_ON(unshare_pfn_hyp(pfn));
		418	}
		419	}
		420
299	/**	421	/**
300	* create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode	422	* create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode
301	* @from: The virtual kernel start address of the range	423	* @from: The virtual kernel start address of the range
@@ -316,12 +438,8 @@ int create_hyp_mappings(void from, void to, enum kvm_pgtable_prot prot) · 14 unmodified lines
316	if (is_kernel_in_hyp_mode())	438	if (is_kernel_in_hyp_mode())
317	return 0;	439	return 0;
318		440
319	if (!kvm_host_owns_hyp_mappings()) {	441	if (!kvm_host_owns_hyp_mappings())
320	if (WARN_ON(prot != PAGE_HYP))	442	return -EPERM;
321	return -EPERM;
322	return pkvm_share_hyp(kvm_kaddr_to_phys(from),
323	kvm_kaddr_to_phys(to));
324	}
325		443
326	start = start & PAGE_MASK;	444	start = start & PAGE_MASK;
327	end = PAGE_ALIGN(end);	445	end = PAGE_ALIGN(end);
@@ -407,6 +525,9 @@ int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size, · 79 unmodified lines
407	unsigned long addr;	525	unsigned long addr;
408	int ret;	526	int ret;
409		527
		528	if (is_protected_kvm_enabled())
		529	return -EPERM;
		530
410	*kaddr = ioremap(phys_addr, size);	531	*kaddr = ioremap(phys_addr, size);
411	if (!*kaddr)	532	if (!*kaddr)
412	return -ENOMEM;	533	return -ENOMEM;
@@ -516,7 +637,8 @@ int kvm_init_stage2_mmu(struct kvm kvm, struct kvm_s2_mmu mmu) · 103 unmodified lines
516	if (!pgt)	637	if (!pgt)
517	return -ENOMEM;	638	return -ENOMEM;
518		639
519	err = kvm_pgtable_stage2_init(pgt, &kvm->arch, &kvm_s2_mm_ops);	640	mmu->arch = &kvm->arch;
		641	err = kvm_pgtable_stage2_init(pgt, mmu, &kvm_s2_mm_ops);
520	if (err)	642	if (err)
521	goto out_free_pgtable;	643	goto out_free_pgtable;
522		644
@@ -529,7 +651,6 @@ int kvm_init_stage2_mmu(struct kvm kvm, struct kvm_s2_mmu mmu) · 6 unmodified lines
529	for_each_possible_cpu(cpu)	651	for_each_possible_cpu(cpu)
530	*per_cpu_ptr(mmu->last_vcpu_ran, cpu) = -1;	652	*per_cpu_ptr(mmu->last_vcpu_ran, cpu) = -1;
531		653
532	mmu->arch = &kvm->arch;
533	mmu->pgt = pgt;	654	mmu->pgt = pgt;
534	mmu->pgd_phys = __pa(pgt->pgd);	655	mmu->pgd_phys = __pa(pgt->pgd);
535	WRITE_ONCE(mmu->vmid.vmid_gen, 0);	656	WRITE_ONCE(mmu->vmid.vmid_gen, 0);
@@ -650,6 +771,9 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, · 114 unmodified lines
650	KVM_PGTABLE_PROT_R \|	771	KVM_PGTABLE_PROT_R \|
651	(writable ? KVM_PGTABLE_PROT_W : 0);	772	(writable ? KVM_PGTABLE_PROT_W : 0);
652		773
		774	if (is_protected_kvm_enabled())
		775	return -EPERM;
		776
653	size += offset_in_page(guest_ipa);	777	size += offset_in_page(guest_ipa);
654	guest_ipa &= PAGE_MASK;	778	guest_ipa &= PAGE_MASK;
655		779

arch/arm64/kvm/pkvm.crenamed from arch/arm64/kvm/hyp/reserved_mem.c+4−4

 #include <linux/memblock.h>
 #include <linux/sort.h>
-#include <asm/kvm_host.h>
+#include <asm/kvm_pkvm.h>
-#include <nvhe/memory.h>
+#include "hyp_constants.h"
-#include <nvhe/mm.h>
 static struct memblock_region *hyp_memory = kvm_nvhe_sym(hyp_memory);
 static unsigned int *hyp_memblock_nr_ptr = &kvm_nvhe_sym(hyp_memblock_nr);
 	do {
 		prev = nr_pages;
 		nr_pages = hyp_mem_pages + prev;
-		nr_pages = DIV_ROUND_UP(nr_pages * sizeof(struct hyp_page), PAGE_SIZE);
+		nr_pages = DIV_ROUND_UP(nr_pages * STRUCT_HYP_PAGE_SIZE,
+					PAGE_SIZE);
 		nr_pages += __hyp_pgtable_max_pages(nr_pages);
 	} while (nr_pages != prev);
 	hyp_mem_pages += nr_pages;

arch/arm64/kvm/pmu-emul.c+1−0

@@ -28,6 +28,7 @@ static u32 kvm_pmu_event_mask(struct kvm *kvm) · 27 unmodified lines
28	case ID_AA64DFR0_PMUVER_8_1:	28	case ID_AA64DFR0_PMUVER_8_1:
29	case ID_AA64DFR0_PMUVER_8_4:	29	case ID_AA64DFR0_PMUVER_8_4:
30	case ID_AA64DFR0_PMUVER_8_5:	30	case ID_AA64DFR0_PMUVER_8_5:
		31	case ID_AA64DFR0_PMUVER_8_7:
31	return GENMASK(15, 0);	32	return GENMASK(15, 0);
32	default: /* Shouldn't be here, just for sanity */	33	default: /* Shouldn't be here, just for sanity */
33	WARN_ONCE(1, "Unknown PMU version %d\n", kvm->arch.pmuver);	34	WARN_ONCE(1, "Unknown PMU version %d\n", kvm->arch.pmuver);

arch/arm64/kvm/psci.c+1−1

@@ -109,7 +109,7 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) · 108 unmodified lines
109		109
110	/*	110	/*
111	* Make sure the reset request is observed if the change to	111	* Make sure the reset request is observed if the change to
112	* power_state is observed.	112	* power_off is observed.
113	*/	113	*/
114	smp_wmb();	114	smp_wmb();
115		115

arch/arm64/kvm/reset.c+21−7

 {
 	void *buf;
 	unsigned int vl;
+	size_t reg_sz;
+	int ret;
 	vl = vcpu->arch.sve_max_vl;
 	/*
 	 * Responsibility for these properties is shared between
-	 * kvm_arm_init_arch_resources(), kvm_vcpu_enable_sve() and
+	 * kvm_arm_init_sve(), kvm_vcpu_enable_sve() and
 	 * set_sve_vls().  Double-check here just to be sure:
 	 */
 	if (WARN_ON(!sve_vl_valid(vl) || vl > sve_max_virtualisable_vl() ||
 		    vl > SVE_VL_ARCH_MAX))
 		return -EIO;
-	buf = kzalloc(SVE_SIG_REGS_SIZE(sve_vq_from_vl(vl)), GFP_KERNEL_ACCOUNT);
+	reg_sz = vcpu_sve_state_size(vcpu);
+	buf = kzalloc(reg_sz, GFP_KERNEL_ACCOUNT);
 	if (!buf)
 		return -ENOMEM;
+	ret = kvm_share_hyp(buf, buf + reg_sz);
+	if (ret) {
+		kfree(buf);
+		return ret;
+	}
 	vcpu->arch.sve_state = buf;
 	vcpu->arch.flags |= KVM_ARM64_VCPU_SVE_FINALIZED;
 	return 0;
 void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu)
 {
-	kfree(vcpu->arch.sve_state);
+	void *sve_state = vcpu->arch.sve_state;
+	kvm_vcpu_unshare_task_fp(vcpu);
+	kvm_unshare_hyp(vcpu, vcpu + 1);
+	if (sve_state)
+		kvm_unshare_hyp(sve_state, sve_state + vcpu_sve_state_size(vcpu));
+	kfree(sve_state);
 }
 static void kvm_vcpu_reset_sve(struct kvm_vcpu *vcpu)
  * kvm_reset_vcpu - sets core registers and sys_regs to reset value
  * @vcpu: The VCPU pointer
  *
- * This function finds the right table above and sets the registers on
+ * This function sets the registers on the virtual CPU struct to their
- * the virtual CPU struct to their architecturally defined reset
+ * architecturally defined reset values, except for registers whose reset is
- * values, except for registers whose reset is deferred until
+ * deferred until kvm_arm_vcpu_finalize().
- * kvm_arm_vcpu_finalize().
  *
  * Note: This function can be called from two paths: The KVM_ARM_VCPU_INIT
  * ioctl or as part of handling a request issued by another VCPU in the PSCI

arch/arm64/kvm/vgic/vgic-init.c+1−1

@@ -92,7 +92,7 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) · 91 unmodified lines
92	return ret;	92	return ret;
93		93
94	kvm_for_each_vcpu(i, vcpu, kvm) {	94	kvm_for_each_vcpu(i, vcpu, kvm) {
95	if (vcpu->arch.has_run_once)	95	if (vcpu_has_run_once(vcpu))
96	goto out_unlock;	96	goto out_unlock;
97	}	97	}
98	ret = 0;	98	ret = 0;

arch/arm64/kvm/vgic/vgic-mmio-v3.c+5−3

 	}
 	if (ret) {
-		/* The current c failed, so we start with the previous one. */
+		/* The current c failed, so iterate over the previous ones. */
+		int i;
 		mutex_lock(&kvm->slots_lock);
-		for (c--; c >= 0; c--) {
+		for (i = 0; i < c; i++) {
-			vcpu = kvm_get_vcpu(kvm, c);
+			vcpu = kvm_get_vcpu(kvm, i);
 			vgic_unregister_redist_iodev(vcpu);
 		}
 		mutex_unlock(&kvm->slots_lock);

arch/arm64/kvm/vgic/vgic-mmio.c+1−1

@@ -1050,7 +1050,7 @@ static int dispatch_mmio_write(struct kvm_vcpu vcpu, struct kvm_io_device dev, · 1049 unmodified lines
1050	return 0;	1050	return 0;
1051	}	1051	}
1052		1052
1053	struct kvm_io_device_ops kvm_io_gic_ops = {	1053	const struct kvm_io_device_ops kvm_io_gic_ops = {
1054	.read = dispatch_mmio_read,	1054	.read = dispatch_mmio_read,
1055	.write = dispatch_mmio_write,	1055	.write = dispatch_mmio_write,
1056	};	1056	};

arch/arm64/kvm/vgic/vgic-mmio.h+1−1

@@ -34,7 +34,7 @@ struct vgic_register_region { · 33 unmodified lines
34	};	34	};
35	};	35	};
36		36
37	extern struct kvm_io_device_ops kvm_io_gic_ops;	37	extern const struct kvm_io_device_ops kvm_io_gic_ops;
38		38
39	#define VGIC_ACCESS_8bit 1	39	#define VGIC_ACCESS_8bit 1
40	#define VGIC_ACCESS_32bit 2	40	#define VGIC_ACCESS_32bit 2

arch/arm64/kvm/vgic/vgic-v2.c+7−2

 	if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base) ||
 	    IS_VGIC_ADDR_UNDEF(dist->vgic_cpu_base)) {
-		kvm_err("Need to set vgic cpu and dist addresses first\n");
+		kvm_debug("Need to set vgic cpu and dist addresses first\n");
 		return -ENXIO;
 	}
 	if (!vgic_v2_check_base(dist->vgic_dist_base, dist->vgic_cpu_base)) {
-		kvm_err("VGIC CPU and dist frames overlap\n");
+		kvm_debug("VGIC CPU and dist frames overlap\n");
 		return -EINVAL;
 	}
 	int ret;
 	u32 vtr;
+	if (is_protected_kvm_enabled()) {
+		kvm_err("GICv2 not supported in protected mode\n");
+		return -ENXIO;
+	}
 	if (!info->vctrl.start) {
 		kvm_err("GICH not present in the firmware table\n");
 		return -ENXIO;

arch/arm64/kvm/vgic/vgic-v3.c+3−3

include/kvm/arm_vgic.h+3−1

 #ifndef __KVM_ARM_VGIC_H
 #define __KVM_ARM_VGIC_H
-#include <linux/kernel.h>
+#include <linux/bits.h>
 #include <linux/kvm.h>
 #include <linux/irqreturn.h>
+#include <linux/kref.h>
+#include <linux/mutex.h>
 #include <linux/spinlock.h>
 #include <linux/static_key.h>
 #include <linux/types.h>

include/linux/kvm_host.h+0−1

@@ -1322,7 +1322,6 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu); · 1321 unmodified lines
1322		1322
1323	bool kvm_is_reserved_pfn(kvm_pfn_t pfn);	1323	bool kvm_is_reserved_pfn(kvm_pfn_t pfn);
1324	bool kvm_is_zone_device_pfn(kvm_pfn_t pfn);	1324	bool kvm_is_zone_device_pfn(kvm_pfn_t pfn);
1325	bool kvm_is_transparent_hugepage(kvm_pfn_t pfn);
1326		1325
1327	struct kvm_irq_ack_notifier {	1326	struct kvm_irq_ack_notifier {
1328	struct hlist_node link;	1327	struct hlist_node link;

tools/testing/selftests/kvm/.gitignore+1−0

@@ -4,6 +4,7 @@ · 3 unmodified lines
4	/aarch64/get-reg-list	4	/aarch64/get-reg-list
5	/aarch64/psci_cpu_on_test	5	/aarch64/psci_cpu_on_test
6	/aarch64/vgic_init	6	/aarch64/vgic_init
		7	/aarch64/vgic_irq
7	/s390x/memop	8	/s390x/memop
8	/s390x/resets	9	/s390x/resets
9	/s390x/sync_regs_test	10	/s390x/sync_regs_test

tools/testing/selftests/kvm/Makefile+1−0

@@ -96,6 +96,7 @@ TEST_GEN_PROGS_aarch64 += aarch64/debug-exceptions · 95 unmodified lines
96	TEST_GEN_PROGS_aarch64 += aarch64/get-reg-list	96	TEST_GEN_PROGS_aarch64 += aarch64/get-reg-list
97	TEST_GEN_PROGS_aarch64 += aarch64/psci_cpu_on_test	97	TEST_GEN_PROGS_aarch64 += aarch64/psci_cpu_on_test
98	TEST_GEN_PROGS_aarch64 += aarch64/vgic_init	98	TEST_GEN_PROGS_aarch64 += aarch64/vgic_init
		99	TEST_GEN_PROGS_aarch64 += aarch64/vgic_irq
99	TEST_GEN_PROGS_aarch64 += demand_paging_test	100	TEST_GEN_PROGS_aarch64 += demand_paging_test
100	TEST_GEN_PROGS_aarch64 += dirty_log_test	101	TEST_GEN_PROGS_aarch64 += dirty_log_test
101	TEST_GEN_PROGS_aarch64 += dirty_log_perf_test	102	TEST_GEN_PROGS_aarch64 += dirty_log_perf_test

tools/testing/selftests/kvm/aarch64/arch_timer.c+1−1

@@ -382,7 +382,7 @@ static struct kvm_vm *test_vm_create(void) · 381 unmodified lines
382		382
383	ucall_init(vm, NULL);	383	ucall_init(vm, NULL);
384	test_init_timer_irq(vm);	384	test_init_timer_irq(vm);
385	vgic_v3_setup(vm, nr_vcpus, GICD_BASE_GPA, GICR_BASE_GPA);	385	vgic_v3_setup(vm, nr_vcpus, 64, GICD_BASE_GPA, GICR_BASE_GPA);
386		386
387	/* Make all the test's cmdline args visible to the guest */	387	/* Make all the test's cmdline args visible to the guest */
388	sync_global_to_guest(vm, test_args);	388	sync_global_to_guest(vm, test_args);

tools/testing/selftests/kvm/aarch64/get-reg-list.c+50−0

@@ -1014,6 +1014,22 @@ static __u64 sve_rejects_set[] = { · 1013 unmodified lines
1014	KVM_REG_ARM64_SVE_VLS,	1014	KVM_REG_ARM64_SVE_VLS,
1015	};	1015	};
1016		1016
		1017	static __u64 pauth_addr_regs[] = {
		1018	ARM64_SYS_REG(3, 0, 2, 1, 0), /* APIAKEYLO_EL1 */
		1019	ARM64_SYS_REG(3, 0, 2, 1, 1), /* APIAKEYHI_EL1 */
		1020	ARM64_SYS_REG(3, 0, 2, 1, 2), /* APIBKEYLO_EL1 */
		1021	ARM64_SYS_REG(3, 0, 2, 1, 3), /* APIBKEYHI_EL1 */
		1022	ARM64_SYS_REG(3, 0, 2, 2, 0), /* APDAKEYLO_EL1 */
		1023	ARM64_SYS_REG(3, 0, 2, 2, 1), /* APDAKEYHI_EL1 */
		1024	ARM64_SYS_REG(3, 0, 2, 2, 2), /* APDBKEYLO_EL1 */
		1025	ARM64_SYS_REG(3, 0, 2, 2, 3) /* APDBKEYHI_EL1 */
		1026	};
		1027
		1028	static __u64 pauth_generic_regs[] = {
		1029	ARM64_SYS_REG(3, 0, 2, 3, 0), /* APGAKEYLO_EL1 */
		1030	ARM64_SYS_REG(3, 0, 2, 3, 1), /* APGAKEYHI_EL1 */
		1031	};
		1032
1017	#define BASE_SUBLIST \	1033	#define BASE_SUBLIST \
1018	{ "base", .regs = base_regs, .regs_n = ARRAY_SIZE(base_regs), }	1034	{ "base", .regs = base_regs, .regs_n = ARRAY_SIZE(base_regs), }
1019	#define VREGS_SUBLIST \	1035	#define VREGS_SUBLIST \
@@ -1025,6 +1041,21 @@ static __u64 sve_rejects_set[] = { · 5 unmodified lines
1025	{ "sve", .capability = KVM_CAP_ARM_SVE, .feature = KVM_ARM_VCPU_SVE, .finalize = true, \	1041	{ "sve", .capability = KVM_CAP_ARM_SVE, .feature = KVM_ARM_VCPU_SVE, .finalize = true, \
1026	.regs = sve_regs, .regs_n = ARRAY_SIZE(sve_regs), \	1042	.regs = sve_regs, .regs_n = ARRAY_SIZE(sve_regs), \
1027	.rejects_set = sve_rejects_set, .rejects_set_n = ARRAY_SIZE(sve_rejects_set), }	1043	.rejects_set = sve_rejects_set, .rejects_set_n = ARRAY_SIZE(sve_rejects_set), }
		1044	#define PAUTH_SUBLIST \
		1045	{ \
		1046	.name = "pauth_address", \
		1047	.capability = KVM_CAP_ARM_PTRAUTH_ADDRESS, \
		1048	.feature = KVM_ARM_VCPU_PTRAUTH_ADDRESS, \
		1049	.regs = pauth_addr_regs, \
		1050	.regs_n = ARRAY_SIZE(pauth_addr_regs), \
		1051	}, \
		1052	{ \
		1053	.name = "pauth_generic", \
		1054	.capability = KVM_CAP_ARM_PTRAUTH_GENERIC, \
		1055	.feature = KVM_ARM_VCPU_PTRAUTH_GENERIC, \
		1056	.regs = pauth_generic_regs, \
		1057	.regs_n = ARRAY_SIZE(pauth_generic_regs), \
		1058	}
1028		1059
1029	static struct vcpu_config vregs_config = {	1060	static struct vcpu_config vregs_config = {
1030	.sublists = {	1061	.sublists = {
@@ -1056,11 +1087,30 @@ static struct vcpu_config sve_pmu_config = { · 25 unmodified lines
1056	{0},	1087	{0},
1057	},	1088	},
1058	};	1089	};
		1090	static struct vcpu_config pauth_config = {
		1091	.sublists = {
		1092	BASE_SUBLIST,
		1093	VREGS_SUBLIST,
		1094	PAUTH_SUBLIST,
		1095	{0},
		1096	},
		1097	};
		1098	static struct vcpu_config pauth_pmu_config = {
		1099	.sublists = {
		1100	BASE_SUBLIST,
		1101	VREGS_SUBLIST,
		1102	PAUTH_SUBLIST,
		1103	PMU_SUBLIST,
		1104	{0},
		1105	},
		1106	};
1059		1107
1060	static struct vcpu_config *vcpu_configs[] = {	1108	static struct vcpu_config *vcpu_configs[] = {
1061	&vregs_config,	1109	&vregs_config,
1062	&vregs_pmu_config,	1110	&vregs_pmu_config,
1063	&sve_config,	1111	&sve_config,
1064	&sve_pmu_config,	1112	&sve_pmu_config,
		1113	&pauth_config,
		1114	&pauth_pmu_config,
1065	};	1115	};
1066	static int vcpu_configs_n = ARRAY_SIZE(vcpu_configs);	1116	static int vcpu_configs_n = ARRAY_SIZE(vcpu_configs);

tools/testing/selftests/kvm/aarch64/vgic_irq.c+853−0

@@ -0,0 +1,853 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* vgic_irq.c - Test userspace injection of IRQs
	4	*
	5	* This test validates the injection of IRQs from userspace using various
	6	* methods (e.g., KVM_IRQ_LINE) and modes (e.g., EOI). The guest "asks" the
	7	* host to inject a specific intid via a GUEST_SYNC call, and then checks that
	8	* it received it.
	9	*/
	10
	11	#include <asm/kvm.h>
	12	#include <asm/kvm_para.h>
	13	#include <sys/eventfd.h>
	14	#include <linux/sizes.h>
	15
	16	#include "processor.h"
	17	#include "test_util.h"
	18	#include "kvm_util.h"
	19	#include "gic.h"
	20	#include "gic_v3.h"
	21	#include "vgic.h"
	22
	23	#define GICD_BASE_GPA 0x08000000ULL
	24	#define GICR_BASE_GPA 0x080A0000ULL
	25	#define VCPU_ID 0
	26
	27	/*
	28	* Stores the user specified args; it's passed to the guest and to every test
	29	* function.
	30	*/
	31	struct test_args {
	32	uint32_t nr_irqs; /* number of KVM supported IRQs. */
	33	bool eoi_split; /* 1 is eoir+dir, 0 is eoir only */
	34	bool level_sensitive; /* 1 is level, 0 is edge */
	35	int kvm_max_routes; /* output of KVM_CAP_IRQ_ROUTING */
	36	bool kvm_supports_irqfd; /* output of KVM_CAP_IRQFD */
	37	};
	38
	39	/*
	40	* KVM implements 32 priority levels:
	41	* 0x00 (highest priority) - 0xF8 (lowest priority), in steps of 8
	42	*
	43	* Note that these macros will still be correct in the case that KVM implements
	44	* more priority levels. Also note that 32 is the minimum for GICv3 and GICv2.
	45	*/
	46	#define KVM_NUM_PRIOS 32
	47	#define KVM_PRIO_SHIFT 3 /* steps of 8 = 1 << 3 */
	48	#define KVM_PRIO_STEPS (1 << KVM_PRIO_SHIFT) /* 8 */
	49	#define LOWEST_PRIO (KVM_NUM_PRIOS - 1)
	50	#define CPU_PRIO_MASK (LOWEST_PRIO << KVM_PRIO_SHIFT) /* 0xf8 */
	51	#define IRQ_DEFAULT_PRIO (LOWEST_PRIO - 1)
	52	#define IRQ_DEFAULT_PRIO_REG (IRQ_DEFAULT_PRIO << KVM_PRIO_SHIFT) /* 0xf0 */
	53
	54	static void dist = (void )GICD_BASE_GPA;
	55	static void redist = (void )GICR_BASE_GPA;
	56
	57	/*
	58	* The kvm_inject_* utilities are used by the guest to ask the host to inject
	59	* interrupts (e.g., using the KVM_IRQ_LINE ioctl).
	60	*/
	61
	62	typedef enum {
	63	KVM_INJECT_EDGE_IRQ_LINE = 1,
	64	KVM_SET_IRQ_LINE,
	65	KVM_SET_IRQ_LINE_HIGH,
	66	KVM_SET_LEVEL_INFO_HIGH,
	67	KVM_INJECT_IRQFD,
	68	KVM_WRITE_ISPENDR,
	69	KVM_WRITE_ISACTIVER,
	70	} kvm_inject_cmd;
	71
	72	struct kvm_inject_args {
	73	kvm_inject_cmd cmd;
	74	uint32_t first_intid;
	75	uint32_t num;
	76	int level;
	77	bool expect_failure;
	78	};
	79
	80	/* Used on the guest side to perform the hypercall. */
	81	static void kvm_inject_call(kvm_inject_cmd cmd, uint32_t first_intid,
	82	uint32_t num, int level, bool expect_failure);
	83
	84	/* Used on the host side to get the hypercall info. */
	85	static void kvm_inject_get_call(struct kvm_vm vm, struct ucall uc,
	86	struct kvm_inject_args *args);
	87
	88	#define _KVM_INJECT_MULTI(cmd, intid, num, expect_failure) \
	89	kvm_inject_call(cmd, intid, num, -1 /* not used */, expect_failure)
	90
	91	#define KVM_INJECT_MULTI(cmd, intid, num) \
	92	_KVM_INJECT_MULTI(cmd, intid, num, false)
	93
	94	#define _KVM_INJECT(cmd, intid, expect_failure) \
	95	_KVM_INJECT_MULTI(cmd, intid, 1, expect_failure)
	96
	97	#define KVM_INJECT(cmd, intid) \
	98	_KVM_INJECT_MULTI(cmd, intid, 1, false)
	99
	100	#define KVM_ACTIVATE(cmd, intid) \
	101	kvm_inject_call(cmd, intid, 1, 1, false);
	102
	103	struct kvm_inject_desc {
	104	kvm_inject_cmd cmd;
	105	/* can inject PPIs, PPIs, and/or SPIs. */
	106	bool sgi, ppi, spi;
	107	};
	108
	109	static struct kvm_inject_desc inject_edge_fns[] = {
	110	/* sgi ppi spi */
	111	{ KVM_INJECT_EDGE_IRQ_LINE, false, false, true },
	112	{ KVM_INJECT_IRQFD, false, false, true },
	113	{ KVM_WRITE_ISPENDR, true, false, true },
	114	{ 0, },
	115	};
	116
	117	static struct kvm_inject_desc inject_level_fns[] = {
	118	/* sgi ppi spi */
	119	{ KVM_SET_IRQ_LINE_HIGH, false, true, true },
	120	{ KVM_SET_LEVEL_INFO_HIGH, false, true, true },
	121	{ KVM_INJECT_IRQFD, false, false, true },
	122	{ KVM_WRITE_ISPENDR, false, true, true },
	123	{ 0, },
	124	};
	125
	126	static struct kvm_inject_desc set_active_fns[] = {
	127	/* sgi ppi spi */
	128	{ KVM_WRITE_ISACTIVER, true, true, true },
	129	{ 0, },
	130	};
	131
	132	#define for_each_inject_fn(t, f) \
	133	for ((f) = (t); (f)->cmd; (f)++)
	134
	135	#define for_each_supported_inject_fn(args, t, f) \
	136	for_each_inject_fn(t, f) \
	137	if ((args)->kvm_supports_irqfd \|\| (f)->cmd != KVM_INJECT_IRQFD)
	138
	139	#define for_each_supported_activate_fn(args, t, f) \
	140	for_each_supported_inject_fn((args), (t), (f))
	141
	142	/* Shared between the guest main thread and the IRQ handlers. */
	143	volatile uint64_t irq_handled;
	144	volatile uint32_t irqnr_received[MAX_SPI + 1];
	145
	146	static void reset_stats(void)
	147	{
	148	int i;
	149
	150	irq_handled = 0;
	151	for (i = 0; i <= MAX_SPI; i++)
	152	irqnr_received[i] = 0;
	153	}
	154
	155	static uint64_t gic_read_ap1r0(void)
	156	{
	157	uint64_t reg = read_sysreg_s(SYS_ICV_AP1R0_EL1);
	158
	159	dsb(sy);
	160	return reg;
	161	}
	162
	163	static void gic_write_ap1r0(uint64_t val)
	164	{
	165	write_sysreg_s(val, SYS_ICV_AP1R0_EL1);
	166	isb();
	167	}
	168
	169	static void guest_set_irq_line(uint32_t intid, uint32_t level);
	170
	171	static void guest_irq_generic_handler(bool eoi_split, bool level_sensitive)
	172	{
	173	uint32_t intid = gic_get_and_ack_irq();
	174
	175	if (intid == IAR_SPURIOUS)
	176	return;
	177
	178	GUEST_ASSERT(gic_irq_get_active(intid));
	179
	180	if (!level_sensitive)
	181	GUEST_ASSERT(!gic_irq_get_pending(intid));
	182
	183	if (level_sensitive)
	184	guest_set_irq_line(intid, 0);
	185
	186	GUEST_ASSERT(intid < MAX_SPI);
	187	irqnr_received[intid] += 1;
	188	irq_handled += 1;
	189
	190	gic_set_eoi(intid);
	191	GUEST_ASSERT_EQ(gic_read_ap1r0(), 0);
	192	if (eoi_split)
	193	gic_set_dir(intid);
	194
	195	GUEST_ASSERT(!gic_irq_get_active(intid));
	196	GUEST_ASSERT(!gic_irq_get_pending(intid));
	197	}
	198
	199	static void kvm_inject_call(kvm_inject_cmd cmd, uint32_t first_intid,
	200	uint32_t num, int level, bool expect_failure)
	201	{
	202	struct kvm_inject_args args = {
	203	.cmd = cmd,
	204	.first_intid = first_intid,
	205	.num = num,
	206	.level = level,
	207	.expect_failure = expect_failure,
	208	};
	209	GUEST_SYNC(&args);
	210	}
	211
	212	#define GUEST_ASSERT_IAR_EMPTY() \
	213	do { \
	214	uint32_t _intid; \
	215	_intid = gic_get_and_ack_irq(); \
	216	GUEST_ASSERT(_intid == 0 \|\| _intid == IAR_SPURIOUS); \
	217	} while (0)
	218
	219	#define CAT_HELPER(a, b) a ## b
	220	#define CAT(a, b) CAT_HELPER(a, b)
	221	#define PREFIX guest_irq_handler_
	222	#define GUEST_IRQ_HANDLER_NAME(split, lev) CAT(PREFIX, CAT(split, lev))
	223	#define GENERATE_GUEST_IRQ_HANDLER(split, lev) \
	224	static void CAT(PREFIX, CAT(split, lev))(struct ex_regs *regs) \
	225	{ \
	226	guest_irq_generic_handler(split, lev); \
	227	}
	228
	229	GENERATE_GUEST_IRQ_HANDLER(0, 0);
	230	GENERATE_GUEST_IRQ_HANDLER(0, 1);
	231	GENERATE_GUEST_IRQ_HANDLER(1, 0);
	232	GENERATE_GUEST_IRQ_HANDLER(1, 1);
	233
	234	static void (guest_irq_handlers[2][2])(struct ex_regs ) = {
	235	{GUEST_IRQ_HANDLER_NAME(0, 0), GUEST_IRQ_HANDLER_NAME(0, 1),},
	236	{GUEST_IRQ_HANDLER_NAME(1, 0), GUEST_IRQ_HANDLER_NAME(1, 1),},
	237	};
	238
	239	static void reset_priorities(struct test_args *args)
	240	{
	241	int i;
	242
	243	for (i = 0; i < args->nr_irqs; i++)
	244	gic_set_priority(i, IRQ_DEFAULT_PRIO_REG);
	245	}
	246
	247	static void guest_set_irq_line(uint32_t intid, uint32_t level)
	248	{
	249	kvm_inject_call(KVM_SET_IRQ_LINE, intid, 1, level, false);
	250	}
	251
	252	static void test_inject_fail(struct test_args *args,
	253	uint32_t intid, kvm_inject_cmd cmd)
	254	{
	255	reset_stats();
	256
	257	_KVM_INJECT(cmd, intid, true);
	258	/* no IRQ to handle on entry */
	259
	260	GUEST_ASSERT_EQ(irq_handled, 0);
	261	GUEST_ASSERT_IAR_EMPTY();
	262	}
	263
	264	static void guest_inject(struct test_args *args,
	265	uint32_t first_intid, uint32_t num,
	266	kvm_inject_cmd cmd)
	267	{
	268	uint32_t i;
	269
	270	reset_stats();
	271
	272	/* Cycle over all priorities to make things more interesting. */
	273	for (i = first_intid; i < num + first_intid; i++)
	274	gic_set_priority(i, (i % (KVM_NUM_PRIOS - 1)) << 3);
	275
	276	asm volatile("msr daifset, #2" : : : "memory");
	277	KVM_INJECT_MULTI(cmd, first_intid, num);
	278
	279	while (irq_handled < num) {
	280	asm volatile("wfi\n"
	281	"msr daifclr, #2\n"
	282	/* handle IRQ */
	283	"msr daifset, #2\n"
	284	: : : "memory");
	285	}
	286	asm volatile("msr daifclr, #2" : : : "memory");
	287
	288	GUEST_ASSERT_EQ(irq_handled, num);
	289	for (i = first_intid; i < num + first_intid; i++)
	290	GUEST_ASSERT_EQ(irqnr_received[i], 1);
	291	GUEST_ASSERT_IAR_EMPTY();
	292
	293	reset_priorities(args);
	294	}
	295
	296	/*
	297	* Restore the active state of multiple concurrent IRQs (given by
	298	* concurrent_irqs). This does what a live-migration would do on the
	299	* destination side assuming there are some active IRQs that were not
	300	* deactivated yet.
	301	*/
	302	static void guest_restore_active(struct test_args *args,
	303	uint32_t first_intid, uint32_t num,
	304	kvm_inject_cmd cmd)
	305	{
	306	uint32_t prio, intid, ap1r;
	307	int i;
	308
	309	/* Set the priorities of the first (KVM_NUM_PRIOS - 1) IRQs
	310	* in descending order, so intid+1 can preempt intid.
	311	*/
	312	for (i = 0, prio = (num - 1) * 8; i < num; i++, prio -= 8) {
	313	GUEST_ASSERT(prio >= 0);
	314	intid = i + first_intid;
	315	gic_set_priority(intid, prio);
	316	}
	317
	318	/* In a real migration, KVM would restore all GIC state before running
	319	* guest code.
	320	*/
	321	for (i = 0; i < num; i++) {
	322	intid = i + first_intid;
	323	KVM_ACTIVATE(cmd, intid);
	324	ap1r = gic_read_ap1r0();
	325	ap1r \|= 1U << i;
	326	gic_write_ap1r0(ap1r);
	327	}
	328
	329	/* This is where the "migration" would occur. */
	330
	331	/* finish handling the IRQs starting with the highest priority one. */
	332	for (i = 0; i < num; i++) {
	333	intid = num - i - 1 + first_intid;
	334	gic_set_eoi(intid);
	335	if (args->eoi_split)
	336	gic_set_dir(intid);
	337	}
	338
	339	for (i = 0; i < num; i++)
	340	GUEST_ASSERT(!gic_irq_get_active(i + first_intid));
	341	GUEST_ASSERT_EQ(gic_read_ap1r0(), 0);
	342	GUEST_ASSERT_IAR_EMPTY();
	343	}
	344
	345	/*
	346	* Polls the IAR until it's not a spurious interrupt.
	347	*
	348	* This function should only be used in test_inject_preemption (with IRQs
	349	* masked).
	350	*/
	351	static uint32_t wait_for_and_activate_irq(void)
	352	{
	353	uint32_t intid;
	354
	355	do {
	356	asm volatile("wfi" : : : "memory");
	357	intid = gic_get_and_ack_irq();
	358	} while (intid == IAR_SPURIOUS);
	359
	360	return intid;
	361	}
	362
	363	/*
	364	* Inject multiple concurrent IRQs (num IRQs starting at first_intid) and
	365	* handle them without handling the actual exceptions. This is done by masking
	366	* interrupts for the whole test.
	367	*/
	368	static void test_inject_preemption(struct test_args *args,
	369	uint32_t first_intid, int num,
	370	kvm_inject_cmd cmd)
	371	{
	372	uint32_t intid, prio, step = KVM_PRIO_STEPS;
	373	int i;
	374
	375	/* Set the priorities of the first (KVM_NUM_PRIOS - 1) IRQs
	376	* in descending order, so intid+1 can preempt intid.
	377	*/
	378	for (i = 0, prio = (num - 1) * step; i < num; i++, prio -= step) {
	379	GUEST_ASSERT(prio >= 0);
	380	intid = i + first_intid;
	381	gic_set_priority(intid, prio);
	382	}
	383
	384	local_irq_disable();
	385
	386	for (i = 0; i < num; i++) {
	387	uint32_t tmp;
	388	intid = i + first_intid;
	389	KVM_INJECT(cmd, intid);
	390	/* Each successive IRQ will preempt the previous one. */
	391	tmp = wait_for_and_activate_irq();
	392	GUEST_ASSERT_EQ(tmp, intid);
	393	if (args->level_sensitive)
	394	guest_set_irq_line(intid, 0);
	395	}
	396
	397	/* finish handling the IRQs starting with the highest priority one. */
	398	for (i = 0; i < num; i++) {
	399	intid = num - i - 1 + first_intid;
	400	gic_set_eoi(intid);
	401	if (args->eoi_split)
	402	gic_set_dir(intid);
	403	}
	404
	405	local_irq_enable();
	406
	407	for (i = 0; i < num; i++)
	408	GUEST_ASSERT(!gic_irq_get_active(i + first_intid));
	409	GUEST_ASSERT_EQ(gic_read_ap1r0(), 0);
	410	GUEST_ASSERT_IAR_EMPTY();
	411
	412	reset_priorities(args);
	413	}
	414
	415	static void test_injection(struct test_args args, struct kvm_inject_desc f)
	416	{
	417	uint32_t nr_irqs = args->nr_irqs;
	418
	419	if (f->sgi) {
	420	guest_inject(args, MIN_SGI, 1, f->cmd);
	421	guest_inject(args, 0, 16, f->cmd);
	422	}
	423
	424	if (f->ppi)
	425	guest_inject(args, MIN_PPI, 1, f->cmd);
	426
	427	if (f->spi) {
	428	guest_inject(args, MIN_SPI, 1, f->cmd);
	429	guest_inject(args, nr_irqs - 1, 1, f->cmd);
	430	guest_inject(args, MIN_SPI, nr_irqs - MIN_SPI, f->cmd);
	431	}
	432	}
	433
	434	static void test_injection_failure(struct test_args *args,
	435	struct kvm_inject_desc *f)
	436	{
	437	uint32_t bad_intid[] = { args->nr_irqs, 1020, 1024, 1120, 5120, ~0U, };
	438	int i;
	439
	440	for (i = 0; i < ARRAY_SIZE(bad_intid); i++)
	441	test_inject_fail(args, bad_intid[i], f->cmd);
	442	}
	443
	444	static void test_preemption(struct test_args args, struct kvm_inject_desc f)
	445	{
	446	/*
	447	* Test up to 4 levels of preemption. The reason is that KVM doesn't
	448	* currently implement the ability to have more than the number-of-LRs
	449	* number of concurrently active IRQs. The number of LRs implemented is
	450	* IMPLEMENTATION DEFINED, however, it seems that most implement 4.
	451	*/
	452	if (f->sgi)
	453	test_inject_preemption(args, MIN_SGI, 4, f->cmd);
	454
	455	if (f->ppi)
	456	test_inject_preemption(args, MIN_PPI, 4, f->cmd);
	457
	458	if (f->spi)
	459	test_inject_preemption(args, MIN_SPI, 4, f->cmd);
	460	}
	461
	462	static void test_restore_active(struct test_args args, struct kvm_inject_desc f)
	463	{
	464	/* Test up to 4 active IRQs. Same reason as in test_preemption. */
	465	if (f->sgi)
	466	guest_restore_active(args, MIN_SGI, 4, f->cmd);
	467
	468	if (f->ppi)
	469	guest_restore_active(args, MIN_PPI, 4, f->cmd);
	470
	471	if (f->spi)
	472	guest_restore_active(args, MIN_SPI, 4, f->cmd);
	473	}
	474
	475	static void guest_code(struct test_args args)
	476	{
	477	uint32_t i, nr_irqs = args.nr_irqs;
	478	bool level_sensitive = args.level_sensitive;
	479	struct kvm_inject_desc f, inject_fns;
	480
	481	gic_init(GIC_V3, 1, dist, redist);
	482
	483	for (i = 0; i < nr_irqs; i++)
	484	gic_irq_enable(i);
	485
	486	for (i = MIN_SPI; i < nr_irqs; i++)
	487	gic_irq_set_config(i, !args.level_sensitive);
	488
	489	gic_set_eoi_split(args.eoi_split);
	490
	491	reset_priorities(&args);
	492	gic_set_priority_mask(CPU_PRIO_MASK);
	493
	494	inject_fns = level_sensitive ? inject_level_fns
	495	: inject_edge_fns;
	496
	497	local_irq_enable();
	498
	499	/* Start the tests. */
	500	for_each_supported_inject_fn(&args, inject_fns, f) {
	501	test_injection(&args, f);
	502	test_preemption(&args, f);
	503	test_injection_failure(&args, f);
	504	}
	505
	506	/* Restore the active state of IRQs. This would happen when live
	507	* migrating IRQs in the middle of being handled.
	508	*/
	509	for_each_supported_activate_fn(&args, set_active_fns, f)
	510	test_restore_active(&args, f);
	511
	512	GUEST_DONE();
	513	}
	514
	515	static void kvm_irq_line_check(struct kvm_vm *vm, uint32_t intid, int level,
	516	struct test_args *test_args, bool expect_failure)
	517	{
	518	int ret;
	519
	520	if (!expect_failure) {
	521	kvm_arm_irq_line(vm, intid, level);
	522	} else {
	523	/* The interface doesn't allow larger intid's. */
	524	if (intid > KVM_ARM_IRQ_NUM_MASK)
	525	return;
	526
	527	ret = _kvm_arm_irq_line(vm, intid, level);
	528	TEST_ASSERT(ret != 0 && errno == EINVAL,
	529	"Bad intid %i did not cause KVM_IRQ_LINE "
	530	"error: rc: %i errno: %i", intid, ret, errno);
	531	}
	532	}
	533
	534	void kvm_irq_set_level_info_check(int gic_fd, uint32_t intid, int level,
	535	bool expect_failure)
	536	{
	537	if (!expect_failure) {
	538	kvm_irq_set_level_info(gic_fd, intid, level);
	539	} else {
	540	int ret = _kvm_irq_set_level_info(gic_fd, intid, level);
	541	/*
	542	* The kernel silently fails for invalid SPIs and SGIs (which
	543	* are not level-sensitive). It only checks for intid to not
	544	* spill over 1U << 10 (the max reserved SPI). Also, callers
	545	* are supposed to mask the intid with 0x3ff (1023).
	546	*/
	547	if (intid > VGIC_MAX_RESERVED)
	548	TEST_ASSERT(ret != 0 && errno == EINVAL,
	549	"Bad intid %i did not cause VGIC_GRP_LEVEL_INFO "
	550	"error: rc: %i errno: %i", intid, ret, errno);
	551	else
	552	TEST_ASSERT(!ret, "KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO "
	553	"for intid %i failed, rc: %i errno: %i",
	554	intid, ret, errno);
	555	}
	556	}
	557
	558	static void kvm_set_gsi_routing_irqchip_check(struct kvm_vm *vm,
	559	uint32_t intid, uint32_t num, uint32_t kvm_max_routes,
	560	bool expect_failure)
	561	{
	562	struct kvm_irq_routing *routing;
	563	int ret;
	564	uint64_t i;
	565
	566	assert(num <= kvm_max_routes && kvm_max_routes <= KVM_MAX_IRQ_ROUTES);
	567
	568	routing = kvm_gsi_routing_create();
	569	for (i = intid; i < (uint64_t)intid + num; i++)
	570	kvm_gsi_routing_irqchip_add(routing, i - MIN_SPI, i - MIN_SPI);
	571
	572	if (!expect_failure) {
	573	kvm_gsi_routing_write(vm, routing);
	574	} else {
	575	ret = _kvm_gsi_routing_write(vm, routing);
	576	/* The kernel only checks for KVM_IRQCHIP_NUM_PINS. */
	577	if (intid >= KVM_IRQCHIP_NUM_PINS)
	578	TEST_ASSERT(ret != 0 && errno == EINVAL,
	579	"Bad intid %u did not cause KVM_SET_GSI_ROUTING "
	580	"error: rc: %i errno: %i", intid, ret, errno);
	581	else
	582	TEST_ASSERT(ret == 0, "KVM_SET_GSI_ROUTING "
	583	"for intid %i failed, rc: %i errno: %i",
	584	intid, ret, errno);
	585	}
	586	}
	587
	588	static void kvm_irq_write_ispendr_check(int gic_fd, uint32_t intid,
	589	uint32_t vcpu, bool expect_failure)
	590	{
	591	/*
	592	* Ignore this when expecting failure as invalid intids will lead to
	593	* either trying to inject SGIs when we configured the test to be
	594	* level_sensitive (or the reverse), or inject large intids which
	595	* will lead to writing above the ISPENDR register space (and we
	596	* don't want to do that either).
	597	*/
	598	if (!expect_failure)
	599	kvm_irq_write_ispendr(gic_fd, intid, vcpu);
	600	}
	601
	602	static void kvm_routing_and_irqfd_check(struct kvm_vm *vm,
	603	uint32_t intid, uint32_t num, uint32_t kvm_max_routes,
	604	bool expect_failure)
	605	{
	606	int fd[MAX_SPI];
	607	uint64_t val;
	608	int ret, f;
	609	uint64_t i;
	610
	611	/*
	612	* There is no way to try injecting an SGI or PPI as the interface
	613	* starts counting from the first SPI (above the private ones), so just
	614	* exit.
	615	*/
	616	if (INTID_IS_SGI(intid) \|\| INTID_IS_PPI(intid))
	617	return;
	618
	619	kvm_set_gsi_routing_irqchip_check(vm, intid, num,
	620	kvm_max_routes, expect_failure);
	621
	622	/*
	623	* If expect_failure, then just to inject anyway. These
	624	* will silently fail. And in any case, the guest will check
	625	* that no actual interrupt was injected for those cases.
	626	*/
	627
	628	for (f = 0, i = intid; i < (uint64_t)intid + num; i++, f++) {
	629	fd[f] = eventfd(0, 0);
	630	TEST_ASSERT(fd[f] != -1,
	631	"eventfd failed, errno: %i\n", errno);
	632	}
	633
	634	for (f = 0, i = intid; i < (uint64_t)intid + num; i++, f++) {
	635	struct kvm_irqfd irqfd = {
	636	.fd = fd[f],
	637	.gsi = i - MIN_SPI,
	638	};
	639	assert(i <= (uint64_t)UINT_MAX);
	640	vm_ioctl(vm, KVM_IRQFD, &irqfd);
	641	}
	642
	643	for (f = 0, i = intid; i < (uint64_t)intid + num; i++, f++) {
	644	val = 1;
	645	ret = write(fd[f], &val, sizeof(uint64_t));
	646	TEST_ASSERT(ret == sizeof(uint64_t),
	647	"Write to KVM_IRQFD failed with ret: %d\n", ret);
	648	}
	649
	650	for (f = 0, i = intid; i < (uint64_t)intid + num; i++, f++)
	651	close(fd[f]);
	652	}
	653
	654	/* handles the valid case: intid=0xffffffff num=1 */
	655	#define for_each_intid(first, num, tmp, i) \
	656	for ((tmp) = (i) = (first); \
	657	(tmp) < (uint64_t)(first) + (uint64_t)(num); \
	658	(tmp)++, (i)++)
	659
	660	static void run_guest_cmd(struct kvm_vm *vm, int gic_fd,
	661	struct kvm_inject_args *inject_args,
	662	struct test_args *test_args)
	663	{
	664	kvm_inject_cmd cmd = inject_args->cmd;
	665	uint32_t intid = inject_args->first_intid;
	666	uint32_t num = inject_args->num;
	667	int level = inject_args->level;
	668	bool expect_failure = inject_args->expect_failure;
	669	uint64_t tmp;
	670	uint32_t i;
	671
	672	/* handles the valid case: intid=0xffffffff num=1 */
	673	assert(intid < UINT_MAX - num \|\| num == 1);
	674
	675	switch (cmd) {
	676	case KVM_INJECT_EDGE_IRQ_LINE:
	677	for_each_intid(intid, num, tmp, i)
	678	kvm_irq_line_check(vm, i, 1, test_args,
	679	expect_failure);
	680	for_each_intid(intid, num, tmp, i)
	681	kvm_irq_line_check(vm, i, 0, test_args,
	682	expect_failure);
	683	break;
	684	case KVM_SET_IRQ_LINE:
	685	for_each_intid(intid, num, tmp, i)
	686	kvm_irq_line_check(vm, i, level, test_args,
	687	expect_failure);
	688	break;
	689	case KVM_SET_IRQ_LINE_HIGH:
	690	for_each_intid(intid, num, tmp, i)
	691	kvm_irq_line_check(vm, i, 1, test_args,
	692	expect_failure);
	693	break;
	694	case KVM_SET_LEVEL_INFO_HIGH:
	695	for_each_intid(intid, num, tmp, i)
	696	kvm_irq_set_level_info_check(gic_fd, i, 1,
	697	expect_failure);
	698	break;
	699	case KVM_INJECT_IRQFD:
	700	kvm_routing_and_irqfd_check(vm, intid, num,
	701	test_args->kvm_max_routes,
	702	expect_failure);
	703	break;
	704	case KVM_WRITE_ISPENDR:
	705	for (i = intid; i < intid + num; i++)
	706	kvm_irq_write_ispendr_check(gic_fd, i,
	707	VCPU_ID, expect_failure);
	708	break;
	709	case KVM_WRITE_ISACTIVER:
	710	for (i = intid; i < intid + num; i++)
	711	kvm_irq_write_isactiver(gic_fd, i, VCPU_ID);
	712	break;
	713	default:
	714	break;
	715	}
	716	}
	717
	718	static void kvm_inject_get_call(struct kvm_vm vm, struct ucall uc,
	719	struct kvm_inject_args *args)
	720	{
	721	struct kvm_inject_args *kvm_args_hva;
	722	vm_vaddr_t kvm_args_gva;
	723
	724	kvm_args_gva = uc->args[1];
	725	kvm_args_hva = (struct kvm_inject_args *)addr_gva2hva(vm, kvm_args_gva);
	726	memcpy(args, kvm_args_hva, sizeof(struct kvm_inject_args));
	727	}
	728
	729	static void print_args(struct test_args *args)
	730	{
	731	printf("nr-irqs=%d level-sensitive=%d eoi-split=%d\n",
	732	args->nr_irqs, args->level_sensitive,
	733	args->eoi_split);
	734	}
	735
	736	static void test_vgic(uint32_t nr_irqs, bool level_sensitive, bool eoi_split)
	737	{
	738	struct ucall uc;
	739	int gic_fd;
	740	struct kvm_vm *vm;
	741	struct kvm_inject_args inject_args;
	742
	743	struct test_args args = {
	744	.nr_irqs = nr_irqs,
	745	.level_sensitive = level_sensitive,
	746	.eoi_split = eoi_split,
	747	.kvm_max_routes = kvm_check_cap(KVM_CAP_IRQ_ROUTING),
	748	.kvm_supports_irqfd = kvm_check_cap(KVM_CAP_IRQFD),
	749	};
	750
	751	print_args(&args);
	752
	753	vm = vm_create_default(VCPU_ID, 0, guest_code);
	754	ucall_init(vm, NULL);
	755
	756	vm_init_descriptor_tables(vm);
	757	vcpu_init_descriptor_tables(vm, VCPU_ID);
	758
	759	/* Setup the guest args page (so it gets the args). */
	760	vcpu_args_set(vm, 0, 1, args);
	761
	762	gic_fd = vgic_v3_setup(vm, 1, nr_irqs,
	763	GICD_BASE_GPA, GICR_BASE_GPA);
	764
	765	vm_install_exception_handler(vm, VECTOR_IRQ_CURRENT,
	766	guest_irq_handlers[args.eoi_split][args.level_sensitive]);
	767
	768	while (1) {
	769	vcpu_run(vm, VCPU_ID);
	770
	771	switch (get_ucall(vm, VCPU_ID, &uc)) {
	772	case UCALL_SYNC:
	773	kvm_inject_get_call(vm, &uc, &inject_args);
	774	run_guest_cmd(vm, gic_fd, &inject_args, &args);
	775	break;
	776	case UCALL_ABORT:
	777	TEST_FAIL("%s at %s:%ld\n\tvalues: %#lx, %#lx",
	778	(const char *)uc.args[0],
	779	__FILE__, uc.args[1], uc.args[2], uc.args[3]);
	780	break;
	781	case UCALL_DONE:
	782	goto done;
	783	default:
	784	TEST_FAIL("Unknown ucall %lu", uc.cmd);
	785	}
	786	}
	787
	788	done:
	789	close(gic_fd);
	790	kvm_vm_free(vm);
	791	}
	792
	793	static void help(const char *name)
	794	{
	795	printf(
	796	"\n"
	797	"usage: %s [-n num_irqs] [-e eoi_split] [-l level_sensitive]\n", name);
	798	printf(" -n: specify number of IRQs to setup the vgic with. "
	799	"It has to be a multiple of 32 and between 64 and 1024.\n");
	800	printf(" -e: if 1 then EOI is split into a write to DIR on top "
	801	"of writing EOI.\n");
	802	printf(" -l: specify whether the IRQs are level-sensitive (1) or not (0).");
	803	puts("");
	804	exit(1);
	805	}
	806
	807	int main(int argc, char **argv)
	808	{
	809	uint32_t nr_irqs = 64;
	810	bool default_args = true;
	811	bool level_sensitive = false;
	812	int opt;
	813	bool eoi_split = false;
	814
	815	/* Tell stdout not to buffer its content */
	816	setbuf(stdout, NULL);
	817
	818	while ((opt = getopt(argc, argv, "hn:e:l:")) != -1) {
	819	switch (opt) {
	820	case 'n':
	821	nr_irqs = atoi(optarg);
	822	if (nr_irqs > 1024 \|\| nr_irqs % 32)
	823	help(argv[0]);
	824	break;
	825	case 'e':
	826	eoi_split = (bool)atoi(optarg);
	827	default_args = false;
	828	break;
	829	case 'l':
	830	level_sensitive = (bool)atoi(optarg);
	831	default_args = false;
	832	break;
	833	case 'h':
	834	default:
	835	help(argv[0]);
	836	break;
	837	}
	838	}
	839
	840	/* If the user just specified nr_irqs and/or gic_version, then run all
	841	* combinations.
	842	*/
	843	if (default_args) {
	844	test_vgic(nr_irqs, false /* level /, false / eoi_split */);
	845	test_vgic(nr_irqs, false /* level /, true / eoi_split */);
	846	test_vgic(nr_irqs, true /* level /, false / eoi_split */);
	847	test_vgic(nr_irqs, true /* level /, true / eoi_split */);
	848	} else {
	849	test_vgic(nr_irqs, level_sensitive, eoi_split);
	850	}
	851
	852	return 0;
	853	}

tools/testing/selftests/kvm/include/aarch64/gic.h+26−0

@@ -11,11 +11,37 @@ enum gic_type { · 10 unmodified lines
11	GIC_TYPE_MAX,	11	GIC_TYPE_MAX,
12	};	12	};
13		13
		14	#define MIN_SGI 0
		15	#define MIN_PPI 16
		16	#define MIN_SPI 32
		17	#define MAX_SPI 1019
		18	#define IAR_SPURIOUS 1023
		19
		20	#define INTID_IS_SGI(intid) (0 <= (intid) && (intid) < MIN_PPI)
		21	#define INTID_IS_PPI(intid) (MIN_PPI <= (intid) && (intid) < MIN_SPI)
		22	#define INTID_IS_SPI(intid) (MIN_SPI <= (intid) && (intid) <= MAX_SPI)
		23
14	void gic_init(enum gic_type type, unsigned int nr_cpus,	24	void gic_init(enum gic_type type, unsigned int nr_cpus,
15	void dist_base, void redist_base);	25	void dist_base, void redist_base);
16	void gic_irq_enable(unsigned int intid);	26	void gic_irq_enable(unsigned int intid);
17	void gic_irq_disable(unsigned int intid);	27	void gic_irq_disable(unsigned int intid);
18	unsigned int gic_get_and_ack_irq(void);	28	unsigned int gic_get_and_ack_irq(void);
19	void gic_set_eoi(unsigned int intid);	29	void gic_set_eoi(unsigned int intid);
		30	void gic_set_dir(unsigned int intid);
		31
		32	/*
		33	* Sets the EOI mode. When split is false, EOI just drops the priority. When
		34	* split is true, EOI drops the priority and deactivates the interrupt.
		35	*/
		36	void gic_set_eoi_split(bool split);
		37	void gic_set_priority_mask(uint64_t mask);
		38	void gic_set_priority(uint32_t intid, uint32_t prio);
		39	void gic_irq_set_active(unsigned int intid);
		40	void gic_irq_clear_active(unsigned int intid);
		41	bool gic_irq_get_active(unsigned int intid);
		42	void gic_irq_set_pending(unsigned int intid);
		43	void gic_irq_clear_pending(unsigned int intid);
		44	bool gic_irq_get_pending(unsigned int intid);
		45	void gic_irq_set_config(unsigned int intid, bool is_edge);
20		46
21	#endif /* SELFTEST_KVM_GIC_H */	47	#endif /* SELFTEST_KVM_GIC_H */

tools/testing/selftests/kvm/include/aarch64/gic_v3.hrenamed from tools/testing/selftests/kvm/lib/aarch64/gic_v3.h+12−0

 #define GICD_IGROUPR			0x0080
 #define GICD_ISENABLER			0x0100
 #define GICD_ICENABLER			0x0180
+#define GICD_ISPENDR			0x0200
+#define GICD_ICPENDR			0x0280
 #define GICD_ICACTIVER			0x0380
+#define GICD_ISACTIVER			0x0300
 #define GICD_IPRIORITYR			0x0400
+#define GICD_ICFGR			0x0C00
 /*
  * The assumption is that the guest runs in a non-secure mode.
 #define GICR_IGROUPR0			GICD_IGROUPR
 #define GICR_ISENABLER0			GICD_ISENABLER
 #define GICR_ICENABLER0			GICD_ICENABLER
+#define GICR_ISPENDR0			GICD_ISPENDR
+#define GICR_ISACTIVER0			GICD_ISACTIVER
 #define GICR_ICACTIVER0			GICD_ICACTIVER
+#define GICR_ICENABLER			GICD_ICENABLER
+#define GICR_ICACTIVER			GICD_ICACTIVER
 #define GICR_IPRIORITYR0		GICD_IPRIORITYR
 /* CPU interface registers */
 #define SYS_ICC_PMR_EL1			sys_reg(3, 0, 4, 6, 0)
 #define SYS_ICC_IAR1_EL1		sys_reg(3, 0, 12, 12, 0)
 #define SYS_ICC_EOIR1_EL1		sys_reg(3, 0, 12, 12, 1)
+#define SYS_ICC_DIR_EL1			sys_reg(3, 0, 12, 11, 1)
+#define SYS_ICC_CTLR_EL1		sys_reg(3, 0, 12, 12, 4)
 #define SYS_ICC_SRE_EL1			sys_reg(3, 0, 12, 12, 5)
 #define SYS_ICC_GRPEN1_EL1		sys_reg(3, 0, 12, 12, 7)
+#define SYS_ICV_AP1R0_EL1		sys_reg(3, 0, 12, 9, 0)
 #define ICC_PMR_DEF_PRIO		0xf0
 #define ICC_SRE_EL1_SRE			(1U << 0)

tools/testing/selftests/kvm/include/aarch64/processor.h+3−0

@@ -113,6 +113,9 @@ enum { · 112 unmodified lines
113	#define ESR_EC_WP_CURRENT 0x35	113	#define ESR_EC_WP_CURRENT 0x35
114	#define ESR_EC_BRK_INS 0x3c	114	#define ESR_EC_BRK_INS 0x3c
115		115
		116	void aarch64_get_supported_page_sizes(uint32_t ipa,
		117	bool ps4k, bool ps16k, bool *ps64k);
		118
116	void vm_init_descriptor_tables(struct kvm_vm *vm);	119	void vm_init_descriptor_tables(struct kvm_vm *vm);
117	void vcpu_init_descriptor_tables(struct kvm_vm *vm, uint32_t vcpuid);	120	void vcpu_init_descriptor_tables(struct kvm_vm *vm, uint32_t vcpuid);
118		121

tools/testing/selftests/kvm/include/aarch64/vgic.h+16−2

@@ -14,7 +14,21 @@ · 13 unmodified lines
14	((uint64_t)(flags) << 12) \| \	14	((uint64_t)(flags) << 12) \| \
15	index)	15	index)
16		16
17	int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus,	17	int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs,
18	uint64_t gicd_base_gpa, uint64_t gicr_base_gpa);	18	uint64_t gicd_base_gpa, uint64_t gicr_base_gpa);
19		19
20	#endif /* SELFTEST_KVM_VGIC_H */	20	#define VGIC_MAX_RESERVED 1023
		21
		22	void kvm_irq_set_level_info(int gic_fd, uint32_t intid, int level);
		23	int _kvm_irq_set_level_info(int gic_fd, uint32_t intid, int level);
		24
		25	void kvm_arm_irq_line(struct kvm_vm *vm, uint32_t intid, int level);
		26	int _kvm_arm_irq_line(struct kvm_vm *vm, uint32_t intid, int level);
		27
		28	/* The vcpu arg only applies to private interrupts. */
		29	void kvm_irq_write_ispendr(int gic_fd, uint32_t intid, uint32_t vcpu);
		30	void kvm_irq_write_isactiver(int gic_fd, uint32_t intid, uint32_t vcpu);
		31
		32	#define KVM_IRQCHIP_NUM_PINS (1020 - 32)
		33
		34	#endif // SELFTEST_KVM_VGIC_H

tools/testing/selftests/kvm/include/kvm_util.h+19−1

 	VM_MODE_P52V48_4K,
 	VM_MODE_P52V48_64K,
 	VM_MODE_P48V48_4K,
+	VM_MODE_P48V48_16K,
 	VM_MODE_P48V48_64K,
 	VM_MODE_P40V48_4K,
+	VM_MODE_P40V48_16K,
 	VM_MODE_P40V48_64K,
 	VM_MODE_PXXV48_4K,	/* For 48bits VA but ANY bits PA */
 	VM_MODE_P47V64_4K,
 	VM_MODE_P44V64_4K,
+	VM_MODE_P36V48_4K,
+	VM_MODE_P36V48_16K,
+	VM_MODE_P36V48_64K,
+	VM_MODE_P36V47_16K,
 	NUM_VM_MODES,
 };
 #if defined(__aarch64__)
-#define VM_MODE_DEFAULT			VM_MODE_P40V48_4K
+extern enum vm_guest_mode vm_mode_default;
+#define VM_MODE_DEFAULT			vm_mode_default
 #define MIN_PAGE_SHIFT			12U
 #define ptes_per_page(page_size)	((page_size) / 8)
 		       void *val, bool write);
 int kvm_device_access(int dev_fd, uint32_t group, uint64_t attr,
 		      void *val, bool write);
+void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);
+int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);
 int _vcpu_has_device_attr(struct kvm_vm *vm, uint32_t vcpuid, uint32_t group,
 			  uint64_t attr);
 int vcpu_access_device_attr(struct kvm_vm *vm, uint32_t vcpuid, uint32_t group,
 			 uint64_t attr, void *val, bool write);
+#define KVM_MAX_IRQ_ROUTES		4096
+struct kvm_irq_routing *kvm_gsi_routing_create(void);
+void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing,
+		uint32_t gsi, uint32_t pin);
+int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
+void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
 const char *exit_reason_str(unsigned int exit_reason);
 void virt_pgd_alloc(struct kvm_vm *vm);

tools/testing/selftests/kvm/lib/aarch64/gic.c+66−0

@@ -93,3 +93,69 @@ void gic_set_eoi(unsigned int intid) · 92 unmodified lines
93	GUEST_ASSERT(gic_common_ops);	93	GUEST_ASSERT(gic_common_ops);
94	gic_common_ops->gic_write_eoir(intid);	94	gic_common_ops->gic_write_eoir(intid);
95	}	95	}
		96
		97	void gic_set_dir(unsigned int intid)
		98	{
		99	GUEST_ASSERT(gic_common_ops);
		100	gic_common_ops->gic_write_dir(intid);
		101	}
		102
		103	void gic_set_eoi_split(bool split)
		104	{
		105	GUEST_ASSERT(gic_common_ops);
		106	gic_common_ops->gic_set_eoi_split(split);
		107	}
		108
		109	void gic_set_priority_mask(uint64_t pmr)
		110	{
		111	GUEST_ASSERT(gic_common_ops);
		112	gic_common_ops->gic_set_priority_mask(pmr);
		113	}
		114
		115	void gic_set_priority(unsigned int intid, unsigned int prio)
		116	{
		117	GUEST_ASSERT(gic_common_ops);
		118	gic_common_ops->gic_set_priority(intid, prio);
		119	}
		120
		121	void gic_irq_set_active(unsigned int intid)
		122	{
		123	GUEST_ASSERT(gic_common_ops);
		124	gic_common_ops->gic_irq_set_active(intid);
		125	}
		126
		127	void gic_irq_clear_active(unsigned int intid)
		128	{
		129	GUEST_ASSERT(gic_common_ops);
		130	gic_common_ops->gic_irq_clear_active(intid);
		131	}
		132
		133	bool gic_irq_get_active(unsigned int intid)
		134	{
		135	GUEST_ASSERT(gic_common_ops);
		136	return gic_common_ops->gic_irq_get_active(intid);
		137	}
		138
		139	void gic_irq_set_pending(unsigned int intid)
		140	{
		141	GUEST_ASSERT(gic_common_ops);
		142	gic_common_ops->gic_irq_set_pending(intid);
		143	}
		144
		145	void gic_irq_clear_pending(unsigned int intid)
		146	{
		147	GUEST_ASSERT(gic_common_ops);
		148	gic_common_ops->gic_irq_clear_pending(intid);
		149	}
		150
		151	bool gic_irq_get_pending(unsigned int intid)
		152	{
		153	GUEST_ASSERT(gic_common_ops);
		154	return gic_common_ops->gic_irq_get_pending(intid);
		155	}
		156
		157	void gic_irq_set_config(unsigned int intid, bool is_edge)
		158	{
		159	GUEST_ASSERT(gic_common_ops);
		160	gic_common_ops->gic_irq_set_config(intid, is_edge);
		161	}

tools/testing/selftests/kvm/lib/aarch64/gic_private.h+11−0

@@ -14,6 +14,17 @@ struct gic_common_ops { · 13 unmodified lines
14	void (*gic_irq_disable)(unsigned int intid);	14	void (*gic_irq_disable)(unsigned int intid);
15	uint64_t (*gic_read_iar)(void);	15	uint64_t (*gic_read_iar)(void);
16	void (*gic_write_eoir)(uint32_t irq);	16	void (*gic_write_eoir)(uint32_t irq);
		17	void (*gic_write_dir)(uint32_t irq);
		18	void (*gic_set_eoi_split)(bool split);
		19	void (*gic_set_priority_mask)(uint64_t mask);
		20	void (*gic_set_priority)(uint32_t intid, uint32_t prio);
		21	void (*gic_irq_set_active)(uint32_t intid);
		22	void (*gic_irq_clear_active)(uint32_t intid);
		23	bool (*gic_irq_get_active)(uint32_t intid);
		24	void (*gic_irq_set_pending)(uint32_t intid);
		25	void (*gic_irq_clear_pending)(uint32_t intid);
		26	bool (*gic_irq_get_pending)(uint32_t intid);
		27	void (*gic_irq_set_config)(uint32_t intid, bool is_edge);
17	};	28	};
18		29
19	extern const struct gic_common_ops gicv3_ops;	30	extern const struct gic_common_ops gicv3_ops;

tools/testing/selftests/kvm/lib/aarch64/gic_v3.c+181−25

@@ -19,7 +19,8 @@ struct gicv3_data { · 18 unmodified lines
19	unsigned int nr_spis;	19	unsigned int nr_spis;
20	};	20	};
21		21
22	#define sgi_base_from_redist(redist_base) (redist_base + SZ_64K)	22	#define sgi_base_from_redist(redist_base) (redist_base + SZ_64K)
		23	#define DIST_BIT (1U << 31)
23		24
24	enum gicv3_intid_range {	25	enum gicv3_intid_range {
25	SGI_RANGE,	26	SGI_RANGE,
@@ -50,6 +51,14 @@ static void gicv3_gicr_wait_for_rwp(void *redist_base) · 24 unmodified lines
50	}	51	}
51	}	52	}
52		53
		54	static void gicv3_wait_for_rwp(uint32_t cpu_or_dist)
		55	{
		56	if (cpu_or_dist & DIST_BIT)
		57	gicv3_gicd_wait_for_rwp();
		58	else
		59	gicv3_gicr_wait_for_rwp(gicv3_data.redist_base[cpu_or_dist]);
		60	}
		61
53	static enum gicv3_intid_range get_intid_range(unsigned int intid)	62	static enum gicv3_intid_range get_intid_range(unsigned int intid)
54	{	63	{
55	switch (intid) {	64	switch (intid) {
@@ -81,39 +90,175 @@ static void gicv3_write_eoir(uint32_t irq) · 25 unmodified lines
81	isb();	90	isb();
82	}	91	}
83		92
84	static void	93	static void gicv3_write_dir(uint32_t irq)
85	gicv3_config_irq(unsigned int intid, unsigned int offset)	94	{
		95	write_sysreg_s(irq, SYS_ICC_DIR_EL1);
		96	isb();
		97	}
		98
		99	static void gicv3_set_priority_mask(uint64_t mask)
		100	{
		101	write_sysreg_s(mask, SYS_ICC_PMR_EL1);
		102	}
		103
		104	static void gicv3_set_eoi_split(bool split)
		105	{
		106	uint32_t val;
		107
		108	/* All other fields are read-only, so no need to read CTLR first. In
		109	* fact, the kernel does the same.
		110	*/
		111	val = split ? (1U << 1) : 0;
		112	write_sysreg_s(val, SYS_ICC_CTLR_EL1);
		113	isb();
		114	}
		115
		116	uint32_t gicv3_reg_readl(uint32_t cpu_or_dist, uint64_t offset)
		117	{
		118	void *base = cpu_or_dist & DIST_BIT ? gicv3_data.dist_base
		119	: sgi_base_from_redist(gicv3_data.redist_base[cpu_or_dist]);
		120	return readl(base + offset);
		121	}
		122
		123	void gicv3_reg_writel(uint32_t cpu_or_dist, uint64_t offset, uint32_t reg_val)
		124	{
		125	void *base = cpu_or_dist & DIST_BIT ? gicv3_data.dist_base
		126	: sgi_base_from_redist(gicv3_data.redist_base[cpu_or_dist]);
		127	writel(reg_val, base + offset);
		128	}
		129
		130	uint32_t gicv3_getl_fields(uint32_t cpu_or_dist, uint64_t offset, uint32_t mask)
		131	{
		132	return gicv3_reg_readl(cpu_or_dist, offset) & mask;
		133	}
		134
		135	void gicv3_setl_fields(uint32_t cpu_or_dist, uint64_t offset,
		136	uint32_t mask, uint32_t reg_val)
		137	{
		138	uint32_t tmp = gicv3_reg_readl(cpu_or_dist, offset) & ~mask;
		139
		140	tmp \|= (reg_val & mask);
		141	gicv3_reg_writel(cpu_or_dist, offset, tmp);
		142	}
		143
		144	/*
		145	* We use a single offset for the distributor and redistributor maps as they
		146	* have the same value in both. The only exceptions are registers that only
		147	* exist in one and not the other, like GICR_WAKER that doesn't exist in the
		148	* distributor map. Such registers are conveniently marked as reserved in the
		149	* map that doesn't implement it; like GICR_WAKER's offset of 0x0014 being
		150	* marked as "Reserved" in the Distributor map.
		151	*/
		152	static void gicv3_access_reg(uint32_t intid, uint64_t offset,
		153	uint32_t reg_bits, uint32_t bits_per_field,
		154	bool write, uint32_t *val)
86	{	155	{
87	uint32_t cpu = guest_get_vcpuid();	156	uint32_t cpu = guest_get_vcpuid();
88	uint32_t mask = 1 << (intid % 32);
89	enum gicv3_intid_range intid_range = get_intid_range(intid);	157	enum gicv3_intid_range intid_range = get_intid_range(intid);
90	void *reg;	158	uint32_t fields_per_reg, index, mask, shift;
91		159	uint32_t cpu_or_dist;
92	/* We care about 'cpu' only for SGIs or PPIs */	160
93	if (intid_range == SGI_RANGE \|\| intid_range == PPI_RANGE) {	161	GUEST_ASSERT(bits_per_field <= reg_bits);
94	GUEST_ASSERT(cpu < gicv3_data.nr_cpus);	162	GUEST_ASSERT(*val < (1U << bits_per_field));
95		163	/* Some registers like IROUTER are 64 bit long. Those are currently not
96	reg = sgi_base_from_redist(gicv3_data.redist_base[cpu]) +	164	* supported by readl nor writel, so just asserting here until then.
97	offset;	165	*/
98	writel(mask, reg);	166	GUEST_ASSERT(reg_bits == 32);
99	gicv3_gicr_wait_for_rwp(gicv3_data.redist_base[cpu]);	167
100	} else if (intid_range == SPI_RANGE) {	168	fields_per_reg = reg_bits / bits_per_field;
101	reg = gicv3_data.dist_base + offset + (intid / 32) * 4;	169	index = intid % fields_per_reg;
102	writel(mask, reg);	170	shift = index * bits_per_field;
103	gicv3_gicd_wait_for_rwp();	171	mask = ((1U << bits_per_field) - 1) << shift;
104	} else {	172
105	GUEST_ASSERT(0);	173	/* Set offset to the actual register holding intid's config. */
106	}	174	offset += (intid / fields_per_reg) * (reg_bits / 8);
		175
		176	cpu_or_dist = (intid_range == SPI_RANGE) ? DIST_BIT : cpu;
		177
		178	if (write)
		179	gicv3_setl_fields(cpu_or_dist, offset, mask, *val << shift);
		180	*val = gicv3_getl_fields(cpu_or_dist, offset, mask) >> shift;
		181	}
		182
		183	static void gicv3_write_reg(uint32_t intid, uint64_t offset,
		184	uint32_t reg_bits, uint32_t bits_per_field, uint32_t val)
		185	{
		186	gicv3_access_reg(intid, offset, reg_bits,
		187	bits_per_field, true, &val);
		188	}
		189
		190	static uint32_t gicv3_read_reg(uint32_t intid, uint64_t offset,
		191	uint32_t reg_bits, uint32_t bits_per_field)
		192	{
		193	uint32_t val;
		194
		195	gicv3_access_reg(intid, offset, reg_bits,
		196	bits_per_field, false, &val);
		197	return val;
		198	}
		199
		200	static void gicv3_set_priority(uint32_t intid, uint32_t prio)
		201	{
		202	gicv3_write_reg(intid, GICD_IPRIORITYR, 32, 8, prio);
		203	}
		204
		205	/* Sets the intid to be level-sensitive or edge-triggered. */
		206	static void gicv3_irq_set_config(uint32_t intid, bool is_edge)
		207	{
		208	uint32_t val;
		209
		210	/* N/A for private interrupts. */
		211	GUEST_ASSERT(get_intid_range(intid) == SPI_RANGE);
		212	val = is_edge ? 2 : 0;
		213	gicv3_write_reg(intid, GICD_ICFGR, 32, 2, val);
		214	}
		215
		216	static void gicv3_irq_enable(uint32_t intid)
		217	{
		218	bool is_spi = get_intid_range(intid) == SPI_RANGE;
		219	uint32_t cpu = guest_get_vcpuid();
		220
		221	gicv3_write_reg(intid, GICD_ISENABLER, 32, 1, 1);
		222	gicv3_wait_for_rwp(is_spi ? DIST_BIT : cpu);
		223	}
		224
		225	static void gicv3_irq_disable(uint32_t intid)
		226	{
		227	bool is_spi = get_intid_range(intid) == SPI_RANGE;
		228	uint32_t cpu = guest_get_vcpuid();
		229
		230	gicv3_write_reg(intid, GICD_ICENABLER, 32, 1, 1);
		231	gicv3_wait_for_rwp(is_spi ? DIST_BIT : cpu);
		232	}
		233
		234	static void gicv3_irq_set_active(uint32_t intid)
		235	{
		236	gicv3_write_reg(intid, GICD_ISACTIVER, 32, 1, 1);
		237	}
		238
		239	static void gicv3_irq_clear_active(uint32_t intid)
		240	{
		241	gicv3_write_reg(intid, GICD_ICACTIVER, 32, 1, 1);
		242	}
		243
		244	static bool gicv3_irq_get_active(uint32_t intid)
		245	{
		246	return gicv3_read_reg(intid, GICD_ISACTIVER, 32, 1);
		247	}
		248
		249	static void gicv3_irq_set_pending(uint32_t intid)
		250	{
		251	gicv3_write_reg(intid, GICD_ISPENDR, 32, 1, 1);
107	}	252	}
108		253
109	static void gicv3_irq_enable(unsigned int intid)	254	static void gicv3_irq_clear_pending(uint32_t intid)
110	{	255	{
111	gicv3_config_irq(intid, GICD_ISENABLER);	256	gicv3_write_reg(intid, GICD_ICPENDR, 32, 1, 1);
112	}	257	}
113		258
114	static void gicv3_irq_disable(unsigned int intid)	259	static bool gicv3_irq_get_pending(uint32_t intid)
115	{	260	{
116	gicv3_config_irq(intid, GICD_ICENABLER);	261	return gicv3_read_reg(intid, GICD_ISPENDR, 32, 1);
117	}	262	}
118		263
119	static void gicv3_enable_redist(void *redist_base)	264	static void gicv3_enable_redist(void *redist_base)
@@ -237,4 +382,15 @@ const struct gic_common_ops gicv3_ops = { · 117 unmodified lines
237	.gic_irq_disable = gicv3_irq_disable,	382	.gic_irq_disable = gicv3_irq_disable,
238	.gic_read_iar = gicv3_read_iar,	383	.gic_read_iar = gicv3_read_iar,
239	.gic_write_eoir = gicv3_write_eoir,	384	.gic_write_eoir = gicv3_write_eoir,
		385	.gic_write_dir = gicv3_write_dir,
		386	.gic_set_priority_mask = gicv3_set_priority_mask,
		387	.gic_set_eoi_split = gicv3_set_eoi_split,
		388	.gic_set_priority = gicv3_set_priority,
		389	.gic_irq_set_active = gicv3_irq_set_active,
		390	.gic_irq_clear_active = gicv3_irq_clear_active,
		391	.gic_irq_get_active = gicv3_irq_get_active,
		392	.gic_irq_set_pending = gicv3_irq_set_pending,
		393	.gic_irq_clear_pending = gicv3_irq_clear_pending,
		394	.gic_irq_get_pending = gicv3_irq_get_pending,
		395	.gic_irq_set_config = gicv3_irq_set_config,
240	};	396	};

tools/testing/selftests/kvm/lib/aarch64/processor.c+75−7

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	#include <linux/compiler.h>	8	#include <linux/compiler.h>
9	#include <assert.h>	9	#include <assert.h>
10		10
		11	#include "guest_modes.h"
11	#include "kvm_util.h"	12	#include "kvm_util.h"
12	#include "../kvm_util_internal.h"	13	#include "../kvm_util_internal.h"
13	#include "processor.h"	14	#include "processor.h"
@@ -237,6 +238,7 @@ void aarch64_vcpu_setup(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_vcpu_init · 223 unmodified lines
237	get_reg(vm, vcpuid, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), &sctlr_el1);	238	get_reg(vm, vcpuid, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), &sctlr_el1);
238	get_reg(vm, vcpuid, KVM_ARM64_SYS_REG(SYS_TCR_EL1), &tcr_el1);	239	get_reg(vm, vcpuid, KVM_ARM64_SYS_REG(SYS_TCR_EL1), &tcr_el1);
239		240
		241	/* Configure base granule size */
240	switch (vm->mode) {	242	switch (vm->mode) {
241	case VM_MODE_P52V48_4K:	243	case VM_MODE_P52V48_4K:
242	TEST_FAIL("AArch64 does not support 4K sized pages "	244	TEST_FAIL("AArch64 does not support 4K sized pages "
@@ -245,25 +247,47 @@ void aarch64_vcpu_setup(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_vcpu_init · 2 unmodified lines
245	TEST_FAIL("AArch64 does not support 4K sized pages "	247	TEST_FAIL("AArch64 does not support 4K sized pages "
246	"with ANY-bit physical address ranges");	248	"with ANY-bit physical address ranges");
247	case VM_MODE_P52V48_64K:	249	case VM_MODE_P52V48_64K:
		250	case VM_MODE_P48V48_64K:
		251	case VM_MODE_P40V48_64K:
		252	case VM_MODE_P36V48_64K:
248	tcr_el1 \|= 1ul << 14; /* TG0 = 64KB */	253	tcr_el1 \|= 1ul << 14; /* TG0 = 64KB */
249	tcr_el1 \|= 6ul << 32; /* IPS = 52 bits */	254	break;
		255	case VM_MODE_P48V48_16K:
		256	case VM_MODE_P40V48_16K:
		257	case VM_MODE_P36V48_16K:
		258	case VM_MODE_P36V47_16K:
		259	tcr_el1 \|= 2ul << 14; /* TG0 = 16KB */
250	break;	260	break;
251	case VM_MODE_P48V48_4K:	261	case VM_MODE_P48V48_4K:
		262	case VM_MODE_P40V48_4K:
		263	case VM_MODE_P36V48_4K:
252	tcr_el1 \|= 0ul << 14; /* TG0 = 4KB */	264	tcr_el1 \|= 0ul << 14; /* TG0 = 4KB */
253	tcr_el1 \|= 5ul << 32; /* IPS = 48 bits */
254	break;	265	break;
		266	default:
		267	TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);
		268	}
		269
		270	/* Configure output size */
		271	switch (vm->mode) {
		272	case VM_MODE_P52V48_64K:
		273	tcr_el1 \|= 6ul << 32; /* IPS = 52 bits */
		274	break;
		275	case VM_MODE_P48V48_4K:
		276	case VM_MODE_P48V48_16K:
255	case VM_MODE_P48V48_64K:	277	case VM_MODE_P48V48_64K:
256	tcr_el1 \|= 1ul << 14; /* TG0 = 64KB */
257	tcr_el1 \|= 5ul << 32; /* IPS = 48 bits */	278	tcr_el1 \|= 5ul << 32; /* IPS = 48 bits */
258	break;	279	break;
259	case VM_MODE_P40V48_4K:	280	case VM_MODE_P40V48_4K:
260	tcr_el1 \|= 0ul << 14; /* TG0 = 4KB */	281	case VM_MODE_P40V48_16K:
261	tcr_el1 \|= 2ul << 32; /* IPS = 40 bits */
262	break;
263	case VM_MODE_P40V48_64K:	282	case VM_MODE_P40V48_64K:
264	tcr_el1 \|= 1ul << 14; /* TG0 = 64KB */
265	tcr_el1 \|= 2ul << 32; /* IPS = 40 bits */	283	tcr_el1 \|= 2ul << 32; /* IPS = 40 bits */
266	break;	284	break;
		285	case VM_MODE_P36V48_4K:
		286	case VM_MODE_P36V48_16K:
		287	case VM_MODE_P36V48_64K:
		288	case VM_MODE_P36V47_16K:
		289	tcr_el1 \|= 1ul << 32; /* IPS = 36 bits */
		290	break;
267	default:	291	default:
268	TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);	292	TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);
269	}	293	}
@@ -432,3 +456,47 @@ uint32_t guest_get_vcpuid(void) · 162 unmodified lines
432	{	456	{
433	return read_sysreg(tpidr_el1);	457	return read_sysreg(tpidr_el1);
434	}	458	}
		459
		460	void aarch64_get_supported_page_sizes(uint32_t ipa,
		461	bool ps4k, bool ps16k, bool *ps64k)
		462	{
		463	struct kvm_vcpu_init preferred_init;
		464	int kvm_fd, vm_fd, vcpu_fd, err;
		465	uint64_t val;
		466	struct kvm_one_reg reg = {
		467	.id = KVM_ARM64_SYS_REG(SYS_ID_AA64MMFR0_EL1),
		468	.addr = (uint64_t)&val,
		469	};
		470
		471	kvm_fd = open_kvm_dev_path_or_exit();
		472	vm_fd = ioctl(kvm_fd, KVM_CREATE_VM, ipa);
		473	TEST_ASSERT(vm_fd >= 0, "Can't create VM");
		474
		475	vcpu_fd = ioctl(vm_fd, KVM_CREATE_VCPU, 0);
		476	TEST_ASSERT(vcpu_fd >= 0, "Can't create vcpu");
		477
		478	err = ioctl(vm_fd, KVM_ARM_PREFERRED_TARGET, &preferred_init);
		479	TEST_ASSERT(err == 0, "Can't get target");
		480	err = ioctl(vcpu_fd, KVM_ARM_VCPU_INIT, &preferred_init);
		481	TEST_ASSERT(err == 0, "Can't get init vcpu");
		482
		483	err = ioctl(vcpu_fd, KVM_GET_ONE_REG, &reg);
		484	TEST_ASSERT(err == 0, "Can't get MMFR0");
		485
		486	*ps4k = ((val >> 28) & 0xf) != 0xf;
		487	*ps64k = ((val >> 24) & 0xf) == 0;
		488	*ps16k = ((val >> 20) & 0xf) != 0;
		489
		490	close(vcpu_fd);
		491	close(vm_fd);
		492	close(kvm_fd);
		493	}
		494
		495	/*
		496	* arm64 doesn't have a true default mode, so start by computing the
		497	* available IPA space and page sizes early.
		498	*/
		499	void __attribute__((constructor)) init_guest_modes(void)
		500	{
		501	guest_modes_append_default();
		502	}

tools/testing/selftests/kvm/lib/aarch64/vgic.c+102−1

@@ -5,11 +5,14 @@ · 4 unmodified lines
5		5
6	#include <linux/kvm.h>	6	#include <linux/kvm.h>
7	#include <linux/sizes.h>	7	#include <linux/sizes.h>
		8	#include <asm/kvm_para.h>
8	#include <asm/kvm.h>	9	#include <asm/kvm.h>
9		10
10	#include "kvm_util.h"	11	#include "kvm_util.h"
11	#include "../kvm_util_internal.h"	12	#include "../kvm_util_internal.h"
12	#include "vgic.h"	13	#include "vgic.h"
		14	#include "gic.h"
		15	#include "gic_v3.h"
13		16
14	/*	17	/*
15	* vGIC-v3 default host setup	18	* vGIC-v3 default host setup
@@ -28,7 +31,7 @@ · 12 unmodified lines
28	* redistributor regions of the guest. Since it depends on the number of	31	* redistributor regions of the guest. Since it depends on the number of
29	* vCPUs for the VM, it must be called after all the vCPUs have been created.	32	* vCPUs for the VM, it must be called after all the vCPUs have been created.
30	*/	33	*/
31	int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus,	34	int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs,
32	uint64_t gicd_base_gpa, uint64_t gicr_base_gpa)	35	uint64_t gicd_base_gpa, uint64_t gicr_base_gpa)
33	{	36	{
34	int gic_fd;	37	int gic_fd;
@@ -50,6 +53,13 @@ int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, · 15 unmodified lines
50		53
51	/* Distributor setup */	54	/* Distributor setup */
52	gic_fd = kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_V3, false);	55	gic_fd = kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_V3, false);
		56
		57	kvm_device_access(gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS,
		58	0, &nr_irqs, true);
		59
		60	kvm_device_access(gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
		61	KVM_DEV_ARM_VGIC_CTRL_INIT, NULL, true);
		62
53	kvm_device_access(gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR,	63	kvm_device_access(gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR,
54	KVM_VGIC_V3_ADDR_TYPE_DIST, &gicd_base_gpa, true);	64	KVM_VGIC_V3_ADDR_TYPE_DIST, &gicd_base_gpa, true);
55	nr_gic_pages = vm_calc_num_guest_pages(vm->mode, KVM_VGIC_V3_DIST_SIZE);	65	nr_gic_pages = vm_calc_num_guest_pages(vm->mode, KVM_VGIC_V3_DIST_SIZE);
@@ -68,3 +78,94 @@ int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, · 12 unmodified lines
68		78
69	return gic_fd;	79	return gic_fd;
70	}	80	}
		81
		82	/* should only work for level sensitive interrupts */
		83	int _kvm_irq_set_level_info(int gic_fd, uint32_t intid, int level)
		84	{
		85	uint64_t attr = 32 * (intid / 32);
		86	uint64_t index = intid % 32;
		87	uint64_t val;
		88	int ret;
		89
		90	ret = _kvm_device_access(gic_fd, KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO,
		91	attr, &val, false);
		92	if (ret != 0)
		93	return ret;
		94
		95	val \|= 1U << index;
		96	ret = _kvm_device_access(gic_fd, KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO,
		97	attr, &val, true);
		98	return ret;
		99	}
		100
		101	void kvm_irq_set_level_info(int gic_fd, uint32_t intid, int level)
		102	{
		103	int ret = _kvm_irq_set_level_info(gic_fd, intid, level);
		104
		105	TEST_ASSERT(ret == 0, "KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO failed, "
		106	"rc: %i errno: %i", ret, errno);
		107	}
		108
		109	int _kvm_arm_irq_line(struct kvm_vm *vm, uint32_t intid, int level)
		110	{
		111	uint32_t irq = intid & KVM_ARM_IRQ_NUM_MASK;
		112
		113	TEST_ASSERT(!INTID_IS_SGI(intid), "KVM_IRQ_LINE's interface itself "
		114	"doesn't allow injecting SGIs. There's no mask for it.");
		115
		116	if (INTID_IS_PPI(intid))
		117	irq \|= KVM_ARM_IRQ_TYPE_PPI << KVM_ARM_IRQ_TYPE_SHIFT;
		118	else
		119	irq \|= KVM_ARM_IRQ_TYPE_SPI << KVM_ARM_IRQ_TYPE_SHIFT;
		120
		121	return _kvm_irq_line(vm, irq, level);
		122	}
		123
		124	void kvm_arm_irq_line(struct kvm_vm *vm, uint32_t intid, int level)
		125	{
		126	int ret = _kvm_arm_irq_line(vm, intid, level);
		127
		128	TEST_ASSERT(ret == 0, "KVM_IRQ_LINE failed, rc: %i errno: %i",
		129	ret, errno);
		130	}
		131
		132	static void vgic_poke_irq(int gic_fd, uint32_t intid,
		133	uint32_t vcpu, uint64_t reg_off)
		134	{
		135	uint64_t reg = intid / 32;
		136	uint64_t index = intid % 32;
		137	uint64_t attr = reg_off + reg * 4;
		138	uint64_t val;
		139	bool intid_is_private = INTID_IS_SGI(intid) \|\| INTID_IS_PPI(intid);
		140
		141	/* Check that the addr part of the attr is within 32 bits. */
		142	assert(attr <= KVM_DEV_ARM_VGIC_OFFSET_MASK);
		143
		144	uint32_t group = intid_is_private ? KVM_DEV_ARM_VGIC_GRP_REDIST_REGS
		145	: KVM_DEV_ARM_VGIC_GRP_DIST_REGS;
		146
		147	if (intid_is_private) {
		148	/* TODO: only vcpu 0 implemented for now. */
		149	assert(vcpu == 0);
		150	attr += SZ_64K;
		151	}
		152
		153	/* All calls will succeed, even with invalid intid's, as long as the
		154	* addr part of the attr is within 32 bits (checked above). An invalid
		155	* intid will just make the read/writes point to above the intended
		156	* register space (i.e., ICPENDR after ISPENDR).
		157	*/
		158	kvm_device_access(gic_fd, group, attr, &val, false);
		159	val \|= 1ULL << index;
		160	kvm_device_access(gic_fd, group, attr, &val, true);
		161	}
		162
		163	void kvm_irq_write_ispendr(int gic_fd, uint32_t intid, uint32_t vcpu)
		164	{
		165	vgic_poke_irq(gic_fd, intid, vcpu, GICD_ISPENDR);
		166	}
		167
		168	void kvm_irq_write_isactiver(int gic_fd, uint32_t intid, uint32_t vcpu)
		169	{
		170	vgic_poke_irq(gic_fd, intid, vcpu, GICD_ISACTIVER);
		171	}

tools/testing/selftests/kvm/lib/guest_modes.c+43−6

@@ -4,22 +4,59 @@ · 3 unmodified lines
4	*/	4	*/
5	#include "guest_modes.h"	5	#include "guest_modes.h"
6		6
		7	#ifdef __aarch64__
		8	#include "processor.h"
		9	enum vm_guest_mode vm_mode_default;
		10	#endif
		11
7	struct guest_mode guest_modes[NUM_VM_MODES];	12	struct guest_mode guest_modes[NUM_VM_MODES];
8		13
9	void guest_modes_append_default(void)	14	void guest_modes_append_default(void)
10	{	15	{
		16	#ifndef __aarch64__
11	guest_mode_append(VM_MODE_DEFAULT, true, true);	17	guest_mode_append(VM_MODE_DEFAULT, true, true);
12		18	#else
13	#ifdef __aarch64__
14	guest_mode_append(VM_MODE_P40V48_64K, true, true);
15	{	19	{
16	unsigned int limit = kvm_check_cap(KVM_CAP_ARM_VM_IPA_SIZE);	20	unsigned int limit = kvm_check_cap(KVM_CAP_ARM_VM_IPA_SIZE);
		21	bool ps4k, ps16k, ps64k;
		22	int i;
		23
		24	aarch64_get_supported_page_sizes(limit, &ps4k, &ps16k, &ps64k);
		25
		26	vm_mode_default = NUM_VM_MODES;
		27
17	if (limit >= 52)	28	if (limit >= 52)
18	guest_mode_append(VM_MODE_P52V48_64K, true, true);	29	guest_mode_append(VM_MODE_P52V48_64K, ps64k, ps64k);
19	if (limit >= 48) {	30	if (limit >= 48) {
20	guest_mode_append(VM_MODE_P48V48_4K, true, true);	31	guest_mode_append(VM_MODE_P48V48_4K, ps4k, ps4k);
21	guest_mode_append(VM_MODE_P48V48_64K, true, true);	32	guest_mode_append(VM_MODE_P48V48_16K, ps16k, ps16k);
		33	guest_mode_append(VM_MODE_P48V48_64K, ps64k, ps64k);
		34	}
		35	if (limit >= 40) {
		36	guest_mode_append(VM_MODE_P40V48_4K, ps4k, ps4k);
		37	guest_mode_append(VM_MODE_P40V48_16K, ps16k, ps16k);
		38	guest_mode_append(VM_MODE_P40V48_64K, ps64k, ps64k);
		39	if (ps4k)
		40	vm_mode_default = VM_MODE_P40V48_4K;
		41	}
		42	if (limit >= 36) {
		43	guest_mode_append(VM_MODE_P36V48_4K, ps4k, ps4k);
		44	guest_mode_append(VM_MODE_P36V48_16K, ps16k, ps16k);
		45	guest_mode_append(VM_MODE_P36V48_64K, ps64k, ps64k);
		46	guest_mode_append(VM_MODE_P36V47_16K, ps16k, ps16k);
22	}	47	}
		48
		49	/*
		50	* Pick the first supported IPA size if the default
		51	* isn't available.
		52	*/
		53	for (i = 0; vm_mode_default == NUM_VM_MODES && i < NUM_VM_MODES; i++) {
		54	if (guest_modes[i].supported && guest_modes[i].enabled)
		55	vm_mode_default = i;
		56	}
		57
		58	TEST_ASSERT(vm_mode_default != NUM_VM_MODES,
		59	"No supported mode!");
23	}	60	}
24	#endif	61	#endif
25	#ifdef __s390x__	62	#ifdef __s390x__

tools/testing/selftests/kvm/lib/kvm_util.c+94−0

@@ -166,12 +166,18 @@ const char *vm_guest_mode_string(uint32_t i) · 165 unmodified lines
166	[VM_MODE_P52V48_4K] = "PA-bits:52, VA-bits:48, 4K pages",	166	[VM_MODE_P52V48_4K] = "PA-bits:52, VA-bits:48, 4K pages",
167	[VM_MODE_P52V48_64K] = "PA-bits:52, VA-bits:48, 64K pages",	167	[VM_MODE_P52V48_64K] = "PA-bits:52, VA-bits:48, 64K pages",
168	[VM_MODE_P48V48_4K] = "PA-bits:48, VA-bits:48, 4K pages",	168	[VM_MODE_P48V48_4K] = "PA-bits:48, VA-bits:48, 4K pages",
		169	[VM_MODE_P48V48_16K] = "PA-bits:48, VA-bits:48, 16K pages",
169	[VM_MODE_P48V48_64K] = "PA-bits:48, VA-bits:48, 64K pages",	170	[VM_MODE_P48V48_64K] = "PA-bits:48, VA-bits:48, 64K pages",
170	[VM_MODE_P40V48_4K] = "PA-bits:40, VA-bits:48, 4K pages",	171	[VM_MODE_P40V48_4K] = "PA-bits:40, VA-bits:48, 4K pages",
		172	[VM_MODE_P40V48_16K] = "PA-bits:40, VA-bits:48, 16K pages",
171	[VM_MODE_P40V48_64K] = "PA-bits:40, VA-bits:48, 64K pages",	173	[VM_MODE_P40V48_64K] = "PA-bits:40, VA-bits:48, 64K pages",
172	[VM_MODE_PXXV48_4K] = "PA-bits:ANY, VA-bits:48, 4K pages",	174	[VM_MODE_PXXV48_4K] = "PA-bits:ANY, VA-bits:48, 4K pages",
173	[VM_MODE_P47V64_4K] = "PA-bits:47, VA-bits:64, 4K pages",	175	[VM_MODE_P47V64_4K] = "PA-bits:47, VA-bits:64, 4K pages",
174	[VM_MODE_P44V64_4K] = "PA-bits:44, VA-bits:64, 4K pages",	176	[VM_MODE_P44V64_4K] = "PA-bits:44, VA-bits:64, 4K pages",
		177	[VM_MODE_P36V48_4K] = "PA-bits:36, VA-bits:48, 4K pages",
		178	[VM_MODE_P36V48_16K] = "PA-bits:36, VA-bits:48, 16K pages",
		179	[VM_MODE_P36V48_64K] = "PA-bits:36, VA-bits:48, 64K pages",
		180	[VM_MODE_P36V47_16K] = "PA-bits:36, VA-bits:47, 16K pages",
175	};	181	};
176	_Static_assert(sizeof(strings)/sizeof(char *) == NUM_VM_MODES,	182	_Static_assert(sizeof(strings)/sizeof(char *) == NUM_VM_MODES,
177	"Missing new mode strings?");	183	"Missing new mode strings?");
@@ -185,12 +191,18 @@ const struct vm_guest_mode_params vm_guest_mode_params[] = { · 7 unmodified lines
185	[VM_MODE_P52V48_4K] = { 52, 48, 0x1000, 12 },	191	[VM_MODE_P52V48_4K] = { 52, 48, 0x1000, 12 },
186	[VM_MODE_P52V48_64K] = { 52, 48, 0x10000, 16 },	192	[VM_MODE_P52V48_64K] = { 52, 48, 0x10000, 16 },
187	[VM_MODE_P48V48_4K] = { 48, 48, 0x1000, 12 },	193	[VM_MODE_P48V48_4K] = { 48, 48, 0x1000, 12 },
		194	[VM_MODE_P48V48_16K] = { 48, 48, 0x4000, 14 },
188	[VM_MODE_P48V48_64K] = { 48, 48, 0x10000, 16 },	195	[VM_MODE_P48V48_64K] = { 48, 48, 0x10000, 16 },
189	[VM_MODE_P40V48_4K] = { 40, 48, 0x1000, 12 },	196	[VM_MODE_P40V48_4K] = { 40, 48, 0x1000, 12 },
		197	[VM_MODE_P40V48_16K] = { 40, 48, 0x4000, 14 },
190	[VM_MODE_P40V48_64K] = { 40, 48, 0x10000, 16 },	198	[VM_MODE_P40V48_64K] = { 40, 48, 0x10000, 16 },
191	[VM_MODE_PXXV48_4K] = { 0, 0, 0x1000, 12 },	199	[VM_MODE_PXXV48_4K] = { 0, 0, 0x1000, 12 },
192	[VM_MODE_P47V64_4K] = { 47, 64, 0x1000, 12 },	200	[VM_MODE_P47V64_4K] = { 47, 64, 0x1000, 12 },
193	[VM_MODE_P44V64_4K] = { 44, 64, 0x1000, 12 },	201	[VM_MODE_P44V64_4K] = { 44, 64, 0x1000, 12 },
		202	[VM_MODE_P36V48_4K] = { 36, 48, 0x1000, 12 },
		203	[VM_MODE_P36V48_16K] = { 36, 48, 0x4000, 14 },
		204	[VM_MODE_P36V48_64K] = { 36, 48, 0x10000, 16 },
		205	[VM_MODE_P36V47_16K] = { 36, 47, 0x4000, 14 },
194	};	206	};
195	_Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES,	207	_Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES,
196	"Missing new mode params?");	208	"Missing new mode params?");
@@ -252,9 +264,19 @@ struct kvm_vm *vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm) · 55 unmodified lines
252	vm->pgtable_levels = 3;	264	vm->pgtable_levels = 3;
253	break;	265	break;
254	case VM_MODE_P40V48_4K:	266	case VM_MODE_P40V48_4K:
		267	case VM_MODE_P36V48_4K:
255	vm->pgtable_levels = 4;	268	vm->pgtable_levels = 4;
256	break;	269	break;
257	case VM_MODE_P40V48_64K:	270	case VM_MODE_P40V48_64K:
		271	case VM_MODE_P36V48_64K:
		272	vm->pgtable_levels = 3;
		273	break;
		274	case VM_MODE_P48V48_16K:
		275	case VM_MODE_P40V48_16K:
		276	case VM_MODE_P36V48_16K:
		277	vm->pgtable_levels = 4;
		278	break;
		279	case VM_MODE_P36V47_16K:
258	vm->pgtable_levels = 3;	280	vm->pgtable_levels = 3;
259	break;	281	break;
260	case VM_MODE_PXXV48_4K:	282	case VM_MODE_PXXV48_4K:
@@ -2086,6 +2108,78 @@ int vcpu_access_device_attr(struct kvm_vm *vm, uint32_t vcpuid, uint32_t group, · 1825 unmodified lines
2086	return ret;	2108	return ret;
2087	}	2109	}
2088		2110
		2111	/*
		2112	* IRQ related functions.
		2113	*/
		2114
		2115	int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level)
		2116	{
		2117	struct kvm_irq_level irq_level = {
		2118	.irq = irq,
		2119	.level = level,
		2120	};
		2121
		2122	return _vm_ioctl(vm, KVM_IRQ_LINE, &irq_level);
		2123	}
		2124
		2125	void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level)
		2126	{
		2127	int ret = _kvm_irq_line(vm, irq, level);
		2128
		2129	TEST_ASSERT(ret >= 0, "KVM_IRQ_LINE failed, rc: %i errno: %i", ret, errno);
		2130	}
		2131
		2132	struct kvm_irq_routing *kvm_gsi_routing_create(void)
		2133	{
		2134	struct kvm_irq_routing *routing;
		2135	size_t size;
		2136
		2137	size = sizeof(struct kvm_irq_routing);
		2138	/* Allocate space for the max number of entries: this wastes 196 KBs. */
		2139	size += KVM_MAX_IRQ_ROUTES * sizeof(struct kvm_irq_routing_entry);
		2140	routing = calloc(1, size);
		2141	assert(routing);
		2142
		2143	return routing;
		2144	}
		2145
		2146	void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing,
		2147	uint32_t gsi, uint32_t pin)
		2148	{
		2149	int i;
		2150
		2151	assert(routing);
		2152	assert(routing->nr < KVM_MAX_IRQ_ROUTES);
		2153
		2154	i = routing->nr;
		2155	routing->entries[i].gsi = gsi;
		2156	routing->entries[i].type = KVM_IRQ_ROUTING_IRQCHIP;
		2157	routing->entries[i].flags = 0;
		2158	routing->entries[i].u.irqchip.irqchip = 0;
		2159	routing->entries[i].u.irqchip.pin = pin;
		2160	routing->nr++;
		2161	}
		2162
		2163	int _kvm_gsi_routing_write(struct kvm_vm vm, struct kvm_irq_routing routing)
		2164	{
		2165	int ret;
		2166
		2167	assert(routing);
		2168	ret = ioctl(vm_get_fd(vm), KVM_SET_GSI_ROUTING, routing);
		2169	free(routing);
		2170
		2171	return ret;
		2172	}
		2173
		2174	void kvm_gsi_routing_write(struct kvm_vm vm, struct kvm_irq_routing routing)
		2175	{
		2176	int ret;
		2177
		2178	ret = _kvm_gsi_routing_write(vm, routing);
		2179	TEST_ASSERT(ret == 0, "KVM_SET_GSI_ROUTING failed, rc: %i errno: %i",
		2180	ret, errno);
		2181	}
		2182
2089	/*	2183	/*
2090	* VM Dump	2184	* VM Dump
2091	*	2185	*

Merge tag 'kvmarm-5.17' of git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into HEAD

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code

@@ -554,12 +554,12 @@ int vgic_v3_map_resources(struct kvm *kvm) · 553 unmodified lines
554	}	554	}
555		555
556	if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base)) {	556	if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base)) {
557	kvm_err("Need to set vgic distributor addresses first\n");	557	kvm_debug("Need to set vgic distributor addresses first\n");
558	return -ENXIO;	558	return -ENXIO;
559	}	559	}
560		560
561	if (!vgic_v3_check_base(kvm)) {	561	if (!vgic_v3_check_base(kvm)) {
562	kvm_err("VGIC redist and dist frames overlap\n");	562	kvm_debug("VGIC redist and dist frames overlap\n");
563	return -EINVAL;	563	return -EINVAL;
564	}	564	}
565		565
@@ -651,7 +651,7 @@ int vgic_v3_probe(const struct gic_kvm_info *info) · 85 unmodified lines
651	} else if (!PAGE_ALIGNED(info->vcpu.start)) {	651	} else if (!PAGE_ALIGNED(info->vcpu.start)) {
652	pr_warn("GICV physical address 0x%llx not page aligned\n",	652	pr_warn("GICV physical address 0x%llx not page aligned\n",
653	(unsigned long long)info->vcpu.start);	653	(unsigned long long)info->vcpu.start);
654	} else {	654	} else if (kvm_get_mode() != KVM_MODE_PROTECTED) {
655	kvm_vgic_global_state.vcpu_base = info->vcpu.start;	655	kvm_vgic_global_state.vcpu_base = info->vcpu.start;
656	kvm_vgic_global_state.can_emulate_gicv2 = true;	656	kvm_vgic_global_state.can_emulate_gicv2 = true;
657	ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V2);	657	ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V2);