lael-labs/linux

Documentation/virt/kvm/api.rst+66−4

@@ -7286,6 +7286,41 @@ exit, even without calls to ``KVM_ENABLE_CAP`` or similar. In this case, · 7285 unmodified lines
7286	it will enter with output fields already valid; in the common case, the	7286	it will enter with output fields already valid; in the common case, the
7287	``unknown.ret`` field of the union will be ``TDVMCALL_STATUS_SUBFUNC_UNSUPPORTED``.	7287	``unknown.ret`` field of the union will be ``TDVMCALL_STATUS_SUBFUNC_UNSUPPORTED``.
7288	Userspace need not do anything if it does not wish to support a TDVMCALL.	7288	Userspace need not do anything if it does not wish to support a TDVMCALL.
		7289
		7290	::
		7291
		7292	/* KVM_EXIT_ARM_SEA */
		7293	struct {
		7294	#define KVM_EXIT_ARM_SEA_FLAG_GPA_VALID (1ULL << 0)
		7295	__u64 flags;
		7296	__u64 esr;
		7297	__u64 gva;
		7298	__u64 gpa;
		7299	} arm_sea;
		7300
		7301	Used on arm64 systems. When the VM capability ``KVM_CAP_ARM_SEA_TO_USER`` is
		7302	enabled, a KVM exits to userspace if a guest access causes a synchronous
		7303	external abort (SEA) and the host APEI fails to handle the SEA.
		7304
		7305	``esr`` is set to a sanitized value of ESR_EL2 from the exception taken to KVM,
		7306	consisting of the following fields:
		7307
		7308	- ``ESR_EL2.EC``
		7309	- ``ESR_EL2.IL``
		7310	- ``ESR_EL2.FnV``
		7311	- ``ESR_EL2.EA``
		7312	- ``ESR_EL2.CM``
		7313	- ``ESR_EL2.WNR``
		7314	- ``ESR_EL2.FSC``
		7315	- ``ESR_EL2.SET`` (when FEAT_RAS is implemented for the VM)
		7316
		7317	``gva`` is set to the value of FAR_EL2 from the exception taken to KVM when
		7318	``ESR_EL2.FnV == 0``. Otherwise, the value of ``gva`` is unknown.
		7319
		7320	``gpa`` is set to the faulting IPA from the exception taken to KVM when
		7321	the ``KVM_EXIT_ARM_SEA_FLAG_GPA_VALID`` flag is set. Otherwise, the value of
		7322	``gpa`` is unknown.
		7323
7289	::	7324	::
7290		7325
7291	/* Fix the size of the union. */	7326	/* Fix the size of the union. */
@@ -7820,7 +7855,7 @@ where 0xff represents CPUs 0-7 in cluster 0. · 528 unmodified lines
7820	:Architectures: s390	7855	:Architectures: s390
7821	:Parameters: none	7856	:Parameters: none
7822		7857
7823	With this capability enabled, all illegal instructions 0x0000 (2 bytes) will	7858	With this capability enabled, the illegal instruction 0x0000 (2 bytes) will
7824	be intercepted and forwarded to user space. User space can use this	7859	be intercepted and forwarded to user space. User space can use this
7825	mechanism e.g. to realize 2-byte software breakpoints. The kernel will	7860	mechanism e.g. to realize 2-byte software breakpoints. The kernel will
7826	not inject an operating exception for these instructions, user space has	7861	not inject an operating exception for these instructions, user space has
@@ -8028,7 +8063,7 @@ will be initialized to 1 when created. This also improves performance because · 201 unmodified lines
8028	dirty logging can be enabled gradually in small chunks on the first call	8063	dirty logging can be enabled gradually in small chunks on the first call
8029	to KVM_CLEAR_DIRTY_LOG. KVM_DIRTY_LOG_INITIALLY_SET depends on	8064	to KVM_CLEAR_DIRTY_LOG. KVM_DIRTY_LOG_INITIALLY_SET depends on
8030	KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE (it is also only available on	8065	KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE (it is also only available on
8031	x86 and arm64 for now).	8066	x86, arm64 and riscv for now).
8032		8067
8033	KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 was previously available under the name	8068	KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 was previously available under the name
8034	KVM_CAP_MANUAL_DIRTY_LOG_PROTECT, but the implementation had bugs that make	8069	KVM_CAP_MANUAL_DIRTY_LOG_PROTECT, but the implementation had bugs that make
@@ -8524,7 +8559,7 @@ Therefore, the ioctl must be called before reading the content of · 489 unmodified lines
8524	the dirty pages.	8559	the dirty pages.
8525		8560
8526	The dirty ring can get full. When it happens, the KVM_RUN of the	8561	The dirty ring can get full. When it happens, the KVM_RUN of the
8527	vcpu will return with exit reason KVM_EXIT_DIRTY_LOG_FULL.	8562	vcpu will return with exit reason KVM_EXIT_DIRTY_RING_FULL.
8528		8563
8529	The dirty ring interface has a major difference comparing to the	8564	The dirty ring interface has a major difference comparing to the
8530	KVM_GET_DIRTY_LOG interface in that, when reading the dirty ring from	8565	KVM_GET_DIRTY_LOG interface in that, when reading the dirty ring from
@@ -8692,7 +8727,7 @@ given VM. · 161 unmodified lines
8692	When this capability is enabled, KVM resets the VCPU when setting	8727	When this capability is enabled, KVM resets the VCPU when setting
8693	MP_STATE_INIT_RECEIVED through IOCTL. The original MP_STATE is preserved.	8728	MP_STATE_INIT_RECEIVED through IOCTL. The original MP_STATE is preserved.
8694		8729
8695	7.43 KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED	8730	7.44 KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED
8696	-------------------------------------------	8731	-------------------------------------------
8697		8732
8698	:Architectures: arm64	8733	:Architectures: arm64
@@ -8703,6 +8738,33 @@ This capability indicate to the userspace whether a PFNMAP memory region · 4 unmodified lines
8703	can be safely mapped as cacheable. This relies on the presence of	8738	can be safely mapped as cacheable. This relies on the presence of
8704	force write back (FWB) feature support on the hardware.	8739	force write back (FWB) feature support on the hardware.
8705		8740
		8741	7.45 KVM_CAP_ARM_SEA_TO_USER
		8742	----------------------------
		8743
		8744	:Architecture: arm64
		8745	:Target: VM
		8746	:Parameters: none
		8747	:Returns: 0 on success, -EINVAL if unsupported.
		8748
		8749	When this capability is enabled, KVM may exit to userspace for SEAs taken to
		8750	EL2 resulting from a guest access. See ``KVM_EXIT_ARM_SEA`` for more
		8751	information.
		8752
		8753	7.46 KVM_CAP_S390_USER_OPEREXEC
		8754	-------------------------------
		8755
		8756	:Architectures: s390
		8757	:Parameters: none
		8758
		8759	When this capability is enabled KVM forwards all operation exceptions
		8760	that it doesn't handle itself to user space. This also includes the
		8761	0x0000 instructions managed by KVM_CAP_S390_USER_INSTR0. This is
		8762	helpful if user space wants to emulate instructions which are not
		8763	(yet) implemented in hardware.
		8764
		8765	This capability can be enabled dynamically even if VCPUs were already
		8766	created and are running.
		8767
8706	8. Other capabilities.	8768	8. Other capabilities.
8707	======================	8769	======================
8708		8770

Documentation/virt/kvm/x86/errata.rst+8−1

@@ -48,7 +48,14 @@ versus "has_error_code", i.e. KVM's ABI follows AMD behavior. · 47 unmodified lines
48	Nested virtualization features	48	Nested virtualization features
49	------------------------------	49	------------------------------
50		50
51	TBD	51	On AMD CPUs, when GIF is cleared, #DB exceptions or traps due to a breakpoint
		52	register match are ignored and discarded by the CPU. The CPU relies on the VMM
		53	to fully virtualize this behavior, even when vGIF is enabled for the guest
		54	(i.e. vGIF=0 does not cause the CPU to drop #DBs when the guest is running).
		55	KVM does not virtualize this behavior as the complexity is unjustified given
		56	the rarity of the use case. One way to handle this would be for KVM to
		57	intercept the #DB, temporarily disable the breakpoint, single-step over the
		58	instruction, then re-enable the breakpoint.
52		59
53	x2APIC	60	x2APIC
54	------	61	------

arch/arm64/include/asm/kvm_arm.h+1−0

@@ -111,6 +111,7 @@ · 110 unmodified lines
111	#define TCR_EL2_DS (1UL << 32)	111	#define TCR_EL2_DS (1UL << 32)
112	#define TCR_EL2_RES1 ((1U << 31) \| (1 << 23))	112	#define TCR_EL2_RES1 ((1U << 31) \| (1 << 23))
113	#define TCR_EL2_HPD (1 << 24)	113	#define TCR_EL2_HPD (1 << 24)
		114	#define TCR_EL2_HA (1 << 21)
114	#define TCR_EL2_TBI (1 << 20)	115	#define TCR_EL2_TBI (1 << 20)
115	#define TCR_EL2_PS_SHIFT 16	116	#define TCR_EL2_PS_SHIFT 16
116	#define TCR_EL2_PS_MASK (7 << TCR_EL2_PS_SHIFT)	117	#define TCR_EL2_PS_MASK (7 << TCR_EL2_PS_SHIFT)

arch/arm64/include/asm/kvm_asm.h+4−4

arch/arm64/include/asm/kvm_host.h+3−0

 #define KVM_REQ_NESTED_S2_UNMAP		KVM_ARCH_REQ(8)
 #define KVM_REQ_GUEST_HYP_IRQ_PENDING	KVM_ARCH_REQ(9)
 #define KVM_REQ_MAP_L1_VNCR_EL2		KVM_ARCH_REQ(10)
+#define KVM_REQ_VGIC_PROCESS_UPDATE	KVM_ARCH_REQ(11)
 #define KVM_DIRTY_LOG_MANUAL_CAPS   (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \
 				     KVM_DIRTY_LOG_INITIALLY_SET)
 #define KVM_ARCH_FLAG_GUEST_HAS_SVE			9
 	/* MIDR_EL1, REVIDR_EL1, and AIDR_EL1 are writable from userspace */
 #define KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS		10
+	/* Unhandled SEAs are taken to userspace */
+#define KVM_ARCH_FLAG_EXIT_SEA				11
 	unsigned long flags;
 	/* VM-wide vCPU feature set */

arch/arm64/include/asm/kvm_hyp.h+2−1

 int __vgic_v2_perform_cpuif_access(struct kvm_vcpu *vcpu);
 u64 __gic_v3_get_lr(unsigned int lr);
+void __gic_v3_set_lr(u64 val, int lr);
 void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if);
 void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if);
 void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if);
 void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if);
-void __vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if);
+void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if);
 void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if);
 int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu);

arch/arm64/include/asm/kvm_nested.h+38−2

@@ -120,9 +120,42 @@ static inline bool kvm_s2_trans_writable(struct kvm_s2_trans *trans) · 119 unmodified lines
120	return trans->writable;	120	return trans->writable;
121	}	121	}
122		122
123	static inline bool kvm_s2_trans_executable(struct kvm_s2_trans *trans)	123	static inline bool kvm_has_xnx(struct kvm *kvm)
124	{	124	{
125	return !(trans->desc & BIT(54));	125	return cpus_have_final_cap(ARM64_HAS_XNX) &&
		126	kvm_has_feat(kvm, ID_AA64MMFR1_EL1, XNX, IMP);
		127	}
		128
		129	static inline bool kvm_s2_trans_exec_el0(struct kvm kvm, struct kvm_s2_trans trans)
		130	{
		131	u8 xn = FIELD_GET(KVM_PTE_LEAF_ATTR_HI_S2_XN, trans->desc);
		132
		133	if (!kvm_has_xnx(kvm))
		134	xn &= FIELD_PREP(KVM_PTE_LEAF_ATTR_HI_S2_XN, 0b10);
		135
		136	switch (xn) {
		137	case 0b00:
		138	case 0b01:
		139	return true;
		140	default:
		141	return false;
		142	}
		143	}
		144
		145	static inline bool kvm_s2_trans_exec_el1(struct kvm kvm, struct kvm_s2_trans trans)
		146	{
		147	u8 xn = FIELD_GET(KVM_PTE_LEAF_ATTR_HI_S2_XN, trans->desc);
		148
		149	if (!kvm_has_xnx(kvm))
		150	xn &= FIELD_PREP(KVM_PTE_LEAF_ATTR_HI_S2_XN, 0b10);
		151
		152	switch (xn) {
		153	case 0b00:
		154	case 0b11:
		155	return true;
		156	default:
		157	return false;
		158	}
126	}	159	}
127		160
128	extern int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,	161	extern int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
@@ -320,6 +353,7 @@ struct s1_walk_info { · 191 unmodified lines
320	bool be;	353	bool be;
321	bool s2;	354	bool s2;
322	bool pa52bit;	355	bool pa52bit;
		356	bool ha;
323	};	357	};
324		358
325	struct s1_walk_result {	359	struct s1_walk_result {
@@ -370,4 +404,6 @@ void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val); · 44 unmodified lines
370	(FIX_VNCR - __c); \	404	(FIX_VNCR - __c); \
371	})	405	})
372		406
		407	int __kvm_at_swap_desc(struct kvm *kvm, gpa_t ipa, u64 old, u64 new);
		408
373	#endif /* __ARM64_KVM_NESTED_H */	409	#endif /* __ARM64_KVM_NESTED_H */

arch/arm64/include/asm/kvm_pgtable.h+42−7

 #define KVM_PTE_LEAF_ATTR_HI_S1_XN	BIT(54)
-#define KVM_PTE_LEAF_ATTR_HI_S2_XN	BIT(54)
+#define KVM_PTE_LEAF_ATTR_HI_S2_XN	GENMASK(54, 53)
 #define KVM_PTE_LEAF_ATTR_HI_S1_GP	BIT(50)
 /**
  * enum kvm_pgtable_prot - Page-table permissions and attributes.
- * @KVM_PGTABLE_PROT_X:		Execute permission.
+ * @KVM_PGTABLE_PROT_UX:	Unprivileged execute permission.
+ * @KVM_PGTABLE_PROT_PX:	Privileged execute permission.
+ * @KVM_PGTABLE_PROT_X:		Privileged and unprivileged execute permission.
  * @KVM_PGTABLE_PROT_W:		Write permission.
  * @KVM_PGTABLE_PROT_R:		Read permission.
  * @KVM_PGTABLE_PROT_DEVICE:	Device attributes.
  * @KVM_PGTABLE_PROT_SW3:	Software bit 3.
  */
 enum kvm_pgtable_prot {
-	KVM_PGTABLE_PROT_X			= BIT(0),
+	KVM_PGTABLE_PROT_PX			= BIT(0),
-	KVM_PGTABLE_PROT_W			= BIT(1),
+	KVM_PGTABLE_PROT_UX			= BIT(1),
-	KVM_PGTABLE_PROT_R			= BIT(2),
+	KVM_PGTABLE_PROT_X			= KVM_PGTABLE_PROT_PX	|
+						  KVM_PGTABLE_PROT_UX,
+	KVM_PGTABLE_PROT_W			= BIT(2),
+	KVM_PGTABLE_PROT_R			= BIT(3),
-	KVM_PGTABLE_PROT_DEVICE			= BIT(3),
+	KVM_PGTABLE_PROT_DEVICE			= BIT(4),
-	KVM_PGTABLE_PROT_NORMAL_NC		= BIT(4),
+	KVM_PGTABLE_PROT_NORMAL_NC		= BIT(5),
 	KVM_PGTABLE_PROT_SW0			= BIT(55),
 	KVM_PGTABLE_PROT_SW1			= BIT(56),
 	return pteref;
 }
+static inline kvm_pte_t *kvm_dereference_pteref_raw(kvm_pteref_t pteref)
+{
+	return pteref;
+}
 static inline int kvm_pgtable_walk_begin(struct kvm_pgtable_walker *walker)
 {
 	/*
 	return rcu_dereference_check(pteref, !(walker->flags & KVM_PGTABLE_WALK_SHARED));
 }
+static inline kvm_pte_t *kvm_dereference_pteref_raw(kvm_pteref_t pteref)
+{
+	return rcu_dereference_raw(pteref);
+}
 static inline int kvm_pgtable_walk_begin(struct kvm_pgtable_walker *walker)
 {
 	if (walker->flags & KVM_PGTABLE_WALK_SHARED)
  */
 void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt);
+/**
+ * kvm_pgtable_stage2_destroy_range() - Destroy the unlinked range of addresses.
+ * @pgt:	Page-table structure initialised by kvm_pgtable_stage2_init*().
+ * @addr:      Intermediate physical address at which to place the mapping.
+ * @size:      Size of the mapping.
+ *
+ * The page-table is assumed to be unreachable by any hardware walkers prior
+ * to freeing and therefore no TLB invalidation is performed.
+ */
+void kvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt,
+					u64 addr, u64 size);
+/**
+ * kvm_pgtable_stage2_destroy_pgd() - Destroy the PGD of guest stage-2 page-table.
+ * @pgt:       Page-table structure initialised by kvm_pgtable_stage2_init*().
+ *
+ * It is assumed that the rest of the page-table is freed before this operation.
+ */
+void kvm_pgtable_stage2_destroy_pgd(struct kvm_pgtable *pgt);
 /**
  * kvm_pgtable_stage2_free_unlinked() - Free an unlinked stage-2 paging structure.
  * @mm_ops:	Memory management callbacks.

arch/arm64/include/asm/kvm_pkvm.h+3−1

 int pkvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,
 			     struct kvm_pgtable_mm_ops *mm_ops);
-void pkvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt);
+void pkvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt,
+					u64 addr, u64 size);
+void pkvm_pgtable_stage2_destroy_pgd(struct kvm_pgtable *pgt);
 int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,
 			    enum kvm_pgtable_prot prot, void *mc,
 			    enum kvm_pgtable_walk_flags flags);

arch/arm64/include/asm/virt.h+6−1

  */
 #define HVC_FINALISE_EL2	3
+/*
+ * HVC_GET_ICH_VTR_EL2 - Retrieve the ICH_VTR_EL2 value
+ */
+#define HVC_GET_ICH_VTR_EL2	4
 /* Max number of HYP stub hypercalls */
-#define HVC_STUB_HCALL_NR 4
+#define HVC_STUB_HCALL_NR 5
 /* Error returned when an invalid stub number is passed into x0 */
 #define HVC_STUB_ERR	0xbadca11

arch/arm64/kernel/cpufeature.c+59−0

@@ -2304,6 +2304,49 @@ static bool has_gic_prio_relaxed_sync(const struct arm64_cpu_capabilities *entry · 2303 unmodified lines
2304	}	2304	}
2305	#endif	2305	#endif
2306		2306
		2307	static bool can_trap_icv_dir_el1(const struct arm64_cpu_capabilities *entry,
		2308	int scope)
		2309	{
		2310	static const struct midr_range has_vgic_v3[] = {
		2311	MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM),
		2312	MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM),
		2313	MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM_PRO),
		2314	MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM_PRO),
		2315	MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM_MAX),
		2316	MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM_MAX),
		2317	MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD),
		2318	MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE),
		2319	MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD_PRO),
		2320	MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE_PRO),
		2321	MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD_MAX),
		2322	MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE_MAX),
		2323	{},
		2324	};
		2325	struct arm_smccc_res res = {};
		2326
		2327	BUILD_BUG_ON(ARM64_HAS_ICH_HCR_EL2_TDIR <= ARM64_HAS_GICV3_CPUIF);
		2328	BUILD_BUG_ON(ARM64_HAS_ICH_HCR_EL2_TDIR <= ARM64_HAS_GICV5_LEGACY);
		2329	if (!this_cpu_has_cap(ARM64_HAS_GICV3_CPUIF) &&
		2330	!is_midr_in_range_list(has_vgic_v3))
		2331	return false;
		2332
		2333	if (!is_hyp_mode_available())
		2334	return false;
		2335
		2336	if (this_cpu_has_cap(ARM64_HAS_GICV5_LEGACY))
		2337	return true;
		2338
		2339	if (is_kernel_in_hyp_mode())
		2340	res.a1 = read_sysreg_s(SYS_ICH_VTR_EL2);
		2341	else
		2342	arm_smccc_1_1_hvc(HVC_GET_ICH_VTR_EL2, &res);
		2343
		2344	if (res.a0 == HVC_STUB_ERR)
		2345	return false;
		2346
		2347	return res.a1 & ICH_VTR_EL2_TDS;
		2348	}
		2349
2307	#ifdef CONFIG_ARM64_BTI	2350	#ifdef CONFIG_ARM64_BTI
2308	static void bti_enable(const struct arm64_cpu_capabilities *__unused)	2351	static void bti_enable(const struct arm64_cpu_capabilities *__unused)
2309	{	2352	{
@@ -2815,6 +2858,15 @@ static const struct arm64_cpu_capabilities arm64_features[] = { · 505 unmodified lines
2815	.matches = has_gic_prio_relaxed_sync,	2858	.matches = has_gic_prio_relaxed_sync,
2816	},	2859	},
2817	#endif	2860	#endif
		2861	{
		2862	/*
		2863	* Depends on having GICv3
		2864	*/
		2865	.desc = "ICV_DIR_EL1 trapping",
		2866	.capability = ARM64_HAS_ICH_HCR_EL2_TDIR,
		2867	.type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE,
		2868	.matches = can_trap_icv_dir_el1,
		2869	},
2818	#ifdef CONFIG_ARM64_E0PD	2870	#ifdef CONFIG_ARM64_E0PD
2819	{	2871	{
2820	.desc = "E0PD",	2872	.desc = "E0PD",
@@ -3089,6 +3141,13 @@ static const struct arm64_cpu_capabilities arm64_features[] = { · 268 unmodified lines
3089	.capability = ARM64_HAS_GICV5_LEGACY,	3141	.capability = ARM64_HAS_GICV5_LEGACY,
3090	.matches = test_has_gicv5_legacy,	3142	.matches = test_has_gicv5_legacy,
3091	},	3143	},
		3144	{
		3145	.desc = "XNX",
		3146	.capability = ARM64_HAS_XNX,
		3147	.type = ARM64_CPUCAP_SYSTEM_FEATURE,
		3148	.matches = has_cpuid_feature,
		3149	ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, XNX, IMP)
		3150	},
3092	{},	3151	{},
3093	};	3152	};
3094		3153

arch/arm64/kernel/hyp-stub.S+5−0

 1:	cmp	x0, #HVC_FINALISE_EL2
 	b.eq	__finalise_el2
+	cmp	x0, #HVC_GET_ICH_VTR_EL2
+	b.ne	2f
+	mrs_s	x1, SYS_ICH_VTR_EL2
+	b	9f
 2:	cmp	x0, #HVC_SOFT_RESTART
 	b.ne	3f
 	mov	x0, x2

arch/arm64/kernel/image-vars.h+1−0

@@ -91,6 +91,7 @@ KVM_NVHE_ALIAS(spectre_bhb_patch_loop_mitigation_enable); · 90 unmodified lines
91	KVM_NVHE_ALIAS(spectre_bhb_patch_wa3);	91	KVM_NVHE_ALIAS(spectre_bhb_patch_wa3);
92	KVM_NVHE_ALIAS(spectre_bhb_patch_clearbhb);	92	KVM_NVHE_ALIAS(spectre_bhb_patch_clearbhb);
93	KVM_NVHE_ALIAS(alt_cb_patch_nops);	93	KVM_NVHE_ALIAS(alt_cb_patch_nops);
		94	KVM_NVHE_ALIAS(kvm_compute_ich_hcr_trap_bits);
94		95
95	/* Global kernel state accessed by nVHE hyp code. */	96	/* Global kernel state accessed by nVHE hyp code. */
96	KVM_NVHE_ALIAS(kvm_vgic_global_state);	97	KVM_NVHE_ALIAS(kvm_vgic_global_state);

arch/arm64/kvm/arm.c+17−3

 		}
 		mutex_unlock(&kvm->lock);
 		break;
+	case KVM_CAP_ARM_SEA_TO_USER:
+		r = 0;
+		set_bit(KVM_ARCH_FLAG_EXIT_SEA, &kvm->arch.flags);
+		break;
 	default:
 		break;
 	}
 	case KVM_CAP_IRQFD_RESAMPLE:
 	case KVM_CAP_COUNTER_OFFSET:
 	case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS:
+	case KVM_CAP_ARM_SEA_TO_USER:
 		r = 1;
 		break;
 	case KVM_CAP_SET_GUEST_DEBUG2:
 	if (!has_vhe())
 		return kzalloc(sz, GFP_KERNEL_ACCOUNT);
-	return __vmalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_HIGHMEM | __GFP_ZERO);
+	return kvzalloc(sz, GFP_KERNEL_ACCOUNT);
 }
 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 {
 	if (is_protected_kvm_enabled()) {
-		kvm_call_hyp(__vgic_v3_save_vmcr_aprs,
+		kvm_call_hyp(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3);
-			     &vcpu->arch.vgic_cpu.vgic_v3);
 		kvm_call_hyp_nvhe(__pkvm_vcpu_put);
 	}
 		 */
 		kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu);
+		/* Process interrupts deactivated through a trap */
+		if (kvm_check_request(KVM_REQ_VGIC_PROCESS_UPDATE, vcpu))
+			kvm_vgic_process_async_update(vcpu);
 		if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu))
 			kvm_update_stolen_time(vcpu);
 	return r;
 }
+long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl,
+				  unsigned long arg)
+{
+	return -ENOIOCTLCMD;
+}
 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
 {

arch/arm64/kvm/at.c+176−20

@@ -346,6 +346,11 @@ static int setup_s1_walk(struct kvm_vcpu vcpu, struct s1_walk_info wi, · 345 unmodified lines
346		346
347	wi->baddr &= GENMASK_ULL(wi->max_oa_bits - 1, x);	347	wi->baddr &= GENMASK_ULL(wi->max_oa_bits - 1, x);
348		348
		349	wi->ha = kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, HAFDBS, AF);
		350	wi->ha &= (wi->regime == TR_EL2 ?
		351	FIELD_GET(TCR_EL2_HA, tcr) :
		352	FIELD_GET(TCR_HA, tcr));
		353
349	return 0;	354	return 0;
350		355
351	addrsz:	356	addrsz:
@@ -362,10 +367,42 @@ transfault: · 10 unmodified lines
362	return -EFAULT;	367	return -EFAULT;
363	}	368	}
364		369
		370	static int kvm_read_s1_desc(struct kvm_vcpu vcpu, u64 pa, u64 desc,
		371	struct s1_walk_info *wi)
		372	{
		373	u64 val;
		374	int r;
		375
		376	r = kvm_read_guest(vcpu->kvm, pa, &val, sizeof(val));
		377	if (r)
		378	return r;
		379
		380	if (wi->be)
		381	*desc = be64_to_cpu((__force __be64)val);
		382	else
		383	*desc = le64_to_cpu((__force __le64)val);
		384
		385	return 0;
		386	}
		387
		388	static int kvm_swap_s1_desc(struct kvm_vcpu *vcpu, u64 pa, u64 old, u64 new,
		389	struct s1_walk_info *wi)
		390	{
		391	if (wi->be) {
		392	old = (__force u64)cpu_to_be64(old);
		393	new = (__force u64)cpu_to_be64(new);
		394	} else {
		395	old = (__force u64)cpu_to_le64(old);
		396	new = (__force u64)cpu_to_le64(new);
		397	}
		398
		399	return __kvm_at_swap_desc(vcpu->kvm, pa, old, new);
		400	}
		401
365	static int walk_s1(struct kvm_vcpu vcpu, struct s1_walk_info wi,	402	static int walk_s1(struct kvm_vcpu vcpu, struct s1_walk_info wi,
366	struct s1_walk_result *wr, u64 va)	403	struct s1_walk_result *wr, u64 va)
367	{	404	{
368	u64 va_top, va_bottom, baddr, desc;	405	u64 va_top, va_bottom, baddr, desc, new_desc, ipa;
369	int level, stride, ret;	406	int level, stride, ret;
370		407
371	level = wi->sl;	408	level = wi->sl;
@@ -375,7 +412,7 @@ static int walk_s1(struct kvm_vcpu vcpu, struct s1_walk_info wi, · 3 unmodified lines
375	va_top = get_ia_size(wi) - 1;	412	va_top = get_ia_size(wi) - 1;
376		413
377	while (1) {	414	while (1) {
378	u64 index, ipa;	415	u64 index;
379		416
380	va_bottom = (3 - level) * stride + wi->pgshift;	417	va_bottom = (3 - level) * stride + wi->pgshift;
381	index = (va & GENMASK_ULL(va_top, va_bottom)) >> (va_bottom - 3);	418	index = (va & GENMASK_ULL(va_top, va_bottom)) >> (va_bottom - 3);
@@ -414,16 +451,13 @@ static int walk_s1(struct kvm_vcpu vcpu, struct s1_walk_info wi, · 32 unmodified lines
414	return ret;	451	return ret;
415	}	452	}
416		453
417	ret = kvm_read_guest(vcpu->kvm, ipa, &desc, sizeof(desc));	454	ret = kvm_read_s1_desc(vcpu, ipa, &desc, wi);
418	if (ret) {	455	if (ret) {
419	fail_s1_walk(wr, ESR_ELx_FSC_SEA_TTW(level), false);	456	fail_s1_walk(wr, ESR_ELx_FSC_SEA_TTW(level), false);
420	return ret;	457	return ret;
421	}	458	}
422		459
423	if (wi->be)	460	new_desc = desc;
424	desc = be64_to_cpu((__force __be64)desc);
425	else
426	desc = le64_to_cpu((__force __le64)desc);
427		461
428	/* Invalid descriptor */	462	/* Invalid descriptor */
429	if (!(desc & BIT(0)))	463	if (!(desc & BIT(0)))
@@ -477,6 +511,17 @@ static int walk_s1(struct kvm_vcpu vcpu, struct s1_walk_info wi, · 47 unmodified lines
477	if (check_output_size(baddr & GENMASK(52, va_bottom), wi))	511	if (check_output_size(baddr & GENMASK(52, va_bottom), wi))
478	goto addrsz;	512	goto addrsz;
479		513
		514	if (wi->ha)
		515	new_desc \|= PTE_AF;
		516
		517	if (new_desc != desc) {
		518	ret = kvm_swap_s1_desc(vcpu, ipa, desc, new_desc, wi);
		519	if (ret)
		520	return ret;
		521
		522	desc = new_desc;
		523	}
		524
480	if (!(desc & PTE_AF)) {	525	if (!(desc & PTE_AF)) {
481	fail_s1_walk(wr, ESR_ELx_FSC_ACCESS_L(level), false);	526	fail_s1_walk(wr, ESR_ELx_FSC_ACCESS_L(level), false);
482	return -EACCES;	527	return -EACCES;
@@ -1221,7 +1266,7 @@ static void compute_s1_permissions(struct kvm_vcpu *vcpu, · 738 unmodified lines
1221	wr->pr &= !pan;	1266	wr->pr &= !pan;
1222	}	1267	}
1223		1268
1224	static u64 handle_at_slow(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)	1269	static int handle_at_slow(struct kvm_vcpu vcpu, u32 op, u64 vaddr, u64 par)
1225	{	1270	{
1226	struct s1_walk_result wr = {};	1271	struct s1_walk_result wr = {};
1227	struct s1_walk_info wi = {};	1272	struct s1_walk_info wi = {};
@@ -1246,6 +1291,11 @@ static u64 handle_at_slow(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) · 18 unmodified lines
1246		1291
1247	srcu_read_unlock(&vcpu->kvm->srcu, idx);	1292	srcu_read_unlock(&vcpu->kvm->srcu, idx);
1248		1293
		1294	/*
		1295	* Race to update a descriptor -- restart the walk.
		1296	*/
		1297	if (ret == -EAGAIN)
		1298	return ret;
1249	if (ret)	1299	if (ret)
1250	goto compute_par;	1300	goto compute_par;
1251		1301
@@ -1279,7 +1329,8 @@ static u64 handle_at_slow(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) · 27 unmodified lines
1279	fail_s1_walk(&wr, ESR_ELx_FSC_PERM_L(wr.level), false);	1329	fail_s1_walk(&wr, ESR_ELx_FSC_PERM_L(wr.level), false);
1280		1330
1281	compute_par:	1331	compute_par:
1282	return compute_par_s1(vcpu, &wi, &wr);	1332	*par = compute_par_s1(vcpu, &wi, &wr);
		1333	return 0;
1283	}	1334	}
1284		1335
1285	/*	1336	/*
@@ -1407,9 +1458,10 @@ static bool par_check_s1_access_fault(u64 par) · 121 unmodified lines
1407	!(par & SYS_PAR_EL1_S));	1458	!(par & SYS_PAR_EL1_S));
1408	}	1459	}
1409		1460
1410	void __kvm_at_s1e01(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)	1461	int __kvm_at_s1e01(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)
1411	{	1462	{
1412	u64 par = __kvm_at_s1e01_fast(vcpu, op, vaddr);	1463	u64 par = __kvm_at_s1e01_fast(vcpu, op, vaddr);
		1464	int ret;
1413		1465
1414	/*	1466	/*
1415	* If PAR_EL1 reports that AT failed on a S1 permission or access	1467	* If PAR_EL1 reports that AT failed on a S1 permission or access
@@ -1421,15 +1473,20 @@ void __kvm_at_s1e01(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) · 5 unmodified lines
1421	*/	1473	*/
1422	if ((par & SYS_PAR_EL1_F) &&	1474	if ((par & SYS_PAR_EL1_F) &&
1423	!par_check_s1_perm_fault(par) &&	1475	!par_check_s1_perm_fault(par) &&
1424	!par_check_s1_access_fault(par))	1476	!par_check_s1_access_fault(par)) {
1425	par = handle_at_slow(vcpu, op, vaddr);	1477	ret = handle_at_slow(vcpu, op, vaddr, &par);
		1478	if (ret)
		1479	return ret;
		1480	}
1426		1481
1427	vcpu_write_sys_reg(vcpu, par, PAR_EL1);	1482	vcpu_write_sys_reg(vcpu, par, PAR_EL1);
		1483	return 0;
1428	}	1484	}
1429		1485
1430	void __kvm_at_s1e2(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)	1486	int __kvm_at_s1e2(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)
1431	{	1487	{
1432	u64 par;	1488	u64 par;
		1489	int ret;
1433		1490
1434	/*	1491	/*
1435	* We've trapped, so everything is live on the CPU. As we will be	1492	* We've trapped, so everything is live on the CPU. As we will be
@@ -1476,13 +1533,17 @@ void __kvm_at_s1e2(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) · 40 unmodified lines
1476	}	1533	}
1477		1534
1478	/* We failed the translation, let's replay it in slow motion */	1535	/* We failed the translation, let's replay it in slow motion */
1479	if ((par & SYS_PAR_EL1_F) && !par_check_s1_perm_fault(par))	1536	if ((par & SYS_PAR_EL1_F) && !par_check_s1_perm_fault(par)) {
1480	par = handle_at_slow(vcpu, op, vaddr);	1537	ret = handle_at_slow(vcpu, op, vaddr, &par);
		1538	if (ret)
		1539	return ret;
		1540	}
1481		1541
1482	vcpu_write_sys_reg(vcpu, par, PAR_EL1);	1542	vcpu_write_sys_reg(vcpu, par, PAR_EL1);
		1543	return 0;
1483	}	1544	}
1484		1545
1485	void __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)	1546	int __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)
1486	{	1547	{
1487	struct kvm_s2_trans out = {};	1548	struct kvm_s2_trans out = {};
1488	u64 ipa, par;	1549	u64 ipa, par;
@@ -1509,13 +1570,13 @@ void __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) · 20 unmodified lines
1509	break;	1570	break;
1510	default:	1571	default:
1511	WARN_ON_ONCE(1);	1572	WARN_ON_ONCE(1);
1512	return;	1573	return 0;
1513	}	1574	}
1514		1575
1515	__kvm_at_s1e01(vcpu, op, vaddr);	1576	__kvm_at_s1e01(vcpu, op, vaddr);
1516	par = vcpu_read_sys_reg(vcpu, PAR_EL1);	1577	par = vcpu_read_sys_reg(vcpu, PAR_EL1);
1517	if (par & SYS_PAR_EL1_F)	1578	if (par & SYS_PAR_EL1_F)
1518	return;	1579	return 0;
1519		1580
1520	/*	1581	/*
1521	* If we only have a single stage of translation (EL2&0), exit	1582	* If we only have a single stage of translation (EL2&0), exit
@@ -1523,14 +1584,14 @@ void __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) · 1 unmodified line
1523	*/	1584	*/
1524	if (compute_translation_regime(vcpu, op) == TR_EL20 \|\|	1585	if (compute_translation_regime(vcpu, op) == TR_EL20 \|\|
1525	!(vcpu_read_sys_reg(vcpu, HCR_EL2) & (HCR_VM \| HCR_DC)))	1586	!(vcpu_read_sys_reg(vcpu, HCR_EL2) & (HCR_VM \| HCR_DC)))
1526	return;	1587	return 0;
1527		1588
1528	/* Do the stage-2 translation */	1589	/* Do the stage-2 translation */
1529	ipa = (par & GENMASK_ULL(47, 12)) \| (vaddr & GENMASK_ULL(11, 0));	1590	ipa = (par & GENMASK_ULL(47, 12)) \| (vaddr & GENMASK_ULL(11, 0));
1530	out.esr = 0;	1591	out.esr = 0;
1531	ret = kvm_walk_nested_s2(vcpu, ipa, &out);	1592	ret = kvm_walk_nested_s2(vcpu, ipa, &out);
1532	if (ret < 0)	1593	if (ret < 0)
1533	return;	1594	return ret;
1534		1595
1535	/* Check the access permission */	1596	/* Check the access permission */
1536	if (!out.esr &&	1597	if (!out.esr &&
@@ -1539,6 +1600,7 @@ void __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr) · 2 unmodified lines
1539		1600
1540	par = compute_par_s12(vcpu, par, &out);	1601	par = compute_par_s12(vcpu, par, &out);
1541	vcpu_write_sys_reg(vcpu, par, PAR_EL1);	1602	vcpu_write_sys_reg(vcpu, par, PAR_EL1);
		1603	return 0;
1542	}	1604	}
1543		1605
1544	/*	1606	/*
@@ -1637,3 +1699,97 @@ int __kvm_find_s1_desc_level(struct kvm_vcpu vcpu, u64 va, u64 ipa, int level) · 92 unmodified lines
1637	return ret;	1699	return ret;
1638	}	1700	}
1639	}	1701	}
		1702
		1703	#ifdef CONFIG_ARM64_LSE_ATOMICS
		1704	static int __lse_swap_desc(u64 __user *ptep, u64 old, u64 new)
		1705	{
		1706	u64 tmp = old;
		1707	int ret = 0;
		1708
		1709	uaccess_enable_privileged();
		1710
		1711	asm volatile(__LSE_PREAMBLE
		1712	"1: cas %[old], %[new], %[addr]\n"
		1713	"2:\n"
		1714	_ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret])
		1715	: [old] "+r" (old), [addr] "+Q" (*ptep), [ret] "+r" (ret)
		1716	: [new] "r" (new)
		1717	: "memory");
		1718
		1719	uaccess_disable_privileged();
		1720
		1721	if (ret)
		1722	return ret;
		1723	if (tmp != old)
		1724	return -EAGAIN;
		1725
		1726	return ret;
		1727	}
		1728	#else
		1729	static int __lse_swap_desc(u64 __user *ptep, u64 old, u64 new)
		1730	{
		1731	return -EINVAL;
		1732	}
		1733	#endif
		1734
		1735	static int __llsc_swap_desc(u64 __user *ptep, u64 old, u64 new)
		1736	{
		1737	int ret = 1;
		1738	u64 tmp;
		1739
		1740	uaccess_enable_privileged();
		1741
		1742	asm volatile("prfm pstl1strm, %[addr]\n"
		1743	"1: ldxr %[tmp], %[addr]\n"
		1744	"sub %[tmp], %[tmp], %[old]\n"
		1745	"cbnz %[tmp], 3f\n"
		1746	"2: stlxr %w[ret], %[new], %[addr]\n"
		1747	"3:\n"
		1748	_ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w[ret])
		1749	_ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w[ret])
		1750	: [ret] "+r" (ret), [addr] "+Q" (*ptep), [tmp] "=&r" (tmp)
		1751	: [old] "r" (old), [new] "r" (new)
		1752	: "memory");
		1753
		1754	uaccess_disable_privileged();
		1755
		1756	/* STLXR didn't update the descriptor, or the compare failed */
		1757	if (ret == 1)
		1758	return -EAGAIN;
		1759
		1760	return ret;
		1761	}
		1762
		1763	int __kvm_at_swap_desc(struct kvm *kvm, gpa_t ipa, u64 old, u64 new)
		1764	{
		1765	struct kvm_memory_slot *slot;
		1766	unsigned long hva;
		1767	u64 __user *ptep;
		1768	bool writable;
		1769	int offset;
		1770	gfn_t gfn;
		1771	int r;
		1772
		1773	lockdep_assert(srcu_read_lock_held(&kvm->srcu));
		1774
		1775	gfn = ipa >> PAGE_SHIFT;
		1776	offset = offset_in_page(ipa);
		1777	slot = gfn_to_memslot(kvm, gfn);
		1778	hva = gfn_to_hva_memslot_prot(slot, gfn, &writable);
		1779	if (kvm_is_error_hva(hva))
		1780	return -EINVAL;
		1781	if (!writable)
		1782	return -EPERM;
		1783
		1784	ptep = (u64 __user *)hva + offset;
		1785	if (cpus_have_final_cap(ARM64_HAS_LSE_ATOMICS))
		1786	r = __lse_swap_desc(ptep, old, new);
		1787	else
		1788	r = __llsc_swap_desc(ptep, old, new);
		1789
		1790	if (r < 0)
		1791	return r;
		1792
		1793	mark_page_dirty_in_slot(kvm, slot, gfn);
		1794	return 0;
		1795	}

arch/arm64/kvm/hyp/nvhe/hyp-main.c+4−3

 	host_vcpu->arch.iflags		= hyp_vcpu->vcpu.arch.iflags;
 	host_cpu_if->vgic_hcr		= hyp_cpu_if->vgic_hcr;
+	host_cpu_if->vgic_vmcr		= hyp_cpu_if->vgic_vmcr;
 	for (i = 0; i < hyp_cpu_if->used_lrs; ++i)
 		host_cpu_if->vgic_lr[i] = hyp_cpu_if->vgic_lr[i];
 }
 	__vgic_v3_init_lrs();
 }
-static void handle___vgic_v3_save_vmcr_aprs(struct kvm_cpu_context *host_ctxt)
+static void handle___vgic_v3_save_aprs(struct kvm_cpu_context *host_ctxt)
 {
 	DECLARE_REG(struct vgic_v3_cpu_if *, cpu_if, host_ctxt, 1);
-	__vgic_v3_save_vmcr_aprs(kern_hyp_va(cpu_if));
+	__vgic_v3_save_aprs(kern_hyp_va(cpu_if));
 }
 static void handle___vgic_v3_restore_vmcr_aprs(struct kvm_cpu_context *host_ctxt)
 	HANDLE_FUNC(__kvm_tlb_flush_vmid_range),
 	HANDLE_FUNC(__kvm_flush_cpu_context),
 	HANDLE_FUNC(__kvm_timer_set_cntvoff),
-	HANDLE_FUNC(__vgic_v3_save_vmcr_aprs),
+	HANDLE_FUNC(__vgic_v3_save_aprs),
 	HANDLE_FUNC(__vgic_v3_restore_vmcr_aprs),
 	HANDLE_FUNC(__pkvm_reserve_vm),
 	HANDLE_FUNC(__pkvm_unreserve_vm),

arch/arm64/kvm/hyp/nvhe/pkvm.c+3−0

@@ -337,6 +337,9 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc · 336 unmodified lines
337	/* CTR_EL0 is always under host control, even for protected VMs. */	337	/* CTR_EL0 is always under host control, even for protected VMs. */
338	hyp_vm->kvm.arch.ctr_el0 = host_kvm->arch.ctr_el0;	338	hyp_vm->kvm.arch.ctr_el0 = host_kvm->arch.ctr_el0;
339		339
		340	/* Preserve the vgic model so that GICv3 emulation works */
		341	hyp_vm->kvm.arch.vgic.vgic_model = host_kvm->arch.vgic.vgic_model;
		342
340	if (test_bit(KVM_ARCH_FLAG_MTE_ENABLED, &host_kvm->arch.flags))	343	if (test_bit(KVM_ARCH_FLAG_MTE_ENABLED, &host_kvm->arch.flags))
341	set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags);	344	set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags);
342		345

arch/arm64/kvm/hyp/nvhe/sys_regs.c+5−0

 	/* Scalable Vector Registers are restricted. */
+	HOST_HANDLED(SYS_ICC_PMR_EL1),
 	RAZ_WI(SYS_ERRIDR_EL1),
 	RAZ_WI(SYS_ERRSELR_EL1),
 	RAZ_WI(SYS_ERXFR_EL1),
 	/* Limited Ordering Regions Registers are restricted. */
+	HOST_HANDLED(SYS_ICC_DIR_EL1),
+	HOST_HANDLED(SYS_ICC_RPR_EL1),
 	HOST_HANDLED(SYS_ICC_SGI1R_EL1),
 	HOST_HANDLED(SYS_ICC_ASGI1R_EL1),
 	HOST_HANDLED(SYS_ICC_SGI0R_EL1),
+	HOST_HANDLED(SYS_ICC_CTLR_EL1),
 	{ SYS_DESC(SYS_ICC_SRE_EL1), .access = pvm_gic_read_sre, },
 	HOST_HANDLED(SYS_CCSIDR_EL1),

arch/arm64/kvm/hyp/pgtable.c+104−18

@@ -661,11 +661,37 @@ void kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, · 660 unmodified lines
661		661
662	#define KVM_S2_MEMATTR(pgt, attr) PAGE_S2_MEMATTR(attr, stage2_has_fwb(pgt))	662	#define KVM_S2_MEMATTR(pgt, attr) PAGE_S2_MEMATTR(attr, stage2_has_fwb(pgt))
663		663
		664	static int stage2_set_xn_attr(enum kvm_pgtable_prot prot, kvm_pte_t *attr)
		665	{
		666	bool px, ux;
		667	u8 xn;
		668
		669	px = prot & KVM_PGTABLE_PROT_PX;
		670	ux = prot & KVM_PGTABLE_PROT_UX;
		671
		672	if (!cpus_have_final_cap(ARM64_HAS_XNX) && px != ux)
		673	return -EINVAL;
		674
		675	if (px && ux)
		676	xn = 0b00;
		677	else if (!px && ux)
		678	xn = 0b01;
		679	else if (!px && !ux)
		680	xn = 0b10;
		681	else
		682	xn = 0b11;
		683
		684	*attr &= ~KVM_PTE_LEAF_ATTR_HI_S2_XN;
		685	*attr \|= FIELD_PREP(KVM_PTE_LEAF_ATTR_HI_S2_XN, xn);
		686	return 0;
		687	}
		688
664	static int stage2_set_prot_attr(struct kvm_pgtable *pgt, enum kvm_pgtable_prot prot,	689	static int stage2_set_prot_attr(struct kvm_pgtable *pgt, enum kvm_pgtable_prot prot,
665	kvm_pte_t *ptep)	690	kvm_pte_t *ptep)
666	{	691	{
667	kvm_pte_t attr;	692	kvm_pte_t attr;
668	u32 sh = KVM_PTE_LEAF_ATTR_LO_S2_SH_IS;	693	u32 sh = KVM_PTE_LEAF_ATTR_LO_S2_SH_IS;
		694	int r;
669		695
670	switch (prot & (KVM_PGTABLE_PROT_DEVICE \|	696	switch (prot & (KVM_PGTABLE_PROT_DEVICE \|
671	KVM_PGTABLE_PROT_NORMAL_NC)) {	697	KVM_PGTABLE_PROT_NORMAL_NC)) {
@@ -685,8 +711,9 @@ static int stage2_set_prot_attr(struct kvm_pgtable *pgt, enum kvm_pgtable_prot p · 13 unmodified lines
685	attr = KVM_S2_MEMATTR(pgt, NORMAL);	711	attr = KVM_S2_MEMATTR(pgt, NORMAL);
686	}	712	}
687		713
688	if (!(prot & KVM_PGTABLE_PROT_X))	714	r = stage2_set_xn_attr(prot, &attr);
689	attr \|= KVM_PTE_LEAF_ATTR_HI_S2_XN;	715	if (r)
		716	return r;
690		717
691	if (prot & KVM_PGTABLE_PROT_R)	718	if (prot & KVM_PGTABLE_PROT_R)
692	attr \|= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R;	719	attr \|= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R;
@@ -715,8 +742,20 @@ enum kvm_pgtable_prot kvm_pgtable_stage2_pte_prot(kvm_pte_t pte) · 22 unmodified lines
715	prot \|= KVM_PGTABLE_PROT_R;	742	prot \|= KVM_PGTABLE_PROT_R;
716	if (pte & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W)	743	if (pte & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W)
717	prot \|= KVM_PGTABLE_PROT_W;	744	prot \|= KVM_PGTABLE_PROT_W;
718	if (!(pte & KVM_PTE_LEAF_ATTR_HI_S2_XN))	745
719	prot \|= KVM_PGTABLE_PROT_X;	746	switch (FIELD_GET(KVM_PTE_LEAF_ATTR_HI_S2_XN, pte)) {
		747	case 0b00:
		748	prot \|= KVM_PGTABLE_PROT_PX \| KVM_PGTABLE_PROT_UX;
		749	break;
		750	case 0b01:
		751	prot \|= KVM_PGTABLE_PROT_UX;
		752	break;
		753	case 0b11:
		754	prot \|= KVM_PGTABLE_PROT_PX;
		755	break;
		756	default:
		757	break;
		758	}
720		759
721	return prot;	760	return prot;
722	}	761	}
@@ -1290,9 +1329,9 @@ bool kvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr, · 567 unmodified lines
1290	int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr,	1329	int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr,
1291	enum kvm_pgtable_prot prot, enum kvm_pgtable_walk_flags flags)	1330	enum kvm_pgtable_prot prot, enum kvm_pgtable_walk_flags flags)
1292	{	1331	{
1293	int ret;	1332	kvm_pte_t xn = 0, set = 0, clr = 0;
1294	s8 level;	1333	s8 level;
1295	kvm_pte_t set = 0, clr = 0;	1334	int ret;
1296		1335
1297	if (prot & KVM_PTE_LEAF_ATTR_HI_SW)	1336	if (prot & KVM_PTE_LEAF_ATTR_HI_SW)
1298	return -EINVAL;	1337	return -EINVAL;
@@ -1303,8 +1342,12 @@ int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, · 4 unmodified lines
1303	if (prot & KVM_PGTABLE_PROT_W)	1342	if (prot & KVM_PGTABLE_PROT_W)
1304	set \|= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W;	1343	set \|= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W;
1305		1344
1306	if (prot & KVM_PGTABLE_PROT_X)	1345	ret = stage2_set_xn_attr(prot, &xn);
1307	clr \|= KVM_PTE_LEAF_ATTR_HI_S2_XN;	1346	if (ret)
		1347	return ret;
		1348
		1349	set \|= xn & KVM_PTE_LEAF_ATTR_HI_S2_XN;
		1350	clr \|= ~xn & KVM_PTE_LEAF_ATTR_HI_S2_XN;
1308		1351
1309	ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level, flags);	1352	ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level, flags);
1310	if (!ret \|\| ret == -EAGAIN)	1353	if (!ret \|\| ret == -EAGAIN)
@@ -1535,37 +1578,80 @@ size_t kvm_pgtable_stage2_pgd_size(u64 vtcr) · 224 unmodified lines
1535	return kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE;	1578	return kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE;
1536	}	1579	}
1537		1580
1538	static int stage2_free_walker(const struct kvm_pgtable_visit_ctx *ctx,	1581	static int stage2_free_leaf(const struct kvm_pgtable_visit_ctx *ctx)
1539	enum kvm_pgtable_walk_flags visit)
1540	{	1582	{
1541	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;	1583	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1542		1584
1543	if (!stage2_pte_is_counted(ctx->old))	1585	mm_ops->put_page(ctx->ptep);
		1586	return 0;
		1587	}
		1588
		1589	static int stage2_free_table_post(const struct kvm_pgtable_visit_ctx *ctx)
		1590	{
		1591	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
		1592	kvm_pte_t *childp = kvm_pte_follow(ctx->old, mm_ops);
		1593
		1594	if (mm_ops->page_count(childp) != 1)
1544	return 0;	1595	return 0;
1545		1596
		1597	/*
		1598	* Drop references and clear the now stale PTE to avoid rewalking the
		1599	* freed page table.
		1600	*/
1546	mm_ops->put_page(ctx->ptep);	1601	mm_ops->put_page(ctx->ptep);
		1602	mm_ops->put_page(childp);
		1603	kvm_clear_pte(ctx->ptep);
		1604	return 0;
		1605	}
1547		1606
1548	if (kvm_pte_table(ctx->old, ctx->level))	1607	static int stage2_free_walker(const struct kvm_pgtable_visit_ctx *ctx,
1549	mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops));	1608	enum kvm_pgtable_walk_flags visit)
		1609	{
		1610	if (!stage2_pte_is_counted(ctx->old))
		1611	return 0;
1550		1612
1551	return 0;	1613	switch (visit) {
		1614	case KVM_PGTABLE_WALK_LEAF:
		1615	return stage2_free_leaf(ctx);
		1616	case KVM_PGTABLE_WALK_TABLE_POST:
		1617	return stage2_free_table_post(ctx);
		1618	default:
		1619	return -EINVAL;
		1620	}
1552	}	1621	}
1553		1622
1554	void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt)	1623	void kvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt,
		1624	u64 addr, u64 size)
1555	{	1625	{
1556	size_t pgd_sz;
1557	struct kvm_pgtable_walker walker = {	1626	struct kvm_pgtable_walker walker = {
1558	.cb = stage2_free_walker,	1627	.cb = stage2_free_walker,
1559	.flags = KVM_PGTABLE_WALK_LEAF \|	1628	.flags = KVM_PGTABLE_WALK_LEAF \|
1560	KVM_PGTABLE_WALK_TABLE_POST,	1629	KVM_PGTABLE_WALK_TABLE_POST,
1561	};	1630	};
1562		1631
1563	WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));	1632	WARN_ON(kvm_pgtable_walk(pgt, addr, size, &walker));
		1633	}
		1634
		1635	void kvm_pgtable_stage2_destroy_pgd(struct kvm_pgtable *pgt)
		1636	{
		1637	size_t pgd_sz;
		1638
1564	pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE;	1639	pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE;
1565	pgt->mm_ops->free_pages_exact(kvm_dereference_pteref(&walker, pgt->pgd), pgd_sz);	1640
		1641	/*
		1642	* Since the pgtable is unlinked at this point, and not shared with
		1643	* other walkers, safely deference pgd with kvm_dereference_pteref_raw()
		1644	*/
		1645	pgt->mm_ops->free_pages_exact(kvm_dereference_pteref_raw(pgt->pgd), pgd_sz);
1566	pgt->pgd = NULL;	1646	pgt->pgd = NULL;
1567	}	1647	}
1568		1648
		1649	void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt)
		1650	{
		1651	kvm_pgtable_stage2_destroy_range(pgt, 0, BIT(pgt->ia_bits));
		1652	kvm_pgtable_stage2_destroy_pgd(pgt);
		1653	}
		1654
1569	void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops mm_ops, void pgtable, s8 level)	1655	void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops mm_ops, void pgtable, s8 level)
1570	{	1656	{
1571	kvm_pteref_t ptep = (kvm_pteref_t)pgtable;	1657	kvm_pteref_t ptep = (kvm_pteref_t)pgtable;

arch/arm64/kvm/hyp/vgic-v2-cpuif-proxy.c+4−0

 		return -1;
 	}
+	/* Handle deactivation as a normal exit */
+	if ((fault_ipa - vgic->vgic_cpu_base) >= GIC_CPU_DEACTIVATE)
+		return 0;
 	rd = kvm_vcpu_dabt_get_rd(vcpu);
 	addr  = kvm_vgic_global_state.vcpu_hyp_va;
 	addr += fault_ipa - vgic->vgic_cpu_base;

arch/arm64/kvm/hyp/vgic-v3-sr.c+63−33

 #include <asm/kvm_hyp.h>
 #include <asm/kvm_mmu.h>
+#include "../../vgic/vgic.h"
 #define vtr_to_max_lr_idx(v)		((v) & 0xf)
 #define vtr_to_nr_pre_bits(v)		((((u32)(v) >> 26) & 7) + 1)
 #define vtr_to_nr_apr_regs(v)		(1 << (vtr_to_nr_pre_bits(v) - 5))
 	unreachable();
 }
-static void __gic_v3_set_lr(u64 val, int lr)
+void __gic_v3_set_lr(u64 val, int lr)
 {
 	switch (lr & 0xf) {
 	case 0:
 	return val;
 }
+static u64 compute_ich_hcr(struct vgic_v3_cpu_if *cpu_if)
+{
+	return cpu_if->vgic_hcr | vgic_ich_hcr_trap_bits();
+}
 void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if)
 {
 	u64 used_lrs = cpu_if->used_lrs;
 		}
 	}
-	if (used_lrs || cpu_if->its_vpe.its_vm) {
+	if (used_lrs) {
 		int i;
 		u32 elrsr;
 		elrsr = read_gicreg(ICH_ELRSR_EL2);
-		write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EL2_En, ICH_HCR_EL2);
 		for (i = 0; i < used_lrs; i++) {
 			if (elrsr & (1 << i))
 				cpu_if->vgic_lr[i] &= ~ICH_LR_STATE;
 			__gic_v3_set_lr(0, i);
 		}
 	}
+	cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
+	if (cpu_if->vgic_hcr & ICH_HCR_EL2_LRENPIE) {
+		u64 val = read_gicreg(ICH_HCR_EL2);
+		cpu_if->vgic_hcr &= ~ICH_HCR_EL2_EOIcount;
+		cpu_if->vgic_hcr |= val & ICH_HCR_EL2_EOIcount;
+	}
+	write_gicreg(0, ICH_HCR_EL2);
+	/*
+	 * Hack alert: On NV, this results in a trap so that the above write
+	 * actually takes effect... No synchronisation is necessary, as we
+	 * only care about the effects when this traps.
+	 */
+	read_gicreg(ICH_MISR_EL2);
 }
 void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if)
 	u64 used_lrs = cpu_if->used_lrs;
 	int i;
-	if (used_lrs || cpu_if->its_vpe.its_vm) {
+	write_gicreg(compute_ich_hcr(cpu_if), ICH_HCR_EL2);
-		write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
-		for (i = 0; i < used_lrs; i++)
+	for (i = 0; i < used_lrs; i++)
-			__gic_v3_set_lr(cpu_if->vgic_lr[i], i);
+		__gic_v3_set_lr(cpu_if->vgic_lr[i], i);
-	}
 	/*
 	 * Ensure that writes to the LRs, and on non-VHE systems ensure that
 	}
 	/*
-	 * If we need to trap system registers, we must write
+	 * If we need to trap system registers, we must write ICH_HCR_EL2
-	 * ICH_HCR_EL2 anyway, even if no interrupts are being
+	 * anyway, even if no interrupts are being injected. Note that this
-	 * injected. Note that this also applies if we don't expect
+	 * also applies if we don't expect any system register access (no
-	 * any system register access (no vgic at all).
+	 * vgic at all). In any case, no need to provide MI configuration.
 	 */
 	if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
 	    cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre)
-		write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
+		write_gicreg(vgic_ich_hcr_trap_bits() | ICH_HCR_EL2_En, ICH_HCR_EL2);
 }
 void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if)
 {
 	u64 val;
-	if (!cpu_if->vgic_sre) {
-		cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
-	}
 	/* Only restore SRE if the host implements the GICv2 interface */
 	if (static_branch_unlikely(&vgic_v3_has_v2_compat)) {
 		val = read_gicreg(ICC_SRE_EL2);
 		write_gicreg(0, ICH_HCR_EL2);
 }
-static void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if)
+void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if)
 {
 	u64 val;
 	u32 nr_pre_bits;
 	write_gicreg(vmcr, ICH_VMCR_EL2);
 }
-void __vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
-{
-	__vgic_v3_save_aprs(cpu_if);
-	if (cpu_if->vgic_sre)
-		cpu_if->vgic_vmcr = __vgic_v3_read_vmcr();
-}
 void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
 {
 	__vgic_v3_compat_mode_enable();
 	write_gicreg(hcr, ICH_HCR_EL2);
 }
-static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
+static int ___vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 {
 	u32 vid = vcpu_get_reg(vcpu, rt);
 	u64 lr_val;
 	/* EOImode == 0, nothing to be done here */
 	if (!(vmcr & ICH_VMCR_EOIM_MASK))
-		return;
+		return 1;
 	/* No deactivate to be performed on an LPI */
 	if (vid >= VGIC_MIN_LPI)
-		return;
+		return 1;
 	lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
-	if (lr == -1) {
+	if (lr != -1) {
-		__vgic_v3_bump_eoicount();
+		__vgic_v3_clear_active_lr(lr, lr_val);
-		return;
+		return 1;
 	}
-	__vgic_v3_clear_active_lr(lr, lr_val);
+	return 0;
+}
+static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
+{
+	if (!___vgic_v3_write_dir(vcpu, vmcr, rt))
+		__vgic_v3_bump_eoicount();
 }
 static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 	case SYS_ICC_DIR_EL1:
 		if (unlikely(is_read))
 			return 0;
+		/*
+		 * Full exit if required to handle overflow deactivation,
+		 * unless we can emulate it in the LRs (likely the majority
+		 * of the cases).
+		 */
+		if (vcpu->arch.vgic_cpu.vgic_v3.vgic_hcr & ICH_HCR_EL2_TDIR) {
+			int ret;
+			ret = ___vgic_v3_write_dir(vcpu, __vgic_v3_read_vmcr(),
+						   kvm_vcpu_sys_get_rt(vcpu));
+			if (ret)
+				__kvm_skip_instr(vcpu);
+			return ret;
+		}
 		fn = __vgic_v3_write_dir;
 		break;
 	case SYS_ICC_RPR_EL1:

arch/arm64/kvm/mmu.c+124−8

@@ -904,6 +904,38 @@ static int kvm_init_ipa_range(struct kvm_s2_mmu *mmu, unsigned long type) · 903 unmodified lines
904	return 0;	904	return 0;
905	}	905	}
906		906
		907	/*
		908	* Assume that @pgt is valid and unlinked from the KVM MMU to free the
		909	* page-table without taking the kvm_mmu_lock and without performing any
		910	* TLB invalidations.
		911	*
		912	* Also, the range of addresses can be large enough to cause need_resched
		913	* warnings, for instance on CONFIG_PREEMPT_NONE kernels. Hence, invoke
		914	* cond_resched() periodically to prevent hogging the CPU for a long time
		915	* and schedule something else, if required.
		916	*/
		917	static void stage2_destroy_range(struct kvm_pgtable *pgt, phys_addr_t addr,
		918	phys_addr_t end)
		919	{
		920	u64 next;
		921
		922	do {
		923	next = stage2_range_addr_end(addr, end);
		924	KVM_PGT_FN(kvm_pgtable_stage2_destroy_range)(pgt, addr,
		925	next - addr);
		926	if (next != end)
		927	cond_resched();
		928	} while (addr = next, addr != end);
		929	}
		930
		931	static void kvm_stage2_destroy(struct kvm_pgtable *pgt)
		932	{
		933	unsigned int ia_bits = VTCR_EL2_IPA(pgt->mmu->vtcr);
		934
		935	stage2_destroy_range(pgt, 0, BIT(ia_bits));
		936	KVM_PGT_FN(kvm_pgtable_stage2_destroy_pgd)(pgt);
		937	}
		938
907	/**	939	/**
908	* kvm_init_stage2_mmu - Initialise a S2 MMU structure	940	* kvm_init_stage2_mmu - Initialise a S2 MMU structure
909	* @kvm: The pointer to the KVM structure	941	* @kvm: The pointer to the KVM structure
@@ -980,7 +1012,7 @@ int kvm_init_stage2_mmu(struct kvm kvm, struct kvm_s2_mmu mmu, unsigned long t · 70 unmodified lines
980	return 0;	1012	return 0;
981		1013
982	out_destroy_pgtable:	1014	out_destroy_pgtable:
983	KVM_PGT_FN(kvm_pgtable_stage2_destroy)(pgt);	1015	kvm_stage2_destroy(pgt);
984	out_free_pgtable:	1016	out_free_pgtable:
985	kfree(pgt);	1017	kfree(pgt);
986	return err;	1018	return err;
@@ -1081,7 +1113,7 @@ void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu) · 94 unmodified lines
1081	write_unlock(&kvm->mmu_lock);	1113	write_unlock(&kvm->mmu_lock);
1082		1114
1083	if (pgt) {	1115	if (pgt) {
1084	KVM_PGT_FN(kvm_pgtable_stage2_destroy)(pgt);	1116	kvm_stage2_destroy(pgt);
1085	kfree(pgt);	1117	kfree(pgt);
1086	}	1118	}
1087	}	1119	}
@@ -1521,6 +1553,16 @@ static void adjust_nested_fault_perms(struct kvm_s2_trans *nested, · 433 unmodified lines
1521	*prot \|= kvm_encode_nested_level(nested);	1553	*prot \|= kvm_encode_nested_level(nested);
1522	}	1554	}
1523		1555
		1556	static void adjust_nested_exec_perms(struct kvm *kvm,
		1557	struct kvm_s2_trans *nested,
		1558	enum kvm_pgtable_prot *prot)
		1559	{
		1560	if (!kvm_s2_trans_exec_el0(kvm, nested))
		1561	*prot &= ~KVM_PGTABLE_PROT_UX;
		1562	if (!kvm_s2_trans_exec_el1(kvm, nested))
		1563	*prot &= ~KVM_PGTABLE_PROT_PX;
		1564	}
		1565
1524	#define KVM_PGTABLE_WALK_MEMABORT_FLAGS (KVM_PGTABLE_WALK_HANDLE_FAULT \| KVM_PGTABLE_WALK_SHARED)	1566	#define KVM_PGTABLE_WALK_MEMABORT_FLAGS (KVM_PGTABLE_WALK_HANDLE_FAULT \| KVM_PGTABLE_WALK_SHARED)
1525		1567
1526	static int gmem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,	1568	static int gmem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
@@ -1572,11 +1614,12 @@ static int gmem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 45 unmodified lines
1572	if (writable)	1614	if (writable)
1573	prot \|= KVM_PGTABLE_PROT_W;	1615	prot \|= KVM_PGTABLE_PROT_W;
1574		1616
1575	if (exec_fault \|\|	1617	if (exec_fault \|\| cpus_have_final_cap(ARM64_HAS_CACHE_DIC))
1576	(cpus_have_final_cap(ARM64_HAS_CACHE_DIC) &&
1577	(!nested \|\| kvm_s2_trans_executable(nested))))
1578	prot \|= KVM_PGTABLE_PROT_X;	1618	prot \|= KVM_PGTABLE_PROT_X;
1579		1619
		1620	if (nested)
		1621	adjust_nested_exec_perms(kvm, nested, &prot);
		1622
1580	kvm_fault_lock(kvm);	1623	kvm_fault_lock(kvm);
1581	if (mmu_invalidate_retry(kvm, mmu_seq)) {	1624	if (mmu_invalidate_retry(kvm, mmu_seq)) {
1582	ret = -EAGAIN;	1625	ret = -EAGAIN;
@@ -1851,11 +1894,13 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 268 unmodified lines
1851	prot \|= KVM_PGTABLE_PROT_NORMAL_NC;	1894	prot \|= KVM_PGTABLE_PROT_NORMAL_NC;
1852	else	1895	else
1853	prot \|= KVM_PGTABLE_PROT_DEVICE;	1896	prot \|= KVM_PGTABLE_PROT_DEVICE;
1854	} else if (cpus_have_final_cap(ARM64_HAS_CACHE_DIC) &&	1897	} else if (cpus_have_final_cap(ARM64_HAS_CACHE_DIC)) {
1855	(!nested \|\| kvm_s2_trans_executable(nested))) {
1856	prot \|= KVM_PGTABLE_PROT_X;	1898	prot \|= KVM_PGTABLE_PROT_X;
1857	}	1899	}
1858		1900
		1901	if (nested)
		1902	adjust_nested_exec_perms(kvm, nested, &prot);
		1903
1859	/*	1904	/*
1860	* Under the premise of getting a FSC_PERM fault, we just need to relax	1905	* Under the premise of getting a FSC_PERM fault, we just need to relax
1861	* permissions only if vma_pagesize equals fault_granule. Otherwise,	1906	* permissions only if vma_pagesize equals fault_granule. Otherwise,
@@ -1899,8 +1944,48 @@ static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa) · 37 unmodified lines
1899	read_unlock(&vcpu->kvm->mmu_lock);	1944	read_unlock(&vcpu->kvm->mmu_lock);
1900	}	1945	}
1901		1946
		1947	/*
		1948	* Returns true if the SEA should be handled locally within KVM if the abort
		1949	* is caused by a kernel memory allocation (e.g. stage-2 table memory).
		1950	*/
		1951	static bool host_owns_sea(struct kvm_vcpu *vcpu, u64 esr)
		1952	{
		1953	/*
		1954	* Without FEAT_RAS HCR_EL2.TEA is RES0, meaning any external abort
		1955	* taken from a guest EL to EL2 is due to a host-imposed access (e.g.
		1956	* stage-2 PTW).
		1957	*/
		1958	if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
		1959	return true;
		1960
		1961	/* KVM owns the VNCR when the vCPU isn't in a nested context. */
		1962	if (is_hyp_ctxt(vcpu) && !kvm_vcpu_trap_is_iabt(vcpu) && (esr & ESR_ELx_VNCR))
		1963	return true;
		1964
		1965	/*
		1966	* Determining if an external abort during a table walk happened at
		1967	* stage-2 is only possible with S1PTW is set. Otherwise, since KVM
		1968	* sets HCR_EL2.TEA, SEAs due to a stage-1 walk (i.e. accessing the
		1969	* PA of the stage-1 descriptor) can reach here and are reported
		1970	* with a TTW ESR value.
		1971	*/
		1972	return (esr_fsc_is_sea_ttw(esr) && (esr & ESR_ELx_S1PTW));
		1973	}
		1974
1902	int kvm_handle_guest_sea(struct kvm_vcpu *vcpu)	1975	int kvm_handle_guest_sea(struct kvm_vcpu *vcpu)
1903	{	1976	{
		1977	struct kvm *kvm = vcpu->kvm;
		1978	struct kvm_run *run = vcpu->run;
		1979	u64 esr = kvm_vcpu_get_esr(vcpu);
		1980	u64 esr_mask = ESR_ELx_EC_MASK \|
		1981	ESR_ELx_IL \|
		1982	ESR_ELx_FnV \|
		1983	ESR_ELx_EA \|
		1984	ESR_ELx_CM \|
		1985	ESR_ELx_WNR \|
		1986	ESR_ELx_FSC;
		1987	u64 ipa;
		1988
1904	/*	1989	/*
1905	* Give APEI the opportunity to claim the abort before handling it	1990	* Give APEI the opportunity to claim the abort before handling it
1906	* within KVM. apei_claim_sea() expects to be called with IRQs enabled.	1991	* within KVM. apei_claim_sea() expects to be called with IRQs enabled.
@@ -1909,7 +1994,33 @@ int kvm_handle_guest_sea(struct kvm_vcpu *vcpu) · 2 unmodified lines
1909	if (apei_claim_sea(NULL) == 0)	1994	if (apei_claim_sea(NULL) == 0)
1910	return 1;	1995	return 1;
1911		1996
1912	return kvm_inject_serror(vcpu);	1997	if (host_owns_sea(vcpu, esr) \|\|
		1998	!test_bit(KVM_ARCH_FLAG_EXIT_SEA, &vcpu->kvm->arch.flags))
		1999	return kvm_inject_serror(vcpu);
		2000
		2001	/* ESR_ELx.SET is RES0 when FEAT_RAS isn't implemented. */
		2002	if (kvm_has_ras(kvm))
		2003	esr_mask \|= ESR_ELx_SET_MASK;
		2004
		2005	/*
		2006	* Exit to userspace, and provide faulting guest virtual and physical
		2007	* addresses in case userspace wants to emulate SEA to guest by
		2008	* writing to FAR_ELx and HPFAR_ELx registers.
		2009	*/
		2010	memset(&run->arm_sea, 0, sizeof(run->arm_sea));
		2011	run->exit_reason = KVM_EXIT_ARM_SEA;
		2012	run->arm_sea.esr = esr & esr_mask;
		2013
		2014	if (!(esr & ESR_ELx_FnV))
		2015	run->arm_sea.gva = kvm_vcpu_get_hfar(vcpu);
		2016
		2017	ipa = kvm_vcpu_get_fault_ipa(vcpu);
		2018	if (ipa != INVALID_GPA) {
		2019	run->arm_sea.flags \|= KVM_EXIT_ARM_SEA_FLAG_GPA_VALID;
		2020	run->arm_sea.gpa = ipa;
		2021	}
		2022
		2023	return 0;
1913	}	2024	}
1914		2025
1915	/**	2026	/**
@@ -1999,6 +2110,11 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) · 83 unmodified lines
1999	u32 esr;	2110	u32 esr;
2000		2111
2001	ret = kvm_walk_nested_s2(vcpu, fault_ipa, &nested_trans);	2112	ret = kvm_walk_nested_s2(vcpu, fault_ipa, &nested_trans);
		2113	if (ret == -EAGAIN) {
		2114	ret = 1;
		2115	goto out_unlock;
		2116	}
		2117
2002	if (ret) {	2118	if (ret) {
2003	esr = kvm_s2_trans_esr(&nested_trans);	2119	esr = kvm_s2_trans_esr(&nested_trans);
2004	kvm_inject_s2_fault(vcpu, esr);	2120	kvm_inject_s2_fault(vcpu, esr);

arch/arm64/kvm/nested.c+84−39

@@ -124,14 +124,13 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) · 123 unmodified lines
124	}	124	}
125		125
126	struct s2_walk_info {	126	struct s2_walk_info {
127	int (read_desc)(phys_addr_t pa, u64 desc, void *data);	127	u64 baddr;
128	void *data;	128	unsigned int max_oa_bits;
129	u64 baddr;	129	unsigned int pgshift;
130	unsigned int max_oa_bits;	130	unsigned int sl;
131	unsigned int pgshift;	131	unsigned int t0sz;
132	unsigned int sl;	132	bool be;
133	unsigned int t0sz;	133	bool ha;
134	bool be;
135	};	134	};
136		135
137	static u32 compute_fsc(int level, u32 fsc)	136	static u32 compute_fsc(int level, u32 fsc)
@@ -199,6 +198,42 @@ static int check_output_size(struct s2_walk_info *wi, phys_addr_t output) · 61 unmodified lines
199	return 0;	198	return 0;
200	}	199	}
201		200
		201	static int read_guest_s2_desc(struct kvm_vcpu vcpu, phys_addr_t pa, u64 desc,
		202	struct s2_walk_info *wi)
		203	{
		204	u64 val;
		205	int r;
		206
		207	r = kvm_read_guest(vcpu->kvm, pa, &val, sizeof(val));
		208	if (r)
		209	return r;
		210
		211	/*
		212	* Handle reversedescriptors if endianness differs between the
		213	* host and the guest hypervisor.
		214	*/
		215	if (wi->be)
		216	*desc = be64_to_cpu((__force __be64)val);
		217	else
		218	*desc = le64_to_cpu((__force __le64)val);
		219
		220	return 0;
		221	}
		222
		223	static int swap_guest_s2_desc(struct kvm_vcpu *vcpu, phys_addr_t pa, u64 old, u64 new,
		224	struct s2_walk_info *wi)
		225	{
		226	if (wi->be) {
		227	old = (__force u64)cpu_to_be64(old);
		228	new = (__force u64)cpu_to_be64(new);
		229	} else {
		230	old = (__force u64)cpu_to_le64(old);
		231	new = (__force u64)cpu_to_le64(new);
		232	}
		233
		234	return __kvm_at_swap_desc(vcpu->kvm, pa, old, new);
		235	}
		236
202	/*	237	/*
203	* This is essentially a C-version of the pseudo code from the ARM ARM	238	* This is essentially a C-version of the pseudo code from the ARM ARM
204	* AArch64.TranslationTableWalk function. I strongly recommend looking at	239	* AArch64.TranslationTableWalk function. I strongly recommend looking at
@@ -206,13 +241,13 @@ static int check_output_size(struct s2_walk_info *wi, phys_addr_t output) · 1 unmodified line
206	*	241	*
207	* Must be called with the kvm->srcu read lock held	242	* Must be called with the kvm->srcu read lock held
208	*/	243	*/
209	static int walk_nested_s2_pgd(phys_addr_t ipa,	244	static int walk_nested_s2_pgd(struct kvm_vcpu *vcpu, phys_addr_t ipa,
210	struct s2_walk_info wi, struct kvm_s2_trans out)	245	struct s2_walk_info wi, struct kvm_s2_trans out)
211	{	246	{
212	int first_block_level, level, stride, input_size, base_lower_bound;	247	int first_block_level, level, stride, input_size, base_lower_bound;
213	phys_addr_t base_addr;	248	phys_addr_t base_addr;
214	unsigned int addr_top, addr_bottom;	249	unsigned int addr_top, addr_bottom;
215	u64 desc; /* page table entry */	250	u64 desc, new_desc; /* page table entry */
216	int ret;	251	int ret;
217	phys_addr_t paddr;	252	phys_addr_t paddr;
218		253
@@ -257,28 +292,30 @@ static int walk_nested_s2_pgd(phys_addr_t ipa, · 38 unmodified lines
257	>> (addr_bottom - 3);	292	>> (addr_bottom - 3);
258		293
259	paddr = base_addr \| index;	294	paddr = base_addr \| index;
260	ret = wi->read_desc(paddr, &desc, wi->data);	295	ret = read_guest_s2_desc(vcpu, paddr, &desc, wi);
261	if (ret < 0)	296	if (ret < 0)
262	return ret;	297	return ret;
263		298
264	/*	299	new_desc = desc;
265	* Handle reversedescriptors if endianness differs between the
266	* host and the guest hypervisor.
267	*/
268	if (wi->be)
269	desc = be64_to_cpu((__force __be64)desc);
270	else
271	desc = le64_to_cpu((__force __le64)desc);
272		300
273	/* Check for valid descriptor at this point */	301	/* Check for valid descriptor at this point */
274	if (!(desc & 1) \|\| ((desc & 3) == 1 && level == 3)) {	302	if (!(desc & KVM_PTE_VALID)) {
275	out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);	303	out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);
276	out->desc = desc;	304	out->desc = desc;
277	return 1;	305	return 1;
278	}	306	}
279		307
280	/* We're at the final level or block translation level */	308	if (FIELD_GET(KVM_PTE_TYPE, desc) == KVM_PTE_TYPE_BLOCK) {
281	if ((desc & 3) == 1 \|\| level == 3)	309	if (level < 3)
		310	break;
		311
		312	out->esr = compute_fsc(level, ESR_ELx_FSC_FAULT);
		313	out->desc = desc;
		314	return 1;
		315	}
		316
		317	/* We're at the final level */
		318	if (level == 3)
282	break;	319	break;
283		320
284	if (check_output_size(wi, desc)) {	321	if (check_output_size(wi, desc)) {
@@ -305,7 +342,18 @@ static int walk_nested_s2_pgd(phys_addr_t ipa, · 20 unmodified lines
305	return 1;	342	return 1;
306	}	343	}
307		344
308	if (!(desc & BIT(10))) {	345	if (wi->ha)
		346	new_desc \|= KVM_PTE_LEAF_ATTR_LO_S2_AF;
		347
		348	if (new_desc != desc) {
		349	ret = swap_guest_s2_desc(vcpu, paddr, desc, new_desc, wi);
		350	if (ret)
		351	return ret;
		352
		353	desc = new_desc;
		354	}
		355
		356	if (!(desc & KVM_PTE_LEAF_ATTR_LO_S2_AF)) {
309	out->esr = compute_fsc(level, ESR_ELx_FSC_ACCESS);	357	out->esr = compute_fsc(level, ESR_ELx_FSC_ACCESS);
310	out->desc = desc;	358	out->desc = desc;
311	return 1;	359	return 1;
@@ -318,20 +366,13 @@ static int walk_nested_s2_pgd(phys_addr_t ipa, · 6 unmodified lines
318	(ipa & GENMASK_ULL(addr_bottom - 1, 0));	366	(ipa & GENMASK_ULL(addr_bottom - 1, 0));
319	out->output = paddr;	367	out->output = paddr;
320	out->block_size = 1UL << ((3 - level) * stride + wi->pgshift);	368	out->block_size = 1UL << ((3 - level) * stride + wi->pgshift);
321	out->readable = desc & (0b01 << 6);	369	out->readable = desc & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R;
322	out->writable = desc & (0b10 << 6);	370	out->writable = desc & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W;
323	out->level = level;	371	out->level = level;
324	out->desc = desc;	372	out->desc = desc;
325	return 0;	373	return 0;
326	}	374	}
327		375
328	static int read_guest_s2_desc(phys_addr_t pa, u64 desc, void data)
329	{
330	struct kvm_vcpu *vcpu = data;
331
332	return kvm_read_guest(vcpu->kvm, pa, desc, sizeof(*desc));
333	}
334
335	static void vtcr_to_walk_info(u64 vtcr, struct s2_walk_info *wi)	376	static void vtcr_to_walk_info(u64 vtcr, struct s2_walk_info *wi)
336	{	377	{
337	wi->t0sz = vtcr & TCR_EL2_T0SZ_MASK;	378	wi->t0sz = vtcr & TCR_EL2_T0SZ_MASK;
@@ -350,6 +391,8 @@ static void vtcr_to_walk_info(u64 vtcr, struct s2_walk_info *wi) · 12 unmodified lines
350	/* Global limit for now, should eventually be per-VM */	391	/* Global limit for now, should eventually be per-VM */
351	wi->max_oa_bits = min(get_kvm_ipa_limit(),	392	wi->max_oa_bits = min(get_kvm_ipa_limit(),
352	ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr), false));	393	ps_to_output_size(FIELD_GET(VTCR_EL2_PS_MASK, vtcr), false));
		394
		395	wi->ha = vtcr & VTCR_EL2_HA;
353	}	396	}
354		397
355	int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,	398	int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
@@ -364,15 +407,13 @@ int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, · 8 unmodified lines
364	if (!vcpu_has_nv(vcpu))	407	if (!vcpu_has_nv(vcpu))
365	return 0;	408	return 0;
366		409
367	wi.read_desc = read_guest_s2_desc;
368	wi.data = vcpu;
369	wi.baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);	410	wi.baddr = vcpu_read_sys_reg(vcpu, VTTBR_EL2);
370		411
371	vtcr_to_walk_info(vtcr, &wi);	412	vtcr_to_walk_info(vtcr, &wi);
372		413
373	wi.be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE;	414	wi.be = vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_EE;
374		415
375	ret = walk_nested_s2_pgd(gipa, &wi, result);	416	ret = walk_nested_s2_pgd(vcpu, gipa, &wi, result);
376	if (ret)	417	if (ret)
377	result->esr \|= (kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC);	418	result->esr \|= (kvm_vcpu_get_esr(vcpu) & ~ESR_ELx_FSC);
378		419
@@ -788,7 +829,10 @@ int kvm_s2_handle_perm_fault(struct kvm_vcpu vcpu, struct kvm_s2_trans trans) · 409 unmodified lines
788	return 0;	829	return 0;
789		830
790	if (kvm_vcpu_trap_is_iabt(vcpu)) {	831	if (kvm_vcpu_trap_is_iabt(vcpu)) {
791	forward_fault = !kvm_s2_trans_executable(trans);	832	if (vcpu_mode_priv(vcpu))
		833	forward_fault = !kvm_s2_trans_exec_el1(vcpu->kvm, trans);
		834	else
		835	forward_fault = !kvm_s2_trans_exec_el0(vcpu->kvm, trans);
792	} else {	836	} else {
793	bool write_fault = kvm_is_write_fault(vcpu);	837	bool write_fault = kvm_is_write_fault(vcpu);
794		838
@@ -1555,12 +1599,13 @@ u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val) · 760 unmodified lines
1555	case SYS_ID_AA64MMFR1_EL1:	1599	case SYS_ID_AA64MMFR1_EL1:
1556	val &= ~(ID_AA64MMFR1_EL1_CMOW \|	1600	val &= ~(ID_AA64MMFR1_EL1_CMOW \|
1557	ID_AA64MMFR1_EL1_nTLBPA \|	1601	ID_AA64MMFR1_EL1_nTLBPA \|
1558	ID_AA64MMFR1_EL1_ETS \|	1602	ID_AA64MMFR1_EL1_ETS);
1559	ID_AA64MMFR1_EL1_XNX \|	1603
1560	ID_AA64MMFR1_EL1_HAFDBS);
1561	/* FEAT_E2H0 implies no VHE */	1604	/* FEAT_E2H0 implies no VHE */
1562	if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features))	1605	if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features))
1563	val &= ~ID_AA64MMFR1_EL1_VH;	1606	val &= ~ID_AA64MMFR1_EL1_VH;
		1607
		1608	val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR1_EL1, HAFDBS, AF);
1564	break;	1609	break;
1565		1610
1566	case SYS_ID_AA64MMFR2_EL1:	1611	case SYS_ID_AA64MMFR2_EL1:

arch/arm64/kvm/pkvm.c+9−2

 	return 0;
 }
-void pkvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt)
+void pkvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt,
+					u64 addr, u64 size)
 {
-	__pkvm_pgtable_stage2_unmap(pgt, 0, ~(0ULL));
+	__pkvm_pgtable_stage2_unmap(pgt, addr, addr + size);
+}
+void pkvm_pgtable_stage2_destroy_pgd(struct kvm_pgtable *pgt)
+{
+	/* Expected to be called after all pKVM mappings have been released. */
+	WARN_ON_ONCE(!RB_EMPTY_ROOT(&pgt->pkvm_mappings.rb_root));
 }
 int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,

arch/arm64/kvm/ptdump.c+27−8

 		.val	= PTE_VALID,
 		.set	= " ",
 		.clear	= "F",
-	}, {
+	},
+	{
 		.mask	= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R,
 		.val	= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R,
 		.set	= "R",
 		.clear	= " ",
-	}, {
+	},
+	{
 		.mask	= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W,
 		.val	= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W,
 		.set	= "W",
 		.clear	= " ",
-	}, {
+	},
+	{
 		.mask	= KVM_PTE_LEAF_ATTR_HI_S2_XN,
-		.val	= KVM_PTE_LEAF_ATTR_HI_S2_XN,
+		.val	= 0b00UL << __bf_shf(KVM_PTE_LEAF_ATTR_HI_S2_XN),
-		.set	= "NX",
+		.set	= "px ux ",
-		.clear	= "x ",
+	},
-	}, {
+	{
+		.mask	= KVM_PTE_LEAF_ATTR_HI_S2_XN,
+		.val	= 0b01UL << __bf_shf(KVM_PTE_LEAF_ATTR_HI_S2_XN),
+		.set	= "PXNux ",
+	},
+	{
+		.mask	= KVM_PTE_LEAF_ATTR_HI_S2_XN,
+		.val	= 0b10UL << __bf_shf(KVM_PTE_LEAF_ATTR_HI_S2_XN),
+		.set	= "PXNUXN",
+	},
+	{
+		.mask	= KVM_PTE_LEAF_ATTR_HI_S2_XN,
+		.val	= 0b11UL << __bf_shf(KVM_PTE_LEAF_ATTR_HI_S2_XN),
+		.set	= "px UXN",
+	},
+	{
 		.mask	= KVM_PTE_LEAF_ATTR_LO_S2_AF,
 		.val	= KVM_PTE_LEAF_ATTR_LO_S2_AF,
 		.set	= "AF",
 		.clear	= "  ",
-	}, {
+	},
+	{
 		.mask	= PMD_TYPE_MASK,
 		.val	= PMD_TYPE_SECT,
 		.set	= "BLK",

arch/arm64/kvm/sys_regs.c+23−5

 	return true;
 }
+static bool access_gic_dir(struct kvm_vcpu *vcpu,
+			   struct sys_reg_params *p,
+			   const struct sys_reg_desc *r)
+{
+	if (!kvm_has_gicv3(vcpu->kvm))
+		return undef_access(vcpu, p, r);
+	if (!p->is_write)
+		return undef_access(vcpu, p, r);
+	vgic_v3_deactivate(vcpu, p->regval);
+	return true;
+}
 static bool trap_raz_wi(struct kvm_vcpu *vcpu,
 			struct sys_reg_params *p,
 			const struct sys_reg_desc *r)
 	{ SYS_DESC(SYS_ICC_AP1R1_EL1), undef_access },
 	{ SYS_DESC(SYS_ICC_AP1R2_EL1), undef_access },
 	{ SYS_DESC(SYS_ICC_AP1R3_EL1), undef_access },
-	{ SYS_DESC(SYS_ICC_DIR_EL1), undef_access },
+	{ SYS_DESC(SYS_ICC_DIR_EL1), access_gic_dir },
 	{ SYS_DESC(SYS_ICC_RPR_EL1), undef_access },
 	{ SYS_DESC(SYS_ICC_SGI1R_EL1), access_gic_sgi },
 	{ SYS_DESC(SYS_ICC_ASGI1R_EL1), access_gic_sgi },
 {
 	u32 op = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2);
-	__kvm_at_s1e01(vcpu, op, p->regval);
+	if (__kvm_at_s1e01(vcpu, op, p->regval))
+		return false;
 	return true;
 }
 		return false;
 	}
-	__kvm_at_s1e2(vcpu, op, p->regval);
+	if (__kvm_at_s1e2(vcpu, op, p->regval))
+		return false;
 	return true;
 }
 {
 	u32 op = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2);
-	__kvm_at_s12(vcpu, op, p->regval);
+	if (__kvm_at_s12(vcpu, op, p->regval))
+		return false;
 	return true;
 }
 	{ CP15_SYS_DESC(SYS_ICC_AP1R1_EL1), undef_access },
 	{ CP15_SYS_DESC(SYS_ICC_AP1R2_EL1), undef_access },
 	{ CP15_SYS_DESC(SYS_ICC_AP1R3_EL1), undef_access },
-	{ CP15_SYS_DESC(SYS_ICC_DIR_EL1), undef_access },
+	{ CP15_SYS_DESC(SYS_ICC_DIR_EL1), access_gic_dir },
 	{ CP15_SYS_DESC(SYS_ICC_RPR_EL1), undef_access },
 	{ CP15_SYS_DESC(SYS_ICC_IAR1_EL1), undef_access },
 	{ CP15_SYS_DESC(SYS_ICC_EOIR1_EL1), undef_access },

arch/arm64/kvm/vgic/vgic-init.c+5−4

 	struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
 	int i;
+	dist->active_spis = (atomic_t)ATOMIC_INIT(0);
 	dist->spis = kcalloc(nr_spis, sizeof(struct vgic_irq), GFP_KERNEL_ACCOUNT);
 	if (!dist->spis)
 		return  -ENOMEM;
 	return ret;
 }
-static void kvm_vgic_vcpu_enable(struct kvm_vcpu *vcpu)
+static void kvm_vgic_vcpu_reset(struct kvm_vcpu *vcpu)
 {
 	if (kvm_vgic_global_state.type == VGIC_V2)
-		vgic_v2_enable(vcpu);
+		vgic_v2_reset(vcpu);
 	else
-		vgic_v3_enable(vcpu);
+		vgic_v3_reset(vcpu);
 }
 /*
 	}
 	kvm_for_each_vcpu(idx, vcpu, kvm)
-		kvm_vgic_vcpu_enable(vcpu);
+		kvm_vgic_vcpu_reset(vcpu);
 	ret = kvm_vgic_setup_default_irq_routing(kvm);
 	if (ret)

arch/arm64/kvm/vgic/vgic-mmio-v2.c+24−0

@@ -359,6 +359,16 @@ static void vgic_mmio_write_vcpuif(struct kvm_vcpu *vcpu, · 358 unmodified lines
359	vgic_set_vmcr(vcpu, &vmcr);	359	vgic_set_vmcr(vcpu, &vmcr);
360	}	360	}
361		361
		362	static void vgic_mmio_write_dir(struct kvm_vcpu *vcpu,
		363	gpa_t addr, unsigned int len,
		364	unsigned long val)
		365	{
		366	if (kvm_vgic_global_state.type == VGIC_V2)
		367	vgic_v2_deactivate(vcpu, val);
		368	else
		369	vgic_v3_deactivate(vcpu, val);
		370	}
		371
362	static unsigned long vgic_mmio_read_apr(struct kvm_vcpu *vcpu,	372	static unsigned long vgic_mmio_read_apr(struct kvm_vcpu *vcpu,
363	gpa_t addr, unsigned int len)	373	gpa_t addr, unsigned int len)
364	{	374	{
@@ -482,6 +492,10 @@ static const struct vgic_register_region vgic_v2_cpu_registers[] = { · 117 unmodified lines
482	REGISTER_DESC_WITH_LENGTH(GIC_CPU_IDENT,	492	REGISTER_DESC_WITH_LENGTH(GIC_CPU_IDENT,
483	vgic_mmio_read_vcpuif, vgic_mmio_write_vcpuif, 4,	493	vgic_mmio_read_vcpuif, vgic_mmio_write_vcpuif, 4,
484	VGIC_ACCESS_32bit),	494	VGIC_ACCESS_32bit),
		495	REGISTER_DESC_WITH_LENGTH_UACCESS(GIC_CPU_DEACTIVATE,
		496	vgic_mmio_read_raz, vgic_mmio_write_dir,
		497	vgic_mmio_read_raz, vgic_mmio_uaccess_write_wi,
		498	4, VGIC_ACCESS_32bit),
485	};	499	};
486		500
487	unsigned int vgic_v2_init_dist_iodev(struct vgic_io_device *dev)	501	unsigned int vgic_v2_init_dist_iodev(struct vgic_io_device *dev)
@@ -494,6 +508,16 @@ unsigned int vgic_v2_init_dist_iodev(struct vgic_io_device *dev) · 6 unmodified lines
494	return SZ_4K;	508	return SZ_4K;
495	}	509	}
496		510
		511	unsigned int vgic_v2_init_cpuif_iodev(struct vgic_io_device *dev)
		512	{
		513	dev->regions = vgic_v2_cpu_registers;
		514	dev->nr_regions = ARRAY_SIZE(vgic_v2_cpu_registers);
		515
		516	kvm_iodevice_init(&dev->dev, &kvm_io_gic_ops);
		517
		518	return KVM_VGIC_V2_CPU_SIZE;
		519	}
		520
497	int vgic_v2_has_attr_regs(struct kvm_device dev, struct kvm_device_attr attr)	521	int vgic_v2_has_attr_regs(struct kvm_device dev, struct kvm_device_attr attr)
498	{	522	{
499	const struct vgic_register_region *region;	523	const struct vgic_register_region *region;

arch/arm64/kvm/vgic/vgic-mmio.h+1−0

@@ -213,6 +213,7 @@ void vgic_write_irq_line_level_info(struct kvm_vcpu *vcpu, u32 intid, · 212 unmodified lines
213	const u32 val);	213	const u32 val);
214		214
215	unsigned int vgic_v2_init_dist_iodev(struct vgic_io_device *dev);	215	unsigned int vgic_v2_init_dist_iodev(struct vgic_io_device *dev);
		216	unsigned int vgic_v2_init_cpuif_iodev(struct vgic_io_device *dev);
216		217
217	unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev);	218	unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev);
218		219

arch/arm64/kvm/vgic/vgic-v2.c+220−71

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	#include <kvm/arm_vgic.h>	9	#include <kvm/arm_vgic.h>
10	#include <asm/kvm_mmu.h>	10	#include <asm/kvm_mmu.h>
11		11
		12	#include "vgic-mmio.h"
12	#include "vgic.h"	13	#include "vgic.h"
13		14
14	static inline void vgic_v2_write_lr(int lr, u32 val)	15	static inline void vgic_v2_write_lr(int lr, u32 val)
@@ -26,11 +27,24 @@ void vgic_v2_init_lrs(void) · 11 unmodified lines
26	vgic_v2_write_lr(i, 0);	27	vgic_v2_write_lr(i, 0);
27	}	28	}
28		29
29	void vgic_v2_set_underflow(struct kvm_vcpu *vcpu)	30	void vgic_v2_configure_hcr(struct kvm_vcpu *vcpu,
		31	struct ap_list_summary *als)
30	{	32	{
31	struct vgic_v2_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v2;	33	struct vgic_v2_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v2;
32		34
33	cpuif->vgic_hcr \|= GICH_HCR_UIE;	35	cpuif->vgic_hcr = GICH_HCR_EN;
		36
		37	if (irqs_pending_outside_lrs(als))
		38	cpuif->vgic_hcr \|= GICH_HCR_NPIE;
		39	if (irqs_active_outside_lrs(als))
		40	cpuif->vgic_hcr \|= GICH_HCR_LRENPIE;
		41	if (irqs_outside_lrs(als))
		42	cpuif->vgic_hcr \|= GICH_HCR_UIE;
		43
		44	cpuif->vgic_hcr \|= (cpuif->vgic_vmcr & GICH_VMCR_ENABLE_GRP0_MASK) ?
		45	GICH_HCR_VGrp0DIE : GICH_HCR_VGrp0EIE;
		46	cpuif->vgic_hcr \|= (cpuif->vgic_vmcr & GICH_VMCR_ENABLE_GRP1_MASK) ?
		47	GICH_HCR_VGrp1DIE : GICH_HCR_VGrp1EIE;
34	}	48	}
35		49
36	static bool lr_signals_eoi_mi(u32 lr_val)	50	static bool lr_signals_eoi_mi(u32 lr_val)
@@ -39,43 +53,23 @@ static bool lr_signals_eoi_mi(u32 lr_val) · 2 unmodified lines
39	!(lr_val & GICH_LR_HW);	53	!(lr_val & GICH_LR_HW);
40	}	54	}
41		55
42	/*	56	static void vgic_v2_fold_lr(struct kvm_vcpu *vcpu, u32 val)
43	* transfer the content of the LRs back into the corresponding ap_list:
44	* - active bit is transferred as is
45	* - pending bit is
46	* - transferred as is in case of edge sensitive IRQs
47	* - set to the line-level (resample time) for level sensitive IRQs
48	*/
49	void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu)
50	{	57	{
51	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;	58	u32 cpuid, intid = val & GICH_LR_VIRTUALID;
52	struct vgic_v2_cpu_if *cpuif = &vgic_cpu->vgic_v2;	59	struct vgic_irq *irq;
53	int lr;	60	bool deactivated;
54
55	DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
56
57	cpuif->vgic_hcr &= ~GICH_HCR_UIE;
58
59	for (lr = 0; lr < vgic_cpu->vgic_v2.used_lrs; lr++) {
60	u32 val = cpuif->vgic_lr[lr];
61	u32 cpuid, intid = val & GICH_LR_VIRTUALID;
62	struct vgic_irq *irq;
63	bool deactivated;
64
65	/* Extract the source vCPU id from the LR */
66	cpuid = val & GICH_LR_PHYSID_CPUID;
67	cpuid >>= GICH_LR_PHYSID_CPUID_SHIFT;
68	cpuid &= 7;
69		61
70	/* Notify fds when the guest EOI'ed a level-triggered SPI */	62	/* Extract the source vCPU id from the LR */
71	if (lr_signals_eoi_mi(val) && vgic_valid_spi(vcpu->kvm, intid))	63	cpuid = FIELD_GET(GICH_LR_PHYSID_CPUID, val) & 7;
72	kvm_notify_acked_irq(vcpu->kvm, 0,
73	intid - VGIC_NR_PRIVATE_IRQS);
74		64
75	irq = vgic_get_vcpu_irq(vcpu, intid);	65	/* Notify fds when the guest EOI'ed a level-triggered SPI */
		66	if (lr_signals_eoi_mi(val) && vgic_valid_spi(vcpu->kvm, intid))
		67	kvm_notify_acked_irq(vcpu->kvm, 0,
		68	intid - VGIC_NR_PRIVATE_IRQS);
76		69
77	raw_spin_lock(&irq->irq_lock);	70	irq = vgic_get_vcpu_irq(vcpu, intid);
78		71
		72	scoped_guard(raw_spinlock, &irq->irq_lock) {
79	/* Always preserve the active bit, note deactivation */	73	/* Always preserve the active bit, note deactivation */
80	deactivated = irq->active && !(val & GICH_LR_ACTIVE_BIT);	74	deactivated = irq->active && !(val & GICH_LR_ACTIVE_BIT);
81	irq->active = !!(val & GICH_LR_ACTIVE_BIT);	75	irq->active = !!(val & GICH_LR_ACTIVE_BIT);
@@ -101,29 +95,139 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu) · 19 unmodified lines
101	/* Handle resampling for mapped interrupts if required */	95	/* Handle resampling for mapped interrupts if required */
102	vgic_irq_handle_resampling(irq, deactivated, val & GICH_LR_PENDING_BIT);	96	vgic_irq_handle_resampling(irq, deactivated, val & GICH_LR_PENDING_BIT);
103		97
104	raw_spin_unlock(&irq->irq_lock);	98	irq->on_lr = false;
105	vgic_put_irq(vcpu->kvm, irq);
106	}	99	}
107		100
108	cpuif->used_lrs = 0;	101	vgic_put_irq(vcpu->kvm, irq);
109	}	102	}
110		103
		104	static u32 vgic_v2_compute_lr(struct kvm_vcpu vcpu, struct vgic_irq irq);
		105
111	/*	106	/*
112	* Populates the particular LR with the state of a given IRQ:	107	* transfer the content of the LRs back into the corresponding ap_list:
113	* - for an edge sensitive IRQ the pending state is cleared in struct vgic_irq	108	* - active bit is transferred as is
114	* - for a level sensitive IRQ the pending state value is unchanged;	109	* - pending bit is
115	* it is dictated directly by the input level	110	* - transferred as is in case of edge sensitive IRQs
116	*	111	* - set to the line-level (resample time) for level sensitive IRQs
117	* If @irq describes an SGI with multiple sources, we choose the
118	* lowest-numbered source VCPU and clear that bit in the source bitmap.
119	*
120	* The irq_lock must be held by the caller.
121	*/	112	*/
122	void vgic_v2_populate_lr(struct kvm_vcpu vcpu, struct vgic_irq irq, int lr)	113	void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu)
		114	{
		115	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
		116	struct vgic_v2_cpu_if *cpuif = &vgic_cpu->vgic_v2;
		117	u32 eoicount = FIELD_GET(GICH_HCR_EOICOUNT, cpuif->vgic_hcr);
		118	struct vgic_irq *irq;
		119
		120	DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
		121
		122	for (int lr = 0; lr < vgic_cpu->vgic_v2.used_lrs; lr++)
		123	vgic_v2_fold_lr(vcpu, cpuif->vgic_lr[lr]);
		124
		125	/* See the GICv3 equivalent for the EOIcount handling rationale */
		126	list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
		127	u32 lr;
		128
		129	if (!eoicount) {
		130	break;
		131	} else {
		132	guard(raw_spinlock)(&irq->irq_lock);
		133
		134	if (!(likely(vgic_target_oracle(irq) == vcpu) &&
		135	irq->active))
		136	continue;
		137
		138	lr = vgic_v2_compute_lr(vcpu, irq) & ~GICH_LR_ACTIVE_BIT;
		139	}
		140
		141	if (lr & GICH_LR_HW)
		142	writel_relaxed(FIELD_GET(GICH_LR_PHYSID_CPUID, lr),
		143	kvm_vgic_global_state.gicc_base + GIC_CPU_DEACTIVATE);
		144	vgic_v2_fold_lr(vcpu, lr);
		145	eoicount--;
		146	}
		147
		148	cpuif->used_lrs = 0;
		149	}
		150
		151	void vgic_v2_deactivate(struct kvm_vcpu *vcpu, u32 val)
		152	{
		153	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
		154	struct vgic_v2_cpu_if *cpuif = &vgic_cpu->vgic_v2;
		155	struct kvm_vcpu *target_vcpu = NULL;
		156	bool mmio = false;
		157	struct vgic_irq *irq;
		158	unsigned long flags;
		159	u64 lr = 0;
		160	u8 cpuid;
		161
		162	/* Snapshot CPUID, and remove it from the INTID */
		163	cpuid = FIELD_GET(GENMASK_ULL(12, 10), val);
		164	val &= ~GENMASK_ULL(12, 10);
		165
		166	/* We only deal with DIR when EOIMode==1 */
		167	if (!(cpuif->vgic_vmcr & GICH_VMCR_EOI_MODE_MASK))
		168	return;
		169
		170	/* Make sure we're in the same context as LR handling */
		171	local_irq_save(flags);
		172
		173	irq = vgic_get_vcpu_irq(vcpu, val);
		174	if (WARN_ON_ONCE(!irq))
		175	goto out;
		176
		177	/* See the corresponding v3 code for the rationale */
		178	scoped_guard(raw_spinlock, &irq->irq_lock) {
		179	target_vcpu = irq->vcpu;
		180
		181	/* Not on any ap_list? */
		182	if (!target_vcpu)
		183	goto put;
		184
		185	/*
		186	* Urgh. We're deactivating something that we cannot
		187	* observe yet... Big hammer time.
		188	*/
		189	if (irq->on_lr) {
		190	mmio = true;
		191	goto put;
		192	}
		193
		194	/* SGI: check that the cpuid matches */
		195	if (val < VGIC_NR_SGIS && irq->active_source != cpuid) {
		196	target_vcpu = NULL;
		197	goto put;
		198	}
		199
		200	/* (with a Dalek voice) DEACTIVATE!!!! */
		201	lr = vgic_v2_compute_lr(vcpu, irq) & ~GICH_LR_ACTIVE_BIT;
		202	}
		203
		204	if (lr & GICH_LR_HW)
		205	writel_relaxed(FIELD_GET(GICH_LR_PHYSID_CPUID, lr),
		206	kvm_vgic_global_state.gicc_base + GIC_CPU_DEACTIVATE);
		207
		208	vgic_v2_fold_lr(vcpu, lr);
		209
		210	put:
		211	vgic_put_irq(vcpu->kvm, irq);
		212
		213	out:
		214	local_irq_restore(flags);
		215
		216	if (mmio)
		217	vgic_mmio_write_cactive(vcpu, (val / 32) * 4, 4, BIT(val % 32));
		218
		219	/* Force the ap_list to be pruned */
		220	if (target_vcpu)
		221	kvm_make_request(KVM_REQ_VGIC_PROCESS_UPDATE, target_vcpu);
		222	}
		223
		224	static u32 vgic_v2_compute_lr(struct kvm_vcpu vcpu, struct vgic_irq irq)
123	{	225	{
124	u32 val = irq->intid;	226	u32 val = irq->intid;
125	bool allow_pending = true;	227	bool allow_pending = true;
126		228
		229	WARN_ON(irq->on_lr);
		230
127	if (irq->active) {	231	if (irq->active) {
128	val \|= GICH_LR_ACTIVE_BIT;	232	val \|= GICH_LR_ACTIVE_BIT;
129	if (vgic_irq_is_sgi(irq->intid))	233	if (vgic_irq_is_sgi(irq->intid))
@@ -163,22 +267,52 @@ void vgic_v2_populate_lr(struct kvm_vcpu vcpu, struct vgic_irq irq, int lr) · 33 unmodified lines
163	if (allow_pending && irq_is_pending(irq)) {	267	if (allow_pending && irq_is_pending(irq)) {
164	val \|= GICH_LR_PENDING_BIT;	268	val \|= GICH_LR_PENDING_BIT;
165		269
166	if (irq->config == VGIC_CONFIG_EDGE)
167	irq->pending_latch = false;
168
169	if (vgic_irq_is_sgi(irq->intid)) {	270	if (vgic_irq_is_sgi(irq->intid)) {
170	u32 src = ffs(irq->source);	271	u32 src = ffs(irq->source);
171		272
172	if (WARN_RATELIMIT(!src, "No SGI source for INTID %d\n",	273	if (WARN_RATELIMIT(!src, "No SGI source for INTID %d\n",
173	irq->intid))	274	irq->intid))
174	return;	275	return 0;
175		276
176	val \|= (src - 1) << GICH_LR_PHYSID_CPUID_SHIFT;	277	val \|= (src - 1) << GICH_LR_PHYSID_CPUID_SHIFT;
177	irq->source &= ~(1 << (src - 1));	278	if (irq->source & ~BIT(src - 1))
178	if (irq->source) {
179	irq->pending_latch = true;
180	val \|= GICH_LR_EOI;	279	val \|= GICH_LR_EOI;
181	}	280	}
		281	}
		282
		283	/* The GICv2 LR only holds five bits of priority. */
		284	val \|= (irq->priority >> 3) << GICH_LR_PRIORITY_SHIFT;
		285
		286	return val;
		287	}
		288
		289	/*
		290	* Populates the particular LR with the state of a given IRQ:
		291	* - for an edge sensitive IRQ the pending state is cleared in struct vgic_irq
		292	* - for a level sensitive IRQ the pending state value is unchanged;
		293	* it is dictated directly by the input level
		294	*
		295	* If @irq describes an SGI with multiple sources, we choose the
		296	* lowest-numbered source VCPU and clear that bit in the source bitmap.
		297	*
		298	* The irq_lock must be held by the caller.
		299	*/
		300	void vgic_v2_populate_lr(struct kvm_vcpu vcpu, struct vgic_irq irq, int lr)
		301	{
		302	u32 val = vgic_v2_compute_lr(vcpu, irq);
		303
		304	vcpu->arch.vgic_cpu.vgic_v2.vgic_lr[lr] = val;
		305
		306	if (val & GICH_LR_PENDING_BIT) {
		307	if (irq->config == VGIC_CONFIG_EDGE)
		308	irq->pending_latch = false;
		309
		310	if (vgic_irq_is_sgi(irq->intid)) {
		311	u32 src = ffs(irq->source);
		312
		313	irq->source &= ~BIT(src - 1);
		314	if (irq->source)
		315	irq->pending_latch = true;
182	}	316	}
183	}	317	}
184		318
@@ -194,7 +328,7 @@ void vgic_v2_populate_lr(struct kvm_vcpu vcpu, struct vgic_irq irq, int lr) · 9 unmodified lines
194	/* The GICv2 LR only holds five bits of priority. */	328	/* The GICv2 LR only holds five bits of priority. */
195	val \|= (irq->priority >> 3) << GICH_LR_PRIORITY_SHIFT;	329	val \|= (irq->priority >> 3) << GICH_LR_PRIORITY_SHIFT;
196		330
197	vcpu->arch.vgic_cpu.vgic_v2.vgic_lr[lr] = val;	331	irq->on_lr = true;
198	}	332	}
199		333
200	void vgic_v2_clear_lr(struct kvm_vcpu *vcpu, int lr)	334	void vgic_v2_clear_lr(struct kvm_vcpu *vcpu, int lr)
@@ -257,7 +391,7 @@ void vgic_v2_get_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp) · 56 unmodified lines
257	GICH_VMCR_PRIMASK_SHIFT) << GICV_PMR_PRIORITY_SHIFT;	391	GICH_VMCR_PRIMASK_SHIFT) << GICV_PMR_PRIORITY_SHIFT;
258	}	392	}
259		393
260	void vgic_v2_enable(struct kvm_vcpu *vcpu)	394	void vgic_v2_reset(struct kvm_vcpu *vcpu)
261	{	395	{
262	/*	396	/*
263	* By forcing VMCR to zero, the GIC will restore the binary	397	* By forcing VMCR to zero, the GIC will restore the binary
@@ -265,9 +399,6 @@ void vgic_v2_enable(struct kvm_vcpu *vcpu) · 1 unmodified line
265	* anyway.	399	* anyway.
266	*/	400	*/
267	vcpu->arch.vgic_cpu.vgic_v2.vgic_vmcr = 0;	401	vcpu->arch.vgic_cpu.vgic_v2.vgic_vmcr = 0;
268
269	/* Get the show on the road... */
270	vcpu->arch.vgic_cpu.vgic_v2.vgic_hcr = GICH_HCR_EN;
271	}	402	}
272		403
273	/* check for overlapping regions and for regions crossing the end of memory */	404	/* check for overlapping regions and for regions crossing the end of memory */
@@ -289,6 +420,7 @@ static bool vgic_v2_check_base(gpa_t dist_base, gpa_t cpu_base) · 15 unmodified lines
289	int vgic_v2_map_resources(struct kvm *kvm)	420	int vgic_v2_map_resources(struct kvm *kvm)
290	{	421	{
291	struct vgic_dist *dist = &kvm->arch.vgic;	422	struct vgic_dist *dist = &kvm->arch.vgic;
		423	unsigned int len;
292	int ret = 0;	424	int ret = 0;
293		425
294	if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base) \|\|	426	if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base) \|\|
@@ -312,10 +444,20 @@ int vgic_v2_map_resources(struct kvm *kvm) · 17 unmodified lines
312	return ret;	444	return ret;
313	}	445	}
314		446
		447	len = vgic_v2_init_cpuif_iodev(&dist->cpuif_iodev);
		448	dist->cpuif_iodev.base_addr = dist->vgic_cpu_base;
		449	dist->cpuif_iodev.iodev_type = IODEV_CPUIF;
		450	dist->cpuif_iodev.redist_vcpu = NULL;
		451
		452	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, dist->vgic_cpu_base,
		453	len, &dist->cpuif_iodev.dev);
		454	if (ret)
		455	return ret;
		456
315	if (!static_branch_unlikely(&vgic_v2_cpuif_trap)) {	457	if (!static_branch_unlikely(&vgic_v2_cpuif_trap)) {
316	ret = kvm_phys_addr_ioremap(kvm, dist->vgic_cpu_base,	458	ret = kvm_phys_addr_ioremap(kvm, dist->vgic_cpu_base,
317	kvm_vgic_global_state.vcpu_base,	459	kvm_vgic_global_state.vcpu_base,
318	KVM_VGIC_V2_CPU_SIZE, true);	460	KVM_VGIC_V2_CPU_SIZE - SZ_4K, true);
319	if (ret) {	461	if (ret) {
320	kvm_err("Unable to remap VGIC CPU to VCPU\n");	462	kvm_err("Unable to remap VGIC CPU to VCPU\n");
321	return ret;	463	return ret;
@@ -385,6 +527,7 @@ int vgic_v2_probe(const struct gic_kvm_info *info) · 63 unmodified lines
385		527
386	kvm_vgic_global_state.can_emulate_gicv2 = true;	528	kvm_vgic_global_state.can_emulate_gicv2 = true;
387	kvm_vgic_global_state.vcpu_base = info->vcpu.start;	529	kvm_vgic_global_state.vcpu_base = info->vcpu.start;
		530	kvm_vgic_global_state.gicc_base = info->gicc_base;
388	kvm_vgic_global_state.type = VGIC_V2;	531	kvm_vgic_global_state.type = VGIC_V2;
389	kvm_vgic_global_state.max_gic_vcpus = VGIC_V2_MAX_CPUS;	532	kvm_vgic_global_state.max_gic_vcpus = VGIC_V2_MAX_CPUS;
390		533
@@ -423,16 +566,26 @@ static void save_lrs(struct kvm_vcpu vcpu, void __iomem base) · 32 unmodified lines
423		566
424	void vgic_v2_save_state(struct kvm_vcpu *vcpu)	567	void vgic_v2_save_state(struct kvm_vcpu *vcpu)
425	{	568	{
		569	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
426	void __iomem *base = kvm_vgic_global_state.vctrl_base;	570	void __iomem *base = kvm_vgic_global_state.vctrl_base;
427	u64 used_lrs = vcpu->arch.vgic_cpu.vgic_v2.used_lrs;	571	u64 used_lrs = vcpu->arch.vgic_cpu.vgic_v2.used_lrs;
428		572
429	if (!base)	573	if (!base)
430	return;	574	return;
431		575
432	if (used_lrs) {	576	cpu_if->vgic_vmcr = readl_relaxed(kvm_vgic_global_state.vctrl_base + GICH_VMCR);
		577
		578	if (used_lrs)
433	save_lrs(vcpu, base);	579	save_lrs(vcpu, base);
434	writel_relaxed(0, base + GICH_HCR);	580
		581	if (cpu_if->vgic_hcr & GICH_HCR_LRENPIE) {
		582	u32 val = readl_relaxed(base + GICH_HCR);
		583
		584	cpu_if->vgic_hcr &= ~GICH_HCR_EOICOUNT;
		585	cpu_if->vgic_hcr \|= val & GICH_HCR_EOICOUNT;
435	}	586	}
		587
		588	writel_relaxed(0, base + GICH_HCR);
436	}	589	}
437		590
438	void vgic_v2_restore_state(struct kvm_vcpu *vcpu)	591	void vgic_v2_restore_state(struct kvm_vcpu *vcpu)
@@ -445,13 +598,10 @@ void vgic_v2_restore_state(struct kvm_vcpu *vcpu) · 6 unmodified lines
445	if (!base)	598	if (!base)
446	return;	599	return;
447		600
448	if (used_lrs) {	601	writel_relaxed(cpu_if->vgic_hcr, base + GICH_HCR);
449	writel_relaxed(cpu_if->vgic_hcr, base + GICH_HCR);	602
450	for (i = 0; i < used_lrs; i++) {	603	for (i = 0; i < used_lrs; i++)
451	writel_relaxed(cpu_if->vgic_lr[i],	604	writel_relaxed(cpu_if->vgic_lr[i], base + GICH_LR0 + (i * 4));
452	base + GICH_LR0 + (i * 4));
453	}
454	}
455	}	605	}
456		606
457	void vgic_v2_load(struct kvm_vcpu *vcpu)	607	void vgic_v2_load(struct kvm_vcpu *vcpu)
@@ -468,6 +618,5 @@ void vgic_v2_put(struct kvm_vcpu *vcpu) · 10 unmodified lines
468	{	618	{
469	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;	619	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
470		620
471	cpu_if->vgic_vmcr = readl_relaxed(kvm_vgic_global_state.vctrl_base + GICH_VMCR);
472	cpu_if->vgic_apr = readl_relaxed(kvm_vgic_global_state.vctrl_base + GICH_APR);	621	cpu_if->vgic_apr = readl_relaxed(kvm_vgic_global_state.vctrl_base + GICH_APR);
473	}	622	}

arch/arm64/kvm/vgic/vgic-v3-nested.c+52−52

@@ -70,13 +70,14 @@ static int lr_map_idx_to_shadow_idx(struct shadow_if *shadow_if, int idx) · 69 unmodified lines
70	* - on L2 put: perform the inverse transformation, so that the result of L2	70	* - on L2 put: perform the inverse transformation, so that the result of L2
71	* running becomes visible to L1 in the VNCR-accessible registers.	71	* running becomes visible to L1 in the VNCR-accessible registers.
72	*	72	*
73	* - there is nothing to do on L2 entry, as everything will have happened	73	* - there is nothing to do on L2 entry apart from enabling the vgic, as
74	* on load. However, this is the point where we detect that an interrupt	74	* everything will have happened on load. However, this is the point where
75	* targeting L1 and prepare the grand switcheroo.	75	* we detect that an interrupt targeting L1 and prepare the grand
		76	* switcheroo.
76	*	77	*
77	* - on L2 exit: emulate the HW bit, and deactivate corresponding the L1	78	* - on L2 exit: resync the LRs and VMCR, emulate the HW bit, and deactivate
78	* interrupt. The L0 active state will be cleared by the HW if the L1	79	* corresponding the L1 interrupt. The L0 active state will be cleared by
79	* interrupt was itself backed by a HW interrupt.	80	* the HW if the L1 interrupt was itself backed by a HW interrupt.
80	*	81	*
81	* Maintenance Interrupt (MI) management:	82	* Maintenance Interrupt (MI) management:
82	*	83	*
@@ -93,8 +94,10 @@ static int lr_map_idx_to_shadow_idx(struct shadow_if *shadow_if, int idx) · 10 unmodified lines
93	*	94	*
94	* - because most of the ICH_*_EL2 registers live in the VNCR page, the	95	* - because most of the ICH_*_EL2 registers live in the VNCR page, the
95	* quality of emulation is poor: L1 can setup the vgic so that an MI would	96	* quality of emulation is poor: L1 can setup the vgic so that an MI would
96	* immediately fire, and not observe anything until the next exit. Trying	97	* immediately fire, and not observe anything until the next exit.
97	* to read ICH_MISR_EL2 would do the trick, for example.	98	* Similarly, a pending MI is not immediately disabled by clearing
		99	* ICH_HCR_EL2.En. Trying to read ICH_MISR_EL2 would do the trick, for
		100	* example.
98	*	101	*
99	* System register emulation:	102	* System register emulation:
100	*	103	*
@@ -265,16 +268,37 @@ static void vgic_v3_create_shadow_lr(struct kvm_vcpu *vcpu, · 164 unmodified lines
265	s_cpu_if->used_lrs = hweight16(shadow_if->lr_map);	268	s_cpu_if->used_lrs = hweight16(shadow_if->lr_map);
266	}	269	}
267		270
		271	void vgic_v3_flush_nested(struct kvm_vcpu *vcpu)
		272	{
		273	u64 val = __vcpu_sys_reg(vcpu, ICH_HCR_EL2);
		274
		275	write_sysreg_s(val \| vgic_ich_hcr_trap_bits(), SYS_ICH_HCR_EL2);
		276	}
		277
268	void vgic_v3_sync_nested(struct kvm_vcpu *vcpu)	278	void vgic_v3_sync_nested(struct kvm_vcpu *vcpu)
269	{	279	{
270	struct shadow_if *shadow_if = get_shadow_if();	280	struct shadow_if *shadow_if = get_shadow_if();
271	int i;	281	int i;
272		282
273	for_each_set_bit(i, &shadow_if->lr_map, kvm_vgic_global_state.nr_lr) {	283	for_each_set_bit(i, &shadow_if->lr_map, kvm_vgic_global_state.nr_lr) {
274	u64 lr = __vcpu_sys_reg(vcpu, ICH_LRN(i));	284	u64 val, host_lr, lr;
275	struct vgic_irq *irq;	285
		286	host_lr = __gic_v3_get_lr(lr_map_idx_to_shadow_idx(shadow_if, i));
		287
		288	/* Propagate the new LR state */
		289	lr = __vcpu_sys_reg(vcpu, ICH_LRN(i));
		290	val = lr & ~ICH_LR_STATE;
		291	val \|= host_lr & ICH_LR_STATE;
		292	__vcpu_assign_sys_reg(vcpu, ICH_LRN(i), val);
276		293
277	if (!(lr & ICH_LR_HW) \|\| !(lr & ICH_LR_STATE))	294	/*
		295	* Deactivation of a HW interrupt: the LR must have the HW
		296	* bit set, have been in a non-invalid state before the run,
		297	* and now be in an invalid state. If any of that doesn't
		298	* hold, we're done with this LR.
		299	*/
		300	if (!((lr & ICH_LR_HW) && (lr & ICH_LR_STATE) &&
		301	!(host_lr & ICH_LR_STATE)))
278	continue;	302	continue;
279		303
280	/*	304	/*
@@ -282,35 +306,27 @@ void vgic_v3_sync_nested(struct kvm_vcpu *vcpu) · 1 unmodified line
282	* need to emulate the HW effect between the guest hypervisor	306	* need to emulate the HW effect between the guest hypervisor
283	* and the nested guest.	307	* and the nested guest.
284	*/	308	*/
285	irq = vgic_get_vcpu_irq(vcpu, FIELD_GET(ICH_LR_PHYS_ID_MASK, lr));	309	vgic_v3_deactivate(vcpu, FIELD_GET(ICH_LR_PHYS_ID_MASK, lr));
286	if (WARN_ON(!irq)) /* Shouldn't happen as we check on load */	310	}
287	continue;
288		311
289	lr = __gic_v3_get_lr(lr_map_idx_to_shadow_idx(shadow_if, i));	312	/* We need these to be synchronised to generate the MI */
290	if (!(lr & ICH_LR_STATE))	313	__vcpu_assign_sys_reg(vcpu, ICH_VMCR_EL2, read_sysreg_s(SYS_ICH_VMCR_EL2));
291	irq->active = false;	314	__vcpu_rmw_sys_reg(vcpu, ICH_HCR_EL2, &=, ~ICH_HCR_EL2_EOIcount);
		315	__vcpu_rmw_sys_reg(vcpu, ICH_HCR_EL2, \|=, read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EL2_EOIcount);
292		316
293	vgic_put_irq(vcpu->kvm, irq);	317	write_sysreg_s(0, SYS_ICH_HCR_EL2);
294	}	318	isb();
		319
		320	vgic_v3_nested_update_mi(vcpu);
295	}	321	}
296		322
297	static void vgic_v3_create_shadow_state(struct kvm_vcpu *vcpu,	323	static void vgic_v3_create_shadow_state(struct kvm_vcpu *vcpu,
298	struct vgic_v3_cpu_if *s_cpu_if)	324	struct vgic_v3_cpu_if *s_cpu_if)
299	{	325	{
300	struct vgic_v3_cpu_if *host_if = &vcpu->arch.vgic_cpu.vgic_v3;	326	struct vgic_v3_cpu_if *host_if = &vcpu->arch.vgic_cpu.vgic_v3;
301	u64 val = 0;
302	int i;	327	int i;
303		328
304	/*	329	s_cpu_if->vgic_hcr = __vcpu_sys_reg(vcpu, ICH_HCR_EL2);
305	* If we're on a system with a broken vgic that requires
306	* trapping, propagate the trapping requirements.
307	*
308	* Ah, the smell of rotten fruits...
309	*/
310	if (static_branch_unlikely(&vgic_v3_cpuif_trap))
311	val = host_if->vgic_hcr & (ICH_HCR_EL2_TALL0 \| ICH_HCR_EL2_TALL1 \|
312	ICH_HCR_EL2_TC \| ICH_HCR_EL2_TDIR);
313	s_cpu_if->vgic_hcr = __vcpu_sys_reg(vcpu, ICH_HCR_EL2) \| val;
314	s_cpu_if->vgic_vmcr = __vcpu_sys_reg(vcpu, ICH_VMCR_EL2);	330	s_cpu_if->vgic_vmcr = __vcpu_sys_reg(vcpu, ICH_VMCR_EL2);
315	s_cpu_if->vgic_sre = host_if->vgic_sre;	331	s_cpu_if->vgic_sre = host_if->vgic_sre;
316		332
@@ -334,7 +350,8 @@ void vgic_v3_load_nested(struct kvm_vcpu *vcpu) · 17 unmodified lines
334	__vgic_v3_restore_vmcr_aprs(cpu_if);	350	__vgic_v3_restore_vmcr_aprs(cpu_if);
335	__vgic_v3_activate_traps(cpu_if);	351	__vgic_v3_activate_traps(cpu_if);
336		352
337	__vgic_v3_restore_state(cpu_if);	353	for (int i = 0; i < cpu_if->used_lrs; i++)
		354	__gic_v3_set_lr(cpu_if->vgic_lr[i], i);
338		355
339	/*	356	/*
340	* Propagate the number of used LRs for the benefit of the HYP	357	* Propagate the number of used LRs for the benefit of the HYP
@@ -347,36 +364,19 @@ void vgic_v3_put_nested(struct kvm_vcpu *vcpu) · 6 unmodified lines
347	{	364	{
348	struct shadow_if *shadow_if = get_shadow_if();	365	struct shadow_if *shadow_if = get_shadow_if();
349	struct vgic_v3_cpu_if *s_cpu_if = &shadow_if->cpuif;	366	struct vgic_v3_cpu_if *s_cpu_if = &shadow_if->cpuif;
350	u64 val;
351	int i;	367	int i;
352		368
353	__vgic_v3_save_vmcr_aprs(s_cpu_if);	369	__vgic_v3_save_aprs(s_cpu_if);
354	__vgic_v3_deactivate_traps(s_cpu_if);
355	__vgic_v3_save_state(s_cpu_if);
356
357	/*
358	* Translate the shadow state HW fields back to the virtual ones
359	* before copying the shadow struct back to the nested one.
360	*/
361	val = __vcpu_sys_reg(vcpu, ICH_HCR_EL2);
362	val &= ~ICH_HCR_EL2_EOIcount_MASK;
363	val \|= (s_cpu_if->vgic_hcr & ICH_HCR_EL2_EOIcount_MASK);
364	__vcpu_assign_sys_reg(vcpu, ICH_HCR_EL2, val);
365	__vcpu_assign_sys_reg(vcpu, ICH_VMCR_EL2, s_cpu_if->vgic_vmcr);
366		370
367	for (i = 0; i < 4; i++) {	371	for (i = 0; i < 4; i++) {
368	__vcpu_assign_sys_reg(vcpu, ICH_AP0RN(i), s_cpu_if->vgic_ap0r[i]);	372	__vcpu_assign_sys_reg(vcpu, ICH_AP0RN(i), s_cpu_if->vgic_ap0r[i]);
369	__vcpu_assign_sys_reg(vcpu, ICH_AP1RN(i), s_cpu_if->vgic_ap1r[i]);	373	__vcpu_assign_sys_reg(vcpu, ICH_AP1RN(i), s_cpu_if->vgic_ap1r[i]);
370	}	374	}
371		375
372	for_each_set_bit(i, &shadow_if->lr_map, kvm_vgic_global_state.nr_lr) {	376	for (i = 0; i < s_cpu_if->used_lrs; i++)
373	val = __vcpu_sys_reg(vcpu, ICH_LRN(i));	377	__gic_v3_set_lr(0, i);
374
375	val &= ~ICH_LR_STATE;
376	val \|= s_cpu_if->vgic_lr[lr_map_idx_to_shadow_idx(shadow_if, i)] & ICH_LR_STATE;
377		378
378	__vcpu_assign_sys_reg(vcpu, ICH_LRN(i), val);	379	__vgic_v3_deactivate_traps(s_cpu_if);
379	}
380		380
381	vcpu->arch.vgic_cpu.vgic_v3.used_lrs = 0;	381	vcpu->arch.vgic_cpu.vgic_v3.used_lrs = 0;
382	}	382	}

arch/arm64/kvm/vgic/vgic-v3.c+328−98

@@ -12,6 +12,7 @@ · 11 unmodified lines
12	#include <asm/kvm_mmu.h>	12	#include <asm/kvm_mmu.h>
13	#include <asm/kvm_asm.h>	13	#include <asm/kvm_asm.h>
14		14
		15	#include "vgic-mmio.h"
15	#include "vgic.h"	16	#include "vgic.h"
16		17
17	static bool group0_trap;	18	static bool group0_trap;
@@ -20,11 +21,48 @@ static bool common_trap; · 2 unmodified lines
20	static bool dir_trap;	21	static bool dir_trap;
21	static bool gicv4_enable;	22	static bool gicv4_enable;
22		23
23	void vgic_v3_set_underflow(struct kvm_vcpu *vcpu)	24	void vgic_v3_configure_hcr(struct kvm_vcpu *vcpu,
		25	struct ap_list_summary *als)
24	{	26	{
25	struct vgic_v3_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v3;	27	struct vgic_v3_cpu_if *cpuif = &vcpu->arch.vgic_cpu.vgic_v3;
26		28
27	cpuif->vgic_hcr \|= ICH_HCR_EL2_UIE;	29	if (!irqchip_in_kernel(vcpu->kvm))
		30	return;
		31
		32	cpuif->vgic_hcr = ICH_HCR_EL2_En;
		33
		34	if (irqs_pending_outside_lrs(als))
		35	cpuif->vgic_hcr \|= ICH_HCR_EL2_NPIE;
		36	if (irqs_active_outside_lrs(als))
		37	cpuif->vgic_hcr \|= ICH_HCR_EL2_LRENPIE;
		38	if (irqs_outside_lrs(als))
		39	cpuif->vgic_hcr \|= ICH_HCR_EL2_UIE;
		40
		41	if (!als->nr_sgi)
		42	cpuif->vgic_hcr \|= ICH_HCR_EL2_vSGIEOICount;
		43
		44	cpuif->vgic_hcr \|= (cpuif->vgic_vmcr & ICH_VMCR_ENG0_MASK) ?
		45	ICH_HCR_EL2_VGrp0DIE : ICH_HCR_EL2_VGrp0EIE;
		46	cpuif->vgic_hcr \|= (cpuif->vgic_vmcr & ICH_VMCR_ENG1_MASK) ?
		47	ICH_HCR_EL2_VGrp1DIE : ICH_HCR_EL2_VGrp1EIE;
		48
		49	/*
		50	* Dealing with EOImode=1 is a massive source of headache. Not
		51	* only do we need to track that we have active interrupts
		52	* outside of the LRs and force DIR to be trapped, we also
		53	* need to deal with SPIs that can be deactivated on another
		54	* CPU.
		55	*
		56	* On systems that do not implement TDIR, force the bit in the
		57	* shadow state anyway to avoid IPI-ing on these poor sods.
		58	*
		59	* Note that we set the trap irrespective of EOIMode, as that
		60	* can change behind our back without any warning...
		61	*/
		62	if (!cpus_have_final_cap(ARM64_HAS_ICH_HCR_EL2_TDIR) \|\|
		63	irqs_active_outside_lrs(als) \|\|
		64	atomic_read(&vcpu->kvm->arch.vgic.active_spis))
		65	cpuif->vgic_hcr \|= ICH_HCR_EL2_TDIR;
28	}	66	}
29		67
30	static bool lr_signals_eoi_mi(u64 lr_val)	68	static bool lr_signals_eoi_mi(u64 lr_val)
@@ -33,84 +71,238 @@ static bool lr_signals_eoi_mi(u64 lr_val) · 2 unmodified lines
33	!(lr_val & ICH_LR_HW);	71	!(lr_val & ICH_LR_HW);
34	}	72	}
35		73
		74	static void vgic_v3_fold_lr(struct kvm_vcpu *vcpu, u64 val)
		75	{
		76	struct vgic_irq *irq;
		77	bool is_v2_sgi = false;
		78	bool deactivated;
		79	u32 intid;
		80
		81	if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
		82	intid = val & ICH_LR_VIRTUAL_ID_MASK;
		83	} else {
		84	intid = val & GICH_LR_VIRTUALID;
		85	is_v2_sgi = vgic_irq_is_sgi(intid);
		86	}
		87
		88	irq = vgic_get_vcpu_irq(vcpu, intid);
		89	if (!irq) /* An LPI could have been unmapped. */
		90	return;
		91
		92	scoped_guard(raw_spinlock, &irq->irq_lock) {
		93	/* Always preserve the active bit for !LPIs, note deactivation */
		94	if (irq->intid >= VGIC_MIN_LPI)
		95	val &= ~ICH_LR_ACTIVE_BIT;
		96	deactivated = irq->active && !(val & ICH_LR_ACTIVE_BIT);
		97	irq->active = !!(val & ICH_LR_ACTIVE_BIT);
		98
		99	/* Edge is the only case where we preserve the pending bit */
		100	if (irq->config == VGIC_CONFIG_EDGE &&
		101	(val & ICH_LR_PENDING_BIT))
		102	irq->pending_latch = true;
		103
		104	/*
		105	* Clear soft pending state when level irqs have been acked.
		106	*/
		107	if (irq->config == VGIC_CONFIG_LEVEL && !(val & ICH_LR_STATE))
		108	irq->pending_latch = false;
		109
		110	if (is_v2_sgi) {
		111	u8 cpuid = FIELD_GET(GICH_LR_PHYSID_CPUID, val);
		112
		113	if (irq->active)
		114	irq->active_source = cpuid;
		115
		116	if (val & ICH_LR_PENDING_BIT)
		117	irq->source \|= BIT(cpuid);
		118	}
		119
		120	/* Handle resampling for mapped interrupts if required */
		121	vgic_irq_handle_resampling(irq, deactivated, val & ICH_LR_PENDING_BIT);
		122
		123	irq->on_lr = false;
		124	}
		125
		126	/* Notify fds when the guest EOI'ed a level-triggered SPI, and drop the refcount */
		127	if (deactivated && lr_signals_eoi_mi(val) && vgic_valid_spi(vcpu->kvm, intid)) {
		128	kvm_notify_acked_irq(vcpu->kvm, 0,
		129	intid - VGIC_NR_PRIVATE_IRQS);
		130	atomic_dec_if_positive(&vcpu->kvm->arch.vgic.active_spis);
		131	}
		132
		133	vgic_put_irq(vcpu->kvm, irq);
		134	}
		135
		136	static u64 vgic_v3_compute_lr(struct kvm_vcpu vcpu, struct vgic_irq irq);
		137
		138	static void vgic_v3_deactivate_phys(u32 intid)
		139	{
		140	if (cpus_have_final_cap(ARM64_HAS_GICV5_LEGACY))
		141	gic_insn(intid \| FIELD_PREP(GICV5_GIC_CDDI_TYPE_MASK, 1), CDDI);
		142	else
		143	gic_write_dir(intid);
		144	}
		145
36	void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu)	146	void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu)
37	{	147	{
38	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;	148	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
39	struct vgic_v3_cpu_if *cpuif = &vgic_cpu->vgic_v3;	149	struct vgic_v3_cpu_if *cpuif = &vgic_cpu->vgic_v3;
40	u32 model = vcpu->kvm->arch.vgic.vgic_model;	150	u32 eoicount = FIELD_GET(ICH_HCR_EL2_EOIcount, cpuif->vgic_hcr);
41	int lr;	151	struct vgic_irq *irq;
42		152
43	DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());	153	DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
44		154
45	cpuif->vgic_hcr &= ~ICH_HCR_EL2_UIE;	155	for (int lr = 0; lr < cpuif->used_lrs; lr++)
46		156	vgic_v3_fold_lr(vcpu, cpuif->vgic_lr[lr]);
47	for (lr = 0; lr < cpuif->used_lrs; lr++) {
48	u64 val = cpuif->vgic_lr[lr];
49	u32 intid, cpuid;
50	struct vgic_irq *irq;
51	bool is_v2_sgi = false;
52	bool deactivated;
53		157
54	cpuid = val & GICH_LR_PHYSID_CPUID;	158	/*
55	cpuid >>= GICH_LR_PHYSID_CPUID_SHIFT;	159	* EOIMode=0: use EOIcount to emulate deactivation. We are
		160	* guaranteed to deactivate in reverse order of the activation, so
		161	* just pick one active interrupt after the other in the ap_list,
		162	* and replay the deactivation as if the CPU was doing it. We also
		163	* rely on priority drop to have taken place, and the list to be
		164	* sorted by priority.
		165	*/
		166	list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
		167	u64 lr;
56		168
57	if (model == KVM_DEV_TYPE_ARM_VGIC_V3) {	169	/*
58	intid = val & ICH_LR_VIRTUAL_ID_MASK;	170	* I would have loved to write this using a scoped_guard(),
		171	* but using 'continue' here is a total train wreck.
		172	*/
		173	if (!eoicount) {
		174	break;
59	} else {	175	} else {
60	intid = val & GICH_LR_VIRTUALID;	176	guard(raw_spinlock)(&irq->irq_lock);
61	is_v2_sgi = vgic_irq_is_sgi(intid);	177
		178	if (!(likely(vgic_target_oracle(irq) == vcpu) &&
		179	irq->active))
		180	continue;
		181
		182	lr = vgic_v3_compute_lr(vcpu, irq) & ~ICH_LR_ACTIVE_BIT;
62	}	183	}
63		184
64	/* Notify fds when the guest EOI'ed a level-triggered IRQ */	185	if (lr & ICH_LR_HW)
65	if (lr_signals_eoi_mi(val) && vgic_valid_spi(vcpu->kvm, intid))	186	vgic_v3_deactivate_phys(FIELD_GET(ICH_LR_PHYS_ID_MASK, lr));
66	kvm_notify_acked_irq(vcpu->kvm, 0,
67	intid - VGIC_NR_PRIVATE_IRQS);
68		187
69	irq = vgic_get_vcpu_irq(vcpu, intid);	188	vgic_v3_fold_lr(vcpu, lr);
70	if (!irq) /* An LPI could have been unmapped. */	189	eoicount--;
71	continue;	190	}
72		191
73	raw_spin_lock(&irq->irq_lock);	192	cpuif->used_lrs = 0;
		193	}
74		194
75	/* Always preserve the active bit, note deactivation */	195	void vgic_v3_deactivate(struct kvm_vcpu *vcpu, u64 val)
76	deactivated = irq->active && !(val & ICH_LR_ACTIVE_BIT);	196	{
77	irq->active = !!(val & ICH_LR_ACTIVE_BIT);	197	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
		198	struct vgic_v3_cpu_if *cpuif = &vgic_cpu->vgic_v3;
		199	u32 model = vcpu->kvm->arch.vgic.vgic_model;
		200	struct kvm_vcpu *target_vcpu = NULL;
		201	bool mmio = false, is_v2_sgi;
		202	struct vgic_irq *irq;
		203	unsigned long flags;
		204	u64 lr = 0;
		205	u8 cpuid;
78		206
79	if (irq->active && is_v2_sgi)	207	/* Snapshot CPUID, and remove it from the INTID */
80	irq->active_source = cpuid;	208	cpuid = FIELD_GET(GENMASK_ULL(12, 10), val);
		209	val &= ~GENMASK_ULL(12, 10);
81		210
82	/* Edge is the only case where we preserve the pending bit */	211	is_v2_sgi = (model == KVM_DEV_TYPE_ARM_VGIC_V2 &&
83	if (irq->config == VGIC_CONFIG_EDGE &&	212	val < VGIC_NR_SGIS);
84	(val & ICH_LR_PENDING_BIT)) {
85	irq->pending_latch = true;
86		213
87	if (is_v2_sgi)	214	/*
88	irq->source \|= (1 << cpuid);	215	* We only deal with DIR when EOIMode==1, and only for SGI,
89	}	216	* PPI or SPI.
		217	*/
		218	if (!(cpuif->vgic_vmcr & ICH_VMCR_EOIM_MASK) \|\|
		219	val >= vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS)
		220	return;
		221
		222	/* Make sure we're in the same context as LR handling */
		223	local_irq_save(flags);
		224
		225	irq = vgic_get_vcpu_irq(vcpu, val);
		226	if (WARN_ON_ONCE(!irq))
		227	goto out;
		228
		229	/*
		230	* EOIMode=1: we must rely on traps to handle deactivate of
		231	* overflowing interrupts, as there is no ordering guarantee and
		232	* EOIcount isn't being incremented. Priority drop will have taken
		233	* place, as ICV_EOIxR_EL1 only affects the APRs and not the LRs.
		234	*
		235	* Three possibities:
		236	*
		237	* - The irq is not queued on any CPU, and there is nothing to
		238	* do,
		239	*
		240	* - Or the irq is in an LR, meaning that its state is not
		241	* directly observable. Treat it bluntly by making it as if
		242	* this was a write to GICD_ICACTIVER, which will force an
		243	* exit on all vcpus. If it hurts, don't do that.
		244	*
		245	* - Or the irq is active, but not in an LR, and we can
		246	* directly deactivate it by building a pseudo-LR, fold it,
		247	* and queue a request to prune the resulting ap_list,
		248	*
		249	* Special care must be taken to match the source CPUID when
		250	* deactivating a GICv2 SGI.
		251	*/
		252	scoped_guard(raw_spinlock, &irq->irq_lock) {
		253	target_vcpu = irq->vcpu;
		254
		255	/* Not on any ap_list? */
		256	if (!target_vcpu)
		257	goto put;
90		258
91	/*	259	/*
92	* Clear soft pending state when level irqs have been acked.	260	* Urgh. We're deactivating something that we cannot
		261	* observe yet... Big hammer time.
93	*/	262	*/
94	if (irq->config == VGIC_CONFIG_LEVEL && !(val & ICH_LR_STATE))	263	if (irq->on_lr) {
95	irq->pending_latch = false;	264	mmio = true;
		265	goto put;
		266	}
96		267
97	/* Handle resampling for mapped interrupts if required */	268	/* GICv2 SGI: check that the cpuid matches */
98	vgic_irq_handle_resampling(irq, deactivated, val & ICH_LR_PENDING_BIT);	269	if (is_v2_sgi && irq->active_source != cpuid) {
		270	target_vcpu = NULL;
		271	goto put;
		272	}
99		273
100	raw_spin_unlock(&irq->irq_lock);	274	/* (with a Dalek voice) DEACTIVATE!!!! */
101	vgic_put_irq(vcpu->kvm, irq);	275	lr = vgic_v3_compute_lr(vcpu, irq) & ~ICH_LR_ACTIVE_BIT;
102	}	276	}
103		277
104	cpuif->used_lrs = 0;	278	if (lr & ICH_LR_HW)
		279	vgic_v3_deactivate_phys(FIELD_GET(ICH_LR_PHYS_ID_MASK, lr));
		280
		281	vgic_v3_fold_lr(vcpu, lr);
		282
		283	put:
		284	vgic_put_irq(vcpu->kvm, irq);
		285
		286	out:
		287	local_irq_restore(flags);
		288
		289	if (mmio)
		290	vgic_mmio_write_cactive(vcpu, (val / 32) * 4, 4, BIT(val % 32));
		291
		292	/* Force the ap_list to be pruned */
		293	if (target_vcpu)
		294	kvm_make_request(KVM_REQ_VGIC_PROCESS_UPDATE, target_vcpu);
105	}	295	}
106		296
107	/* Requires the irq to be locked already */	297	/* Requires the irq to be locked already */
108	void vgic_v3_populate_lr(struct kvm_vcpu vcpu, struct vgic_irq irq, int lr)	298	static u64 vgic_v3_compute_lr(struct kvm_vcpu vcpu, struct vgic_irq irq)
109	{	299	{
110	u32 model = vcpu->kvm->arch.vgic.vgic_model;	300	u32 model = vcpu->kvm->arch.vgic.vgic_model;
111	u64 val = irq->intid;	301	u64 val = irq->intid;
112	bool allow_pending = true, is_v2_sgi;	302	bool allow_pending = true, is_v2_sgi;
113		303
		304	WARN_ON(irq->on_lr);
		305
114	is_v2_sgi = (vgic_irq_is_sgi(irq->intid) &&	306	is_v2_sgi = (vgic_irq_is_sgi(irq->intid) &&
115	model == KVM_DEV_TYPE_ARM_VGIC_V2);	307	model == KVM_DEV_TYPE_ARM_VGIC_V2);
116		308
@@ -150,6 +342,35 @@ void vgic_v3_populate_lr(struct kvm_vcpu vcpu, struct vgic_irq irq, int lr) · 33 unmodified lines
150	if (allow_pending && irq_is_pending(irq)) {	342	if (allow_pending && irq_is_pending(irq)) {
151	val \|= ICH_LR_PENDING_BIT;	343	val \|= ICH_LR_PENDING_BIT;
152		344
		345	if (is_v2_sgi) {
		346	u32 src = ffs(irq->source);
		347
		348	if (WARN_RATELIMIT(!src, "No SGI source for INTID %d\n",
		349	irq->intid))
		350	return 0;
		351
		352	val \|= (src - 1) << GICH_LR_PHYSID_CPUID_SHIFT;
		353	if (irq->source & ~BIT(src - 1))
		354	val \|= ICH_LR_EOI;
		355	}
		356	}
		357
		358	if (irq->group)
		359	val \|= ICH_LR_GROUP;
		360
		361	val \|= (u64)irq->priority << ICH_LR_PRIORITY_SHIFT;
		362
		363	return val;
		364	}
		365
		366	void vgic_v3_populate_lr(struct kvm_vcpu vcpu, struct vgic_irq irq, int lr)
		367	{
		368	u32 model = vcpu->kvm->arch.vgic.vgic_model;
		369	u64 val = vgic_v3_compute_lr(vcpu, irq);
		370
		371	vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[lr] = val;
		372
		373	if (val & ICH_LR_PENDING_BIT) {
153	if (irq->config == VGIC_CONFIG_EDGE)	374	if (irq->config == VGIC_CONFIG_EDGE)
154	irq->pending_latch = false;	375	irq->pending_latch = false;
155		376
@@ -157,16 +378,9 @@ void vgic_v3_populate_lr(struct kvm_vcpu vcpu, struct vgic_irq irq, int lr) · 1 unmodified line
157	model == KVM_DEV_TYPE_ARM_VGIC_V2) {	378	model == KVM_DEV_TYPE_ARM_VGIC_V2) {
158	u32 src = ffs(irq->source);	379	u32 src = ffs(irq->source);
159		380
160	if (WARN_RATELIMIT(!src, "No SGI source for INTID %d\n",	381	irq->source &= ~BIT(src - 1);
161	irq->intid))	382	if (irq->source)
162	return;
163
164	val \|= (src - 1) << GICH_LR_PHYSID_CPUID_SHIFT;
165	irq->source &= ~(1 << (src - 1));
166	if (irq->source) {
167	irq->pending_latch = true;	383	irq->pending_latch = true;
168	val \|= ICH_LR_EOI;
169	}
170	}	384	}
171	}	385	}
172		386
@@ -179,12 +393,7 @@ void vgic_v3_populate_lr(struct kvm_vcpu vcpu, struct vgic_irq irq, int lr) · 6 unmodified lines
179	if (vgic_irq_is_mapped_level(irq) && (val & ICH_LR_PENDING_BIT))	393	if (vgic_irq_is_mapped_level(irq) && (val & ICH_LR_PENDING_BIT))
180	irq->line_level = false;	394	irq->line_level = false;
181		395
182	if (irq->group)	396	irq->on_lr = true;
183	val \|= ICH_LR_GROUP;
184
185	val \|= (u64)irq->priority << ICH_LR_PRIORITY_SHIFT;
186
187	vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[lr] = val;
188	}	397	}
189		398
190	void vgic_v3_clear_lr(struct kvm_vcpu *vcpu, int lr)	399	void vgic_v3_clear_lr(struct kvm_vcpu *vcpu, int lr)
@@ -258,7 +467,7 @@ void vgic_v3_get_vmcr(struct kvm_vcpu vcpu, struct vgic_vmcr vmcrp) · 67 unmodified lines
258	GIC_BASER_CACHEABILITY(GICR_PENDBASER, OUTER, SameAsInner) \| \	467	GIC_BASER_CACHEABILITY(GICR_PENDBASER, OUTER, SameAsInner) \| \
259	GIC_BASER_SHAREABILITY(GICR_PENDBASER, InnerShareable))	468	GIC_BASER_SHAREABILITY(GICR_PENDBASER, InnerShareable))
260		469
261	void vgic_v3_enable(struct kvm_vcpu *vcpu)	470	void vgic_v3_reset(struct kvm_vcpu *vcpu)
262	{	471	{
263	struct vgic_v3_cpu_if *vgic_v3 = &vcpu->arch.vgic_cpu.vgic_v3;	472	struct vgic_v3_cpu_if *vgic_v3 = &vcpu->arch.vgic_cpu.vgic_v3;
264		473
@@ -288,9 +497,6 @@ void vgic_v3_enable(struct kvm_vcpu *vcpu) · 23 unmodified lines
288	kvm_vgic_global_state.ich_vtr_el2);	497	kvm_vgic_global_state.ich_vtr_el2);
289	vcpu->arch.vgic_cpu.num_pri_bits = FIELD_GET(ICH_VTR_EL2_PRIbits,	498	vcpu->arch.vgic_cpu.num_pri_bits = FIELD_GET(ICH_VTR_EL2_PRIbits,
290	kvm_vgic_global_state.ich_vtr_el2) + 1;	499	kvm_vgic_global_state.ich_vtr_el2) + 1;
291
292	/* Get the show on the road... */
293	vgic_v3->vgic_hcr = ICH_HCR_EL2_En;
294	}	500	}
295		501
296	void vcpu_set_ich_hcr(struct kvm_vcpu *vcpu)	502	void vcpu_set_ich_hcr(struct kvm_vcpu *vcpu)
@@ -302,20 +508,9 @@ void vcpu_set_ich_hcr(struct kvm_vcpu *vcpu) · 5 unmodified lines
302		508
303	/* Hide GICv3 sysreg if necessary */	509	/* Hide GICv3 sysreg if necessary */
304	if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2 \|\|	510	if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2 \|\|
305	!irqchip_in_kernel(vcpu->kvm)) {	511	!irqchip_in_kernel(vcpu->kvm))
306	vgic_v3->vgic_hcr \|= (ICH_HCR_EL2_TALL0 \| ICH_HCR_EL2_TALL1 \|	512	vgic_v3->vgic_hcr \|= (ICH_HCR_EL2_TALL0 \| ICH_HCR_EL2_TALL1 \|
307	ICH_HCR_EL2_TC);	513	ICH_HCR_EL2_TC);
308	return;
309	}
310
311	if (group0_trap)
312	vgic_v3->vgic_hcr \|= ICH_HCR_EL2_TALL0;
313	if (group1_trap)
314	vgic_v3->vgic_hcr \|= ICH_HCR_EL2_TALL1;
315	if (common_trap)
316	vgic_v3->vgic_hcr \|= ICH_HCR_EL2_TC;
317	if (dir_trap)
318	vgic_v3->vgic_hcr \|= ICH_HCR_EL2_TDIR;
319	}	514	}
320		515
321	int vgic_v3_lpi_sync_pending_status(struct kvm kvm, struct vgic_irq irq)	516	int vgic_v3_lpi_sync_pending_status(struct kvm kvm, struct vgic_irq irq)
@@ -636,8 +831,53 @@ static const struct midr_range broken_seis[] = { · 314 unmodified lines
636		831
637	static bool vgic_v3_broken_seis(void)	832	static bool vgic_v3_broken_seis(void)
638	{	833	{
639	return ((kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_EL2_SEIS) &&	834	return (is_kernel_in_hyp_mode() &&
640	is_midr_in_range_list(broken_seis));	835	is_midr_in_range_list(broken_seis) &&
		836	(read_sysreg_s(SYS_ICH_VTR_EL2) & ICH_VTR_EL2_SEIS));
		837	}
		838
		839	void noinstr kvm_compute_ich_hcr_trap_bits(struct alt_instr *alt,
		840	__le32 origptr, __le32 updptr,
		841	int nr_inst)
		842	{
		843	u32 insn, oinsn, rd;
		844	u64 hcr = 0;
		845
		846	if (cpus_have_cap(ARM64_WORKAROUND_CAVIUM_30115)) {
		847	group0_trap = true;
		848	group1_trap = true;
		849	}
		850
		851	if (vgic_v3_broken_seis()) {
		852	/* We know that these machines have ICH_HCR_EL2.TDIR */
		853	group0_trap = true;
		854	group1_trap = true;
		855	dir_trap = true;
		856	}
		857
		858	if (!cpus_have_cap(ARM64_HAS_ICH_HCR_EL2_TDIR))
		859	common_trap = true;
		860
		861	if (group0_trap)
		862	hcr \|= ICH_HCR_EL2_TALL0;
		863	if (group1_trap)
		864	hcr \|= ICH_HCR_EL2_TALL1;
		865	if (common_trap)
		866	hcr \|= ICH_HCR_EL2_TC;
		867	if (dir_trap)
		868	hcr \|= ICH_HCR_EL2_TDIR;
		869
		870	/* Compute target register */
		871	oinsn = le32_to_cpu(*origptr);
		872	rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
		873
		874	/* movz rd, #(val & 0xffff) */
		875	insn = aarch64_insn_gen_movewide(rd,
		876	(u16)hcr,
		877	0,
		878	AARCH64_INSN_VARIANT_64BIT,
		879	AARCH64_INSN_MOVEWIDE_ZERO);
		880	*updptr = cpu_to_le32(insn);
641	}	881	}
642		882
643	/**	883	/**
@@ -651,6 +891,7 @@ int vgic_v3_probe(const struct gic_kvm_info *info) · 7 unmodified lines
651	{	891	{
652	u64 ich_vtr_el2 = kvm_call_hyp_ret(__vgic_v3_get_gic_config);	892	u64 ich_vtr_el2 = kvm_call_hyp_ret(__vgic_v3_get_gic_config);
653	bool has_v2;	893	bool has_v2;
		894	u64 traps;
654	int ret;	895	int ret;
655		896
656	has_v2 = ich_vtr_el2 >> 63;	897	has_v2 = ich_vtr_el2 >> 63;
@@ -709,29 +950,18 @@ int vgic_v3_probe(const struct gic_kvm_info *info) · 52 unmodified lines
709	if (has_v2)	950	if (has_v2)
710	static_branch_enable(&vgic_v3_has_v2_compat);	951	static_branch_enable(&vgic_v3_has_v2_compat);
711		952
712	if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_30115)) {
713	group0_trap = true;
714	group1_trap = true;
715	}
716
717	if (vgic_v3_broken_seis()) {	953	if (vgic_v3_broken_seis()) {
718	kvm_info("GICv3 with broken locally generated SEI\n");	954	kvm_info("GICv3 with broken locally generated SEI\n");
719
720	kvm_vgic_global_state.ich_vtr_el2 &= ~ICH_VTR_EL2_SEIS;	955	kvm_vgic_global_state.ich_vtr_el2 &= ~ICH_VTR_EL2_SEIS;
721	group0_trap = true;
722	group1_trap = true;
723	if (ich_vtr_el2 & ICH_VTR_EL2_TDS)
724	dir_trap = true;
725	else
726	common_trap = true;
727	}	956	}
728		957
729	if (group0_trap \|\| group1_trap \|\| common_trap \| dir_trap) {	958	traps = vgic_ich_hcr_trap_bits();
		959	if (traps) {
730	kvm_info("GICv3 sysreg trapping enabled ([%s%s%s%s], reduced performance)\n",	960	kvm_info("GICv3 sysreg trapping enabled ([%s%s%s%s], reduced performance)\n",
731	group0_trap ? "G0" : "",	961	(traps & ICH_HCR_EL2_TALL0) ? "G0" : "",
732	group1_trap ? "G1" : "",	962	(traps & ICH_HCR_EL2_TALL1) ? "G1" : "",
733	common_trap ? "C" : "",	963	(traps & ICH_HCR_EL2_TC) ? "C" : "",
734	dir_trap ? "D" : "");	964	(traps & ICH_HCR_EL2_TDIR) ? "D" : "");
735	static_branch_enable(&vgic_v3_cpuif_trap);	965	static_branch_enable(&vgic_v3_cpuif_trap);
736	}	966	}
737		967
@@ -771,7 +1001,7 @@ void vgic_v3_put(struct kvm_vcpu *vcpu) · 33 unmodified lines
771	}	1001	}
772		1002
773	if (likely(!is_protected_kvm_enabled()))	1003	if (likely(!is_protected_kvm_enabled()))
774	kvm_call_hyp(__vgic_v3_save_vmcr_aprs, cpu_if);	1004	kvm_call_hyp(__vgic_v3_save_aprs, cpu_if);
775	WARN_ON(vgic_v4_put(vcpu));	1005	WARN_ON(vgic_v4_put(vcpu));
776		1006
777	if (has_vhe())	1007	if (has_vhe())

arch/arm64/kvm/vgic/vgic-v4.c+4−1

 		struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, i);
 		struct irq_desc *desc;
 		unsigned long flags;
+		bool pending;
 		int ret;
 		raw_spin_lock_irqsave(&irq->irq_lock, flags);
 		irq->hw = false;
 		ret = irq_get_irqchip_state(irq->host_irq,
 					    IRQCHIP_STATE_PENDING,
-					    &irq->pending_latch);
+					    &pending);
 		WARN_ON(ret);
+		irq->pending_latch = pending;
 		desc = irq_to_desc(irq->host_irq);
 		irq_domain_deactivate_irq(irq_desc_get_irq_data(desc));
 	unlock:

arch/arm64/kvm/vgic/vgic.c+186−112

@@ -244,7 +244,7 @@ void vgic_irq_set_phys_active(struct vgic_irq *irq, bool active) · 243 unmodified lines
244	*	244	*
245	* Requires the IRQ lock to be held.	245	* Requires the IRQ lock to be held.
246	*/	246	*/
247	static struct kvm_vcpu vgic_target_oracle(struct vgic_irq irq)	247	struct kvm_vcpu vgic_target_oracle(struct vgic_irq irq)
248	{	248	{
249	lockdep_assert_held(&irq->irq_lock);	249	lockdep_assert_held(&irq->irq_lock);
250		250
@@ -272,17 +272,20 @@ static struct kvm_vcpu vgic_target_oracle(struct vgic_irq irq) · 21 unmodified lines
272	return NULL;	272	return NULL;
273	}	273	}
274		274
		275	struct vgic_sort_info {
		276	struct kvm_vcpu *vcpu;
		277	struct vgic_vmcr vmcr;
		278	};
		279
275	/*	280	/*
276	* The order of items in the ap_lists defines how we'll pack things in LRs as	281	* The order of items in the ap_lists defines how we'll pack things in LRs as
277	* well, the first items in the list being the first things populated in the	282	* well, the first items in the list being the first things populated in the
278	* LRs.	283	* LRs.
279	*	284	*
280	* A hard rule is that active interrupts can never be pushed out of the LRs	285	* Pending, non-active interrupts must be placed at the head of the list.
281	* (and therefore take priority) since we cannot reliably trap on deactivation
282	* of IRQs and therefore they have to be present in the LRs.
283	*
284	* Otherwise things should be sorted by the priority field and the GIC	286	* Otherwise things should be sorted by the priority field and the GIC
285	* hardware support will take care of preemption of priority groups etc.	287	* hardware support will take care of preemption of priority groups etc.
		288	* Interrupts that are not deliverable should be at the end of the list.
286	*	289	*
287	* Return negative if "a" sorts before "b", 0 to preserve order, and positive	290	* Return negative if "a" sorts before "b", 0 to preserve order, and positive
288	* to sort "b" before "a".	291	* to sort "b" before "a".
@@ -292,6 +295,8 @@ static int vgic_irq_cmp(void priv, const struct list_head a, · 3 unmodified lines
292	{	295	{
293	struct vgic_irq *irqa = container_of(a, struct vgic_irq, ap_list);	296	struct vgic_irq *irqa = container_of(a, struct vgic_irq, ap_list);
294	struct vgic_irq *irqb = container_of(b, struct vgic_irq, ap_list);	297	struct vgic_irq *irqb = container_of(b, struct vgic_irq, ap_list);
		298	struct vgic_sort_info *info = priv;
		299	struct kvm_vcpu *vcpu = info->vcpu;
295	bool penda, pendb;	300	bool penda, pendb;
296	int ret;	301	int ret;
297		302
@@ -305,21 +310,32 @@ static int vgic_irq_cmp(void priv, const struct list_head a, · 7 unmodified lines
305	raw_spin_lock(&irqa->irq_lock);	310	raw_spin_lock(&irqa->irq_lock);
306	raw_spin_lock_nested(&irqb->irq_lock, SINGLE_DEPTH_NESTING);	311	raw_spin_lock_nested(&irqb->irq_lock, SINGLE_DEPTH_NESTING);
307		312
308	if (irqa->active \|\| irqb->active) {	313	/* Undeliverable interrupts should be last */
309	ret = (int)irqb->active - (int)irqa->active;	314	ret = (int)(vgic_target_oracle(irqb) == vcpu) - (int)(vgic_target_oracle(irqa) == vcpu);
		315	if (ret)
		316	goto out;
		317
		318	/* Same thing for interrupts targeting a disabled group */
		319	ret = (int)(irqb->group ? info->vmcr.grpen1 : info->vmcr.grpen0);
		320	ret -= (int)(irqa->group ? info->vmcr.grpen1 : info->vmcr.grpen0);
		321	if (ret)
310	goto out;	322	goto out;
311	}
312		323
313	penda = irqa->enabled && irq_is_pending(irqa);	324	penda = irqa->enabled && irq_is_pending(irqa) && !irqa->active;
314	pendb = irqb->enabled && irq_is_pending(irqb);	325	pendb = irqb->enabled && irq_is_pending(irqb) && !irqb->active;
315		326
316	if (!penda \|\| !pendb) {	327	ret = (int)pendb - (int)penda;
317	ret = (int)pendb - (int)penda;	328	if (ret)
318	goto out;	329	goto out;
319	}
320		330
321	/* Both pending and enabled, sort by priority */	331	/* Both pending and enabled, sort by priority (lower number first) */
322	ret = irqa->priority - irqb->priority;	332	ret = (int)irqa->priority - (int)irqb->priority;
		333	if (ret)
		334	goto out;
		335
		336	/* Finally, HW bit active interrupts have priority over non-HW ones */
		337	ret = (int)irqb->hw - (int)irqa->hw;
		338
323	out:	339	out:
324	raw_spin_unlock(&irqb->irq_lock);	340	raw_spin_unlock(&irqb->irq_lock);
325	raw_spin_unlock(&irqa->irq_lock);	341	raw_spin_unlock(&irqa->irq_lock);
@@ -330,10 +346,12 @@ out: · 4 unmodified lines
330	static void vgic_sort_ap_list(struct kvm_vcpu *vcpu)	346	static void vgic_sort_ap_list(struct kvm_vcpu *vcpu)
331	{	347	{
332	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;	348	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
		349	struct vgic_sort_info info = { .vcpu = vcpu, };
333		350
334	lockdep_assert_held(&vgic_cpu->ap_list_lock);	351	lockdep_assert_held(&vgic_cpu->ap_list_lock);
335		352
336	list_sort(NULL, &vgic_cpu->ap_list_head, vgic_irq_cmp);	353	vgic_get_vmcr(vcpu, &info.vmcr);
		354	list_sort(&info, &vgic_cpu->ap_list_head, vgic_irq_cmp);
337	}	355	}
338		356
339	/*	357	/*
@@ -356,6 +374,20 @@ static bool vgic_validate_injection(struct vgic_irq irq, bool level, void owne · 16 unmodified lines
356	return false;	374	return false;
357	}	375	}
358		376
		377	static bool vgic_model_needs_bcst_kick(struct kvm *kvm)
		378	{
		379	/*
		380	* A GICv3 (or GICv3-like) system exposing a GICv3 to the guest
		381	* needs a broadcast kick to set TDIR globally.
		382	*
		383	* For systems that do not have TDIR (ARM's own v8.0 CPUs), the
		384	* shadow TDIR bit is always set, and so is the register's TC bit,
		385	* so no need to kick the CPUs.
		386	*/
		387	return (cpus_have_final_cap(ARM64_HAS_ICH_HCR_EL2_TDIR) &&
		388	kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3);
		389	}
		390
359	/*	391	/*
360	* Check whether an IRQ needs to (and can) be queued to a VCPU's ap list.	392	* Check whether an IRQ needs to (and can) be queued to a VCPU's ap list.
361	* Do the queuing if necessary, taking the right locks in the right order.	393	* Do the queuing if necessary, taking the right locks in the right order.
@@ -368,6 +400,7 @@ bool vgic_queue_irq_unlock(struct kvm kvm, struct vgic_irq irq, · 6 unmodified lines
368	unsigned long flags) __releases(&irq->irq_lock)	400	unsigned long flags) __releases(&irq->irq_lock)
369	{	401	{
370	struct kvm_vcpu *vcpu;	402	struct kvm_vcpu *vcpu;
		403	bool bcast;
371		404
372	lockdep_assert_held(&irq->irq_lock);	405	lockdep_assert_held(&irq->irq_lock);
373		406
@@ -442,11 +475,20 @@ retry: · 68 unmodified lines
442	list_add_tail(&irq->ap_list, &vcpu->arch.vgic_cpu.ap_list_head);	475	list_add_tail(&irq->ap_list, &vcpu->arch.vgic_cpu.ap_list_head);
443	irq->vcpu = vcpu;	476	irq->vcpu = vcpu;
444		477
		478	/* A new SPI may result in deactivation trapping on all vcpus */
		479	bcast = (vgic_model_needs_bcst_kick(vcpu->kvm) &&
		480	vgic_valid_spi(vcpu->kvm, irq->intid) &&
		481	atomic_fetch_inc(&vcpu->kvm->arch.vgic.active_spis) == 0);
		482
445	raw_spin_unlock(&irq->irq_lock);	483	raw_spin_unlock(&irq->irq_lock);
446	raw_spin_unlock_irqrestore(&vcpu->arch.vgic_cpu.ap_list_lock, flags);	484	raw_spin_unlock_irqrestore(&vcpu->arch.vgic_cpu.ap_list_lock, flags);
447		485
448	kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);	486	if (!bcast) {
449	kvm_vcpu_kick(vcpu);	487	kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
		488	kvm_vcpu_kick(vcpu);
		489	} else {
		490	kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_IRQ_PENDING);
		491	}
450		492
451	return true;	493	return true;
452	}	494	}
@@ -798,98 +840,148 @@ static inline void vgic_clear_lr(struct kvm_vcpu *vcpu, int lr) · 345 unmodified lines
798	vgic_v3_clear_lr(vcpu, lr);	840	vgic_v3_clear_lr(vcpu, lr);
799	}	841	}
800		842
801	static inline void vgic_set_underflow(struct kvm_vcpu *vcpu)	843	static void summarize_ap_list(struct kvm_vcpu *vcpu,
802	{	844	struct ap_list_summary *als)
803	if (kvm_vgic_global_state.type == VGIC_V2)
804	vgic_v2_set_underflow(vcpu);
805	else
806	vgic_v3_set_underflow(vcpu);
807	}
808
809	/* Requires the ap_list_lock to be held. */
810	static int compute_ap_list_depth(struct kvm_vcpu *vcpu,
811	bool *multi_sgi)
812	{	845	{
813	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;	846	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
814	struct vgic_irq *irq;	847	struct vgic_irq *irq;
815	int count = 0;
816
817	*multi_sgi = false;
818		848
819	lockdep_assert_held(&vgic_cpu->ap_list_lock);	849	lockdep_assert_held(&vgic_cpu->ap_list_lock);
820		850
		851	als = (typeof(als)){};
		852
821	list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {	853	list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
822	int w;	854	guard(raw_spinlock)(&irq->irq_lock);
823		855
824	raw_spin_lock(&irq->irq_lock);	856	if (unlikely(vgic_target_oracle(irq) != vcpu))
825	/* GICv2 SGIs can count for more than one... */	857	continue;
826	w = vgic_irq_get_lr_count(irq);	858
827	raw_spin_unlock(&irq->irq_lock);	859	if (!irq->active)
		860	als->nr_pend++;
		861	else
		862	als->nr_act++;
828		863
829	count += w;	864	if (irq->intid < VGIC_NR_SGIS)
830	*multi_sgi \|= (w > 1);	865	als->nr_sgi++;
831	}	866	}
832	return count;
833	}	867	}
834		868
835	/* Requires the VCPU's ap_list_lock to be held. */	869	/*
		870	* Dealing with LR overflow is close to black magic -- dress accordingly.
		871	*
		872	* We have to present an almost infinite number of interrupts through a very
		873	* limited number of registers. Therefore crucial decisions must be made to
		874	* ensure we feed the most relevant interrupts into the LRs, and yet have
		875	* some facilities to let the guest interact with those that are not there.
		876	*
		877	* All considerations below are in the context of interrupts targeting a
		878	* single vcpu with non-idle state (either pending, active, or both),
		879	* colloquially called the ap_list:
		880	*
		881	* - Pending interrupts must have priority over active interrupts. This also
		882	* excludes pending+active interrupts. This ensures that a guest can
		883	* perform priority drops on any number of interrupts, and yet be
		884	* presented the next pending one.
		885	*
		886	* - Deactivation of interrupts outside of the LRs must be tracked by using
		887	* either the EOIcount-driven maintenance interrupt, and sometimes by
		888	* trapping the DIR register.
		889	*
		890	* - For EOImode=0, a non-zero EOIcount means walking the ap_list past the
		891	* point that made it into the LRs, and deactivate interrupts that would
		892	* have made it onto the LRs if we had the space.
		893	*
		894	* - The MI-generation bits must be used to try and force an exit when the
		895	* guest has done enough changes to the LRs that we want to reevaluate the
		896	* situation:
		897	*
		898	* - if the total number of pending interrupts exceeds the number of
		899	* LR, NPIE must be set in order to exit once no pending interrupts
		900	* are present in the LRs, allowing us to populate the next batch.
		901	*
		902	* - if there are active interrupts outside of the LRs, then LRENPIE
		903	* must be set so that we exit on deactivation of one of these, and
		904	* work out which one is to be deactivated. Note that this is not
		905	* enough to deal with EOImode=1, see below.
		906	*
		907	* - if the overall number of interrupts exceeds the number of LRs,
		908	* then UIE must be set to allow refilling of the LRs once the
		909	* majority of them has been processed.
		910	*
		911	* - as usual, MI triggers are only an optimisation, since we cannot
		912	* rely on the MI being delivered in timely manner...
		913	*
		914	* - EOImode=1 creates some additional problems:
		915	*
		916	* - deactivation can happen in any order, and we cannot rely on
		917	* EOImode=0's coupling of priority-drop and deactivation which
		918	* imposes strict reverse Ack order. This means that DIR must
		919	* trap if we have active interrupts outside of the LRs.
		920	*
		921	* - deactivation of SPIs can occur on any CPU, while the SPI is only
		922	* present in the ap_list of the CPU that actually ack-ed it. In that
		923	* case, EOIcount doesn't provide enough information, and we must
		924	* resort to trapping DIR even if we don't overflow the LRs. Bonus
		925	* point for not trapping DIR when no SPIs are pending or active in
		926	* the whole VM.
		927	*
		928	* - LPIs do not suffer the same problem as SPIs on deactivation, as we
		929	* have to essentially discard the active state, see below.
		930	*
		931	* - Virtual LPIs have an active state (surprise!), which gets removed on
		932	* priority drop (EOI). However, EOIcount doesn't get bumped when the LPI
		933	* is not present in the LR (surprise again!). Special care must therefore
		934	* be taken to remove the active state from any activated LPI when exiting
		935	* from the guest. This is in a way no different from what happens on the
		936	* physical side. We still rely on the running priority to have been
		937	* removed from the APRs, irrespective of the LPI being present in the LRs
		938	* or not.
		939	*
		940	* - Virtual SGIs directly injected via GICv4.1 must not affect EOIcount, as
		941	* they are not managed in SW and don't have a true active state. So only
		942	* set vSGIEOICount when no SGIs are in the ap_list.
		943	*
		944	* - GICv2 SGIs with multiple sources are injected one source at a time, as
		945	* if they were made pending sequentially. This may mean that we don't
		946	* always present the HPPI if other interrupts with lower priority are
		947	* pending in the LRs. Big deal.
		948	*/
836	static void vgic_flush_lr_state(struct kvm_vcpu *vcpu)	949	static void vgic_flush_lr_state(struct kvm_vcpu *vcpu)
837	{	950	{
838	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;	951	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
		952	struct ap_list_summary als;
839	struct vgic_irq *irq;	953	struct vgic_irq *irq;
840	int count;	954	int count = 0;
841	bool multi_sgi;
842	u8 prio = 0xff;
843	int i = 0;
844		955
845	lockdep_assert_held(&vgic_cpu->ap_list_lock);	956	lockdep_assert_held(&vgic_cpu->ap_list_lock);
846		957
847	count = compute_ap_list_depth(vcpu, &multi_sgi);	958	summarize_ap_list(vcpu, &als);
848	if (count > kvm_vgic_global_state.nr_lr \|\| multi_sgi)
849	vgic_sort_ap_list(vcpu);
850		959
851	count = 0;	960	if (irqs_outside_lrs(&als))
		961	vgic_sort_ap_list(vcpu);
852		962
853	list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {	963	list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
854	raw_spin_lock(&irq->irq_lock);	964	scoped_guard(raw_spinlock, &irq->irq_lock) {
855		965	if (likely(vgic_target_oracle(irq) == vcpu)) {
856	/*	966	vgic_populate_lr(vcpu, irq, count++);
857	* If we have multi-SGIs in the pipeline, we need to	967	}
858	* guarantee that they are all seen before any IRQ of
859	* lower priority. In that case, we need to filter out
860	* these interrupts by exiting early. This is easy as
861	* the AP list has been sorted already.
862	*/
863	if (multi_sgi && irq->priority > prio) {
864	raw_spin_unlock(&irq->irq_lock);
865	break;
866	}
867
868	if (likely(vgic_target_oracle(irq) == vcpu)) {
869	vgic_populate_lr(vcpu, irq, count++);
870
871	if (irq->source)
872	prio = irq->priority;
873	}	968	}
874		969
875	raw_spin_unlock(&irq->irq_lock);	970	if (count == kvm_vgic_global_state.nr_lr)
876
877	if (count == kvm_vgic_global_state.nr_lr) {
878	if (!list_is_last(&irq->ap_list,
879	&vgic_cpu->ap_list_head))
880	vgic_set_underflow(vcpu);
881	break;	971	break;
882	}
883	}	972	}
884		973
885	/* Nuke remaining LRs */	974	/* Nuke remaining LRs */
886	for (i = count ; i < kvm_vgic_global_state.nr_lr; i++)	975	for (int i = count ; i < kvm_vgic_global_state.nr_lr; i++)
887	vgic_clear_lr(vcpu, i);	976	vgic_clear_lr(vcpu, i);
888		977
889	if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))	978	if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
890	vcpu->arch.vgic_cpu.vgic_v2.used_lrs = count;	979	vcpu->arch.vgic_cpu.vgic_v2.used_lrs = count;
891	else	980	vgic_v2_configure_hcr(vcpu, &als);
		981	} else {
892	vcpu->arch.vgic_cpu.vgic_v3.used_lrs = count;	982	vcpu->arch.vgic_cpu.vgic_v3.used_lrs = count;
		983	vgic_v3_configure_hcr(vcpu, &als);
		984	}
893	}	985	}
894		986
895	static inline bool can_access_vgic_from_kernel(void)	987	static inline bool can_access_vgic_from_kernel(void)
@@ -913,8 +1005,6 @@ static inline void vgic_save_state(struct kvm_vcpu *vcpu) · 17 unmodified lines
913	/* Sync back the hardware VGIC state into our emulation after a guest's run. */	1005	/* Sync back the hardware VGIC state into our emulation after a guest's run. */
914	void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)	1006	void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
915	{	1007	{
916	int used_lrs;
917
918	/* If nesting, emulate the HW effect from L0 to L1 */	1008	/* If nesting, emulate the HW effect from L0 to L1 */
919	if (vgic_state_is_nested(vcpu)) {	1009	if (vgic_state_is_nested(vcpu)) {
920	vgic_v3_sync_nested(vcpu);	1010	vgic_v3_sync_nested(vcpu);
@@ -924,21 +1014,22 @@ void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu) · 3 unmodified lines
924	if (vcpu_has_nv(vcpu))	1014	if (vcpu_has_nv(vcpu))
925	vgic_v3_nested_update_mi(vcpu);	1015	vgic_v3_nested_update_mi(vcpu);
926		1016
927	/* An empty ap_list_head implies used_lrs == 0 */
928	if (list_empty(&vcpu->arch.vgic_cpu.ap_list_head))
929	return;
930
931	if (can_access_vgic_from_kernel())	1017	if (can_access_vgic_from_kernel())
932	vgic_save_state(vcpu);	1018	vgic_save_state(vcpu);
933		1019
934	if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif))	1020	vgic_fold_lr_state(vcpu);
935	used_lrs = vcpu->arch.vgic_cpu.vgic_v2.used_lrs;	1021	vgic_prune_ap_list(vcpu);
936	else	1022	}
937	used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;	1023
		1024	/* Sync interrupts that were deactivated through a DIR trap */
		1025	void kvm_vgic_process_async_update(struct kvm_vcpu *vcpu)
		1026	{
		1027	unsigned long flags;
938		1028
939	if (used_lrs)	1029	/* Make sure we're in the same context as LR handling */
940	vgic_fold_lr_state(vcpu);	1030	local_irq_save(flags);
941	vgic_prune_ap_list(vcpu);	1031	vgic_prune_ap_list(vcpu);
		1032	local_irq_restore(flags);
942	}	1033	}
943		1034
944	static inline void vgic_restore_state(struct kvm_vcpu *vcpu)	1035	static inline void vgic_restore_state(struct kvm_vcpu *vcpu)
@@ -965,8 +1056,9 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu) · 20 unmodified lines
965	* abort the entry procedure and inject the exception at the	1056	* abort the entry procedure and inject the exception at the
966	* beginning of the run loop.	1057	* beginning of the run loop.
967	*	1058	*
968	* - Otherwise, do exactly NOTHING. The guest state is	1059	* - Otherwise, do exactly NOTHING apart from enabling the virtual
969	* already loaded, and we can carry on with running it.	1060	* CPU interface. The guest state is already loaded, and we can
		1061	* carry on with running it.
970	*	1062	*
971	* If we have NV, but are not in a nested state, compute the	1063	* If we have NV, but are not in a nested state, compute the
972	* maintenance interrupt state, as it may fire.	1064	* maintenance interrupt state, as it may fire.
@@ -975,35 +1067,17 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu) · 2 unmodified lines
975	if (kvm_vgic_vcpu_pending_irq(vcpu))	1067	if (kvm_vgic_vcpu_pending_irq(vcpu))
976	kvm_make_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu);	1068	kvm_make_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu);
977		1069
		1070	vgic_v3_flush_nested(vcpu);
978	return;	1071	return;
979	}	1072	}
980		1073
981	if (vcpu_has_nv(vcpu))	1074	if (vcpu_has_nv(vcpu))
982	vgic_v3_nested_update_mi(vcpu);	1075	vgic_v3_nested_update_mi(vcpu);
983		1076
984	/*
985	* If there are no virtual interrupts active or pending for this
986	* VCPU, then there is no work to do and we can bail out without
987	* taking any lock. There is a potential race with someone injecting
988	* interrupts to the VCPU, but it is a benign race as the VCPU will
989	* either observe the new interrupt before or after doing this check,
990	* and introducing additional synchronization mechanism doesn't change
991	* this.
992	*
993	* Note that we still need to go through the whole thing if anything
994	* can be directly injected (GICv4).
995	*/
996	if (list_empty(&vcpu->arch.vgic_cpu.ap_list_head) &&
997	!vgic_supports_direct_irqs(vcpu->kvm))
998	return;
999
1000	DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());	1077	DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
1001		1078
1002	if (!list_empty(&vcpu->arch.vgic_cpu.ap_list_head)) {	1079	scoped_guard(raw_spinlock, &vcpu->arch.vgic_cpu.ap_list_lock)
1003	raw_spin_lock(&vcpu->arch.vgic_cpu.ap_list_lock);
1004	vgic_flush_lr_state(vcpu);	1080	vgic_flush_lr_state(vcpu);
1005	raw_spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock);
1006	}
1007		1081
1008	if (can_access_vgic_from_kernel())	1082	if (can_access_vgic_from_kernel())
1009	vgic_restore_state(vcpu);	1083	vgic_restore_state(vcpu);

arch/arm64/kvm/vgic/vgic.h+39−4

 	return ret;
 }
+void kvm_compute_ich_hcr_trap_bits(struct alt_instr *alt,
+				   __le32 *origptr, __le32 *updptr, int nr_inst);
+static inline u64 vgic_ich_hcr_trap_bits(void)
+{
+	u64 hcr;
+	/* All the traps are in the bottom 16bits */
+	asm volatile(ALTERNATIVE_CB("movz %0, #0\n",
+				    ARM64_ALWAYS_SYSTEM,
+				    kvm_compute_ich_hcr_trap_bits)
+		     : "=r" (hcr));
+	return hcr;
+}
 /*
  * This struct provides an intermediate representation of the fields contained
  * in the GICH_VMCR and ICH_VMCR registers, such that code exporting the GIC
 	u32 event_id;
 };
+struct ap_list_summary {
+	unsigned int	nr_pend;	/* purely pending, not active */
+	unsigned int	nr_act;		/* active, or active+pending */
+	unsigned int	nr_sgi;		/* any SGI */
+};
+#define irqs_outside_lrs(s)						\
+	 (((s)->nr_pend + (s)->nr_act) > kvm_vgic_global_state.nr_lr)
+#define irqs_pending_outside_lrs(s)			\
+	((s)->nr_pend > kvm_vgic_global_state.nr_lr)
+#define irqs_active_outside_lrs(s)		\
+	((s)->nr_act &&	irqs_outside_lrs(s))
 int vgic_v3_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr,
 		       struct vgic_reg_attr *reg_attr);
 int vgic_v2_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr,
 struct vgic_irq *vgic_get_irq(struct kvm *kvm, u32 intid);
 struct vgic_irq *vgic_get_vcpu_irq(struct kvm_vcpu *vcpu, u32 intid);
 void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq);
+struct kvm_vcpu *vgic_target_oracle(struct vgic_irq *irq);
 bool vgic_get_phys_line_level(struct vgic_irq *irq);
 void vgic_irq_set_phys_pending(struct vgic_irq *irq, bool pending);
 void vgic_irq_set_phys_active(struct vgic_irq *irq, bool active);
 void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu);
 void vgic_v2_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr);
+void vgic_v2_deactivate(struct kvm_vcpu *vcpu, u32 val);
 void vgic_v2_clear_lr(struct kvm_vcpu *vcpu, int lr);
-void vgic_v2_set_underflow(struct kvm_vcpu *vcpu);
+void vgic_v2_configure_hcr(struct kvm_vcpu *vcpu, struct ap_list_summary *als);
 int vgic_v2_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr);
 int vgic_v2_dist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
 			 int offset, u32 *val);
 			  int offset, u32 *val);
 void vgic_v2_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
 void vgic_v2_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
-void vgic_v2_enable(struct kvm_vcpu *vcpu);
+void vgic_v2_reset(struct kvm_vcpu *vcpu);
 int vgic_v2_probe(const struct gic_kvm_info *info);
 int vgic_v2_map_resources(struct kvm *kvm);
 int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address,
 void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu);
 void vgic_v3_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr);
 void vgic_v3_clear_lr(struct kvm_vcpu *vcpu, int lr);
-void vgic_v3_set_underflow(struct kvm_vcpu *vcpu);
+void vgic_v3_deactivate(struct kvm_vcpu *vcpu, u64 val);
+void vgic_v3_configure_hcr(struct kvm_vcpu *vcpu, struct ap_list_summary *als);
 void vgic_v3_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
 void vgic_v3_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
-void vgic_v3_enable(struct kvm_vcpu *vcpu);
+void vgic_v3_reset(struct kvm_vcpu *vcpu);
 int vgic_v3_probe(const struct gic_kvm_info *info);
 int vgic_v3_map_resources(struct kvm *kvm);
 int vgic_v3_lpi_sync_pending_status(struct kvm *kvm, struct vgic_irq *irq);
 	return kvm_has_feat(kvm, ID_AA64PFR0_EL1, GIC, IMP);
 }
+void vgic_v3_flush_nested(struct kvm_vcpu *vcpu);
 void vgic_v3_sync_nested(struct kvm_vcpu *vcpu);
 void vgic_v3_load_nested(struct kvm_vcpu *vcpu);
 void vgic_v3_put_nested(struct kvm_vcpu *vcpu);

arch/arm64/tools/cpucaps+2−0

 HAS_GICV5_LEGACY
 HAS_GIC_PRIO_MASKING
 HAS_GIC_PRIO_RELAXED_SYNC
+HAS_ICH_HCR_EL2_TDIR
 HAS_HCR_NV1
 HAS_HCX
 HAS_LDAPR
 HAS_VA52
 HAS_VIRT_HOST_EXTN
 HAS_WFXT
+HAS_XNX
 HAFT
 HW_DBM
 KVM_HVHE

arch/loongarch/include/asm/kvm_eiointc.h+8−47

@@ -10,10 +10,7 @@ · 9 unmodified lines
10		10
11	#define EIOINTC_IRQS 256	11	#define EIOINTC_IRQS 256
12	#define EIOINTC_ROUTE_MAX_VCPUS 256	12	#define EIOINTC_ROUTE_MAX_VCPUS 256
13	#define EIOINTC_IRQS_U8_NUMS (EIOINTC_IRQS / 8)	13	#define EIOINTC_IRQS_U64_NUMS (EIOINTC_IRQS / 64)
14	#define EIOINTC_IRQS_U16_NUMS (EIOINTC_IRQS_U8_NUMS / 2)
15	#define EIOINTC_IRQS_U32_NUMS (EIOINTC_IRQS_U8_NUMS / 4)
16	#define EIOINTC_IRQS_U64_NUMS (EIOINTC_IRQS_U8_NUMS / 8)
17	/* map to ipnum per 32 irqs */	14	/* map to ipnum per 32 irqs */
18	#define EIOINTC_IRQS_NODETYPE_COUNT 16	15	#define EIOINTC_IRQS_NODETYPE_COUNT 16
19		16
@@ -64,54 +61,18 @@ struct loongarch_eiointc { · 44 unmodified lines
64	uint32_t status;	61	uint32_t status;
65		62
66	/* hardware state */	63	/* hardware state */
67	union nodetype {	64	u64 nodetype[EIOINTC_IRQS_NODETYPE_COUNT / 4];
68	u64 reg_u64[EIOINTC_IRQS_NODETYPE_COUNT / 4];
69	u32 reg_u32[EIOINTC_IRQS_NODETYPE_COUNT / 2];
70	u16 reg_u16[EIOINTC_IRQS_NODETYPE_COUNT];
71	u8 reg_u8[EIOINTC_IRQS_NODETYPE_COUNT * 2];
72	} nodetype;
73		65
74	/* one bit shows the state of one irq */	66	/* one bit shows the state of one irq */
75	union bounce {	67	u64 bounce[EIOINTC_IRQS_U64_NUMS];
76	u64 reg_u64[EIOINTC_IRQS_U64_NUMS];	68	u64 isr[EIOINTC_IRQS_U64_NUMS];
77	u32 reg_u32[EIOINTC_IRQS_U32_NUMS];	69	u64 coreisr[EIOINTC_ROUTE_MAX_VCPUS][EIOINTC_IRQS_U64_NUMS];
78	u16 reg_u16[EIOINTC_IRQS_U16_NUMS];	70	u64 enable[EIOINTC_IRQS_U64_NUMS];
79	u8 reg_u8[EIOINTC_IRQS_U8_NUMS];
80	} bounce;
81
82	union isr {
83	u64 reg_u64[EIOINTC_IRQS_U64_NUMS];
84	u32 reg_u32[EIOINTC_IRQS_U32_NUMS];
85	u16 reg_u16[EIOINTC_IRQS_U16_NUMS];
86	u8 reg_u8[EIOINTC_IRQS_U8_NUMS];
87	} isr;
88	union coreisr {
89	u64 reg_u64[EIOINTC_ROUTE_MAX_VCPUS][EIOINTC_IRQS_U64_NUMS];
90	u32 reg_u32[EIOINTC_ROUTE_MAX_VCPUS][EIOINTC_IRQS_U32_NUMS];
91	u16 reg_u16[EIOINTC_ROUTE_MAX_VCPUS][EIOINTC_IRQS_U16_NUMS];
92	u8 reg_u8[EIOINTC_ROUTE_MAX_VCPUS][EIOINTC_IRQS_U8_NUMS];
93	} coreisr;
94	union enable {
95	u64 reg_u64[EIOINTC_IRQS_U64_NUMS];
96	u32 reg_u32[EIOINTC_IRQS_U32_NUMS];
97	u16 reg_u16[EIOINTC_IRQS_U16_NUMS];
98	u8 reg_u8[EIOINTC_IRQS_U8_NUMS];
99	} enable;
100		71
101	/* use one byte to config ipmap for 32 irqs at once */	72	/* use one byte to config ipmap for 32 irqs at once */
102	union ipmap {	73	u64 ipmap;
103	u64 reg_u64;
104	u32 reg_u32[EIOINTC_IRQS_U32_NUMS / 4];
105	u16 reg_u16[EIOINTC_IRQS_U16_NUMS / 4];
106	u8 reg_u8[EIOINTC_IRQS_U8_NUMS / 4];
107	} ipmap;
108	/* use one byte to config coremap for one irq */	74	/* use one byte to config coremap for one irq */
109	union coremap {	75	u64 coremap[EIOINTC_IRQS / 8];
110	u64 reg_u64[EIOINTC_IRQS / 8];
111	u32 reg_u32[EIOINTC_IRQS / 4];
112	u16 reg_u16[EIOINTC_IRQS / 2];
113	u8 reg_u8[EIOINTC_IRQS];
114	} coremap;
115		76
116	DECLARE_BITMAP(sw_coreisr[EIOINTC_ROUTE_MAX_VCPUS][LOONGSON_IP_NUM], EIOINTC_IRQS);	77	DECLARE_BITMAP(sw_coreisr[EIOINTC_ROUTE_MAX_VCPUS][LOONGSON_IP_NUM], EIOINTC_IRQS);
117	uint8_t sw_coremap[EIOINTC_IRQS];	78	uint8_t sw_coremap[EIOINTC_IRQS];

arch/loongarch/include/asm/kvm_host.h+8−0

 	struct kvm_phyid_map  *phyid_map;
 	/* Enabled PV features */
 	unsigned long pv_features;
+	/* Supported KVM features */
+	unsigned long kvm_features;
 	s64 time_offset;
 	struct kvm_context __percpu *vmcs;
 	return (arch->cpucfg[6] & CPUCFG6_PMNUM) >> CPUCFG6_PMNUM_SHIFT;
 }
+/* Check whether KVM support this feature (VMM may disable it) */
+static inline bool kvm_vm_support(struct kvm_arch *arch, int feature)
+{
+	return !!(arch->kvm_features & BIT_ULL(feature));
+}
 bool kvm_arch_pmi_in_guest(struct kvm_vcpu *vcpu);
 /* Debug: dump vcpu state */

arch/loongarch/include/asm/kvm_vcpu.h+1−0

@@ -15,6 +15,7 @@ · 14 unmodified lines
15	#define CPU_PMU (_ULCAST_(1) << 10)	15	#define CPU_PMU (_ULCAST_(1) << 10)
16	#define CPU_TIMER (_ULCAST_(1) << 11)	16	#define CPU_TIMER (_ULCAST_(1) << 11)
17	#define CPU_IPI (_ULCAST_(1) << 12)	17	#define CPU_IPI (_ULCAST_(1) << 12)
		18	#define CPU_AVEC (_ULCAST_(1) << 14)
18		19
19	/* Controlled by 0x52 guest exception VIP aligned to estat bit 5~12 */	20	/* Controlled by 0x52 guest exception VIP aligned to estat bit 5~12 */
20	#define CPU_IP0 (_ULCAST_(1))	21	#define CPU_IP0 (_ULCAST_(1))

arch/loongarch/include/asm/loongarch.h+2−0

 #define  CSR_GCFG_GPERF_SHIFT		24
 #define  CSR_GCFG_GPERF_WIDTH		3
 #define  CSR_GCFG_GPERF			(_ULCAST_(0x7) << CSR_GCFG_GPERF_SHIFT)
+#define  CSR_GCFG_GPMP_SHIFT		23
+#define  CSR_GCFG_GPMP			(_ULCAST_(0x1) << CSR_GCFG_GPMP_SHIFT)
 #define  CSR_GCFG_GCI_SHIFT		20
 #define  CSR_GCFG_GCI_WIDTH		2
 #define  CSR_GCFG_GCI			(_ULCAST_(0x3) << CSR_GCFG_GCI_SHIFT)

arch/loongarch/include/uapi/asm/kvm.h+1−0

@@ -104,6 +104,7 @@ struct kvm_fpu { · 103 unmodified lines
104	#define KVM_LOONGARCH_VM_FEAT_PV_IPI 6	104	#define KVM_LOONGARCH_VM_FEAT_PV_IPI 6
105	#define KVM_LOONGARCH_VM_FEAT_PV_STEALTIME 7	105	#define KVM_LOONGARCH_VM_FEAT_PV_STEALTIME 7
106	#define KVM_LOONGARCH_VM_FEAT_PTW 8	106	#define KVM_LOONGARCH_VM_FEAT_PTW 8
		107	#define KVM_LOONGARCH_VM_FEAT_MSGINT 9
107		108
108	/* Device Control API on vcpu fd */	109	/* Device Control API on vcpu fd */
109	#define KVM_LOONGARCH_VCPU_CPUCFG 0	110	#define KVM_LOONGARCH_VCPU_CPUCFG 0

arch/loongarch/kvm/Kconfig+0−1

@@ -25,7 +25,6 @@ config KVM · 24 unmodified lines
25	select HAVE_KVM_IRQCHIP	25	select HAVE_KVM_IRQCHIP
26	select HAVE_KVM_MSI	26	select HAVE_KVM_MSI
27	select HAVE_KVM_READONLY_MEM	27	select HAVE_KVM_READONLY_MEM
28	select HAVE_KVM_VCPU_ASYNC_IOCTL
29	select KVM_COMMON	28	select KVM_COMMON
30	select KVM_GENERIC_DIRTYLOG_READ_PROTECT	29	select KVM_GENERIC_DIRTYLOG_READ_PROTECT
31	select KVM_GENERIC_HARDWARE_ENABLING	30	select KVM_GENERIC_HARDWARE_ENABLING

arch/loongarch/kvm/intc/eiointc.c+40−40

@@ -13,19 +13,19 @@ static void eiointc_set_sw_coreisr(struct loongarch_eiointc *s) · 12 unmodified lines
13	struct kvm_vcpu *vcpu;	13	struct kvm_vcpu *vcpu;
14		14
15	for (irq = 0; irq < EIOINTC_IRQS; irq++) {	15	for (irq = 0; irq < EIOINTC_IRQS; irq++) {
16	ipnum = s->ipmap.reg_u8[irq / 32];	16	ipnum = (s->ipmap >> (irq / 32 * 8)) & 0xff;
17	if (!(s->status & BIT(EIOINTC_ENABLE_INT_ENCODE))) {	17	if (!(s->status & BIT(EIOINTC_ENABLE_INT_ENCODE))) {
18	ipnum = count_trailing_zeros(ipnum);	18	ipnum = count_trailing_zeros(ipnum);
19	ipnum = (ipnum >= 0 && ipnum < 4) ? ipnum : 0;	19	ipnum = (ipnum >= 0 && ipnum < 4) ? ipnum : 0;
20	}	20	}
21		21
22	cpuid = s->coremap.reg_u8[irq];	22	cpuid = ((u8 *)s->coremap)[irq];
23	vcpu = kvm_get_vcpu_by_cpuid(s->kvm, cpuid);	23	vcpu = kvm_get_vcpu_by_cpuid(s->kvm, cpuid);
24	if (!vcpu)	24	if (!vcpu)
25	continue;	25	continue;
26		26
27	cpu = vcpu->vcpu_id;	27	cpu = vcpu->vcpu_id;
28	if (test_bit(irq, (unsigned long *)s->coreisr.reg_u32[cpu]))	28	if (test_bit(irq, (unsigned long *)s->coreisr[cpu]))
29	__set_bit(irq, s->sw_coreisr[cpu][ipnum]);	29	__set_bit(irq, s->sw_coreisr[cpu][ipnum]);
30	else	30	else
31	__clear_bit(irq, s->sw_coreisr[cpu][ipnum]);	31	__clear_bit(irq, s->sw_coreisr[cpu][ipnum]);
@@ -38,7 +38,7 @@ static void eiointc_update_irq(struct loongarch_eiointc *s, int irq, int level) · 6 unmodified lines
38	struct kvm_vcpu *vcpu;	38	struct kvm_vcpu *vcpu;
39	struct kvm_interrupt vcpu_irq;	39	struct kvm_interrupt vcpu_irq;
40		40
41	ipnum = s->ipmap.reg_u8[irq / 32];	41	ipnum = (s->ipmap >> (irq / 32 * 8)) & 0xff;
42	if (!(s->status & BIT(EIOINTC_ENABLE_INT_ENCODE))) {	42	if (!(s->status & BIT(EIOINTC_ENABLE_INT_ENCODE))) {
43	ipnum = count_trailing_zeros(ipnum);	43	ipnum = count_trailing_zeros(ipnum);
44	ipnum = (ipnum >= 0 && ipnum < 4) ? ipnum : 0;	44	ipnum = (ipnum >= 0 && ipnum < 4) ? ipnum : 0;
@@ -53,13 +53,13 @@ static void eiointc_update_irq(struct loongarch_eiointc *s, int irq, int level) · 8 unmodified lines
53		53
54	if (level) {	54	if (level) {
55	/* if not enable return false */	55	/* if not enable return false */
56	if (!test_bit(irq, (unsigned long *)s->enable.reg_u32))	56	if (!test_bit(irq, (unsigned long *)s->enable))
57	return;	57	return;
58	__set_bit(irq, (unsigned long *)s->coreisr.reg_u32[cpu]);	58	__set_bit(irq, (unsigned long *)s->coreisr[cpu]);
59	found = find_first_bit(s->sw_coreisr[cpu][ipnum], EIOINTC_IRQS);	59	found = find_first_bit(s->sw_coreisr[cpu][ipnum], EIOINTC_IRQS);
60	__set_bit(irq, s->sw_coreisr[cpu][ipnum]);	60	__set_bit(irq, s->sw_coreisr[cpu][ipnum]);
61	} else {	61	} else {
62	__clear_bit(irq, (unsigned long *)s->coreisr.reg_u32[cpu]);	62	__clear_bit(irq, (unsigned long *)s->coreisr[cpu]);
63	__clear_bit(irq, s->sw_coreisr[cpu][ipnum]);	63	__clear_bit(irq, s->sw_coreisr[cpu][ipnum]);
64	found = find_first_bit(s->sw_coreisr[cpu][ipnum], EIOINTC_IRQS);	64	found = find_first_bit(s->sw_coreisr[cpu][ipnum], EIOINTC_IRQS);
65	}	65	}
@@ -94,7 +94,7 @@ static inline void eiointc_update_sw_coremap(struct loongarch_eiointc *s, · 28 unmodified lines
94	if (s->sw_coremap[irq + i] == cpu)	94	if (s->sw_coremap[irq + i] == cpu)
95	continue;	95	continue;
96		96
97	if (notify && test_bit(irq + i, (unsigned long *)s->isr.reg_u8)) {	97	if (notify && test_bit(irq + i, (unsigned long *)s->isr)) {
98	/* lower irq at old cpu and raise irq at new cpu */	98	/* lower irq at old cpu and raise irq at new cpu */
99	eiointc_update_irq(s, irq + i, 0);	99	eiointc_update_irq(s, irq + i, 0);
100	s->sw_coremap[irq + i] = cpu;	100	s->sw_coremap[irq + i] = cpu;
@@ -108,7 +108,7 @@ static inline void eiointc_update_sw_coremap(struct loongarch_eiointc *s, · 7 unmodified lines
108	void eiointc_set_irq(struct loongarch_eiointc *s, int irq, int level)	108	void eiointc_set_irq(struct loongarch_eiointc *s, int irq, int level)
109	{	109	{
110	unsigned long flags;	110	unsigned long flags;
111	unsigned long isr = (unsigned long )s->isr.reg_u8;	111	unsigned long isr = (unsigned long )s->isr;
112		112
113	spin_lock_irqsave(&s->lock, flags);	113	spin_lock_irqsave(&s->lock, flags);
114	level ? __set_bit(irq, isr) : __clear_bit(irq, isr);	114	level ? __set_bit(irq, isr) : __clear_bit(irq, isr);
@@ -127,27 +127,27 @@ static int loongarch_eiointc_read(struct kvm_vcpu *vcpu, struct loongarch_eioint · 12 unmodified lines
127	switch (offset) {	127	switch (offset) {
128	case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END:	128	case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END:
129	index = (offset - EIOINTC_NODETYPE_START) >> 3;	129	index = (offset - EIOINTC_NODETYPE_START) >> 3;
130	data = s->nodetype.reg_u64[index];	130	data = s->nodetype[index];
131	break;	131	break;
132	case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END:	132	case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END:
133	index = (offset - EIOINTC_IPMAP_START) >> 3;	133	index = (offset - EIOINTC_IPMAP_START) >> 3;
134	data = s->ipmap.reg_u64;	134	data = s->ipmap;
135	break;	135	break;
136	case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END:	136	case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END:
137	index = (offset - EIOINTC_ENABLE_START) >> 3;	137	index = (offset - EIOINTC_ENABLE_START) >> 3;
138	data = s->enable.reg_u64[index];	138	data = s->enable[index];
139	break;	139	break;
140	case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END:	140	case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END:
141	index = (offset - EIOINTC_BOUNCE_START) >> 3;	141	index = (offset - EIOINTC_BOUNCE_START) >> 3;
142	data = s->bounce.reg_u64[index];	142	data = s->bounce[index];
143	break;	143	break;
144	case EIOINTC_COREISR_START ... EIOINTC_COREISR_END:	144	case EIOINTC_COREISR_START ... EIOINTC_COREISR_END:
145	index = (offset - EIOINTC_COREISR_START) >> 3;	145	index = (offset - EIOINTC_COREISR_START) >> 3;
146	data = s->coreisr.reg_u64[vcpu->vcpu_id][index];	146	data = s->coreisr[vcpu->vcpu_id][index];
147	break;	147	break;
148	case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END:	148	case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END:
149	index = (offset - EIOINTC_COREMAP_START) >> 3;	149	index = (offset - EIOINTC_COREMAP_START) >> 3;
150	data = s->coremap.reg_u64[index];	150	data = s->coremap[index];
151	break;	151	break;
152	default:	152	default:
153	ret = -EINVAL;	153	ret = -EINVAL;
@@ -223,26 +223,26 @@ static int loongarch_eiointc_write(struct kvm_vcpu *vcpu, · 69 unmodified lines
223	switch (offset) {	223	switch (offset) {
224	case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END:	224	case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END:
225	index = (offset - EIOINTC_NODETYPE_START) >> 3;	225	index = (offset - EIOINTC_NODETYPE_START) >> 3;
226	old = s->nodetype.reg_u64[index];	226	old = s->nodetype[index];
227	s->nodetype.reg_u64[index] = (old & ~mask) \| data;	227	s->nodetype[index] = (old & ~mask) \| data;
228	break;	228	break;
229	case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END:	229	case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END:
230	/*	230	/*
231	* ipmap cannot be set at runtime, can be set only at the beginning	231	* ipmap cannot be set at runtime, can be set only at the beginning
232	* of irqchip driver, need not update upper irq level	232	* of irqchip driver, need not update upper irq level
233	*/	233	*/
234	old = s->ipmap.reg_u64;	234	old = s->ipmap;
235	s->ipmap.reg_u64 = (old & ~mask) \| data;	235	s->ipmap = (old & ~mask) \| data;
236	break;	236	break;
237	case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END:	237	case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END:
238	index = (offset - EIOINTC_ENABLE_START) >> 3;	238	index = (offset - EIOINTC_ENABLE_START) >> 3;
239	old = s->enable.reg_u64[index];	239	old = s->enable[index];
240	s->enable.reg_u64[index] = (old & ~mask) \| data;	240	s->enable[index] = (old & ~mask) \| data;
241	/*	241	/*
242	* 1: enable irq.	242	* 1: enable irq.
243	* update irq when isr is set.	243	* update irq when isr is set.
244	*/	244	*/
245	data = s->enable.reg_u64[index] & ~old & s->isr.reg_u64[index];	245	data = s->enable[index] & ~old & s->isr[index];
246	while (data) {	246	while (data) {
247	irq = __ffs(data);	247	irq = __ffs(data);
248	eiointc_update_irq(s, irq + index * 64, 1);	248	eiointc_update_irq(s, irq + index * 64, 1);
@@ -252,7 +252,7 @@ static int loongarch_eiointc_write(struct kvm_vcpu *vcpu, · 3 unmodified lines
252	* 0: disable irq.	252	* 0: disable irq.
253	* update irq when isr is set.	253	* update irq when isr is set.
254	*/	254	*/
255	data = ~s->enable.reg_u64[index] & old & s->isr.reg_u64[index];	255	data = ~s->enable[index] & old & s->isr[index];
256	while (data) {	256	while (data) {
257	irq = __ffs(data);	257	irq = __ffs(data);
258	eiointc_update_irq(s, irq + index * 64, 0);	258	eiointc_update_irq(s, irq + index * 64, 0);
@@ -262,16 +262,16 @@ static int loongarch_eiointc_write(struct kvm_vcpu *vcpu, · 3 unmodified lines
262	case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END:	262	case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END:
263	/* do not emulate hw bounced irq routing */	263	/* do not emulate hw bounced irq routing */
264	index = (offset - EIOINTC_BOUNCE_START) >> 3;	264	index = (offset - EIOINTC_BOUNCE_START) >> 3;
265	old = s->bounce.reg_u64[index];	265	old = s->bounce[index];
266	s->bounce.reg_u64[index] = (old & ~mask) \| data;	266	s->bounce[index] = (old & ~mask) \| data;
267	break;	267	break;
268	case EIOINTC_COREISR_START ... EIOINTC_COREISR_END:	268	case EIOINTC_COREISR_START ... EIOINTC_COREISR_END:
269	index = (offset - EIOINTC_COREISR_START) >> 3;	269	index = (offset - EIOINTC_COREISR_START) >> 3;
270	/* use attrs to get current cpu index */	270	/* use attrs to get current cpu index */
271	cpu = vcpu->vcpu_id;	271	cpu = vcpu->vcpu_id;
272	old = s->coreisr.reg_u64[cpu][index];	272	old = s->coreisr[cpu][index];
273	/* write 1 to clear interrupt */	273	/* write 1 to clear interrupt */
274	s->coreisr.reg_u64[cpu][index] = old & ~data;	274	s->coreisr[cpu][index] = old & ~data;
275	data &= old;	275	data &= old;
276	while (data) {	276	while (data) {
277	irq = __ffs(data);	277	irq = __ffs(data);
@@ -281,9 +281,9 @@ static int loongarch_eiointc_write(struct kvm_vcpu *vcpu, · 3 unmodified lines
281	break;	281	break;
282	case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END:	282	case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END:
283	index = (offset - EIOINTC_COREMAP_START) >> 3;	283	index = (offset - EIOINTC_COREMAP_START) >> 3;
284	old = s->coremap.reg_u64[index];	284	old = s->coremap[index];
285	s->coremap.reg_u64[index] = (old & ~mask) \| data;	285	s->coremap[index] = (old & ~mask) \| data;
286	data = s->coremap.reg_u64[index];	286	data = s->coremap[index];
287	eiointc_update_sw_coremap(s, index * 8, data, sizeof(data), true);	287	eiointc_update_sw_coremap(s, index * 8, data, sizeof(data), true);
288	break;	288	break;
289	default:	289	default:
@@ -451,10 +451,10 @@ static int kvm_eiointc_ctrl_access(struct kvm_device *dev, · 161 unmodified lines
451	break;	451	break;
452	case KVM_DEV_LOONGARCH_EXTIOI_CTRL_LOAD_FINISHED:	452	case KVM_DEV_LOONGARCH_EXTIOI_CTRL_LOAD_FINISHED:
453	eiointc_set_sw_coreisr(s);	453	eiointc_set_sw_coreisr(s);
454	for (i = 0; i < (EIOINTC_IRQS / 4); i++) {	454	for (i = 0; i < (EIOINTC_IRQS / 8); i++) {
455	start_irq = i * 4;	455	start_irq = i * 8;
456	eiointc_update_sw_coremap(s, start_irq,	456	eiointc_update_sw_coremap(s, start_irq,
457	s->coremap.reg_u32[i], sizeof(u32), false);	457	s->coremap[i], sizeof(u64), false);
458	}	458	}
459	break;	459	break;
460	default:	460	default:
@@ -481,34 +481,34 @@ static int kvm_eiointc_regs_access(struct kvm_device *dev, · 20 unmodified lines
481	switch (addr) {	481	switch (addr) {
482	case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END:	482	case EIOINTC_NODETYPE_START ... EIOINTC_NODETYPE_END:
483	offset = (addr - EIOINTC_NODETYPE_START) / 4;	483	offset = (addr - EIOINTC_NODETYPE_START) / 4;
484	p = &s->nodetype.reg_u32[offset];	484	p = s->nodetype + offset * 4;
485	break;	485	break;
486	case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END:	486	case EIOINTC_IPMAP_START ... EIOINTC_IPMAP_END:
487	offset = (addr - EIOINTC_IPMAP_START) / 4;	487	offset = (addr - EIOINTC_IPMAP_START) / 4;
488	p = &s->ipmap.reg_u32[offset];	488	p = &s->ipmap + offset * 4;
489	break;	489	break;
490	case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END:	490	case EIOINTC_ENABLE_START ... EIOINTC_ENABLE_END:
491	offset = (addr - EIOINTC_ENABLE_START) / 4;	491	offset = (addr - EIOINTC_ENABLE_START) / 4;
492	p = &s->enable.reg_u32[offset];	492	p = s->enable + offset * 4;
493	break;	493	break;
494	case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END:	494	case EIOINTC_BOUNCE_START ... EIOINTC_BOUNCE_END:
495	offset = (addr - EIOINTC_BOUNCE_START) / 4;	495	offset = (addr - EIOINTC_BOUNCE_START) / 4;
496	p = &s->bounce.reg_u32[offset];	496	p = s->bounce + offset * 4;
497	break;	497	break;
498	case EIOINTC_ISR_START ... EIOINTC_ISR_END:	498	case EIOINTC_ISR_START ... EIOINTC_ISR_END:
499	offset = (addr - EIOINTC_ISR_START) / 4;	499	offset = (addr - EIOINTC_ISR_START) / 4;
500	p = &s->isr.reg_u32[offset];	500	p = s->isr + offset * 4;
501	break;	501	break;
502	case EIOINTC_COREISR_START ... EIOINTC_COREISR_END:	502	case EIOINTC_COREISR_START ... EIOINTC_COREISR_END:
503	if (cpu >= s->num_cpu)	503	if (cpu >= s->num_cpu)
504	return -EINVAL;	504	return -EINVAL;
505		505
506	offset = (addr - EIOINTC_COREISR_START) / 4;	506	offset = (addr - EIOINTC_COREISR_START) / 4;
507	p = &s->coreisr.reg_u32[cpu][offset];	507	p = s->coreisr[cpu] + offset * 4;
508	break;	508	break;
509	case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END:	509	case EIOINTC_COREMAP_START ... EIOINTC_COREMAP_END:
510	offset = (addr - EIOINTC_COREMAP_START) / 4;	510	offset = (addr - EIOINTC_COREMAP_START) / 4;
511	p = &s->coremap.reg_u32[offset];	511	p = s->coremap + offset * 4;
512	break;	512	break;
513	default:	513	default:
514	kvm_err("%s: unknown eiointc register, addr = %d\n", __func__, addr);	514	kvm_err("%s: unknown eiointc register, addr = %d\n", __func__, addr);

arch/loongarch/kvm/interrupt.c+13−2

 	[INT_HWI5]	= CPU_IP5,
 	[INT_HWI6]	= CPU_IP6,
 	[INT_HWI7]	= CPU_IP7,
+	[INT_AVEC]	= CPU_AVEC,
 };
 static int kvm_irq_deliver(struct kvm_vcpu *vcpu, unsigned int priority)
 	if (priority < EXCCODE_INT_NUM)
 		irq = priority_to_irq[priority];
+	if (cpu_has_msgint && (priority == INT_AVEC)) {
+		set_gcsr_estat(irq);
+		return 1;
+	}
 	switch (priority) {
 	case INT_TI:
 	case INT_IPI:
 	if (priority < EXCCODE_INT_NUM)
 		irq = priority_to_irq[priority];
+	if (cpu_has_msgint && (priority == INT_AVEC)) {
+		clear_gcsr_estat(irq);
+		return 1;
+	}
 	switch (priority) {
 	case INT_TI:
 	case INT_IPI:
 	unsigned long *pending = &vcpu->arch.irq_pending;
 	unsigned long *pending_clr = &vcpu->arch.irq_clear;
-	for_each_set_bit(priority, pending_clr, INT_IPI + 1)
+	for_each_set_bit(priority, pending_clr, EXCCODE_INT_NUM)
 		kvm_irq_clear(vcpu, priority);
-	for_each_set_bit(priority, pending, INT_IPI + 1)
+	for_each_set_bit(priority, pending, EXCCODE_INT_NUM)
 		kvm_irq_deliver(vcpu, priority);
 }

arch/loongarch/kvm/vcpu.c+19−4

 		*v = GENMASK(31, 0);
 		return 0;
 	case LOONGARCH_CPUCFG1:
-		/* CPUCFG1_MSGINT is not supported by KVM */
+		*v = GENMASK(26, 0);
-		*v = GENMASK(25, 0);
 		return 0;
 	case LOONGARCH_CPUCFG2:
 		/* CPUCFG2 features unconditionally supported by KVM */
 		return -EINVAL;
 	switch (id) {
+	case LOONGARCH_CPUCFG1:
+		if ((val & CPUCFG1_MSGINT) && !cpu_has_msgint)
+			return -EINVAL;
+		return 0;
 	case LOONGARCH_CPUCFG2:
 		if (!(val & CPUCFG2_LLFTP))
 			/* Guests must have a constant timer */
 	return 0;
 }
-long kvm_arch_vcpu_async_ioctl(struct file *filp,
+long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl,
-			       unsigned int ioctl, unsigned long arg)
+				  unsigned long arg)
 {
 	void __user *argp = (void __user *)arg;
 	struct kvm_vcpu *vcpu = filp->private_data;
 	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN2);
 	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN3);
 	kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_LLBCTL);
+	if (cpu_has_msgint) {
+		kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ISR0);
+		kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ISR1);
+		kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ISR2);
+		kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ISR3);
+	}
 	/* Restore Root.GINTC from unused Guest.GINTC register */
 	write_csr_gintc(csr->csrs[LOONGARCH_CSR_GINTC]);
 	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN1);
 	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN2);
 	kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN3);
+	if (cpu_has_msgint) {
+		kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ISR0);
+		kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ISR1);
+		kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ISR2);
+		kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ISR3);
+	}
 	vcpu->arch.aux_inuse |= KVM_LARCH_SWCSR_LATEST;

arch/loongarch/kvm/vm.c+27−13

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	#include <linux/kvm_host.h>	6	#include <linux/kvm_host.h>
7	#include <asm/kvm_mmu.h>	7	#include <asm/kvm_mmu.h>
8	#include <asm/kvm_vcpu.h>	8	#include <asm/kvm_vcpu.h>
		9	#include <asm/kvm_csr.h>
9	#include <asm/kvm_eiointc.h>	10	#include <asm/kvm_eiointc.h>
10	#include <asm/kvm_pch_pic.h>	11	#include <asm/kvm_pch_pic.h>
11		12
@@ -24,6 +25,23 @@ const struct kvm_stats_header kvm_vm_stats_header = { · 12 unmodified lines
24	sizeof(kvm_vm_stats_desc),	25	sizeof(kvm_vm_stats_desc),
25	};	26	};
26		27
		28	static void kvm_vm_init_features(struct kvm *kvm)
		29	{
		30	unsigned long val;
		31
		32	val = read_csr_gcfg();
		33	if (val & CSR_GCFG_GPMP)
		34	kvm->arch.kvm_features \|= BIT(KVM_LOONGARCH_VM_FEAT_PMU);
		35
		36	/* Enable all PV features by default */
		37	kvm->arch.pv_features = BIT(KVM_FEATURE_IPI);
		38	kvm->arch.kvm_features = BIT(KVM_LOONGARCH_VM_FEAT_PV_IPI);
		39	if (kvm_pvtime_supported()) {
		40	kvm->arch.pv_features \|= BIT(KVM_FEATURE_STEAL_TIME);
		41	kvm->arch.kvm_features \|= BIT(KVM_LOONGARCH_VM_FEAT_PV_STEALTIME);
		42	}
		43	}
		44
27	int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)	45	int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
28	{	46	{
29	int i;	47	int i;
@@ -42,11 +60,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 12 unmodified lines
42	spin_lock_init(&kvm->arch.phyid_map_lock);	60	spin_lock_init(&kvm->arch.phyid_map_lock);
43		61
44	kvm_init_vmcs(kvm);	62	kvm_init_vmcs(kvm);
45		63	kvm_vm_init_features(kvm);
46	/* Enable all PV features by default */
47	kvm->arch.pv_features = BIT(KVM_FEATURE_IPI);
48	if (kvm_pvtime_supported())
49	kvm->arch.pv_features \|= BIT(KVM_FEATURE_STEAL_TIME);
50		64
51	/*	65	/*
52	* cpu_vabits means user address space only (a half of total).	66	* cpu_vabits means user address space only (a half of total).
@@ -136,18 +150,18 @@ static int kvm_vm_feature_has_attr(struct kvm kvm, struct kvm_device_attr attr · 83 unmodified lines
136	if (cpu_has_lbt_mips)	150	if (cpu_has_lbt_mips)
137	return 0;	151	return 0;
138	return -ENXIO;	152	return -ENXIO;
139	case KVM_LOONGARCH_VM_FEAT_PMU:	153	case KVM_LOONGARCH_VM_FEAT_PTW:
140	if (cpu_has_pmp)	154	if (cpu_has_ptw)
141	return 0;	155	return 0;
142	return -ENXIO;	156	return -ENXIO;
143	case KVM_LOONGARCH_VM_FEAT_PV_IPI:	157	case KVM_LOONGARCH_VM_FEAT_MSGINT:
144	return 0;	158	if (cpu_has_msgint)
145	case KVM_LOONGARCH_VM_FEAT_PV_STEALTIME:
146	if (kvm_pvtime_supported())
147	return 0;	159	return 0;
148	return -ENXIO;	160	return -ENXIO;
149	case KVM_LOONGARCH_VM_FEAT_PTW:	161	case KVM_LOONGARCH_VM_FEAT_PMU:
150	if (cpu_has_ptw)	162	case KVM_LOONGARCH_VM_FEAT_PV_IPI:
		163	case KVM_LOONGARCH_VM_FEAT_PV_STEALTIME:
		164	if (kvm_vm_support(&kvm->arch, attr->attr))
151	return 0;	165	return 0;
152	return -ENXIO;	166	return -ENXIO;
153	default:	167	default:

arch/mips/kvm/Kconfig+0−1

@@ -22,7 +22,6 @@ config KVM · 21 unmodified lines
22	select EXPORT_UASM	22	select EXPORT_UASM
23	select KVM_COMMON	23	select KVM_COMMON
24	select KVM_GENERIC_DIRTYLOG_READ_PROTECT	24	select KVM_GENERIC_DIRTYLOG_READ_PROTECT
25	select HAVE_KVM_VCPU_ASYNC_IOCTL
26	select KVM_MMIO	25	select KVM_MMIO
27	select KVM_GENERIC_MMU_NOTIFIER	26	select KVM_GENERIC_MMU_NOTIFIER
28	select KVM_GENERIC_HARDWARE_ENABLING	27	select KVM_GENERIC_HARDWARE_ENABLING

arch/mips/kvm/mips.c+2−2

@@ -895,8 +895,8 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, · 894 unmodified lines
895	return r;	895	return r;
896	}	896	}
897		897
898	long kvm_arch_vcpu_async_ioctl(struct file *filp, unsigned int ioctl,	898	long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl,
899	unsigned long arg)	899	unsigned long arg)
900	{	900	{
901	struct kvm_vcpu *vcpu = filp->private_data;	901	struct kvm_vcpu *vcpu = filp->private_data;
902	void __user argp = (void __user )arg;	902	void __user argp = (void __user )arg;

arch/powerpc/kvm/Kconfig+0−1

@@ -20,7 +20,6 @@ if VIRTUALIZATION · 19 unmodified lines
20	config KVM	20	config KVM
21	bool	21	bool
22	select KVM_COMMON	22	select KVM_COMMON
23	select HAVE_KVM_VCPU_ASYNC_IOCTL
24	select KVM_VFIO	23	select KVM_VFIO
25	select HAVE_KVM_IRQ_BYPASS	24	select HAVE_KVM_IRQ_BYPASS
26		25

arch/powerpc/kvm/powerpc.c+2−2

@@ -2028,8 +2028,8 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, · 2027 unmodified lines
2028	return -EINVAL;	2028	return -EINVAL;
2029	}	2029	}
2030		2030
2031	long kvm_arch_vcpu_async_ioctl(struct file *filp,	2031	long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl,
2032	unsigned int ioctl, unsigned long arg)	2032	unsigned long arg)
2033	{	2033	{
2034	struct kvm_vcpu *vcpu = filp->private_data;	2034	struct kvm_vcpu *vcpu = filp->private_data;
2035	void __user argp = (void __user )arg;	2035	void __user argp = (void __user )arg;

arch/riscv/include/asm/kvm_host.h+6−0

 					 BIT(IRQ_VS_TIMER) | \
 					 BIT(IRQ_VS_EXT))
+#define KVM_DIRTY_LOG_MANUAL_CAPS	(KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \
+					 KVM_DIRTY_LOG_INITIALLY_SET)
 struct kvm_vm_stat {
 	struct kvm_vm_stat_generic generic;
 };
 void kvm_riscv_vcpu_record_steal_time(struct kvm_vcpu *vcpu);
+/* Flags representing implementation specific details */
+DECLARE_STATIC_KEY_FALSE(kvm_riscv_vsstage_tlb_no_gpa);
 #endif /* __RISCV_KVM_HOST_H__ */

arch/riscv/include/asm/kvm_tlb.h+1−0

@@ -49,6 +49,7 @@ void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, · 48 unmodified lines
49	unsigned long gva, unsigned long gvsz,	49	unsigned long gva, unsigned long gvsz,
50	unsigned long order);	50	unsigned long order);
51	void kvm_riscv_local_hfence_vvma_all(unsigned long vmid);	51	void kvm_riscv_local_hfence_vvma_all(unsigned long vmid);
		52	void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu);
52		53
53	void kvm_riscv_tlb_flush_process(struct kvm_vcpu *vcpu);	54	void kvm_riscv_tlb_flush_process(struct kvm_vcpu *vcpu);
54		55

arch/riscv/include/asm/kvm_vcpu_sbi.h+4−1

 			     unsigned long reg_size, const void *reg_val);
 };
-void kvm_riscv_vcpu_sbi_forward(struct kvm_vcpu *vcpu, struct kvm_run *run);
+int kvm_riscv_vcpu_sbi_forward_handler(struct kvm_vcpu *vcpu,
+				       struct kvm_run *run,
+				       struct kvm_vcpu_sbi_return *retdata);
 void kvm_riscv_vcpu_sbi_system_reset(struct kvm_vcpu *vcpu,
 				     struct kvm_run *run,
 				     u32 type, u64 flags);
 extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_susp;
 extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_sta;
 extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_fwft;
+extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_mpxy;
 extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_experimental;
 extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_vendor;

arch/riscv/include/asm/kvm_vmid.h+0−1

@@ -22,6 +22,5 @@ unsigned long kvm_riscv_gstage_vmid_bits(void); · 21 unmodified lines
22	int kvm_riscv_gstage_vmid_init(struct kvm *kvm);	22	int kvm_riscv_gstage_vmid_init(struct kvm *kvm);
23	bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid);	23	bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid);
24	void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu);	24	void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu);
25	void kvm_riscv_gstage_vmid_sanitize(struct kvm_vcpu *vcpu);
26		25
27	#endif	26	#endif

arch/riscv/include/uapi/asm/kvm.h+3−0

 #define KVM_INTERRUPT_SET	-1U
 #define KVM_INTERRUPT_UNSET	-2U
+#define KVM_EXIT_FAIL_ENTRY_NO_VSFILE	(1ULL << 0)
 /* for KVM_GET_REGS and KVM_SET_REGS */
 struct kvm_regs {
 };
 	KVM_RISCV_SBI_EXT_STA,
 	KVM_RISCV_SBI_EXT_SUSP,
 	KVM_RISCV_SBI_EXT_FWFT,
+	KVM_RISCV_SBI_EXT_MPXY,
 	KVM_RISCV_SBI_EXT_MAX,
 };

arch/riscv/kvm/Kconfig+0−1

@@ -23,7 +23,6 @@ config KVM · 22 unmodified lines
23	select HAVE_KVM_IRQCHIP	23	select HAVE_KVM_IRQCHIP
24	select HAVE_KVM_IRQ_ROUTING	24	select HAVE_KVM_IRQ_ROUTING
25	select HAVE_KVM_MSI	25	select HAVE_KVM_MSI
26	select HAVE_KVM_VCPU_ASYNC_IOCTL
27	select HAVE_KVM_READONLY_MEM	26	select HAVE_KVM_READONLY_MEM
28	select HAVE_KVM_DIRTY_RING_ACQ_REL	27	select HAVE_KVM_DIRTY_RING_ACQ_REL
29	select KVM_COMMON	28	select KVM_COMMON

arch/riscv/kvm/Makefile+1−0

@@ -27,6 +27,7 @@ kvm-y += vcpu_onereg.o · 26 unmodified lines
27	kvm-$(CONFIG_RISCV_PMU_SBI) += vcpu_pmu.o	27	kvm-$(CONFIG_RISCV_PMU_SBI) += vcpu_pmu.o
28	kvm-y += vcpu_sbi.o	28	kvm-y += vcpu_sbi.o
29	kvm-y += vcpu_sbi_base.o	29	kvm-y += vcpu_sbi_base.o
		30	kvm-y += vcpu_sbi_forward.o
30	kvm-y += vcpu_sbi_fwft.o	31	kvm-y += vcpu_sbi_fwft.o
31	kvm-y += vcpu_sbi_hsm.o	32	kvm-y += vcpu_sbi_hsm.o
32	kvm-$(CONFIG_RISCV_PMU_SBI) += vcpu_sbi_pmu.o	33	kvm-$(CONFIG_RISCV_PMU_SBI) += vcpu_sbi_pmu.o

arch/riscv/kvm/aia_imsic.c+1−1

@@ -814,7 +814,7 @@ int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu) · 813 unmodified lines
814	/* For HW acceleration mode, we can't continue */	814	/* For HW acceleration mode, we can't continue */
815	if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_HWACCEL) {	815	if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_HWACCEL) {
816	run->fail_entry.hardware_entry_failure_reason =	816	run->fail_entry.hardware_entry_failure_reason =
817	CSR_HSTATUS;	817	KVM_EXIT_FAIL_ENTRY_NO_VSFILE;
818	run->fail_entry.cpu = vcpu->cpu;	818	run->fail_entry.cpu = vcpu->cpu;
819	run->exit_reason = KVM_EXIT_FAIL_ENTRY;	819	run->exit_reason = KVM_EXIT_FAIL_ENTRY;
820	return 0;	820	return 0;

arch/riscv/kvm/main.c+14−0

@@ -15,6 +15,18 @@ · 14 unmodified lines
15	#include <asm/kvm_nacl.h>	15	#include <asm/kvm_nacl.h>
16	#include <asm/sbi.h>	16	#include <asm/sbi.h>
17		17
		18	DEFINE_STATIC_KEY_FALSE(kvm_riscv_vsstage_tlb_no_gpa);
		19
		20	static void kvm_riscv_setup_vendor_features(void)
		21	{
		22	/* Andes AX66: split two-stage TLBs */
		23	if (riscv_cached_mvendorid(0) == ANDES_VENDOR_ID &&
		24	(riscv_cached_marchid(0) & 0xFFFF) == 0x8A66) {
		25	static_branch_enable(&kvm_riscv_vsstage_tlb_no_gpa);
		26	kvm_info("VS-stage TLB does not cache guest physical address and VMID\n");
		27	}
		28	}
		29
18	long kvm_arch_dev_ioctl(struct file *filp,	30	long kvm_arch_dev_ioctl(struct file *filp,
19	unsigned int ioctl, unsigned long arg)	31	unsigned int ioctl, unsigned long arg)
20	{	32	{
@@ -160,6 +172,8 @@ static int __init riscv_kvm_init(void) · 139 unmodified lines
160	kvm_info("AIA available with %d guest external interrupts\n",	172	kvm_info("AIA available with %d guest external interrupts\n",
161	kvm_riscv_aia_nr_hgei);	173	kvm_riscv_aia_nr_hgei);
162		174
		175	kvm_riscv_setup_vendor_features();
		176
163	kvm_register_perf_callbacks(NULL);	177	kvm_register_perf_callbacks(NULL);
164		178
165	rc = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);	179	rc = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);

arch/riscv/kvm/mmu.c+4−1

 	 * allocated dirty_bitmap[], dirty pages will be tracked while
 	 * the memory slot is write protected.
 	 */
-	if (change != KVM_MR_DELETE && new->flags & KVM_MEM_LOG_DIRTY_PAGES)
+	if (change != KVM_MR_DELETE && new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
+		if (kvm_dirty_log_manual_protect_and_init_set(kvm))
+			return;
 		mmu_wp_memory_region(kvm, new->id);
+	}
 }
 int kvm_arch_prepare_memory_region(struct kvm *kvm,

arch/riscv/kvm/tlb.c+30−0

@@ -158,6 +158,36 @@ void kvm_riscv_local_hfence_vvma_all(unsigned long vmid) · 157 unmodified lines
158	csr_write(CSR_HGATP, hgatp);	158	csr_write(CSR_HGATP, hgatp);
159	}	159	}
160		160
		161	void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu)
		162	{
		163	unsigned long vmid;
		164
		165	if (!kvm_riscv_gstage_vmid_bits() \|\|
		166	vcpu->arch.last_exit_cpu == vcpu->cpu)
		167	return;
		168
		169	/*
		170	* On RISC-V platforms with hardware VMID support, we share same
		171	* VMID for all VCPUs of a particular Guest/VM. This means we might
		172	* have stale G-stage TLB entries on the current Host CPU due to
		173	* some other VCPU of the same Guest which ran previously on the
		174	* current Host CPU.
		175	*
		176	* To cleanup stale TLB entries, we simply flush all G-stage TLB
		177	* entries by VMID whenever underlying Host CPU changes for a VCPU.
		178	*/
		179
		180	vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
		181	kvm_riscv_local_hfence_gvma_vmid_all(vmid);
		182
		183	/*
		184	* Flush VS-stage TLB entries for implementation where VS-stage
		185	* TLB does not cahce guest physical address and VMID.
		186	*/
		187	if (static_branch_unlikely(&kvm_riscv_vsstage_tlb_no_gpa))
		188	kvm_riscv_local_hfence_vvma_all(vmid);
		189	}
		190
161	void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu)	191	void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu)
162	{	192	{
163	kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_FENCE_I_RCVD);	193	kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_FENCE_I_RCVD);

arch/riscv/kvm/vcpu.c+3−3

@@ -238,8 +238,8 @@ vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu vcpu, struct vm_fault vmf) · 237 unmodified lines
238	return VM_FAULT_SIGBUS;	238	return VM_FAULT_SIGBUS;
239	}	239	}
240		240
241	long kvm_arch_vcpu_async_ioctl(struct file *filp,	241	long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl,
242	unsigned int ioctl, unsigned long arg)	242	unsigned long arg)
243	{	243	{
244	struct kvm_vcpu *vcpu = filp->private_data;	244	struct kvm_vcpu *vcpu = filp->private_data;
245	void __user argp = (void __user )arg;	245	void __user argp = (void __user )arg;
@@ -968,7 +968,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) · 722 unmodified lines
968	* Note: This should be done after G-stage VMID has been	968	* Note: This should be done after G-stage VMID has been
969	* updated using kvm_riscv_gstage_vmid_ver_changed()	969	* updated using kvm_riscv_gstage_vmid_ver_changed()
970	*/	970	*/
971	kvm_riscv_gstage_vmid_sanitize(vcpu);	971	kvm_riscv_local_tlb_sanitize(vcpu);
972		972
973	trace_kvm_entry(vcpu);	973	trace_kvm_entry(vcpu);
974		974

arch/riscv/kvm/vcpu_insn.c+22−0

 	return (rc <= 0) ? rc : 1;
 }
+static bool is_load_guest_page_fault(unsigned long scause)
+{
+	/**
+	 * If a g-stage page fault occurs, the direct approach
+	 * is to let the g-stage page fault handler handle it
+	 * naturally, however, calling the g-stage page fault
+	 * handler here seems rather strange.
+	 * Considering this is a corner case, we can directly
+	 * return to the guest and re-execute the same PC, this
+	 * will trigger a g-stage page fault again and then the
+	 * regular g-stage page fault handler will populate
+	 * g-stage page table.
+	 */
+	return (scause == EXC_LOAD_GUEST_PAGE_FAULT);
+}
 /**
  * kvm_riscv_vcpu_virtual_insn -- Handle virtual instruction trap
  *
 							  ct->sepc,
 							  &utrap);
 			if (utrap.scause) {
+				if (is_load_guest_page_fault(utrap.scause))
+					return 1;
 				utrap.sepc = ct->sepc;
 				kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
 				return 1;
 		insn = kvm_riscv_vcpu_unpriv_read(vcpu, true, ct->sepc,
 						  &utrap);
 		if (utrap.scause) {
+			if (is_load_guest_page_fault(utrap.scause))
+				return 1;
 			/* Redirect trap if we failed to read instruction */
 			utrap.sepc = ct->sepc;
 			kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);
 		insn = kvm_riscv_vcpu_unpriv_read(vcpu, true, ct->sepc,
 						  &utrap);
 		if (utrap.scause) {
+			if (is_load_guest_page_fault(utrap.scause))
+				return 1;
 			/* Redirect trap if we failed to read instruction */
 			utrap.sepc = ct->sepc;
 			kvm_riscv_vcpu_trap_redirect(vcpu, &utrap);

arch/riscv/kvm/vcpu_sbi.c+9−1

 		.ext_idx = KVM_RISCV_SBI_EXT_FWFT,
 		.ext_ptr = &vcpu_sbi_ext_fwft,
 	},
+	{
+		.ext_idx = KVM_RISCV_SBI_EXT_MPXY,
+		.ext_ptr = &vcpu_sbi_ext_mpxy,
+	},
 	{
 		.ext_idx = KVM_RISCV_SBI_EXT_EXPERIMENTAL,
 		.ext_ptr = &vcpu_sbi_ext_experimental,
 	return sext && scontext->ext_status[sext->ext_idx] != KVM_RISCV_SBI_EXT_STATUS_UNAVAILABLE;
 }
-void kvm_riscv_vcpu_sbi_forward(struct kvm_vcpu *vcpu, struct kvm_run *run)
+int kvm_riscv_vcpu_sbi_forward_handler(struct kvm_vcpu *vcpu,
+				       struct kvm_run *run,
+				       struct kvm_vcpu_sbi_return *retdata)
 {
 	struct kvm_cpu_context *cp = &vcpu->arch.guest_context;
 	run->riscv_sbi.args[5] = cp->a5;
 	run->riscv_sbi.ret[0] = SBI_ERR_NOT_SUPPORTED;
 	run->riscv_sbi.ret[1] = 0;
+	retdata->uexit = true;
+	return 0;
 }
 void kvm_riscv_vcpu_sbi_system_reset(struct kvm_vcpu *vcpu,

arch/riscv/kvm/vcpu_sbi_base.c+1−27

@@ -41,8 +41,7 @@ static int kvm_sbi_ext_base_handler(struct kvm_vcpu vcpu, struct kvm_run run, · 40 unmodified lines
41	* For experimental/vendor extensions	41	* For experimental/vendor extensions
42	* forward it to the userspace	42	* forward it to the userspace
43	*/	43	*/
44	kvm_riscv_vcpu_sbi_forward(vcpu, run);	44	return kvm_riscv_vcpu_sbi_forward_handler(vcpu, run, retdata);
45	retdata->uexit = true;
46	} else {	45	} else {
47	sbi_ext = kvm_vcpu_sbi_find_ext(vcpu, cp->a0);	46	sbi_ext = kvm_vcpu_sbi_find_ext(vcpu, cp->a0);
48	*out_val = sbi_ext && sbi_ext->probe ?	47	*out_val = sbi_ext && sbi_ext->probe ?
@@ -71,28 +70,3 @@ const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_base = { · 22 unmodified lines
71	.extid_end = SBI_EXT_BASE,	70	.extid_end = SBI_EXT_BASE,
72	.handler = kvm_sbi_ext_base_handler,	71	.handler = kvm_sbi_ext_base_handler,
73	};	72	};
74
75	static int kvm_sbi_ext_forward_handler(struct kvm_vcpu *vcpu,
76	struct kvm_run *run,
77	struct kvm_vcpu_sbi_return *retdata)
78	{
79	/*
80	* Both SBI experimental and vendor extensions are
81	* unconditionally forwarded to userspace.
82	*/
83	kvm_riscv_vcpu_sbi_forward(vcpu, run);
84	retdata->uexit = true;
85	return 0;
86	}
87
88	const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_experimental = {
89	.extid_start = SBI_EXT_EXPERIMENTAL_START,
90	.extid_end = SBI_EXT_EXPERIMENTAL_END,
91	.handler = kvm_sbi_ext_forward_handler,
92	};
93
94	const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_vendor = {
95	.extid_start = SBI_EXT_VENDOR_START,
96	.extid_end = SBI_EXT_VENDOR_END,
97	.handler = kvm_sbi_ext_forward_handler,
98	};

arch/riscv/kvm/vcpu_sbi_forward.c+34−0

@@ -0,0 +1,34 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Copyright (c) 2025 Ventana Micro Systems Inc.
	4	*/
	5
	6	#include <linux/kvm_host.h>
	7	#include <asm/kvm_vcpu_sbi.h>
	8	#include <asm/sbi.h>
	9
	10	const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_experimental = {
	11	.extid_start = SBI_EXT_EXPERIMENTAL_START,
	12	.extid_end = SBI_EXT_EXPERIMENTAL_END,
	13	.handler = kvm_riscv_vcpu_sbi_forward_handler,
	14	};
	15
	16	const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_vendor = {
	17	.extid_start = SBI_EXT_VENDOR_START,
	18	.extid_end = SBI_EXT_VENDOR_END,
	19	.handler = kvm_riscv_vcpu_sbi_forward_handler,
	20	};
	21
	22	const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_dbcn = {
	23	.extid_start = SBI_EXT_DBCN,
	24	.extid_end = SBI_EXT_DBCN,
	25	.default_disabled = true,
	26	.handler = kvm_riscv_vcpu_sbi_forward_handler,
	27	};
	28
	29	const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_mpxy = {
	30	.extid_start = SBI_EXT_MPXY,
	31	.extid_end = SBI_EXT_MPXY,
	32	.default_disabled = true,
	33	.handler = kvm_riscv_vcpu_sbi_forward_handler,
	34	};

arch/riscv/kvm/vcpu_sbi_replace.c+0−32

@@ -185,35 +185,3 @@ const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_srst = { · 184 unmodified lines
185	.extid_end = SBI_EXT_SRST,	185	.extid_end = SBI_EXT_SRST,
186	.handler = kvm_sbi_ext_srst_handler,	186	.handler = kvm_sbi_ext_srst_handler,
187	};	187	};
188
189	static int kvm_sbi_ext_dbcn_handler(struct kvm_vcpu *vcpu,
190	struct kvm_run *run,
191	struct kvm_vcpu_sbi_return *retdata)
192	{
193	struct kvm_cpu_context *cp = &vcpu->arch.guest_context;
194	unsigned long funcid = cp->a6;
195
196	switch (funcid) {
197	case SBI_EXT_DBCN_CONSOLE_WRITE:
198	case SBI_EXT_DBCN_CONSOLE_READ:
199	case SBI_EXT_DBCN_CONSOLE_WRITE_BYTE:
200	/*
201	* The SBI debug console functions are unconditionally
202	* forwarded to the userspace.
203	*/
204	kvm_riscv_vcpu_sbi_forward(vcpu, run);
205	retdata->uexit = true;
206	break;
207	default:
208	retdata->err_val = SBI_ERR_NOT_SUPPORTED;
209	}
210
211	return 0;
212	}
213
214	const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_dbcn = {
215	.extid_start = SBI_EXT_DBCN,
216	.extid_end = SBI_EXT_DBCN,
217	.default_disabled = true,
218	.handler = kvm_sbi_ext_dbcn_handler,
219	};

arch/riscv/kvm/vcpu_sbi_system.c+1−3

 		kvm_riscv_vcpu_sbi_request_reset(vcpu, cp->a1, cp->a2);
 		/* userspace provides the suspend implementation */
-		kvm_riscv_vcpu_sbi_forward(vcpu, run);
+		return kvm_riscv_vcpu_sbi_forward_handler(vcpu, run, retdata);
-		retdata->uexit = true;
-		break;
 	default:
 		retdata->err_val = SBI_ERR_NOT_SUPPORTED;
 		break;

arch/riscv/kvm/vcpu_sbi_v01.c+1−2

 		 * The CONSOLE_GETCHAR/CONSOLE_PUTCHAR SBI calls cannot be
 		 * handled in kernel so we forward these to user-space
 		 */
-		kvm_riscv_vcpu_sbi_forward(vcpu, run);
+		ret = kvm_riscv_vcpu_sbi_forward_handler(vcpu, run, retdata);
-		retdata->uexit = true;
 		break;
 	case SBI_EXT_0_1_SET_TIMER:
 #if __riscv_xlen == 32

arch/riscv/kvm/vmid.c+0−23

@@ -122,26 +122,3 @@ void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu) · 121 unmodified lines
122	kvm_for_each_vcpu(i, v, vcpu->kvm)	122	kvm_for_each_vcpu(i, v, vcpu->kvm)
123	kvm_make_request(KVM_REQ_UPDATE_HGATP, v);	123	kvm_make_request(KVM_REQ_UPDATE_HGATP, v);
124	}	124	}
125
126	void kvm_riscv_gstage_vmid_sanitize(struct kvm_vcpu *vcpu)
127	{
128	unsigned long vmid;
129
130	if (!kvm_riscv_gstage_vmid_bits() \|\|
131	vcpu->arch.last_exit_cpu == vcpu->cpu)
132	return;
133
134	/*
135	* On RISC-V platforms with hardware VMID support, we share same
136	* VMID for all VCPUs of a particular Guest/VM. This means we might
137	* have stale G-stage TLB entries on the current Host CPU due to
138	* some other VCPU of the same Guest which ran previously on the
139	* current Host CPU.
140	*
141	* To cleanup stale TLB entries, we simply flush all G-stage TLB
142	* entries by VMID whenever underlying Host CPU changes for a VCPU.
143	*/
144
145	vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
146	kvm_riscv_local_hfence_gvma_vmid_all(vmid);
147	}

arch/s390/include/asm/kvm_host.h+4−4

 	u64 instruction_diagnose_500;
 	u64 instruction_diagnose_other;
 	u64 pfault_sync;
+	u64 signal_exits;
 };
 #define PGM_OPERATION			0x01
 	struct mmu_notifier mmu_notifier;
 };
-struct kvm_arch{
+struct kvm_arch {
-	void *sca;
+	struct esca_block *sca;
-	int use_esca;
-	rwlock_t sca_lock;
 	debug_info_t *dbf;
 	struct kvm_s390_float_interrupt float_int;
 	struct kvm_device *flic;
 	int user_sigp;
 	int user_stsi;
 	int user_instr0;
+	int user_operexec;
 	struct s390_io_adapter *adapters[MAX_S390_IO_ADAPTERS];
 	wait_queue_head_t ipte_wq;
 	int ipte_lock_count;

arch/s390/include/asm/stacktrace.h+1−0

@@ -66,6 +66,7 @@ struct stack_frame { · 65 unmodified lines
66	unsigned long sie_flags;	66	unsigned long sie_flags;
67	unsigned long sie_control_block_phys;	67	unsigned long sie_control_block_phys;
68	unsigned long sie_guest_asce;	68	unsigned long sie_guest_asce;
		69	unsigned long sie_irq;
69	};	70	};
70	};	71	};
71	unsigned long gprs[10];	72	unsigned long gprs[10];

arch/s390/kernel/asm-offsets.c+1−0

@@ -67,6 +67,7 @@ int main(void) · 66 unmodified lines
67	OFFSET(__SF_SIE_FLAGS, stack_frame, sie_flags);	67	OFFSET(__SF_SIE_FLAGS, stack_frame, sie_flags);
68	OFFSET(__SF_SIE_CONTROL_PHYS, stack_frame, sie_control_block_phys);	68	OFFSET(__SF_SIE_CONTROL_PHYS, stack_frame, sie_control_block_phys);
69	OFFSET(__SF_SIE_GUEST_ASCE, stack_frame, sie_guest_asce);	69	OFFSET(__SF_SIE_GUEST_ASCE, stack_frame, sie_guest_asce);
		70	OFFSET(__SF_SIE_IRQ, stack_frame, sie_irq);
70	DEFINE(STACK_FRAME_OVERHEAD, sizeof(struct stack_frame));	71	DEFINE(STACK_FRAME_OVERHEAD, sizeof(struct stack_frame));
71	BLANK();	72	BLANK();
72	OFFSET(__SFUSER_BACKCHAIN, stack_frame_user, back_chain);	73	OFFSET(__SFUSER_BACKCHAIN, stack_frame_user, back_chain);

arch/s390/kernel/entry.S+2−0

 	mvc	__SF_SIE_FLAGS(8,%r15),__TI_flags(%r14) # copy thread flags
 	lmg	%r0,%r13,0(%r4)			# load guest gprs 0-13
 	mvi	__TI_sie(%r14),1
+	stosm	__SF_SIE_IRQ(%r15),0x03		# enable interrupts
 	lctlg	%c1,%c1,__SF_SIE_GUEST_ASCE(%r15) # load primary asce
 	lg	%r14,__SF_SIE_CONTROL(%r15)	# get control block pointer
 	oi	__SIE_PROG0C+3(%r14),1		# we are going into SIE now
 	lg	%r14,__LC_CURRENT(%r14)
 	mvi	__TI_sie(%r14),0
 SYM_INNER_LABEL(sie_exit, SYM_L_GLOBAL)
+	stnsm	__SF_SIE_IRQ(%r15),0xfc		# disable interrupts
 	lg	%r14,__SF_SIE_SAVEAREA(%r15)	# load guest register save area
 	stmg	%r0,%r13,0(%r14)		# save guest gprs 0-13
 	xgr	%r0,%r0				# clear guest registers to

arch/s390/kvm/Kconfig+1−1

 	def_tristate y
 	prompt "Kernel-based Virtual Machine (KVM) support"
 	select HAVE_KVM_CPU_RELAX_INTERCEPT
-	select HAVE_KVM_VCPU_ASYNC_IOCTL
 	select KVM_ASYNC_PF
 	select KVM_ASYNC_PF_SYNC
 	select KVM_COMMON
 	select HAVE_KVM_NO_POLL
 	select KVM_VFIO
 	select MMU_NOTIFIER
+	select VIRT_XFER_TO_GUEST_WORK
 	help
 	  Support hosting paravirtualized guest machines using the SIE
 	  virtualization capability on the mainframe. This should work

arch/s390/kvm/gaccess.c+6−21

 int ipte_lock_held(struct kvm *kvm)
 {
-	if (sclp.has_siif) {
+	if (sclp.has_siif)
-		int rc;
+		return kvm->arch.sca->ipte_control.kh != 0;
-		read_lock(&kvm->arch.sca_lock);
-		rc = kvm_s390_get_ipte_control(kvm)->kh != 0;
-		read_unlock(&kvm->arch.sca_lock);
-		return rc;
-	}
 	return kvm->arch.ipte_lock_count != 0;
 }
 	if (kvm->arch.ipte_lock_count > 1)
 		goto out;
 retry:
-	read_lock(&kvm->arch.sca_lock);
+	ic = &kvm->arch.sca->ipte_control;
-	ic = kvm_s390_get_ipte_control(kvm);
 	old = READ_ONCE(*ic);
 	do {
 		if (old.k) {
-			read_unlock(&kvm->arch.sca_lock);
 			cond_resched();
 			goto retry;
 		}
 		new = old;
 		new.k = 1;
 	} while (!try_cmpxchg(&ic->val, &old.val, new.val));
-	read_unlock(&kvm->arch.sca_lock);
 out:
 	mutex_unlock(&kvm->arch.ipte_mutex);
 }
 	kvm->arch.ipte_lock_count--;
 	if (kvm->arch.ipte_lock_count)
 		goto out;
-	read_lock(&kvm->arch.sca_lock);
+	ic = &kvm->arch.sca->ipte_control;
-	ic = kvm_s390_get_ipte_control(kvm);
 	old = READ_ONCE(*ic);
 	do {
 		new = old;
 		new.k = 0;
 	} while (!try_cmpxchg(&ic->val, &old.val, new.val));
-	read_unlock(&kvm->arch.sca_lock);
 	wake_up(&kvm->arch.ipte_wq);
 out:
 	mutex_unlock(&kvm->arch.ipte_mutex);
 	union ipte_control old, new, *ic;
 retry:
-	read_lock(&kvm->arch.sca_lock);
+	ic = &kvm->arch.sca->ipte_control;
-	ic = kvm_s390_get_ipte_control(kvm);
 	old = READ_ONCE(*ic);
 	do {
 		if (old.kg) {
-			read_unlock(&kvm->arch.sca_lock);
 			cond_resched();
 			goto retry;
 		}
 		new.k = 1;
 		new.kh++;
 	} while (!try_cmpxchg(&ic->val, &old.val, new.val));
-	read_unlock(&kvm->arch.sca_lock);
 }
 static void ipte_unlock_siif(struct kvm *kvm)
 {
 	union ipte_control old, new, *ic;
-	read_lock(&kvm->arch.sca_lock);
+	ic = &kvm->arch.sca->ipte_control;
-	ic = kvm_s390_get_ipte_control(kvm);
 	old = READ_ONCE(*ic);
 	do {
 		new = old;
 		if (!new.kh)
 			new.k = 0;
 	} while (!try_cmpxchg(&ic->val, &old.val, new.val));
-	read_unlock(&kvm->arch.sca_lock);
 	if (!new.kh)
 		wake_up(&kvm->arch.ipte_wq);
 }

arch/s390/kvm/intercept.c+3−0

@@ -471,6 +471,9 @@ static int handle_operexc(struct kvm_vcpu *vcpu) · 470 unmodified lines
471	if (vcpu->arch.sie_block->ipa == 0xb256)	471	if (vcpu->arch.sie_block->ipa == 0xb256)
472	return handle_sthyi(vcpu);	472	return handle_sthyi(vcpu);
473		473
		474	if (vcpu->kvm->arch.user_operexec)
		475	return -EOPNOTSUPP;
		476
474	if (vcpu->arch.sie_block->ipa == 0 && vcpu->kvm->arch.user_instr0)	477	if (vcpu->arch.sie_block->ipa == 0 && vcpu->kvm->arch.user_instr0)
475	return -EOPNOTSUPP;	478	return -EOPNOTSUPP;
476	rc = read_guest_lc(vcpu, __LC_PGM_NEW_PSW, &newpsw, sizeof(psw_t));	479	rc = read_guest_lc(vcpu, __LC_PGM_NEW_PSW, &newpsw, sizeof(psw_t));

arch/s390/kvm/interrupt.c+17−63

 /* handle external calls via sigp interpretation facility */
 static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id)
 {
-	int c, scn;
+	struct esca_block *sca = vcpu->kvm->arch.sca;
+	union esca_sigp_ctrl sigp_ctrl = sca->cpu[vcpu->vcpu_id].sigp_ctrl;
 	if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND))
 		return 0;
 	BUG_ON(!kvm_s390_use_sca_entries());
-	read_lock(&vcpu->kvm->arch.sca_lock);
-	if (vcpu->kvm->arch.use_esca) {
-		struct esca_block *sca = vcpu->kvm->arch.sca;
-		union esca_sigp_ctrl sigp_ctrl =
-			sca->cpu[vcpu->vcpu_id].sigp_ctrl;
-		c = sigp_ctrl.c;
-		scn = sigp_ctrl.scn;
-	} else {
-		struct bsca_block *sca = vcpu->kvm->arch.sca;
-		union bsca_sigp_ctrl sigp_ctrl =
-			sca->cpu[vcpu->vcpu_id].sigp_ctrl;
-		c = sigp_ctrl.c;
-		scn = sigp_ctrl.scn;
-	}
-	read_unlock(&vcpu->kvm->arch.sca_lock);
 	if (src_id)
-		*src_id = scn;
+		*src_id = sigp_ctrl.scn;
-	return c;
+	return sigp_ctrl.c;
 }
 static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id)
 {
+	struct esca_block *sca = vcpu->kvm->arch.sca;
+	union esca_sigp_ctrl *sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl;
+	union esca_sigp_ctrl old_val, new_val = {.scn = src_id, .c = 1};
 	int expect, rc;
 	BUG_ON(!kvm_s390_use_sca_entries());
-	read_lock(&vcpu->kvm->arch.sca_lock);
-	if (vcpu->kvm->arch.use_esca) {
-		struct esca_block *sca = vcpu->kvm->arch.sca;
-		union esca_sigp_ctrl *sigp_ctrl =
-			&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
-		union esca_sigp_ctrl new_val = {0}, old_val;
-		old_val = READ_ONCE(*sigp_ctrl);
-		new_val.scn = src_id;
-		new_val.c = 1;
-		old_val.c = 0;
-		expect = old_val.value;
-		rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
-	} else {
-		struct bsca_block *sca = vcpu->kvm->arch.sca;
-		union bsca_sigp_ctrl *sigp_ctrl =
-			&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
-		union bsca_sigp_ctrl new_val = {0}, old_val;
-		old_val = READ_ONCE(*sigp_ctrl);
+	old_val = READ_ONCE(*sigp_ctrl);
-		new_val.scn = src_id;
+	old_val.c = 0;
-		new_val.c = 1;
-		old_val.c = 0;
-		expect = old_val.value;
+	expect = old_val.value;
-		rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
+	rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
-	}
-	read_unlock(&vcpu->kvm->arch.sca_lock);
 	if (rc != expect) {
 		/* another external call is pending */
 static void sca_clear_ext_call(struct kvm_vcpu *vcpu)
 {
+	struct esca_block *sca = vcpu->kvm->arch.sca;
+	union esca_sigp_ctrl *sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl;
 	if (!kvm_s390_use_sca_entries())
 		return;
 	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
-	read_lock(&vcpu->kvm->arch.sca_lock);
-	if (vcpu->kvm->arch.use_esca) {
-		struct esca_block *sca = vcpu->kvm->arch.sca;
-		union esca_sigp_ctrl *sigp_ctrl =
-			&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
-		WRITE_ONCE(sigp_ctrl->value, 0);
-	} else {
-		struct bsca_block *sca = vcpu->kvm->arch.sca;
-		union bsca_sigp_ctrl *sigp_ctrl =
-			&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
-		WRITE_ONCE(sigp_ctrl->value, 0);
+	WRITE_ONCE(sigp_ctrl->value, 0);
-	}
-	read_unlock(&vcpu->kvm->arch.sca_lock);
 }
 int psw_extint_disabled(struct kvm_vcpu *vcpu)
 {
 	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
-	if (!sclp.has_sigpif)
+	if (!kvm_s390_use_sca_entries())
 		return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
 	return sca_ext_call_pending(vcpu, NULL);
 	if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL)
 		return -EINVAL;
-	if (sclp.has_sigpif && !kvm_s390_pv_cpu_get_handle(vcpu))
+	if (kvm_s390_use_sca_entries() && !kvm_s390_pv_cpu_get_handle(vcpu))
 		return sca_inject_ext_call(vcpu, src_id);
 	if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs))

arch/s390/kvm/kvm-s390.c+64−169

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	#define pr_fmt(fmt) "kvm-s390: " fmt	13	#define pr_fmt(fmt) "kvm-s390: " fmt
14		14
15	#include <linux/compiler.h>	15	#include <linux/compiler.h>
		16	#include <linux/entry-virt.h>
16	#include <linux/export.h>	17	#include <linux/export.h>
17	#include <linux/err.h>	18	#include <linux/err.h>
18	#include <linux/fs.h>	19	#include <linux/fs.h>
@@ -184,7 +185,8 @@ const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { · 165 unmodified lines
184	STATS_DESC_COUNTER(VCPU, instruction_diagnose_308),	185	STATS_DESC_COUNTER(VCPU, instruction_diagnose_308),
185	STATS_DESC_COUNTER(VCPU, instruction_diagnose_500),	186	STATS_DESC_COUNTER(VCPU, instruction_diagnose_500),
186	STATS_DESC_COUNTER(VCPU, instruction_diagnose_other),	187	STATS_DESC_COUNTER(VCPU, instruction_diagnose_other),
187	STATS_DESC_COUNTER(VCPU, pfault_sync)	188	STATS_DESC_COUNTER(VCPU, pfault_sync),
		189	STATS_DESC_COUNTER(VCPU, signal_exits)
188	};	190	};
189		191
190	const struct kvm_stats_header kvm_vcpu_stats_header = {	192	const struct kvm_stats_header kvm_vcpu_stats_header = {
@@ -271,7 +273,6 @@ debug_info_t *kvm_s390_dbf_uv; · 80 unmodified lines
271	/* forward declarations */	273	/* forward declarations */
272	static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,	274	static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
273	unsigned long end);	275	unsigned long end);
274	static int sca_switch_to_extended(struct kvm *kvm);
275		276
276	static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)	277	static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
277	{	278	{
@@ -606,6 +607,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 328 unmodified lines
606	case KVM_CAP_SET_GUEST_DEBUG:	607	case KVM_CAP_SET_GUEST_DEBUG:
607	case KVM_CAP_S390_DIAG318:	608	case KVM_CAP_S390_DIAG318:
608	case KVM_CAP_IRQFD_RESAMPLE:	609	case KVM_CAP_IRQFD_RESAMPLE:
		610	case KVM_CAP_S390_USER_OPEREXEC:
609	r = 1;	611	r = 1;
610	break;	612	break;
611	case KVM_CAP_SET_GUEST_DEBUG2:	613	case KVM_CAP_SET_GUEST_DEBUG2:
@@ -631,11 +633,13 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 19 unmodified lines
631	case KVM_CAP_NR_VCPUS:	633	case KVM_CAP_NR_VCPUS:
632	case KVM_CAP_MAX_VCPUS:	634	case KVM_CAP_MAX_VCPUS:
633	case KVM_CAP_MAX_VCPU_ID:	635	case KVM_CAP_MAX_VCPU_ID:
634	r = KVM_S390_BSCA_CPU_SLOTS;	636	/*
		637	* Return the same value for KVM_CAP_MAX_VCPUS and
		638	* KVM_CAP_MAX_VCPU_ID to conform with the KVM API.
		639	*/
		640	r = KVM_S390_ESCA_CPU_SLOTS;
635	if (!kvm_s390_use_sca_entries())	641	if (!kvm_s390_use_sca_entries())
636	r = KVM_MAX_VCPUS;	642	r = KVM_MAX_VCPUS;
637	else if (sclp.has_esca && sclp.has_64bscao)
638	r = KVM_S390_ESCA_CPU_SLOTS;
639	if (ext == KVM_CAP_NR_VCPUS)	643	if (ext == KVM_CAP_NR_VCPUS)
640	r = min_t(unsigned int, num_online_cpus(), r);	644	r = min_t(unsigned int, num_online_cpus(), r);
641	break;	645	break;
@@ -919,6 +923,12 @@ int kvm_vm_ioctl_enable_cap(struct kvm kvm, struct kvm_enable_cap cap) · 277 unmodified lines
919	VM_EVENT(kvm, 3, "ENABLE: CAP_S390_CPU_TOPOLOGY %s",	923	VM_EVENT(kvm, 3, "ENABLE: CAP_S390_CPU_TOPOLOGY %s",
920	r ? "(not available)" : "(success)");	924	r ? "(not available)" : "(success)");
921	break;	925	break;
		926	case KVM_CAP_S390_USER_OPEREXEC:
		927	VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_OPEREXEC");
		928	kvm->arch.user_operexec = 1;
		929	icpt_operexc_on_all_vcpus(kvm);
		930	r = 0;
		931	break;
922	default:	932	default:
923	r = -EINVAL;	933	r = -EINVAL;
924	break;	934	break;
@@ -1930,22 +1940,18 @@ static int kvm_s390_get_cpu_model(struct kvm kvm, struct kvm_device_attr attr) · 1005 unmodified lines
1930	* Updates the Multiprocessor Topology-Change-Report bit to signal	1940	* Updates the Multiprocessor Topology-Change-Report bit to signal
1931	* the guest with a topology change.	1941	* the guest with a topology change.
1932	* This is only relevant if the topology facility is present.	1942	* This is only relevant if the topology facility is present.
1933	*
1934	* The SCA version, bsca or esca, doesn't matter as offset is the same.
1935	*/	1943	*/
1936	static void kvm_s390_update_topology_change_report(struct kvm *kvm, bool val)	1944	static void kvm_s390_update_topology_change_report(struct kvm *kvm, bool val)
1937	{	1945	{
1938	union sca_utility new, old;	1946	union sca_utility new, old;
1939	struct bsca_block *sca;	1947	struct esca_block *sca;
1940		1948
1941	read_lock(&kvm->arch.sca_lock);
1942	sca = kvm->arch.sca;	1949	sca = kvm->arch.sca;
1943	old = READ_ONCE(sca->utility);	1950	old = READ_ONCE(sca->utility);
1944	do {	1951	do {
1945	new = old;	1952	new = old;
1946	new.mtcr = val;	1953	new.mtcr = val;
1947	} while (!try_cmpxchg(&sca->utility.val, &old.val, new.val));	1954	} while (!try_cmpxchg(&sca->utility.val, &old.val, new.val));
1948	read_unlock(&kvm->arch.sca_lock);
1949	}	1955	}
1950		1956
1951	static int kvm_s390_set_topo_change_indication(struct kvm *kvm,	1957	static int kvm_s390_set_topo_change_indication(struct kvm *kvm,
@@ -1966,9 +1972,7 @@ static int kvm_s390_get_topo_change_indication(struct kvm *kvm, · 14 unmodified lines
1966	if (!test_kvm_facility(kvm, 11))	1972	if (!test_kvm_facility(kvm, 11))
1967	return -ENXIO;	1973	return -ENXIO;
1968		1974
1969	read_lock(&kvm->arch.sca_lock);	1975	topo = kvm->arch.sca->utility.mtcr;
1970	topo = ((struct bsca_block *)kvm->arch.sca)->utility.mtcr;
1971	read_unlock(&kvm->arch.sca_lock);
1972		1976
1973	return put_user(topo, (u8 __user *)attr->addr);	1977	return put_user(topo, (u8 __user *)attr->addr);
1974	}	1978	}
@@ -2666,14 +2670,6 @@ static int kvm_s390_handle_pv(struct kvm kvm, struct kvm_pv_cmd cmd) · 691 unmodified lines
2666	if (kvm_s390_pv_is_protected(kvm))	2670	if (kvm_s390_pv_is_protected(kvm))
2667	break;	2671	break;
2668		2672
2669	/*
2670	* FMT 4 SIE needs esca. As we never switch back to bsca from
2671	* esca, we need no cleanup in the error cases below
2672	*/
2673	r = sca_switch_to_extended(kvm);
2674	if (r)
2675	break;
2676
2677	mmap_write_lock(kvm->mm);	2673	mmap_write_lock(kvm->mm);
2678	r = gmap_helper_disable_cow_sharing();	2674	r = gmap_helper_disable_cow_sharing();
2679	mmap_write_unlock(kvm->mm);	2675	mmap_write_unlock(kvm->mm);
@@ -3316,10 +3312,7 @@ static void kvm_s390_crypto_init(struct kvm *kvm) · 636 unmodified lines
3316		3312
3317	static void sca_dispose(struct kvm *kvm)	3313	static void sca_dispose(struct kvm *kvm)
3318	{	3314	{
3319	if (kvm->arch.use_esca)	3315	free_pages_exact(kvm->arch.sca, sizeof(*kvm->arch.sca));
3320	free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
3321	else
3322	free_page((unsigned long)(kvm->arch.sca));
3323	kvm->arch.sca = NULL;	3316	kvm->arch.sca = NULL;
3324	}	3317	}
3325		3318
@@ -3333,10 +3326,9 @@ void kvm_arch_free_vm(struct kvm *kvm) · 7 unmodified lines
3333		3326
3334	int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)	3327	int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
3335	{	3328	{
3336	gfp_t alloc_flags = GFP_KERNEL_ACCOUNT;	3329	gfp_t alloc_flags = GFP_KERNEL_ACCOUNT \| __GFP_ZERO;
3337	int i, rc;
3338	char debug_name[16];	3330	char debug_name[16];
3339	static unsigned long sca_offset;	3331	int i, rc;
3340		3332
3341	rc = -EINVAL;	3333	rc = -EINVAL;
3342	#ifdef CONFIG_KVM_S390_UCONTROL	3334	#ifdef CONFIG_KVM_S390_UCONTROL
@@ -3357,20 +3349,14 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 14 unmodified lines
3357		3349
3358	if (!sclp.has_64bscao)	3350	if (!sclp.has_64bscao)
3359	alloc_flags \|= GFP_DMA;	3351	alloc_flags \|= GFP_DMA;
3360	rwlock_init(&kvm->arch.sca_lock);
3361	/* start with basic SCA */
3362	kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
3363	if (!kvm->arch.sca)
3364	goto out_err;
3365	mutex_lock(&kvm_lock);	3352	mutex_lock(&kvm_lock);
3366	sca_offset += 16;	3353
3367	if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)	3354	kvm->arch.sca = alloc_pages_exact(sizeof(*kvm->arch.sca), alloc_flags);
3368	sca_offset = 0;
3369	kvm->arch.sca = (struct bsca_block *)
3370	((char *) kvm->arch.sca + sca_offset);
3371	mutex_unlock(&kvm_lock);	3355	mutex_unlock(&kvm_lock);
		3356	if (!kvm->arch.sca)
		3357	goto out_err;
3372		3358
3373	sprintf(debug_name, "kvm-%u", current->pid);	3359	snprintf(debug_name, sizeof(debug_name), "kvm-%u", current->pid);
3374		3360
3375	kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));	3361	kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
3376	if (!kvm->arch.dbf)	3362	if (!kvm->arch.dbf)
@@ -3547,133 +3533,38 @@ static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu) · 170 unmodified lines
3547		3533
3548	static void sca_del_vcpu(struct kvm_vcpu *vcpu)	3534	static void sca_del_vcpu(struct kvm_vcpu *vcpu)
3549	{	3535	{
		3536	struct esca_block *sca = vcpu->kvm->arch.sca;
		3537
3550	if (!kvm_s390_use_sca_entries())	3538	if (!kvm_s390_use_sca_entries())
3551	return;	3539	return;
3552	read_lock(&vcpu->kvm->arch.sca_lock);
3553	if (vcpu->kvm->arch.use_esca) {
3554	struct esca_block *sca = vcpu->kvm->arch.sca;
3555		3540
3556	clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);	3541	clear_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn);
3557	sca->cpu[vcpu->vcpu_id].sda = 0;	3542	sca->cpu[vcpu->vcpu_id].sda = 0;
3558	} else {
3559	struct bsca_block *sca = vcpu->kvm->arch.sca;
3560
3561	clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
3562	sca->cpu[vcpu->vcpu_id].sda = 0;
3563	}
3564	read_unlock(&vcpu->kvm->arch.sca_lock);
3565	}	3543	}
3566		3544
3567	static void sca_add_vcpu(struct kvm_vcpu *vcpu)	3545	static void sca_add_vcpu(struct kvm_vcpu *vcpu)
3568	{	3546	{
3569	if (!kvm_s390_use_sca_entries()) {	3547	struct esca_block *sca = vcpu->kvm->arch.sca;
3570	phys_addr_t sca_phys = virt_to_phys(vcpu->kvm->arch.sca);	3548	phys_addr_t sca_phys = virt_to_phys(sca);
3571
3572	/* we still need the basic sca for the ipte control */
3573	vcpu->arch.sie_block->scaoh = sca_phys >> 32;
3574	vcpu->arch.sie_block->scaol = sca_phys;
3575	return;
3576	}
3577	read_lock(&vcpu->kvm->arch.sca_lock);
3578	if (vcpu->kvm->arch.use_esca) {
3579	struct esca_block *sca = vcpu->kvm->arch.sca;
3580	phys_addr_t sca_phys = virt_to_phys(sca);
3581
3582	sca->cpu[vcpu->vcpu_id].sda = virt_to_phys(vcpu->arch.sie_block);
3583	vcpu->arch.sie_block->scaoh = sca_phys >> 32;
3584	vcpu->arch.sie_block->scaol = sca_phys & ESCA_SCAOL_MASK;
3585	vcpu->arch.sie_block->ecb2 \|= ECB2_ESCA;
3586	set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
3587	} else {
3588	struct bsca_block *sca = vcpu->kvm->arch.sca;
3589	phys_addr_t sca_phys = virt_to_phys(sca);
3590
3591	sca->cpu[vcpu->vcpu_id].sda = virt_to_phys(vcpu->arch.sie_block);
3592	vcpu->arch.sie_block->scaoh = sca_phys >> 32;
3593	vcpu->arch.sie_block->scaol = sca_phys;
3594	set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
3595	}
3596	read_unlock(&vcpu->kvm->arch.sca_lock);
3597	}
3598
3599	/* Basic SCA to Extended SCA data copy routines */
3600	static inline void sca_copy_entry(struct esca_entry d, struct bsca_entry s)
3601	{
3602	d->sda = s->sda;
3603	d->sigp_ctrl.c = s->sigp_ctrl.c;
3604	d->sigp_ctrl.scn = s->sigp_ctrl.scn;
3605	}
3606
3607	static void sca_copy_b_to_e(struct esca_block d, struct bsca_block s)
3608	{
3609	int i;
3610		3549
3611	d->ipte_control = s->ipte_control;	3550	/* we still need the sca header for the ipte control */
3612	d->mcn[0] = s->mcn;	3551	vcpu->arch.sie_block->scaoh = sca_phys >> 32;
3613	for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)	3552	vcpu->arch.sie_block->scaol = sca_phys & ESCA_SCAOL_MASK;
3614	sca_copy_entry(&d->cpu[i], &s->cpu[i]);	3553	vcpu->arch.sie_block->ecb2 \|= ECB2_ESCA;
3615	}
3616		3554
3617	static int sca_switch_to_extended(struct kvm *kvm)	3555	if (!kvm_s390_use_sca_entries())
3618	{	3556	return;
3619	struct bsca_block *old_sca = kvm->arch.sca;
3620	struct esca_block *new_sca;
3621	struct kvm_vcpu *vcpu;
3622	unsigned long vcpu_idx;
3623	u32 scaol, scaoh;
3624	phys_addr_t new_sca_phys;
3625
3626	if (kvm->arch.use_esca)
3627	return 0;
3628
3629	new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL_ACCOUNT \| __GFP_ZERO);
3630	if (!new_sca)
3631	return -ENOMEM;
3632
3633	new_sca_phys = virt_to_phys(new_sca);
3634	scaoh = new_sca_phys >> 32;
3635	scaol = new_sca_phys & ESCA_SCAOL_MASK;
3636
3637	kvm_s390_vcpu_block_all(kvm);
3638	write_lock(&kvm->arch.sca_lock);
3639
3640	sca_copy_b_to_e(new_sca, old_sca);
3641
3642	kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
3643	vcpu->arch.sie_block->scaoh = scaoh;
3644	vcpu->arch.sie_block->scaol = scaol;
3645	vcpu->arch.sie_block->ecb2 \|= ECB2_ESCA;
3646	}
3647	kvm->arch.sca = new_sca;
3648	kvm->arch.use_esca = 1;
3649
3650	write_unlock(&kvm->arch.sca_lock);
3651	kvm_s390_vcpu_unblock_all(kvm);
3652
3653	free_page((unsigned long)old_sca);
3654		3557
3655	VM_EVENT(kvm, 2, "Switched to ESCA (0x%p -> 0x%p)",	3558	set_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn);
3656	old_sca, kvm->arch.sca);	3559	sca->cpu[vcpu->vcpu_id].sda = virt_to_phys(vcpu->arch.sie_block);
3657	return 0;
3658	}	3560	}
3659		3561
3660	static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)	3562	static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
3661	{	3563	{
3662	int rc;	3564	if (!kvm_s390_use_sca_entries())
3663		3565	return id < KVM_MAX_VCPUS;
3664	if (!kvm_s390_use_sca_entries()) {
3665	if (id < KVM_MAX_VCPUS)
3666	return true;
3667	return false;
3668	}
3669	if (id < KVM_S390_BSCA_CPU_SLOTS)
3670	return true;
3671	if (!sclp.has_esca \|\| !sclp.has_64bscao)
3672	return false;
3673
3674	rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
3675		3566
3676	return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;	3567	return id < KVM_S390_ESCA_CPU_SLOTS;
3677	}	3568	}
3678		3569
3679	/* needs disabled preemption to protect from TOD sync and vcpu_load/put */	3570	/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
@@ -3919,7 +3810,7 @@ static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu) · 239 unmodified lines
3919	vcpu->arch.sie_block->eca \|= ECA_IB;	3810	vcpu->arch.sie_block->eca \|= ECA_IB;
3920	if (sclp.has_siif)	3811	if (sclp.has_siif)
3921	vcpu->arch.sie_block->eca \|= ECA_SII;	3812	vcpu->arch.sie_block->eca \|= ECA_SII;
3922	if (sclp.has_sigpif)	3813	if (kvm_s390_use_sca_entries())
3923	vcpu->arch.sie_block->eca \|= ECA_SIGPI;	3814	vcpu->arch.sie_block->eca \|= ECA_SIGPI;
3924	if (test_kvm_facility(vcpu->kvm, 129)) {	3815	if (test_kvm_facility(vcpu->kvm, 129)) {
3925	vcpu->arch.sie_block->eca \|= ECA_VX;	3816	vcpu->arch.sie_block->eca \|= ECA_VX;
@@ -4366,8 +4257,6 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, · 440 unmodified lines
4366		4257
4367	int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu vcpu, struct kvm_fpu fpu)	4258	int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu vcpu, struct kvm_fpu fpu)
4368	{	4259	{
4369	int ret = 0;
4370
4371	vcpu_load(vcpu);	4260	vcpu_load(vcpu);
4372		4261
4373	vcpu->run->s.regs.fpc = fpu->fpc;	4262	vcpu->run->s.regs.fpc = fpu->fpc;
@@ -4378,7 +4267,7 @@ int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu vcpu, struct kvm_fpu fpu) · 4 unmodified lines
4378	memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));	4267	memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
4379		4268
4380	vcpu_put(vcpu);	4269	vcpu_put(vcpu);
4381	return ret;	4270	return 0;
4382	}	4271	}
4383		4272
4384	int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu vcpu, struct kvm_fpu fpu)	4273	int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu vcpu, struct kvm_fpu fpu)
@@ -4786,9 +4675,6 @@ static int vcpu_pre_run(struct kvm_vcpu *vcpu) · 401 unmodified lines
4786	vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];	4675	vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
4787	vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];	4676	vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
4788		4677
4789	if (need_resched())
4790	schedule();
4791
4792	if (!kvm_is_ucontrol(vcpu->kvm)) {	4678	if (!kvm_is_ucontrol(vcpu->kvm)) {
4793	rc = kvm_s390_deliver_pending_interrupts(vcpu);	4679	rc = kvm_s390_deliver_pending_interrupts(vcpu);
4794	if (rc \|\| guestdbg_exit_pending(vcpu))	4680	if (rc \|\| guestdbg_exit_pending(vcpu))
@@ -5073,13 +4959,8 @@ int noinstr kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb, · 278 unmodified lines
5073	* The guest_state_{enter,exit}_irqoff() functions inform lockdep and	4959	* The guest_state_{enter,exit}_irqoff() functions inform lockdep and
5074	* tracing that entry to the guest will enable host IRQs, and exit from	4960	* tracing that entry to the guest will enable host IRQs, and exit from
5075	* the guest will disable host IRQs.	4961	* the guest will disable host IRQs.
5076	*
5077	* We must not use lockdep/tracing/RCU in this critical section, so we
5078	* use the low-level arch_local_irq_*() helpers to enable/disable IRQs.
5079	*/	4962	*/
5080	arch_local_irq_enable();
5081	ret = sie64a(scb, gprs, gasce);	4963	ret = sie64a(scb, gprs, gasce);
5082	arch_local_irq_disable();
5083		4964
5084	guest_state_exit_irqoff();	4965	guest_state_exit_irqoff();
5085		4966
@@ -5098,12 +4979,12 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) · 12 unmodified lines
5098	*/	4979	*/
5099	kvm_vcpu_srcu_read_lock(vcpu);	4980	kvm_vcpu_srcu_read_lock(vcpu);
5100		4981
5101	do {	4982	while (true) {
5102	rc = vcpu_pre_run(vcpu);	4983	rc = vcpu_pre_run(vcpu);
		4984	kvm_vcpu_srcu_read_unlock(vcpu);
5103	if (rc \|\| guestdbg_exit_pending(vcpu))	4985	if (rc \|\| guestdbg_exit_pending(vcpu))
5104	break;	4986	break;
5105		4987
5106	kvm_vcpu_srcu_read_unlock(vcpu);
5107	/*	4988	/*
5108	* As PF_VCPU will be used in fault handler, between	4989	* As PF_VCPU will be used in fault handler, between
5109	* guest_timing_enter_irqoff and guest_timing_exit_irqoff	4990	* guest_timing_enter_irqoff and guest_timing_exit_irqoff
@@ -5115,7 +4996,17 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) · 5 unmodified lines
5115	sizeof(sie_page->pv_grregs));	4996	sizeof(sie_page->pv_grregs));
5116	}	4997	}
5117		4998
		4999	xfer_to_guest_mode_check:
5118	local_irq_disable();	5000	local_irq_disable();
		5001	xfer_to_guest_mode_prepare();
		5002	if (xfer_to_guest_mode_work_pending()) {
		5003	local_irq_enable();
		5004	rc = kvm_xfer_to_guest_mode_handle_work(vcpu);
		5005	if (rc)
		5006	break;
		5007	goto xfer_to_guest_mode_check;
		5008	}
		5009
5119	guest_timing_enter_irqoff();	5010	guest_timing_enter_irqoff();
5120	__disable_cpu_timer_accounting(vcpu);	5011	__disable_cpu_timer_accounting(vcpu);
5121		5012
@@ -5145,9 +5036,12 @@ static int __vcpu_run(struct kvm_vcpu *vcpu) · 23 unmodified lines
5145	kvm_vcpu_srcu_read_lock(vcpu);	5036	kvm_vcpu_srcu_read_lock(vcpu);
5146		5037
5147	rc = vcpu_post_run(vcpu, exit_reason);	5038	rc = vcpu_post_run(vcpu, exit_reason);
5148	} while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);	5039	if (rc \|\| guestdbg_exit_pending(vcpu)) {
		5040	kvm_vcpu_srcu_read_unlock(vcpu);
		5041	break;
		5042	}
		5043	}
5149		5044
5150	kvm_vcpu_srcu_read_unlock(vcpu);
5151	return rc;	5045	return rc;
5152	}	5046	}
5153		5047
@@ -5363,6 +5257,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) · 209 unmodified lines
5363		5257
5364	if (signal_pending(current) && !rc) {	5258	if (signal_pending(current) && !rc) {
5365	kvm_run->exit_reason = KVM_EXIT_INTR;	5259	kvm_run->exit_reason = KVM_EXIT_INTR;
		5260	vcpu->stat.signal_exits++;
5366	rc = -EINTR;	5261	rc = -EINTR;
5367	}	5262	}
5368		5263
@@ -5729,8 +5624,8 @@ static long kvm_s390_vcpu_memsida_op(struct kvm_vcpu *vcpu, · 360 unmodified lines
5729	return r;	5624	return r;
5730	}	5625	}
5731		5626
5732	long kvm_arch_vcpu_async_ioctl(struct file *filp,	5627	long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl,
5733	unsigned int ioctl, unsigned long arg)	5628	unsigned long arg)
5734	{	5629	{
5735	struct kvm_vcpu *vcpu = filp->private_data;	5630	struct kvm_vcpu *vcpu = filp->private_data;
5736	void __user argp = (void __user )arg;	5631	void __user argp = (void __user )arg;

arch/s390/kvm/kvm-s390.h+1−8

 int kvm_s390_handle_per_ifetch_icpt(struct kvm_vcpu *vcpu);
 int kvm_s390_handle_per_event(struct kvm_vcpu *vcpu);
-/* support for Basic/Extended SCA handling */
-static inline union ipte_control *kvm_s390_get_ipte_control(struct kvm *kvm)
-{
-	struct bsca_block *sca = kvm->arch.sca; /* SCA version doesn't matter */
-	return &sca->ipte_control;
-}
 static inline int kvm_s390_use_sca_entries(void)
 {
 	/*
 	 * might use the entries. By not setting the entries and keeping them
 	 * invalid, hardware will not access them but intercept.
 	 */
-	return sclp.has_sigpif;
+	return sclp.has_sigpif && sclp.has_esca;
 }
 void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu,
 				     struct mcck_volatile_info *mcck_info);

arch/s390/kvm/vsie.c+14−6

 		else if ((gpa & ~0x1fffUL) == kvm_s390_get_prefix(vcpu))
 			rc = set_validity_icpt(scb_s, 0x0011U);
 		else if ((gpa & PAGE_MASK) !=
-			 ((gpa + sizeof(struct bsca_block) - 1) & PAGE_MASK))
+			 ((gpa + offsetof(struct bsca_block, cpu[0]) - 1) & PAGE_MASK))
 			rc = set_validity_icpt(scb_s, 0x003bU);
 		if (!rc) {
 			rc = pin_guest_page(vcpu->kvm, gpa, &hpa);
 	current->thread.gmap_int_code = 0;
 	barrier();
 	if (!kvm_s390_vcpu_sie_inhibited(vcpu)) {
+xfer_to_guest_mode_check:
 		local_irq_disable();
+		xfer_to_guest_mode_prepare();
+		if (xfer_to_guest_mode_work_pending()) {
+			local_irq_enable();
+			rc = kvm_xfer_to_guest_mode_handle_work(vcpu);
+			if (rc)
+				goto skip_sie;
+			goto xfer_to_guest_mode_check;
+		}
 		guest_timing_enter_irqoff();
 		rc = kvm_s390_enter_exit_sie(scb_s, vcpu->run->s.regs.gprs, vsie_page->gmap->asce);
 		guest_timing_exit_irqoff();
 		local_irq_enable();
 	}
+skip_sie:
 	barrier();
 	vcpu->arch.sie_block->prog0c &= ~PROG_IN_SIE;
 		 * but rewind the PSW to re-enter SIE once that's completed
 		 * instead of passing a "no action" intercept to the guest.
 		 */
-		if (signal_pending(current) ||
+		if (kvm_s390_vcpu_has_irq(vcpu, 0) ||
-		    kvm_s390_vcpu_has_irq(vcpu, 0) ||
 		    kvm_s390_vcpu_sie_inhibited(vcpu)) {
 			kvm_s390_rewind_psw(vcpu, 4);
 			break;
 		}
-		cond_resched();
 	}
 	if (rc == -EFAULT) {
 	if (unlikely(scb_addr & 0x1ffUL))
 		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
-	if (signal_pending(current) || kvm_s390_vcpu_has_irq(vcpu, 0) ||
+	if (kvm_s390_vcpu_has_irq(vcpu, 0) || kvm_s390_vcpu_sie_inhibited(vcpu)) {
-	    kvm_s390_vcpu_sie_inhibited(vcpu)) {
 		kvm_s390_rewind_psw(vcpu, 4);
 		return 0;
 	}

arch/x86/include/asm/cpufeatures.h+7−0

 #define X86_FEATURE_AMD_STIBP		(13*32+15) /* Single Thread Indirect Branch Predictors */
 #define X86_FEATURE_AMD_STIBP_ALWAYS_ON	(13*32+17) /* Single Thread Indirect Branch Predictors always-on preferred */
 #define X86_FEATURE_AMD_IBRS_SAME_MODE	(13*32+19) /* Indirect Branch Restricted Speculation same mode protection*/
+#define X86_FEATURE_EFER_LMSLE_MBZ	(13*32+20) /* EFER.LMSLE must be zero */
 #define X86_FEATURE_AMD_PPIN		(13*32+23) /* "amd_ppin" Protected Processor Inventory Number */
 #define X86_FEATURE_AMD_SSBD		(13*32+24) /* Speculative Store Bypass Disable */
 #define X86_FEATURE_VIRT_SSBD		(13*32+25) /* "virt_ssbd" Virtualized Speculative Store Bypass Disable */
 #define X86_FEATURE_SGX_EUPDATESVN	(21*32+17) /* Support for ENCLS[EUPDATESVN] instruction */
 #define X86_FEATURE_SDCIAE		(21*32+18) /* L3 Smart Data Cache Injection Allocation Enforcement */
+#define X86_FEATURE_CLEAR_CPU_BUF_VM_MMIO (21*32+19) /*
+						      * Clear CPU buffers before VM-Enter if the vCPU
+						      * can access host MMIO (ignored for all intents
+						      * and purposes if CLEAR_CPU_BUF_VM is set).
+						      */
+#define X86_FEATURE_X2AVIC_EXT		(21*32+20) /* AMD SVM x2AVIC support for 4k vCPUs */
 /*
  * BUG word(s)

arch/x86/include/asm/hardirq.h+2−2

@@ -5,7 +5,7 @@ · 4 unmodified lines
5	#include <linux/threads.h>	5	#include <linux/threads.h>
6		6
7	typedef struct {	7	typedef struct {
8	#if IS_ENABLED(CONFIG_KVM_INTEL)	8	#if IS_ENABLED(CONFIG_CPU_MITIGATIONS) && IS_ENABLED(CONFIG_KVM_INTEL)
9	u8 kvm_cpu_l1tf_flush_l1d;	9	u8 kvm_cpu_l1tf_flush_l1d;
10	#endif	10	#endif
11	unsigned int __nmi_count; /* arch dependent */	11	unsigned int __nmi_count; /* arch dependent */
@@ -68,7 +68,7 @@ extern u64 arch_irq_stat(void); · 56 unmodified lines
68	DECLARE_PER_CPU_CACHE_HOT(u16, __softirq_pending);	68	DECLARE_PER_CPU_CACHE_HOT(u16, __softirq_pending);
69	#define local_softirq_pending_ref __softirq_pending	69	#define local_softirq_pending_ref __softirq_pending
70		70
71	#if IS_ENABLED(CONFIG_KVM_INTEL)	71	#if IS_ENABLED(CONFIG_CPU_MITIGATIONS) && IS_ENABLED(CONFIG_KVM_INTEL)
72	/*	72	/*
73	* This function is called from noinstr interrupt contexts	73	* This function is called from noinstr interrupt contexts
74	* and must be inlined to not get instrumentation.	74	* and must be inlined to not get instrumentation.

arch/x86/include/asm/kvm-x86-ops.h+1−0

@@ -128,6 +128,7 @@ KVM_X86_OP(enable_smi_window) · 127 unmodified lines
128	KVM_X86_OP_OPTIONAL(dev_get_attr)	128	KVM_X86_OP_OPTIONAL(dev_get_attr)
129	KVM_X86_OP_OPTIONAL(mem_enc_ioctl)	129	KVM_X86_OP_OPTIONAL(mem_enc_ioctl)
130	KVM_X86_OP_OPTIONAL(vcpu_mem_enc_ioctl)	130	KVM_X86_OP_OPTIONAL(vcpu_mem_enc_ioctl)
		131	KVM_X86_OP_OPTIONAL(vcpu_mem_enc_unlocked_ioctl)
131	KVM_X86_OP_OPTIONAL(mem_enc_register_region)	132	KVM_X86_OP_OPTIONAL(mem_enc_register_region)
132	KVM_X86_OP_OPTIONAL(mem_enc_unregister_region)	133	KVM_X86_OP_OPTIONAL(mem_enc_unregister_region)
133	KVM_X86_OP_OPTIONAL(vm_copy_enc_context_from)	134	KVM_X86_OP_OPTIONAL(vm_copy_enc_context_from)

arch/x86/include/asm/kvm_host.h+14−9

 	/* be preempted when it's in kernel-mode(cpl=0) */
 	bool preempted_in_kernel;
-	/* Flush the L1 Data cache for L1TF mitigation on VMENTER */
-	bool l1tf_flush_l1d;
 	/* Host CPU on which VM-entry was most recently attempted */
 	int last_vmentry_cpu;
 	bool use_master_clock;
 	u64 master_kernel_ns;
 	u64 master_cycle_now;
-	struct delayed_work kvmclock_update_work;
-	struct delayed_work kvmclock_sync_work;
 #ifdef CONFIG_KVM_HYPERV
 	struct kvm_hv hyperv;
 				void *external_spt);
 	/* Update the external page table from spte getting set. */
 	int (*set_external_spte)(struct kvm *kvm, gfn_t gfn, enum pg_level level,
-				 kvm_pfn_t pfn_for_gfn);
+				 u64 mirror_spte);
 	/* Update external page tables for page table about to be freed. */
 	int (*free_external_spt)(struct kvm *kvm, gfn_t gfn, enum pg_level level,
 				 void *external_spt);
 	/* Update external page table from spte getting removed, and flush TLB. */
-	int (*remove_external_spte)(struct kvm *kvm, gfn_t gfn, enum pg_level level,
+	void (*remove_external_spte)(struct kvm *kvm, gfn_t gfn, enum pg_level level,
-				    kvm_pfn_t pfn_for_gfn);
+				     u64 mirror_spte);
 	bool (*has_wbinvd_exit)(void);
 	int (*dev_get_attr)(u32 group, u64 attr, u64 *val);
 	int (*mem_enc_ioctl)(struct kvm *kvm, void __user *argp);
 	int (*vcpu_mem_enc_ioctl)(struct kvm_vcpu *vcpu, void __user *argp);
+	int (*vcpu_mem_enc_unlocked_ioctl)(struct kvm_vcpu *vcpu, void __user *argp);
 	int (*mem_enc_register_region)(struct kvm *kvm, struct kvm_enc_region *argp);
 	int (*mem_enc_unregister_region)(struct kvm *kvm, struct kvm_enc_region *argp);
 	int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd);
  *			     the gfn, i.e. retrying the instruction will hit a
  *			     !PRESENT fault, which results in a new shadow page
  *			     and sends KVM back to square one.
+ *
+ * EMULTYPE_SKIP_SOFT_INT - Set in combination with EMULTYPE_SKIP to only skip
+ *                          an instruction if it could generate a given software
+ *                          interrupt, which must be encoded via
+ *                          EMULTYPE_SET_SOFT_INT_VECTOR().
  */
 #define EMULTYPE_NO_DECODE	    (1 << 0)
 #define EMULTYPE_TRAP_UD	    (1 << 1)
 #define EMULTYPE_PF		    (1 << 6)
 #define EMULTYPE_COMPLETE_USER_EXIT (1 << 7)
 #define EMULTYPE_WRITE_PF_TO_SP	    (1 << 8)
+#define EMULTYPE_SKIP_SOFT_INT	    (1 << 9)
+#define EMULTYPE_SET_SOFT_INT_VECTOR(v)	((u32)((v) & 0xff) << 16)
+#define EMULTYPE_GET_SOFT_INT_VECTOR(e)	(((e) >> 16) & 0xff)
 static inline bool kvm_can_emulate_event_vectoring(int emul_type)
 {
 void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu);
 void kvm_prepare_event_vectoring_exit(struct kvm_vcpu *vcpu, gpa_t gpa);
+void kvm_prepare_unexpected_reason_exit(struct kvm_vcpu *vcpu, u64 exit_reason);
 void kvm_enable_efer_bits(u64);
 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer);
 int kvm_add_user_return_msr(u32 msr);
 int kvm_find_user_return_msr(u32 msr);
 int kvm_set_user_return_msr(unsigned index, u64 val, u64 mask);
-void kvm_user_return_msr_update_cache(unsigned int index, u64 val);
 u64 kvm_get_user_return_msr(unsigned int slot);
 static inline bool kvm_is_supported_user_return_msr(u32 msr)

arch/x86/include/asm/nospec-branch.h+15−15

  * CFLAGS.ZF.
  * Note: Only the memory operand variant of VERW clears the CPU buffers.
  */
-.macro __CLEAR_CPU_BUFFERS feature
 #ifdef CONFIG_X86_64
-	ALTERNATIVE "", "verw x86_verw_sel(%rip)", \feature
+#define VERW	verw x86_verw_sel(%rip)
 #else
-	/*
+/*
-	 * In 32bit mode, the memory operand must be a %cs reference. The data
+ * In 32bit mode, the memory operand must be a %cs reference. The data segments
-	 * segments may not be usable (vm86 mode), and the stack segment may not
+ * may not be usable (vm86 mode), and the stack segment may not be flat (ESPFIX32).
-	 * be flat (ESPFIX32).
+ */
-	 */
+#define VERW	verw %cs:x86_verw_sel
-	ALTERNATIVE "", "verw %cs:x86_verw_sel", \feature
 #endif
-.endm
-#define CLEAR_CPU_BUFFERS \
+/*
-	__CLEAR_CPU_BUFFERS X86_FEATURE_CLEAR_CPU_BUF
+ * Provide a stringified VERW macro for simple usage, and a non-stringified
+ * VERW macro for use in more elaborate sequences, e.g. to encode a conditional
+ * VERW within an ALTERNATIVE.
+ */
+#define __CLEAR_CPU_BUFFERS	__stringify(VERW)
-#define VM_CLEAR_CPU_BUFFERS \
+/* If necessary, emit VERW on exit-to-userspace to clear CPU buffers. */
-	__CLEAR_CPU_BUFFERS X86_FEATURE_CLEAR_CPU_BUF_VM
+#define CLEAR_CPU_BUFFERS \
+	ALTERNATIVE "", __CLEAR_CPU_BUFFERS, X86_FEATURE_CLEAR_CPU_BUF
 #ifdef CONFIG_X86_64
 .macro CLEAR_BRANCH_HISTORY
 DECLARE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush);
-DECLARE_STATIC_KEY_FALSE(cpu_buf_vm_clear);
 extern u16 x86_verw_sel;
 #include <asm/segment.h>

arch/x86/include/asm/svm.h+4−1

 	AVIC_IPI_FAILURE_INVALID_IPI_VECTOR,
 };
-#define AVIC_PHYSICAL_MAX_INDEX_MASK	GENMASK_ULL(8, 0)
+#define AVIC_PHYSICAL_MAX_INDEX_MASK	GENMASK_ULL(11, 0)
 /*
  * For AVIC, the max index allowed for physical APIC ID table is 0xfe (254), as
 /*
  * For x2AVIC, the max index allowed for physical APIC ID table is 0x1ff (511).
+ * With X86_FEATURE_X2AVIC_EXT, the max index is increased to 0xfff (4095).
  */
 #define X2AVIC_MAX_PHYSICAL_ID		0x1FFUL
+#define X2AVIC_4K_MAX_PHYSICAL_ID	0xFFFUL
 static_assert((AVIC_MAX_PHYSICAL_ID & AVIC_PHYSICAL_MAX_INDEX_MASK) == AVIC_MAX_PHYSICAL_ID);
 static_assert((X2AVIC_MAX_PHYSICAL_ID & AVIC_PHYSICAL_MAX_INDEX_MASK) == X2AVIC_MAX_PHYSICAL_ID);
+static_assert((X2AVIC_4K_MAX_PHYSICAL_ID & AVIC_PHYSICAL_MAX_INDEX_MASK) == X2AVIC_4K_MAX_PHYSICAL_ID);
 #define SVM_SEV_FEAT_SNP_ACTIVE				BIT(0)
 #define SVM_SEV_FEAT_RESTRICTED_INJECTION		BIT(3)

arch/x86/include/uapi/asm/kvm.h+1−0

@@ -502,6 +502,7 @@ struct kvm_sync_regs { · 501 unmodified lines
502	/* vendor-specific groups and attributes for system fd */	502	/* vendor-specific groups and attributes for system fd */
503	#define KVM_X86_GRP_SEV 1	503	#define KVM_X86_GRP_SEV 1
504	# define KVM_X86_SEV_VMSA_FEATURES 0	504	# define KVM_X86_SEV_VMSA_FEATURES 0
		505	# define KVM_X86_SNP_POLICY_BITS 1
505		506
506	struct kvm_vmx_nested_state_data {	507	struct kvm_vmx_nested_state_data {
507	__u8 vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE];	508	__u8 vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE];

arch/x86/kernel/cpu/bugs.c+9−13

  */
 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush);
-/*
- * Controls CPU Fill buffer clear before VMenter. This is a subset of
- * X86_FEATURE_CLEAR_CPU_BUF, and should only be enabled when KVM-only
- * mitigation is required.
- */
-DEFINE_STATIC_KEY_FALSE(cpu_buf_vm_clear);
-EXPORT_SYMBOL_FOR_KVM(cpu_buf_vm_clear);
 #undef pr_fmt
 #define pr_fmt(fmt)	"mitigations: " fmt
 	IS_ENABLED(CONFIG_MITIGATION_RFDS) ? RFDS_MITIGATION_AUTO : RFDS_MITIGATION_OFF;
 /*
- * Set if any of MDS/TAA/MMIO/RFDS are going to enable VERW clearing
+ * Set if any of MDS/TAA/MMIO/RFDS are going to enable VERW clearing on exit to
- * through X86_FEATURE_CLEAR_CPU_BUF on kernel and guest entry.
+ * userspace *and* on entry to KVM guests.
  */
 static bool verw_clear_cpu_buf_mitigation_selected __ro_after_init;
 	if (mds_mitigation == MDS_MITIGATION_FULL ||
 	    mds_mitigation == MDS_MITIGATION_VMWERV) {
 		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
+		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM);
 		if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) &&
 		    (mds_nosmt || smt_mitigations == SMT_MITIGATIONS_ON))
 			cpu_smt_disable(false);
 		 * present on host, enable the mitigation for UCODE_NEEDED as well.
 		 */
 		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
+		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM);
 		if (taa_nosmt || smt_mitigations == SMT_MITIGATIONS_ON)
 			cpu_smt_disable(false);
 	 */
 	if (verw_clear_cpu_buf_mitigation_selected) {
 		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
-		static_branch_disable(&cpu_buf_vm_clear);
+		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM);
 	} else {
-		static_branch_enable(&cpu_buf_vm_clear);
+		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM_MMIO);
 	}
 	/*
 static void __init rfds_apply_mitigation(void)
 {
-	if (rfds_mitigation == RFDS_MITIGATION_VERW)
+	if (rfds_mitigation == RFDS_MITIGATION_VERW) {
 		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
+		setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF_VM);
+	}
 }
 static __init int rfds_parse_cmdline(char *str)

arch/x86/kernel/cpu/scattered.c+1−0

@@ -53,6 +53,7 @@ static const struct cpuid_bit cpuid_bits[] = { · 52 unmodified lines
53	{ X86_FEATURE_PROC_FEEDBACK, CPUID_EDX, 11, 0x80000007, 0 },	53	{ X86_FEATURE_PROC_FEEDBACK, CPUID_EDX, 11, 0x80000007, 0 },
54	{ X86_FEATURE_AMD_FAST_CPPC, CPUID_EDX, 15, 0x80000007, 0 },	54	{ X86_FEATURE_AMD_FAST_CPPC, CPUID_EDX, 15, 0x80000007, 0 },
55	{ X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 },	55	{ X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 },
		56	{ X86_FEATURE_X2AVIC_EXT, CPUID_ECX, 6, 0x8000000a, 0 },
56	{ X86_FEATURE_COHERENCY_SFW_NO, CPUID_EBX, 31, 0x8000001f, 0 },	57	{ X86_FEATURE_COHERENCY_SFW_NO, CPUID_EBX, 31, 0x8000001f, 0 },
57	{ X86_FEATURE_SMBA, CPUID_EBX, 2, 0x80000020, 0 },	58	{ X86_FEATURE_SMBA, CPUID_EBX, 2, 0x80000020, 0 },
58	{ X86_FEATURE_BMEC, CPUID_EBX, 3, 0x80000020, 0 },	59	{ X86_FEATURE_BMEC, CPUID_EBX, 3, 0x80000020, 0 },

arch/x86/kvm/cpuid.c+1−0

@@ -1135,6 +1135,7 @@ void kvm_set_cpu_caps(void) · 1134 unmodified lines
1135	F(AMD_STIBP),	1135	F(AMD_STIBP),
1136	F(AMD_STIBP_ALWAYS_ON),	1136	F(AMD_STIBP_ALWAYS_ON),
1137	F(AMD_IBRS_SAME_MODE),	1137	F(AMD_IBRS_SAME_MODE),
		1138	PASSTHROUGH_F(EFER_LMSLE_MBZ),
1138	F(AMD_PSFD),	1139	F(AMD_PSFD),
1139	F(AMD_IBPB_RET),	1140	F(AMD_IBPB_RET),
1140	);	1141	);

arch/x86/kvm/emulate.c+223−96

@@ -81,9 +81,8 @@ · 80 unmodified lines
81	*/	81	*/
82		82
83	/* Operand sizes: 8-bit operands or specified/overridden size. */	83	/* Operand sizes: 8-bit operands or specified/overridden size. */
84	#define ByteOp (1<<0) /* 8-bit operands. */	84	#define ByteOp (1<<0) /* 8-bit operands. */
85	/* Destination operand type. */	85	#define DstShift 1 /* Destination operand type at bits 1-5 */
86	#define DstShift 1
87	#define ImplicitOps (OpImplicit << DstShift)	86	#define ImplicitOps (OpImplicit << DstShift)
88	#define DstReg (OpReg << DstShift)	87	#define DstReg (OpReg << DstShift)
89	#define DstMem (OpMem << DstShift)	88	#define DstMem (OpMem << DstShift)
@@ -95,8 +94,7 @@ · 5 unmodified lines
95	#define DstDX (OpDX << DstShift)	94	#define DstDX (OpDX << DstShift)
96	#define DstAccLo (OpAccLo << DstShift)	95	#define DstAccLo (OpAccLo << DstShift)
97	#define DstMask (OpMask << DstShift)	96	#define DstMask (OpMask << DstShift)
98	/* Source operand type. */	97	#define SrcShift 6 /* Source operand type at bits 6-10 */
99	#define SrcShift 6
100	#define SrcNone (OpNone << SrcShift)	98	#define SrcNone (OpNone << SrcShift)
101	#define SrcReg (OpReg << SrcShift)	99	#define SrcReg (OpReg << SrcShift)
102	#define SrcMem (OpMem << SrcShift)	100	#define SrcMem (OpMem << SrcShift)
@@ -119,10 +117,10 @@ · 16 unmodified lines
119	#define SrcAccHi (OpAccHi << SrcShift)	117	#define SrcAccHi (OpAccHi << SrcShift)
120	#define SrcMask (OpMask << SrcShift)	118	#define SrcMask (OpMask << SrcShift)
121	#define BitOp (1<<11)	119	#define BitOp (1<<11)
122	#define MemAbs (1<<12) /* Memory operand is absolute displacement */	120	#define MemAbs (1<<12) /* Memory operand is absolute displacement */
123	#define String (1<<13) /* String instruction (rep capable) */	121	#define String (1<<13) /* String instruction (rep capable) */
124	#define Stack (1<<14) /* Stack instruction (push/pop) */	122	#define Stack (1<<14) /* Stack instruction (push/pop) */
125	#define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */	123	#define GroupMask (7<<15) /* Group mechanisms, at bits 15-17 */
126	#define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */	124	#define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
127	#define GroupDual (2<<15) /* Alternate decoding of mod == 3 */	125	#define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
128	#define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */	126	#define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
@@ -131,11 +129,8 @@ · 2 unmodified lines
131	#define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */	129	#define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
132	#define ModeDual (7<<15) /* Different instruction for 32/64 bit */	130	#define ModeDual (7<<15) /* Different instruction for 32/64 bit */
133	#define Sse (1<<18) /* SSE Vector instruction */	131	#define Sse (1<<18) /* SSE Vector instruction */
134	/* Generic ModRM decode. */	132	#define ModRM (1<<19) /* Generic ModRM decode. */
135	#define ModRM (1<<19)	133	#define Mov (1<<20) /* Destination is only written; never read. */
136	/* Destination is only written; never read. */
137	#define Mov (1<<20)
138	/* Misc flags */
139	#define Prot (1<<21) /* instruction generates #UD if not in prot-mode */	134	#define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
140	#define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */	135	#define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
141	#define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */	136	#define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
@@ -143,11 +138,11 @@ · 1 unmodified line
143	#define Undefined (1<<25) /* No Such Instruction */	138	#define Undefined (1<<25) /* No Such Instruction */
144	#define Lock (1<<26) /* lock prefix is allowed for the instruction */	139	#define Lock (1<<26) /* lock prefix is allowed for the instruction */
145	#define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */	140	#define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
146	#define No64 (1<<28)	141	#define No64 (1<<28) /* Instruction generates #UD in 64-bit mode */
147	#define PageTable (1 << 29) /* instruction used to write page table */	142	#define PageTable (1 << 29) /* instruction used to write page table */
148	#define NotImpl (1 << 30) /* instruction is not implemented */	143	#define NotImpl (1 << 30) /* instruction is not implemented */
149	/* Source 2 operand type */	144	#define Avx ((u64)1 << 31) /* Instruction uses VEX prefix */
150	#define Src2Shift (31)	145	#define Src2Shift (32) /* Source 2 operand type at bits 32-36 */
151	#define Src2None (OpNone << Src2Shift)	146	#define Src2None (OpNone << Src2Shift)
152	#define Src2Mem (OpMem << Src2Shift)	147	#define Src2Mem (OpMem << Src2Shift)
153	#define Src2CL (OpCL << Src2Shift)	148	#define Src2CL (OpCL << Src2Shift)
@@ -161,12 +156,13 @@ · 7 unmodified lines
161	#define Src2FS (OpFS << Src2Shift)	156	#define Src2FS (OpFS << Src2Shift)
162	#define Src2GS (OpGS << Src2Shift)	157	#define Src2GS (OpGS << Src2Shift)
163	#define Src2Mask (OpMask << Src2Shift)	158	#define Src2Mask (OpMask << Src2Shift)
		159	/* free: 37-39 */
164	#define Mmx ((u64)1 << 40) /* MMX Vector instruction */	160	#define Mmx ((u64)1 << 40) /* MMX Vector instruction */
165	#define AlignMask ((u64)7 << 41)	161	#define AlignMask ((u64)3 << 41) /* Memory alignment requirement at bits 41-42 */
166	#define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */	162	#define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
167	#define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */	163	#define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
168	#define Avx ((u64)3 << 41) /* Advanced Vector Extensions */	164	#define Aligned16 ((u64)3 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
169	#define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */	165	/* free: 43-44 */
170	#define NoWrite ((u64)1 << 45) /* No writeback */	166	#define NoWrite ((u64)1 << 45) /* No writeback */
171	#define SrcWrite ((u64)1 << 46) /* Write back src operand */	167	#define SrcWrite ((u64)1 << 46) /* Write back src operand */
172	#define NoMod ((u64)1 << 47) /* Mod field is ignored */	168	#define NoMod ((u64)1 << 47) /* Mod field is ignored */
@@ -243,6 +239,13 @@ enum x86_transfer_type { · 70 unmodified lines
243	X86_TRANSFER_TASK_SWITCH,	239	X86_TRANSFER_TASK_SWITCH,
244	};	240	};
245		241
		242	enum rex_bits {
		243	REX_B = 1,
		244	REX_X = 2,
		245	REX_R = 4,
		246	REX_W = 8,
		247	};
		248
246	static void writeback_registers(struct x86_emulate_ctxt *ctxt)	249	static void writeback_registers(struct x86_emulate_ctxt *ctxt)
247	{	250	{
248	unsigned long dirty = ctxt->regs_dirty;	251	unsigned long dirty = ctxt->regs_dirty;
@@ -622,7 +625,6 @@ static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size) · 373 unmodified lines
622		625
623	switch (alignment) {	626	switch (alignment) {
624	case Unaligned:	627	case Unaligned:
625	case Avx:
626	return 1;	628	return 1;
627	case Aligned16:	629	case Aligned16:
628	return 16;	630	return 16;
@@ -924,7 +926,7 @@ static void decode_register(struct x86_emulate_ctxt ctxt, u8 modrm_reg, · 295 unmodified lines
924	int byteop)	926	int byteop)
925	{	927	{
926	void *p;	928	void *p;
927	int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;	929	int highbyte_regs = (ctxt->rex_prefix == REX_NONE) && byteop;
928		930
929	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)	931	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
930	p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;	932	p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
@@ -1030,6 +1032,7 @@ static void fetch_register_operand(struct operand *op) · 99 unmodified lines
1030	op->val = (u64 )op->addr.reg;	1032	op->val = (u64 )op->addr.reg;
1031	break;	1033	break;
1032	}	1034	}
		1035	op->orig_val = op->val;
1033	}	1036	}
1034		1037
1035	static int em_fninit(struct x86_emulate_ctxt *ctxt)	1038	static int em_fninit(struct x86_emulate_ctxt *ctxt)
@@ -1075,17 +1078,17 @@ static int em_fnstsw(struct x86_emulate_ctxt *ctxt) · 39 unmodified lines
1075	return X86EMUL_CONTINUE;	1078	return X86EMUL_CONTINUE;
1076	}	1079	}
1077		1080
1078	static void decode_register_operand(struct x86_emulate_ctxt *ctxt,	1081	static void __decode_register_operand(struct x86_emulate_ctxt *ctxt,
1079	struct operand *op)	1082	struct operand *op, int reg)
1080	{	1083	{
1081	unsigned int reg;	1084	if ((ctxt->d & Avx) && ctxt->op_bytes == 32) {
1082		1085	op->type = OP_YMM;
1083	if (ctxt->d & ModRM)	1086	op->bytes = 32;
1084	reg = ctxt->modrm_reg;	1087	op->addr.xmm = reg;
1085	else	1088	kvm_read_avx_reg(reg, &op->vec_val2);
1086	reg = (ctxt->b & 7) \| ((ctxt->rex_prefix & 1) << 3);	1089	return;
1087		1090	}
1088	if (ctxt->d & Sse) {	1091	if (ctxt->d & (Avx\|Sse)) {
1089	op->type = OP_XMM;	1092	op->type = OP_XMM;
1090	op->bytes = 16;	1093	op->bytes = 16;
1091	op->addr.xmm = reg;	1094	op->addr.xmm = reg;
@@ -1103,9 +1106,20 @@ static void decode_register_operand(struct x86_emulate_ctxt *ctxt, · 11 unmodified lines
1103	op->type = OP_REG;	1106	op->type = OP_REG;
1104	op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;	1107	op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1105	op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);	1108	op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1106
1107	fetch_register_operand(op);	1109	fetch_register_operand(op);
1108	op->orig_val = op->val;	1110	}
		1111
		1112	static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
		1113	struct operand *op)
		1114	{
		1115	unsigned int reg;
		1116
		1117	if (ctxt->d & ModRM)
		1118	reg = ctxt->modrm_reg;
		1119	else
		1120	reg = (ctxt->b & 7) \| (ctxt->rex_bits & REX_B ? 8 : 0);
		1121
		1122	__decode_register_operand(ctxt, op, reg);
1109	}	1123	}
1110		1124
1111	static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)	1125	static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
@@ -1122,9 +1136,9 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt, · 10 unmodified lines
1122	int rc = X86EMUL_CONTINUE;	1136	int rc = X86EMUL_CONTINUE;
1123	ulong modrm_ea = 0;	1137	ulong modrm_ea = 0;
1124		1138
1125	ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */	1139	ctxt->modrm_reg = (ctxt->rex_bits & REX_R ? 8 : 0);
1126	index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */	1140	index_reg = (ctxt->rex_bits & REX_X ? 8 : 0);
1127	base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */	1141	base_reg = (ctxt->rex_bits & REX_B ? 8 : 0);
1128		1142
1129	ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;	1143	ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1130	ctxt->modrm_reg \|= (ctxt->modrm & 0x38) >> 3;	1144	ctxt->modrm_reg \|= (ctxt->modrm & 0x38) >> 3;
@@ -1132,24 +1146,7 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt, · 1 unmodified line
1132	ctxt->modrm_seg = VCPU_SREG_DS;	1146	ctxt->modrm_seg = VCPU_SREG_DS;
1133		1147
1134	if (ctxt->modrm_mod == 3 \|\| (ctxt->d & NoMod)) {	1148	if (ctxt->modrm_mod == 3 \|\| (ctxt->d & NoMod)) {
1135	op->type = OP_REG;	1149	__decode_register_operand(ctxt, op, ctxt->modrm_rm);
1136	op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1137	op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1138	ctxt->d & ByteOp);
1139	if (ctxt->d & Sse) {
1140	op->type = OP_XMM;
1141	op->bytes = 16;
1142	op->addr.xmm = ctxt->modrm_rm;
1143	kvm_read_sse_reg(ctxt->modrm_rm, &op->vec_val);
1144	return rc;
1145	}
1146	if (ctxt->d & Mmx) {
1147	op->type = OP_MM;
1148	op->bytes = 8;
1149	op->addr.mm = ctxt->modrm_rm & 7;
1150	return rc;
1151	}
1152	fetch_register_operand(op);
1153	return rc;	1150	return rc;
1154	}	1151	}
1155		1152
@@ -1783,7 +1780,15 @@ static int writeback(struct x86_emulate_ctxt ctxt, struct operand op) · 627 unmodified lines
1783	op->data,	1780	op->data,
1784	op->bytes * op->count);	1781	op->bytes * op->count);
1785	case OP_XMM:	1782	case OP_XMM:
1786	kvm_write_sse_reg(op->addr.xmm, &op->vec_val);	1783	if (!(ctxt->d & Avx)) {
		1784	kvm_write_sse_reg(op->addr.xmm, &op->vec_val);
		1785	break;
		1786	}
		1787	/* full YMM write but with high bytes cleared */
		1788	memset(op->valptr + 16, 0, 16);
		1789	fallthrough;
		1790	case OP_YMM:
		1791	kvm_write_avx_reg(op->addr.xmm, &op->vec_val2);
1787	break;	1792	break;
1788	case OP_MM:	1793	case OP_MM:
1789	kvm_write_mmx_reg(op->addr.mm, &op->mm_val);	1794	kvm_write_mmx_reg(op->addr.mm, &op->mm_val);
@@ -2466,7 +2471,7 @@ static int em_sysexit(struct x86_emulate_ctxt *ctxt) · 676 unmodified lines
2466		2471
2467	setup_syscalls_segments(&cs, &ss);	2472	setup_syscalls_segments(&cs, &ss);
2468		2473
2469	if ((ctxt->rex_prefix & 0x8) != 0x0)	2474	if (ctxt->rex_bits & REX_W)
2470	usermode = X86EMUL_MODE_PROT64;	2475	usermode = X86EMUL_MODE_PROT64;
2471	else	2476	else
2472	usermode = X86EMUL_MODE_PROT32;	2477	usermode = X86EMUL_MODE_PROT32;
@@ -3958,6 +3963,8 @@ static int check_perm_out(struct x86_emulate_ctxt *ctxt) · 1485 unmodified lines
3958	I2bv(((_f) \| DstReg \| SrcMem \| ModRM) & ~Lock, _e), \	3963	I2bv(((_f) \| DstReg \| SrcMem \| ModRM) & ~Lock, _e), \
3959	I2bv(((_f) & ~Lock) \| DstAcc \| SrcImm, _e)	3964	I2bv(((_f) & ~Lock) \| DstAcc \| SrcImm, _e)
3960		3965
		3966	static const struct opcode ud = I(SrcNone, emulate_ud);
		3967
3961	static const struct opcode group7_rm0[] = {	3968	static const struct opcode group7_rm0[] = {
3962	N,	3969	N,
3963	I(SrcNone \| Priv \| EmulateOnUD, em_hypercall),	3970	I(SrcNone \| Priv \| EmulateOnUD, em_hypercall),
@@ -4114,7 +4121,7 @@ static const struct group_dual group15 = { { · 150 unmodified lines
4114	} };	4121	} };
4115		4122
4116	static const struct gprefix pfx_0f_6f_0f_7f = {	4123	static const struct gprefix pfx_0f_6f_0f_7f = {
4117	I(Mmx, em_mov), I(Sse \| Aligned, em_mov), N, I(Sse \| Unaligned, em_mov),	4124	I(Mmx, em_mov), I(Sse \| Avx \| Aligned, em_mov), N, I(Sse \| Avx \| Unaligned, em_mov),
4118	};	4125	};
4119		4126
4120	static const struct instr_dual instr_dual_0f_2b = {	4127	static const struct instr_dual instr_dual_0f_2b = {
@@ -4133,8 +4140,8 @@ static const struct gprefix pfx_0f_28_0f_29 = { · 12 unmodified lines
4133	I(Aligned, em_mov), I(Aligned, em_mov), N, N,	4140	I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4134	};	4141	};
4135		4142
4136	static const struct gprefix pfx_0f_e7 = {	4143	static const struct gprefix pfx_0f_e7_0f_38_2a = {
4137	N, I(Sse, em_mov), N, N,	4144	N, I(Sse \| Avx, em_mov), N, N,
4138	};	4145	};
4139		4146
4140	static const struct escape escape_d9 = { {	4147	static const struct escape escape_d9 = { {
@@ -4347,8 +4354,8 @@ static const struct opcode twobyte_table[256] = { · 206 unmodified lines
4347	DI(ImplicitOps \| Priv, invd), DI(ImplicitOps \| Priv, wbinvd), N, N,	4354	DI(ImplicitOps \| Priv, invd), DI(ImplicitOps \| Priv, wbinvd), N, N,
4348	N, D(ImplicitOps \| ModRM \| SrcMem \| NoAccess), N, N,	4355	N, D(ImplicitOps \| ModRM \| SrcMem \| NoAccess), N, N,
4349	/* 0x10 - 0x1F */	4356	/* 0x10 - 0x1F */
4350	GP(ModRM \| DstReg \| SrcMem \| Mov \| Sse, &pfx_0f_10_0f_11),	4357	GP(ModRM \| DstReg \| SrcMem \| Mov \| Sse \| Avx, &pfx_0f_10_0f_11),
4351	GP(ModRM \| DstMem \| SrcReg \| Mov \| Sse, &pfx_0f_10_0f_11),	4358	GP(ModRM \| DstMem \| SrcReg \| Mov \| Sse \| Avx, &pfx_0f_10_0f_11),
4352	N, N, N, N, N, N,	4359	N, N, N, N, N, N,
4353	D(ImplicitOps \| ModRM \| SrcMem \| NoAccess), /* 4 * prefetch + 4 * reserved NOP */	4360	D(ImplicitOps \| ModRM \| SrcMem \| NoAccess), /* 4 * prefetch + 4 * reserved NOP */
4354	D(ImplicitOps \| ModRM \| SrcMem \| NoAccess), N, N,	4361	D(ImplicitOps \| ModRM \| SrcMem \| NoAccess), N, N,
@@ -4364,9 +4371,9 @@ static const struct opcode twobyte_table[256] = { · 9 unmodified lines
4364	IIP(ModRM \| SrcMem \| Priv \| Op3264 \| NoMod, em_dr_write, dr_write,	4371	IIP(ModRM \| SrcMem \| Priv \| Op3264 \| NoMod, em_dr_write, dr_write,
4365	check_dr_write),	4372	check_dr_write),
4366	N, N, N, N,	4373	N, N, N, N,
4367	GP(ModRM \| DstReg \| SrcMem \| Mov \| Sse, &pfx_0f_28_0f_29),	4374	GP(ModRM \| DstReg \| SrcMem \| Mov \| Sse \| Avx, &pfx_0f_28_0f_29),
4368	GP(ModRM \| DstMem \| SrcReg \| Mov \| Sse, &pfx_0f_28_0f_29),	4375	GP(ModRM \| DstMem \| SrcReg \| Mov \| Sse \| Avx, &pfx_0f_28_0f_29),
4369	N, GP(ModRM \| DstMem \| SrcReg \| Mov \| Sse, &pfx_0f_2b),	4376	N, GP(ModRM \| DstMem \| SrcReg \| Mov \| Sse \| Avx, &pfx_0f_2b),
4370	N, N, N, N,	4377	N, N, N, N,
4371	/* 0x30 - 0x3F */	4378	/* 0x30 - 0x3F */
4372	II(ImplicitOps \| Priv, em_wrmsr, wrmsr),	4379	II(ImplicitOps \| Priv, em_wrmsr, wrmsr),
@@ -4431,7 +4438,7 @@ static const struct opcode twobyte_table[256] = { · 58 unmodified lines
4431	/* 0xD0 - 0xDF */	4438	/* 0xD0 - 0xDF */
4432	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,	4439	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4433	/* 0xE0 - 0xEF */	4440	/* 0xE0 - 0xEF */
4434	N, N, N, N, N, N, N, GP(SrcReg \| DstMem \| ModRM \| Mov, &pfx_0f_e7),	4441	N, N, N, N, N, N, N, GP(SrcReg \| DstMem \| ModRM \| Mov, &pfx_0f_e7_0f_38_2a),
4435	N, N, N, N, N, N, N, N,	4442	N, N, N, N, N, N, N, N,
4436	/* 0xF0 - 0xFF */	4443	/* 0xF0 - 0xFF */
4437	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N	4444	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
@@ -4458,8 +4465,13 @@ static const struct gprefix three_byte_0f_38_f1 = { · 20 unmodified lines
4458	* byte.	4465	* byte.
4459	*/	4466	*/
4460	static const struct opcode opcode_map_0f_38[256] = {	4467	static const struct opcode opcode_map_0f_38[256] = {
4461	/* 0x00 - 0x7f */	4468	/* 0x00 - 0x1f */
4462	X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),	4469	X16(N), X16(N),
		4470	/* 0x20 - 0x2f */
		4471	X8(N),
		4472	X2(N), GP(SrcReg \| DstMem \| ModRM \| Mov \| Aligned, &pfx_0f_e7_0f_38_2a), N, N, N, N, N,
		4473	/* 0x30 - 0x7f */
		4474	X16(N), X16(N), X16(N), X16(N), X16(N),
4463	/* 0x80 - 0xef */	4475	/* 0x80 - 0xef */
4464	X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),	4476	X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4465	/* 0xf0 - 0xf1 */	4477	/* 0xf0 - 0xf1 */
@@ -4618,14 +4630,12 @@ static int decode_operand(struct x86_emulate_ctxt ctxt, struct operand op, · 152 unmodified lines
4618	op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;	4630	op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4619	op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);	4631	op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4620	fetch_register_operand(op);	4632	fetch_register_operand(op);
4621	op->orig_val = op->val;
4622	break;	4633	break;
4623	case OpAccLo:	4634	case OpAccLo:
4624	op->type = OP_REG;	4635	op->type = OP_REG;
4625	op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;	4636	op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4626	op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);	4637	op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4627	fetch_register_operand(op);	4638	fetch_register_operand(op);
4628	op->orig_val = op->val;
4629	break;	4639	break;
4630	case OpAccHi:	4640	case OpAccHi:
4631	if (ctxt->d & ByteOp) {	4641	if (ctxt->d & ByteOp) {
@@ -4636,7 +4646,6 @@ static int decode_operand(struct x86_emulate_ctxt ctxt, struct operand op, · 4 unmodified lines
4636	op->bytes = ctxt->op_bytes;	4646	op->bytes = ctxt->op_bytes;
4637	op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);	4647	op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4638	fetch_register_operand(op);	4648	fetch_register_operand(op);
4639	op->orig_val = op->val;
4640	break;	4649	break;
4641	case OpDI:	4650	case OpDI:
4642	op->type = OP_MEM;	4651	op->type = OP_MEM;
@@ -4755,12 +4764,87 @@ done: · 112 unmodified lines
4755	return rc;	4764	return rc;
4756	}	4765	}
4757		4766
		4767	static int x86_decode_avx(struct x86_emulate_ctxt *ctxt,
		4768	u8 vex_1st, u8 vex_2nd, struct opcode *opcode)
		4769	{
		4770	u8 vex_3rd, map, pp, l, v;
		4771	int rc = X86EMUL_CONTINUE;
		4772
		4773	if (ctxt->rep_prefix \|\| ctxt->op_prefix \|\| ctxt->rex_prefix)
		4774	goto ud;
		4775
		4776	if (vex_1st == 0xc5) {
		4777	/* Expand RVVVVlpp to VEX3 format */
		4778	vex_3rd = vex_2nd & ~0x80; /* VVVVlpp from VEX2, w=0 */
		4779	vex_2nd = (vex_2nd & 0x80) \| 0x61; /* R from VEX2, X=1 B=1 mmmmm=00001 */
		4780	} else {
		4781	vex_3rd = insn_fetch(u8, ctxt);
		4782	}
		4783
		4784	/* vex_2nd = RXBmmmmm, vex_3rd = wVVVVlpp. Fix polarity */
		4785	vex_2nd ^= 0xE0; /* binary 11100000 */
		4786	vex_3rd ^= 0x78; /* binary 01111000 */
		4787
		4788	ctxt->rex_prefix = REX_PREFIX;
		4789	ctxt->rex_bits = (vex_2nd & 0xE0) >> 5; /* RXB */
		4790	ctxt->rex_bits \|= (vex_3rd & 0x80) >> 4; /* w */
		4791	if (ctxt->rex_bits && ctxt->mode != X86EMUL_MODE_PROT64)
		4792	goto ud;
		4793
		4794	map = vex_2nd & 0x1f;
		4795	v = (vex_3rd >> 3) & 0xf;
		4796	l = vex_3rd & 0x4;
		4797	pp = vex_3rd & 0x3;
		4798
		4799	ctxt->b = insn_fetch(u8, ctxt);
		4800	switch (map) {
		4801	case 1:
		4802	ctxt->opcode_len = 2;
		4803	*opcode = twobyte_table[ctxt->b];
		4804	break;
		4805	case 2:
		4806	ctxt->opcode_len = 3;
		4807	*opcode = opcode_map_0f_38[ctxt->b];
		4808	break;
		4809	case 3:
		4810	/* no 0f 3a instructions are supported yet */
		4811	return X86EMUL_UNHANDLEABLE;
		4812	default:
		4813	goto ud;
		4814	}
		4815
		4816	/*
		4817	* No three operand instructions are supported yet; those that
		4818	* are marked with the Avx flag reserve the VVVV flag.
		4819	*/
		4820	if (v)
		4821	goto ud;
		4822
		4823	if (l)
		4824	ctxt->op_bytes = 32;
		4825	else
		4826	ctxt->op_bytes = 16;
		4827
		4828	switch (pp) {
		4829	case 0: break;
		4830	case 1: ctxt->op_prefix = true; break;
		4831	case 2: ctxt->rep_prefix = 0xf3; break;
		4832	case 3: ctxt->rep_prefix = 0xf2; break;
		4833	}
		4834
		4835	done:
		4836	return rc;
		4837	ud:
		4838	*opcode = ud;
		4839	return rc;
		4840	}
		4841
4758	int x86_decode_insn(struct x86_emulate_ctxt ctxt, void insn, int insn_len, int emulation_type)	4842	int x86_decode_insn(struct x86_emulate_ctxt ctxt, void insn, int insn_len, int emulation_type)
4759	{	4843	{
4760	int rc = X86EMUL_CONTINUE;	4844	int rc = X86EMUL_CONTINUE;
4761	int mode = ctxt->mode;	4845	int mode = ctxt->mode;
4762	int def_op_bytes, def_ad_bytes, goffset, simd_prefix;	4846	int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4763	bool op_prefix = false;	4847	bool vex_prefix = false;
4764	bool has_seg_override = false;	4848	bool has_seg_override = false;
4765	struct opcode opcode;	4849	struct opcode opcode;
4766	u16 dummy;	4850	u16 dummy;
@@ -4812,7 +4896,7 @@ int x86_decode_insn(struct x86_emulate_ctxt ctxt, void insn, int insn_len, int · 45 unmodified lines
4812	for (;;) {	4896	for (;;) {
4813	switch (ctxt->b = insn_fetch(u8, ctxt)) {	4897	switch (ctxt->b = insn_fetch(u8, ctxt)) {
4814	case 0x66: /* operand-size override */	4898	case 0x66: /* operand-size override */
4815	op_prefix = true;	4899	ctxt->op_prefix = true;
4816	/* switch between 2/4 bytes */	4900	/* switch between 2/4 bytes */
4817	ctxt->op_bytes = def_op_bytes ^ 6;	4901	ctxt->op_bytes = def_op_bytes ^ 6;
4818	break;	4902	break;
@@ -4851,7 +4935,8 @@ int x86_decode_insn(struct x86_emulate_ctxt ctxt, void insn, int insn_len, int · 32 unmodified lines
4851	case 0x40 ... 0x4f: /* REX */	4935	case 0x40 ... 0x4f: /* REX */
4852	if (mode != X86EMUL_MODE_PROT64)	4936	if (mode != X86EMUL_MODE_PROT64)
4853	goto done_prefixes;	4937	goto done_prefixes;
4854	ctxt->rex_prefix = ctxt->b;	4938	ctxt->rex_prefix = REX_PREFIX;
		4939	ctxt->rex_bits = ctxt->b & 0xf;
4855	continue;	4940	continue;
4856	case 0xf0: /* LOCK */	4941	case 0xf0: /* LOCK */
4857	ctxt->lock_prefix = 1;	4942	ctxt->lock_prefix = 1;
@@ -4865,20 +4950,33 @@ int x86_decode_insn(struct x86_emulate_ctxt ctxt, void insn, int insn_len, int · 7 unmodified lines
4865	}	4950	}
4866		4951
4867	/* Any legacy prefix after a REX prefix nullifies its effect. */	4952	/* Any legacy prefix after a REX prefix nullifies its effect. */
4868		4953	ctxt->rex_prefix = REX_NONE;
4869	ctxt->rex_prefix = 0;	4954	ctxt->rex_bits = 0;
4870	}	4955	}
4871		4956
4872	done_prefixes:	4957	done_prefixes:
4873		4958
4874	/* REX prefix. */	4959	/* REX prefix. */
4875	if (ctxt->rex_prefix & 8)	4960	if (ctxt->rex_bits & REX_W)
4876	ctxt->op_bytes = 8; /* REX.W */	4961	ctxt->op_bytes = 8;
4877		4962
4878	/* Opcode byte(s). */	4963	/* Opcode byte(s). */
4879	opcode = opcode_table[ctxt->b];	4964	if (ctxt->b == 0xc4 \|\| ctxt->b == 0xc5) {
4880	/* Two-byte opcode? */	4965	/* VEX or LDS/LES */
4881	if (ctxt->b == 0x0f) {	4966	u8 vex_2nd = insn_fetch(u8, ctxt);
		4967	if (mode != X86EMUL_MODE_PROT64 && (vex_2nd & 0xc0) != 0xc0) {
		4968	opcode = opcode_table[ctxt->b];
		4969	ctxt->modrm = vex_2nd;
		4970	/* the Mod/RM byte has been fetched already! */
		4971	goto done_modrm;
		4972	}
		4973
		4974	vex_prefix = true;
		4975	rc = x86_decode_avx(ctxt, ctxt->b, vex_2nd, &opcode);
		4976	if (rc != X86EMUL_CONTINUE)
		4977	goto done;
		4978	} else if (ctxt->b == 0x0f) {
		4979	/* Two- or three-byte opcode */
4882	ctxt->opcode_len = 2;	4980	ctxt->opcode_len = 2;
4883	ctxt->b = insn_fetch(u8, ctxt);	4981	ctxt->b = insn_fetch(u8, ctxt);
4884	opcode = twobyte_table[ctxt->b];	4982	opcode = twobyte_table[ctxt->b];
@@ -4889,18 +4987,16 @@ done_prefixes: · 4 unmodified lines
4889	ctxt->b = insn_fetch(u8, ctxt);	4987	ctxt->b = insn_fetch(u8, ctxt);
4890	opcode = opcode_map_0f_38[ctxt->b];	4988	opcode = opcode_map_0f_38[ctxt->b];
4891	}	4989	}
		4990	} else {
		4991	/* Opcode byte(s). */
		4992	opcode = opcode_table[ctxt->b];
4892	}	4993	}
4893	ctxt->d = opcode.flags;
4894		4994
4895	if (ctxt->d & ModRM)	4995	if (opcode.flags & ModRM)
4896	ctxt->modrm = insn_fetch(u8, ctxt);	4996	ctxt->modrm = insn_fetch(u8, ctxt);
4897		4997
4898	/* vex-prefix instructions are not implemented */	4998	done_modrm:
4899	if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 \|\| ctxt->b == 0xc4) &&	4999	ctxt->d = opcode.flags;
4900	(mode == X86EMUL_MODE_PROT64 \|\| (ctxt->modrm & 0xc0) == 0xc0)) {
4901	ctxt->d = NotImpl;
4902	}
4903
4904	while (ctxt->d & GroupMask) {	5000	while (ctxt->d & GroupMask) {
4905	switch (ctxt->d & GroupMask) {	5001	switch (ctxt->d & GroupMask) {
4906	case Group:	5002	case Group:
@@ -4919,9 +5015,9 @@ done_prefixes: · 12 unmodified lines
4919	opcode = opcode.u.group[goffset];	5015	opcode = opcode.u.group[goffset];
4920	break;	5016	break;
4921	case Prefix:	5017	case Prefix:
4922	if (ctxt->rep_prefix && op_prefix)	5018	if (ctxt->rep_prefix && ctxt->op_prefix)
4923	return EMULATION_FAILED;	5019	return EMULATION_FAILED;
4924	simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;	5020	simd_prefix = ctxt->op_prefix ? 0x66 : ctxt->rep_prefix;
4925	switch (simd_prefix) {	5021	switch (simd_prefix) {
4926	case 0x00: opcode = opcode.u.gprefix->pfx_no; break;	5022	case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4927	case 0x66: opcode = opcode.u.gprefix->pfx_66; break;	5023	case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
@@ -4966,6 +5062,19 @@ done_prefixes: · 38 unmodified lines
4966	if (ctxt->d == 0)	5062	if (ctxt->d == 0)
4967	return EMULATION_FAILED;	5063	return EMULATION_FAILED;
4968		5064
		5065	if (unlikely(vex_prefix)) {
		5066	/*
		5067	* Only specifically marked instructions support VEX. Since many
		5068	* instructions support it but are not annotated, return not implemented
		5069	* rather than #UD.
		5070	*/
		5071	if (!(ctxt->d & Avx))
		5072	return EMULATION_FAILED;
		5073
		5074	if (!(ctxt->d & AlignMask))
		5075	ctxt->d \|= Unaligned;
		5076	}
		5077
4969	ctxt->execute = opcode.u.execute;	5078	ctxt->execute = opcode.u.execute;
4970		5079
4971	/*	5080	/*
@@ -5036,8 +5145,10 @@ done_prefixes: · 64 unmodified lines
5036	if ((ctxt->d & No16) && ctxt->op_bytes == 2)	5145	if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5037	ctxt->op_bytes = 4;	5146	ctxt->op_bytes = 4;
5038		5147
5039	if (ctxt->d & Sse)	5148	if (vex_prefix)
5040	ctxt->op_bytes = 16;	5149	;
		5150	else if (ctxt->d & Sse)
		5151	ctxt->op_bytes = 16, ctxt->d &= ~Avx;
5041	else if (ctxt->d & Mmx)	5152	else if (ctxt->d & Mmx)
5042	ctxt->op_bytes = 8;	5153	ctxt->op_bytes = 8;
5043	}	5154	}
@@ -5137,8 +5248,10 @@ void init_decode_cache(struct x86_emulate_ctxt *ctxt) · 93 unmodified lines
5137	{	5248	{
5138	/* Clear fields that are set conditionally but read without a guard. */	5249	/* Clear fields that are set conditionally but read without a guard. */
5139	ctxt->rip_relative = false;	5250	ctxt->rip_relative = false;
5140	ctxt->rex_prefix = 0;	5251	ctxt->rex_prefix = REX_NONE;
		5252	ctxt->rex_bits = 0;
5141	ctxt->lock_prefix = 0;	5253	ctxt->lock_prefix = 0;
		5254	ctxt->op_prefix = false;
5142	ctxt->rep_prefix = 0;	5255	ctxt->rep_prefix = 0;
5143	ctxt->regs_valid = 0;	5256	ctxt->regs_valid = 0;
5144	ctxt->regs_dirty = 0;	5257	ctxt->regs_dirty = 0;
@@ -5168,20 +5281,34 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt, bool check_intercepts) · 23 unmodified lines
5168	}	5281	}
5169		5282
5170	if (unlikely(ctxt->d &	5283	if (unlikely(ctxt->d &
5171	(No64\|Undefined\|Sse\|Mmx\|Intercept\|CheckPerm\|Priv\|Prot\|String))) {	5284	(No64\|Undefined\|Avx\|Sse\|Mmx\|Intercept\|CheckPerm\|Priv\|Prot\|String))) {
5172	if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) \|\|	5285	if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) \|\|
5173	(ctxt->d & Undefined)) {	5286	(ctxt->d & Undefined)) {
5174	rc = emulate_ud(ctxt);	5287	rc = emulate_ud(ctxt);
5175	goto done;	5288	goto done;
5176	}	5289	}
5177		5290
5178	if (((ctxt->d & (Sse\|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))	5291	if ((ctxt->d & (Avx\|Sse\|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM))) {
5179	\|\| ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5180	rc = emulate_ud(ctxt);	5292	rc = emulate_ud(ctxt);
5181	goto done;	5293	goto done;
5182	}	5294	}
5183		5295
5184	if ((ctxt->d & (Sse\|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {	5296	if (ctxt->d & Avx) {
		5297	u64 xcr = 0;
		5298	if (!(ops->get_cr(ctxt, 4) & X86_CR4_OSXSAVE)
		5299	\|\| ops->get_xcr(ctxt, 0, &xcr)
		5300	\|\| !(xcr & XFEATURE_MASK_YMM)) {
		5301	rc = emulate_ud(ctxt);
		5302	goto done;
		5303	}
		5304	} else if (ctxt->d & Sse) {
		5305	if (!(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR)) {
		5306	rc = emulate_ud(ctxt);
		5307	goto done;
		5308	}
		5309	}
		5310
		5311	if ((ctxt->d & (Avx\|Sse\|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5185	rc = emulate_nm(ctxt);	5312	rc = emulate_nm(ctxt);
5186	goto done;	5313	goto done;
5187	}	5314	}

arch/x86/kvm/fpu.h+66−0

@@ -15,6 +15,58 @@ typedef u32 __attribute__((vector_size(16))) sse128_t; · 14 unmodified lines
15	#define sse128_l3(x) ({ __sse128_u t; t.vec = x; t.as_u32[3]; })	15	#define sse128_l3(x) ({ __sse128_u t; t.vec = x; t.as_u32[3]; })
16	#define sse128(lo, hi) ({ __sse128_u t; t.as_u64[0] = lo; t.as_u64[1] = hi; t.vec; })	16	#define sse128(lo, hi) ({ __sse128_u t; t.as_u64[0] = lo; t.as_u64[1] = hi; t.vec; })
17		17
		18	typedef u32 __attribute__((vector_size(32))) avx256_t;
		19
		20	static inline void _kvm_read_avx_reg(int reg, avx256_t *data)
		21	{
		22	switch (reg) {
		23	case 0: asm("vmovdqa %%ymm0, %0" : "=m"(*data)); break;
		24	case 1: asm("vmovdqa %%ymm1, %0" : "=m"(*data)); break;
		25	case 2: asm("vmovdqa %%ymm2, %0" : "=m"(*data)); break;
		26	case 3: asm("vmovdqa %%ymm3, %0" : "=m"(*data)); break;
		27	case 4: asm("vmovdqa %%ymm4, %0" : "=m"(*data)); break;
		28	case 5: asm("vmovdqa %%ymm5, %0" : "=m"(*data)); break;
		29	case 6: asm("vmovdqa %%ymm6, %0" : "=m"(*data)); break;
		30	case 7: asm("vmovdqa %%ymm7, %0" : "=m"(*data)); break;
		31	#ifdef CONFIG_X86_64
		32	case 8: asm("vmovdqa %%ymm8, %0" : "=m"(*data)); break;
		33	case 9: asm("vmovdqa %%ymm9, %0" : "=m"(*data)); break;
		34	case 10: asm("vmovdqa %%ymm10, %0" : "=m"(*data)); break;
		35	case 11: asm("vmovdqa %%ymm11, %0" : "=m"(*data)); break;
		36	case 12: asm("vmovdqa %%ymm12, %0" : "=m"(*data)); break;
		37	case 13: asm("vmovdqa %%ymm13, %0" : "=m"(*data)); break;
		38	case 14: asm("vmovdqa %%ymm14, %0" : "=m"(*data)); break;
		39	case 15: asm("vmovdqa %%ymm15, %0" : "=m"(*data)); break;
		40	#endif
		41	default: BUG();
		42	}
		43	}
		44
		45	static inline void _kvm_write_avx_reg(int reg, const avx256_t *data)
		46	{
		47	switch (reg) {
		48	case 0: asm("vmovdqa %0, %%ymm0" : : "m"(*data)); break;
		49	case 1: asm("vmovdqa %0, %%ymm1" : : "m"(*data)); break;
		50	case 2: asm("vmovdqa %0, %%ymm2" : : "m"(*data)); break;
		51	case 3: asm("vmovdqa %0, %%ymm3" : : "m"(*data)); break;
		52	case 4: asm("vmovdqa %0, %%ymm4" : : "m"(*data)); break;
		53	case 5: asm("vmovdqa %0, %%ymm5" : : "m"(*data)); break;
		54	case 6: asm("vmovdqa %0, %%ymm6" : : "m"(*data)); break;
		55	case 7: asm("vmovdqa %0, %%ymm7" : : "m"(*data)); break;
		56	#ifdef CONFIG_X86_64
		57	case 8: asm("vmovdqa %0, %%ymm8" : : "m"(*data)); break;
		58	case 9: asm("vmovdqa %0, %%ymm9" : : "m"(*data)); break;
		59	case 10: asm("vmovdqa %0, %%ymm10" : : "m"(*data)); break;
		60	case 11: asm("vmovdqa %0, %%ymm11" : : "m"(*data)); break;
		61	case 12: asm("vmovdqa %0, %%ymm12" : : "m"(*data)); break;
		62	case 13: asm("vmovdqa %0, %%ymm13" : : "m"(*data)); break;
		63	case 14: asm("vmovdqa %0, %%ymm14" : : "m"(*data)); break;
		64	case 15: asm("vmovdqa %0, %%ymm15" : : "m"(*data)); break;
		65	#endif
		66	default: BUG();
		67	}
		68	}
		69
18	static inline void _kvm_read_sse_reg(int reg, sse128_t *data)	70	static inline void _kvm_read_sse_reg(int reg, sse128_t *data)
19	{	71	{
20	switch (reg) {	72	switch (reg) {
@@ -109,6 +161,20 @@ static inline void kvm_fpu_put(void) · 88 unmodified lines
109	fpregs_unlock();	161	fpregs_unlock();
110	}	162	}
111		163
		164	static inline void kvm_read_avx_reg(int reg, avx256_t *data)
		165	{
		166	kvm_fpu_get();
		167	_kvm_read_avx_reg(reg, data);
		168	kvm_fpu_put();
		169	}
		170
		171	static inline void kvm_write_avx_reg(int reg, const avx256_t *data)
		172	{
		173	kvm_fpu_get();
		174	_kvm_write_avx_reg(reg, data);
		175	kvm_fpu_put();
		176	}
		177
112	static inline void kvm_read_sse_reg(int reg, sse128_t *data)	178	static inline void kvm_read_sse_reg(int reg, sse128_t *data)
113	{	179	{
114	kvm_fpu_get();	180	kvm_fpu_get();

arch/x86/kvm/hyperv.c+1−1

@@ -1568,7 +1568,7 @@ static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) · 1567 unmodified lines
1568	* only, there can be valuable data in the rest which needs	1568	* only, there can be valuable data in the rest which needs
1569	* to be preserved e.g. on migration.	1569	* to be preserved e.g. on migration.
1570	*/	1570	*/
1571	if (__put_user(0, (u32 __user *)addr))	1571	if (put_user(0, (u32 __user *)addr))
1572	return 1;	1572	return 1;
1573	hv_vcpu->hv_vapic = data;	1573	hv_vcpu->hv_vapic = data;
1574	kvm_vcpu_mark_page_dirty(vcpu, gfn);	1574	kvm_vcpu_mark_page_dirty(vcpu, gfn);

arch/x86/kvm/kvm_emulate.h+16−4

 	bool (*is_smm)(struct x86_emulate_ctxt *ctxt);
 	int (*leave_smm)(struct x86_emulate_ctxt *ctxt);
 	void (*triple_fault)(struct x86_emulate_ctxt *ctxt);
+	int (*get_xcr)(struct x86_emulate_ctxt *ctxt, u32 index, u64 *xcr);
 	int (*set_xcr)(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr);
 	gva_t (*get_untagged_addr)(struct x86_emulate_ctxt *ctxt, gva_t addr,
 /* Type, address-of, and value of an instruction's operand. */
 struct operand {
-	enum { OP_REG, OP_MEM, OP_MEM_STR, OP_IMM, OP_XMM, OP_MM, OP_NONE } type;
+	enum { OP_REG, OP_MEM, OP_MEM_STR, OP_IMM, OP_XMM, OP_YMM, OP_MM, OP_NONE } type;
 	unsigned int bytes;
 	unsigned int count;
 	union {
 	union {
 		unsigned long val;
 		u64 val64;
-		char valptr[sizeof(sse128_t)];
+		char valptr[sizeof(avx256_t)];
 		sse128_t vec_val;
+		avx256_t vec_val2;
 		u64 mm_val;
 		void *data;
-	};
+	} __aligned(32);
 };
 #define X86_MAX_INSTRUCTION_LENGTH	15
 #define NR_EMULATOR_GPRS	8
 #endif
+/*
+ * Distinguish between no prefix, REX, or in the future REX2.
+ */
+enum rex_type {
+	REX_NONE,
+	REX_PREFIX,
+};
 struct x86_emulate_ctxt {
 	void *vcpu;
 	const struct x86_emulate_ops *ops;
 	u8 opcode_len;
 	u8 b;
 	u8 intercept;
+	bool op_prefix;
 	u8 op_bytes;
 	u8 ad_bytes;
 	union {
 	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
 	bool rip_relative;
-	u8 rex_prefix;
+	enum rex_type rex_prefix;
+	u8 rex_bits;
 	u8 lock_prefix;
 	u8 rep_prefix;
 	/* bitmaps of registers in _regs[] that can be read */

arch/x86/kvm/lapic.c+31−13

@@ -2126,23 +2126,41 @@ static bool set_target_expiration(struct kvm_lapic *apic, u32 count_reg) · 2125 unmodified lines
2126		2126
2127	static void advance_periodic_target_expiration(struct kvm_lapic *apic)	2127	static void advance_periodic_target_expiration(struct kvm_lapic *apic)
2128	{	2128	{
		2129	struct kvm_timer *ktimer = &apic->lapic_timer;
2129	ktime_t now = ktime_get();	2130	ktime_t now = ktime_get();
2130	u64 tscl = rdtsc();	2131	u64 tscl = rdtsc();
2131	ktime_t delta;	2132	ktime_t delta;
2132		2133
2133	/*	2134	/*
2134	* Synchronize both deadlines to the same time source or	2135	* Use kernel time as the time source for both the hrtimer deadline and
2135	* differences in the periods (caused by differences in the	2136	* TSC-based deadline so that they stay synchronized. Computing each
2136	* underlying clocks or numerical approximation errors) will	2137	* deadline independently will cause the two deadlines to drift apart
2137	* cause the two to drift apart over time as the errors	2138	* over time as differences in the periods accumulate, e.g. due to
2138	* accumulate.	2139	* differences in the underlying clocks or numerical approximation errors.
2139	*/	2140	*/
2140	apic->lapic_timer.target_expiration =	2141	ktimer->target_expiration = ktime_add_ns(ktimer->target_expiration,
2141	ktime_add_ns(apic->lapic_timer.target_expiration,	2142	ktimer->period);
2142	apic->lapic_timer.period);	2143
2143	delta = ktime_sub(apic->lapic_timer.target_expiration, now);	2144	/*
2144	apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +	2145	* If the new expiration is in the past, e.g. because userspace stopped
2145	nsec_to_cycles(apic->vcpu, delta);	2146	* running the VM for an extended duration, then force the expiration
		2147	* to "now" and don't try to play catch-up with the missed events. KVM
		2148	* will only deliver a single interrupt regardless of how many events
		2149	* are pending, i.e. restarting the timer with an expiration in the
		2150	* past will do nothing more than waste host cycles, and can even lead
		2151	* to a hard lockup in extreme cases.
		2152	*/
		2153	if (ktime_before(ktimer->target_expiration, now))
		2154	ktimer->target_expiration = now;
		2155
		2156	/*
		2157	* Note, ensuring the expiration isn't in the past also prevents delta
		2158	* from going negative, which could cause the TSC deadline to become
		2159	* excessively large due to it an unsigned value.
		2160	*/
		2161	delta = ktime_sub(ktimer->target_expiration, now);
		2162	ktimer->tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
		2163	nsec_to_cycles(apic->vcpu, delta);
2146	}	2164	}
2147		2165
2148	static void start_sw_period(struct kvm_lapic *apic)	2166	static void start_sw_period(struct kvm_lapic *apic)
@@ -2970,9 +2988,9 @@ static enum hrtimer_restart apic_timer_fn(struct hrtimer *data) · 821 unmodified lines
2970		2988
2971	apic_timer_expired(apic, true);	2989	apic_timer_expired(apic, true);
2972		2990
2973	if (lapic_is_periodic(apic)) {	2991	if (lapic_is_periodic(apic) && !WARN_ON_ONCE(!apic->lapic_timer.period)) {
2974	advance_periodic_target_expiration(apic);	2992	advance_periodic_target_expiration(apic);
2975	hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);	2993	hrtimer_set_expires(&ktimer->timer, ktimer->target_expiration);
2976	return HRTIMER_RESTART;	2994	return HRTIMER_RESTART;
2977	} else	2995	} else
2978	return HRTIMER_NORESTART;	2996	return HRTIMER_NORESTART;

arch/x86/kvm/mmu.h+1−4

 	return -(u32)fault & errcode;
 }
-bool kvm_mmu_may_ignore_guest_pat(struct kvm *kvm);
 int kvm_mmu_post_init_vm(struct kvm *kvm);
 void kvm_mmu_pre_destroy_vm(struct kvm *kvm);
 #define tdp_mmu_enabled false
 #endif
-bool kvm_tdp_mmu_gpa_is_mapped(struct kvm_vcpu *vcpu, u64 gpa);
+int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn);
-int kvm_tdp_map_page(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code, u8 *level);
 static inline bool kvm_memslots_have_rmaps(struct kvm *kvm)
 {

arch/x86/kvm/mmu/mmu.c+88−6

@@ -4859,7 +4859,7 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, · 4858 unmodified lines
4859	*/	4859	*/
4860	BUILD_BUG_ON(lower_32_bits(PFERR_SYNTHETIC_MASK));	4860	BUILD_BUG_ON(lower_32_bits(PFERR_SYNTHETIC_MASK));
4861		4861
4862	vcpu->arch.l1tf_flush_l1d = true;	4862	kvm_request_l1tf_flush_l1d();
4863	if (!flags) {	4863	if (!flags) {
4864	trace_kvm_page_fault(vcpu, fault_address, error_code);	4864	trace_kvm_page_fault(vcpu, fault_address, error_code);
4865		4865
@@ -4924,7 +4924,8 @@ int kvm_tdp_page_fault(struct kvm_vcpu vcpu, struct kvm_page_fault fault) · 58 unmodified lines
4924	return direct_page_fault(vcpu, fault);	4924	return direct_page_fault(vcpu, fault);
4925	}	4925	}
4926		4926
4927	int kvm_tdp_map_page(struct kvm_vcpu vcpu, gpa_t gpa, u64 error_code, u8 level)	4927	static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa,
		4928	u64 error_code, u8 *level)
4928	{	4929	{
4929	int r;	4930	int r;
4930		4931
@@ -4966,7 +4967,6 @@ int kvm_tdp_map_page(struct kvm_vcpu vcpu, gpa_t gpa, u64 error_code, u8 level · 35 unmodified lines
4966	return -EIO;	4967	return -EIO;
4967	}	4968	}
4968	}	4969	}
4969	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_tdp_map_page);
4970		4970
4971	long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,	4971	long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
4972	struct kvm_pre_fault_memory *range)	4972	struct kvm_pre_fault_memory *range)
@@ -5002,7 +5002,7 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, · 29 unmodified lines
5002	* Shadow paging uses GVA for kvm page fault, so restrict to	5002	* Shadow paging uses GVA for kvm page fault, so restrict to
5003	* two-dimensional paging.	5003	* two-dimensional paging.
5004	*/	5004	*/
5005	r = kvm_tdp_map_page(vcpu, range->gpa \| direct_bits, error_code, &level);	5005	r = kvm_tdp_page_prefault(vcpu, range->gpa \| direct_bits, error_code, &level);
5006	if (r < 0)	5006	if (r < 0)
5007	return r;	5007	return r;
5008		5008
@@ -5014,6 +5014,86 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, · 5 unmodified lines
5014	return min(range->size, end - range->gpa);	5014	return min(range->size, end - range->gpa);
5015	}	5015	}
5016		5016
		5017	#ifdef CONFIG_KVM_GUEST_MEMFD
		5018	static void kvm_assert_gmem_invalidate_lock_held(struct kvm_memory_slot *slot)
		5019	{
		5020	#ifdef CONFIG_PROVE_LOCKING
		5021	if (WARN_ON_ONCE(!kvm_slot_has_gmem(slot)) \|\|
		5022	WARN_ON_ONCE(!slot->gmem.file) \|\|
		5023	WARN_ON_ONCE(!file_count(slot->gmem.file)))
		5024	return;
		5025
		5026	lockdep_assert_held(&file_inode(slot->gmem.file)->i_mapping->invalidate_lock);
		5027	#endif
		5028	}
		5029
		5030	int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
		5031	{
		5032	struct kvm_page_fault fault = {
		5033	.addr = gfn_to_gpa(gfn),
		5034	.error_code = PFERR_GUEST_FINAL_MASK \| PFERR_PRIVATE_ACCESS,
		5035	.prefetch = true,
		5036	.is_tdp = true,
		5037	.nx_huge_page_workaround_enabled = is_nx_huge_page_enabled(vcpu->kvm),
		5038
		5039	.max_level = PG_LEVEL_4K,
		5040	.req_level = PG_LEVEL_4K,
		5041	.goal_level = PG_LEVEL_4K,
		5042	.is_private = true,
		5043
		5044	.gfn = gfn,
		5045	.slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn),
		5046	.pfn = pfn,
		5047	.map_writable = true,
		5048	};
		5049	struct kvm *kvm = vcpu->kvm;
		5050	int r;
		5051
		5052	lockdep_assert_held(&kvm->slots_lock);
		5053
		5054	/*
		5055	* Mapping a pre-determined private pfn is intended only for use when
		5056	* populating a guest_memfd instance. Assert that the slot is backed
		5057	* by guest_memfd and that the gmem instance's invalidate_lock is held.
		5058	*/
		5059	kvm_assert_gmem_invalidate_lock_held(fault.slot);
		5060
		5061	if (KVM_BUG_ON(!tdp_mmu_enabled, kvm))
		5062	return -EIO;
		5063
		5064	if (kvm_gfn_is_write_tracked(kvm, fault.slot, fault.gfn))
		5065	return -EPERM;
		5066
		5067	r = kvm_mmu_reload(vcpu);
		5068	if (r)
		5069	return r;
		5070
		5071	r = mmu_topup_memory_caches(vcpu, false);
		5072	if (r)
		5073	return r;
		5074
		5075	do {
		5076	if (signal_pending(current))
		5077	return -EINTR;
		5078
		5079	if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu))
		5080	return -EIO;
		5081
		5082	cond_resched();
		5083
		5084	guard(read_lock)(&kvm->mmu_lock);
		5085
		5086	r = kvm_tdp_mmu_map(vcpu, &fault);
		5087	} while (r == RET_PF_RETRY);
		5088
		5089	if (r != RET_PF_FIXED)
		5090	return -EIO;
		5091
		5092	return 0;
		5093	}
		5094	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_tdp_mmu_map_private_pfn);
		5095	#endif
		5096
5017	static void nonpaging_init_context(struct kvm_mmu *context)	5097	static void nonpaging_init_context(struct kvm_mmu *context)
5018	{	5098	{
5019	context->page_fault = nonpaging_page_fault;	5099	context->page_fault = nonpaging_page_fault;
@@ -5997,7 +6077,6 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu) · 977 unmodified lines
5997	out:	6077	out:
5998	return r;	6078	return r;
5999	}	6079	}
6000	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_load);
6001		6080
6002	void kvm_mmu_unload(struct kvm_vcpu *vcpu)	6081	void kvm_mmu_unload(struct kvm_vcpu *vcpu)
6003	{	6082	{
@@ -6863,6 +6942,7 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) · 859 unmodified lines
6863		6942
6864	write_unlock(&kvm->mmu_lock);	6943	write_unlock(&kvm->mmu_lock);
6865	}	6944	}
		6945	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_zap_gfn_range);
6866		6946
6867	static bool slot_rmap_write_protect(struct kvm *kvm,	6947	static bool slot_rmap_write_protect(struct kvm *kvm,
6868	struct kvm_rmap_head *rmap_head,	6948	struct kvm_rmap_head *rmap_head,
@@ -7204,7 +7284,6 @@ restart: · 335 unmodified lines
7204		7284
7205	return need_tlb_flush;	7285	return need_tlb_flush;
7206	}	7286	}
7207	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_zap_gfn_range);
7208		7287
7209	static void kvm_rmap_zap_collapsible_sptes(struct kvm *kvm,	7288	static void kvm_rmap_zap_collapsible_sptes(struct kvm *kvm,
7210	const struct kvm_memory_slot *slot)	7289	const struct kvm_memory_slot *slot)
@@ -7364,6 +7443,9 @@ void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen) · 153 unmodified lines
7364	{	7443	{
7365	WARN_ON_ONCE(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);	7444	WARN_ON_ONCE(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
7366		7445
		7446	if (!enable_mmio_caching)
		7447	return;
		7448
7367	gen &= MMIO_SPTE_GEN_MASK;	7449	gen &= MMIO_SPTE_GEN_MASK;
7368		7450
7369	/*	7451	/*

arch/x86/kvm/mmu/mmu_internal.h+0−10

@@ -39,16 +39,6 @@ · 38 unmodified lines
39	#define INVALID_PAE_ROOT 0	39	#define INVALID_PAE_ROOT 0
40	#define IS_VALID_PAE_ROOT(x) (!!(x))	40	#define IS_VALID_PAE_ROOT(x) (!!(x))
41		41
42	static inline hpa_t kvm_mmu_get_dummy_root(void)
43	{
44	return my_zero_pfn(0) << PAGE_SHIFT;
45	}
46
47	static inline bool kvm_mmu_is_dummy_root(hpa_t shadow_page)
48	{
49	return is_zero_pfn(shadow_page >> PAGE_SHIFT);
50	}
51
52	typedef u64 __rcu *tdp_ptep_t;	42	typedef u64 __rcu *tdp_ptep_t;
53		43
54	struct kvm_mmu_page {	44	struct kvm_mmu_page {

arch/x86/kvm/mmu/paging_tmpl.h+1−1

@@ -402,7 +402,7 @@ retry_walk: · 401 unmodified lines
402	goto error;	402	goto error;
403		403
404	ptep_user = (pt_element_t __user )((void )host_addr + offset);	404	ptep_user = (pt_element_t __user )((void )host_addr + offset);
405	if (unlikely(__get_user(pte, ptep_user)))	405	if (unlikely(get_user(pte, ptep_user)))
406	goto error;	406	goto error;
407	walker->ptep_user[walker->level - 1] = ptep_user;	407	walker->ptep_user[walker->level - 1] = ptep_user;
408		408

arch/x86/kvm/mmu/spte.c+1−1

@@ -292,7 +292,7 @@ bool make_spte(struct kvm_vcpu vcpu, struct kvm_mmu_page sp, · 291 unmodified lines
292	mark_page_dirty_in_slot(vcpu->kvm, slot, gfn);	292	mark_page_dirty_in_slot(vcpu->kvm, slot, gfn);
293	}	293	}
294		294
295	if (static_branch_unlikely(&cpu_buf_vm_clear) &&	295	if (cpu_feature_enabled(X86_FEATURE_CLEAR_CPU_BUF_VM_MMIO) &&
296	!kvm_vcpu_can_access_host_mmio(vcpu) &&	296	!kvm_vcpu_can_access_host_mmio(vcpu) &&
297	kvm_is_mmio_pfn(pfn, &is_host_mmio))	297	kvm_is_mmio_pfn(pfn, &is_host_mmio))
298	kvm_track_host_mmio_mapping(vcpu);	298	kvm_track_host_mmio_mapping(vcpu);

arch/x86/kvm/mmu/spte.h+10−0

@@ -246,6 +246,16 @@ static inline int spte_index(u64 *sptep) · 245 unmodified lines
246	*/	246	*/
247	extern u64 __read_mostly shadow_nonpresent_or_rsvd_lower_gfn_mask;	247	extern u64 __read_mostly shadow_nonpresent_or_rsvd_lower_gfn_mask;
248		248
		249	static inline hpa_t kvm_mmu_get_dummy_root(void)
		250	{
		251	return my_zero_pfn(0) << PAGE_SHIFT;
		252	}
		253
		254	static inline bool kvm_mmu_is_dummy_root(hpa_t shadow_page)
		255	{
		256	return is_zero_pfn(shadow_page >> PAGE_SHIFT);
		257	}
		258
249	static inline struct kvm_mmu_page *to_shadow_page(hpa_t shadow_page)	259	static inline struct kvm_mmu_page *to_shadow_page(hpa_t shadow_page)
250	{	260	{
251	struct page *page = pfn_to_page((shadow_page) >> PAGE_SHIFT);	261	struct page *page = pfn_to_page((shadow_page) >> PAGE_SHIFT);

arch/x86/kvm/mmu/tdp_mmu.c+10−40

@@ -362,9 +362,6 @@ static void tdp_mmu_unlink_sp(struct kvm kvm, struct kvm_mmu_page sp) · 361 unmodified lines
362	static void remove_external_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte,	362	static void remove_external_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte,
363	int level)	363	int level)
364	{	364	{
365	kvm_pfn_t old_pfn = spte_to_pfn(old_spte);
366	int ret;
367
368	/*	365	/*
369	* External (TDX) SPTEs are limited to PG_LEVEL_4K, and external	366	* External (TDX) SPTEs are limited to PG_LEVEL_4K, and external
370	* PTs are removed in a special order, involving free_external_spt().	367	* PTs are removed in a special order, involving free_external_spt().
@@ -377,9 +374,8 @@ static void remove_external_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte, · 6 unmodified lines
377		374
378	/* Zapping leaf spte is allowed only when write lock is held. */	375	/* Zapping leaf spte is allowed only when write lock is held. */
379	lockdep_assert_held_write(&kvm->mmu_lock);	376	lockdep_assert_held_write(&kvm->mmu_lock);
380	/* Because write lock is held, operation should success. */	377
381	ret = kvm_x86_call(remove_external_spte)(kvm, gfn, level, old_pfn);	378	kvm_x86_call(remove_external_spte)(kvm, gfn, level, old_spte);
382	KVM_BUG_ON(ret, kvm);
383	}	379	}
384		380
385	/**	381	/**
@@ -519,7 +515,6 @@ static int __must_check set_external_spte_present(struct kvm *kvm, tdp_ptep_t sp · 133 unmodified lines
519	bool was_present = is_shadow_present_pte(old_spte);	515	bool was_present = is_shadow_present_pte(old_spte);
520	bool is_present = is_shadow_present_pte(new_spte);	516	bool is_present = is_shadow_present_pte(new_spte);
521	bool is_leaf = is_present && is_last_spte(new_spte, level);	517	bool is_leaf = is_present && is_last_spte(new_spte, level);
522	kvm_pfn_t new_pfn = spte_to_pfn(new_spte);
523	int ret = 0;	518	int ret = 0;
524		519
525	KVM_BUG_ON(was_present, kvm);	520	KVM_BUG_ON(was_present, kvm);
@@ -538,7 +533,7 @@ static int __must_check set_external_spte_present(struct kvm *kvm, tdp_ptep_t sp · 12 unmodified lines
538	* external page table, or leaf.	533	* external page table, or leaf.
539	*/	534	*/
540	if (is_leaf) {	535	if (is_leaf) {
541	ret = kvm_x86_call(set_external_spte)(kvm, gfn, level, new_pfn);	536	ret = kvm_x86_call(set_external_spte)(kvm, gfn, level, new_spte);
542	} else {	537	} else {
543	void *external_spt = get_external_spt(gfn, new_spte, level);	538	void *external_spt = get_external_spt(gfn, new_spte, level);
544		539
@@ -1273,6 +1268,8 @@ int kvm_tdp_mmu_map(struct kvm_vcpu vcpu, struct kvm_page_fault fault) · 728 unmodified lines
1273	struct kvm_mmu_page *sp;	1268	struct kvm_mmu_page *sp;
1274	int ret = RET_PF_RETRY;	1269	int ret = RET_PF_RETRY;
1275		1270
		1271	KVM_MMU_WARN_ON(!root \|\| root->role.invalid);
		1272
1276	kvm_mmu_hugepage_adjust(vcpu, fault);	1273	kvm_mmu_hugepage_adjust(vcpu, fault);
1277		1274
1278	trace_kvm_mmu_spte_requested(fault);	1275	trace_kvm_mmu_spte_requested(fault);
@@ -1939,13 +1936,16 @@ bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm, · 660 unmodified lines
1939	*	1936	*
1940	* Must be called between kvm_tdp_mmu_walk_lockless_{begin,end}.	1937	* Must be called between kvm_tdp_mmu_walk_lockless_{begin,end}.
1941	*/	1938	*/
1942	static int __kvm_tdp_mmu_get_walk(struct kvm_vcpu vcpu, u64 addr, u64 sptes,	1939	int kvm_tdp_mmu_get_walk(struct kvm_vcpu vcpu, u64 addr, u64 sptes,
1943	struct kvm_mmu_page *root)	1940	int *root_level)
1944	{	1941	{
		1942	struct kvm_mmu_page *root = root_to_sp(vcpu->arch.mmu->root.hpa);
1945	struct tdp_iter iter;	1943	struct tdp_iter iter;
1946	gfn_t gfn = addr >> PAGE_SHIFT;	1944	gfn_t gfn = addr >> PAGE_SHIFT;
1947	int leaf = -1;	1945	int leaf = -1;
1948		1946
		1947	*root_level = vcpu->arch.mmu->root_role.level;
		1948
1949	for_each_tdp_pte(iter, vcpu->kvm, root, gfn, gfn + 1) {	1949	for_each_tdp_pte(iter, vcpu->kvm, root, gfn, gfn + 1) {
1950	leaf = iter.level;	1950	leaf = iter.level;
1951	sptes[leaf] = iter.old_spte;	1951	sptes[leaf] = iter.old_spte;
@@ -1954,36 +1954,6 @@ static int __kvm_tdp_mmu_get_walk(struct kvm_vcpu vcpu, u64 addr, u64 sptes, · 2 unmodified lines
1954	return leaf;	1954	return leaf;
1955	}	1955	}
1956		1956
1957	int kvm_tdp_mmu_get_walk(struct kvm_vcpu vcpu, u64 addr, u64 sptes,
1958	int *root_level)
1959	{
1960	struct kvm_mmu_page *root = root_to_sp(vcpu->arch.mmu->root.hpa);
1961	*root_level = vcpu->arch.mmu->root_role.level;
1962
1963	return __kvm_tdp_mmu_get_walk(vcpu, addr, sptes, root);
1964	}
1965
1966	bool kvm_tdp_mmu_gpa_is_mapped(struct kvm_vcpu *vcpu, u64 gpa)
1967	{
1968	struct kvm *kvm = vcpu->kvm;
1969	bool is_direct = kvm_is_addr_direct(kvm, gpa);
1970	hpa_t root = is_direct ? vcpu->arch.mmu->root.hpa :
1971	vcpu->arch.mmu->mirror_root_hpa;
1972	u64 sptes[PT64_ROOT_MAX_LEVEL + 1], spte;
1973	int leaf;
1974
1975	lockdep_assert_held(&kvm->mmu_lock);
1976	rcu_read_lock();
1977	leaf = __kvm_tdp_mmu_get_walk(vcpu, gpa, sptes, root_to_sp(root));
1978	rcu_read_unlock();
1979	if (leaf < 0)
1980	return false;
1981
1982	spte = sptes[leaf];
1983	return is_shadow_present_pte(spte) && is_last_spte(spte, leaf);
1984	}
1985	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_tdp_mmu_gpa_is_mapped);
1986
1987	/*	1957	/*
1988	* Returns the last level spte pointer of the shadow page walk for the given	1958	* Returns the last level spte pointer of the shadow page walk for the given
1989	* gpa, and sets *spte to the spte value. This spte may be non-preset. If no	1959	* gpa, and sets *spte to the spte value. This spte may be non-preset. If no

arch/x86/kvm/svm/avic.c+70−16

@@ -106,7 +106,7 @@ static u32 next_vm_id = 0; · 105 unmodified lines
106	static bool next_vm_id_wrapped = 0;	106	static bool next_vm_id_wrapped = 0;
107	static DEFINE_SPINLOCK(svm_vm_data_hash_lock);	107	static DEFINE_SPINLOCK(svm_vm_data_hash_lock);
108	static bool x2avic_enabled;	108	static bool x2avic_enabled;
109		109	static u32 x2avic_max_physical_id;
110		110
111	static void avic_set_x2apic_msr_interception(struct vcpu_svm *svm,	111	static void avic_set_x2apic_msr_interception(struct vcpu_svm *svm,
112	bool intercept)	112	bool intercept)
@@ -158,12 +158,40 @@ static void avic_set_x2apic_msr_interception(struct vcpu_svm *svm, · 45 unmodified lines
158	svm->x2avic_msrs_intercepted = intercept;	158	svm->x2avic_msrs_intercepted = intercept;
159	}	159	}
160		160
		161	static u32 __avic_get_max_physical_id(struct kvm kvm, struct kvm_vcpu vcpu)
		162	{
		163	u32 arch_max;
		164
		165	/*
		166	* Return the largest size (x2APIC) when querying without a vCPU, e.g.
		167	* to allocate the per-VM table..
		168	*/
		169	if (x2avic_enabled && (!vcpu \|\| apic_x2apic_mode(vcpu->arch.apic)))
		170	arch_max = x2avic_max_physical_id;
		171	else
		172	arch_max = AVIC_MAX_PHYSICAL_ID;
		173
		174	/*
		175	* Despite its name, KVM_CAP_MAX_VCPU_ID represents the maximum APIC ID
		176	* plus one, so the max possible APIC ID is one less than that.
		177	*/
		178	return min(kvm->arch.max_vcpu_ids - 1, arch_max);
		179	}
		180
		181	static u32 avic_get_max_physical_id(struct kvm_vcpu *vcpu)
		182	{
		183	return __avic_get_max_physical_id(vcpu->kvm, vcpu);
		184	}
		185
161	static void avic_activate_vmcb(struct vcpu_svm *svm)	186	static void avic_activate_vmcb(struct vcpu_svm *svm)
162	{	187	{
163	struct vmcb *vmcb = svm->vmcb01.ptr;	188	struct vmcb *vmcb = svm->vmcb01.ptr;
		189	struct kvm_vcpu *vcpu = &svm->vcpu;
164		190
165	vmcb->control.int_ctl &= ~(AVIC_ENABLE_MASK \| X2APIC_MODE_MASK);	191	vmcb->control.int_ctl &= ~(AVIC_ENABLE_MASK \| X2APIC_MODE_MASK);
		192
166	vmcb->control.avic_physical_id &= ~AVIC_PHYSICAL_MAX_INDEX_MASK;	193	vmcb->control.avic_physical_id &= ~AVIC_PHYSICAL_MAX_INDEX_MASK;
		194	vmcb->control.avic_physical_id \|= avic_get_max_physical_id(vcpu);
167		195
168	vmcb->control.int_ctl \|= AVIC_ENABLE_MASK;	196	vmcb->control.int_ctl \|= AVIC_ENABLE_MASK;
169		197
@@ -176,7 +204,7 @@ static void avic_activate_vmcb(struct vcpu_svm *svm) · 6 unmodified lines
176	*/	204	*/
177	if (x2avic_enabled && apic_x2apic_mode(svm->vcpu.arch.apic)) {	205	if (x2avic_enabled && apic_x2apic_mode(svm->vcpu.arch.apic)) {
178	vmcb->control.int_ctl \|= X2APIC_MODE_MASK;	206	vmcb->control.int_ctl \|= X2APIC_MODE_MASK;
179	vmcb->control.avic_physical_id \|= X2AVIC_MAX_PHYSICAL_ID;	207
180	/* Disabling MSR intercept for x2APIC registers */	208	/* Disabling MSR intercept for x2APIC registers */
181	avic_set_x2apic_msr_interception(svm, false);	209	avic_set_x2apic_msr_interception(svm, false);
182	} else {	210	} else {
@@ -186,8 +214,6 @@ static void avic_activate_vmcb(struct vcpu_svm *svm) · 3 unmodified lines
186	*/	214	*/
187	kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, &svm->vcpu);	215	kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, &svm->vcpu);
188		216
189	/* For xAVIC and hybrid-xAVIC modes */
190	vmcb->control.avic_physical_id \|= AVIC_MAX_PHYSICAL_ID;
191	/* Enabling MSR intercept for x2APIC registers */	217	/* Enabling MSR intercept for x2APIC registers */
192	avic_set_x2apic_msr_interception(svm, true);	218	avic_set_x2apic_msr_interception(svm, true);
193	}	219	}
@@ -247,6 +273,30 @@ static int avic_ga_log_notifier(u32 ga_tag) · 53 unmodified lines
247	return 0;	273	return 0;
248	}	274	}
249		275
		276	static int avic_get_physical_id_table_order(struct kvm *kvm)
		277	{
		278	/* Provision for the maximum physical ID supported in x2avic mode */
		279	return get_order((__avic_get_max_physical_id(kvm, NULL) + 1) * sizeof(u64));
		280	}
		281
		282	int avic_alloc_physical_id_table(struct kvm *kvm)
		283	{
		284	struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
		285
		286	if (!irqchip_in_kernel(kvm) \|\| !enable_apicv)
		287	return 0;
		288
		289	if (kvm_svm->avic_physical_id_table)
		290	return 0;
		291
		292	kvm_svm->avic_physical_id_table = (void *)__get_free_pages(GFP_KERNEL_ACCOUNT \| __GFP_ZERO,
		293	avic_get_physical_id_table_order(kvm));
		294	if (!kvm_svm->avic_physical_id_table)
		295	return -ENOMEM;
		296
		297	return 0;
		298	}
		299
250	void avic_vm_destroy(struct kvm *kvm)	300	void avic_vm_destroy(struct kvm *kvm)
251	{	301	{
252	unsigned long flags;	302	unsigned long flags;
@@ -256,7 +306,8 @@ void avic_vm_destroy(struct kvm *kvm) · 3 unmodified lines
256	return;	306	return;
257		307
258	free_page((unsigned long)kvm_svm->avic_logical_id_table);	308	free_page((unsigned long)kvm_svm->avic_logical_id_table);
259	free_page((unsigned long)kvm_svm->avic_physical_id_table);	309	free_pages((unsigned long)kvm_svm->avic_physical_id_table,
		310	avic_get_physical_id_table_order(kvm));
260		311
261	spin_lock_irqsave(&svm_vm_data_hash_lock, flags);	312	spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
262	hash_del(&kvm_svm->hnode);	313	hash_del(&kvm_svm->hnode);
@@ -274,10 +325,6 @@ int avic_vm_init(struct kvm *kvm) · 11 unmodified lines
274	if (!enable_apicv)	325	if (!enable_apicv)
275	return 0;	326	return 0;
276		327
277	kvm_svm->avic_physical_id_table = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
278	if (!kvm_svm->avic_physical_id_table)
279	goto free_avic;
280
281	kvm_svm->avic_logical_id_table = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);	328	kvm_svm->avic_logical_id_table = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
282	if (!kvm_svm->avic_logical_id_table)	329	if (!kvm_svm->avic_logical_id_table)
283	goto free_avic;	330	goto free_avic;
@@ -342,7 +389,7 @@ static int avic_init_backing_page(struct kvm_vcpu *vcpu) · 58 unmodified lines
342	* fully initialized AVIC.	389	* fully initialized AVIC.
343	*/	390	*/
344	if ((!x2avic_enabled && id > AVIC_MAX_PHYSICAL_ID) \|\|	391	if ((!x2avic_enabled && id > AVIC_MAX_PHYSICAL_ID) \|\|
345	(id > X2AVIC_MAX_PHYSICAL_ID)) {	392	(id > x2avic_max_physical_id)) {
346	kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_PHYSICAL_ID_TOO_BIG);	393	kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_PHYSICAL_ID_TOO_BIG);
347	vcpu->arch.apic->apicv_active = false;	394	vcpu->arch.apic->apicv_active = false;
348	return 0;	395	return 0;
@@ -562,7 +609,7 @@ int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu) · 213 unmodified lines
562	u32 icrh = svm->vmcb->control.exit_info_1 >> 32;	609	u32 icrh = svm->vmcb->control.exit_info_1 >> 32;
563	u32 icrl = svm->vmcb->control.exit_info_1;	610	u32 icrl = svm->vmcb->control.exit_info_1;
564	u32 id = svm->vmcb->control.exit_info_2 >> 32;	611	u32 id = svm->vmcb->control.exit_info_2 >> 32;
565	u32 index = svm->vmcb->control.exit_info_2 & 0x1FF;	612	u32 index = svm->vmcb->control.exit_info_2 & AVIC_PHYSICAL_MAX_INDEX_MASK;
566	struct kvm_lapic *apic = vcpu->arch.apic;	613	struct kvm_lapic *apic = vcpu->arch.apic;
567		614
568	trace_kvm_avic_incomplete_ipi(vcpu->vcpu_id, icrh, icrl, id, index);	615	trace_kvm_avic_incomplete_ipi(vcpu->vcpu_id, icrh, icrl, id, index);
@@ -962,7 +1009,8 @@ static void __avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu, · 393 unmodified lines
962	if (WARN_ON(h_physical_id & ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK))	1009	if (WARN_ON(h_physical_id & ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK))
963	return;	1010	return;
964		1011
965	if (WARN_ON_ONCE(vcpu->vcpu_id * sizeof(entry) >= PAGE_SIZE))	1012	if (WARN_ON_ONCE(vcpu->vcpu_id * sizeof(entry) >=
		1013	PAGE_SIZE << avic_get_physical_id_table_order(vcpu->kvm)))
966	return;	1014	return;
967		1015
968	/*	1016	/*
@@ -1024,7 +1072,8 @@ static void __avic_vcpu_put(struct kvm_vcpu *vcpu, enum avic_vcpu_action action) · 55 unmodified lines
1024		1072
1025	lockdep_assert_preemption_disabled();	1073	lockdep_assert_preemption_disabled();
1026		1074
1027	if (WARN_ON_ONCE(vcpu->vcpu_id * sizeof(entry) >= PAGE_SIZE))	1075	if (WARN_ON_ONCE(vcpu->vcpu_id * sizeof(entry) >=
		1076	PAGE_SIZE << avic_get_physical_id_table_order(vcpu->kvm)))
1028	return;	1077	return;
1029		1078
1030	/*	1079	/*
@@ -1226,10 +1275,15 @@ bool __init avic_hardware_setup(void) · 195 unmodified lines
1226		1275
1227	/* AVIC is a prerequisite for x2AVIC. */	1276	/* AVIC is a prerequisite for x2AVIC. */
1228	x2avic_enabled = boot_cpu_has(X86_FEATURE_X2AVIC);	1277	x2avic_enabled = boot_cpu_has(X86_FEATURE_X2AVIC);
1229	if (x2avic_enabled)	1278	if (x2avic_enabled) {
1230	pr_info("x2AVIC enabled\n");	1279	if (cpu_feature_enabled(X86_FEATURE_X2AVIC_EXT))
1231	else	1280	x2avic_max_physical_id = X2AVIC_4K_MAX_PHYSICAL_ID;
		1281	else
		1282	x2avic_max_physical_id = X2AVIC_MAX_PHYSICAL_ID;
		1283	pr_info("x2AVIC enabled (max %u vCPUs)\n", x2avic_max_physical_id + 1);
		1284	} else {
1232	svm_x86_ops.allow_apicv_in_x2apic_without_x2apic_virtualization = true;	1285	svm_x86_ops.allow_apicv_in_x2apic_without_x2apic_virtualization = true;
		1286	}
1233		1287
1234	/*	1288	/*
1235	* Disable IPI virtualization for AMD Family 17h CPUs (Zen1 and Zen2)	1289	* Disable IPI virtualization for AMD Family 17h CPUs (Zen1 and Zen2)

arch/x86/kvm/svm/nested.c+2−10

 	struct kvm_vcpu *vcpu = &svm->vcpu;
 	nested_vmcb02_compute_g_pat(svm);
+	vmcb_mark_dirty(vmcb02, VMCB_NPT);
 	/* Load the nested guest state */
 	if (svm->nested.vmcb12_gpa != svm->nested.last_vmcb12_gpa) {
 	vmcb02->control.nested_ctl = vmcb01->control.nested_ctl;
 	vmcb02->control.iopm_base_pa = vmcb01->control.iopm_base_pa;
 	vmcb02->control.msrpm_base_pa = vmcb01->control.msrpm_base_pa;
+	vmcb_mark_dirty(vmcb02, VMCB_PERM_MAP);
 	/*
 	 * Stash vmcb02's counter if the guest hasn't moved past the guilty
 	case SVM_EXIT_IOIO:
 		vmexit = nested_svm_intercept_ioio(svm);
 		break;
-	case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
-		if (vmcb12_is_intercept(&svm->nested.ctl, exit_code))
-			vmexit = NESTED_EXIT_DONE;
-		break;
-	}
-	case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
-		if (vmcb12_is_intercept(&svm->nested.ctl, exit_code))
-			vmexit = NESTED_EXIT_DONE;
-		break;
-	}
 	case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
 		/*
 		 * Host-intercepted exceptions have been checked already in

arch/x86/kvm/svm/sev.c+29−18

@@ -65,20 +65,24 @@ module_param_named(ciphertext_hiding_asids, nr_ciphertext_hiding_asids, uint, 04 · 64 unmodified lines
65	#define AP_RESET_HOLD_NAE_EVENT 1	65	#define AP_RESET_HOLD_NAE_EVENT 1
66	#define AP_RESET_HOLD_MSR_PROTO 2	66	#define AP_RESET_HOLD_MSR_PROTO 2
67		67
68	/* As defined by SEV-SNP Firmware ABI, under "Guest Policy". */	68	/*
69	#define SNP_POLICY_MASK_API_MINOR GENMASK_ULL(7, 0)	69	* SEV-SNP policy bits that can be supported by KVM. These include policy bits
70	#define SNP_POLICY_MASK_API_MAJOR GENMASK_ULL(15, 8)	70	* that have implementation support within KVM or policy bits that do not
71	#define SNP_POLICY_MASK_SMT BIT_ULL(16)	71	* require implementation support within KVM to enforce the policy.
72	#define SNP_POLICY_MASK_RSVD_MBO BIT_ULL(17)	72	*/
73	#define SNP_POLICY_MASK_DEBUG BIT_ULL(19)	73	#define KVM_SNP_POLICY_MASK_VALID (SNP_POLICY_MASK_API_MINOR \| \
74	#define SNP_POLICY_MASK_SINGLE_SOCKET BIT_ULL(20)	74	SNP_POLICY_MASK_API_MAJOR \| \
75		75	SNP_POLICY_MASK_SMT \| \
76	#define SNP_POLICY_MASK_VALID (SNP_POLICY_MASK_API_MINOR \| \	76	SNP_POLICY_MASK_RSVD_MBO \| \
77	SNP_POLICY_MASK_API_MAJOR \| \	77	SNP_POLICY_MASK_DEBUG \| \
78	SNP_POLICY_MASK_SMT \| \	78	SNP_POLICY_MASK_SINGLE_SOCKET \| \
79	SNP_POLICY_MASK_RSVD_MBO \| \	79	SNP_POLICY_MASK_CXL_ALLOW \| \
80	SNP_POLICY_MASK_DEBUG \| \	80	SNP_POLICY_MASK_MEM_AES_256_XTS \| \
81	SNP_POLICY_MASK_SINGLE_SOCKET)	81	SNP_POLICY_MASK_RAPL_DIS \| \
		82	SNP_POLICY_MASK_CIPHERTEXT_HIDING_DRAM \| \
		83	SNP_POLICY_MASK_PAGE_SWAP_DISABLE)
		84
		85	static u64 snp_supported_policy_bits __ro_after_init;
82		86
83	#define INITIAL_VMSA_GPA 0xFFFFFFFFF000	87	#define INITIAL_VMSA_GPA 0xFFFFFFFFF000
84		88
@@ -2143,6 +2147,10 @@ int sev_dev_get_attr(u32 group, u64 attr, u64 *val) · 2058 unmodified lines
2143	*val = sev_supported_vmsa_features;	2147	*val = sev_supported_vmsa_features;
2144	return 0;	2148	return 0;
2145		2149
		2150	case KVM_X86_SNP_POLICY_BITS:
		2151	*val = snp_supported_policy_bits;
		2152	return 0;
		2153
2146	default:	2154	default:
2147	return -ENXIO;	2155	return -ENXIO;
2148	}	2156	}
@@ -2207,7 +2215,7 @@ static int snp_launch_start(struct kvm kvm, struct kvm_sev_cmd argp) · 58 unmodified lines
2207	if (params.flags)	2215	if (params.flags)
2208	return -EINVAL;	2216	return -EINVAL;
2209		2217
2210	if (params.policy & ~SNP_POLICY_MASK_VALID)	2218	if (params.policy & ~snp_supported_policy_bits)
2211	return -EINVAL;	2219	return -EINVAL;
2212		2220
2213	/* Check for policy bits that must be set */	2221	/* Check for policy bits that must be set */
@@ -3100,8 +3108,11 @@ out: · 886 unmodified lines
3100	else if (sev_snp_supported)	3108	else if (sev_snp_supported)
3101	sev_snp_supported = is_sev_snp_initialized();	3109	sev_snp_supported = is_sev_snp_initialized();
3102		3110
3103	if (sev_snp_supported)	3111	if (sev_snp_supported) {
		3112	snp_supported_policy_bits = sev_get_snp_policy_bits() &
		3113	KVM_SNP_POLICY_MASK_VALID;
3104	nr_ciphertext_hiding_asids = init_args.max_snp_asid;	3114	nr_ciphertext_hiding_asids = init_args.max_snp_asid;
		3115	}
3105		3116
3106	/*	3117	/*
3107	* If ciphertext hiding is enabled, the joint SEV-ES/SEV-SNP	3118	* If ciphertext hiding is enabled, the joint SEV-ES/SEV-SNP
@@ -5085,10 +5096,10 @@ struct vmcb_save_area sev_decrypt_vmsa(struct kvm_vcpu vcpu) · 1977 unmodified lines
5085		5096
5086	/* Check if the SEV policy allows debugging */	5097	/* Check if the SEV policy allows debugging */
5087	if (sev_snp_guest(vcpu->kvm)) {	5098	if (sev_snp_guest(vcpu->kvm)) {
5088	if (!(sev->policy & SNP_POLICY_DEBUG))	5099	if (!(sev->policy & SNP_POLICY_MASK_DEBUG))
5089	return NULL;	5100	return NULL;
5090	} else {	5101	} else {
5091	if (sev->policy & SEV_POLICY_NODBG)	5102	if (sev->policy & SEV_POLICY_MASK_NODBG)
5092	return NULL;	5103	return NULL;
5093	}	5104	}
5094		5105

arch/x86/kvm/svm/svm.c+60−43

 }
 static int __svm_skip_emulated_instruction(struct kvm_vcpu *vcpu,
+					   int emul_type,
 					   bool commit_side_effects)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 		if (unlikely(!commit_side_effects))
 			old_rflags = svm->vmcb->save.rflags;
-		if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
+		if (!kvm_emulate_instruction(vcpu, emul_type))
 			return 0;
 		if (unlikely(!commit_side_effects))
 static int svm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
 {
-	return __svm_skip_emulated_instruction(vcpu, true);
+	return __svm_skip_emulated_instruction(vcpu, EMULTYPE_SKIP, true);
 }
-static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu)
+static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu, u8 vector)
 {
+	const int emul_type = EMULTYPE_SKIP | EMULTYPE_SKIP_SOFT_INT |
+			      EMULTYPE_SET_SOFT_INT_VECTOR(vector);
 	unsigned long rip, old_rip = kvm_rip_read(vcpu);
 	struct vcpu_svm *svm = to_svm(vcpu);
 	 * in use, the skip must not commit any side effects such as clearing
 	 * the interrupt shadow or RFLAGS.RF.
 	 */
-	if (!__svm_skip_emulated_instruction(vcpu, !nrips))
+	if (!__svm_skip_emulated_instruction(vcpu, emul_type, !nrips))
 		return -EIO;
 	rip = kvm_rip_read(vcpu);
 	kvm_deliver_exception_payload(vcpu, ex);
 	if (kvm_exception_is_soft(ex->vector) &&
-	    svm_update_soft_interrupt_rip(vcpu))
+	    svm_update_soft_interrupt_rip(vcpu, ex->vector))
 		return;
 	svm->vmcb->control.event_inj = ex->vector
 	svm->vmcb = target_vmcb->ptr;
 }
+static int svm_vcpu_precreate(struct kvm *kvm)
+{
+	return avic_alloc_physical_id_table(kvm);
+}
 static int svm_vcpu_create(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm;
 static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code)
 {
-	vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%llx\n", exit_code);
 	dump_vmcb(vcpu);
-	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+	kvm_prepare_unexpected_reason_exit(vcpu, exit_code);
-	vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
-	vcpu->run->internal.ndata = 2;
-	vcpu->run->internal.data[0] = exit_code;
-	vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
 	return 0;
 }
 static void svm_inject_irq(struct kvm_vcpu *vcpu, bool reinjected)
 {
+	struct kvm_queued_interrupt *intr = &vcpu->arch.interrupt;
 	struct vcpu_svm *svm = to_svm(vcpu);
 	u32 type;
-	if (vcpu->arch.interrupt.soft) {
+	if (intr->soft) {
-		if (svm_update_soft_interrupt_rip(vcpu))
+		if (svm_update_soft_interrupt_rip(vcpu, intr->nr))
 			return;
 		type = SVM_EVTINJ_TYPE_SOFT;
 		type = SVM_EVTINJ_TYPE_INTR;
 	}
-	trace_kvm_inj_virq(vcpu->arch.interrupt.nr,
+	trace_kvm_inj_virq(intr->nr, intr->soft, reinjected);
-			   vcpu->arch.interrupt.soft, reinjected);
 	++vcpu->stat.irq_injections;
-	svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
+	svm->vmcb->control.event_inj = intr->nr | SVM_EVTINJ_VALID | type;
-				       SVM_EVTINJ_VALID | type;
 }
 void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode,
 		svm_set_dr6(vcpu, DR6_ACTIVE_LOW);
 	clgi();
-	kvm_load_guest_xsave_state(vcpu);
 	/*
 	 * Hardware only context switches DEBUGCTL if LBR virtualization is
 	    vcpu->arch.host_debugctl != svm->vmcb->save.dbgctl)
 		update_debugctlmsr(vcpu->arch.host_debugctl);
-	kvm_load_host_xsave_state(vcpu);
 	stgi();
 	/* Any pending NMI will happen here */
 	vcpu->arch.regs_avail &= ~SVM_REGS_LAZY_LOAD_SET;
-	/*
-	 * We need to handle MC intercepts here before the vcpu has a chance to
-	 * change the physical cpu
-	 */
-	if (unlikely(svm->vmcb->control.exit_code ==
-		     SVM_EXIT_EXCP_BASE + MC_VECTOR))
-		svm_handle_mce(vcpu);
 	trace_kvm_exit(vcpu, KVM_ISA_SVM);
 	svm_complete_interrupts(vcpu);
 	case SVM_EXIT_WRITE_CR0: {
 		unsigned long cr0, val;
-		if (info->intercept == x86_intercept_cr_write)
+		/*
+		 * Adjust the exit code accordingly if a CR other than CR0 is
+		 * being written, and skip straight to the common handling as
+		 * only CR0 has an additional selective intercept.
+		 */
+		if (info->intercept == x86_intercept_cr_write && info->modrm_reg) {
 			icpt_info.exit_code += info->modrm_reg;
-		if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
-		    info->intercept == x86_intercept_clts)
 			break;
+		}
-		if (!(vmcb12_is_intercept(&svm->nested.ctl,
+		/*
-					INTERCEPT_SELECTIVE_CR0)))
+		 * Convert the exit_code to SVM_EXIT_CR0_SEL_WRITE if a
+		 * selective CR0 intercept is triggered (the common logic will
+		 * treat the selective intercept as being enabled).  Note, the
+		 * unconditional intercept has higher priority, i.e. this is
+		 * only relevant if *only* the selective intercept is enabled.
+		 */
+		if (vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_CR0_WRITE) ||
+		    !(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0)))
 			break;
-		cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
+		/* CLTS never triggers INTERCEPT_SELECTIVE_CR0 */
-		val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
+		if (info->intercept == x86_intercept_clts)
+			break;
+		/* LMSW always triggers INTERCEPT_SELECTIVE_CR0 */
 		if (info->intercept == x86_intercept_lmsw) {
-			cr0 &= 0xfUL;
+			icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
-			val &= 0xfUL;
+			break;
-			/* lmsw can't clear PE - catch this here */
-			if (cr0 & X86_CR0_PE)
-				val |= X86_CR0_PE;
 		}
+		/*
+		 * MOV-to-CR0 only triggers INTERCEPT_SELECTIVE_CR0 if any bit
+		 * other than SVM_CR0_SELECTIVE_MASK is changed.
+		 */
+		cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
+		val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
 		if (cr0 ^ val)
 			icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
 		break;
 	}
 	case SVM_EXIT_READ_DR0:
 static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu)
 {
-	if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_INTR)
+	switch (to_svm(vcpu)->vmcb->control.exit_code) {
+	case SVM_EXIT_EXCP_BASE + MC_VECTOR:
+		svm_handle_mce(vcpu);
+		break;
+	case SVM_EXIT_INTR:
 		vcpu->arch.at_instruction_boundary = true;
+		break;
+	default:
+		break;
+	}
 }
 static void svm_setup_mce(struct kvm_vcpu *vcpu)
 	.emergency_disable_virtualization_cpu = svm_emergency_disable_virtualization_cpu,
 	.has_emulated_msr = svm_has_emulated_msr,
+	.vcpu_precreate = svm_vcpu_precreate,
 	.vcpu_create = svm_vcpu_create,
 	.vcpu_free = svm_vcpu_free,
 	.vcpu_reset = svm_vcpu_reset,
 	if (nested) {
 		pr_info("Nested Virtualization enabled\n");
-		kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
+		kvm_enable_efer_bits(EFER_SVME);
+		if (!boot_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ))
+			kvm_enable_efer_bits(EFER_LMSLE);
 		r = nested_svm_init_msrpm_merge_offsets();
 		if (r)

arch/x86/kvm/svm/svm.h+1−3

 	cpumask_var_t have_run_cpus; /* CPUs that have done VMRUN for this VM. */
 };
-#define SEV_POLICY_NODBG	BIT_ULL(0)
-#define SNP_POLICY_DEBUG	BIT_ULL(19)
 struct kvm_svm {
 	struct kvm kvm;
 bool __init avic_hardware_setup(void);
 void avic_hardware_unsetup(void);
+int avic_alloc_physical_id_table(struct kvm *kvm);
 void avic_vm_destroy(struct kvm *kvm);
 int avic_vm_init(struct kvm *kvm);
 void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb);

arch/x86/kvm/svm/vmenter.S+41−12

 	 * there must not be any returns or indirect branches between this code
 	 * and vmentry.
 	 */
-	movl SVM_spec_ctrl(%_ASM_DI), %eax
+#ifdef CONFIG_X86_64
-	cmp PER_CPU_VAR(x86_spec_ctrl_current), %eax
+	mov SVM_spec_ctrl(%rdi), %rdx
+	cmp PER_CPU_VAR(x86_spec_ctrl_current), %rdx
+	je 801b
+	movl %edx, %eax
+	shr $32, %rdx
+#else
+	mov SVM_spec_ctrl(%edi), %eax
+	mov PER_CPU_VAR(x86_spec_ctrl_current), %ecx
+	xor %eax, %ecx
+	mov SVM_spec_ctrl + 4(%edi), %edx
+	mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %esi
+	xor %edx, %esi
+	or %esi, %ecx
 	je 801b
+#endif
 	mov $MSR_IA32_SPEC_CTRL, %ecx
-	xor %edx, %edx
 	wrmsr
 	jmp 801b
 .endm
 	jnz 998f
 	rdmsr
 	movl %eax, SVM_spec_ctrl(%_ASM_DI)
+	movl %edx, SVM_spec_ctrl + 4(%_ASM_DI)
 998:
 	/* Now restore the host value of the MSR if different from the guest's.  */
-	movl PER_CPU_VAR(x86_spec_ctrl_current), %eax
+#ifdef CONFIG_X86_64
-	cmp SVM_spec_ctrl(%_ASM_DI), %eax
+	mov PER_CPU_VAR(x86_spec_ctrl_current), %rdx
+	cmp SVM_spec_ctrl(%rdi), %rdx
 	je 901b
-	xor %edx, %edx
+	movl %edx, %eax
+	shr $32, %rdx
+#else
+	mov PER_CPU_VAR(x86_spec_ctrl_current), %eax
+	mov SVM_spec_ctrl(%edi), %esi
+	xor %eax, %esi
+	mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %edx
+	mov SVM_spec_ctrl + 4(%edi), %edi
+	xor %edx, %edi
+	or %edi, %esi
+	je 901b
+#endif
 	wrmsr
 	jmp 901b
 .endm
+#define SVM_CLEAR_CPU_BUFFERS \
+	ALTERNATIVE "", __CLEAR_CPU_BUFFERS, X86_FEATURE_CLEAR_CPU_BUF_VM
 /**
  * __svm_vcpu_run - Run a vCPU via a transition to SVM guest mode
 	mov %_ASM_ARG1, %_ASM_DI
 .endif
-	/* Clobbers RAX, RCX, RDX.  */
+	/* Clobbers RAX, RCX, RDX (and ESI on 32-bit), consumes RDI (@svm). */
 	RESTORE_GUEST_SPEC_CTRL
 	/*
 	mov VCPU_RDI(%_ASM_DI), %_ASM_DI
 	/* Clobbers EFLAGS.ZF */
-	VM_CLEAR_CPU_BUFFERS
+	SVM_CLEAR_CPU_BUFFERS
 	/* Enter guest mode */
 3:	vmrun %_ASM_AX
 	/* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */
 	FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_VMEXIT
-	/* Clobbers RAX, RCX, RDX.  */
+	/*
+	 * Clobbers RAX, RCX, RDX (and ESI, EDI on 32-bit), consumes RDI (@svm)
+	 * and RSP (pointer to @spec_ctrl_intercepted).
+	 */
 	RESTORE_HOST_SPEC_CTRL
 	/*
 	mov %rdi, SEV_ES_RDI (%rdx)
 	mov %rsi, SEV_ES_RSI (%rdx)
-	/* Clobbers RAX, RCX, RDX (@hostsa). */
+	/* Clobbers RAX, RCX, and RDX (@hostsa), consumes RDI (@svm). */
 	RESTORE_GUEST_SPEC_CTRL
 	/* Get svm->current_vmcb->pa into RAX. */
 	mov KVM_VMCB_pa(%rax), %rax
 	/* Clobbers EFLAGS.ZF */
-	VM_CLEAR_CPU_BUFFERS
+	SVM_CLEAR_CPU_BUFFERS
 	/* Enter guest mode */
 1:	vmrun %rax

arch/x86/kvm/vmx/main.c+9−0

 	return tdx_vcpu_ioctl(vcpu, argp);
 }
+static int vt_vcpu_mem_enc_unlocked_ioctl(struct kvm_vcpu *vcpu, void __user *argp)
+{
+	if (!is_td_vcpu(vcpu))
+		return -EINVAL;
+	return tdx_vcpu_unlocked_ioctl(vcpu, argp);
+}
 static int vt_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn,
 				     bool is_private)
 {
 	.mem_enc_ioctl = vt_op_tdx_only(mem_enc_ioctl),
 	.vcpu_mem_enc_ioctl = vt_op_tdx_only(vcpu_mem_enc_ioctl),
+	.vcpu_mem_enc_unlocked_ioctl = vt_op_tdx_only(vcpu_mem_enc_unlocked_ioctl),
 	.gmem_max_mapping_level = vt_op_tdx_only(gmem_max_mapping_level)
 };

arch/x86/kvm/vmx/nested.c+61−112

@@ -23,8 +23,8 @@ · 22 unmodified lines
23	static bool __read_mostly enable_shadow_vmcs = 1;	23	static bool __read_mostly enable_shadow_vmcs = 1;
24	module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);	24	module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);
25		25
26	static bool __read_mostly nested_early_check = 0;	26	static bool __ro_after_init warn_on_missed_cc;
27	module_param(nested_early_check, bool, S_IRUGO);	27	module_param(warn_on_missed_cc, bool, 0444);
28		28
29	#define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK	29	#define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK
30		30
@@ -555,6 +555,9 @@ static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, · 524 unmodified lines
555	if (CC(!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr)))	555	if (CC(!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr)))
556	return -EINVAL;	556	return -EINVAL;
557		557
		558	if (CC(!nested_cpu_has_vid(vmcs12) && vmcs12->tpr_threshold >> 4))
		559	return -EINVAL;
		560
558	return 0;	561	return 0;
559	}	562	}
560		563
@@ -761,7 +764,7 @@ static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu, · 200 unmodified lines
761	vmcs12->vmcs_link_pointer, VMCS12_SIZE))	764	vmcs12->vmcs_link_pointer, VMCS12_SIZE))
762	return;	765	return;
763		766
764	kvm_read_guest_cached(vmx->vcpu.kvm, ghc, get_shadow_vmcs12(vcpu),	767	kvm_read_guest_cached(vcpu->kvm, ghc, get_shadow_vmcs12(vcpu),
765	VMCS12_SIZE);	768	VMCS12_SIZE);
766	}	769	}
767		770
@@ -780,7 +783,7 @@ static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu, · 12 unmodified lines
780	vmcs12->vmcs_link_pointer, VMCS12_SIZE))	783	vmcs12->vmcs_link_pointer, VMCS12_SIZE))
781	return;	784	return;
782		785
783	kvm_write_guest_cached(vmx->vcpu.kvm, ghc, get_shadow_vmcs12(vcpu),	786	kvm_write_guest_cached(vcpu->kvm, ghc, get_shadow_vmcs12(vcpu),
784	VMCS12_SIZE);	787	VMCS12_SIZE);
785	}	788	}
786		789
@@ -2296,15 +2299,6 @@ static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx) · 1509 unmodified lines
2296	return;	2299	return;
2297	vmx->nested.vmcs02_initialized = true;	2300	vmx->nested.vmcs02_initialized = true;
2298		2301
2299	/*
2300	* We don't care what the EPTP value is we just need to guarantee
2301	* it's valid so we don't get a false positive when doing early
2302	* consistency checks.
2303	*/
2304	if (enable_ept && nested_early_check)
2305	vmcs_write64(EPT_POINTER,
2306	construct_eptp(&vmx->vcpu, 0, PT64_ROOT_4LEVEL));
2307
2308	if (vmx->ve_info)	2302	if (vmx->ve_info)
2309	vmcs_write64(VE_INFORMATION_ADDRESS, __pa(vmx->ve_info));	2303	vmcs_write64(VE_INFORMATION_ADDRESS, __pa(vmx->ve_info));
2310		2304
@@ -2749,7 +2743,7 @@ static int prepare_vmcs02(struct kvm_vcpu vcpu, struct vmcs12 vmcs12, · 438 unmodified lines
2749	vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat);	2743	vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat);
2750	vcpu->arch.pat = vmcs12->guest_ia32_pat;	2744	vcpu->arch.pat = vmcs12->guest_ia32_pat;
2751	} else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {	2745	} else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
2752	vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);	2746	vmcs_write64(GUEST_IA32_PAT, vcpu->arch.pat);
2753	}	2747	}
2754		2748
2755	vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(	2749	vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(
@@ -2961,6 +2955,10 @@ static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, · 205 unmodified lines
2961	}	2955	}
2962	}	2956	}
2963		2957
		2958	if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING) &&
		2959	CC(!vmcs12->tsc_multiplier))
		2960	return -EINVAL;
		2961
2964	return 0;	2962	return 0;
2965	}	2963	}
2966		2964
@@ -3078,6 +3076,38 @@ static int nested_vmx_check_controls(struct kvm_vcpu *vcpu, · 111 unmodified lines
3078	return 0;	3076	return 0;
3079	}	3077	}
3080		3078
		3079	static int nested_vmx_check_controls_late(struct kvm_vcpu *vcpu,
		3080	struct vmcs12 *vmcs12)
		3081	{
		3082	void *vapic = to_vmx(vcpu)->nested.virtual_apic_map.hva;
		3083	u32 vtpr = vapic ? ((u32 )(vapic + APIC_TASKPRI)) >> 4 : 0;
		3084
		3085	/*
		3086	* Don't bother with the consistency checks if KVM isn't configured to
		3087	* WARN on missed consistency checks, as KVM needs to rely on hardware
		3088	* to fully detect an illegal vTPR vs. TRP Threshold combination due to
		3089	* the vTPR being writable by L1 at all times (it's an in-memory value,
		3090	* not a VMCS field). I.e. even if the check passes now, it might fail
		3091	* at the actual VM-Enter.
		3092	*
		3093	* Keying off the module param also allows treating an invalid vAPIC
		3094	* mapping as a consistency check failure without increasing the risk
		3095	* of breaking a "real" VM.
		3096	*/
		3097	if (!warn_on_missed_cc)
		3098	return 0;
		3099
		3100	if ((exec_controls_get(to_vmx(vcpu)) & CPU_BASED_TPR_SHADOW) &&
		3101	nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW) &&
		3102	!nested_cpu_has_vid(vmcs12) &&
		3103	!nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) &&
		3104	(CC(!vapic) \|\|
		3105	CC((vmcs12->tpr_threshold & GENMASK(3, 0)) > (vtpr & GENMASK(3, 0)))))
		3106	return -EINVAL;
		3107
		3108	return 0;
		3109	}
		3110
3081	static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu,	3111	static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu,
3082	struct vmcs12 *vmcs12)	3112	struct vmcs12 *vmcs12)
3083	{	3113	{
@@ -3333,84 +3363,6 @@ static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, · 249 unmodified lines
3333	return 0;	3363	return 0;
3334	}	3364	}
3335		3365
3336	static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu)
3337	{
3338	struct vcpu_vmx *vmx = to_vmx(vcpu);
3339	unsigned long cr3, cr4;
3340	bool vm_fail;
3341
3342	if (!nested_early_check)
3343	return 0;
3344
3345	if (vmx->msr_autoload.host.nr)
3346	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
3347	if (vmx->msr_autoload.guest.nr)
3348	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
3349
3350	preempt_disable();
3351
3352	vmx_prepare_switch_to_guest(vcpu);
3353
3354	/*
3355	* Induce a consistency check VMExit by clearing bit 1 in GUEST_RFLAGS,
3356	* which is reserved to '1' by hardware. GUEST_RFLAGS is guaranteed to
3357	* be written (by prepare_vmcs02()) before the "real" VMEnter, i.e.
3358	* there is no need to preserve other bits or save/restore the field.
3359	*/
3360	vmcs_writel(GUEST_RFLAGS, 0);
3361
3362	cr3 = __get_current_cr3_fast();
3363	if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) {
3364	vmcs_writel(HOST_CR3, cr3);
3365	vmx->loaded_vmcs->host_state.cr3 = cr3;
3366	}
3367
3368	cr4 = cr4_read_shadow();
3369	if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) {
3370	vmcs_writel(HOST_CR4, cr4);
3371	vmx->loaded_vmcs->host_state.cr4 = cr4;
3372	}
3373
3374	vm_fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs,
3375	__vmx_vcpu_run_flags(vmx));
3376
3377	if (vmx->msr_autoload.host.nr)
3378	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
3379	if (vmx->msr_autoload.guest.nr)
3380	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
3381
3382	if (vm_fail) {
3383	u32 error = vmcs_read32(VM_INSTRUCTION_ERROR);
3384
3385	preempt_enable();
3386
3387	trace_kvm_nested_vmenter_failed(
3388	"early hardware check VM-instruction error: ", error);
3389	WARN_ON_ONCE(error != VMXERR_ENTRY_INVALID_CONTROL_FIELD);
3390	return 1;
3391	}
3392
3393	/*
3394	* VMExit clears RFLAGS.IF and DR7, even on a consistency check.
3395	*/
3396	if (hw_breakpoint_active())
3397	set_debugreg(__this_cpu_read(cpu_dr7), 7);
3398	local_irq_enable();
3399	preempt_enable();
3400
3401	/*
3402	* A non-failing VMEntry means we somehow entered guest mode with
3403	* an illegal RIP, and that's just the tip of the iceberg. There
3404	* is no telling what memory has been modified or what state has
3405	* been exposed to unknown code. Hitting this all but guarantees
3406	* a (very critical) hardware issue.
3407	*/
3408	WARN_ON(!(vmcs_read32(VM_EXIT_REASON) &
3409	VMX_EXIT_REASONS_FAILED_VMENTRY));
3410
3411	return 0;
3412	}
3413
3414	#ifdef CONFIG_KVM_HYPERV	3366	#ifdef CONFIG_KVM_HYPERV
3415	static bool nested_get_evmcs_page(struct kvm_vcpu *vcpu)	3367	static bool nested_get_evmcs_page(struct kvm_vcpu *vcpu)
3416	{	3368	{
@@ -3667,22 +3619,18 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, · 250 unmodified lines
3667	&vmx->nested.pre_vmenter_ssp_tbl);	3619	&vmx->nested.pre_vmenter_ssp_tbl);
3668		3620
3669	/*	3621	/*
3670	* Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled and	3622	* Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled. In the
3671	* nested early checks are disabled. In the event of a "late" VM-Fail,	3623	* event of a "late" VM-Fail, i.e. a VM-Fail detected by hardware but
3672	* i.e. a VM-Fail detected by hardware but not KVM, KVM must unwind its	3624	* not KVM, KVM must unwind its software model to the pre-VM-Entry host
3673	* software model to the pre-VMEntry host state. When EPT is disabled,	3625	* state. When EPT is disabled, GUEST_CR3 holds KVM's shadow CR3, not
3674	* GUEST_CR3 holds KVM's shadow CR3, not L1's "real" CR3, which causes	3626	* L1's "real" CR3, which causes nested_vmx_restore_host_state() to
3675	* nested_vmx_restore_host_state() to corrupt vcpu->arch.cr3. Stuffing	3627	* corrupt vcpu->arch.cr3. Stuffing vmcs01.GUEST_CR3 results in the
3676	* vmcs01.GUEST_CR3 results in the unwind naturally setting arch.cr3 to	3628	* unwind naturally setting arch.cr3 to the correct value. Smashing
3677	* the correct value. Smashing vmcs01.GUEST_CR3 is safe because nested	3629	* vmcs01.GUEST_CR3 is safe because nested VM-Exits, and the unwind,
3678	* VM-Exits, and the unwind, reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is	3630	* reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is guaranteed to be
3679	* guaranteed to be overwritten with a shadow CR3 prior to re-entering	3631	* overwritten with a shadow CR3 prior to re-entering L1.
3680	* L1. Don't stuff vmcs01.GUEST_CR3 when using nested early checks as
3681	* KVM modifies vcpu->arch.cr3 if and only if the early hardware checks
3682	* pass, and early VM-Fails do not reset KVM's MMU, i.e. the VM-Fail
3683	* path would need to manually save/restore vmcs01.GUEST_CR3.
3684	*/	3632	*/
3685	if (!enable_ept && !nested_early_check)	3633	if (!enable_ept)
3686	vmcs_writel(GUEST_CR3, vcpu->arch.cr3);	3634	vmcs_writel(GUEST_CR3, vcpu->arch.cr3);
3687		3635
3688	vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02);	3636	vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02);
@@ -3695,7 +3643,7 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, · 6 unmodified lines
3695	return NVMX_VMENTRY_KVM_INTERNAL_ERROR;	3643	return NVMX_VMENTRY_KVM_INTERNAL_ERROR;
3696	}	3644	}
3697		3645
3698	if (nested_vmx_check_vmentry_hw(vcpu)) {	3646	if (nested_vmx_check_controls_late(vcpu, vmcs12)) {
3699	vmx_switch_vmcs(vcpu, &vmx->vmcs01);	3647	vmx_switch_vmcs(vcpu, &vmx->vmcs01);
3700	return NVMX_VMENTRY_VMFAIL;	3648	return NVMX_VMENTRY_VMFAIL;
3701	}	3649	}
@@ -3880,7 +3828,7 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) · 178 unmodified lines
3880	goto vmentry_failed;	3828	goto vmentry_failed;
3881		3829
3882	/* Hide L1D cache contents from the nested guest. */	3830	/* Hide L1D cache contents from the nested guest. */
3883	vmx->vcpu.arch.l1tf_flush_l1d = true;	3831	kvm_request_l1tf_flush_l1d();
3884		3832
3885	/*	3833	/*
3886	* Must happen outside of nested_vmx_enter_non_root_mode() as it will	3834	* Must happen outside of nested_vmx_enter_non_root_mode() as it will
@@ -5164,12 +5112,13 @@ void __nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, · 1277 unmodified lines
5164	/*	5112	/*
5165	* The only expected VM-instruction error is "VM entry with	5113	* The only expected VM-instruction error is "VM entry with
5166	* invalid control field(s)." Anything else indicates a	5114	* invalid control field(s)." Anything else indicates a
5167	* problem with L0. And we should never get here with a	5115	* problem with L0.
5168	* VMFail of any type if early consistency checks are enabled.
5169	*/	5116	*/
5170	WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) !=	5117	WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) !=
5171	VMXERR_ENTRY_INVALID_CONTROL_FIELD);	5118	VMXERR_ENTRY_INVALID_CONTROL_FIELD);
5172	WARN_ON_ONCE(nested_early_check);	5119
		5120	/* VM-Fail at VM-Entry means KVM missed a consistency check. */
		5121	WARN_ON_ONCE(warn_on_missed_cc);
5173	}	5122	}
5174		5123
5175	/*	5124	/*

arch/x86/kvm/vmx/run_flags.h+3−7

 #ifndef __KVM_X86_VMX_RUN_FLAGS_H
 #define __KVM_X86_VMX_RUN_FLAGS_H
-#define VMX_RUN_VMRESUME_SHIFT				0
+#define VMX_RUN_VMRESUME			BIT(0)
-#define VMX_RUN_SAVE_SPEC_CTRL_SHIFT			1
+#define VMX_RUN_SAVE_SPEC_CTRL			BIT(1)
-#define VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO_SHIFT	2
+#define VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO	BIT(2)
-#define VMX_RUN_VMRESUME			BIT(VMX_RUN_VMRESUME_SHIFT)
-#define VMX_RUN_SAVE_SPEC_CTRL			BIT(VMX_RUN_SAVE_SPEC_CTRL_SHIFT)
-#define VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO	BIT(VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO_SHIFT)
 #endif /* __KVM_X86_VMX_RUN_FLAGS_H */

arch/x86/kvm/vmx/tdx.c+380−425

@@ -24,20 +24,33 @@ · 23 unmodified lines
24	#undef pr_fmt	24	#undef pr_fmt
25	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt	25	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
26		26
27	#define pr_tdx_error(__fn, __err) \	27	#define __TDX_BUG_ON(__err, __f, __kvm, __fmt, __args...) \
28	pr_err_ratelimited("SEAMCALL %s failed: 0x%llx\n", #__fn, __err)	28	({ \
		29	struct kvm *_kvm = (__kvm); \
		30	bool __ret = !!(__err); \
		31	\
		32	if (WARN_ON_ONCE(__ret && (!_kvm \|\| !_kvm->vm_bugged))) { \
		33	if (_kvm) \
		34	kvm_vm_bugged(_kvm); \
		35	pr_err_ratelimited("SEAMCALL " __f " failed: 0x%llx" __fmt "\n",\
		36	__err, __args); \
		37	} \
		38	unlikely(__ret); \
		39	})
		40
		41	#define TDX_BUG_ON(__err, __fn, __kvm) \
		42	__TDX_BUG_ON(__err, #__fn, __kvm, "%s", "")
		43
		44	#define TDX_BUG_ON_1(__err, __fn, a1, __kvm) \
		45	__TDX_BUG_ON(__err, #__fn, __kvm, ", " #a1 " 0x%llx", a1)
		46
		47	#define TDX_BUG_ON_2(__err, __fn, a1, a2, __kvm) \
		48	__TDX_BUG_ON(__err, #__fn, __kvm, ", " #a1 " 0x%llx, " #a2 " 0x%llx", a1, a2)
		49
		50	#define TDX_BUG_ON_3(__err, __fn, a1, a2, a3, __kvm) \
		51	__TDX_BUG_ON(__err, #__fn, __kvm, ", " #a1 " 0x%llx, " #a2 ", 0x%llx, " #a3 " 0x%llx", \
		52	a1, a2, a3)
29		53
30	#define __pr_tdx_error_N(__fn_str, __err, __fmt, ...) \
31	pr_err_ratelimited("SEAMCALL " __fn_str " failed: 0x%llx, " __fmt, __err, __VA_ARGS__)
32
33	#define pr_tdx_error_1(__fn, __err, __rcx) \
34	__pr_tdx_error_N(#__fn, __err, "rcx 0x%llx\n", __rcx)
35
36	#define pr_tdx_error_2(__fn, __err, __rcx, __rdx) \
37	__pr_tdx_error_N(#__fn, __err, "rcx 0x%llx, rdx 0x%llx\n", __rcx, __rdx)
38
39	#define pr_tdx_error_3(__fn, __err, __rcx, __rdx, __r8) \
40	__pr_tdx_error_N(#__fn, __err, "rcx 0x%llx, rdx 0x%llx, r8 0x%llx\n", __rcx, __rdx, __r8)
41		54
42	bool enable_tdx __ro_after_init;	55	bool enable_tdx __ro_after_init;
43	module_param_named(tdx, enable_tdx, bool, 0444);	56	module_param_named(tdx, enable_tdx, bool, 0444);
@@ -281,25 +294,34 @@ static inline void tdx_disassociate_vp(struct kvm_vcpu *vcpu) · 237 unmodified lines
281	vcpu->cpu = -1;	294	vcpu->cpu = -1;
282	}	295	}
283		296
284	static void tdx_no_vcpus_enter_start(struct kvm *kvm)	297	/*
285	{	298	* Execute a SEAMCALL related to removing/blocking S-EPT entries, with a single
286	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);	299	* retry (if necessary) after forcing vCPUs to exit and wait for the operation
287		300	* to complete. All flows that remove/block S-EPT entries run with mmu_lock
288	lockdep_assert_held_write(&kvm->mmu_lock);	301	* held for write, i.e. are mutually exclusive with each other, but they aren't
289		302	* mutually exclusive with running vCPUs, and so can fail with "operand busy"
290	WRITE_ONCE(kvm_tdx->wait_for_sept_zap, true);	303	* if a vCPU acquires a relevant lock in the TDX-Module, e.g. when doing TDCALL.
291		304	*
292	kvm_make_all_cpus_request(kvm, KVM_REQ_OUTSIDE_GUEST_MODE);	305	* Note, the retry is guaranteed to succeed, absent KVM and/or TDX-Module bugs.
293	}	306	*/
294		307	#define tdh_do_no_vcpus(tdh_func, kvm, args...) \
295	static void tdx_no_vcpus_enter_stop(struct kvm *kvm)	308	({ \
296	{	309	struct kvm_tdx *__kvm_tdx = to_kvm_tdx(kvm); \
297	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);	310	u64 __err; \
298		311	\
299	lockdep_assert_held_write(&kvm->mmu_lock);	312	lockdep_assert_held_write(&kvm->mmu_lock); \
300		313	\
301	WRITE_ONCE(kvm_tdx->wait_for_sept_zap, false);	314	__err = tdh_func(args); \
302	}	315	if (unlikely(tdx_operand_busy(__err))) { \
		316	WRITE_ONCE(__kvm_tdx->wait_for_sept_zap, true); \
		317	kvm_make_all_cpus_request(kvm, KVM_REQ_OUTSIDE_GUEST_MODE); \
		318	\
		319	__err = tdh_func(args); \
		320	\
		321	WRITE_ONCE(__kvm_tdx->wait_for_sept_zap, false); \
		322	} \
		323	__err; \
		324	})
303		325
304	/* TDH.PHYMEM.PAGE.RECLAIM is allowed only when destroying the TD. */	326	/* TDH.PHYMEM.PAGE.RECLAIM is allowed only when destroying the TD. */
305	static int __tdx_reclaim_page(struct page *page)	327	static int __tdx_reclaim_page(struct page *page)
@@ -313,10 +335,9 @@ static int __tdx_reclaim_page(struct page *page) · 7 unmodified lines
313	* before the HKID is released and control pages have also been	335	* before the HKID is released and control pages have also been
314	* released at this point, so there is no possibility of contention.	336	* released at this point, so there is no possibility of contention.
315	*/	337	*/
316	if (WARN_ON_ONCE(err)) {	338	if (TDX_BUG_ON_3(err, TDH_PHYMEM_PAGE_RECLAIM, rcx, rdx, r8, NULL))
317	pr_tdx_error_3(TDH_PHYMEM_PAGE_RECLAIM, err, rcx, rdx, r8);
318	return -EIO;	339	return -EIO;
319	}	340
320	return 0;	341	return 0;
321	}	342	}
322		343
@@ -404,8 +425,8 @@ static void tdx_flush_vp_on_cpu(struct kvm_vcpu *vcpu) · 81 unmodified lines
404	return;	425	return;
405		426
406	smp_call_function_single(cpu, tdx_flush_vp, &arg, 1);	427	smp_call_function_single(cpu, tdx_flush_vp, &arg, 1);
407	if (KVM_BUG_ON(arg.err, vcpu->kvm))	428
408	pr_tdx_error(TDH_VP_FLUSH, arg.err);	429	TDX_BUG_ON(arg.err, TDH_VP_FLUSH, vcpu->kvm);
409	}	430	}
410		431
411	void tdx_disable_virtualization_cpu(void)	432	void tdx_disable_virtualization_cpu(void)
@@ -464,8 +485,7 @@ static void smp_func_do_phymem_cache_wb(void *unused) · 52 unmodified lines
464	}	485	}
465		486
466	out:	487	out:
467	if (WARN_ON_ONCE(err))	488	TDX_BUG_ON(err, TDH_PHYMEM_CACHE_WB, NULL);
468	pr_tdx_error(TDH_PHYMEM_CACHE_WB, err);
469	}	489	}
470		490
471	void tdx_mmu_release_hkid(struct kvm *kvm)	491	void tdx_mmu_release_hkid(struct kvm *kvm)
@@ -504,8 +524,7 @@ void tdx_mmu_release_hkid(struct kvm *kvm) · 32 unmodified lines
504	err = tdh_mng_vpflushdone(&kvm_tdx->td);	524	err = tdh_mng_vpflushdone(&kvm_tdx->td);
505	if (err == TDX_FLUSHVP_NOT_DONE)	525	if (err == TDX_FLUSHVP_NOT_DONE)
506	goto out;	526	goto out;
507	if (KVM_BUG_ON(err, kvm)) {	527	if (TDX_BUG_ON(err, TDH_MNG_VPFLUSHDONE, kvm)) {
508	pr_tdx_error(TDH_MNG_VPFLUSHDONE, err);
509	pr_err("tdh_mng_vpflushdone() failed. HKID %d is leaked.\n",	528	pr_err("tdh_mng_vpflushdone() failed. HKID %d is leaked.\n",
510	kvm_tdx->hkid);	529	kvm_tdx->hkid);
511	goto out;	530	goto out;
@@ -528,8 +547,7 @@ void tdx_mmu_release_hkid(struct kvm *kvm) · 16 unmodified lines
528	* tdh_mng_key_freeid() will fail.	547	* tdh_mng_key_freeid() will fail.
529	*/	548	*/
530	err = tdh_mng_key_freeid(&kvm_tdx->td);	549	err = tdh_mng_key_freeid(&kvm_tdx->td);
531	if (KVM_BUG_ON(err, kvm)) {	550	if (TDX_BUG_ON(err, TDH_MNG_KEY_FREEID, kvm)) {
532	pr_tdx_error(TDH_MNG_KEY_FREEID, err);
533	pr_err("tdh_mng_key_freeid() failed. HKID %d is leaked.\n",	551	pr_err("tdh_mng_key_freeid() failed. HKID %d is leaked.\n",
534	kvm_tdx->hkid);	552	kvm_tdx->hkid);
535	} else {	553	} else {
@@ -580,10 +598,9 @@ static void tdx_reclaim_td_control_pages(struct kvm *kvm) · 44 unmodified lines
580	* when it is reclaiming TDCS).	598	* when it is reclaiming TDCS).
581	*/	599	*/
582	err = tdh_phymem_page_wbinvd_tdr(&kvm_tdx->td);	600	err = tdh_phymem_page_wbinvd_tdr(&kvm_tdx->td);
583	if (KVM_BUG_ON(err, kvm)) {	601	if (TDX_BUG_ON(err, TDH_PHYMEM_PAGE_WBINVD, kvm))
584	pr_tdx_error(TDH_PHYMEM_PAGE_WBINVD, err);
585	return;	602	return;
586	}	603
587	tdx_quirk_reset_page(kvm_tdx->td.tdr_page);	604	tdx_quirk_reset_page(kvm_tdx->td.tdr_page);
588		605
589	__free_page(kvm_tdx->td.tdr_page);	606	__free_page(kvm_tdx->td.tdr_page);
@@ -606,11 +623,8 @@ static int tdx_do_tdh_mng_key_config(void *param) · 16 unmodified lines
606		623
607	/* TDX_RND_NO_ENTROPY related retries are handled by sc_retry() */	624	/* TDX_RND_NO_ENTROPY related retries are handled by sc_retry() */
608	err = tdh_mng_key_config(&kvm_tdx->td);	625	err = tdh_mng_key_config(&kvm_tdx->td);
609		626	if (TDX_BUG_ON(err, TDH_MNG_KEY_CONFIG, &kvm_tdx->kvm))
610	if (KVM_BUG_ON(err, &kvm_tdx->kvm)) {
611	pr_tdx_error(TDH_MNG_KEY_CONFIG, err);
612	return -EIO;	627	return -EIO;
613	}
614		628
615	return 0;	629	return 0;
616	}	630	}
@@ -763,25 +777,6 @@ static bool tdx_protected_apic_has_interrupt(struct kvm_vcpu *vcpu) · 146 unmodified lines
763	return tdx_vcpu_state_details_intr_pending(vcpu_state_details);	777	return tdx_vcpu_state_details_intr_pending(vcpu_state_details);
764	}	778	}
765		779
766	/*
767	* Compared to vmx_prepare_switch_to_guest(), there is not much to do
768	* as SEAMCALL/SEAMRET calls take care of most of save and restore.
769	*/
770	void tdx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
771	{
772	struct vcpu_vt *vt = to_vt(vcpu);
773
774	if (vt->guest_state_loaded)
775	return;
776
777	if (likely(is_64bit_mm(current->mm)))
778	vt->msr_host_kernel_gs_base = current->thread.gsbase;
779	else
780	vt->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE);
781
782	vt->guest_state_loaded = true;
783	}
784
785	struct tdx_uret_msr {	780	struct tdx_uret_msr {
786	u32 msr;	781	u32 msr;
787	unsigned int slot;	782	unsigned int slot;
@@ -795,19 +790,38 @@ static struct tdx_uret_msr tdx_uret_msrs[] = { · 7 unmodified lines
795	{.msr = MSR_TSC_AUX,},	790	{.msr = MSR_TSC_AUX,},
796	};	791	};
797		792
798	static void tdx_user_return_msr_update_cache(void)	793	void tdx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
799	{	794	{
		795	struct vcpu_vt *vt = to_vt(vcpu);
800	int i;	796	int i;
801		797
		798	if (vt->guest_state_loaded)
		799	return;
		800
		801	if (likely(is_64bit_mm(current->mm)))
		802	vt->msr_host_kernel_gs_base = current->thread.gsbase;
		803	else
		804	vt->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE);
		805
		806	vt->guest_state_loaded = true;
		807
		808	/*
		809	* Explicitly set user-return MSRs that are clobbered by the TDX-Module
		810	* if VP.ENTER succeeds, i.e. on TD-Exit, with the values that would be
		811	* written by the TDX-Module. Don't rely on the TDX-Module to actually
		812	* clobber the MSRs, as the contract is poorly defined and not upheld.
		813	* E.g. the TDX-Module will synthesize an EPT Violation without doing
		814	* VM-Enter if it suspects a zero-step attack, and never "restore" VMM
		815	* state.
		816	*/
802	for (i = 0; i < ARRAY_SIZE(tdx_uret_msrs); i++)	817	for (i = 0; i < ARRAY_SIZE(tdx_uret_msrs); i++)
803	kvm_user_return_msr_update_cache(tdx_uret_msrs[i].slot,	818	kvm_set_user_return_msr(tdx_uret_msrs[i].slot,
804	tdx_uret_msrs[i].defval);	819	tdx_uret_msrs[i].defval, -1ull);
805	}	820	}
806		821
807	static void tdx_prepare_switch_to_host(struct kvm_vcpu *vcpu)	822	static void tdx_prepare_switch_to_host(struct kvm_vcpu *vcpu)
808	{	823	{
809	struct vcpu_vt *vt = to_vt(vcpu);	824	struct vcpu_vt *vt = to_vt(vcpu);
810	struct vcpu_tdx *tdx = to_tdx(vcpu);
811		825
812	if (!vt->guest_state_loaded)	826	if (!vt->guest_state_loaded)
813	return;	827	return;
@@ -815,11 +829,6 @@ static void tdx_prepare_switch_to_host(struct kvm_vcpu *vcpu) · 1 unmodified line
815	++vcpu->stat.host_state_reload;	829	++vcpu->stat.host_state_reload;
816	wrmsrl(MSR_KERNEL_GS_BASE, vt->msr_host_kernel_gs_base);	830	wrmsrl(MSR_KERNEL_GS_BASE, vt->msr_host_kernel_gs_base);
817		831
818	if (tdx->guest_entered) {
819	tdx_user_return_msr_update_cache();
820	tdx->guest_entered = false;
821	}
822
823	vt->guest_state_loaded = false;	832	vt->guest_state_loaded = false;
824	}	833	}
825		834
@@ -829,19 +838,52 @@ void tdx_vcpu_put(struct kvm_vcpu *vcpu) · 3 unmodified lines
829	tdx_prepare_switch_to_host(vcpu);	838	tdx_prepare_switch_to_host(vcpu);
830	}	839	}
831		840
		841	/*
		842	* Life cycles for a TD and a vCPU:
		843	* 1. KVM_CREATE_VM ioctl.
		844	* TD state is TD_STATE_UNINITIALIZED.
		845	* hkid is not assigned at this stage.
		846	* 2. KVM_TDX_INIT_VM ioctl.
		847	* TD transitions to TD_STATE_INITIALIZED.
		848	* hkid is assigned after this stage.
		849	* 3. KVM_CREATE_VCPU ioctl. (only when TD is TD_STATE_INITIALIZED).
		850	* 3.1 tdx_vcpu_create() transitions vCPU state to VCPU_TD_STATE_UNINITIALIZED.
		851	* 3.2 vcpu_load() and vcpu_put() in kvm_arch_vcpu_create().
		852	* 3.3 (conditional) if any error encountered after kvm_arch_vcpu_create()
		853	* kvm_arch_vcpu_destroy() --> tdx_vcpu_free().
		854	* 4. KVM_TDX_INIT_VCPU ioctl.
		855	* tdx_vcpu_init() transitions vCPU state to VCPU_TD_STATE_INITIALIZED.
		856	* vCPU control structures are allocated at this stage.
		857	* 5. kvm_destroy_vm().
		858	* 5.1 tdx_mmu_release_hkid(): (1) tdh_vp_flush(), disassociates all vCPUs.
		859	* (2) puts hkid to !assigned state.
		860	* 5.2 kvm_destroy_vcpus() --> tdx_vcpu_free():
		861	* transitions vCPU to VCPU_TD_STATE_UNINITIALIZED state.
		862	* 5.3 tdx_vm_destroy()
		863	* transitions TD to TD_STATE_UNINITIALIZED state.
		864	*
		865	* tdx_vcpu_free() can be invoked only at 3.3 or 5.2.
		866	* - If at 3.3, hkid is still assigned, but the vCPU must be in
		867	* VCPU_TD_STATE_UNINITIALIZED state.
		868	* - if at 5.2, hkid must be !assigned and all vCPUs must be in
		869	* VCPU_TD_STATE_INITIALIZED state and have been dissociated.
		870	*/
832	void tdx_vcpu_free(struct kvm_vcpu *vcpu)	871	void tdx_vcpu_free(struct kvm_vcpu *vcpu)
833	{	872	{
834	struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);	873	struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);
835	struct vcpu_tdx *tdx = to_tdx(vcpu);	874	struct vcpu_tdx *tdx = to_tdx(vcpu);
836	int i;	875	int i;
837		876
		877	if (vcpu->cpu != -1) {
		878	KVM_BUG_ON(tdx->state == VCPU_TD_STATE_INITIALIZED, vcpu->kvm);
		879	tdx_flush_vp_on_cpu(vcpu);
		880	return;
		881	}
		882
838	/*	883	/*
839	* It is not possible to reclaim pages while hkid is assigned. It might	884	* It is not possible to reclaim pages while hkid is assigned. It might
840	* be assigned if:	885	* be assigned if the TD VM is being destroyed but freeing hkid failed,
841	* 1. the TD VM is being destroyed but freeing hkid failed, in which	886	* in which case the pages are leaked.
842	* case the pages are leaked
843	* 2. TD VCPU creation failed and this on the error path, in which case
844	* there is nothing to do anyway
845	*/	887	*/
846	if (is_hkid_assigned(kvm_tdx))	888	if (is_hkid_assigned(kvm_tdx))
847	return;	889	return;
@@ -856,7 +898,7 @@ void tdx_vcpu_free(struct kvm_vcpu *vcpu) · 8 unmodified lines
856	}	898	}
857	if (tdx->vp.tdvpr_page) {	899	if (tdx->vp.tdvpr_page) {
858	tdx_reclaim_control_page(tdx->vp.tdvpr_page);	900	tdx_reclaim_control_page(tdx->vp.tdvpr_page);
859	tdx->vp.tdvpr_page = 0;	901	tdx->vp.tdvpr_page = NULL;
860	tdx->vp.tdvpr_pa = 0;	902	tdx->vp.tdvpr_pa = 0;
861	}	903	}
862		904
@@ -1059,7 +1101,6 @@ fastpath_t tdx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) · 196 unmodified lines
1059	update_debugctlmsr(vcpu->arch.host_debugctl);	1101	update_debugctlmsr(vcpu->arch.host_debugctl);
1060		1102
1061	tdx_load_host_xsave_state(vcpu);	1103	tdx_load_host_xsave_state(vcpu);
1062	tdx->guest_entered = true;
1063		1104
1064	vcpu->arch.regs_avail &= TDX_REGS_AVAIL_SET;	1105	vcpu->arch.regs_avail &= TDX_REGS_AVAIL_SET;
1065		1106
@@ -1069,9 +1110,6 @@ fastpath_t tdx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) · 3 unmodified lines
1069	if (unlikely((tdx->vp_enter_ret & TDX_SW_ERROR) == TDX_SW_ERROR))	1110	if (unlikely((tdx->vp_enter_ret & TDX_SW_ERROR) == TDX_SW_ERROR))
1070	return EXIT_FASTPATH_NONE;	1111	return EXIT_FASTPATH_NONE;
1071		1112
1072	if (unlikely(vmx_get_exit_reason(vcpu).basic == EXIT_REASON_MCE_DURING_VMENTRY))
1073	kvm_machine_check();
1074
1075	trace_kvm_exit(vcpu, KVM_ISA_VMX);	1113	trace_kvm_exit(vcpu, KVM_ISA_VMX);
1076		1114
1077	if (unlikely(tdx_failed_vmentry(vcpu)))	1115	if (unlikely(tdx_failed_vmentry(vcpu)))
@@ -1583,137 +1621,79 @@ void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int pgd_level) · 505 unmodified lines
1583	td_vmcs_write64(to_tdx(vcpu), SHARED_EPT_POINTER, root_hpa);	1621	td_vmcs_write64(to_tdx(vcpu), SHARED_EPT_POINTER, root_hpa);
1584	}	1622	}
1585		1623
1586	static void tdx_unpin(struct kvm kvm, struct page page)	1624	static int tdx_mem_page_add(struct kvm *kvm, gfn_t gfn, enum pg_level level,
		1625	kvm_pfn_t pfn)
1587	{	1626	{
1588	put_page(page);	1627	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
		1628	u64 err, entry, level_state;
		1629	gpa_t gpa = gfn_to_gpa(gfn);
		1630
		1631	lockdep_assert_held(&kvm->slots_lock);
		1632
		1633	if (KVM_BUG_ON(kvm->arch.pre_fault_allowed, kvm) \|\|
		1634	KVM_BUG_ON(!kvm_tdx->page_add_src, kvm))
		1635	return -EIO;
		1636
		1637	err = tdh_mem_page_add(&kvm_tdx->td, gpa, pfn_to_page(pfn),
		1638	kvm_tdx->page_add_src, &entry, &level_state);
		1639	if (unlikely(tdx_operand_busy(err)))
		1640	return -EBUSY;
		1641
		1642	if (TDX_BUG_ON_2(err, TDH_MEM_PAGE_ADD, entry, level_state, kvm))
		1643	return -EIO;
		1644
		1645	return 0;
1589	}	1646	}
1590		1647
1591	static int tdx_mem_page_aug(struct kvm *kvm, gfn_t gfn,	1648	static int tdx_mem_page_aug(struct kvm *kvm, gfn_t gfn,
1592	enum pg_level level, struct page *page)	1649	enum pg_level level, kvm_pfn_t pfn)
1593	{	1650	{
1594	int tdx_level = pg_level_to_tdx_sept_level(level);	1651	int tdx_level = pg_level_to_tdx_sept_level(level);
1595	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);	1652	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
		1653	struct page *page = pfn_to_page(pfn);
1596	gpa_t gpa = gfn_to_gpa(gfn);	1654	gpa_t gpa = gfn_to_gpa(gfn);
1597	u64 entry, level_state;	1655	u64 entry, level_state;
1598	u64 err;	1656	u64 err;
1599		1657
1600	err = tdh_mem_page_aug(&kvm_tdx->td, gpa, tdx_level, page, &entry, &level_state);	1658	err = tdh_mem_page_aug(&kvm_tdx->td, gpa, tdx_level, page, &entry, &level_state);
1601	if (unlikely(tdx_operand_busy(err))) {	1659	if (unlikely(tdx_operand_busy(err)))
1602	tdx_unpin(kvm, page);
1603	return -EBUSY;	1660	return -EBUSY;
1604	}
1605		1661
1606	if (KVM_BUG_ON(err, kvm)) {	1662	if (TDX_BUG_ON_2(err, TDH_MEM_PAGE_AUG, entry, level_state, kvm))
1607	pr_tdx_error_2(TDH_MEM_PAGE_AUG, err, entry, level_state);
1608	tdx_unpin(kvm, page);
1609	return -EIO;	1663	return -EIO;
1610	}
1611
1612	return 0;
1613	}
1614
1615	/*
1616	* KVM_TDX_INIT_MEM_REGION calls kvm_gmem_populate() to map guest pages; the
1617	* callback tdx_gmem_post_populate() then maps pages into private memory.
1618	* through the a seamcall TDH.MEM.PAGE.ADD(). The SEAMCALL also requires the
1619	* private EPT structures for the page to have been built before, which is
1620	* done via kvm_tdp_map_page(). nr_premapped counts the number of pages that
1621	* were added to the EPT structures but not added with TDH.MEM.PAGE.ADD().
1622	* The counter has to be zero on KVM_TDX_FINALIZE_VM, to ensure that there
1623	* are no half-initialized shared EPT pages.
1624	*/
1625	static int tdx_mem_page_record_premap_cnt(struct kvm *kvm, gfn_t gfn,
1626	enum pg_level level, kvm_pfn_t pfn)
1627	{
1628	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
1629
1630	if (KVM_BUG_ON(kvm->arch.pre_fault_allowed, kvm))
1631	return -EINVAL;
1632		1664
1633	/* nr_premapped will be decreased when tdh_mem_page_add() is called. */
1634	atomic64_inc(&kvm_tdx->nr_premapped);
1635	return 0;	1665	return 0;
1636	}	1666	}
1637		1667
1638	static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn,	1668	static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn,
1639	enum pg_level level, kvm_pfn_t pfn)	1669	enum pg_level level, u64 mirror_spte)
1640	{	1670	{
1641	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);	1671	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
1642	struct page *page = pfn_to_page(pfn);	1672	kvm_pfn_t pfn = spte_to_pfn(mirror_spte);
1643		1673
1644	/* TODO: handle large pages. */	1674	/* TODO: handle large pages. */
1645	if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))	1675	if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))
1646	return -EINVAL;	1676	return -EIO;
1647		1677
1648	/*	1678	WARN_ON_ONCE(!is_shadow_present_pte(mirror_spte) \|\|
1649	* Because guest_memfd doesn't support page migration with	1679	(mirror_spte & VMX_EPT_RWX_MASK) != VMX_EPT_RWX_MASK);
1650	* a_ops->migrate_folio (yet), no callback is triggered for KVM on page
1651	* migration. Until guest_memfd supports page migration, prevent page
1652	* migration.
1653	* TODO: Once guest_memfd introduces callback on page migration,
1654	* implement it and remove get_page/put_page().
1655	*/
1656	get_page(page);
1657		1680
1658	/*	1681	/*
1659	* Read 'pre_fault_allowed' before 'kvm_tdx->state'; see matching	1682	* Ensure pre_fault_allowed is read by kvm_arch_vcpu_pre_fault_memory()
1660	* barrier in tdx_td_finalize().	1683	* before kvm_tdx->state. Userspace must not be allowed to pre-fault
		1684	* arbitrary memory until the initial memory image is finalized. Pairs
		1685	* with the smp_wmb() in tdx_td_finalize().
1661	*/	1686	*/
1662	smp_rmb();	1687	smp_rmb();
1663	if (likely(kvm_tdx->state == TD_STATE_RUNNABLE))
1664	return tdx_mem_page_aug(kvm, gfn, level, page);
1665
1666	return tdx_mem_page_record_premap_cnt(kvm, gfn, level, pfn);
1667	}
1668
1669	static int tdx_sept_drop_private_spte(struct kvm *kvm, gfn_t gfn,
1670	enum pg_level level, struct page *page)
1671	{
1672	int tdx_level = pg_level_to_tdx_sept_level(level);
1673	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
1674	gpa_t gpa = gfn_to_gpa(gfn);
1675	u64 err, entry, level_state;
1676
1677	/* TODO: handle large pages. */
1678	if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))
1679	return -EINVAL;
1680
1681	if (KVM_BUG_ON(!is_hkid_assigned(kvm_tdx), kvm))
1682	return -EINVAL;
1683		1688
1684	/*	1689	/*
1685	* When zapping private page, write lock is held. So no race condition	1690	* If the TD isn't finalized/runnable, then userspace is initializing
1686	* with other vcpu sept operation.	1691	* the VM image via KVM_TDX_INIT_MEM_REGION; ADD the page to the TD.
1687	* Race with TDH.VP.ENTER due to (0-step mitigation) and Guest TDCALLs.
1688	*/	1692	*/
1689	err = tdh_mem_page_remove(&kvm_tdx->td, gpa, tdx_level, &entry,	1693	if (unlikely(kvm_tdx->state != TD_STATE_RUNNABLE))
1690	&level_state);	1694	return tdx_mem_page_add(kvm, gfn, level, pfn);
1691
1692	if (unlikely(tdx_operand_busy(err))) {
1693	/*
1694	* The second retry is expected to succeed after kicking off all
1695	* other vCPUs and prevent them from invoking TDH.VP.ENTER.
1696	*/
1697	tdx_no_vcpus_enter_start(kvm);
1698	err = tdh_mem_page_remove(&kvm_tdx->td, gpa, tdx_level, &entry,
1699	&level_state);
1700	tdx_no_vcpus_enter_stop(kvm);
1701	}
1702
1703	if (KVM_BUG_ON(err, kvm)) {
1704	pr_tdx_error_2(TDH_MEM_PAGE_REMOVE, err, entry, level_state);
1705	return -EIO;
1706	}
1707
1708	err = tdh_phymem_page_wbinvd_hkid((u16)kvm_tdx->hkid, page);
1709		1695
1710	if (KVM_BUG_ON(err, kvm)) {	1696	return tdx_mem_page_aug(kvm, gfn, level, pfn);
1711	pr_tdx_error(TDH_PHYMEM_PAGE_WBINVD, err);
1712	return -EIO;
1713	}
1714	tdx_quirk_reset_page(page);
1715	tdx_unpin(kvm, page);
1716	return 0;
1717	}	1697	}
1718		1698
1719	static int tdx_sept_link_private_spt(struct kvm *kvm, gfn_t gfn,	1699	static int tdx_sept_link_private_spt(struct kvm *kvm, gfn_t gfn,
@@ -1729,80 +1709,12 @@ static int tdx_sept_link_private_spt(struct kvm *kvm, gfn_t gfn, · 9 unmodified lines
1729	if (unlikely(tdx_operand_busy(err)))	1709	if (unlikely(tdx_operand_busy(err)))
1730	return -EBUSY;	1710	return -EBUSY;
1731		1711
1732	if (KVM_BUG_ON(err, kvm)) {	1712	if (TDX_BUG_ON_2(err, TDH_MEM_SEPT_ADD, entry, level_state, kvm))
1733	pr_tdx_error_2(TDH_MEM_SEPT_ADD, err, entry, level_state);
1734	return -EIO;	1713	return -EIO;
1735	}
1736		1714
1737	return 0;	1715	return 0;
1738	}	1716	}
1739		1717
1740	/*
1741	* Check if the error returned from a SEPT zap SEAMCALL is due to that a page is
1742	* mapped by KVM_TDX_INIT_MEM_REGION without tdh_mem_page_add() being called
1743	* successfully.
1744	*
1745	* Since tdh_mem_sept_add() must have been invoked successfully before a
1746	* non-leaf entry present in the mirrored page table, the SEPT ZAP related
1747	* SEAMCALLs should not encounter err TDX_EPT_WALK_FAILED. They should instead
1748	* find TDX_EPT_ENTRY_STATE_INCORRECT due to an empty leaf entry found in the
1749	* SEPT.
1750	*
1751	* Further check if the returned entry from SEPT walking is with RWX permissions
1752	* to filter out anything unexpected.
1753	*
1754	* Note: @level is pg_level, not the tdx_level. The tdx_level extracted from
1755	* level_state returned from a SEAMCALL error is the same as that passed into
1756	* the SEAMCALL.
1757	*/
1758	static int tdx_is_sept_zap_err_due_to_premap(struct kvm_tdx *kvm_tdx, u64 err,
1759	u64 entry, int level)
1760	{
1761	if (!err \|\| kvm_tdx->state == TD_STATE_RUNNABLE)
1762	return false;
1763
1764	if (err != (TDX_EPT_ENTRY_STATE_INCORRECT \| TDX_OPERAND_ID_RCX))
1765	return false;
1766
1767	if ((is_last_spte(entry, level) && (entry & VMX_EPT_RWX_MASK)))
1768	return false;
1769
1770	return true;
1771	}
1772
1773	static int tdx_sept_zap_private_spte(struct kvm *kvm, gfn_t gfn,
1774	enum pg_level level, struct page *page)
1775	{
1776	int tdx_level = pg_level_to_tdx_sept_level(level);
1777	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
1778	gpa_t gpa = gfn_to_gpa(gfn) & KVM_HPAGE_MASK(level);
1779	u64 err, entry, level_state;
1780
1781	/* For now large page isn't supported yet. */
1782	WARN_ON_ONCE(level != PG_LEVEL_4K);
1783
1784	err = tdh_mem_range_block(&kvm_tdx->td, gpa, tdx_level, &entry, &level_state);
1785
1786	if (unlikely(tdx_operand_busy(err))) {
1787	/* After no vCPUs enter, the second retry is expected to succeed */
1788	tdx_no_vcpus_enter_start(kvm);
1789	err = tdh_mem_range_block(&kvm_tdx->td, gpa, tdx_level, &entry, &level_state);
1790	tdx_no_vcpus_enter_stop(kvm);
1791	}
1792	if (tdx_is_sept_zap_err_due_to_premap(kvm_tdx, err, entry, level) &&
1793	!KVM_BUG_ON(!atomic64_read(&kvm_tdx->nr_premapped), kvm)) {
1794	atomic64_dec(&kvm_tdx->nr_premapped);
1795	tdx_unpin(kvm, page);
1796	return 0;
1797	}
1798
1799	if (KVM_BUG_ON(err, kvm)) {
1800	pr_tdx_error_2(TDH_MEM_RANGE_BLOCK, err, entry, level_state);
1801	return -EIO;
1802	}
1803	return 1;
1804	}
1805
1806	/*	1718	/*
1807	* Ensure shared and private EPTs to be flushed on all vCPUs.	1719	* Ensure shared and private EPTs to be flushed on all vCPUs.
1808	* tdh_mem_track() is the only caller that increases TD epoch. An increase in	1720	* tdh_mem_track() is the only caller that increases TD epoch. An increase in
@@ -1836,18 +1748,15 @@ static void tdx_track(struct kvm *kvm) · 27 unmodified lines
1836	if (unlikely(kvm_tdx->state != TD_STATE_RUNNABLE))	1748	if (unlikely(kvm_tdx->state != TD_STATE_RUNNABLE))
1837	return;	1749	return;
1838		1750
		1751	/*
		1752	* The full sequence of TDH.MEM.TRACK and forcing vCPUs out of guest
		1753	* mode must be serialized, as TDH.MEM.TRACK will fail if the previous
		1754	* tracking epoch hasn't completed.
		1755	*/
1839	lockdep_assert_held_write(&kvm->mmu_lock);	1756	lockdep_assert_held_write(&kvm->mmu_lock);
1840		1757
1841	err = tdh_mem_track(&kvm_tdx->td);	1758	err = tdh_do_no_vcpus(tdh_mem_track, kvm, &kvm_tdx->td);
1842	if (unlikely(tdx_operand_busy(err))) {	1759	TDX_BUG_ON(err, TDH_MEM_TRACK, kvm);
1843	/* After no vCPUs enter, the second retry is expected to succeed */
1844	tdx_no_vcpus_enter_start(kvm);
1845	err = tdh_mem_track(&kvm_tdx->td);
1846	tdx_no_vcpus_enter_stop(kvm);
1847	}
1848
1849	if (KVM_BUG_ON(err, kvm))
1850	pr_tdx_error(TDH_MEM_TRACK, err);
1851		1760
1852	kvm_make_all_cpus_request(kvm, KVM_REQ_OUTSIDE_GUEST_MODE);	1761	kvm_make_all_cpus_request(kvm, KVM_REQ_OUTSIDE_GUEST_MODE);
1853	}	1762	}
@@ -1866,7 +1775,7 @@ static int tdx_sept_free_private_spt(struct kvm *kvm, gfn_t gfn, · 12 unmodified lines
1866	* and slot move/deletion.	1775	* and slot move/deletion.
1867	*/	1776	*/
1868	if (KVM_BUG_ON(is_hkid_assigned(kvm_tdx), kvm))	1777	if (KVM_BUG_ON(is_hkid_assigned(kvm_tdx), kvm))
1869	return -EINVAL;	1778	return -EIO;
1870		1779
1871	/*	1780	/*
1872	* The HKID assigned to this TD was already freed and cache was	1781	* The HKID assigned to this TD was already freed and cache was
@@ -1875,11 +1784,16 @@ static int tdx_sept_free_private_spt(struct kvm *kvm, gfn_t gfn, · 2 unmodified lines
1875	return tdx_reclaim_page(virt_to_page(private_spt));	1784	return tdx_reclaim_page(virt_to_page(private_spt));
1876	}	1785	}
1877		1786
1878	static int tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn,	1787	static void tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn,
1879	enum pg_level level, kvm_pfn_t pfn)	1788	enum pg_level level, u64 mirror_spte)
1880	{	1789	{
1881	struct page *page = pfn_to_page(pfn);	1790	struct page *page = pfn_to_page(spte_to_pfn(mirror_spte));
1882	int ret;	1791	int tdx_level = pg_level_to_tdx_sept_level(level);
		1792	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
		1793	gpa_t gpa = gfn_to_gpa(gfn);
		1794	u64 err, entry, level_state;
		1795
		1796	lockdep_assert_held_write(&kvm->mmu_lock);
1883		1797
1884	/*	1798	/*
1885	* HKID is released after all private pages have been removed, and set	1799	* HKID is released after all private pages have been removed, and set
@@ -1887,11 +1801,16 @@ static int tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn, · 1 unmodified line
1887	* there can't be anything populated in the private EPT.	1801	* there can't be anything populated in the private EPT.
1888	*/	1802	*/
1889	if (KVM_BUG_ON(!is_hkid_assigned(to_kvm_tdx(kvm)), kvm))	1803	if (KVM_BUG_ON(!is_hkid_assigned(to_kvm_tdx(kvm)), kvm))
1890	return -EINVAL;	1804	return;
1891		1805
1892	ret = tdx_sept_zap_private_spte(kvm, gfn, level, page);	1806	/* TODO: handle large pages. */
1893	if (ret <= 0)	1807	if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))
1894	return ret;	1808	return;
		1809
		1810	err = tdh_do_no_vcpus(tdh_mem_range_block, kvm, &kvm_tdx->td, gpa,
		1811	tdx_level, &entry, &level_state);
		1812	if (TDX_BUG_ON_2(err, TDH_MEM_RANGE_BLOCK, entry, level_state, kvm))
		1813	return;
1895		1814
1896	/*	1815	/*
1897	* TDX requires TLB tracking before dropping private page. Do	1816	* TDX requires TLB tracking before dropping private page. Do
@@ -1899,7 +1818,21 @@ static int tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn, · 1 unmodified line
1899	*/	1818	*/
1900	tdx_track(kvm);	1819	tdx_track(kvm);
1901		1820
1902	return tdx_sept_drop_private_spte(kvm, gfn, level, page);	1821	/*
		1822	* When zapping private page, write lock is held. So no race condition
		1823	* with other vcpu sept operation.
		1824	* Race with TDH.VP.ENTER due to (0-step mitigation) and Guest TDCALLs.
		1825	*/
		1826	err = tdh_do_no_vcpus(tdh_mem_page_remove, kvm, &kvm_tdx->td, gpa,
		1827	tdx_level, &entry, &level_state);
		1828	if (TDX_BUG_ON_2(err, TDH_MEM_PAGE_REMOVE, entry, level_state, kvm))
		1829	return;
		1830
		1831	err = tdh_phymem_page_wbinvd_hkid((u16)kvm_tdx->hkid, page);
		1832	if (TDX_BUG_ON(err, TDH_PHYMEM_PAGE_WBINVD, kvm))
		1833	return;
		1834
		1835	tdx_quirk_reset_page(page);
1903	}	1836	}
1904		1837
1905	void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode,	1838	void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode,
@@ -2145,11 +2078,7 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) · 239 unmodified lines
2145	}	2078	}
2146		2079
2147	unhandled_exit:	2080	unhandled_exit:
2148	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;	2081	kvm_prepare_unexpected_reason_exit(vcpu, vp_enter_ret);
2149	vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
2150	vcpu->run->internal.ndata = 2;
2151	vcpu->run->internal.data[0] = vp_enter_ret;
2152	vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
2153	return 0;	2082	return 0;
2154	}	2083	}
2155		2084
@@ -2282,37 +2211,28 @@ static int tdx_get_capabilities(struct kvm_tdx_cmd *cmd) · 126 unmodified lines
2282	if (cmd->flags)	2211	if (cmd->flags)
2283	return -EINVAL;	2212	return -EINVAL;
2284		2213
2285	caps = kzalloc(sizeof(*caps) +
2286	sizeof(struct kvm_cpuid_entry2) * td_conf->num_cpuid_config,
2287	GFP_KERNEL);
2288	if (!caps)
2289	return -ENOMEM;
2290
2291	user_caps = u64_to_user_ptr(cmd->data);	2214	user_caps = u64_to_user_ptr(cmd->data);
2292	if (get_user(nr_user_entries, &user_caps->cpuid.nent)) {	2215	if (get_user(nr_user_entries, &user_caps->cpuid.nent))
2293	ret = -EFAULT;	2216	return -EFAULT;
2294	goto out;
2295	}
2296		2217
2297	if (nr_user_entries < td_conf->num_cpuid_config) {	2218	if (nr_user_entries < td_conf->num_cpuid_config)
2298	ret = -E2BIG;	2219	return -E2BIG;
2299	goto out;	2220
2300	}	2221	caps = kzalloc(struct_size(caps, cpuid.entries,
		2222	td_conf->num_cpuid_config), GFP_KERNEL);
		2223	if (!caps)
		2224	return -ENOMEM;
2301		2225
2302	ret = init_kvm_tdx_caps(td_conf, caps);	2226	ret = init_kvm_tdx_caps(td_conf, caps);
2303	if (ret)	2227	if (ret)
2304	goto out;	2228	goto out;
2305		2229
2306	if (copy_to_user(user_caps, caps, sizeof(*caps))) {	2230	if (copy_to_user(user_caps, caps, struct_size(caps, cpuid.entries,
		2231	caps->cpuid.nent))) {
2307	ret = -EFAULT;	2232	ret = -EFAULT;
2308	goto out;	2233	goto out;
2309	}	2234	}
2310		2235
2311	if (copy_to_user(user_caps->cpuid.entries, caps->cpuid.entries,
2312	caps->cpuid.nent *
2313	sizeof(caps->cpuid.entries[0])))
2314	ret = -EFAULT;
2315
2316	out:	2236	out:
2317	/* kfree() accepts NULL. */	2237	/* kfree() accepts NULL. */
2318	kfree(caps);	2238	kfree(caps);
@@ -2537,8 +2457,7 @@ static int __tdx_td_init(struct kvm kvm, struct td_params td_params, · 218 unmodified lines
2537	goto free_packages;	2457	goto free_packages;
2538	}	2458	}
2539		2459
2540	if (WARN_ON_ONCE(err)) {	2460	if (TDX_BUG_ON(err, TDH_MNG_CREATE, kvm)) {
2541	pr_tdx_error(TDH_MNG_CREATE, err);
2542	ret = -EIO;	2461	ret = -EIO;
2543	goto free_packages;	2462	goto free_packages;
2544	}	2463	}
@@ -2579,8 +2498,7 @@ static int __tdx_td_init(struct kvm kvm, struct td_params td_params, · 34 unmodified lines
2579	ret = -EAGAIN;	2498	ret = -EAGAIN;
2580	goto teardown;	2499	goto teardown;
2581	}	2500	}
2582	if (WARN_ON_ONCE(err)) {	2501	if (TDX_BUG_ON(err, TDH_MNG_ADDCX, kvm)) {
2583	pr_tdx_error(TDH_MNG_ADDCX, err);
2584	ret = -EIO;	2502	ret = -EIO;
2585	goto teardown;	2503	goto teardown;
2586	}	2504	}
@@ -2597,8 +2515,7 @@ static int __tdx_td_init(struct kvm kvm, struct td_params td_params, · 10 unmodified lines
2597	*seamcall_err = err;	2515	*seamcall_err = err;
2598	ret = -EINVAL;	2516	ret = -EINVAL;
2599	goto teardown;	2517	goto teardown;
2600	} else if (WARN_ON_ONCE(err)) {	2518	} else if (TDX_BUG_ON_1(err, TDH_MNG_INIT, rcx, kvm)) {
2601	pr_tdx_error_1(TDH_MNG_INIT, err, rcx);
2602	ret = -EIO;	2519	ret = -EIO;
2603	goto teardown;	2520	goto teardown;
2604	}	2521	}
@@ -2642,7 +2559,7 @@ free_tdcs: · 37 unmodified lines
2642	free_tdr:	2559	free_tdr:
2643	if (tdr_page)	2560	if (tdr_page)
2644	__free_page(tdr_page);	2561	__free_page(tdr_page);
2645	kvm_tdx->td.tdr_page = 0;	2562	kvm_tdx->td.tdr_page = NULL;
2646		2563
2647	free_hkid:	2564	free_hkid:
2648	tdx_hkid_free(kvm_tdx);	2565	tdx_hkid_free(kvm_tdx);
@@ -2747,11 +2664,53 @@ err_out: · 98 unmodified lines
2747	return -EIO;	2664	return -EIO;
2748	}	2665	}
2749		2666
		2667	typedef void *tdx_vm_state_guard_t;
		2668
		2669	static tdx_vm_state_guard_t tdx_acquire_vm_state_locks(struct kvm *kvm)
		2670	{
		2671	int r;
		2672
		2673	mutex_lock(&kvm->lock);
		2674
		2675	if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus)) {
		2676	r = -EBUSY;
		2677	goto out_err;
		2678	}
		2679
		2680	r = kvm_lock_all_vcpus(kvm);
		2681	if (r)
		2682	goto out_err;
		2683
		2684	/*
		2685	* Note the unintuitive ordering! vcpu->mutex must be taken outside
		2686	* kvm->slots_lock!
		2687	*/
		2688	mutex_lock(&kvm->slots_lock);
		2689	return kvm;
		2690
		2691	out_err:
		2692	mutex_unlock(&kvm->lock);
		2693	return ERR_PTR(r);
		2694	}
		2695
		2696	static void tdx_release_vm_state_locks(struct kvm *kvm)
		2697	{
		2698	mutex_unlock(&kvm->slots_lock);
		2699	kvm_unlock_all_vcpus(kvm);
		2700	mutex_unlock(&kvm->lock);
		2701	}
		2702
		2703	DEFINE_CLASS(tdx_vm_state_guard, tdx_vm_state_guard_t,
		2704	if (!IS_ERR(_T)) tdx_release_vm_state_locks(_T),
		2705	tdx_acquire_vm_state_locks(kvm), struct kvm *kvm);
		2706
2750	static int tdx_td_init(struct kvm kvm, struct kvm_tdx_cmd cmd)	2707	static int tdx_td_init(struct kvm kvm, struct kvm_tdx_cmd cmd)
2751	{	2708	{
		2709	struct kvm_tdx_init_vm __user *user_data = u64_to_user_ptr(cmd->data);
2752	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);	2710	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
2753	struct kvm_tdx_init_vm *init_vm;	2711	struct kvm_tdx_init_vm *init_vm;
2754	struct td_params *td_params = NULL;	2712	struct td_params *td_params = NULL;
		2713	u32 nr_user_entries;
2755	int ret;	2714	int ret;
2756		2715
2757	BUILD_BUG_ON(sizeof(*init_vm) != 256 + sizeof_field(struct kvm_tdx_init_vm, cpuid));	2716	BUILD_BUG_ON(sizeof(*init_vm) != 256 + sizeof_field(struct kvm_tdx_init_vm, cpuid));
@@ -2763,28 +2722,16 @@ static int tdx_td_init(struct kvm kvm, struct kvm_tdx_cmd cmd) · 5 unmodified lines
2763	if (cmd->flags)	2722	if (cmd->flags)
2764	return -EINVAL;	2723	return -EINVAL;
2765		2724
2766	init_vm = kmalloc(sizeof(*init_vm) +	2725	if (get_user(nr_user_entries, &user_data->cpuid.nent))
2767	sizeof(init_vm->cpuid.entries[0]) * KVM_MAX_CPUID_ENTRIES,	2726	return -EFAULT;
2768	GFP_KERNEL);
2769	if (!init_vm)
2770	return -ENOMEM;
2771
2772	if (copy_from_user(init_vm, u64_to_user_ptr(cmd->data), sizeof(*init_vm))) {
2773	ret = -EFAULT;
2774	goto out;
2775	}
2776		2727
2777	if (init_vm->cpuid.nent > KVM_MAX_CPUID_ENTRIES) {	2728	if (nr_user_entries > KVM_MAX_CPUID_ENTRIES)
2778	ret = -E2BIG;	2729	return -E2BIG;
2779	goto out;
2780	}
2781		2730
2782	if (copy_from_user(init_vm->cpuid.entries,	2731	init_vm = memdup_user(user_data,
2783	u64_to_user_ptr(cmd->data) + sizeof(*init_vm),	2732	struct_size(user_data, cpuid.entries, nr_user_entries));
2784	flex_array_size(init_vm, cpuid.entries, init_vm->cpuid.nent))) {	2733	if (IS_ERR(init_vm))
2785	ret = -EFAULT;	2734	return PTR_ERR(init_vm);
2786	goto out;
2787	}
2788		2735
2789	if (memchr_inv(init_vm->reserved, 0, sizeof(init_vm->reserved))) {	2736	if (memchr_inv(init_vm->reserved, 0, sizeof(init_vm->reserved))) {
2790	ret = -EINVAL;	2737	ret = -EINVAL;
@@ -2868,24 +2815,14 @@ static int tdx_td_finalize(struct kvm kvm, struct kvm_tdx_cmd cmd) · 77 unmodified lines
2868	{	2815	{
2869	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);	2816	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
2870		2817
2871	guard(mutex)(&kvm->slots_lock);
2872
2873	if (!is_hkid_assigned(kvm_tdx) \|\| kvm_tdx->state == TD_STATE_RUNNABLE)	2818	if (!is_hkid_assigned(kvm_tdx) \|\| kvm_tdx->state == TD_STATE_RUNNABLE)
2874	return -EINVAL;	2819	return -EINVAL;
2875	/*
2876	* Pages are pending for KVM_TDX_INIT_MEM_REGION to issue
2877	* TDH.MEM.PAGE.ADD().
2878	*/
2879	if (atomic64_read(&kvm_tdx->nr_premapped))
2880	return -EINVAL;
2881		2820
2882	cmd->hw_error = tdh_mr_finalize(&kvm_tdx->td);	2821	cmd->hw_error = tdh_mr_finalize(&kvm_tdx->td);
2883	if (tdx_operand_busy(cmd->hw_error))	2822	if (tdx_operand_busy(cmd->hw_error))
2884	return -EBUSY;	2823	return -EBUSY;
2885	if (KVM_BUG_ON(cmd->hw_error, kvm)) {	2824	if (TDX_BUG_ON(cmd->hw_error, TDH_MR_FINALIZE, kvm))
2886	pr_tdx_error(TDH_MR_FINALIZE, cmd->hw_error);
2887	return -EIO;	2825	return -EIO;
2888	}
2889		2826
2890	kvm_tdx->state = TD_STATE_RUNNABLE;	2827	kvm_tdx->state = TD_STATE_RUNNABLE;
2891	/* TD_STATE_RUNNABLE must be set before 'pre_fault_allowed' */	2828	/* TD_STATE_RUNNABLE must be set before 'pre_fault_allowed' */
@@ -2894,27 +2831,38 @@ static int tdx_td_finalize(struct kvm kvm, struct kvm_tdx_cmd cmd) · 2 unmodified lines
2894	return 0;	2831	return 0;
2895	}	2832	}
2896		2833
2897	int tdx_vm_ioctl(struct kvm kvm, void __user argp)	2834	static int tdx_get_cmd(void __user argp, struct kvm_tdx_cmd cmd)
2898	{	2835	{
2899	struct kvm_tdx_cmd tdx_cmd;	2836	if (copy_from_user(cmd, argp, sizeof(*cmd)))
2900	int r;
2901
2902	if (copy_from_user(&tdx_cmd, argp, sizeof(struct kvm_tdx_cmd)))
2903	return -EFAULT;	2837	return -EFAULT;
2904		2838
2905	/*	2839	/*
2906	* Userspace should never set hw_error. It is used to fill	2840	* Userspace should never set hw_error. KVM writes hw_error to report
2907	* hardware-defined error by the kernel.	2841	* hardware-defined error back to userspace.
2908	*/	2842	*/
2909	if (tdx_cmd.hw_error)	2843	if (cmd->hw_error)
2910	return -EINVAL;	2844	return -EINVAL;
2911		2845
2912	mutex_lock(&kvm->lock);	2846	return 0;
		2847	}
		2848
		2849	int tdx_vm_ioctl(struct kvm kvm, void __user argp)
		2850	{
		2851	struct kvm_tdx_cmd tdx_cmd;
		2852	int r;
		2853
		2854	r = tdx_get_cmd(argp, &tdx_cmd);
		2855	if (r)
		2856	return r;
		2857
		2858	if (tdx_cmd.id == KVM_TDX_CAPABILITIES)
		2859	return tdx_get_capabilities(&tdx_cmd);
		2860
		2861	CLASS(tdx_vm_state_guard, guard)(kvm);
		2862	if (IS_ERR(guard))
		2863	return PTR_ERR(guard);
2913		2864
2914	switch (tdx_cmd.id) {	2865	switch (tdx_cmd.id) {
2915	case KVM_TDX_CAPABILITIES:
2916	r = tdx_get_capabilities(&tdx_cmd);
2917	break;
2918	case KVM_TDX_INIT_VM:	2866	case KVM_TDX_INIT_VM:
2919	r = tdx_td_init(kvm, &tdx_cmd);	2867	r = tdx_td_init(kvm, &tdx_cmd);
2920	break;	2868	break;
@@ -2922,15 +2870,12 @@ int tdx_vm_ioctl(struct kvm kvm, void __user argp) · 1 unmodified line
2922	r = tdx_td_finalize(kvm, &tdx_cmd);	2870	r = tdx_td_finalize(kvm, &tdx_cmd);
2923	break;	2871	break;
2924	default:	2872	default:
2925	r = -EINVAL;	2873	return -EINVAL;
2926	goto out;
2927	}	2874	}
2928		2875
2929	if (copy_to_user(argp, &tdx_cmd, sizeof(struct kvm_tdx_cmd)))	2876	if (copy_to_user(argp, &tdx_cmd, sizeof(struct kvm_tdx_cmd)))
2930	r = -EFAULT;	2877	return -EFAULT;
2931		2878
2932	out:
2933	mutex_unlock(&kvm->lock);
2934	return r;	2879	return r;
2935	}	2880	}
2936		2881
@@ -2972,16 +2917,14 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx) · 35 unmodified lines
2972	}	2917	}
2973		2918
2974	err = tdh_vp_create(&kvm_tdx->td, &tdx->vp);	2919	err = tdh_vp_create(&kvm_tdx->td, &tdx->vp);
2975	if (KVM_BUG_ON(err, vcpu->kvm)) {	2920	if (TDX_BUG_ON(err, TDH_VP_CREATE, vcpu->kvm)) {
2976	ret = -EIO;	2921	ret = -EIO;
2977	pr_tdx_error(TDH_VP_CREATE, err);
2978	goto free_tdcx;	2922	goto free_tdcx;
2979	}	2923	}
2980		2924
2981	for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) {	2925	for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) {
2982	err = tdh_vp_addcx(&tdx->vp, tdx->vp.tdcx_pages[i]);	2926	err = tdh_vp_addcx(&tdx->vp, tdx->vp.tdcx_pages[i]);
2983	if (KVM_BUG_ON(err, vcpu->kvm)) {	2927	if (TDX_BUG_ON(err, TDH_VP_ADDCX, vcpu->kvm)) {
2984	pr_tdx_error(TDH_VP_ADDCX, err);
2985	/*	2928	/*
2986	* Pages already added are reclaimed by the vcpu_free	2929	* Pages already added are reclaimed by the vcpu_free
2987	* method, but the rest are freed here.	2930	* method, but the rest are freed here.
@@ -2994,10 +2937,19 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx) · 6 unmodified lines
2994	}	2937	}
2995	}	2938	}
2996		2939
2997	err = tdh_vp_init(&tdx->vp, vcpu_rcx, vcpu->vcpu_id);	2940	/*
2998	if (KVM_BUG_ON(err, vcpu->kvm)) {	2941	* tdh_vp_init() can take an exclusive lock of the TDR resource inside
2999	pr_tdx_error(TDH_VP_INIT, err);	2942	* the TDX-Module. The TDR resource is also taken as shared in several
3000	return -EIO;	2943	* no-fail MMU paths, which could return TDX_OPERAND_BUSY on contention
		2944	* (TDX-Module locks are try-lock implementations with no slow path).
		2945	* Take mmu_lock for write to reflect the nature of the lock taken by
		2946	* the TDX-Module, and to ensure the no-fail MMU paths succeed, e.g. if
		2947	* a concurrent PUNCH_HOLE on guest_memfd triggers removal of SPTEs.
		2948	*/
		2949	scoped_guard(write_lock, &vcpu->kvm->mmu_lock) {
		2950	err = tdh_vp_init(&tdx->vp, vcpu_rcx, vcpu->vcpu_id);
		2951	if (TDX_BUG_ON(err, TDH_VP_INIT, vcpu->kvm))
		2952	return -EIO;
3001	}	2953	}
3002		2954
3003	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;	2955	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
@@ -3016,7 +2968,7 @@ free_tdcx: · 12 unmodified lines
3016	free_tdvpr:	2968	free_tdvpr:
3017	if (tdx->vp.tdvpr_page)	2969	if (tdx->vp.tdvpr_page)
3018	__free_page(tdx->vp.tdvpr_page);	2970	__free_page(tdx->vp.tdvpr_page);
3019	tdx->vp.tdvpr_page = 0;	2971	tdx->vp.tdvpr_page = NULL;
3020	tdx->vp.tdvpr_pa = 0;	2972	tdx->vp.tdvpr_pa = 0;
3021		2973
3022	return ret;	2974	return ret;
@@ -3054,7 +3006,8 @@ static int tdx_vcpu_get_cpuid_leaf(struct kvm_vcpu vcpu, u32 leaf, int entry_i · 31 unmodified lines
3054		3006
3055	static int tdx_vcpu_get_cpuid(struct kvm_vcpu vcpu, struct kvm_tdx_cmd cmd)	3007	static int tdx_vcpu_get_cpuid(struct kvm_vcpu vcpu, struct kvm_tdx_cmd cmd)
3056	{	3008	{
3057	struct kvm_cpuid2 __user output, td_cpuid;	3009	struct kvm_cpuid2 __user *output;
		3010	struct kvm_cpuid2 *td_cpuid;
3058	int r = 0, i = 0, leaf;	3011	int r = 0, i = 0, leaf;
3059	u32 level;	3012	u32 level;
3060		3013
@@ -3167,15 +3120,15 @@ struct tdx_gmem_post_populate_arg { · 106 unmodified lines
3167	static int tdx_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,	3120	static int tdx_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
3168	void __user src, int order, void _arg)	3121	void __user src, int order, void _arg)
3169	{	3122	{
3170	u64 error_code = PFERR_GUEST_FINAL_MASK \| PFERR_PRIVATE_ACCESS;
3171	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
3172	struct tdx_gmem_post_populate_arg *arg = _arg;	3123	struct tdx_gmem_post_populate_arg *arg = _arg;
3173	struct kvm_vcpu *vcpu = arg->vcpu;	3124	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
		3125	u64 err, entry, level_state;
3174	gpa_t gpa = gfn_to_gpa(gfn);	3126	gpa_t gpa = gfn_to_gpa(gfn);
3175	u8 level = PG_LEVEL_4K;
3176	struct page *src_page;	3127	struct page *src_page;
3177	int ret, i;	3128	int ret, i;
3178	u64 err, entry, level_state;	3129
		3130	if (KVM_BUG_ON(kvm_tdx->page_add_src, kvm))
		3131	return -EIO;
3179		3132
3180	/*	3133	/*
3181	* Get the source page if it has been faulted in. Return failure if the	3134	* Get the source page if it has been faulted in. Return failure if the
@@ -3187,49 +3140,29 @@ static int tdx_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, · 5 unmodified lines
3187	if (ret != 1)	3140	if (ret != 1)
3188	return -ENOMEM;	3141	return -ENOMEM;
3189		3142
3190	ret = kvm_tdp_map_page(vcpu, gpa, error_code, &level);	3143	kvm_tdx->page_add_src = src_page;
3191	if (ret < 0)	3144	ret = kvm_tdp_mmu_map_private_pfn(arg->vcpu, gfn, pfn);
3192	goto out;	3145	kvm_tdx->page_add_src = NULL;
3193
3194	/*
3195	* The private mem cannot be zapped after kvm_tdp_map_page()
3196	* because all paths are covered by slots_lock and the
3197	* filemap invalidate lock. Check that they are indeed enough.
3198	*/
3199	if (IS_ENABLED(CONFIG_KVM_PROVE_MMU)) {
3200	scoped_guard(read_lock, &kvm->mmu_lock) {
3201	if (KVM_BUG_ON(!kvm_tdp_mmu_gpa_is_mapped(vcpu, gpa), kvm)) {
3202	ret = -EIO;
3203	goto out;
3204	}
3205	}
3206	}
3207		3146
3208	ret = 0;	3147	put_page(src_page);
3209	err = tdh_mem_page_add(&kvm_tdx->td, gpa, pfn_to_page(pfn),
3210	src_page, &entry, &level_state);
3211	if (err) {
3212	ret = unlikely(tdx_operand_busy(err)) ? -EBUSY : -EIO;
3213	goto out;
3214	}
3215		3148
3216	if (!KVM_BUG_ON(!atomic64_read(&kvm_tdx->nr_premapped), kvm))	3149	if (ret \|\| !(arg->flags & KVM_TDX_MEASURE_MEMORY_REGION))
3217	atomic64_dec(&kvm_tdx->nr_premapped);	3150	return ret;
3218		3151
3219	if (arg->flags & KVM_TDX_MEASURE_MEMORY_REGION) {	3152	/*
3220	for (i = 0; i < PAGE_SIZE; i += TDX_EXTENDMR_CHUNKSIZE) {	3153	* Note, MR.EXTEND can fail if the S-EPT mapping is somehow removed
3221	err = tdh_mr_extend(&kvm_tdx->td, gpa + i, &entry,	3154	* between mapping the pfn and now, but slots_lock prevents memslot
3222	&level_state);	3155	* updates, filemap_invalidate_lock() prevents guest_memfd updates,
3223	if (err) {	3156	* mmu_notifier events can't reach S-EPT entries, and KVM's internal
3224	ret = -EIO;	3157	* zapping flows are mutually exclusive with S-EPT mappings.
3225	break;	3158	*/
3226	}	3159	for (i = 0; i < PAGE_SIZE; i += TDX_EXTENDMR_CHUNKSIZE) {
3227	}	3160	err = tdh_mr_extend(&kvm_tdx->td, gpa + i, &entry, &level_state);
		3161	if (TDX_BUG_ON_2(err, TDH_MR_EXTEND, entry, level_state, kvm))
		3162	return -EIO;
3228	}	3163	}
3229		3164
3230	out:	3165	return 0;
3231	put_page(src_page);
3232	return ret;
3233	}	3166	}
3234		3167
3235	static int tdx_vcpu_init_mem_region(struct kvm_vcpu vcpu, struct kvm_tdx_cmd cmd)	3168	static int tdx_vcpu_init_mem_region(struct kvm_vcpu vcpu, struct kvm_tdx_cmd cmd)
@@ -3245,8 +3178,6 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu vcpu, struct kvm_tdx_cmd c · 9 unmodified lines
3245	if (tdx->state != VCPU_TD_STATE_INITIALIZED)	3178	if (tdx->state != VCPU_TD_STATE_INITIALIZED)
3246	return -EINVAL;	3179	return -EINVAL;
3247		3180
3248	guard(mutex)(&kvm->slots_lock);
3249
3250	/* Once TD is finalized, the initial guest memory is fixed. */	3181	/* Once TD is finalized, the initial guest memory is fixed. */
3251	if (kvm_tdx->state == TD_STATE_RUNNABLE)	3182	if (kvm_tdx->state == TD_STATE_RUNNABLE)
3252	return -EINVAL;	3183	return -EINVAL;
@@ -3264,7 +3195,6 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu vcpu, struct kvm_tdx_cmd c · 11 unmodified lines
3264	!vt_is_tdx_private_gpa(kvm, region.gpa + (region.nr_pages << PAGE_SHIFT) - 1))	3195	!vt_is_tdx_private_gpa(kvm, region.gpa + (region.nr_pages << PAGE_SHIFT) - 1))
3265	return -EINVAL;	3196	return -EINVAL;
3266		3197
3267	kvm_mmu_reload(vcpu);
3268	ret = 0;	3198	ret = 0;
3269	while (region.nr_pages) {	3199	while (region.nr_pages) {
3270	if (signal_pending(current)) {	3200	if (signal_pending(current)) {
@@ -3301,28 +3231,57 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu vcpu, struct kvm_tdx_cmd c · 30 unmodified lines
3301	return ret;	3231	return ret;
3302	}	3232	}
3303		3233
3304	int tdx_vcpu_ioctl(struct kvm_vcpu vcpu, void __user argp)	3234	int tdx_vcpu_unlocked_ioctl(struct kvm_vcpu vcpu, void __user argp)
3305	{	3235	{
3306	struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);	3236	struct kvm *kvm = vcpu->kvm;
		3237	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
3307	struct kvm_tdx_cmd cmd;	3238	struct kvm_tdx_cmd cmd;
3308	int ret;	3239	int r;
3309		3240
3310	if (!is_hkid_assigned(kvm_tdx) \|\| kvm_tdx->state == TD_STATE_RUNNABLE)	3241	r = tdx_get_cmd(argp, &cmd);
3311	return -EINVAL;	3242	if (r)
		3243	return r;
3312		3244
3313	if (copy_from_user(&cmd, argp, sizeof(cmd)))	3245	CLASS(tdx_vm_state_guard, guard)(kvm);
3314	return -EFAULT;	3246	if (IS_ERR(guard))
		3247	return PTR_ERR(guard);
3315		3248
3316	if (cmd.hw_error)	3249	if (!is_hkid_assigned(kvm_tdx) \|\| kvm_tdx->state == TD_STATE_RUNNABLE)
3317	return -EINVAL;	3250	return -EINVAL;
3318		3251
		3252	vcpu_load(vcpu);
		3253
3319	switch (cmd.id) {	3254	switch (cmd.id) {
		3255	case KVM_TDX_INIT_MEM_REGION:
		3256	r = tdx_vcpu_init_mem_region(vcpu, &cmd);
		3257	break;
3320	case KVM_TDX_INIT_VCPU:	3258	case KVM_TDX_INIT_VCPU:
3321	ret = tdx_vcpu_init(vcpu, &cmd);	3259	r = tdx_vcpu_init(vcpu, &cmd);
3322	break;	3260	break;
3323	case KVM_TDX_INIT_MEM_REGION:	3261	default:
3324	ret = tdx_vcpu_init_mem_region(vcpu, &cmd);	3262	r = -ENOIOCTLCMD;
3325	break;	3263	break;
		3264	}
		3265
		3266	vcpu_put(vcpu);
		3267
		3268	return r;
		3269	}
		3270
		3271	int tdx_vcpu_ioctl(struct kvm_vcpu vcpu, void __user argp)
		3272	{
		3273	struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);
		3274	struct kvm_tdx_cmd cmd;
		3275	int ret;
		3276
		3277	if (!is_hkid_assigned(kvm_tdx) \|\| kvm_tdx->state == TD_STATE_RUNNABLE)
		3278	return -EINVAL;
		3279
		3280	ret = tdx_get_cmd(argp, &cmd);
		3281	if (ret)
		3282	return ret;
		3283
		3284	switch (cmd.id) {
3326	case KVM_TDX_GET_CPUID:	3285	case KVM_TDX_GET_CPUID:
3327	ret = tdx_vcpu_get_cpuid(vcpu, &cmd);	3286	ret = tdx_vcpu_get_cpuid(vcpu, &cmd);
3328	break;	3287	break;
@@ -3447,10 +3406,6 @@ static int __init __tdx_bringup(void) · 118 unmodified lines
3447	/*	3406	/*
3448	* Check if MSRs (tdx_uret_msrs) can be saved/restored	3407	* Check if MSRs (tdx_uret_msrs) can be saved/restored
3449	* before returning to user space.	3408	* before returning to user space.
3450	*
3451	* this_cpu_ptr(user_return_msrs)->registered isn't checked
3452	* because the registration is done at vcpu runtime by
3453	* tdx_user_return_msr_update_cache().
3454	*/	3409	*/
3455	tdx_uret_msrs[i].slot = kvm_find_user_return_msr(tdx_uret_msrs[i].msr);	3410	tdx_uret_msrs[i].slot = kvm_find_user_return_msr(tdx_uret_msrs[i].msr);
3456	if (tdx_uret_msrs[i].slot == -1) {	3411	if (tdx_uret_msrs[i].slot == -1) {

arch/x86/kvm/vmx/tdx.h+6−3

 	struct tdx_td td;
-	/* For KVM_TDX_INIT_MEM_REGION. */
+	/*
-	atomic64_t nr_premapped;
+	 * Scratch pointer used to pass the source page to tdx_mem_page_add().
+	 * Protected by slots_lock, and non-NULL only when mapping a private
+	 * pfn via tdx_gmem_post_populate().
+	 */
+	struct page *page_add_src;
 	/*
 	 * Prevent vCPUs from TD entry to ensure SEPT zap related SEAMCALLs do
 	u64 vp_enter_ret;
 	enum vcpu_tdx_state state;
-	bool guest_entered;
 	u64 map_gpa_next;
 	u64 map_gpa_end;

arch/x86/kvm/vmx/vmenter.S+34−17

@@ -71,6 +71,7 @@ · 70 unmodified lines
71	* @regs: unsigned long * (to guest registers)	71	* @regs: unsigned long * (to guest registers)
72	* @flags: VMX_RUN_VMRESUME: use VMRESUME instead of VMLAUNCH	72	* @flags: VMX_RUN_VMRESUME: use VMRESUME instead of VMLAUNCH
73	* VMX_RUN_SAVE_SPEC_CTRL: save guest SPEC_CTRL into vmx->spec_ctrl	73	* VMX_RUN_SAVE_SPEC_CTRL: save guest SPEC_CTRL into vmx->spec_ctrl
		74	* VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO: vCPU can access host MMIO
74	*	75	*
75	* Returns:	76	* Returns:
76	* 0 on VM-Exit, 1 on VM-Fail	77	* 0 on VM-Exit, 1 on VM-Fail
@@ -92,7 +93,7 @@ SYM_FUNC_START(__vmx_vcpu_run) · 15 unmodified lines
92	/* Save @vmx for SPEC_CTRL handling */	93	/* Save @vmx for SPEC_CTRL handling */
93	push %_ASM_ARG1	94	push %_ASM_ARG1
94		95
95	/* Save @flags for SPEC_CTRL handling */	96	/* Save @flags (used for VMLAUNCH vs. VMRESUME and mitigations). */
96	push %_ASM_ARG3	97	push %_ASM_ARG3
97		98
98	/*	99	/*
@@ -101,9 +102,6 @@ SYM_FUNC_START(__vmx_vcpu_run) · 2 unmodified lines
101	*/	102	*/
102	push %_ASM_ARG2	103	push %_ASM_ARG2
103		104
104	/* Copy @flags to EBX, _ASM_ARG3 is volatile. */
105	mov %_ASM_ARG3L, %ebx
106
107	lea (%_ASM_SP), %_ASM_ARG2	105	lea (%_ASM_SP), %_ASM_ARG2
108	call vmx_update_host_rsp	106	call vmx_update_host_rsp
109		107
@@ -118,13 +116,23 @@ SYM_FUNC_START(__vmx_vcpu_run) · 8 unmodified lines
118	* and vmentry.	116	* and vmentry.
119	*/	117	*/
120	mov 2*WORD_SIZE(%_ASM_SP), %_ASM_DI	118	mov 2*WORD_SIZE(%_ASM_SP), %_ASM_DI
121	movl VMX_spec_ctrl(%_ASM_DI), %edi	119	#ifdef CONFIG_X86_64
122	movl PER_CPU_VAR(x86_spec_ctrl_current), %esi	120	mov VMX_spec_ctrl(%rdi), %rdx
123	cmp %edi, %esi	121	cmp PER_CPU_VAR(x86_spec_ctrl_current), %rdx
124	je .Lspec_ctrl_done	122	je .Lspec_ctrl_done
		123	movl %edx, %eax
		124	shr $32, %rdx
		125	#else
		126	mov VMX_spec_ctrl(%edi), %eax
		127	mov PER_CPU_VAR(x86_spec_ctrl_current), %ecx
		128	xor %eax, %ecx
		129	mov VMX_spec_ctrl + 4(%edi), %edx
		130	mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %edi
		131	xor %edx, %edi
		132	or %edi, %ecx
		133	je .Lspec_ctrl_done
		134	#endif
125	mov $MSR_IA32_SPEC_CTRL, %ecx	135	mov $MSR_IA32_SPEC_CTRL, %ecx
126	xor %edx, %edx
127	mov %edi, %eax
128	wrmsr	136	wrmsr
129		137
130	.Lspec_ctrl_done:	138	.Lspec_ctrl_done:
@@ -137,9 +145,6 @@ SYM_FUNC_START(__vmx_vcpu_run) · 6 unmodified lines
137	/* Load @regs to RAX. */	145	/* Load @regs to RAX. */
138	mov (%_ASM_SP), %_ASM_AX	146	mov (%_ASM_SP), %_ASM_AX
139		147
140	/* Check if vmlaunch or vmresume is needed */
141	bt $VMX_RUN_VMRESUME_SHIFT, %ebx
142
143	/* Load guest registers. Don't clobber flags. */	148	/* Load guest registers. Don't clobber flags. */
144	mov VCPU_RCX(%_ASM_AX), %_ASM_CX	149	mov VCPU_RCX(%_ASM_AX), %_ASM_CX
145	mov VCPU_RDX(%_ASM_AX), %_ASM_DX	150	mov VCPU_RDX(%_ASM_AX), %_ASM_DX
@@ -160,11 +165,23 @@ SYM_FUNC_START(__vmx_vcpu_run) · 14 unmodified lines
160	/* Load guest RAX. This kills the @regs pointer! */	165	/* Load guest RAX. This kills the @regs pointer! */
161	mov VCPU_RAX(%_ASM_AX), %_ASM_AX	166	mov VCPU_RAX(%_ASM_AX), %_ASM_AX
162		167
163	/* Clobbers EFLAGS.ZF */	168	/*
164	CLEAR_CPU_BUFFERS	169	* Note, ALTERNATIVE_2 works in reverse order. If CLEAR_CPU_BUF_VM is
165		170	* enabled, do VERW unconditionally. If CPU_BUF_VM_MMIO is enabled,
166	/* Check EFLAGS.CF from the VMX_RUN_VMRESUME bit test above. */	171	* check @flags to see if the vCPU has access to host MMIO, and if so,
167	jnc .Lvmlaunch	172	* do VERW. Else, do nothing (no mitigations needed/enabled).
		173	*/
		174	ALTERNATIVE_2 "", \
		175	__stringify(testl $VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO, WORD_SIZE(%_ASM_SP); \
		176	jz .Lskip_mmio_verw; \
		177	VERW; \
		178	.Lskip_mmio_verw:), \
		179	X86_FEATURE_CLEAR_CPU_BUF_VM_MMIO, \
		180	__stringify(VERW), X86_FEATURE_CLEAR_CPU_BUF_VM
		181
		182	/* Check @flags to see if VMLAUNCH or VMRESUME is needed. */
		183	testl $VMX_RUN_VMRESUME, WORD_SIZE(%_ASM_SP)
		184	jz .Lvmlaunch
168		185
169	/*	186	/*
170	* After a successful VMRESUME/VMLAUNCH, control flow "magically"	187	* After a successful VMRESUME/VMLAUNCH, control flow "magically"

arch/x86/kvm/vmx/vmx.c+173−150

 struct x86_pmu_lbr __ro_after_init vmx_lbr_caps;
+#ifdef CONFIG_CPU_MITIGATIONS
 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_should_flush);
 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_flush_cond);
 static DEFINE_MUTEX(vmx_l1d_flush_mutex);
 #define L1D_CACHE_ORDER 4
 static void *vmx_l1d_flush_pages;
-static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf)
+static int __vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf)
 {
 	struct page *page;
 	unsigned int i;
 	return 0;
 }
+static int vmx_setup_l1d_flush(void)
+{
+	/*
+	 * Hand the parameter mitigation value in which was stored in the pre
+	 * module init parser. If no parameter was given, it will contain
+	 * 'auto' which will be turned into the default 'cond' mitigation mode.
+	 */
+	return __vmx_setup_l1d_flush(vmentry_l1d_flush_param);
+}
+static void vmx_cleanup_l1d_flush(void)
+{
+	if (vmx_l1d_flush_pages) {
+		free_pages((unsigned long)vmx_l1d_flush_pages, L1D_CACHE_ORDER);
+		vmx_l1d_flush_pages = NULL;
+	}
+	/* Restore state so sysfs ignores VMX */
+	l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
+}
 static int vmentry_l1d_flush_parse(const char *s)
 {
 	unsigned int i;
 	}
 	mutex_lock(&vmx_l1d_flush_mutex);
-	ret = vmx_setup_l1d_flush(l1tf);
+	ret = __vmx_setup_l1d_flush(l1tf);
 	mutex_unlock(&vmx_l1d_flush_mutex);
 	return ret;
 }
 	return sysfs_emit(s, "%s\n", vmentry_l1d_param[l1tf_vmx_mitigation].option);
 }
+/*
+ * Software based L1D cache flush which is used when microcode providing
+ * the cache control MSR is not loaded.
+ *
+ * The L1D cache is 32 KiB on Nehalem and later microarchitectures, but to
+ * flush it is required to read in 64 KiB because the replacement algorithm
+ * is not exactly LRU. This could be sized at runtime via topology
+ * information but as all relevant affected CPUs have 32KiB L1D cache size
+ * there is no point in doing so.
+ */
+static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu)
+{
+	int size = PAGE_SIZE << L1D_CACHE_ORDER;
+	if (!static_branch_unlikely(&vmx_l1d_should_flush))
+		return;
+	/*
+	 * This code is only executed when the flush mode is 'cond' or
+	 * 'always'
+	 */
+	if (static_branch_likely(&vmx_l1d_flush_cond)) {
+		/*
+		 * Clear the per-cpu flush bit, it gets set again if the vCPU
+		 * is reloaded, i.e. if the vCPU is scheduled out or if KVM
+		 * exits to userspace, or if KVM reaches one of the unsafe
+		 * VMEXIT handlers, e.g. if KVM calls into the emulator,
+		 * or from the interrupt handlers.
+		 */
+		if (!kvm_get_cpu_l1tf_flush_l1d())
+			return;
+		kvm_clear_cpu_l1tf_flush_l1d();
+	}
+	vcpu->stat.l1d_flush++;
+	if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) {
+		native_wrmsrq(MSR_IA32_FLUSH_CMD, L1D_FLUSH);
+		return;
+	}
+	asm volatile(
+		/* First ensure the pages are in the TLB */
+		"xorl	%%eax, %%eax\n"
+		".Lpopulate_tlb:\n\t"
+		"movzbl	(%[flush_pages], %%" _ASM_AX "), %%ecx\n\t"
+		"addl	$4096, %%eax\n\t"
+		"cmpl	%%eax, %[size]\n\t"
+		"jne	.Lpopulate_tlb\n\t"
+		"xorl	%%eax, %%eax\n\t"
+		"cpuid\n\t"
+		/* Now fill the cache */
+		"xorl	%%eax, %%eax\n"
+		".Lfill_cache:\n"
+		"movzbl	(%[flush_pages], %%" _ASM_AX "), %%ecx\n\t"
+		"addl	$64, %%eax\n\t"
+		"cmpl	%%eax, %[size]\n\t"
+		"jne	.Lfill_cache\n\t"
+		"lfence\n"
+		:: [flush_pages] "r" (vmx_l1d_flush_pages),
+		    [size] "r" (size)
+		: "eax", "ebx", "ecx", "edx");
+}
+#else /* CONFIG_CPU_MITIGATIONS*/
+static int vmx_setup_l1d_flush(void)
+{
+	l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NEVER;
+	return 0;
+}
+static void vmx_cleanup_l1d_flush(void)
+{
+	l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
+}
+static __always_inline void vmx_l1d_flush(struct kvm_vcpu *vcpu)
+{
+}
+static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp)
+{
+	pr_warn_once("Kernel compiled without mitigations, ignoring vmentry_l1d_flush\n");
+	return 0;
+}
+static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp)
+{
+	return sysfs_emit(s, "never\n");
+}
+#endif
+static const struct kernel_param_ops vmentry_l1d_flush_ops = {
+	.set = vmentry_l1d_flush_set,
+	.get = vmentry_l1d_flush_get,
+};
+module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, NULL, 0644);
 static __always_inline void vmx_disable_fb_clear(struct vcpu_vmx *vmx)
 {
 	u64 msr;
 		vmx->disable_fb_clear = false;
 }
-static const struct kernel_param_ops vmentry_l1d_flush_ops = {
-	.set = vmentry_l1d_flush_set,
-	.get = vmentry_l1d_flush_get,
-};
-module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, NULL, 0644);
 static u32 vmx_segment_access_rights(struct kvm_segment *var);
 void vmx_vmexit(void);
 	loaded_vmcs->launched = 0;
 }
-void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs)
+static void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs)
 {
 	int cpu = loaded_vmcs->cpu;
 	if (!msr_write_intercepted(vmx, MSR_IA32_SPEC_CTRL))
 		flags |= VMX_RUN_SAVE_SPEC_CTRL;
-	if (static_branch_unlikely(&cpu_buf_vm_clear) &&
+	if (cpu_feature_enabled(X86_FEATURE_CLEAR_CPU_BUF_VM_MMIO) &&
 	    kvm_vcpu_can_access_host_mmio(&vmx->vcpu))
 		flags |= VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO;
 	return to_vmx(vcpu)->vpid;
 }
+static u64 construct_eptp(hpa_t root_hpa)
+{
+	u64 eptp = root_hpa | VMX_EPTP_MT_WB;
+	struct kvm_mmu_page *root;
+	if (kvm_mmu_is_dummy_root(root_hpa))
+		return eptp | VMX_EPTP_PWL_4;
+	/*
+	 * EPT roots should always have an associated MMU page.  Return a "bad"
+	 * EPTP to induce VM-Fail instead of continuing on in a unknown state.
+	 */
+	root = root_to_sp(root_hpa);
+	if (WARN_ON_ONCE(!root))
+		return INVALID_PAGE;
+	eptp |= (root->role.level == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4;
+	if (enable_ept_ad_bits && !root->role.ad_disabled)
+		eptp |= VMX_EPTP_AD_ENABLE_BIT;
+	return eptp;
+}
+static void vmx_flush_tlb_ept_root(hpa_t root_hpa)
+{
+	u64 eptp = construct_eptp(root_hpa);
+	if (VALID_PAGE(eptp))
+		ept_sync_context(eptp);
+	else
+		ept_sync_global();
+}
 void vmx_flush_tlb_current(struct kvm_vcpu *vcpu)
 {
 	struct kvm_mmu *mmu = vcpu->arch.mmu;
 		return;
 	if (enable_ept)
-		ept_sync_context(construct_eptp(vcpu, root_hpa,
+		vmx_flush_tlb_ept_root(root_hpa);
-						mmu->root_role.level));
 	else
 		vpid_sync_context(vmx_get_current_vpid(vcpu));
 }
 	return 4;
 }
-u64 construct_eptp(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level)
-{
-	u64 eptp = VMX_EPTP_MT_WB;
-	eptp |= (root_level == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4;
-	if (enable_ept_ad_bits &&
-	    (!is_guest_mode(vcpu) || nested_ept_ad_enabled(vcpu)))
-		eptp |= VMX_EPTP_AD_ENABLE_BIT;
-	eptp |= root_hpa;
-	return eptp;
-}
 void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level)
 {
 	struct kvm *kvm = vcpu->kvm;
 	bool update_guest_cr3 = true;
 	unsigned long guest_cr3;
-	u64 eptp;
 	if (enable_ept) {
-		eptp = construct_eptp(vcpu, root_hpa, root_level);
+		KVM_MMU_WARN_ON(root_to_sp(root_hpa) &&
-		vmcs_write64(EPT_POINTER, eptp);
+				root_level != root_to_sp(root_hpa)->role.level);
+		vmcs_write64(EPT_POINTER, construct_eptp(root_hpa));
 		hv_track_root_tdp(vcpu, root_hpa);
 	return kvm_vmx_exit_handlers[exit_handler_index](vcpu);
 unexpected_vmexit:
-	vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n",
-		    exit_reason.full);
 	dump_vmcs(vcpu);
-	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+	kvm_prepare_unexpected_reason_exit(vcpu, exit_reason.full);
-	vcpu->run->internal.suberror =
-			KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
-	vcpu->run->internal.ndata = 2;
-	vcpu->run->internal.data[0] = exit_reason.full;
-	vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
 	return 0;
 }
 	return ret;
 }
-/*
- * Software based L1D cache flush which is used when microcode providing
- * the cache control MSR is not loaded.
- *
- * The L1D cache is 32 KiB on Nehalem and later microarchitectures, but to
- * flush it is required to read in 64 KiB because the replacement algorithm
- * is not exactly LRU. This could be sized at runtime via topology
- * information but as all relevant affected CPUs have 32KiB L1D cache size
- * there is no point in doing so.
- */
-static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu)
-{
-	int size = PAGE_SIZE << L1D_CACHE_ORDER;
-	/*
-	 * This code is only executed when the flush mode is 'cond' or
-	 * 'always'
-	 */
-	if (static_branch_likely(&vmx_l1d_flush_cond)) {
-		bool flush_l1d;
-		/*
-		 * Clear the per-vcpu flush bit, it gets set again if the vCPU
-		 * is reloaded, i.e. if the vCPU is scheduled out or if KVM
-		 * exits to userspace, or if KVM reaches one of the unsafe
-		 * VMEXIT handlers, e.g. if KVM calls into the emulator.
-		 */
-		flush_l1d = vcpu->arch.l1tf_flush_l1d;
-		vcpu->arch.l1tf_flush_l1d = false;
-		/*
-		 * Clear the per-cpu flush bit, it gets set again from
-		 * the interrupt handlers.
-		 */
-		flush_l1d |= kvm_get_cpu_l1tf_flush_l1d();
-		kvm_clear_cpu_l1tf_flush_l1d();
-		if (!flush_l1d)
-			return;
-	}
-	vcpu->stat.l1d_flush++;
-	if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) {
-		native_wrmsrq(MSR_IA32_FLUSH_CMD, L1D_FLUSH);
-		return;
-	}
-	asm volatile(
-		/* First ensure the pages are in the TLB */
-		"xorl	%%eax, %%eax\n"
-		".Lpopulate_tlb:\n\t"
-		"movzbl	(%[flush_pages], %%" _ASM_AX "), %%ecx\n\t"
-		"addl	$4096, %%eax\n\t"
-		"cmpl	%%eax, %[size]\n\t"
-		"jne	.Lpopulate_tlb\n\t"
-		"xorl	%%eax, %%eax\n\t"
-		"cpuid\n\t"
-		/* Now fill the cache */
-		"xorl	%%eax, %%eax\n"
-		".Lfill_cache:\n"
-		"movzbl	(%[flush_pages], %%" _ASM_AX "), %%ecx\n\t"
-		"addl	$64, %%eax\n\t"
-		"cmpl	%%eax, %[size]\n\t"
-		"jne	.Lfill_cache\n\t"
-		"lfence\n"
-		:: [flush_pages] "r" (vmx_l1d_flush_pages),
-		    [size] "r" (size)
-		: "eax", "ebx", "ecx", "edx");
-}
 void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
 {
 	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
 	if (to_vt(vcpu)->emulation_required)
 		return;
-	if (vmx_get_exit_reason(vcpu).basic == EXIT_REASON_EXTERNAL_INTERRUPT)
+	switch (vmx_get_exit_reason(vcpu).basic) {
+	case EXIT_REASON_EXTERNAL_INTERRUPT:
 		handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu));
-	else if (vmx_get_exit_reason(vcpu).basic == EXIT_REASON_EXCEPTION_NMI)
+		break;
+	case EXIT_REASON_EXCEPTION_NMI:
 		handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu));
+		break;
+	case EXIT_REASON_MCE_DURING_VMENTRY:
+		kvm_machine_check();
+		break;
+	default:
+		break;
+	}
 }
 /*
 	guest_state_enter_irqoff();
-	/*
+	vmx_l1d_flush(vcpu);
-	 * L1D Flush includes CPU buffer clear to mitigate MDS, but VERW
-	 * mitigation for MDS is done late in VMentry and is still
-	 * executed in spite of L1D Flush. This is because an extra VERW
-	 * should not matter much after the big hammer L1D Flush.
-	 *
-	 * cpu_buf_vm_clear is used when system is not vulnerable to MDS/TAA,
-	 * and is affected by MMIO Stale Data. In such cases mitigation in only
-	 * needed against an MMIO capable guest.
-	 */
-	if (static_branch_unlikely(&vmx_l1d_should_flush))
-		vmx_l1d_flush(vcpu);
-	else if (static_branch_unlikely(&cpu_buf_vm_clear) &&
-		 (flags & VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO))
-		x86_clear_cpu_buffers();
 	vmx_disable_fb_clear(vmx);
 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
 		vmx_set_interrupt_shadow(vcpu, 0);
-	kvm_load_guest_xsave_state(vcpu);
 	pt_guest_enter(vmx);
 	atomic_switch_perf_msrs(vmx);
 	pt_guest_exit(vmx);
-	kvm_load_host_xsave_state(vcpu);
 	if (is_guest_mode(vcpu)) {
 		/*
 		 * Track VMLAUNCH/VMRESUME that have made past guest state
 	if (unlikely(vmx->fail))
 		return EXIT_FASTPATH_NONE;
-	if (unlikely((u16)vmx_get_exit_reason(vcpu).basic == EXIT_REASON_MCE_DURING_VMENTRY))
-		kvm_machine_check();
 	trace_kvm_exit(vcpu, KVM_ISA_VMX);
 	if (unlikely(vmx_get_exit_reason(vcpu).failed_vmentry))
 	return r;
 }
-static void vmx_cleanup_l1d_flush(void)
-{
-	if (vmx_l1d_flush_pages) {
-		free_pages((unsigned long)vmx_l1d_flush_pages, L1D_CACHE_ORDER);
-		vmx_l1d_flush_pages = NULL;
-	}
-	/* Restore state so sysfs ignores VMX */
-	l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
-}
 void vmx_exit(void)
 {
 	allow_smaller_maxphyaddr = false;
 	if (r)
 		return r;
-	/*
+	/* Must be called after common x86 init so enable_ept is setup. */
-	 * Must be called after common x86 init so enable_ept is properly set
+	r = vmx_setup_l1d_flush();
-	 * up. Hand the parameter mitigation value in which was stored in
-	 * the pre module init parser. If no parameter was given, it will
-	 * contain 'auto' which will be turned into the default 'cond'
-	 * mitigation mode.
-	 */
-	r = vmx_setup_l1d_flush(vmentry_l1d_flush_param);
 	if (r)
 		goto err_l1d_flush;

arch/x86/kvm/vmx/vmx.h+0−2

 void ept_save_pdptrs(struct kvm_vcpu *vcpu);
 void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
 void __vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
-u64 construct_eptp(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level);
 bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu);
 void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu);
 void free_vmcs(struct vmcs *vmcs);
 int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs);
 void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs);
-void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs);
 static inline struct vmcs *alloc_vmcs(bool shadow)
 {

arch/x86/kvm/vmx/x86_ops.h+1−1

 void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
 void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
 void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
-void vmx_set_dr6(struct kvm_vcpu *vcpu, unsigned long val);
 void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val);
 void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu);
 void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg);
 int tdx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr);
 int tdx_vcpu_ioctl(struct kvm_vcpu *vcpu, void __user *argp);
+int tdx_vcpu_unlocked_ioctl(struct kvm_vcpu *vcpu, void __user *argp);
 void tdx_flush_tlb_current(struct kvm_vcpu *vcpu);
 void tdx_flush_tlb_all(struct kvm_vcpu *vcpu);

arch/x86/kvm/x86.c+156−129

@@ -159,9 +159,6 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(report_ignored_msrs); · 158 unmodified lines
159	unsigned int min_timer_period_us = 200;	159	unsigned int min_timer_period_us = 200;
160	module_param(min_timer_period_us, uint, 0644);	160	module_param(min_timer_period_us, uint, 0644);
161		161
162	static bool __read_mostly kvmclock_periodic_sync = true;
163	module_param(kvmclock_periodic_sync, bool, 0444);
164
165	/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */	162	/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
166	static u32 __read_mostly tsc_tolerance_ppm = 250;	163	static u32 __read_mostly tsc_tolerance_ppm = 250;
167	module_param(tsc_tolerance_ppm, uint, 0644);	164	module_param(tsc_tolerance_ppm, uint, 0644);
@@ -212,7 +209,7 @@ struct kvm_user_return_msrs { · 44 unmodified lines
212	u32 __read_mostly kvm_nr_uret_msrs;	209	u32 __read_mostly kvm_nr_uret_msrs;
213	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_nr_uret_msrs);	210	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_nr_uret_msrs);
214	static u32 __read_mostly kvm_uret_msrs_list[KVM_MAX_NR_USER_RETURN_MSRS];	211	static u32 __read_mostly kvm_uret_msrs_list[KVM_MAX_NR_USER_RETURN_MSRS];
215	static struct kvm_user_return_msrs __percpu *user_return_msrs;	212	static DEFINE_PER_CPU(struct kvm_user_return_msrs, user_return_msrs);
216		213
217	#define KVM_SUPPORTED_XCR0 (XFEATURE_MASK_FP \| XFEATURE_MASK_SSE \	214	#define KVM_SUPPORTED_XCR0 (XFEATURE_MASK_FP \| XFEATURE_MASK_SSE \
218	\| XFEATURE_MASK_YMM \| XFEATURE_MASK_BNDREGS \	215	\| XFEATURE_MASK_YMM \| XFEATURE_MASK_BNDREGS \
@@ -575,24 +572,26 @@ static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu) · 356 unmodified lines
575	vcpu->arch.apf.gfns[i] = ~0;	572	vcpu->arch.apf.gfns[i] = ~0;
576	}	573	}
577		574
		575	static void kvm_destroy_user_return_msrs(void)
		576	{
		577	int cpu;
		578
		579	for_each_possible_cpu(cpu)
		580	WARN_ON_ONCE(per_cpu(user_return_msrs, cpu).registered);
		581
		582	kvm_nr_uret_msrs = 0;
		583	}
		584
578	static void kvm_on_user_return(struct user_return_notifier *urn)	585	static void kvm_on_user_return(struct user_return_notifier *urn)
579	{	586	{
580	unsigned slot;	587	unsigned slot;
581	struct kvm_user_return_msrs *msrs	588	struct kvm_user_return_msrs *msrs
582	= container_of(urn, struct kvm_user_return_msrs, urn);	589	= container_of(urn, struct kvm_user_return_msrs, urn);
583	struct kvm_user_return_msr_values *values;	590	struct kvm_user_return_msr_values *values;
584	unsigned long flags;
585		591
586	/*	592	msrs->registered = false;
587	* Disabling irqs at this point since the following code could be	593	user_return_notifier_unregister(urn);
588	* interrupted and executed through kvm_arch_disable_virtualization_cpu()	594
589	*/
590	local_irq_save(flags);
591	if (msrs->registered) {
592	msrs->registered = false;
593	user_return_notifier_unregister(urn);
594	}
595	local_irq_restore(flags);
596	for (slot = 0; slot < kvm_nr_uret_msrs; ++slot) {	595	for (slot = 0; slot < kvm_nr_uret_msrs; ++slot) {
597	values = &msrs->values[slot];	596	values = &msrs->values[slot];
598	if (values->host != values->curr) {	597	if (values->host != values->curr) {
@@ -643,7 +642,7 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_find_user_return_msr); · 44 unmodified lines
643		642
644	static void kvm_user_return_msr_cpu_online(void)	643	static void kvm_user_return_msr_cpu_online(void)
645	{	644	{
646	struct kvm_user_return_msrs *msrs = this_cpu_ptr(user_return_msrs);	645	struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs);
647	u64 value;	646	u64 value;
648	int i;	647	int i;
649		648
@@ -665,7 +664,7 @@ static void kvm_user_return_register_notifier(struct kvm_user_return_msrs *msrs) · 15 unmodified lines
665		664
666	int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask)	665	int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask)
667	{	666	{
668	struct kvm_user_return_msrs *msrs = this_cpu_ptr(user_return_msrs);	667	struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs);
669	int err;	668	int err;
670		669
671	value = (value & mask) \| (msrs->values[slot].host & ~mask);	670	value = (value & mask) \| (msrs->values[slot].host & ~mask);
@@ -681,24 +680,15 @@ int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask) · 9 unmodified lines
681	}	680	}
682	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_user_return_msr);	681	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_user_return_msr);
683		682
684	void kvm_user_return_msr_update_cache(unsigned int slot, u64 value)
685	{
686	struct kvm_user_return_msrs *msrs = this_cpu_ptr(user_return_msrs);
687
688	msrs->values[slot].curr = value;
689	kvm_user_return_register_notifier(msrs);
690	}
691	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_user_return_msr_update_cache);
692
693	u64 kvm_get_user_return_msr(unsigned int slot)	683	u64 kvm_get_user_return_msr(unsigned int slot)
694	{	684	{
695	return this_cpu_ptr(user_return_msrs)->values[slot].curr;	685	return this_cpu_ptr(&user_return_msrs)->values[slot].curr;
696	}	686	}
697	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_user_return_msr);	687	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_user_return_msr);
698		688
699	static void drop_user_return_notifiers(void)	689	static void drop_user_return_notifiers(void)
700	{	690	{
701	struct kvm_user_return_msrs *msrs = this_cpu_ptr(user_return_msrs);	691	struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs);
702		692
703	if (msrs->registered)	693	if (msrs->registered)
704	kvm_on_user_return(&msrs->urn);	694	kvm_on_user_return(&msrs->urn);
@@ -1045,6 +1035,13 @@ bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr) · 340 unmodified lines
1045	}	1035	}
1046	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_require_dr);	1036	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_require_dr);
1047		1037
		1038	static bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)
		1039	{
		1040	u64 mask = KVM_ASYNC_PF_ENABLED \| KVM_ASYNC_PF_DELIVERY_AS_INT;
		1041
		1042	return (vcpu->arch.apf.msr_en_val & mask) == mask;
		1043	}
		1044
1048	static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)	1045	static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
1049	{	1046	{
1050	return vcpu->arch.reserved_gpa_bits \| rsvd_bits(5, 8) \| rsvd_bits(1, 2);	1047	return vcpu->arch.reserved_gpa_bits \| rsvd_bits(5, 8) \| rsvd_bits(1, 2);
@@ -1137,15 +1134,20 @@ void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned lon · 86 unmodified lines
1137	}	1134	}
1138		1135
1139	if ((cr0 ^ old_cr0) & X86_CR0_PG) {	1136	if ((cr0 ^ old_cr0) & X86_CR0_PG) {
1140	kvm_clear_async_pf_completion_queue(vcpu);
1141	kvm_async_pf_hash_reset(vcpu);
1142
1143	/*	1137	/*
1144	* Clearing CR0.PG is defined to flush the TLB from the guest's	1138	* Clearing CR0.PG is defined to flush the TLB from the guest's
1145	* perspective.	1139	* perspective.
1146	*/	1140	*/
1147	if (!(cr0 & X86_CR0_PG))	1141	if (!(cr0 & X86_CR0_PG))
1148	kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);	1142	kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
		1143	/*
		1144	* Check for async #PF completion events when enabling paging,
		1145	* as the vCPU may have previously encountered async #PFs (it's
		1146	* entirely legal for the guest to toggle paging on/off without
		1147	* waiting for the async #PF queue to drain).
		1148	*/
		1149	else if (kvm_pv_async_pf_enabled(vcpu))
		1150	kvm_make_request(KVM_REQ_APF_READY, vcpu);
1149	}	1151	}
1150		1152
1151	if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS)	1153	if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS)
@@ -1203,20 +1205,27 @@ void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw) · 51 unmodified lines
1203	}	1205	}
1204	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_lmsw);	1206	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_lmsw);
1205		1207
1206	void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu)	1208	static void kvm_load_xfeatures(struct kvm_vcpu *vcpu, bool load_guest)
1207	{	1209	{
1208	if (vcpu->arch.guest_state_protected)	1210	if (vcpu->arch.guest_state_protected)
1209	return;	1211	return;
1210		1212
1211	if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) {	1213	if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE))
		1214	return;
1212		1215
1213	if (vcpu->arch.xcr0 != kvm_host.xcr0)	1216	if (vcpu->arch.xcr0 != kvm_host.xcr0)
1214	xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);	1217	xsetbv(XCR_XFEATURE_ENABLED_MASK,
		1218	load_guest ? vcpu->arch.xcr0 : kvm_host.xcr0);
1215		1219
1216	if (guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVES) &&	1220	if (guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVES) &&
1217	vcpu->arch.ia32_xss != kvm_host.xss)	1221	vcpu->arch.ia32_xss != kvm_host.xss)
1218	wrmsrq(MSR_IA32_XSS, vcpu->arch.ia32_xss);	1222	wrmsrq(MSR_IA32_XSS, load_guest ? vcpu->arch.ia32_xss : kvm_host.xss);
1219	}	1223	}
		1224
		1225	static void kvm_load_guest_pkru(struct kvm_vcpu *vcpu)
		1226	{
		1227	if (vcpu->arch.guest_state_protected)
		1228	return;
1220		1229
1221	if (cpu_feature_enabled(X86_FEATURE_PKU) &&	1230	if (cpu_feature_enabled(X86_FEATURE_PKU) &&
1222	vcpu->arch.pkru != vcpu->arch.host_pkru &&	1231	vcpu->arch.pkru != vcpu->arch.host_pkru &&
@@ -1224,9 +1233,8 @@ void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu) · 1 unmodified line
1224	kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE)))	1233	kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE)))
1225	wrpkru(vcpu->arch.pkru);	1234	wrpkru(vcpu->arch.pkru);
1226	}	1235	}
1227	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_load_guest_xsave_state);
1228		1236
1229	void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu)	1237	static void kvm_load_host_pkru(struct kvm_vcpu *vcpu)
1230	{	1238	{
1231	if (vcpu->arch.guest_state_protected)	1239	if (vcpu->arch.guest_state_protected)
1232	return;	1240	return;
@@ -1238,19 +1246,7 @@ void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu) · 5 unmodified lines
1238	if (vcpu->arch.pkru != vcpu->arch.host_pkru)	1246	if (vcpu->arch.pkru != vcpu->arch.host_pkru)
1239	wrpkru(vcpu->arch.host_pkru);	1247	wrpkru(vcpu->arch.host_pkru);
1240	}	1248	}
1241
1242	if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) {
1243
1244	if (vcpu->arch.xcr0 != kvm_host.xcr0)
1245	xsetbv(XCR_XFEATURE_ENABLED_MASK, kvm_host.xcr0);
1246
1247	if (guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVES) &&
1248	vcpu->arch.ia32_xss != kvm_host.xss)
1249	wrmsrq(MSR_IA32_XSS, kvm_host.xss);
1250	}
1251
1252	}	1249	}
1253	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_load_host_xsave_state);
1254		1250
1255	#ifdef CONFIG_X86_64	1251	#ifdef CONFIG_X86_64
1256	static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu)	1252	static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu)
@@ -3505,27 +3501,17 @@ uint64_t kvm_get_wall_clock_epoch(struct kvm *kvm) · 2248 unmodified lines
3505	/*	3501	/*
3506	* kvmclock updates which are isolated to a given vcpu, such as	3502	* kvmclock updates which are isolated to a given vcpu, such as
3507	* vcpu->cpu migration, should not allow system_timestamp from	3503	* vcpu->cpu migration, should not allow system_timestamp from
3508	* the rest of the vcpus to remain static. Otherwise ntp frequency	3504	* the rest of the vcpus to remain static.
3509	* correction applies to one vcpu's system_timestamp but not
3510	* the others.
3511	*	3505	*
3512	* So in those cases, request a kvmclock update for all vcpus.	3506	* So in those cases, request a kvmclock update for all vcpus.
3513	* We need to rate-limit these requests though, as they can	3507	* The worst case for a remote vcpu to update its kvmclock
3514	* considerably slow guests that have a large number of vcpus.	3508	* is then bounded by maximum nohz sleep latency.
3515	* The time for a remote vcpu to update its kvmclock is bound
3516	* by the delay we use to rate-limit the updates.
3517	*/	3509	*/
3518		3510	static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
3519	#define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100)
3520
3521	static void kvmclock_update_fn(struct work_struct *work)
3522	{	3511	{
3523	unsigned long i;	3512	unsigned long i;
3524	struct delayed_work *dwork = to_delayed_work(work);
3525	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
3526	kvmclock_update_work);
3527	struct kvm *kvm = container_of(ka, struct kvm, arch);
3528	struct kvm_vcpu *vcpu;	3513	struct kvm_vcpu *vcpu;
		3514	struct kvm *kvm = v->kvm;
3529		3515
3530	kvm_for_each_vcpu(i, vcpu, kvm) {	3516	kvm_for_each_vcpu(i, vcpu, kvm) {
3531	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);	3517	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
@@ -3533,29 +3519,6 @@ static void kvmclock_update_fn(struct work_struct *work) · 1 unmodified line
3533	}	3519	}
3534	}	3520	}
3535		3521
3536	static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
3537	{
3538	struct kvm *kvm = v->kvm;
3539
3540	kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
3541	schedule_delayed_work(&kvm->arch.kvmclock_update_work,
3542	KVMCLOCK_UPDATE_DELAY);
3543	}
3544
3545	#define KVMCLOCK_SYNC_PERIOD (300 * HZ)
3546
3547	static void kvmclock_sync_fn(struct work_struct *work)
3548	{
3549	struct delayed_work *dwork = to_delayed_work(work);
3550	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
3551	kvmclock_sync_work);
3552	struct kvm *kvm = container_of(ka, struct kvm, arch);
3553
3554	schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0);
3555	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
3556	KVMCLOCK_SYNC_PERIOD);
3557	}
3558
3559	/* These helpers are safe iff @msr is known to be an MCx bank MSR. */	3522	/* These helpers are safe iff @msr is known to be an MCx bank MSR. */
3560	static bool is_mci_control_msr(u32 msr)	3523	static bool is_mci_control_msr(u32 msr)
3561	{	3524	{
@@ -3650,13 +3613,6 @@ static int set_msr_mce(struct kvm_vcpu vcpu, struct msr_data msr_info) · 88 unmodified lines
3650	return 0;	3613	return 0;
3651	}	3614	}
3652		3615
3653	static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)
3654	{
3655	u64 mask = KVM_ASYNC_PF_ENABLED \| KVM_ASYNC_PF_DELIVERY_AS_INT;
3656
3657	return (vcpu->arch.apf.msr_en_val & mask) == mask;
3658	}
3659
3660	static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)	3616	static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
3661	{	3617	{
3662	gpa_t gpa = data & ~0x3f;	3618	gpa_t gpa = data & ~0x3f;
@@ -4182,7 +4138,12 @@ int kvm_set_msr_common(struct kvm_vcpu vcpu, struct msr_data msr_info) · 519 unmodified lines
4182	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))	4138	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
4183	return 1;	4139	return 1;
4184	if (data & 0x1) {	4140	if (data & 0x1) {
4185	vcpu->arch.apf.pageready_pending = false;	4141	/*
		4142	* Pairs with the smp_mb__after_atomic() in
		4143	* kvm_arch_async_page_present_queued().
		4144	*/
		4145	smp_store_mb(vcpu->arch.apf.pageready_pending, false);
		4146
4186	kvm_check_async_pf_completion(vcpu);	4147	kvm_check_async_pf_completion(vcpu);
4187	}	4148	}
4188	break;	4149	break;
@@ -5188,7 +5149,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) · 999 unmodified lines
5188	{	5149	{
5189	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);	5150	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
5190		5151
5191	vcpu->arch.l1tf_flush_l1d = true;	5152	kvm_request_l1tf_flush_l1d();
5192		5153
5193	if (vcpu->scheduled_out && pmu->version && pmu->event_count) {	5154	if (vcpu->scheduled_out && pmu->version && pmu->event_count) {
5194	pmu->need_cleanup = true;	5155	pmu->need_cleanup = true;
@@ -7239,6 +7200,19 @@ static int kvm_vm_ioctl_set_clock(struct kvm kvm, void __user argp) · 2044 unmodified lines
7239	return 0;	7200	return 0;
7240	}	7201	}
7241		7202
		7203	long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl,
		7204	unsigned long arg)
		7205	{
		7206	struct kvm_vcpu *vcpu = filp->private_data;
		7207	void __user argp = (void __user )arg;
		7208
		7209	if (ioctl == KVM_MEMORY_ENCRYPT_OP &&
		7210	kvm_x86_ops.vcpu_mem_enc_unlocked_ioctl)
		7211	return kvm_x86_call(vcpu_mem_enc_unlocked_ioctl)(vcpu, argp);
		7212
		7213	return -ENOIOCTLCMD;
		7214	}
		7215
7242	int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)	7216	int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
7243	{	7217	{
7244	struct kvm *kvm = filp->private_data;	7218	struct kvm *kvm = filp->private_data;
@@ -7998,7 +7972,7 @@ int kvm_write_guest_virt_system(struct kvm_vcpu vcpu, gva_t addr, void val, · 753 unmodified lines
7998	unsigned int bytes, struct x86_exception *exception)	7972	unsigned int bytes, struct x86_exception *exception)
7999	{	7973	{
8000	/* kvm_write_guest_virt_system can pull in tons of pages. */	7974	/* kvm_write_guest_virt_system can pull in tons of pages. */
8001	vcpu->arch.l1tf_flush_l1d = true;	7975	kvm_request_l1tf_flush_l1d();
8002		7976
8003	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,	7977	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
8004	PFERR_WRITE_MASK, exception);	7978	PFERR_WRITE_MASK, exception);
@@ -8842,6 +8816,14 @@ static void emulator_triple_fault(struct x86_emulate_ctxt *ctxt) · 837 unmodified lines
8842	kvm_make_request(KVM_REQ_TRIPLE_FAULT, emul_to_vcpu(ctxt));	8816	kvm_make_request(KVM_REQ_TRIPLE_FAULT, emul_to_vcpu(ctxt));
8843	}	8817	}
8844		8818
		8819	static int emulator_get_xcr(struct x86_emulate_ctxt ctxt, u32 index, u64 xcr)
		8820	{
		8821	if (index != XCR_XFEATURE_ENABLED_MASK)
		8822	return 1;
		8823	*xcr = emul_to_vcpu(ctxt)->arch.xcr0;
		8824	return 0;
		8825	}
		8826
8845	static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr)	8827	static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr)
8846	{	8828	{
8847	return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr);	8829	return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr);
@@ -8914,6 +8896,7 @@ static const struct x86_emulate_ops emulate_ops = { · 66 unmodified lines
8914	.is_smm = emulator_is_smm,	8896	.is_smm = emulator_is_smm,
8915	.leave_smm = emulator_leave_smm,	8897	.leave_smm = emulator_leave_smm,
8916	.triple_fault = emulator_triple_fault,	8898	.triple_fault = emulator_triple_fault,
		8899	.get_xcr = emulator_get_xcr,
8917	.set_xcr = emulator_set_xcr,	8900	.set_xcr = emulator_set_xcr,
8918	.get_untagged_addr = emulator_get_untagged_addr,	8901	.get_untagged_addr = emulator_get_untagged_addr,
8919	.is_canonical_addr = emulator_is_canonical_addr,	8902	.is_canonical_addr = emulator_is_canonical_addr,
@@ -9109,6 +9092,18 @@ void kvm_prepare_event_vectoring_exit(struct kvm_vcpu *vcpu, gpa_t gpa) · 189 unmodified lines
9109	}	9092	}
9110	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_prepare_event_vectoring_exit);	9093	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_prepare_event_vectoring_exit);
9111		9094
		9095	void kvm_prepare_unexpected_reason_exit(struct kvm_vcpu *vcpu, u64 exit_reason)
		9096	{
		9097	vcpu_unimpl(vcpu, "unexpected exit reason 0x%llx\n", exit_reason);
		9098
		9099	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		9100	vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
		9101	vcpu->run->internal.ndata = 2;
		9102	vcpu->run->internal.data[0] = exit_reason;
		9103	vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
		9104	}
		9105	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_prepare_unexpected_reason_exit);
		9106
9112	static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)	9107	static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)
9113	{	9108	{
9114	struct kvm *kvm = vcpu->kvm;	9109	struct kvm *kvm = vcpu->kvm;
@@ -9337,6 +9332,23 @@ static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt) · 222 unmodified lines
9337	return false;	9332	return false;
9338	}	9333	}
9339		9334
		9335	static bool is_soft_int_instruction(struct x86_emulate_ctxt *ctxt,
		9336	int emulation_type)
		9337	{
		9338	u8 vector = EMULTYPE_GET_SOFT_INT_VECTOR(emulation_type);
		9339
		9340	switch (ctxt->b) {
		9341	case 0xcc:
		9342	return vector == BP_VECTOR;
		9343	case 0xcd:
		9344	return vector == ctxt->src.val;
		9345	case 0xce:
		9346	return vector == OF_VECTOR;
		9347	default:
		9348	return false;
		9349	}
		9350	}
		9351
9340	/*	9352	/*
9341	* Decode an instruction for emulation. The caller is responsible for handling	9353	* Decode an instruction for emulation. The caller is responsible for handling
9342	* code breakpoints. Note, manually detecting code breakpoints is unnecessary	9354	* code breakpoints. Note, manually detecting code breakpoints is unnecessary
@@ -9394,7 +9406,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, · 51 unmodified lines
9394	return handle_emulation_failure(vcpu, emulation_type);	9406	return handle_emulation_failure(vcpu, emulation_type);
9395	}	9407	}
9396		9408
9397	vcpu->arch.l1tf_flush_l1d = true;	9409	kvm_request_l1tf_flush_l1d();
9398		9410
9399	if (!(emulation_type & EMULTYPE_NO_DECODE)) {	9411	if (!(emulation_type & EMULTYPE_NO_DECODE)) {
9400	kvm_clear_exception_queue(vcpu);	9412	kvm_clear_exception_queue(vcpu);
@@ -9447,6 +9459,10 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, · 46 unmodified lines
9447	* injecting single-step #DBs.	9459	* injecting single-step #DBs.
9448	*/	9460	*/
9449	if (emulation_type & EMULTYPE_SKIP) {	9461	if (emulation_type & EMULTYPE_SKIP) {
		9462	if (emulation_type & EMULTYPE_SKIP_SOFT_INT &&
		9463	!is_soft_int_instruction(ctxt, emulation_type))
		9464	return 0;
		9465
9450	if (ctxt->mode != X86EMUL_MODE_PROT64)	9466	if (ctxt->mode != X86EMUL_MODE_PROT64)
9451	ctxt->eip = (u32)ctxt->_eip;	9467	ctxt->eip = (u32)ctxt->_eip;
9452	else	9468	else
@@ -10031,17 +10047,9 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops) · 578 unmodified lines
10031	return -ENOMEM;	10047	return -ENOMEM;
10032	}	10048	}
10033		10049
10034	user_return_msrs = alloc_percpu(struct kvm_user_return_msrs);
10035	if (!user_return_msrs) {
10036	pr_err("failed to allocate percpu kvm_user_return_msrs\n");
10037	r = -ENOMEM;
10038	goto out_free_x86_emulator_cache;
10039	}
10040	kvm_nr_uret_msrs = 0;
10041
10042	r = kvm_mmu_vendor_module_init();	10050	r = kvm_mmu_vendor_module_init();
10043	if (r)	10051	if (r)
10044	goto out_free_percpu;	10052	goto out_free_x86_emulator_cache;
10045		10053
10046	kvm_caps.supported_vm_types = BIT(KVM_X86_DEFAULT_VM);	10054	kvm_caps.supported_vm_types = BIT(KVM_X86_DEFAULT_VM);
10047	kvm_caps.supported_mce_cap = MCG_CTL_P \| MCG_SER_P;	10055	kvm_caps.supported_mce_cap = MCG_CTL_P \| MCG_SER_P;
@@ -10066,6 +10074,8 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops) · 18 unmodified lines
10066	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))	10074	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
10067	rdmsrq(MSR_IA32_ARCH_CAPABILITIES, kvm_host.arch_capabilities);	10075	rdmsrq(MSR_IA32_ARCH_CAPABILITIES, kvm_host.arch_capabilities);
10068		10076
		10077	WARN_ON_ONCE(kvm_nr_uret_msrs);
		10078
10069	r = ops->hardware_setup();	10079	r = ops->hardware_setup();
10070	if (r != 0)	10080	if (r != 0)
10071	goto out_mmu_exit;	10081	goto out_mmu_exit;
@@ -10138,9 +10148,8 @@ out_unwind_ops: · 66 unmodified lines
10138	kvm_x86_ops.enable_virtualization_cpu = NULL;	10148	kvm_x86_ops.enable_virtualization_cpu = NULL;
10139	kvm_x86_call(hardware_unsetup)();	10149	kvm_x86_call(hardware_unsetup)();
10140	out_mmu_exit:	10150	out_mmu_exit:
		10151	kvm_destroy_user_return_msrs();
10141	kvm_mmu_vendor_module_exit();	10152	kvm_mmu_vendor_module_exit();
10142	out_free_percpu:
10143	free_percpu(user_return_msrs);
10144	out_free_x86_emulator_cache:	10153	out_free_x86_emulator_cache:
10145	kmem_cache_destroy(x86_emulator_cache);	10154	kmem_cache_destroy(x86_emulator_cache);
10146	return r;	10155	return r;
@@ -10168,8 +10177,8 @@ void kvm_x86_vendor_exit(void) · 21 unmodified lines
10168	cancel_work_sync(&pvclock_gtod_work);	10177	cancel_work_sync(&pvclock_gtod_work);
10169	#endif	10178	#endif
10170	kvm_x86_call(hardware_unsetup)();	10179	kvm_x86_call(hardware_unsetup)();
		10180	kvm_destroy_user_return_msrs();
10171	kvm_mmu_vendor_module_exit();	10181	kvm_mmu_vendor_module_exit();
10172	free_percpu(user_return_msrs);
10173	kmem_cache_destroy(x86_emulator_cache);	10182	kmem_cache_destroy(x86_emulator_cache);
10174	#ifdef CONFIG_KVM_XEN	10183	#ifdef CONFIG_KVM_XEN
10175	static_key_deferred_flush(&kvm_xen_enabled);	10184	static_key_deferred_flush(&kvm_xen_enabled);
@@ -11291,6 +11300,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 1115 unmodified lines
11291	if (vcpu->arch.guest_fpu.xfd_err)	11300	if (vcpu->arch.guest_fpu.xfd_err)
11292	wrmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);	11301	wrmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);
11293		11302
		11303	kvm_load_xfeatures(vcpu, true);
		11304
11294	if (unlikely(vcpu->arch.switch_db_regs &&	11305	if (unlikely(vcpu->arch.switch_db_regs &&
11295	!(vcpu->arch.switch_db_regs & KVM_DEBUGREG_AUTO_SWITCH))) {	11306	!(vcpu->arch.switch_db_regs & KVM_DEBUGREG_AUTO_SWITCH))) {
11296	set_debugreg(DR7_FIXED_1, 7);	11307	set_debugreg(DR7_FIXED_1, 7);
@@ -11319,6 +11330,12 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 22 unmodified lines
11319		11330
11320	guest_timing_enter_irqoff();	11331	guest_timing_enter_irqoff();
11321		11332
		11333	/*
		11334	* Swap PKRU with hardware breakpoints disabled to minimize the number
		11335	* of flows where non-KVM code can run with guest state loaded.
		11336	*/
		11337	kvm_load_guest_pkru(vcpu);
		11338
11322	for (;;) {	11339	for (;;) {
11323	/*	11340	/*
11324	* Assert that vCPU vs. VM APICv state is consistent. An APICv	11341	* Assert that vCPU vs. VM APICv state is consistent. An APICv
@@ -11347,6 +11364,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 22 unmodified lines
11347	++vcpu->stat.exits;	11364	++vcpu->stat.exits;
11348	}	11365	}
11349		11366
		11367	kvm_load_host_pkru(vcpu);
		11368
11350	/*	11369	/*
11351	* Do this here before restoring debug registers on the host. And	11370	* Do this here before restoring debug registers on the host. And
11352	* since we do this before handling the vmexit, a DR access vmexit	11371	* since we do this before handling the vmexit, a DR access vmexit
@@ -11377,6 +11396,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 24 unmodified lines
11377	vcpu->mode = OUTSIDE_GUEST_MODE;	11396	vcpu->mode = OUTSIDE_GUEST_MODE;
11378	smp_wmb();	11397	smp_wmb();
11379		11398
		11399	kvm_load_xfeatures(vcpu, false);
		11400
11380	/*	11401	/*
11381	* Sync xfd before calling handle_exit_irqoff() which may	11402	* Sync xfd before calling handle_exit_irqoff() which may
11382	* rely on the fact that guest_fpu::xfd is up-to-date (e.g.	11403	* rely on the fact that guest_fpu::xfd is up-to-date (e.g.
@@ -12734,8 +12755,6 @@ fail_mmu_destroy: · 1351 unmodified lines
12734		12755
12735	void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)	12756	void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
12736	{	12757	{
12737	struct kvm *kvm = vcpu->kvm;
12738
12739	if (mutex_lock_killable(&vcpu->mutex))	12758	if (mutex_lock_killable(&vcpu->mutex))
12740	return;	12759	return;
12741	vcpu_load(vcpu);	12760	vcpu_load(vcpu);
@@ -12746,10 +12765,6 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) · 4 unmodified lines
12746	vcpu->arch.msr_kvm_poll_control = 1;	12765	vcpu->arch.msr_kvm_poll_control = 1;
12747		12766
12748	mutex_unlock(&vcpu->mutex);	12767	mutex_unlock(&vcpu->mutex);
12749
12750	if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0)
12751	schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
12752	KVMCLOCK_SYNC_PERIOD);
12753	}	12768	}
12754		12769
12755	void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)	12770	void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
@@ -13088,7 +13103,21 @@ int kvm_arch_enable_virtualization_cpu(void) · 332 unmodified lines
13088	void kvm_arch_disable_virtualization_cpu(void)	13103	void kvm_arch_disable_virtualization_cpu(void)
13089	{	13104	{
13090	kvm_x86_call(disable_virtualization_cpu)();	13105	kvm_x86_call(disable_virtualization_cpu)();
13091	drop_user_return_notifiers();	13106
		13107	/*
		13108	* Leave the user-return notifiers as-is when disabling virtualization
		13109	* for reboot, i.e. when disabling via IPI function call, and instead
		13110	* pin kvm.ko (if it's a module) to defend against use-after-free (in
		13111	* the very unlikely scenario module unload is racing with reboot).
		13112	* On a forced reboot, tasks aren't frozen before shutdown, and so KVM
		13113	* could be actively modifying user-return MSR state when the IPI to
		13114	* disable virtualization arrives. Handle the extreme edge case here
		13115	* instead of trying to account for it in the normal flows.
		13116	*/
		13117	if (in_task() \|\| WARN_ON_ONCE(!kvm_rebooting))
		13118	drop_user_return_notifiers();
		13119	else
		13120	__module_get(THIS_MODULE);
13092	}	13121	}
13093		13122
13094	bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)	13123	bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)
@@ -13160,9 +13189,6 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 65 unmodified lines
13160	kvm->arch.hv_root_tdp = INVALID_PAGE;	13189	kvm->arch.hv_root_tdp = INVALID_PAGE;
13161	#endif	13190	#endif
13162		13191
13163	INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn);
13164	INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn);
13165
13166	kvm_apicv_init(kvm);	13192	kvm_apicv_init(kvm);
13167	kvm_hv_init_vm(kvm);	13193	kvm_hv_init_vm(kvm);
13168	kvm_xen_init_vm(kvm);	13194	kvm_xen_init_vm(kvm);
@@ -13269,9 +13295,6 @@ void kvm_arch_pre_destroy_vm(struct kvm *kvm) · 100 unmodified lines
13269	* is unsafe, i.e. will lead to use-after-free. The PIT also needs to	13295	* is unsafe, i.e. will lead to use-after-free. The PIT also needs to
13270	* be stopped before IRQ routing is freed.	13296	* be stopped before IRQ routing is freed.
13271	*/	13297	*/
13272	cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work);
13273	cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work);
13274
13275	#ifdef CONFIG_KVM_IOAPIC	13298	#ifdef CONFIG_KVM_IOAPIC
13276	kvm_free_pit(kvm);	13299	kvm_free_pit(kvm);
13277	#endif	13300	#endif
@@ -13888,7 +13911,7 @@ void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, · 610 unmodified lines
13888	if ((work->wakeup_all \|\| work->notpresent_injected) &&	13911	if ((work->wakeup_all \|\| work->notpresent_injected) &&
13889	kvm_pv_async_pf_enabled(vcpu) &&	13912	kvm_pv_async_pf_enabled(vcpu) &&
13890	!apf_put_user_ready(vcpu, work->arch.token)) {	13913	!apf_put_user_ready(vcpu, work->arch.token)) {
13891	vcpu->arch.apf.pageready_pending = true;	13914	WRITE_ONCE(vcpu->arch.apf.pageready_pending, true);
13892	kvm_apic_set_irq(vcpu, &irq, NULL);	13915	kvm_apic_set_irq(vcpu, &irq, NULL);
13893	}	13916	}
13894		13917
@@ -13899,7 +13922,11 @@ void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, · 4 unmodified lines
13899	void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu)	13922	void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu)
13900	{	13923	{
13901	kvm_make_request(KVM_REQ_APF_READY, vcpu);	13924	kvm_make_request(KVM_REQ_APF_READY, vcpu);
13902	if (!vcpu->arch.apf.pageready_pending)	13925
		13926	/* Pairs with smp_store_mb() in kvm_set_msr_common(). */
		13927	smp_mb__after_atomic();
		13928
		13929	if (!READ_ONCE(vcpu->arch.apf.pageready_pending))
13903	kvm_vcpu_kick(vcpu);	13930	kvm_vcpu_kick(vcpu);
13904	}	13931	}
13905		13932

arch/x86/kvm/x86.h+14−2

 	return !(kvm->arch.disabled_quirks & quirk);
 }
+static __always_inline void kvm_request_l1tf_flush_l1d(void)
+{
+#if IS_ENABLED(CONFIG_CPU_MITIGATIONS) && IS_ENABLED(CONFIG_KVM_INTEL)
+	/*
+	 * Use a raw write to set the per-CPU flag, as KVM will ensure a flush
+	 * even if preemption is currently enabled..  If the current vCPU task
+	 * is migrated to a different CPU (or userspace runs the vCPU on a
+	 * different task) before the next VM-Entry, then kvm_arch_vcpu_load()
+	 * will request a flush on the new CPU.
+	 */
+	raw_cpu_write(irq_stat.kvm_cpu_l1tf_flush_l1d, 1);
+#endif
+}
 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip);
 u64 get_kvmclock_ns(struct kvm *kvm);
 #endif
 }
-void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu);
-void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu);
 int kvm_spec_ctrl_test_value(u64 value);
 int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r,
 			      struct x86_exception *e);

drivers/crypto/ccp/sev-dev.c+37−0

@@ -2770,6 +2770,43 @@ void sev_platform_shutdown(void) · 2769 unmodified lines
2770	}	2770	}
2771	EXPORT_SYMBOL_GPL(sev_platform_shutdown);	2771	EXPORT_SYMBOL_GPL(sev_platform_shutdown);
2772		2772
		2773	u64 sev_get_snp_policy_bits(void)
		2774	{
		2775	struct psp_device *psp = psp_master;
		2776	struct sev_device *sev;
		2777	u64 policy_bits;
		2778
		2779	if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
		2780	return 0;
		2781
		2782	if (!psp \|\| !psp->sev_data)
		2783	return 0;
		2784
		2785	sev = psp->sev_data;
		2786
		2787	policy_bits = SNP_POLICY_MASK_BASE;
		2788
		2789	if (sev->snp_plat_status.feature_info) {
		2790	if (sev->snp_feat_info_0.ecx & SNP_RAPL_DISABLE_SUPPORTED)
		2791	policy_bits \|= SNP_POLICY_MASK_RAPL_DIS;
		2792
		2793	if (sev->snp_feat_info_0.ecx & SNP_CIPHER_TEXT_HIDING_SUPPORTED)
		2794	policy_bits \|= SNP_POLICY_MASK_CIPHERTEXT_HIDING_DRAM;
		2795
		2796	if (sev->snp_feat_info_0.ecx & SNP_AES_256_XTS_POLICY_SUPPORTED)
		2797	policy_bits \|= SNP_POLICY_MASK_MEM_AES_256_XTS;
		2798
		2799	if (sev->snp_feat_info_0.ecx & SNP_CXL_ALLOW_POLICY_SUPPORTED)
		2800	policy_bits \|= SNP_POLICY_MASK_CXL_ALLOW;
		2801
		2802	if (sev_version_greater_or_equal(1, 58))
		2803	policy_bits \|= SNP_POLICY_MASK_PAGE_SWAP_DISABLE;
		2804	}
		2805
		2806	return policy_bits;
		2807	}
		2808	EXPORT_SYMBOL_GPL(sev_get_snp_policy_bits);
		2809
2773	void sev_dev_destroy(struct psp_device *psp)	2810	void sev_dev_destroy(struct psp_device *psp)
2774	{	2811	{
2775	struct sev_device *sev = psp->sev_data;	2812	struct sev_device *sev = psp->sev_data;

drivers/irqchip/irq-apple-aic.c+5−2

 	if (is_kernel_in_hyp_mode() &&
 	    (read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EL2_En) &&
 	    read_sysreg_s(SYS_ICH_MISR_EL2) != 0) {
+		u64 val;
 		generic_handle_domain_irq(aic_irqc->hw_domain,
 					  AIC_FIQ_HWIRQ(AIC_VGIC_MI));
 		if (unlikely((read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EL2_En) &&
-			     read_sysreg_s(SYS_ICH_MISR_EL2))) {
+			     (val = read_sysreg_s(SYS_ICH_MISR_EL2)))) {
-			pr_err_ratelimited("vGIC IRQ fired and not handled by KVM, disabling.\n");
+			pr_err_ratelimited("vGIC IRQ fired and not handled by KVM (MISR=%llx), disabling.\n",
+					   val);
 			sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EL2_En, 0);
 		}
 	}

drivers/irqchip/irq-gic.c+3−0

 	if (ret)
 		return;
+	gic_v2_kvm_info.gicc_base = gic_data[0].cpu_base.common_base;
 	if (static_branch_likely(&supports_deactivate_key))
 		vgic_set_kvm_info(&gic_v2_kvm_info);
 }
 		return;
 	gic_v2_kvm_info.maint_irq = irq;
+	gic_v2_kvm_info.gicc_base = gic_data[0].cpu_base.common_base;
 	vgic_set_kvm_info(&gic_v2_kvm_info);
 }

fs/btrfs/compression.c+2−2

@@ -475,8 +475,8 @@ static noinline int add_ra_bio_pages(struct inode *inode, · 474 unmodified lines
475	continue;	475	continue;
476	}	476	}
477		477
478	folio = filemap_alloc_folio(mapping_gfp_constraint(mapping,	478	folio = filemap_alloc_folio(mapping_gfp_constraint(mapping, ~__GFP_FS),
479	~__GFP_FS), 0);	479	0, NULL);
480	if (!folio)	480	if (!folio)
481	break;	481	break;
482		482

fs/btrfs/verity.c+1−1

@@ -736,7 +736,7 @@ again: · 735 unmodified lines
736	}	736	}
737		737
738	folio = filemap_alloc_folio(mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS),	738	folio = filemap_alloc_folio(mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS),
739	0);	739	0, NULL);
740	if (!folio)	740	if (!folio)
741	return ERR_PTR(-ENOMEM);	741	return ERR_PTR(-ENOMEM);
742		742

fs/erofs/zdata.c+1−1

@@ -562,7 +562,7 @@ static void z_erofs_bind_cache(struct z_erofs_frontend *fe) · 561 unmodified lines
562	* Allocate a managed folio for cached I/O, or it may be	562	* Allocate a managed folio for cached I/O, or it may be
563	* then filled with a file-backed folio for in-place I/O	563	* then filled with a file-backed folio for in-place I/O
564	*/	564	*/
565	newfolio = filemap_alloc_folio(gfp, 0);	565	newfolio = filemap_alloc_folio(gfp, 0, NULL);
566	if (!newfolio)	566	if (!newfolio)
567	continue;	567	continue;
568	newfolio->private = Z_EROFS_PREALLOCATED_FOLIO;	568	newfolio->private = Z_EROFS_PREALLOCATED_FOLIO;

fs/f2fs/compress.c+1−1

@@ -1947,7 +1947,7 @@ static void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi, · 1946 unmodified lines
1947	return;	1947	return;
1948	}	1948	}
1949		1949
1950	cfolio = filemap_alloc_folio(__GFP_NOWARN \| __GFP_IO, 0);	1950	cfolio = filemap_alloc_folio(__GFP_NOWARN \| __GFP_IO, 0, NULL);
1951	if (!cfolio)	1951	if (!cfolio)
1952	return;	1952	return;
1953		1953

include/kvm/arm_vgic.h+19−10

 	/* virtual control interface mapping, HYP VA */
 	void __iomem		*vctrl_hyp;
+	/* Physical CPU interface, kernel VA */
+	void __iomem		*gicc_base;
 	/* Number of implemented list registers */
 	int			nr_lr;
 struct vgic_irq {
 	raw_spinlock_t irq_lock;	/* Protects the content of the struct */
+	u32 intid;			/* Guest visible INTID */
 	struct rcu_head rcu;
 	struct list_head ap_list;
 					 * affinity reg (v3).
 					 */
-	u32 intid;			/* Guest visible INTID */
+	bool pending_release:1;		/* Used for LPIs only, unreferenced IRQ
-	bool line_level;		/* Level only */
-	bool pending_latch;		/* The pending latch state used to calculate
-					 * the pending state for both level
-					 * and edge triggered IRQs. */
-	bool active;
-	bool pending_release;		/* Used for LPIs only, unreferenced IRQ
 					 * pending a release */
-	bool enabled;
+	bool pending_latch:1;		/* The pending latch state used to calculate
-	bool hw;			/* Tied to HW IRQ */
+					 * the pending state for both level
+					 * and edge triggered IRQs. */
+	enum vgic_irq_config config:1;	/* Level or edge */
+	bool line_level:1;		/* Level only */
+	bool enabled:1;
+	bool active:1;
+	bool hw:1;			/* Tied to HW IRQ */
+	bool on_lr:1;			/* Present in a CPU LR */
 	refcount_t refcount;		/* Used for LPIs */
 	u32 hwintid;			/* HW INTID number */
 	unsigned int host_irq;		/* linux irq corresponding to hwintid */
 	u8 active_source;		/* GICv2 SGIs only */
 	u8 priority;
 	u8 group;			/* 0 == group 0, 1 == group 1 */
-	enum vgic_irq_config config;	/* Level or edge */
 	struct irq_ops *ops;
 	/* The GIC maintenance IRQ for nested hypervisors. */
 	u32			mi_intid;
+	/* Track the number of in-flight active SPIs */
+	atomic_t		active_spis;
 	/* base addresses in guest physical address space: */
 	gpa_t			vgic_dist_base;		/* distributor */
 	union {
 	struct vgic_irq		*spis;
 	struct vgic_io_device	dist_iodev;
+	struct vgic_io_device	cpuif_iodev;
 	bool			has_its;
 	bool			table_write_in_progress;
 void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu);
 void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu);
 void kvm_vgic_reset_mapped_irq(struct kvm_vcpu *vcpu, u32 vintid);
+void kvm_vgic_process_async_update(struct kvm_vcpu *vcpu);
 void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1);

include/linux/irqchip/arm-gic.h+6−0

 #define GICH_HCR_EN			(1 << 0)
 #define GICH_HCR_UIE			(1 << 1)
+#define GICH_HCR_LRENPIE		(1 << 2)
 #define GICH_HCR_NPIE			(1 << 3)
+#define GICH_HCR_VGrp0EIE		(1 << 4)
+#define GICH_HCR_VGrp0DIE		(1 << 5)
+#define GICH_HCR_VGrp1EIE		(1 << 6)
+#define GICH_HCR_VGrp1DIE		(1 << 7)
+#define GICH_HCR_EOICOUNT		GENMASK(31, 27)
 #define GICH_LR_VIRTUALID		(0x3ff << 0)
 #define GICH_LR_PHYSID_CPUID_SHIFT	(10)

include/linux/irqchip/arm-vgic-info.h+2−0

 	enum gic_type	type;
 	/* Virtual CPU interface */
 	struct resource vcpu;
+	/* GICv2 GICC VA */
+	void __iomem	*gicc_base;
 	/* Interrupt number */
 	unsigned int	maint_irq;
 	/* No interrupt mask, no need to use the above field */

include/linux/kvm_host.h+2−12

 			unsigned int ioctl, unsigned long arg);
 long kvm_arch_vcpu_ioctl(struct file *filp,
 			 unsigned int ioctl, unsigned long arg);
+long kvm_arch_vcpu_unlocked_ioctl(struct file *filp,
+				  unsigned int ioctl, unsigned long arg);
 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf);
 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext);
 }
 #endif /* CONFIG_HAVE_KVM_NO_POLL */
-#ifdef CONFIG_HAVE_KVM_VCPU_ASYNC_IOCTL
-long kvm_arch_vcpu_async_ioctl(struct file *filp,
-			       unsigned int ioctl, unsigned long arg);
-#else
-static inline long kvm_arch_vcpu_async_ioctl(struct file *filp,
-					     unsigned int ioctl,
-					     unsigned long arg)
-{
-	return -ENOIOCTLCMD;
-}
-#endif /* CONFIG_HAVE_KVM_VCPU_ASYNC_IOCTL */
 void kvm_arch_guest_memory_reclaimed(struct kvm *kvm);
 #ifdef CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE

include/linux/pagemap.h+13−5

 }
 #ifdef CONFIG_NUMA
-struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order);
+struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order,
+		struct mempolicy *policy);
 #else
-static inline struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order)
+static inline struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order,
+		struct mempolicy *policy)
 {
 	return folio_alloc_noprof(gfp, order);
 }
 static inline struct page *__page_cache_alloc(gfp_t gfp)
 {
-	return &filemap_alloc_folio(gfp, 0)->page;
+	return &filemap_alloc_folio(gfp, 0, NULL)->page;
 }
 static inline gfp_t readahead_gfp_mask(struct address_space *x)
 }
 void *filemap_get_entry(struct address_space *mapping, pgoff_t index);
-struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,
+struct folio *__filemap_get_folio_mpol(struct address_space *mapping,
-		fgf_t fgp_flags, gfp_t gfp);
+		pgoff_t index, fgf_t fgf_flags, gfp_t gfp, struct mempolicy *policy);
 struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index,
 		fgf_t fgp_flags, gfp_t gfp);
+static inline struct folio *__filemap_get_folio(struct address_space *mapping,
+		pgoff_t index, fgf_t fgf_flags, gfp_t gfp)
+{
+	return __filemap_get_folio_mpol(mapping, index, fgf_flags, gfp, NULL);
+}
 /**
  * write_begin_get_folio - Get folio for write_begin with flags.
  * @iocb: The kiocb passed from write_begin (may be NULL).

include/linux/psp-sev.h+37−0

@@ -14,6 +14,39 @@ · 13 unmodified lines
14		14
15	#include <uapi/linux/psp-sev.h>	15	#include <uapi/linux/psp-sev.h>
16		16
		17	/* As defined by SEV API, under "Guest Policy". */
		18	#define SEV_POLICY_MASK_NODBG BIT(0)
		19	#define SEV_POLICY_MASK_NOKS BIT(1)
		20	#define SEV_POLICY_MASK_ES BIT(2)
		21	#define SEV_POLICY_MASK_NOSEND BIT(3)
		22	#define SEV_POLICY_MASK_DOMAIN BIT(4)
		23	#define SEV_POLICY_MASK_SEV BIT(5)
		24	#define SEV_POLICY_MASK_API_MAJOR GENMASK(23, 16)
		25	#define SEV_POLICY_MASK_API_MINOR GENMASK(31, 24)
		26
		27	/* As defined by SEV-SNP Firmware ABI, under "Guest Policy". */
		28	#define SNP_POLICY_MASK_API_MINOR GENMASK_ULL(7, 0)
		29	#define SNP_POLICY_MASK_API_MAJOR GENMASK_ULL(15, 8)
		30	#define SNP_POLICY_MASK_SMT BIT_ULL(16)
		31	#define SNP_POLICY_MASK_RSVD_MBO BIT_ULL(17)
		32	#define SNP_POLICY_MASK_MIGRATE_MA BIT_ULL(18)
		33	#define SNP_POLICY_MASK_DEBUG BIT_ULL(19)
		34	#define SNP_POLICY_MASK_SINGLE_SOCKET BIT_ULL(20)
		35	#define SNP_POLICY_MASK_CXL_ALLOW BIT_ULL(21)
		36	#define SNP_POLICY_MASK_MEM_AES_256_XTS BIT_ULL(22)
		37	#define SNP_POLICY_MASK_RAPL_DIS BIT_ULL(23)
		38	#define SNP_POLICY_MASK_CIPHERTEXT_HIDING_DRAM BIT_ULL(24)
		39	#define SNP_POLICY_MASK_PAGE_SWAP_DISABLE BIT_ULL(25)
		40
		41	/* Base SEV-SNP policy bitmask for minimum supported SEV firmware version */
		42	#define SNP_POLICY_MASK_BASE (SNP_POLICY_MASK_API_MINOR \| \
		43	SNP_POLICY_MASK_API_MAJOR \| \
		44	SNP_POLICY_MASK_SMT \| \
		45	SNP_POLICY_MASK_RSVD_MBO \| \
		46	SNP_POLICY_MASK_MIGRATE_MA \| \
		47	SNP_POLICY_MASK_DEBUG \| \
		48	SNP_POLICY_MASK_SINGLE_SOCKET)
		49
17	#define SEV_FW_BLOB_MAX_SIZE 0x4000 /* 16KB */	50	#define SEV_FW_BLOB_MAX_SIZE 0x4000 /* 16KB */
18		51
19	/**	52	/**
@@ -849,7 +882,10 @@ struct snp_feature_info { · 829 unmodified lines
849	u32 edx;	882	u32 edx;
850	} __packed;	883	} __packed;
851		884
		885	#define SNP_RAPL_DISABLE_SUPPORTED BIT(2)
852	#define SNP_CIPHER_TEXT_HIDING_SUPPORTED BIT(3)	886	#define SNP_CIPHER_TEXT_HIDING_SUPPORTED BIT(3)
		887	#define SNP_AES_256_XTS_POLICY_SUPPORTED BIT(4)
		888	#define SNP_CXL_ALLOW_POLICY_SUPPORTED BIT(5)
853		889
854	#ifdef CONFIG_CRYPTO_DEV_SP_PSP	890	#ifdef CONFIG_CRYPTO_DEV_SP_PSP
855		891
@@ -995,6 +1031,7 @@ void *snp_alloc_firmware_page(gfp_t mask); · 139 unmodified lines
995	void snp_free_firmware_page(void *addr);	1031	void snp_free_firmware_page(void *addr);
996	void sev_platform_shutdown(void);	1032	void sev_platform_shutdown(void);
997	bool sev_is_snp_ciphertext_hiding_supported(void);	1033	bool sev_is_snp_ciphertext_hiding_supported(void);
		1034	u64 sev_get_snp_policy_bits(void);
998		1035
999	#else /* !CONFIG_CRYPTO_DEV_SP_PSP */	1036	#else /* !CONFIG_CRYPTO_DEV_SP_PSP */
1000		1037

include/uapi/linux/kvm.h+11−0

 #define KVM_EXIT_LOONGARCH_IOCSR  38
 #define KVM_EXIT_MEMORY_FAULT     39
 #define KVM_EXIT_TDX              40
+#define KVM_EXIT_ARM_SEA          41
 /* For KVM_EXIT_INTERNAL_ERROR */
 /* Emulate instruction failed. */
 				} setup_event_notify;
 			};
 		} tdx;
+		/* KVM_EXIT_ARM_SEA */
+		struct {
+#define KVM_EXIT_ARM_SEA_FLAG_GPA_VALID	(1ULL << 0)
+			__u64 flags;
+			__u64 esr;
+			__u64 gva;
+			__u64 gpa;
+		} arm_sea;
 		/* Fix the size of the union. */
 		char padding[256];
 	};
 #define KVM_CAP_RISCV_MP_STATE_RESET 242
 #define KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED 243
 #define KVM_CAP_GUEST_MEMFD_FLAGS 244
+#define KVM_CAP_ARM_SEA_TO_USER 245
+#define KVM_CAP_S390_USER_OPEREXEC 246
 struct kvm_irq_routing_irqchip {
 	__u32 irqchip;

include/uapi/linux/magic.h+1−0

@@ -103,5 +103,6 @@ · 102 unmodified lines
103	#define DEVMEM_MAGIC 0x454d444d /* "DMEM" */	103	#define DEVMEM_MAGIC 0x454d444d /* "DMEM" */
104	#define SECRETMEM_MAGIC 0x5345434d /* "SECM" */	104	#define SECRETMEM_MAGIC 0x5345434d /* "SECM" */
105	#define PID_FS_MAGIC 0x50494446 /* "PIDF" */	105	#define PID_FS_MAGIC 0x50494446 /* "PIDF" */
		106	#define GUEST_MEMFD_MAGIC 0x474d454d /* "GMEM" */
106		107
107	#endif /* __LINUX_MAGIC_H__ */	108	#endif /* __LINUX_MAGIC_H__ */

mm/filemap.c+14−9

 EXPORT_SYMBOL_GPL(filemap_add_folio);
 #ifdef CONFIG_NUMA
-struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order)
+struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order,
+		struct mempolicy *policy)
 {
 	int n;
 	struct folio *folio;
+	if (policy)
+		return folio_alloc_mpol_noprof(gfp, order, policy,
+				NO_INTERLEAVE_INDEX, numa_node_id());
 	if (cpuset_do_page_mem_spread()) {
 		unsigned int cpuset_mems_cookie;
 		do {
 }
 /**
- * __filemap_get_folio - Find and get a reference to a folio.
+ * __filemap_get_folio_mpol - Find and get a reference to a folio.
  * @mapping: The address_space to search.
  * @index: The page index.
  * @fgp_flags: %FGP flags modify how the folio is returned.
  * @gfp: Memory allocation flags to use if %FGP_CREAT is specified.
+ * @policy: NUMA memory allocation policy to follow.
  *
  * Looks up the page cache entry at @mapping & @index.
  *
  *
  * Return: The found folio or an ERR_PTR() otherwise.
  */
-struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,
+struct folio *__filemap_get_folio_mpol(struct address_space *mapping,
-		fgf_t fgp_flags, gfp_t gfp)
+		pgoff_t index, fgf_t fgp_flags, gfp_t gfp, struct mempolicy *policy)
 {
 	struct folio *folio;
 			err = -ENOMEM;
 			if (order > min_order)
 				alloc_gfp |= __GFP_NORETRY | __GFP_NOWARN;
-			folio = filemap_alloc_folio(alloc_gfp, order);
+			folio = filemap_alloc_folio(alloc_gfp, order, policy);
 			if (!folio)
 				continue;
 		folio_clear_dropbehind(folio);
 	return folio;
 }
-EXPORT_SYMBOL(__filemap_get_folio);
+EXPORT_SYMBOL(__filemap_get_folio_mpol);
 static inline struct folio *find_get_entry(struct xa_state *xas, pgoff_t max,
 		xa_mark_t mark)
 	if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_WAITQ))
 		return -EAGAIN;
-	folio = filemap_alloc_folio(mapping_gfp_mask(mapping), min_order);
+	folio = filemap_alloc_folio(mapping_gfp_mask(mapping), min_order, NULL);
 	if (!folio)
 		return -ENOMEM;
 	if (iocb->ki_flags & IOCB_DONTCACHE)
 repeat:
 	folio = filemap_get_folio(mapping, index);
 	if (IS_ERR(folio)) {
-		folio = filemap_alloc_folio(gfp,
+		folio = filemap_alloc_folio(gfp, mapping_min_folio_order(mapping), NULL);
-					    mapping_min_folio_order(mapping));
 		if (!folio)
 			return ERR_PTR(-ENOMEM);
 		index = mapping_align_index(mapping, index);

mm/mempolicy.c+6−0

 	return &default_policy;
 }
+EXPORT_SYMBOL_FOR_MODULES(get_task_policy, "kvm");
 static const struct mempolicy_operations {
 	int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
 		return;
 	kmem_cache_free(policy_cache, pol);
 }
+EXPORT_SYMBOL_FOR_MODULES(__mpol_put, "kvm");
 static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)
 {
 	read_unlock(&sp->lock);
 	return pol;
 }
+EXPORT_SYMBOL_FOR_MODULES(mpol_shared_policy_lookup, "kvm");
 static void sp_free(struct sp_node *n)
 {
 		mpol_put(mpol);	/* drop our incoming ref on sb mpol */
 	}
 }
+EXPORT_SYMBOL_FOR_MODULES(mpol_shared_policy_init, "kvm");
 int mpol_set_shared_policy(struct shared_policy *sp,
 			struct vm_area_struct *vma, struct mempolicy *pol)
 		sp_free(new);
 	return err;
 }
+EXPORT_SYMBOL_FOR_MODULES(mpol_set_shared_policy, "kvm");
 /* Free a backing policy store on inode delete. */
 void mpol_free_shared_policy(struct shared_policy *sp)
 	}
 	write_unlock(&sp->lock);
 }
+EXPORT_SYMBOL_FOR_MODULES(mpol_free_shared_policy, "kvm");
 #ifdef CONFIG_NUMA_BALANCING
 static int __initdata numabalancing_override;

mm/readahead.c+1−1

@@ -186,7 +186,7 @@ static struct folio ractl_alloc_folio(struct readahead_control ractl, · 185 unmodified lines
186	{	186	{
187	struct folio *folio;	187	struct folio *folio;
188		188
189	folio = filemap_alloc_folio(gfp_mask, order);	189	folio = filemap_alloc_folio(gfp_mask, order, NULL);
190	if (folio && ractl->dropbehind)	190	if (folio && ractl->dropbehind)
191	__folio_set_dropbehind(folio);	191	__folio_set_dropbehind(folio);
192		192

tools/arch/arm64/include/asm/esr.h+2−0

@@ -141,6 +141,8 @@ · 140 unmodified lines
141	#define ESR_ELx_SF (UL(1) << ESR_ELx_SF_SHIFT)	141	#define ESR_ELx_SF (UL(1) << ESR_ELx_SF_SHIFT)
142	#define ESR_ELx_AR_SHIFT (14)	142	#define ESR_ELx_AR_SHIFT (14)
143	#define ESR_ELx_AR (UL(1) << ESR_ELx_AR_SHIFT)	143	#define ESR_ELx_AR (UL(1) << ESR_ELx_AR_SHIFT)
		144	#define ESR_ELx_VNCR_SHIFT (13)
		145	#define ESR_ELx_VNCR (UL(1) << ESR_ELx_VNCR_SHIFT)
144	#define ESR_ELx_CM_SHIFT (8)	146	#define ESR_ELx_CM_SHIFT (8)
145	#define ESR_ELx_CM (UL(1) << ESR_ELx_CM_SHIFT)	147	#define ESR_ELx_CM (UL(1) << ESR_ELx_CM_SHIFT)
146		148

tools/testing/selftests/kvm/Makefile+1−1

@@ -6,7 +6,7 @@ ARCH ?= $(SUBARCH) · 5 unmodified lines
6	ifeq ($(ARCH),$(filter $(ARCH),arm64 s390 riscv x86 x86_64 loongarch))	6	ifeq ($(ARCH),$(filter $(ARCH),arm64 s390 riscv x86 x86_64 loongarch))
7	# Top-level selftests allows ARCH=x86_64 :-(	7	# Top-level selftests allows ARCH=x86_64 :-(
8	ifeq ($(ARCH),x86_64)	8	ifeq ($(ARCH),x86_64)
9	ARCH := x86	9	override ARCH := x86
10	endif	10	endif
11	include Makefile.kvm	11	include Makefile.kvm
12	else	12	else

tools/testing/selftests/kvm/Makefile.kvm+9−3

 TEST_GEN_PROGS_x86 += x86/kvm_buslock_test
 TEST_GEN_PROGS_x86 += x86/monitor_mwait_test
 TEST_GEN_PROGS_x86 += x86/msrs_test
+TEST_GEN_PROGS_x86 += x86/nested_close_kvm_test
 TEST_GEN_PROGS_x86 += x86/nested_emulation_test
 TEST_GEN_PROGS_x86 += x86/nested_exceptions_test
+TEST_GEN_PROGS_x86 += x86/nested_invalid_cr3_test
+TEST_GEN_PROGS_x86 += x86/nested_tsc_adjust_test
+TEST_GEN_PROGS_x86 += x86/nested_tsc_scaling_test
 TEST_GEN_PROGS_x86 += x86/platform_info_test
 TEST_GEN_PROGS_x86 += x86/pmu_counters_test
 TEST_GEN_PROGS_x86 += x86/pmu_event_filter_test
 TEST_GEN_PROGS_x86 += x86/userspace_io_test
 TEST_GEN_PROGS_x86 += x86/userspace_msr_exit_test
 TEST_GEN_PROGS_x86 += x86/vmx_apic_access_test
-TEST_GEN_PROGS_x86 += x86/vmx_close_while_nested_test
 TEST_GEN_PROGS_x86 += x86/vmx_dirty_log_test
 TEST_GEN_PROGS_x86 += x86/vmx_exception_with_invalid_guest_state
 TEST_GEN_PROGS_x86 += x86/vmx_msrs_test
 TEST_GEN_PROGS_x86 += x86/vmx_invalid_nested_guest_state
+TEST_GEN_PROGS_x86 += x86/vmx_nested_la57_state_test
 TEST_GEN_PROGS_x86 += x86/vmx_set_nested_state_test
-TEST_GEN_PROGS_x86 += x86/vmx_tsc_adjust_test
-TEST_GEN_PROGS_x86 += x86/vmx_nested_tsc_scaling_test
 TEST_GEN_PROGS_x86 += x86/apic_bus_clock_test
 TEST_GEN_PROGS_x86 += x86/xapic_ipi_test
 TEST_GEN_PROGS_x86 += x86/xapic_state_test
 TEST_GEN_PROGS_arm64 = $(TEST_GEN_PROGS_COMMON)
 TEST_GEN_PROGS_arm64 += arm64/aarch32_id_regs
 TEST_GEN_PROGS_arm64 += arm64/arch_timer_edge_cases
+TEST_GEN_PROGS_arm64 += arm64/at
 TEST_GEN_PROGS_arm64 += arm64/debug-exceptions
 TEST_GEN_PROGS_arm64 += arm64/hello_el2
 TEST_GEN_PROGS_arm64 += arm64/host_sve
 TEST_GEN_PROGS_arm64 += arm64/external_aborts
 TEST_GEN_PROGS_arm64 += arm64/page_fault_test
 TEST_GEN_PROGS_arm64 += arm64/psci_test
+TEST_GEN_PROGS_arm64 += arm64/sea_to_user
 TEST_GEN_PROGS_arm64 += arm64/set_id_regs
 TEST_GEN_PROGS_arm64 += arm64/smccc_filter
 TEST_GEN_PROGS_arm64 += arm64/vcpu_width_config
 TEST_GEN_PROGS_s390 += s390/cpumodel_subfuncs_test
 TEST_GEN_PROGS_s390 += s390/shared_zeropage_test
 TEST_GEN_PROGS_s390 += s390/ucontrol_test
+TEST_GEN_PROGS_s390 += s390/user_operexec
 TEST_GEN_PROGS_s390 += rseq_test
 TEST_GEN_PROGS_riscv = $(TEST_GEN_PROGS_COMMON)
 TEST_GEN_PROGS_riscv += rseq_test
 TEST_GEN_PROGS_riscv += steal_time
+TEST_GEN_PROGS_loongarch = arch_timer
 TEST_GEN_PROGS_loongarch += coalesced_io_test
 TEST_GEN_PROGS_loongarch += demand_paging_test
 TEST_GEN_PROGS_loongarch += dirty_log_perf_test

tools/testing/selftests/kvm/arm64/at.c+166−0

@@ -0,0 +1,166 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* at - Test for KVM's AT emulation in the EL2&0 and EL1&0 translation regimes.
	4	*/
	5	#include "kvm_util.h"
	6	#include "processor.h"
	7	#include "test_util.h"
	8	#include "ucall.h"
	9
	10	#include <asm/sysreg.h>
	11
	12	#define TEST_ADDR 0x80000000
	13
	14	enum {
	15	CLEAR_ACCESS_FLAG,
	16	TEST_ACCESS_FLAG,
	17	};
	18
	19	static u64 *ptep_hva;
	20
	21	#define copy_el2_to_el1(reg) \
	22	write_sysreg_s(read_sysreg_s(SYS_##reg##_EL1), SYS_##reg##_EL12)
	23
	24	/* Yes, this is an ugly hack */
	25	#define __at(op, addr) write_sysreg_s(addr, op)
	26
	27	#define test_at_insn(op, expect_fault) \
	28	do { \
	29	u64 par, fsc; \
	30	bool fault; \
	31	\
	32	GUEST_SYNC(CLEAR_ACCESS_FLAG); \
	33	\
	34	__at(OP_AT_##op, TEST_ADDR); \
	35	isb(); \
	36	par = read_sysreg(par_el1); \
	37	\
	38	fault = par & SYS_PAR_EL1_F; \
	39	fsc = FIELD_GET(SYS_PAR_EL1_FST, par); \
	40	\
	41	__GUEST_ASSERT((expect_fault) == fault, \
	42	"AT "#op": %sexpected fault (par: %lx)1", \
	43	(expect_fault) ? "" : "un", par); \
	44	if ((expect_fault)) { \
	45	__GUEST_ASSERT(fsc == ESR_ELx_FSC_ACCESS_L(3), \
	46	"AT "#op": expected access flag fault (par: %lx)", \
	47	par); \
	48	} else { \
	49	GUEST_ASSERT_EQ(FIELD_GET(SYS_PAR_EL1_ATTR, par), MAIR_ATTR_NORMAL); \
	50	GUEST_ASSERT_EQ(FIELD_GET(SYS_PAR_EL1_SH, par), PTE_SHARED >> 8); \
	51	GUEST_ASSERT_EQ(par & SYS_PAR_EL1_PA, TEST_ADDR); \
	52	GUEST_SYNC(TEST_ACCESS_FLAG); \
	53	} \
	54	} while (0)
	55
	56	static void test_at(bool expect_fault)
	57	{
	58	test_at_insn(S1E2R, expect_fault);
	59	test_at_insn(S1E2W, expect_fault);
	60
	61	/* Reuse the stage-1 MMU context from EL2 at EL1 */
	62	copy_el2_to_el1(SCTLR);
	63	copy_el2_to_el1(MAIR);
	64	copy_el2_to_el1(TCR);
	65	copy_el2_to_el1(TTBR0);
	66	copy_el2_to_el1(TTBR1);
	67
	68	/* Disable stage-2 translation and enter a non-host context */
	69	write_sysreg(0, vtcr_el2);
	70	write_sysreg(0, vttbr_el2);
	71	sysreg_clear_set(hcr_el2, HCR_EL2_TGE \| HCR_EL2_VM, 0);
	72	isb();
	73
	74	test_at_insn(S1E1R, expect_fault);
	75	test_at_insn(S1E1W, expect_fault);
	76	}
	77
	78	static void guest_code(void)
	79	{
	80	sysreg_clear_set(tcr_el1, TCR_HA, 0);
	81	isb();
	82
	83	test_at(true);
	84
	85	if (!SYS_FIELD_GET(ID_AA64MMFR1_EL1, HAFDBS, read_sysreg(id_aa64mmfr1_el1)))
	86	GUEST_DONE();
	87
	88	/*
	89	* KVM's software PTW makes the implementation choice that the AT
	90	* instruction sets the access flag.
	91	*/
	92	sysreg_clear_set(tcr_el1, 0, TCR_HA);
	93	isb();
	94	test_at(false);
	95
	96	GUEST_DONE();
	97	}
	98
	99	static void handle_sync(struct kvm_vcpu vcpu, struct ucall uc)
	100	{
	101	switch (uc->args[1]) {
	102	case CLEAR_ACCESS_FLAG:
	103	/*
	104	* Delete + reinstall the memslot to invalidate stage-2
	105	* mappings of the stage-1 page tables, forcing KVM to
	106	* use the 'slow' AT emulation path.
	107	*
	108	* This and clearing the access flag from host userspace
	109	* ensures that the access flag cannot be set speculatively
	110	* and is reliably cleared at the time of the AT instruction.
	111	*/
	112	clear_bit(__ffs(PTE_AF), ptep_hva);
	113	vm_mem_region_reload(vcpu->vm, vcpu->vm->memslots[MEM_REGION_PT]);
	114	break;
	115	case TEST_ACCESS_FLAG:
	116	TEST_ASSERT(test_bit(__ffs(PTE_AF), ptep_hva),
	117	"Expected access flag to be set (desc: %lu)", *ptep_hva);
	118	break;
	119	default:
	120	TEST_FAIL("Unexpected SYNC arg: %lu", uc->args[1]);
	121	}
	122	}
	123
	124	static void run_test(struct kvm_vcpu *vcpu)
	125	{
	126	struct ucall uc;
	127
	128	while (true) {
	129	vcpu_run(vcpu);
	130	switch (get_ucall(vcpu, &uc)) {
	131	case UCALL_DONE:
	132	return;
	133	case UCALL_SYNC:
	134	handle_sync(vcpu, &uc);
	135	continue;
	136	case UCALL_ABORT:
	137	REPORT_GUEST_ASSERT(uc);
	138	return;
	139	default:
	140	TEST_FAIL("Unexpected ucall: %lu", uc.cmd);
	141	}
	142	}
	143	}
	144
	145	int main(void)
	146	{
	147	struct kvm_vcpu_init init;
	148	struct kvm_vcpu *vcpu;
	149	struct kvm_vm *vm;
	150
	151	TEST_REQUIRE(kvm_check_cap(KVM_CAP_ARM_EL2));
	152
	153	vm = vm_create(1);
	154
	155	kvm_get_default_vcpu_target(vm, &init);
	156	init.features[0] \|= BIT(KVM_ARM_VCPU_HAS_EL2);
	157	vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code);
	158	kvm_arch_vm_finalize_vcpus(vm);
	159
	160	virt_map(vm, TEST_ADDR, TEST_ADDR, 1);
	161	ptep_hva = virt_get_pte_hva_at_level(vm, TEST_ADDR, 3);
	162	run_test(vcpu);
	163
	164	kvm_vm_free(vm);
	165	return 0;
	166	}

tools/testing/selftests/kvm/arm64/sea_to_user.c+331−0

@@ -0,0 +1,331 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Test KVM returns to userspace with KVM_EXIT_ARM_SEA if host APEI fails
	4	* to handle SEA and userspace has opt-ed in KVM_CAP_ARM_SEA_TO_USER.
	5	*
	6	* After reaching userspace with expected arm_sea info, also test userspace
	7	* injecting a synchronous external data abort into the guest.
	8	*
	9	* This test utilizes EINJ to generate a REAL synchronous external data
	10	* abort by consuming a recoverable uncorrectable memory error. Therefore
	11	* the device under test must support EINJ in both firmware and host kernel,
	12	* including the notrigger feature. Otherwise the test will be skipped.
	13	* The under-test platform's APEI should be unable to claim SEA. Otherwise
	14	* the test will also be skipped.
	15	*/
	16
	17	#include <signal.h>
	18	#include <stdio.h>
	19	#include <stdlib.h>
	20	#include <unistd.h>
	21
	22	#include "test_util.h"
	23	#include "kvm_util.h"
	24	#include "processor.h"
	25	#include "guest_modes.h"
	26
	27	#define PAGE_PRESENT (1ULL << 63)
	28	#define PAGE_PHYSICAL 0x007fffffffffffffULL
	29	#define PAGE_ADDR_MASK (~(0xfffULL))
	30
	31	/* Group ISV and ISS[23:14]. */
	32	#define ESR_ELx_INST_SYNDROME ((ESR_ELx_ISV) \| (ESR_ELx_SAS) \| \
	33	(ESR_ELx_SSE) \| (ESR_ELx_SRT_MASK) \| \
	34	(ESR_ELx_SF) \| (ESR_ELx_AR))
	35
	36	#define EINJ_ETYPE "/sys/kernel/debug/apei/einj/error_type"
	37	#define EINJ_ADDR "/sys/kernel/debug/apei/einj/param1"
	38	#define EINJ_MASK "/sys/kernel/debug/apei/einj/param2"
	39	#define EINJ_FLAGS "/sys/kernel/debug/apei/einj/flags"
	40	#define EINJ_NOTRIGGER "/sys/kernel/debug/apei/einj/notrigger"
	41	#define EINJ_DOIT "/sys/kernel/debug/apei/einj/error_inject"
	42	/* Memory Uncorrectable non-fatal. */
	43	#define ERROR_TYPE_MEMORY_UER 0x10
	44	/* Memory address and mask valid (param1 and param2). */
	45	#define MASK_MEMORY_UER 0b10
	46
	47	/* Guest virtual address region = [2G, 3G). */
	48	#define START_GVA 0x80000000UL
	49	#define VM_MEM_SIZE 0x40000000UL
	50	/* Note: EINJ_OFFSET must < VM_MEM_SIZE. */
	51	#define EINJ_OFFSET 0x01234badUL
	52	#define EINJ_GVA ((START_GVA) + (EINJ_OFFSET))
	53
	54	static vm_paddr_t einj_gpa;
	55	static void *einj_hva;
	56	static uint64_t einj_hpa;
	57	static bool far_invalid;
	58
	59	static uint64_t translate_to_host_paddr(unsigned long vaddr)
	60	{
	61	uint64_t pinfo;
	62	int64_t offset = vaddr / getpagesize() * sizeof(pinfo);
	63	int fd;
	64	uint64_t page_addr;
	65	uint64_t paddr;
	66
	67	fd = open("/proc/self/pagemap", O_RDONLY);
	68	if (fd < 0)
	69	ksft_exit_fail_perror("Failed to open /proc/self/pagemap");
	70	if (pread(fd, &pinfo, sizeof(pinfo), offset) != sizeof(pinfo)) {
	71	close(fd);
	72	ksft_exit_fail_perror("Failed to read /proc/self/pagemap");
	73	}
	74
	75	close(fd);
	76
	77	if ((pinfo & PAGE_PRESENT) == 0)
	78	ksft_exit_fail_perror("Page not present");
	79
	80	page_addr = (pinfo & PAGE_PHYSICAL) << MIN_PAGE_SHIFT;
	81	paddr = page_addr + (vaddr & (getpagesize() - 1));
	82	return paddr;
	83	}
	84
	85	static void write_einj_entry(const char *einj_path, uint64_t val)
	86	{
	87	char cmd[256] = {0};
	88	FILE *cmdfile = NULL;
	89
	90	sprintf(cmd, "echo %#lx > %s", val, einj_path);
	91	cmdfile = popen(cmd, "r");
	92
	93	if (pclose(cmdfile) == 0)
	94	ksft_print_msg("echo %#lx > %s - done\n", val, einj_path);
	95	else
	96	ksft_exit_fail_perror("Failed to write EINJ entry");
	97	}
	98
	99	static void inject_uer(uint64_t paddr)
	100	{
	101	if (access("/sys/firmware/acpi/tables/EINJ", R_OK) == -1)
	102	ksft_test_result_skip("EINJ table no available in firmware");
	103
	104	if (access(EINJ_ETYPE, R_OK \| W_OK) == -1)
	105	ksft_test_result_skip("EINJ module probably not loaded?");
	106
	107	write_einj_entry(EINJ_ETYPE, ERROR_TYPE_MEMORY_UER);
	108	write_einj_entry(EINJ_FLAGS, MASK_MEMORY_UER);
	109	write_einj_entry(EINJ_ADDR, paddr);
	110	write_einj_entry(EINJ_MASK, ~0x0UL);
	111	write_einj_entry(EINJ_NOTRIGGER, 1);
	112	write_einj_entry(EINJ_DOIT, 1);
	113	}
	114
	115	/*
	116	* When host APEI successfully claims the SEA caused by guest_code, kernel
	117	* will send SIGBUS signal with BUS_MCEERR_AR to test thread.
	118	*
	119	* We set up this SIGBUS handler to skip the test for that case.
	120	*/
	121	static void sigbus_signal_handler(int sig, siginfo_t si, void v)
	122	{
	123	ksft_print_msg("SIGBUS (%d) received, dumping siginfo...\n", sig);
	124	ksft_print_msg("si_signo=%d, si_errno=%d, si_code=%d, si_addr=%p\n",
	125	si->si_signo, si->si_errno, si->si_code, si->si_addr);
	126	if (si->si_code == BUS_MCEERR_AR)
	127	ksft_test_result_skip("SEA is claimed by host APEI\n");
	128	else
	129	ksft_test_result_fail("Exit with signal unhandled\n");
	130
	131	exit(0);
	132	}
	133
	134	static void setup_sigbus_handler(void)
	135	{
	136	struct sigaction act;
	137
	138	memset(&act, 0, sizeof(act));
	139	sigemptyset(&act.sa_mask);
	140	act.sa_sigaction = sigbus_signal_handler;
	141	act.sa_flags = SA_SIGINFO;
	142	TEST_ASSERT(sigaction(SIGBUS, &act, NULL) == 0,
	143	"Failed to setup SIGBUS handler");
	144	}
	145
	146	static void guest_code(void)
	147	{
	148	uint64_t guest_data;
	149
	150	/* Consumes error will cause a SEA. */
	151	guest_data = (uint64_t )EINJ_GVA;
	152
	153	GUEST_FAIL("Poison not protected by SEA: gva=%#lx, guest_data=%#lx\n",
	154	EINJ_GVA, guest_data);
	155	}
	156
	157	static void expect_sea_handler(struct ex_regs *regs)
	158	{
	159	u64 esr = read_sysreg(esr_el1);
	160	u64 far = read_sysreg(far_el1);
	161	bool expect_far_invalid = far_invalid;
	162
	163	GUEST_PRINTF("Handling Guest SEA\n");
	164	GUEST_PRINTF("ESR_EL1=%#lx, FAR_EL1=%#lx\n", esr, far);
	165
	166	GUEST_ASSERT_EQ(ESR_ELx_EC(esr), ESR_ELx_EC_DABT_CUR);
	167	GUEST_ASSERT_EQ(esr & ESR_ELx_FSC_TYPE, ESR_ELx_FSC_EXTABT);
	168
	169	if (expect_far_invalid) {
	170	GUEST_ASSERT_EQ(esr & ESR_ELx_FnV, ESR_ELx_FnV);
	171	GUEST_PRINTF("Guest observed garbage value in FAR\n");
	172	} else {
	173	GUEST_ASSERT_EQ(esr & ESR_ELx_FnV, 0);
	174	GUEST_ASSERT_EQ(far, EINJ_GVA);
	175	}
	176
	177	GUEST_DONE();
	178	}
	179
	180	static void vcpu_inject_sea(struct kvm_vcpu *vcpu)
	181	{
	182	struct kvm_vcpu_events events = {};
	183
	184	events.exception.ext_dabt_pending = true;
	185	vcpu_events_set(vcpu, &events);
	186	}
	187
	188	static void run_vm(struct kvm_vm vm, struct kvm_vcpu vcpu)
	189	{
	190	struct ucall uc;
	191	bool guest_done = false;
	192	struct kvm_run *run = vcpu->run;
	193	u64 esr;
	194
	195	/* Resume the vCPU after error injection to consume the error. */
	196	vcpu_run(vcpu);
	197
	198	ksft_print_msg("Dump kvm_run info about KVM_EXIT_%s\n",
	199	exit_reason_str(run->exit_reason));
	200	ksft_print_msg("kvm_run.arm_sea: esr=%#llx, flags=%#llx\n",
	201	run->arm_sea.esr, run->arm_sea.flags);
	202	ksft_print_msg("kvm_run.arm_sea: gva=%#llx, gpa=%#llx\n",
	203	run->arm_sea.gva, run->arm_sea.gpa);
	204
	205	TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_ARM_SEA);
	206
	207	esr = run->arm_sea.esr;
	208	TEST_ASSERT_EQ(ESR_ELx_EC(esr), ESR_ELx_EC_DABT_LOW);
	209	TEST_ASSERT_EQ(esr & ESR_ELx_FSC_TYPE, ESR_ELx_FSC_EXTABT);
	210	TEST_ASSERT_EQ(ESR_ELx_ISS2(esr), 0);
	211	TEST_ASSERT_EQ((esr & ESR_ELx_INST_SYNDROME), 0);
	212	TEST_ASSERT_EQ(esr & ESR_ELx_VNCR, 0);
	213
	214	if (!(esr & ESR_ELx_FnV)) {
	215	ksft_print_msg("Expect gva to match given FnV bit is 0\n");
	216	TEST_ASSERT_EQ(run->arm_sea.gva, EINJ_GVA);
	217	}
	218
	219	if (run->arm_sea.flags & KVM_EXIT_ARM_SEA_FLAG_GPA_VALID) {
	220	ksft_print_msg("Expect gpa to match given KVM_EXIT_ARM_SEA_FLAG_GPA_VALID is set\n");
	221	TEST_ASSERT_EQ(run->arm_sea.gpa, einj_gpa & PAGE_ADDR_MASK);
	222	}
	223
	224	far_invalid = esr & ESR_ELx_FnV;
	225
	226	/* Inject a SEA into guest and expect handled in SEA handler. */
	227	vcpu_inject_sea(vcpu);
	228
	229	/* Expect the guest to reach GUEST_DONE gracefully. */
	230	do {
	231	vcpu_run(vcpu);
	232	switch (get_ucall(vcpu, &uc)) {
	233	case UCALL_PRINTF:
	234	ksft_print_msg("From guest: %s", uc.buffer);
	235	break;
	236	case UCALL_DONE:
	237	ksft_print_msg("Guest done gracefully!\n");
	238	guest_done = 1;
	239	break;
	240	case UCALL_ABORT:
	241	ksft_print_msg("Guest aborted!\n");
	242	guest_done = 1;
	243	REPORT_GUEST_ASSERT(uc);
	244	break;
	245	default:
	246	TEST_FAIL("Unexpected ucall: %lu\n", uc.cmd);
	247	}
	248	} while (!guest_done);
	249	}
	250
	251	static struct kvm_vm vm_create_with_sea_handler(struct kvm_vcpu *vcpu)
	252	{
	253	size_t backing_page_size;
	254	size_t guest_page_size;
	255	size_t alignment;
	256	uint64_t num_guest_pages;
	257	vm_paddr_t start_gpa;
	258	enum vm_mem_backing_src_type src_type = VM_MEM_SRC_ANONYMOUS_HUGETLB_1GB;
	259	struct kvm_vm *vm;
	260
	261	backing_page_size = get_backing_src_pagesz(src_type);
	262	guest_page_size = vm_guest_mode_params[VM_MODE_DEFAULT].page_size;
	263	alignment = max(backing_page_size, guest_page_size);
	264	num_guest_pages = VM_MEM_SIZE / guest_page_size;
	265
	266	vm = __vm_create_with_one_vcpu(vcpu, num_guest_pages, guest_code);
	267	vm_init_descriptor_tables(vm);
	268	vcpu_init_descriptor_tables(*vcpu);
	269
	270	vm_install_sync_handler(vm,
	271	/vector=/VECTOR_SYNC_CURRENT,
	272	/ec=/ESR_ELx_EC_DABT_CUR,
	273	/handler=/expect_sea_handler);
	274
	275	start_gpa = (vm->max_gfn - num_guest_pages) * guest_page_size;
	276	start_gpa = align_down(start_gpa, alignment);
	277
	278	vm_userspace_mem_region_add(
	279	/vm=/vm,
	280	/src_type=/src_type,
	281	/guest_paddr=/start_gpa,
	282	/slot=/1,
	283	/npages=/num_guest_pages,
	284	/flags=/0);
	285
	286	virt_map(vm, START_GVA, start_gpa, num_guest_pages);
	287
	288	ksft_print_msg("Mapped %#lx pages: gva=%#lx to gpa=%#lx\n",
	289	num_guest_pages, START_GVA, start_gpa);
	290	return vm;
	291	}
	292
	293	static void vm_inject_memory_uer(struct kvm_vm *vm)
	294	{
	295	uint64_t guest_data;
	296
	297	einj_gpa = addr_gva2gpa(vm, EINJ_GVA);
	298	einj_hva = addr_gva2hva(vm, EINJ_GVA);
	299
	300	/* Populate certain data before injecting UER. */
	301	(uint64_t )einj_hva = 0xBAADCAFE;
	302	guest_data = (uint64_t )einj_hva;
	303	ksft_print_msg("Before EINJect: data=%#lx\n",
	304	guest_data);
	305
	306	einj_hpa = translate_to_host_paddr((unsigned long)einj_hva);
	307
	308	ksft_print_msg("EINJ_GVA=%#lx, einj_gpa=%#lx, einj_hva=%p, einj_hpa=%#lx\n",
	309	EINJ_GVA, einj_gpa, einj_hva, einj_hpa);
	310
	311	inject_uer(einj_hpa);
	312	ksft_print_msg("Memory UER EINJected\n");
	313	}
	314
	315	int main(int argc, char *argv[])
	316	{
	317	struct kvm_vm *vm;
	318	struct kvm_vcpu *vcpu;
	319
	320	TEST_REQUIRE(kvm_has_cap(KVM_CAP_ARM_SEA_TO_USER));
	321
	322	setup_sigbus_handler();
	323
	324	vm = vm_create_with_sea_handler(&vcpu);
	325	vm_enable_cap(vm, KVM_CAP_ARM_SEA_TO_USER, 0);
	326	vm_inject_memory_uer(vm);
	327	run_vm(vm, vcpu);
	328	kvm_vm_free(vm);
	329
	330	return 0;
	331	}

tools/testing/selftests/kvm/arm64/vgic_irq.c+264−23

@@ -29,6 +29,7 @@ struct test_args { · 28 unmodified lines
29	bool level_sensitive; /* 1 is level, 0 is edge */	29	bool level_sensitive; /* 1 is level, 0 is edge */
30	int kvm_max_routes; /* output of KVM_CAP_IRQ_ROUTING */	30	int kvm_max_routes; /* output of KVM_CAP_IRQ_ROUTING */
31	bool kvm_supports_irqfd; /* output of KVM_CAP_IRQFD */	31	bool kvm_supports_irqfd; /* output of KVM_CAP_IRQFD */
		32	uint32_t shared_data;
32	};	33	};
33		34
34	/*	35	/*
@@ -205,7 +206,7 @@ static void kvm_inject_call(kvm_inject_cmd cmd, uint32_t first_intid, · 170 unmodified lines
205	do { \	206	do { \
206	uint32_t _intid; \	207	uint32_t _intid; \
207	_intid = gic_get_and_ack_irq(); \	208	_intid = gic_get_and_ack_irq(); \
208	GUEST_ASSERT(_intid == 0 \|\| _intid == IAR_SPURIOUS); \	209	GUEST_ASSERT(_intid == IAR_SPURIOUS); \
209	} while (0)	210	} while (0)
210		211
211	#define CAT_HELPER(a, b) a ## b	212	#define CAT_HELPER(a, b) a ## b
@@ -359,8 +360,9 @@ static uint32_t wait_for_and_activate_irq(void) · 147 unmodified lines
359	* interrupts for the whole test.	360	* interrupts for the whole test.
360	*/	361	*/
361	static void test_inject_preemption(struct test_args *args,	362	static void test_inject_preemption(struct test_args *args,
362	uint32_t first_intid, int num,	363	uint32_t first_intid, int num,
363	kvm_inject_cmd cmd)	364	const unsigned long *exclude,
		365	kvm_inject_cmd cmd)
364	{	366	{
365	uint32_t intid, prio, step = KVM_PRIO_STEPS;	367	uint32_t intid, prio, step = KVM_PRIO_STEPS;
366	int i;	368	int i;
@@ -379,6 +381,10 @@ static void test_inject_preemption(struct test_args *args, · 12 unmodified lines
379	for (i = 0; i < num; i++) {	381	for (i = 0; i < num; i++) {
380	uint32_t tmp;	382	uint32_t tmp;
381	intid = i + first_intid;	383	intid = i + first_intid;
		384
		385	if (exclude && test_bit(i, exclude))
		386	continue;
		387
382	KVM_INJECT(cmd, intid);	388	KVM_INJECT(cmd, intid);
383	/* Each successive IRQ will preempt the previous one. */	389	/* Each successive IRQ will preempt the previous one. */
384	tmp = wait_for_and_activate_irq();	390	tmp = wait_for_and_activate_irq();
@@ -390,15 +396,33 @@ static void test_inject_preemption(struct test_args *args, · 5 unmodified lines
390	/* finish handling the IRQs starting with the highest priority one. */	396	/* finish handling the IRQs starting with the highest priority one. */
391	for (i = 0; i < num; i++) {	397	for (i = 0; i < num; i++) {
392	intid = num - i - 1 + first_intid;	398	intid = num - i - 1 + first_intid;
		399
		400	if (exclude && test_bit(intid - first_intid, exclude))
		401	continue;
		402
393	gic_set_eoi(intid);	403	gic_set_eoi(intid);
394	if (args->eoi_split)	404	}
395	gic_set_dir(intid);	405
		406	if (args->eoi_split) {
		407	for (i = 0; i < num; i++) {
		408	intid = i + first_intid;
		409
		410	if (exclude && test_bit(i, exclude))
		411	continue;
		412
		413	if (args->eoi_split)
		414	gic_set_dir(intid);
		415	}
396	}	416	}
397		417
398	local_irq_enable();	418	local_irq_enable();
399		419
400	for (i = 0; i < num; i++)	420	for (i = 0; i < num; i++) {
		421	if (exclude && test_bit(i, exclude))
		422	continue;
		423
401	GUEST_ASSERT(!gic_irq_get_active(i + first_intid));	424	GUEST_ASSERT(!gic_irq_get_active(i + first_intid));
		425	}
402	GUEST_ASSERT_EQ(gic_read_ap1r0(), 0);	426	GUEST_ASSERT_EQ(gic_read_ap1r0(), 0);
403	GUEST_ASSERT_IAR_EMPTY();	427	GUEST_ASSERT_IAR_EMPTY();
404		428
@@ -436,33 +460,32 @@ static void test_injection_failure(struct test_args *args, · 31 unmodified lines
436		460
437	static void test_preemption(struct test_args args, struct kvm_inject_desc f)	461	static void test_preemption(struct test_args args, struct kvm_inject_desc f)
438	{	462	{
439	/*	463	/* Timer PPIs cannot be injected from userspace */
440	* Test up to 4 levels of preemption. The reason is that KVM doesn't	464	static const unsigned long ppi_exclude = (BIT(27 - MIN_PPI) \|
441	* currently implement the ability to have more than the number-of-LRs	465	BIT(30 - MIN_PPI) \|
442	* number of concurrently active IRQs. The number of LRs implemented is	466	BIT(28 - MIN_PPI) \|
443	* IMPLEMENTATION DEFINED, however, it seems that most implement 4.	467	BIT(26 - MIN_PPI));
444	*/	468
445	if (f->sgi)	469	if (f->sgi)
446	test_inject_preemption(args, MIN_SGI, 4, f->cmd);	470	test_inject_preemption(args, MIN_SGI, 16, NULL, f->cmd);
447		471
448	if (f->ppi)	472	if (f->ppi)
449	test_inject_preemption(args, MIN_PPI, 4, f->cmd);	473	test_inject_preemption(args, MIN_PPI, 16, &ppi_exclude, f->cmd);
450		474
451	if (f->spi)	475	if (f->spi)
452	test_inject_preemption(args, MIN_SPI, 4, f->cmd);	476	test_inject_preemption(args, MIN_SPI, 31, NULL, f->cmd);
453	}	477	}
454		478
455	static void test_restore_active(struct test_args args, struct kvm_inject_desc f)	479	static void test_restore_active(struct test_args args, struct kvm_inject_desc f)
456	{	480	{
457	/* Test up to 4 active IRQs. Same reason as in test_preemption. */
458	if (f->sgi)	481	if (f->sgi)
459	guest_restore_active(args, MIN_SGI, 4, f->cmd);	482	guest_restore_active(args, MIN_SGI, 16, f->cmd);
460		483
461	if (f->ppi)	484	if (f->ppi)
462	guest_restore_active(args, MIN_PPI, 4, f->cmd);	485	guest_restore_active(args, MIN_PPI, 16, f->cmd);
463		486
464	if (f->spi)	487	if (f->spi)
465	guest_restore_active(args, MIN_SPI, 4, f->cmd);	488	guest_restore_active(args, MIN_SPI, 31, f->cmd);
466	}	489	}
467		490
468	static void guest_code(struct test_args *args)	491	static void guest_code(struct test_args *args)
@@ -473,12 +496,12 @@ static void guest_code(struct test_args *args) · 4 unmodified lines
473		496
474	gic_init(GIC_V3, 1);	497	gic_init(GIC_V3, 1);
475		498
476	for (i = 0; i < nr_irqs; i++)
477	gic_irq_enable(i);
478
479	for (i = MIN_SPI; i < nr_irqs; i++)	499	for (i = MIN_SPI; i < nr_irqs; i++)
480	gic_irq_set_config(i, !level_sensitive);	500	gic_irq_set_config(i, !level_sensitive);
481		501
		502	for (i = 0; i < nr_irqs; i++)
		503	gic_irq_enable(i);
		504
482	gic_set_eoi_split(args->eoi_split);	505	gic_set_eoi_split(args->eoi_split);
483		506
484	reset_priorities(args);	507	reset_priorities(args);
@@ -636,7 +659,7 @@ static void kvm_routing_and_irqfd_check(struct kvm_vm *vm, · 151 unmodified lines
636	}	659	}
637		660
638	for (f = 0, i = intid; i < (uint64_t)intid + num; i++, f++)	661	for (f = 0, i = intid; i < (uint64_t)intid + num; i++, f++)
639	close(fd[f]);	662	kvm_close(fd[f]);
640	}	663	}
641		664
642	/* handles the valid case: intid=0xffffffff num=1 */	665	/* handles the valid case: intid=0xffffffff num=1 */
@@ -779,6 +802,221 @@ done: · 136 unmodified lines
779	kvm_vm_free(vm);	802	kvm_vm_free(vm);
780	}	803	}
781		804
		805	static void guest_code_asym_dir(struct test_args *args, int cpuid)
		806	{
		807	gic_init(GIC_V3, 2);
		808
		809	gic_set_eoi_split(1);
		810	gic_set_priority_mask(CPU_PRIO_MASK);
		811
		812	if (cpuid == 0) {
		813	uint32_t intid;
		814
		815	local_irq_disable();
		816
		817	gic_set_priority(MIN_PPI, IRQ_DEFAULT_PRIO);
		818	gic_irq_enable(MIN_SPI);
		819	gic_irq_set_pending(MIN_SPI);
		820
		821	intid = wait_for_and_activate_irq();
		822	GUEST_ASSERT_EQ(intid, MIN_SPI);
		823
		824	gic_set_eoi(intid);
		825	isb();
		826
		827	WRITE_ONCE(args->shared_data, MIN_SPI);
		828	dsb(ishst);
		829
		830	do {
		831	dsb(ishld);
		832	} while (READ_ONCE(args->shared_data) == MIN_SPI);
		833	GUEST_ASSERT(!gic_irq_get_active(MIN_SPI));
		834	} else {
		835	do {
		836	dsb(ishld);
		837	} while (READ_ONCE(args->shared_data) != MIN_SPI);
		838
		839	gic_set_dir(MIN_SPI);
		840	isb();
		841
		842	WRITE_ONCE(args->shared_data, 0);
		843	dsb(ishst);
		844	}
		845
		846	GUEST_DONE();
		847	}
		848
		849	static void guest_code_group_en(struct test_args *args, int cpuid)
		850	{
		851	uint32_t intid;
		852
		853	gic_init(GIC_V3, 2);
		854
		855	gic_set_eoi_split(0);
		856	gic_set_priority_mask(CPU_PRIO_MASK);
		857	/* SGI0 is G0, which is disabled */
		858	gic_irq_set_group(0, 0);
		859
		860	/* Configure all SGIs with decreasing priority */
		861	for (intid = 0; intid < MIN_PPI; intid++) {
		862	gic_set_priority(intid, (intid + 1) * 8);
		863	gic_irq_enable(intid);
		864	gic_irq_set_pending(intid);
		865	}
		866
		867	/* Ack and EOI all G1 interrupts */
		868	for (int i = 1; i < MIN_PPI; i++) {
		869	intid = wait_for_and_activate_irq();
		870
		871	GUEST_ASSERT(intid < MIN_PPI);
		872	gic_set_eoi(intid);
		873	isb();
		874	}
		875
		876	/*
		877	* Check that SGI0 is still pending, inactive, and that we cannot
		878	* ack anything.
		879	*/
		880	GUEST_ASSERT(gic_irq_get_pending(0));
		881	GUEST_ASSERT(!gic_irq_get_active(0));
		882	GUEST_ASSERT_IAR_EMPTY();
		883	GUEST_ASSERT(read_sysreg_s(SYS_ICC_IAR0_EL1) == IAR_SPURIOUS);
		884
		885	/* Open the G0 gates, and verify we can ack SGI0 */
		886	write_sysreg_s(1, SYS_ICC_IGRPEN0_EL1);
		887	isb();
		888
		889	do {
		890	intid = read_sysreg_s(SYS_ICC_IAR0_EL1);
		891	} while (intid == IAR_SPURIOUS);
		892
		893	GUEST_ASSERT(intid == 0);
		894	GUEST_DONE();
		895	}
		896
		897	static void guest_code_timer_spi(struct test_args *args, int cpuid)
		898	{
		899	uint32_t intid;
		900	u64 val;
		901
		902	gic_init(GIC_V3, 2);
		903
		904	gic_set_eoi_split(1);
		905	gic_set_priority_mask(CPU_PRIO_MASK);
		906
		907	/* Add a pending SPI so that KVM starts trapping DIR */
		908	gic_set_priority(MIN_SPI + cpuid, IRQ_DEFAULT_PRIO);
		909	gic_irq_set_pending(MIN_SPI + cpuid);
		910
		911	/* Configure the timer with a higher priority, make it pending */
		912	gic_set_priority(27, IRQ_DEFAULT_PRIO - 8);
		913
		914	isb();
		915	val = read_sysreg(cntvct_el0);
		916	write_sysreg(val, cntv_cval_el0);
		917	write_sysreg(1, cntv_ctl_el0);
		918	isb();
		919
		920	GUEST_ASSERT(gic_irq_get_pending(27));
		921
		922	/* Enable both interrupts */
		923	gic_irq_enable(MIN_SPI + cpuid);
		924	gic_irq_enable(27);
		925
		926	/* The timer must fire */
		927	intid = wait_for_and_activate_irq();
		928	GUEST_ASSERT(intid == 27);
		929
		930	/* Check that we can deassert it */
		931	write_sysreg(0, cntv_ctl_el0);
		932	isb();
		933
		934	GUEST_ASSERT(!gic_irq_get_pending(27));
		935
		936	/*
		937	* Priority drop, deactivation -- we expect that the host
		938	* deactivation will have been effective
		939	*/
		940	gic_set_eoi(27);
		941	gic_set_dir(27);
		942
		943	GUEST_ASSERT(!gic_irq_get_active(27));
		944
		945	/* Do it one more time */
		946	isb();
		947	val = read_sysreg(cntvct_el0);
		948	write_sysreg(val, cntv_cval_el0);
		949	write_sysreg(1, cntv_ctl_el0);
		950	isb();
		951
		952	GUEST_ASSERT(gic_irq_get_pending(27));
		953
		954	/* The timer must fire again */
		955	intid = wait_for_and_activate_irq();
		956	GUEST_ASSERT(intid == 27);
		957
		958	GUEST_DONE();
		959	}
		960
		961	static void test_vcpu_run(void arg)
		962	{
		963	struct kvm_vcpu *vcpu = arg;
		964	struct ucall uc;
		965
		966	while (1) {
		967	vcpu_run(vcpu);
		968
		969	switch (get_ucall(vcpu, &uc)) {
		970	case UCALL_ABORT:
		971	REPORT_GUEST_ASSERT(uc);
		972	break;
		973	case UCALL_DONE:
		974	return NULL;
		975	default:
		976	TEST_FAIL("Unknown ucall %lu", uc.cmd);
		977	}
		978	}
		979
		980	return NULL;
		981	}
		982
		983	static void test_vgic_two_cpus(void *gcode)
		984	{
		985	pthread_t thr[2];
		986	struct kvm_vcpu *vcpus[2];
		987	struct test_args args = {};
		988	struct kvm_vm *vm;
		989	vm_vaddr_t args_gva;
		990	int gic_fd, ret;
		991
		992	vm = vm_create_with_vcpus(2, gcode, vcpus);
		993
		994	vm_init_descriptor_tables(vm);
		995	vcpu_init_descriptor_tables(vcpus[0]);
		996	vcpu_init_descriptor_tables(vcpus[1]);
		997
		998	/* Setup the guest args page (so it gets the args). */
		999	args_gva = vm_vaddr_alloc_page(vm);
		1000	memcpy(addr_gva2hva(vm, args_gva), &args, sizeof(args));
		1001	vcpu_args_set(vcpus[0], 2, args_gva, 0);
		1002	vcpu_args_set(vcpus[1], 2, args_gva, 1);
		1003
		1004	gic_fd = vgic_v3_setup(vm, 2, 64);
		1005
		1006	ret = pthread_create(&thr[0], NULL, test_vcpu_run, vcpus[0]);
		1007	if (ret)
		1008	TEST_FAIL("Can't create thread for vcpu 0 (%d)\n", ret);
		1009	ret = pthread_create(&thr[1], NULL, test_vcpu_run, vcpus[1]);
		1010	if (ret)
		1011	TEST_FAIL("Can't create thread for vcpu 1 (%d)\n", ret);
		1012
		1013	pthread_join(thr[0], NULL);
		1014	pthread_join(thr[1], NULL);
		1015
		1016	close(gic_fd);
		1017	kvm_vm_free(vm);
		1018	}
		1019
782	static void help(const char *name)	1020	static void help(const char *name)
783	{	1021	{
784	printf(	1022	printf(
@@ -835,6 +1073,9 @@ int main(int argc, char **argv) · 50 unmodified lines
835	test_vgic(nr_irqs, false /* level /, true / eoi_split */);	1073	test_vgic(nr_irqs, false /* level /, true / eoi_split */);
836	test_vgic(nr_irqs, true /* level /, false / eoi_split */);	1074	test_vgic(nr_irqs, true /* level /, false / eoi_split */);
837	test_vgic(nr_irqs, true /* level /, true / eoi_split */);	1075	test_vgic(nr_irqs, true /* level /, true / eoi_split */);
		1076	test_vgic_two_cpus(guest_code_asym_dir);
		1077	test_vgic_two_cpus(guest_code_group_en);
		1078	test_vgic_two_cpus(guest_code_timer_spi);
838	} else {	1079	} else {
839	test_vgic(nr_irqs, level_sensitive, eoi_split);	1080	test_vgic(nr_irqs, level_sensitive, eoi_split);
840	}	1081	}

tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c+4−0

@@ -118,6 +118,10 @@ static void guest_setup_gic(void) · 117 unmodified lines
118		118
119	guest_setup_its_mappings();	119	guest_setup_its_mappings();
120	guest_invalidate_all_rdists();	120	guest_invalidate_all_rdists();
		121
		122	/* SYNC to ensure ITS setup is complete */
		123	for (cpuid = 0; cpuid < test_data.nr_cpus; cpuid++)
		124	its_send_sync_cmd(test_data.cmdq_base_va, cpuid);
121	}	125	}
122		126
123	static void guest_code(size_t nr_lpis)	127	static void guest_code(size_t nr_lpis)

tools/testing/selftests/kvm/guest_memfd_test.c+98−0

@@ -19,6 +19,7 @@ · 18 unmodified lines
19	#include <sys/stat.h>	19	#include <sys/stat.h>
20		20
21	#include "kvm_util.h"	21	#include "kvm_util.h"
		22	#include "numaif.h"
22	#include "test_util.h"	23	#include "test_util.h"
23	#include "ucall_common.h"	24	#include "ucall_common.h"
24		25
@@ -75,6 +76,101 @@ static void test_mmap_supported(int fd, size_t total_size) · 50 unmodified lines
75	kvm_munmap(mem, total_size);	76	kvm_munmap(mem, total_size);
76	}	77	}
77		78
		79	static void test_mbind(int fd, size_t total_size)
		80	{
		81	const unsigned long nodemask_0 = 1; /* nid: 0 */
		82	unsigned long nodemask = 0;
		83	unsigned long maxnode = 8;
		84	int policy;
		85	char *mem;
		86	int ret;
		87
		88	if (!is_multi_numa_node_system())
		89	return;
		90
		91	mem = kvm_mmap(total_size, PROT_READ \| PROT_WRITE, MAP_SHARED, fd);
		92
		93	/* Test MPOL_INTERLEAVE policy */
		94	kvm_mbind(mem, page_size * 2, MPOL_INTERLEAVE, &nodemask_0, maxnode, 0);
		95	kvm_get_mempolicy(&policy, &nodemask, maxnode, mem, MPOL_F_ADDR);
		96	TEST_ASSERT(policy == MPOL_INTERLEAVE && nodemask == nodemask_0,
		97	"Wanted MPOL_INTERLEAVE (%u) and nodemask 0x%lx, got %u and 0x%lx",
		98	MPOL_INTERLEAVE, nodemask_0, policy, nodemask);
		99
		100	/* Test basic MPOL_BIND policy */
		101	kvm_mbind(mem + page_size * 2, page_size * 2, MPOL_BIND, &nodemask_0, maxnode, 0);
		102	kvm_get_mempolicy(&policy, &nodemask, maxnode, mem + page_size * 2, MPOL_F_ADDR);
		103	TEST_ASSERT(policy == MPOL_BIND && nodemask == nodemask_0,
		104	"Wanted MPOL_BIND (%u) and nodemask 0x%lx, got %u and 0x%lx",
		105	MPOL_BIND, nodemask_0, policy, nodemask);
		106
		107	/* Test MPOL_DEFAULT policy */
		108	kvm_mbind(mem, total_size, MPOL_DEFAULT, NULL, 0, 0);
		109	kvm_get_mempolicy(&policy, &nodemask, maxnode, mem, MPOL_F_ADDR);
		110	TEST_ASSERT(policy == MPOL_DEFAULT && !nodemask,
		111	"Wanted MPOL_DEFAULT (%u) and nodemask 0x0, got %u and 0x%lx",
		112	MPOL_DEFAULT, policy, nodemask);
		113
		114	/* Test with invalid policy */
		115	ret = mbind(mem, page_size, 999, &nodemask_0, maxnode, 0);
		116	TEST_ASSERT(ret == -1 && errno == EINVAL,
		117	"mbind with invalid policy should fail with EINVAL");
		118
		119	kvm_munmap(mem, total_size);
		120	}
		121
		122	static void test_numa_allocation(int fd, size_t total_size)
		123	{
		124	unsigned long node0_mask = 1; /* Node 0 */
		125	unsigned long node1_mask = 2; /* Node 1 */
		126	unsigned long maxnode = 8;
		127	void *pages[4];
		128	int status[4];
		129	char *mem;
		130	int i;
		131
		132	if (!is_multi_numa_node_system())
		133	return;
		134
		135	mem = kvm_mmap(total_size, PROT_READ \| PROT_WRITE, MAP_SHARED, fd);
		136
		137	for (i = 0; i < 4; i++)
		138	pages[i] = (char )mem + page_size i;
		139
		140	/* Set NUMA policy after allocation */
		141	memset(mem, 0xaa, page_size);
		142	kvm_mbind(pages[0], page_size, MPOL_BIND, &node0_mask, maxnode, 0);
		143	kvm_fallocate(fd, FALLOC_FL_PUNCH_HOLE \| FALLOC_FL_KEEP_SIZE, 0, page_size);
		144
		145	/* Set NUMA policy before allocation */
		146	kvm_mbind(pages[0], page_size * 2, MPOL_BIND, &node1_mask, maxnode, 0);
		147	kvm_mbind(pages[2], page_size * 2, MPOL_BIND, &node0_mask, maxnode, 0);
		148	memset(mem, 0xaa, total_size);
		149
		150	/* Validate if pages are allocated on specified NUMA nodes */
		151	kvm_move_pages(0, 4, pages, NULL, status, 0);
		152	TEST_ASSERT(status[0] == 1, "Expected page 0 on node 1, got it on node %d", status[0]);
		153	TEST_ASSERT(status[1] == 1, "Expected page 1 on node 1, got it on node %d", status[1]);
		154	TEST_ASSERT(status[2] == 0, "Expected page 2 on node 0, got it on node %d", status[2]);
		155	TEST_ASSERT(status[3] == 0, "Expected page 3 on node 0, got it on node %d", status[3]);
		156
		157	/* Punch hole for all pages */
		158	kvm_fallocate(fd, FALLOC_FL_PUNCH_HOLE \| FALLOC_FL_KEEP_SIZE, 0, total_size);
		159
		160	/* Change NUMA policy nodes and reallocate */
		161	kvm_mbind(pages[0], page_size * 2, MPOL_BIND, &node0_mask, maxnode, 0);
		162	kvm_mbind(pages[2], page_size * 2, MPOL_BIND, &node1_mask, maxnode, 0);
		163	memset(mem, 0xaa, total_size);
		164
		165	kvm_move_pages(0, 4, pages, NULL, status, 0);
		166	TEST_ASSERT(status[0] == 0, "Expected page 0 on node 0, got it on node %d", status[0]);
		167	TEST_ASSERT(status[1] == 0, "Expected page 1 on node 0, got it on node %d", status[1]);
		168	TEST_ASSERT(status[2] == 1, "Expected page 2 on node 1, got it on node %d", status[2]);
		169	TEST_ASSERT(status[3] == 1, "Expected page 3 on node 1, got it on node %d", status[3]);
		170
		171	kvm_munmap(mem, total_size);
		172	}
		173
78	static void test_fault_sigbus(int fd, size_t accessible_size, size_t map_size)	174	static void test_fault_sigbus(int fd, size_t accessible_size, size_t map_size)
79	{	175	{
80	const char val = 0xaa;	176	const char val = 0xaa;
@@ -273,11 +369,13 @@ static void __test_guest_memfd(struct kvm_vm *vm, uint64_t flags) · 192 unmodified lines
273	if (flags & GUEST_MEMFD_FLAG_INIT_SHARED) {	369	if (flags & GUEST_MEMFD_FLAG_INIT_SHARED) {
274	gmem_test(mmap_supported, vm, flags);	370	gmem_test(mmap_supported, vm, flags);
275	gmem_test(fault_overflow, vm, flags);	371	gmem_test(fault_overflow, vm, flags);
		372	gmem_test(numa_allocation, vm, flags);
276	} else {	373	} else {
277	gmem_test(fault_private, vm, flags);	374	gmem_test(fault_private, vm, flags);
278	}	375	}
279		376
280	gmem_test(mmap_cow, vm, flags);	377	gmem_test(mmap_cow, vm, flags);
		378	gmem_test(mbind, vm, flags);
281	} else {	379	} else {
282	gmem_test(mmap_not_supported, vm, flags);	380	gmem_test(mmap_not_supported, vm, flags);
283	}	381	}

tools/testing/selftests/kvm/include/arm64/gic.h+1−0

@@ -57,6 +57,7 @@ void gic_irq_set_pending(unsigned int intid); · 56 unmodified lines
57	void gic_irq_clear_pending(unsigned int intid);	57	void gic_irq_clear_pending(unsigned int intid);
58	bool gic_irq_get_pending(unsigned int intid);	58	bool gic_irq_get_pending(unsigned int intid);
59	void gic_irq_set_config(unsigned int intid, bool is_edge);	59	void gic_irq_set_config(unsigned int intid, bool is_edge);
		60	void gic_irq_set_group(unsigned int intid, bool group);
60		61
61	void gic_rdist_enable_lpis(vm_paddr_t cfg_table, size_t cfg_table_size,	62	void gic_rdist_enable_lpis(vm_paddr_t cfg_table, size_t cfg_table_size,
62	vm_paddr_t pend_table);	63	vm_paddr_t pend_table);

tools/testing/selftests/kvm/include/arm64/gic_v3_its.h+1−0

@@ -15,5 +15,6 @@ void its_send_mapc_cmd(void *cmdq_base, u32 vcpu_id, u32 collection_id, bool val · 14 unmodified lines
15	void its_send_mapti_cmd(void *cmdq_base, u32 device_id, u32 event_id,	15	void its_send_mapti_cmd(void *cmdq_base, u32 device_id, u32 event_id,
16	u32 collection_id, u32 intid);	16	u32 collection_id, u32 intid);
17	void its_send_invall_cmd(void *cmdq_base, u32 collection_id);	17	void its_send_invall_cmd(void *cmdq_base, u32 collection_id);
		18	void its_send_sync_cmd(void *cmdq_base, u32 vcpu_id);
18		19
19	#endif // __SELFTESTS_GIC_V3_ITS_H__	20	#endif // __SELFTESTS_GIC_V3_ITS_H__

tools/testing/selftests/kvm/include/kvm_syscalls.h+81−0

@@ -0,0 +1,81 @@
	1	/* SPDX-License-Identifier: GPL-2.0-only */
	2	#ifndef SELFTEST_KVM_SYSCALLS_H
	3	#define SELFTEST_KVM_SYSCALLS_H
	4
	5	#include <sys/syscall.h>
	6
	7	#define MAP_ARGS0(m,...)
	8	#define MAP_ARGS1(m,t,a,...) m(t,a)
	9	#define MAP_ARGS2(m,t,a,...) m(t,a), MAP_ARGS1(m,__VA_ARGS__)
	10	#define MAP_ARGS3(m,t,a,...) m(t,a), MAP_ARGS2(m,__VA_ARGS__)
	11	#define MAP_ARGS4(m,t,a,...) m(t,a), MAP_ARGS3(m,__VA_ARGS__)
	12	#define MAP_ARGS5(m,t,a,...) m(t,a), MAP_ARGS4(m,__VA_ARGS__)
	13	#define MAP_ARGS6(m,t,a,...) m(t,a), MAP_ARGS5(m,__VA_ARGS__)
	14	#define MAP_ARGS(n,...) MAP_ARGS##n(__VA_ARGS__)
	15
	16	#define __DECLARE_ARGS(t, a) t a
	17	#define __UNPACK_ARGS(t, a) a
	18
	19	#define DECLARE_ARGS(nr_args, args...) MAP_ARGS(nr_args, __DECLARE_ARGS, args)
	20	#define UNPACK_ARGS(nr_args, args...) MAP_ARGS(nr_args, __UNPACK_ARGS, args)
	21
	22	#define __KVM_SYSCALL_ERROR(_name, _ret) \
	23	"%s failed, rc: %i errno: %i (%s)", (_name), (_ret), errno, strerror(errno)
	24
	25	/* Define a kvm_<syscall>() API to assert success. */
	26	#define __KVM_SYSCALL_DEFINE(name, nr_args, args...) \
	27	static inline void kvm_##name(DECLARE_ARGS(nr_args, args)) \
	28	{ \
	29	int r; \
	30	\
	31	r = name(UNPACK_ARGS(nr_args, args)); \
	32	TEST_ASSERT(!r, __KVM_SYSCALL_ERROR(#name, r)); \
	33	}
	34
	35	/*
	36	* Macro to define syscall APIs, either because KVM selftests doesn't link to
	37	* the standard library, e.g. libnuma, or because there is no library that yet
	38	* provides the syscall. These
	39	*/
	40	#define KVM_SYSCALL_DEFINE(name, nr_args, args...) \
	41	static inline long name(DECLARE_ARGS(nr_args, args)) \
	42	{ \
	43	return syscall(__NR_##name, UNPACK_ARGS(nr_args, args)); \
	44	} \
	45	__KVM_SYSCALL_DEFINE(name, nr_args, args)
	46
	47	/*
	48	* Special case mmap(), as KVM selftest rarely/never specific an address,
	49	* rarely specify an offset, and because the unique return code requires
	50	* special handling anyways.
	51	*/
	52	static inline void *__kvm_mmap(size_t size, int prot, int flags, int fd,
	53	off_t offset)
	54	{
	55	void *mem;
	56
	57	mem = mmap(NULL, size, prot, flags, fd, offset);
	58	TEST_ASSERT(mem != MAP_FAILED, __KVM_SYSCALL_ERROR("mmap()",
	59	(int)(unsigned long)MAP_FAILED));
	60	return mem;
	61	}
	62
	63	static inline void *kvm_mmap(size_t size, int prot, int flags, int fd)
	64	{
	65	return __kvm_mmap(size, prot, flags, fd, 0);
	66	}
	67
	68	static inline int kvm_dup(int fd)
	69	{
	70	int new_fd = dup(fd);
	71
	72	TEST_ASSERT(new_fd >= 0, __KVM_SYSCALL_ERROR("dup()", new_fd));
	73	return new_fd;
	74	}
	75
	76	__KVM_SYSCALL_DEFINE(munmap, 2, void *, mem, size_t, size);
	77	__KVM_SYSCALL_DEFINE(close, 1, int, fd);
	78	__KVM_SYSCALL_DEFINE(fallocate, 4, int, fd, int, mode, loff_t, offset, loff_t, len);
	79	__KVM_SYSCALL_DEFINE(ftruncate, 2, unsigned int, fd, off_t, length);
	80
	81	#endif /* SELFTEST_KVM_SYSCALLS_H */

tools/testing/selftests/kvm/include/kvm_util.h+10−35

 #include <pthread.h>
+#include "kvm_syscalls.h"
 #include "kvm_util_arch.h"
 #include "kvm_util_types.h"
 #include "sparsebit.h"
 	VM_MODE_P40V48_4K,
 	VM_MODE_P40V48_16K,
 	VM_MODE_P40V48_64K,
-	VM_MODE_PXXV48_4K,	/* For 48bits VA but ANY bits PA */
+	VM_MODE_PXXVYY_4K,	/* For 48-bit or 57-bit VA, depending on host support */
 	VM_MODE_P47V64_4K,
 	VM_MODE_P44V64_4K,
 	VM_MODE_P36V48_4K,
 #elif defined(__x86_64__)
-#define VM_MODE_DEFAULT			VM_MODE_PXXV48_4K
+#define VM_MODE_DEFAULT			VM_MODE_PXXVYY_4K
 #define MIN_PAGE_SHIFT			12U
 #define ptes_per_page(page_size)	((page_size) / 8)
 	return kvm_check_cap(cap);
 }
-#define __KVM_SYSCALL_ERROR(_name, _ret) \
-	"%s failed, rc: %i errno: %i (%s)", (_name), (_ret), errno, strerror(errno)
-static inline void *__kvm_mmap(size_t size, int prot, int flags, int fd,
-			       off_t offset)
-{
-	void *mem;
-	mem = mmap(NULL, size, prot, flags, fd, offset);
-	TEST_ASSERT(mem != MAP_FAILED, __KVM_SYSCALL_ERROR("mmap()",
-		    (int)(unsigned long)MAP_FAILED));
-	return mem;
-}
-static inline void *kvm_mmap(size_t size, int prot, int flags, int fd)
-{
-	return __kvm_mmap(size, prot, flags, fd, 0);
-}
-static inline void kvm_munmap(void *mem, size_t size)
-{
-	int ret;
-	ret = munmap(mem, size);
-	TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret));
-}
 /*
  * Use the "inner", double-underscore macro when reporting errors from within
  * other macros so that the name of ioctl() and not its literal numeric value
 				 uint32_t guest_memfd, uint64_t guest_memfd_offset);
 void vm_userspace_mem_region_add(struct kvm_vm *vm,
-	enum vm_mem_backing_src_type src_type,
+				 enum vm_mem_backing_src_type src_type,
-	uint64_t guest_paddr, uint32_t slot, uint64_t npages,
+				 uint64_t gpa, uint32_t slot, uint64_t npages,
-	uint32_t flags);
+				 uint32_t flags);
 void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
-		uint64_t guest_paddr, uint32_t slot, uint64_t npages,
+		uint64_t gpa, uint32_t slot, uint64_t npages, uint32_t flags,
-		uint32_t flags, int guest_memfd_fd, uint64_t guest_memfd_offset);
+		int guest_memfd_fd, uint64_t guest_memfd_offset);
 #ifndef vm_arch_has_protected_memory
 static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm)
 #endif
 void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags);
+void vm_mem_region_reload(struct kvm_vm *vm, uint32_t slot);
 void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa);
 void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot);
 struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id);
 static inline void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
 {
 	virt_arch_pg_map(vm, vaddr, paddr);
+	sparsebit_set(vm->vpages_mapped, vaddr >> vm->page_shift);
 }

tools/testing/selftests/kvm/include/loongarch/arch_timer.h+85−0

@@ -0,0 +1,85 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/*
	3	* LoongArch Constant Timer specific interface
	4	*/
	5	#ifndef SELFTEST_KVM_ARCH_TIMER_H
	6	#define SELFTEST_KVM_ARCH_TIMER_H
	7
	8	#include "processor.h"
	9
	10	/* LoongArch timer frequency is constant 100MHZ */
	11	#define TIMER_FREQ (100UL << 20)
	12	#define msec_to_cycles(msec) (TIMER_FREQ * (unsigned long)(msec) / 1000)
	13	#define usec_to_cycles(usec) (TIMER_FREQ * (unsigned long)(usec) / 1000000)
	14	#define cycles_to_usec(cycles) ((unsigned long)(cycles) * 1000000 / TIMER_FREQ)
	15
	16	static inline unsigned long timer_get_cycles(void)
	17	{
	18	unsigned long val = 0;
	19
	20	__asm__ __volatile__(
	21	"rdtime.d %0, $zero\n\t"
	22	: "=r"(val)
	23	:
	24	);
	25
	26	return val;
	27	}
	28
	29	static inline unsigned long timer_get_cfg(void)
	30	{
	31	return csr_read(LOONGARCH_CSR_TCFG);
	32	}
	33
	34	static inline unsigned long timer_get_val(void)
	35	{
	36	return csr_read(LOONGARCH_CSR_TVAL);
	37	}
	38
	39	static inline void disable_timer(void)
	40	{
	41	csr_write(0, LOONGARCH_CSR_TCFG);
	42	}
	43
	44	static inline void timer_irq_enable(void)
	45	{
	46	unsigned long val;
	47
	48	val = csr_read(LOONGARCH_CSR_ECFG);
	49	val \|= ECFGF_TIMER;
	50	csr_write(val, LOONGARCH_CSR_ECFG);
	51	}
	52
	53	static inline void timer_irq_disable(void)
	54	{
	55	unsigned long val;
	56
	57	val = csr_read(LOONGARCH_CSR_ECFG);
	58	val &= ~ECFGF_TIMER;
	59	csr_write(val, LOONGARCH_CSR_ECFG);
	60	}
	61
	62	static inline void timer_set_next_cmp_ms(unsigned int msec, bool period)
	63	{
	64	unsigned long val;
	65
	66	val = msec_to_cycles(msec) & CSR_TCFG_VAL;
	67	val \|= CSR_TCFG_EN;
	68	if (period)
	69	val \|= CSR_TCFG_PERIOD;
	70	csr_write(val, LOONGARCH_CSR_TCFG);
	71	}
	72
	73	static inline void __delay(uint64_t cycles)
	74	{
	75	uint64_t start = timer_get_cycles();
	76
	77	while ((timer_get_cycles() - start) < cycles)
	78	cpu_relax();
	79	}
	80
	81	static inline void udelay(unsigned long usec)
	82	{
	83	__delay(usec_to_cycles(usec));
	84	}
	85	#endif /* SELFTEST_KVM_ARCH_TIMER_H */

tools/testing/selftests/kvm/include/loongarch/processor.h+80−1

@@ -83,7 +83,14 @@ · 82 unmodified lines
83	#define LOONGARCH_CSR_PRMD 0x1	83	#define LOONGARCH_CSR_PRMD 0x1
84	#define LOONGARCH_CSR_EUEN 0x2	84	#define LOONGARCH_CSR_EUEN 0x2
85	#define LOONGARCH_CSR_ECFG 0x4	85	#define LOONGARCH_CSR_ECFG 0x4
		86	#define ECFGB_TIMER 11
		87	#define ECFGF_TIMER (BIT_ULL(ECFGB_TIMER))
86	#define LOONGARCH_CSR_ESTAT 0x5 /* Exception status */	88	#define LOONGARCH_CSR_ESTAT 0x5 /* Exception status */
		89	#define CSR_ESTAT_EXC_SHIFT 16
		90	#define CSR_ESTAT_EXC_WIDTH 6
		91	#define CSR_ESTAT_EXC (0x3f << CSR_ESTAT_EXC_SHIFT)
		92	#define EXCCODE_INT 0 /* Interrupt */
		93	#define INT_TI 11 /* Timer interrupt*/
87	#define LOONGARCH_CSR_ERA 0x6 /* ERA */	94	#define LOONGARCH_CSR_ERA 0x6 /* ERA */
88	#define LOONGARCH_CSR_BADV 0x7 /* Bad virtual address */	95	#define LOONGARCH_CSR_BADV 0x7 /* Bad virtual address */
89	#define LOONGARCH_CSR_EENTRY 0xc	96	#define LOONGARCH_CSR_EENTRY 0xc
@@ -106,6 +113,14 @@ · 16 unmodified lines
106	#define LOONGARCH_CSR_KS1 0x31	113	#define LOONGARCH_CSR_KS1 0x31
107	#define LOONGARCH_CSR_TMID 0x40	114	#define LOONGARCH_CSR_TMID 0x40
108	#define LOONGARCH_CSR_TCFG 0x41	115	#define LOONGARCH_CSR_TCFG 0x41
		116	#define CSR_TCFG_VAL (BIT_ULL(48) - BIT_ULL(2))
		117	#define CSR_TCFG_PERIOD_SHIFT 1
		118	#define CSR_TCFG_PERIOD (0x1UL << CSR_TCFG_PERIOD_SHIFT)
		119	#define CSR_TCFG_EN (0x1UL)
		120	#define LOONGARCH_CSR_TVAL 0x42
		121	#define LOONGARCH_CSR_TINTCLR 0x44 /* Timer interrupt clear */
		122	#define CSR_TINTCLR_TI_SHIFT 0
		123	#define CSR_TINTCLR_TI (1 << CSR_TINTCLR_TI_SHIFT)
109	/* TLB refill exception entry */	124	/* TLB refill exception entry */
110	#define LOONGARCH_CSR_TLBRENTRY 0x88	125	#define LOONGARCH_CSR_TLBRENTRY 0x88
111	#define LOONGARCH_CSR_TLBRSAVE 0x8b	126	#define LOONGARCH_CSR_TLBRSAVE 0x8b
@@ -113,6 +128,28 @@ · 1 unmodified line
113	#define CSR_TLBREHI_PS_SHIFT 0	128	#define CSR_TLBREHI_PS_SHIFT 0
114	#define CSR_TLBREHI_PS (0x3fUL << CSR_TLBREHI_PS_SHIFT)	129	#define CSR_TLBREHI_PS (0x3fUL << CSR_TLBREHI_PS_SHIFT)
115		130
		131	#define csr_read(csr) \
		132	({ \
		133	register unsigned long __v; \
		134	__asm__ __volatile__( \
		135	"csrrd %[val], %[reg]\n\t" \
		136	: [val] "=r" (__v) \
		137	: [reg] "i" (csr) \
		138	: "memory"); \
		139	__v; \
		140	})
		141
		142	#define csr_write(v, csr) \
		143	({ \
		144	register unsigned long __v = v; \
		145	__asm__ __volatile__ ( \
		146	"csrwr %[val], %[reg]\n\t" \
		147	: [val] "+r" (__v) \
		148	: [reg] "i" (csr) \
		149	: "memory"); \
		150	__v; \
		151	})
		152
116	#define EXREGS_GPRS (32)	153	#define EXREGS_GPRS (32)
117		154
118	#ifndef __ASSEMBLER__	155	#ifndef __ASSEMBLER__
@@ -124,18 +161,60 @@ struct ex_regs { · 5 unmodified lines
124	unsigned long pc;	161	unsigned long pc;
125	unsigned long estat;	162	unsigned long estat;
126	unsigned long badv;	163	unsigned long badv;
		164	unsigned long prmd;
127	};	165	};
128		166
129	#define PC_OFFSET_EXREGS offsetof(struct ex_regs, pc)	167	#define PC_OFFSET_EXREGS offsetof(struct ex_regs, pc)
130	#define ESTAT_OFFSET_EXREGS offsetof(struct ex_regs, estat)	168	#define ESTAT_OFFSET_EXREGS offsetof(struct ex_regs, estat)
131	#define BADV_OFFSET_EXREGS offsetof(struct ex_regs, badv)	169	#define BADV_OFFSET_EXREGS offsetof(struct ex_regs, badv)
		170	#define PRMD_OFFSET_EXREGS offsetof(struct ex_regs, prmd)
132	#define EXREGS_SIZE sizeof(struct ex_regs)	171	#define EXREGS_SIZE sizeof(struct ex_regs)
133		172
		173	#define VECTOR_NUM 64
		174
		175	typedef void(handler_fn)(struct ex_regs );
		176
		177	struct handlers {
		178	handler_fn exception_handlers[VECTOR_NUM];
		179	};
		180
		181	void vm_init_descriptor_tables(struct kvm_vm *vm);
		182	void vm_install_exception_handler(struct kvm_vm *vm, int vector, handler_fn handler);
		183
		184	static inline void cpu_relax(void)
		185	{
		186	asm volatile("nop" ::: "memory");
		187	}
		188
		189	static inline void local_irq_enable(void)
		190	{
		191	unsigned int flags = CSR_CRMD_IE;
		192	register unsigned int mask asm("$t0") = CSR_CRMD_IE;
		193
		194	__asm__ __volatile__(
		195	"csrxchg %[val], %[mask], %[reg]\n\t"
		196	: [val] "+r" (flags)
		197	: [mask] "r" (mask), [reg] "i" (LOONGARCH_CSR_CRMD)
		198	: "memory");
		199	}
		200
		201	static inline void local_irq_disable(void)
		202	{
		203	unsigned int flags = 0;
		204	register unsigned int mask asm("$t0") = CSR_CRMD_IE;
		205
		206	__asm__ __volatile__(
		207	"csrxchg %[val], %[mask], %[reg]\n\t"
		208	: [val] "+r" (flags)
		209	: [mask] "r" (mask), [reg] "i" (LOONGARCH_CSR_CRMD)
		210	: "memory");
		211	}
134	#else	212	#else
135	#define PC_OFFSET_EXREGS ((EXREGS_GPRS + 0) * 8)	213	#define PC_OFFSET_EXREGS ((EXREGS_GPRS + 0) * 8)
136	#define ESTAT_OFFSET_EXREGS ((EXREGS_GPRS + 1) * 8)	214	#define ESTAT_OFFSET_EXREGS ((EXREGS_GPRS + 1) * 8)
137	#define BADV_OFFSET_EXREGS ((EXREGS_GPRS + 2) * 8)	215	#define BADV_OFFSET_EXREGS ((EXREGS_GPRS + 2) * 8)
138	#define EXREGS_SIZE ((EXREGS_GPRS + 3) * 8)	216	#define PRMD_OFFSET_EXREGS ((EXREGS_GPRS + 3) * 8)
		217	#define EXREGS_SIZE ((EXREGS_GPRS + 4) * 8)
139	#endif	218	#endif
140		219
141	#endif /* SELFTEST_KVM_PROCESSOR_H */	220	#endif /* SELFTEST_KVM_PROCESSOR_H */

tools/testing/selftests/kvm/include/numaif.h+69−41

@@ -1,55 +1,83 @@
1	/* SPDX-License-Identifier: GPL-2.0-only */	1	/* SPDX-License-Identifier: GPL-2.0-only */
2	/*	2	/* Copyright (C) 2020, Google LLC. */
3	* tools/testing/selftests/kvm/include/numaif.h
4	*
5	* Copyright (C) 2020, Google LLC.
6	*
7	* This work is licensed under the terms of the GNU GPL, version 2.
8	*
9	* Header file that provides access to NUMA API functions not explicitly
10	* exported to user space.
11	*/
12		3
13	#ifndef SELFTEST_KVM_NUMAIF_H	4	#ifndef SELFTEST_KVM_NUMAIF_H
14	#define SELFTEST_KVM_NUMAIF_H	5	#define SELFTEST_KVM_NUMAIF_H
15		6
16	#define __NR_get_mempolicy 239	7	#include <dirent.h>
17	#define __NR_migrate_pages 256
18		8
19	/* System calls */	9	#include <linux/mempolicy.h>
20	long get_mempolicy(int policy, const unsigned long nmask,	10
21	unsigned long maxnode, void *addr, int flags)	11	#include "kvm_syscalls.h"
		12
		13	KVM_SYSCALL_DEFINE(get_mempolicy, 5, int , policy, const unsigned long , nmask,
		14	unsigned long, maxnode, void *, addr, int, flags);
		15
		16	KVM_SYSCALL_DEFINE(set_mempolicy, 3, int, mode, const unsigned long *, nmask,
		17	unsigned long, maxnode);
		18
		19	KVM_SYSCALL_DEFINE(set_mempolicy_home_node, 4, unsigned long, start,
		20	unsigned long, len, unsigned long, home_node,
		21	unsigned long, flags);
		22
		23	KVM_SYSCALL_DEFINE(migrate_pages, 4, int, pid, unsigned long, maxnode,
		24	const unsigned long , frommask, const unsigned long , tomask);
		25
		26	KVM_SYSCALL_DEFINE(move_pages, 6, int, pid, unsigned long, count, void *, pages,
		27	const int , nodes, int , status, int, flags);
		28
		29	KVM_SYSCALL_DEFINE(mbind, 6, void *, addr, unsigned long, size, int, mode,
		30	const unsigned long *, nodemask, unsigned long, maxnode,
		31	unsigned int, flags);
		32
		33	static inline int get_max_numa_node(void)
22	{	34	{
23	return syscall(__NR_get_mempolicy, policy, nmask,	35	struct dirent *de;
24	maxnode, addr, flags);	36	int max_node = 0;
		37	DIR *d;
		38
		39	/*
		40	* Assume there's a single node if the kernel doesn't support NUMA,
		41	* or if no nodes are found.
		42	*/
		43	d = opendir("/sys/devices/system/node");
		44	if (!d)
		45	return 0;
		46
		47	while ((de = readdir(d)) != NULL) {
		48	int node_id;
		49	char *endptr;
		50
		51	if (strncmp(de->d_name, "node", 4) != 0)
		52	continue;
		53
		54	node_id = strtol(de->d_name + 4, &endptr, 10);
		55	if (*endptr != '\0')
		56	continue;
		57
		58	if (node_id > max_node)
		59	max_node = node_id;
		60	}
		61	closedir(d);
		62
		63	return max_node;
25	}	64	}
26		65
27	long migrate_pages(int pid, unsigned long maxnode,	66	static bool is_numa_available(void)
28	const unsigned long *frommask,
29	const unsigned long *tomask)
30	{	67	{
31	return syscall(__NR_migrate_pages, pid, maxnode, frommask, tomask);	68	/*
		69	* Probe for NUMA by doing a dummy get_mempolicy(). If the syscall
		70	* fails with ENOSYS, then the kernel was built without NUMA support.
		71	* if the syscall fails with EPERM, then the process/user lacks the
		72	* necessary capabilities (CAP_SYS_NICE).
		73	*/
		74	return !get_mempolicy(NULL, NULL, 0, NULL, 0) \|\|
		75	(errno != ENOSYS && errno != EPERM);
32	}	76	}
33		77
34	/* Policies */	78	static inline bool is_multi_numa_node_system(void)
35	#define MPOL_DEFAULT 0	79	{
36	#define MPOL_PREFERRED 1	80	return is_numa_available() && get_max_numa_node() >= 1;
37	#define MPOL_BIND 2	81	}
38	#define MPOL_INTERLEAVE 3
39
40	#define MPOL_MAX MPOL_INTERLEAVE
41
42	/* Flags for get_mem_policy */
43	#define MPOL_F_NODE (1<<0) /* return next il node or node of address */
44	/* Warning: MPOL_F_NODE is unsupported and
45	* subject to change. Don't use.
46	*/
47	#define MPOL_F_ADDR (1<<1) /* look up vma using address */
48	#define MPOL_F_MEMS_ALLOWED (1<<2) /* query nodes allowed in cpuset */
49
50	/* Flags for mbind */
51	#define MPOL_MF_STRICT (1<<0) /* Verify existing pages in the mapping */
52	#define MPOL_MF_MOVE (1<<1) /* Move pages owned by this process to conform to mapping */
53	#define MPOL_MF_MOVE_ALL (1<<2) /* Move every page to conform to mapping */
54		82
55	#endif /* SELFTEST_KVM_NUMAIF_H */	83	#endif /* SELFTEST_KVM_NUMAIF_H */

tools/testing/selftests/kvm/include/x86/processor.h+1−1

@@ -1441,7 +1441,7 @@ enum pg_level { · 1440 unmodified lines
1441	PG_LEVEL_2M,	1441	PG_LEVEL_2M,
1442	PG_LEVEL_1G,	1442	PG_LEVEL_1G,
1443	PG_LEVEL_512G,	1443	PG_LEVEL_512G,
1444	PG_LEVEL_NUM	1444	PG_LEVEL_256T
1445	};	1445	};
1446		1446
1447	#define PG_LEVEL_SHIFT(_level) ((_level - 1) * 9 + 12)	1447	#define PG_LEVEL_SHIFT(_level) ((_level - 1) * 9 + 12)

tools/testing/selftests/kvm/include/x86/vmx.h+1−2

@@ -568,8 +568,7 @@ void nested_map_memslot(struct vmx_pages vmx, struct kvm_vm vm, · 567 unmodified lines
568	void nested_identity_map_1g(struct vmx_pages vmx, struct kvm_vm vm,	568	void nested_identity_map_1g(struct vmx_pages vmx, struct kvm_vm vm,
569	uint64_t addr, uint64_t size);	569	uint64_t addr, uint64_t size);
570	bool kvm_cpu_has_ept(void);	570	bool kvm_cpu_has_ept(void);
571	void prepare_eptp(struct vmx_pages vmx, struct kvm_vm vm,	571	void prepare_eptp(struct vmx_pages vmx, struct kvm_vm vm);
572	uint32_t eptp_memslot);
573	void prepare_virtualize_apic_accesses(struct vmx_pages vmx, struct kvm_vm vm);	572	void prepare_virtualize_apic_accesses(struct vmx_pages vmx, struct kvm_vm vm);
574		573
575	#endif /* SELFTEST_KVM_VMX_H */	574	#endif /* SELFTEST_KVM_VMX_H */

tools/testing/selftests/kvm/kvm_binary_stats_test.c+2−2

@@ -239,14 +239,14 @@ int main(int argc, char *argv[]) · 238 unmodified lines
239	* single stats file works and doesn't cause explosions.	239	* single stats file works and doesn't cause explosions.
240	*/	240	*/
241	vm_stats_fds = vm_get_stats_fd(vms[i]);	241	vm_stats_fds = vm_get_stats_fd(vms[i]);
242	stats_test(dup(vm_stats_fds));	242	stats_test(kvm_dup(vm_stats_fds));
243		243
244	/* Verify userspace can instantiate multiple stats files. */	244	/* Verify userspace can instantiate multiple stats files. */
245	stats_test(vm_get_stats_fd(vms[i]));	245	stats_test(vm_get_stats_fd(vms[i]));
246		246
247	for (j = 0; j < max_vcpu; ++j) {	247	for (j = 0; j < max_vcpu; ++j) {
248	vcpu_stats_fds[j] = vcpu_get_stats_fd(vcpus[i * max_vcpu + j]);	248	vcpu_stats_fds[j] = vcpu_get_stats_fd(vcpus[i * max_vcpu + j]);
249	stats_test(dup(vcpu_stats_fds[j]));	249	stats_test(kvm_dup(vcpu_stats_fds[j]));
250	stats_test(vcpu_get_stats_fd(vcpus[i * max_vcpu + j]));	250	stats_test(vcpu_get_stats_fd(vcpus[i * max_vcpu + j]));
251	}	251	}
252		252

tools/testing/selftests/kvm/lib/arm64/gic.c+6−0

@@ -155,3 +155,9 @@ void gic_irq_set_config(unsigned int intid, bool is_edge) · 154 unmodified lines
155	GUEST_ASSERT(gic_common_ops);	155	GUEST_ASSERT(gic_common_ops);
156	gic_common_ops->gic_irq_set_config(intid, is_edge);	156	gic_common_ops->gic_irq_set_config(intid, is_edge);
157	}	157	}
		158
		159	void gic_irq_set_group(unsigned int intid, bool group)
		160	{
		161	GUEST_ASSERT(gic_common_ops);
		162	gic_common_ops->gic_irq_set_group(intid, group);
		163	}

tools/testing/selftests/kvm/lib/arm64/gic_private.h+1−0

@@ -25,6 +25,7 @@ struct gic_common_ops { · 24 unmodified lines
25	void (*gic_irq_clear_pending)(uint32_t intid);	25	void (*gic_irq_clear_pending)(uint32_t intid);
26	bool (*gic_irq_get_pending)(uint32_t intid);	26	bool (*gic_irq_get_pending)(uint32_t intid);
27	void (*gic_irq_set_config)(uint32_t intid, bool is_edge);	27	void (*gic_irq_set_config)(uint32_t intid, bool is_edge);
		28	void (*gic_irq_set_group)(uint32_t intid, bool group);
28	};	29	};
29		30
30	extern const struct gic_common_ops gicv3_ops;	31	extern const struct gic_common_ops gicv3_ops;

tools/testing/selftests/kvm/lib/arm64/gic_v3.c+22−0

 	}
 }
+static void gicv3_set_group(uint32_t intid, bool grp)
+{
+	uint32_t cpu_or_dist;
+	uint32_t val;
+	cpu_or_dist = (get_intid_range(intid) == SPI_RANGE) ? DIST_BIT : guest_get_vcpuid();
+	val = gicv3_reg_readl(cpu_or_dist, GICD_IGROUPR + (intid / 32) * 4);
+	if (grp)
+		val |= BIT(intid % 32);
+	else
+		val &= ~BIT(intid % 32);
+	gicv3_reg_writel(cpu_or_dist, GICD_IGROUPR + (intid / 32) * 4, val);
+}
 static void gicv3_cpu_init(unsigned int cpu)
 {
 	volatile void *sgi_base;
 	unsigned int i;
 	volatile void *redist_base_cpu;
+	u64 typer;
 	GUEST_ASSERT(cpu < gicv3_data.nr_cpus);
 	redist_base_cpu = gicr_base_cpu(cpu);
 	sgi_base = sgi_base_from_redist(redist_base_cpu);
+	/* Verify assumption that GICR_TYPER.Processor_number == cpu */
+	typer = readq_relaxed(redist_base_cpu + GICR_TYPER);
+	GUEST_ASSERT_EQ(GICR_TYPER_CPU_NUMBER(typer), cpu);
 	gicv3_enable_redist(redist_base_cpu);
 	/*
 	/* Set a default priority threshold */
 	write_sysreg_s(ICC_PMR_DEF_PRIO, SYS_ICC_PMR_EL1);
+	/* Disable Group-0 interrupts */
+	write_sysreg_s(ICC_IGRPEN0_EL1_MASK, SYS_ICC_IGRPEN1_EL1);
 	/* Enable non-secure Group-1 interrupts */
 	write_sysreg_s(ICC_IGRPEN1_EL1_MASK, SYS_ICC_IGRPEN1_EL1);
 }
 	.gic_irq_clear_pending = gicv3_irq_clear_pending,
 	.gic_irq_get_pending = gicv3_irq_get_pending,
 	.gic_irq_set_config = gicv3_irq_set_config,
+	.gic_irq_set_group = gicv3_set_group,
 };
 void gic_rdist_enable_lpis(vm_paddr_t cfg_table, size_t cfg_table_size,

tools/testing/selftests/kvm/lib/arm64/gic_v3_its.c+10−0

@@ -253,3 +253,13 @@ void its_send_invall_cmd(void *cmdq_base, u32 collection_id) · 252 unmodified lines
253		253
254	its_send_cmd(cmdq_base, &cmd);	254	its_send_cmd(cmdq_base, &cmd);
255	}	255	}
		256
		257	void its_send_sync_cmd(void *cmdq_base, u32 vcpu_id)
		258	{
		259	struct its_cmd_block cmd = {};
		260
		261	its_encode_cmd(&cmd, GITS_CMD_SYNC);
		262	its_encode_target(&cmd, procnum_to_rdbase(vcpu_id));
		263
		264	its_send_cmd(cmdq_base, &cmd);
		265	}

tools/testing/selftests/kvm/lib/arm64/processor.c+1−1

@@ -324,7 +324,7 @@ void aarch64_vcpu_setup(struct kvm_vcpu vcpu, struct kvm_vcpu_init init) · 323 unmodified lines
324		324
325	/* Configure base granule size */	325	/* Configure base granule size */
326	switch (vm->mode) {	326	switch (vm->mode) {
327	case VM_MODE_PXXV48_4K:	327	case VM_MODE_PXXVYY_4K:
328	TEST_FAIL("AArch64 does not support 4K sized pages "	328	TEST_FAIL("AArch64 does not support 4K sized pages "
329	"with ANY-bit physical address ranges");	329	"with ANY-bit physical address ranges");
330	case VM_MODE_P52V48_64K:	330	case VM_MODE_P52V48_64K:

tools/testing/selftests/kvm/lib/kvm_util.c+80−65

@@ -201,7 +201,7 @@ const char *vm_guest_mode_string(uint32_t i) · 200 unmodified lines
201	[VM_MODE_P40V48_4K] = "PA-bits:40, VA-bits:48, 4K pages",	201	[VM_MODE_P40V48_4K] = "PA-bits:40, VA-bits:48, 4K pages",
202	[VM_MODE_P40V48_16K] = "PA-bits:40, VA-bits:48, 16K pages",	202	[VM_MODE_P40V48_16K] = "PA-bits:40, VA-bits:48, 16K pages",
203	[VM_MODE_P40V48_64K] = "PA-bits:40, VA-bits:48, 64K pages",	203	[VM_MODE_P40V48_64K] = "PA-bits:40, VA-bits:48, 64K pages",
204	[VM_MODE_PXXV48_4K] = "PA-bits:ANY, VA-bits:48, 4K pages",	204	[VM_MODE_PXXVYY_4K] = "PA-bits:ANY, VA-bits:48 or 57, 4K pages",
205	[VM_MODE_P47V64_4K] = "PA-bits:47, VA-bits:64, 4K pages",	205	[VM_MODE_P47V64_4K] = "PA-bits:47, VA-bits:64, 4K pages",
206	[VM_MODE_P44V64_4K] = "PA-bits:44, VA-bits:64, 4K pages",	206	[VM_MODE_P44V64_4K] = "PA-bits:44, VA-bits:64, 4K pages",
207	[VM_MODE_P36V48_4K] = "PA-bits:36, VA-bits:48, 4K pages",	207	[VM_MODE_P36V48_4K] = "PA-bits:36, VA-bits:48, 4K pages",
@@ -228,7 +228,7 @@ const struct vm_guest_mode_params vm_guest_mode_params[] = { · 20 unmodified lines
228	[VM_MODE_P40V48_4K] = { 40, 48, 0x1000, 12 },	228	[VM_MODE_P40V48_4K] = { 40, 48, 0x1000, 12 },
229	[VM_MODE_P40V48_16K] = { 40, 48, 0x4000, 14 },	229	[VM_MODE_P40V48_16K] = { 40, 48, 0x4000, 14 },
230	[VM_MODE_P40V48_64K] = { 40, 48, 0x10000, 16 },	230	[VM_MODE_P40V48_64K] = { 40, 48, 0x10000, 16 },
231	[VM_MODE_PXXV48_4K] = { 0, 0, 0x1000, 12 },	231	[VM_MODE_PXXVYY_4K] = { 0, 0, 0x1000, 12 },
232	[VM_MODE_P47V64_4K] = { 47, 64, 0x1000, 12 },	232	[VM_MODE_P47V64_4K] = { 47, 64, 0x1000, 12 },
233	[VM_MODE_P44V64_4K] = { 44, 64, 0x1000, 12 },	233	[VM_MODE_P44V64_4K] = { 44, 64, 0x1000, 12 },
234	[VM_MODE_P36V48_4K] = { 36, 48, 0x1000, 12 },	234	[VM_MODE_P36V48_4K] = { 36, 48, 0x1000, 12 },
@@ -310,24 +310,26 @@ struct kvm_vm *____vm_create(struct vm_shape shape) · 75 unmodified lines
310	case VM_MODE_P36V47_16K:	310	case VM_MODE_P36V47_16K:
311	vm->pgtable_levels = 3;	311	vm->pgtable_levels = 3;
312	break;	312	break;
313	case VM_MODE_PXXV48_4K:	313	case VM_MODE_PXXVYY_4K:
314	#ifdef __x86_64__	314	#ifdef __x86_64__
315	kvm_get_cpu_address_width(&vm->pa_bits, &vm->va_bits);	315	kvm_get_cpu_address_width(&vm->pa_bits, &vm->va_bits);
316	kvm_init_vm_address_properties(vm);	316	kvm_init_vm_address_properties(vm);
317	/*	317
318	* Ignore KVM support for 5-level paging (vm->va_bits == 57),
319	* it doesn't take effect unless a CR4.LA57 is set, which it
320	* isn't for this mode (48-bit virtual address space).
321	*/
322	TEST_ASSERT(vm->va_bits == 48 \|\| vm->va_bits == 57,
323	"Linear address width (%d bits) not supported",
324	vm->va_bits);
325	pr_debug("Guest physical address width detected: %d\n",	318	pr_debug("Guest physical address width detected: %d\n",
326	vm->pa_bits);	319	vm->pa_bits);
327	vm->pgtable_levels = 4;	320	pr_debug("Guest virtual address width detected: %d\n",
328	vm->va_bits = 48;	321	vm->va_bits);
		322
		323	if (vm->va_bits == 57) {
		324	vm->pgtable_levels = 5;
		325	} else {
		326	TEST_ASSERT(vm->va_bits == 48,
		327	"Unexpected guest virtual address width: %d",
		328	vm->va_bits);
		329	vm->pgtable_levels = 4;
		330	}
329	#else	331	#else
330	TEST_FAIL("VM_MODE_PXXV48_4K not supported on non-x86 platforms");	332	TEST_FAIL("VM_MODE_PXXVYY_4K not supported on non-x86 platforms");
331	#endif	333	#endif
332	break;	334	break;
333	case VM_MODE_P47V64_4K:	335	case VM_MODE_P47V64_4K:
@@ -704,8 +706,6 @@ userspace_mem_region_find(struct kvm_vm *vm, uint64_t start, uint64_t end) · 370 unmodified lines
704		706
705	static void kvm_stats_release(struct kvm_binary_stats *stats)	707	static void kvm_stats_release(struct kvm_binary_stats *stats)
706	{	708	{
707	int ret;
708
709	if (stats->fd < 0)	709	if (stats->fd < 0)
710	return;	710	return;
711		711
@@ -714,8 +714,7 @@ static void kvm_stats_release(struct kvm_binary_stats *stats) · 2 unmodified lines
714	stats->desc = NULL;	714	stats->desc = NULL;
715	}	715	}
716		716
717	ret = close(stats->fd);	717	kvm_close(stats->fd);
718	TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("close()", ret));
719	stats->fd = -1;	718	stats->fd = -1;
720	}	719	}
721		720
@@ -738,8 +737,6 @@ __weak void vcpu_arch_free(struct kvm_vcpu *vcpu) · 16 unmodified lines
738	*/	737	*/
739	static void vm_vcpu_rm(struct kvm_vm vm, struct kvm_vcpu vcpu)	738	static void vm_vcpu_rm(struct kvm_vm vm, struct kvm_vcpu vcpu)
740	{	739	{
741	int ret;
742
743	if (vcpu->dirty_gfns) {	740	if (vcpu->dirty_gfns) {
744	kvm_munmap(vcpu->dirty_gfns, vm->dirty_ring_size);	741	kvm_munmap(vcpu->dirty_gfns, vm->dirty_ring_size);
745	vcpu->dirty_gfns = NULL;	742	vcpu->dirty_gfns = NULL;
@@ -747,9 +744,7 @@ static void vm_vcpu_rm(struct kvm_vm vm, struct kvm_vcpu vcpu) · 1 unmodified line
747		744
748	kvm_munmap(vcpu->run, vcpu_mmap_sz());	745	kvm_munmap(vcpu->run, vcpu_mmap_sz());
749		746
750	ret = close(vcpu->fd);	747	kvm_close(vcpu->fd);
751	TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("close()", ret));
752
753	kvm_stats_release(&vcpu->stats);	748	kvm_stats_release(&vcpu->stats);
754		749
755	list_del(&vcpu->list);	750	list_del(&vcpu->list);
@@ -761,16 +756,12 @@ static void vm_vcpu_rm(struct kvm_vm vm, struct kvm_vcpu vcpu) · 5 unmodified lines
761	void kvm_vm_release(struct kvm_vm *vmp)	756	void kvm_vm_release(struct kvm_vm *vmp)
762	{	757	{
763	struct kvm_vcpu vcpu, tmp;	758	struct kvm_vcpu vcpu, tmp;
764	int ret;
765		759
766	list_for_each_entry_safe(vcpu, tmp, &vmp->vcpus, list)	760	list_for_each_entry_safe(vcpu, tmp, &vmp->vcpus, list)
767	vm_vcpu_rm(vmp, vcpu);	761	vm_vcpu_rm(vmp, vcpu);
768		762
769	ret = close(vmp->fd);	763	kvm_close(vmp->fd);
770	TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("close()", ret));	764	kvm_close(vmp->kvm_fd);
771
772	ret = close(vmp->kvm_fd);
773	TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("close()", ret));
774		765
775	/* Free cached stats metadata and close FD */	766	/* Free cached stats metadata and close FD */
776	kvm_stats_release(&vmp->stats);	767	kvm_stats_release(&vmp->stats);
@@ -828,7 +819,7 @@ void kvm_vm_free(struct kvm_vm *vmp) · 51 unmodified lines
828	int kvm_memfd_alloc(size_t size, bool hugepages)	819	int kvm_memfd_alloc(size_t size, bool hugepages)
829	{	820	{
830	int memfd_flags = MFD_CLOEXEC;	821	int memfd_flags = MFD_CLOEXEC;
831	int fd, r;	822	int fd;
832		823
833	if (hugepages)	824	if (hugepages)
834	memfd_flags \|= MFD_HUGETLB;	825	memfd_flags \|= MFD_HUGETLB;
@@ -836,11 +827,8 @@ int kvm_memfd_alloc(size_t size, bool hugepages) · 1 unmodified line
836	fd = memfd_create("kvm_selftest", memfd_flags);	827	fd = memfd_create("kvm_selftest", memfd_flags);
837	TEST_ASSERT(fd != -1, __KVM_SYSCALL_ERROR("memfd_create()", fd));	828	TEST_ASSERT(fd != -1, __KVM_SYSCALL_ERROR("memfd_create()", fd));
838		829
839	r = ftruncate(fd, size);	830	kvm_ftruncate(fd, size);
840	TEST_ASSERT(!r, __KVM_SYSCALL_ERROR("ftruncate()", r));	831	kvm_fallocate(fd, FALLOC_FL_PUNCH_HOLE \| FALLOC_FL_KEEP_SIZE, 0, size);
841
842	r = fallocate(fd, FALLOC_FL_PUNCH_HOLE \| FALLOC_FL_KEEP_SIZE, 0, size);
843	TEST_ASSERT(!r, __KVM_SYSCALL_ERROR("fallocate()", r));
844		832
845	return fd;	833	return fd;
846	}	834	}
@@ -957,8 +945,8 @@ void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags · 110 unmodified lines
957		945
958	/* FIXME: This thing needs to be ripped apart and rewritten. */	946	/* FIXME: This thing needs to be ripped apart and rewritten. */
959	void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,	947	void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
960	uint64_t guest_paddr, uint32_t slot, uint64_t npages,	948	uint64_t gpa, uint32_t slot, uint64_t npages, uint32_t flags,
961	uint32_t flags, int guest_memfd, uint64_t guest_memfd_offset)	949	int guest_memfd, uint64_t guest_memfd_offset)
962	{	950	{
963	int ret;	951	int ret;
964	struct userspace_mem_region *region;	952	struct userspace_mem_region *region;
@@ -972,30 +960,29 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, · 7 unmodified lines
972	"Number of guest pages is not compatible with the host. "	960	"Number of guest pages is not compatible with the host. "
973	"Try npages=%d", vm_adjust_num_guest_pages(vm->mode, npages));	961	"Try npages=%d", vm_adjust_num_guest_pages(vm->mode, npages));
974		962
975	TEST_ASSERT((guest_paddr % vm->page_size) == 0, "Guest physical "	963	TEST_ASSERT((gpa % vm->page_size) == 0, "Guest physical "
976	"address not on a page boundary.\n"	964	"address not on a page boundary.\n"
977	" guest_paddr: 0x%lx vm->page_size: 0x%x",	965	" gpa: 0x%lx vm->page_size: 0x%x",
978	guest_paddr, vm->page_size);	966	gpa, vm->page_size);
979	TEST_ASSERT((((guest_paddr >> vm->page_shift) + npages) - 1)	967	TEST_ASSERT((((gpa >> vm->page_shift) + npages) - 1)
980	<= vm->max_gfn, "Physical range beyond maximum "	968	<= vm->max_gfn, "Physical range beyond maximum "
981	"supported physical address,\n"	969	"supported physical address,\n"
982	" guest_paddr: 0x%lx npages: 0x%lx\n"	970	" gpa: 0x%lx npages: 0x%lx\n"
983	" vm->max_gfn: 0x%lx vm->page_size: 0x%x",	971	" vm->max_gfn: 0x%lx vm->page_size: 0x%x",
984	guest_paddr, npages, vm->max_gfn, vm->page_size);	972	gpa, npages, vm->max_gfn, vm->page_size);
985		973
986	/*	974	/*
987	* Confirm a mem region with an overlapping address doesn't	975	* Confirm a mem region with an overlapping address doesn't
988	* already exist.	976	* already exist.
989	*/	977	*/
990	region = (struct userspace_mem_region *) userspace_mem_region_find(	978	region = (struct userspace_mem_region *) userspace_mem_region_find(
991	vm, guest_paddr, (guest_paddr + npages * vm->page_size) - 1);	979	vm, gpa, (gpa + npages * vm->page_size) - 1);
992	if (region != NULL)	980	if (region != NULL)
993	TEST_FAIL("overlapping userspace_mem_region already "	981	TEST_FAIL("overlapping userspace_mem_region already "
994	"exists\n"	982	"exists\n"
995	" requested guest_paddr: 0x%lx npages: 0x%lx "	983	" requested gpa: 0x%lx npages: 0x%lx page_size: 0x%x\n"
996	"page_size: 0x%x\n"	984	" existing gpa: 0x%lx size: 0x%lx",
997	" existing guest_paddr: 0x%lx size: 0x%lx",	985	gpa, npages, vm->page_size,
998	guest_paddr, npages, vm->page_size,
999	(uint64_t) region->region.guest_phys_addr,	986	(uint64_t) region->region.guest_phys_addr,
1000	(uint64_t) region->region.memory_size);	987	(uint64_t) region->region.memory_size);
1001		988
@@ -1009,8 +996,7 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, · 7 unmodified lines
1009	"already exists.\n"	996	"already exists.\n"
1010	" requested slot: %u paddr: 0x%lx npages: 0x%lx\n"	997	" requested slot: %u paddr: 0x%lx npages: 0x%lx\n"
1011	" existing slot: %u paddr: 0x%lx size: 0x%lx",	998	" existing slot: %u paddr: 0x%lx size: 0x%lx",
1012	slot, guest_paddr, npages,	999	slot, gpa, npages, region->region.slot,
1013	region->region.slot,
1014	(uint64_t) region->region.guest_phys_addr,	1000	(uint64_t) region->region.guest_phys_addr,
1015	(uint64_t) region->region.memory_size);	1001	(uint64_t) region->region.memory_size);
1016	}	1002	}
@@ -1036,7 +1022,7 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, · 19 unmodified lines
1036	if (src_type == VM_MEM_SRC_ANONYMOUS_THP)	1022	if (src_type == VM_MEM_SRC_ANONYMOUS_THP)
1037	alignment = max(backing_src_pagesz, alignment);	1023	alignment = max(backing_src_pagesz, alignment);
1038		1024
1039	TEST_ASSERT_EQ(guest_paddr, align_up(guest_paddr, backing_src_pagesz));	1025	TEST_ASSERT_EQ(gpa, align_up(gpa, backing_src_pagesz));
1040		1026
1041	/* Add enough memory to align up if necessary */	1027	/* Add enough memory to align up if necessary */
1042	if (alignment > 1)	1028	if (alignment > 1)
@@ -1084,8 +1070,7 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, · 41 unmodified lines
1084	* needing to track if the fd is owned by the framework	1070	* needing to track if the fd is owned by the framework
1085	* or by the caller.	1071	* or by the caller.
1086	*/	1072	*/
1087	guest_memfd = dup(guest_memfd);	1073	guest_memfd = kvm_dup(guest_memfd);
1088	TEST_ASSERT(guest_memfd >= 0, __KVM_SYSCALL_ERROR("dup()", guest_memfd));
1089	}	1074	}
1090		1075
1091	region->region.guest_memfd = guest_memfd;	1076	region->region.guest_memfd = guest_memfd;
@@ -1097,20 +1082,18 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, · 5 unmodified lines
1097	region->unused_phy_pages = sparsebit_alloc();	1082	region->unused_phy_pages = sparsebit_alloc();
1098	if (vm_arch_has_protected_memory(vm))	1083	if (vm_arch_has_protected_memory(vm))
1099	region->protected_phy_pages = sparsebit_alloc();	1084	region->protected_phy_pages = sparsebit_alloc();
1100	sparsebit_set_num(region->unused_phy_pages,	1085	sparsebit_set_num(region->unused_phy_pages, gpa >> vm->page_shift, npages);
1101	guest_paddr >> vm->page_shift, npages);
1102	region->region.slot = slot;	1086	region->region.slot = slot;
1103	region->region.flags = flags;	1087	region->region.flags = flags;
1104	region->region.guest_phys_addr = guest_paddr;	1088	region->region.guest_phys_addr = gpa;
1105	region->region.memory_size = npages * vm->page_size;	1089	region->region.memory_size = npages * vm->page_size;
1106	region->region.userspace_addr = (uintptr_t) region->host_mem;	1090	region->region.userspace_addr = (uintptr_t) region->host_mem;
1107	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &region->region);	1091	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &region->region);
1108	TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n"	1092	TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n"
1109	" rc: %i errno: %i\n"	1093	" rc: %i errno: %i\n"
1110	" slot: %u flags: 0x%x\n"	1094	" slot: %u flags: 0x%x\n"
1111	" guest_phys_addr: 0x%lx size: 0x%lx guest_memfd: %d",	1095	" guest_phys_addr: 0x%lx size: 0x%llx guest_memfd: %d",
1112	ret, errno, slot, flags,	1096	ret, errno, slot, flags, gpa, region->region.memory_size,
1113	guest_paddr, (uint64_t) region->region.memory_size,
1114	region->region.guest_memfd);	1097	region->region.guest_memfd);
1115		1098
1116	/* Add to quick lookup data structures */	1099	/* Add to quick lookup data structures */
@@ -1132,10 +1115,10 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, · 15 unmodified lines
1132		1115
1133	void vm_userspace_mem_region_add(struct kvm_vm *vm,	1116	void vm_userspace_mem_region_add(struct kvm_vm *vm,
1134	enum vm_mem_backing_src_type src_type,	1117	enum vm_mem_backing_src_type src_type,
1135	uint64_t guest_paddr, uint32_t slot,	1118	uint64_t gpa, uint32_t slot, uint64_t npages,
1136	uint64_t npages, uint32_t flags)	1119	uint32_t flags)
1137	{	1120	{
1138	vm_mem_add(vm, src_type, guest_paddr, slot, npages, flags, -1, 0);	1121	vm_mem_add(vm, src_type, gpa, slot, npages, flags, -1, 0);
1139	}	1122	}
1140		1123
1141	/*	1124	/*
@@ -1201,6 +1184,16 @@ void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags) · 59 unmodified lines
1201	ret, errno, slot, flags);	1184	ret, errno, slot, flags);
1202	}	1185	}
1203		1186
		1187	void vm_mem_region_reload(struct kvm_vm *vm, uint32_t slot)
		1188	{
		1189	struct userspace_mem_region *region = memslot2region(vm, slot);
		1190	struct kvm_userspace_memory_region2 tmp = region->region;
		1191
		1192	tmp.memory_size = 0;
		1193	vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &tmp);
		1194	vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &region->region);
		1195	}
		1196
1204	/*	1197	/*
1205	* VM Memory Region Move	1198	* VM Memory Region Move
1206	*	1199	*
@@ -1456,8 +1449,6 @@ static vm_vaddr_t ____vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, · 249 unmodified lines
1456	pages--, vaddr += vm->page_size, paddr += vm->page_size) {	1449	pages--, vaddr += vm->page_size, paddr += vm->page_size) {
1457		1450
1458	virt_pg_map(vm, vaddr, paddr);	1451	virt_pg_map(vm, vaddr, paddr);
1459
1460	sparsebit_set(vm->vpages_mapped, vaddr >> vm->page_shift);
1461	}	1452	}
1462		1453
1463	return vaddr_start;	1454	return vaddr_start;
@@ -1571,7 +1562,6 @@ void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, · 107 unmodified lines
1571		1562
1572	while (npages--) {	1563	while (npages--) {
1573	virt_pg_map(vm, vaddr, paddr);	1564	virt_pg_map(vm, vaddr, paddr);
1574	sparsebit_set(vm->vpages_mapped, vaddr >> vm->page_shift);
1575		1565
1576	vaddr += page_size;	1566	vaddr += page_size;
1577	paddr += page_size;	1567	paddr += page_size;
@@ -2025,6 +2015,7 @@ static struct exit_reason { · 447 unmodified lines
2025	KVM_EXIT_STRING(NOTIFY),	2015	KVM_EXIT_STRING(NOTIFY),
2026	KVM_EXIT_STRING(LOONGARCH_IOCSR),	2016	KVM_EXIT_STRING(LOONGARCH_IOCSR),
2027	KVM_EXIT_STRING(MEMORY_FAULT),	2017	KVM_EXIT_STRING(MEMORY_FAULT),
		2018	KVM_EXIT_STRING(ARM_SEA),
2028	};	2019	};
2029		2020
2030	/*	2021	/*
@@ -2305,11 +2296,35 @@ __weak void kvm_selftest_arch_init(void) · 274 unmodified lines
2305	{	2296	{
2306	}	2297	}
2307		2298
		2299	static void report_unexpected_signal(int signum)
		2300	{
		2301	#define KVM_CASE_SIGNUM(sig) \
		2302	case sig: TEST_FAIL("Unexpected " #sig " (%d)\n", signum)
		2303
		2304	switch (signum) {
		2305	KVM_CASE_SIGNUM(SIGBUS);
		2306	KVM_CASE_SIGNUM(SIGSEGV);
		2307	KVM_CASE_SIGNUM(SIGILL);
		2308	KVM_CASE_SIGNUM(SIGFPE);
		2309	default:
		2310	TEST_FAIL("Unexpected signal %d\n", signum);
		2311	}
		2312	}
		2313
2308	void __attribute((constructor)) kvm_selftest_init(void)	2314	void __attribute((constructor)) kvm_selftest_init(void)
2309	{	2315	{
		2316	struct sigaction sig_sa = {
		2317	.sa_handler = report_unexpected_signal,
		2318	};
		2319
2310	/* Tell stdout not to buffer its content. */	2320	/* Tell stdout not to buffer its content. */
2311	setbuf(stdout, NULL);	2321	setbuf(stdout, NULL);
2312		2322
		2323	sigaction(SIGBUS, &sig_sa, NULL);
		2324	sigaction(SIGSEGV, &sig_sa, NULL);
		2325	sigaction(SIGILL, &sig_sa, NULL);
		2326	sigaction(SIGFPE, &sig_sa, NULL);
		2327
2313	guest_random_seed = last_guest_seed = random();	2328	guest_random_seed = last_guest_seed = random();
2314	pr_info("Random seed: 0x%x\n", guest_random_seed);	2329	pr_info("Random seed: 0x%x\n", guest_random_seed);
2315		2330

tools/testing/selftests/kvm/lib/loongarch/exception.S+6−0

 	st.d   t0, sp, ESTAT_OFFSET_EXREGS
 	csrrd  t0, LOONGARCH_CSR_BADV
 	st.d   t0, sp, BADV_OFFSET_EXREGS
+	csrrd  t0, LOONGARCH_CSR_PRMD
+	st.d   t0, sp, PRMD_OFFSET_EXREGS
 	or     a0, sp, zero
 	bl route_exception
+	ld.d   t0, sp, PC_OFFSET_EXREGS
+	csrwr  t0, LOONGARCH_CSR_ERA
+	ld.d   t0, sp, PRMD_OFFSET_EXREGS
+	csrwr  t0, LOONGARCH_CSR_PRMD
 	restore_gprs sp
 	csrrd  sp, LOONGARCH_CSR_KS0
 	ertn

tools/testing/selftests/kvm/lib/loongarch/processor.c+45−2

 #include <assert.h>
 #include <linux/compiler.h>
+#include <asm/kvm.h>
 #include "kvm_util.h"
 #include "processor.h"
 #include "ucall_common.h"
 #define LOONGARCH_GUEST_STACK_VADDR_MIN		0x200000
 static vm_paddr_t invalid_pgtable[4];
+static vm_vaddr_t exception_handlers;
 static uint64_t virt_pte_index(struct kvm_vm *vm, vm_vaddr_t gva, int level)
 {
 void route_exception(struct ex_regs *regs)
 {
+	int vector;
 	unsigned long pc, estat, badv;
+	struct handlers *handlers;
+	handlers = (struct handlers *)exception_handlers;
+	vector = (regs->estat & CSR_ESTAT_EXC) >> CSR_ESTAT_EXC_SHIFT;
+	if (handlers && handlers->exception_handlers[vector])
+		return handlers->exception_handlers[vector](regs);
 	pc = regs->pc;
 	badv  = regs->badv;
 	while (1) ;
 }
+void vm_init_descriptor_tables(struct kvm_vm *vm)
+{
+	void *addr;
+	vm->handlers = __vm_vaddr_alloc(vm, sizeof(struct handlers),
+			LOONGARCH_GUEST_STACK_VADDR_MIN, MEM_REGION_DATA);
+	addr = addr_gva2hva(vm, vm->handlers);
+	memset(addr, 0, vm->page_size);
+	exception_handlers = vm->handlers;
+	sync_global_to_guest(vm, exception_handlers);
+}
+void vm_install_exception_handler(struct kvm_vm *vm, int vector, handler_fn handler)
+{
+	struct handlers *handlers = addr_gva2hva(vm, vm->handlers);
+	assert(vector < VECTOR_NUM);
+	handlers->exception_handlers[vector] = handler;
+}
+uint32_t guest_get_vcpuid(void)
+{
+	return csr_read(LOONGARCH_CSR_CPUID);
+}
 void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...)
 {
 	int i;
 	vcpu_regs_set(vcpu, &regs);
 }
+static void loongarch_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val)
+{
+	__vcpu_set_reg(vcpu, id, val);
+}
 static void loongarch_get_csr(struct kvm_vcpu *vcpu, uint64_t id, void *addr)
 {
 	uint64_t csrid;
 		TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);
 	}
-	/* user mode and page enable mode */
+	/* kernel mode and page enable mode */
-	val = PLV_USER | CSR_CRMD_PG;
+	val = PLV_KERN | CSR_CRMD_PG;
 	loongarch_set_csr(vcpu, LOONGARCH_CSR_CRMD, val);
 	loongarch_set_csr(vcpu, LOONGARCH_CSR_PRMD, val);
 	loongarch_set_csr(vcpu, LOONGARCH_CSR_EUEN, 1);
 	loongarch_set_csr(vcpu, LOONGARCH_CSR_TCFG, 0);
 	loongarch_set_csr(vcpu, LOONGARCH_CSR_ASID, 1);
+	/* time count start from 0 */
 	val = 0;
+	loongarch_set_reg(vcpu, KVM_REG_LOONGARCH_COUNTER, val);
 	width = vm->page_shift - 3;
 	switch (vm->pgtable_levels) {

tools/testing/selftests/kvm/lib/x86/memstress.c+1−1

@@ -63,7 +63,7 @@ void memstress_setup_ept(struct vmx_pages vmx, struct kvm_vm vm) · 62 unmodified lines
63	{	63	{
64	uint64_t start, end;	64	uint64_t start, end;
65		65
66	prepare_eptp(vmx, vm, 0);	66	prepare_eptp(vmx, vm);
67		67
68	/*	68	/*
69	* Identity map the first 4G and the test region with 1G pages so that	69	* Identity map the first 4G and the test region with 1G pages so that

tools/testing/selftests/kvm/lib/x86/processor.c+41−41

@@ -158,10 +158,10 @@ bool kvm_is_tdp_enabled(void) · 157 unmodified lines
158		158
159	void virt_arch_pgd_alloc(struct kvm_vm *vm)	159	void virt_arch_pgd_alloc(struct kvm_vm *vm)
160	{	160	{
161	TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "	161	TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K,
162	"unknown or unsupported guest mode, mode: 0x%x", vm->mode);	162	"Unknown or unsupported guest mode: 0x%x", vm->mode);
163		163
164	/* If needed, create page map l4 table. */	164	/* If needed, create the top-level page table. */
165	if (!vm->pgd_created) {	165	if (!vm->pgd_created) {
166	vm->pgd = vm_alloc_page_table(vm);	166	vm->pgd = vm_alloc_page_table(vm);
167	vm->pgd_created = true;	167	vm->pgd_created = true;
@@ -218,11 +218,11 @@ static uint64_t virt_create_upper_pte(struct kvm_vm vm, · 50 unmodified lines
218	void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, int level)	218	void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, int level)
219	{	219	{
220	const uint64_t pg_size = PG_LEVEL_SIZE(level);	220	const uint64_t pg_size = PG_LEVEL_SIZE(level);
221	uint64_t pml4e, pdpe, *pde;	221	uint64_t *pte = &vm->pgd;
222	uint64_t *pte;	222	int current_level;
223		223
224	TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K,	224	TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K,
225	"Unknown or unsupported guest mode, mode: 0x%x", vm->mode);	225	"Unknown or unsupported guest mode: 0x%x", vm->mode);
226		226
227	TEST_ASSERT((vaddr % pg_size) == 0,	227	TEST_ASSERT((vaddr % pg_size) == 0,
228	"Virtual address not aligned,\n"	228	"Virtual address not aligned,\n"
@@ -243,20 +243,17 @@ void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, int level) · 14 unmodified lines
243	* Allocate upper level page tables, if not already present. Return	243	* Allocate upper level page tables, if not already present. Return
244	* early if a hugepage was created.	244	* early if a hugepage was created.
245	*/	245	*/
246	pml4e = virt_create_upper_pte(vm, &vm->pgd, vaddr, paddr, PG_LEVEL_512G, level);	246	for (current_level = vm->pgtable_levels;
247	if (*pml4e & PTE_LARGE_MASK)	247	current_level > PG_LEVEL_4K;
248	return;	248	current_level--) {
249		249	pte = virt_create_upper_pte(vm, pte, vaddr, paddr,
250	pdpe = virt_create_upper_pte(vm, pml4e, vaddr, paddr, PG_LEVEL_1G, level);	250	current_level, level);
251	if (*pdpe & PTE_LARGE_MASK)	251	if (*pte & PTE_LARGE_MASK)
252	return;	252	return;
253		253	}
254	pde = virt_create_upper_pte(vm, pdpe, vaddr, paddr, PG_LEVEL_2M, level);
255	if (*pde & PTE_LARGE_MASK)
256	return;
257		254
258	/* Fill in page table entry. */	255	/* Fill in page table entry. */
259	pte = virt_get_pte(vm, pde, vaddr, PG_LEVEL_4K);	256	pte = virt_get_pte(vm, pte, vaddr, PG_LEVEL_4K);
260	TEST_ASSERT(!(*pte & PTE_PRESENT_MASK),	257	TEST_ASSERT(!(*pte & PTE_PRESENT_MASK),
261	"PTE already present for 4k page at vaddr: 0x%lx", vaddr);	258	"PTE already present for 4k page at vaddr: 0x%lx", vaddr);
262	*pte = PTE_PRESENT_MASK \| PTE_WRITABLE_MASK \| (paddr & PHYSICAL_PAGE_MASK);	259	*pte = PTE_PRESENT_MASK \| PTE_WRITABLE_MASK \| (paddr & PHYSICAL_PAGE_MASK);
@@ -289,6 +286,8 @@ void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, · 26 unmodified lines
289		286
290	for (i = 0; i < nr_pages; i++) {	287	for (i = 0; i < nr_pages; i++) {
291	__virt_pg_map(vm, vaddr, paddr, level);	288	__virt_pg_map(vm, vaddr, paddr, level);
		289	sparsebit_set_num(vm->vpages_mapped, vaddr >> vm->page_shift,
		290	nr_bytes / PAGE_SIZE);
292		291
293	vaddr += pg_size;	292	vaddr += pg_size;
294	paddr += pg_size;	293	paddr += pg_size;
@@ -310,40 +309,38 @@ static bool vm_is_target_pte(uint64_t pte, int level, int current_level) · 15 unmodified lines
310	uint64_t __vm_get_page_table_entry(struct kvm_vm vm, uint64_t vaddr,	309	uint64_t __vm_get_page_table_entry(struct kvm_vm vm, uint64_t vaddr,
311	int *level)	310	int *level)
312	{	311	{
313	uint64_t pml4e, pdpe, *pde;	312	int va_width = 12 + (vm->pgtable_levels) * 9;
		313	uint64_t *pte = &vm->pgd;
		314	int current_level;
314		315
315	TEST_ASSERT(!vm->arch.is_pt_protected,	316	TEST_ASSERT(!vm->arch.is_pt_protected,
316	"Walking page tables of protected guests is impossible");	317	"Walking page tables of protected guests is impossible");
317		318
318	TEST_ASSERT(level >= PG_LEVEL_NONE && level < PG_LEVEL_NUM,	319	TEST_ASSERT(level >= PG_LEVEL_NONE && level <= vm->pgtable_levels,
319	"Invalid PG_LEVEL_* '%d'", *level);	320	"Invalid PG_LEVEL_* '%d'", *level);
320		321
321	TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "	322	TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K,
322	"unknown or unsupported guest mode, mode: 0x%x", vm->mode);	323	"Unknown or unsupported guest mode: 0x%x", vm->mode);
323	TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,	324	TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,
324	(vaddr >> vm->page_shift)),	325	(vaddr >> vm->page_shift)),
325	"Invalid virtual address, vaddr: 0x%lx",	326	"Invalid virtual address, vaddr: 0x%lx",
326	vaddr);	327	vaddr);
327	/*	328	/*
328	* Based on the mode check above there are 48 bits in the vaddr, so	329	* Check that the vaddr is a sign-extended va_width value.
329	* shift 16 to sign extend the last bit (bit-47),
330	*/	330	*/
331	TEST_ASSERT(vaddr == (((int64_t)vaddr << 16) >> 16),	331	TEST_ASSERT(vaddr ==
332	"Canonical check failed. The virtual address is invalid.");	332	(((int64_t)vaddr << (64 - va_width) >> (64 - va_width))),
333		333	"Canonical check failed. The virtual address is invalid.");
334	pml4e = virt_get_pte(vm, &vm->pgd, vaddr, PG_LEVEL_512G);	334
335	if (vm_is_target_pte(pml4e, level, PG_LEVEL_512G))	335	for (current_level = vm->pgtable_levels;
336	return pml4e;	336	current_level > PG_LEVEL_4K;
337		337	current_level--) {
338	pdpe = virt_get_pte(vm, pml4e, vaddr, PG_LEVEL_1G);	338	pte = virt_get_pte(vm, pte, vaddr, current_level);
339	if (vm_is_target_pte(pdpe, level, PG_LEVEL_1G))	339	if (vm_is_target_pte(pte, level, current_level))
340	return pdpe;	340	return pte;
341		341	}
342	pde = virt_get_pte(vm, pdpe, vaddr, PG_LEVEL_2M);
343	if (vm_is_target_pte(pde, level, PG_LEVEL_2M))
344	return pde;
345		342
346	return virt_get_pte(vm, pde, vaddr, PG_LEVEL_4K);	343	return virt_get_pte(vm, pte, vaddr, PG_LEVEL_4K);
347	}	344	}
348		345
349	uint64_t vm_get_page_table_entry(struct kvm_vm vm, uint64_t vaddr)	346	uint64_t vm_get_page_table_entry(struct kvm_vm vm, uint64_t vaddr)
@@ -526,7 +523,8 @@ static void vcpu_init_sregs(struct kvm_vm vm, struct kvm_vcpu vcpu) · 176 unmodified lines
526	{	523	{
527	struct kvm_sregs sregs;	524	struct kvm_sregs sregs;
528		525
529	TEST_ASSERT_EQ(vm->mode, VM_MODE_PXXV48_4K);	526	TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K,
		527	"Unknown or unsupported guest mode: 0x%x", vm->mode);
530		528
531	/* Set mode specific system register values. */	529	/* Set mode specific system register values. */
532	vcpu_sregs_get(vcpu, &sregs);	530	vcpu_sregs_get(vcpu, &sregs);
@@ -540,6 +538,8 @@ static void vcpu_init_sregs(struct kvm_vm vm, struct kvm_vcpu vcpu) · 7 unmodified lines
540	sregs.cr4 \|= X86_CR4_PAE \| X86_CR4_OSFXSR;	538	sregs.cr4 \|= X86_CR4_PAE \| X86_CR4_OSFXSR;
541	if (kvm_cpu_has(X86_FEATURE_XSAVE))	539	if (kvm_cpu_has(X86_FEATURE_XSAVE))
542	sregs.cr4 \|= X86_CR4_OSXSAVE;	540	sregs.cr4 \|= X86_CR4_OSXSAVE;
		541	if (vm->pgtable_levels == 5)
		542	sregs.cr4 \|= X86_CR4_LA57;
543	sregs.efer \|= (EFER_LME \| EFER_LMA \| EFER_NX);	543	sregs.efer \|= (EFER_LME \| EFER_LMA \| EFER_NX);
544		544
545	kvm_seg_set_unusable(&sregs.ldt);	545	kvm_seg_set_unusable(&sregs.ldt);

tools/testing/selftests/kvm/lib/x86/vmx.c+4−5

 	struct eptPageTableEntry *pt = vmx->eptp_hva, *pte;
 	uint16_t index;
-	TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "
+	TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K,
-		    "unknown or unsupported guest mode, mode: 0x%x", vm->mode);
+		    "Unknown or unsupported guest mode: 0x%x", vm->mode);
 	TEST_ASSERT((nested_paddr >> 48) == 0,
-		    "Nested physical address 0x%lx requires 5-level paging",
+		    "Nested physical address 0x%lx is > 48-bits and requires 5-level EPT",
 		    nested_paddr);
 	TEST_ASSERT((nested_paddr % page_size) == 0,
 		    "Nested physical address not on page boundary,\n"
 	return ctrl & SECONDARY_EXEC_ENABLE_EPT;
 }
-void prepare_eptp(struct vmx_pages *vmx, struct kvm_vm *vm,
+void prepare_eptp(struct vmx_pages *vmx, struct kvm_vm *vm)
-		  uint32_t eptp_memslot)
 {
 	TEST_ASSERT(kvm_cpu_has_ept(), "KVM doesn't support nested EPT");

tools/testing/selftests/kvm/loongarch/arch_timer.c+200−0

@@ -0,0 +1,200 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* The test validates periodic/one-shot constant timer IRQ using
	4	* CSR.TCFG and CSR.TVAL registers.
	5	*/
	6	#include "arch_timer.h"
	7	#include "kvm_util.h"
	8	#include "processor.h"
	9	#include "timer_test.h"
	10	#include "ucall_common.h"
	11
	12	static void do_idle(void)
	13	{
	14	unsigned int intid;
	15	unsigned long estat;
	16
	17	__asm__ __volatile__("idle 0" : : : "memory");
	18
	19	estat = csr_read(LOONGARCH_CSR_ESTAT);
	20	intid = !!(estat & BIT(INT_TI));
	21
	22	/* Make sure pending timer IRQ arrived */
	23	GUEST_ASSERT_EQ(intid, 1);
	24	csr_write(CSR_TINTCLR_TI, LOONGARCH_CSR_TINTCLR);
	25	}
	26
	27	static void guest_irq_handler(struct ex_regs *regs)
	28	{
	29	unsigned int intid;
	30	uint32_t cpu = guest_get_vcpuid();
	31	uint64_t xcnt, val, cfg, xcnt_diff_us;
	32	struct test_vcpu_shared_data *shared_data = &vcpu_shared_data[cpu];
	33
	34	intid = !!(regs->estat & BIT(INT_TI));
	35
	36	/* Make sure we are dealing with the correct timer IRQ */
	37	GUEST_ASSERT_EQ(intid, 1);
	38
	39	cfg = timer_get_cfg();
	40	if (cfg & CSR_TCFG_PERIOD) {
	41	WRITE_ONCE(shared_data->nr_iter, shared_data->nr_iter - 1);
	42	if (shared_data->nr_iter == 0)
	43	disable_timer();
	44	csr_write(CSR_TINTCLR_TI, LOONGARCH_CSR_TINTCLR);
	45	return;
	46	}
	47
	48	/*
	49	* On real machine, value of LOONGARCH_CSR_TVAL is BIT_ULL(48) - 1
	50	* On virtual machine, its value counts down from BIT_ULL(48) - 1
	51	*/
	52	val = timer_get_val();
	53	xcnt = timer_get_cycles();
	54	xcnt_diff_us = cycles_to_usec(xcnt - shared_data->xcnt);
	55
	56	/* Basic 'timer condition met' check */
	57	__GUEST_ASSERT(val > cfg,
	58	"val = 0x%lx, cfg = 0x%lx, xcnt_diff_us = 0x%lx",
	59	val, cfg, xcnt_diff_us);
	60
	61	csr_write(CSR_TINTCLR_TI, LOONGARCH_CSR_TINTCLR);
	62	WRITE_ONCE(shared_data->nr_iter, shared_data->nr_iter + 1);
	63	}
	64
	65	static void guest_test_period_timer(uint32_t cpu)
	66	{
	67	uint32_t irq_iter, config_iter;
	68	uint64_t us;
	69	struct test_vcpu_shared_data *shared_data = &vcpu_shared_data[cpu];
	70
	71	shared_data->nr_iter = test_args.nr_iter;
	72	shared_data->xcnt = timer_get_cycles();
	73	us = msecs_to_usecs(test_args.timer_period_ms) + test_args.timer_err_margin_us;
	74	timer_set_next_cmp_ms(test_args.timer_period_ms, true);
	75
	76	for (config_iter = 0; config_iter < test_args.nr_iter; config_iter++) {
	77	/* Setup a timeout for the interrupt to arrive */
	78	udelay(us);
	79	}
	80
	81	irq_iter = READ_ONCE(shared_data->nr_iter);
	82	__GUEST_ASSERT(irq_iter == 0,
	83	"irq_iter = 0x%x.\n"
	84	" Guest period timer interrupt was not triggered within the specified\n"
	85	" interval, try to increase the error margin by [-e] option.\n",
	86	irq_iter);
	87	}
	88
	89	static void guest_test_oneshot_timer(uint32_t cpu)
	90	{
	91	uint32_t irq_iter, config_iter;
	92	uint64_t us;
	93	struct test_vcpu_shared_data *shared_data = &vcpu_shared_data[cpu];
	94
	95	shared_data->nr_iter = 0;
	96	shared_data->guest_stage = 0;
	97	us = msecs_to_usecs(test_args.timer_period_ms) + test_args.timer_err_margin_us;
	98	for (config_iter = 0; config_iter < test_args.nr_iter; config_iter++) {
	99	shared_data->xcnt = timer_get_cycles();
	100
	101	/* Setup the next interrupt */
	102	timer_set_next_cmp_ms(test_args.timer_period_ms, false);
	103	/* Setup a timeout for the interrupt to arrive */
	104	udelay(us);
	105
	106	irq_iter = READ_ONCE(shared_data->nr_iter);
	107	__GUEST_ASSERT(config_iter + 1 == irq_iter,
	108	"config_iter + 1 = 0x%x, irq_iter = 0x%x.\n"
	109	" Guest timer interrupt was not triggered within the specified\n"
	110	" interval, try to increase the error margin by [-e] option.\n",
	111	config_iter + 1, irq_iter);
	112	}
	113	}
	114
	115	static void guest_test_emulate_timer(uint32_t cpu)
	116	{
	117	uint32_t config_iter;
	118	uint64_t xcnt_diff_us, us;
	119	struct test_vcpu_shared_data *shared_data = &vcpu_shared_data[cpu];
	120
	121	local_irq_disable();
	122	shared_data->nr_iter = 0;
	123	us = msecs_to_usecs(test_args.timer_period_ms);
	124	for (config_iter = 0; config_iter < test_args.nr_iter; config_iter++) {
	125	shared_data->xcnt = timer_get_cycles();
	126
	127	/* Setup the next interrupt */
	128	timer_set_next_cmp_ms(test_args.timer_period_ms, false);
	129	do_idle();
	130
	131	xcnt_diff_us = cycles_to_usec(timer_get_cycles() - shared_data->xcnt);
	132	__GUEST_ASSERT(xcnt_diff_us >= us,
	133	"xcnt_diff_us = 0x%lx, us = 0x%lx.\n",
	134	xcnt_diff_us, us);
	135	}
	136	local_irq_enable();
	137	}
	138
	139	static void guest_time_count_test(uint32_t cpu)
	140	{
	141	uint32_t config_iter;
	142	unsigned long start, end, prev, us;
	143
	144	/* Assuming that test case starts to run in 1 second */
	145	start = timer_get_cycles();
	146	us = msec_to_cycles(1000);
	147	__GUEST_ASSERT(start <= us,
	148	"start = 0x%lx, us = 0x%lx.\n",
	149	start, us);
	150
	151	us = msec_to_cycles(test_args.timer_period_ms);
	152	for (config_iter = 0; config_iter < test_args.nr_iter; config_iter++) {
	153	start = timer_get_cycles();
	154	end = start + us;
	155	/* test time count growing up always */
	156	while (start < end) {
	157	prev = start;
	158	start = timer_get_cycles();
	159	__GUEST_ASSERT(prev <= start,
	160	"prev = 0x%lx, start = 0x%lx.\n",
	161	prev, start);
	162	}
	163	}
	164	}
	165
	166	static void guest_code(void)
	167	{
	168	uint32_t cpu = guest_get_vcpuid();
	169
	170	/* must run at first */
	171	guest_time_count_test(cpu);
	172
	173	timer_irq_enable();
	174	local_irq_enable();
	175	guest_test_period_timer(cpu);
	176	guest_test_oneshot_timer(cpu);
	177	guest_test_emulate_timer(cpu);
	178
	179	GUEST_DONE();
	180	}
	181
	182	struct kvm_vm *test_vm_create(void)
	183	{
	184	struct kvm_vm *vm;
	185	int nr_vcpus = test_args.nr_vcpus;
	186
	187	vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus);
	188	vm_init_descriptor_tables(vm);
	189	vm_install_exception_handler(vm, EXCCODE_INT, guest_irq_handler);
	190
	191	/* Make all the test's cmdline args visible to the guest */
	192	sync_global_to_guest(vm, test_args);
	193
	194	return vm;
	195	}
	196
	197	void test_vm_cleanup(struct kvm_vm *vm)
	198	{
	199	kvm_vm_free(vm);
	200	}

tools/testing/selftests/kvm/mmu_stress_test.c+5−5

 	TEST_ASSERT(!r, "sched_getaffinity failed, errno = %d (%s)",
 		    errno, strerror(errno));
-	nr_vcpus = CPU_COUNT(&possible_mask) * 3/4;
+	nr_vcpus = CPU_COUNT(&possible_mask);
 	TEST_ASSERT(nr_vcpus > 0, "Uh, no CPUs?");
+	if (nr_vcpus >= 2)
+		nr_vcpus = nr_vcpus * 3/4;
 }
 int main(int argc, char *argv[])
 #ifdef __x86_64__
 		/* Identity map memory in the guest using 1gb pages. */
-		for (i = 0; i < slot_size; i += SZ_1G)
+		virt_map_level(vm, gpa, gpa, slot_size, PG_LEVEL_1G);
-			__virt_pg_map(vm, gpa + i, gpa + i, PG_LEVEL_1G);
 #else
-		for (i = 0; i < slot_size; i += vm->page_size)
+		virt_map(vm, gpa, gpa, slot_size >> vm->page_shift);
-			virt_pg_map(vm, gpa + i, gpa + i);
 #endif
 	}

tools/testing/selftests/kvm/pre_fault_memory_test.c+14−18

 #define TEST_NPAGES		(TEST_SIZE / PAGE_SIZE)
 #define TEST_SLOT		10
-static void guest_code(uint64_t base_gpa)
+static void guest_code(uint64_t base_gva)
 {
 	volatile uint64_t val __used;
 	int i;
 	for (i = 0; i < TEST_NPAGES; i++) {
-		uint64_t *src = (uint64_t *)(base_gpa + i * PAGE_SIZE);
+		uint64_t *src = (uint64_t *)(base_gva + i * PAGE_SIZE);
 		val = *src;
 	}
 static void __test_pre_fault_memory(unsigned long vm_type, bool private)
 {
+	uint64_t gpa, gva, alignment, guest_page_size;
 	const struct vm_shape shape = {
 		.mode = VM_MODE_DEFAULT,
 		.type = vm_type,
 	struct kvm_vm *vm;
 	struct ucall uc;
-	uint64_t guest_test_phys_mem;
-	uint64_t guest_test_virt_mem;
-	uint64_t alignment, guest_page_size;
 	vm = vm_create_shape_with_one_vcpu(shape, &vcpu, guest_code);
 	alignment = guest_page_size = vm_guest_mode_params[VM_MODE_DEFAULT].page_size;
-	guest_test_phys_mem = (vm->max_gfn - TEST_NPAGES) * guest_page_size;
+	gpa = (vm->max_gfn - TEST_NPAGES) * guest_page_size;
 #ifdef __s390x__
 	alignment = max(0x100000UL, guest_page_size);
 #else
 	alignment = SZ_2M;
 #endif
-	guest_test_phys_mem = align_down(guest_test_phys_mem, alignment);
+	gpa = align_down(gpa, alignment);
-	guest_test_virt_mem = guest_test_phys_mem & ((1ULL << (vm->va_bits - 1)) - 1);
+	gva = gpa & ((1ULL << (vm->va_bits - 1)) - 1);
-	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
+	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, gpa, TEST_SLOT,
-				    guest_test_phys_mem, TEST_SLOT, TEST_NPAGES,
+				    TEST_NPAGES, private ? KVM_MEM_GUEST_MEMFD : 0);
-				    private ? KVM_MEM_GUEST_MEMFD : 0);
+	virt_map(vm, gva, gpa, TEST_NPAGES);
-	virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, TEST_NPAGES);
 	if (private)
-		vm_mem_set_private(vm, guest_test_phys_mem, TEST_SIZE);
+		vm_mem_set_private(vm, gpa, TEST_SIZE);
-	pre_fault_memory(vcpu, guest_test_phys_mem, 0, SZ_2M, 0, private);
+	pre_fault_memory(vcpu, gpa, 0, SZ_2M, 0, private);
-	pre_fault_memory(vcpu, guest_test_phys_mem, SZ_2M, PAGE_SIZE * 2, PAGE_SIZE, private);
+	pre_fault_memory(vcpu, gpa, SZ_2M, PAGE_SIZE * 2, PAGE_SIZE, private);
-	pre_fault_memory(vcpu, guest_test_phys_mem, TEST_SIZE, PAGE_SIZE, PAGE_SIZE, private);
+	pre_fault_memory(vcpu, gpa, TEST_SIZE, PAGE_SIZE, PAGE_SIZE, private);
-	vcpu_args_set(vcpu, 1, guest_test_virt_mem);
+	vcpu_args_set(vcpu, 1, gva);
 	vcpu_run(vcpu);
 	run = vcpu->run;

tools/testing/selftests/kvm/riscv/get-reg-list.c+4−0

 	case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_SUSP:
 	case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_STA:
 	case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_FWFT:
+	case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_MPXY:
 	case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_EXPERIMENTAL:
 	case KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_VENDOR:
 		return true;
 		KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_SUSP),
 		KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_STA),
 		KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_FWFT),
+		KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_MPXY),
 		KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_EXPERIMENTAL),
 		KVM_SBI_EXT_ARR(KVM_RISCV_SBI_EXT_VENDOR),
 	};
 KVM_SBI_EXT_SIMPLE_CONFIG(pmu, PMU);
 KVM_SBI_EXT_SIMPLE_CONFIG(dbcn, DBCN);
 KVM_SBI_EXT_SIMPLE_CONFIG(susp, SUSP);
+KVM_SBI_EXT_SIMPLE_CONFIG(mpxy, MPXY);
 KVM_SBI_EXT_SUBLIST_CONFIG(fwft, FWFT);
 KVM_ISA_EXT_SUBLIST_CONFIG(aia, AIA);
 	&config_sbi_pmu,
 	&config_sbi_dbcn,
 	&config_sbi_susp,
+	&config_sbi_mpxy,
 	&config_sbi_fwft,
 	&config_aia,
 	&config_fp_f,

tools/testing/selftests/kvm/s390/user_operexec.c+140−0

@@ -0,0 +1,140 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Test operation exception forwarding.
	3	*
	4	* Copyright IBM Corp. 2025
	5	*
	6	* Authors:
	7	* Janosch Frank <frankja@linux.ibm.com>
	8	*/
	9	#include "kselftest.h"
	10	#include "kvm_util.h"
	11	#include "test_util.h"
	12	#include "sie.h"
	13
	14	#include <linux/kvm.h>
	15
	16	static void guest_code_instr0(void)
	17	{
	18	asm(".word 0x0000");
	19	}
	20
	21	static void test_user_instr0(void)
	22	{
	23	struct kvm_vcpu *vcpu;
	24	struct kvm_vm *vm;
	25	int rc;
	26
	27	vm = vm_create_with_one_vcpu(&vcpu, guest_code_instr0);
	28	rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_INSTR0, 0);
	29	TEST_ASSERT_EQ(0, rc);
	30
	31	vcpu_run(vcpu);
	32	TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_S390_SIEIC);
	33	TEST_ASSERT_EQ(vcpu->run->s390_sieic.icptcode, ICPT_OPEREXC);
	34	TEST_ASSERT_EQ(vcpu->run->s390_sieic.ipa, 0);
	35
	36	kvm_vm_free(vm);
	37	}
	38
	39	static void guest_code_user_operexec(void)
	40	{
	41	asm(".word 0x0807");
	42	}
	43
	44	static void test_user_operexec(void)
	45	{
	46	struct kvm_vcpu *vcpu;
	47	struct kvm_vm *vm;
	48	int rc;
	49
	50	vm = vm_create_with_one_vcpu(&vcpu, guest_code_user_operexec);
	51	rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_OPEREXEC, 0);
	52	TEST_ASSERT_EQ(0, rc);
	53
	54	vcpu_run(vcpu);
	55	TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_S390_SIEIC);
	56	TEST_ASSERT_EQ(vcpu->run->s390_sieic.icptcode, ICPT_OPEREXC);
	57	TEST_ASSERT_EQ(vcpu->run->s390_sieic.ipa, 0x0807);
	58
	59	kvm_vm_free(vm);
	60
	61	/*
	62	* Since user_operexec is the superset it can be used for the
	63	* 0 instruction.
	64	*/
	65	vm = vm_create_with_one_vcpu(&vcpu, guest_code_instr0);
	66	rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_OPEREXEC, 0);
	67	TEST_ASSERT_EQ(0, rc);
	68
	69	vcpu_run(vcpu);
	70	TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_S390_SIEIC);
	71	TEST_ASSERT_EQ(vcpu->run->s390_sieic.icptcode, ICPT_OPEREXC);
	72	TEST_ASSERT_EQ(vcpu->run->s390_sieic.ipa, 0);
	73
	74	kvm_vm_free(vm);
	75	}
	76
	77	/* combine user_instr0 and user_operexec */
	78	static void test_user_operexec_combined(void)
	79	{
	80	struct kvm_vcpu *vcpu;
	81	struct kvm_vm *vm;
	82	int rc;
	83
	84	vm = vm_create_with_one_vcpu(&vcpu, guest_code_user_operexec);
	85	rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_INSTR0, 0);
	86	TEST_ASSERT_EQ(0, rc);
	87	rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_OPEREXEC, 0);
	88	TEST_ASSERT_EQ(0, rc);
	89
	90	vcpu_run(vcpu);
	91	TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_S390_SIEIC);
	92	TEST_ASSERT_EQ(vcpu->run->s390_sieic.icptcode, ICPT_OPEREXC);
	93	TEST_ASSERT_EQ(vcpu->run->s390_sieic.ipa, 0x0807);
	94
	95	kvm_vm_free(vm);
	96
	97	/* Reverse enablement order */
	98	vm = vm_create_with_one_vcpu(&vcpu, guest_code_user_operexec);
	99	rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_OPEREXEC, 0);
	100	TEST_ASSERT_EQ(0, rc);
	101	rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_INSTR0, 0);
	102	TEST_ASSERT_EQ(0, rc);
	103
	104	vcpu_run(vcpu);
	105	TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_S390_SIEIC);
	106	TEST_ASSERT_EQ(vcpu->run->s390_sieic.icptcode, ICPT_OPEREXC);
	107	TEST_ASSERT_EQ(vcpu->run->s390_sieic.ipa, 0x0807);
	108
	109	kvm_vm_free(vm);
	110	}
	111
	112	/*
	113	* Run all tests above.
	114	*
	115	* Enablement after VCPU has been added is automatically tested since
	116	* we enable the capability after VCPU creation.
	117	*/
	118	static struct testdef {
	119	const char *name;
	120	void (*test)(void);
	121	} testlist[] = {
	122	{ "instr0", test_user_instr0 },
	123	{ "operexec", test_user_operexec },
	124	{ "operexec_combined", test_user_operexec_combined},
	125	};
	126
	127	int main(int argc, char *argv[])
	128	{
	129	int idx;
	130
	131	TEST_REQUIRE(kvm_has_cap(KVM_CAP_S390_USER_INSTR0));
	132
	133	ksft_print_header();
	134	ksft_set_plan(ARRAY_SIZE(testlist));
	135	for (idx = 0; idx < ARRAY_SIZE(testlist); idx++) {
	136	testlist[idx].test();
	137	ksft_test_result_pass("%s\n", testlist[idx].name);
	138	}
	139	ksft_finished();
	140	}

tools/testing/selftests/kvm/x86/hyperv_features.c+1−1

@@ -94,7 +94,7 @@ static void guest_hcall(vm_vaddr_t pgs_gpa, struct hcall_data *hcall) · 93 unmodified lines
94		94
95	if (!(hcall->control & HV_HYPERCALL_FAST_BIT)) {	95	if (!(hcall->control & HV_HYPERCALL_FAST_BIT)) {
96	input = pgs_gpa;	96	input = pgs_gpa;
97	output = pgs_gpa + 4096;	97	output = pgs_gpa + PAGE_SIZE;
98	} else {	98	} else {
99	input = output = 0;	99	input = output = 0;
100	}	100	}

tools/testing/selftests/kvm/x86/hyperv_ipi.c+9−9

@@ -102,7 +102,7 @@ static void sender_guest_code(void *hcall_page, vm_vaddr_t pgs_gpa) · 101 unmodified lines
102	/* 'Slow' HvCallSendSyntheticClusterIpi to RECEIVER_VCPU_ID_1 */	102	/* 'Slow' HvCallSendSyntheticClusterIpi to RECEIVER_VCPU_ID_1 */
103	ipi->vector = IPI_VECTOR;	103	ipi->vector = IPI_VECTOR;
104	ipi->cpu_mask = 1 << RECEIVER_VCPU_ID_1;	104	ipi->cpu_mask = 1 << RECEIVER_VCPU_ID_1;
105	hyperv_hypercall(HVCALL_SEND_IPI, pgs_gpa, pgs_gpa + 4096);	105	hyperv_hypercall(HVCALL_SEND_IPI, pgs_gpa, pgs_gpa + PAGE_SIZE);
106	nop_loop();	106	nop_loop();
107	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_1] == ++ipis_expected[0]);	107	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_1] == ++ipis_expected[0]);
108	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_2] == ipis_expected[1]);	108	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_2] == ipis_expected[1]);
@@ -116,13 +116,13 @@ static void sender_guest_code(void *hcall_page, vm_vaddr_t pgs_gpa) · 7 unmodified lines
116	GUEST_SYNC(stage++);	116	GUEST_SYNC(stage++);
117		117
118	/* 'Slow' HvCallSendSyntheticClusterIpiEx to RECEIVER_VCPU_ID_1 */	118	/* 'Slow' HvCallSendSyntheticClusterIpiEx to RECEIVER_VCPU_ID_1 */
119	memset(hcall_page, 0, 4096);	119	memset(hcall_page, 0, PAGE_SIZE);
120	ipi_ex->vector = IPI_VECTOR;	120	ipi_ex->vector = IPI_VECTOR;
121	ipi_ex->vp_set.format = HV_GENERIC_SET_SPARSE_4K;	121	ipi_ex->vp_set.format = HV_GENERIC_SET_SPARSE_4K;
122	ipi_ex->vp_set.valid_bank_mask = 1 << 0;	122	ipi_ex->vp_set.valid_bank_mask = 1 << 0;
123	ipi_ex->vp_set.bank_contents[0] = BIT(RECEIVER_VCPU_ID_1);	123	ipi_ex->vp_set.bank_contents[0] = BIT(RECEIVER_VCPU_ID_1);
124	hyperv_hypercall(HVCALL_SEND_IPI_EX \| (1 << HV_HYPERCALL_VARHEAD_OFFSET),	124	hyperv_hypercall(HVCALL_SEND_IPI_EX \| (1 << HV_HYPERCALL_VARHEAD_OFFSET),
125	pgs_gpa, pgs_gpa + 4096);	125	pgs_gpa, pgs_gpa + PAGE_SIZE);
126	nop_loop();	126	nop_loop();
127	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_1] == ++ipis_expected[0]);	127	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_1] == ++ipis_expected[0]);
128	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_2] == ipis_expected[1]);	128	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_2] == ipis_expected[1]);
@@ -138,13 +138,13 @@ static void sender_guest_code(void *hcall_page, vm_vaddr_t pgs_gpa) · 9 unmodified lines
138	GUEST_SYNC(stage++);	138	GUEST_SYNC(stage++);
139		139
140	/* 'Slow' HvCallSendSyntheticClusterIpiEx to RECEIVER_VCPU_ID_2 */	140	/* 'Slow' HvCallSendSyntheticClusterIpiEx to RECEIVER_VCPU_ID_2 */
141	memset(hcall_page, 0, 4096);	141	memset(hcall_page, 0, PAGE_SIZE);
142	ipi_ex->vector = IPI_VECTOR;	142	ipi_ex->vector = IPI_VECTOR;
143	ipi_ex->vp_set.format = HV_GENERIC_SET_SPARSE_4K;	143	ipi_ex->vp_set.format = HV_GENERIC_SET_SPARSE_4K;
144	ipi_ex->vp_set.valid_bank_mask = 1 << 1;	144	ipi_ex->vp_set.valid_bank_mask = 1 << 1;
145	ipi_ex->vp_set.bank_contents[0] = BIT(RECEIVER_VCPU_ID_2 - 64);	145	ipi_ex->vp_set.bank_contents[0] = BIT(RECEIVER_VCPU_ID_2 - 64);
146	hyperv_hypercall(HVCALL_SEND_IPI_EX \| (1 << HV_HYPERCALL_VARHEAD_OFFSET),	146	hyperv_hypercall(HVCALL_SEND_IPI_EX \| (1 << HV_HYPERCALL_VARHEAD_OFFSET),
147	pgs_gpa, pgs_gpa + 4096);	147	pgs_gpa, pgs_gpa + PAGE_SIZE);
148	nop_loop();	148	nop_loop();
149	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_1] == ipis_expected[0]);	149	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_1] == ipis_expected[0]);
150	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_2] == ++ipis_expected[1]);	150	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_2] == ++ipis_expected[1]);
@@ -160,14 +160,14 @@ static void sender_guest_code(void *hcall_page, vm_vaddr_t pgs_gpa) · 9 unmodified lines
160	GUEST_SYNC(stage++);	160	GUEST_SYNC(stage++);
161		161
162	/* 'Slow' HvCallSendSyntheticClusterIpiEx to both RECEIVER_VCPU_ID_{1,2} */	162	/* 'Slow' HvCallSendSyntheticClusterIpiEx to both RECEIVER_VCPU_ID_{1,2} */
163	memset(hcall_page, 0, 4096);	163	memset(hcall_page, 0, PAGE_SIZE);
164	ipi_ex->vector = IPI_VECTOR;	164	ipi_ex->vector = IPI_VECTOR;
165	ipi_ex->vp_set.format = HV_GENERIC_SET_SPARSE_4K;	165	ipi_ex->vp_set.format = HV_GENERIC_SET_SPARSE_4K;
166	ipi_ex->vp_set.valid_bank_mask = 1 << 1 \| 1;	166	ipi_ex->vp_set.valid_bank_mask = 1 << 1 \| 1;
167	ipi_ex->vp_set.bank_contents[0] = BIT(RECEIVER_VCPU_ID_1);	167	ipi_ex->vp_set.bank_contents[0] = BIT(RECEIVER_VCPU_ID_1);
168	ipi_ex->vp_set.bank_contents[1] = BIT(RECEIVER_VCPU_ID_2 - 64);	168	ipi_ex->vp_set.bank_contents[1] = BIT(RECEIVER_VCPU_ID_2 - 64);
169	hyperv_hypercall(HVCALL_SEND_IPI_EX \| (2 << HV_HYPERCALL_VARHEAD_OFFSET),	169	hyperv_hypercall(HVCALL_SEND_IPI_EX \| (2 << HV_HYPERCALL_VARHEAD_OFFSET),
170	pgs_gpa, pgs_gpa + 4096);	170	pgs_gpa, pgs_gpa + PAGE_SIZE);
171	nop_loop();	171	nop_loop();
172	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_1] == ++ipis_expected[0]);	172	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_1] == ++ipis_expected[0]);
173	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_2] == ++ipis_expected[1]);	173	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_2] == ++ipis_expected[1]);
@@ -183,10 +183,10 @@ static void sender_guest_code(void *hcall_page, vm_vaddr_t pgs_gpa) · 9 unmodified lines
183	GUEST_SYNC(stage++);	183	GUEST_SYNC(stage++);
184		184
185	/* 'Slow' HvCallSendSyntheticClusterIpiEx to HV_GENERIC_SET_ALL */	185	/* 'Slow' HvCallSendSyntheticClusterIpiEx to HV_GENERIC_SET_ALL */
186	memset(hcall_page, 0, 4096);	186	memset(hcall_page, 0, PAGE_SIZE);
187	ipi_ex->vector = IPI_VECTOR;	187	ipi_ex->vector = IPI_VECTOR;
188	ipi_ex->vp_set.format = HV_GENERIC_SET_ALL;	188	ipi_ex->vp_set.format = HV_GENERIC_SET_ALL;
189	hyperv_hypercall(HVCALL_SEND_IPI_EX, pgs_gpa, pgs_gpa + 4096);	189	hyperv_hypercall(HVCALL_SEND_IPI_EX, pgs_gpa, pgs_gpa + PAGE_SIZE);
190	nop_loop();	190	nop_loop();
191	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_1] == ++ipis_expected[0]);	191	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_1] == ++ipis_expected[0]);
192	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_2] == ++ipis_expected[1]);	192	GUEST_ASSERT(ipis_rcvd[RECEIVER_VCPU_ID_2] == ++ipis_expected[1]);

tools/testing/selftests/kvm/x86/hyperv_tlb_flush.c+1−1

@@ -621,7 +621,7 @@ int main(int argc, char *argv[]) · 620 unmodified lines
621	for (i = 0; i < NTEST_PAGES; i++) {	621	for (i = 0; i < NTEST_PAGES; i++) {
622	pte = vm_get_page_table_entry(vm, data->test_pages + i * PAGE_SIZE);	622	pte = vm_get_page_table_entry(vm, data->test_pages + i * PAGE_SIZE);
623	gpa = addr_hva2gpa(vm, pte);	623	gpa = addr_hva2gpa(vm, pte);
624	__virt_pg_map(vm, gva + PAGE_SIZE * i, gpa & PAGE_MASK, PG_LEVEL_4K);	624	virt_pg_map(vm, gva + PAGE_SIZE * i, gpa & PAGE_MASK);
625	data->test_pages_pte[i] = gva + (gpa & ~PAGE_MASK);	625	data->test_pages_pte[i] = gva + (gpa & ~PAGE_MASK);
626	}	626	}
627		627

tools/testing/selftests/kvm/x86/nested_close_kvm_test.crenamed from tools/testing/selftests/kvm/x86/vmx_close_while_nested_test.c+33−9

 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * vmx_close_while_nested
- *
  * Copyright (C) 2019, Red Hat, Inc.
  *
  * Verify that nothing bad happens if a KVM user exits with open
 #include "kvm_util.h"
 #include "processor.h"
 #include "vmx.h"
+#include "svm_util.h"
 #include <string.h>
 #include <sys/ioctl.h>
 	PORT_L0_EXIT = 0x2000,
 };
+#define L2_GUEST_STACK_SIZE 64
 static void l2_guest_code(void)
 {
 	/* Exit to L0 */
 		     : : [port] "d" (PORT_L0_EXIT) : "rax");
 }
-static void l1_guest_code(struct vmx_pages *vmx_pages)
+static void l1_vmx_code(struct vmx_pages *vmx_pages)
 {
-#define L2_GUEST_STACK_SIZE 64
 	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
 	GUEST_ASSERT(0);
 }
+static void l1_svm_code(struct svm_test_data *svm)
+{
+	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
+	/* Prepare the VMCB for L2 execution. */
+	generic_svm_setup(svm, l2_guest_code,
+			  &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	run_guest(svm->vmcb, svm->vmcb_gpa);
+	GUEST_ASSERT(0);
+}
+static void l1_guest_code(void *data)
+{
+	if (this_cpu_has(X86_FEATURE_VMX))
+		l1_vmx_code(data);
+	else
+		l1_svm_code(data);
+}
 int main(int argc, char *argv[])
 {
-	vm_vaddr_t vmx_pages_gva;
+	vm_vaddr_t guest_gva;
 	struct kvm_vcpu *vcpu;
 	struct kvm_vm *vm;
-	TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX));
+	TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) ||
+		     kvm_cpu_has(X86_FEATURE_SVM));
 	vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code);
-	/* Allocate VMX pages and shared descriptors (vmx_pages). */
+	if (kvm_cpu_has(X86_FEATURE_VMX))
-	vcpu_alloc_vmx(vm, &vmx_pages_gva);
+		vcpu_alloc_vmx(vm, &guest_gva);
-	vcpu_args_set(vcpu, 1, vmx_pages_gva);
+	else
+		vcpu_alloc_svm(vm, &guest_gva);
+	vcpu_args_set(vcpu, 1, guest_gva);
 	for (;;) {
 		volatile struct kvm_run *run = vcpu->run;

tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c+116−0

@@ -0,0 +1,116 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (C) 2025, Google LLC.
	4	*
	5	* This test verifies that L1 fails to enter L2 with an invalid CR3, and
	6	* succeeds otherwise.
	7	*/
	8	#include "kvm_util.h"
	9	#include "vmx.h"
	10	#include "svm_util.h"
	11	#include "kselftest.h"
	12
	13
	14	#define L2_GUEST_STACK_SIZE 64
	15
	16	static void l2_guest_code(void)
	17	{
	18	vmcall();
	19	}
	20
	21	static void l1_svm_code(struct svm_test_data *svm)
	22	{
	23	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
	24	uintptr_t save_cr3;
	25
	26	generic_svm_setup(svm, l2_guest_code,
	27	&l2_guest_stack[L2_GUEST_STACK_SIZE]);
	28
	29	/* Try to run L2 with invalid CR3 and make sure it fails */
	30	save_cr3 = svm->vmcb->save.cr3;
	31	svm->vmcb->save.cr3 = -1ull;
	32	run_guest(svm->vmcb, svm->vmcb_gpa);
	33	GUEST_ASSERT(svm->vmcb->control.exit_code == SVM_EXIT_ERR);
	34
	35	/* Now restore CR3 and make sure L2 runs successfully */
	36	svm->vmcb->save.cr3 = save_cr3;
	37	run_guest(svm->vmcb, svm->vmcb_gpa);
	38	GUEST_ASSERT(svm->vmcb->control.exit_code == SVM_EXIT_VMMCALL);
	39
	40	GUEST_DONE();
	41	}
	42
	43	static void l1_vmx_code(struct vmx_pages *vmx_pages)
	44	{
	45	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
	46	uintptr_t save_cr3;
	47
	48	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
	49	GUEST_ASSERT(load_vmcs(vmx_pages));
	50
	51	prepare_vmcs(vmx_pages, l2_guest_code,
	52	&l2_guest_stack[L2_GUEST_STACK_SIZE]);
	53
	54	/* Try to run L2 with invalid CR3 and make sure it fails */
	55	save_cr3 = vmreadz(GUEST_CR3);
	56	vmwrite(GUEST_CR3, -1ull);
	57	GUEST_ASSERT(!vmlaunch());
	58	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) ==
	59	(EXIT_REASON_FAILED_VMENTRY \| EXIT_REASON_INVALID_STATE));
	60
	61	/* Now restore CR3 and make sure L2 runs successfully */
	62	vmwrite(GUEST_CR3, save_cr3);
	63	GUEST_ASSERT(!vmlaunch());
	64	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
	65
	66	GUEST_DONE();
	67	}
	68
	69	static void l1_guest_code(void *data)
	70	{
	71	if (this_cpu_has(X86_FEATURE_VMX))
	72	l1_vmx_code(data);
	73	else
	74	l1_svm_code(data);
	75	}
	76
	77	int main(int argc, char *argv[])
	78	{
	79	struct kvm_vcpu *vcpu;
	80	struct kvm_vm *vm;
	81	vm_vaddr_t guest_gva = 0;
	82
	83	TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) \|\|
	84	kvm_cpu_has(X86_FEATURE_SVM));
	85
	86	vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code);
	87
	88	if (kvm_cpu_has(X86_FEATURE_VMX))
	89	vcpu_alloc_vmx(vm, &guest_gva);
	90	else
	91	vcpu_alloc_svm(vm, &guest_gva);
	92
	93	vcpu_args_set(vcpu, 1, guest_gva);
	94
	95	for (;;) {
	96	struct ucall uc;
	97
	98	vcpu_run(vcpu);
	99	TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
	100
	101	switch (get_ucall(vcpu, &uc)) {
	102	case UCALL_ABORT:
	103	REPORT_GUEST_ASSERT(uc);
	104	case UCALL_SYNC:
	105	break;
	106	case UCALL_DONE:
	107	goto done;
	108	default:
	109	TEST_FAIL("Unknown ucall %lu", uc.cmd);
	110	}
	111	}
	112
	113	done:
	114	kvm_vm_free(vm);
	115	return 0;
	116	}

tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.crenamed from tools/testing/selftests/kvm/x86/vmx_tsc_adjust_test.c+44−35

@@ -1,7 +1,5 @@
1	// SPDX-License-Identifier: GPL-2.0-only	1	// SPDX-License-Identifier: GPL-2.0-only
2	/*	2	/*
3	* vmx_tsc_adjust_test
4	*
5	* Copyright (C) 2018, Google LLC.	3	* Copyright (C) 2018, Google LLC.
6	*	4	*
7	* IA32_TSC_ADJUST test	5	* IA32_TSC_ADJUST test
@@ -22,6 +20,7 @@ · 14 unmodified lines
22	#include "kvm_util.h"	20	#include "kvm_util.h"
23	#include "processor.h"	21	#include "processor.h"
24	#include "vmx.h"	22	#include "vmx.h"
		23	#include "svm_util.h"
25		24
26	#include <string.h>	25	#include <string.h>
27	#include <sys/ioctl.h>	26	#include <sys/ioctl.h>
@@ -35,6 +34,8 @@ · 7 unmodified lines
35	#define TSC_ADJUST_VALUE (1ll << 32)	34	#define TSC_ADJUST_VALUE (1ll << 32)
36	#define TSC_OFFSET_VALUE -(1ll << 48)	35	#define TSC_OFFSET_VALUE -(1ll << 48)
37		36
		37	#define L2_GUEST_STACK_SIZE 64
		38
38	enum {	39	enum {
39	PORT_ABORT = 0x1000,	40	PORT_ABORT = 0x1000,
40	PORT_REPORT,	41	PORT_REPORT,
@@ -72,42 +73,47 @@ static void l2_guest_code(void) · 31 unmodified lines
72	__asm__ __volatile__("vmcall");	73	__asm__ __volatile__("vmcall");
73	}	74	}
74		75
75	static void l1_guest_code(struct vmx_pages *vmx_pages)	76	static void l1_guest_code(void *data)
76	{	77	{
77	#define L2_GUEST_STACK_SIZE 64
78	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];	78	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
79	uint32_t control;
80	uintptr_t save_cr3;
81		79
		80	/* Set TSC from L1 and make sure TSC_ADJUST is updated correctly */
82	GUEST_ASSERT(rdtsc() < TSC_ADJUST_VALUE);	81	GUEST_ASSERT(rdtsc() < TSC_ADJUST_VALUE);
83	wrmsr(MSR_IA32_TSC, rdtsc() - TSC_ADJUST_VALUE);	82	wrmsr(MSR_IA32_TSC, rdtsc() - TSC_ADJUST_VALUE);
84	check_ia32_tsc_adjust(-1 * TSC_ADJUST_VALUE);	83	check_ia32_tsc_adjust(-1 * TSC_ADJUST_VALUE);
85		84
86	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));	85	/*
87	GUEST_ASSERT(load_vmcs(vmx_pages));	86	* Run L2 with TSC_OFFSET. L2 will write to TSC, and L1 is not
88		87	* intercepting the write so it should update L1's TSC_ADJUST.
89	/* Prepare the VMCS for L2 execution. */	88	*/
90	prepare_vmcs(vmx_pages, l2_guest_code,	89	if (this_cpu_has(X86_FEATURE_VMX)) {
91	&l2_guest_stack[L2_GUEST_STACK_SIZE]);	90	struct vmx_pages *vmx_pages = data;
92	control = vmreadz(CPU_BASED_VM_EXEC_CONTROL);	91	uint32_t control;
93	control \|= CPU_BASED_USE_MSR_BITMAPS \| CPU_BASED_USE_TSC_OFFSETTING;	92
94	vmwrite(CPU_BASED_VM_EXEC_CONTROL, control);	93	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
95	vmwrite(TSC_OFFSET, TSC_OFFSET_VALUE);	94	GUEST_ASSERT(load_vmcs(vmx_pages));
96		95
97	/* Jump into L2. First, test failure to load guest CR3. */	96	prepare_vmcs(vmx_pages, l2_guest_code,
98	save_cr3 = vmreadz(GUEST_CR3);	97	&l2_guest_stack[L2_GUEST_STACK_SIZE]);
99	vmwrite(GUEST_CR3, -1ull);	98	control = vmreadz(CPU_BASED_VM_EXEC_CONTROL);
100	GUEST_ASSERT(!vmlaunch());	99	control \|= CPU_BASED_USE_MSR_BITMAPS \| CPU_BASED_USE_TSC_OFFSETTING;
101	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) ==	100	vmwrite(CPU_BASED_VM_EXEC_CONTROL, control);
102	(EXIT_REASON_FAILED_VMENTRY \| EXIT_REASON_INVALID_STATE));	101	vmwrite(TSC_OFFSET, TSC_OFFSET_VALUE);
103	check_ia32_tsc_adjust(-1 * TSC_ADJUST_VALUE);	102
104	vmwrite(GUEST_CR3, save_cr3);	103	GUEST_ASSERT(!vmlaunch());
105		104	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);
106	GUEST_ASSERT(!vmlaunch());	105	} else {
107	GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL);	106	struct svm_test_data *svm = data;
		107
		108	generic_svm_setup(svm, l2_guest_code,
		109	&l2_guest_stack[L2_GUEST_STACK_SIZE]);
		110
		111	svm->vmcb->control.tsc_offset = TSC_OFFSET_VALUE;
		112	run_guest(svm->vmcb, svm->vmcb_gpa);
		113	GUEST_ASSERT(svm->vmcb->control.exit_code == SVM_EXIT_VMMCALL);
		114	}
108		115
109	check_ia32_tsc_adjust(-2 * TSC_ADJUST_VALUE);	116	check_ia32_tsc_adjust(-2 * TSC_ADJUST_VALUE);
110
111	GUEST_DONE();	117	GUEST_DONE();
112	}	118	}
113		119
@@ -119,16 +125,19 @@ static void report(int64_t val) · 5 unmodified lines
119		125
120	int main(int argc, char *argv[])	126	int main(int argc, char *argv[])
121	{	127	{
122	vm_vaddr_t vmx_pages_gva;	128	vm_vaddr_t nested_gva;
123	struct kvm_vcpu *vcpu;	129	struct kvm_vcpu *vcpu;
124		130
125	TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX));	131	TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) \|\|
		132	kvm_cpu_has(X86_FEATURE_SVM));
126		133
127	vm = vm_create_with_one_vcpu(&vcpu, (void *) l1_guest_code);	134	vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code);
		135	if (kvm_cpu_has(X86_FEATURE_VMX))
		136	vcpu_alloc_vmx(vm, &nested_gva);
		137	else
		138	vcpu_alloc_svm(vm, &nested_gva);
128		139
129	/* Allocate VMX pages and shared descriptors (vmx_pages). */	140	vcpu_args_set(vcpu, 1, nested_gva);
130	vcpu_alloc_vmx(vm, &vmx_pages_gva);
131	vcpu_args_set(vcpu, 1, vmx_pages_gva);
132		141
133	for (;;) {	142	for (;;) {
134	struct ucall uc;	143	struct ucall uc;

tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.crenamed from tools/testing/selftests/kvm/x86/vmx_nested_tsc_scaling_test.c+43−5

@@ -13,6 +13,7 @@ · 12 unmodified lines
13		13
14	#include "kvm_util.h"	14	#include "kvm_util.h"
15	#include "vmx.h"	15	#include "vmx.h"
		16	#include "svm_util.h"
16	#include "kselftest.h"	17	#include "kselftest.h"
17		18
18	/* L2 is scaled up (from L1's perspective) by this factor */	19	/* L2 is scaled up (from L1's perspective) by this factor */
@@ -79,7 +80,30 @@ static void l2_guest_code(void) · 60 unmodified lines
79	__asm__ __volatile__("vmcall");	80	__asm__ __volatile__("vmcall");
80	}	81	}
81		82
82	static void l1_guest_code(struct vmx_pages *vmx_pages)	83	static void l1_svm_code(struct svm_test_data *svm)
		84	{
		85	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
		86
		87	/* check that L1's frequency looks alright before launching L2 */
		88	check_tsc_freq(UCHECK_L1);
		89
		90	generic_svm_setup(svm, l2_guest_code,
		91	&l2_guest_stack[L2_GUEST_STACK_SIZE]);
		92
		93	/* enable TSC scaling for L2 */
		94	wrmsr(MSR_AMD64_TSC_RATIO, L2_SCALE_FACTOR << 32);
		95
		96	/* launch L2 */
		97	run_guest(svm->vmcb, svm->vmcb_gpa);
		98	GUEST_ASSERT(svm->vmcb->control.exit_code == SVM_EXIT_VMMCALL);
		99
		100	/* check that L1's frequency still looks good */
		101	check_tsc_freq(UCHECK_L1);
		102
		103	GUEST_DONE();
		104	}
		105
		106	static void l1_vmx_code(struct vmx_pages *vmx_pages)
83	{	107	{
84	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];	108	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
85	uint32_t control;	109	uint32_t control;
@@ -116,11 +140,19 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) · 30 unmodified lines
116	GUEST_DONE();	140	GUEST_DONE();
117	}	141	}
118		142
		143	static void l1_guest_code(void *data)
		144	{
		145	if (this_cpu_has(X86_FEATURE_VMX))
		146	l1_vmx_code(data);
		147	else
		148	l1_svm_code(data);
		149	}
		150
119	int main(int argc, char *argv[])	151	int main(int argc, char *argv[])
120	{	152	{
121	struct kvm_vcpu *vcpu;	153	struct kvm_vcpu *vcpu;
122	struct kvm_vm *vm;	154	struct kvm_vm *vm;
123	vm_vaddr_t vmx_pages_gva;	155	vm_vaddr_t guest_gva = 0;
124		156
125	uint64_t tsc_start, tsc_end;	157	uint64_t tsc_start, tsc_end;
126	uint64_t tsc_khz;	158	uint64_t tsc_khz;
@@ -129,7 +161,8 @@ int main(int argc, char *argv[]) · 2 unmodified lines
129	uint64_t l1_tsc_freq = 0;	161	uint64_t l1_tsc_freq = 0;
130	uint64_t l2_tsc_freq = 0;	162	uint64_t l2_tsc_freq = 0;
131		163
132	TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX));	164	TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) \|\|
		165	kvm_cpu_has(X86_FEATURE_SVM));
133	TEST_REQUIRE(kvm_has_cap(KVM_CAP_TSC_CONTROL));	166	TEST_REQUIRE(kvm_has_cap(KVM_CAP_TSC_CONTROL));
134	TEST_REQUIRE(sys_clocksource_is_based_on_tsc());	167	TEST_REQUIRE(sys_clocksource_is_based_on_tsc());
135		168
@@ -152,8 +185,13 @@ int main(int argc, char *argv[]) · 16 unmodified lines
152	printf("real TSC frequency is around: %"PRIu64"\n", l0_tsc_freq);	185	printf("real TSC frequency is around: %"PRIu64"\n", l0_tsc_freq);
153		186
154	vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code);	187	vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code);
155	vcpu_alloc_vmx(vm, &vmx_pages_gva);	188
156	vcpu_args_set(vcpu, 1, vmx_pages_gva);	189	if (kvm_cpu_has(X86_FEATURE_VMX))
		190	vcpu_alloc_vmx(vm, &guest_gva);
		191	else
		192	vcpu_alloc_svm(vm, &guest_gva);
		193
		194	vcpu_args_set(vcpu, 1, guest_gva);
157		195
158	tsc_khz = __vcpu_ioctl(vcpu, KVM_GET_TSC_KHZ, NULL);	196	tsc_khz = __vcpu_ioctl(vcpu, KVM_GET_TSC_KHZ, NULL);
159	TEST_ASSERT(tsc_khz != -1, "vcpu ioctl KVM_GET_TSC_KHZ failed");	197	TEST_ASSERT(tsc_khz != -1, "vcpu ioctl KVM_GET_TSC_KHZ failed");

tools/testing/selftests/kvm/x86/private_mem_conversions_test.c+3−6

 	struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
 	pthread_t threads[KVM_MAX_VCPUS];
 	struct kvm_vm *vm;
-	int memfd, i, r;
+	int memfd, i;
 	const struct vm_shape shape = {
 		.mode = VM_MODE_DEFAULT,
 	 * should prevent the VM from being fully destroyed until the last
 	 * reference to the guest_memfd is also put.
 	 */
-	r = fallocate(memfd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, memfd_size);
+	kvm_fallocate(memfd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, memfd_size);
-	TEST_ASSERT(!r, __KVM_SYSCALL_ERROR("fallocate()", r));
+	kvm_fallocate(memfd, FALLOC_FL_KEEP_SIZE, 0, memfd_size);
-	r = fallocate(memfd, FALLOC_FL_KEEP_SIZE, 0, memfd_size);
-	TEST_ASSERT(!r, __KVM_SYSCALL_ERROR("fallocate()", r));
 	close(memfd);
 }

tools/testing/selftests/kvm/x86/sev_smoke_test.c+1−1

@@ -104,7 +104,7 @@ static void test_sync_vmsa(uint32_t type, uint64_t policy) · 103 unmodified lines
104	vm_sev_launch(vm, policy, NULL);	104	vm_sev_launch(vm, policy, NULL);
105		105
106	/* This page is shared, so make it decrypted. */	106	/* This page is shared, so make it decrypted. */
107	memset(hva, 0, 4096);	107	memset(hva, 0, PAGE_SIZE);
108		108
109	vcpu_run(vcpu);	109	vcpu_run(vcpu);
110		110

tools/testing/selftests/kvm/x86/state_test.c+1−1

@@ -141,7 +141,7 @@ static void __attribute__((__flatten__)) guest_code(void *arg) · 140 unmodified lines
141		141
142	if (this_cpu_has(X86_FEATURE_XSAVE)) {	142	if (this_cpu_has(X86_FEATURE_XSAVE)) {
143	uint64_t supported_xcr0 = this_cpu_supported_xcr0();	143	uint64_t supported_xcr0 = this_cpu_supported_xcr0();
144	uint8_t buffer[4096];	144	uint8_t buffer[PAGE_SIZE];
145		145
146	memset(buffer, 0xcc, sizeof(buffer));	146	memset(buffer, 0xcc, sizeof(buffer));
147		147

tools/testing/selftests/kvm/x86/userspace_io_test.c+1−1

@@ -85,7 +85,7 @@ int main(int argc, char *argv[]) · 84 unmodified lines
85	regs.rcx = 1;	85	regs.rcx = 1;
86	if (regs.rcx == 3)	86	if (regs.rcx == 3)
87	regs.rcx = 8192;	87	regs.rcx = 8192;
88	memset((void *)run + run->io.data_offset, 0xaa, 4096);	88	memset((void *)run + run->io.data_offset, 0xaa, PAGE_SIZE);
89	vcpu_regs_set(vcpu, &regs);	89	vcpu_regs_set(vcpu, &regs);
90	}	90	}
91		91

tools/testing/selftests/kvm/x86/vmx_dirty_log_test.c+6−6

@@ -120,17 +120,17 @@ static void test_vmx_dirty_log(bool enable_ept) · 119 unmodified lines
120	* GPAs as the EPT enabled case.	120	* GPAs as the EPT enabled case.
121	*/	121	*/
122	if (enable_ept) {	122	if (enable_ept) {
123	prepare_eptp(vmx, vm, 0);	123	prepare_eptp(vmx, vm);
124	nested_map_memslot(vmx, vm, 0);	124	nested_map_memslot(vmx, vm, 0);
125	nested_map(vmx, vm, NESTED_TEST_MEM1, GUEST_TEST_MEM, 4096);	125	nested_map(vmx, vm, NESTED_TEST_MEM1, GUEST_TEST_MEM, PAGE_SIZE);
126	nested_map(vmx, vm, NESTED_TEST_MEM2, GUEST_TEST_MEM, 4096);	126	nested_map(vmx, vm, NESTED_TEST_MEM2, GUEST_TEST_MEM, PAGE_SIZE);
127	}	127	}
128		128
129	bmap = bitmap_zalloc(TEST_MEM_PAGES);	129	bmap = bitmap_zalloc(TEST_MEM_PAGES);
130	host_test_mem = addr_gpa2hva(vm, GUEST_TEST_MEM);	130	host_test_mem = addr_gpa2hva(vm, GUEST_TEST_MEM);
131		131
132	while (!done) {	132	while (!done) {
133	memset(host_test_mem, 0xaa, TEST_MEM_PAGES * 4096);	133	memset(host_test_mem, 0xaa, TEST_MEM_PAGES * PAGE_SIZE);
134	vcpu_run(vcpu);	134	vcpu_run(vcpu);
135	TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);	135	TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
136		136
@@ -153,9 +153,9 @@ static void test_vmx_dirty_log(bool enable_ept) · 16 unmodified lines
153	}	153	}
154		154
155	TEST_ASSERT(!test_bit(1, bmap), "Page 1 incorrectly reported dirty");	155	TEST_ASSERT(!test_bit(1, bmap), "Page 1 incorrectly reported dirty");
156	TEST_ASSERT(host_test_mem[4096 / 8] == 0xaaaaaaaaaaaaaaaaULL, "Page 1 written by guest");	156	TEST_ASSERT(host_test_mem[PAGE_SIZE / 8] == 0xaaaaaaaaaaaaaaaaULL, "Page 1 written by guest");
157	TEST_ASSERT(!test_bit(2, bmap), "Page 2 incorrectly reported dirty");	157	TEST_ASSERT(!test_bit(2, bmap), "Page 2 incorrectly reported dirty");
158	TEST_ASSERT(host_test_mem[8192 / 8] == 0xaaaaaaaaaaaaaaaaULL, "Page 2 written by guest");	158	TEST_ASSERT(host_test_mem[PAGE_SIZE*2 / 8] == 0xaaaaaaaaaaaaaaaaULL, "Page 2 written by guest");
159	break;	159	break;
160	case UCALL_DONE:	160	case UCALL_DONE:
161	done = true;	161	done = true;

tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c+132−0

@@ -0,0 +1,132 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (C) 2025, Google LLC.
	4	*
	5	* Test KVM's ability to save and restore nested state when the L1 guest
	6	* is using 5-level paging and the L2 guest is using 4-level paging.
	7	*
	8	* This test would have failed prior to commit 9245fd6b8531 ("KVM: x86:
	9	* model canonical checks more precisely").
	10	*/
	11	#include "test_util.h"
	12	#include "kvm_util.h"
	13	#include "processor.h"
	14	#include "vmx.h"
	15
	16	#define LA57_GS_BASE 0xff2bc0311fb00000ull
	17
	18	static void l2_guest_code(void)
	19	{
	20	/*
	21	* Sync with L0 to trigger save/restore. After
	22	* resuming, execute VMCALL to exit back to L1.
	23	*/
	24	GUEST_SYNC(1);
	25	vmcall();
	26	}
	27
	28	static void l1_guest_code(struct vmx_pages *vmx_pages)
	29	{
	30	#define L2_GUEST_STACK_SIZE 64
	31	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
	32	u64 guest_cr4;
	33	vm_paddr_t pml5_pa, pml4_pa;
	34	u64 *pml5;
	35	u64 exit_reason;
	36
	37	/* Set GS_BASE to a value that is only canonical with LA57. */
	38	wrmsr(MSR_GS_BASE, LA57_GS_BASE);
	39	GUEST_ASSERT(rdmsr(MSR_GS_BASE) == LA57_GS_BASE);
	40
	41	GUEST_ASSERT(vmx_pages->vmcs_gpa);
	42	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
	43	GUEST_ASSERT(load_vmcs(vmx_pages));
	44
	45	prepare_vmcs(vmx_pages, l2_guest_code,
	46	&l2_guest_stack[L2_GUEST_STACK_SIZE]);
	47
	48	/*
	49	* Set up L2 with a 4-level page table by pointing its CR3 to
	50	* L1's first PML4 table and clearing CR4.LA57. This creates
	51	* the CR4.LA57 mismatch that exercises the bug.
	52	*/
	53	pml5_pa = get_cr3() & PHYSICAL_PAGE_MASK;
	54	pml5 = (u64 *)pml5_pa;
	55	pml4_pa = pml5[0] & PHYSICAL_PAGE_MASK;
	56	vmwrite(GUEST_CR3, pml4_pa);
	57
	58	guest_cr4 = vmreadz(GUEST_CR4);
	59	guest_cr4 &= ~X86_CR4_LA57;
	60	vmwrite(GUEST_CR4, guest_cr4);
	61
	62	GUEST_ASSERT(!vmlaunch());
	63
	64	exit_reason = vmreadz(VM_EXIT_REASON);
	65	GUEST_ASSERT(exit_reason == EXIT_REASON_VMCALL);
	66	}
	67
	68	void guest_code(struct vmx_pages *vmx_pages)
	69	{
	70	l1_guest_code(vmx_pages);
	71	GUEST_DONE();
	72	}
	73
	74	int main(int argc, char *argv[])
	75	{
	76	vm_vaddr_t vmx_pages_gva = 0;
	77	struct kvm_vm *vm;
	78	struct kvm_vcpu *vcpu;
	79	struct kvm_x86_state *state;
	80	struct ucall uc;
	81	int stage;
	82
	83	TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX));
	84	TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_LA57));
	85	TEST_REQUIRE(kvm_has_cap(KVM_CAP_NESTED_STATE));
	86
	87	vm = vm_create_with_one_vcpu(&vcpu, guest_code);
	88
	89	/*
	90	* L1 needs to read its own PML5 table to set up L2. Identity map
	91	* the PML5 table to facilitate this.
	92	*/
	93	virt_map(vm, vm->pgd, vm->pgd, 1);
	94
	95	vcpu_alloc_vmx(vm, &vmx_pages_gva);
	96	vcpu_args_set(vcpu, 1, vmx_pages_gva);
	97
	98	for (stage = 1;; stage++) {
	99	vcpu_run(vcpu);
	100	TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
	101
	102	switch (get_ucall(vcpu, &uc)) {
	103	case UCALL_ABORT:
	104	REPORT_GUEST_ASSERT(uc);
	105	/* NOT REACHED */
	106	case UCALL_SYNC:
	107	break;
	108	case UCALL_DONE:
	109	goto done;
	110	default:
	111	TEST_FAIL("Unknown ucall %lu", uc.cmd);
	112	}
	113
	114	TEST_ASSERT(uc.args[1] == stage,
	115	"Expected stage %d, got stage %lu", stage, (ulong)uc.args[1]);
	116	if (stage == 1) {
	117	pr_info("L2 is active; performing save/restore.\n");
	118	state = vcpu_save_state(vcpu);
	119
	120	kvm_vm_release(vm);
	121
	122	/* Restore state in a new VM. */
	123	vcpu = vm_recreate_with_one_vcpu(vm);
	124	vcpu_load_state(vcpu, state);
	125	kvm_x86_state_cleanup(state);
	126	}
	127	}
	128
	129	done:
	130	kvm_vm_free(vm);
	131	return 0;
	132	}

tools/testing/selftests/kvm/x86/xapic_ipi_test.c+2−3

 	int nodes = 0;
 	time_t start_time, last_update, now;
 	time_t interval_secs = 1;
-	int i, r;
+	int i;
 	int from, to;
 	unsigned long bit;
 	uint64_t hlt_count;
 		delay_usecs);
 	/* Get set of first 64 numa nodes available */
-	r = get_mempolicy(NULL, &nodemask, sizeof(nodemask) * 8,
+	kvm_get_mempolicy(NULL, &nodemask, sizeof(nodemask) * 8,
 			  0, MPOL_F_MEMS_ALLOWED);
-	TEST_ASSERT(r == 0, "get_mempolicy failed errno=%d", errno);
 	fprintf(stderr, "Numa nodes found amongst first %lu possible nodes "
 		"(each 1-bit indicates node is present): %#lx\n",

virt/kvm/Kconfig+0−3

@@ -78,9 +78,6 @@ config HAVE_KVM_IRQ_BYPASS · 77 unmodified lines
78	tristate	78	tristate
79	select IRQ_BYPASS_MANAGER	79	select IRQ_BYPASS_MANAGER
80		80
81	config HAVE_KVM_VCPU_ASYNC_IOCTL
82	bool
83
84	config HAVE_KVM_VCPU_RUN_PID_CHANGE	81	config HAVE_KVM_VCPU_RUN_PID_CHANGE
85	bool	82	bool
86		83

virt/kvm/eventfd.c+1−1

@@ -707,7 +707,7 @@ bool kvm_notify_irqfd_resampler(struct kvm *kvm, · 706 unmodified lines
707	*/	707	*/
708	int kvm_irqfd_init(void)	708	int kvm_irqfd_init(void)
709	{	709	{
710	irqfd_cleanup_wq = alloc_workqueue("kvm-irqfd-cleanup", 0, 0);	710	irqfd_cleanup_wq = alloc_workqueue("kvm-irqfd-cleanup", WQ_PERCPU, 0);
711	if (!irqfd_cleanup_wq)	711	if (!irqfd_cleanup_wq)
712	return -ENOMEM;	712	return -ENOMEM;
713		713

virt/kvm/guest_memfd.c+279−94

@@ -1,18 +1,47 @@
1	// SPDX-License-Identifier: GPL-2.0	1	// SPDX-License-Identifier: GPL-2.0
		2	#include <linux/anon_inodes.h>
2	#include <linux/backing-dev.h>	3	#include <linux/backing-dev.h>
3	#include <linux/falloc.h>	4	#include <linux/falloc.h>
		5	#include <linux/fs.h>
4	#include <linux/kvm_host.h>	6	#include <linux/kvm_host.h>
		7	#include <linux/mempolicy.h>
		8	#include <linux/pseudo_fs.h>
5	#include <linux/pagemap.h>	9	#include <linux/pagemap.h>
6	#include <linux/anon_inodes.h>
7		10
8	#include "kvm_mm.h"	11	#include "kvm_mm.h"
9		12
10	struct kvm_gmem {	13	static struct vfsmount *kvm_gmem_mnt;
		14
		15	/*
		16	* A guest_memfd instance can be associated multiple VMs, each with its own
		17	* "view" of the underlying physical memory.
		18	*
		19	* The gmem's inode is effectively the raw underlying physical storage, and is
		20	* used to track properties of the physical memory, while each gmem file is
		21	* effectively a single VM's view of that storage, and is used to track assets
		22	* specific to its associated VM, e.g. memslots=>gmem bindings.
		23	*/
		24	struct gmem_file {
11	struct kvm *kvm;	25	struct kvm *kvm;
12	struct xarray bindings;	26	struct xarray bindings;
13	struct list_head entry;	27	struct list_head entry;
14	};	28	};
15		29
		30	struct gmem_inode {
		31	struct shared_policy policy;
		32	struct inode vfs_inode;
		33
		34	u64 flags;
		35	};
		36
		37	static __always_inline struct gmem_inode GMEM_I(struct inode inode)
		38	{
		39	return container_of(inode, struct gmem_inode, vfs_inode);
		40	}
		41
		42	#define kvm_gmem_for_each_file(f, mapping) \
		43	list_for_each_entry(f, &(mapping)->i_private_list, entry)
		44
16	/**	45	/**
17	* folio_file_pfn - like folio_file_page, but return a pfn.	46	* folio_file_pfn - like folio_file_page, but return a pfn.
18	* @folio: The folio which contains this index.	47	* @folio: The folio which contains this index.
@@ -25,6 +54,11 @@ static inline kvm_pfn_t folio_file_pfn(struct folio *folio, pgoff_t index) · 6 unmodified lines
25	return folio_pfn(folio) + (index & (folio_nr_pages(folio) - 1));	54	return folio_pfn(folio) + (index & (folio_nr_pages(folio) - 1));
26	}	55	}
27		56
		57	static pgoff_t kvm_gmem_get_index(struct kvm_memory_slot *slot, gfn_t gfn)
		58	{
		59	return gfn - slot->base_gfn + slot->gmem.pgoff;
		60	}
		61
28	static int __kvm_gmem_prepare_folio(struct kvm kvm, struct kvm_memory_slot slot,	62	static int __kvm_gmem_prepare_folio(struct kvm kvm, struct kvm_memory_slot slot,
29	pgoff_t index, struct folio *folio)	63	pgoff_t index, struct folio *folio)
30	{	64	{
@@ -77,9 +111,9 @@ static int kvm_gmem_prepare_folio(struct kvm kvm, struct kvm_memory_slot slot, · 46 unmodified lines
77	* The order will be passed when creating the guest_memfd, and	111	* The order will be passed when creating the guest_memfd, and
78	* checked when creating memslots.	112	* checked when creating memslots.
79	*/	113	*/
80	WARN_ON(!IS_ALIGNED(slot->gmem.pgoff, 1 << folio_order(folio)));	114	WARN_ON(!IS_ALIGNED(slot->gmem.pgoff, folio_nr_pages(folio)));
81	index = gfn - slot->base_gfn + slot->gmem.pgoff;	115	index = kvm_gmem_get_index(slot, gfn);
82	index = ALIGN_DOWN(index, 1 << folio_order(folio));	116	index = ALIGN_DOWN(index, folio_nr_pages(folio));
83	r = __kvm_gmem_prepare_folio(kvm, slot, index, folio);	117	r = __kvm_gmem_prepare_folio(kvm, slot, index, folio);
84	if (!r)	118	if (!r)
85	kvm_gmem_mark_prepared(folio);	119	kvm_gmem_mark_prepared(folio);
@@ -99,27 +133,45 @@ static int kvm_gmem_prepare_folio(struct kvm kvm, struct kvm_memory_slot slot, · 13 unmodified lines
99	static struct folio kvm_gmem_get_folio(struct inode inode, pgoff_t index)	133	static struct folio kvm_gmem_get_folio(struct inode inode, pgoff_t index)
100	{	134	{
101	/* TODO: Support huge pages. */	135	/* TODO: Support huge pages. */
102	return filemap_grab_folio(inode->i_mapping, index);	136	struct mempolicy *policy;
		137	struct folio *folio;
		138
		139	/*
		140	* Fast-path: See if folio is already present in mapping to avoid
		141	* policy_lookup.
		142	*/
		143	folio = __filemap_get_folio(inode->i_mapping, index,
		144	FGP_LOCK \| FGP_ACCESSED, 0);
		145	if (!IS_ERR(folio))
		146	return folio;
		147
		148	policy = mpol_shared_policy_lookup(&GMEM_I(inode)->policy, index);
		149	folio = __filemap_get_folio_mpol(inode->i_mapping, index,
		150	FGP_LOCK \| FGP_ACCESSED \| FGP_CREAT,
		151	mapping_gfp_mask(inode->i_mapping), policy);
		152	mpol_cond_put(policy);
		153
		154	return folio;
103	}	155	}
104		156
105	static enum kvm_gfn_range_filter kvm_gmem_get_invalidate_filter(struct inode *inode)	157	static enum kvm_gfn_range_filter kvm_gmem_get_invalidate_filter(struct inode *inode)
106	{	158	{
107	if ((u64)inode->i_private & GUEST_MEMFD_FLAG_INIT_SHARED)	159	if (GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED)
108	return KVM_FILTER_SHARED;	160	return KVM_FILTER_SHARED;
109		161
110	return KVM_FILTER_PRIVATE;	162	return KVM_FILTER_PRIVATE;
111	}	163	}
112		164
113	static void __kvm_gmem_invalidate_begin(struct kvm_gmem *gmem, pgoff_t start,	165	static void __kvm_gmem_invalidate_begin(struct gmem_file *f, pgoff_t start,
114	pgoff_t end,	166	pgoff_t end,
115	enum kvm_gfn_range_filter attr_filter)	167	enum kvm_gfn_range_filter attr_filter)
116	{	168	{
117	bool flush = false, found_memslot = false;	169	bool flush = false, found_memslot = false;
118	struct kvm_memory_slot *slot;	170	struct kvm_memory_slot *slot;
119	struct kvm *kvm = gmem->kvm;	171	struct kvm *kvm = f->kvm;
120	unsigned long index;	172	unsigned long index;
121		173
122	xa_for_each_range(&gmem->bindings, index, slot, start, end - 1) {	174	xa_for_each_range(&f->bindings, index, slot, start, end - 1) {
123	pgoff_t pgoff = slot->gmem.pgoff;	175	pgoff_t pgoff = slot->gmem.pgoff;
124		176
125	struct kvm_gfn_range gfn_range = {	177	struct kvm_gfn_range gfn_range = {
@@ -150,22 +202,21 @@ static void __kvm_gmem_invalidate_begin(struct kvm_gmem *gmem, pgoff_t start, · 24 unmodified lines
150	static void kvm_gmem_invalidate_begin(struct inode *inode, pgoff_t start,	202	static void kvm_gmem_invalidate_begin(struct inode *inode, pgoff_t start,
151	pgoff_t end)	203	pgoff_t end)
152	{	204	{
153	struct list_head *gmem_list = &inode->i_mapping->i_private_list;
154	enum kvm_gfn_range_filter attr_filter;	205	enum kvm_gfn_range_filter attr_filter;
155	struct kvm_gmem *gmem;	206	struct gmem_file *f;
156		207
157	attr_filter = kvm_gmem_get_invalidate_filter(inode);	208	attr_filter = kvm_gmem_get_invalidate_filter(inode);
158		209
159	list_for_each_entry(gmem, gmem_list, entry)	210	kvm_gmem_for_each_file(f, inode->i_mapping)
160	__kvm_gmem_invalidate_begin(gmem, start, end, attr_filter);	211	__kvm_gmem_invalidate_begin(f, start, end, attr_filter);
161	}	212	}
162		213
163	static void __kvm_gmem_invalidate_end(struct kvm_gmem *gmem, pgoff_t start,	214	static void __kvm_gmem_invalidate_end(struct gmem_file *f, pgoff_t start,
164	pgoff_t end)	215	pgoff_t end)
165	{	216	{
166	struct kvm *kvm = gmem->kvm;	217	struct kvm *kvm = f->kvm;
167		218
168	if (xa_find(&gmem->bindings, &start, end - 1, XA_PRESENT)) {	219	if (xa_find(&f->bindings, &start, end - 1, XA_PRESENT)) {
169	KVM_MMU_LOCK(kvm);	220	KVM_MMU_LOCK(kvm);
170	kvm_mmu_invalidate_end(kvm);	221	kvm_mmu_invalidate_end(kvm);
171	KVM_MMU_UNLOCK(kvm);	222	KVM_MMU_UNLOCK(kvm);
@@ -175,11 +226,10 @@ static void __kvm_gmem_invalidate_end(struct kvm_gmem *gmem, pgoff_t start, · 3 unmodified lines
175	static void kvm_gmem_invalidate_end(struct inode *inode, pgoff_t start,	226	static void kvm_gmem_invalidate_end(struct inode *inode, pgoff_t start,
176	pgoff_t end)	227	pgoff_t end)
177	{	228	{
178	struct list_head *gmem_list = &inode->i_mapping->i_private_list;	229	struct gmem_file *f;
179	struct kvm_gmem *gmem;
180		230
181	list_for_each_entry(gmem, gmem_list, entry)	231	kvm_gmem_for_each_file(f, inode->i_mapping)
182	__kvm_gmem_invalidate_end(gmem, start, end);	232	__kvm_gmem_invalidate_end(f, start, end);
183	}	233	}
184		234
185	static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len)	235	static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len)
@@ -277,9 +327,9 @@ static long kvm_gmem_fallocate(struct file *file, int mode, loff_t offset, · 91 unmodified lines
277		327
278	static int kvm_gmem_release(struct inode inode, struct file file)	328	static int kvm_gmem_release(struct inode inode, struct file file)
279	{	329	{
280	struct kvm_gmem *gmem = file->private_data;	330	struct gmem_file *f = file->private_data;
281	struct kvm_memory_slot *slot;	331	struct kvm_memory_slot *slot;
282	struct kvm *kvm = gmem->kvm;	332	struct kvm *kvm = f->kvm;
283	unsigned long index;	333	unsigned long index;
284		334
285	/*	335	/*
@@ -299,7 +349,7 @@ static int kvm_gmem_release(struct inode inode, struct file file) · 13 unmodified lines
299		349
300	filemap_invalidate_lock(inode->i_mapping);	350	filemap_invalidate_lock(inode->i_mapping);
301		351
302	xa_for_each(&gmem->bindings, index, slot)	352	xa_for_each(&f->bindings, index, slot)
303	WRITE_ONCE(slot->gmem.file, NULL);	353	WRITE_ONCE(slot->gmem.file, NULL);
304		354
305	/*	355	/*
@@ -307,18 +357,18 @@ static int kvm_gmem_release(struct inode inode, struct file file) · 1 unmodified line
307	* Zap all SPTEs pointed at by this file. Do not free the backing	357	* Zap all SPTEs pointed at by this file. Do not free the backing
308	* memory, as its lifetime is associated with the inode, not the file.	358	* memory, as its lifetime is associated with the inode, not the file.
309	*/	359	*/
310	__kvm_gmem_invalidate_begin(gmem, 0, -1ul,	360	__kvm_gmem_invalidate_begin(f, 0, -1ul,
311	kvm_gmem_get_invalidate_filter(inode));	361	kvm_gmem_get_invalidate_filter(inode));
312	__kvm_gmem_invalidate_end(gmem, 0, -1ul);	362	__kvm_gmem_invalidate_end(f, 0, -1ul);
313		363
314	list_del(&gmem->entry);	364	list_del(&f->entry);
315		365
316	filemap_invalidate_unlock(inode->i_mapping);	366	filemap_invalidate_unlock(inode->i_mapping);
317		367
318	mutex_unlock(&kvm->slots_lock);	368	mutex_unlock(&kvm->slots_lock);
319		369
320	xa_destroy(&gmem->bindings);	370	xa_destroy(&f->bindings);
321	kfree(gmem);	371	kfree(f);
322		372
323	kvm_put_kvm(kvm);	373	kvm_put_kvm(kvm);
324		374
@@ -335,16 +385,12 @@ static inline struct file kvm_gmem_get_file(struct kvm_memory_slot slot) · 10 unmodified lines
335	return get_file_active(&slot->gmem.file);	385	return get_file_active(&slot->gmem.file);
336	}	386	}
337		387
338	static pgoff_t kvm_gmem_get_index(struct kvm_memory_slot *slot, gfn_t gfn)	388	DEFINE_CLASS(gmem_get_file, struct file *, if (_T) fput(_T),
339	{	389	kvm_gmem_get_file(slot), struct kvm_memory_slot *slot);
340	return gfn - slot->base_gfn + slot->gmem.pgoff;
341	}
342		390
343	static bool kvm_gmem_supports_mmap(struct inode *inode)	391	static bool kvm_gmem_supports_mmap(struct inode *inode)
344	{	392	{
345	const u64 flags = (u64)inode->i_private;	393	return GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_MMAP;
346
347	return flags & GUEST_MEMFD_FLAG_MMAP;
348	}	394	}
349		395
350	static vm_fault_t kvm_gmem_fault_user_mapping(struct vm_fault *vmf)	396	static vm_fault_t kvm_gmem_fault_user_mapping(struct vm_fault *vmf)
@@ -356,17 +402,15 @@ static vm_fault_t kvm_gmem_fault_user_mapping(struct vm_fault *vmf) · 5 unmodified lines
356	if (((loff_t)vmf->pgoff << PAGE_SHIFT) >= i_size_read(inode))	402	if (((loff_t)vmf->pgoff << PAGE_SHIFT) >= i_size_read(inode))
357	return VM_FAULT_SIGBUS;	403	return VM_FAULT_SIGBUS;
358		404
359	if (!((u64)inode->i_private & GUEST_MEMFD_FLAG_INIT_SHARED))	405	if (!(GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED))
360	return VM_FAULT_SIGBUS;	406	return VM_FAULT_SIGBUS;
361		407
362	folio = kvm_gmem_get_folio(inode, vmf->pgoff);	408	folio = kvm_gmem_get_folio(inode, vmf->pgoff);
363	if (IS_ERR(folio)) {	409	if (IS_ERR(folio)) {
364	int err = PTR_ERR(folio);	410	if (PTR_ERR(folio) == -EAGAIN)
365
366	if (err == -EAGAIN)
367	return VM_FAULT_RETRY;	411	return VM_FAULT_RETRY;
368		412
369	return vmf_error(err);	413	return vmf_error(PTR_ERR(folio));
370	}	414	}
371		415
372	if (WARN_ON_ONCE(folio_test_large(folio))) {	416	if (WARN_ON_ONCE(folio_test_large(folio))) {
@@ -390,8 +434,40 @@ out_folio: · 17 unmodified lines
390	return ret;	434	return ret;
391	}	435	}
392		436
		437	#ifdef CONFIG_NUMA
		438	static int kvm_gmem_set_policy(struct vm_area_struct vma, struct mempolicy mpol)
		439	{
		440	struct inode *inode = file_inode(vma->vm_file);
		441
		442	return mpol_set_shared_policy(&GMEM_I(inode)->policy, vma, mpol);
		443	}
		444
		445	static struct mempolicy kvm_gmem_get_policy(struct vm_area_struct vma,
		446	unsigned long addr, pgoff_t *pgoff)
		447	{
		448	struct inode *inode = file_inode(vma->vm_file);
		449
		450	*pgoff = vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT);
		451
		452	/*
		453	* Return the memory policy for this index, or NULL if none is set.
		454	*
		455	* Returning NULL, e.g. instead of the current task's memory policy, is
		456	* important for the .get_policy kernel ABI: it indicates that no
		457	* explicit policy has been set via mbind() for this memory. The caller
		458	* can then replace NULL with the default memory policy instead of the
		459	* current task's memory policy.
		460	*/
		461	return mpol_shared_policy_lookup(&GMEM_I(inode)->policy, *pgoff);
		462	}
		463	#endif /* CONFIG_NUMA */
		464
393	static const struct vm_operations_struct kvm_gmem_vm_ops = {	465	static const struct vm_operations_struct kvm_gmem_vm_ops = {
394	.fault = kvm_gmem_fault_user_mapping,	466	.fault = kvm_gmem_fault_user_mapping,
		467	#ifdef CONFIG_NUMA
		468	.get_policy = kvm_gmem_get_policy,
		469	.set_policy = kvm_gmem_set_policy,
		470	#endif
395	};	471	};
396		472
397	static int kvm_gmem_mmap(struct file file, struct vm_area_struct vma)	473	static int kvm_gmem_mmap(struct file file, struct vm_area_struct vma)
@@ -416,11 +492,6 @@ static struct file_operations kvm_gmem_fops = { · 18 unmodified lines
416	.fallocate = kvm_gmem_fallocate,	492	.fallocate = kvm_gmem_fallocate,
417	};	493	};
418		494
419	void kvm_gmem_init(struct module *module)
420	{
421	kvm_gmem_fops.owner = module;
422	}
423
424	static int kvm_gmem_migrate_folio(struct address_space *mapping,	495	static int kvm_gmem_migrate_folio(struct address_space *mapping,
425	struct folio dst, struct folio src,	496	struct folio dst, struct folio src,
426	enum migrate_mode mode)	497	enum migrate_mode mode)
@@ -492,8 +563,8 @@ bool __weak kvm_arch_supports_gmem_init_shared(struct kvm *kvm) · 65 unmodified lines
492		563
493	static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)	564	static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
494	{	565	{
495	const char *anon_name = "[kvm-gmem]";	566	static const char *name = "[kvm-gmem]";
496	struct kvm_gmem *gmem;	567	struct gmem_file *f;
497	struct inode *inode;	568	struct inode *inode;
498	struct file *file;	569	struct file *file;
499	int fd, err;	570	int fd, err;
@@ -502,25 +573,24 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) · 2 unmodified lines
502	if (fd < 0)	573	if (fd < 0)
503	return fd;	574	return fd;
504		575
505	gmem = kzalloc(sizeof(*gmem), GFP_KERNEL);	576	f = kzalloc(sizeof(*f), GFP_KERNEL);
506	if (!gmem) {	577	if (!f) {
507	err = -ENOMEM;	578	err = -ENOMEM;
508	goto err_fd;	579	goto err_fd;
509	}	580	}
510		581
511	file = anon_inode_create_getfile(anon_name, &kvm_gmem_fops, gmem,	582	/* __fput() will take care of fops_put(). */
512	O_RDWR, NULL);	583	if (!fops_get(&kvm_gmem_fops)) {
513	if (IS_ERR(file)) {	584	err = -ENOENT;
514	err = PTR_ERR(file);
515	goto err_gmem;	585	goto err_gmem;
516	}	586	}
517		587
518	file->f_flags \|= O_LARGEFILE;	588	inode = anon_inode_make_secure_inode(kvm_gmem_mnt->mnt_sb, name, NULL);
519		589	if (IS_ERR(inode)) {
520	inode = file->f_inode;	590	err = PTR_ERR(inode);
521	WARN_ON(file->f_mapping != inode->i_mapping);	591	goto err_fops;
		592	}
522		593
523	inode->i_private = (void *)(unsigned long)flags;
524	inode->i_op = &kvm_gmem_iops;	594	inode->i_op = &kvm_gmem_iops;
525	inode->i_mapping->a_ops = &kvm_gmem_aops;	595	inode->i_mapping->a_ops = &kvm_gmem_aops;
526	inode->i_mode \|= S_IFREG;	596	inode->i_mode \|= S_IFREG;
@@ -530,16 +600,31 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) · 3 unmodified lines
530	/* Unmovable mappings are supposed to be marked unevictable as well. */	600	/* Unmovable mappings are supposed to be marked unevictable as well. */
531	WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping));	601	WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping));
532		602
		603	GMEM_I(inode)->flags = flags;
		604
		605	file = alloc_file_pseudo(inode, kvm_gmem_mnt, name, O_RDWR, &kvm_gmem_fops);
		606	if (IS_ERR(file)) {
		607	err = PTR_ERR(file);
		608	goto err_inode;
		609	}
		610
		611	file->f_flags \|= O_LARGEFILE;
		612	file->private_data = f;
		613
533	kvm_get_kvm(kvm);	614	kvm_get_kvm(kvm);
534	gmem->kvm = kvm;	615	f->kvm = kvm;
535	xa_init(&gmem->bindings);	616	xa_init(&f->bindings);
536	list_add(&gmem->entry, &inode->i_mapping->i_private_list);	617	list_add(&f->entry, &inode->i_mapping->i_private_list);
537		618
538	fd_install(fd, file);	619	fd_install(fd, file);
539	return fd;	620	return fd;
540		621
		622	err_inode:
		623	iput(inode);
		624	err_fops:
		625	fops_put(&kvm_gmem_fops);
541	err_gmem:	626	err_gmem:
542	kfree(gmem);	627	kfree(f);
543	err_fd:	628	err_fd:
544	put_unused_fd(fd);	629	put_unused_fd(fd);
545	return err;	630	return err;
@@ -564,7 +649,7 @@ int kvm_gmem_bind(struct kvm kvm, struct kvm_memory_slot slot, · 18 unmodified lines
564	{	649	{
565	loff_t size = slot->npages << PAGE_SHIFT;	650	loff_t size = slot->npages << PAGE_SHIFT;
566	unsigned long start, end;	651	unsigned long start, end;
567	struct kvm_gmem *gmem;	652	struct gmem_file *f;
568	struct inode *inode;	653	struct inode *inode;
569	struct file *file;	654	struct file *file;
570	int r = -EINVAL;	655	int r = -EINVAL;
@@ -578,8 +663,8 @@ int kvm_gmem_bind(struct kvm kvm, struct kvm_memory_slot slot, · 7 unmodified lines
578	if (file->f_op != &kvm_gmem_fops)	663	if (file->f_op != &kvm_gmem_fops)
579	goto err;	664	goto err;
580		665
581	gmem = file->private_data;	666	f = file->private_data;
582	if (gmem->kvm != kvm)	667	if (f->kvm != kvm)
583	goto err;	668	goto err;
584		669
585	inode = file_inode(file);	670	inode = file_inode(file);
@@ -593,8 +678,8 @@ int kvm_gmem_bind(struct kvm kvm, struct kvm_memory_slot slot, · 7 unmodified lines
593	start = offset >> PAGE_SHIFT;	678	start = offset >> PAGE_SHIFT;
594	end = start + slot->npages;	679	end = start + slot->npages;
595		680
596	if (!xa_empty(&gmem->bindings) &&	681	if (!xa_empty(&f->bindings) &&
597	xa_find(&gmem->bindings, &start, end - 1, XA_PRESENT)) {	682	xa_find(&f->bindings, &start, end - 1, XA_PRESENT)) {
598	filemap_invalidate_unlock(inode->i_mapping);	683	filemap_invalidate_unlock(inode->i_mapping);
599	goto err;	684	goto err;
600	}	685	}
@@ -609,7 +694,7 @@ int kvm_gmem_bind(struct kvm kvm, struct kvm_memory_slot slot, · 8 unmodified lines
609	if (kvm_gmem_supports_mmap(inode))	694	if (kvm_gmem_supports_mmap(inode))
610	slot->flags \|= KVM_MEMSLOT_GMEM_ONLY;	695	slot->flags \|= KVM_MEMSLOT_GMEM_ONLY;
611		696
612	xa_store_range(&gmem->bindings, start, end - 1, slot, GFP_KERNEL);	697	xa_store_range(&f->bindings, start, end - 1, slot, GFP_KERNEL);
613	filemap_invalidate_unlock(inode->i_mapping);	698	filemap_invalidate_unlock(inode->i_mapping);
614		699
615	/*	700	/*
@@ -623,12 +708,12 @@ err: · 7 unmodified lines
623	return r;	708	return r;
624	}	709	}
625		710
626	static void __kvm_gmem_unbind(struct kvm_memory_slot slot, struct kvm_gmem gmem)	711	static void __kvm_gmem_unbind(struct kvm_memory_slot slot, struct gmem_file f)
627	{	712	{
628	unsigned long start = slot->gmem.pgoff;	713	unsigned long start = slot->gmem.pgoff;
629	unsigned long end = start + slot->npages;	714	unsigned long end = start + slot->npages;
630		715
631	xa_store_range(&gmem->bindings, start, end - 1, NULL, GFP_KERNEL);	716	xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL);
632		717
633	/*	718	/*
634	* synchronize_srcu(&kvm->srcu) ensured that kvm_gmem_get_pfn()	719	* synchronize_srcu(&kvm->srcu) ensured that kvm_gmem_get_pfn()
@@ -639,8 +724,6 @@ static void __kvm_gmem_unbind(struct kvm_memory_slot slot, struct kvm_gmem gme · 4 unmodified lines
639		724
640	void kvm_gmem_unbind(struct kvm_memory_slot *slot)	725	void kvm_gmem_unbind(struct kvm_memory_slot *slot)
641	{	726	{
642	struct file *file;
643
644	/*	727	/*
645	* Nothing to do if the underlying file was _already_ closed, as	728	* Nothing to do if the underlying file was _already_ closed, as
646	* kvm_gmem_release() invalidates and nullifies all bindings.	729	* kvm_gmem_release() invalidates and nullifies all bindings.
@@ -648,7 +731,7 @@ void kvm_gmem_unbind(struct kvm_memory_slot *slot) · 1 unmodified line
648	if (!slot->gmem.file)	731	if (!slot->gmem.file)
649	return;	732	return;
650		733
651	file = kvm_gmem_get_file(slot);	734	CLASS(gmem_get_file, file)(slot);
652		735
653	/*	736	/*
654	* However, if the file is _being_ closed, then the bindings need to be	737	* However, if the file is _being_ closed, then the bindings need to be
@@ -668,8 +751,6 @@ void kvm_gmem_unbind(struct kvm_memory_slot *slot) · 13 unmodified lines
668	filemap_invalidate_lock(file->f_mapping);	751	filemap_invalidate_lock(file->f_mapping);
669	__kvm_gmem_unbind(slot, file->private_data);	752	__kvm_gmem_unbind(slot, file->private_data);
670	filemap_invalidate_unlock(file->f_mapping);	753	filemap_invalidate_unlock(file->f_mapping);
671
672	fput(file);
673	}	754	}
674		755
675	/* Returns a locked folio on success. */	756	/* Returns a locked folio on success. */
@@ -678,18 +759,17 @@ static struct folio __kvm_gmem_get_pfn(struct file file, · 2 unmodified lines
678	pgoff_t index, kvm_pfn_t *pfn,	759	pgoff_t index, kvm_pfn_t *pfn,
679	bool is_prepared, int max_order)	760	bool is_prepared, int max_order)
680	{	761	{
681	struct file *gmem_file = READ_ONCE(slot->gmem.file);	762	struct file *slot_file = READ_ONCE(slot->gmem.file);
682	struct kvm_gmem *gmem = file->private_data;	763	struct gmem_file *f = file->private_data;
683	struct folio *folio;	764	struct folio *folio;
684		765
685	if (file != gmem_file) {	766	if (file != slot_file) {
686	WARN_ON_ONCE(gmem_file);	767	WARN_ON_ONCE(slot_file);
687	return ERR_PTR(-EFAULT);	768	return ERR_PTR(-EFAULT);
688	}	769	}
689		770
690	gmem = file->private_data;	771	if (xa_load(&f->bindings, index) != slot) {
691	if (xa_load(&gmem->bindings, index) != slot) {	772	WARN_ON_ONCE(xa_load(&f->bindings, index));
692	WARN_ON_ONCE(xa_load(&gmem->bindings, index));
693	return ERR_PTR(-EIO);	773	return ERR_PTR(-EIO);
694	}	774	}
695		775
@@ -716,19 +796,17 @@ int kvm_gmem_get_pfn(struct kvm kvm, struct kvm_memory_slot slot, · 20 unmodified lines
716	int *max_order)	796	int *max_order)
717	{	797	{
718	pgoff_t index = kvm_gmem_get_index(slot, gfn);	798	pgoff_t index = kvm_gmem_get_index(slot, gfn);
719	struct file *file = kvm_gmem_get_file(slot);
720	struct folio *folio;	799	struct folio *folio;
721	bool is_prepared = false;	800	bool is_prepared = false;
722	int r = 0;	801	int r = 0;
723		802
		803	CLASS(gmem_get_file, file)(slot);
724	if (!file)	804	if (!file)
725	return -EFAULT;	805	return -EFAULT;
726		806
727	folio = __kvm_gmem_get_pfn(file, slot, index, pfn, &is_prepared, max_order);	807	folio = __kvm_gmem_get_pfn(file, slot, index, pfn, &is_prepared, max_order);
728	if (IS_ERR(folio)) {	808	if (IS_ERR(folio))
729	r = PTR_ERR(folio);	809	return PTR_ERR(folio);
730	goto out;
731	}
732		810
733	if (!is_prepared)	811	if (!is_prepared)
734	r = kvm_gmem_prepare_folio(kvm, slot, gfn, folio);	812	r = kvm_gmem_prepare_folio(kvm, slot, gfn, folio);
@@ -740,8 +818,6 @@ int kvm_gmem_get_pfn(struct kvm kvm, struct kvm_memory_slot slot, · 5 unmodified lines
740	else	818	else
741	folio_put(folio);	819	folio_put(folio);
742		820
743	out:
744	fput(file);
745	return r;	821	return r;
746	}	822	}
747	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_get_pfn);	823	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_get_pfn);
@@ -750,7 +826,6 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_get_pfn); · 2 unmodified lines
750	long kvm_gmem_populate(struct kvm kvm, gfn_t start_gfn, void __user src, long npages,	826	long kvm_gmem_populate(struct kvm kvm, gfn_t start_gfn, void __user src, long npages,
751	kvm_gmem_populate_cb post_populate, void *opaque)	827	kvm_gmem_populate_cb post_populate, void *opaque)
752	{	828	{
753	struct file *file;
754	struct kvm_memory_slot *slot;	829	struct kvm_memory_slot *slot;
755	void __user *p;	830	void __user *p;
756		831
@@ -766,7 +841,7 @@ long kvm_gmem_populate(struct kvm kvm, gfn_t start_gfn, void __user src, long · 9 unmodified lines
766	if (!kvm_slot_has_gmem(slot))	841	if (!kvm_slot_has_gmem(slot))
767	return -EINVAL;	842	return -EINVAL;
768		843
769	file = kvm_gmem_get_file(slot);	844	CLASS(gmem_get_file, file)(slot);
770	if (!file)	845	if (!file)
771	return -EFAULT;	846	return -EFAULT;
772		847
@@ -824,8 +899,118 @@ put_folio_and_exit: · 51 unmodified lines
824		899
825	filemap_invalidate_unlock(file->f_mapping);	900	filemap_invalidate_unlock(file->f_mapping);
826		901
827	fput(file);
828	return ret && !i ? ret : i;	902	return ret && !i ? ret : i;
829	}	903	}
830	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_populate);	904	EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_populate);
831	#endif	905	#endif
		906
		907	static struct kmem_cache *kvm_gmem_inode_cachep;
		908
		909	static void kvm_gmem_init_inode_once(void *__gi)
		910	{
		911	struct gmem_inode *gi = __gi;
		912
		913	/*
		914	* Note! Don't initialize the inode with anything specific to the
		915	* guest_memfd instance, or that might be specific to how the inode is
		916	* used (from the VFS-layer's perspective). This hook is called only
		917	* during the initial slab allocation, i.e. only fields/state that are
		918	* idempotent across _all_ use of the inode _object_ can be initialized
		919	* at this time!
		920	*/
		921	inode_init_once(&gi->vfs_inode);
		922	}
		923
		924	static struct inode kvm_gmem_alloc_inode(struct super_block sb)
		925	{
		926	struct gmem_inode *gi;
		927
		928	gi = alloc_inode_sb(sb, kvm_gmem_inode_cachep, GFP_KERNEL);
		929	if (!gi)
		930	return NULL;
		931
		932	mpol_shared_policy_init(&gi->policy, NULL);
		933
		934	gi->flags = 0;
		935	return &gi->vfs_inode;
		936	}
		937
		938	static void kvm_gmem_destroy_inode(struct inode *inode)
		939	{
		940	mpol_free_shared_policy(&GMEM_I(inode)->policy);
		941	}
		942
		943	static void kvm_gmem_free_inode(struct inode *inode)
		944	{
		945	kmem_cache_free(kvm_gmem_inode_cachep, GMEM_I(inode));
		946	}
		947
		948	static const struct super_operations kvm_gmem_super_operations = {
		949	.statfs = simple_statfs,
		950	.alloc_inode = kvm_gmem_alloc_inode,
		951	.destroy_inode = kvm_gmem_destroy_inode,
		952	.free_inode = kvm_gmem_free_inode,
		953	};
		954
		955	static int kvm_gmem_init_fs_context(struct fs_context *fc)
		956	{
		957	struct pseudo_fs_context *ctx;
		958
		959	if (!init_pseudo(fc, GUEST_MEMFD_MAGIC))
		960	return -ENOMEM;
		961
		962	fc->s_iflags \|= SB_I_NOEXEC;
		963	fc->s_iflags \|= SB_I_NODEV;
		964	ctx = fc->fs_private;
		965	ctx->ops = &kvm_gmem_super_operations;
		966
		967	return 0;
		968	}
		969
		970	static struct file_system_type kvm_gmem_fs = {
		971	.name = "guest_memfd",
		972	.init_fs_context = kvm_gmem_init_fs_context,
		973	.kill_sb = kill_anon_super,
		974	};
		975
		976	static int kvm_gmem_init_mount(void)
		977	{
		978	kvm_gmem_mnt = kern_mount(&kvm_gmem_fs);
		979
		980	if (IS_ERR(kvm_gmem_mnt))
		981	return PTR_ERR(kvm_gmem_mnt);
		982
		983	kvm_gmem_mnt->mnt_flags \|= MNT_NOEXEC;
		984	return 0;
		985	}
		986
		987	int kvm_gmem_init(struct module *module)
		988	{
		989	struct kmem_cache_args args = {
		990	.align = 0,
		991	.ctor = kvm_gmem_init_inode_once,
		992	};
		993	int ret;
		994
		995	kvm_gmem_fops.owner = module;
		996	kvm_gmem_inode_cachep = kmem_cache_create("kvm_gmem_inode_cache",
		997	sizeof(struct gmem_inode),
		998	&args, SLAB_ACCOUNT);
		999	if (!kvm_gmem_inode_cachep)
		1000	return -ENOMEM;
		1001
		1002	ret = kvm_gmem_init_mount();
		1003	if (ret) {
		1004	kmem_cache_destroy(kvm_gmem_inode_cachep);
		1005	return ret;
		1006	}
		1007	return 0;
		1008	}
		1009
		1010	void kvm_gmem_exit(void)
		1011	{
		1012	kern_unmount(kvm_gmem_mnt);
		1013	kvm_gmem_mnt = NULL;
		1014	rcu_barrier();
		1015	kmem_cache_destroy(kvm_gmem_inode_cachep);
		1016	}

virt/kvm/kvm_main.c+10−5

 		yielded = kvm_vcpu_yield_to(vcpu);
 		if (yielded > 0) {
-			WRITE_ONCE(kvm->last_boosted_vcpu, i);
+			WRITE_ONCE(kvm->last_boosted_vcpu, idx);
 			break;
 		} else if (yielded < 0 && !--try) {
 			break;
 		return r;
 	/*
-	 * Some architectures have vcpu ioctls that are asynchronous to vcpu
+	 * Let arch code handle select vCPU ioctls without holding vcpu->mutex,
-	 * execution; mutex_lock() would break them.
+	 * e.g. to support ioctls that can run asynchronous to vCPU execution.
 	 */
-	r = kvm_arch_vcpu_async_ioctl(filp, ioctl, arg);
+	r = kvm_arch_vcpu_unlocked_ioctl(filp, ioctl, arg);
 	if (r != -ENOIOCTLCMD)
 		return r;
 	if (WARN_ON_ONCE(r))
 		goto err_vfio;
-	kvm_gmem_init(module);
+	r = kvm_gmem_init(module);
+	if (r)
+		goto err_gmem;
 	r = kvm_init_virtualization();
 	if (r)
 err_register:
 	kvm_uninit_virtualization();
 err_virt:
+	kvm_gmem_exit();
+err_gmem:
 	kvm_vfio_ops_exit();
 err_vfio:
 	kvm_async_pf_deinit();
 	for_each_possible_cpu(cpu)
 		free_cpumask_var(per_cpu(cpu_kick_mask, cpu));
 	kmem_cache_destroy(kvm_vcpu_cache);
+	kvm_gmem_exit();
 	kvm_vfio_ops_exit();
 	kvm_async_pf_deinit();
 	kvm_irqfd_exit();

virt/kvm/kvm_mm.h+5−4

@@ -68,17 +68,18 @@ static inline void gfn_to_pfn_cache_invalidate_start(struct kvm *kvm, · 67 unmodified lines
68	#endif /* HAVE_KVM_PFNCACHE */	68	#endif /* HAVE_KVM_PFNCACHE */
69		69
70	#ifdef CONFIG_KVM_GUEST_MEMFD	70	#ifdef CONFIG_KVM_GUEST_MEMFD
71	void kvm_gmem_init(struct module *module);	71	int kvm_gmem_init(struct module *module);
		72	void kvm_gmem_exit(void);
72	int kvm_gmem_create(struct kvm kvm, struct kvm_create_guest_memfd args);	73	int kvm_gmem_create(struct kvm kvm, struct kvm_create_guest_memfd args);
73	int kvm_gmem_bind(struct kvm kvm, struct kvm_memory_slot slot,	74	int kvm_gmem_bind(struct kvm kvm, struct kvm_memory_slot slot,
74	unsigned int fd, loff_t offset);	75	unsigned int fd, loff_t offset);
75	void kvm_gmem_unbind(struct kvm_memory_slot *slot);	76	void kvm_gmem_unbind(struct kvm_memory_slot *slot);
76	#else	77	#else
77	static inline void kvm_gmem_init(struct module *module)	78	static inline int kvm_gmem_init(struct module *module)
78	{	79	{
79		80	return 0;
80	}	81	}
81		82	static inline void kvm_gmem_exit(void) {};
82	static inline int kvm_gmem_bind(struct kvm *kvm,	83	static inline int kvm_gmem_bind(struct kvm *kvm,
83	struct kvm_memory_slot *slot,	84	struct kvm_memory_slot *slot,
84	unsigned int fd, loff_t offset)	85	unsigned int fd, loff_t offset)

@@ -79,7 +79,7 @@ enum __kvm_host_smccc_func { · 78 unmodified lines
79	__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_range,	79	__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_range,
80	__KVM_HOST_SMCCC_FUNC___kvm_flush_cpu_context,	80	__KVM_HOST_SMCCC_FUNC___kvm_flush_cpu_context,
81	__KVM_HOST_SMCCC_FUNC___kvm_timer_set_cntvoff,	81	__KVM_HOST_SMCCC_FUNC___kvm_timer_set_cntvoff,
82	__KVM_HOST_SMCCC_FUNC___vgic_v3_save_vmcr_aprs,	82	__KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs,
83	__KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs,	83	__KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs,
84	__KVM_HOST_SMCCC_FUNC___pkvm_reserve_vm,	84	__KVM_HOST_SMCCC_FUNC___pkvm_reserve_vm,
85	__KVM_HOST_SMCCC_FUNC___pkvm_unreserve_vm,	85	__KVM_HOST_SMCCC_FUNC___pkvm_unreserve_vm,
@@ -246,9 +246,9 @@ extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu); · 160 unmodified lines
246	extern int __kvm_tlbi_s1e2(struct kvm_s2_mmu *mmu, u64 va, u64 sys_encoding);	246	extern int __kvm_tlbi_s1e2(struct kvm_s2_mmu *mmu, u64 va, u64 sys_encoding);
247		247
248	extern void __kvm_timer_set_cntvoff(u64 cntvoff);	248	extern void __kvm_timer_set_cntvoff(u64 cntvoff);
249	extern void __kvm_at_s1e01(struct kvm_vcpu *vcpu, u32 op, u64 vaddr);	249	extern int __kvm_at_s1e01(struct kvm_vcpu *vcpu, u32 op, u64 vaddr);
250	extern void __kvm_at_s1e2(struct kvm_vcpu *vcpu, u32 op, u64 vaddr);	250	extern int __kvm_at_s1e2(struct kvm_vcpu *vcpu, u32 op, u64 vaddr);
251	extern void __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr);	251	extern int __kvm_at_s12(struct kvm_vcpu *vcpu, u32 op, u64 vaddr);
252		252
253	extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);	253	extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
254		254