lael-labs/linux

Documentation/virt/kvm/amd-memory-encryption.rst+21−0

@@ -263,6 +263,27 @@ Returns: 0 on success, -negative on error · 262 unmodified lines
263	__u32 trans_len;	263	__u32 trans_len;
264	};	264	};
265		265
		266	10. KVM_SEV_GET_ATTESTATION_REPORT
		267	----------------------------------
		268
		269	The KVM_SEV_GET_ATTESTATION_REPORT command can be used by the hypervisor to query the attestation
		270	report containing the SHA-256 digest of the guest memory and VMSA passed through the KVM_SEV_LAUNCH
		271	commands and signed with the PEK. The digest returned by the command should match the digest
		272	used by the guest owner with the KVM_SEV_LAUNCH_MEASURE.
		273
		274	Parameters (in): struct kvm_sev_attestation
		275
		276	Returns: 0 on success, -negative on error
		277
		278	::
		279
		280	struct kvm_sev_attestation_report {
		281	__u8 mnonce[16]; /* A random mnonce that will be placed in the report */
		282
		283	__u64 uaddr; /* userspace address where the report should be copied */
		284	__u32 len;
		285	};
		286
266	References	287	References
267	==========	288	==========
268		289

Documentation/virt/kvm/api.rst+220−8

@@ -960,6 +960,14 @@ memory. · 959 unmodified lines
960	__u8 pad2[30];	960	__u8 pad2[30];
961	};	961	};
962		962
		963	If the KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL flag is returned from the
		964	KVM_CAP_XEN_HVM check, it may be set in the flags field of this ioctl.
		965	This requests KVM to generate the contents of the hypercall page
		966	automatically; hypercalls will be intercepted and passed to userspace
		967	through KVM_EXIT_XEN. In this case, all of the blob size and address
		968	fields must be zero.
		969
		970	No other flags are currently valid in the struct kvm_xen_hvm_config.
963		971
964	4.29 KVM_GET_CLOCK	972	4.29 KVM_GET_CLOCK
965	------------------	973	------------------
@@ -2268,6 +2276,8 @@ registers, find a list below: · 1302 unmodified lines
2268	PPC KVM_REG_PPC_PSSCR 64	2276	PPC KVM_REG_PPC_PSSCR 64
2269	PPC KVM_REG_PPC_DEC_EXPIRY 64	2277	PPC KVM_REG_PPC_DEC_EXPIRY 64
2270	PPC KVM_REG_PPC_PTCR 64	2278	PPC KVM_REG_PPC_PTCR 64
		2279	PPC KVM_REG_PPC_DAWR1 64
		2280	PPC KVM_REG_PPC_DAWRX1 64
2271	PPC KVM_REG_PPC_TM_GPR0 64	2281	PPC KVM_REG_PPC_TM_GPR0 64
2272	...	2282	...
2273	PPC KVM_REG_PPC_TM_GPR31 64	2283	PPC KVM_REG_PPC_TM_GPR31 64
@@ -4831,6 +4841,101 @@ into user space. · 2557 unmodified lines
4831	If a vCPU is in running state while this ioctl is invoked, the vCPU may	4841	If a vCPU is in running state while this ioctl is invoked, the vCPU may
4832	experience inconsistent filtering behavior on MSR accesses.	4842	experience inconsistent filtering behavior on MSR accesses.
4833		4843
		4844	4.127 KVM_XEN_HVM_SET_ATTR
		4845	--------------------------
		4846
		4847	:Capability: KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_SHARED_INFO
		4848	:Architectures: x86
		4849	:Type: vm ioctl
		4850	:Parameters: struct kvm_xen_hvm_attr
		4851	:Returns: 0 on success, < 0 on error
		4852
		4853	::
		4854
		4855	struct kvm_xen_hvm_attr {
		4856	__u16 type;
		4857	__u16 pad[3];
		4858	union {
		4859	__u8 long_mode;
		4860	__u8 vector;
		4861	struct {
		4862	__u64 gfn;
		4863	} shared_info;
		4864	__u64 pad[4];
		4865	} u;
		4866	};
		4867
		4868	type values:
		4869
		4870	KVM_XEN_ATTR_TYPE_LONG_MODE
		4871	Sets the ABI mode of the VM to 32-bit or 64-bit (long mode). This
		4872	determines the layout of the shared info pages exposed to the VM.
		4873
		4874	KVM_XEN_ATTR_TYPE_SHARED_INFO
		4875	Sets the guest physical frame number at which the Xen "shared info"
		4876	page resides. Note that although Xen places vcpu_info for the first
		4877	32 vCPUs in the shared_info page, KVM does not automatically do so
		4878	and instead requires that KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO be used
		4879	explicitly even when the vcpu_info for a given vCPU resides at the
		4880	"default" location in the shared_info page. This is because KVM is
		4881	not aware of the Xen CPU id which is used as the index into the
		4882	vcpu_info[] array, so cannot know the correct default location.
		4883
		4884	KVM_XEN_ATTR_TYPE_UPCALL_VECTOR
		4885	Sets the exception vector used to deliver Xen event channel upcalls.
		4886
		4887	4.128 KVM_XEN_HVM_GET_ATTR
		4888	--------------------------
		4889
		4890	:Capability: KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_SHARED_INFO
		4891	:Architectures: x86
		4892	:Type: vm ioctl
		4893	:Parameters: struct kvm_xen_hvm_attr
		4894	:Returns: 0 on success, < 0 on error
		4895
		4896	Allows Xen VM attributes to be read. For the structure and types,
		4897	see KVM_XEN_HVM_SET_ATTR above.
		4898
		4899	4.129 KVM_XEN_VCPU_SET_ATTR
		4900	---------------------------
		4901
		4902	:Capability: KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_SHARED_INFO
		4903	:Architectures: x86
		4904	:Type: vcpu ioctl
		4905	:Parameters: struct kvm_xen_vcpu_attr
		4906	:Returns: 0 on success, < 0 on error
		4907
		4908	::
		4909
		4910	struct kvm_xen_vcpu_attr {
		4911	__u16 type;
		4912	__u16 pad[3];
		4913	union {
		4914	__u64 gpa;
		4915	__u64 pad[4];
		4916	} u;
		4917	};
		4918
		4919	type values:
		4920
		4921	KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO
		4922	Sets the guest physical address of the vcpu_info for a given vCPU.
		4923
		4924	KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO
		4925	Sets the guest physical address of an additional pvclock structure
		4926	for a given vCPU. This is typically used for guest vsyscall support.
		4927
		4928	4.130 KVM_XEN_VCPU_GET_ATTR
		4929	---------------------------
		4930
		4931	:Capability: KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_SHARED_INFO
		4932	:Architectures: x86
		4933	:Type: vcpu ioctl
		4934	:Parameters: struct kvm_xen_vcpu_attr
		4935	:Returns: 0 on success, < 0 on error
		4936
		4937	Allows Xen vCPU attributes to be read. For the structure and types,
		4938	see KVM_XEN_VCPU_SET_ATTR above.
4834		4939
4835	5. The kvm_run structure	4940	5. The kvm_run structure
4836	========================	4941	========================
@@ -4893,9 +4998,11 @@ local APIC is not used. · 56 unmodified lines
4893	__u16 flags;	4998	__u16 flags;
4894		4999
4895	More architecture-specific flags detailing state of the VCPU that may	5000	More architecture-specific flags detailing state of the VCPU that may
4896	affect the device's behavior. The only currently defined flag is	5001	affect the device's behavior. Current defined flags:
4897	KVM_RUN_X86_SMM, which is valid on x86 machines and is set if the	5002	/* x86, set if the VCPU is in system management mode */
4898	VCPU is in system management mode.	5003	#define KVM_RUN_X86_SMM (1 << 0)
		5004	/* x86, set if bus lock detected in VM */
		5005	#define KVM_RUN_BUS_LOCK (1 << 1)
4899		5006
4900	::	5007	::
4901		5008
@@ -4996,13 +5103,18 @@ to the byte array. · 94 unmodified lines
4996		5103
4997	.. note::	5104	.. note::
4998		5105
4999	For KVM_EXIT_IO, KVM_EXIT_MMIO, KVM_EXIT_OSI, KVM_EXIT_PAPR,	5106	For KVM_EXIT_IO, KVM_EXIT_MMIO, KVM_EXIT_OSI, KVM_EXIT_PAPR, KVM_EXIT_XEN,
5000	KVM_EXIT_EPR, KVM_EXIT_X86_RDMSR and KVM_EXIT_X86_WRMSR the corresponding	5107	KVM_EXIT_EPR, KVM_EXIT_X86_RDMSR and KVM_EXIT_X86_WRMSR the corresponding
5001	operations are complete (and guest state is consistent) only after userspace	5108	operations are complete (and guest state is consistent) only after userspace
5002	has re-entered the kernel with KVM_RUN. The kernel side will first finish	5109	has re-entered the kernel with KVM_RUN. The kernel side will first finish
5003	incomplete operations and then check for pending signals. Userspace	5110	incomplete operations and then check for pending signals.
5004	can re-enter the guest with an unmasked signal pending to complete	5111
5005	pending operations.	5112	The pending state of the operation is not preserved in state which is
		5113	visible to userspace, thus userspace should ensure that the operation is
		5114	completed before performing a live migration. Userspace can re-enter the
		5115	guest with an unmasked signal pending or with the immediate_exit field set
		5116	to complete pending operations without allowing any further instructions
		5117	to be executed.
5006		5118
5007	::	5119	::
5008		5120
@@ -5327,6 +5439,34 @@ wants to write. Once finished processing the event, user space must continue · 318 unmodified lines
5327	vCPU execution. If the MSR write was unsuccessful, user space also sets the	5439	vCPU execution. If the MSR write was unsuccessful, user space also sets the
5328	"error" field to "1".	5440	"error" field to "1".
5329		5441
		5442	::
		5443
		5444
		5445	struct kvm_xen_exit {
		5446	#define KVM_EXIT_XEN_HCALL 1
		5447	__u32 type;
		5448	union {
		5449	struct {
		5450	__u32 longmode;
		5451	__u32 cpl;
		5452	__u64 input;
		5453	__u64 result;
		5454	__u64 params[6];
		5455	} hcall;
		5456	} u;
		5457	};
		5458	/* KVM_EXIT_XEN */
		5459	struct kvm_hyperv_exit xen;
		5460
		5461	Indicates that the VCPU exits into userspace to process some tasks
		5462	related to Xen emulation.
		5463
		5464	Valid values for 'type' are:
		5465
		5466	- KVM_EXIT_XEN_HCALL -- synchronously notify user-space about Xen hypercall.
		5467	Userspace is expected to place the hypercall result into the appropriate
		5468	field before invoking KVM_RUN again.
		5469
5330	::	5470	::
5331		5471
5332	/* Fix the size of the union. */	5472	/* Fix the size of the union. */
@@ -6038,6 +6178,53 @@ KVM_EXIT_X86_RDMSR and KVM_EXIT_X86_WRMSR exit notifications which user space · 705 unmodified lines
6038	can then handle to implement model specific MSR handling and/or user notifications	6178	can then handle to implement model specific MSR handling and/or user notifications
6039	to inform a user that an MSR was not handled.	6179	to inform a user that an MSR was not handled.
6040		6180
		6181	7.22 KVM_CAP_X86_BUS_LOCK_EXIT
		6182	-------------------------------
		6183
		6184	:Architectures: x86
		6185	:Target: VM
		6186	:Parameters: args[0] defines the policy used when bus locks detected in guest
		6187	:Returns: 0 on success, -EINVAL when args[0] contains invalid bits
		6188
		6189	Valid bits in args[0] are::
		6190
		6191	#define KVM_BUS_LOCK_DETECTION_OFF (1 << 0)
		6192	#define KVM_BUS_LOCK_DETECTION_EXIT (1 << 1)
		6193
		6194	Enabling this capability on a VM provides userspace with a way to select
		6195	a policy to handle the bus locks detected in guest. Userspace can obtain
		6196	the supported modes from the result of KVM_CHECK_EXTENSION and define it
		6197	through the KVM_ENABLE_CAP.
		6198
		6199	KVM_BUS_LOCK_DETECTION_OFF and KVM_BUS_LOCK_DETECTION_EXIT are supported
		6200	currently and mutually exclusive with each other. More bits can be added in
		6201	the future.
		6202
		6203	With KVM_BUS_LOCK_DETECTION_OFF set, bus locks in guest will not cause vm exits
		6204	so that no additional actions are needed. This is the default mode.
		6205
		6206	With KVM_BUS_LOCK_DETECTION_EXIT set, vm exits happen when bus lock detected
		6207	in VM. KVM just exits to userspace when handling them. Userspace can enforce
		6208	its own throttling or other policy based mitigations.
		6209
		6210	This capability is aimed to address the thread that VM can exploit bus locks to
		6211	degree the performance of the whole system. Once the userspace enable this
		6212	capability and select the KVM_BUS_LOCK_DETECTION_EXIT mode, KVM will set the
		6213	KVM_RUN_BUS_LOCK flag in vcpu-run->flags field and exit to userspace. Concerning
		6214	the bus lock vm exit can be preempted by a higher priority VM exit, the exit
		6215	notifications to userspace can be KVM_EXIT_BUS_LOCK or other reasons.
		6216	KVM_RUN_BUS_LOCK flag is used to distinguish between them.
		6217
		6218	7.22 KVM_CAP_PPC_DAWR1
		6219	----------------------
		6220
		6221	:Architectures: ppc
		6222	:Parameters: none
		6223	:Returns: 0 on success, -EINVAL when CPU doesn't support 2nd DAWR
		6224
		6225	This capability can be used to check / enable 2nd DAWR feature provided
		6226	by POWER10 processor.
		6227
6041	8. Other capabilities.	6228	8. Other capabilities.
6042	======================	6229	======================
6043		6230
@@ -6415,7 +6602,6 @@ guest according to the bits in the KVM_CPUID_FEATURES CPUID leaf · 371 unmodified lines
6415	(0x40000001). Otherwise, a guest may use the paravirtual features	6602	(0x40000001). Otherwise, a guest may use the paravirtual features
6416	regardless of what has actually been exposed through the CPUID leaf.	6603	regardless of what has actually been exposed through the CPUID leaf.
6417		6604
6418
6419	8.29 KVM_CAP_DIRTY_LOG_RING	6605	8.29 KVM_CAP_DIRTY_LOG_RING
6420	---------------------------	6606	---------------------------
6421		6607
@@ -6502,3 +6688,29 @@ KVM_GET_DIRTY_LOG and KVM_CLEAR_DIRTY_LOG. After enabling · 80 unmodified lines
6502	KVM_CAP_DIRTY_LOG_RING with an acceptable dirty ring size, the virtual	6688	KVM_CAP_DIRTY_LOG_RING with an acceptable dirty ring size, the virtual
6503	machine will switch to ring-buffer dirty page tracking and further	6689	machine will switch to ring-buffer dirty page tracking and further
6504	KVM_GET_DIRTY_LOG or KVM_CLEAR_DIRTY_LOG ioctls will fail.	6690	KVM_GET_DIRTY_LOG or KVM_CLEAR_DIRTY_LOG ioctls will fail.
		6691
		6692	8.30 KVM_CAP_XEN_HVM
		6693	--------------------
		6694
		6695	:Architectures: x86
		6696
		6697	This capability indicates the features that Xen supports for hosting Xen
		6698	PVHVM guests. Valid flags are::
		6699
		6700	#define KVM_XEN_HVM_CONFIG_HYPERCALL_MSR (1 << 0)
		6701	#define KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL (1 << 1)
		6702	#define KVM_XEN_HVM_CONFIG_SHARED_INFO (1 << 2)
		6703
		6704	The KVM_XEN_HVM_CONFIG_HYPERCALL_MSR flag indicates that the KVM_XEN_HVM_CONFIG
		6705	ioctl is available, for the guest to set its hypercall page.
		6706
		6707	If KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL is also set, the same flag may also be
		6708	provided in the flags to KVM_XEN_HVM_CONFIG, without providing hypercall page
		6709	contents, to request that KVM generate hypercall page content automatically
		6710	and also enable interception of guest hypercalls with KVM_EXIT_XEN.
		6711
		6712	The KVM_XEN_HVM_CONFIG_SHARED_INFO flag indicates the availability of the
		6713	KVM_XEN_HVM_SET_ATTR, KVM_XEN_HVM_GET_ATTR, KVM_XEN_VCPU_SET_ATTR and
		6714	KVM_XEN_VCPU_GET_ATTR ioctls, as well as the delivery of exception vectors
		6715	for event channel upcalls when the evtchn_upcall_pending field of a vcpu's
		6716	vcpu_info is set.

Documentation/virt/kvm/locking.rst+8−1

 - kvm->slots_lock is taken outside kvm->irq_lock, though acquiring
   them together is quite rare.
-On x86, vcpu->mutex is taken outside kvm->arch.hyperv.hv_lock.
+On x86:
+- vcpu->mutex is taken outside kvm->arch.hyperv.hv_lock
+- kvm->arch.mmu_lock is an rwlock.  kvm->arch.tdp_mmu_pages_lock is
+  taken inside kvm->arch.mmu_lock, and cannot be taken without already
+  holding kvm->arch.mmu_lock (typically with ``read_lock``, otherwise
+  there's no need to take kvm->arch.tdp_mmu_pages_lock at all).
 Everything else is a leaf: no other lock is taken inside the critical
 sections.

arch/arm64/include/asm/hyp_image.h+27−2

@@ -7,6 +7,9 @@ · 6 unmodified lines
7	#ifndef __ARM64_HYP_IMAGE_H__	7	#ifndef __ARM64_HYP_IMAGE_H__
8	#define __ARM64_HYP_IMAGE_H__	8	#define __ARM64_HYP_IMAGE_H__
9		9
		10	#define __HYP_CONCAT(a, b) a ## b
		11	#define HYP_CONCAT(a, b) __HYP_CONCAT(a, b)
		12
10	/*	13	/*
11	* KVM nVHE code has its own symbol namespace prefixed with __kvm_nvhe_,	14	* KVM nVHE code has its own symbol namespace prefixed with __kvm_nvhe_,
12	* to separate it from the kernel proper.	15	* to separate it from the kernel proper.
@@ -21,9 +24,31 @@ · 8 unmodified lines
21	*/	24	*/
22	#define HYP_SECTION_NAME(NAME) .hyp##NAME	25	#define HYP_SECTION_NAME(NAME) .hyp##NAME
23		26
		27	/* Symbol defined at the beginning of each hyp section. */
		28	#define HYP_SECTION_SYMBOL_NAME(NAME) \
		29	HYP_CONCAT(__hyp_section_, HYP_SECTION_NAME(NAME))
		30
		31	/*
		32	* Helper to generate linker script statements starting a hyp section.
		33	*
		34	* A symbol with a well-known name is defined at the first byte. This
		35	* is used as a base for hyp relocations (see gen-hyprel.c). It must
		36	* be defined inside the section so the linker of `vmlinux` cannot
		37	* separate it from the section data.
		38	*/
		39	#define BEGIN_HYP_SECTION(NAME) \
		40	HYP_SECTION_NAME(NAME) : { \
		41	HYP_SECTION_SYMBOL_NAME(NAME) = .;
		42
		43	/* Helper to generate linker script statements ending a hyp section. */
		44	#define END_HYP_SECTION \
		45	}
		46
24	/* Defines an ELF hyp section from input section @NAME and its subsections. */	47	/* Defines an ELF hyp section from input section @NAME and its subsections. */
25	#define HYP_SECTION(NAME) \	48	#define HYP_SECTION(NAME) \
26	HYP_SECTION_NAME(NAME) : { (NAME NAME##.) }	49	BEGIN_HYP_SECTION(NAME) \
		50	(NAME NAME##.) \
		51	END_HYP_SECTION
27		52
28	/*	53	/*
29	* Defines a linker script alias of a kernel-proper symbol referenced by	54	* Defines a linker script alias of a kernel-proper symbol referenced by

arch/arm64/include/asm/kvm_asm.h+0−20

@@ -199,26 +199,6 @@ extern void __vgic_v3_init_lrs(void); · 198 unmodified lines
199		199
200	extern u32 __kvm_get_mdcr_el2(void);	200	extern u32 __kvm_get_mdcr_el2(void);
201		201
202	/*
203	* Obtain the PC-relative address of a kernel symbol
204	* s: symbol
205	*
206	* The goal of this macro is to return a symbol's address based on a
207	* PC-relative computation, as opposed to a loading the VA from a
208	* constant pool or something similar. This works well for HYP, as an
209	* absolute VA is guaranteed to be wrong. Only use this if trying to
210	* obtain the address of a symbol (i.e. not something you obtained by
211	* following a pointer).
212	*/
213	#define hyp_symbol_addr(s) \
214	({ \
215	typeof(s) *addr; \
216	asm("adrp %0, %1\n" \
217	"add %0, %0, :lo12:%1\n" \
218	: "=r" (addr) : "S" (&s)); \
219	addr; \
220	})
221
222	#define __KVM_EXTABLE(from, to) \	202	#define __KVM_EXTABLE(from, to) \
223	" .pushsection __kvm_ex_table, \"a\"\n" \	203	" .pushsection __kvm_ex_table, \"a\"\n" \
224	" .align 3\n" \	204	" .align 3\n" \

arch/arm64/include/asm/kvm_host.h+2−1

 #define __KVM_HAVE_ARCH_INTC_INITIALIZED
-#define KVM_USER_MEM_SLOTS 512
 #define KVM_HALT_POLL_NS_DEFAULT 500000
 #include <kvm/arm_vgic.h>
 #define kvm_vcpu_has_pmu(vcpu)					\
 	(test_bit(KVM_ARM_VCPU_PMU_V3, (vcpu)->arch.features))
+int kvm_trng_call(struct kvm_vcpu *vcpu);
 #endif /* __ARM64_KVM_HOST_H__ */

arch/arm64/include/asm/kvm_mmu.h+17−44

 .endm
 /*
- * Convert a kernel image address to a PA
+ * Convert a hypervisor VA to a PA
- * reg: kernel address to be converted in place
+ * reg: hypervisor address to be converted in place
  * tmp: temporary register
- *
- * The actual code generation takes place in kvm_get_kimage_voffset, and
- * the instructions below are only there to reserve the space and
- * perform the register allocation (kvm_get_kimage_voffset uses the
- * specific registers encoded in the instructions).
  */
-.macro kimg_pa reg, tmp
+.macro hyp_pa reg, tmp
-alternative_cb kvm_get_kimage_voffset
+	ldr_l	\tmp, hyp_physvirt_offset
-	movz	\tmp, #0
+	add	\reg, \reg, \tmp
-	movk	\tmp, #0, lsl #16
-	movk	\tmp, #0, lsl #32
-	movk	\tmp, #0, lsl #48
-alternative_cb_end
-	/* reg = __pa(reg) */
-	sub	\reg, \reg, \tmp
 .endm
 /*
- * Convert a kernel image address to a hyp VA
+ * Convert a hypervisor VA to a kernel image address
- * reg: kernel address to be converted in place
+ * reg: hypervisor address to be converted in place
  * tmp: temporary register
  *
  * The actual code generation takes place in kvm_get_kimage_voffset, and
  * the instructions below are only there to reserve the space and
- * perform the register allocation (kvm_update_kimg_phys_offset uses the
+ * perform the register allocation (kvm_get_kimage_voffset uses the
  * specific registers encoded in the instructions).
  */
-.macro kimg_hyp_va reg, tmp
+.macro hyp_kimg_va reg, tmp
-alternative_cb kvm_update_kimg_phys_offset
+	/* Convert hyp VA -> PA. */
+	hyp_pa	\reg, \tmp
+	/* Load kimage_voffset. */
+alternative_cb kvm_get_kimage_voffset
 	movz	\tmp, #0
 	movk	\tmp, #0, lsl #16
 	movk	\tmp, #0, lsl #32
 	movk	\tmp, #0, lsl #48
 alternative_cb_end
-	sub	\reg, \reg, \tmp
+	/* Convert PA -> kimg VA. */
-	mov_q	\tmp, PAGE_OFFSET
+	add	\reg, \reg, \tmp
-	orr	\reg, \reg, \tmp
-	kern_hyp_va \reg
 .endm
 #else
 void kvm_update_va_mask(struct alt_instr *alt,
 			__le32 *origptr, __le32 *updptr, int nr_inst);
 void kvm_compute_layout(void);
+void kvm_apply_hyp_relocations(void);
 static __always_inline unsigned long __kern_hyp_va(unsigned long v)
 {
 #define kern_hyp_va(v) 	((typeof(v))(__kern_hyp_va((unsigned long)(v))))
-static __always_inline unsigned long __kimg_hyp_va(unsigned long v)
-{
-	unsigned long offset;
-	asm volatile(ALTERNATIVE_CB("movz %0, #0\n"
-				    "movk %0, #0, lsl #16\n"
-				    "movk %0, #0, lsl #32\n"
-				    "movk %0, #0, lsl #48\n",
-				    kvm_update_kimg_phys_offset)
-		     : "=r" (offset));
-	return __kern_hyp_va((v - offset) | PAGE_OFFSET);
-}
-#define kimg_fn_hyp_va(v) 	((typeof(*v))(__kimg_hyp_va((unsigned long)(v))))
-#define kimg_fn_ptr(x)	(typeof(x) **)(x)
 /*
  * We currently support using a VM-specified IPA size. For backward
  * compatibility, the default IPA size is fixed to 40bits.

arch/arm64/include/asm/kvm_pgtable.h+5−0

  * If device attributes are not explicitly requested in @prot, then the
  * mapping will be normal, cacheable.
  *
+ * Note that the update of a valid leaf PTE in this function will be aborted,
+ * if it's trying to recreate the exact same mapping or only change the access
+ * permissions. Instead, the vCPU will exit one more time from guest if still
+ * needed and then go through the path of relaxing permissions.
+ *
  * Note that this function will both coalesce existing table entries and split
  * existing block mappings, relying on page-faults to fault back areas outside
  * of the new mapping lazily.

arch/arm64/include/asm/sections.h+2−1

 extern char __hibernate_exit_text_start[], __hibernate_exit_text_end[];
 extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
 extern char __hyp_text_start[], __hyp_text_end[];
-extern char __hyp_data_ro_after_init_start[], __hyp_data_ro_after_init_end[];
+extern char __hyp_rodata_start[], __hyp_rodata_end[];
+extern char __hyp_reloc_begin[], __hyp_reloc_end[];
 extern char __idmap_text_start[], __idmap_text_end[];
 extern char __initdata_begin[], __initdata_end[];
 extern char __inittext_begin[], __inittext_end[];

arch/arm64/include/asm/spinlock.h+1−1

@@ -5,8 +5,8 @@ · 4 unmodified lines
5	#ifndef __ASM_SPINLOCK_H	5	#ifndef __ASM_SPINLOCK_H
6	#define __ASM_SPINLOCK_H	6	#define __ASM_SPINLOCK_H
7		7
8	#include <asm/qrwlock.h>
9	#include <asm/qspinlock.h>	8	#include <asm/qspinlock.h>
		9	#include <asm/qrwlock.h>
10		10
11	/* See include/linux/spinlock.h */	11	/* See include/linux/spinlock.h */
12	#define smp_mb__after_spinlock() smp_mb()	12	#define smp_mb__after_spinlock() smp_mb()

arch/arm64/include/asm/sysreg.h+3−0

@@ -853,7 +853,10 @@ · 852 unmodified lines
853		853
854	#define ID_DFR0_PERFMON_SHIFT 24	854	#define ID_DFR0_PERFMON_SHIFT 24
855		855
		856	#define ID_DFR0_PERFMON_8_0 0x3
856	#define ID_DFR0_PERFMON_8_1 0x4	857	#define ID_DFR0_PERFMON_8_1 0x4
		858	#define ID_DFR0_PERFMON_8_4 0x5
		859	#define ID_DFR0_PERFMON_8_5 0x6
857		860
858	#define ID_ISAR4_SWP_FRAC_SHIFT 28	861	#define ID_ISAR4_SWP_FRAC_SHIFT 28
859	#define ID_ISAR4_PSR_M_SHIFT 24	862	#define ID_ISAR4_PSR_M_SHIFT 24

arch/arm64/kernel/image-vars.h+0−1

@@ -64,7 +64,6 @@ __efistub__ctype = _ctype; · 63 unmodified lines
64	/* Alternative callbacks for init-time patching of nVHE hyp code. */	64	/* Alternative callbacks for init-time patching of nVHE hyp code. */
65	KVM_NVHE_ALIAS(kvm_patch_vector_branch);	65	KVM_NVHE_ALIAS(kvm_patch_vector_branch);
66	KVM_NVHE_ALIAS(kvm_update_va_mask);	66	KVM_NVHE_ALIAS(kvm_update_va_mask);
67	KVM_NVHE_ALIAS(kvm_update_kimg_phys_offset);
68	KVM_NVHE_ALIAS(kvm_get_kimage_voffset);	67	KVM_NVHE_ALIAS(kvm_get_kimage_voffset);
69		68
70	/* Global kernel state accessed by nVHE hyp code. */	69	/* Global kernel state accessed by nVHE hyp code. */

arch/arm64/kernel/smp.c+3−1

 			   "CPU: CPUs started in inconsistent modes");
 	else
 		pr_info("CPU: All CPU(s) started at EL1\n");
-	if (IS_ENABLED(CONFIG_KVM) && !is_kernel_in_hyp_mode())
+	if (IS_ENABLED(CONFIG_KVM) && !is_kernel_in_hyp_mode()) {
 		kvm_compute_layout();
+		kvm_apply_hyp_relocations();
+	}
 }
 void __init smp_cpus_done(unsigned int max_cpus)

arch/arm64/kernel/vmlinux.lds.S+15−3

 	__stop___kvm_ex_table = .;
 #define HYPERVISOR_DATA_SECTIONS				\
-	HYP_SECTION_NAME(.data..ro_after_init) : {		\
+	HYP_SECTION_NAME(.rodata) : {				\
-		__hyp_data_ro_after_init_start = .;		\
+		__hyp_rodata_start = .;				\
 		*(HYP_SECTION_NAME(.data..ro_after_init))	\
-		__hyp_data_ro_after_init_end = .;		\
+		*(HYP_SECTION_NAME(.rodata))			\
+		__hyp_rodata_end = .;				\
 	}
 #define HYPERVISOR_PERCPU_SECTION				\
 	HYP_SECTION_NAME(.data..percpu) : {			\
 		*(HYP_SECTION_NAME(.data..percpu))		\
 	}
+#define HYPERVISOR_RELOC_SECTION				\
+	.hyp.reloc : ALIGN(4) {					\
+		__hyp_reloc_begin = .;				\
+		*(.hyp.reloc)					\
+		__hyp_reloc_end = .;				\
+	}
 #else /* CONFIG_KVM */
 #define HYPERVISOR_EXTABLE
 #define HYPERVISOR_DATA_SECTIONS
 #define HYPERVISOR_PERCPU_SECTION
+#define HYPERVISOR_RELOC_SECTION
 #endif
 #define HYPERVISOR_TEXT					\
 	PERCPU_SECTION(L1_CACHE_BYTES)
 	HYPERVISOR_PERCPU_SECTION
+	HYPERVISOR_RELOC_SECTION
 	.rela.dyn : ALIGN(8) {
 		*(.rela .rela*)
 	}

arch/arm64/kvm/Makefile+1−1

@@ -16,7 +16,7 @@ kvm-y := $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o \ · 15 unmodified lines
16	inject_fault.o va_layout.o handle_exit.o \	16	inject_fault.o va_layout.o handle_exit.o \
17	guest.o debug.o reset.o sys_regs.o \	17	guest.o debug.o reset.o sys_regs.o \
18	vgic-sys-reg-v3.o fpsimd.o pmu.o \	18	vgic-sys-reg-v3.o fpsimd.o pmu.o \
19	arch_timer.o \	19	arch_timer.o trng.o\
20	vgic/vgic.o vgic/vgic-init.o \	20	vgic/vgic.o vgic/vgic-init.o \
21	vgic/vgic-irqfd.o vgic/vgic-v2.o \	21	vgic/vgic-irqfd.o vgic/vgic-v2.o \
22	vgic/vgic-v3.o vgic/vgic-v4.o \	22	vgic/vgic-v3.o vgic/vgic-v4.o \

arch/arm64/kvm/arm.c+3−4

 		goto out_err;
 	}
-	err = create_hyp_mappings(kvm_ksym_ref(__hyp_data_ro_after_init_start),
+	err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start),
-				  kvm_ksym_ref(__hyp_data_ro_after_init_end),
+				  kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO);
-				  PAGE_HYP_RO);
 	if (err) {
-		kvm_err("Cannot map .hyp.data..ro_after_init section\n");
+		kvm_err("Cannot map .hyp.rodata section\n");
 		goto out_err;
 	}

arch/arm64/kvm/hyp/include/hyp/switch.h+2−2

@@ -505,8 +505,8 @@ static inline void __kvm_unexpected_el2_exception(void) · 504 unmodified lines
505	struct exception_table_entry entry, end;	505	struct exception_table_entry entry, end;
506	unsigned long elr_el2 = read_sysreg(elr_el2);	506	unsigned long elr_el2 = read_sysreg(elr_el2);
507		507
508	entry = hyp_symbol_addr(__start___kvm_ex_table);	508	entry = &__start___kvm_ex_table;
509	end = hyp_symbol_addr(__stop___kvm_ex_table);	509	end = &__stop___kvm_ex_table;
510		510
511	while (entry < end) {	511	while (entry < end) {
512	addr = (unsigned long)&entry->insn + entry->insn;	512	addr = (unsigned long)&entry->insn + entry->insn;

arch/arm64/kvm/hyp/nvhe/.gitignore+2−0

@@ -1,2 +1,4 @@
1	# SPDX-License-Identifier: GPL-2.0-only	1	# SPDX-License-Identifier: GPL-2.0-only
		2	gen-hyprel
2	hyp.lds	3	hyp.lds
		4	hyp-reloc.S

arch/arm64/kvm/hyp/nvhe/Makefile+28−5

 # Makefile for Kernel-based Virtual Machine module, HYP/nVHE part
 #
-asflags-y := -D__KVM_NVHE_HYPERVISOR__
+asflags-y := -D__KVM_NVHE_HYPERVISOR__ -D__DISABLE_EXPORTS
-ccflags-y := -D__KVM_NVHE_HYPERVISOR__
+ccflags-y := -D__KVM_NVHE_HYPERVISOR__ -D__DISABLE_EXPORTS
+hostprogs := gen-hyprel
+HOST_EXTRACFLAGS += -I$(objtree)/include
 obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \
 	 hyp-main.o hyp-smp.o psci-relay.o
 hyp-obj := $(patsubst %.o,%.nvhe.o,$(obj-y))
 obj-y := kvm_nvhe.o
-extra-y := $(hyp-obj) kvm_nvhe.tmp.o hyp.lds
+extra-y := $(hyp-obj) kvm_nvhe.tmp.o kvm_nvhe.rel.o hyp.lds hyp-reloc.S hyp-reloc.o
 # 1) Compile all source files to `.nvhe.o` object files. The file extension
 #    avoids file name clashes for files shared with VHE.
 $(obj)/kvm_nvhe.tmp.o: $(obj)/hyp.lds $(addprefix $(obj)/,$(hyp-obj)) FORCE
 	$(call if_changed,ld)
-# 4) Produce the final 'kvm_nvhe.o', ready to be linked into 'vmlinux'.
+# 4) Generate list of hyp code/data positions that need to be relocated at
+#    runtime. Because the hypervisor is part of the kernel binary, relocations
+#    produce a kernel VA. We enumerate relocations targeting hyp at build time
+#    and convert the kernel VAs at those positions to hyp VAs.
+$(obj)/hyp-reloc.S: $(obj)/kvm_nvhe.tmp.o $(obj)/gen-hyprel
+	$(call if_changed,hyprel)
+# 5) Compile hyp-reloc.S and link it into the existing partially linked object.
+#    The object file now contains a section with pointers to hyp positions that
+#    will contain kernel VAs at runtime. These pointers have relocations on them
+#    so that they get updated as the hyp object is linked into `vmlinux`.
+LDFLAGS_kvm_nvhe.rel.o := -r
+$(obj)/kvm_nvhe.rel.o: $(obj)/kvm_nvhe.tmp.o $(obj)/hyp-reloc.o FORCE
+	$(call if_changed,ld)
+# 6) Produce the final 'kvm_nvhe.o', ready to be linked into 'vmlinux'.
 #    Prefixes names of ELF symbols with '__kvm_nvhe_'.
-$(obj)/kvm_nvhe.o: $(obj)/kvm_nvhe.tmp.o FORCE
+$(obj)/kvm_nvhe.o: $(obj)/kvm_nvhe.rel.o FORCE
 	$(call if_changed,hypcopy)
+# The HYPREL command calls `gen-hyprel` to generate an assembly file with
+# a list of relocations targeting hyp code/data.
+quiet_cmd_hyprel = HYPREL  $@
+      cmd_hyprel = $(obj)/gen-hyprel $< > $@
 # The HYPCOPY command uses `objcopy` to prefix all ELF symbol names
 # to avoid clashes with VHE code/data.
 quiet_cmd_hypcopy = HYPCOPY $@

arch/arm64/kvm/hyp/nvhe/gen-hyprel.c+438−0

@@ -0,0 +1,438 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (C) 2020 - Google LLC
	4	* Author: David Brazdil <dbrazdil@google.com>
	5	*
	6	* Generates relocation information used by the kernel to convert
	7	* absolute addresses in hyp data from kernel VAs to hyp VAs.
	8	*
	9	* This is necessary because hyp code is linked into the same binary
	10	* as the kernel but executes under different memory mappings.
	11	* If the compiler used absolute addressing, those addresses need to
	12	* be converted before they are used by hyp code.
	13	*
	14	* The input of this program is the relocatable ELF object containing
	15	* all hyp code/data, not yet linked into vmlinux. Hyp section names
	16	* should have been prefixed with `.hyp` at this point.
	17	*
	18	* The output (printed to stdout) is an assembly file containing
	19	* an array of 32-bit integers and static relocations that instruct
	20	* the linker of `vmlinux` to populate the array entries with offsets
	21	* to positions in the kernel binary containing VAs used by hyp code.
	22	*
	23	* Note that dynamic relocations could be used for the same purpose.
	24	* However, those are only generated if CONFIG_RELOCATABLE=y.
	25	*/
	26
	27	#include <elf.h>
	28	#include <endian.h>
	29	#include <errno.h>
	30	#include <fcntl.h>
	31	#include <stdbool.h>
	32	#include <stdio.h>
	33	#include <stdlib.h>
	34	#include <string.h>
	35	#include <sys/mman.h>
	36	#include <sys/types.h>
	37	#include <sys/stat.h>
	38	#include <unistd.h>
	39
	40	#include <generated/autoconf.h>
	41
	42	#define HYP_SECTION_PREFIX ".hyp"
	43	#define HYP_RELOC_SECTION ".hyp.reloc"
	44	#define HYP_SECTION_SYMBOL_PREFIX "__hyp_section_"
	45
	46	/*
	47	* AArch64 relocation type constants.
	48	* Included in case these are not defined in the host toolchain.
	49	*/
	50	#ifndef R_AARCH64_ABS64
	51	#define R_AARCH64_ABS64 257
	52	#endif
	53	#ifndef R_AARCH64_LD_PREL_LO19
	54	#define R_AARCH64_LD_PREL_LO19 273
	55	#endif
	56	#ifndef R_AARCH64_ADR_PREL_LO21
	57	#define R_AARCH64_ADR_PREL_LO21 274
	58	#endif
	59	#ifndef R_AARCH64_ADR_PREL_PG_HI21
	60	#define R_AARCH64_ADR_PREL_PG_HI21 275
	61	#endif
	62	#ifndef R_AARCH64_ADR_PREL_PG_HI21_NC
	63	#define R_AARCH64_ADR_PREL_PG_HI21_NC 276
	64	#endif
	65	#ifndef R_AARCH64_ADD_ABS_LO12_NC
	66	#define R_AARCH64_ADD_ABS_LO12_NC 277
	67	#endif
	68	#ifndef R_AARCH64_LDST8_ABS_LO12_NC
	69	#define R_AARCH64_LDST8_ABS_LO12_NC 278
	70	#endif
	71	#ifndef R_AARCH64_TSTBR14
	72	#define R_AARCH64_TSTBR14 279
	73	#endif
	74	#ifndef R_AARCH64_CONDBR19
	75	#define R_AARCH64_CONDBR19 280
	76	#endif
	77	#ifndef R_AARCH64_JUMP26
	78	#define R_AARCH64_JUMP26 282
	79	#endif
	80	#ifndef R_AARCH64_CALL26
	81	#define R_AARCH64_CALL26 283
	82	#endif
	83	#ifndef R_AARCH64_LDST16_ABS_LO12_NC
	84	#define R_AARCH64_LDST16_ABS_LO12_NC 284
	85	#endif
	86	#ifndef R_AARCH64_LDST32_ABS_LO12_NC
	87	#define R_AARCH64_LDST32_ABS_LO12_NC 285
	88	#endif
	89	#ifndef R_AARCH64_LDST64_ABS_LO12_NC
	90	#define R_AARCH64_LDST64_ABS_LO12_NC 286
	91	#endif
	92	#ifndef R_AARCH64_MOVW_PREL_G0
	93	#define R_AARCH64_MOVW_PREL_G0 287
	94	#endif
	95	#ifndef R_AARCH64_MOVW_PREL_G0_NC
	96	#define R_AARCH64_MOVW_PREL_G0_NC 288
	97	#endif
	98	#ifndef R_AARCH64_MOVW_PREL_G1
	99	#define R_AARCH64_MOVW_PREL_G1 289
	100	#endif
	101	#ifndef R_AARCH64_MOVW_PREL_G1_NC
	102	#define R_AARCH64_MOVW_PREL_G1_NC 290
	103	#endif
	104	#ifndef R_AARCH64_MOVW_PREL_G2
	105	#define R_AARCH64_MOVW_PREL_G2 291
	106	#endif
	107	#ifndef R_AARCH64_MOVW_PREL_G2_NC
	108	#define R_AARCH64_MOVW_PREL_G2_NC 292
	109	#endif
	110	#ifndef R_AARCH64_MOVW_PREL_G3
	111	#define R_AARCH64_MOVW_PREL_G3 293
	112	#endif
	113	#ifndef R_AARCH64_LDST128_ABS_LO12_NC
	114	#define R_AARCH64_LDST128_ABS_LO12_NC 299
	115	#endif
	116
	117	/* Global state of the processed ELF. */
	118	static struct {
	119	const char *path;
	120	char *begin;
	121	size_t size;
	122	Elf64_Ehdr *ehdr;
	123	Elf64_Shdr *sh_table;
	124	const char *sh_string;
	125	} elf;
	126
	127	#if defined(CONFIG_CPU_LITTLE_ENDIAN)
	128
	129	#define elf16toh(x) le16toh(x)
	130	#define elf32toh(x) le32toh(x)
	131	#define elf64toh(x) le64toh(x)
	132
	133	#define ELFENDIAN ELFDATA2LSB
	134
	135	#elif defined(CONFIG_CPU_BIG_ENDIAN)
	136
	137	#define elf16toh(x) be16toh(x)
	138	#define elf32toh(x) be32toh(x)
	139	#define elf64toh(x) be64toh(x)
	140
	141	#define ELFENDIAN ELFDATA2MSB
	142
	143	#else
	144
	145	#error PDP-endian sadly unsupported...
	146
	147	#endif
	148
	149	#define fatal_error(fmt, ...) \
	150	({ \
	151	fprintf(stderr, "error: %s: " fmt "\n", \
	152	elf.path, ## __VA_ARGS__); \
	153	exit(EXIT_FAILURE); \
	154	__builtin_unreachable(); \
	155	})
	156
	157	#define fatal_perror(msg) \
	158	({ \
	159	fprintf(stderr, "error: %s: " msg ": %s\n", \
	160	elf.path, strerror(errno)); \
	161	exit(EXIT_FAILURE); \
	162	__builtin_unreachable(); \
	163	})
	164
	165	#define assert_op(lhs, rhs, fmt, op) \
	166	({ \
	167	typeof(lhs) _lhs = (lhs); \
	168	typeof(rhs) _rhs = (rhs); \
	169	\
	170	if (!(_lhs op _rhs)) { \
	171	fatal_error("assertion " #lhs " " #op " " #rhs \
	172	" failed (lhs=" fmt ", rhs=" fmt \
	173	", line=%d)", _lhs, _rhs, __LINE__); \
	174	} \
	175	})
	176
	177	#define assert_eq(lhs, rhs, fmt) assert_op(lhs, rhs, fmt, ==)
	178	#define assert_ne(lhs, rhs, fmt) assert_op(lhs, rhs, fmt, !=)
	179	#define assert_lt(lhs, rhs, fmt) assert_op(lhs, rhs, fmt, <)
	180	#define assert_ge(lhs, rhs, fmt) assert_op(lhs, rhs, fmt, >=)
	181
	182	/*
	183	* Return a pointer of a given type at a given offset from
	184	* the beginning of the ELF file.
	185	*/
	186	#define elf_ptr(type, off) ((type *)(elf.begin + (off)))
	187
	188	/* Iterate over all sections in the ELF. */
	189	#define for_each_section(var) \
	190	for (var = elf.sh_table; var < elf.sh_table + elf16toh(elf.ehdr->e_shnum); ++var)
	191
	192	/* Iterate over all Elf64_Rela relocations in a given section. */
	193	#define for_each_rela(shdr, var) \
	194	for (var = elf_ptr(Elf64_Rela, elf64toh(shdr->sh_offset)); \
	195	var < elf_ptr(Elf64_Rela, elf64toh(shdr->sh_offset) + elf64toh(shdr->sh_size)); var++)
	196
	197	/* True if a string starts with a given prefix. */
	198	static inline bool starts_with(const char str, const char prefix)
	199	{
	200	return memcmp(str, prefix, strlen(prefix)) == 0;
	201	}
	202
	203	/* Returns a string containing the name of a given section. */
	204	static inline const char section_name(Elf64_Shdr shdr)
	205	{
	206	return elf.sh_string + elf32toh(shdr->sh_name);
	207	}
	208
	209	/* Returns a pointer to the first byte of section data. */
	210	static inline const char section_begin(Elf64_Shdr shdr)
	211	{
	212	return elf_ptr(char, elf64toh(shdr->sh_offset));
	213	}
	214
	215	/* Find a section by its offset from the beginning of the file. */
	216	static inline Elf64_Shdr *section_by_off(Elf64_Off off)
	217	{
	218	assert_ne(off, 0UL, "%lu");
	219	return elf_ptr(Elf64_Shdr, off);
	220	}
	221
	222	/* Find a section by its index. */
	223	static inline Elf64_Shdr *section_by_idx(uint16_t idx)
	224	{
	225	assert_ne(idx, SHN_UNDEF, "%u");
	226	return &elf.sh_table[idx];
	227	}
	228
	229	/*
	230	* Memory-map the given ELF file, perform sanity checks, and
	231	* populate global state.
	232	*/
	233	static void init_elf(const char *path)
	234	{
	235	int fd, ret;
	236	struct stat stat;
	237
	238	/* Store path in the global struct for error printing. */
	239	elf.path = path;
	240
	241	/* Open the ELF file. */
	242	fd = open(path, O_RDONLY);
	243	if (fd < 0)
	244	fatal_perror("Could not open ELF file");
	245
	246	/* Get status of ELF file to obtain its size. */
	247	ret = fstat(fd, &stat);
	248	if (ret < 0) {
	249	close(fd);
	250	fatal_perror("Could not get status of ELF file");
	251	}
	252
	253	/* mmap() the entire ELF file read-only at an arbitrary address. */
	254	elf.begin = mmap(0, stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
	255	if (elf.begin == MAP_FAILED) {
	256	close(fd);
	257	fatal_perror("Could not mmap ELF file");
	258	}
	259
	260	/* mmap() was successful, close the FD. */
	261	close(fd);
	262
	263	/* Get pointer to the ELF header. */
	264	assert_ge(stat.st_size, sizeof(*elf.ehdr), "%lu");
	265	elf.ehdr = elf_ptr(Elf64_Ehdr, 0);
	266
	267	/* Check the ELF magic. */
	268	assert_eq(elf.ehdr->e_ident[EI_MAG0], ELFMAG0, "0x%x");
	269	assert_eq(elf.ehdr->e_ident[EI_MAG1], ELFMAG1, "0x%x");
	270	assert_eq(elf.ehdr->e_ident[EI_MAG2], ELFMAG2, "0x%x");
	271	assert_eq(elf.ehdr->e_ident[EI_MAG3], ELFMAG3, "0x%x");
	272
	273	/* Sanity check that this is an ELF64 relocatable object for AArch64. */
	274	assert_eq(elf.ehdr->e_ident[EI_CLASS], ELFCLASS64, "%u");
	275	assert_eq(elf.ehdr->e_ident[EI_DATA], ELFENDIAN, "%u");
	276	assert_eq(elf16toh(elf.ehdr->e_type), ET_REL, "%u");
	277	assert_eq(elf16toh(elf.ehdr->e_machine), EM_AARCH64, "%u");
	278
	279	/* Populate fields of the global struct. */
	280	elf.sh_table = section_by_off(elf64toh(elf.ehdr->e_shoff));
	281	elf.sh_string = section_begin(section_by_idx(elf16toh(elf.ehdr->e_shstrndx)));
	282	}
	283
	284	/* Print the prologue of the output ASM file. */
	285	static void emit_prologue(void)
	286	{
	287	printf(".data\n"
	288	".pushsection " HYP_RELOC_SECTION ", \"a\"\n");
	289	}
	290
	291	/* Print ASM statements needed as a prologue to a processed hyp section. */
	292	static void emit_section_prologue(const char *sh_orig_name)
	293	{
	294	/* Declare the hyp section symbol. */
	295	printf(".global %s%s\n", HYP_SECTION_SYMBOL_PREFIX, sh_orig_name);
	296	}
	297
	298	/*
	299	* Print ASM statements to create a hyp relocation entry for a given
	300	* R_AARCH64_ABS64 relocation.
	301	*
	302	* The linker of vmlinux will populate the position given by `rela` with
	303	* an absolute 64-bit kernel VA. If the kernel is relocatable, it will
	304	* also generate a dynamic relocation entry so that the kernel can shift
	305	* the address at runtime for KASLR.
	306	*
	307	* Emit a 32-bit offset from the current address to the position given
	308	* by `rela`. This way the kernel can iterate over all kernel VAs used
	309	* by hyp at runtime and convert them to hyp VAs. However, that offset
	310	* will not be known until linking of `vmlinux`, so emit a PREL32
	311	* relocation referencing a symbol that the hyp linker script put at
	312	* the beginning of the relocated section + the offset from `rela`.
	313	*/
	314	static void emit_rela_abs64(Elf64_Rela rela, const char sh_orig_name)
	315	{
	316	/* Offset of this reloc from the beginning of HYP_RELOC_SECTION. */
	317	static size_t reloc_offset;
	318
	319	/* Create storage for the 32-bit offset. */
	320	printf(".word 0\n");
	321
	322	/*
	323	* Create a PREL32 relocation which instructs the linker of `vmlinux`
	324	* to insert offset to position <base> + <offset>, where <base> is
	325	* a symbol at the beginning of the relocated section, and <offset>
	326	* is `rela->r_offset`.
	327	*/
	328	printf(".reloc %lu, R_AARCH64_PREL32, %s%s + 0x%lx\n",
	329	reloc_offset, HYP_SECTION_SYMBOL_PREFIX, sh_orig_name,
	330	elf64toh(rela->r_offset));
	331
	332	reloc_offset += 4;
	333	}
	334
	335	/* Print the epilogue of the output ASM file. */
	336	static void emit_epilogue(void)
	337	{
	338	printf(".popsection\n");
	339	}
	340
	341	/*
	342	* Iterate over all RELA relocations in a given section and emit
	343	* hyp relocation data for all absolute addresses in hyp code/data.
	344	*
	345	* Static relocations that generate PC-relative-addressing are ignored.
	346	* Failure is reported for unexpected relocation types.
	347	*/
	348	static void emit_rela_section(Elf64_Shdr *sh_rela)
	349	{
	350	Elf64_Shdr *sh_orig = &elf.sh_table[elf32toh(sh_rela->sh_info)];
	351	const char *sh_orig_name = section_name(sh_orig);
	352	Elf64_Rela *rela;
	353
	354	/* Skip all non-hyp sections. */
	355	if (!starts_with(sh_orig_name, HYP_SECTION_PREFIX))
	356	return;
	357
	358	emit_section_prologue(sh_orig_name);
	359
	360	for_each_rela(sh_rela, rela) {
	361	uint32_t type = (uint32_t)elf64toh(rela->r_info);
	362
	363	/* Check that rela points inside the relocated section. */
	364	assert_lt(elf64toh(rela->r_offset), elf64toh(sh_orig->sh_size), "0x%lx");
	365
	366	switch (type) {
	367	/*
	368	* Data relocations to generate absolute addressing.
	369	* Emit a hyp relocation.
	370	*/
	371	case R_AARCH64_ABS64:
	372	emit_rela_abs64(rela, sh_orig_name);
	373	break;
	374	/* Allow relocations to generate PC-relative addressing. */
	375	case R_AARCH64_LD_PREL_LO19:
	376	case R_AARCH64_ADR_PREL_LO21:
	377	case R_AARCH64_ADR_PREL_PG_HI21:
	378	case R_AARCH64_ADR_PREL_PG_HI21_NC:
	379	case R_AARCH64_ADD_ABS_LO12_NC:
	380	case R_AARCH64_LDST8_ABS_LO12_NC:
	381	case R_AARCH64_LDST16_ABS_LO12_NC:
	382	case R_AARCH64_LDST32_ABS_LO12_NC:
	383	case R_AARCH64_LDST64_ABS_LO12_NC:
	384	case R_AARCH64_LDST128_ABS_LO12_NC:
	385	break;
	386	/* Allow relative relocations for control-flow instructions. */
	387	case R_AARCH64_TSTBR14:
	388	case R_AARCH64_CONDBR19:
	389	case R_AARCH64_JUMP26:
	390	case R_AARCH64_CALL26:
	391	break;
	392	/* Allow group relocations to create PC-relative offset inline. */
	393	case R_AARCH64_MOVW_PREL_G0:
	394	case R_AARCH64_MOVW_PREL_G0_NC:
	395	case R_AARCH64_MOVW_PREL_G1:
	396	case R_AARCH64_MOVW_PREL_G1_NC:
	397	case R_AARCH64_MOVW_PREL_G2:
	398	case R_AARCH64_MOVW_PREL_G2_NC:
	399	case R_AARCH64_MOVW_PREL_G3:
	400	break;
	401	default:
	402	fatal_error("Unexpected RELA type %u", type);
	403	}
	404	}
	405	}
	406
	407	/* Iterate over all sections and emit hyp relocation data for RELA sections. */
	408	static void emit_all_relocs(void)
	409	{
	410	Elf64_Shdr *shdr;
	411
	412	for_each_section(shdr) {
	413	switch (elf32toh(shdr->sh_type)) {
	414	case SHT_REL:
	415	fatal_error("Unexpected SHT_REL section \"%s\"",
	416	section_name(shdr));
	417	case SHT_RELA:
	418	emit_rela_section(shdr);
	419	break;
	420	}
	421	}
	422	}
	423
	424	int main(int argc, const char **argv)
	425	{
	426	if (argc != 2) {
	427	fprintf(stderr, "Usage: %s <elf_input>\n", argv[0]);
	428	return EXIT_FAILURE;
	429	}
	430
	431	init_elf(argv[1]);
	432
	433	emit_prologue();
	434	emit_all_relocs();
	435	emit_epilogue();
	436
	437	return EXIT_SUCCESS;
	438	}

arch/arm64/kvm/hyp/nvhe/host.S+15−14

@@ -74,27 +74,28 @@ SYM_FUNC_END(__host_enter) · 73 unmodified lines
74	* void __noreturn __hyp_do_panic(bool restore_host, u64 spsr, u64 elr, u64 par);	74	* void __noreturn __hyp_do_panic(bool restore_host, u64 spsr, u64 elr, u64 par);
75	*/	75	*/
76	SYM_FUNC_START(__hyp_do_panic)	76	SYM_FUNC_START(__hyp_do_panic)
77	/* Load the format arguments into x1-7 */
78	mov x6, x3
79	get_vcpu_ptr x7, x3
80
81	mrs x3, esr_el2
82	mrs x4, far_el2
83	mrs x5, hpfar_el2
84
85	/* Prepare and exit to the host's panic funciton. */	77	/* Prepare and exit to the host's panic funciton. */
86	mov lr, #(PSR_F_BIT \| PSR_I_BIT \| PSR_A_BIT \| PSR_D_BIT \|\	78	mov lr, #(PSR_F_BIT \| PSR_I_BIT \| PSR_A_BIT \| PSR_D_BIT \|\
87	PSR_MODE_EL1h)	79	PSR_MODE_EL1h)
88	msr spsr_el2, lr	80	msr spsr_el2, lr
89	ldr lr, =panic	81	ldr lr, =panic
		82	hyp_kimg_va lr, x6
90	msr elr_el2, lr	83	msr elr_el2, lr
91		84
92	/*	85	/* Set the panic format string. Use the, now free, LR as scratch. */
93	* Set the panic format string and enter the host, conditionally	86	ldr lr, =__hyp_panic_string
94	* restoring the host context.	87	hyp_kimg_va lr, x6
95	*/	88
		89	/* Load the format arguments into x1-7. */
		90	mov x6, x3
		91	get_vcpu_ptr x7, x3
		92	mrs x3, esr_el2
		93	mrs x4, far_el2
		94	mrs x5, hpfar_el2
		95
		96	/* Enter the host, conditionally restoring the host context. */
96	cmp x0, xzr	97	cmp x0, xzr
97	ldr x0, =__hyp_panic_string	98	mov x0, lr
98	b.eq __host_enter_without_restoring	99	b.eq __host_enter_without_restoring
99	b __host_enter_for_panic	100	b __host_enter_for_panic
100	SYM_FUNC_END(__hyp_do_panic)	101	SYM_FUNC_END(__hyp_do_panic)
@@ -124,7 +125,7 @@ SYM_FUNC_END(__hyp_do_panic) · 23 unmodified lines
124	* Preserve x0-x4, which may contain stub parameters.	125	* Preserve x0-x4, which may contain stub parameters.
125	*/	126	*/
126	ldr x5, =__kvm_handle_stub_hvc	127	ldr x5, =__kvm_handle_stub_hvc
127	kimg_pa x5, x6	128	hyp_pa x5, x6
128	br x5	129	br x5
129	.L__vect_end\@:	130	.L__vect_end\@:
130	.if ((.L__vect_end\@ - .L__vect_start\@) > 0x80)	131	.if ((.L__vect_end\@ - .L__vect_start\@) > 0x80)

arch/arm64/kvm/hyp/nvhe/hyp-init.S+5−14

@@ -18,7 +18,7 @@ · 17 unmodified lines
18	#include <asm/virt.h>	18	#include <asm/virt.h>
19		19
20	.text	20	.text
21	.pushsection .hyp.idmap.text, "ax"	21	.pushsection .idmap.text, "ax"
22		22
23	.align 11	23	.align 11
24		24
@@ -57,17 +57,10 @@ __do_hyp_init: · 32 unmodified lines
57	cmp x0, #HVC_STUB_HCALL_NR	57	cmp x0, #HVC_STUB_HCALL_NR
58	b.lo __kvm_handle_stub_hvc	58	b.lo __kvm_handle_stub_hvc
59		59
60	// We only actively check bits [24:31], and everything	60	mov x3, #KVM_HOST_SMCCC_FUNC(__kvm_hyp_init)
61	// else has to be zero, which we check at build time.	61	cmp x0, x3
62	#if (KVM_HOST_SMCCC_FUNC(__kvm_hyp_init) & 0xFFFFFFFF00FFFFFF)	62	b.eq 1f
63	#error Unexpected __KVM_HOST_SMCCC_FUNC___kvm_hyp_init value	63
64	#endif
65
66	ror x0, x0, #24
67	eor x0, x0, #((KVM_HOST_SMCCC_FUNC(__kvm_hyp_init) >> 24) & 0xF)
68	ror x0, x0, #4
69	eor x0, x0, #((KVM_HOST_SMCCC_FUNC(__kvm_hyp_init) >> 28) & 0xF)
70	cbz x0, 1f
71	mov x0, #SMCCC_RET_NOT_SUPPORTED	64	mov x0, #SMCCC_RET_NOT_SUPPORTED
72	eret	65	eret
73		66
@@ -141,7 +134,6 @@ alternative_else_nop_endif · 67 unmodified lines
141		134
142	/* Set the host vector */	135	/* Set the host vector */
143	ldr x0, =__kvm_hyp_host_vector	136	ldr x0, =__kvm_hyp_host_vector
144	kimg_hyp_va x0, x1
145	msr vbar_el2, x0	137	msr vbar_el2, x0
146		138
147	ret	139	ret
@@ -200,7 +192,6 @@ SYM_CODE_START_LOCAL(__kvm_hyp_init_cpu) · 52 unmodified lines
200	/* Leave idmap. */	192	/* Leave idmap. */
201	mov x0, x29	193	mov x0, x29
202	ldr x1, =kvm_host_psci_cpu_entry	194	ldr x1, =kvm_host_psci_cpu_entry
203	kimg_hyp_va x1, x2
204	br x1	195	br x1
205	SYM_CODE_END(__kvm_hyp_init_cpu)	196	SYM_CODE_END(__kvm_hyp_init_cpu)
206		197

arch/arm64/kvm/hyp/nvhe/hyp-main.c+4−7

 typedef void (*hcall_t)(struct kvm_cpu_context *);
-#define HANDLE_FUNC(x)	[__KVM_HOST_SMCCC_FUNC_##x] = kimg_fn_ptr(handle_##x)
+#define HANDLE_FUNC(x)	[__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x
-static const hcall_t *host_hcall[] = {
+static const hcall_t host_hcall[] = {
 	HANDLE_FUNC(__kvm_vcpu_run),
 	HANDLE_FUNC(__kvm_flush_vm_context),
 	HANDLE_FUNC(__kvm_tlb_flush_vmid_ipa),
 static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)
 {
 	DECLARE_REG(unsigned long, id, host_ctxt, 0);
-	const hcall_t *kfn;
 	hcall_t hfn;
 	id -= KVM_HOST_SMCCC_ID(0);
 	if (unlikely(id >= ARRAY_SIZE(host_hcall)))
 		goto inval;
-	kfn = host_hcall[id];
+	hfn = host_hcall[id];
-	if (unlikely(!kfn))
+	if (unlikely(!hfn))
 		goto inval;
 	cpu_reg(host_ctxt, 0) = SMCCC_RET_SUCCESS;
-	hfn = kimg_fn_hyp_va(kfn);
 	hfn(host_ctxt);
 	return;

arch/arm64/kvm/hyp/nvhe/hyp-smp.c+2−2

@@ -33,8 +33,8 @@ unsigned long __hyp_per_cpu_offset(unsigned int cpu) · 32 unmodified lines
33	if (cpu >= ARRAY_SIZE(kvm_arm_hyp_percpu_base))	33	if (cpu >= ARRAY_SIZE(kvm_arm_hyp_percpu_base))
34	hyp_panic();	34	hyp_panic();
35		35
36	cpu_base_array = (unsigned long *)hyp_symbol_addr(kvm_arm_hyp_percpu_base);	36	cpu_base_array = (unsigned long *)&kvm_arm_hyp_percpu_base;
37	this_cpu_base = kern_hyp_va(cpu_base_array[cpu]);	37	this_cpu_base = kern_hyp_va(cpu_base_array[cpu]);
38	elf_base = (unsigned long)hyp_symbol_addr(__per_cpu_start);	38	elf_base = (unsigned long)&__per_cpu_start;
39	return this_cpu_base - elf_base;	39	return this_cpu_base - elf_base;
40	}	40	}

arch/arm64/kvm/hyp/nvhe/hyp.lds.S+6−3

 #include <asm/memory.h>
 SECTIONS {
+	HYP_SECTION(.idmap.text)
 	HYP_SECTION(.text)
+	HYP_SECTION(.data..ro_after_init)
+	HYP_SECTION(.rodata)
 	/*
 	 * .hyp..data..percpu needs to be page aligned to maintain the same
 	 * alignment for when linking into vmlinux.
 	 */
 	. = ALIGN(PAGE_SIZE);
-	HYP_SECTION_NAME(.data..percpu) : {
+	BEGIN_HYP_SECTION(.data..percpu)
 		PERCPU_INPUT(L1_CACHE_BYTES)
-	}
+	END_HYP_SECTION
-	HYP_SECTION(.data..ro_after_init)
 }

arch/arm64/kvm/hyp/nvhe/psci-relay.c+12−12

@@ -128,8 +128,8 @@ static int psci_cpu_on(u64 func_id, struct kvm_cpu_context *host_ctxt) · 127 unmodified lines
128	if (cpu_id == INVALID_CPU_ID)	128	if (cpu_id == INVALID_CPU_ID)
129	return PSCI_RET_INVALID_PARAMS;	129	return PSCI_RET_INVALID_PARAMS;
130		130
131	boot_args = per_cpu_ptr(hyp_symbol_addr(cpu_on_args), cpu_id);	131	boot_args = per_cpu_ptr(&cpu_on_args, cpu_id);
132	init_params = per_cpu_ptr(hyp_symbol_addr(kvm_init_params), cpu_id);	132	init_params = per_cpu_ptr(&kvm_init_params, cpu_id);
133		133
134	/* Check if the target CPU is already being booted. */	134	/* Check if the target CPU is already being booted. */
135	if (!try_acquire_boot_args(boot_args))	135	if (!try_acquire_boot_args(boot_args))
@@ -140,7 +140,7 @@ static int psci_cpu_on(u64 func_id, struct kvm_cpu_context *host_ctxt) · 4 unmodified lines
140	wmb();	140	wmb();
141		141
142	ret = psci_call(func_id, mpidr,	142	ret = psci_call(func_id, mpidr,
143	__hyp_pa(hyp_symbol_addr(kvm_hyp_cpu_entry)),	143	__hyp_pa(&kvm_hyp_cpu_entry),
144	__hyp_pa(init_params));	144	__hyp_pa(init_params));
145		145
146	/* If successful, the lock will be released by the target CPU. */	146	/* If successful, the lock will be released by the target CPU. */
@@ -159,8 +159,8 @@ static int psci_cpu_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt) · 12 unmodified lines
159	struct psci_boot_args *boot_args;	159	struct psci_boot_args *boot_args;
160	struct kvm_nvhe_init_params *init_params;	160	struct kvm_nvhe_init_params *init_params;
161		161
162	boot_args = this_cpu_ptr(hyp_symbol_addr(suspend_args));	162	boot_args = this_cpu_ptr(&suspend_args);
163	init_params = this_cpu_ptr(hyp_symbol_addr(kvm_init_params));	163	init_params = this_cpu_ptr(&kvm_init_params);
164		164
165	/*	165	/*
166	* No need to acquire a lock before writing to boot_args because a core	166	* No need to acquire a lock before writing to boot_args because a core
@@ -174,7 +174,7 @@ static int psci_cpu_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt) · 7 unmodified lines
174	* point if it is a deep sleep state.	174	* point if it is a deep sleep state.
175	*/	175	*/
176	return psci_call(func_id, power_state,	176	return psci_call(func_id, power_state,
177	__hyp_pa(hyp_symbol_addr(kvm_hyp_cpu_resume)),	177	__hyp_pa(&kvm_hyp_cpu_resume),
178	__hyp_pa(init_params));	178	__hyp_pa(init_params));
179	}	179	}
180		180
@@ -186,8 +186,8 @@ static int psci_system_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt) · 5 unmodified lines
186	struct psci_boot_args *boot_args;	186	struct psci_boot_args *boot_args;
187	struct kvm_nvhe_init_params *init_params;	187	struct kvm_nvhe_init_params *init_params;
188		188
189	boot_args = this_cpu_ptr(hyp_symbol_addr(suspend_args));	189	boot_args = this_cpu_ptr(&suspend_args);
190	init_params = this_cpu_ptr(hyp_symbol_addr(kvm_init_params));	190	init_params = this_cpu_ptr(&kvm_init_params);
191		191
192	/*	192	/*
193	* No need to acquire a lock before writing to boot_args because a core	193	* No need to acquire a lock before writing to boot_args because a core
@@ -198,7 +198,7 @@ static int psci_system_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt) · 4 unmodified lines
198		198
199	/* Will only return on error. */	199	/* Will only return on error. */
200	return psci_call(func_id,	200	return psci_call(func_id,
201	__hyp_pa(hyp_symbol_addr(kvm_hyp_cpu_resume)),	201	__hyp_pa(&kvm_hyp_cpu_resume),
202	__hyp_pa(init_params), 0);	202	__hyp_pa(init_params), 0);
203	}	203	}
204		204
@@ -207,12 +207,12 @@ asmlinkage void __noreturn kvm_host_psci_cpu_entry(bool is_cpu_on) · 2 unmodified lines
207	struct psci_boot_args *boot_args;	207	struct psci_boot_args *boot_args;
208	struct kvm_cpu_context *host_ctxt;	208	struct kvm_cpu_context *host_ctxt;
209		209
210	host_ctxt = &this_cpu_ptr(hyp_symbol_addr(kvm_host_data))->host_ctxt;	210	host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
211		211
212	if (is_cpu_on)	212	if (is_cpu_on)
213	boot_args = this_cpu_ptr(hyp_symbol_addr(cpu_on_args));	213	boot_args = this_cpu_ptr(&cpu_on_args);
214	else	214	else
215	boot_args = this_cpu_ptr(hyp_symbol_addr(suspend_args));	215	boot_args = this_cpu_ptr(&suspend_args);
216		216
217	cpu_reg(host_ctxt, 0) = boot_args->r0;	217	cpu_reg(host_ctxt, 0) = boot_args->r0;
218	write_sysreg_el2(boot_args->pc, SYS_ELR);	218	write_sysreg_el2(boot_args->pc, SYS_ELR);

arch/arm64/kvm/hyp/pgtable.c+47−36

@@ -45,6 +45,10 @@ · 44 unmodified lines
45		45
46	#define KVM_PTE_LEAF_ATTR_HI_S2_XN BIT(54)	46	#define KVM_PTE_LEAF_ATTR_HI_S2_XN BIT(54)
47		47
		48	#define KVM_PTE_LEAF_ATTR_S2_PERMS (KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R \| \
		49	KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W \| \
		50	KVM_PTE_LEAF_ATTR_HI_S2_XN)
		51
48	struct kvm_pgtable_walk_data {	52	struct kvm_pgtable_walk_data {
49	struct kvm_pgtable *pgt;	53	struct kvm_pgtable *pgt;
50	struct kvm_pgtable_walker *walker;	54	struct kvm_pgtable_walker *walker;
@@ -170,10 +174,9 @@ static void kvm_set_table_pte(kvm_pte_t ptep, kvm_pte_t childp) · 119 unmodified lines
170	smp_store_release(ptep, pte);	174	smp_store_release(ptep, pte);
171	}	175	}
172		176
173	static bool kvm_set_valid_leaf_pte(kvm_pte_t *ptep, u64 pa, kvm_pte_t attr,	177	static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, u32 level)
174	u32 level)
175	{	178	{
176	kvm_pte_t old = *ptep, pte = kvm_phys_to_pte(pa);	179	kvm_pte_t pte = kvm_phys_to_pte(pa);
177	u64 type = (level == KVM_PGTABLE_MAX_LEVELS - 1) ? KVM_PTE_TYPE_PAGE :	180	u64 type = (level == KVM_PGTABLE_MAX_LEVELS - 1) ? KVM_PTE_TYPE_PAGE :
178	KVM_PTE_TYPE_BLOCK;	181	KVM_PTE_TYPE_BLOCK;
179		182
@@ -181,12 +184,7 @@ static bool kvm_set_valid_leaf_pte(kvm_pte_t *ptep, u64 pa, kvm_pte_t attr, · 1 unmodified line
181	pte \|= FIELD_PREP(KVM_PTE_TYPE, type);	184	pte \|= FIELD_PREP(KVM_PTE_TYPE, type);
182	pte \|= KVM_PTE_VALID;	185	pte \|= KVM_PTE_VALID;
183		186
184	/* Tolerate KVM recreating the exact same mapping. */	187	return pte;
185	if (kvm_pte_valid(old))
186	return old == pte;
187
188	smp_store_release(ptep, pte);
189	return true;
190	}	188	}
191		189
192	static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data *data, u64 addr,	190	static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data *data, u64 addr,
@@ -341,12 +339,17 @@ static int hyp_map_set_prot_attr(enum kvm_pgtable_prot prot, · 148 unmodified lines
341	static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level,	339	static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level,
342	kvm_pte_t ptep, struct hyp_map_data data)	340	kvm_pte_t ptep, struct hyp_map_data data)
343	{	341	{
		342	kvm_pte_t new, old = *ptep;
344	u64 granule = kvm_granule_size(level), phys = data->phys;	343	u64 granule = kvm_granule_size(level), phys = data->phys;
345		344
346	if (!kvm_block_mapping_supported(addr, end, phys, level))	345	if (!kvm_block_mapping_supported(addr, end, phys, level))
347	return false;	346	return false;
348		347
349	WARN_ON(!kvm_set_valid_leaf_pte(ptep, phys, data->attr, level));	348	/* Tolerate KVM recreating the exact same mapping */
		349	new = kvm_init_valid_leaf_pte(phys, data->attr, level);
		350	if (old != new && !WARN_ON(kvm_pte_valid(old)))
		351	smp_store_release(ptep, new);
		352
350	data->phys += granule;	353	data->phys += granule;
351	return true;	354	return true;
352	}	355	}
@@ -461,34 +464,41 @@ static int stage2_map_set_prot_attr(enum kvm_pgtable_prot prot, · 108 unmodified lines
461	return 0;	464	return 0;
462	}	465	}
463		466
464	static bool stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level,	467	static int stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level,
465	kvm_pte_t *ptep,	468	kvm_pte_t *ptep,
466	struct stage2_map_data *data)	469	struct stage2_map_data *data)
467	{	470	{
		471	kvm_pte_t new, old = *ptep;
468	u64 granule = kvm_granule_size(level), phys = data->phys;	472	u64 granule = kvm_granule_size(level), phys = data->phys;
		473	struct page *page = virt_to_page(ptep);
469		474
470	if (!kvm_block_mapping_supported(addr, end, phys, level))	475	if (!kvm_block_mapping_supported(addr, end, phys, level))
471	return false;	476	return -E2BIG;
472		477
473	/*	478	new = kvm_init_valid_leaf_pte(phys, data->attr, level);
474	* If the PTE was already valid, drop the refcount on the table	479	if (kvm_pte_valid(old)) {
475	* early, as it will be bumped-up again in stage2_map_walk_leaf().	480	/*
476	* This ensures that the refcount stays constant across a valid to	481	* Skip updating the PTE if we are trying to recreate the exact
477	* valid PTE update.	482	* same mapping or only change the access permissions. Instead,
478	*/	483	* the vCPU will exit one more time from guest if still needed
479	if (kvm_pte_valid(*ptep))	484	* and then go through the path of relaxing permissions.
480	put_page(virt_to_page(ptep));	485	*/
481		486	if (!((old ^ new) & (~KVM_PTE_LEAF_ATTR_S2_PERMS)))
482	if (kvm_set_valid_leaf_pte(ptep, phys, data->attr, level))	487	return -EAGAIN;
483	goto out;	488
		489	/*
		490	* There's an existing different valid leaf entry, so perform
		491	* break-before-make.
		492	*/
		493	kvm_set_invalid_pte(ptep);
		494	kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, level);
		495	put_page(page);
		496	}
484		497
485	/* There's an existing valid leaf entry, so perform break-before-make */	498	smp_store_release(ptep, new);
486	kvm_set_invalid_pte(ptep);	499	get_page(page);
487	kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, level);
488	kvm_set_valid_leaf_pte(ptep, phys, data->attr, level);
489	out:
490	data->phys += granule;	500	data->phys += granule;
491	return true;	501	return 0;
492	}	502	}
493		503
494	static int stage2_map_walk_table_pre(u64 addr, u64 end, u32 level,	504	static int stage2_map_walk_table_pre(u64 addr, u64 end, u32 level,
@@ -516,6 +526,7 @@ static int stage2_map_walk_table_pre(u64 addr, u64 end, u32 level, · 21 unmodified lines
516	static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,	526	static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
517	struct stage2_map_data *data)	527	struct stage2_map_data *data)
518	{	528	{
		529	int ret;
519	kvm_pte_t childp, pte = ptep;	530	kvm_pte_t childp, pte = ptep;
520	struct page *page = virt_to_page(ptep);	531	struct page *page = virt_to_page(ptep);
521		532
@@ -526,8 +537,9 @@ static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 4 unmodified lines
526	return 0;	537	return 0;
527	}	538	}
528		539
529	if (stage2_map_walker_try_leaf(addr, end, level, ptep, data))	540	ret = stage2_map_walker_try_leaf(addr, end, level, ptep, data);
530	goto out_get_page;	541	if (ret != -E2BIG)
		542	return ret;
531		543
532	if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1))	544	if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1))
533	return -EINVAL;	545	return -EINVAL;
@@ -551,9 +563,8 @@ static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 17 unmodified lines
551	}	563	}
552		564
553	kvm_set_table_pte(ptep, childp);	565	kvm_set_table_pte(ptep, childp);
554
555	out_get_page:
556	get_page(page);	566	get_page(page);
		567
557	return 0;	568	return 0;
558	}	569	}
559		570

arch/arm64/kvm/hyp/vgic-v2-cpuif-proxy.c+1−1

@@ -64,7 +64,7 @@ int __vgic_v2_perform_cpuif_access(struct kvm_vcpu *vcpu) · 63 unmodified lines
64	}	64	}
65		65
66	rd = kvm_vcpu_dabt_get_rd(vcpu);	66	rd = kvm_vcpu_dabt_get_rd(vcpu);
67	addr = hyp_symbol_addr(kvm_vgic_global_state)->vcpu_hyp_va;	67	addr = kvm_vgic_global_state.vcpu_hyp_va;
68	addr += fault_ipa - vgic->vgic_cpu_base;	68	addr += fault_ipa - vgic->vgic_cpu_base;
69		69
70	if (kvm_vcpu_dabt_iswrite(vcpu)) {	70	if (kvm_vcpu_dabt_iswrite(vcpu)) {

arch/arm64/kvm/hypercalls.c+6−0

 		if (gpa != GPA_INVALID)
 			val = gpa;
 		break;
+	case ARM_SMCCC_TRNG_VERSION:
+	case ARM_SMCCC_TRNG_FEATURES:
+	case ARM_SMCCC_TRNG_GET_UUID:
+	case ARM_SMCCC_TRNG_RND32:
+	case ARM_SMCCC_TRNG_RND64:
+		return kvm_trng_call(vcpu);
 	default:
 		return kvm_psci_call(vcpu);
 	}

arch/arm64/kvm/mmu.c+8−5

 	if (vma_pagesize == PAGE_SIZE && !force_pte)
 		vma_pagesize = transparent_hugepage_adjust(memslot, hva,
 							   &pfn, &fault_ipa);
-	if (writable) {
+	if (writable)
 		prot |= KVM_PGTABLE_PROT_W;
-		kvm_set_pfn_dirty(pfn);
-		mark_page_dirty(kvm, gfn);
-	}
 	if (fault_status != FSC_PERM && !device)
 		clean_dcache_guest_page(pfn, vma_pagesize);
 					     memcache);
 	}
+	/* Mark the page dirty only if the fault is handled successfully */
+	if (writable && !ret) {
+		kvm_set_pfn_dirty(pfn);
+		mark_page_dirty(kvm, gfn);
+	}
 out_unlock:
 	spin_unlock(&kvm->mmu_lock);
 	kvm_set_pfn_accessed(pfn);
 	kvm_release_pfn_clean(pfn);
-	return ret;
+	return ret != -EAGAIN ? ret : 0;
 }
 /* Resolve the access fault by making the page young again. */

arch/arm64/kvm/pmu-emul.c+10−4

 static u32 kvm_pmu_event_mask(struct kvm *kvm)
 {
 	switch (kvm->arch.pmuver) {
-	case 1:			/* ARMv8.0 */
+	case ID_AA64DFR0_PMUVER_8_0:
 		return GENMASK(9, 0);
-	case 4:			/* ARMv8.1 */
+	case ID_AA64DFR0_PMUVER_8_1:
-	case 5:			/* ARMv8.4 */
+	case ID_AA64DFR0_PMUVER_8_4:
-	case 6:			/* ARMv8.5 */
+	case ID_AA64DFR0_PMUVER_8_5:
 		return GENMASK(15, 0);
 	default:		/* Shouldn't be here, just for sanity */
 		WARN_ONCE(1, "Unknown PMU version %d\n", kvm->arch.pmuver);
 		base = 0;
 	} else {
 		val = read_sysreg(pmceid1_el0);
+		/*
+		 * Don't advertise STALL_SLOT, as PMMIR_EL0 is handled
+		 * as RAZ
+		 */
+		if (vcpu->kvm->arch.pmuver >= ID_AA64DFR0_PMUVER_8_4)
+			val &= ~BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT - 32);
 		base = 32;
 	}

arch/arm64/kvm/sys_regs.c+51−34

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	* Christoffer Dall <c.dall@virtualopensystems.com>	9	* Christoffer Dall <c.dall@virtualopensystems.com>
10	*/	10	*/
11		11
		12	#include <linux/bitfield.h>
12	#include <linux/bsearch.h>	13	#include <linux/bsearch.h>
13	#include <linux/kvm_host.h>	14	#include <linux/kvm_host.h>
14	#include <linux/mm.h>	15	#include <linux/mm.h>
@@ -700,14 +701,18 @@ static bool access_pmselr(struct kvm_vcpu vcpu, struct sys_reg_params p, · 685 unmodified lines
700	static bool access_pmceid(struct kvm_vcpu vcpu, struct sys_reg_params p,	701	static bool access_pmceid(struct kvm_vcpu vcpu, struct sys_reg_params p,
701	const struct sys_reg_desc *r)	702	const struct sys_reg_desc *r)
702	{	703	{
703	u64 pmceid;	704	u64 pmceid, mask, shift;
704		705
705	BUG_ON(p->is_write);	706	BUG_ON(p->is_write);
706		707
707	if (pmu_access_el0_disabled(vcpu))	708	if (pmu_access_el0_disabled(vcpu))
708	return false;	709	return false;
709		710
		711	get_access_mask(r, &mask, &shift);
		712
710	pmceid = kvm_pmu_get_pmceid(vcpu, (p->Op2 & 1));	713	pmceid = kvm_pmu_get_pmceid(vcpu, (p->Op2 & 1));
		714	pmceid &= mask;
		715	pmceid >>= shift;
711		716
712	p->regval = pmceid;	717	p->regval = pmceid;
713		718
@@ -1021,6 +1026,8 @@ static bool access_arch_timer(struct kvm_vcpu *vcpu, · 307 unmodified lines
1021	return true;	1026	return true;
1022	}	1027	}
1023		1028
		1029	#define FEATURE(x) (GENMASK_ULL(x##_SHIFT + 3, x##_SHIFT))
		1030
1024	/* Read a sanitised cpufeature ID register by sys_reg_desc */	1031	/* Read a sanitised cpufeature ID register by sys_reg_desc */
1025	static u64 read_id_reg(const struct kvm_vcpu *vcpu,	1032	static u64 read_id_reg(const struct kvm_vcpu *vcpu,
1026	struct sys_reg_desc const *r, bool raz)	1033	struct sys_reg_desc const *r, bool raz)
@@ -1028,36 +1035,41 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, · 1 unmodified line
1028	u32 id = reg_to_encoding(r);	1035	u32 id = reg_to_encoding(r);
1029	u64 val = raz ? 0 : read_sanitised_ftr_reg(id);	1036	u64 val = raz ? 0 : read_sanitised_ftr_reg(id);
1030		1037
1031	if (id == SYS_ID_AA64PFR0_EL1) {	1038	switch (id) {
		1039	case SYS_ID_AA64PFR0_EL1:
1032	if (!vcpu_has_sve(vcpu))	1040	if (!vcpu_has_sve(vcpu))
1033	val &= ~(0xfUL << ID_AA64PFR0_SVE_SHIFT);	1041	val &= ~FEATURE(ID_AA64PFR0_SVE);
1034	val &= ~(0xfUL << ID_AA64PFR0_AMU_SHIFT);	1042	val &= ~FEATURE(ID_AA64PFR0_AMU);
1035	val &= ~(0xfUL << ID_AA64PFR0_CSV2_SHIFT);	1043	val &= ~FEATURE(ID_AA64PFR0_CSV2);
1036	val \|= ((u64)vcpu->kvm->arch.pfr0_csv2 << ID_AA64PFR0_CSV2_SHIFT);	1044	val \|= FIELD_PREP(FEATURE(ID_AA64PFR0_CSV2), (u64)vcpu->kvm->arch.pfr0_csv2);
1037	val &= ~(0xfUL << ID_AA64PFR0_CSV3_SHIFT);	1045	val &= ~FEATURE(ID_AA64PFR0_CSV3);
1038	val \|= ((u64)vcpu->kvm->arch.pfr0_csv3 << ID_AA64PFR0_CSV3_SHIFT);	1046	val \|= FIELD_PREP(FEATURE(ID_AA64PFR0_CSV3), (u64)vcpu->kvm->arch.pfr0_csv3);
1039	} else if (id == SYS_ID_AA64PFR1_EL1) {	1047	break;
1040	val &= ~(0xfUL << ID_AA64PFR1_MTE_SHIFT);	1048	case SYS_ID_AA64PFR1_EL1:
1041	} else if (id == SYS_ID_AA64ISAR1_EL1 && !vcpu_has_ptrauth(vcpu)) {	1049	val &= ~FEATURE(ID_AA64PFR1_MTE);
1042	val &= ~((0xfUL << ID_AA64ISAR1_APA_SHIFT) \|	1050	break;
1043	(0xfUL << ID_AA64ISAR1_API_SHIFT) \|	1051	case SYS_ID_AA64ISAR1_EL1:
1044	(0xfUL << ID_AA64ISAR1_GPA_SHIFT) \|	1052	if (!vcpu_has_ptrauth(vcpu))
1045	(0xfUL << ID_AA64ISAR1_GPI_SHIFT));	1053	val &= ~(FEATURE(ID_AA64ISAR1_APA) \|
1046	} else if (id == SYS_ID_AA64DFR0_EL1) {	1054	FEATURE(ID_AA64ISAR1_API) \|
1047	u64 cap = 0;	1055	FEATURE(ID_AA64ISAR1_GPA) \|
1048		1056	FEATURE(ID_AA64ISAR1_GPI));
1049	/* Limit guests to PMUv3 for ARMv8.1 */	1057	break;
1050	if (kvm_vcpu_has_pmu(vcpu))	1058	case SYS_ID_AA64DFR0_EL1:
1051	cap = ID_AA64DFR0_PMUVER_8_1;	1059	/* Limit debug to ARMv8.0 */
1052		1060	val &= ~FEATURE(ID_AA64DFR0_DEBUGVER);
		1061	val \|= FIELD_PREP(FEATURE(ID_AA64DFR0_DEBUGVER), 6);
		1062	/* Limit guests to PMUv3 for ARMv8.4 */
1053	val = cpuid_feature_cap_perfmon_field(val,	1063	val = cpuid_feature_cap_perfmon_field(val,
1054	ID_AA64DFR0_PMUVER_SHIFT,	1064	ID_AA64DFR0_PMUVER_SHIFT,
1055	cap);	1065	kvm_vcpu_has_pmu(vcpu) ? ID_AA64DFR0_PMUVER_8_4 : 0);
1056	} else if (id == SYS_ID_DFR0_EL1) {	1066	break;
1057	/* Limit guests to PMUv3 for ARMv8.1 */	1067	case SYS_ID_DFR0_EL1:
		1068	/* Limit guests to PMUv3 for ARMv8.4 */
1058	val = cpuid_feature_cap_perfmon_field(val,	1069	val = cpuid_feature_cap_perfmon_field(val,
1059	ID_DFR0_PERFMON_SHIFT,	1070	ID_DFR0_PERFMON_SHIFT,
1060	ID_DFR0_PERFMON_8_1);	1071	kvm_vcpu_has_pmu(vcpu) ? ID_DFR0_PERFMON_8_4 : 0);
		1072	break;
1061	}	1073	}
1062		1074
1063	return val;	1075	return val;
@@ -1493,6 +1505,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 429 unmodified lines
1493	.access = access_pminten, .reg = PMINTENSET_EL1 },	1505	.access = access_pminten, .reg = PMINTENSET_EL1 },
1494	{ PMU_SYS_REG(SYS_PMINTENCLR_EL1),	1506	{ PMU_SYS_REG(SYS_PMINTENCLR_EL1),
1495	.access = access_pminten, .reg = PMINTENSET_EL1 },	1507	.access = access_pminten, .reg = PMINTENSET_EL1 },
		1508	{ SYS_DESC(SYS_PMMIR_EL1), trap_raz_wi },
1496		1509
1497	{ SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 },	1510	{ SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 },
1498	{ SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 },	1511	{ SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 },
@@ -1720,7 +1733,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 221 unmodified lines
1720	{ SYS_DESC(SYS_FPEXC32_EL2), NULL, reset_val, FPEXC32_EL2, 0x700 },	1733	{ SYS_DESC(SYS_FPEXC32_EL2), NULL, reset_val, FPEXC32_EL2, 0x700 },
1721	};	1734	};
1722		1735
1723	static bool trap_dbgidr(struct kvm_vcpu *vcpu,	1736	static bool trap_dbgdidr(struct kvm_vcpu *vcpu,
1724	struct sys_reg_params *p,	1737	struct sys_reg_params *p,
1725	const struct sys_reg_desc *r)	1738	const struct sys_reg_desc *r)
1726	{	1739	{
@@ -1734,7 +1747,7 @@ static bool trap_dbgidr(struct kvm_vcpu *vcpu, · 7 unmodified lines
1734	p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) \|	1747	p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) \|
1735	(((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) \|	1748	(((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) \|
1736	(((dfr >> ID_AA64DFR0_CTX_CMPS_SHIFT) & 0xf) << 20)	1749	(((dfr >> ID_AA64DFR0_CTX_CMPS_SHIFT) & 0xf) << 20)
1737	\| (6 << 16) \| (el3 << 14) \| (el3 << 12));	1750	\| (6 << 16) \| (1 << 15) \| (el3 << 14) \| (el3 << 12));
1738	return true;	1751	return true;
1739	}	1752	}
1740	}	1753	}
@@ -1767,8 +1780,8 @@ static bool trap_dbgidr(struct kvm_vcpu *vcpu, · 26 unmodified lines
1767	* guest. Revisit this one day, would this principle change.	1780	* guest. Revisit this one day, would this principle change.
1768	*/	1781	*/
1769	static const struct sys_reg_desc cp14_regs[] = {	1782	static const struct sys_reg_desc cp14_regs[] = {
1770	/* DBGIDR */	1783	/* DBGDIDR */
1771	{ Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgidr },	1784	{ Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgdidr },
1772	/* DBGDTRRXext */	1785	/* DBGDTRRXext */
1773	{ Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },	1786	{ Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },
1774		1787
@@ -1918,8 +1931,8 @@ static const struct sys_reg_desc cp15_regs[] = { · 143 unmodified lines
1918	{ Op1( 0), CRn( 9), CRm(12), Op2( 3), access_pmovs },	1931	{ Op1( 0), CRn( 9), CRm(12), Op2( 3), access_pmovs },
1919	{ Op1( 0), CRn( 9), CRm(12), Op2( 4), access_pmswinc },	1932	{ Op1( 0), CRn( 9), CRm(12), Op2( 4), access_pmswinc },
1920	{ Op1( 0), CRn( 9), CRm(12), Op2( 5), access_pmselr },	1933	{ Op1( 0), CRn( 9), CRm(12), Op2( 5), access_pmselr },
1921	{ Op1( 0), CRn( 9), CRm(12), Op2( 6), access_pmceid },	1934	{ AA32(LO), Op1( 0), CRn( 9), CRm(12), Op2( 6), access_pmceid },
1922	{ Op1( 0), CRn( 9), CRm(12), Op2( 7), access_pmceid },	1935	{ AA32(LO), Op1( 0), CRn( 9), CRm(12), Op2( 7), access_pmceid },
1923	{ Op1( 0), CRn( 9), CRm(13), Op2( 0), access_pmu_evcntr },	1936	{ Op1( 0), CRn( 9), CRm(13), Op2( 0), access_pmu_evcntr },
1924	{ Op1( 0), CRn( 9), CRm(13), Op2( 1), access_pmu_evtyper },	1937	{ Op1( 0), CRn( 9), CRm(13), Op2( 1), access_pmu_evtyper },
1925	{ Op1( 0), CRn( 9), CRm(13), Op2( 2), access_pmu_evcntr },	1938	{ Op1( 0), CRn( 9), CRm(13), Op2( 2), access_pmu_evcntr },
@@ -1927,6 +1940,10 @@ static const struct sys_reg_desc cp15_regs[] = { · 1 unmodified line
1927	{ Op1( 0), CRn( 9), CRm(14), Op2( 1), access_pminten },	1940	{ Op1( 0), CRn( 9), CRm(14), Op2( 1), access_pminten },
1928	{ Op1( 0), CRn( 9), CRm(14), Op2( 2), access_pminten },	1941	{ Op1( 0), CRn( 9), CRm(14), Op2( 2), access_pminten },
1929	{ Op1( 0), CRn( 9), CRm(14), Op2( 3), access_pmovs },	1942	{ Op1( 0), CRn( 9), CRm(14), Op2( 3), access_pmovs },
		1943	{ AA32(HI), Op1( 0), CRn( 9), CRm(14), Op2( 4), access_pmceid },
		1944	{ AA32(HI), Op1( 0), CRn( 9), CRm(14), Op2( 5), access_pmceid },
		1945	/* PMMIR */
		1946	{ Op1( 0), CRn( 9), CRm(14), Op2( 6), trap_raz_wi },
1930		1947
1931	/* PRRR/MAIR0 */	1948	/* PRRR/MAIR0 */
1932	{ AA32(LO), Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, MAIR_EL1 },	1949	{ AA32(LO), Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, MAIR_EL1 },

arch/arm64/kvm/trng.c+85−0

@@ -0,0 +1,85 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	// Copyright (C) 2020 Arm Ltd.
	3
	4	#include <linux/arm-smccc.h>
	5	#include <linux/kvm_host.h>
	6
	7	#include <asm/kvm_emulate.h>
	8
	9	#include <kvm/arm_hypercalls.h>
	10
	11	#define ARM_SMCCC_TRNG_VERSION_1_0 0x10000UL
	12
	13	/* Those values are deliberately separate from the generic SMCCC definitions. */
	14	#define TRNG_SUCCESS 0UL
	15	#define TRNG_NOT_SUPPORTED ((unsigned long)-1)
	16	#define TRNG_INVALID_PARAMETER ((unsigned long)-2)
	17	#define TRNG_NO_ENTROPY ((unsigned long)-3)
	18
	19	#define TRNG_MAX_BITS64 192
	20
	21	static const uuid_t arm_smc_trng_uuid __aligned(4) = UUID_INIT(
	22	0x0d21e000, 0x4384, 0x11eb, 0x80, 0x70, 0x52, 0x44, 0x55, 0x4e, 0x5a, 0x4c);
	23
	24	static int kvm_trng_do_rnd(struct kvm_vcpu *vcpu, int size)
	25	{
	26	DECLARE_BITMAP(bits, TRNG_MAX_BITS64);
	27	u32 num_bits = smccc_get_arg1(vcpu);
	28	int i;
	29
	30	if (num_bits > 3 * size) {
	31	smccc_set_retval(vcpu, TRNG_INVALID_PARAMETER, 0, 0, 0);
	32	return 1;
	33	}
	34
	35	/* get as many bits as we need to fulfil the request */
	36	for (i = 0; i < DIV_ROUND_UP(num_bits, BITS_PER_LONG); i++)
	37	bits[i] = get_random_long();
	38
	39	bitmap_clear(bits, num_bits, TRNG_MAX_BITS64 - num_bits);
	40
	41	if (size == 32)
	42	smccc_set_retval(vcpu, TRNG_SUCCESS, lower_32_bits(bits[1]),
	43	upper_32_bits(bits[0]), lower_32_bits(bits[0]));
	44	else
	45	smccc_set_retval(vcpu, TRNG_SUCCESS, bits[2], bits[1], bits[0]);
	46
	47	memzero_explicit(bits, sizeof(bits));
	48	return 1;
	49	}
	50
	51	int kvm_trng_call(struct kvm_vcpu *vcpu)
	52	{
	53	const __le32 u = (__le32 )arm_smc_trng_uuid.b;
	54	u32 func_id = smccc_get_function(vcpu);
	55	unsigned long val = TRNG_NOT_SUPPORTED;
	56	int size = 64;
	57
	58	switch (func_id) {
	59	case ARM_SMCCC_TRNG_VERSION:
	60	val = ARM_SMCCC_TRNG_VERSION_1_0;
	61	break;
	62	case ARM_SMCCC_TRNG_FEATURES:
	63	switch (smccc_get_arg1(vcpu)) {
	64	case ARM_SMCCC_TRNG_VERSION:
	65	case ARM_SMCCC_TRNG_FEATURES:
	66	case ARM_SMCCC_TRNG_GET_UUID:
	67	case ARM_SMCCC_TRNG_RND32:
	68	case ARM_SMCCC_TRNG_RND64:
	69	val = TRNG_SUCCESS;
	70	}
	71	break;
	72	case ARM_SMCCC_TRNG_GET_UUID:
	73	smccc_set_retval(vcpu, le32_to_cpu(u[0]), le32_to_cpu(u[1]),
	74	le32_to_cpu(u[2]), le32_to_cpu(u[3]));
	75	return 1;
	76	case ARM_SMCCC_TRNG_RND32:
	77	size = 32;
	78	fallthrough;
	79	case ARM_SMCCC_TRNG_RND64:
	80	return kvm_trng_do_rnd(vcpu, size);
	81	}
	82
	83	smccc_set_retval(vcpu, val, 0, 0, 0);
	84	return 1;
	85	}

arch/arm64/kvm/va_layout.c+28−6

@@ -81,6 +81,34 @@ __init void kvm_compute_layout(void) · 80 unmodified lines
81	init_hyp_physvirt_offset();	81	init_hyp_physvirt_offset();
82	}	82	}
83		83
		84	/*
		85	* The .hyp.reloc ELF section contains a list of kimg positions that
		86	* contains kimg VAs but will be accessed only in hyp execution context.
		87	* Convert them to hyp VAs. See gen-hyprel.c for more details.
		88	*/
		89	__init void kvm_apply_hyp_relocations(void)
		90	{
		91	int32_t *rel;
		92	int32_t begin = (int32_t )__hyp_reloc_begin;
		93	int32_t end = (int32_t )__hyp_reloc_end;
		94
		95	for (rel = begin; rel < end; ++rel) {
		96	uintptr_t *ptr, kimg_va;
		97
		98	/*
		99	* Each entry contains a 32-bit relative offset from itself
		100	* to a kimg VA position.
		101	*/
		102	ptr = (uintptr_t )lm_alias((char )rel + *rel);
		103
		104	/* Read the kimg VA value at the relocation address. */
		105	kimg_va = *ptr;
		106
		107	/* Convert to hyp VA and store back to the relocation address. */
		108	*ptr = __early_kern_hyp_va((uintptr_t)lm_alias(kimg_va));
		109	}
		110	}
		111
84	static u32 compute_instruction(int n, u32 rd, u32 rn)	112	static u32 compute_instruction(int n, u32 rd, u32 rn)
85	{	113	{
86	u32 insn = AARCH64_BREAK_FAULT;	114	u32 insn = AARCH64_BREAK_FAULT;
@@ -255,12 +283,6 @@ static void generate_mov_q(u64 val, __le32 origptr, __le32 updptr, int nr_inst · 168 unmodified lines
255	*updptr++ = cpu_to_le32(insn);	283	*updptr++ = cpu_to_le32(insn);
256	}	284	}
257		285
258	void kvm_update_kimg_phys_offset(struct alt_instr *alt,
259	__le32 origptr, __le32 updptr, int nr_inst)
260	{
261	generate_mov_q(kimage_voffset + PHYS_OFFSET, origptr, updptr, nr_inst);
262	}
263
264	void kvm_get_kimage_voffset(struct alt_instr *alt,	286	void kvm_get_kimage_voffset(struct alt_instr *alt,
265	__le32 origptr, __le32 updptr, int nr_inst)	287	__le32 origptr, __le32 updptr, int nr_inst)
266	{	288	{

arch/mips/include/asm/kvm_host.h+0−1

@@ -83,7 +83,6 @@ · 82 unmodified lines
83		83
84		84
85	#define KVM_MAX_VCPUS 16	85	#define KVM_MAX_VCPUS 16
86	#define KVM_USER_MEM_SLOTS 16
87	/* memory slots that does not exposed to userspace */	86	/* memory slots that does not exposed to userspace */
88	#define KVM_PRIVATE_MEM_SLOTS 0	87	#define KVM_PRIVATE_MEM_SLOTS 0
89		88

arch/mips/include/asm/spinlock.h+1−1

 #define _ASM_SPINLOCK_H
 #include <asm/processor.h>
-#include <asm/qrwlock.h>
 #include <asm-generic/qspinlock_types.h>
 }
 #include <asm/qspinlock.h>
+#include <asm/qrwlock.h>
 #endif /* _ASM_SPINLOCK_H */

arch/powerpc/include/asm/hvcall.h+22−3

 	u64 behaviour;
 };
-/* Register state for entering a nested guest with H_ENTER_NESTED */
+/*
+ * Register state for entering a nested guest with H_ENTER_NESTED.
+ * New member must be added at the end.
+ */
 struct hv_guest_state {
-	u64 version;		/* version of this structure layout */
+	u64 version;		/* version of this structure layout, must be first */
 	u32 lpid;
 	u32 vcpu_token;
 	/* These registers are hypervisor privileged (at least for writing) */
 	u64 pidr;
 	u64 cfar;
 	u64 ppr;
+	/* Version 1 ends here */
+	u64 dawr1;
+	u64 dawrx1;
+	/* Version 2 ends here */
 };
 /* Latest version of hv_guest_state structure */
-#define HV_GUEST_STATE_VERSION	1
+#define HV_GUEST_STATE_VERSION	2
+static inline int hv_guest_state_size(unsigned int version)
+{
+	switch (version) {
+	case 1:
+		return offsetofend(struct hv_guest_state, ppr);
+	case 2:
+		return offsetofend(struct hv_guest_state, dawrx1);
+	default:
+		return -1;
+	}
+}
 /*
  * From the document "H_GetPerformanceCounterInfo Interface" v1.07

arch/powerpc/include/asm/kvm_book3s_asm.h+0−11

 	u8		do_nap;
 	u8		napped[MAX_SMT_THREADS];
 	struct kvmppc_vcore *vc[MAX_SUBCORES];
-	/* Bits for changing lpcr on P9 */
-	unsigned long	lpcr_req;
-	unsigned long	lpidr_req;
-	unsigned long	host_lpcr;
-	u32		do_set;
-	u32		do_restore;
-	union {
-		u32	allphases;
-		u8	phase[4];
-	} lpcr_sync;
 };
 /*
 	u8 hwthread_state;
 	u8 host_ipi;
 	u8 ptid;		/* thread number within subcore when split */
-	u8 tid;			/* thread number within whole core */
 	u8 fake_suspend;
 	struct kvm_vcpu *kvm_vcpu;
 	struct kvmppc_vcore *kvm_vcore;

arch/powerpc/include/asm/kvm_host.h+5−3

 #define KVM_MAX_VCPUS		NR_CPUS
 #define KVM_MAX_VCORES		NR_CPUS
-#define KVM_USER_MEM_SLOTS	512
 #include <asm/cputhreads.h>
 	u8 svm_enabled;
 	bool threads_indep;
 	bool nested_enable;
+	bool dawr1_enabled;
 	pgd_t *pgtable;
 	u64 process_table;
 	struct dentry *debugfs_dir;
 	u32 ctrl;
 	u32 dabrx;
 	ulong dabr;
-	ulong dawr;
+	ulong dawr0;
-	ulong dawrx;
+	ulong dawrx0;
+	ulong dawr1;
+	ulong dawrx1;
 	ulong ciabr;
 	ulong cfar;
 	ulong ppr;

arch/powerpc/include/asm/kvm_ppc.h+2−0

@@ -314,6 +314,8 @@ struct kvmppc_ops { · 313 unmodified lines
314	int size);	314	int size);
315	int (enable_svm)(struct kvm kvm);	315	int (enable_svm)(struct kvm kvm);
316	int (svm_off)(struct kvm kvm);	316	int (svm_off)(struct kvm kvm);
		317	int (enable_dawr1)(struct kvm kvm);
		318	bool (*hash_v3_possible)(void);
317	};	319	};
318		320
319	extern struct kvmppc_ops *kvmppc_hv_ops;	321	extern struct kvmppc_ops *kvmppc_hv_ops;

arch/powerpc/include/uapi/asm/kvm.h+2−0

@@ -644,6 +644,8 @@ struct kvm_ppc_cpu_char { · 643 unmodified lines
644	#define KVM_REG_PPC_MMCR3 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc1)	644	#define KVM_REG_PPC_MMCR3 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc1)
645	#define KVM_REG_PPC_SIER2 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc2)	645	#define KVM_REG_PPC_SIER2 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc2)
646	#define KVM_REG_PPC_SIER3 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc3)	646	#define KVM_REG_PPC_SIER3 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc3)
		647	#define KVM_REG_PPC_DAWR1 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc4)
		648	#define KVM_REG_PPC_DAWRX1 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc5)
647		649
648	/* Transactional Memory checkpointed state:	650	/* Transactional Memory checkpointed state:
649	* This is all GPRs, all VSX regs and a subset of SPRs	651	* This is all GPRs, all VSX regs and a subset of SPRs

arch/powerpc/kernel/asm-offsets.c+4−5

 	OFFSET(VCPU_CTRL, kvm_vcpu, arch.ctrl);
 	OFFSET(VCPU_DABR, kvm_vcpu, arch.dabr);
 	OFFSET(VCPU_DABRX, kvm_vcpu, arch.dabrx);
-	OFFSET(VCPU_DAWR, kvm_vcpu, arch.dawr);
+	OFFSET(VCPU_DAWR0, kvm_vcpu, arch.dawr0);
-	OFFSET(VCPU_DAWRX, kvm_vcpu, arch.dawrx);
+	OFFSET(VCPU_DAWRX0, kvm_vcpu, arch.dawrx0);
+	OFFSET(VCPU_DAWR1, kvm_vcpu, arch.dawr1);
+	OFFSET(VCPU_DAWRX1, kvm_vcpu, arch.dawrx1);
 	OFFSET(VCPU_CIABR, kvm_vcpu, arch.ciabr);
 	OFFSET(VCPU_HFLAGS, kvm_vcpu, arch.hflags);
 	OFFSET(VCPU_DEC, kvm_vcpu, arch.dec);
 	HSTATE_FIELD(HSTATE_SAVED_XIRR, saved_xirr);
 	HSTATE_FIELD(HSTATE_HOST_IPI, host_ipi);
 	HSTATE_FIELD(HSTATE_PTID, ptid);
-	HSTATE_FIELD(HSTATE_TID, tid);
 	HSTATE_FIELD(HSTATE_FAKE_SUSPEND, fake_suspend);
 	HSTATE_FIELD(HSTATE_MMCR0, host_mmcr[0]);
 	HSTATE_FIELD(HSTATE_MMCR1, host_mmcr[1]);
 	OFFSET(KVM_SPLIT_LDBAR, kvm_split_mode, ldbar);
 	OFFSET(KVM_SPLIT_DO_NAP, kvm_split_mode, do_nap);
 	OFFSET(KVM_SPLIT_NAPPED, kvm_split_mode, napped);
-	OFFSET(KVM_SPLIT_DO_SET, kvm_split_mode, do_set);
-	OFFSET(KVM_SPLIT_DO_RESTORE, kvm_split_mode, do_restore);
 #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */
 #ifdef CONFIG_PPC_BOOK3S_64

arch/powerpc/kvm/book3s_hv.c+93−56

 }
 /* If set, the threads on each CPU core have to be in the same MMU mode */
-static bool no_mixing_hpt_and_radix;
+static bool no_mixing_hpt_and_radix __read_mostly;
 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
 			return H_UNSUPPORTED_FLAG_START;
 		if (value2 & DABRX_HYP)
 			return H_P4;
-		vcpu->arch.dawr  = value1;
+		vcpu->arch.dawr0  = value1;
-		vcpu->arch.dawrx = value2;
+		vcpu->arch.dawrx0 = value2;
+		return H_SUCCESS;
+	case H_SET_MODE_RESOURCE_SET_DAWR1:
+		if (!kvmppc_power8_compatible(vcpu))
+			return H_P2;
+		if (!ppc_breakpoint_available())
+			return H_P2;
+		if (!cpu_has_feature(CPU_FTR_DAWR1))
+			return H_P2;
+		if (!vcpu->kvm->arch.dawr1_enabled)
+			return H_FUNCTION;
+		if (mflags)
+			return H_UNSUPPORTED_FLAG_START;
+		if (value2 & DABRX_HYP)
+			return H_P4;
+		vcpu->arch.dawr1  = value1;
+		vcpu->arch.dawrx1 = value2;
 		return H_SUCCESS;
 	case H_SET_MODE_RESOURCE_ADDR_TRANS_MODE:
 		/* KVM does not support mflags=2 (AIL=2) */
 		*val = get_reg_val(id, vcpu->arch.vcore->vtb);
 		break;
 	case KVM_REG_PPC_DAWR:
-		*val = get_reg_val(id, vcpu->arch.dawr);
+		*val = get_reg_val(id, vcpu->arch.dawr0);
 		break;
 	case KVM_REG_PPC_DAWRX:
-		*val = get_reg_val(id, vcpu->arch.dawrx);
+		*val = get_reg_val(id, vcpu->arch.dawrx0);
+		break;
+	case KVM_REG_PPC_DAWR1:
+		*val = get_reg_val(id, vcpu->arch.dawr1);
+		break;
+	case KVM_REG_PPC_DAWRX1:
+		*val = get_reg_val(id, vcpu->arch.dawrx1);
 		break;
 	case KVM_REG_PPC_CIABR:
 		*val = get_reg_val(id, vcpu->arch.ciabr);
 		vcpu->arch.vcore->vtb = set_reg_val(id, *val);
 		break;
 	case KVM_REG_PPC_DAWR:
-		vcpu->arch.dawr = set_reg_val(id, *val);
+		vcpu->arch.dawr0 = set_reg_val(id, *val);
 		break;
 	case KVM_REG_PPC_DAWRX:
-		vcpu->arch.dawrx = set_reg_val(id, *val) & ~DAWRX_HYP;
+		vcpu->arch.dawrx0 = set_reg_val(id, *val) & ~DAWRX_HYP;
+		break;
+	case KVM_REG_PPC_DAWR1:
+		vcpu->arch.dawr1 = set_reg_val(id, *val);
+		break;
+	case KVM_REG_PPC_DAWRX1:
+		vcpu->arch.dawrx1 = set_reg_val(id, *val) & ~DAWRX_HYP;
 		break;
 	case KVM_REG_PPC_CIABR:
 		vcpu->arch.ciabr = set_reg_val(id, *val);
 	if (one_vm_per_core && vc->kvm != cip->vc[0]->kvm)
 		return false;
-	/* Some POWER9 chips require all threads to be in the same MMU mode */
-	if (no_mixing_hpt_and_radix &&
-	    kvm_is_radix(vc->kvm) != kvm_is_radix(cip->vc[0]->kvm))
-		return false;
 	if (n_threads < cip->max_subcore_threads)
 		n_threads = cip->max_subcore_threads;
 	if (!subcore_config_ok(cip->n_subcores + 1, n_threads))
 	for_each_runnable_thread(i, vcpu, vc) {
 		if (signal_pending(vcpu->arch.run_task))
 			vcpu->arch.ret = -EINTR;
+		else if (no_mixing_hpt_and_radix &&
+			 kvm_is_radix(vc->kvm) != radix_enabled())
+			vcpu->arch.ret = -EINVAL;
 		else if (vcpu->arch.vpa.update_pending ||
 			 vcpu->arch.slb_shadow.update_pending ||
 			 vcpu->arch.dtl.update_pending)
 	int controlled_threads;
 	int trap;
 	bool is_power8;
-	bool hpt_on_radix;
 	/*
 	 * Remove from the list any threads that have a signal pending
 	 * this is a HPT guest on a radix host machine where the
 	 * CPU threads may not be in different MMU modes.
 	 */
-	hpt_on_radix = no_mixing_hpt_and_radix && radix_enabled() &&
+	if ((controlled_threads > 1) &&
-		!kvm_is_radix(vc->kvm);
+	    ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) {
-	if (((controlled_threads > 1) &&
-	     ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) ||
-	    (hpt_on_radix && vc->kvm->arch.threads_indep)) {
 		for_each_runnable_thread(i, vcpu, vc) {
 			vcpu->arch.ret = -EBUSY;
 			kvmppc_remove_runnable(vc, vcpu);
 	is_power8 = cpu_has_feature(CPU_FTR_ARCH_207S)
 		&& !cpu_has_feature(CPU_FTR_ARCH_300);
-	if (split > 1 || hpt_on_radix) {
+	if (split > 1) {
 		sip = &split_info;
 		memset(&split_info, 0, sizeof(split_info));
 		for (sub = 0; sub < core_info.n_subcores; ++sub)
 			split_info.subcore_size = subcore_size;
 		} else {
 			split_info.subcore_size = 1;
-			if (hpt_on_radix) {
-				/* Use the split_info for LPCR/LPIDR changes */
-				split_info.lpcr_req = vc->lpcr;
-				split_info.lpidr_req = vc->kvm->arch.lpid;
-				split_info.host_lpcr = vc->kvm->arch.host_lpcr;
-				split_info.do_set = 1;
-			}
 		}
 		/* order writes to split_info before kvm_split_mode pointer */
 	for (thr = 0; thr < controlled_threads; ++thr) {
 		struct paca_struct *paca = paca_ptrs[pcpu + thr];
-		paca->kvm_hstate.tid = thr;
 		paca->kvm_hstate.napping = 0;
 		paca->kvm_hstate.kvm_split_mode = sip;
 	}
 	 * When doing micro-threading, poke the inactive threads as well.
 	 * This gets them to the nap instruction after kvm_do_nap,
 	 * which reduces the time taken to unsplit later.
-	 * For POWER9 HPT guest on radix host, we need all the secondary
-	 * threads woken up so they can do the LPCR/LPIDR change.
 	 */
-	if (cmd_bit || hpt_on_radix) {
+	if (cmd_bit) {
 		split_info.do_nap = 1;	/* ask secondaries to nap when done */
 		for (thr = 1; thr < threads_per_subcore; ++thr)
 			if (!(active & (1 << thr)))
 			cpu_relax();
 			++loops;
 		}
-	} else if (hpt_on_radix) {
+		split_info.do_nap = 0;
-		/* Wait for all threads to have seen final sync */
-		for (thr = 1; thr < controlled_threads; ++thr) {
-			struct paca_struct *paca = paca_ptrs[pcpu + thr];
-			while (paca->kvm_hstate.kvm_split_mode) {
-				HMT_low();
-				barrier();
-			}
-			HMT_medium();
-		}
 	}
-	split_info.do_nap = 0;
 	kvmppc_set_host_core(pcpu);
 	int trap;
 	unsigned long host_hfscr = mfspr(SPRN_HFSCR);
 	unsigned long host_ciabr = mfspr(SPRN_CIABR);
-	unsigned long host_dawr = mfspr(SPRN_DAWR0);
+	unsigned long host_dawr0 = mfspr(SPRN_DAWR0);
-	unsigned long host_dawrx = mfspr(SPRN_DAWRX0);
+	unsigned long host_dawrx0 = mfspr(SPRN_DAWRX0);
 	unsigned long host_psscr = mfspr(SPRN_PSSCR);
 	unsigned long host_pidr = mfspr(SPRN_PID);
+	unsigned long host_dawr1 = 0;
+	unsigned long host_dawrx1 = 0;
+	if (cpu_has_feature(CPU_FTR_DAWR1)) {
+		host_dawr1 = mfspr(SPRN_DAWR1);
+		host_dawrx1 = mfspr(SPRN_DAWRX1);
+	}
 	/*
 	 * P8 and P9 suppress the HDEC exception when LPCR[HDICE] = 0,
 	mtspr(SPRN_SPURR, vcpu->arch.spurr);
 	if (dawr_enabled()) {
-		mtspr(SPRN_DAWR0, vcpu->arch.dawr);
+		mtspr(SPRN_DAWR0, vcpu->arch.dawr0);
-		mtspr(SPRN_DAWRX0, vcpu->arch.dawrx);
+		mtspr(SPRN_DAWRX0, vcpu->arch.dawrx0);
+		if (cpu_has_feature(CPU_FTR_DAWR1)) {
+			mtspr(SPRN_DAWR1, vcpu->arch.dawr1);
+			mtspr(SPRN_DAWRX1, vcpu->arch.dawrx1);
+		}
 	}
 	mtspr(SPRN_CIABR, vcpu->arch.ciabr);
 	mtspr(SPRN_IC, vcpu->arch.ic);
 	      (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG));
 	mtspr(SPRN_HFSCR, host_hfscr);
 	mtspr(SPRN_CIABR, host_ciabr);
-	mtspr(SPRN_DAWR0, host_dawr);
+	mtspr(SPRN_DAWR0, host_dawr0);
-	mtspr(SPRN_DAWRX0, host_dawrx);
+	mtspr(SPRN_DAWRX0, host_dawrx0);
+	if (cpu_has_feature(CPU_FTR_DAWR1)) {
+		mtspr(SPRN_DAWR1, host_dawr1);
+		mtspr(SPRN_DAWRX1, host_dawrx1);
+	}
 	mtspr(SPRN_PID, host_pidr);
 	/*
 	unsigned long host_tidr = mfspr(SPRN_TIDR);
 	unsigned long host_iamr = mfspr(SPRN_IAMR);
 	unsigned long host_amr = mfspr(SPRN_AMR);
+	unsigned long host_fscr = mfspr(SPRN_FSCR);
 	s64 dec;
 	u64 tb;
 	int trap, save_pmu;
 	if (host_amr != vcpu->arch.amr)
 		mtspr(SPRN_AMR, host_amr);
+	if (host_fscr != vcpu->arch.fscr)
+		mtspr(SPRN_FSCR, host_fscr);
 	msr_check_and_set(MSR_FP | MSR_VEC | MSR_VSX);
 	store_fp_state(&vcpu->arch.fp);
 #ifdef CONFIG_ALTIVEC
 	kvmppc_clear_host_core(pcpu);
-	local_paca->kvm_hstate.tid = 0;
 	local_paca->kvm_hstate.napping = 0;
 	local_paca->kvm_hstate.kvm_split_mode = NULL;
 	kvmppc_start_thread(vcpu, vc);
 	do {
 		/*
-		 * The early POWER9 chips that can't mix radix and HPT threads
+		 * The TLB prefetch bug fixup is only in the kvmppc_run_vcpu
-		 * on the same core also need the workaround for the problem
+		 * path, which also handles hash and dependent threads mode.
-		 * where the TLB would prefetch entries in the guest exit path
-		 * for radix guests using the guest PIDR value and LPID 0.
-		 * The workaround is in the old path (kvmppc_run_vcpu())
-		 * but not the new path (kvmhv_run_single_vcpu()).
 		 */
 		if (kvm->arch.threads_indep && kvm_is_radix(kvm) &&
-		    !no_mixing_hpt_and_radix)
+		    !cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG))
 			r = kvmhv_run_single_vcpu(vcpu, ~(u64)0,
 						  vcpu->arch.vcore->lpcr);
 		else
 	return ret;
 }
+static int kvmhv_enable_dawr1(struct kvm *kvm)
+{
+	if (!cpu_has_feature(CPU_FTR_DAWR1))
+		return -ENODEV;
+	/* kvm == NULL means the caller is testing if the capability exists */
+	if (kvm)
+		kvm->arch.dawr1_enabled = true;
+	return 0;
+}
+static bool kvmppc_hash_v3_possible(void)
+{
+	if (radix_enabled() && no_mixing_hpt_and_radix)
+		return false;
+	return cpu_has_feature(CPU_FTR_ARCH_300) &&
+		cpu_has_feature(CPU_FTR_HVMODE);
+}
 static struct kvmppc_ops kvm_ops_hv = {
 	.get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv,
 	.set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv,
 	.store_to_eaddr = kvmhv_store_to_eaddr,
 	.enable_svm = kvmhv_enable_svm,
 	.svm_off = kvmhv_svm_off,
+	.enable_dawr1 = kvmhv_enable_dawr1,
+	.hash_v3_possible = kvmppc_hash_v3_possible,
 };
 static int kvm_init_subcore_bitmap(void)

arch/powerpc/kvm/book3s_hv_builtin.c+1−107

@@ -277,8 +277,7 @@ void kvmhv_commence_exit(int trap) · 276 unmodified lines
277	struct kvmppc_vcore *vc = local_paca->kvm_hstate.kvm_vcore;	277	struct kvmppc_vcore *vc = local_paca->kvm_hstate.kvm_vcore;
278	int ptid = local_paca->kvm_hstate.ptid;	278	int ptid = local_paca->kvm_hstate.ptid;
279	struct kvm_split_mode *sip = local_paca->kvm_hstate.kvm_split_mode;	279	struct kvm_split_mode *sip = local_paca->kvm_hstate.kvm_split_mode;
280	int me, ee, i, t;	280	int me, ee, i;
281	int cpu0;
282		281
283	/* Set our bit in the threads-exiting-guest map in the 0xff00	282	/* Set our bit in the threads-exiting-guest map in the 0xff00
284	bits of vcore->entry_exit_map */	283	bits of vcore->entry_exit_map */
@@ -320,22 +319,6 @@ void kvmhv_commence_exit(int trap) · 35 unmodified lines
320	if ((ee >> 8) == 0)	319	if ((ee >> 8) == 0)
321	kvmhv_interrupt_vcore(vc, ee);	320	kvmhv_interrupt_vcore(vc, ee);
322	}	321	}
323
324	/*
325	* On POWER9 when running a HPT guest on a radix host (sip != NULL),
326	* we have to interrupt inactive CPU threads to get them to
327	* restore the host LPCR value.
328	*/
329	if (sip->lpcr_req) {
330	if (cmpxchg(&sip->do_restore, 0, 1) == 0) {
331	vc = local_paca->kvm_hstate.kvm_vcore;
332	cpu0 = vc->pcpu + ptid - local_paca->kvm_hstate.tid;
333	for (t = 1; t < threads_per_core; ++t) {
334	if (sip->napped[t])
335	kvmhv_rm_send_ipi(cpu0 + t);
336	}
337	}
338	}
339	}	322	}
340		323
341	struct kvmppc_host_rm_ops *kvmppc_host_rm_ops_hv;	324	struct kvmppc_host_rm_ops *kvmppc_host_rm_ops_hv;
@@ -667,95 +650,6 @@ void kvmppc_bad_interrupt(struct pt_regs *regs) · 325 unmodified lines
667	panic("Bad KVM trap");	650	panic("Bad KVM trap");
668	}	651	}
669		652
670	/*
671	* Functions used to switch LPCR HR and UPRT bits on all threads
672	* when entering and exiting HPT guests on a radix host.
673	*/
674
675	#define PHASE_REALMODE 1 /* in real mode */
676	#define PHASE_SET_LPCR 2 /* have set LPCR */
677	#define PHASE_OUT_OF_GUEST 4 /* have finished executing in guest */
678	#define PHASE_RESET_LPCR 8 /* have reset LPCR to host value */
679
680	#define ALL(p) (((p) << 24) \| ((p) << 16) \| ((p) << 8) \| (p))
681
682	static void wait_for_sync(struct kvm_split_mode *sip, int phase)
683	{
684	int thr = local_paca->kvm_hstate.tid;
685
686	sip->lpcr_sync.phase[thr] \|= phase;
687	phase = ALL(phase);
688	while ((sip->lpcr_sync.allphases & phase) != phase) {
689	HMT_low();
690	barrier();
691	}
692	HMT_medium();
693	}
694
695	void kvmhv_p9_set_lpcr(struct kvm_split_mode *sip)
696	{
697	int num_sets;
698	unsigned long rb, set;
699
700	/* wait for every other thread to get to real mode */
701	wait_for_sync(sip, PHASE_REALMODE);
702
703	/* Set LPCR and LPIDR */
704	mtspr(SPRN_LPCR, sip->lpcr_req);
705	mtspr(SPRN_LPID, sip->lpidr_req);
706	isync();
707
708	/*
709	* P10 will flush all the congruence class with a single tlbiel
710	*/
711	if (cpu_has_feature(CPU_FTR_ARCH_31))
712	num_sets = 1;
713	else
714	num_sets = POWER9_TLB_SETS_RADIX;
715
716	/* Invalidate the TLB on thread 0 */
717	if (local_paca->kvm_hstate.tid == 0) {
718	sip->do_set = 0;
719	asm volatile("ptesync" : : : "memory");
720	for (set = 0; set < num_sets; ++set) {
721	rb = TLBIEL_INVAL_SET_LPID +
722	(set << TLBIEL_INVAL_SET_SHIFT);
723	asm volatile(PPC_TLBIEL(%0, %1, 0, 0, 0) : :
724	"r" (rb), "r" (0));
725	}
726	asm volatile("ptesync" : : : "memory");
727	}
728
729	/* indicate that we have done so and wait for others */
730	wait_for_sync(sip, PHASE_SET_LPCR);
731	/* order read of sip->lpcr_sync.allphases vs. sip->do_set */
732	smp_rmb();
733	}
734
735	/*
736	* Called when a thread that has been in the guest needs
737	* to reload the host LPCR value - but only on POWER9 when
738	* running a HPT guest on a radix host.
739	*/
740	void kvmhv_p9_restore_lpcr(struct kvm_split_mode *sip)
741	{
742	/* we're out of the guest... */
743	wait_for_sync(sip, PHASE_OUT_OF_GUEST);
744
745	mtspr(SPRN_LPID, 0);
746	mtspr(SPRN_LPCR, sip->host_lpcr);
747	isync();
748
749	if (local_paca->kvm_hstate.tid == 0) {
750	sip->do_restore = 0;
751	smp_wmb(); /* order store of do_restore vs. phase */
752	}
753
754	wait_for_sync(sip, PHASE_RESET_LPCR);
755	smp_mb();
756	local_paca->kvm_hstate.kvm_split_mode = NULL;
757	}
758
759	static void kvmppc_end_cede(struct kvm_vcpu *vcpu)	653	static void kvmppc_end_cede(struct kvm_vcpu *vcpu)
760	{	654	{
761	vcpu->arch.ceded = 0;	655	vcpu->arch.ceded = 0;

arch/powerpc/kvm/book3s_hv_nested.c+56−14

 	hr->dpdes = vc->dpdes;
 	hr->hfscr = vcpu->arch.hfscr;
 	hr->tb_offset = vc->tb_offset;
-	hr->dawr0 = vcpu->arch.dawr;
+	hr->dawr0 = vcpu->arch.dawr0;
-	hr->dawrx0 = vcpu->arch.dawrx;
+	hr->dawrx0 = vcpu->arch.dawrx0;
 	hr->ciabr = vcpu->arch.ciabr;
 	hr->purr = vcpu->arch.purr;
 	hr->spurr = vcpu->arch.spurr;
 	hr->pidr = vcpu->arch.pid;
 	hr->cfar = vcpu->arch.cfar;
 	hr->ppr = vcpu->arch.ppr;
+	hr->dawr1 = vcpu->arch.dawr1;
+	hr->dawrx1 = vcpu->arch.dawrx1;
 }
 static void byteswap_pt_regs(struct pt_regs *regs)
 	hr->pidr = swab64(hr->pidr);
 	hr->cfar = swab64(hr->cfar);
 	hr->ppr = swab64(hr->ppr);
+	hr->dawr1 = swab64(hr->dawr1);
+	hr->dawrx1 = swab64(hr->dawrx1);
 }
 static void save_hv_return_state(struct kvm_vcpu *vcpu, int trap,
 	/* Don't let data address watchpoint match in hypervisor state */
 	hr->dawrx0 &= ~DAWRX_HYP;
+	hr->dawrx1 &= ~DAWRX_HYP;
 	/* Don't let completed instruction address breakpt match in HV state */
 	if ((hr->ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER)
 	vc->pcr = hr->pcr | PCR_MASK;
 	vc->dpdes = hr->dpdes;
 	vcpu->arch.hfscr = hr->hfscr;
-	vcpu->arch.dawr = hr->dawr0;
+	vcpu->arch.dawr0 = hr->dawr0;
-	vcpu->arch.dawrx = hr->dawrx0;
+	vcpu->arch.dawrx0 = hr->dawrx0;
 	vcpu->arch.ciabr = hr->ciabr;
 	vcpu->arch.purr = hr->purr;
 	vcpu->arch.spurr = hr->spurr;
 	vcpu->arch.pid = hr->pidr;
 	vcpu->arch.cfar = hr->cfar;
 	vcpu->arch.ppr = hr->ppr;
+	vcpu->arch.dawr1 = hr->dawr1;
+	vcpu->arch.dawrx1 = hr->dawrx1;
 }
 void kvmhv_restore_hv_return_state(struct kvm_vcpu *vcpu,
 	}
 }
+static int kvmhv_read_guest_state_and_regs(struct kvm_vcpu *vcpu,
+					   struct hv_guest_state *l2_hv,
+					   struct pt_regs *l2_regs,
+					   u64 hv_ptr, u64 regs_ptr)
+{
+	int size;
+	if (kvm_vcpu_read_guest(vcpu, hv_ptr, &l2_hv->version,
+				sizeof(l2_hv->version)))
+		return -1;
+	if (kvmppc_need_byteswap(vcpu))
+		l2_hv->version = swab64(l2_hv->version);
+	size = hv_guest_state_size(l2_hv->version);
+	if (size < 0)
+		return -1;
+	return kvm_vcpu_read_guest(vcpu, hv_ptr, l2_hv, size) ||
+		kvm_vcpu_read_guest(vcpu, regs_ptr, l2_regs,
+				    sizeof(struct pt_regs));
+}
+static int kvmhv_write_guest_state_and_regs(struct kvm_vcpu *vcpu,
+					    struct hv_guest_state *l2_hv,
+					    struct pt_regs *l2_regs,
+					    u64 hv_ptr, u64 regs_ptr)
+{
+	int size;
+	size = hv_guest_state_size(l2_hv->version);
+	if (size < 0)
+		return -1;
+	return kvm_vcpu_write_guest(vcpu, hv_ptr, l2_hv, size) ||
+		kvm_vcpu_write_guest(vcpu, regs_ptr, l2_regs,
+				     sizeof(struct pt_regs));
+}
 long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
 {
 	long int err, r;
 	struct kvm_nested_guest *l2;
 	struct pt_regs l2_regs, saved_l1_regs;
-	struct hv_guest_state l2_hv, saved_l1_hv;
+	struct hv_guest_state l2_hv = {0}, saved_l1_hv;
 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
 	u64 hv_ptr, regs_ptr;
 	u64 hdec_exp;
 	hv_ptr = kvmppc_get_gpr(vcpu, 4);
 	regs_ptr = kvmppc_get_gpr(vcpu, 5);
 	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
-	err = kvm_vcpu_read_guest(vcpu, hv_ptr, &l2_hv,
+	err = kvmhv_read_guest_state_and_regs(vcpu, &l2_hv, &l2_regs,
-				  sizeof(struct hv_guest_state)) ||
+					      hv_ptr, regs_ptr);
-		kvm_vcpu_read_guest(vcpu, regs_ptr, &l2_regs,
-				    sizeof(struct pt_regs));
 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
 	if (err)
 		return H_PARAMETER;
 	if (kvmppc_need_byteswap(vcpu))
 		byteswap_hv_regs(&l2_hv);
-	if (l2_hv.version != HV_GUEST_STATE_VERSION)
+	if (l2_hv.version > HV_GUEST_STATE_VERSION)
 		return H_P2;
 	if (kvmppc_need_byteswap(vcpu))
 		byteswap_pt_regs(&l2_regs);
 	}
 	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
-	err = kvm_vcpu_write_guest(vcpu, hv_ptr, &l2_hv,
+	err = kvmhv_write_guest_state_and_regs(vcpu, &l2_hv, &l2_regs,
-				   sizeof(struct hv_guest_state)) ||
+					       hv_ptr, regs_ptr);
-		kvm_vcpu_write_guest(vcpu, regs_ptr, &l2_regs,
-				   sizeof(struct pt_regs));
 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
 	if (err)
 		return H_AUTHORITY;

arch/powerpc/kvm/book3s_hv_rmhandlers.S+95−80

@@ -52,11 +52,13 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) · 51 unmodified lines
52	#define STACK_SLOT_PID (SFS-32)	52	#define STACK_SLOT_PID (SFS-32)
53	#define STACK_SLOT_IAMR (SFS-40)	53	#define STACK_SLOT_IAMR (SFS-40)
54	#define STACK_SLOT_CIABR (SFS-48)	54	#define STACK_SLOT_CIABR (SFS-48)
55	#define STACK_SLOT_DAWR (SFS-56)	55	#define STACK_SLOT_DAWR0 (SFS-56)
56	#define STACK_SLOT_DAWRX (SFS-64)	56	#define STACK_SLOT_DAWRX0 (SFS-64)
57	#define STACK_SLOT_HFSCR (SFS-72)	57	#define STACK_SLOT_HFSCR (SFS-72)
58	#define STACK_SLOT_AMR (SFS-80)	58	#define STACK_SLOT_AMR (SFS-80)
59	#define STACK_SLOT_UAMOR (SFS-88)	59	#define STACK_SLOT_UAMOR (SFS-88)
		60	#define STACK_SLOT_DAWR1 (SFS-96)
		61	#define STACK_SLOT_DAWRX1 (SFS-104)
60	/* the following is used by the P9 short path */	62	/* the following is used by the P9 short path */
61	#define STACK_SLOT_NVGPRS (SFS-152) /* 18 gprs */	63	#define STACK_SLOT_NVGPRS (SFS-152) /* 18 gprs */
62		64
@@ -85,19 +87,6 @@ _GLOBAL_TOC(kvmppc_hv_entry_trampoline) · 22 unmodified lines
85	RFI_TO_KERNEL	87	RFI_TO_KERNEL
86		88
87	kvmppc_call_hv_entry:	89	kvmppc_call_hv_entry:
88	BEGIN_FTR_SECTION
89	/* On P9, do LPCR setting, if necessary */
90	ld r3, HSTATE_SPLIT_MODE(r13)
91	cmpdi r3, 0
92	beq 46f
93	lwz r4, KVM_SPLIT_DO_SET(r3)
94	cmpwi r4, 0
95	beq 46f
96	bl kvmhv_p9_set_lpcr
97	nop
98	46:
99	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
100
101	ld r4, HSTATE_KVM_VCPU(r13)	90	ld r4, HSTATE_KVM_VCPU(r13)
102	bl kvmppc_hv_entry	91	bl kvmppc_hv_entry
103		92
@@ -361,11 +350,11 @@ kvm_secondary_got_guest: · 257 unmodified lines
361	LOAD_REG_ADDR(r6, decrementer_max)	350	LOAD_REG_ADDR(r6, decrementer_max)
362	ld r6, 0(r6)	351	ld r6, 0(r6)
363	mtspr SPRN_HDEC, r6	352	mtspr SPRN_HDEC, r6
		353	BEGIN_FTR_SECTION
364	/* and set per-LPAR registers, if doing dynamic micro-threading */	354	/* and set per-LPAR registers, if doing dynamic micro-threading */
365	ld r6, HSTATE_SPLIT_MODE(r13)	355	ld r6, HSTATE_SPLIT_MODE(r13)
366	cmpdi r6, 0	356	cmpdi r6, 0
367	beq 63f	357	beq 63f
368	BEGIN_FTR_SECTION
369	ld r0, KVM_SPLIT_RPR(r6)	358	ld r0, KVM_SPLIT_RPR(r6)
370	mtspr SPRN_RPR, r0	359	mtspr SPRN_RPR, r0
371	ld r0, KVM_SPLIT_PMMAR(r6)	360	ld r0, KVM_SPLIT_PMMAR(r6)
@@ -373,16 +362,7 @@ BEGIN_FTR_SECTION · 1 unmodified line
373	ld r0, KVM_SPLIT_LDBAR(r6)	362	ld r0, KVM_SPLIT_LDBAR(r6)
374	mtspr SPRN_LDBAR, r0	363	mtspr SPRN_LDBAR, r0
375	isync	364	isync
376	FTR_SECTION_ELSE	365	END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
377	/* On P9 we use the split_info for coordinating LPCR changes */
378	lwz r4, KVM_SPLIT_DO_SET(r6)
379	cmpwi r4, 0
380	beq 1f
381	mr r3, r6
382	bl kvmhv_p9_set_lpcr
383	nop
384	1:
385	ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
386	63:	366	63:
387	/* Order load of vcpu after load of vcore */	367	/* Order load of vcpu after load of vcore */
388	lwsync	368	lwsync
@@ -452,19 +432,15 @@ kvm_no_guest: · 63 unmodified lines
452	mtcr r5	432	mtcr r5
453	blr	433	blr
454		434
455	53: HMT_LOW	435	53:
		436	BEGIN_FTR_SECTION
		437	HMT_LOW
456	ld r5, HSTATE_KVM_VCORE(r13)	438	ld r5, HSTATE_KVM_VCORE(r13)
457	cmpdi r5, 0	439	cmpdi r5, 0
458	bne 60f	440	bne 60f
459	ld r3, HSTATE_SPLIT_MODE(r13)	441	ld r3, HSTATE_SPLIT_MODE(r13)
460	cmpdi r3, 0	442	cmpdi r3, 0
461	beq kvm_no_guest	443	beq kvm_no_guest
462	lwz r0, KVM_SPLIT_DO_SET(r3)
463	cmpwi r0, 0
464	bne kvmhv_do_set
465	lwz r0, KVM_SPLIT_DO_RESTORE(r3)
466	cmpwi r0, 0
467	bne kvmhv_do_restore
468	lbz r0, KVM_SPLIT_DO_NAP(r3)	444	lbz r0, KVM_SPLIT_DO_NAP(r3)
469	cmpwi r0, 0	445	cmpwi r0, 0
470	beq kvm_no_guest	446	beq kvm_no_guest
@@ -472,24 +448,19 @@ kvm_no_guest: · 1 unmodified line
472	b kvm_unsplit_nap	448	b kvm_unsplit_nap
473	60: HMT_MEDIUM	449	60: HMT_MEDIUM
474	b kvm_secondary_got_guest	450	b kvm_secondary_got_guest
		451	FTR_SECTION_ELSE
		452	HMT_LOW
		453	ld r5, HSTATE_KVM_VCORE(r13)
		454	cmpdi r5, 0
		455	beq kvm_no_guest
		456	HMT_MEDIUM
		457	b kvm_secondary_got_guest
		458	ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
475		459
476	54: li r0, KVM_HWTHREAD_IN_KVM	460	54: li r0, KVM_HWTHREAD_IN_KVM
477	stb r0, HSTATE_HWTHREAD_STATE(r13)	461	stb r0, HSTATE_HWTHREAD_STATE(r13)
478	b kvm_no_guest	462	b kvm_no_guest
479		463
480	kvmhv_do_set:
481	/* Set LPCR, LPIDR etc. on P9 */
482	HMT_MEDIUM
483	bl kvmhv_p9_set_lpcr
484	nop
485	b kvm_no_guest
486
487	kvmhv_do_restore:
488	HMT_MEDIUM
489	bl kvmhv_p9_restore_lpcr
490	nop
491	b kvm_no_guest
492
493	/*	464	/*
494	* Here the primary thread is trying to return the core to	465	* Here the primary thread is trying to return the core to
495	* whole-core mode, so we need to nap.	466	* whole-core mode, so we need to nap.
@@ -527,7 +498,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) · 31 unmodified lines
527	/* Set kvm_split_mode.napped[tid] = 1 */	498	/* Set kvm_split_mode.napped[tid] = 1 */
528	ld r3, HSTATE_SPLIT_MODE(r13)	499	ld r3, HSTATE_SPLIT_MODE(r13)
529	li r0, 1	500	li r0, 1
530	lbz r4, HSTATE_TID(r13)	501	lhz r4, PACAPACAINDEX(r13)
		502	clrldi r4, r4, 61 /* micro-threading => P8 => 8 threads/core */
531	addi r4, r4, KVM_SPLIT_NAPPED	503	addi r4, r4, KVM_SPLIT_NAPPED
532	stbx r0, r3, r4	504	stbx r0, r3, r4
533	/* Check the do_nap flag again after setting napped[] */	505	/* Check the do_nap flag again after setting napped[] */
@@ -711,10 +683,16 @@ BEGIN_FTR_SECTION · 177 unmodified lines
711	mfspr r7, SPRN_DAWRX0	683	mfspr r7, SPRN_DAWRX0
712	mfspr r8, SPRN_IAMR	684	mfspr r8, SPRN_IAMR
713	std r5, STACK_SLOT_CIABR(r1)	685	std r5, STACK_SLOT_CIABR(r1)
714	std r6, STACK_SLOT_DAWR(r1)	686	std r6, STACK_SLOT_DAWR0(r1)
715	std r7, STACK_SLOT_DAWRX(r1)	687	std r7, STACK_SLOT_DAWRX0(r1)
716	std r8, STACK_SLOT_IAMR(r1)	688	std r8, STACK_SLOT_IAMR(r1)
717	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)	689	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
		690	BEGIN_FTR_SECTION
		691	mfspr r6, SPRN_DAWR1
		692	mfspr r7, SPRN_DAWRX1
		693	std r6, STACK_SLOT_DAWR1(r1)
		694	std r7, STACK_SLOT_DAWRX1(r1)
		695	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S \| CPU_FTR_DAWR1)
718		696
719	mfspr r5, SPRN_AMR	697	mfspr r5, SPRN_AMR
720	std r5, STACK_SLOT_AMR(r1)	698	std r5, STACK_SLOT_AMR(r1)
@@ -801,10 +779,16 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) · 80 unmodified lines
801	lbz r5, 0(r5)	779	lbz r5, 0(r5)
802	cmpdi r5, 0	780	cmpdi r5, 0
803	beq 1f	781	beq 1f
804	ld r5, VCPU_DAWR(r4)	782	ld r5, VCPU_DAWR0(r4)
805	ld r6, VCPU_DAWRX(r4)	783	ld r6, VCPU_DAWRX0(r4)
806	mtspr SPRN_DAWR0, r5	784	mtspr SPRN_DAWR0, r5
807	mtspr SPRN_DAWRX0, r6	785	mtspr SPRN_DAWRX0, r6
		786	BEGIN_FTR_SECTION
		787	ld r5, VCPU_DAWR1(r4)
		788	ld r6, VCPU_DAWRX1(r4)
		789	mtspr SPRN_DAWR1, r5
		790	mtspr SPRN_DAWRX1, r6
		791	END_FTR_SECTION_IFSET(CPU_FTR_DAWR1)
808	1:	792	1:
809	ld r7, VCPU_CIABR(r4)	793	ld r7, VCPU_CIABR(r4)
810	ld r8, VCPU_TAR(r4)	794	ld r8, VCPU_TAR(r4)
@@ -918,15 +902,19 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) · 107 unmodified lines
918	cmpdi r3, 512 /* 1 microsecond */	902	cmpdi r3, 512 /* 1 microsecond */
919	blt hdec_soon	903	blt hdec_soon
920		904
921	/* For hash guest, clear out and reload the SLB */
922	ld r6, VCPU_KVM(r4)	905	ld r6, VCPU_KVM(r4)
923	lbz r0, KVM_RADIX(r6)	906	lbz r0, KVM_RADIX(r6)
924	cmpwi r0, 0	907	cmpwi r0, 0
925	bne 9f	908	bne 9f
		909
		910	/* For hash guest, clear out and reload the SLB */
		911	BEGIN_MMU_FTR_SECTION
		912	/* Radix host won't have populated the SLB, so no need to clear */
926	li r6, 0	913	li r6, 0
927	slbmte r6, r6	914	slbmte r6, r6
928	slbia	915	PPC_SLBIA(6)
929	ptesync	916	ptesync
		917	END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX)
930		918
931	/* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */	919	/* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */
932	lwz r5,VCPU_SLB_MAX(r4)	920	lwz r5,VCPU_SLB_MAX(r4)
@@ -1187,6 +1175,20 @@ EXPORT_SYMBOL_GPL(__kvmhv_vcpu_entry_p9) · 254 unmodified lines
1187	mr r4, r3	1175	mr r4, r3
1188	b fast_guest_entry_c	1176	b fast_guest_entry_c
1189	guest_exit_short_path:	1177	guest_exit_short_path:
		1178	/*
		1179	* Malicious or buggy radix guests may have inserted SLB entries
		1180	* (only 0..3 because radix always runs with UPRT=1), so these must
		1181	* be cleared here to avoid side-channels. slbmte is used rather
		1182	* than slbia, as it won't clear cached translations.
		1183	*/
		1184	li r0,0
		1185	slbmte r0,r0
		1186	li r4,1
		1187	slbmte r0,r4
		1188	li r4,2
		1189	slbmte r0,r4
		1190	li r4,3
		1191	slbmte r0,r4
1190		1192
1191	li r0, KVM_GUEST_MODE_NONE	1193	li r0, KVM_GUEST_MODE_NONE
1192	stb r0, HSTATE_IN_GUEST(r13)	1194	stb r0, HSTATE_IN_GUEST(r13)
@@ -1499,7 +1501,7 @@ guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ · 306 unmodified lines
1499	lbz r0, KVM_RADIX(r5)	1501	lbz r0, KVM_RADIX(r5)
1500	li r5, 0	1502	li r5, 0
1501	cmpwi r0, 0	1503	cmpwi r0, 0
1502	bne 3f /* for radix, save 0 entries */	1504	bne 0f /* for radix, save 0 entries */
1503	lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */	1505	lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */
1504	mtctr r0	1506	mtctr r0
1505	li r6,0	1507	li r6,0
@@ -1518,13 +1520,13 @@ guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ · 12 unmodified lines
1518	/* Finally clear out the SLB */	1520	/* Finally clear out the SLB */
1519	li r0,0	1521	li r0,0
1520	slbmte r0,r0	1522	slbmte r0,r0
1521	slbia	1523	PPC_SLBIA(6)
1522	ptesync	1524	ptesync
1523	3: stw r5,VCPU_SLB_MAX(r9)	1525	stw r5,VCPU_SLB_MAX(r9)
1524		1526
1525	/* load host SLB entries */	1527	/* load host SLB entries */
1526	BEGIN_MMU_FTR_SECTION	1528	BEGIN_MMU_FTR_SECTION
1527	b 0f	1529	b guest_bypass
1528	END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)	1530	END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
1529	ld r8,PACA_SLBSHADOWPTR(r13)	1531	ld r8,PACA_SLBSHADOWPTR(r13)
1530		1532
@@ -1538,7 +1540,21 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) · 7 unmodified lines
1538	slbmte r6,r5	1540	slbmte r6,r5
1539	1: addi r8,r8,16	1541	1: addi r8,r8,16
1540	.endr	1542	.endr
1541	0:	1543	b guest_bypass
		1544
		1545	0: /*
		1546	* Sanitise radix guest SLB, see guest_exit_short_path comment.
		1547	* We clear vcpu->arch.slb_max to match earlier behaviour.
		1548	*/
		1549	li r0,0
		1550	stw r0,VCPU_SLB_MAX(r9)
		1551	slbmte r0,r0
		1552	li r4,1
		1553	slbmte r0,r4
		1554	li r4,2
		1555	slbmte r0,r4
		1556	li r4,3
		1557	slbmte r0,r4
1542		1558
1543	guest_bypass:	1559	guest_bypass:
1544	stw r12, STACK_SLOT_TRAP(r1)	1560	stw r12, STACK_SLOT_TRAP(r1)
@@ -1759,8 +1775,8 @@ END_FTR_SECTION(CPU_FTR_TM \| CPU_FTR_P9_TM_HV_ASSIST, 0) · 214 unmodified lines
1759	/* Restore host values of some registers */	1775	/* Restore host values of some registers */
1760	BEGIN_FTR_SECTION	1776	BEGIN_FTR_SECTION
1761	ld r5, STACK_SLOT_CIABR(r1)	1777	ld r5, STACK_SLOT_CIABR(r1)
1762	ld r6, STACK_SLOT_DAWR(r1)	1778	ld r6, STACK_SLOT_DAWR0(r1)
1763	ld r7, STACK_SLOT_DAWRX(r1)	1779	ld r7, STACK_SLOT_DAWRX0(r1)
1764	mtspr SPRN_CIABR, r5	1780	mtspr SPRN_CIABR, r5
1765	/*	1781	/*
1766	* If the DAWR doesn't work, it's ok to write these here as	1782	* If the DAWR doesn't work, it's ok to write these here as
@@ -1769,6 +1785,12 @@ BEGIN_FTR_SECTION · 2 unmodified lines
1769	mtspr SPRN_DAWR0, r6	1785	mtspr SPRN_DAWR0, r6
1770	mtspr SPRN_DAWRX0, r7	1786	mtspr SPRN_DAWRX0, r7
1771	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)	1787	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
		1788	BEGIN_FTR_SECTION
		1789	ld r6, STACK_SLOT_DAWR1(r1)
		1790	ld r7, STACK_SLOT_DAWRX1(r1)
		1791	mtspr SPRN_DAWR1, r6
		1792	mtspr SPRN_DAWRX1, r7
		1793	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S \| CPU_FTR_DAWR1)
1772	BEGIN_FTR_SECTION	1794	BEGIN_FTR_SECTION
1773	ld r5, STACK_SLOT_TID(r1)	1795	ld r5, STACK_SLOT_TID(r1)
1774	ld r6, STACK_SLOT_PSSCR(r1)	1796	ld r6, STACK_SLOT_PSSCR(r1)
@@ -1938,24 +1960,10 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) · 163 unmodified lines
1938	19: lis r8,0x7fff /* MAX_INT@h */	1960	19: lis r8,0x7fff /* MAX_INT@h */
1939	mtspr SPRN_HDEC,r8	1961	mtspr SPRN_HDEC,r8
1940		1962
1941	16:	1963	16: ld r8,KVM_HOST_LPCR(r4)
1942	BEGIN_FTR_SECTION
1943	/* On POWER9 with HPT-on-radix we need to wait for all other threads */
1944	ld r3, HSTATE_SPLIT_MODE(r13)
1945	cmpdi r3, 0
1946	beq 47f
1947	lwz r8, KVM_SPLIT_DO_RESTORE(r3)
1948	cmpwi r8, 0
1949	beq 47f
1950	bl kvmhv_p9_restore_lpcr
1951	nop
1952	b 48f
1953	47:
1954	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
1955	ld r8,KVM_HOST_LPCR(r4)
1956	mtspr SPRN_LPCR,r8	1964	mtspr SPRN_LPCR,r8
1957	isync	1965	isync
1958	48:	1966
1959	#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING	1967	#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1960	/* Finish timing, if we have a vcpu */	1968	/* Finish timing, if we have a vcpu */
1961	ld r4, HSTATE_KVM_VCPU(r13)	1969	ld r4, HSTATE_KVM_VCPU(r13)
@@ -2574,8 +2582,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) · 612 unmodified lines
2574	rlwimi r5, r4, 5, DAWRX_DR \| DAWRX_DW	2582	rlwimi r5, r4, 5, DAWRX_DR \| DAWRX_DW
2575	rlwimi r5, r4, 2, DAWRX_WT	2583	rlwimi r5, r4, 2, DAWRX_WT
2576	clrrdi r4, r4, 3	2584	clrrdi r4, r4, 3
2577	std r4, VCPU_DAWR(r3)	2585	std r4, VCPU_DAWR0(r3)
2578	std r5, VCPU_DAWRX(r3)	2586	std r5, VCPU_DAWRX0(r3)
2579	/*	2587	/*
2580	* If came in through the real mode hcall handler then it is necessary	2588	* If came in through the real mode hcall handler then it is necessary
2581	* to write the registers since the return path won't. Otherwise it is	2589	* to write the registers since the return path won't. Otherwise it is
@@ -2779,8 +2787,10 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300) · 197 unmodified lines
2779	beq kvm_end_cede	2787	beq kvm_end_cede
2780	cmpwi r0, NAPPING_NOVCPU	2788	cmpwi r0, NAPPING_NOVCPU
2781	beq kvm_novcpu_wakeup	2789	beq kvm_novcpu_wakeup
		2790	BEGIN_FTR_SECTION
2782	cmpwi r0, NAPPING_UNSPLIT	2791	cmpwi r0, NAPPING_UNSPLIT
2783	beq kvm_unsplit_wakeup	2792	beq kvm_unsplit_wakeup
		2793	END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
2784	twi 31,0,0 /* Nap state must not be zero */	2794	twi 31,0,0 /* Nap state must not be zero */
2785		2795
2786	33: mr r4, r3	2796	33: mr r4, r3
@@ -3343,13 +3353,18 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) · 556 unmodified lines
3343	mtspr SPRN_IAMR, r0	3353	mtspr SPRN_IAMR, r0
3344	mtspr SPRN_CIABR, r0	3354	mtspr SPRN_CIABR, r0
3345	mtspr SPRN_DAWRX0, r0	3355	mtspr SPRN_DAWRX0, r0
		3356	BEGIN_FTR_SECTION
		3357	mtspr SPRN_DAWRX1, r0
		3358	END_FTR_SECTION_IFSET(CPU_FTR_DAWR1)
		3359
		3360	/* Clear hash and radix guest SLB, see guest_exit_short_path comment. */
		3361	slbmte r0, r0
		3362	PPC_SLBIA(6)
3346		3363
3347	BEGIN_MMU_FTR_SECTION	3364	BEGIN_MMU_FTR_SECTION
3348	b 4f	3365	b 4f
3349	END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)	3366	END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
3350		3367
3351	slbmte r0, r0
3352	slbia
3353	ptesync	3368	ptesync
3354	ld r8, PACA_SLBSHADOWPTR(r13)	3369	ld r8, PACA_SLBSHADOWPTR(r13)
3355	.rept SLB_NUM_BOLTED	3370	.rept SLB_NUM_BOLTED

arch/powerpc/kvm/booke.c+1−1

@@ -698,7 +698,7 @@ int kvmppc_core_prepare_to_enter(struct kvm_vcpu *vcpu) · 697 unmodified lines
698		698
699	kvmppc_set_exit_type(vcpu, EMULATED_MTMSRWE_EXITS);	699	kvmppc_set_exit_type(vcpu, EMULATED_MTMSRWE_EXITS);
700	r = 1;	700	r = 1;
701	};	701	}
702		702
703	return r;	703	return r;
704	}	704	}

arch/powerpc/kvm/powerpc.c+12−2

 		r = !!(hv_enabled && radix_enabled());
 		break;
 	case KVM_CAP_PPC_MMU_HASH_V3:
-		r = !!(hv_enabled && cpu_has_feature(CPU_FTR_ARCH_300) &&
+		r = !!(hv_enabled && kvmppc_hv_ops->hash_v3_possible &&
-		       cpu_has_feature(CPU_FTR_HVMODE));
+		       kvmppc_hv_ops->hash_v3_possible());
 		break;
 	case KVM_CAP_PPC_NESTED_HV:
 		r = !!(hv_enabled && kvmppc_hv_ops->enable_nested &&
 		r = hv_enabled && kvmppc_hv_ops->enable_svm &&
 			!kvmppc_hv_ops->enable_svm(NULL);
 		break;
+	case KVM_CAP_PPC_DAWR1:
+		r = !!(hv_enabled && kvmppc_hv_ops->enable_dawr1 &&
+		       !kvmppc_hv_ops->enable_dawr1(NULL));
+		break;
 #endif
 	default:
 		r = 0;
 			break;
 		r = kvm->arch.kvm_ops->enable_svm(kvm);
 		break;
+	case KVM_CAP_PPC_DAWR1:
+		r = -EINVAL;
+		if (!is_kvmppc_hv_enabled(kvm) || !kvm->arch.kvm_ops->enable_dawr1)
+			break;
+		r = kvm->arch.kvm_ops->enable_dawr1(kvm);
+		break;
 #endif
 	default:
 		r = -EINVAL;

arch/s390/include/asm/kvm_host.h+0−1

@@ -28,7 +28,6 @@ · 27 unmodified lines
28	#define KVM_S390_BSCA_CPU_SLOTS 64	28	#define KVM_S390_BSCA_CPU_SLOTS 64
29	#define KVM_S390_ESCA_CPU_SLOTS 248	29	#define KVM_S390_ESCA_CPU_SLOTS 248
30	#define KVM_MAX_VCPUS 255	30	#define KVM_MAX_VCPUS 255
31	#define KVM_USER_MEM_SLOTS 32
32		31
33	/*	32	/*
34	* These seem to be used for allocating ->chip in the routing table, which we	33	* These seem to be used for allocating ->chip in the routing table, which we

arch/s390/pci/pci_mmio.c+2−2

@@ -170,7 +170,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, · 169 unmodified lines
170	if (!(vma->vm_flags & VM_WRITE))	170	if (!(vma->vm_flags & VM_WRITE))
171	goto out_unlock_mmap;	171	goto out_unlock_mmap;
172		172
173	ret = follow_pte(vma->vm_mm, mmio_addr, NULL, &ptep, NULL, &ptl);	173	ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl);
174	if (ret)	174	if (ret)
175	goto out_unlock_mmap;	175	goto out_unlock_mmap;
176		176
@@ -311,7 +311,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, · 134 unmodified lines
311	if (!(vma->vm_flags & VM_WRITE))	311	if (!(vma->vm_flags & VM_WRITE))
312	goto out_unlock_mmap;	312	goto out_unlock_mmap;
313		313
314	ret = follow_pte(vma->vm_mm, mmio_addr, NULL, &ptep, NULL, &ptl);	314	ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl);
315	if (ret)	315	if (ret)
316	goto out_unlock_mmap;	316	goto out_unlock_mmap;
317		317

arch/sparc/include/asm/spinlock_64.h+1−1

@@ -11,8 +11,8 @@ · 10 unmodified lines
11		11
12	#include <asm/processor.h>	12	#include <asm/processor.h>
13	#include <asm/barrier.h>	13	#include <asm/barrier.h>
14	#include <asm/qrwlock.h>
15	#include <asm/qspinlock.h>	14	#include <asm/qspinlock.h>
		15	#include <asm/qrwlock.h>
16		16
17	#endif /* !(__ASSEMBLY__) */	17	#endif /* !(__ASSEMBLY__) */
18		18

arch/x86/include/asm/cpufeatures.h+2−0

 #define X86_FEATURE_PER_THREAD_MBA	(11*32+ 7) /* "" Per-thread Memory Bandwidth Allocation */
 /* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */
+#define X86_FEATURE_AVX_VNNI		(12*32+ 4) /* AVX VNNI instructions */
 #define X86_FEATURE_AVX512_BF16		(12*32+ 5) /* AVX512 BFLOAT16 instructions */
 /* AMD-defined CPU features, CPUID level 0x80000008 (EBX), word 13 */
 #define X86_FEATURE_AVIC		(15*32+13) /* Virtual Interrupt Controller */
 #define X86_FEATURE_V_VMSAVE_VMLOAD	(15*32+15) /* Virtual VMSAVE VMLOAD */
 #define X86_FEATURE_VGIF		(15*32+16) /* Virtual GIF */
+#define X86_FEATURE_SVME_ADDR_CHK	(15*32+28) /* "" SVME addr check */
 /* Intel-defined CPU features, CPUID level 0x00000007:0 (ECX), word 16 */
 #define X86_FEATURE_AVX512VBMI		(16*32+ 1) /* AVX512 Vector Bit Manipulation instructions*/

arch/x86/include/asm/kvm-x86-ops.h+127−0

@@ -0,0 +1,127 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#if !defined(KVM_X86_OP) \|\| !defined(KVM_X86_OP_NULL)
	3	BUILD_BUG_ON(1)
	4	#endif
	5
	6	/*
	7	* KVM_X86_OP() and KVM_X86_OP_NULL() are used to help generate
	8	* "static_call()"s. They are also intended for use when defining
	9	* the vmx/svm kvm_x86_ops. KVM_X86_OP() can be used for those
	10	* functions that follow the [svm\|vmx]_func_name convention.
	11	* KVM_X86_OP_NULL() can leave a NULL definition for the
	12	* case where there is no definition or a function name that
	13	* doesn't match the typical naming convention is supplied.
	14	*/
	15	KVM_X86_OP_NULL(hardware_enable)
	16	KVM_X86_OP_NULL(hardware_disable)
	17	KVM_X86_OP_NULL(hardware_unsetup)
	18	KVM_X86_OP_NULL(cpu_has_accelerated_tpr)
	19	KVM_X86_OP(has_emulated_msr)
	20	KVM_X86_OP(vcpu_after_set_cpuid)
	21	KVM_X86_OP(vm_init)
	22	KVM_X86_OP_NULL(vm_destroy)
	23	KVM_X86_OP(vcpu_create)
	24	KVM_X86_OP(vcpu_free)
	25	KVM_X86_OP(vcpu_reset)
	26	KVM_X86_OP(prepare_guest_switch)
	27	KVM_X86_OP(vcpu_load)
	28	KVM_X86_OP(vcpu_put)
	29	KVM_X86_OP(update_exception_bitmap)
	30	KVM_X86_OP(get_msr)
	31	KVM_X86_OP(set_msr)
	32	KVM_X86_OP(get_segment_base)
	33	KVM_X86_OP(get_segment)
	34	KVM_X86_OP(get_cpl)
	35	KVM_X86_OP(set_segment)
	36	KVM_X86_OP_NULL(get_cs_db_l_bits)
	37	KVM_X86_OP(set_cr0)
	38	KVM_X86_OP(is_valid_cr4)
	39	KVM_X86_OP(set_cr4)
	40	KVM_X86_OP(set_efer)
	41	KVM_X86_OP(get_idt)
	42	KVM_X86_OP(set_idt)
	43	KVM_X86_OP(get_gdt)
	44	KVM_X86_OP(set_gdt)
	45	KVM_X86_OP(sync_dirty_debug_regs)
	46	KVM_X86_OP(set_dr7)
	47	KVM_X86_OP(cache_reg)
	48	KVM_X86_OP(get_rflags)
	49	KVM_X86_OP(set_rflags)
	50	KVM_X86_OP(tlb_flush_all)
	51	KVM_X86_OP(tlb_flush_current)
	52	KVM_X86_OP_NULL(tlb_remote_flush)
	53	KVM_X86_OP_NULL(tlb_remote_flush_with_range)
	54	KVM_X86_OP(tlb_flush_gva)
	55	KVM_X86_OP(tlb_flush_guest)
	56	KVM_X86_OP(run)
	57	KVM_X86_OP_NULL(handle_exit)
	58	KVM_X86_OP_NULL(skip_emulated_instruction)
	59	KVM_X86_OP_NULL(update_emulated_instruction)
	60	KVM_X86_OP(set_interrupt_shadow)
	61	KVM_X86_OP(get_interrupt_shadow)
	62	KVM_X86_OP(patch_hypercall)
	63	KVM_X86_OP(set_irq)
	64	KVM_X86_OP(set_nmi)
	65	KVM_X86_OP(queue_exception)
	66	KVM_X86_OP(cancel_injection)
	67	KVM_X86_OP(interrupt_allowed)
	68	KVM_X86_OP(nmi_allowed)
	69	KVM_X86_OP(get_nmi_mask)
	70	KVM_X86_OP(set_nmi_mask)
	71	KVM_X86_OP(enable_nmi_window)
	72	KVM_X86_OP(enable_irq_window)
	73	KVM_X86_OP(update_cr8_intercept)
	74	KVM_X86_OP(check_apicv_inhibit_reasons)
	75	KVM_X86_OP_NULL(pre_update_apicv_exec_ctrl)
	76	KVM_X86_OP(refresh_apicv_exec_ctrl)
	77	KVM_X86_OP(hwapic_irr_update)
	78	KVM_X86_OP(hwapic_isr_update)
	79	KVM_X86_OP_NULL(guest_apic_has_interrupt)
	80	KVM_X86_OP(load_eoi_exitmap)
	81	KVM_X86_OP(set_virtual_apic_mode)
	82	KVM_X86_OP_NULL(set_apic_access_page_addr)
	83	KVM_X86_OP(deliver_posted_interrupt)
	84	KVM_X86_OP_NULL(sync_pir_to_irr)
	85	KVM_X86_OP(set_tss_addr)
	86	KVM_X86_OP(set_identity_map_addr)
	87	KVM_X86_OP(get_mt_mask)
	88	KVM_X86_OP(load_mmu_pgd)
	89	KVM_X86_OP_NULL(has_wbinvd_exit)
	90	KVM_X86_OP(write_l1_tsc_offset)
	91	KVM_X86_OP(get_exit_info)
	92	KVM_X86_OP(check_intercept)
	93	KVM_X86_OP(handle_exit_irqoff)
	94	KVM_X86_OP_NULL(request_immediate_exit)
	95	KVM_X86_OP(sched_in)
	96	KVM_X86_OP_NULL(slot_enable_log_dirty)
	97	KVM_X86_OP_NULL(slot_disable_log_dirty)
	98	KVM_X86_OP_NULL(flush_log_dirty)
	99	KVM_X86_OP_NULL(enable_log_dirty_pt_masked)
	100	KVM_X86_OP_NULL(cpu_dirty_log_size)
	101	KVM_X86_OP_NULL(pre_block)
	102	KVM_X86_OP_NULL(post_block)
	103	KVM_X86_OP_NULL(vcpu_blocking)
	104	KVM_X86_OP_NULL(vcpu_unblocking)
	105	KVM_X86_OP_NULL(update_pi_irte)
	106	KVM_X86_OP_NULL(apicv_post_state_restore)
	107	KVM_X86_OP_NULL(dy_apicv_has_pending_interrupt)
	108	KVM_X86_OP_NULL(set_hv_timer)
	109	KVM_X86_OP_NULL(cancel_hv_timer)
	110	KVM_X86_OP(setup_mce)
	111	KVM_X86_OP(smi_allowed)
	112	KVM_X86_OP(pre_enter_smm)
	113	KVM_X86_OP(pre_leave_smm)
	114	KVM_X86_OP(enable_smi_window)
	115	KVM_X86_OP_NULL(mem_enc_op)
	116	KVM_X86_OP_NULL(mem_enc_reg_region)
	117	KVM_X86_OP_NULL(mem_enc_unreg_region)
	118	KVM_X86_OP(get_msr_feature)
	119	KVM_X86_OP(can_emulate_instruction)
	120	KVM_X86_OP(apic_init_signal_blocked)
	121	KVM_X86_OP_NULL(enable_direct_tlbflush)
	122	KVM_X86_OP_NULL(migrate_timers)
	123	KVM_X86_OP(msr_filter_changed)
	124	KVM_X86_OP_NULL(complete_emulated_msr)
	125
	126	#undef KVM_X86_OP
	127	#undef KVM_X86_OP_NULL

arch/x86/include/asm/kvm_host.h+74−15

 #define KVM_MAX_VCPUS 288
 #define KVM_SOFT_MAX_VCPUS 240
 #define KVM_MAX_VCPU_ID 1023
-#define KVM_USER_MEM_SLOTS 509
 /* memory slots that are not exposed to userspace */
 #define KVM_PRIVATE_MEM_SLOTS 3
-#define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS)
 #define KVM_HALT_POLL_NS_DEFAULT 200000
 #define KVM_DIRTY_LOG_MANUAL_CAPS   (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \
 					KVM_DIRTY_LOG_INITIALLY_SET)
+#define KVM_BUS_LOCK_DETECTION_VALID_MODE	(KVM_BUS_LOCK_DETECTION_OFF | \
+						 KVM_BUS_LOCK_DETECTION_EXIT)
 /* x86-specific vcpu->requests bit members */
 #define KVM_REQ_MIGRATE_TIMER		KVM_ARCH_REQ(0)
 #define KVM_REQ_REPORT_TPR_ACCESS	KVM_ARCH_REQ(1)
 #define DR6_BS		(1 << 14)
 #define DR6_BT		(1 << 15)
 #define DR6_RTM		(1 << 16)
-#define DR6_FIXED_1	0xfffe0ff0
+/*
-#define DR6_INIT	0xffff0ff0
+ * DR6_ACTIVE_LOW combines fixed-1 and active-low bits.
+ * We can regard all the bits in DR6_FIXED_1 as active_low bits;
+ * they will never be 0 for now, but when they are defined
+ * in the future it will require no code change.
+ *
+ * DR6_ACTIVE_LOW is also used as the init/reset value for DR6.
+ */
+#define DR6_ACTIVE_LOW	0xffff0ff0
 #define DR6_VOLATILE	0x0001e00f
+#define DR6_FIXED_1	(DR6_ACTIVE_LOW & ~DR6_VOLATILE)
 #define DR7_BP_EN_MASK	0x000000ff
 #define DR7_GE		(1 << 9)
 #define KVM_MMU_NUM_PREV_ROOTS 3
+#define KVM_HAVE_MMU_RWLOCK
 struct kvm_mmu_page;
 /*
 	int (*sync_page)(struct kvm_vcpu *vcpu,
 			 struct kvm_mmu_page *sp);
 	void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa);
-	void (*update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
-			   u64 *spte, const void *pte);
 	hpa_t root_hpa;
 	gpa_t root_pgd;
 	union kvm_mmu_role mmu_role;
 /* Hyper-V per vcpu emulation context */
 struct kvm_vcpu_hv {
+	struct kvm_vcpu *vcpu;
 	u32 vp_index;
 	u64 hv_vapic;
 	s64 runtime_offset;
 	cpumask_t tlb_flush;
 };
+/* Xen HVM per vcpu emulation context */
+struct kvm_vcpu_xen {
+	u64 hypercall_rip;
+	bool vcpu_info_set;
+	bool vcpu_time_info_set;
+	struct gfn_to_hva_cache vcpu_info_cache;
+	struct gfn_to_hva_cache vcpu_time_info_cache;
+};
 struct kvm_vcpu_arch {
 	/*
 	 * rip and regs accesses must go through
 	int cpuid_nent;
 	struct kvm_cpuid_entry2 *cpuid_entries;
-	unsigned long cr3_lm_rsvd_bits;
+	u64 reserved_gpa_bits;
 	int maxphyaddr;
 	int max_tdp_level;
 	/* used for guest single stepping over the given code position */
 	unsigned long singlestep_rip;
-	struct kvm_vcpu_hv hyperv;
+	bool hyperv_enabled;
+	struct kvm_vcpu_hv *hyperv;
+	struct kvm_vcpu_xen xen;
 	cpumask_var_t wbinvd_dirty_mask;
 	unsigned long *bitmap;
 };
+/* Xen emulation context */
+struct kvm_xen {
+	bool long_mode;
+	bool shinfo_set;
+	u8 upcall_vector;
+	struct gfn_to_hva_cache shinfo_cache;
+};
 enum kvm_irqchip_mode {
 	KVM_IRQCHIP_NONE,
 	KVM_IRQCHIP_KERNEL,       /* created with KVM_CREATE_IRQCHIP */
 	struct hlist_head mask_notifier_list;
 	struct kvm_hv hyperv;
+	struct kvm_xen xen;
 	#ifdef CONFIG_KVM_MMU_AUDIT
 	int audit_point;
 		struct msr_bitmap_range ranges[16];
 	} msr_filter;
+	bool bus_lock_detection_enabled;
 	struct kvm_pmu_event_filter *pmu_event_filter;
 	struct task_struct *nx_lpage_recovery_thread;
+#ifdef CONFIG_X86_64
 	/*
 	 * Whether the TDP MMU is enabled for this VM. This contains a
 	 * snapshot of the TDP MMU module parameter from when the VM was
 	 * tdp_mmu_page set and a root_count of 0.
 	 */
 	struct list_head tdp_mmu_pages;
+	/*
+	 * Protects accesses to the following fields when the MMU lock
+	 * is held in read mode:
+	 *  - tdp_mmu_pages (above)
+	 *  - the link field of struct kvm_mmu_pages used by the TDP MMU
+	 *  - lpage_disallowed_mmu_pages
+	 *  - the lpage_disallowed_link field of struct kvm_mmu_pages used
+	 *    by the TDP MMU
+	 * It is acceptable, but not necessary, to acquire this lock when
+	 * the thread holds the MMU lock in write mode.
+	 */
+	spinlock_t tdp_mmu_pages_lock;
+#endif /* CONFIG_X86_64 */
 };
 struct kvm_vm_stat {
 	ulong mmu_shadow_zapped;
 	ulong mmu_pte_write;
-	ulong mmu_pte_updated;
 	ulong mmu_pde_zapped;
 	ulong mmu_flooded;
 	ulong mmu_recycled;
 extern bool __read_mostly allow_smaller_maxphyaddr;
 extern struct kvm_x86_ops kvm_x86_ops;
+#define KVM_X86_OP(func) \
+	DECLARE_STATIC_CALL(kvm_x86_##func, *(((struct kvm_x86_ops *)0)->func));
+#define KVM_X86_OP_NULL KVM_X86_OP
+#include <asm/kvm-x86-ops.h>
+static inline void kvm_ops_static_call_update(void)
+{
+#define KVM_X86_OP(func) \
+	static_call_update(kvm_x86_##func, kvm_x86_ops.func);
+#define KVM_X86_OP_NULL KVM_X86_OP
+#include <asm/kvm-x86-ops.h>
+}
 #define __KVM_HAVE_ARCH_VM_ALLOC
 static inline struct kvm *kvm_arch_alloc_vm(void)
 {
 static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm)
 {
 	if (kvm_x86_ops.tlb_remote_flush &&
-	    !kvm_x86_ops.tlb_remote_flush(kvm))
+	    !static_call(kvm_x86_tlb_remote_flush)(kvm))
 		return 0;
 	else
 		return -ENOTSUPP;
 extern u64  kvm_max_tsc_scaling_ratio;
 /* 1ull << kvm_tsc_scaling_ratio_frac_bits */
 extern u64  kvm_default_tsc_scaling_ratio;
+/* bus lock detection supported? */
+extern bool kvm_has_bus_lock_exit;
 extern u64 kvm_mce_cap_supported;
 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8);
 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val);
-int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val);
+void kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val);
 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu);
 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw);
 void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l);
 static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
 {
-	if (kvm_x86_ops.vcpu_blocking)
+	static_call_cond(kvm_x86_vcpu_blocking)(vcpu);
-		kvm_x86_ops.vcpu_blocking(vcpu);
 }
 static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
 {
-	if (kvm_x86_ops.vcpu_unblocking)
+	static_call_cond(kvm_x86_vcpu_unblocking)(vcpu);
-		kvm_x86_ops.vcpu_unblocking(vcpu);
 }
 static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}

arch/x86/include/asm/virtext.h+19−6

 }
-/** Disable VMX on the current CPU
+/**
+ * cpu_vmxoff() - Disable VMX on the current CPU
  *
- * vmxoff causes a undefined-opcode exception if vmxon was not run
+ * Disable VMX and clear CR4.VMXE (even if VMXOFF faults)
- * on the CPU previously. Only call this function if you know VMX
+ *
- * is enabled.
+ * Note, VMXOFF causes a #UD if the CPU is !post-VMXON, but it's impossible to
+ * atomically track post-VMXON state, e.g. this may be called in NMI context.
+ * Eat all faults as all other faults on VMXOFF faults are mode related, i.e.
+ * faults are guaranteed to be due to the !post-VMXON check unless the CPU is
+ * magically in RM, VM86, compat mode, or at CPL>0.
  */
-static inline void cpu_vmxoff(void)
+static inline int cpu_vmxoff(void)
 {
-	asm volatile ("vmxoff");
+	asm_volatile_goto("1: vmxoff\n\t"
+			  _ASM_EXTABLE(1b, %l[fault])
+			  ::: "cc", "memory" : fault);
+	cr4_clear_bits(X86_CR4_VMXE);
+	return 0;
+fault:
 	cr4_clear_bits(X86_CR4_VMXE);
+	return -EIO;
 }
 static inline int cpu_vmx_enabled(void)

arch/x86/include/asm/vmx.h+1−0

@@ -73,6 +73,7 @@ · 72 unmodified lines
73	#define SECONDARY_EXEC_PT_USE_GPA VMCS_CONTROL_BIT(PT_USE_GPA)	73	#define SECONDARY_EXEC_PT_USE_GPA VMCS_CONTROL_BIT(PT_USE_GPA)
74	#define SECONDARY_EXEC_TSC_SCALING VMCS_CONTROL_BIT(TSC_SCALING)	74	#define SECONDARY_EXEC_TSC_SCALING VMCS_CONTROL_BIT(TSC_SCALING)
75	#define SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE VMCS_CONTROL_BIT(USR_WAIT_PAUSE)	75	#define SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE VMCS_CONTROL_BIT(USR_WAIT_PAUSE)
		76	#define SECONDARY_EXEC_BUS_LOCK_DETECTION VMCS_CONTROL_BIT(BUS_LOCK_DETECTION)
76		77
77	#define PIN_BASED_EXT_INTR_MASK VMCS_CONTROL_BIT(INTR_EXITING)	78	#define PIN_BASED_EXT_INTR_MASK VMCS_CONTROL_BIT(INTR_EXITING)
78	#define PIN_BASED_NMI_EXITING VMCS_CONTROL_BIT(NMI_EXITING)	79	#define PIN_BASED_NMI_EXITING VMCS_CONTROL_BIT(NMI_EXITING)

arch/x86/include/asm/vmxfeatures.h+1−0

@@ -83,5 +83,6 @@ · 82 unmodified lines
83	#define VMX_FEATURE_TSC_SCALING ( 232+ 25) / Scale hardware TSC when read in guest */	83	#define VMX_FEATURE_TSC_SCALING ( 232+ 25) / Scale hardware TSC when read in guest */
84	#define VMX_FEATURE_USR_WAIT_PAUSE ( 232+ 26) / Enable TPAUSE, UMONITOR, UMWAIT in guest */	84	#define VMX_FEATURE_USR_WAIT_PAUSE ( 232+ 26) / Enable TPAUSE, UMONITOR, UMWAIT in guest */
85	#define VMX_FEATURE_ENCLV_EXITING ( 232+ 28) / "" VM-Exit on ENCLV (leaf dependent) */	85	#define VMX_FEATURE_ENCLV_EXITING ( 232+ 28) / "" VM-Exit on ENCLV (leaf dependent) */
		86	#define VMX_FEATURE_BUS_LOCK_DETECTION ( 232+ 30) / "" VM-Exit when bus lock caused */
86		87
87	#endif /* _ASM_X86_VMXFEATURES_H */	88	#endif /* _ASM_X86_VMXFEATURES_H */

arch/x86/include/asm/xen/interface.h+3−0

 	unsigned long p2m_cr3;		/* cr3 value of the p2m address space */
 	unsigned long p2m_vaddr;	/* virtual address of the p2m list */
 	unsigned long p2m_generation;	/* generation count of p2m mapping */
+#ifdef CONFIG_X86_32
+	uint32_t wc_sec_hi;
+#endif
 };
 #endif	/* !__ASSEMBLY__ */

arch/x86/include/uapi/asm/kvm.h+1−0

@@ -112,6 +112,7 @@ struct kvm_ioapic_state { · 111 unmodified lines
112	#define KVM_NR_IRQCHIPS 3	112	#define KVM_NR_IRQCHIPS 3
113		113
114	#define KVM_RUN_X86_SMM (1 << 0)	114	#define KVM_RUN_X86_SMM (1 << 0)
		115	#define KVM_RUN_X86_BUS_LOCK (1 << 1)
115		116
116	/* for KVM_GET_REGS and KVM_SET_REGS */	117	/* for KVM_GET_REGS and KVM_SET_REGS */
117	struct kvm_regs {	118	struct kvm_regs {

arch/x86/include/uapi/asm/vmx.h+3−1

 #define EXIT_REASON_XRSTORS             64
 #define EXIT_REASON_UMWAIT              67
 #define EXIT_REASON_TPAUSE              68
+#define EXIT_REASON_BUS_LOCK            74
 #define VMX_EXIT_REASONS \
 	{ EXIT_REASON_EXCEPTION_NMI,         "EXCEPTION_NMI" }, \
 	{ EXIT_REASON_XSAVES,                "XSAVES" }, \
 	{ EXIT_REASON_XRSTORS,               "XRSTORS" }, \
 	{ EXIT_REASON_UMWAIT,                "UMWAIT" }, \
-	{ EXIT_REASON_TPAUSE,                "TPAUSE" }
+	{ EXIT_REASON_TPAUSE,                "TPAUSE" }, \
+	{ EXIT_REASON_BUS_LOCK,              "BUS_LOCK" }
 #define VMX_EXIT_REASON_FLAGS \
 	{ VMX_EXIT_REASONS_FAILED_VMENTRY,	"FAILED_VMENTRY" }

arch/x86/kernel/apic/apic.c+1−0

@@ -1747,6 +1747,7 @@ void apic_ap_setup(void) · 1746 unmodified lines
1747		1747
1748	#ifdef CONFIG_X86_X2APIC	1748	#ifdef CONFIG_X86_X2APIC
1749	int x2apic_mode;	1749	int x2apic_mode;
		1750	EXPORT_SYMBOL_GPL(x2apic_mode);
1750		1751
1751	enum {	1752	enum {
1752	X2APIC_OFF,	1753	X2APIC_OFF,

arch/x86/kernel/reboot.c+10−20

 	local_irq_disable();
 	/*
-	 * We need to disable VMX on all CPUs before rebooting, otherwise
+	 * Disable VMX on all CPUs before rebooting, otherwise we risk hanging
-	 * we risk hanging up the machine, because the CPU ignores INIT
+	 * the machine, because the CPU blocks INIT when it's in VMX root.
-	 * signals when VMX is enabled.
 	 *
-	 * We can't take any locks and we may be on an inconsistent
+	 * We can't take any locks and we may be on an inconsistent state, so
-	 * state, so we use NMIs as IPIs to tell the other CPUs to disable
+	 * use NMIs as IPIs to tell the other CPUs to exit VMX root and halt.
-	 * VMX and halt.
 	 *
-	 * For safety, we will avoid running the nmi_shootdown_cpus()
+	 * Do the NMI shootdown even if VMX if off on _this_ CPU, as that
-	 * stuff unnecessarily, but we don't have a way to check
+	 * doesn't prevent a different CPU from being in VMX root operation.
-	 * if other CPUs have VMX enabled. So we will call it only if the
-	 * CPU we are running on has VMX enabled.
-	 *
-	 * We will miss cases where VMX is not enabled on all CPUs. This
-	 * shouldn't do much harm because KVM always enable VMX on all
-	 * CPUs anyway. But we can miss it on the small window where KVM
-	 * is still enabling VMX.
 	 */
-	if (cpu_has_vmx() && cpu_vmx_enabled()) {
+	if (cpu_has_vmx()) {
-		/* Disable VMX on this CPU. */
+		/* Safely force _this_ CPU out of VMX root operation. */
-		cpu_vmxoff();
+		__cpu_emergency_vmxoff();
-		/* Halt and disable VMX on the other CPUs */
+		/* Halt and exit VMX root operation on the other CPUs. */
 		nmi_shootdown_cpus(vmxoff_nmi);
 	}
 }

arch/x86/kvm/Makefile+3−2

 				$(KVM)/dirty_ring.o
 kvm-$(CONFIG_KVM_ASYNC_PF)	+= $(KVM)/async_pf.o
-kvm-y			+= x86.o emulate.o i8259.o irq.o lapic.o \
+kvm-y			+= x86.o emulate.o i8259.o irq.o lapic.o xen.o \
 			   i8254.o ioapic.o irq_comm.o cpuid.o pmu.o mtrr.o \
 			   hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o \
-			   mmu/spte.o mmu/tdp_iter.o mmu/tdp_mmu.o
+			   mmu/spte.o
+kvm-$(CONFIG_X86_64) += mmu/tdp_iter.o mmu/tdp_mmu.o
 kvm-intel-y		+= vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \
 			   vmx/evmcs.o vmx/nested.o vmx/posted_intr.o

arch/x86/kvm/cpuid.c+20−4

 	kvm_update_pv_runtime(vcpu);
 	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
-	kvm_mmu_reset_context(vcpu);
+	vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu);
 	kvm_pmu_refresh(vcpu);
 	vcpu->arch.cr4_guest_rsvd_bits =
 	    __cr4_reserved_bits(guest_cpuid_has, vcpu);
-	vcpu->arch.cr3_lm_rsvd_bits = rsvd_bits(cpuid_maxphyaddr(vcpu), 63);
+	kvm_hv_set_cpuid(vcpu);
 	/* Invoke the vendor callback only after the above state is updated. */
-	kvm_x86_ops.vcpu_after_set_cpuid(vcpu);
+	static_call(kvm_x86_vcpu_after_set_cpuid)(vcpu);
+	/*
+	 * Except for the MMU, which needs to be reset after any vendor
+	 * specific adjustments to the reserved GPA bits.
+	 */
+	kvm_mmu_reset_context(vcpu);
 }
 static int is_efer_nx(void)
 	return 36;
 }
+/*
+ * This "raw" version returns the reserved GPA bits without any adjustments for
+ * encryption technologies that usurp bits.  The raw mask should be used if and
+ * only if hardware does _not_ strip the usurped bits, e.g. in virtual MTRRs.
+ */
+u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu)
+{
+	return rsvd_bits(cpuid_maxphyaddr(vcpu), 63);
+}
 /* when an old userspace process fills a new kernel module */
 int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
 			     struct kvm_cpuid *cpuid,
 		kvm_cpu_cap_set(X86_FEATURE_SPEC_CTRL_SSBD);
 	kvm_cpu_cap_mask(CPUID_7_1_EAX,
-		F(AVX512_BF16)
+		F(AVX_VNNI) | F(AVX512_BF16)
 	);
 	kvm_cpu_cap_mask(CPUID_D_1_EAX,

arch/x86/kvm/cpuid.h+18−6

 	       u32 *ecx, u32 *edx, bool exact_only);
 int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu);
+u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu);
 static inline int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)
 {
 	return vcpu->arch.maxphyaddr;
 }
+static inline bool kvm_vcpu_is_legal_gpa(struct kvm_vcpu *vcpu, gpa_t gpa)
+{
+	return !(gpa & vcpu->arch.reserved_gpa_bits);
+}
 static inline bool kvm_vcpu_is_illegal_gpa(struct kvm_vcpu *vcpu, gpa_t gpa)
 {
-	return (gpa >= BIT_ULL(cpuid_maxphyaddr(vcpu)));
+	return !kvm_vcpu_is_legal_gpa(vcpu, gpa);
+}
+static inline bool kvm_vcpu_is_legal_aligned_gpa(struct kvm_vcpu *vcpu,
+						 gpa_t gpa, gpa_t alignment)
+{
+	return IS_ALIGNED(gpa, alignment) && kvm_vcpu_is_legal_gpa(vcpu, gpa);
+}
+static inline bool page_address_valid(struct kvm_vcpu *vcpu, gpa_t gpa)
+{
+	return kvm_vcpu_is_legal_aligned_gpa(vcpu, gpa, PAGE_SIZE);
 }
 struct cpuid_reg {
 		kvm_cpu_cap_set(x86_feature);
 }
-static inline bool page_address_valid(struct kvm_vcpu *vcpu, gpa_t gpa)
-{
-	return PAGE_ALIGNED(gpa) && !(gpa >> cpuid_maxphyaddr(vcpu));
-}
 static __always_inline bool guest_pv_has(struct kvm_vcpu *vcpu,
 					 unsigned int kvm_feature)
 {

arch/x86/kvm/emulate.c+7−7

@@ -2506,12 +2506,12 @@ static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, · 2505 unmodified lines
2506		2506
2507	val = GET_SMSTATE(u32, smstate, 0x7fcc);	2507	val = GET_SMSTATE(u32, smstate, 0x7fcc);
2508		2508
2509	if (ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) \| DR6_FIXED_1))	2509	if (ctxt->ops->set_dr(ctxt, 6, val))
2510	return X86EMUL_UNHANDLEABLE;	2510	return X86EMUL_UNHANDLEABLE;
2511		2511
2512	val = GET_SMSTATE(u32, smstate, 0x7fc8);	2512	val = GET_SMSTATE(u32, smstate, 0x7fc8);
2513		2513
2514	if (ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) \| DR7_FIXED_1))	2514	if (ctxt->ops->set_dr(ctxt, 7, val))
2515	return X86EMUL_UNHANDLEABLE;	2515	return X86EMUL_UNHANDLEABLE;
2516		2516
2517	selector = GET_SMSTATE(u32, smstate, 0x7fc4);	2517	selector = GET_SMSTATE(u32, smstate, 0x7fc4);
@@ -2564,14 +2564,14 @@ static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, · 46 unmodified lines
2564	ctxt->_eip = GET_SMSTATE(u64, smstate, 0x7f78);	2564	ctxt->_eip = GET_SMSTATE(u64, smstate, 0x7f78);
2565	ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) \| X86_EFLAGS_FIXED;	2565	ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) \| X86_EFLAGS_FIXED;
2566		2566
2567	val = GET_SMSTATE(u32, smstate, 0x7f68);	2567	val = GET_SMSTATE(u64, smstate, 0x7f68);
2568		2568
2569	if (ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) \| DR6_FIXED_1))	2569	if (ctxt->ops->set_dr(ctxt, 6, val))
2570	return X86EMUL_UNHANDLEABLE;	2570	return X86EMUL_UNHANDLEABLE;
2571		2571
2572	val = GET_SMSTATE(u32, smstate, 0x7f60);	2572	val = GET_SMSTATE(u64, smstate, 0x7f60);
2573		2573
2574	if (ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) \| DR7_FIXED_1))	2574	if (ctxt->ops->set_dr(ctxt, 7, val))
2575	return X86EMUL_UNHANDLEABLE;	2575	return X86EMUL_UNHANDLEABLE;
2576		2576
2577	cr0 = GET_SMSTATE(u64, smstate, 0x7f58);	2577	cr0 = GET_SMSTATE(u64, smstate, 0x7f58);
@@ -4329,7 +4329,7 @@ static int check_dr_read(struct x86_emulate_ctxt *ctxt) · 1751 unmodified lines
4329		4329
4330	ctxt->ops->get_dr(ctxt, 6, &dr6);	4330	ctxt->ops->get_dr(ctxt, 6, &dr6);
4331	dr6 &= ~DR_TRAP_BITS;	4331	dr6 &= ~DR_TRAP_BITS;
4332	dr6 \|= DR6_BD \| DR6_RTM;	4332	dr6 \|= DR6_BD \| DR6_ACTIVE_LOW;
4333	ctxt->ops->set_dr(ctxt, 6, dr6);	4333	ctxt->ops->set_dr(ctxt, 6, dr6);
4334	return emulate_db(ctxt);	4334	return emulate_db(ctxt);
4335	}	4335	}

arch/x86/kvm/hyperv.c+209−134

@@ -23,6 +23,7 @@ · 22 unmodified lines
23	#include "ioapic.h"	23	#include "ioapic.h"
24	#include "cpuid.h"	24	#include "cpuid.h"
25	#include "hyperv.h"	25	#include "hyperv.h"
		26	#include "xen.h"
26		27
27	#include <linux/cpu.h>	28	#include <linux/cpu.h>
28	#include <linux/kvm_host.h>	29	#include <linux/kvm_host.h>
@@ -36,6 +37,9 @@ · 7 unmodified lines
36	#include "trace.h"	37	#include "trace.h"
37	#include "irq.h"	38	#include "irq.h"
38		39
		40	/* "Hv#1" signature */
		41	#define HYPERV_CPUID_SIGNATURE_EAX 0x31237648
		42
39	#define KVM_HV_MAX_SPARSE_VCPU_SET_BITS DIV_ROUND_UP(KVM_MAX_VCPUS, 64)	43	#define KVM_HV_MAX_SPARSE_VCPU_SET_BITS DIV_ROUND_UP(KVM_MAX_VCPUS, 64)
40		44
41	static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer,	45	static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer,
@@ -128,7 +132,7 @@ static int synic_set_sint(struct kvm_vcpu_hv_synic *synic, int sint, · 86 unmodified lines
128	synic_update_vector(synic, vector);	132	synic_update_vector(synic, vector);
129		133
130	/* Load SynIC vectors into EOI exit bitmap */	134	/* Load SynIC vectors into EOI exit bitmap */
131	kvm_make_request(KVM_REQ_SCAN_IOAPIC, synic_to_vcpu(synic));	135	kvm_make_request(KVM_REQ_SCAN_IOAPIC, hv_synic_to_vcpu(synic));
132	return 0;	136	return 0;
133	}	137	}
134		138
@@ -141,10 +145,10 @@ static struct kvm_vcpu get_vcpu_by_vpidx(struct kvm kvm, u32 vpidx) · 6 unmodified lines
141	return NULL;	145	return NULL;
142		146
143	vcpu = kvm_get_vcpu(kvm, vpidx);	147	vcpu = kvm_get_vcpu(kvm, vpidx);
144	if (vcpu && vcpu_to_hv_vcpu(vcpu)->vp_index == vpidx)	148	if (vcpu && kvm_hv_get_vpindex(vcpu) == vpidx)
145	return vcpu;	149	return vcpu;
146	kvm_for_each_vcpu(i, vcpu, kvm)	150	kvm_for_each_vcpu(i, vcpu, kvm)
147	if (vcpu_to_hv_vcpu(vcpu)->vp_index == vpidx)	151	if (kvm_hv_get_vpindex(vcpu) == vpidx)
148	return vcpu;	152	return vcpu;
149	return NULL;	153	return NULL;
150	}	154	}
@@ -157,15 +161,15 @@ static struct kvm_vcpu_hv_synic synic_get(struct kvm kvm, u32 vpidx) · 6 unmodified lines
157	vcpu = get_vcpu_by_vpidx(kvm, vpidx);	161	vcpu = get_vcpu_by_vpidx(kvm, vpidx);
158	if (!vcpu)	162	if (!vcpu)
159	return NULL;	163	return NULL;
160	synic = vcpu_to_synic(vcpu);	164	synic = to_hv_synic(vcpu);
161	return (synic->active) ? synic : NULL;	165	return (synic->active) ? synic : NULL;
162	}	166	}
163		167
164	static void kvm_hv_notify_acked_sint(struct kvm_vcpu *vcpu, u32 sint)	168	static void kvm_hv_notify_acked_sint(struct kvm_vcpu *vcpu, u32 sint)
165	{	169	{
166	struct kvm *kvm = vcpu->kvm;	170	struct kvm *kvm = vcpu->kvm;
167	struct kvm_vcpu_hv_synic *synic = vcpu_to_synic(vcpu);	171	struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu);
168	struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);	172	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
169	struct kvm_vcpu_hv_stimer *stimer;	173	struct kvm_vcpu_hv_stimer *stimer;
170	int gsi, idx;	174	int gsi, idx;
171		175
@@ -189,8 +193,8 @@ static void kvm_hv_notify_acked_sint(struct kvm_vcpu *vcpu, u32 sint) · 17 unmodified lines
189		193
190	static void synic_exit(struct kvm_vcpu_hv_synic *synic, u32 msr)	194	static void synic_exit(struct kvm_vcpu_hv_synic *synic, u32 msr)
191	{	195	{
192	struct kvm_vcpu *vcpu = synic_to_vcpu(synic);	196	struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic);
193	struct kvm_vcpu_hv *hv_vcpu = &vcpu->arch.hyperv;	197	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
194		198
195	hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNIC;	199	hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNIC;
196	hv_vcpu->exit.u.synic.msr = msr;	200	hv_vcpu->exit.u.synic.msr = msr;
@@ -204,7 +208,7 @@ static void synic_exit(struct kvm_vcpu_hv_synic *synic, u32 msr) · 7 unmodified lines
204	static int synic_set_msr(struct kvm_vcpu_hv_synic *synic,	208	static int synic_set_msr(struct kvm_vcpu_hv_synic *synic,
205	u32 msr, u64 data, bool host)	209	u32 msr, u64 data, bool host)
206	{	210	{
207	struct kvm_vcpu *vcpu = synic_to_vcpu(synic);	211	struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic);
208	int ret;	212	int ret;
209		213
210	if (!synic->active && !host)	214	if (!synic->active && !host)
@@ -282,8 +286,7 @@ static bool kvm_hv_is_syndbg_enabled(struct kvm_vcpu *vcpu) · 71 unmodified lines
282		286
283	static int kvm_hv_syndbg_complete_userspace(struct kvm_vcpu *vcpu)	287	static int kvm_hv_syndbg_complete_userspace(struct kvm_vcpu *vcpu)
284	{	288	{
285	struct kvm *kvm = vcpu->kvm;	289	struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
286	struct kvm_hv *hv = &kvm->arch.hyperv;
287		290
288	if (vcpu->run->hyperv.u.syndbg.msr == HV_X64_MSR_SYNDBG_CONTROL)	291	if (vcpu->run->hyperv.u.syndbg.msr == HV_X64_MSR_SYNDBG_CONTROL)
289	hv->hv_syndbg.control.status =	292	hv->hv_syndbg.control.status =
@@ -293,8 +296,8 @@ static int kvm_hv_syndbg_complete_userspace(struct kvm_vcpu *vcpu) · 3 unmodified lines
293		296
294	static void syndbg_exit(struct kvm_vcpu *vcpu, u32 msr)	297	static void syndbg_exit(struct kvm_vcpu *vcpu, u32 msr)
295	{	298	{
296	struct kvm_hv_syndbg *syndbg = vcpu_to_hv_syndbg(vcpu);	299	struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu);
297	struct kvm_vcpu_hv *hv_vcpu = &vcpu->arch.hyperv;	300	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
298		301
299	hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNDBG;	302	hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNDBG;
300	hv_vcpu->exit.u.syndbg.msr = msr;	303	hv_vcpu->exit.u.syndbg.msr = msr;
@@ -310,13 +313,13 @@ static void syndbg_exit(struct kvm_vcpu *vcpu, u32 msr) · 9 unmodified lines
310		313
311	static int syndbg_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host)	314	static int syndbg_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host)
312	{	315	{
313	struct kvm_hv_syndbg *syndbg = vcpu_to_hv_syndbg(vcpu);	316	struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu);
314		317
315	if (!kvm_hv_is_syndbg_enabled(vcpu) && !host)	318	if (!kvm_hv_is_syndbg_enabled(vcpu) && !host)
316	return 1;	319	return 1;
317		320
318	trace_kvm_hv_syndbg_set_msr(vcpu->vcpu_id,	321	trace_kvm_hv_syndbg_set_msr(vcpu->vcpu_id,
319	vcpu_to_hv_vcpu(vcpu)->vp_index, msr, data);	322	to_hv_vcpu(vcpu)->vp_index, msr, data);
320	switch (msr) {	323	switch (msr) {
321	case HV_X64_MSR_SYNDBG_CONTROL:	324	case HV_X64_MSR_SYNDBG_CONTROL:
322	syndbg->control.control = data;	325	syndbg->control.control = data;
@@ -349,7 +352,7 @@ static int syndbg_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) · 26 unmodified lines
349		352
350	static int syndbg_get_msr(struct kvm_vcpu vcpu, u32 msr, u64 pdata, bool host)	353	static int syndbg_get_msr(struct kvm_vcpu vcpu, u32 msr, u64 pdata, bool host)
351	{	354	{
352	struct kvm_hv_syndbg *syndbg = vcpu_to_hv_syndbg(vcpu);	355	struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu);
353		356
354	if (!kvm_hv_is_syndbg_enabled(vcpu) && !host)	357	if (!kvm_hv_is_syndbg_enabled(vcpu) && !host)
355	return 1;	358	return 1;
@@ -377,9 +380,7 @@ static int syndbg_get_msr(struct kvm_vcpu vcpu, u32 msr, u64 pdata, bool host) · 21 unmodified lines
377	break;	380	break;
378	}	381	}
379		382
380	trace_kvm_hv_syndbg_get_msr(vcpu->vcpu_id,	383	trace_kvm_hv_syndbg_get_msr(vcpu->vcpu_id, kvm_hv_get_vpindex(vcpu), msr, *pdata);
381	vcpu_to_hv_vcpu(vcpu)->vp_index, msr,
382	*pdata);
383		384
384	return 0;	385	return 0;
385	}	386	}
@@ -421,7 +422,7 @@ static int synic_get_msr(struct kvm_vcpu_hv_synic synic, u32 msr, u64 pdata, · 35 unmodified lines
421		422
422	static int synic_set_irq(struct kvm_vcpu_hv_synic *synic, u32 sint)	423	static int synic_set_irq(struct kvm_vcpu_hv_synic *synic, u32 sint)
423	{	424	{
424	struct kvm_vcpu *vcpu = synic_to_vcpu(synic);	425	struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic);
425	struct kvm_lapic_irq irq;	426	struct kvm_lapic_irq irq;
426	int ret, vector;	427	int ret, vector;
427		428
@@ -457,7 +458,7 @@ int kvm_hv_synic_set_irq(struct kvm *kvm, u32 vpidx, u32 sint) · 29 unmodified lines
457		458
458	void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector)	459	void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector)
459	{	460	{
460	struct kvm_vcpu_hv_synic *synic = vcpu_to_synic(vcpu);	461	struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu);
461	int i;	462	int i;
462		463
463	trace_kvm_hv_synic_send_eoi(vcpu->vcpu_id, vector);	464	trace_kvm_hv_synic_send_eoi(vcpu->vcpu_id, vector);
@@ -514,7 +515,7 @@ static void synic_init(struct kvm_vcpu_hv_synic *synic) · 50 unmodified lines
514		515
515	static u64 get_time_ref_counter(struct kvm *kvm)	516	static u64 get_time_ref_counter(struct kvm *kvm)
516	{	517	{
517	struct kvm_hv *hv = &kvm->arch.hyperv;	518	struct kvm_hv *hv = to_kvm_hv(kvm);
518	struct kvm_vcpu *vcpu;	519	struct kvm_vcpu *vcpu;
519	u64 tsc;	520	u64 tsc;
520		521
@@ -534,10 +535,10 @@ static u64 get_time_ref_counter(struct kvm *kvm) · 13 unmodified lines
534	static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer,	535	static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer,
535	bool vcpu_kick)	536	bool vcpu_kick)
536	{	537	{
537	struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);	538	struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
538		539
539	set_bit(stimer->index,	540	set_bit(stimer->index,
540	vcpu_to_hv_vcpu(vcpu)->stimer_pending_bitmap);	541	to_hv_vcpu(vcpu)->stimer_pending_bitmap);
541	kvm_make_request(KVM_REQ_HV_STIMER, vcpu);	542	kvm_make_request(KVM_REQ_HV_STIMER, vcpu);
542	if (vcpu_kick)	543	if (vcpu_kick)
543	kvm_vcpu_kick(vcpu);	544	kvm_vcpu_kick(vcpu);
@@ -545,14 +546,14 @@ static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer, · 1 unmodified line
545		546
546	static void stimer_cleanup(struct kvm_vcpu_hv_stimer *stimer)	547	static void stimer_cleanup(struct kvm_vcpu_hv_stimer *stimer)
547	{	548	{
548	struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);	549	struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
549		550
550	trace_kvm_hv_stimer_cleanup(stimer_to_vcpu(stimer)->vcpu_id,	551	trace_kvm_hv_stimer_cleanup(hv_stimer_to_vcpu(stimer)->vcpu_id,
551	stimer->index);	552	stimer->index);
552		553
553	hrtimer_cancel(&stimer->timer);	554	hrtimer_cancel(&stimer->timer);
554	clear_bit(stimer->index,	555	clear_bit(stimer->index,
555	vcpu_to_hv_vcpu(vcpu)->stimer_pending_bitmap);	556	to_hv_vcpu(vcpu)->stimer_pending_bitmap);
556	stimer->msg_pending = false;	557	stimer->msg_pending = false;
557	stimer->exp_time = 0;	558	stimer->exp_time = 0;
558	}	559	}
@@ -562,7 +563,7 @@ static enum hrtimer_restart stimer_timer_callback(struct hrtimer *timer) · 3 unmodified lines
562	struct kvm_vcpu_hv_stimer *stimer;	563	struct kvm_vcpu_hv_stimer *stimer;
563		564
564	stimer = container_of(timer, struct kvm_vcpu_hv_stimer, timer);	565	stimer = container_of(timer, struct kvm_vcpu_hv_stimer, timer);
565	trace_kvm_hv_stimer_callback(stimer_to_vcpu(stimer)->vcpu_id,	566	trace_kvm_hv_stimer_callback(hv_stimer_to_vcpu(stimer)->vcpu_id,
566	stimer->index);	567	stimer->index);
567	stimer_mark_pending(stimer, true);	568	stimer_mark_pending(stimer, true);
568		569
@@ -579,7 +580,7 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) · 10 unmodified lines
579	u64 time_now;	580	u64 time_now;
580	ktime_t ktime_now;	581	ktime_t ktime_now;
581		582
582	time_now = get_time_ref_counter(stimer_to_vcpu(stimer)->kvm);	583	time_now = get_time_ref_counter(hv_stimer_to_vcpu(stimer)->kvm);
583	ktime_now = ktime_get();	584	ktime_now = ktime_get();
584		585
585	if (stimer->config.periodic) {	586	if (stimer->config.periodic) {
@@ -596,7 +597,7 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) · 10 unmodified lines
596	stimer->exp_time = time_now + stimer->count;	597	stimer->exp_time = time_now + stimer->count;
597		598
598	trace_kvm_hv_stimer_start_periodic(	599	trace_kvm_hv_stimer_start_periodic(
599	stimer_to_vcpu(stimer)->vcpu_id,	600	hv_stimer_to_vcpu(stimer)->vcpu_id,
600	stimer->index,	601	stimer->index,
601	time_now, stimer->exp_time);	602	time_now, stimer->exp_time);
602		603
@@ -618,7 +619,7 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) · 15 unmodified lines
618	return 0;	619	return 0;
619	}	620	}
620		621
621	trace_kvm_hv_stimer_start_one_shot(stimer_to_vcpu(stimer)->vcpu_id,	622	trace_kvm_hv_stimer_start_one_shot(hv_stimer_to_vcpu(stimer)->vcpu_id,
622	stimer->index,	623	stimer->index,
623	time_now, stimer->count);	624	time_now, stimer->count);
624		625
@@ -633,13 +634,13 @@ static int stimer_set_config(struct kvm_vcpu_hv_stimer *stimer, u64 config, · 8 unmodified lines
633	{	634	{
634	union hv_stimer_config new_config = {.as_uint64 = config},	635	union hv_stimer_config new_config = {.as_uint64 = config},
635	old_config = {.as_uint64 = stimer->config.as_uint64};	636	old_config = {.as_uint64 = stimer->config.as_uint64};
636	struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);	637	struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
637	struct kvm_vcpu_hv_synic *synic = vcpu_to_synic(vcpu);	638	struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu);
638		639
639	if (!synic->active && !host)	640	if (!synic->active && !host)
640	return 1;	641	return 1;
641		642
642	trace_kvm_hv_stimer_set_config(stimer_to_vcpu(stimer)->vcpu_id,	643	trace_kvm_hv_stimer_set_config(hv_stimer_to_vcpu(stimer)->vcpu_id,
643	stimer->index, config, host);	644	stimer->index, config, host);
644		645
645	stimer_cleanup(stimer);	646	stimer_cleanup(stimer);
@@ -657,13 +658,13 @@ static int stimer_set_config(struct kvm_vcpu_hv_stimer *stimer, u64 config, · 11 unmodified lines
657	static int stimer_set_count(struct kvm_vcpu_hv_stimer *stimer, u64 count,	658	static int stimer_set_count(struct kvm_vcpu_hv_stimer *stimer, u64 count,
658	bool host)	659	bool host)
659	{	660	{
660	struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);	661	struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
661	struct kvm_vcpu_hv_synic *synic = vcpu_to_synic(vcpu);	662	struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu);
662		663
663	if (!synic->active && !host)	664	if (!synic->active && !host)
664	return 1;	665	return 1;
665		666
666	trace_kvm_hv_stimer_set_count(stimer_to_vcpu(stimer)->vcpu_id,	667	trace_kvm_hv_stimer_set_count(hv_stimer_to_vcpu(stimer)->vcpu_id,
667	stimer->index, count, host);	668	stimer->index, count, host);
668		669
669	stimer_cleanup(stimer);	670	stimer_cleanup(stimer);
@@ -694,7 +695,7 @@ static int stimer_get_count(struct kvm_vcpu_hv_stimer stimer, u64 pcount) · 24 unmodified lines
694	static int synic_deliver_msg(struct kvm_vcpu_hv_synic *synic, u32 sint,	695	static int synic_deliver_msg(struct kvm_vcpu_hv_synic *synic, u32 sint,
695	struct hv_message *src_msg, bool no_retry)	696	struct hv_message *src_msg, bool no_retry)
696	{	697	{
697	struct kvm_vcpu *vcpu = synic_to_vcpu(synic);	698	struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic);
698	int msg_off = offsetof(struct hv_message_page, sint_message[sint]);	699	int msg_off = offsetof(struct hv_message_page, sint_message[sint]);
699	gfn_t msg_page_gfn;	700	gfn_t msg_page_gfn;
700	struct hv_message_header hv_hdr;	701	struct hv_message_header hv_hdr;
@@ -750,7 +751,7 @@ static int synic_deliver_msg(struct kvm_vcpu_hv_synic *synic, u32 sint, · 49 unmodified lines
750		751
751	static int stimer_send_msg(struct kvm_vcpu_hv_stimer *stimer)	752	static int stimer_send_msg(struct kvm_vcpu_hv_stimer *stimer)
752	{	753	{
753	struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);	754	struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
754	struct hv_message *msg = &stimer->msg;	755	struct hv_message *msg = &stimer->msg;
755	struct hv_timer_message_payload *payload =	756	struct hv_timer_message_payload *payload =
756	(struct hv_timer_message_payload *)&msg->u.payload;	757	(struct hv_timer_message_payload *)&msg->u.payload;
@@ -763,14 +764,14 @@ static int stimer_send_msg(struct kvm_vcpu_hv_stimer *stimer) · 6 unmodified lines
763		764
764	payload->expiration_time = stimer->exp_time;	765	payload->expiration_time = stimer->exp_time;
765	payload->delivery_time = get_time_ref_counter(vcpu->kvm);	766	payload->delivery_time = get_time_ref_counter(vcpu->kvm);
766	return synic_deliver_msg(vcpu_to_synic(vcpu),	767	return synic_deliver_msg(to_hv_synic(vcpu),
767	stimer->config.sintx, msg,	768	stimer->config.sintx, msg,
768	no_retry);	769	no_retry);
769	}	770	}
770		771
771	static int stimer_notify_direct(struct kvm_vcpu_hv_stimer *stimer)	772	static int stimer_notify_direct(struct kvm_vcpu_hv_stimer *stimer)
772	{	773	{
773	struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);	774	struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
774	struct kvm_lapic_irq irq = {	775	struct kvm_lapic_irq irq = {
775	.delivery_mode = APIC_DM_FIXED,	776	.delivery_mode = APIC_DM_FIXED,
776	.vector = stimer->config.apic_vector	777	.vector = stimer->config.apic_vector
@@ -790,7 +791,7 @@ static void stimer_expiration(struct kvm_vcpu_hv_stimer *stimer) · 13 unmodified lines
790	r = stimer_send_msg(stimer);	791	r = stimer_send_msg(stimer);
791	else	792	else
792	r = stimer_notify_direct(stimer);	793	r = stimer_notify_direct(stimer);
793	trace_kvm_hv_stimer_expiration(stimer_to_vcpu(stimer)->vcpu_id,	794	trace_kvm_hv_stimer_expiration(hv_stimer_to_vcpu(stimer)->vcpu_id,
794	stimer->index, direct, r);	795	stimer->index, direct, r);
795	if (!r) {	796	if (!r) {
796	stimer->msg_pending = false;	797	stimer->msg_pending = false;
@@ -801,11 +802,14 @@ static void stimer_expiration(struct kvm_vcpu_hv_stimer *stimer) · 4 unmodified lines
801		802
802	void kvm_hv_process_stimers(struct kvm_vcpu *vcpu)	803	void kvm_hv_process_stimers(struct kvm_vcpu *vcpu)
803	{	804	{
804	struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);	805	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
805	struct kvm_vcpu_hv_stimer *stimer;	806	struct kvm_vcpu_hv_stimer *stimer;
806	u64 time_now, exp_time;	807	u64 time_now, exp_time;
807	int i;	808	int i;
808		809
		810	if (!hv_vcpu)
		811	return;
		812
809	for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++)	813	for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++)
810	if (test_and_clear_bit(i, hv_vcpu->stimer_pending_bitmap)) {	814	if (test_and_clear_bit(i, hv_vcpu->stimer_pending_bitmap)) {
811	stimer = &hv_vcpu->stimer[i];	815	stimer = &hv_vcpu->stimer[i];
@@ -831,16 +835,27 @@ void kvm_hv_process_stimers(struct kvm_vcpu *vcpu) · 19 unmodified lines
831		835
832	void kvm_hv_vcpu_uninit(struct kvm_vcpu *vcpu)	836	void kvm_hv_vcpu_uninit(struct kvm_vcpu *vcpu)
833	{	837	{
834	struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);	838	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
835	int i;	839	int i;
836		840
		841	if (!hv_vcpu)
		842	return;
		843
837	for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++)	844	for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++)
838	stimer_cleanup(&hv_vcpu->stimer[i]);	845	stimer_cleanup(&hv_vcpu->stimer[i]);
		846
		847	kfree(hv_vcpu);
		848	vcpu->arch.hyperv = NULL;
839	}	849	}
840		850
841	bool kvm_hv_assist_page_enabled(struct kvm_vcpu *vcpu)	851	bool kvm_hv_assist_page_enabled(struct kvm_vcpu *vcpu)
842	{	852	{
843	if (!(vcpu->arch.hyperv.hv_vapic & HV_X64_MSR_VP_ASSIST_PAGE_ENABLE))	853	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
		854
		855	if (!hv_vcpu)
		856	return false;
		857
		858	if (!(hv_vcpu->hv_vapic & HV_X64_MSR_VP_ASSIST_PAGE_ENABLE))
844	return false;	859	return false;
845	return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;	860	return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
846	}	861	}
@@ -880,28 +895,41 @@ static void stimer_init(struct kvm_vcpu_hv_stimer *stimer, int timer_index) · 33 unmodified lines
880	stimer_prepare_msg(stimer);	895	stimer_prepare_msg(stimer);
881	}	896	}
882		897
883	void kvm_hv_vcpu_init(struct kvm_vcpu *vcpu)	898	static int kvm_hv_vcpu_init(struct kvm_vcpu *vcpu)
884	{	899	{
885	struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);	900	struct kvm_vcpu_hv *hv_vcpu;
886	int i;	901	int i;
887		902
		903	hv_vcpu = kzalloc(sizeof(struct kvm_vcpu_hv), GFP_KERNEL_ACCOUNT);
		904	if (!hv_vcpu)
		905	return -ENOMEM;
		906
		907	vcpu->arch.hyperv = hv_vcpu;
		908	hv_vcpu->vcpu = vcpu;
		909
888	synic_init(&hv_vcpu->synic);	910	synic_init(&hv_vcpu->synic);
889		911
890	bitmap_zero(hv_vcpu->stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT);	912	bitmap_zero(hv_vcpu->stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT);
891	for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++)	913	for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++)
892	stimer_init(&hv_vcpu->stimer[i], i);	914	stimer_init(&hv_vcpu->stimer[i], i);
893	}
894
895	void kvm_hv_vcpu_postcreate(struct kvm_vcpu *vcpu)
896	{
897	struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);
898		915
899	hv_vcpu->vp_index = kvm_vcpu_get_idx(vcpu);	916	hv_vcpu->vp_index = kvm_vcpu_get_idx(vcpu);
		917
		918	return 0;
900	}	919	}
901		920
902	int kvm_hv_activate_synic(struct kvm_vcpu *vcpu, bool dont_zero_synic_pages)	921	int kvm_hv_activate_synic(struct kvm_vcpu *vcpu, bool dont_zero_synic_pages)
903	{	922	{
904	struct kvm_vcpu_hv_synic *synic = vcpu_to_synic(vcpu);	923	struct kvm_vcpu_hv_synic *synic;
		924	int r;
		925
		926	if (!to_hv_vcpu(vcpu)) {
		927	r = kvm_hv_vcpu_init(vcpu);
		928	if (r)
		929	return r;
		930	}
		931
		932	synic = to_hv_synic(vcpu);
905		933
906	/*	934	/*
907	* Hyper-V SynIC auto EOI SINT's are	935	* Hyper-V SynIC auto EOI SINT's are
@@ -939,10 +967,9 @@ static bool kvm_hv_msr_partition_wide(u32 msr) · 31 unmodified lines
939	return r;	967	return r;
940	}	968	}
941		969
942	static int kvm_hv_msr_get_crash_data(struct kvm_vcpu *vcpu,	970	static int kvm_hv_msr_get_crash_data(struct kvm kvm, u32 index, u64 pdata)
943	u32 index, u64 *pdata)
944	{	971	{
945	struct kvm_hv *hv = &vcpu->kvm->arch.hyperv;	972	struct kvm_hv *hv = to_kvm_hv(kvm);
946	size_t size = ARRAY_SIZE(hv->hv_crash_param);	973	size_t size = ARRAY_SIZE(hv->hv_crash_param);
947		974
948	if (WARN_ON_ONCE(index >= size))	975	if (WARN_ON_ONCE(index >= size))
@@ -952,41 +979,26 @@ static int kvm_hv_msr_get_crash_data(struct kvm_vcpu *vcpu, · 3 unmodified lines
952	return 0;	979	return 0;
953	}	980	}
954		981
955	static int kvm_hv_msr_get_crash_ctl(struct kvm_vcpu vcpu, u64 pdata)	982	static int kvm_hv_msr_get_crash_ctl(struct kvm kvm, u64 pdata)
956	{	983	{
957	struct kvm_hv *hv = &vcpu->kvm->arch.hyperv;	984	struct kvm_hv *hv = to_kvm_hv(kvm);
958		985
959	*pdata = hv->hv_crash_ctl;	986	*pdata = hv->hv_crash_ctl;
960	return 0;	987	return 0;
961	}	988	}
962		989
963	static int kvm_hv_msr_set_crash_ctl(struct kvm_vcpu *vcpu, u64 data, bool host)	990	static int kvm_hv_msr_set_crash_ctl(struct kvm *kvm, u64 data)
964	{	991	{
965	struct kvm_hv *hv = &vcpu->kvm->arch.hyperv;	992	struct kvm_hv *hv = to_kvm_hv(kvm);
966
967	if (host)
968	hv->hv_crash_ctl = data & HV_CRASH_CTL_CRASH_NOTIFY;
969		993
970	if (!host && (data & HV_CRASH_CTL_CRASH_NOTIFY)) {	994	hv->hv_crash_ctl = data & HV_CRASH_CTL_CRASH_NOTIFY;
971
972	vcpu_debug(vcpu, "hv crash (0x%llx 0x%llx 0x%llx 0x%llx 0x%llx)\n",
973	hv->hv_crash_param[0],
974	hv->hv_crash_param[1],
975	hv->hv_crash_param[2],
976	hv->hv_crash_param[3],
977	hv->hv_crash_param[4]);
978
979	/* Send notification about crash to user space */
980	kvm_make_request(KVM_REQ_HV_CRASH, vcpu);
981	}
982		995
983	return 0;	996	return 0;
984	}	997	}
985		998
986	static int kvm_hv_msr_set_crash_data(struct kvm_vcpu *vcpu,	999	static int kvm_hv_msr_set_crash_data(struct kvm *kvm, u32 index, u64 data)
987	u32 index, u64 data)
988	{	1000	{
989	struct kvm_hv *hv = &vcpu->kvm->arch.hyperv;	1001	struct kvm_hv *hv = to_kvm_hv(kvm);
990	size_t size = ARRAY_SIZE(hv->hv_crash_param);	1002	size_t size = ARRAY_SIZE(hv->hv_crash_param);
991		1003
992	if (WARN_ON_ONCE(index >= size))	1004	if (WARN_ON_ONCE(index >= size))
@@ -1068,7 +1080,7 @@ static bool compute_tsc_page_parameters(struct pvclock_vcpu_time_info *hv_clock, · 75 unmodified lines
1068	void kvm_hv_setup_tsc_page(struct kvm *kvm,	1080	void kvm_hv_setup_tsc_page(struct kvm *kvm,
1069	struct pvclock_vcpu_time_info *hv_clock)	1081	struct pvclock_vcpu_time_info *hv_clock)
1070	{	1082	{
1071	struct kvm_hv *hv = &kvm->arch.hyperv;	1083	struct kvm_hv *hv = to_kvm_hv(kvm);
1072	u32 tsc_seq;	1084	u32 tsc_seq;
1073	u64 gfn;	1085	u64 gfn;
1074		1086
@@ -1078,7 +1090,7 @@ void kvm_hv_setup_tsc_page(struct kvm *kvm, · 3 unmodified lines
1078	if (!(hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE))	1090	if (!(hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE))
1079	return;	1091	return;
1080		1092
1081	mutex_lock(&kvm->arch.hyperv.hv_lock);	1093	mutex_lock(&hv->hv_lock);
1082	if (!(hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE))	1094	if (!(hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE))
1083	goto out_unlock;	1095	goto out_unlock;
1084		1096
@@ -1122,14 +1134,14 @@ void kvm_hv_setup_tsc_page(struct kvm *kvm, · 37 unmodified lines
1122	kvm_write_guest(kvm, gfn_to_gpa(gfn),	1134	kvm_write_guest(kvm, gfn_to_gpa(gfn),
1123	&hv->tsc_ref, sizeof(hv->tsc_ref.tsc_sequence));	1135	&hv->tsc_ref, sizeof(hv->tsc_ref.tsc_sequence));
1124	out_unlock:	1136	out_unlock:
1125	mutex_unlock(&kvm->arch.hyperv.hv_lock);	1137	mutex_unlock(&hv->hv_lock);
1126	}	1138	}
1127		1139
1128	static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data,	1140	static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data,
1129	bool host)	1141	bool host)
1130	{	1142	{
1131	struct kvm *kvm = vcpu->kvm;	1143	struct kvm *kvm = vcpu->kvm;
1132	struct kvm_hv *hv = &kvm->arch.hyperv;	1144	struct kvm_hv *hv = to_kvm_hv(kvm);
1133		1145
1134	switch (msr) {	1146	switch (msr) {
1135	case HV_X64_MSR_GUEST_OS_ID:	1147	case HV_X64_MSR_GUEST_OS_ID:
@@ -1139,9 +1151,9 @@ static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data, · 3 unmodified lines
1139	hv->hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE;	1151	hv->hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE;
1140	break;	1152	break;
1141	case HV_X64_MSR_HYPERCALL: {	1153	case HV_X64_MSR_HYPERCALL: {
1142	u64 gfn;	1154	u8 instructions[9];
1143	unsigned long addr;	1155	int i = 0;
1144	u8 instructions[4];	1156	u64 addr;
1145		1157
1146	/* if guest os id is not set hypercall should remain disabled */	1158	/* if guest os id is not set hypercall should remain disabled */
1147	if (!hv->hv_guest_os_id)	1159	if (!hv->hv_guest_os_id)
@@ -1150,16 +1162,33 @@ static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data, · 2 unmodified lines
1150	hv->hv_hypercall = data;	1162	hv->hv_hypercall = data;
1151	break;	1163	break;
1152	}	1164	}
1153	gfn = data >> HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT;	1165
1154	addr = gfn_to_hva(kvm, gfn);	1166	/*
1155	if (kvm_is_error_hva(addr))	1167	* If Xen and Hyper-V hypercalls are both enabled, disambiguate
1156	return 1;	1168	* the same way Xen itself does, by setting the bit 31 of EAX
1157	kvm_x86_ops.patch_hypercall(vcpu, instructions);	1169	* which is RsvdZ in the 32-bit Hyper-V hypercall ABI and just
1158	((unsigned char )instructions)[3] = 0xc3; / ret */	1170	* going to be clobbered on 64-bit.
1159	if (__copy_to_user((void __user *)addr, instructions, 4))	1171	*/
		1172	if (kvm_xen_hypercall_enabled(kvm)) {
		1173	/* orl $0x80000000, %eax */
		1174	instructions[i++] = 0x0d;
		1175	instructions[i++] = 0x00;
		1176	instructions[i++] = 0x00;
		1177	instructions[i++] = 0x00;
		1178	instructions[i++] = 0x80;
		1179	}
		1180
		1181	/* vmcall/vmmcall */
		1182	static_call(kvm_x86_patch_hypercall)(vcpu, instructions + i);
		1183	i += 3;
		1184
		1185	/* ret */
		1186	((unsigned char *)instructions)[i++] = 0xc3;
		1187
		1188	addr = data & HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK;
		1189	if (kvm_vcpu_write_guest(vcpu, addr, instructions, i))
1160	return 1;	1190	return 1;
1161	hv->hv_hypercall = data;	1191	hv->hv_hypercall = data;
1162	mark_page_dirty(kvm, gfn);
1163	break;	1192	break;
1164	}	1193	}
1165	case HV_X64_MSR_REFERENCE_TSC:	1194	case HV_X64_MSR_REFERENCE_TSC:
@@ -1168,11 +1197,25 @@ static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data, · 2 unmodified lines
1168	kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);	1197	kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
1169	break;	1198	break;
1170	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:	1199	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
1171	return kvm_hv_msr_set_crash_data(vcpu,	1200	return kvm_hv_msr_set_crash_data(kvm,
1172	msr - HV_X64_MSR_CRASH_P0,	1201	msr - HV_X64_MSR_CRASH_P0,
1173	data);	1202	data);
1174	case HV_X64_MSR_CRASH_CTL:	1203	case HV_X64_MSR_CRASH_CTL:
1175	return kvm_hv_msr_set_crash_ctl(vcpu, data, host);	1204	if (host)
		1205	return kvm_hv_msr_set_crash_ctl(kvm, data);
		1206
		1207	if (data & HV_CRASH_CTL_CRASH_NOTIFY) {
		1208	vcpu_debug(vcpu, "hv crash (0x%llx 0x%llx 0x%llx 0x%llx 0x%llx)\n",
		1209	hv->hv_crash_param[0],
		1210	hv->hv_crash_param[1],
		1211	hv->hv_crash_param[2],
		1212	hv->hv_crash_param[3],
		1213	hv->hv_crash_param[4]);
		1214
		1215	/* Send notification about crash to user space */
		1216	kvm_make_request(KVM_REQ_HV_CRASH, vcpu);
		1217	}
		1218	break;
1176	case HV_X64_MSR_RESET:	1219	case HV_X64_MSR_RESET:
1177	if (data == 1) {	1220	if (data == 1) {
1178	vcpu_debug(vcpu, "hyper-v reset requested\n");	1221	vcpu_debug(vcpu, "hyper-v reset requested\n");
@@ -1216,11 +1259,11 @@ static u64 current_task_runtime_100ns(void) · 37 unmodified lines
1216		1259
1217	static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host)	1260	static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host)
1218	{	1261	{
1219	struct kvm_vcpu_hv *hv_vcpu = &vcpu->arch.hyperv;	1262	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
1220		1263
1221	switch (msr) {	1264	switch (msr) {
1222	case HV_X64_MSR_VP_INDEX: {	1265	case HV_X64_MSR_VP_INDEX: {
1223	struct kvm_hv *hv = &vcpu->kvm->arch.hyperv;	1266	struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
1224	int vcpu_idx = kvm_vcpu_get_idx(vcpu);	1267	int vcpu_idx = kvm_vcpu_get_idx(vcpu);
1225	u32 new_vp_index = (u32)data;	1268	u32 new_vp_index = (u32)data;
1226		1269
@@ -1291,14 +1334,14 @@ static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) · 64 unmodified lines
1291	case HV_X64_MSR_SIMP:	1334	case HV_X64_MSR_SIMP:
1292	case HV_X64_MSR_EOM:	1335	case HV_X64_MSR_EOM:
1293	case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:	1336	case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:
1294	return synic_set_msr(vcpu_to_synic(vcpu), msr, data, host);	1337	return synic_set_msr(to_hv_synic(vcpu), msr, data, host);
1295	case HV_X64_MSR_STIMER0_CONFIG:	1338	case HV_X64_MSR_STIMER0_CONFIG:
1296	case HV_X64_MSR_STIMER1_CONFIG:	1339	case HV_X64_MSR_STIMER1_CONFIG:
1297	case HV_X64_MSR_STIMER2_CONFIG:	1340	case HV_X64_MSR_STIMER2_CONFIG:
1298	case HV_X64_MSR_STIMER3_CONFIG: {	1341	case HV_X64_MSR_STIMER3_CONFIG: {
1299	int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2;	1342	int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2;
1300		1343
1301	return stimer_set_config(vcpu_to_stimer(vcpu, timer_index),	1344	return stimer_set_config(to_hv_stimer(vcpu, timer_index),
1302	data, host);	1345	data, host);
1303	}	1346	}
1304	case HV_X64_MSR_STIMER0_COUNT:	1347	case HV_X64_MSR_STIMER0_COUNT:
@@ -1307,7 +1350,7 @@ static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) · 2 unmodified lines
1307	case HV_X64_MSR_STIMER3_COUNT: {	1350	case HV_X64_MSR_STIMER3_COUNT: {
1308	int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2;	1351	int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2;
1309		1352
1310	return stimer_set_count(vcpu_to_stimer(vcpu, timer_index),	1353	return stimer_set_count(to_hv_stimer(vcpu, timer_index),
1311	data, host);	1354	data, host);
1312	}	1355	}
1313	case HV_X64_MSR_TSC_FREQUENCY:	1356	case HV_X64_MSR_TSC_FREQUENCY:
@@ -1330,7 +1373,7 @@ static int kvm_hv_get_msr_pw(struct kvm_vcpu vcpu, u32 msr, u64 pdata, · 16 unmodified lines
1330	{	1373	{
1331	u64 data = 0;	1374	u64 data = 0;
1332	struct kvm *kvm = vcpu->kvm;	1375	struct kvm *kvm = vcpu->kvm;
1333	struct kvm_hv *hv = &kvm->arch.hyperv;	1376	struct kvm_hv *hv = to_kvm_hv(kvm);
1334		1377
1335	switch (msr) {	1378	switch (msr) {
1336	case HV_X64_MSR_GUEST_OS_ID:	1379	case HV_X64_MSR_GUEST_OS_ID:
@@ -1346,11 +1389,11 @@ static int kvm_hv_get_msr_pw(struct kvm_vcpu vcpu, u32 msr, u64 pdata, · 9 unmodified lines
1346	data = hv->hv_tsc_page;	1389	data = hv->hv_tsc_page;
1347	break;	1390	break;
1348	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:	1391	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
1349	return kvm_hv_msr_get_crash_data(vcpu,	1392	return kvm_hv_msr_get_crash_data(kvm,
1350	msr - HV_X64_MSR_CRASH_P0,	1393	msr - HV_X64_MSR_CRASH_P0,
1351	pdata);	1394	pdata);
1352	case HV_X64_MSR_CRASH_CTL:	1395	case HV_X64_MSR_CRASH_CTL:
1353	return kvm_hv_msr_get_crash_ctl(vcpu, pdata);	1396	return kvm_hv_msr_get_crash_ctl(kvm, pdata);
1354	case HV_X64_MSR_RESET:	1397	case HV_X64_MSR_RESET:
1355	data = 0;	1398	data = 0;
1356	break;	1399	break;
@@ -1379,7 +1422,7 @@ static int kvm_hv_get_msr(struct kvm_vcpu vcpu, u32 msr, u64 pdata, · 22 unmodified lines
1379	bool host)	1422	bool host)
1380	{	1423	{
1381	u64 data = 0;	1424	u64 data = 0;
1382	struct kvm_vcpu_hv *hv_vcpu = &vcpu->arch.hyperv;	1425	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
1383		1426
1384	switch (msr) {	1427	switch (msr) {
1385	case HV_X64_MSR_VP_INDEX:	1428	case HV_X64_MSR_VP_INDEX:
@@ -1403,14 +1446,14 @@ static int kvm_hv_get_msr(struct kvm_vcpu vcpu, u32 msr, u64 pdata, · 17 unmodified lines
1403	case HV_X64_MSR_SIMP:	1446	case HV_X64_MSR_SIMP:
1404	case HV_X64_MSR_EOM:	1447	case HV_X64_MSR_EOM:
1405	case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:	1448	case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:
1406	return synic_get_msr(vcpu_to_synic(vcpu), msr, pdata, host);	1449	return synic_get_msr(to_hv_synic(vcpu), msr, pdata, host);
1407	case HV_X64_MSR_STIMER0_CONFIG:	1450	case HV_X64_MSR_STIMER0_CONFIG:
1408	case HV_X64_MSR_STIMER1_CONFIG:	1451	case HV_X64_MSR_STIMER1_CONFIG:
1409	case HV_X64_MSR_STIMER2_CONFIG:	1452	case HV_X64_MSR_STIMER2_CONFIG:
1410	case HV_X64_MSR_STIMER3_CONFIG: {	1453	case HV_X64_MSR_STIMER3_CONFIG: {
1411	int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2;	1454	int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2;
1412		1455
1413	return stimer_get_config(vcpu_to_stimer(vcpu, timer_index),	1456	return stimer_get_config(to_hv_stimer(vcpu, timer_index),
1414	pdata);	1457	pdata);
1415	}	1458	}
1416	case HV_X64_MSR_STIMER0_COUNT:	1459	case HV_X64_MSR_STIMER0_COUNT:
@@ -1419,7 +1462,7 @@ static int kvm_hv_get_msr(struct kvm_vcpu vcpu, u32 msr, u64 pdata, · 2 unmodified lines
1419	case HV_X64_MSR_STIMER3_COUNT: {	1462	case HV_X64_MSR_STIMER3_COUNT: {
1420	int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2;	1463	int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2;
1421		1464
1422	return stimer_get_count(vcpu_to_stimer(vcpu, timer_index),	1465	return stimer_get_count(to_hv_stimer(vcpu, timer_index),
1423	pdata);	1466	pdata);
1424	}	1467	}
1425	case HV_X64_MSR_TSC_FREQUENCY:	1468	case HV_X64_MSR_TSC_FREQUENCY:
@@ -1438,12 +1481,22 @@ static int kvm_hv_get_msr(struct kvm_vcpu vcpu, u32 msr, u64 pdata, · 12 unmodified lines
1438		1481
1439	int kvm_hv_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host)	1482	int kvm_hv_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host)
1440	{	1483	{
		1484	struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
		1485
		1486	if (!host && !vcpu->arch.hyperv_enabled)
		1487	return 1;
		1488
		1489	if (!to_hv_vcpu(vcpu)) {
		1490	if (kvm_hv_vcpu_init(vcpu))
		1491	return 1;
		1492	}
		1493
1441	if (kvm_hv_msr_partition_wide(msr)) {	1494	if (kvm_hv_msr_partition_wide(msr)) {
1442	int r;	1495	int r;
1443		1496
1444	mutex_lock(&vcpu->kvm->arch.hyperv.hv_lock);	1497	mutex_lock(&hv->hv_lock);
1445	r = kvm_hv_set_msr_pw(vcpu, msr, data, host);	1498	r = kvm_hv_set_msr_pw(vcpu, msr, data, host);
1446	mutex_unlock(&vcpu->kvm->arch.hyperv.hv_lock);	1499	mutex_unlock(&hv->hv_lock);
1447	return r;	1500	return r;
1448	} else	1501	} else
1449	return kvm_hv_set_msr(vcpu, msr, data, host);	1502	return kvm_hv_set_msr(vcpu, msr, data, host);
@@ -1451,12 +1504,22 @@ int kvm_hv_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) · 1 unmodified line
1451		1504
1452	int kvm_hv_get_msr_common(struct kvm_vcpu vcpu, u32 msr, u64 pdata, bool host)	1505	int kvm_hv_get_msr_common(struct kvm_vcpu vcpu, u32 msr, u64 pdata, bool host)
1453	{	1506	{
		1507	struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
		1508
		1509	if (!host && !vcpu->arch.hyperv_enabled)
		1510	return 1;
		1511
		1512	if (!to_hv_vcpu(vcpu)) {
		1513	if (kvm_hv_vcpu_init(vcpu))
		1514	return 1;
		1515	}
		1516
1454	if (kvm_hv_msr_partition_wide(msr)) {	1517	if (kvm_hv_msr_partition_wide(msr)) {
1455	int r;	1518	int r;
1456		1519
1457	mutex_lock(&vcpu->kvm->arch.hyperv.hv_lock);	1520	mutex_lock(&hv->hv_lock);
1458	r = kvm_hv_get_msr_pw(vcpu, msr, pdata, host);	1521	r = kvm_hv_get_msr_pw(vcpu, msr, pdata, host);
1459	mutex_unlock(&vcpu->kvm->arch.hyperv.hv_lock);	1522	mutex_unlock(&hv->hv_lock);
1460	return r;	1523	return r;
1461	} else	1524	} else
1462	return kvm_hv_get_msr(vcpu, msr, pdata, host);	1525	return kvm_hv_get_msr(vcpu, msr, pdata, host);
@@ -1466,7 +1529,7 @@ static __always_inline unsigned long *sparse_set_to_vcpu_mask( · 3 unmodified lines
1466	struct kvm kvm, u64 sparse_banks, u64 valid_bank_mask,	1529	struct kvm kvm, u64 sparse_banks, u64 valid_bank_mask,
1467	u64 vp_bitmap, unsigned long vcpu_bitmap)	1530	u64 vp_bitmap, unsigned long vcpu_bitmap)
1468	{	1531	{
1469	struct kvm_hv *hv = &kvm->arch.hyperv;	1532	struct kvm_hv *hv = to_kvm_hv(kvm);
1470	struct kvm_vcpu *vcpu;	1533	struct kvm_vcpu *vcpu;
1471	int i, bank, sbank = 0;	1534	int i, bank, sbank = 0;
1472		1535
@@ -1483,18 +1546,16 @@ static __always_inline unsigned long *sparse_set_to_vcpu_mask( · 10 unmodified lines
1483		1546
1484	bitmap_zero(vcpu_bitmap, KVM_MAX_VCPUS);	1547	bitmap_zero(vcpu_bitmap, KVM_MAX_VCPUS);
1485	kvm_for_each_vcpu(i, vcpu, kvm) {	1548	kvm_for_each_vcpu(i, vcpu, kvm) {
1486	if (test_bit(vcpu_to_hv_vcpu(vcpu)->vp_index,	1549	if (test_bit(kvm_hv_get_vpindex(vcpu), (unsigned long *)vp_bitmap))
1487	(unsigned long *)vp_bitmap))
1488	__set_bit(i, vcpu_bitmap);	1550	__set_bit(i, vcpu_bitmap);
1489	}	1551	}
1490	return vcpu_bitmap;	1552	return vcpu_bitmap;
1491	}	1553	}
1492		1554
1493	static u64 kvm_hv_flush_tlb(struct kvm_vcpu *current_vcpu, u64 ingpa,	1555	static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, u64 ingpa, u16 rep_cnt, bool ex)
1494	u16 rep_cnt, bool ex)
1495	{	1556	{
1496	struct kvm *kvm = current_vcpu->kvm;	1557	struct kvm *kvm = vcpu->kvm;
1497	struct kvm_vcpu_hv *hv_vcpu = &current_vcpu->arch.hyperv;	1558	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
1498	struct hv_tlb_flush_ex flush_ex;	1559	struct hv_tlb_flush_ex flush_ex;
1499	struct hv_tlb_flush flush;	1560	struct hv_tlb_flush flush;
1500	u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS];	1561	u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS];
@@ -1592,10 +1653,10 @@ static void kvm_send_ipi_to_many(struct kvm *kvm, u32 vector, · 91 unmodified lines
1592	}	1653	}
1593	}	1654	}
1594		1655
1595	static u64 kvm_hv_send_ipi(struct kvm_vcpu *current_vcpu, u64 ingpa, u64 outgpa,	1656	static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, u64 ingpa, u64 outgpa,
1596	bool ex, bool fast)	1657	bool ex, bool fast)
1597	{	1658	{
1598	struct kvm *kvm = current_vcpu->kvm;	1659	struct kvm *kvm = vcpu->kvm;
1599	struct hv_send_ipi_ex send_ipi_ex;	1660	struct hv_send_ipi_ex send_ipi_ex;
1600	struct hv_send_ipi send_ipi;	1661	struct hv_send_ipi send_ipi;
1601	u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS];	1662	u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS];
@@ -1666,9 +1727,20 @@ ret_success: · 64 unmodified lines
1666	return HV_STATUS_SUCCESS;	1727	return HV_STATUS_SUCCESS;
1667	}	1728	}
1668		1729
1669	bool kvm_hv_hypercall_enabled(struct kvm *kvm)	1730	void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu)
1670	{	1731	{
1671	return READ_ONCE(kvm->arch.hyperv.hv_guest_os_id) != 0;	1732	struct kvm_cpuid_entry2 *entry;
		1733
		1734	entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_INTERFACE, 0);
		1735	if (entry && entry->eax == HYPERV_CPUID_SIGNATURE_EAX)
		1736	vcpu->arch.hyperv_enabled = true;
		1737	else
		1738	vcpu->arch.hyperv_enabled = false;
		1739	}
		1740
		1741	bool kvm_hv_hypercall_enabled(struct kvm_vcpu *vcpu)
		1742	{
		1743	return vcpu->arch.hyperv_enabled && to_kvm_hv(vcpu->kvm)->hv_guest_os_id;
1672	}	1744	}
1673		1745
1674	static void kvm_hv_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result)	1746	static void kvm_hv_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result)
@@ -1698,6 +1770,7 @@ static int kvm_hv_hypercall_complete_userspace(struct kvm_vcpu *vcpu) · 23 unmodified lines
1698		1770
1699	static u16 kvm_hvcall_signal_event(struct kvm_vcpu *vcpu, bool fast, u64 param)	1771	static u16 kvm_hvcall_signal_event(struct kvm_vcpu *vcpu, bool fast, u64 param)
1700	{	1772	{
		1773	struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
1701	struct eventfd_ctx *eventfd;	1774	struct eventfd_ctx *eventfd;
1702		1775
1703	if (unlikely(!fast)) {	1776	if (unlikely(!fast)) {
@@ -1726,7 +1799,7 @@ static u16 kvm_hvcall_signal_event(struct kvm_vcpu *vcpu, bool fast, u64 param) · 22 unmodified lines
1726		1799
1727	/* the eventfd is protected by vcpu->kvm->srcu, but conn_to_evt isn't */	1800	/* the eventfd is protected by vcpu->kvm->srcu, but conn_to_evt isn't */
1728	rcu_read_lock();	1801	rcu_read_lock();
1729	eventfd = idr_find(&vcpu->kvm->arch.hyperv.conn_to_evt, param);	1802	eventfd = idr_find(&hv->conn_to_evt, param);
1730	rcu_read_unlock();	1803	rcu_read_unlock();
1731	if (!eventfd)	1804	if (!eventfd)
1732	return HV_STATUS_INVALID_PORT_ID;	1805	return HV_STATUS_INVALID_PORT_ID;
@@ -1745,7 +1818,7 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu) · 12 unmodified lines
1745	* hypercall generates UD from non zero cpl and real mode	1818	* hypercall generates UD from non zero cpl and real mode
1746	* per HYPER-V spec	1819	* per HYPER-V spec
1747	*/	1820	*/
1748	if (kvm_x86_ops.get_cpl(vcpu) != 0 \|\| !is_protmode(vcpu)) {	1821	if (static_call(kvm_x86_get_cpl)(vcpu) != 0 \|\| !is_protmode(vcpu)) {
1749	kvm_queue_exception(vcpu, UD_VECTOR);	1822	kvm_queue_exception(vcpu, UD_VECTOR);
1750	return 1;	1823	return 1;
1751	}	1824	}
@@ -1793,7 +1866,7 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu) · 41 unmodified lines
1793	fallthrough; /* maybe userspace knows this conn_id */	1866	fallthrough; /* maybe userspace knows this conn_id */
1794	case HVCALL_POST_MESSAGE:	1867	case HVCALL_POST_MESSAGE:
1795	/* don't bother userspace if it has no way to handle it */	1868	/* don't bother userspace if it has no way to handle it */
1796	if (unlikely(rep \|\| !vcpu_to_synic(vcpu)->active)) {	1869	if (unlikely(rep \|\| !to_hv_synic(vcpu)->active)) {
1797	ret = HV_STATUS_INVALID_HYPERCALL_INPUT;	1870	ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
1798	break;	1871	break;
1799	}	1872	}
@@ -1855,7 +1928,7 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu) · 55 unmodified lines
1855	}	1928	}
1856	fallthrough;	1929	fallthrough;
1857	case HVCALL_RESET_DEBUG_SESSION: {	1930	case HVCALL_RESET_DEBUG_SESSION: {
1858	struct kvm_hv_syndbg *syndbg = vcpu_to_hv_syndbg(vcpu);	1931	struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu);
1859		1932
1860	if (!kvm_hv_is_syndbg_enabled(vcpu)) {	1933	if (!kvm_hv_is_syndbg_enabled(vcpu)) {
1861	ret = HV_STATUS_INVALID_HYPERCALL_CODE;	1934	ret = HV_STATUS_INVALID_HYPERCALL_CODE;
@@ -1885,23 +1958,26 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu) · 23 unmodified lines
1885		1958
1886	void kvm_hv_init_vm(struct kvm *kvm)	1959	void kvm_hv_init_vm(struct kvm *kvm)
1887	{	1960	{
1888	mutex_init(&kvm->arch.hyperv.hv_lock);	1961	struct kvm_hv *hv = to_kvm_hv(kvm);
1889	idr_init(&kvm->arch.hyperv.conn_to_evt);	1962
		1963	mutex_init(&hv->hv_lock);
		1964	idr_init(&hv->conn_to_evt);
1890	}	1965	}
1891		1966
1892	void kvm_hv_destroy_vm(struct kvm *kvm)	1967	void kvm_hv_destroy_vm(struct kvm *kvm)
1893	{	1968	{
		1969	struct kvm_hv *hv = to_kvm_hv(kvm);
1894	struct eventfd_ctx *eventfd;	1970	struct eventfd_ctx *eventfd;
1895	int i;	1971	int i;
1896		1972
1897	idr_for_each_entry(&kvm->arch.hyperv.conn_to_evt, eventfd, i)	1973	idr_for_each_entry(&hv->conn_to_evt, eventfd, i)
1898	eventfd_ctx_put(eventfd);	1974	eventfd_ctx_put(eventfd);
1899	idr_destroy(&kvm->arch.hyperv.conn_to_evt);	1975	idr_destroy(&hv->conn_to_evt);
1900	}	1976	}
1901		1977
1902	static int kvm_hv_eventfd_assign(struct kvm *kvm, u32 conn_id, int fd)	1978	static int kvm_hv_eventfd_assign(struct kvm *kvm, u32 conn_id, int fd)
1903	{	1979	{
1904	struct kvm_hv *hv = &kvm->arch.hyperv;	1980	struct kvm_hv *hv = to_kvm_hv(kvm);
1905	struct eventfd_ctx *eventfd;	1981	struct eventfd_ctx *eventfd;
1906	int ret;	1982	int ret;
1907		1983
@@ -1925,7 +2001,7 @@ static int kvm_hv_eventfd_assign(struct kvm *kvm, u32 conn_id, int fd) · 17 unmodified lines
1925		2001
1926	static int kvm_hv_eventfd_deassign(struct kvm *kvm, u32 conn_id)	2002	static int kvm_hv_eventfd_deassign(struct kvm *kvm, u32 conn_id)
1927	{	2003	{
1928	struct kvm_hv *hv = &kvm->arch.hyperv;	2004	struct kvm_hv *hv = to_kvm_hv(kvm);
1929	struct eventfd_ctx *eventfd;	2005	struct eventfd_ctx *eventfd;
1930		2006
1931	mutex_lock(&hv->hv_lock);	2007	mutex_lock(&hv->hv_lock);
@@ -1997,8 +2073,7 @@ int kvm_get_hv_cpuid(struct kvm_vcpu vcpu, struct kvm_cpuid2 cpuid, · 65 unmodified lines
1997	break;	2073	break;
1998		2074
1999	case HYPERV_CPUID_INTERFACE:	2075	case HYPERV_CPUID_INTERFACE:
2000	memcpy(signature, "Hv#1\0\0\0\0\0\0\0\0", 12);	2076	ent->eax = HYPERV_CPUID_SIGNATURE_EAX;
2001	ent->eax = signature[0];
2002	break;	2077	break;
2003		2078
2004	case HYPERV_CPUID_VERSION:	2079	case HYPERV_CPUID_VERSION:

arch/x86/kvm/hyperv.h+33−21

 /* Hyper-V HV_X64_MSR_SYNDBG_OPTIONS bits */
 #define HV_X64_SYNDBG_OPTION_USE_HCALLS		BIT(2)
-static inline struct kvm_vcpu_hv *vcpu_to_hv_vcpu(struct kvm_vcpu *vcpu)
+static inline struct kvm_hv *to_kvm_hv(struct kvm *kvm)
 {
-	return &vcpu->arch.hyperv;
+	return &kvm->arch.hyperv;
 }
-static inline struct kvm_vcpu *hv_vcpu_to_vcpu(struct kvm_vcpu_hv *hv_vcpu)
+static inline struct kvm_vcpu_hv *to_hv_vcpu(struct kvm_vcpu *vcpu)
 {
-	struct kvm_vcpu_arch *arch;
+	return vcpu->arch.hyperv;
-	arch = container_of(hv_vcpu, struct kvm_vcpu_arch, hyperv);
-	return container_of(arch, struct kvm_vcpu, arch);
 }
-static inline struct kvm_vcpu_hv_synic *vcpu_to_synic(struct kvm_vcpu *vcpu)
+static inline struct kvm_vcpu_hv_synic *to_hv_synic(struct kvm_vcpu *vcpu)
 {
-	return &vcpu->arch.hyperv.synic;
+	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
+	return &hv_vcpu->synic;
 }
-static inline struct kvm_vcpu *synic_to_vcpu(struct kvm_vcpu_hv_synic *synic)
+static inline struct kvm_vcpu *hv_synic_to_vcpu(struct kvm_vcpu_hv_synic *synic)
 {
-	return hv_vcpu_to_vcpu(container_of(synic, struct kvm_vcpu_hv, synic));
+	struct kvm_vcpu_hv *hv_vcpu = container_of(synic, struct kvm_vcpu_hv, synic);
+	return hv_vcpu->vcpu;
 }
-static inline struct kvm_hv_syndbg *vcpu_to_hv_syndbg(struct kvm_vcpu *vcpu)
+static inline struct kvm_hv_syndbg *to_hv_syndbg(struct kvm_vcpu *vcpu)
 {
 	return &vcpu->kvm->arch.hyperv.hv_syndbg;
 }
+static inline u32 kvm_hv_get_vpindex(struct kvm_vcpu *vcpu)
+{
+	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
+	return hv_vcpu ? hv_vcpu->vp_index : kvm_vcpu_get_idx(vcpu);
+}
 int kvm_hv_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host);
 int kvm_hv_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host);
-bool kvm_hv_hypercall_enabled(struct kvm *kvm);
+bool kvm_hv_hypercall_enabled(struct kvm_vcpu *vcpu);
 int kvm_hv_hypercall(struct kvm_vcpu *vcpu);
 void kvm_hv_irq_routing_update(struct kvm *kvm);
 void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector);
 int kvm_hv_activate_synic(struct kvm_vcpu *vcpu, bool dont_zero_synic_pages);
-void kvm_hv_vcpu_init(struct kvm_vcpu *vcpu);
-void kvm_hv_vcpu_postcreate(struct kvm_vcpu *vcpu);
 void kvm_hv_vcpu_uninit(struct kvm_vcpu *vcpu);
 bool kvm_hv_assist_page_enabled(struct kvm_vcpu *vcpu);
 bool kvm_hv_get_assist_page(struct kvm_vcpu *vcpu,
 			    struct hv_vp_assist_page *assist_page);
-static inline struct kvm_vcpu_hv_stimer *vcpu_to_stimer(struct kvm_vcpu *vcpu,
+static inline struct kvm_vcpu_hv_stimer *to_hv_stimer(struct kvm_vcpu *vcpu,
-							int timer_index)
+						      int timer_index)
 {
-	return &vcpu_to_hv_vcpu(vcpu)->stimer[timer_index];
+	return &to_hv_vcpu(vcpu)->stimer[timer_index];
 }
-static inline struct kvm_vcpu *stimer_to_vcpu(struct kvm_vcpu_hv_stimer *stimer)
+static inline struct kvm_vcpu *hv_stimer_to_vcpu(struct kvm_vcpu_hv_stimer *stimer)
 {
 	struct kvm_vcpu_hv *hv_vcpu;
 	hv_vcpu = container_of(stimer - stimer->index, struct kvm_vcpu_hv,
 			       stimer[0]);
-	return hv_vcpu_to_vcpu(hv_vcpu);
+	return hv_vcpu->vcpu;
 }
 static inline bool kvm_hv_has_stimer_pending(struct kvm_vcpu *vcpu)
 {
-	return !bitmap_empty(vcpu->arch.hyperv.stimer_pending_bitmap,
+	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
+	if (!hv_vcpu)
+		return false;
+	return !bitmap_empty(hv_vcpu->stimer_pending_bitmap,
 			     HV_SYNIC_STIMER_COUNT);
 }
 void kvm_hv_init_vm(struct kvm *kvm);
 void kvm_hv_destroy_vm(struct kvm *kvm);
+void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu);
 int kvm_vm_ioctl_hv_eventfd(struct kvm *kvm, struct kvm_hyperv_eventfd *args);
 int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid,
 		     struct kvm_cpuid_entry2 __user *entries);

arch/x86/kvm/irq.c+8−2

 #include "irq.h"
 #include "i8254.h"
 #include "x86.h"
+#include "xen.h"
 /*
  * check if there are pending timer events
 	if (!lapic_in_kernel(v))
 		return v->arch.interrupt.injected;
+	if (kvm_xen_has_interrupt(v))
+		return 1;
 	if (!kvm_apic_accept_pic_intr(v))
 		return 0;
 	if (!lapic_in_kernel(v))
 		return v->arch.interrupt.nr;
+	if (kvm_xen_has_interrupt(v))
+		return v->kvm->arch.xen.upcall_vector;
 	if (irqchip_split(v->kvm)) {
 		int vector = v->arch.pending_external_vector;
 {
 	__kvm_migrate_apic_timer(vcpu);
 	__kvm_migrate_pit_timer(vcpu);
-	if (kvm_x86_ops.migrate_timers)
+	static_call_cond(kvm_x86_migrate_timers)(vcpu);
-		kvm_x86_ops.migrate_timers(vcpu);
 }
 bool kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args)

arch/x86/kvm/kvm_cache_regs.h+5−5

@@ -68,7 +68,7 @@ static inline unsigned long kvm_register_read(struct kvm_vcpu *vcpu, int reg) · 67 unmodified lines
68	return 0;	68	return 0;
69		69
70	if (!kvm_register_is_available(vcpu, reg))	70	if (!kvm_register_is_available(vcpu, reg))
71	kvm_x86_ops.cache_reg(vcpu, reg);	71	static_call(kvm_x86_cache_reg)(vcpu, reg);
72		72
73	return vcpu->arch.regs[reg];	73	return vcpu->arch.regs[reg];
74	}	74	}
@@ -108,7 +108,7 @@ static inline u64 kvm_pdptr_read(struct kvm_vcpu *vcpu, int index) · 33 unmodified lines
108	might_sleep(); /* on svm */	108	might_sleep(); /* on svm */
109		109
110	if (!kvm_register_is_available(vcpu, VCPU_EXREG_PDPTR))	110	if (!kvm_register_is_available(vcpu, VCPU_EXREG_PDPTR))
111	kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_PDPTR);	111	static_call(kvm_x86_cache_reg)(vcpu, VCPU_EXREG_PDPTR);
112		112
113	return vcpu->arch.walk_mmu->pdptrs[index];	113	return vcpu->arch.walk_mmu->pdptrs[index];
114	}	114	}
@@ -118,7 +118,7 @@ static inline ulong kvm_read_cr0_bits(struct kvm_vcpu *vcpu, ulong mask) · 3 unmodified lines
118	ulong tmask = mask & KVM_POSSIBLE_CR0_GUEST_BITS;	118	ulong tmask = mask & KVM_POSSIBLE_CR0_GUEST_BITS;
119	if ((tmask & vcpu->arch.cr0_guest_owned_bits) &&	119	if ((tmask & vcpu->arch.cr0_guest_owned_bits) &&
120	!kvm_register_is_available(vcpu, VCPU_EXREG_CR0))	120	!kvm_register_is_available(vcpu, VCPU_EXREG_CR0))
121	kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_CR0);	121	static_call(kvm_x86_cache_reg)(vcpu, VCPU_EXREG_CR0);
122	return vcpu->arch.cr0 & mask;	122	return vcpu->arch.cr0 & mask;
123	}	123	}
124		124
@@ -132,14 +132,14 @@ static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask) · 7 unmodified lines
132	ulong tmask = mask & KVM_POSSIBLE_CR4_GUEST_BITS;	132	ulong tmask = mask & KVM_POSSIBLE_CR4_GUEST_BITS;
133	if ((tmask & vcpu->arch.cr4_guest_owned_bits) &&	133	if ((tmask & vcpu->arch.cr4_guest_owned_bits) &&
134	!kvm_register_is_available(vcpu, VCPU_EXREG_CR4))	134	!kvm_register_is_available(vcpu, VCPU_EXREG_CR4))
135	kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_CR4);	135	static_call(kvm_x86_cache_reg)(vcpu, VCPU_EXREG_CR4);
136	return vcpu->arch.cr4 & mask;	136	return vcpu->arch.cr4 & mask;
137	}	137	}
138		138
139	static inline ulong kvm_read_cr3(struct kvm_vcpu *vcpu)	139	static inline ulong kvm_read_cr3(struct kvm_vcpu *vcpu)
140	{	140	{
141	if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3))	141	if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3))
142	kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_CR3);	142	static_call(kvm_x86_cache_reg)(vcpu, VCPU_EXREG_CR3);
143	return vcpu->arch.cr3;	143	return vcpu->arch.cr3;
144	}	144	}
145		145

arch/x86/kvm/kvm_emulate.h+1−1

@@ -205,7 +205,7 @@ struct x86_emulate_ops { · 204 unmodified lines
205	ulong (get_cr)(struct x86_emulate_ctxt ctxt, int cr);	205	ulong (get_cr)(struct x86_emulate_ctxt ctxt, int cr);
206	int (set_cr)(struct x86_emulate_ctxt ctxt, int cr, ulong val);	206	int (set_cr)(struct x86_emulate_ctxt ctxt, int cr, ulong val);
207	int (cpl)(struct x86_emulate_ctxt ctxt);	207	int (cpl)(struct x86_emulate_ctxt ctxt);
208	int (get_dr)(struct x86_emulate_ctxt ctxt, int dr, ulong *dest);	208	void (get_dr)(struct x86_emulate_ctxt ctxt, int dr, ulong *dest);
209	int (set_dr)(struct x86_emulate_ctxt ctxt, int dr, ulong value);	209	int (set_dr)(struct x86_emulate_ctxt ctxt, int dr, ulong value);
210	u64 (get_smbase)(struct x86_emulate_ctxt ctxt);	210	u64 (get_smbase)(struct x86_emulate_ctxt ctxt);
211	void (set_smbase)(struct x86_emulate_ctxt ctxt, u64 smbase);	211	void (set_smbase)(struct x86_emulate_ctxt ctxt, u64 smbase);

arch/x86/kvm/lapic.c+27−33

 	return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
 }
-struct static_key_deferred apic_hw_disabled __read_mostly;
+__read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_hw_disabled, HZ);
-struct static_key_deferred apic_sw_disabled __read_mostly;
+__read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_sw_disabled, HZ);
 static inline int apic_enabled(struct kvm_lapic *apic)
 {
 	if (enabled != apic->sw_enabled) {
 		apic->sw_enabled = enabled;
 		if (enabled)
-			static_key_slow_dec_deferred(&apic_sw_disabled);
+			static_branch_slow_dec_deferred(&apic_sw_disabled);
 		else
-			static_key_slow_inc(&apic_sw_disabled.key);
+			static_branch_inc(&apic_sw_disabled.key);
 		atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
 	}
 	if (unlikely(vcpu->arch.apicv_active)) {
 		/* need to update RVI */
 		kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
-		kvm_x86_ops.hwapic_irr_update(vcpu,
+		static_call(kvm_x86_hwapic_irr_update)(vcpu,
 				apic_find_highest_irr(apic));
 	} else {
 		apic->irr_pending = false;
 	 * just set SVI.
 	 */
 	if (unlikely(vcpu->arch.apicv_active))
-		kvm_x86_ops.hwapic_isr_update(vcpu, vec);
+		static_call(kvm_x86_hwapic_isr_update)(vcpu, vec);
 	else {
 		++apic->isr_count;
 		BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
 	 * and must be left alone.
 	 */
 	if (unlikely(vcpu->arch.apicv_active))
-		kvm_x86_ops.hwapic_isr_update(vcpu,
+		static_call(kvm_x86_hwapic_isr_update)(vcpu,
-					       apic_find_highest_isr(apic));
+						apic_find_highest_isr(apic));
 	else {
 		--apic->isr_count;
 		BUG_ON(apic->isr_count < 0);
 {
 	int highest_irr;
 	if (apic->vcpu->arch.apicv_active)
-		highest_irr = kvm_x86_ops.sync_pir_to_irr(apic->vcpu);
+		highest_irr = static_call(kvm_x86_sync_pir_to_irr)(apic->vcpu);
 	else
 		highest_irr = apic_find_highest_irr(apic);
 	if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr)
 						       apic->regs + APIC_TMR);
 		}
-		if (kvm_x86_ops.deliver_posted_interrupt(vcpu, vector)) {
+		if (static_call(kvm_x86_deliver_posted_interrupt)(vcpu, vector)) {
 			kvm_lapic_set_irr(vector, apic);
 			kvm_make_request(KVM_REQ_EVENT, vcpu);
 			kvm_vcpu_kick(vcpu);
 	apic_clear_isr(vector, apic);
 	apic_update_ppr(apic);
-	if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap))
+	if (to_hv_vcpu(apic->vcpu) &&
+	    test_bit(vector, to_hv_synic(apic->vcpu)->vec_bitmap))
 		kvm_hv_synic_send_eoi(apic->vcpu, vector);
 	kvm_ioapic_send_eoi(apic, vector);
 {
 	WARN_ON(preemptible());
 	WARN_ON(!apic->lapic_timer.hv_timer_in_use);
-	kvm_x86_ops.cancel_hv_timer(apic->vcpu);
+	static_call(kvm_x86_cancel_hv_timer)(apic->vcpu);
 	apic->lapic_timer.hv_timer_in_use = false;
 }
 	if (!ktimer->tscdeadline)
 		return false;
-	if (kvm_x86_ops.set_hv_timer(vcpu, ktimer->tscdeadline, &expired))
+	if (static_call(kvm_x86_set_hv_timer)(vcpu, ktimer->tscdeadline, &expired))
 		return false;
 	ktimer->hv_timer_in_use = true;
 	hrtimer_cancel(&apic->lapic_timer.timer);
 	if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
-		static_key_slow_dec_deferred(&apic_hw_disabled);
+		static_branch_slow_dec_deferred(&apic_hw_disabled);
 	if (!apic->sw_enabled)
-		static_key_slow_dec_deferred(&apic_sw_disabled);
+		static_branch_slow_dec_deferred(&apic_sw_disabled);
 	if (apic->regs)
 		free_page((unsigned long)apic->regs);
 	if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
 		if (value & MSR_IA32_APICBASE_ENABLE) {
 			kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
-			static_key_slow_dec_deferred(&apic_hw_disabled);
+			static_branch_slow_dec_deferred(&apic_hw_disabled);
 		} else {
-			static_key_slow_inc(&apic_hw_disabled.key);
+			static_branch_inc(&apic_hw_disabled.key);
 			atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
 		}
 	}
 		kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
 	if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE))
-		kvm_x86_ops.set_virtual_apic_mode(vcpu);
+		static_call(kvm_x86_set_virtual_apic_mode)(vcpu);
 	apic->base_address = apic->vcpu->arch.apic_base &
 			     MSR_IA32_APICBASE_BASE;
 	vcpu->arch.pv_eoi.msr_val = 0;
 	apic_update_ppr(apic);
 	if (vcpu->arch.apicv_active) {
-		kvm_x86_ops.apicv_post_state_restore(vcpu);
+		static_call(kvm_x86_apicv_post_state_restore)(vcpu);
-		kvm_x86_ops.hwapic_irr_update(vcpu, -1);
+		static_call(kvm_x86_hwapic_irr_update)(vcpu, -1);
-		kvm_x86_ops.hwapic_isr_update(vcpu, -1);
+		static_call(kvm_x86_hwapic_isr_update)(vcpu, -1);
 	}
 	vcpu->arch.apic_arb_prio = 0;
 	 * thinking that APIC state has changed.
 	 */
 	vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
-	static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
+	static_branch_inc(&apic_sw_disabled.key); /* sw disabled at reset */
 	kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
 	return 0;
 	 */
 	apic_clear_irr(vector, apic);
-	if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) {
+	if (to_hv_vcpu(vcpu) && test_bit(vector, to_hv_synic(vcpu)->auto_eoi_bitmap)) {
 		/*
 		 * For auto-EOI interrupts, there might be another pending
 		 * interrupt above PPR, so check whether to raise another
 	kvm_apic_update_apicv(vcpu);
 	apic->highest_isr_cache = -1;
 	if (vcpu->arch.apicv_active) {
-		kvm_x86_ops.apicv_post_state_restore(vcpu);
+		static_call(kvm_x86_apicv_post_state_restore)(vcpu);
-		kvm_x86_ops.hwapic_irr_update(vcpu,
+		static_call(kvm_x86_hwapic_irr_update)(vcpu,
 				apic_find_highest_irr(apic));
-		kvm_x86_ops.hwapic_isr_update(vcpu,
+		static_call(kvm_x86_hwapic_isr_update)(vcpu,
 				apic_find_highest_isr(apic));
 	}
 	kvm_make_request(KVM_REQ_EVENT, vcpu);
 	}
 }
-void kvm_lapic_init(void)
-{
-	/* do not patch jump label more than once per second */
-	jump_label_rate_limit(&apic_hw_disabled, HZ);
-	jump_label_rate_limit(&apic_sw_disabled, HZ);
-}
 void kvm_lapic_exit(void)
 {
 	static_key_deferred_flush(&apic_hw_disabled);

arch/x86/kvm/lapic.h+8−12

 #include <linux/kvm_host.h>
+#include "hyperv.h"
 #define KVM_APIC_INIT		0
 #define KVM_APIC_SIPI		1
 #define KVM_APIC_LVT_NUM	6
 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data);
 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data);
-static inline bool kvm_hv_vapic_assist_page_enabled(struct kvm_vcpu *vcpu)
-{
-	return vcpu->arch.hyperv.hv_vapic & HV_X64_MSR_VP_ASSIST_PAGE_ENABLE;
-}
 int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len);
-void kvm_lapic_init(void);
 void kvm_lapic_exit(void);
 #define VEC_POS(v) ((v) & (32 - 1))
 	__kvm_lapic_set_reg(apic->regs, reg_off, val);
 }
-extern struct static_key kvm_no_apic_vcpu;
+DECLARE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu);
 static inline bool lapic_in_kernel(struct kvm_vcpu *vcpu)
 {
-	if (static_key_false(&kvm_no_apic_vcpu))
+	if (static_branch_unlikely(&kvm_has_noapic_vcpu))
 		return vcpu->arch.apic;
 	return true;
 }
-extern struct static_key_deferred apic_hw_disabled;
+extern struct static_key_false_deferred apic_hw_disabled;
 static inline int kvm_apic_hw_enabled(struct kvm_lapic *apic)
 {
-	if (static_key_false(&apic_hw_disabled.key))
+	if (static_branch_unlikely(&apic_hw_disabled.key))
 		return apic->vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE;
 	return MSR_IA32_APICBASE_ENABLE;
 }
-extern struct static_key_deferred apic_sw_disabled;
+extern struct static_key_false_deferred apic_sw_disabled;
 static inline bool kvm_apic_sw_enabled(struct kvm_lapic *apic)
 {
-	if (static_key_false(&apic_sw_disabled.key))
+	if (static_branch_unlikely(&apic_sw_disabled.key))
 		return apic->sw_enabled;
 	return true;
 }

arch/x86/kvm/mmu.h+4−4

@@ -102,7 +102,7 @@ static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu) · 101 unmodified lines
102	if (!VALID_PAGE(root_hpa))	102	if (!VALID_PAGE(root_hpa))
103	return;	103	return;
104		104
105	kvm_x86_ops.load_mmu_pgd(vcpu, root_hpa \| kvm_get_active_pcid(vcpu),	105	static_call(kvm_x86_load_mmu_pgd)(vcpu, root_hpa \| kvm_get_active_pcid(vcpu),
106	vcpu->arch.mmu->shadow_root_level);	106	vcpu->arch.mmu->shadow_root_level);
107	}	107	}
108		108
@@ -152,7 +152,7 @@ static inline int kvm_mmu_do_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, · 43 unmodified lines
152	*	152	*
153	* TODO: introduce APIs to split these two cases.	153	* TODO: introduce APIs to split these two cases.
154	*/	154	*/
155	static inline int is_writable_pte(unsigned long pte)	155	static inline bool is_writable_pte(unsigned long pte)
156	{	156	{
157	return pte & PT_WRITABLE_MASK;	157	return pte & PT_WRITABLE_MASK;
158	}	158	}
@@ -174,8 +174,8 @@ static inline u8 permission_fault(struct kvm_vcpu vcpu, struct kvm_mmu mmu, · 15 unmodified lines
174	unsigned pte_access, unsigned pte_pkey,	174	unsigned pte_access, unsigned pte_pkey,
175	unsigned pfec)	175	unsigned pfec)
176	{	176	{
177	int cpl = kvm_x86_ops.get_cpl(vcpu);	177	int cpl = static_call(kvm_x86_get_cpl)(vcpu);
178	unsigned long rflags = kvm_x86_ops.get_rflags(vcpu);	178	unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
179		179
180	/*	180	/*
181	* If CPL < 3, SMAP prevention are disabled if EFLAGS.AC = 1.	181	* If CPL < 3, SMAP prevention are disabled if EFLAGS.AC = 1.

arch/x86/kvm/mmu/mmu.c+159−194

@@ -190,7 +190,7 @@ static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm, · 189 unmodified lines
190	int ret = -ENOTSUPP;	190	int ret = -ENOTSUPP;
191		191
192	if (range && kvm_x86_ops.tlb_remote_flush_with_range)	192	if (range && kvm_x86_ops.tlb_remote_flush_with_range)
193	ret = kvm_x86_ops.tlb_remote_flush_with_range(kvm, range);	193	ret = static_call(kvm_x86_tlb_remote_flush_with_range)(kvm, range);
194		194
195	if (ret)	195	if (ret)
196	kvm_flush_remote_tlbs(kvm);	196	kvm_flush_remote_tlbs(kvm);
@@ -844,17 +844,17 @@ static int pte_list_add(struct kvm_vcpu vcpu, u64 spte, · 647 unmodified lines
844	int i, count = 0;	844	int i, count = 0;
845		845
846	if (!rmap_head->val) {	846	if (!rmap_head->val) {
847	rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);	847	rmap_printk("%p %llx 0->1\n", spte, *spte);
848	rmap_head->val = (unsigned long)spte;	848	rmap_head->val = (unsigned long)spte;
849	} else if (!(rmap_head->val & 1)) {	849	} else if (!(rmap_head->val & 1)) {
850	rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);	850	rmap_printk("%p %llx 1->many\n", spte, *spte);
851	desc = mmu_alloc_pte_list_desc(vcpu);	851	desc = mmu_alloc_pte_list_desc(vcpu);
852	desc->sptes[0] = (u64 *)rmap_head->val;	852	desc->sptes[0] = (u64 *)rmap_head->val;
853	desc->sptes[1] = spte;	853	desc->sptes[1] = spte;
854	rmap_head->val = (unsigned long)desc \| 1;	854	rmap_head->val = (unsigned long)desc \| 1;
855	++count;	855	++count;
856	} else {	856	} else {
857	rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);	857	rmap_printk("%p %llx many->many\n", spte, *spte);
858	desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);	858	desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
859	while (desc->sptes[PTE_LIST_EXT-1]) {	859	while (desc->sptes[PTE_LIST_EXT-1]) {
860	count += PTE_LIST_EXT;	860	count += PTE_LIST_EXT;
@@ -906,14 +906,14 @@ static void __pte_list_remove(u64 spte, struct kvm_rmap_head rmap_head) · 45 unmodified lines
906	pr_err("%s: %p 0->BUG\n", __func__, spte);	906	pr_err("%s: %p 0->BUG\n", __func__, spte);
907	BUG();	907	BUG();
908	} else if (!(rmap_head->val & 1)) {	908	} else if (!(rmap_head->val & 1)) {
909	rmap_printk("%s: %p 1->0\n", __func__, spte);	909	rmap_printk("%p 1->0\n", spte);
910	if ((u64 *)rmap_head->val != spte) {	910	if ((u64 *)rmap_head->val != spte) {
911	pr_err("%s: %p 1->BUG\n", __func__, spte);	911	pr_err("%s: %p 1->BUG\n", __func__, spte);
912	BUG();	912	BUG();
913	}	913	}
914	rmap_head->val = 0;	914	rmap_head->val = 0;
915	} else {	915	} else {
916	rmap_printk("%s: %p many->many\n", __func__, spte);	916	rmap_printk("%p many->many\n", spte);
917	desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);	917	desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
918	prev_desc = NULL;	918	prev_desc = NULL;
919	while (desc) {	919	while (desc) {
@@ -1115,7 +1115,7 @@ static bool spte_write_protect(u64 *sptep, bool pt_protect) · 195 unmodified lines
1115	!(pt_protect && spte_can_locklessly_be_made_writable(spte)))	1115	!(pt_protect && spte_can_locklessly_be_made_writable(spte)))
1116	return false;	1116	return false;
1117		1117
1118	rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep);	1118	rmap_printk("spte %p %llx\n", sptep, *sptep);
1119		1119
1120	if (pt_protect)	1120	if (pt_protect)
1121	spte &= ~SPTE_MMU_WRITEABLE;	1121	spte &= ~SPTE_MMU_WRITEABLE;
@@ -1142,7 +1142,7 @@ static bool spte_clear_dirty(u64 *sptep) · 20 unmodified lines
1142	{	1142	{
1143	u64 spte = *sptep;	1143	u64 spte = *sptep;
1144		1144
1145	rmap_printk("rmap_clear_dirty: spte %p %llx\n", sptep, *sptep);	1145	rmap_printk("spte %p %llx\n", sptep, *sptep);
1146		1146
1147	MMU_WARN_ON(!spte_ad_enabled(spte));	1147	MMU_WARN_ON(!spte_ad_enabled(spte));
1148	spte &= ~shadow_dirty_mask;	1148	spte &= ~shadow_dirty_mask;
@@ -1184,7 +1184,7 @@ static bool spte_set_dirty(u64 *sptep) · 35 unmodified lines
1184	{	1184	{
1185	u64 spte = *sptep;	1185	u64 spte = *sptep;
1186		1186
1187	rmap_printk("rmap_set_dirty: spte %p %llx\n", sptep, *sptep);	1187	rmap_printk("spte %p %llx\n", sptep, *sptep);
1188		1188
1189	/*	1189	/*
1190	* Similar to the !kvm_x86_ops.slot_disable_log_dirty case,	1190	* Similar to the !kvm_x86_ops.slot_disable_log_dirty case,
@@ -1225,7 +1225,7 @@ static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, · 34 unmodified lines
1225	{	1225	{
1226	struct kvm_rmap_head *rmap_head;	1226	struct kvm_rmap_head *rmap_head;
1227		1227
1228	if (kvm->arch.tdp_mmu_enabled)	1228	if (is_tdp_mmu_enabled(kvm))
1229	kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,	1229	kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,
1230	slot->base_gfn + gfn_offset, mask, true);	1230	slot->base_gfn + gfn_offset, mask, true);
1231	while (mask) {	1231	while (mask) {
@@ -1254,7 +1254,7 @@ void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, · 22 unmodified lines
1254	{	1254	{
1255	struct kvm_rmap_head *rmap_head;	1255	struct kvm_rmap_head *rmap_head;
1256		1256
1257	if (kvm->arch.tdp_mmu_enabled)	1257	if (is_tdp_mmu_enabled(kvm))
1258	kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,	1258	kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,
1259	slot->base_gfn + gfn_offset, mask, false);	1259	slot->base_gfn + gfn_offset, mask, false);
1260	while (mask) {	1260	while (mask) {
@@ -1283,8 +1283,9 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, · 22 unmodified lines
1283	gfn_t gfn_offset, unsigned long mask)	1283	gfn_t gfn_offset, unsigned long mask)
1284	{	1284	{
1285	if (kvm_x86_ops.enable_log_dirty_pt_masked)	1285	if (kvm_x86_ops.enable_log_dirty_pt_masked)
1286	kvm_x86_ops.enable_log_dirty_pt_masked(kvm, slot, gfn_offset,	1286	static_call(kvm_x86_enable_log_dirty_pt_masked)(kvm, slot,
1287	mask);	1287	gfn_offset,
		1288	mask);
1288	else	1289	else
1289	kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);	1290	kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
1290	}	1291	}
@@ -1292,7 +1293,7 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, · 1 unmodified line
1292	int kvm_cpu_dirty_log_size(void)	1293	int kvm_cpu_dirty_log_size(void)
1293	{	1294	{
1294	if (kvm_x86_ops.cpu_dirty_log_size)	1295	if (kvm_x86_ops.cpu_dirty_log_size)
1295	return kvm_x86_ops.cpu_dirty_log_size();	1296	return static_call(kvm_x86_cpu_dirty_log_size)();
1296		1297
1297	return 0;	1298	return 0;
1298	}	1299	}
@@ -1309,7 +1310,7 @@ bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm, · 10 unmodified lines
1309	write_protected \|= __rmap_write_protect(kvm, rmap_head, true);	1310	write_protected \|= __rmap_write_protect(kvm, rmap_head, true);
1310	}	1311	}
1311		1312
1312	if (kvm->arch.tdp_mmu_enabled)	1313	if (is_tdp_mmu_enabled(kvm))
1313	write_protected \|=	1314	write_protected \|=
1314	kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn);	1315	kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn);
1315		1316
@@ -1331,7 +1332,7 @@ static bool kvm_zap_rmapp(struct kvm kvm, struct kvm_rmap_head rmap_head) · 15 unmodified lines
1331	bool flush = false;	1332	bool flush = false;
1332		1333
1333	while ((sptep = rmap_get_first(rmap_head, &iter))) {	1334	while ((sptep = rmap_get_first(rmap_head, &iter))) {
1334	rmap_printk("%s: spte %p %llx.\n", __func__, sptep, *sptep);	1335	rmap_printk("spte %p %llx.\n", sptep, *sptep);
1335		1336
1336	pte_list_remove(rmap_head, sptep);	1337	pte_list_remove(rmap_head, sptep);
1337	flush = true;	1338	flush = true;
@@ -1363,7 +1364,7 @@ static int kvm_set_pte_rmapp(struct kvm kvm, struct kvm_rmap_head rmap_head, · 25 unmodified lines
1363		1364
1364	restart:	1365	restart:
1365	for_each_rmap_spte(rmap_head, &iter, sptep) {	1366	for_each_rmap_spte(rmap_head, &iter, sptep) {
1366	rmap_printk("kvm_set_pte_rmapp: spte %p %llx gfn %llx (%d)\n",	1367	rmap_printk("spte %p %llx gfn %llx (%d)\n",
1367	sptep, *sptep, gfn, level);	1368	sptep, *sptep, gfn, level);
1368		1369
1369	need_flush = 1;	1370	need_flush = 1;
@@ -1456,16 +1457,17 @@ static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator) · 86 unmodified lines
1456	slot_rmap_walk_okay(_iter_); \	1457	slot_rmap_walk_okay(_iter_); \
1457	slot_rmap_walk_next(_iter_))	1458	slot_rmap_walk_next(_iter_))
1458		1459
1459	static int kvm_handle_hva_range(struct kvm *kvm,	1460	static __always_inline int
1460	unsigned long start,	1461	kvm_handle_hva_range(struct kvm *kvm,
1461	unsigned long end,	1462	unsigned long start,
1462	unsigned long data,	1463	unsigned long end,
1463	int (handler)(struct kvm kvm,	1464	unsigned long data,
1464	struct kvm_rmap_head *rmap_head,	1465	int (handler)(struct kvm kvm,
1465	struct kvm_memory_slot *slot,	1466	struct kvm_rmap_head *rmap_head,
1466	gfn_t gfn,	1467	struct kvm_memory_slot *slot,
1467	int level,	1468	gfn_t gfn,
1468	unsigned long data))	1469	int level,
		1470	unsigned long data))
1469	{	1471	{
1470	struct kvm_memslots *slots;	1472	struct kvm_memslots *slots;
1471	struct kvm_memory_slot *memslot;	1473	struct kvm_memory_slot *memslot;
@@ -1521,7 +1523,7 @@ int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end, · 49 unmodified lines
1521		1523
1522	r = kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);	1524	r = kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
1523		1525
1524	if (kvm->arch.tdp_mmu_enabled)	1526	if (is_tdp_mmu_enabled(kvm))
1525	r \|= kvm_tdp_mmu_zap_hva_range(kvm, start, end);	1527	r \|= kvm_tdp_mmu_zap_hva_range(kvm, start, end);
1526		1528
1527	return r;	1529	return r;
@@ -1533,7 +1535,7 @@ int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) · 5 unmodified lines
1533		1535
1534	r = kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);	1536	r = kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
1535		1537
1536	if (kvm->arch.tdp_mmu_enabled)	1538	if (is_tdp_mmu_enabled(kvm))
1537	r \|= kvm_tdp_mmu_set_spte_hva(kvm, hva, &pte);	1539	r \|= kvm_tdp_mmu_set_spte_hva(kvm, hva, &pte);
1538		1540
1539	return r;	1541	return r;
@@ -1588,7 +1590,7 @@ int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end) · 48 unmodified lines
1588	int young = false;	1590	int young = false;
1589		1591
1590	young = kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp);	1592	young = kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp);
1591	if (kvm->arch.tdp_mmu_enabled)	1593	if (is_tdp_mmu_enabled(kvm))
1592	young \|= kvm_tdp_mmu_age_hva_range(kvm, start, end);	1594	young \|= kvm_tdp_mmu_age_hva_range(kvm, start, end);
1593		1595
1594	return young;	1596	return young;
@@ -1599,7 +1601,7 @@ int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) · 4 unmodified lines
1599	int young = false;	1601	int young = false;
1600		1602
1601	young = kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);	1603	young = kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
1602	if (kvm->arch.tdp_mmu_enabled)	1604	if (is_tdp_mmu_enabled(kvm))
1603	young \|= kvm_tdp_mmu_test_age_hva(kvm, hva);	1605	young \|= kvm_tdp_mmu_test_age_hva(kvm, hva);
1604		1606
1605	return young;	1607	return young;
@@ -1723,13 +1725,6 @@ static int nonpaging_sync_page(struct kvm_vcpu *vcpu, · 117 unmodified lines
1723	return 0;	1725	return 0;
1724	}	1726	}
1725		1727
1726	static void nonpaging_update_pte(struct kvm_vcpu *vcpu,
1727	struct kvm_mmu_page sp, u64 spte,
1728	const void *pte)
1729	{
1730	WARN_ON(1);
1731	}
1732
1733	#define KVM_PAGE_ARRAY_NR 16	1728	#define KVM_PAGE_ARRAY_NR 16
1734		1729
1735	struct kvm_mmu_pages {	1730	struct kvm_mmu_pages {
@@ -2016,9 +2011,9 @@ static void mmu_sync_children(struct kvm_vcpu *vcpu, · 280 unmodified lines
2016	flush \|= kvm_sync_page(vcpu, sp, &invalid_list);	2011	flush \|= kvm_sync_page(vcpu, sp, &invalid_list);
2017	mmu_pages_clear_parents(&parents);	2012	mmu_pages_clear_parents(&parents);
2018	}	2013	}
2019	if (need_resched() \|\| spin_needbreak(&vcpu->kvm->mmu_lock)) {	2014	if (need_resched() \|\| rwlock_needbreak(&vcpu->kvm->mmu_lock)) {
2020	kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);	2015	kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);
2021	cond_resched_lock(&vcpu->kvm->mmu_lock);	2016	cond_resched_rwlock_write(&vcpu->kvm->mmu_lock);
2022	flush = false;	2017	flush = false;
2023	}	2018	}
2024	}	2019	}
@@ -2417,7 +2412,7 @@ static unsigned long kvm_mmu_zap_oldest_mmu_pages(struct kvm *kvm, · 392 unmodified lines
2417	return 0;	2412	return 0;
2418		2413
2419	restart:	2414	restart:
2420	list_for_each_entry_safe(sp, tmp, &kvm->arch.active_mmu_pages, link) {	2415	list_for_each_entry_safe_reverse(sp, tmp, &kvm->arch.active_mmu_pages, link) {
2421	/*	2416	/*
2422	* Don't zap active root pages, the page itself can't be freed	2417	* Don't zap active root pages, the page itself can't be freed
2423	* and zapping it will just force vCPUs to realloc and reload.	2418	* and zapping it will just force vCPUs to realloc and reload.
@@ -2470,7 +2465,7 @@ static int make_mmu_pages_available(struct kvm_vcpu *vcpu) · 46 unmodified lines
2470	*/	2465	*/
2471	void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages)	2466	void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages)
2472	{	2467	{
2473	spin_lock(&kvm->mmu_lock);	2468	write_lock(&kvm->mmu_lock);
2474		2469
2475	if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {	2470	if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
2476	kvm_mmu_zap_oldest_mmu_pages(kvm, kvm->arch.n_used_mmu_pages -	2471	kvm_mmu_zap_oldest_mmu_pages(kvm, kvm->arch.n_used_mmu_pages -
@@ -2481,7 +2476,7 @@ void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages) · 4 unmodified lines
2481		2476
2482	kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;	2477	kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
2483		2478
2484	spin_unlock(&kvm->mmu_lock);	2479	write_unlock(&kvm->mmu_lock);
2485	}	2480	}
2486		2481
2487	int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)	2482	int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
@@ -2492,7 +2487,7 @@ int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) · 4 unmodified lines
2492		2487
2493	pgprintk("%s: looking for gfn %llx\n", __func__, gfn);	2488	pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
2494	r = 0;	2489	r = 0;
2495	spin_lock(&kvm->mmu_lock);	2490	write_lock(&kvm->mmu_lock);
2496	for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {	2491	for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {
2497	pgprintk("%s: gfn %llx role %x\n", __func__, gfn,	2492	pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
2498	sp->role.word);	2493	sp->role.word);
@@ -2500,7 +2495,7 @@ int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) · 1 unmodified line
2500	kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);	2495	kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
2501	}	2496	}
2502	kvm_mmu_commit_zap_page(kvm, &invalid_list);	2497	kvm_mmu_commit_zap_page(kvm, &invalid_list);
2503	spin_unlock(&kvm->mmu_lock);	2498	write_unlock(&kvm->mmu_lock);
2504		2499
2505	return r;	2500	return r;
2506	}	2501	}
@@ -3161,7 +3156,7 @@ static void mmu_free_root_page(struct kvm kvm, hpa_t root_hpa, · 654 unmodified lines
3161	sp = to_shadow_page(*root_hpa & PT64_BASE_ADDR_MASK);	3156	sp = to_shadow_page(*root_hpa & PT64_BASE_ADDR_MASK);
3162		3157
3163	if (kvm_mmu_put_root(kvm, sp)) {	3158	if (kvm_mmu_put_root(kvm, sp)) {
3164	if (sp->tdp_mmu_page)	3159	if (is_tdp_mmu_page(sp))
3165	kvm_tdp_mmu_free_root(kvm, sp);	3160	kvm_tdp_mmu_free_root(kvm, sp);
3166	else if (sp->role.invalid)	3161	else if (sp->role.invalid)
3167	kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);	3162	kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
@@ -3192,7 +3187,7 @@ void kvm_mmu_free_roots(struct kvm_vcpu vcpu, struct kvm_mmu mmu, · 24 unmodified lines
3192	return;	3187	return;
3193	}	3188	}
3194		3189
3195	spin_lock(&kvm->mmu_lock);	3190	write_lock(&kvm->mmu_lock);
3196		3191
3197	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)	3192	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
3198	if (roots_to_free & KVM_MMU_ROOT_PREVIOUS(i))	3193	if (roots_to_free & KVM_MMU_ROOT_PREVIOUS(i))
@@ -3215,7 +3210,7 @@ void kvm_mmu_free_roots(struct kvm_vcpu vcpu, struct kvm_mmu mmu, · 16 unmodified lines
3215	}	3210	}
3216		3211
3217	kvm_mmu_commit_zap_page(kvm, &invalid_list);	3212	kvm_mmu_commit_zap_page(kvm, &invalid_list);
3218	spin_unlock(&kvm->mmu_lock);	3213	write_unlock(&kvm->mmu_lock);
3219	}	3214	}
3220	EXPORT_SYMBOL_GPL(kvm_mmu_free_roots);	3215	EXPORT_SYMBOL_GPL(kvm_mmu_free_roots);
3221		3216
@@ -3236,16 +3231,16 @@ static hpa_t mmu_alloc_root(struct kvm_vcpu *vcpu, gfn_t gfn, gva_t gva, · 14 unmodified lines
3236	{	3231	{
3237	struct kvm_mmu_page *sp;	3232	struct kvm_mmu_page *sp;
3238		3233
3239	spin_lock(&vcpu->kvm->mmu_lock);	3234	write_lock(&vcpu->kvm->mmu_lock);
3240		3235
3241	if (make_mmu_pages_available(vcpu)) {	3236	if (make_mmu_pages_available(vcpu)) {
3242	spin_unlock(&vcpu->kvm->mmu_lock);	3237	write_unlock(&vcpu->kvm->mmu_lock);
3243	return INVALID_PAGE;	3238	return INVALID_PAGE;
3244	}	3239	}
3245	sp = kvm_mmu_get_page(vcpu, gfn, gva, level, direct, ACC_ALL);	3240	sp = kvm_mmu_get_page(vcpu, gfn, gva, level, direct, ACC_ALL);
3246	++sp->root_count;	3241	++sp->root_count;
3247		3242
3248	spin_unlock(&vcpu->kvm->mmu_lock);	3243	write_unlock(&vcpu->kvm->mmu_lock);
3249	return __pa(sp->spt);	3244	return __pa(sp->spt);
3250	}	3245	}
3251		3246
@@ -3255,7 +3250,7 @@ static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu) · 3 unmodified lines
3255	hpa_t root;	3250	hpa_t root;
3256	unsigned i;	3251	unsigned i;
3257		3252
3258	if (vcpu->kvm->arch.tdp_mmu_enabled) {	3253	if (is_tdp_mmu_enabled(vcpu->kvm)) {
3259	root = kvm_tdp_mmu_get_vcpu_root_hpa(vcpu);	3254	root = kvm_tdp_mmu_get_vcpu_root_hpa(vcpu);
3260		3255
3261	if (!VALID_PAGE(root))	3256	if (!VALID_PAGE(root))
@@ -3416,17 +3411,17 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) · 154 unmodified lines
3416	!smp_load_acquire(&sp->unsync_children))	3411	!smp_load_acquire(&sp->unsync_children))
3417	return;	3412	return;
3418		3413
3419	spin_lock(&vcpu->kvm->mmu_lock);	3414	write_lock(&vcpu->kvm->mmu_lock);
3420	kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);	3415	kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
3421		3416
3422	mmu_sync_children(vcpu, sp);	3417	mmu_sync_children(vcpu, sp);
3423		3418
3424	kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);	3419	kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3425	spin_unlock(&vcpu->kvm->mmu_lock);	3420	write_unlock(&vcpu->kvm->mmu_lock);
3426	return;	3421	return;
3427	}	3422	}
3428		3423
3429	spin_lock(&vcpu->kvm->mmu_lock);	3424	write_lock(&vcpu->kvm->mmu_lock);
3430	kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);	3425	kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
3431		3426
3432	for (i = 0; i < 4; ++i) {	3427	for (i = 0; i < 4; ++i) {
@@ -3440,7 +3435,7 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) · 7 unmodified lines
3440	}	3435	}
3441		3436
3442	kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);	3437	kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3443	spin_unlock(&vcpu->kvm->mmu_lock);	3438	write_unlock(&vcpu->kvm->mmu_lock);
3444	}	3439	}
3445	EXPORT_SYMBOL_GPL(kvm_mmu_sync_roots);	3440	EXPORT_SYMBOL_GPL(kvm_mmu_sync_roots);
3446		3441
@@ -3724,7 +3719,12 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, · 277 unmodified lines
3724	return r;	3719	return r;
3725		3720
3726	r = RET_PF_RETRY;	3721	r = RET_PF_RETRY;
3727	spin_lock(&vcpu->kvm->mmu_lock);	3722
		3723	if (is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa))
		3724	read_lock(&vcpu->kvm->mmu_lock);
		3725	else
		3726	write_lock(&vcpu->kvm->mmu_lock);
		3727
3728	if (mmu_notifier_retry(vcpu->kvm, mmu_seq))	3728	if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
3729	goto out_unlock;	3729	goto out_unlock;
3730	r = make_mmu_pages_available(vcpu);	3730	r = make_mmu_pages_available(vcpu);
@@ -3739,7 +3739,10 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, · 8 unmodified lines
3739	prefault, is_tdp);	3739	prefault, is_tdp);
3740		3740
3741	out_unlock:	3741	out_unlock:
3742	spin_unlock(&vcpu->kvm->mmu_lock);	3742	if (is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa))
		3743	read_unlock(&vcpu->kvm->mmu_lock);
		3744	else
		3745	write_unlock(&vcpu->kvm->mmu_lock);
3743	kvm_release_pfn_clean(pfn);	3746	kvm_release_pfn_clean(pfn);
3744	return r;	3747	return r;
3745	}	3748	}
@@ -3813,7 +3816,6 @@ static void nonpaging_init_context(struct kvm_vcpu *vcpu, · 67 unmodified lines
3813	context->gva_to_gpa = nonpaging_gva_to_gpa;	3816	context->gva_to_gpa = nonpaging_gva_to_gpa;
3814	context->sync_page = nonpaging_sync_page;	3817	context->sync_page = nonpaging_sync_page;
3815	context->invlpg = NULL;	3818	context->invlpg = NULL;
3816	context->update_pte = nonpaging_update_pte;
3817	context->root_level = 0;	3819	context->root_level = 0;
3818	context->shadow_root_level = PT32E_ROOT_LEVEL;	3820	context->shadow_root_level = PT32E_ROOT_LEVEL;
3819	context->direct_map = true;	3821	context->direct_map = true;
@@ -3984,20 +3986,27 @@ static inline bool is_last_gpte(struct kvm_mmu *mmu, · 164 unmodified lines
3984	static void	3986	static void
3985	__reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,	3987	__reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
3986	struct rsvd_bits_validate *rsvd_check,	3988	struct rsvd_bits_validate *rsvd_check,
3987	int maxphyaddr, int level, bool nx, bool gbpages,	3989	u64 pa_bits_rsvd, int level, bool nx, bool gbpages,
3988	bool pse, bool amd)	3990	bool pse, bool amd)
3989	{	3991	{
3990	u64 exb_bit_rsvd = 0;
3991	u64 gbpages_bit_rsvd = 0;	3992	u64 gbpages_bit_rsvd = 0;
3992	u64 nonleaf_bit8_rsvd = 0;	3993	u64 nonleaf_bit8_rsvd = 0;
		3994	u64 high_bits_rsvd;
3993		3995
3994	rsvd_check->bad_mt_xwr = 0;	3996	rsvd_check->bad_mt_xwr = 0;
3995		3997
3996	if (!nx)
3997	exb_bit_rsvd = rsvd_bits(63, 63);
3998	if (!gbpages)	3998	if (!gbpages)
3999	gbpages_bit_rsvd = rsvd_bits(7, 7);	3999	gbpages_bit_rsvd = rsvd_bits(7, 7);
4000		4000
		4001	if (level == PT32E_ROOT_LEVEL)
		4002	high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 62);
		4003	else
		4004	high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 51);
		4005
		4006	/* Note, NX doesn't exist in PDPTEs, this is handled below. */
		4007	if (!nx)
		4008	high_bits_rsvd \|= rsvd_bits(63, 63);
		4009
4001	/*	4010	/*
4002	* Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for	4011	* Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for
4003	* leaf entries) on AMD CPUs only.	4012	* leaf entries) on AMD CPUs only.
@@ -4026,45 +4035,39 @@ __reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, · 22 unmodified lines
4026	rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);	4035	rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
4027	break;	4036	break;
4028	case PT32E_ROOT_LEVEL:	4037	case PT32E_ROOT_LEVEL:
4029	rsvd_check->rsvd_bits_mask[0][2] =	4038	rsvd_check->rsvd_bits_mask[0][2] = rsvd_bits(63, 63) \|
4030	rsvd_bits(maxphyaddr, 63) \|	4039	high_bits_rsvd \|
4031	rsvd_bits(5, 8) \| rsvd_bits(1, 2); /* PDPTE */	4040	rsvd_bits(5, 8) \|
4032	rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd \|	4041	rsvd_bits(1, 2); /* PDPTE */
4033	rsvd_bits(maxphyaddr, 62); /* PDE */	4042	rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd; /* PDE */
4034	rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd \|	4043	rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd; /* PTE */
4035	rsvd_bits(maxphyaddr, 62); /* PTE */	4044	rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd \|
4036	rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd \|	4045	rsvd_bits(13, 20); /* large page */
4037	rsvd_bits(maxphyaddr, 62) \|
4038	rsvd_bits(13, 20); /* large page */
4039	rsvd_check->rsvd_bits_mask[1][0] =	4046	rsvd_check->rsvd_bits_mask[1][0] =
4040	rsvd_check->rsvd_bits_mask[0][0];	4047	rsvd_check->rsvd_bits_mask[0][0];
4041	break;	4048	break;
4042	case PT64_ROOT_5LEVEL:	4049	case PT64_ROOT_5LEVEL:
4043	rsvd_check->rsvd_bits_mask[0][4] = exb_bit_rsvd \|	4050	rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd \|
4044	nonleaf_bit8_rsvd \| rsvd_bits(7, 7) \|	4051	nonleaf_bit8_rsvd \|
4045	rsvd_bits(maxphyaddr, 51);	4052	rsvd_bits(7, 7);
4046	rsvd_check->rsvd_bits_mask[1][4] =	4053	rsvd_check->rsvd_bits_mask[1][4] =
4047	rsvd_check->rsvd_bits_mask[0][4];	4054	rsvd_check->rsvd_bits_mask[0][4];
4048	fallthrough;	4055	fallthrough;
4049	case PT64_ROOT_4LEVEL:	4056	case PT64_ROOT_4LEVEL:
4050	rsvd_check->rsvd_bits_mask[0][3] = exb_bit_rsvd \|	4057	rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd \|
4051	nonleaf_bit8_rsvd \| rsvd_bits(7, 7) \|	4058	nonleaf_bit8_rsvd \|
4052	rsvd_bits(maxphyaddr, 51);	4059	rsvd_bits(7, 7);
4053	rsvd_check->rsvd_bits_mask[0][2] = exb_bit_rsvd \|	4060	rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd \|
4054	gbpages_bit_rsvd \|	4061	gbpages_bit_rsvd;
4055	rsvd_bits(maxphyaddr, 51);	4062	rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;
4056	rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd \|	4063	rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
4057	rsvd_bits(maxphyaddr, 51);
4058	rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd \|
4059	rsvd_bits(maxphyaddr, 51);
4060	rsvd_check->rsvd_bits_mask[1][3] =	4064	rsvd_check->rsvd_bits_mask[1][3] =
4061	rsvd_check->rsvd_bits_mask[0][3];	4065	rsvd_check->rsvd_bits_mask[0][3];
4062	rsvd_check->rsvd_bits_mask[1][2] = exb_bit_rsvd \|	4066	rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd \|
4063	gbpages_bit_rsvd \| rsvd_bits(maxphyaddr, 51) \|	4067	gbpages_bit_rsvd \|
4064	rsvd_bits(13, 29);	4068	rsvd_bits(13, 29);
4065	rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd \|	4069	rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd \|
4066	rsvd_bits(maxphyaddr, 51) \|	4070	rsvd_bits(13, 20); /* large page */
4067	rsvd_bits(13, 20); /* large page */
4068	rsvd_check->rsvd_bits_mask[1][0] =	4071	rsvd_check->rsvd_bits_mask[1][0] =
4069	rsvd_check->rsvd_bits_mask[0][0];	4072	rsvd_check->rsvd_bits_mask[0][0];
4070	break;	4073	break;
@@ -4075,8 +4078,8 @@ static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, · 4 unmodified lines
4075	struct kvm_mmu *context)	4078	struct kvm_mmu *context)
4076	{	4079	{
4077	__reset_rsvds_bits_mask(vcpu, &context->guest_rsvd_check,	4080	__reset_rsvds_bits_mask(vcpu, &context->guest_rsvd_check,
4078	cpuid_maxphyaddr(vcpu), context->root_level,	4081	vcpu->arch.reserved_gpa_bits,
4079	context->nx,	4082	context->root_level, context->nx,
4080	guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES),	4083	guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES),
4081	is_pse(vcpu),	4084	is_pse(vcpu),
4082	guest_cpuid_is_amd_or_hygon(vcpu));	4085	guest_cpuid_is_amd_or_hygon(vcpu));
@@ -4084,27 +4087,22 @@ static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, · 1 unmodified line
4084		4087
4085	static void	4088	static void
4086	__reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,	4089	__reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
4087	int maxphyaddr, bool execonly)	4090	u64 pa_bits_rsvd, bool execonly)
4088	{	4091	{
		4092	u64 high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 51);
4089	u64 bad_mt_xwr;	4093	u64 bad_mt_xwr;
4090		4094
4091	rsvd_check->rsvd_bits_mask[0][4] =	4095	rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd \| rsvd_bits(3, 7);
4092	rsvd_bits(maxphyaddr, 51) \| rsvd_bits(3, 7);	4096	rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd \| rsvd_bits(3, 7);
4093	rsvd_check->rsvd_bits_mask[0][3] =	4097	rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd \| rsvd_bits(3, 6);
4094	rsvd_bits(maxphyaddr, 51) \| rsvd_bits(3, 7);	4098	rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd \| rsvd_bits(3, 6);
4095	rsvd_check->rsvd_bits_mask[0][2] =	4099	rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
4096	rsvd_bits(maxphyaddr, 51) \| rsvd_bits(3, 6);
4097	rsvd_check->rsvd_bits_mask[0][1] =
4098	rsvd_bits(maxphyaddr, 51) \| rsvd_bits(3, 6);
4099	rsvd_check->rsvd_bits_mask[0][0] = rsvd_bits(maxphyaddr, 51);
4100		4100
4101	/* large page */	4101	/* large page */
4102	rsvd_check->rsvd_bits_mask[1][4] = rsvd_check->rsvd_bits_mask[0][4];	4102	rsvd_check->rsvd_bits_mask[1][4] = rsvd_check->rsvd_bits_mask[0][4];
4103	rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];	4103	rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];
4104	rsvd_check->rsvd_bits_mask[1][2] =	4104	rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd \| rsvd_bits(12, 29);
4105	rsvd_bits(maxphyaddr, 51) \| rsvd_bits(12, 29);	4105	rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd \| rsvd_bits(12, 20);
4106	rsvd_check->rsvd_bits_mask[1][1] =
4107	rsvd_bits(maxphyaddr, 51) \| rsvd_bits(12, 20);
4108	rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];	4106	rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];
4109		4107
4110	bad_mt_xwr = 0xFFull << (2 * 8); /* bits 3..5 must not be 2 */	4108	bad_mt_xwr = 0xFFull << (2 * 8); /* bits 3..5 must not be 2 */
@@ -4123,7 +4121,12 @@ static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu, · 12 unmodified lines
4123	struct kvm_mmu *context, bool execonly)	4121	struct kvm_mmu *context, bool execonly)
4124	{	4122	{
4125	__reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,	4123	__reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,
4126	cpuid_maxphyaddr(vcpu), execonly);	4124	vcpu->arch.reserved_gpa_bits, execonly);
		4125	}
		4126
		4127	static inline u64 reserved_hpa_bits(void)
		4128	{
		4129	return rsvd_bits(shadow_phys_bits, 63);
4127	}	4130	}
4128		4131
4129	/*	4132	/*
@@ -4145,7 +4148,7 @@ reset_shadow_zero_bits_mask(struct kvm_vcpu vcpu, struct kvm_mmu context) · 15 unmodified lines
4145	*/	4148	*/
4146	shadow_zero_check = &context->shadow_zero_check;	4149	shadow_zero_check = &context->shadow_zero_check;
4147	__reset_rsvds_bits_mask(vcpu, shadow_zero_check,	4150	__reset_rsvds_bits_mask(vcpu, shadow_zero_check,
4148	shadow_phys_bits,	4151	reserved_hpa_bits(),
4149	context->shadow_root_level, uses_nx,	4152	context->shadow_root_level, uses_nx,
4150	guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES),	4153	guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES),
4151	is_pse(vcpu), true);	4154	is_pse(vcpu), true);
@@ -4182,14 +4185,13 @@ reset_tdp_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, · 30 unmodified lines
4182		4185
4183	if (boot_cpu_is_amd())	4186	if (boot_cpu_is_amd())
4184	__reset_rsvds_bits_mask(vcpu, shadow_zero_check,	4187	__reset_rsvds_bits_mask(vcpu, shadow_zero_check,
4185	shadow_phys_bits,	4188	reserved_hpa_bits(),
4186	context->shadow_root_level, false,	4189	context->shadow_root_level, false,
4187	boot_cpu_has(X86_FEATURE_GBPAGES),	4190	boot_cpu_has(X86_FEATURE_GBPAGES),
4188	true, true);	4191	true, true);
4189	else	4192	else
4190	__reset_rsvds_bits_mask_ept(shadow_zero_check,	4193	__reset_rsvds_bits_mask_ept(shadow_zero_check,
4191	shadow_phys_bits,	4194	reserved_hpa_bits(), false);
4192	false);
4193		4195
4194	if (!shadow_me_mask)	4196	if (!shadow_me_mask)
4195	return;	4197	return;
@@ -4209,7 +4211,7 @@ reset_ept_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, · 13 unmodified lines
4209	struct kvm_mmu *context, bool execonly)	4211	struct kvm_mmu *context, bool execonly)
4210	{	4212	{
4211	__reset_rsvds_bits_mask_ept(&context->shadow_zero_check,	4213	__reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
4212	shadow_phys_bits, execonly);	4214	reserved_hpa_bits(), execonly);
4213	}	4215	}
4214		4216
4215	#define BYTE_MASK(access) \	4217	#define BYTE_MASK(access) \
@@ -4395,7 +4397,6 @@ static void paging64_init_context_common(struct kvm_vcpu *vcpu, · 179 unmodified lines
4395	context->gva_to_gpa = paging64_gva_to_gpa;	4397	context->gva_to_gpa = paging64_gva_to_gpa;
4396	context->sync_page = paging64_sync_page;	4398	context->sync_page = paging64_sync_page;
4397	context->invlpg = paging64_invlpg;	4399	context->invlpg = paging64_invlpg;
4398	context->update_pte = paging64_update_pte;
4399	context->shadow_root_level = level;	4400	context->shadow_root_level = level;
4400	context->direct_map = false;	4401	context->direct_map = false;
4401	}	4402	}
@@ -4424,7 +4425,6 @@ static void paging32_init_context(struct kvm_vcpu *vcpu, · 22 unmodified lines
4424	context->gva_to_gpa = paging32_gva_to_gpa;	4425	context->gva_to_gpa = paging32_gva_to_gpa;
4425	context->sync_page = paging32_sync_page;	4426	context->sync_page = paging32_sync_page;
4426	context->invlpg = paging32_invlpg;	4427	context->invlpg = paging32_invlpg;
4427	context->update_pte = paging32_update_pte;
4428	context->shadow_root_level = PT32E_ROOT_LEVEL;	4428	context->shadow_root_level = PT32E_ROOT_LEVEL;
4429	context->direct_map = false;	4429	context->direct_map = false;
4430	}	4430	}
@@ -4506,7 +4506,6 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) · 75 unmodified lines
4506	context->page_fault = kvm_tdp_page_fault;	4506	context->page_fault = kvm_tdp_page_fault;
4507	context->sync_page = nonpaging_sync_page;	4507	context->sync_page = nonpaging_sync_page;
4508	context->invlpg = NULL;	4508	context->invlpg = NULL;
4509	context->update_pte = nonpaging_update_pte;
4510	context->shadow_root_level = kvm_mmu_get_tdp_level(vcpu);	4509	context->shadow_root_level = kvm_mmu_get_tdp_level(vcpu);
4511	context->direct_map = true;	4510	context->direct_map = true;
4512	context->get_guest_pgd = get_cr3;	4511	context->get_guest_pgd = get_cr3;
@@ -4678,7 +4677,6 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, · 165 unmodified lines
4678	context->gva_to_gpa = ept_gva_to_gpa;	4677	context->gva_to_gpa = ept_gva_to_gpa;
4679	context->sync_page = ept_sync_page;	4678	context->sync_page = ept_sync_page;
4680	context->invlpg = ept_invlpg;	4679	context->invlpg = ept_invlpg;
4681	context->update_pte = ept_update_pte;
4682	context->root_level = level;	4680	context->root_level = level;
4683	context->direct_map = false;	4681	context->direct_map = false;
4684	context->mmu_role.as_u64 = new_role.as_u64;	4682	context->mmu_role.as_u64 = new_role.as_u64;
@@ -4811,7 +4809,7 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu) · 126 unmodified lines
4811	if (r)	4809	if (r)
4812	goto out;	4810	goto out;
4813	kvm_mmu_load_pgd(vcpu);	4811	kvm_mmu_load_pgd(vcpu);
4814	kvm_x86_ops.tlb_flush_current(vcpu);	4812	static_call(kvm_x86_tlb_flush_current)(vcpu);
4815	out:	4813	out:
4816	return r;	4814	return r;
4817	}	4815	}
@@ -4826,19 +4824,6 @@ void kvm_mmu_unload(struct kvm_vcpu *vcpu) · 8 unmodified lines
4826	}	4824	}
4827	EXPORT_SYMBOL_GPL(kvm_mmu_unload);	4825	EXPORT_SYMBOL_GPL(kvm_mmu_unload);
4828		4826
4829	static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
4830	struct kvm_mmu_page sp, u64 spte,
4831	const void *new)
4832	{
4833	if (sp->role.level != PG_LEVEL_4K) {
4834	++vcpu->kvm->stat.mmu_pde_zapped;
4835	return;
4836	}
4837
4838	++vcpu->kvm->stat.mmu_pte_updated;
4839	vcpu->arch.mmu->update_pte(vcpu, sp, spte, new);
4840	}
4841
4842	static bool need_remote_flush(u64 old, u64 new)	4827	static bool need_remote_flush(u64 old, u64 new)
4843	{	4828	{
4844	if (!is_shadow_present_pte(old))	4829	if (!is_shadow_present_pte(old))
@@ -4954,22 +4939,6 @@ static u64 get_written_sptes(struct kvm_mmu_page sp, gpa_t gpa, int *nspte) · 109 unmodified lines
4954	return spte;	4939	return spte;
4955	}	4940	}
4956		4941
4957	/*
4958	* Ignore various flags when determining if a SPTE can be immediately
4959	* overwritten for the current MMU.
4960	* - level: explicitly checked in mmu_pte_write_new_pte(), and will never
4961	* match the current MMU role, as MMU's level tracks the root level.
4962	* - access: updated based on the new guest PTE
4963	* - quadrant: handled by get_written_sptes()
4964	* - invalid: always false (loop only walks valid shadow pages)
4965	*/
4966	static const union kvm_mmu_page_role role_ign = {
4967	.level = 0xf,
4968	.access = 0x7,
4969	.quadrant = 0x3,
4970	.invalid = 0x1,
4971	};
4972
4973	static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,	4942	static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
4974	const u8 *new, int bytes,	4943	const u8 *new, int bytes,
4975	struct kvm_page_track_notifier_node *node)	4944	struct kvm_page_track_notifier_node *node)
@@ -4999,7 +4968,7 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, · 23 unmodified lines
4999	*/	4968	*/
5000	mmu_topup_memory_caches(vcpu, true);	4969	mmu_topup_memory_caches(vcpu, true);
5001		4970
5002	spin_lock(&vcpu->kvm->mmu_lock);	4971	write_lock(&vcpu->kvm->mmu_lock);
5003		4972
5004	gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, &bytes);	4973	gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, &bytes);
5005		4974
@@ -5020,14 +4989,10 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, · 14 unmodified lines
5020		4989
5021	local_flush = true;	4990	local_flush = true;
5022	while (npte--) {	4991	while (npte--) {
5023	u32 base_role = vcpu->arch.mmu->mmu_role.base.word;
5024
5025	entry = *spte;	4992	entry = *spte;
5026	mmu_page_zap_pte(vcpu->kvm, sp, spte, NULL);	4993	mmu_page_zap_pte(vcpu->kvm, sp, spte, NULL);
5027	if (gentry &&	4994	if (gentry && sp->role.level != PG_LEVEL_4K)
5028	!((sp->role.word ^ base_role) & ~role_ign.word) &&	4995	++vcpu->kvm->stat.mmu_pde_zapped;
5029	rmap_can_add(vcpu))
5030	mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
5031	if (need_remote_flush(entry, *spte))	4996	if (need_remote_flush(entry, *spte))
5032	remote_flush = true;	4997	remote_flush = true;
5033	++spte;	4998	++spte;
@@ -5035,7 +5000,7 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, · 1 unmodified line
5035	}	5000	}
5036	kvm_mmu_flush_or_zap(vcpu, &invalid_list, remote_flush, local_flush);	5001	kvm_mmu_flush_or_zap(vcpu, &invalid_list, remote_flush, local_flush);
5037	kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);	5002	kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
5038	spin_unlock(&vcpu->kvm->mmu_lock);	5003	write_unlock(&vcpu->kvm->mmu_lock);
5039	}	5004	}
5040		5005
5041	int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)	5006	int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
@@ -5125,7 +5090,7 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu vcpu, struct kvm_mmu mmu, · 83 unmodified lines
5125	if (is_noncanonical_address(gva, vcpu))	5090	if (is_noncanonical_address(gva, vcpu))
5126	return;	5091	return;
5127		5092
5128	kvm_x86_ops.tlb_flush_gva(vcpu, gva);	5093	static_call(kvm_x86_tlb_flush_gva)(vcpu, gva);
5129	}	5094	}
5130		5095
5131	if (!mmu->invlpg)	5096	if (!mmu->invlpg)
@@ -5182,7 +5147,7 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) · 50 unmodified lines
5182	}	5147	}
5183		5148
5184	if (tlb_flush)	5149	if (tlb_flush)
5185	kvm_x86_ops.tlb_flush_gva(vcpu, gva);	5150	static_call(kvm_x86_tlb_flush_gva)(vcpu, gva);
5186		5151
5187	++vcpu->stat.invlpg;	5152	++vcpu->stat.invlpg;
5188		5153
@@ -5233,14 +5198,14 @@ slot_handle_level_range(struct kvm kvm, struct kvm_memory_slot memslot, · 44 unmodified lines
5233	if (iterator.rmap)	5198	if (iterator.rmap)
5234	flush \|= fn(kvm, iterator.rmap);	5199	flush \|= fn(kvm, iterator.rmap);
5235		5200
5236	if (need_resched() \|\| spin_needbreak(&kvm->mmu_lock)) {	5201	if (need_resched() \|\| rwlock_needbreak(&kvm->mmu_lock)) {
5237	if (flush && lock_flush_tlb) {	5202	if (flush && lock_flush_tlb) {
5238	kvm_flush_remote_tlbs_with_address(kvm,	5203	kvm_flush_remote_tlbs_with_address(kvm,
5239	start_gfn,	5204	start_gfn,
5240	iterator.gfn - start_gfn + 1);	5205	iterator.gfn - start_gfn + 1);
5241	flush = false;	5206	flush = false;
5242	}	5207	}
5243	cond_resched_lock(&kvm->mmu_lock);	5208	cond_resched_rwlock_write(&kvm->mmu_lock);
5244	}	5209	}
5245	}	5210	}
5246		5211
@@ -5390,7 +5355,7 @@ restart: · 143 unmodified lines
5390	* be in active use by the guest.	5355	* be in active use by the guest.
5391	*/	5356	*/
5392	if (batch >= BATCH_ZAP_PAGES &&	5357	if (batch >= BATCH_ZAP_PAGES &&
5393	cond_resched_lock(&kvm->mmu_lock)) {	5358	cond_resched_rwlock_write(&kvm->mmu_lock)) {
5394	batch = 0;	5359	batch = 0;
5395	goto restart;	5360	goto restart;
5396	}	5361	}
@@ -5423,7 +5388,7 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm) · 26 unmodified lines
5423	{	5388	{
5424	lockdep_assert_held(&kvm->slots_lock);	5389	lockdep_assert_held(&kvm->slots_lock);
5425		5390
5426	spin_lock(&kvm->mmu_lock);	5391	write_lock(&kvm->mmu_lock);
5427	trace_kvm_mmu_zap_all_fast(kvm);	5392	trace_kvm_mmu_zap_all_fast(kvm);
5428		5393
5429	/*	5394	/*
@@ -5447,10 +5412,10 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm) · 17 unmodified lines
5447		5412
5448	kvm_zap_obsolete_pages(kvm);	5413	kvm_zap_obsolete_pages(kvm);
5449		5414
5450	if (kvm->arch.tdp_mmu_enabled)	5415	if (is_tdp_mmu_enabled(kvm))
5451	kvm_tdp_mmu_zap_all(kvm);	5416	kvm_tdp_mmu_zap_all(kvm);
5452		5417
5453	spin_unlock(&kvm->mmu_lock);	5418	write_unlock(&kvm->mmu_lock);
5454	}	5419	}
5455		5420
5456	static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm)	5421	static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm)
@@ -5492,7 +5457,7 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) · 35 unmodified lines
5492	int i;	5457	int i;
5493	bool flush;	5458	bool flush;
5494		5459
5495	spin_lock(&kvm->mmu_lock);	5460	write_lock(&kvm->mmu_lock);
5496	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {	5461	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
5497	slots = __kvm_memslots(kvm, i);	5462	slots = __kvm_memslots(kvm, i);
5498	kvm_for_each_memslot(memslot, slots) {	5463	kvm_for_each_memslot(memslot, slots) {
@@ -5510,13 +5475,13 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) · 11 unmodified lines
5510	}	5475	}
5511	}	5476	}
5512		5477
5513	if (kvm->arch.tdp_mmu_enabled) {	5478	if (is_tdp_mmu_enabled(kvm)) {
5514	flush = kvm_tdp_mmu_zap_gfn_range(kvm, gfn_start, gfn_end);	5479	flush = kvm_tdp_mmu_zap_gfn_range(kvm, gfn_start, gfn_end);
5515	if (flush)	5480	if (flush)
5516	kvm_flush_remote_tlbs(kvm);	5481	kvm_flush_remote_tlbs(kvm);
5517	}	5482	}
5518		5483
5519	spin_unlock(&kvm->mmu_lock);	5484	write_unlock(&kvm->mmu_lock);
5520	}	5485	}
5521		5486
5522	static bool slot_rmap_write_protect(struct kvm *kvm,	5487	static bool slot_rmap_write_protect(struct kvm *kvm,
@@ -5531,12 +5496,12 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, · 8 unmodified lines
5531	{	5496	{
5532	bool flush;	5497	bool flush;
5533		5498
5534	spin_lock(&kvm->mmu_lock);	5499	write_lock(&kvm->mmu_lock);
5535	flush = slot_handle_level(kvm, memslot, slot_rmap_write_protect,	5500	flush = slot_handle_level(kvm, memslot, slot_rmap_write_protect,
5536	start_level, KVM_MAX_HUGEPAGE_LEVEL, false);	5501	start_level, KVM_MAX_HUGEPAGE_LEVEL, false);
5537	if (kvm->arch.tdp_mmu_enabled)	5502	if (is_tdp_mmu_enabled(kvm))
5538	flush \|= kvm_tdp_mmu_wrprot_slot(kvm, memslot, PG_LEVEL_4K);	5503	flush \|= kvm_tdp_mmu_wrprot_slot(kvm, memslot, PG_LEVEL_4K);
5539	spin_unlock(&kvm->mmu_lock);	5504	write_unlock(&kvm->mmu_lock);
5540		5505
5541	/*	5506	/*
5542	* We can flush all the TLBs out of the mmu lock without TLB	5507	* We can flush all the TLBs out of the mmu lock without TLB
@@ -5596,13 +5561,13 @@ void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm, · 53 unmodified lines
5596	const struct kvm_memory_slot *memslot)	5561	const struct kvm_memory_slot *memslot)
5597	{	5562	{
5598	/* FIXME: const-ify all uses of struct kvm_memory_slot. */	5563	/* FIXME: const-ify all uses of struct kvm_memory_slot. */
5599	spin_lock(&kvm->mmu_lock);	5564	write_lock(&kvm->mmu_lock);
5600	slot_handle_leaf(kvm, (struct kvm_memory_slot *)memslot,	5565	slot_handle_leaf(kvm, (struct kvm_memory_slot *)memslot,
5601	kvm_mmu_zap_collapsible_spte, true);	5566	kvm_mmu_zap_collapsible_spte, true);
5602		5567
5603	if (kvm->arch.tdp_mmu_enabled)	5568	if (is_tdp_mmu_enabled(kvm))
5604	kvm_tdp_mmu_zap_collapsible_sptes(kvm, memslot);	5569	kvm_tdp_mmu_zap_collapsible_sptes(kvm, memslot);
5605	spin_unlock(&kvm->mmu_lock);	5570	write_unlock(&kvm->mmu_lock);
5606	}	5571	}
5607		5572
5608	void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,	5573	void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,
@@ -5625,11 +5590,11 @@ void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, · 16 unmodified lines
5625	{	5590	{
5626	bool flush;	5591	bool flush;
5627		5592
5628	spin_lock(&kvm->mmu_lock);	5593	write_lock(&kvm->mmu_lock);
5629	flush = slot_handle_leaf(kvm, memslot, __rmap_clear_dirty, false);	5594	flush = slot_handle_leaf(kvm, memslot, __rmap_clear_dirty, false);
5630	if (kvm->arch.tdp_mmu_enabled)	5595	if (is_tdp_mmu_enabled(kvm))
5631	flush \|= kvm_tdp_mmu_clear_dirty_slot(kvm, memslot);	5596	flush \|= kvm_tdp_mmu_clear_dirty_slot(kvm, memslot);
5632	spin_unlock(&kvm->mmu_lock);	5597	write_unlock(&kvm->mmu_lock);
5633		5598
5634	/*	5599	/*
5635	* It's also safe to flush TLBs out of mmu lock here as currently this	5600	* It's also safe to flush TLBs out of mmu lock here as currently this
@@ -5647,12 +5612,12 @@ void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm, · 11 unmodified lines
5647	{	5612	{
5648	bool flush;	5613	bool flush;
5649		5614
5650	spin_lock(&kvm->mmu_lock);	5615	write_lock(&kvm->mmu_lock);
5651	flush = slot_handle_large_level(kvm, memslot, slot_rmap_write_protect,	5616	flush = slot_handle_large_level(kvm, memslot, slot_rmap_write_protect,
5652	false);	5617	false);
5653	if (kvm->arch.tdp_mmu_enabled)	5618	if (is_tdp_mmu_enabled(kvm))
5654	flush \|= kvm_tdp_mmu_wrprot_slot(kvm, memslot, PG_LEVEL_2M);	5619	flush \|= kvm_tdp_mmu_wrprot_slot(kvm, memslot, PG_LEVEL_2M);
5655	spin_unlock(&kvm->mmu_lock);	5620	write_unlock(&kvm->mmu_lock);
5656		5621
5657	if (flush)	5622	if (flush)
5658	kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);	5623	kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);
@@ -5664,11 +5629,11 @@ void kvm_mmu_slot_set_dirty(struct kvm *kvm, · 5 unmodified lines
5664	{	5629	{
5665	bool flush;	5630	bool flush;
5666		5631
5667	spin_lock(&kvm->mmu_lock);	5632	write_lock(&kvm->mmu_lock);
5668	flush = slot_handle_all_level(kvm, memslot, __rmap_set_dirty, false);	5633	flush = slot_handle_all_level(kvm, memslot, __rmap_set_dirty, false);
5669	if (kvm->arch.tdp_mmu_enabled)	5634	if (is_tdp_mmu_enabled(kvm))
5670	flush \|= kvm_tdp_mmu_slot_set_dirty(kvm, memslot);	5635	flush \|= kvm_tdp_mmu_slot_set_dirty(kvm, memslot);
5671	spin_unlock(&kvm->mmu_lock);	5636	write_unlock(&kvm->mmu_lock);
5672		5637
5673	if (flush)	5638	if (flush)
5674	kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);	5639	kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);
@@ -5681,23 +5646,23 @@ void kvm_mmu_zap_all(struct kvm *kvm) · 6 unmodified lines
5681	LIST_HEAD(invalid_list);	5646	LIST_HEAD(invalid_list);
5682	int ign;	5647	int ign;
5683		5648
5684	spin_lock(&kvm->mmu_lock);	5649	write_lock(&kvm->mmu_lock);
5685	restart:	5650	restart:
5686	list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) {	5651	list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) {
5687	if (WARN_ON(sp->role.invalid))	5652	if (WARN_ON(sp->role.invalid))
5688	continue;	5653	continue;
5689	if (__kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list, &ign))	5654	if (__kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list, &ign))
5690	goto restart;	5655	goto restart;
5691	if (cond_resched_lock(&kvm->mmu_lock))	5656	if (cond_resched_rwlock_write(&kvm->mmu_lock))
5692	goto restart;	5657	goto restart;
5693	}	5658	}
5694		5659
5695	kvm_mmu_commit_zap_page(kvm, &invalid_list);	5660	kvm_mmu_commit_zap_page(kvm, &invalid_list);
5696		5661
5697	if (kvm->arch.tdp_mmu_enabled)	5662	if (is_tdp_mmu_enabled(kvm))
5698	kvm_tdp_mmu_zap_all(kvm);	5663	kvm_tdp_mmu_zap_all(kvm);
5699		5664
5700	spin_unlock(&kvm->mmu_lock);	5665	write_unlock(&kvm->mmu_lock);
5701	}	5666	}
5702		5667
5703	void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen)	5668	void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen)
@@ -5757,7 +5722,7 @@ mmu_shrink_scan(struct shrinker shrink, struct shrink_control sc) · 53 unmodified lines
5757	continue;	5722	continue;
5758		5723
5759	idx = srcu_read_lock(&kvm->srcu);	5724	idx = srcu_read_lock(&kvm->srcu);
5760	spin_lock(&kvm->mmu_lock);	5725	write_lock(&kvm->mmu_lock);
5761		5726
5762	if (kvm_has_zapped_obsolete_pages(kvm)) {	5727	if (kvm_has_zapped_obsolete_pages(kvm)) {
5763	kvm_mmu_commit_zap_page(kvm,	5728	kvm_mmu_commit_zap_page(kvm,
@@ -5768,7 +5733,7 @@ mmu_shrink_scan(struct shrinker shrink, struct shrink_control sc) · 4 unmodified lines
5768	freed = kvm_mmu_zap_oldest_mmu_pages(kvm, sc->nr_to_scan);	5733	freed = kvm_mmu_zap_oldest_mmu_pages(kvm, sc->nr_to_scan);
5769		5734
5770	unlock:	5735	unlock:
5771	spin_unlock(&kvm->mmu_lock);	5736	write_unlock(&kvm->mmu_lock);
5772	srcu_read_unlock(&kvm->srcu, idx);	5737	srcu_read_unlock(&kvm->srcu, idx);
5773		5738
5774	/*	5739	/*
@@ -5988,7 +5953,7 @@ static void kvm_recover_nx_lpages(struct kvm *kvm) · 213 unmodified lines
5988	ulong to_zap;	5953	ulong to_zap;
5989		5954
5990	rcu_idx = srcu_read_lock(&kvm->srcu);	5955	rcu_idx = srcu_read_lock(&kvm->srcu);
5991	spin_lock(&kvm->mmu_lock);	5956	write_lock(&kvm->mmu_lock);
5992		5957
5993	ratio = READ_ONCE(nx_huge_pages_recovery_ratio);	5958	ratio = READ_ONCE(nx_huge_pages_recovery_ratio);
5994	to_zap = ratio ? DIV_ROUND_UP(kvm->stat.nx_lpage_splits, ratio) : 0;	5959	to_zap = ratio ? DIV_ROUND_UP(kvm->stat.nx_lpage_splits, ratio) : 0;
@@ -6005,22 +5970,22 @@ static void kvm_recover_nx_lpages(struct kvm *kvm) · 10 unmodified lines
6005	struct kvm_mmu_page,	5970	struct kvm_mmu_page,
6006	lpage_disallowed_link);	5971	lpage_disallowed_link);
6007	WARN_ON_ONCE(!sp->lpage_disallowed);	5972	WARN_ON_ONCE(!sp->lpage_disallowed);
6008	if (sp->tdp_mmu_page)	5973	if (is_tdp_mmu_page(sp)) {
6009	kvm_tdp_mmu_zap_gfn_range(kvm, sp->gfn,	5974	kvm_tdp_mmu_zap_gfn_range(kvm, sp->gfn,
6010	sp->gfn + KVM_PAGES_PER_HPAGE(sp->role.level));	5975	sp->gfn + KVM_PAGES_PER_HPAGE(sp->role.level));
6011	else {	5976	} else {
6012	kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);	5977	kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
6013	WARN_ON_ONCE(sp->lpage_disallowed);	5978	WARN_ON_ONCE(sp->lpage_disallowed);
6014	}	5979	}
6015		5980
6016	if (need_resched() \|\| spin_needbreak(&kvm->mmu_lock)) {	5981	if (need_resched() \|\| rwlock_needbreak(&kvm->mmu_lock)) {
6017	kvm_mmu_commit_zap_page(kvm, &invalid_list);	5982	kvm_mmu_commit_zap_page(kvm, &invalid_list);
6018	cond_resched_lock(&kvm->mmu_lock);	5983	cond_resched_rwlock_write(&kvm->mmu_lock);
6019	}	5984	}
6020	}	5985	}
6021	kvm_mmu_commit_zap_page(kvm, &invalid_list);	5986	kvm_mmu_commit_zap_page(kvm, &invalid_list);
6022		5987
6023	spin_unlock(&kvm->mmu_lock);	5988	write_unlock(&kvm->mmu_lock);
6024	srcu_read_unlock(&kvm->srcu, rcu_idx);	5989	srcu_read_unlock(&kvm->srcu, rcu_idx);
6025	}	5990	}
6026		5991

arch/x86/kvm/mmu/mmu_audit.c+4−4

@@ -234,7 +234,7 @@ static void audit_vcpu_spte(struct kvm_vcpu *vcpu) · 233 unmodified lines
234	}	234	}
235		235
236	static bool mmu_audit;	236	static bool mmu_audit;
237	static struct static_key mmu_audit_key;	237	static DEFINE_STATIC_KEY_FALSE(mmu_audit_key);
238		238
239	static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)	239	static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
240	{	240	{
@@ -250,7 +250,7 @@ static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) · 9 unmodified lines
250		250
251	static inline void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)	251	static inline void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
252	{	252	{
253	if (static_key_false((&mmu_audit_key)))	253	if (static_branch_unlikely((&mmu_audit_key)))
254	__kvm_mmu_audit(vcpu, point);	254	__kvm_mmu_audit(vcpu, point);
255	}	255	}
256		256
@@ -259,7 +259,7 @@ static void mmu_audit_enable(void) · 2 unmodified lines
259	if (mmu_audit)	259	if (mmu_audit)
260	return;	260	return;
261		261
262	static_key_slow_inc(&mmu_audit_key);	262	static_branch_inc(&mmu_audit_key);
263	mmu_audit = true;	263	mmu_audit = true;
264	}	264	}
265		265
@@ -268,7 +268,7 @@ static void mmu_audit_disable(void) · 2 unmodified lines
268	if (!mmu_audit)	268	if (!mmu_audit)
269	return;	269	return;
270		270
271	static_key_slow_dec(&mmu_audit_key);	271	static_branch_dec(&mmu_audit_key);
272	mmu_audit = false;	272	mmu_audit = false;
273	}	273	}
274		274

arch/x86/kvm/mmu/mmu_internal.h+6−1

 extern bool dbg;
 #define pgprintk(x...) do { if (dbg) printk(x); } while (0)
-#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
+#define rmap_printk(fmt, args...) do { if (dbg) printk("%s: " fmt, __func__, ## args); } while (0)
 #define MMU_WARN_ON(x) WARN_ON(x)
 #else
 #define pgprintk(x...) do { } while (0)
 	/* Number of writes since the last time traversal visited this page.  */
 	atomic_t write_flooding_count;
+#ifdef CONFIG_X86_64
 	bool tdp_mmu_page;
+	/* Used for freeing the page asyncronously if it is a TDP MMU page. */
+	struct rcu_head rcu_head;
+#endif
 };
 extern struct kmem_cache *mmu_page_header_cache;

arch/x86/kvm/mmu/page_track.c+4−4

@@ -184,9 +184,9 @@ kvm_page_track_register_notifier(struct kvm *kvm, · 183 unmodified lines
184		184
185	head = &kvm->arch.track_notifier_head;	185	head = &kvm->arch.track_notifier_head;
186		186
187	spin_lock(&kvm->mmu_lock);	187	write_lock(&kvm->mmu_lock);
188	hlist_add_head_rcu(&n->node, &head->track_notifier_list);	188	hlist_add_head_rcu(&n->node, &head->track_notifier_list);
189	spin_unlock(&kvm->mmu_lock);	189	write_unlock(&kvm->mmu_lock);
190	}	190	}
191	EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier);	191	EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier);
192		192
@@ -202,9 +202,9 @@ kvm_page_track_unregister_notifier(struct kvm *kvm, · 9 unmodified lines
202		202
203	head = &kvm->arch.track_notifier_head;	203	head = &kvm->arch.track_notifier_head;
204		204
205	spin_lock(&kvm->mmu_lock);	205	write_lock(&kvm->mmu_lock);
206	hlist_del_rcu(&n->node);	206	hlist_del_rcu(&n->node);
207	spin_unlock(&kvm->mmu_lock);	207	write_unlock(&kvm->mmu_lock);
208	synchronize_srcu(&head->track_srcu);	208	synchronize_srcu(&head->track_srcu);
209	}	209	}
210	EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier);	210	EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier);

arch/x86/kvm/mmu/paging_tmpl.h+4−4

@@ -868,7 +868,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, · 867 unmodified lines
868	}	868	}
869		869
870	r = RET_PF_RETRY;	870	r = RET_PF_RETRY;
871	spin_lock(&vcpu->kvm->mmu_lock);	871	write_lock(&vcpu->kvm->mmu_lock);
872	if (mmu_notifier_retry(vcpu->kvm, mmu_seq))	872	if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
873	goto out_unlock;	873	goto out_unlock;
874		874
@@ -881,7 +881,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, · 6 unmodified lines
881	kvm_mmu_audit(vcpu, AUDIT_POST_PAGE_FAULT);	881	kvm_mmu_audit(vcpu, AUDIT_POST_PAGE_FAULT);
882		882
883	out_unlock:	883	out_unlock:
884	spin_unlock(&vcpu->kvm->mmu_lock);	884	write_unlock(&vcpu->kvm->mmu_lock);
885	kvm_release_pfn_clean(pfn);	885	kvm_release_pfn_clean(pfn);
886	return r;	886	return r;
887	}	887	}
@@ -919,7 +919,7 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa) · 31 unmodified lines
919	return;	919	return;
920	}	920	}
921		921
922	spin_lock(&vcpu->kvm->mmu_lock);	922	write_lock(&vcpu->kvm->mmu_lock);
923	for_each_shadow_entry_using_root(vcpu, root_hpa, gva, iterator) {	923	for_each_shadow_entry_using_root(vcpu, root_hpa, gva, iterator) {
924	level = iterator.level;	924	level = iterator.level;
925	sptep = iterator.sptep;	925	sptep = iterator.sptep;
@@ -954,7 +954,7 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa) · 28 unmodified lines
954	if (!is_shadow_present_pte(*sptep) \|\| !sp->unsync_children)	954	if (!is_shadow_present_pte(*sptep) \|\| !sp->unsync_children)
955	break;	955	break;
956	}	956	}
957	spin_unlock(&vcpu->kvm->mmu_lock);	957	write_unlock(&vcpu->kvm->mmu_lock);
958	}	958	}
959		959
960	/* Note, @addr is a GPA when gva_to_gpa() translates an L2 GPA to an L1 GPA. */	960	/* Note, @addr is a GPA when gva_to_gpa() translates an L2 GPA to an L1 GPA. */

arch/x86/kvm/mmu/spte.c+1−1

@@ -120,7 +120,7 @@ int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level, · 119 unmodified lines
120	if (level > PG_LEVEL_4K)	120	if (level > PG_LEVEL_4K)
121	spte \|= PT_PAGE_SIZE_MASK;	121	spte \|= PT_PAGE_SIZE_MASK;
122	if (tdp_enabled)	122	if (tdp_enabled)
123	spte \|= kvm_x86_ops.get_mt_mask(vcpu, gfn,	123	spte \|= static_call(kvm_x86_get_mt_mask)(vcpu, gfn,
124	kvm_is_mmio_pfn(pfn));	124	kvm_is_mmio_pfn(pfn));
125		125
126	if (host_writable)	126	if (host_writable)

arch/x86/kvm/mmu/spte.h+24−9

 					  PT64_EPT_EXECUTABLE_MASK)
 #define SHADOW_ACC_TRACK_SAVED_BITS_SHIFT PT64_SECOND_AVAIL_BITS_SHIFT
+/*
+ * If a thread running without exclusive control of the MMU lock must perform a
+ * multi-part operation on an SPTE, it can set the SPTE to REMOVED_SPTE as a
+ * non-present intermediate value. Other threads which encounter this value
+ * should not modify the SPTE.
+ *
+ * This constant works because it is considered non-present on both AMD and
+ * Intel CPUs and does not create a L1TF vulnerability because the pfn section
+ * is zeroed out.
+ *
+ * Only used by the TDP MMU.
+ */
+#define REMOVED_SPTE (1ull << 59)
+static inline bool is_removed_spte(u64 spte)
+{
+	return spte == REMOVED_SPTE;
+}
 /*
  * In some cases, we need to preserve the GFN of a non-present or reserved
  * SPTE when we usurp the upper five bits of the physical address space to
 	return !spte_ad_enabled(spte) && (spte & shadow_acc_track_mask) == 0;
 }
-static inline int is_shadow_present_pte(u64 pte)
+static inline bool is_shadow_present_pte(u64 pte)
 {
-	return (pte != 0) && !is_mmio_spte(pte);
+	return (pte != 0) && !is_mmio_spte(pte) && !is_removed_spte(pte);
 }
-static inline int is_large_pte(u64 pte)
+static inline bool is_large_pte(u64 pte)
 {
 	return pte & PT_PAGE_SIZE_MASK;
 }
-static inline int is_last_spte(u64 pte, int level)
+static inline bool is_last_spte(u64 pte, int level)
 {
-	if (level == PG_LEVEL_4K)
+	return (level == PG_LEVEL_4K) || is_large_pte(pte);
-		return 1;
-	if (is_large_pte(pte))
-		return 1;
-	return 0;
 }
 static inline bool is_executable_pte(u64 spte)

arch/x86/kvm/mmu/tdp_iter.c+15−31

 {
 	iter->sptep = iter->pt_path[iter->level - 1] +
 		SHADOW_PT_INDEX(iter->gfn << PAGE_SHIFT, iter->level);
-	iter->old_spte = READ_ONCE(*iter->sptep);
+	iter->old_spte = READ_ONCE(*rcu_dereference(iter->sptep));
 }
 static gfn_t round_gfn_for_level(gfn_t gfn, int level)
 /*
  * Sets a TDP iterator to walk a pre-order traversal of the paging structure
- * rooted at root_pt, starting with the walk to translate goal_gfn.
+ * rooted at root_pt, starting with the walk to translate next_last_level_gfn.
  */
 void tdp_iter_start(struct tdp_iter *iter, u64 *root_pt, int root_level,
-		    int min_level, gfn_t goal_gfn)
+		    int min_level, gfn_t next_last_level_gfn)
 {
 	WARN_ON(root_level < 1);
 	WARN_ON(root_level > PT64_ROOT_MAX_LEVEL);
-	iter->goal_gfn = goal_gfn;
+	iter->next_last_level_gfn = next_last_level_gfn;
+	iter->yielded_gfn = iter->next_last_level_gfn;
 	iter->root_level = root_level;
 	iter->min_level = min_level;
 	iter->level = root_level;
-	iter->pt_path[iter->level - 1] = root_pt;
+	iter->pt_path[iter->level - 1] = (tdp_ptep_t)root_pt;
-	iter->gfn = round_gfn_for_level(iter->goal_gfn, iter->level);
+	iter->gfn = round_gfn_for_level(iter->next_last_level_gfn, iter->level);
 	tdp_iter_refresh_sptep(iter);
 	iter->valid = true;
  * address of the child page table referenced by the SPTE. Returns null if
  * there is no such entry.
  */
-u64 *spte_to_child_pt(u64 spte, int level)
+tdp_ptep_t spte_to_child_pt(u64 spte, int level)
 {
 	/*
 	 * There's no child entry if this entry isn't present or is a
 	if (!is_shadow_present_pte(spte) || is_last_spte(spte, level))
 		return NULL;
-	return __va(spte_to_pfn(spte) << PAGE_SHIFT);
+	return (tdp_ptep_t)__va(spte_to_pfn(spte) << PAGE_SHIFT);
 }
 /*
  */
 static bool try_step_down(struct tdp_iter *iter)
 {
-	u64 *child_pt;
+	tdp_ptep_t child_pt;
 	if (iter->level == iter->min_level)
 		return false;
 	 * Reread the SPTE before stepping down to avoid traversing into page
 	 * tables that are no longer linked from this entry.
 	 */
-	iter->old_spte = READ_ONCE(*iter->sptep);
+	iter->old_spte = READ_ONCE(*rcu_dereference(iter->sptep));
 	child_pt = spte_to_child_pt(iter->old_spte, iter->level);
 	if (!child_pt)
 	iter->level--;
 	iter->pt_path[iter->level - 1] = child_pt;
-	iter->gfn = round_gfn_for_level(iter->goal_gfn, iter->level);
+	iter->gfn = round_gfn_for_level(iter->next_last_level_gfn, iter->level);
 	tdp_iter_refresh_sptep(iter);
 	return true;
 		return false;
 	iter->gfn += KVM_PAGES_PER_HPAGE(iter->level);
-	iter->goal_gfn = iter->gfn;
+	iter->next_last_level_gfn = iter->gfn;
 	iter->sptep++;
-	iter->old_spte = READ_ONCE(*iter->sptep);
+	iter->old_spte = READ_ONCE(*rcu_dereference(iter->sptep));
 	return true;
 }
 	iter->valid = false;
 }
-/*
+tdp_ptep_t tdp_iter_root_pt(struct tdp_iter *iter)
- * Restart the walk over the paging structure from the root, starting from the
- * highest gfn the iterator had previously reached. Assumes that the entire
- * paging structure, except the root page, may have been completely torn down
- * and rebuilt.
- */
-void tdp_iter_refresh_walk(struct tdp_iter *iter)
-{
-	gfn_t goal_gfn = iter->goal_gfn;
-	if (iter->gfn > goal_gfn)
-		goal_gfn = iter->gfn;
-	tdp_iter_start(iter, iter->pt_path[iter->root_level - 1],
-		       iter->root_level, iter->min_level, goal_gfn);
-}
-u64 *tdp_iter_root_pt(struct tdp_iter *iter)
 {
 	return iter->pt_path[iter->root_level - 1];
 }

arch/x86/kvm/mmu/tdp_iter.h+14−7

 #include "mmu.h"
+typedef u64 __rcu *tdp_ptep_t;
 /*
  * A TDP iterator performs a pre-order walk over a TDP paging structure.
  */
 	 * The iterator will traverse the paging structure towards the mapping
 	 * for this GFN.
 	 */
-	gfn_t goal_gfn;
+	gfn_t next_last_level_gfn;
+	/*
+	 * The next_last_level_gfn at the time when the thread last
+	 * yielded. Only yielding when the next_last_level_gfn !=
+	 * yielded_gfn helps ensure forward progress.
+	 */
+	gfn_t yielded_gfn;
 	/* Pointers to the page tables traversed to reach the current SPTE */
-	u64 *pt_path[PT64_ROOT_MAX_LEVEL];
+	tdp_ptep_t pt_path[PT64_ROOT_MAX_LEVEL];
 	/* A pointer to the current SPTE */
-	u64 *sptep;
+	tdp_ptep_t sptep;
 	/* The lowest GFN mapped by the current SPTE */
 	gfn_t gfn;
 	/* The level of the root page given to the iterator */
 #define for_each_tdp_pte(iter, root, root_level, start, end) \
 	for_each_tdp_pte_min_level(iter, root, root_level, PG_LEVEL_4K, start, end)
-u64 *spte_to_child_pt(u64 pte, int level);
+tdp_ptep_t spte_to_child_pt(u64 pte, int level);
 void tdp_iter_start(struct tdp_iter *iter, u64 *root_pt, int root_level,
-		    int min_level, gfn_t goal_gfn);
+		    int min_level, gfn_t next_last_level_gfn);
 void tdp_iter_next(struct tdp_iter *iter);
-void tdp_iter_refresh_walk(struct tdp_iter *iter);
+tdp_ptep_t tdp_iter_root_pt(struct tdp_iter *iter);
-u64 *tdp_iter_root_pt(struct tdp_iter *iter);
 #endif /* __KVM_X86_MMU_TDP_ITER_H */

arch/x86/kvm/mmu/tdp_mmu.c+422−132

@@ -7,32 +7,23 @@ · 6 unmodified lines
7	#include "tdp_mmu.h"	7	#include "tdp_mmu.h"
8	#include "spte.h"	8	#include "spte.h"
9		9
		10	#include <asm/cmpxchg.h>
10	#include <trace/events/kvm.h>	11	#include <trace/events/kvm.h>
11		12
12	#ifdef CONFIG_X86_64
13	static bool __read_mostly tdp_mmu_enabled = false;	13	static bool __read_mostly tdp_mmu_enabled = false;
14	module_param_named(tdp_mmu, tdp_mmu_enabled, bool, 0644);	14	module_param_named(tdp_mmu, tdp_mmu_enabled, bool, 0644);
15	#endif
16
17	static bool is_tdp_mmu_enabled(void)
18	{
19	#ifdef CONFIG_X86_64
20	return tdp_enabled && READ_ONCE(tdp_mmu_enabled);
21	#else
22	return false;
23	#endif /* CONFIG_X86_64 */
24	}
25		15
26	/* Initializes the TDP MMU for the VM, if enabled. */	16	/* Initializes the TDP MMU for the VM, if enabled. */
27	void kvm_mmu_init_tdp_mmu(struct kvm *kvm)	17	void kvm_mmu_init_tdp_mmu(struct kvm *kvm)
28	{	18	{
29	if (!is_tdp_mmu_enabled())	19	if (!tdp_enabled \|\| !READ_ONCE(tdp_mmu_enabled))
30	return;	20	return;
31		21
32	/* This should not be changed for the lifetime of the VM. */	22	/* This should not be changed for the lifetime of the VM. */
33	kvm->arch.tdp_mmu_enabled = true;	23	kvm->arch.tdp_mmu_enabled = true;
34		24
35	INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots);	25	INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots);
		26	spin_lock_init(&kvm->arch.tdp_mmu_pages_lock);
36	INIT_LIST_HEAD(&kvm->arch.tdp_mmu_pages);	27	INIT_LIST_HEAD(&kvm->arch.tdp_mmu_pages);
37	}	28	}
38		29
@@ -42,6 +33,12 @@ void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) · 3 unmodified lines
42	return;	33	return;
43		34
44	WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots));	35	WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots));
		36
		37	/*
		38	* Ensure that all the outstanding RCU callbacks to free shadow pages
		39	* can run before the VM is torn down.
		40	*/
		41	rcu_barrier();
45	}	42	}
46		43
47	static void tdp_mmu_put_root(struct kvm kvm, struct kvm_mmu_page root)	44	static void tdp_mmu_put_root(struct kvm kvm, struct kvm_mmu_page root)
@@ -53,7 +50,7 @@ static void tdp_mmu_put_root(struct kvm kvm, struct kvm_mmu_page root) · 5 unmodified lines
53	static inline bool tdp_mmu_next_root_valid(struct kvm *kvm,	50	static inline bool tdp_mmu_next_root_valid(struct kvm *kvm,
54	struct kvm_mmu_page *root)	51	struct kvm_mmu_page *root)
55	{	52	{
56	lockdep_assert_held(&kvm->mmu_lock);	53	lockdep_assert_held_write(&kvm->mmu_lock);
57		54
58	if (list_entry_is_head(root, &kvm->arch.tdp_mmu_roots, link))	55	if (list_entry_is_head(root, &kvm->arch.tdp_mmu_roots, link))
59	return false;	56	return false;
@@ -88,22 +85,6 @@ static inline struct kvm_mmu_page tdp_mmu_next_root(struct kvm kvm, · 28 unmodified lines
88	#define for_each_tdp_mmu_root(_kvm, _root) \	85	#define for_each_tdp_mmu_root(_kvm, _root) \
89	list_for_each_entry(_root, &_kvm->arch.tdp_mmu_roots, link)	86	list_for_each_entry(_root, &_kvm->arch.tdp_mmu_roots, link)
90		87
91	bool is_tdp_mmu_root(struct kvm *kvm, hpa_t hpa)
92	{
93	struct kvm_mmu_page *sp;
94
95	if (!kvm->arch.tdp_mmu_enabled)
96	return false;
97	if (WARN_ON(!VALID_PAGE(hpa)))
98	return false;
99
100	sp = to_shadow_page(hpa);
101	if (WARN_ON(!sp))
102	return false;
103
104	return sp->tdp_mmu_page && sp->root_count;
105	}
106
107	static bool zap_gfn_range(struct kvm kvm, struct kvm_mmu_page root,	88	static bool zap_gfn_range(struct kvm kvm, struct kvm_mmu_page root,
108	gfn_t start, gfn_t end, bool can_yield);	89	gfn_t start, gfn_t end, bool can_yield);
109		90
@@ -111,7 +92,7 @@ void kvm_tdp_mmu_free_root(struct kvm kvm, struct kvm_mmu_page root) · 1 unmodified line
111	{	92	{
112	gfn_t max_gfn = 1ULL << (shadow_phys_bits - PAGE_SHIFT);	93	gfn_t max_gfn = 1ULL << (shadow_phys_bits - PAGE_SHIFT);
113		94
114	lockdep_assert_held(&kvm->mmu_lock);	95	lockdep_assert_held_write(&kvm->mmu_lock);
115		96
116	WARN_ON(root->root_count);	97	WARN_ON(root->root_count);
117	WARN_ON(!root->tdp_mmu_page);	98	WARN_ON(!root->tdp_mmu_page);
@@ -164,13 +145,13 @@ static struct kvm_mmu_page get_tdp_mmu_vcpu_root(struct kvm_vcpu vcpu) · 46 unmodified lines
164		145
165	role = page_role_for_level(vcpu, vcpu->arch.mmu->shadow_root_level);	146	role = page_role_for_level(vcpu, vcpu->arch.mmu->shadow_root_level);
166		147
167	spin_lock(&kvm->mmu_lock);	148	write_lock(&kvm->mmu_lock);
168		149
169	/* Check for an existing root before allocating a new one. */	150	/* Check for an existing root before allocating a new one. */
170	for_each_tdp_mmu_root(kvm, root) {	151	for_each_tdp_mmu_root(kvm, root) {
171	if (root->role.word == role.word) {	152	if (root->role.word == role.word) {
172	kvm_mmu_get_root(kvm, root);	153	kvm_mmu_get_root(kvm, root);
173	spin_unlock(&kvm->mmu_lock);	154	write_unlock(&kvm->mmu_lock);
174	return root;	155	return root;
175	}	156	}
176	}	157	}
@@ -180,7 +161,7 @@ static struct kvm_mmu_page get_tdp_mmu_vcpu_root(struct kvm_vcpu vcpu) · 3 unmodified lines
180		161
181	list_add(&root->link, &kvm->arch.tdp_mmu_roots);	162	list_add(&root->link, &kvm->arch.tdp_mmu_roots);
182		163
183	spin_unlock(&kvm->mmu_lock);	164	write_unlock(&kvm->mmu_lock);
184		165
185	return root;	166	return root;
186	}	167	}
@@ -196,8 +177,31 @@ hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu) · 9 unmodified lines
196	return __pa(root->spt);	177	return __pa(root->spt);
197	}	178	}
198		179
		180	static void tdp_mmu_free_sp(struct kvm_mmu_page *sp)
		181	{
		182	free_page((unsigned long)sp->spt);
		183	kmem_cache_free(mmu_page_header_cache, sp);
		184	}
		185
		186	/*
		187	* This is called through call_rcu in order to free TDP page table memory
		188	* safely with respect to other kernel threads that may be operating on
		189	* the memory.
		190	* By only accessing TDP MMU page table memory in an RCU read critical
		191	* section, and freeing it after a grace period, lockless access to that
		192	* memory won't use it after it is freed.
		193	*/
		194	static void tdp_mmu_free_sp_rcu_callback(struct rcu_head *head)
		195	{
		196	struct kvm_mmu_page *sp = container_of(head, struct kvm_mmu_page,
		197	rcu_head);
		198
		199	tdp_mmu_free_sp(sp);
		200	}
		201
199	static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,	202	static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,
200	u64 old_spte, u64 new_spte, int level);	203	u64 old_spte, u64 new_spte, int level,
		204	bool shared);
201		205
202	static int kvm_mmu_page_as_id(struct kvm_mmu_page *sp)	206	static int kvm_mmu_page_as_id(struct kvm_mmu_page *sp)
203	{	207	{
@@ -234,6 +238,128 @@ static void handle_changed_spte_dirty_log(struct kvm *kvm, int as_id, gfn_t gfn, · 30 unmodified lines
234	}	238	}
235	}	239	}
236		240
		241	/**
		242	* tdp_mmu_link_page - Add a new page to the list of pages used by the TDP MMU
		243	*
		244	* @kvm: kvm instance
		245	* @sp: the new page
		246	* @shared: This operation may not be running under the exclusive use of
		247	* the MMU lock and the operation must synchronize with other
		248	* threads that might be adding or removing pages.
		249	* @account_nx: This page replaces a NX large page and should be marked for
		250	* eventual reclaim.
		251	*/
		252	static void tdp_mmu_link_page(struct kvm kvm, struct kvm_mmu_page sp,
		253	bool shared, bool account_nx)
		254	{
		255	if (shared)
		256	spin_lock(&kvm->arch.tdp_mmu_pages_lock);
		257	else
		258	lockdep_assert_held_write(&kvm->mmu_lock);
		259
		260	list_add(&sp->link, &kvm->arch.tdp_mmu_pages);
		261	if (account_nx)
		262	account_huge_nx_page(kvm, sp);
		263
		264	if (shared)
		265	spin_unlock(&kvm->arch.tdp_mmu_pages_lock);
		266	}
		267
		268	/**
		269	* tdp_mmu_unlink_page - Remove page from the list of pages used by the TDP MMU
		270	*
		271	* @kvm: kvm instance
		272	* @sp: the page to be removed
		273	* @shared: This operation may not be running under the exclusive use of
		274	* the MMU lock and the operation must synchronize with other
		275	* threads that might be adding or removing pages.
		276	*/
		277	static void tdp_mmu_unlink_page(struct kvm kvm, struct kvm_mmu_page sp,
		278	bool shared)
		279	{
		280	if (shared)
		281	spin_lock(&kvm->arch.tdp_mmu_pages_lock);
		282	else
		283	lockdep_assert_held_write(&kvm->mmu_lock);
		284
		285	list_del(&sp->link);
		286	if (sp->lpage_disallowed)
		287	unaccount_huge_nx_page(kvm, sp);
		288
		289	if (shared)
		290	spin_unlock(&kvm->arch.tdp_mmu_pages_lock);
		291	}
		292
		293	/**
		294	* handle_removed_tdp_mmu_page - handle a pt removed from the TDP structure
		295	*
		296	* @kvm: kvm instance
		297	* @pt: the page removed from the paging structure
		298	* @shared: This operation may not be running under the exclusive use
		299	* of the MMU lock and the operation must synchronize with other
		300	* threads that might be modifying SPTEs.
		301	*
		302	* Given a page table that has been removed from the TDP paging structure,
		303	* iterates through the page table to clear SPTEs and free child page tables.
		304	*/
		305	static void handle_removed_tdp_mmu_page(struct kvm kvm, u64 pt,
		306	bool shared)
		307	{
		308	struct kvm_mmu_page *sp = sptep_to_sp(pt);
		309	int level = sp->role.level;
		310	gfn_t base_gfn = sp->gfn;
		311	u64 old_child_spte;
		312	u64 *sptep;
		313	gfn_t gfn;
		314	int i;
		315
		316	trace_kvm_mmu_prepare_zap_page(sp);
		317
		318	tdp_mmu_unlink_page(kvm, sp, shared);
		319
		320	for (i = 0; i < PT64_ENT_PER_PAGE; i++) {
		321	sptep = pt + i;
		322	gfn = base_gfn + (i * KVM_PAGES_PER_HPAGE(level - 1));
		323
		324	if (shared) {
		325	/*
		326	* Set the SPTE to a nonpresent value that other
		327	* threads will not overwrite. If the SPTE was
		328	* already marked as removed then another thread
		329	* handling a page fault could overwrite it, so
		330	* set the SPTE until it is set from some other
		331	* value to the removed SPTE value.
		332	*/
		333	for (;;) {
		334	old_child_spte = xchg(sptep, REMOVED_SPTE);
		335	if (!is_removed_spte(old_child_spte))
		336	break;
		337	cpu_relax();
		338	}
		339	} else {
		340	old_child_spte = READ_ONCE(*sptep);
		341
		342	/*
		343	* Marking the SPTE as a removed SPTE is not
		344	* strictly necessary here as the MMU lock will
		345	* stop other threads from concurrently modifying
		346	* this SPTE. Using the removed SPTE value keeps
		347	* the two branches consistent and simplifies
		348	* the function.
		349	*/
		350	WRITE_ONCE(*sptep, REMOVED_SPTE);
		351	}
		352	handle_changed_spte(kvm, kvm_mmu_page_as_id(sp), gfn,
		353	old_child_spte, REMOVED_SPTE, level - 1,
		354	shared);
		355	}
		356
		357	kvm_flush_remote_tlbs_with_address(kvm, gfn,
		358	KVM_PAGES_PER_HPAGE(level));
		359
		360	call_rcu(&sp->rcu_head, tdp_mmu_free_sp_rcu_callback);
		361	}
		362
237	/**	363	/**
238	* handle_changed_spte - handle bookkeeping associated with an SPTE change	364	* handle_changed_spte - handle bookkeeping associated with an SPTE change
239	* @kvm: kvm instance	365	* @kvm: kvm instance
@@ -242,22 +368,22 @@ static void handle_changed_spte_dirty_log(struct kvm *kvm, int as_id, gfn_t gfn, · 2 unmodified lines
242	* @old_spte: The value of the SPTE before the change	368	* @old_spte: The value of the SPTE before the change
243	* @new_spte: The value of the SPTE after the change	369	* @new_spte: The value of the SPTE after the change
244	* @level: the level of the PT the SPTE is part of in the paging structure	370	* @level: the level of the PT the SPTE is part of in the paging structure
		371	* @shared: This operation may not be running under the exclusive use of
		372	* the MMU lock and the operation must synchronize with other
		373	* threads that might be modifying SPTEs.
245	*	374	*
246	* Handle bookkeeping that might result from the modification of a SPTE.	375	* Handle bookkeeping that might result from the modification of a SPTE.
247	* This function must be called for all TDP SPTE modifications.	376	* This function must be called for all TDP SPTE modifications.
248	*/	377	*/
249	static void __handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,	378	static void __handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,
250	u64 old_spte, u64 new_spte, int level)	379	u64 old_spte, u64 new_spte, int level,
		380	bool shared)
251	{	381	{
252	bool was_present = is_shadow_present_pte(old_spte);	382	bool was_present = is_shadow_present_pte(old_spte);
253	bool is_present = is_shadow_present_pte(new_spte);	383	bool is_present = is_shadow_present_pte(new_spte);
254	bool was_leaf = was_present && is_last_spte(old_spte, level);	384	bool was_leaf = was_present && is_last_spte(old_spte, level);
255	bool is_leaf = is_present && is_last_spte(new_spte, level);	385	bool is_leaf = is_present && is_last_spte(new_spte, level);
256	bool pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte);	386	bool pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte);
257	u64 *pt;
258	struct kvm_mmu_page *sp;
259	u64 old_child_spte;
260	int i;
261		387
262	WARN_ON(level > PT64_ROOT_MAX_LEVEL);	388	WARN_ON(level > PT64_ROOT_MAX_LEVEL);
263	WARN_ON(level < PG_LEVEL_4K);	389	WARN_ON(level < PG_LEVEL_4K);
@@ -298,15 +424,19 @@ static void __handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, · 34 unmodified lines
298	*/	424	*/
299	if (!was_present && !is_present) {	425	if (!was_present && !is_present) {
300	/*	426	/*
301	* If this change does not involve a MMIO SPTE, it is	427	* If this change does not involve a MMIO SPTE or removed SPTE,
302	* unexpected. Log the change, though it should not impact the	428	* it is unexpected. Log the change, though it should not
303	* guest since both the former and current SPTEs are nonpresent.	429	* impact the guest since both the former and current SPTEs
		430	* are nonpresent.
304	*/	431	*/
305	if (WARN_ON(!is_mmio_spte(old_spte) && !is_mmio_spte(new_spte)))	432	if (WARN_ON(!is_mmio_spte(old_spte) &&
		433	!is_mmio_spte(new_spte) &&
		434	!is_removed_spte(new_spte)))
306	pr_err("Unexpected SPTE change! Nonpresent SPTEs\n"	435	pr_err("Unexpected SPTE change! Nonpresent SPTEs\n"
307	"should not be replaced with another,\n"	436	"should not be replaced with another,\n"
308	"different nonpresent SPTE, unless one or both\n"	437	"different nonpresent SPTE, unless one or both\n"
309	"are MMIO SPTEs.\n"	438	"are MMIO SPTEs, or the new SPTE is\n"
		439	"a temporary removed SPTE.\n"
310	"as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d",	440	"as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d",
311	as_id, gfn, old_spte, new_spte, level);	441	as_id, gfn, old_spte, new_spte, level);
312	return;	442	return;
@@ -321,54 +451,127 @@ static void __handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, · 8 unmodified lines
321	* Recursively handle child PTs if the change removed a subtree from	451	* Recursively handle child PTs if the change removed a subtree from
322	* the paging structure.	452	* the paging structure.
323	*/	453	*/
324	if (was_present && !was_leaf && (pfn_changed \|\| !is_present)) {	454	if (was_present && !was_leaf && (pfn_changed \|\| !is_present))
325	pt = spte_to_child_pt(old_spte, level);	455	handle_removed_tdp_mmu_page(kvm,
326	sp = sptep_to_sp(pt);	456	spte_to_child_pt(old_spte, level), shared);
		457	}
327		458
328	trace_kvm_mmu_prepare_zap_page(sp);	459	static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,
		460	u64 old_spte, u64 new_spte, int level,
		461	bool shared)
		462	{
		463	__handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level,
		464	shared);
		465	handle_changed_spte_acc_track(old_spte, new_spte, level);
		466	handle_changed_spte_dirty_log(kvm, as_id, gfn, old_spte,
		467	new_spte, level);
		468	}
329		469
330	list_del(&sp->link);	470	/*
		471	* tdp_mmu_set_spte_atomic - Set a TDP MMU SPTE atomically and handle the
		472	* associated bookkeeping
		473	*
		474	* @kvm: kvm instance
		475	* @iter: a tdp_iter instance currently on the SPTE that should be set
		476	* @new_spte: The value the SPTE should be set to
		477	* Returns: true if the SPTE was set, false if it was not. If false is returned,
		478	* this function will have no side-effects.
		479	*/
		480	static inline bool tdp_mmu_set_spte_atomic(struct kvm *kvm,
		481	struct tdp_iter *iter,
		482	u64 new_spte)
		483	{
		484	u64 *root_pt = tdp_iter_root_pt(iter);
		485	struct kvm_mmu_page *root = sptep_to_sp(root_pt);
		486	int as_id = kvm_mmu_page_as_id(root);
331		487
332	if (sp->lpage_disallowed)	488	lockdep_assert_held_read(&kvm->mmu_lock);
333	unaccount_huge_nx_page(kvm, sp);
334		489
335	for (i = 0; i < PT64_ENT_PER_PAGE; i++) {	490	/*
336	old_child_spte = READ_ONCE(*(pt + i));	491	* Do not change removed SPTEs. Only the thread that froze the SPTE
337	WRITE_ONCE(*(pt + i), 0);	492	* may modify it.
338	handle_changed_spte(kvm, as_id,	493	*/
339	gfn + (i * KVM_PAGES_PER_HPAGE(level - 1)),	494	if (iter->old_spte == REMOVED_SPTE)
340	old_child_spte, 0, level - 1);	495	return false;
341	}
342		496
343	kvm_flush_remote_tlbs_with_address(kvm, gfn,	497	if (cmpxchg64(rcu_dereference(iter->sptep), iter->old_spte,
344	KVM_PAGES_PER_HPAGE(level));	498	new_spte) != iter->old_spte)
		499	return false;
345		500
346	free_page((unsigned long)pt);	501	handle_changed_spte(kvm, as_id, iter->gfn, iter->old_spte, new_spte,
347	kmem_cache_free(mmu_page_header_cache, sp);	502	iter->level, true);
348	}	503
		504	return true;
349	}	505	}
350		506
351	static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,	507	static inline bool tdp_mmu_zap_spte_atomic(struct kvm *kvm,
352	u64 old_spte, u64 new_spte, int level)	508	struct tdp_iter *iter)
353	{	509	{
354	__handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level);	510	/*
355	handle_changed_spte_acc_track(old_spte, new_spte, level);	511	* Freeze the SPTE by setting it to a special,
356	handle_changed_spte_dirty_log(kvm, as_id, gfn, old_spte,	512	* non-present value. This will stop other threads from
357	new_spte, level);	513	* immediately installing a present entry in its place
		514	* before the TLBs are flushed.
		515	*/
		516	if (!tdp_mmu_set_spte_atomic(kvm, iter, REMOVED_SPTE))
		517	return false;
		518
		519	kvm_flush_remote_tlbs_with_address(kvm, iter->gfn,
		520	KVM_PAGES_PER_HPAGE(iter->level));
		521
		522	/*
		523	* No other thread can overwrite the removed SPTE as they
		524	* must either wait on the MMU lock or use
		525	* tdp_mmu_set_spte_atomic which will not overrite the
		526	* special removed SPTE value. No bookkeeping is needed
		527	* here since the SPTE is going from non-present
		528	* to non-present.
		529	*/
		530	WRITE_ONCE(*iter->sptep, 0);
		531
		532	return true;
358	}	533	}
359		534
		535
		536	/*
		537	* __tdp_mmu_set_spte - Set a TDP MMU SPTE and handle the associated bookkeeping
		538	* @kvm: kvm instance
		539	* @iter: a tdp_iter instance currently on the SPTE that should be set
		540	* @new_spte: The value the SPTE should be set to
		541	* @record_acc_track: Notify the MM subsystem of changes to the accessed state
		542	* of the page. Should be set unless handling an MMU
		543	* notifier for access tracking. Leaving record_acc_track
		544	* unset in that case prevents page accesses from being
		545	* double counted.
		546	* @record_dirty_log: Record the page as dirty in the dirty bitmap if
		547	* appropriate for the change being made. Should be set
		548	* unless performing certain dirty logging operations.
		549	* Leaving record_dirty_log unset in that case prevents page
		550	* writes from being double counted.
		551	*/
360	static inline void __tdp_mmu_set_spte(struct kvm kvm, struct tdp_iter iter,	552	static inline void __tdp_mmu_set_spte(struct kvm kvm, struct tdp_iter iter,
361	u64 new_spte, bool record_acc_track,	553	u64 new_spte, bool record_acc_track,
362	bool record_dirty_log)	554	bool record_dirty_log)
363	{	555	{
364	u64 *root_pt = tdp_iter_root_pt(iter);	556	tdp_ptep_t root_pt = tdp_iter_root_pt(iter);
365	struct kvm_mmu_page *root = sptep_to_sp(root_pt);	557	struct kvm_mmu_page *root = sptep_to_sp(root_pt);
366	int as_id = kvm_mmu_page_as_id(root);	558	int as_id = kvm_mmu_page_as_id(root);
367		559
368	WRITE_ONCE(*iter->sptep, new_spte);	560	lockdep_assert_held_write(&kvm->mmu_lock);
		561
		562	/*
		563	* No thread should be using this function to set SPTEs to the
		564	* temporary removed SPTE value.
		565	* If operating under the MMU lock in read mode, tdp_mmu_set_spte_atomic
		566	* should be used. If operating under the MMU lock in write mode, the
		567	* use of the removed SPTE should not be necessary.
		568	*/
		569	WARN_ON(iter->old_spte == REMOVED_SPTE);
		570
		571	WRITE_ONCE(*rcu_dereference(iter->sptep), new_spte);
369		572
370	__handle_changed_spte(kvm, as_id, iter->gfn, iter->old_spte, new_spte,	573	__handle_changed_spte(kvm, as_id, iter->gfn, iter->old_spte, new_spte,
371	iter->level);	574	iter->level, false);
372	if (record_acc_track)	575	if (record_acc_track)
373	handle_changed_spte_acc_track(iter->old_spte, new_spte,	576	handle_changed_spte_acc_track(iter->old_spte, new_spte,
374	iter->level);	577	iter->level);
@@ -413,27 +616,46 @@ static inline void tdp_mmu_set_spte_no_dirty_log(struct kvm *kvm, · 38 unmodified lines
413	_mmu->shadow_root_level, _start, _end)	616	_mmu->shadow_root_level, _start, _end)
414		617
415	/*	618	/*
416	* Flush the TLB if the process should drop kvm->mmu_lock.	619	* Yield if the MMU lock is contended or this thread needs to return control
417	* Return whether the caller still needs to flush the tlb.	620	* to the scheduler.
		621	*
		622	* If this function should yield and flush is set, it will perform a remote
		623	* TLB flush before yielding.
		624	*
		625	* If this function yields, it will also reset the tdp_iter's walk over the
		626	* paging structure and the calling function should skip to the next
		627	* iteration to allow the iterator to continue its traversal from the
		628	* paging structure root.
		629	*
		630	* Return true if this function yielded and the iterator's traversal was reset.
		631	* Return false if a yield was not needed.
418	*/	632	*/
419	static bool tdp_mmu_iter_flush_cond_resched(struct kvm kvm, struct tdp_iter iter)	633	static inline bool tdp_mmu_iter_cond_resched(struct kvm *kvm,
		634	struct tdp_iter *iter, bool flush)
420	{	635	{
421	if (need_resched() \|\| spin_needbreak(&kvm->mmu_lock)) {	636	/* Ensure forward progress has been made before yielding. */
422	kvm_flush_remote_tlbs(kvm);	637	if (iter->next_last_level_gfn == iter->yielded_gfn)
423	cond_resched_lock(&kvm->mmu_lock);
424	tdp_iter_refresh_walk(iter);
425	return false;	638	return false;
426	} else {	639
		640	if (need_resched() \|\| rwlock_needbreak(&kvm->mmu_lock)) {
		641	rcu_read_unlock();
		642
		643	if (flush)
		644	kvm_flush_remote_tlbs(kvm);
		645
		646	cond_resched_rwlock_write(&kvm->mmu_lock);
		647	rcu_read_lock();
		648
		649	WARN_ON(iter->gfn > iter->next_last_level_gfn);
		650
		651	tdp_iter_start(iter, iter->pt_path[iter->root_level - 1],
		652	iter->root_level, iter->min_level,
		653	iter->next_last_level_gfn);
		654
427	return true;	655	return true;
428	}	656	}
429	}
430		657
431	static void tdp_mmu_iter_cond_resched(struct kvm kvm, struct tdp_iter iter)	658	return false;
432	{
433	if (need_resched() \|\| spin_needbreak(&kvm->mmu_lock)) {
434	cond_resched_lock(&kvm->mmu_lock);
435	tdp_iter_refresh_walk(iter);
436	}
437	}	659	}
438		660
439	/*	661	/*
@@ -453,7 +675,15 @@ static bool zap_gfn_range(struct kvm kvm, struct kvm_mmu_page root, · 13 unmodified lines
453	struct tdp_iter iter;	675	struct tdp_iter iter;
454	bool flush_needed = false;	676	bool flush_needed = false;
455		677
		678	rcu_read_lock();
		679
456	tdp_root_for_each_pte(iter, root, start, end) {	680	tdp_root_for_each_pte(iter, root, start, end) {
		681	if (can_yield &&
		682	tdp_mmu_iter_cond_resched(kvm, &iter, flush_needed)) {
		683	flush_needed = false;
		684	continue;
		685	}
		686
457	if (!is_shadow_present_pte(iter.old_spte))	687	if (!is_shadow_present_pte(iter.old_spte))
458	continue;	688	continue;
459		689
@@ -468,12 +698,10 @@ static bool zap_gfn_range(struct kvm kvm, struct kvm_mmu_page root, · 8 unmodified lines
468	continue;	698	continue;
469		699
470	tdp_mmu_set_spte(kvm, &iter, 0);	700	tdp_mmu_set_spte(kvm, &iter, 0);
471		701	flush_needed = true;
472	if (can_yield)
473	flush_needed = tdp_mmu_iter_flush_cond_resched(kvm, &iter);
474	else
475	flush_needed = true;
476	}	702	}
		703
		704	rcu_read_unlock();
477	return flush_needed;	705	return flush_needed;
478	}	706	}
479		707
@@ -517,21 +745,18 @@ static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu, int write, · 37 unmodified lines
517	int ret = 0;	745	int ret = 0;
518	int make_spte_ret = 0;	746	int make_spte_ret = 0;
519		747
520	if (unlikely(is_noslot_pfn(pfn))) {	748	if (unlikely(is_noslot_pfn(pfn)))
521	new_spte = make_mmio_spte(vcpu, iter->gfn, ACC_ALL);	749	new_spte = make_mmio_spte(vcpu, iter->gfn, ACC_ALL);
522	trace_mark_mmio_spte(iter->sptep, iter->gfn, new_spte);	750	else
523	} else {
524	make_spte_ret = make_spte(vcpu, ACC_ALL, iter->level, iter->gfn,	751	make_spte_ret = make_spte(vcpu, ACC_ALL, iter->level, iter->gfn,
525	pfn, iter->old_spte, prefault, true,	752	pfn, iter->old_spte, prefault, true,
526	map_writable, !shadow_accessed_mask,	753	map_writable, !shadow_accessed_mask,
527	&new_spte);	754	&new_spte);
528	trace_kvm_mmu_set_spte(iter->level, iter->gfn, iter->sptep);
529	}
530		755
531	if (new_spte == iter->old_spte)	756	if (new_spte == iter->old_spte)
532	ret = RET_PF_SPURIOUS;	757	ret = RET_PF_SPURIOUS;
533	else	758	else if (!tdp_mmu_set_spte_atomic(vcpu->kvm, iter, new_spte))
534	tdp_mmu_set_spte(vcpu->kvm, iter, new_spte);	759	return RET_PF_RETRY;
535		760
536	/*	761	/*
537	* If the page fault was caused by a write but the page is write	762	* If the page fault was caused by a write but the page is write
@@ -545,10 +770,16 @@ static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu, int write, · 7 unmodified lines
545	}	770	}
546		771
547	/* If a MMIO SPTE is installed, the MMIO will need to be emulated. */	772	/* If a MMIO SPTE is installed, the MMIO will need to be emulated. */
548	if (unlikely(is_mmio_spte(new_spte)))	773	if (unlikely(is_mmio_spte(new_spte))) {
		774	trace_mark_mmio_spte(rcu_dereference(iter->sptep), iter->gfn,
		775	new_spte);
549	ret = RET_PF_EMULATE;	776	ret = RET_PF_EMULATE;
		777	} else
		778	trace_kvm_mmu_set_spte(iter->level, iter->gfn,
		779	rcu_dereference(iter->sptep));
550		780
551	trace_kvm_mmu_set_spte(iter->level, iter->gfn, iter->sptep);	781	trace_kvm_mmu_set_spte(iter->level, iter->gfn,
		782	rcu_dereference(iter->sptep));
552	if (!prefault)	783	if (!prefault)
553	vcpu->stat.pf_fixed++;	784	vcpu->stat.pf_fixed++;
554		785
@@ -586,6 +817,9 @@ int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, · 31 unmodified lines
586	huge_page_disallowed, &req_level);	817	huge_page_disallowed, &req_level);
587		818
588	trace_kvm_mmu_spte_requested(gpa, level, pfn);	819	trace_kvm_mmu_spte_requested(gpa, level, pfn);
		820
		821	rcu_read_lock();
		822
589	tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) {	823	tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) {
590	if (nx_huge_page_workaround_enabled)	824	if (nx_huge_page_workaround_enabled)
591	disallowed_hugepage_adjust(iter.old_spte, gfn,	825	disallowed_hugepage_adjust(iter.old_spte, gfn,
@@ -601,49 +835,61 @@ int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, · 9 unmodified lines
601	*/	835	*/
602	if (is_shadow_present_pte(iter.old_spte) &&	836	if (is_shadow_present_pte(iter.old_spte) &&
603	is_large_pte(iter.old_spte)) {	837	is_large_pte(iter.old_spte)) {
604	tdp_mmu_set_spte(vcpu->kvm, &iter, 0);	838	if (!tdp_mmu_zap_spte_atomic(vcpu->kvm, &iter))
605		839	break;
606	kvm_flush_remote_tlbs_with_address(vcpu->kvm, iter.gfn,
607	KVM_PAGES_PER_HPAGE(iter.level));
608		840
609	/*	841	/*
610	* The iter must explicitly re-read the spte here	842	* The iter must explicitly re-read the spte here
611	* because the new value informs the !present	843	* because the new value informs the !present
612	* path below.	844	* path below.
613	*/	845	*/
614	iter.old_spte = READ_ONCE(*iter.sptep);	846	iter.old_spte = READ_ONCE(*rcu_dereference(iter.sptep));
615	}	847	}
616		848
617	if (!is_shadow_present_pte(iter.old_spte)) {	849	if (!is_shadow_present_pte(iter.old_spte)) {
618	sp = alloc_tdp_mmu_page(vcpu, iter.gfn, iter.level);	850	sp = alloc_tdp_mmu_page(vcpu, iter.gfn, iter.level);
619	list_add(&sp->link, &vcpu->kvm->arch.tdp_mmu_pages);
620	child_pt = sp->spt;	851	child_pt = sp->spt;
621	clear_page(child_pt);	852
622	new_spte = make_nonleaf_spte(child_pt,	853	new_spte = make_nonleaf_spte(child_pt,
623	!shadow_accessed_mask);	854	!shadow_accessed_mask);
624		855
625	trace_kvm_mmu_get_page(sp, true);	856	if (tdp_mmu_set_spte_atomic(vcpu->kvm, &iter,
626	if (huge_page_disallowed && req_level >= iter.level)	857	new_spte)) {
627	account_huge_nx_page(vcpu->kvm, sp);	858	tdp_mmu_link_page(vcpu->kvm, sp, true,
628		859	huge_page_disallowed &&
629	tdp_mmu_set_spte(vcpu->kvm, &iter, new_spte);	860	req_level >= iter.level);
		861
		862	trace_kvm_mmu_get_page(sp, true);
		863	} else {
		864	tdp_mmu_free_sp(sp);
		865	break;
		866	}
630	}	867	}
631	}	868	}
632		869
633	if (WARN_ON(iter.level != level))	870	if (iter.level != level) {
		871	rcu_read_unlock();
634	return RET_PF_RETRY;	872	return RET_PF_RETRY;
		873	}
635		874
636	ret = tdp_mmu_map_handle_target_level(vcpu, write, map_writable, &iter,	875	ret = tdp_mmu_map_handle_target_level(vcpu, write, map_writable, &iter,
637	pfn, prefault);	876	pfn, prefault);
		877	rcu_read_unlock();
638		878
639	return ret;	879	return ret;
640	}	880	}
641		881
642	static int kvm_tdp_mmu_handle_hva_range(struct kvm *kvm, unsigned long start,	882	static __always_inline int
643	unsigned long end, unsigned long data,	883	kvm_tdp_mmu_handle_hva_range(struct kvm *kvm,
644	int (handler)(struct kvm kvm, struct kvm_memory_slot *slot,	884	unsigned long start,
645	struct kvm_mmu_page *root, gfn_t start,	885	unsigned long end,
646	gfn_t end, unsigned long data))	886	unsigned long data,
		887	int (handler)(struct kvm kvm,
		888	struct kvm_memory_slot *slot,
		889	struct kvm_mmu_page *root,
		890	gfn_t start,
		891	gfn_t end,
		892	unsigned long data))
647	{	893	{
648	struct kvm_memslots *slots;	894	struct kvm_memslots *slots;
649	struct kvm_memory_slot *memslot;	895	struct kvm_memory_slot *memslot;
@@ -705,6 +951,8 @@ static int age_gfn_range(struct kvm kvm, struct kvm_memory_slot slot, · 55 unmodified lines
705	int young = 0;	951	int young = 0;
706	u64 new_spte = 0;	952	u64 new_spte = 0;
707		953
		954	rcu_read_lock();
		955
708	tdp_root_for_each_leaf_pte(iter, root, start, end) {	956	tdp_root_for_each_leaf_pte(iter, root, start, end) {
709	/*	957	/*
710	* If we have a non-accessed entry we don't need to change the	958	* If we have a non-accessed entry we don't need to change the
@@ -736,6 +984,8 @@ static int age_gfn_range(struct kvm kvm, struct kvm_memory_slot slot, · 25 unmodified lines
736	trace_kvm_age_page(iter.gfn, iter.level, slot, young);	984	trace_kvm_age_page(iter.gfn, iter.level, slot, young);
737	}	985	}
738		986
		987	rcu_read_unlock();
		988
739	return young;	989	return young;
740	}	990	}
741		991
@@ -781,6 +1031,8 @@ static int set_tdp_spte(struct kvm kvm, struct kvm_memory_slot slot, · 39 unmodified lines
781	u64 new_spte;	1031	u64 new_spte;
782	int need_flush = 0;	1032	int need_flush = 0;
783		1033
		1034	rcu_read_lock();
		1035
784	WARN_ON(pte_huge(*ptep));	1036	WARN_ON(pte_huge(*ptep));
785		1037
786	new_pfn = pte_pfn(*ptep);	1038	new_pfn = pte_pfn(*ptep);
@@ -809,6 +1061,8 @@ static int set_tdp_spte(struct kvm kvm, struct kvm_memory_slot slot, · 22 unmodified lines
809	if (need_flush)	1061	if (need_flush)
810	kvm_flush_remote_tlbs_with_address(kvm, gfn, 1);	1062	kvm_flush_remote_tlbs_with_address(kvm, gfn, 1);
811		1063
		1064	rcu_read_unlock();
		1065
812	return 0;	1066	return 0;
813	}	1067	}
814		1068
@@ -832,21 +1086,27 @@ static bool wrprot_gfn_range(struct kvm kvm, struct kvm_mmu_page root, · 17 unmodified lines
832	u64 new_spte;	1086	u64 new_spte;
833	bool spte_set = false;	1087	bool spte_set = false;
834		1088
		1089	rcu_read_lock();
		1090
835	BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL);	1091	BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL);
836		1092
837	for_each_tdp_pte_min_level(iter, root->spt, root->role.level,	1093	for_each_tdp_pte_min_level(iter, root->spt, root->role.level,
838	min_level, start, end) {	1094	min_level, start, end) {
		1095	if (tdp_mmu_iter_cond_resched(kvm, &iter, false))
		1096	continue;
		1097
839	if (!is_shadow_present_pte(iter.old_spte) \|\|	1098	if (!is_shadow_present_pte(iter.old_spte) \|\|
840	!is_last_spte(iter.old_spte, iter.level))	1099	!is_last_spte(iter.old_spte, iter.level) \|\|
		1100	!(iter.old_spte & PT_WRITABLE_MASK))
841	continue;	1101	continue;
842		1102
843	new_spte = iter.old_spte & ~PT_WRITABLE_MASK;	1103	new_spte = iter.old_spte & ~PT_WRITABLE_MASK;
844		1104
845	tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte);	1105	tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte);
846	spte_set = true;	1106	spte_set = true;
847
848	tdp_mmu_iter_cond_resched(kvm, &iter);
849	}	1107	}
		1108
		1109	rcu_read_unlock();
850	return spte_set;	1110	return spte_set;
851	}	1111	}
852		1112
@@ -888,7 +1148,12 @@ static bool clear_dirty_gfn_range(struct kvm kvm, struct kvm_mmu_page root, · 35 unmodified lines
888	u64 new_spte;	1148	u64 new_spte;
889	bool spte_set = false;	1149	bool spte_set = false;
890		1150
		1151	rcu_read_lock();
		1152
891	tdp_root_for_each_leaf_pte(iter, root, start, end) {	1153	tdp_root_for_each_leaf_pte(iter, root, start, end) {
		1154	if (tdp_mmu_iter_cond_resched(kvm, &iter, false))
		1155	continue;
		1156
892	if (spte_ad_need_write_protect(iter.old_spte)) {	1157	if (spte_ad_need_write_protect(iter.old_spte)) {
893	if (is_writable_pte(iter.old_spte))	1158	if (is_writable_pte(iter.old_spte))
894	new_spte = iter.old_spte & ~PT_WRITABLE_MASK;	1159	new_spte = iter.old_spte & ~PT_WRITABLE_MASK;
@@ -903,9 +1168,9 @@ static bool clear_dirty_gfn_range(struct kvm kvm, struct kvm_mmu_page root, · 8 unmodified lines
903		1168
904	tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte);	1169	tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte);
905	spte_set = true;	1170	spte_set = true;
906
907	tdp_mmu_iter_cond_resched(kvm, &iter);
908	}	1171	}
		1172
		1173	rcu_read_unlock();
909	return spte_set;	1174	return spte_set;
910	}	1175	}
911		1176
@@ -947,6 +1212,8 @@ static void clear_dirty_pt_masked(struct kvm kvm, struct kvm_mmu_page root, · 35 unmodified lines
947	struct tdp_iter iter;	1212	struct tdp_iter iter;
948	u64 new_spte;	1213	u64 new_spte;
949		1214
		1215	rcu_read_lock();
		1216
950	tdp_root_for_each_leaf_pte(iter, root, gfn + __ffs(mask),	1217	tdp_root_for_each_leaf_pte(iter, root, gfn + __ffs(mask),
951	gfn + BITS_PER_LONG) {	1218	gfn + BITS_PER_LONG) {
952	if (!mask)	1219	if (!mask)
@@ -956,6 +1223,8 @@ static void clear_dirty_pt_masked(struct kvm kvm, struct kvm_mmu_page root, · 3 unmodified lines
956	!(mask & (1UL << (iter.gfn - gfn))))	1223	!(mask & (1UL << (iter.gfn - gfn))))
957	continue;	1224	continue;
958		1225
		1226	mask &= ~(1UL << (iter.gfn - gfn));
		1227
959	if (wrprot \|\| spte_ad_need_write_protect(iter.old_spte)) {	1228	if (wrprot \|\| spte_ad_need_write_protect(iter.old_spte)) {
960	if (is_writable_pte(iter.old_spte))	1229	if (is_writable_pte(iter.old_spte))
961	new_spte = iter.old_spte & ~PT_WRITABLE_MASK;	1230	new_spte = iter.old_spte & ~PT_WRITABLE_MASK;
@@ -969,9 +1238,9 @@ static void clear_dirty_pt_masked(struct kvm kvm, struct kvm_mmu_page root, · 7 unmodified lines
969	}	1238	}
970		1239
971	tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte);	1240	tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte);
972
973	mask &= ~(1UL << (iter.gfn - gfn));
974	}	1241	}
		1242
		1243	rcu_read_unlock();
975	}	1244	}
976		1245
977	/*	1246	/*
@@ -989,7 +1258,7 @@ void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm, · 11 unmodified lines
989	struct kvm_mmu_page *root;	1258	struct kvm_mmu_page *root;
990	int root_as_id;	1259	int root_as_id;
991		1260
992	lockdep_assert_held(&kvm->mmu_lock);	1261	lockdep_assert_held_write(&kvm->mmu_lock);
993	for_each_tdp_mmu_root(kvm, root) {	1262	for_each_tdp_mmu_root(kvm, root) {
994	root_as_id = kvm_mmu_page_as_id(root);	1263	root_as_id = kvm_mmu_page_as_id(root);
995	if (root_as_id != slot->as_id)	1264	if (root_as_id != slot->as_id)
@@ -1011,18 +1280,23 @@ static bool set_dirty_gfn_range(struct kvm kvm, struct kvm_mmu_page root, · 15 unmodified lines
1011	u64 new_spte;	1280	u64 new_spte;
1012	bool spte_set = false;	1281	bool spte_set = false;
1013		1282
		1283	rcu_read_lock();
		1284
1014	tdp_root_for_each_pte(iter, root, start, end) {	1285	tdp_root_for_each_pte(iter, root, start, end) {
1015	if (!is_shadow_present_pte(iter.old_spte))	1286	if (tdp_mmu_iter_cond_resched(kvm, &iter, false))
		1287	continue;
		1288
		1289	if (!is_shadow_present_pte(iter.old_spte) \|\|
		1290	iter.old_spte & shadow_dirty_mask)
1016	continue;	1291	continue;
1017		1292
1018	new_spte = iter.old_spte \| shadow_dirty_mask;	1293	new_spte = iter.old_spte \| shadow_dirty_mask;
1019		1294
1020	tdp_mmu_set_spte(kvm, &iter, new_spte);	1295	tdp_mmu_set_spte(kvm, &iter, new_spte);
1021	spte_set = true;	1296	spte_set = true;
1022
1023	tdp_mmu_iter_cond_resched(kvm, &iter);
1024	}	1297	}
1025		1298
		1299	rcu_read_unlock();
1026	return spte_set;	1300	return spte_set;
1027	}	1301	}
1028		1302
@@ -1060,7 +1334,14 @@ static void zap_collapsible_spte_range(struct kvm *kvm, · 31 unmodified lines
1060	kvm_pfn_t pfn;	1334	kvm_pfn_t pfn;
1061	bool spte_set = false;	1335	bool spte_set = false;
1062		1336
		1337	rcu_read_lock();
		1338
1063	tdp_root_for_each_pte(iter, root, start, end) {	1339	tdp_root_for_each_pte(iter, root, start, end) {
		1340	if (tdp_mmu_iter_cond_resched(kvm, &iter, spte_set)) {
		1341	spte_set = false;
		1342	continue;
		1343	}
		1344
1064	if (!is_shadow_present_pte(iter.old_spte) \|\|	1345	if (!is_shadow_present_pte(iter.old_spte) \|\|
1065	!is_last_spte(iter.old_spte, iter.level))	1346	!is_last_spte(iter.old_spte, iter.level))
1066	continue;	1347	continue;
@@ -1072,9 +1353,10 @@ static void zap_collapsible_spte_range(struct kvm *kvm, · 5 unmodified lines
1072		1353
1073	tdp_mmu_set_spte(kvm, &iter, 0);	1354	tdp_mmu_set_spte(kvm, &iter, 0);
1074		1355
1075	spte_set = tdp_mmu_iter_flush_cond_resched(kvm, &iter);	1356	spte_set = true;
1076	}	1357	}
1077		1358
		1359	rcu_read_unlock();
1078	if (spte_set)	1360	if (spte_set)
1079	kvm_flush_remote_tlbs(kvm);	1361	kvm_flush_remote_tlbs(kvm);
1080	}	1362	}
@@ -1111,6 +1393,8 @@ static bool write_protect_gfn(struct kvm kvm, struct kvm_mmu_page root, · 30 unmodified lines
1111	u64 new_spte;	1393	u64 new_spte;
1112	bool spte_set = false;	1394	bool spte_set = false;
1113		1395
		1396	rcu_read_lock();
		1397
1114	tdp_root_for_each_leaf_pte(iter, root, gfn, gfn + 1) {	1398	tdp_root_for_each_leaf_pte(iter, root, gfn, gfn + 1) {
1115	if (!is_writable_pte(iter.old_spte))	1399	if (!is_writable_pte(iter.old_spte))
1116	break;	1400	break;
@@ -1122,6 +1406,8 @@ static bool write_protect_gfn(struct kvm kvm, struct kvm_mmu_page root, · 5 unmodified lines
1122	spte_set = true;	1406	spte_set = true;
1123	}	1407	}
1124		1408
		1409	rcu_read_unlock();
		1410
1125	return spte_set;	1411	return spte_set;
1126	}	1412	}
1127		1413
@@ -1137,7 +1423,7 @@ bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm, · 9 unmodified lines
1137	int root_as_id;	1423	int root_as_id;
1138	bool spte_set = false;	1424	bool spte_set = false;
1139		1425
1140	lockdep_assert_held(&kvm->mmu_lock);	1426	lockdep_assert_held_write(&kvm->mmu_lock);
1141	for_each_tdp_mmu_root(kvm, root) {	1427	for_each_tdp_mmu_root(kvm, root) {
1142	root_as_id = kvm_mmu_page_as_id(root);	1428	root_as_id = kvm_mmu_page_as_id(root);
1143	if (root_as_id != slot->as_id)	1429	if (root_as_id != slot->as_id)
@@ -1162,10 +1448,14 @@ int kvm_tdp_mmu_get_walk(struct kvm_vcpu vcpu, u64 addr, u64 sptes, · 18 unmodified lines
1162		1448
1163	*root_level = vcpu->arch.mmu->shadow_root_level;	1449	*root_level = vcpu->arch.mmu->shadow_root_level;
1164		1450
		1451	rcu_read_lock();
		1452
1165	tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) {	1453	tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) {
1166	leaf = iter.level;	1454	leaf = iter.level;
1167	sptes[leaf] = iter.old_spte;	1455	sptes[leaf] = iter.old_spte;
1168	}	1456	}
1169		1457
		1458	rcu_read_unlock();
		1459
1170	return leaf;	1460	return leaf;
1171	}	1461	}

arch/x86/kvm/mmu/tdp_mmu.h+28−4

@@ -5,10 +5,6 @@ · 4 unmodified lines
5		5
6	#include <linux/kvm_host.h>	6	#include <linux/kvm_host.h>
7		7
8	void kvm_mmu_init_tdp_mmu(struct kvm *kvm);
9	void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm);
10
11	bool is_tdp_mmu_root(struct kvm *kvm, hpa_t root);
12	hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu);	8	hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu);
13	void kvm_tdp_mmu_free_root(struct kvm kvm, struct kvm_mmu_page root);	9	void kvm_tdp_mmu_free_root(struct kvm kvm, struct kvm_mmu_page root);
14		10
@@ -47,4 +43,32 @@ bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm, · 32 unmodified lines
47	int kvm_tdp_mmu_get_walk(struct kvm_vcpu vcpu, u64 addr, u64 sptes,	43	int kvm_tdp_mmu_get_walk(struct kvm_vcpu vcpu, u64 addr, u64 sptes,
48	int *root_level);	44	int *root_level);
49		45
		46	#ifdef CONFIG_X86_64
		47	void kvm_mmu_init_tdp_mmu(struct kvm *kvm);
		48	void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm);
		49	static inline bool is_tdp_mmu_enabled(struct kvm *kvm) { return kvm->arch.tdp_mmu_enabled; }
		50	static inline bool is_tdp_mmu_page(struct kvm_mmu_page *sp) { return sp->tdp_mmu_page; }
		51	#else
		52	static inline void kvm_mmu_init_tdp_mmu(struct kvm *kvm) {}
		53	static inline void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) {}
		54	static inline bool is_tdp_mmu_enabled(struct kvm *kvm) { return false; }
		55	static inline bool is_tdp_mmu_page(struct kvm_mmu_page *sp) { return false; }
		56	#endif
		57
		58	static inline bool is_tdp_mmu_root(struct kvm *kvm, hpa_t hpa)
		59	{
		60	struct kvm_mmu_page *sp;
		61
		62	if (!is_tdp_mmu_enabled(kvm))
		63	return false;
		64	if (WARN_ON(!VALID_PAGE(hpa)))
		65	return false;
		66
		67	sp = to_shadow_page(hpa);
		68	if (WARN_ON(!sp))
		69	return false;
		70
		71	return is_tdp_mmu_page(sp) && sp->root_count;
		72	}
		73
50	#endif /* __KVM_X86_MMU_TDP_MMU_H */	74	#endif /* __KVM_X86_MMU_TDP_MMU_H */

arch/x86/kvm/mtrr.c+6−6

@@ -75,7 +75,7 @@ bool kvm_mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data) · 74 unmodified lines
75	/* variable MTRRs */	75	/* variable MTRRs */
76	WARN_ON(!(msr >= 0x200 && msr < 0x200 + 2 * KVM_NR_VAR_MTRR));	76	WARN_ON(!(msr >= 0x200 && msr < 0x200 + 2 * KVM_NR_VAR_MTRR));
77		77
78	mask = (~0ULL) << cpuid_maxphyaddr(vcpu);	78	mask = kvm_vcpu_reserved_gpa_bits_raw(vcpu);
79	if ((msr & 1) == 0) {	79	if ((msr & 1) == 0) {
80	/* MTRR base */	80	/* MTRR base */
81	if (!valid_mtrr_type(data & 0xff))	81	if (!valid_mtrr_type(data & 0xff))
@@ -351,14 +351,14 @@ static void set_var_mtrr_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data) · 269 unmodified lines
351	if (var_mtrr_range_is_valid(cur))	351	if (var_mtrr_range_is_valid(cur))
352	list_del(&mtrr_state->var_ranges[index].node);	352	list_del(&mtrr_state->var_ranges[index].node);
353		353
354	/* Extend the mask with all 1 bits to the left, since those	354	/*
355	* bits must implicitly be 0. The bits are then cleared	355	* Set all illegal GPA bits in the mask, since those bits must
356	* when reading them.	356	* implicitly be 0. The bits are then cleared when reading them.
357	*/	357	*/
358	if (!is_mtrr_mask)	358	if (!is_mtrr_mask)
359	cur->base = data;	359	cur->base = data;
360	else	360	else
361	cur->mask = data \| (-1LL << cpuid_maxphyaddr(vcpu));	361	cur->mask = data \| kvm_vcpu_reserved_gpa_bits_raw(vcpu);
362		362
363	/* add it to the list if it's enabled. */	363	/* add it to the list if it's enabled. */
364	if (var_mtrr_range_is_valid(cur)) {	364	if (var_mtrr_range_is_valid(cur)) {
@@ -426,7 +426,7 @@ int kvm_mtrr_get_msr(struct kvm_vcpu vcpu, u32 msr, u64 pdata) · 61 unmodified lines
426	else	426	else
427	*pdata = vcpu->arch.mtrr_state.var_ranges[index].mask;	427	*pdata = vcpu->arch.mtrr_state.var_ranges[index].mask;
428		428
429	*pdata &= (1ULL << cpuid_maxphyaddr(vcpu)) - 1;	429	*pdata &= ~kvm_vcpu_reserved_gpa_bits_raw(vcpu);
430	}	430	}
431		431
432	return 0;	432	return 0;

arch/x86/kvm/pmu.c+8−2

 		return 1;
 	if (!(kvm_read_cr4(vcpu) & X86_CR4_PCE) &&
-	    (kvm_x86_ops.get_cpl(vcpu) != 0) &&
+	    (static_call(kvm_x86_get_cpl)(vcpu) != 0) &&
 	    (kvm_read_cr0(vcpu) & X86_CR0_PE))
 		return 1;
 void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
 {
-	if (lapic_in_kernel(vcpu))
+	if (lapic_in_kernel(vcpu)) {
+		if (kvm_x86_ops.pmu_ops->deliver_pmi)
+			kvm_x86_ops.pmu_ops->deliver_pmi(vcpu);
 		kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC);
+	}
 }
 bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
 			pmc_stop_counter(pmc);
 	}
+	if (kvm_x86_ops.pmu_ops->cleanup)
+		kvm_x86_ops.pmu_ops->cleanup(vcpu);
 	bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX);
 }

arch/x86/kvm/pmu.h+2−0

@@ -39,6 +39,8 @@ struct kvm_pmu_ops { · 38 unmodified lines
39	void (refresh)(struct kvm_vcpu vcpu);	39	void (refresh)(struct kvm_vcpu vcpu);
40	void (init)(struct kvm_vcpu vcpu);	40	void (init)(struct kvm_vcpu vcpu);
41	void (reset)(struct kvm_vcpu vcpu);	41	void (reset)(struct kvm_vcpu vcpu);
		42	void (deliver_pmi)(struct kvm_vcpu vcpu);
		43	void (cleanup)(struct kvm_vcpu vcpu);
42	};	44	};
43		45
44	static inline u64 pmc_bitmask(struct kvm_pmc *pmc)	46	static inline u64 pmc_bitmask(struct kvm_pmc *pmc)

arch/x86/kvm/svm/avic.c+19−16

 	return 0;
 }
+static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source,
+				   u32 icrl, u32 icrh)
+{
+	struct kvm_vcpu *vcpu;
+	int i;
+	kvm_for_each_vcpu(i, vcpu, kvm) {
+		bool m = kvm_apic_match_dest(vcpu, source,
+					     icrl & APIC_SHORT_MASK,
+					     GET_APIC_DEST_FIELD(icrh),
+					     icrl & APIC_DEST_MASK);
+		if (m && !avic_vcpu_is_running(vcpu))
+			kvm_vcpu_wake_up(vcpu);
+	}
+}
 int avic_incomplete_ipi_interception(struct vcpu_svm *svm)
 {
 	u32 icrh = svm->vmcb->control.exit_info_1 >> 32;
 		kvm_lapic_reg_write(apic, APIC_ICR2, icrh);
 		kvm_lapic_reg_write(apic, APIC_ICR, icrl);
 		break;
-	case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: {
+	case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING:
-		int i;
-		struct kvm_vcpu *vcpu;
-		struct kvm *kvm = svm->vcpu.kvm;
-		struct kvm_lapic *apic = svm->vcpu.arch.apic;
 		/*
 		 * At this point, we expect that the AVIC HW has already
 		 * set the appropriate IRR bits on the valid target
 		 * vcpus. So, we just need to kick the appropriate vcpu.
 		 */
-		kvm_for_each_vcpu(i, vcpu, kvm) {
+		avic_kick_target_vcpus(svm->vcpu.kvm, apic, icrl, icrh);
-			bool m = kvm_apic_match_dest(vcpu, apic,
-						     icrl & APIC_SHORT_MASK,
-						     GET_APIC_DEST_FIELD(icrh),
-						     icrl & APIC_DEST_MASK);
-			if (m && !avic_vcpu_is_running(vcpu))
-				kvm_vcpu_wake_up(vcpu);
-		}
 		break;
-	}
 	case AVIC_IPI_FAILURE_INVALID_TARGET:
 		WARN_ONCE(1, "Invalid IPI target: index=%u, vcpu=%d, icr=%#0x:%#0x\n",
 			  index, svm->vcpu.vcpu_id, icrh, icrl);

arch/x86/kvm/svm/nested.c+4−4

@@ -58,7 +58,7 @@ static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) · 57 unmodified lines
58	u64 pdpte;	58	u64 pdpte;
59	int ret;	59	int ret;
60		60
61	ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(__sme_clr(cr3)), &pdpte,	61	ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte,
62	offset_in_page(cr3) + index * 8, 8);	62	offset_in_page(cr3) + index * 8, 8);
63	if (ret)	63	if (ret)
64	return 0;	64	return 0;
@@ -248,7 +248,7 @@ static bool nested_vmcb_checks(struct vcpu_svm svm, struct vmcb vmcb12) · 183 unmodified lines
248	if (vmcb12_lma) {	248	if (vmcb12_lma) {
249	if (!(vmcb12->save.cr4 & X86_CR4_PAE) \|\|	249	if (!(vmcb12->save.cr4 & X86_CR4_PAE) \|\|
250	!(vmcb12->save.cr0 & X86_CR0_PE) \|\|	250	!(vmcb12->save.cr0 & X86_CR0_PE) \|\|
251	(vmcb12->save.cr3 & vcpu->arch.cr3_lm_rsvd_bits))	251	kvm_vcpu_is_illegal_gpa(vcpu, vmcb12->save.cr3))
252	return false;	252	return false;
253	}	253	}
254	if (!kvm_is_valid_cr4(&svm->vcpu, vmcb12->save.cr4))	254	if (!kvm_is_valid_cr4(&svm->vcpu, vmcb12->save.cr4))
@@ -345,7 +345,7 @@ static inline bool nested_npt_enabled(struct vcpu_svm *svm) · 90 unmodified lines
345	static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3,	345	static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3,
346	bool nested_npt)	346	bool nested_npt)
347	{	347	{
348	if (cr3 & rsvd_bits(cpuid_maxphyaddr(vcpu), 63))	348	if (kvm_vcpu_is_illegal_gpa(vcpu, cr3))
349	return -EINVAL;	349	return -EINVAL;
350		350
351	if (!nested_npt && is_pae_paging(vcpu) &&	351	if (!nested_npt && is_pae_paging(vcpu) &&
@@ -392,7 +392,7 @@ static void nested_prepare_vmcb_save(struct vcpu_svm svm, struct vmcb vmcb12) · 40 unmodified lines
392	svm->vmcb->save.rsp = vmcb12->save.rsp;	392	svm->vmcb->save.rsp = vmcb12->save.rsp;
393	svm->vmcb->save.rip = vmcb12->save.rip;	393	svm->vmcb->save.rip = vmcb12->save.rip;
394	svm->vmcb->save.dr7 = vmcb12->save.dr7 \| DR7_FIXED_1;	394	svm->vmcb->save.dr7 = vmcb12->save.dr7 \| DR7_FIXED_1;
395	svm->vcpu.arch.dr6 = vmcb12->save.dr6 \| DR6_FIXED_1 \| DR6_RTM;	395	svm->vcpu.arch.dr6 = vmcb12->save.dr6 \| DR6_ACTIVE_LOW;
396	svm->vmcb->save.cpl = vmcb12->save.cpl;	396	svm->vmcb->save.cpl = vmcb12->save.cpl;
397	}	397	}
398		398

arch/x86/kvm/svm/sev.c+74−30

@@ -22,6 +22,7 @@ · 21 unmodified lines
22		22
23	#include "x86.h"	23	#include "x86.h"
24	#include "svm.h"	24	#include "svm.h"
		25	#include "svm_ops.h"
25	#include "cpuid.h"	26	#include "cpuid.h"
26	#include "trace.h"	27	#include "trace.h"
27		28
@@ -1041,6 +1042,74 @@ e_unpin_memory: · 1013 unmodified lines
1041	return ret;	1042	return ret;
1042	}	1043	}
1043		1044
		1045	static int sev_get_attestation_report(struct kvm kvm, struct kvm_sev_cmd argp)
		1046	{
		1047	void __user report = (void __user )(uintptr_t)argp->data;
		1048	struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
		1049	struct sev_data_attestation_report *data;
		1050	struct kvm_sev_attestation_report params;
		1051	void __user *p;
		1052	void *blob = NULL;
		1053	int ret;
		1054
		1055	if (!sev_guest(kvm))
		1056	return -ENOTTY;
		1057
		1058	if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
		1059	return -EFAULT;
		1060
		1061	data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
		1062	if (!data)
		1063	return -ENOMEM;
		1064
		1065	/* User wants to query the blob length */
		1066	if (!params.len)
		1067	goto cmd;
		1068
		1069	p = (void __user *)(uintptr_t)params.uaddr;
		1070	if (p) {
		1071	if (params.len > SEV_FW_BLOB_MAX_SIZE) {
		1072	ret = -EINVAL;
		1073	goto e_free;
		1074	}
		1075
		1076	ret = -ENOMEM;
		1077	blob = kmalloc(params.len, GFP_KERNEL);
		1078	if (!blob)
		1079	goto e_free;
		1080
		1081	data->address = __psp_pa(blob);
		1082	data->len = params.len;
		1083	memcpy(data->mnonce, params.mnonce, sizeof(params.mnonce));
		1084	}
		1085	cmd:
		1086	data->handle = sev->handle;
		1087	ret = sev_issue_cmd(kvm, SEV_CMD_ATTESTATION_REPORT, data, &argp->error);
		1088	/*
		1089	* If we query the session length, FW responded with expected data.
		1090	*/
		1091	if (!params.len)
		1092	goto done;
		1093
		1094	if (ret)
		1095	goto e_free_blob;
		1096
		1097	if (blob) {
		1098	if (copy_to_user(p, blob, params.len))
		1099	ret = -EFAULT;
		1100	}
		1101
		1102	done:
		1103	params.len = data->len;
		1104	if (copy_to_user(report, &params, sizeof(params)))
		1105	ret = -EFAULT;
		1106	e_free_blob:
		1107	kfree(blob);
		1108	e_free:
		1109	kfree(data);
		1110	return ret;
		1111	}
		1112
1044	int svm_mem_enc_op(struct kvm kvm, void __user argp)	1113	int svm_mem_enc_op(struct kvm kvm, void __user argp)
1045	{	1114	{
1046	struct kvm_sev_cmd sev_cmd;	1115	struct kvm_sev_cmd sev_cmd;
@@ -1091,6 +1160,9 @@ int svm_mem_enc_op(struct kvm kvm, void __user argp) · 44 unmodified lines
1091	case KVM_SEV_LAUNCH_SECRET:	1160	case KVM_SEV_LAUNCH_SECRET:
1092	r = sev_launch_secret(kvm, &sev_cmd);	1161	r = sev_launch_secret(kvm, &sev_cmd);
1093	break;	1162	break;
		1163	case KVM_SEV_GET_ATTESTATION_REPORT:
		1164	r = sev_get_attestation_report(kvm, &sev_cmd);
		1165	break;
1094	default:	1166	default:
1095	r = -EINVAL;	1167	r = -EINVAL;
1096	goto out;	1168	goto out;
@@ -1994,29 +2066,17 @@ void sev_es_create_vcpu(struct vcpu_svm *svm) · 897 unmodified lines
1994	sev_enc_bit));	2066	sev_enc_bit));
1995	}	2067	}
1996		2068
1997	void sev_es_vcpu_load(struct vcpu_svm *svm, int cpu)	2069	void sev_es_prepare_guest_switch(struct vcpu_svm *svm, unsigned int cpu)
1998	{	2070	{
1999	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);	2071	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
2000	struct vmcb_save_area *hostsa;	2072	struct vmcb_save_area *hostsa;
2001	unsigned int i;
2002		2073
2003	/*	2074	/*
2004	* As an SEV-ES guest, hardware will restore the host state on VMEXIT,	2075	* As an SEV-ES guest, hardware will restore the host state on VMEXIT,
2005	* of which one step is to perform a VMLOAD. Since hardware does not	2076	* of which one step is to perform a VMLOAD. Since hardware does not
2006	* perform a VMSAVE on VMRUN, the host savearea must be updated.	2077	* perform a VMSAVE on VMRUN, the host savearea must be updated.
2007	*/	2078	*/
2008	asm volatile(__ex("vmsave %0") : : "a" (__sme_page_pa(sd->save_area)) : "memory");	2079	vmsave(__sme_page_pa(sd->save_area));
2009
2010	/*
2011	* Certain MSRs are restored on VMEXIT, only save ones that aren't
2012	* restored.
2013	*/
2014	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) {
2015	if (host_save_user_msrs[i].sev_es_restored)
2016	continue;
2017
2018	rdmsrl(host_save_user_msrs[i].index, svm->host_user_msrs[i]);
2019	}
2020		2080
2021	/* XCR0 is restored on VMEXIT, save the current host value */	2081	/* XCR0 is restored on VMEXIT, save the current host value */
2022	hostsa = (struct vmcb_save_area *)(page_address(sd->save_area) + 0x400);	2082	hostsa = (struct vmcb_save_area *)(page_address(sd->save_area) + 0x400);
@@ -2029,22 +2089,6 @@ void sev_es_vcpu_load(struct vcpu_svm *svm, int cpu) · 6 unmodified lines
2029	hostsa->xss = host_xss;	2089	hostsa->xss = host_xss;
2030	}	2090	}
2031		2091
2032	void sev_es_vcpu_put(struct vcpu_svm *svm)
2033	{
2034	unsigned int i;
2035
2036	/*
2037	* Certain MSRs are restored on VMEXIT and were saved with vmsave in
2038	* sev_es_vcpu_load() above. Only restore ones that weren't.
2039	*/
2040	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) {
2041	if (host_save_user_msrs[i].sev_es_restored)
2042	continue;
2043
2044	wrmsrl(host_save_user_msrs[i].index, svm->host_user_msrs[i]);
2045	}
2046	}
2047
2048	void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)	2092	void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
2049	{	2093	{
2050	struct vcpu_svm *svm = to_svm(vcpu);	2094	struct vcpu_svm *svm = to_svm(vcpu);

arch/x86/kvm/svm/svm.c+189−114

@@ -41,6 +41,7 @@ · 40 unmodified lines
41	#include "trace.h"	41	#include "trace.h"
42		42
43	#include "svm.h"	43	#include "svm.h"
		44	#include "svm_ops.h"
44		45
45	#define __ex(x) __kvm_handle_fault_on_reboot(x)	46	#define __ex(x) __kvm_handle_fault_on_reboot(x)
46		47
@@ -200,9 +201,9 @@ module_param(sev_es, int, 0444); · 153 unmodified lines
200	bool __read_mostly dump_invalid_vmcb;	201	bool __read_mostly dump_invalid_vmcb;
201	module_param(dump_invalid_vmcb, bool, 0644);	202	module_param(dump_invalid_vmcb, bool, 0644);
202		203
203	static u8 rsm_ins_bytes[] = "\x0f\xaa";	204	static bool svm_gp_erratum_intercept = true;
204		205
205	static void svm_complete_interrupts(struct vcpu_svm *svm);	206	static u8 rsm_ins_bytes[] = "\x0f\xaa";
206		207
207	static unsigned long iopm_base;	208	static unsigned long iopm_base;
208		209
@@ -246,21 +247,6 @@ u32 svm_msrpm_offset(u32 msr) · 37 unmodified lines
246		247
247	#define MAX_INST_SIZE 15	248	#define MAX_INST_SIZE 15
248		249
249	static inline void clgi(void)
250	{
251	asm volatile (__ex("clgi"));
252	}
253
254	static inline void stgi(void)
255	{
256	asm volatile (__ex("stgi"));
257	}
258
259	static inline void invlpga(unsigned long addr, u32 asid)
260	{
261	asm volatile (__ex("invlpga %1, %0") : : "c"(asid), "a"(addr));
262	}
263
264	static int get_max_npt_level(void)	250	static int get_max_npt_level(void)
265	{	251	{
266	#ifdef CONFIG_X86_64	252	#ifdef CONFIG_X86_64
@@ -288,6 +274,9 @@ int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) · 21 unmodified lines
288	if (!(efer & EFER_SVME)) {	274	if (!(efer & EFER_SVME)) {
289	svm_leave_nested(svm);	275	svm_leave_nested(svm);
290	svm_set_gif(svm, true);	276	svm_set_gif(svm, true);
		277	/* #GP intercept is still needed for vmware backdoor */
		278	if (!enable_vmware_backdoor)
		279	clr_exception_intercept(svm, GP_VECTOR);
291		280
292	/*	281	/*
293	* Free the nested guest state, unless we are in SMM.	282	* Free the nested guest state, unless we are in SMM.
@@ -304,6 +293,9 @@ int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) · 10 unmodified lines
304	vcpu->arch.efer = old_efer;	293	vcpu->arch.efer = old_efer;
305	return ret;	294	return ret;
306	}	295	}
		296
		297	if (svm_gp_erratum_intercept)
		298	set_exception_intercept(svm, GP_VECTOR);
307	}	299	}
308	}	300	}
309		301
@@ -925,6 +917,9 @@ static __init void svm_set_cpu_caps(void) · 615 unmodified lines
925		917
926	if (npt_enabled)	918	if (npt_enabled)
927	kvm_cpu_cap_set(X86_FEATURE_NPT);	919	kvm_cpu_cap_set(X86_FEATURE_NPT);
		920
		921	/* Nested VM can receive #VMEXIT instead of triggering #GP */
		922	kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK);
928	}	923	}
929		924
930	/* CPUID 0x80000008 */	925	/* CPUID 0x80000008 */
@@ -1032,6 +1027,9 @@ static __init int svm_hardware_setup(void) · 101 unmodified lines
1032	}	1027	}
1033	}	1028	}
1034		1029
		1030	if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK))
		1031	svm_gp_erratum_intercept = false;
		1032
1035	if (vgif) {	1033	if (vgif) {
1036	if (!boot_cpu_has(X86_FEATURE_VGIF))	1034	if (!boot_cpu_has(X86_FEATURE_VGIF))
1037	vgif = false;	1035	vgif = false;
@@ -1207,7 +1205,7 @@ static void init_vmcb(struct vcpu_svm *svm) · 169 unmodified lines
1207		1205
1208	svm_set_efer(&svm->vcpu, 0);	1206	svm_set_efer(&svm->vcpu, 0);
1209	save->dr6 = 0xffff0ff0;	1207	save->dr6 = 0xffff0ff0;
1210	kvm_set_rflags(&svm->vcpu, 2);	1208	kvm_set_rflags(&svm->vcpu, X86_EFLAGS_FIXED);
1211	save->rip = 0x0000fff0;	1209	save->rip = 0x0000fff0;
1212	svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;	1210	svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
1213		1211
@@ -1366,6 +1364,7 @@ static int svm_create_vcpu(struct kvm_vcpu *vcpu) · 152 unmodified lines
1366	svm->vmsa = page_address(vmsa_page);	1364	svm->vmsa = page_address(vmsa_page);
1367		1365
1368	svm->asid_generation = 0;	1366	svm->asid_generation = 0;
		1367	svm->guest_state_loaded = false;
1369	init_vmcb(svm);	1368	init_vmcb(svm);
1370		1369
1371	svm_init_osvw(vcpu);	1370	svm_init_osvw(vcpu);
@@ -1413,30 +1412,31 @@ static void svm_free_vcpu(struct kvm_vcpu *vcpu) · 41 unmodified lines
1413	__free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);	1412	__free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
1414	}	1413	}
1415		1414
1416	static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)	1415	static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
1417	{	1416	{
1418	struct vcpu_svm *svm = to_svm(vcpu);	1417	struct vcpu_svm *svm = to_svm(vcpu);
1419	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);	1418	struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
1420	int i;	1419	unsigned int i;
1421		1420
1422	if (unlikely(cpu != vcpu->cpu)) {	1421	if (svm->guest_state_loaded)
1423	svm->asid_generation = 0;	1422	return;
1424	vmcb_mark_all_dirty(svm->vmcb);
1425	}
1426		1423
		1424	/*
		1425	* Certain MSRs are restored on VMEXIT (sev-es), or vmload of host save
		1426	* area (non-sev-es). Save ones that aren't so we can restore them
		1427	* individually later.
		1428	*/
		1429	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
		1430	rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
		1431
		1432	/*
		1433	* Save additional host state that will be restored on VMEXIT (sev-es)
		1434	* or subsequent vmload of host save area.
		1435	*/
1427	if (sev_es_guest(svm->vcpu.kvm)) {	1436	if (sev_es_guest(svm->vcpu.kvm)) {
1428	sev_es_vcpu_load(svm, cpu);	1437	sev_es_prepare_guest_switch(svm, vcpu->cpu);
1429	} else {	1438	} else {
1430	#ifdef CONFIG_X86_64	1439	vmsave(__sme_page_pa(sd->save_area));
1431	rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
1432	#endif
1433	savesegment(fs, svm->host.fs);
1434	savesegment(gs, svm->host.gs);
1435	svm->host.ldt = kvm_read_ldt();
1436
1437	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1438	rdmsrl(host_save_user_msrs[i].index,
1439	svm->host_user_msrs[i]);
1440	}	1440	}
1441		1441
1442	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {	1442	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
@@ -1446,10 +1446,42 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) · 3 unmodified lines
1446	wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);	1446	wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
1447	}	1447	}
1448	}	1448	}
		1449
1449	/* This assumes that the kernel never uses MSR_TSC_AUX */	1450	/* This assumes that the kernel never uses MSR_TSC_AUX */
1450	if (static_cpu_has(X86_FEATURE_RDTSCP))	1451	if (static_cpu_has(X86_FEATURE_RDTSCP))
1451	wrmsrl(MSR_TSC_AUX, svm->tsc_aux);	1452	wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
1452		1453
		1454	svm->guest_state_loaded = true;
		1455	}
		1456
		1457	static void svm_prepare_host_switch(struct kvm_vcpu *vcpu)
		1458	{
		1459	struct vcpu_svm *svm = to_svm(vcpu);
		1460	unsigned int i;
		1461
		1462	if (!svm->guest_state_loaded)
		1463	return;
		1464
		1465	/*
		1466	* Certain MSRs are restored on VMEXIT (sev-es), or vmload of host save
		1467	* area (non-sev-es). Restore the ones that weren't.
		1468	*/
		1469	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
		1470	wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
		1471
		1472	svm->guest_state_loaded = false;
		1473	}
		1474
		1475	static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
		1476	{
		1477	struct vcpu_svm *svm = to_svm(vcpu);
		1478	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
		1479
		1480	if (unlikely(cpu != vcpu->cpu)) {
		1481	svm->asid_generation = 0;
		1482	vmcb_mark_all_dirty(svm->vmcb);
		1483	}
		1484
1453	if (sd->current_vmcb != svm->vmcb) {	1485	if (sd->current_vmcb != svm->vmcb) {
1454	sd->current_vmcb = svm->vmcb;	1486	sd->current_vmcb = svm->vmcb;
1455	indirect_branch_prediction_barrier();	1487	indirect_branch_prediction_barrier();
@@ -1459,30 +1491,10 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) · 3 unmodified lines
1459		1491
1460	static void svm_vcpu_put(struct kvm_vcpu *vcpu)	1492	static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1461	{	1493	{
1462	struct vcpu_svm *svm = to_svm(vcpu);
1463	int i;
1464
1465	avic_vcpu_put(vcpu);	1494	avic_vcpu_put(vcpu);
		1495	svm_prepare_host_switch(vcpu);
1466		1496
1467	++vcpu->stat.host_state_reload;	1497	++vcpu->stat.host_state_reload;
1468	if (sev_es_guest(svm->vcpu.kvm)) {
1469	sev_es_vcpu_put(svm);
1470	} else {
1471	kvm_load_ldt(svm->host.ldt);
1472	#ifdef CONFIG_X86_64
1473	loadsegment(fs, svm->host.fs);
1474	wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gsbase);
1475	load_gs_index(svm->host.gs);
1476	#else
1477	#ifdef CONFIG_X86_32_LAZY_GS
1478	loadsegment(gs, svm->host.gs);
1479	#endif
1480	#endif
1481
1482	for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1483	wrmsrl(host_save_user_msrs[i].index,
1484	svm->host_user_msrs[i]);
1485	}
1486	}	1498	}
1487		1499
1488	static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)	1500	static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
@@ -1815,7 +1827,7 @@ static void svm_set_segment(struct kvm_vcpu *vcpu, · 326 unmodified lines
1815	vmcb_mark_dirty(svm->vmcb, VMCB_SEG);	1827	vmcb_mark_dirty(svm->vmcb, VMCB_SEG);
1816	}	1828	}
1817		1829
1818	static void update_exception_bitmap(struct kvm_vcpu *vcpu)	1830	static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu)
1819	{	1831	{
1820	struct vcpu_svm *svm = to_svm(vcpu);	1832	struct vcpu_svm *svm = to_svm(vcpu);
1821		1833
@@ -1865,7 +1877,7 @@ static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) · 43 unmodified lines
1865	get_debugreg(vcpu->arch.db[2], 2);	1877	get_debugreg(vcpu->arch.db[2], 2);
1866	get_debugreg(vcpu->arch.db[3], 3);	1878	get_debugreg(vcpu->arch.db[3], 3);
1867	/*	1879	/*
1868	* We cannot reset svm->vmcb->save.dr6 to DR6_FIXED_1\|DR6_RTM here,	1880	* We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here,
1869	* because db_interception might need it. We can do it before vmentry.	1881	* because db_interception might need it. We can do it before vmentry.
1870	*/	1882	*/
1871	vcpu->arch.dr6 = svm->vmcb->save.dr6;	1883	vcpu->arch.dr6 = svm->vmcb->save.dr6;
@@ -1916,7 +1928,7 @@ static int db_interception(struct vcpu_svm *svm) · 44 unmodified lines
1916	if (!(svm->vcpu.guest_debug &	1928	if (!(svm->vcpu.guest_debug &
1917	(KVM_GUESTDBG_SINGLESTEP \| KVM_GUESTDBG_USE_HW_BP)) &&	1929	(KVM_GUESTDBG_SINGLESTEP \| KVM_GUESTDBG_USE_HW_BP)) &&
1918	!svm->nmi_singlestep) {	1930	!svm->nmi_singlestep) {
1919	u32 payload = (svm->vmcb->save.dr6 ^ DR6_RTM) & ~DR6_FIXED_1;	1931	u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW;
1920	kvm_queue_exception_p(&svm->vcpu, DB_VECTOR, payload);	1932	kvm_queue_exception_p(&svm->vcpu, DB_VECTOR, payload);
1921	return 1;	1933	return 1;
1922	}	1934	}
@@ -1962,24 +1974,6 @@ static int ac_interception(struct vcpu_svm *svm) · 39 unmodified lines
1962	return 1;	1974	return 1;
1963	}	1975	}
1964		1976
1965	static int gp_interception(struct vcpu_svm *svm)
1966	{
1967	struct kvm_vcpu *vcpu = &svm->vcpu;
1968	u32 error_code = svm->vmcb->control.exit_info_1;
1969
1970	WARN_ON_ONCE(!enable_vmware_backdoor);
1971
1972	/*
1973	* VMware backdoor emulation on #GP interception only handles IN{S},
1974	* OUT{S}, and RDPMC, none of which generate a non-zero error code.
1975	*/
1976	if (error_code) {
1977	kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
1978	return 1;
1979	}
1980	return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP);
1981	}
1982
1983	static bool is_erratum_383(void)	1977	static bool is_erratum_383(void)
1984	{	1978	{
1985	int err, i;	1979	int err, i;
@@ -2178,6 +2172,102 @@ static int vmrun_interception(struct vcpu_svm *svm) · 192 unmodified lines
2178	return nested_svm_vmrun(svm);	2172	return nested_svm_vmrun(svm);
2179	}	2173	}
2180		2174
		2175	enum {
		2176	NONE_SVM_INSTR,
		2177	SVM_INSTR_VMRUN,
		2178	SVM_INSTR_VMLOAD,
		2179	SVM_INSTR_VMSAVE,
		2180	};
		2181
		2182	/* Return NONE_SVM_INSTR if not SVM instrs, otherwise return decode result */
		2183	static int svm_instr_opcode(struct kvm_vcpu *vcpu)
		2184	{
		2185	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
		2186
		2187	if (ctxt->b != 0x1 \|\| ctxt->opcode_len != 2)
		2188	return NONE_SVM_INSTR;
		2189
		2190	switch (ctxt->modrm) {
		2191	case 0xd8: /* VMRUN */
		2192	return SVM_INSTR_VMRUN;
		2193	case 0xda: /* VMLOAD */
		2194	return SVM_INSTR_VMLOAD;
		2195	case 0xdb: /* VMSAVE */
		2196	return SVM_INSTR_VMSAVE;
		2197	default:
		2198	break;
		2199	}
		2200
		2201	return NONE_SVM_INSTR;
		2202	}
		2203
		2204	static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode)
		2205	{
		2206	const int guest_mode_exit_codes[] = {
		2207	[SVM_INSTR_VMRUN] = SVM_EXIT_VMRUN,
		2208	[SVM_INSTR_VMLOAD] = SVM_EXIT_VMLOAD,
		2209	[SVM_INSTR_VMSAVE] = SVM_EXIT_VMSAVE,
		2210	};
		2211	int (const svm_instr_handlers[])(struct vcpu_svm svm) = {
		2212	[SVM_INSTR_VMRUN] = vmrun_interception,
		2213	[SVM_INSTR_VMLOAD] = vmload_interception,
		2214	[SVM_INSTR_VMSAVE] = vmsave_interception,
		2215	};
		2216	struct vcpu_svm *svm = to_svm(vcpu);
		2217
		2218	if (is_guest_mode(vcpu)) {
		2219	svm->vmcb->control.exit_code = guest_mode_exit_codes[opcode];
		2220	svm->vmcb->control.exit_info_1 = 0;
		2221	svm->vmcb->control.exit_info_2 = 0;
		2222
		2223	return nested_svm_vmexit(svm);
		2224	} else
		2225	return svm_instr_handlers[opcode](svm);
		2226	}
		2227
		2228	/*
		2229	* #GP handling code. Note that #GP can be triggered under the following two
		2230	* cases:
		2231	* 1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on
		2232	* some AMD CPUs when EAX of these instructions are in the reserved memory
		2233	* regions (e.g. SMM memory on host).
		2234	* 2) VMware backdoor
		2235	*/
		2236	static int gp_interception(struct vcpu_svm *svm)
		2237	{
		2238	struct kvm_vcpu *vcpu = &svm->vcpu;
		2239	u32 error_code = svm->vmcb->control.exit_info_1;
		2240	int opcode;
		2241
		2242	/* Both #GP cases have zero error_code */
		2243	if (error_code)
		2244	goto reinject;
		2245
		2246	/* Decode the instruction for usage later */
		2247	if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK)
		2248	goto reinject;
		2249
		2250	opcode = svm_instr_opcode(vcpu);
		2251
		2252	if (opcode == NONE_SVM_INSTR) {
		2253	if (!enable_vmware_backdoor)
		2254	goto reinject;
		2255
		2256	/*
		2257	* VMware backdoor emulation on #GP interception only handles
		2258	* IN{S}, OUT{S}, and RDPMC.
		2259	*/
		2260	if (!is_guest_mode(vcpu))
		2261	return kvm_emulate_instruction(vcpu,
		2262	EMULTYPE_VMWARE_GP \| EMULTYPE_NO_DECODE);
		2263	} else
		2264	return emulate_svm_instr(vcpu, opcode);
		2265
		2266	reinject:
		2267	kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
		2268	return 1;
		2269	}
		2270
2181	void svm_set_gif(struct vcpu_svm *svm, bool value)	2271	void svm_set_gif(struct vcpu_svm *svm, bool value)
2182	{	2272	{
2183	if (value) {	2273	if (value) {
@@ -2265,11 +2355,8 @@ static int xsetbv_interception(struct vcpu_svm *svm) · 81 unmodified lines
2265	u64 new_bv = kvm_read_edx_eax(&svm->vcpu);	2355	u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
2266	u32 index = kvm_rcx_read(&svm->vcpu);	2356	u32 index = kvm_rcx_read(&svm->vcpu);
2267		2357
2268	if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {	2358	int err = kvm_set_xcr(&svm->vcpu, index, new_bv);
2269	return kvm_skip_emulated_instruction(&svm->vcpu);	2359	return kvm_complete_insn_gp(&svm->vcpu, err);
2270	}
2271
2272	return 1;
2273	}	2360	}
2274		2361
2275	static int rdpru_interception(struct vcpu_svm *svm)	2362	static int rdpru_interception(struct vcpu_svm *svm)
@@ -2530,6 +2617,7 @@ static int dr_interception(struct vcpu_svm *svm) · 254 unmodified lines
2530	{	2617	{
2531	int reg, dr;	2618	int reg, dr;
2532	unsigned long val;	2619	unsigned long val;
		2620	int err = 0;
2533		2621
2534	if (svm->vcpu.guest_debug == 0) {	2622	if (svm->vcpu.guest_debug == 0) {
2535	/*	2623	/*
@@ -2547,20 +2635,16 @@ static int dr_interception(struct vcpu_svm *svm) · 11 unmodified lines
2547		2635
2548	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;	2636	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2549	dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;	2637	dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
2550		2638	if (dr >= 16) { /* mov to DRn */
2551	if (dr >= 16) { /* mov to DRn */	2639	dr -= 16;
2552	if (!kvm_require_dr(&svm->vcpu, dr - 16))
2553	return 1;
2554	val = kvm_register_read(&svm->vcpu, reg);	2640	val = kvm_register_read(&svm->vcpu, reg);
2555	kvm_set_dr(&svm->vcpu, dr - 16, val);	2641	err = kvm_set_dr(&svm->vcpu, dr, val);
2556	} else {	2642	} else {
2557	if (!kvm_require_dr(&svm->vcpu, dr))
2558	return 1;
2559	kvm_get_dr(&svm->vcpu, dr, &val);	2643	kvm_get_dr(&svm->vcpu, dr, &val);
2560	kvm_register_write(&svm->vcpu, reg, val);	2644	kvm_register_write(&svm->vcpu, reg, val);
2561	}	2645	}
2562		2646
2563	return kvm_skip_emulated_instruction(&svm->vcpu);	2647	return kvm_complete_insn_gp(&svm->vcpu, err);
2564	}	2648	}
2565		2649
2566	static int cr8_write_interception(struct vcpu_svm *svm)	2650	static int cr8_write_interception(struct vcpu_svm *svm)
@@ -3354,7 +3438,7 @@ static void svm_set_irq(struct kvm_vcpu *vcpu) · 787 unmodified lines
3354	SVM_EVTINJ_VALID \| SVM_EVTINJ_TYPE_INTR;	3438	SVM_EVTINJ_VALID \| SVM_EVTINJ_TYPE_INTR;
3355	}	3439	}
3356		3440
3357	static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)	3441	static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3358	{	3442	{
3359	struct vcpu_svm *svm = to_svm(vcpu);	3443	struct vcpu_svm *svm = to_svm(vcpu);
3360		3444
@@ -3479,7 +3563,7 @@ static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) · 118 unmodified lines
3479	return !svm_interrupt_blocked(vcpu);	3563	return !svm_interrupt_blocked(vcpu);
3480	}	3564	}
3481		3565
3482	static void enable_irq_window(struct kvm_vcpu *vcpu)	3566	static void svm_enable_irq_window(struct kvm_vcpu *vcpu)
3483	{	3567	{
3484	struct vcpu_svm *svm = to_svm(vcpu);	3568	struct vcpu_svm *svm = to_svm(vcpu);
3485		3569
@@ -3503,7 +3587,7 @@ static void enable_irq_window(struct kvm_vcpu *vcpu) · 17 unmodified lines
3503	}	3587	}
3504	}	3588	}
3505		3589
3506	static void enable_nmi_window(struct kvm_vcpu *vcpu)	3590	static void svm_enable_nmi_window(struct kvm_vcpu *vcpu)
3507	{	3591	{
3508	struct vcpu_svm *svm = to_svm(vcpu);	3592	struct vcpu_svm *svm = to_svm(vcpu);
3509		3593
@@ -3560,10 +3644,6 @@ static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva) · 50 unmodified lines
3560	invlpga(gva, svm->vmcb->control.asid);	3644	invlpga(gva, svm->vmcb->control.asid);
3561	}	3645	}
3562		3646
3563	static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
3564	{
3565	}
3566
3567	static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)	3647	static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
3568	{	3648	{
3569	struct vcpu_svm *svm = to_svm(vcpu);	3649	struct vcpu_svm *svm = to_svm(vcpu);
@@ -3708,16 +3788,11 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, · 138 unmodified lines
3708	if (sev_es_guest(svm->vcpu.kvm)) {	3788	if (sev_es_guest(svm->vcpu.kvm)) {
3709	__svm_sev_es_vcpu_run(svm->vmcb_pa);	3789	__svm_sev_es_vcpu_run(svm->vmcb_pa);
3710	} else {	3790	} else {
		3791	struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
		3792
3711	__svm_vcpu_run(svm->vmcb_pa, (unsigned long *)&svm->vcpu.arch.regs);	3793	__svm_vcpu_run(svm->vmcb_pa, (unsigned long *)&svm->vcpu.arch.regs);
3712		3794
3713	#ifdef CONFIG_X86_64	3795	vmload(__sme_page_pa(sd->save_area));
3714	native_wrmsrl(MSR_GS_BASE, svm->host.gs_base);
3715	#else
3716	loadsegment(fs, svm->host.fs);
3717	#ifndef CONFIG_X86_32_LAZY_GS
3718	loadsegment(gs, svm->host.gs);
3719	#endif
3720	#endif
3721	}	3796	}
3722		3797
3723	/*	3798	/*
@@ -3783,7 +3858,7 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) · 59 unmodified lines
3783	if (unlikely(svm->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))	3858	if (unlikely(svm->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))
3784	svm_set_dr6(svm, vcpu->arch.dr6);	3859	svm_set_dr6(svm, vcpu->arch.dr6);
3785	else	3860	else
3786	svm_set_dr6(svm, DR6_FIXED_1 \| DR6_RTM);	3861	svm_set_dr6(svm, DR6_ACTIVE_LOW);
3787		3862
3788	clgi();	3863	clgi();
3789	kvm_load_guest_xsave_state(vcpu);	3864	kvm_load_guest_xsave_state(vcpu);
@@ -3978,7 +4053,7 @@ static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) · 188 unmodified lines
3978	if (sev_guest(vcpu->kvm)) {	4053	if (sev_guest(vcpu->kvm)) {
3979	best = kvm_find_cpuid_entry(vcpu, 0x8000001F, 0);	4054	best = kvm_find_cpuid_entry(vcpu, 0x8000001F, 0);
3980	if (best)	4055	if (best)
3981	vcpu->arch.cr3_lm_rsvd_bits &= ~(1UL << (best->ebx & 0x3f));	4056	vcpu->arch.reserved_gpa_bits &= ~(1UL << (best->ebx & 0x3f));
3982	}	4057	}
3983		4058
3984	if (!kvm_vcpu_apicv_active(vcpu))	4059	if (!kvm_vcpu_apicv_active(vcpu))
@@ -4285,7 +4360,7 @@ static int svm_pre_leave_smm(struct kvm_vcpu vcpu, const char smstate) · 300 unmodified lines
4285	return ret;	4360	return ret;
4286	}	4361	}
4287		4362
4288	static void enable_smi_window(struct kvm_vcpu *vcpu)	4363	static void svm_enable_smi_window(struct kvm_vcpu *vcpu)
4289	{	4364	{
4290	struct vcpu_svm *svm = to_svm(vcpu);	4365	struct vcpu_svm *svm = to_svm(vcpu);
4291		4366
@@ -4439,7 +4514,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { · 147 unmodified lines
4439	.vcpu_blocking = svm_vcpu_blocking,	4514	.vcpu_blocking = svm_vcpu_blocking,
4440	.vcpu_unblocking = svm_vcpu_unblocking,	4515	.vcpu_unblocking = svm_vcpu_unblocking,
4441		4516
4442	.update_exception_bitmap = update_exception_bitmap,	4517	.update_exception_bitmap = svm_update_exception_bitmap,
4443	.get_msr_feature = svm_get_msr_feature,	4518	.get_msr_feature = svm_get_msr_feature,
4444	.get_msr = svm_get_msr,	4519	.get_msr = svm_get_msr,
4445	.set_msr = svm_set_msr,	4520	.set_msr = svm_set_msr,
@@ -4482,9 +4557,9 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { · 36 unmodified lines
4482	.nmi_allowed = svm_nmi_allowed,	4557	.nmi_allowed = svm_nmi_allowed,
4483	.get_nmi_mask = svm_get_nmi_mask,	4558	.get_nmi_mask = svm_get_nmi_mask,
4484	.set_nmi_mask = svm_set_nmi_mask,	4559	.set_nmi_mask = svm_set_nmi_mask,
4485	.enable_nmi_window = enable_nmi_window,	4560	.enable_nmi_window = svm_enable_nmi_window,
4486	.enable_irq_window = enable_irq_window,	4561	.enable_irq_window = svm_enable_irq_window,
4487	.update_cr8_intercept = update_cr8_intercept,	4562	.update_cr8_intercept = svm_update_cr8_intercept,
4488	.set_virtual_apic_mode = svm_set_virtual_apic_mode,	4563	.set_virtual_apic_mode = svm_set_virtual_apic_mode,
4489	.refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,	4564	.refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
4490	.check_apicv_inhibit_reasons = svm_check_apicv_inhibit_reasons,	4565	.check_apicv_inhibit_reasons = svm_check_apicv_inhibit_reasons,
@@ -4527,7 +4602,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { · 36 unmodified lines
4527	.smi_allowed = svm_smi_allowed,	4602	.smi_allowed = svm_smi_allowed,
4528	.pre_enter_smm = svm_pre_enter_smm,	4603	.pre_enter_smm = svm_pre_enter_smm,
4529	.pre_leave_smm = svm_pre_leave_smm,	4604	.pre_leave_smm = svm_pre_leave_smm,
4530	.enable_smi_window = enable_smi_window,	4605	.enable_smi_window = svm_enable_smi_window,
4531		4606
4532	.mem_enc_op = svm_mem_enc_op,	4607	.mem_enc_op = svm_mem_enc_op,
4533	.mem_enc_reg_region = svm_register_enc_region,	4608	.mem_enc_reg_region = svm_register_enc_region,

arch/x86/kvm/svm/svm.h+5−24

 #define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT)
-static const struct svm_host_save_msrs {
+static const u32 host_save_user_msrs[] = {
-	u32 index;		/* Index of the MSR */
+	MSR_TSC_AUX,
-	bool sev_es_restored;	/* True if MSR is restored on SEV-ES VMEXIT */
-} host_save_user_msrs[] = {
-#ifdef CONFIG_X86_64
-	{ .index = MSR_STAR,			.sev_es_restored = true },
-	{ .index = MSR_LSTAR,			.sev_es_restored = true },
-	{ .index = MSR_CSTAR,			.sev_es_restored = true },
-	{ .index = MSR_SYSCALL_MASK,		.sev_es_restored = true },
-	{ .index = MSR_KERNEL_GS_BASE,		.sev_es_restored = true },
-	{ .index = MSR_FS_BASE,			.sev_es_restored = true },
-#endif
-	{ .index = MSR_IA32_SYSENTER_CS,	.sev_es_restored = true },
-	{ .index = MSR_IA32_SYSENTER_ESP,	.sev_es_restored = true },
-	{ .index = MSR_IA32_SYSENTER_EIP,	.sev_es_restored = true },
-	{ .index = MSR_TSC_AUX,			.sev_es_restored = false },
 };
 #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
 	u64 next_rip;
 	u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
-	struct {
-		u16 fs;
-		u16 gs;
-		u16 ldt;
-		u64 gs_base;
-	} host;
 	u64 spec_ctrl;
 	/*
 	u64 ghcb_sa_len;
 	bool ghcb_sa_sync;
 	bool ghcb_sa_free;
+	bool guest_state_loaded;
 };
 struct svm_cpu_data {
 int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in);
 void sev_es_init_vmcb(struct vcpu_svm *svm);
 void sev_es_create_vcpu(struct vcpu_svm *svm);
-void sev_es_vcpu_load(struct vcpu_svm *svm, int cpu);
-void sev_es_vcpu_put(struct vcpu_svm *svm);
 void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector);
+void sev_es_prepare_guest_switch(struct vcpu_svm *svm, unsigned int cpu);
 /* vmenter.S */

arch/x86/kvm/svm/svm_ops.h+69−0

@@ -0,0 +1,69 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef __KVM_X86_SVM_OPS_H
	3	#define __KVM_X86_SVM_OPS_H
	4
	5	#include <linux/compiler_types.h>
	6
	7	#include <asm/kvm_host.h>
	8
	9	#define svm_asm(insn, clobber...) \
	10	do { \
	11	asm_volatile_goto("1: " __stringify(insn) "\n\t" \
	12	_ASM_EXTABLE(1b, %l[fault]) \
	13	::: clobber : fault); \
	14	return; \
	15	fault: \
	16	kvm_spurious_fault(); \
	17	} while (0)
	18
	19	#define svm_asm1(insn, op1, clobber...) \
	20	do { \
	21	asm_volatile_goto("1: " __stringify(insn) " %0\n\t" \
	22	_ASM_EXTABLE(1b, %l[fault]) \
	23	:: op1 : clobber : fault); \
	24	return; \
	25	fault: \
	26	kvm_spurious_fault(); \
	27	} while (0)
	28
	29	#define svm_asm2(insn, op1, op2, clobber...) \
	30	do { \
	31	asm_volatile_goto("1: " __stringify(insn) " %1, %0\n\t" \
	32	_ASM_EXTABLE(1b, %l[fault]) \
	33	:: op1, op2 : clobber : fault); \
	34	return; \
	35	fault: \
	36	kvm_spurious_fault(); \
	37	} while (0)
	38
	39	static inline void clgi(void)
	40	{
	41	svm_asm(clgi);
	42	}
	43
	44	static inline void stgi(void)
	45	{
	46	svm_asm(stgi);
	47	}
	48
	49	static inline void invlpga(unsigned long addr, u32 asid)
	50	{
	51	svm_asm2(invlpga, "c"(asid), "a"(addr));
	52	}
	53
	54	/*
	55	* Despite being a physical address, the portion of rAX that is consumed by
	56	* VMSAVE, VMLOAD, etc... is still controlled by the effective address size,
	57	* hence 'unsigned long' instead of 'hpa_t'.
	58	*/
	59	static inline void vmsave(unsigned long pa)
	60	{
	61	svm_asm1(vmsave, "a" (pa), "memory");
	62	}
	63
	64	static inline void vmload(unsigned long pa)
	65	{
	66	svm_asm1(vmload, "a" (pa), "memory");
	67	}
	68
	69	#endif /* __KVM_X86_SVM_OPS_H */

arch/x86/kvm/trace.h+38−2

@@ -92,6 +92,42 @@ TRACE_EVENT(kvm_hv_hypercall, · 91 unmodified lines
92	__entry->outgpa)	92	__entry->outgpa)
93	);	93	);
94		94
		95	/*
		96	* Tracepoint for Xen hypercall.
		97	*/
		98	TRACE_EVENT(kvm_xen_hypercall,
		99	TP_PROTO(unsigned long nr, unsigned long a0, unsigned long a1,
		100	unsigned long a2, unsigned long a3, unsigned long a4,
		101	unsigned long a5),
		102	TP_ARGS(nr, a0, a1, a2, a3, a4, a5),
		103
		104	TP_STRUCT__entry(
		105	__field(unsigned long, nr)
		106	__field(unsigned long, a0)
		107	__field(unsigned long, a1)
		108	__field(unsigned long, a2)
		109	__field(unsigned long, a3)
		110	__field(unsigned long, a4)
		111	__field(unsigned long, a5)
		112	),
		113
		114	TP_fast_assign(
		115	__entry->nr = nr;
		116	__entry->a0 = a0;
		117	__entry->a1 = a1;
		118	__entry->a2 = a2;
		119	__entry->a3 = a3;
		120	__entry->a4 = a4;
		121	__entry->a4 = a5;
		122	),
		123
		124	TP_printk("nr 0x%lx a0 0x%lx a1 0x%lx a2 0x%lx a3 0x%lx a4 0x%lx a5 %lx",
		125	__entry->nr, __entry->a0, __entry->a1, __entry->a2,
		126	__entry->a3, __entry->a4, __entry->a5)
		127	);
		128
		129
		130
95	/*	131	/*
96	* Tracepoint for PIO.	132	* Tracepoint for PIO.
97	*/	133	*/
@@ -256,7 +292,7 @@ TRACE_EVENT(name, \ · 158 unmodified lines
256	__entry->guest_rip = kvm_rip_read(vcpu); \	292	__entry->guest_rip = kvm_rip_read(vcpu); \
257	__entry->isa = isa; \	293	__entry->isa = isa; \
258	__entry->vcpu_id = vcpu->vcpu_id; \	294	__entry->vcpu_id = vcpu->vcpu_id; \
259	kvm_x86_ops.get_exit_info(vcpu, &__entry->info1, \	295	static_call(kvm_x86_get_exit_info)(vcpu, &__entry->info1, \
260	&__entry->info2, \	296	&__entry->info2, \
261	&__entry->intr_info, \	297	&__entry->intr_info, \
262	&__entry->error_code); \	298	&__entry->error_code); \
@@ -738,7 +774,7 @@ TRACE_EVENT(kvm_emulate_insn, · 475 unmodified lines
738	),	774	),
739		775
740	TP_fast_assign(	776	TP_fast_assign(
741	__entry->csbase = kvm_x86_ops.get_segment_base(vcpu, VCPU_SREG_CS);	777	__entry->csbase = static_call(kvm_x86_get_segment_base)(vcpu, VCPU_SREG_CS);
742	__entry->len = vcpu->arch.emulate_ctxt->fetch.ptr	778	__entry->len = vcpu->arch.emulate_ctxt->fetch.ptr
743	- vcpu->arch.emulate_ctxt->fetch.data;	779	- vcpu->arch.emulate_ctxt->fetch.data;
744	__entry->rip = vcpu->arch.emulate_ctxt->_eip - __entry->len;	780	__entry->rip = vcpu->arch.emulate_ctxt->_eip - __entry->len;

arch/x86/kvm/vmx/capabilities.h+27−1

@@ -19,6 +19,9 @@ extern int __read_mostly pt_mode; · 18 unmodified lines
19	#define PT_MODE_HOST_GUEST 1	19	#define PT_MODE_HOST_GUEST 1
20		20
21	#define PMU_CAP_FW_WRITES (1ULL << 13)	21	#define PMU_CAP_FW_WRITES (1ULL << 13)
		22	#define PMU_CAP_LBR_FMT 0x3f
		23
		24	#define DEBUGCTLMSR_LBR_MASK (DEBUGCTLMSR_LBR \| DEBUGCTLMSR_FREEZE_LBRS_ON_PMI)
22		25
23	struct nested_vmx_msrs {	26	struct nested_vmx_msrs {
24	/*	27	/*
@@ -262,6 +265,12 @@ static inline bool cpu_has_vmx_tsc_scaling(void) · 237 unmodified lines
262	SECONDARY_EXEC_TSC_SCALING;	265	SECONDARY_EXEC_TSC_SCALING;
263	}	266	}
264		267
		268	static inline bool cpu_has_vmx_bus_lock_detection(void)
		269	{
		270	return vmcs_config.cpu_based_2nd_exec_ctrl &
		271	SECONDARY_EXEC_BUS_LOCK_DETECTION;
		272	}
		273
265	static inline bool cpu_has_vmx_apicv(void)	274	static inline bool cpu_has_vmx_apicv(void)
266	{	275	{
267	return cpu_has_vmx_apic_register_virt() &&	276	return cpu_has_vmx_apic_register_virt() &&
@@ -371,11 +380,28 @@ static inline bool vmx_pt_mode_is_host_guest(void) · 103 unmodified lines
371		380
372	static inline u64 vmx_get_perf_capabilities(void)	381	static inline u64 vmx_get_perf_capabilities(void)
373	{	382	{
		383	u64 perf_cap = 0;
		384
		385	if (boot_cpu_has(X86_FEATURE_PDCM))
		386	rdmsrl(MSR_IA32_PERF_CAPABILITIES, perf_cap);
		387
		388	perf_cap &= PMU_CAP_LBR_FMT;
		389
374	/*	390	/*
375	* Since counters are virtualized, KVM would support full	391	* Since counters are virtualized, KVM would support full
376	* width counting unconditionally, even if the host lacks it.	392	* width counting unconditionally, even if the host lacks it.
377	*/	393	*/
378	return PMU_CAP_FW_WRITES;	394	return PMU_CAP_FW_WRITES \| perf_cap;
		395	}
		396
		397	static inline u64 vmx_supported_debugctl(void)
		398	{
		399	u64 debugctl = 0;
		400
		401	if (vmx_get_perf_capabilities() & PMU_CAP_LBR_FMT)
		402	debugctl \|= DEBUGCTLMSR_LBR_MASK;
		403
		404	return debugctl;
379	}	405	}
380		406
381	#endif /* __KVM_X86_VMX_CAPS_H */	407	#endif /* __KVM_X86_VMX_CAPS_H */

arch/x86/kvm/vmx/nested.c+40−66

@@ -12,6 +12,7 @@ · 11 unmodified lines
12	#include "nested.h"	12	#include "nested.h"
13	#include "pmu.h"	13	#include "pmu.h"
14	#include "trace.h"	14	#include "trace.h"
		15	#include "vmx.h"
15	#include "x86.h"	16	#include "x86.h"
16		17
17	static bool __read_mostly enable_shadow_vmcs = 1;	18	static bool __read_mostly enable_shadow_vmcs = 1;
@@ -411,8 +412,8 @@ static int nested_vmx_check_exception(struct kvm_vcpu vcpu, unsigned long exit · 393 unmodified lines
411	if (nr == DB_VECTOR) {	412	if (nr == DB_VECTOR) {
412	if (!has_payload) {	413	if (!has_payload) {
413	payload = vcpu->arch.dr6;	414	payload = vcpu->arch.dr6;
414	payload &= ~(DR6_FIXED_1 \| DR6_BT);	415	payload &= ~DR6_BT;
415	payload ^= DR6_RTM;	416	payload ^= DR6_ACTIVE_LOW;
416	}	417	}
417	*exit_qual = payload;	418	*exit_qual = payload;
418	} else	419	} else
@@ -744,8 +745,7 @@ static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, · 325 unmodified lines
744	(CC(!nested_cpu_has_vid(vmcs12)) \|\|	745	(CC(!nested_cpu_has_vid(vmcs12)) \|\|
745	CC(!nested_exit_intr_ack_set(vcpu)) \|\|	746	CC(!nested_exit_intr_ack_set(vcpu)) \|\|
746	CC((vmcs12->posted_intr_nv & 0xff00)) \|\|	747	CC((vmcs12->posted_intr_nv & 0xff00)) \|\|
747	CC((vmcs12->posted_intr_desc_addr & 0x3f)) \|\|	748	CC(!kvm_vcpu_is_legal_aligned_gpa(vcpu, vmcs12->posted_intr_desc_addr, 64))))
748	CC((vmcs12->posted_intr_desc_addr >> cpuid_maxphyaddr(vcpu)))))
749	return -EINVAL;	749	return -EINVAL;
750		750
751	/* tpr shadow is needed by all apicv features. */	751	/* tpr shadow is needed by all apicv features. */
@@ -758,13 +758,11 @@ static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, · 6 unmodified lines
758	static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,	758	static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
759	u32 count, u64 addr)	759	u32 count, u64 addr)
760	{	760	{
761	int maxphyaddr;
762
763	if (count == 0)	761	if (count == 0)
764	return 0;	762	return 0;
765	maxphyaddr = cpuid_maxphyaddr(vcpu);	763
766	if (!IS_ALIGNED(addr, 16) \|\| addr >> maxphyaddr \|\|	764	if (!kvm_vcpu_is_legal_aligned_gpa(vcpu, addr, 16) \|\|
767	(addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr)	765	!kvm_vcpu_is_legal_gpa(vcpu, (addr + count * sizeof(struct vmx_msr_entry) - 1)))
768	return -EINVAL;	766	return -EINVAL;
769		767
770	return 0;	768	return 0;
@@ -1062,14 +1060,6 @@ static void prepare_vmx_msr_autostore_list(struct kvm_vcpu *vcpu, · 291 unmodified lines
1062	}	1060	}
1063	}	1061	}
1064		1062
1065	static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val)
1066	{
1067	unsigned long invalid_mask;
1068
1069	invalid_mask = (~0ULL) << cpuid_maxphyaddr(vcpu);
1070	return (val & invalid_mask) == 0;
1071	}
1072
1073	/*	1063	/*
1074	* Returns true if the MMU needs to be sync'd on nested VM-Enter/VM-Exit.	1064	* Returns true if the MMU needs to be sync'd on nested VM-Enter/VM-Exit.
1075	* tl;dr: the MMU needs a sync if L0 is using shadow paging and L1 didn't	1065	* tl;dr: the MMU needs a sync if L0 is using shadow paging and L1 didn't
@@ -1121,7 +1111,7 @@ static bool nested_vmx_transition_mmu_sync(struct kvm_vcpu *vcpu) · 45 unmodified lines
1121	static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept,	1111	static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept,
1122	enum vm_entry_failure_code *entry_failure_code)	1112	enum vm_entry_failure_code *entry_failure_code)
1123	{	1113	{
1124	if (CC(!nested_cr3_valid(vcpu, cr3))) {	1114	if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3))) {
1125	*entry_failure_code = ENTRY_FAIL_DEFAULT;	1115	*entry_failure_code = ENTRY_FAIL_DEFAULT;
1126	return -EINVAL;	1116	return -EINVAL;
1127	}	1117	}
@@ -2532,7 +2522,7 @@ static int prepare_vmcs02(struct kvm_vcpu vcpu, struct vmcs12 vmcs12, · 1404 unmodified lines
2532	* bitwise-or of what L1 wants to trap for L2, and what we want to	2522	* bitwise-or of what L1 wants to trap for L2, and what we want to
2533	* trap. Note that CR0.TS also needs updating - we do this later.	2523	* trap. Note that CR0.TS also needs updating - we do this later.
2534	*/	2524	*/
2535	update_exception_bitmap(vcpu);	2525	vmx_update_exception_bitmap(vcpu);
2536	vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask;	2526	vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask;
2537	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);	2527	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
2538		2528
@@ -2635,7 +2625,6 @@ static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12) · 96 unmodified lines
2635	static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp)	2625	static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp)
2636	{	2626	{
2637	struct vcpu_vmx *vmx = to_vmx(vcpu);	2627	struct vcpu_vmx *vmx = to_vmx(vcpu);
2638	int maxphyaddr = cpuid_maxphyaddr(vcpu);
2639		2628
2640	/* Check for memory type validity */	2629	/* Check for memory type validity */
2641	switch (new_eptp & VMX_EPTP_MT_MASK) {	2630	switch (new_eptp & VMX_EPTP_MT_MASK) {
@@ -2666,7 +2655,7 @@ static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp) · 24 unmodified lines
2666	}	2655	}
2667		2656
2668	/* Reserved bits should not be set */	2657	/* Reserved bits should not be set */
2669	if (CC(new_eptp >> maxphyaddr \|\| ((new_eptp >> 7) & 0x1f)))	2658	if (CC(kvm_vcpu_is_illegal_gpa(vcpu, new_eptp) \|\| ((new_eptp >> 7) & 0x1f)))
2670	return false;	2659	return false;
2671		2660
2672	/* AD, if set, should be supported */	2661	/* AD, if set, should be supported */
@@ -2850,7 +2839,7 @@ static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu, · 177 unmodified lines
2850		2839
2851	if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) \|\|	2840	if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) \|\|
2852	CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) \|\|	2841	CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) \|\|
2853	CC(!nested_cr3_valid(vcpu, vmcs12->host_cr3)))	2842	CC(kvm_vcpu_is_illegal_gpa(vcpu, vmcs12->host_cr3)))
2854	return -EINVAL;	2843	return -EINVAL;
2855		2844
2856	if (CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu)) \|\|	2845	if (CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu)) \|\|
@@ -3057,35 +3046,8 @@ static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) · 200 unmodified lines
3057	vmx->loaded_vmcs->host_state.cr4 = cr4;	3046	vmx->loaded_vmcs->host_state.cr4 = cr4;
3058	}	3047	}
3059		3048
3060	asm(	3049	vm_fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs,
3061	"sub $%c[wordsize], %%" _ASM_SP "\n\t" /* temporarily adjust RSP for CALL */	3050	vmx->loaded_vmcs->launched);
3062	"cmp %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t"
3063	"je 1f \n\t"
3064	__ex("vmwrite %%" _ASM_SP ", %[HOST_RSP]") "\n\t"
3065	"mov %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t"
3066	"1: \n\t"
3067	"add $%c[wordsize], %%" _ASM_SP "\n\t" /* un-adjust RSP */
3068
3069	/* Check if vmlaunch or vmresume is needed */
3070	"cmpb $0, %c[launched](%[loaded_vmcs])\n\t"
3071
3072	/*
3073	* VMLAUNCH and VMRESUME clear RFLAGS.{CF,ZF} on VM-Exit, set
3074	* RFLAGS.CF on VM-Fail Invalid and set RFLAGS.ZF on VM-Fail
3075	* Valid. vmx_vmenter() directly "returns" RFLAGS, and so the
3076	* results of VM-Enter is captured via CC_{SET,OUT} to vm_fail.
3077	*/
3078	"call vmx_vmenter\n\t"
3079
3080	CC_SET(be)
3081	: ASM_CALL_CONSTRAINT, CC_OUT(be) (vm_fail)
3082	: [HOST_RSP]"r"((unsigned long)HOST_RSP),
3083	[loaded_vmcs]"r"(vmx->loaded_vmcs),
3084	[launched]"i"(offsetof(struct loaded_vmcs, launched)),
3085	[host_state_rsp]"i"(offsetof(struct loaded_vmcs, host_state.rsp)),
3086	[wordsize]"i"(sizeof(ulong))
3087	: "memory"
3088	);
3089		3051
3090	if (vmx->msr_autoload.host.nr)	3052	if (vmx->msr_autoload.host.nr)
3091	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);	3053	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
@@ -3330,7 +3292,11 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, · 238 unmodified lines
3330	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);	3292	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3331	enum vm_entry_failure_code entry_failure_code;	3293	enum vm_entry_failure_code entry_failure_code;
3332	bool evaluate_pending_interrupts;	3294	bool evaluate_pending_interrupts;
3333	u32 exit_reason, failed_index;	3295	union vmx_exit_reason exit_reason = {
		3296	.basic = EXIT_REASON_INVALID_STATE,
		3297	.failed_vmentry = 1,
		3298	};
		3299	u32 failed_index;
3334		3300
3335	if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))	3301	if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
3336	kvm_vcpu_flush_tlb_current(vcpu);	3302	kvm_vcpu_flush_tlb_current(vcpu);
@@ -3382,7 +3348,7 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, · 45 unmodified lines
3382		3348
3383	if (nested_vmx_check_guest_state(vcpu, vmcs12,	3349	if (nested_vmx_check_guest_state(vcpu, vmcs12,
3384	&entry_failure_code)) {	3350	&entry_failure_code)) {
3385	exit_reason = EXIT_REASON_INVALID_STATE;	3351	exit_reason.basic = EXIT_REASON_INVALID_STATE;
3386	vmcs12->exit_qualification = entry_failure_code;	3352	vmcs12->exit_qualification = entry_failure_code;
3387	goto vmentry_fail_vmexit;	3353	goto vmentry_fail_vmexit;
3388	}	3354	}
@@ -3393,7 +3359,7 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, · 4 unmodified lines
3393	vcpu->arch.tsc_offset += vmcs12->tsc_offset;	3359	vcpu->arch.tsc_offset += vmcs12->tsc_offset;
3394		3360
3395	if (prepare_vmcs02(vcpu, vmcs12, &entry_failure_code)) {	3361	if (prepare_vmcs02(vcpu, vmcs12, &entry_failure_code)) {
3396	exit_reason = EXIT_REASON_INVALID_STATE;	3362	exit_reason.basic = EXIT_REASON_INVALID_STATE;
3397	vmcs12->exit_qualification = entry_failure_code;	3363	vmcs12->exit_qualification = entry_failure_code;
3398	goto vmentry_fail_vmexit_guest_mode;	3364	goto vmentry_fail_vmexit_guest_mode;
3399	}	3365	}
@@ -3403,7 +3369,7 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, · 3 unmodified lines
3403	vmcs12->vm_entry_msr_load_addr,	3369	vmcs12->vm_entry_msr_load_addr,
3404	vmcs12->vm_entry_msr_load_count);	3370	vmcs12->vm_entry_msr_load_count);
3405	if (failed_index) {	3371	if (failed_index) {
3406	exit_reason = EXIT_REASON_MSR_LOAD_FAIL;	3372	exit_reason.basic = EXIT_REASON_MSR_LOAD_FAIL;
3407	vmcs12->exit_qualification = failed_index;	3373	vmcs12->exit_qualification = failed_index;
3408	goto vmentry_fail_vmexit_guest_mode;	3374	goto vmentry_fail_vmexit_guest_mode;
3409	}	3375	}
@@ -3471,7 +3437,7 @@ vmentry_fail_vmexit: · 61 unmodified lines
3471	return NVMX_VMENTRY_VMEXIT;	3437	return NVMX_VMENTRY_VMEXIT;
3472		3438
3473	load_vmcs12_host_state(vcpu, vmcs12);	3439	load_vmcs12_host_state(vcpu, vmcs12);
3474	vmcs12->vm_exit_reason = exit_reason \| VMX_EXIT_REASONS_FAILED_VMENTRY;	3440	vmcs12->vm_exit_reason = exit_reason.full;
3475	if (enable_shadow_vmcs \|\| vmx->nested.hv_evmcs)	3441	if (enable_shadow_vmcs \|\| vmx->nested.hv_evmcs)
3476	vmx->nested.need_vmcs12_to_shadow_sync = true;	3442	vmx->nested.need_vmcs12_to_shadow_sync = true;
3477	return NVMX_VMENTRY_VMEXIT;	3443	return NVMX_VMENTRY_VMEXIT;
@@ -5559,7 +5525,12 @@ static int handle_vmfunc(struct kvm_vcpu *vcpu) · 2081 unmodified lines
5559	return kvm_skip_emulated_instruction(vcpu);	5525	return kvm_skip_emulated_instruction(vcpu);
5560		5526
5561	fail:	5527	fail:
5562	nested_vmx_vmexit(vcpu, vmx->exit_reason,	5528	/*
		5529	* This is effectively a reflected VM-Exit, as opposed to a synthesized
		5530	* nested VM-Exit. Pass the original exit reason, i.e. don't hardcode
		5531	* EXIT_REASON_VMFUNC as the exit reason.
		5532	*/
		5533	nested_vmx_vmexit(vcpu, vmx->exit_reason.full,
5563	vmx_get_intr_info(vcpu),	5534	vmx_get_intr_info(vcpu),
5564	vmx_get_exit_qual(vcpu));	5535	vmx_get_exit_qual(vcpu));
5565	return 1;	5536	return 1;
@@ -5627,7 +5598,8 @@ static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, · 61 unmodified lines
5627	* MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps.	5598	* MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps.
5628	*/	5599	*/
5629	static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,	5600	static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
5630	struct vmcs12 *vmcs12, u32 exit_reason)	5601	struct vmcs12 *vmcs12,
		5602	union vmx_exit_reason exit_reason)
5631	{	5603	{
5632	u32 msr_index = kvm_rcx_read(vcpu);	5604	u32 msr_index = kvm_rcx_read(vcpu);
5633	gpa_t bitmap;	5605	gpa_t bitmap;
@@ -5641,7 +5613,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, · 7 unmodified lines
5641	* First we need to figure out which of the four to use:	5613	* First we need to figure out which of the four to use:
5642	*/	5614	*/
5643	bitmap = vmcs12->msr_bitmap;	5615	bitmap = vmcs12->msr_bitmap;
5644	if (exit_reason == EXIT_REASON_MSR_WRITE)	5616	if (exit_reason.basic == EXIT_REASON_MSR_WRITE)
5645	bitmap += 2048;	5617	bitmap += 2048;
5646	if (msr_index >= 0xc0000000) {	5618	if (msr_index >= 0xc0000000) {
5647	msr_index -= 0xc0000000;	5619	msr_index -= 0xc0000000;
@@ -5778,11 +5750,12 @@ static bool nested_vmx_exit_handled_mtf(struct vmcs12 *vmcs12) · 130 unmodified lines
5778	* Return true if L0 wants to handle an exit from L2 regardless of whether or not	5750	* Return true if L0 wants to handle an exit from L2 regardless of whether or not
5779	* L1 wants the exit. Only call this when in is_guest_mode (L2).	5751	* L1 wants the exit. Only call this when in is_guest_mode (L2).
5780	*/	5752	*/
5781	static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason)	5753	static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu,
		5754	union vmx_exit_reason exit_reason)
5782	{	5755	{
5783	u32 intr_info;	5756	u32 intr_info;
5784		5757
5785	switch ((u16)exit_reason) {	5758	switch ((u16)exit_reason.basic) {
5786	case EXIT_REASON_EXCEPTION_NMI:	5759	case EXIT_REASON_EXCEPTION_NMI:
5787	intr_info = vmx_get_intr_info(vcpu);	5760	intr_info = vmx_get_intr_info(vcpu);
5788	if (is_nmi(intr_info))	5761	if (is_nmi(intr_info))
@@ -5838,12 +5811,13 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason) · 49 unmodified lines
5838	* Return 1 if L1 wants to intercept an exit from L2. Only call this when in	5811	* Return 1 if L1 wants to intercept an exit from L2. Only call this when in
5839	* is_guest_mode (L2).	5812	* is_guest_mode (L2).
5840	*/	5813	*/
5841	static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason)	5814	static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu,
		5815	union vmx_exit_reason exit_reason)
5842	{	5816	{
5843	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);	5817	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
5844	u32 intr_info;	5818	u32 intr_info;
5845		5819
5846	switch ((u16)exit_reason) {	5820	switch ((u16)exit_reason.basic) {
5847	case EXIT_REASON_EXCEPTION_NMI:	5821	case EXIT_REASON_EXCEPTION_NMI:
5848	intr_info = vmx_get_intr_info(vcpu);	5822	intr_info = vmx_get_intr_info(vcpu);
5849	if (is_nmi(intr_info))	5823	if (is_nmi(intr_info))
@@ -5962,7 +5936,7 @@ static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason) · 112 unmodified lines
5962	bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu)	5936	bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu)
5963	{	5937	{
5964	struct vcpu_vmx *vmx = to_vmx(vcpu);	5938	struct vcpu_vmx *vmx = to_vmx(vcpu);
5965	u32 exit_reason = vmx->exit_reason;	5939	union vmx_exit_reason exit_reason = vmx->exit_reason;
5966	unsigned long exit_qual;	5940	unsigned long exit_qual;
5967	u32 exit_intr_info;	5941	u32 exit_intr_info;
5968		5942
@@ -5981,7 +5955,7 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) · 12 unmodified lines
5981	goto reflect_vmexit;	5955	goto reflect_vmexit;
5982	}	5956	}
5983		5957
5984	trace_kvm_nested_vmexit(exit_reason, vcpu, KVM_ISA_VMX);	5958	trace_kvm_nested_vmexit(exit_reason.full, vcpu, KVM_ISA_VMX);
5985		5959
5986	/* If L0 (KVM) wants the exit, it trumps L1's desires. */	5960	/* If L0 (KVM) wants the exit, it trumps L1's desires. */
5987	if (nested_vmx_l0_wants_exit(vcpu, exit_reason))	5961	if (nested_vmx_l0_wants_exit(vcpu, exit_reason))
@@ -6007,7 +5981,7 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) · 19 unmodified lines
6007	exit_qual = vmx_get_exit_qual(vcpu);	5981	exit_qual = vmx_get_exit_qual(vcpu);
6008		5982
6009	reflect_vmexit:	5983	reflect_vmexit:
6010	nested_vmx_vmexit(vcpu, exit_reason, exit_intr_info, exit_qual);	5984	nested_vmx_vmexit(vcpu, exit_reason.full, exit_intr_info, exit_qual);
6011	return true;	5985	return true;
6012	}	5986	}
6013		5987

arch/x86/kvm/vmx/pmu_intel.c+285−9

@@ -152,12 +152,17 @@ static struct kvm_pmc intel_rdpmc_ecx_to_pmc(struct kvm_vcpu vcpu, · 151 unmodified lines
152	return &counters[array_index_nospec(idx, num_counters)];	152	return &counters[array_index_nospec(idx, num_counters)];
153	}	153	}
154		154
155	static inline bool fw_writes_is_enabled(struct kvm_vcpu *vcpu)	155	static inline u64 vcpu_get_perf_capabilities(struct kvm_vcpu *vcpu)
156	{	156	{
157	if (!guest_cpuid_has(vcpu, X86_FEATURE_PDCM))	157	if (!guest_cpuid_has(vcpu, X86_FEATURE_PDCM))
158	return false;	158	return 0;
159		159
160	return vcpu->arch.perf_capabilities & PMU_CAP_FW_WRITES;	160	return vcpu->arch.perf_capabilities;
		161	}
		162
		163	static inline bool fw_writes_is_enabled(struct kvm_vcpu *vcpu)
		164	{
		165	return (vcpu_get_perf_capabilities(vcpu) & PMU_CAP_FW_WRITES) != 0;
161	}	166	}
162		167
163	static inline struct kvm_pmc get_fw_gp_pmc(struct kvm_pmu pmu, u32 msr)	168	static inline struct kvm_pmc get_fw_gp_pmc(struct kvm_pmu pmu, u32 msr)
@@ -168,6 +173,41 @@ static inline struct kvm_pmc get_fw_gp_pmc(struct kvm_pmu pmu, u32 msr) · 4 unmodified lines
168	return get_gp_pmc(pmu, msr, MSR_IA32_PMC0);	173	return get_gp_pmc(pmu, msr, MSR_IA32_PMC0);
169	}	174	}
170		175
		176	bool intel_pmu_lbr_is_compatible(struct kvm_vcpu *vcpu)
		177	{
		178	/*
		179	* As a first step, a guest could only enable LBR feature if its
		180	* cpu model is the same as the host because the LBR registers
		181	* would be pass-through to the guest and they're model specific.
		182	*/
		183	return boot_cpu_data.x86_model == guest_cpuid_model(vcpu);
		184	}
		185
		186	bool intel_pmu_lbr_is_enabled(struct kvm_vcpu *vcpu)
		187	{
		188	struct x86_pmu_lbr *lbr = vcpu_to_lbr_records(vcpu);
		189
		190	return lbr->nr && (vcpu_get_perf_capabilities(vcpu) & PMU_CAP_LBR_FMT);
		191	}
		192
		193	static bool intel_pmu_is_valid_lbr_msr(struct kvm_vcpu *vcpu, u32 index)
		194	{
		195	struct x86_pmu_lbr *records = vcpu_to_lbr_records(vcpu);
		196	bool ret = false;
		197
		198	if (!intel_pmu_lbr_is_enabled(vcpu))
		199	return ret;
		200
		201	ret = (index == MSR_LBR_SELECT) \|\| (index == MSR_LBR_TOS) \|\|
		202	(index >= records->from && index < records->from + records->nr) \|\|
		203	(index >= records->to && index < records->to + records->nr);
		204
		205	if (!ret && records->info)
		206	ret = (index >= records->info && index < records->info + records->nr);
		207
		208	return ret;
		209	}
		210
171	static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)	211	static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
172	{	212	{
173	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);	213	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
@@ -183,7 +223,8 @@ static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) · 9 unmodified lines
183	default:	223	default:
184	ret = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0) \|\|	224	ret = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0) \|\|
185	get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0) \|\|	225	get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0) \|\|
186	get_fixed_pmc(pmu, msr) \|\| get_fw_gp_pmc(pmu, msr);	226	get_fixed_pmc(pmu, msr) \|\| get_fw_gp_pmc(pmu, msr) \|\|
		227	intel_pmu_is_valid_lbr_msr(vcpu, msr);
187	break;	228	break;
188	}	229	}
189		230
@@ -202,6 +243,111 @@ static struct kvm_pmc intel_msr_idx_to_pmc(struct kvm_vcpu vcpu, u32 msr) · 12 unmodified lines
202	return pmc;	243	return pmc;
203	}	244	}
204		245
		246	static inline void intel_pmu_release_guest_lbr_event(struct kvm_vcpu *vcpu)
		247	{
		248	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
		249
		250	if (lbr_desc->event) {
		251	perf_event_release_kernel(lbr_desc->event);
		252	lbr_desc->event = NULL;
		253	vcpu_to_pmu(vcpu)->event_count--;
		254	}
		255	}
		256
		257	int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu)
		258	{
		259	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
		260	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
		261	struct perf_event *event;
		262
		263	/*
		264	* The perf_event_attr is constructed in the minimum efficient way:
		265	* - set 'pinned = true' to make it task pinned so that if another
		266	* cpu pinned event reclaims LBR, the event->oncpu will be set to -1;
		267	* - set '.exclude_host = true' to record guest branches behavior;
		268	*
		269	* - set '.config = INTEL_FIXED_VLBR_EVENT' to indicates host perf
		270	* schedule the event without a real HW counter but a fake one;
		271	* check is_guest_lbr_event() and __intel_get_event_constraints();
		272	*
		273	* - set 'sample_type = PERF_SAMPLE_BRANCH_STACK' and
		274	* 'branch_sample_type = PERF_SAMPLE_BRANCH_CALL_STACK \|
		275	* PERF_SAMPLE_BRANCH_USER' to configure it as a LBR callstack
		276	* event, which helps KVM to save/restore guest LBR records
		277	* during host context switches and reduces quite a lot overhead,
		278	* check branch_user_callstack() and intel_pmu_lbr_sched_task();
		279	*/
		280	struct perf_event_attr attr = {
		281	.type = PERF_TYPE_RAW,
		282	.size = sizeof(attr),
		283	.config = INTEL_FIXED_VLBR_EVENT,
		284	.sample_type = PERF_SAMPLE_BRANCH_STACK,
		285	.pinned = true,
		286	.exclude_host = true,
		287	.branch_sample_type = PERF_SAMPLE_BRANCH_CALL_STACK \|
		288	PERF_SAMPLE_BRANCH_USER,
		289	};
		290
		291	if (unlikely(lbr_desc->event)) {
		292	__set_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use);
		293	return 0;
		294	}
		295
		296	event = perf_event_create_kernel_counter(&attr, -1,
		297	current, NULL, NULL);
		298	if (IS_ERR(event)) {
		299	pr_debug_ratelimited("%s: failed %ld\n",
		300	__func__, PTR_ERR(event));
		301	return -ENOENT;
		302	}
		303	lbr_desc->event = event;
		304	pmu->event_count++;
		305	__set_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use);
		306	return 0;
		307	}
		308
		309	/*
		310	* It's safe to access LBR msrs from guest when they have not
		311	* been passthrough since the host would help restore or reset
		312	* the LBR msrs records when the guest LBR event is scheduled in.
		313	*/
		314	static bool intel_pmu_handle_lbr_msrs_access(struct kvm_vcpu *vcpu,
		315	struct msr_data *msr_info, bool read)
		316	{
		317	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
		318	u32 index = msr_info->index;
		319
		320	if (!intel_pmu_is_valid_lbr_msr(vcpu, index))
		321	return false;
		322
		323	if (!lbr_desc->event && !intel_pmu_create_guest_lbr_event(vcpu))
		324	goto dummy;
		325
		326	/*
		327	* Disable irq to ensure the LBR feature doesn't get reclaimed by the
		328	* host at the time the value is read from the msr, and this avoids the
		329	* host LBR value to be leaked to the guest. If LBR has been reclaimed,
		330	* return 0 on guest reads.
		331	*/
		332	local_irq_disable();
		333	if (lbr_desc->event->state == PERF_EVENT_STATE_ACTIVE) {
		334	if (read)
		335	rdmsrl(index, msr_info->data);
		336	else
		337	wrmsrl(index, msr_info->data);
		338	__set_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use);
		339	local_irq_enable();
		340	return true;
		341	}
		342	clear_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use);
		343	local_irq_enable();
		344
		345	dummy:
		346	if (read)
		347	msr_info->data = 0;
		348	return true;
		349	}
		350
205	static int intel_pmu_get_msr(struct kvm_vcpu vcpu, struct msr_data msr_info)	351	static int intel_pmu_get_msr(struct kvm_vcpu vcpu, struct msr_data msr_info)
206	{	352	{
207	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);	353	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
@@ -236,7 +382,8 @@ static int intel_pmu_get_msr(struct kvm_vcpu vcpu, struct msr_data msr_info) · 28 unmodified lines
236	} else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) {	382	} else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) {
237	msr_info->data = pmc->eventsel;	383	msr_info->data = pmc->eventsel;
238	return 0;	384	return 0;
239	}	385	} else if (intel_pmu_handle_lbr_msrs_access(vcpu, msr_info, true))
		386	return 0;
240	}	387	}
241		388
242	return 1;	389	return 1;
@@ -307,7 +454,8 @@ static int intel_pmu_set_msr(struct kvm_vcpu vcpu, struct msr_data msr_info) · 64 unmodified lines
307	reprogram_gp_counter(pmc, data);	454	reprogram_gp_counter(pmc, data);
308	return 0;	455	return 0;
309	}	456	}
310	}	457	} else if (intel_pmu_handle_lbr_msrs_access(vcpu, msr_info, false))
		458	return 0;
311	}	459	}
312		460
313	return 1;	461	return 1;
@@ -316,6 +464,8 @@ static int intel_pmu_set_msr(struct kvm_vcpu vcpu, struct msr_data msr_info) · 2 unmodified lines
316	static void intel_pmu_refresh(struct kvm_vcpu *vcpu)	464	static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
317	{	465	{
318	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);	466	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
		467	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
		468
319	struct x86_pmu_capability x86_pmu;	469	struct x86_pmu_capability x86_pmu;
320	struct kvm_cpuid_entry2 *entry;	470	struct kvm_cpuid_entry2 *entry;
321	union cpuid10_eax eax;	471	union cpuid10_eax eax;
@@ -327,7 +477,6 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) · 5 unmodified lines
327	pmu->counter_bitmask[KVM_PMC_FIXED] = 0;	477	pmu->counter_bitmask[KVM_PMC_FIXED] = 0;
328	pmu->version = 0;	478	pmu->version = 0;
329	pmu->reserved_bits = 0xffffffff00200000ull;	479	pmu->reserved_bits = 0xffffffff00200000ull;
330	vcpu->arch.perf_capabilities = 0;
331		480
332	entry = kvm_find_cpuid_entry(vcpu, 0xa, 0);	481	entry = kvm_find_cpuid_entry(vcpu, 0xa, 0);
333	if (!entry)	482	if (!entry)
@@ -340,8 +489,6 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) · 6 unmodified lines
340	return;	489	return;
341		490
342	perf_get_x86_pmu_capability(&x86_pmu);	491	perf_get_x86_pmu_capability(&x86_pmu);
343	if (guest_cpuid_has(vcpu, X86_FEATURE_PDCM))
344	vcpu->arch.perf_capabilities = vmx_get_perf_capabilities();
345		492
346	pmu->nr_arch_gp_counters = min_t(int, eax.split.num_counters,	493	pmu->nr_arch_gp_counters = min_t(int, eax.split.num_counters,
347	x86_pmu.num_counters_gp);	494	x86_pmu.num_counters_gp);
@@ -385,12 +532,21 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) · 37 unmodified lines
385	INTEL_PMC_MAX_GENERIC, pmu->nr_arch_fixed_counters);	532	INTEL_PMC_MAX_GENERIC, pmu->nr_arch_fixed_counters);
386		533
387	nested_vmx_pmu_entry_exit_ctls_update(vcpu);	534	nested_vmx_pmu_entry_exit_ctls_update(vcpu);
		535
		536	if (intel_pmu_lbr_is_compatible(vcpu))
		537	x86_perf_get_lbr(&lbr_desc->records);
		538	else
		539	lbr_desc->records.nr = 0;
		540
		541	if (lbr_desc->records.nr)
		542	bitmap_set(pmu->all_valid_pmc_idx, INTEL_PMC_IDX_FIXED_VLBR, 1);
388	}	543	}
389		544
390	static void intel_pmu_init(struct kvm_vcpu *vcpu)	545	static void intel_pmu_init(struct kvm_vcpu *vcpu)
391	{	546	{
392	int i;	547	int i;
393	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);	548	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
		549	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
394		550
395	for (i = 0; i < INTEL_PMC_MAX_GENERIC; i++) {	551	for (i = 0; i < INTEL_PMC_MAX_GENERIC; i++) {
396	pmu->gp_counters[i].type = KVM_PMC_GP;	552	pmu->gp_counters[i].type = KVM_PMC_GP;
@@ -405,6 +561,11 @@ static void intel_pmu_init(struct kvm_vcpu *vcpu) · 8 unmodified lines
405	pmu->fixed_counters[i].idx = i + INTEL_PMC_IDX_FIXED;	561	pmu->fixed_counters[i].idx = i + INTEL_PMC_IDX_FIXED;
406	pmu->fixed_counters[i].current_config = 0;	562	pmu->fixed_counters[i].current_config = 0;
407	}	563	}
		564
		565	vcpu->arch.perf_capabilities = vmx_get_perf_capabilities();
		566	lbr_desc->records.nr = 0;
		567	lbr_desc->event = NULL;
		568	lbr_desc->msr_passthrough = false;
408	}	569	}
409		570
410	static void intel_pmu_reset(struct kvm_vcpu *vcpu)	571	static void intel_pmu_reset(struct kvm_vcpu *vcpu)
@@ -429,6 +590,119 @@ static void intel_pmu_reset(struct kvm_vcpu *vcpu) · 18 unmodified lines
429		590
430	pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status =	591	pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status =
431	pmu->global_ovf_ctrl = 0;	592	pmu->global_ovf_ctrl = 0;
		593
		594	intel_pmu_release_guest_lbr_event(vcpu);
		595	}
		596
		597	/*
		598	* Emulate LBR_On_PMI behavior for 1 < pmu.version < 4.
		599	*
		600	* If Freeze_LBR_On_PMI = 1, the LBR is frozen on PMI and
		601	* the KVM emulates to clear the LBR bit (bit 0) in IA32_DEBUGCTL.
		602	*
		603	* Guest needs to re-enable LBR to resume branches recording.
		604	*/
		605	static void intel_pmu_legacy_freezing_lbrs_on_pmi(struct kvm_vcpu *vcpu)
		606	{
		607	u64 data = vmcs_read64(GUEST_IA32_DEBUGCTL);
		608
		609	if (data & DEBUGCTLMSR_FREEZE_LBRS_ON_PMI) {
		610	data &= ~DEBUGCTLMSR_LBR;
		611	vmcs_write64(GUEST_IA32_DEBUGCTL, data);
		612	}
		613	}
		614
		615	static void intel_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
		616	{
		617	u8 version = vcpu_to_pmu(vcpu)->version;
		618
		619	if (!intel_pmu_lbr_is_enabled(vcpu))
		620	return;
		621
		622	if (version > 1 && version < 4)
		623	intel_pmu_legacy_freezing_lbrs_on_pmi(vcpu);
		624	}
		625
		626	static void vmx_update_intercept_for_lbr_msrs(struct kvm_vcpu *vcpu, bool set)
		627	{
		628	struct x86_pmu_lbr *lbr = vcpu_to_lbr_records(vcpu);
		629	int i;
		630
		631	for (i = 0; i < lbr->nr; i++) {
		632	vmx_set_intercept_for_msr(vcpu, lbr->from + i, MSR_TYPE_RW, set);
		633	vmx_set_intercept_for_msr(vcpu, lbr->to + i, MSR_TYPE_RW, set);
		634	if (lbr->info)
		635	vmx_set_intercept_for_msr(vcpu, lbr->info + i, MSR_TYPE_RW, set);
		636	}
		637
		638	vmx_set_intercept_for_msr(vcpu, MSR_LBR_SELECT, MSR_TYPE_RW, set);
		639	vmx_set_intercept_for_msr(vcpu, MSR_LBR_TOS, MSR_TYPE_RW, set);
		640	}
		641
		642	static inline void vmx_disable_lbr_msrs_passthrough(struct kvm_vcpu *vcpu)
		643	{
		644	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
		645
		646	if (!lbr_desc->msr_passthrough)
		647	return;
		648
		649	vmx_update_intercept_for_lbr_msrs(vcpu, true);
		650	lbr_desc->msr_passthrough = false;
		651	}
		652
		653	static inline void vmx_enable_lbr_msrs_passthrough(struct kvm_vcpu *vcpu)
		654	{
		655	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
		656
		657	if (lbr_desc->msr_passthrough)
		658	return;
		659
		660	vmx_update_intercept_for_lbr_msrs(vcpu, false);
		661	lbr_desc->msr_passthrough = true;
		662	}
		663
		664	/*
		665	* Higher priority host perf events (e.g. cpu pinned) could reclaim the
		666	* pmu resources (e.g. LBR) that were assigned to the guest. This is
		667	* usually done via ipi calls (more details in perf_install_in_context).
		668	*
		669	* Before entering the non-root mode (with irq disabled here), double
		670	* confirm that the pmu features enabled to the guest are not reclaimed
		671	* by higher priority host events. Otherwise, disallow vcpu's access to
		672	* the reclaimed features.
		673	*/
		674	void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu)
		675	{
		676	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
		677	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
		678
		679	if (!lbr_desc->event) {
		680	vmx_disable_lbr_msrs_passthrough(vcpu);
		681	if (vmcs_read64(GUEST_IA32_DEBUGCTL) & DEBUGCTLMSR_LBR)
		682	goto warn;
		683	if (test_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use))
		684	goto warn;
		685	return;
		686	}
		687
		688	if (lbr_desc->event->state < PERF_EVENT_STATE_ACTIVE) {
		689	vmx_disable_lbr_msrs_passthrough(vcpu);
		690	__clear_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use);
		691	goto warn;
		692	} else
		693	vmx_enable_lbr_msrs_passthrough(vcpu);
		694
		695	return;
		696
		697	warn:
		698	pr_warn_ratelimited("kvm: vcpu-%d: fail to passthrough LBR.\n",
		699	vcpu->vcpu_id);
		700	}
		701
		702	static void intel_pmu_cleanup(struct kvm_vcpu *vcpu)
		703	{
		704	if (!(vmcs_read64(GUEST_IA32_DEBUGCTL) & DEBUGCTLMSR_LBR))
		705	intel_pmu_release_guest_lbr_event(vcpu);
432	}	706	}
433		707
434	struct kvm_pmu_ops intel_pmu_ops = {	708	struct kvm_pmu_ops intel_pmu_ops = {
@@ -445,4 +719,6 @@ struct kvm_pmu_ops intel_pmu_ops = { · 10 unmodified lines
445	.refresh = intel_pmu_refresh,	719	.refresh = intel_pmu_refresh,
446	.init = intel_pmu_init,	720	.init = intel_pmu_init,
447	.reset = intel_pmu_reset,	721	.reset = intel_pmu_reset,
		722	.deliver_pmi = intel_pmu_deliver_pmi,
		723	.cleanup = intel_pmu_cleanup,
448	};	724	};

arch/x86/kvm/vmx/posted_intr.c+3−3

@@ -54,7 +54,7 @@ void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) · 53 unmodified lines
54		54
55	dest = cpu_physical_id(cpu);	55	dest = cpu_physical_id(cpu);
56		56
57	if (x2apic_enabled())	57	if (x2apic_mode)
58	new.ndst = dest;	58	new.ndst = dest;
59	else	59	else
60	new.ndst = (dest << 8) & 0xFF00;	60	new.ndst = (dest << 8) & 0xFF00;
@@ -104,7 +104,7 @@ static void __pi_post_block(struct kvm_vcpu *vcpu) · 43 unmodified lines
104		104
105	dest = cpu_physical_id(vcpu->cpu);	105	dest = cpu_physical_id(vcpu->cpu);
106		106
107	if (x2apic_enabled())	107	if (x2apic_mode)
108	new.ndst = dest;	108	new.ndst = dest;
109	else	109	else
110	new.ndst = (dest << 8) & 0xFF00;	110	new.ndst = (dest << 8) & 0xFF00;
@@ -174,7 +174,7 @@ int pi_pre_block(struct kvm_vcpu *vcpu) · 63 unmodified lines
174	*/	174	*/
175	dest = cpu_physical_id(vcpu->pre_pcpu);	175	dest = cpu_physical_id(vcpu->pre_pcpu);
176		176
177	if (x2apic_enabled())	177	if (x2apic_mode)
178	new.ndst = dest;	178	new.ndst = dest;
179	else	179	else
180	new.ndst = (dest << 8) & 0xFF00;	180	new.ndst = (dest << 8) & 0xFF00;

arch/x86/kvm/vmx/vmenter.S+1−1

@@ -44,7 +44,7 @@ · 43 unmodified lines
44	* they VM-Fail, whereas a successful VM-Enter + VM-Exit will jump	44	* they VM-Fail, whereas a successful VM-Enter + VM-Exit will jump
45	* to vmx_vmexit.	45	* to vmx_vmexit.
46	*/	46	*/
47	SYM_FUNC_START(vmx_vmenter)	47	SYM_FUNC_START_LOCAL(vmx_vmenter)
48	/* EFLAGS.ZF is set if VMCS.LAUNCHED == 0 */	48	/* EFLAGS.ZF is set if VMCS.LAUNCHED == 0 */
49	je 2f	49	je 2f
50		50

arch/x86/kvm/vmx/vmx.c+172−110

@@ -50,6 +50,7 @@ · 49 unmodified lines
50	#include "capabilities.h"	50	#include "capabilities.h"
51	#include "cpuid.h"	51	#include "cpuid.h"
52	#include "evmcs.h"	52	#include "evmcs.h"
		53	#include "hyperv.h"
53	#include "irq.h"	54	#include "irq.h"
54	#include "kvm_cache_regs.h"	55	#include "kvm_cache_regs.h"
55	#include "lapic.h"	56	#include "lapic.h"
@@ -552,7 +553,7 @@ static int hv_enable_direct_tlbflush(struct kvm_vcpu *vcpu) · 496 unmodified lines
552	{	553	{
553	struct hv_enlightened_vmcs *evmcs;	554	struct hv_enlightened_vmcs *evmcs;
554	struct hv_partition_assist_pg **p_hv_pa_pg =	555	struct hv_partition_assist_pg **p_hv_pa_pg =
555	&vcpu->kvm->arch.hyperv.hv_pa_pg;	556	&to_kvm_hv(vcpu->kvm)->hv_pa_pg;
556	/*	557	/*
557	* Synthetic VM-Exit is not enabled in current code and so All	558	* Synthetic VM-Exit is not enabled in current code and so All
558	* evmcs in singe VM shares same assist page.	559	* evmcs in singe VM shares same assist page.
@@ -658,6 +659,14 @@ static bool is_valid_passthrough_msr(u32 msr) · 99 unmodified lines
658	case MSR_IA32_RTIT_CR3_MATCH:	659	case MSR_IA32_RTIT_CR3_MATCH:
659	case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:	660	case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
660	/* PT MSRs. These are handled in pt_update_intercept_for_msr() */	661	/* PT MSRs. These are handled in pt_update_intercept_for_msr() */
		662	case MSR_LBR_SELECT:
		663	case MSR_LBR_TOS:
		664	case MSR_LBR_INFO_0 ... MSR_LBR_INFO_0 + 31:
		665	case MSR_LBR_NHM_FROM ... MSR_LBR_NHM_FROM + 31:
		666	case MSR_LBR_NHM_TO ... MSR_LBR_NHM_TO + 31:
		667	case MSR_LBR_CORE_FROM ... MSR_LBR_CORE_FROM + 8:
		668	case MSR_LBR_CORE_TO ... MSR_LBR_CORE_TO + 8:
		669	/* LBR MSRs. These are handled in vmx_update_intercept_for_lbr_msrs() */
661	return true;	670	return true;
662	}	671	}
663		672
@@ -806,7 +815,7 @@ static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg) · 142 unmodified lines
806	return *p;	815	return *p;
807	}	816	}
808		817
809	void update_exception_bitmap(struct kvm_vcpu *vcpu)	818	void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu)
810	{	819	{
811	u32 eb;	820	u32 eb;
812		821
@@ -1102,7 +1111,7 @@ static inline bool pt_can_write_msr(struct vcpu_vmx *vmx) · 289 unmodified lines
1102	static inline bool pt_output_base_valid(struct kvm_vcpu *vcpu, u64 base)	1111	static inline bool pt_output_base_valid(struct kvm_vcpu *vcpu, u64 base)
1103	{	1112	{
1104	/* The base must be 128-byte aligned and a legal physical address. */	1113	/* The base must be 128-byte aligned and a legal physical address. */
1105	return !kvm_vcpu_is_illegal_gpa(vcpu, base) && !(base & 0x7f);	1114	return kvm_vcpu_is_legal_aligned_gpa(vcpu, base, 128);
1106	}	1115	}
1107		1116
1108	static inline void pt_load_msr(struct pt_ctx *ctx, u32 addr_range)	1117	static inline void pt_load_msr(struct pt_ctx *ctx, u32 addr_range)
@@ -1577,7 +1586,7 @@ static int skip_emulated_instruction(struct kvm_vcpu *vcpu) · 468 unmodified lines
1577	* i.e. we end up advancing IP with some random value.	1586	* i.e. we end up advancing IP with some random value.
1578	*/	1587	*/
1579	if (!static_cpu_has(X86_FEATURE_HYPERVISOR) \|\|	1588	if (!static_cpu_has(X86_FEATURE_HYPERVISOR) \|\|
1580	to_vmx(vcpu)->exit_reason != EXIT_REASON_EPT_MISCONFIG) {	1589	to_vmx(vcpu)->exit_reason.basic != EXIT_REASON_EPT_MISCONFIG) {
1581	orig_rip = kvm_rip_read(vcpu);	1590	orig_rip = kvm_rip_read(vcpu);
1582	rip = orig_rip + vmcs_read32(VM_EXIT_INSTRUCTION_LEN);	1591	rip = orig_rip + vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
1583	#ifdef CONFIG_X86_64	1592	#ifdef CONFIG_X86_64
@@ -1924,6 +1933,9 @@ static int vmx_get_msr(struct kvm_vcpu vcpu, struct msr_data msr_info) · 340 unmodified lines
1924	!guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))	1933	!guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
1925	return 1;	1934	return 1;
1926	goto find_uret_msr;	1935	goto find_uret_msr;
		1936	case MSR_IA32_DEBUGCTLMSR:
		1937	msr_info->data = vmcs_read64(GUEST_IA32_DEBUGCTL);
		1938	break;
1927	default:	1939	default:
1928	find_uret_msr:	1940	find_uret_msr:
1929	msr = vmx_find_uret_msr(vmx, msr_info->index);	1941	msr = vmx_find_uret_msr(vmx, msr_info->index);
@@ -1947,6 +1959,16 @@ static u64 nested_vmx_truncate_sysenter_addr(struct kvm_vcpu *vcpu, · 17 unmodified lines
1947	return (unsigned long)data;	1959	return (unsigned long)data;
1948	}	1960	}
1949		1961
		1962	static u64 vcpu_supported_debugctl(struct kvm_vcpu *vcpu)
		1963	{
		1964	u64 debugctl = vmx_supported_debugctl();
		1965
		1966	if (!intel_pmu_lbr_is_enabled(vcpu))
		1967	debugctl &= ~DEBUGCTLMSR_LBR_MASK;
		1968
		1969	return debugctl;
		1970	}
		1971
1950	/*	1972	/*
1951	* Writes msr value into the appropriate "register".	1973	* Writes msr value into the appropriate "register".
1952	* Returns 0 on success, non-0 otherwise.	1974	* Returns 0 on success, non-0 otherwise.
@@ -1997,14 +2019,29 @@ static int vmx_set_msr(struct kvm_vcpu vcpu, struct msr_data msr_info) · 44 unmodified lines
1997	}	2019	}
1998	vmcs_writel(GUEST_SYSENTER_ESP, data);	2020	vmcs_writel(GUEST_SYSENTER_ESP, data);
1999	break;	2021	break;
2000	case MSR_IA32_DEBUGCTLMSR:	2022	case MSR_IA32_DEBUGCTLMSR: {
		2023	u64 invalid = data & ~vcpu_supported_debugctl(vcpu);
		2024	if (invalid & (DEBUGCTLMSR_BTF\|DEBUGCTLMSR_LBR)) {
		2025	if (report_ignored_msrs)
		2026	vcpu_unimpl(vcpu, "%s: BTF\|LBR in IA32_DEBUGCTLMSR 0x%llx, nop\n",
		2027	__func__, data);
		2028	data &= ~(DEBUGCTLMSR_BTF\|DEBUGCTLMSR_LBR);
		2029	invalid &= ~(DEBUGCTLMSR_BTF\|DEBUGCTLMSR_LBR);
		2030	}
		2031
		2032	if (invalid)
		2033	return 1;
		2034
2001	if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls &	2035	if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls &
2002	VM_EXIT_SAVE_DEBUG_CONTROLS)	2036	VM_EXIT_SAVE_DEBUG_CONTROLS)
2003	get_vmcs12(vcpu)->guest_ia32_debugctl = data;	2037	get_vmcs12(vcpu)->guest_ia32_debugctl = data;
2004		2038
2005	ret = kvm_set_msr_common(vcpu, msr_info);	2039	vmcs_write64(GUEST_IA32_DEBUGCTL, data);
2006	break;	2040	if (intel_pmu_lbr_is_enabled(vcpu) && !to_vmx(vcpu)->lbr_desc.event &&
2007		2041	(data & DEBUGCTLMSR_LBR))
		2042	intel_pmu_create_guest_lbr_event(vcpu);
		2043	return 0;
		2044	}
2008	case MSR_IA32_BNDCFGS:	2045	case MSR_IA32_BNDCFGS:
2009	if (!kvm_mpx_supported() \|\|	2046	if (!kvm_mpx_supported() \|\|
2010	(!msr_info->host_initiated &&	2047	(!msr_info->host_initiated &&
@@ -2196,6 +2233,18 @@ static int vmx_set_msr(struct kvm_vcpu vcpu, struct msr_data msr_info) · 185 unmodified lines
2196	if ((data >> 32) != 0)	2233	if ((data >> 32) != 0)
2197	return 1;	2234	return 1;
2198	goto find_uret_msr;	2235	goto find_uret_msr;
		2236	case MSR_IA32_PERF_CAPABILITIES:
		2237	if (data && !vcpu_to_pmu(vcpu)->version)
		2238	return 1;
		2239	if (data & PMU_CAP_LBR_FMT) {
		2240	if ((data & PMU_CAP_LBR_FMT) !=
		2241	(vmx_get_perf_capabilities() & PMU_CAP_LBR_FMT))
		2242	return 1;
		2243	if (!intel_pmu_lbr_is_compatible(vcpu))
		2244	return 1;
		2245	}
		2246	ret = kvm_set_msr_common(vcpu, msr_info);
		2247	break;
2199		2248
2200	default:	2249	default:
2201	find_uret_msr:	2250	find_uret_msr:
@@ -2265,7 +2314,6 @@ static int kvm_cpu_vmxon(u64 vmxon_pointer) · 63 unmodified lines
2265	u64 msr;	2314	u64 msr;
2266		2315
2267	cr4_set_bits(X86_CR4_VMXE);	2316	cr4_set_bits(X86_CR4_VMXE);
2268	intel_pt_handle_vmx(1);
2269		2317
2270	asm_volatile_goto("1: vmxon %[vmxon_pointer]\n\t"	2318	asm_volatile_goto("1: vmxon %[vmxon_pointer]\n\t"
2271	_ASM_EXTABLE(1b, %l[fault])	2319	_ASM_EXTABLE(1b, %l[fault])
@@ -2276,7 +2324,6 @@ static int kvm_cpu_vmxon(u64 vmxon_pointer) · 4 unmodified lines
2276	fault:	2324	fault:
2277	WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n",	2325	WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n",
2278	rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr);	2326	rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr);
2279	intel_pt_handle_vmx(0);
2280	cr4_clear_bits(X86_CR4_VMXE);	2327	cr4_clear_bits(X86_CR4_VMXE);
2281		2328
2282	return -EFAULT;	2329	return -EFAULT;
@@ -2299,9 +2346,13 @@ static int hardware_enable(void) · 16 unmodified lines
2299	!hv_get_vp_assist_page(cpu))	2346	!hv_get_vp_assist_page(cpu))
2300	return -EFAULT;	2347	return -EFAULT;
2301		2348
		2349	intel_pt_handle_vmx(1);
		2350
2302	r = kvm_cpu_vmxon(phys_addr);	2351	r = kvm_cpu_vmxon(phys_addr);
2303	if (r)	2352	if (r) {
		2353	intel_pt_handle_vmx(0);
2304	return r;	2354	return r;
		2355	}
2305		2356
2306	if (enable_ept)	2357	if (enable_ept)
2307	ept_sync_global();	2358	ept_sync_global();
@@ -2319,22 +2370,14 @@ static void vmclear_local_loaded_vmcss(void) · 11 unmodified lines
2319	__loaded_vmcs_clear(v);	2370	__loaded_vmcs_clear(v);
2320	}	2371	}
2321		2372
2322
2323	/* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
2324	* tricks.
2325	*/
2326	static void kvm_cpu_vmxoff(void)
2327	{
2328	asm volatile (__ex("vmxoff"));
2329
2330	intel_pt_handle_vmx(0);
2331	cr4_clear_bits(X86_CR4_VMXE);
2332	}
2333
2334	static void hardware_disable(void)	2373	static void hardware_disable(void)
2335	{	2374	{
2336	vmclear_local_loaded_vmcss();	2375	vmclear_local_loaded_vmcss();
2337	kvm_cpu_vmxoff();	2376
		2377	if (cpu_vmxoff())
		2378	kvm_spurious_fault();
		2379
		2380	intel_pt_handle_vmx(0);
2338	}	2381	}
2339		2382
2340	/*	2383	/*
@@ -2428,7 +2471,8 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf, · 87 unmodified lines
2428	SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE \|	2471	SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE \|
2429	SECONDARY_EXEC_PT_USE_GPA \|	2472	SECONDARY_EXEC_PT_USE_GPA \|
2430	SECONDARY_EXEC_PT_CONCEAL_VMX \|	2473	SECONDARY_EXEC_PT_CONCEAL_VMX \|
2431	SECONDARY_EXEC_ENABLE_VMFUNC;	2474	SECONDARY_EXEC_ENABLE_VMFUNC \|
		2475	SECONDARY_EXEC_BUS_LOCK_DETECTION;
2432	if (cpu_has_sgx())	2476	if (cpu_has_sgx())
2433	opt2 \|= SECONDARY_EXEC_ENCLS_EXITING;	2477	opt2 \|= SECONDARY_EXEC_ENCLS_EXITING;
2434	if (adjust_vmx_controls(min2, opt2,	2478	if (adjust_vmx_controls(min2, opt2,
@@ -2739,7 +2783,7 @@ static void enter_pmode(struct kvm_vcpu *vcpu) · 304 unmodified lines
2739	vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) \|	2783	vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) \|
2740	(vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));	2784	(vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
2741		2785
2742	update_exception_bitmap(vcpu);	2786	vmx_update_exception_bitmap(vcpu);
2743		2787
2744	fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);	2788	fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
2745	fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);	2789	fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
@@ -2819,7 +2863,7 @@ static void enter_rmode(struct kvm_vcpu *vcpu) · 73 unmodified lines
2819		2863
2820	vmcs_writel(GUEST_RFLAGS, flags);	2864	vmcs_writel(GUEST_RFLAGS, flags);
2821	vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) \| X86_CR4_VME);	2865	vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) \| X86_CR4_VME);
2822	update_exception_bitmap(vcpu);	2866	vmx_update_exception_bitmap(vcpu);
2823		2867
2824	fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);	2868	fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
2825	fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);	2869	fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
@@ -3774,7 +3818,7 @@ static __always_inline void vmx_enable_intercept_for_msr(struct kvm_vcpu *vcpu, · 948 unmodified lines
3774	vmx_set_msr_bitmap_write(msr_bitmap, msr);	3818	vmx_set_msr_bitmap_write(msr_bitmap, msr);
3775	}	3819	}
3776		3820
3777	static __always_inline void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu,	3821	void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu,
3778	u32 msr, int type, bool value)	3822	u32 msr, int type, bool value)
3779	{	3823	{
3780	if (value)	3824	if (value)
@@ -4269,6 +4313,9 @@ static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx) · 488 unmodified lines
4269	vmx_adjust_sec_exec_control(vmx, &exec_control, waitpkg, WAITPKG,	4313	vmx_adjust_sec_exec_control(vmx, &exec_control, waitpkg, WAITPKG,
4270	ENABLE_USR_WAIT_PAUSE, false);	4314	ENABLE_USR_WAIT_PAUSE, false);
4271		4315
		4316	if (!vcpu->kvm->arch.bus_lock_detection_enabled)
		4317	exec_control &= ~SECONDARY_EXEC_BUS_LOCK_DETECTION;
		4318
4272	vmx->secondary_exec_control = exec_control;	4319	vmx->secondary_exec_control = exec_control;
4273	}	4320	}
4274		4321
@@ -4467,23 +4514,23 @@ static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) · 192 unmodified lines
4467	vmx_set_cr4(vcpu, 0);	4514	vmx_set_cr4(vcpu, 0);
4468	vmx_set_efer(vcpu, 0);	4515	vmx_set_efer(vcpu, 0);
4469		4516
4470	update_exception_bitmap(vcpu);	4517	vmx_update_exception_bitmap(vcpu);
4471		4518
4472	vpid_sync_context(vmx->vpid);	4519	vpid_sync_context(vmx->vpid);
4473	if (init_event)	4520	if (init_event)
4474	vmx_clear_hlt(vcpu);	4521	vmx_clear_hlt(vcpu);
4475	}	4522	}
4476		4523
4477	static void enable_irq_window(struct kvm_vcpu *vcpu)	4524	static void vmx_enable_irq_window(struct kvm_vcpu *vcpu)
4478	{	4525	{
4479	exec_controls_setbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING);	4526	exec_controls_setbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING);
4480	}	4527	}
4481		4528
4482	static void enable_nmi_window(struct kvm_vcpu *vcpu)	4529	static void vmx_enable_nmi_window(struct kvm_vcpu *vcpu)
4483	{	4530	{
4484	if (!enable_vnmi \|\|	4531	if (!enable_vnmi \|\|
4485	vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {	4532	vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
4486	enable_irq_window(vcpu);	4533	vmx_enable_irq_window(vcpu);
4487	return;	4534	return;
4488	}	4535	}
4489		4536
@@ -4824,7 +4871,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) · 334 unmodified lines
4824	kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);	4871	kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
4825	return 1;	4872	return 1;
4826	}	4873	}
4827	kvm_run->debug.arch.dr6 = dr6 \| DR6_FIXED_1 \| DR6_RTM;	4874	kvm_run->debug.arch.dr6 = dr6 \| DR6_ACTIVE_LOW;
4828	kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);	4875	kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
4829	fallthrough;	4876	fallthrough;
4830	case BP_VECTOR:	4877	case BP_VECTOR:
@@ -5049,6 +5096,7 @@ static int handle_dr(struct kvm_vcpu *vcpu) · 218 unmodified lines
5049	{	5096	{
5050	unsigned long exit_qualification;	5097	unsigned long exit_qualification;
5051	int dr, dr7, reg;	5098	int dr, dr7, reg;
		5099	int err = 1;
5052		5100
5053	exit_qualification = vmx_get_exit_qual(vcpu);	5101	exit_qualification = vmx_get_exit_qual(vcpu);
5054	dr = exit_qualification & DEBUG_REG_ACCESS_NUM;	5102	dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
@@ -5057,9 +5105,9 @@ static int handle_dr(struct kvm_vcpu *vcpu) · 2 unmodified lines
5057	if (!kvm_require_dr(vcpu, dr))	5105	if (!kvm_require_dr(vcpu, dr))
5058	return 1;	5106	return 1;
5059		5107
5060	/* Do not handle if the CPL > 0, will trigger GP on re-entry */	5108	if (kvm_x86_ops.get_cpl(vcpu) > 0)
5061	if (!kvm_require_cpl(vcpu, 0))	5109	goto out;
5062	return 1;	5110
5063	dr7 = vmcs_readl(GUEST_DR7);	5111	dr7 = vmcs_readl(GUEST_DR7);
5064	if (dr7 & DR7_GD) {	5112	if (dr7 & DR7_GD) {
5065	/*	5113	/*
@@ -5068,7 +5116,7 @@ static int handle_dr(struct kvm_vcpu *vcpu) · 2 unmodified lines
5068	* guest debugging itself.	5116	* guest debugging itself.
5069	*/	5117	*/
5070	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {	5118	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
5071	vcpu->run->debug.arch.dr6 = DR6_BD \| DR6_RTM \| DR6_FIXED_1;	5119	vcpu->run->debug.arch.dr6 = DR6_BD \| DR6_ACTIVE_LOW;
5072	vcpu->run->debug.arch.dr7 = dr7;	5120	vcpu->run->debug.arch.dr7 = dr7;
5073	vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu);	5121	vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu);
5074	vcpu->run->debug.arch.exception = DB_VECTOR;	5122	vcpu->run->debug.arch.exception = DB_VECTOR;
@@ -5096,14 +5144,15 @@ static int handle_dr(struct kvm_vcpu *vcpu) · 21 unmodified lines
5096	if (exit_qualification & TYPE_MOV_FROM_DR) {	5144	if (exit_qualification & TYPE_MOV_FROM_DR) {
5097	unsigned long val;	5145	unsigned long val;
5098		5146
5099	if (kvm_get_dr(vcpu, dr, &val))	5147	kvm_get_dr(vcpu, dr, &val);
5100	return 1;
5101	kvm_register_write(vcpu, reg, val);	5148	kvm_register_write(vcpu, reg, val);
5102	} else	5149	err = 0;
5103	if (kvm_set_dr(vcpu, dr, kvm_register_readl(vcpu, reg)))	5150	} else {
5104	return 1;	5151	err = kvm_set_dr(vcpu, dr, kvm_register_readl(vcpu, reg));
		5152	}
5105		5153
5106	return kvm_skip_emulated_instruction(vcpu);	5154	out:
		5155	return kvm_complete_insn_gp(vcpu, err);
5107	}	5156	}
5108		5157
5109	static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)	5158	static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
@@ -5177,9 +5226,8 @@ static int handle_xsetbv(struct kvm_vcpu *vcpu) · 67 unmodified lines
5177	u64 new_bv = kvm_read_edx_eax(vcpu);	5226	u64 new_bv = kvm_read_edx_eax(vcpu);
5178	u32 index = kvm_rcx_read(vcpu);	5227	u32 index = kvm_rcx_read(vcpu);
5179		5228
5180	if (kvm_set_xcr(vcpu, index, new_bv) == 0)	5229	int err = kvm_set_xcr(vcpu, index, new_bv);
5181	return kvm_skip_emulated_instruction(vcpu);	5230	return kvm_complete_insn_gp(vcpu, err);
5182	return 1;
5183	}	5231	}
5184		5232
5185	static int handle_apic_access(struct kvm_vcpu *vcpu)	5233	static int handle_apic_access(struct kvm_vcpu *vcpu)
@@ -5600,6 +5648,13 @@ static int handle_encls(struct kvm_vcpu *vcpu) · 414 unmodified lines
5600	return 1;	5648	return 1;
5601	}	5649	}
5602		5650
		5651	static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu)
		5652	{
		5653	vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK;
		5654	vcpu->run->flags \|= KVM_RUN_X86_BUS_LOCK;
		5655	return 0;
		5656	}
		5657
5603	/*	5658	/*
5604	* The exit handlers return 1 if the exit was handled fully and guest execution	5659	* The exit handlers return 1 if the exit was handled fully and guest execution
5605	* may resume. Otherwise they set the kvm_run parameter to indicate what needs	5660	* may resume. Otherwise they set the kvm_run parameter to indicate what needs
@@ -5656,6 +5711,7 @@ static int (kvm_vmx_exit_handlers[])(struct kvm_vcpu vcpu) = { · 50 unmodified lines
5656	[EXIT_REASON_VMFUNC] = handle_vmx_instruction,	5711	[EXIT_REASON_VMFUNC] = handle_vmx_instruction,
5657	[EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer,	5712	[EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer,
5658	[EXIT_REASON_ENCLS] = handle_encls,	5713	[EXIT_REASON_ENCLS] = handle_encls,
		5714	[EXIT_REASON_BUS_LOCK] = handle_bus_lock_vmexit,
5659	};	5715	};
5660		5716
5661	static const int kvm_vmx_max_exit_handlers =	5717	static const int kvm_vmx_max_exit_handlers =
@@ -5667,7 +5723,7 @@ static void vmx_get_exit_info(struct kvm_vcpu vcpu, u64 info1, u64 *info2, · 5 unmodified lines
5667	struct vcpu_vmx *vmx = to_vmx(vcpu);	5723	struct vcpu_vmx *vmx = to_vmx(vcpu);
5668		5724
5669	*info1 = vmx_get_exit_qual(vcpu);	5725	*info1 = vmx_get_exit_qual(vcpu);
5670	if (!(vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) {	5726	if (!(vmx->exit_reason.failed_vmentry)) {
5671	*info2 = vmx->idt_vectoring_info;	5727	*info2 = vmx->idt_vectoring_info;
5672	*intr_info = vmx_get_intr_info(vcpu);	5728	*intr_info = vmx_get_intr_info(vcpu);
5673	if (is_exception_with_error_code(*intr_info))	5729	if (is_exception_with_error_code(*intr_info))
@@ -5908,11 +5964,12 @@ void dump_vmcs(void) · 234 unmodified lines
5908	* The guest has exited. See if we can fix it or if we need userspace	5964	* The guest has exited. See if we can fix it or if we need userspace
5909	* assistance.	5965	* assistance.
5910	*/	5966	*/
5911	static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)	5967	static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
5912	{	5968	{
5913	struct vcpu_vmx *vmx = to_vmx(vcpu);	5969	struct vcpu_vmx *vmx = to_vmx(vcpu);
5914	u32 exit_reason = vmx->exit_reason;	5970	union vmx_exit_reason exit_reason = vmx->exit_reason;
5915	u32 vectoring_info = vmx->idt_vectoring_info;	5971	u32 vectoring_info = vmx->idt_vectoring_info;
		5972	u16 exit_handler_index;
5916		5973
5917	/*	5974	/*
5918	* Flush logged GPAs PML buffer, this will make dirty_bitmap more	5975	* Flush logged GPAs PML buffer, this will make dirty_bitmap more
@@ -5954,11 +6011,11 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) · 35 unmodified lines
5954	return 1;	6011	return 1;
5955	}	6012	}
5956		6013
5957	if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) {	6014	if (exit_reason.failed_vmentry) {
5958	dump_vmcs();	6015	dump_vmcs();
5959	vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;	6016	vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
5960	vcpu->run->fail_entry.hardware_entry_failure_reason	6017	vcpu->run->fail_entry.hardware_entry_failure_reason
5961	= exit_reason;	6018	= exit_reason.full;
5962	vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;	6019	vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;
5963	return 0;	6020	return 0;
5964	}	6021	}
@@ -5980,18 +6037,18 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) · 15 unmodified lines
5980	* will cause infinite loop.	6037	* will cause infinite loop.
5981	*/	6038	*/
5982	if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&	6039	if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
5983	(exit_reason != EXIT_REASON_EXCEPTION_NMI &&	6040	(exit_reason.basic != EXIT_REASON_EXCEPTION_NMI &&
5984	exit_reason != EXIT_REASON_EPT_VIOLATION &&	6041	exit_reason.basic != EXIT_REASON_EPT_VIOLATION &&
5985	exit_reason != EXIT_REASON_PML_FULL &&	6042	exit_reason.basic != EXIT_REASON_PML_FULL &&
5986	exit_reason != EXIT_REASON_APIC_ACCESS &&	6043	exit_reason.basic != EXIT_REASON_APIC_ACCESS &&
5987	exit_reason != EXIT_REASON_TASK_SWITCH)) {	6044	exit_reason.basic != EXIT_REASON_TASK_SWITCH)) {
5988	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;	6045	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
5989	vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV;	6046	vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV;
5990	vcpu->run->internal.ndata = 3;	6047	vcpu->run->internal.ndata = 3;
5991	vcpu->run->internal.data[0] = vectoring_info;	6048	vcpu->run->internal.data[0] = vectoring_info;
5992	vcpu->run->internal.data[1] = exit_reason;	6049	vcpu->run->internal.data[1] = exit_reason.full;
5993	vcpu->run->internal.data[2] = vcpu->arch.exit_qualification;	6050	vcpu->run->internal.data[2] = vcpu->arch.exit_qualification;
5994	if (exit_reason == EXIT_REASON_EPT_MISCONFIG) {	6051	if (exit_reason.basic == EXIT_REASON_EPT_MISCONFIG) {
5995	vcpu->run->internal.ndata++;	6052	vcpu->run->internal.ndata++;
5996	vcpu->run->internal.data[3] =	6053	vcpu->run->internal.data[3] =
5997	vmcs_read64(GUEST_PHYSICAL_ADDRESS);	6054	vmcs_read64(GUEST_PHYSICAL_ADDRESS);
@@ -6023,42 +6080,62 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) · 25 unmodified lines
6023	if (exit_fastpath != EXIT_FASTPATH_NONE)	6080	if (exit_fastpath != EXIT_FASTPATH_NONE)
6024	return 1;	6081	return 1;
6025		6082
6026	if (exit_reason >= kvm_vmx_max_exit_handlers)	6083	if (exit_reason.basic >= kvm_vmx_max_exit_handlers)
6027	goto unexpected_vmexit;	6084	goto unexpected_vmexit;
6028	#ifdef CONFIG_RETPOLINE	6085	#ifdef CONFIG_RETPOLINE
6029	if (exit_reason == EXIT_REASON_MSR_WRITE)	6086	if (exit_reason.basic == EXIT_REASON_MSR_WRITE)
6030	return kvm_emulate_wrmsr(vcpu);	6087	return kvm_emulate_wrmsr(vcpu);
6031	else if (exit_reason == EXIT_REASON_PREEMPTION_TIMER)	6088	else if (exit_reason.basic == EXIT_REASON_PREEMPTION_TIMER)
6032	return handle_preemption_timer(vcpu);	6089	return handle_preemption_timer(vcpu);
6033	else if (exit_reason == EXIT_REASON_INTERRUPT_WINDOW)	6090	else if (exit_reason.basic == EXIT_REASON_INTERRUPT_WINDOW)
6034	return handle_interrupt_window(vcpu);	6091	return handle_interrupt_window(vcpu);
6035	else if (exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT)	6092	else if (exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT)
6036	return handle_external_interrupt(vcpu);	6093	return handle_external_interrupt(vcpu);
6037	else if (exit_reason == EXIT_REASON_HLT)	6094	else if (exit_reason.basic == EXIT_REASON_HLT)
6038	return kvm_emulate_halt(vcpu);	6095	return kvm_emulate_halt(vcpu);
6039	else if (exit_reason == EXIT_REASON_EPT_MISCONFIG)	6096	else if (exit_reason.basic == EXIT_REASON_EPT_MISCONFIG)
6040	return handle_ept_misconfig(vcpu);	6097	return handle_ept_misconfig(vcpu);
6041	#endif	6098	#endif
6042		6099
6043	exit_reason = array_index_nospec(exit_reason,	6100	exit_handler_index = array_index_nospec((u16)exit_reason.basic,
6044	kvm_vmx_max_exit_handlers);	6101	kvm_vmx_max_exit_handlers);
6045	if (!kvm_vmx_exit_handlers[exit_reason])	6102	if (!kvm_vmx_exit_handlers[exit_handler_index])
6046	goto unexpected_vmexit;	6103	goto unexpected_vmexit;
6047		6104
6048	return kvm_vmx_exit_handlers[exit_reason](vcpu);	6105	return kvm_vmx_exit_handlers[exit_handler_index](vcpu);
6049		6106
6050	unexpected_vmexit:	6107	unexpected_vmexit:
6051	vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n", exit_reason);	6108	vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n",
		6109	exit_reason.full);
6052	dump_vmcs();	6110	dump_vmcs();
6053	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;	6111	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
6054	vcpu->run->internal.suberror =	6112	vcpu->run->internal.suberror =
6055	KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;	6113	KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
6056	vcpu->run->internal.ndata = 2;	6114	vcpu->run->internal.ndata = 2;
6057	vcpu->run->internal.data[0] = exit_reason;	6115	vcpu->run->internal.data[0] = exit_reason.full;
6058	vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;	6116	vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
6059	return 0;	6117	return 0;
6060	}	6118	}
6061		6119
		6120	static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
		6121	{
		6122	int ret = __vmx_handle_exit(vcpu, exit_fastpath);
		6123
		6124	/*
		6125	* Even when current exit reason is handled by KVM internally, we
		6126	* still need to exit to user space when bus lock detected to inform
		6127	* that there is a bus lock in guest.
		6128	*/
		6129	if (to_vmx(vcpu)->exit_reason.bus_lock_detected) {
		6130	if (ret > 0)
		6131	vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK;
		6132
		6133	vcpu->run->flags \|= KVM_RUN_X86_BUS_LOCK;
		6134	return 0;
		6135	}
		6136	return ret;
		6137	}
		6138
6062	/*	6139	/*
6063	* Software based L1D cache flush which is used when microcode providing	6140	* Software based L1D cache flush which is used when microcode providing
6064	* the cache control MSR is not loaded.	6141	* the cache control MSR is not loaded.
@@ -6129,7 +6206,7 @@ static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu) · 64 unmodified lines
6129	: "eax", "ebx", "ecx", "edx");	6206	: "eax", "ebx", "ecx", "edx");
6130	}	6207	}
6131		6208
6132	static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)	6209	static void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
6133	{	6210	{
6134	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);	6211	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
6135	int tpr_threshold;	6212	int tpr_threshold;
@@ -6373,9 +6450,9 @@ static void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu) · 237 unmodified lines
6373	{	6450	{
6374	struct vcpu_vmx *vmx = to_vmx(vcpu);	6451	struct vcpu_vmx *vmx = to_vmx(vcpu);
6375		6452
6376	if (vmx->exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT)	6453	if (vmx->exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT)
6377	handle_external_interrupt_irqoff(vcpu);	6454	handle_external_interrupt_irqoff(vcpu);
6378	else if (vmx->exit_reason == EXIT_REASON_EXCEPTION_NMI)	6455	else if (vmx->exit_reason.basic == EXIT_REASON_EXCEPTION_NMI)
6379	handle_exception_nmi_irqoff(vmx);	6456	handle_exception_nmi_irqoff(vmx);
6380	}	6457	}
6381		6458
@@ -6567,7 +6644,7 @@ void noinstr vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp) · 185 unmodified lines
6567		6644
6568	static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu)	6645	static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
6569	{	6646	{
6570	switch (to_vmx(vcpu)->exit_reason) {	6647	switch (to_vmx(vcpu)->exit_reason.basic) {
6571	case EXIT_REASON_MSR_WRITE:	6648	case EXIT_REASON_MSR_WRITE:
6572	return handle_fastpath_set_msr_irqoff(vcpu);	6649	return handle_fastpath_set_msr_irqoff(vcpu);
6573	case EXIT_REASON_PREEMPTION_TIMER:	6650	case EXIT_REASON_PREEMPTION_TIMER:
@@ -6577,8 +6654,6 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) · 3 unmodified lines
6577	}	6654	}
6578	}	6655	}
6579		6656
6580	bool __vmx_vcpu_run(struct vcpu_vmx vmx, unsigned long regs, bool launched);
6581
6582	static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,	6657	static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,
6583	struct vcpu_vmx *vmx)	6658	struct vcpu_vmx *vmx)
6584	{	6659	{
@@ -6638,11 +6713,9 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, · 53 unmodified lines
6638		6713
6639	static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu)	6714	static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu)
6640	{	6715	{
6641	fastpath_t exit_fastpath;
6642	struct vcpu_vmx *vmx = to_vmx(vcpu);	6716	struct vcpu_vmx *vmx = to_vmx(vcpu);
6643	unsigned long cr3, cr4;	6717	unsigned long cr3, cr4;
6644		6718
6645	reenter_guest:
6646	/* Record the guest's net vcpu time for enforced NMI injections. */	6719	/* Record the guest's net vcpu time for enforced NMI injections. */
6647	if (unlikely(!enable_vnmi &&	6720	if (unlikely(!enable_vnmi &&
6648	vmx->loaded_vmcs->soft_vnmi_blocked))	6721	vmx->loaded_vmcs->soft_vnmi_blocked))
@@ -6696,6 +6769,8 @@ reenter_guest: · 47 unmodified lines
6696	pt_guest_enter(vmx);	6769	pt_guest_enter(vmx);
6697		6770
6698	atomic_switch_perf_msrs(vmx);	6771	atomic_switch_perf_msrs(vmx);
		6772	if (intel_pmu_lbr_is_enabled(vcpu))
		6773	vmx_passthrough_lbr_msrs(vcpu);
6699		6774
6700	if (enable_preemption_timer)	6775	if (enable_preemption_timer)
6701	vmx_update_hv_timer(vcpu);	6776	vmx_update_hv_timer(vcpu);
@@ -6734,12 +6809,12 @@ reenter_guest: · 32 unmodified lines
6734	x86_spec_ctrl_restore_host(vmx->spec_ctrl, 0);	6809	x86_spec_ctrl_restore_host(vmx->spec_ctrl, 0);
6735		6810
6736	/* All fields are clean at this point */	6811	/* All fields are clean at this point */
6737	if (static_branch_unlikely(&enable_evmcs))	6812	if (static_branch_unlikely(&enable_evmcs)) {
6738	current_evmcs->hv_clean_fields \|=	6813	current_evmcs->hv_clean_fields \|=
6739	HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;	6814	HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;
6740		6815
6741	if (static_branch_unlikely(&enable_evmcs))	6816	current_evmcs->hv_vp_id = kvm_hv_get_vpindex(vcpu);
6742	current_evmcs->hv_vp_id = vcpu->arch.hyperv.vp_index;	6817	}
6743		6818
6744	/* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */	6819	/* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
6745	if (vmx->host_debugctlmsr)	6820	if (vmx->host_debugctlmsr)
@@ -6768,17 +6843,17 @@ reenter_guest: · 22 unmodified lines
6768	vmx->idt_vectoring_info = 0;	6843	vmx->idt_vectoring_info = 0;
6769		6844
6770	if (unlikely(vmx->fail)) {	6845	if (unlikely(vmx->fail)) {
6771	vmx->exit_reason = 0xdead;	6846	vmx->exit_reason.full = 0xdead;
6772	return EXIT_FASTPATH_NONE;	6847	return EXIT_FASTPATH_NONE;
6773	}	6848	}
6774		6849
6775	vmx->exit_reason = vmcs_read32(VM_EXIT_REASON);	6850	vmx->exit_reason.full = vmcs_read32(VM_EXIT_REASON);
6776	if (unlikely((u16)vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY))	6851	if (unlikely((u16)vmx->exit_reason.basic == EXIT_REASON_MCE_DURING_VMENTRY))
6777	kvm_machine_check();	6852	kvm_machine_check();
6778		6853
6779	trace_kvm_exit(vmx->exit_reason, vcpu, KVM_ISA_VMX);	6854	trace_kvm_exit(vmx->exit_reason.full, vcpu, KVM_ISA_VMX);
6780		6855
6781	if (unlikely(vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY))	6856	if (unlikely(vmx->exit_reason.failed_vmentry))
6782	return EXIT_FASTPATH_NONE;	6857	return EXIT_FASTPATH_NONE;
6783		6858
6784	vmx->loaded_vmcs->launched = 1;	6859	vmx->loaded_vmcs->launched = 1;
@@ -6790,22 +6865,7 @@ reenter_guest: · 5 unmodified lines
6790	if (is_guest_mode(vcpu))	6865	if (is_guest_mode(vcpu))
6791	return EXIT_FASTPATH_NONE;	6866	return EXIT_FASTPATH_NONE;
6792		6867
6793	exit_fastpath = vmx_exit_handlers_fastpath(vcpu);	6868	return vmx_exit_handlers_fastpath(vcpu);
6794	if (exit_fastpath == EXIT_FASTPATH_REENTER_GUEST) {
6795	if (!kvm_vcpu_exit_request(vcpu)) {
6796	/*
6797	* FIXME: this goto should be a loop in vcpu_enter_guest,
6798	* but it would incur the cost of a retpoline for now.
6799	* Revisit once static calls are available.
6800	*/
6801	if (vcpu->arch.apicv_active)
6802	vmx_sync_pir_to_irr(vcpu);
6803	goto reenter_guest;
6804	}
6805	exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED;
6806	}
6807
6808	return exit_fastpath;
6809	}	6869	}
6810		6870
6811	static void vmx_free_vcpu(struct kvm_vcpu *vcpu)	6871	static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
@@ -7256,7 +7316,7 @@ static void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) · 444 unmodified lines
7256	set_cr4_guest_host_mask(vmx);	7316	set_cr4_guest_host_mask(vmx);
7257		7317
7258	/* Refresh #PF interception to account for MAXPHYADDR changes. */	7318	/* Refresh #PF interception to account for MAXPHYADDR changes. */
7259	update_exception_bitmap(vcpu);	7319	vmx_update_exception_bitmap(vcpu);
7260	}	7320	}
7261		7321
7262	static __init void vmx_set_cpu_caps(void)	7322	static __init void vmx_set_cpu_caps(void)
@@ -7546,7 +7606,7 @@ static int vmx_pre_leave_smm(struct kvm_vcpu vcpu, const char smstate) · 283 unmodified lines
7546	return 0;	7606	return 0;
7547	}	7607	}
7548		7608
7549	static void enable_smi_window(struct kvm_vcpu *vcpu)	7609	static void vmx_enable_smi_window(struct kvm_vcpu *vcpu)
7550	{	7610	{
7551	/* RSM will cause a vmexit anyway. */	7611	/* RSM will cause a vmexit anyway. */
7552	}	7612	}
@@ -7606,7 +7666,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { · 53 unmodified lines
7606	.vcpu_load = vmx_vcpu_load,	7666	.vcpu_load = vmx_vcpu_load,
7607	.vcpu_put = vmx_vcpu_put,	7667	.vcpu_put = vmx_vcpu_put,
7608		7668
7609	.update_exception_bitmap = update_exception_bitmap,	7669	.update_exception_bitmap = vmx_update_exception_bitmap,
7610	.get_msr_feature = vmx_get_msr_feature,	7670	.get_msr_feature = vmx_get_msr_feature,
7611	.get_msr = vmx_get_msr,	7671	.get_msr = vmx_get_msr,
7612	.set_msr = vmx_set_msr,	7672	.set_msr = vmx_set_msr,
@@ -7649,9 +7709,9 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { · 36 unmodified lines
7649	.nmi_allowed = vmx_nmi_allowed,	7709	.nmi_allowed = vmx_nmi_allowed,
7650	.get_nmi_mask = vmx_get_nmi_mask,	7710	.get_nmi_mask = vmx_get_nmi_mask,
7651	.set_nmi_mask = vmx_set_nmi_mask,	7711	.set_nmi_mask = vmx_set_nmi_mask,
7652	.enable_nmi_window = enable_nmi_window,	7712	.enable_nmi_window = vmx_enable_nmi_window,
7653	.enable_irq_window = enable_irq_window,	7713	.enable_irq_window = vmx_enable_irq_window,
7654	.update_cr8_intercept = update_cr8_intercept,	7714	.update_cr8_intercept = vmx_update_cr8_intercept,
7655	.set_virtual_apic_mode = vmx_set_virtual_apic_mode,	7715	.set_virtual_apic_mode = vmx_set_virtual_apic_mode,
7656	.set_apic_access_page_addr = vmx_set_apic_access_page_addr,	7716	.set_apic_access_page_addr = vmx_set_apic_access_page_addr,
7657	.refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,	7717	.refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
@@ -7709,7 +7769,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { · 51 unmodified lines
7709	.smi_allowed = vmx_smi_allowed,	7769	.smi_allowed = vmx_smi_allowed,
7710	.pre_enter_smm = vmx_pre_enter_smm,	7770	.pre_enter_smm = vmx_pre_enter_smm,
7711	.pre_leave_smm = vmx_pre_leave_smm,	7771	.pre_leave_smm = vmx_pre_leave_smm,
7712	.enable_smi_window = enable_smi_window,	7772	.enable_smi_window = vmx_enable_smi_window,
7713		7773
7714	.can_emulate_instruction = vmx_can_emulate_instruction,	7774	.can_emulate_instruction = vmx_can_emulate_instruction,
7715	.apic_init_signal_blocked = vmx_apic_init_signal_blocked,	7775	.apic_init_signal_blocked = vmx_apic_init_signal_blocked,
@@ -7810,6 +7870,8 @@ static __init int hardware_setup(void) · 94 unmodified lines
7810	kvm_tsc_scaling_ratio_frac_bits = 48;	7870	kvm_tsc_scaling_ratio_frac_bits = 48;
7811	}	7871	}
7812		7872
		7873	kvm_has_bus_lock_exit = cpu_has_vmx_bus_lock_detection();
		7874
7813	set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */	7875	set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
7814		7876
7815	if (enable_ept)	7877	if (enable_ept)

arch/x86/kvm/vmx/vmx.h+54−2

@@ -70,6 +70,54 @@ struct pt_desc { · 69 unmodified lines
70	struct pt_ctx guest;	70	struct pt_ctx guest;
71	};	71	};
72		72
		73	union vmx_exit_reason {
		74	struct {
		75	u32 basic : 16;
		76	u32 reserved16 : 1;
		77	u32 reserved17 : 1;
		78	u32 reserved18 : 1;
		79	u32 reserved19 : 1;
		80	u32 reserved20 : 1;
		81	u32 reserved21 : 1;
		82	u32 reserved22 : 1;
		83	u32 reserved23 : 1;
		84	u32 reserved24 : 1;
		85	u32 reserved25 : 1;
		86	u32 bus_lock_detected : 1;
		87	u32 enclave_mode : 1;
		88	u32 smi_pending_mtf : 1;
		89	u32 smi_from_vmx_root : 1;
		90	u32 reserved30 : 1;
		91	u32 failed_vmentry : 1;
		92	};
		93	u32 full;
		94	};
		95
		96	#define vcpu_to_lbr_desc(vcpu) (&to_vmx(vcpu)->lbr_desc)
		97	#define vcpu_to_lbr_records(vcpu) (&to_vmx(vcpu)->lbr_desc.records)
		98
		99	bool intel_pmu_lbr_is_compatible(struct kvm_vcpu *vcpu);
		100	bool intel_pmu_lbr_is_enabled(struct kvm_vcpu *vcpu);
		101
		102	int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu);
		103	void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu);
		104
		105	struct lbr_desc {
		106	/* Basic info about guest LBR records. */
		107	struct x86_pmu_lbr records;
		108
		109	/*
		110	* Emulate LBR feature via passthrough LBR registers when the
		111	* per-vcpu guest LBR event is scheduled on the current pcpu.
		112	*
		113	* The records may be inaccurate if the host reclaims the LBR.
		114	*/
		115	struct perf_event *event;
		116
		117	/* True if LBRs are marked as not intercepted in the MSR bitmap */
		118	bool msr_passthrough;
		119	};
		120
73	/*	121	/*
74	* The nested_vmx structure is part of vcpu_vmx, and holds information we need	122	* The nested_vmx structure is part of vcpu_vmx, and holds information we need
75	* for correct emulation of VMX (i.e., nested VMX) on this vcpu.	123	* for correct emulation of VMX (i.e., nested VMX) on this vcpu.
@@ -244,7 +292,7 @@ struct vcpu_vmx { · 168 unmodified lines
244	int vpid;	292	int vpid;
245	bool emulation_required;	293	bool emulation_required;
246		294
247	u32 exit_reason;	295	union vmx_exit_reason exit_reason;
248		296
249	/* Posted interrupt descriptor */	297	/* Posted interrupt descriptor */
250	struct pi_desc pi_desc;	298	struct pi_desc pi_desc;
@@ -279,6 +327,7 @@ struct vcpu_vmx { · 28 unmodified lines
279	u64 ept_pointer;	327	u64 ept_pointer;
280		328
281	struct pt_desc pt_desc;	329	struct pt_desc pt_desc;
		330	struct lbr_desc lbr_desc;
282		331
283	/* Save desired MSR intercept (read: pass-through) state */	332	/* Save desired MSR intercept (read: pass-through) state */
284	#define MAX_POSSIBLE_PASSTHROUGH_MSRS 13	333	#define MAX_POSSIBLE_PASSTHROUGH_MSRS 13
@@ -329,7 +378,7 @@ void vmx_set_segment(struct kvm_vcpu vcpu, struct kvm_segment var, int seg); · 44 unmodified lines
329	u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa,	378	u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa,
330	int root_level);	379	int root_level);
331		380
332	void update_exception_bitmap(struct kvm_vcpu *vcpu);	381	void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu);
333	void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu);	382	void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu);
334	bool vmx_nmi_blocked(struct kvm_vcpu *vcpu);	383	bool vmx_nmi_blocked(struct kvm_vcpu *vcpu);
335	bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu);	384	bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu);
@@ -339,8 +388,11 @@ void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu); · 3 unmodified lines
339	struct vmx_uret_msr vmx_find_uret_msr(struct vcpu_vmx vmx, u32 msr);	388	struct vmx_uret_msr vmx_find_uret_msr(struct vcpu_vmx vmx, u32 msr);
340	void pt_update_intercept_for_msr(struct kvm_vcpu *vcpu);	389	void pt_update_intercept_for_msr(struct kvm_vcpu *vcpu);
341	void vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp);	390	void vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp);
		391	bool __vmx_vcpu_run(struct vcpu_vmx vmx, unsigned long regs, bool launched);
342	int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr);	392	int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr);
343	void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu);	393	void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu);
		394	void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu,
		395	u32 msr, int type, bool value);
344		396
345	static inline u8 vmx_get_rvi(void)	397	static inline u8 vmx_get_rvi(void)
346	{	398	{

arch/x86/kvm/x86.c+392−326

@@ -29,6 +29,7 @@ · 28 unmodified lines
29	#include "pmu.h"	29	#include "pmu.h"
30	#include "hyperv.h"	30	#include "hyperv.h"
31	#include "lapic.h"	31	#include "lapic.h"
		32	#include "xen.h"
32		33
33	#include <linux/clocksource.h>	34	#include <linux/clocksource.h>
34	#include <linux/interrupt.h>	35	#include <linux/interrupt.h>
@@ -114,11 +115,21 @@ static int sync_regs(struct kvm_vcpu *vcpu); · 79 unmodified lines
114	struct kvm_x86_ops kvm_x86_ops __read_mostly;	115	struct kvm_x86_ops kvm_x86_ops __read_mostly;
115	EXPORT_SYMBOL_GPL(kvm_x86_ops);	116	EXPORT_SYMBOL_GPL(kvm_x86_ops);
116		117
		118	#define KVM_X86_OP(func) \
		119	DEFINE_STATIC_CALL_NULL(kvm_x86_##func, \
		120	(((struct kvm_x86_ops )0)->func));
		121	#define KVM_X86_OP_NULL KVM_X86_OP
		122	#include <asm/kvm-x86-ops.h>
		123	EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits);
		124	EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg);
		125	EXPORT_STATIC_CALL_GPL(kvm_x86_tlb_flush_current);
		126
117	static bool __read_mostly ignore_msrs = 0;	127	static bool __read_mostly ignore_msrs = 0;
118	module_param(ignore_msrs, bool, S_IRUGO \| S_IWUSR);	128	module_param(ignore_msrs, bool, S_IRUGO \| S_IWUSR);
119		129
120	static bool __read_mostly report_ignored_msrs = true;	130	bool __read_mostly report_ignored_msrs = true;
121	module_param(report_ignored_msrs, bool, S_IRUGO \| S_IWUSR);	131	module_param(report_ignored_msrs, bool, S_IRUGO \| S_IWUSR);
		132	EXPORT_SYMBOL_GPL(report_ignored_msrs);
122		133
123	unsigned int min_timer_period_us = 200;	134	unsigned int min_timer_period_us = 200;
124	module_param(min_timer_period_us, uint, S_IRUGO \| S_IWUSR);	135	module_param(min_timer_period_us, uint, S_IRUGO \| S_IWUSR);
@@ -136,6 +147,8 @@ u64 __read_mostly kvm_max_tsc_scaling_ratio; · 11 unmodified lines
136	EXPORT_SYMBOL_GPL(kvm_max_tsc_scaling_ratio);	147	EXPORT_SYMBOL_GPL(kvm_max_tsc_scaling_ratio);
137	u64 __read_mostly kvm_default_tsc_scaling_ratio;	148	u64 __read_mostly kvm_default_tsc_scaling_ratio;
138	EXPORT_SYMBOL_GPL(kvm_default_tsc_scaling_ratio);	149	EXPORT_SYMBOL_GPL(kvm_default_tsc_scaling_ratio);
		150	bool __read_mostly kvm_has_bus_lock_exit;
		151	EXPORT_SYMBOL_GPL(kvm_has_bus_lock_exit);
139		152
140	/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */	153	/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
141	static u32 __read_mostly tsc_tolerance_ppm = 250;	154	static u32 __read_mostly tsc_tolerance_ppm = 250;
@@ -234,7 +247,6 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { · 92 unmodified lines
234	VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns),	247	VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns),
235	VM_STAT("mmu_shadow_zapped", mmu_shadow_zapped),	248	VM_STAT("mmu_shadow_zapped", mmu_shadow_zapped),
236	VM_STAT("mmu_pte_write", mmu_pte_write),	249	VM_STAT("mmu_pte_write", mmu_pte_write),
237	VM_STAT("mmu_pte_updated", mmu_pte_updated),
238	VM_STAT("mmu_pde_zapped", mmu_pde_zapped),	250	VM_STAT("mmu_pde_zapped", mmu_pde_zapped),
239	VM_STAT("mmu_flooded", mmu_flooded),	251	VM_STAT("mmu_flooded", mmu_flooded),
240	VM_STAT("mmu_recycled", mmu_recycled),	252	VM_STAT("mmu_recycled", mmu_recycled),
@@ -395,7 +407,7 @@ int kvm_set_apic_base(struct kvm_vcpu vcpu, struct msr_data msr_info) · 154 unmodified lines
395	{	407	{
396	enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);	408	enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
397	enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);	409	enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);
398	u64 reserved_bits = ((~0ULL) << cpuid_maxphyaddr(vcpu)) \| 0x2ff \|	410	u64 reserved_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu) \| 0x2ff \|
399	(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);	411	(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);
400		412
401	if ((msr_info->data & reserved_bits) != 0 \|\| new_mode == LAPIC_MODE_INVALID)	413	if ((msr_info->data & reserved_bits) != 0 \|\| new_mode == LAPIC_MODE_INVALID)
@@ -484,19 +496,24 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu) · 82 unmodified lines
484	*/	496	*/
485	vcpu->arch.dr6 &= ~DR_TRAP_BITS;	497	vcpu->arch.dr6 &= ~DR_TRAP_BITS;
486	/*	498	/*
487	* DR6.RTM is set by all #DB exceptions that don't clear it.	499	* In order to reflect the #DB exception payload in guest
		500	* dr6, three components need to be considered: active low
		501	* bit, FIXED_1 bits and active high bits (e.g. DR6_BD,
		502	* DR6_BS and DR6_BT)
		503	* DR6_ACTIVE_LOW contains the FIXED_1 and active low bits.
		504	* In the target guest dr6:
		505	* FIXED_1 bits should always be set.
		506	* Active low bits should be cleared if 1-setting in payload.
		507	* Active high bits should be set if 1-setting in payload.
		508	*
		509	* Note, the payload is compatible with the pending debug
		510	* exceptions/exit qualification under VMX, that active_low bits
		511	* are active high in payload.
		512	* So they need to be flipped for DR6.
488	*/	513	*/
489	vcpu->arch.dr6 \|= DR6_RTM;	514	vcpu->arch.dr6 \|= DR6_ACTIVE_LOW;
490	vcpu->arch.dr6 \|= payload;	515	vcpu->arch.dr6 \|= payload;
491	/*	516	vcpu->arch.dr6 ^= payload & DR6_ACTIVE_LOW;
492	* Bit 16 should be set in the payload whenever the #DB
493	* exception should clear DR6.RTM. This makes the payload
494	* compatible with the pending debug exceptions under VMX.
495	* Though not currently documented in the SDM, this also
496	* makes the payload compatible with the exit qualification
497	* for #DB exceptions under VMX.
498	*/
499	vcpu->arch.dr6 ^= payload & DR6_RTM;
500		517
501	/*	518	/*
502	* The #DB payload is defined as compatible with the 'pending	519	* The #DB payload is defined as compatible with the 'pending
@@ -692,7 +709,7 @@ EXPORT_SYMBOL_GPL(kvm_requeue_exception_e); · 189 unmodified lines
692	*/	709	*/
693	bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)	710	bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
694	{	711	{
695	if (kvm_x86_ops.get_cpl(vcpu) <= required_cpl)	712	if (static_call(kvm_x86_get_cpl)(vcpu) <= required_cpl)
696	return true;	713	return true;
697	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);	714	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
698	return false;	715	return false;
@@ -742,8 +759,7 @@ static int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, · 43 unmodified lines
742		759
743	static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)	760	static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
744	{	761	{
745	return rsvd_bits(cpuid_maxphyaddr(vcpu), 63) \| rsvd_bits(5, 8) \|	762	return vcpu->arch.reserved_gpa_bits \| rsvd_bits(5, 8) \| rsvd_bits(1, 2);
746	rsvd_bits(1, 2);
747	}	763	}
748		764
749	/*	765	/*
@@ -852,7 +868,7 @@ int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) · 102 unmodified lines
852		868
853	if (!is_pae(vcpu))	869	if (!is_pae(vcpu))
854	return 1;	870	return 1;
855	kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l);	871	static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
856	if (cs_l)	872	if (cs_l)
857	return 1;	873	return 1;
858	}	874	}
@@ -865,7 +881,7 @@ int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) · 6 unmodified lines
865	if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))	881	if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
866	return 1;	882	return 1;
867		883
868	kvm_x86_ops.set_cr0(vcpu, cr0);	884	static_call(kvm_x86_set_cr0)(vcpu, cr0);
869		885
870	kvm_post_set_cr0(vcpu, old_cr0, cr0);	886	kvm_post_set_cr0(vcpu, old_cr0, cr0);
871		887
@@ -970,12 +986,10 @@ static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) · 98 unmodified lines
970		986
971	int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)	987	int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
972	{	988	{
973	if (kvm_x86_ops.get_cpl(vcpu) != 0 \|\|	989	if (static_call(kvm_x86_get_cpl)(vcpu) == 0)
974	__kvm_set_xcr(vcpu, index, xcr)) {	990	return __kvm_set_xcr(vcpu, index, xcr);
975	kvm_inject_gp(vcpu, 0);	991
976	return 1;	992	return 1;
977	}
978	return 0;
979	}	993	}
980	EXPORT_SYMBOL_GPL(kvm_set_xcr);	994	EXPORT_SYMBOL_GPL(kvm_set_xcr);
981		995
@@ -987,7 +1001,7 @@ bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) · 5 unmodified lines
987	if (cr4 & vcpu->arch.cr4_guest_rsvd_bits)	1001	if (cr4 & vcpu->arch.cr4_guest_rsvd_bits)
988	return false;	1002	return false;
989		1003
990	return kvm_x86_ops.is_valid_cr4(vcpu, cr4);	1004	return static_call(kvm_x86_is_valid_cr4)(vcpu, cr4);
991	}	1005	}
992	EXPORT_SYMBOL_GPL(kvm_is_valid_cr4);	1006	EXPORT_SYMBOL_GPL(kvm_is_valid_cr4);
993		1007
@@ -1031,7 +1045,7 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) · 37 unmodified lines
1031	return 1;	1045	return 1;
1032	}	1046	}
1033		1047
1034	kvm_x86_ops.set_cr4(vcpu, cr4);	1048	static_call(kvm_x86_set_cr4)(vcpu, cr4);
1035		1049
1036	kvm_post_set_cr4(vcpu, old_cr4, cr4);	1050	kvm_post_set_cr4(vcpu, old_cr4, cr4);
1037		1051
@@ -1059,8 +1073,7 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) · 21 unmodified lines
1059	return 0;	1073	return 0;
1060	}	1074	}
1061		1075
1062	if (is_long_mode(vcpu) &&	1076	if (is_long_mode(vcpu) && kvm_vcpu_is_illegal_gpa(vcpu, cr3))
1063	(cr3 & vcpu->arch.cr3_lm_rsvd_bits))
1064	return 1;	1077	return 1;
1065	else if (is_pae_paging(vcpu) &&	1078	else if (is_pae_paging(vcpu) &&
1066	!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))	1079	!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
@@ -1114,7 +1127,7 @@ void kvm_update_dr7(struct kvm_vcpu *vcpu) · 47 unmodified lines
1114	dr7 = vcpu->arch.guest_debug_dr7;	1127	dr7 = vcpu->arch.guest_debug_dr7;
1115	else	1128	else
1116	dr7 = vcpu->arch.dr7;	1129	dr7 = vcpu->arch.dr7;
1117	kvm_x86_ops.set_dr7(vcpu, dr7);	1130	static_call(kvm_x86_set_dr7)(vcpu, dr7);
1118	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;	1131	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
1119	if (dr7 & DR7_BP_EN_MASK)	1132	if (dr7 & DR7_BP_EN_MASK)
1120	vcpu->arch.switch_db_regs \|= KVM_DEBUGREG_BP_ENABLED;	1133	vcpu->arch.switch_db_regs \|= KVM_DEBUGREG_BP_ENABLED;
@@ -1130,7 +1143,7 @@ static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) · 9 unmodified lines
1130	return fixed;	1143	return fixed;
1131	}	1144	}
1132		1145
1133	static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)	1146	int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1134	{	1147	{
1135	size_t size = ARRAY_SIZE(vcpu->arch.db);	1148	size_t size = ARRAY_SIZE(vcpu->arch.db);
1136		1149
@@ -1143,13 +1156,13 @@ static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) · 6 unmodified lines
1143	case 4:	1156	case 4:
1144	case 6:	1157	case 6:
1145	if (!kvm_dr6_valid(val))	1158	if (!kvm_dr6_valid(val))
1146	return -1; /* #GP */	1159	return 1; /* #GP */
1147	vcpu->arch.dr6 = (val & DR6_VOLATILE) \| kvm_dr6_fixed(vcpu);	1160	vcpu->arch.dr6 = (val & DR6_VOLATILE) \| kvm_dr6_fixed(vcpu);
1148	break;	1161	break;
1149	case 5:	1162	case 5:
1150	default: /* 7 */	1163	default: /* 7 */
1151	if (!kvm_dr7_valid(val))	1164	if (!kvm_dr7_valid(val))
1152	return -1; /* #GP */	1165	return 1; /* #GP */
1153	vcpu->arch.dr7 = (val & DR7_VOLATILE) \| DR7_FIXED_1;	1166	vcpu->arch.dr7 = (val & DR7_VOLATILE) \| DR7_FIXED_1;
1154	kvm_update_dr7(vcpu);	1167	kvm_update_dr7(vcpu);
1155	break;	1168	break;
@@ -1157,18 +1170,9 @@ static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) · 1 unmodified line
1157		1170
1158	return 0;	1171	return 0;
1159	}	1172	}
1160
1161	int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1162	{
1163	if (__kvm_set_dr(vcpu, dr, val)) {
1164	kvm_inject_gp(vcpu, 0);
1165	return 1;
1166	}
1167	return 0;
1168	}
1169	EXPORT_SYMBOL_GPL(kvm_set_dr);	1173	EXPORT_SYMBOL_GPL(kvm_set_dr);
1170		1174
1171	int kvm_get_dr(struct kvm_vcpu vcpu, int dr, unsigned long val)	1175	void kvm_get_dr(struct kvm_vcpu vcpu, int dr, unsigned long val)
1172	{	1176	{
1173	size_t size = ARRAY_SIZE(vcpu->arch.db);	1177	size_t size = ARRAY_SIZE(vcpu->arch.db);
1174		1178
@@ -1185,7 +1189,6 @@ int kvm_get_dr(struct kvm_vcpu vcpu, int dr, unsigned long val) · 10 unmodified lines
1185	*val = vcpu->arch.dr7;	1189	*val = vcpu->arch.dr7;
1186	break;	1190	break;
1187	}	1191	}
1188	return 0;
1189	}	1192	}
1190	EXPORT_SYMBOL_GPL(kvm_get_dr);	1193	EXPORT_SYMBOL_GPL(kvm_get_dr);
1191		1194
@@ -1426,7 +1429,7 @@ static int kvm_get_msr_feature(struct kvm_msr_entry *msr) · 234 unmodified lines
1426	rdmsrl_safe(msr->index, &msr->data);	1429	rdmsrl_safe(msr->index, &msr->data);
1427	break;	1430	break;
1428	default:	1431	default:
1429	return kvm_x86_ops.get_msr_feature(msr);	1432	return static_call(kvm_x86_get_msr_feature)(msr);
1430	}	1433	}
1431	return 0;	1434	return 0;
1432	}	1435	}
@@ -1502,7 +1505,7 @@ static int set_efer(struct kvm_vcpu vcpu, struct msr_data msr_info) · 69 unmodified lines
1502	efer &= ~EFER_LMA;	1505	efer &= ~EFER_LMA;
1503	efer \|= vcpu->arch.efer & EFER_LMA;	1506	efer \|= vcpu->arch.efer & EFER_LMA;
1504		1507
1505	r = kvm_x86_ops.set_efer(vcpu, efer);	1508	r = static_call(kvm_x86_set_efer)(vcpu, efer);
1506	if (r) {	1509	if (r) {
1507	WARN_ON(r > 0);	1510	WARN_ON(r > 0);
1508	return r;	1511	return r;
@@ -1599,7 +1602,7 @@ static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data, · 90 unmodified lines
1599	msr.index = index;	1602	msr.index = index;
1600	msr.host_initiated = host_initiated;	1603	msr.host_initiated = host_initiated;
1601		1604
1602	return kvm_x86_ops.set_msr(vcpu, &msr);	1605	return static_call(kvm_x86_set_msr)(vcpu, &msr);
1603	}	1606	}
1604		1607
1605	static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,	1608	static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,
@@ -1632,7 +1635,7 @@ int __kvm_get_msr(struct kvm_vcpu vcpu, u32 index, u64 data, · 26 unmodified lines
1632	msr.index = index;	1635	msr.index = index;
1633	msr.host_initiated = host_initiated;	1636	msr.host_initiated = host_initiated;
1634		1637
1635	ret = kvm_x86_ops.get_msr(vcpu, &msr);	1638	ret = static_call(kvm_x86_get_msr)(vcpu, &msr);
1636	if (!ret)	1639	if (!ret)
1637	*data = msr.data;	1640	*data = msr.data;
1638	return ret;	1641	return ret;
@@ -1673,12 +1676,12 @@ static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu) · 34 unmodified lines
1673	kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);	1676	kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);
1674	}	1677	}
1675		1678
1676	return kvm_x86_ops.complete_emulated_msr(vcpu, err);	1679	return static_call(kvm_x86_complete_emulated_msr)(vcpu, err);
1677	}	1680	}
1678		1681
1679	static int complete_emulated_wrmsr(struct kvm_vcpu *vcpu)	1682	static int complete_emulated_wrmsr(struct kvm_vcpu *vcpu)
1680	{	1683	{
1681	return kvm_x86_ops.complete_emulated_msr(vcpu, vcpu->run->msr.error);	1684	return static_call(kvm_x86_complete_emulated_msr)(vcpu, vcpu->run->msr.error);
1682	}	1685	}
1683		1686
1684	static u64 kvm_msr_reason(int r)	1687	static u64 kvm_msr_reason(int r)
@@ -1750,7 +1753,7 @@ int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu) · 65 unmodified lines
1750	trace_kvm_msr_read_ex(ecx);	1753	trace_kvm_msr_read_ex(ecx);
1751	}	1754	}
1752		1755
1753	return kvm_x86_ops.complete_emulated_msr(vcpu, r);	1756	return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);
1754	}	1757	}
1755	EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);	1758	EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);
1756		1759
@@ -1776,17 +1779,16 @@ int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu) · 19 unmodified lines
1776	else	1779	else
1777	trace_kvm_msr_write_ex(ecx, data);	1780	trace_kvm_msr_write_ex(ecx, data);
1778		1781
1779	return kvm_x86_ops.complete_emulated_msr(vcpu, r);	1782	return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);
1780	}	1783	}
1781	EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);	1784	EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);
1782		1785
1783	bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)	1786	static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)
1784	{	1787	{
1785	xfer_to_guest_mode_prepare();	1788	xfer_to_guest_mode_prepare();
1786	return vcpu->mode == EXITING_GUEST_MODE \|\| kvm_request_pending(vcpu) \|\|	1789	return vcpu->mode == EXITING_GUEST_MODE \|\| kvm_request_pending(vcpu) \|\|
1787	xfer_to_guest_mode_work_pending();	1790	xfer_to_guest_mode_work_pending();
1788	}	1791	}
1789	EXPORT_SYMBOL_GPL(kvm_vcpu_exit_request);
1790		1792
1791	/*	1793	/*
1792	* The fast path for frequent and performance sensitive wrmsr emulation,	1794	* The fast path for frequent and performance sensitive wrmsr emulation,
@@ -1936,15 +1938,14 @@ static s64 get_kvmclock_base_ns(void) · 143 unmodified lines
1936	}	1938	}
1937	#endif	1939	#endif
1938		1940
1939	static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)	1941	void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs)
1940	{	1942	{
1941	int version;	1943	int version;
1942	int r;	1944	int r;
1943	struct pvclock_wall_clock wc;	1945	struct pvclock_wall_clock wc;
		1946	u32 wc_sec_hi;
1944	u64 wall_nsec;	1947	u64 wall_nsec;
1945		1948
1946	kvm->arch.wall_clock = wall_clock;
1947
1948	if (!wall_clock)	1949	if (!wall_clock)
1949	return;	1950	return;
1950		1951
@@ -1973,6 +1974,12 @@ static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock) · 22 unmodified lines
1973		1974
1974	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));	1975	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
1975		1976
		1977	if (sec_hi_ofs) {
		1978	wc_sec_hi = wall_nsec >> 32;
		1979	kvm_write_guest(kvm, wall_clock + sec_hi_ofs,
		1980	&wc_sec_hi, sizeof(wc_sec_hi));
		1981	}
		1982
1976	version++;	1983	version++;
1977	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));	1984	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
1978	}	1985	}
@@ -2209,7 +2216,7 @@ EXPORT_SYMBOL_GPL(kvm_read_l1_tsc); · 230 unmodified lines
2209	static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)	2216	static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
2210	{	2217	{
2211	vcpu->arch.l1_tsc_offset = offset;	2218	vcpu->arch.l1_tsc_offset = offset;
2212	vcpu->arch.tsc_offset = kvm_x86_ops.write_l1_tsc_offset(vcpu, offset);	2219	vcpu->arch.tsc_offset = static_call(kvm_x86_write_l1_tsc_offset)(vcpu, offset);
2213	}	2220	}
2214		2221
2215	static inline bool kvm_check_tsc_unstable(void)	2222	static inline bool kvm_check_tsc_unstable(void)
@@ -2592,13 +2599,15 @@ u64 get_kvmclock_ns(struct kvm *kvm) · 376 unmodified lines
2592	return ret;	2599	return ret;
2593	}	2600	}
2594		2601
2595	static void kvm_setup_pvclock_page(struct kvm_vcpu *v)	2602	static void kvm_setup_pvclock_page(struct kvm_vcpu *v,
		2603	struct gfn_to_hva_cache *cache,
		2604	unsigned int offset)
2596	{	2605	{
2597	struct kvm_vcpu_arch *vcpu = &v->arch;	2606	struct kvm_vcpu_arch *vcpu = &v->arch;
2598	struct pvclock_vcpu_time_info guest_hv_clock;	2607	struct pvclock_vcpu_time_info guest_hv_clock;
2599		2608
2600	if (unlikely(kvm_read_guest_cached(v->kvm, &vcpu->pv_time,	2609	if (unlikely(kvm_read_guest_offset_cached(v->kvm, cache,
2601	&guest_hv_clock, sizeof(guest_hv_clock))))	2610	&guest_hv_clock, offset, sizeof(guest_hv_clock))))
2602	return;	2611	return;
2603		2612
2604	/* This VCPU is paused, but it's legal for a guest to read another	2613	/* This VCPU is paused, but it's legal for a guest to read another
@@ -2621,9 +2630,9 @@ static void kvm_setup_pvclock_page(struct kvm_vcpu *v) · 16 unmodified lines
2621	++guest_hv_clock.version; /* first time write, random junk */	2630	++guest_hv_clock.version; /* first time write, random junk */
2622		2631
2623	vcpu->hv_clock.version = guest_hv_clock.version + 1;	2632	vcpu->hv_clock.version = guest_hv_clock.version + 1;
2624	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,	2633	kvm_write_guest_offset_cached(v->kvm, cache,
2625	&vcpu->hv_clock,	2634	&vcpu->hv_clock, offset,
2626	sizeof(vcpu->hv_clock.version));	2635	sizeof(vcpu->hv_clock.version));
2627		2636
2628	smp_wmb();	2637	smp_wmb();
2629		2638
@@ -2637,16 +2646,16 @@ static void kvm_setup_pvclock_page(struct kvm_vcpu *v) · 7 unmodified lines
2637		2646
2638	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);	2647	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);
2639		2648
2640	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,	2649	kvm_write_guest_offset_cached(v->kvm, cache,
2641	&vcpu->hv_clock,	2650	&vcpu->hv_clock, offset,
2642	sizeof(vcpu->hv_clock));	2651	sizeof(vcpu->hv_clock));
2643		2652
2644	smp_wmb();	2653	smp_wmb();
2645		2654
2646	vcpu->hv_clock.version++;	2655	vcpu->hv_clock.version++;
2647	kvm_write_guest_cached(v->kvm, &vcpu->pv_time,	2656	kvm_write_guest_offset_cached(v->kvm, cache,
2648	&vcpu->hv_clock,	2657	&vcpu->hv_clock, offset,
2649	sizeof(vcpu->hv_clock.version));	2658	sizeof(vcpu->hv_clock.version));
2650	}	2659	}
2651		2660
2652	static int kvm_guest_time_update(struct kvm_vcpu *v)	2661	static int kvm_guest_time_update(struct kvm_vcpu *v)
@@ -2733,7 +2742,12 @@ static int kvm_guest_time_update(struct kvm_vcpu *v) · 80 unmodified lines
2733	vcpu->hv_clock.flags = pvclock_flags;	2742	vcpu->hv_clock.flags = pvclock_flags;
2734		2743
2735	if (vcpu->pv_time_enabled)	2744	if (vcpu->pv_time_enabled)
2736	kvm_setup_pvclock_page(v);	2745	kvm_setup_pvclock_page(v, &vcpu->pv_time, 0);
		2746	if (vcpu->xen.vcpu_info_set)
		2747	kvm_setup_pvclock_page(v, &vcpu->xen.vcpu_info_cache,
		2748	offsetof(struct compat_vcpu_info, time));
		2749	if (vcpu->xen.vcpu_time_info_set)
		2750	kvm_setup_pvclock_page(v, &vcpu->xen.vcpu_time_info_cache, 0);
2737	if (v == kvm_get_vcpu(v->kvm, 0))	2751	if (v == kvm_get_vcpu(v->kvm, 0))
2738	kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);	2752	kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);
2739	return 0;	2753	return 0;
@@ -2858,32 +2872,6 @@ static int set_msr_mce(struct kvm_vcpu vcpu, struct msr_data msr_info) · 118 unmodified lines
2858	return 0;	2872	return 0;
2859	}	2873	}
2860		2874
2861	static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
2862	{
2863	struct kvm *kvm = vcpu->kvm;
2864	int lm = is_long_mode(vcpu);
2865	u8 blob_addr = lm ? (u8 )(long)kvm->arch.xen_hvm_config.blob_addr_64
2866	: (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
2867	u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
2868	: kvm->arch.xen_hvm_config.blob_size_32;
2869	u32 page_num = data & ~PAGE_MASK;
2870	u64 page_addr = data & PAGE_MASK;
2871	u8 *page;
2872
2873	if (page_num >= blob_size)
2874	return 1;
2875
2876	page = memdup_user(blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE);
2877	if (IS_ERR(page))
2878	return PTR_ERR(page);
2879
2880	if (kvm_vcpu_write_guest(vcpu, page_addr, page, PAGE_SIZE)) {
2881	kfree(page);
2882	return 1;
2883	}
2884	return 0;
2885	}
2886
2887	static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)	2875	static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)
2888	{	2876	{
2889	u64 mask = KVM_ASYNC_PF_ENABLED \| KVM_ASYNC_PF_DELIVERY_AS_INT;	2877	u64 mask = KVM_ASYNC_PF_ENABLED \| KVM_ASYNC_PF_DELIVERY_AS_INT;
@@ -2955,13 +2943,13 @@ static void kvmclock_reset(struct kvm_vcpu *vcpu) · 65 unmodified lines
2955	static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)	2943	static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)
2956	{	2944	{
2957	++vcpu->stat.tlb_flush;	2945	++vcpu->stat.tlb_flush;
2958	kvm_x86_ops.tlb_flush_all(vcpu);	2946	static_call(kvm_x86_tlb_flush_all)(vcpu);
2959	}	2947	}
2960		2948
2961	static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)	2949	static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)
2962	{	2950	{
2963	++vcpu->stat.tlb_flush;	2951	++vcpu->stat.tlb_flush;
2964	kvm_x86_ops.tlb_flush_guest(vcpu);	2952	static_call(kvm_x86_tlb_flush_guest)(vcpu);
2965	}	2953	}
2966		2954
2967	static void record_steal_time(struct kvm_vcpu *vcpu)	2955	static void record_steal_time(struct kvm_vcpu *vcpu)
@@ -3017,6 +3005,9 @@ int kvm_set_msr_common(struct kvm_vcpu vcpu, struct msr_data msr_info) · 49 unmodified lines
3017	u32 msr = msr_info->index;	3005	u32 msr = msr_info->index;
3018	u64 data = msr_info->data;	3006	u64 data = msr_info->data;
3019		3007
		3008	if (msr && msr == vcpu->kvm->arch.xen_hvm_config.msr)
		3009	return kvm_xen_write_hypercall_page(vcpu, data);
		3010
3020	switch (msr) {	3011	switch (msr) {
3021	case MSR_AMD64_NB_CFG:	3012	case MSR_AMD64_NB_CFG:
3022	case MSR_IA32_UCODE_WRITE:	3013	case MSR_IA32_UCODE_WRITE:
@@ -3073,18 +3064,6 @@ int kvm_set_msr_common(struct kvm_vcpu vcpu, struct msr_data msr_info) · 50 unmodified lines
3073	return 1;	3064	return 1;
3074	}	3065	}
3075	break;	3066	break;
3076	case MSR_IA32_DEBUGCTLMSR:
3077	if (!data) {
3078	/* We support the non-activated case already */
3079	break;
3080	} else if (data & ~(DEBUGCTLMSR_LBR \| DEBUGCTLMSR_BTF)) {
3081	/* Values other than LBR and BTF are vendor-specific,
3082	thus reserved and should throw a #GP */
3083	return 1;
3084	} else if (report_ignored_msrs)
3085	vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
3086	__func__, data);
3087	break;
3088	case 0x200 ... 0x2ff:	3067	case 0x200 ... 0x2ff:
3089	return kvm_mtrr_set_msr(vcpu, msr, data);	3068	return kvm_mtrr_set_msr(vcpu, msr, data);
3090	case MSR_IA32_APICBASE:	3069	case MSR_IA32_APICBASE:
@@ -3153,13 +3132,15 @@ int kvm_set_msr_common(struct kvm_vcpu vcpu, struct msr_data msr_info) · 62 unmodified lines
3153	if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))	3132	if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3154	return 1;	3133	return 1;
3155		3134
3156	kvm_write_wall_clock(vcpu->kvm, data);	3135	vcpu->kvm->arch.wall_clock = data;
		3136	kvm_write_wall_clock(vcpu->kvm, data, 0);
3157	break;	3137	break;
3158	case MSR_KVM_WALL_CLOCK:	3138	case MSR_KVM_WALL_CLOCK:
3159	if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))	3139	if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3160	return 1;	3140	return 1;
3161		3141
3162	kvm_write_wall_clock(vcpu->kvm, data);	3142	vcpu->kvm->arch.wall_clock = data;
		3143	kvm_write_wall_clock(vcpu->kvm, data, 0);
3163	break;	3144	break;
3164	case MSR_KVM_SYSTEM_TIME_NEW:	3145	case MSR_KVM_SYSTEM_TIME_NEW:
3165	if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))	3146	if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
@@ -3304,8 +3285,6 @@ int kvm_set_msr_common(struct kvm_vcpu vcpu, struct msr_data msr_info) · 138 unmodified lines
3304	vcpu->arch.msr_misc_features_enables = data;	3285	vcpu->arch.msr_misc_features_enables = data;
3305	break;	3286	break;
3306	default:	3287	default:
3307	if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
3308	return xen_hvm_config(vcpu, data);
3309	if (kvm_pmu_is_valid_msr(vcpu, msr))	3288	if (kvm_pmu_is_valid_msr(vcpu, msr))
3310	return kvm_pmu_set_msr(vcpu, msr_info);	3289	return kvm_pmu_set_msr(vcpu, msr_info);
3311	return KVM_MSR_RET_INVALID;	3290	return KVM_MSR_RET_INVALID;
@@ -3357,7 +3336,6 @@ int kvm_get_msr_common(struct kvm_vcpu vcpu, struct msr_data msr_info) · 45 unmodified lines
3357	switch (msr_info->index) {	3336	switch (msr_info->index) {
3358	case MSR_IA32_PLATFORM_ID:	3337	case MSR_IA32_PLATFORM_ID:
3359	case MSR_IA32_EBL_CR_POWERON:	3338	case MSR_IA32_EBL_CR_POWERON:
3360	case MSR_IA32_DEBUGCTLMSR:
3361	case MSR_IA32_LASTBRANCHFROMIP:	3339	case MSR_IA32_LASTBRANCHFROMIP:
3362	case MSR_IA32_LASTBRANCHTOIP:	3340	case MSR_IA32_LASTBRANCHTOIP:
3363	case MSR_IA32_LASTINTFROMIP:	3341	case MSR_IA32_LASTINTFROMIP:
@@ -3739,7 +3717,6 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 375 unmodified lines
3739	case KVM_CAP_PIT2:	3717	case KVM_CAP_PIT2:
3740	case KVM_CAP_PIT_STATE2:	3718	case KVM_CAP_PIT_STATE2:
3741	case KVM_CAP_SET_IDENTITY_MAP_ADDR:	3719	case KVM_CAP_SET_IDENTITY_MAP_ADDR:
3742	case KVM_CAP_XEN_HVM:
3743	case KVM_CAP_VCPU_EVENTS:	3720	case KVM_CAP_VCPU_EVENTS:
3744	case KVM_CAP_HYPERV:	3721	case KVM_CAP_HYPERV:
3745	case KVM_CAP_HYPERV_VAPIC:	3722	case KVM_CAP_HYPERV_VAPIC:
@@ -3779,6 +3756,11 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 33 unmodified lines
3779	case KVM_CAP_ENFORCE_PV_FEATURE_CPUID:	3756	case KVM_CAP_ENFORCE_PV_FEATURE_CPUID:
3780	r = 1;	3757	r = 1;
3781	break;	3758	break;
		3759	case KVM_CAP_XEN_HVM:
		3760	r = KVM_XEN_HVM_CONFIG_HYPERCALL_MSR \|
		3761	KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL \|
		3762	KVM_XEN_HVM_CONFIG_SHARED_INFO;
		3763	break;
3782	case KVM_CAP_SYNC_REGS:	3764	case KVM_CAP_SYNC_REGS:
3783	r = KVM_SYNC_X86_VALID_FIELDS;	3765	r = KVM_SYNC_X86_VALID_FIELDS;
3784	break;	3766	break;
@@ -3800,10 +3782,10 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 15 unmodified lines
3800	* fringe case that is not enabled except via specific settings	3782	* fringe case that is not enabled except via specific settings
3801	* of the module parameters.	3783	* of the module parameters.
3802	*/	3784	*/
3803	r = kvm_x86_ops.has_emulated_msr(kvm, MSR_IA32_SMBASE);	3785	r = static_call(kvm_x86_has_emulated_msr)(kvm, MSR_IA32_SMBASE);
3804	break;	3786	break;
3805	case KVM_CAP_VAPIC:	3787	case KVM_CAP_VAPIC:
3806	r = !kvm_x86_ops.cpu_has_accelerated_tpr();	3788	r = !static_call(kvm_x86_cpu_has_accelerated_tpr)();
3807	break;	3789	break;
3808	case KVM_CAP_NR_VCPUS:	3790	case KVM_CAP_NR_VCPUS:
3809	r = KVM_SOFT_MAX_VCPUS;	3791	r = KVM_SOFT_MAX_VCPUS;
@@ -3845,6 +3827,13 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 35 unmodified lines
3845	case KVM_CAP_STEAL_TIME:	3827	case KVM_CAP_STEAL_TIME:
3846	r = sched_info_on();	3828	r = sched_info_on();
3847	break;	3829	break;
		3830	case KVM_CAP_X86_BUS_LOCK_EXIT:
		3831	if (kvm_has_bus_lock_exit)
		3832	r = KVM_BUS_LOCK_DETECTION_OFF \|
		3833	KVM_BUS_LOCK_DETECTION_EXIT;
		3834	else
		3835	r = 0;
		3836	break;
3848	default:	3837	default:
3849	break;	3838	break;
3850	}	3839	}
@@ -3962,14 +3951,14 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) · 111 unmodified lines
3962	{	3951	{
3963	/* Address WBINVD may be executed by guest */	3952	/* Address WBINVD may be executed by guest */
3964	if (need_emulate_wbinvd(vcpu)) {	3953	if (need_emulate_wbinvd(vcpu)) {
3965	if (kvm_x86_ops.has_wbinvd_exit())	3954	if (static_call(kvm_x86_has_wbinvd_exit)())
3966	cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);	3955	cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
3967	else if (vcpu->cpu != -1 && vcpu->cpu != cpu)	3956	else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
3968	smp_call_function_single(vcpu->cpu,	3957	smp_call_function_single(vcpu->cpu,
3969	wbinvd_ipi, NULL, 1);	3958	wbinvd_ipi, NULL, 1);
3970	}	3959	}
3971		3960
3972	kvm_x86_ops.vcpu_load(vcpu, cpu);	3961	static_call(kvm_x86_vcpu_load)(vcpu, cpu);
3973		3962
3974	/* Save host pkru register if supported */	3963	/* Save host pkru register if supported */
3975	vcpu->arch.host_pkru = read_pkru();	3964	vcpu->arch.host_pkru = read_pkru();
@@ -4015,6 +4004,7 @@ static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu) · 39 unmodified lines
4015	{	4004	{
4016	struct kvm_host_map map;	4005	struct kvm_host_map map;
4017	struct kvm_steal_time *st;	4006	struct kvm_steal_time *st;
		4007	int idx;
4018		4008
4019	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))	4009	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
4020	return;	4010	return;
@@ -4022,9 +4012,15 @@ static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu) · 1 unmodified line
4022	if (vcpu->arch.st.preempted)	4012	if (vcpu->arch.st.preempted)
4023	return;	4013	return;
4024		4014
		4015	/*
		4016	* Take the srcu lock as memslots will be accessed to check the gfn
		4017	* cache generation against the memslots generation.
		4018	*/
		4019	idx = srcu_read_lock(&vcpu->kvm->srcu);
		4020
4025	if (kvm_map_gfn(vcpu, vcpu->arch.st.msr_val >> PAGE_SHIFT, &map,	4021	if (kvm_map_gfn(vcpu, vcpu->arch.st.msr_val >> PAGE_SHIFT, &map,
4026	&vcpu->arch.st.cache, true))	4022	&vcpu->arch.st.cache, true))
4027	return;	4023	goto out;
4028		4024
4029	st = map.hva +	4025	st = map.hva +
4030	offset_in_page(vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS);	4026	offset_in_page(vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS);
@@ -4032,33 +4028,18 @@ static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu) · 1 unmodified line
4032	st->preempted = vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;	4028	st->preempted = vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;
4033		4029
4034	kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, true);	4030	kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, true);
		4031
		4032	out:
		4033	srcu_read_unlock(&vcpu->kvm->srcu, idx);
4035	}	4034	}
4036		4035
4037	void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)	4036	void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
4038	{	4037	{
4039	int idx;
4040
4041	if (vcpu->preempted && !vcpu->arch.guest_state_protected)	4038	if (vcpu->preempted && !vcpu->arch.guest_state_protected)
4042	vcpu->arch.preempted_in_kernel = !kvm_x86_ops.get_cpl(vcpu);	4039	vcpu->arch.preempted_in_kernel = !static_call(kvm_x86_get_cpl)(vcpu);
4043		4040
4044	/*
4045	* Disable page faults because we're in atomic context here.
4046	* kvm_write_guest_offset_cached() would call might_fault()
4047	* that relies on pagefault_disable() to tell if there's a
4048	* bug. NOTE: the write to guest memory may not go through if
4049	* during postcopy live migration or if there's heavy guest
4050	* paging.
4051	*/
4052	pagefault_disable();
4053	/*
4054	* kvm_memslots() will be called by
4055	* kvm_write_guest_offset_cached() so take the srcu lock.
4056	*/
4057	idx = srcu_read_lock(&vcpu->kvm->srcu);
4058	kvm_steal_time_set_preempted(vcpu);	4041	kvm_steal_time_set_preempted(vcpu);
4059	srcu_read_unlock(&vcpu->kvm->srcu, idx);	4042	static_call(kvm_x86_vcpu_put)(vcpu);
4060	pagefault_enable();
4061	kvm_x86_ops.vcpu_put(vcpu);
4062	vcpu->arch.last_host_tsc = rdtsc();	4043	vcpu->arch.last_host_tsc = rdtsc();
4063	/*	4044	/*
4064	* If userspace has set any breakpoints or watchpoints, dr6 is restored	4045	* If userspace has set any breakpoints or watchpoints, dr6 is restored
@@ -4072,7 +4053,7 @@ static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, · 7 unmodified lines
4072	struct kvm_lapic_state *s)	4053	struct kvm_lapic_state *s)
4073	{	4054	{
4074	if (vcpu->arch.apicv_active)	4055	if (vcpu->arch.apicv_active)
4075	kvm_x86_ops.sync_pir_to_irr(vcpu);	4056	static_call(kvm_x86_sync_pir_to_irr)(vcpu);
4076		4057
4077	return kvm_apic_get_state(vcpu, s);	4058	return kvm_apic_get_state(vcpu, s);
4078	}	4059	}
@@ -4182,7 +4163,7 @@ static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu, · 103 unmodified lines
4182	for (bank = 0; bank < bank_num; bank++)	4163	for (bank = 0; bank < bank_num; bank++)
4183	vcpu->arch.mce_banks[bank*4] = ~(u64)0;	4164	vcpu->arch.mce_banks[bank*4] = ~(u64)0;
4184		4165
4185	kvm_x86_ops.setup_mce(vcpu);	4166	static_call(kvm_x86_setup_mce)(vcpu);
4186	out:	4167	out:
4187	return r;	4168	return r;
4188	}	4169	}
@@ -4289,11 +4270,11 @@ static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, · 100 unmodified lines
4289	vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;	4270	vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;
4290	events->interrupt.nr = vcpu->arch.interrupt.nr;	4271	events->interrupt.nr = vcpu->arch.interrupt.nr;
4291	events->interrupt.soft = 0;	4272	events->interrupt.soft = 0;
4292	events->interrupt.shadow = kvm_x86_ops.get_interrupt_shadow(vcpu);	4273	events->interrupt.shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
4293		4274
4294	events->nmi.injected = vcpu->arch.nmi_injected;	4275	events->nmi.injected = vcpu->arch.nmi_injected;
4295	events->nmi.pending = vcpu->arch.nmi_pending != 0;	4276	events->nmi.pending = vcpu->arch.nmi_pending != 0;
4296	events->nmi.masked = kvm_x86_ops.get_nmi_mask(vcpu);	4277	events->nmi.masked = static_call(kvm_x86_get_nmi_mask)(vcpu);
4297	events->nmi.pad = 0;	4278	events->nmi.pad = 0;
4298		4279
4299	events->sipi_vector = 0; /* never valid when reporting to user space */	4280	events->sipi_vector = 0; /* never valid when reporting to user space */
@@ -4360,13 +4341,13 @@ static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, · 60 unmodified lines
4360	vcpu->arch.interrupt.nr = events->interrupt.nr;	4341	vcpu->arch.interrupt.nr = events->interrupt.nr;
4361	vcpu->arch.interrupt.soft = events->interrupt.soft;	4342	vcpu->arch.interrupt.soft = events->interrupt.soft;
4362	if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)	4343	if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
4363	kvm_x86_ops.set_interrupt_shadow(vcpu,	4344	static_call(kvm_x86_set_interrupt_shadow)(vcpu,
4364	events->interrupt.shadow);	4345	events->interrupt.shadow);
4365		4346
4366	vcpu->arch.nmi_injected = events->nmi.injected;	4347	vcpu->arch.nmi_injected = events->nmi.injected;
4367	if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)	4348	if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
4368	vcpu->arch.nmi_pending = events->nmi.pending;	4349	vcpu->arch.nmi_pending = events->nmi.pending;
4369	kvm_x86_ops.set_nmi_mask(vcpu, events->nmi.masked);	4350	static_call(kvm_x86_set_nmi_mask)(vcpu, events->nmi.masked);
4370		4351
4371	if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&	4352	if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&
4372	lapic_in_kernel(vcpu))	4353	lapic_in_kernel(vcpu))
@@ -4422,9 +4403,9 @@ static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, · 49 unmodified lines
4422	if (dbgregs->flags)	4403	if (dbgregs->flags)
4423	return -EINVAL;	4404	return -EINVAL;
4424		4405
4425	if (dbgregs->dr6 & ~0xffffffffull)	4406	if (!kvm_dr6_valid(dbgregs->dr6))
4426	return -EINVAL;	4407	return -EINVAL;
4427	if (dbgregs->dr7 & ~0xffffffffull)	4408	if (!kvm_dr7_valid(dbgregs->dr7))
4428	return -EINVAL;	4409	return -EINVAL;
4429		4410
4430	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));	4411	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
@@ -4661,7 +4642,7 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, · 230 unmodified lines
4661	if (!kvm_x86_ops.enable_direct_tlbflush)	4642	if (!kvm_x86_ops.enable_direct_tlbflush)
4662	return -ENOTTY;	4643	return -ENOTTY;
4663		4644
4664	return kvm_x86_ops.enable_direct_tlbflush(vcpu);	4645	return static_call(kvm_x86_enable_direct_tlbflush)(vcpu);
4665		4646
4666	case KVM_CAP_ENFORCE_PV_FEATURE_CPUID:	4647	case KVM_CAP_ENFORCE_PV_FEATURE_CPUID:
4667	vcpu->arch.pv_cpuid.enforce = cap->args[0];	4648	vcpu->arch.pv_cpuid.enforce = cap->args[0];
@@ -5032,6 +5013,26 @@ long kvm_arch_vcpu_ioctl(struct file *filp, · 364 unmodified lines
5032	case KVM_GET_SUPPORTED_HV_CPUID:	5013	case KVM_GET_SUPPORTED_HV_CPUID:
5033	r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp);	5014	r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp);
5034	break;	5015	break;
		5016	case KVM_XEN_VCPU_GET_ATTR: {
		5017	struct kvm_xen_vcpu_attr xva;
		5018
		5019	r = -EFAULT;
		5020	if (copy_from_user(&xva, argp, sizeof(xva)))
		5021	goto out;
		5022	r = kvm_xen_vcpu_get_attr(vcpu, &xva);
		5023	if (!r && copy_to_user(argp, &xva, sizeof(xva)))
		5024	r = -EFAULT;
		5025	break;
		5026	}
		5027	case KVM_XEN_VCPU_SET_ATTR: {
		5028	struct kvm_xen_vcpu_attr xva;
		5029
		5030	r = -EFAULT;
		5031	if (copy_from_user(&xva, argp, sizeof(xva)))
		5032	goto out;
		5033	r = kvm_xen_vcpu_set_attr(vcpu, &xva);
		5034	break;
		5035	}
5035	default:	5036	default:
5036	r = -EINVAL;	5037	r = -EINVAL;
5037	}	5038	}
@@ -5053,14 +5054,14 @@ static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr) · 15 unmodified lines
5053		5054
5054	if (addr > (unsigned int)(-3 * PAGE_SIZE))	5055	if (addr > (unsigned int)(-3 * PAGE_SIZE))
5055	return -EINVAL;	5056	return -EINVAL;
5056	ret = kvm_x86_ops.set_tss_addr(kvm, addr);	5057	ret = static_call(kvm_x86_set_tss_addr)(kvm, addr);
5057	return ret;	5058	return ret;
5058	}	5059	}
5059		5060
5060	static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,	5061	static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
5061	u64 ident_addr)	5062	u64 ident_addr)
5062	{	5063	{
5063	return kvm_x86_ops.set_identity_map_addr(kvm, ident_addr);	5064	return static_call(kvm_x86_set_identity_map_addr)(kvm, ident_addr);
5064	}	5065	}
5065		5066
5066	static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,	5067	static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
@@ -5217,8 +5218,7 @@ void kvm_arch_sync_dirty_log(struct kvm kvm, struct kvm_memory_slot memslot) · 150 unmodified lines
5217	/*	5218	/*
5218	* Flush potentially hardware-cached dirty pages to dirty_bitmap.	5219	* Flush potentially hardware-cached dirty pages to dirty_bitmap.
5219	*/	5220	*/
5220	if (kvm_x86_ops.flush_log_dirty)	5221	static_call_cond(kvm_x86_flush_log_dirty)(kvm);
5221	kvm_x86_ops.flush_log_dirty(kvm);
5222	}	5222	}
5223		5223
5224	int kvm_vm_ioctl_irq_line(struct kvm kvm, struct kvm_irq_level irq_event,	5224	int kvm_vm_ioctl_irq_line(struct kvm kvm, struct kvm_irq_level irq_event,
@@ -5308,6 +5308,20 @@ split_irqchip_unlock: · 83 unmodified lines
5308	kvm->arch.user_space_msr_mask = cap->args[0];	5308	kvm->arch.user_space_msr_mask = cap->args[0];
5309	r = 0;	5309	r = 0;
5310	break;	5310	break;
		5311	case KVM_CAP_X86_BUS_LOCK_EXIT:
		5312	r = -EINVAL;
		5313	if (cap->args[0] & ~KVM_BUS_LOCK_DETECTION_VALID_MODE)
		5314	break;
		5315
		5316	if ((cap->args[0] & KVM_BUS_LOCK_DETECTION_OFF) &&
		5317	(cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT))
		5318	break;
		5319
		5320	if (kvm_has_bus_lock_exit &&
		5321	cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT)
		5322	kvm->arch.bus_lock_detection_enabled = true;
		5323	r = 0;
		5324	break;
5311	default:	5325	default:
5312	r = -EINVAL;	5326	r = -EINVAL;
5313	break;	5327	break;
@@ -5637,11 +5651,27 @@ set_pit2_out: · 323 unmodified lines
5637	r = -EFAULT;	5651	r = -EFAULT;
5638	if (copy_from_user(&xhc, argp, sizeof(xhc)))	5652	if (copy_from_user(&xhc, argp, sizeof(xhc)))
5639	goto out;	5653	goto out;
5640	r = -EINVAL;	5654	r = kvm_xen_hvm_config(kvm, &xhc);
5641	if (xhc.flags)	5655	break;
		5656	}
		5657	case KVM_XEN_HVM_GET_ATTR: {
		5658	struct kvm_xen_hvm_attr xha;
		5659
		5660	r = -EFAULT;
		5661	if (copy_from_user(&xha, argp, sizeof(xha)))
5642	goto out;	5662	goto out;
5643	memcpy(&kvm->arch.xen_hvm_config, &xhc, sizeof(xhc));	5663	r = kvm_xen_hvm_get_attr(kvm, &xha);
5644	r = 0;	5664	if (!r && copy_to_user(argp, &xha, sizeof(xha)))
		5665	r = -EFAULT;
		5666	break;
		5667	}
		5668	case KVM_XEN_HVM_SET_ATTR: {
		5669	struct kvm_xen_hvm_attr xha;
		5670
		5671	r = -EFAULT;
		5672	if (copy_from_user(&xha, argp, sizeof(xha)))
		5673	goto out;
		5674	r = kvm_xen_hvm_set_attr(kvm, &xha);
5645	break;	5675	break;
5646	}	5676	}
5647	case KVM_SET_CLOCK: {	5677	case KVM_SET_CLOCK: {
@@ -5686,7 +5716,7 @@ set_pit2_out: · 38 unmodified lines
5686	case KVM_MEMORY_ENCRYPT_OP: {	5716	case KVM_MEMORY_ENCRYPT_OP: {
5687	r = -ENOTTY;	5717	r = -ENOTTY;
5688	if (kvm_x86_ops.mem_enc_op)	5718	if (kvm_x86_ops.mem_enc_op)
5689	r = kvm_x86_ops.mem_enc_op(kvm, argp);	5719	r = static_call(kvm_x86_mem_enc_op)(kvm, argp);
5690	break;	5720	break;
5691	}	5721	}
5692	case KVM_MEMORY_ENCRYPT_REG_REGION: {	5722	case KVM_MEMORY_ENCRYPT_REG_REGION: {
@@ -5698,7 +5728,7 @@ set_pit2_out: · 5 unmodified lines
5698		5728
5699	r = -ENOTTY;	5729	r = -ENOTTY;
5700	if (kvm_x86_ops.mem_enc_reg_region)	5730	if (kvm_x86_ops.mem_enc_reg_region)
5701	r = kvm_x86_ops.mem_enc_reg_region(kvm, &region);	5731	r = static_call(kvm_x86_mem_enc_reg_region)(kvm, &region);
5702	break;	5732	break;
5703	}	5733	}
5704	case KVM_MEMORY_ENCRYPT_UNREG_REGION: {	5734	case KVM_MEMORY_ENCRYPT_UNREG_REGION: {
@@ -5710,7 +5740,7 @@ set_pit2_out: · 5 unmodified lines
5710		5740
5711	r = -ENOTTY;	5741	r = -ENOTTY;
5712	if (kvm_x86_ops.mem_enc_unreg_region)	5742	if (kvm_x86_ops.mem_enc_unreg_region)
5713	r = kvm_x86_ops.mem_enc_unreg_region(kvm, &region);	5743	r = static_call(kvm_x86_mem_enc_unreg_region)(kvm, &region);
5714	break;	5744	break;
5715	}	5745	}
5716	case KVM_HYPERV_EVENTFD: {	5746	case KVM_HYPERV_EVENTFD: {
@@ -5812,7 +5842,7 @@ static void kvm_init_msr_list(void) · 95 unmodified lines
5812	}	5842	}
5813		5843
5814	for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) {	5844	for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) {
5815	if (!kvm_x86_ops.has_emulated_msr(NULL, emulated_msrs_all[i]))	5845	if (!static_call(kvm_x86_has_emulated_msr)(NULL, emulated_msrs_all[i]))
5816	continue;	5846	continue;
5817		5847
5818	emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i];	5848	emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i];
@@ -5875,13 +5905,13 @@ static int vcpu_mmio_read(struct kvm_vcpu vcpu, gpa_t addr, int len, void v) · 56 unmodified lines
5875	static void kvm_set_segment(struct kvm_vcpu *vcpu,	5905	static void kvm_set_segment(struct kvm_vcpu *vcpu,
5876	struct kvm_segment *var, int seg)	5906	struct kvm_segment *var, int seg)
5877	{	5907	{
5878	kvm_x86_ops.set_segment(vcpu, var, seg);	5908	static_call(kvm_x86_set_segment)(vcpu, var, seg);
5879	}	5909	}
5880		5910
5881	void kvm_get_segment(struct kvm_vcpu *vcpu,	5911	void kvm_get_segment(struct kvm_vcpu *vcpu,
5882	struct kvm_segment *var, int seg)	5912	struct kvm_segment *var, int seg)
5883	{	5913	{
5884	kvm_x86_ops.get_segment(vcpu, var, seg);	5914	static_call(kvm_x86_get_segment)(vcpu, var, seg);
5885	}	5915	}
5886		5916
5887	gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,	5917	gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
@@ -5901,14 +5931,14 @@ gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, · 13 unmodified lines
5901	gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,	5931	gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
5902	struct x86_exception *exception)	5932	struct x86_exception *exception)
5903	{	5933	{
5904	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;	5934	u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
5905	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);	5935	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5906	}	5936	}
5907		5937
5908	gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,	5938	gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
5909	struct x86_exception *exception)	5939	struct x86_exception *exception)
5910	{	5940	{
5911	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;	5941	u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
5912	access \|= PFERR_FETCH_MASK;	5942	access \|= PFERR_FETCH_MASK;
5913	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);	5943	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5914	}	5944	}
@@ -5916,7 +5946,7 @@ gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, · 1 unmodified line
5916	gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,	5946	gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
5917	struct x86_exception *exception)	5947	struct x86_exception *exception)
5918	{	5948	{
5919	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;	5949	u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
5920	access \|= PFERR_WRITE_MASK;	5950	access \|= PFERR_WRITE_MASK;
5921	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);	5951	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5922	}	5952	}
@@ -5965,7 +5995,7 @@ static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt, · 42 unmodified lines
5965	struct x86_exception *exception)	5995	struct x86_exception *exception)
5966	{	5996	{
5967	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);	5997	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5968	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;	5998	u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
5969	unsigned offset;	5999	unsigned offset;
5970	int ret;	6000	int ret;
5971		6001
@@ -5990,7 +6020,7 @@ int kvm_read_guest_virt(struct kvm_vcpu *vcpu, · 18 unmodified lines
5990	gva_t addr, void *val, unsigned int bytes,	6020	gva_t addr, void *val, unsigned int bytes,
5991	struct x86_exception *exception)	6021	struct x86_exception *exception)
5992	{	6022	{
5993	u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;	6023	u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
5994		6024
5995	/*	6025	/*
5996	* FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED	6026	* FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
@@ -6011,7 +6041,7 @@ static int emulator_read_std(struct x86_emulate_ctxt *ctxt, · 14 unmodified lines
6011	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);	6041	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6012	u32 access = 0;	6042	u32 access = 0;
6013		6043
6014	if (!system && kvm_x86_ops.get_cpl(vcpu) == 3)	6044	if (!system && static_call(kvm_x86_get_cpl)(vcpu) == 3)
6015	access \|= PFERR_USER_MASK;	6045	access \|= PFERR_USER_MASK;
6016		6046
6017	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);	6047	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);
@@ -6064,7 +6094,7 @@ static int emulator_write_std(struct x86_emulate_ctxt ctxt, gva_t addr, void v · 46 unmodified lines
6064	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);	6094	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6065	u32 access = PFERR_WRITE_MASK;	6095	u32 access = PFERR_WRITE_MASK;
6066		6096
6067	if (!system && kvm_x86_ops.get_cpl(vcpu) == 3)	6097	if (!system && static_call(kvm_x86_get_cpl)(vcpu) == 3)
6068	access \|= PFERR_USER_MASK;	6098	access \|= PFERR_USER_MASK;
6069		6099
6070	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,	6100	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
@@ -6089,7 +6119,7 @@ int handle_ud(struct kvm_vcpu *vcpu) · 18 unmodified lines
6089	char sig[5]; /* ud2; .ascii "kvm" */	6119	char sig[5]; /* ud2; .ascii "kvm" */
6090	struct x86_exception e;	6120	struct x86_exception e;
6091		6121
6092	if (unlikely(!kvm_x86_ops.can_emulate_instruction(vcpu, NULL, 0)))	6122	if (unlikely(!static_call(kvm_x86_can_emulate_instruction)(vcpu, NULL, 0)))
6093	return 1;	6123	return 1;
6094		6124
6095	if (force_emulation_prefix &&	6125	if (force_emulation_prefix &&
@@ -6123,7 +6153,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, · 27 unmodified lines
6123	gpa_t gpa, struct x86_exception exception,	6153	gpa_t gpa, struct x86_exception exception,
6124	bool write)	6154	bool write)
6125	{	6155	{
6126	u32 access = ((kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)	6156	u32 access = ((static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0)
6127	\| (write ? PFERR_WRITE_MASK : 0);	6157	\| (write ? PFERR_WRITE_MASK : 0);
6128		6158
6129	/*	6159	/*
@@ -6531,7 +6561,7 @@ static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt, · 401 unmodified lines
6531		6561
6532	static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)	6562	static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
6533	{	6563	{
6534	return kvm_x86_ops.get_segment_base(vcpu, seg);	6564	return static_call(kvm_x86_get_segment_base)(vcpu, seg);
6535	}	6565	}
6536		6566
6537	static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)	6567	static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
@@ -6544,7 +6574,7 @@ static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu) · 6 unmodified lines
6544	if (!need_emulate_wbinvd(vcpu))	6574	if (!need_emulate_wbinvd(vcpu))
6545	return X86EMUL_CONTINUE;	6575	return X86EMUL_CONTINUE;
6546		6576
6547	if (kvm_x86_ops.has_wbinvd_exit()) {	6577	if (static_call(kvm_x86_has_wbinvd_exit)()) {
6548	int cpu = get_cpu();	6578	int cpu = get_cpu();
6549		6579
6550	cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);	6580	cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
@@ -6571,17 +6601,17 @@ static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt) · 20 unmodified lines
6571	kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));	6601	kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));
6572	}	6602	}
6573		6603
6574	static int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,	6604	static void emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,
6575	unsigned long *dest)	6605	unsigned long *dest)
6576	{	6606	{
6577	return kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);	6607	kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
6578	}	6608	}
6579		6609
6580	static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,	6610	static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,
6581	unsigned long value)	6611	unsigned long value)
6582	{	6612	{
6583		6613
6584	return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);	6614	return kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
6585	}	6615	}
6586		6616
6587	static u64 mk_cr_64(u64 curr_cr, u32 new_val)	6617	static u64 mk_cr_64(u64 curr_cr, u32 new_val)
@@ -6649,27 +6679,27 @@ static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val) · 61 unmodified lines
6649		6679
6650	static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)	6680	static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
6651	{	6681	{
6652	return kvm_x86_ops.get_cpl(emul_to_vcpu(ctxt));	6682	return static_call(kvm_x86_get_cpl)(emul_to_vcpu(ctxt));
6653	}	6683	}
6654		6684
6655	static void emulator_get_gdt(struct x86_emulate_ctxt ctxt, struct desc_ptr dt)	6685	static void emulator_get_gdt(struct x86_emulate_ctxt ctxt, struct desc_ptr dt)
6656	{	6686	{
6657	kvm_x86_ops.get_gdt(emul_to_vcpu(ctxt), dt);	6687	static_call(kvm_x86_get_gdt)(emul_to_vcpu(ctxt), dt);
6658	}	6688	}
6659		6689
6660	static void emulator_get_idt(struct x86_emulate_ctxt ctxt, struct desc_ptr dt)	6690	static void emulator_get_idt(struct x86_emulate_ctxt ctxt, struct desc_ptr dt)
6661	{	6691	{
6662	kvm_x86_ops.get_idt(emul_to_vcpu(ctxt), dt);	6692	static_call(kvm_x86_get_idt)(emul_to_vcpu(ctxt), dt);
6663	}	6693	}
6664		6694
6665	static void emulator_set_gdt(struct x86_emulate_ctxt ctxt, struct desc_ptr dt)	6695	static void emulator_set_gdt(struct x86_emulate_ctxt ctxt, struct desc_ptr dt)
6666	{	6696	{
6667	kvm_x86_ops.set_gdt(emul_to_vcpu(ctxt), dt);	6697	static_call(kvm_x86_set_gdt)(emul_to_vcpu(ctxt), dt);
6668	}	6698	}
6669		6699
6670	static void emulator_set_idt(struct x86_emulate_ctxt ctxt, struct desc_ptr dt)	6700	static void emulator_set_idt(struct x86_emulate_ctxt ctxt, struct desc_ptr dt)
6671	{	6701	{
6672	kvm_x86_ops.set_idt(emul_to_vcpu(ctxt), dt);	6702	static_call(kvm_x86_set_idt)(emul_to_vcpu(ctxt), dt);
6673	}	6703	}
6674		6704
6675	static unsigned long emulator_get_cached_segment_base(	6705	static unsigned long emulator_get_cached_segment_base(
@@ -6811,7 +6841,7 @@ static int emulator_intercept(struct x86_emulate_ctxt *ctxt, · 135 unmodified lines
6811	struct x86_instruction_info *info,	6841	struct x86_instruction_info *info,
6812	enum x86_intercept_stage stage)	6842	enum x86_intercept_stage stage)
6813	{	6843	{
6814	return kvm_x86_ops.check_intercept(emul_to_vcpu(ctxt), info, stage,	6844	return static_call(kvm_x86_check_intercept)(emul_to_vcpu(ctxt), info, stage,
6815	&ctxt->exception);	6845	&ctxt->exception);
6816	}	6846	}
6817		6847
@@ -6849,7 +6879,7 @@ static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulon · 31 unmodified lines
6849		6879
6850	static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked)	6880	static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked)
6851	{	6881	{
6852	kvm_x86_ops.set_nmi_mask(emul_to_vcpu(ctxt), masked);	6882	static_call(kvm_x86_set_nmi_mask)(emul_to_vcpu(ctxt), masked);
6853	}	6883	}
6854		6884
6855	static unsigned emulator_get_hflags(struct x86_emulate_ctxt *ctxt)	6885	static unsigned emulator_get_hflags(struct x86_emulate_ctxt *ctxt)
@@ -6865,7 +6895,7 @@ static void emulator_set_hflags(struct x86_emulate_ctxt *ctxt, unsigned emul_fla · 9 unmodified lines
6865	static int emulator_pre_leave_smm(struct x86_emulate_ctxt *ctxt,	6895	static int emulator_pre_leave_smm(struct x86_emulate_ctxt *ctxt,
6866	const char *smstate)	6896	const char *smstate)
6867	{	6897	{
6868	return kvm_x86_ops.pre_leave_smm(emul_to_vcpu(ctxt), smstate);	6898	return static_call(kvm_x86_pre_leave_smm)(emul_to_vcpu(ctxt), smstate);
6869	}	6899	}
6870		6900
6871	static void emulator_post_leave_smm(struct x86_emulate_ctxt *ctxt)	6901	static void emulator_post_leave_smm(struct x86_emulate_ctxt *ctxt)
@@ -6927,7 +6957,7 @@ static const struct x86_emulate_ops emulate_ops = { · 55 unmodified lines
6927		6957
6928	static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)	6958	static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
6929	{	6959	{
6930	u32 int_shadow = kvm_x86_ops.get_interrupt_shadow(vcpu);	6960	u32 int_shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
6931	/*	6961	/*
6932	* an sti; sti; sequence only disable interrupts for the first	6962	* an sti; sti; sequence only disable interrupts for the first
6933	* instruction. So, if the last instruction, be it emulated or	6963	* instruction. So, if the last instruction, be it emulated or
@@ -6938,7 +6968,7 @@ static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) · 4 unmodified lines
6938	if (int_shadow & mask)	6968	if (int_shadow & mask)
6939	mask = 0;	6969	mask = 0;
6940	if (unlikely(int_shadow \|\| mask)) {	6970	if (unlikely(int_shadow \|\| mask)) {
6941	kvm_x86_ops.set_interrupt_shadow(vcpu, mask);	6971	static_call(kvm_x86_set_interrupt_shadow)(vcpu, mask);
6942	if (!mask)	6972	if (!mask)
6943	kvm_make_request(KVM_REQ_EVENT, vcpu);	6973	kvm_make_request(KVM_REQ_EVENT, vcpu);
6944	}	6974	}
@@ -6980,7 +7010,7 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu) · 35 unmodified lines
6980	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;	7010	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
6981	int cs_db, cs_l;	7011	int cs_db, cs_l;
6982		7012
6983	kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l);	7013	static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
6984		7014
6985	ctxt->gpa_available = false;	7015	ctxt->gpa_available = false;
6986	ctxt->eflags = kvm_get_rflags(vcpu);	7016	ctxt->eflags = kvm_get_rflags(vcpu);
@@ -7041,7 +7071,7 @@ static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type) · 54 unmodified lines
7041		7071
7042	kvm_queue_exception(vcpu, UD_VECTOR);	7072	kvm_queue_exception(vcpu, UD_VECTOR);
7043		7073
7044	if (!is_guest_mode(vcpu) && kvm_x86_ops.get_cpl(vcpu) == 0) {	7074	if (!is_guest_mode(vcpu) && static_call(kvm_x86_get_cpl)(vcpu) == 0) {
7045	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;	7075	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
7046	vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;	7076	vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
7047	vcpu->run->internal.ndata = 0;	7077	vcpu->run->internal.ndata = 0;
@@ -7101,9 +7131,9 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, · 53 unmodified lines
7101	if (vcpu->arch.mmu->direct_map) {	7131	if (vcpu->arch.mmu->direct_map) {
7102	unsigned int indirect_shadow_pages;	7132	unsigned int indirect_shadow_pages;
7103		7133
7104	spin_lock(&vcpu->kvm->mmu_lock);	7134	write_lock(&vcpu->kvm->mmu_lock);
7105	indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;	7135	indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
7106	spin_unlock(&vcpu->kvm->mmu_lock);	7136	write_unlock(&vcpu->kvm->mmu_lock);
7107		7137
7108	if (indirect_shadow_pages)	7138	if (indirect_shadow_pages)
7109	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));	7139	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
@@ -7210,7 +7240,7 @@ static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu) · 100 unmodified lines
7210	struct kvm_run *kvm_run = vcpu->run;	7240	struct kvm_run *kvm_run = vcpu->run;
7211		7241
7212	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {	7242	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
7213	kvm_run->debug.arch.dr6 = DR6_BS \| DR6_FIXED_1 \| DR6_RTM;	7243	kvm_run->debug.arch.dr6 = DR6_BS \| DR6_ACTIVE_LOW;
7214	kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);	7244	kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);
7215	kvm_run->debug.arch.exception = DB_VECTOR;	7245	kvm_run->debug.arch.exception = DB_VECTOR;
7216	kvm_run->exit_reason = KVM_EXIT_DEBUG;	7246	kvm_run->exit_reason = KVM_EXIT_DEBUG;
@@ -7222,10 +7252,10 @@ static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu) · 5 unmodified lines
7222		7252
7223	int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)	7253	int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
7224	{	7254	{
7225	unsigned long rflags = kvm_x86_ops.get_rflags(vcpu);	7255	unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
7226	int r;	7256	int r;
7227		7257
7228	r = kvm_x86_ops.skip_emulated_instruction(vcpu);	7258	r = static_call(kvm_x86_skip_emulated_instruction)(vcpu);
7229	if (unlikely(!r))	7259	if (unlikely(!r))
7230	return 0;	7260	return 0;
7231		7261
@@ -7254,7 +7284,7 @@ static bool kvm_vcpu_check_breakpoint(struct kvm_vcpu vcpu, int r) · 22 unmodified lines
7254	vcpu->arch.eff_db);	7284	vcpu->arch.eff_db);
7255		7285
7256	if (dr6 != 0) {	7286	if (dr6 != 0) {
7257	kvm_run->debug.arch.dr6 = dr6 \| DR6_FIXED_1 \| DR6_RTM;	7287	kvm_run->debug.arch.dr6 = dr6 \| DR6_ACTIVE_LOW;
7258	kvm_run->debug.arch.pc = eip;	7288	kvm_run->debug.arch.pc = eip;
7259	kvm_run->debug.arch.exception = DB_VECTOR;	7289	kvm_run->debug.arch.exception = DB_VECTOR;
7260	kvm_run->exit_reason = KVM_EXIT_DEBUG;	7290	kvm_run->exit_reason = KVM_EXIT_DEBUG;
@@ -7311,6 +7341,42 @@ static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt) · 50 unmodified lines
7311	return false;	7341	return false;
7312	}	7342	}
7313		7343
		7344	/*
		7345	* Decode to be emulated instruction. Return EMULATION_OK if success.
		7346	*/
		7347	int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type,
		7348	void *insn, int insn_len)
		7349	{
		7350	int r = EMULATION_OK;
		7351	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
		7352
		7353	init_emulate_ctxt(vcpu);
		7354
		7355	/*
		7356	* We will reenter on the same instruction since we do not set
		7357	* complete_userspace_io. This does not handle watchpoints yet,
		7358	* those would be handled in the emulate_ops.
		7359	*/
		7360	if (!(emulation_type & EMULTYPE_SKIP) &&
		7361	kvm_vcpu_check_breakpoint(vcpu, &r))
		7362	return r;
		7363
		7364	ctxt->interruptibility = 0;
		7365	ctxt->have_exception = false;
		7366	ctxt->exception.vector = -1;
		7367	ctxt->perm_ok = false;
		7368
		7369	ctxt->ud = emulation_type & EMULTYPE_TRAP_UD;
		7370
		7371	r = x86_decode_insn(ctxt, insn, insn_len);
		7372
		7373	trace_kvm_emulate_insn_start(vcpu);
		7374	++vcpu->stat.insn_emulation;
		7375
		7376	return r;
		7377	}
		7378	EXPORT_SYMBOL_GPL(x86_decode_emulated_instruction);
		7379
7314	int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,	7380	int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
7315	int emulation_type, void *insn, int insn_len)	7381	int emulation_type, void *insn, int insn_len)
7316	{	7382	{
@@ -7319,7 +7385,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, · 2 unmodified lines
7319	bool writeback = true;	7385	bool writeback = true;
7320	bool write_fault_to_spt;	7386	bool write_fault_to_spt;
7321		7387
7322	if (unlikely(!kvm_x86_ops.can_emulate_instruction(vcpu, insn, insn_len)))	7388	if (unlikely(!static_call(kvm_x86_can_emulate_instruction)(vcpu, insn, insn_len)))
7323	return 1;	7389	return 1;
7324		7390
7325	vcpu->arch.l1tf_flush_l1d = true;	7391	vcpu->arch.l1tf_flush_l1d = true;
@@ -7330,32 +7396,12 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, · 4 unmodified lines
7330	*/	7396	*/
7331	write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;	7397	write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
7332	vcpu->arch.write_fault_to_shadow_pgtable = false;	7398	vcpu->arch.write_fault_to_shadow_pgtable = false;
7333	kvm_clear_exception_queue(vcpu);
7334		7399
7335	if (!(emulation_type & EMULTYPE_NO_DECODE)) {	7400	if (!(emulation_type & EMULTYPE_NO_DECODE)) {
7336	init_emulate_ctxt(vcpu);	7401	kvm_clear_exception_queue(vcpu);
7337
7338	/*
7339	* We will reenter on the same instruction since
7340	* we do not set complete_userspace_io. This does not
7341	* handle watchpoints yet, those would be handled in
7342	* the emulate_ops.
7343	*/
7344	if (!(emulation_type & EMULTYPE_SKIP) &&
7345	kvm_vcpu_check_breakpoint(vcpu, &r))
7346	return r;
7347		7402
7348	ctxt->interruptibility = 0;	7403	r = x86_decode_emulated_instruction(vcpu, emulation_type,
7349	ctxt->have_exception = false;	7404	insn, insn_len);
7350	ctxt->exception.vector = -1;
7351	ctxt->perm_ok = false;
7352
7353	ctxt->ud = emulation_type & EMULTYPE_TRAP_UD;
7354
7355	r = x86_decode_insn(ctxt, insn, insn_len);
7356
7357	trace_kvm_emulate_insn_start(vcpu);
7358	++vcpu->stat.insn_emulation;
7359	if (r != EMULATION_OK) {	7405	if (r != EMULATION_OK) {
7360	if ((emulation_type & EMULTYPE_TRAP_UD) \|\|	7406	if ((emulation_type & EMULTYPE_TRAP_UD) \|\|
7361	(emulation_type & EMULTYPE_TRAP_UD_FORCED)) {	7407	(emulation_type & EMULTYPE_TRAP_UD_FORCED)) {
@@ -7462,7 +7508,7 @@ restart: · 100 unmodified lines
7462	r = 1;	7508	r = 1;
7463		7509
7464	if (writeback) {	7510	if (writeback) {
7465	unsigned long rflags = kvm_x86_ops.get_rflags(vcpu);	7511	unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
7466	toggle_interruptibility(vcpu, ctxt->interruptibility);	7512	toggle_interruptibility(vcpu, ctxt->interruptibility);
7467	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;	7513	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
7468	if (!ctxt->have_exception \|\|	7514	if (!ctxt->have_exception \|\|
@@ -7471,7 +7517,7 @@ restart: · 2 unmodified lines
7471	if (r && (ctxt->tf \|\| (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))	7517	if (r && (ctxt->tf \|\| (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))
7472	r = kvm_vcpu_do_singlestep(vcpu);	7518	r = kvm_vcpu_do_singlestep(vcpu);
7473	if (kvm_x86_ops.update_emulated_instruction)	7519	if (kvm_x86_ops.update_emulated_instruction)
7474	kvm_x86_ops.update_emulated_instruction(vcpu);	7520	static_call(kvm_x86_update_emulated_instruction)(vcpu);
7475	__kvm_set_rflags(vcpu, ctxt->eflags);	7521	__kvm_set_rflags(vcpu, ctxt->eflags);
7476	}	7522	}
7477		7523
@@ -7800,7 +7846,7 @@ static int kvm_is_user_mode(void) · 322 unmodified lines
7800	int user_mode = 3;	7846	int user_mode = 3;
7801		7847
7802	if (__this_cpu_read(current_vcpu))	7848	if (__this_cpu_read(current_vcpu))
7803	user_mode = kvm_x86_ops.get_cpl(__this_cpu_read(current_vcpu));	7849	user_mode = static_call(kvm_x86_get_cpl)(__this_cpu_read(current_vcpu));
7804		7850
7805	return user_mode != 0;	7851	return user_mode != 0;
7806	}	7852	}
@@ -7945,7 +7991,6 @@ int kvm_arch_init(void *opaque) · 138 unmodified lines
7945	supported_xcr0 = host_xcr0 & KVM_SUPPORTED_XCR0;	7991	supported_xcr0 = host_xcr0 & KVM_SUPPORTED_XCR0;
7946	}	7992	}
7947		7993
7948	kvm_lapic_init();
7949	if (pi_inject_timer == -1)	7994	if (pi_inject_timer == -1)
7950	pi_inject_timer = housekeeping_enabled(HK_FLAG_TIMER);	7995	pi_inject_timer = housekeeping_enabled(HK_FLAG_TIMER);
7951	#ifdef CONFIG_X86_64	7996	#ifdef CONFIG_X86_64
@@ -7987,6 +8032,7 @@ void kvm_arch_exit(void) · 35 unmodified lines
7987	kvm_mmu_module_exit();	8032	kvm_mmu_module_exit();
7988	free_percpu(user_return_msrs);	8033	free_percpu(user_return_msrs);
7989	kmem_cache_destroy(x86_fpu_cache);	8034	kmem_cache_destroy(x86_fpu_cache);
		8035	WARN_ON(static_branch_unlikely(&kvm_xen_enabled.key));
7990	}	8036	}
7991		8037
7992	static int __kvm_vcpu_halt(struct kvm_vcpu *vcpu, int state, int reason)	8038	static int __kvm_vcpu_halt(struct kvm_vcpu *vcpu, int state, int reason)
@@ -8038,7 +8084,7 @@ static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr, · 45 unmodified lines
8038	if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)	8084	if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
8039	return -KVM_EOPNOTSUPP;	8085	return -KVM_EOPNOTSUPP;
8040		8086
8041	if (kvm_get_walltime_and_clockread(&ts, &cycle) == false)	8087	if (!kvm_get_walltime_and_clockread(&ts, &cycle))
8042	return -KVM_EOPNOTSUPP;	8088	return -KVM_EOPNOTSUPP;
8043		8089
8044	clock_pairing.sec = ts.tv_sec;	8090	clock_pairing.sec = ts.tv_sec;
@@ -8114,7 +8160,10 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) · 69 unmodified lines
8114	unsigned long nr, a0, a1, a2, a3, ret;	8160	unsigned long nr, a0, a1, a2, a3, ret;
8115	int op_64_bit;	8161	int op_64_bit;
8116		8162
8117	if (kvm_hv_hypercall_enabled(vcpu->kvm))	8163	if (kvm_xen_hypercall_enabled(vcpu->kvm))
		8164	return kvm_xen_hypercall(vcpu);
		8165
		8166	if (kvm_hv_hypercall_enabled(vcpu))
8118	return kvm_hv_hypercall(vcpu);	8167	return kvm_hv_hypercall(vcpu);
8119		8168
8120	nr = kvm_rax_read(vcpu);	8169	nr = kvm_rax_read(vcpu);
@@ -8134,7 +8183,7 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) · 13 unmodified lines
8134	a3 &= 0xFFFFFFFF;	8183	a3 &= 0xFFFFFFFF;
8135	}	8184	}
8136		8185
8137	if (kvm_x86_ops.get_cpl(vcpu) != 0) {	8186	if (static_call(kvm_x86_get_cpl)(vcpu) != 0) {
8138	ret = -KVM_EPERM;	8187	ret = -KVM_EPERM;
8139	goto out;	8188	goto out;
8140	}	8189	}
@@ -8191,7 +8240,7 @@ static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt) · 50 unmodified lines
8191	char instruction[3];	8240	char instruction[3];
8192	unsigned long rip = kvm_rip_read(vcpu);	8241	unsigned long rip = kvm_rip_read(vcpu);
8193		8242
8194	kvm_x86_ops.patch_hypercall(vcpu, instruction);	8243	static_call(kvm_x86_patch_hypercall)(vcpu, instruction);
8195		8244
8196	return emulator_write_emulated(ctxt, rip, instruction, 3,	8245	return emulator_write_emulated(ctxt, rip, instruction, 3,
8197	&ctxt->exception);	8246	&ctxt->exception);
@@ -8215,12 +8264,14 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu) · 17 unmodified lines
8215	kvm_run->if_flag = !vcpu->arch.guest_state_protected	8264	kvm_run->if_flag = !vcpu->arch.guest_state_protected
8216	&& (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;	8265	&& (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
8217		8266
8218	kvm_run->flags = is_smm(vcpu) ? KVM_RUN_X86_SMM : 0;
8219	kvm_run->cr8 = kvm_get_cr8(vcpu);	8267	kvm_run->cr8 = kvm_get_cr8(vcpu);
8220	kvm_run->apic_base = kvm_get_apic_base(vcpu);	8268	kvm_run->apic_base = kvm_get_apic_base(vcpu);
8221	kvm_run->ready_for_interrupt_injection =	8269	kvm_run->ready_for_interrupt_injection =
8222	pic_in_kernel(vcpu->kvm) \|\|	8270	pic_in_kernel(vcpu->kvm) \|\|
8223	kvm_vcpu_ready_for_interrupt_injection(vcpu);	8271	kvm_vcpu_ready_for_interrupt_injection(vcpu);
		8272
		8273	if (is_smm(vcpu))
		8274	kvm_run->flags \|= KVM_RUN_X86_SMM;
8224	}	8275	}
8225		8276
8226	static void update_cr8_intercept(struct kvm_vcpu *vcpu)	8277	static void update_cr8_intercept(struct kvm_vcpu *vcpu)
@@ -8246,7 +8297,7 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu) · 19 unmodified lines
8246		8297
8247	tpr = kvm_lapic_get_cr8(vcpu);	8298	tpr = kvm_lapic_get_cr8(vcpu);
8248		8299
8249	kvm_x86_ops.update_cr8_intercept(vcpu, tpr, max_irr);	8300	static_call(kvm_x86_update_cr8_intercept)(vcpu, tpr, max_irr);
8250	}	8301	}
8251		8302
8252	static void inject_pending_event(struct kvm_vcpu vcpu, bool req_immediate_exit)	8303	static void inject_pending_event(struct kvm_vcpu vcpu, bool req_immediate_exit)
@@ -8257,7 +8308,7 @@ static void inject_pending_event(struct kvm_vcpu vcpu, bool req_immediate_exit · 4 unmodified lines
8257	/* try to reinject previous events if any */	8308	/* try to reinject previous events if any */
8258		8309
8259	if (vcpu->arch.exception.injected) {	8310	if (vcpu->arch.exception.injected) {
8260	kvm_x86_ops.queue_exception(vcpu);	8311	static_call(kvm_x86_queue_exception)(vcpu);
8261	can_inject = false;	8312	can_inject = false;
8262	}	8313	}
8263	/*	8314	/*
@@ -8276,10 +8327,10 @@ static void inject_pending_event(struct kvm_vcpu vcpu, bool req_immediate_exit · 12 unmodified lines
8276	*/	8327	*/
8277	else if (!vcpu->arch.exception.pending) {	8328	else if (!vcpu->arch.exception.pending) {
8278	if (vcpu->arch.nmi_injected) {	8329	if (vcpu->arch.nmi_injected) {
8279	kvm_x86_ops.set_nmi(vcpu);	8330	static_call(kvm_x86_set_nmi)(vcpu);
8280	can_inject = false;	8331	can_inject = false;
8281	} else if (vcpu->arch.interrupt.injected) {	8332	} else if (vcpu->arch.interrupt.injected) {
8282	kvm_x86_ops.set_irq(vcpu);	8333	static_call(kvm_x86_set_irq)(vcpu);
8283	can_inject = false;	8334	can_inject = false;
8284	}	8335	}
8285	}	8336	}
@@ -8320,7 +8371,7 @@ static void inject_pending_event(struct kvm_vcpu vcpu, bool req_immediate_exit · 34 unmodified lines
8320	}	8371	}
8321	}	8372	}
8322		8373
8323	kvm_x86_ops.queue_exception(vcpu);	8374	static_call(kvm_x86_queue_exception)(vcpu);
8324	can_inject = false;	8375	can_inject = false;
8325	}	8376	}
8326		8377
@@ -8336,7 +8387,7 @@ static void inject_pending_event(struct kvm_vcpu vcpu, bool req_immediate_exit · 9 unmodified lines
8336	* The kvm_x86_ops hooks communicate this by returning -EBUSY.	8387	* The kvm_x86_ops hooks communicate this by returning -EBUSY.
8337	*/	8388	*/
8338	if (vcpu->arch.smi_pending) {	8389	if (vcpu->arch.smi_pending) {
8339	r = can_inject ? kvm_x86_ops.smi_allowed(vcpu, true) : -EBUSY;	8390	r = can_inject ? static_call(kvm_x86_smi_allowed)(vcpu, true) : -EBUSY;
8340	if (r < 0)	8391	if (r < 0)
8341	goto busy;	8392	goto busy;
8342	if (r) {	8393	if (r) {
@@ -8345,35 +8396,35 @@ static void inject_pending_event(struct kvm_vcpu vcpu, bool req_immediate_exit · 2 unmodified lines
8345	enter_smm(vcpu);	8396	enter_smm(vcpu);
8346	can_inject = false;	8397	can_inject = false;
8347	} else	8398	} else
8348	kvm_x86_ops.enable_smi_window(vcpu);	8399	static_call(kvm_x86_enable_smi_window)(vcpu);
8349	}	8400	}
8350		8401
8351	if (vcpu->arch.nmi_pending) {	8402	if (vcpu->arch.nmi_pending) {
8352	r = can_inject ? kvm_x86_ops.nmi_allowed(vcpu, true) : -EBUSY;	8403	r = can_inject ? static_call(kvm_x86_nmi_allowed)(vcpu, true) : -EBUSY;
8353	if (r < 0)	8404	if (r < 0)
8354	goto busy;	8405	goto busy;
8355	if (r) {	8406	if (r) {
8356	--vcpu->arch.nmi_pending;	8407	--vcpu->arch.nmi_pending;
8357	vcpu->arch.nmi_injected = true;	8408	vcpu->arch.nmi_injected = true;
8358	kvm_x86_ops.set_nmi(vcpu);	8409	static_call(kvm_x86_set_nmi)(vcpu);
8359	can_inject = false;	8410	can_inject = false;
8360	WARN_ON(kvm_x86_ops.nmi_allowed(vcpu, true) < 0);	8411	WARN_ON(static_call(kvm_x86_nmi_allowed)(vcpu, true) < 0);
8361	}	8412	}
8362	if (vcpu->arch.nmi_pending)	8413	if (vcpu->arch.nmi_pending)
8363	kvm_x86_ops.enable_nmi_window(vcpu);	8414	static_call(kvm_x86_enable_nmi_window)(vcpu);
8364	}	8415	}
8365		8416
8366	if (kvm_cpu_has_injectable_intr(vcpu)) {	8417	if (kvm_cpu_has_injectable_intr(vcpu)) {
8367	r = can_inject ? kvm_x86_ops.interrupt_allowed(vcpu, true) : -EBUSY;	8418	r = can_inject ? static_call(kvm_x86_interrupt_allowed)(vcpu, true) : -EBUSY;
8368	if (r < 0)	8419	if (r < 0)
8369	goto busy;	8420	goto busy;
8370	if (r) {	8421	if (r) {
8371	kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), false);	8422	kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), false);
8372	kvm_x86_ops.set_irq(vcpu);	8423	static_call(kvm_x86_set_irq)(vcpu);
8373	WARN_ON(kvm_x86_ops.interrupt_allowed(vcpu, true) < 0);	8424	WARN_ON(static_call(kvm_x86_interrupt_allowed)(vcpu, true) < 0);
8374	}	8425	}
8375	if (kvm_cpu_has_injectable_intr(vcpu))	8426	if (kvm_cpu_has_injectable_intr(vcpu))
8376	kvm_x86_ops.enable_irq_window(vcpu);	8427	static_call(kvm_x86_enable_irq_window)(vcpu);
8377	}	8428	}
8378		8429
8379	if (is_guest_mode(vcpu) &&	8430	if (is_guest_mode(vcpu) &&
@@ -8398,7 +8449,7 @@ static void process_nmi(struct kvm_vcpu *vcpu) · 18 unmodified lines
8398	* If an NMI is already in progress, limit further NMIs to just one.	8449	* If an NMI is already in progress, limit further NMIs to just one.
8399	* Otherwise, allow two (and we'll inject the first one immediately).	8450	* Otherwise, allow two (and we'll inject the first one immediately).
8400	*/	8451	*/
8401	if (kvm_x86_ops.get_nmi_mask(vcpu) \|\| vcpu->arch.nmi_injected)	8452	if (static_call(kvm_x86_get_nmi_mask)(vcpu) \|\| vcpu->arch.nmi_injected)
8402	limit = 1;	8453	limit = 1;
8403		8454
8404	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);	8455	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
@@ -8488,11 +8539,11 @@ static void enter_smm_save_state_32(struct kvm_vcpu vcpu, char buf) · 83 unmodified lines
8488	put_smstate(u32, buf, 0x7f7c, seg.limit);	8539	put_smstate(u32, buf, 0x7f7c, seg.limit);
8489	put_smstate(u32, buf, 0x7f78, enter_smm_get_segment_flags(&seg));	8540	put_smstate(u32, buf, 0x7f78, enter_smm_get_segment_flags(&seg));
8490		8541
8491	kvm_x86_ops.get_gdt(vcpu, &dt);	8542	static_call(kvm_x86_get_gdt)(vcpu, &dt);
8492	put_smstate(u32, buf, 0x7f74, dt.address);	8543	put_smstate(u32, buf, 0x7f74, dt.address);
8493	put_smstate(u32, buf, 0x7f70, dt.size);	8544	put_smstate(u32, buf, 0x7f70, dt.size);
8494		8545
8495	kvm_x86_ops.get_idt(vcpu, &dt);	8546	static_call(kvm_x86_get_idt)(vcpu, &dt);
8496	put_smstate(u32, buf, 0x7f58, dt.address);	8547	put_smstate(u32, buf, 0x7f58, dt.address);
8497	put_smstate(u32, buf, 0x7f54, dt.size);	8548	put_smstate(u32, buf, 0x7f54, dt.size);
8498		8549
@@ -8542,7 +8593,7 @@ static void enter_smm_save_state_64(struct kvm_vcpu vcpu, char buf) · 43 unmodified lines
8542	put_smstate(u32, buf, 0x7e94, seg.limit);	8593	put_smstate(u32, buf, 0x7e94, seg.limit);
8543	put_smstate(u64, buf, 0x7e98, seg.base);	8594	put_smstate(u64, buf, 0x7e98, seg.base);
8544		8595
8545	kvm_x86_ops.get_idt(vcpu, &dt);	8596	static_call(kvm_x86_get_idt)(vcpu, &dt);
8546	put_smstate(u32, buf, 0x7e84, dt.size);	8597	put_smstate(u32, buf, 0x7e84, dt.size);
8547	put_smstate(u64, buf, 0x7e88, dt.address);	8598	put_smstate(u64, buf, 0x7e88, dt.address);
8548		8599
@@ -8552,7 +8603,7 @@ static void enter_smm_save_state_64(struct kvm_vcpu vcpu, char buf) · 3 unmodified lines
8552	put_smstate(u32, buf, 0x7e74, seg.limit);	8603	put_smstate(u32, buf, 0x7e74, seg.limit);
8553	put_smstate(u64, buf, 0x7e78, seg.base);	8604	put_smstate(u64, buf, 0x7e78, seg.base);
8554		8605
8555	kvm_x86_ops.get_gdt(vcpu, &dt);	8606	static_call(kvm_x86_get_gdt)(vcpu, &dt);
8556	put_smstate(u32, buf, 0x7e64, dt.size);	8607	put_smstate(u32, buf, 0x7e64, dt.size);
8557	put_smstate(u64, buf, 0x7e68, dt.address);	8608	put_smstate(u64, buf, 0x7e68, dt.address);
8558		8609
@@ -8582,30 +8633,30 @@ static void enter_smm(struct kvm_vcpu *vcpu) · 23 unmodified lines
8582	* vCPU state (e.g. leave guest mode) after we've saved the state into	8633	* vCPU state (e.g. leave guest mode) after we've saved the state into
8583	* the SMM state-save area.	8634	* the SMM state-save area.
8584	*/	8635	*/
8585	kvm_x86_ops.pre_enter_smm(vcpu, buf);	8636	static_call(kvm_x86_pre_enter_smm)(vcpu, buf);
8586		8637
8587	vcpu->arch.hflags \|= HF_SMM_MASK;	8638	vcpu->arch.hflags \|= HF_SMM_MASK;
8588	kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf));	8639	kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf));
8589		8640
8590	if (kvm_x86_ops.get_nmi_mask(vcpu))	8641	if (static_call(kvm_x86_get_nmi_mask)(vcpu))
8591	vcpu->arch.hflags \|= HF_SMM_INSIDE_NMI_MASK;	8642	vcpu->arch.hflags \|= HF_SMM_INSIDE_NMI_MASK;
8592	else	8643	else
8593	kvm_x86_ops.set_nmi_mask(vcpu, true);	8644	static_call(kvm_x86_set_nmi_mask)(vcpu, true);
8594		8645
8595	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);	8646	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
8596	kvm_rip_write(vcpu, 0x8000);	8647	kvm_rip_write(vcpu, 0x8000);
8597		8648
8598	cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE \| X86_CR0_EM \| X86_CR0_TS \| X86_CR0_PG);	8649	cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE \| X86_CR0_EM \| X86_CR0_TS \| X86_CR0_PG);
8599	kvm_x86_ops.set_cr0(vcpu, cr0);	8650	static_call(kvm_x86_set_cr0)(vcpu, cr0);
8600	vcpu->arch.cr0 = cr0;	8651	vcpu->arch.cr0 = cr0;
8601		8652
8602	kvm_x86_ops.set_cr4(vcpu, 0);	8653	static_call(kvm_x86_set_cr4)(vcpu, 0);
8603		8654
8604	/* Undocumented: IDT limit is set to zero on entry to SMM. */	8655	/* Undocumented: IDT limit is set to zero on entry to SMM. */
8605	dt.address = dt.size = 0;	8656	dt.address = dt.size = 0;
8606	kvm_x86_ops.set_idt(vcpu, &dt);	8657	static_call(kvm_x86_set_idt)(vcpu, &dt);
8607		8658
8608	__kvm_set_dr(vcpu, 7, DR7_FIXED_1);	8659	kvm_set_dr(vcpu, 7, DR7_FIXED_1);
8609		8660
8610	cs.selector = (vcpu->arch.smbase >> 4) & 0xffff;	8661	cs.selector = (vcpu->arch.smbase >> 4) & 0xffff;
8611	cs.base = vcpu->arch.smbase;	8662	cs.base = vcpu->arch.smbase;
@@ -8634,7 +8685,7 @@ static void enter_smm(struct kvm_vcpu *vcpu) · 22 unmodified lines
8634		8685
8635	#ifdef CONFIG_X86_64	8686	#ifdef CONFIG_X86_64
8636	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))	8687	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
8637	kvm_x86_ops.set_efer(vcpu, 0);	8688	static_call(kvm_x86_set_efer)(vcpu, 0);
8638	#endif	8689	#endif
8639		8690
8640	kvm_update_cpuid_runtime(vcpu);	8691	kvm_update_cpuid_runtime(vcpu);
@@ -8672,7 +8723,7 @@ void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu) · 31 unmodified lines
8672		8723
8673	vcpu->arch.apicv_active = kvm_apicv_activated(vcpu->kvm);	8724	vcpu->arch.apicv_active = kvm_apicv_activated(vcpu->kvm);
8674	kvm_apic_update_apicv(vcpu);	8725	kvm_apic_update_apicv(vcpu);
8675	kvm_x86_ops.refresh_apicv_exec_ctrl(vcpu);	8726	static_call(kvm_x86_refresh_apicv_exec_ctrl)(vcpu);
8676	}	8727	}
8677	EXPORT_SYMBOL_GPL(kvm_vcpu_update_apicv);	8728	EXPORT_SYMBOL_GPL(kvm_vcpu_update_apicv);
8678		8729
@@ -8689,7 +8740,7 @@ void kvm_request_apicv_update(struct kvm *kvm, bool activate, ulong bit) · 10 unmodified lines
8689	unsigned long old, new, expected;	8740	unsigned long old, new, expected;
8690		8741
8691	if (!kvm_x86_ops.check_apicv_inhibit_reasons \|\|	8742	if (!kvm_x86_ops.check_apicv_inhibit_reasons \|\|
8692	!kvm_x86_ops.check_apicv_inhibit_reasons(bit))	8743	!static_call(kvm_x86_check_apicv_inhibit_reasons)(bit))
8693	return;	8744	return;
8694		8745
8695	old = READ_ONCE(kvm->arch.apicv_inhibit_reasons);	8746	old = READ_ONCE(kvm->arch.apicv_inhibit_reasons);
@@ -8709,7 +8760,7 @@ void kvm_request_apicv_update(struct kvm *kvm, bool activate, ulong bit) · 13 unmodified lines
8709		8760
8710	trace_kvm_apicv_update_request(activate, bit);	8761	trace_kvm_apicv_update_request(activate, bit);
8711	if (kvm_x86_ops.pre_update_apicv_exec_ctrl)	8762	if (kvm_x86_ops.pre_update_apicv_exec_ctrl)
8712	kvm_x86_ops.pre_update_apicv_exec_ctrl(kvm, activate);	8763	static_call(kvm_x86_pre_update_apicv_exec_ctrl)(kvm, activate);
8713		8764
8714	/*	8765	/*
8715	* Sending request to update APICV for all other vcpus,	8766	* Sending request to update APICV for all other vcpus,
@@ -8735,7 +8786,7 @@ static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu) · 19 unmodified lines
8735	kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);	8786	kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
8736	else {	8787	else {
8737	if (vcpu->arch.apicv_active)	8788	if (vcpu->arch.apicv_active)
8738	kvm_x86_ops.sync_pir_to_irr(vcpu);	8789	static_call(kvm_x86_sync_pir_to_irr)(vcpu);
8739	if (ioapic_in_kernel(vcpu->kvm))	8790	if (ioapic_in_kernel(vcpu->kvm))
8740	kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);	8791	kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
8741	}	8792	}
@@ -8753,9 +8804,12 @@ static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu) · 11 unmodified lines
8753	if (!kvm_apic_hw_enabled(vcpu->arch.apic))	8804	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
8754	return;	8805	return;
8755		8806
8756	bitmap_or((ulong *)eoi_exit_bitmap, vcpu->arch.ioapic_handled_vectors,	8807	if (to_hv_vcpu(vcpu))
8757	vcpu_to_synic(vcpu)->vec_bitmap, 256);	8808	bitmap_or((ulong *)eoi_exit_bitmap,
8758	kvm_x86_ops.load_eoi_exitmap(vcpu, eoi_exit_bitmap);	8809	vcpu->arch.ioapic_handled_vectors,
		8810	to_hv_synic(vcpu)->vec_bitmap, 256);
		8811
		8812	static_call(kvm_x86_load_eoi_exitmap)(vcpu, eoi_exit_bitmap);
8759	}	8813	}
8760		8814
8761	void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,	8815	void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
@@ -8780,7 +8834,7 @@ void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) · 18 unmodified lines
8780	if (!kvm_x86_ops.set_apic_access_page_addr)	8834	if (!kvm_x86_ops.set_apic_access_page_addr)
8781	return;	8835	return;
8782		8836
8783	kvm_x86_ops.set_apic_access_page_addr(vcpu);	8837	static_call(kvm_x86_set_apic_access_page_addr)(vcpu);
8784	}	8838	}
8785		8839
8786	void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)	8840	void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)
@@ -8905,8 +8959,10 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 118 unmodified lines
8905	goto out;	8959	goto out;
8906	}	8960	}
8907	if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {	8961	if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {
		8962	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
		8963
8908	vcpu->run->exit_reason = KVM_EXIT_HYPERV;	8964	vcpu->run->exit_reason = KVM_EXIT_HYPERV;
8909	vcpu->run->hyperv = vcpu->arch.hyperv.exit;	8965	vcpu->run->hyperv = hv_vcpu->exit;
8910	r = 0;	8966	r = 0;
8911	goto out;	8967	goto out;
8912	}	8968	}
@@ -8923,10 +8979,11 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 10 unmodified lines
8923	if (kvm_check_request(KVM_REQ_APF_READY, vcpu))	8979	if (kvm_check_request(KVM_REQ_APF_READY, vcpu))
8924	kvm_check_async_pf_completion(vcpu);	8980	kvm_check_async_pf_completion(vcpu);
8925	if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu))	8981	if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu))
8926	kvm_x86_ops.msr_filter_changed(vcpu);	8982	static_call(kvm_x86_msr_filter_changed)(vcpu);
8927	}	8983	}
8928		8984
8929	if (kvm_check_request(KVM_REQ_EVENT, vcpu) \|\| req_int_win) {	8985	if (kvm_check_request(KVM_REQ_EVENT, vcpu) \|\| req_int_win \|\|
		8986	kvm_xen_has_interrupt(vcpu)) {
8930	++vcpu->stat.req_event;	8987	++vcpu->stat.req_event;
8931	kvm_apic_accept_events(vcpu);	8988	kvm_apic_accept_events(vcpu);
8932	if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {	8989	if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
@@ -8936,7 +8993,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 3 unmodified lines
8936		8993
8937	inject_pending_event(vcpu, &req_immediate_exit);	8994	inject_pending_event(vcpu, &req_immediate_exit);
8938	if (req_int_win)	8995	if (req_int_win)
8939	kvm_x86_ops.enable_irq_window(vcpu);	8996	static_call(kvm_x86_enable_irq_window)(vcpu);
8940		8997
8941	if (kvm_lapic_enabled(vcpu)) {	8998	if (kvm_lapic_enabled(vcpu)) {
8942	update_cr8_intercept(vcpu);	8999	update_cr8_intercept(vcpu);
@@ -8951,7 +9008,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 8 unmodified lines
8951		9008
8952	preempt_disable();	9009	preempt_disable();
8953		9010
8954	kvm_x86_ops.prepare_guest_switch(vcpu);	9011	static_call(kvm_x86_prepare_guest_switch)(vcpu);
8955		9012
8956	/*	9013	/*
8957	* Disable IRQs before setting IN_GUEST_MODE. Posted interrupt	9014	* Disable IRQs before setting IN_GUEST_MODE. Posted interrupt
@@ -8982,7 +9039,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 24 unmodified lines
8982	* notified with kvm_vcpu_kick.	9039	* notified with kvm_vcpu_kick.
8983	*/	9040	*/
8984	if (kvm_lapic_enabled(vcpu) && vcpu->arch.apicv_active)	9041	if (kvm_lapic_enabled(vcpu) && vcpu->arch.apicv_active)
8985	kvm_x86_ops.sync_pir_to_irr(vcpu);	9042	static_call(kvm_x86_sync_pir_to_irr)(vcpu);
8986		9043
8987	if (kvm_vcpu_exit_request(vcpu)) {	9044	if (kvm_vcpu_exit_request(vcpu)) {
8988	vcpu->mode = OUTSIDE_GUEST_MODE;	9045	vcpu->mode = OUTSIDE_GUEST_MODE;
@@ -8996,7 +9053,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 7 unmodified lines
8996		9053
8997	if (req_immediate_exit) {	9054	if (req_immediate_exit) {
8998	kvm_make_request(KVM_REQ_EVENT, vcpu);	9055	kvm_make_request(KVM_REQ_EVENT, vcpu);
8999	kvm_x86_ops.request_immediate_exit(vcpu);	9056	static_call(kvm_x86_request_immediate_exit)(vcpu);
9000	}	9057	}
9001		9058
9002	fpregs_assert_state_consistent();	9059	fpregs_assert_state_consistent();
@@ -9013,7 +9070,19 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 10 unmodified lines
9013	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;	9070	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
9014	}	9071	}
9015		9072
9016	exit_fastpath = kvm_x86_ops.run(vcpu);	9073	for (;;) {
		9074	exit_fastpath = static_call(kvm_x86_run)(vcpu);
		9075	if (likely(exit_fastpath != EXIT_FASTPATH_REENTER_GUEST))
		9076	break;
		9077
		9078	if (unlikely(kvm_vcpu_exit_request(vcpu))) {
		9079	exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED;
		9080	break;
		9081	}
		9082
		9083	if (vcpu->arch.apicv_active)
		9084	static_call(kvm_x86_sync_pir_to_irr)(vcpu);
		9085	}
9017		9086
9018	/*	9087	/*
9019	* Do this here before restoring debug registers on the host. And	9088	* Do this here before restoring debug registers on the host. And
@@ -9023,7 +9092,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 3 unmodified lines
9023	*/	9092	*/
9024	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {	9093	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
9025	WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);	9094	WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
9026	kvm_x86_ops.sync_dirty_debug_regs(vcpu);	9095	static_call(kvm_x86_sync_dirty_debug_regs)(vcpu);
9027	kvm_update_dr0123(vcpu);	9096	kvm_update_dr0123(vcpu);
9028	kvm_update_dr7(vcpu);	9097	kvm_update_dr7(vcpu);
9029	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;	9098	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
@@ -9045,7 +9114,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 15 unmodified lines
9045	vcpu->mode = OUTSIDE_GUEST_MODE;	9114	vcpu->mode = OUTSIDE_GUEST_MODE;
9046	smp_wmb();	9115	smp_wmb();
9047		9116
9048	kvm_x86_ops.handle_exit_irqoff(vcpu);	9117	static_call(kvm_x86_handle_exit_irqoff)(vcpu);
9049		9118
9050	/*	9119	/*
9051	* Consume any pending interrupts, including the possible source of	9120	* Consume any pending interrupts, including the possible source of
@@ -9087,13 +9156,13 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 35 unmodified lines
9087	if (vcpu->arch.apic_attention)	9156	if (vcpu->arch.apic_attention)
9088	kvm_lapic_sync_from_vapic(vcpu);	9157	kvm_lapic_sync_from_vapic(vcpu);
9089		9158
9090	r = kvm_x86_ops.handle_exit(vcpu, exit_fastpath);	9159	r = static_call(kvm_x86_handle_exit)(vcpu, exit_fastpath);
9091	return r;	9160	return r;
9092		9161
9093	cancel_injection:	9162	cancel_injection:
9094	if (req_immediate_exit)	9163	if (req_immediate_exit)
9095	kvm_make_request(KVM_REQ_EVENT, vcpu);	9164	kvm_make_request(KVM_REQ_EVENT, vcpu);
9096	kvm_x86_ops.cancel_injection(vcpu);	9165	static_call(kvm_x86_cancel_injection)(vcpu);
9097	if (unlikely(vcpu->arch.apic_attention))	9166	if (unlikely(vcpu->arch.apic_attention))
9098	kvm_lapic_sync_from_vapic(vcpu);	9167	kvm_lapic_sync_from_vapic(vcpu);
9099	out:	9168	out:
@@ -9103,13 +9172,13 @@ out: · 3 unmodified lines
9103	static inline int vcpu_block(struct kvm kvm, struct kvm_vcpu vcpu)	9172	static inline int vcpu_block(struct kvm kvm, struct kvm_vcpu vcpu)
9104	{	9173	{
9105	if (!kvm_arch_vcpu_runnable(vcpu) &&	9174	if (!kvm_arch_vcpu_runnable(vcpu) &&
9106	(!kvm_x86_ops.pre_block \|\| kvm_x86_ops.pre_block(vcpu) == 0)) {	9175	(!kvm_x86_ops.pre_block \|\| static_call(kvm_x86_pre_block)(vcpu) == 0)) {
9107	srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);	9176	srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
9108	kvm_vcpu_block(vcpu);	9177	kvm_vcpu_block(vcpu);
9109	vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);	9178	vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
9110		9179
9111	if (kvm_x86_ops.post_block)	9180	if (kvm_x86_ops.post_block)
9112	kvm_x86_ops.post_block(vcpu);	9181	static_call(kvm_x86_post_block)(vcpu);
9113		9182
9114	if (!kvm_check_request(KVM_REQ_UNHALT, vcpu))	9183	if (!kvm_check_request(KVM_REQ_UNHALT, vcpu))
9115	return 1;	9184	return 1;
@@ -9330,6 +9399,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) · 214 unmodified lines
9330		9399
9331	vcpu_load(vcpu);	9400	vcpu_load(vcpu);
9332	kvm_sigset_activate(vcpu);	9401	kvm_sigset_activate(vcpu);
		9402	kvm_run->flags = 0;
9333	kvm_load_guest_fpu(vcpu);	9403	kvm_load_guest_fpu(vcpu);
9334		9404
9335	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {	9405	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
@@ -9504,10 +9574,10 @@ static void __get_sregs(struct kvm_vcpu vcpu, struct kvm_sregs sregs) · 168 unmodified lines
9504	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);	9574	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
9505	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);	9575	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
9506		9576
9507	kvm_x86_ops.get_idt(vcpu, &dt);	9577	static_call(kvm_x86_get_idt)(vcpu, &dt);
9508	sregs->idt.limit = dt.size;	9578	sregs->idt.limit = dt.size;
9509	sregs->idt.base = dt.address;	9579	sregs->idt.base = dt.address;
9510	kvm_x86_ops.get_gdt(vcpu, &dt);	9580	static_call(kvm_x86_get_gdt)(vcpu, &dt);
9511	sregs->gdt.limit = dt.size;	9581	sregs->gdt.limit = dt.size;
9512	sregs->gdt.base = dt.address;	9582	sregs->gdt.base = dt.address;
9513		9583
@@ -9625,7 +9695,7 @@ static bool kvm_is_valid_sregs(struct kvm_vcpu vcpu, struct kvm_sregs sregs) · 111 unmodified lines
9625	*/	9695	*/
9626	if (!(sregs->cr4 & X86_CR4_PAE) \|\| !(sregs->efer & EFER_LMA))	9696	if (!(sregs->cr4 & X86_CR4_PAE) \|\| !(sregs->efer & EFER_LMA))
9627	return false;	9697	return false;
9628	if (sregs->cr3 & vcpu->arch.cr3_lm_rsvd_bits)	9698	if (kvm_vcpu_is_illegal_gpa(vcpu, sregs->cr3))
9629	return false;	9699	return false;
9630	} else {	9700	} else {
9631	/*	9701	/*
@@ -9660,10 +9730,10 @@ static int __set_sregs(struct kvm_vcpu vcpu, struct kvm_sregs sregs) · 28 unmodified lines
9660		9730
9661	dt.size = sregs->idt.limit;	9731	dt.size = sregs->idt.limit;
9662	dt.address = sregs->idt.base;	9732	dt.address = sregs->idt.base;
9663	kvm_x86_ops.set_idt(vcpu, &dt);	9733	static_call(kvm_x86_set_idt)(vcpu, &dt);
9664	dt.size = sregs->gdt.limit;	9734	dt.size = sregs->gdt.limit;
9665	dt.address = sregs->gdt.base;	9735	dt.address = sregs->gdt.base;
9666	kvm_x86_ops.set_gdt(vcpu, &dt);	9736	static_call(kvm_x86_set_gdt)(vcpu, &dt);
9667		9737
9668	vcpu->arch.cr2 = sregs->cr2;	9738	vcpu->arch.cr2 = sregs->cr2;
9669	mmu_reset_needed \|= kvm_read_cr3(vcpu) != sregs->cr3;	9739	mmu_reset_needed \|= kvm_read_cr3(vcpu) != sregs->cr3;
@@ -9673,14 +9743,14 @@ static int __set_sregs(struct kvm_vcpu vcpu, struct kvm_sregs sregs) · 3 unmodified lines
9673	kvm_set_cr8(vcpu, sregs->cr8);	9743	kvm_set_cr8(vcpu, sregs->cr8);
9674		9744
9675	mmu_reset_needed \|= vcpu->arch.efer != sregs->efer;	9745	mmu_reset_needed \|= vcpu->arch.efer != sregs->efer;
9676	kvm_x86_ops.set_efer(vcpu, sregs->efer);	9746	static_call(kvm_x86_set_efer)(vcpu, sregs->efer);
9677		9747
9678	mmu_reset_needed \|= kvm_read_cr0(vcpu) != sregs->cr0;	9748	mmu_reset_needed \|= kvm_read_cr0(vcpu) != sregs->cr0;
9679	kvm_x86_ops.set_cr0(vcpu, sregs->cr0);	9749	static_call(kvm_x86_set_cr0)(vcpu, sregs->cr0);
9680	vcpu->arch.cr0 = sregs->cr0;	9750	vcpu->arch.cr0 = sregs->cr0;
9681		9751
9682	mmu_reset_needed \|= kvm_read_cr4(vcpu) != sregs->cr4;	9752	mmu_reset_needed \|= kvm_read_cr4(vcpu) != sregs->cr4;
9683	kvm_x86_ops.set_cr4(vcpu, sregs->cr4);	9753	static_call(kvm_x86_set_cr4)(vcpu, sregs->cr4);
9684		9754
9685	idx = srcu_read_lock(&vcpu->kvm->srcu);	9755	idx = srcu_read_lock(&vcpu->kvm->srcu);
9686	if (is_pae_paging(vcpu)) {	9756	if (is_pae_paging(vcpu)) {
@@ -9788,7 +9858,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, · 101 unmodified lines
9788	*/	9858	*/
9789	kvm_set_rflags(vcpu, rflags);	9859	kvm_set_rflags(vcpu, rflags);
9790		9860
9791	kvm_x86_ops.update_exception_bitmap(vcpu);	9861	static_call(kvm_x86_update_exception_bitmap)(vcpu);
9792		9862
9793	r = 0;	9863	r = 0;
9794		9864
@@ -9966,7 +10036,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) · 171 unmodified lines
9966	if (kvm_apicv_activated(vcpu->kvm))	10036	if (kvm_apicv_activated(vcpu->kvm))
9967	vcpu->arch.apicv_active = true;	10037	vcpu->arch.apicv_active = true;
9968	} else	10038	} else
9969	static_key_slow_inc(&kvm_no_apic_vcpu);	10039	static_branch_inc(&kvm_has_noapic_vcpu);
9970		10040
9971	r = -ENOMEM;	10041	r = -ENOMEM;
9972		10042
@@ -10004,7 +10074,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) · 31 unmodified lines
10004	fx_init(vcpu);	10074	fx_init(vcpu);
10005		10075
10006	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);	10076	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
10007	vcpu->arch.cr3_lm_rsvd_bits = rsvd_bits(cpuid_maxphyaddr(vcpu), 63);	10077	vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu);
10008		10078
10009	vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;	10079	vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;
10010		10080
@@ -10014,9 +10084,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) · 3 unmodified lines
10014	vcpu->arch.pending_external_vector = -1;	10084	vcpu->arch.pending_external_vector = -1;
10015	vcpu->arch.preempted_in_kernel = false;	10085	vcpu->arch.preempted_in_kernel = false;
10016		10086
10017	kvm_hv_vcpu_init(vcpu);	10087	r = static_call(kvm_x86_vcpu_create)(vcpu);
10018
10019	r = kvm_x86_ops.vcpu_create(vcpu);
10020	if (r)	10088	if (r)
10021	goto free_guest_fpu;	10089	goto free_guest_fpu;
10022		10090
@@ -10052,8 +10120,6 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) · 29 unmodified lines
10052	{	10120	{
10053	struct kvm *kvm = vcpu->kvm;	10121	struct kvm *kvm = vcpu->kvm;
10054		10122
10055	kvm_hv_vcpu_postcreate(vcpu);
10056
10057	if (mutex_lock_killable(&vcpu->mutex))	10123	if (mutex_lock_killable(&vcpu->mutex))
10058	return;	10124	return;
10059	vcpu_load(vcpu);	10125	vcpu_load(vcpu);
@@ -10079,7 +10145,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) · 19 unmodified lines
10079		10145
10080	kvmclock_reset(vcpu);	10146	kvmclock_reset(vcpu);
10081		10147
10082	kvm_x86_ops.vcpu_free(vcpu);	10148	static_call(kvm_x86_vcpu_free)(vcpu);
10083		10149
10084	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);	10150	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
10085	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);	10151	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
@@ -10096,7 +10162,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) · 10 unmodified lines
10096	free_page((unsigned long)vcpu->arch.pio_data);	10162	free_page((unsigned long)vcpu->arch.pio_data);
10097	kvfree(vcpu->arch.cpuid_entries);	10163	kvfree(vcpu->arch.cpuid_entries);
10098	if (!lapic_in_kernel(vcpu))	10164	if (!lapic_in_kernel(vcpu))
10099	static_key_slow_dec(&kvm_no_apic_vcpu);	10165	static_branch_dec(&kvm_has_noapic_vcpu);
10100	}	10166	}
10101		10167
10102	void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)	10168	void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
@@ -10115,7 +10181,7 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) · 12 unmodified lines
10115		10181
10116	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));	10182	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
10117	kvm_update_dr0123(vcpu);	10183	kvm_update_dr0123(vcpu);
10118	vcpu->arch.dr6 = DR6_INIT;	10184	vcpu->arch.dr6 = DR6_ACTIVE_LOW;
10119	vcpu->arch.dr7 = DR7_FIXED_1;	10185	vcpu->arch.dr7 = DR7_FIXED_1;
10120	kvm_update_dr7(vcpu);	10186	kvm_update_dr7(vcpu);
10121		10187
@@ -10168,7 +10234,7 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) · 46 unmodified lines
10168		10234
10169	vcpu->arch.ia32_xss = 0;	10235	vcpu->arch.ia32_xss = 0;
10170		10236
10171	kvm_x86_ops.vcpu_reset(vcpu, init_event);	10237	static_call(kvm_x86_vcpu_reset)(vcpu, init_event);
10172	}	10238	}
10173		10239
10174	void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)	10240	void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
@@ -10194,7 +10260,7 @@ int kvm_arch_hardware_enable(void) · 19 unmodified lines
10194	bool stable, backwards_tsc = false;	10260	bool stable, backwards_tsc = false;
10195		10261
10196	kvm_user_return_msr_cpu_online();	10262	kvm_user_return_msr_cpu_online();
10197	ret = kvm_x86_ops.hardware_enable();	10263	ret = static_call(kvm_x86_hardware_enable)();
10198	if (ret != 0)	10264	if (ret != 0)
10199	return ret;	10265	return ret;
10200		10266
@@ -10276,7 +10342,7 @@ int kvm_arch_hardware_enable(void) · 75 unmodified lines
10276		10342
10277	void kvm_arch_hardware_disable(void)	10343	void kvm_arch_hardware_disable(void)
10278	{	10344	{
10279	kvm_x86_ops.hardware_disable();	10345	static_call(kvm_x86_hardware_disable)();
10280	drop_user_return_notifiers();	10346	drop_user_return_notifiers();
10281	}	10347	}
10282		10348
@@ -10295,6 +10361,7 @@ int kvm_arch_hardware_setup(void *opaque) · 12 unmodified lines
10295	return r;	10361	return r;
10296		10362
10297	memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));	10363	memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
		10364	kvm_ops_static_call_update();
10298		10365
10299	if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))	10366	if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
10300	supported_xss = 0;	10367	supported_xss = 0;
@@ -10323,7 +10390,7 @@ int kvm_arch_hardware_setup(void *opaque) · 22 unmodified lines
10323		10390
10324	void kvm_arch_hardware_unsetup(void)	10391	void kvm_arch_hardware_unsetup(void)
10325	{	10392	{
10326	kvm_x86_ops.hardware_unsetup();	10393	static_call(kvm_x86_hardware_unsetup)();
10327	}	10394	}
10328		10395
10329	int kvm_arch_check_processor_compat(void *opaque)	10396	int kvm_arch_check_processor_compat(void *opaque)
@@ -10351,8 +10418,8 @@ bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu) · 21 unmodified lines
10351	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;	10418	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
10352	}	10419	}
10353		10420
10354	struct static_key kvm_no_apic_vcpu __read_mostly;	10421	__read_mostly DEFINE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu);
10355	EXPORT_SYMBOL_GPL(kvm_no_apic_vcpu);	10422	EXPORT_SYMBOL_GPL(kvm_has_noapic_vcpu);
10356		10423
10357	void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)	10424	void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
10358	{	10425	{
@@ -10363,12 +10430,12 @@ void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) · 4 unmodified lines
10363	pmu->need_cleanup = true;	10430	pmu->need_cleanup = true;
10364	kvm_make_request(KVM_REQ_PMU, vcpu);	10431	kvm_make_request(KVM_REQ_PMU, vcpu);
10365	}	10432	}
10366	kvm_x86_ops.sched_in(vcpu, cpu);	10433	static_call(kvm_x86_sched_in)(vcpu, cpu);
10367	}	10434	}
10368		10435
10369	void kvm_arch_free_vm(struct kvm *kvm)	10436	void kvm_arch_free_vm(struct kvm *kvm)
10370	{	10437	{
10371	kfree(kvm->arch.hyperv.hv_pa_pg);	10438	kfree(to_kvm_hv(kvm)->hv_pa_pg);
10372	vfree(kvm);	10439	vfree(kvm);
10373	}	10440	}
10374		10441
@@ -10407,7 +10474,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 32 unmodified lines
10407	kvm_page_track_init(kvm);	10474	kvm_page_track_init(kvm);
10408	kvm_mmu_init_vm(kvm);	10475	kvm_mmu_init_vm(kvm);
10409		10476
10410	return kvm_x86_ops.vm_init(kvm);	10477	return static_call(kvm_x86_vm_init)(kvm);
10411	}	10478	}
10412		10479
10413	int kvm_arch_post_init_vm(struct kvm *kvm)	10480	int kvm_arch_post_init_vm(struct kvm *kvm)
@@ -10552,8 +10619,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm) · 138 unmodified lines
10552	__x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);	10619	__x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);
10553	mutex_unlock(&kvm->slots_lock);	10620	mutex_unlock(&kvm->slots_lock);
10554	}	10621	}
10555	if (kvm_x86_ops.vm_destroy)	10622	static_call_cond(kvm_x86_vm_destroy)(kvm);
10556	kvm_x86_ops.vm_destroy(kvm);
10557	for (i = 0; i < kvm->arch.msr_filter.count; i++)	10623	for (i = 0; i < kvm->arch.msr_filter.count; i++)
10558	kfree(kvm->arch.msr_filter.ranges[i].bitmap);	10624	kfree(kvm->arch.msr_filter.ranges[i].bitmap);
10559	kvm_pic_destroy(kvm);	10625	kvm_pic_destroy(kvm);
@@ -10563,6 +10629,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm) · 3 unmodified lines
10563	kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));	10629	kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));
10564	kvm_mmu_uninit_vm(kvm);	10630	kvm_mmu_uninit_vm(kvm);
10565	kvm_page_track_cleanup(kvm);	10631	kvm_page_track_cleanup(kvm);
		10632	kvm_xen_destroy_vm(kvm);
10566	kvm_hv_destroy_vm(kvm);	10633	kvm_hv_destroy_vm(kvm);
10567	}	10634	}
10568		10635
@@ -10744,7 +10811,7 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, · 175 unmodified lines
10744	*/	10811	*/
10745	if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) {	10812	if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
10746	if (kvm_x86_ops.slot_enable_log_dirty) {	10813	if (kvm_x86_ops.slot_enable_log_dirty) {
10747	kvm_x86_ops.slot_enable_log_dirty(kvm, new);	10814	static_call(kvm_x86_slot_enable_log_dirty)(kvm, new);
10748	} else {	10815	} else {
10749	int level =	10816	int level =
10750	kvm_dirty_log_manual_protect_and_init_set(kvm) ?	10817	kvm_dirty_log_manual_protect_and_init_set(kvm) ?
@@ -10761,8 +10828,7 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, · 10 unmodified lines
10761	kvm_mmu_slot_remove_write_access(kvm, new, level);	10828	kvm_mmu_slot_remove_write_access(kvm, new, level);
10762	}	10829	}
10763	} else {	10830	} else {
10764	if (kvm_x86_ops.slot_disable_log_dirty)	10831	static_call_cond(kvm_x86_slot_disable_log_dirty)(kvm, new);
10765	kvm_x86_ops.slot_disable_log_dirty(kvm, new);
10766	}	10832	}
10767	}	10833	}
10768		10834
@@ -10801,7 +10867,7 @@ static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu) · 32 unmodified lines
10801	{	10867	{
10802	return (is_guest_mode(vcpu) &&	10868	return (is_guest_mode(vcpu) &&
10803	kvm_x86_ops.guest_apic_has_interrupt &&	10869	kvm_x86_ops.guest_apic_has_interrupt &&
10804	kvm_x86_ops.guest_apic_has_interrupt(vcpu));	10870	static_call(kvm_x86_guest_apic_has_interrupt)(vcpu));
10805	}	10871	}
10806		10872
10807	static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)	10873	static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
@@ -10820,12 +10886,12 @@ static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) · 12 unmodified lines
10820		10886
10821	if (kvm_test_request(KVM_REQ_NMI, vcpu) \|\|	10887	if (kvm_test_request(KVM_REQ_NMI, vcpu) \|\|
10822	(vcpu->arch.nmi_pending &&	10888	(vcpu->arch.nmi_pending &&
10823	kvm_x86_ops.nmi_allowed(vcpu, false)))	10889	static_call(kvm_x86_nmi_allowed)(vcpu, false)))
10824	return true;	10890	return true;
10825		10891
10826	if (kvm_test_request(KVM_REQ_SMI, vcpu) \|\|	10892	if (kvm_test_request(KVM_REQ_SMI, vcpu) \|\|
10827	(vcpu->arch.smi_pending &&	10893	(vcpu->arch.smi_pending &&
10828	kvm_x86_ops.smi_allowed(vcpu, false)))	10894	static_call(kvm_x86_smi_allowed)(vcpu, false)))
10829	return true;	10895	return true;
10830		10896
10831	if (kvm_arch_interrupt_allowed(vcpu) &&	10897	if (kvm_arch_interrupt_allowed(vcpu) &&
@@ -10859,7 +10925,7 @@ bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu) · 27 unmodified lines
10859	kvm_test_request(KVM_REQ_EVENT, vcpu))	10925	kvm_test_request(KVM_REQ_EVENT, vcpu))
10860	return true;	10926	return true;
10861		10927
10862	if (vcpu->arch.apicv_active && kvm_x86_ops.dy_apicv_has_pending_interrupt(vcpu))	10928	if (vcpu->arch.apicv_active && static_call(kvm_x86_dy_apicv_has_pending_interrupt)(vcpu))
10863	return true;	10929	return true;
10864		10930
10865	return false;	10931	return false;
@@ -10877,7 +10943,7 @@ int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) · 11 unmodified lines
10877		10943
10878	int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)	10944	int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
10879	{	10945	{
10880	return kvm_x86_ops.interrupt_allowed(vcpu, false);	10946	return static_call(kvm_x86_interrupt_allowed)(vcpu, false);
10881	}	10947	}
10882		10948
10883	unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)	10949	unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
@@ -10903,7 +10969,7 @@ unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu) · 19 unmodified lines
10903	{	10969	{
10904	unsigned long rflags;	10970	unsigned long rflags;
10905		10971
10906	rflags = kvm_x86_ops.get_rflags(vcpu);	10972	rflags = static_call(kvm_x86_get_rflags)(vcpu);
10907	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)	10973	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
10908	rflags &= ~X86_EFLAGS_TF;	10974	rflags &= ~X86_EFLAGS_TF;
10909	return rflags;	10975	return rflags;
@@ -10915,7 +10981,7 @@ static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) · 5 unmodified lines
10915	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&	10981	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
10916	kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))	10982	kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
10917	rflags \|= X86_EFLAGS_TF;	10983	rflags \|= X86_EFLAGS_TF;
10918	kvm_x86_ops.set_rflags(vcpu, rflags);	10984	static_call(kvm_x86_set_rflags)(vcpu, rflags);
10919	}	10985	}
10920		10986
10921	void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)	10987	void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
@@ -11045,7 +11111,7 @@ static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu) · 123 unmodified lines
11045	return false;	11111	return false;
11046		11112
11047	if (!kvm_pv_async_pf_enabled(vcpu) \|\|	11113	if (!kvm_pv_async_pf_enabled(vcpu) \|\|
11048	(vcpu->arch.apf.send_user_only && kvm_x86_ops.get_cpl(vcpu) == 0))	11114	(vcpu->arch.apf.send_user_only && static_call(kvm_x86_get_cpl)(vcpu) == 0))
11049	return false;	11115	return false;
11050		11116
11051	return true;	11117	return true;
@@ -11190,7 +11256,7 @@ int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons, · 138 unmodified lines
11190		11256
11191	irqfd->producer = prod;	11257	irqfd->producer = prod;
11192	kvm_arch_start_assignment(irqfd->kvm);	11258	kvm_arch_start_assignment(irqfd->kvm);
11193	ret = kvm_x86_ops.update_pi_irte(irqfd->kvm,	11259	ret = static_call(kvm_x86_update_pi_irte)(irqfd->kvm,
11194	prod->irq, irqfd->gsi, 1);	11260	prod->irq, irqfd->gsi, 1);
11195		11261
11196	if (ret)	11262	if (ret)
@@ -11215,7 +11281,7 @@ void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, · 18 unmodified lines
11215	* when the irq is masked/disabled or the consumer side (KVM	11281	* when the irq is masked/disabled or the consumer side (KVM
11216	* int this case doesn't want to receive the interrupts.	11282	* int this case doesn't want to receive the interrupts.
11217	*/	11283	*/
11218	ret = kvm_x86_ops.update_pi_irte(irqfd->kvm, prod->irq, irqfd->gsi, 0);	11284	ret = static_call(kvm_x86_update_pi_irte)(irqfd->kvm, prod->irq, irqfd->gsi, 0);
11219	if (ret)	11285	if (ret)
11220	printk(KERN_INFO "irq bypass consumer (token %p) unregistration"	11286	printk(KERN_INFO "irq bypass consumer (token %p) unregistration"
11221	" fails: %d\n", irqfd->consumer.token, ret);	11287	" fails: %d\n", irqfd->consumer.token, ret);
@@ -11226,7 +11292,7 @@ void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, · 4 unmodified lines
11226	int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,	11292	int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
11227	uint32_t guest_irq, bool set)	11293	uint32_t guest_irq, bool set)
11228	{	11294	{
11229	return kvm_x86_ops.update_pi_irte(kvm, host_irq, guest_irq, set);	11295	return static_call(kvm_x86_update_pi_irte)(kvm, host_irq, guest_irq, set);
11230	}	11296	}
11231		11297
11232	bool kvm_vector_hashing_enabled(void)	11298	bool kvm_vector_hashing_enabled(void)

arch/x86/kvm/x86.h+8−4

 	if (!is_long_mode(vcpu))
 		return false;
-	kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
+	static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
 	return cs_l;
 }
 static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu)
 {
 	++vcpu->stat.tlb_flush;
-	kvm_x86_ops.tlb_flush_current(vcpu);
+	static_call(kvm_x86_tlb_flush_current)(vcpu);
 }
 static inline int is_pae(struct kvm_vcpu *vcpu)
 static inline bool kvm_vcpu_latch_init(struct kvm_vcpu *vcpu)
 {
-	return is_smm(vcpu) || kvm_x86_ops.apic_init_signal_blocked(vcpu);
+	return is_smm(vcpu) || static_call(kvm_x86_apic_init_signal_blocked)(vcpu);
 }
+void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs);
 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip);
 void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr);
 					  int page_num);
 bool kvm_vector_hashing_enabled(void);
 void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code);
+int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type,
+				    void *insn, int insn_len);
 int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
 			    int emulation_type, void *insn, int insn_len);
 fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu);
 extern struct static_key kvm_no_apic_vcpu;
+extern bool report_ignored_msrs;
 static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec)
 {
 	return pvclock_scale_delta(nsec, vcpu->arch.virtual_tsc_mult,
 void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu);
 int kvm_spec_ctrl_test_value(u64 value);
 bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
-bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu);
 int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r,
 			      struct x86_exception *e);
 int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva);

arch/x86/kvm/xen.c+431−0

@@ -0,0 +1,431 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Copyright © 2019 Oracle and/or its affiliates. All rights reserved.
	4	* Copyright © 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
	5	*
	6	* KVM Xen emulation
	7	*/
	8
	9	#include "x86.h"
	10	#include "xen.h"
	11	#include "hyperv.h"
	12
	13	#include <linux/kvm_host.h>
	14
	15	#include <trace/events/kvm.h>
	16	#include <xen/interface/xen.h>
	17
	18	#include "trace.h"
	19
	20	DEFINE_STATIC_KEY_DEFERRED_FALSE(kvm_xen_enabled, HZ);
	21
	22	static int kvm_xen_shared_info_init(struct kvm *kvm, gfn_t gfn)
	23	{
	24	gpa_t gpa = gfn_to_gpa(gfn);
	25	int wc_ofs, sec_hi_ofs;
	26	int ret;
	27	int idx = srcu_read_lock(&kvm->srcu);
	28
	29	ret = kvm_gfn_to_hva_cache_init(kvm, &kvm->arch.xen.shinfo_cache,
	30	gpa, PAGE_SIZE);
	31	if (ret)
	32	goto out;
	33
	34	kvm->arch.xen.shinfo_set = true;
	35
	36	/* Paranoia checks on the 32-bit struct layout */
	37	BUILD_BUG_ON(offsetof(struct compat_shared_info, wc) != 0x900);
	38	BUILD_BUG_ON(offsetof(struct compat_shared_info, arch.wc_sec_hi) != 0x924);
	39	BUILD_BUG_ON(offsetof(struct pvclock_vcpu_time_info, version) != 0);
	40
	41	/* 32-bit location by default */
	42	wc_ofs = offsetof(struct compat_shared_info, wc);
	43	sec_hi_ofs = offsetof(struct compat_shared_info, arch.wc_sec_hi);
	44
	45	#ifdef CONFIG_X86_64
	46	/* Paranoia checks on the 64-bit struct layout */
	47	BUILD_BUG_ON(offsetof(struct shared_info, wc) != 0xc00);
	48	BUILD_BUG_ON(offsetof(struct shared_info, wc_sec_hi) != 0xc0c);
	49
	50	if (kvm->arch.xen.long_mode) {
	51	wc_ofs = offsetof(struct shared_info, wc);
	52	sec_hi_ofs = offsetof(struct shared_info, wc_sec_hi);
	53	}
	54	#endif
	55
	56	kvm_write_wall_clock(kvm, gpa + wc_ofs, sec_hi_ofs - wc_ofs);
	57	kvm_make_all_cpus_request(kvm, KVM_REQ_MASTERCLOCK_UPDATE);
	58
	59	out:
	60	srcu_read_unlock(&kvm->srcu, idx);
	61	return ret;
	62	}
	63
	64	int __kvm_xen_has_interrupt(struct kvm_vcpu *v)
	65	{
	66	u8 rc = 0;
	67
	68	/*
	69	* If the global upcall vector (HVMIRQ_callback_vector) is set and
	70	* the vCPU's evtchn_upcall_pending flag is set, the IRQ is pending.
	71	*/
	72	struct gfn_to_hva_cache *ghc = &v->arch.xen.vcpu_info_cache;
	73	struct kvm_memslots *slots = kvm_memslots(v->kvm);
	74	unsigned int offset = offsetof(struct vcpu_info, evtchn_upcall_pending);
	75
	76	/* No need for compat handling here */
	77	BUILD_BUG_ON(offsetof(struct vcpu_info, evtchn_upcall_pending) !=
	78	offsetof(struct compat_vcpu_info, evtchn_upcall_pending));
	79	BUILD_BUG_ON(sizeof(rc) !=
	80	sizeof(((struct vcpu_info *)0)->evtchn_upcall_pending));
	81	BUILD_BUG_ON(sizeof(rc) !=
	82	sizeof(((struct compat_vcpu_info *)0)->evtchn_upcall_pending));
	83
	84	/*
	85	* For efficiency, this mirrors the checks for using the valid
	86	* cache in kvm_read_guest_offset_cached(), but just uses
	87	* __get_user() instead. And falls back to the slow path.
	88	*/
	89	if (likely(slots->generation == ghc->generation &&
	90	!kvm_is_error_hva(ghc->hva) && ghc->memslot)) {
	91	/* Fast path */
	92	__get_user(rc, (u8 __user *)ghc->hva + offset);
	93	} else {
	94	/* Slow path */
	95	kvm_read_guest_offset_cached(v->kvm, ghc, &rc, offset,
	96	sizeof(rc));
	97	}
	98
	99	return rc;
	100	}
	101
	102	int kvm_xen_hvm_set_attr(struct kvm kvm, struct kvm_xen_hvm_attr data)
	103	{
	104	int r = -ENOENT;
	105
	106	mutex_lock(&kvm->lock);
	107
	108	switch (data->type) {
	109	case KVM_XEN_ATTR_TYPE_LONG_MODE:
	110	if (!IS_ENABLED(CONFIG_64BIT) && data->u.long_mode) {
	111	r = -EINVAL;
	112	} else {
	113	kvm->arch.xen.long_mode = !!data->u.long_mode;
	114	r = 0;
	115	}
	116	break;
	117
	118	case KVM_XEN_ATTR_TYPE_SHARED_INFO:
	119	if (data->u.shared_info.gfn == GPA_INVALID) {
	120	kvm->arch.xen.shinfo_set = false;
	121	r = 0;
	122	break;
	123	}
	124	r = kvm_xen_shared_info_init(kvm, data->u.shared_info.gfn);
	125	break;
	126
	127
	128	case KVM_XEN_ATTR_TYPE_UPCALL_VECTOR:
	129	if (data->u.vector && data->u.vector < 0x10)
	130	r = -EINVAL;
	131	else {
	132	kvm->arch.xen.upcall_vector = data->u.vector;
	133	r = 0;
	134	}
	135	break;
	136
	137	default:
	138	break;
	139	}
	140
	141	mutex_unlock(&kvm->lock);
	142	return r;
	143	}
	144
	145	int kvm_xen_hvm_get_attr(struct kvm kvm, struct kvm_xen_hvm_attr data)
	146	{
	147	int r = -ENOENT;
	148
	149	mutex_lock(&kvm->lock);
	150
	151	switch (data->type) {
	152	case KVM_XEN_ATTR_TYPE_LONG_MODE:
	153	data->u.long_mode = kvm->arch.xen.long_mode;
	154	r = 0;
	155	break;
	156
	157	case KVM_XEN_ATTR_TYPE_SHARED_INFO:
	158	if (kvm->arch.xen.shinfo_set)
	159	data->u.shared_info.gfn = gpa_to_gfn(kvm->arch.xen.shinfo_cache.gpa);
	160	else
	161	data->u.shared_info.gfn = GPA_INVALID;
	162	r = 0;
	163	break;
	164
	165	case KVM_XEN_ATTR_TYPE_UPCALL_VECTOR:
	166	data->u.vector = kvm->arch.xen.upcall_vector;
	167	r = 0;
	168	break;
	169
	170	default:
	171	break;
	172	}
	173
	174	mutex_unlock(&kvm->lock);
	175	return r;
	176	}
	177
	178	int kvm_xen_vcpu_set_attr(struct kvm_vcpu vcpu, struct kvm_xen_vcpu_attr data)
	179	{
	180	int idx, r = -ENOENT;
	181
	182	mutex_lock(&vcpu->kvm->lock);
	183	idx = srcu_read_lock(&vcpu->kvm->srcu);
	184
	185	switch (data->type) {
	186	case KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO:
	187	/* No compat necessary here. */
	188	BUILD_BUG_ON(sizeof(struct vcpu_info) !=
	189	sizeof(struct compat_vcpu_info));
	190
	191	if (data->u.gpa == GPA_INVALID) {
	192	vcpu->arch.xen.vcpu_info_set = false;
	193	break;
	194	}
	195
	196	r = kvm_gfn_to_hva_cache_init(vcpu->kvm,
	197	&vcpu->arch.xen.vcpu_info_cache,
	198	data->u.gpa,
	199	sizeof(struct vcpu_info));
	200	if (!r) {
	201	vcpu->arch.xen.vcpu_info_set = true;
	202	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
	203	}
	204	break;
	205
	206	case KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO:
	207	if (data->u.gpa == GPA_INVALID) {
	208	vcpu->arch.xen.vcpu_time_info_set = false;
	209	break;
	210	}
	211
	212	r = kvm_gfn_to_hva_cache_init(vcpu->kvm,
	213	&vcpu->arch.xen.vcpu_time_info_cache,
	214	data->u.gpa,
	215	sizeof(struct pvclock_vcpu_time_info));
	216	if (!r) {
	217	vcpu->arch.xen.vcpu_time_info_set = true;
	218	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
	219	}
	220	break;
	221
	222	default:
	223	break;
	224	}
	225
	226	srcu_read_unlock(&vcpu->kvm->srcu, idx);
	227	mutex_unlock(&vcpu->kvm->lock);
	228	return r;
	229	}
	230
	231	int kvm_xen_vcpu_get_attr(struct kvm_vcpu vcpu, struct kvm_xen_vcpu_attr data)
	232	{
	233	int r = -ENOENT;
	234
	235	mutex_lock(&vcpu->kvm->lock);
	236
	237	switch (data->type) {
	238	case KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO:
	239	if (vcpu->arch.xen.vcpu_info_set)
	240	data->u.gpa = vcpu->arch.xen.vcpu_info_cache.gpa;
	241	else
	242	data->u.gpa = GPA_INVALID;
	243	r = 0;
	244	break;
	245
	246	case KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO:
	247	if (vcpu->arch.xen.vcpu_time_info_set)
	248	data->u.gpa = vcpu->arch.xen.vcpu_time_info_cache.gpa;
	249	else
	250	data->u.gpa = GPA_INVALID;
	251	r = 0;
	252	break;
	253
	254	default:
	255	break;
	256	}
	257
	258	mutex_unlock(&vcpu->kvm->lock);
	259	return r;
	260	}
	261
	262	int kvm_xen_write_hypercall_page(struct kvm_vcpu *vcpu, u64 data)
	263	{
	264	struct kvm *kvm = vcpu->kvm;
	265	u32 page_num = data & ~PAGE_MASK;
	266	u64 page_addr = data & PAGE_MASK;
	267	bool lm = is_long_mode(vcpu);
	268
	269	/* Latch long_mode for shared_info pages etc. */
	270	vcpu->kvm->arch.xen.long_mode = lm;
	271
	272	/*
	273	* If Xen hypercall intercept is enabled, fill the hypercall
	274	* page with VMCALL/VMMCALL instructions since that's what
	275	* we catch. Else the VMM has provided the hypercall pages
	276	* with instructions of its own choosing, so use those.
	277	*/
	278	if (kvm_xen_hypercall_enabled(kvm)) {
	279	u8 instructions[32];
	280	int i;
	281
	282	if (page_num)
	283	return 1;
	284
	285	/* mov imm32, %eax */
	286	instructions[0] = 0xb8;
	287
	288	/* vmcall / vmmcall */
	289	kvm_x86_ops.patch_hypercall(vcpu, instructions + 5);
	290
	291	/* ret */
	292	instructions[8] = 0xc3;
	293
	294	/* int3 to pad */
	295	memset(instructions + 9, 0xcc, sizeof(instructions) - 9);
	296
	297	for (i = 0; i < PAGE_SIZE / sizeof(instructions); i++) {
	298	(u32 )&instructions[1] = i;
	299	if (kvm_vcpu_write_guest(vcpu,
	300	page_addr + (i * sizeof(instructions)),
	301	instructions, sizeof(instructions)))
	302	return 1;
	303	}
	304	} else {
	305	/*
	306	* Note, truncation is a non-issue as 'lm' is guaranteed to be
	307	* false for a 32-bit kernel, i.e. when hva_t is only 4 bytes.
	308	*/
	309	hva_t blob_addr = lm ? kvm->arch.xen_hvm_config.blob_addr_64
	310	: kvm->arch.xen_hvm_config.blob_addr_32;
	311	u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
	312	: kvm->arch.xen_hvm_config.blob_size_32;
	313	u8 *page;
	314
	315	if (page_num >= blob_size)
	316	return 1;
	317
	318	blob_addr += page_num * PAGE_SIZE;
	319
	320	page = memdup_user((u8 __user *)blob_addr, PAGE_SIZE);
	321	if (IS_ERR(page))
	322	return PTR_ERR(page);
	323
	324	if (kvm_vcpu_write_guest(vcpu, page_addr, page, PAGE_SIZE)) {
	325	kfree(page);
	326	return 1;
	327	}
	328	}
	329	return 0;
	330	}
	331
	332	int kvm_xen_hvm_config(struct kvm kvm, struct kvm_xen_hvm_config xhc)
	333	{
	334	if (xhc->flags & ~KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL)
	335	return -EINVAL;
	336
	337	/*
	338	* With hypercall interception the kernel generates its own
	339	* hypercall page so it must not be provided.
	340	*/
	341	if ((xhc->flags & KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL) &&
	342	(xhc->blob_addr_32 \|\| xhc->blob_addr_64 \|\|
	343	xhc->blob_size_32 \|\| xhc->blob_size_64))
	344	return -EINVAL;
	345
	346	mutex_lock(&kvm->lock);
	347
	348	if (xhc->msr && !kvm->arch.xen_hvm_config.msr)
	349	static_branch_inc(&kvm_xen_enabled.key);
	350	else if (!xhc->msr && kvm->arch.xen_hvm_config.msr)
	351	static_branch_slow_dec_deferred(&kvm_xen_enabled);
	352
	353	memcpy(&kvm->arch.xen_hvm_config, xhc, sizeof(*xhc));
	354
	355	mutex_unlock(&kvm->lock);
	356	return 0;
	357	}
	358
	359	void kvm_xen_destroy_vm(struct kvm *kvm)
	360	{
	361	if (kvm->arch.xen_hvm_config.msr)
	362	static_branch_slow_dec_deferred(&kvm_xen_enabled);
	363	}
	364
	365	static int kvm_xen_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result)
	366	{
	367	kvm_rax_write(vcpu, result);
	368	return kvm_skip_emulated_instruction(vcpu);
	369	}
	370
	371	static int kvm_xen_hypercall_complete_userspace(struct kvm_vcpu *vcpu)
	372	{
	373	struct kvm_run *run = vcpu->run;
	374
	375	if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.xen.hypercall_rip)))
	376	return 1;
	377
	378	return kvm_xen_hypercall_set_result(vcpu, run->xen.u.hcall.result);
	379	}
	380
	381	int kvm_xen_hypercall(struct kvm_vcpu *vcpu)
	382	{
	383	bool longmode;
	384	u64 input, params[6];
	385
	386	input = (u64)kvm_register_read(vcpu, VCPU_REGS_RAX);
	387
	388	/* Hyper-V hypercalls get bit 31 set in EAX */
	389	if ((input & 0x80000000) &&
	390	kvm_hv_hypercall_enabled(vcpu))
	391	return kvm_hv_hypercall(vcpu);
	392
	393	longmode = is_64_bit_mode(vcpu);
	394	if (!longmode) {
	395	params[0] = (u32)kvm_rbx_read(vcpu);
	396	params[1] = (u32)kvm_rcx_read(vcpu);
	397	params[2] = (u32)kvm_rdx_read(vcpu);
	398	params[3] = (u32)kvm_rsi_read(vcpu);
	399	params[4] = (u32)kvm_rdi_read(vcpu);
	400	params[5] = (u32)kvm_rbp_read(vcpu);
	401	}
	402	#ifdef CONFIG_X86_64
	403	else {
	404	params[0] = (u64)kvm_rdi_read(vcpu);
	405	params[1] = (u64)kvm_rsi_read(vcpu);
	406	params[2] = (u64)kvm_rdx_read(vcpu);
	407	params[3] = (u64)kvm_r10_read(vcpu);
	408	params[4] = (u64)kvm_r8_read(vcpu);
	409	params[5] = (u64)kvm_r9_read(vcpu);
	410	}
	411	#endif
	412	trace_kvm_xen_hypercall(input, params[0], params[1], params[2],
	413	params[3], params[4], params[5]);
	414
	415	vcpu->run->exit_reason = KVM_EXIT_XEN;
	416	vcpu->run->xen.type = KVM_EXIT_XEN_HCALL;
	417	vcpu->run->xen.u.hcall.longmode = longmode;
	418	vcpu->run->xen.u.hcall.cpl = kvm_x86_ops.get_cpl(vcpu);
	419	vcpu->run->xen.u.hcall.input = input;
	420	vcpu->run->xen.u.hcall.params[0] = params[0];
	421	vcpu->run->xen.u.hcall.params[1] = params[1];
	422	vcpu->run->xen.u.hcall.params[2] = params[2];
	423	vcpu->run->xen.u.hcall.params[3] = params[3];
	424	vcpu->run->xen.u.hcall.params[4] = params[4];
	425	vcpu->run->xen.u.hcall.params[5] = params[5];
	426	vcpu->arch.xen.hypercall_rip = kvm_get_linear_rip(vcpu);
	427	vcpu->arch.complete_userspace_io =
	428	kvm_xen_hypercall_complete_userspace;
	429
	430	return 0;
	431	}

arch/x86/kvm/xen.h+78−0

@@ -0,0 +1,78 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Copyright © 2019 Oracle and/or its affiliates. All rights reserved.
	4	* Copyright © 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
	5	*
	6	* KVM Xen emulation
	7	*/
	8
	9	#ifndef __ARCH_X86_KVM_XEN_H__
	10	#define __ARCH_X86_KVM_XEN_H__
	11
	12	#include <linux/jump_label_ratelimit.h>
	13
	14	extern struct static_key_false_deferred kvm_xen_enabled;
	15
	16	int __kvm_xen_has_interrupt(struct kvm_vcpu *vcpu);
	17	int kvm_xen_vcpu_set_attr(struct kvm_vcpu vcpu, struct kvm_xen_vcpu_attr data);
	18	int kvm_xen_vcpu_get_attr(struct kvm_vcpu vcpu, struct kvm_xen_vcpu_attr data);
	19	int kvm_xen_hvm_set_attr(struct kvm kvm, struct kvm_xen_hvm_attr data);
	20	int kvm_xen_hvm_get_attr(struct kvm kvm, struct kvm_xen_hvm_attr data);
	21	int kvm_xen_hypercall(struct kvm_vcpu *vcpu);
	22	int kvm_xen_write_hypercall_page(struct kvm_vcpu *vcpu, u64 data);
	23	int kvm_xen_hvm_config(struct kvm kvm, struct kvm_xen_hvm_config xhc);
	24	void kvm_xen_destroy_vm(struct kvm *kvm);
	25
	26	static inline bool kvm_xen_hypercall_enabled(struct kvm *kvm)
	27	{
	28	return static_branch_unlikely(&kvm_xen_enabled.key) &&
	29	(kvm->arch.xen_hvm_config.flags &
	30	KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL);
	31	}
	32
	33	static inline int kvm_xen_has_interrupt(struct kvm_vcpu *vcpu)
	34	{
	35	if (static_branch_unlikely(&kvm_xen_enabled.key) &&
	36	vcpu->arch.xen.vcpu_info_set && vcpu->kvm->arch.xen.upcall_vector)
	37	return __kvm_xen_has_interrupt(vcpu);
	38
	39	return 0;
	40	}
	41
	42	/* 32-bit compatibility definitions, also used natively in 32-bit build */
	43	#include <asm/pvclock-abi.h>
	44	#include <asm/xen/interface.h>
	45
	46	struct compat_arch_vcpu_info {
	47	unsigned int cr2;
	48	unsigned int pad[5];
	49	};
	50
	51	struct compat_vcpu_info {
	52	uint8_t evtchn_upcall_pending;
	53	uint8_t evtchn_upcall_mask;
	54	uint16_t pad;
	55	uint32_t evtchn_pending_sel;
	56	struct compat_arch_vcpu_info arch;
	57	struct pvclock_vcpu_time_info time;
	58	}; /* 64 bytes (x86) */
	59
	60	struct compat_arch_shared_info {
	61	unsigned int max_pfn;
	62	unsigned int pfn_to_mfn_frame_list_list;
	63	unsigned int nmi_reason;
	64	unsigned int p2m_cr3;
	65	unsigned int p2m_vaddr;
	66	unsigned int p2m_generation;
	67	uint32_t wc_sec_hi;
	68	};
	69
	70	struct compat_shared_info {
	71	struct compat_vcpu_info vcpu_info[MAX_VIRT_CPUS];
	72	uint32_t evtchn_pending[32];
	73	uint32_t evtchn_mask[32];
	74	struct pvclock_wall_clock wc;
	75	struct compat_arch_shared_info arch;
	76	};
	77
	78	#endif /* __ARCH_X86_KVM_XEN_H__ */

arch/xtensa/include/asm/spinlock.h+1−1

@@ -12,8 +12,8 @@ · 11 unmodified lines
12	#define _XTENSA_SPINLOCK_H	12	#define _XTENSA_SPINLOCK_H
13		13
14	#include <asm/barrier.h>	14	#include <asm/barrier.h>
15	#include <asm/qrwlock.h>
16	#include <asm/qspinlock.h>	15	#include <asm/qspinlock.h>
		16	#include <asm/qrwlock.h>
17		17
18	#define smp_mb__after_spinlock() smp_mb()	18	#define smp_mb__after_spinlock() smp_mb()
19		19

drivers/crypto/ccp/sev-dev.c+1−0

@@ -128,6 +128,7 @@ static int sev_cmd_buffer_len(int cmd) · 127 unmodified lines
128	case SEV_CMD_LAUNCH_UPDATE_SECRET: return sizeof(struct sev_data_launch_secret);	128	case SEV_CMD_LAUNCH_UPDATE_SECRET: return sizeof(struct sev_data_launch_secret);
129	case SEV_CMD_DOWNLOAD_FIRMWARE: return sizeof(struct sev_data_download_firmware);	129	case SEV_CMD_DOWNLOAD_FIRMWARE: return sizeof(struct sev_data_download_firmware);
130	case SEV_CMD_GET_ID: return sizeof(struct sev_data_get_id);	130	case SEV_CMD_GET_ID: return sizeof(struct sev_data_get_id);
		131	case SEV_CMD_ATTESTATION_REPORT: return sizeof(struct sev_data_attestation_report);
131	default: return 0;	132	default: return 0;
132	}	133	}
133		134

drivers/gpu/drm/i915/gvt/kvmgt.c+6−6

@@ -1703,7 +1703,7 @@ static int kvmgt_page_track_add(unsigned long handle, u64 gfn) · 1702 unmodified lines
1703	return -EINVAL;	1703	return -EINVAL;
1704	}	1704	}
1705		1705
1706	spin_lock(&kvm->mmu_lock);	1706	write_lock(&kvm->mmu_lock);
1707		1707
1708	if (kvmgt_gfn_is_write_protected(info, gfn))	1708	if (kvmgt_gfn_is_write_protected(info, gfn))
1709	goto out;	1709	goto out;
@@ -1712,7 +1712,7 @@ static int kvmgt_page_track_add(unsigned long handle, u64 gfn) · 2 unmodified lines
1712	kvmgt_protect_table_add(info, gfn);	1712	kvmgt_protect_table_add(info, gfn);
1713		1713
1714	out:	1714	out:
1715	spin_unlock(&kvm->mmu_lock);	1715	write_unlock(&kvm->mmu_lock);
1716	srcu_read_unlock(&kvm->srcu, idx);	1716	srcu_read_unlock(&kvm->srcu, idx);
1717	return 0;	1717	return 0;
1718	}	1718	}
@@ -1737,7 +1737,7 @@ static int kvmgt_page_track_remove(unsigned long handle, u64 gfn) · 18 unmodified lines
1737	return -EINVAL;	1737	return -EINVAL;
1738	}	1738	}
1739		1739
1740	spin_lock(&kvm->mmu_lock);	1740	write_lock(&kvm->mmu_lock);
1741		1741
1742	if (!kvmgt_gfn_is_write_protected(info, gfn))	1742	if (!kvmgt_gfn_is_write_protected(info, gfn))
1743	goto out;	1743	goto out;
@@ -1746,7 +1746,7 @@ static int kvmgt_page_track_remove(unsigned long handle, u64 gfn) · 2 unmodified lines
1746	kvmgt_protect_table_del(info, gfn);	1746	kvmgt_protect_table_del(info, gfn);
1747		1747
1748	out:	1748	out:
1749	spin_unlock(&kvm->mmu_lock);	1749	write_unlock(&kvm->mmu_lock);
1750	srcu_read_unlock(&kvm->srcu, idx);	1750	srcu_read_unlock(&kvm->srcu, idx);
1751	return 0;	1751	return 0;
1752	}	1752	}
@@ -1772,7 +1772,7 @@ static void kvmgt_page_track_flush_slot(struct kvm *kvm, · 19 unmodified lines
1772	struct kvmgt_guest_info *info = container_of(node,	1772	struct kvmgt_guest_info *info = container_of(node,
1773	struct kvmgt_guest_info, track_node);	1773	struct kvmgt_guest_info, track_node);
1774		1774
1775	spin_lock(&kvm->mmu_lock);	1775	write_lock(&kvm->mmu_lock);
1776	for (i = 0; i < slot->npages; i++) {	1776	for (i = 0; i < slot->npages; i++) {
1777	gfn = slot->base_gfn + i;	1777	gfn = slot->base_gfn + i;
1778	if (kvmgt_gfn_is_write_protected(info, gfn)) {	1778	if (kvmgt_gfn_is_write_protected(info, gfn)) {
@@ -1781,7 +1781,7 @@ static void kvmgt_page_track_flush_slot(struct kvm *kvm, · 2 unmodified lines
1781	kvmgt_protect_table_del(info, gfn);	1781	kvmgt_protect_table_del(info, gfn);
1782	}	1782	}
1783	}	1783	}
1784	spin_unlock(&kvm->mmu_lock);	1784	write_unlock(&kvm->mmu_lock);
1785	}	1785	}
1786		1786
1787	static bool __kvmgt_vgpu_exist(struct intel_vgpu vgpu, struct kvm kvm)	1787	static bool __kvmgt_vgpu_exist(struct intel_vgpu vgpu, struct kvm kvm)

fs/dax.c+3−2

 		address = pgoff_address(index, vma);
 		/*
-		 * Note because we provide range to follow_pte it will call
+		 * follow_invalidate_pte() will use the range to call
 		 * mmu_notifier_invalidate_range_start() on our behalf before
 		 * taking any lock.
 		 */
-		if (follow_pte(vma->vm_mm, address, &range, &ptep, &pmdp, &ptl))
+		if (follow_invalidate_pte(vma->vm_mm, address, &range, &ptep,
+					  &pmdp, &ptl))
 			continue;
 		/*

include/asm-generic/export.h+1−1

@@ -33,7 +33,7 @@ · 32 unmodified lines
33	*/	33	*/
34		34
35	.macro ___EXPORT_SYMBOL name,val,sec	35	.macro ___EXPORT_SYMBOL name,val,sec
36	#ifdef CONFIG_MODULES	36	#if defined(CONFIG_MODULES) && !defined(__DISABLE_EXPORTS)
37	.section ___ksymtab\sec+\name,"a"	37	.section ___ksymtab\sec+\name,"a"
38	.balign KSYM_ALIGN	38	.balign KSYM_ALIGN
39	__ksymtab_\name:	39	__ksymtab_\name:

include/asm-generic/qrwlock.h+19−6

 #include <asm-generic/qrwlock_types.h>
+/* Must be included from asm/spinlock.h after defining arch_spin_is_locked.  */
 /*
  * Writer states & reader shift and bias.
  */
 	smp_store_release(&lock->wlocked, 0);
 }
+/**
+ * queued_rwlock_is_contended - check if the lock is contended
+ * @lock : Pointer to queue rwlock structure
+ * Return: 1 if lock contended, 0 otherwise
+ */
+static inline int queued_rwlock_is_contended(struct qrwlock *lock)
+{
+	return arch_spin_is_locked(&lock->wait_lock);
+}
 /*
  * Remapping rwlock architecture specific functions to the corresponding
  * queue rwlock functions.
  */
-#define arch_read_lock(l)	queued_read_lock(l)
+#define arch_read_lock(l)		queued_read_lock(l)
-#define arch_write_lock(l)	queued_write_lock(l)
+#define arch_write_lock(l)		queued_write_lock(l)
-#define arch_read_trylock(l)	queued_read_trylock(l)
+#define arch_read_trylock(l)		queued_read_trylock(l)
-#define arch_write_trylock(l)	queued_write_trylock(l)
+#define arch_write_trylock(l)		queued_write_trylock(l)
-#define arch_read_unlock(l)	queued_read_unlock(l)
+#define arch_read_unlock(l)		queued_read_unlock(l)
-#define arch_write_unlock(l)	queued_write_unlock(l)
+#define arch_write_unlock(l)		queued_write_unlock(l)
+#define arch_rwlock_is_contended(l)	queued_rwlock_is_contended(l)
 #endif /* __ASM_GENERIC_QRWLOCK_H */

include/linux/kvm_host.h+7−3

 #define KVM_PRIVATE_MEM_SLOTS 0
 #endif
-#ifndef KVM_MEM_SLOTS_NUM
+#define KVM_MEM_SLOTS_NUM SHRT_MAX
-#define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS)
+#define KVM_USER_MEM_SLOTS (KVM_MEM_SLOTS_NUM - KVM_PRIVATE_MEM_SLOTS)
-#endif
 #ifndef __KVM_VCPU_MULTIPLE_ADDRESS_SPACE
 static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu)
 };
 struct kvm {
+#ifdef KVM_HAVE_MMU_RWLOCK
+	rwlock_t mmu_lock;
+#else
 	spinlock_t mmu_lock;
+#endif /* KVM_HAVE_MMU_RWLOCK */
 	struct mutex slots_lock;
 	struct mm_struct *mm; /* userspace tied to this vm */
 	struct kvm_memslots __rcu *memslots[KVM_ADDRESS_SPACE_NUM];

include/linux/mm.h+4−2

 		unsigned long end, unsigned long floor, unsigned long ceiling);
 int
 copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma);
+int follow_invalidate_pte(struct mm_struct *mm, unsigned long address,
+			  struct mmu_notifier_range *range, pte_t **ptepp,
+			  pmd_t **pmdpp, spinlock_t **ptlp);
 int follow_pte(struct mm_struct *mm, unsigned long address,
-		struct mmu_notifier_range *range, pte_t **ptepp, pmd_t **pmdpp,
+	       pte_t **ptepp, spinlock_t **ptlp);
-		spinlock_t **ptlp);
 int follow_pfn(struct vm_area_struct *vma, unsigned long address,
 	unsigned long *pfn);
 int follow_phys(struct vm_area_struct *vma, unsigned long address,

include/linux/psp-sev.h+17−0

@@ -66,6 +66,7 @@ enum sev_cmd { · 65 unmodified lines
66	SEV_CMD_LAUNCH_MEASURE = 0x033,	66	SEV_CMD_LAUNCH_MEASURE = 0x033,
67	SEV_CMD_LAUNCH_UPDATE_SECRET = 0x034,	67	SEV_CMD_LAUNCH_UPDATE_SECRET = 0x034,
68	SEV_CMD_LAUNCH_FINISH = 0x035,	68	SEV_CMD_LAUNCH_FINISH = 0x035,
		69	SEV_CMD_ATTESTATION_REPORT = 0x036,
69		70
70	/* Guest migration commands (outgoing) */	71	/* Guest migration commands (outgoing) */
71	SEV_CMD_SEND_START = 0x040,	72	SEV_CMD_SEND_START = 0x040,
@@ -483,6 +484,22 @@ struct sev_data_dbg { · 411 unmodified lines
483	u32 len; /* In */	484	u32 len; /* In */
484	} __packed;	485	} __packed;
485		486
		487	/**
		488	* struct sev_data_attestation_report - SEV_ATTESTATION_REPORT command parameters
		489	*
		490	* @handle: handle of the VM
		491	* @mnonce: a random nonce that will be included in the report.
		492	* @address: physical address where the report will be copied.
		493	* @len: length of the physical buffer.
		494	*/
		495	struct sev_data_attestation_report {
		496	u32 handle; /* In */
		497	u32 reserved;
		498	u64 address; /* In */
		499	u8 mnonce[16]; /* In */
		500	u32 len; /* In/Out */
		501	} __packed;
		502
486	#ifdef CONFIG_CRYPTO_DEV_SP_PSP	503	#ifdef CONFIG_CRYPTO_DEV_SP_PSP
487		504
488	/**	505	/**

include/linux/rwlock.h+7−0

@@ -128,4 +128,11 @@ do { \ · 127 unmodified lines
128	1 : ({ local_irq_restore(flags); 0; }); \	128	1 : ({ local_irq_restore(flags); 0; }); \
129	})	129	})
130		130
		131	#ifdef arch_rwlock_is_contended
		132	#define rwlock_is_contended(lock) \
		133	arch_rwlock_is_contended(&(lock)->raw_lock)
		134	#else
		135	#define rwlock_is_contended(lock) ((void)(lock), 0)
		136	#endif /* arch_rwlock_is_contended */
		137
131	#endif /* __LINUX_RWLOCK_H */	138	#endif /* __LINUX_RWLOCK_H */

include/linux/sched.h+29−0

@@ -1904,12 +1904,24 @@ static inline int _cond_resched(void) { return 0; } · 1903 unmodified lines
1904	})	1904	})
1905		1905
1906	extern int __cond_resched_lock(spinlock_t *lock);	1906	extern int __cond_resched_lock(spinlock_t *lock);
		1907	extern int __cond_resched_rwlock_read(rwlock_t *lock);
		1908	extern int __cond_resched_rwlock_write(rwlock_t *lock);
1907		1909
1908	#define cond_resched_lock(lock) ({ \	1910	#define cond_resched_lock(lock) ({ \
1909	___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\	1911	___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\
1910	__cond_resched_lock(lock); \	1912	__cond_resched_lock(lock); \
1911	})	1913	})
1912		1914
		1915	#define cond_resched_rwlock_read(lock) ({ \
		1916	__might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \
		1917	__cond_resched_rwlock_read(lock); \
		1918	})
		1919
		1920	#define cond_resched_rwlock_write(lock) ({ \
		1921	__might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \
		1922	__cond_resched_rwlock_write(lock); \
		1923	})
		1924
1913	static inline void cond_resched_rcu(void)	1925	static inline void cond_resched_rcu(void)
1914	{	1926	{
1915	#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) \|\| !defined(CONFIG_PREEMPT_RCU)	1927	#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) \|\| !defined(CONFIG_PREEMPT_RCU)
@@ -1933,6 +1945,23 @@ static inline int spin_needbreak(spinlock_t *lock) · 17 unmodified lines
1933	#endif	1945	#endif
1934	}	1946	}
1935		1947
		1948	/*
		1949	* Check if a rwlock is contended.
		1950	* Returns non-zero if there is another task waiting on the rwlock.
		1951	* Returns zero if the lock is not contended or the system / underlying
		1952	* rwlock implementation does not support contention detection.
		1953	* Technically does not depend on CONFIG_PREEMPTION, but a general need
		1954	* for low latency.
		1955	*/
		1956	static inline int rwlock_needbreak(rwlock_t *lock)
		1957	{
		1958	#ifdef CONFIG_PREEMPTION
		1959	return rwlock_is_contended(lock);
		1960	#else
		1961	return 0;
		1962	#endif
		1963	}
		1964
1936	static __always_inline bool need_resched(void)	1965	static __always_inline bool need_resched(void)
1937	{	1966	{
1938	return unlikely(tif_need_resched());	1967	return unlikely(tif_need_resched());

include/uapi/linux/kvm.h+74−0

@@ -216,6 +216,20 @@ struct kvm_hyperv_exit { · 215 unmodified lines
216	} u;	216	} u;
217	};	217	};
218		218
		219	struct kvm_xen_exit {
		220	#define KVM_EXIT_XEN_HCALL 1
		221	__u32 type;
		222	union {
		223	struct {
		224	__u32 longmode;
		225	__u32 cpl;
		226	__u64 input;
		227	__u64 result;
		228	__u64 params[6];
		229	} hcall;
		230	} u;
		231	};
		232
219	#define KVM_S390_GET_SKEYS_NONE 1	233	#define KVM_S390_GET_SKEYS_NONE 1
220	#define KVM_S390_SKEYS_MAX 1048576	234	#define KVM_S390_SKEYS_MAX 1048576
221		235
@@ -252,6 +266,8 @@ struct kvm_hyperv_exit { · 30 unmodified lines
252	#define KVM_EXIT_X86_WRMSR 30	266	#define KVM_EXIT_X86_WRMSR 30
253	#define KVM_EXIT_DIRTY_RING_FULL 31	267	#define KVM_EXIT_DIRTY_RING_FULL 31
254	#define KVM_EXIT_AP_RESET_HOLD 32	268	#define KVM_EXIT_AP_RESET_HOLD 32
		269	#define KVM_EXIT_X86_BUS_LOCK 33
		270	#define KVM_EXIT_XEN 34
255		271
256	/* For KVM_EXIT_INTERNAL_ERROR */	272	/* For KVM_EXIT_INTERNAL_ERROR */
257	/* Emulate instruction failed. */	273	/* Emulate instruction failed. */
@@ -428,6 +444,8 @@ struct kvm_run { · 170 unmodified lines
428	__u32 index; /* kernel -> user */	444	__u32 index; /* kernel -> user */
429	__u64 data; /* kernel <-> user */	445	__u64 data; /* kernel <-> user */
430	} msr;	446	} msr;
		447	/* KVM_EXIT_XEN */
		448	struct kvm_xen_exit xen;
431	/* Fix the size of the union. */	449	/* Fix the size of the union. */
432	char padding[256];	450	char padding[256];
433	};	451	};
@@ -1058,6 +1076,8 @@ struct kvm_ppc_resize_hpt { · 624 unmodified lines
1058	#define KVM_CAP_ENFORCE_PV_FEATURE_CPUID 190	1076	#define KVM_CAP_ENFORCE_PV_FEATURE_CPUID 190
1059	#define KVM_CAP_SYS_HYPERV_CPUID 191	1077	#define KVM_CAP_SYS_HYPERV_CPUID 191
1060	#define KVM_CAP_DIRTY_LOG_RING 192	1078	#define KVM_CAP_DIRTY_LOG_RING 192
		1079	#define KVM_CAP_X86_BUS_LOCK_EXIT 193
		1080	#define KVM_CAP_PPC_DAWR1 194
1061		1081
1062	#ifdef KVM_CAP_IRQ_ROUTING	1082	#ifdef KVM_CAP_IRQ_ROUTING
1063		1083
@@ -1131,6 +1151,10 @@ struct kvm_x86_mce { · 67 unmodified lines
1131	#endif	1151	#endif
1132		1152
1133	#ifdef KVM_CAP_XEN_HVM	1153	#ifdef KVM_CAP_XEN_HVM
		1154	#define KVM_XEN_HVM_CONFIG_HYPERCALL_MSR (1 << 0)
		1155	#define KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL (1 << 1)
		1156	#define KVM_XEN_HVM_CONFIG_SHARED_INFO (1 << 2)
		1157
1134	struct kvm_xen_hvm_config {	1158	struct kvm_xen_hvm_config {
1135	__u32 flags;	1159	__u32 flags;
1136	__u32 msr;	1160	__u32 msr;
@@ -1565,6 +1589,45 @@ struct kvm_pv_cmd { · 428 unmodified lines
1565	/* Available with KVM_CAP_DIRTY_LOG_RING */	1589	/* Available with KVM_CAP_DIRTY_LOG_RING */
1566	#define KVM_RESET_DIRTY_RINGS _IO(KVMIO, 0xc7)	1590	#define KVM_RESET_DIRTY_RINGS _IO(KVMIO, 0xc7)
1567		1591
		1592	/* Per-VM Xen attributes */
		1593	#define KVM_XEN_HVM_GET_ATTR _IOWR(KVMIO, 0xc8, struct kvm_xen_hvm_attr)
		1594	#define KVM_XEN_HVM_SET_ATTR _IOW(KVMIO, 0xc9, struct kvm_xen_hvm_attr)
		1595
		1596	struct kvm_xen_hvm_attr {
		1597	__u16 type;
		1598	__u16 pad[3];
		1599	union {
		1600	__u8 long_mode;
		1601	__u8 vector;
		1602	struct {
		1603	__u64 gfn;
		1604	} shared_info;
		1605	__u64 pad[8];
		1606	} u;
		1607	};
		1608
		1609	/* Available with KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_SHARED_INFO */
		1610	#define KVM_XEN_ATTR_TYPE_LONG_MODE 0x0
		1611	#define KVM_XEN_ATTR_TYPE_SHARED_INFO 0x1
		1612	#define KVM_XEN_ATTR_TYPE_UPCALL_VECTOR 0x2
		1613
		1614	/* Per-vCPU Xen attributes */
		1615	#define KVM_XEN_VCPU_GET_ATTR _IOWR(KVMIO, 0xca, struct kvm_xen_vcpu_attr)
		1616	#define KVM_XEN_VCPU_SET_ATTR _IOW(KVMIO, 0xcb, struct kvm_xen_vcpu_attr)
		1617
		1618	struct kvm_xen_vcpu_attr {
		1619	__u16 type;
		1620	__u16 pad[3];
		1621	union {
		1622	__u64 gpa;
		1623	__u64 pad[8];
		1624	} u;
		1625	};
		1626
		1627	/* Available with KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_SHARED_INFO */
		1628	#define KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO 0x0
		1629	#define KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO 0x1
		1630
1568	/* Secure Encrypted Virtualization command */	1631	/* Secure Encrypted Virtualization command */
1569	enum sev_cmd_id {	1632	enum sev_cmd_id {
1570	/* Guest initialization commands */	1633	/* Guest initialization commands */
@@ -1593,6 +1656,8 @@ enum sev_cmd_id { · 22 unmodified lines
1593	KVM_SEV_DBG_ENCRYPT,	1656	KVM_SEV_DBG_ENCRYPT,
1594	/* Guest certificates commands */	1657	/* Guest certificates commands */
1595	KVM_SEV_CERT_EXPORT,	1658	KVM_SEV_CERT_EXPORT,
		1659	/* Attestation report */
		1660	KVM_SEV_GET_ATTESTATION_REPORT,
1596		1661
1597	KVM_SEV_NR_MAX,	1662	KVM_SEV_NR_MAX,
1598	};	1663	};
@@ -1645,6 +1710,12 @@ struct kvm_sev_dbg { · 46 unmodified lines
1645	__u32 len;	1710	__u32 len;
1646	};	1711	};
1647		1712
		1713	struct kvm_sev_attestation_report {
		1714	__u8 mnonce[16];
		1715	__u64 uaddr;
		1716	__u32 len;
		1717	};
		1718
1648	#define KVM_DEV_ASSIGN_ENABLE_IOMMU (1 << 0)	1719	#define KVM_DEV_ASSIGN_ENABLE_IOMMU (1 << 0)
1649	#define KVM_DEV_ASSIGN_PCI_2_3 (1 << 1)	1720	#define KVM_DEV_ASSIGN_PCI_2_3 (1 << 1)
1650	#define KVM_DEV_ASSIGN_MASK_INTX (1 << 2)	1721	#define KVM_DEV_ASSIGN_MASK_INTX (1 << 2)
@@ -1766,4 +1837,7 @@ struct kvm_dirty_gfn { · 115 unmodified lines
1766	__u64 offset;	1837	__u64 offset;
1767	};	1838	};
1768		1839
		1840	#define KVM_BUS_LOCK_DETECTION_OFF (1 << 0)
		1841	#define KVM_BUS_LOCK_DETECTION_EXIT (1 << 1)
		1842
1769	#endif /* __LINUX_KVM_H */	1843	#endif /* __LINUX_KVM_H */

include/xen/interface/xen.h+3−1

@@ -598,7 +598,9 @@ struct shared_info { · 597 unmodified lines
598	* their gettimeofday() syscall on this wallclock-base value.	598	* their gettimeofday() syscall on this wallclock-base value.
599	*/	599	*/
600	struct pvclock_wall_clock wc;	600	struct pvclock_wall_clock wc;
601		601	#ifndef CONFIG_X86_32
		602	uint32_t wc_sec_hi;
		603	#endif
602	struct arch_shared_info arch;	604	struct arch_shared_info arch;
603		605
604	};	606	};

kernel/locking/qrwlock.c+0−1

@@ -12,7 +12,6 @@ · 11 unmodified lines
12	#include <linux/percpu.h>	12	#include <linux/percpu.h>
13	#include <linux/hardirq.h>	13	#include <linux/hardirq.h>
14	#include <linux/spinlock.h>	14	#include <linux/spinlock.h>
15	#include <asm/qrwlock.h>
16		15
17	/**	16	/**
18	* queued_read_lock_slowpath - acquire read lock of a queue rwlock	17	* queued_read_lock_slowpath - acquire read lock of a queue rwlock

kernel/sched/core.c+40−0

@@ -7028,6 +7028,46 @@ int __cond_resched_lock(spinlock_t *lock) · 7027 unmodified lines
7028	}	7028	}
7029	EXPORT_SYMBOL(__cond_resched_lock);	7029	EXPORT_SYMBOL(__cond_resched_lock);
7030		7030
		7031	int __cond_resched_rwlock_read(rwlock_t *lock)
		7032	{
		7033	int resched = should_resched(PREEMPT_LOCK_OFFSET);
		7034	int ret = 0;
		7035
		7036	lockdep_assert_held_read(lock);
		7037
		7038	if (rwlock_needbreak(lock) \|\| resched) {
		7039	read_unlock(lock);
		7040	if (resched)
		7041	preempt_schedule_common();
		7042	else
		7043	cpu_relax();
		7044	ret = 1;
		7045	read_lock(lock);
		7046	}
		7047	return ret;
		7048	}
		7049	EXPORT_SYMBOL(__cond_resched_rwlock_read);
		7050
		7051	int __cond_resched_rwlock_write(rwlock_t *lock)
		7052	{
		7053	int resched = should_resched(PREEMPT_LOCK_OFFSET);
		7054	int ret = 0;
		7055
		7056	lockdep_assert_held_write(lock);
		7057
		7058	if (rwlock_needbreak(lock) \|\| resched) {
		7059	write_unlock(lock);
		7060	if (resched)
		7061	preempt_schedule_common();
		7062	else
		7063	cpu_relax();
		7064	ret = 1;
		7065	write_lock(lock);
		7066	}
		7067	return ret;
		7068	}
		7069	EXPORT_SYMBOL(__cond_resched_rwlock_write);
		7070
7031	/**	7071	/**
7032	* yield - yield the current processor to other threads.	7072	* yield - yield the current processor to other threads.
7033	*	7073	*

mm/memory.c+36−5

 }
 #endif /* __PAGETABLE_PMD_FOLDED */
-int follow_pte(struct mm_struct *mm, unsigned long address,
+int follow_invalidate_pte(struct mm_struct *mm, unsigned long address,
-	       struct mmu_notifier_range *range, pte_t **ptepp, pmd_t **pmdpp,
+			  struct mmu_notifier_range *range, pte_t **ptepp,
-	       spinlock_t **ptlp)
+			  pmd_t **pmdpp, spinlock_t **ptlp)
 {
 	pgd_t *pgd;
 	p4d_t *p4d;
 	return -EINVAL;
 }
+/**
+ * follow_pte - look up PTE at a user virtual address
+ * @mm: the mm_struct of the target address space
+ * @address: user virtual address
+ * @ptepp: location to store found PTE
+ * @ptlp: location to store the lock for the PTE
+ *
+ * On a successful return, the pointer to the PTE is stored in @ptepp;
+ * the corresponding lock is taken and its location is stored in @ptlp.
+ * The contents of the PTE are only stable until @ptlp is released;
+ * any further use, if any, must be protected against invalidation
+ * with MMU notifiers.
+ *
+ * Only IO mappings and raw PFN mappings are allowed.  The mmap semaphore
+ * should be taken for read.
+ *
+ * KVM uses this function.  While it is arguably less bad than ``follow_pfn``,
+ * it is not a good general-purpose API.
+ *
+ * Return: zero on success, -ve otherwise.
+ */
+int follow_pte(struct mm_struct *mm, unsigned long address,
+	       pte_t **ptepp, spinlock_t **ptlp)
+{
+	return follow_invalidate_pte(mm, address, NULL, ptepp, NULL, ptlp);
+}
+EXPORT_SYMBOL_GPL(follow_pte);
 /**
  * follow_pfn - look up PFN at a user virtual address
  * @vma: memory mapping
  *
  * Only IO mappings and raw PFN mappings are allowed.
  *
+ * This function does not allow the caller to read the permissions
+ * of the PTE.  Do not use it.
+ *
  * Return: zero and the pfn at @pfn on success, -ve otherwise.
  */
 int follow_pfn(struct vm_area_struct *vma, unsigned long address,
 	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
 		return ret;
-	ret = follow_pte(vma->vm_mm, address, NULL, &ptep, NULL, &ptl);
+	ret = follow_pte(vma->vm_mm, address, &ptep, &ptl);
 	if (ret)
 		return ret;
 	*pfn = pte_pfn(*ptep);
 	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
 		goto out;
-	if (follow_pte(vma->vm_mm, address, NULL, &ptep, NULL, &ptl))
+	if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
 		goto out;
 	pte = *ptep;

tools/arch/powerpc/include/uapi/asm/kvm.h+2−0

@@ -644,6 +644,8 @@ struct kvm_ppc_cpu_char { · 643 unmodified lines
644	#define KVM_REG_PPC_MMCR3 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc1)	644	#define KVM_REG_PPC_MMCR3 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc1)
645	#define KVM_REG_PPC_SIER2 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc2)	645	#define KVM_REG_PPC_SIER2 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc2)
646	#define KVM_REG_PPC_SIER3 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc3)	646	#define KVM_REG_PPC_SIER3 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc3)
		647	#define KVM_REG_PPC_DAWR1 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc4)
		648	#define KVM_REG_PPC_DAWRX1 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc5)
647		649
648	/* Transactional Memory checkpointed state:	650	/* Transactional Memory checkpointed state:
649	* This is all GPRs, all VSX regs and a subset of SPRs	651	* This is all GPRs, all VSX regs and a subset of SPRs

tools/include/uapi/linux/kvm.h+1−0

@@ -1058,6 +1058,7 @@ struct kvm_ppc_resize_hpt { · 1057 unmodified lines
1058	#define KVM_CAP_ENFORCE_PV_FEATURE_CPUID 190	1058	#define KVM_CAP_ENFORCE_PV_FEATURE_CPUID 190
1059	#define KVM_CAP_SYS_HYPERV_CPUID 191	1059	#define KVM_CAP_SYS_HYPERV_CPUID 191
1060	#define KVM_CAP_DIRTY_LOG_RING 192	1060	#define KVM_CAP_DIRTY_LOG_RING 192
		1061	#define KVM_CAP_PPC_DAWR1 194
1061		1062
1062	#ifdef KVM_CAP_IRQ_ROUTING	1063	#ifdef KVM_CAP_IRQ_ROUTING
1063		1064

tools/testing/selftests/kvm/.gitignore+6−0

 /x86_64/cr4_cpuid_sync_test
 /x86_64/debug_regs
 /x86_64/evmcs_test
+/x86_64/get_cpuid_test
 /x86_64/kvm_pv_test
 /x86_64/hyperv_cpuid
 /x86_64/mmio_warning_test
 /x86_64/vmx_preemption_timer_test
 /x86_64/vmx_set_nested_state_test
 /x86_64/vmx_tsc_adjust_test
+/x86_64/xapic_ipi_test
+/x86_64/xen_shinfo_test
+/x86_64/xen_vmcall_test
 /x86_64/xss_msr_test
+/x86_64/vmx_pmu_msrs_test
 /demand_paging_test
 /dirty_log_test
 /dirty_log_perf_test
 /kvm_create_max_vcpus
+/memslot_modification_stress_test
 /set_memory_region_test
 /steal_time

tools/testing/selftests/kvm/Makefile+6−0

 TEST_GEN_PROGS_x86_64 = x86_64/cr4_cpuid_sync_test
 TEST_GEN_PROGS_x86_64 += x86_64/evmcs_test
+TEST_GEN_PROGS_x86_64 += x86_64/get_cpuid_test
 TEST_GEN_PROGS_x86_64 += x86_64/hyperv_cpuid
 TEST_GEN_PROGS_x86_64 += x86_64/kvm_pv_test
 TEST_GEN_PROGS_x86_64 += x86_64/mmio_warning_test
 TEST_GEN_PROGS_x86_64 += x86_64/vmx_dirty_log_test
 TEST_GEN_PROGS_x86_64 += x86_64/vmx_set_nested_state_test
 TEST_GEN_PROGS_x86_64 += x86_64/vmx_tsc_adjust_test
+TEST_GEN_PROGS_x86_64 += x86_64/xapic_ipi_test
 TEST_GEN_PROGS_x86_64 += x86_64/xss_msr_test
 TEST_GEN_PROGS_x86_64 += x86_64/debug_regs
 TEST_GEN_PROGS_x86_64 += x86_64/tsc_msrs_test
+TEST_GEN_PROGS_x86_64 += x86_64/vmx_pmu_msrs_test
+TEST_GEN_PROGS_x86_64 += x86_64/xen_shinfo_test
+TEST_GEN_PROGS_x86_64 += x86_64/xen_vmcall_test
 TEST_GEN_PROGS_x86_64 += demand_paging_test
 TEST_GEN_PROGS_x86_64 += dirty_log_test
 TEST_GEN_PROGS_x86_64 += dirty_log_perf_test
 TEST_GEN_PROGS_x86_64 += kvm_create_max_vcpus
+TEST_GEN_PROGS_x86_64 += memslot_modification_stress_test
 TEST_GEN_PROGS_x86_64 += set_memory_region_test
 TEST_GEN_PROGS_x86_64 += steal_time

tools/testing/selftests/kvm/demand_paging_test.c+31−12

@@ -64,7 +64,7 @@ static void vcpu_worker(void data) · 63 unmodified lines
64	exit_reason_str(run->exit_reason));	64	exit_reason_str(run->exit_reason));
65	}	65	}
66		66
67	ts_diff = timespec_diff_now(start);	67	ts_diff = timespec_elapsed(start);
68	PER_VCPU_DEBUG("vCPU %d execution time: %ld.%.9lds\n", vcpu_id,	68	PER_VCPU_DEBUG("vCPU %d execution time: %ld.%.9lds\n", vcpu_id,
69	ts_diff.tv_sec, ts_diff.tv_nsec);	69	ts_diff.tv_sec, ts_diff.tv_nsec);
70		70
@@ -95,7 +95,7 @@ static int handle_uffd_page_request(int uffd, uint64_t addr) · 24 unmodified lines
95	return r;	95	return r;
96	}	96	}
97		97
98	ts_diff = timespec_diff_now(start);	98	ts_diff = timespec_elapsed(start);
99		99
100	PER_PAGE_DEBUG("UFFDIO_COPY %d \t%ld ns\n", tid,	100	PER_PAGE_DEBUG("UFFDIO_COPY %d \t%ld ns\n", tid,
101	timespec_to_ns(ts_diff));	101	timespec_to_ns(ts_diff));
@@ -190,7 +190,7 @@ static void uffd_handler_thread_fn(void arg) · 88 unmodified lines
190	pages++;	190	pages++;
191	}	191	}
192		192
193	ts_diff = timespec_diff_now(start);	193	ts_diff = timespec_elapsed(start);
194	PER_VCPU_DEBUG("userfaulted %ld pages over %ld.%.9lds. (%f/sec)\n",	194	PER_VCPU_DEBUG("userfaulted %ld pages over %ld.%.9lds. (%f/sec)\n",
195	pages, ts_diff.tv_sec, ts_diff.tv_nsec,	195	pages, ts_diff.tv_sec, ts_diff.tv_nsec,
196	pages / ((double)ts_diff.tv_sec + (double)ts_diff.tv_nsec / 100000000.0));	196	pages / ((double)ts_diff.tv_sec + (double)ts_diff.tv_nsec / 100000000.0));
@@ -250,6 +250,7 @@ static int setup_demand_paging(struct kvm_vm *vm, · 53 unmodified lines
250	struct test_params {	250	struct test_params {
251	bool use_uffd;	251	bool use_uffd;
252	useconds_t uffd_delay;	252	useconds_t uffd_delay;
		253	bool partition_vcpu_memory_access;
253	};	254	};
254		255
255	static void run_test(enum vm_guest_mode mode, void *arg)	256	static void run_test(enum vm_guest_mode mode, void *arg)
@@ -265,7 +266,8 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 9 unmodified lines
265	int vcpu_id;	266	int vcpu_id;
266	int r;	267	int r;
267		268
268	vm = perf_test_create_vm(mode, nr_vcpus, guest_percpu_mem_size);	269	vm = perf_test_create_vm(mode, nr_vcpus, guest_percpu_mem_size,
		270	VM_MEM_SRC_ANONYMOUS);
269		271
270	perf_test_args.wr_fract = 1;	272	perf_test_args.wr_fract = 1;
271		273
@@ -277,7 +279,8 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 5 unmodified lines
277	vcpu_threads = malloc(nr_vcpus * sizeof(*vcpu_threads));	279	vcpu_threads = malloc(nr_vcpus * sizeof(*vcpu_threads));
278	TEST_ASSERT(vcpu_threads, "Memory allocation failed");	280	TEST_ASSERT(vcpu_threads, "Memory allocation failed");
279		281
280	perf_test_setup_vcpus(vm, nr_vcpus, guest_percpu_mem_size);	282	perf_test_setup_vcpus(vm, nr_vcpus, guest_percpu_mem_size,
		283	p->partition_vcpu_memory_access);
281		284
282	if (p->use_uffd) {	285	if (p->use_uffd) {
283	uffd_handler_threads =	286	uffd_handler_threads =
@@ -293,10 +296,19 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 9 unmodified lines
293	for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++) {	296	for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++) {
294	vm_paddr_t vcpu_gpa;	297	vm_paddr_t vcpu_gpa;
295	void *vcpu_hva;	298	void *vcpu_hva;
		299	uint64_t vcpu_mem_size;
296		300
297	vcpu_gpa = guest_test_phys_mem + (vcpu_id * guest_percpu_mem_size);	301
		302	if (p->partition_vcpu_memory_access) {
		303	vcpu_gpa = guest_test_phys_mem +
		304	(vcpu_id * guest_percpu_mem_size);
		305	vcpu_mem_size = guest_percpu_mem_size;
		306	} else {
		307	vcpu_gpa = guest_test_phys_mem;
		308	vcpu_mem_size = guest_percpu_mem_size * nr_vcpus;
		309	}
298	PER_VCPU_DEBUG("Added VCPU %d with test mem gpa [%lx, %lx)\n",	310	PER_VCPU_DEBUG("Added VCPU %d with test mem gpa [%lx, %lx)\n",
299	vcpu_id, vcpu_gpa, vcpu_gpa + guest_percpu_mem_size);	311	vcpu_id, vcpu_gpa, vcpu_gpa + vcpu_mem_size);
300		312
301	/* Cache the HVA pointer of the region */	313	/* Cache the HVA pointer of the region */
302	vcpu_hva = addr_gpa2hva(vm, vcpu_gpa);	314	vcpu_hva = addr_gpa2hva(vm, vcpu_gpa);
@@ -313,7 +325,7 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 10 unmodified lines
313	&uffd_handler_threads[vcpu_id],	325	&uffd_handler_threads[vcpu_id],
314	pipefds[vcpu_id * 2],	326	pipefds[vcpu_id * 2],
315	p->uffd_delay, &uffd_args[vcpu_id],	327	p->uffd_delay, &uffd_args[vcpu_id],
316	vcpu_hva, guest_percpu_mem_size);	328	vcpu_hva, vcpu_mem_size);
317	if (r < 0)	329	if (r < 0)
318	exit(-r);	330	exit(-r);
319	}	331	}
@@ -339,7 +351,7 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 19 unmodified lines
339	PER_VCPU_DEBUG("Joined thread for vCPU %d\n", vcpu_id);	351	PER_VCPU_DEBUG("Joined thread for vCPU %d\n", vcpu_id);
340	}	352	}
341		353
342	ts_diff = timespec_diff_now(start);	354	ts_diff = timespec_elapsed(start);
343		355
344	pr_info("All vCPU threads joined\n");	356	pr_info("All vCPU threads joined\n");
345		357
@@ -376,7 +388,7 @@ static void help(char *name) · 30 unmodified lines
376	{	388	{
377	puts("");	389	puts("");
378	printf("usage: %s [-h] [-m mode] [-u] [-d uffd_delay_usec]\n"	390	printf("usage: %s [-h] [-m mode] [-u] [-d uffd_delay_usec]\n"
379	" [-b memory] [-v vcpus]\n", name);	391	" [-b memory] [-v vcpus] [-o]\n", name);
380	guest_modes_help();	392	guest_modes_help();
381	printf(" -u: use User Fault FD to handle vCPU page\n"	393	printf(" -u: use User Fault FD to handle vCPU page\n"
382	" faults.\n");	394	" faults.\n");
@@ -387,6 +399,8 @@ static void help(char *name) · 4 unmodified lines
387	" demand paged by each vCPU. e.g. 10M or 3G.\n"	399	" demand paged by each vCPU. e.g. 10M or 3G.\n"
388	" Default: 1G\n");	400	" Default: 1G\n");
389	printf(" -v: specify the number of vCPUs to run.\n");	401	printf(" -v: specify the number of vCPUs to run.\n");
		402	printf(" -o: Overlap guest memory accesses instead of partitioning\n"
		403	" them into a separate region of memory for each vCPU.\n");
390	puts("");	404	puts("");
391	exit(0);	405	exit(0);
392	}	406	}
@@ -394,12 +408,14 @@ static void help(char *name) · 1 unmodified line
394	int main(int argc, char *argv[])	408	int main(int argc, char *argv[])
395	{	409	{
396	int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS);	410	int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS);
397	struct test_params p = {};	411	struct test_params p = {
		412	.partition_vcpu_memory_access = true,
		413	};
398	int opt;	414	int opt;
399		415
400	guest_modes_append_default();	416	guest_modes_append_default();
401		417
402	while ((opt = getopt(argc, argv, "hm:ud:b:v:")) != -1) {	418	while ((opt = getopt(argc, argv, "hm:ud:b:v:o")) != -1) {
403	switch (opt) {	419	switch (opt) {
404	case 'm':	420	case 'm':
405	guest_modes_cmdline(optarg);	421	guest_modes_cmdline(optarg);
@@ -419,6 +435,9 @@ int main(int argc, char *argv[]) · 13 unmodified lines
419	TEST_ASSERT(nr_vcpus > 0 && nr_vcpus <= max_vcpus,	435	TEST_ASSERT(nr_vcpus > 0 && nr_vcpus <= max_vcpus,
420	"Invalid number of vcpus, must be between 1 and %d", max_vcpus);	436	"Invalid number of vcpus, must be between 1 and %d", max_vcpus);
421	break;	437	break;
		438	case 'o':
		439	p.partition_vcpu_memory_access = false;
		440	break;
422	case 'h':	441	case 'h':
423	default:	442	default:
424	help(argv[0]);	443	help(argv[0]);

tools/testing/selftests/kvm/dirty_log_perf_test.c+56−36

 /* Host variables */
 static u64 dirty_log_manual_caps;
 static bool host_quit;
-static uint64_t iteration;
+static int iteration;
-static uint64_t vcpu_last_completed_iteration[KVM_MAX_VCPUS];
+static int vcpu_last_completed_iteration[KVM_MAX_VCPUS];
 static void *vcpu_worker(void *data)
 {
 	run = vcpu_state(vm, vcpu_id);
 	while (!READ_ONCE(host_quit)) {
-		uint64_t current_iteration = READ_ONCE(iteration);
+		int current_iteration = READ_ONCE(iteration);
 		clock_gettime(CLOCK_MONOTONIC, &start);
 		ret = _vcpu_run(vm, vcpu_id);
-		ts_diff = timespec_diff_now(start);
+		ts_diff = timespec_elapsed(start);
 		TEST_ASSERT(ret == 0, "vcpu_run failed: %d\n", ret);
 		TEST_ASSERT(get_ucall(vm, vcpu_id, NULL) == UCALL_SYNC,
 		pr_debug("Got sync event from vCPU %d\n", vcpu_id);
 		vcpu_last_completed_iteration[vcpu_id] = current_iteration;
-		pr_debug("vCPU %d updated last completed iteration to %lu\n",
+		pr_debug("vCPU %d updated last completed iteration to %d\n",
 			 vcpu_id, vcpu_last_completed_iteration[vcpu_id]);
 		if (current_iteration) {
 			pages_count += vcpu_args->pages;
 			total = timespec_add(total, ts_diff);
-			pr_debug("vCPU %d iteration %lu dirty memory time: %ld.%.9lds\n",
+			pr_debug("vCPU %d iteration %d dirty memory time: %ld.%.9lds\n",
 				vcpu_id, current_iteration, ts_diff.tv_sec,
 				ts_diff.tv_nsec);
 		} else {
-			pr_debug("vCPU %d iteration %lu populate memory time: %ld.%.9lds\n",
+			pr_debug("vCPU %d iteration %d populate memory time: %ld.%.9lds\n",
 				vcpu_id, current_iteration, ts_diff.tv_sec,
 				ts_diff.tv_nsec);
 		}
 	}
 	avg = timespec_div(total, vcpu_last_completed_iteration[vcpu_id]);
-	pr_debug("\nvCPU %d dirtied 0x%lx pages over %lu iterations in %ld.%.9lds. (Avg %ld.%.9lds/iteration)\n",
+	pr_debug("\nvCPU %d dirtied 0x%lx pages over %d iterations in %ld.%.9lds. (Avg %ld.%.9lds/iteration)\n",
 		vcpu_id, pages_count, vcpu_last_completed_iteration[vcpu_id],
 		total.tv_sec, total.tv_nsec, avg.tv_sec, avg.tv_nsec);
 	unsigned long iterations;
 	uint64_t phys_offset;
 	int wr_fract;
+	bool partition_vcpu_memory_access;
+	enum vm_mem_backing_src_type backing_src;
 };
 static void run_test(enum vm_guest_mode mode, void *arg)
 	struct kvm_enable_cap cap = {};
 	struct timespec clear_dirty_log_total = (struct timespec){0};
-	vm = perf_test_create_vm(mode, nr_vcpus, guest_percpu_mem_size);
+	vm = perf_test_create_vm(mode, nr_vcpus, guest_percpu_mem_size,
+				 p->backing_src);
 	perf_test_args.wr_fract = p->wr_fract;
 	vcpu_threads = malloc(nr_vcpus * sizeof(*vcpu_threads));
 	TEST_ASSERT(vcpu_threads, "Memory allocation failed");
-	perf_test_setup_vcpus(vm, nr_vcpus, guest_percpu_mem_size);
+	perf_test_setup_vcpus(vm, nr_vcpus, guest_percpu_mem_size,
+			      p->partition_vcpu_memory_access);
 	sync_global_to_guest(vm, perf_test_args);
 	clock_gettime(CLOCK_MONOTONIC, &start);
 	for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++) {
+		vcpu_last_completed_iteration[vcpu_id] = -1;
 		pthread_create(&vcpu_threads[vcpu_id], NULL, vcpu_worker,
 			       &perf_test_args.vcpu_args[vcpu_id]);
 	}
-	/* Allow the vCPU to populate memory */
+	/* Allow the vCPUs to populate memory */
-	pr_debug("Starting iteration %lu - Populating\n", iteration);
+	pr_debug("Starting iteration %d - Populating\n", iteration);
-	while (READ_ONCE(vcpu_last_completed_iteration[vcpu_id]) != iteration)
+	for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++) {
-		pr_debug("Waiting for vcpu_last_completed_iteration == %lu\n",
+		while (READ_ONCE(vcpu_last_completed_iteration[vcpu_id]) !=
-			iteration);
+		       iteration)
+			;
+	}
-	ts_diff = timespec_diff_now(start);
+	ts_diff = timespec_elapsed(start);
 	pr_info("Populate memory time: %ld.%.9lds\n",
 		ts_diff.tv_sec, ts_diff.tv_nsec);
 	clock_gettime(CLOCK_MONOTONIC, &start);
 	vm_mem_region_set_flags(vm, PERF_TEST_MEM_SLOT_INDEX,
 				KVM_MEM_LOG_DIRTY_PAGES);
-	ts_diff = timespec_diff_now(start);
+	ts_diff = timespec_elapsed(start);
 	pr_info("Enabling dirty logging time: %ld.%.9lds\n\n",
 		ts_diff.tv_sec, ts_diff.tv_nsec);
 		clock_gettime(CLOCK_MONOTONIC, &start);
 		iteration++;
-		pr_debug("Starting iteration %lu\n", iteration);
+		pr_debug("Starting iteration %d\n", iteration);
 		for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++) {
-			while (READ_ONCE(vcpu_last_completed_iteration[vcpu_id]) != iteration)
+			while (READ_ONCE(vcpu_last_completed_iteration[vcpu_id])
-				pr_debug("Waiting for vCPU %d vcpu_last_completed_iteration == %lu\n",
+			       != iteration)
-					 vcpu_id, iteration);
+				;
 		}
-		ts_diff = timespec_diff_now(start);
+		ts_diff = timespec_elapsed(start);
 		vcpu_dirty_total = timespec_add(vcpu_dirty_total, ts_diff);
-		pr_info("Iteration %lu dirty memory time: %ld.%.9lds\n",
+		pr_info("Iteration %d dirty memory time: %ld.%.9lds\n",
 			iteration, ts_diff.tv_sec, ts_diff.tv_nsec);
 		clock_gettime(CLOCK_MONOTONIC, &start);
 		kvm_vm_get_dirty_log(vm, PERF_TEST_MEM_SLOT_INDEX, bmap);
-		ts_diff = timespec_diff_now(start);
+		ts_diff = timespec_elapsed(start);
 		get_dirty_log_total = timespec_add(get_dirty_log_total,
 						   ts_diff);
-		pr_info("Iteration %lu get dirty log time: %ld.%.9lds\n",
+		pr_info("Iteration %d get dirty log time: %ld.%.9lds\n",
 			iteration, ts_diff.tv_sec, ts_diff.tv_nsec);
 		if (dirty_log_manual_caps) {
 			kvm_vm_clear_dirty_log(vm, PERF_TEST_MEM_SLOT_INDEX, bmap, 0,
 					       host_num_pages);
-			ts_diff = timespec_diff_now(start);
+			ts_diff = timespec_elapsed(start);
 			clear_dirty_log_total = timespec_add(clear_dirty_log_total,
 							     ts_diff);
-			pr_info("Iteration %lu clear dirty log time: %ld.%.9lds\n",
+			pr_info("Iteration %d clear dirty log time: %ld.%.9lds\n",
 				iteration, ts_diff.tv_sec, ts_diff.tv_nsec);
 		}
 	}
-	/* Tell the vcpu thread to quit */
-	host_quit = true;
-	for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++)
-		pthread_join(vcpu_threads[vcpu_id], NULL);
 	/* Disable dirty logging */
 	clock_gettime(CLOCK_MONOTONIC, &start);
 	vm_mem_region_set_flags(vm, PERF_TEST_MEM_SLOT_INDEX, 0);
-	ts_diff = timespec_diff_now(start);
+	ts_diff = timespec_elapsed(start);
 	pr_info("Disabling dirty logging time: %ld.%.9lds\n",
 		ts_diff.tv_sec, ts_diff.tv_nsec);
+	/* Tell the vcpu thread to quit */
+	host_quit = true;
+	for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++)
+		pthread_join(vcpu_threads[vcpu_id], NULL);
 	avg = timespec_div(get_dirty_log_total, p->iterations);
 	pr_info("Get dirty log over %lu iterations took %ld.%.9lds. (Avg %ld.%.9lds/iteration)\n",
 		p->iterations, get_dirty_log_total.tv_sec,
 {
 	puts("");
 	printf("usage: %s [-h] [-i iterations] [-p offset] "
-	       "[-m mode] [-b vcpu bytes] [-v vcpus]\n", name);
+	       "[-m mode] [-b vcpu bytes] [-v vcpus] [-o] [-s mem type]\n", name);
 	puts("");
 	printf(" -i: specify iteration counts (default: %"PRIu64")\n",
 	       TEST_HOST_LOOP_N);
 	       "     1/<fraction of pages to write>.\n"
 	       "     (default: 1 i.e. all pages are written to.)\n");
 	printf(" -v: specify the number of vCPUs to run.\n");
+	printf(" -o: Overlap guest memory accesses instead of partitioning\n"
+	       "     them into a separate region of memory for each vCPU.\n");
+	printf(" -s: specify the type of memory that should be used to\n"
+	       "     back the guest data region.\n\n");
+	backing_src_help();
 	puts("");
 	exit(0);
 }
 	struct test_params p = {
 		.iterations = TEST_HOST_LOOP_N,
 		.wr_fract = 1,
+		.partition_vcpu_memory_access = true,
+		.backing_src = VM_MEM_SRC_ANONYMOUS,
 	};
 	int opt;
 	guest_modes_append_default();
-	while ((opt = getopt(argc, argv, "hi:p:m:b:f:v:")) != -1) {
+	while ((opt = getopt(argc, argv, "hi:p:m:b:f:v:os:")) != -1) {
 		switch (opt) {
 		case 'i':
-			p.iterations = strtol(optarg, NULL, 10);
+			p.iterations = atoi(optarg);
 			break;
 		case 'p':
 			p.phys_offset = strtoull(optarg, NULL, 0);
 			TEST_ASSERT(nr_vcpus > 0 && nr_vcpus <= max_vcpus,
 				    "Invalid number of vcpus, must be between 1 and %d", max_vcpus);
 			break;
+		case 'o':
+			p.partition_vcpu_memory_access = false;
+		case 's':
+			p.backing_src = parse_backing_src_type(optarg);
+			break;
 		case 'h':
 		default:
 			help(argv[0]);

tools/testing/selftests/kvm/include/kvm_util.h+0−6

 };
 extern const struct vm_guest_mode_params vm_guest_mode_params[];
-enum vm_mem_backing_src_type {
-	VM_MEM_SRC_ANONYMOUS,
-	VM_MEM_SRC_ANONYMOUS_THP,
-	VM_MEM_SRC_ANONYMOUS_HUGETLB,
-};
 int kvm_check_cap(long cap);
 int vm_enable_cap(struct kvm_vm *vm, struct kvm_enable_cap *cap);
 int vcpu_enable_cap(struct kvm_vm *vm, uint32_t vcpu_id,

tools/testing/selftests/kvm/include/numaif.h+55−0

@@ -0,0 +1,55 @@
	1	/* SPDX-License-Identifier: GPL-2.0-only */
	2	/*
	3	* tools/testing/selftests/kvm/include/numaif.h
	4	*
	5	* Copyright (C) 2020, Google LLC.
	6	*
	7	* This work is licensed under the terms of the GNU GPL, version 2.
	8	*
	9	* Header file that provides access to NUMA API functions not explicitly
	10	* exported to user space.
	11	*/
	12
	13	#ifndef SELFTEST_KVM_NUMAIF_H
	14	#define SELFTEST_KVM_NUMAIF_H
	15
	16	#define __NR_get_mempolicy 239
	17	#define __NR_migrate_pages 256
	18
	19	/* System calls */
	20	long get_mempolicy(int policy, const unsigned long nmask,
	21	unsigned long maxnode, void *addr, int flags)
	22	{
	23	return syscall(__NR_get_mempolicy, policy, nmask,
	24	maxnode, addr, flags);
	25	}
	26
	27	long migrate_pages(int pid, unsigned long maxnode,
	28	const unsigned long *frommask,
	29	const unsigned long *tomask)
	30	{
	31	return syscall(__NR_migrate_pages, pid, maxnode, frommask, tomask);
	32	}
	33
	34	/* Policies */
	35	#define MPOL_DEFAULT 0
	36	#define MPOL_PREFERRED 1
	37	#define MPOL_BIND 2
	38	#define MPOL_INTERLEAVE 3
	39
	40	#define MPOL_MAX MPOL_INTERLEAVE
	41
	42	/* Flags for get_mem_policy */
	43	#define MPOL_F_NODE (1<<0) /* return next il node or node of address */
	44	/* Warning: MPOL_F_NODE is unsupported and
	45	* subject to change. Don't use.
	46	*/
	47	#define MPOL_F_ADDR (1<<1) /* look up vma using address */
	48	#define MPOL_F_MEMS_ALLOWED (1<<2) /* query nodes allowed in cpuset */
	49
	50	/* Flags for mbind */
	51	#define MPOL_MF_STRICT (1<<0) /* Verify existing pages in the mapping */
	52	#define MPOL_MF_MOVE (1<<1) /* Move pages owned by this process to conform to mapping */
	53	#define MPOL_MF_MOVE_ALL (1<<2) /* Move every page to conform to mapping */
	54
	55	#endif /* SELFTEST_KVM_NUMAIF_H */

tools/testing/selftests/kvm/include/perf_test_util.h+5−2

 extern uint64_t guest_test_phys_mem;
 struct kvm_vm *perf_test_create_vm(enum vm_guest_mode mode, int vcpus,
-				uint64_t vcpu_memory_bytes);
+				   uint64_t vcpu_memory_bytes,
+				   enum vm_mem_backing_src_type backing_src);
 void perf_test_destroy_vm(struct kvm_vm *vm);
-void perf_test_setup_vcpus(struct kvm_vm *vm, int vcpus, uint64_t vcpu_memory_bytes);
+void perf_test_setup_vcpus(struct kvm_vm *vm, int vcpus,
+			   uint64_t vcpu_memory_bytes,
+			   bool partition_vcpu_memory_access);
 #endif /* SELFTEST_KVM_PERF_TEST_UTIL_H */

tools/testing/selftests/kvm/include/test_util.h+15−1

@@ -64,7 +64,21 @@ int64_t timespec_to_ns(struct timespec ts); · 63 unmodified lines
64	struct timespec timespec_add_ns(struct timespec ts, int64_t ns);	64	struct timespec timespec_add_ns(struct timespec ts, int64_t ns);
65	struct timespec timespec_add(struct timespec ts1, struct timespec ts2);	65	struct timespec timespec_add(struct timespec ts1, struct timespec ts2);
66	struct timespec timespec_sub(struct timespec ts1, struct timespec ts2);	66	struct timespec timespec_sub(struct timespec ts1, struct timespec ts2);
67	struct timespec timespec_diff_now(struct timespec start);	67	struct timespec timespec_elapsed(struct timespec start);
68	struct timespec timespec_div(struct timespec ts, int divisor);	68	struct timespec timespec_div(struct timespec ts, int divisor);
69		69
		70	enum vm_mem_backing_src_type {
		71	VM_MEM_SRC_ANONYMOUS,
		72	VM_MEM_SRC_ANONYMOUS_THP,
		73	VM_MEM_SRC_ANONYMOUS_HUGETLB,
		74	};
		75
		76	struct vm_mem_backing_src_alias {
		77	const char *name;
		78	enum vm_mem_backing_src_type type;
		79	};
		80
		81	void backing_src_help(void);
		82	enum vm_mem_backing_src_type parse_backing_src_type(const char *type_name);
		83
70	#endif /* SELFTEST_KVM_TEST_UTIL_H */	84	#endif /* SELFTEST_KVM_TEST_UTIL_H */

tools/testing/selftests/kvm/include/x86_64/processor.h+40−1

@@ -263,6 +263,19 @@ static inline void outl(uint16_t port, uint32_t value) · 262 unmodified lines
263	__asm__ __volatile__("outl %%eax, %%dx" : : "d"(port), "a"(value));	263	__asm__ __volatile__("outl %%eax, %%dx" : : "d"(port), "a"(value));
264	}	264	}
265		265
		266	static inline void cpuid(uint32_t eax, uint32_t ebx,
		267	uint32_t ecx, uint32_t edx)
		268	{
		269	/* ecx is often an input as well as an output. */
		270	asm volatile("cpuid"
		271	: "=a" (*eax),
		272	"=b" (*ebx),
		273	"=c" (*ecx),
		274	"=d" (*edx)
		275	: "0" (eax), "2" (ecx)
		276	: "memory");
		277	}
		278
266	#define SET_XMM(__var, __xmm) \	279	#define SET_XMM(__var, __xmm) \
267	asm volatile("movq %0, %%"#__xmm : : "r"(__var) : #__xmm)	280	asm volatile("movq %0, %%"#__xmm : : "r"(__var) : #__xmm)
268		281
@@ -338,8 +351,10 @@ void vcpu_load_state(struct kvm_vm *vm, uint32_t vcpuid, · 69 unmodified lines
338	struct kvm_x86_state *state);	351	struct kvm_x86_state *state);
339		352
340	struct kvm_msr_list *kvm_get_msr_index_list(void);	353	struct kvm_msr_list *kvm_get_msr_index_list(void);
341		354	uint64_t kvm_get_feature_msr(uint64_t msr_index);
342	struct kvm_cpuid2 *kvm_get_supported_cpuid(void);	355	struct kvm_cpuid2 *kvm_get_supported_cpuid(void);
		356
		357	struct kvm_cpuid2 vcpu_get_cpuid(struct kvm_vm vm, uint32_t vcpuid);
343	void vcpu_set_cpuid(struct kvm_vm *vm, uint32_t vcpuid,	358	void vcpu_set_cpuid(struct kvm_vm *vm, uint32_t vcpuid,
344	struct kvm_cpuid2 *cpuid);	359	struct kvm_cpuid2 *cpuid);
345		360
@@ -391,6 +406,10 @@ bool set_cpuid(struct kvm_cpuid2 cpuid, struct kvm_cpuid_entry2 ent); · 45 unmodified lines
391	uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2,	406	uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2,
392	uint64_t a3);	407	uint64_t a3);
393		408
		409	struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(void);
		410	void vcpu_set_hv_cpuid(struct kvm_vm *vm, uint32_t vcpuid);
		411	struct kvm_cpuid2 vcpu_get_supported_hv_cpuid(struct kvm_vm vm, uint32_t vcpuid);
		412
394	/*	413	/*
395	* Basic CPU control in CR0	414	* Basic CPU control in CR0
396	*/	415	*/
@@ -406,8 +425,27 @@ uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2, · 9 unmodified lines
406	#define X86_CR0_CD (1UL<<30) /* Cache Disable */	425	#define X86_CR0_CD (1UL<<30) /* Cache Disable */
407	#define X86_CR0_PG (1UL<<31) /* Paging */	426	#define X86_CR0_PG (1UL<<31) /* Paging */
408		427
		428	#define APIC_DEFAULT_GPA 0xfee00000ULL
		429
		430	/* APIC base address MSR and fields */
		431	#define MSR_IA32_APICBASE 0x0000001b
		432	#define MSR_IA32_APICBASE_BSP (1<<8)
		433	#define MSR_IA32_APICBASE_EXTD (1<<10)
		434	#define MSR_IA32_APICBASE_ENABLE (1<<11)
		435	#define MSR_IA32_APICBASE_BASE (0xfffff<<12)
		436	#define GET_APIC_BASE(x) (((x) >> 12) << 12)
		437
409	#define APIC_BASE_MSR 0x800	438	#define APIC_BASE_MSR 0x800
410	#define X2APIC_ENABLE (1UL << 10)	439	#define X2APIC_ENABLE (1UL << 10)
		440	#define APIC_ID 0x20
		441	#define APIC_LVR 0x30
		442	#define GET_APIC_ID_FIELD(x) (((x) >> 24) & 0xFF)
		443	#define APIC_TASKPRI 0x80
		444	#define APIC_PROCPRI 0xA0
		445	#define APIC_EOI 0xB0
		446	#define APIC_SPIV 0xF0
		447	#define APIC_SPIV_FOCUS_DISABLED (1 << 9)
		448	#define APIC_SPIV_APIC_ENABLED (1 << 8)
411	#define APIC_ICR 0x300	449	#define APIC_ICR 0x300
412	#define APIC_DEST_SELF 0x40000	450	#define APIC_DEST_SELF 0x40000
413	#define APIC_DEST_ALLINC 0x80000	451	#define APIC_DEST_ALLINC 0x80000
@@ -432,6 +470,7 @@ uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2, · 18 unmodified lines
432	#define APIC_DM_EXTINT 0x00700	470	#define APIC_DM_EXTINT 0x00700
433	#define APIC_VECTOR_MASK 0x000FF	471	#define APIC_VECTOR_MASK 0x000FF
434	#define APIC_ICR2 0x310	472	#define APIC_ICR2 0x310
		473	#define SET_APIC_DEST_FIELD(x) ((x) << 24)
435		474
436	/* VMX_EPT_VPID_CAP bits */	475	/* VMX_EPT_VPID_CAP bits */
437	#define VMX_EPT_VPID_CAP_AD_BITS (1ULL << 21)	476	#define VMX_EPT_VPID_CAP_AD_BITS (1ULL << 21)

tools/testing/selftests/kvm/lib/kvm_util.c+1−0

@@ -1801,6 +1801,7 @@ static struct exit_reason { · 1800 unmodified lines
1801	{KVM_EXIT_DIRTY_RING_FULL, "DIRTY_RING_FULL"},	1801	{KVM_EXIT_DIRTY_RING_FULL, "DIRTY_RING_FULL"},
1802	{KVM_EXIT_X86_RDMSR, "RDMSR"},	1802	{KVM_EXIT_X86_RDMSR, "RDMSR"},
1803	{KVM_EXIT_X86_WRMSR, "WRMSR"},	1803	{KVM_EXIT_X86_WRMSR, "WRMSR"},
		1804	{KVM_EXIT_XEN, "XEN"},
1804	#ifdef KVM_EXIT_MEMORY_NOT_PRESENT	1805	#ifdef KVM_EXIT_MEMORY_NOT_PRESENT
1805	{KVM_EXIT_MEMORY_NOT_PRESENT, "MEMORY_NOT_PRESENT"},	1806	{KVM_EXIT_MEMORY_NOT_PRESENT, "MEMORY_NOT_PRESENT"},
1806	#endif	1807	#endif

tools/testing/selftests/kvm/lib/perf_test_util.c+21−10

 }
 struct kvm_vm *perf_test_create_vm(enum vm_guest_mode mode, int vcpus,
-				   uint64_t vcpu_memory_bytes)
+				   uint64_t vcpu_memory_bytes,
+				   enum vm_mem_backing_src_type backing_src)
 {
 	struct kvm_vm *vm;
 	uint64_t guest_num_pages;
 	pr_info("guest physical test memory offset: 0x%lx\n", guest_test_phys_mem);
 	/* Add an extra memory slot for testing */
-	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
+	vm_userspace_mem_region_add(vm, backing_src, guest_test_phys_mem,
-				    guest_test_phys_mem,
 				    PERF_TEST_MEM_SLOT_INDEX,
 				    guest_num_pages, 0);
 	kvm_vm_free(vm);
 }
-void perf_test_setup_vcpus(struct kvm_vm *vm, int vcpus, uint64_t vcpu_memory_bytes)
+void perf_test_setup_vcpus(struct kvm_vm *vm, int vcpus,
+			   uint64_t vcpu_memory_bytes,
+			   bool partition_vcpu_memory_access)
 {
 	vm_paddr_t vcpu_gpa;
 	struct perf_test_vcpu_args *vcpu_args;
 		vcpu_args = &perf_test_args.vcpu_args[vcpu_id];
 		vcpu_args->vcpu_id = vcpu_id;
-		vcpu_args->gva = guest_test_virt_mem +
+		if (partition_vcpu_memory_access) {
-				 (vcpu_id * vcpu_memory_bytes);
+			vcpu_args->gva = guest_test_virt_mem +
-		vcpu_args->pages = vcpu_memory_bytes /
+					 (vcpu_id * vcpu_memory_bytes);
-				   perf_test_args.guest_page_size;
+			vcpu_args->pages = vcpu_memory_bytes /
+					   perf_test_args.guest_page_size;
+			vcpu_gpa = guest_test_phys_mem +
+				   (vcpu_id * vcpu_memory_bytes);
+		} else {
+			vcpu_args->gva = guest_test_virt_mem;
+			vcpu_args->pages = (vcpus * vcpu_memory_bytes) /
+					   perf_test_args.guest_page_size;
+			vcpu_gpa = guest_test_phys_mem;
+		}
-		vcpu_gpa = guest_test_phys_mem + (vcpu_id * vcpu_memory_bytes);
 		pr_debug("Added VCPU %d with test mem gpa [%lx, %lx)\n",
-			 vcpu_id, vcpu_gpa, vcpu_gpa + vcpu_memory_bytes);
+			 vcpu_id, vcpu_gpa, vcpu_gpa +
+			 (vcpu_args->pages * perf_test_args.guest_page_size));
 	}
 }

tools/testing/selftests/kvm/lib/test_util.c+30−1

@@ -10,6 +10,7 @@ · 9 unmodified lines
10	#include <limits.h>	10	#include <limits.h>
11	#include <stdlib.h>	11	#include <stdlib.h>
12	#include <time.h>	12	#include <time.h>
		13	#include "linux/kernel.h"
13		14
14	#include "test_util.h"	15	#include "test_util.h"
15		16
@@ -84,7 +85,7 @@ struct timespec timespec_sub(struct timespec ts1, struct timespec ts2) · 68 unmodified lines
84	return timespec_add_ns((struct timespec){0}, ns1 - ns2);	85	return timespec_add_ns((struct timespec){0}, ns1 - ns2);
85	}	86	}
86		87
87	struct timespec timespec_diff_now(struct timespec start)	88	struct timespec timespec_elapsed(struct timespec start)
88	{	89	{
89	struct timespec end;	90	struct timespec end;
90		91
@@ -109,3 +110,31 @@ void print_skip(const char *fmt, ...) · 18 unmodified lines
109	va_end(ap);	110	va_end(ap);
110	puts(", skipping test");	111	puts(", skipping test");
111	}	112	}
		113
		114	const struct vm_mem_backing_src_alias backing_src_aliases[] = {
		115	{"anonymous", VM_MEM_SRC_ANONYMOUS,},
		116	{"anonymous_thp", VM_MEM_SRC_ANONYMOUS_THP,},
		117	{"anonymous_hugetlb", VM_MEM_SRC_ANONYMOUS_HUGETLB,},
		118	};
		119
		120	void backing_src_help(void)
		121	{
		122	int i;
		123
		124	printf("Available backing src types:\n");
		125	for (i = 0; i < ARRAY_SIZE(backing_src_aliases); i++)
		126	printf("\t%s\n", backing_src_aliases[i].name);
		127	}
		128
		129	enum vm_mem_backing_src_type parse_backing_src_type(const char *type_name)
		130	{
		131	int i;
		132
		133	for (i = 0; i < ARRAY_SIZE(backing_src_aliases); i++)
		134	if (!strcmp(type_name, backing_src_aliases[i].name))
		135	return backing_src_aliases[i].type;
		136
		137	backing_src_help();
		138	TEST_FAIL("Unknown backing src type: %s", type_name);
		139	return -1;
		140	}

tools/testing/selftests/kvm/lib/x86_64/processor.c+144−0

@@ -669,6 +669,82 @@ struct kvm_cpuid2 *kvm_get_supported_cpuid(void) · 668 unmodified lines
669	return cpuid;	669	return cpuid;
670	}	670	}
671		671
		672	/*
		673	* KVM Get MSR
		674	*
		675	* Input Args:
		676	* msr_index - Index of MSR
		677	*
		678	* Output Args: None
		679	*
		680	* Return: On success, value of the MSR. On failure a TEST_ASSERT is produced.
		681	*
		682	* Get value of MSR for VCPU.
		683	*/
		684	uint64_t kvm_get_feature_msr(uint64_t msr_index)
		685	{
		686	struct {
		687	struct kvm_msrs header;
		688	struct kvm_msr_entry entry;
		689	} buffer = {};
		690	int r, kvm_fd;
		691
		692	buffer.header.nmsrs = 1;
		693	buffer.entry.index = msr_index;
		694	kvm_fd = open(KVM_DEV_PATH, O_RDONLY);
		695	if (kvm_fd < 0)
		696	exit(KSFT_SKIP);
		697
		698	r = ioctl(kvm_fd, KVM_GET_MSRS, &buffer.header);
		699	TEST_ASSERT(r == 1, "KVM_GET_MSRS IOCTL failed,\n"
		700	" rc: %i errno: %i", r, errno);
		701
		702	close(kvm_fd);
		703	return buffer.entry.data;
		704	}
		705
		706	/*
		707	* VM VCPU CPUID Set
		708	*
		709	* Input Args:
		710	* vm - Virtual Machine
		711	* vcpuid - VCPU id
		712	*
		713	* Output Args: None
		714	*
		715	* Return: KVM CPUID (KVM_GET_CPUID2)
		716	*
		717	* Set the VCPU's CPUID.
		718	*/
		719	struct kvm_cpuid2 vcpu_get_cpuid(struct kvm_vm vm, uint32_t vcpuid)
		720	{
		721	struct vcpu *vcpu = vcpu_find(vm, vcpuid);
		722	struct kvm_cpuid2 *cpuid;
		723	int rc, max_ent;
		724
		725	TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
		726
		727	cpuid = allocate_kvm_cpuid2();
		728	max_ent = cpuid->nent;
		729
		730	for (cpuid->nent = 1; cpuid->nent <= max_ent; cpuid->nent++) {
		731	rc = ioctl(vcpu->fd, KVM_GET_CPUID2, cpuid);
		732	if (!rc)
		733	break;
		734
		735	TEST_ASSERT(rc == -1 && errno == E2BIG,
		736	"KVM_GET_CPUID2 should either succeed or give E2BIG: %d %d",
		737	rc, errno);
		738	}
		739
		740	TEST_ASSERT(rc == 0, "KVM_GET_CPUID2 failed, rc: %i errno: %i",
		741	rc, errno);
		742
		743	return cpuid;
		744	}
		745
		746
		747
672	/*	748	/*
673	* Locate a cpuid entry.	749	* Locate a cpuid entry.
674	*	750	*
@@ -1224,3 +1300,71 @@ uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2, · 549 unmodified lines
1224	: "b"(a0), "c"(a1), "d"(a2), "S"(a3));	1300	: "b"(a0), "c"(a1), "d"(a2), "S"(a3));
1225	return r;	1301	return r;
1226	}	1302	}
		1303
		1304	struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(void)
		1305	{
		1306	static struct kvm_cpuid2 *cpuid;
		1307	int ret;
		1308	int kvm_fd;
		1309
		1310	if (cpuid)
		1311	return cpuid;
		1312
		1313	cpuid = allocate_kvm_cpuid2();
		1314	kvm_fd = open(KVM_DEV_PATH, O_RDONLY);
		1315	if (kvm_fd < 0)
		1316	exit(KSFT_SKIP);
		1317
		1318	ret = ioctl(kvm_fd, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
		1319	TEST_ASSERT(ret == 0, "KVM_GET_SUPPORTED_HV_CPUID failed %d %d\n",
		1320	ret, errno);
		1321
		1322	close(kvm_fd);
		1323	return cpuid;
		1324	}
		1325
		1326	void vcpu_set_hv_cpuid(struct kvm_vm *vm, uint32_t vcpuid)
		1327	{
		1328	static struct kvm_cpuid2 *cpuid_full;
		1329	struct kvm_cpuid2 cpuid_sys, cpuid_hv;
		1330	int i, nent = 0;
		1331
		1332	if (!cpuid_full) {
		1333	cpuid_sys = kvm_get_supported_cpuid();
		1334	cpuid_hv = kvm_get_supported_hv_cpuid();
		1335
		1336	cpuid_full = malloc(sizeof(*cpuid_full) +
		1337	(cpuid_sys->nent + cpuid_hv->nent) *
		1338	sizeof(struct kvm_cpuid_entry2));
		1339	if (!cpuid_full) {
		1340	perror("malloc");
		1341	abort();
		1342	}
		1343
		1344	/* Need to skip KVM CPUID leaves 0x400000xx */
		1345	for (i = 0; i < cpuid_sys->nent; i++) {
		1346	if (cpuid_sys->entries[i].function >= 0x40000000 &&
		1347	cpuid_sys->entries[i].function < 0x40000100)
		1348	continue;
		1349	cpuid_full->entries[nent] = cpuid_sys->entries[i];
		1350	nent++;
		1351	}
		1352
		1353	memcpy(&cpuid_full->entries[nent], cpuid_hv->entries,
		1354	cpuid_hv->nent * sizeof(struct kvm_cpuid_entry2));
		1355	cpuid_full->nent = nent + cpuid_hv->nent;
		1356	}
		1357
		1358	vcpu_set_cpuid(vm, vcpuid, cpuid_full);
		1359	}
		1360
		1361	struct kvm_cpuid2 vcpu_get_supported_hv_cpuid(struct kvm_vm vm, uint32_t vcpuid)
		1362	{
		1363	static struct kvm_cpuid2 *cpuid;
		1364
		1365	cpuid = allocate_kvm_cpuid2();
		1366
		1367	vcpu_ioctl(vm, vcpuid, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
		1368
		1369	return cpuid;
		1370	}

tools/testing/selftests/kvm/lib/x86_64/svm.c+4−4

@@ -74,7 +74,7 @@ void generic_svm_setup(struct svm_test_data svm, void guest_rip, void *guest_r · 73 unmodified lines
74	wrmsr(MSR_VM_HSAVE_PA, svm->save_area_gpa);	74	wrmsr(MSR_VM_HSAVE_PA, svm->save_area_gpa);
75		75
76	memset(vmcb, 0, sizeof(*vmcb));	76	memset(vmcb, 0, sizeof(*vmcb));
77	asm volatile ("vmsave\n\t" : : "a" (vmcb_gpa) : "memory");	77	asm volatile ("vmsave %0\n\t" : : "a" (vmcb_gpa) : "memory");
78	vmcb_set_seg(&save->es, get_es(), 0, -1U, data_seg_attr);	78	vmcb_set_seg(&save->es, get_es(), 0, -1U, data_seg_attr);
79	vmcb_set_seg(&save->cs, get_cs(), 0, -1U, code_seg_attr);	79	vmcb_set_seg(&save->cs, get_cs(), 0, -1U, code_seg_attr);
80	vmcb_set_seg(&save->ss, get_ss(), 0, -1U, data_seg_attr);	80	vmcb_set_seg(&save->ss, get_ss(), 0, -1U, data_seg_attr);
@@ -131,19 +131,19 @@ void generic_svm_setup(struct svm_test_data svm, void guest_rip, void *guest_r · 50 unmodified lines
131	void run_guest(struct vmcb *vmcb, uint64_t vmcb_gpa)	131	void run_guest(struct vmcb *vmcb, uint64_t vmcb_gpa)
132	{	132	{
133	asm volatile (	133	asm volatile (
134	"vmload\n\t"	134	"vmload %[vmcb_gpa]\n\t"
135	"mov rflags, %%r15\n\t" // rflags	135	"mov rflags, %%r15\n\t" // rflags
136	"mov %%r15, 0x170(%[vmcb])\n\t"	136	"mov %%r15, 0x170(%[vmcb])\n\t"
137	"mov guest_regs, %%r15\n\t" // rax	137	"mov guest_regs, %%r15\n\t" // rax
138	"mov %%r15, 0x1f8(%[vmcb])\n\t"	138	"mov %%r15, 0x1f8(%[vmcb])\n\t"
139	LOAD_GPR_C	139	LOAD_GPR_C
140	"vmrun\n\t"	140	"vmrun %[vmcb_gpa]\n\t"
141	SAVE_GPR_C	141	SAVE_GPR_C
142	"mov 0x170(%[vmcb]), %%r15\n\t" // rflags	142	"mov 0x170(%[vmcb]), %%r15\n\t" // rflags
143	"mov %%r15, rflags\n\t"	143	"mov %%r15, rflags\n\t"
144	"mov 0x1f8(%[vmcb]), %%r15\n\t" // rax	144	"mov 0x1f8(%[vmcb]), %%r15\n\t" // rax
145	"mov %%r15, guest_regs\n\t"	145	"mov %%r15, guest_regs\n\t"
146	"vmsave\n\t"	146	"vmsave %[vmcb_gpa]\n\t"
147	: : [vmcb] "r" (vmcb), [vmcb_gpa] "a" (vmcb_gpa)	147	: : [vmcb] "r" (vmcb), [vmcb_gpa] "a" (vmcb_gpa)
148	: "r15", "memory");	148	: "r15", "memory");
149	}	149	}

tools/testing/selftests/kvm/memslot_modification_stress_test.c+212−0

@@ -0,0 +1,212 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* KVM memslot modification stress test
	4	* Adapted from demand_paging_test.c
	5	*
	6	* Copyright (C) 2018, Red Hat, Inc.
	7	* Copyright (C) 2020, Google, Inc.
	8	*/
	9
	10	#define _GNU_SOURCE /* for program_invocation_name */
	11
	12	#include <stdio.h>
	13	#include <stdlib.h>
	14	#include <sys/syscall.h>
	15	#include <unistd.h>
	16	#include <asm/unistd.h>
	17	#include <time.h>
	18	#include <poll.h>
	19	#include <pthread.h>
	20	#include <linux/bitmap.h>
	21	#include <linux/bitops.h>
	22	#include <linux/userfaultfd.h>
	23
	24	#include "perf_test_util.h"
	25	#include "processor.h"
	26	#include "test_util.h"
	27	#include "guest_modes.h"
	28
	29	#define DUMMY_MEMSLOT_INDEX 7
	30
	31	#define DEFAULT_MEMSLOT_MODIFICATION_ITERATIONS 10
	32
	33
	34	static int nr_vcpus = 1;
	35	static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
	36
	37	static bool run_vcpus = true;
	38
	39	static void vcpu_worker(void data)
	40	{
	41	int ret;
	42	struct perf_test_vcpu_args *vcpu_args =
	43	(struct perf_test_vcpu_args *)data;
	44	int vcpu_id = vcpu_args->vcpu_id;
	45	struct kvm_vm *vm = perf_test_args.vm;
	46	struct kvm_run *run;
	47
	48	vcpu_args_set(vm, vcpu_id, 1, vcpu_id);
	49	run = vcpu_state(vm, vcpu_id);
	50
	51	/* Let the guest access its memory until a stop signal is received */
	52	while (READ_ONCE(run_vcpus)) {
	53	ret = _vcpu_run(vm, vcpu_id);
	54	TEST_ASSERT(ret == 0, "vcpu_run failed: %d\n", ret);
	55
	56	if (get_ucall(vm, vcpu_id, NULL) == UCALL_SYNC)
	57	continue;
	58
	59	TEST_ASSERT(false,
	60	"Invalid guest sync status: exit_reason=%s\n",
	61	exit_reason_str(run->exit_reason));
	62	}
	63
	64	return NULL;
	65	}
	66
	67	struct memslot_antagonist_args {
	68	struct kvm_vm *vm;
	69	useconds_t delay;
	70	uint64_t nr_modifications;
	71	};
	72
	73	static void add_remove_memslot(struct kvm_vm *vm, useconds_t delay,
	74	uint64_t nr_modifications, uint64_t gpa)
	75	{
	76	int i;
	77
	78	for (i = 0; i < nr_modifications; i++) {
	79	usleep(delay);
	80	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, gpa,
	81	DUMMY_MEMSLOT_INDEX, 1, 0);
	82
	83	vm_mem_region_delete(vm, DUMMY_MEMSLOT_INDEX);
	84	}
	85	}
	86
	87	struct test_params {
	88	useconds_t memslot_modification_delay;
	89	uint64_t nr_memslot_modifications;
	90	bool partition_vcpu_memory_access;
	91	};
	92
	93	static void run_test(enum vm_guest_mode mode, void *arg)
	94	{
	95	struct test_params *p = arg;
	96	pthread_t *vcpu_threads;
	97	struct kvm_vm *vm;
	98	int vcpu_id;
	99
	100	vm = perf_test_create_vm(mode, nr_vcpus, guest_percpu_mem_size,
	101	VM_MEM_SRC_ANONYMOUS);
	102
	103	perf_test_args.wr_fract = 1;
	104
	105	vcpu_threads = malloc(nr_vcpus * sizeof(*vcpu_threads));
	106	TEST_ASSERT(vcpu_threads, "Memory allocation failed");
	107
	108	perf_test_setup_vcpus(vm, nr_vcpus, guest_percpu_mem_size,
	109	p->partition_vcpu_memory_access);
	110
	111	/* Export the shared variables to the guest */
	112	sync_global_to_guest(vm, perf_test_args);
	113
	114	pr_info("Finished creating vCPUs\n");
	115
	116	for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++)
	117	pthread_create(&vcpu_threads[vcpu_id], NULL, vcpu_worker,
	118	&perf_test_args.vcpu_args[vcpu_id]);
	119
	120	pr_info("Started all vCPUs\n");
	121
	122	add_remove_memslot(vm, p->memslot_modification_delay,
	123	p->nr_memslot_modifications,
	124	guest_test_phys_mem +
	125	(guest_percpu_mem_size * nr_vcpus) +
	126	perf_test_args.host_page_size +
	127	perf_test_args.guest_page_size);
	128
	129	run_vcpus = false;
	130
	131	/* Wait for the vcpu threads to quit */
	132	for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++)
	133	pthread_join(vcpu_threads[vcpu_id], NULL);
	134
	135	pr_info("All vCPU threads joined\n");
	136
	137	ucall_uninit(vm);
	138	kvm_vm_free(vm);
	139
	140	free(vcpu_threads);
	141	}
	142
	143	static void help(char *name)
	144	{
	145	puts("");
	146	printf("usage: %s [-h] [-m mode] [-d delay_usec]\n"
	147	" [-b memory] [-v vcpus] [-o] [-i iterations]\n", name);
	148	guest_modes_help();
	149	printf(" -d: add a delay between each iteration of adding and\n"
	150	" deleting a memslot in usec.\n");
	151	printf(" -b: specify the size of the memory region which should be\n"
	152	" accessed by each vCPU. e.g. 10M or 3G.\n"
	153	" Default: 1G\n");
	154	printf(" -v: specify the number of vCPUs to run.\n");
	155	printf(" -o: Overlap guest memory accesses instead of partitioning\n"
	156	" them into a separate region of memory for each vCPU.\n");
	157	printf(" -i: specify the number of iterations of adding and removing\n"
	158	" a memslot.\n"
	159	" Default: %d\n", DEFAULT_MEMSLOT_MODIFICATION_ITERATIONS);
	160	puts("");
	161	exit(0);
	162	}
	163
	164	int main(int argc, char *argv[])
	165	{
	166	int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS);
	167	int opt;
	168	struct test_params p = {
	169	.memslot_modification_delay = 0,
	170	.nr_memslot_modifications =
	171	DEFAULT_MEMSLOT_MODIFICATION_ITERATIONS,
	172	.partition_vcpu_memory_access = true
	173	};
	174
	175	guest_modes_append_default();
	176
	177	while ((opt = getopt(argc, argv, "hm:d:b:v:oi:")) != -1) {
	178	switch (opt) {
	179	case 'm':
	180	guest_modes_cmdline(optarg);
	181	break;
	182	case 'd':
	183	p.memslot_modification_delay = strtoul(optarg, NULL, 0);
	184	TEST_ASSERT(p.memslot_modification_delay >= 0,
	185	"A negative delay is not supported.");
	186	break;
	187	case 'b':
	188	guest_percpu_mem_size = parse_size(optarg);
	189	break;
	190	case 'v':
	191	nr_vcpus = atoi(optarg);
	192	TEST_ASSERT(nr_vcpus > 0 && nr_vcpus <= max_vcpus,
	193	"Invalid number of vcpus, must be between 1 and %d",
	194	max_vcpus);
	195	break;
	196	case 'o':
	197	p.partition_vcpu_memory_access = false;
	198	break;
	199	case 'i':
	200	p.nr_memslot_modifications = atoi(optarg);
	201	break;
	202	case 'h':
	203	default:
	204	help(argv[0]);
	205	break;
	206	}
	207	}
	208
	209	for_each_guest_mode(run_test, &p);
	210
	211	return 0;
	212	}

tools/testing/selftests/kvm/settings+1−0

@@ -0,0 +1 @@
		1	timeout=120

tools/testing/selftests/kvm/x86_64/evmcs_test.c+2−1

 		exit(KSFT_SKIP);
 	}
+	vcpu_set_hv_cpuid(vm, VCPU_ID);
 	vcpu_enable_evmcs(vm, VCPU_ID);
 	run = vcpu_state(vm, VCPU_ID);
 		/* Restore state in a new VM.  */
 		kvm_vm_restart(vm, O_RDWR);
 		vm_vcpu_add(vm, VCPU_ID);
-		vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid());
+		vcpu_set_hv_cpuid(vm, VCPU_ID);
 		vcpu_enable_evmcs(vm, VCPU_ID);
 		vcpu_load_state(vm, VCPU_ID, state);
 		run = vcpu_state(vm, VCPU_ID);

tools/testing/selftests/kvm/x86_64/get_cpuid_test.c+175−0

@@ -0,0 +1,175 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (C) 2021, Red Hat Inc.
	4	*
	5	* Generic tests for KVM CPUID set/get ioctls
	6	*/
	7	#include <asm/kvm_para.h>
	8	#include <linux/kvm_para.h>
	9	#include <stdint.h>
	10
	11	#include "test_util.h"
	12	#include "kvm_util.h"
	13	#include "processor.h"
	14
	15	#define VCPU_ID 0
	16
	17	/* CPUIDs known to differ */
	18	struct {
	19	u32 function;
	20	u32 index;
	21	} mangled_cpuids[] = {
	22	{.function = 0xd, .index = 0},
	23	};
	24
	25	static void test_guest_cpuids(struct kvm_cpuid2 *guest_cpuid)
	26	{
	27	int i;
	28	u32 eax, ebx, ecx, edx;
	29
	30	for (i = 0; i < guest_cpuid->nent; i++) {
	31	eax = guest_cpuid->entries[i].function;
	32	ecx = guest_cpuid->entries[i].index;
	33
	34	cpuid(&eax, &ebx, &ecx, &edx);
	35
	36	GUEST_ASSERT(eax == guest_cpuid->entries[i].eax &&
	37	ebx == guest_cpuid->entries[i].ebx &&
	38	ecx == guest_cpuid->entries[i].ecx &&
	39	edx == guest_cpuid->entries[i].edx);
	40	}
	41
	42	}
	43
	44	static void test_cpuid_40000000(struct kvm_cpuid2 *guest_cpuid)
	45	{
	46	u32 eax = 0x40000000, ebx, ecx = 0, edx;
	47
	48	cpuid(&eax, &ebx, &ecx, &edx);
	49
	50	GUEST_ASSERT(eax == 0x40000001);
	51	}
	52
	53	static void guest_main(struct kvm_cpuid2 *guest_cpuid)
	54	{
	55	GUEST_SYNC(1);
	56
	57	test_guest_cpuids(guest_cpuid);
	58
	59	GUEST_SYNC(2);
	60
	61	test_cpuid_40000000(guest_cpuid);
	62
	63	GUEST_DONE();
	64	}
	65
	66	static bool is_cpuid_mangled(struct kvm_cpuid_entry2 *entrie)
	67	{
	68	int i;
	69
	70	for (i = 0; i < sizeof(mangled_cpuids); i++) {
	71	if (mangled_cpuids[i].function == entrie->function &&
	72	mangled_cpuids[i].index == entrie->index)
	73	return true;
	74	}
	75
	76	return false;
	77	}
	78
	79	static void check_cpuid(struct kvm_cpuid2 cpuid, struct kvm_cpuid_entry2 entrie)
	80	{
	81	int i;
	82
	83	for (i = 0; i < cpuid->nent; i++) {
	84	if (cpuid->entries[i].function == entrie->function &&
	85	cpuid->entries[i].index == entrie->index) {
	86	if (is_cpuid_mangled(entrie))
	87	return;
	88
	89	TEST_ASSERT(cpuid->entries[i].eax == entrie->eax &&
	90	cpuid->entries[i].ebx == entrie->ebx &&
	91	cpuid->entries[i].ecx == entrie->ecx &&
	92	cpuid->entries[i].edx == entrie->edx,
	93	"CPUID 0x%x.%x differ: 0x%x:0x%x:0x%x:0x%x vs 0x%x:0x%x:0x%x:0x%x",
	94	entrie->function, entrie->index,
	95	cpuid->entries[i].eax, cpuid->entries[i].ebx,
	96	cpuid->entries[i].ecx, cpuid->entries[i].edx,
	97	entrie->eax, entrie->ebx, entrie->ecx, entrie->edx);
	98	return;
	99	}
	100	}
	101
	102	TEST_ASSERT(false, "CPUID 0x%x.%x not found", entrie->function, entrie->index);
	103	}
	104
	105	static void compare_cpuids(struct kvm_cpuid2 cpuid1, struct kvm_cpuid2 cpuid2)
	106	{
	107	int i;
	108
	109	for (i = 0; i < cpuid1->nent; i++)
	110	check_cpuid(cpuid2, &cpuid1->entries[i]);
	111
	112	for (i = 0; i < cpuid2->nent; i++)
	113	check_cpuid(cpuid1, &cpuid2->entries[i]);
	114	}
	115
	116	static void run_vcpu(struct kvm_vm *vm, uint32_t vcpuid, int stage)
	117	{
	118	struct ucall uc;
	119
	120	_vcpu_run(vm, vcpuid);
	121
	122	switch (get_ucall(vm, vcpuid, &uc)) {
	123	case UCALL_SYNC:
	124	TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") &&
	125	uc.args[1] == stage + 1,
	126	"Stage %d: Unexpected register values vmexit, got %lx",
	127	stage + 1, (ulong)uc.args[1]);
	128	return;
	129	case UCALL_DONE:
	130	return;
	131	case UCALL_ABORT:
	132	TEST_ASSERT(false, "%s at %s:%ld\n\tvalues: %#lx, %#lx", (const char *)uc.args[0],
	133	__FILE__, uc.args[1], uc.args[2], uc.args[3]);
	134	default:
	135	TEST_ASSERT(false, "Unexpected exit: %s",
	136	exit_reason_str(vcpu_state(vm, vcpuid)->exit_reason));
	137	}
	138	}
	139
	140	struct kvm_cpuid2 vcpu_alloc_cpuid(struct kvm_vm vm, vm_vaddr_t p_gva, struct kvm_cpuid2 cpuid)
	141	{
	142	int size = sizeof(cpuid) + cpuid->nent sizeof(cpuid->entries[0]);
	143	vm_vaddr_t gva = vm_vaddr_alloc(vm, size,
	144	getpagesize(), 0, 0);
	145	struct kvm_cpuid2 *guest_cpuids = addr_gva2hva(vm, gva);
	146
	147	memcpy(guest_cpuids, cpuid, size);
	148
	149	*p_gva = gva;
	150	return guest_cpuids;
	151	}
	152
	153	int main(void)
	154	{
	155	struct kvm_cpuid2 supp_cpuid, cpuid2;
	156	vm_vaddr_t cpuid_gva;
	157	struct kvm_vm *vm;
	158	int stage;
	159
	160	vm = vm_create_default(VCPU_ID, 0, guest_main);
	161
	162	supp_cpuid = kvm_get_supported_cpuid();
	163	cpuid2 = vcpu_get_cpuid(vm, VCPU_ID);
	164
	165	compare_cpuids(supp_cpuid, cpuid2);
	166
	167	vcpu_alloc_cpuid(vm, &cpuid_gva, cpuid2);
	168
	169	vcpu_args_set(vm, VCPU_ID, 1, cpuid_gva);
	170
	171	for (stage = 0; stage < 3; stage++)
	172	run_vcpu(vm, VCPU_ID, stage);
	173
	174	kvm_vm_free(vm);
	175	}

tools/testing/selftests/kvm/x86_64/hyperv_cpuid.c+3−28

 		    " it should have: %d %d", system ? "KVM" : "vCPU", ret, errno);
 }
-struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(struct kvm_vm *vm, bool system)
-{
-	int nent = 20; /* should be enough */
-	static struct kvm_cpuid2 *cpuid;
-	cpuid = malloc(sizeof(*cpuid) + nent * sizeof(struct kvm_cpuid_entry2));
-	if (!cpuid) {
-		perror("malloc");
-		abort();
-	}
-	cpuid->nent = nent;
-	if (!system)
-		vcpu_ioctl(vm, VCPU_ID, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
-	else
-		kvm_ioctl(vm, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
-	return cpuid;
-}
 int main(int argc, char *argv[])
 {
 	struct kvm_vm *vm;
 	/* Test vCPU ioctl version */
 	test_hv_cpuid_e2big(vm, false);
-	hv_cpuid_entries = kvm_get_supported_hv_cpuid(vm, false);
+	hv_cpuid_entries = vcpu_get_supported_hv_cpuid(vm, VCPU_ID);
 	test_hv_cpuid(hv_cpuid_entries, false);
 	free(hv_cpuid_entries);
 		goto do_sys;
 	}
 	vcpu_enable_evmcs(vm, VCPU_ID);
-	hv_cpuid_entries = kvm_get_supported_hv_cpuid(vm, false);
+	hv_cpuid_entries = vcpu_get_supported_hv_cpuid(vm, VCPU_ID);
 	test_hv_cpuid(hv_cpuid_entries, true);
 	free(hv_cpuid_entries);
 	test_hv_cpuid_e2big(vm, true);
-	hv_cpuid_entries = kvm_get_supported_hv_cpuid(vm, true);
+	hv_cpuid_entries = kvm_get_supported_hv_cpuid();
 	test_hv_cpuid(hv_cpuid_entries, nested_vmx_supported());
-	free(hv_cpuid_entries);
 out:
 	kvm_vm_free(vm);

tools/testing/selftests/kvm/x86_64/vmx_pmu_msrs_test.c+131−0

@@ -0,0 +1,131 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* VMX-pmu related msrs test
	4	*
	5	* Copyright (C) 2021 Intel Corporation
	6	*
	7	* Test to check the effect of various CPUID settings
	8	* on the MSR_IA32_PERF_CAPABILITIES MSR, and check that
	9	* whatever we write with KVM_SET_MSR is _not_ modified
	10	* in the guest and test it can be retrieved with KVM_GET_MSR.
	11	*
	12	* Test to check that invalid LBR formats are rejected.
	13	*/
	14
	15	#define _GNU_SOURCE /* for program_invocation_short_name */
	16	#include <sys/ioctl.h>
	17
	18	#include "kvm_util.h"
	19	#include "vmx.h"
	20
	21	#define VCPU_ID 0
	22
	23	#define X86_FEATURE_PDCM (1<<15)
	24	#define PMU_CAP_FW_WRITES (1ULL << 13)
	25	#define PMU_CAP_LBR_FMT 0x3f
	26
	27	union cpuid10_eax {
	28	struct {
	29	unsigned int version_id:8;
	30	unsigned int num_counters:8;
	31	unsigned int bit_width:8;
	32	unsigned int mask_length:8;
	33	} split;
	34	unsigned int full;
	35	};
	36
	37	union perf_capabilities {
	38	struct {
	39	u64 lbr_format:6;
	40	u64 pebs_trap:1;
	41	u64 pebs_arch_reg:1;
	42	u64 pebs_format:4;
	43	u64 smm_freeze:1;
	44	u64 full_width_write:1;
	45	u64 pebs_baseline:1;
	46	u64 perf_metrics:1;
	47	u64 pebs_output_pt_available:1;
	48	u64 anythread_deprecated:1;
	49	};
	50	u64 capabilities;
	51	};
	52
	53	static void guest_code(void)
	54	{
	55	wrmsr(MSR_IA32_PERF_CAPABILITIES, PMU_CAP_LBR_FMT);
	56	}
	57
	58	int main(int argc, char *argv[])
	59	{
	60	struct kvm_cpuid2 *cpuid;
	61	struct kvm_cpuid_entry2 *entry_1_0;
	62	struct kvm_cpuid_entry2 *entry_a_0;
	63	bool pdcm_supported = false;
	64	struct kvm_vm *vm;
	65	int ret;
	66	union cpuid10_eax eax;
	67	union perf_capabilities host_cap;
	68
	69	host_cap.capabilities = kvm_get_feature_msr(MSR_IA32_PERF_CAPABILITIES);
	70	host_cap.capabilities &= (PMU_CAP_FW_WRITES \| PMU_CAP_LBR_FMT);
	71
	72	/* Create VM */
	73	vm = vm_create_default(VCPU_ID, 0, guest_code);
	74	cpuid = kvm_get_supported_cpuid();
	75
	76	if (kvm_get_cpuid_max_basic() >= 0xa) {
	77	entry_1_0 = kvm_get_supported_cpuid_index(1, 0);
	78	entry_a_0 = kvm_get_supported_cpuid_index(0xa, 0);
	79	pdcm_supported = entry_1_0 && !!(entry_1_0->ecx & X86_FEATURE_PDCM);
	80	eax.full = entry_a_0->eax;
	81	}
	82	if (!pdcm_supported) {
	83	print_skip("MSR_IA32_PERF_CAPABILITIES is not supported by the vCPU");
	84	exit(KSFT_SKIP);
	85	}
	86	if (!eax.split.version_id) {
	87	print_skip("PMU is not supported by the vCPU");
	88	exit(KSFT_SKIP);
	89	}
	90
	91	/* testcase 1, set capabilities when we have PDCM bit */
	92	vcpu_set_cpuid(vm, VCPU_ID, cpuid);
	93	vcpu_set_msr(vm, 0, MSR_IA32_PERF_CAPABILITIES, PMU_CAP_FW_WRITES);
	94
	95	/* check capabilities can be retrieved with KVM_GET_MSR */
	96	ASSERT_EQ(vcpu_get_msr(vm, VCPU_ID, MSR_IA32_PERF_CAPABILITIES), PMU_CAP_FW_WRITES);
	97
	98	/* check whatever we write with KVM_SET_MSR is _not_ modified */
	99	vcpu_run(vm, VCPU_ID);
	100	ASSERT_EQ(vcpu_get_msr(vm, VCPU_ID, MSR_IA32_PERF_CAPABILITIES), PMU_CAP_FW_WRITES);
	101
	102	/* testcase 2, check valid LBR formats are accepted */
	103	vcpu_set_msr(vm, 0, MSR_IA32_PERF_CAPABILITIES, 0);
	104	ASSERT_EQ(vcpu_get_msr(vm, VCPU_ID, MSR_IA32_PERF_CAPABILITIES), 0);
	105
	106	vcpu_set_msr(vm, 0, MSR_IA32_PERF_CAPABILITIES, host_cap.lbr_format);
	107	ASSERT_EQ(vcpu_get_msr(vm, VCPU_ID, MSR_IA32_PERF_CAPABILITIES), (u64)host_cap.lbr_format);
	108
	109	/* testcase 3, check invalid LBR format is rejected */
	110	ret = _vcpu_set_msr(vm, 0, MSR_IA32_PERF_CAPABILITIES, PMU_CAP_LBR_FMT);
	111	TEST_ASSERT(ret == 0, "Bad PERF_CAPABILITIES didn't fail.");
	112
	113	/* testcase 4, set capabilities when we don't have PDCM bit */
	114	entry_1_0->ecx &= ~X86_FEATURE_PDCM;
	115	vcpu_set_cpuid(vm, VCPU_ID, cpuid);
	116	ret = _vcpu_set_msr(vm, 0, MSR_IA32_PERF_CAPABILITIES, host_cap.capabilities);
	117	TEST_ASSERT(ret == 0, "Bad PERF_CAPABILITIES didn't fail.");
	118
	119	/* testcase 5, set capabilities when we don't have PMU version bits */
	120	entry_1_0->ecx \|= X86_FEATURE_PDCM;
	121	eax.split.version_id = 0;
	122	entry_1_0->ecx = eax.full;
	123	vcpu_set_cpuid(vm, VCPU_ID, cpuid);
	124	ret = _vcpu_set_msr(vm, 0, MSR_IA32_PERF_CAPABILITIES, PMU_CAP_FW_WRITES);
	125	TEST_ASSERT(ret == 0, "Bad PERF_CAPABILITIES didn't fail.");
	126
	127	vcpu_set_msr(vm, 0, MSR_IA32_PERF_CAPABILITIES, 0);
	128	ASSERT_EQ(vcpu_get_msr(vm, VCPU_ID, MSR_IA32_PERF_CAPABILITIES), 0);
	129
	130	kvm_vm_free(vm);
	131	}

tools/testing/selftests/kvm/x86_64/xapic_ipi_test.c+544−0

@@ -0,0 +1,544 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* xapic_ipi_test
	4	*
	5	* Copyright (C) 2020, Google LLC.
	6	*
	7	* This work is licensed under the terms of the GNU GPL, version 2.
	8	*
	9	* Test that when the APIC is in xAPIC mode, a vCPU can send an IPI to wake
	10	* another vCPU that is halted when KVM's backing page for the APIC access
	11	* address has been moved by mm.
	12	*
	13	* The test starts two vCPUs: one that sends IPIs and one that continually
	14	* executes HLT. The sender checks that the halter has woken from the HLT and
	15	* has reentered HLT before sending the next IPI. While the vCPUs are running,
	16	* the host continually calls migrate_pages to move all of the process' pages
	17	* amongst the available numa nodes on the machine.
	18	*
	19	* Migration is a command line option. When used on non-numa machines will
	20	* exit with error. Test is still usefull on non-numa for testing IPIs.
	21	*/
	22
	23	#define _GNU_SOURCE /* for program_invocation_short_name */
	24	#include <getopt.h>
	25	#include <pthread.h>
	26	#include <inttypes.h>
	27	#include <string.h>
	28	#include <time.h>
	29
	30	#include "kvm_util.h"
	31	#include "numaif.h"
	32	#include "processor.h"
	33	#include "test_util.h"
	34	#include "vmx.h"
	35
	36	/* Default running time for the test */
	37	#define DEFAULT_RUN_SECS 3
	38
	39	/* Default delay between migrate_pages calls (microseconds) */
	40	#define DEFAULT_DELAY_USECS 500000
	41
	42	#define HALTER_VCPU_ID 0
	43	#define SENDER_VCPU_ID 1
	44
	45	volatile uint32_t apic_base = (volatile uint32_t )APIC_DEFAULT_GPA;
	46
	47	/*
	48	* Vector for IPI from sender vCPU to halting vCPU.
	49	* Value is arbitrary and was chosen for the alternating bit pattern. Any
	50	* value should work.
	51	*/
	52	#define IPI_VECTOR 0xa5
	53
	54	/*
	55	* Incremented in the IPI handler. Provides evidence to the sender that the IPI
	56	* arrived at the destination
	57	*/
	58	static volatile uint64_t ipis_rcvd;
	59
	60	/* Data struct shared between host main thread and vCPUs */
	61	struct test_data_page {
	62	uint32_t halter_apic_id;
	63	volatile uint64_t hlt_count;
	64	volatile uint64_t wake_count;
	65	uint64_t ipis_sent;
	66	uint64_t migrations_attempted;
	67	uint64_t migrations_completed;
	68	uint32_t icr;
	69	uint32_t icr2;
	70	uint32_t halter_tpr;
	71	uint32_t halter_ppr;
	72
	73	/*
	74	* Record local version register as a cross-check that APIC access
	75	* worked. Value should match what KVM reports (APIC_VERSION in
	76	* arch/x86/kvm/lapic.c). If test is failing, check that values match
	77	* to determine whether APIC access exits are working.
	78	*/
	79	uint32_t halter_lvr;
	80	};
	81
	82	struct thread_params {
	83	struct test_data_page *data;
	84	struct kvm_vm *vm;
	85	uint32_t vcpu_id;
	86	uint64_t pipis_rcvd; / host address of ipis_rcvd global */
	87	};
	88
	89	uint32_t read_apic_reg(uint reg)
	90	{
	91	return apic_base[reg >> 2];
	92	}
	93
	94	void write_apic_reg(uint reg, uint32_t val)
	95	{
	96	apic_base[reg >> 2] = val;
	97	}
	98
	99	void disable_apic(void)
	100	{
	101	wrmsr(MSR_IA32_APICBASE,
	102	rdmsr(MSR_IA32_APICBASE) &
	103	~(MSR_IA32_APICBASE_ENABLE \| MSR_IA32_APICBASE_EXTD));
	104	}
	105
	106	void enable_xapic(void)
	107	{
	108	uint64_t val = rdmsr(MSR_IA32_APICBASE);
	109
	110	/* Per SDM: to enable xAPIC when in x2APIC must first disable APIC */
	111	if (val & MSR_IA32_APICBASE_EXTD) {
	112	disable_apic();
	113	wrmsr(MSR_IA32_APICBASE,
	114	rdmsr(MSR_IA32_APICBASE) \| MSR_IA32_APICBASE_ENABLE);
	115	} else if (!(val & MSR_IA32_APICBASE_ENABLE)) {
	116	wrmsr(MSR_IA32_APICBASE, val \| MSR_IA32_APICBASE_ENABLE);
	117	}
	118
	119	/*
	120	* Per SDM: reset value of spurious interrupt vector register has the
	121	* APIC software enabled bit=0. It must be enabled in addition to the
	122	* enable bit in the MSR.
	123	*/
	124	val = read_apic_reg(APIC_SPIV) \| APIC_SPIV_APIC_ENABLED;
	125	write_apic_reg(APIC_SPIV, val);
	126	}
	127
	128	void verify_apic_base_addr(void)
	129	{
	130	uint64_t msr = rdmsr(MSR_IA32_APICBASE);
	131	uint64_t base = GET_APIC_BASE(msr);
	132
	133	GUEST_ASSERT(base == APIC_DEFAULT_GPA);
	134	}
	135
	136	static void halter_guest_code(struct test_data_page *data)
	137	{
	138	verify_apic_base_addr();
	139	enable_xapic();
	140
	141	data->halter_apic_id = GET_APIC_ID_FIELD(read_apic_reg(APIC_ID));
	142	data->halter_lvr = read_apic_reg(APIC_LVR);
	143
	144	/*
	145	* Loop forever HLTing and recording halts & wakes. Disable interrupts
	146	* each time around to minimize window between signaling the pending
	147	* halt to the sender vCPU and executing the halt. No need to disable on
	148	* first run as this vCPU executes first and the host waits for it to
	149	* signal going into first halt before starting the sender vCPU. Record
	150	* TPR and PPR for diagnostic purposes in case the test fails.
	151	*/
	152	for (;;) {
	153	data->halter_tpr = read_apic_reg(APIC_TASKPRI);
	154	data->halter_ppr = read_apic_reg(APIC_PROCPRI);
	155	data->hlt_count++;
	156	asm volatile("sti; hlt; cli");
	157	data->wake_count++;
	158	}
	159	}
	160
	161	/*
	162	* Runs on halter vCPU when IPI arrives. Write an arbitrary non-zero value to
	163	* enable diagnosing errant writes to the APIC access address backing page in
	164	* case of test failure.
	165	*/
	166	static void guest_ipi_handler(struct ex_regs *regs)
	167	{
	168	ipis_rcvd++;
	169	write_apic_reg(APIC_EOI, 77);
	170	}
	171
	172	static void sender_guest_code(struct test_data_page *data)
	173	{
	174	uint64_t last_wake_count;
	175	uint64_t last_hlt_count;
	176	uint64_t last_ipis_rcvd_count;
	177	uint32_t icr_val;
	178	uint32_t icr2_val;
	179	uint64_t tsc_start;
	180
	181	verify_apic_base_addr();
	182	enable_xapic();
	183
	184	/*
	185	* Init interrupt command register for sending IPIs
	186	*
	187	* Delivery mode=fixed, per SDM:
	188	* "Delivers the interrupt specified in the vector field to the target
	189	* processor."
	190	*
	191	* Destination mode=physical i.e. specify target by its local APIC
	192	* ID. This vCPU assumes that the halter vCPU has already started and
	193	* set data->halter_apic_id.
	194	*/
	195	icr_val = (APIC_DEST_PHYSICAL \| APIC_DM_FIXED \| IPI_VECTOR);
	196	icr2_val = SET_APIC_DEST_FIELD(data->halter_apic_id);
	197	data->icr = icr_val;
	198	data->icr2 = icr2_val;
	199
	200	last_wake_count = data->wake_count;
	201	last_hlt_count = data->hlt_count;
	202	last_ipis_rcvd_count = ipis_rcvd;
	203	for (;;) {
	204	/*
	205	* Send IPI to halter vCPU.
	206	* First IPI can be sent unconditionally because halter vCPU
	207	* starts earlier.
	208	*/
	209	write_apic_reg(APIC_ICR2, icr2_val);
	210	write_apic_reg(APIC_ICR, icr_val);
	211	data->ipis_sent++;
	212
	213	/*
	214	* Wait up to ~1 sec for halter to indicate that it has:
	215	* 1. Received the IPI
	216	* 2. Woken up from the halt
	217	* 3. Gone back into halt
	218	* Current CPUs typically run at 2.x Ghz which is ~2
	219	* billion ticks per second.
	220	*/
	221	tsc_start = rdtsc();
	222	while (rdtsc() - tsc_start < 2000000000) {
	223	if ((ipis_rcvd != last_ipis_rcvd_count) &&
	224	(data->wake_count != last_wake_count) &&
	225	(data->hlt_count != last_hlt_count))
	226	break;
	227	}
	228
	229	GUEST_ASSERT((ipis_rcvd != last_ipis_rcvd_count) &&
	230	(data->wake_count != last_wake_count) &&
	231	(data->hlt_count != last_hlt_count));
	232
	233	last_wake_count = data->wake_count;
	234	last_hlt_count = data->hlt_count;
	235	last_ipis_rcvd_count = ipis_rcvd;
	236	}
	237	}
	238
	239	static void vcpu_thread(void arg)
	240	{
	241	struct thread_params params = (struct thread_params )arg;
	242	struct ucall uc;
	243	int old;
	244	int r;
	245	unsigned int exit_reason;
	246
	247	r = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old);
	248	TEST_ASSERT(r == 0,
	249	"pthread_setcanceltype failed on vcpu_id=%u with errno=%d",
	250	params->vcpu_id, r);
	251
	252	fprintf(stderr, "vCPU thread running vCPU %u\n", params->vcpu_id);
	253	vcpu_run(params->vm, params->vcpu_id);
	254	exit_reason = vcpu_state(params->vm, params->vcpu_id)->exit_reason;
	255
	256	TEST_ASSERT(exit_reason == KVM_EXIT_IO,
	257	"vCPU %u exited with unexpected exit reason %u-%s, expected KVM_EXIT_IO",
	258	params->vcpu_id, exit_reason, exit_reason_str(exit_reason));
	259
	260	if (get_ucall(params->vm, params->vcpu_id, &uc) == UCALL_ABORT) {
	261	TEST_ASSERT(false,
	262	"vCPU %u exited with error: %s.\n"
	263	"Sending vCPU sent %lu IPIs to halting vCPU\n"
	264	"Halting vCPU halted %lu times, woke %lu times, received %lu IPIs.\n"
	265	"Halter TPR=%#x PPR=%#x LVR=%#x\n"
	266	"Migrations attempted: %lu\n"
	267	"Migrations completed: %lu\n",
	268	params->vcpu_id, (const char *)uc.args[0],
	269	params->data->ipis_sent, params->data->hlt_count,
	270	params->data->wake_count,
	271	*params->pipis_rcvd, params->data->halter_tpr,
	272	params->data->halter_ppr, params->data->halter_lvr,
	273	params->data->migrations_attempted,
	274	params->data->migrations_completed);
	275	}
	276
	277	return NULL;
	278	}
	279
	280	static void cancel_join_vcpu_thread(pthread_t thread, uint32_t vcpu_id)
	281	{
	282	void *retval;
	283	int r;
	284
	285	r = pthread_cancel(thread);
	286	TEST_ASSERT(r == 0,
	287	"pthread_cancel on vcpu_id=%d failed with errno=%d",
	288	vcpu_id, r);
	289
	290	r = pthread_join(thread, &retval);
	291	TEST_ASSERT(r == 0,
	292	"pthread_join on vcpu_id=%d failed with errno=%d",
	293	vcpu_id, r);
	294	TEST_ASSERT(retval == PTHREAD_CANCELED,
	295	"expected retval=%p, got %p", PTHREAD_CANCELED,
	296	retval);
	297	}
	298
	299	void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs,
	300	uint64_t *pipis_rcvd)
	301	{
	302	long pages_not_moved;
	303	unsigned long nodemask = 0;
	304	unsigned long nodemasks[sizeof(nodemask) * 8];
	305	int nodes = 0;
	306	time_t start_time, last_update, now;
	307	time_t interval_secs = 1;
	308	int i, r;
	309	int from, to;
	310	unsigned long bit;
	311	uint64_t hlt_count;
	312	uint64_t wake_count;
	313	uint64_t ipis_sent;
	314
	315	fprintf(stderr, "Calling migrate_pages every %d microseconds\n",
	316	delay_usecs);
	317
	318	/* Get set of first 64 numa nodes available */
	319	r = get_mempolicy(NULL, &nodemask, sizeof(nodemask) * 8,
	320	0, MPOL_F_MEMS_ALLOWED);
	321	TEST_ASSERT(r == 0, "get_mempolicy failed errno=%d", errno);
	322
	323	fprintf(stderr, "Numa nodes found amongst first %lu possible nodes "
	324	"(each 1-bit indicates node is present): %#lx\n",
	325	sizeof(nodemask) * 8, nodemask);
	326
	327	/* Init array of masks containing a single-bit in each, one for each
	328	* available node. migrate_pages called below requires specifying nodes
	329	* as bit masks.
	330	*/
	331	for (i = 0, bit = 1; i < sizeof(nodemask) * 8; i++, bit <<= 1) {
	332	if (nodemask & bit) {
	333	nodemasks[nodes] = nodemask & bit;
	334	nodes++;
	335	}
	336	}
	337
	338	TEST_ASSERT(nodes > 1,
	339	"Did not find at least 2 numa nodes. Can't do migration\n");
	340
	341	fprintf(stderr, "Migrating amongst %d nodes found\n", nodes);
	342
	343	from = 0;
	344	to = 1;
	345	start_time = time(NULL);
	346	last_update = start_time;
	347
	348	ipis_sent = data->ipis_sent;
	349	hlt_count = data->hlt_count;
	350	wake_count = data->wake_count;
	351
	352	while ((int)(time(NULL) - start_time) < run_secs) {
	353	data->migrations_attempted++;
	354
	355	/*
	356	* migrate_pages with PID=0 will migrate all pages of this
	357	* process between the nodes specified as bitmasks. The page
	358	* backing the APIC access address belongs to this process
	359	* because it is allocated by KVM in the context of the
	360	* KVM_CREATE_VCPU ioctl. If that assumption ever changes this
	361	* test may break or give a false positive signal.
	362	*/
	363	pages_not_moved = migrate_pages(0, sizeof(nodemasks[from]),
	364	&nodemasks[from],
	365	&nodemasks[to]);
	366	if (pages_not_moved < 0)
	367	fprintf(stderr,
	368	"migrate_pages failed, errno=%d\n", errno);
	369	else if (pages_not_moved > 0)
	370	fprintf(stderr,
	371	"migrate_pages could not move %ld pages\n",
	372	pages_not_moved);
	373	else
	374	data->migrations_completed++;
	375
	376	from = to;
	377	to++;
	378	if (to == nodes)
	379	to = 0;
	380
	381	now = time(NULL);
	382	if (((now - start_time) % interval_secs == 0) &&
	383	(now != last_update)) {
	384	last_update = now;
	385	fprintf(stderr,
	386	"%lu seconds: Migrations attempted=%lu completed=%lu, "
	387	"IPIs sent=%lu received=%lu, HLTs=%lu wakes=%lu\n",
	388	now - start_time, data->migrations_attempted,
	389	data->migrations_completed,
	390	data->ipis_sent, *pipis_rcvd,
	391	data->hlt_count, data->wake_count);
	392
	393	TEST_ASSERT(ipis_sent != data->ipis_sent &&
	394	hlt_count != data->hlt_count &&
	395	wake_count != data->wake_count,
	396	"IPI, HLT and wake count have not increased "
	397	"in the last %lu seconds. "
	398	"HLTer is likely hung.\n", interval_secs);
	399
	400	ipis_sent = data->ipis_sent;
	401	hlt_count = data->hlt_count;
	402	wake_count = data->wake_count;
	403	}
	404	usleep(delay_usecs);
	405	}
	406	}
	407
	408	void get_cmdline_args(int argc, char argv[], int run_secs,
	409	bool migrate, int delay_usecs)
	410	{
	411	for (;;) {
	412	int opt = getopt(argc, argv, "s:d:m");
	413
	414	if (opt == -1)
	415	break;
	416	switch (opt) {
	417	case 's':
	418	*run_secs = parse_size(optarg);
	419	break;
	420	case 'm':
	421	*migrate = true;
	422	break;
	423	case 'd':
	424	*delay_usecs = parse_size(optarg);
	425	break;
	426	default:
	427	TEST_ASSERT(false,
	428	"Usage: -s <runtime seconds>. Default is %d seconds.\n"
	429	"-m adds calls to migrate_pages while vCPUs are running."
	430	" Default is no migrations.\n"
	431	"-d <delay microseconds> - delay between migrate_pages() calls."
	432	" Default is %d microseconds.\n",
	433	DEFAULT_RUN_SECS, DEFAULT_DELAY_USECS);
	434	}
	435	}
	436	}
	437
	438	int main(int argc, char *argv[])
	439	{
	440	int r;
	441	int wait_secs;
	442	const int max_halter_wait = 10;
	443	int run_secs = 0;
	444	int delay_usecs = 0;
	445	struct test_data_page *data;
	446	vm_vaddr_t test_data_page_vaddr;
	447	bool migrate = false;
	448	pthread_t threads[2];
	449	struct thread_params params[2];
	450	struct kvm_vm *vm;
	451	uint64_t *pipis_rcvd;
	452
	453	get_cmdline_args(argc, argv, &run_secs, &migrate, &delay_usecs);
	454	if (run_secs <= 0)
	455	run_secs = DEFAULT_RUN_SECS;
	456	if (delay_usecs <= 0)
	457	delay_usecs = DEFAULT_DELAY_USECS;
	458
	459	vm = vm_create_default(HALTER_VCPU_ID, 0, halter_guest_code);
	460	params[0].vm = vm;
	461	params[1].vm = vm;
	462
	463	vm_init_descriptor_tables(vm);
	464	vcpu_init_descriptor_tables(vm, HALTER_VCPU_ID);
	465	vm_handle_exception(vm, IPI_VECTOR, guest_ipi_handler);
	466
	467	virt_pg_map(vm, APIC_DEFAULT_GPA, APIC_DEFAULT_GPA, 0);
	468
	469	vm_vcpu_add_default(vm, SENDER_VCPU_ID, sender_guest_code);
	470
	471	test_data_page_vaddr = vm_vaddr_alloc(vm, 0x1000, 0x1000, 0, 0);
	472	data =
	473	(struct test_data_page *)addr_gva2hva(vm, test_data_page_vaddr);
	474	memset(data, 0, sizeof(*data));
	475	params[0].data = data;
	476	params[1].data = data;
	477
	478	vcpu_args_set(vm, HALTER_VCPU_ID, 1, test_data_page_vaddr);
	479	vcpu_args_set(vm, SENDER_VCPU_ID, 1, test_data_page_vaddr);
	480
	481	pipis_rcvd = (uint64_t *)addr_gva2hva(vm, (uint64_t)&ipis_rcvd);
	482	params[0].pipis_rcvd = pipis_rcvd;
	483	params[1].pipis_rcvd = pipis_rcvd;
	484
	485	/* Start halter vCPU thread and wait for it to execute first HLT. */
	486	params[0].vcpu_id = HALTER_VCPU_ID;
	487	r = pthread_create(&threads[0], NULL, vcpu_thread, &params[0]);
	488	TEST_ASSERT(r == 0,
	489	"pthread_create halter failed errno=%d", errno);
	490	fprintf(stderr, "Halter vCPU thread started\n");
	491
	492	wait_secs = 0;
	493	while ((wait_secs < max_halter_wait) && !data->hlt_count) {
	494	sleep(1);
	495	wait_secs++;
	496	}
	497
	498	TEST_ASSERT(data->hlt_count,
	499	"Halter vCPU did not execute first HLT within %d seconds",
	500	max_halter_wait);
	501
	502	fprintf(stderr,
	503	"Halter vCPU thread reported its APIC ID: %u after %d seconds.\n",
	504	data->halter_apic_id, wait_secs);
	505
	506	params[1].vcpu_id = SENDER_VCPU_ID;
	507	r = pthread_create(&threads[1], NULL, vcpu_thread, &params[1]);
	508	TEST_ASSERT(r == 0, "pthread_create sender failed errno=%d", errno);
	509
	510	fprintf(stderr,
	511	"IPI sender vCPU thread started. Letting vCPUs run for %d seconds.\n",
	512	run_secs);
	513
	514	if (!migrate)
	515	sleep(run_secs);
	516	else
	517	do_migrations(data, run_secs, delay_usecs, pipis_rcvd);
	518
	519	/*
	520	* Cancel threads and wait for them to stop.
	521	*/
	522	cancel_join_vcpu_thread(threads[0], HALTER_VCPU_ID);
	523	cancel_join_vcpu_thread(threads[1], SENDER_VCPU_ID);
	524
	525	fprintf(stderr,
	526	"Test successful after running for %d seconds.\n"
	527	"Sending vCPU sent %lu IPIs to halting vCPU\n"
	528	"Halting vCPU halted %lu times, woke %lu times, received %lu IPIs.\n"
	529	"Halter APIC ID=%#x\n"
	530	"Sender ICR value=%#x ICR2 value=%#x\n"
	531	"Halter TPR=%#x PPR=%#x LVR=%#x\n"
	532	"Migrations attempted: %lu\n"
	533	"Migrations completed: %lu\n",
	534	run_secs, data->ipis_sent,
	535	data->hlt_count, data->wake_count, *pipis_rcvd,
	536	data->halter_apic_id,
	537	data->icr, data->icr2,
	538	data->halter_tpr, data->halter_ppr, data->halter_lvr,
	539	data->migrations_attempted, data->migrations_completed);
	540
	541	kvm_vm_free(vm);
	542
	543	return 0;
	544	}

tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c+167−0

@@ -0,0 +1,167 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* svm_vmcall_test
	4	*
	5	* Copyright © 2021 Amazon.com, Inc. or its affiliates.
	6	*
	7	* Xen shared_info / pvclock testing
	8	*/
	9
	10	#include "test_util.h"
	11	#include "kvm_util.h"
	12	#include "processor.h"
	13
	14	#include <stdint.h>
	15	#include <time.h>
	16
	17	#define VCPU_ID 5
	18
	19	#define SHINFO_REGION_GPA 0xc0000000ULL
	20	#define SHINFO_REGION_SLOT 10
	21	#define PAGE_SIZE 4096
	22
	23	#define PVTIME_ADDR (SHINFO_REGION_GPA + PAGE_SIZE)
	24
	25	static struct kvm_vm *vm;
	26
	27	#define XEN_HYPERCALL_MSR 0x40000000
	28
	29	struct pvclock_vcpu_time_info {
	30	u32 version;
	31	u32 pad0;
	32	u64 tsc_timestamp;
	33	u64 system_time;
	34	u32 tsc_to_system_mul;
	35	s8 tsc_shift;
	36	u8 flags;
	37	u8 pad[2];
	38	} __attribute__((__packed__)); /* 32 bytes */
	39
	40	struct pvclock_wall_clock {
	41	u32 version;
	42	u32 sec;
	43	u32 nsec;
	44	} __attribute__((__packed__));
	45
	46	static void guest_code(void)
	47	{
	48	GUEST_DONE();
	49	}
	50
	51	static int cmp_timespec(struct timespec a, struct timespec b)
	52	{
	53	if (a->tv_sec > b->tv_sec)
	54	return 1;
	55	else if (a->tv_sec < b->tv_sec)
	56	return -1;
	57	else if (a->tv_nsec > b->tv_nsec)
	58	return 1;
	59	else if (a->tv_nsec < b->tv_nsec)
	60	return -1;
	61	else
	62	return 0;
	63	}
	64
	65	int main(int argc, char *argv[])
	66	{
	67	struct timespec min_ts, max_ts, vm_ts;
	68
	69	if (!(kvm_check_cap(KVM_CAP_XEN_HVM) &
	70	KVM_XEN_HVM_CONFIG_SHARED_INFO) ) {
	71	print_skip("KVM_XEN_HVM_CONFIG_SHARED_INFO not available");
	72	exit(KSFT_SKIP);
	73	}
	74
	75	clock_gettime(CLOCK_REALTIME, &min_ts);
	76
	77	vm = vm_create_default(VCPU_ID, 0, (void *) guest_code);
	78	vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid());
	79
	80	/* Map a region for the shared_info page */
	81	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
	82	SHINFO_REGION_GPA, SHINFO_REGION_SLOT, 2, 0);
	83
	84	struct kvm_xen_hvm_config hvmc = {
	85	.flags = KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL,
	86	.msr = XEN_HYPERCALL_MSR,
	87	};
	88	vm_ioctl(vm, KVM_XEN_HVM_CONFIG, &hvmc);
	89
	90	struct kvm_xen_hvm_attr lm = {
	91	.type = KVM_XEN_ATTR_TYPE_LONG_MODE,
	92	.u.long_mode = 1,
	93	};
	94	vm_ioctl(vm, KVM_XEN_HVM_SET_ATTR, &lm);
	95
	96	struct kvm_xen_hvm_attr ha = {
	97	.type = KVM_XEN_ATTR_TYPE_SHARED_INFO,
	98	.u.shared_info.gfn = SHINFO_REGION_GPA / PAGE_SIZE,
	99	};
	100	vm_ioctl(vm, KVM_XEN_HVM_SET_ATTR, &ha);
	101
	102	struct kvm_xen_vcpu_attr vi = {
	103	.type = KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO,
	104	.u.gpa = SHINFO_REGION_GPA + 0x40,
	105	};
	106	vcpu_ioctl(vm, VCPU_ID, KVM_XEN_VCPU_SET_ATTR, &vi);
	107
	108	struct kvm_xen_vcpu_attr pvclock = {
	109	.type = KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO,
	110	.u.gpa = PVTIME_ADDR,
	111	};
	112	vcpu_ioctl(vm, VCPU_ID, KVM_XEN_VCPU_SET_ATTR, &pvclock);
	113
	114	for (;;) {
	115	volatile struct kvm_run *run = vcpu_state(vm, VCPU_ID);
	116	struct ucall uc;
	117
	118	vcpu_run(vm, VCPU_ID);
	119
	120	TEST_ASSERT(run->exit_reason == KVM_EXIT_IO,
	121	"Got exit_reason other than KVM_EXIT_IO: %u (%s)\n",
	122	run->exit_reason,
	123	exit_reason_str(run->exit_reason));
	124
	125	switch (get_ucall(vm, VCPU_ID, &uc)) {
	126	case UCALL_ABORT:
	127	TEST_FAIL("%s", (const char *)uc.args[0]);
	128	/* NOT REACHED */
	129	case UCALL_SYNC:
	130	break;
	131	case UCALL_DONE:
	132	goto done;
	133	default:
	134	TEST_FAIL("Unknown ucall 0x%lx.", uc.cmd);
	135	}
	136	}
	137
	138	done:
	139	clock_gettime(CLOCK_REALTIME, &max_ts);
	140
	141	/*
	142	* Just a really basic check that things are being put in the
	143	* right place. The actual calculations are much the same for
	144	* Xen as they are for the KVM variants, so no need to check.
	145	*/
	146	struct pvclock_wall_clock *wc;
	147	struct pvclock_vcpu_time_info ti, ti2;
	148
	149	wc = addr_gpa2hva(vm, SHINFO_REGION_GPA + 0xc00);
	150	ti = addr_gpa2hva(vm, SHINFO_REGION_GPA + 0x40 + 0x20);
	151	ti2 = addr_gpa2hva(vm, PVTIME_ADDR);
	152
	153	vm_ts.tv_sec = wc->sec;
	154	vm_ts.tv_nsec = wc->nsec;
	155	TEST_ASSERT(wc->version && !(wc->version & 1),
	156	"Bad wallclock version %x", wc->version);
	157	TEST_ASSERT(cmp_timespec(&min_ts, &vm_ts) <= 0, "VM time too old");
	158	TEST_ASSERT(cmp_timespec(&max_ts, &vm_ts) >= 0, "VM time too new");
	159
	160	TEST_ASSERT(ti->version && !(ti->version & 1),
	161	"Bad time_info version %x", ti->version);
	162	TEST_ASSERT(ti2->version && !(ti2->version & 1),
	163	"Bad time_info version %x", ti->version);
	164
	165	kvm_vm_free(vm);
	166	return 0;
	167	}

tools/testing/selftests/kvm/x86_64/xen_vmcall_test.c+149−0

@@ -0,0 +1,149 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* xen_vmcall_test
	4	*
	5	* Copyright © 2020 Amazon.com, Inc. or its affiliates.
	6	*
	7	* Userspace hypercall testing
	8	*/
	9
	10	#include "test_util.h"
	11	#include "kvm_util.h"
	12	#include "processor.h"
	13
	14	#define VCPU_ID 5
	15
	16	#define HCALL_REGION_GPA 0xc0000000ULL
	17	#define HCALL_REGION_SLOT 10
	18	#define PAGE_SIZE 4096
	19
	20	static struct kvm_vm *vm;
	21
	22	#define INPUTVALUE 17
	23	#define ARGVALUE(x) (0xdeadbeef5a5a0000UL + x)
	24	#define RETVALUE 0xcafef00dfbfbffffUL
	25
	26	#define XEN_HYPERCALL_MSR 0x40000200
	27	#define HV_GUEST_OS_ID_MSR 0x40000000
	28	#define HV_HYPERCALL_MSR 0x40000001
	29
	30	#define HVCALL_SIGNAL_EVENT 0x005d
	31	#define HV_STATUS_INVALID_ALIGNMENT 4
	32
	33	static void guest_code(void)
	34	{
	35	unsigned long rax = INPUTVALUE;
	36	unsigned long rdi = ARGVALUE(1);
	37	unsigned long rsi = ARGVALUE(2);
	38	unsigned long rdx = ARGVALUE(3);
	39	unsigned long rcx;
	40	register unsigned long r10 __asm__("r10") = ARGVALUE(4);
	41	register unsigned long r8 __asm__("r8") = ARGVALUE(5);
	42	register unsigned long r9 __asm__("r9") = ARGVALUE(6);
	43
	44	/* First a direct invocation of 'vmcall' */
	45	__asm__ __volatile__("vmcall" :
	46	"=a"(rax) :
	47	"a"(rax), "D"(rdi), "S"(rsi), "d"(rdx),
	48	"r"(r10), "r"(r8), "r"(r9));
	49	GUEST_ASSERT(rax == RETVALUE);
	50
	51	/* Fill in the Xen hypercall page */
	52	__asm__ __volatile__("wrmsr" : : "c" (XEN_HYPERCALL_MSR),
	53	"a" (HCALL_REGION_GPA & 0xffffffff),
	54	"d" (HCALL_REGION_GPA >> 32));
	55
	56	/* Set Hyper-V Guest OS ID */
	57	__asm__ __volatile__("wrmsr" : : "c" (HV_GUEST_OS_ID_MSR),
	58	"a" (0x5a), "d" (0));
	59
	60	/* Hyper-V hypercall page */
	61	u64 msrval = HCALL_REGION_GPA + PAGE_SIZE + 1;
	62	__asm__ __volatile__("wrmsr" : : "c" (HV_HYPERCALL_MSR),
	63	"a" (msrval & 0xffffffff),
	64	"d" (msrval >> 32));
	65
	66	/* Invoke a Xen hypercall */
	67	__asm__ __volatile__("call *%1" : "=a"(rax) :
	68	"r"(HCALL_REGION_GPA + INPUTVALUE * 32),
	69	"a"(rax), "D"(rdi), "S"(rsi), "d"(rdx),
	70	"r"(r10), "r"(r8), "r"(r9));
	71	GUEST_ASSERT(rax == RETVALUE);
	72
	73	/* Invoke a Hyper-V hypercall */
	74	rax = 0;
	75	rcx = HVCALL_SIGNAL_EVENT; /* code */
	76	rdx = 0x5a5a5a5a; /* ingpa (badly aligned) */
	77	__asm__ __volatile__("call *%1" : "=a"(rax) :
	78	"r"(HCALL_REGION_GPA + PAGE_SIZE),
	79	"a"(rax), "c"(rcx), "d"(rdx),
	80	"r"(r8));
	81	GUEST_ASSERT(rax == HV_STATUS_INVALID_ALIGNMENT);
	82
	83	GUEST_DONE();
	84	}
	85
	86	int main(int argc, char *argv[])
	87	{
	88	if (!(kvm_check_cap(KVM_CAP_XEN_HVM) &
	89	KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL) ) {
	90	print_skip("KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL not available");
	91	exit(KSFT_SKIP);
	92	}
	93
	94	vm = vm_create_default(VCPU_ID, 0, (void *) guest_code);
	95	vcpu_set_hv_cpuid(vm, VCPU_ID);
	96
	97	struct kvm_xen_hvm_config hvmc = {
	98	.flags = KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL,
	99	.msr = XEN_HYPERCALL_MSR,
	100	};
	101	vm_ioctl(vm, KVM_XEN_HVM_CONFIG, &hvmc);
	102
	103	/* Map a region for the hypercall pages */
	104	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
	105	HCALL_REGION_GPA, HCALL_REGION_SLOT, 2, 0);
	106	virt_map(vm, HCALL_REGION_GPA, HCALL_REGION_GPA, 2, 0);
	107
	108	for (;;) {
	109	volatile struct kvm_run *run = vcpu_state(vm, VCPU_ID);
	110	struct ucall uc;
	111
	112	vcpu_run(vm, VCPU_ID);
	113
	114	if (run->exit_reason == KVM_EXIT_XEN) {
	115	ASSERT_EQ(run->xen.type, KVM_EXIT_XEN_HCALL);
	116	ASSERT_EQ(run->xen.u.hcall.cpl, 0);
	117	ASSERT_EQ(run->xen.u.hcall.longmode, 1);
	118	ASSERT_EQ(run->xen.u.hcall.input, INPUTVALUE);
	119	ASSERT_EQ(run->xen.u.hcall.params[0], ARGVALUE(1));
	120	ASSERT_EQ(run->xen.u.hcall.params[1], ARGVALUE(2));
	121	ASSERT_EQ(run->xen.u.hcall.params[2], ARGVALUE(3));
	122	ASSERT_EQ(run->xen.u.hcall.params[3], ARGVALUE(4));
	123	ASSERT_EQ(run->xen.u.hcall.params[4], ARGVALUE(5));
	124	ASSERT_EQ(run->xen.u.hcall.params[5], ARGVALUE(6));
	125	run->xen.u.hcall.result = RETVALUE;
	126	continue;
	127	}
	128
	129	TEST_ASSERT(run->exit_reason == KVM_EXIT_IO,
	130	"Got exit_reason other than KVM_EXIT_IO: %u (%s)\n",
	131	run->exit_reason,
	132	exit_reason_str(run->exit_reason));
	133
	134	switch (get_ucall(vm, VCPU_ID, &uc)) {
	135	case UCALL_ABORT:
	136	TEST_FAIL("%s", (const char *)uc.args[0]);
	137	/* NOT REACHED */
	138	case UCALL_SYNC:
	139	break;
	140	case UCALL_DONE:
	141	goto done;
	142	default:
	143	TEST_FAIL("Unknown ucall 0x%lx.", uc.cmd);
	144	}
	145	}
	146	done:
	147	kvm_vm_free(vm);
	148	return 0;
	149	}

virt/kvm/dirty_ring.c+4−4

 #include <linux/vmalloc.h>
 #include <linux/kvm_dirty_ring.h>
 #include <trace/events/kvm.h>
+#include "mmu_lock.h"
 int __weak kvm_cpu_dirty_log_size(void)
 {
 	if (!memslot || (offset + __fls(mask)) >= memslot->npages)
 		return;
-	spin_lock(&kvm->mmu_lock);
+	KVM_MMU_LOCK(kvm);
 	kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, offset, mask);
-	spin_unlock(&kvm->mmu_lock);
+	KVM_MMU_UNLOCK(kvm);
 }
 int kvm_dirty_ring_alloc(struct kvm_dirty_ring *ring, int index, u32 size)
 {
-	ring->dirty_gfns = vmalloc(size);
+	ring->dirty_gfns = vzalloc(size);
 	if (!ring->dirty_gfns)
 		return -ENOMEM;
-	memset(ring->dirty_gfns, 0, size);
 	ring->size = size / sizeof(struct kvm_dirty_gfn);
 	ring->soft_limit = ring->size - kvm_dirty_ring_get_rsvd_entries();

virt/kvm/kvm_main.c+33−21

@@ -58,6 +58,7 @@ · 57 unmodified lines
58		58
59	#include "coalesced_mmio.h"	59	#include "coalesced_mmio.h"
60	#include "async_pf.h"	60	#include "async_pf.h"
		61	#include "mmu_lock.h"
61	#include "vfio.h"	62	#include "vfio.h"
62		63
63	#define CREATE_TRACE_POINTS	64	#define CREATE_TRACE_POINTS
@@ -459,13 +460,15 @@ static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, · 395 unmodified lines
459	int idx;	460	int idx;
460		461
461	idx = srcu_read_lock(&kvm->srcu);	462	idx = srcu_read_lock(&kvm->srcu);
462	spin_lock(&kvm->mmu_lock);	463
		464	KVM_MMU_LOCK(kvm);
		465
463	kvm->mmu_notifier_seq++;	466	kvm->mmu_notifier_seq++;
464		467
465	if (kvm_set_spte_hva(kvm, address, pte))	468	if (kvm_set_spte_hva(kvm, address, pte))
466	kvm_flush_remote_tlbs(kvm);	469	kvm_flush_remote_tlbs(kvm);
467		470
468	spin_unlock(&kvm->mmu_lock);	471	KVM_MMU_UNLOCK(kvm);
469	srcu_read_unlock(&kvm->srcu, idx);	472	srcu_read_unlock(&kvm->srcu, idx);
470	}	473	}
471		474
@@ -476,7 +479,7 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, · 4 unmodified lines
476	int need_tlb_flush = 0, idx;	479	int need_tlb_flush = 0, idx;
477		480
478	idx = srcu_read_lock(&kvm->srcu);	481	idx = srcu_read_lock(&kvm->srcu);
479	spin_lock(&kvm->mmu_lock);	482	KVM_MMU_LOCK(kvm);
480	/*	483	/*
481	* The count increase must become visible at unlock time as no	484	* The count increase must become visible at unlock time as no
482	* spte can be established without taking the mmu_lock and	485	* spte can be established without taking the mmu_lock and
@@ -489,7 +492,7 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, · 6 unmodified lines
489	if (need_tlb_flush \|\| kvm->tlbs_dirty)	492	if (need_tlb_flush \|\| kvm->tlbs_dirty)
490	kvm_flush_remote_tlbs(kvm);	493	kvm_flush_remote_tlbs(kvm);
491		494
492	spin_unlock(&kvm->mmu_lock);	495	KVM_MMU_UNLOCK(kvm);
493	srcu_read_unlock(&kvm->srcu, idx);	496	srcu_read_unlock(&kvm->srcu, idx);
494		497
495	return 0;	498	return 0;
@@ -500,7 +503,7 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, · 4 unmodified lines
500	{	503	{
501	struct kvm *kvm = mmu_notifier_to_kvm(mn);	504	struct kvm *kvm = mmu_notifier_to_kvm(mn);
502		505
503	spin_lock(&kvm->mmu_lock);	506	KVM_MMU_LOCK(kvm);
504	/*	507	/*
505	* This sequence increase will notify the kvm page fault that	508	* This sequence increase will notify the kvm page fault that
506	* the page that is going to be mapped in the spte could have	509	* the page that is going to be mapped in the spte could have
@@ -514,7 +517,7 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, · 7 unmodified lines
514	* in conjunction with the smp_rmb in mmu_notifier_retry().	517	* in conjunction with the smp_rmb in mmu_notifier_retry().
515	*/	518	*/
516	kvm->mmu_notifier_count--;	519	kvm->mmu_notifier_count--;
517	spin_unlock(&kvm->mmu_lock);	520	KVM_MMU_UNLOCK(kvm);
518		521
519	BUG_ON(kvm->mmu_notifier_count < 0);	522	BUG_ON(kvm->mmu_notifier_count < 0);
520	}	523	}
@@ -528,13 +531,13 @@ static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, · 7 unmodified lines
528	int young, idx;	531	int young, idx;
529		532
530	idx = srcu_read_lock(&kvm->srcu);	533	idx = srcu_read_lock(&kvm->srcu);
531	spin_lock(&kvm->mmu_lock);	534	KVM_MMU_LOCK(kvm);
532		535
533	young = kvm_age_hva(kvm, start, end);	536	young = kvm_age_hva(kvm, start, end);
534	if (young)	537	if (young)
535	kvm_flush_remote_tlbs(kvm);	538	kvm_flush_remote_tlbs(kvm);
536		539
537	spin_unlock(&kvm->mmu_lock);	540	KVM_MMU_UNLOCK(kvm);
538	srcu_read_unlock(&kvm->srcu, idx);	541	srcu_read_unlock(&kvm->srcu, idx);
539		542
540	return young;	543	return young;
@@ -549,7 +552,7 @@ static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn, · 8 unmodified lines
549	int young, idx;	552	int young, idx;
550		553
551	idx = srcu_read_lock(&kvm->srcu);	554	idx = srcu_read_lock(&kvm->srcu);
552	spin_lock(&kvm->mmu_lock);	555	KVM_MMU_LOCK(kvm);
553	/*	556	/*
554	* Even though we do not flush TLB, this will still adversely	557	* Even though we do not flush TLB, this will still adversely
555	* affect performance on pre-Haswell Intel EPT, where there is	558	* affect performance on pre-Haswell Intel EPT, where there is
@@ -564,7 +567,7 @@ static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn, · 8 unmodified lines
564	* more sophisticated heuristic later.	567	* more sophisticated heuristic later.
565	*/	568	*/
566	young = kvm_age_hva(kvm, start, end);	569	young = kvm_age_hva(kvm, start, end);
567	spin_unlock(&kvm->mmu_lock);	570	KVM_MMU_UNLOCK(kvm);
568	srcu_read_unlock(&kvm->srcu, idx);	571	srcu_read_unlock(&kvm->srcu, idx);
569		572
570	return young;	573	return young;
@@ -578,9 +581,9 @@ static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, · 7 unmodified lines
578	int young, idx;	581	int young, idx;
579		582
580	idx = srcu_read_lock(&kvm->srcu);	583	idx = srcu_read_lock(&kvm->srcu);
581	spin_lock(&kvm->mmu_lock);	584	KVM_MMU_LOCK(kvm);
582	young = kvm_test_age_hva(kvm, address);	585	young = kvm_test_age_hva(kvm, address);
583	spin_unlock(&kvm->mmu_lock);	586	KVM_MMU_UNLOCK(kvm);
584	srcu_read_unlock(&kvm->srcu, idx);	587	srcu_read_unlock(&kvm->srcu, idx);
585		588
586	return young;	589	return young;
@@ -745,7 +748,7 @@ static struct kvm *kvm_create_vm(unsigned long type) · 158 unmodified lines
745	if (!kvm)	748	if (!kvm)
746	return ERR_PTR(-ENOMEM);	749	return ERR_PTR(-ENOMEM);
747		750
748	spin_lock_init(&kvm->mmu_lock);	751	KVM_MMU_LOCK_INIT(kvm);
749	mmgrab(current->mm);	752	mmgrab(current->mm);
750	kvm->mm = current->mm;	753	kvm->mm = current->mm;
751	kvm_eventfd_init(kvm);	754	kvm_eventfd_init(kvm);
@@ -1525,7 +1528,7 @@ static int kvm_get_dirty_log_protect(struct kvm kvm, struct kvm_dirty_log log) · 773 unmodified lines
1525	dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);	1528	dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
1526	memset(dirty_bitmap_buffer, 0, n);	1529	memset(dirty_bitmap_buffer, 0, n);
1527		1530
1528	spin_lock(&kvm->mmu_lock);	1531	KVM_MMU_LOCK(kvm);
1529	for (i = 0; i < n / sizeof(long); i++) {	1532	for (i = 0; i < n / sizeof(long); i++) {
1530	unsigned long mask;	1533	unsigned long mask;
1531	gfn_t offset;	1534	gfn_t offset;
@@ -1541,7 +1544,7 @@ static int kvm_get_dirty_log_protect(struct kvm kvm, struct kvm_dirty_log log) · 9 unmodified lines
1541	kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,	1544	kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
1542	offset, mask);	1545	offset, mask);
1543	}	1546	}
1544	spin_unlock(&kvm->mmu_lock);	1547	KVM_MMU_UNLOCK(kvm);
1545	}	1548	}
1546		1549
1547	if (flush)	1550	if (flush)
@@ -1636,7 +1639,7 @@ static int kvm_clear_dirty_log_protect(struct kvm *kvm, · 88 unmodified lines
1636	if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n))	1639	if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n))
1637	return -EFAULT;	1640	return -EFAULT;
1638		1641
1639	spin_lock(&kvm->mmu_lock);	1642	KVM_MMU_LOCK(kvm);
1640	for (offset = log->first_page, i = offset / BITS_PER_LONG,	1643	for (offset = log->first_page, i = offset / BITS_PER_LONG,
1641	n = DIV_ROUND_UP(log->num_pages, BITS_PER_LONG); n--;	1644	n = DIV_ROUND_UP(log->num_pages, BITS_PER_LONG); n--;
1642	i++, offset += BITS_PER_LONG) {	1645	i++, offset += BITS_PER_LONG) {
@@ -1659,7 +1662,7 @@ static int kvm_clear_dirty_log_protect(struct kvm *kvm, · 16 unmodified lines
1659	offset, mask);	1662	offset, mask);
1660	}	1663	}
1661	}	1664	}
1662	spin_unlock(&kvm->mmu_lock);	1665	KVM_MMU_UNLOCK(kvm);
1663		1666
1664	if (flush)	1667	if (flush)
1665	kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);	1668	kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);
@@ -1903,10 +1906,12 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, · 237 unmodified lines
1903	bool write_fault, bool *writable,	1906	bool write_fault, bool *writable,
1904	kvm_pfn_t *p_pfn)	1907	kvm_pfn_t *p_pfn)
1905	{	1908	{
1906	unsigned long pfn;	1909	kvm_pfn_t pfn;
		1910	pte_t *ptep;
		1911	spinlock_t *ptl;
1907	int r;	1912	int r;
1908		1913
1909	r = follow_pfn(vma, addr, &pfn);	1914	r = follow_pte(vma->vm_mm, addr, &ptep, &ptl);
1910	if (r) {	1915	if (r) {
1911	/*	1916	/*
1912	* get_user_pages fails for VM_IO and VM_PFNMAP vmas and does	1917	* get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
@@ -1921,14 +1926,19 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, · 8 unmodified lines
1921	if (r)	1926	if (r)
1922	return r;	1927	return r;
1923		1928
1924	r = follow_pfn(vma, addr, &pfn);	1929	r = follow_pte(vma->vm_mm, addr, &ptep, &ptl);
1925	if (r)	1930	if (r)
1926	return r;	1931	return r;
		1932	}
1927		1933
		1934	if (write_fault && !pte_write(*ptep)) {
		1935	pfn = KVM_PFN_ERR_RO_FAULT;
		1936	goto out;
1928	}	1937	}
1929		1938
1930	if (writable)	1939	if (writable)
1931	*writable = true;	1940	writable = pte_write(ptep);
		1941	pfn = pte_pfn(*ptep);
1932		1942
1933	/*	1943	/*
1934	* Get a reference here because callers of hva_to_pfn and	1944	* Get a reference here because callers of hva_to_pfn and
@@ -1943,6 +1953,8 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, · 8 unmodified lines
1943	*/	1953	*/
1944	kvm_get_pfn(pfn);	1954	kvm_get_pfn(pfn);
1945		1955
		1956	out:
		1957	pte_unmap_unlock(ptep, ptl);
1946	*p_pfn = pfn;	1958	*p_pfn = pfn;
1947	return 0;	1959	return 0;
1948	}	1960	}

virt/kvm/mmu_lock.h+23−0

@@ -0,0 +1,23 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2
	3	#ifndef KVM_MMU_LOCK_H
	4	#define KVM_MMU_LOCK_H 1
	5
	6	/*
	7	* Architectures can choose whether to use an rwlock or spinlock
	8	* for the mmu_lock. These macros, for use in common code
	9	* only, avoids using #ifdefs in places that must deal with
	10	* multiple architectures.
	11	*/
	12
	13	#ifdef KVM_HAVE_MMU_RWLOCK
	14	#define KVM_MMU_LOCK_INIT(kvm) rwlock_init(&(kvm)->mmu_lock)
	15	#define KVM_MMU_LOCK(kvm) write_lock(&(kvm)->mmu_lock)
	16	#define KVM_MMU_UNLOCK(kvm) write_unlock(&(kvm)->mmu_lock)
	17	#else
	18	#define KVM_MMU_LOCK_INIT(kvm) spin_lock_init(&(kvm)->mmu_lock)
	19	#define KVM_MMU_LOCK(kvm) spin_lock(&(kvm)->mmu_lock)
	20	#define KVM_MMU_UNLOCK(kvm) spin_unlock(&(kvm)->mmu_lock)
	21	#endif /* KVM_HAVE_MMU_RWLOCK */
	22
	23	#endif