Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

Pull KVM updates from Paolo Bonzini: "x86: - Support for userspace to emulate Xen hypercalls - Raise the maximum number of user memslots - Scalability improvements for the new MMU. Instead of the complex "fast page fault" logic that is used in mmu.c, tdp_mmu.c uses an rwlock so that page faults are concurrent, but the code that can run against page faults is limited. Right now only page faults take the lock for reading; in the future this will be extended to some cases of page table destruction. I hope to switch the default MMU around 5.12-rc3 (some testing was delayed due to Chinese New Year). - Cleanups for MAXPHYADDR checks - Use static calls for vendor-specific callbacks - On AMD, use VMLOAD/VMSAVE to save and restore host state - Stop using deprecated jump label APIs - Workaround for AMD erratum that made nested virtualization unreliable - Support for LBR emulation in the guest - Support for communicating bus lock vmexits to userspace - Add support for SEV attestation command - Miscellaneous cleanups PPC: - Support for second data watchpoint on POWER10 - Remove some complex workarounds for buggy early versions of POWER9 - Guest entry/exit fixes ARM64: - Make the nVHE EL2 object relocatable - Cleanups for concurrent translation faults hitting the same page - Support for the standard TRNG hypervisor call - A bunch of small PMU/Debug fixes - Simplification of the early init hypercall handling Non-KVM changes (with acks): - Detection of contended rwlocks (implemented only for qrwlocks, because KVM only needs it for x86) - Allow __DISABLE_EXPORTS from assembly code - Provide a saner follow_pfn replacements for modules" * tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (192 commits) KVM: x86/xen: Explicitly pad struct compat_vcpu_info to 64 bytes KVM: selftests: Don't bother mapping GVA for Xen shinfo test KVM: selftests: Fix hex vs. decimal snafu in Xen test KVM: selftests: Fix size of memslots created by Xen tests KVM: selftests: Ignore recently added Xen tests' build output KVM: selftests: Add missing header file needed by xAPIC IPI tests KVM: selftests: Add operand to vmsave/vmload/vmrun in svm.c KVM: SVM: Make symbol 'svm_gp_erratum_intercept' static locking/arch: Move qrwlock.h include after qspinlock.h KVM: PPC: Book3S HV: Fix host radix SLB optimisation with hash guests KVM: PPC: Book3S HV: Ensure radix guest has no SLB entries KVM: PPC: Don't always report hash MMU capability for P9 < DD2.2 KVM: PPC: Book3S HV: Save and restore FSCR in the P9 path KVM: PPC: remove unneeded semicolon KVM: PPC: Book3S HV: Use POWER9 SLBIA IH=6 variant to clear SLB KVM: PPC: Book3S HV: No need to clear radix host SLB before loading HPT guest KVM: PPC: Book3S HV: Fix radix guest SLB side channel KVM: PPC: Book3S HV: Remove support for running HPT guest on RPT host without mixed mode support KVM: PPC: Book3S HV: Introduce new capability for 2nd DAWR KVM: PPC: Book3S HV: Add infrastructure to support 2nd DAWR ...

Linus Torvalds committed 5 years ago

Commit 3e105852 parents 9c5b80b 8c6e67b150 files · +6655 −2060Browse files

Unified · Split

150 files

Documentation/virt/kvm/amd-memory-encryption.rst +21 −0
Documentation/virt/kvm/api.rst +220 −8
Documentation/virt/kvm/locking.rst +8 −1
arch/arm64/include/asm/hyp_image.h +27 −2
arch/arm64/include/asm/kvm_asm.h +0 −20
arch/arm64/include/asm/kvm_host.h +2 −1
arch/arm64/include/asm/kvm_mmu.h +17 −44
arch/arm64/include/asm/kvm_pgtable.h +5 −0
arch/arm64/include/asm/sections.h +2 −1
arch/arm64/include/asm/spinlock.h +1 −1
arch/arm64/include/asm/sysreg.h +3 −0
arch/arm64/kernel/image-vars.h +0 −1
arch/arm64/kernel/smp.c +3 −1
arch/arm64/kernel/vmlinux.lds.S +15 −3
arch/arm64/kvm/Makefile +1 −1
arch/arm64/kvm/arm.c +3 −4
arch/arm64/kvm/hyp/include/hyp/switch.h +2 −2
arch/arm64/kvm/hyp/nvhe/.gitignore +2 −0
arch/arm64/kvm/hyp/nvhe/Makefile +28 −5
arch/arm64/kvm/hyp/nvhe/gen-hyprel.c +438 −0
arch/arm64/kvm/hyp/nvhe/host.S +15 −14
arch/arm64/kvm/hyp/nvhe/hyp-init.S +5 −14
arch/arm64/kvm/hyp/nvhe/hyp-main.c +4 −7
arch/arm64/kvm/hyp/nvhe/hyp-smp.c +2 −2
arch/arm64/kvm/hyp/nvhe/hyp.lds.S +6 −3
arch/arm64/kvm/hyp/nvhe/psci-relay.c +12 −12
arch/arm64/kvm/hyp/pgtable.c +47 −36
arch/arm64/kvm/hyp/vgic-v2-cpuif-proxy.c +1 −1
arch/arm64/kvm/hypercalls.c +6 −0
arch/arm64/kvm/mmu.c +8 −5
arch/arm64/kvm/pmu-emul.c +10 −4
arch/arm64/kvm/sys_regs.c +51 −34
arch/arm64/kvm/trng.c +85 −0
arch/arm64/kvm/va_layout.c +28 −6
arch/mips/include/asm/kvm_host.h +0 −1
arch/mips/include/asm/spinlock.h +1 −1
arch/powerpc/include/asm/hvcall.h +22 −3
arch/powerpc/include/asm/kvm_book3s_asm.h +0 −11
arch/powerpc/include/asm/kvm_host.h +5 −3
arch/powerpc/include/asm/kvm_ppc.h +2 −0
arch/powerpc/include/uapi/asm/kvm.h +2 −0
arch/powerpc/kernel/asm-offsets.c +4 −5
arch/powerpc/kvm/book3s_hv.c +93 −56
arch/powerpc/kvm/book3s_hv_builtin.c +1 −107
arch/powerpc/kvm/book3s_hv_nested.c +56 −14
arch/powerpc/kvm/book3s_hv_rmhandlers.S +95 −80
arch/powerpc/kvm/booke.c +1 −1
arch/powerpc/kvm/powerpc.c +12 −2
arch/s390/include/asm/kvm_host.h +0 −1
arch/s390/pci/pci_mmio.c +2 −2
arch/sparc/include/asm/spinlock_64.h +1 −1
arch/x86/include/asm/cpufeatures.h +2 −0
arch/x86/include/asm/kvm-x86-ops.h +127 −0
arch/x86/include/asm/kvm_host.h +74 −15
arch/x86/include/asm/virtext.h +19 −6
arch/x86/include/asm/vmx.h +1 −0
arch/x86/include/asm/vmxfeatures.h +1 −0
arch/x86/include/asm/xen/interface.h +3 −0
arch/x86/include/uapi/asm/kvm.h +1 −0
arch/x86/include/uapi/asm/vmx.h +3 −1
arch/x86/kernel/apic/apic.c +1 −0
arch/x86/kernel/reboot.c +10 −20
arch/x86/kvm/Makefile +3 −2
arch/x86/kvm/cpuid.c +20 −4
arch/x86/kvm/cpuid.h +18 −6
arch/x86/kvm/emulate.c +7 −7
arch/x86/kvm/hyperv.c +209 −134
arch/x86/kvm/hyperv.h +33 −21
arch/x86/kvm/irq.c +8 −2
arch/x86/kvm/kvm_cache_regs.h +5 −5
arch/x86/kvm/kvm_emulate.h +1 −1
arch/x86/kvm/lapic.c +27 −33
arch/x86/kvm/lapic.h +8 −12
arch/x86/kvm/mmu.h +4 −4
arch/x86/kvm/mmu/mmu.c +159 −194
arch/x86/kvm/mmu/mmu_audit.c +4 −4
arch/x86/kvm/mmu/mmu_internal.h +6 −1
arch/x86/kvm/mmu/page_track.c +4 −4
arch/x86/kvm/mmu/paging_tmpl.h +4 −4
arch/x86/kvm/mmu/spte.c +1 −1
arch/x86/kvm/mmu/spte.h +24 −9
arch/x86/kvm/mmu/tdp_iter.c +15 −31
arch/x86/kvm/mmu/tdp_iter.h +14 −7
arch/x86/kvm/mmu/tdp_mmu.c +422 −132
arch/x86/kvm/mmu/tdp_mmu.h +28 −4
arch/x86/kvm/mtrr.c +6 −6
arch/x86/kvm/pmu.c +8 −2
arch/x86/kvm/pmu.h +2 −0
arch/x86/kvm/svm/avic.c +19 −16
arch/x86/kvm/svm/nested.c +4 −4
arch/x86/kvm/svm/sev.c +74 −30
arch/x86/kvm/svm/svm.c +189 −114
arch/x86/kvm/svm/svm.h +5 −24
arch/x86/kvm/svm/svm_ops.h +69 −0
arch/x86/kvm/trace.h +38 −2
arch/x86/kvm/vmx/capabilities.h +27 −1
arch/x86/kvm/vmx/nested.c +40 −66
arch/x86/kvm/vmx/pmu_intel.c +285 −9
arch/x86/kvm/vmx/posted_intr.c +3 −3
arch/x86/kvm/vmx/vmenter.S +1 −1
arch/x86/kvm/vmx/vmx.c +172 −110
arch/x86/kvm/vmx/vmx.h +54 −2
arch/x86/kvm/x86.c +392 −326
arch/x86/kvm/x86.h +8 −4
arch/x86/kvm/xen.c +431 −0
arch/x86/kvm/xen.h +78 −0
arch/xtensa/include/asm/spinlock.h +1 −1
drivers/crypto/ccp/sev-dev.c +1 −0
drivers/gpu/drm/i915/gvt/kvmgt.c +6 −6
fs/dax.c +3 −2
include/asm-generic/export.h +1 −1
include/asm-generic/qrwlock.h +19 −6
include/linux/kvm_host.h +7 −3
include/linux/mm.h +4 −2
include/linux/psp-sev.h +17 −0
include/linux/rwlock.h +7 −0
include/linux/sched.h +29 −0
include/uapi/linux/kvm.h +74 −0
include/xen/interface/xen.h +3 −1
kernel/locking/qrwlock.c +0 −1
kernel/sched/core.c +40 −0
mm/memory.c +36 −5
tools/arch/powerpc/include/uapi/asm/kvm.h +2 −0
tools/include/uapi/linux/kvm.h +1 −0
tools/testing/selftests/kvm/.gitignore +6 −0
tools/testing/selftests/kvm/Makefile +6 −0
tools/testing/selftests/kvm/demand_paging_test.c +31 −12
tools/testing/selftests/kvm/dirty_log_perf_test.c +56 −36
tools/testing/selftests/kvm/include/kvm_util.h +0 −6
tools/testing/selftests/kvm/include/numaif.h +55 −0
tools/testing/selftests/kvm/include/perf_test_util.h +5 −2
tools/testing/selftests/kvm/include/test_util.h +15 −1
tools/testing/selftests/kvm/include/x86_64/processor.h +40 −1
tools/testing/selftests/kvm/lib/kvm_util.c +1 −0
tools/testing/selftests/kvm/lib/perf_test_util.c +21 −10
tools/testing/selftests/kvm/lib/test_util.c +30 −1
tools/testing/selftests/kvm/lib/x86_64/processor.c +144 −0
tools/testing/selftests/kvm/lib/x86_64/svm.c +4 −4
tools/testing/selftests/kvm/memslot_modification_stress_test.c +212 −0
tools/testing/selftests/kvm/settings +1 −0
tools/testing/selftests/kvm/x86_64/evmcs_test.c +2 −1
tools/testing/selftests/kvm/x86_64/get_cpuid_test.c +175 −0
tools/testing/selftests/kvm/x86_64/hyperv_cpuid.c +3 −28
tools/testing/selftests/kvm/x86_64/vmx_pmu_msrs_test.c +131 −0
tools/testing/selftests/kvm/x86_64/xapic_ipi_test.c +544 −0
tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c +167 −0
tools/testing/selftests/kvm/x86_64/xen_vmcall_test.c +149 −0
virt/kvm/dirty_ring.c +4 −4
virt/kvm/kvm_main.c +33 −21
virt/kvm/mmu_lock.h +23 −0

Documentation/virt/kvm/amd-memory-encryption.rst+21−0

@@ -263,6 +263,27 @@ Returns: 0 on success, -negative on error · 262 unmodified lines
263	263	__u32 trans_len;
264	264	};
265	265
	266	+10. KVM_SEV_GET_ATTESTATION_REPORT
	267	+----------------------------------
	268	+
	269	+The KVM_SEV_GET_ATTESTATION_REPORT command can be used by the hypervisor to query the attestation
	270	+report containing the SHA-256 digest of the guest memory and VMSA passed through the KVM_SEV_LAUNCH
	271	+commands and signed with the PEK. The digest returned by the command should match the digest
	272	+used by the guest owner with the KVM_SEV_LAUNCH_MEASURE.
	273	+
	274	+Parameters (in): struct kvm_sev_attestation
	275	+
	276	+Returns: 0 on success, -negative on error
	277	+
	278	+::
	279	+
	280	+ struct kvm_sev_attestation_report {
	281	+ __u8 mnonce[16]; /* A random mnonce that will be placed in the report */
	282	+
	283	+ __u64 uaddr; /* userspace address where the report should be copied */
	284	+ __u32 len;
	285	+ };
	286	+
266	287	References
267	288	==========
268	289

Documentation/virt/kvm/api.rst+220−8

@@ -960,6 +960,14 @@ memory. · 959 unmodified lines
960	960	__u8 pad2[30];
961	961	};
962	962
	963	+If the KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL flag is returned from the
	964	+KVM_CAP_XEN_HVM check, it may be set in the flags field of this ioctl.
	965	+This requests KVM to generate the contents of the hypercall page
	966	+automatically; hypercalls will be intercepted and passed to userspace
	967	+through KVM_EXIT_XEN. In this case, all of the blob size and address
	968	+fields must be zero.
	969	+
	970	+No other flags are currently valid in the struct kvm_xen_hvm_config.
963	971
964	972	4.29 KVM_GET_CLOCK
965	973	------------------
@@ -2268,6 +2276,8 @@ registers, find a list below: · 1302 unmodified lines
2268	2276	PPC KVM_REG_PPC_PSSCR 64
2269	2277	PPC KVM_REG_PPC_DEC_EXPIRY 64
2270	2278	PPC KVM_REG_PPC_PTCR 64
	2279	+ PPC KVM_REG_PPC_DAWR1 64
	2280	+ PPC KVM_REG_PPC_DAWRX1 64
2271	2281	PPC KVM_REG_PPC_TM_GPR0 64
2272	2282	...
2273	2283	PPC KVM_REG_PPC_TM_GPR31 64
@@ -4831,6 +4841,101 @@ into user space. · 2557 unmodified lines
4831	4841	If a vCPU is in running state while this ioctl is invoked, the vCPU may
4832	4842	experience inconsistent filtering behavior on MSR accesses.
4833	4843
	4844	+4.127 KVM_XEN_HVM_SET_ATTR
	4845	+--------------------------
	4846	+
	4847	+:Capability: KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_SHARED_INFO
	4848	+:Architectures: x86
	4849	+:Type: vm ioctl
	4850	+:Parameters: struct kvm_xen_hvm_attr
	4851	+:Returns: 0 on success, < 0 on error
	4852	+
	4853	+::
	4854	+
	4855	+ struct kvm_xen_hvm_attr {
	4856	+ __u16 type;
	4857	+ __u16 pad[3];
	4858	+ union {
	4859	+ __u8 long_mode;
	4860	+ __u8 vector;
	4861	+ struct {
	4862	+ __u64 gfn;
	4863	+ } shared_info;
	4864	+ __u64 pad[4];
	4865	+ } u;
	4866	+ };
	4867	+
	4868	+type values:
	4869	+
	4870	+KVM_XEN_ATTR_TYPE_LONG_MODE
	4871	+ Sets the ABI mode of the VM to 32-bit or 64-bit (long mode). This
	4872	+ determines the layout of the shared info pages exposed to the VM.
	4873	+
	4874	+KVM_XEN_ATTR_TYPE_SHARED_INFO
	4875	+ Sets the guest physical frame number at which the Xen "shared info"
	4876	+ page resides. Note that although Xen places vcpu_info for the first
	4877	+ 32 vCPUs in the shared_info page, KVM does not automatically do so
	4878	+ and instead requires that KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO be used
	4879	+ explicitly even when the vcpu_info for a given vCPU resides at the
	4880	+ "default" location in the shared_info page. This is because KVM is
	4881	+ not aware of the Xen CPU id which is used as the index into the
	4882	+ vcpu_info[] array, so cannot know the correct default location.
	4883	+
	4884	+KVM_XEN_ATTR_TYPE_UPCALL_VECTOR
	4885	+ Sets the exception vector used to deliver Xen event channel upcalls.
	4886	+
	4887	+4.128 KVM_XEN_HVM_GET_ATTR
	4888	+--------------------------
	4889	+
	4890	+:Capability: KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_SHARED_INFO
	4891	+:Architectures: x86
	4892	+:Type: vm ioctl
	4893	+:Parameters: struct kvm_xen_hvm_attr
	4894	+:Returns: 0 on success, < 0 on error
	4895	+
	4896	+Allows Xen VM attributes to be read. For the structure and types,
	4897	+see KVM_XEN_HVM_SET_ATTR above.
	4898	+
	4899	+4.129 KVM_XEN_VCPU_SET_ATTR
	4900	+---------------------------
	4901	+
	4902	+:Capability: KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_SHARED_INFO
	4903	+:Architectures: x86
	4904	+:Type: vcpu ioctl
	4905	+:Parameters: struct kvm_xen_vcpu_attr
	4906	+:Returns: 0 on success, < 0 on error
	4907	+
	4908	+::
	4909	+
	4910	+ struct kvm_xen_vcpu_attr {
	4911	+ __u16 type;
	4912	+ __u16 pad[3];
	4913	+ union {
	4914	+ __u64 gpa;
	4915	+ __u64 pad[4];
	4916	+ } u;
	4917	+ };
	4918	+
	4919	+type values:
	4920	+
	4921	+KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO
	4922	+ Sets the guest physical address of the vcpu_info for a given vCPU.
	4923	+
	4924	+KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO
	4925	+ Sets the guest physical address of an additional pvclock structure
	4926	+ for a given vCPU. This is typically used for guest vsyscall support.
	4927	+
	4928	+4.130 KVM_XEN_VCPU_GET_ATTR
	4929	+---------------------------
	4930	+
	4931	+:Capability: KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_SHARED_INFO
	4932	+:Architectures: x86
	4933	+:Type: vcpu ioctl
	4934	+:Parameters: struct kvm_xen_vcpu_attr
	4935	+:Returns: 0 on success, < 0 on error
	4936	+
	4937	+Allows Xen vCPU attributes to be read. For the structure and types,
	4938	+see KVM_XEN_VCPU_SET_ATTR above.
4834	4939
4835	4940	5. The kvm_run structure
4836	4941	========================
@@ -4893,9 +4998,11 @@ local APIC is not used. · 56 unmodified lines
4893	4998	__u16 flags;
4894	4999
4895	5000	More architecture-specific flags detailing state of the VCPU that may
4896		−affect the device's behavior. The only currently defined flag is
4897		−KVM_RUN_X86_SMM, which is valid on x86 machines and is set if the
4898		−VCPU is in system management mode.
	5001	+affect the device's behavior. Current defined flags:
	5002	+ /* x86, set if the VCPU is in system management mode */
	5003	+ #define KVM_RUN_X86_SMM (1 << 0)
	5004	+ /* x86, set if bus lock detected in VM */
	5005	+ #define KVM_RUN_BUS_LOCK (1 << 1)
4899	5006
4900	5007	::
4901	5008
@@ -4996,13 +5103,18 @@ to the byte array. · 94 unmodified lines
4996	5103
4997	5104	.. note::
4998	5105
4999		− For KVM_EXIT_IO, KVM_EXIT_MMIO, KVM_EXIT_OSI, KVM_EXIT_PAPR,
	5106	+ For KVM_EXIT_IO, KVM_EXIT_MMIO, KVM_EXIT_OSI, KVM_EXIT_PAPR, KVM_EXIT_XEN,
5000	5107	KVM_EXIT_EPR, KVM_EXIT_X86_RDMSR and KVM_EXIT_X86_WRMSR the corresponding
5001	5108	operations are complete (and guest state is consistent) only after userspace
5002	5109	has re-entered the kernel with KVM_RUN. The kernel side will first finish
5003		− incomplete operations and then check for pending signals. Userspace
5004		− can re-enter the guest with an unmasked signal pending to complete
5005		− pending operations.
	5110	+ incomplete operations and then check for pending signals.
	5111	+
	5112	+ The pending state of the operation is not preserved in state which is
	5113	+ visible to userspace, thus userspace should ensure that the operation is
	5114	+ completed before performing a live migration. Userspace can re-enter the
	5115	+ guest with an unmasked signal pending or with the immediate_exit field set
	5116	+ to complete pending operations without allowing any further instructions
	5117	+ to be executed.
5006	5118
5007	5119	::
5008	5120
@@ -5327,6 +5439,34 @@ wants to write. Once finished processing the event, user space must continue · 318 unmodified lines
5327	5439	vCPU execution. If the MSR write was unsuccessful, user space also sets the
5328	5440	"error" field to "1".
5329	5441
	5442	+::
	5443	+
	5444	+
	5445	+ struct kvm_xen_exit {
	5446	+ #define KVM_EXIT_XEN_HCALL 1
	5447	+ __u32 type;
	5448	+ union {
	5449	+ struct {
	5450	+ __u32 longmode;
	5451	+ __u32 cpl;
	5452	+ __u64 input;
	5453	+ __u64 result;
	5454	+ __u64 params[6];
	5455	+ } hcall;
	5456	+ } u;
	5457	+ };
	5458	+ /* KVM_EXIT_XEN */
	5459	+ struct kvm_hyperv_exit xen;
	5460	+
	5461	+Indicates that the VCPU exits into userspace to process some tasks
	5462	+related to Xen emulation.
	5463	+
	5464	+Valid values for 'type' are:
	5465	+
	5466	+ - KVM_EXIT_XEN_HCALL -- synchronously notify user-space about Xen hypercall.
	5467	+ Userspace is expected to place the hypercall result into the appropriate
	5468	+ field before invoking KVM_RUN again.
	5469	+
5330	5470	::
5331	5471
5332	5472	/* Fix the size of the union. */
@@ -6038,6 +6178,53 @@ KVM_EXIT_X86_RDMSR and KVM_EXIT_X86_WRMSR exit notifications which user space · 705 unmodified lines
6038	6178	can then handle to implement model specific MSR handling and/or user notifications
6039	6179	to inform a user that an MSR was not handled.
6040	6180
	6181	+7.22 KVM_CAP_X86_BUS_LOCK_EXIT
	6182	+-------------------------------
	6183	+
	6184	+:Architectures: x86
	6185	+:Target: VM
	6186	+:Parameters: args[0] defines the policy used when bus locks detected in guest
	6187	+:Returns: 0 on success, -EINVAL when args[0] contains invalid bits
	6188	+
	6189	+Valid bits in args[0] are::
	6190	+
	6191	+ #define KVM_BUS_LOCK_DETECTION_OFF (1 << 0)
	6192	+ #define KVM_BUS_LOCK_DETECTION_EXIT (1 << 1)
	6193	+
	6194	+Enabling this capability on a VM provides userspace with a way to select
	6195	+a policy to handle the bus locks detected in guest. Userspace can obtain
	6196	+the supported modes from the result of KVM_CHECK_EXTENSION and define it
	6197	+through the KVM_ENABLE_CAP.
	6198	+
	6199	+KVM_BUS_LOCK_DETECTION_OFF and KVM_BUS_LOCK_DETECTION_EXIT are supported
	6200	+currently and mutually exclusive with each other. More bits can be added in
	6201	+the future.
	6202	+
	6203	+With KVM_BUS_LOCK_DETECTION_OFF set, bus locks in guest will not cause vm exits
	6204	+so that no additional actions are needed. This is the default mode.
	6205	+
	6206	+With KVM_BUS_LOCK_DETECTION_EXIT set, vm exits happen when bus lock detected
	6207	+in VM. KVM just exits to userspace when handling them. Userspace can enforce
	6208	+its own throttling or other policy based mitigations.
	6209	+
	6210	+This capability is aimed to address the thread that VM can exploit bus locks to
	6211	+degree the performance of the whole system. Once the userspace enable this
	6212	+capability and select the KVM_BUS_LOCK_DETECTION_EXIT mode, KVM will set the
	6213	+KVM_RUN_BUS_LOCK flag in vcpu-run->flags field and exit to userspace. Concerning
	6214	+the bus lock vm exit can be preempted by a higher priority VM exit, the exit
	6215	+notifications to userspace can be KVM_EXIT_BUS_LOCK or other reasons.
	6216	+KVM_RUN_BUS_LOCK flag is used to distinguish between them.
	6217	+
	6218	+7.22 KVM_CAP_PPC_DAWR1
	6219	+----------------------
	6220	+
	6221	+:Architectures: ppc
	6222	+:Parameters: none
	6223	+:Returns: 0 on success, -EINVAL when CPU doesn't support 2nd DAWR
	6224	+
	6225	+This capability can be used to check / enable 2nd DAWR feature provided
	6226	+by POWER10 processor.
	6227	+
6041	6228	8. Other capabilities.
6042	6229	======================
6043	6230
@@ -6415,7 +6602,6 @@ guest according to the bits in the KVM_CPUID_FEATURES CPUID leaf · 371 unmodified lines
6415	6602	(0x40000001). Otherwise, a guest may use the paravirtual features
6416	6603	regardless of what has actually been exposed through the CPUID leaf.
6417	6604
6418		−
6419	6605	8.29 KVM_CAP_DIRTY_LOG_RING
6420	6606	---------------------------
6421	6607
@@ -6502,3 +6688,29 @@ KVM_GET_DIRTY_LOG and KVM_CLEAR_DIRTY_LOG. After enabling · 80 unmodified lines
6502	6688	KVM_CAP_DIRTY_LOG_RING with an acceptable dirty ring size, the virtual
6503	6689	machine will switch to ring-buffer dirty page tracking and further
6504	6690	KVM_GET_DIRTY_LOG or KVM_CLEAR_DIRTY_LOG ioctls will fail.
	6691	+
	6692	+8.30 KVM_CAP_XEN_HVM
	6693	+--------------------
	6694	+
	6695	+:Architectures: x86
	6696	+
	6697	+This capability indicates the features that Xen supports for hosting Xen
	6698	+PVHVM guests. Valid flags are::
	6699	+
	6700	+ #define KVM_XEN_HVM_CONFIG_HYPERCALL_MSR (1 << 0)
	6701	+ #define KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL (1 << 1)
	6702	+ #define KVM_XEN_HVM_CONFIG_SHARED_INFO (1 << 2)
	6703	+
	6704	+The KVM_XEN_HVM_CONFIG_HYPERCALL_MSR flag indicates that the KVM_XEN_HVM_CONFIG
	6705	+ioctl is available, for the guest to set its hypercall page.
	6706	+
	6707	+If KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL is also set, the same flag may also be
	6708	+provided in the flags to KVM_XEN_HVM_CONFIG, without providing hypercall page
	6709	+contents, to request that KVM generate hypercall page content automatically
	6710	+and also enable interception of guest hypercalls with KVM_EXIT_XEN.
	6711	+
	6712	+The KVM_XEN_HVM_CONFIG_SHARED_INFO flag indicates the availability of the
	6713	+KVM_XEN_HVM_SET_ATTR, KVM_XEN_HVM_GET_ATTR, KVM_XEN_VCPU_SET_ATTR and
	6714	+KVM_XEN_VCPU_GET_ATTR ioctls, as well as the delivery of exception vectors
	6715	+for event channel upcalls when the evtchn_upcall_pending field of a vcpu's
	6716	+vcpu_info is set.

Documentation/virt/kvm/locking.rst+8−1

@@ -16,7 +16,14 @@ The acquisition orders for mutexes are as follows: · 15 unmodified lines
16	16	- kvm->slots_lock is taken outside kvm->irq_lock, though acquiring
17	17	them together is quite rare.
18	18
19		−On x86, vcpu->mutex is taken outside kvm->arch.hyperv.hv_lock.
	19	+On x86:
	20	+
	21	+- vcpu->mutex is taken outside kvm->arch.hyperv.hv_lock
	22	+
	23	+- kvm->arch.mmu_lock is an rwlock. kvm->arch.tdp_mmu_pages_lock is
	24	+ taken inside kvm->arch.mmu_lock, and cannot be taken without already
	25	+ holding kvm->arch.mmu_lock (typically with ``read_lock``, otherwise
	26	+ there's no need to take kvm->arch.tdp_mmu_pages_lock at all).
20	27
21	28	Everything else is a leaf: no other lock is taken inside the critical
22	29	sections.

arch/arm64/include/asm/hyp_image.h+27−2

@@ -7,6 +7,9 @@ · 6 unmodified lines
7	7	#ifndef __ARM64_HYP_IMAGE_H__
8	8	#define __ARM64_HYP_IMAGE_H__
9	9
	10	+#define __HYP_CONCAT(a, b) a ## b
	11	+#define HYP_CONCAT(a, b) __HYP_CONCAT(a, b)
	12	+
10	13	/*
11	14	* KVM nVHE code has its own symbol namespace prefixed with __kvm_nvhe_,
12	15	* to separate it from the kernel proper.
@@ -21,9 +24,31 @@ · 8 unmodified lines
21	24	*/
22	25	#define HYP_SECTION_NAME(NAME) .hyp##NAME
23	26
	27	+/* Symbol defined at the beginning of each hyp section. */
	28	+#define HYP_SECTION_SYMBOL_NAME(NAME) \
	29	+ HYP_CONCAT(__hyp_section_, HYP_SECTION_NAME(NAME))
	30	+
	31	+/*
	32	+ * Helper to generate linker script statements starting a hyp section.
	33	+ *
	34	+ * A symbol with a well-known name is defined at the first byte. This
	35	+ * is used as a base for hyp relocations (see gen-hyprel.c). It must
	36	+ * be defined inside the section so the linker of `vmlinux` cannot
	37	+ * separate it from the section data.
	38	+ */
	39	+#define BEGIN_HYP_SECTION(NAME) \
	40	+ HYP_SECTION_NAME(NAME) : { \
	41	+ HYP_SECTION_SYMBOL_NAME(NAME) = .;
	42	+
	43	+/* Helper to generate linker script statements ending a hyp section. */
	44	+#define END_HYP_SECTION \
	45	+ }
	46	+
24	47	/* Defines an ELF hyp section from input section @NAME and its subsections. */
25		−#define HYP_SECTION(NAME) \
26		− HYP_SECTION_NAME(NAME) : { (NAME NAME##.) }
	48	+#define HYP_SECTION(NAME) \
	49	+ BEGIN_HYP_SECTION(NAME) \
	50	+ (NAME NAME##.) \
	51	+ END_HYP_SECTION
27	52
28	53	/*
29	54	* Defines a linker script alias of a kernel-proper symbol referenced by

arch/arm64/include/asm/kvm_asm.h+0−20

@@ -199,26 +199,6 @@ extern void __vgic_v3_init_lrs(void); · 198 unmodified lines
199	199
200	200	extern u32 __kvm_get_mdcr_el2(void);
201	201
202		−/*
203		− * Obtain the PC-relative address of a kernel symbol
204		− * s: symbol
205		− *
206		− * The goal of this macro is to return a symbol's address based on a
207		− * PC-relative computation, as opposed to a loading the VA from a
208		− * constant pool or something similar. This works well for HYP, as an
209		− * absolute VA is guaranteed to be wrong. Only use this if trying to
210		− * obtain the address of a symbol (i.e. not something you obtained by
211		− * following a pointer).
212		− */
213		−#define hyp_symbol_addr(s) \
214		− ({ \
215		− typeof(s) *addr; \
216		− asm("adrp %0, %1\n" \
217		− "add %0, %0, :lo12:%1\n" \
218		− : "=r" (addr) : "S" (&s)); \
219		− addr; \
220		− })
221		−
222	202	#define __KVM_EXTABLE(from, to) \
223	203	" .pushsection __kvm_ex_table, \"a\"\n" \
224	204	" .align 3\n" \

arch/arm64/include/asm/kvm_host.h+2−1

@@ -30,7 +30,6 @@ · 29 unmodified lines
30	30
31	31	#define __KVM_HAVE_ARCH_INTC_INITIALIZED
32	32
33		−#define KVM_USER_MEM_SLOTS 512
34	33	#define KVM_HALT_POLL_NS_DEFAULT 500000
35	34
36	35	#include <kvm/arm_vgic.h>
@@ -771,4 +770,6 @@ bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu); · 734 unmodified lines
771	770	#define kvm_vcpu_has_pmu(vcpu) \
772	771	(test_bit(KVM_ARM_VCPU_PMU_V3, (vcpu)->arch.features))
773	772
	773	+int kvm_trng_call(struct kvm_vcpu *vcpu);
	774	+
774	775	#endif /* __ARM64_KVM_HOST_H__ */

arch/arm64/include/asm/kvm_mmu.h+17−44

@@ -73,49 +73,39 @@ alternative_cb_end · 72 unmodified lines
73	73	.endm
74	74
75	75	/*
76		− * Convert a kernel image address to a PA
77		− * reg: kernel address to be converted in place
	76	+ * Convert a hypervisor VA to a PA
	77	+ * reg: hypervisor address to be converted in place
78	78	* tmp: temporary register
79		− *
80		− * The actual code generation takes place in kvm_get_kimage_voffset, and
81		− * the instructions below are only there to reserve the space and
82		− * perform the register allocation (kvm_get_kimage_voffset uses the
83		− * specific registers encoded in the instructions).
84	79	*/
85		−.macro kimg_pa reg, tmp
86		−alternative_cb kvm_get_kimage_voffset
87		− movz \tmp, #0
88		− movk \tmp, #0, lsl #16
89		− movk \tmp, #0, lsl #32
90		− movk \tmp, #0, lsl #48
91		−alternative_cb_end
92		−
93		− /* reg = __pa(reg) */
94		− sub \reg, \reg, \tmp
	80	+.macro hyp_pa reg, tmp
	81	+ ldr_l \tmp, hyp_physvirt_offset
	82	+ add \reg, \reg, \tmp
95	83	.endm
96	84
97	85	/*
98		− * Convert a kernel image address to a hyp VA
99		− * reg: kernel address to be converted in place
	86	+ * Convert a hypervisor VA to a kernel image address
	87	+ * reg: hypervisor address to be converted in place
100	88	* tmp: temporary register
101	89	*
102	90	* The actual code generation takes place in kvm_get_kimage_voffset, and
103	91	* the instructions below are only there to reserve the space and
104		− * perform the register allocation (kvm_update_kimg_phys_offset uses the
	92	+ * perform the register allocation (kvm_get_kimage_voffset uses the
105	93	* specific registers encoded in the instructions).
106	94	*/
107		−.macro kimg_hyp_va reg, tmp
108		−alternative_cb kvm_update_kimg_phys_offset
	95	+.macro hyp_kimg_va reg, tmp
	96	+ /* Convert hyp VA -> PA. */
	97	+ hyp_pa \reg, \tmp
	98	+
	99	+ /* Load kimage_voffset. */
	100	+alternative_cb kvm_get_kimage_voffset
109	101	movz \tmp, #0
110	102	movk \tmp, #0, lsl #16
111	103	movk \tmp, #0, lsl #32
112	104	movk \tmp, #0, lsl #48
113	105	alternative_cb_end
114	106
115		− sub \reg, \reg, \tmp
116		− mov_q \tmp, PAGE_OFFSET
117		− orr \reg, \reg, \tmp
118		− kern_hyp_va \reg
	107	+ /* Convert PA -> kimg VA. */
	108	+ add \reg, \reg, \tmp
119	109	.endm
120	110
121	111	#else
@@ -129,6 +119,7 @@ alternative_cb_end · 7 unmodified lines
129	119	void kvm_update_va_mask(struct alt_instr *alt,
130	120	__le32 origptr, __le32 updptr, int nr_inst);
131	121	void kvm_compute_layout(void);
	122	+void kvm_apply_hyp_relocations(void);
132	123
133	124	static __always_inline unsigned long __kern_hyp_va(unsigned long v)
134	125	{
@@ -144,24 +135,6 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v) · 9 unmodified lines
144	135
145	136	#define kern_hyp_va(v) ((typeof(v))(__kern_hyp_va((unsigned long)(v))))
146	137
147		−static __always_inline unsigned long __kimg_hyp_va(unsigned long v)
148		−{
149		− unsigned long offset;
150		−
151		− asm volatile(ALTERNATIVE_CB("movz %0, #0\n"
152		− "movk %0, #0, lsl #16\n"
153		− "movk %0, #0, lsl #32\n"
154		− "movk %0, #0, lsl #48\n",
155		− kvm_update_kimg_phys_offset)
156		− : "=r" (offset));
157		−
158		− return __kern_hyp_va((v - offset) \| PAGE_OFFSET);
159		−}
160		−
161		−#define kimg_fn_hyp_va(v) ((typeof(*v))(__kimg_hyp_va((unsigned long)(v))))
162		−
163		−#define kimg_fn_ptr(x) (typeof(x) **)(x)
164		−
165	138	/*
166	139	* We currently support using a VM-specified IPA size. For backward
167	140	* compatibility, the default IPA size is fixed to 40bits.

arch/arm64/include/asm/kvm_pgtable.h+5−0

@@ -157,6 +157,11 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt); · 156 unmodified lines
157	157	* If device attributes are not explicitly requested in @prot, then the
158	158	* mapping will be normal, cacheable.
159	159	*
	160	+ * Note that the update of a valid leaf PTE in this function will be aborted,
	161	+ * if it's trying to recreate the exact same mapping or only change the access
	162	+ * permissions. Instead, the vCPU will exit one more time from guest if still
	163	+ * needed and then go through the path of relaxing permissions.
	164	+ *
160	165	* Note that this function will both coalesce existing table entries and split
161	166	* existing block mappings, relying on page-faults to fault back areas outside
162	167	* of the new mapping lazily.

arch/arm64/include/asm/sections.h+2−1

@@ -11,7 +11,8 @@ extern char __alt_instructions[], __alt_instructions_end[]; · 10 unmodified lines
11	11	extern char __hibernate_exit_text_start[], __hibernate_exit_text_end[];
12	12	extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
13	13	extern char __hyp_text_start[], __hyp_text_end[];
14		−extern char __hyp_data_ro_after_init_start[], __hyp_data_ro_after_init_end[];
	14	+extern char __hyp_rodata_start[], __hyp_rodata_end[];
	15	+extern char __hyp_reloc_begin[], __hyp_reloc_end[];
15	16	extern char __idmap_text_start[], __idmap_text_end[];
16	17	extern char __initdata_begin[], __initdata_end[];
17	18	extern char __inittext_begin[], __inittext_end[];

arch/arm64/include/asm/spinlock.h+1−1

@@ -5,8 +5,8 @@ · 4 unmodified lines
5	5	#ifndef __ASM_SPINLOCK_H
6	6	#define __ASM_SPINLOCK_H
7	7
8		−#include <asm/qrwlock.h>
9	8	#include <asm/qspinlock.h>
	9	+#include <asm/qrwlock.h>
10	10
11	11	/* See include/linux/spinlock.h */
12	12	#define smp_mb__after_spinlock() smp_mb()

arch/arm64/include/asm/sysreg.h+3−0

@@ -853,7 +853,10 @@ · 852 unmodified lines
853	853
854	854	#define ID_DFR0_PERFMON_SHIFT 24
855	855
	856	+#define ID_DFR0_PERFMON_8_0 0x3
856	857	#define ID_DFR0_PERFMON_8_1 0x4
	858	+#define ID_DFR0_PERFMON_8_4 0x5
	859	+#define ID_DFR0_PERFMON_8_5 0x6
857	860
858	861	#define ID_ISAR4_SWP_FRAC_SHIFT 28
859	862	#define ID_ISAR4_PSR_M_SHIFT 24

arch/arm64/kernel/image-vars.h+0−1

@@ -64,7 +64,6 @@ __efistub__ctype = _ctype; · 63 unmodified lines
64	64	/* Alternative callbacks for init-time patching of nVHE hyp code. */
65	65	KVM_NVHE_ALIAS(kvm_patch_vector_branch);
66	66	KVM_NVHE_ALIAS(kvm_update_va_mask);
67		−KVM_NVHE_ALIAS(kvm_update_kimg_phys_offset);
68	67	KVM_NVHE_ALIAS(kvm_get_kimage_voffset);
69	68
70	69	/* Global kernel state accessed by nVHE hyp code. */

arch/arm64/kernel/smp.c+3−1

@@ -434,8 +434,10 @@ static void __init hyp_mode_check(void) · 433 unmodified lines
434	434	"CPU: CPUs started in inconsistent modes");
435	435	else
436	436	pr_info("CPU: All CPU(s) started at EL1\n");
437		− if (IS_ENABLED(CONFIG_KVM) && !is_kernel_in_hyp_mode())
	437	+ if (IS_ENABLED(CONFIG_KVM) && !is_kernel_in_hyp_mode()) {
438	438	kvm_compute_layout();
	439	+ kvm_apply_hyp_relocations();
	440	+ }
439	441	}
440	442
441	443	void __init smp_cpus_done(unsigned int max_cpus)

arch/arm64/kernel/vmlinux.lds.S+15−3

@@ -31,10 +31,11 @@ jiffies = jiffies_64; · 30 unmodified lines
31	31	__stop___kvm_ex_table = .;
32	32
33	33	#define HYPERVISOR_DATA_SECTIONS \
34		− HYP_SECTION_NAME(.data..ro_after_init) : { \
35		− __hyp_data_ro_after_init_start = .; \
	34	+ HYP_SECTION_NAME(.rodata) : { \
	35	+ __hyp_rodata_start = .; \
36	36	*(HYP_SECTION_NAME(.data..ro_after_init)) \
37		− __hyp_data_ro_after_init_end = .; \
	37	+ *(HYP_SECTION_NAME(.rodata)) \
	38	+ __hyp_rodata_end = .; \
38	39	}
39	40
40	41	#define HYPERVISOR_PERCPU_SECTION \
@@ -42,10 +43,19 @@ jiffies = jiffies_64; · 1 unmodified line
42	43	HYP_SECTION_NAME(.data..percpu) : { \
43	44	*(HYP_SECTION_NAME(.data..percpu)) \
44	45	}
	46	+
	47	+#define HYPERVISOR_RELOC_SECTION \
	48	+ .hyp.reloc : ALIGN(4) { \
	49	+ __hyp_reloc_begin = .; \
	50	+ *(.hyp.reloc) \
	51	+ __hyp_reloc_end = .; \
	52	+ }
	53	+
45	54	#else /* CONFIG_KVM */
46	55	#define HYPERVISOR_EXTABLE
47	56	#define HYPERVISOR_DATA_SECTIONS
48	57	#define HYPERVISOR_PERCPU_SECTION
	58	+#define HYPERVISOR_RELOC_SECTION
49	59	#endif
50	60
51	61	#define HYPERVISOR_TEXT \
@@ -216,6 +226,8 @@ SECTIONS · 164 unmodified lines
216	226	PERCPU_SECTION(L1_CACHE_BYTES)
217	227	HYPERVISOR_PERCPU_SECTION
218	228
	229	+ HYPERVISOR_RELOC_SECTION
	230	+
219	231	.rela.dyn : ALIGN(8) {
220	232	(.rela .rela)
221	233	}

arch/arm64/kvm/Makefile+1−1

@@ -16,7 +16,7 @@ kvm-y := $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o \ · 15 unmodified lines
16	16	inject_fault.o va_layout.o handle_exit.o \
17	17	guest.o debug.o reset.o sys_regs.o \
18	18	vgic-sys-reg-v3.o fpsimd.o pmu.o \
19		− arch_timer.o \
	19	+ arch_timer.o trng.o\
20	20	vgic/vgic.o vgic/vgic-init.o \
21	21	vgic/vgic-irqfd.o vgic/vgic-v2.o \
22	22	vgic/vgic-v3.o vgic/vgic-v4.o \

arch/arm64/kvm/arm.c+3−4

@@ -1750,11 +1750,10 @@ static int init_hyp_mode(void) · 1749 unmodified lines
1750	1750	goto out_err;
1751	1751	}
1752	1752
1753		− err = create_hyp_mappings(kvm_ksym_ref(__hyp_data_ro_after_init_start),
1754		− kvm_ksym_ref(__hyp_data_ro_after_init_end),
1755		− PAGE_HYP_RO);
	1753	+ err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start),
	1754	+ kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO);
1756	1755	if (err) {
1757		− kvm_err("Cannot map .hyp.data..ro_after_init section\n");
	1756	+ kvm_err("Cannot map .hyp.rodata section\n");
1758	1757	goto out_err;
1759	1758	}
1760	1759

arch/arm64/kvm/hyp/include/hyp/switch.h+2−2

@@ -505,8 +505,8 @@ static inline void __kvm_unexpected_el2_exception(void) · 504 unmodified lines
505	505	struct exception_table_entry entry, end;
506	506	unsigned long elr_el2 = read_sysreg(elr_el2);
507	507
508		− entry = hyp_symbol_addr(__start___kvm_ex_table);
509		− end = hyp_symbol_addr(__stop___kvm_ex_table);
	508	+ entry = &__start___kvm_ex_table;
	509	+ end = &__stop___kvm_ex_table;
510	510
511	511	while (entry < end) {
512	512	addr = (unsigned long)&entry->insn + entry->insn;

arch/arm64/kvm/hyp/nvhe/.gitignore+2−0

@@ -1,2 +1,4 @@
1	1	# SPDX-License-Identifier: GPL-2.0-only
	2	+gen-hyprel
2	3	hyp.lds
	4	+hyp-reloc.S

arch/arm64/kvm/hyp/nvhe/Makefile+28−5

@@ -3,8 +3,11 @@ · 2 unmodified lines
3	3	# Makefile for Kernel-based Virtual Machine module, HYP/nVHE part
4	4	#
5	5
6		−asflags-y := -D__KVM_NVHE_HYPERVISOR__
7		−ccflags-y := -D__KVM_NVHE_HYPERVISOR__
	6	+asflags-y := -D__KVM_NVHE_HYPERVISOR__ -D__DISABLE_EXPORTS
	7	+ccflags-y := -D__KVM_NVHE_HYPERVISOR__ -D__DISABLE_EXPORTS
	8	+
	9	+hostprogs := gen-hyprel
	10	+HOST_EXTRACFLAGS += -I$(objtree)/include
8	11
9	12	obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \
10	13	hyp-main.o hyp-smp.o psci-relay.o
@@ -19,7 +22,7 @@ obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ · 8 unmodified lines
19	22
20	23	hyp-obj := $(patsubst %.o,%.nvhe.o,$(obj-y))
21	24	obj-y := kvm_nvhe.o
22		−extra-y := $(hyp-obj) kvm_nvhe.tmp.o hyp.lds
	25	+extra-y := $(hyp-obj) kvm_nvhe.tmp.o kvm_nvhe.rel.o hyp.lds hyp-reloc.S hyp-reloc.o
23	26
24	27	# 1) Compile all source files to `.nvhe.o` object files. The file extension
25	28	# avoids file name clashes for files shared with VHE.
@@ -42,11 +45,31 @@ LDFLAGS_kvm_nvhe.tmp.o := -r -T · 16 unmodified lines
42	45	$(obj)/kvm_nvhe.tmp.o: $(obj)/hyp.lds $(addprefix $(obj)/,$(hyp-obj)) FORCE
43	46	$(call if_changed,ld)
44	47
45		−# 4) Produce the final 'kvm_nvhe.o', ready to be linked into 'vmlinux'.
	48	+# 4) Generate list of hyp code/data positions that need to be relocated at
	49	+# runtime. Because the hypervisor is part of the kernel binary, relocations
	50	+# produce a kernel VA. We enumerate relocations targeting hyp at build time
	51	+# and convert the kernel VAs at those positions to hyp VAs.
	52	+$(obj)/hyp-reloc.S: $(obj)/kvm_nvhe.tmp.o $(obj)/gen-hyprel
	53	+ $(call if_changed,hyprel)
	54	+
	55	+# 5) Compile hyp-reloc.S and link it into the existing partially linked object.
	56	+# The object file now contains a section with pointers to hyp positions that
	57	+# will contain kernel VAs at runtime. These pointers have relocations on them
	58	+# so that they get updated as the hyp object is linked into `vmlinux`.
	59	+LDFLAGS_kvm_nvhe.rel.o := -r
	60	+$(obj)/kvm_nvhe.rel.o: $(obj)/kvm_nvhe.tmp.o $(obj)/hyp-reloc.o FORCE
	61	+ $(call if_changed,ld)
	62	+
	63	+# 6) Produce the final 'kvm_nvhe.o', ready to be linked into 'vmlinux'.
46	64	# Prefixes names of ELF symbols with '__kvm_nvhe_'.
47		−$(obj)/kvm_nvhe.o: $(obj)/kvm_nvhe.tmp.o FORCE
	65	+$(obj)/kvm_nvhe.o: $(obj)/kvm_nvhe.rel.o FORCE
48	66	$(call if_changed,hypcopy)
49	67
	68	+# The HYPREL command calls `gen-hyprel` to generate an assembly file with
	69	+# a list of relocations targeting hyp code/data.
	70	+quiet_cmd_hyprel = HYPREL $@
	71	+ cmd_hyprel = $(obj)/gen-hyprel $< > $@
	72	+
50	73	# The HYPCOPY command uses `objcopy` to prefix all ELF symbol names
51	74	# to avoid clashes with VHE code/data.
52	75	quiet_cmd_hypcopy = HYPCOPY $@

arch/arm64/kvm/hyp/nvhe/gen-hyprel.c+438−0

@@ -0,0 +1,438 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/*
	3	+ * Copyright (C) 2020 - Google LLC
	4	+ * Author: David Brazdil <dbrazdil@google.com>
	5	+ *
	6	+ * Generates relocation information used by the kernel to convert
	7	+ * absolute addresses in hyp data from kernel VAs to hyp VAs.
	8	+ *
	9	+ * This is necessary because hyp code is linked into the same binary
	10	+ * as the kernel but executes under different memory mappings.
	11	+ * If the compiler used absolute addressing, those addresses need to
	12	+ * be converted before they are used by hyp code.
	13	+ *
	14	+ * The input of this program is the relocatable ELF object containing
	15	+ * all hyp code/data, not yet linked into vmlinux. Hyp section names
	16	+ * should have been prefixed with `.hyp` at this point.
	17	+ *
	18	+ * The output (printed to stdout) is an assembly file containing
	19	+ * an array of 32-bit integers and static relocations that instruct
	20	+ * the linker of `vmlinux` to populate the array entries with offsets
	21	+ * to positions in the kernel binary containing VAs used by hyp code.
	22	+ *
	23	+ * Note that dynamic relocations could be used for the same purpose.
	24	+ * However, those are only generated if CONFIG_RELOCATABLE=y.
	25	+ */
	26	+
	27	+#include <elf.h>
	28	+#include <endian.h>
	29	+#include <errno.h>
	30	+#include <fcntl.h>
	31	+#include <stdbool.h>
	32	+#include <stdio.h>
	33	+#include <stdlib.h>
	34	+#include <string.h>
	35	+#include <sys/mman.h>
	36	+#include <sys/types.h>
	37	+#include <sys/stat.h>
	38	+#include <unistd.h>
	39	+
	40	+#include <generated/autoconf.h>
	41	+
	42	+#define HYP_SECTION_PREFIX ".hyp"
	43	+#define HYP_RELOC_SECTION ".hyp.reloc"
	44	+#define HYP_SECTION_SYMBOL_PREFIX "__hyp_section_"
	45	+
	46	+/*
	47	+ * AArch64 relocation type constants.
	48	+ * Included in case these are not defined in the host toolchain.
	49	+ */
	50	+#ifndef R_AARCH64_ABS64
	51	+#define R_AARCH64_ABS64 257
	52	+#endif
	53	+#ifndef R_AARCH64_LD_PREL_LO19
	54	+#define R_AARCH64_LD_PREL_LO19 273
	55	+#endif
	56	+#ifndef R_AARCH64_ADR_PREL_LO21
	57	+#define R_AARCH64_ADR_PREL_LO21 274
	58	+#endif
	59	+#ifndef R_AARCH64_ADR_PREL_PG_HI21
	60	+#define R_AARCH64_ADR_PREL_PG_HI21 275
	61	+#endif
	62	+#ifndef R_AARCH64_ADR_PREL_PG_HI21_NC
	63	+#define R_AARCH64_ADR_PREL_PG_HI21_NC 276
	64	+#endif
	65	+#ifndef R_AARCH64_ADD_ABS_LO12_NC
	66	+#define R_AARCH64_ADD_ABS_LO12_NC 277
	67	+#endif
	68	+#ifndef R_AARCH64_LDST8_ABS_LO12_NC
	69	+#define R_AARCH64_LDST8_ABS_LO12_NC 278
	70	+#endif
	71	+#ifndef R_AARCH64_TSTBR14
	72	+#define R_AARCH64_TSTBR14 279
	73	+#endif
	74	+#ifndef R_AARCH64_CONDBR19
	75	+#define R_AARCH64_CONDBR19 280
	76	+#endif
	77	+#ifndef R_AARCH64_JUMP26
	78	+#define R_AARCH64_JUMP26 282
	79	+#endif
	80	+#ifndef R_AARCH64_CALL26
	81	+#define R_AARCH64_CALL26 283
	82	+#endif
	83	+#ifndef R_AARCH64_LDST16_ABS_LO12_NC
	84	+#define R_AARCH64_LDST16_ABS_LO12_NC 284
	85	+#endif
	86	+#ifndef R_AARCH64_LDST32_ABS_LO12_NC
	87	+#define R_AARCH64_LDST32_ABS_LO12_NC 285
	88	+#endif
	89	+#ifndef R_AARCH64_LDST64_ABS_LO12_NC
	90	+#define R_AARCH64_LDST64_ABS_LO12_NC 286
	91	+#endif
	92	+#ifndef R_AARCH64_MOVW_PREL_G0
	93	+#define R_AARCH64_MOVW_PREL_G0 287
	94	+#endif
	95	+#ifndef R_AARCH64_MOVW_PREL_G0_NC
	96	+#define R_AARCH64_MOVW_PREL_G0_NC 288
	97	+#endif
	98	+#ifndef R_AARCH64_MOVW_PREL_G1
	99	+#define R_AARCH64_MOVW_PREL_G1 289
	100	+#endif
	101	+#ifndef R_AARCH64_MOVW_PREL_G1_NC
	102	+#define R_AARCH64_MOVW_PREL_G1_NC 290
	103	+#endif
	104	+#ifndef R_AARCH64_MOVW_PREL_G2
	105	+#define R_AARCH64_MOVW_PREL_G2 291
	106	+#endif
	107	+#ifndef R_AARCH64_MOVW_PREL_G2_NC
	108	+#define R_AARCH64_MOVW_PREL_G2_NC 292
	109	+#endif
	110	+#ifndef R_AARCH64_MOVW_PREL_G3
	111	+#define R_AARCH64_MOVW_PREL_G3 293
	112	+#endif
	113	+#ifndef R_AARCH64_LDST128_ABS_LO12_NC
	114	+#define R_AARCH64_LDST128_ABS_LO12_NC 299
	115	+#endif
	116	+
	117	+/* Global state of the processed ELF. */
	118	+static struct {
	119	+ const char *path;
	120	+ char *begin;
	121	+ size_t size;
	122	+ Elf64_Ehdr *ehdr;
	123	+ Elf64_Shdr *sh_table;
	124	+ const char *sh_string;
	125	+} elf;
	126	+
	127	+#if defined(CONFIG_CPU_LITTLE_ENDIAN)
	128	+
	129	+#define elf16toh(x) le16toh(x)
	130	+#define elf32toh(x) le32toh(x)
	131	+#define elf64toh(x) le64toh(x)
	132	+
	133	+#define ELFENDIAN ELFDATA2LSB
	134	+
	135	+#elif defined(CONFIG_CPU_BIG_ENDIAN)
	136	+
	137	+#define elf16toh(x) be16toh(x)
	138	+#define elf32toh(x) be32toh(x)
	139	+#define elf64toh(x) be64toh(x)
	140	+
	141	+#define ELFENDIAN ELFDATA2MSB
	142	+
	143	+#else
	144	+
	145	+#error PDP-endian sadly unsupported...
	146	+
	147	+#endif
	148	+
	149	+#define fatal_error(fmt, ...) \
	150	+ ({ \
	151	+ fprintf(stderr, "error: %s: " fmt "\n", \
	152	+ elf.path, ## __VA_ARGS__); \
	153	+ exit(EXIT_FAILURE); \
	154	+ __builtin_unreachable(); \
	155	+ })
	156	+
	157	+#define fatal_perror(msg) \
	158	+ ({ \
	159	+ fprintf(stderr, "error: %s: " msg ": %s\n", \
	160	+ elf.path, strerror(errno)); \
	161	+ exit(EXIT_FAILURE); \
	162	+ __builtin_unreachable(); \
	163	+ })
	164	+
	165	+#define assert_op(lhs, rhs, fmt, op) \
	166	+ ({ \
	167	+ typeof(lhs) _lhs = (lhs); \
	168	+ typeof(rhs) _rhs = (rhs); \
	169	+ \
	170	+ if (!(_lhs op _rhs)) { \
	171	+ fatal_error("assertion " #lhs " " #op " " #rhs \
	172	+ " failed (lhs=" fmt ", rhs=" fmt \
	173	+ ", line=%d)", _lhs, _rhs, __LINE__); \
	174	+ } \
	175	+ })
	176	+
	177	+#define assert_eq(lhs, rhs, fmt) assert_op(lhs, rhs, fmt, ==)
	178	+#define assert_ne(lhs, rhs, fmt) assert_op(lhs, rhs, fmt, !=)
	179	+#define assert_lt(lhs, rhs, fmt) assert_op(lhs, rhs, fmt, <)
	180	+#define assert_ge(lhs, rhs, fmt) assert_op(lhs, rhs, fmt, >=)
	181	+
	182	+/*
	183	+ * Return a pointer of a given type at a given offset from
	184	+ * the beginning of the ELF file.
	185	+ */
	186	+#define elf_ptr(type, off) ((type *)(elf.begin + (off)))
	187	+
	188	+/* Iterate over all sections in the ELF. */
	189	+#define for_each_section(var) \
	190	+ for (var = elf.sh_table; var < elf.sh_table + elf16toh(elf.ehdr->e_shnum); ++var)
	191	+
	192	+/* Iterate over all Elf64_Rela relocations in a given section. */
	193	+#define for_each_rela(shdr, var) \
	194	+ for (var = elf_ptr(Elf64_Rela, elf64toh(shdr->sh_offset)); \
	195	+ var < elf_ptr(Elf64_Rela, elf64toh(shdr->sh_offset) + elf64toh(shdr->sh_size)); var++)
	196	+
	197	+/* True if a string starts with a given prefix. */
	198	+static inline bool starts_with(const char str, const char prefix)
	199	+{
	200	+ return memcmp(str, prefix, strlen(prefix)) == 0;
	201	+}
	202	+
	203	+/* Returns a string containing the name of a given section. */
	204	+static inline const char section_name(Elf64_Shdr shdr)
	205	+{
	206	+ return elf.sh_string + elf32toh(shdr->sh_name);
	207	+}
	208	+
	209	+/* Returns a pointer to the first byte of section data. */
	210	+static inline const char section_begin(Elf64_Shdr shdr)
	211	+{
	212	+ return elf_ptr(char, elf64toh(shdr->sh_offset));
	213	+}
	214	+
	215	+/* Find a section by its offset from the beginning of the file. */
	216	+static inline Elf64_Shdr *section_by_off(Elf64_Off off)
	217	+{
	218	+ assert_ne(off, 0UL, "%lu");
	219	+ return elf_ptr(Elf64_Shdr, off);
	220	+}
	221	+
	222	+/* Find a section by its index. */
	223	+static inline Elf64_Shdr *section_by_idx(uint16_t idx)
	224	+{
	225	+ assert_ne(idx, SHN_UNDEF, "%u");
	226	+ return &elf.sh_table[idx];
	227	+}
	228	+
	229	+/*
	230	+ * Memory-map the given ELF file, perform sanity checks, and
	231	+ * populate global state.
	232	+ */
	233	+static void init_elf(const char *path)
	234	+{
	235	+ int fd, ret;
	236	+ struct stat stat;
	237	+
	238	+ /* Store path in the global struct for error printing. */
	239	+ elf.path = path;
	240	+
	241	+ /* Open the ELF file. */
	242	+ fd = open(path, O_RDONLY);
	243	+ if (fd < 0)
	244	+ fatal_perror("Could not open ELF file");
	245	+
	246	+ /* Get status of ELF file to obtain its size. */
	247	+ ret = fstat(fd, &stat);
	248	+ if (ret < 0) {
	249	+ close(fd);
	250	+ fatal_perror("Could not get status of ELF file");
	251	+ }
	252	+
	253	+ /* mmap() the entire ELF file read-only at an arbitrary address. */
	254	+ elf.begin = mmap(0, stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
	255	+ if (elf.begin == MAP_FAILED) {
	256	+ close(fd);
	257	+ fatal_perror("Could not mmap ELF file");
	258	+ }
	259	+
	260	+ /* mmap() was successful, close the FD. */
	261	+ close(fd);
	262	+
	263	+ /* Get pointer to the ELF header. */
	264	+ assert_ge(stat.st_size, sizeof(*elf.ehdr), "%lu");
	265	+ elf.ehdr = elf_ptr(Elf64_Ehdr, 0);
	266	+
	267	+ /* Check the ELF magic. */
	268	+ assert_eq(elf.ehdr->e_ident[EI_MAG0], ELFMAG0, "0x%x");
	269	+ assert_eq(elf.ehdr->e_ident[EI_MAG1], ELFMAG1, "0x%x");
	270	+ assert_eq(elf.ehdr->e_ident[EI_MAG2], ELFMAG2, "0x%x");
	271	+ assert_eq(elf.ehdr->e_ident[EI_MAG3], ELFMAG3, "0x%x");
	272	+
	273	+ /* Sanity check that this is an ELF64 relocatable object for AArch64. */
	274	+ assert_eq(elf.ehdr->e_ident[EI_CLASS], ELFCLASS64, "%u");
	275	+ assert_eq(elf.ehdr->e_ident[EI_DATA], ELFENDIAN, "%u");
	276	+ assert_eq(elf16toh(elf.ehdr->e_type), ET_REL, "%u");
	277	+ assert_eq(elf16toh(elf.ehdr->e_machine), EM_AARCH64, "%u");
	278	+
	279	+ /* Populate fields of the global struct. */
	280	+ elf.sh_table = section_by_off(elf64toh(elf.ehdr->e_shoff));
	281	+ elf.sh_string = section_begin(section_by_idx(elf16toh(elf.ehdr->e_shstrndx)));
	282	+}
	283	+
	284	+/* Print the prologue of the output ASM file. */
	285	+static void emit_prologue(void)
	286	+{
	287	+ printf(".data\n"
	288	+ ".pushsection " HYP_RELOC_SECTION ", \"a\"\n");
	289	+}
	290	+
	291	+/* Print ASM statements needed as a prologue to a processed hyp section. */
	292	+static void emit_section_prologue(const char *sh_orig_name)
	293	+{
	294	+ /* Declare the hyp section symbol. */
	295	+ printf(".global %s%s\n", HYP_SECTION_SYMBOL_PREFIX, sh_orig_name);
	296	+}
	297	+
	298	+/*
	299	+ * Print ASM statements to create a hyp relocation entry for a given
	300	+ * R_AARCH64_ABS64 relocation.
	301	+ *
	302	+ * The linker of vmlinux will populate the position given by `rela` with
	303	+ * an absolute 64-bit kernel VA. If the kernel is relocatable, it will
	304	+ * also generate a dynamic relocation entry so that the kernel can shift
	305	+ * the address at runtime for KASLR.
	306	+ *
	307	+ * Emit a 32-bit offset from the current address to the position given
	308	+ * by `rela`. This way the kernel can iterate over all kernel VAs used
	309	+ * by hyp at runtime and convert them to hyp VAs. However, that offset
	310	+ * will not be known until linking of `vmlinux`, so emit a PREL32
	311	+ * relocation referencing a symbol that the hyp linker script put at
	312	+ * the beginning of the relocated section + the offset from `rela`.
	313	+ */
	314	+static void emit_rela_abs64(Elf64_Rela rela, const char sh_orig_name)
	315	+{
	316	+ /* Offset of this reloc from the beginning of HYP_RELOC_SECTION. */
	317	+ static size_t reloc_offset;
	318	+
	319	+ /* Create storage for the 32-bit offset. */
	320	+ printf(".word 0\n");
	321	+
	322	+ /*
	323	+ * Create a PREL32 relocation which instructs the linker of `vmlinux`
	324	+ * to insert offset to position <base> + <offset>, where <base> is
	325	+ * a symbol at the beginning of the relocated section, and <offset>
	326	+ * is `rela->r_offset`.
	327	+ */
	328	+ printf(".reloc %lu, R_AARCH64_PREL32, %s%s + 0x%lx\n",
	329	+ reloc_offset, HYP_SECTION_SYMBOL_PREFIX, sh_orig_name,
	330	+ elf64toh(rela->r_offset));
	331	+
	332	+ reloc_offset += 4;
	333	+}
	334	+
	335	+/* Print the epilogue of the output ASM file. */
	336	+static void emit_epilogue(void)
	337	+{
	338	+ printf(".popsection\n");
	339	+}
	340	+
	341	+/*
	342	+ * Iterate over all RELA relocations in a given section and emit
	343	+ * hyp relocation data for all absolute addresses in hyp code/data.
	344	+ *
	345	+ * Static relocations that generate PC-relative-addressing are ignored.
	346	+ * Failure is reported for unexpected relocation types.
	347	+ */
	348	+static void emit_rela_section(Elf64_Shdr *sh_rela)
	349	+{
	350	+ Elf64_Shdr *sh_orig = &elf.sh_table[elf32toh(sh_rela->sh_info)];
	351	+ const char *sh_orig_name = section_name(sh_orig);
	352	+ Elf64_Rela *rela;
	353	+
	354	+ /* Skip all non-hyp sections. */
	355	+ if (!starts_with(sh_orig_name, HYP_SECTION_PREFIX))
	356	+ return;
	357	+
	358	+ emit_section_prologue(sh_orig_name);
	359	+
	360	+ for_each_rela(sh_rela, rela) {
	361	+ uint32_t type = (uint32_t)elf64toh(rela->r_info);
	362	+
	363	+ /* Check that rela points inside the relocated section. */
	364	+ assert_lt(elf64toh(rela->r_offset), elf64toh(sh_orig->sh_size), "0x%lx");
	365	+
	366	+ switch (type) {
	367	+ /*
	368	+ * Data relocations to generate absolute addressing.
	369	+ * Emit a hyp relocation.
	370	+ */
	371	+ case R_AARCH64_ABS64:
	372	+ emit_rela_abs64(rela, sh_orig_name);
	373	+ break;
	374	+ /* Allow relocations to generate PC-relative addressing. */
	375	+ case R_AARCH64_LD_PREL_LO19:
	376	+ case R_AARCH64_ADR_PREL_LO21:
	377	+ case R_AARCH64_ADR_PREL_PG_HI21:
	378	+ case R_AARCH64_ADR_PREL_PG_HI21_NC:
	379	+ case R_AARCH64_ADD_ABS_LO12_NC:
	380	+ case R_AARCH64_LDST8_ABS_LO12_NC:
	381	+ case R_AARCH64_LDST16_ABS_LO12_NC:
	382	+ case R_AARCH64_LDST32_ABS_LO12_NC:
	383	+ case R_AARCH64_LDST64_ABS_LO12_NC:
	384	+ case R_AARCH64_LDST128_ABS_LO12_NC:
	385	+ break;
	386	+ /* Allow relative relocations for control-flow instructions. */
	387	+ case R_AARCH64_TSTBR14:
	388	+ case R_AARCH64_CONDBR19:
	389	+ case R_AARCH64_JUMP26:
	390	+ case R_AARCH64_CALL26:
	391	+ break;
	392	+ /* Allow group relocations to create PC-relative offset inline. */
	393	+ case R_AARCH64_MOVW_PREL_G0:
	394	+ case R_AARCH64_MOVW_PREL_G0_NC:
	395	+ case R_AARCH64_MOVW_PREL_G1:
	396	+ case R_AARCH64_MOVW_PREL_G1_NC:
	397	+ case R_AARCH64_MOVW_PREL_G2:
	398	+ case R_AARCH64_MOVW_PREL_G2_NC:
	399	+ case R_AARCH64_MOVW_PREL_G3:
	400	+ break;
	401	+ default:
	402	+ fatal_error("Unexpected RELA type %u", type);
	403	+ }
	404	+ }
	405	+}
	406	+
	407	+/* Iterate over all sections and emit hyp relocation data for RELA sections. */
	408	+static void emit_all_relocs(void)
	409	+{
	410	+ Elf64_Shdr *shdr;
	411	+
	412	+ for_each_section(shdr) {
	413	+ switch (elf32toh(shdr->sh_type)) {
	414	+ case SHT_REL:
	415	+ fatal_error("Unexpected SHT_REL section \"%s\"",
	416	+ section_name(shdr));
	417	+ case SHT_RELA:
	418	+ emit_rela_section(shdr);
	419	+ break;
	420	+ }
	421	+ }
	422	+}
	423	+
	424	+int main(int argc, const char **argv)
	425	+{
	426	+ if (argc != 2) {
	427	+ fprintf(stderr, "Usage: %s <elf_input>\n", argv[0]);
	428	+ return EXIT_FAILURE;
	429	+ }
	430	+
	431	+ init_elf(argv[1]);
	432	+
	433	+ emit_prologue();
	434	+ emit_all_relocs();
	435	+ emit_epilogue();
	436	+
	437	+ return EXIT_SUCCESS;
	438	+}

arch/arm64/kvm/hyp/nvhe/host.S+15−14

@@ -74,27 +74,28 @@ SYM_FUNC_END(__host_enter) · 73 unmodified lines
74	74	* void __noreturn __hyp_do_panic(bool restore_host, u64 spsr, u64 elr, u64 par);
75	75	*/
76	76	SYM_FUNC_START(__hyp_do_panic)
77		− /* Load the format arguments into x1-7 */
78		− mov x6, x3
79		− get_vcpu_ptr x7, x3
80		−
81		− mrs x3, esr_el2
82		− mrs x4, far_el2
83		− mrs x5, hpfar_el2
84		−
85	77	/* Prepare and exit to the host's panic funciton. */
86	78	mov lr, #(PSR_F_BIT \| PSR_I_BIT \| PSR_A_BIT \| PSR_D_BIT \|\
87	79	PSR_MODE_EL1h)
88	80	msr spsr_el2, lr
89	81	ldr lr, =panic
	82	+ hyp_kimg_va lr, x6
90	83	msr elr_el2, lr
91	84
92		− /*
93		− * Set the panic format string and enter the host, conditionally
94		− * restoring the host context.
95		− */
	85	+ /* Set the panic format string. Use the, now free, LR as scratch. */
	86	+ ldr lr, =__hyp_panic_string
	87	+ hyp_kimg_va lr, x6
	88	+
	89	+ /* Load the format arguments into x1-7. */
	90	+ mov x6, x3
	91	+ get_vcpu_ptr x7, x3
	92	+ mrs x3, esr_el2
	93	+ mrs x4, far_el2
	94	+ mrs x5, hpfar_el2
	95	+
	96	+ /* Enter the host, conditionally restoring the host context. */
96	97	cmp x0, xzr
97		− ldr x0, =__hyp_panic_string
	98	+ mov x0, lr
98	99	b.eq __host_enter_without_restoring
99	100	b __host_enter_for_panic
100	101	SYM_FUNC_END(__hyp_do_panic)
@@ -124,7 +125,7 @@ SYM_FUNC_END(__hyp_do_panic) · 23 unmodified lines
124	125	* Preserve x0-x4, which may contain stub parameters.
125	126	*/
126	127	ldr x5, =__kvm_handle_stub_hvc
127		− kimg_pa x5, x6
	128	+ hyp_pa x5, x6
128	129	br x5
129	130	.L__vect_end\@:
130	131	.if ((.L__vect_end\@ - .L__vect_start\@) > 0x80)

arch/arm64/kvm/hyp/nvhe/hyp-init.S+5−14

@@ -18,7 +18,7 @@ · 17 unmodified lines
18	18	#include <asm/virt.h>
19	19
20	20	.text
21		− .pushsection .hyp.idmap.text, "ax"
	21	+ .pushsection .idmap.text, "ax"
22	22
23	23	.align 11
24	24
@@ -57,17 +57,10 @@ __do_hyp_init: · 32 unmodified lines
57	57	cmp x0, #HVC_STUB_HCALL_NR
58	58	b.lo __kvm_handle_stub_hvc
59	59
60		− // We only actively check bits [24:31], and everything
61		− // else has to be zero, which we check at build time.
62		−#if (KVM_HOST_SMCCC_FUNC(__kvm_hyp_init) & 0xFFFFFFFF00FFFFFF)
63		−#error Unexpected __KVM_HOST_SMCCC_FUNC___kvm_hyp_init value
64		−#endif
65		−
66		− ror x0, x0, #24
67		− eor x0, x0, #((KVM_HOST_SMCCC_FUNC(__kvm_hyp_init) >> 24) & 0xF)
68		− ror x0, x0, #4
69		− eor x0, x0, #((KVM_HOST_SMCCC_FUNC(__kvm_hyp_init) >> 28) & 0xF)
70		− cbz x0, 1f
	60	+ mov x3, #KVM_HOST_SMCCC_FUNC(__kvm_hyp_init)
	61	+ cmp x0, x3
	62	+ b.eq 1f
	63	+
71	64	mov x0, #SMCCC_RET_NOT_SUPPORTED
72	65	eret
73	66
@@ -141,7 +134,6 @@ alternative_else_nop_endif · 67 unmodified lines
141	134
142	135	/* Set the host vector */
143	136	ldr x0, =__kvm_hyp_host_vector
144		− kimg_hyp_va x0, x1
145	137	msr vbar_el2, x0
146	138
147	139	ret
@@ -200,7 +192,6 @@ SYM_CODE_START_LOCAL(__kvm_hyp_init_cpu) · 52 unmodified lines
200	192	/* Leave idmap. */
201	193	mov x0, x29
202	194	ldr x1, =kvm_host_psci_cpu_entry
203		− kimg_hyp_va x1, x2
204	195	br x1
205	196	SYM_CODE_END(__kvm_hyp_init_cpu)
206	197

arch/arm64/kvm/hyp/nvhe/hyp-main.c+4−7

@@ -108,9 +108,9 @@ static void handle___vgic_v3_restore_aprs(struct kvm_cpu_context *host_ctxt) · 107 unmodified lines
108	108
109	109	typedef void (hcall_t)(struct kvm_cpu_context );
110	110
111		−#define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = kimg_fn_ptr(handle_##x)
	111	+#define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x
112	112
113		−static const hcall_t *host_hcall[] = {
	113	+static const hcall_t host_hcall[] = {
114	114	HANDLE_FUNC(__kvm_vcpu_run),
115	115	HANDLE_FUNC(__kvm_flush_vm_context),
116	116	HANDLE_FUNC(__kvm_tlb_flush_vmid_ipa),
@@ -130,7 +130,6 @@ static const hcall_t *host_hcall[] = { · 13 unmodified lines
130	130	static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)
131	131	{
132	132	DECLARE_REG(unsigned long, id, host_ctxt, 0);
133		− const hcall_t *kfn;
134	133	hcall_t hfn;
135	134
136	135	id -= KVM_HOST_SMCCC_ID(0);
@@ -138,13 +137,11 @@ static void handle_host_hcall(struct kvm_cpu_context *host_ctxt) · 1 unmodified line
138	137	if (unlikely(id >= ARRAY_SIZE(host_hcall)))
139	138	goto inval;
140	139
141		− kfn = host_hcall[id];
142		− if (unlikely(!kfn))
	140	+ hfn = host_hcall[id];
	141	+ if (unlikely(!hfn))
143	142	goto inval;
144	143
145	144	cpu_reg(host_ctxt, 0) = SMCCC_RET_SUCCESS;
146		−
147		− hfn = kimg_fn_hyp_va(kfn);
148	145	hfn(host_ctxt);
149	146
150	147	return;

arch/arm64/kvm/hyp/nvhe/hyp-smp.c+2−2

@@ -33,8 +33,8 @@ unsigned long __hyp_per_cpu_offset(unsigned int cpu) · 32 unmodified lines
33	33	if (cpu >= ARRAY_SIZE(kvm_arm_hyp_percpu_base))
34	34	hyp_panic();
35	35
36		− cpu_base_array = (unsigned long *)hyp_symbol_addr(kvm_arm_hyp_percpu_base);
	36	+ cpu_base_array = (unsigned long *)&kvm_arm_hyp_percpu_base;
37	37	this_cpu_base = kern_hyp_va(cpu_base_array[cpu]);
38		− elf_base = (unsigned long)hyp_symbol_addr(__per_cpu_start);
	38	+ elf_base = (unsigned long)&__per_cpu_start;
39	39	return this_cpu_base - elf_base;
40	40	}

arch/arm64/kvm/hyp/nvhe/hyp.lds.S+6−3

@@ -12,14 +12,17 @@ · 11 unmodified lines
12	12	#include <asm/memory.h>
13	13
14	14	SECTIONS {
	15	+ HYP_SECTION(.idmap.text)
15	16	HYP_SECTION(.text)
	17	+ HYP_SECTION(.data..ro_after_init)
	18	+ HYP_SECTION(.rodata)
	19	+
16	20	/*
17	21	* .hyp..data..percpu needs to be page aligned to maintain the same
18	22	* alignment for when linking into vmlinux.
19	23	*/
20	24	. = ALIGN(PAGE_SIZE);
21		− HYP_SECTION_NAME(.data..percpu) : {
	25	+ BEGIN_HYP_SECTION(.data..percpu)
22	26	PERCPU_INPUT(L1_CACHE_BYTES)
23		− }
24		− HYP_SECTION(.data..ro_after_init)
	27	+ END_HYP_SECTION
25	28	}

arch/arm64/kvm/hyp/nvhe/psci-relay.c+12−12

@@ -128,8 +128,8 @@ static int psci_cpu_on(u64 func_id, struct kvm_cpu_context *host_ctxt) · 127 unmodified lines
128	128	if (cpu_id == INVALID_CPU_ID)
129	129	return PSCI_RET_INVALID_PARAMS;
130	130
131		− boot_args = per_cpu_ptr(hyp_symbol_addr(cpu_on_args), cpu_id);
132		− init_params = per_cpu_ptr(hyp_symbol_addr(kvm_init_params), cpu_id);
	131	+ boot_args = per_cpu_ptr(&cpu_on_args, cpu_id);
	132	+ init_params = per_cpu_ptr(&kvm_init_params, cpu_id);
133	133
134	134	/* Check if the target CPU is already being booted. */
135	135	if (!try_acquire_boot_args(boot_args))
@@ -140,7 +140,7 @@ static int psci_cpu_on(u64 func_id, struct kvm_cpu_context *host_ctxt) · 4 unmodified lines
140	140	wmb();
141	141
142	142	ret = psci_call(func_id, mpidr,
143		− __hyp_pa(hyp_symbol_addr(kvm_hyp_cpu_entry)),
	143	+ __hyp_pa(&kvm_hyp_cpu_entry),
144	144	__hyp_pa(init_params));
145	145
146	146	/* If successful, the lock will be released by the target CPU. */
@@ -159,8 +159,8 @@ static int psci_cpu_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt) · 12 unmodified lines
159	159	struct psci_boot_args *boot_args;
160	160	struct kvm_nvhe_init_params *init_params;
161	161
162		− boot_args = this_cpu_ptr(hyp_symbol_addr(suspend_args));
163		− init_params = this_cpu_ptr(hyp_symbol_addr(kvm_init_params));
	162	+ boot_args = this_cpu_ptr(&suspend_args);
	163	+ init_params = this_cpu_ptr(&kvm_init_params);
164	164
165	165	/*
166	166	* No need to acquire a lock before writing to boot_args because a core
@@ -174,7 +174,7 @@ static int psci_cpu_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt) · 7 unmodified lines
174	174	* point if it is a deep sleep state.
175	175	*/
176	176	return psci_call(func_id, power_state,
177		− __hyp_pa(hyp_symbol_addr(kvm_hyp_cpu_resume)),
	177	+ __hyp_pa(&kvm_hyp_cpu_resume),
178	178	__hyp_pa(init_params));
179	179	}
180	180
@@ -186,8 +186,8 @@ static int psci_system_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt) · 5 unmodified lines
186	186	struct psci_boot_args *boot_args;
187	187	struct kvm_nvhe_init_params *init_params;
188	188
189		− boot_args = this_cpu_ptr(hyp_symbol_addr(suspend_args));
190		− init_params = this_cpu_ptr(hyp_symbol_addr(kvm_init_params));
	189	+ boot_args = this_cpu_ptr(&suspend_args);
	190	+ init_params = this_cpu_ptr(&kvm_init_params);
191	191
192	192	/*
193	193	* No need to acquire a lock before writing to boot_args because a core
@@ -198,7 +198,7 @@ static int psci_system_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt) · 4 unmodified lines
198	198
199	199	/* Will only return on error. */
200	200	return psci_call(func_id,
201		− __hyp_pa(hyp_symbol_addr(kvm_hyp_cpu_resume)),
	201	+ __hyp_pa(&kvm_hyp_cpu_resume),
202	202	__hyp_pa(init_params), 0);
203	203	}
204	204
@@ -207,12 +207,12 @@ asmlinkage void __noreturn kvm_host_psci_cpu_entry(bool is_cpu_on) · 2 unmodified lines
207	207	struct psci_boot_args *boot_args;
208	208	struct kvm_cpu_context *host_ctxt;
209	209
210		− host_ctxt = &this_cpu_ptr(hyp_symbol_addr(kvm_host_data))->host_ctxt;
	210	+ host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
211	211
212	212	if (is_cpu_on)
213		− boot_args = this_cpu_ptr(hyp_symbol_addr(cpu_on_args));
	213	+ boot_args = this_cpu_ptr(&cpu_on_args);
214	214	else
215		− boot_args = this_cpu_ptr(hyp_symbol_addr(suspend_args));
	215	+ boot_args = this_cpu_ptr(&suspend_args);
216	216
217	217	cpu_reg(host_ctxt, 0) = boot_args->r0;
218	218	write_sysreg_el2(boot_args->pc, SYS_ELR);

arch/arm64/kvm/hyp/pgtable.c+47−36

@@ -45,6 +45,10 @@ · 44 unmodified lines
45	45
46	46	#define KVM_PTE_LEAF_ATTR_HI_S2_XN BIT(54)
47	47
	48	+#define KVM_PTE_LEAF_ATTR_S2_PERMS (KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R \| \
	49	+ KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W \| \
	50	+ KVM_PTE_LEAF_ATTR_HI_S2_XN)
	51	+
48	52	struct kvm_pgtable_walk_data {
49	53	struct kvm_pgtable *pgt;
50	54	struct kvm_pgtable_walker *walker;
@@ -170,10 +174,9 @@ static void kvm_set_table_pte(kvm_pte_t ptep, kvm_pte_t childp) · 119 unmodified lines
170	174	smp_store_release(ptep, pte);
171	175	}
172	176
173		−static bool kvm_set_valid_leaf_pte(kvm_pte_t *ptep, u64 pa, kvm_pte_t attr,
174		− u32 level)
	177	+static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, u32 level)
175	178	{
176		− kvm_pte_t old = *ptep, pte = kvm_phys_to_pte(pa);
	179	+ kvm_pte_t pte = kvm_phys_to_pte(pa);
177	180	u64 type = (level == KVM_PGTABLE_MAX_LEVELS - 1) ? KVM_PTE_TYPE_PAGE :
178	181	KVM_PTE_TYPE_BLOCK;
179	182
@@ -181,12 +184,7 @@ static bool kvm_set_valid_leaf_pte(kvm_pte_t *ptep, u64 pa, kvm_pte_t attr, · 1 unmodified line
181	184	pte \|= FIELD_PREP(KVM_PTE_TYPE, type);
182	185	pte \|= KVM_PTE_VALID;
183	186
184		− /* Tolerate KVM recreating the exact same mapping. */
185		− if (kvm_pte_valid(old))
186		− return old == pte;
187		−
188		− smp_store_release(ptep, pte);
189		− return true;
	187	+ return pte;
190	188	}
191	189
192	190	static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data *data, u64 addr,
@@ -341,12 +339,17 @@ static int hyp_map_set_prot_attr(enum kvm_pgtable_prot prot, · 148 unmodified lines
341	339	static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level,
342	340	kvm_pte_t ptep, struct hyp_map_data data)
343	341	{
	342	+ kvm_pte_t new, old = *ptep;
344	343	u64 granule = kvm_granule_size(level), phys = data->phys;
345	344
346	345	if (!kvm_block_mapping_supported(addr, end, phys, level))
347	346	return false;
348	347
349		− WARN_ON(!kvm_set_valid_leaf_pte(ptep, phys, data->attr, level));
	348	+ /* Tolerate KVM recreating the exact same mapping */
	349	+ new = kvm_init_valid_leaf_pte(phys, data->attr, level);
	350	+ if (old != new && !WARN_ON(kvm_pte_valid(old)))
	351	+ smp_store_release(ptep, new);
	352	+
350	353	data->phys += granule;
351	354	return true;
352	355	}
@@ -461,34 +464,41 @@ static int stage2_map_set_prot_attr(enum kvm_pgtable_prot prot, · 108 unmodified lines
461	464	return 0;
462	465	}
463	466
464		−static bool stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level,
465		− kvm_pte_t *ptep,
466		− struct stage2_map_data *data)
	467	+static int stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level,
	468	+ kvm_pte_t *ptep,
	469	+ struct stage2_map_data *data)
467	470	{
	471	+ kvm_pte_t new, old = *ptep;
468	472	u64 granule = kvm_granule_size(level), phys = data->phys;
	473	+ struct page *page = virt_to_page(ptep);
469	474
470	475	if (!kvm_block_mapping_supported(addr, end, phys, level))
471		− return false;
472		−
473		− /*
474		− * If the PTE was already valid, drop the refcount on the table
475		− * early, as it will be bumped-up again in stage2_map_walk_leaf().
476		− * This ensures that the refcount stays constant across a valid to
477		− * valid PTE update.
478		− */
479		− if (kvm_pte_valid(*ptep))
480		− put_page(virt_to_page(ptep));
481		−
482		− if (kvm_set_valid_leaf_pte(ptep, phys, data->attr, level))
483		− goto out;
	476	+ return -E2BIG;
	477	+
	478	+ new = kvm_init_valid_leaf_pte(phys, data->attr, level);
	479	+ if (kvm_pte_valid(old)) {
	480	+ /*
	481	+ * Skip updating the PTE if we are trying to recreate the exact
	482	+ * same mapping or only change the access permissions. Instead,
	483	+ * the vCPU will exit one more time from guest if still needed
	484	+ * and then go through the path of relaxing permissions.
	485	+ */
	486	+ if (!((old ^ new) & (~KVM_PTE_LEAF_ATTR_S2_PERMS)))
	487	+ return -EAGAIN;
	488	+
	489	+ /*
	490	+ * There's an existing different valid leaf entry, so perform
	491	+ * break-before-make.
	492	+ */
	493	+ kvm_set_invalid_pte(ptep);
	494	+ kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, level);
	495	+ put_page(page);
	496	+ }
484	497
485		− /* There's an existing valid leaf entry, so perform break-before-make */
486		− kvm_set_invalid_pte(ptep);
487		− kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, level);
488		− kvm_set_valid_leaf_pte(ptep, phys, data->attr, level);
489		−out:
	498	+ smp_store_release(ptep, new);
	499	+ get_page(page);
490	500	data->phys += granule;
491		− return true;
	501	+ return 0;
492	502	}
493	503
494	504	static int stage2_map_walk_table_pre(u64 addr, u64 end, u32 level,
@@ -516,6 +526,7 @@ static int stage2_map_walk_table_pre(u64 addr, u64 end, u32 level, · 21 unmodified lines
516	526	static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
517	527	struct stage2_map_data *data)
518	528	{
	529	+ int ret;
519	530	kvm_pte_t childp, pte = ptep;
520	531	struct page *page = virt_to_page(ptep);
521	532
@@ -526,8 +537,9 @@ static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 4 unmodified lines
526	537	return 0;
527	538	}
528	539
529		− if (stage2_map_walker_try_leaf(addr, end, level, ptep, data))
530		− goto out_get_page;
	540	+ ret = stage2_map_walker_try_leaf(addr, end, level, ptep, data);
	541	+ if (ret != -E2BIG)
	542	+ return ret;
531	543
532	544	if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1))
533	545	return -EINVAL;
@@ -551,9 +563,8 @@ static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 17 unmodified lines
551	563	}
552	564
553	565	kvm_set_table_pte(ptep, childp);
554		−
555		−out_get_page:
556	566	get_page(page);
	567	+
557	568	return 0;
558	569	}
559	570

arch/arm64/kvm/hyp/vgic-v2-cpuif-proxy.c+1−1

@@ -64,7 +64,7 @@ int __vgic_v2_perform_cpuif_access(struct kvm_vcpu *vcpu) · 63 unmodified lines
64	64	}
65	65
66	66	rd = kvm_vcpu_dabt_get_rd(vcpu);
67		− addr = hyp_symbol_addr(kvm_vgic_global_state)->vcpu_hyp_va;
	67	+ addr = kvm_vgic_global_state.vcpu_hyp_va;
68	68	addr += fault_ipa - vgic->vgic_cpu_base;
69	69
70	70	if (kvm_vcpu_dabt_iswrite(vcpu)) {

arch/arm64/kvm/hypercalls.c+6−0

@@ -71,6 +71,12 @@ int kvm_hvc_call_handler(struct kvm_vcpu *vcpu) · 70 unmodified lines
71	71	if (gpa != GPA_INVALID)
72	72	val = gpa;
73	73	break;
	74	+ case ARM_SMCCC_TRNG_VERSION:
	75	+ case ARM_SMCCC_TRNG_FEATURES:
	76	+ case ARM_SMCCC_TRNG_GET_UUID:
	77	+ case ARM_SMCCC_TRNG_RND32:
	78	+ case ARM_SMCCC_TRNG_RND64:
	79	+ return kvm_trng_call(vcpu);
74	80	default:
75	81	return kvm_psci_call(vcpu);
76	82	}

arch/arm64/kvm/mmu.c+8−5

@@ -879,11 +879,8 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 878 unmodified lines
879	879	if (vma_pagesize == PAGE_SIZE && !force_pte)
880	880	vma_pagesize = transparent_hugepage_adjust(memslot, hva,
881	881	&pfn, &fault_ipa);
882		− if (writable) {
	882	+ if (writable)
883	883	prot \|= KVM_PGTABLE_PROT_W;
884		− kvm_set_pfn_dirty(pfn);
885		− mark_page_dirty(kvm, gfn);
886		− }
887	884
888	885	if (fault_status != FSC_PERM && !device)
889	886	clean_dcache_guest_page(pfn, vma_pagesize);
@@ -911,11 +908,17 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 21 unmodified lines
911	908	memcache);
912	909	}
913	910
	911	+ /* Mark the page dirty only if the fault is handled successfully */
	912	+ if (writable && !ret) {
	913	+ kvm_set_pfn_dirty(pfn);
	914	+ mark_page_dirty(kvm, gfn);
	915	+ }
	916	+
914	917	out_unlock:
915	918	spin_unlock(&kvm->mmu_lock);
916	919	kvm_set_pfn_accessed(pfn);
917	920	kvm_release_pfn_clean(pfn);
918		− return ret;
	921	+ return ret != -EAGAIN ? ret : 0;
919	922	}
920	923
921	924	/* Resolve the access fault by making the page young again. */

arch/arm64/kvm/pmu-emul.c+10−4

@@ -23,11 +23,11 @@ static void kvm_pmu_stop_counter(struct kvm_vcpu vcpu, struct kvm_pmc pmc); · 22 unmodified lines
23	23	static u32 kvm_pmu_event_mask(struct kvm *kvm)
24	24	{
25	25	switch (kvm->arch.pmuver) {
26		− case 1: /* ARMv8.0 */
	26	+ case ID_AA64DFR0_PMUVER_8_0:
27	27	return GENMASK(9, 0);
28		− case 4: /* ARMv8.1 */
29		− case 5: /* ARMv8.4 */
30		− case 6: /* ARMv8.5 */
	28	+ case ID_AA64DFR0_PMUVER_8_1:
	29	+ case ID_AA64DFR0_PMUVER_8_4:
	30	+ case ID_AA64DFR0_PMUVER_8_5:
31	31	return GENMASK(15, 0);
32	32	default: /* Shouldn't be here, just for sanity */
33	33	WARN_ONCE(1, "Unknown PMU version %d\n", kvm->arch.pmuver);
@@ -795,6 +795,12 @@ u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1) · 761 unmodified lines
795	795	base = 0;
796	796	} else {
797	797	val = read_sysreg(pmceid1_el0);
	798	+ /*
	799	+ * Don't advertise STALL_SLOT, as PMMIR_EL0 is handled
	800	+ * as RAZ
	801	+ */
	802	+ if (vcpu->kvm->arch.pmuver >= ID_AA64DFR0_PMUVER_8_4)
	803	+ val &= ~BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT - 32);
798	804	base = 32;
799	805	}
800	806

arch/arm64/kvm/sys_regs.c+51−34

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	9	* Christoffer Dall <c.dall@virtualopensystems.com>
10	10	*/
11	11
	12	+#include <linux/bitfield.h>
12	13	#include <linux/bsearch.h>
13	14	#include <linux/kvm_host.h>
14	15	#include <linux/mm.h>
@@ -700,14 +701,18 @@ static bool access_pmselr(struct kvm_vcpu vcpu, struct sys_reg_params p, · 685 unmodified lines
700	701	static bool access_pmceid(struct kvm_vcpu vcpu, struct sys_reg_params p,
701	702	const struct sys_reg_desc *r)
702	703	{
703		− u64 pmceid;
	704	+ u64 pmceid, mask, shift;
704	705
705	706	BUG_ON(p->is_write);
706	707
707	708	if (pmu_access_el0_disabled(vcpu))
708	709	return false;
709	710
	711	+ get_access_mask(r, &mask, &shift);
	712	+
710	713	pmceid = kvm_pmu_get_pmceid(vcpu, (p->Op2 & 1));
	714	+ pmceid &= mask;
	715	+ pmceid >>= shift;
711	716
712	717	p->regval = pmceid;
713	718
@@ -1021,6 +1026,8 @@ static bool access_arch_timer(struct kvm_vcpu *vcpu, · 307 unmodified lines
1021	1026	return true;
1022	1027	}
1023	1028
	1029	+#define FEATURE(x) (GENMASK_ULL(x##_SHIFT + 3, x##_SHIFT))
	1030	+
1024	1031	/* Read a sanitised cpufeature ID register by sys_reg_desc */
1025	1032	static u64 read_id_reg(const struct kvm_vcpu *vcpu,
1026	1033	struct sys_reg_desc const *r, bool raz)
@@ -1028,36 +1035,41 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, · 1 unmodified line
1028	1035	u32 id = reg_to_encoding(r);
1029	1036	u64 val = raz ? 0 : read_sanitised_ftr_reg(id);
1030	1037
1031		− if (id == SYS_ID_AA64PFR0_EL1) {
	1038	+ switch (id) {
	1039	+ case SYS_ID_AA64PFR0_EL1:
1032	1040	if (!vcpu_has_sve(vcpu))
1033		− val &= ~(0xfUL << ID_AA64PFR0_SVE_SHIFT);
1034		− val &= ~(0xfUL << ID_AA64PFR0_AMU_SHIFT);
1035		− val &= ~(0xfUL << ID_AA64PFR0_CSV2_SHIFT);
1036		− val \|= ((u64)vcpu->kvm->arch.pfr0_csv2 << ID_AA64PFR0_CSV2_SHIFT);
1037		− val &= ~(0xfUL << ID_AA64PFR0_CSV3_SHIFT);
1038		− val \|= ((u64)vcpu->kvm->arch.pfr0_csv3 << ID_AA64PFR0_CSV3_SHIFT);
1039		− } else if (id == SYS_ID_AA64PFR1_EL1) {
1040		− val &= ~(0xfUL << ID_AA64PFR1_MTE_SHIFT);
1041		− } else if (id == SYS_ID_AA64ISAR1_EL1 && !vcpu_has_ptrauth(vcpu)) {
1042		− val &= ~((0xfUL << ID_AA64ISAR1_APA_SHIFT) \|
1043		− (0xfUL << ID_AA64ISAR1_API_SHIFT) \|
1044		− (0xfUL << ID_AA64ISAR1_GPA_SHIFT) \|
1045		− (0xfUL << ID_AA64ISAR1_GPI_SHIFT));
1046		− } else if (id == SYS_ID_AA64DFR0_EL1) {
1047		− u64 cap = 0;
1048		−
1049		− /* Limit guests to PMUv3 for ARMv8.1 */
1050		− if (kvm_vcpu_has_pmu(vcpu))
1051		− cap = ID_AA64DFR0_PMUVER_8_1;
1052		−
	1041	+ val &= ~FEATURE(ID_AA64PFR0_SVE);
	1042	+ val &= ~FEATURE(ID_AA64PFR0_AMU);
	1043	+ val &= ~FEATURE(ID_AA64PFR0_CSV2);
	1044	+ val \|= FIELD_PREP(FEATURE(ID_AA64PFR0_CSV2), (u64)vcpu->kvm->arch.pfr0_csv2);
	1045	+ val &= ~FEATURE(ID_AA64PFR0_CSV3);
	1046	+ val \|= FIELD_PREP(FEATURE(ID_AA64PFR0_CSV3), (u64)vcpu->kvm->arch.pfr0_csv3);
	1047	+ break;
	1048	+ case SYS_ID_AA64PFR1_EL1:
	1049	+ val &= ~FEATURE(ID_AA64PFR1_MTE);
	1050	+ break;
	1051	+ case SYS_ID_AA64ISAR1_EL1:
	1052	+ if (!vcpu_has_ptrauth(vcpu))
	1053	+ val &= ~(FEATURE(ID_AA64ISAR1_APA) \|
	1054	+ FEATURE(ID_AA64ISAR1_API) \|
	1055	+ FEATURE(ID_AA64ISAR1_GPA) \|
	1056	+ FEATURE(ID_AA64ISAR1_GPI));
	1057	+ break;
	1058	+ case SYS_ID_AA64DFR0_EL1:
	1059	+ /* Limit debug to ARMv8.0 */
	1060	+ val &= ~FEATURE(ID_AA64DFR0_DEBUGVER);
	1061	+ val \|= FIELD_PREP(FEATURE(ID_AA64DFR0_DEBUGVER), 6);
	1062	+ /* Limit guests to PMUv3 for ARMv8.4 */
1053	1063	val = cpuid_feature_cap_perfmon_field(val,
1054		− ID_AA64DFR0_PMUVER_SHIFT,
1055		− cap);
1056		− } else if (id == SYS_ID_DFR0_EL1) {
1057		− /* Limit guests to PMUv3 for ARMv8.1 */
	1064	+ ID_AA64DFR0_PMUVER_SHIFT,
	1065	+ kvm_vcpu_has_pmu(vcpu) ? ID_AA64DFR0_PMUVER_8_4 : 0);
	1066	+ break;
	1067	+ case SYS_ID_DFR0_EL1:
	1068	+ /* Limit guests to PMUv3 for ARMv8.4 */
1058	1069	val = cpuid_feature_cap_perfmon_field(val,
1059		− ID_DFR0_PERFMON_SHIFT,
1060		− ID_DFR0_PERFMON_8_1);
	1070	+ ID_DFR0_PERFMON_SHIFT,
	1071	+ kvm_vcpu_has_pmu(vcpu) ? ID_DFR0_PERFMON_8_4 : 0);
	1072	+ break;
1061	1073	}
1062	1074
1063	1075	return val;
@@ -1493,6 +1505,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 429 unmodified lines
1493	1505	.access = access_pminten, .reg = PMINTENSET_EL1 },
1494	1506	{ PMU_SYS_REG(SYS_PMINTENCLR_EL1),
1495	1507	.access = access_pminten, .reg = PMINTENSET_EL1 },
	1508	+ { SYS_DESC(SYS_PMMIR_EL1), trap_raz_wi },
1496	1509
1497	1510	{ SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 },
1498	1511	{ SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 },
@@ -1720,7 +1733,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 221 unmodified lines
1720	1733	{ SYS_DESC(SYS_FPEXC32_EL2), NULL, reset_val, FPEXC32_EL2, 0x700 },
1721	1734	};
1722	1735
1723		−static bool trap_dbgidr(struct kvm_vcpu *vcpu,
	1736	+static bool trap_dbgdidr(struct kvm_vcpu *vcpu,
1724	1737	struct sys_reg_params *p,
1725	1738	const struct sys_reg_desc *r)
1726	1739	{
@@ -1734,7 +1747,7 @@ static bool trap_dbgidr(struct kvm_vcpu *vcpu, · 7 unmodified lines
1734	1747	p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) \|
1735	1748	(((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) \|
1736	1749	(((dfr >> ID_AA64DFR0_CTX_CMPS_SHIFT) & 0xf) << 20)
1737		− \| (6 << 16) \| (el3 << 14) \| (el3 << 12));
	1750	+ \| (6 << 16) \| (1 << 15) \| (el3 << 14) \| (el3 << 12));
1738	1751	return true;
1739	1752	}
1740	1753	}
@@ -1767,8 +1780,8 @@ static bool trap_dbgidr(struct kvm_vcpu *vcpu, · 26 unmodified lines
1767	1780	* guest. Revisit this one day, would this principle change.
1768	1781	*/
1769	1782	static const struct sys_reg_desc cp14_regs[] = {
1770		− /* DBGIDR */
1771		− { Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgidr },
	1783	+ /* DBGDIDR */
	1784	+ { Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgdidr },
1772	1785	/* DBGDTRRXext */
1773	1786	{ Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },
1774	1787
@@ -1918,8 +1931,8 @@ static const struct sys_reg_desc cp15_regs[] = { · 143 unmodified lines
1918	1931	{ Op1( 0), CRn( 9), CRm(12), Op2( 3), access_pmovs },
1919	1932	{ Op1( 0), CRn( 9), CRm(12), Op2( 4), access_pmswinc },
1920	1933	{ Op1( 0), CRn( 9), CRm(12), Op2( 5), access_pmselr },
1921		− { Op1( 0), CRn( 9), CRm(12), Op2( 6), access_pmceid },
1922		− { Op1( 0), CRn( 9), CRm(12), Op2( 7), access_pmceid },
	1934	+ { AA32(LO), Op1( 0), CRn( 9), CRm(12), Op2( 6), access_pmceid },
	1935	+ { AA32(LO), Op1( 0), CRn( 9), CRm(12), Op2( 7), access_pmceid },
1923	1936	{ Op1( 0), CRn( 9), CRm(13), Op2( 0), access_pmu_evcntr },
1924	1937	{ Op1( 0), CRn( 9), CRm(13), Op2( 1), access_pmu_evtyper },
1925	1938	{ Op1( 0), CRn( 9), CRm(13), Op2( 2), access_pmu_evcntr },
@@ -1927,6 +1940,10 @@ static const struct sys_reg_desc cp15_regs[] = { · 1 unmodified line
1927	1940	{ Op1( 0), CRn( 9), CRm(14), Op2( 1), access_pminten },
1928	1941	{ Op1( 0), CRn( 9), CRm(14), Op2( 2), access_pminten },
1929	1942	{ Op1( 0), CRn( 9), CRm(14), Op2( 3), access_pmovs },
	1943	+ { AA32(HI), Op1( 0), CRn( 9), CRm(14), Op2( 4), access_pmceid },
	1944	+ { AA32(HI), Op1( 0), CRn( 9), CRm(14), Op2( 5), access_pmceid },
	1945	+ /* PMMIR */
	1946	+ { Op1( 0), CRn( 9), CRm(14), Op2( 6), trap_raz_wi },
1930	1947
1931	1948	/* PRRR/MAIR0 */
1932	1949	{ AA32(LO), Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, MAIR_EL1 },

arch/arm64/kvm/trng.c+85−0

@@ -0,0 +1,85 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+// Copyright (C) 2020 Arm Ltd.
	3	+
	4	+#include <linux/arm-smccc.h>
	5	+#include <linux/kvm_host.h>
	6	+
	7	+#include <asm/kvm_emulate.h>
	8	+
	9	+#include <kvm/arm_hypercalls.h>
	10	+
	11	+#define ARM_SMCCC_TRNG_VERSION_1_0 0x10000UL
	12	+
	13	+/* Those values are deliberately separate from the generic SMCCC definitions. */
	14	+#define TRNG_SUCCESS 0UL
	15	+#define TRNG_NOT_SUPPORTED ((unsigned long)-1)
	16	+#define TRNG_INVALID_PARAMETER ((unsigned long)-2)
	17	+#define TRNG_NO_ENTROPY ((unsigned long)-3)
	18	+
	19	+#define TRNG_MAX_BITS64 192
	20	+
	21	+static const uuid_t arm_smc_trng_uuid __aligned(4) = UUID_INIT(
	22	+ 0x0d21e000, 0x4384, 0x11eb, 0x80, 0x70, 0x52, 0x44, 0x55, 0x4e, 0x5a, 0x4c);
	23	+
	24	+static int kvm_trng_do_rnd(struct kvm_vcpu *vcpu, int size)
	25	+{
	26	+ DECLARE_BITMAP(bits, TRNG_MAX_BITS64);
	27	+ u32 num_bits = smccc_get_arg1(vcpu);
	28	+ int i;
	29	+
	30	+ if (num_bits > 3 * size) {
	31	+ smccc_set_retval(vcpu, TRNG_INVALID_PARAMETER, 0, 0, 0);
	32	+ return 1;
	33	+ }
	34	+
	35	+ /* get as many bits as we need to fulfil the request */
	36	+ for (i = 0; i < DIV_ROUND_UP(num_bits, BITS_PER_LONG); i++)
	37	+ bits[i] = get_random_long();
	38	+
	39	+ bitmap_clear(bits, num_bits, TRNG_MAX_BITS64 - num_bits);
	40	+
	41	+ if (size == 32)
	42	+ smccc_set_retval(vcpu, TRNG_SUCCESS, lower_32_bits(bits[1]),
	43	+ upper_32_bits(bits[0]), lower_32_bits(bits[0]));
	44	+ else
	45	+ smccc_set_retval(vcpu, TRNG_SUCCESS, bits[2], bits[1], bits[0]);
	46	+
	47	+ memzero_explicit(bits, sizeof(bits));
	48	+ return 1;
	49	+}
	50	+
	51	+int kvm_trng_call(struct kvm_vcpu *vcpu)
	52	+{
	53	+ const __le32 u = (__le32 )arm_smc_trng_uuid.b;
	54	+ u32 func_id = smccc_get_function(vcpu);
	55	+ unsigned long val = TRNG_NOT_SUPPORTED;
	56	+ int size = 64;
	57	+
	58	+ switch (func_id) {
	59	+ case ARM_SMCCC_TRNG_VERSION:
	60	+ val = ARM_SMCCC_TRNG_VERSION_1_0;
	61	+ break;
	62	+ case ARM_SMCCC_TRNG_FEATURES:
	63	+ switch (smccc_get_arg1(vcpu)) {
	64	+ case ARM_SMCCC_TRNG_VERSION:
	65	+ case ARM_SMCCC_TRNG_FEATURES:
	66	+ case ARM_SMCCC_TRNG_GET_UUID:
	67	+ case ARM_SMCCC_TRNG_RND32:
	68	+ case ARM_SMCCC_TRNG_RND64:
	69	+ val = TRNG_SUCCESS;
	70	+ }
	71	+ break;
	72	+ case ARM_SMCCC_TRNG_GET_UUID:
	73	+ smccc_set_retval(vcpu, le32_to_cpu(u[0]), le32_to_cpu(u[1]),
	74	+ le32_to_cpu(u[2]), le32_to_cpu(u[3]));
	75	+ return 1;
	76	+ case ARM_SMCCC_TRNG_RND32:
	77	+ size = 32;
	78	+ fallthrough;
	79	+ case ARM_SMCCC_TRNG_RND64:
	80	+ return kvm_trng_do_rnd(vcpu, size);
	81	+ }
	82	+
	83	+ smccc_set_retval(vcpu, val, 0, 0, 0);
	84	+ return 1;
	85	+}

arch/arm64/kvm/va_layout.c+28−6

@@ -81,6 +81,34 @@ __init void kvm_compute_layout(void) · 80 unmodified lines
81	81	init_hyp_physvirt_offset();
82	82	}
83	83
	84	+/*
	85	+ * The .hyp.reloc ELF section contains a list of kimg positions that
	86	+ * contains kimg VAs but will be accessed only in hyp execution context.
	87	+ * Convert them to hyp VAs. See gen-hyprel.c for more details.
	88	+ */
	89	+__init void kvm_apply_hyp_relocations(void)
	90	+{
	91	+ int32_t *rel;
	92	+ int32_t begin = (int32_t )__hyp_reloc_begin;
	93	+ int32_t end = (int32_t )__hyp_reloc_end;
	94	+
	95	+ for (rel = begin; rel < end; ++rel) {
	96	+ uintptr_t *ptr, kimg_va;
	97	+
	98	+ /*
	99	+ * Each entry contains a 32-bit relative offset from itself
	100	+ * to a kimg VA position.
	101	+ */
	102	+ ptr = (uintptr_t )lm_alias((char )rel + *rel);
	103	+
	104	+ /* Read the kimg VA value at the relocation address. */
	105	+ kimg_va = *ptr;
	106	+
	107	+ /* Convert to hyp VA and store back to the relocation address. */
	108	+ *ptr = __early_kern_hyp_va((uintptr_t)lm_alias(kimg_va));
	109	+ }
	110	+}
	111	+
84	112	static u32 compute_instruction(int n, u32 rd, u32 rn)
85	113	{
86	114	u32 insn = AARCH64_BREAK_FAULT;
@@ -255,12 +283,6 @@ static void generate_mov_q(u64 val, __le32 origptr, __le32 updptr, int nr_inst · 168 unmodified lines
255	283	*updptr++ = cpu_to_le32(insn);
256	284	}
257	285
258		−void kvm_update_kimg_phys_offset(struct alt_instr *alt,
259		− __le32 origptr, __le32 updptr, int nr_inst)
260		−{
261		− generate_mov_q(kimage_voffset + PHYS_OFFSET, origptr, updptr, nr_inst);
262		−}
263		−
264	286	void kvm_get_kimage_voffset(struct alt_instr *alt,
265	287	__le32 origptr, __le32 updptr, int nr_inst)
266	288	{

arch/mips/include/asm/kvm_host.h+0−1

@@ -83,7 +83,6 @@ · 82 unmodified lines
83	83
84	84
85	85	#define KVM_MAX_VCPUS 16
86		−#define KVM_USER_MEM_SLOTS 16
87	86	/* memory slots that does not exposed to userspace */
88	87	#define KVM_PRIVATE_MEM_SLOTS 0
89	88

arch/mips/include/asm/spinlock.h+1−1

@@ -10,7 +10,6 @@ · 9 unmodified lines
10	10	#define _ASM_SPINLOCK_H
11	11
12	12	#include <asm/processor.h>
13		−#include <asm/qrwlock.h>
14	13
15	14	#include <asm-generic/qspinlock_types.h>
16	15
@@ -27,5 +26,6 @@ static inline void queued_spin_unlock(struct qspinlock *lock) · 10 unmodified lines
27	26	}
28	27
29	28	#include <asm/qspinlock.h>
	29	+#include <asm/qrwlock.h>
30	30
31	31	#endif /* _ASM_SPINLOCK_H */

arch/powerpc/include/asm/hvcall.h+22−3

@@ -535,9 +535,12 @@ struct h_cpu_char_result { · 534 unmodified lines
535	535	u64 behaviour;
536	536	};
537	537
538		−/* Register state for entering a nested guest with H_ENTER_NESTED */
	538	+/*
	539	+ * Register state for entering a nested guest with H_ENTER_NESTED.
	540	+ * New member must be added at the end.
	541	+ */
539	542	struct hv_guest_state {
540		− u64 version; /* version of this structure layout */
	543	+ u64 version; /* version of this structure layout, must be first */
541	544	u32 lpid;
542	545	u32 vcpu_token;
543	546	/* These registers are hypervisor privileged (at least for writing) */
@@ -566,10 +569,26 @@ struct hv_guest_state { · 22 unmodified lines
566	569	u64 pidr;
567	570	u64 cfar;
568	571	u64 ppr;
	572	+ /* Version 1 ends here */
	573	+ u64 dawr1;
	574	+ u64 dawrx1;
	575	+ /* Version 2 ends here */
569	576	};
570	577
571	578	/* Latest version of hv_guest_state structure */
572		−#define HV_GUEST_STATE_VERSION 1
	579	+#define HV_GUEST_STATE_VERSION 2
	580	+
	581	+static inline int hv_guest_state_size(unsigned int version)
	582	+{
	583	+ switch (version) {
	584	+ case 1:
	585	+ return offsetofend(struct hv_guest_state, ppr);
	586	+ case 2:
	587	+ return offsetofend(struct hv_guest_state, dawrx1);
	588	+ default:
	589	+ return -1;
	590	+ }
	591	+}
573	592
574	593	/*
575	594	* From the document "H_GetPerformanceCounterInfo Interface" v1.07

arch/powerpc/include/asm/kvm_book3s_asm.h+0−11

@@ -74,16 +74,6 @@ struct kvm_split_mode { · 73 unmodified lines
74	74	u8 do_nap;
75	75	u8 napped[MAX_SMT_THREADS];
76	76	struct kvmppc_vcore *vc[MAX_SUBCORES];
77		− /* Bits for changing lpcr on P9 */
78		− unsigned long lpcr_req;
79		− unsigned long lpidr_req;
80		− unsigned long host_lpcr;
81		− u32 do_set;
82		− u32 do_restore;
83		− union {
84		− u32 allphases;
85		− u8 phase[4];
86		− } lpcr_sync;
87	77	};
88	78
89	79	/*
@@ -110,7 +100,6 @@ struct kvmppc_host_state { · 20 unmodified lines
110	100	u8 hwthread_state;
111	101	u8 host_ipi;
112	102	u8 ptid; /* thread number within subcore when split */
113		− u8 tid; /* thread number within whole core */
114	103	u8 fake_suspend;
115	104	struct kvm_vcpu *kvm_vcpu;
116	105	struct kvmppc_vcore *kvm_vcore;

arch/powerpc/include/asm/kvm_host.h+5−3

@@ -28,7 +28,6 @@ · 27 unmodified lines
28	28
29	29	#define KVM_MAX_VCPUS NR_CPUS
30	30	#define KVM_MAX_VCORES NR_CPUS
31		−#define KVM_USER_MEM_SLOTS 512
32	31
33	32	#include <asm/cputhreads.h>
34	33
@@ -307,6 +306,7 @@ struct kvm_arch { · 272 unmodified lines
307	306	u8 svm_enabled;
308	307	bool threads_indep;
309	308	bool nested_enable;
	309	+ bool dawr1_enabled;
310	310	pgd_t *pgtable;
311	311	u64 process_table;
312	312	struct dentry *debugfs_dir;
@@ -584,8 +584,10 @@ struct kvm_vcpu_arch { · 271 unmodified lines
584	584	u32 ctrl;
585	585	u32 dabrx;
586	586	ulong dabr;
587		− ulong dawr;
588		− ulong dawrx;
	587	+ ulong dawr0;
	588	+ ulong dawrx0;
	589	+ ulong dawr1;
	590	+ ulong dawrx1;
589	591	ulong ciabr;
590	592	ulong cfar;
591	593	ulong ppr;

arch/powerpc/include/asm/kvm_ppc.h+2−0

@@ -314,6 +314,8 @@ struct kvmppc_ops { · 313 unmodified lines
314	314	int size);
315	315	int (enable_svm)(struct kvm kvm);
316	316	int (svm_off)(struct kvm kvm);
	317	+ int (enable_dawr1)(struct kvm kvm);
	318	+ bool (*hash_v3_possible)(void);
317	319	};
318	320
319	321	extern struct kvmppc_ops *kvmppc_hv_ops;

arch/powerpc/include/uapi/asm/kvm.h+2−0

@@ -644,6 +644,8 @@ struct kvm_ppc_cpu_char { · 643 unmodified lines
644	644	#define KVM_REG_PPC_MMCR3 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc1)
645	645	#define KVM_REG_PPC_SIER2 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc2)
646	646	#define KVM_REG_PPC_SIER3 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc3)
	647	+#define KVM_REG_PPC_DAWR1 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc4)
	648	+#define KVM_REG_PPC_DAWRX1 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc5)
647	649
648	650	/* Transactional Memory checkpointed state:
649	651	* This is all GPRs, all VSX regs and a subset of SPRs

arch/powerpc/kernel/asm-offsets.c+4−5

@@ -526,8 +526,10 @@ int main(void) · 525 unmodified lines
526	526	OFFSET(VCPU_CTRL, kvm_vcpu, arch.ctrl);
527	527	OFFSET(VCPU_DABR, kvm_vcpu, arch.dabr);
528	528	OFFSET(VCPU_DABRX, kvm_vcpu, arch.dabrx);
529		− OFFSET(VCPU_DAWR, kvm_vcpu, arch.dawr);
530		− OFFSET(VCPU_DAWRX, kvm_vcpu, arch.dawrx);
	529	+ OFFSET(VCPU_DAWR0, kvm_vcpu, arch.dawr0);
	530	+ OFFSET(VCPU_DAWRX0, kvm_vcpu, arch.dawrx0);
	531	+ OFFSET(VCPU_DAWR1, kvm_vcpu, arch.dawr1);
	532	+ OFFSET(VCPU_DAWRX1, kvm_vcpu, arch.dawrx1);
531	533	OFFSET(VCPU_CIABR, kvm_vcpu, arch.ciabr);
532	534	OFFSET(VCPU_HFLAGS, kvm_vcpu, arch.hflags);
533	535	OFFSET(VCPU_DEC, kvm_vcpu, arch.dec);
@@ -668,7 +670,6 @@ int main(void) · 134 unmodified lines
668	670	HSTATE_FIELD(HSTATE_SAVED_XIRR, saved_xirr);
669	671	HSTATE_FIELD(HSTATE_HOST_IPI, host_ipi);
670	672	HSTATE_FIELD(HSTATE_PTID, ptid);
671		− HSTATE_FIELD(HSTATE_TID, tid);
672	673	HSTATE_FIELD(HSTATE_FAKE_SUSPEND, fake_suspend);
673	674	HSTATE_FIELD(HSTATE_MMCR0, host_mmcr[0]);
674	675	HSTATE_FIELD(HSTATE_MMCR1, host_mmcr[1]);
@@ -698,8 +699,6 @@ int main(void) · 23 unmodified lines
698	699	OFFSET(KVM_SPLIT_LDBAR, kvm_split_mode, ldbar);
699	700	OFFSET(KVM_SPLIT_DO_NAP, kvm_split_mode, do_nap);
700	701	OFFSET(KVM_SPLIT_NAPPED, kvm_split_mode, napped);
701		− OFFSET(KVM_SPLIT_DO_SET, kvm_split_mode, do_set);
702		− OFFSET(KVM_SPLIT_DO_RESTORE, kvm_split_mode, do_restore);
703	702	#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */
704	703
705	704	#ifdef CONFIG_PPC_BOOK3S_64

arch/powerpc/kvm/book3s_hv.c+93−56

@@ -134,7 +134,7 @@ static inline bool nesting_enabled(struct kvm *kvm) · 133 unmodified lines
134	134	}
135	135
136	136	/* If set, the threads on each CPU core have to be in the same MMU mode */
137		−static bool no_mixing_hpt_and_radix;
	137	+static bool no_mixing_hpt_and_radix __read_mostly;
138	138
139	139	static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
140	140
@@ -782,8 +782,24 @@ static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags, · 641 unmodified lines
782	782	return H_UNSUPPORTED_FLAG_START;
783	783	if (value2 & DABRX_HYP)
784	784	return H_P4;
785		− vcpu->arch.dawr = value1;
786		− vcpu->arch.dawrx = value2;
	785	+ vcpu->arch.dawr0 = value1;
	786	+ vcpu->arch.dawrx0 = value2;
	787	+ return H_SUCCESS;
	788	+ case H_SET_MODE_RESOURCE_SET_DAWR1:
	789	+ if (!kvmppc_power8_compatible(vcpu))
	790	+ return H_P2;
	791	+ if (!ppc_breakpoint_available())
	792	+ return H_P2;
	793	+ if (!cpu_has_feature(CPU_FTR_DAWR1))
	794	+ return H_P2;
	795	+ if (!vcpu->kvm->arch.dawr1_enabled)
	796	+ return H_FUNCTION;
	797	+ if (mflags)
	798	+ return H_UNSUPPORTED_FLAG_START;
	799	+ if (value2 & DABRX_HYP)
	800	+ return H_P4;
	801	+ vcpu->arch.dawr1 = value1;
	802	+ vcpu->arch.dawrx1 = value2;
787	803	return H_SUCCESS;
788	804	case H_SET_MODE_RESOURCE_ADDR_TRANS_MODE:
789	805	/* KVM does not support mflags=2 (AIL=2) */
@@ -1759,10 +1775,16 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, · 969 unmodified lines
1759	1775	*val = get_reg_val(id, vcpu->arch.vcore->vtb);
1760	1776	break;
1761	1777	case KVM_REG_PPC_DAWR:
1762		− *val = get_reg_val(id, vcpu->arch.dawr);
	1778	+ *val = get_reg_val(id, vcpu->arch.dawr0);
1763	1779	break;
1764	1780	case KVM_REG_PPC_DAWRX:
1765		− *val = get_reg_val(id, vcpu->arch.dawrx);
	1781	+ *val = get_reg_val(id, vcpu->arch.dawrx0);
	1782	+ break;
	1783	+ case KVM_REG_PPC_DAWR1:
	1784	+ *val = get_reg_val(id, vcpu->arch.dawr1);
	1785	+ break;
	1786	+ case KVM_REG_PPC_DAWRX1:
	1787	+ *val = get_reg_val(id, vcpu->arch.dawrx1);
1766	1788	break;
1767	1789	case KVM_REG_PPC_CIABR:
1768	1790	*val = get_reg_val(id, vcpu->arch.ciabr);
@@ -1991,10 +2013,16 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, · 222 unmodified lines
1991	2013	vcpu->arch.vcore->vtb = set_reg_val(id, *val);
1992	2014	break;
1993	2015	case KVM_REG_PPC_DAWR:
1994		− vcpu->arch.dawr = set_reg_val(id, *val);
	2016	+ vcpu->arch.dawr0 = set_reg_val(id, *val);
1995	2017	break;
1996	2018	case KVM_REG_PPC_DAWRX:
1997		− vcpu->arch.dawrx = set_reg_val(id, *val) & ~DAWRX_HYP;
	2019	+ vcpu->arch.dawrx0 = set_reg_val(id, *val) & ~DAWRX_HYP;
	2020	+ break;
	2021	+ case KVM_REG_PPC_DAWR1:
	2022	+ vcpu->arch.dawr1 = set_reg_val(id, *val);
	2023	+ break;
	2024	+ case KVM_REG_PPC_DAWRX1:
	2025	+ vcpu->arch.dawrx1 = set_reg_val(id, *val) & ~DAWRX_HYP;
1998	2026	break;
1999	2027	case KVM_REG_PPC_CIABR:
2000	2028	vcpu->arch.ciabr = set_reg_val(id, *val);
@@ -2862,11 +2890,6 @@ static bool can_dynamic_split(struct kvmppc_vcore vc, struct core_info cip) · 861 unmodified lines
2862	2890	if (one_vm_per_core && vc->kvm != cip->vc[0]->kvm)
2863	2891	return false;
2864	2892
2865		− /* Some POWER9 chips require all threads to be in the same MMU mode */
2866		− if (no_mixing_hpt_and_radix &&
2867		− kvm_is_radix(vc->kvm) != kvm_is_radix(cip->vc[0]->kvm))
2868		− return false;
2869		−
2870	2893	if (n_threads < cip->max_subcore_threads)
2871	2894	n_threads = cip->max_subcore_threads;
2872	2895	if (!subcore_config_ok(cip->n_subcores + 1, n_threads))
@@ -2905,6 +2928,9 @@ static void prepare_threads(struct kvmppc_vcore *vc) · 32 unmodified lines
2905	2928	for_each_runnable_thread(i, vcpu, vc) {
2906	2929	if (signal_pending(vcpu->arch.run_task))
2907	2930	vcpu->arch.ret = -EINTR;
	2931	+ else if (no_mixing_hpt_and_radix &&
	2932	+ kvm_is_radix(vc->kvm) != radix_enabled())
	2933	+ vcpu->arch.ret = -EINVAL;
2908	2934	else if (vcpu->arch.vpa.update_pending \|\|
2909	2935	vcpu->arch.slb_shadow.update_pending \|\|
2910	2936	vcpu->arch.dtl.update_pending)
@@ -3110,7 +3136,6 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) · 199 unmodified lines
3110	3136	int controlled_threads;
3111	3137	int trap;
3112	3138	bool is_power8;
3113		− bool hpt_on_radix;
3114	3139
3115	3140	/*
3116	3141	* Remove from the list any threads that have a signal pending
@@ -3143,11 +3168,8 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) · 26 unmodified lines
3143	3168	* this is a HPT guest on a radix host machine where the
3144	3169	* CPU threads may not be in different MMU modes.
3145	3170	*/
3146		− hpt_on_radix = no_mixing_hpt_and_radix && radix_enabled() &&
3147		− !kvm_is_radix(vc->kvm);
3148		− if (((controlled_threads > 1) &&
3149		− ((vc->num_threads > threads_per_subcore) \|\| !on_primary_thread())) \|\|
3150		− (hpt_on_radix && vc->kvm->arch.threads_indep)) {
	3171	+ if ((controlled_threads > 1) &&
	3172	+ ((vc->num_threads > threads_per_subcore) \|\| !on_primary_thread())) {
3151	3173	for_each_runnable_thread(i, vcpu, vc) {
3152	3174	vcpu->arch.ret = -EBUSY;
3153	3175	kvmppc_remove_runnable(vc, vcpu);
@@ -3215,7 +3237,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) · 61 unmodified lines
3215	3237	is_power8 = cpu_has_feature(CPU_FTR_ARCH_207S)
3216	3238	&& !cpu_has_feature(CPU_FTR_ARCH_300);
3217	3239
3218		− if (split > 1 \|\| hpt_on_radix) {
	3240	+ if (split > 1) {
3219	3241	sip = &split_info;
3220	3242	memset(&split_info, 0, sizeof(split_info));
3221	3243	for (sub = 0; sub < core_info.n_subcores; ++sub)
@@ -3237,13 +3259,6 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) · 15 unmodified lines
3237	3259	split_info.subcore_size = subcore_size;
3238	3260	} else {
3239	3261	split_info.subcore_size = 1;
3240		− if (hpt_on_radix) {
3241		− /* Use the split_info for LPCR/LPIDR changes */
3242		− split_info.lpcr_req = vc->lpcr;
3243		− split_info.lpidr_req = vc->kvm->arch.lpid;
3244		− split_info.host_lpcr = vc->kvm->arch.host_lpcr;
3245		− split_info.do_set = 1;
3246		− }
3247	3262	}
3248	3263
3249	3264	/* order writes to split_info before kvm_split_mode pointer */
@@ -3253,7 +3268,6 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) · 3 unmodified lines
3253	3268	for (thr = 0; thr < controlled_threads; ++thr) {
3254	3269	struct paca_struct *paca = paca_ptrs[pcpu + thr];
3255	3270
3256		− paca->kvm_hstate.tid = thr;
3257	3271	paca->kvm_hstate.napping = 0;
3258	3272	paca->kvm_hstate.kvm_split_mode = sip;
3259	3273	}
@@ -3327,10 +3341,8 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) · 67 unmodified lines
3327	3341	* When doing micro-threading, poke the inactive threads as well.
3328	3342	* This gets them to the nap instruction after kvm_do_nap,
3329	3343	* which reduces the time taken to unsplit later.
3330		− * For POWER9 HPT guest on radix host, we need all the secondary
3331		− * threads woken up so they can do the LPCR/LPIDR change.
3332	3344	*/
3333		− if (cmd_bit \|\| hpt_on_radix) {
	3345	+ if (cmd_bit) {
3334	3346	split_info.do_nap = 1; /* ask secondaries to nap when done */
3335	3347	for (thr = 1; thr < threads_per_subcore; ++thr)
3336	3348	if (!(active & (1 << thr)))
@@ -3391,19 +3403,8 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) · 54 unmodified lines
3391	3403	cpu_relax();
3392	3404	++loops;
3393	3405	}
3394		− } else if (hpt_on_radix) {
3395		− /* Wait for all threads to have seen final sync */
3396		− for (thr = 1; thr < controlled_threads; ++thr) {
3397		− struct paca_struct *paca = paca_ptrs[pcpu + thr];
3398		−
3399		− while (paca->kvm_hstate.kvm_split_mode) {
3400		− HMT_low();
3401		− barrier();
3402		− }
3403		− HMT_medium();
3404		− }
	3406	+ split_info.do_nap = 0;
3405	3407	}
3406		− split_info.do_nap = 0;
3407	3408
3408	3409	kvmppc_set_host_core(pcpu);
3409	3410
@@ -3449,10 +3450,17 @@ static int kvmhv_load_hv_regs_and_go(struct kvm_vcpu *vcpu, u64 time_limit, · 39 unmodified lines
3449	3450	int trap;
3450	3451	unsigned long host_hfscr = mfspr(SPRN_HFSCR);
3451	3452	unsigned long host_ciabr = mfspr(SPRN_CIABR);
3452		− unsigned long host_dawr = mfspr(SPRN_DAWR0);
3453		− unsigned long host_dawrx = mfspr(SPRN_DAWRX0);
	3453	+ unsigned long host_dawr0 = mfspr(SPRN_DAWR0);
	3454	+ unsigned long host_dawrx0 = mfspr(SPRN_DAWRX0);
3454	3455	unsigned long host_psscr = mfspr(SPRN_PSSCR);
3455	3456	unsigned long host_pidr = mfspr(SPRN_PID);
	3457	+ unsigned long host_dawr1 = 0;
	3458	+ unsigned long host_dawrx1 = 0;
	3459	+
	3460	+ if (cpu_has_feature(CPU_FTR_DAWR1)) {
	3461	+ host_dawr1 = mfspr(SPRN_DAWR1);
	3462	+ host_dawrx1 = mfspr(SPRN_DAWRX1);
	3463	+ }
3456	3464
3457	3465	/*
3458	3466	* P8 and P9 suppress the HDEC exception when LPCR[HDICE] = 0,
@@ -3489,8 +3497,12 @@ static int kvmhv_load_hv_regs_and_go(struct kvm_vcpu *vcpu, u64 time_limit, · 30 unmodified lines
3489	3497	mtspr(SPRN_SPURR, vcpu->arch.spurr);
3490	3498
3491	3499	if (dawr_enabled()) {
3492		− mtspr(SPRN_DAWR0, vcpu->arch.dawr);
3493		− mtspr(SPRN_DAWRX0, vcpu->arch.dawrx);
	3500	+ mtspr(SPRN_DAWR0, vcpu->arch.dawr0);
	3501	+ mtspr(SPRN_DAWRX0, vcpu->arch.dawrx0);
	3502	+ if (cpu_has_feature(CPU_FTR_DAWR1)) {
	3503	+ mtspr(SPRN_DAWR1, vcpu->arch.dawr1);
	3504	+ mtspr(SPRN_DAWRX1, vcpu->arch.dawrx1);
	3505	+ }
3494	3506	}
3495	3507	mtspr(SPRN_CIABR, vcpu->arch.ciabr);
3496	3508	mtspr(SPRN_IC, vcpu->arch.ic);
@@ -3542,8 +3554,12 @@ static int kvmhv_load_hv_regs_and_go(struct kvm_vcpu *vcpu, u64 time_limit, · 45 unmodified lines
3542	3554	(local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG));
3543	3555	mtspr(SPRN_HFSCR, host_hfscr);
3544	3556	mtspr(SPRN_CIABR, host_ciabr);
3545		− mtspr(SPRN_DAWR0, host_dawr);
3546		− mtspr(SPRN_DAWRX0, host_dawrx);
	3557	+ mtspr(SPRN_DAWR0, host_dawr0);
	3558	+ mtspr(SPRN_DAWRX0, host_dawrx0);
	3559	+ if (cpu_has_feature(CPU_FTR_DAWR1)) {
	3560	+ mtspr(SPRN_DAWR1, host_dawr1);
	3561	+ mtspr(SPRN_DAWRX1, host_dawrx1);
	3562	+ }
3547	3563	mtspr(SPRN_PID, host_pidr);
3548	3564
3549	3565	/*
@@ -3595,6 +3611,7 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, · 45 unmodified lines
3595	3611	unsigned long host_tidr = mfspr(SPRN_TIDR);
3596	3612	unsigned long host_iamr = mfspr(SPRN_IAMR);
3597	3613	unsigned long host_amr = mfspr(SPRN_AMR);
	3614	+ unsigned long host_fscr = mfspr(SPRN_FSCR);
3598	3615	s64 dec;
3599	3616	u64 tb;
3600	3617	int trap, save_pmu;
@@ -3735,6 +3752,9 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, · 134 unmodified lines
3735	3752	if (host_amr != vcpu->arch.amr)
3736	3753	mtspr(SPRN_AMR, host_amr);
3737	3754
	3755	+ if (host_fscr != vcpu->arch.fscr)
	3756	+ mtspr(SPRN_FSCR, host_fscr);
	3757	+
3738	3758	msr_check_and_set(MSR_FP \| MSR_VEC \| MSR_VSX);
3739	3759	store_fp_state(&vcpu->arch.fp);
3740	3760	#ifdef CONFIG_ALTIVEC
@@ -4173,7 +4193,6 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, · 432 unmodified lines
4173	4193
4174	4194	kvmppc_clear_host_core(pcpu);
4175	4195
4176		− local_paca->kvm_hstate.tid = 0;
4177	4196	local_paca->kvm_hstate.napping = 0;
4178	4197	local_paca->kvm_hstate.kvm_split_mode = NULL;
4179	4198	kvmppc_start_thread(vcpu, vc);
@@ -4358,15 +4377,11 @@ static int kvmppc_vcpu_run_hv(struct kvm_vcpu *vcpu) · 178 unmodified lines
4358	4377
4359	4378	do {
4360	4379	/*
4361		− * The early POWER9 chips that can't mix radix and HPT threads
4362		− * on the same core also need the workaround for the problem
4363		− * where the TLB would prefetch entries in the guest exit path
4364		− * for radix guests using the guest PIDR value and LPID 0.
4365		− * The workaround is in the old path (kvmppc_run_vcpu())
4366		− * but not the new path (kvmhv_run_single_vcpu()).
	4380	+ * The TLB prefetch bug fixup is only in the kvmppc_run_vcpu
	4381	+ * path, which also handles hash and dependent threads mode.
4367	4382	*/
4368	4383	if (kvm->arch.threads_indep && kvm_is_radix(kvm) &&
4369		− !no_mixing_hpt_and_radix)
	4384	+ !cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG))
4370	4385	r = kvmhv_run_single_vcpu(vcpu, ~(u64)0,
4371	4386	vcpu->arch.vcore->lpcr);
4372	4387	else
@@ -5599,6 +5614,26 @@ out: · 1226 unmodified lines
5599	5614	return ret;
5600	5615	}
5601	5616
	5617	+static int kvmhv_enable_dawr1(struct kvm *kvm)
	5618	+{
	5619	+ if (!cpu_has_feature(CPU_FTR_DAWR1))
	5620	+ return -ENODEV;
	5621	+
	5622	+ /* kvm == NULL means the caller is testing if the capability exists */
	5623	+ if (kvm)
	5624	+ kvm->arch.dawr1_enabled = true;
	5625	+ return 0;
	5626	+}
	5627	+
	5628	+static bool kvmppc_hash_v3_possible(void)
	5629	+{
	5630	+ if (radix_enabled() && no_mixing_hpt_and_radix)
	5631	+ return false;
	5632	+
	5633	+ return cpu_has_feature(CPU_FTR_ARCH_300) &&
	5634	+ cpu_has_feature(CPU_FTR_HVMODE);
	5635	+}
	5636	+
5602	5637	static struct kvmppc_ops kvm_ops_hv = {
5603	5638	.get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv,
5604	5639	.set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv,
@@ -5642,6 +5677,8 @@ static struct kvmppc_ops kvm_ops_hv = { · 37 unmodified lines
5642	5677	.store_to_eaddr = kvmhv_store_to_eaddr,
5643	5678	.enable_svm = kvmhv_enable_svm,
5644	5679	.svm_off = kvmhv_svm_off,
	5680	+ .enable_dawr1 = kvmhv_enable_dawr1,
	5681	+ .hash_v3_possible = kvmppc_hash_v3_possible,
5645	5682	};
5646	5683
5647	5684	static int kvm_init_subcore_bitmap(void)

arch/powerpc/kvm/book3s_hv_builtin.c+1−107

@@ -277,8 +277,7 @@ void kvmhv_commence_exit(int trap) · 276 unmodified lines
277	277	struct kvmppc_vcore *vc = local_paca->kvm_hstate.kvm_vcore;
278	278	int ptid = local_paca->kvm_hstate.ptid;
279	279	struct kvm_split_mode *sip = local_paca->kvm_hstate.kvm_split_mode;
280		− int me, ee, i, t;
281		− int cpu0;
	280	+ int me, ee, i;
282	281
283	282	/* Set our bit in the threads-exiting-guest map in the 0xff00
284	283	bits of vcore->entry_exit_map */
@@ -320,22 +319,6 @@ void kvmhv_commence_exit(int trap) · 35 unmodified lines
320	319	if ((ee >> 8) == 0)
321	320	kvmhv_interrupt_vcore(vc, ee);
322	321	}
323		−
324		− /*
325		− * On POWER9 when running a HPT guest on a radix host (sip != NULL),
326		− * we have to interrupt inactive CPU threads to get them to
327		− * restore the host LPCR value.
328		− */
329		− if (sip->lpcr_req) {
330		− if (cmpxchg(&sip->do_restore, 0, 1) == 0) {
331		− vc = local_paca->kvm_hstate.kvm_vcore;
332		− cpu0 = vc->pcpu + ptid - local_paca->kvm_hstate.tid;
333		− for (t = 1; t < threads_per_core; ++t) {
334		− if (sip->napped[t])
335		− kvmhv_rm_send_ipi(cpu0 + t);
336		− }
337		− }
338		− }
339	322	}
340	323
341	324	struct kvmppc_host_rm_ops *kvmppc_host_rm_ops_hv;
@@ -667,95 +650,6 @@ void kvmppc_bad_interrupt(struct pt_regs *regs) · 325 unmodified lines
667	650	panic("Bad KVM trap");
668	651	}
669	652
670		−/*
671		− * Functions used to switch LPCR HR and UPRT bits on all threads
672		− * when entering and exiting HPT guests on a radix host.
673		− */
674		−
675		−#define PHASE_REALMODE 1 /* in real mode */
676		−#define PHASE_SET_LPCR 2 /* have set LPCR */
677		−#define PHASE_OUT_OF_GUEST 4 /* have finished executing in guest */
678		−#define PHASE_RESET_LPCR 8 /* have reset LPCR to host value */
679		−
680		−#define ALL(p) (((p) << 24) \| ((p) << 16) \| ((p) << 8) \| (p))
681		−
682		−static void wait_for_sync(struct kvm_split_mode *sip, int phase)
683		−{
684		− int thr = local_paca->kvm_hstate.tid;
685		−
686		− sip->lpcr_sync.phase[thr] \|= phase;
687		− phase = ALL(phase);
688		− while ((sip->lpcr_sync.allphases & phase) != phase) {
689		− HMT_low();
690		− barrier();
691		− }
692		− HMT_medium();
693		−}
694		−
695		−void kvmhv_p9_set_lpcr(struct kvm_split_mode *sip)
696		−{
697		− int num_sets;
698		− unsigned long rb, set;
699		−
700		− /* wait for every other thread to get to real mode */
701		− wait_for_sync(sip, PHASE_REALMODE);
702		−
703		− /* Set LPCR and LPIDR */
704		− mtspr(SPRN_LPCR, sip->lpcr_req);
705		− mtspr(SPRN_LPID, sip->lpidr_req);
706		− isync();
707		−
708		− /*
709		− * P10 will flush all the congruence class with a single tlbiel
710		− */
711		− if (cpu_has_feature(CPU_FTR_ARCH_31))
712		− num_sets = 1;
713		− else
714		− num_sets = POWER9_TLB_SETS_RADIX;
715		−
716		− /* Invalidate the TLB on thread 0 */
717		− if (local_paca->kvm_hstate.tid == 0) {
718		− sip->do_set = 0;
719		− asm volatile("ptesync" : : : "memory");
720		− for (set = 0; set < num_sets; ++set) {
721		− rb = TLBIEL_INVAL_SET_LPID +
722		− (set << TLBIEL_INVAL_SET_SHIFT);
723		− asm volatile(PPC_TLBIEL(%0, %1, 0, 0, 0) : :
724		− "r" (rb), "r" (0));
725		− }
726		− asm volatile("ptesync" : : : "memory");
727		− }
728		−
729		− /* indicate that we have done so and wait for others */
730		− wait_for_sync(sip, PHASE_SET_LPCR);
731		− /* order read of sip->lpcr_sync.allphases vs. sip->do_set */
732		− smp_rmb();
733		−}
734		−
735		−/*
736		− * Called when a thread that has been in the guest needs
737		− * to reload the host LPCR value - but only on POWER9 when
738		− * running a HPT guest on a radix host.
739		− */
740		−void kvmhv_p9_restore_lpcr(struct kvm_split_mode *sip)
741		−{
742		− /* we're out of the guest... */
743		− wait_for_sync(sip, PHASE_OUT_OF_GUEST);
744		−
745		− mtspr(SPRN_LPID, 0);
746		− mtspr(SPRN_LPCR, sip->host_lpcr);
747		− isync();
748		−
749		− if (local_paca->kvm_hstate.tid == 0) {
750		− sip->do_restore = 0;
751		− smp_wmb(); /* order store of do_restore vs. phase */
752		− }
753		−
754		− wait_for_sync(sip, PHASE_RESET_LPCR);
755		− smp_mb();
756		− local_paca->kvm_hstate.kvm_split_mode = NULL;
757		−}
758		−
759	653	static void kvmppc_end_cede(struct kvm_vcpu *vcpu)
760	654	{
761	655	vcpu->arch.ceded = 0;

arch/powerpc/kvm/book3s_hv_nested.c+56−14

@@ -33,8 +33,8 @@ void kvmhv_save_hv_regs(struct kvm_vcpu vcpu, struct hv_guest_state hr) · 32 unmodified lines
33	33	hr->dpdes = vc->dpdes;
34	34	hr->hfscr = vcpu->arch.hfscr;
35	35	hr->tb_offset = vc->tb_offset;
36		− hr->dawr0 = vcpu->arch.dawr;
37		− hr->dawrx0 = vcpu->arch.dawrx;
	36	+ hr->dawr0 = vcpu->arch.dawr0;
	37	+ hr->dawrx0 = vcpu->arch.dawrx0;
38	38	hr->ciabr = vcpu->arch.ciabr;
39	39	hr->purr = vcpu->arch.purr;
40	40	hr->spurr = vcpu->arch.spurr;
@@ -49,6 +49,8 @@ void kvmhv_save_hv_regs(struct kvm_vcpu vcpu, struct hv_guest_state hr) · 8 unmodified lines
49	49	hr->pidr = vcpu->arch.pid;
50	50	hr->cfar = vcpu->arch.cfar;
51	51	hr->ppr = vcpu->arch.ppr;
	52	+ hr->dawr1 = vcpu->arch.dawr1;
	53	+ hr->dawrx1 = vcpu->arch.dawrx1;
52	54	}
53	55
54	56	static void byteswap_pt_regs(struct pt_regs *regs)
@@ -91,6 +93,8 @@ static void byteswap_hv_regs(struct hv_guest_state *hr) · 36 unmodified lines
91	93	hr->pidr = swab64(hr->pidr);
92	94	hr->cfar = swab64(hr->cfar);
93	95	hr->ppr = swab64(hr->ppr);
	96	+ hr->dawr1 = swab64(hr->dawr1);
	97	+ hr->dawrx1 = swab64(hr->dawrx1);
94	98	}
95	99
96	100	static void save_hv_return_state(struct kvm_vcpu *vcpu, int trap,
@@ -138,6 +142,7 @@ static void sanitise_hv_regs(struct kvm_vcpu vcpu, struct hv_guest_state hr) · 41 unmodified lines
138	142
139	143	/* Don't let data address watchpoint match in hypervisor state */
140	144	hr->dawrx0 &= ~DAWRX_HYP;
	145	+ hr->dawrx1 &= ~DAWRX_HYP;
141	146
142	147	/* Don't let completed instruction address breakpt match in HV state */
143	148	if ((hr->ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER)
@@ -151,8 +156,8 @@ static void restore_hv_regs(struct kvm_vcpu vcpu, struct hv_guest_state hr) · 7 unmodified lines
151	156	vc->pcr = hr->pcr \| PCR_MASK;
152	157	vc->dpdes = hr->dpdes;
153	158	vcpu->arch.hfscr = hr->hfscr;
154		− vcpu->arch.dawr = hr->dawr0;
155		− vcpu->arch.dawrx = hr->dawrx0;
	159	+ vcpu->arch.dawr0 = hr->dawr0;
	160	+ vcpu->arch.dawrx0 = hr->dawrx0;
156	161	vcpu->arch.ciabr = hr->ciabr;
157	162	vcpu->arch.purr = hr->purr;
158	163	vcpu->arch.spurr = hr->spurr;
@@ -167,6 +172,8 @@ static void restore_hv_regs(struct kvm_vcpu vcpu, struct hv_guest_state hr) · 8 unmodified lines
167	172	vcpu->arch.pid = hr->pidr;
168	173	vcpu->arch.cfar = hr->cfar;
169	174	vcpu->arch.ppr = hr->ppr;
	175	+ vcpu->arch.dawr1 = hr->dawr1;
	176	+ vcpu->arch.dawrx1 = hr->dawrx1;
170	177	}
171	178
172	179	void kvmhv_restore_hv_return_state(struct kvm_vcpu *vcpu,
@@ -215,12 +222,51 @@ static void kvmhv_nested_mmio_needed(struct kvm_vcpu *vcpu, u64 regs_ptr) · 42 unmodified lines
215	222	}
216	223	}
217	224
	225	+static int kvmhv_read_guest_state_and_regs(struct kvm_vcpu *vcpu,
	226	+ struct hv_guest_state *l2_hv,
	227	+ struct pt_regs *l2_regs,
	228	+ u64 hv_ptr, u64 regs_ptr)
	229	+{
	230	+ int size;
	231	+
	232	+ if (kvm_vcpu_read_guest(vcpu, hv_ptr, &l2_hv->version,
	233	+ sizeof(l2_hv->version)))
	234	+ return -1;
	235	+
	236	+ if (kvmppc_need_byteswap(vcpu))
	237	+ l2_hv->version = swab64(l2_hv->version);
	238	+
	239	+ size = hv_guest_state_size(l2_hv->version);
	240	+ if (size < 0)
	241	+ return -1;
	242	+
	243	+ return kvm_vcpu_read_guest(vcpu, hv_ptr, l2_hv, size) \|\|
	244	+ kvm_vcpu_read_guest(vcpu, regs_ptr, l2_regs,
	245	+ sizeof(struct pt_regs));
	246	+}
	247	+
	248	+static int kvmhv_write_guest_state_and_regs(struct kvm_vcpu *vcpu,
	249	+ struct hv_guest_state *l2_hv,
	250	+ struct pt_regs *l2_regs,
	251	+ u64 hv_ptr, u64 regs_ptr)
	252	+{
	253	+ int size;
	254	+
	255	+ size = hv_guest_state_size(l2_hv->version);
	256	+ if (size < 0)
	257	+ return -1;
	258	+
	259	+ return kvm_vcpu_write_guest(vcpu, hv_ptr, l2_hv, size) \|\|
	260	+ kvm_vcpu_write_guest(vcpu, regs_ptr, l2_regs,
	261	+ sizeof(struct pt_regs));
	262	+}
	263	+
218	264	long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
219	265	{
220	266	long int err, r;
221	267	struct kvm_nested_guest *l2;
222	268	struct pt_regs l2_regs, saved_l1_regs;
223		− struct hv_guest_state l2_hv, saved_l1_hv;
	269	+ struct hv_guest_state l2_hv = {0}, saved_l1_hv;
224	270	struct kvmppc_vcore *vc = vcpu->arch.vcore;
225	271	u64 hv_ptr, regs_ptr;
226	272	u64 hdec_exp;
@@ -235,17 +281,15 @@ long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu) · 8 unmodified lines
235	281	hv_ptr = kvmppc_get_gpr(vcpu, 4);
236	282	regs_ptr = kvmppc_get_gpr(vcpu, 5);
237	283	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
238		− err = kvm_vcpu_read_guest(vcpu, hv_ptr, &l2_hv,
239		− sizeof(struct hv_guest_state)) \|\|
240		− kvm_vcpu_read_guest(vcpu, regs_ptr, &l2_regs,
241		− sizeof(struct pt_regs));
	284	+ err = kvmhv_read_guest_state_and_regs(vcpu, &l2_hv, &l2_regs,
	285	+ hv_ptr, regs_ptr);
242	286	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
243	287	if (err)
244	288	return H_PARAMETER;
245	289
246	290	if (kvmppc_need_byteswap(vcpu))
247	291	byteswap_hv_regs(&l2_hv);
248		− if (l2_hv.version != HV_GUEST_STATE_VERSION)
	292	+ if (l2_hv.version > HV_GUEST_STATE_VERSION)
249	293	return H_P2;
250	294
251	295	if (kvmppc_need_byteswap(vcpu))
@@ -325,10 +369,8 @@ long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu) · 73 unmodified lines
325	369	byteswap_pt_regs(&l2_regs);
326	370	}
327	371	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
328		− err = kvm_vcpu_write_guest(vcpu, hv_ptr, &l2_hv,
329		− sizeof(struct hv_guest_state)) \|\|
330		− kvm_vcpu_write_guest(vcpu, regs_ptr, &l2_regs,
331		− sizeof(struct pt_regs));
	372	+ err = kvmhv_write_guest_state_and_regs(vcpu, &l2_hv, &l2_regs,
	373	+ hv_ptr, regs_ptr);
332	374	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
333	375	if (err)
334	376	return H_AUTHORITY;

arch/powerpc/kvm/book3s_hv_rmhandlers.S+95−80

@@ -52,11 +52,13 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) · 51 unmodified lines
52	52	#define STACK_SLOT_PID (SFS-32)
53	53	#define STACK_SLOT_IAMR (SFS-40)
54	54	#define STACK_SLOT_CIABR (SFS-48)
55		−#define STACK_SLOT_DAWR (SFS-56)
56		−#define STACK_SLOT_DAWRX (SFS-64)
	55	+#define STACK_SLOT_DAWR0 (SFS-56)
	56	+#define STACK_SLOT_DAWRX0 (SFS-64)
57	57	#define STACK_SLOT_HFSCR (SFS-72)
58	58	#define STACK_SLOT_AMR (SFS-80)
59	59	#define STACK_SLOT_UAMOR (SFS-88)
	60	+#define STACK_SLOT_DAWR1 (SFS-96)
	61	+#define STACK_SLOT_DAWRX1 (SFS-104)
60	62	/* the following is used by the P9 short path */
61	63	#define STACK_SLOT_NVGPRS (SFS-152) /* 18 gprs */
62	64
@@ -85,19 +87,6 @@ _GLOBAL_TOC(kvmppc_hv_entry_trampoline) · 22 unmodified lines
85	87	RFI_TO_KERNEL
86	88
87	89	kvmppc_call_hv_entry:
88		−BEGIN_FTR_SECTION
89		− /* On P9, do LPCR setting, if necessary */
90		− ld r3, HSTATE_SPLIT_MODE(r13)
91		− cmpdi r3, 0
92		− beq 46f
93		− lwz r4, KVM_SPLIT_DO_SET(r3)
94		− cmpwi r4, 0
95		− beq 46f
96		− bl kvmhv_p9_set_lpcr
97		− nop
98		−46:
99		−END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
100		−
101	90	ld r4, HSTATE_KVM_VCPU(r13)
102	91	bl kvmppc_hv_entry
103	92
@@ -361,11 +350,11 @@ kvm_secondary_got_guest: · 257 unmodified lines
361	350	LOAD_REG_ADDR(r6, decrementer_max)
362	351	ld r6, 0(r6)
363	352	mtspr SPRN_HDEC, r6
	353	+BEGIN_FTR_SECTION
364	354	/* and set per-LPAR registers, if doing dynamic micro-threading */
365	355	ld r6, HSTATE_SPLIT_MODE(r13)
366	356	cmpdi r6, 0
367	357	beq 63f
368		−BEGIN_FTR_SECTION
369	358	ld r0, KVM_SPLIT_RPR(r6)
370	359	mtspr SPRN_RPR, r0
371	360	ld r0, KVM_SPLIT_PMMAR(r6)
@@ -373,16 +362,7 @@ BEGIN_FTR_SECTION · 1 unmodified line
373	362	ld r0, KVM_SPLIT_LDBAR(r6)
374	363	mtspr SPRN_LDBAR, r0
375	364	isync
376		−FTR_SECTION_ELSE
377		− /* On P9 we use the split_info for coordinating LPCR changes */
378		− lwz r4, KVM_SPLIT_DO_SET(r6)
379		− cmpwi r4, 0
380		− beq 1f
381		− mr r3, r6
382		− bl kvmhv_p9_set_lpcr
383		− nop
384		−1:
385		−ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
	365	+END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
386	366	63:
387	367	/* Order load of vcpu after load of vcore */
388	368	lwsync
@@ -452,19 +432,15 @@ kvm_no_guest: · 63 unmodified lines
452	432	mtcr r5
453	433	blr
454	434
455		−53: HMT_LOW
	435	+53:
	436	+BEGIN_FTR_SECTION
	437	+ HMT_LOW
456	438	ld r5, HSTATE_KVM_VCORE(r13)
457	439	cmpdi r5, 0
458	440	bne 60f
459	441	ld r3, HSTATE_SPLIT_MODE(r13)
460	442	cmpdi r3, 0
461	443	beq kvm_no_guest
462		− lwz r0, KVM_SPLIT_DO_SET(r3)
463		− cmpwi r0, 0
464		− bne kvmhv_do_set
465		− lwz r0, KVM_SPLIT_DO_RESTORE(r3)
466		− cmpwi r0, 0
467		− bne kvmhv_do_restore
468	444	lbz r0, KVM_SPLIT_DO_NAP(r3)
469	445	cmpwi r0, 0
470	446	beq kvm_no_guest
@@ -472,24 +448,19 @@ kvm_no_guest: · 1 unmodified line
472	448	b kvm_unsplit_nap
473	449	60: HMT_MEDIUM
474	450	b kvm_secondary_got_guest
	451	+FTR_SECTION_ELSE
	452	+ HMT_LOW
	453	+ ld r5, HSTATE_KVM_VCORE(r13)
	454	+ cmpdi r5, 0
	455	+ beq kvm_no_guest
	456	+ HMT_MEDIUM
	457	+ b kvm_secondary_got_guest
	458	+ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
475	459
476	460	54: li r0, KVM_HWTHREAD_IN_KVM
477	461	stb r0, HSTATE_HWTHREAD_STATE(r13)
478	462	b kvm_no_guest
479	463
480		−kvmhv_do_set:
481		− /* Set LPCR, LPIDR etc. on P9 */
482		− HMT_MEDIUM
483		− bl kvmhv_p9_set_lpcr
484		− nop
485		− b kvm_no_guest
486		−
487		−kvmhv_do_restore:
488		− HMT_MEDIUM
489		− bl kvmhv_p9_restore_lpcr
490		− nop
491		− b kvm_no_guest
492		−
493	464	/*
494	465	* Here the primary thread is trying to return the core to
495	466	* whole-core mode, so we need to nap.
@@ -527,7 +498,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) · 31 unmodified lines
527	498	/* Set kvm_split_mode.napped[tid] = 1 */
528	499	ld r3, HSTATE_SPLIT_MODE(r13)
529	500	li r0, 1
530		− lbz r4, HSTATE_TID(r13)
	501	+ lhz r4, PACAPACAINDEX(r13)
	502	+ clrldi r4, r4, 61 /* micro-threading => P8 => 8 threads/core */
531	503	addi r4, r4, KVM_SPLIT_NAPPED
532	504	stbx r0, r3, r4
533	505	/* Check the do_nap flag again after setting napped[] */
@@ -711,10 +683,16 @@ BEGIN_FTR_SECTION · 177 unmodified lines
711	683	mfspr r7, SPRN_DAWRX0
712	684	mfspr r8, SPRN_IAMR
713	685	std r5, STACK_SLOT_CIABR(r1)
714		− std r6, STACK_SLOT_DAWR(r1)
715		− std r7, STACK_SLOT_DAWRX(r1)
	686	+ std r6, STACK_SLOT_DAWR0(r1)
	687	+ std r7, STACK_SLOT_DAWRX0(r1)
716	688	std r8, STACK_SLOT_IAMR(r1)
717	689	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
	690	+BEGIN_FTR_SECTION
	691	+ mfspr r6, SPRN_DAWR1
	692	+ mfspr r7, SPRN_DAWRX1
	693	+ std r6, STACK_SLOT_DAWR1(r1)
	694	+ std r7, STACK_SLOT_DAWRX1(r1)
	695	+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S \| CPU_FTR_DAWR1)
718	696
719	697	mfspr r5, SPRN_AMR
720	698	std r5, STACK_SLOT_AMR(r1)
@@ -801,10 +779,16 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) · 80 unmodified lines
801	779	lbz r5, 0(r5)
802	780	cmpdi r5, 0
803	781	beq 1f
804		− ld r5, VCPU_DAWR(r4)
805		− ld r6, VCPU_DAWRX(r4)
	782	+ ld r5, VCPU_DAWR0(r4)
	783	+ ld r6, VCPU_DAWRX0(r4)
806	784	mtspr SPRN_DAWR0, r5
807	785	mtspr SPRN_DAWRX0, r6
	786	+BEGIN_FTR_SECTION
	787	+ ld r5, VCPU_DAWR1(r4)
	788	+ ld r6, VCPU_DAWRX1(r4)
	789	+ mtspr SPRN_DAWR1, r5
	790	+ mtspr SPRN_DAWRX1, r6
	791	+END_FTR_SECTION_IFSET(CPU_FTR_DAWR1)
808	792	1:
809	793	ld r7, VCPU_CIABR(r4)
810	794	ld r8, VCPU_TAR(r4)
@@ -918,15 +902,19 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) · 107 unmodified lines
918	902	cmpdi r3, 512 /* 1 microsecond */
919	903	blt hdec_soon
920	904
921		− /* For hash guest, clear out and reload the SLB */
922	905	ld r6, VCPU_KVM(r4)
923	906	lbz r0, KVM_RADIX(r6)
924	907	cmpwi r0, 0
925	908	bne 9f
	909	+
	910	+ /* For hash guest, clear out and reload the SLB */
	911	+BEGIN_MMU_FTR_SECTION
	912	+ /* Radix host won't have populated the SLB, so no need to clear */
926	913	li r6, 0
927	914	slbmte r6, r6
928		− slbia
	915	+ PPC_SLBIA(6)
929	916	ptesync
	917	+END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX)
930	918
931	919	/* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */
932	920	lwz r5,VCPU_SLB_MAX(r4)
@@ -1187,6 +1175,20 @@ EXPORT_SYMBOL_GPL(__kvmhv_vcpu_entry_p9) · 254 unmodified lines
1187	1175	mr r4, r3
1188	1176	b fast_guest_entry_c
1189	1177	guest_exit_short_path:
	1178	+ /*
	1179	+ * Malicious or buggy radix guests may have inserted SLB entries
	1180	+ * (only 0..3 because radix always runs with UPRT=1), so these must
	1181	+ * be cleared here to avoid side-channels. slbmte is used rather
	1182	+ * than slbia, as it won't clear cached translations.
	1183	+ */
	1184	+ li r0,0
	1185	+ slbmte r0,r0
	1186	+ li r4,1
	1187	+ slbmte r0,r4
	1188	+ li r4,2
	1189	+ slbmte r0,r4
	1190	+ li r4,3
	1191	+ slbmte r0,r4
1190	1192
1191	1193	li r0, KVM_GUEST_MODE_NONE
1192	1194	stb r0, HSTATE_IN_GUEST(r13)
@@ -1499,7 +1501,7 @@ guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ · 306 unmodified lines
1499	1501	lbz r0, KVM_RADIX(r5)
1500	1502	li r5, 0
1501	1503	cmpwi r0, 0
1502		− bne 3f /* for radix, save 0 entries */
	1504	+ bne 0f /* for radix, save 0 entries */
1503	1505	lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */
1504	1506	mtctr r0
1505	1507	li r6,0
@@ -1518,13 +1520,13 @@ guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ · 12 unmodified lines
1518	1520	/* Finally clear out the SLB */
1519	1521	li r0,0
1520	1522	slbmte r0,r0
1521		− slbia
	1523	+ PPC_SLBIA(6)
1522	1524	ptesync
1523		−3: stw r5,VCPU_SLB_MAX(r9)
	1525	+ stw r5,VCPU_SLB_MAX(r9)
1524	1526
1525	1527	/* load host SLB entries */
1526	1528	BEGIN_MMU_FTR_SECTION
1527		− b 0f
	1529	+ b guest_bypass
1528	1530	END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
1529	1531	ld r8,PACA_SLBSHADOWPTR(r13)
1530	1532
@@ -1538,7 +1540,21 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) · 7 unmodified lines
1538	1540	slbmte r6,r5
1539	1541	1: addi r8,r8,16
1540	1542	.endr
1541		−0:
	1543	+ b guest_bypass
	1544	+
	1545	+0: /*
	1546	+ * Sanitise radix guest SLB, see guest_exit_short_path comment.
	1547	+ * We clear vcpu->arch.slb_max to match earlier behaviour.
	1548	+ */
	1549	+ li r0,0
	1550	+ stw r0,VCPU_SLB_MAX(r9)
	1551	+ slbmte r0,r0
	1552	+ li r4,1
	1553	+ slbmte r0,r4
	1554	+ li r4,2
	1555	+ slbmte r0,r4
	1556	+ li r4,3
	1557	+ slbmte r0,r4
1542	1558
1543	1559	guest_bypass:
1544	1560	stw r12, STACK_SLOT_TRAP(r1)
@@ -1759,8 +1775,8 @@ END_FTR_SECTION(CPU_FTR_TM \| CPU_FTR_P9_TM_HV_ASSIST, 0) · 214 unmodified lines
1759	1775	/* Restore host values of some registers */
1760	1776	BEGIN_FTR_SECTION
1761	1777	ld r5, STACK_SLOT_CIABR(r1)
1762		− ld r6, STACK_SLOT_DAWR(r1)
1763		− ld r7, STACK_SLOT_DAWRX(r1)
	1778	+ ld r6, STACK_SLOT_DAWR0(r1)
	1779	+ ld r7, STACK_SLOT_DAWRX0(r1)
1764	1780	mtspr SPRN_CIABR, r5
1765	1781	/*
1766	1782	* If the DAWR doesn't work, it's ok to write these here as
@@ -1769,6 +1785,12 @@ BEGIN_FTR_SECTION · 2 unmodified lines
1769	1785	mtspr SPRN_DAWR0, r6
1770	1786	mtspr SPRN_DAWRX0, r7
1771	1787	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
	1788	+BEGIN_FTR_SECTION
	1789	+ ld r6, STACK_SLOT_DAWR1(r1)
	1790	+ ld r7, STACK_SLOT_DAWRX1(r1)
	1791	+ mtspr SPRN_DAWR1, r6
	1792	+ mtspr SPRN_DAWRX1, r7
	1793	+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S \| CPU_FTR_DAWR1)
1772	1794	BEGIN_FTR_SECTION
1773	1795	ld r5, STACK_SLOT_TID(r1)
1774	1796	ld r6, STACK_SLOT_PSSCR(r1)
@@ -1938,24 +1960,10 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) · 163 unmodified lines
1938	1960	19: lis r8,0x7fff /* MAX_INT@h */
1939	1961	mtspr SPRN_HDEC,r8
1940	1962
1941		−16:
1942		−BEGIN_FTR_SECTION
1943		− /* On POWER9 with HPT-on-radix we need to wait for all other threads */
1944		− ld r3, HSTATE_SPLIT_MODE(r13)
1945		− cmpdi r3, 0
1946		− beq 47f
1947		− lwz r8, KVM_SPLIT_DO_RESTORE(r3)
1948		− cmpwi r8, 0
1949		− beq 47f
1950		− bl kvmhv_p9_restore_lpcr
1951		− nop
1952		− b 48f
1953		−47:
1954		−END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
1955		− ld r8,KVM_HOST_LPCR(r4)
	1963	+16: ld r8,KVM_HOST_LPCR(r4)
1956	1964	mtspr SPRN_LPCR,r8
1957	1965	isync
1958		−48:
	1966	+
1959	1967	#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1960	1968	/* Finish timing, if we have a vcpu */
1961	1969	ld r4, HSTATE_KVM_VCPU(r13)
@@ -2574,8 +2582,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) · 612 unmodified lines
2574	2582	rlwimi r5, r4, 5, DAWRX_DR \| DAWRX_DW
2575	2583	rlwimi r5, r4, 2, DAWRX_WT
2576	2584	clrrdi r4, r4, 3
2577		− std r4, VCPU_DAWR(r3)
2578		− std r5, VCPU_DAWRX(r3)
	2585	+ std r4, VCPU_DAWR0(r3)
	2586	+ std r5, VCPU_DAWRX0(r3)
2579	2587	/*
2580	2588	* If came in through the real mode hcall handler then it is necessary
2581	2589	* to write the registers since the return path won't. Otherwise it is
@@ -2779,8 +2787,10 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300) · 197 unmodified lines
2779	2787	beq kvm_end_cede
2780	2788	cmpwi r0, NAPPING_NOVCPU
2781	2789	beq kvm_novcpu_wakeup
	2790	+BEGIN_FTR_SECTION
2782	2791	cmpwi r0, NAPPING_UNSPLIT
2783	2792	beq kvm_unsplit_wakeup
	2793	+END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
2784	2794	twi 31,0,0 /* Nap state must not be zero */
2785	2795
2786	2796	33: mr r4, r3
@@ -3343,13 +3353,18 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) · 556 unmodified lines
3343	3353	mtspr SPRN_IAMR, r0
3344	3354	mtspr SPRN_CIABR, r0
3345	3355	mtspr SPRN_DAWRX0, r0
	3356	+BEGIN_FTR_SECTION
	3357	+ mtspr SPRN_DAWRX1, r0
	3358	+END_FTR_SECTION_IFSET(CPU_FTR_DAWR1)
	3359	+
	3360	+ /* Clear hash and radix guest SLB, see guest_exit_short_path comment. */
	3361	+ slbmte r0, r0
	3362	+ PPC_SLBIA(6)
3346	3363
3347	3364	BEGIN_MMU_FTR_SECTION
3348	3365	b 4f
3349	3366	END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
3350	3367
3351		− slbmte r0, r0
3352		− slbia
3353	3368	ptesync
3354	3369	ld r8, PACA_SLBSHADOWPTR(r13)
3355	3370	.rept SLB_NUM_BOLTED

arch/powerpc/kvm/booke.c+1−1

@@ -698,7 +698,7 @@ int kvmppc_core_prepare_to_enter(struct kvm_vcpu *vcpu) · 697 unmodified lines
698	698
699	699	kvmppc_set_exit_type(vcpu, EMULATED_MTMSRWE_EXITS);
700	700	r = 1;
701		− };
	701	+ }
702	702
703	703	return r;
704	704	}

arch/powerpc/kvm/powerpc.c+12−2

@@ -611,8 +611,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 610 unmodified lines
611	611	r = !!(hv_enabled && radix_enabled());
612	612	break;
613	613	case KVM_CAP_PPC_MMU_HASH_V3:
614		− r = !!(hv_enabled && cpu_has_feature(CPU_FTR_ARCH_300) &&
615		− cpu_has_feature(CPU_FTR_HVMODE));
	614	+ r = !!(hv_enabled && kvmppc_hv_ops->hash_v3_possible &&
	615	+ kvmppc_hv_ops->hash_v3_possible());
616	616	break;
617	617	case KVM_CAP_PPC_NESTED_HV:
618	618	r = !!(hv_enabled && kvmppc_hv_ops->enable_nested &&
@@ -678,6 +678,10 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 59 unmodified lines
678	678	r = hv_enabled && kvmppc_hv_ops->enable_svm &&
679	679	!kvmppc_hv_ops->enable_svm(NULL);
680	680	break;
	681	+ case KVM_CAP_PPC_DAWR1:
	682	+ r = !!(hv_enabled && kvmppc_hv_ops->enable_dawr1 &&
	683	+ !kvmppc_hv_ops->enable_dawr1(NULL));
	684	+ break;
681	685	#endif
682	686	default:
683	687	r = 0;
@@ -2187,6 +2191,12 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, · 1503 unmodified lines
2187	2191	break;
2188	2192	r = kvm->arch.kvm_ops->enable_svm(kvm);
2189	2193	break;
	2194	+ case KVM_CAP_PPC_DAWR1:
	2195	+ r = -EINVAL;
	2196	+ if (!is_kvmppc_hv_enabled(kvm) \|\| !kvm->arch.kvm_ops->enable_dawr1)
	2197	+ break;
	2198	+ r = kvm->arch.kvm_ops->enable_dawr1(kvm);
	2199	+ break;
2190	2200	#endif
2191	2201	default:
2192	2202	r = -EINVAL;

arch/s390/include/asm/kvm_host.h+0−1

@@ -28,7 +28,6 @@ · 27 unmodified lines
28	28	#define KVM_S390_BSCA_CPU_SLOTS 64
29	29	#define KVM_S390_ESCA_CPU_SLOTS 248
30	30	#define KVM_MAX_VCPUS 255
31		−#define KVM_USER_MEM_SLOTS 32
32	31
33	32	/*
34	33	* These seem to be used for allocating ->chip in the routing table, which we

arch/s390/pci/pci_mmio.c+2−2

@@ -170,7 +170,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, · 169 unmodified lines
170	170	if (!(vma->vm_flags & VM_WRITE))
171	171	goto out_unlock_mmap;
172	172
173		− ret = follow_pte(vma->vm_mm, mmio_addr, NULL, &ptep, NULL, &ptl);
	173	+ ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl);
174	174	if (ret)
175	175	goto out_unlock_mmap;
176	176
@@ -311,7 +311,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, · 134 unmodified lines
311	311	if (!(vma->vm_flags & VM_WRITE))
312	312	goto out_unlock_mmap;
313	313
314		− ret = follow_pte(vma->vm_mm, mmio_addr, NULL, &ptep, NULL, &ptl);
	314	+ ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl);
315	315	if (ret)
316	316	goto out_unlock_mmap;
317	317

arch/sparc/include/asm/spinlock_64.h+1−1

@@ -11,8 +11,8 @@ · 10 unmodified lines
11	11
12	12	#include <asm/processor.h>
13	13	#include <asm/barrier.h>
14		−#include <asm/qrwlock.h>
15	14	#include <asm/qspinlock.h>
	15	+#include <asm/qrwlock.h>
16	16
17	17	#endif /* !(__ASSEMBLY__) */
18	18

arch/x86/include/asm/cpufeatures.h+2−0

@@ -292,6 +292,7 @@ · 291 unmodified lines
292	292	#define X86_FEATURE_PER_THREAD_MBA (1132+ 7) / "" Per-thread Memory Bandwidth Allocation */
293	293
294	294	/* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */
	295	+#define X86_FEATURE_AVX_VNNI (1232+ 4) / AVX VNNI instructions */
295	296	#define X86_FEATURE_AVX512_BF16 (1232+ 5) / AVX512 BFLOAT16 instructions */
296	297
297	298	/* AMD-defined CPU features, CPUID level 0x80000008 (EBX), word 13 */
@@ -335,6 +336,7 @@ · 37 unmodified lines
335	336	#define X86_FEATURE_AVIC (1532+13) / Virtual Interrupt Controller */
336	337	#define X86_FEATURE_V_VMSAVE_VMLOAD (1532+15) / Virtual VMSAVE VMLOAD */
337	338	#define X86_FEATURE_VGIF (1532+16) / Virtual GIF */
	339	+#define X86_FEATURE_SVME_ADDR_CHK (1532+28) / "" SVME addr check */
338	340
339	341	/* Intel-defined CPU features, CPUID level 0x00000007:0 (ECX), word 16 */
340	342	#define X86_FEATURE_AVX512VBMI (1632+ 1) / AVX512 Vector Bit Manipulation instructions*/

arch/x86/include/asm/kvm-x86-ops.h+127−0

@@ -0,0 +1,127 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+#if !defined(KVM_X86_OP) \|\| !defined(KVM_X86_OP_NULL)
	3	+BUILD_BUG_ON(1)
	4	+#endif
	5	+
	6	+/*
	7	+ * KVM_X86_OP() and KVM_X86_OP_NULL() are used to help generate
	8	+ * "static_call()"s. They are also intended for use when defining
	9	+ * the vmx/svm kvm_x86_ops. KVM_X86_OP() can be used for those
	10	+ * functions that follow the [svm\|vmx]_func_name convention.
	11	+ * KVM_X86_OP_NULL() can leave a NULL definition for the
	12	+ * case where there is no definition or a function name that
	13	+ * doesn't match the typical naming convention is supplied.
	14	+ */
	15	+KVM_X86_OP_NULL(hardware_enable)
	16	+KVM_X86_OP_NULL(hardware_disable)
	17	+KVM_X86_OP_NULL(hardware_unsetup)
	18	+KVM_X86_OP_NULL(cpu_has_accelerated_tpr)
	19	+KVM_X86_OP(has_emulated_msr)
	20	+KVM_X86_OP(vcpu_after_set_cpuid)
	21	+KVM_X86_OP(vm_init)
	22	+KVM_X86_OP_NULL(vm_destroy)
	23	+KVM_X86_OP(vcpu_create)
	24	+KVM_X86_OP(vcpu_free)
	25	+KVM_X86_OP(vcpu_reset)
	26	+KVM_X86_OP(prepare_guest_switch)
	27	+KVM_X86_OP(vcpu_load)
	28	+KVM_X86_OP(vcpu_put)
	29	+KVM_X86_OP(update_exception_bitmap)
	30	+KVM_X86_OP(get_msr)
	31	+KVM_X86_OP(set_msr)
	32	+KVM_X86_OP(get_segment_base)
	33	+KVM_X86_OP(get_segment)
	34	+KVM_X86_OP(get_cpl)
	35	+KVM_X86_OP(set_segment)
	36	+KVM_X86_OP_NULL(get_cs_db_l_bits)
	37	+KVM_X86_OP(set_cr0)
	38	+KVM_X86_OP(is_valid_cr4)
	39	+KVM_X86_OP(set_cr4)
	40	+KVM_X86_OP(set_efer)
	41	+KVM_X86_OP(get_idt)
	42	+KVM_X86_OP(set_idt)
	43	+KVM_X86_OP(get_gdt)
	44	+KVM_X86_OP(set_gdt)
	45	+KVM_X86_OP(sync_dirty_debug_regs)
	46	+KVM_X86_OP(set_dr7)
	47	+KVM_X86_OP(cache_reg)
	48	+KVM_X86_OP(get_rflags)
	49	+KVM_X86_OP(set_rflags)
	50	+KVM_X86_OP(tlb_flush_all)
	51	+KVM_X86_OP(tlb_flush_current)
	52	+KVM_X86_OP_NULL(tlb_remote_flush)
	53	+KVM_X86_OP_NULL(tlb_remote_flush_with_range)
	54	+KVM_X86_OP(tlb_flush_gva)
	55	+KVM_X86_OP(tlb_flush_guest)
	56	+KVM_X86_OP(run)
	57	+KVM_X86_OP_NULL(handle_exit)
	58	+KVM_X86_OP_NULL(skip_emulated_instruction)
	59	+KVM_X86_OP_NULL(update_emulated_instruction)
	60	+KVM_X86_OP(set_interrupt_shadow)
	61	+KVM_X86_OP(get_interrupt_shadow)
	62	+KVM_X86_OP(patch_hypercall)
	63	+KVM_X86_OP(set_irq)
	64	+KVM_X86_OP(set_nmi)
	65	+KVM_X86_OP(queue_exception)
	66	+KVM_X86_OP(cancel_injection)
	67	+KVM_X86_OP(interrupt_allowed)
	68	+KVM_X86_OP(nmi_allowed)
	69	+KVM_X86_OP(get_nmi_mask)
	70	+KVM_X86_OP(set_nmi_mask)
	71	+KVM_X86_OP(enable_nmi_window)
	72	+KVM_X86_OP(enable_irq_window)
	73	+KVM_X86_OP(update_cr8_intercept)
	74	+KVM_X86_OP(check_apicv_inhibit_reasons)
	75	+KVM_X86_OP_NULL(pre_update_apicv_exec_ctrl)
	76	+KVM_X86_OP(refresh_apicv_exec_ctrl)
	77	+KVM_X86_OP(hwapic_irr_update)
	78	+KVM_X86_OP(hwapic_isr_update)
	79	+KVM_X86_OP_NULL(guest_apic_has_interrupt)
	80	+KVM_X86_OP(load_eoi_exitmap)
	81	+KVM_X86_OP(set_virtual_apic_mode)
	82	+KVM_X86_OP_NULL(set_apic_access_page_addr)
	83	+KVM_X86_OP(deliver_posted_interrupt)
	84	+KVM_X86_OP_NULL(sync_pir_to_irr)
	85	+KVM_X86_OP(set_tss_addr)
	86	+KVM_X86_OP(set_identity_map_addr)
	87	+KVM_X86_OP(get_mt_mask)
	88	+KVM_X86_OP(load_mmu_pgd)
	89	+KVM_X86_OP_NULL(has_wbinvd_exit)
	90	+KVM_X86_OP(write_l1_tsc_offset)
	91	+KVM_X86_OP(get_exit_info)
	92	+KVM_X86_OP(check_intercept)
	93	+KVM_X86_OP(handle_exit_irqoff)
	94	+KVM_X86_OP_NULL(request_immediate_exit)
	95	+KVM_X86_OP(sched_in)
	96	+KVM_X86_OP_NULL(slot_enable_log_dirty)
	97	+KVM_X86_OP_NULL(slot_disable_log_dirty)
	98	+KVM_X86_OP_NULL(flush_log_dirty)
	99	+KVM_X86_OP_NULL(enable_log_dirty_pt_masked)
	100	+KVM_X86_OP_NULL(cpu_dirty_log_size)
	101	+KVM_X86_OP_NULL(pre_block)
	102	+KVM_X86_OP_NULL(post_block)
	103	+KVM_X86_OP_NULL(vcpu_blocking)
	104	+KVM_X86_OP_NULL(vcpu_unblocking)
	105	+KVM_X86_OP_NULL(update_pi_irte)
	106	+KVM_X86_OP_NULL(apicv_post_state_restore)
	107	+KVM_X86_OP_NULL(dy_apicv_has_pending_interrupt)
	108	+KVM_X86_OP_NULL(set_hv_timer)
	109	+KVM_X86_OP_NULL(cancel_hv_timer)
	110	+KVM_X86_OP(setup_mce)
	111	+KVM_X86_OP(smi_allowed)
	112	+KVM_X86_OP(pre_enter_smm)
	113	+KVM_X86_OP(pre_leave_smm)
	114	+KVM_X86_OP(enable_smi_window)
	115	+KVM_X86_OP_NULL(mem_enc_op)
	116	+KVM_X86_OP_NULL(mem_enc_reg_region)
	117	+KVM_X86_OP_NULL(mem_enc_unreg_region)
	118	+KVM_X86_OP(get_msr_feature)
	119	+KVM_X86_OP(can_emulate_instruction)
	120	+KVM_X86_OP(apic_init_signal_blocked)
	121	+KVM_X86_OP_NULL(enable_direct_tlbflush)
	122	+KVM_X86_OP_NULL(migrate_timers)
	123	+KVM_X86_OP(msr_filter_changed)
	124	+KVM_X86_OP_NULL(complete_emulated_msr)
	125	+
	126	+#undef KVM_X86_OP
	127	+#undef KVM_X86_OP_NULL

arch/x86/include/asm/kvm_host.h+74−15

@@ -40,10 +40,8 @@ · 39 unmodified lines
40	40	#define KVM_MAX_VCPUS 288
41	41	#define KVM_SOFT_MAX_VCPUS 240
42	42	#define KVM_MAX_VCPU_ID 1023
43		−#define KVM_USER_MEM_SLOTS 509
44	43	/* memory slots that are not exposed to userspace */
45	44	#define KVM_PRIVATE_MEM_SLOTS 3
46		−#define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS)
47	45
48	46	#define KVM_HALT_POLL_NS_DEFAULT 200000
49	47
@@ -52,6 +50,9 @@ · 2 unmodified lines
52	50	#define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE \| \
53	51	KVM_DIRTY_LOG_INITIALLY_SET)
54	52
	53	+#define KVM_BUS_LOCK_DETECTION_VALID_MODE (KVM_BUS_LOCK_DETECTION_OFF \| \
	54	+ KVM_BUS_LOCK_DETECTION_EXIT)
	55	+
55	56	/* x86-specific vcpu->requests bit members */
56	57	#define KVM_REQ_MIGRATE_TIMER KVM_ARCH_REQ(0)
57	58	#define KVM_REQ_REPORT_TPR_ACCESS KVM_ARCH_REQ(1)
@@ -200,9 +201,17 @@ enum x86_intercept_stage; · 142 unmodified lines
200	201	#define DR6_BS (1 << 14)
201	202	#define DR6_BT (1 << 15)
202	203	#define DR6_RTM (1 << 16)
203		−#define DR6_FIXED_1 0xfffe0ff0
204		−#define DR6_INIT 0xffff0ff0
	204	+/*
	205	+ * DR6_ACTIVE_LOW combines fixed-1 and active-low bits.
	206	+ * We can regard all the bits in DR6_FIXED_1 as active_low bits;
	207	+ * they will never be 0 for now, but when they are defined
	208	+ * in the future it will require no code change.
	209	+ *
	210	+ * DR6_ACTIVE_LOW is also used as the init/reset value for DR6.
	211	+ */
	212	+#define DR6_ACTIVE_LOW 0xffff0ff0
205	213	#define DR6_VOLATILE 0x0001e00f
	214	+#define DR6_FIXED_1 (DR6_ACTIVE_LOW & ~DR6_VOLATILE)
206	215
207	216	#define DR7_BP_EN_MASK 0x000000ff
208	217	#define DR7_GE (1 << 9)
@@ -337,6 +346,8 @@ struct kvm_mmu_root_info { · 128 unmodified lines
337	346
338	347	#define KVM_MMU_NUM_PREV_ROOTS 3
339	348
	349	+#define KVM_HAVE_MMU_RWLOCK
	350	+
340	351	struct kvm_mmu_page;
341	352
342	353	/*
@@ -358,8 +369,6 @@ struct kvm_mmu { · 15 unmodified lines
358	369	int (sync_page)(struct kvm_vcpu vcpu,
359	370	struct kvm_mmu_page *sp);
360	371	void (invlpg)(struct kvm_vcpu vcpu, gva_t gva, hpa_t root_hpa);
361		− void (update_pte)(struct kvm_vcpu vcpu, struct kvm_mmu_page *sp,
362		− u64 spte, const void pte);
363	372	hpa_t root_hpa;
364	373	gpa_t root_pgd;
365	374	union kvm_mmu_role mmu_role;
@@ -510,6 +519,7 @@ struct kvm_vcpu_hv_synic { · 144 unmodified lines
510	519
511	520	/* Hyper-V per vcpu emulation context */
512	521	struct kvm_vcpu_hv {
	522	+ struct kvm_vcpu *vcpu;
513	523	u32 vp_index;
514	524	u64 hv_vapic;
515	525	s64 runtime_offset;
@@ -520,6 +530,15 @@ struct kvm_vcpu_hv { · 4 unmodified lines
520	530	cpumask_t tlb_flush;
521	531	};
522	532
	533	+/* Xen HVM per vcpu emulation context */
	534	+struct kvm_vcpu_xen {
	535	+ u64 hypercall_rip;
	536	+ bool vcpu_info_set;
	537	+ bool vcpu_time_info_set;
	538	+ struct gfn_to_hva_cache vcpu_info_cache;
	539	+ struct gfn_to_hva_cache vcpu_time_info_cache;
	540	+};
	541	+
523	542	struct kvm_vcpu_arch {
524	543	/*
525	544	* rip and regs accesses must go through
@@ -640,7 +659,7 @@ struct kvm_vcpu_arch { · 114 unmodified lines
640	659	int cpuid_nent;
641	660	struct kvm_cpuid_entry2 *cpuid_entries;
642	661
643		− unsigned long cr3_lm_rsvd_bits;
	662	+ u64 reserved_gpa_bits;
644	663	int maxphyaddr;
645	664	int max_tdp_level;
646	665
@@ -717,7 +736,9 @@ struct kvm_vcpu_arch { · 70 unmodified lines
717	736	/* used for guest single stepping over the given code position */
718	737	unsigned long singlestep_rip;
719	738
720		− struct kvm_vcpu_hv hyperv;
	739	+ bool hyperv_enabled;
	740	+ struct kvm_vcpu_hv *hyperv;
	741	+ struct kvm_vcpu_xen xen;
721	742
722	743	cpumask_var_t wbinvd_dirty_mask;
723	744
@@ -888,6 +909,14 @@ struct msr_bitmap_range { · 164 unmodified lines
888	909	unsigned long *bitmap;
889	910	};
890	911
	912	+/* Xen emulation context */
	913	+struct kvm_xen {
	914	+ bool long_mode;
	915	+ bool shinfo_set;
	916	+ u8 upcall_vector;
	917	+ struct gfn_to_hva_cache shinfo_cache;
	918	+};
	919	+
891	920	enum kvm_irqchip_mode {
892	921	KVM_IRQCHIP_NONE,
893	922	KVM_IRQCHIP_KERNEL, /* created with KVM_CREATE_IRQCHIP */
@@ -967,6 +996,7 @@ struct kvm_arch { · 73 unmodified lines
967	996	struct hlist_head mask_notifier_list;
968	997
969	998	struct kvm_hv hyperv;
	999	+ struct kvm_xen xen;
970	1000
971	1001	#ifdef CONFIG_KVM_MMU_AUDIT
972	1002	int audit_point;
@@ -998,9 +1028,12 @@ struct kvm_arch { · 25 unmodified lines
998	1028	struct msr_bitmap_range ranges[16];
999	1029	} msr_filter;
1000	1030
	1031	+ bool bus_lock_detection_enabled;
	1032	+
1001	1033	struct kvm_pmu_event_filter *pmu_event_filter;
1002	1034	struct task_struct *nx_lpage_recovery_thread;
1003	1035
	1036	+#ifdef CONFIG_X86_64
1004	1037	/*
1005	1038	* Whether the TDP MMU is enabled for this VM. This contains a
1006	1039	* snapshot of the TDP MMU module parameter from when the VM was
@@ -1026,12 +1059,25 @@ struct kvm_arch { · 19 unmodified lines
1026	1059	* tdp_mmu_page set and a root_count of 0.
1027	1060	*/
1028	1061	struct list_head tdp_mmu_pages;
	1062	+
	1063	+ /*
	1064	+ * Protects accesses to the following fields when the MMU lock
	1065	+ * is held in read mode:
	1066	+ * - tdp_mmu_pages (above)
	1067	+ * - the link field of struct kvm_mmu_pages used by the TDP MMU
	1068	+ * - lpage_disallowed_mmu_pages
	1069	+ * - the lpage_disallowed_link field of struct kvm_mmu_pages used
	1070	+ * by the TDP MMU
	1071	+ * It is acceptable, but not necessary, to acquire this lock when
	1072	+ * the thread holds the MMU lock in write mode.
	1073	+ */
	1074	+ spinlock_t tdp_mmu_pages_lock;
	1075	+#endif /* CONFIG_X86_64 */
1029	1076	};
1030	1077
1031	1078	struct kvm_vm_stat {
1032	1079	ulong mmu_shadow_zapped;
1033	1080	ulong mmu_pte_write;
1034		− ulong mmu_pte_updated;
1035	1081	ulong mmu_pde_zapped;
1036	1082	ulong mmu_flooded;
1037	1083	ulong mmu_recycled;
@@ -1340,6 +1386,19 @@ extern u64 __read_mostly host_efer; · 302 unmodified lines
1340	1386	extern bool __read_mostly allow_smaller_maxphyaddr;
1341	1387	extern struct kvm_x86_ops kvm_x86_ops;
1342	1388
	1389	+#define KVM_X86_OP(func) \
	1390	+ DECLARE_STATIC_CALL(kvm_x86_##func, (((struct kvm_x86_ops )0)->func));
	1391	+#define KVM_X86_OP_NULL KVM_X86_OP
	1392	+#include <asm/kvm-x86-ops.h>
	1393	+
	1394	+static inline void kvm_ops_static_call_update(void)
	1395	+{
	1396	+#define KVM_X86_OP(func) \
	1397	+ static_call_update(kvm_x86_##func, kvm_x86_ops.func);
	1398	+#define KVM_X86_OP_NULL KVM_X86_OP
	1399	+#include <asm/kvm-x86-ops.h>
	1400	+}
	1401	+
1343	1402	#define __KVM_HAVE_ARCH_VM_ALLOC
1344	1403	static inline struct kvm *kvm_arch_alloc_vm(void)
1345	1404	{
@@ -1351,7 +1410,7 @@ void kvm_arch_free_vm(struct kvm *kvm); · 5 unmodified lines
1351	1410	static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm)
1352	1411	{
1353	1412	if (kvm_x86_ops.tlb_remote_flush &&
1354		− !kvm_x86_ops.tlb_remote_flush(kvm))
	1413	+ !static_call(kvm_x86_tlb_remote_flush)(kvm))
1355	1414	return 0;
1356	1415	else
1357	1416	return -ENOTSUPP;
@@ -1421,6 +1480,8 @@ extern u8 kvm_tsc_scaling_ratio_frac_bits; · 63 unmodified lines
1421	1480	extern u64 kvm_max_tsc_scaling_ratio;
1422	1481	/* 1ull << kvm_tsc_scaling_ratio_frac_bits */
1423	1482	extern u64 kvm_default_tsc_scaling_ratio;
	1483	+/* bus lock detection supported? */
	1484	+extern bool kvm_has_bus_lock_exit;
1424	1485
1425	1486	extern u64 kvm_mce_cap_supported;
1426	1487
@@ -1501,7 +1562,7 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3); · 74 unmodified lines
1501	1562	int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
1502	1563	int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8);
1503	1564	int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val);
1504		−int kvm_get_dr(struct kvm_vcpu vcpu, int dr, unsigned long val);
	1565	+void kvm_get_dr(struct kvm_vcpu vcpu, int dr, unsigned long val);
1505	1566	unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu);
1506	1567	void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw);
1507	1568	void kvm_get_cs_db_l_bits(struct kvm_vcpu vcpu, int db, int *l);
@@ -1742,14 +1803,12 @@ static inline bool kvm_irq_is_postable(struct kvm_lapic_irq *irq) · 234 unmodified lines
1742	1803
1743	1804	static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
1744	1805	{
1745		− if (kvm_x86_ops.vcpu_blocking)
1746		− kvm_x86_ops.vcpu_blocking(vcpu);
	1806	+ static_call_cond(kvm_x86_vcpu_blocking)(vcpu);
1747	1807	}
1748	1808
1749	1809	static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
1750	1810	{
1751		− if (kvm_x86_ops.vcpu_unblocking)
1752		− kvm_x86_ops.vcpu_unblocking(vcpu);
	1811	+ static_call_cond(kvm_x86_vcpu_unblocking)(vcpu);
1753	1812	}
1754	1813
1755	1814	static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}

arch/x86/include/asm/virtext.h+19−6

@@ -30,16 +30,29 @@ static inline int cpu_has_vmx(void) · 29 unmodified lines
30	30	}
31	31
32	32
33		−/** Disable VMX on the current CPU
	33	+/**
	34	+ * cpu_vmxoff() - Disable VMX on the current CPU
34	35	*
35		− * vmxoff causes a undefined-opcode exception if vmxon was not run
36		− * on the CPU previously. Only call this function if you know VMX
37		− * is enabled.
	36	+ * Disable VMX and clear CR4.VMXE (even if VMXOFF faults)
	37	+ *
	38	+ * Note, VMXOFF causes a #UD if the CPU is !post-VMXON, but it's impossible to
	39	+ * atomically track post-VMXON state, e.g. this may be called in NMI context.
	40	+ * Eat all faults as all other faults on VMXOFF faults are mode related, i.e.
	41	+ * faults are guaranteed to be due to the !post-VMXON check unless the CPU is
	42	+ * magically in RM, VM86, compat mode, or at CPL>0.
38	43	*/
39		−static inline void cpu_vmxoff(void)
	44	+static inline int cpu_vmxoff(void)
40	45	{
41		− asm volatile ("vmxoff");
	46	+ asm_volatile_goto("1: vmxoff\n\t"
	47	+ _ASM_EXTABLE(1b, %l[fault])
	48	+ ::: "cc", "memory" : fault);
	49	+
	50	+ cr4_clear_bits(X86_CR4_VMXE);
	51	+ return 0;
	52	+
	53	+fault:
42	54	cr4_clear_bits(X86_CR4_VMXE);
	55	+ return -EIO;
43	56	}
44	57
45	58	static inline int cpu_vmx_enabled(void)

arch/x86/include/asm/vmx.h+1−0

@@ -73,6 +73,7 @@ · 72 unmodified lines
73	73	#define SECONDARY_EXEC_PT_USE_GPA VMCS_CONTROL_BIT(PT_USE_GPA)
74	74	#define SECONDARY_EXEC_TSC_SCALING VMCS_CONTROL_BIT(TSC_SCALING)
75	75	#define SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE VMCS_CONTROL_BIT(USR_WAIT_PAUSE)
	76	+#define SECONDARY_EXEC_BUS_LOCK_DETECTION VMCS_CONTROL_BIT(BUS_LOCK_DETECTION)
76	77
77	78	#define PIN_BASED_EXT_INTR_MASK VMCS_CONTROL_BIT(INTR_EXITING)
78	79	#define PIN_BASED_NMI_EXITING VMCS_CONTROL_BIT(NMI_EXITING)

arch/x86/include/asm/vmxfeatures.h+1−0

@@ -83,5 +83,6 @@ · 82 unmodified lines
83	83	#define VMX_FEATURE_TSC_SCALING ( 232+ 25) / Scale hardware TSC when read in guest */
84	84	#define VMX_FEATURE_USR_WAIT_PAUSE ( 232+ 26) / Enable TPAUSE, UMONITOR, UMWAIT in guest */
85	85	#define VMX_FEATURE_ENCLV_EXITING ( 232+ 28) / "" VM-Exit on ENCLV (leaf dependent) */
	86	+#define VMX_FEATURE_BUS_LOCK_DETECTION ( 232+ 30) / "" VM-Exit when bus lock caused */
86	87
87	88	#endif /* _ASM_X86_VMXFEATURES_H */

arch/x86/include/asm/xen/interface.h+3−0

@@ -182,6 +182,9 @@ struct arch_shared_info { · 181 unmodified lines
182	182	unsigned long p2m_cr3; /* cr3 value of the p2m address space */
183	183	unsigned long p2m_vaddr; /* virtual address of the p2m list */
184	184	unsigned long p2m_generation; /* generation count of p2m mapping */
	185	+#ifdef CONFIG_X86_32
	186	+ uint32_t wc_sec_hi;
	187	+#endif
185	188	};
186	189	#endif /* !__ASSEMBLY__ */
187	190

arch/x86/include/uapi/asm/kvm.h+1−0

@@ -112,6 +112,7 @@ struct kvm_ioapic_state { · 111 unmodified lines
112	112	#define KVM_NR_IRQCHIPS 3
113	113
114	114	#define KVM_RUN_X86_SMM (1 << 0)
	115	+#define KVM_RUN_X86_BUS_LOCK (1 << 1)
115	116
116	117	/* for KVM_GET_REGS and KVM_SET_REGS */
117	118	struct kvm_regs {

arch/x86/include/uapi/asm/vmx.h+3−1

@@ -89,6 +89,7 @@ · 88 unmodified lines
89	89	#define EXIT_REASON_XRSTORS 64
90	90	#define EXIT_REASON_UMWAIT 67
91	91	#define EXIT_REASON_TPAUSE 68
	92	+#define EXIT_REASON_BUS_LOCK 74
92	93
93	94	#define VMX_EXIT_REASONS \
94	95	{ EXIT_REASON_EXCEPTION_NMI, "EXCEPTION_NMI" }, \
@@ -150,7 +151,8 @@ · 55 unmodified lines
150	151	{ EXIT_REASON_XSAVES, "XSAVES" }, \
151	152	{ EXIT_REASON_XRSTORS, "XRSTORS" }, \
152	153	{ EXIT_REASON_UMWAIT, "UMWAIT" }, \
153		− { EXIT_REASON_TPAUSE, "TPAUSE" }
	154	+ { EXIT_REASON_TPAUSE, "TPAUSE" }, \
	155	+ { EXIT_REASON_BUS_LOCK, "BUS_LOCK" }
154	156
155	157	#define VMX_EXIT_REASON_FLAGS \
156	158	{ VMX_EXIT_REASONS_FAILED_VMENTRY, "FAILED_VMENTRY" }

arch/x86/kernel/apic/apic.c+1−0

@@ -1747,6 +1747,7 @@ void apic_ap_setup(void) · 1746 unmodified lines
1747	1747
1748	1748	#ifdef CONFIG_X86_X2APIC
1749	1749	int x2apic_mode;
	1750	+EXPORT_SYMBOL_GPL(x2apic_mode);
1750	1751
1751	1752	enum {
1752	1753	X2APIC_OFF,

arch/x86/kernel/reboot.c+10−20

@@ -547,31 +547,21 @@ static void emergency_vmx_disable_all(void) · 546 unmodified lines
547	547	local_irq_disable();
548	548
549	549	/*
550		− * We need to disable VMX on all CPUs before rebooting, otherwise
551		− * we risk hanging up the machine, because the CPU ignores INIT
552		− * signals when VMX is enabled.
	550	+ * Disable VMX on all CPUs before rebooting, otherwise we risk hanging
	551	+ * the machine, because the CPU blocks INIT when it's in VMX root.
553	552	*
554		− * We can't take any locks and we may be on an inconsistent
555		− * state, so we use NMIs as IPIs to tell the other CPUs to disable
556		− * VMX and halt.
	553	+ * We can't take any locks and we may be on an inconsistent state, so
	554	+ * use NMIs as IPIs to tell the other CPUs to exit VMX root and halt.
557	555	*
558		− * For safety, we will avoid running the nmi_shootdown_cpus()
559		− * stuff unnecessarily, but we don't have a way to check
560		− * if other CPUs have VMX enabled. So we will call it only if the
561		− * CPU we are running on has VMX enabled.
562		− *
563		− * We will miss cases where VMX is not enabled on all CPUs. This
564		− * shouldn't do much harm because KVM always enable VMX on all
565		− * CPUs anyway. But we can miss it on the small window where KVM
566		− * is still enabling VMX.
	556	+ * Do the NMI shootdown even if VMX if off on _this_ CPU, as that
	557	+ * doesn't prevent a different CPU from being in VMX root operation.
567	558	*/
568		− if (cpu_has_vmx() && cpu_vmx_enabled()) {
569		− /* Disable VMX on this CPU. */
570		− cpu_vmxoff();
	559	+ if (cpu_has_vmx()) {
	560	+ /* Safely force _this_ CPU out of VMX root operation. */
	561	+ __cpu_emergency_vmxoff();
571	562
572		− /* Halt and disable VMX on the other CPUs */
	563	+ /* Halt and exit VMX root operation on the other CPUs. */
573	564	nmi_shootdown_cpus(vmxoff_nmi);
574		−
575	565	}
576	566	}
577	567

arch/x86/kvm/Makefile+3−2

@@ -14,10 +14,11 @@ kvm-y += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o \ · 13 unmodified lines
14	14	$(KVM)/dirty_ring.o
15	15	kvm-$(CONFIG_KVM_ASYNC_PF) += $(KVM)/async_pf.o
16	16
17		−kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \
	17	+kvm-y += x86.o emulate.o i8259.o irq.o lapic.o xen.o \
18	18	i8254.o ioapic.o irq_comm.o cpuid.o pmu.o mtrr.o \
19	19	hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o \
20		− mmu/spte.o mmu/tdp_iter.o mmu/tdp_mmu.o
	20	+ mmu/spte.o
	21	+kvm-$(CONFIG_X86_64) += mmu/tdp_iter.o mmu/tdp_mmu.o
21	22
22	23	kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \
23	24	vmx/evmcs.o vmx/nested.o vmx/posted_intr.o

arch/x86/kvm/cpuid.c+20−4

@@ -173,16 +173,22 @@ static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) · 172 unmodified lines
173	173	kvm_update_pv_runtime(vcpu);
174	174
175	175	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
176		− kvm_mmu_reset_context(vcpu);
	176	+ vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu);
177	177
178	178	kvm_pmu_refresh(vcpu);
179	179	vcpu->arch.cr4_guest_rsvd_bits =
180	180	__cr4_reserved_bits(guest_cpuid_has, vcpu);
181	181
182		− vcpu->arch.cr3_lm_rsvd_bits = rsvd_bits(cpuid_maxphyaddr(vcpu), 63);
	182	+ kvm_hv_set_cpuid(vcpu);
183	183
184	184	/* Invoke the vendor callback only after the above state is updated. */
185		− kvm_x86_ops.vcpu_after_set_cpuid(vcpu);
	185	+ static_call(kvm_x86_vcpu_after_set_cpuid)(vcpu);
	186	+
	187	+ /*
	188	+ * Except for the MMU, which needs to be reset after any vendor
	189	+ * specific adjustments to the reserved GPA bits.
	190	+ */
	191	+ kvm_mmu_reset_context(vcpu);
186	192	}
187	193
188	194	static int is_efer_nx(void)
@@ -223,6 +229,16 @@ not_found: · 34 unmodified lines
223	229	return 36;
224	230	}
225	231
	232	+/*
	233	+ * This "raw" version returns the reserved GPA bits without any adjustments for
	234	+ * encryption technologies that usurp bits. The raw mask should be used if and
	235	+ * only if hardware does _not_ strip the usurped bits, e.g. in virtual MTRRs.
	236	+ */
	237	+u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu)
	238	+{
	239	+ return rsvd_bits(cpuid_maxphyaddr(vcpu), 63);
	240	+}
	241	+
226	242	/* when an old userspace process fills a new kernel module */
227	243	int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
228	244	struct kvm_cpuid *cpuid,
@@ -434,7 +450,7 @@ void kvm_set_cpu_caps(void) · 205 unmodified lines
434	450	kvm_cpu_cap_set(X86_FEATURE_SPEC_CTRL_SSBD);
435	451
436	452	kvm_cpu_cap_mask(CPUID_7_1_EAX,
437		− F(AVX512_BF16)
	453	+ F(AVX_VNNI) \| F(AVX512_BF16)
438	454	);
439	455
440	456	kvm_cpu_cap_mask(CPUID_D_1_EAX,

arch/x86/kvm/cpuid.h+18−6

@@ -30,15 +30,32 @@ bool kvm_cpuid(struct kvm_vcpu vcpu, u32 eax, u32 *ebx, · 29 unmodified lines
30	30	u32 ecx, u32 edx, bool exact_only);
31	31
32	32	int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu);
	33	+u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu);
33	34
34	35	static inline int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)
35	36	{
36	37	return vcpu->arch.maxphyaddr;
37	38	}
38	39
	40	+static inline bool kvm_vcpu_is_legal_gpa(struct kvm_vcpu *vcpu, gpa_t gpa)
	41	+{
	42	+ return !(gpa & vcpu->arch.reserved_gpa_bits);
	43	+}
	44	+
39	45	static inline bool kvm_vcpu_is_illegal_gpa(struct kvm_vcpu *vcpu, gpa_t gpa)
40	46	{
41		− return (gpa >= BIT_ULL(cpuid_maxphyaddr(vcpu)));
	47	+ return !kvm_vcpu_is_legal_gpa(vcpu, gpa);
	48	+}
	49	+
	50	+static inline bool kvm_vcpu_is_legal_aligned_gpa(struct kvm_vcpu *vcpu,
	51	+ gpa_t gpa, gpa_t alignment)
	52	+{
	53	+ return IS_ALIGNED(gpa, alignment) && kvm_vcpu_is_legal_gpa(vcpu, gpa);
	54	+}
	55	+
	56	+static inline bool page_address_valid(struct kvm_vcpu *vcpu, gpa_t gpa)
	57	+{
	58	+ return kvm_vcpu_is_legal_aligned_gpa(vcpu, gpa, PAGE_SIZE);
42	59	}
43	60
44	61	struct cpuid_reg {
@@ -324,11 +341,6 @@ static __always_inline void kvm_cpu_cap_check_and_set(unsigned int x86_feature) · 279 unmodified lines
324	341	kvm_cpu_cap_set(x86_feature);
325	342	}
326	343
327		−static inline bool page_address_valid(struct kvm_vcpu *vcpu, gpa_t gpa)
328		−{
329		− return PAGE_ALIGNED(gpa) && !(gpa >> cpuid_maxphyaddr(vcpu));
330		−}
331		−
332	344	static __always_inline bool guest_pv_has(struct kvm_vcpu *vcpu,
333	345	unsigned int kvm_feature)
334	346	{

arch/x86/kvm/emulate.c+7−7

@@ -2506,12 +2506,12 @@ static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, · 2505 unmodified lines
2506	2506
2507	2507	val = GET_SMSTATE(u32, smstate, 0x7fcc);
2508	2508
2509		− if (ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) \| DR6_FIXED_1))
	2509	+ if (ctxt->ops->set_dr(ctxt, 6, val))
2510	2510	return X86EMUL_UNHANDLEABLE;
2511	2511
2512	2512	val = GET_SMSTATE(u32, smstate, 0x7fc8);
2513	2513
2514		− if (ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) \| DR7_FIXED_1))
	2514	+ if (ctxt->ops->set_dr(ctxt, 7, val))
2515	2515	return X86EMUL_UNHANDLEABLE;
2516	2516
2517	2517	selector = GET_SMSTATE(u32, smstate, 0x7fc4);
@@ -2564,14 +2564,14 @@ static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, · 46 unmodified lines
2564	2564	ctxt->_eip = GET_SMSTATE(u64, smstate, 0x7f78);
2565	2565	ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) \| X86_EFLAGS_FIXED;
2566	2566
2567		− val = GET_SMSTATE(u32, smstate, 0x7f68);
	2567	+ val = GET_SMSTATE(u64, smstate, 0x7f68);
2568	2568
2569		− if (ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) \| DR6_FIXED_1))
	2569	+ if (ctxt->ops->set_dr(ctxt, 6, val))
2570	2570	return X86EMUL_UNHANDLEABLE;
2571	2571
2572		− val = GET_SMSTATE(u32, smstate, 0x7f60);
	2572	+ val = GET_SMSTATE(u64, smstate, 0x7f60);
2573	2573
2574		− if (ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) \| DR7_FIXED_1))
	2574	+ if (ctxt->ops->set_dr(ctxt, 7, val))
2575	2575	return X86EMUL_UNHANDLEABLE;
2576	2576
2577	2577	cr0 = GET_SMSTATE(u64, smstate, 0x7f58);
@@ -4329,7 +4329,7 @@ static int check_dr_read(struct x86_emulate_ctxt *ctxt) · 1751 unmodified lines
4329	4329
4330	4330	ctxt->ops->get_dr(ctxt, 6, &dr6);
4331	4331	dr6 &= ~DR_TRAP_BITS;
4332		− dr6 \|= DR6_BD \| DR6_RTM;
	4332	+ dr6 \|= DR6_BD \| DR6_ACTIVE_LOW;
4333	4333	ctxt->ops->set_dr(ctxt, 6, dr6);
4334	4334	return emulate_db(ctxt);
4335	4335	}

arch/x86/kvm/hyperv.c+209−134

@@ -23,6 +23,7 @@ · 22 unmodified lines
23	23	#include "ioapic.h"
24	24	#include "cpuid.h"
25	25	#include "hyperv.h"
	26	+#include "xen.h"
26	27
27	28	#include <linux/cpu.h>
28	29	#include <linux/kvm_host.h>
@@ -36,6 +37,9 @@ · 7 unmodified lines
36	37	#include "trace.h"
37	38	#include "irq.h"
38	39
	40	+/* "Hv#1" signature */
	41	+#define HYPERV_CPUID_SIGNATURE_EAX 0x31237648
	42	+
39	43	#define KVM_HV_MAX_SPARSE_VCPU_SET_BITS DIV_ROUND_UP(KVM_MAX_VCPUS, 64)
40	44
41	45	static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer,
@@ -128,7 +132,7 @@ static int synic_set_sint(struct kvm_vcpu_hv_synic *synic, int sint, · 86 unmodified lines
128	132	synic_update_vector(synic, vector);
129	133
130	134	/* Load SynIC vectors into EOI exit bitmap */
131		− kvm_make_request(KVM_REQ_SCAN_IOAPIC, synic_to_vcpu(synic));
	135	+ kvm_make_request(KVM_REQ_SCAN_IOAPIC, hv_synic_to_vcpu(synic));
132	136	return 0;
133	137	}
134	138
@@ -141,10 +145,10 @@ static struct kvm_vcpu get_vcpu_by_vpidx(struct kvm kvm, u32 vpidx) · 6 unmodified lines
141	145	return NULL;
142	146
143	147	vcpu = kvm_get_vcpu(kvm, vpidx);
144		− if (vcpu && vcpu_to_hv_vcpu(vcpu)->vp_index == vpidx)
	148	+ if (vcpu && kvm_hv_get_vpindex(vcpu) == vpidx)
145	149	return vcpu;
146	150	kvm_for_each_vcpu(i, vcpu, kvm)
147		− if (vcpu_to_hv_vcpu(vcpu)->vp_index == vpidx)
	151	+ if (kvm_hv_get_vpindex(vcpu) == vpidx)
148	152	return vcpu;
149	153	return NULL;
150	154	}
@@ -157,15 +161,15 @@ static struct kvm_vcpu_hv_synic synic_get(struct kvm kvm, u32 vpidx) · 6 unmodified lines
157	161	vcpu = get_vcpu_by_vpidx(kvm, vpidx);
158	162	if (!vcpu)
159	163	return NULL;
160		− synic = vcpu_to_synic(vcpu);
	164	+ synic = to_hv_synic(vcpu);
161	165	return (synic->active) ? synic : NULL;
162	166	}
163	167
164	168	static void kvm_hv_notify_acked_sint(struct kvm_vcpu *vcpu, u32 sint)
165	169	{
166	170	struct kvm *kvm = vcpu->kvm;
167		− struct kvm_vcpu_hv_synic *synic = vcpu_to_synic(vcpu);
168		− struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);
	171	+ struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu);
	172	+ struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
169	173	struct kvm_vcpu_hv_stimer *stimer;
170	174	int gsi, idx;
171	175
@@ -189,8 +193,8 @@ static void kvm_hv_notify_acked_sint(struct kvm_vcpu *vcpu, u32 sint) · 17 unmodified lines
189	193
190	194	static void synic_exit(struct kvm_vcpu_hv_synic *synic, u32 msr)
191	195	{
192		− struct kvm_vcpu *vcpu = synic_to_vcpu(synic);
193		− struct kvm_vcpu_hv *hv_vcpu = &vcpu->arch.hyperv;
	196	+ struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic);
	197	+ struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
194	198
195	199	hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNIC;
196	200	hv_vcpu->exit.u.synic.msr = msr;
@@ -204,7 +208,7 @@ static void synic_exit(struct kvm_vcpu_hv_synic *synic, u32 msr) · 7 unmodified lines
204	208	static int synic_set_msr(struct kvm_vcpu_hv_synic *synic,
205	209	u32 msr, u64 data, bool host)
206	210	{
207		− struct kvm_vcpu *vcpu = synic_to_vcpu(synic);
	211	+ struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic);
208	212	int ret;
209	213
210	214	if (!synic->active && !host)
@@ -282,8 +286,7 @@ static bool kvm_hv_is_syndbg_enabled(struct kvm_vcpu *vcpu) · 71 unmodified lines
282	286
283	287	static int kvm_hv_syndbg_complete_userspace(struct kvm_vcpu *vcpu)
284	288	{
285		− struct kvm *kvm = vcpu->kvm;
286		− struct kvm_hv *hv = &kvm->arch.hyperv;
	289	+ struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
287	290
288	291	if (vcpu->run->hyperv.u.syndbg.msr == HV_X64_MSR_SYNDBG_CONTROL)
289	292	hv->hv_syndbg.control.status =
@@ -293,8 +296,8 @@ static int kvm_hv_syndbg_complete_userspace(struct kvm_vcpu *vcpu) · 3 unmodified lines
293	296
294	297	static void syndbg_exit(struct kvm_vcpu *vcpu, u32 msr)
295	298	{
296		− struct kvm_hv_syndbg *syndbg = vcpu_to_hv_syndbg(vcpu);
297		− struct kvm_vcpu_hv *hv_vcpu = &vcpu->arch.hyperv;
	299	+ struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu);
	300	+ struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
298	301
299	302	hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNDBG;
300	303	hv_vcpu->exit.u.syndbg.msr = msr;
@@ -310,13 +313,13 @@ static void syndbg_exit(struct kvm_vcpu *vcpu, u32 msr) · 9 unmodified lines
310	313
311	314	static int syndbg_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host)
312	315	{
313		− struct kvm_hv_syndbg *syndbg = vcpu_to_hv_syndbg(vcpu);
	316	+ struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu);
314	317
315	318	if (!kvm_hv_is_syndbg_enabled(vcpu) && !host)
316	319	return 1;
317	320
318	321	trace_kvm_hv_syndbg_set_msr(vcpu->vcpu_id,
319		− vcpu_to_hv_vcpu(vcpu)->vp_index, msr, data);
	322	+ to_hv_vcpu(vcpu)->vp_index, msr, data);
320	323	switch (msr) {
321	324	case HV_X64_MSR_SYNDBG_CONTROL:
322	325	syndbg->control.control = data;
@@ -349,7 +352,7 @@ static int syndbg_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) · 26 unmodified lines
349	352
350	353	static int syndbg_get_msr(struct kvm_vcpu vcpu, u32 msr, u64 pdata, bool host)
351	354	{
352		− struct kvm_hv_syndbg *syndbg = vcpu_to_hv_syndbg(vcpu);
	355	+ struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu);
353	356
354	357	if (!kvm_hv_is_syndbg_enabled(vcpu) && !host)
355	358	return 1;
@@ -377,9 +380,7 @@ static int syndbg_get_msr(struct kvm_vcpu vcpu, u32 msr, u64 pdata, bool host) · 21 unmodified lines
377	380	break;
378	381	}
379	382
380		− trace_kvm_hv_syndbg_get_msr(vcpu->vcpu_id,
381		− vcpu_to_hv_vcpu(vcpu)->vp_index, msr,
382		− *pdata);
	383	+ trace_kvm_hv_syndbg_get_msr(vcpu->vcpu_id, kvm_hv_get_vpindex(vcpu), msr, *pdata);
383	384
384	385	return 0;
385	386	}
@@ -421,7 +422,7 @@ static int synic_get_msr(struct kvm_vcpu_hv_synic synic, u32 msr, u64 pdata, · 35 unmodified lines
421	422
422	423	static int synic_set_irq(struct kvm_vcpu_hv_synic *synic, u32 sint)
423	424	{
424		− struct kvm_vcpu *vcpu = synic_to_vcpu(synic);
	425	+ struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic);
425	426	struct kvm_lapic_irq irq;
426	427	int ret, vector;
427	428
@@ -457,7 +458,7 @@ int kvm_hv_synic_set_irq(struct kvm *kvm, u32 vpidx, u32 sint) · 29 unmodified lines
457	458
458	459	void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector)
459	460	{
460		− struct kvm_vcpu_hv_synic *synic = vcpu_to_synic(vcpu);
	461	+ struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu);
461	462	int i;
462	463
463	464	trace_kvm_hv_synic_send_eoi(vcpu->vcpu_id, vector);
@@ -514,7 +515,7 @@ static void synic_init(struct kvm_vcpu_hv_synic *synic) · 50 unmodified lines
514	515
515	516	static u64 get_time_ref_counter(struct kvm *kvm)
516	517	{
517		− struct kvm_hv *hv = &kvm->arch.hyperv;
	518	+ struct kvm_hv *hv = to_kvm_hv(kvm);
518	519	struct kvm_vcpu *vcpu;
519	520	u64 tsc;
520	521
@@ -534,10 +535,10 @@ static u64 get_time_ref_counter(struct kvm *kvm) · 13 unmodified lines
534	535	static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer,
535	536	bool vcpu_kick)
536	537	{
537		− struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);
	538	+ struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
538	539
539	540	set_bit(stimer->index,
540		− vcpu_to_hv_vcpu(vcpu)->stimer_pending_bitmap);
	541	+ to_hv_vcpu(vcpu)->stimer_pending_bitmap);
541	542	kvm_make_request(KVM_REQ_HV_STIMER, vcpu);
542	543	if (vcpu_kick)
543	544	kvm_vcpu_kick(vcpu);
@@ -545,14 +546,14 @@ static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer, · 1 unmodified line
545	546
546	547	static void stimer_cleanup(struct kvm_vcpu_hv_stimer *stimer)
547	548	{
548		− struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);
	549	+ struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
549	550
550		− trace_kvm_hv_stimer_cleanup(stimer_to_vcpu(stimer)->vcpu_id,
	551	+ trace_kvm_hv_stimer_cleanup(hv_stimer_to_vcpu(stimer)->vcpu_id,
551	552	stimer->index);
552	553
553	554	hrtimer_cancel(&stimer->timer);
554	555	clear_bit(stimer->index,
555		− vcpu_to_hv_vcpu(vcpu)->stimer_pending_bitmap);
	556	+ to_hv_vcpu(vcpu)->stimer_pending_bitmap);
556	557	stimer->msg_pending = false;
557	558	stimer->exp_time = 0;
558	559	}
@@ -562,7 +563,7 @@ static enum hrtimer_restart stimer_timer_callback(struct hrtimer *timer) · 3 unmodified lines
562	563	struct kvm_vcpu_hv_stimer *stimer;
563	564
564	565	stimer = container_of(timer, struct kvm_vcpu_hv_stimer, timer);
565		− trace_kvm_hv_stimer_callback(stimer_to_vcpu(stimer)->vcpu_id,
	566	+ trace_kvm_hv_stimer_callback(hv_stimer_to_vcpu(stimer)->vcpu_id,
566	567	stimer->index);
567	568	stimer_mark_pending(stimer, true);
568	569
@@ -579,7 +580,7 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) · 10 unmodified lines
579	580	u64 time_now;
580	581	ktime_t ktime_now;
581	582
582		− time_now = get_time_ref_counter(stimer_to_vcpu(stimer)->kvm);
	583	+ time_now = get_time_ref_counter(hv_stimer_to_vcpu(stimer)->kvm);
583	584	ktime_now = ktime_get();
584	585
585	586	if (stimer->config.periodic) {
@@ -596,7 +597,7 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) · 10 unmodified lines
596	597	stimer->exp_time = time_now + stimer->count;
597	598
598	599	trace_kvm_hv_stimer_start_periodic(
599		− stimer_to_vcpu(stimer)->vcpu_id,
	600	+ hv_stimer_to_vcpu(stimer)->vcpu_id,
600	601	stimer->index,
601	602	time_now, stimer->exp_time);
602	603
@@ -618,7 +619,7 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) · 15 unmodified lines
618	619	return 0;
619	620	}
620	621
621		− trace_kvm_hv_stimer_start_one_shot(stimer_to_vcpu(stimer)->vcpu_id,
	622	+ trace_kvm_hv_stimer_start_one_shot(hv_stimer_to_vcpu(stimer)->vcpu_id,
622	623	stimer->index,
623	624	time_now, stimer->count);
624	625
@@ -633,13 +634,13 @@ static int stimer_set_config(struct kvm_vcpu_hv_stimer *stimer, u64 config, · 8 unmodified lines
633	634	{
634	635	union hv_stimer_config new_config = {.as_uint64 = config},
635	636	old_config = {.as_uint64 = stimer->config.as_uint64};
636		− struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);
637		− struct kvm_vcpu_hv_synic *synic = vcpu_to_synic(vcpu);
	637	+ struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
	638	+ struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu);
638	639
639	640	if (!synic->active && !host)
640	641	return 1;
641	642
642		− trace_kvm_hv_stimer_set_config(stimer_to_vcpu(stimer)->vcpu_id,
	643	+ trace_kvm_hv_stimer_set_config(hv_stimer_to_vcpu(stimer)->vcpu_id,
643	644	stimer->index, config, host);
644	645
645	646	stimer_cleanup(stimer);
@@ -657,13 +658,13 @@ static int stimer_set_config(struct kvm_vcpu_hv_stimer *stimer, u64 config, · 11 unmodified lines
657	658	static int stimer_set_count(struct kvm_vcpu_hv_stimer *stimer, u64 count,
658	659	bool host)
659	660	{
660		− struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);
661		− struct kvm_vcpu_hv_synic *synic = vcpu_to_synic(vcpu);
	661	+ struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
	662	+ struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu);
662	663
663	664	if (!synic->active && !host)
664	665	return 1;
665	666
666		− trace_kvm_hv_stimer_set_count(stimer_to_vcpu(stimer)->vcpu_id,
	667	+ trace_kvm_hv_stimer_set_count(hv_stimer_to_vcpu(stimer)->vcpu_id,
667	668	stimer->index, count, host);
668	669
669	670	stimer_cleanup(stimer);
@@ -694,7 +695,7 @@ static int stimer_get_count(struct kvm_vcpu_hv_stimer stimer, u64 pcount) · 24 unmodified lines
694	695	static int synic_deliver_msg(struct kvm_vcpu_hv_synic *synic, u32 sint,
695	696	struct hv_message *src_msg, bool no_retry)
696	697	{
697		− struct kvm_vcpu *vcpu = synic_to_vcpu(synic);
	698	+ struct kvm_vcpu *vcpu = hv_synic_to_vcpu(synic);
698	699	int msg_off = offsetof(struct hv_message_page, sint_message[sint]);
699	700	gfn_t msg_page_gfn;
700	701	struct hv_message_header hv_hdr;
@@ -750,7 +751,7 @@ static int synic_deliver_msg(struct kvm_vcpu_hv_synic *synic, u32 sint, · 49 unmodified lines
750	751
751	752	static int stimer_send_msg(struct kvm_vcpu_hv_stimer *stimer)
752	753	{
753		− struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);
	754	+ struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
754	755	struct hv_message *msg = &stimer->msg;
755	756	struct hv_timer_message_payload *payload =
756	757	(struct hv_timer_message_payload *)&msg->u.payload;
@@ -763,14 +764,14 @@ static int stimer_send_msg(struct kvm_vcpu_hv_stimer *stimer) · 6 unmodified lines
763	764
764	765	payload->expiration_time = stimer->exp_time;
765	766	payload->delivery_time = get_time_ref_counter(vcpu->kvm);
766		− return synic_deliver_msg(vcpu_to_synic(vcpu),
	767	+ return synic_deliver_msg(to_hv_synic(vcpu),
767	768	stimer->config.sintx, msg,
768	769	no_retry);
769	770	}
770	771
771	772	static int stimer_notify_direct(struct kvm_vcpu_hv_stimer *stimer)
772	773	{
773		− struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);
	774	+ struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
774	775	struct kvm_lapic_irq irq = {
775	776	.delivery_mode = APIC_DM_FIXED,
776	777	.vector = stimer->config.apic_vector
@@ -790,7 +791,7 @@ static void stimer_expiration(struct kvm_vcpu_hv_stimer *stimer) · 13 unmodified lines
790	791	r = stimer_send_msg(stimer);
791	792	else
792	793	r = stimer_notify_direct(stimer);
793		− trace_kvm_hv_stimer_expiration(stimer_to_vcpu(stimer)->vcpu_id,
	794	+ trace_kvm_hv_stimer_expiration(hv_stimer_to_vcpu(stimer)->vcpu_id,
794	795	stimer->index, direct, r);
795	796	if (!r) {
796	797	stimer->msg_pending = false;
@@ -801,11 +802,14 @@ static void stimer_expiration(struct kvm_vcpu_hv_stimer *stimer) · 4 unmodified lines
801	802
802	803	void kvm_hv_process_stimers(struct kvm_vcpu *vcpu)
803	804	{
804		− struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);
	805	+ struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
805	806	struct kvm_vcpu_hv_stimer *stimer;
806	807	u64 time_now, exp_time;
807	808	int i;
808	809
	810	+ if (!hv_vcpu)
	811	+ return;
	812	+
809	813	for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++)
810	814	if (test_and_clear_bit(i, hv_vcpu->stimer_pending_bitmap)) {
811	815	stimer = &hv_vcpu->stimer[i];
@@ -831,16 +835,27 @@ void kvm_hv_process_stimers(struct kvm_vcpu *vcpu) · 19 unmodified lines
831	835
832	836	void kvm_hv_vcpu_uninit(struct kvm_vcpu *vcpu)
833	837	{
834		− struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);
	838	+ struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
835	839	int i;
836	840
	841	+ if (!hv_vcpu)
	842	+ return;
	843	+
837	844	for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++)
838	845	stimer_cleanup(&hv_vcpu->stimer[i]);
	846	+
	847	+ kfree(hv_vcpu);
	848	+ vcpu->arch.hyperv = NULL;
839	849	}
840	850
841	851	bool kvm_hv_assist_page_enabled(struct kvm_vcpu *vcpu)
842	852	{
843		− if (!(vcpu->arch.hyperv.hv_vapic & HV_X64_MSR_VP_ASSIST_PAGE_ENABLE))
	853	+ struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
	854	+
	855	+ if (!hv_vcpu)
	856	+ return false;
	857	+
	858	+ if (!(hv_vcpu->hv_vapic & HV_X64_MSR_VP_ASSIST_PAGE_ENABLE))
844	859	return false;
845	860	return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
846	861	}
@@ -880,28 +895,41 @@ static void stimer_init(struct kvm_vcpu_hv_stimer *stimer, int timer_index) · 33 unmodified lines
880	895	stimer_prepare_msg(stimer);
881	896	}
882	897
883		−void kvm_hv_vcpu_init(struct kvm_vcpu *vcpu)
	898	+static int kvm_hv_vcpu_init(struct kvm_vcpu *vcpu)
884	899	{
885		− struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);
	900	+ struct kvm_vcpu_hv *hv_vcpu;
886	901	int i;
887	902
	903	+ hv_vcpu = kzalloc(sizeof(struct kvm_vcpu_hv), GFP_KERNEL_ACCOUNT);
	904	+ if (!hv_vcpu)
	905	+ return -ENOMEM;
	906	+
	907	+ vcpu->arch.hyperv = hv_vcpu;
	908	+ hv_vcpu->vcpu = vcpu;
	909	+
888	910	synic_init(&hv_vcpu->synic);
889	911
890	912	bitmap_zero(hv_vcpu->stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT);
891	913	for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++)
892	914	stimer_init(&hv_vcpu->stimer[i], i);
893		−}
894		−
895		−void kvm_hv_vcpu_postcreate(struct kvm_vcpu *vcpu)
896		−{
897		− struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);
898	915
899	916	hv_vcpu->vp_index = kvm_vcpu_get_idx(vcpu);
	917	+
	918	+ return 0;
900	919	}
901	920
902	921	int kvm_hv_activate_synic(struct kvm_vcpu *vcpu, bool dont_zero_synic_pages)
903	922	{
904		− struct kvm_vcpu_hv_synic *synic = vcpu_to_synic(vcpu);
	923	+ struct kvm_vcpu_hv_synic *synic;
	924	+ int r;
	925	+
	926	+ if (!to_hv_vcpu(vcpu)) {
	927	+ r = kvm_hv_vcpu_init(vcpu);
	928	+ if (r)
	929	+ return r;
	930	+ }
	931	+
	932	+ synic = to_hv_synic(vcpu);
905	933
906	934	/*
907	935	* Hyper-V SynIC auto EOI SINT's are
@@ -939,10 +967,9 @@ static bool kvm_hv_msr_partition_wide(u32 msr) · 31 unmodified lines
939	967	return r;
940	968	}
941	969
942		−static int kvm_hv_msr_get_crash_data(struct kvm_vcpu *vcpu,
943		− u32 index, u64 *pdata)
	970	+static int kvm_hv_msr_get_crash_data(struct kvm kvm, u32 index, u64 pdata)
944	971	{
945		− struct kvm_hv *hv = &vcpu->kvm->arch.hyperv;
	972	+ struct kvm_hv *hv = to_kvm_hv(kvm);
946	973	size_t size = ARRAY_SIZE(hv->hv_crash_param);
947	974
948	975	if (WARN_ON_ONCE(index >= size))
@@ -952,41 +979,26 @@ static int kvm_hv_msr_get_crash_data(struct kvm_vcpu *vcpu, · 3 unmodified lines
952	979	return 0;
953	980	}
954	981
955		−static int kvm_hv_msr_get_crash_ctl(struct kvm_vcpu vcpu, u64 pdata)
	982	+static int kvm_hv_msr_get_crash_ctl(struct kvm kvm, u64 pdata)
956	983	{
957		− struct kvm_hv *hv = &vcpu->kvm->arch.hyperv;
	984	+ struct kvm_hv *hv = to_kvm_hv(kvm);
958	985
959	986	*pdata = hv->hv_crash_ctl;
960	987	return 0;
961	988	}
962	989
963		−static int kvm_hv_msr_set_crash_ctl(struct kvm_vcpu *vcpu, u64 data, bool host)
	990	+static int kvm_hv_msr_set_crash_ctl(struct kvm *kvm, u64 data)
964	991	{
965		− struct kvm_hv *hv = &vcpu->kvm->arch.hyperv;
966		−
967		− if (host)
968		− hv->hv_crash_ctl = data & HV_CRASH_CTL_CRASH_NOTIFY;
	992	+ struct kvm_hv *hv = to_kvm_hv(kvm);
969	993
970		− if (!host && (data & HV_CRASH_CTL_CRASH_NOTIFY)) {
971		−
972		− vcpu_debug(vcpu, "hv crash (0x%llx 0x%llx 0x%llx 0x%llx 0x%llx)\n",
973		− hv->hv_crash_param[0],
974		− hv->hv_crash_param[1],
975		− hv->hv_crash_param[2],
976		− hv->hv_crash_param[3],
977		− hv->hv_crash_param[4]);
978		−
979		− /* Send notification about crash to user space */
980		− kvm_make_request(KVM_REQ_HV_CRASH, vcpu);
981		− }
	994	+ hv->hv_crash_ctl = data & HV_CRASH_CTL_CRASH_NOTIFY;
982	995
983	996	return 0;
984	997	}
985	998
986		−static int kvm_hv_msr_set_crash_data(struct kvm_vcpu *vcpu,
987		− u32 index, u64 data)
	999	+static int kvm_hv_msr_set_crash_data(struct kvm *kvm, u32 index, u64 data)
988	1000	{
989		− struct kvm_hv *hv = &vcpu->kvm->arch.hyperv;
	1001	+ struct kvm_hv *hv = to_kvm_hv(kvm);
990	1002	size_t size = ARRAY_SIZE(hv->hv_crash_param);
991	1003
992	1004	if (WARN_ON_ONCE(index >= size))
@@ -1068,7 +1080,7 @@ static bool compute_tsc_page_parameters(struct pvclock_vcpu_time_info *hv_clock, · 75 unmodified lines
1068	1080	void kvm_hv_setup_tsc_page(struct kvm *kvm,
1069	1081	struct pvclock_vcpu_time_info *hv_clock)
1070	1082	{
1071		− struct kvm_hv *hv = &kvm->arch.hyperv;
	1083	+ struct kvm_hv *hv = to_kvm_hv(kvm);
1072	1084	u32 tsc_seq;
1073	1085	u64 gfn;
1074	1086
@@ -1078,7 +1090,7 @@ void kvm_hv_setup_tsc_page(struct kvm *kvm, · 3 unmodified lines
1078	1090	if (!(hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE))
1079	1091	return;
1080	1092
1081		− mutex_lock(&kvm->arch.hyperv.hv_lock);
	1093	+ mutex_lock(&hv->hv_lock);
1082	1094	if (!(hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE))
1083	1095	goto out_unlock;
1084	1096
@@ -1122,14 +1134,14 @@ void kvm_hv_setup_tsc_page(struct kvm *kvm, · 37 unmodified lines
1122	1134	kvm_write_guest(kvm, gfn_to_gpa(gfn),
1123	1135	&hv->tsc_ref, sizeof(hv->tsc_ref.tsc_sequence));
1124	1136	out_unlock:
1125		− mutex_unlock(&kvm->arch.hyperv.hv_lock);
	1137	+ mutex_unlock(&hv->hv_lock);
1126	1138	}
1127	1139
1128	1140	static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data,
1129	1141	bool host)
1130	1142	{
1131	1143	struct kvm *kvm = vcpu->kvm;
1132		− struct kvm_hv *hv = &kvm->arch.hyperv;
	1144	+ struct kvm_hv *hv = to_kvm_hv(kvm);
1133	1145
1134	1146	switch (msr) {
1135	1147	case HV_X64_MSR_GUEST_OS_ID:
@@ -1139,9 +1151,9 @@ static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data, · 3 unmodified lines
1139	1151	hv->hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE;
1140	1152	break;
1141	1153	case HV_X64_MSR_HYPERCALL: {
1142		− u64 gfn;
1143		− unsigned long addr;
1144		− u8 instructions[4];
	1154	+ u8 instructions[9];
	1155	+ int i = 0;
	1156	+ u64 addr;
1145	1157
1146	1158	/* if guest os id is not set hypercall should remain disabled */
1147	1159	if (!hv->hv_guest_os_id)
@@ -1150,16 +1162,33 @@ static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data, · 2 unmodified lines
1150	1162	hv->hv_hypercall = data;
1151	1163	break;
1152	1164	}
1153		− gfn = data >> HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT;
1154		− addr = gfn_to_hva(kvm, gfn);
1155		− if (kvm_is_error_hva(addr))
1156		− return 1;
1157		− kvm_x86_ops.patch_hypercall(vcpu, instructions);
1158		− ((unsigned char )instructions)[3] = 0xc3; / ret */
1159		− if (__copy_to_user((void __user *)addr, instructions, 4))
	1165	+
	1166	+ /*
	1167	+ * If Xen and Hyper-V hypercalls are both enabled, disambiguate
	1168	+ * the same way Xen itself does, by setting the bit 31 of EAX
	1169	+ * which is RsvdZ in the 32-bit Hyper-V hypercall ABI and just
	1170	+ * going to be clobbered on 64-bit.
	1171	+ */
	1172	+ if (kvm_xen_hypercall_enabled(kvm)) {
	1173	+ /* orl $0x80000000, %eax */
	1174	+ instructions[i++] = 0x0d;
	1175	+ instructions[i++] = 0x00;
	1176	+ instructions[i++] = 0x00;
	1177	+ instructions[i++] = 0x00;
	1178	+ instructions[i++] = 0x80;
	1179	+ }
	1180	+
	1181	+ /* vmcall/vmmcall */
	1182	+ static_call(kvm_x86_patch_hypercall)(vcpu, instructions + i);
	1183	+ i += 3;
	1184	+
	1185	+ /* ret */
	1186	+ ((unsigned char *)instructions)[i++] = 0xc3;
	1187	+
	1188	+ addr = data & HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK;
	1189	+ if (kvm_vcpu_write_guest(vcpu, addr, instructions, i))
1160	1190	return 1;
1161	1191	hv->hv_hypercall = data;
1162		− mark_page_dirty(kvm, gfn);
1163	1192	break;
1164	1193	}
1165	1194	case HV_X64_MSR_REFERENCE_TSC:
@@ -1168,11 +1197,25 @@ static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data, · 2 unmodified lines
1168	1197	kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
1169	1198	break;
1170	1199	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
1171		− return kvm_hv_msr_set_crash_data(vcpu,
	1200	+ return kvm_hv_msr_set_crash_data(kvm,
1172	1201	msr - HV_X64_MSR_CRASH_P0,
1173	1202	data);
1174	1203	case HV_X64_MSR_CRASH_CTL:
1175		− return kvm_hv_msr_set_crash_ctl(vcpu, data, host);
	1204	+ if (host)
	1205	+ return kvm_hv_msr_set_crash_ctl(kvm, data);
	1206	+
	1207	+ if (data & HV_CRASH_CTL_CRASH_NOTIFY) {
	1208	+ vcpu_debug(vcpu, "hv crash (0x%llx 0x%llx 0x%llx 0x%llx 0x%llx)\n",
	1209	+ hv->hv_crash_param[0],
	1210	+ hv->hv_crash_param[1],
	1211	+ hv->hv_crash_param[2],
	1212	+ hv->hv_crash_param[3],
	1213	+ hv->hv_crash_param[4]);
	1214	+
	1215	+ /* Send notification about crash to user space */
	1216	+ kvm_make_request(KVM_REQ_HV_CRASH, vcpu);
	1217	+ }
	1218	+ break;
1176	1219	case HV_X64_MSR_RESET:
1177	1220	if (data == 1) {
1178	1221	vcpu_debug(vcpu, "hyper-v reset requested\n");
@@ -1216,11 +1259,11 @@ static u64 current_task_runtime_100ns(void) · 37 unmodified lines
1216	1259
1217	1260	static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host)
1218	1261	{
1219		− struct kvm_vcpu_hv *hv_vcpu = &vcpu->arch.hyperv;
	1262	+ struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
1220	1263
1221	1264	switch (msr) {
1222	1265	case HV_X64_MSR_VP_INDEX: {
1223		− struct kvm_hv *hv = &vcpu->kvm->arch.hyperv;
	1266	+ struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
1224	1267	int vcpu_idx = kvm_vcpu_get_idx(vcpu);
1225	1268	u32 new_vp_index = (u32)data;
1226	1269
@@ -1291,14 +1334,14 @@ static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) · 64 unmodified lines
1291	1334	case HV_X64_MSR_SIMP:
1292	1335	case HV_X64_MSR_EOM:
1293	1336	case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:
1294		− return synic_set_msr(vcpu_to_synic(vcpu), msr, data, host);
	1337	+ return synic_set_msr(to_hv_synic(vcpu), msr, data, host);
1295	1338	case HV_X64_MSR_STIMER0_CONFIG:
1296	1339	case HV_X64_MSR_STIMER1_CONFIG:
1297	1340	case HV_X64_MSR_STIMER2_CONFIG:
1298	1341	case HV_X64_MSR_STIMER3_CONFIG: {
1299	1342	int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2;
1300	1343
1301		− return stimer_set_config(vcpu_to_stimer(vcpu, timer_index),
	1344	+ return stimer_set_config(to_hv_stimer(vcpu, timer_index),
1302	1345	data, host);
1303	1346	}
1304	1347	case HV_X64_MSR_STIMER0_COUNT:
@@ -1307,7 +1350,7 @@ static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) · 2 unmodified lines
1307	1350	case HV_X64_MSR_STIMER3_COUNT: {
1308	1351	int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2;
1309	1352
1310		− return stimer_set_count(vcpu_to_stimer(vcpu, timer_index),
	1353	+ return stimer_set_count(to_hv_stimer(vcpu, timer_index),
1311	1354	data, host);
1312	1355	}
1313	1356	case HV_X64_MSR_TSC_FREQUENCY:
@@ -1330,7 +1373,7 @@ static int kvm_hv_get_msr_pw(struct kvm_vcpu vcpu, u32 msr, u64 pdata, · 16 unmodified lines
1330	1373	{
1331	1374	u64 data = 0;
1332	1375	struct kvm *kvm = vcpu->kvm;
1333		− struct kvm_hv *hv = &kvm->arch.hyperv;
	1376	+ struct kvm_hv *hv = to_kvm_hv(kvm);
1334	1377
1335	1378	switch (msr) {
1336	1379	case HV_X64_MSR_GUEST_OS_ID:
@@ -1346,11 +1389,11 @@ static int kvm_hv_get_msr_pw(struct kvm_vcpu vcpu, u32 msr, u64 pdata, · 9 unmodified lines
1346	1389	data = hv->hv_tsc_page;
1347	1390	break;
1348	1391	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
1349		− return kvm_hv_msr_get_crash_data(vcpu,
	1392	+ return kvm_hv_msr_get_crash_data(kvm,
1350	1393	msr - HV_X64_MSR_CRASH_P0,
1351	1394	pdata);
1352	1395	case HV_X64_MSR_CRASH_CTL:
1353		− return kvm_hv_msr_get_crash_ctl(vcpu, pdata);
	1396	+ return kvm_hv_msr_get_crash_ctl(kvm, pdata);
1354	1397	case HV_X64_MSR_RESET:
1355	1398	data = 0;
1356	1399	break;
@@ -1379,7 +1422,7 @@ static int kvm_hv_get_msr(struct kvm_vcpu vcpu, u32 msr, u64 pdata, · 22 unmodified lines
1379	1422	bool host)
1380	1423	{
1381	1424	u64 data = 0;
1382		− struct kvm_vcpu_hv *hv_vcpu = &vcpu->arch.hyperv;
	1425	+ struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
1383	1426
1384	1427	switch (msr) {
1385	1428	case HV_X64_MSR_VP_INDEX:
@@ -1403,14 +1446,14 @@ static int kvm_hv_get_msr(struct kvm_vcpu vcpu, u32 msr, u64 pdata, · 17 unmodified lines
1403	1446	case HV_X64_MSR_SIMP:
1404	1447	case HV_X64_MSR_EOM:
1405	1448	case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:
1406		− return synic_get_msr(vcpu_to_synic(vcpu), msr, pdata, host);
	1449	+ return synic_get_msr(to_hv_synic(vcpu), msr, pdata, host);
1407	1450	case HV_X64_MSR_STIMER0_CONFIG:
1408	1451	case HV_X64_MSR_STIMER1_CONFIG:
1409	1452	case HV_X64_MSR_STIMER2_CONFIG:
1410	1453	case HV_X64_MSR_STIMER3_CONFIG: {
1411	1454	int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2;
1412	1455
1413		− return stimer_get_config(vcpu_to_stimer(vcpu, timer_index),
	1456	+ return stimer_get_config(to_hv_stimer(vcpu, timer_index),
1414	1457	pdata);
1415	1458	}
1416	1459	case HV_X64_MSR_STIMER0_COUNT:
@@ -1419,7 +1462,7 @@ static int kvm_hv_get_msr(struct kvm_vcpu vcpu, u32 msr, u64 pdata, · 2 unmodified lines
1419	1462	case HV_X64_MSR_STIMER3_COUNT: {
1420	1463	int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2;
1421	1464
1422		− return stimer_get_count(vcpu_to_stimer(vcpu, timer_index),
	1465	+ return stimer_get_count(to_hv_stimer(vcpu, timer_index),
1423	1466	pdata);
1424	1467	}
1425	1468	case HV_X64_MSR_TSC_FREQUENCY:
@@ -1438,12 +1481,22 @@ static int kvm_hv_get_msr(struct kvm_vcpu vcpu, u32 msr, u64 pdata, · 12 unmodified lines
1438	1481
1439	1482	int kvm_hv_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host)
1440	1483	{
	1484	+ struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
	1485	+
	1486	+ if (!host && !vcpu->arch.hyperv_enabled)
	1487	+ return 1;
	1488	+
	1489	+ if (!to_hv_vcpu(vcpu)) {
	1490	+ if (kvm_hv_vcpu_init(vcpu))
	1491	+ return 1;
	1492	+ }
	1493	+
1441	1494	if (kvm_hv_msr_partition_wide(msr)) {
1442	1495	int r;
1443	1496
1444		− mutex_lock(&vcpu->kvm->arch.hyperv.hv_lock);
	1497	+ mutex_lock(&hv->hv_lock);
1445	1498	r = kvm_hv_set_msr_pw(vcpu, msr, data, host);
1446		− mutex_unlock(&vcpu->kvm->arch.hyperv.hv_lock);
	1499	+ mutex_unlock(&hv->hv_lock);
1447	1500	return r;
1448	1501	} else
1449	1502	return kvm_hv_set_msr(vcpu, msr, data, host);
@@ -1451,12 +1504,22 @@ int kvm_hv_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) · 1 unmodified line
1451	1504
1452	1505	int kvm_hv_get_msr_common(struct kvm_vcpu vcpu, u32 msr, u64 pdata, bool host)
1453	1506	{
	1507	+ struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
	1508	+
	1509	+ if (!host && !vcpu->arch.hyperv_enabled)
	1510	+ return 1;
	1511	+
	1512	+ if (!to_hv_vcpu(vcpu)) {
	1513	+ if (kvm_hv_vcpu_init(vcpu))
	1514	+ return 1;
	1515	+ }
	1516	+
1454	1517	if (kvm_hv_msr_partition_wide(msr)) {
1455	1518	int r;
1456	1519
1457		− mutex_lock(&vcpu->kvm->arch.hyperv.hv_lock);
	1520	+ mutex_lock(&hv->hv_lock);
1458	1521	r = kvm_hv_get_msr_pw(vcpu, msr, pdata, host);
1459		− mutex_unlock(&vcpu->kvm->arch.hyperv.hv_lock);
	1522	+ mutex_unlock(&hv->hv_lock);
1460	1523	return r;
1461	1524	} else
1462	1525	return kvm_hv_get_msr(vcpu, msr, pdata, host);
@@ -1466,7 +1529,7 @@ static __always_inline unsigned long *sparse_set_to_vcpu_mask( · 3 unmodified lines
1466	1529	struct kvm kvm, u64 sparse_banks, u64 valid_bank_mask,
1467	1530	u64 vp_bitmap, unsigned long vcpu_bitmap)
1468	1531	{
1469		− struct kvm_hv *hv = &kvm->arch.hyperv;
	1532	+ struct kvm_hv *hv = to_kvm_hv(kvm);
1470	1533	struct kvm_vcpu *vcpu;
1471	1534	int i, bank, sbank = 0;
1472	1535
@@ -1483,18 +1546,16 @@ static __always_inline unsigned long *sparse_set_to_vcpu_mask( · 10 unmodified lines
1483	1546
1484	1547	bitmap_zero(vcpu_bitmap, KVM_MAX_VCPUS);
1485	1548	kvm_for_each_vcpu(i, vcpu, kvm) {
1486		− if (test_bit(vcpu_to_hv_vcpu(vcpu)->vp_index,
1487		− (unsigned long *)vp_bitmap))
	1549	+ if (test_bit(kvm_hv_get_vpindex(vcpu), (unsigned long *)vp_bitmap))
1488	1550	__set_bit(i, vcpu_bitmap);
1489	1551	}
1490	1552	return vcpu_bitmap;
1491	1553	}
1492	1554
1493		−static u64 kvm_hv_flush_tlb(struct kvm_vcpu *current_vcpu, u64 ingpa,
1494		− u16 rep_cnt, bool ex)
	1555	+static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, u64 ingpa, u16 rep_cnt, bool ex)
1495	1556	{
1496		− struct kvm *kvm = current_vcpu->kvm;
1497		− struct kvm_vcpu_hv *hv_vcpu = &current_vcpu->arch.hyperv;
	1557	+ struct kvm *kvm = vcpu->kvm;
	1558	+ struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
1498	1559	struct hv_tlb_flush_ex flush_ex;
1499	1560	struct hv_tlb_flush flush;
1500	1561	u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS];
@@ -1592,10 +1653,10 @@ static void kvm_send_ipi_to_many(struct kvm *kvm, u32 vector, · 91 unmodified lines
1592	1653	}
1593	1654	}
1594	1655
1595		−static u64 kvm_hv_send_ipi(struct kvm_vcpu *current_vcpu, u64 ingpa, u64 outgpa,
	1656	+static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, u64 ingpa, u64 outgpa,
1596	1657	bool ex, bool fast)
1597	1658	{
1598		− struct kvm *kvm = current_vcpu->kvm;
	1659	+ struct kvm *kvm = vcpu->kvm;
1599	1660	struct hv_send_ipi_ex send_ipi_ex;
1600	1661	struct hv_send_ipi send_ipi;
1601	1662	u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS];
@@ -1666,9 +1727,20 @@ ret_success: · 64 unmodified lines
1666	1727	return HV_STATUS_SUCCESS;
1667	1728	}
1668	1729
1669		−bool kvm_hv_hypercall_enabled(struct kvm *kvm)
	1730	+void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu)
1670	1731	{
1671		− return READ_ONCE(kvm->arch.hyperv.hv_guest_os_id) != 0;
	1732	+ struct kvm_cpuid_entry2 *entry;
	1733	+
	1734	+ entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_INTERFACE, 0);
	1735	+ if (entry && entry->eax == HYPERV_CPUID_SIGNATURE_EAX)
	1736	+ vcpu->arch.hyperv_enabled = true;
	1737	+ else
	1738	+ vcpu->arch.hyperv_enabled = false;
	1739	+}
	1740	+
	1741	+bool kvm_hv_hypercall_enabled(struct kvm_vcpu *vcpu)
	1742	+{
	1743	+ return vcpu->arch.hyperv_enabled && to_kvm_hv(vcpu->kvm)->hv_guest_os_id;
1672	1744	}
1673	1745
1674	1746	static void kvm_hv_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result)
@@ -1698,6 +1770,7 @@ static int kvm_hv_hypercall_complete_userspace(struct kvm_vcpu *vcpu) · 23 unmodified lines
1698	1770
1699	1771	static u16 kvm_hvcall_signal_event(struct kvm_vcpu *vcpu, bool fast, u64 param)
1700	1772	{
	1773	+ struct kvm_hv *hv = to_kvm_hv(vcpu->kvm);
1701	1774	struct eventfd_ctx *eventfd;
1702	1775
1703	1776	if (unlikely(!fast)) {
@@ -1726,7 +1799,7 @@ static u16 kvm_hvcall_signal_event(struct kvm_vcpu *vcpu, bool fast, u64 param) · 22 unmodified lines
1726	1799
1727	1800	/* the eventfd is protected by vcpu->kvm->srcu, but conn_to_evt isn't */
1728	1801	rcu_read_lock();
1729		− eventfd = idr_find(&vcpu->kvm->arch.hyperv.conn_to_evt, param);
	1802	+ eventfd = idr_find(&hv->conn_to_evt, param);
1730	1803	rcu_read_unlock();
1731	1804	if (!eventfd)
1732	1805	return HV_STATUS_INVALID_PORT_ID;
@@ -1745,7 +1818,7 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu) · 12 unmodified lines
1745	1818	* hypercall generates UD from non zero cpl and real mode
1746	1819	* per HYPER-V spec
1747	1820	*/
1748		− if (kvm_x86_ops.get_cpl(vcpu) != 0 \|\| !is_protmode(vcpu)) {
	1821	+ if (static_call(kvm_x86_get_cpl)(vcpu) != 0 \|\| !is_protmode(vcpu)) {
1749	1822	kvm_queue_exception(vcpu, UD_VECTOR);
1750	1823	return 1;
1751	1824	}
@@ -1793,7 +1866,7 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu) · 41 unmodified lines
1793	1866	fallthrough; /* maybe userspace knows this conn_id */
1794	1867	case HVCALL_POST_MESSAGE:
1795	1868	/* don't bother userspace if it has no way to handle it */
1796		− if (unlikely(rep \|\| !vcpu_to_synic(vcpu)->active)) {
	1869	+ if (unlikely(rep \|\| !to_hv_synic(vcpu)->active)) {
1797	1870	ret = HV_STATUS_INVALID_HYPERCALL_INPUT;
1798	1871	break;
1799	1872	}
@@ -1855,7 +1928,7 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu) · 55 unmodified lines
1855	1928	}
1856	1929	fallthrough;
1857	1930	case HVCALL_RESET_DEBUG_SESSION: {
1858		− struct kvm_hv_syndbg *syndbg = vcpu_to_hv_syndbg(vcpu);
	1931	+ struct kvm_hv_syndbg *syndbg = to_hv_syndbg(vcpu);
1859	1932
1860	1933	if (!kvm_hv_is_syndbg_enabled(vcpu)) {
1861	1934	ret = HV_STATUS_INVALID_HYPERCALL_CODE;
@@ -1885,23 +1958,26 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu) · 23 unmodified lines
1885	1958
1886	1959	void kvm_hv_init_vm(struct kvm *kvm)
1887	1960	{
1888		− mutex_init(&kvm->arch.hyperv.hv_lock);
1889		− idr_init(&kvm->arch.hyperv.conn_to_evt);
	1961	+ struct kvm_hv *hv = to_kvm_hv(kvm);
	1962	+
	1963	+ mutex_init(&hv->hv_lock);
	1964	+ idr_init(&hv->conn_to_evt);
1890	1965	}
1891	1966
1892	1967	void kvm_hv_destroy_vm(struct kvm *kvm)
1893	1968	{
	1969	+ struct kvm_hv *hv = to_kvm_hv(kvm);
1894	1970	struct eventfd_ctx *eventfd;
1895	1971	int i;
1896	1972
1897		− idr_for_each_entry(&kvm->arch.hyperv.conn_to_evt, eventfd, i)
	1973	+ idr_for_each_entry(&hv->conn_to_evt, eventfd, i)
1898	1974	eventfd_ctx_put(eventfd);
1899		− idr_destroy(&kvm->arch.hyperv.conn_to_evt);
	1975	+ idr_destroy(&hv->conn_to_evt);
1900	1976	}
1901	1977
1902	1978	static int kvm_hv_eventfd_assign(struct kvm *kvm, u32 conn_id, int fd)
1903	1979	{
1904		− struct kvm_hv *hv = &kvm->arch.hyperv;
	1980	+ struct kvm_hv *hv = to_kvm_hv(kvm);
1905	1981	struct eventfd_ctx *eventfd;
1906	1982	int ret;
1907	1983
@@ -1925,7 +2001,7 @@ static int kvm_hv_eventfd_assign(struct kvm *kvm, u32 conn_id, int fd) · 17 unmodified lines
1925	2001
1926	2002	static int kvm_hv_eventfd_deassign(struct kvm *kvm, u32 conn_id)
1927	2003	{
1928		− struct kvm_hv *hv = &kvm->arch.hyperv;
	2004	+ struct kvm_hv *hv = to_kvm_hv(kvm);
1929	2005	struct eventfd_ctx *eventfd;
1930	2006
1931	2007	mutex_lock(&hv->hv_lock);
@@ -1997,8 +2073,7 @@ int kvm_get_hv_cpuid(struct kvm_vcpu vcpu, struct kvm_cpuid2 cpuid, · 65 unmodified lines
1997	2073	break;
1998	2074
1999	2075	case HYPERV_CPUID_INTERFACE:
2000		− memcpy(signature, "Hv#1\0\0\0\0\0\0\0\0", 12);
2001		− ent->eax = signature[0];
	2076	+ ent->eax = HYPERV_CPUID_SIGNATURE_EAX;
2002	2077	break;
2003	2078
2004	2079	case HYPERV_CPUID_VERSION:

arch/x86/kvm/hyperv.h+33−21

@@ -50,38 +50,46 @@ · 49 unmodified lines
50	50	/* Hyper-V HV_X64_MSR_SYNDBG_OPTIONS bits */
51	51	#define HV_X64_SYNDBG_OPTION_USE_HCALLS BIT(2)
52	52
53		−static inline struct kvm_vcpu_hv vcpu_to_hv_vcpu(struct kvm_vcpu vcpu)
	53	+static inline struct kvm_hv to_kvm_hv(struct kvm kvm)
54	54	{
55		− return &vcpu->arch.hyperv;
	55	+ return &kvm->arch.hyperv;
56	56	}
57	57
58		−static inline struct kvm_vcpu hv_vcpu_to_vcpu(struct kvm_vcpu_hv hv_vcpu)
	58	+static inline struct kvm_vcpu_hv to_hv_vcpu(struct kvm_vcpu vcpu)
59	59	{
60		− struct kvm_vcpu_arch *arch;
61		−
62		− arch = container_of(hv_vcpu, struct kvm_vcpu_arch, hyperv);
63		− return container_of(arch, struct kvm_vcpu, arch);
	60	+ return vcpu->arch.hyperv;
64	61	}
65	62
66		−static inline struct kvm_vcpu_hv_synic vcpu_to_synic(struct kvm_vcpu vcpu)
	63	+static inline struct kvm_vcpu_hv_synic to_hv_synic(struct kvm_vcpu vcpu)
67	64	{
68		− return &vcpu->arch.hyperv.synic;
	65	+ struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
	66	+
	67	+ return &hv_vcpu->synic;
69	68	}
70	69
71		−static inline struct kvm_vcpu synic_to_vcpu(struct kvm_vcpu_hv_synic synic)
	70	+static inline struct kvm_vcpu hv_synic_to_vcpu(struct kvm_vcpu_hv_synic synic)
72	71	{
73		− return hv_vcpu_to_vcpu(container_of(synic, struct kvm_vcpu_hv, synic));
	72	+ struct kvm_vcpu_hv *hv_vcpu = container_of(synic, struct kvm_vcpu_hv, synic);
	73	+
	74	+ return hv_vcpu->vcpu;
74	75	}
75	76
76		−static inline struct kvm_hv_syndbg vcpu_to_hv_syndbg(struct kvm_vcpu vcpu)
	77	+static inline struct kvm_hv_syndbg to_hv_syndbg(struct kvm_vcpu vcpu)
77	78	{
78	79	return &vcpu->kvm->arch.hyperv.hv_syndbg;
79	80	}
80	81
	82	+static inline u32 kvm_hv_get_vpindex(struct kvm_vcpu *vcpu)
	83	+{
	84	+ struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
	85	+
	86	+ return hv_vcpu ? hv_vcpu->vp_index : kvm_vcpu_get_idx(vcpu);
	87	+}
	88	+
81	89	int kvm_hv_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host);
82	90	int kvm_hv_get_msr_common(struct kvm_vcpu vcpu, u32 msr, u64 pdata, bool host);
83	91
84		−bool kvm_hv_hypercall_enabled(struct kvm *kvm);
	92	+bool kvm_hv_hypercall_enabled(struct kvm_vcpu *vcpu);
85	93	int kvm_hv_hypercall(struct kvm_vcpu *vcpu);
86	94
87	95	void kvm_hv_irq_routing_update(struct kvm *kvm);
@@ -89,32 +97,35 @@ int kvm_hv_synic_set_irq(struct kvm *kvm, u32 vcpu_id, u32 sint); · 1 unmodified line
89	97	void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector);
90	98	int kvm_hv_activate_synic(struct kvm_vcpu *vcpu, bool dont_zero_synic_pages);
91	99
92		−void kvm_hv_vcpu_init(struct kvm_vcpu *vcpu);
93		−void kvm_hv_vcpu_postcreate(struct kvm_vcpu *vcpu);
94	100	void kvm_hv_vcpu_uninit(struct kvm_vcpu *vcpu);
95	101
96	102	bool kvm_hv_assist_page_enabled(struct kvm_vcpu *vcpu);
97	103	bool kvm_hv_get_assist_page(struct kvm_vcpu *vcpu,
98	104	struct hv_vp_assist_page *assist_page);
99	105
100		−static inline struct kvm_vcpu_hv_stimer vcpu_to_stimer(struct kvm_vcpu vcpu,
101		− int timer_index)
	106	+static inline struct kvm_vcpu_hv_stimer to_hv_stimer(struct kvm_vcpu vcpu,
	107	+ int timer_index)
102	108	{
103		− return &vcpu_to_hv_vcpu(vcpu)->stimer[timer_index];
	109	+ return &to_hv_vcpu(vcpu)->stimer[timer_index];
104	110	}
105	111
106		−static inline struct kvm_vcpu stimer_to_vcpu(struct kvm_vcpu_hv_stimer stimer)
	112	+static inline struct kvm_vcpu hv_stimer_to_vcpu(struct kvm_vcpu_hv_stimer stimer)
107	113	{
108	114	struct kvm_vcpu_hv *hv_vcpu;
109	115
110	116	hv_vcpu = container_of(stimer - stimer->index, struct kvm_vcpu_hv,
111	117	stimer[0]);
112		− return hv_vcpu_to_vcpu(hv_vcpu);
	118	+ return hv_vcpu->vcpu;
113	119	}
114	120
115	121	static inline bool kvm_hv_has_stimer_pending(struct kvm_vcpu *vcpu)
116	122	{
117		− return !bitmap_empty(vcpu->arch.hyperv.stimer_pending_bitmap,
	123	+ struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
	124	+
	125	+ if (!hv_vcpu)
	126	+ return false;
	127	+
	128	+ return !bitmap_empty(hv_vcpu->stimer_pending_bitmap,
118	129	HV_SYNIC_STIMER_COUNT);
119	130	}
120	131
@@ -125,6 +136,7 @@ void kvm_hv_setup_tsc_page(struct kvm *kvm, · 4 unmodified lines
125	136
126	137	void kvm_hv_init_vm(struct kvm *kvm);
127	138	void kvm_hv_destroy_vm(struct kvm *kvm);
	139	+void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu);
128	140	int kvm_vm_ioctl_hv_eventfd(struct kvm kvm, struct kvm_hyperv_eventfd args);
129	141	int kvm_get_hv_cpuid(struct kvm_vcpu vcpu, struct kvm_cpuid2 cpuid,
130	142	struct kvm_cpuid_entry2 __user *entries);

arch/x86/kvm/irq.c+8−2

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	#include "irq.h"
15	15	#include "i8254.h"
16	16	#include "x86.h"
	17	+#include "xen.h"
17	18
18	19	/*
19	20	* check if there are pending timer events
@@ -56,6 +57,9 @@ int kvm_cpu_has_extint(struct kvm_vcpu *v) · 36 unmodified lines
56	57	if (!lapic_in_kernel(v))
57	58	return v->arch.interrupt.injected;
58	59
	60	+ if (kvm_xen_has_interrupt(v))
	61	+ return 1;
	62	+
59	63	if (!kvm_apic_accept_pic_intr(v))
60	64	return 0;
61	65
@@ -110,6 +114,9 @@ static int kvm_cpu_get_extint(struct kvm_vcpu *v) · 48 unmodified lines
110	114	if (!lapic_in_kernel(v))
111	115	return v->arch.interrupt.nr;
112	116
	117	+ if (kvm_xen_has_interrupt(v))
	118	+ return v->kvm->arch.xen.upcall_vector;
	119	+
113	120	if (irqchip_split(v->kvm)) {
114	121	int vector = v->arch.pending_external_vector;
115	122
@@ -143,8 +150,7 @@ void __kvm_migrate_timers(struct kvm_vcpu *vcpu) · 27 unmodified lines
143	150	{
144	151	__kvm_migrate_apic_timer(vcpu);
145	152	__kvm_migrate_pit_timer(vcpu);
146		− if (kvm_x86_ops.migrate_timers)
147		− kvm_x86_ops.migrate_timers(vcpu);
	153	+ static_call_cond(kvm_x86_migrate_timers)(vcpu);
148	154	}
149	155
150	156	bool kvm_arch_irqfd_allowed(struct kvm kvm, struct kvm_irqfd args)

arch/x86/kvm/kvm_cache_regs.h+5−5

@@ -68,7 +68,7 @@ static inline unsigned long kvm_register_read(struct kvm_vcpu *vcpu, int reg) · 67 unmodified lines
68	68	return 0;
69	69
70	70	if (!kvm_register_is_available(vcpu, reg))
71		− kvm_x86_ops.cache_reg(vcpu, reg);
	71	+ static_call(kvm_x86_cache_reg)(vcpu, reg);
72	72
73	73	return vcpu->arch.regs[reg];
74	74	}
@@ -108,7 +108,7 @@ static inline u64 kvm_pdptr_read(struct kvm_vcpu *vcpu, int index) · 33 unmodified lines
108	108	might_sleep(); /* on svm */
109	109
110	110	if (!kvm_register_is_available(vcpu, VCPU_EXREG_PDPTR))
111		− kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_PDPTR);
	111	+ static_call(kvm_x86_cache_reg)(vcpu, VCPU_EXREG_PDPTR);
112	112
113	113	return vcpu->arch.walk_mmu->pdptrs[index];
114	114	}
@@ -118,7 +118,7 @@ static inline ulong kvm_read_cr0_bits(struct kvm_vcpu *vcpu, ulong mask) · 3 unmodified lines
118	118	ulong tmask = mask & KVM_POSSIBLE_CR0_GUEST_BITS;
119	119	if ((tmask & vcpu->arch.cr0_guest_owned_bits) &&
120	120	!kvm_register_is_available(vcpu, VCPU_EXREG_CR0))
121		− kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_CR0);
	121	+ static_call(kvm_x86_cache_reg)(vcpu, VCPU_EXREG_CR0);
122	122	return vcpu->arch.cr0 & mask;
123	123	}
124	124
@@ -132,14 +132,14 @@ static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask) · 7 unmodified lines
132	132	ulong tmask = mask & KVM_POSSIBLE_CR4_GUEST_BITS;
133	133	if ((tmask & vcpu->arch.cr4_guest_owned_bits) &&
134	134	!kvm_register_is_available(vcpu, VCPU_EXREG_CR4))
135		− kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_CR4);
	135	+ static_call(kvm_x86_cache_reg)(vcpu, VCPU_EXREG_CR4);
136	136	return vcpu->arch.cr4 & mask;
137	137	}
138	138
139	139	static inline ulong kvm_read_cr3(struct kvm_vcpu *vcpu)
140	140	{
141	141	if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3))
142		− kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_CR3);
	142	+ static_call(kvm_x86_cache_reg)(vcpu, VCPU_EXREG_CR3);
143	143	return vcpu->arch.cr3;
144	144	}
145	145

arch/x86/kvm/kvm_emulate.h+1−1

@@ -205,7 +205,7 @@ struct x86_emulate_ops { · 204 unmodified lines
205	205	ulong (get_cr)(struct x86_emulate_ctxt ctxt, int cr);
206	206	int (set_cr)(struct x86_emulate_ctxt ctxt, int cr, ulong val);
207	207	int (cpl)(struct x86_emulate_ctxt ctxt);
208		− int (get_dr)(struct x86_emulate_ctxt ctxt, int dr, ulong *dest);
	208	+ void (get_dr)(struct x86_emulate_ctxt ctxt, int dr, ulong *dest);
209	209	int (set_dr)(struct x86_emulate_ctxt ctxt, int dr, ulong value);
210	210	u64 (get_smbase)(struct x86_emulate_ctxt ctxt);
211	211	void (set_smbase)(struct x86_emulate_ctxt ctxt, u64 smbase);

arch/x86/kvm/lapic.c+27−33

@@ -91,8 +91,8 @@ static inline int __apic_test_and_clear_vector(int vec, void *bitmap) · 90 unmodified lines
91	91	return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
92	92	}
93	93
94		−struct static_key_deferred apic_hw_disabled __read_mostly;
95		−struct static_key_deferred apic_sw_disabled __read_mostly;
	94	+__read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_hw_disabled, HZ);
	95	+__read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_sw_disabled, HZ);
96	96
97	97	static inline int apic_enabled(struct kvm_lapic *apic)
98	98	{
@@ -290,9 +290,9 @@ static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val) · 191 unmodified lines
290	290	if (enabled != apic->sw_enabled) {
291	291	apic->sw_enabled = enabled;
292	292	if (enabled)
293		− static_key_slow_dec_deferred(&apic_sw_disabled);
	293	+ static_branch_slow_dec_deferred(&apic_sw_disabled);
294	294	else
295		− static_key_slow_inc(&apic_sw_disabled.key);
	295	+ static_branch_inc(&apic_sw_disabled.key);
296	296
297	297	atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
298	298	}
@@ -484,7 +484,7 @@ static inline void apic_clear_irr(int vec, struct kvm_lapic *apic) · 185 unmodified lines
484	484	if (unlikely(vcpu->arch.apicv_active)) {
485	485	/* need to update RVI */
486	486	kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
487		− kvm_x86_ops.hwapic_irr_update(vcpu,
	487	+ static_call(kvm_x86_hwapic_irr_update)(vcpu,
488	488	apic_find_highest_irr(apic));
489	489	} else {
490	490	apic->irr_pending = false;
@@ -515,7 +515,7 @@ static inline void apic_set_isr(int vec, struct kvm_lapic *apic) · 24 unmodified lines
515	515	* just set SVI.
516	516	*/
517	517	if (unlikely(vcpu->arch.apicv_active))
518		− kvm_x86_ops.hwapic_isr_update(vcpu, vec);
	518	+ static_call(kvm_x86_hwapic_isr_update)(vcpu, vec);
519	519	else {
520	520	++apic->isr_count;
521	521	BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
@@ -563,8 +563,8 @@ static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) · 41 unmodified lines
563	563	* and must be left alone.
564	564	*/
565	565	if (unlikely(vcpu->arch.apicv_active))
566		− kvm_x86_ops.hwapic_isr_update(vcpu,
567		− apic_find_highest_isr(apic));
	566	+ static_call(kvm_x86_hwapic_isr_update)(vcpu,
	567	+ apic_find_highest_isr(apic));
568	568	else {
569	569	--apic->isr_count;
570	570	BUG_ON(apic->isr_count < 0);
@@ -701,7 +701,7 @@ static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr) · 130 unmodified lines
701	701	{
702	702	int highest_irr;
703	703	if (apic->vcpu->arch.apicv_active)
704		− highest_irr = kvm_x86_ops.sync_pir_to_irr(apic->vcpu);
	704	+ highest_irr = static_call(kvm_x86_sync_pir_to_irr)(apic->vcpu);
705	705	else
706	706	highest_irr = apic_find_highest_irr(apic);
707	707	if (highest_irr == -1 \|\| (highest_irr & 0xF0) <= ppr)
@@ -1090,7 +1090,7 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, · 382 unmodified lines
1090	1090	apic->regs + APIC_TMR);
1091	1091	}
1092	1092
1093		− if (kvm_x86_ops.deliver_posted_interrupt(vcpu, vector)) {
	1093	+ if (static_call(kvm_x86_deliver_posted_interrupt)(vcpu, vector)) {
1094	1094	kvm_lapic_set_irr(vector, apic);
1095	1095	kvm_make_request(KVM_REQ_EVENT, vcpu);
1096	1096	kvm_vcpu_kick(vcpu);
@@ -1245,7 +1245,8 @@ static int apic_set_eoi(struct kvm_lapic *apic) · 148 unmodified lines
1245	1245	apic_clear_isr(vector, apic);
1246	1246	apic_update_ppr(apic);
1247	1247
1248		− if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap))
	1248	+ if (to_hv_vcpu(apic->vcpu) &&
	1249	+ test_bit(vector, to_hv_synic(apic->vcpu)->vec_bitmap))
1249	1250	kvm_hv_synic_send_eoi(apic->vcpu, vector);
1250	1251
1251	1252	kvm_ioapic_send_eoi(apic, vector);
@@ -1814,7 +1815,7 @@ static void cancel_hv_timer(struct kvm_lapic *apic) · 562 unmodified lines
1814	1815	{
1815	1816	WARN_ON(preemptible());
1816	1817	WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1817		− kvm_x86_ops.cancel_hv_timer(apic->vcpu);
	1818	+ static_call(kvm_x86_cancel_hv_timer)(apic->vcpu);
1818	1819	apic->lapic_timer.hv_timer_in_use = false;
1819	1820	}
1820	1821
@@ -1831,7 +1832,7 @@ static bool start_hv_timer(struct kvm_lapic *apic) · 10 unmodified lines
1831	1832	if (!ktimer->tscdeadline)
1832	1833	return false;
1833	1834
1834		− if (kvm_x86_ops.set_hv_timer(vcpu, ktimer->tscdeadline, &expired))
	1835	+ if (static_call(kvm_x86_set_hv_timer)(vcpu, ktimer->tscdeadline, &expired))
1835	1836	return false;
1836	1837
1837	1838	ktimer->hv_timer_in_use = true;
@@ -2175,10 +2176,10 @@ void kvm_free_lapic(struct kvm_vcpu *vcpu) · 337 unmodified lines
2175	2176	hrtimer_cancel(&apic->lapic_timer.timer);
2176	2177
2177	2178	if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
2178		− static_key_slow_dec_deferred(&apic_hw_disabled);
	2179	+ static_branch_slow_dec_deferred(&apic_hw_disabled);
2179	2180
2180	2181	if (!apic->sw_enabled)
2181		− static_key_slow_dec_deferred(&apic_sw_disabled);
	2182	+ static_branch_slow_dec_deferred(&apic_sw_disabled);
2182	2183
2183	2184	if (apic->regs)
2184	2185	free_page((unsigned long)apic->regs);
@@ -2250,9 +2251,9 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) · 65 unmodified lines
2250	2251	if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
2251	2252	if (value & MSR_IA32_APICBASE_ENABLE) {
2252	2253	kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2253		− static_key_slow_dec_deferred(&apic_hw_disabled);
	2254	+ static_branch_slow_dec_deferred(&apic_hw_disabled);
2254	2255	} else {
2255		− static_key_slow_inc(&apic_hw_disabled.key);
	2256	+ static_branch_inc(&apic_hw_disabled.key);
2256	2257	atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
2257	2258	}
2258	2259	}
@@ -2261,7 +2262,7 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) · 2 unmodified lines
2261	2262	kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
2262	2263
2263	2264	if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE \| X2APIC_ENABLE))
2264		− kvm_x86_ops.set_virtual_apic_mode(vcpu);
	2265	+ static_call(kvm_x86_set_virtual_apic_mode)(vcpu);
2265	2266
2266	2267	apic->base_address = apic->vcpu->arch.apic_base &
2267	2268	MSR_IA32_APICBASE_BASE;
@@ -2338,9 +2339,9 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event) · 70 unmodified lines
2338	2339	vcpu->arch.pv_eoi.msr_val = 0;
2339	2340	apic_update_ppr(apic);
2340	2341	if (vcpu->arch.apicv_active) {
2341		− kvm_x86_ops.apicv_post_state_restore(vcpu);
2342		− kvm_x86_ops.hwapic_irr_update(vcpu, -1);
2343		− kvm_x86_ops.hwapic_isr_update(vcpu, -1);
	2342	+ static_call(kvm_x86_apicv_post_state_restore)(vcpu);
	2343	+ static_call(kvm_x86_hwapic_irr_update)(vcpu, -1);
	2344	+ static_call(kvm_x86_hwapic_isr_update)(vcpu, -1);
2344	2345	}
2345	2346
2346	2347	vcpu->arch.apic_arb_prio = 0;
@@ -2449,7 +2450,7 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns) · 102 unmodified lines
2449	2450	* thinking that APIC state has changed.
2450	2451	*/
2451	2452	vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
2452		− static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
	2453	+ static_branch_inc(&apic_sw_disabled.key); /* sw disabled at reset */
2453	2454	kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
2454	2455
2455	2456	return 0;
@@ -2512,7 +2513,7 @@ int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) · 56 unmodified lines
2512	2513	*/
2513	2514
2514	2515	apic_clear_irr(vector, apic);
2515		− if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) {
	2516	+ if (to_hv_vcpu(vcpu) && test_bit(vector, to_hv_synic(vcpu)->auto_eoi_bitmap)) {
2516	2517	/*
2517	2518	* For auto-EOI interrupts, there might be another pending
2518	2519	* interrupt above PPR, so check whether to raise another
@@ -2601,10 +2602,10 @@ int kvm_apic_set_state(struct kvm_vcpu vcpu, struct kvm_lapic_state s) · 82 unmodified lines
2601	2602	kvm_apic_update_apicv(vcpu);
2602	2603	apic->highest_isr_cache = -1;
2603	2604	if (vcpu->arch.apicv_active) {
2604		− kvm_x86_ops.apicv_post_state_restore(vcpu);
2605		− kvm_x86_ops.hwapic_irr_update(vcpu,
	2605	+ static_call(kvm_x86_apicv_post_state_restore)(vcpu);
	2606	+ static_call(kvm_x86_hwapic_irr_update)(vcpu,
2606	2607	apic_find_highest_irr(apic));
2607		− kvm_x86_ops.hwapic_isr_update(vcpu,
	2608	+ static_call(kvm_x86_hwapic_isr_update)(vcpu,
2608	2609	apic_find_highest_isr(apic));
2609	2610	}
2610	2611	kvm_make_request(KVM_REQ_EVENT, vcpu);
@@ -2904,13 +2905,6 @@ void kvm_apic_accept_events(struct kvm_vcpu *vcpu) · 293 unmodified lines
2904	2905	}
2905	2906	}
2906	2907
2907		−void kvm_lapic_init(void)
2908		−{
2909		− /* do not patch jump label more than once per second */
2910		− jump_label_rate_limit(&apic_hw_disabled, HZ);
2911		− jump_label_rate_limit(&apic_sw_disabled, HZ);
2912		−}
2913		−
2914	2908	void kvm_lapic_exit(void)
2915	2909	{
2916	2910	static_key_deferred_flush(&apic_hw_disabled);

arch/x86/kvm/lapic.h+8−12

@@ -6,6 +6,8 @@ · 5 unmodified lines
6	6
7	7	#include <linux/kvm_host.h>
8	8
	9	+#include "hyperv.h"
	10	+
9	11	#define KVM_APIC_INIT 0
10	12	#define KVM_APIC_SIPI 1
11	13	#define KVM_APIC_LVT_NUM 6
@@ -125,13 +127,7 @@ int kvm_x2apic_msr_read(struct kvm_vcpu vcpu, u32 msr, u64 data); · 113 unmodified lines
125	127	int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data);
126	128	int kvm_hv_vapic_msr_read(struct kvm_vcpu vcpu, u32 msr, u64 data);
127	129
128		−static inline bool kvm_hv_vapic_assist_page_enabled(struct kvm_vcpu *vcpu)
129		−{
130		− return vcpu->arch.hyperv.hv_vapic & HV_X64_MSR_VP_ASSIST_PAGE_ENABLE;
131		−}
132		−
133	130	int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len);
134		−void kvm_lapic_init(void);
135	131	void kvm_lapic_exit(void);
136	132
137	133	#define VEC_POS(v) ((v) & (32 - 1))
@@ -172,29 +168,29 @@ static inline void kvm_lapic_set_reg(struct kvm_lapic *apic, int reg_off, u32 va · 34 unmodified lines
172	168	__kvm_lapic_set_reg(apic->regs, reg_off, val);
173	169	}
174	170
175		−extern struct static_key kvm_no_apic_vcpu;
	171	+DECLARE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu);
176	172
177	173	static inline bool lapic_in_kernel(struct kvm_vcpu *vcpu)
178	174	{
179		− if (static_key_false(&kvm_no_apic_vcpu))
	175	+ if (static_branch_unlikely(&kvm_has_noapic_vcpu))
180	176	return vcpu->arch.apic;
181	177	return true;
182	178	}
183	179
184		−extern struct static_key_deferred apic_hw_disabled;
	180	+extern struct static_key_false_deferred apic_hw_disabled;
185	181
186	182	static inline int kvm_apic_hw_enabled(struct kvm_lapic *apic)
187	183	{
188		− if (static_key_false(&apic_hw_disabled.key))
	184	+ if (static_branch_unlikely(&apic_hw_disabled.key))
189	185	return apic->vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE;
190	186	return MSR_IA32_APICBASE_ENABLE;
191	187	}
192	188
193		−extern struct static_key_deferred apic_sw_disabled;
	189	+extern struct static_key_false_deferred apic_sw_disabled;
194	190
195	191	static inline bool kvm_apic_sw_enabled(struct kvm_lapic *apic)
196	192	{
197		− if (static_key_false(&apic_sw_disabled.key))
	193	+ if (static_branch_unlikely(&apic_sw_disabled.key))
198	194	return apic->sw_enabled;
199	195	return true;
200	196	}

arch/x86/kvm/mmu.h+4−4

@@ -102,7 +102,7 @@ static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu) · 101 unmodified lines
102	102	if (!VALID_PAGE(root_hpa))
103	103	return;
104	104
105		− kvm_x86_ops.load_mmu_pgd(vcpu, root_hpa \| kvm_get_active_pcid(vcpu),
	105	+ static_call(kvm_x86_load_mmu_pgd)(vcpu, root_hpa \| kvm_get_active_pcid(vcpu),
106	106	vcpu->arch.mmu->shadow_root_level);
107	107	}
108	108
@@ -152,7 +152,7 @@ static inline int kvm_mmu_do_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, · 43 unmodified lines
152	152	*
153	153	* TODO: introduce APIs to split these two cases.
154	154	*/
155		−static inline int is_writable_pte(unsigned long pte)
	155	+static inline bool is_writable_pte(unsigned long pte)
156	156	{
157	157	return pte & PT_WRITABLE_MASK;
158	158	}
@@ -174,8 +174,8 @@ static inline u8 permission_fault(struct kvm_vcpu vcpu, struct kvm_mmu mmu, · 15 unmodified lines
174	174	unsigned pte_access, unsigned pte_pkey,
175	175	unsigned pfec)
176	176	{
177		− int cpl = kvm_x86_ops.get_cpl(vcpu);
178		− unsigned long rflags = kvm_x86_ops.get_rflags(vcpu);
	177	+ int cpl = static_call(kvm_x86_get_cpl)(vcpu);
	178	+ unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
179	179
180	180	/*
181	181	* If CPL < 3, SMAP prevention are disabled if EFLAGS.AC = 1.

arch/x86/kvm/mmu/mmu.c+159−194

@@ -190,7 +190,7 @@ static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm, · 189 unmodified lines
190	190	int ret = -ENOTSUPP;
191	191
192	192	if (range && kvm_x86_ops.tlb_remote_flush_with_range)
193		− ret = kvm_x86_ops.tlb_remote_flush_with_range(kvm, range);
	193	+ ret = static_call(kvm_x86_tlb_remote_flush_with_range)(kvm, range);
194	194
195	195	if (ret)
196	196	kvm_flush_remote_tlbs(kvm);
@@ -844,17 +844,17 @@ static int pte_list_add(struct kvm_vcpu vcpu, u64 spte, · 647 unmodified lines
844	844	int i, count = 0;
845	845
846	846	if (!rmap_head->val) {
847		− rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);
	847	+ rmap_printk("%p %llx 0->1\n", spte, *spte);
848	848	rmap_head->val = (unsigned long)spte;
849	849	} else if (!(rmap_head->val & 1)) {
850		− rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);
	850	+ rmap_printk("%p %llx 1->many\n", spte, *spte);
851	851	desc = mmu_alloc_pte_list_desc(vcpu);
852	852	desc->sptes[0] = (u64 *)rmap_head->val;
853	853	desc->sptes[1] = spte;
854	854	rmap_head->val = (unsigned long)desc \| 1;
855	855	++count;
856	856	} else {
857		− rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);
	857	+ rmap_printk("%p %llx many->many\n", spte, *spte);
858	858	desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
859	859	while (desc->sptes[PTE_LIST_EXT-1]) {
860	860	count += PTE_LIST_EXT;
@@ -906,14 +906,14 @@ static void __pte_list_remove(u64 spte, struct kvm_rmap_head rmap_head) · 45 unmodified lines
906	906	pr_err("%s: %p 0->BUG\n", __func__, spte);
907	907	BUG();
908	908	} else if (!(rmap_head->val & 1)) {
909		− rmap_printk("%s: %p 1->0\n", __func__, spte);
	909	+ rmap_printk("%p 1->0\n", spte);
910	910	if ((u64 *)rmap_head->val != spte) {
911	911	pr_err("%s: %p 1->BUG\n", __func__, spte);
912	912	BUG();
913	913	}
914	914	rmap_head->val = 0;
915	915	} else {
916		− rmap_printk("%s: %p many->many\n", __func__, spte);
	916	+ rmap_printk("%p many->many\n", spte);
917	917	desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
918	918	prev_desc = NULL;
919	919	while (desc) {
@@ -1115,7 +1115,7 @@ static bool spte_write_protect(u64 *sptep, bool pt_protect) · 195 unmodified lines
1115	1115	!(pt_protect && spte_can_locklessly_be_made_writable(spte)))
1116	1116	return false;
1117	1117
1118		− rmap_printk("rmap_write_protect: spte %p %llx\n", sptep, *sptep);
	1118	+ rmap_printk("spte %p %llx\n", sptep, *sptep);
1119	1119
1120	1120	if (pt_protect)
1121	1121	spte &= ~SPTE_MMU_WRITEABLE;
@@ -1142,7 +1142,7 @@ static bool spte_clear_dirty(u64 *sptep) · 20 unmodified lines
1142	1142	{
1143	1143	u64 spte = *sptep;
1144	1144
1145		− rmap_printk("rmap_clear_dirty: spte %p %llx\n", sptep, *sptep);
	1145	+ rmap_printk("spte %p %llx\n", sptep, *sptep);
1146	1146
1147	1147	MMU_WARN_ON(!spte_ad_enabled(spte));
1148	1148	spte &= ~shadow_dirty_mask;
@@ -1184,7 +1184,7 @@ static bool spte_set_dirty(u64 *sptep) · 35 unmodified lines
1184	1184	{
1185	1185	u64 spte = *sptep;
1186	1186
1187		− rmap_printk("rmap_set_dirty: spte %p %llx\n", sptep, *sptep);
	1187	+ rmap_printk("spte %p %llx\n", sptep, *sptep);
1188	1188
1189	1189	/*
1190	1190	* Similar to the !kvm_x86_ops.slot_disable_log_dirty case,
@@ -1225,7 +1225,7 @@ static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, · 34 unmodified lines
1225	1225	{
1226	1226	struct kvm_rmap_head *rmap_head;
1227	1227
1228		− if (kvm->arch.tdp_mmu_enabled)
	1228	+ if (is_tdp_mmu_enabled(kvm))
1229	1229	kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,
1230	1230	slot->base_gfn + gfn_offset, mask, true);
1231	1231	while (mask) {
@@ -1254,7 +1254,7 @@ void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, · 22 unmodified lines
1254	1254	{
1255	1255	struct kvm_rmap_head *rmap_head;
1256	1256
1257		− if (kvm->arch.tdp_mmu_enabled)
	1257	+ if (is_tdp_mmu_enabled(kvm))
1258	1258	kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,
1259	1259	slot->base_gfn + gfn_offset, mask, false);
1260	1260	while (mask) {
@@ -1283,8 +1283,9 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, · 22 unmodified lines
1283	1283	gfn_t gfn_offset, unsigned long mask)
1284	1284	{
1285	1285	if (kvm_x86_ops.enable_log_dirty_pt_masked)
1286		− kvm_x86_ops.enable_log_dirty_pt_masked(kvm, slot, gfn_offset,
1287		− mask);
	1286	+ static_call(kvm_x86_enable_log_dirty_pt_masked)(kvm, slot,
	1287	+ gfn_offset,
	1288	+ mask);
1288	1289	else
1289	1290	kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
1290	1291	}
@@ -1292,7 +1293,7 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, · 1 unmodified line
1292	1293	int kvm_cpu_dirty_log_size(void)
1293	1294	{
1294	1295	if (kvm_x86_ops.cpu_dirty_log_size)
1295		− return kvm_x86_ops.cpu_dirty_log_size();
	1296	+ return static_call(kvm_x86_cpu_dirty_log_size)();
1296	1297
1297	1298	return 0;
1298	1299	}
@@ -1309,7 +1310,7 @@ bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm, · 10 unmodified lines
1309	1310	write_protected \|= __rmap_write_protect(kvm, rmap_head, true);
1310	1311	}
1311	1312
1312		− if (kvm->arch.tdp_mmu_enabled)
	1313	+ if (is_tdp_mmu_enabled(kvm))
1313	1314	write_protected \|=
1314	1315	kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn);
1315	1316
@@ -1331,7 +1332,7 @@ static bool kvm_zap_rmapp(struct kvm kvm, struct kvm_rmap_head rmap_head) · 15 unmodified lines
1331	1332	bool flush = false;
1332	1333
1333	1334	while ((sptep = rmap_get_first(rmap_head, &iter))) {
1334		− rmap_printk("%s: spte %p %llx.\n", __func__, sptep, *sptep);
	1335	+ rmap_printk("spte %p %llx.\n", sptep, *sptep);
1335	1336
1336	1337	pte_list_remove(rmap_head, sptep);
1337	1338	flush = true;
@@ -1363,7 +1364,7 @@ static int kvm_set_pte_rmapp(struct kvm kvm, struct kvm_rmap_head rmap_head, · 25 unmodified lines
1363	1364
1364	1365	restart:
1365	1366	for_each_rmap_spte(rmap_head, &iter, sptep) {
1366		− rmap_printk("kvm_set_pte_rmapp: spte %p %llx gfn %llx (%d)\n",
	1367	+ rmap_printk("spte %p %llx gfn %llx (%d)\n",
1367	1368	sptep, *sptep, gfn, level);
1368	1369
1369	1370	need_flush = 1;
@@ -1456,16 +1457,17 @@ static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator) · 86 unmodified lines
1456	1457	slot_rmap_walk_okay(_iter_); \
1457	1458	slot_rmap_walk_next(_iter_))
1458	1459
1459		−static int kvm_handle_hva_range(struct kvm *kvm,
1460		− unsigned long start,
1461		− unsigned long end,
1462		− unsigned long data,
1463		− int (handler)(struct kvm kvm,
1464		− struct kvm_rmap_head *rmap_head,
1465		− struct kvm_memory_slot *slot,
1466		− gfn_t gfn,
1467		− int level,
1468		− unsigned long data))
	1460	+static __always_inline int
	1461	+kvm_handle_hva_range(struct kvm *kvm,
	1462	+ unsigned long start,
	1463	+ unsigned long end,
	1464	+ unsigned long data,
	1465	+ int (handler)(struct kvm kvm,
	1466	+ struct kvm_rmap_head *rmap_head,
	1467	+ struct kvm_memory_slot *slot,
	1468	+ gfn_t gfn,
	1469	+ int level,
	1470	+ unsigned long data))
1469	1471	{
1470	1472	struct kvm_memslots *slots;
1471	1473	struct kvm_memory_slot *memslot;
@@ -1521,7 +1523,7 @@ int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end, · 49 unmodified lines
1521	1523
1522	1524	r = kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
1523	1525
1524		− if (kvm->arch.tdp_mmu_enabled)
	1526	+ if (is_tdp_mmu_enabled(kvm))
1525	1527	r \|= kvm_tdp_mmu_zap_hva_range(kvm, start, end);
1526	1528
1527	1529	return r;
@@ -1533,7 +1535,7 @@ int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) · 5 unmodified lines
1533	1535
1534	1536	r = kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
1535	1537
1536		− if (kvm->arch.tdp_mmu_enabled)
	1538	+ if (is_tdp_mmu_enabled(kvm))
1537	1539	r \|= kvm_tdp_mmu_set_spte_hva(kvm, hva, &pte);
1538	1540
1539	1541	return r;
@@ -1588,7 +1590,7 @@ int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end) · 48 unmodified lines
1588	1590	int young = false;
1589	1591
1590	1592	young = kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp);
1591		− if (kvm->arch.tdp_mmu_enabled)
	1593	+ if (is_tdp_mmu_enabled(kvm))
1592	1594	young \|= kvm_tdp_mmu_age_hva_range(kvm, start, end);
1593	1595
1594	1596	return young;
@@ -1599,7 +1601,7 @@ int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) · 4 unmodified lines
1599	1601	int young = false;
1600	1602
1601	1603	young = kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
1602		− if (kvm->arch.tdp_mmu_enabled)
	1604	+ if (is_tdp_mmu_enabled(kvm))
1603	1605	young \|= kvm_tdp_mmu_test_age_hva(kvm, hva);
1604	1606
1605	1607	return young;
@@ -1723,13 +1725,6 @@ static int nonpaging_sync_page(struct kvm_vcpu *vcpu, · 117 unmodified lines
1723	1725	return 0;
1724	1726	}
1725	1727
1726		−static void nonpaging_update_pte(struct kvm_vcpu *vcpu,
1727		− struct kvm_mmu_page sp, u64 spte,
1728		− const void *pte)
1729		−{
1730		− WARN_ON(1);
1731		−}
1732		−
1733	1728	#define KVM_PAGE_ARRAY_NR 16
1734	1729
1735	1730	struct kvm_mmu_pages {
@@ -2016,9 +2011,9 @@ static void mmu_sync_children(struct kvm_vcpu *vcpu, · 280 unmodified lines
2016	2011	flush \|= kvm_sync_page(vcpu, sp, &invalid_list);
2017	2012	mmu_pages_clear_parents(&parents);
2018	2013	}
2019		− if (need_resched() \|\| spin_needbreak(&vcpu->kvm->mmu_lock)) {
	2014	+ if (need_resched() \|\| rwlock_needbreak(&vcpu->kvm->mmu_lock)) {
2020	2015	kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush);
2021		− cond_resched_lock(&vcpu->kvm->mmu_lock);
	2016	+ cond_resched_rwlock_write(&vcpu->kvm->mmu_lock);
2022	2017	flush = false;
2023	2018	}
2024	2019	}
@@ -2417,7 +2412,7 @@ static unsigned long kvm_mmu_zap_oldest_mmu_pages(struct kvm *kvm, · 392 unmodified lines
2417	2412	return 0;
2418	2413
2419	2414	restart:
2420		− list_for_each_entry_safe(sp, tmp, &kvm->arch.active_mmu_pages, link) {
	2415	+ list_for_each_entry_safe_reverse(sp, tmp, &kvm->arch.active_mmu_pages, link) {
2421	2416	/*
2422	2417	* Don't zap active root pages, the page itself can't be freed
2423	2418	* and zapping it will just force vCPUs to realloc and reload.
@@ -2470,7 +2465,7 @@ static int make_mmu_pages_available(struct kvm_vcpu *vcpu) · 46 unmodified lines
2470	2465	*/
2471	2466	void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages)
2472	2467	{
2473		− spin_lock(&kvm->mmu_lock);
	2468	+ write_lock(&kvm->mmu_lock);
2474	2469
2475	2470	if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
2476	2471	kvm_mmu_zap_oldest_mmu_pages(kvm, kvm->arch.n_used_mmu_pages -
@@ -2481,7 +2476,7 @@ void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages) · 4 unmodified lines
2481	2476
2482	2477	kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
2483	2478
2484		− spin_unlock(&kvm->mmu_lock);
	2479	+ write_unlock(&kvm->mmu_lock);
2485	2480	}
2486	2481
2487	2482	int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
@@ -2492,7 +2487,7 @@ int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) · 4 unmodified lines
2492	2487
2493	2488	pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
2494	2489	r = 0;
2495		− spin_lock(&kvm->mmu_lock);
	2490	+ write_lock(&kvm->mmu_lock);
2496	2491	for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {
2497	2492	pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
2498	2493	sp->role.word);
@@ -2500,7 +2495,7 @@ int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) · 1 unmodified line
2500	2495	kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
2501	2496	}
2502	2497	kvm_mmu_commit_zap_page(kvm, &invalid_list);
2503		− spin_unlock(&kvm->mmu_lock);
	2498	+ write_unlock(&kvm->mmu_lock);
2504	2499
2505	2500	return r;
2506	2501	}
@@ -3161,7 +3156,7 @@ static void mmu_free_root_page(struct kvm kvm, hpa_t root_hpa, · 654 unmodified lines
3161	3156	sp = to_shadow_page(*root_hpa & PT64_BASE_ADDR_MASK);
3162	3157
3163	3158	if (kvm_mmu_put_root(kvm, sp)) {
3164		− if (sp->tdp_mmu_page)
	3159	+ if (is_tdp_mmu_page(sp))
3165	3160	kvm_tdp_mmu_free_root(kvm, sp);
3166	3161	else if (sp->role.invalid)
3167	3162	kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
@@ -3192,7 +3187,7 @@ void kvm_mmu_free_roots(struct kvm_vcpu vcpu, struct kvm_mmu mmu, · 24 unmodified lines
3192	3187	return;
3193	3188	}
3194	3189
3195		− spin_lock(&kvm->mmu_lock);
	3190	+ write_lock(&kvm->mmu_lock);
3196	3191
3197	3192	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
3198	3193	if (roots_to_free & KVM_MMU_ROOT_PREVIOUS(i))
@@ -3215,7 +3210,7 @@ void kvm_mmu_free_roots(struct kvm_vcpu vcpu, struct kvm_mmu mmu, · 16 unmodified lines
3215	3210	}
3216	3211
3217	3212	kvm_mmu_commit_zap_page(kvm, &invalid_list);
3218		− spin_unlock(&kvm->mmu_lock);
	3213	+ write_unlock(&kvm->mmu_lock);
3219	3214	}
3220	3215	EXPORT_SYMBOL_GPL(kvm_mmu_free_roots);
3221	3216
@@ -3236,16 +3231,16 @@ static hpa_t mmu_alloc_root(struct kvm_vcpu *vcpu, gfn_t gfn, gva_t gva, · 14 unmodified lines
3236	3231	{
3237	3232	struct kvm_mmu_page *sp;
3238	3233
3239		− spin_lock(&vcpu->kvm->mmu_lock);
	3234	+ write_lock(&vcpu->kvm->mmu_lock);
3240	3235
3241	3236	if (make_mmu_pages_available(vcpu)) {
3242		− spin_unlock(&vcpu->kvm->mmu_lock);
	3237	+ write_unlock(&vcpu->kvm->mmu_lock);
3243	3238	return INVALID_PAGE;
3244	3239	}
3245	3240	sp = kvm_mmu_get_page(vcpu, gfn, gva, level, direct, ACC_ALL);
3246	3241	++sp->root_count;
3247	3242
3248		− spin_unlock(&vcpu->kvm->mmu_lock);
	3243	+ write_unlock(&vcpu->kvm->mmu_lock);
3249	3244	return __pa(sp->spt);
3250	3245	}
3251	3246
@@ -3255,7 +3250,7 @@ static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu) · 3 unmodified lines
3255	3250	hpa_t root;
3256	3251	unsigned i;
3257	3252
3258		− if (vcpu->kvm->arch.tdp_mmu_enabled) {
	3253	+ if (is_tdp_mmu_enabled(vcpu->kvm)) {
3259	3254	root = kvm_tdp_mmu_get_vcpu_root_hpa(vcpu);
3260	3255
3261	3256	if (!VALID_PAGE(root))
@@ -3416,17 +3411,17 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) · 154 unmodified lines
3416	3411	!smp_load_acquire(&sp->unsync_children))
3417	3412	return;
3418	3413
3419		− spin_lock(&vcpu->kvm->mmu_lock);
	3414	+ write_lock(&vcpu->kvm->mmu_lock);
3420	3415	kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
3421	3416
3422	3417	mmu_sync_children(vcpu, sp);
3423	3418
3424	3419	kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3425		− spin_unlock(&vcpu->kvm->mmu_lock);
	3420	+ write_unlock(&vcpu->kvm->mmu_lock);
3426	3421	return;
3427	3422	}
3428	3423
3429		− spin_lock(&vcpu->kvm->mmu_lock);
	3424	+ write_lock(&vcpu->kvm->mmu_lock);
3430	3425	kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);
3431	3426
3432	3427	for (i = 0; i < 4; ++i) {
@@ -3440,7 +3435,7 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) · 7 unmodified lines
3440	3435	}
3441	3436
3442	3437	kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3443		− spin_unlock(&vcpu->kvm->mmu_lock);
	3438	+ write_unlock(&vcpu->kvm->mmu_lock);
3444	3439	}
3445	3440	EXPORT_SYMBOL_GPL(kvm_mmu_sync_roots);
3446	3441
@@ -3724,7 +3719,12 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, · 277 unmodified lines
3724	3719	return r;
3725	3720
3726	3721	r = RET_PF_RETRY;
3727		− spin_lock(&vcpu->kvm->mmu_lock);
	3722	+
	3723	+ if (is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa))
	3724	+ read_lock(&vcpu->kvm->mmu_lock);
	3725	+ else
	3726	+ write_lock(&vcpu->kvm->mmu_lock);
	3727	+
3728	3728	if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
3729	3729	goto out_unlock;
3730	3730	r = make_mmu_pages_available(vcpu);
@@ -3739,7 +3739,10 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, · 8 unmodified lines
3739	3739	prefault, is_tdp);
3740	3740
3741	3741	out_unlock:
3742		− spin_unlock(&vcpu->kvm->mmu_lock);
	3742	+ if (is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa))
	3743	+ read_unlock(&vcpu->kvm->mmu_lock);
	3744	+ else
	3745	+ write_unlock(&vcpu->kvm->mmu_lock);
3743	3746	kvm_release_pfn_clean(pfn);
3744	3747	return r;
3745	3748	}
@@ -3813,7 +3816,6 @@ static void nonpaging_init_context(struct kvm_vcpu *vcpu, · 67 unmodified lines
3813	3816	context->gva_to_gpa = nonpaging_gva_to_gpa;
3814	3817	context->sync_page = nonpaging_sync_page;
3815	3818	context->invlpg = NULL;
3816		− context->update_pte = nonpaging_update_pte;
3817	3819	context->root_level = 0;
3818	3820	context->shadow_root_level = PT32E_ROOT_LEVEL;
3819	3821	context->direct_map = true;
@@ -3984,20 +3986,27 @@ static inline bool is_last_gpte(struct kvm_mmu *mmu, · 164 unmodified lines
3984	3986	static void
3985	3987	__reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
3986	3988	struct rsvd_bits_validate *rsvd_check,
3987		− int maxphyaddr, int level, bool nx, bool gbpages,
	3989	+ u64 pa_bits_rsvd, int level, bool nx, bool gbpages,
3988	3990	bool pse, bool amd)
3989	3991	{
3990		− u64 exb_bit_rsvd = 0;
3991	3992	u64 gbpages_bit_rsvd = 0;
3992	3993	u64 nonleaf_bit8_rsvd = 0;
	3994	+ u64 high_bits_rsvd;
3993	3995
3994	3996	rsvd_check->bad_mt_xwr = 0;
3995	3997
3996		− if (!nx)
3997		− exb_bit_rsvd = rsvd_bits(63, 63);
3998	3998	if (!gbpages)
3999	3999	gbpages_bit_rsvd = rsvd_bits(7, 7);
4000	4000
	4001	+ if (level == PT32E_ROOT_LEVEL)
	4002	+ high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 62);
	4003	+ else
	4004	+ high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 51);
	4005	+
	4006	+ /* Note, NX doesn't exist in PDPTEs, this is handled below. */
	4007	+ if (!nx)
	4008	+ high_bits_rsvd \|= rsvd_bits(63, 63);
	4009	+
4001	4010	/*
4002	4011	* Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for
4003	4012	* leaf entries) on AMD CPUs only.
@@ -4026,45 +4035,39 @@ __reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, · 22 unmodified lines
4026	4035	rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
4027	4036	break;
4028	4037	case PT32E_ROOT_LEVEL:
4029		− rsvd_check->rsvd_bits_mask[0][2] =
4030		− rsvd_bits(maxphyaddr, 63) \|
4031		− rsvd_bits(5, 8) \| rsvd_bits(1, 2); /* PDPTE */
4032		− rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd \|
4033		− rsvd_bits(maxphyaddr, 62); /* PDE */
4034		− rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd \|
4035		− rsvd_bits(maxphyaddr, 62); /* PTE */
4036		− rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd \|
4037		− rsvd_bits(maxphyaddr, 62) \|
4038		− rsvd_bits(13, 20); /* large page */
	4038	+ rsvd_check->rsvd_bits_mask[0][2] = rsvd_bits(63, 63) \|
	4039	+ high_bits_rsvd \|
	4040	+ rsvd_bits(5, 8) \|
	4041	+ rsvd_bits(1, 2); /* PDPTE */
	4042	+ rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd; /* PDE */
	4043	+ rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd; /* PTE */
	4044	+ rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd \|
	4045	+ rsvd_bits(13, 20); /* large page */
4039	4046	rsvd_check->rsvd_bits_mask[1][0] =
4040	4047	rsvd_check->rsvd_bits_mask[0][0];
4041	4048	break;
4042	4049	case PT64_ROOT_5LEVEL:
4043		− rsvd_check->rsvd_bits_mask[0][4] = exb_bit_rsvd \|
4044		− nonleaf_bit8_rsvd \| rsvd_bits(7, 7) \|
4045		− rsvd_bits(maxphyaddr, 51);
	4050	+ rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd \|
	4051	+ nonleaf_bit8_rsvd \|
	4052	+ rsvd_bits(7, 7);
4046	4053	rsvd_check->rsvd_bits_mask[1][4] =
4047	4054	rsvd_check->rsvd_bits_mask[0][4];
4048	4055	fallthrough;
4049	4056	case PT64_ROOT_4LEVEL:
4050		− rsvd_check->rsvd_bits_mask[0][3] = exb_bit_rsvd \|
4051		− nonleaf_bit8_rsvd \| rsvd_bits(7, 7) \|
4052		− rsvd_bits(maxphyaddr, 51);
4053		− rsvd_check->rsvd_bits_mask[0][2] = exb_bit_rsvd \|
4054		− gbpages_bit_rsvd \|
4055		− rsvd_bits(maxphyaddr, 51);
4056		− rsvd_check->rsvd_bits_mask[0][1] = exb_bit_rsvd \|
4057		− rsvd_bits(maxphyaddr, 51);
4058		− rsvd_check->rsvd_bits_mask[0][0] = exb_bit_rsvd \|
4059		− rsvd_bits(maxphyaddr, 51);
	4057	+ rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd \|
	4058	+ nonleaf_bit8_rsvd \|
	4059	+ rsvd_bits(7, 7);
	4060	+ rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd \|
	4061	+ gbpages_bit_rsvd;
	4062	+ rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;
	4063	+ rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
4060	4064	rsvd_check->rsvd_bits_mask[1][3] =
4061	4065	rsvd_check->rsvd_bits_mask[0][3];
4062		− rsvd_check->rsvd_bits_mask[1][2] = exb_bit_rsvd \|
4063		− gbpages_bit_rsvd \| rsvd_bits(maxphyaddr, 51) \|
4064		− rsvd_bits(13, 29);
4065		− rsvd_check->rsvd_bits_mask[1][1] = exb_bit_rsvd \|
4066		− rsvd_bits(maxphyaddr, 51) \|
4067		− rsvd_bits(13, 20); /* large page */
	4066	+ rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd \|
	4067	+ gbpages_bit_rsvd \|
	4068	+ rsvd_bits(13, 29);
	4069	+ rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd \|
	4070	+ rsvd_bits(13, 20); /* large page */
4068	4071	rsvd_check->rsvd_bits_mask[1][0] =
4069	4072	rsvd_check->rsvd_bits_mask[0][0];
4070	4073	break;
@@ -4075,8 +4078,8 @@ static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, · 4 unmodified lines
4075	4078	struct kvm_mmu *context)
4076	4079	{
4077	4080	__reset_rsvds_bits_mask(vcpu, &context->guest_rsvd_check,
4078		− cpuid_maxphyaddr(vcpu), context->root_level,
4079		− context->nx,
	4081	+ vcpu->arch.reserved_gpa_bits,
	4082	+ context->root_level, context->nx,
4080	4083	guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES),
4081	4084	is_pse(vcpu),
4082	4085	guest_cpuid_is_amd_or_hygon(vcpu));
@@ -4084,27 +4087,22 @@ static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, · 1 unmodified line
4084	4087
4085	4088	static void
4086	4089	__reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
4087		− int maxphyaddr, bool execonly)
	4090	+ u64 pa_bits_rsvd, bool execonly)
4088	4091	{
	4092	+ u64 high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 51);
4089	4093	u64 bad_mt_xwr;
4090	4094
4091		− rsvd_check->rsvd_bits_mask[0][4] =
4092		− rsvd_bits(maxphyaddr, 51) \| rsvd_bits(3, 7);
4093		− rsvd_check->rsvd_bits_mask[0][3] =
4094		− rsvd_bits(maxphyaddr, 51) \| rsvd_bits(3, 7);
4095		− rsvd_check->rsvd_bits_mask[0][2] =
4096		− rsvd_bits(maxphyaddr, 51) \| rsvd_bits(3, 6);
4097		− rsvd_check->rsvd_bits_mask[0][1] =
4098		− rsvd_bits(maxphyaddr, 51) \| rsvd_bits(3, 6);
4099		− rsvd_check->rsvd_bits_mask[0][0] = rsvd_bits(maxphyaddr, 51);
	4095	+ rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd \| rsvd_bits(3, 7);
	4096	+ rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd \| rsvd_bits(3, 7);
	4097	+ rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd \| rsvd_bits(3, 6);
	4098	+ rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd \| rsvd_bits(3, 6);
	4099	+ rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
4100	4100
4101	4101	/* large page */
4102	4102	rsvd_check->rsvd_bits_mask[1][4] = rsvd_check->rsvd_bits_mask[0][4];
4103	4103	rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];
4104		− rsvd_check->rsvd_bits_mask[1][2] =
4105		− rsvd_bits(maxphyaddr, 51) \| rsvd_bits(12, 29);
4106		− rsvd_check->rsvd_bits_mask[1][1] =
4107		− rsvd_bits(maxphyaddr, 51) \| rsvd_bits(12, 20);
	4104	+ rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd \| rsvd_bits(12, 29);
	4105	+ rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd \| rsvd_bits(12, 20);
4108	4106	rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];
4109	4107
4110	4108	bad_mt_xwr = 0xFFull << (2 * 8); /* bits 3..5 must not be 2 */
@@ -4123,7 +4121,12 @@ static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu, · 12 unmodified lines
4123	4121	struct kvm_mmu *context, bool execonly)
4124	4122	{
4125	4123	__reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,
4126		− cpuid_maxphyaddr(vcpu), execonly);
	4124	+ vcpu->arch.reserved_gpa_bits, execonly);
	4125	+}
	4126	+
	4127	+static inline u64 reserved_hpa_bits(void)
	4128	+{
	4129	+ return rsvd_bits(shadow_phys_bits, 63);
4127	4130	}
4128	4131
4129	4132	/*
@@ -4145,7 +4148,7 @@ reset_shadow_zero_bits_mask(struct kvm_vcpu vcpu, struct kvm_mmu context) · 15 unmodified lines
4145	4148	*/
4146	4149	shadow_zero_check = &context->shadow_zero_check;
4147	4150	__reset_rsvds_bits_mask(vcpu, shadow_zero_check,
4148		− shadow_phys_bits,
	4151	+ reserved_hpa_bits(),
4149	4152	context->shadow_root_level, uses_nx,
4150	4153	guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES),
4151	4154	is_pse(vcpu), true);
@@ -4182,14 +4185,13 @@ reset_tdp_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, · 30 unmodified lines
4182	4185
4183	4186	if (boot_cpu_is_amd())
4184	4187	__reset_rsvds_bits_mask(vcpu, shadow_zero_check,
4185		− shadow_phys_bits,
	4188	+ reserved_hpa_bits(),
4186	4189	context->shadow_root_level, false,
4187	4190	boot_cpu_has(X86_FEATURE_GBPAGES),
4188	4191	true, true);
4189	4192	else
4190	4193	__reset_rsvds_bits_mask_ept(shadow_zero_check,
4191		− shadow_phys_bits,
4192		− false);
	4194	+ reserved_hpa_bits(), false);
4193	4195
4194	4196	if (!shadow_me_mask)
4195	4197	return;
@@ -4209,7 +4211,7 @@ reset_ept_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, · 13 unmodified lines
4209	4211	struct kvm_mmu *context, bool execonly)
4210	4212	{
4211	4213	__reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
4212		− shadow_phys_bits, execonly);
	4214	+ reserved_hpa_bits(), execonly);
4213	4215	}
4214	4216
4215	4217	#define BYTE_MASK(access) \
@@ -4395,7 +4397,6 @@ static void paging64_init_context_common(struct kvm_vcpu *vcpu, · 179 unmodified lines
4395	4397	context->gva_to_gpa = paging64_gva_to_gpa;
4396	4398	context->sync_page = paging64_sync_page;
4397	4399	context->invlpg = paging64_invlpg;
4398		− context->update_pte = paging64_update_pte;
4399	4400	context->shadow_root_level = level;
4400	4401	context->direct_map = false;
4401	4402	}
@@ -4424,7 +4425,6 @@ static void paging32_init_context(struct kvm_vcpu *vcpu, · 22 unmodified lines
4424	4425	context->gva_to_gpa = paging32_gva_to_gpa;
4425	4426	context->sync_page = paging32_sync_page;
4426	4427	context->invlpg = paging32_invlpg;
4427		− context->update_pte = paging32_update_pte;
4428	4428	context->shadow_root_level = PT32E_ROOT_LEVEL;
4429	4429	context->direct_map = false;
4430	4430	}
@@ -4506,7 +4506,6 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) · 75 unmodified lines
4506	4506	context->page_fault = kvm_tdp_page_fault;
4507	4507	context->sync_page = nonpaging_sync_page;
4508	4508	context->invlpg = NULL;
4509		− context->update_pte = nonpaging_update_pte;
4510	4509	context->shadow_root_level = kvm_mmu_get_tdp_level(vcpu);
4511	4510	context->direct_map = true;
4512	4511	context->get_guest_pgd = get_cr3;
@@ -4678,7 +4677,6 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, · 165 unmodified lines
4678	4677	context->gva_to_gpa = ept_gva_to_gpa;
4679	4678	context->sync_page = ept_sync_page;
4680	4679	context->invlpg = ept_invlpg;
4681		− context->update_pte = ept_update_pte;
4682	4680	context->root_level = level;
4683	4681	context->direct_map = false;
4684	4682	context->mmu_role.as_u64 = new_role.as_u64;
@@ -4811,7 +4809,7 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu) · 126 unmodified lines
4811	4809	if (r)
4812	4810	goto out;
4813	4811	kvm_mmu_load_pgd(vcpu);
4814		− kvm_x86_ops.tlb_flush_current(vcpu);
	4812	+ static_call(kvm_x86_tlb_flush_current)(vcpu);
4815	4813	out:
4816	4814	return r;
4817	4815	}
@@ -4826,19 +4824,6 @@ void kvm_mmu_unload(struct kvm_vcpu *vcpu) · 8 unmodified lines
4826	4824	}
4827	4825	EXPORT_SYMBOL_GPL(kvm_mmu_unload);
4828	4826
4829		−static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
4830		− struct kvm_mmu_page sp, u64 spte,
4831		− const void *new)
4832		−{
4833		− if (sp->role.level != PG_LEVEL_4K) {
4834		− ++vcpu->kvm->stat.mmu_pde_zapped;
4835		− return;
4836		− }
4837		−
4838		− ++vcpu->kvm->stat.mmu_pte_updated;
4839		− vcpu->arch.mmu->update_pte(vcpu, sp, spte, new);
4840		−}
4841		−
4842	4827	static bool need_remote_flush(u64 old, u64 new)
4843	4828	{
4844	4829	if (!is_shadow_present_pte(old))
@@ -4954,22 +4939,6 @@ static u64 get_written_sptes(struct kvm_mmu_page sp, gpa_t gpa, int *nspte) · 109 unmodified lines
4954	4939	return spte;
4955	4940	}
4956	4941
4957		−/*
4958		− * Ignore various flags when determining if a SPTE can be immediately
4959		− * overwritten for the current MMU.
4960		− * - level: explicitly checked in mmu_pte_write_new_pte(), and will never
4961		− * match the current MMU role, as MMU's level tracks the root level.
4962		− * - access: updated based on the new guest PTE
4963		− * - quadrant: handled by get_written_sptes()
4964		− * - invalid: always false (loop only walks valid shadow pages)
4965		− */
4966		−static const union kvm_mmu_page_role role_ign = {
4967		− .level = 0xf,
4968		− .access = 0x7,
4969		− .quadrant = 0x3,
4970		− .invalid = 0x1,
4971		−};
4972		−
4973	4942	static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
4974	4943	const u8 *new, int bytes,
4975	4944	struct kvm_page_track_notifier_node *node)
@@ -4999,7 +4968,7 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, · 23 unmodified lines
4999	4968	*/
5000	4969	mmu_topup_memory_caches(vcpu, true);
5001	4970
5002		− spin_lock(&vcpu->kvm->mmu_lock);
	4971	+ write_lock(&vcpu->kvm->mmu_lock);
5003	4972
5004	4973	gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, &bytes);
5005	4974
@@ -5020,14 +4989,10 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, · 14 unmodified lines
5020	4989
5021	4990	local_flush = true;
5022	4991	while (npte--) {
5023		− u32 base_role = vcpu->arch.mmu->mmu_role.base.word;
5024		−
5025	4992	entry = *spte;
5026	4993	mmu_page_zap_pte(vcpu->kvm, sp, spte, NULL);
5027		− if (gentry &&
5028		− !((sp->role.word ^ base_role) & ~role_ign.word) &&
5029		− rmap_can_add(vcpu))
5030		− mmu_pte_write_new_pte(vcpu, sp, spte, &gentry);
	4994	+ if (gentry && sp->role.level != PG_LEVEL_4K)
	4995	+ ++vcpu->kvm->stat.mmu_pde_zapped;
5031	4996	if (need_remote_flush(entry, *spte))
5032	4997	remote_flush = true;
5033	4998	++spte;
@@ -5035,7 +5000,7 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, · 1 unmodified line
5035	5000	}
5036	5001	kvm_mmu_flush_or_zap(vcpu, &invalid_list, remote_flush, local_flush);
5037	5002	kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
5038		− spin_unlock(&vcpu->kvm->mmu_lock);
	5003	+ write_unlock(&vcpu->kvm->mmu_lock);
5039	5004	}
5040	5005
5041	5006	int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
@@ -5125,7 +5090,7 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu vcpu, struct kvm_mmu mmu, · 83 unmodified lines
5125	5090	if (is_noncanonical_address(gva, vcpu))
5126	5091	return;
5127	5092
5128		− kvm_x86_ops.tlb_flush_gva(vcpu, gva);
	5093	+ static_call(kvm_x86_tlb_flush_gva)(vcpu, gva);
5129	5094	}
5130	5095
5131	5096	if (!mmu->invlpg)
@@ -5182,7 +5147,7 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) · 50 unmodified lines
5182	5147	}
5183	5148
5184	5149	if (tlb_flush)
5185		− kvm_x86_ops.tlb_flush_gva(vcpu, gva);
	5150	+ static_call(kvm_x86_tlb_flush_gva)(vcpu, gva);
5186	5151
5187	5152	++vcpu->stat.invlpg;
5188	5153
@@ -5233,14 +5198,14 @@ slot_handle_level_range(struct kvm kvm, struct kvm_memory_slot memslot, · 44 unmodified lines
5233	5198	if (iterator.rmap)
5234	5199	flush \|= fn(kvm, iterator.rmap);
5235	5200
5236		− if (need_resched() \|\| spin_needbreak(&kvm->mmu_lock)) {
	5201	+ if (need_resched() \|\| rwlock_needbreak(&kvm->mmu_lock)) {
5237	5202	if (flush && lock_flush_tlb) {
5238	5203	kvm_flush_remote_tlbs_with_address(kvm,
5239	5204	start_gfn,
5240	5205	iterator.gfn - start_gfn + 1);
5241	5206	flush = false;
5242	5207	}
5243		− cond_resched_lock(&kvm->mmu_lock);
	5208	+ cond_resched_rwlock_write(&kvm->mmu_lock);
5244	5209	}
5245	5210	}
5246	5211
@@ -5390,7 +5355,7 @@ restart: · 143 unmodified lines
5390	5355	* be in active use by the guest.
5391	5356	*/
5392	5357	if (batch >= BATCH_ZAP_PAGES &&
5393		− cond_resched_lock(&kvm->mmu_lock)) {
	5358	+ cond_resched_rwlock_write(&kvm->mmu_lock)) {
5394	5359	batch = 0;
5395	5360	goto restart;
5396	5361	}
@@ -5423,7 +5388,7 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm) · 26 unmodified lines
5423	5388	{
5424	5389	lockdep_assert_held(&kvm->slots_lock);
5425	5390
5426		− spin_lock(&kvm->mmu_lock);
	5391	+ write_lock(&kvm->mmu_lock);
5427	5392	trace_kvm_mmu_zap_all_fast(kvm);
5428	5393
5429	5394	/*
@@ -5447,10 +5412,10 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm) · 17 unmodified lines
5447	5412
5448	5413	kvm_zap_obsolete_pages(kvm);
5449	5414
5450		− if (kvm->arch.tdp_mmu_enabled)
	5415	+ if (is_tdp_mmu_enabled(kvm))
5451	5416	kvm_tdp_mmu_zap_all(kvm);
5452	5417
5453		− spin_unlock(&kvm->mmu_lock);
	5418	+ write_unlock(&kvm->mmu_lock);
5454	5419	}
5455	5420
5456	5421	static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm)
@@ -5492,7 +5457,7 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) · 35 unmodified lines
5492	5457	int i;
5493	5458	bool flush;
5494	5459
5495		− spin_lock(&kvm->mmu_lock);
	5460	+ write_lock(&kvm->mmu_lock);
5496	5461	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
5497	5462	slots = __kvm_memslots(kvm, i);
5498	5463	kvm_for_each_memslot(memslot, slots) {
@@ -5510,13 +5475,13 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) · 11 unmodified lines
5510	5475	}
5511	5476	}
5512	5477
5513		− if (kvm->arch.tdp_mmu_enabled) {
	5478	+ if (is_tdp_mmu_enabled(kvm)) {
5514	5479	flush = kvm_tdp_mmu_zap_gfn_range(kvm, gfn_start, gfn_end);
5515	5480	if (flush)
5516	5481	kvm_flush_remote_tlbs(kvm);
5517	5482	}
5518	5483
5519		− spin_unlock(&kvm->mmu_lock);
	5484	+ write_unlock(&kvm->mmu_lock);
5520	5485	}
5521	5486
5522	5487	static bool slot_rmap_write_protect(struct kvm *kvm,
@@ -5531,12 +5496,12 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, · 8 unmodified lines
5531	5496	{
5532	5497	bool flush;
5533	5498
5534		− spin_lock(&kvm->mmu_lock);
	5499	+ write_lock(&kvm->mmu_lock);
5535	5500	flush = slot_handle_level(kvm, memslot, slot_rmap_write_protect,
5536	5501	start_level, KVM_MAX_HUGEPAGE_LEVEL, false);
5537		− if (kvm->arch.tdp_mmu_enabled)
	5502	+ if (is_tdp_mmu_enabled(kvm))
5538	5503	flush \|= kvm_tdp_mmu_wrprot_slot(kvm, memslot, PG_LEVEL_4K);
5539		− spin_unlock(&kvm->mmu_lock);
	5504	+ write_unlock(&kvm->mmu_lock);
5540	5505
5541	5506	/*
5542	5507	* We can flush all the TLBs out of the mmu lock without TLB
@@ -5596,13 +5561,13 @@ void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm, · 53 unmodified lines
5596	5561	const struct kvm_memory_slot *memslot)
5597	5562	{
5598	5563	/* FIXME: const-ify all uses of struct kvm_memory_slot. */
5599		− spin_lock(&kvm->mmu_lock);
	5564	+ write_lock(&kvm->mmu_lock);
5600	5565	slot_handle_leaf(kvm, (struct kvm_memory_slot *)memslot,
5601	5566	kvm_mmu_zap_collapsible_spte, true);
5602	5567
5603		− if (kvm->arch.tdp_mmu_enabled)
	5568	+ if (is_tdp_mmu_enabled(kvm))
5604	5569	kvm_tdp_mmu_zap_collapsible_sptes(kvm, memslot);
5605		− spin_unlock(&kvm->mmu_lock);
	5570	+ write_unlock(&kvm->mmu_lock);
5606	5571	}
5607	5572
5608	5573	void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,
@@ -5625,11 +5590,11 @@ void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, · 16 unmodified lines
5625	5590	{
5626	5591	bool flush;
5627	5592
5628		− spin_lock(&kvm->mmu_lock);
	5593	+ write_lock(&kvm->mmu_lock);
5629	5594	flush = slot_handle_leaf(kvm, memslot, __rmap_clear_dirty, false);
5630		− if (kvm->arch.tdp_mmu_enabled)
	5595	+ if (is_tdp_mmu_enabled(kvm))
5631	5596	flush \|= kvm_tdp_mmu_clear_dirty_slot(kvm, memslot);
5632		− spin_unlock(&kvm->mmu_lock);
	5597	+ write_unlock(&kvm->mmu_lock);
5633	5598
5634	5599	/*
5635	5600	* It's also safe to flush TLBs out of mmu lock here as currently this
@@ -5647,12 +5612,12 @@ void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm, · 11 unmodified lines
5647	5612	{
5648	5613	bool flush;
5649	5614
5650		− spin_lock(&kvm->mmu_lock);
	5615	+ write_lock(&kvm->mmu_lock);
5651	5616	flush = slot_handle_large_level(kvm, memslot, slot_rmap_write_protect,
5652	5617	false);
5653		− if (kvm->arch.tdp_mmu_enabled)
	5618	+ if (is_tdp_mmu_enabled(kvm))
5654	5619	flush \|= kvm_tdp_mmu_wrprot_slot(kvm, memslot, PG_LEVEL_2M);
5655		− spin_unlock(&kvm->mmu_lock);
	5620	+ write_unlock(&kvm->mmu_lock);
5656	5621
5657	5622	if (flush)
5658	5623	kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);
@@ -5664,11 +5629,11 @@ void kvm_mmu_slot_set_dirty(struct kvm *kvm, · 5 unmodified lines
5664	5629	{
5665	5630	bool flush;
5666	5631
5667		− spin_lock(&kvm->mmu_lock);
	5632	+ write_lock(&kvm->mmu_lock);
5668	5633	flush = slot_handle_all_level(kvm, memslot, __rmap_set_dirty, false);
5669		− if (kvm->arch.tdp_mmu_enabled)
	5634	+ if (is_tdp_mmu_enabled(kvm))
5670	5635	flush \|= kvm_tdp_mmu_slot_set_dirty(kvm, memslot);
5671		− spin_unlock(&kvm->mmu_lock);
	5636	+ write_unlock(&kvm->mmu_lock);
5672	5637
5673	5638	if (flush)
5674	5639	kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);
@@ -5681,23 +5646,23 @@ void kvm_mmu_zap_all(struct kvm *kvm) · 6 unmodified lines
5681	5646	LIST_HEAD(invalid_list);
5682	5647	int ign;
5683	5648
5684		− spin_lock(&kvm->mmu_lock);
	5649	+ write_lock(&kvm->mmu_lock);
5685	5650	restart:
5686	5651	list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) {
5687	5652	if (WARN_ON(sp->role.invalid))
5688	5653	continue;
5689	5654	if (__kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list, &ign))
5690	5655	goto restart;
5691		− if (cond_resched_lock(&kvm->mmu_lock))
	5656	+ if (cond_resched_rwlock_write(&kvm->mmu_lock))
5692	5657	goto restart;
5693	5658	}
5694	5659
5695	5660	kvm_mmu_commit_zap_page(kvm, &invalid_list);
5696	5661
5697		− if (kvm->arch.tdp_mmu_enabled)
	5662	+ if (is_tdp_mmu_enabled(kvm))
5698	5663	kvm_tdp_mmu_zap_all(kvm);
5699	5664
5700		− spin_unlock(&kvm->mmu_lock);
	5665	+ write_unlock(&kvm->mmu_lock);
5701	5666	}
5702	5667
5703	5668	void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen)
@@ -5757,7 +5722,7 @@ mmu_shrink_scan(struct shrinker shrink, struct shrink_control sc) · 53 unmodified lines
5757	5722	continue;
5758	5723
5759	5724	idx = srcu_read_lock(&kvm->srcu);
5760		− spin_lock(&kvm->mmu_lock);
	5725	+ write_lock(&kvm->mmu_lock);
5761	5726
5762	5727	if (kvm_has_zapped_obsolete_pages(kvm)) {
5763	5728	kvm_mmu_commit_zap_page(kvm,
@@ -5768,7 +5733,7 @@ mmu_shrink_scan(struct shrinker shrink, struct shrink_control sc) · 4 unmodified lines
5768	5733	freed = kvm_mmu_zap_oldest_mmu_pages(kvm, sc->nr_to_scan);
5769	5734
5770	5735	unlock:
5771		− spin_unlock(&kvm->mmu_lock);
	5736	+ write_unlock(&kvm->mmu_lock);
5772	5737	srcu_read_unlock(&kvm->srcu, idx);
5773	5738
5774	5739	/*
@@ -5988,7 +5953,7 @@ static void kvm_recover_nx_lpages(struct kvm *kvm) · 213 unmodified lines
5988	5953	ulong to_zap;
5989	5954
5990	5955	rcu_idx = srcu_read_lock(&kvm->srcu);
5991		− spin_lock(&kvm->mmu_lock);
	5956	+ write_lock(&kvm->mmu_lock);
5992	5957
5993	5958	ratio = READ_ONCE(nx_huge_pages_recovery_ratio);
5994	5959	to_zap = ratio ? DIV_ROUND_UP(kvm->stat.nx_lpage_splits, ratio) : 0;
@@ -6005,22 +5970,22 @@ static void kvm_recover_nx_lpages(struct kvm *kvm) · 10 unmodified lines
6005	5970	struct kvm_mmu_page,
6006	5971	lpage_disallowed_link);
6007	5972	WARN_ON_ONCE(!sp->lpage_disallowed);
6008		− if (sp->tdp_mmu_page)
	5973	+ if (is_tdp_mmu_page(sp)) {
6009	5974	kvm_tdp_mmu_zap_gfn_range(kvm, sp->gfn,
6010	5975	sp->gfn + KVM_PAGES_PER_HPAGE(sp->role.level));
6011		− else {
	5976	+ } else {
6012	5977	kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
6013	5978	WARN_ON_ONCE(sp->lpage_disallowed);
6014	5979	}
6015	5980
6016		− if (need_resched() \|\| spin_needbreak(&kvm->mmu_lock)) {
	5981	+ if (need_resched() \|\| rwlock_needbreak(&kvm->mmu_lock)) {
6017	5982	kvm_mmu_commit_zap_page(kvm, &invalid_list);
6018		− cond_resched_lock(&kvm->mmu_lock);
	5983	+ cond_resched_rwlock_write(&kvm->mmu_lock);
6019	5984	}
6020	5985	}
6021	5986	kvm_mmu_commit_zap_page(kvm, &invalid_list);
6022	5987
6023		− spin_unlock(&kvm->mmu_lock);
	5988	+ write_unlock(&kvm->mmu_lock);
6024	5989	srcu_read_unlock(&kvm->srcu, rcu_idx);
6025	5990	}
6026	5991

arch/x86/kvm/mmu/mmu_audit.c+4−4

@@ -234,7 +234,7 @@ static void audit_vcpu_spte(struct kvm_vcpu *vcpu) · 233 unmodified lines
234	234	}
235	235
236	236	static bool mmu_audit;
237		−static struct static_key mmu_audit_key;
	237	+static DEFINE_STATIC_KEY_FALSE(mmu_audit_key);
238	238
239	239	static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
240	240	{
@@ -250,7 +250,7 @@ static void __kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) · 9 unmodified lines
250	250
251	251	static inline void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point)
252	252	{
253		− if (static_key_false((&mmu_audit_key)))
	253	+ if (static_branch_unlikely((&mmu_audit_key)))
254	254	__kvm_mmu_audit(vcpu, point);
255	255	}
256	256
@@ -259,7 +259,7 @@ static void mmu_audit_enable(void) · 2 unmodified lines
259	259	if (mmu_audit)
260	260	return;
261	261
262		− static_key_slow_inc(&mmu_audit_key);
	262	+ static_branch_inc(&mmu_audit_key);
263	263	mmu_audit = true;
264	264	}
265	265
@@ -268,7 +268,7 @@ static void mmu_audit_disable(void) · 2 unmodified lines
268	268	if (!mmu_audit)
269	269	return;
270	270
271		− static_key_slow_dec(&mmu_audit_key);
	271	+ static_branch_dec(&mmu_audit_key);
272	272	mmu_audit = false;
273	273	}
274	274

arch/x86/kvm/mmu/mmu_internal.h+6−1

@@ -12,7 +12,7 @@ · 11 unmodified lines
12	12	extern bool dbg;
13	13
14	14	#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
15		−#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
	15	+#define rmap_printk(fmt, args...) do { if (dbg) printk("%s: " fmt, __func__, ## args); } while (0)
16	16	#define MMU_WARN_ON(x) WARN_ON(x)
17	17	#else
18	18	#define pgprintk(x...) do { } while (0)
@@ -56,7 +56,12 @@ struct kvm_mmu_page { · 37 unmodified lines
56	56	/* Number of writes since the last time traversal visited this page. */
57	57	atomic_t write_flooding_count;
58	58
	59	+#ifdef CONFIG_X86_64
59	60	bool tdp_mmu_page;
	61	+
	62	+ /* Used for freeing the page asyncronously if it is a TDP MMU page. */
	63	+ struct rcu_head rcu_head;
	64	+#endif
60	65	};
61	66
62	67	extern struct kmem_cache *mmu_page_header_cache;

arch/x86/kvm/mmu/page_track.c+4−4

@@ -184,9 +184,9 @@ kvm_page_track_register_notifier(struct kvm *kvm, · 183 unmodified lines
184	184
185	185	head = &kvm->arch.track_notifier_head;
186	186
187		− spin_lock(&kvm->mmu_lock);
	187	+ write_lock(&kvm->mmu_lock);
188	188	hlist_add_head_rcu(&n->node, &head->track_notifier_list);
189		− spin_unlock(&kvm->mmu_lock);
	189	+ write_unlock(&kvm->mmu_lock);
190	190	}
191	191	EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier);
192	192
@@ -202,9 +202,9 @@ kvm_page_track_unregister_notifier(struct kvm *kvm, · 9 unmodified lines
202	202
203	203	head = &kvm->arch.track_notifier_head;
204	204
205		− spin_lock(&kvm->mmu_lock);
	205	+ write_lock(&kvm->mmu_lock);
206	206	hlist_del_rcu(&n->node);
207		− spin_unlock(&kvm->mmu_lock);
	207	+ write_unlock(&kvm->mmu_lock);
208	208	synchronize_srcu(&head->track_srcu);
209	209	}
210	210	EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier);

arch/x86/kvm/mmu/paging_tmpl.h+4−4

@@ -868,7 +868,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, · 867 unmodified lines
868	868	}
869	869
870	870	r = RET_PF_RETRY;
871		− spin_lock(&vcpu->kvm->mmu_lock);
	871	+ write_lock(&vcpu->kvm->mmu_lock);
872	872	if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
873	873	goto out_unlock;
874	874
@@ -881,7 +881,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, · 6 unmodified lines
881	881	kvm_mmu_audit(vcpu, AUDIT_POST_PAGE_FAULT);
882	882
883	883	out_unlock:
884		− spin_unlock(&vcpu->kvm->mmu_lock);
	884	+ write_unlock(&vcpu->kvm->mmu_lock);
885	885	kvm_release_pfn_clean(pfn);
886	886	return r;
887	887	}
@@ -919,7 +919,7 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa) · 31 unmodified lines
919	919	return;
920	920	}
921	921
922		− spin_lock(&vcpu->kvm->mmu_lock);
	922	+ write_lock(&vcpu->kvm->mmu_lock);
923	923	for_each_shadow_entry_using_root(vcpu, root_hpa, gva, iterator) {
924	924	level = iterator.level;
925	925	sptep = iterator.sptep;
@@ -954,7 +954,7 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa) · 28 unmodified lines
954	954	if (!is_shadow_present_pte(*sptep) \|\| !sp->unsync_children)
955	955	break;
956	956	}
957		− spin_unlock(&vcpu->kvm->mmu_lock);
	957	+ write_unlock(&vcpu->kvm->mmu_lock);
958	958	}
959	959
960	960	/* Note, @addr is a GPA when gva_to_gpa() translates an L2 GPA to an L1 GPA. */

arch/x86/kvm/mmu/spte.c+1−1

@@ -120,7 +120,7 @@ int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level, · 119 unmodified lines
120	120	if (level > PG_LEVEL_4K)
121	121	spte \|= PT_PAGE_SIZE_MASK;
122	122	if (tdp_enabled)
123		− spte \|= kvm_x86_ops.get_mt_mask(vcpu, gfn,
	123	+ spte \|= static_call(kvm_x86_get_mt_mask)(vcpu, gfn,
124	124	kvm_is_mmio_pfn(pfn));
125	125
126	126	if (host_writable)

arch/x86/kvm/mmu/spte.h+24−9

@@ -130,6 +130,25 @@ extern u64 __read_mostly shadow_nonpresent_or_rsvd_mask; · 129 unmodified lines
130	130	PT64_EPT_EXECUTABLE_MASK)
131	131	#define SHADOW_ACC_TRACK_SAVED_BITS_SHIFT PT64_SECOND_AVAIL_BITS_SHIFT
132	132
	133	+/*
	134	+ * If a thread running without exclusive control of the MMU lock must perform a
	135	+ * multi-part operation on an SPTE, it can set the SPTE to REMOVED_SPTE as a
	136	+ * non-present intermediate value. Other threads which encounter this value
	137	+ * should not modify the SPTE.
	138	+ *
	139	+ * This constant works because it is considered non-present on both AMD and
	140	+ * Intel CPUs and does not create a L1TF vulnerability because the pfn section
	141	+ * is zeroed out.
	142	+ *
	143	+ * Only used by the TDP MMU.
	144	+ */
	145	+#define REMOVED_SPTE (1ull << 59)
	146	+
	147	+static inline bool is_removed_spte(u64 spte)
	148	+{
	149	+ return spte == REMOVED_SPTE;
	150	+}
	151	+
133	152	/*
134	153	* In some cases, we need to preserve the GFN of a non-present or reserved
135	154	* SPTE when we usurp the upper five bits of the physical address space to
@@ -185,23 +204,19 @@ static inline bool is_access_track_spte(u64 spte) · 49 unmodified lines
185	204	return !spte_ad_enabled(spte) && (spte & shadow_acc_track_mask) == 0;
186	205	}
187	206
188		−static inline int is_shadow_present_pte(u64 pte)
	207	+static inline bool is_shadow_present_pte(u64 pte)
189	208	{
190		− return (pte != 0) && !is_mmio_spte(pte);
	209	+ return (pte != 0) && !is_mmio_spte(pte) && !is_removed_spte(pte);
191	210	}
192	211
193		−static inline int is_large_pte(u64 pte)
	212	+static inline bool is_large_pte(u64 pte)
194	213	{
195	214	return pte & PT_PAGE_SIZE_MASK;
196	215	}
197	216
198		−static inline int is_last_spte(u64 pte, int level)
	217	+static inline bool is_last_spte(u64 pte, int level)
199	218	{
200		− if (level == PG_LEVEL_4K)
201		− return 1;
202		− if (is_large_pte(pte))
203		− return 1;
204		− return 0;
	219	+ return (level == PG_LEVEL_4K) \|\| is_large_pte(pte);
205	220	}
206	221
207	222	static inline bool is_executable_pte(u64 spte)

arch/x86/kvm/mmu/tdp_iter.c+15−31

@@ -12,7 +12,7 @@ static void tdp_iter_refresh_sptep(struct tdp_iter *iter) · 11 unmodified lines
12	12	{
13	13	iter->sptep = iter->pt_path[iter->level - 1] +
14	14	SHADOW_PT_INDEX(iter->gfn << PAGE_SHIFT, iter->level);
15		− iter->old_spte = READ_ONCE(*iter->sptep);
	15	+ iter->old_spte = READ_ONCE(*rcu_dereference(iter->sptep));
16	16	}
17	17
18	18	static gfn_t round_gfn_for_level(gfn_t gfn, int level)
@@ -22,21 +22,22 @@ static gfn_t round_gfn_for_level(gfn_t gfn, int level) · 3 unmodified lines
22	22
23	23	/*
24	24	* Sets a TDP iterator to walk a pre-order traversal of the paging structure
25		− * rooted at root_pt, starting with the walk to translate goal_gfn.
	25	+ * rooted at root_pt, starting with the walk to translate next_last_level_gfn.
26	26	*/
27	27	void tdp_iter_start(struct tdp_iter iter, u64 root_pt, int root_level,
28		− int min_level, gfn_t goal_gfn)
	28	+ int min_level, gfn_t next_last_level_gfn)
29	29	{
30	30	WARN_ON(root_level < 1);
31	31	WARN_ON(root_level > PT64_ROOT_MAX_LEVEL);
32	32
33		− iter->goal_gfn = goal_gfn;
	33	+ iter->next_last_level_gfn = next_last_level_gfn;
	34	+ iter->yielded_gfn = iter->next_last_level_gfn;
34	35	iter->root_level = root_level;
35	36	iter->min_level = min_level;
36	37	iter->level = root_level;
37		− iter->pt_path[iter->level - 1] = root_pt;
	38	+ iter->pt_path[iter->level - 1] = (tdp_ptep_t)root_pt;
38	39
39		− iter->gfn = round_gfn_for_level(iter->goal_gfn, iter->level);
	40	+ iter->gfn = round_gfn_for_level(iter->next_last_level_gfn, iter->level);
40	41	tdp_iter_refresh_sptep(iter);
41	42
42	43	iter->valid = true;
@@ -47,7 +48,7 @@ void tdp_iter_start(struct tdp_iter iter, u64 root_pt, int root_level, · 4 unmodified lines
47	48	* address of the child page table referenced by the SPTE. Returns null if
48	49	* there is no such entry.
49	50	*/
50		−u64 *spte_to_child_pt(u64 spte, int level)
	51	+tdp_ptep_t spte_to_child_pt(u64 spte, int level)
51	52	{
52	53	/*
53	54	* There's no child entry if this entry isn't present or is a
@@ -56,7 +57,7 @@ u64 *spte_to_child_pt(u64 spte, int level) · 2 unmodified lines
56	57	if (!is_shadow_present_pte(spte) \|\| is_last_spte(spte, level))
57	58	return NULL;
58	59
59		− return __va(spte_to_pfn(spte) << PAGE_SHIFT);
	60	+ return (tdp_ptep_t)__va(spte_to_pfn(spte) << PAGE_SHIFT);
60	61	}
61	62
62	63	/*
@@ -65,7 +66,7 @@ u64 *spte_to_child_pt(u64 spte, int level) · 2 unmodified lines
65	66	*/
66	67	static bool try_step_down(struct tdp_iter *iter)
67	68	{
68		− u64 *child_pt;
	69	+ tdp_ptep_t child_pt;
69	70
70	71	if (iter->level == iter->min_level)
71	72	return false;
@@ -74,7 +75,7 @@ static bool try_step_down(struct tdp_iter *iter) · 2 unmodified lines
74	75	* Reread the SPTE before stepping down to avoid traversing into page
75	76	* tables that are no longer linked from this entry.
76	77	*/
77		− iter->old_spte = READ_ONCE(*iter->sptep);
	78	+ iter->old_spte = READ_ONCE(*rcu_dereference(iter->sptep));
78	79
79	80	child_pt = spte_to_child_pt(iter->old_spte, iter->level);
80	81	if (!child_pt)
@@ -82,7 +83,7 @@ static bool try_step_down(struct tdp_iter *iter) · 1 unmodified line
82	83
83	84	iter->level--;
84	85	iter->pt_path[iter->level - 1] = child_pt;
85		− iter->gfn = round_gfn_for_level(iter->goal_gfn, iter->level);
	86	+ iter->gfn = round_gfn_for_level(iter->next_last_level_gfn, iter->level);
86	87	tdp_iter_refresh_sptep(iter);
87	88
88	89	return true;
@@ -106,9 +107,9 @@ static bool try_step_side(struct tdp_iter *iter) · 17 unmodified lines
106	107	return false;
107	108
108	109	iter->gfn += KVM_PAGES_PER_HPAGE(iter->level);
109		− iter->goal_gfn = iter->gfn;
	110	+ iter->next_last_level_gfn = iter->gfn;
110	111	iter->sptep++;
111		− iter->old_spte = READ_ONCE(*iter->sptep);
	112	+ iter->old_spte = READ_ONCE(*rcu_dereference(iter->sptep));
112	113
113	114	return true;
114	115	}
@@ -158,24 +159,7 @@ void tdp_iter_next(struct tdp_iter *iter) · 43 unmodified lines
158	159	iter->valid = false;
159	160	}
160	161
161		−/*
162		− * Restart the walk over the paging structure from the root, starting from the
163		− * highest gfn the iterator had previously reached. Assumes that the entire
164		− * paging structure, except the root page, may have been completely torn down
165		− * and rebuilt.
166		− */
167		−void tdp_iter_refresh_walk(struct tdp_iter *iter)
168		−{
169		− gfn_t goal_gfn = iter->goal_gfn;
170		−
171		− if (iter->gfn > goal_gfn)
172		− goal_gfn = iter->gfn;
173		−
174		− tdp_iter_start(iter, iter->pt_path[iter->root_level - 1],
175		− iter->root_level, iter->min_level, goal_gfn);
176		−}
177		−
178		−u64 tdp_iter_root_pt(struct tdp_iter iter)
	162	+tdp_ptep_t tdp_iter_root_pt(struct tdp_iter *iter)
179	163	{
180	164	return iter->pt_path[iter->root_level - 1];
181	165	}

arch/x86/kvm/mmu/tdp_iter.h+14−7

@@ -7,6 +7,8 @@ · 6 unmodified lines
7	7
8	8	#include "mmu.h"
9	9
	10	+typedef u64 __rcu *tdp_ptep_t;
	11	+
10	12	/*
11	13	* A TDP iterator performs a pre-order walk over a TDP paging structure.
12	14	*/
@@ -15,11 +17,17 @@ struct tdp_iter { · 2 unmodified lines
15	17	* The iterator will traverse the paging structure towards the mapping
16	18	* for this GFN.
17	19	*/
18		− gfn_t goal_gfn;
	20	+ gfn_t next_last_level_gfn;
	21	+ /*
	22	+ * The next_last_level_gfn at the time when the thread last
	23	+ * yielded. Only yielding when the next_last_level_gfn !=
	24	+ * yielded_gfn helps ensure forward progress.
	25	+ */
	26	+ gfn_t yielded_gfn;
19	27	/* Pointers to the page tables traversed to reach the current SPTE */
20		− u64 *pt_path[PT64_ROOT_MAX_LEVEL];
	28	+ tdp_ptep_t pt_path[PT64_ROOT_MAX_LEVEL];
21	29	/* A pointer to the current SPTE */
22		− u64 *sptep;
	30	+ tdp_ptep_t sptep;
23	31	/* The lowest GFN mapped by the current SPTE */
24	32	gfn_t gfn;
25	33	/* The level of the root page given to the iterator */
@@ -49,12 +57,11 @@ struct tdp_iter { · 23 unmodified lines
49	57	#define for_each_tdp_pte(iter, root, root_level, start, end) \
50	58	for_each_tdp_pte_min_level(iter, root, root_level, PG_LEVEL_4K, start, end)
51	59
52		−u64 *spte_to_child_pt(u64 pte, int level);
	60	+tdp_ptep_t spte_to_child_pt(u64 pte, int level);
53	61
54	62	void tdp_iter_start(struct tdp_iter iter, u64 root_pt, int root_level,
55		− int min_level, gfn_t goal_gfn);
	63	+ int min_level, gfn_t next_last_level_gfn);
56	64	void tdp_iter_next(struct tdp_iter *iter);
57		−void tdp_iter_refresh_walk(struct tdp_iter *iter);
58		−u64 tdp_iter_root_pt(struct tdp_iter iter);
	65	+tdp_ptep_t tdp_iter_root_pt(struct tdp_iter *iter);
59	66
60	67	#endif /* __KVM_X86_MMU_TDP_ITER_H */

arch/x86/kvm/mmu/tdp_mmu.c+422−132

@@ -7,32 +7,23 @@ · 6 unmodified lines
7	7	#include "tdp_mmu.h"
8	8	#include "spte.h"
9	9
	10	+#include <asm/cmpxchg.h>
10	11	#include <trace/events/kvm.h>
11	12
12		−#ifdef CONFIG_X86_64
13	13	static bool __read_mostly tdp_mmu_enabled = false;
14	14	module_param_named(tdp_mmu, tdp_mmu_enabled, bool, 0644);
15		−#endif
16		−
17		−static bool is_tdp_mmu_enabled(void)
18		−{
19		−#ifdef CONFIG_X86_64
20		− return tdp_enabled && READ_ONCE(tdp_mmu_enabled);
21		−#else
22		− return false;
23		−#endif /* CONFIG_X86_64 */
24		−}
25	15
26	16	/* Initializes the TDP MMU for the VM, if enabled. */
27	17	void kvm_mmu_init_tdp_mmu(struct kvm *kvm)
28	18	{
29		− if (!is_tdp_mmu_enabled())
	19	+ if (!tdp_enabled \|\| !READ_ONCE(tdp_mmu_enabled))
30	20	return;
31	21
32	22	/* This should not be changed for the lifetime of the VM. */
33	23	kvm->arch.tdp_mmu_enabled = true;
34	24
35	25	INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots);
	26	+ spin_lock_init(&kvm->arch.tdp_mmu_pages_lock);
36	27	INIT_LIST_HEAD(&kvm->arch.tdp_mmu_pages);
37	28	}
38	29
@@ -42,6 +33,12 @@ void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) · 3 unmodified lines
42	33	return;
43	34
44	35	WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots));
	36	+
	37	+ /*
	38	+ * Ensure that all the outstanding RCU callbacks to free shadow pages
	39	+ * can run before the VM is torn down.
	40	+ */
	41	+ rcu_barrier();
45	42	}
46	43
47	44	static void tdp_mmu_put_root(struct kvm kvm, struct kvm_mmu_page root)
@@ -53,7 +50,7 @@ static void tdp_mmu_put_root(struct kvm kvm, struct kvm_mmu_page root) · 5 unmodified lines
53	50	static inline bool tdp_mmu_next_root_valid(struct kvm *kvm,
54	51	struct kvm_mmu_page *root)
55	52	{
56		− lockdep_assert_held(&kvm->mmu_lock);
	53	+ lockdep_assert_held_write(&kvm->mmu_lock);
57	54
58	55	if (list_entry_is_head(root, &kvm->arch.tdp_mmu_roots, link))
59	56	return false;
@@ -88,22 +85,6 @@ static inline struct kvm_mmu_page tdp_mmu_next_root(struct kvm kvm, · 28 unmodified lines
88	85	#define for_each_tdp_mmu_root(_kvm, _root) \
89	86	list_for_each_entry(_root, &_kvm->arch.tdp_mmu_roots, link)
90	87
91		−bool is_tdp_mmu_root(struct kvm *kvm, hpa_t hpa)
92		−{
93		− struct kvm_mmu_page *sp;
94		−
95		− if (!kvm->arch.tdp_mmu_enabled)
96		− return false;
97		− if (WARN_ON(!VALID_PAGE(hpa)))
98		− return false;
99		−
100		− sp = to_shadow_page(hpa);
101		− if (WARN_ON(!sp))
102		− return false;
103		−
104		− return sp->tdp_mmu_page && sp->root_count;
105		−}
106		−
107	88	static bool zap_gfn_range(struct kvm kvm, struct kvm_mmu_page root,
108	89	gfn_t start, gfn_t end, bool can_yield);
109	90
@@ -111,7 +92,7 @@ void kvm_tdp_mmu_free_root(struct kvm kvm, struct kvm_mmu_page root) · 1 unmodified line
111	92	{
112	93	gfn_t max_gfn = 1ULL << (shadow_phys_bits - PAGE_SHIFT);
113	94
114		− lockdep_assert_held(&kvm->mmu_lock);
	95	+ lockdep_assert_held_write(&kvm->mmu_lock);
115	96
116	97	WARN_ON(root->root_count);
117	98	WARN_ON(!root->tdp_mmu_page);
@@ -164,13 +145,13 @@ static struct kvm_mmu_page get_tdp_mmu_vcpu_root(struct kvm_vcpu vcpu) · 46 unmodified lines
164	145
165	146	role = page_role_for_level(vcpu, vcpu->arch.mmu->shadow_root_level);
166	147
167		− spin_lock(&kvm->mmu_lock);
	148	+ write_lock(&kvm->mmu_lock);
168	149
169	150	/* Check for an existing root before allocating a new one. */
170	151	for_each_tdp_mmu_root(kvm, root) {
171	152	if (root->role.word == role.word) {
172	153	kvm_mmu_get_root(kvm, root);
173		− spin_unlock(&kvm->mmu_lock);
	154	+ write_unlock(&kvm->mmu_lock);
174	155	return root;
175	156	}
176	157	}
@@ -180,7 +161,7 @@ static struct kvm_mmu_page get_tdp_mmu_vcpu_root(struct kvm_vcpu vcpu) · 3 unmodified lines
180	161
181	162	list_add(&root->link, &kvm->arch.tdp_mmu_roots);
182	163
183		− spin_unlock(&kvm->mmu_lock);
	164	+ write_unlock(&kvm->mmu_lock);
184	165
185	166	return root;
186	167	}
@@ -196,8 +177,31 @@ hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu) · 9 unmodified lines
196	177	return __pa(root->spt);
197	178	}
198	179
	180	+static void tdp_mmu_free_sp(struct kvm_mmu_page *sp)
	181	+{
	182	+ free_page((unsigned long)sp->spt);
	183	+ kmem_cache_free(mmu_page_header_cache, sp);
	184	+}
	185	+
	186	+/*
	187	+ * This is called through call_rcu in order to free TDP page table memory
	188	+ * safely with respect to other kernel threads that may be operating on
	189	+ * the memory.
	190	+ * By only accessing TDP MMU page table memory in an RCU read critical
	191	+ * section, and freeing it after a grace period, lockless access to that
	192	+ * memory won't use it after it is freed.
	193	+ */
	194	+static void tdp_mmu_free_sp_rcu_callback(struct rcu_head *head)
	195	+{
	196	+ struct kvm_mmu_page *sp = container_of(head, struct kvm_mmu_page,
	197	+ rcu_head);
	198	+
	199	+ tdp_mmu_free_sp(sp);
	200	+}
	201	+
199	202	static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,
200		− u64 old_spte, u64 new_spte, int level);
	203	+ u64 old_spte, u64 new_spte, int level,
	204	+ bool shared);
201	205
202	206	static int kvm_mmu_page_as_id(struct kvm_mmu_page *sp)
203	207	{
@@ -234,6 +238,128 @@ static void handle_changed_spte_dirty_log(struct kvm *kvm, int as_id, gfn_t gfn, · 30 unmodified lines
234	238	}
235	239	}
236	240
	241	+/**
	242	+ * tdp_mmu_link_page - Add a new page to the list of pages used by the TDP MMU
	243	+ *
	244	+ * @kvm: kvm instance
	245	+ * @sp: the new page
	246	+ * @shared: This operation may not be running under the exclusive use of
	247	+ * the MMU lock and the operation must synchronize with other
	248	+ * threads that might be adding or removing pages.
	249	+ * @account_nx: This page replaces a NX large page and should be marked for
	250	+ * eventual reclaim.
	251	+ */
	252	+static void tdp_mmu_link_page(struct kvm kvm, struct kvm_mmu_page sp,
	253	+ bool shared, bool account_nx)
	254	+{
	255	+ if (shared)
	256	+ spin_lock(&kvm->arch.tdp_mmu_pages_lock);
	257	+ else
	258	+ lockdep_assert_held_write(&kvm->mmu_lock);
	259	+
	260	+ list_add(&sp->link, &kvm->arch.tdp_mmu_pages);
	261	+ if (account_nx)
	262	+ account_huge_nx_page(kvm, sp);
	263	+
	264	+ if (shared)
	265	+ spin_unlock(&kvm->arch.tdp_mmu_pages_lock);
	266	+}
	267	+
	268	+/**
	269	+ * tdp_mmu_unlink_page - Remove page from the list of pages used by the TDP MMU
	270	+ *
	271	+ * @kvm: kvm instance
	272	+ * @sp: the page to be removed
	273	+ * @shared: This operation may not be running under the exclusive use of
	274	+ * the MMU lock and the operation must synchronize with other
	275	+ * threads that might be adding or removing pages.
	276	+ */
	277	+static void tdp_mmu_unlink_page(struct kvm kvm, struct kvm_mmu_page sp,
	278	+ bool shared)
	279	+{
	280	+ if (shared)
	281	+ spin_lock(&kvm->arch.tdp_mmu_pages_lock);
	282	+ else
	283	+ lockdep_assert_held_write(&kvm->mmu_lock);
	284	+
	285	+ list_del(&sp->link);
	286	+ if (sp->lpage_disallowed)
	287	+ unaccount_huge_nx_page(kvm, sp);
	288	+
	289	+ if (shared)
	290	+ spin_unlock(&kvm->arch.tdp_mmu_pages_lock);
	291	+}
	292	+
	293	+/**
	294	+ * handle_removed_tdp_mmu_page - handle a pt removed from the TDP structure
	295	+ *
	296	+ * @kvm: kvm instance
	297	+ * @pt: the page removed from the paging structure
	298	+ * @shared: This operation may not be running under the exclusive use
	299	+ * of the MMU lock and the operation must synchronize with other
	300	+ * threads that might be modifying SPTEs.
	301	+ *
	302	+ * Given a page table that has been removed from the TDP paging structure,
	303	+ * iterates through the page table to clear SPTEs and free child page tables.
	304	+ */
	305	+static void handle_removed_tdp_mmu_page(struct kvm kvm, u64 pt,
	306	+ bool shared)
	307	+{
	308	+ struct kvm_mmu_page *sp = sptep_to_sp(pt);
	309	+ int level = sp->role.level;
	310	+ gfn_t base_gfn = sp->gfn;
	311	+ u64 old_child_spte;
	312	+ u64 *sptep;
	313	+ gfn_t gfn;
	314	+ int i;
	315	+
	316	+ trace_kvm_mmu_prepare_zap_page(sp);
	317	+
	318	+ tdp_mmu_unlink_page(kvm, sp, shared);
	319	+
	320	+ for (i = 0; i < PT64_ENT_PER_PAGE; i++) {
	321	+ sptep = pt + i;
	322	+ gfn = base_gfn + (i * KVM_PAGES_PER_HPAGE(level - 1));
	323	+
	324	+ if (shared) {
	325	+ /*
	326	+ * Set the SPTE to a nonpresent value that other
	327	+ * threads will not overwrite. If the SPTE was
	328	+ * already marked as removed then another thread
	329	+ * handling a page fault could overwrite it, so
	330	+ * set the SPTE until it is set from some other
	331	+ * value to the removed SPTE value.
	332	+ */
	333	+ for (;;) {
	334	+ old_child_spte = xchg(sptep, REMOVED_SPTE);
	335	+ if (!is_removed_spte(old_child_spte))
	336	+ break;
	337	+ cpu_relax();
	338	+ }
	339	+ } else {
	340	+ old_child_spte = READ_ONCE(*sptep);
	341	+
	342	+ /*
	343	+ * Marking the SPTE as a removed SPTE is not
	344	+ * strictly necessary here as the MMU lock will
	345	+ * stop other threads from concurrently modifying
	346	+ * this SPTE. Using the removed SPTE value keeps
	347	+ * the two branches consistent and simplifies
	348	+ * the function.
	349	+ */
	350	+ WRITE_ONCE(*sptep, REMOVED_SPTE);
	351	+ }
	352	+ handle_changed_spte(kvm, kvm_mmu_page_as_id(sp), gfn,
	353	+ old_child_spte, REMOVED_SPTE, level - 1,
	354	+ shared);
	355	+ }
	356	+
	357	+ kvm_flush_remote_tlbs_with_address(kvm, gfn,
	358	+ KVM_PAGES_PER_HPAGE(level));
	359	+
	360	+ call_rcu(&sp->rcu_head, tdp_mmu_free_sp_rcu_callback);
	361	+}
	362	+
237	363	/**
238	364	* handle_changed_spte - handle bookkeeping associated with an SPTE change
239	365	* @kvm: kvm instance
@@ -242,22 +368,22 @@ static void handle_changed_spte_dirty_log(struct kvm *kvm, int as_id, gfn_t gfn, · 2 unmodified lines
242	368	* @old_spte: The value of the SPTE before the change
243	369	* @new_spte: The value of the SPTE after the change
244	370	* @level: the level of the PT the SPTE is part of in the paging structure
	371	+ * @shared: This operation may not be running under the exclusive use of
	372	+ * the MMU lock and the operation must synchronize with other
	373	+ * threads that might be modifying SPTEs.
245	374	*
246	375	* Handle bookkeeping that might result from the modification of a SPTE.
247	376	* This function must be called for all TDP SPTE modifications.
248	377	*/
249	378	static void __handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,
250		− u64 old_spte, u64 new_spte, int level)
	379	+ u64 old_spte, u64 new_spte, int level,
	380	+ bool shared)
251	381	{
252	382	bool was_present = is_shadow_present_pte(old_spte);
253	383	bool is_present = is_shadow_present_pte(new_spte);
254	384	bool was_leaf = was_present && is_last_spte(old_spte, level);
255	385	bool is_leaf = is_present && is_last_spte(new_spte, level);
256	386	bool pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte);
257		− u64 *pt;
258		− struct kvm_mmu_page *sp;
259		− u64 old_child_spte;
260		− int i;
261	387
262	388	WARN_ON(level > PT64_ROOT_MAX_LEVEL);
263	389	WARN_ON(level < PG_LEVEL_4K);
@@ -298,15 +424,19 @@ static void __handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, · 34 unmodified lines
298	424	*/
299	425	if (!was_present && !is_present) {
300	426	/*
301		− * If this change does not involve a MMIO SPTE, it is
302		− * unexpected. Log the change, though it should not impact the
303		− * guest since both the former and current SPTEs are nonpresent.
	427	+ * If this change does not involve a MMIO SPTE or removed SPTE,
	428	+ * it is unexpected. Log the change, though it should not
	429	+ * impact the guest since both the former and current SPTEs
	430	+ * are nonpresent.
304	431	*/
305		− if (WARN_ON(!is_mmio_spte(old_spte) && !is_mmio_spte(new_spte)))
	432	+ if (WARN_ON(!is_mmio_spte(old_spte) &&
	433	+ !is_mmio_spte(new_spte) &&
	434	+ !is_removed_spte(new_spte)))
306	435	pr_err("Unexpected SPTE change! Nonpresent SPTEs\n"
307	436	"should not be replaced with another,\n"
308	437	"different nonpresent SPTE, unless one or both\n"
309		− "are MMIO SPTEs.\n"
	438	+ "are MMIO SPTEs, or the new SPTE is\n"
	439	+ "a temporary removed SPTE.\n"
310	440	"as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d",
311	441	as_id, gfn, old_spte, new_spte, level);
312	442	return;
@@ -321,54 +451,127 @@ static void __handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn, · 8 unmodified lines
321	451	* Recursively handle child PTs if the change removed a subtree from
322	452	* the paging structure.
323	453	*/
324		− if (was_present && !was_leaf && (pfn_changed \|\| !is_present)) {
325		− pt = spte_to_child_pt(old_spte, level);
326		− sp = sptep_to_sp(pt);
	454	+ if (was_present && !was_leaf && (pfn_changed \|\| !is_present))
	455	+ handle_removed_tdp_mmu_page(kvm,
	456	+ spte_to_child_pt(old_spte, level), shared);
	457	+}
327	458
328		− trace_kvm_mmu_prepare_zap_page(sp);
	459	+static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,
	460	+ u64 old_spte, u64 new_spte, int level,
	461	+ bool shared)
	462	+{
	463	+ __handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level,
	464	+ shared);
	465	+ handle_changed_spte_acc_track(old_spte, new_spte, level);
	466	+ handle_changed_spte_dirty_log(kvm, as_id, gfn, old_spte,
	467	+ new_spte, level);
	468	+}
329	469
330		− list_del(&sp->link);
	470	+/*
	471	+ * tdp_mmu_set_spte_atomic - Set a TDP MMU SPTE atomically and handle the
	472	+ * associated bookkeeping
	473	+ *
	474	+ * @kvm: kvm instance
	475	+ * @iter: a tdp_iter instance currently on the SPTE that should be set
	476	+ * @new_spte: The value the SPTE should be set to
	477	+ * Returns: true if the SPTE was set, false if it was not. If false is returned,
	478	+ * this function will have no side-effects.
	479	+ */
	480	+static inline bool tdp_mmu_set_spte_atomic(struct kvm *kvm,
	481	+ struct tdp_iter *iter,
	482	+ u64 new_spte)
	483	+{
	484	+ u64 *root_pt = tdp_iter_root_pt(iter);
	485	+ struct kvm_mmu_page *root = sptep_to_sp(root_pt);
	486	+ int as_id = kvm_mmu_page_as_id(root);
331	487
332		− if (sp->lpage_disallowed)
333		− unaccount_huge_nx_page(kvm, sp);
	488	+ lockdep_assert_held_read(&kvm->mmu_lock);
334	489
335		− for (i = 0; i < PT64_ENT_PER_PAGE; i++) {
336		− old_child_spte = READ_ONCE(*(pt + i));
337		− WRITE_ONCE(*(pt + i), 0);
338		− handle_changed_spte(kvm, as_id,
339		− gfn + (i * KVM_PAGES_PER_HPAGE(level - 1)),
340		− old_child_spte, 0, level - 1);
341		− }
	490	+ /*
	491	+ * Do not change removed SPTEs. Only the thread that froze the SPTE
	492	+ * may modify it.
	493	+ */
	494	+ if (iter->old_spte == REMOVED_SPTE)
	495	+ return false;
342	496
343		− kvm_flush_remote_tlbs_with_address(kvm, gfn,
344		− KVM_PAGES_PER_HPAGE(level));
	497	+ if (cmpxchg64(rcu_dereference(iter->sptep), iter->old_spte,
	498	+ new_spte) != iter->old_spte)
	499	+ return false;
345	500
346		− free_page((unsigned long)pt);
347		− kmem_cache_free(mmu_page_header_cache, sp);
348		− }
	501	+ handle_changed_spte(kvm, as_id, iter->gfn, iter->old_spte, new_spte,
	502	+ iter->level, true);
	503	+
	504	+ return true;
349	505	}
350	506
351		−static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,
352		− u64 old_spte, u64 new_spte, int level)
	507	+static inline bool tdp_mmu_zap_spte_atomic(struct kvm *kvm,
	508	+ struct tdp_iter *iter)
353	509	{
354		− __handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level);
355		− handle_changed_spte_acc_track(old_spte, new_spte, level);
356		− handle_changed_spte_dirty_log(kvm, as_id, gfn, old_spte,
357		− new_spte, level);
	510	+ /*
	511	+ * Freeze the SPTE by setting it to a special,
	512	+ * non-present value. This will stop other threads from
	513	+ * immediately installing a present entry in its place
	514	+ * before the TLBs are flushed.
	515	+ */
	516	+ if (!tdp_mmu_set_spte_atomic(kvm, iter, REMOVED_SPTE))
	517	+ return false;
	518	+
	519	+ kvm_flush_remote_tlbs_with_address(kvm, iter->gfn,
	520	+ KVM_PAGES_PER_HPAGE(iter->level));
	521	+
	522	+ /*
	523	+ * No other thread can overwrite the removed SPTE as they
	524	+ * must either wait on the MMU lock or use
	525	+ * tdp_mmu_set_spte_atomic which will not overrite the
	526	+ * special removed SPTE value. No bookkeeping is needed
	527	+ * here since the SPTE is going from non-present
	528	+ * to non-present.
	529	+ */
	530	+ WRITE_ONCE(*iter->sptep, 0);
	531	+
	532	+ return true;
358	533	}
359	534
	535	+
	536	+/*
	537	+ * __tdp_mmu_set_spte - Set a TDP MMU SPTE and handle the associated bookkeeping
	538	+ * @kvm: kvm instance
	539	+ * @iter: a tdp_iter instance currently on the SPTE that should be set
	540	+ * @new_spte: The value the SPTE should be set to
	541	+ * @record_acc_track: Notify the MM subsystem of changes to the accessed state
	542	+ * of the page. Should be set unless handling an MMU
	543	+ * notifier for access tracking. Leaving record_acc_track
	544	+ * unset in that case prevents page accesses from being
	545	+ * double counted.
	546	+ * @record_dirty_log: Record the page as dirty in the dirty bitmap if
	547	+ * appropriate for the change being made. Should be set
	548	+ * unless performing certain dirty logging operations.
	549	+ * Leaving record_dirty_log unset in that case prevents page
	550	+ * writes from being double counted.
	551	+ */
360	552	static inline void __tdp_mmu_set_spte(struct kvm kvm, struct tdp_iter iter,
361	553	u64 new_spte, bool record_acc_track,
362	554	bool record_dirty_log)
363	555	{
364		− u64 *root_pt = tdp_iter_root_pt(iter);
	556	+ tdp_ptep_t root_pt = tdp_iter_root_pt(iter);
365	557	struct kvm_mmu_page *root = sptep_to_sp(root_pt);
366	558	int as_id = kvm_mmu_page_as_id(root);
367	559
368		− WRITE_ONCE(*iter->sptep, new_spte);
	560	+ lockdep_assert_held_write(&kvm->mmu_lock);
	561	+
	562	+ /*
	563	+ * No thread should be using this function to set SPTEs to the
	564	+ * temporary removed SPTE value.
	565	+ * If operating under the MMU lock in read mode, tdp_mmu_set_spte_atomic
	566	+ * should be used. If operating under the MMU lock in write mode, the
	567	+ * use of the removed SPTE should not be necessary.
	568	+ */
	569	+ WARN_ON(iter->old_spte == REMOVED_SPTE);
	570	+
	571	+ WRITE_ONCE(*rcu_dereference(iter->sptep), new_spte);
369	572
370	573	__handle_changed_spte(kvm, as_id, iter->gfn, iter->old_spte, new_spte,
371		− iter->level);
	574	+ iter->level, false);
372	575	if (record_acc_track)
373	576	handle_changed_spte_acc_track(iter->old_spte, new_spte,
374	577	iter->level);
@@ -413,27 +616,46 @@ static inline void tdp_mmu_set_spte_no_dirty_log(struct kvm *kvm, · 38 unmodified lines
413	616	_mmu->shadow_root_level, _start, _end)
414	617
415	618	/*
416		− * Flush the TLB if the process should drop kvm->mmu_lock.
417		− * Return whether the caller still needs to flush the tlb.
	619	+ * Yield if the MMU lock is contended or this thread needs to return control
	620	+ * to the scheduler.
	621	+ *
	622	+ * If this function should yield and flush is set, it will perform a remote
	623	+ * TLB flush before yielding.
	624	+ *
	625	+ * If this function yields, it will also reset the tdp_iter's walk over the
	626	+ * paging structure and the calling function should skip to the next
	627	+ * iteration to allow the iterator to continue its traversal from the
	628	+ * paging structure root.
	629	+ *
	630	+ * Return true if this function yielded and the iterator's traversal was reset.
	631	+ * Return false if a yield was not needed.
418	632	*/
419		−static bool tdp_mmu_iter_flush_cond_resched(struct kvm kvm, struct tdp_iter iter)
	633	+static inline bool tdp_mmu_iter_cond_resched(struct kvm *kvm,
	634	+ struct tdp_iter *iter, bool flush)
420	635	{
421		− if (need_resched() \|\| spin_needbreak(&kvm->mmu_lock)) {
422		− kvm_flush_remote_tlbs(kvm);
423		− cond_resched_lock(&kvm->mmu_lock);
424		− tdp_iter_refresh_walk(iter);
	636	+ /* Ensure forward progress has been made before yielding. */
	637	+ if (iter->next_last_level_gfn == iter->yielded_gfn)
425	638	return false;
426		− } else {
	639	+
	640	+ if (need_resched() \|\| rwlock_needbreak(&kvm->mmu_lock)) {
	641	+ rcu_read_unlock();
	642	+
	643	+ if (flush)
	644	+ kvm_flush_remote_tlbs(kvm);
	645	+
	646	+ cond_resched_rwlock_write(&kvm->mmu_lock);
	647	+ rcu_read_lock();
	648	+
	649	+ WARN_ON(iter->gfn > iter->next_last_level_gfn);
	650	+
	651	+ tdp_iter_start(iter, iter->pt_path[iter->root_level - 1],
	652	+ iter->root_level, iter->min_level,
	653	+ iter->next_last_level_gfn);
	654	+
427	655	return true;
428	656	}
429		−}
430	657
431		−static void tdp_mmu_iter_cond_resched(struct kvm kvm, struct tdp_iter iter)
432		−{
433		− if (need_resched() \|\| spin_needbreak(&kvm->mmu_lock)) {
434		− cond_resched_lock(&kvm->mmu_lock);
435		− tdp_iter_refresh_walk(iter);
436		− }
	658	+ return false;
437	659	}
438	660
439	661	/*
@@ -453,7 +675,15 @@ static bool zap_gfn_range(struct kvm kvm, struct kvm_mmu_page root, · 13 unmodified lines
453	675	struct tdp_iter iter;
454	676	bool flush_needed = false;
455	677
	678	+ rcu_read_lock();
	679	+
456	680	tdp_root_for_each_pte(iter, root, start, end) {
	681	+ if (can_yield &&
	682	+ tdp_mmu_iter_cond_resched(kvm, &iter, flush_needed)) {
	683	+ flush_needed = false;
	684	+ continue;
	685	+ }
	686	+
457	687	if (!is_shadow_present_pte(iter.old_spte))
458	688	continue;
459	689
@@ -468,12 +698,10 @@ static bool zap_gfn_range(struct kvm kvm, struct kvm_mmu_page root, · 8 unmodified lines
468	698	continue;
469	699
470	700	tdp_mmu_set_spte(kvm, &iter, 0);
471		−
472		− if (can_yield)
473		− flush_needed = tdp_mmu_iter_flush_cond_resched(kvm, &iter);
474		− else
475		− flush_needed = true;
	701	+ flush_needed = true;
476	702	}
	703	+
	704	+ rcu_read_unlock();
477	705	return flush_needed;
478	706	}
479	707
@@ -517,21 +745,18 @@ static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu, int write, · 37 unmodified lines
517	745	int ret = 0;
518	746	int make_spte_ret = 0;
519	747
520		− if (unlikely(is_noslot_pfn(pfn))) {
	748	+ if (unlikely(is_noslot_pfn(pfn)))
521	749	new_spte = make_mmio_spte(vcpu, iter->gfn, ACC_ALL);
522		− trace_mark_mmio_spte(iter->sptep, iter->gfn, new_spte);
523		− } else {
	750	+ else
524	751	make_spte_ret = make_spte(vcpu, ACC_ALL, iter->level, iter->gfn,
525	752	pfn, iter->old_spte, prefault, true,
526	753	map_writable, !shadow_accessed_mask,
527	754	&new_spte);
528		− trace_kvm_mmu_set_spte(iter->level, iter->gfn, iter->sptep);
529		− }
530	755
531	756	if (new_spte == iter->old_spte)
532	757	ret = RET_PF_SPURIOUS;
533		− else
534		− tdp_mmu_set_spte(vcpu->kvm, iter, new_spte);
	758	+ else if (!tdp_mmu_set_spte_atomic(vcpu->kvm, iter, new_spte))
	759	+ return RET_PF_RETRY;
535	760
536	761	/*
537	762	* If the page fault was caused by a write but the page is write
@@ -545,10 +770,16 @@ static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu, int write, · 7 unmodified lines
545	770	}
546	771
547	772	/* If a MMIO SPTE is installed, the MMIO will need to be emulated. */
548		− if (unlikely(is_mmio_spte(new_spte)))
	773	+ if (unlikely(is_mmio_spte(new_spte))) {
	774	+ trace_mark_mmio_spte(rcu_dereference(iter->sptep), iter->gfn,
	775	+ new_spte);
549	776	ret = RET_PF_EMULATE;
	777	+ } else
	778	+ trace_kvm_mmu_set_spte(iter->level, iter->gfn,
	779	+ rcu_dereference(iter->sptep));
550	780
551		− trace_kvm_mmu_set_spte(iter->level, iter->gfn, iter->sptep);
	781	+ trace_kvm_mmu_set_spte(iter->level, iter->gfn,
	782	+ rcu_dereference(iter->sptep));
552	783	if (!prefault)
553	784	vcpu->stat.pf_fixed++;
554	785
@@ -586,6 +817,9 @@ int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, · 31 unmodified lines
586	817	huge_page_disallowed, &req_level);
587	818
588	819	trace_kvm_mmu_spte_requested(gpa, level, pfn);
	820	+
	821	+ rcu_read_lock();
	822	+
589	823	tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) {
590	824	if (nx_huge_page_workaround_enabled)
591	825	disallowed_hugepage_adjust(iter.old_spte, gfn,
@@ -601,49 +835,61 @@ int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, · 9 unmodified lines
601	835	*/
602	836	if (is_shadow_present_pte(iter.old_spte) &&
603	837	is_large_pte(iter.old_spte)) {
604		− tdp_mmu_set_spte(vcpu->kvm, &iter, 0);
605		−
606		− kvm_flush_remote_tlbs_with_address(vcpu->kvm, iter.gfn,
607		− KVM_PAGES_PER_HPAGE(iter.level));
	838	+ if (!tdp_mmu_zap_spte_atomic(vcpu->kvm, &iter))
	839	+ break;
608	840
609	841	/*
610	842	* The iter must explicitly re-read the spte here
611	843	* because the new value informs the !present
612	844	* path below.
613	845	*/
614		− iter.old_spte = READ_ONCE(*iter.sptep);
	846	+ iter.old_spte = READ_ONCE(*rcu_dereference(iter.sptep));
615	847	}
616	848
617	849	if (!is_shadow_present_pte(iter.old_spte)) {
618	850	sp = alloc_tdp_mmu_page(vcpu, iter.gfn, iter.level);
619		− list_add(&sp->link, &vcpu->kvm->arch.tdp_mmu_pages);
620	851	child_pt = sp->spt;
621		− clear_page(child_pt);
	852	+
622	853	new_spte = make_nonleaf_spte(child_pt,
623	854	!shadow_accessed_mask);
624	855
625		− trace_kvm_mmu_get_page(sp, true);
626		− if (huge_page_disallowed && req_level >= iter.level)
627		− account_huge_nx_page(vcpu->kvm, sp);
628		−
629		− tdp_mmu_set_spte(vcpu->kvm, &iter, new_spte);
	856	+ if (tdp_mmu_set_spte_atomic(vcpu->kvm, &iter,
	857	+ new_spte)) {
	858	+ tdp_mmu_link_page(vcpu->kvm, sp, true,
	859	+ huge_page_disallowed &&
	860	+ req_level >= iter.level);
	861	+
	862	+ trace_kvm_mmu_get_page(sp, true);
	863	+ } else {
	864	+ tdp_mmu_free_sp(sp);
	865	+ break;
	866	+ }
630	867	}
631	868	}
632	869
633		− if (WARN_ON(iter.level != level))
	870	+ if (iter.level != level) {
	871	+ rcu_read_unlock();
634	872	return RET_PF_RETRY;
	873	+ }
635	874
636	875	ret = tdp_mmu_map_handle_target_level(vcpu, write, map_writable, &iter,
637	876	pfn, prefault);
	877	+ rcu_read_unlock();
638	878
639	879	return ret;
640	880	}
641	881
642		−static int kvm_tdp_mmu_handle_hva_range(struct kvm *kvm, unsigned long start,
643		− unsigned long end, unsigned long data,
644		− int (handler)(struct kvm kvm, struct kvm_memory_slot *slot,
645		− struct kvm_mmu_page *root, gfn_t start,
646		− gfn_t end, unsigned long data))
	882	+static __always_inline int
	883	+kvm_tdp_mmu_handle_hva_range(struct kvm *kvm,
	884	+ unsigned long start,
	885	+ unsigned long end,
	886	+ unsigned long data,
	887	+ int (handler)(struct kvm kvm,
	888	+ struct kvm_memory_slot *slot,
	889	+ struct kvm_mmu_page *root,
	890	+ gfn_t start,
	891	+ gfn_t end,
	892	+ unsigned long data))
647	893	{
648	894	struct kvm_memslots *slots;
649	895	struct kvm_memory_slot *memslot;
@@ -705,6 +951,8 @@ static int age_gfn_range(struct kvm kvm, struct kvm_memory_slot slot, · 55 unmodified lines
705	951	int young = 0;
706	952	u64 new_spte = 0;
707	953
	954	+ rcu_read_lock();
	955	+
708	956	tdp_root_for_each_leaf_pte(iter, root, start, end) {
709	957	/*
710	958	* If we have a non-accessed entry we don't need to change the
@@ -736,6 +984,8 @@ static int age_gfn_range(struct kvm kvm, struct kvm_memory_slot slot, · 25 unmodified lines
736	984	trace_kvm_age_page(iter.gfn, iter.level, slot, young);
737	985	}
738	986
	987	+ rcu_read_unlock();
	988	+
739	989	return young;
740	990	}
741	991
@@ -781,6 +1031,8 @@ static int set_tdp_spte(struct kvm kvm, struct kvm_memory_slot slot, · 39 unmodified lines
781	1031	u64 new_spte;
782	1032	int need_flush = 0;
783	1033
	1034	+ rcu_read_lock();
	1035	+
784	1036	WARN_ON(pte_huge(*ptep));
785	1037
786	1038	new_pfn = pte_pfn(*ptep);
@@ -809,6 +1061,8 @@ static int set_tdp_spte(struct kvm kvm, struct kvm_memory_slot slot, · 22 unmodified lines
809	1061	if (need_flush)
810	1062	kvm_flush_remote_tlbs_with_address(kvm, gfn, 1);
811	1063
	1064	+ rcu_read_unlock();
	1065	+
812	1066	return 0;
813	1067	}
814	1068
@@ -832,21 +1086,27 @@ static bool wrprot_gfn_range(struct kvm kvm, struct kvm_mmu_page root, · 17 unmodified lines
832	1086	u64 new_spte;
833	1087	bool spte_set = false;
834	1088
	1089	+ rcu_read_lock();
	1090	+
835	1091	BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL);
836	1092
837	1093	for_each_tdp_pte_min_level(iter, root->spt, root->role.level,
838	1094	min_level, start, end) {
	1095	+ if (tdp_mmu_iter_cond_resched(kvm, &iter, false))
	1096	+ continue;
	1097	+
839	1098	if (!is_shadow_present_pte(iter.old_spte) \|\|
840		− !is_last_spte(iter.old_spte, iter.level))
	1099	+ !is_last_spte(iter.old_spte, iter.level) \|\|
	1100	+ !(iter.old_spte & PT_WRITABLE_MASK))
841	1101	continue;
842	1102
843	1103	new_spte = iter.old_spte & ~PT_WRITABLE_MASK;
844	1104
845	1105	tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte);
846	1106	spte_set = true;
847		−
848		− tdp_mmu_iter_cond_resched(kvm, &iter);
849	1107	}
	1108	+
	1109	+ rcu_read_unlock();
850	1110	return spte_set;
851	1111	}
852	1112
@@ -888,7 +1148,12 @@ static bool clear_dirty_gfn_range(struct kvm kvm, struct kvm_mmu_page root, · 35 unmodified lines
888	1148	u64 new_spte;
889	1149	bool spte_set = false;
890	1150
	1151	+ rcu_read_lock();
	1152	+
891	1153	tdp_root_for_each_leaf_pte(iter, root, start, end) {
	1154	+ if (tdp_mmu_iter_cond_resched(kvm, &iter, false))
	1155	+ continue;
	1156	+
892	1157	if (spte_ad_need_write_protect(iter.old_spte)) {
893	1158	if (is_writable_pte(iter.old_spte))
894	1159	new_spte = iter.old_spte & ~PT_WRITABLE_MASK;
@@ -903,9 +1168,9 @@ static bool clear_dirty_gfn_range(struct kvm kvm, struct kvm_mmu_page root, · 8 unmodified lines
903	1168
904	1169	tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte);
905	1170	spte_set = true;
906		−
907		− tdp_mmu_iter_cond_resched(kvm, &iter);
908	1171	}
	1172	+
	1173	+ rcu_read_unlock();
909	1174	return spte_set;
910	1175	}
911	1176
@@ -947,6 +1212,8 @@ static void clear_dirty_pt_masked(struct kvm kvm, struct kvm_mmu_page root, · 35 unmodified lines
947	1212	struct tdp_iter iter;
948	1213	u64 new_spte;
949	1214
	1215	+ rcu_read_lock();
	1216	+
950	1217	tdp_root_for_each_leaf_pte(iter, root, gfn + __ffs(mask),
951	1218	gfn + BITS_PER_LONG) {
952	1219	if (!mask)
@@ -956,6 +1223,8 @@ static void clear_dirty_pt_masked(struct kvm kvm, struct kvm_mmu_page root, · 3 unmodified lines
956	1223	!(mask & (1UL << (iter.gfn - gfn))))
957	1224	continue;
958	1225
	1226	+ mask &= ~(1UL << (iter.gfn - gfn));
	1227	+
959	1228	if (wrprot \|\| spte_ad_need_write_protect(iter.old_spte)) {
960	1229	if (is_writable_pte(iter.old_spte))
961	1230	new_spte = iter.old_spte & ~PT_WRITABLE_MASK;
@@ -969,9 +1238,9 @@ static void clear_dirty_pt_masked(struct kvm kvm, struct kvm_mmu_page root, · 7 unmodified lines
969	1238	}
970	1239
971	1240	tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte);
972		−
973		− mask &= ~(1UL << (iter.gfn - gfn));
974	1241	}
	1242	+
	1243	+ rcu_read_unlock();
975	1244	}
976	1245
977	1246	/*
@@ -989,7 +1258,7 @@ void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm, · 11 unmodified lines
989	1258	struct kvm_mmu_page *root;
990	1259	int root_as_id;
991	1260
992		− lockdep_assert_held(&kvm->mmu_lock);
	1261	+ lockdep_assert_held_write(&kvm->mmu_lock);
993	1262	for_each_tdp_mmu_root(kvm, root) {
994	1263	root_as_id = kvm_mmu_page_as_id(root);
995	1264	if (root_as_id != slot->as_id)
@@ -1011,18 +1280,23 @@ static bool set_dirty_gfn_range(struct kvm kvm, struct kvm_mmu_page root, · 15 unmodified lines
1011	1280	u64 new_spte;
1012	1281	bool spte_set = false;
1013	1282
	1283	+ rcu_read_lock();
	1284	+
1014	1285	tdp_root_for_each_pte(iter, root, start, end) {
1015		− if (!is_shadow_present_pte(iter.old_spte))
	1286	+ if (tdp_mmu_iter_cond_resched(kvm, &iter, false))
	1287	+ continue;
	1288	+
	1289	+ if (!is_shadow_present_pte(iter.old_spte) \|\|
	1290	+ iter.old_spte & shadow_dirty_mask)
1016	1291	continue;
1017	1292
1018	1293	new_spte = iter.old_spte \| shadow_dirty_mask;
1019	1294
1020	1295	tdp_mmu_set_spte(kvm, &iter, new_spte);
1021	1296	spte_set = true;
1022		−
1023		− tdp_mmu_iter_cond_resched(kvm, &iter);
1024	1297	}
1025	1298
	1299	+ rcu_read_unlock();
1026	1300	return spte_set;
1027	1301	}
1028	1302
@@ -1060,7 +1334,14 @@ static void zap_collapsible_spte_range(struct kvm *kvm, · 31 unmodified lines
1060	1334	kvm_pfn_t pfn;
1061	1335	bool spte_set = false;
1062	1336
	1337	+ rcu_read_lock();
	1338	+
1063	1339	tdp_root_for_each_pte(iter, root, start, end) {
	1340	+ if (tdp_mmu_iter_cond_resched(kvm, &iter, spte_set)) {
	1341	+ spte_set = false;
	1342	+ continue;
	1343	+ }
	1344	+
1064	1345	if (!is_shadow_present_pte(iter.old_spte) \|\|
1065	1346	!is_last_spte(iter.old_spte, iter.level))
1066	1347	continue;
@@ -1072,9 +1353,10 @@ static void zap_collapsible_spte_range(struct kvm *kvm, · 5 unmodified lines
1072	1353
1073	1354	tdp_mmu_set_spte(kvm, &iter, 0);
1074	1355
1075		− spte_set = tdp_mmu_iter_flush_cond_resched(kvm, &iter);
	1356	+ spte_set = true;
1076	1357	}
1077	1358
	1359	+ rcu_read_unlock();
1078	1360	if (spte_set)
1079	1361	kvm_flush_remote_tlbs(kvm);
1080	1362	}
@@ -1111,6 +1393,8 @@ static bool write_protect_gfn(struct kvm kvm, struct kvm_mmu_page root, · 30 unmodified lines
1111	1393	u64 new_spte;
1112	1394	bool spte_set = false;
1113	1395
	1396	+ rcu_read_lock();
	1397	+
1114	1398	tdp_root_for_each_leaf_pte(iter, root, gfn, gfn + 1) {
1115	1399	if (!is_writable_pte(iter.old_spte))
1116	1400	break;
@@ -1122,6 +1406,8 @@ static bool write_protect_gfn(struct kvm kvm, struct kvm_mmu_page root, · 5 unmodified lines
1122	1406	spte_set = true;
1123	1407	}
1124	1408
	1409	+ rcu_read_unlock();
	1410	+
1125	1411	return spte_set;
1126	1412	}
1127	1413
@@ -1137,7 +1423,7 @@ bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm, · 9 unmodified lines
1137	1423	int root_as_id;
1138	1424	bool spte_set = false;
1139	1425
1140		− lockdep_assert_held(&kvm->mmu_lock);
	1426	+ lockdep_assert_held_write(&kvm->mmu_lock);
1141	1427	for_each_tdp_mmu_root(kvm, root) {
1142	1428	root_as_id = kvm_mmu_page_as_id(root);
1143	1429	if (root_as_id != slot->as_id)
@@ -1162,10 +1448,14 @@ int kvm_tdp_mmu_get_walk(struct kvm_vcpu vcpu, u64 addr, u64 sptes, · 18 unmodified lines
1162	1448
1163	1449	*root_level = vcpu->arch.mmu->shadow_root_level;
1164	1450
	1451	+ rcu_read_lock();
	1452	+
1165	1453	tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) {
1166	1454	leaf = iter.level;
1167	1455	sptes[leaf] = iter.old_spte;
1168	1456	}
1169	1457
	1458	+ rcu_read_unlock();
	1459	+
1170	1460	return leaf;
1171	1461	}

arch/x86/kvm/mmu/tdp_mmu.h+28−4

@@ -5,10 +5,6 @@ · 4 unmodified lines
5	5
6	6	#include <linux/kvm_host.h>
7	7
8		−void kvm_mmu_init_tdp_mmu(struct kvm *kvm);
9		−void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm);
10		−
11		−bool is_tdp_mmu_root(struct kvm *kvm, hpa_t root);
12	8	hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu);
13	9	void kvm_tdp_mmu_free_root(struct kvm kvm, struct kvm_mmu_page root);
14	10
@@ -47,4 +43,32 @@ bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm, · 32 unmodified lines
47	43	int kvm_tdp_mmu_get_walk(struct kvm_vcpu vcpu, u64 addr, u64 sptes,
48	44	int *root_level);
49	45
	46	+#ifdef CONFIG_X86_64
	47	+void kvm_mmu_init_tdp_mmu(struct kvm *kvm);
	48	+void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm);
	49	+static inline bool is_tdp_mmu_enabled(struct kvm *kvm) { return kvm->arch.tdp_mmu_enabled; }
	50	+static inline bool is_tdp_mmu_page(struct kvm_mmu_page *sp) { return sp->tdp_mmu_page; }
	51	+#else
	52	+static inline void kvm_mmu_init_tdp_mmu(struct kvm *kvm) {}
	53	+static inline void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) {}
	54	+static inline bool is_tdp_mmu_enabled(struct kvm *kvm) { return false; }
	55	+static inline bool is_tdp_mmu_page(struct kvm_mmu_page *sp) { return false; }
	56	+#endif
	57	+
	58	+static inline bool is_tdp_mmu_root(struct kvm *kvm, hpa_t hpa)
	59	+{
	60	+ struct kvm_mmu_page *sp;
	61	+
	62	+ if (!is_tdp_mmu_enabled(kvm))
	63	+ return false;
	64	+ if (WARN_ON(!VALID_PAGE(hpa)))
	65	+ return false;
	66	+
	67	+ sp = to_shadow_page(hpa);
	68	+ if (WARN_ON(!sp))
	69	+ return false;
	70	+
	71	+ return is_tdp_mmu_page(sp) && sp->root_count;
	72	+}
	73	+
50	74	#endif /* __KVM_X86_MMU_TDP_MMU_H */

arch/x86/kvm/mtrr.c+6−6

@@ -75,7 +75,7 @@ bool kvm_mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data) · 74 unmodified lines
75	75	/* variable MTRRs */
76	76	WARN_ON(!(msr >= 0x200 && msr < 0x200 + 2 * KVM_NR_VAR_MTRR));
77	77
78		− mask = (~0ULL) << cpuid_maxphyaddr(vcpu);
	78	+ mask = kvm_vcpu_reserved_gpa_bits_raw(vcpu);
79	79	if ((msr & 1) == 0) {
80	80	/* MTRR base */
81	81	if (!valid_mtrr_type(data & 0xff))
@@ -351,14 +351,14 @@ static void set_var_mtrr_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data) · 269 unmodified lines
351	351	if (var_mtrr_range_is_valid(cur))
352	352	list_del(&mtrr_state->var_ranges[index].node);
353	353
354		− /* Extend the mask with all 1 bits to the left, since those
355		− * bits must implicitly be 0. The bits are then cleared
356		− * when reading them.
	354	+ /*
	355	+ * Set all illegal GPA bits in the mask, since those bits must
	356	+ * implicitly be 0. The bits are then cleared when reading them.
357	357	*/
358	358	if (!is_mtrr_mask)
359	359	cur->base = data;
360	360	else
361		− cur->mask = data \| (-1LL << cpuid_maxphyaddr(vcpu));
	361	+ cur->mask = data \| kvm_vcpu_reserved_gpa_bits_raw(vcpu);
362	362
363	363	/* add it to the list if it's enabled. */
364	364	if (var_mtrr_range_is_valid(cur)) {
@@ -426,7 +426,7 @@ int kvm_mtrr_get_msr(struct kvm_vcpu vcpu, u32 msr, u64 pdata) · 61 unmodified lines
426	426	else
427	427	*pdata = vcpu->arch.mtrr_state.var_ranges[index].mask;
428	428
429		− *pdata &= (1ULL << cpuid_maxphyaddr(vcpu)) - 1;
	429	+ *pdata &= ~kvm_vcpu_reserved_gpa_bits_raw(vcpu);
430	430	}
431	431
432	432	return 0;

arch/x86/kvm/pmu.c+8−2

@@ -373,7 +373,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu vcpu, unsigned idx, u64 data) · 372 unmodified lines
373	373	return 1;
374	374
375	375	if (!(kvm_read_cr4(vcpu) & X86_CR4_PCE) &&
376		− (kvm_x86_ops.get_cpl(vcpu) != 0) &&
	376	+ (static_call(kvm_x86_get_cpl)(vcpu) != 0) &&
377	377	(kvm_read_cr0(vcpu) & X86_CR0_PE))
378	378	return 1;
379	379
@@ -383,8 +383,11 @@ int kvm_pmu_rdpmc(struct kvm_vcpu vcpu, unsigned idx, u64 data) · 3 unmodified lines
383	383
384	384	void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
385	385	{
386		− if (lapic_in_kernel(vcpu))
	386	+ if (lapic_in_kernel(vcpu)) {
	387	+ if (kvm_x86_ops.pmu_ops->deliver_pmi)
	388	+ kvm_x86_ops.pmu_ops->deliver_pmi(vcpu);
387	389	kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC);
	390	+ }
388	391	}
389	392
390	393	bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
@@ -473,6 +476,9 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu) · 82 unmodified lines
473	476	pmc_stop_counter(pmc);
474	477	}
475	478
	479	+ if (kvm_x86_ops.pmu_ops->cleanup)
	480	+ kvm_x86_ops.pmu_ops->cleanup(vcpu);
	481	+
476	482	bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX);
477	483	}
478	484

arch/x86/kvm/pmu.h+2−0

@@ -39,6 +39,8 @@ struct kvm_pmu_ops { · 38 unmodified lines
39	39	void (refresh)(struct kvm_vcpu vcpu);
40	40	void (init)(struct kvm_vcpu vcpu);
41	41	void (reset)(struct kvm_vcpu vcpu);
	42	+ void (deliver_pmi)(struct kvm_vcpu vcpu);
	43	+ void (cleanup)(struct kvm_vcpu vcpu);
42	44	};
43	45
44	46	static inline u64 pmc_bitmask(struct kvm_pmc *pmc)

arch/x86/kvm/svm/avic.c+19−16

@@ -298,6 +298,23 @@ static int avic_init_backing_page(struct kvm_vcpu *vcpu) · 297 unmodified lines
298	298	return 0;
299	299	}
300	300
	301	+static void avic_kick_target_vcpus(struct kvm kvm, struct kvm_lapic source,
	302	+ u32 icrl, u32 icrh)
	303	+{
	304	+ struct kvm_vcpu *vcpu;
	305	+ int i;
	306	+
	307	+ kvm_for_each_vcpu(i, vcpu, kvm) {
	308	+ bool m = kvm_apic_match_dest(vcpu, source,
	309	+ icrl & APIC_SHORT_MASK,
	310	+ GET_APIC_DEST_FIELD(icrh),
	311	+ icrl & APIC_DEST_MASK);
	312	+
	313	+ if (m && !avic_vcpu_is_running(vcpu))
	314	+ kvm_vcpu_wake_up(vcpu);
	315	+ }
	316	+}
	317	+
301	318	int avic_incomplete_ipi_interception(struct vcpu_svm *svm)
302	319	{
303	320	u32 icrh = svm->vmcb->control.exit_info_1 >> 32;
@@ -324,28 +341,14 @@ int avic_incomplete_ipi_interception(struct vcpu_svm *svm) · 20 unmodified lines
324	341	kvm_lapic_reg_write(apic, APIC_ICR2, icrh);
325	342	kvm_lapic_reg_write(apic, APIC_ICR, icrl);
326	343	break;
327		− case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: {
328		− int i;
329		− struct kvm_vcpu *vcpu;
330		− struct kvm *kvm = svm->vcpu.kvm;
331		− struct kvm_lapic *apic = svm->vcpu.arch.apic;
332		−
	344	+ case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING:
333	345	/*
334	346	* At this point, we expect that the AVIC HW has already
335	347	* set the appropriate IRR bits on the valid target
336	348	* vcpus. So, we just need to kick the appropriate vcpu.
337	349	*/
338		− kvm_for_each_vcpu(i, vcpu, kvm) {
339		− bool m = kvm_apic_match_dest(vcpu, apic,
340		− icrl & APIC_SHORT_MASK,
341		− GET_APIC_DEST_FIELD(icrh),
342		− icrl & APIC_DEST_MASK);
343		−
344		− if (m && !avic_vcpu_is_running(vcpu))
345		− kvm_vcpu_wake_up(vcpu);
346		− }
	350	+ avic_kick_target_vcpus(svm->vcpu.kvm, apic, icrl, icrh);
347	351	break;
348		− }
349	352	case AVIC_IPI_FAILURE_INVALID_TARGET:
350	353	WARN_ONCE(1, "Invalid IPI target: index=%u, vcpu=%d, icr=%#0x:%#0x\n",
351	354	index, svm->vcpu.vcpu_id, icrh, icrl);

arch/x86/kvm/svm/nested.c+4−4

@@ -58,7 +58,7 @@ static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) · 57 unmodified lines
58	58	u64 pdpte;
59	59	int ret;
60	60
61		− ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(__sme_clr(cr3)), &pdpte,
	61	+ ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte,
62	62	offset_in_page(cr3) + index * 8, 8);
63	63	if (ret)
64	64	return 0;
@@ -248,7 +248,7 @@ static bool nested_vmcb_checks(struct vcpu_svm svm, struct vmcb vmcb12) · 183 unmodified lines
248	248	if (vmcb12_lma) {
249	249	if (!(vmcb12->save.cr4 & X86_CR4_PAE) \|\|
250	250	!(vmcb12->save.cr0 & X86_CR0_PE) \|\|
251		− (vmcb12->save.cr3 & vcpu->arch.cr3_lm_rsvd_bits))
	251	+ kvm_vcpu_is_illegal_gpa(vcpu, vmcb12->save.cr3))
252	252	return false;
253	253	}
254	254	if (!kvm_is_valid_cr4(&svm->vcpu, vmcb12->save.cr4))
@@ -345,7 +345,7 @@ static inline bool nested_npt_enabled(struct vcpu_svm *svm) · 90 unmodified lines
345	345	static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3,
346	346	bool nested_npt)
347	347	{
348		− if (cr3 & rsvd_bits(cpuid_maxphyaddr(vcpu), 63))
	348	+ if (kvm_vcpu_is_illegal_gpa(vcpu, cr3))
349	349	return -EINVAL;
350	350
351	351	if (!nested_npt && is_pae_paging(vcpu) &&
@@ -392,7 +392,7 @@ static void nested_prepare_vmcb_save(struct vcpu_svm svm, struct vmcb vmcb12) · 40 unmodified lines
392	392	svm->vmcb->save.rsp = vmcb12->save.rsp;
393	393	svm->vmcb->save.rip = vmcb12->save.rip;
394	394	svm->vmcb->save.dr7 = vmcb12->save.dr7 \| DR7_FIXED_1;
395		− svm->vcpu.arch.dr6 = vmcb12->save.dr6 \| DR6_FIXED_1 \| DR6_RTM;
	395	+ svm->vcpu.arch.dr6 = vmcb12->save.dr6 \| DR6_ACTIVE_LOW;
396	396	svm->vmcb->save.cpl = vmcb12->save.cpl;
397	397	}
398	398

arch/x86/kvm/svm/sev.c+74−30

@@ -22,6 +22,7 @@ · 21 unmodified lines
22	22
23	23	#include "x86.h"
24	24	#include "svm.h"
	25	+#include "svm_ops.h"
25	26	#include "cpuid.h"
26	27	#include "trace.h"
27	28
@@ -1041,6 +1042,74 @@ e_unpin_memory: · 1013 unmodified lines
1041	1042	return ret;
1042	1043	}
1043	1044
	1045	+static int sev_get_attestation_report(struct kvm kvm, struct kvm_sev_cmd argp)
	1046	+{
	1047	+ void __user report = (void __user )(uintptr_t)argp->data;
	1048	+ struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
	1049	+ struct sev_data_attestation_report *data;
	1050	+ struct kvm_sev_attestation_report params;
	1051	+ void __user *p;
	1052	+ void *blob = NULL;
	1053	+ int ret;
	1054	+
	1055	+ if (!sev_guest(kvm))
	1056	+ return -ENOTTY;
	1057	+
	1058	+ if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
	1059	+ return -EFAULT;
	1060	+
	1061	+ data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
	1062	+ if (!data)
	1063	+ return -ENOMEM;
	1064	+
	1065	+ /* User wants to query the blob length */
	1066	+ if (!params.len)
	1067	+ goto cmd;
	1068	+
	1069	+ p = (void __user *)(uintptr_t)params.uaddr;
	1070	+ if (p) {
	1071	+ if (params.len > SEV_FW_BLOB_MAX_SIZE) {
	1072	+ ret = -EINVAL;
	1073	+ goto e_free;
	1074	+ }
	1075	+
	1076	+ ret = -ENOMEM;
	1077	+ blob = kmalloc(params.len, GFP_KERNEL);
	1078	+ if (!blob)
	1079	+ goto e_free;
	1080	+
	1081	+ data->address = __psp_pa(blob);
	1082	+ data->len = params.len;
	1083	+ memcpy(data->mnonce, params.mnonce, sizeof(params.mnonce));
	1084	+ }
	1085	+cmd:
	1086	+ data->handle = sev->handle;
	1087	+ ret = sev_issue_cmd(kvm, SEV_CMD_ATTESTATION_REPORT, data, &argp->error);
	1088	+ /*
	1089	+ * If we query the session length, FW responded with expected data.
	1090	+ */
	1091	+ if (!params.len)
	1092	+ goto done;
	1093	+
	1094	+ if (ret)
	1095	+ goto e_free_blob;
	1096	+
	1097	+ if (blob) {
	1098	+ if (copy_to_user(p, blob, params.len))
	1099	+ ret = -EFAULT;
	1100	+ }
	1101	+
	1102	+done:
	1103	+ params.len = data->len;
	1104	+ if (copy_to_user(report, &params, sizeof(params)))
	1105	+ ret = -EFAULT;
	1106	+e_free_blob:
	1107	+ kfree(blob);
	1108	+e_free:
	1109	+ kfree(data);
	1110	+ return ret;
	1111	+}
	1112	+
1044	1113	int svm_mem_enc_op(struct kvm kvm, void __user argp)
1045	1114	{
1046	1115	struct kvm_sev_cmd sev_cmd;
@@ -1091,6 +1160,9 @@ int svm_mem_enc_op(struct kvm kvm, void __user argp) · 44 unmodified lines
1091	1160	case KVM_SEV_LAUNCH_SECRET:
1092	1161	r = sev_launch_secret(kvm, &sev_cmd);
1093	1162	break;
	1163	+ case KVM_SEV_GET_ATTESTATION_REPORT:
	1164	+ r = sev_get_attestation_report(kvm, &sev_cmd);
	1165	+ break;
1094	1166	default:
1095	1167	r = -EINVAL;
1096	1168	goto out;
@@ -1994,29 +2066,17 @@ void sev_es_create_vcpu(struct vcpu_svm *svm) · 897 unmodified lines
1994	2066	sev_enc_bit));
1995	2067	}
1996	2068
1997		−void sev_es_vcpu_load(struct vcpu_svm *svm, int cpu)
	2069	+void sev_es_prepare_guest_switch(struct vcpu_svm *svm, unsigned int cpu)
1998	2070	{
1999	2071	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
2000	2072	struct vmcb_save_area *hostsa;
2001		− unsigned int i;
2002	2073
2003	2074	/*
2004	2075	* As an SEV-ES guest, hardware will restore the host state on VMEXIT,
2005	2076	* of which one step is to perform a VMLOAD. Since hardware does not
2006	2077	* perform a VMSAVE on VMRUN, the host savearea must be updated.
2007	2078	*/
2008		− asm volatile(__ex("vmsave %0") : : "a" (__sme_page_pa(sd->save_area)) : "memory");
2009		−
2010		− /*
2011		− * Certain MSRs are restored on VMEXIT, only save ones that aren't
2012		− * restored.
2013		− */
2014		− for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) {
2015		− if (host_save_user_msrs[i].sev_es_restored)
2016		− continue;
2017		−
2018		− rdmsrl(host_save_user_msrs[i].index, svm->host_user_msrs[i]);
2019		− }
	2079	+ vmsave(__sme_page_pa(sd->save_area));
2020	2080
2021	2081	/* XCR0 is restored on VMEXIT, save the current host value */
2022	2082	hostsa = (struct vmcb_save_area *)(page_address(sd->save_area) + 0x400);
@@ -2029,22 +2089,6 @@ void sev_es_vcpu_load(struct vcpu_svm *svm, int cpu) · 6 unmodified lines
2029	2089	hostsa->xss = host_xss;
2030	2090	}
2031	2091
2032		−void sev_es_vcpu_put(struct vcpu_svm *svm)
2033		−{
2034		− unsigned int i;
2035		−
2036		− /*
2037		− * Certain MSRs are restored on VMEXIT and were saved with vmsave in
2038		− * sev_es_vcpu_load() above. Only restore ones that weren't.
2039		− */
2040		− for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) {
2041		− if (host_save_user_msrs[i].sev_es_restored)
2042		− continue;
2043		−
2044		− wrmsrl(host_save_user_msrs[i].index, svm->host_user_msrs[i]);
2045		− }
2046		−}
2047		−
2048	2092	void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
2049	2093	{
2050	2094	struct vcpu_svm *svm = to_svm(vcpu);

arch/x86/kvm/svm/svm.c+189−114

@@ -41,6 +41,7 @@ · 40 unmodified lines
41	41	#include "trace.h"
42	42
43	43	#include "svm.h"
	44	+#include "svm_ops.h"
44	45
45	46	#define __ex(x) __kvm_handle_fault_on_reboot(x)
46	47
@@ -200,9 +201,9 @@ module_param(sev_es, int, 0444); · 153 unmodified lines
200	201	bool __read_mostly dump_invalid_vmcb;
201	202	module_param(dump_invalid_vmcb, bool, 0644);
202	203
203		−static u8 rsm_ins_bytes[] = "\x0f\xaa";
	204	+static bool svm_gp_erratum_intercept = true;
204	205
205		−static void svm_complete_interrupts(struct vcpu_svm *svm);
	206	+static u8 rsm_ins_bytes[] = "\x0f\xaa";
206	207
207	208	static unsigned long iopm_base;
208	209
@@ -246,21 +247,6 @@ u32 svm_msrpm_offset(u32 msr) · 37 unmodified lines
246	247
247	248	#define MAX_INST_SIZE 15
248	249
249		−static inline void clgi(void)
250		−{
251		− asm volatile (__ex("clgi"));
252		−}
253		−
254		−static inline void stgi(void)
255		−{
256		− asm volatile (__ex("stgi"));
257		−}
258		−
259		−static inline void invlpga(unsigned long addr, u32 asid)
260		−{
261		− asm volatile (__ex("invlpga %1, %0") : : "c"(asid), "a"(addr));
262		−}
263		−
264	250	static int get_max_npt_level(void)
265	251	{
266	252	#ifdef CONFIG_X86_64
@@ -288,6 +274,9 @@ int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) · 21 unmodified lines
288	274	if (!(efer & EFER_SVME)) {
289	275	svm_leave_nested(svm);
290	276	svm_set_gif(svm, true);
	277	+ /* #GP intercept is still needed for vmware backdoor */
	278	+ if (!enable_vmware_backdoor)
	279	+ clr_exception_intercept(svm, GP_VECTOR);
291	280
292	281	/*
293	282	* Free the nested guest state, unless we are in SMM.
@@ -304,6 +293,9 @@ int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) · 10 unmodified lines
304	293	vcpu->arch.efer = old_efer;
305	294	return ret;
306	295	}
	296	+
	297	+ if (svm_gp_erratum_intercept)
	298	+ set_exception_intercept(svm, GP_VECTOR);
307	299	}
308	300	}
309	301
@@ -925,6 +917,9 @@ static __init void svm_set_cpu_caps(void) · 615 unmodified lines
925	917
926	918	if (npt_enabled)
927	919	kvm_cpu_cap_set(X86_FEATURE_NPT);
	920	+
	921	+ /* Nested VM can receive #VMEXIT instead of triggering #GP */
	922	+ kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK);
928	923	}
929	924
930	925	/* CPUID 0x80000008 */
@@ -1032,6 +1027,9 @@ static __init int svm_hardware_setup(void) · 101 unmodified lines
1032	1027	}
1033	1028	}
1034	1029
	1030	+ if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK))
	1031	+ svm_gp_erratum_intercept = false;
	1032	+
1035	1033	if (vgif) {
1036	1034	if (!boot_cpu_has(X86_FEATURE_VGIF))
1037	1035	vgif = false;
@@ -1207,7 +1205,7 @@ static void init_vmcb(struct vcpu_svm *svm) · 169 unmodified lines
1207	1205
1208	1206	svm_set_efer(&svm->vcpu, 0);
1209	1207	save->dr6 = 0xffff0ff0;
1210		− kvm_set_rflags(&svm->vcpu, 2);
	1208	+ kvm_set_rflags(&svm->vcpu, X86_EFLAGS_FIXED);
1211	1209	save->rip = 0x0000fff0;
1212	1210	svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
1213	1211
@@ -1366,6 +1364,7 @@ static int svm_create_vcpu(struct kvm_vcpu *vcpu) · 152 unmodified lines
1366	1364	svm->vmsa = page_address(vmsa_page);
1367	1365
1368	1366	svm->asid_generation = 0;
	1367	+ svm->guest_state_loaded = false;
1369	1368	init_vmcb(svm);
1370	1369
1371	1370	svm_init_osvw(vcpu);
@@ -1413,30 +1412,31 @@ static void svm_free_vcpu(struct kvm_vcpu *vcpu) · 41 unmodified lines
1413	1412	__free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
1414	1413	}
1415	1414
1416		−static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
	1415	+static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
1417	1416	{
1418	1417	struct vcpu_svm *svm = to_svm(vcpu);
1419		− struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
1420		− int i;
	1418	+ struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
	1419	+ unsigned int i;
1421	1420
1422		− if (unlikely(cpu != vcpu->cpu)) {
1423		− svm->asid_generation = 0;
1424		− vmcb_mark_all_dirty(svm->vmcb);
1425		− }
	1421	+ if (svm->guest_state_loaded)
	1422	+ return;
1426	1423
	1424	+ /*
	1425	+ * Certain MSRs are restored on VMEXIT (sev-es), or vmload of host save
	1426	+ * area (non-sev-es). Save ones that aren't so we can restore them
	1427	+ * individually later.
	1428	+ */
	1429	+ for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
	1430	+ rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
	1431	+
	1432	+ /*
	1433	+ * Save additional host state that will be restored on VMEXIT (sev-es)
	1434	+ * or subsequent vmload of host save area.
	1435	+ */
1427	1436	if (sev_es_guest(svm->vcpu.kvm)) {
1428		− sev_es_vcpu_load(svm, cpu);
	1437	+ sev_es_prepare_guest_switch(svm, vcpu->cpu);
1429	1438	} else {
1430		−#ifdef CONFIG_X86_64
1431		− rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
1432		−#endif
1433		− savesegment(fs, svm->host.fs);
1434		− savesegment(gs, svm->host.gs);
1435		− svm->host.ldt = kvm_read_ldt();
1436		−
1437		− for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1438		− rdmsrl(host_save_user_msrs[i].index,
1439		− svm->host_user_msrs[i]);
	1439	+ vmsave(__sme_page_pa(sd->save_area));
1440	1440	}
1441	1441
1442	1442	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
@@ -1446,10 +1446,42 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) · 3 unmodified lines
1446	1446	wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
1447	1447	}
1448	1448	}
	1449	+
1449	1450	/* This assumes that the kernel never uses MSR_TSC_AUX */
1450	1451	if (static_cpu_has(X86_FEATURE_RDTSCP))
1451	1452	wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
1452	1453
	1454	+ svm->guest_state_loaded = true;
	1455	+}
	1456	+
	1457	+static void svm_prepare_host_switch(struct kvm_vcpu *vcpu)
	1458	+{
	1459	+ struct vcpu_svm *svm = to_svm(vcpu);
	1460	+ unsigned int i;
	1461	+
	1462	+ if (!svm->guest_state_loaded)
	1463	+ return;
	1464	+
	1465	+ /*
	1466	+ * Certain MSRs are restored on VMEXIT (sev-es), or vmload of host save
	1467	+ * area (non-sev-es). Restore the ones that weren't.
	1468	+ */
	1469	+ for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
	1470	+ wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
	1471	+
	1472	+ svm->guest_state_loaded = false;
	1473	+}
	1474	+
	1475	+static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
	1476	+{
	1477	+ struct vcpu_svm *svm = to_svm(vcpu);
	1478	+ struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
	1479	+
	1480	+ if (unlikely(cpu != vcpu->cpu)) {
	1481	+ svm->asid_generation = 0;
	1482	+ vmcb_mark_all_dirty(svm->vmcb);
	1483	+ }
	1484	+
1453	1485	if (sd->current_vmcb != svm->vmcb) {
1454	1486	sd->current_vmcb = svm->vmcb;
1455	1487	indirect_branch_prediction_barrier();
@@ -1459,30 +1491,10 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) · 3 unmodified lines
1459	1491
1460	1492	static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1461	1493	{
1462		− struct vcpu_svm *svm = to_svm(vcpu);
1463		− int i;
1464		−
1465	1494	avic_vcpu_put(vcpu);
	1495	+ svm_prepare_host_switch(vcpu);
1466	1496
1467	1497	++vcpu->stat.host_state_reload;
1468		− if (sev_es_guest(svm->vcpu.kvm)) {
1469		− sev_es_vcpu_put(svm);
1470		− } else {
1471		− kvm_load_ldt(svm->host.ldt);
1472		−#ifdef CONFIG_X86_64
1473		− loadsegment(fs, svm->host.fs);
1474		− wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gsbase);
1475		− load_gs_index(svm->host.gs);
1476		−#else
1477		−#ifdef CONFIG_X86_32_LAZY_GS
1478		− loadsegment(gs, svm->host.gs);
1479		−#endif
1480		−#endif
1481		−
1482		− for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1483		− wrmsrl(host_save_user_msrs[i].index,
1484		− svm->host_user_msrs[i]);
1485		− }
1486	1498	}
1487	1499
1488	1500	static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
@@ -1815,7 +1827,7 @@ static void svm_set_segment(struct kvm_vcpu *vcpu, · 326 unmodified lines
1815	1827	vmcb_mark_dirty(svm->vmcb, VMCB_SEG);
1816	1828	}
1817	1829
1818		−static void update_exception_bitmap(struct kvm_vcpu *vcpu)
	1830	+static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu)
1819	1831	{
1820	1832	struct vcpu_svm *svm = to_svm(vcpu);
1821	1833
@@ -1865,7 +1877,7 @@ static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) · 43 unmodified lines
1865	1877	get_debugreg(vcpu->arch.db[2], 2);
1866	1878	get_debugreg(vcpu->arch.db[3], 3);
1867	1879	/*
1868		− * We cannot reset svm->vmcb->save.dr6 to DR6_FIXED_1\|DR6_RTM here,
	1880	+ * We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here,
1869	1881	* because db_interception might need it. We can do it before vmentry.
1870	1882	*/
1871	1883	vcpu->arch.dr6 = svm->vmcb->save.dr6;
@@ -1916,7 +1928,7 @@ static int db_interception(struct vcpu_svm *svm) · 44 unmodified lines
1916	1928	if (!(svm->vcpu.guest_debug &
1917	1929	(KVM_GUESTDBG_SINGLESTEP \| KVM_GUESTDBG_USE_HW_BP)) &&
1918	1930	!svm->nmi_singlestep) {
1919		− u32 payload = (svm->vmcb->save.dr6 ^ DR6_RTM) & ~DR6_FIXED_1;
	1931	+ u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW;
1920	1932	kvm_queue_exception_p(&svm->vcpu, DB_VECTOR, payload);
1921	1933	return 1;
1922	1934	}
@@ -1962,24 +1974,6 @@ static int ac_interception(struct vcpu_svm *svm) · 39 unmodified lines
1962	1974	return 1;
1963	1975	}
1964	1976
1965		−static int gp_interception(struct vcpu_svm *svm)
1966		−{
1967		− struct kvm_vcpu *vcpu = &svm->vcpu;
1968		− u32 error_code = svm->vmcb->control.exit_info_1;
1969		−
1970		− WARN_ON_ONCE(!enable_vmware_backdoor);
1971		−
1972		− /*
1973		− * VMware backdoor emulation on #GP interception only handles IN{S},
1974		− * OUT{S}, and RDPMC, none of which generate a non-zero error code.
1975		− */
1976		− if (error_code) {
1977		− kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
1978		− return 1;
1979		− }
1980		− return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP);
1981		−}
1982		−
1983	1977	static bool is_erratum_383(void)
1984	1978	{
1985	1979	int err, i;
@@ -2178,6 +2172,102 @@ static int vmrun_interception(struct vcpu_svm *svm) · 192 unmodified lines
2178	2172	return nested_svm_vmrun(svm);
2179	2173	}
2180	2174
	2175	+enum {
	2176	+ NONE_SVM_INSTR,
	2177	+ SVM_INSTR_VMRUN,
	2178	+ SVM_INSTR_VMLOAD,
	2179	+ SVM_INSTR_VMSAVE,
	2180	+};
	2181	+
	2182	+/* Return NONE_SVM_INSTR if not SVM instrs, otherwise return decode result */
	2183	+static int svm_instr_opcode(struct kvm_vcpu *vcpu)
	2184	+{
	2185	+ struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
	2186	+
	2187	+ if (ctxt->b != 0x1 \|\| ctxt->opcode_len != 2)
	2188	+ return NONE_SVM_INSTR;
	2189	+
	2190	+ switch (ctxt->modrm) {
	2191	+ case 0xd8: /* VMRUN */
	2192	+ return SVM_INSTR_VMRUN;
	2193	+ case 0xda: /* VMLOAD */
	2194	+ return SVM_INSTR_VMLOAD;
	2195	+ case 0xdb: /* VMSAVE */
	2196	+ return SVM_INSTR_VMSAVE;
	2197	+ default:
	2198	+ break;
	2199	+ }
	2200	+
	2201	+ return NONE_SVM_INSTR;
	2202	+}
	2203	+
	2204	+static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode)
	2205	+{
	2206	+ const int guest_mode_exit_codes[] = {
	2207	+ [SVM_INSTR_VMRUN] = SVM_EXIT_VMRUN,
	2208	+ [SVM_INSTR_VMLOAD] = SVM_EXIT_VMLOAD,
	2209	+ [SVM_INSTR_VMSAVE] = SVM_EXIT_VMSAVE,
	2210	+ };
	2211	+ int (const svm_instr_handlers[])(struct vcpu_svm svm) = {
	2212	+ [SVM_INSTR_VMRUN] = vmrun_interception,
	2213	+ [SVM_INSTR_VMLOAD] = vmload_interception,
	2214	+ [SVM_INSTR_VMSAVE] = vmsave_interception,
	2215	+ };
	2216	+ struct vcpu_svm *svm = to_svm(vcpu);
	2217	+
	2218	+ if (is_guest_mode(vcpu)) {
	2219	+ svm->vmcb->control.exit_code = guest_mode_exit_codes[opcode];
	2220	+ svm->vmcb->control.exit_info_1 = 0;
	2221	+ svm->vmcb->control.exit_info_2 = 0;
	2222	+
	2223	+ return nested_svm_vmexit(svm);
	2224	+ } else
	2225	+ return svm_instr_handlers[opcode](svm);
	2226	+}
	2227	+
	2228	+/*
	2229	+ * #GP handling code. Note that #GP can be triggered under the following two
	2230	+ * cases:
	2231	+ * 1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on
	2232	+ * some AMD CPUs when EAX of these instructions are in the reserved memory
	2233	+ * regions (e.g. SMM memory on host).
	2234	+ * 2) VMware backdoor
	2235	+ */
	2236	+static int gp_interception(struct vcpu_svm *svm)
	2237	+{
	2238	+ struct kvm_vcpu *vcpu = &svm->vcpu;
	2239	+ u32 error_code = svm->vmcb->control.exit_info_1;
	2240	+ int opcode;
	2241	+
	2242	+ /* Both #GP cases have zero error_code */
	2243	+ if (error_code)
	2244	+ goto reinject;
	2245	+
	2246	+ /* Decode the instruction for usage later */
	2247	+ if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK)
	2248	+ goto reinject;
	2249	+
	2250	+ opcode = svm_instr_opcode(vcpu);
	2251	+
	2252	+ if (opcode == NONE_SVM_INSTR) {
	2253	+ if (!enable_vmware_backdoor)
	2254	+ goto reinject;
	2255	+
	2256	+ /*
	2257	+ * VMware backdoor emulation on #GP interception only handles
	2258	+ * IN{S}, OUT{S}, and RDPMC.
	2259	+ */
	2260	+ if (!is_guest_mode(vcpu))
	2261	+ return kvm_emulate_instruction(vcpu,
	2262	+ EMULTYPE_VMWARE_GP \| EMULTYPE_NO_DECODE);
	2263	+ } else
	2264	+ return emulate_svm_instr(vcpu, opcode);
	2265	+
	2266	+reinject:
	2267	+ kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
	2268	+ return 1;
	2269	+}
	2270	+
2181	2271	void svm_set_gif(struct vcpu_svm *svm, bool value)
2182	2272	{
2183	2273	if (value) {
@@ -2265,11 +2355,8 @@ static int xsetbv_interception(struct vcpu_svm *svm) · 81 unmodified lines
2265	2355	u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
2266	2356	u32 index = kvm_rcx_read(&svm->vcpu);
2267	2357
2268		− if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
2269		− return kvm_skip_emulated_instruction(&svm->vcpu);
2270		− }
2271		−
2272		− return 1;
	2358	+ int err = kvm_set_xcr(&svm->vcpu, index, new_bv);
	2359	+ return kvm_complete_insn_gp(&svm->vcpu, err);
2273	2360	}
2274	2361
2275	2362	static int rdpru_interception(struct vcpu_svm *svm)
@@ -2530,6 +2617,7 @@ static int dr_interception(struct vcpu_svm *svm) · 254 unmodified lines
2530	2617	{
2531	2618	int reg, dr;
2532	2619	unsigned long val;
	2620	+ int err = 0;
2533	2621
2534	2622	if (svm->vcpu.guest_debug == 0) {
2535	2623	/*
@@ -2547,20 +2635,16 @@ static int dr_interception(struct vcpu_svm *svm) · 11 unmodified lines
2547	2635
2548	2636	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2549	2637	dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
2550		−
2551		− if (dr >= 16) { /* mov to DRn */
2552		− if (!kvm_require_dr(&svm->vcpu, dr - 16))
2553		− return 1;
	2638	+ if (dr >= 16) { /* mov to DRn */
	2639	+ dr -= 16;
2554	2640	val = kvm_register_read(&svm->vcpu, reg);
2555		− kvm_set_dr(&svm->vcpu, dr - 16, val);
	2641	+ err = kvm_set_dr(&svm->vcpu, dr, val);
2556	2642	} else {
2557		− if (!kvm_require_dr(&svm->vcpu, dr))
2558		− return 1;
2559	2643	kvm_get_dr(&svm->vcpu, dr, &val);
2560	2644	kvm_register_write(&svm->vcpu, reg, val);
2561	2645	}
2562	2646
2563		− return kvm_skip_emulated_instruction(&svm->vcpu);
	2647	+ return kvm_complete_insn_gp(&svm->vcpu, err);
2564	2648	}
2565	2649
2566	2650	static int cr8_write_interception(struct vcpu_svm *svm)
@@ -3354,7 +3438,7 @@ static void svm_set_irq(struct kvm_vcpu *vcpu) · 787 unmodified lines
3354	3438	SVM_EVTINJ_VALID \| SVM_EVTINJ_TYPE_INTR;
3355	3439	}
3356	3440
3357		−static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
	3441	+static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3358	3442	{
3359	3443	struct vcpu_svm *svm = to_svm(vcpu);
3360	3444
@@ -3479,7 +3563,7 @@ static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) · 118 unmodified lines
3479	3563	return !svm_interrupt_blocked(vcpu);
3480	3564	}
3481	3565
3482		−static void enable_irq_window(struct kvm_vcpu *vcpu)
	3566	+static void svm_enable_irq_window(struct kvm_vcpu *vcpu)
3483	3567	{
3484	3568	struct vcpu_svm *svm = to_svm(vcpu);
3485	3569
@@ -3503,7 +3587,7 @@ static void enable_irq_window(struct kvm_vcpu *vcpu) · 17 unmodified lines
3503	3587	}
3504	3588	}
3505	3589
3506		−static void enable_nmi_window(struct kvm_vcpu *vcpu)
	3590	+static void svm_enable_nmi_window(struct kvm_vcpu *vcpu)
3507	3591	{
3508	3592	struct vcpu_svm *svm = to_svm(vcpu);
3509	3593
@@ -3560,10 +3644,6 @@ static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva) · 50 unmodified lines
3560	3644	invlpga(gva, svm->vmcb->control.asid);
3561	3645	}
3562	3646
3563		−static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
3564		−{
3565		−}
3566		−
3567	3647	static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
3568	3648	{
3569	3649	struct vcpu_svm *svm = to_svm(vcpu);
@@ -3708,16 +3788,11 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, · 138 unmodified lines
3708	3788	if (sev_es_guest(svm->vcpu.kvm)) {
3709	3789	__svm_sev_es_vcpu_run(svm->vmcb_pa);
3710	3790	} else {
	3791	+ struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
	3792	+
3711	3793	__svm_vcpu_run(svm->vmcb_pa, (unsigned long *)&svm->vcpu.arch.regs);
3712	3794
3713		−#ifdef CONFIG_X86_64
3714		− native_wrmsrl(MSR_GS_BASE, svm->host.gs_base);
3715		−#else
3716		− loadsegment(fs, svm->host.fs);
3717		−#ifndef CONFIG_X86_32_LAZY_GS
3718		− loadsegment(gs, svm->host.gs);
3719		−#endif
3720		−#endif
	3795	+ vmload(__sme_page_pa(sd->save_area));
3721	3796	}
3722	3797
3723	3798	/*
@@ -3783,7 +3858,7 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) · 59 unmodified lines
3783	3858	if (unlikely(svm->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))
3784	3859	svm_set_dr6(svm, vcpu->arch.dr6);
3785	3860	else
3786		− svm_set_dr6(svm, DR6_FIXED_1 \| DR6_RTM);
	3861	+ svm_set_dr6(svm, DR6_ACTIVE_LOW);
3787	3862
3788	3863	clgi();
3789	3864	kvm_load_guest_xsave_state(vcpu);
@@ -3978,7 +4053,7 @@ static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) · 188 unmodified lines
3978	4053	if (sev_guest(vcpu->kvm)) {
3979	4054	best = kvm_find_cpuid_entry(vcpu, 0x8000001F, 0);
3980	4055	if (best)
3981		− vcpu->arch.cr3_lm_rsvd_bits &= ~(1UL << (best->ebx & 0x3f));
	4056	+ vcpu->arch.reserved_gpa_bits &= ~(1UL << (best->ebx & 0x3f));
3982	4057	}
3983	4058
3984	4059	if (!kvm_vcpu_apicv_active(vcpu))
@@ -4285,7 +4360,7 @@ static int svm_pre_leave_smm(struct kvm_vcpu vcpu, const char smstate) · 300 unmodified lines
4285	4360	return ret;
4286	4361	}
4287	4362
4288		−static void enable_smi_window(struct kvm_vcpu *vcpu)
	4363	+static void svm_enable_smi_window(struct kvm_vcpu *vcpu)
4289	4364	{
4290	4365	struct vcpu_svm *svm = to_svm(vcpu);
4291	4366
@@ -4439,7 +4514,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { · 147 unmodified lines
4439	4514	.vcpu_blocking = svm_vcpu_blocking,
4440	4515	.vcpu_unblocking = svm_vcpu_unblocking,
4441	4516
4442		− .update_exception_bitmap = update_exception_bitmap,
	4517	+ .update_exception_bitmap = svm_update_exception_bitmap,
4443	4518	.get_msr_feature = svm_get_msr_feature,
4444	4519	.get_msr = svm_get_msr,
4445	4520	.set_msr = svm_set_msr,
@@ -4482,9 +4557,9 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { · 36 unmodified lines
4482	4557	.nmi_allowed = svm_nmi_allowed,
4483	4558	.get_nmi_mask = svm_get_nmi_mask,
4484	4559	.set_nmi_mask = svm_set_nmi_mask,
4485		− .enable_nmi_window = enable_nmi_window,
4486		− .enable_irq_window = enable_irq_window,
4487		− .update_cr8_intercept = update_cr8_intercept,
	4560	+ .enable_nmi_window = svm_enable_nmi_window,
	4561	+ .enable_irq_window = svm_enable_irq_window,
	4562	+ .update_cr8_intercept = svm_update_cr8_intercept,
4488	4563	.set_virtual_apic_mode = svm_set_virtual_apic_mode,
4489	4564	.refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
4490	4565	.check_apicv_inhibit_reasons = svm_check_apicv_inhibit_reasons,
@@ -4527,7 +4602,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { · 36 unmodified lines
4527	4602	.smi_allowed = svm_smi_allowed,
4528	4603	.pre_enter_smm = svm_pre_enter_smm,
4529	4604	.pre_leave_smm = svm_pre_leave_smm,
4530		− .enable_smi_window = enable_smi_window,
	4605	+ .enable_smi_window = svm_enable_smi_window,
4531	4606
4532	4607	.mem_enc_op = svm_mem_enc_op,
4533	4608	.mem_enc_reg_region = svm_register_enc_region,

arch/x86/kvm/svm/svm.h+5−24

@@ -23,22 +23,8 @@ · 22 unmodified lines
23	23
24	24	#define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT)
25	25
26		−static const struct svm_host_save_msrs {
27		− u32 index; /* Index of the MSR */
28		− bool sev_es_restored; /* True if MSR is restored on SEV-ES VMEXIT */
29		−} host_save_user_msrs[] = {
30		−#ifdef CONFIG_X86_64
31		− { .index = MSR_STAR, .sev_es_restored = true },
32		− { .index = MSR_LSTAR, .sev_es_restored = true },
33		− { .index = MSR_CSTAR, .sev_es_restored = true },
34		− { .index = MSR_SYSCALL_MASK, .sev_es_restored = true },
35		− { .index = MSR_KERNEL_GS_BASE, .sev_es_restored = true },
36		− { .index = MSR_FS_BASE, .sev_es_restored = true },
37		−#endif
38		− { .index = MSR_IA32_SYSENTER_CS, .sev_es_restored = true },
39		− { .index = MSR_IA32_SYSENTER_ESP, .sev_es_restored = true },
40		− { .index = MSR_IA32_SYSENTER_EIP, .sev_es_restored = true },
41		− { .index = MSR_TSC_AUX, .sev_es_restored = false },
	26	+static const u32 host_save_user_msrs[] = {
	27	+ MSR_TSC_AUX,
42	28	};
43	29	#define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
44	30
@@ -130,12 +116,6 @@ struct vcpu_svm { · 85 unmodified lines
130	116	u64 next_rip;
131	117
132	118	u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
133		− struct {
134		− u16 fs;
135		− u16 gs;
136		− u16 ldt;
137		− u64 gs_base;
138		− } host;
139	119
140	120	u64 spec_ctrl;
141	121	/*
@@ -192,6 +172,8 @@ struct vcpu_svm { · 50 unmodified lines
192	172	u64 ghcb_sa_len;
193	173	bool ghcb_sa_sync;
194	174	bool ghcb_sa_free;
	175	+
	176	+ bool guest_state_loaded;
195	177	};
196	178
197	179	struct svm_cpu_data {
@@ -587,9 +569,8 @@ int sev_handle_vmgexit(struct vcpu_svm *svm); · 389 unmodified lines
587	569	int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in);
588	570	void sev_es_init_vmcb(struct vcpu_svm *svm);
589	571	void sev_es_create_vcpu(struct vcpu_svm *svm);
590		−void sev_es_vcpu_load(struct vcpu_svm *svm, int cpu);
591		−void sev_es_vcpu_put(struct vcpu_svm *svm);
592	572	void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector);
	573	+void sev_es_prepare_guest_switch(struct vcpu_svm *svm, unsigned int cpu);
593	574
594	575	/* vmenter.S */
595	576

arch/x86/kvm/svm/svm_ops.h+69−0

@@ -0,0 +1,69 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+#ifndef __KVM_X86_SVM_OPS_H
	3	+#define __KVM_X86_SVM_OPS_H
	4	+
	5	+#include <linux/compiler_types.h>
	6	+
	7	+#include <asm/kvm_host.h>
	8	+
	9	+#define svm_asm(insn, clobber...) \
	10	+do { \
	11	+ asm_volatile_goto("1: " __stringify(insn) "\n\t" \
	12	+ _ASM_EXTABLE(1b, %l[fault]) \
	13	+ ::: clobber : fault); \
	14	+ return; \
	15	+fault: \
	16	+ kvm_spurious_fault(); \
	17	+} while (0)
	18	+
	19	+#define svm_asm1(insn, op1, clobber...) \
	20	+do { \
	21	+ asm_volatile_goto("1: " __stringify(insn) " %0\n\t" \
	22	+ _ASM_EXTABLE(1b, %l[fault]) \
	23	+ :: op1 : clobber : fault); \
	24	+ return; \
	25	+fault: \
	26	+ kvm_spurious_fault(); \
	27	+} while (0)
	28	+
	29	+#define svm_asm2(insn, op1, op2, clobber...) \
	30	+do { \
	31	+ asm_volatile_goto("1: " __stringify(insn) " %1, %0\n\t" \
	32	+ _ASM_EXTABLE(1b, %l[fault]) \
	33	+ :: op1, op2 : clobber : fault); \
	34	+ return; \
	35	+fault: \
	36	+ kvm_spurious_fault(); \
	37	+} while (0)
	38	+
	39	+static inline void clgi(void)
	40	+{
	41	+ svm_asm(clgi);
	42	+}
	43	+
	44	+static inline void stgi(void)
	45	+{
	46	+ svm_asm(stgi);
	47	+}
	48	+
	49	+static inline void invlpga(unsigned long addr, u32 asid)
	50	+{
	51	+ svm_asm2(invlpga, "c"(asid), "a"(addr));
	52	+}
	53	+
	54	+/*
	55	+ * Despite being a physical address, the portion of rAX that is consumed by
	56	+ * VMSAVE, VMLOAD, etc... is still controlled by the effective address size,
	57	+ * hence 'unsigned long' instead of 'hpa_t'.
	58	+ */
	59	+static inline void vmsave(unsigned long pa)
	60	+{
	61	+ svm_asm1(vmsave, "a" (pa), "memory");
	62	+}
	63	+
	64	+static inline void vmload(unsigned long pa)
	65	+{
	66	+ svm_asm1(vmload, "a" (pa), "memory");
	67	+}
	68	+
	69	+#endif /* __KVM_X86_SVM_OPS_H */

arch/x86/kvm/trace.h+38−2

@@ -92,6 +92,42 @@ TRACE_EVENT(kvm_hv_hypercall, · 91 unmodified lines
92	92	__entry->outgpa)
93	93	);
94	94
	95	+/*
	96	+ * Tracepoint for Xen hypercall.
	97	+ */
	98	+TRACE_EVENT(kvm_xen_hypercall,
	99	+ TP_PROTO(unsigned long nr, unsigned long a0, unsigned long a1,
	100	+ unsigned long a2, unsigned long a3, unsigned long a4,
	101	+ unsigned long a5),
	102	+ TP_ARGS(nr, a0, a1, a2, a3, a4, a5),
	103	+
	104	+ TP_STRUCT__entry(
	105	+ __field(unsigned long, nr)
	106	+ __field(unsigned long, a0)
	107	+ __field(unsigned long, a1)
	108	+ __field(unsigned long, a2)
	109	+ __field(unsigned long, a3)
	110	+ __field(unsigned long, a4)
	111	+ __field(unsigned long, a5)
	112	+ ),
	113	+
	114	+ TP_fast_assign(
	115	+ __entry->nr = nr;
	116	+ __entry->a0 = a0;
	117	+ __entry->a1 = a1;
	118	+ __entry->a2 = a2;
	119	+ __entry->a3 = a3;
	120	+ __entry->a4 = a4;
	121	+ __entry->a4 = a5;
	122	+ ),
	123	+
	124	+ TP_printk("nr 0x%lx a0 0x%lx a1 0x%lx a2 0x%lx a3 0x%lx a4 0x%lx a5 %lx",
	125	+ __entry->nr, __entry->a0, __entry->a1, __entry->a2,
	126	+ __entry->a3, __entry->a4, __entry->a5)
	127	+);
	128	+
	129	+
	130	+
95	131	/*
96	132	* Tracepoint for PIO.
97	133	*/
@@ -256,7 +292,7 @@ TRACE_EVENT(name, \ · 158 unmodified lines
256	292	__entry->guest_rip = kvm_rip_read(vcpu); \
257	293	__entry->isa = isa; \
258	294	__entry->vcpu_id = vcpu->vcpu_id; \
259		− kvm_x86_ops.get_exit_info(vcpu, &__entry->info1, \
	295	+ static_call(kvm_x86_get_exit_info)(vcpu, &__entry->info1, \
260	296	&__entry->info2, \
261	297	&__entry->intr_info, \
262	298	&__entry->error_code); \
@@ -738,7 +774,7 @@ TRACE_EVENT(kvm_emulate_insn, · 475 unmodified lines
738	774	),
739	775
740	776	TP_fast_assign(
741		− __entry->csbase = kvm_x86_ops.get_segment_base(vcpu, VCPU_SREG_CS);
	777	+ __entry->csbase = static_call(kvm_x86_get_segment_base)(vcpu, VCPU_SREG_CS);
742	778	__entry->len = vcpu->arch.emulate_ctxt->fetch.ptr
743	779	- vcpu->arch.emulate_ctxt->fetch.data;
744	780	__entry->rip = vcpu->arch.emulate_ctxt->_eip - __entry->len;

arch/x86/kvm/vmx/capabilities.h+27−1

@@ -19,6 +19,9 @@ extern int __read_mostly pt_mode; · 18 unmodified lines
19	19	#define PT_MODE_HOST_GUEST 1
20	20
21	21	#define PMU_CAP_FW_WRITES (1ULL << 13)
	22	+#define PMU_CAP_LBR_FMT 0x3f
	23	+
	24	+#define DEBUGCTLMSR_LBR_MASK (DEBUGCTLMSR_LBR \| DEBUGCTLMSR_FREEZE_LBRS_ON_PMI)
22	25
23	26	struct nested_vmx_msrs {
24	27	/*
@@ -262,6 +265,12 @@ static inline bool cpu_has_vmx_tsc_scaling(void) · 237 unmodified lines
262	265	SECONDARY_EXEC_TSC_SCALING;
263	266	}
264	267
	268	+static inline bool cpu_has_vmx_bus_lock_detection(void)
	269	+{
	270	+ return vmcs_config.cpu_based_2nd_exec_ctrl &
	271	+ SECONDARY_EXEC_BUS_LOCK_DETECTION;
	272	+}
	273	+
265	274	static inline bool cpu_has_vmx_apicv(void)
266	275	{
267	276	return cpu_has_vmx_apic_register_virt() &&
@@ -371,11 +380,28 @@ static inline bool vmx_pt_mode_is_host_guest(void) · 103 unmodified lines
371	380
372	381	static inline u64 vmx_get_perf_capabilities(void)
373	382	{
	383	+ u64 perf_cap = 0;
	384	+
	385	+ if (boot_cpu_has(X86_FEATURE_PDCM))
	386	+ rdmsrl(MSR_IA32_PERF_CAPABILITIES, perf_cap);
	387	+
	388	+ perf_cap &= PMU_CAP_LBR_FMT;
	389	+
374	390	/*
375	391	* Since counters are virtualized, KVM would support full
376	392	* width counting unconditionally, even if the host lacks it.
377	393	*/
378		− return PMU_CAP_FW_WRITES;
	394	+ return PMU_CAP_FW_WRITES \| perf_cap;
	395	+}
	396	+
	397	+static inline u64 vmx_supported_debugctl(void)
	398	+{
	399	+ u64 debugctl = 0;
	400	+
	401	+ if (vmx_get_perf_capabilities() & PMU_CAP_LBR_FMT)
	402	+ debugctl \|= DEBUGCTLMSR_LBR_MASK;
	403	+
	404	+ return debugctl;
379	405	}
380	406
381	407	#endif /* __KVM_X86_VMX_CAPS_H */

arch/x86/kvm/vmx/nested.c+40−66

@@ -12,6 +12,7 @@ · 11 unmodified lines
12	12	#include "nested.h"
13	13	#include "pmu.h"
14	14	#include "trace.h"
	15	+#include "vmx.h"
15	16	#include "x86.h"
16	17
17	18	static bool __read_mostly enable_shadow_vmcs = 1;
@@ -411,8 +412,8 @@ static int nested_vmx_check_exception(struct kvm_vcpu vcpu, unsigned long exit · 393 unmodified lines
411	412	if (nr == DB_VECTOR) {
412	413	if (!has_payload) {
413	414	payload = vcpu->arch.dr6;
414		− payload &= ~(DR6_FIXED_1 \| DR6_BT);
415		− payload ^= DR6_RTM;
	415	+ payload &= ~DR6_BT;
	416	+ payload ^= DR6_ACTIVE_LOW;
416	417	}
417	418	*exit_qual = payload;
418	419	} else
@@ -744,8 +745,7 @@ static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, · 325 unmodified lines
744	745	(CC(!nested_cpu_has_vid(vmcs12)) \|\|
745	746	CC(!nested_exit_intr_ack_set(vcpu)) \|\|
746	747	CC((vmcs12->posted_intr_nv & 0xff00)) \|\|
747		− CC((vmcs12->posted_intr_desc_addr & 0x3f)) \|\|
748		− CC((vmcs12->posted_intr_desc_addr >> cpuid_maxphyaddr(vcpu)))))
	748	+ CC(!kvm_vcpu_is_legal_aligned_gpa(vcpu, vmcs12->posted_intr_desc_addr, 64))))
749	749	return -EINVAL;
750	750
751	751	/* tpr shadow is needed by all apicv features. */
@@ -758,13 +758,11 @@ static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, · 6 unmodified lines
758	758	static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
759	759	u32 count, u64 addr)
760	760	{
761		− int maxphyaddr;
762		−
763	761	if (count == 0)
764	762	return 0;
765		− maxphyaddr = cpuid_maxphyaddr(vcpu);
766		− if (!IS_ALIGNED(addr, 16) \|\| addr >> maxphyaddr \|\|
767		− (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr)
	763	+
	764	+ if (!kvm_vcpu_is_legal_aligned_gpa(vcpu, addr, 16) \|\|
	765	+ !kvm_vcpu_is_legal_gpa(vcpu, (addr + count * sizeof(struct vmx_msr_entry) - 1)))
768	766	return -EINVAL;
769	767
770	768	return 0;
@@ -1062,14 +1060,6 @@ static void prepare_vmx_msr_autostore_list(struct kvm_vcpu *vcpu, · 291 unmodified lines
1062	1060	}
1063	1061	}
1064	1062
1065		−static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val)
1066		−{
1067		− unsigned long invalid_mask;
1068		−
1069		− invalid_mask = (~0ULL) << cpuid_maxphyaddr(vcpu);
1070		− return (val & invalid_mask) == 0;
1071		−}
1072		−
1073	1063	/*
1074	1064	* Returns true if the MMU needs to be sync'd on nested VM-Enter/VM-Exit.
1075	1065	* tl;dr: the MMU needs a sync if L0 is using shadow paging and L1 didn't
@@ -1121,7 +1111,7 @@ static bool nested_vmx_transition_mmu_sync(struct kvm_vcpu *vcpu) · 45 unmodified lines
1121	1111	static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept,
1122	1112	enum vm_entry_failure_code *entry_failure_code)
1123	1113	{
1124		− if (CC(!nested_cr3_valid(vcpu, cr3))) {
	1114	+ if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3))) {
1125	1115	*entry_failure_code = ENTRY_FAIL_DEFAULT;
1126	1116	return -EINVAL;
1127	1117	}
@@ -2532,7 +2522,7 @@ static int prepare_vmcs02(struct kvm_vcpu vcpu, struct vmcs12 vmcs12, · 1404 unmodified lines
2532	2522	* bitwise-or of what L1 wants to trap for L2, and what we want to
2533	2523	* trap. Note that CR0.TS also needs updating - we do this later.
2534	2524	*/
2535		− update_exception_bitmap(vcpu);
	2525	+ vmx_update_exception_bitmap(vcpu);
2536	2526	vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask;
2537	2527	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
2538	2528
@@ -2635,7 +2625,6 @@ static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12) · 96 unmodified lines
2635	2625	static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp)
2636	2626	{
2637	2627	struct vcpu_vmx *vmx = to_vmx(vcpu);
2638		− int maxphyaddr = cpuid_maxphyaddr(vcpu);
2639	2628
2640	2629	/* Check for memory type validity */
2641	2630	switch (new_eptp & VMX_EPTP_MT_MASK) {
@@ -2666,7 +2655,7 @@ static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp) · 24 unmodified lines
2666	2655	}
2667	2656
2668	2657	/* Reserved bits should not be set */
2669		− if (CC(new_eptp >> maxphyaddr \|\| ((new_eptp >> 7) & 0x1f)))
	2658	+ if (CC(kvm_vcpu_is_illegal_gpa(vcpu, new_eptp) \|\| ((new_eptp >> 7) & 0x1f)))
2670	2659	return false;
2671	2660
2672	2661	/* AD, if set, should be supported */
@@ -2850,7 +2839,7 @@ static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu, · 177 unmodified lines
2850	2839
2851	2840	if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) \|\|
2852	2841	CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) \|\|
2853		− CC(!nested_cr3_valid(vcpu, vmcs12->host_cr3)))
	2842	+ CC(kvm_vcpu_is_illegal_gpa(vcpu, vmcs12->host_cr3)))
2854	2843	return -EINVAL;
2855	2844
2856	2845	if (CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu)) \|\|
@@ -3057,35 +3046,8 @@ static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) · 200 unmodified lines
3057	3046	vmx->loaded_vmcs->host_state.cr4 = cr4;
3058	3047	}
3059	3048
3060		− asm(
3061		− "sub $%c[wordsize], %%" _ASM_SP "\n\t" /* temporarily adjust RSP for CALL */
3062		− "cmp %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t"
3063		− "je 1f \n\t"
3064		− __ex("vmwrite %%" _ASM_SP ", %[HOST_RSP]") "\n\t"
3065		− "mov %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t"
3066		− "1: \n\t"
3067		− "add $%c[wordsize], %%" _ASM_SP "\n\t" /* un-adjust RSP */
3068		−
3069		− /* Check if vmlaunch or vmresume is needed */
3070		− "cmpb $0, %c[launched](%[loaded_vmcs])\n\t"
3071		−
3072		− /*
3073		− * VMLAUNCH and VMRESUME clear RFLAGS.{CF,ZF} on VM-Exit, set
3074		− * RFLAGS.CF on VM-Fail Invalid and set RFLAGS.ZF on VM-Fail
3075		− * Valid. vmx_vmenter() directly "returns" RFLAGS, and so the
3076		− * results of VM-Enter is captured via CC_{SET,OUT} to vm_fail.
3077		− */
3078		− "call vmx_vmenter\n\t"
3079		−
3080		− CC_SET(be)
3081		− : ASM_CALL_CONSTRAINT, CC_OUT(be) (vm_fail)
3082		− : [HOST_RSP]"r"((unsigned long)HOST_RSP),
3083		− [loaded_vmcs]"r"(vmx->loaded_vmcs),
3084		− [launched]"i"(offsetof(struct loaded_vmcs, launched)),
3085		− [host_state_rsp]"i"(offsetof(struct loaded_vmcs, host_state.rsp)),
3086		− [wordsize]"i"(sizeof(ulong))
3087		− : "memory"
3088		− );
	3049	+ vm_fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs,
	3050	+ vmx->loaded_vmcs->launched);
3089	3051
3090	3052	if (vmx->msr_autoload.host.nr)
3091	3053	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
@@ -3330,7 +3292,11 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, · 238 unmodified lines
3330	3292	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3331	3293	enum vm_entry_failure_code entry_failure_code;
3332	3294	bool evaluate_pending_interrupts;
3333		− u32 exit_reason, failed_index;
	3295	+ union vmx_exit_reason exit_reason = {
	3296	+ .basic = EXIT_REASON_INVALID_STATE,
	3297	+ .failed_vmentry = 1,
	3298	+ };
	3299	+ u32 failed_index;
3334	3300
3335	3301	if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
3336	3302	kvm_vcpu_flush_tlb_current(vcpu);
@@ -3382,7 +3348,7 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, · 45 unmodified lines
3382	3348
3383	3349	if (nested_vmx_check_guest_state(vcpu, vmcs12,
3384	3350	&entry_failure_code)) {
3385		− exit_reason = EXIT_REASON_INVALID_STATE;
	3351	+ exit_reason.basic = EXIT_REASON_INVALID_STATE;
3386	3352	vmcs12->exit_qualification = entry_failure_code;
3387	3353	goto vmentry_fail_vmexit;
3388	3354	}
@@ -3393,7 +3359,7 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, · 4 unmodified lines
3393	3359	vcpu->arch.tsc_offset += vmcs12->tsc_offset;
3394	3360
3395	3361	if (prepare_vmcs02(vcpu, vmcs12, &entry_failure_code)) {
3396		− exit_reason = EXIT_REASON_INVALID_STATE;
	3362	+ exit_reason.basic = EXIT_REASON_INVALID_STATE;
3397	3363	vmcs12->exit_qualification = entry_failure_code;
3398	3364	goto vmentry_fail_vmexit_guest_mode;
3399	3365	}
@@ -3403,7 +3369,7 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, · 3 unmodified lines
3403	3369	vmcs12->vm_entry_msr_load_addr,
3404	3370	vmcs12->vm_entry_msr_load_count);
3405	3371	if (failed_index) {
3406		− exit_reason = EXIT_REASON_MSR_LOAD_FAIL;
	3372	+ exit_reason.basic = EXIT_REASON_MSR_LOAD_FAIL;
3407	3373	vmcs12->exit_qualification = failed_index;
3408	3374	goto vmentry_fail_vmexit_guest_mode;
3409	3375	}
@@ -3471,7 +3437,7 @@ vmentry_fail_vmexit: · 61 unmodified lines
3471	3437	return NVMX_VMENTRY_VMEXIT;
3472	3438
3473	3439	load_vmcs12_host_state(vcpu, vmcs12);
3474		− vmcs12->vm_exit_reason = exit_reason \| VMX_EXIT_REASONS_FAILED_VMENTRY;
	3440	+ vmcs12->vm_exit_reason = exit_reason.full;
3475	3441	if (enable_shadow_vmcs \|\| vmx->nested.hv_evmcs)
3476	3442	vmx->nested.need_vmcs12_to_shadow_sync = true;
3477	3443	return NVMX_VMENTRY_VMEXIT;
@@ -5559,7 +5525,12 @@ static int handle_vmfunc(struct kvm_vcpu *vcpu) · 2081 unmodified lines
5559	5525	return kvm_skip_emulated_instruction(vcpu);
5560	5526
5561	5527	fail:
5562		− nested_vmx_vmexit(vcpu, vmx->exit_reason,
	5528	+ /*
	5529	+ * This is effectively a reflected VM-Exit, as opposed to a synthesized
	5530	+ * nested VM-Exit. Pass the original exit reason, i.e. don't hardcode
	5531	+ * EXIT_REASON_VMFUNC as the exit reason.
	5532	+ */
	5533	+ nested_vmx_vmexit(vcpu, vmx->exit_reason.full,
5563	5534	vmx_get_intr_info(vcpu),
5564	5535	vmx_get_exit_qual(vcpu));
5565	5536	return 1;
@@ -5627,7 +5598,8 @@ static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, · 61 unmodified lines
5627	5598	* MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps.
5628	5599	*/
5629	5600	static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
5630		− struct vmcs12 *vmcs12, u32 exit_reason)
	5601	+ struct vmcs12 *vmcs12,
	5602	+ union vmx_exit_reason exit_reason)
5631	5603	{
5632	5604	u32 msr_index = kvm_rcx_read(vcpu);
5633	5605	gpa_t bitmap;
@@ -5641,7 +5613,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, · 7 unmodified lines
5641	5613	* First we need to figure out which of the four to use:
5642	5614	*/
5643	5615	bitmap = vmcs12->msr_bitmap;
5644		− if (exit_reason == EXIT_REASON_MSR_WRITE)
	5616	+ if (exit_reason.basic == EXIT_REASON_MSR_WRITE)
5645	5617	bitmap += 2048;
5646	5618	if (msr_index >= 0xc0000000) {
5647	5619	msr_index -= 0xc0000000;
@@ -5778,11 +5750,12 @@ static bool nested_vmx_exit_handled_mtf(struct vmcs12 *vmcs12) · 130 unmodified lines
5778	5750	* Return true if L0 wants to handle an exit from L2 regardless of whether or not
5779	5751	* L1 wants the exit. Only call this when in is_guest_mode (L2).
5780	5752	*/
5781		−static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason)
	5753	+static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu,
	5754	+ union vmx_exit_reason exit_reason)
5782	5755	{
5783	5756	u32 intr_info;
5784	5757
5785		− switch ((u16)exit_reason) {
	5758	+ switch ((u16)exit_reason.basic) {
5786	5759	case EXIT_REASON_EXCEPTION_NMI:
5787	5760	intr_info = vmx_get_intr_info(vcpu);
5788	5761	if (is_nmi(intr_info))
@@ -5838,12 +5811,13 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason) · 49 unmodified lines
5838	5811	* Return 1 if L1 wants to intercept an exit from L2. Only call this when in
5839	5812	* is_guest_mode (L2).
5840	5813	*/
5841		−static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason)
	5814	+static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu,
	5815	+ union vmx_exit_reason exit_reason)
5842	5816	{
5843	5817	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
5844	5818	u32 intr_info;
5845	5819
5846		− switch ((u16)exit_reason) {
	5820	+ switch ((u16)exit_reason.basic) {
5847	5821	case EXIT_REASON_EXCEPTION_NMI:
5848	5822	intr_info = vmx_get_intr_info(vcpu);
5849	5823	if (is_nmi(intr_info))
@@ -5962,7 +5936,7 @@ static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason) · 112 unmodified lines
5962	5936	bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu)
5963	5937	{
5964	5938	struct vcpu_vmx *vmx = to_vmx(vcpu);
5965		− u32 exit_reason = vmx->exit_reason;
	5939	+ union vmx_exit_reason exit_reason = vmx->exit_reason;
5966	5940	unsigned long exit_qual;
5967	5941	u32 exit_intr_info;
5968	5942
@@ -5981,7 +5955,7 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) · 12 unmodified lines
5981	5955	goto reflect_vmexit;
5982	5956	}
5983	5957
5984		− trace_kvm_nested_vmexit(exit_reason, vcpu, KVM_ISA_VMX);
	5958	+ trace_kvm_nested_vmexit(exit_reason.full, vcpu, KVM_ISA_VMX);
5985	5959
5986	5960	/* If L0 (KVM) wants the exit, it trumps L1's desires. */
5987	5961	if (nested_vmx_l0_wants_exit(vcpu, exit_reason))
@@ -6007,7 +5981,7 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) · 19 unmodified lines
6007	5981	exit_qual = vmx_get_exit_qual(vcpu);
6008	5982
6009	5983	reflect_vmexit:
6010		− nested_vmx_vmexit(vcpu, exit_reason, exit_intr_info, exit_qual);
	5984	+ nested_vmx_vmexit(vcpu, exit_reason.full, exit_intr_info, exit_qual);
6011	5985	return true;
6012	5986	}
6013	5987

arch/x86/kvm/vmx/pmu_intel.c+285−9

@@ -152,12 +152,17 @@ static struct kvm_pmc intel_rdpmc_ecx_to_pmc(struct kvm_vcpu vcpu, · 151 unmodified lines
152	152	return &counters[array_index_nospec(idx, num_counters)];
153	153	}
154	154
155		−static inline bool fw_writes_is_enabled(struct kvm_vcpu *vcpu)
	155	+static inline u64 vcpu_get_perf_capabilities(struct kvm_vcpu *vcpu)
156	156	{
157	157	if (!guest_cpuid_has(vcpu, X86_FEATURE_PDCM))
158		− return false;
	158	+ return 0;
159	159
160		− return vcpu->arch.perf_capabilities & PMU_CAP_FW_WRITES;
	160	+ return vcpu->arch.perf_capabilities;
	161	+}
	162	+
	163	+static inline bool fw_writes_is_enabled(struct kvm_vcpu *vcpu)
	164	+{
	165	+ return (vcpu_get_perf_capabilities(vcpu) & PMU_CAP_FW_WRITES) != 0;
161	166	}
162	167
163	168	static inline struct kvm_pmc get_fw_gp_pmc(struct kvm_pmu pmu, u32 msr)
@@ -168,6 +173,41 @@ static inline struct kvm_pmc get_fw_gp_pmc(struct kvm_pmu pmu, u32 msr) · 4 unmodified lines
168	173	return get_gp_pmc(pmu, msr, MSR_IA32_PMC0);
169	174	}
170	175
	176	+bool intel_pmu_lbr_is_compatible(struct kvm_vcpu *vcpu)
	177	+{
	178	+ /*
	179	+ * As a first step, a guest could only enable LBR feature if its
	180	+ * cpu model is the same as the host because the LBR registers
	181	+ * would be pass-through to the guest and they're model specific.
	182	+ */
	183	+ return boot_cpu_data.x86_model == guest_cpuid_model(vcpu);
	184	+}
	185	+
	186	+bool intel_pmu_lbr_is_enabled(struct kvm_vcpu *vcpu)
	187	+{
	188	+ struct x86_pmu_lbr *lbr = vcpu_to_lbr_records(vcpu);
	189	+
	190	+ return lbr->nr && (vcpu_get_perf_capabilities(vcpu) & PMU_CAP_LBR_FMT);
	191	+}
	192	+
	193	+static bool intel_pmu_is_valid_lbr_msr(struct kvm_vcpu *vcpu, u32 index)
	194	+{
	195	+ struct x86_pmu_lbr *records = vcpu_to_lbr_records(vcpu);
	196	+ bool ret = false;
	197	+
	198	+ if (!intel_pmu_lbr_is_enabled(vcpu))
	199	+ return ret;
	200	+
	201	+ ret = (index == MSR_LBR_SELECT) \|\| (index == MSR_LBR_TOS) \|\|
	202	+ (index >= records->from && index < records->from + records->nr) \|\|
	203	+ (index >= records->to && index < records->to + records->nr);
	204	+
	205	+ if (!ret && records->info)
	206	+ ret = (index >= records->info && index < records->info + records->nr);
	207	+
	208	+ return ret;
	209	+}
	210	+
171	211	static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
172	212	{
173	213	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
@@ -183,7 +223,8 @@ static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) · 9 unmodified lines
183	223	default:
184	224	ret = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0) \|\|
185	225	get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0) \|\|
186		− get_fixed_pmc(pmu, msr) \|\| get_fw_gp_pmc(pmu, msr);
	226	+ get_fixed_pmc(pmu, msr) \|\| get_fw_gp_pmc(pmu, msr) \|\|
	227	+ intel_pmu_is_valid_lbr_msr(vcpu, msr);
187	228	break;
188	229	}
189	230
@@ -202,6 +243,111 @@ static struct kvm_pmc intel_msr_idx_to_pmc(struct kvm_vcpu vcpu, u32 msr) · 12 unmodified lines
202	243	return pmc;
203	244	}
204	245
	246	+static inline void intel_pmu_release_guest_lbr_event(struct kvm_vcpu *vcpu)
	247	+{
	248	+ struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
	249	+
	250	+ if (lbr_desc->event) {
	251	+ perf_event_release_kernel(lbr_desc->event);
	252	+ lbr_desc->event = NULL;
	253	+ vcpu_to_pmu(vcpu)->event_count--;
	254	+ }
	255	+}
	256	+
	257	+int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu)
	258	+{
	259	+ struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
	260	+ struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
	261	+ struct perf_event *event;
	262	+
	263	+ /*
	264	+ * The perf_event_attr is constructed in the minimum efficient way:
	265	+ * - set 'pinned = true' to make it task pinned so that if another
	266	+ * cpu pinned event reclaims LBR, the event->oncpu will be set to -1;
	267	+ * - set '.exclude_host = true' to record guest branches behavior;
	268	+ *
	269	+ * - set '.config = INTEL_FIXED_VLBR_EVENT' to indicates host perf
	270	+ * schedule the event without a real HW counter but a fake one;
	271	+ * check is_guest_lbr_event() and __intel_get_event_constraints();
	272	+ *
	273	+ * - set 'sample_type = PERF_SAMPLE_BRANCH_STACK' and
	274	+ * 'branch_sample_type = PERF_SAMPLE_BRANCH_CALL_STACK \|
	275	+ * PERF_SAMPLE_BRANCH_USER' to configure it as a LBR callstack
	276	+ * event, which helps KVM to save/restore guest LBR records
	277	+ * during host context switches and reduces quite a lot overhead,
	278	+ * check branch_user_callstack() and intel_pmu_lbr_sched_task();
	279	+ */
	280	+ struct perf_event_attr attr = {
	281	+ .type = PERF_TYPE_RAW,
	282	+ .size = sizeof(attr),
	283	+ .config = INTEL_FIXED_VLBR_EVENT,
	284	+ .sample_type = PERF_SAMPLE_BRANCH_STACK,
	285	+ .pinned = true,
	286	+ .exclude_host = true,
	287	+ .branch_sample_type = PERF_SAMPLE_BRANCH_CALL_STACK \|
	288	+ PERF_SAMPLE_BRANCH_USER,
	289	+ };
	290	+
	291	+ if (unlikely(lbr_desc->event)) {
	292	+ __set_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use);
	293	+ return 0;
	294	+ }
	295	+
	296	+ event = perf_event_create_kernel_counter(&attr, -1,
	297	+ current, NULL, NULL);
	298	+ if (IS_ERR(event)) {
	299	+ pr_debug_ratelimited("%s: failed %ld\n",
	300	+ __func__, PTR_ERR(event));
	301	+ return -ENOENT;
	302	+ }
	303	+ lbr_desc->event = event;
	304	+ pmu->event_count++;
	305	+ __set_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use);
	306	+ return 0;
	307	+}
	308	+
	309	+/*
	310	+ * It's safe to access LBR msrs from guest when they have not
	311	+ * been passthrough since the host would help restore or reset
	312	+ * the LBR msrs records when the guest LBR event is scheduled in.
	313	+ */
	314	+static bool intel_pmu_handle_lbr_msrs_access(struct kvm_vcpu *vcpu,
	315	+ struct msr_data *msr_info, bool read)
	316	+{
	317	+ struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
	318	+ u32 index = msr_info->index;
	319	+
	320	+ if (!intel_pmu_is_valid_lbr_msr(vcpu, index))
	321	+ return false;
	322	+
	323	+ if (!lbr_desc->event && !intel_pmu_create_guest_lbr_event(vcpu))
	324	+ goto dummy;
	325	+
	326	+ /*
	327	+ * Disable irq to ensure the LBR feature doesn't get reclaimed by the
	328	+ * host at the time the value is read from the msr, and this avoids the
	329	+ * host LBR value to be leaked to the guest. If LBR has been reclaimed,
	330	+ * return 0 on guest reads.
	331	+ */
	332	+ local_irq_disable();
	333	+ if (lbr_desc->event->state == PERF_EVENT_STATE_ACTIVE) {
	334	+ if (read)
	335	+ rdmsrl(index, msr_info->data);
	336	+ else
	337	+ wrmsrl(index, msr_info->data);
	338	+ __set_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use);
	339	+ local_irq_enable();
	340	+ return true;
	341	+ }
	342	+ clear_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use);
	343	+ local_irq_enable();
	344	+
	345	+dummy:
	346	+ if (read)
	347	+ msr_info->data = 0;
	348	+ return true;
	349	+}
	350	+
205	351	static int intel_pmu_get_msr(struct kvm_vcpu vcpu, struct msr_data msr_info)
206	352	{
207	353	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
@@ -236,7 +382,8 @@ static int intel_pmu_get_msr(struct kvm_vcpu vcpu, struct msr_data msr_info) · 28 unmodified lines
236	382	} else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) {
237	383	msr_info->data = pmc->eventsel;
238	384	return 0;
239		− }
	385	+ } else if (intel_pmu_handle_lbr_msrs_access(vcpu, msr_info, true))
	386	+ return 0;
240	387	}
241	388
242	389	return 1;
@@ -307,7 +454,8 @@ static int intel_pmu_set_msr(struct kvm_vcpu vcpu, struct msr_data msr_info) · 64 unmodified lines
307	454	reprogram_gp_counter(pmc, data);
308	455	return 0;
309	456	}
310		− }
	457	+ } else if (intel_pmu_handle_lbr_msrs_access(vcpu, msr_info, false))
	458	+ return 0;
311	459	}
312	460
313	461	return 1;
@@ -316,6 +464,8 @@ static int intel_pmu_set_msr(struct kvm_vcpu vcpu, struct msr_data msr_info) · 2 unmodified lines
316	464	static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
317	465	{
318	466	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
	467	+ struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
	468	+
319	469	struct x86_pmu_capability x86_pmu;
320	470	struct kvm_cpuid_entry2 *entry;
321	471	union cpuid10_eax eax;
@@ -327,7 +477,6 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) · 5 unmodified lines
327	477	pmu->counter_bitmask[KVM_PMC_FIXED] = 0;
328	478	pmu->version = 0;
329	479	pmu->reserved_bits = 0xffffffff00200000ull;
330		− vcpu->arch.perf_capabilities = 0;
331	480
332	481	entry = kvm_find_cpuid_entry(vcpu, 0xa, 0);
333	482	if (!entry)
@@ -340,8 +489,6 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) · 6 unmodified lines
340	489	return;
341	490
342	491	perf_get_x86_pmu_capability(&x86_pmu);
343		− if (guest_cpuid_has(vcpu, X86_FEATURE_PDCM))
344		− vcpu->arch.perf_capabilities = vmx_get_perf_capabilities();
345	492
346	493	pmu->nr_arch_gp_counters = min_t(int, eax.split.num_counters,
347	494	x86_pmu.num_counters_gp);
@@ -385,12 +532,21 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) · 37 unmodified lines
385	532	INTEL_PMC_MAX_GENERIC, pmu->nr_arch_fixed_counters);
386	533
387	534	nested_vmx_pmu_entry_exit_ctls_update(vcpu);
	535	+
	536	+ if (intel_pmu_lbr_is_compatible(vcpu))
	537	+ x86_perf_get_lbr(&lbr_desc->records);
	538	+ else
	539	+ lbr_desc->records.nr = 0;
	540	+
	541	+ if (lbr_desc->records.nr)
	542	+ bitmap_set(pmu->all_valid_pmc_idx, INTEL_PMC_IDX_FIXED_VLBR, 1);
388	543	}
389	544
390	545	static void intel_pmu_init(struct kvm_vcpu *vcpu)
391	546	{
392	547	int i;
393	548	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
	549	+ struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
394	550
395	551	for (i = 0; i < INTEL_PMC_MAX_GENERIC; i++) {
396	552	pmu->gp_counters[i].type = KVM_PMC_GP;
@@ -405,6 +561,11 @@ static void intel_pmu_init(struct kvm_vcpu *vcpu) · 8 unmodified lines
405	561	pmu->fixed_counters[i].idx = i + INTEL_PMC_IDX_FIXED;
406	562	pmu->fixed_counters[i].current_config = 0;
407	563	}
	564	+
	565	+ vcpu->arch.perf_capabilities = vmx_get_perf_capabilities();
	566	+ lbr_desc->records.nr = 0;
	567	+ lbr_desc->event = NULL;
	568	+ lbr_desc->msr_passthrough = false;
408	569	}
409	570
410	571	static void intel_pmu_reset(struct kvm_vcpu *vcpu)
@@ -429,6 +590,119 @@ static void intel_pmu_reset(struct kvm_vcpu *vcpu) · 18 unmodified lines
429	590
430	591	pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status =
431	592	pmu->global_ovf_ctrl = 0;
	593	+
	594	+ intel_pmu_release_guest_lbr_event(vcpu);
	595	+}
	596	+
	597	+/*
	598	+ * Emulate LBR_On_PMI behavior for 1 < pmu.version < 4.
	599	+ *
	600	+ * If Freeze_LBR_On_PMI = 1, the LBR is frozen on PMI and
	601	+ * the KVM emulates to clear the LBR bit (bit 0) in IA32_DEBUGCTL.
	602	+ *
	603	+ * Guest needs to re-enable LBR to resume branches recording.
	604	+ */
	605	+static void intel_pmu_legacy_freezing_lbrs_on_pmi(struct kvm_vcpu *vcpu)
	606	+{
	607	+ u64 data = vmcs_read64(GUEST_IA32_DEBUGCTL);
	608	+
	609	+ if (data & DEBUGCTLMSR_FREEZE_LBRS_ON_PMI) {
	610	+ data &= ~DEBUGCTLMSR_LBR;
	611	+ vmcs_write64(GUEST_IA32_DEBUGCTL, data);
	612	+ }
	613	+}
	614	+
	615	+static void intel_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
	616	+{
	617	+ u8 version = vcpu_to_pmu(vcpu)->version;
	618	+
	619	+ if (!intel_pmu_lbr_is_enabled(vcpu))
	620	+ return;
	621	+
	622	+ if (version > 1 && version < 4)
	623	+ intel_pmu_legacy_freezing_lbrs_on_pmi(vcpu);
	624	+}
	625	+
	626	+static void vmx_update_intercept_for_lbr_msrs(struct kvm_vcpu *vcpu, bool set)
	627	+{
	628	+ struct x86_pmu_lbr *lbr = vcpu_to_lbr_records(vcpu);
	629	+ int i;
	630	+
	631	+ for (i = 0; i < lbr->nr; i++) {
	632	+ vmx_set_intercept_for_msr(vcpu, lbr->from + i, MSR_TYPE_RW, set);
	633	+ vmx_set_intercept_for_msr(vcpu, lbr->to + i, MSR_TYPE_RW, set);
	634	+ if (lbr->info)
	635	+ vmx_set_intercept_for_msr(vcpu, lbr->info + i, MSR_TYPE_RW, set);
	636	+ }
	637	+
	638	+ vmx_set_intercept_for_msr(vcpu, MSR_LBR_SELECT, MSR_TYPE_RW, set);
	639	+ vmx_set_intercept_for_msr(vcpu, MSR_LBR_TOS, MSR_TYPE_RW, set);
	640	+}
	641	+
	642	+static inline void vmx_disable_lbr_msrs_passthrough(struct kvm_vcpu *vcpu)
	643	+{
	644	+ struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
	645	+
	646	+ if (!lbr_desc->msr_passthrough)
	647	+ return;
	648	+
	649	+ vmx_update_intercept_for_lbr_msrs(vcpu, true);
	650	+ lbr_desc->msr_passthrough = false;
	651	+}
	652	+
	653	+static inline void vmx_enable_lbr_msrs_passthrough(struct kvm_vcpu *vcpu)
	654	+{
	655	+ struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
	656	+
	657	+ if (lbr_desc->msr_passthrough)
	658	+ return;
	659	+
	660	+ vmx_update_intercept_for_lbr_msrs(vcpu, false);
	661	+ lbr_desc->msr_passthrough = true;
	662	+}
	663	+
	664	+/*
	665	+ * Higher priority host perf events (e.g. cpu pinned) could reclaim the
	666	+ * pmu resources (e.g. LBR) that were assigned to the guest. This is
	667	+ * usually done via ipi calls (more details in perf_install_in_context).
	668	+ *
	669	+ * Before entering the non-root mode (with irq disabled here), double
	670	+ * confirm that the pmu features enabled to the guest are not reclaimed
	671	+ * by higher priority host events. Otherwise, disallow vcpu's access to
	672	+ * the reclaimed features.
	673	+ */
	674	+void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu)
	675	+{
	676	+ struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
	677	+ struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
	678	+
	679	+ if (!lbr_desc->event) {
	680	+ vmx_disable_lbr_msrs_passthrough(vcpu);
	681	+ if (vmcs_read64(GUEST_IA32_DEBUGCTL) & DEBUGCTLMSR_LBR)
	682	+ goto warn;
	683	+ if (test_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use))
	684	+ goto warn;
	685	+ return;
	686	+ }
	687	+
	688	+ if (lbr_desc->event->state < PERF_EVENT_STATE_ACTIVE) {
	689	+ vmx_disable_lbr_msrs_passthrough(vcpu);
	690	+ __clear_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use);
	691	+ goto warn;
	692	+ } else
	693	+ vmx_enable_lbr_msrs_passthrough(vcpu);
	694	+
	695	+ return;
	696	+
	697	+warn:
	698	+ pr_warn_ratelimited("kvm: vcpu-%d: fail to passthrough LBR.\n",
	699	+ vcpu->vcpu_id);
	700	+}
	701	+
	702	+static void intel_pmu_cleanup(struct kvm_vcpu *vcpu)
	703	+{
	704	+ if (!(vmcs_read64(GUEST_IA32_DEBUGCTL) & DEBUGCTLMSR_LBR))
	705	+ intel_pmu_release_guest_lbr_event(vcpu);
432	706	}
433	707
434	708	struct kvm_pmu_ops intel_pmu_ops = {
@@ -445,4 +719,6 @@ struct kvm_pmu_ops intel_pmu_ops = { · 10 unmodified lines
445	719	.refresh = intel_pmu_refresh,
446	720	.init = intel_pmu_init,
447	721	.reset = intel_pmu_reset,
	722	+ .deliver_pmi = intel_pmu_deliver_pmi,
	723	+ .cleanup = intel_pmu_cleanup,
448	724	};

arch/x86/kvm/vmx/posted_intr.c+3−3

@@ -54,7 +54,7 @@ void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) · 53 unmodified lines
54	54
55	55	dest = cpu_physical_id(cpu);
56	56
57		− if (x2apic_enabled())
	57	+ if (x2apic_mode)
58	58	new.ndst = dest;
59	59	else
60	60	new.ndst = (dest << 8) & 0xFF00;
@@ -104,7 +104,7 @@ static void __pi_post_block(struct kvm_vcpu *vcpu) · 43 unmodified lines
104	104
105	105	dest = cpu_physical_id(vcpu->cpu);
106	106
107		− if (x2apic_enabled())
	107	+ if (x2apic_mode)
108	108	new.ndst = dest;
109	109	else
110	110	new.ndst = (dest << 8) & 0xFF00;
@@ -174,7 +174,7 @@ int pi_pre_block(struct kvm_vcpu *vcpu) · 63 unmodified lines
174	174	*/
175	175	dest = cpu_physical_id(vcpu->pre_pcpu);
176	176
177		− if (x2apic_enabled())
	177	+ if (x2apic_mode)
178	178	new.ndst = dest;
179	179	else
180	180	new.ndst = (dest << 8) & 0xFF00;

arch/x86/kvm/vmx/vmenter.S+1−1

@@ -44,7 +44,7 @@ · 43 unmodified lines
44	44	* they VM-Fail, whereas a successful VM-Enter + VM-Exit will jump
45	45	* to vmx_vmexit.
46	46	*/
47		−SYM_FUNC_START(vmx_vmenter)
	47	+SYM_FUNC_START_LOCAL(vmx_vmenter)
48	48	/* EFLAGS.ZF is set if VMCS.LAUNCHED == 0 */
49	49	je 2f
50	50

arch/x86/kvm/vmx/vmx.c+172−110

@@ -50,6 +50,7 @@ · 49 unmodified lines
50	50	#include "capabilities.h"
51	51	#include "cpuid.h"
52	52	#include "evmcs.h"
	53	+#include "hyperv.h"
53	54	#include "irq.h"
54	55	#include "kvm_cache_regs.h"
55	56	#include "lapic.h"
@@ -552,7 +553,7 @@ static int hv_enable_direct_tlbflush(struct kvm_vcpu *vcpu) · 496 unmodified lines
552	553	{
553	554	struct hv_enlightened_vmcs *evmcs;
554	555	struct hv_partition_assist_pg **p_hv_pa_pg =
555		− &vcpu->kvm->arch.hyperv.hv_pa_pg;
	556	+ &to_kvm_hv(vcpu->kvm)->hv_pa_pg;
556	557	/*
557	558	* Synthetic VM-Exit is not enabled in current code and so All
558	559	* evmcs in singe VM shares same assist page.
@@ -658,6 +659,14 @@ static bool is_valid_passthrough_msr(u32 msr) · 99 unmodified lines
658	659	case MSR_IA32_RTIT_CR3_MATCH:
659	660	case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
660	661	/* PT MSRs. These are handled in pt_update_intercept_for_msr() */
	662	+ case MSR_LBR_SELECT:
	663	+ case MSR_LBR_TOS:
	664	+ case MSR_LBR_INFO_0 ... MSR_LBR_INFO_0 + 31:
	665	+ case MSR_LBR_NHM_FROM ... MSR_LBR_NHM_FROM + 31:
	666	+ case MSR_LBR_NHM_TO ... MSR_LBR_NHM_TO + 31:
	667	+ case MSR_LBR_CORE_FROM ... MSR_LBR_CORE_FROM + 8:
	668	+ case MSR_LBR_CORE_TO ... MSR_LBR_CORE_TO + 8:
	669	+ /* LBR MSRs. These are handled in vmx_update_intercept_for_lbr_msrs() */
661	670	return true;
662	671	}
663	672
@@ -806,7 +815,7 @@ static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg) · 142 unmodified lines
806	815	return *p;
807	816	}
808	817
809		−void update_exception_bitmap(struct kvm_vcpu *vcpu)
	818	+void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu)
810	819	{
811	820	u32 eb;
812	821
@@ -1102,7 +1111,7 @@ static inline bool pt_can_write_msr(struct vcpu_vmx *vmx) · 289 unmodified lines
1102	1111	static inline bool pt_output_base_valid(struct kvm_vcpu *vcpu, u64 base)
1103	1112	{
1104	1113	/* The base must be 128-byte aligned and a legal physical address. */
1105		− return !kvm_vcpu_is_illegal_gpa(vcpu, base) && !(base & 0x7f);
	1114	+ return kvm_vcpu_is_legal_aligned_gpa(vcpu, base, 128);
1106	1115	}
1107	1116
1108	1117	static inline void pt_load_msr(struct pt_ctx *ctx, u32 addr_range)
@@ -1577,7 +1586,7 @@ static int skip_emulated_instruction(struct kvm_vcpu *vcpu) · 468 unmodified lines
1577	1586	* i.e. we end up advancing IP with some random value.
1578	1587	*/
1579	1588	if (!static_cpu_has(X86_FEATURE_HYPERVISOR) \|\|
1580		− to_vmx(vcpu)->exit_reason != EXIT_REASON_EPT_MISCONFIG) {
	1589	+ to_vmx(vcpu)->exit_reason.basic != EXIT_REASON_EPT_MISCONFIG) {
1581	1590	orig_rip = kvm_rip_read(vcpu);
1582	1591	rip = orig_rip + vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
1583	1592	#ifdef CONFIG_X86_64
@@ -1924,6 +1933,9 @@ static int vmx_get_msr(struct kvm_vcpu vcpu, struct msr_data msr_info) · 340 unmodified lines
1924	1933	!guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
1925	1934	return 1;
1926	1935	goto find_uret_msr;
	1936	+ case MSR_IA32_DEBUGCTLMSR:
	1937	+ msr_info->data = vmcs_read64(GUEST_IA32_DEBUGCTL);
	1938	+ break;
1927	1939	default:
1928	1940	find_uret_msr:
1929	1941	msr = vmx_find_uret_msr(vmx, msr_info->index);
@@ -1947,6 +1959,16 @@ static u64 nested_vmx_truncate_sysenter_addr(struct kvm_vcpu *vcpu, · 17 unmodified lines
1947	1959	return (unsigned long)data;
1948	1960	}
1949	1961
	1962	+static u64 vcpu_supported_debugctl(struct kvm_vcpu *vcpu)
	1963	+{
	1964	+ u64 debugctl = vmx_supported_debugctl();
	1965	+
	1966	+ if (!intel_pmu_lbr_is_enabled(vcpu))
	1967	+ debugctl &= ~DEBUGCTLMSR_LBR_MASK;
	1968	+
	1969	+ return debugctl;
	1970	+}
	1971	+
1950	1972	/*
1951	1973	* Writes msr value into the appropriate "register".
1952	1974	* Returns 0 on success, non-0 otherwise.
@@ -1997,14 +2019,29 @@ static int vmx_set_msr(struct kvm_vcpu vcpu, struct msr_data msr_info) · 44 unmodified lines
1997	2019	}
1998	2020	vmcs_writel(GUEST_SYSENTER_ESP, data);
1999	2021	break;
2000		− case MSR_IA32_DEBUGCTLMSR:
	2022	+ case MSR_IA32_DEBUGCTLMSR: {
	2023	+ u64 invalid = data & ~vcpu_supported_debugctl(vcpu);
	2024	+ if (invalid & (DEBUGCTLMSR_BTF\|DEBUGCTLMSR_LBR)) {
	2025	+ if (report_ignored_msrs)
	2026	+ vcpu_unimpl(vcpu, "%s: BTF\|LBR in IA32_DEBUGCTLMSR 0x%llx, nop\n",
	2027	+ __func__, data);
	2028	+ data &= ~(DEBUGCTLMSR_BTF\|DEBUGCTLMSR_LBR);
	2029	+ invalid &= ~(DEBUGCTLMSR_BTF\|DEBUGCTLMSR_LBR);
	2030	+ }
	2031	+
	2032	+ if (invalid)
	2033	+ return 1;
	2034	+
2001	2035	if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls &
2002	2036	VM_EXIT_SAVE_DEBUG_CONTROLS)
2003	2037	get_vmcs12(vcpu)->guest_ia32_debugctl = data;
2004	2038
2005		− ret = kvm_set_msr_common(vcpu, msr_info);
2006		− break;
2007		−
	2039	+ vmcs_write64(GUEST_IA32_DEBUGCTL, data);
	2040	+ if (intel_pmu_lbr_is_enabled(vcpu) && !to_vmx(vcpu)->lbr_desc.event &&
	2041	+ (data & DEBUGCTLMSR_LBR))
	2042	+ intel_pmu_create_guest_lbr_event(vcpu);
	2043	+ return 0;
	2044	+ }
2008	2045	case MSR_IA32_BNDCFGS:
2009	2046	if (!kvm_mpx_supported() \|\|
2010	2047	(!msr_info->host_initiated &&
@@ -2196,6 +2233,18 @@ static int vmx_set_msr(struct kvm_vcpu vcpu, struct msr_data msr_info) · 185 unmodified lines
2196	2233	if ((data >> 32) != 0)
2197	2234	return 1;
2198	2235	goto find_uret_msr;
	2236	+ case MSR_IA32_PERF_CAPABILITIES:
	2237	+ if (data && !vcpu_to_pmu(vcpu)->version)
	2238	+ return 1;
	2239	+ if (data & PMU_CAP_LBR_FMT) {
	2240	+ if ((data & PMU_CAP_LBR_FMT) !=
	2241	+ (vmx_get_perf_capabilities() & PMU_CAP_LBR_FMT))
	2242	+ return 1;
	2243	+ if (!intel_pmu_lbr_is_compatible(vcpu))
	2244	+ return 1;
	2245	+ }
	2246	+ ret = kvm_set_msr_common(vcpu, msr_info);
	2247	+ break;
2199	2248
2200	2249	default:
2201	2250	find_uret_msr:
@@ -2265,7 +2314,6 @@ static int kvm_cpu_vmxon(u64 vmxon_pointer) · 63 unmodified lines
2265	2314	u64 msr;
2266	2315
2267	2316	cr4_set_bits(X86_CR4_VMXE);
2268		− intel_pt_handle_vmx(1);
2269	2317
2270	2318	asm_volatile_goto("1: vmxon %[vmxon_pointer]\n\t"
2271	2319	_ASM_EXTABLE(1b, %l[fault])
@@ -2276,7 +2324,6 @@ static int kvm_cpu_vmxon(u64 vmxon_pointer) · 4 unmodified lines
2276	2324	fault:
2277	2325	WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n",
2278	2326	rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr);
2279		− intel_pt_handle_vmx(0);
2280	2327	cr4_clear_bits(X86_CR4_VMXE);
2281	2328
2282	2329	return -EFAULT;
@@ -2299,9 +2346,13 @@ static int hardware_enable(void) · 16 unmodified lines
2299	2346	!hv_get_vp_assist_page(cpu))
2300	2347	return -EFAULT;
2301	2348
	2349	+ intel_pt_handle_vmx(1);
	2350	+
2302	2351	r = kvm_cpu_vmxon(phys_addr);
2303		− if (r)
	2352	+ if (r) {
	2353	+ intel_pt_handle_vmx(0);
2304	2354	return r;
	2355	+ }
2305	2356
2306	2357	if (enable_ept)
2307	2358	ept_sync_global();
@@ -2319,22 +2370,14 @@ static void vmclear_local_loaded_vmcss(void) · 11 unmodified lines
2319	2370	__loaded_vmcs_clear(v);
2320	2371	}
2321	2372
2322		−
2323		−/* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
2324		− * tricks.
2325		− */
2326		−static void kvm_cpu_vmxoff(void)
2327		−{
2328		− asm volatile (__ex("vmxoff"));
2329		−
2330		− intel_pt_handle_vmx(0);
2331		− cr4_clear_bits(X86_CR4_VMXE);
2332		−}
2333		−
2334	2373	static void hardware_disable(void)
2335	2374	{
2336	2375	vmclear_local_loaded_vmcss();
2337		− kvm_cpu_vmxoff();
	2376	+
	2377	+ if (cpu_vmxoff())
	2378	+ kvm_spurious_fault();
	2379	+
	2380	+ intel_pt_handle_vmx(0);
2338	2381	}
2339	2382
2340	2383	/*
@@ -2428,7 +2471,8 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf, · 87 unmodified lines
2428	2471	SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE \|
2429	2472	SECONDARY_EXEC_PT_USE_GPA \|
2430	2473	SECONDARY_EXEC_PT_CONCEAL_VMX \|
2431		− SECONDARY_EXEC_ENABLE_VMFUNC;
	2474	+ SECONDARY_EXEC_ENABLE_VMFUNC \|
	2475	+ SECONDARY_EXEC_BUS_LOCK_DETECTION;
2432	2476	if (cpu_has_sgx())
2433	2477	opt2 \|= SECONDARY_EXEC_ENCLS_EXITING;
2434	2478	if (adjust_vmx_controls(min2, opt2,
@@ -2739,7 +2783,7 @@ static void enter_pmode(struct kvm_vcpu *vcpu) · 304 unmodified lines
2739	2783	vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) \|
2740	2784	(vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
2741	2785
2742		− update_exception_bitmap(vcpu);
	2786	+ vmx_update_exception_bitmap(vcpu);
2743	2787
2744	2788	fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
2745	2789	fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
@@ -2819,7 +2863,7 @@ static void enter_rmode(struct kvm_vcpu *vcpu) · 73 unmodified lines
2819	2863
2820	2864	vmcs_writel(GUEST_RFLAGS, flags);
2821	2865	vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) \| X86_CR4_VME);
2822		− update_exception_bitmap(vcpu);
	2866	+ vmx_update_exception_bitmap(vcpu);
2823	2867
2824	2868	fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
2825	2869	fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
@@ -3774,7 +3818,7 @@ static __always_inline void vmx_enable_intercept_for_msr(struct kvm_vcpu *vcpu, · 948 unmodified lines
3774	3818	vmx_set_msr_bitmap_write(msr_bitmap, msr);
3775	3819	}
3776	3820
3777		−static __always_inline void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu,
	3821	+void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu,
3778	3822	u32 msr, int type, bool value)
3779	3823	{
3780	3824	if (value)
@@ -4269,6 +4313,9 @@ static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx) · 488 unmodified lines
4269	4313	vmx_adjust_sec_exec_control(vmx, &exec_control, waitpkg, WAITPKG,
4270	4314	ENABLE_USR_WAIT_PAUSE, false);
4271	4315
	4316	+ if (!vcpu->kvm->arch.bus_lock_detection_enabled)
	4317	+ exec_control &= ~SECONDARY_EXEC_BUS_LOCK_DETECTION;
	4318	+
4272	4319	vmx->secondary_exec_control = exec_control;
4273	4320	}
4274	4321
@@ -4467,23 +4514,23 @@ static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) · 192 unmodified lines
4467	4514	vmx_set_cr4(vcpu, 0);
4468	4515	vmx_set_efer(vcpu, 0);
4469	4516
4470		− update_exception_bitmap(vcpu);
	4517	+ vmx_update_exception_bitmap(vcpu);
4471	4518
4472	4519	vpid_sync_context(vmx->vpid);
4473	4520	if (init_event)
4474	4521	vmx_clear_hlt(vcpu);
4475	4522	}
4476	4523
4477		−static void enable_irq_window(struct kvm_vcpu *vcpu)
	4524	+static void vmx_enable_irq_window(struct kvm_vcpu *vcpu)
4478	4525	{
4479	4526	exec_controls_setbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING);
4480	4527	}
4481	4528
4482		−static void enable_nmi_window(struct kvm_vcpu *vcpu)
	4529	+static void vmx_enable_nmi_window(struct kvm_vcpu *vcpu)
4483	4530	{
4484	4531	if (!enable_vnmi \|\|
4485	4532	vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
4486		− enable_irq_window(vcpu);
	4533	+ vmx_enable_irq_window(vcpu);
4487	4534	return;
4488	4535	}
4489	4536
@@ -4824,7 +4871,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) · 334 unmodified lines
4824	4871	kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
4825	4872	return 1;
4826	4873	}
4827		− kvm_run->debug.arch.dr6 = dr6 \| DR6_FIXED_1 \| DR6_RTM;
	4874	+ kvm_run->debug.arch.dr6 = dr6 \| DR6_ACTIVE_LOW;
4828	4875	kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
4829	4876	fallthrough;
4830	4877	case BP_VECTOR:
@@ -5049,6 +5096,7 @@ static int handle_dr(struct kvm_vcpu *vcpu) · 218 unmodified lines
5049	5096	{
5050	5097	unsigned long exit_qualification;
5051	5098	int dr, dr7, reg;
	5099	+ int err = 1;
5052	5100
5053	5101	exit_qualification = vmx_get_exit_qual(vcpu);
5054	5102	dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
@@ -5057,9 +5105,9 @@ static int handle_dr(struct kvm_vcpu *vcpu) · 2 unmodified lines
5057	5105	if (!kvm_require_dr(vcpu, dr))
5058	5106	return 1;
5059	5107
5060		− /* Do not handle if the CPL > 0, will trigger GP on re-entry */
5061		− if (!kvm_require_cpl(vcpu, 0))
5062		− return 1;
	5108	+ if (kvm_x86_ops.get_cpl(vcpu) > 0)
	5109	+ goto out;
	5110	+
5063	5111	dr7 = vmcs_readl(GUEST_DR7);
5064	5112	if (dr7 & DR7_GD) {
5065	5113	/*
@@ -5068,7 +5116,7 @@ static int handle_dr(struct kvm_vcpu *vcpu) · 2 unmodified lines
5068	5116	* guest debugging itself.
5069	5117	*/
5070	5118	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
5071		− vcpu->run->debug.arch.dr6 = DR6_BD \| DR6_RTM \| DR6_FIXED_1;
	5119	+ vcpu->run->debug.arch.dr6 = DR6_BD \| DR6_ACTIVE_LOW;
5072	5120	vcpu->run->debug.arch.dr7 = dr7;
5073	5121	vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu);
5074	5122	vcpu->run->debug.arch.exception = DB_VECTOR;
@@ -5096,14 +5144,15 @@ static int handle_dr(struct kvm_vcpu *vcpu) · 21 unmodified lines
5096	5144	if (exit_qualification & TYPE_MOV_FROM_DR) {
5097	5145	unsigned long val;
5098	5146
5099		− if (kvm_get_dr(vcpu, dr, &val))
5100		− return 1;
	5147	+ kvm_get_dr(vcpu, dr, &val);
5101	5148	kvm_register_write(vcpu, reg, val);
5102		− } else
5103		− if (kvm_set_dr(vcpu, dr, kvm_register_readl(vcpu, reg)))
5104		− return 1;
	5149	+ err = 0;
	5150	+ } else {
	5151	+ err = kvm_set_dr(vcpu, dr, kvm_register_readl(vcpu, reg));
	5152	+ }
5105	5153
5106		− return kvm_skip_emulated_instruction(vcpu);
	5154	+out:
	5155	+ return kvm_complete_insn_gp(vcpu, err);
5107	5156	}
5108	5157
5109	5158	static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
@@ -5177,9 +5226,8 @@ static int handle_xsetbv(struct kvm_vcpu *vcpu) · 67 unmodified lines
5177	5226	u64 new_bv = kvm_read_edx_eax(vcpu);
5178	5227	u32 index = kvm_rcx_read(vcpu);
5179	5228
5180		− if (kvm_set_xcr(vcpu, index, new_bv) == 0)
5181		− return kvm_skip_emulated_instruction(vcpu);
5182		− return 1;
	5229	+ int err = kvm_set_xcr(vcpu, index, new_bv);
	5230	+ return kvm_complete_insn_gp(vcpu, err);
5183	5231	}
5184	5232
5185	5233	static int handle_apic_access(struct kvm_vcpu *vcpu)
@@ -5600,6 +5648,13 @@ static int handle_encls(struct kvm_vcpu *vcpu) · 414 unmodified lines
5600	5648	return 1;
5601	5649	}
5602	5650
	5651	+static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu)
	5652	+{
	5653	+ vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK;
	5654	+ vcpu->run->flags \|= KVM_RUN_X86_BUS_LOCK;
	5655	+ return 0;
	5656	+}
	5657	+
5603	5658	/*
5604	5659	* The exit handlers return 1 if the exit was handled fully and guest execution
5605	5660	* may resume. Otherwise they set the kvm_run parameter to indicate what needs
@@ -5656,6 +5711,7 @@ static int (kvm_vmx_exit_handlers[])(struct kvm_vcpu vcpu) = { · 50 unmodified lines
5656	5711	[EXIT_REASON_VMFUNC] = handle_vmx_instruction,
5657	5712	[EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer,
5658	5713	[EXIT_REASON_ENCLS] = handle_encls,
	5714	+ [EXIT_REASON_BUS_LOCK] = handle_bus_lock_vmexit,
5659	5715	};
5660	5716
5661	5717	static const int kvm_vmx_max_exit_handlers =
@@ -5667,7 +5723,7 @@ static void vmx_get_exit_info(struct kvm_vcpu vcpu, u64 info1, u64 *info2, · 5 unmodified lines
5667	5723	struct vcpu_vmx *vmx = to_vmx(vcpu);
5668	5724
5669	5725	*info1 = vmx_get_exit_qual(vcpu);
5670		− if (!(vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) {
	5726	+ if (!(vmx->exit_reason.failed_vmentry)) {
5671	5727	*info2 = vmx->idt_vectoring_info;
5672	5728	*intr_info = vmx_get_intr_info(vcpu);
5673	5729	if (is_exception_with_error_code(*intr_info))
@@ -5908,11 +5964,12 @@ void dump_vmcs(void) · 234 unmodified lines
5908	5964	* The guest has exited. See if we can fix it or if we need userspace
5909	5965	* assistance.
5910	5966	*/
5911		−static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
	5967	+static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
5912	5968	{
5913	5969	struct vcpu_vmx *vmx = to_vmx(vcpu);
5914		− u32 exit_reason = vmx->exit_reason;
	5970	+ union vmx_exit_reason exit_reason = vmx->exit_reason;
5915	5971	u32 vectoring_info = vmx->idt_vectoring_info;
	5972	+ u16 exit_handler_index;
5916	5973
5917	5974	/*
5918	5975	* Flush logged GPAs PML buffer, this will make dirty_bitmap more
@@ -5954,11 +6011,11 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) · 35 unmodified lines
5954	6011	return 1;
5955	6012	}
5956	6013
5957		− if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) {
	6014	+ if (exit_reason.failed_vmentry) {
5958	6015	dump_vmcs();
5959	6016	vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
5960	6017	vcpu->run->fail_entry.hardware_entry_failure_reason
5961		− = exit_reason;
	6018	+ = exit_reason.full;
5962	6019	vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;
5963	6020	return 0;
5964	6021	}
@@ -5980,18 +6037,18 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) · 15 unmodified lines
5980	6037	* will cause infinite loop.
5981	6038	*/
5982	6039	if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
5983		− (exit_reason != EXIT_REASON_EXCEPTION_NMI &&
5984		− exit_reason != EXIT_REASON_EPT_VIOLATION &&
5985		− exit_reason != EXIT_REASON_PML_FULL &&
5986		− exit_reason != EXIT_REASON_APIC_ACCESS &&
5987		− exit_reason != EXIT_REASON_TASK_SWITCH)) {
	6040	+ (exit_reason.basic != EXIT_REASON_EXCEPTION_NMI &&
	6041	+ exit_reason.basic != EXIT_REASON_EPT_VIOLATION &&
	6042	+ exit_reason.basic != EXIT_REASON_PML_FULL &&
	6043	+ exit_reason.basic != EXIT_REASON_APIC_ACCESS &&
	6044	+ exit_reason.basic != EXIT_REASON_TASK_SWITCH)) {
5988	6045	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
5989	6046	vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV;
5990	6047	vcpu->run->internal.ndata = 3;
5991	6048	vcpu->run->internal.data[0] = vectoring_info;
5992		− vcpu->run->internal.data[1] = exit_reason;
	6049	+ vcpu->run->internal.data[1] = exit_reason.full;
5993	6050	vcpu->run->internal.data[2] = vcpu->arch.exit_qualification;
5994		− if (exit_reason == EXIT_REASON_EPT_MISCONFIG) {
	6051	+ if (exit_reason.basic == EXIT_REASON_EPT_MISCONFIG) {
5995	6052	vcpu->run->internal.ndata++;
5996	6053	vcpu->run->internal.data[3] =
5997	6054	vmcs_read64(GUEST_PHYSICAL_ADDRESS);
@@ -6023,42 +6080,62 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) · 25 unmodified lines
6023	6080	if (exit_fastpath != EXIT_FASTPATH_NONE)
6024	6081	return 1;
6025	6082
6026		− if (exit_reason >= kvm_vmx_max_exit_handlers)
	6083	+ if (exit_reason.basic >= kvm_vmx_max_exit_handlers)
6027	6084	goto unexpected_vmexit;
6028	6085	#ifdef CONFIG_RETPOLINE
6029		− if (exit_reason == EXIT_REASON_MSR_WRITE)
	6086	+ if (exit_reason.basic == EXIT_REASON_MSR_WRITE)
6030	6087	return kvm_emulate_wrmsr(vcpu);
6031		− else if (exit_reason == EXIT_REASON_PREEMPTION_TIMER)
	6088	+ else if (exit_reason.basic == EXIT_REASON_PREEMPTION_TIMER)
6032	6089	return handle_preemption_timer(vcpu);
6033		− else if (exit_reason == EXIT_REASON_INTERRUPT_WINDOW)
	6090	+ else if (exit_reason.basic == EXIT_REASON_INTERRUPT_WINDOW)
6034	6091	return handle_interrupt_window(vcpu);
6035		− else if (exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT)
	6092	+ else if (exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT)
6036	6093	return handle_external_interrupt(vcpu);
6037		− else if (exit_reason == EXIT_REASON_HLT)
	6094	+ else if (exit_reason.basic == EXIT_REASON_HLT)
6038	6095	return kvm_emulate_halt(vcpu);
6039		− else if (exit_reason == EXIT_REASON_EPT_MISCONFIG)
	6096	+ else if (exit_reason.basic == EXIT_REASON_EPT_MISCONFIG)
6040	6097	return handle_ept_misconfig(vcpu);
6041	6098	#endif
6042	6099
6043		− exit_reason = array_index_nospec(exit_reason,
6044		− kvm_vmx_max_exit_handlers);
6045		− if (!kvm_vmx_exit_handlers[exit_reason])
	6100	+ exit_handler_index = array_index_nospec((u16)exit_reason.basic,
	6101	+ kvm_vmx_max_exit_handlers);
	6102	+ if (!kvm_vmx_exit_handlers[exit_handler_index])
6046	6103	goto unexpected_vmexit;
6047	6104
6048		− return kvm_vmx_exit_handlers[exit_reason](vcpu);
	6105	+ return kvm_vmx_exit_handlers[exit_handler_index](vcpu);
6049	6106
6050	6107	unexpected_vmexit:
6051		− vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n", exit_reason);
	6108	+ vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n",
	6109	+ exit_reason.full);
6052	6110	dump_vmcs();
6053	6111	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
6054	6112	vcpu->run->internal.suberror =
6055	6113	KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
6056	6114	vcpu->run->internal.ndata = 2;
6057		− vcpu->run->internal.data[0] = exit_reason;
	6115	+ vcpu->run->internal.data[0] = exit_reason.full;
6058	6116	vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
6059	6117	return 0;
6060	6118	}
6061	6119
	6120	+static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
	6121	+{
	6122	+ int ret = __vmx_handle_exit(vcpu, exit_fastpath);
	6123	+
	6124	+ /*
	6125	+ * Even when current exit reason is handled by KVM internally, we
	6126	+ * still need to exit to user space when bus lock detected to inform
	6127	+ * that there is a bus lock in guest.
	6128	+ */
	6129	+ if (to_vmx(vcpu)->exit_reason.bus_lock_detected) {
	6130	+ if (ret > 0)
	6131	+ vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK;
	6132	+
	6133	+ vcpu->run->flags \|= KVM_RUN_X86_BUS_LOCK;
	6134	+ return 0;
	6135	+ }
	6136	+ return ret;
	6137	+}
	6138	+
6062	6139	/*
6063	6140	* Software based L1D cache flush which is used when microcode providing
6064	6141	* the cache control MSR is not loaded.
@@ -6129,7 +6206,7 @@ static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu) · 64 unmodified lines
6129	6206	: "eax", "ebx", "ecx", "edx");
6130	6207	}
6131	6208
6132		−static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
	6209	+static void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
6133	6210	{
6134	6211	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
6135	6212	int tpr_threshold;
@@ -6373,9 +6450,9 @@ static void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu) · 237 unmodified lines
6373	6450	{
6374	6451	struct vcpu_vmx *vmx = to_vmx(vcpu);
6375	6452
6376		− if (vmx->exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT)
	6453	+ if (vmx->exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT)
6377	6454	handle_external_interrupt_irqoff(vcpu);
6378		− else if (vmx->exit_reason == EXIT_REASON_EXCEPTION_NMI)
	6455	+ else if (vmx->exit_reason.basic == EXIT_REASON_EXCEPTION_NMI)
6379	6456	handle_exception_nmi_irqoff(vmx);
6380	6457	}
6381	6458
@@ -6567,7 +6644,7 @@ void noinstr vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp) · 185 unmodified lines
6567	6644
6568	6645	static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
6569	6646	{
6570		− switch (to_vmx(vcpu)->exit_reason) {
	6647	+ switch (to_vmx(vcpu)->exit_reason.basic) {
6571	6648	case EXIT_REASON_MSR_WRITE:
6572	6649	return handle_fastpath_set_msr_irqoff(vcpu);
6573	6650	case EXIT_REASON_PREEMPTION_TIMER:
@@ -6577,8 +6654,6 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) · 3 unmodified lines
6577	6654	}
6578	6655	}
6579	6656
6580		−bool __vmx_vcpu_run(struct vcpu_vmx vmx, unsigned long regs, bool launched);
6581		−
6582	6657	static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,
6583	6658	struct vcpu_vmx *vmx)
6584	6659	{
@@ -6638,11 +6713,9 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, · 53 unmodified lines
6638	6713
6639	6714	static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu)
6640	6715	{
6641		− fastpath_t exit_fastpath;
6642	6716	struct vcpu_vmx *vmx = to_vmx(vcpu);
6643	6717	unsigned long cr3, cr4;
6644	6718
6645		−reenter_guest:
6646	6719	/* Record the guest's net vcpu time for enforced NMI injections. */
6647	6720	if (unlikely(!enable_vnmi &&
6648	6721	vmx->loaded_vmcs->soft_vnmi_blocked))
@@ -6696,6 +6769,8 @@ reenter_guest: · 47 unmodified lines
6696	6769	pt_guest_enter(vmx);
6697	6770
6698	6771	atomic_switch_perf_msrs(vmx);
	6772	+ if (intel_pmu_lbr_is_enabled(vcpu))
	6773	+ vmx_passthrough_lbr_msrs(vcpu);
6699	6774
6700	6775	if (enable_preemption_timer)
6701	6776	vmx_update_hv_timer(vcpu);
@@ -6734,12 +6809,12 @@ reenter_guest: · 32 unmodified lines
6734	6809	x86_spec_ctrl_restore_host(vmx->spec_ctrl, 0);
6735	6810
6736	6811	/* All fields are clean at this point */
6737		− if (static_branch_unlikely(&enable_evmcs))
	6812	+ if (static_branch_unlikely(&enable_evmcs)) {
6738	6813	current_evmcs->hv_clean_fields \|=
6739	6814	HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;
6740	6815
6741		− if (static_branch_unlikely(&enable_evmcs))
6742		− current_evmcs->hv_vp_id = vcpu->arch.hyperv.vp_index;
	6816	+ current_evmcs->hv_vp_id = kvm_hv_get_vpindex(vcpu);
	6817	+ }
6743	6818
6744	6819	/* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
6745	6820	if (vmx->host_debugctlmsr)
@@ -6768,17 +6843,17 @@ reenter_guest: · 22 unmodified lines
6768	6843	vmx->idt_vectoring_info = 0;
6769	6844
6770	6845	if (unlikely(vmx->fail)) {
6771		− vmx->exit_reason = 0xdead;
	6846	+ vmx->exit_reason.full = 0xdead;
6772	6847	return EXIT_FASTPATH_NONE;
6773	6848	}
6774	6849
6775		− vmx->exit_reason = vmcs_read32(VM_EXIT_REASON);
6776		− if (unlikely((u16)vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY))
	6850	+ vmx->exit_reason.full = vmcs_read32(VM_EXIT_REASON);
	6851	+ if (unlikely((u16)vmx->exit_reason.basic == EXIT_REASON_MCE_DURING_VMENTRY))
6777	6852	kvm_machine_check();
6778	6853
6779		− trace_kvm_exit(vmx->exit_reason, vcpu, KVM_ISA_VMX);
	6854	+ trace_kvm_exit(vmx->exit_reason.full, vcpu, KVM_ISA_VMX);
6780	6855
6781		− if (unlikely(vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY))
	6856	+ if (unlikely(vmx->exit_reason.failed_vmentry))
6782	6857	return EXIT_FASTPATH_NONE;
6783	6858
6784	6859	vmx->loaded_vmcs->launched = 1;
@@ -6790,22 +6865,7 @@ reenter_guest: · 5 unmodified lines
6790	6865	if (is_guest_mode(vcpu))
6791	6866	return EXIT_FASTPATH_NONE;
6792	6867
6793		− exit_fastpath = vmx_exit_handlers_fastpath(vcpu);
6794		− if (exit_fastpath == EXIT_FASTPATH_REENTER_GUEST) {
6795		− if (!kvm_vcpu_exit_request(vcpu)) {
6796		− /*
6797		− * FIXME: this goto should be a loop in vcpu_enter_guest,
6798		− * but it would incur the cost of a retpoline for now.
6799		− * Revisit once static calls are available.
6800		− */
6801		− if (vcpu->arch.apicv_active)
6802		− vmx_sync_pir_to_irr(vcpu);
6803		− goto reenter_guest;
6804		− }
6805		− exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED;
6806		− }
6807		−
6808		− return exit_fastpath;
	6868	+ return vmx_exit_handlers_fastpath(vcpu);
6809	6869	}
6810	6870
6811	6871	static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
@@ -7256,7 +7316,7 @@ static void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) · 444 unmodified lines
7256	7316	set_cr4_guest_host_mask(vmx);
7257	7317
7258	7318	/* Refresh #PF interception to account for MAXPHYADDR changes. */
7259		− update_exception_bitmap(vcpu);
	7319	+ vmx_update_exception_bitmap(vcpu);
7260	7320	}
7261	7321
7262	7322	static __init void vmx_set_cpu_caps(void)
@@ -7546,7 +7606,7 @@ static int vmx_pre_leave_smm(struct kvm_vcpu vcpu, const char smstate) · 283 unmodified lines
7546	7606	return 0;
7547	7607	}
7548	7608
7549		−static void enable_smi_window(struct kvm_vcpu *vcpu)
	7609	+static void vmx_enable_smi_window(struct kvm_vcpu *vcpu)
7550	7610	{
7551	7611	/* RSM will cause a vmexit anyway. */
7552	7612	}
@@ -7606,7 +7666,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { · 53 unmodified lines
7606	7666	.vcpu_load = vmx_vcpu_load,
7607	7667	.vcpu_put = vmx_vcpu_put,
7608	7668
7609		− .update_exception_bitmap = update_exception_bitmap,
	7669	+ .update_exception_bitmap = vmx_update_exception_bitmap,
7610	7670	.get_msr_feature = vmx_get_msr_feature,
7611	7671	.get_msr = vmx_get_msr,
7612	7672	.set_msr = vmx_set_msr,
@@ -7649,9 +7709,9 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { · 36 unmodified lines
7649	7709	.nmi_allowed = vmx_nmi_allowed,
7650	7710	.get_nmi_mask = vmx_get_nmi_mask,
7651	7711	.set_nmi_mask = vmx_set_nmi_mask,
7652		− .enable_nmi_window = enable_nmi_window,
7653		− .enable_irq_window = enable_irq_window,
7654		− .update_cr8_intercept = update_cr8_intercept,
	7712	+ .enable_nmi_window = vmx_enable_nmi_window,
	7713	+ .enable_irq_window = vmx_enable_irq_window,
	7714	+ .update_cr8_intercept = vmx_update_cr8_intercept,
7655	7715	.set_virtual_apic_mode = vmx_set_virtual_apic_mode,
7656	7716	.set_apic_access_page_addr = vmx_set_apic_access_page_addr,
7657	7717	.refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
@@ -7709,7 +7769,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { · 51 unmodified lines
7709	7769	.smi_allowed = vmx_smi_allowed,
7710	7770	.pre_enter_smm = vmx_pre_enter_smm,
7711	7771	.pre_leave_smm = vmx_pre_leave_smm,
7712		− .enable_smi_window = enable_smi_window,
	7772	+ .enable_smi_window = vmx_enable_smi_window,
7713	7773
7714	7774	.can_emulate_instruction = vmx_can_emulate_instruction,
7715	7775	.apic_init_signal_blocked = vmx_apic_init_signal_blocked,
@@ -7810,6 +7870,8 @@ static __init int hardware_setup(void) · 94 unmodified lines
7810	7870	kvm_tsc_scaling_ratio_frac_bits = 48;
7811	7871	}
7812	7872
	7873	+ kvm_has_bus_lock_exit = cpu_has_vmx_bus_lock_detection();
	7874	+
7813	7875	set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
7814	7876
7815	7877	if (enable_ept)

arch/x86/kvm/vmx/vmx.h+54−2

@@ -70,6 +70,54 @@ struct pt_desc { · 69 unmodified lines
70	70	struct pt_ctx guest;
71	71	};
72	72
	73	+union vmx_exit_reason {
	74	+ struct {
	75	+ u32 basic : 16;
	76	+ u32 reserved16 : 1;
	77	+ u32 reserved17 : 1;
	78	+ u32 reserved18 : 1;
	79	+ u32 reserved19 : 1;
	80	+ u32 reserved20 : 1;
	81	+ u32 reserved21 : 1;
	82	+ u32 reserved22 : 1;
	83	+ u32 reserved23 : 1;
	84	+ u32 reserved24 : 1;
	85	+ u32 reserved25 : 1;
	86	+ u32 bus_lock_detected : 1;
	87	+ u32 enclave_mode : 1;
	88	+ u32 smi_pending_mtf : 1;
	89	+ u32 smi_from_vmx_root : 1;
	90	+ u32 reserved30 : 1;
	91	+ u32 failed_vmentry : 1;
	92	+ };
	93	+ u32 full;
	94	+};
	95	+
	96	+#define vcpu_to_lbr_desc(vcpu) (&to_vmx(vcpu)->lbr_desc)
	97	+#define vcpu_to_lbr_records(vcpu) (&to_vmx(vcpu)->lbr_desc.records)
	98	+
	99	+bool intel_pmu_lbr_is_compatible(struct kvm_vcpu *vcpu);
	100	+bool intel_pmu_lbr_is_enabled(struct kvm_vcpu *vcpu);
	101	+
	102	+int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu);
	103	+void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu);
	104	+
	105	+struct lbr_desc {
	106	+ /* Basic info about guest LBR records. */
	107	+ struct x86_pmu_lbr records;
	108	+
	109	+ /*
	110	+ * Emulate LBR feature via passthrough LBR registers when the
	111	+ * per-vcpu guest LBR event is scheduled on the current pcpu.
	112	+ *
	113	+ * The records may be inaccurate if the host reclaims the LBR.
	114	+ */
	115	+ struct perf_event *event;
	116	+
	117	+ /* True if LBRs are marked as not intercepted in the MSR bitmap */
	118	+ bool msr_passthrough;
	119	+};
	120	+
73	121	/*
74	122	* The nested_vmx structure is part of vcpu_vmx, and holds information we need
75	123	* for correct emulation of VMX (i.e., nested VMX) on this vcpu.
@@ -244,7 +292,7 @@ struct vcpu_vmx { · 168 unmodified lines
244	292	int vpid;
245	293	bool emulation_required;
246	294
247		− u32 exit_reason;
	295	+ union vmx_exit_reason exit_reason;
248	296
249	297	/* Posted interrupt descriptor */
250	298	struct pi_desc pi_desc;
@@ -279,6 +327,7 @@ struct vcpu_vmx { · 28 unmodified lines
279	327	u64 ept_pointer;
280	328
281	329	struct pt_desc pt_desc;
	330	+ struct lbr_desc lbr_desc;
282	331
283	332	/* Save desired MSR intercept (read: pass-through) state */
284	333	#define MAX_POSSIBLE_PASSTHROUGH_MSRS 13
@@ -329,7 +378,7 @@ void vmx_set_segment(struct kvm_vcpu vcpu, struct kvm_segment var, int seg); · 44 unmodified lines
329	378	u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa,
330	379	int root_level);
331	380
332		−void update_exception_bitmap(struct kvm_vcpu *vcpu);
	381	+void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu);
333	382	void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu);
334	383	bool vmx_nmi_blocked(struct kvm_vcpu *vcpu);
335	384	bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu);
@@ -339,8 +388,11 @@ void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu); · 3 unmodified lines
339	388	struct vmx_uret_msr vmx_find_uret_msr(struct vcpu_vmx vmx, u32 msr);
340	389	void pt_update_intercept_for_msr(struct kvm_vcpu *vcpu);
341	390	void vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp);
	391	+bool __vmx_vcpu_run(struct vcpu_vmx vmx, unsigned long regs, bool launched);
342	392	int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr);
343	393	void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu);
	394	+void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu,
	395	+ u32 msr, int type, bool value);
344	396
345	397	static inline u8 vmx_get_rvi(void)
346	398	{

arch/x86/kvm/x86.c+392−326

@@ -29,6 +29,7 @@ · 28 unmodified lines
29	29	#include "pmu.h"
30	30	#include "hyperv.h"
31	31	#include "lapic.h"
	32	+#include "xen.h"
32	33
33	34	#include <linux/clocksource.h>
34	35	#include <linux/interrupt.h>
@@ -114,11 +115,21 @@ static int sync_regs(struct kvm_vcpu *vcpu); · 79 unmodified lines
114	115	struct kvm_x86_ops kvm_x86_ops __read_mostly;
115	116	EXPORT_SYMBOL_GPL(kvm_x86_ops);
116	117
	118	+#define KVM_X86_OP(func) \
	119	+ DEFINE_STATIC_CALL_NULL(kvm_x86_##func, \
	120	+ (((struct kvm_x86_ops )0)->func));
	121	+#define KVM_X86_OP_NULL KVM_X86_OP
	122	+#include <asm/kvm-x86-ops.h>
	123	+EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits);
	124	+EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg);
	125	+EXPORT_STATIC_CALL_GPL(kvm_x86_tlb_flush_current);
	126	+
117	127	static bool __read_mostly ignore_msrs = 0;
118	128	module_param(ignore_msrs, bool, S_IRUGO \| S_IWUSR);
119	129
120		−static bool __read_mostly report_ignored_msrs = true;
	130	+bool __read_mostly report_ignored_msrs = true;
121	131	module_param(report_ignored_msrs, bool, S_IRUGO \| S_IWUSR);
	132	+EXPORT_SYMBOL_GPL(report_ignored_msrs);
122	133
123	134	unsigned int min_timer_period_us = 200;
124	135	module_param(min_timer_period_us, uint, S_IRUGO \| S_IWUSR);
@@ -136,6 +147,8 @@ u64 __read_mostly kvm_max_tsc_scaling_ratio; · 11 unmodified lines
136	147	EXPORT_SYMBOL_GPL(kvm_max_tsc_scaling_ratio);
137	148	u64 __read_mostly kvm_default_tsc_scaling_ratio;
138	149	EXPORT_SYMBOL_GPL(kvm_default_tsc_scaling_ratio);
	150	+bool __read_mostly kvm_has_bus_lock_exit;
	151	+EXPORT_SYMBOL_GPL(kvm_has_bus_lock_exit);
139	152
140	153	/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
141	154	static u32 __read_mostly tsc_tolerance_ppm = 250;
@@ -234,7 +247,6 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { · 92 unmodified lines
234	247	VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns),
235	248	VM_STAT("mmu_shadow_zapped", mmu_shadow_zapped),
236	249	VM_STAT("mmu_pte_write", mmu_pte_write),
237		− VM_STAT("mmu_pte_updated", mmu_pte_updated),
238	250	VM_STAT("mmu_pde_zapped", mmu_pde_zapped),
239	251	VM_STAT("mmu_flooded", mmu_flooded),
240	252	VM_STAT("mmu_recycled", mmu_recycled),
@@ -395,7 +407,7 @@ int kvm_set_apic_base(struct kvm_vcpu vcpu, struct msr_data msr_info) · 154 unmodified lines
395	407	{
396	408	enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
397	409	enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);
398		− u64 reserved_bits = ((~0ULL) << cpuid_maxphyaddr(vcpu)) \| 0x2ff \|
	410	+ u64 reserved_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu) \| 0x2ff \|
399	411	(guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);
400	412
401	413	if ((msr_info->data & reserved_bits) != 0 \|\| new_mode == LAPIC_MODE_INVALID)
@@ -484,19 +496,24 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu) · 82 unmodified lines
484	496	*/
485	497	vcpu->arch.dr6 &= ~DR_TRAP_BITS;
486	498	/*
487		− * DR6.RTM is set by all #DB exceptions that don't clear it.
	499	+ * In order to reflect the #DB exception payload in guest
	500	+ * dr6, three components need to be considered: active low
	501	+ * bit, FIXED_1 bits and active high bits (e.g. DR6_BD,
	502	+ * DR6_BS and DR6_BT)
	503	+ * DR6_ACTIVE_LOW contains the FIXED_1 and active low bits.
	504	+ * In the target guest dr6:
	505	+ * FIXED_1 bits should always be set.
	506	+ * Active low bits should be cleared if 1-setting in payload.
	507	+ * Active high bits should be set if 1-setting in payload.
	508	+ *
	509	+ * Note, the payload is compatible with the pending debug
	510	+ * exceptions/exit qualification under VMX, that active_low bits
	511	+ * are active high in payload.
	512	+ * So they need to be flipped for DR6.
488	513	*/
489		− vcpu->arch.dr6 \|= DR6_RTM;
	514	+ vcpu->arch.dr6 \|= DR6_ACTIVE_LOW;
490	515	vcpu->arch.dr6 \|= payload;
491		− /*
492		− * Bit 16 should be set in the payload whenever the #DB
493		− * exception should clear DR6.RTM. This makes the payload
494		− * compatible with the pending debug exceptions under VMX.
495		− * Though not currently documented in the SDM, this also
496		− * makes the payload compatible with the exit qualification
497		− * for #DB exceptions under VMX.
498		− */
499		− vcpu->arch.dr6 ^= payload & DR6_RTM;
	516	+ vcpu->arch.dr6 ^= payload & DR6_ACTIVE_LOW;
500	517
501	518	/*
502	519	* The #DB payload is defined as compatible with the 'pending
@@ -692,7 +709,7 @@ EXPORT_SYMBOL_GPL(kvm_requeue_exception_e); · 189 unmodified lines
692	709	*/
693	710	bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
694	711	{
695		− if (kvm_x86_ops.get_cpl(vcpu) <= required_cpl)
	712	+ if (static_call(kvm_x86_get_cpl)(vcpu) <= required_cpl)
696	713	return true;
697	714	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
698	715	return false;
@@ -742,8 +759,7 @@ static int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, · 43 unmodified lines
742	759
743	760	static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
744	761	{
745		− return rsvd_bits(cpuid_maxphyaddr(vcpu), 63) \| rsvd_bits(5, 8) \|
746		− rsvd_bits(1, 2);
	762	+ return vcpu->arch.reserved_gpa_bits \| rsvd_bits(5, 8) \| rsvd_bits(1, 2);
747	763	}
748	764
749	765	/*
@@ -852,7 +868,7 @@ int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) · 102 unmodified lines
852	868
853	869	if (!is_pae(vcpu))
854	870	return 1;
855		− kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
	871	+ static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
856	872	if (cs_l)
857	873	return 1;
858	874	}
@@ -865,7 +881,7 @@ int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) · 6 unmodified lines
865	881	if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
866	882	return 1;
867	883
868		− kvm_x86_ops.set_cr0(vcpu, cr0);
	884	+ static_call(kvm_x86_set_cr0)(vcpu, cr0);
869	885
870	886	kvm_post_set_cr0(vcpu, old_cr0, cr0);
871	887
@@ -970,12 +986,10 @@ static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) · 98 unmodified lines
970	986
971	987	int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
972	988	{
973		− if (kvm_x86_ops.get_cpl(vcpu) != 0 \|\|
974		− __kvm_set_xcr(vcpu, index, xcr)) {
975		− kvm_inject_gp(vcpu, 0);
976		− return 1;
977		− }
978		− return 0;
	989	+ if (static_call(kvm_x86_get_cpl)(vcpu) == 0)
	990	+ return __kvm_set_xcr(vcpu, index, xcr);
	991	+
	992	+ return 1;
979	993	}
980	994	EXPORT_SYMBOL_GPL(kvm_set_xcr);
981	995
@@ -987,7 +1001,7 @@ bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) · 5 unmodified lines
987	1001	if (cr4 & vcpu->arch.cr4_guest_rsvd_bits)
988	1002	return false;
989	1003
990		− return kvm_x86_ops.is_valid_cr4(vcpu, cr4);
	1004	+ return static_call(kvm_x86_is_valid_cr4)(vcpu, cr4);
991	1005	}
992	1006	EXPORT_SYMBOL_GPL(kvm_is_valid_cr4);
993	1007
@@ -1031,7 +1045,7 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) · 37 unmodified lines
1031	1045	return 1;
1032	1046	}
1033	1047
1034		− kvm_x86_ops.set_cr4(vcpu, cr4);
	1048	+ static_call(kvm_x86_set_cr4)(vcpu, cr4);
1035	1049
1036	1050	kvm_post_set_cr4(vcpu, old_cr4, cr4);
1037	1051
@@ -1059,8 +1073,7 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) · 21 unmodified lines
1059	1073	return 0;
1060	1074	}
1061	1075
1062		− if (is_long_mode(vcpu) &&
1063		− (cr3 & vcpu->arch.cr3_lm_rsvd_bits))
	1076	+ if (is_long_mode(vcpu) && kvm_vcpu_is_illegal_gpa(vcpu, cr3))
1064	1077	return 1;
1065	1078	else if (is_pae_paging(vcpu) &&
1066	1079	!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
@@ -1114,7 +1127,7 @@ void kvm_update_dr7(struct kvm_vcpu *vcpu) · 47 unmodified lines
1114	1127	dr7 = vcpu->arch.guest_debug_dr7;
1115	1128	else
1116	1129	dr7 = vcpu->arch.dr7;
1117		− kvm_x86_ops.set_dr7(vcpu, dr7);
	1130	+ static_call(kvm_x86_set_dr7)(vcpu, dr7);
1118	1131	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
1119	1132	if (dr7 & DR7_BP_EN_MASK)
1120	1133	vcpu->arch.switch_db_regs \|= KVM_DEBUGREG_BP_ENABLED;
@@ -1130,7 +1143,7 @@ static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) · 9 unmodified lines
1130	1143	return fixed;
1131	1144	}
1132	1145
1133		−static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
	1146	+int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1134	1147	{
1135	1148	size_t size = ARRAY_SIZE(vcpu->arch.db);
1136	1149
@@ -1143,13 +1156,13 @@ static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) · 6 unmodified lines
1143	1156	case 4:
1144	1157	case 6:
1145	1158	if (!kvm_dr6_valid(val))
1146		− return -1; /* #GP */
	1159	+ return 1; /* #GP */
1147	1160	vcpu->arch.dr6 = (val & DR6_VOLATILE) \| kvm_dr6_fixed(vcpu);
1148	1161	break;
1149	1162	case 5:
1150	1163	default: /* 7 */
1151	1164	if (!kvm_dr7_valid(val))
1152		− return -1; /* #GP */
	1165	+ return 1; /* #GP */
1153	1166	vcpu->arch.dr7 = (val & DR7_VOLATILE) \| DR7_FIXED_1;
1154	1167	kvm_update_dr7(vcpu);
1155	1168	break;
@@ -1157,18 +1170,9 @@ static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) · 1 unmodified line
1157	1170
1158	1171	return 0;
1159	1172	}
1160		−
1161		−int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1162		−{
1163		− if (__kvm_set_dr(vcpu, dr, val)) {
1164		− kvm_inject_gp(vcpu, 0);
1165		− return 1;
1166		− }
1167		− return 0;
1168		−}
1169	1173	EXPORT_SYMBOL_GPL(kvm_set_dr);
1170	1174
1171		−int kvm_get_dr(struct kvm_vcpu vcpu, int dr, unsigned long val)
	1175	+void kvm_get_dr(struct kvm_vcpu vcpu, int dr, unsigned long val)
1172	1176	{
1173	1177	size_t size = ARRAY_SIZE(vcpu->arch.db);
1174	1178
@@ -1185,7 +1189,6 @@ int kvm_get_dr(struct kvm_vcpu vcpu, int dr, unsigned long val) · 10 unmodified lines
1185	1189	*val = vcpu->arch.dr7;
1186	1190	break;
1187	1191	}
1188		− return 0;
1189	1192	}
1190	1193	EXPORT_SYMBOL_GPL(kvm_get_dr);
1191	1194
@@ -1426,7 +1429,7 @@ static int kvm_get_msr_feature(struct kvm_msr_entry *msr) · 234 unmodified lines
1426	1429	rdmsrl_safe(msr->index, &msr->data);
1427	1430	break;
1428	1431	default:
1429		− return kvm_x86_ops.get_msr_feature(msr);
	1432	+ return static_call(kvm_x86_get_msr_feature)(msr);
1430	1433	}
1431	1434	return 0;
1432	1435	}
@@ -1502,7 +1505,7 @@ static int set_efer(struct kvm_vcpu vcpu, struct msr_data msr_info) · 69 unmodified lines
1502	1505	efer &= ~EFER_LMA;
1503	1506	efer \|= vcpu->arch.efer & EFER_LMA;
1504	1507
1505		− r = kvm_x86_ops.set_efer(vcpu, efer);
	1508	+ r = static_call(kvm_x86_set_efer)(vcpu, efer);
1506	1509	if (r) {
1507	1510	WARN_ON(r > 0);
1508	1511	return r;
@@ -1599,7 +1602,7 @@ static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data, · 90 unmodified lines
1599	1602	msr.index = index;
1600	1603	msr.host_initiated = host_initiated;
1601	1604
1602		− return kvm_x86_ops.set_msr(vcpu, &msr);
	1605	+ return static_call(kvm_x86_set_msr)(vcpu, &msr);
1603	1606	}
1604	1607
1605	1608	static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,
@@ -1632,7 +1635,7 @@ int __kvm_get_msr(struct kvm_vcpu vcpu, u32 index, u64 data, · 26 unmodified lines
1632	1635	msr.index = index;
1633	1636	msr.host_initiated = host_initiated;
1634	1637
1635		− ret = kvm_x86_ops.get_msr(vcpu, &msr);
	1638	+ ret = static_call(kvm_x86_get_msr)(vcpu, &msr);
1636	1639	if (!ret)
1637	1640	*data = msr.data;
1638	1641	return ret;
@@ -1673,12 +1676,12 @@ static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu) · 34 unmodified lines
1673	1676	kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);
1674	1677	}
1675	1678
1676		− return kvm_x86_ops.complete_emulated_msr(vcpu, err);
	1679	+ return static_call(kvm_x86_complete_emulated_msr)(vcpu, err);
1677	1680	}
1678	1681
1679	1682	static int complete_emulated_wrmsr(struct kvm_vcpu *vcpu)
1680	1683	{
1681		− return kvm_x86_ops.complete_emulated_msr(vcpu, vcpu->run->msr.error);
	1684	+ return static_call(kvm_x86_complete_emulated_msr)(vcpu, vcpu->run->msr.error);
1682	1685	}
1683	1686
1684	1687	static u64 kvm_msr_reason(int r)
@@ -1750,7 +1753,7 @@ int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu) · 65 unmodified lines
1750	1753	trace_kvm_msr_read_ex(ecx);
1751	1754	}
1752	1755
1753		− return kvm_x86_ops.complete_emulated_msr(vcpu, r);
	1756	+ return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);
1754	1757	}
1755	1758	EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);
1756	1759
@@ -1776,17 +1779,16 @@ int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu) · 19 unmodified lines
1776	1779	else
1777	1780	trace_kvm_msr_write_ex(ecx, data);
1778	1781
1779		− return kvm_x86_ops.complete_emulated_msr(vcpu, r);
	1782	+ return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);
1780	1783	}
1781	1784	EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);
1782	1785
1783		−bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)
	1786	+static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)
1784	1787	{
1785	1788	xfer_to_guest_mode_prepare();
1786	1789	return vcpu->mode == EXITING_GUEST_MODE \|\| kvm_request_pending(vcpu) \|\|
1787	1790	xfer_to_guest_mode_work_pending();
1788	1791	}
1789		−EXPORT_SYMBOL_GPL(kvm_vcpu_exit_request);
1790	1792
1791	1793	/*
1792	1794	* The fast path for frequent and performance sensitive wrmsr emulation,
@@ -1936,15 +1938,14 @@ static s64 get_kvmclock_base_ns(void) · 143 unmodified lines
1936	1938	}
1937	1939	#endif
1938	1940
1939		−static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
	1941	+void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs)
1940	1942	{
1941	1943	int version;
1942	1944	int r;
1943	1945	struct pvclock_wall_clock wc;
	1946	+ u32 wc_sec_hi;
1944	1947	u64 wall_nsec;
1945	1948
1946		− kvm->arch.wall_clock = wall_clock;
1947		−
1948	1949	if (!wall_clock)
1949	1950	return;
1950	1951
@@ -1973,6 +1974,12 @@ static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock) · 22 unmodified lines
1973	1974
1974	1975	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
1975	1976
	1977	+ if (sec_hi_ofs) {
	1978	+ wc_sec_hi = wall_nsec >> 32;
	1979	+ kvm_write_guest(kvm, wall_clock + sec_hi_ofs,
	1980	+ &wc_sec_hi, sizeof(wc_sec_hi));
	1981	+ }
	1982	+
1976	1983	version++;
1977	1984	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
1978	1985	}
@@ -2209,7 +2216,7 @@ EXPORT_SYMBOL_GPL(kvm_read_l1_tsc); · 230 unmodified lines
2209	2216	static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
2210	2217	{
2211	2218	vcpu->arch.l1_tsc_offset = offset;
2212		− vcpu->arch.tsc_offset = kvm_x86_ops.write_l1_tsc_offset(vcpu, offset);
	2219	+ vcpu->arch.tsc_offset = static_call(kvm_x86_write_l1_tsc_offset)(vcpu, offset);
2213	2220	}
2214	2221
2215	2222	static inline bool kvm_check_tsc_unstable(void)
@@ -2592,13 +2599,15 @@ u64 get_kvmclock_ns(struct kvm *kvm) · 376 unmodified lines
2592	2599	return ret;
2593	2600	}
2594	2601
2595		−static void kvm_setup_pvclock_page(struct kvm_vcpu *v)
	2602	+static void kvm_setup_pvclock_page(struct kvm_vcpu *v,
	2603	+ struct gfn_to_hva_cache *cache,
	2604	+ unsigned int offset)
2596	2605	{
2597	2606	struct kvm_vcpu_arch *vcpu = &v->arch;
2598	2607	struct pvclock_vcpu_time_info guest_hv_clock;
2599	2608
2600		− if (unlikely(kvm_read_guest_cached(v->kvm, &vcpu->pv_time,
2601		− &guest_hv_clock, sizeof(guest_hv_clock))))
	2609	+ if (unlikely(kvm_read_guest_offset_cached(v->kvm, cache,
	2610	+ &guest_hv_clock, offset, sizeof(guest_hv_clock))))
2602	2611	return;
2603	2612
2604	2613	/* This VCPU is paused, but it's legal for a guest to read another
@@ -2621,9 +2630,9 @@ static void kvm_setup_pvclock_page(struct kvm_vcpu *v) · 16 unmodified lines
2621	2630	++guest_hv_clock.version; /* first time write, random junk */
2622	2631
2623	2632	vcpu->hv_clock.version = guest_hv_clock.version + 1;
2624		− kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
2625		− &vcpu->hv_clock,
2626		− sizeof(vcpu->hv_clock.version));
	2633	+ kvm_write_guest_offset_cached(v->kvm, cache,
	2634	+ &vcpu->hv_clock, offset,
	2635	+ sizeof(vcpu->hv_clock.version));
2627	2636
2628	2637	smp_wmb();
2629	2638
@@ -2637,16 +2646,16 @@ static void kvm_setup_pvclock_page(struct kvm_vcpu *v) · 7 unmodified lines
2637	2646
2638	2647	trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);
2639	2648
2640		− kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
2641		− &vcpu->hv_clock,
2642		− sizeof(vcpu->hv_clock));
	2649	+ kvm_write_guest_offset_cached(v->kvm, cache,
	2650	+ &vcpu->hv_clock, offset,
	2651	+ sizeof(vcpu->hv_clock));
2643	2652
2644	2653	smp_wmb();
2645	2654
2646	2655	vcpu->hv_clock.version++;
2647		− kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
2648		− &vcpu->hv_clock,
2649		− sizeof(vcpu->hv_clock.version));
	2656	+ kvm_write_guest_offset_cached(v->kvm, cache,
	2657	+ &vcpu->hv_clock, offset,
	2658	+ sizeof(vcpu->hv_clock.version));
2650	2659	}
2651	2660
2652	2661	static int kvm_guest_time_update(struct kvm_vcpu *v)
@@ -2733,7 +2742,12 @@ static int kvm_guest_time_update(struct kvm_vcpu *v) · 80 unmodified lines
2733	2742	vcpu->hv_clock.flags = pvclock_flags;
2734	2743
2735	2744	if (vcpu->pv_time_enabled)
2736		− kvm_setup_pvclock_page(v);
	2745	+ kvm_setup_pvclock_page(v, &vcpu->pv_time, 0);
	2746	+ if (vcpu->xen.vcpu_info_set)
	2747	+ kvm_setup_pvclock_page(v, &vcpu->xen.vcpu_info_cache,
	2748	+ offsetof(struct compat_vcpu_info, time));
	2749	+ if (vcpu->xen.vcpu_time_info_set)
	2750	+ kvm_setup_pvclock_page(v, &vcpu->xen.vcpu_time_info_cache, 0);
2737	2751	if (v == kvm_get_vcpu(v->kvm, 0))
2738	2752	kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);
2739	2753	return 0;
@@ -2858,32 +2872,6 @@ static int set_msr_mce(struct kvm_vcpu vcpu, struct msr_data msr_info) · 118 unmodified lines
2858	2872	return 0;
2859	2873	}
2860	2874
2861		−static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
2862		−{
2863		− struct kvm *kvm = vcpu->kvm;
2864		− int lm = is_long_mode(vcpu);
2865		− u8 blob_addr = lm ? (u8 )(long)kvm->arch.xen_hvm_config.blob_addr_64
2866		− : (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
2867		− u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
2868		− : kvm->arch.xen_hvm_config.blob_size_32;
2869		− u32 page_num = data & ~PAGE_MASK;
2870		− u64 page_addr = data & PAGE_MASK;
2871		− u8 *page;
2872		−
2873		− if (page_num >= blob_size)
2874		− return 1;
2875		−
2876		− page = memdup_user(blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE);
2877		− if (IS_ERR(page))
2878		− return PTR_ERR(page);
2879		−
2880		− if (kvm_vcpu_write_guest(vcpu, page_addr, page, PAGE_SIZE)) {
2881		− kfree(page);
2882		− return 1;
2883		− }
2884		− return 0;
2885		−}
2886		−
2887	2875	static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)
2888	2876	{
2889	2877	u64 mask = KVM_ASYNC_PF_ENABLED \| KVM_ASYNC_PF_DELIVERY_AS_INT;
@@ -2955,13 +2943,13 @@ static void kvmclock_reset(struct kvm_vcpu *vcpu) · 65 unmodified lines
2955	2943	static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)
2956	2944	{
2957	2945	++vcpu->stat.tlb_flush;
2958		− kvm_x86_ops.tlb_flush_all(vcpu);
	2946	+ static_call(kvm_x86_tlb_flush_all)(vcpu);
2959	2947	}
2960	2948
2961	2949	static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)
2962	2950	{
2963	2951	++vcpu->stat.tlb_flush;
2964		− kvm_x86_ops.tlb_flush_guest(vcpu);
	2952	+ static_call(kvm_x86_tlb_flush_guest)(vcpu);
2965	2953	}
2966	2954
2967	2955	static void record_steal_time(struct kvm_vcpu *vcpu)
@@ -3017,6 +3005,9 @@ int kvm_set_msr_common(struct kvm_vcpu vcpu, struct msr_data msr_info) · 49 unmodified lines
3017	3005	u32 msr = msr_info->index;
3018	3006	u64 data = msr_info->data;
3019	3007
	3008	+ if (msr && msr == vcpu->kvm->arch.xen_hvm_config.msr)
	3009	+ return kvm_xen_write_hypercall_page(vcpu, data);
	3010	+
3020	3011	switch (msr) {
3021	3012	case MSR_AMD64_NB_CFG:
3022	3013	case MSR_IA32_UCODE_WRITE:
@@ -3073,18 +3064,6 @@ int kvm_set_msr_common(struct kvm_vcpu vcpu, struct msr_data msr_info) · 50 unmodified lines
3073	3064	return 1;
3074	3065	}
3075	3066	break;
3076		− case MSR_IA32_DEBUGCTLMSR:
3077		− if (!data) {
3078		− /* We support the non-activated case already */
3079		− break;
3080		− } else if (data & ~(DEBUGCTLMSR_LBR \| DEBUGCTLMSR_BTF)) {
3081		− /* Values other than LBR and BTF are vendor-specific,
3082		− thus reserved and should throw a #GP */
3083		− return 1;
3084		− } else if (report_ignored_msrs)
3085		− vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
3086		− __func__, data);
3087		− break;
3088	3067	case 0x200 ... 0x2ff:
3089	3068	return kvm_mtrr_set_msr(vcpu, msr, data);
3090	3069	case MSR_IA32_APICBASE:
@@ -3153,13 +3132,15 @@ int kvm_set_msr_common(struct kvm_vcpu vcpu, struct msr_data msr_info) · 62 unmodified lines
3153	3132	if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3154	3133	return 1;
3155	3134
3156		− kvm_write_wall_clock(vcpu->kvm, data);
	3135	+ vcpu->kvm->arch.wall_clock = data;
	3136	+ kvm_write_wall_clock(vcpu->kvm, data, 0);
3157	3137	break;
3158	3138	case MSR_KVM_WALL_CLOCK:
3159	3139	if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3160	3140	return 1;
3161	3141
3162		− kvm_write_wall_clock(vcpu->kvm, data);
	3142	+ vcpu->kvm->arch.wall_clock = data;
	3143	+ kvm_write_wall_clock(vcpu->kvm, data, 0);
3163	3144	break;
3164	3145	case MSR_KVM_SYSTEM_TIME_NEW:
3165	3146	if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
@@ -3304,8 +3285,6 @@ int kvm_set_msr_common(struct kvm_vcpu vcpu, struct msr_data msr_info) · 138 unmodified lines
3304	3285	vcpu->arch.msr_misc_features_enables = data;
3305	3286	break;
3306	3287	default:
3307		− if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
3308		− return xen_hvm_config(vcpu, data);
3309	3288	if (kvm_pmu_is_valid_msr(vcpu, msr))
3310	3289	return kvm_pmu_set_msr(vcpu, msr_info);
3311	3290	return KVM_MSR_RET_INVALID;
@@ -3357,7 +3336,6 @@ int kvm_get_msr_common(struct kvm_vcpu vcpu, struct msr_data msr_info) · 45 unmodified lines
3357	3336	switch (msr_info->index) {
3358	3337	case MSR_IA32_PLATFORM_ID:
3359	3338	case MSR_IA32_EBL_CR_POWERON:
3360		− case MSR_IA32_DEBUGCTLMSR:
3361	3339	case MSR_IA32_LASTBRANCHFROMIP:
3362	3340	case MSR_IA32_LASTBRANCHTOIP:
3363	3341	case MSR_IA32_LASTINTFROMIP:
@@ -3739,7 +3717,6 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 375 unmodified lines
3739	3717	case KVM_CAP_PIT2:
3740	3718	case KVM_CAP_PIT_STATE2:
3741	3719	case KVM_CAP_SET_IDENTITY_MAP_ADDR:
3742		− case KVM_CAP_XEN_HVM:
3743	3720	case KVM_CAP_VCPU_EVENTS:
3744	3721	case KVM_CAP_HYPERV:
3745	3722	case KVM_CAP_HYPERV_VAPIC:
@@ -3779,6 +3756,11 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 33 unmodified lines
3779	3756	case KVM_CAP_ENFORCE_PV_FEATURE_CPUID:
3780	3757	r = 1;
3781	3758	break;
	3759	+ case KVM_CAP_XEN_HVM:
	3760	+ r = KVM_XEN_HVM_CONFIG_HYPERCALL_MSR \|
	3761	+ KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL \|
	3762	+ KVM_XEN_HVM_CONFIG_SHARED_INFO;
	3763	+ break;
3782	3764	case KVM_CAP_SYNC_REGS:
3783	3765	r = KVM_SYNC_X86_VALID_FIELDS;
3784	3766	break;
@@ -3800,10 +3782,10 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 15 unmodified lines
3800	3782	* fringe case that is not enabled except via specific settings
3801	3783	* of the module parameters.
3802	3784	*/
3803		− r = kvm_x86_ops.has_emulated_msr(kvm, MSR_IA32_SMBASE);
	3785	+ r = static_call(kvm_x86_has_emulated_msr)(kvm, MSR_IA32_SMBASE);
3804	3786	break;
3805	3787	case KVM_CAP_VAPIC:
3806		− r = !kvm_x86_ops.cpu_has_accelerated_tpr();
	3788	+ r = !static_call(kvm_x86_cpu_has_accelerated_tpr)();
3807	3789	break;
3808	3790	case KVM_CAP_NR_VCPUS:
3809	3791	r = KVM_SOFT_MAX_VCPUS;
@@ -3845,6 +3827,13 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 35 unmodified lines
3845	3827	case KVM_CAP_STEAL_TIME:
3846	3828	r = sched_info_on();
3847	3829	break;
	3830	+ case KVM_CAP_X86_BUS_LOCK_EXIT:
	3831	+ if (kvm_has_bus_lock_exit)
	3832	+ r = KVM_BUS_LOCK_DETECTION_OFF \|
	3833	+ KVM_BUS_LOCK_DETECTION_EXIT;
	3834	+ else
	3835	+ r = 0;
	3836	+ break;
3848	3837	default:
3849	3838	break;
3850	3839	}
@@ -3962,14 +3951,14 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) · 111 unmodified lines
3962	3951	{
3963	3952	/* Address WBINVD may be executed by guest */
3964	3953	if (need_emulate_wbinvd(vcpu)) {
3965		− if (kvm_x86_ops.has_wbinvd_exit())
	3954	+ if (static_call(kvm_x86_has_wbinvd_exit)())
3966	3955	cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
3967	3956	else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
3968	3957	smp_call_function_single(vcpu->cpu,
3969	3958	wbinvd_ipi, NULL, 1);
3970	3959	}
3971	3960
3972		− kvm_x86_ops.vcpu_load(vcpu, cpu);
	3961	+ static_call(kvm_x86_vcpu_load)(vcpu, cpu);
3973	3962
3974	3963	/* Save host pkru register if supported */
3975	3964	vcpu->arch.host_pkru = read_pkru();
@@ -4015,6 +4004,7 @@ static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu) · 39 unmodified lines
4015	4004	{
4016	4005	struct kvm_host_map map;
4017	4006	struct kvm_steal_time *st;
	4007	+ int idx;
4018	4008
4019	4009	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
4020	4010	return;
@@ -4022,9 +4012,15 @@ static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu) · 1 unmodified line
4022	4012	if (vcpu->arch.st.preempted)
4023	4013	return;
4024	4014
	4015	+ /*
	4016	+ * Take the srcu lock as memslots will be accessed to check the gfn
	4017	+ * cache generation against the memslots generation.
	4018	+ */
	4019	+ idx = srcu_read_lock(&vcpu->kvm->srcu);
	4020	+
4025	4021	if (kvm_map_gfn(vcpu, vcpu->arch.st.msr_val >> PAGE_SHIFT, &map,
4026	4022	&vcpu->arch.st.cache, true))
4027		− return;
	4023	+ goto out;
4028	4024
4029	4025	st = map.hva +
4030	4026	offset_in_page(vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS);
@@ -4032,33 +4028,18 @@ static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu) · 1 unmodified line
4032	4028	st->preempted = vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;
4033	4029
4034	4030	kvm_unmap_gfn(vcpu, &map, &vcpu->arch.st.cache, true, true);
	4031	+
	4032	+out:
	4033	+ srcu_read_unlock(&vcpu->kvm->srcu, idx);
4035	4034	}
4036	4035
4037	4036	void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
4038	4037	{
4039		− int idx;
4040		−
4041	4038	if (vcpu->preempted && !vcpu->arch.guest_state_protected)
4042		− vcpu->arch.preempted_in_kernel = !kvm_x86_ops.get_cpl(vcpu);
	4039	+ vcpu->arch.preempted_in_kernel = !static_call(kvm_x86_get_cpl)(vcpu);
4043	4040
4044		− /*
4045		− * Disable page faults because we're in atomic context here.
4046		− * kvm_write_guest_offset_cached() would call might_fault()
4047		− * that relies on pagefault_disable() to tell if there's a
4048		− * bug. NOTE: the write to guest memory may not go through if
4049		− * during postcopy live migration or if there's heavy guest
4050		− * paging.
4051		− */
4052		− pagefault_disable();
4053		− /*
4054		− * kvm_memslots() will be called by
4055		− * kvm_write_guest_offset_cached() so take the srcu lock.
4056		− */
4057		− idx = srcu_read_lock(&vcpu->kvm->srcu);
4058	4041	kvm_steal_time_set_preempted(vcpu);
4059		− srcu_read_unlock(&vcpu->kvm->srcu, idx);
4060		− pagefault_enable();
4061		− kvm_x86_ops.vcpu_put(vcpu);
	4042	+ static_call(kvm_x86_vcpu_put)(vcpu);
4062	4043	vcpu->arch.last_host_tsc = rdtsc();
4063	4044	/*
4064	4045	* If userspace has set any breakpoints or watchpoints, dr6 is restored
@@ -4072,7 +4053,7 @@ static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, · 7 unmodified lines
4072	4053	struct kvm_lapic_state *s)
4073	4054	{
4074	4055	if (vcpu->arch.apicv_active)
4075		− kvm_x86_ops.sync_pir_to_irr(vcpu);
	4056	+ static_call(kvm_x86_sync_pir_to_irr)(vcpu);
4076	4057
4077	4058	return kvm_apic_get_state(vcpu, s);
4078	4059	}
@@ -4182,7 +4163,7 @@ static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu, · 103 unmodified lines
4182	4163	for (bank = 0; bank < bank_num; bank++)
4183	4164	vcpu->arch.mce_banks[bank*4] = ~(u64)0;
4184	4165
4185		− kvm_x86_ops.setup_mce(vcpu);
	4166	+ static_call(kvm_x86_setup_mce)(vcpu);
4186	4167	out:
4187	4168	return r;
4188	4169	}
@@ -4289,11 +4270,11 @@ static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, · 100 unmodified lines
4289	4270	vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;
4290	4271	events->interrupt.nr = vcpu->arch.interrupt.nr;
4291	4272	events->interrupt.soft = 0;
4292		− events->interrupt.shadow = kvm_x86_ops.get_interrupt_shadow(vcpu);
	4273	+ events->interrupt.shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
4293	4274
4294	4275	events->nmi.injected = vcpu->arch.nmi_injected;
4295	4276	events->nmi.pending = vcpu->arch.nmi_pending != 0;
4296		− events->nmi.masked = kvm_x86_ops.get_nmi_mask(vcpu);
	4277	+ events->nmi.masked = static_call(kvm_x86_get_nmi_mask)(vcpu);
4297	4278	events->nmi.pad = 0;
4298	4279
4299	4280	events->sipi_vector = 0; /* never valid when reporting to user space */
@@ -4360,13 +4341,13 @@ static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, · 60 unmodified lines
4360	4341	vcpu->arch.interrupt.nr = events->interrupt.nr;
4361	4342	vcpu->arch.interrupt.soft = events->interrupt.soft;
4362	4343	if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
4363		− kvm_x86_ops.set_interrupt_shadow(vcpu,
4364		− events->interrupt.shadow);
	4344	+ static_call(kvm_x86_set_interrupt_shadow)(vcpu,
	4345	+ events->interrupt.shadow);
4365	4346
4366	4347	vcpu->arch.nmi_injected = events->nmi.injected;
4367	4348	if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
4368	4349	vcpu->arch.nmi_pending = events->nmi.pending;
4369		− kvm_x86_ops.set_nmi_mask(vcpu, events->nmi.masked);
	4350	+ static_call(kvm_x86_set_nmi_mask)(vcpu, events->nmi.masked);
4370	4351
4371	4352	if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&
4372	4353	lapic_in_kernel(vcpu))
@@ -4422,9 +4403,9 @@ static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, · 49 unmodified lines
4422	4403	if (dbgregs->flags)
4423	4404	return -EINVAL;
4424	4405
4425		− if (dbgregs->dr6 & ~0xffffffffull)
	4406	+ if (!kvm_dr6_valid(dbgregs->dr6))
4426	4407	return -EINVAL;
4427		− if (dbgregs->dr7 & ~0xffffffffull)
	4408	+ if (!kvm_dr7_valid(dbgregs->dr7))
4428	4409	return -EINVAL;
4429	4410
4430	4411	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
@@ -4661,7 +4642,7 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, · 230 unmodified lines
4661	4642	if (!kvm_x86_ops.enable_direct_tlbflush)
4662	4643	return -ENOTTY;
4663	4644
4664		− return kvm_x86_ops.enable_direct_tlbflush(vcpu);
	4645	+ return static_call(kvm_x86_enable_direct_tlbflush)(vcpu);
4665	4646
4666	4647	case KVM_CAP_ENFORCE_PV_FEATURE_CPUID:
4667	4648	vcpu->arch.pv_cpuid.enforce = cap->args[0];
@@ -5032,6 +5013,26 @@ long kvm_arch_vcpu_ioctl(struct file *filp, · 364 unmodified lines
5032	5013	case KVM_GET_SUPPORTED_HV_CPUID:
5033	5014	r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp);
5034	5015	break;
	5016	+ case KVM_XEN_VCPU_GET_ATTR: {
	5017	+ struct kvm_xen_vcpu_attr xva;
	5018	+
	5019	+ r = -EFAULT;
	5020	+ if (copy_from_user(&xva, argp, sizeof(xva)))
	5021	+ goto out;
	5022	+ r = kvm_xen_vcpu_get_attr(vcpu, &xva);
	5023	+ if (!r && copy_to_user(argp, &xva, sizeof(xva)))
	5024	+ r = -EFAULT;
	5025	+ break;
	5026	+ }
	5027	+ case KVM_XEN_VCPU_SET_ATTR: {
	5028	+ struct kvm_xen_vcpu_attr xva;
	5029	+
	5030	+ r = -EFAULT;
	5031	+ if (copy_from_user(&xva, argp, sizeof(xva)))
	5032	+ goto out;
	5033	+ r = kvm_xen_vcpu_set_attr(vcpu, &xva);
	5034	+ break;
	5035	+ }
5035	5036	default:
5036	5037	r = -EINVAL;
5037	5038	}
@@ -5053,14 +5054,14 @@ static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr) · 15 unmodified lines
5053	5054
5054	5055	if (addr > (unsigned int)(-3 * PAGE_SIZE))
5055	5056	return -EINVAL;
5056		− ret = kvm_x86_ops.set_tss_addr(kvm, addr);
	5057	+ ret = static_call(kvm_x86_set_tss_addr)(kvm, addr);
5057	5058	return ret;
5058	5059	}
5059	5060
5060	5061	static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
5061	5062	u64 ident_addr)
5062	5063	{
5063		− return kvm_x86_ops.set_identity_map_addr(kvm, ident_addr);
	5064	+ return static_call(kvm_x86_set_identity_map_addr)(kvm, ident_addr);
5064	5065	}
5065	5066
5066	5067	static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
@@ -5217,8 +5218,7 @@ void kvm_arch_sync_dirty_log(struct kvm kvm, struct kvm_memory_slot memslot) · 150 unmodified lines
5217	5218	/*
5218	5219	* Flush potentially hardware-cached dirty pages to dirty_bitmap.
5219	5220	*/
5220		− if (kvm_x86_ops.flush_log_dirty)
5221		− kvm_x86_ops.flush_log_dirty(kvm);
	5221	+ static_call_cond(kvm_x86_flush_log_dirty)(kvm);
5222	5222	}
5223	5223
5224	5224	int kvm_vm_ioctl_irq_line(struct kvm kvm, struct kvm_irq_level irq_event,
@@ -5308,6 +5308,20 @@ split_irqchip_unlock: · 83 unmodified lines
5308	5308	kvm->arch.user_space_msr_mask = cap->args[0];
5309	5309	r = 0;
5310	5310	break;
	5311	+ case KVM_CAP_X86_BUS_LOCK_EXIT:
	5312	+ r = -EINVAL;
	5313	+ if (cap->args[0] & ~KVM_BUS_LOCK_DETECTION_VALID_MODE)
	5314	+ break;
	5315	+
	5316	+ if ((cap->args[0] & KVM_BUS_LOCK_DETECTION_OFF) &&
	5317	+ (cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT))
	5318	+ break;
	5319	+
	5320	+ if (kvm_has_bus_lock_exit &&
	5321	+ cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT)
	5322	+ kvm->arch.bus_lock_detection_enabled = true;
	5323	+ r = 0;
	5324	+ break;
5311	5325	default:
5312	5326	r = -EINVAL;
5313	5327	break;
@@ -5637,11 +5651,27 @@ set_pit2_out: · 323 unmodified lines
5637	5651	r = -EFAULT;
5638	5652	if (copy_from_user(&xhc, argp, sizeof(xhc)))
5639	5653	goto out;
5640		− r = -EINVAL;
5641		− if (xhc.flags)
	5654	+ r = kvm_xen_hvm_config(kvm, &xhc);
	5655	+ break;
	5656	+ }
	5657	+ case KVM_XEN_HVM_GET_ATTR: {
	5658	+ struct kvm_xen_hvm_attr xha;
	5659	+
	5660	+ r = -EFAULT;
	5661	+ if (copy_from_user(&xha, argp, sizeof(xha)))
5642	5662	goto out;
5643		− memcpy(&kvm->arch.xen_hvm_config, &xhc, sizeof(xhc));
5644		− r = 0;
	5663	+ r = kvm_xen_hvm_get_attr(kvm, &xha);
	5664	+ if (!r && copy_to_user(argp, &xha, sizeof(xha)))
	5665	+ r = -EFAULT;
	5666	+ break;
	5667	+ }
	5668	+ case KVM_XEN_HVM_SET_ATTR: {
	5669	+ struct kvm_xen_hvm_attr xha;
	5670	+
	5671	+ r = -EFAULT;
	5672	+ if (copy_from_user(&xha, argp, sizeof(xha)))
	5673	+ goto out;
	5674	+ r = kvm_xen_hvm_set_attr(kvm, &xha);
5645	5675	break;
5646	5676	}
5647	5677	case KVM_SET_CLOCK: {
@@ -5686,7 +5716,7 @@ set_pit2_out: · 38 unmodified lines
5686	5716	case KVM_MEMORY_ENCRYPT_OP: {
5687	5717	r = -ENOTTY;
5688	5718	if (kvm_x86_ops.mem_enc_op)
5689		− r = kvm_x86_ops.mem_enc_op(kvm, argp);
	5719	+ r = static_call(kvm_x86_mem_enc_op)(kvm, argp);
5690	5720	break;
5691	5721	}
5692	5722	case KVM_MEMORY_ENCRYPT_REG_REGION: {
@@ -5698,7 +5728,7 @@ set_pit2_out: · 5 unmodified lines
5698	5728
5699	5729	r = -ENOTTY;
5700	5730	if (kvm_x86_ops.mem_enc_reg_region)
5701		− r = kvm_x86_ops.mem_enc_reg_region(kvm, &region);
	5731	+ r = static_call(kvm_x86_mem_enc_reg_region)(kvm, &region);
5702	5732	break;
5703	5733	}
5704	5734	case KVM_MEMORY_ENCRYPT_UNREG_REGION: {
@@ -5710,7 +5740,7 @@ set_pit2_out: · 5 unmodified lines
5710	5740
5711	5741	r = -ENOTTY;
5712	5742	if (kvm_x86_ops.mem_enc_unreg_region)
5713		− r = kvm_x86_ops.mem_enc_unreg_region(kvm, &region);
	5743	+ r = static_call(kvm_x86_mem_enc_unreg_region)(kvm, &region);
5714	5744	break;
5715	5745	}
5716	5746	case KVM_HYPERV_EVENTFD: {
@@ -5812,7 +5842,7 @@ static void kvm_init_msr_list(void) · 95 unmodified lines
5812	5842	}
5813	5843
5814	5844	for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) {
5815		− if (!kvm_x86_ops.has_emulated_msr(NULL, emulated_msrs_all[i]))
	5845	+ if (!static_call(kvm_x86_has_emulated_msr)(NULL, emulated_msrs_all[i]))
5816	5846	continue;
5817	5847
5818	5848	emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i];
@@ -5875,13 +5905,13 @@ static int vcpu_mmio_read(struct kvm_vcpu vcpu, gpa_t addr, int len, void v) · 56 unmodified lines
5875	5905	static void kvm_set_segment(struct kvm_vcpu *vcpu,
5876	5906	struct kvm_segment *var, int seg)
5877	5907	{
5878		− kvm_x86_ops.set_segment(vcpu, var, seg);
	5908	+ static_call(kvm_x86_set_segment)(vcpu, var, seg);
5879	5909	}
5880	5910
5881	5911	void kvm_get_segment(struct kvm_vcpu *vcpu,
5882	5912	struct kvm_segment *var, int seg)
5883	5913	{
5884		− kvm_x86_ops.get_segment(vcpu, var, seg);
	5914	+ static_call(kvm_x86_get_segment)(vcpu, var, seg);
5885	5915	}
5886	5916
5887	5917	gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
@@ -5901,14 +5931,14 @@ gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, · 13 unmodified lines
5901	5931	gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
5902	5932	struct x86_exception *exception)
5903	5933	{
5904		− u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	5934	+ u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
5905	5935	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5906	5936	}
5907	5937
5908	5938	gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
5909	5939	struct x86_exception *exception)
5910	5940	{
5911		− u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	5941	+ u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
5912	5942	access \|= PFERR_FETCH_MASK;
5913	5943	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5914	5944	}
@@ -5916,7 +5946,7 @@ gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, · 1 unmodified line
5916	5946	gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
5917	5947	struct x86_exception *exception)
5918	5948	{
5919		− u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	5949	+ u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
5920	5950	access \|= PFERR_WRITE_MASK;
5921	5951	return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
5922	5952	}
@@ -5965,7 +5995,7 @@ static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt, · 42 unmodified lines
5965	5995	struct x86_exception *exception)
5966	5996	{
5967	5997	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5968		− u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	5998	+ u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
5969	5999	unsigned offset;
5970	6000	int ret;
5971	6001
@@ -5990,7 +6020,7 @@ int kvm_read_guest_virt(struct kvm_vcpu *vcpu, · 18 unmodified lines
5990	6020	gva_t addr, void *val, unsigned int bytes,
5991	6021	struct x86_exception *exception)
5992	6022	{
5993		− u32 access = (kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
	6023	+ u32 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
5994	6024
5995	6025	/*
5996	6026	* FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
@@ -6011,7 +6041,7 @@ static int emulator_read_std(struct x86_emulate_ctxt *ctxt, · 14 unmodified lines
6011	6041	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6012	6042	u32 access = 0;
6013	6043
6014		− if (!system && kvm_x86_ops.get_cpl(vcpu) == 3)
	6044	+ if (!system && static_call(kvm_x86_get_cpl)(vcpu) == 3)
6015	6045	access \|= PFERR_USER_MASK;
6016	6046
6017	6047	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);
@@ -6064,7 +6094,7 @@ static int emulator_write_std(struct x86_emulate_ctxt ctxt, gva_t addr, void v · 46 unmodified lines
6064	6094	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
6065	6095	u32 access = PFERR_WRITE_MASK;
6066	6096
6067		− if (!system && kvm_x86_ops.get_cpl(vcpu) == 3)
	6097	+ if (!system && static_call(kvm_x86_get_cpl)(vcpu) == 3)
6068	6098	access \|= PFERR_USER_MASK;
6069	6099
6070	6100	return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
@@ -6089,7 +6119,7 @@ int handle_ud(struct kvm_vcpu *vcpu) · 18 unmodified lines
6089	6119	char sig[5]; /* ud2; .ascii "kvm" */
6090	6120	struct x86_exception e;
6091	6121
6092		− if (unlikely(!kvm_x86_ops.can_emulate_instruction(vcpu, NULL, 0)))
	6122	+ if (unlikely(!static_call(kvm_x86_can_emulate_instruction)(vcpu, NULL, 0)))
6093	6123	return 1;
6094	6124
6095	6125	if (force_emulation_prefix &&
@@ -6123,7 +6153,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, · 27 unmodified lines
6123	6153	gpa_t gpa, struct x86_exception exception,
6124	6154	bool write)
6125	6155	{
6126		− u32 access = ((kvm_x86_ops.get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
	6156	+ u32 access = ((static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0)
6127	6157	\| (write ? PFERR_WRITE_MASK : 0);
6128	6158
6129	6159	/*
@@ -6531,7 +6561,7 @@ static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt, · 401 unmodified lines
6531	6561
6532	6562	static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
6533	6563	{
6534		− return kvm_x86_ops.get_segment_base(vcpu, seg);
	6564	+ return static_call(kvm_x86_get_segment_base)(vcpu, seg);
6535	6565	}
6536	6566
6537	6567	static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
@@ -6544,7 +6574,7 @@ static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu) · 6 unmodified lines
6544	6574	if (!need_emulate_wbinvd(vcpu))
6545	6575	return X86EMUL_CONTINUE;
6546	6576
6547		− if (kvm_x86_ops.has_wbinvd_exit()) {
	6577	+ if (static_call(kvm_x86_has_wbinvd_exit)()) {
6548	6578	int cpu = get_cpu();
6549	6579
6550	6580	cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
@@ -6571,17 +6601,17 @@ static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt) · 20 unmodified lines
6571	6601	kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));
6572	6602	}
6573	6603
6574		−static int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,
6575		− unsigned long *dest)
	6604	+static void emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,
	6605	+ unsigned long *dest)
6576	6606	{
6577		− return kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
	6607	+ kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
6578	6608	}
6579	6609
6580	6610	static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,
6581	6611	unsigned long value)
6582	6612	{
6583	6613
6584		− return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
	6614	+ return kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
6585	6615	}
6586	6616
6587	6617	static u64 mk_cr_64(u64 curr_cr, u32 new_val)
@@ -6649,27 +6679,27 @@ static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val) · 61 unmodified lines
6649	6679
6650	6680	static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
6651	6681	{
6652		− return kvm_x86_ops.get_cpl(emul_to_vcpu(ctxt));
	6682	+ return static_call(kvm_x86_get_cpl)(emul_to_vcpu(ctxt));
6653	6683	}
6654	6684
6655	6685	static void emulator_get_gdt(struct x86_emulate_ctxt ctxt, struct desc_ptr dt)
6656	6686	{
6657		− kvm_x86_ops.get_gdt(emul_to_vcpu(ctxt), dt);
	6687	+ static_call(kvm_x86_get_gdt)(emul_to_vcpu(ctxt), dt);
6658	6688	}
6659	6689
6660	6690	static void emulator_get_idt(struct x86_emulate_ctxt ctxt, struct desc_ptr dt)
6661	6691	{
6662		− kvm_x86_ops.get_idt(emul_to_vcpu(ctxt), dt);
	6692	+ static_call(kvm_x86_get_idt)(emul_to_vcpu(ctxt), dt);
6663	6693	}
6664	6694
6665	6695	static void emulator_set_gdt(struct x86_emulate_ctxt ctxt, struct desc_ptr dt)
6666	6696	{
6667		− kvm_x86_ops.set_gdt(emul_to_vcpu(ctxt), dt);
	6697	+ static_call(kvm_x86_set_gdt)(emul_to_vcpu(ctxt), dt);
6668	6698	}
6669	6699
6670	6700	static void emulator_set_idt(struct x86_emulate_ctxt ctxt, struct desc_ptr dt)
6671	6701	{
6672		− kvm_x86_ops.set_idt(emul_to_vcpu(ctxt), dt);
	6702	+ static_call(kvm_x86_set_idt)(emul_to_vcpu(ctxt), dt);
6673	6703	}
6674	6704
6675	6705	static unsigned long emulator_get_cached_segment_base(
@@ -6811,7 +6841,7 @@ static int emulator_intercept(struct x86_emulate_ctxt *ctxt, · 135 unmodified lines
6811	6841	struct x86_instruction_info *info,
6812	6842	enum x86_intercept_stage stage)
6813	6843	{
6814		− return kvm_x86_ops.check_intercept(emul_to_vcpu(ctxt), info, stage,
	6844	+ return static_call(kvm_x86_check_intercept)(emul_to_vcpu(ctxt), info, stage,
6815	6845	&ctxt->exception);
6816	6846	}
6817	6847
@@ -6849,7 +6879,7 @@ static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulon · 31 unmodified lines
6849	6879
6850	6880	static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked)
6851	6881	{
6852		− kvm_x86_ops.set_nmi_mask(emul_to_vcpu(ctxt), masked);
	6882	+ static_call(kvm_x86_set_nmi_mask)(emul_to_vcpu(ctxt), masked);
6853	6883	}
6854	6884
6855	6885	static unsigned emulator_get_hflags(struct x86_emulate_ctxt *ctxt)
@@ -6865,7 +6895,7 @@ static void emulator_set_hflags(struct x86_emulate_ctxt *ctxt, unsigned emul_fla · 9 unmodified lines
6865	6895	static int emulator_pre_leave_smm(struct x86_emulate_ctxt *ctxt,
6866	6896	const char *smstate)
6867	6897	{
6868		− return kvm_x86_ops.pre_leave_smm(emul_to_vcpu(ctxt), smstate);
	6898	+ return static_call(kvm_x86_pre_leave_smm)(emul_to_vcpu(ctxt), smstate);
6869	6899	}
6870	6900
6871	6901	static void emulator_post_leave_smm(struct x86_emulate_ctxt *ctxt)
@@ -6927,7 +6957,7 @@ static const struct x86_emulate_ops emulate_ops = { · 55 unmodified lines
6927	6957
6928	6958	static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
6929	6959	{
6930		− u32 int_shadow = kvm_x86_ops.get_interrupt_shadow(vcpu);
	6960	+ u32 int_shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
6931	6961	/*
6932	6962	* an sti; sti; sequence only disable interrupts for the first
6933	6963	* instruction. So, if the last instruction, be it emulated or
@@ -6938,7 +6968,7 @@ static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) · 4 unmodified lines
6938	6968	if (int_shadow & mask)
6939	6969	mask = 0;
6940	6970	if (unlikely(int_shadow \|\| mask)) {
6941		− kvm_x86_ops.set_interrupt_shadow(vcpu, mask);
	6971	+ static_call(kvm_x86_set_interrupt_shadow)(vcpu, mask);
6942	6972	if (!mask)
6943	6973	kvm_make_request(KVM_REQ_EVENT, vcpu);
6944	6974	}
@@ -6980,7 +7010,7 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu) · 35 unmodified lines
6980	7010	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
6981	7011	int cs_db, cs_l;
6982	7012
6983		− kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
	7013	+ static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
6984	7014
6985	7015	ctxt->gpa_available = false;
6986	7016	ctxt->eflags = kvm_get_rflags(vcpu);
@@ -7041,7 +7071,7 @@ static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type) · 54 unmodified lines
7041	7071
7042	7072	kvm_queue_exception(vcpu, UD_VECTOR);
7043	7073
7044		− if (!is_guest_mode(vcpu) && kvm_x86_ops.get_cpl(vcpu) == 0) {
	7074	+ if (!is_guest_mode(vcpu) && static_call(kvm_x86_get_cpl)(vcpu) == 0) {
7045	7075	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
7046	7076	vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
7047	7077	vcpu->run->internal.ndata = 0;
@@ -7101,9 +7131,9 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, · 53 unmodified lines
7101	7131	if (vcpu->arch.mmu->direct_map) {
7102	7132	unsigned int indirect_shadow_pages;
7103	7133
7104		− spin_lock(&vcpu->kvm->mmu_lock);
	7134	+ write_lock(&vcpu->kvm->mmu_lock);
7105	7135	indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
7106		− spin_unlock(&vcpu->kvm->mmu_lock);
	7136	+ write_unlock(&vcpu->kvm->mmu_lock);
7107	7137
7108	7138	if (indirect_shadow_pages)
7109	7139	kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
@@ -7210,7 +7240,7 @@ static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu) · 100 unmodified lines
7210	7240	struct kvm_run *kvm_run = vcpu->run;
7211	7241
7212	7242	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
7213		− kvm_run->debug.arch.dr6 = DR6_BS \| DR6_FIXED_1 \| DR6_RTM;
	7243	+ kvm_run->debug.arch.dr6 = DR6_BS \| DR6_ACTIVE_LOW;
7214	7244	kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);
7215	7245	kvm_run->debug.arch.exception = DB_VECTOR;
7216	7246	kvm_run->exit_reason = KVM_EXIT_DEBUG;
@@ -7222,10 +7252,10 @@ static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu) · 5 unmodified lines
7222	7252
7223	7253	int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
7224	7254	{
7225		− unsigned long rflags = kvm_x86_ops.get_rflags(vcpu);
	7255	+ unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
7226	7256	int r;
7227	7257
7228		− r = kvm_x86_ops.skip_emulated_instruction(vcpu);
	7258	+ r = static_call(kvm_x86_skip_emulated_instruction)(vcpu);
7229	7259	if (unlikely(!r))
7230	7260	return 0;
7231	7261
@@ -7254,7 +7284,7 @@ static bool kvm_vcpu_check_breakpoint(struct kvm_vcpu vcpu, int r) · 22 unmodified lines
7254	7284	vcpu->arch.eff_db);
7255	7285
7256	7286	if (dr6 != 0) {
7257		− kvm_run->debug.arch.dr6 = dr6 \| DR6_FIXED_1 \| DR6_RTM;
	7287	+ kvm_run->debug.arch.dr6 = dr6 \| DR6_ACTIVE_LOW;
7258	7288	kvm_run->debug.arch.pc = eip;
7259	7289	kvm_run->debug.arch.exception = DB_VECTOR;
7260	7290	kvm_run->exit_reason = KVM_EXIT_DEBUG;
@@ -7311,6 +7341,42 @@ static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt) · 50 unmodified lines
7311	7341	return false;
7312	7342	}
7313	7343
	7344	+/*
	7345	+ * Decode to be emulated instruction. Return EMULATION_OK if success.
	7346	+ */
	7347	+int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type,
	7348	+ void *insn, int insn_len)
	7349	+{
	7350	+ int r = EMULATION_OK;
	7351	+ struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
	7352	+
	7353	+ init_emulate_ctxt(vcpu);
	7354	+
	7355	+ /*
	7356	+ * We will reenter on the same instruction since we do not set
	7357	+ * complete_userspace_io. This does not handle watchpoints yet,
	7358	+ * those would be handled in the emulate_ops.
	7359	+ */
	7360	+ if (!(emulation_type & EMULTYPE_SKIP) &&
	7361	+ kvm_vcpu_check_breakpoint(vcpu, &r))
	7362	+ return r;
	7363	+
	7364	+ ctxt->interruptibility = 0;
	7365	+ ctxt->have_exception = false;
	7366	+ ctxt->exception.vector = -1;
	7367	+ ctxt->perm_ok = false;
	7368	+
	7369	+ ctxt->ud = emulation_type & EMULTYPE_TRAP_UD;
	7370	+
	7371	+ r = x86_decode_insn(ctxt, insn, insn_len);
	7372	+
	7373	+ trace_kvm_emulate_insn_start(vcpu);
	7374	+ ++vcpu->stat.insn_emulation;
	7375	+
	7376	+ return r;
	7377	+}
	7378	+EXPORT_SYMBOL_GPL(x86_decode_emulated_instruction);
	7379	+
7314	7380	int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
7315	7381	int emulation_type, void *insn, int insn_len)
7316	7382	{
@@ -7319,7 +7385,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, · 2 unmodified lines
7319	7385	bool writeback = true;
7320	7386	bool write_fault_to_spt;
7321	7387
7322		− if (unlikely(!kvm_x86_ops.can_emulate_instruction(vcpu, insn, insn_len)))
	7388	+ if (unlikely(!static_call(kvm_x86_can_emulate_instruction)(vcpu, insn, insn_len)))
7323	7389	return 1;
7324	7390
7325	7391	vcpu->arch.l1tf_flush_l1d = true;
@@ -7330,32 +7396,12 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, · 4 unmodified lines
7330	7396	*/
7331	7397	write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
7332	7398	vcpu->arch.write_fault_to_shadow_pgtable = false;
7333		− kvm_clear_exception_queue(vcpu);
7334	7399
7335	7400	if (!(emulation_type & EMULTYPE_NO_DECODE)) {
7336		− init_emulate_ctxt(vcpu);
7337		−
7338		− /*
7339		− * We will reenter on the same instruction since
7340		− * we do not set complete_userspace_io. This does not
7341		− * handle watchpoints yet, those would be handled in
7342		− * the emulate_ops.
7343		− */
7344		− if (!(emulation_type & EMULTYPE_SKIP) &&
7345		− kvm_vcpu_check_breakpoint(vcpu, &r))
7346		− return r;
	7401	+ kvm_clear_exception_queue(vcpu);
7347	7402
7348		− ctxt->interruptibility = 0;
7349		− ctxt->have_exception = false;
7350		− ctxt->exception.vector = -1;
7351		− ctxt->perm_ok = false;
7352		−
7353		− ctxt->ud = emulation_type & EMULTYPE_TRAP_UD;
7354		−
7355		− r = x86_decode_insn(ctxt, insn, insn_len);
7356		−
7357		− trace_kvm_emulate_insn_start(vcpu);
7358		− ++vcpu->stat.insn_emulation;
	7403	+ r = x86_decode_emulated_instruction(vcpu, emulation_type,
	7404	+ insn, insn_len);
7359	7405	if (r != EMULATION_OK) {
7360	7406	if ((emulation_type & EMULTYPE_TRAP_UD) \|\|
7361	7407	(emulation_type & EMULTYPE_TRAP_UD_FORCED)) {
@@ -7462,7 +7508,7 @@ restart: · 100 unmodified lines
7462	7508	r = 1;
7463	7509
7464	7510	if (writeback) {
7465		− unsigned long rflags = kvm_x86_ops.get_rflags(vcpu);
	7511	+ unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
7466	7512	toggle_interruptibility(vcpu, ctxt->interruptibility);
7467	7513	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
7468	7514	if (!ctxt->have_exception \|\|
@@ -7471,7 +7517,7 @@ restart: · 2 unmodified lines
7471	7517	if (r && (ctxt->tf \|\| (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))
7472	7518	r = kvm_vcpu_do_singlestep(vcpu);
7473	7519	if (kvm_x86_ops.update_emulated_instruction)
7474		− kvm_x86_ops.update_emulated_instruction(vcpu);
	7520	+ static_call(kvm_x86_update_emulated_instruction)(vcpu);
7475	7521	__kvm_set_rflags(vcpu, ctxt->eflags);
7476	7522	}
7477	7523
@@ -7800,7 +7846,7 @@ static int kvm_is_user_mode(void) · 322 unmodified lines
7800	7846	int user_mode = 3;
7801	7847
7802	7848	if (__this_cpu_read(current_vcpu))
7803		− user_mode = kvm_x86_ops.get_cpl(__this_cpu_read(current_vcpu));
	7849	+ user_mode = static_call(kvm_x86_get_cpl)(__this_cpu_read(current_vcpu));
7804	7850
7805	7851	return user_mode != 0;
7806	7852	}
@@ -7945,7 +7991,6 @@ int kvm_arch_init(void *opaque) · 138 unmodified lines
7945	7991	supported_xcr0 = host_xcr0 & KVM_SUPPORTED_XCR0;
7946	7992	}
7947	7993
7948		− kvm_lapic_init();
7949	7994	if (pi_inject_timer == -1)
7950	7995	pi_inject_timer = housekeeping_enabled(HK_FLAG_TIMER);
7951	7996	#ifdef CONFIG_X86_64
@@ -7987,6 +8032,7 @@ void kvm_arch_exit(void) · 35 unmodified lines
7987	8032	kvm_mmu_module_exit();
7988	8033	free_percpu(user_return_msrs);
7989	8034	kmem_cache_destroy(x86_fpu_cache);
	8035	+ WARN_ON(static_branch_unlikely(&kvm_xen_enabled.key));
7990	8036	}
7991	8037
7992	8038	static int __kvm_vcpu_halt(struct kvm_vcpu *vcpu, int state, int reason)
@@ -8038,7 +8084,7 @@ static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr, · 45 unmodified lines
8038	8084	if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
8039	8085	return -KVM_EOPNOTSUPP;
8040	8086
8041		− if (kvm_get_walltime_and_clockread(&ts, &cycle) == false)
	8087	+ if (!kvm_get_walltime_and_clockread(&ts, &cycle))
8042	8088	return -KVM_EOPNOTSUPP;
8043	8089
8044	8090	clock_pairing.sec = ts.tv_sec;
@@ -8114,7 +8160,10 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) · 69 unmodified lines
8114	8160	unsigned long nr, a0, a1, a2, a3, ret;
8115	8161	int op_64_bit;
8116	8162
8117		− if (kvm_hv_hypercall_enabled(vcpu->kvm))
	8163	+ if (kvm_xen_hypercall_enabled(vcpu->kvm))
	8164	+ return kvm_xen_hypercall(vcpu);
	8165	+
	8166	+ if (kvm_hv_hypercall_enabled(vcpu))
8118	8167	return kvm_hv_hypercall(vcpu);
8119	8168
8120	8169	nr = kvm_rax_read(vcpu);
@@ -8134,7 +8183,7 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) · 13 unmodified lines
8134	8183	a3 &= 0xFFFFFFFF;
8135	8184	}
8136	8185
8137		− if (kvm_x86_ops.get_cpl(vcpu) != 0) {
	8186	+ if (static_call(kvm_x86_get_cpl)(vcpu) != 0) {
8138	8187	ret = -KVM_EPERM;
8139	8188	goto out;
8140	8189	}
@@ -8191,7 +8240,7 @@ static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt) · 50 unmodified lines
8191	8240	char instruction[3];
8192	8241	unsigned long rip = kvm_rip_read(vcpu);
8193	8242
8194		− kvm_x86_ops.patch_hypercall(vcpu, instruction);
	8243	+ static_call(kvm_x86_patch_hypercall)(vcpu, instruction);
8195	8244
8196	8245	return emulator_write_emulated(ctxt, rip, instruction, 3,
8197	8246	&ctxt->exception);
@@ -8215,12 +8264,14 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu) · 17 unmodified lines
8215	8264	kvm_run->if_flag = !vcpu->arch.guest_state_protected
8216	8265	&& (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
8217	8266
8218		− kvm_run->flags = is_smm(vcpu) ? KVM_RUN_X86_SMM : 0;
8219	8267	kvm_run->cr8 = kvm_get_cr8(vcpu);
8220	8268	kvm_run->apic_base = kvm_get_apic_base(vcpu);
8221	8269	kvm_run->ready_for_interrupt_injection =
8222	8270	pic_in_kernel(vcpu->kvm) \|\|
8223	8271	kvm_vcpu_ready_for_interrupt_injection(vcpu);
	8272	+
	8273	+ if (is_smm(vcpu))
	8274	+ kvm_run->flags \|= KVM_RUN_X86_SMM;
8224	8275	}
8225	8276
8226	8277	static void update_cr8_intercept(struct kvm_vcpu *vcpu)
@@ -8246,7 +8297,7 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu) · 19 unmodified lines
8246	8297
8247	8298	tpr = kvm_lapic_get_cr8(vcpu);
8248	8299
8249		− kvm_x86_ops.update_cr8_intercept(vcpu, tpr, max_irr);
	8300	+ static_call(kvm_x86_update_cr8_intercept)(vcpu, tpr, max_irr);
8250	8301	}
8251	8302
8252	8303	static void inject_pending_event(struct kvm_vcpu vcpu, bool req_immediate_exit)
@@ -8257,7 +8308,7 @@ static void inject_pending_event(struct kvm_vcpu vcpu, bool req_immediate_exit · 4 unmodified lines
8257	8308	/* try to reinject previous events if any */
8258	8309
8259	8310	if (vcpu->arch.exception.injected) {
8260		− kvm_x86_ops.queue_exception(vcpu);
	8311	+ static_call(kvm_x86_queue_exception)(vcpu);
8261	8312	can_inject = false;
8262	8313	}
8263	8314	/*
@@ -8276,10 +8327,10 @@ static void inject_pending_event(struct kvm_vcpu vcpu, bool req_immediate_exit · 12 unmodified lines
8276	8327	*/
8277	8328	else if (!vcpu->arch.exception.pending) {
8278	8329	if (vcpu->arch.nmi_injected) {
8279		− kvm_x86_ops.set_nmi(vcpu);
	8330	+ static_call(kvm_x86_set_nmi)(vcpu);
8280	8331	can_inject = false;
8281	8332	} else if (vcpu->arch.interrupt.injected) {
8282		− kvm_x86_ops.set_irq(vcpu);
	8333	+ static_call(kvm_x86_set_irq)(vcpu);
8283	8334	can_inject = false;
8284	8335	}
8285	8336	}
@@ -8320,7 +8371,7 @@ static void inject_pending_event(struct kvm_vcpu vcpu, bool req_immediate_exit · 34 unmodified lines
8320	8371	}
8321	8372	}
8322	8373
8323		− kvm_x86_ops.queue_exception(vcpu);
	8374	+ static_call(kvm_x86_queue_exception)(vcpu);
8324	8375	can_inject = false;
8325	8376	}
8326	8377
@@ -8336,7 +8387,7 @@ static void inject_pending_event(struct kvm_vcpu vcpu, bool req_immediate_exit · 9 unmodified lines
8336	8387	* The kvm_x86_ops hooks communicate this by returning -EBUSY.
8337	8388	*/
8338	8389	if (vcpu->arch.smi_pending) {
8339		− r = can_inject ? kvm_x86_ops.smi_allowed(vcpu, true) : -EBUSY;
	8390	+ r = can_inject ? static_call(kvm_x86_smi_allowed)(vcpu, true) : -EBUSY;
8340	8391	if (r < 0)
8341	8392	goto busy;
8342	8393	if (r) {
@@ -8345,35 +8396,35 @@ static void inject_pending_event(struct kvm_vcpu vcpu, bool req_immediate_exit · 2 unmodified lines
8345	8396	enter_smm(vcpu);
8346	8397	can_inject = false;
8347	8398	} else
8348		− kvm_x86_ops.enable_smi_window(vcpu);
	8399	+ static_call(kvm_x86_enable_smi_window)(vcpu);
8349	8400	}
8350	8401
8351	8402	if (vcpu->arch.nmi_pending) {
8352		− r = can_inject ? kvm_x86_ops.nmi_allowed(vcpu, true) : -EBUSY;
	8403	+ r = can_inject ? static_call(kvm_x86_nmi_allowed)(vcpu, true) : -EBUSY;
8353	8404	if (r < 0)
8354	8405	goto busy;
8355	8406	if (r) {
8356	8407	--vcpu->arch.nmi_pending;
8357	8408	vcpu->arch.nmi_injected = true;
8358		− kvm_x86_ops.set_nmi(vcpu);
	8409	+ static_call(kvm_x86_set_nmi)(vcpu);
8359	8410	can_inject = false;
8360		− WARN_ON(kvm_x86_ops.nmi_allowed(vcpu, true) < 0);
	8411	+ WARN_ON(static_call(kvm_x86_nmi_allowed)(vcpu, true) < 0);
8361	8412	}
8362	8413	if (vcpu->arch.nmi_pending)
8363		− kvm_x86_ops.enable_nmi_window(vcpu);
	8414	+ static_call(kvm_x86_enable_nmi_window)(vcpu);
8364	8415	}
8365	8416
8366	8417	if (kvm_cpu_has_injectable_intr(vcpu)) {
8367		− r = can_inject ? kvm_x86_ops.interrupt_allowed(vcpu, true) : -EBUSY;
	8418	+ r = can_inject ? static_call(kvm_x86_interrupt_allowed)(vcpu, true) : -EBUSY;
8368	8419	if (r < 0)
8369	8420	goto busy;
8370	8421	if (r) {
8371	8422	kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), false);
8372		− kvm_x86_ops.set_irq(vcpu);
8373		− WARN_ON(kvm_x86_ops.interrupt_allowed(vcpu, true) < 0);
	8423	+ static_call(kvm_x86_set_irq)(vcpu);
	8424	+ WARN_ON(static_call(kvm_x86_interrupt_allowed)(vcpu, true) < 0);
8374	8425	}
8375	8426	if (kvm_cpu_has_injectable_intr(vcpu))
8376		− kvm_x86_ops.enable_irq_window(vcpu);
	8427	+ static_call(kvm_x86_enable_irq_window)(vcpu);
8377	8428	}
8378	8429
8379	8430	if (is_guest_mode(vcpu) &&
@@ -8398,7 +8449,7 @@ static void process_nmi(struct kvm_vcpu *vcpu) · 18 unmodified lines
8398	8449	* If an NMI is already in progress, limit further NMIs to just one.
8399	8450	* Otherwise, allow two (and we'll inject the first one immediately).
8400	8451	*/
8401		− if (kvm_x86_ops.get_nmi_mask(vcpu) \|\| vcpu->arch.nmi_injected)
	8452	+ if (static_call(kvm_x86_get_nmi_mask)(vcpu) \|\| vcpu->arch.nmi_injected)
8402	8453	limit = 1;
8403	8454
8404	8455	vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
@@ -8488,11 +8539,11 @@ static void enter_smm_save_state_32(struct kvm_vcpu vcpu, char buf) · 83 unmodified lines
8488	8539	put_smstate(u32, buf, 0x7f7c, seg.limit);
8489	8540	put_smstate(u32, buf, 0x7f78, enter_smm_get_segment_flags(&seg));
8490	8541
8491		− kvm_x86_ops.get_gdt(vcpu, &dt);
	8542	+ static_call(kvm_x86_get_gdt)(vcpu, &dt);
8492	8543	put_smstate(u32, buf, 0x7f74, dt.address);
8493	8544	put_smstate(u32, buf, 0x7f70, dt.size);
8494	8545
8495		− kvm_x86_ops.get_idt(vcpu, &dt);
	8546	+ static_call(kvm_x86_get_idt)(vcpu, &dt);
8496	8547	put_smstate(u32, buf, 0x7f58, dt.address);
8497	8548	put_smstate(u32, buf, 0x7f54, dt.size);
8498	8549
@@ -8542,7 +8593,7 @@ static void enter_smm_save_state_64(struct kvm_vcpu vcpu, char buf) · 43 unmodified lines
8542	8593	put_smstate(u32, buf, 0x7e94, seg.limit);
8543	8594	put_smstate(u64, buf, 0x7e98, seg.base);
8544	8595
8545		− kvm_x86_ops.get_idt(vcpu, &dt);
	8596	+ static_call(kvm_x86_get_idt)(vcpu, &dt);
8546	8597	put_smstate(u32, buf, 0x7e84, dt.size);
8547	8598	put_smstate(u64, buf, 0x7e88, dt.address);
8548	8599
@@ -8552,7 +8603,7 @@ static void enter_smm_save_state_64(struct kvm_vcpu vcpu, char buf) · 3 unmodified lines
8552	8603	put_smstate(u32, buf, 0x7e74, seg.limit);
8553	8604	put_smstate(u64, buf, 0x7e78, seg.base);
8554	8605
8555		− kvm_x86_ops.get_gdt(vcpu, &dt);
	8606	+ static_call(kvm_x86_get_gdt)(vcpu, &dt);
8556	8607	put_smstate(u32, buf, 0x7e64, dt.size);
8557	8608	put_smstate(u64, buf, 0x7e68, dt.address);
8558	8609
@@ -8582,30 +8633,30 @@ static void enter_smm(struct kvm_vcpu *vcpu) · 23 unmodified lines
8582	8633	* vCPU state (e.g. leave guest mode) after we've saved the state into
8583	8634	* the SMM state-save area.
8584	8635	*/
8585		− kvm_x86_ops.pre_enter_smm(vcpu, buf);
	8636	+ static_call(kvm_x86_pre_enter_smm)(vcpu, buf);
8586	8637
8587	8638	vcpu->arch.hflags \|= HF_SMM_MASK;
8588	8639	kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf));
8589	8640
8590		− if (kvm_x86_ops.get_nmi_mask(vcpu))
	8641	+ if (static_call(kvm_x86_get_nmi_mask)(vcpu))
8591	8642	vcpu->arch.hflags \|= HF_SMM_INSIDE_NMI_MASK;
8592	8643	else
8593		− kvm_x86_ops.set_nmi_mask(vcpu, true);
	8644	+ static_call(kvm_x86_set_nmi_mask)(vcpu, true);
8594	8645
8595	8646	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
8596	8647	kvm_rip_write(vcpu, 0x8000);
8597	8648
8598	8649	cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE \| X86_CR0_EM \| X86_CR0_TS \| X86_CR0_PG);
8599		− kvm_x86_ops.set_cr0(vcpu, cr0);
	8650	+ static_call(kvm_x86_set_cr0)(vcpu, cr0);
8600	8651	vcpu->arch.cr0 = cr0;
8601	8652
8602		− kvm_x86_ops.set_cr4(vcpu, 0);
	8653	+ static_call(kvm_x86_set_cr4)(vcpu, 0);
8603	8654
8604	8655	/* Undocumented: IDT limit is set to zero on entry to SMM. */
8605	8656	dt.address = dt.size = 0;
8606		− kvm_x86_ops.set_idt(vcpu, &dt);
	8657	+ static_call(kvm_x86_set_idt)(vcpu, &dt);
8607	8658
8608		− __kvm_set_dr(vcpu, 7, DR7_FIXED_1);
	8659	+ kvm_set_dr(vcpu, 7, DR7_FIXED_1);
8609	8660
8610	8661	cs.selector = (vcpu->arch.smbase >> 4) & 0xffff;
8611	8662	cs.base = vcpu->arch.smbase;
@@ -8634,7 +8685,7 @@ static void enter_smm(struct kvm_vcpu *vcpu) · 22 unmodified lines
8634	8685
8635	8686	#ifdef CONFIG_X86_64
8636	8687	if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
8637		− kvm_x86_ops.set_efer(vcpu, 0);
	8688	+ static_call(kvm_x86_set_efer)(vcpu, 0);
8638	8689	#endif
8639	8690
8640	8691	kvm_update_cpuid_runtime(vcpu);
@@ -8672,7 +8723,7 @@ void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu) · 31 unmodified lines
8672	8723
8673	8724	vcpu->arch.apicv_active = kvm_apicv_activated(vcpu->kvm);
8674	8725	kvm_apic_update_apicv(vcpu);
8675		− kvm_x86_ops.refresh_apicv_exec_ctrl(vcpu);
	8726	+ static_call(kvm_x86_refresh_apicv_exec_ctrl)(vcpu);
8676	8727	}
8677	8728	EXPORT_SYMBOL_GPL(kvm_vcpu_update_apicv);
8678	8729
@@ -8689,7 +8740,7 @@ void kvm_request_apicv_update(struct kvm *kvm, bool activate, ulong bit) · 10 unmodified lines
8689	8740	unsigned long old, new, expected;
8690	8741
8691	8742	if (!kvm_x86_ops.check_apicv_inhibit_reasons \|\|
8692		− !kvm_x86_ops.check_apicv_inhibit_reasons(bit))
	8743	+ !static_call(kvm_x86_check_apicv_inhibit_reasons)(bit))
8693	8744	return;
8694	8745
8695	8746	old = READ_ONCE(kvm->arch.apicv_inhibit_reasons);
@@ -8709,7 +8760,7 @@ void kvm_request_apicv_update(struct kvm *kvm, bool activate, ulong bit) · 13 unmodified lines
8709	8760
8710	8761	trace_kvm_apicv_update_request(activate, bit);
8711	8762	if (kvm_x86_ops.pre_update_apicv_exec_ctrl)
8712		− kvm_x86_ops.pre_update_apicv_exec_ctrl(kvm, activate);
	8763	+ static_call(kvm_x86_pre_update_apicv_exec_ctrl)(kvm, activate);
8713	8764
8714	8765	/*
8715	8766	* Sending request to update APICV for all other vcpus,
@@ -8735,7 +8786,7 @@ static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu) · 19 unmodified lines
8735	8786	kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
8736	8787	else {
8737	8788	if (vcpu->arch.apicv_active)
8738		− kvm_x86_ops.sync_pir_to_irr(vcpu);
	8789	+ static_call(kvm_x86_sync_pir_to_irr)(vcpu);
8739	8790	if (ioapic_in_kernel(vcpu->kvm))
8740	8791	kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
8741	8792	}
@@ -8753,9 +8804,12 @@ static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu) · 11 unmodified lines
8753	8804	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
8754	8805	return;
8755	8806
8756		− bitmap_or((ulong *)eoi_exit_bitmap, vcpu->arch.ioapic_handled_vectors,
8757		− vcpu_to_synic(vcpu)->vec_bitmap, 256);
8758		− kvm_x86_ops.load_eoi_exitmap(vcpu, eoi_exit_bitmap);
	8807	+ if (to_hv_vcpu(vcpu))
	8808	+ bitmap_or((ulong *)eoi_exit_bitmap,
	8809	+ vcpu->arch.ioapic_handled_vectors,
	8810	+ to_hv_synic(vcpu)->vec_bitmap, 256);
	8811	+
	8812	+ static_call(kvm_x86_load_eoi_exitmap)(vcpu, eoi_exit_bitmap);
8759	8813	}
8760	8814
8761	8815	void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
@@ -8780,7 +8834,7 @@ void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) · 18 unmodified lines
8780	8834	if (!kvm_x86_ops.set_apic_access_page_addr)
8781	8835	return;
8782	8836
8783		− kvm_x86_ops.set_apic_access_page_addr(vcpu);
	8837	+ static_call(kvm_x86_set_apic_access_page_addr)(vcpu);
8784	8838	}
8785	8839
8786	8840	void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)
@@ -8905,8 +8959,10 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 118 unmodified lines
8905	8959	goto out;
8906	8960	}
8907	8961	if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {
	8962	+ struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
	8963	+
8908	8964	vcpu->run->exit_reason = KVM_EXIT_HYPERV;
8909		− vcpu->run->hyperv = vcpu->arch.hyperv.exit;
	8965	+ vcpu->run->hyperv = hv_vcpu->exit;
8910	8966	r = 0;
8911	8967	goto out;
8912	8968	}
@@ -8923,10 +8979,11 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 10 unmodified lines
8923	8979	if (kvm_check_request(KVM_REQ_APF_READY, vcpu))
8924	8980	kvm_check_async_pf_completion(vcpu);
8925	8981	if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu))
8926		− kvm_x86_ops.msr_filter_changed(vcpu);
	8982	+ static_call(kvm_x86_msr_filter_changed)(vcpu);
8927	8983	}
8928	8984
8929		− if (kvm_check_request(KVM_REQ_EVENT, vcpu) \|\| req_int_win) {
	8985	+ if (kvm_check_request(KVM_REQ_EVENT, vcpu) \|\| req_int_win \|\|
	8986	+ kvm_xen_has_interrupt(vcpu)) {
8930	8987	++vcpu->stat.req_event;
8931	8988	kvm_apic_accept_events(vcpu);
8932	8989	if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
@@ -8936,7 +8993,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 3 unmodified lines
8936	8993
8937	8994	inject_pending_event(vcpu, &req_immediate_exit);
8938	8995	if (req_int_win)
8939		− kvm_x86_ops.enable_irq_window(vcpu);
	8996	+ static_call(kvm_x86_enable_irq_window)(vcpu);
8940	8997
8941	8998	if (kvm_lapic_enabled(vcpu)) {
8942	8999	update_cr8_intercept(vcpu);
@@ -8951,7 +9008,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 8 unmodified lines
8951	9008
8952	9009	preempt_disable();
8953	9010
8954		− kvm_x86_ops.prepare_guest_switch(vcpu);
	9011	+ static_call(kvm_x86_prepare_guest_switch)(vcpu);
8955	9012
8956	9013	/*
8957	9014	* Disable IRQs before setting IN_GUEST_MODE. Posted interrupt
@@ -8982,7 +9039,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 24 unmodified lines
8982	9039	* notified with kvm_vcpu_kick.
8983	9040	*/
8984	9041	if (kvm_lapic_enabled(vcpu) && vcpu->arch.apicv_active)
8985		− kvm_x86_ops.sync_pir_to_irr(vcpu);
	9042	+ static_call(kvm_x86_sync_pir_to_irr)(vcpu);
8986	9043
8987	9044	if (kvm_vcpu_exit_request(vcpu)) {
8988	9045	vcpu->mode = OUTSIDE_GUEST_MODE;
@@ -8996,7 +9053,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 7 unmodified lines
8996	9053
8997	9054	if (req_immediate_exit) {
8998	9055	kvm_make_request(KVM_REQ_EVENT, vcpu);
8999		− kvm_x86_ops.request_immediate_exit(vcpu);
	9056	+ static_call(kvm_x86_request_immediate_exit)(vcpu);
9000	9057	}
9001	9058
9002	9059	fpregs_assert_state_consistent();
@@ -9013,7 +9070,19 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 10 unmodified lines
9013	9070	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
9014	9071	}
9015	9072
9016		− exit_fastpath = kvm_x86_ops.run(vcpu);
	9073	+ for (;;) {
	9074	+ exit_fastpath = static_call(kvm_x86_run)(vcpu);
	9075	+ if (likely(exit_fastpath != EXIT_FASTPATH_REENTER_GUEST))
	9076	+ break;
	9077	+
	9078	+ if (unlikely(kvm_vcpu_exit_request(vcpu))) {
	9079	+ exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED;
	9080	+ break;
	9081	+ }
	9082	+
	9083	+ if (vcpu->arch.apicv_active)
	9084	+ static_call(kvm_x86_sync_pir_to_irr)(vcpu);
	9085	+ }
9017	9086
9018	9087	/*
9019	9088	* Do this here before restoring debug registers on the host. And
@@ -9023,7 +9092,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 3 unmodified lines
9023	9092	*/
9024	9093	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
9025	9094	WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
9026		− kvm_x86_ops.sync_dirty_debug_regs(vcpu);
	9095	+ static_call(kvm_x86_sync_dirty_debug_regs)(vcpu);
9027	9096	kvm_update_dr0123(vcpu);
9028	9097	kvm_update_dr7(vcpu);
9029	9098	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD;
@@ -9045,7 +9114,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 15 unmodified lines
9045	9114	vcpu->mode = OUTSIDE_GUEST_MODE;
9046	9115	smp_wmb();
9047	9116
9048		− kvm_x86_ops.handle_exit_irqoff(vcpu);
	9117	+ static_call(kvm_x86_handle_exit_irqoff)(vcpu);
9049	9118
9050	9119	/*
9051	9120	* Consume any pending interrupts, including the possible source of
@@ -9087,13 +9156,13 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 35 unmodified lines
9087	9156	if (vcpu->arch.apic_attention)
9088	9157	kvm_lapic_sync_from_vapic(vcpu);
9089	9158
9090		− r = kvm_x86_ops.handle_exit(vcpu, exit_fastpath);
	9159	+ r = static_call(kvm_x86_handle_exit)(vcpu, exit_fastpath);
9091	9160	return r;
9092	9161
9093	9162	cancel_injection:
9094	9163	if (req_immediate_exit)
9095	9164	kvm_make_request(KVM_REQ_EVENT, vcpu);
9096		− kvm_x86_ops.cancel_injection(vcpu);
	9165	+ static_call(kvm_x86_cancel_injection)(vcpu);
9097	9166	if (unlikely(vcpu->arch.apic_attention))
9098	9167	kvm_lapic_sync_from_vapic(vcpu);
9099	9168	out:
@@ -9103,13 +9172,13 @@ out: · 3 unmodified lines
9103	9172	static inline int vcpu_block(struct kvm kvm, struct kvm_vcpu vcpu)
9104	9173	{
9105	9174	if (!kvm_arch_vcpu_runnable(vcpu) &&
9106		− (!kvm_x86_ops.pre_block \|\| kvm_x86_ops.pre_block(vcpu) == 0)) {
	9175	+ (!kvm_x86_ops.pre_block \|\| static_call(kvm_x86_pre_block)(vcpu) == 0)) {
9107	9176	srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
9108	9177	kvm_vcpu_block(vcpu);
9109	9178	vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
9110	9179
9111	9180	if (kvm_x86_ops.post_block)
9112		− kvm_x86_ops.post_block(vcpu);
	9181	+ static_call(kvm_x86_post_block)(vcpu);
9113	9182
9114	9183	if (!kvm_check_request(KVM_REQ_UNHALT, vcpu))
9115	9184	return 1;
@@ -9330,6 +9399,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) · 214 unmodified lines
9330	9399
9331	9400	vcpu_load(vcpu);
9332	9401	kvm_sigset_activate(vcpu);
	9402	+ kvm_run->flags = 0;
9333	9403	kvm_load_guest_fpu(vcpu);
9334	9404
9335	9405	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
@@ -9504,10 +9574,10 @@ static void __get_sregs(struct kvm_vcpu vcpu, struct kvm_sregs sregs) · 168 unmodified lines
9504	9574	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
9505	9575	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
9506	9576
9507		− kvm_x86_ops.get_idt(vcpu, &dt);
	9577	+ static_call(kvm_x86_get_idt)(vcpu, &dt);
9508	9578	sregs->idt.limit = dt.size;
9509	9579	sregs->idt.base = dt.address;
9510		− kvm_x86_ops.get_gdt(vcpu, &dt);
	9580	+ static_call(kvm_x86_get_gdt)(vcpu, &dt);
9511	9581	sregs->gdt.limit = dt.size;
9512	9582	sregs->gdt.base = dt.address;
9513	9583
@@ -9625,7 +9695,7 @@ static bool kvm_is_valid_sregs(struct kvm_vcpu vcpu, struct kvm_sregs sregs) · 111 unmodified lines
9625	9695	*/
9626	9696	if (!(sregs->cr4 & X86_CR4_PAE) \|\| !(sregs->efer & EFER_LMA))
9627	9697	return false;
9628		− if (sregs->cr3 & vcpu->arch.cr3_lm_rsvd_bits)
	9698	+ if (kvm_vcpu_is_illegal_gpa(vcpu, sregs->cr3))
9629	9699	return false;
9630	9700	} else {
9631	9701	/*
@@ -9660,10 +9730,10 @@ static int __set_sregs(struct kvm_vcpu vcpu, struct kvm_sregs sregs) · 28 unmodified lines
9660	9730
9661	9731	dt.size = sregs->idt.limit;
9662	9732	dt.address = sregs->idt.base;
9663		− kvm_x86_ops.set_idt(vcpu, &dt);
	9733	+ static_call(kvm_x86_set_idt)(vcpu, &dt);
9664	9734	dt.size = sregs->gdt.limit;
9665	9735	dt.address = sregs->gdt.base;
9666		− kvm_x86_ops.set_gdt(vcpu, &dt);
	9736	+ static_call(kvm_x86_set_gdt)(vcpu, &dt);
9667	9737
9668	9738	vcpu->arch.cr2 = sregs->cr2;
9669	9739	mmu_reset_needed \|= kvm_read_cr3(vcpu) != sregs->cr3;
@@ -9673,14 +9743,14 @@ static int __set_sregs(struct kvm_vcpu vcpu, struct kvm_sregs sregs) · 3 unmodified lines
9673	9743	kvm_set_cr8(vcpu, sregs->cr8);
9674	9744
9675	9745	mmu_reset_needed \|= vcpu->arch.efer != sregs->efer;
9676		− kvm_x86_ops.set_efer(vcpu, sregs->efer);
	9746	+ static_call(kvm_x86_set_efer)(vcpu, sregs->efer);
9677	9747
9678	9748	mmu_reset_needed \|= kvm_read_cr0(vcpu) != sregs->cr0;
9679		− kvm_x86_ops.set_cr0(vcpu, sregs->cr0);
	9749	+ static_call(kvm_x86_set_cr0)(vcpu, sregs->cr0);
9680	9750	vcpu->arch.cr0 = sregs->cr0;
9681	9751
9682	9752	mmu_reset_needed \|= kvm_read_cr4(vcpu) != sregs->cr4;
9683		− kvm_x86_ops.set_cr4(vcpu, sregs->cr4);
	9753	+ static_call(kvm_x86_set_cr4)(vcpu, sregs->cr4);
9684	9754
9685	9755	idx = srcu_read_lock(&vcpu->kvm->srcu);
9686	9756	if (is_pae_paging(vcpu)) {
@@ -9788,7 +9858,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, · 101 unmodified lines
9788	9858	*/
9789	9859	kvm_set_rflags(vcpu, rflags);
9790	9860
9791		− kvm_x86_ops.update_exception_bitmap(vcpu);
	9861	+ static_call(kvm_x86_update_exception_bitmap)(vcpu);
9792	9862
9793	9863	r = 0;
9794	9864
@@ -9966,7 +10036,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) · 171 unmodified lines
9966	10036	if (kvm_apicv_activated(vcpu->kvm))
9967	10037	vcpu->arch.apicv_active = true;
9968	10038	} else
9969		− static_key_slow_inc(&kvm_no_apic_vcpu);
	10039	+ static_branch_inc(&kvm_has_noapic_vcpu);
9970	10040
9971	10041	r = -ENOMEM;
9972	10042
@@ -10004,7 +10074,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) · 31 unmodified lines
10004	10074	fx_init(vcpu);
10005	10075
10006	10076	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
10007		− vcpu->arch.cr3_lm_rsvd_bits = rsvd_bits(cpuid_maxphyaddr(vcpu), 63);
	10077	+ vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu);
10008	10078
10009	10079	vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;
10010	10080
@@ -10014,9 +10084,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) · 3 unmodified lines
10014	10084	vcpu->arch.pending_external_vector = -1;
10015	10085	vcpu->arch.preempted_in_kernel = false;
10016	10086
10017		− kvm_hv_vcpu_init(vcpu);
10018		−
10019		− r = kvm_x86_ops.vcpu_create(vcpu);
	10087	+ r = static_call(kvm_x86_vcpu_create)(vcpu);
10020	10088	if (r)
10021	10089	goto free_guest_fpu;
10022	10090
@@ -10052,8 +10120,6 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) · 29 unmodified lines
10052	10120	{
10053	10121	struct kvm *kvm = vcpu->kvm;
10054	10122
10055		− kvm_hv_vcpu_postcreate(vcpu);
10056		−
10057	10123	if (mutex_lock_killable(&vcpu->mutex))
10058	10124	return;
10059	10125	vcpu_load(vcpu);
@@ -10079,7 +10145,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) · 19 unmodified lines
10079	10145
10080	10146	kvmclock_reset(vcpu);
10081	10147
10082		− kvm_x86_ops.vcpu_free(vcpu);
	10148	+ static_call(kvm_x86_vcpu_free)(vcpu);
10083	10149
10084	10150	kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
10085	10151	free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
@@ -10096,7 +10162,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) · 10 unmodified lines
10096	10162	free_page((unsigned long)vcpu->arch.pio_data);
10097	10163	kvfree(vcpu->arch.cpuid_entries);
10098	10164	if (!lapic_in_kernel(vcpu))
10099		− static_key_slow_dec(&kvm_no_apic_vcpu);
	10165	+ static_branch_dec(&kvm_has_noapic_vcpu);
10100	10166	}
10101	10167
10102	10168	void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
@@ -10115,7 +10181,7 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) · 12 unmodified lines
10115	10181
10116	10182	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
10117	10183	kvm_update_dr0123(vcpu);
10118		− vcpu->arch.dr6 = DR6_INIT;
	10184	+ vcpu->arch.dr6 = DR6_ACTIVE_LOW;
10119	10185	vcpu->arch.dr7 = DR7_FIXED_1;
10120	10186	kvm_update_dr7(vcpu);
10121	10187
@@ -10168,7 +10234,7 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) · 46 unmodified lines
10168	10234
10169	10235	vcpu->arch.ia32_xss = 0;
10170	10236
10171		− kvm_x86_ops.vcpu_reset(vcpu, init_event);
	10237	+ static_call(kvm_x86_vcpu_reset)(vcpu, init_event);
10172	10238	}
10173	10239
10174	10240	void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
@@ -10194,7 +10260,7 @@ int kvm_arch_hardware_enable(void) · 19 unmodified lines
10194	10260	bool stable, backwards_tsc = false;
10195	10261
10196	10262	kvm_user_return_msr_cpu_online();
10197		− ret = kvm_x86_ops.hardware_enable();
	10263	+ ret = static_call(kvm_x86_hardware_enable)();
10198	10264	if (ret != 0)
10199	10265	return ret;
10200	10266
@@ -10276,7 +10342,7 @@ int kvm_arch_hardware_enable(void) · 75 unmodified lines
10276	10342
10277	10343	void kvm_arch_hardware_disable(void)
10278	10344	{
10279		− kvm_x86_ops.hardware_disable();
	10345	+ static_call(kvm_x86_hardware_disable)();
10280	10346	drop_user_return_notifiers();
10281	10347	}
10282	10348
@@ -10295,6 +10361,7 @@ int kvm_arch_hardware_setup(void *opaque) · 12 unmodified lines
10295	10361	return r;
10296	10362
10297	10363	memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
	10364	+ kvm_ops_static_call_update();
10298	10365
10299	10366	if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
10300	10367	supported_xss = 0;
@@ -10323,7 +10390,7 @@ int kvm_arch_hardware_setup(void *opaque) · 22 unmodified lines
10323	10390
10324	10391	void kvm_arch_hardware_unsetup(void)
10325	10392	{
10326		− kvm_x86_ops.hardware_unsetup();
	10393	+ static_call(kvm_x86_hardware_unsetup)();
10327	10394	}
10328	10395
10329	10396	int kvm_arch_check_processor_compat(void *opaque)
@@ -10351,8 +10418,8 @@ bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu) · 21 unmodified lines
10351	10418	return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
10352	10419	}
10353	10420
10354		−struct static_key kvm_no_apic_vcpu __read_mostly;
10355		−EXPORT_SYMBOL_GPL(kvm_no_apic_vcpu);
	10421	+__read_mostly DEFINE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu);
	10422	+EXPORT_SYMBOL_GPL(kvm_has_noapic_vcpu);
10356	10423
10357	10424	void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
10358	10425	{
@@ -10363,12 +10430,12 @@ void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) · 4 unmodified lines
10363	10430	pmu->need_cleanup = true;
10364	10431	kvm_make_request(KVM_REQ_PMU, vcpu);
10365	10432	}
10366		− kvm_x86_ops.sched_in(vcpu, cpu);
	10433	+ static_call(kvm_x86_sched_in)(vcpu, cpu);
10367	10434	}
10368	10435
10369	10436	void kvm_arch_free_vm(struct kvm *kvm)
10370	10437	{
10371		− kfree(kvm->arch.hyperv.hv_pa_pg);
	10438	+ kfree(to_kvm_hv(kvm)->hv_pa_pg);
10372	10439	vfree(kvm);
10373	10440	}
10374	10441
@@ -10407,7 +10474,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 32 unmodified lines
10407	10474	kvm_page_track_init(kvm);
10408	10475	kvm_mmu_init_vm(kvm);
10409	10476
10410		− return kvm_x86_ops.vm_init(kvm);
	10477	+ return static_call(kvm_x86_vm_init)(kvm);
10411	10478	}
10412	10479
10413	10480	int kvm_arch_post_init_vm(struct kvm *kvm)
@@ -10552,8 +10619,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm) · 138 unmodified lines
10552	10619	__x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);
10553	10620	mutex_unlock(&kvm->slots_lock);
10554	10621	}
10555		− if (kvm_x86_ops.vm_destroy)
10556		− kvm_x86_ops.vm_destroy(kvm);
	10622	+ static_call_cond(kvm_x86_vm_destroy)(kvm);
10557	10623	for (i = 0; i < kvm->arch.msr_filter.count; i++)
10558	10624	kfree(kvm->arch.msr_filter.ranges[i].bitmap);
10559	10625	kvm_pic_destroy(kvm);
@@ -10563,6 +10629,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm) · 3 unmodified lines
10563	10629	kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));
10564	10630	kvm_mmu_uninit_vm(kvm);
10565	10631	kvm_page_track_cleanup(kvm);
	10632	+ kvm_xen_destroy_vm(kvm);
10566	10633	kvm_hv_destroy_vm(kvm);
10567	10634	}
10568	10635
@@ -10744,7 +10811,7 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, · 175 unmodified lines
10744	10811	*/
10745	10812	if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
10746	10813	if (kvm_x86_ops.slot_enable_log_dirty) {
10747		− kvm_x86_ops.slot_enable_log_dirty(kvm, new);
	10814	+ static_call(kvm_x86_slot_enable_log_dirty)(kvm, new);
10748	10815	} else {
10749	10816	int level =
10750	10817	kvm_dirty_log_manual_protect_and_init_set(kvm) ?
@@ -10761,8 +10828,7 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, · 10 unmodified lines
10761	10828	kvm_mmu_slot_remove_write_access(kvm, new, level);
10762	10829	}
10763	10830	} else {
10764		− if (kvm_x86_ops.slot_disable_log_dirty)
10765		− kvm_x86_ops.slot_disable_log_dirty(kvm, new);
	10831	+ static_call_cond(kvm_x86_slot_disable_log_dirty)(kvm, new);
10766	10832	}
10767	10833	}
10768	10834
@@ -10801,7 +10867,7 @@ static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu) · 32 unmodified lines
10801	10867	{
10802	10868	return (is_guest_mode(vcpu) &&
10803	10869	kvm_x86_ops.guest_apic_has_interrupt &&
10804		− kvm_x86_ops.guest_apic_has_interrupt(vcpu));
	10870	+ static_call(kvm_x86_guest_apic_has_interrupt)(vcpu));
10805	10871	}
10806	10872
10807	10873	static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
@@ -10820,12 +10886,12 @@ static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) · 12 unmodified lines
10820	10886
10821	10887	if (kvm_test_request(KVM_REQ_NMI, vcpu) \|\|
10822	10888	(vcpu->arch.nmi_pending &&
10823		− kvm_x86_ops.nmi_allowed(vcpu, false)))
	10889	+ static_call(kvm_x86_nmi_allowed)(vcpu, false)))
10824	10890	return true;
10825	10891
10826	10892	if (kvm_test_request(KVM_REQ_SMI, vcpu) \|\|
10827	10893	(vcpu->arch.smi_pending &&
10828		− kvm_x86_ops.smi_allowed(vcpu, false)))
	10894	+ static_call(kvm_x86_smi_allowed)(vcpu, false)))
10829	10895	return true;
10830	10896
10831	10897	if (kvm_arch_interrupt_allowed(vcpu) &&
@@ -10859,7 +10925,7 @@ bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu) · 27 unmodified lines
10859	10925	kvm_test_request(KVM_REQ_EVENT, vcpu))
10860	10926	return true;
10861	10927
10862		− if (vcpu->arch.apicv_active && kvm_x86_ops.dy_apicv_has_pending_interrupt(vcpu))
	10928	+ if (vcpu->arch.apicv_active && static_call(kvm_x86_dy_apicv_has_pending_interrupt)(vcpu))
10863	10929	return true;
10864	10930
10865	10931	return false;
@@ -10877,7 +10943,7 @@ int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) · 11 unmodified lines
10877	10943
10878	10944	int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
10879	10945	{
10880		− return kvm_x86_ops.interrupt_allowed(vcpu, false);
	10946	+ return static_call(kvm_x86_interrupt_allowed)(vcpu, false);
10881	10947	}
10882	10948
10883	10949	unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
@@ -10903,7 +10969,7 @@ unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu) · 19 unmodified lines
10903	10969	{
10904	10970	unsigned long rflags;
10905	10971
10906		− rflags = kvm_x86_ops.get_rflags(vcpu);
	10972	+ rflags = static_call(kvm_x86_get_rflags)(vcpu);
10907	10973	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
10908	10974	rflags &= ~X86_EFLAGS_TF;
10909	10975	return rflags;
@@ -10915,7 +10981,7 @@ static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) · 5 unmodified lines
10915	10981	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
10916	10982	kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
10917	10983	rflags \|= X86_EFLAGS_TF;
10918		− kvm_x86_ops.set_rflags(vcpu, rflags);
	10984	+ static_call(kvm_x86_set_rflags)(vcpu, rflags);
10919	10985	}
10920	10986
10921	10987	void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
@@ -11045,7 +11111,7 @@ static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu) · 123 unmodified lines
11045	11111	return false;
11046	11112
11047	11113	if (!kvm_pv_async_pf_enabled(vcpu) \|\|
11048		− (vcpu->arch.apf.send_user_only && kvm_x86_ops.get_cpl(vcpu) == 0))
	11114	+ (vcpu->arch.apf.send_user_only && static_call(kvm_x86_get_cpl)(vcpu) == 0))
11049	11115	return false;
11050	11116
11051	11117	return true;
@@ -11190,7 +11256,7 @@ int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons, · 138 unmodified lines
11190	11256
11191	11257	irqfd->producer = prod;
11192	11258	kvm_arch_start_assignment(irqfd->kvm);
11193		− ret = kvm_x86_ops.update_pi_irte(irqfd->kvm,
	11259	+ ret = static_call(kvm_x86_update_pi_irte)(irqfd->kvm,
11194	11260	prod->irq, irqfd->gsi, 1);
11195	11261
11196	11262	if (ret)
@@ -11215,7 +11281,7 @@ void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, · 18 unmodified lines
11215	11281	* when the irq is masked/disabled or the consumer side (KVM
11216	11282	* int this case doesn't want to receive the interrupts.
11217	11283	*/
11218		− ret = kvm_x86_ops.update_pi_irte(irqfd->kvm, prod->irq, irqfd->gsi, 0);
	11284	+ ret = static_call(kvm_x86_update_pi_irte)(irqfd->kvm, prod->irq, irqfd->gsi, 0);
11219	11285	if (ret)
11220	11286	printk(KERN_INFO "irq bypass consumer (token %p) unregistration"
11221	11287	" fails: %d\n", irqfd->consumer.token, ret);
@@ -11226,7 +11292,7 @@ void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, · 4 unmodified lines
11226	11292	int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
11227	11293	uint32_t guest_irq, bool set)
11228	11294	{
11229		− return kvm_x86_ops.update_pi_irte(kvm, host_irq, guest_irq, set);
	11295	+ return static_call(kvm_x86_update_pi_irte)(kvm, host_irq, guest_irq, set);
11230	11296	}
11231	11297
11232	11298	bool kvm_vector_hashing_enabled(void)

arch/x86/kvm/x86.h+8−4

@@ -98,7 +98,7 @@ static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu) · 97 unmodified lines
98	98
99	99	if (!is_long_mode(vcpu))
100	100	return false;
101		− kvm_x86_ops.get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
	101	+ static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
102	102	return cs_l;
103	103	}
104	104
@@ -129,7 +129,7 @@ static inline bool mmu_is_nested(struct kvm_vcpu *vcpu) · 24 unmodified lines
129	129	static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu)
130	130	{
131	131	++vcpu->stat.tlb_flush;
132		− kvm_x86_ops.tlb_flush_current(vcpu);
	132	+ static_call(kvm_x86_tlb_flush_current)(vcpu);
133	133	}
134	134
135	135	static inline int is_pae(struct kvm_vcpu *vcpu)
@@ -244,9 +244,10 @@ static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk) · 108 unmodified lines
244	244
245	245	static inline bool kvm_vcpu_latch_init(struct kvm_vcpu *vcpu)
246	246	{
247		− return is_smm(vcpu) \|\| kvm_x86_ops.apic_init_signal_blocked(vcpu);
	247	+ return is_smm(vcpu) \|\| static_call(kvm_x86_apic_init_signal_blocked)(vcpu);
248	248	}
249	249
	250	+void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs);
250	251	void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip);
251	252
252	253	void kvm_write_tsc(struct kvm_vcpu vcpu, struct msr_data msr);
@@ -273,6 +274,8 @@ bool kvm_mtrr_check_gfn_range_consistency(struct kvm_vcpu *vcpu, gfn_t gfn, · 20 unmodified lines
273	274	int page_num);
274	275	bool kvm_vector_hashing_enabled(void);
275	276	void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code);
	277	+int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type,
	278	+ void *insn, int insn_len);
276	279	int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
277	280	int emulation_type, void *insn, int insn_len);
278	281	fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu);
@@ -296,6 +299,8 @@ extern int pi_inject_timer; · 17 unmodified lines
296	299
297	300	extern struct static_key kvm_no_apic_vcpu;
298	301
	302	+extern bool report_ignored_msrs;
	303	+
299	304	static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec)
300	305	{
301	306	return pvclock_scale_delta(nsec, vcpu->arch.virtual_tsc_mult,
@@ -391,7 +396,6 @@ void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu); · 89 unmodified lines
391	396	void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu);
392	397	int kvm_spec_ctrl_test_value(u64 value);
393	398	bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
394		−bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu);
395	399	int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r,
396	400	struct x86_exception *e);
397	401	int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva);

arch/x86/kvm/xen.c+431−0

@@ -0,0 +1,431 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * Copyright © 2019 Oracle and/or its affiliates. All rights reserved.
	4	+ * Copyright © 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
	5	+ *
	6	+ * KVM Xen emulation
	7	+ */
	8	+
	9	+#include "x86.h"
	10	+#include "xen.h"
	11	+#include "hyperv.h"
	12	+
	13	+#include <linux/kvm_host.h>
	14	+
	15	+#include <trace/events/kvm.h>
	16	+#include <xen/interface/xen.h>
	17	+
	18	+#include "trace.h"
	19	+
	20	+DEFINE_STATIC_KEY_DEFERRED_FALSE(kvm_xen_enabled, HZ);
	21	+
	22	+static int kvm_xen_shared_info_init(struct kvm *kvm, gfn_t gfn)
	23	+{
	24	+ gpa_t gpa = gfn_to_gpa(gfn);
	25	+ int wc_ofs, sec_hi_ofs;
	26	+ int ret;
	27	+ int idx = srcu_read_lock(&kvm->srcu);
	28	+
	29	+ ret = kvm_gfn_to_hva_cache_init(kvm, &kvm->arch.xen.shinfo_cache,
	30	+ gpa, PAGE_SIZE);
	31	+ if (ret)
	32	+ goto out;
	33	+
	34	+ kvm->arch.xen.shinfo_set = true;
	35	+
	36	+ /* Paranoia checks on the 32-bit struct layout */
	37	+ BUILD_BUG_ON(offsetof(struct compat_shared_info, wc) != 0x900);
	38	+ BUILD_BUG_ON(offsetof(struct compat_shared_info, arch.wc_sec_hi) != 0x924);
	39	+ BUILD_BUG_ON(offsetof(struct pvclock_vcpu_time_info, version) != 0);
	40	+
	41	+ /* 32-bit location by default */
	42	+ wc_ofs = offsetof(struct compat_shared_info, wc);
	43	+ sec_hi_ofs = offsetof(struct compat_shared_info, arch.wc_sec_hi);
	44	+
	45	+#ifdef CONFIG_X86_64
	46	+ /* Paranoia checks on the 64-bit struct layout */
	47	+ BUILD_BUG_ON(offsetof(struct shared_info, wc) != 0xc00);
	48	+ BUILD_BUG_ON(offsetof(struct shared_info, wc_sec_hi) != 0xc0c);
	49	+
	50	+ if (kvm->arch.xen.long_mode) {
	51	+ wc_ofs = offsetof(struct shared_info, wc);
	52	+ sec_hi_ofs = offsetof(struct shared_info, wc_sec_hi);
	53	+ }
	54	+#endif
	55	+
	56	+ kvm_write_wall_clock(kvm, gpa + wc_ofs, sec_hi_ofs - wc_ofs);
	57	+ kvm_make_all_cpus_request(kvm, KVM_REQ_MASTERCLOCK_UPDATE);
	58	+
	59	+out:
	60	+ srcu_read_unlock(&kvm->srcu, idx);
	61	+ return ret;
	62	+}
	63	+
	64	+int __kvm_xen_has_interrupt(struct kvm_vcpu *v)
	65	+{
	66	+ u8 rc = 0;
	67	+
	68	+ /*
	69	+ * If the global upcall vector (HVMIRQ_callback_vector) is set and
	70	+ * the vCPU's evtchn_upcall_pending flag is set, the IRQ is pending.
	71	+ */
	72	+ struct gfn_to_hva_cache *ghc = &v->arch.xen.vcpu_info_cache;
	73	+ struct kvm_memslots *slots = kvm_memslots(v->kvm);
	74	+ unsigned int offset = offsetof(struct vcpu_info, evtchn_upcall_pending);
	75	+
	76	+ /* No need for compat handling here */
	77	+ BUILD_BUG_ON(offsetof(struct vcpu_info, evtchn_upcall_pending) !=
	78	+ offsetof(struct compat_vcpu_info, evtchn_upcall_pending));
	79	+ BUILD_BUG_ON(sizeof(rc) !=
	80	+ sizeof(((struct vcpu_info *)0)->evtchn_upcall_pending));
	81	+ BUILD_BUG_ON(sizeof(rc) !=
	82	+ sizeof(((struct compat_vcpu_info *)0)->evtchn_upcall_pending));
	83	+
	84	+ /*
	85	+ * For efficiency, this mirrors the checks for using the valid
	86	+ * cache in kvm_read_guest_offset_cached(), but just uses
	87	+ * __get_user() instead. And falls back to the slow path.
	88	+ */
	89	+ if (likely(slots->generation == ghc->generation &&
	90	+ !kvm_is_error_hva(ghc->hva) && ghc->memslot)) {
	91	+ /* Fast path */
	92	+ __get_user(rc, (u8 __user *)ghc->hva + offset);
	93	+ } else {
	94	+ /* Slow path */
	95	+ kvm_read_guest_offset_cached(v->kvm, ghc, &rc, offset,
	96	+ sizeof(rc));
	97	+ }
	98	+
	99	+ return rc;
	100	+}
	101	+
	102	+int kvm_xen_hvm_set_attr(struct kvm kvm, struct kvm_xen_hvm_attr data)
	103	+{
	104	+ int r = -ENOENT;
	105	+
	106	+ mutex_lock(&kvm->lock);
	107	+
	108	+ switch (data->type) {
	109	+ case KVM_XEN_ATTR_TYPE_LONG_MODE:
	110	+ if (!IS_ENABLED(CONFIG_64BIT) && data->u.long_mode) {
	111	+ r = -EINVAL;
	112	+ } else {
	113	+ kvm->arch.xen.long_mode = !!data->u.long_mode;
	114	+ r = 0;
	115	+ }
	116	+ break;
	117	+
	118	+ case KVM_XEN_ATTR_TYPE_SHARED_INFO:
	119	+ if (data->u.shared_info.gfn == GPA_INVALID) {
	120	+ kvm->arch.xen.shinfo_set = false;
	121	+ r = 0;
	122	+ break;
	123	+ }
	124	+ r = kvm_xen_shared_info_init(kvm, data->u.shared_info.gfn);
	125	+ break;
	126	+
	127	+
	128	+ case KVM_XEN_ATTR_TYPE_UPCALL_VECTOR:
	129	+ if (data->u.vector && data->u.vector < 0x10)
	130	+ r = -EINVAL;
	131	+ else {
	132	+ kvm->arch.xen.upcall_vector = data->u.vector;
	133	+ r = 0;
	134	+ }
	135	+ break;
	136	+
	137	+ default:
	138	+ break;
	139	+ }
	140	+
	141	+ mutex_unlock(&kvm->lock);
	142	+ return r;
	143	+}
	144	+
	145	+int kvm_xen_hvm_get_attr(struct kvm kvm, struct kvm_xen_hvm_attr data)
	146	+{
	147	+ int r = -ENOENT;
	148	+
	149	+ mutex_lock(&kvm->lock);
	150	+
	151	+ switch (data->type) {
	152	+ case KVM_XEN_ATTR_TYPE_LONG_MODE:
	153	+ data->u.long_mode = kvm->arch.xen.long_mode;
	154	+ r = 0;
	155	+ break;
	156	+
	157	+ case KVM_XEN_ATTR_TYPE_SHARED_INFO:
	158	+ if (kvm->arch.xen.shinfo_set)
	159	+ data->u.shared_info.gfn = gpa_to_gfn(kvm->arch.xen.shinfo_cache.gpa);
	160	+ else
	161	+ data->u.shared_info.gfn = GPA_INVALID;
	162	+ r = 0;
	163	+ break;
	164	+
	165	+ case KVM_XEN_ATTR_TYPE_UPCALL_VECTOR:
	166	+ data->u.vector = kvm->arch.xen.upcall_vector;
	167	+ r = 0;
	168	+ break;
	169	+
	170	+ default:
	171	+ break;
	172	+ }
	173	+
	174	+ mutex_unlock(&kvm->lock);
	175	+ return r;
	176	+}
	177	+
	178	+int kvm_xen_vcpu_set_attr(struct kvm_vcpu vcpu, struct kvm_xen_vcpu_attr data)
	179	+{
	180	+ int idx, r = -ENOENT;
	181	+
	182	+ mutex_lock(&vcpu->kvm->lock);
	183	+ idx = srcu_read_lock(&vcpu->kvm->srcu);
	184	+
	185	+ switch (data->type) {
	186	+ case KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO:
	187	+ /* No compat necessary here. */
	188	+ BUILD_BUG_ON(sizeof(struct vcpu_info) !=
	189	+ sizeof(struct compat_vcpu_info));
	190	+
	191	+ if (data->u.gpa == GPA_INVALID) {
	192	+ vcpu->arch.xen.vcpu_info_set = false;
	193	+ break;
	194	+ }
	195	+
	196	+ r = kvm_gfn_to_hva_cache_init(vcpu->kvm,
	197	+ &vcpu->arch.xen.vcpu_info_cache,
	198	+ data->u.gpa,
	199	+ sizeof(struct vcpu_info));
	200	+ if (!r) {
	201	+ vcpu->arch.xen.vcpu_info_set = true;
	202	+ kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
	203	+ }
	204	+ break;
	205	+
	206	+ case KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO:
	207	+ if (data->u.gpa == GPA_INVALID) {
	208	+ vcpu->arch.xen.vcpu_time_info_set = false;
	209	+ break;
	210	+ }
	211	+
	212	+ r = kvm_gfn_to_hva_cache_init(vcpu->kvm,
	213	+ &vcpu->arch.xen.vcpu_time_info_cache,
	214	+ data->u.gpa,
	215	+ sizeof(struct pvclock_vcpu_time_info));
	216	+ if (!r) {
	217	+ vcpu->arch.xen.vcpu_time_info_set = true;
	218	+ kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
	219	+ }
	220	+ break;
	221	+
	222	+ default:
	223	+ break;
	224	+ }
	225	+
	226	+ srcu_read_unlock(&vcpu->kvm->srcu, idx);
	227	+ mutex_unlock(&vcpu->kvm->lock);
	228	+ return r;
	229	+}
	230	+
	231	+int kvm_xen_vcpu_get_attr(struct kvm_vcpu vcpu, struct kvm_xen_vcpu_attr data)
	232	+{
	233	+ int r = -ENOENT;
	234	+
	235	+ mutex_lock(&vcpu->kvm->lock);
	236	+
	237	+ switch (data->type) {
	238	+ case KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO:
	239	+ if (vcpu->arch.xen.vcpu_info_set)
	240	+ data->u.gpa = vcpu->arch.xen.vcpu_info_cache.gpa;
	241	+ else
	242	+ data->u.gpa = GPA_INVALID;
	243	+ r = 0;
	244	+ break;
	245	+
	246	+ case KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO:
	247	+ if (vcpu->arch.xen.vcpu_time_info_set)
	248	+ data->u.gpa = vcpu->arch.xen.vcpu_time_info_cache.gpa;
	249	+ else
	250	+ data->u.gpa = GPA_INVALID;
	251	+ r = 0;
	252	+ break;
	253	+
	254	+ default:
	255	+ break;
	256	+ }
	257	+
	258	+ mutex_unlock(&vcpu->kvm->lock);
	259	+ return r;
	260	+}
	261	+
	262	+int kvm_xen_write_hypercall_page(struct kvm_vcpu *vcpu, u64 data)
	263	+{
	264	+ struct kvm *kvm = vcpu->kvm;
	265	+ u32 page_num = data & ~PAGE_MASK;
	266	+ u64 page_addr = data & PAGE_MASK;
	267	+ bool lm = is_long_mode(vcpu);
	268	+
	269	+ /* Latch long_mode for shared_info pages etc. */
	270	+ vcpu->kvm->arch.xen.long_mode = lm;
	271	+
	272	+ /*
	273	+ * If Xen hypercall intercept is enabled, fill the hypercall
	274	+ * page with VMCALL/VMMCALL instructions since that's what
	275	+ * we catch. Else the VMM has provided the hypercall pages
	276	+ * with instructions of its own choosing, so use those.
	277	+ */
	278	+ if (kvm_xen_hypercall_enabled(kvm)) {
	279	+ u8 instructions[32];
	280	+ int i;
	281	+
	282	+ if (page_num)
	283	+ return 1;
	284	+
	285	+ /* mov imm32, %eax */
	286	+ instructions[0] = 0xb8;
	287	+
	288	+ /* vmcall / vmmcall */
	289	+ kvm_x86_ops.patch_hypercall(vcpu, instructions + 5);
	290	+
	291	+ /* ret */
	292	+ instructions[8] = 0xc3;
	293	+
	294	+ /* int3 to pad */
	295	+ memset(instructions + 9, 0xcc, sizeof(instructions) - 9);
	296	+
	297	+ for (i = 0; i < PAGE_SIZE / sizeof(instructions); i++) {
	298	+ (u32 )&instructions[1] = i;
	299	+ if (kvm_vcpu_write_guest(vcpu,
	300	+ page_addr + (i * sizeof(instructions)),
	301	+ instructions, sizeof(instructions)))
	302	+ return 1;
	303	+ }
	304	+ } else {
	305	+ /*
	306	+ * Note, truncation is a non-issue as 'lm' is guaranteed to be
	307	+ * false for a 32-bit kernel, i.e. when hva_t is only 4 bytes.
	308	+ */
	309	+ hva_t blob_addr = lm ? kvm->arch.xen_hvm_config.blob_addr_64
	310	+ : kvm->arch.xen_hvm_config.blob_addr_32;
	311	+ u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
	312	+ : kvm->arch.xen_hvm_config.blob_size_32;
	313	+ u8 *page;
	314	+
	315	+ if (page_num >= blob_size)
	316	+ return 1;
	317	+
	318	+ blob_addr += page_num * PAGE_SIZE;
	319	+
	320	+ page = memdup_user((u8 __user *)blob_addr, PAGE_SIZE);
	321	+ if (IS_ERR(page))
	322	+ return PTR_ERR(page);
	323	+
	324	+ if (kvm_vcpu_write_guest(vcpu, page_addr, page, PAGE_SIZE)) {
	325	+ kfree(page);
	326	+ return 1;
	327	+ }
	328	+ }
	329	+ return 0;
	330	+}
	331	+
	332	+int kvm_xen_hvm_config(struct kvm kvm, struct kvm_xen_hvm_config xhc)
	333	+{
	334	+ if (xhc->flags & ~KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL)
	335	+ return -EINVAL;
	336	+
	337	+ /*
	338	+ * With hypercall interception the kernel generates its own
	339	+ * hypercall page so it must not be provided.
	340	+ */
	341	+ if ((xhc->flags & KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL) &&
	342	+ (xhc->blob_addr_32 \|\| xhc->blob_addr_64 \|\|
	343	+ xhc->blob_size_32 \|\| xhc->blob_size_64))
	344	+ return -EINVAL;
	345	+
	346	+ mutex_lock(&kvm->lock);
	347	+
	348	+ if (xhc->msr && !kvm->arch.xen_hvm_config.msr)
	349	+ static_branch_inc(&kvm_xen_enabled.key);
	350	+ else if (!xhc->msr && kvm->arch.xen_hvm_config.msr)
	351	+ static_branch_slow_dec_deferred(&kvm_xen_enabled);
	352	+
	353	+ memcpy(&kvm->arch.xen_hvm_config, xhc, sizeof(*xhc));
	354	+
	355	+ mutex_unlock(&kvm->lock);
	356	+ return 0;
	357	+}
	358	+
	359	+void kvm_xen_destroy_vm(struct kvm *kvm)
	360	+{
	361	+ if (kvm->arch.xen_hvm_config.msr)
	362	+ static_branch_slow_dec_deferred(&kvm_xen_enabled);
	363	+}
	364	+
	365	+static int kvm_xen_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result)
	366	+{
	367	+ kvm_rax_write(vcpu, result);
	368	+ return kvm_skip_emulated_instruction(vcpu);
	369	+}
	370	+
	371	+static int kvm_xen_hypercall_complete_userspace(struct kvm_vcpu *vcpu)
	372	+{
	373	+ struct kvm_run *run = vcpu->run;
	374	+
	375	+ if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.xen.hypercall_rip)))
	376	+ return 1;
	377	+
	378	+ return kvm_xen_hypercall_set_result(vcpu, run->xen.u.hcall.result);
	379	+}
	380	+
	381	+int kvm_xen_hypercall(struct kvm_vcpu *vcpu)
	382	+{
	383	+ bool longmode;
	384	+ u64 input, params[6];
	385	+
	386	+ input = (u64)kvm_register_read(vcpu, VCPU_REGS_RAX);
	387	+
	388	+ /* Hyper-V hypercalls get bit 31 set in EAX */
	389	+ if ((input & 0x80000000) &&
	390	+ kvm_hv_hypercall_enabled(vcpu))
	391	+ return kvm_hv_hypercall(vcpu);
	392	+
	393	+ longmode = is_64_bit_mode(vcpu);
	394	+ if (!longmode) {
	395	+ params[0] = (u32)kvm_rbx_read(vcpu);
	396	+ params[1] = (u32)kvm_rcx_read(vcpu);
	397	+ params[2] = (u32)kvm_rdx_read(vcpu);
	398	+ params[3] = (u32)kvm_rsi_read(vcpu);
	399	+ params[4] = (u32)kvm_rdi_read(vcpu);
	400	+ params[5] = (u32)kvm_rbp_read(vcpu);
	401	+ }
	402	+#ifdef CONFIG_X86_64
	403	+ else {
	404	+ params[0] = (u64)kvm_rdi_read(vcpu);
	405	+ params[1] = (u64)kvm_rsi_read(vcpu);
	406	+ params[2] = (u64)kvm_rdx_read(vcpu);
	407	+ params[3] = (u64)kvm_r10_read(vcpu);
	408	+ params[4] = (u64)kvm_r8_read(vcpu);
	409	+ params[5] = (u64)kvm_r9_read(vcpu);
	410	+ }
	411	+#endif
	412	+ trace_kvm_xen_hypercall(input, params[0], params[1], params[2],
	413	+ params[3], params[4], params[5]);
	414	+
	415	+ vcpu->run->exit_reason = KVM_EXIT_XEN;
	416	+ vcpu->run->xen.type = KVM_EXIT_XEN_HCALL;
	417	+ vcpu->run->xen.u.hcall.longmode = longmode;
	418	+ vcpu->run->xen.u.hcall.cpl = kvm_x86_ops.get_cpl(vcpu);
	419	+ vcpu->run->xen.u.hcall.input = input;
	420	+ vcpu->run->xen.u.hcall.params[0] = params[0];
	421	+ vcpu->run->xen.u.hcall.params[1] = params[1];
	422	+ vcpu->run->xen.u.hcall.params[2] = params[2];
	423	+ vcpu->run->xen.u.hcall.params[3] = params[3];
	424	+ vcpu->run->xen.u.hcall.params[4] = params[4];
	425	+ vcpu->run->xen.u.hcall.params[5] = params[5];
	426	+ vcpu->arch.xen.hypercall_rip = kvm_get_linear_rip(vcpu);
	427	+ vcpu->arch.complete_userspace_io =
	428	+ kvm_xen_hypercall_complete_userspace;
	429	+
	430	+ return 0;
	431	+}

arch/x86/kvm/xen.h+78−0

@@ -0,0 +1,78 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * Copyright © 2019 Oracle and/or its affiliates. All rights reserved.
	4	+ * Copyright © 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
	5	+ *
	6	+ * KVM Xen emulation
	7	+ */
	8	+
	9	+#ifndef __ARCH_X86_KVM_XEN_H__
	10	+#define __ARCH_X86_KVM_XEN_H__
	11	+
	12	+#include <linux/jump_label_ratelimit.h>
	13	+
	14	+extern struct static_key_false_deferred kvm_xen_enabled;
	15	+
	16	+int __kvm_xen_has_interrupt(struct kvm_vcpu *vcpu);
	17	+int kvm_xen_vcpu_set_attr(struct kvm_vcpu vcpu, struct kvm_xen_vcpu_attr data);
	18	+int kvm_xen_vcpu_get_attr(struct kvm_vcpu vcpu, struct kvm_xen_vcpu_attr data);
	19	+int kvm_xen_hvm_set_attr(struct kvm kvm, struct kvm_xen_hvm_attr data);
	20	+int kvm_xen_hvm_get_attr(struct kvm kvm, struct kvm_xen_hvm_attr data);
	21	+int kvm_xen_hypercall(struct kvm_vcpu *vcpu);
	22	+int kvm_xen_write_hypercall_page(struct kvm_vcpu *vcpu, u64 data);
	23	+int kvm_xen_hvm_config(struct kvm kvm, struct kvm_xen_hvm_config xhc);
	24	+void kvm_xen_destroy_vm(struct kvm *kvm);
	25	+
	26	+static inline bool kvm_xen_hypercall_enabled(struct kvm *kvm)
	27	+{
	28	+ return static_branch_unlikely(&kvm_xen_enabled.key) &&
	29	+ (kvm->arch.xen_hvm_config.flags &
	30	+ KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL);
	31	+}
	32	+
	33	+static inline int kvm_xen_has_interrupt(struct kvm_vcpu *vcpu)
	34	+{
	35	+ if (static_branch_unlikely(&kvm_xen_enabled.key) &&
	36	+ vcpu->arch.xen.vcpu_info_set && vcpu->kvm->arch.xen.upcall_vector)
	37	+ return __kvm_xen_has_interrupt(vcpu);
	38	+
	39	+ return 0;
	40	+}
	41	+
	42	+/* 32-bit compatibility definitions, also used natively in 32-bit build */
	43	+#include <asm/pvclock-abi.h>
	44	+#include <asm/xen/interface.h>
	45	+
	46	+struct compat_arch_vcpu_info {
	47	+ unsigned int cr2;
	48	+ unsigned int pad[5];
	49	+};
	50	+
	51	+struct compat_vcpu_info {
	52	+ uint8_t evtchn_upcall_pending;
	53	+ uint8_t evtchn_upcall_mask;
	54	+ uint16_t pad;
	55	+ uint32_t evtchn_pending_sel;
	56	+ struct compat_arch_vcpu_info arch;
	57	+ struct pvclock_vcpu_time_info time;
	58	+}; /* 64 bytes (x86) */
	59	+
	60	+struct compat_arch_shared_info {
	61	+ unsigned int max_pfn;
	62	+ unsigned int pfn_to_mfn_frame_list_list;
	63	+ unsigned int nmi_reason;
	64	+ unsigned int p2m_cr3;
	65	+ unsigned int p2m_vaddr;
	66	+ unsigned int p2m_generation;
	67	+ uint32_t wc_sec_hi;
	68	+};
	69	+
	70	+struct compat_shared_info {
	71	+ struct compat_vcpu_info vcpu_info[MAX_VIRT_CPUS];
	72	+ uint32_t evtchn_pending[32];
	73	+ uint32_t evtchn_mask[32];
	74	+ struct pvclock_wall_clock wc;
	75	+ struct compat_arch_shared_info arch;
	76	+};
	77	+
	78	+#endif /* __ARCH_X86_KVM_XEN_H__ */

arch/xtensa/include/asm/spinlock.h+1−1

@@ -12,8 +12,8 @@ · 11 unmodified lines
12	12	#define _XTENSA_SPINLOCK_H
13	13
14	14	#include <asm/barrier.h>
15		−#include <asm/qrwlock.h>
16	15	#include <asm/qspinlock.h>
	16	+#include <asm/qrwlock.h>
17	17
18	18	#define smp_mb__after_spinlock() smp_mb()
19	19

drivers/crypto/ccp/sev-dev.c+1−0

@@ -128,6 +128,7 @@ static int sev_cmd_buffer_len(int cmd) · 127 unmodified lines
128	128	case SEV_CMD_LAUNCH_UPDATE_SECRET: return sizeof(struct sev_data_launch_secret);
129	129	case SEV_CMD_DOWNLOAD_FIRMWARE: return sizeof(struct sev_data_download_firmware);
130	130	case SEV_CMD_GET_ID: return sizeof(struct sev_data_get_id);
	131	+ case SEV_CMD_ATTESTATION_REPORT: return sizeof(struct sev_data_attestation_report);
131	132	default: return 0;
132	133	}
133	134

drivers/gpu/drm/i915/gvt/kvmgt.c+6−6

@@ -1703,7 +1703,7 @@ static int kvmgt_page_track_add(unsigned long handle, u64 gfn) · 1702 unmodified lines
1703	1703	return -EINVAL;
1704	1704	}
1705	1705
1706		− spin_lock(&kvm->mmu_lock);
	1706	+ write_lock(&kvm->mmu_lock);
1707	1707
1708	1708	if (kvmgt_gfn_is_write_protected(info, gfn))
1709	1709	goto out;
@@ -1712,7 +1712,7 @@ static int kvmgt_page_track_add(unsigned long handle, u64 gfn) · 2 unmodified lines
1712	1712	kvmgt_protect_table_add(info, gfn);
1713	1713
1714	1714	out:
1715		− spin_unlock(&kvm->mmu_lock);
	1715	+ write_unlock(&kvm->mmu_lock);
1716	1716	srcu_read_unlock(&kvm->srcu, idx);
1717	1717	return 0;
1718	1718	}
@@ -1737,7 +1737,7 @@ static int kvmgt_page_track_remove(unsigned long handle, u64 gfn) · 18 unmodified lines
1737	1737	return -EINVAL;
1738	1738	}
1739	1739
1740		− spin_lock(&kvm->mmu_lock);
	1740	+ write_lock(&kvm->mmu_lock);
1741	1741
1742	1742	if (!kvmgt_gfn_is_write_protected(info, gfn))
1743	1743	goto out;
@@ -1746,7 +1746,7 @@ static int kvmgt_page_track_remove(unsigned long handle, u64 gfn) · 2 unmodified lines
1746	1746	kvmgt_protect_table_del(info, gfn);
1747	1747
1748	1748	out:
1749		− spin_unlock(&kvm->mmu_lock);
	1749	+ write_unlock(&kvm->mmu_lock);
1750	1750	srcu_read_unlock(&kvm->srcu, idx);
1751	1751	return 0;
1752	1752	}
@@ -1772,7 +1772,7 @@ static void kvmgt_page_track_flush_slot(struct kvm *kvm, · 19 unmodified lines
1772	1772	struct kvmgt_guest_info *info = container_of(node,
1773	1773	struct kvmgt_guest_info, track_node);
1774	1774
1775		− spin_lock(&kvm->mmu_lock);
	1775	+ write_lock(&kvm->mmu_lock);
1776	1776	for (i = 0; i < slot->npages; i++) {
1777	1777	gfn = slot->base_gfn + i;
1778	1778	if (kvmgt_gfn_is_write_protected(info, gfn)) {
@@ -1781,7 +1781,7 @@ static void kvmgt_page_track_flush_slot(struct kvm *kvm, · 2 unmodified lines
1781	1781	kvmgt_protect_table_del(info, gfn);
1782	1782	}
1783	1783	}
1784		− spin_unlock(&kvm->mmu_lock);
	1784	+ write_unlock(&kvm->mmu_lock);
1785	1785	}
1786	1786
1787	1787	static bool __kvmgt_vgpu_exist(struct intel_vgpu vgpu, struct kvm kvm)

fs/dax.c+3−2

@@ -810,11 +810,12 @@ static void dax_entry_mkclean(struct address_space *mapping, pgoff_t index, · 809 unmodified lines
810	810	address = pgoff_address(index, vma);
811	811
812	812	/*
813		− * Note because we provide range to follow_pte it will call
	813	+ * follow_invalidate_pte() will use the range to call
814	814	* mmu_notifier_invalidate_range_start() on our behalf before
815	815	* taking any lock.
816	816	*/
817		− if (follow_pte(vma->vm_mm, address, &range, &ptep, &pmdp, &ptl))
	817	+ if (follow_invalidate_pte(vma->vm_mm, address, &range, &ptep,
	818	+ &pmdp, &ptl))
818	819	continue;
819	820
820	821	/*

include/asm-generic/export.h+1−1

@@ -33,7 +33,7 @@ · 32 unmodified lines
33	33	*/
34	34
35	35	.macro ___EXPORT_SYMBOL name,val,sec
36		−#ifdef CONFIG_MODULES
	36	+#if defined(CONFIG_MODULES) && !defined(__DISABLE_EXPORTS)
37	37	.section ___ksymtab\sec+\name,"a"
38	38	.balign KSYM_ALIGN
39	39	__ksymtab_\name:

include/asm-generic/qrwlock.h+19−6

@@ -15,6 +15,8 @@ · 14 unmodified lines
15	15
16	16	#include <asm-generic/qrwlock_types.h>
17	17
	18	+/* Must be included from asm/spinlock.h after defining arch_spin_is_locked. */
	19	+
18	20	/*
19	21	* Writer states & reader shift and bias.
20	22	*/
@@ -116,15 +118,26 @@ static inline void queued_write_unlock(struct qrwlock *lock) · 95 unmodified lines
116	118	smp_store_release(&lock->wlocked, 0);
117	119	}
118	120
	121	+/**
	122	+ * queued_rwlock_is_contended - check if the lock is contended
	123	+ * @lock : Pointer to queue rwlock structure
	124	+ * Return: 1 if lock contended, 0 otherwise
	125	+ */
	126	+static inline int queued_rwlock_is_contended(struct qrwlock *lock)
	127	+{
	128	+ return arch_spin_is_locked(&lock->wait_lock);
	129	+}
	130	+
119	131	/*
120	132	* Remapping rwlock architecture specific functions to the corresponding
121	133	* queue rwlock functions.
122	134	*/
123		−#define arch_read_lock(l) queued_read_lock(l)
124		−#define arch_write_lock(l) queued_write_lock(l)
125		−#define arch_read_trylock(l) queued_read_trylock(l)
126		−#define arch_write_trylock(l) queued_write_trylock(l)
127		−#define arch_read_unlock(l) queued_read_unlock(l)
128		−#define arch_write_unlock(l) queued_write_unlock(l)
	135	+#define arch_read_lock(l) queued_read_lock(l)
	136	+#define arch_write_lock(l) queued_write_lock(l)
	137	+#define arch_read_trylock(l) queued_read_trylock(l)
	138	+#define arch_write_trylock(l) queued_write_trylock(l)
	139	+#define arch_read_unlock(l) queued_read_unlock(l)
	140	+#define arch_write_unlock(l) queued_write_unlock(l)
	141	+#define arch_rwlock_is_contended(l) queued_rwlock_is_contended(l)
129	142
130	143	#endif /* __ASM_GENERIC_QRWLOCK_H */

include/linux/kvm_host.h+7−3

@@ -425,9 +425,8 @@ struct kvm_irq_routing_table { · 424 unmodified lines
425	425	#define KVM_PRIVATE_MEM_SLOTS 0
426	426	#endif
427	427
428		−#ifndef KVM_MEM_SLOTS_NUM
429		−#define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS)
430		−#endif
	428	+#define KVM_MEM_SLOTS_NUM SHRT_MAX
	429	+#define KVM_USER_MEM_SLOTS (KVM_MEM_SLOTS_NUM - KVM_PRIVATE_MEM_SLOTS)
431	430
432	431	#ifndef __KVM_VCPU_MULTIPLE_ADDRESS_SPACE
433	432	static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu)
@@ -451,7 +450,12 @@ struct kvm_memslots { · 17 unmodified lines
451	450	};
452	451
453	452	struct kvm {
	453	+#ifdef KVM_HAVE_MMU_RWLOCK
	454	+ rwlock_t mmu_lock;
	455	+#else
454	456	spinlock_t mmu_lock;
	457	+#endif /* KVM_HAVE_MMU_RWLOCK */
	458	+
455	459	struct mutex slots_lock;
456	460	struct mm_struct mm; / userspace tied to this vm */
457	461	struct kvm_memslots __rcu *memslots[KVM_ADDRESS_SPACE_NUM];

include/linux/mm.h+4−2

@@ -1663,9 +1663,11 @@ void free_pgd_range(struct mmu_gather *tlb, unsigned long addr, · 1662 unmodified lines
1663	1663	unsigned long end, unsigned long floor, unsigned long ceiling);
1664	1664	int
1665	1665	copy_page_range(struct vm_area_struct dst_vma, struct vm_area_struct src_vma);
	1666	+int follow_invalidate_pte(struct mm_struct *mm, unsigned long address,
	1667	+ struct mmu_notifier_range range, pte_t *ptepp,
	1668	+ pmd_t pmdpp, spinlock_t ptlp);
1666	1669	int follow_pte(struct mm_struct *mm, unsigned long address,
1667		− struct mmu_notifier_range range, pte_t ptepp, pmd_t *pmdpp,
1668		− spinlock_t **ptlp);
	1670	+ pte_t ptepp, spinlock_t ptlp);
1669	1671	int follow_pfn(struct vm_area_struct *vma, unsigned long address,
1670	1672	unsigned long *pfn);
1671	1673	int follow_phys(struct vm_area_struct *vma, unsigned long address,

include/linux/psp-sev.h+17−0

@@ -66,6 +66,7 @@ enum sev_cmd { · 65 unmodified lines
66	66	SEV_CMD_LAUNCH_MEASURE = 0x033,
67	67	SEV_CMD_LAUNCH_UPDATE_SECRET = 0x034,
68	68	SEV_CMD_LAUNCH_FINISH = 0x035,
	69	+ SEV_CMD_ATTESTATION_REPORT = 0x036,
69	70
70	71	/* Guest migration commands (outgoing) */
71	72	SEV_CMD_SEND_START = 0x040,
@@ -483,6 +484,22 @@ struct sev_data_dbg { · 411 unmodified lines
483	484	u32 len; /* In */
484	485	} __packed;
485	486
	487	+/**
	488	+ * struct sev_data_attestation_report - SEV_ATTESTATION_REPORT command parameters
	489	+ *
	490	+ * @handle: handle of the VM
	491	+ * @mnonce: a random nonce that will be included in the report.
	492	+ * @address: physical address where the report will be copied.
	493	+ * @len: length of the physical buffer.
	494	+ */
	495	+struct sev_data_attestation_report {
	496	+ u32 handle; /* In */
	497	+ u32 reserved;
	498	+ u64 address; /* In */
	499	+ u8 mnonce[16]; /* In */
	500	+ u32 len; /* In/Out */
	501	+} __packed;
	502	+
486	503	#ifdef CONFIG_CRYPTO_DEV_SP_PSP
487	504
488	505	/**

include/linux/rwlock.h+7−0

@@ -128,4 +128,11 @@ do { \ · 127 unmodified lines
128	128	1 : ({ local_irq_restore(flags); 0; }); \
129	129	})
130	130
	131	+#ifdef arch_rwlock_is_contended
	132	+#define rwlock_is_contended(lock) \
	133	+ arch_rwlock_is_contended(&(lock)->raw_lock)
	134	+#else
	135	+#define rwlock_is_contended(lock) ((void)(lock), 0)
	136	+#endif /* arch_rwlock_is_contended */
	137	+
131	138	#endif /* __LINUX_RWLOCK_H */

include/linux/sched.h+29−0

@@ -1904,12 +1904,24 @@ static inline int _cond_resched(void) { return 0; } · 1903 unmodified lines
1904	1904	})
1905	1905
1906	1906	extern int __cond_resched_lock(spinlock_t *lock);
	1907	+extern int __cond_resched_rwlock_read(rwlock_t *lock);
	1908	+extern int __cond_resched_rwlock_write(rwlock_t *lock);
1907	1909
1908	1910	#define cond_resched_lock(lock) ({ \
1909	1911	___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\
1910	1912	__cond_resched_lock(lock); \
1911	1913	})
1912	1914
	1915	+#define cond_resched_rwlock_read(lock) ({ \
	1916	+ __might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \
	1917	+ __cond_resched_rwlock_read(lock); \
	1918	+})
	1919	+
	1920	+#define cond_resched_rwlock_write(lock) ({ \
	1921	+ __might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \
	1922	+ __cond_resched_rwlock_write(lock); \
	1923	+})
	1924	+
1913	1925	static inline void cond_resched_rcu(void)
1914	1926	{
1915	1927	#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) \|\| !defined(CONFIG_PREEMPT_RCU)
@@ -1933,6 +1945,23 @@ static inline int spin_needbreak(spinlock_t *lock) · 17 unmodified lines
1933	1945	#endif
1934	1946	}
1935	1947
	1948	+/*
	1949	+ * Check if a rwlock is contended.
	1950	+ * Returns non-zero if there is another task waiting on the rwlock.
	1951	+ * Returns zero if the lock is not contended or the system / underlying
	1952	+ * rwlock implementation does not support contention detection.
	1953	+ * Technically does not depend on CONFIG_PREEMPTION, but a general need
	1954	+ * for low latency.
	1955	+ */
	1956	+static inline int rwlock_needbreak(rwlock_t *lock)
	1957	+{
	1958	+#ifdef CONFIG_PREEMPTION
	1959	+ return rwlock_is_contended(lock);
	1960	+#else
	1961	+ return 0;
	1962	+#endif
	1963	+}
	1964	+
1936	1965	static __always_inline bool need_resched(void)
1937	1966	{
1938	1967	return unlikely(tif_need_resched());

include/uapi/linux/kvm.h+74−0

@@ -216,6 +216,20 @@ struct kvm_hyperv_exit { · 215 unmodified lines
216	216	} u;
217	217	};
218	218
	219	+struct kvm_xen_exit {
	220	+#define KVM_EXIT_XEN_HCALL 1
	221	+ __u32 type;
	222	+ union {
	223	+ struct {
	224	+ __u32 longmode;
	225	+ __u32 cpl;
	226	+ __u64 input;
	227	+ __u64 result;
	228	+ __u64 params[6];
	229	+ } hcall;
	230	+ } u;
	231	+};
	232	+
219	233	#define KVM_S390_GET_SKEYS_NONE 1
220	234	#define KVM_S390_SKEYS_MAX 1048576
221	235
@@ -252,6 +266,8 @@ struct kvm_hyperv_exit { · 30 unmodified lines
252	266	#define KVM_EXIT_X86_WRMSR 30
253	267	#define KVM_EXIT_DIRTY_RING_FULL 31
254	268	#define KVM_EXIT_AP_RESET_HOLD 32
	269	+#define KVM_EXIT_X86_BUS_LOCK 33
	270	+#define KVM_EXIT_XEN 34
255	271
256	272	/* For KVM_EXIT_INTERNAL_ERROR */
257	273	/* Emulate instruction failed. */
@@ -428,6 +444,8 @@ struct kvm_run { · 170 unmodified lines
428	444	__u32 index; /* kernel -> user */
429	445	__u64 data; /* kernel <-> user */
430	446	} msr;
	447	+ /* KVM_EXIT_XEN */
	448	+ struct kvm_xen_exit xen;
431	449	/* Fix the size of the union. */
432	450	char padding[256];
433	451	};
@@ -1058,6 +1076,8 @@ struct kvm_ppc_resize_hpt { · 624 unmodified lines
1058	1076	#define KVM_CAP_ENFORCE_PV_FEATURE_CPUID 190
1059	1077	#define KVM_CAP_SYS_HYPERV_CPUID 191
1060	1078	#define KVM_CAP_DIRTY_LOG_RING 192
	1079	+#define KVM_CAP_X86_BUS_LOCK_EXIT 193
	1080	+#define KVM_CAP_PPC_DAWR1 194
1061	1081
1062	1082	#ifdef KVM_CAP_IRQ_ROUTING
1063	1083
@@ -1131,6 +1151,10 @@ struct kvm_x86_mce { · 67 unmodified lines
1131	1151	#endif
1132	1152
1133	1153	#ifdef KVM_CAP_XEN_HVM
	1154	+#define KVM_XEN_HVM_CONFIG_HYPERCALL_MSR (1 << 0)
	1155	+#define KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL (1 << 1)
	1156	+#define KVM_XEN_HVM_CONFIG_SHARED_INFO (1 << 2)
	1157	+
1134	1158	struct kvm_xen_hvm_config {
1135	1159	__u32 flags;
1136	1160	__u32 msr;
@@ -1565,6 +1589,45 @@ struct kvm_pv_cmd { · 428 unmodified lines
1565	1589	/* Available with KVM_CAP_DIRTY_LOG_RING */
1566	1590	#define KVM_RESET_DIRTY_RINGS _IO(KVMIO, 0xc7)
1567	1591
	1592	+/* Per-VM Xen attributes */
	1593	+#define KVM_XEN_HVM_GET_ATTR _IOWR(KVMIO, 0xc8, struct kvm_xen_hvm_attr)
	1594	+#define KVM_XEN_HVM_SET_ATTR _IOW(KVMIO, 0xc9, struct kvm_xen_hvm_attr)
	1595	+
	1596	+struct kvm_xen_hvm_attr {
	1597	+ __u16 type;
	1598	+ __u16 pad[3];
	1599	+ union {
	1600	+ __u8 long_mode;
	1601	+ __u8 vector;
	1602	+ struct {
	1603	+ __u64 gfn;
	1604	+ } shared_info;
	1605	+ __u64 pad[8];
	1606	+ } u;
	1607	+};
	1608	+
	1609	+/* Available with KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_SHARED_INFO */
	1610	+#define KVM_XEN_ATTR_TYPE_LONG_MODE 0x0
	1611	+#define KVM_XEN_ATTR_TYPE_SHARED_INFO 0x1
	1612	+#define KVM_XEN_ATTR_TYPE_UPCALL_VECTOR 0x2
	1613	+
	1614	+/* Per-vCPU Xen attributes */
	1615	+#define KVM_XEN_VCPU_GET_ATTR _IOWR(KVMIO, 0xca, struct kvm_xen_vcpu_attr)
	1616	+#define KVM_XEN_VCPU_SET_ATTR _IOW(KVMIO, 0xcb, struct kvm_xen_vcpu_attr)
	1617	+
	1618	+struct kvm_xen_vcpu_attr {
	1619	+ __u16 type;
	1620	+ __u16 pad[3];
	1621	+ union {
	1622	+ __u64 gpa;
	1623	+ __u64 pad[8];
	1624	+ } u;
	1625	+};
	1626	+
	1627	+/* Available with KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_SHARED_INFO */
	1628	+#define KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO 0x0
	1629	+#define KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO 0x1
	1630	+
1568	1631	/* Secure Encrypted Virtualization command */
1569	1632	enum sev_cmd_id {
1570	1633	/* Guest initialization commands */
@@ -1593,6 +1656,8 @@ enum sev_cmd_id { · 22 unmodified lines
1593	1656	KVM_SEV_DBG_ENCRYPT,
1594	1657	/* Guest certificates commands */
1595	1658	KVM_SEV_CERT_EXPORT,
	1659	+ /* Attestation report */
	1660	+ KVM_SEV_GET_ATTESTATION_REPORT,
1596	1661
1597	1662	KVM_SEV_NR_MAX,
1598	1663	};
@@ -1645,6 +1710,12 @@ struct kvm_sev_dbg { · 46 unmodified lines
1645	1710	__u32 len;
1646	1711	};
1647	1712
	1713	+struct kvm_sev_attestation_report {
	1714	+ __u8 mnonce[16];
	1715	+ __u64 uaddr;
	1716	+ __u32 len;
	1717	+};
	1718	+
1648	1719	#define KVM_DEV_ASSIGN_ENABLE_IOMMU (1 << 0)
1649	1720	#define KVM_DEV_ASSIGN_PCI_2_3 (1 << 1)
1650	1721	#define KVM_DEV_ASSIGN_MASK_INTX (1 << 2)
@@ -1766,4 +1837,7 @@ struct kvm_dirty_gfn { · 115 unmodified lines
1766	1837	__u64 offset;
1767	1838	};
1768	1839
	1840	+#define KVM_BUS_LOCK_DETECTION_OFF (1 << 0)
	1841	+#define KVM_BUS_LOCK_DETECTION_EXIT (1 << 1)
	1842	+
1769	1843	#endif /* __LINUX_KVM_H */

include/xen/interface/xen.h+3−1

@@ -598,7 +598,9 @@ struct shared_info { · 597 unmodified lines
598	598	* their gettimeofday() syscall on this wallclock-base value.
599	599	*/
600	600	struct pvclock_wall_clock wc;
601		−
	601	+#ifndef CONFIG_X86_32
	602	+ uint32_t wc_sec_hi;
	603	+#endif
602	604	struct arch_shared_info arch;
603	605
604	606	};

kernel/locking/qrwlock.c+0−1

@@ -12,7 +12,6 @@ · 11 unmodified lines
12	12	#include <linux/percpu.h>
13	13	#include <linux/hardirq.h>
14	14	#include <linux/spinlock.h>
15		−#include <asm/qrwlock.h>
16	15
17	16	/**
18	17	* queued_read_lock_slowpath - acquire read lock of a queue rwlock

kernel/sched/core.c+40−0

@@ -7028,6 +7028,46 @@ int __cond_resched_lock(spinlock_t *lock) · 7027 unmodified lines
7028	7028	}
7029	7029	EXPORT_SYMBOL(__cond_resched_lock);
7030	7030
	7031	+int __cond_resched_rwlock_read(rwlock_t *lock)
	7032	+{
	7033	+ int resched = should_resched(PREEMPT_LOCK_OFFSET);
	7034	+ int ret = 0;
	7035	+
	7036	+ lockdep_assert_held_read(lock);
	7037	+
	7038	+ if (rwlock_needbreak(lock) \|\| resched) {
	7039	+ read_unlock(lock);
	7040	+ if (resched)
	7041	+ preempt_schedule_common();
	7042	+ else
	7043	+ cpu_relax();
	7044	+ ret = 1;
	7045	+ read_lock(lock);
	7046	+ }
	7047	+ return ret;
	7048	+}
	7049	+EXPORT_SYMBOL(__cond_resched_rwlock_read);
	7050	+
	7051	+int __cond_resched_rwlock_write(rwlock_t *lock)
	7052	+{
	7053	+ int resched = should_resched(PREEMPT_LOCK_OFFSET);
	7054	+ int ret = 0;
	7055	+
	7056	+ lockdep_assert_held_write(lock);
	7057	+
	7058	+ if (rwlock_needbreak(lock) \|\| resched) {
	7059	+ write_unlock(lock);
	7060	+ if (resched)
	7061	+ preempt_schedule_common();
	7062	+ else
	7063	+ cpu_relax();
	7064	+ ret = 1;
	7065	+ write_lock(lock);
	7066	+ }
	7067	+ return ret;
	7068	+}
	7069	+EXPORT_SYMBOL(__cond_resched_rwlock_write);
	7070	+
7031	7071	/**
7032	7072	* yield - yield the current processor to other threads.
7033	7073	*

mm/memory.c+36−5

@@ -4642,9 +4642,9 @@ int __pmd_alloc(struct mm_struct mm, pud_t pud, unsigned long address) · 4641 unmodified lines
4642	4642	}
4643	4643	#endif /* __PAGETABLE_PMD_FOLDED */
4644	4644
4645		−int follow_pte(struct mm_struct *mm, unsigned long address,
4646		− struct mmu_notifier_range range, pte_t ptepp, pmd_t *pmdpp,
4647		− spinlock_t **ptlp)
	4645	+int follow_invalidate_pte(struct mm_struct *mm, unsigned long address,
	4646	+ struct mmu_notifier_range range, pte_t *ptepp,
	4647	+ pmd_t pmdpp, spinlock_t ptlp)
4648	4648	{
4649	4649	pgd_t *pgd;
4650	4650	p4d_t *p4d;
@@ -4709,6 +4709,34 @@ out: · 58 unmodified lines
4709	4709	return -EINVAL;
4710	4710	}
4711	4711
	4712	+/**
	4713	+ * follow_pte - look up PTE at a user virtual address
	4714	+ * @mm: the mm_struct of the target address space
	4715	+ * @address: user virtual address
	4716	+ * @ptepp: location to store found PTE
	4717	+ * @ptlp: location to store the lock for the PTE
	4718	+ *
	4719	+ * On a successful return, the pointer to the PTE is stored in @ptepp;
	4720	+ * the corresponding lock is taken and its location is stored in @ptlp.
	4721	+ * The contents of the PTE are only stable until @ptlp is released;
	4722	+ * any further use, if any, must be protected against invalidation
	4723	+ * with MMU notifiers.
	4724	+ *
	4725	+ * Only IO mappings and raw PFN mappings are allowed. The mmap semaphore
	4726	+ * should be taken for read.
	4727	+ *
	4728	+ * KVM uses this function. While it is arguably less bad than ``follow_pfn``,
	4729	+ * it is not a good general-purpose API.
	4730	+ *
	4731	+ * Return: zero on success, -ve otherwise.
	4732	+ */
	4733	+int follow_pte(struct mm_struct *mm, unsigned long address,
	4734	+ pte_t ptepp, spinlock_t ptlp)
	4735	+{
	4736	+ return follow_invalidate_pte(mm, address, NULL, ptepp, NULL, ptlp);
	4737	+}
	4738	+EXPORT_SYMBOL_GPL(follow_pte);
	4739	+
4712	4740	/**
4713	4741	* follow_pfn - look up PFN at a user virtual address
4714	4742	* @vma: memory mapping
@@ -4717,6 +4745,9 @@ out: · 2 unmodified lines
4717	4745	*
4718	4746	* Only IO mappings and raw PFN mappings are allowed.
4719	4747	*
	4748	+ * This function does not allow the caller to read the permissions
	4749	+ * of the PTE. Do not use it.
	4750	+ *
4720	4751	* Return: zero and the pfn at @pfn on success, -ve otherwise.
4721	4752	*/
4722	4753	int follow_pfn(struct vm_area_struct *vma, unsigned long address,
@@ -4729,7 +4760,7 @@ int follow_pfn(struct vm_area_struct *vma, unsigned long address, · 6 unmodified lines
4729	4760	if (!(vma->vm_flags & (VM_IO \| VM_PFNMAP)))
4730	4761	return ret;
4731	4762
4732		− ret = follow_pte(vma->vm_mm, address, NULL, &ptep, NULL, &ptl);
	4763	+ ret = follow_pte(vma->vm_mm, address, &ptep, &ptl);
4733	4764	if (ret)
4734	4765	return ret;
4735	4766	pfn = pte_pfn(ptep);
@@ -4750,7 +4781,7 @@ int follow_phys(struct vm_area_struct *vma, · 14 unmodified lines
4750	4781	if (!(vma->vm_flags & (VM_IO \| VM_PFNMAP)))
4751	4782	goto out;
4752	4783
4753		− if (follow_pte(vma->vm_mm, address, NULL, &ptep, NULL, &ptl))
	4784	+ if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
4754	4785	goto out;
4755	4786	pte = *ptep;
4756	4787

tools/arch/powerpc/include/uapi/asm/kvm.h+2−0

@@ -644,6 +644,8 @@ struct kvm_ppc_cpu_char { · 643 unmodified lines
644	644	#define KVM_REG_PPC_MMCR3 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc1)
645	645	#define KVM_REG_PPC_SIER2 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc2)
646	646	#define KVM_REG_PPC_SIER3 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc3)
	647	+#define KVM_REG_PPC_DAWR1 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc4)
	648	+#define KVM_REG_PPC_DAWRX1 (KVM_REG_PPC \| KVM_REG_SIZE_U64 \| 0xc5)
647	649
648	650	/* Transactional Memory checkpointed state:
649	651	* This is all GPRs, all VSX regs and a subset of SPRs

tools/include/uapi/linux/kvm.h+1−0

@@ -1058,6 +1058,7 @@ struct kvm_ppc_resize_hpt { · 1057 unmodified lines
1058	1058	#define KVM_CAP_ENFORCE_PV_FEATURE_CPUID 190
1059	1059	#define KVM_CAP_SYS_HYPERV_CPUID 191
1060	1060	#define KVM_CAP_DIRTY_LOG_RING 192
	1061	+#define KVM_CAP_PPC_DAWR1 194
1061	1062
1062	1063	#ifdef KVM_CAP_IRQ_ROUTING
1063	1064

tools/testing/selftests/kvm/.gitignore+6−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	/x86_64/cr4_cpuid_sync_test
8	8	/x86_64/debug_regs
9	9	/x86_64/evmcs_test
	10	+/x86_64/get_cpuid_test
10	11	/x86_64/kvm_pv_test
11	12	/x86_64/hyperv_cpuid
12	13	/x86_64/mmio_warning_test
@@ -24,10 +25,15 @@ · 11 unmodified lines
24	25	/x86_64/vmx_preemption_timer_test
25	26	/x86_64/vmx_set_nested_state_test
26	27	/x86_64/vmx_tsc_adjust_test
	28	+/x86_64/xapic_ipi_test
	29	+/x86_64/xen_shinfo_test
	30	+/x86_64/xen_vmcall_test
27	31	/x86_64/xss_msr_test
	32	+/x86_64/vmx_pmu_msrs_test
28	33	/demand_paging_test
29	34	/dirty_log_test
30	35	/dirty_log_perf_test
31	36	/kvm_create_max_vcpus
	37	+/memslot_modification_stress_test
32	38	/set_memory_region_test
33	39	/steal_time

tools/testing/selftests/kvm/Makefile+6−0

@@ -40,6 +40,7 @@ LIBKVM_s390x = lib/s390x/processor.c lib/s390x/ucall.c lib/s390x/diag318_test_ha · 39 unmodified lines
40	40
41	41	TEST_GEN_PROGS_x86_64 = x86_64/cr4_cpuid_sync_test
42	42	TEST_GEN_PROGS_x86_64 += x86_64/evmcs_test
	43	+TEST_GEN_PROGS_x86_64 += x86_64/get_cpuid_test
43	44	TEST_GEN_PROGS_x86_64 += x86_64/hyperv_cpuid
44	45	TEST_GEN_PROGS_x86_64 += x86_64/kvm_pv_test
45	46	TEST_GEN_PROGS_x86_64 += x86_64/mmio_warning_test
@@ -56,13 +57,18 @@ TEST_GEN_PROGS_x86_64 += x86_64/vmx_close_while_nested_test · 10 unmodified lines
56	57	TEST_GEN_PROGS_x86_64 += x86_64/vmx_dirty_log_test
57	58	TEST_GEN_PROGS_x86_64 += x86_64/vmx_set_nested_state_test
58	59	TEST_GEN_PROGS_x86_64 += x86_64/vmx_tsc_adjust_test
	60	+TEST_GEN_PROGS_x86_64 += x86_64/xapic_ipi_test
59	61	TEST_GEN_PROGS_x86_64 += x86_64/xss_msr_test
60	62	TEST_GEN_PROGS_x86_64 += x86_64/debug_regs
61	63	TEST_GEN_PROGS_x86_64 += x86_64/tsc_msrs_test
	64	+TEST_GEN_PROGS_x86_64 += x86_64/vmx_pmu_msrs_test
	65	+TEST_GEN_PROGS_x86_64 += x86_64/xen_shinfo_test
	66	+TEST_GEN_PROGS_x86_64 += x86_64/xen_vmcall_test
62	67	TEST_GEN_PROGS_x86_64 += demand_paging_test
63	68	TEST_GEN_PROGS_x86_64 += dirty_log_test
64	69	TEST_GEN_PROGS_x86_64 += dirty_log_perf_test
65	70	TEST_GEN_PROGS_x86_64 += kvm_create_max_vcpus
	71	+TEST_GEN_PROGS_x86_64 += memslot_modification_stress_test
66	72	TEST_GEN_PROGS_x86_64 += set_memory_region_test
67	73	TEST_GEN_PROGS_x86_64 += steal_time
68	74

tools/testing/selftests/kvm/demand_paging_test.c+31−12

@@ -64,7 +64,7 @@ static void vcpu_worker(void data) · 63 unmodified lines
64	64	exit_reason_str(run->exit_reason));
65	65	}
66	66
67		− ts_diff = timespec_diff_now(start);
	67	+ ts_diff = timespec_elapsed(start);
68	68	PER_VCPU_DEBUG("vCPU %d execution time: %ld.%.9lds\n", vcpu_id,
69	69	ts_diff.tv_sec, ts_diff.tv_nsec);
70	70
@@ -95,7 +95,7 @@ static int handle_uffd_page_request(int uffd, uint64_t addr) · 24 unmodified lines
95	95	return r;
96	96	}
97	97
98		− ts_diff = timespec_diff_now(start);
	98	+ ts_diff = timespec_elapsed(start);
99	99
100	100	PER_PAGE_DEBUG("UFFDIO_COPY %d \t%ld ns\n", tid,
101	101	timespec_to_ns(ts_diff));
@@ -190,7 +190,7 @@ static void uffd_handler_thread_fn(void arg) · 88 unmodified lines
190	190	pages++;
191	191	}
192	192
193		− ts_diff = timespec_diff_now(start);
	193	+ ts_diff = timespec_elapsed(start);
194	194	PER_VCPU_DEBUG("userfaulted %ld pages over %ld.%.9lds. (%f/sec)\n",
195	195	pages, ts_diff.tv_sec, ts_diff.tv_nsec,
196	196	pages / ((double)ts_diff.tv_sec + (double)ts_diff.tv_nsec / 100000000.0));
@@ -250,6 +250,7 @@ static int setup_demand_paging(struct kvm_vm *vm, · 53 unmodified lines
250	250	struct test_params {
251	251	bool use_uffd;
252	252	useconds_t uffd_delay;
	253	+ bool partition_vcpu_memory_access;
253	254	};
254	255
255	256	static void run_test(enum vm_guest_mode mode, void *arg)
@@ -265,7 +266,8 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 9 unmodified lines
265	266	int vcpu_id;
266	267	int r;
267	268
268		− vm = perf_test_create_vm(mode, nr_vcpus, guest_percpu_mem_size);
	269	+ vm = perf_test_create_vm(mode, nr_vcpus, guest_percpu_mem_size,
	270	+ VM_MEM_SRC_ANONYMOUS);
269	271
270	272	perf_test_args.wr_fract = 1;
271	273
@@ -277,7 +279,8 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 5 unmodified lines
277	279	vcpu_threads = malloc(nr_vcpus * sizeof(*vcpu_threads));
278	280	TEST_ASSERT(vcpu_threads, "Memory allocation failed");
279	281
280		− perf_test_setup_vcpus(vm, nr_vcpus, guest_percpu_mem_size);
	282	+ perf_test_setup_vcpus(vm, nr_vcpus, guest_percpu_mem_size,
	283	+ p->partition_vcpu_memory_access);
281	284
282	285	if (p->use_uffd) {
283	286	uffd_handler_threads =
@@ -293,10 +296,19 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 9 unmodified lines
293	296	for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++) {
294	297	vm_paddr_t vcpu_gpa;
295	298	void *vcpu_hva;
	299	+ uint64_t vcpu_mem_size;
296	300
297		− vcpu_gpa = guest_test_phys_mem + (vcpu_id * guest_percpu_mem_size);
	301	+
	302	+ if (p->partition_vcpu_memory_access) {
	303	+ vcpu_gpa = guest_test_phys_mem +
	304	+ (vcpu_id * guest_percpu_mem_size);
	305	+ vcpu_mem_size = guest_percpu_mem_size;
	306	+ } else {
	307	+ vcpu_gpa = guest_test_phys_mem;
	308	+ vcpu_mem_size = guest_percpu_mem_size * nr_vcpus;
	309	+ }
298	310	PER_VCPU_DEBUG("Added VCPU %d with test mem gpa [%lx, %lx)\n",
299		− vcpu_id, vcpu_gpa, vcpu_gpa + guest_percpu_mem_size);
	311	+ vcpu_id, vcpu_gpa, vcpu_gpa + vcpu_mem_size);
300	312
301	313	/* Cache the HVA pointer of the region */
302	314	vcpu_hva = addr_gpa2hva(vm, vcpu_gpa);
@@ -313,7 +325,7 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 10 unmodified lines
313	325	&uffd_handler_threads[vcpu_id],
314	326	pipefds[vcpu_id * 2],
315	327	p->uffd_delay, &uffd_args[vcpu_id],
316		− vcpu_hva, guest_percpu_mem_size);
	328	+ vcpu_hva, vcpu_mem_size);
317	329	if (r < 0)
318	330	exit(-r);
319	331	}
@@ -339,7 +351,7 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 19 unmodified lines
339	351	PER_VCPU_DEBUG("Joined thread for vCPU %d\n", vcpu_id);
340	352	}
341	353
342		− ts_diff = timespec_diff_now(start);
	354	+ ts_diff = timespec_elapsed(start);
343	355
344	356	pr_info("All vCPU threads joined\n");
345	357
@@ -376,7 +388,7 @@ static void help(char *name) · 30 unmodified lines
376	388	{
377	389	puts("");
378	390	printf("usage: %s [-h] [-m mode] [-u] [-d uffd_delay_usec]\n"
379		− " [-b memory] [-v vcpus]\n", name);
	391	+ " [-b memory] [-v vcpus] [-o]\n", name);
380	392	guest_modes_help();
381	393	printf(" -u: use User Fault FD to handle vCPU page\n"
382	394	" faults.\n");
@@ -387,6 +399,8 @@ static void help(char *name) · 4 unmodified lines
387	399	" demand paged by each vCPU. e.g. 10M or 3G.\n"
388	400	" Default: 1G\n");
389	401	printf(" -v: specify the number of vCPUs to run.\n");
	402	+ printf(" -o: Overlap guest memory accesses instead of partitioning\n"
	403	+ " them into a separate region of memory for each vCPU.\n");
390	404	puts("");
391	405	exit(0);
392	406	}
@@ -394,12 +408,14 @@ static void help(char *name) · 1 unmodified line
394	408	int main(int argc, char *argv[])
395	409	{
396	410	int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS);
397		− struct test_params p = {};
	411	+ struct test_params p = {
	412	+ .partition_vcpu_memory_access = true,
	413	+ };
398	414	int opt;
399	415
400	416	guest_modes_append_default();
401	417
402		− while ((opt = getopt(argc, argv, "hm:ud:b:v:")) != -1) {
	418	+ while ((opt = getopt(argc, argv, "hm:ud:b:v:o")) != -1) {
403	419	switch (opt) {
404	420	case 'm':
405	421	guest_modes_cmdline(optarg);
@@ -419,6 +435,9 @@ int main(int argc, char *argv[]) · 13 unmodified lines
419	435	TEST_ASSERT(nr_vcpus > 0 && nr_vcpus <= max_vcpus,
420	436	"Invalid number of vcpus, must be between 1 and %d", max_vcpus);
421	437	break;
	438	+ case 'o':
	439	+ p.partition_vcpu_memory_access = false;
	440	+ break;
422	441	case 'h':
423	442	default:
424	443	help(argv[0]);

tools/testing/selftests/kvm/dirty_log_perf_test.c+56−36

@@ -28,8 +28,8 @@ static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE; · 27 unmodified lines
28	28	/* Host variables */
29	29	static u64 dirty_log_manual_caps;
30	30	static bool host_quit;
31		−static uint64_t iteration;
32		−static uint64_t vcpu_last_completed_iteration[KVM_MAX_VCPUS];
	31	+static int iteration;
	32	+static int vcpu_last_completed_iteration[KVM_MAX_VCPUS];
33	33
34	34	static void vcpu_worker(void data)
35	35	{
@@ -48,11 +48,11 @@ static void vcpu_worker(void data) · 12 unmodified lines
48	48	run = vcpu_state(vm, vcpu_id);
49	49
50	50	while (!READ_ONCE(host_quit)) {
51		− uint64_t current_iteration = READ_ONCE(iteration);
	51	+ int current_iteration = READ_ONCE(iteration);
52	52
53	53	clock_gettime(CLOCK_MONOTONIC, &start);
54	54	ret = _vcpu_run(vm, vcpu_id);
55		− ts_diff = timespec_diff_now(start);
	55	+ ts_diff = timespec_elapsed(start);
56	56
57	57	TEST_ASSERT(ret == 0, "vcpu_run failed: %d\n", ret);
58	58	TEST_ASSERT(get_ucall(vm, vcpu_id, NULL) == UCALL_SYNC,
@@ -61,17 +61,17 @@ static void vcpu_worker(void data) · 2 unmodified lines
61	61
62	62	pr_debug("Got sync event from vCPU %d\n", vcpu_id);
63	63	vcpu_last_completed_iteration[vcpu_id] = current_iteration;
64		− pr_debug("vCPU %d updated last completed iteration to %lu\n",
	64	+ pr_debug("vCPU %d updated last completed iteration to %d\n",
65	65	vcpu_id, vcpu_last_completed_iteration[vcpu_id]);
66	66
67	67	if (current_iteration) {
68	68	pages_count += vcpu_args->pages;
69	69	total = timespec_add(total, ts_diff);
70		− pr_debug("vCPU %d iteration %lu dirty memory time: %ld.%.9lds\n",
	70	+ pr_debug("vCPU %d iteration %d dirty memory time: %ld.%.9lds\n",
71	71	vcpu_id, current_iteration, ts_diff.tv_sec,
72	72	ts_diff.tv_nsec);
73	73	} else {
74		− pr_debug("vCPU %d iteration %lu populate memory time: %ld.%.9lds\n",
	74	+ pr_debug("vCPU %d iteration %d populate memory time: %ld.%.9lds\n",
75	75	vcpu_id, current_iteration, ts_diff.tv_sec,
76	76	ts_diff.tv_nsec);
77	77	}
@@ -81,7 +81,7 @@ static void vcpu_worker(void data) · 3 unmodified lines
81	81	}
82	82
83	83	avg = timespec_div(total, vcpu_last_completed_iteration[vcpu_id]);
84		− pr_debug("\nvCPU %d dirtied 0x%lx pages over %lu iterations in %ld.%.9lds. (Avg %ld.%.9lds/iteration)\n",
	84	+ pr_debug("\nvCPU %d dirtied 0x%lx pages over %d iterations in %ld.%.9lds. (Avg %ld.%.9lds/iteration)\n",
85	85	vcpu_id, pages_count, vcpu_last_completed_iteration[vcpu_id],
86	86	total.tv_sec, total.tv_nsec, avg.tv_sec, avg.tv_nsec);
87	87
@@ -92,6 +92,8 @@ struct test_params { · 4 unmodified lines
92	92	unsigned long iterations;
93	93	uint64_t phys_offset;
94	94	int wr_fract;
	95	+ bool partition_vcpu_memory_access;
	96	+ enum vm_mem_backing_src_type backing_src;
95	97	};
96	98
97	99	static void run_test(enum vm_guest_mode mode, void *arg)
@@ -111,7 +113,8 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 13 unmodified lines
111	113	struct kvm_enable_cap cap = {};
112	114	struct timespec clear_dirty_log_total = (struct timespec){0};
113	115
114		− vm = perf_test_create_vm(mode, nr_vcpus, guest_percpu_mem_size);
	116	+ vm = perf_test_create_vm(mode, nr_vcpus, guest_percpu_mem_size,
	117	+ p->backing_src);
115	118
116	119	perf_test_args.wr_fract = p->wr_fract;
117	120
@@ -129,7 +132,8 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 11 unmodified lines
129	132	vcpu_threads = malloc(nr_vcpus * sizeof(*vcpu_threads));
130	133	TEST_ASSERT(vcpu_threads, "Memory allocation failed");
131	134
132		− perf_test_setup_vcpus(vm, nr_vcpus, guest_percpu_mem_size);
	135	+ perf_test_setup_vcpus(vm, nr_vcpus, guest_percpu_mem_size,
	136	+ p->partition_vcpu_memory_access);
133	137
134	138	sync_global_to_guest(vm, perf_test_args);
135	139
@@ -139,17 +143,21 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 3 unmodified lines
139	143
140	144	clock_gettime(CLOCK_MONOTONIC, &start);
141	145	for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++) {
	146	+ vcpu_last_completed_iteration[vcpu_id] = -1;
	147	+
142	148	pthread_create(&vcpu_threads[vcpu_id], NULL, vcpu_worker,
143	149	&perf_test_args.vcpu_args[vcpu_id]);
144	150	}
145	151
146		− /* Allow the vCPU to populate memory */
147		− pr_debug("Starting iteration %lu - Populating\n", iteration);
148		− while (READ_ONCE(vcpu_last_completed_iteration[vcpu_id]) != iteration)
149		− pr_debug("Waiting for vcpu_last_completed_iteration == %lu\n",
150		− iteration);
	152	+ /* Allow the vCPUs to populate memory */
	153	+ pr_debug("Starting iteration %d - Populating\n", iteration);
	154	+ for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++) {
	155	+ while (READ_ONCE(vcpu_last_completed_iteration[vcpu_id]) !=
	156	+ iteration)
	157	+ ;
	158	+ }
151	159
152		− ts_diff = timespec_diff_now(start);
	160	+ ts_diff = timespec_elapsed(start);
153	161	pr_info("Populate memory time: %ld.%.9lds\n",
154	162	ts_diff.tv_sec, ts_diff.tv_nsec);
155	163
@@ -157,7 +165,7 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 1 unmodified line
157	165	clock_gettime(CLOCK_MONOTONIC, &start);
158	166	vm_mem_region_set_flags(vm, PERF_TEST_MEM_SLOT_INDEX,
159	167	KVM_MEM_LOG_DIRTY_PAGES);
160		− ts_diff = timespec_diff_now(start);
	168	+ ts_diff = timespec_elapsed(start);
161	169	pr_info("Enabling dirty logging time: %ld.%.9lds\n\n",
162	170	ts_diff.tv_sec, ts_diff.tv_nsec);
163	171
@@ -169,25 +177,25 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 5 unmodified lines
169	177	clock_gettime(CLOCK_MONOTONIC, &start);
170	178	iteration++;
171	179
172		− pr_debug("Starting iteration %lu\n", iteration);
	180	+ pr_debug("Starting iteration %d\n", iteration);
173	181	for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++) {
174		− while (READ_ONCE(vcpu_last_completed_iteration[vcpu_id]) != iteration)
175		− pr_debug("Waiting for vCPU %d vcpu_last_completed_iteration == %lu\n",
176		− vcpu_id, iteration);
	182	+ while (READ_ONCE(vcpu_last_completed_iteration[vcpu_id])
	183	+ != iteration)
	184	+ ;
177	185	}
178	186
179		− ts_diff = timespec_diff_now(start);
	187	+ ts_diff = timespec_elapsed(start);
180	188	vcpu_dirty_total = timespec_add(vcpu_dirty_total, ts_diff);
181		− pr_info("Iteration %lu dirty memory time: %ld.%.9lds\n",
	189	+ pr_info("Iteration %d dirty memory time: %ld.%.9lds\n",
182	190	iteration, ts_diff.tv_sec, ts_diff.tv_nsec);
183	191
184	192	clock_gettime(CLOCK_MONOTONIC, &start);
185	193	kvm_vm_get_dirty_log(vm, PERF_TEST_MEM_SLOT_INDEX, bmap);
186	194
187		− ts_diff = timespec_diff_now(start);
	195	+ ts_diff = timespec_elapsed(start);
188	196	get_dirty_log_total = timespec_add(get_dirty_log_total,
189	197	ts_diff);
190		− pr_info("Iteration %lu get dirty log time: %ld.%.9lds\n",
	198	+ pr_info("Iteration %d get dirty log time: %ld.%.9lds\n",
191	199	iteration, ts_diff.tv_sec, ts_diff.tv_nsec);
192	200
193	201	if (dirty_log_manual_caps) {
@@ -195,26 +203,26 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 1 unmodified line
195	203	kvm_vm_clear_dirty_log(vm, PERF_TEST_MEM_SLOT_INDEX, bmap, 0,
196	204	host_num_pages);
197	205
198		− ts_diff = timespec_diff_now(start);
	206	+ ts_diff = timespec_elapsed(start);
199	207	clear_dirty_log_total = timespec_add(clear_dirty_log_total,
200	208	ts_diff);
201		− pr_info("Iteration %lu clear dirty log time: %ld.%.9lds\n",
	209	+ pr_info("Iteration %d clear dirty log time: %ld.%.9lds\n",
202	210	iteration, ts_diff.tv_sec, ts_diff.tv_nsec);
203	211	}
204	212	}
205	213
206		− /* Tell the vcpu thread to quit */
207		− host_quit = true;
208		− for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++)
209		− pthread_join(vcpu_threads[vcpu_id], NULL);
210		−
211	214	/* Disable dirty logging */
212	215	clock_gettime(CLOCK_MONOTONIC, &start);
213	216	vm_mem_region_set_flags(vm, PERF_TEST_MEM_SLOT_INDEX, 0);
214		− ts_diff = timespec_diff_now(start);
	217	+ ts_diff = timespec_elapsed(start);
215	218	pr_info("Disabling dirty logging time: %ld.%.9lds\n",
216	219	ts_diff.tv_sec, ts_diff.tv_nsec);
217	220
	221	+ /* Tell the vcpu thread to quit */
	222	+ host_quit = true;
	223	+ for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++)
	224	+ pthread_join(vcpu_threads[vcpu_id], NULL);
	225	+
218	226	avg = timespec_div(get_dirty_log_total, p->iterations);
219	227	pr_info("Get dirty log over %lu iterations took %ld.%.9lds. (Avg %ld.%.9lds/iteration)\n",
220	228	p->iterations, get_dirty_log_total.tv_sec,
@@ -236,7 +244,7 @@ static void help(char *name) · 15 unmodified lines
236	244	{
237	245	puts("");
238	246	printf("usage: %s [-h] [-i iterations] [-p offset] "
239		− "[-m mode] [-b vcpu bytes] [-v vcpus]\n", name);
	247	+ "[-m mode] [-b vcpu bytes] [-v vcpus] [-o] [-s mem type]\n", name);
240	248	puts("");
241	249	printf(" -i: specify iteration counts (default: %"PRIu64")\n",
242	250	TEST_HOST_LOOP_N);
@@ -251,6 +259,11 @@ static void help(char *name) · 8 unmodified lines
251	259	" 1/<fraction of pages to write>.\n"
252	260	" (default: 1 i.e. all pages are written to.)\n");
253	261	printf(" -v: specify the number of vCPUs to run.\n");
	262	+ printf(" -o: Overlap guest memory accesses instead of partitioning\n"
	263	+ " them into a separate region of memory for each vCPU.\n");
	264	+ printf(" -s: specify the type of memory that should be used to\n"
	265	+ " back the guest data region.\n\n");
	266	+ backing_src_help();
254	267	puts("");
255	268	exit(0);
256	269	}
@@ -261,6 +274,8 @@ int main(int argc, char *argv[]) · 4 unmodified lines
261	274	struct test_params p = {
262	275	.iterations = TEST_HOST_LOOP_N,
263	276	.wr_fract = 1,
	277	+ .partition_vcpu_memory_access = true,
	278	+ .backing_src = VM_MEM_SRC_ANONYMOUS,
264	279	};
265	280	int opt;
266	281
@@ -271,10 +286,10 @@ int main(int argc, char *argv[]) · 4 unmodified lines
271	286
272	287	guest_modes_append_default();
273	288
274		− while ((opt = getopt(argc, argv, "hi:p:m:b:f:v:")) != -1) {
	289	+ while ((opt = getopt(argc, argv, "hi:p:m:b:f:v:os:")) != -1) {
275	290	switch (opt) {
276	291	case 'i':
277		− p.iterations = strtol(optarg, NULL, 10);
	292	+ p.iterations = atoi(optarg);
278	293	break;
279	294	case 'p':
280	295	p.phys_offset = strtoull(optarg, NULL, 0);
@@ -295,6 +310,11 @@ int main(int argc, char *argv[]) · 14 unmodified lines
295	310	TEST_ASSERT(nr_vcpus > 0 && nr_vcpus <= max_vcpus,
296	311	"Invalid number of vcpus, must be between 1 and %d", max_vcpus);
297	312	break;
	313	+ case 'o':
	314	+ p.partition_vcpu_memory_access = false;
	315	+ case 's':
	316	+ p.backing_src = parse_backing_src_type(optarg);
	317	+ break;
298	318	case 'h':
299	319	default:
300	320	help(argv[0]);

tools/testing/selftests/kvm/include/kvm_util.h+0−6

@@ -79,12 +79,6 @@ struct vm_guest_mode_params { · 78 unmodified lines
79	79	};
80	80	extern const struct vm_guest_mode_params vm_guest_mode_params[];
81	81
82		−enum vm_mem_backing_src_type {
83		− VM_MEM_SRC_ANONYMOUS,
84		− VM_MEM_SRC_ANONYMOUS_THP,
85		− VM_MEM_SRC_ANONYMOUS_HUGETLB,
86		−};
87		−
88	82	int kvm_check_cap(long cap);
89	83	int vm_enable_cap(struct kvm_vm vm, struct kvm_enable_cap cap);
90	84	int vcpu_enable_cap(struct kvm_vm *vm, uint32_t vcpu_id,

tools/testing/selftests/kvm/include/numaif.h+55−0

@@ -0,0 +1,55 @@
	1	+/* SPDX-License-Identifier: GPL-2.0-only */
	2	+/*
	3	+ * tools/testing/selftests/kvm/include/numaif.h
	4	+ *
	5	+ * Copyright (C) 2020, Google LLC.
	6	+ *
	7	+ * This work is licensed under the terms of the GNU GPL, version 2.
	8	+ *
	9	+ * Header file that provides access to NUMA API functions not explicitly
	10	+ * exported to user space.
	11	+ */
	12	+
	13	+#ifndef SELFTEST_KVM_NUMAIF_H
	14	+#define SELFTEST_KVM_NUMAIF_H
	15	+
	16	+#define __NR_get_mempolicy 239
	17	+#define __NR_migrate_pages 256
	18	+
	19	+/* System calls */
	20	+long get_mempolicy(int policy, const unsigned long nmask,
	21	+ unsigned long maxnode, void *addr, int flags)
	22	+{
	23	+ return syscall(__NR_get_mempolicy, policy, nmask,
	24	+ maxnode, addr, flags);
	25	+}
	26	+
	27	+long migrate_pages(int pid, unsigned long maxnode,
	28	+ const unsigned long *frommask,
	29	+ const unsigned long *tomask)
	30	+{
	31	+ return syscall(__NR_migrate_pages, pid, maxnode, frommask, tomask);
	32	+}
	33	+
	34	+/* Policies */
	35	+#define MPOL_DEFAULT 0
	36	+#define MPOL_PREFERRED 1
	37	+#define MPOL_BIND 2
	38	+#define MPOL_INTERLEAVE 3
	39	+
	40	+#define MPOL_MAX MPOL_INTERLEAVE
	41	+
	42	+/* Flags for get_mem_policy */
	43	+#define MPOL_F_NODE (1<<0) /* return next il node or node of address */
	44	+ /* Warning: MPOL_F_NODE is unsupported and
	45	+ * subject to change. Don't use.
	46	+ */
	47	+#define MPOL_F_ADDR (1<<1) /* look up vma using address */
	48	+#define MPOL_F_MEMS_ALLOWED (1<<2) /* query nodes allowed in cpuset */
	49	+
	50	+/* Flags for mbind */
	51	+#define MPOL_MF_STRICT (1<<0) /* Verify existing pages in the mapping */
	52	+#define MPOL_MF_MOVE (1<<1) /* Move pages owned by this process to conform to mapping */
	53	+#define MPOL_MF_MOVE_ALL (1<<2) /* Move every page to conform to mapping */
	54	+
	55	+#endif /* SELFTEST_KVM_NUMAIF_H */

tools/testing/selftests/kvm/include/perf_test_util.h+5−2

@@ -44,8 +44,11 @@ extern struct perf_test_args perf_test_args; · 43 unmodified lines
44	44	extern uint64_t guest_test_phys_mem;
45	45
46	46	struct kvm_vm *perf_test_create_vm(enum vm_guest_mode mode, int vcpus,
47		− uint64_t vcpu_memory_bytes);
	47	+ uint64_t vcpu_memory_bytes,
	48	+ enum vm_mem_backing_src_type backing_src);
48	49	void perf_test_destroy_vm(struct kvm_vm *vm);
49		−void perf_test_setup_vcpus(struct kvm_vm *vm, int vcpus, uint64_t vcpu_memory_bytes);
	50	+void perf_test_setup_vcpus(struct kvm_vm *vm, int vcpus,
	51	+ uint64_t vcpu_memory_bytes,
	52	+ bool partition_vcpu_memory_access);
50	53
51	54	#endif /* SELFTEST_KVM_PERF_TEST_UTIL_H */

tools/testing/selftests/kvm/include/test_util.h+15−1

@@ -64,7 +64,21 @@ int64_t timespec_to_ns(struct timespec ts); · 63 unmodified lines
64	64	struct timespec timespec_add_ns(struct timespec ts, int64_t ns);
65	65	struct timespec timespec_add(struct timespec ts1, struct timespec ts2);
66	66	struct timespec timespec_sub(struct timespec ts1, struct timespec ts2);
67		−struct timespec timespec_diff_now(struct timespec start);
	67	+struct timespec timespec_elapsed(struct timespec start);
68	68	struct timespec timespec_div(struct timespec ts, int divisor);
69	69
	70	+enum vm_mem_backing_src_type {
	71	+ VM_MEM_SRC_ANONYMOUS,
	72	+ VM_MEM_SRC_ANONYMOUS_THP,
	73	+ VM_MEM_SRC_ANONYMOUS_HUGETLB,
	74	+};
	75	+
	76	+struct vm_mem_backing_src_alias {
	77	+ const char *name;
	78	+ enum vm_mem_backing_src_type type;
	79	+};
	80	+
	81	+void backing_src_help(void);
	82	+enum vm_mem_backing_src_type parse_backing_src_type(const char *type_name);
	83	+
70	84	#endif /* SELFTEST_KVM_TEST_UTIL_H */

tools/testing/selftests/kvm/include/x86_64/processor.h+40−1

@@ -263,6 +263,19 @@ static inline void outl(uint16_t port, uint32_t value) · 262 unmodified lines
263	263	__asm__ __volatile__("outl %%eax, %%dx" : : "d"(port), "a"(value));
264	264	}
265	265
	266	+static inline void cpuid(uint32_t eax, uint32_t ebx,
	267	+ uint32_t ecx, uint32_t edx)
	268	+{
	269	+ /* ecx is often an input as well as an output. */
	270	+ asm volatile("cpuid"
	271	+ : "=a" (*eax),
	272	+ "=b" (*ebx),
	273	+ "=c" (*ecx),
	274	+ "=d" (*edx)
	275	+ : "0" (eax), "2" (ecx)
	276	+ : "memory");
	277	+}
	278	+
266	279	#define SET_XMM(__var, __xmm) \
267	280	asm volatile("movq %0, %%"#__xmm : : "r"(__var) : #__xmm)
268	281
@@ -338,8 +351,10 @@ void vcpu_load_state(struct kvm_vm *vm, uint32_t vcpuid, · 69 unmodified lines
338	351	struct kvm_x86_state *state);
339	352
340	353	struct kvm_msr_list *kvm_get_msr_index_list(void);
341		−
	354	+uint64_t kvm_get_feature_msr(uint64_t msr_index);
342	355	struct kvm_cpuid2 *kvm_get_supported_cpuid(void);
	356	+
	357	+struct kvm_cpuid2 vcpu_get_cpuid(struct kvm_vm vm, uint32_t vcpuid);
343	358	void vcpu_set_cpuid(struct kvm_vm *vm, uint32_t vcpuid,
344	359	struct kvm_cpuid2 *cpuid);
345	360
@@ -391,6 +406,10 @@ bool set_cpuid(struct kvm_cpuid2 cpuid, struct kvm_cpuid_entry2 ent); · 45 unmodified lines
391	406	uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2,
392	407	uint64_t a3);
393	408
	409	+struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(void);
	410	+void vcpu_set_hv_cpuid(struct kvm_vm *vm, uint32_t vcpuid);
	411	+struct kvm_cpuid2 vcpu_get_supported_hv_cpuid(struct kvm_vm vm, uint32_t vcpuid);
	412	+
394	413	/*
395	414	* Basic CPU control in CR0
396	415	*/
@@ -406,8 +425,27 @@ uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2, · 9 unmodified lines
406	425	#define X86_CR0_CD (1UL<<30) /* Cache Disable */
407	426	#define X86_CR0_PG (1UL<<31) /* Paging */
408	427
	428	+#define APIC_DEFAULT_GPA 0xfee00000ULL
	429	+
	430	+/* APIC base address MSR and fields */
	431	+#define MSR_IA32_APICBASE 0x0000001b
	432	+#define MSR_IA32_APICBASE_BSP (1<<8)
	433	+#define MSR_IA32_APICBASE_EXTD (1<<10)
	434	+#define MSR_IA32_APICBASE_ENABLE (1<<11)
	435	+#define MSR_IA32_APICBASE_BASE (0xfffff<<12)
	436	+#define GET_APIC_BASE(x) (((x) >> 12) << 12)
	437	+
409	438	#define APIC_BASE_MSR 0x800
410	439	#define X2APIC_ENABLE (1UL << 10)
	440	+#define APIC_ID 0x20
	441	+#define APIC_LVR 0x30
	442	+#define GET_APIC_ID_FIELD(x) (((x) >> 24) & 0xFF)
	443	+#define APIC_TASKPRI 0x80
	444	+#define APIC_PROCPRI 0xA0
	445	+#define APIC_EOI 0xB0
	446	+#define APIC_SPIV 0xF0
	447	+#define APIC_SPIV_FOCUS_DISABLED (1 << 9)
	448	+#define APIC_SPIV_APIC_ENABLED (1 << 8)
411	449	#define APIC_ICR 0x300
412	450	#define APIC_DEST_SELF 0x40000
413	451	#define APIC_DEST_ALLINC 0x80000
@@ -432,6 +470,7 @@ uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2, · 18 unmodified lines
432	470	#define APIC_DM_EXTINT 0x00700
433	471	#define APIC_VECTOR_MASK 0x000FF
434	472	#define APIC_ICR2 0x310
	473	+#define SET_APIC_DEST_FIELD(x) ((x) << 24)
435	474
436	475	/* VMX_EPT_VPID_CAP bits */
437	476	#define VMX_EPT_VPID_CAP_AD_BITS (1ULL << 21)

tools/testing/selftests/kvm/lib/kvm_util.c+1−0

@@ -1801,6 +1801,7 @@ static struct exit_reason { · 1800 unmodified lines
1801	1801	{KVM_EXIT_DIRTY_RING_FULL, "DIRTY_RING_FULL"},
1802	1802	{KVM_EXIT_X86_RDMSR, "RDMSR"},
1803	1803	{KVM_EXIT_X86_WRMSR, "WRMSR"},
	1804	+ {KVM_EXIT_XEN, "XEN"},
1804	1805	#ifdef KVM_EXIT_MEMORY_NOT_PRESENT
1805	1806	{KVM_EXIT_MEMORY_NOT_PRESENT, "MEMORY_NOT_PRESENT"},
1806	1807	#endif

tools/testing/selftests/kvm/lib/perf_test_util.c+21−10

@@ -49,7 +49,8 @@ static void guest_code(uint32_t vcpu_id) · 48 unmodified lines
49	49	}
50	50
51	51	struct kvm_vm *perf_test_create_vm(enum vm_guest_mode mode, int vcpus,
52		− uint64_t vcpu_memory_bytes)
	52	+ uint64_t vcpu_memory_bytes,
	53	+ enum vm_mem_backing_src_type backing_src)
53	54	{
54	55	struct kvm_vm *vm;
55	56	uint64_t guest_num_pages;
@@ -93,8 +94,7 @@ struct kvm_vm *perf_test_create_vm(enum vm_guest_mode mode, int vcpus, · 37 unmodified lines
93	94	pr_info("guest physical test memory offset: 0x%lx\n", guest_test_phys_mem);
94	95
95	96	/* Add an extra memory slot for testing */
96		− vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
97		− guest_test_phys_mem,
	97	+ vm_userspace_mem_region_add(vm, backing_src, guest_test_phys_mem,
98	98	PERF_TEST_MEM_SLOT_INDEX,
99	99	guest_num_pages, 0);
100	100
@@ -112,7 +112,9 @@ void perf_test_destroy_vm(struct kvm_vm *vm) · 11 unmodified lines
112	112	kvm_vm_free(vm);
113	113	}
114	114
115		−void perf_test_setup_vcpus(struct kvm_vm *vm, int vcpus, uint64_t vcpu_memory_bytes)
	115	+void perf_test_setup_vcpus(struct kvm_vm *vm, int vcpus,
	116	+ uint64_t vcpu_memory_bytes,
	117	+ bool partition_vcpu_memory_access)
116	118	{
117	119	vm_paddr_t vcpu_gpa;
118	120	struct perf_test_vcpu_args *vcpu_args;
@@ -122,13 +124,22 @@ void perf_test_setup_vcpus(struct kvm_vm *vm, int vcpus, uint64_t vcpu_memory_by · 3 unmodified lines
122	124	vcpu_args = &perf_test_args.vcpu_args[vcpu_id];
123	125
124	126	vcpu_args->vcpu_id = vcpu_id;
125		− vcpu_args->gva = guest_test_virt_mem +
126		− (vcpu_id * vcpu_memory_bytes);
127		− vcpu_args->pages = vcpu_memory_bytes /
128		− perf_test_args.guest_page_size;
	127	+ if (partition_vcpu_memory_access) {
	128	+ vcpu_args->gva = guest_test_virt_mem +
	129	+ (vcpu_id * vcpu_memory_bytes);
	130	+ vcpu_args->pages = vcpu_memory_bytes /
	131	+ perf_test_args.guest_page_size;
	132	+ vcpu_gpa = guest_test_phys_mem +
	133	+ (vcpu_id * vcpu_memory_bytes);
	134	+ } else {
	135	+ vcpu_args->gva = guest_test_virt_mem;
	136	+ vcpu_args->pages = (vcpus * vcpu_memory_bytes) /
	137	+ perf_test_args.guest_page_size;
	138	+ vcpu_gpa = guest_test_phys_mem;
	139	+ }
129	140
130		− vcpu_gpa = guest_test_phys_mem + (vcpu_id * vcpu_memory_bytes);
131	141	pr_debug("Added VCPU %d with test mem gpa [%lx, %lx)\n",
132		− vcpu_id, vcpu_gpa, vcpu_gpa + vcpu_memory_bytes);
	142	+ vcpu_id, vcpu_gpa, vcpu_gpa +
	143	+ (vcpu_args->pages * perf_test_args.guest_page_size));
133	144	}
134	145	}

tools/testing/selftests/kvm/lib/test_util.c+30−1

@@ -10,6 +10,7 @@ · 9 unmodified lines
10	10	#include <limits.h>
11	11	#include <stdlib.h>
12	12	#include <time.h>
	13	+#include "linux/kernel.h"
13	14
14	15	#include "test_util.h"
15	16
@@ -84,7 +85,7 @@ struct timespec timespec_sub(struct timespec ts1, struct timespec ts2) · 68 unmodified lines
84	85	return timespec_add_ns((struct timespec){0}, ns1 - ns2);
85	86	}
86	87
87		−struct timespec timespec_diff_now(struct timespec start)
	88	+struct timespec timespec_elapsed(struct timespec start)
88	89	{
89	90	struct timespec end;
90	91
@@ -109,3 +110,31 @@ void print_skip(const char *fmt, ...) · 18 unmodified lines
109	110	va_end(ap);
110	111	puts(", skipping test");
111	112	}
	113	+
	114	+const struct vm_mem_backing_src_alias backing_src_aliases[] = {
	115	+ {"anonymous", VM_MEM_SRC_ANONYMOUS,},
	116	+ {"anonymous_thp", VM_MEM_SRC_ANONYMOUS_THP,},
	117	+ {"anonymous_hugetlb", VM_MEM_SRC_ANONYMOUS_HUGETLB,},
	118	+};
	119	+
	120	+void backing_src_help(void)
	121	+{
	122	+ int i;
	123	+
	124	+ printf("Available backing src types:\n");
	125	+ for (i = 0; i < ARRAY_SIZE(backing_src_aliases); i++)
	126	+ printf("\t%s\n", backing_src_aliases[i].name);
	127	+}
	128	+
	129	+enum vm_mem_backing_src_type parse_backing_src_type(const char *type_name)
	130	+{
	131	+ int i;
	132	+
	133	+ for (i = 0; i < ARRAY_SIZE(backing_src_aliases); i++)
	134	+ if (!strcmp(type_name, backing_src_aliases[i].name))
	135	+ return backing_src_aliases[i].type;
	136	+
	137	+ backing_src_help();
	138	+ TEST_FAIL("Unknown backing src type: %s", type_name);
	139	+ return -1;
	140	+}

tools/testing/selftests/kvm/lib/x86_64/processor.c+144−0

@@ -669,6 +669,82 @@ struct kvm_cpuid2 *kvm_get_supported_cpuid(void) · 668 unmodified lines
669	669	return cpuid;
670	670	}
671	671
	672	+/*
	673	+ * KVM Get MSR
	674	+ *
	675	+ * Input Args:
	676	+ * msr_index - Index of MSR
	677	+ *
	678	+ * Output Args: None
	679	+ *
	680	+ * Return: On success, value of the MSR. On failure a TEST_ASSERT is produced.
	681	+ *
	682	+ * Get value of MSR for VCPU.
	683	+ */
	684	+uint64_t kvm_get_feature_msr(uint64_t msr_index)
	685	+{
	686	+ struct {
	687	+ struct kvm_msrs header;
	688	+ struct kvm_msr_entry entry;
	689	+ } buffer = {};
	690	+ int r, kvm_fd;
	691	+
	692	+ buffer.header.nmsrs = 1;
	693	+ buffer.entry.index = msr_index;
	694	+ kvm_fd = open(KVM_DEV_PATH, O_RDONLY);
	695	+ if (kvm_fd < 0)
	696	+ exit(KSFT_SKIP);
	697	+
	698	+ r = ioctl(kvm_fd, KVM_GET_MSRS, &buffer.header);
	699	+ TEST_ASSERT(r == 1, "KVM_GET_MSRS IOCTL failed,\n"
	700	+ " rc: %i errno: %i", r, errno);
	701	+
	702	+ close(kvm_fd);
	703	+ return buffer.entry.data;
	704	+}
	705	+
	706	+/*
	707	+ * VM VCPU CPUID Set
	708	+ *
	709	+ * Input Args:
	710	+ * vm - Virtual Machine
	711	+ * vcpuid - VCPU id
	712	+ *
	713	+ * Output Args: None
	714	+ *
	715	+ * Return: KVM CPUID (KVM_GET_CPUID2)
	716	+ *
	717	+ * Set the VCPU's CPUID.
	718	+ */
	719	+struct kvm_cpuid2 vcpu_get_cpuid(struct kvm_vm vm, uint32_t vcpuid)
	720	+{
	721	+ struct vcpu *vcpu = vcpu_find(vm, vcpuid);
	722	+ struct kvm_cpuid2 *cpuid;
	723	+ int rc, max_ent;
	724	+
	725	+ TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
	726	+
	727	+ cpuid = allocate_kvm_cpuid2();
	728	+ max_ent = cpuid->nent;
	729	+
	730	+ for (cpuid->nent = 1; cpuid->nent <= max_ent; cpuid->nent++) {
	731	+ rc = ioctl(vcpu->fd, KVM_GET_CPUID2, cpuid);
	732	+ if (!rc)
	733	+ break;
	734	+
	735	+ TEST_ASSERT(rc == -1 && errno == E2BIG,
	736	+ "KVM_GET_CPUID2 should either succeed or give E2BIG: %d %d",
	737	+ rc, errno);
	738	+ }
	739	+
	740	+ TEST_ASSERT(rc == 0, "KVM_GET_CPUID2 failed, rc: %i errno: %i",
	741	+ rc, errno);
	742	+
	743	+ return cpuid;
	744	+}
	745	+
	746	+
	747	+
672	748	/*
673	749	* Locate a cpuid entry.
674	750	*
@@ -1224,3 +1300,71 @@ uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2, · 549 unmodified lines
1224	1300	: "b"(a0), "c"(a1), "d"(a2), "S"(a3));
1225	1301	return r;
1226	1302	}
	1303	+
	1304	+struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(void)
	1305	+{
	1306	+ static struct kvm_cpuid2 *cpuid;
	1307	+ int ret;
	1308	+ int kvm_fd;
	1309	+
	1310	+ if (cpuid)
	1311	+ return cpuid;
	1312	+
	1313	+ cpuid = allocate_kvm_cpuid2();
	1314	+ kvm_fd = open(KVM_DEV_PATH, O_RDONLY);
	1315	+ if (kvm_fd < 0)
	1316	+ exit(KSFT_SKIP);
	1317	+
	1318	+ ret = ioctl(kvm_fd, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
	1319	+ TEST_ASSERT(ret == 0, "KVM_GET_SUPPORTED_HV_CPUID failed %d %d\n",
	1320	+ ret, errno);
	1321	+
	1322	+ close(kvm_fd);
	1323	+ return cpuid;
	1324	+}
	1325	+
	1326	+void vcpu_set_hv_cpuid(struct kvm_vm *vm, uint32_t vcpuid)
	1327	+{
	1328	+ static struct kvm_cpuid2 *cpuid_full;
	1329	+ struct kvm_cpuid2 cpuid_sys, cpuid_hv;
	1330	+ int i, nent = 0;
	1331	+
	1332	+ if (!cpuid_full) {
	1333	+ cpuid_sys = kvm_get_supported_cpuid();
	1334	+ cpuid_hv = kvm_get_supported_hv_cpuid();
	1335	+
	1336	+ cpuid_full = malloc(sizeof(*cpuid_full) +
	1337	+ (cpuid_sys->nent + cpuid_hv->nent) *
	1338	+ sizeof(struct kvm_cpuid_entry2));
	1339	+ if (!cpuid_full) {
	1340	+ perror("malloc");
	1341	+ abort();
	1342	+ }
	1343	+
	1344	+ /* Need to skip KVM CPUID leaves 0x400000xx */
	1345	+ for (i = 0; i < cpuid_sys->nent; i++) {
	1346	+ if (cpuid_sys->entries[i].function >= 0x40000000 &&
	1347	+ cpuid_sys->entries[i].function < 0x40000100)
	1348	+ continue;
	1349	+ cpuid_full->entries[nent] = cpuid_sys->entries[i];
	1350	+ nent++;
	1351	+ }
	1352	+
	1353	+ memcpy(&cpuid_full->entries[nent], cpuid_hv->entries,
	1354	+ cpuid_hv->nent * sizeof(struct kvm_cpuid_entry2));
	1355	+ cpuid_full->nent = nent + cpuid_hv->nent;
	1356	+ }
	1357	+
	1358	+ vcpu_set_cpuid(vm, vcpuid, cpuid_full);
	1359	+}
	1360	+
	1361	+struct kvm_cpuid2 vcpu_get_supported_hv_cpuid(struct kvm_vm vm, uint32_t vcpuid)
	1362	+{
	1363	+ static struct kvm_cpuid2 *cpuid;
	1364	+
	1365	+ cpuid = allocate_kvm_cpuid2();
	1366	+
	1367	+ vcpu_ioctl(vm, vcpuid, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
	1368	+
	1369	+ return cpuid;
	1370	+}

tools/testing/selftests/kvm/lib/x86_64/svm.c+4−4

@@ -74,7 +74,7 @@ void generic_svm_setup(struct svm_test_data svm, void guest_rip, void *guest_r · 73 unmodified lines
74	74	wrmsr(MSR_VM_HSAVE_PA, svm->save_area_gpa);
75	75
76	76	memset(vmcb, 0, sizeof(*vmcb));
77		− asm volatile ("vmsave\n\t" : : "a" (vmcb_gpa) : "memory");
	77	+ asm volatile ("vmsave %0\n\t" : : "a" (vmcb_gpa) : "memory");
78	78	vmcb_set_seg(&save->es, get_es(), 0, -1U, data_seg_attr);
79	79	vmcb_set_seg(&save->cs, get_cs(), 0, -1U, code_seg_attr);
80	80	vmcb_set_seg(&save->ss, get_ss(), 0, -1U, data_seg_attr);
@@ -131,19 +131,19 @@ void generic_svm_setup(struct svm_test_data svm, void guest_rip, void *guest_r · 50 unmodified lines
131	131	void run_guest(struct vmcb *vmcb, uint64_t vmcb_gpa)
132	132	{
133	133	asm volatile (
134		− "vmload\n\t"
	134	+ "vmload %[vmcb_gpa]\n\t"
135	135	"mov rflags, %%r15\n\t" // rflags
136	136	"mov %%r15, 0x170(%[vmcb])\n\t"
137	137	"mov guest_regs, %%r15\n\t" // rax
138	138	"mov %%r15, 0x1f8(%[vmcb])\n\t"
139	139	LOAD_GPR_C
140		− "vmrun\n\t"
	140	+ "vmrun %[vmcb_gpa]\n\t"
141	141	SAVE_GPR_C
142	142	"mov 0x170(%[vmcb]), %%r15\n\t" // rflags
143	143	"mov %%r15, rflags\n\t"
144	144	"mov 0x1f8(%[vmcb]), %%r15\n\t" // rax
145	145	"mov %%r15, guest_regs\n\t"
146		− "vmsave\n\t"
	146	+ "vmsave %[vmcb_gpa]\n\t"
147	147	: : [vmcb] "r" (vmcb), [vmcb_gpa] "a" (vmcb_gpa)
148	148	: "r15", "memory");
149	149	}

tools/testing/selftests/kvm/memslot_modification_stress_test.c+212−0

@@ -0,0 +1,212 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * KVM memslot modification stress test
	4	+ * Adapted from demand_paging_test.c
	5	+ *
	6	+ * Copyright (C) 2018, Red Hat, Inc.
	7	+ * Copyright (C) 2020, Google, Inc.
	8	+ */
	9	+
	10	+#define _GNU_SOURCE /* for program_invocation_name */
	11	+
	12	+#include <stdio.h>
	13	+#include <stdlib.h>
	14	+#include <sys/syscall.h>
	15	+#include <unistd.h>
	16	+#include <asm/unistd.h>
	17	+#include <time.h>
	18	+#include <poll.h>
	19	+#include <pthread.h>
	20	+#include <linux/bitmap.h>
	21	+#include <linux/bitops.h>
	22	+#include <linux/userfaultfd.h>
	23	+
	24	+#include "perf_test_util.h"
	25	+#include "processor.h"
	26	+#include "test_util.h"
	27	+#include "guest_modes.h"
	28	+
	29	+#define DUMMY_MEMSLOT_INDEX 7
	30	+
	31	+#define DEFAULT_MEMSLOT_MODIFICATION_ITERATIONS 10
	32	+
	33	+
	34	+static int nr_vcpus = 1;
	35	+static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
	36	+
	37	+static bool run_vcpus = true;
	38	+
	39	+static void vcpu_worker(void data)
	40	+{
	41	+ int ret;
	42	+ struct perf_test_vcpu_args *vcpu_args =
	43	+ (struct perf_test_vcpu_args *)data;
	44	+ int vcpu_id = vcpu_args->vcpu_id;
	45	+ struct kvm_vm *vm = perf_test_args.vm;
	46	+ struct kvm_run *run;
	47	+
	48	+ vcpu_args_set(vm, vcpu_id, 1, vcpu_id);
	49	+ run = vcpu_state(vm, vcpu_id);
	50	+
	51	+ /* Let the guest access its memory until a stop signal is received */
	52	+ while (READ_ONCE(run_vcpus)) {
	53	+ ret = _vcpu_run(vm, vcpu_id);
	54	+ TEST_ASSERT(ret == 0, "vcpu_run failed: %d\n", ret);
	55	+
	56	+ if (get_ucall(vm, vcpu_id, NULL) == UCALL_SYNC)
	57	+ continue;
	58	+
	59	+ TEST_ASSERT(false,
	60	+ "Invalid guest sync status: exit_reason=%s\n",
	61	+ exit_reason_str(run->exit_reason));
	62	+ }
	63	+
	64	+ return NULL;
	65	+}
	66	+
	67	+struct memslot_antagonist_args {
	68	+ struct kvm_vm *vm;
	69	+ useconds_t delay;
	70	+ uint64_t nr_modifications;
	71	+};
	72	+
	73	+static void add_remove_memslot(struct kvm_vm *vm, useconds_t delay,
	74	+ uint64_t nr_modifications, uint64_t gpa)
	75	+{
	76	+ int i;
	77	+
	78	+ for (i = 0; i < nr_modifications; i++) {
	79	+ usleep(delay);
	80	+ vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, gpa,
	81	+ DUMMY_MEMSLOT_INDEX, 1, 0);
	82	+
	83	+ vm_mem_region_delete(vm, DUMMY_MEMSLOT_INDEX);
	84	+ }
	85	+}
	86	+
	87	+struct test_params {
	88	+ useconds_t memslot_modification_delay;
	89	+ uint64_t nr_memslot_modifications;
	90	+ bool partition_vcpu_memory_access;
	91	+};
	92	+
	93	+static void run_test(enum vm_guest_mode mode, void *arg)
	94	+{
	95	+ struct test_params *p = arg;
	96	+ pthread_t *vcpu_threads;
	97	+ struct kvm_vm *vm;
	98	+ int vcpu_id;
	99	+
	100	+ vm = perf_test_create_vm(mode, nr_vcpus, guest_percpu_mem_size,
	101	+ VM_MEM_SRC_ANONYMOUS);
	102	+
	103	+ perf_test_args.wr_fract = 1;
	104	+
	105	+ vcpu_threads = malloc(nr_vcpus * sizeof(*vcpu_threads));
	106	+ TEST_ASSERT(vcpu_threads, "Memory allocation failed");
	107	+
	108	+ perf_test_setup_vcpus(vm, nr_vcpus, guest_percpu_mem_size,
	109	+ p->partition_vcpu_memory_access);
	110	+
	111	+ /* Export the shared variables to the guest */
	112	+ sync_global_to_guest(vm, perf_test_args);
	113	+
	114	+ pr_info("Finished creating vCPUs\n");
	115	+
	116	+ for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++)
	117	+ pthread_create(&vcpu_threads[vcpu_id], NULL, vcpu_worker,
	118	+ &perf_test_args.vcpu_args[vcpu_id]);
	119	+
	120	+ pr_info("Started all vCPUs\n");
	121	+
	122	+ add_remove_memslot(vm, p->memslot_modification_delay,
	123	+ p->nr_memslot_modifications,
	124	+ guest_test_phys_mem +
	125	+ (guest_percpu_mem_size * nr_vcpus) +
	126	+ perf_test_args.host_page_size +
	127	+ perf_test_args.guest_page_size);
	128	+
	129	+ run_vcpus = false;
	130	+
	131	+ /* Wait for the vcpu threads to quit */
	132	+ for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++)
	133	+ pthread_join(vcpu_threads[vcpu_id], NULL);
	134	+
	135	+ pr_info("All vCPU threads joined\n");
	136	+
	137	+ ucall_uninit(vm);
	138	+ kvm_vm_free(vm);
	139	+
	140	+ free(vcpu_threads);
	141	+}
	142	+
	143	+static void help(char *name)
	144	+{
	145	+ puts("");
	146	+ printf("usage: %s [-h] [-m mode] [-d delay_usec]\n"
	147	+ " [-b memory] [-v vcpus] [-o] [-i iterations]\n", name);
	148	+ guest_modes_help();
	149	+ printf(" -d: add a delay between each iteration of adding and\n"
	150	+ " deleting a memslot in usec.\n");
	151	+ printf(" -b: specify the size of the memory region which should be\n"
	152	+ " accessed by each vCPU. e.g. 10M or 3G.\n"
	153	+ " Default: 1G\n");
	154	+ printf(" -v: specify the number of vCPUs to run.\n");
	155	+ printf(" -o: Overlap guest memory accesses instead of partitioning\n"
	156	+ " them into a separate region of memory for each vCPU.\n");
	157	+ printf(" -i: specify the number of iterations of adding and removing\n"
	158	+ " a memslot.\n"
	159	+ " Default: %d\n", DEFAULT_MEMSLOT_MODIFICATION_ITERATIONS);
	160	+ puts("");
	161	+ exit(0);
	162	+}
	163	+
	164	+int main(int argc, char *argv[])
	165	+{
	166	+ int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS);
	167	+ int opt;
	168	+ struct test_params p = {
	169	+ .memslot_modification_delay = 0,
	170	+ .nr_memslot_modifications =
	171	+ DEFAULT_MEMSLOT_MODIFICATION_ITERATIONS,
	172	+ .partition_vcpu_memory_access = true
	173	+ };
	174	+
	175	+ guest_modes_append_default();
	176	+
	177	+ while ((opt = getopt(argc, argv, "hm:d:b:v:oi:")) != -1) {
	178	+ switch (opt) {
	179	+ case 'm':
	180	+ guest_modes_cmdline(optarg);
	181	+ break;
	182	+ case 'd':
	183	+ p.memslot_modification_delay = strtoul(optarg, NULL, 0);
	184	+ TEST_ASSERT(p.memslot_modification_delay >= 0,
	185	+ "A negative delay is not supported.");
	186	+ break;
	187	+ case 'b':
	188	+ guest_percpu_mem_size = parse_size(optarg);
	189	+ break;
	190	+ case 'v':
	191	+ nr_vcpus = atoi(optarg);
	192	+ TEST_ASSERT(nr_vcpus > 0 && nr_vcpus <= max_vcpus,
	193	+ "Invalid number of vcpus, must be between 1 and %d",
	194	+ max_vcpus);
	195	+ break;
	196	+ case 'o':
	197	+ p.partition_vcpu_memory_access = false;
	198	+ break;
	199	+ case 'i':
	200	+ p.nr_memslot_modifications = atoi(optarg);
	201	+ break;
	202	+ case 'h':
	203	+ default:
	204	+ help(argv[0]);
	205	+ break;
	206	+ }
	207	+ }
	208	+
	209	+ for_each_guest_mode(run_test, &p);
	210	+
	211	+ return 0;
	212	+}

tools/testing/selftests/kvm/settings+1−0

@@ -0,0 +1 @@
	1	+timeout=120

tools/testing/selftests/kvm/x86_64/evmcs_test.c+2−1

@@ -99,6 +99,7 @@ int main(int argc, char *argv[]) · 98 unmodified lines
99	99	exit(KSFT_SKIP);
100	100	}
101	101
	102	+ vcpu_set_hv_cpuid(vm, VCPU_ID);
102	103	vcpu_enable_evmcs(vm, VCPU_ID);
103	104
104	105	run = vcpu_state(vm, VCPU_ID);
@@ -142,7 +143,7 @@ int main(int argc, char *argv[]) · 37 unmodified lines
142	143	/* Restore state in a new VM. */
143	144	kvm_vm_restart(vm, O_RDWR);
144	145	vm_vcpu_add(vm, VCPU_ID);
145		− vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid());
	146	+ vcpu_set_hv_cpuid(vm, VCPU_ID);
146	147	vcpu_enable_evmcs(vm, VCPU_ID);
147	148	vcpu_load_state(vm, VCPU_ID, state);
148	149	run = vcpu_state(vm, VCPU_ID);

tools/testing/selftests/kvm/x86_64/get_cpuid_test.c+175−0

@@ -0,0 +1,175 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/*
	3	+ * Copyright (C) 2021, Red Hat Inc.
	4	+ *
	5	+ * Generic tests for KVM CPUID set/get ioctls
	6	+ */
	7	+#include <asm/kvm_para.h>
	8	+#include <linux/kvm_para.h>
	9	+#include <stdint.h>
	10	+
	11	+#include "test_util.h"
	12	+#include "kvm_util.h"
	13	+#include "processor.h"
	14	+
	15	+#define VCPU_ID 0
	16	+
	17	+/* CPUIDs known to differ */
	18	+struct {
	19	+ u32 function;
	20	+ u32 index;
	21	+} mangled_cpuids[] = {
	22	+ {.function = 0xd, .index = 0},
	23	+};
	24	+
	25	+static void test_guest_cpuids(struct kvm_cpuid2 *guest_cpuid)
	26	+{
	27	+ int i;
	28	+ u32 eax, ebx, ecx, edx;
	29	+
	30	+ for (i = 0; i < guest_cpuid->nent; i++) {
	31	+ eax = guest_cpuid->entries[i].function;
	32	+ ecx = guest_cpuid->entries[i].index;
	33	+
	34	+ cpuid(&eax, &ebx, &ecx, &edx);
	35	+
	36	+ GUEST_ASSERT(eax == guest_cpuid->entries[i].eax &&
	37	+ ebx == guest_cpuid->entries[i].ebx &&
	38	+ ecx == guest_cpuid->entries[i].ecx &&
	39	+ edx == guest_cpuid->entries[i].edx);
	40	+ }
	41	+
	42	+}
	43	+
	44	+static void test_cpuid_40000000(struct kvm_cpuid2 *guest_cpuid)
	45	+{
	46	+ u32 eax = 0x40000000, ebx, ecx = 0, edx;
	47	+
	48	+ cpuid(&eax, &ebx, &ecx, &edx);
	49	+
	50	+ GUEST_ASSERT(eax == 0x40000001);
	51	+}
	52	+
	53	+static void guest_main(struct kvm_cpuid2 *guest_cpuid)
	54	+{
	55	+ GUEST_SYNC(1);
	56	+
	57	+ test_guest_cpuids(guest_cpuid);
	58	+
	59	+ GUEST_SYNC(2);
	60	+
	61	+ test_cpuid_40000000(guest_cpuid);
	62	+
	63	+ GUEST_DONE();
	64	+}
	65	+
	66	+static bool is_cpuid_mangled(struct kvm_cpuid_entry2 *entrie)
	67	+{
	68	+ int i;
	69	+
	70	+ for (i = 0; i < sizeof(mangled_cpuids); i++) {
	71	+ if (mangled_cpuids[i].function == entrie->function &&
	72	+ mangled_cpuids[i].index == entrie->index)
	73	+ return true;
	74	+ }
	75	+
	76	+ return false;
	77	+}
	78	+
	79	+static void check_cpuid(struct kvm_cpuid2 cpuid, struct kvm_cpuid_entry2 entrie)
	80	+{
	81	+ int i;
	82	+
	83	+ for (i = 0; i < cpuid->nent; i++) {
	84	+ if (cpuid->entries[i].function == entrie->function &&
	85	+ cpuid->entries[i].index == entrie->index) {
	86	+ if (is_cpuid_mangled(entrie))
	87	+ return;
	88	+
	89	+ TEST_ASSERT(cpuid->entries[i].eax == entrie->eax &&
	90	+ cpuid->entries[i].ebx == entrie->ebx &&
	91	+ cpuid->entries[i].ecx == entrie->ecx &&
	92	+ cpuid->entries[i].edx == entrie->edx,
	93	+ "CPUID 0x%x.%x differ: 0x%x:0x%x:0x%x:0x%x vs 0x%x:0x%x:0x%x:0x%x",
	94	+ entrie->function, entrie->index,
	95	+ cpuid->entries[i].eax, cpuid->entries[i].ebx,
	96	+ cpuid->entries[i].ecx, cpuid->entries[i].edx,
	97	+ entrie->eax, entrie->ebx, entrie->ecx, entrie->edx);
	98	+ return;
	99	+ }
	100	+ }
	101	+
	102	+ TEST_ASSERT(false, "CPUID 0x%x.%x not found", entrie->function, entrie->index);
	103	+}
	104	+
	105	+static void compare_cpuids(struct kvm_cpuid2 cpuid1, struct kvm_cpuid2 cpuid2)
	106	+{
	107	+ int i;
	108	+
	109	+ for (i = 0; i < cpuid1->nent; i++)
	110	+ check_cpuid(cpuid2, &cpuid1->entries[i]);
	111	+
	112	+ for (i = 0; i < cpuid2->nent; i++)
	113	+ check_cpuid(cpuid1, &cpuid2->entries[i]);
	114	+}
	115	+
	116	+static void run_vcpu(struct kvm_vm *vm, uint32_t vcpuid, int stage)
	117	+{
	118	+ struct ucall uc;
	119	+
	120	+ _vcpu_run(vm, vcpuid);
	121	+
	122	+ switch (get_ucall(vm, vcpuid, &uc)) {
	123	+ case UCALL_SYNC:
	124	+ TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") &&
	125	+ uc.args[1] == stage + 1,
	126	+ "Stage %d: Unexpected register values vmexit, got %lx",
	127	+ stage + 1, (ulong)uc.args[1]);
	128	+ return;
	129	+ case UCALL_DONE:
	130	+ return;
	131	+ case UCALL_ABORT:
	132	+ TEST_ASSERT(false, "%s at %s:%ld\n\tvalues: %#lx, %#lx", (const char *)uc.args[0],
	133	+ __FILE__, uc.args[1], uc.args[2], uc.args[3]);
	134	+ default:
	135	+ TEST_ASSERT(false, "Unexpected exit: %s",
	136	+ exit_reason_str(vcpu_state(vm, vcpuid)->exit_reason));
	137	+ }
	138	+}
	139	+
	140	+struct kvm_cpuid2 vcpu_alloc_cpuid(struct kvm_vm vm, vm_vaddr_t p_gva, struct kvm_cpuid2 cpuid)
	141	+{
	142	+ int size = sizeof(cpuid) + cpuid->nent sizeof(cpuid->entries[0]);
	143	+ vm_vaddr_t gva = vm_vaddr_alloc(vm, size,
	144	+ getpagesize(), 0, 0);
	145	+ struct kvm_cpuid2 *guest_cpuids = addr_gva2hva(vm, gva);
	146	+
	147	+ memcpy(guest_cpuids, cpuid, size);
	148	+
	149	+ *p_gva = gva;
	150	+ return guest_cpuids;
	151	+}
	152	+
	153	+int main(void)
	154	+{
	155	+ struct kvm_cpuid2 supp_cpuid, cpuid2;
	156	+ vm_vaddr_t cpuid_gva;
	157	+ struct kvm_vm *vm;
	158	+ int stage;
	159	+
	160	+ vm = vm_create_default(VCPU_ID, 0, guest_main);
	161	+
	162	+ supp_cpuid = kvm_get_supported_cpuid();
	163	+ cpuid2 = vcpu_get_cpuid(vm, VCPU_ID);
	164	+
	165	+ compare_cpuids(supp_cpuid, cpuid2);
	166	+
	167	+ vcpu_alloc_cpuid(vm, &cpuid_gva, cpuid2);
	168	+
	169	+ vcpu_args_set(vm, VCPU_ID, 1, cpuid_gva);
	170	+
	171	+ for (stage = 0; stage < 3; stage++)
	172	+ run_vcpu(vm, VCPU_ID, stage);
	173	+
	174	+ kvm_vm_free(vm);
	175	+}

tools/testing/selftests/kvm/x86_64/hyperv_cpuid.c+3−28

@@ -125,30 +125,6 @@ void test_hv_cpuid_e2big(struct kvm_vm *vm, bool system) · 124 unmodified lines
125	125	" it should have: %d %d", system ? "KVM" : "vCPU", ret, errno);
126	126	}
127	127
128		−
129		−struct kvm_cpuid2 kvm_get_supported_hv_cpuid(struct kvm_vm vm, bool system)
130		−{
131		− int nent = 20; /* should be enough */
132		− static struct kvm_cpuid2 *cpuid;
133		−
134		− cpuid = malloc(sizeof(cpuid) + nent sizeof(struct kvm_cpuid_entry2));
135		−
136		− if (!cpuid) {
137		− perror("malloc");
138		− abort();
139		− }
140		−
141		− cpuid->nent = nent;
142		−
143		− if (!system)
144		− vcpu_ioctl(vm, VCPU_ID, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
145		− else
146		− kvm_ioctl(vm, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
147		−
148		− return cpuid;
149		−}
150		−
151		−
152	128	int main(int argc, char *argv[])
153	129	{
154	130	struct kvm_vm *vm;
@@ -167,7 +143,7 @@ int main(int argc, char *argv[]) · 12 unmodified lines
167	143	/* Test vCPU ioctl version */
168	144	test_hv_cpuid_e2big(vm, false);
169	145
170		− hv_cpuid_entries = kvm_get_supported_hv_cpuid(vm, false);
	146	+ hv_cpuid_entries = vcpu_get_supported_hv_cpuid(vm, VCPU_ID);
171	147	test_hv_cpuid(hv_cpuid_entries, false);
172	148	free(hv_cpuid_entries);
173	149
@@ -177,7 +153,7 @@ int main(int argc, char *argv[]) · 3 unmodified lines
177	153	goto do_sys;
178	154	}
179	155	vcpu_enable_evmcs(vm, VCPU_ID);
180		− hv_cpuid_entries = kvm_get_supported_hv_cpuid(vm, false);
	156	+ hv_cpuid_entries = vcpu_get_supported_hv_cpuid(vm, VCPU_ID);
181	157	test_hv_cpuid(hv_cpuid_entries, true);
182	158	free(hv_cpuid_entries);
183	159
@@ -190,9 +166,8 @@ do_sys: · 6 unmodified lines
190	166
191	167	test_hv_cpuid_e2big(vm, true);
192	168
193		− hv_cpuid_entries = kvm_get_supported_hv_cpuid(vm, true);
	169	+ hv_cpuid_entries = kvm_get_supported_hv_cpuid();
194	170	test_hv_cpuid(hv_cpuid_entries, nested_vmx_supported());
195		− free(hv_cpuid_entries);
196	171
197	172	out:
198	173	kvm_vm_free(vm);

tools/testing/selftests/kvm/x86_64/vmx_pmu_msrs_test.c+131−0

@@ -0,0 +1,131 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * VMX-pmu related msrs test
	4	+ *
	5	+ * Copyright (C) 2021 Intel Corporation
	6	+ *
	7	+ * Test to check the effect of various CPUID settings
	8	+ * on the MSR_IA32_PERF_CAPABILITIES MSR, and check that
	9	+ * whatever we write with KVM_SET_MSR is _not_ modified
	10	+ * in the guest and test it can be retrieved with KVM_GET_MSR.
	11	+ *
	12	+ * Test to check that invalid LBR formats are rejected.
	13	+ */
	14	+
	15	+#define _GNU_SOURCE /* for program_invocation_short_name */
	16	+#include <sys/ioctl.h>
	17	+
	18	+#include "kvm_util.h"
	19	+#include "vmx.h"
	20	+
	21	+#define VCPU_ID 0
	22	+
	23	+#define X86_FEATURE_PDCM (1<<15)
	24	+#define PMU_CAP_FW_WRITES (1ULL << 13)
	25	+#define PMU_CAP_LBR_FMT 0x3f
	26	+
	27	+union cpuid10_eax {
	28	+ struct {
	29	+ unsigned int version_id:8;
	30	+ unsigned int num_counters:8;
	31	+ unsigned int bit_width:8;
	32	+ unsigned int mask_length:8;
	33	+ } split;
	34	+ unsigned int full;
	35	+};
	36	+
	37	+union perf_capabilities {
	38	+ struct {
	39	+ u64 lbr_format:6;
	40	+ u64 pebs_trap:1;
	41	+ u64 pebs_arch_reg:1;
	42	+ u64 pebs_format:4;
	43	+ u64 smm_freeze:1;
	44	+ u64 full_width_write:1;
	45	+ u64 pebs_baseline:1;
	46	+ u64 perf_metrics:1;
	47	+ u64 pebs_output_pt_available:1;
	48	+ u64 anythread_deprecated:1;
	49	+ };
	50	+ u64 capabilities;
	51	+};
	52	+
	53	+static void guest_code(void)
	54	+{
	55	+ wrmsr(MSR_IA32_PERF_CAPABILITIES, PMU_CAP_LBR_FMT);
	56	+}
	57	+
	58	+int main(int argc, char *argv[])
	59	+{
	60	+ struct kvm_cpuid2 *cpuid;
	61	+ struct kvm_cpuid_entry2 *entry_1_0;
	62	+ struct kvm_cpuid_entry2 *entry_a_0;
	63	+ bool pdcm_supported = false;
	64	+ struct kvm_vm *vm;
	65	+ int ret;
	66	+ union cpuid10_eax eax;
	67	+ union perf_capabilities host_cap;
	68	+
	69	+ host_cap.capabilities = kvm_get_feature_msr(MSR_IA32_PERF_CAPABILITIES);
	70	+ host_cap.capabilities &= (PMU_CAP_FW_WRITES \| PMU_CAP_LBR_FMT);
	71	+
	72	+ /* Create VM */
	73	+ vm = vm_create_default(VCPU_ID, 0, guest_code);
	74	+ cpuid = kvm_get_supported_cpuid();
	75	+
	76	+ if (kvm_get_cpuid_max_basic() >= 0xa) {
	77	+ entry_1_0 = kvm_get_supported_cpuid_index(1, 0);
	78	+ entry_a_0 = kvm_get_supported_cpuid_index(0xa, 0);
	79	+ pdcm_supported = entry_1_0 && !!(entry_1_0->ecx & X86_FEATURE_PDCM);
	80	+ eax.full = entry_a_0->eax;
	81	+ }
	82	+ if (!pdcm_supported) {
	83	+ print_skip("MSR_IA32_PERF_CAPABILITIES is not supported by the vCPU");
	84	+ exit(KSFT_SKIP);
	85	+ }
	86	+ if (!eax.split.version_id) {
	87	+ print_skip("PMU is not supported by the vCPU");
	88	+ exit(KSFT_SKIP);
	89	+ }
	90	+
	91	+ /* testcase 1, set capabilities when we have PDCM bit */
	92	+ vcpu_set_cpuid(vm, VCPU_ID, cpuid);
	93	+ vcpu_set_msr(vm, 0, MSR_IA32_PERF_CAPABILITIES, PMU_CAP_FW_WRITES);
	94	+
	95	+ /* check capabilities can be retrieved with KVM_GET_MSR */
	96	+ ASSERT_EQ(vcpu_get_msr(vm, VCPU_ID, MSR_IA32_PERF_CAPABILITIES), PMU_CAP_FW_WRITES);
	97	+
	98	+ /* check whatever we write with KVM_SET_MSR is _not_ modified */
	99	+ vcpu_run(vm, VCPU_ID);
	100	+ ASSERT_EQ(vcpu_get_msr(vm, VCPU_ID, MSR_IA32_PERF_CAPABILITIES), PMU_CAP_FW_WRITES);
	101	+
	102	+ /* testcase 2, check valid LBR formats are accepted */
	103	+ vcpu_set_msr(vm, 0, MSR_IA32_PERF_CAPABILITIES, 0);
	104	+ ASSERT_EQ(vcpu_get_msr(vm, VCPU_ID, MSR_IA32_PERF_CAPABILITIES), 0);
	105	+
	106	+ vcpu_set_msr(vm, 0, MSR_IA32_PERF_CAPABILITIES, host_cap.lbr_format);
	107	+ ASSERT_EQ(vcpu_get_msr(vm, VCPU_ID, MSR_IA32_PERF_CAPABILITIES), (u64)host_cap.lbr_format);
	108	+
	109	+ /* testcase 3, check invalid LBR format is rejected */
	110	+ ret = _vcpu_set_msr(vm, 0, MSR_IA32_PERF_CAPABILITIES, PMU_CAP_LBR_FMT);
	111	+ TEST_ASSERT(ret == 0, "Bad PERF_CAPABILITIES didn't fail.");
	112	+
	113	+ /* testcase 4, set capabilities when we don't have PDCM bit */
	114	+ entry_1_0->ecx &= ~X86_FEATURE_PDCM;
	115	+ vcpu_set_cpuid(vm, VCPU_ID, cpuid);
	116	+ ret = _vcpu_set_msr(vm, 0, MSR_IA32_PERF_CAPABILITIES, host_cap.capabilities);
	117	+ TEST_ASSERT(ret == 0, "Bad PERF_CAPABILITIES didn't fail.");
	118	+
	119	+ /* testcase 5, set capabilities when we don't have PMU version bits */
	120	+ entry_1_0->ecx \|= X86_FEATURE_PDCM;
	121	+ eax.split.version_id = 0;
	122	+ entry_1_0->ecx = eax.full;
	123	+ vcpu_set_cpuid(vm, VCPU_ID, cpuid);
	124	+ ret = _vcpu_set_msr(vm, 0, MSR_IA32_PERF_CAPABILITIES, PMU_CAP_FW_WRITES);
	125	+ TEST_ASSERT(ret == 0, "Bad PERF_CAPABILITIES didn't fail.");
	126	+
	127	+ vcpu_set_msr(vm, 0, MSR_IA32_PERF_CAPABILITIES, 0);
	128	+ ASSERT_EQ(vcpu_get_msr(vm, VCPU_ID, MSR_IA32_PERF_CAPABILITIES), 0);
	129	+
	130	+ kvm_vm_free(vm);
	131	+}

tools/testing/selftests/kvm/x86_64/xapic_ipi_test.c+544−0

@@ -0,0 +1,544 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * xapic_ipi_test
	4	+ *
	5	+ * Copyright (C) 2020, Google LLC.
	6	+ *
	7	+ * This work is licensed under the terms of the GNU GPL, version 2.
	8	+ *
	9	+ * Test that when the APIC is in xAPIC mode, a vCPU can send an IPI to wake
	10	+ * another vCPU that is halted when KVM's backing page for the APIC access
	11	+ * address has been moved by mm.
	12	+ *
	13	+ * The test starts two vCPUs: one that sends IPIs and one that continually
	14	+ * executes HLT. The sender checks that the halter has woken from the HLT and
	15	+ * has reentered HLT before sending the next IPI. While the vCPUs are running,
	16	+ * the host continually calls migrate_pages to move all of the process' pages
	17	+ * amongst the available numa nodes on the machine.
	18	+ *
	19	+ * Migration is a command line option. When used on non-numa machines will
	20	+ * exit with error. Test is still usefull on non-numa for testing IPIs.
	21	+ */
	22	+
	23	+#define _GNU_SOURCE /* for program_invocation_short_name */
	24	+#include <getopt.h>
	25	+#include <pthread.h>
	26	+#include <inttypes.h>
	27	+#include <string.h>
	28	+#include <time.h>
	29	+
	30	+#include "kvm_util.h"
	31	+#include "numaif.h"
	32	+#include "processor.h"
	33	+#include "test_util.h"
	34	+#include "vmx.h"
	35	+
	36	+/* Default running time for the test */
	37	+#define DEFAULT_RUN_SECS 3
	38	+
	39	+/* Default delay between migrate_pages calls (microseconds) */
	40	+#define DEFAULT_DELAY_USECS 500000
	41	+
	42	+#define HALTER_VCPU_ID 0
	43	+#define SENDER_VCPU_ID 1
	44	+
	45	+volatile uint32_t apic_base = (volatile uint32_t )APIC_DEFAULT_GPA;
	46	+
	47	+/*
	48	+ * Vector for IPI from sender vCPU to halting vCPU.
	49	+ * Value is arbitrary and was chosen for the alternating bit pattern. Any
	50	+ * value should work.
	51	+ */
	52	+#define IPI_VECTOR 0xa5
	53	+
	54	+/*
	55	+ * Incremented in the IPI handler. Provides evidence to the sender that the IPI
	56	+ * arrived at the destination
	57	+ */
	58	+static volatile uint64_t ipis_rcvd;
	59	+
	60	+/* Data struct shared between host main thread and vCPUs */
	61	+struct test_data_page {
	62	+ uint32_t halter_apic_id;
	63	+ volatile uint64_t hlt_count;
	64	+ volatile uint64_t wake_count;
	65	+ uint64_t ipis_sent;
	66	+ uint64_t migrations_attempted;
	67	+ uint64_t migrations_completed;
	68	+ uint32_t icr;
	69	+ uint32_t icr2;
	70	+ uint32_t halter_tpr;
	71	+ uint32_t halter_ppr;
	72	+
	73	+ /*
	74	+ * Record local version register as a cross-check that APIC access
	75	+ * worked. Value should match what KVM reports (APIC_VERSION in
	76	+ * arch/x86/kvm/lapic.c). If test is failing, check that values match
	77	+ * to determine whether APIC access exits are working.
	78	+ */
	79	+ uint32_t halter_lvr;
	80	+};
	81	+
	82	+struct thread_params {
	83	+ struct test_data_page *data;
	84	+ struct kvm_vm *vm;
	85	+ uint32_t vcpu_id;
	86	+ uint64_t pipis_rcvd; / host address of ipis_rcvd global */
	87	+};
	88	+
	89	+uint32_t read_apic_reg(uint reg)
	90	+{
	91	+ return apic_base[reg >> 2];
	92	+}
	93	+
	94	+void write_apic_reg(uint reg, uint32_t val)
	95	+{
	96	+ apic_base[reg >> 2] = val;
	97	+}
	98	+
	99	+void disable_apic(void)
	100	+{
	101	+ wrmsr(MSR_IA32_APICBASE,
	102	+ rdmsr(MSR_IA32_APICBASE) &
	103	+ ~(MSR_IA32_APICBASE_ENABLE \| MSR_IA32_APICBASE_EXTD));
	104	+}
	105	+
	106	+void enable_xapic(void)
	107	+{
	108	+ uint64_t val = rdmsr(MSR_IA32_APICBASE);
	109	+
	110	+ /* Per SDM: to enable xAPIC when in x2APIC must first disable APIC */
	111	+ if (val & MSR_IA32_APICBASE_EXTD) {
	112	+ disable_apic();
	113	+ wrmsr(MSR_IA32_APICBASE,
	114	+ rdmsr(MSR_IA32_APICBASE) \| MSR_IA32_APICBASE_ENABLE);
	115	+ } else if (!(val & MSR_IA32_APICBASE_ENABLE)) {
	116	+ wrmsr(MSR_IA32_APICBASE, val \| MSR_IA32_APICBASE_ENABLE);
	117	+ }
	118	+
	119	+ /*
	120	+ * Per SDM: reset value of spurious interrupt vector register has the
	121	+ * APIC software enabled bit=0. It must be enabled in addition to the
	122	+ * enable bit in the MSR.
	123	+ */
	124	+ val = read_apic_reg(APIC_SPIV) \| APIC_SPIV_APIC_ENABLED;
	125	+ write_apic_reg(APIC_SPIV, val);
	126	+}
	127	+
	128	+void verify_apic_base_addr(void)
	129	+{
	130	+ uint64_t msr = rdmsr(MSR_IA32_APICBASE);
	131	+ uint64_t base = GET_APIC_BASE(msr);
	132	+
	133	+ GUEST_ASSERT(base == APIC_DEFAULT_GPA);
	134	+}
	135	+
	136	+static void halter_guest_code(struct test_data_page *data)
	137	+{
	138	+ verify_apic_base_addr();
	139	+ enable_xapic();
	140	+
	141	+ data->halter_apic_id = GET_APIC_ID_FIELD(read_apic_reg(APIC_ID));
	142	+ data->halter_lvr = read_apic_reg(APIC_LVR);
	143	+
	144	+ /*
	145	+ * Loop forever HLTing and recording halts & wakes. Disable interrupts
	146	+ * each time around to minimize window between signaling the pending
	147	+ * halt to the sender vCPU and executing the halt. No need to disable on
	148	+ * first run as this vCPU executes first and the host waits for it to
	149	+ * signal going into first halt before starting the sender vCPU. Record
	150	+ * TPR and PPR for diagnostic purposes in case the test fails.
	151	+ */
	152	+ for (;;) {
	153	+ data->halter_tpr = read_apic_reg(APIC_TASKPRI);
	154	+ data->halter_ppr = read_apic_reg(APIC_PROCPRI);
	155	+ data->hlt_count++;
	156	+ asm volatile("sti; hlt; cli");
	157	+ data->wake_count++;
	158	+ }
	159	+}
	160	+
	161	+/*
	162	+ * Runs on halter vCPU when IPI arrives. Write an arbitrary non-zero value to
	163	+ * enable diagnosing errant writes to the APIC access address backing page in
	164	+ * case of test failure.
	165	+ */
	166	+static void guest_ipi_handler(struct ex_regs *regs)
	167	+{
	168	+ ipis_rcvd++;
	169	+ write_apic_reg(APIC_EOI, 77);
	170	+}
	171	+
	172	+static void sender_guest_code(struct test_data_page *data)
	173	+{
	174	+ uint64_t last_wake_count;
	175	+ uint64_t last_hlt_count;
	176	+ uint64_t last_ipis_rcvd_count;
	177	+ uint32_t icr_val;
	178	+ uint32_t icr2_val;
	179	+ uint64_t tsc_start;
	180	+
	181	+ verify_apic_base_addr();
	182	+ enable_xapic();
	183	+
	184	+ /*
	185	+ * Init interrupt command register for sending IPIs
	186	+ *
	187	+ * Delivery mode=fixed, per SDM:
	188	+ * "Delivers the interrupt specified in the vector field to the target
	189	+ * processor."
	190	+ *
	191	+ * Destination mode=physical i.e. specify target by its local APIC
	192	+ * ID. This vCPU assumes that the halter vCPU has already started and
	193	+ * set data->halter_apic_id.
	194	+ */
	195	+ icr_val = (APIC_DEST_PHYSICAL \| APIC_DM_FIXED \| IPI_VECTOR);
	196	+ icr2_val = SET_APIC_DEST_FIELD(data->halter_apic_id);
	197	+ data->icr = icr_val;
	198	+ data->icr2 = icr2_val;
	199	+
	200	+ last_wake_count = data->wake_count;
	201	+ last_hlt_count = data->hlt_count;
	202	+ last_ipis_rcvd_count = ipis_rcvd;
	203	+ for (;;) {
	204	+ /*
	205	+ * Send IPI to halter vCPU.
	206	+ * First IPI can be sent unconditionally because halter vCPU
	207	+ * starts earlier.
	208	+ */
	209	+ write_apic_reg(APIC_ICR2, icr2_val);
	210	+ write_apic_reg(APIC_ICR, icr_val);
	211	+ data->ipis_sent++;
	212	+
	213	+ /*
	214	+ * Wait up to ~1 sec for halter to indicate that it has:
	215	+ * 1. Received the IPI
	216	+ * 2. Woken up from the halt
	217	+ * 3. Gone back into halt
	218	+ * Current CPUs typically run at 2.x Ghz which is ~2
	219	+ * billion ticks per second.
	220	+ */
	221	+ tsc_start = rdtsc();
	222	+ while (rdtsc() - tsc_start < 2000000000) {
	223	+ if ((ipis_rcvd != last_ipis_rcvd_count) &&
	224	+ (data->wake_count != last_wake_count) &&
	225	+ (data->hlt_count != last_hlt_count))
	226	+ break;
	227	+ }
	228	+
	229	+ GUEST_ASSERT((ipis_rcvd != last_ipis_rcvd_count) &&
	230	+ (data->wake_count != last_wake_count) &&
	231	+ (data->hlt_count != last_hlt_count));
	232	+
	233	+ last_wake_count = data->wake_count;
	234	+ last_hlt_count = data->hlt_count;
	235	+ last_ipis_rcvd_count = ipis_rcvd;
	236	+ }
	237	+}
	238	+
	239	+static void vcpu_thread(void arg)
	240	+{
	241	+ struct thread_params params = (struct thread_params )arg;
	242	+ struct ucall uc;
	243	+ int old;
	244	+ int r;
	245	+ unsigned int exit_reason;
	246	+
	247	+ r = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old);
	248	+ TEST_ASSERT(r == 0,
	249	+ "pthread_setcanceltype failed on vcpu_id=%u with errno=%d",
	250	+ params->vcpu_id, r);
	251	+
	252	+ fprintf(stderr, "vCPU thread running vCPU %u\n", params->vcpu_id);
	253	+ vcpu_run(params->vm, params->vcpu_id);
	254	+ exit_reason = vcpu_state(params->vm, params->vcpu_id)->exit_reason;
	255	+
	256	+ TEST_ASSERT(exit_reason == KVM_EXIT_IO,
	257	+ "vCPU %u exited with unexpected exit reason %u-%s, expected KVM_EXIT_IO",
	258	+ params->vcpu_id, exit_reason, exit_reason_str(exit_reason));
	259	+
	260	+ if (get_ucall(params->vm, params->vcpu_id, &uc) == UCALL_ABORT) {
	261	+ TEST_ASSERT(false,
	262	+ "vCPU %u exited with error: %s.\n"
	263	+ "Sending vCPU sent %lu IPIs to halting vCPU\n"
	264	+ "Halting vCPU halted %lu times, woke %lu times, received %lu IPIs.\n"
	265	+ "Halter TPR=%#x PPR=%#x LVR=%#x\n"
	266	+ "Migrations attempted: %lu\n"
	267	+ "Migrations completed: %lu\n",
	268	+ params->vcpu_id, (const char *)uc.args[0],
	269	+ params->data->ipis_sent, params->data->hlt_count,
	270	+ params->data->wake_count,
	271	+ *params->pipis_rcvd, params->data->halter_tpr,
	272	+ params->data->halter_ppr, params->data->halter_lvr,
	273	+ params->data->migrations_attempted,
	274	+ params->data->migrations_completed);
	275	+ }
	276	+
	277	+ return NULL;
	278	+}
	279	+
	280	+static void cancel_join_vcpu_thread(pthread_t thread, uint32_t vcpu_id)
	281	+{
	282	+ void *retval;
	283	+ int r;
	284	+
	285	+ r = pthread_cancel(thread);
	286	+ TEST_ASSERT(r == 0,
	287	+ "pthread_cancel on vcpu_id=%d failed with errno=%d",
	288	+ vcpu_id, r);
	289	+
	290	+ r = pthread_join(thread, &retval);
	291	+ TEST_ASSERT(r == 0,
	292	+ "pthread_join on vcpu_id=%d failed with errno=%d",
	293	+ vcpu_id, r);
	294	+ TEST_ASSERT(retval == PTHREAD_CANCELED,
	295	+ "expected retval=%p, got %p", PTHREAD_CANCELED,
	296	+ retval);
	297	+}
	298	+
	299	+void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs,
	300	+ uint64_t *pipis_rcvd)
	301	+{
	302	+ long pages_not_moved;
	303	+ unsigned long nodemask = 0;
	304	+ unsigned long nodemasks[sizeof(nodemask) * 8];
	305	+ int nodes = 0;
	306	+ time_t start_time, last_update, now;
	307	+ time_t interval_secs = 1;
	308	+ int i, r;
	309	+ int from, to;
	310	+ unsigned long bit;
	311	+ uint64_t hlt_count;
	312	+ uint64_t wake_count;
	313	+ uint64_t ipis_sent;
	314	+
	315	+ fprintf(stderr, "Calling migrate_pages every %d microseconds\n",
	316	+ delay_usecs);
	317	+
	318	+ /* Get set of first 64 numa nodes available */
	319	+ r = get_mempolicy(NULL, &nodemask, sizeof(nodemask) * 8,
	320	+ 0, MPOL_F_MEMS_ALLOWED);
	321	+ TEST_ASSERT(r == 0, "get_mempolicy failed errno=%d", errno);
	322	+
	323	+ fprintf(stderr, "Numa nodes found amongst first %lu possible nodes "
	324	+ "(each 1-bit indicates node is present): %#lx\n",
	325	+ sizeof(nodemask) * 8, nodemask);
	326	+
	327	+ /* Init array of masks containing a single-bit in each, one for each
	328	+ * available node. migrate_pages called below requires specifying nodes
	329	+ * as bit masks.
	330	+ */
	331	+ for (i = 0, bit = 1; i < sizeof(nodemask) * 8; i++, bit <<= 1) {
	332	+ if (nodemask & bit) {
	333	+ nodemasks[nodes] = nodemask & bit;
	334	+ nodes++;
	335	+ }
	336	+ }
	337	+
	338	+ TEST_ASSERT(nodes > 1,
	339	+ "Did not find at least 2 numa nodes. Can't do migration\n");
	340	+
	341	+ fprintf(stderr, "Migrating amongst %d nodes found\n", nodes);
	342	+
	343	+ from = 0;
	344	+ to = 1;
	345	+ start_time = time(NULL);
	346	+ last_update = start_time;
	347	+
	348	+ ipis_sent = data->ipis_sent;
	349	+ hlt_count = data->hlt_count;
	350	+ wake_count = data->wake_count;
	351	+
	352	+ while ((int)(time(NULL) - start_time) < run_secs) {
	353	+ data->migrations_attempted++;
	354	+
	355	+ /*
	356	+ * migrate_pages with PID=0 will migrate all pages of this
	357	+ * process between the nodes specified as bitmasks. The page
	358	+ * backing the APIC access address belongs to this process
	359	+ * because it is allocated by KVM in the context of the
	360	+ * KVM_CREATE_VCPU ioctl. If that assumption ever changes this
	361	+ * test may break or give a false positive signal.
	362	+ */
	363	+ pages_not_moved = migrate_pages(0, sizeof(nodemasks[from]),
	364	+ &nodemasks[from],
	365	+ &nodemasks[to]);
	366	+ if (pages_not_moved < 0)
	367	+ fprintf(stderr,
	368	+ "migrate_pages failed, errno=%d\n", errno);
	369	+ else if (pages_not_moved > 0)
	370	+ fprintf(stderr,
	371	+ "migrate_pages could not move %ld pages\n",
	372	+ pages_not_moved);
	373	+ else
	374	+ data->migrations_completed++;
	375	+
	376	+ from = to;
	377	+ to++;
	378	+ if (to == nodes)
	379	+ to = 0;
	380	+
	381	+ now = time(NULL);
	382	+ if (((now - start_time) % interval_secs == 0) &&
	383	+ (now != last_update)) {
	384	+ last_update = now;
	385	+ fprintf(stderr,
	386	+ "%lu seconds: Migrations attempted=%lu completed=%lu, "
	387	+ "IPIs sent=%lu received=%lu, HLTs=%lu wakes=%lu\n",
	388	+ now - start_time, data->migrations_attempted,
	389	+ data->migrations_completed,
	390	+ data->ipis_sent, *pipis_rcvd,
	391	+ data->hlt_count, data->wake_count);
	392	+
	393	+ TEST_ASSERT(ipis_sent != data->ipis_sent &&
	394	+ hlt_count != data->hlt_count &&
	395	+ wake_count != data->wake_count,
	396	+ "IPI, HLT and wake count have not increased "
	397	+ "in the last %lu seconds. "
	398	+ "HLTer is likely hung.\n", interval_secs);
	399	+
	400	+ ipis_sent = data->ipis_sent;
	401	+ hlt_count = data->hlt_count;
	402	+ wake_count = data->wake_count;
	403	+ }
	404	+ usleep(delay_usecs);
	405	+ }
	406	+}
	407	+
	408	+void get_cmdline_args(int argc, char argv[], int run_secs,
	409	+ bool migrate, int delay_usecs)
	410	+{
	411	+ for (;;) {
	412	+ int opt = getopt(argc, argv, "s:d:m");
	413	+
	414	+ if (opt == -1)
	415	+ break;
	416	+ switch (opt) {
	417	+ case 's':
	418	+ *run_secs = parse_size(optarg);
	419	+ break;
	420	+ case 'm':
	421	+ *migrate = true;
	422	+ break;
	423	+ case 'd':
	424	+ *delay_usecs = parse_size(optarg);
	425	+ break;
	426	+ default:
	427	+ TEST_ASSERT(false,
	428	+ "Usage: -s <runtime seconds>. Default is %d seconds.\n"
	429	+ "-m adds calls to migrate_pages while vCPUs are running."
	430	+ " Default is no migrations.\n"
	431	+ "-d <delay microseconds> - delay between migrate_pages() calls."
	432	+ " Default is %d microseconds.\n",
	433	+ DEFAULT_RUN_SECS, DEFAULT_DELAY_USECS);
	434	+ }
	435	+ }
	436	+}
	437	+
	438	+int main(int argc, char *argv[])
	439	+{
	440	+ int r;
	441	+ int wait_secs;
	442	+ const int max_halter_wait = 10;
	443	+ int run_secs = 0;
	444	+ int delay_usecs = 0;
	445	+ struct test_data_page *data;
	446	+ vm_vaddr_t test_data_page_vaddr;
	447	+ bool migrate = false;
	448	+ pthread_t threads[2];
	449	+ struct thread_params params[2];
	450	+ struct kvm_vm *vm;
	451	+ uint64_t *pipis_rcvd;
	452	+
	453	+ get_cmdline_args(argc, argv, &run_secs, &migrate, &delay_usecs);
	454	+ if (run_secs <= 0)
	455	+ run_secs = DEFAULT_RUN_SECS;
	456	+ if (delay_usecs <= 0)
	457	+ delay_usecs = DEFAULT_DELAY_USECS;
	458	+
	459	+ vm = vm_create_default(HALTER_VCPU_ID, 0, halter_guest_code);
	460	+ params[0].vm = vm;
	461	+ params[1].vm = vm;
	462	+
	463	+ vm_init_descriptor_tables(vm);
	464	+ vcpu_init_descriptor_tables(vm, HALTER_VCPU_ID);
	465	+ vm_handle_exception(vm, IPI_VECTOR, guest_ipi_handler);
	466	+
	467	+ virt_pg_map(vm, APIC_DEFAULT_GPA, APIC_DEFAULT_GPA, 0);
	468	+
	469	+ vm_vcpu_add_default(vm, SENDER_VCPU_ID, sender_guest_code);
	470	+
	471	+ test_data_page_vaddr = vm_vaddr_alloc(vm, 0x1000, 0x1000, 0, 0);
	472	+ data =
	473	+ (struct test_data_page *)addr_gva2hva(vm, test_data_page_vaddr);
	474	+ memset(data, 0, sizeof(*data));
	475	+ params[0].data = data;
	476	+ params[1].data = data;
	477	+
	478	+ vcpu_args_set(vm, HALTER_VCPU_ID, 1, test_data_page_vaddr);
	479	+ vcpu_args_set(vm, SENDER_VCPU_ID, 1, test_data_page_vaddr);
	480	+
	481	+ pipis_rcvd = (uint64_t *)addr_gva2hva(vm, (uint64_t)&ipis_rcvd);
	482	+ params[0].pipis_rcvd = pipis_rcvd;
	483	+ params[1].pipis_rcvd = pipis_rcvd;
	484	+
	485	+ /* Start halter vCPU thread and wait for it to execute first HLT. */
	486	+ params[0].vcpu_id = HALTER_VCPU_ID;
	487	+ r = pthread_create(&threads[0], NULL, vcpu_thread, &params[0]);
	488	+ TEST_ASSERT(r == 0,
	489	+ "pthread_create halter failed errno=%d", errno);
	490	+ fprintf(stderr, "Halter vCPU thread started\n");
	491	+
	492	+ wait_secs = 0;
	493	+ while ((wait_secs < max_halter_wait) && !data->hlt_count) {
	494	+ sleep(1);
	495	+ wait_secs++;
	496	+ }
	497	+
	498	+ TEST_ASSERT(data->hlt_count,
	499	+ "Halter vCPU did not execute first HLT within %d seconds",
	500	+ max_halter_wait);
	501	+
	502	+ fprintf(stderr,
	503	+ "Halter vCPU thread reported its APIC ID: %u after %d seconds.\n",
	504	+ data->halter_apic_id, wait_secs);
	505	+
	506	+ params[1].vcpu_id = SENDER_VCPU_ID;
	507	+ r = pthread_create(&threads[1], NULL, vcpu_thread, &params[1]);
	508	+ TEST_ASSERT(r == 0, "pthread_create sender failed errno=%d", errno);
	509	+
	510	+ fprintf(stderr,
	511	+ "IPI sender vCPU thread started. Letting vCPUs run for %d seconds.\n",
	512	+ run_secs);
	513	+
	514	+ if (!migrate)
	515	+ sleep(run_secs);
	516	+ else
	517	+ do_migrations(data, run_secs, delay_usecs, pipis_rcvd);
	518	+
	519	+ /*
	520	+ * Cancel threads and wait for them to stop.
	521	+ */
	522	+ cancel_join_vcpu_thread(threads[0], HALTER_VCPU_ID);
	523	+ cancel_join_vcpu_thread(threads[1], SENDER_VCPU_ID);
	524	+
	525	+ fprintf(stderr,
	526	+ "Test successful after running for %d seconds.\n"
	527	+ "Sending vCPU sent %lu IPIs to halting vCPU\n"
	528	+ "Halting vCPU halted %lu times, woke %lu times, received %lu IPIs.\n"
	529	+ "Halter APIC ID=%#x\n"
	530	+ "Sender ICR value=%#x ICR2 value=%#x\n"
	531	+ "Halter TPR=%#x PPR=%#x LVR=%#x\n"
	532	+ "Migrations attempted: %lu\n"
	533	+ "Migrations completed: %lu\n",
	534	+ run_secs, data->ipis_sent,
	535	+ data->hlt_count, data->wake_count, *pipis_rcvd,
	536	+ data->halter_apic_id,
	537	+ data->icr, data->icr2,
	538	+ data->halter_tpr, data->halter_ppr, data->halter_lvr,
	539	+ data->migrations_attempted, data->migrations_completed);
	540	+
	541	+ kvm_vm_free(vm);
	542	+
	543	+ return 0;
	544	+}

tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c+167−0

@@ -0,0 +1,167 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/*
	3	+ * svm_vmcall_test
	4	+ *
	5	+ * Copyright © 2021 Amazon.com, Inc. or its affiliates.
	6	+ *
	7	+ * Xen shared_info / pvclock testing
	8	+ */
	9	+
	10	+#include "test_util.h"
	11	+#include "kvm_util.h"
	12	+#include "processor.h"
	13	+
	14	+#include <stdint.h>
	15	+#include <time.h>
	16	+
	17	+#define VCPU_ID 5
	18	+
	19	+#define SHINFO_REGION_GPA 0xc0000000ULL
	20	+#define SHINFO_REGION_SLOT 10
	21	+#define PAGE_SIZE 4096
	22	+
	23	+#define PVTIME_ADDR (SHINFO_REGION_GPA + PAGE_SIZE)
	24	+
	25	+static struct kvm_vm *vm;
	26	+
	27	+#define XEN_HYPERCALL_MSR 0x40000000
	28	+
	29	+struct pvclock_vcpu_time_info {
	30	+ u32 version;
	31	+ u32 pad0;
	32	+ u64 tsc_timestamp;
	33	+ u64 system_time;
	34	+ u32 tsc_to_system_mul;
	35	+ s8 tsc_shift;
	36	+ u8 flags;
	37	+ u8 pad[2];
	38	+} __attribute__((__packed__)); /* 32 bytes */
	39	+
	40	+struct pvclock_wall_clock {
	41	+ u32 version;
	42	+ u32 sec;
	43	+ u32 nsec;
	44	+} __attribute__((__packed__));
	45	+
	46	+static void guest_code(void)
	47	+{
	48	+ GUEST_DONE();
	49	+}
	50	+
	51	+static int cmp_timespec(struct timespec a, struct timespec b)
	52	+{
	53	+ if (a->tv_sec > b->tv_sec)
	54	+ return 1;
	55	+ else if (a->tv_sec < b->tv_sec)
	56	+ return -1;
	57	+ else if (a->tv_nsec > b->tv_nsec)
	58	+ return 1;
	59	+ else if (a->tv_nsec < b->tv_nsec)
	60	+ return -1;
	61	+ else
	62	+ return 0;
	63	+}
	64	+
	65	+int main(int argc, char *argv[])
	66	+{
	67	+ struct timespec min_ts, max_ts, vm_ts;
	68	+
	69	+ if (!(kvm_check_cap(KVM_CAP_XEN_HVM) &
	70	+ KVM_XEN_HVM_CONFIG_SHARED_INFO) ) {
	71	+ print_skip("KVM_XEN_HVM_CONFIG_SHARED_INFO not available");
	72	+ exit(KSFT_SKIP);
	73	+ }
	74	+
	75	+ clock_gettime(CLOCK_REALTIME, &min_ts);
	76	+
	77	+ vm = vm_create_default(VCPU_ID, 0, (void *) guest_code);
	78	+ vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid());
	79	+
	80	+ /* Map a region for the shared_info page */
	81	+ vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
	82	+ SHINFO_REGION_GPA, SHINFO_REGION_SLOT, 2, 0);
	83	+
	84	+ struct kvm_xen_hvm_config hvmc = {
	85	+ .flags = KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL,
	86	+ .msr = XEN_HYPERCALL_MSR,
	87	+ };
	88	+ vm_ioctl(vm, KVM_XEN_HVM_CONFIG, &hvmc);
	89	+
	90	+ struct kvm_xen_hvm_attr lm = {
	91	+ .type = KVM_XEN_ATTR_TYPE_LONG_MODE,
	92	+ .u.long_mode = 1,
	93	+ };
	94	+ vm_ioctl(vm, KVM_XEN_HVM_SET_ATTR, &lm);
	95	+
	96	+ struct kvm_xen_hvm_attr ha = {
	97	+ .type = KVM_XEN_ATTR_TYPE_SHARED_INFO,
	98	+ .u.shared_info.gfn = SHINFO_REGION_GPA / PAGE_SIZE,
	99	+ };
	100	+ vm_ioctl(vm, KVM_XEN_HVM_SET_ATTR, &ha);
	101	+
	102	+ struct kvm_xen_vcpu_attr vi = {
	103	+ .type = KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO,
	104	+ .u.gpa = SHINFO_REGION_GPA + 0x40,
	105	+ };
	106	+ vcpu_ioctl(vm, VCPU_ID, KVM_XEN_VCPU_SET_ATTR, &vi);
	107	+
	108	+ struct kvm_xen_vcpu_attr pvclock = {
	109	+ .type = KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO,
	110	+ .u.gpa = PVTIME_ADDR,
	111	+ };
	112	+ vcpu_ioctl(vm, VCPU_ID, KVM_XEN_VCPU_SET_ATTR, &pvclock);
	113	+
	114	+ for (;;) {
	115	+ volatile struct kvm_run *run = vcpu_state(vm, VCPU_ID);
	116	+ struct ucall uc;
	117	+
	118	+ vcpu_run(vm, VCPU_ID);
	119	+
	120	+ TEST_ASSERT(run->exit_reason == KVM_EXIT_IO,
	121	+ "Got exit_reason other than KVM_EXIT_IO: %u (%s)\n",
	122	+ run->exit_reason,
	123	+ exit_reason_str(run->exit_reason));
	124	+
	125	+ switch (get_ucall(vm, VCPU_ID, &uc)) {
	126	+ case UCALL_ABORT:
	127	+ TEST_FAIL("%s", (const char *)uc.args[0]);
	128	+ /* NOT REACHED */
	129	+ case UCALL_SYNC:
	130	+ break;
	131	+ case UCALL_DONE:
	132	+ goto done;
	133	+ default:
	134	+ TEST_FAIL("Unknown ucall 0x%lx.", uc.cmd);
	135	+ }
	136	+ }
	137	+
	138	+ done:
	139	+ clock_gettime(CLOCK_REALTIME, &max_ts);
	140	+
	141	+ /*
	142	+ * Just a really basic check that things are being put in the
	143	+ * right place. The actual calculations are much the same for
	144	+ * Xen as they are for the KVM variants, so no need to check.
	145	+ */
	146	+ struct pvclock_wall_clock *wc;
	147	+ struct pvclock_vcpu_time_info ti, ti2;
	148	+
	149	+ wc = addr_gpa2hva(vm, SHINFO_REGION_GPA + 0xc00);
	150	+ ti = addr_gpa2hva(vm, SHINFO_REGION_GPA + 0x40 + 0x20);
	151	+ ti2 = addr_gpa2hva(vm, PVTIME_ADDR);
	152	+
	153	+ vm_ts.tv_sec = wc->sec;
	154	+ vm_ts.tv_nsec = wc->nsec;
	155	+ TEST_ASSERT(wc->version && !(wc->version & 1),
	156	+ "Bad wallclock version %x", wc->version);
	157	+ TEST_ASSERT(cmp_timespec(&min_ts, &vm_ts) <= 0, "VM time too old");
	158	+ TEST_ASSERT(cmp_timespec(&max_ts, &vm_ts) >= 0, "VM time too new");
	159	+
	160	+ TEST_ASSERT(ti->version && !(ti->version & 1),
	161	+ "Bad time_info version %x", ti->version);
	162	+ TEST_ASSERT(ti2->version && !(ti2->version & 1),
	163	+ "Bad time_info version %x", ti->version);
	164	+
	165	+ kvm_vm_free(vm);
	166	+ return 0;
	167	+}

tools/testing/selftests/kvm/x86_64/xen_vmcall_test.c+149−0

@@ -0,0 +1,149 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/*
	3	+ * xen_vmcall_test
	4	+ *
	5	+ * Copyright © 2020 Amazon.com, Inc. or its affiliates.
	6	+ *
	7	+ * Userspace hypercall testing
	8	+ */
	9	+
	10	+#include "test_util.h"
	11	+#include "kvm_util.h"
	12	+#include "processor.h"
	13	+
	14	+#define VCPU_ID 5
	15	+
	16	+#define HCALL_REGION_GPA 0xc0000000ULL
	17	+#define HCALL_REGION_SLOT 10
	18	+#define PAGE_SIZE 4096
	19	+
	20	+static struct kvm_vm *vm;
	21	+
	22	+#define INPUTVALUE 17
	23	+#define ARGVALUE(x) (0xdeadbeef5a5a0000UL + x)
	24	+#define RETVALUE 0xcafef00dfbfbffffUL
	25	+
	26	+#define XEN_HYPERCALL_MSR 0x40000200
	27	+#define HV_GUEST_OS_ID_MSR 0x40000000
	28	+#define HV_HYPERCALL_MSR 0x40000001
	29	+
	30	+#define HVCALL_SIGNAL_EVENT 0x005d
	31	+#define HV_STATUS_INVALID_ALIGNMENT 4
	32	+
	33	+static void guest_code(void)
	34	+{
	35	+ unsigned long rax = INPUTVALUE;
	36	+ unsigned long rdi = ARGVALUE(1);
	37	+ unsigned long rsi = ARGVALUE(2);
	38	+ unsigned long rdx = ARGVALUE(3);
	39	+ unsigned long rcx;
	40	+ register unsigned long r10 __asm__("r10") = ARGVALUE(4);
	41	+ register unsigned long r8 __asm__("r8") = ARGVALUE(5);
	42	+ register unsigned long r9 __asm__("r9") = ARGVALUE(6);
	43	+
	44	+ /* First a direct invocation of 'vmcall' */
	45	+ __asm__ __volatile__("vmcall" :
	46	+ "=a"(rax) :
	47	+ "a"(rax), "D"(rdi), "S"(rsi), "d"(rdx),
	48	+ "r"(r10), "r"(r8), "r"(r9));
	49	+ GUEST_ASSERT(rax == RETVALUE);
	50	+
	51	+ /* Fill in the Xen hypercall page */
	52	+ __asm__ __volatile__("wrmsr" : : "c" (XEN_HYPERCALL_MSR),
	53	+ "a" (HCALL_REGION_GPA & 0xffffffff),
	54	+ "d" (HCALL_REGION_GPA >> 32));
	55	+
	56	+ /* Set Hyper-V Guest OS ID */
	57	+ __asm__ __volatile__("wrmsr" : : "c" (HV_GUEST_OS_ID_MSR),
	58	+ "a" (0x5a), "d" (0));
	59	+
	60	+ /* Hyper-V hypercall page */
	61	+ u64 msrval = HCALL_REGION_GPA + PAGE_SIZE + 1;
	62	+ __asm__ __volatile__("wrmsr" : : "c" (HV_HYPERCALL_MSR),
	63	+ "a" (msrval & 0xffffffff),
	64	+ "d" (msrval >> 32));
	65	+
	66	+ /* Invoke a Xen hypercall */
	67	+ __asm__ __volatile__("call *%1" : "=a"(rax) :
	68	+ "r"(HCALL_REGION_GPA + INPUTVALUE * 32),
	69	+ "a"(rax), "D"(rdi), "S"(rsi), "d"(rdx),
	70	+ "r"(r10), "r"(r8), "r"(r9));
	71	+ GUEST_ASSERT(rax == RETVALUE);
	72	+
	73	+ /* Invoke a Hyper-V hypercall */
	74	+ rax = 0;
	75	+ rcx = HVCALL_SIGNAL_EVENT; /* code */
	76	+ rdx = 0x5a5a5a5a; /* ingpa (badly aligned) */
	77	+ __asm__ __volatile__("call *%1" : "=a"(rax) :
	78	+ "r"(HCALL_REGION_GPA + PAGE_SIZE),
	79	+ "a"(rax), "c"(rcx), "d"(rdx),
	80	+ "r"(r8));
	81	+ GUEST_ASSERT(rax == HV_STATUS_INVALID_ALIGNMENT);
	82	+
	83	+ GUEST_DONE();
	84	+}
	85	+
	86	+int main(int argc, char *argv[])
	87	+{
	88	+ if (!(kvm_check_cap(KVM_CAP_XEN_HVM) &
	89	+ KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL) ) {
	90	+ print_skip("KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL not available");
	91	+ exit(KSFT_SKIP);
	92	+ }
	93	+
	94	+ vm = vm_create_default(VCPU_ID, 0, (void *) guest_code);
	95	+ vcpu_set_hv_cpuid(vm, VCPU_ID);
	96	+
	97	+ struct kvm_xen_hvm_config hvmc = {
	98	+ .flags = KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL,
	99	+ .msr = XEN_HYPERCALL_MSR,
	100	+ };
	101	+ vm_ioctl(vm, KVM_XEN_HVM_CONFIG, &hvmc);
	102	+
	103	+ /* Map a region for the hypercall pages */
	104	+ vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
	105	+ HCALL_REGION_GPA, HCALL_REGION_SLOT, 2, 0);
	106	+ virt_map(vm, HCALL_REGION_GPA, HCALL_REGION_GPA, 2, 0);
	107	+
	108	+ for (;;) {
	109	+ volatile struct kvm_run *run = vcpu_state(vm, VCPU_ID);
	110	+ struct ucall uc;
	111	+
	112	+ vcpu_run(vm, VCPU_ID);
	113	+
	114	+ if (run->exit_reason == KVM_EXIT_XEN) {
	115	+ ASSERT_EQ(run->xen.type, KVM_EXIT_XEN_HCALL);
	116	+ ASSERT_EQ(run->xen.u.hcall.cpl, 0);
	117	+ ASSERT_EQ(run->xen.u.hcall.longmode, 1);
	118	+ ASSERT_EQ(run->xen.u.hcall.input, INPUTVALUE);
	119	+ ASSERT_EQ(run->xen.u.hcall.params[0], ARGVALUE(1));
	120	+ ASSERT_EQ(run->xen.u.hcall.params[1], ARGVALUE(2));
	121	+ ASSERT_EQ(run->xen.u.hcall.params[2], ARGVALUE(3));
	122	+ ASSERT_EQ(run->xen.u.hcall.params[3], ARGVALUE(4));
	123	+ ASSERT_EQ(run->xen.u.hcall.params[4], ARGVALUE(5));
	124	+ ASSERT_EQ(run->xen.u.hcall.params[5], ARGVALUE(6));
	125	+ run->xen.u.hcall.result = RETVALUE;
	126	+ continue;
	127	+ }
	128	+
	129	+ TEST_ASSERT(run->exit_reason == KVM_EXIT_IO,
	130	+ "Got exit_reason other than KVM_EXIT_IO: %u (%s)\n",
	131	+ run->exit_reason,
	132	+ exit_reason_str(run->exit_reason));
	133	+
	134	+ switch (get_ucall(vm, VCPU_ID, &uc)) {
	135	+ case UCALL_ABORT:
	136	+ TEST_FAIL("%s", (const char *)uc.args[0]);
	137	+ /* NOT REACHED */
	138	+ case UCALL_SYNC:
	139	+ break;
	140	+ case UCALL_DONE:
	141	+ goto done;
	142	+ default:
	143	+ TEST_FAIL("Unknown ucall 0x%lx.", uc.cmd);
	144	+ }
	145	+ }
	146	+done:
	147	+ kvm_vm_free(vm);
	148	+ return 0;
	149	+}

virt/kvm/dirty_ring.c+4−4

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	9	#include <linux/vmalloc.h>
10	10	#include <linux/kvm_dirty_ring.h>
11	11	#include <trace/events/kvm.h>
	12	+#include "mmu_lock.h"
12	13
13	14	int __weak kvm_cpu_dirty_log_size(void)
14	15	{
@@ -60,17 +61,16 @@ static void kvm_reset_dirty_gfn(struct kvm *kvm, u32 slot, u64 offset, u64 mask) · 45 unmodified lines
60	61	if (!memslot \|\| (offset + __fls(mask)) >= memslot->npages)
61	62	return;
62	63
63		− spin_lock(&kvm->mmu_lock);
	64	+ KVM_MMU_LOCK(kvm);
64	65	kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, offset, mask);
65		− spin_unlock(&kvm->mmu_lock);
	66	+ KVM_MMU_UNLOCK(kvm);
66	67	}
67	68
68	69	int kvm_dirty_ring_alloc(struct kvm_dirty_ring *ring, int index, u32 size)
69	70	{
70		− ring->dirty_gfns = vmalloc(size);
	71	+ ring->dirty_gfns = vzalloc(size);
71	72	if (!ring->dirty_gfns)
72	73	return -ENOMEM;
73		− memset(ring->dirty_gfns, 0, size);
74	74
75	75	ring->size = size / sizeof(struct kvm_dirty_gfn);
76	76	ring->soft_limit = ring->size - kvm_dirty_ring_get_rsvd_entries();

virt/kvm/kvm_main.c+33−21

@@ -58,6 +58,7 @@ · 57 unmodified lines
58	58
59	59	#include "coalesced_mmio.h"
60	60	#include "async_pf.h"
	61	+#include "mmu_lock.h"
61	62	#include "vfio.h"
62	63
63	64	#define CREATE_TRACE_POINTS
@@ -459,13 +460,15 @@ static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, · 395 unmodified lines
459	460	int idx;
460	461
461	462	idx = srcu_read_lock(&kvm->srcu);
462		− spin_lock(&kvm->mmu_lock);
	463	+
	464	+ KVM_MMU_LOCK(kvm);
	465	+
463	466	kvm->mmu_notifier_seq++;
464	467
465	468	if (kvm_set_spte_hva(kvm, address, pte))
466	469	kvm_flush_remote_tlbs(kvm);
467	470
468		− spin_unlock(&kvm->mmu_lock);
	471	+ KVM_MMU_UNLOCK(kvm);
469	472	srcu_read_unlock(&kvm->srcu, idx);
470	473	}
471	474
@@ -476,7 +479,7 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, · 4 unmodified lines
476	479	int need_tlb_flush = 0, idx;
477	480
478	481	idx = srcu_read_lock(&kvm->srcu);
479		− spin_lock(&kvm->mmu_lock);
	482	+ KVM_MMU_LOCK(kvm);
480	483	/*
481	484	* The count increase must become visible at unlock time as no
482	485	* spte can be established without taking the mmu_lock and
@@ -489,7 +492,7 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, · 6 unmodified lines
489	492	if (need_tlb_flush \|\| kvm->tlbs_dirty)
490	493	kvm_flush_remote_tlbs(kvm);
491	494
492		− spin_unlock(&kvm->mmu_lock);
	495	+ KVM_MMU_UNLOCK(kvm);
493	496	srcu_read_unlock(&kvm->srcu, idx);
494	497
495	498	return 0;
@@ -500,7 +503,7 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, · 4 unmodified lines
500	503	{
501	504	struct kvm *kvm = mmu_notifier_to_kvm(mn);
502	505
503		− spin_lock(&kvm->mmu_lock);
	506	+ KVM_MMU_LOCK(kvm);
504	507	/*
505	508	* This sequence increase will notify the kvm page fault that
506	509	* the page that is going to be mapped in the spte could have
@@ -514,7 +517,7 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, · 7 unmodified lines
514	517	* in conjunction with the smp_rmb in mmu_notifier_retry().
515	518	*/
516	519	kvm->mmu_notifier_count--;
517		− spin_unlock(&kvm->mmu_lock);
	520	+ KVM_MMU_UNLOCK(kvm);
518	521
519	522	BUG_ON(kvm->mmu_notifier_count < 0);
520	523	}
@@ -528,13 +531,13 @@ static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, · 7 unmodified lines
528	531	int young, idx;
529	532
530	533	idx = srcu_read_lock(&kvm->srcu);
531		− spin_lock(&kvm->mmu_lock);
	534	+ KVM_MMU_LOCK(kvm);
532	535
533	536	young = kvm_age_hva(kvm, start, end);
534	537	if (young)
535	538	kvm_flush_remote_tlbs(kvm);
536	539
537		− spin_unlock(&kvm->mmu_lock);
	540	+ KVM_MMU_UNLOCK(kvm);
538	541	srcu_read_unlock(&kvm->srcu, idx);
539	542
540	543	return young;
@@ -549,7 +552,7 @@ static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn, · 8 unmodified lines
549	552	int young, idx;
550	553
551	554	idx = srcu_read_lock(&kvm->srcu);
552		− spin_lock(&kvm->mmu_lock);
	555	+ KVM_MMU_LOCK(kvm);
553	556	/*
554	557	* Even though we do not flush TLB, this will still adversely
555	558	* affect performance on pre-Haswell Intel EPT, where there is
@@ -564,7 +567,7 @@ static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn, · 8 unmodified lines
564	567	* more sophisticated heuristic later.
565	568	*/
566	569	young = kvm_age_hva(kvm, start, end);
567		− spin_unlock(&kvm->mmu_lock);
	570	+ KVM_MMU_UNLOCK(kvm);
568	571	srcu_read_unlock(&kvm->srcu, idx);
569	572
570	573	return young;
@@ -578,9 +581,9 @@ static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, · 7 unmodified lines
578	581	int young, idx;
579	582
580	583	idx = srcu_read_lock(&kvm->srcu);
581		− spin_lock(&kvm->mmu_lock);
	584	+ KVM_MMU_LOCK(kvm);
582	585	young = kvm_test_age_hva(kvm, address);
583		− spin_unlock(&kvm->mmu_lock);
	586	+ KVM_MMU_UNLOCK(kvm);
584	587	srcu_read_unlock(&kvm->srcu, idx);
585	588
586	589	return young;
@@ -745,7 +748,7 @@ static struct kvm *kvm_create_vm(unsigned long type) · 158 unmodified lines
745	748	if (!kvm)
746	749	return ERR_PTR(-ENOMEM);
747	750
748		− spin_lock_init(&kvm->mmu_lock);
	751	+ KVM_MMU_LOCK_INIT(kvm);
749	752	mmgrab(current->mm);
750	753	kvm->mm = current->mm;
751	754	kvm_eventfd_init(kvm);
@@ -1525,7 +1528,7 @@ static int kvm_get_dirty_log_protect(struct kvm kvm, struct kvm_dirty_log log) · 773 unmodified lines
1525	1528	dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
1526	1529	memset(dirty_bitmap_buffer, 0, n);
1527	1530
1528		− spin_lock(&kvm->mmu_lock);
	1531	+ KVM_MMU_LOCK(kvm);
1529	1532	for (i = 0; i < n / sizeof(long); i++) {
1530	1533	unsigned long mask;
1531	1534	gfn_t offset;
@@ -1541,7 +1544,7 @@ static int kvm_get_dirty_log_protect(struct kvm kvm, struct kvm_dirty_log log) · 9 unmodified lines
1541	1544	kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
1542	1545	offset, mask);
1543	1546	}
1544		− spin_unlock(&kvm->mmu_lock);
	1547	+ KVM_MMU_UNLOCK(kvm);
1545	1548	}
1546	1549
1547	1550	if (flush)
@@ -1636,7 +1639,7 @@ static int kvm_clear_dirty_log_protect(struct kvm *kvm, · 88 unmodified lines
1636	1639	if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n))
1637	1640	return -EFAULT;
1638	1641
1639		− spin_lock(&kvm->mmu_lock);
	1642	+ KVM_MMU_LOCK(kvm);
1640	1643	for (offset = log->first_page, i = offset / BITS_PER_LONG,
1641	1644	n = DIV_ROUND_UP(log->num_pages, BITS_PER_LONG); n--;
1642	1645	i++, offset += BITS_PER_LONG) {
@@ -1659,7 +1662,7 @@ static int kvm_clear_dirty_log_protect(struct kvm *kvm, · 16 unmodified lines
1659	1662	offset, mask);
1660	1663	}
1661	1664	}
1662		− spin_unlock(&kvm->mmu_lock);
	1665	+ KVM_MMU_UNLOCK(kvm);
1663	1666
1664	1667	if (flush)
1665	1668	kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);
@@ -1903,10 +1906,12 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, · 237 unmodified lines
1903	1906	bool write_fault, bool *writable,
1904	1907	kvm_pfn_t *p_pfn)
1905	1908	{
1906		− unsigned long pfn;
	1909	+ kvm_pfn_t pfn;
	1910	+ pte_t *ptep;
	1911	+ spinlock_t *ptl;
1907	1912	int r;
1908	1913
1909		− r = follow_pfn(vma, addr, &pfn);
	1914	+ r = follow_pte(vma->vm_mm, addr, &ptep, &ptl);
1910	1915	if (r) {
1911	1916	/*
1912	1917	* get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
@@ -1921,14 +1926,19 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, · 8 unmodified lines
1921	1926	if (r)
1922	1927	return r;
1923	1928
1924		− r = follow_pfn(vma, addr, &pfn);
	1929	+ r = follow_pte(vma->vm_mm, addr, &ptep, &ptl);
1925	1930	if (r)
1926	1931	return r;
	1932	+ }
1927	1933
	1934	+ if (write_fault && !pte_write(*ptep)) {
	1935	+ pfn = KVM_PFN_ERR_RO_FAULT;
	1936	+ goto out;
1928	1937	}
1929	1938
1930	1939	if (writable)
1931		− *writable = true;
	1940	+ writable = pte_write(ptep);
	1941	+ pfn = pte_pfn(*ptep);
1932	1942
1933	1943	/*
1934	1944	* Get a reference here because callers of hva_to_pfn and
@@ -1943,6 +1953,8 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, · 8 unmodified lines
1943	1953	*/
1944	1954	kvm_get_pfn(pfn);
1945	1955
	1956	+out:
	1957	+ pte_unmap_unlock(ptep, ptl);
1946	1958	*p_pfn = pfn;
1947	1959	return 0;
1948	1960	}

virt/kvm/mmu_lock.h+23−0

@@ -0,0 +1,23 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+
	3	+#ifndef KVM_MMU_LOCK_H
	4	+#define KVM_MMU_LOCK_H 1
	5	+
	6	+/*
	7	+ * Architectures can choose whether to use an rwlock or spinlock
	8	+ * for the mmu_lock. These macros, for use in common code
	9	+ * only, avoids using #ifdefs in places that must deal with
	10	+ * multiple architectures.
	11	+ */
	12	+
	13	+#ifdef KVM_HAVE_MMU_RWLOCK
	14	+#define KVM_MMU_LOCK_INIT(kvm) rwlock_init(&(kvm)->mmu_lock)
	15	+#define KVM_MMU_LOCK(kvm) write_lock(&(kvm)->mmu_lock)
	16	+#define KVM_MMU_UNLOCK(kvm) write_unlock(&(kvm)->mmu_lock)
	17	+#else
	18	+#define KVM_MMU_LOCK_INIT(kvm) spin_lock_init(&(kvm)->mmu_lock)
	19	+#define KVM_MMU_LOCK(kvm) spin_lock(&(kvm)->mmu_lock)
	20	+#define KVM_MMU_UNLOCK(kvm) spin_unlock(&(kvm)->mmu_lock)
	21	+#endif /* KVM_HAVE_MMU_RWLOCK */
	22	+
	23	+#endif