Merge branch 'kvm-tdx-initial' into HEAD

This large commit contains the initial support for TDX in KVM. All x86 parts enable the host-side hypercalls that KVM uses to talk to the TDX module, a software component that runs in a special CPU mode called SEAM (Secure Arbitration Mode). The series is in turn split into multiple sub-series, each with a separate merge commit: - Initialization: basic setup for using the TDX module from KVM, plus ioctls to create TDX VMs and vCPUs. - MMU: in TDX, private and shared halves of the address space are mapped by different EPT roots, and the private half is managed by the TDX module. Using the support that was added to the generic MMU code in 6.14, add support for TDX's secure page tables to the Intel side of KVM. Generic KVM code takes care of maintaining a mirror of the secure page tables so that they can be queried efficiently, and ensuring that changes are applied to both the mirror and the secure EPT. - vCPU enter/exit: implement the callbacks that handle the entry of a TDX vCPU (via the SEAMCALL TDH.VP.ENTER) and the corresponding save/restore of host state. - Userspace exits: introduce support for guest TDVMCALLs that KVM forwards to userspace. These correspond to the usual KVM_EXIT_* "heavyweight vmexits" but are triggered through a different mechanism, similar to VMGEXIT for SEV-ES and SEV-SNP. - Interrupt handling: support for virtual interrupt injection as well as handling VM-Exits that are caused by vectored events. Exclusive to TDX are machine-check SMIs, which the kernel already knows how to handle through the kernel machine check handler (commit 7911f145de5f, "x86/mce: Implement recovery for errors in TDX/SEAM non-root mode") - Loose ends: handling of the remaining exits from the TDX module, including EPT violation/misconfig and several TDVMCALL leaves that are handled in the kernel (CPUID, HLT, RDMSR/WRMSR, GetTdVmCallInfo); plus returning an error or ignoring operations that are not supported by TDX guests

Paolo Bonzini committed 1 year ago

Commit fd02aa42 parents 7d76856 7bcf72455 files · +6822 −583Browse files

Unified · Split

55 files

Documentation/virt/kvm/api.rst +37 −4
Documentation/virt/kvm/x86/index.rst +1 −0
Documentation/virt/kvm/x86/intel-tdx.rst +255 −0
arch/x86/include/asm/kvm-x86-ops.h +4 −1
arch/x86/include/asm/kvm_host.h +26 −8
arch/x86/include/asm/posted_intr.h +5 −0
arch/x86/include/asm/shared/tdx.h +8 −1
arch/x86/include/asm/tdx.h +75 −0
arch/x86/include/asm/tdx_global_metadata.h +19 −0
arch/x86/include/asm/vmx.h +2 −0
arch/x86/include/uapi/asm/kvm.h +71 −0
arch/x86/include/uapi/asm/vmx.h +4 −1
arch/x86/kvm/Kconfig +12 −0
arch/x86/kvm/Makefile +1 −0
arch/x86/kvm/cpuid.c +19 −33
arch/x86/kvm/cpuid.h +29 −4
arch/x86/kvm/irq.c +3 −0
arch/x86/kvm/lapic.c +14 −1
arch/x86/kvm/lapic.h +2 −0
arch/x86/kvm/mmu.h +5 −1
arch/x86/kvm/mmu/mmu.c +20 −19
arch/x86/kvm/mmu/mmu_internal.h +3 −2
arch/x86/kvm/mmu/page_track.c +3 −0
arch/x86/kvm/mmu/spte.c +9 −20
arch/x86/kvm/mmu/spte.h +0 −1
arch/x86/kvm/mmu/tdp_mmu.c +38 −11
arch/x86/kvm/smm.h +3 −0
arch/x86/kvm/svm/svm.c +1 −0
arch/x86/kvm/vmx/common.h +182 −0
arch/x86/kvm/vmx/main.c +1034 −91
arch/x86/kvm/vmx/nested.c +6 −6
arch/x86/kvm/vmx/pmu_intel.c +51 −1
arch/x86/kvm/vmx/pmu_intel.h +28 −0
arch/x86/kvm/vmx/posted_intr.c +16 −12
arch/x86/kvm/vmx/posted_intr.h +2 −0
arch/x86/kvm/vmx/tdx.c +3526 −0
arch/x86/kvm/vmx/tdx.h +204 −0
arch/x86/kvm/vmx/tdx_arch.h +167 −0
arch/x86/kvm/vmx/tdx_errno.h +40 −0
arch/x86/kvm/vmx/vmx.c +111 −180
arch/x86/kvm/vmx/vmx.h +43 −97
arch/x86/kvm/vmx/x86_ops.h +110 −1
arch/x86/kvm/x86.c +68 −31
arch/x86/kvm/x86.h +14 −17
arch/x86/virt/vmx/tdx/seamcall.S +3 −0
arch/x86/virt/vmx/tdx/tdx.c +418 −5
arch/x86/virt/vmx/tdx/tdx.h +40 −8
arch/x86/virt/vmx/tdx/tdx_global_metadata.c +50 −0
include/linux/kvm_dirty_ring.h +6 −5
include/linux/kvm_host.h +10 −0
include/linux/misc_cgroup.h +4 −0
include/uapi/linux/kvm.h +1 −0
kernel/cgroup/misc.c +4 −0
virt/kvm/dirty_ring.c +6 −5
virt/kvm/kvm_main.c +9 −17

Documentation/virt/kvm/api.rst+37−4

@@ -1411,6 +1411,9 @@ the memory region are automatically reflected into the guest. For example, an · 1410 unmodified lines
1411	1411	mmap() that affects the region will be made visible immediately. Another
1412	1412	example is madvise(MADV_DROP).
1413	1413
	1414	+For TDX guest, deleting/moving memory region loses guest memory contents.
	1415	+Read only region isn't supported. Only as-id 0 is supported.
	1416	+
1414	1417	Note: On arm64, a write generated by the page-table walker (to update
1415	1418	the Access and Dirty flags, for example) never results in a
1416	1419	KVM_EXIT_MMIO exit when the slot has the KVM_MEM_READONLY flag. This
@@ -4768,7 +4771,7 @@ H_GET_CPU_CHARACTERISTICS hypercall. · 3351 unmodified lines
4768	4771
4769	4772	:Capability: basic
4770	4773	:Architectures: x86
4771		−:Type: vm
	4774	+:Type: vm ioctl, vcpu ioctl
4772	4775	:Parameters: an opaque platform specific structure (in/out)
4773	4776	:Returns: 0 on success; -1 on error
4774	4777
@@ -4776,9 +4779,11 @@ If the platform supports creating encrypted VMs then this ioctl can be used · 1 unmodified line
4776	4779	for issuing platform-specific memory encryption commands to manage those
4777	4780	encrypted VMs.
4778	4781
4779		−Currently, this ioctl is used for issuing Secure Encrypted Virtualization
4780		−(SEV) commands on AMD Processors. The SEV commands are defined in
4781		−Documentation/virt/kvm/x86/amd-memory-encryption.rst.
	4782	+Currently, this ioctl is used for issuing both Secure Encrypted Virtualization
	4783	+(SEV) commands on AMD Processors and Trusted Domain Extensions (TDX) commands
	4784	+on Intel Processors. The detailed commands are defined in
	4785	+Documentation/virt/kvm/x86/amd-memory-encryption.rst and
	4786	+Documentation/virt/kvm/x86/intel-tdx.rst.
4782	4787
4783	4788	4.111 KVM_MEMORY_ENCRYPT_REG_REGION
4784	4789	-----------------------------------
@@ -6827,6 +6832,7 @@ should put the acknowledged interrupt vector into the 'epr' field. · 2042 unmodified lines
6827	6832	#define KVM_SYSTEM_EVENT_WAKEUP 4
6828	6833	#define KVM_SYSTEM_EVENT_SUSPEND 5
6829	6834	#define KVM_SYSTEM_EVENT_SEV_TERM 6
	6835	+ #define KVM_SYSTEM_EVENT_TDX_FATAL 7
6830	6836	__u32 type;
6831	6837	__u32 ndata;
6832	6838	__u64 data[16];
@@ -6853,6 +6859,11 @@ Valid values for 'type' are: · 20 unmodified lines
6853	6859	reset/shutdown of the VM.
6854	6860	- KVM_SYSTEM_EVENT_SEV_TERM -- an AMD SEV guest requested termination.
6855	6861	The guest physical address of the guest's GHCB is stored in `data[0]`.
	6862	+ - KVM_SYSTEM_EVENT_TDX_FATAL -- a TDX guest reported a fatal error state.
	6863	+ KVM doesn't do any parsing or conversion, it just dumps 16 general-purpose
	6864	+ registers to userspace, in ascending order of the 4-bit indices for x86-64
	6865	+ general-purpose registers in instruction encoding, as defined in the Intel
	6866	+ SDM.
6856	6867	- KVM_SYSTEM_EVENT_WAKEUP -- the exiting vCPU is in a suspended state and
6857	6868	KVM has recognized a wakeup event. Userspace may honor this event by
6858	6869	marking the exiting vCPU as runnable, or deny it and call KVM_RUN again.
@@ -8194,6 +8205,28 @@ KVM_X86_QUIRK_STUFF_FEATURE_MSRS By default, at vCPU creation, KVM sets the · 1335 unmodified lines
8194	8205	and 0x489), as KVM does now allow them to
8195	8206	be set by userspace (KVM sets them based on
8196	8207	guest CPUID, for safety purposes).
	8208	+
	8209	+KVM_X86_QUIRK_IGNORE_GUEST_PAT By default, on Intel platforms, KVM ignores
	8210	+ guest PAT and forces the effective memory
	8211	+ type to WB in EPT. The quirk is not available
	8212	+ on Intel platforms which are incapable of
	8213	+ safely honoring guest PAT (i.e., without CPU
	8214	+ self-snoop, KVM always ignores guest PAT and
	8215	+ forces effective memory type to WB). It is
	8216	+ also ignored on AMD platforms or, on Intel,
	8217	+ when a VM has non-coherent DMA devices
	8218	+ assigned; KVM always honors guest PAT in
	8219	+ such case. The quirk is needed to avoid
	8220	+ slowdowns on certain Intel Xeon platforms
	8221	+ (e.g. ICX, SPR) where self-snoop feature is
	8222	+ supported but UC is slow enough to cause
	8223	+ issues with some older guests that use
	8224	+ UC instead of WC to map the video RAM.
	8225	+ Userspace can disable the quirk to honor
	8226	+ guest PAT if it knows that there is no such
	8227	+ guest software, for example if it does not
	8228	+ expose a bochs graphics device (which is
	8229	+ known to have had a buggy driver).
8197	8230	=================================== ============================================
8198	8231
8199	8232	7.32 KVM_CAP_MAX_VCPU_ID

Documentation/virt/kvm/x86/index.rst+1−0

@@ -11,6 +11,7 @@ KVM for x86 systems · 10 unmodified lines
11	11	cpuid
12	12	errata
13	13	hypercalls
	14	+ intel-tdx
14	15	mmu
15	16	msr
16	17	nested-vmx

Documentation/virt/kvm/x86/intel-tdx.rst+255−0

@@ -0,0 +1,255 @@
	1	+.. SPDX-License-Identifier: GPL-2.0
	2	+
	3	+===================================
	4	+Intel Trust Domain Extensions (TDX)
	5	+===================================
	6	+
	7	+Overview
	8	+========
	9	+Intel's Trust Domain Extensions (TDX) protect confidential guest VMs from the
	10	+host and physical attacks. A CPU-attested software module called 'the TDX
	11	+module' runs inside a new CPU isolated range to provide the functionalities to
	12	+manage and run protected VMs, a.k.a, TDX guests or TDs.
	13	+
	14	+Please refer to [1] for the whitepaper, specifications and other resources.
	15	+
	16	+This documentation describes TDX-specific KVM ABIs. The TDX module needs to be
	17	+initialized before it can be used by KVM to run any TDX guests. The host
	18	+core-kernel provides the support of initializing the TDX module, which is
	19	+described in the Documentation/arch/x86/tdx.rst.
	20	+
	21	+API description
	22	+===============
	23	+
	24	+KVM_MEMORY_ENCRYPT_OP
	25	+---------------------
	26	+:Type: vm ioctl, vcpu ioctl
	27	+
	28	+For TDX operations, KVM_MEMORY_ENCRYPT_OP is re-purposed to be generic
	29	+ioctl with TDX specific sub-ioctl() commands.
	30	+
	31	+::
	32	+
	33	+ /* Trust Domain Extensions sub-ioctl() commands. */
	34	+ enum kvm_tdx_cmd_id {
	35	+ KVM_TDX_CAPABILITIES = 0,
	36	+ KVM_TDX_INIT_VM,
	37	+ KVM_TDX_INIT_VCPU,
	38	+ KVM_TDX_INIT_MEM_REGION,
	39	+ KVM_TDX_FINALIZE_VM,
	40	+ KVM_TDX_GET_CPUID,
	41	+
	42	+ KVM_TDX_CMD_NR_MAX,
	43	+ };
	44	+
	45	+ struct kvm_tdx_cmd {
	46	+ /* enum kvm_tdx_cmd_id */
	47	+ __u32 id;
	48	+ /* flags for sub-command. If sub-command doesn't use this, set zero. */
	49	+ __u32 flags;
	50	+ /*
	51	+ * data for each sub-command. An immediate or a pointer to the actual
	52	+ * data in process virtual address. If sub-command doesn't use it,
	53	+ * set zero.
	54	+ */
	55	+ __u64 data;
	56	+ /*
	57	+ * Auxiliary error code. The sub-command may return TDX SEAMCALL
	58	+ * status code in addition to -Exxx.
	59	+ */
	60	+ __u64 hw_error;
	61	+ };
	62	+
	63	+KVM_TDX_CAPABILITIES
	64	+--------------------
	65	+:Type: vm ioctl
	66	+:Returns: 0 on success, <0 on error
	67	+
	68	+Return the TDX capabilities that current KVM supports with the specific TDX
	69	+module loaded in the system. It reports what features/capabilities are allowed
	70	+to be configured to the TDX guest.
	71	+
	72	+- id: KVM_TDX_CAPABILITIES
	73	+- flags: must be 0
	74	+- data: pointer to struct kvm_tdx_capabilities
	75	+- hw_error: must be 0
	76	+
	77	+::
	78	+
	79	+ struct kvm_tdx_capabilities {
	80	+ __u64 supported_attrs;
	81	+ __u64 supported_xfam;
	82	+ __u64 reserved[254];
	83	+
	84	+ /* Configurable CPUID bits for userspace */
	85	+ struct kvm_cpuid2 cpuid;
	86	+ };
	87	+
	88	+
	89	+KVM_TDX_INIT_VM
	90	+---------------
	91	+:Type: vm ioctl
	92	+:Returns: 0 on success, <0 on error
	93	+
	94	+Perform TDX specific VM initialization. This needs to be called after
	95	+KVM_CREATE_VM and before creating any VCPUs.
	96	+
	97	+- id: KVM_TDX_INIT_VM
	98	+- flags: must be 0
	99	+- data: pointer to struct kvm_tdx_init_vm
	100	+- hw_error: must be 0
	101	+
	102	+::
	103	+
	104	+ struct kvm_tdx_init_vm {
	105	+ __u64 attributes;
	106	+ __u64 xfam;
	107	+ __u64 mrconfigid[6]; /* sha384 digest */
	108	+ __u64 mrowner[6]; /* sha384 digest */
	109	+ __u64 mrownerconfig[6]; /* sha384 digest */
	110	+
	111	+ /* The total space for TD_PARAMS before the CPUIDs is 256 bytes */
	112	+ __u64 reserved[12];
	113	+
	114	+ /*
	115	+ * Call KVM_TDX_INIT_VM before vcpu creation, thus before
	116	+ * KVM_SET_CPUID2.
	117	+ * This configuration supersedes KVM_SET_CPUID2s for VCPUs because the
	118	+ * TDX module directly virtualizes those CPUIDs without VMM. The user
	119	+ * space VMM, e.g. qemu, should make KVM_SET_CPUID2 consistent with
	120	+ * those values. If it doesn't, KVM may have wrong idea of vCPUIDs of
	121	+ * the guest, and KVM may wrongly emulate CPUIDs or MSRs that the TDX
	122	+ * module doesn't virtualize.
	123	+ */
	124	+ struct kvm_cpuid2 cpuid;
	125	+ };
	126	+
	127	+
	128	+KVM_TDX_INIT_VCPU
	129	+-----------------
	130	+:Type: vcpu ioctl
	131	+:Returns: 0 on success, <0 on error
	132	+
	133	+Perform TDX specific VCPU initialization.
	134	+
	135	+- id: KVM_TDX_INIT_VCPU
	136	+- flags: must be 0
	137	+- data: initial value of the guest TD VCPU RCX
	138	+- hw_error: must be 0
	139	+
	140	+KVM_TDX_INIT_MEM_REGION
	141	+-----------------------
	142	+:Type: vcpu ioctl
	143	+:Returns: 0 on success, <0 on error
	144	+
	145	+Initialize @nr_pages TDX guest private memory starting from @gpa with userspace
	146	+provided data from @source_addr.
	147	+
	148	+Note, before calling this sub command, memory attribute of the range
	149	+[gpa, gpa + nr_pages] needs to be private. Userspace can use
	150	+KVM_SET_MEMORY_ATTRIBUTES to set the attribute.
	151	+
	152	+If KVM_TDX_MEASURE_MEMORY_REGION flag is specified, it also extends measurement.
	153	+
	154	+- id: KVM_TDX_INIT_MEM_REGION
	155	+- flags: currently only KVM_TDX_MEASURE_MEMORY_REGION is defined
	156	+- data: pointer to struct kvm_tdx_init_mem_region
	157	+- hw_error: must be 0
	158	+
	159	+::
	160	+
	161	+ #define KVM_TDX_MEASURE_MEMORY_REGION (1UL << 0)
	162	+
	163	+ struct kvm_tdx_init_mem_region {
	164	+ __u64 source_addr;
	165	+ __u64 gpa;
	166	+ __u64 nr_pages;
	167	+ };
	168	+
	169	+
	170	+KVM_TDX_FINALIZE_VM
	171	+-------------------
	172	+:Type: vm ioctl
	173	+:Returns: 0 on success, <0 on error
	174	+
	175	+Complete measurement of the initial TD contents and mark it ready to run.
	176	+
	177	+- id: KVM_TDX_FINALIZE_VM
	178	+- flags: must be 0
	179	+- data: must be 0
	180	+- hw_error: must be 0
	181	+
	182	+
	183	+KVM_TDX_GET_CPUID
	184	+-----------------
	185	+:Type: vcpu ioctl
	186	+:Returns: 0 on success, <0 on error
	187	+
	188	+Get the CPUID values that the TDX module virtualizes for the TD guest.
	189	+When it returns -E2BIG, the user space should allocate a larger buffer and
	190	+retry. The minimum buffer size is updated in the nent field of the
	191	+struct kvm_cpuid2.
	192	+
	193	+- id: KVM_TDX_GET_CPUID
	194	+- flags: must be 0
	195	+- data: pointer to struct kvm_cpuid2 (in/out)
	196	+- hw_error: must be 0 (out)
	197	+
	198	+::
	199	+
	200	+ struct kvm_cpuid2 {
	201	+ __u32 nent;
	202	+ __u32 padding;
	203	+ struct kvm_cpuid_entry2 entries[0];
	204	+ };
	205	+
	206	+ struct kvm_cpuid_entry2 {
	207	+ __u32 function;
	208	+ __u32 index;
	209	+ __u32 flags;
	210	+ __u32 eax;
	211	+ __u32 ebx;
	212	+ __u32 ecx;
	213	+ __u32 edx;
	214	+ __u32 padding[3];
	215	+ };
	216	+
	217	+KVM TDX creation flow
	218	+=====================
	219	+In addition to the standard KVM flow, new TDX ioctls need to be called. The
	220	+control flow is as follows:
	221	+
	222	+#. Check system wide capability
	223	+
	224	+ * KVM_CAP_VM_TYPES: Check if VM type is supported and if KVM_X86_TDX_VM
	225	+ is supported.
	226	+
	227	+#. Create VM
	228	+
	229	+ * KVM_CREATE_VM
	230	+ * KVM_TDX_CAPABILITIES: Query TDX capabilities for creating TDX guests.
	231	+ * KVM_CHECK_EXTENSION(KVM_CAP_MAX_VCPUS): Query maximum VCPUs the TD can
	232	+ support at VM level (TDX has its own limitation on this).
	233	+ * KVM_SET_TSC_KHZ: Configure TD's TSC frequency if a different TSC frequency
	234	+ than host is desired. This is Optional.
	235	+ * KVM_TDX_INIT_VM: Pass TDX specific VM parameters.
	236	+
	237	+#. Create VCPU
	238	+
	239	+ * KVM_CREATE_VCPU
	240	+ * KVM_TDX_INIT_VCPU: Pass TDX specific VCPU parameters.
	241	+ * KVM_SET_CPUID2: Configure TD's CPUIDs.
	242	+ * KVM_SET_MSRS: Configure TD's MSRs.
	243	+
	244	+#. Initialize initial guest memory
	245	+
	246	+ * Prepare content of initial guest memory.
	247	+ * KVM_TDX_INIT_MEM_REGION: Add initial guest memory.
	248	+ * KVM_TDX_FINALIZE_VM: Finalize the measurement of the TDX guest.
	249	+
	250	+#. Run VCPU
	251	+
	252	+References
	253	+==========
	254	+
	255	+.. [1] https://www.intel.com/content/www/us/en/developer/tools/trust-domain-extensions/documentation.html

arch/x86/include/asm/kvm-x86-ops.h+4−1

@@ -21,6 +21,7 @@ KVM_X86_OP(has_emulated_msr) · 20 unmodified lines
21	21	KVM_X86_OP(vcpu_after_set_cpuid)
22	22	KVM_X86_OP(vm_init)
23	23	KVM_X86_OP_OPTIONAL(vm_destroy)
	24	+KVM_X86_OP_OPTIONAL(vm_pre_destroy)
24	25	KVM_X86_OP_OPTIONAL_RET0(vcpu_precreate)
25	26	KVM_X86_OP(vcpu_create)
26	27	KVM_X86_OP(vcpu_free)
@@ -115,6 +116,7 @@ KVM_X86_OP_OPTIONAL(pi_start_assignment) · 88 unmodified lines
115	116	KVM_X86_OP_OPTIONAL(apicv_pre_state_restore)
116	117	KVM_X86_OP_OPTIONAL(apicv_post_state_restore)
117	118	KVM_X86_OP_OPTIONAL_RET0(dy_apicv_has_pending_interrupt)
	119	+KVM_X86_OP_OPTIONAL(protected_apic_has_interrupt)
118	120	KVM_X86_OP_OPTIONAL(set_hv_timer)
119	121	KVM_X86_OP_OPTIONAL(cancel_hv_timer)
120	122	KVM_X86_OP(setup_mce)
@@ -125,7 +127,8 @@ KVM_X86_OP(leave_smm) · 4 unmodified lines
125	127	KVM_X86_OP(enable_smi_window)
126	128	#endif
127	129	KVM_X86_OP_OPTIONAL(dev_get_attr)
128		−KVM_X86_OP_OPTIONAL(mem_enc_ioctl)
	130	+KVM_X86_OP(mem_enc_ioctl)
	131	+KVM_X86_OP_OPTIONAL(vcpu_mem_enc_ioctl)
129	132	KVM_X86_OP_OPTIONAL(mem_enc_register_region)
130	133	KVM_X86_OP_OPTIONAL(mem_enc_unregister_region)
131	134	KVM_X86_OP_OPTIONAL(vm_copy_enc_context_from)

arch/x86/include/asm/kvm_host.h+26−8

@@ -607,8 +607,15 @@ struct kvm_pmu { · 606 unmodified lines
607	607	struct kvm_pmu_ops;
608	608
609	609	enum {
610		− KVM_DEBUGREG_BP_ENABLED = 1,
611		− KVM_DEBUGREG_WONT_EXIT = 2,
	610	+ KVM_DEBUGREG_BP_ENABLED = BIT(0),
	611	+ KVM_DEBUGREG_WONT_EXIT = BIT(1),
	612	+ /*
	613	+ * Guest debug registers (DR0-3, DR6 and DR7) are saved/restored by
	614	+ * hardware on exit from or enter to guest. KVM needn't switch them.
	615	+ * DR0-3, DR6 and DR7 are set to their architectural INIT value on VM
	616	+ * exit, host values need to be restored.
	617	+ */
	618	+ KVM_DEBUGREG_AUTO_SWITCH = BIT(2),
612	619	};
613	620
614	621	struct kvm_mtrr {
@@ -1569,6 +1576,13 @@ struct kvm_arch { · 954 unmodified lines
1569	1576	struct kvm_mmu_memory_cache split_desc_cache;
1570	1577
1571	1578	gfn_t gfn_direct_bits;
	1579	+
	1580	+ /*
	1581	+ * Size of the CPU's dirty log buffer, i.e. VMX's PML buffer. A Zero
	1582	+ * value indicates CPU dirty logging is unsupported or disabled in
	1583	+ * current VM.
	1584	+ */
	1585	+ int cpu_dirty_log_size;
1572	1586	};
1573	1587
1574	1588	struct kvm_vm_stat {
@@ -1672,6 +1686,7 @@ struct kvm_x86_ops { · 97 unmodified lines
1672	1686	unsigned int vm_size;
1673	1687	int (vm_init)(struct kvm kvm);
1674	1688	void (vm_destroy)(struct kvm kvm);
	1689	+ void (vm_pre_destroy)(struct kvm kvm);
1675	1690
1676	1691	/* Create, but do not attach this VCPU */
1677	1692	int (vcpu_precreate)(struct kvm kvm);
@@ -1821,11 +1836,6 @@ struct kvm_x86_ops { · 143 unmodified lines
1821	1836	struct x86_exception *exception);
1822	1837	void (handle_exit_irqoff)(struct kvm_vcpu vcpu);
1823	1838
1824		− /*
1825		− * Size of the CPU's dirty log buffer, i.e. VMX's PML buffer. A zero
1826		− * value indicates CPU dirty logging is unsupported or disabled.
1827		− */
1828		− int cpu_dirty_log_size;
1829	1839	void (update_cpu_dirty_logging)(struct kvm_vcpu vcpu);
1830	1840
1831	1841	const struct kvm_x86_nested_ops *nested_ops;
@@ -1839,6 +1849,7 @@ struct kvm_x86_ops { · 7 unmodified lines
1839	1849	void (apicv_pre_state_restore)(struct kvm_vcpu vcpu);
1840	1850	void (apicv_post_state_restore)(struct kvm_vcpu vcpu);
1841	1851	bool (dy_apicv_has_pending_interrupt)(struct kvm_vcpu vcpu);
	1852	+ bool (protected_apic_has_interrupt)(struct kvm_vcpu vcpu);
1842	1853
1843	1854	int (set_hv_timer)(struct kvm_vcpu vcpu, u64 guest_deadline_tsc,
1844	1855	bool *expired);
@@ -1855,6 +1866,7 @@ struct kvm_x86_ops { · 10 unmodified lines
1855	1866
1856	1867	int (dev_get_attr)(u32 group, u64 attr, u64 val);
1857	1868	int (mem_enc_ioctl)(struct kvm kvm, void __user *argp);
	1869	+ int (vcpu_mem_enc_ioctl)(struct kvm_vcpu vcpu, void __user *argp);
1858	1870	int (mem_enc_register_region)(struct kvm kvm, struct kvm_enc_region *argp);
1859	1871	int (mem_enc_unregister_region)(struct kvm kvm, struct kvm_enc_region *argp);
1860	1872	int (vm_copy_enc_context_from)(struct kvm kvm, unsigned int source_fd);
@@ -2331,6 +2343,7 @@ int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low, · 470 unmodified lines
2331	2343	int kvm_add_user_return_msr(u32 msr);
2332	2344	int kvm_find_user_return_msr(u32 msr);
2333	2345	int kvm_set_user_return_msr(unsigned index, u64 val, u64 mask);
	2346	+void kvm_user_return_msr_update_cache(unsigned int index, u64 val);
2334	2347
2335	2348	static inline bool kvm_is_supported_user_return_msr(u32 msr)
2336	2349	{
@@ -2414,7 +2427,12 @@ int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages); · 77 unmodified lines
2414	2427	KVM_X86_QUIRK_FIX_HYPERCALL_INSN \| \
2415	2428	KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS \| \
2416	2429	KVM_X86_QUIRK_SLOT_ZAP_ALL \| \
2417		− KVM_X86_QUIRK_STUFF_FEATURE_MSRS)
	2430	+ KVM_X86_QUIRK_STUFF_FEATURE_MSRS \| \
	2431	+ KVM_X86_QUIRK_IGNORE_GUEST_PAT)
	2432	+
	2433	+#define KVM_X86_CONDITIONAL_QUIRKS \
	2434	+ (KVM_X86_QUIRK_CD_NW_CLEARED \| \
	2435	+ KVM_X86_QUIRK_IGNORE_GUEST_PAT)
2418	2436
2419	2437	/*
2420	2438	* KVM previously used a u32 field in kvm_run to indicate the hypercall was

arch/x86/include/asm/posted_intr.h+5−0

@@ -81,6 +81,11 @@ static inline bool pi_test_sn(struct pi_desc *pi_desc) · 80 unmodified lines
81	81	return test_bit(POSTED_INTR_SN, (unsigned long *)&pi_desc->control);
82	82	}
83	83
	84	+static inline bool pi_test_pir(int vector, struct pi_desc *pi_desc)
	85	+{
	86	+ return test_bit(vector, (unsigned long *)pi_desc->pir);
	87	+}
	88	+
84	89	/* Non-atomic helpers */
85	90	static inline void __pi_set_sn(struct pi_desc *pi_desc)
86	91	{

arch/x86/include/asm/shared/tdx.h+8−1

@@ -67,11 +67,18 @@ · 66 unmodified lines
67	67	#define TD_CTLS_LOCK BIT_ULL(TD_CTLS_LOCK_BIT)
68	68
69	69	/* TDX hypercall Leaf IDs */
	70	+#define TDVMCALL_GET_TD_VM_CALL_INFO 0x10000
70	71	#define TDVMCALL_MAP_GPA 0x10001
71	72	#define TDVMCALL_GET_QUOTE 0x10002
72	73	#define TDVMCALL_REPORT_FATAL_ERROR 0x10003
73	74
74		−#define TDVMCALL_STATUS_RETRY 1
	75	+/*
	76	+ * TDG.VP.VMCALL Status Codes (returned in R10)
	77	+ */
	78	+#define TDVMCALL_STATUS_SUCCESS 0x0000000000000000ULL
	79	+#define TDVMCALL_STATUS_RETRY 0x0000000000000001ULL
	80	+#define TDVMCALL_STATUS_INVALID_OPERAND 0x8000000000000000ULL
	81	+#define TDVMCALL_STATUS_ALIGN_ERROR 0x8000000000000002ULL
75	82
76	83	/*
77	84	* Bitmasks of exposed registers (with VMM).

arch/x86/include/asm/tdx.h+75−0

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	5
6	6	#include <linux/init.h>
7	7	#include <linux/bits.h>
	8	+#include <linux/mmzone.h>
8	9
9	10	#include <asm/errno.h>
10	11	#include <asm/ptrace.h>
@@ -18,6 +19,7 @@ · 7 unmodified lines
18	19	* TDX module.
19	20	*/
20	21	#define TDX_ERROR _BITUL(63)
	22	+#define TDX_NON_RECOVERABLE _BITUL(62)
21	23	#define TDX_SW_ERROR (TDX_ERROR \| GENMASK_ULL(47, 40))
22	24	#define TDX_SEAMCALL_VMFAILINVALID (TDX_SW_ERROR \| _UL(0xFFFF0000))
23	25
@@ -33,6 +35,8 @@ · 9 unmodified lines
33	35	#ifndef __ASSEMBLER__
34	36
35	37	#include <uapi/asm/mce.h>
	38	+#include <asm/tdx_global_metadata.h>
	39	+#include <linux/pgtable.h>
36	40
37	41	/*
38	42	* Used by the #VE exception handler to gather the #VE exception
@@ -119,11 +123,82 @@ static inline u64 sc_retry(sc_func_t func, u64 fn, · 80 unmodified lines
119	123	int tdx_cpu_enable(void);
120	124	int tdx_enable(void);
121	125	const char tdx_dump_mce_info(struct mce m);
	126	+const struct tdx_sys_info *tdx_get_sysinfo(void);
	127	+
	128	+int tdx_guest_keyid_alloc(void);
	129	+u32 tdx_get_nr_guest_keyids(void);
	130	+void tdx_guest_keyid_free(unsigned int keyid);
	131	+
	132	+struct tdx_td {
	133	+ /* TD root structure: */
	134	+ struct page *tdr_page;
	135	+
	136	+ int tdcs_nr_pages;
	137	+ /* TD control structure: */
	138	+ struct page **tdcs_pages;
	139	+
	140	+ /* Size of `tdcx_pages` in struct tdx_vp */
	141	+ int tdcx_nr_pages;
	142	+};
	143	+
	144	+struct tdx_vp {
	145	+ /* TDVP root page */
	146	+ struct page *tdvpr_page;
	147	+
	148	+ /* TD vCPU control structure: */
	149	+ struct page **tdcx_pages;
	150	+};
	151	+
	152	+static inline u64 mk_keyed_paddr(u16 hkid, struct page *page)
	153	+{
	154	+ u64 ret;
	155	+
	156	+ ret = page_to_phys(page);
	157	+ /* KeyID bits are just above the physical address bits: */
	158	+ ret \|= (u64)hkid << boot_cpu_data.x86_phys_bits;
	159	+
	160	+ return ret;
	161	+}
	162	+
	163	+static inline int pg_level_to_tdx_sept_level(enum pg_level level)
	164	+{
	165	+ WARN_ON_ONCE(level == PG_LEVEL_NONE);
	166	+ return level - 1;
	167	+}
	168	+
	169	+u64 tdh_vp_enter(struct tdx_vp vp, struct tdx_module_args args);
	170	+u64 tdh_mng_addcx(struct tdx_td td, struct page tdcs_page);
	171	+u64 tdh_mem_page_add(struct tdx_td td, u64 gpa, struct page page, struct page source, u64 ext_err1, u64 *ext_err2);
	172	+u64 tdh_mem_sept_add(struct tdx_td td, u64 gpa, int level, struct page page, u64 ext_err1, u64 ext_err2);
	173	+u64 tdh_vp_addcx(struct tdx_vp vp, struct page tdcx_page);
	174	+u64 tdh_mem_page_aug(struct tdx_td td, u64 gpa, int level, struct page page, u64 ext_err1, u64 ext_err2);
	175	+u64 tdh_mem_range_block(struct tdx_td td, u64 gpa, int level, u64 ext_err1, u64 *ext_err2);
	176	+u64 tdh_mng_key_config(struct tdx_td *td);
	177	+u64 tdh_mng_create(struct tdx_td *td, u16 hkid);
	178	+u64 tdh_vp_create(struct tdx_td td, struct tdx_vp vp);
	179	+u64 tdh_mng_rd(struct tdx_td td, u64 field, u64 data);
	180	+u64 tdh_mr_extend(struct tdx_td td, u64 gpa, u64 ext_err1, u64 *ext_err2);
	181	+u64 tdh_mr_finalize(struct tdx_td *td);
	182	+u64 tdh_vp_flush(struct tdx_vp *vp);
	183	+u64 tdh_mng_vpflushdone(struct tdx_td *td);
	184	+u64 tdh_mng_key_freeid(struct tdx_td *td);
	185	+u64 tdh_mng_init(struct tdx_td td, u64 td_params, u64 extended_err);
	186	+u64 tdh_vp_init(struct tdx_vp *vp, u64 initial_rcx, u32 x2apicid);
	187	+u64 tdh_vp_rd(struct tdx_vp vp, u64 field, u64 data);
	188	+u64 tdh_vp_wr(struct tdx_vp *vp, u64 field, u64 data, u64 mask);
	189	+u64 tdh_phymem_page_reclaim(struct page page, u64 tdx_pt, u64 tdx_owner, u64 tdx_size);
	190	+u64 tdh_mem_track(struct tdx_td *tdr);
	191	+u64 tdh_mem_page_remove(struct tdx_td td, u64 gpa, u64 level, u64 ext_err1, u64 *ext_err2);
	192	+u64 tdh_phymem_cache_wb(bool resume);
	193	+u64 tdh_phymem_page_wbinvd_tdr(struct tdx_td *td);
	194	+u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, struct page *page);
122	195	#else
123	196	static inline void tdx_init(void) { }
124	197	static inline int tdx_cpu_enable(void) { return -ENODEV; }
125	198	static inline int tdx_enable(void) { return -ENODEV; }
	199	+static inline u32 tdx_get_nr_guest_keyids(void) { return 0; }
126	200	static inline const char tdx_dump_mce_info(struct mce m) { return NULL; }
	201	+static inline const struct tdx_sys_info *tdx_get_sysinfo(void) { return NULL; }
127	202	#endif /* CONFIG_INTEL_TDX_HOST */
128	203
129	204	#endif /* !__ASSEMBLER__ */

arch/x86/include/asm/tdx_global_metadata.hrenamed from arch/x86/virt/vmx/tdx/tdx_global_metadata.h+19−0

@@ -17,9 +17,28 @@ struct tdx_sys_info_tdmr { · 16 unmodified lines
17	17	u16 pamt_1g_entry_size;
18	18	};
19	19
	20	+struct tdx_sys_info_td_ctrl {
	21	+ u16 tdr_base_size;
	22	+ u16 tdcs_base_size;
	23	+ u16 tdvps_base_size;
	24	+};
	25	+
	26	+struct tdx_sys_info_td_conf {
	27	+ u64 attributes_fixed0;
	28	+ u64 attributes_fixed1;
	29	+ u64 xfam_fixed0;
	30	+ u64 xfam_fixed1;
	31	+ u16 num_cpuid_config;
	32	+ u16 max_vcpus_per_td;
	33	+ u64 cpuid_config_leaves[128];
	34	+ u64 cpuid_config_values[128][2];
	35	+};
	36	+
20	37	struct tdx_sys_info {
21	38	struct tdx_sys_info_features features;
22	39	struct tdx_sys_info_tdmr tdmr;
	40	+ struct tdx_sys_info_td_ctrl td_ctrl;
	41	+ struct tdx_sys_info_td_conf td_conf;
23	42	};
24	43
25	44	#endif

arch/x86/include/asm/vmx.h+2−0

@@ -256,6 +256,7 @@ enum vmcs_field { · 255 unmodified lines
256	256	TSC_MULTIPLIER_HIGH = 0x00002033,
257	257	TERTIARY_VM_EXEC_CONTROL = 0x00002034,
258	258	TERTIARY_VM_EXEC_CONTROL_HIGH = 0x00002035,
	259	+ SHARED_EPT_POINTER = 0x0000203C,
259	260	PID_POINTER_TABLE = 0x00002042,
260	261	PID_POINTER_TABLE_HIGH = 0x00002043,
261	262	GUEST_PHYSICAL_ADDRESS = 0x00002400,
@@ -586,6 +587,7 @@ enum vm_entry_failure_code { · 324 unmodified lines
586	587	#define EPT_VIOLATION_PROT_READ BIT(3)
587	588	#define EPT_VIOLATION_PROT_WRITE BIT(4)
588	589	#define EPT_VIOLATION_PROT_EXEC BIT(5)
	590	+#define EPT_VIOLATION_EXEC_FOR_RING3_LIN BIT(6)
589	591	#define EPT_VIOLATION_PROT_MASK (EPT_VIOLATION_PROT_READ \| \
590	592	EPT_VIOLATION_PROT_WRITE \| \
591	593	EPT_VIOLATION_PROT_EXEC)

arch/x86/include/uapi/asm/kvm.h+71−0

@@ -441,6 +441,7 @@ struct kvm_sync_regs { · 440 unmodified lines
441	441	#define KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS (1 << 6)
442	442	#define KVM_X86_QUIRK_SLOT_ZAP_ALL (1 << 7)
443	443	#define KVM_X86_QUIRK_STUFF_FEATURE_MSRS (1 << 8)
	444	+#define KVM_X86_QUIRK_IGNORE_GUEST_PAT (1 << 9)
444	445
445	446	#define KVM_STATE_NESTED_FORMAT_VMX 0
446	447	#define KVM_STATE_NESTED_FORMAT_SVM 1
@@ -930,4 +931,74 @@ struct kvm_hyperv_eventfd { · 483 unmodified lines
930	931	#define KVM_X86_SNP_VM 4
931	932	#define KVM_X86_TDX_VM 5
932	933
	934	+/* Trust Domain eXtension sub-ioctl() commands. */
	935	+enum kvm_tdx_cmd_id {
	936	+ KVM_TDX_CAPABILITIES = 0,
	937	+ KVM_TDX_INIT_VM,
	938	+ KVM_TDX_INIT_VCPU,
	939	+ KVM_TDX_INIT_MEM_REGION,
	940	+ KVM_TDX_FINALIZE_VM,
	941	+ KVM_TDX_GET_CPUID,
	942	+
	943	+ KVM_TDX_CMD_NR_MAX,
	944	+};
	945	+
	946	+struct kvm_tdx_cmd {
	947	+ /* enum kvm_tdx_cmd_id */
	948	+ __u32 id;
	949	+ /* flags for sub-commend. If sub-command doesn't use this, set zero. */
	950	+ __u32 flags;
	951	+ /*
	952	+ * data for each sub-command. An immediate or a pointer to the actual
	953	+ * data in process virtual address. If sub-command doesn't use it,
	954	+ * set zero.
	955	+ */
	956	+ __u64 data;
	957	+ /*
	958	+ * Auxiliary error code. The sub-command may return TDX SEAMCALL
	959	+ * status code in addition to -Exxx.
	960	+ */
	961	+ __u64 hw_error;
	962	+};
	963	+
	964	+struct kvm_tdx_capabilities {
	965	+ __u64 supported_attrs;
	966	+ __u64 supported_xfam;
	967	+ __u64 reserved[254];
	968	+
	969	+ /* Configurable CPUID bits for userspace */
	970	+ struct kvm_cpuid2 cpuid;
	971	+};
	972	+
	973	+struct kvm_tdx_init_vm {
	974	+ __u64 attributes;
	975	+ __u64 xfam;
	976	+ __u64 mrconfigid[6]; /* sha384 digest */
	977	+ __u64 mrowner[6]; /* sha384 digest */
	978	+ __u64 mrownerconfig[6]; /* sha384 digest */
	979	+
	980	+ /* The total space for TD_PARAMS before the CPUIDs is 256 bytes */
	981	+ __u64 reserved[12];
	982	+
	983	+ /*
	984	+ * Call KVM_TDX_INIT_VM before vcpu creation, thus before
	985	+ * KVM_SET_CPUID2.
	986	+ * This configuration supersedes KVM_SET_CPUID2s for VCPUs because the
	987	+ * TDX module directly virtualizes those CPUIDs without VMM. The user
	988	+ * space VMM, e.g. qemu, should make KVM_SET_CPUID2 consistent with
	989	+ * those values. If it doesn't, KVM may have wrong idea of vCPUIDs of
	990	+ * the guest, and KVM may wrongly emulate CPUIDs or MSRs that the TDX
	991	+ * module doesn't virtualize.
	992	+ */
	993	+ struct kvm_cpuid2 cpuid;
	994	+};
	995	+
	996	+#define KVM_TDX_MEASURE_MEMORY_REGION _BITULL(0)
	997	+
	998	+struct kvm_tdx_init_mem_region {
	999	+ __u64 source_addr;
	1000	+ __u64 gpa;
	1001	+ __u64 nr_pages;
	1002	+};
	1003	+
933	1004	#endif /* _ASM_X86_KVM_H */

arch/x86/include/uapi/asm/vmx.h+4−1

@@ -34,6 +34,7 @@ · 33 unmodified lines
34	34	#define EXIT_REASON_TRIPLE_FAULT 2
35	35	#define EXIT_REASON_INIT_SIGNAL 3
36	36	#define EXIT_REASON_SIPI_SIGNAL 4
	37	+#define EXIT_REASON_OTHER_SMI 6
37	38
38	39	#define EXIT_REASON_INTERRUPT_WINDOW 7
39	40	#define EXIT_REASON_NMI_WINDOW 8
@@ -92,6 +93,7 @@ · 52 unmodified lines
92	93	#define EXIT_REASON_TPAUSE 68
93	94	#define EXIT_REASON_BUS_LOCK 74
94	95	#define EXIT_REASON_NOTIFY 75
	96	+#define EXIT_REASON_TDCALL 77
95	97
96	98	#define VMX_EXIT_REASONS \
97	99	{ EXIT_REASON_EXCEPTION_NMI, "EXCEPTION_NMI" }, \
@@ -155,7 +157,8 @@ · 57 unmodified lines
155	157	{ EXIT_REASON_UMWAIT, "UMWAIT" }, \
156	158	{ EXIT_REASON_TPAUSE, "TPAUSE" }, \
157	159	{ EXIT_REASON_BUS_LOCK, "BUS_LOCK" }, \
158		− { EXIT_REASON_NOTIFY, "NOTIFY" }
	160	+ { EXIT_REASON_NOTIFY, "NOTIFY" }, \
	161	+ { EXIT_REASON_TDCALL, "TDCALL" }
159	162
160	163	#define VMX_EXIT_REASON_FLAGS \
161	164	{ VMX_EXIT_REASONS_FAILED_VMENTRY, "FAILED_VMENTRY" }

arch/x86/kvm/Kconfig+12−0

@@ -95,6 +95,8 @@ config KVM_SW_PROTECTED_VM · 94 unmodified lines
95	95	config KVM_INTEL
96	96	tristate "KVM for Intel (and compatible) processors support"
97	97	depends on KVM && IA32_FEAT_CTL
	98	+ select KVM_GENERIC_PRIVATE_MEM if INTEL_TDX_HOST
	99	+ select KVM_GENERIC_MEMORY_ATTRIBUTES if INTEL_TDX_HOST
98	100	help
99	101	Provides support for KVM on processors equipped with Intel's VT
100	102	extensions, a.k.a. Virtual Machine Extensions (VMX).
@@ -129,6 +131,16 @@ config X86_SGX_KVM · 28 unmodified lines
129	131
130	132	If unsure, say N.
131	133
	134	+config KVM_INTEL_TDX
	135	+ bool "Intel Trust Domain Extensions (TDX) support"
	136	+ default y
	137	+ depends on INTEL_TDX_HOST
	138	+ help
	139	+ Provides support for launching Intel Trust Domain Extensions (TDX)
	140	+ confidential VMs on Intel processors.
	141	+
	142	+ If unsure, say N.
	143	+
132	144	config KVM_AMD
133	145	tristate "KVM for AMD processors support"
134	146	depends on KVM && (CPU_SUP_AMD \|\| CPU_SUP_HYGON)

arch/x86/kvm/Makefile+1−0

@@ -20,6 +20,7 @@ kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \ · 19 unmodified lines
20	20
21	21	kvm-intel-$(CONFIG_X86_SGX_KVM) += vmx/sgx.o
22	22	kvm-intel-$(CONFIG_KVM_HYPERV) += vmx/hyperv.o vmx/hyperv_evmcs.o
	23	+kvm-intel-$(CONFIG_KVM_INTEL_TDX) += vmx/tdx.o
23	24
24	25	kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o
25	26

arch/x86/kvm/cpuid.c+19−33

@@ -81,17 +81,8 @@ u32 xstate_required_size(u64 xstate_bv, bool compacted) · 80 unmodified lines
81	81	return ret;
82	82	}
83	83
84		−/*
85		− * Magic value used by KVM when querying userspace-provided CPUID entries and
86		− * doesn't care about the CPIUD index because the index of the function in
87		− * question is not significant. Note, this magic value must have at least one
88		− * bit set in bits[63:32] and must be consumed as a u64 by cpuid_entry2_find()
89		− * to avoid false positives when processing guest CPUID input.
90		− */
91		−#define KVM_CPUID_INDEX_NOT_SIGNIFICANT -1ull
92		−
93		−static struct kvm_cpuid_entry2 cpuid_entry2_find(struct kvm_vcpu vcpu,
94		− u32 function, u64 index)
	84	+struct kvm_cpuid_entry2 *kvm_find_cpuid_entry2(
	85	+ struct kvm_cpuid_entry2 *entries, int nent, u32 function, u64 index)
95	86	{
96	87	struct kvm_cpuid_entry2 *e;
97	88	int i;
@@ -108,8 +99,8 @@ static struct kvm_cpuid_entry2 cpuid_entry2_find(struct kvm_vcpu vcpu, · 10 unmodified lines
108	99	*/
109	100	lockdep_assert_irqs_enabled();
110	101
111		− for (i = 0; i < vcpu->arch.cpuid_nent; i++) {
112		− e = &vcpu->arch.cpuid_entries[i];
	102	+ for (i = 0; i < nent; i++) {
	103	+ e = &entries[i];
113	104
114	105	if (e->function != function)
115	106	continue;
@@ -140,26 +131,7 @@ static struct kvm_cpuid_entry2 cpuid_entry2_find(struct kvm_vcpu vcpu, · 24 unmodified lines
140	131
141	132	return NULL;
142	133	}
143		−
144		−struct kvm_cpuid_entry2 kvm_find_cpuid_entry_index(struct kvm_vcpu vcpu,
145		− u32 function, u32 index)
146		−{
147		− return cpuid_entry2_find(vcpu, function, index);
148		−}
149		−EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry_index);
150		−
151		−struct kvm_cpuid_entry2 kvm_find_cpuid_entry(struct kvm_vcpu vcpu,
152		− u32 function)
153		−{
154		− return cpuid_entry2_find(vcpu, function, KVM_CPUID_INDEX_NOT_SIGNIFICANT);
155		−}
156		−EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry);
157		−
158		−/*
159		− * cpuid_entry2_find() and KVM_CPUID_INDEX_NOT_SIGNIFICANT should never be used
160		− * directly outside of kvm_find_cpuid_entry() and kvm_find_cpuid_entry_index().
161		− */
162		−#undef KVM_CPUID_INDEX_NOT_SIGNIFICANT
	134	+EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry2);
163	135
164	136	static int kvm_check_cpuid(struct kvm_vcpu *vcpu)
165	137	{
@@ -492,6 +464,20 @@ not_found: · 326 unmodified lines
492	464	return 36;
493	465	}
494	466
	467	+int cpuid_query_maxguestphyaddr(struct kvm_vcpu *vcpu)
	468	+{
	469	+ struct kvm_cpuid_entry2 *best;
	470	+
	471	+ best = kvm_find_cpuid_entry(vcpu, 0x80000000);
	472	+ if (!best \|\| best->eax < 0x80000008)
	473	+ goto not_found;
	474	+ best = kvm_find_cpuid_entry(vcpu, 0x80000008);
	475	+ if (best)
	476	+ return (best->eax >> 16) & 0xff;
	477	+not_found:
	478	+ return 0;
	479	+}
	480	+
495	481	/*
496	482	* This "raw" version returns the reserved GPA bits without any adjustments for
497	483	* encryption technologies that usurp bits. The raw mask should be used if and

arch/x86/kvm/cpuid.h+29−4

@@ -11,10 +11,34 @@ extern u32 kvm_cpu_caps[NR_KVM_CPU_CAPS] __read_mostly; · 10 unmodified lines
11	11	void kvm_set_cpu_caps(void);
12	12
13	13	void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu);
14		−struct kvm_cpuid_entry2 kvm_find_cpuid_entry_index(struct kvm_vcpu vcpu,
15		− u32 function, u32 index);
16		−struct kvm_cpuid_entry2 kvm_find_cpuid_entry(struct kvm_vcpu vcpu,
17		− u32 function);
	14	+struct kvm_cpuid_entry2 kvm_find_cpuid_entry2(struct kvm_cpuid_entry2 entries,
	15	+ int nent, u32 function, u64 index);
	16	+/*
	17	+ * Magic value used by KVM when querying userspace-provided CPUID entries and
	18	+ * doesn't care about the CPIUD index because the index of the function in
	19	+ * question is not significant. Note, this magic value must have at least one
	20	+ * bit set in bits[63:32] and must be consumed as a u64 by kvm_find_cpuid_entry2()
	21	+ * to avoid false positives when processing guest CPUID input.
	22	+ *
	23	+ * KVM_CPUID_INDEX_NOT_SIGNIFICANT should never be used directly outside of
	24	+ * kvm_find_cpuid_entry2() and kvm_find_cpuid_entry().
	25	+ */
	26	+#define KVM_CPUID_INDEX_NOT_SIGNIFICANT -1ull
	27	+
	28	+static inline struct kvm_cpuid_entry2 kvm_find_cpuid_entry_index(struct kvm_vcpu vcpu,
	29	+ u32 function, u32 index)
	30	+{
	31	+ return kvm_find_cpuid_entry2(vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent,
	32	+ function, index);
	33	+}
	34	+
	35	+static inline struct kvm_cpuid_entry2 kvm_find_cpuid_entry(struct kvm_vcpu vcpu,
	36	+ u32 function)
	37	+{
	38	+ return kvm_find_cpuid_entry2(vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent,
	39	+ function, KVM_CPUID_INDEX_NOT_SIGNIFICANT);
	40	+}
	41	+
18	42	int kvm_dev_ioctl_get_cpuid(struct kvm_cpuid2 *cpuid,
19	43	struct kvm_cpuid_entry2 __user *entries,
20	44	unsigned int type);
@@ -34,6 +58,7 @@ void __init kvm_init_xstate_sizes(void); · 13 unmodified lines
34	58	u32 xstate_required_size(u64 xstate_bv, bool compacted);
35	59
36	60	int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu);
	61	+int cpuid_query_maxguestphyaddr(struct kvm_vcpu *vcpu);
37	62	u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu);
38	63
39	64	static inline int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)

arch/x86/kvm/irq.c+3−0

@@ -100,6 +100,9 @@ int kvm_cpu_has_interrupt(struct kvm_vcpu *v) · 99 unmodified lines
100	100	if (kvm_cpu_has_extint(v))
101	101	return 1;
102	102
	103	+ if (lapic_in_kernel(v) && v->arch.apic->guest_apic_protected)
	104	+ return kvm_x86_call(protected_apic_has_interrupt)(v);
	105	+
103	106	return kvm_apic_has_interrupt(v) != -1; /* LAPIC */
104	107	}
105	108	EXPORT_SYMBOL_GPL(kvm_cpu_has_interrupt);

arch/x86/kvm/lapic.c+14−1

@@ -1790,8 +1790,17 @@ static void apic_update_lvtt(struct kvm_lapic *apic) · 1789 unmodified lines
1790	1790	static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu)
1791	1791	{
1792	1792	struct kvm_lapic *apic = vcpu->arch.apic;
1793		− u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT);
	1793	+ u32 reg;
1794	1794
	1795	+ /*
	1796	+ * Assume a timer IRQ was "injected" if the APIC is protected. KVM's
	1797	+ * copy of the vIRR is bogus, it's the responsibility of the caller to
	1798	+ * precisely check whether or not a timer IRQ is pending.
	1799	+ */
	1800	+ if (apic->guest_apic_protected)
	1801	+ return true;
	1802	+
	1803	+ reg = kvm_lapic_get_reg(apic, APIC_LVTT);
1795	1804	if (kvm_apic_hw_enabled(apic)) {
1796	1805	int vec = reg & APIC_VECTOR_MASK;
1797	1806	void *bitmap = apic->regs + APIC_ISR;
@@ -2650,6 +2659,7 @@ int kvm_apic_set_base(struct kvm_vcpu *vcpu, u64 value, bool host_initiated) · 852 unmodified lines
2650	2659	kvm_recalculate_apic_map(vcpu->kvm);
2651	2660	return 0;
2652	2661	}
	2662	+EXPORT_SYMBOL_GPL(kvm_apic_set_base);
2653	2663
2654	2664	void kvm_apic_update_apicv(struct kvm_vcpu *vcpu)
2655	2665	{
@@ -2958,6 +2968,9 @@ int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) · 302 unmodified lines
2958	2968	if (!kvm_apic_present(vcpu))
2959	2969	return -1;
2960	2970
	2971	+ if (apic->guest_apic_protected)
	2972	+ return -1;
	2973	+
2961	2974	__apic_update_ppr(apic, &ppr);
2962	2975	return apic_has_interrupt_for_ppr(apic, ppr);
2963	2976	}

arch/x86/kvm/lapic.h+2−0

@@ -65,6 +65,8 @@ struct kvm_lapic { · 64 unmodified lines
65	65	bool sw_enabled;
66	66	bool irr_pending;
67	67	bool lvt0_in_nmi_mode;
	68	+ /* Select registers in the vAPIC cannot be read/written. */
	69	+ bool guest_apic_protected;
68	70	/* Number of bits set in ISR. */
69	71	s16 isr_count;
70	72	/* The highest vector set in ISR; if -1 - invalid, must scan ISR. */

arch/x86/kvm/mmu.h+5−1

@@ -79,6 +79,7 @@ static inline gfn_t kvm_mmu_max_gfn(void) · 78 unmodified lines
79	79	u8 kvm_mmu_get_max_tdp_level(void);
80	80
81	81	void kvm_mmu_set_mmio_spte_mask(u64 mmio_value, u64 mmio_mask, u64 access_mask);
	82	+void kvm_mmu_set_mmio_spte_value(struct kvm *kvm, u64 mmio_value);
82	83	void kvm_mmu_set_me_spte_mask(u64 me_value, u64 me_mask);
83	84	void kvm_mmu_set_ept_masks(bool has_ad_bits, bool has_exec_only);
84	85
@@ -231,7 +232,7 @@ static inline u8 permission_fault(struct kvm_vcpu vcpu, struct kvm_mmu mmu, · 146 unmodified lines
231	232	return -(u32)fault & errcode;
232	233	}
233	234
234		−bool kvm_mmu_may_ignore_guest_pat(void);
	235	+bool kvm_mmu_may_ignore_guest_pat(struct kvm *kvm);
235	236
236	237	int kvm_mmu_post_init_vm(struct kvm *kvm);
237	238	void kvm_mmu_pre_destroy_vm(struct kvm *kvm);
@@ -253,6 +254,9 @@ extern bool tdp_mmu_enabled; · 15 unmodified lines
253	254	#define tdp_mmu_enabled false
254	255	#endif
255	256
	257	+bool kvm_tdp_mmu_gpa_is_mapped(struct kvm_vcpu *vcpu, u64 gpa);
	258	+int kvm_tdp_map_page(struct kvm_vcpu vcpu, gpa_t gpa, u64 error_code, u8 level);
	259	+
256	260	static inline bool kvm_memslots_have_rmaps(struct kvm *kvm)
257	261	{
258	262	return !tdp_mmu_enabled \|\| kvm_shadow_root_allocated(kvm);

arch/x86/kvm/mmu/mmu.c+20−19

@@ -110,6 +110,7 @@ static bool __ro_after_init tdp_mmu_allowed; · 109 unmodified lines
110	110	#ifdef CONFIG_X86_64
111	111	bool __read_mostly tdp_mmu_enabled = true;
112	112	module_param_named(tdp_mmu, tdp_mmu_enabled, bool, 0444);
	113	+EXPORT_SYMBOL_GPL(tdp_mmu_enabled);
113	114	#endif
114	115
115	116	static int max_huge_page_level __read_mostly;
@@ -1456,15 +1457,15 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, · 1340 unmodified lines
1456	1457	* enabled but it chooses between clearing the Dirty bit and Writeable
1457	1458	* bit based on the context.
1458	1459	*/
1459		− if (kvm_x86_ops.cpu_dirty_log_size)
	1460	+ if (kvm->arch.cpu_dirty_log_size)
1460	1461	kvm_mmu_clear_dirty_pt_masked(kvm, slot, gfn_offset, mask);
1461	1462	else
1462	1463	kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
1463	1464	}
1464	1465
1465		−int kvm_cpu_dirty_log_size(void)
	1466	+int kvm_cpu_dirty_log_size(struct kvm *kvm)
1466	1467	{
1467		− return kvm_x86_ops.cpu_dirty_log_size;
	1468	+ return kvm->arch.cpu_dirty_log_size;
1468	1469	}
1469	1470
1470	1471	bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
@@ -4835,19 +4836,6 @@ out_unlock: · 3364 unmodified lines
4835	4836	}
4836	4837	#endif
4837	4838
4838		−bool kvm_mmu_may_ignore_guest_pat(void)
4839		−{
4840		− /*
4841		− * When EPT is enabled (shadow_memtype_mask is non-zero), and the VM
4842		− * has non-coherent DMA (DMA doesn't snoop CPU caches), KVM's ABI is to
4843		− * honor the memtype from the guest's PAT so that guest accesses to
4844		− * memory that is DMA'd aren't cached against the guest's wishes. As a
4845		− * result, KVM _may_ ignore guest PAT, whereas without non-coherent DMA,
4846		− * KVM _always_ ignores guest PAT (when EPT is enabled).
4847		− */
4848		− return shadow_memtype_mask;
4849		−}
4850		−
4851	4839	int kvm_tdp_page_fault(struct kvm_vcpu vcpu, struct kvm_page_fault fault)
4852	4840	{
4853	4841	#ifdef CONFIG_X86_64
@@ -4858,8 +4846,7 @@ int kvm_tdp_page_fault(struct kvm_vcpu vcpu, struct kvm_page_fault fault) · 4 unmodified lines
4858	4846	return direct_page_fault(vcpu, fault);
4859	4847	}
4860	4848
4861		−static int kvm_tdp_map_page(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code,
4862		− u8 *level)
	4849	+int kvm_tdp_map_page(struct kvm_vcpu vcpu, gpa_t gpa, u64 error_code, u8 level)
4863	4850	{
4864	4851	int r;
4865	4852
@@ -4873,6 +4860,10 @@ static int kvm_tdp_map_page(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code, · 7 unmodified lines
4873	4860	do {
4874	4861	if (signal_pending(current))
4875	4862	return -EINTR;
	4863	+
	4864	+ if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu))
	4865	+ return -EIO;
	4866	+
4876	4867	cond_resched();
4877	4868	r = kvm_mmu_do_page_fault(vcpu, gpa, error_code, true, NULL, level);
4878	4869	} while (r == RET_PF_RETRY);
@@ -4897,6 +4888,7 @@ static int kvm_tdp_map_page(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code, · 18 unmodified lines
4897	4888	return -EIO;
4898	4889	}
4899	4890	}
	4891	+EXPORT_SYMBOL_GPL(kvm_tdp_map_page);
4900	4892
4901	4893	long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
4902	4894	struct kvm_pre_fault_memory *range)
@@ -5589,12 +5581,19 @@ void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu, · 686 unmodified lines
5589	5581
5590	5582	static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu)
5591	5583	{
	5584	+ int maxpa;
	5585	+
	5586	+ if (vcpu->kvm->arch.vm_type == KVM_X86_TDX_VM)
	5587	+ maxpa = cpuid_query_maxguestphyaddr(vcpu);
	5588	+ else
	5589	+ maxpa = cpuid_maxphyaddr(vcpu);
	5590	+
5592	5591	/* tdp_root_level is architecture forced level, use it if nonzero */
5593	5592	if (tdp_root_level)
5594	5593	return tdp_root_level;
5595	5594
5596	5595	/* Use 5-level TDP if and only if it's useful/necessary. */
5597		− if (max_tdp_level == 5 && cpuid_maxphyaddr(vcpu) <= 48)
	5596	+ if (max_tdp_level == 5 && maxpa <= 48)
5598	5597	return 4;
5599	5598
5600	5599	return max_tdp_level;
@@ -5913,6 +5912,7 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu) · 312 unmodified lines
5913	5912	out:
5914	5913	return r;
5915	5914	}
	5915	+EXPORT_SYMBOL_GPL(kvm_mmu_load);
5916	5916
5917	5917	void kvm_mmu_unload(struct kvm_vcpu *vcpu)
5918	5918	{
@@ -7238,6 +7238,7 @@ static void kvm_mmu_zap_memslot(struct kvm *kvm, · 1319 unmodified lines
7238	7238	.start = slot->base_gfn,
7239	7239	.end = slot->base_gfn + slot->npages,
7240	7240	.may_block = true,
	7241	+ .attr_filter = KVM_FILTER_PRIVATE \| KVM_FILTER_SHARED,
7241	7242	};
7242	7243	bool flush;
7243	7244

arch/x86/kvm/mmu/mmu_internal.h+3−2

@@ -187,7 +187,8 @@ static inline gfn_t kvm_gfn_root_bits(const struct kvm *kvm, const struct kvm_mm · 186 unmodified lines
187	187	return kvm_gfn_direct_bits(kvm);
188	188	}
189	189
190		−static inline bool kvm_mmu_page_ad_need_write_protect(struct kvm_mmu_page *sp)
	190	+static inline bool kvm_mmu_page_ad_need_write_protect(struct kvm *kvm,
	191	+ struct kvm_mmu_page *sp)
191	192	{
192	193	/*
193	194	* When using the EPT page-modification log, the GPAs in the CPU dirty
@@ -197,7 +198,7 @@ static inline bool kvm_mmu_page_ad_need_write_protect(struct kvm_mmu_page *sp) · 3 unmodified lines
197	198	* being enabled is mandatory as the bits used to denote WP-only SPTEs
198	199	* are reserved for PAE paging (32-bit KVM).
199	200	*/
200		− return kvm_x86_ops.cpu_dirty_log_size && sp->role.guest_mode;
	201	+ return kvm->arch.cpu_dirty_log_size && sp->role.guest_mode;
201	202	}
202	203
203	204	static inline gfn_t gfn_round_for_level(gfn_t gfn, int level)

arch/x86/kvm/mmu/page_track.c+3−0

@@ -172,6 +172,9 @@ static int kvm_enable_external_write_tracking(struct kvm *kvm) · 171 unmodified lines
172	172	struct kvm_memory_slot *slot;
173	173	int r = 0, i, bkt;
174	174
	175	+ if (kvm->arch.vm_type == KVM_X86_TDX_VM)
	176	+ return -EOPNOTSUPP;
	177	+
175	178	mutex_lock(&kvm->slots_arch_lock);
176	179
177	180	/*

arch/x86/kvm/mmu/spte.c+9−20

@@ -37,7 +37,6 @@ u64 __read_mostly shadow_mmio_value; · 36 unmodified lines
37	37	u64 __read_mostly shadow_mmio_mask;
38	38	u64 __read_mostly shadow_mmio_access_mask;
39	39	u64 __read_mostly shadow_present_mask;
40		−u64 __read_mostly shadow_memtype_mask;
41	40	u64 __read_mostly shadow_me_value;
42	41	u64 __read_mostly shadow_me_mask;
43	42	u64 __read_mostly shadow_acc_track_mask;
@@ -96,8 +95,6 @@ u64 make_mmio_spte(struct kvm_vcpu *vcpu, u64 gfn, unsigned int access) · 52 unmodified lines
96	95	u64 spte = generation_mmio_spte_mask(gen);
97	96	u64 gpa = gfn << PAGE_SHIFT;
98	97
99		− WARN_ON_ONCE(!vcpu->kvm->arch.shadow_mmio_value);
100		−
101	98	access &= shadow_mmio_access_mask;
102	99	spte \|= vcpu->kvm->arch.shadow_mmio_value \| access;
103	100	spte \|= gpa \| shadow_nonpresent_or_rsvd_mask;
@@ -177,7 +174,7 @@ bool make_spte(struct kvm_vcpu vcpu, struct kvm_mmu_page sp, · 73 unmodified lines
177	174
178	175	if (sp->role.ad_disabled)
179	176	spte \|= SPTE_TDP_AD_DISABLED;
180		− else if (kvm_mmu_page_ad_need_write_protect(sp))
	177	+ else if (kvm_mmu_page_ad_need_write_protect(vcpu->kvm, sp))
181	178	spte \|= SPTE_TDP_AD_WRPROT_ONLY;
182	179
183	180	spte \|= shadow_present_mask;
@@ -212,9 +209,7 @@ bool make_spte(struct kvm_vcpu vcpu, struct kvm_mmu_page sp, · 28 unmodified lines
212	209	if (level > PG_LEVEL_4K)
213	210	spte \|= PT_PAGE_SIZE_MASK;
214	211
215		− if (shadow_memtype_mask)
216		− spte \|= kvm_x86_call(get_mt_mask)(vcpu, gfn,
217		− kvm_is_mmio_pfn(pfn));
	212	+ spte \|= kvm_x86_call(get_mt_mask)(vcpu, gfn, kvm_is_mmio_pfn(pfn));
218	213	if (host_writable)
219	214	spte \|= shadow_host_writable_mask;
220	215	else
@@ -440,6 +435,12 @@ void kvm_mmu_set_mmio_spte_mask(u64 mmio_value, u64 mmio_mask, u64 access_mask) · 219 unmodified lines
440	435	}
441	436	EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask);
442	437
	438	+void kvm_mmu_set_mmio_spte_value(struct kvm *kvm, u64 mmio_value)
	439	+{
	440	+ kvm->arch.shadow_mmio_value = mmio_value;
	441	+}
	442	+EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_value);
	443	+
443	444	void kvm_mmu_set_me_spte_mask(u64 me_value, u64 me_mask)
444	445	{
445	446	/* shadow_me_value must be a subset of shadow_me_mask */
@@ -463,13 +464,7 @@ void kvm_mmu_set_ept_masks(bool has_ad_bits, bool has_exec_only) · 17 unmodified lines
463	464	/* VMX_EPT_SUPPRESS_VE_BIT is needed for W or X violation. */
464	465	shadow_present_mask =
465	466	(has_exec_only ? 0ull : VMX_EPT_READABLE_MASK) \| VMX_EPT_SUPPRESS_VE_BIT;
466		− /*
467		− * EPT overrides the host MTRRs, and so KVM must program the desired
468		− * memtype directly into the SPTEs. Note, this mask is just the mask
469		− * of all bits that factor into the memtype, the actual memtype must be
470		− * dynamically calculated, e.g. to ensure host MMIO is mapped UC.
471		− */
472		− shadow_memtype_mask = VMX_EPT_MT_MASK \| VMX_EPT_IPAT_BIT;
	467	+
473	468	shadow_acc_track_mask = VMX_EPT_RWX_MASK;
474	469	shadow_host_writable_mask = EPT_SPTE_HOST_WRITABLE;
475	470	shadow_mmu_writable_mask = EPT_SPTE_MMU_WRITABLE;
@@ -521,12 +516,6 @@ void kvm_mmu_reset_all_pte_masks(void) · 45 unmodified lines
521	516	shadow_x_mask = 0;
522	517	shadow_present_mask = PT_PRESENT_MASK;
523	518
524		− /*
525		− * For shadow paging and NPT, KVM uses PAT entry '0' to encode WB
526		− * memtype in the SPTEs, i.e. relies on host MTRRs to provide the
527		− * correct memtype (WB is the "weakest" memtype).
528		− */
529		− shadow_memtype_mask = 0;
530	519	shadow_acc_track_mask = 0;
531	520	shadow_me_mask = 0;
532	521	shadow_me_value = 0;

arch/x86/kvm/mmu/spte.h+0−1

@@ -187,7 +187,6 @@ extern u64 __read_mostly shadow_mmio_value; · 186 unmodified lines
187	187	extern u64 __read_mostly shadow_mmio_mask;
188	188	extern u64 __read_mostly shadow_mmio_access_mask;
189	189	extern u64 __read_mostly shadow_present_mask;
190		−extern u64 __read_mostly shadow_memtype_mask;
191	190	extern u64 __read_mostly shadow_me_value;
192	191	extern u64 __read_mostly shadow_me_mask;
193	192

arch/x86/kvm/mmu/tdp_mmu.c+38−11

@@ -1630,21 +1630,21 @@ void kvm_tdp_mmu_try_split_huge_pages(struct kvm *kvm, · 1629 unmodified lines
1630	1630	}
1631	1631	}
1632	1632
1633		−static bool tdp_mmu_need_write_protect(struct kvm_mmu_page *sp)
	1633	+static bool tdp_mmu_need_write_protect(struct kvm kvm, struct kvm_mmu_page sp)
1634	1634	{
1635	1635	/*
1636	1636	* All TDP MMU shadow pages share the same role as their root, aside
1637	1637	* from level, so it is valid to key off any shadow page to determine if
1638	1638	* write protection is needed for an entire tree.
1639	1639	*/
1640		− return kvm_mmu_page_ad_need_write_protect(sp) \|\| !kvm_ad_enabled;
	1640	+ return kvm_mmu_page_ad_need_write_protect(kvm, sp) \|\| !kvm_ad_enabled;
1641	1641	}
1642	1642
1643	1643	static void clear_dirty_gfn_range(struct kvm kvm, struct kvm_mmu_page root,
1644	1644	gfn_t start, gfn_t end)
1645	1645	{
1646		− const u64 dbit = tdp_mmu_need_write_protect(root) ? PT_WRITABLE_MASK :
1647		− shadow_dirty_mask;
	1646	+ const u64 dbit = tdp_mmu_need_write_protect(kvm, root) ?
	1647	+ PT_WRITABLE_MASK : shadow_dirty_mask;
1648	1648	struct tdp_iter iter;
1649	1649
1650	1650	rcu_read_lock();
@@ -1689,8 +1689,8 @@ void kvm_tdp_mmu_clear_dirty_slot(struct kvm *kvm, · 38 unmodified lines
1689	1689	static void clear_dirty_pt_masked(struct kvm kvm, struct kvm_mmu_page root,
1690	1690	gfn_t gfn, unsigned long mask, bool wrprot)
1691	1691	{
1692		− const u64 dbit = (wrprot \|\| tdp_mmu_need_write_protect(root)) ? PT_WRITABLE_MASK :
1693		− shadow_dirty_mask;
	1692	+ const u64 dbit = (wrprot \|\| tdp_mmu_need_write_protect(kvm, root)) ?
	1693	+ PT_WRITABLE_MASK : shadow_dirty_mask;
1694	1694	struct tdp_iter iter;
1695	1695
1696	1696	lockdep_assert_held_write(&kvm->mmu_lock);
@@ -1911,16 +1911,13 @@ bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm, · 214 unmodified lines
1911	1911	*
1912	1912	* Must be called between kvm_tdp_mmu_walk_lockless_{begin,end}.
1913	1913	*/
1914		−int kvm_tdp_mmu_get_walk(struct kvm_vcpu vcpu, u64 addr, u64 sptes,
1915		− int *root_level)
	1914	+static int __kvm_tdp_mmu_get_walk(struct kvm_vcpu vcpu, u64 addr, u64 sptes,
	1915	+ struct kvm_mmu_page *root)
1916	1916	{
1917		− struct kvm_mmu_page *root = root_to_sp(vcpu->arch.mmu->root.hpa);
1918	1917	struct tdp_iter iter;
1919	1918	gfn_t gfn = addr >> PAGE_SHIFT;
1920	1919	int leaf = -1;
1921	1920
1922		− *root_level = vcpu->arch.mmu->root_role.level;
1923		−
1924	1921	for_each_tdp_pte(iter, vcpu->kvm, root, gfn, gfn + 1) {
1925	1922	leaf = iter.level;
1926	1923	sptes[leaf] = iter.old_spte;
@@ -1929,6 +1926,36 @@ int kvm_tdp_mmu_get_walk(struct kvm_vcpu vcpu, u64 addr, u64 sptes, · 2 unmodified lines
1929	1926	return leaf;
1930	1927	}
1931	1928
	1929	+int kvm_tdp_mmu_get_walk(struct kvm_vcpu vcpu, u64 addr, u64 sptes,
	1930	+ int *root_level)
	1931	+{
	1932	+ struct kvm_mmu_page *root = root_to_sp(vcpu->arch.mmu->root.hpa);
	1933	+ *root_level = vcpu->arch.mmu->root_role.level;
	1934	+
	1935	+ return __kvm_tdp_mmu_get_walk(vcpu, addr, sptes, root);
	1936	+}
	1937	+
	1938	+bool kvm_tdp_mmu_gpa_is_mapped(struct kvm_vcpu *vcpu, u64 gpa)
	1939	+{
	1940	+ struct kvm *kvm = vcpu->kvm;
	1941	+ bool is_direct = kvm_is_addr_direct(kvm, gpa);
	1942	+ hpa_t root = is_direct ? vcpu->arch.mmu->root.hpa :
	1943	+ vcpu->arch.mmu->mirror_root_hpa;
	1944	+ u64 sptes[PT64_ROOT_MAX_LEVEL + 1], spte;
	1945	+ int leaf;
	1946	+
	1947	+ lockdep_assert_held(&kvm->mmu_lock);
	1948	+ rcu_read_lock();
	1949	+ leaf = __kvm_tdp_mmu_get_walk(vcpu, gpa, sptes, root_to_sp(root));
	1950	+ rcu_read_unlock();
	1951	+ if (leaf < 0)
	1952	+ return false;
	1953	+
	1954	+ spte = sptes[leaf];
	1955	+ return is_shadow_present_pte(spte) && is_last_spte(spte, leaf);
	1956	+}
	1957	+EXPORT_SYMBOL_GPL(kvm_tdp_mmu_gpa_is_mapped);
	1958	+
1932	1959	/*
1933	1960	* Returns the last level spte pointer of the shadow page walk for the given
1934	1961	* gpa, and sets *spte to the spte value. This spte may be non-preset. If no

arch/x86/kvm/smm.h+3−0

@@ -142,6 +142,9 @@ union kvm_smram { · 141 unmodified lines
142	142
143	143	static inline int kvm_inject_smi(struct kvm_vcpu *vcpu)
144	144	{
	145	+ if (!kvm_x86_call(has_emulated_msr)(vcpu->kvm, MSR_IA32_SMBASE))
	146	+ return -ENOTTY;
	147	+
145	148	kvm_make_request(KVM_REQ_SMI, vcpu);
146	149	return 0;
147	150	}

arch/x86/kvm/svm/svm.c+1−0

@@ -5501,6 +5501,7 @@ static __init int svm_hardware_setup(void) · 5500 unmodified lines
5501	5501	*/
5502	5502	allow_smaller_maxphyaddr = !npt_enabled;
5503	5503
	5504	+ kvm_caps.inapplicable_quirks &= ~KVM_X86_QUIRK_CD_NW_CLEARED;
5504	5505	return 0;
5505	5506
5506	5507	err:

arch/x86/kvm/vmx/common.h+182−0

@@ -0,0 +1,182 @@
	1	+/* SPDX-License-Identifier: GPL-2.0-only */
	2	+#ifndef __KVM_X86_VMX_COMMON_H
	3	+#define __KVM_X86_VMX_COMMON_H
	4	+
	5	+#include <linux/kvm_host.h>
	6	+#include <asm/posted_intr.h>
	7	+
	8	+#include "mmu.h"
	9	+
	10	+union vmx_exit_reason {
	11	+ struct {
	12	+ u32 basic : 16;
	13	+ u32 reserved16 : 1;
	14	+ u32 reserved17 : 1;
	15	+ u32 reserved18 : 1;
	16	+ u32 reserved19 : 1;
	17	+ u32 reserved20 : 1;
	18	+ u32 reserved21 : 1;
	19	+ u32 reserved22 : 1;
	20	+ u32 reserved23 : 1;
	21	+ u32 reserved24 : 1;
	22	+ u32 reserved25 : 1;
	23	+ u32 bus_lock_detected : 1;
	24	+ u32 enclave_mode : 1;
	25	+ u32 smi_pending_mtf : 1;
	26	+ u32 smi_from_vmx_root : 1;
	27	+ u32 reserved30 : 1;
	28	+ u32 failed_vmentry : 1;
	29	+ };
	30	+ u32 full;
	31	+};
	32	+
	33	+struct vcpu_vt {
	34	+ /* Posted interrupt descriptor */
	35	+ struct pi_desc pi_desc;
	36	+
	37	+ /* Used if this vCPU is waiting for PI notification wakeup. */
	38	+ struct list_head pi_wakeup_list;
	39	+
	40	+ union vmx_exit_reason exit_reason;
	41	+
	42	+ unsigned long exit_qualification;
	43	+ u32 exit_intr_info;
	44	+
	45	+ /*
	46	+ * If true, guest state has been loaded into hardware, and host state
	47	+ * saved into vcpu_{vt,vmx,tdx}. If false, host state is loaded into
	48	+ * hardware.
	49	+ */
	50	+ bool guest_state_loaded;
	51	+ bool emulation_required;
	52	+
	53	+#ifdef CONFIG_X86_64
	54	+ u64 msr_host_kernel_gs_base;
	55	+#endif
	56	+
	57	+ unsigned long host_debugctlmsr;
	58	+};
	59	+
	60	+#ifdef CONFIG_KVM_INTEL_TDX
	61	+
	62	+static __always_inline bool is_td(struct kvm *kvm)
	63	+{
	64	+ return kvm->arch.vm_type == KVM_X86_TDX_VM;
	65	+}
	66	+
	67	+static __always_inline bool is_td_vcpu(struct kvm_vcpu *vcpu)
	68	+{
	69	+ return is_td(vcpu->kvm);
	70	+}
	71	+
	72	+#else
	73	+
	74	+static inline bool is_td(struct kvm *kvm) { return false; }
	75	+static inline bool is_td_vcpu(struct kvm_vcpu *vcpu) { return false; }
	76	+
	77	+#endif
	78	+
	79	+static inline bool vt_is_tdx_private_gpa(struct kvm *kvm, gpa_t gpa)
	80	+{
	81	+ /* For TDX the direct mask is the shared mask. */
	82	+ return !kvm_is_addr_direct(kvm, gpa);
	83	+}
	84	+
	85	+static inline int __vmx_handle_ept_violation(struct kvm_vcpu *vcpu, gpa_t gpa,
	86	+ unsigned long exit_qualification)
	87	+{
	88	+ u64 error_code;
	89	+
	90	+ /* Is it a read fault? */
	91	+ error_code = (exit_qualification & EPT_VIOLATION_ACC_READ)
	92	+ ? PFERR_USER_MASK : 0;
	93	+ /* Is it a write fault? */
	94	+ error_code \|= (exit_qualification & EPT_VIOLATION_ACC_WRITE)
	95	+ ? PFERR_WRITE_MASK : 0;
	96	+ /* Is it a fetch fault? */
	97	+ error_code \|= (exit_qualification & EPT_VIOLATION_ACC_INSTR)
	98	+ ? PFERR_FETCH_MASK : 0;
	99	+ /* ept page table entry is present? */
	100	+ error_code \|= (exit_qualification & EPT_VIOLATION_PROT_MASK)
	101	+ ? PFERR_PRESENT_MASK : 0;
	102	+
	103	+ if (error_code & EPT_VIOLATION_GVA_IS_VALID)
	104	+ error_code \|= (exit_qualification & EPT_VIOLATION_GVA_TRANSLATED) ?
	105	+ PFERR_GUEST_FINAL_MASK : PFERR_GUEST_PAGE_MASK;
	106	+
	107	+ if (vt_is_tdx_private_gpa(vcpu->kvm, gpa))
	108	+ error_code \|= PFERR_PRIVATE_ACCESS;
	109	+
	110	+ return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
	111	+}
	112	+
	113	+static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu,
	114	+ int pi_vec)
	115	+{
	116	+#ifdef CONFIG_SMP
	117	+ if (vcpu->mode == IN_GUEST_MODE) {
	118	+ /*
	119	+ * The vector of the virtual has already been set in the PIR.
	120	+ * Send a notification event to deliver the virtual interrupt
	121	+ * unless the vCPU is the currently running vCPU, i.e. the
	122	+ * event is being sent from a fastpath VM-Exit handler, in
	123	+ * which case the PIR will be synced to the vIRR before
	124	+ * re-entering the guest.
	125	+ *
	126	+ * When the target is not the running vCPU, the following
	127	+ * possibilities emerge:
	128	+ *
	129	+ * Case 1: vCPU stays in non-root mode. Sending a notification
	130	+ * event posts the interrupt to the vCPU.
	131	+ *
	132	+ * Case 2: vCPU exits to root mode and is still runnable. The
	133	+ * PIR will be synced to the vIRR before re-entering the guest.
	134	+ * Sending a notification event is ok as the host IRQ handler
	135	+ * will ignore the spurious event.
	136	+ *
	137	+ * Case 3: vCPU exits to root mode and is blocked. vcpu_block()
	138	+ * has already synced PIR to vIRR and never blocks the vCPU if
	139	+ * the vIRR is not empty. Therefore, a blocked vCPU here does
	140	+ * not wait for any requested interrupts in PIR, and sending a
	141	+ * notification event also results in a benign, spurious event.
	142	+ */
	143	+
	144	+ if (vcpu != kvm_get_running_vcpu())
	145	+ __apic_send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec);
	146	+ return;
	147	+ }
	148	+#endif
	149	+ /*
	150	+ * The vCPU isn't in the guest; wake the vCPU in case it is blocking,
	151	+ * otherwise do nothing as KVM will grab the highest priority pending
	152	+ * IRQ via ->sync_pir_to_irr() in vcpu_enter_guest().
	153	+ */
	154	+ kvm_vcpu_wake_up(vcpu);
	155	+}
	156	+
	157	+/*
	158	+ * Post an interrupt to a vCPU's PIR and trigger the vCPU to process the
	159	+ * interrupt if necessary.
	160	+ */
	161	+static inline void __vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu,
	162	+ struct pi_desc *pi_desc, int vector)
	163	+{
	164	+ if (pi_test_and_set_pir(vector, pi_desc))
	165	+ return;
	166	+
	167	+ /* If a previous notification has sent the IPI, nothing to do. */
	168	+ if (pi_test_and_set_on(pi_desc))
	169	+ return;
	170	+
	171	+ /*
	172	+ * The implied barrier in pi_test_and_set_on() pairs with the smp_mb_*()
	173	+ * after setting vcpu->mode in vcpu_enter_guest(), thus the vCPU is
	174	+ * guaranteed to see PID.ON=1 and sync the PIR to IRR if triggering a
	175	+ * posted interrupt "fails" because vcpu->mode != IN_GUEST_MODE.
	176	+ */
	177	+ kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_VECTOR);
	178	+}
	179	+
	180	+noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu);
	181	+
	182	+#endif /* __KVM_X86_VMX_COMMON_H */

arch/x86/kvm/vmx/main.c+1034−91

@@ -3,9 +3,890 @@ · 2 unmodified lines
3	3
4	4	#include "x86_ops.h"
5	5	#include "vmx.h"
	6	+#include "mmu.h"
6	7	#include "nested.h"
7	8	#include "pmu.h"
8	9	#include "posted_intr.h"
	10	+#include "tdx.h"
	11	+#include "tdx_arch.h"
	12	+
	13	+#ifdef CONFIG_KVM_INTEL_TDX
	14	+static_assert(offsetof(struct vcpu_vmx, vt) == offsetof(struct vcpu_tdx, vt));
	15	+#endif
	16	+
	17	+static void vt_disable_virtualization_cpu(void)
	18	+{
	19	+ /* Note, TDX and VMX need to be disabled if TDX is enabled. */
	20	+ if (enable_tdx)
	21	+ tdx_disable_virtualization_cpu();
	22	+ vmx_disable_virtualization_cpu();
	23	+}
	24	+
	25	+static __init int vt_hardware_setup(void)
	26	+{
	27	+ int ret;
	28	+
	29	+ ret = vmx_hardware_setup();
	30	+ if (ret)
	31	+ return ret;
	32	+
	33	+ /*
	34	+ * Update vt_x86_ops::vm_size here so it is ready before
	35	+ * kvm_ops_update() is called in kvm_x86_vendor_init().
	36	+ *
	37	+ * Note, the actual bringing up of TDX must be done after
	38	+ * kvm_ops_update() because enabling TDX requires enabling
	39	+ * hardware virtualization first, i.e., all online CPUs must
	40	+ * be in post-VMXON state. This means the @vm_size here
	41	+ * may be updated to TDX's size but TDX may fail to enable
	42	+ * at later time.
	43	+ *
	44	+ * The VMX/VT code could update kvm_x86_ops::vm_size again
	45	+ * after bringing up TDX, but this would require exporting
	46	+ * either kvm_x86_ops or kvm_ops_update() from the base KVM
	47	+ * module, which looks overkill. Anyway, the worst case here
	48	+ * is KVM may allocate couple of more bytes than needed for
	49	+ * each VM.
	50	+ */
	51	+ if (enable_tdx) {
	52	+ vt_x86_ops.vm_size = max_t(unsigned int, vt_x86_ops.vm_size,
	53	+ sizeof(struct kvm_tdx));
	54	+ /*
	55	+ * Note, TDX may fail to initialize in a later time in
	56	+ * vt_init(), in which case it is not necessary to setup
	57	+ * those callbacks. But making them valid here even
	58	+ * when TDX fails to init later is fine because those
	59	+ * callbacks won't be called if the VM isn't TDX guest.
	60	+ */
	61	+ vt_x86_ops.link_external_spt = tdx_sept_link_private_spt;
	62	+ vt_x86_ops.set_external_spte = tdx_sept_set_private_spte;
	63	+ vt_x86_ops.free_external_spt = tdx_sept_free_private_spt;
	64	+ vt_x86_ops.remove_external_spte = tdx_sept_remove_private_spte;
	65	+ vt_x86_ops.protected_apic_has_interrupt = tdx_protected_apic_has_interrupt;
	66	+ }
	67	+
	68	+ return 0;
	69	+}
	70	+
	71	+static int vt_vm_init(struct kvm *kvm)
	72	+{
	73	+ if (is_td(kvm))
	74	+ return tdx_vm_init(kvm);
	75	+
	76	+ return vmx_vm_init(kvm);
	77	+}
	78	+
	79	+static void vt_vm_pre_destroy(struct kvm *kvm)
	80	+{
	81	+ if (is_td(kvm))
	82	+ return tdx_mmu_release_hkid(kvm);
	83	+}
	84	+
	85	+static void vt_vm_destroy(struct kvm *kvm)
	86	+{
	87	+ if (is_td(kvm))
	88	+ return tdx_vm_destroy(kvm);
	89	+
	90	+ vmx_vm_destroy(kvm);
	91	+}
	92	+
	93	+static int vt_vcpu_precreate(struct kvm *kvm)
	94	+{
	95	+ if (is_td(kvm))
	96	+ return 0;
	97	+
	98	+ return vmx_vcpu_precreate(kvm);
	99	+}
	100	+
	101	+static int vt_vcpu_create(struct kvm_vcpu *vcpu)
	102	+{
	103	+ if (is_td_vcpu(vcpu))
	104	+ return tdx_vcpu_create(vcpu);
	105	+
	106	+ return vmx_vcpu_create(vcpu);
	107	+}
	108	+
	109	+static void vt_vcpu_free(struct kvm_vcpu *vcpu)
	110	+{
	111	+ if (is_td_vcpu(vcpu)) {
	112	+ tdx_vcpu_free(vcpu);
	113	+ return;
	114	+ }
	115	+
	116	+ vmx_vcpu_free(vcpu);
	117	+}
	118	+
	119	+static void vt_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
	120	+{
	121	+ if (is_td_vcpu(vcpu)) {
	122	+ tdx_vcpu_reset(vcpu, init_event);
	123	+ return;
	124	+ }
	125	+
	126	+ vmx_vcpu_reset(vcpu, init_event);
	127	+}
	128	+
	129	+static void vt_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
	130	+{
	131	+ if (is_td_vcpu(vcpu)) {
	132	+ tdx_vcpu_load(vcpu, cpu);
	133	+ return;
	134	+ }
	135	+
	136	+ vmx_vcpu_load(vcpu, cpu);
	137	+}
	138	+
	139	+static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu)
	140	+{
	141	+ /*
	142	+ * Basic TDX does not support feature PML. KVM does not enable PML in
	143	+ * TD's VMCS, nor does it allocate or flush PML buffer for TDX.
	144	+ */
	145	+ if (WARN_ON_ONCE(is_td_vcpu(vcpu)))
	146	+ return;
	147	+
	148	+ vmx_update_cpu_dirty_logging(vcpu);
	149	+}
	150	+
	151	+static void vt_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
	152	+{
	153	+ if (is_td_vcpu(vcpu)) {
	154	+ tdx_prepare_switch_to_guest(vcpu);
	155	+ return;
	156	+ }
	157	+
	158	+ vmx_prepare_switch_to_guest(vcpu);
	159	+}
	160	+
	161	+static void vt_vcpu_put(struct kvm_vcpu *vcpu)
	162	+{
	163	+ if (is_td_vcpu(vcpu)) {
	164	+ tdx_vcpu_put(vcpu);
	165	+ return;
	166	+ }
	167	+
	168	+ vmx_vcpu_put(vcpu);
	169	+}
	170	+
	171	+static int vt_vcpu_pre_run(struct kvm_vcpu *vcpu)
	172	+{
	173	+ if (is_td_vcpu(vcpu))
	174	+ return tdx_vcpu_pre_run(vcpu);
	175	+
	176	+ return vmx_vcpu_pre_run(vcpu);
	177	+}
	178	+
	179	+static fastpath_t vt_vcpu_run(struct kvm_vcpu *vcpu, bool force_immediate_exit)
	180	+{
	181	+ if (is_td_vcpu(vcpu))
	182	+ return tdx_vcpu_run(vcpu, force_immediate_exit);
	183	+
	184	+ return vmx_vcpu_run(vcpu, force_immediate_exit);
	185	+}
	186	+
	187	+static int vt_handle_exit(struct kvm_vcpu *vcpu,
	188	+ enum exit_fastpath_completion fastpath)
	189	+{
	190	+ if (is_td_vcpu(vcpu))
	191	+ return tdx_handle_exit(vcpu, fastpath);
	192	+
	193	+ return vmx_handle_exit(vcpu, fastpath);
	194	+}
	195	+
	196	+static int vt_set_msr(struct kvm_vcpu vcpu, struct msr_data msr_info)
	197	+{
	198	+ if (unlikely(is_td_vcpu(vcpu)))
	199	+ return tdx_set_msr(vcpu, msr_info);
	200	+
	201	+ return vmx_set_msr(vcpu, msr_info);
	202	+}
	203	+
	204	+/*
	205	+ * The kvm parameter can be NULL (module initialization, or invocation before
	206	+ * VM creation). Be sure to check the kvm parameter before using it.
	207	+ */
	208	+static bool vt_has_emulated_msr(struct kvm *kvm, u32 index)
	209	+{
	210	+ if (kvm && is_td(kvm))
	211	+ return tdx_has_emulated_msr(index);
	212	+
	213	+ return vmx_has_emulated_msr(kvm, index);
	214	+}
	215	+
	216	+static int vt_get_msr(struct kvm_vcpu vcpu, struct msr_data msr_info)
	217	+{
	218	+ if (unlikely(is_td_vcpu(vcpu)))
	219	+ return tdx_get_msr(vcpu, msr_info);
	220	+
	221	+ return vmx_get_msr(vcpu, msr_info);
	222	+}
	223	+
	224	+static void vt_msr_filter_changed(struct kvm_vcpu *vcpu)
	225	+{
	226	+ /*
	227	+ * TDX doesn't allow VMM to configure interception of MSR accesses.
	228	+ * TDX guest requests MSR accesses by calling TDVMCALL. The MSR
	229	+ * filters will be applied when handling the TDVMCALL for RDMSR/WRMSR
	230	+ * if the userspace has set any.
	231	+ */
	232	+ if (is_td_vcpu(vcpu))
	233	+ return;
	234	+
	235	+ vmx_msr_filter_changed(vcpu);
	236	+}
	237	+
	238	+static int vt_complete_emulated_msr(struct kvm_vcpu *vcpu, int err)
	239	+{
	240	+ if (is_td_vcpu(vcpu))
	241	+ return tdx_complete_emulated_msr(vcpu, err);
	242	+
	243	+ return kvm_complete_insn_gp(vcpu, err);
	244	+}
	245	+
	246	+#ifdef CONFIG_KVM_SMM
	247	+static int vt_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
	248	+{
	249	+ if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm))
	250	+ return 0;
	251	+
	252	+ return vmx_smi_allowed(vcpu, for_injection);
	253	+}
	254	+
	255	+static int vt_enter_smm(struct kvm_vcpu vcpu, union kvm_smram smram)
	256	+{
	257	+ if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm))
	258	+ return 0;
	259	+
	260	+ return vmx_enter_smm(vcpu, smram);
	261	+}
	262	+
	263	+static int vt_leave_smm(struct kvm_vcpu vcpu, const union kvm_smram smram)
	264	+{
	265	+ if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm))
	266	+ return 0;
	267	+
	268	+ return vmx_leave_smm(vcpu, smram);
	269	+}
	270	+
	271	+static void vt_enable_smi_window(struct kvm_vcpu *vcpu)
	272	+{
	273	+ if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm))
	274	+ return;
	275	+
	276	+ /* RSM will cause a vmexit anyway. */
	277	+ vmx_enable_smi_window(vcpu);
	278	+}
	279	+#endif
	280	+
	281	+static int vt_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
	282	+ void *insn, int insn_len)
	283	+{
	284	+ /*
	285	+ * For TDX, this can only be triggered for MMIO emulation. Let the
	286	+ * guest retry after installing the SPTE with suppress #VE bit cleared,
	287	+ * so that the guest will receive #VE when retry. The guest is expected
	288	+ * to call TDG.VP.VMCALL<MMIO> to request VMM to do MMIO emulation on
	289	+ * #VE.
	290	+ */
	291	+ if (is_td_vcpu(vcpu))
	292	+ return X86EMUL_RETRY_INSTR;
	293	+
	294	+ return vmx_check_emulate_instruction(vcpu, emul_type, insn, insn_len);
	295	+}
	296	+
	297	+static bool vt_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
	298	+{
	299	+ /*
	300	+ * INIT and SIPI are always blocked for TDX, i.e., INIT handling and
	301	+ * the OP vcpu_deliver_sipi_vector() won't be called.
	302	+ */
	303	+ if (is_td_vcpu(vcpu))
	304	+ return true;
	305	+
	306	+ return vmx_apic_init_signal_blocked(vcpu);
	307	+}
	308	+
	309	+static void vt_set_virtual_apic_mode(struct kvm_vcpu *vcpu)
	310	+{
	311	+ /* Only x2APIC mode is supported for TD. */
	312	+ if (is_td_vcpu(vcpu))
	313	+ return;
	314	+
	315	+ return vmx_set_virtual_apic_mode(vcpu);
	316	+}
	317	+
	318	+static void vt_apicv_pre_state_restore(struct kvm_vcpu *vcpu)
	319	+{
	320	+ struct pi_desc *pi = vcpu_to_pi_desc(vcpu);
	321	+
	322	+ pi_clear_on(pi);
	323	+ memset(pi->pir, 0, sizeof(pi->pir));
	324	+}
	325	+
	326	+static void vt_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
	327	+{
	328	+ if (is_td_vcpu(vcpu))
	329	+ return;
	330	+
	331	+ return vmx_hwapic_isr_update(vcpu, max_isr);
	332	+}
	333	+
	334	+static int vt_sync_pir_to_irr(struct kvm_vcpu *vcpu)
	335	+{
	336	+ if (is_td_vcpu(vcpu))
	337	+ return -1;
	338	+
	339	+ return vmx_sync_pir_to_irr(vcpu);
	340	+}
	341	+
	342	+static void vt_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode,
	343	+ int trig_mode, int vector)
	344	+{
	345	+ if (is_td_vcpu(apic->vcpu)) {
	346	+ tdx_deliver_interrupt(apic, delivery_mode, trig_mode,
	347	+ vector);
	348	+ return;
	349	+ }
	350	+
	351	+ vmx_deliver_interrupt(apic, delivery_mode, trig_mode, vector);
	352	+}
	353	+
	354	+static void vt_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
	355	+{
	356	+ if (is_td_vcpu(vcpu))
	357	+ return;
	358	+
	359	+ vmx_vcpu_after_set_cpuid(vcpu);
	360	+}
	361	+
	362	+static void vt_update_exception_bitmap(struct kvm_vcpu *vcpu)
	363	+{
	364	+ if (is_td_vcpu(vcpu))
	365	+ return;
	366	+
	367	+ vmx_update_exception_bitmap(vcpu);
	368	+}
	369	+
	370	+static u64 vt_get_segment_base(struct kvm_vcpu *vcpu, int seg)
	371	+{
	372	+ if (is_td_vcpu(vcpu))
	373	+ return 0;
	374	+
	375	+ return vmx_get_segment_base(vcpu, seg);
	376	+}
	377	+
	378	+static void vt_get_segment(struct kvm_vcpu vcpu, struct kvm_segment var,
	379	+ int seg)
	380	+{
	381	+ if (is_td_vcpu(vcpu)) {
	382	+ memset(var, 0, sizeof(*var));
	383	+ return;
	384	+ }
	385	+
	386	+ vmx_get_segment(vcpu, var, seg);
	387	+}
	388	+
	389	+static void vt_set_segment(struct kvm_vcpu vcpu, struct kvm_segment var,
	390	+ int seg)
	391	+{
	392	+ if (is_td_vcpu(vcpu))
	393	+ return;
	394	+
	395	+ vmx_set_segment(vcpu, var, seg);
	396	+}
	397	+
	398	+static int vt_get_cpl(struct kvm_vcpu *vcpu)
	399	+{
	400	+ if (is_td_vcpu(vcpu))
	401	+ return 0;
	402	+
	403	+ return vmx_get_cpl(vcpu);
	404	+}
	405	+
	406	+static int vt_get_cpl_no_cache(struct kvm_vcpu *vcpu)
	407	+{
	408	+ if (is_td_vcpu(vcpu))
	409	+ return 0;
	410	+
	411	+ return vmx_get_cpl_no_cache(vcpu);
	412	+}
	413	+
	414	+static void vt_get_cs_db_l_bits(struct kvm_vcpu vcpu, int db, int *l)
	415	+{
	416	+ if (is_td_vcpu(vcpu)) {
	417	+ *db = 0;
	418	+ *l = 0;
	419	+ return;
	420	+ }
	421	+
	422	+ vmx_get_cs_db_l_bits(vcpu, db, l);
	423	+}
	424	+
	425	+static bool vt_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
	426	+{
	427	+ if (is_td_vcpu(vcpu))
	428	+ return true;
	429	+
	430	+ return vmx_is_valid_cr0(vcpu, cr0);
	431	+}
	432	+
	433	+static void vt_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
	434	+{
	435	+ if (is_td_vcpu(vcpu))
	436	+ return;
	437	+
	438	+ vmx_set_cr0(vcpu, cr0);
	439	+}
	440	+
	441	+static bool vt_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
	442	+{
	443	+ if (is_td_vcpu(vcpu))
	444	+ return true;
	445	+
	446	+ return vmx_is_valid_cr4(vcpu, cr4);
	447	+}
	448	+
	449	+static void vt_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
	450	+{
	451	+ if (is_td_vcpu(vcpu))
	452	+ return;
	453	+
	454	+ vmx_set_cr4(vcpu, cr4);
	455	+}
	456	+
	457	+static int vt_set_efer(struct kvm_vcpu *vcpu, u64 efer)
	458	+{
	459	+ if (is_td_vcpu(vcpu))
	460	+ return 0;
	461	+
	462	+ return vmx_set_efer(vcpu, efer);
	463	+}
	464	+
	465	+static void vt_get_idt(struct kvm_vcpu vcpu, struct desc_ptr dt)
	466	+{
	467	+ if (is_td_vcpu(vcpu)) {
	468	+ memset(dt, 0, sizeof(*dt));
	469	+ return;
	470	+ }
	471	+
	472	+ vmx_get_idt(vcpu, dt);
	473	+}
	474	+
	475	+static void vt_set_idt(struct kvm_vcpu vcpu, struct desc_ptr dt)
	476	+{
	477	+ if (is_td_vcpu(vcpu))
	478	+ return;
	479	+
	480	+ vmx_set_idt(vcpu, dt);
	481	+}
	482	+
	483	+static void vt_get_gdt(struct kvm_vcpu vcpu, struct desc_ptr dt)
	484	+{
	485	+ if (is_td_vcpu(vcpu)) {
	486	+ memset(dt, 0, sizeof(*dt));
	487	+ return;
	488	+ }
	489	+
	490	+ vmx_get_gdt(vcpu, dt);
	491	+}
	492	+
	493	+static void vt_set_gdt(struct kvm_vcpu vcpu, struct desc_ptr dt)
	494	+{
	495	+ if (is_td_vcpu(vcpu))
	496	+ return;
	497	+
	498	+ vmx_set_gdt(vcpu, dt);
	499	+}
	500	+
	501	+static void vt_set_dr6(struct kvm_vcpu *vcpu, unsigned long val)
	502	+{
	503	+ if (is_td_vcpu(vcpu))
	504	+ return;
	505	+
	506	+ vmx_set_dr6(vcpu, val);
	507	+}
	508	+
	509	+static void vt_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
	510	+{
	511	+ if (is_td_vcpu(vcpu))
	512	+ return;
	513	+
	514	+ vmx_set_dr7(vcpu, val);
	515	+}
	516	+
	517	+static void vt_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
	518	+{
	519	+ /*
	520	+ * MOV-DR exiting is always cleared for TD guest, even in debug mode.
	521	+ * Thus KVM_DEBUGREG_WONT_EXIT can never be set and it should never
	522	+ * reach here for TD vcpu.
	523	+ */
	524	+ if (is_td_vcpu(vcpu))
	525	+ return;
	526	+
	527	+ vmx_sync_dirty_debug_regs(vcpu);
	528	+}
	529	+
	530	+static void vt_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
	531	+{
	532	+ if (WARN_ON_ONCE(is_td_vcpu(vcpu)))
	533	+ return;
	534	+
	535	+ vmx_cache_reg(vcpu, reg);
	536	+}
	537	+
	538	+static unsigned long vt_get_rflags(struct kvm_vcpu *vcpu)
	539	+{
	540	+ if (is_td_vcpu(vcpu))
	541	+ return 0;
	542	+
	543	+ return vmx_get_rflags(vcpu);
	544	+}
	545	+
	546	+static void vt_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
	547	+{
	548	+ if (is_td_vcpu(vcpu))
	549	+ return;
	550	+
	551	+ vmx_set_rflags(vcpu, rflags);
	552	+}
	553	+
	554	+static bool vt_get_if_flag(struct kvm_vcpu *vcpu)
	555	+{
	556	+ if (is_td_vcpu(vcpu))
	557	+ return false;
	558	+
	559	+ return vmx_get_if_flag(vcpu);
	560	+}
	561	+
	562	+static void vt_flush_tlb_all(struct kvm_vcpu *vcpu)
	563	+{
	564	+ if (is_td_vcpu(vcpu)) {
	565	+ tdx_flush_tlb_all(vcpu);
	566	+ return;
	567	+ }
	568	+
	569	+ vmx_flush_tlb_all(vcpu);
	570	+}
	571	+
	572	+static void vt_flush_tlb_current(struct kvm_vcpu *vcpu)
	573	+{
	574	+ if (is_td_vcpu(vcpu)) {
	575	+ tdx_flush_tlb_current(vcpu);
	576	+ return;
	577	+ }
	578	+
	579	+ vmx_flush_tlb_current(vcpu);
	580	+}
	581	+
	582	+static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr)
	583	+{
	584	+ if (is_td_vcpu(vcpu))
	585	+ return;
	586	+
	587	+ vmx_flush_tlb_gva(vcpu, addr);
	588	+}
	589	+
	590	+static void vt_flush_tlb_guest(struct kvm_vcpu *vcpu)
	591	+{
	592	+ if (is_td_vcpu(vcpu))
	593	+ return;
	594	+
	595	+ vmx_flush_tlb_guest(vcpu);
	596	+}
	597	+
	598	+static void vt_inject_nmi(struct kvm_vcpu *vcpu)
	599	+{
	600	+ if (is_td_vcpu(vcpu)) {
	601	+ tdx_inject_nmi(vcpu);
	602	+ return;
	603	+ }
	604	+
	605	+ vmx_inject_nmi(vcpu);
	606	+}
	607	+
	608	+static int vt_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
	609	+{
	610	+ /*
	611	+ * The TDX module manages NMI windows and NMI reinjection, and hides NMI
	612	+ * blocking, all KVM can do is throw an NMI over the wall.
	613	+ */
	614	+ if (is_td_vcpu(vcpu))
	615	+ return true;
	616	+
	617	+ return vmx_nmi_allowed(vcpu, for_injection);
	618	+}
	619	+
	620	+static bool vt_get_nmi_mask(struct kvm_vcpu *vcpu)
	621	+{
	622	+ /*
	623	+ * KVM can't get NMI blocking status for TDX guest, assume NMIs are
	624	+ * always unmasked.
	625	+ */
	626	+ if (is_td_vcpu(vcpu))
	627	+ return false;
	628	+
	629	+ return vmx_get_nmi_mask(vcpu);
	630	+}
	631	+
	632	+static void vt_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
	633	+{
	634	+ if (is_td_vcpu(vcpu))
	635	+ return;
	636	+
	637	+ vmx_set_nmi_mask(vcpu, masked);
	638	+}
	639	+
	640	+static void vt_enable_nmi_window(struct kvm_vcpu *vcpu)
	641	+{
	642	+ /* Refer to the comments in tdx_inject_nmi(). */
	643	+ if (is_td_vcpu(vcpu))
	644	+ return;
	645	+
	646	+ vmx_enable_nmi_window(vcpu);
	647	+}
	648	+
	649	+static void vt_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa,
	650	+ int pgd_level)
	651	+{
	652	+ if (is_td_vcpu(vcpu)) {
	653	+ tdx_load_mmu_pgd(vcpu, root_hpa, pgd_level);
	654	+ return;
	655	+ }
	656	+
	657	+ vmx_load_mmu_pgd(vcpu, root_hpa, pgd_level);
	658	+}
	659	+
	660	+static void vt_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
	661	+{
	662	+ if (is_td_vcpu(vcpu))
	663	+ return;
	664	+
	665	+ vmx_set_interrupt_shadow(vcpu, mask);
	666	+}
	667	+
	668	+static u32 vt_get_interrupt_shadow(struct kvm_vcpu *vcpu)
	669	+{
	670	+ if (is_td_vcpu(vcpu))
	671	+ return 0;
	672	+
	673	+ return vmx_get_interrupt_shadow(vcpu);
	674	+}
	675	+
	676	+static void vt_patch_hypercall(struct kvm_vcpu *vcpu,
	677	+ unsigned char *hypercall)
	678	+{
	679	+ /*
	680	+ * Because guest memory is protected, guest can't be patched. TD kernel
	681	+ * is modified to use TDG.VP.VMCALL for hypercall.
	682	+ */
	683	+ if (is_td_vcpu(vcpu))
	684	+ return;
	685	+
	686	+ vmx_patch_hypercall(vcpu, hypercall);
	687	+}
	688	+
	689	+static void vt_inject_irq(struct kvm_vcpu *vcpu, bool reinjected)
	690	+{
	691	+ if (is_td_vcpu(vcpu))
	692	+ return;
	693	+
	694	+ vmx_inject_irq(vcpu, reinjected);
	695	+}
	696	+
	697	+static void vt_inject_exception(struct kvm_vcpu *vcpu)
	698	+{
	699	+ if (is_td_vcpu(vcpu))
	700	+ return;
	701	+
	702	+ vmx_inject_exception(vcpu);
	703	+}
	704	+
	705	+static void vt_cancel_injection(struct kvm_vcpu *vcpu)
	706	+{
	707	+ if (is_td_vcpu(vcpu))
	708	+ return;
	709	+
	710	+ vmx_cancel_injection(vcpu);
	711	+}
	712	+
	713	+static int vt_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
	714	+{
	715	+ if (is_td_vcpu(vcpu))
	716	+ return tdx_interrupt_allowed(vcpu);
	717	+
	718	+ return vmx_interrupt_allowed(vcpu, for_injection);
	719	+}
	720	+
	721	+static void vt_enable_irq_window(struct kvm_vcpu *vcpu)
	722	+{
	723	+ if (is_td_vcpu(vcpu))
	724	+ return;
	725	+
	726	+ vmx_enable_irq_window(vcpu);
	727	+}
	728	+
	729	+static void vt_get_entry_info(struct kvm_vcpu vcpu, u32 intr_info, u32 *error_code)
	730	+{
	731	+ *intr_info = 0;
	732	+ *error_code = 0;
	733	+
	734	+ if (is_td_vcpu(vcpu))
	735	+ return;
	736	+
	737	+ vmx_get_entry_info(vcpu, intr_info, error_code);
	738	+}
	739	+
	740	+static void vt_get_exit_info(struct kvm_vcpu vcpu, u32 reason,
	741	+ u64 info1, u64 info2, u32 intr_info, u32 error_code)
	742	+{
	743	+ if (is_td_vcpu(vcpu)) {
	744	+ tdx_get_exit_info(vcpu, reason, info1, info2, intr_info,
	745	+ error_code);
	746	+ return;
	747	+ }
	748	+
	749	+ vmx_get_exit_info(vcpu, reason, info1, info2, intr_info, error_code);
	750	+}
	751	+
	752	+static void vt_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
	753	+{
	754	+ if (is_td_vcpu(vcpu))
	755	+ return;
	756	+
	757	+ vmx_update_cr8_intercept(vcpu, tpr, irr);
	758	+}
	759	+
	760	+static void vt_set_apic_access_page_addr(struct kvm_vcpu *vcpu)
	761	+{
	762	+ if (is_td_vcpu(vcpu))
	763	+ return;
	764	+
	765	+ vmx_set_apic_access_page_addr(vcpu);
	766	+}
	767	+
	768	+static void vt_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
	769	+{
	770	+ if (is_td_vcpu(vcpu)) {
	771	+ KVM_BUG_ON(!kvm_vcpu_apicv_active(vcpu), vcpu->kvm);
	772	+ return;
	773	+ }
	774	+
	775	+ vmx_refresh_apicv_exec_ctrl(vcpu);
	776	+}
	777	+
	778	+static void vt_load_eoi_exitmap(struct kvm_vcpu vcpu, u64 eoi_exit_bitmap)
	779	+{
	780	+ if (is_td_vcpu(vcpu))
	781	+ return;
	782	+
	783	+ vmx_load_eoi_exitmap(vcpu, eoi_exit_bitmap);
	784	+}
	785	+
	786	+static int vt_set_tss_addr(struct kvm *kvm, unsigned int addr)
	787	+{
	788	+ if (is_td(kvm))
	789	+ return 0;
	790	+
	791	+ return vmx_set_tss_addr(kvm, addr);
	792	+}
	793	+
	794	+static int vt_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
	795	+{
	796	+ if (is_td(kvm))
	797	+ return 0;
	798	+
	799	+ return vmx_set_identity_map_addr(kvm, ident_addr);
	800	+}
	801	+
	802	+static u64 vt_get_l2_tsc_offset(struct kvm_vcpu *vcpu)
	803	+{
	804	+ /* TDX doesn't support L2 guest at the moment. */
	805	+ if (is_td_vcpu(vcpu))
	806	+ return 0;
	807	+
	808	+ return vmx_get_l2_tsc_offset(vcpu);
	809	+}
	810	+
	811	+static u64 vt_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu)
	812	+{
	813	+ /* TDX doesn't support L2 guest at the moment. */
	814	+ if (is_td_vcpu(vcpu))
	815	+ return 0;
	816	+
	817	+ return vmx_get_l2_tsc_multiplier(vcpu);
	818	+}
	819	+
	820	+static void vt_write_tsc_offset(struct kvm_vcpu *vcpu)
	821	+{
	822	+ /* In TDX, tsc offset can't be changed. */
	823	+ if (is_td_vcpu(vcpu))
	824	+ return;
	825	+
	826	+ vmx_write_tsc_offset(vcpu);
	827	+}
	828	+
	829	+static void vt_write_tsc_multiplier(struct kvm_vcpu *vcpu)
	830	+{
	831	+ /* In TDX, tsc multiplier can't be changed. */
	832	+ if (is_td_vcpu(vcpu))
	833	+ return;
	834	+
	835	+ vmx_write_tsc_multiplier(vcpu);
	836	+}
	837	+
	838	+#ifdef CONFIG_X86_64
	839	+static int vt_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc,
	840	+ bool *expired)
	841	+{
	842	+ /* VMX-preemption timer isn't available for TDX. */
	843	+ if (is_td_vcpu(vcpu))
	844	+ return -EINVAL;
	845	+
	846	+ return vmx_set_hv_timer(vcpu, guest_deadline_tsc, expired);
	847	+}
	848	+
	849	+static void vt_cancel_hv_timer(struct kvm_vcpu *vcpu)
	850	+{
	851	+ /* VMX-preemption timer can't be set. See vt_set_hv_timer(). */
	852	+ if (is_td_vcpu(vcpu))
	853	+ return;
	854	+
	855	+ vmx_cancel_hv_timer(vcpu);
	856	+}
	857	+#endif
	858	+
	859	+static void vt_setup_mce(struct kvm_vcpu *vcpu)
	860	+{
	861	+ if (is_td_vcpu(vcpu))
	862	+ return;
	863	+
	864	+ vmx_setup_mce(vcpu);
	865	+}
	866	+
	867	+static int vt_mem_enc_ioctl(struct kvm kvm, void __user argp)
	868	+{
	869	+ if (!is_td(kvm))
	870	+ return -ENOTTY;
	871	+
	872	+ return tdx_vm_ioctl(kvm, argp);
	873	+}
	874	+
	875	+static int vt_vcpu_mem_enc_ioctl(struct kvm_vcpu vcpu, void __user argp)
	876	+{
	877	+ if (!is_td_vcpu(vcpu))
	878	+ return -EINVAL;
	879	+
	880	+ return tdx_vcpu_ioctl(vcpu, argp);
	881	+}
	882	+
	883	+static int vt_gmem_private_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn)
	884	+{
	885	+ if (is_td(kvm))
	886	+ return tdx_gmem_private_max_mapping_level(kvm, pfn);
	887	+
	888	+ return 0;
	889	+}
9	890
10	891	#define VMX_REQUIRED_APICV_INHIBITS \
11	892	(BIT(APICV_INHIBIT_REASON_DISABLED) \| \
@@ -24,111 +905,113 @@ struct kvm_x86_ops vt_x86_ops __initdata = { · 12 unmodified lines
24	905	.hardware_unsetup = vmx_hardware_unsetup,
25	906
26	907	.enable_virtualization_cpu = vmx_enable_virtualization_cpu,
27		− .disable_virtualization_cpu = vmx_disable_virtualization_cpu,
	908	+ .disable_virtualization_cpu = vt_disable_virtualization_cpu,
28	909	.emergency_disable_virtualization_cpu = vmx_emergency_disable_virtualization_cpu,
29	910
30		− .has_emulated_msr = vmx_has_emulated_msr,
	911	+ .has_emulated_msr = vt_has_emulated_msr,
31	912
32	913	.vm_size = sizeof(struct kvm_vmx),
33		− .vm_init = vmx_vm_init,
34		− .vm_destroy = vmx_vm_destroy,
35	914
36		− .vcpu_precreate = vmx_vcpu_precreate,
37		− .vcpu_create = vmx_vcpu_create,
38		− .vcpu_free = vmx_vcpu_free,
39		− .vcpu_reset = vmx_vcpu_reset,
	915	+ .vm_init = vt_vm_init,
	916	+ .vm_pre_destroy = vt_vm_pre_destroy,
	917	+ .vm_destroy = vt_vm_destroy,
	918	+
	919	+ .vcpu_precreate = vt_vcpu_precreate,
	920	+ .vcpu_create = vt_vcpu_create,
	921	+ .vcpu_free = vt_vcpu_free,
	922	+ .vcpu_reset = vt_vcpu_reset,
40	923
41		− .prepare_switch_to_guest = vmx_prepare_switch_to_guest,
42		− .vcpu_load = vmx_vcpu_load,
43		− .vcpu_put = vmx_vcpu_put,
	924	+ .prepare_switch_to_guest = vt_prepare_switch_to_guest,
	925	+ .vcpu_load = vt_vcpu_load,
	926	+ .vcpu_put = vt_vcpu_put,
44	927
45		− .update_exception_bitmap = vmx_update_exception_bitmap,
	928	+ .update_exception_bitmap = vt_update_exception_bitmap,
46	929	.get_feature_msr = vmx_get_feature_msr,
47		− .get_msr = vmx_get_msr,
48		− .set_msr = vmx_set_msr,
49		− .get_segment_base = vmx_get_segment_base,
50		− .get_segment = vmx_get_segment,
51		− .set_segment = vmx_set_segment,
52		− .get_cpl = vmx_get_cpl,
53		− .get_cpl_no_cache = vmx_get_cpl_no_cache,
54		− .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
55		− .is_valid_cr0 = vmx_is_valid_cr0,
56		− .set_cr0 = vmx_set_cr0,
57		− .is_valid_cr4 = vmx_is_valid_cr4,
58		− .set_cr4 = vmx_set_cr4,
59		− .set_efer = vmx_set_efer,
60		− .get_idt = vmx_get_idt,
61		− .set_idt = vmx_set_idt,
62		− .get_gdt = vmx_get_gdt,
63		− .set_gdt = vmx_set_gdt,
64		− .set_dr6 = vmx_set_dr6,
65		− .set_dr7 = vmx_set_dr7,
66		− .sync_dirty_debug_regs = vmx_sync_dirty_debug_regs,
67		− .cache_reg = vmx_cache_reg,
68		− .get_rflags = vmx_get_rflags,
69		− .set_rflags = vmx_set_rflags,
70		− .get_if_flag = vmx_get_if_flag,
71		−
72		− .flush_tlb_all = vmx_flush_tlb_all,
73		− .flush_tlb_current = vmx_flush_tlb_current,
74		− .flush_tlb_gva = vmx_flush_tlb_gva,
75		− .flush_tlb_guest = vmx_flush_tlb_guest,
76		−
77		− .vcpu_pre_run = vmx_vcpu_pre_run,
78		− .vcpu_run = vmx_vcpu_run,
79		− .handle_exit = vmx_handle_exit,
	930	+ .get_msr = vt_get_msr,
	931	+ .set_msr = vt_set_msr,
	932	+
	933	+ .get_segment_base = vt_get_segment_base,
	934	+ .get_segment = vt_get_segment,
	935	+ .set_segment = vt_set_segment,
	936	+ .get_cpl = vt_get_cpl,
	937	+ .get_cpl_no_cache = vt_get_cpl_no_cache,
	938	+ .get_cs_db_l_bits = vt_get_cs_db_l_bits,
	939	+ .is_valid_cr0 = vt_is_valid_cr0,
	940	+ .set_cr0 = vt_set_cr0,
	941	+ .is_valid_cr4 = vt_is_valid_cr4,
	942	+ .set_cr4 = vt_set_cr4,
	943	+ .set_efer = vt_set_efer,
	944	+ .get_idt = vt_get_idt,
	945	+ .set_idt = vt_set_idt,
	946	+ .get_gdt = vt_get_gdt,
	947	+ .set_gdt = vt_set_gdt,
	948	+ .set_dr6 = vt_set_dr6,
	949	+ .set_dr7 = vt_set_dr7,
	950	+ .sync_dirty_debug_regs = vt_sync_dirty_debug_regs,
	951	+ .cache_reg = vt_cache_reg,
	952	+ .get_rflags = vt_get_rflags,
	953	+ .set_rflags = vt_set_rflags,
	954	+ .get_if_flag = vt_get_if_flag,
	955	+
	956	+ .flush_tlb_all = vt_flush_tlb_all,
	957	+ .flush_tlb_current = vt_flush_tlb_current,
	958	+ .flush_tlb_gva = vt_flush_tlb_gva,
	959	+ .flush_tlb_guest = vt_flush_tlb_guest,
	960	+
	961	+ .vcpu_pre_run = vt_vcpu_pre_run,
	962	+ .vcpu_run = vt_vcpu_run,
	963	+ .handle_exit = vt_handle_exit,
80	964	.skip_emulated_instruction = vmx_skip_emulated_instruction,
81	965	.update_emulated_instruction = vmx_update_emulated_instruction,
82		− .set_interrupt_shadow = vmx_set_interrupt_shadow,
83		− .get_interrupt_shadow = vmx_get_interrupt_shadow,
84		− .patch_hypercall = vmx_patch_hypercall,
85		− .inject_irq = vmx_inject_irq,
86		− .inject_nmi = vmx_inject_nmi,
87		− .inject_exception = vmx_inject_exception,
88		− .cancel_injection = vmx_cancel_injection,
89		− .interrupt_allowed = vmx_interrupt_allowed,
90		− .nmi_allowed = vmx_nmi_allowed,
91		− .get_nmi_mask = vmx_get_nmi_mask,
92		− .set_nmi_mask = vmx_set_nmi_mask,
93		− .enable_nmi_window = vmx_enable_nmi_window,
94		− .enable_irq_window = vmx_enable_irq_window,
95		− .update_cr8_intercept = vmx_update_cr8_intercept,
	966	+ .set_interrupt_shadow = vt_set_interrupt_shadow,
	967	+ .get_interrupt_shadow = vt_get_interrupt_shadow,
	968	+ .patch_hypercall = vt_patch_hypercall,
	969	+ .inject_irq = vt_inject_irq,
	970	+ .inject_nmi = vt_inject_nmi,
	971	+ .inject_exception = vt_inject_exception,
	972	+ .cancel_injection = vt_cancel_injection,
	973	+ .interrupt_allowed = vt_interrupt_allowed,
	974	+ .nmi_allowed = vt_nmi_allowed,
	975	+ .get_nmi_mask = vt_get_nmi_mask,
	976	+ .set_nmi_mask = vt_set_nmi_mask,
	977	+ .enable_nmi_window = vt_enable_nmi_window,
	978	+ .enable_irq_window = vt_enable_irq_window,
	979	+ .update_cr8_intercept = vt_update_cr8_intercept,
96	980
97	981	.x2apic_icr_is_split = false,
98		− .set_virtual_apic_mode = vmx_set_virtual_apic_mode,
99		− .set_apic_access_page_addr = vmx_set_apic_access_page_addr,
100		− .refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
101		− .load_eoi_exitmap = vmx_load_eoi_exitmap,
102		− .apicv_pre_state_restore = vmx_apicv_pre_state_restore,
	982	+ .set_virtual_apic_mode = vt_set_virtual_apic_mode,
	983	+ .set_apic_access_page_addr = vt_set_apic_access_page_addr,
	984	+ .refresh_apicv_exec_ctrl = vt_refresh_apicv_exec_ctrl,
	985	+ .load_eoi_exitmap = vt_load_eoi_exitmap,
	986	+ .apicv_pre_state_restore = vt_apicv_pre_state_restore,
103	987	.required_apicv_inhibits = VMX_REQUIRED_APICV_INHIBITS,
104		− .hwapic_isr_update = vmx_hwapic_isr_update,
105		− .sync_pir_to_irr = vmx_sync_pir_to_irr,
106		− .deliver_interrupt = vmx_deliver_interrupt,
	988	+ .hwapic_isr_update = vt_hwapic_isr_update,
	989	+ .sync_pir_to_irr = vt_sync_pir_to_irr,
	990	+ .deliver_interrupt = vt_deliver_interrupt,
107	991	.dy_apicv_has_pending_interrupt = pi_has_pending_interrupt,
108	992
109		− .set_tss_addr = vmx_set_tss_addr,
110		− .set_identity_map_addr = vmx_set_identity_map_addr,
	993	+ .set_tss_addr = vt_set_tss_addr,
	994	+ .set_identity_map_addr = vt_set_identity_map_addr,
111	995	.get_mt_mask = vmx_get_mt_mask,
112	996
113		− .get_exit_info = vmx_get_exit_info,
114		− .get_entry_info = vmx_get_entry_info,
	997	+ .get_exit_info = vt_get_exit_info,
	998	+ .get_entry_info = vt_get_entry_info,
115	999
116		− .vcpu_after_set_cpuid = vmx_vcpu_after_set_cpuid,
	1000	+ .vcpu_after_set_cpuid = vt_vcpu_after_set_cpuid,
117	1001
118	1002	.has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
119	1003
120		− .get_l2_tsc_offset = vmx_get_l2_tsc_offset,
121		− .get_l2_tsc_multiplier = vmx_get_l2_tsc_multiplier,
122		− .write_tsc_offset = vmx_write_tsc_offset,
123		− .write_tsc_multiplier = vmx_write_tsc_multiplier,
	1004	+ .get_l2_tsc_offset = vt_get_l2_tsc_offset,
	1005	+ .get_l2_tsc_multiplier = vt_get_l2_tsc_multiplier,
	1006	+ .write_tsc_offset = vt_write_tsc_offset,
	1007	+ .write_tsc_multiplier = vt_write_tsc_multiplier,
124	1008
125		− .load_mmu_pgd = vmx_load_mmu_pgd,
	1009	+ .load_mmu_pgd = vt_load_mmu_pgd,
126	1010
127	1011	.check_intercept = vmx_check_intercept,
128	1012	.handle_exit_irqoff = vmx_handle_exit_irqoff,
129	1013
130		− .cpu_dirty_log_size = PML_LOG_NR_ENTRIES,
131		− .update_cpu_dirty_logging = vmx_update_cpu_dirty_logging,
	1014	+ .update_cpu_dirty_logging = vt_update_cpu_dirty_logging,
132	1015
133	1016	.nested_ops = &vmx_nested_ops,
134	1017
@@ -136,35 +1019,95 @@ struct kvm_x86_ops vt_x86_ops __initdata = { · 1 unmodified line
136	1019	.pi_start_assignment = vmx_pi_start_assignment,
137	1020
138	1021	#ifdef CONFIG_X86_64
139		− .set_hv_timer = vmx_set_hv_timer,
140		− .cancel_hv_timer = vmx_cancel_hv_timer,
	1022	+ .set_hv_timer = vt_set_hv_timer,
	1023	+ .cancel_hv_timer = vt_cancel_hv_timer,
141	1024	#endif
142	1025
143		− .setup_mce = vmx_setup_mce,
	1026	+ .setup_mce = vt_setup_mce,
144	1027
145	1028	#ifdef CONFIG_KVM_SMM
146		− .smi_allowed = vmx_smi_allowed,
147		− .enter_smm = vmx_enter_smm,
148		− .leave_smm = vmx_leave_smm,
149		− .enable_smi_window = vmx_enable_smi_window,
	1029	+ .smi_allowed = vt_smi_allowed,
	1030	+ .enter_smm = vt_enter_smm,
	1031	+ .leave_smm = vt_leave_smm,
	1032	+ .enable_smi_window = vt_enable_smi_window,
150	1033	#endif
151	1034
152		− .check_emulate_instruction = vmx_check_emulate_instruction,
153		− .apic_init_signal_blocked = vmx_apic_init_signal_blocked,
	1035	+ .check_emulate_instruction = vt_check_emulate_instruction,
	1036	+ .apic_init_signal_blocked = vt_apic_init_signal_blocked,
154	1037	.migrate_timers = vmx_migrate_timers,
155	1038
156		− .msr_filter_changed = vmx_msr_filter_changed,
157		− .complete_emulated_msr = kvm_complete_insn_gp,
	1039	+ .msr_filter_changed = vt_msr_filter_changed,
	1040	+ .complete_emulated_msr = vt_complete_emulated_msr,
158	1041
159	1042	.vcpu_deliver_sipi_vector = kvm_vcpu_deliver_sipi_vector,
160	1043
161	1044	.get_untagged_addr = vmx_get_untagged_addr,
	1045	+
	1046	+ .mem_enc_ioctl = vt_mem_enc_ioctl,
	1047	+ .vcpu_mem_enc_ioctl = vt_vcpu_mem_enc_ioctl,
	1048	+
	1049	+ .private_max_mapping_level = vt_gmem_private_max_mapping_level
162	1050	};
163	1051
164	1052	struct kvm_x86_init_ops vt_init_ops __initdata = {
165		− .hardware_setup = vmx_hardware_setup,
	1053	+ .hardware_setup = vt_hardware_setup,
166	1054	.handle_intel_pt_intr = NULL,
167	1055
168	1056	.runtime_ops = &vt_x86_ops,
169	1057	.pmu_ops = &intel_pmu_ops,
170	1058	};
	1059	+
	1060	+static void __exit vt_exit(void)
	1061	+{
	1062	+ kvm_exit();
	1063	+ tdx_cleanup();
	1064	+ vmx_exit();
	1065	+}
	1066	+module_exit(vt_exit);
	1067	+
	1068	+static int __init vt_init(void)
	1069	+{
	1070	+ unsigned vcpu_size, vcpu_align;
	1071	+ int r;
	1072	+
	1073	+ r = vmx_init();
	1074	+ if (r)
	1075	+ return r;
	1076	+
	1077	+ /* tdx_init() has been taken */
	1078	+ r = tdx_bringup();
	1079	+ if (r)
	1080	+ goto err_tdx_bringup;
	1081	+
	1082	+ /*
	1083	+ * TDX and VMX have different vCPU structures. Calculate the
	1084	+ * maximum size/align so that kvm_init() can use the larger
	1085	+ * values to create the kmem_vcpu_cache.
	1086	+ */
	1087	+ vcpu_size = sizeof(struct vcpu_vmx);
	1088	+ vcpu_align = __alignof__(struct vcpu_vmx);
	1089	+ if (enable_tdx) {
	1090	+ vcpu_size = max_t(unsigned, vcpu_size,
	1091	+ sizeof(struct vcpu_tdx));
	1092	+ vcpu_align = max_t(unsigned, vcpu_align,
	1093	+ __alignof__(struct vcpu_tdx));
	1094	+ kvm_caps.supported_vm_types \|= BIT(KVM_X86_TDX_VM);
	1095	+ }
	1096	+
	1097	+ /*
	1098	+ * Common KVM initialization _must_ come last, after this, /dev/kvm is
	1099	+ * exposed to userspace!
	1100	+ */
	1101	+ r = kvm_init(vcpu_size, vcpu_align, THIS_MODULE);
	1102	+ if (r)
	1103	+ goto err_kvm_init;
	1104	+
	1105	+ return 0;
	1106	+
	1107	+err_kvm_init:
	1108	+ tdx_cleanup();
	1109	+err_tdx_bringup:
	1110	+ vmx_exit();
	1111	+ return r;
	1112	+}
	1113	+module_init(vt_init);

arch/x86/kvm/vmx/nested.c+6−6

@@ -275,7 +275,7 @@ static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx, · 274 unmodified lines
275	275	{
276	276	struct vmcs_host_state dest, src;
277	277
278		− if (unlikely(!vmx->guest_state_loaded))
	278	+ if (unlikely(!vmx->vt.guest_state_loaded))
279	279	return;
280	280
281	281	src = &prev->host_state;
@@ -425,7 +425,7 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, · 143 unmodified lines
425	425	* tables also changed, but KVM should not treat EPT Misconfig
426	426	* VM-Exits as writes.
427	427	*/
428		− WARN_ON_ONCE(vmx->exit_reason.basic != EXIT_REASON_EPT_VIOLATION);
	428	+ WARN_ON_ONCE(vmx->vt.exit_reason.basic != EXIT_REASON_EPT_VIOLATION);
429	429
430	430	/*
431	431	* PML Full and EPT Violation VM-Exits both use bit 12 to report
@@ -4622,7 +4622,7 @@ static void prepare_vmcs12(struct kvm_vcpu vcpu, struct vmcs12 vmcs12, · 4190 unmodified lines
4622	4622	{
4623	4623	/* update exit information fields: */
4624	4624	vmcs12->vm_exit_reason = vm_exit_reason;
4625		− if (to_vmx(vcpu)->exit_reason.enclave_mode)
	4625	+ if (vmx_get_exit_reason(vcpu).enclave_mode)
4626	4626	vmcs12->vm_exit_reason \|= VMX_EXIT_REASONS_SGX_ENCLAVE_MODE;
4627	4627	vmcs12->exit_qualification = exit_qualification;
4628	4628
@@ -4794,7 +4794,7 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, · 165 unmodified lines
4794	4794	vmcs12->vm_exit_msr_load_count))
4795	4795	nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
4796	4796
4797		− to_vmx(vcpu)->emulation_required = vmx_emulation_required(vcpu);
	4797	+ to_vt(vcpu)->emulation_required = vmx_emulation_required(vcpu);
4798	4798	}
4799	4799
4800	4800	static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx)
@@ -6127,7 +6127,7 @@ fail: · 1326 unmodified lines
6127	6127	* nested VM-Exit. Pass the original exit reason, i.e. don't hardcode
6128	6128	* EXIT_REASON_VMFUNC as the exit reason.
6129	6129	*/
6130		− nested_vmx_vmexit(vcpu, vmx->exit_reason.full,
	6130	+ nested_vmx_vmexit(vcpu, vmx->vt.exit_reason.full,
6131	6131	vmx_get_intr_info(vcpu),
6132	6132	vmx_get_exit_qual(vcpu));
6133	6133	return 1;
@@ -6572,7 +6572,7 @@ static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, · 438 unmodified lines
6572	6572	bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu)
6573	6573	{
6574	6574	struct vcpu_vmx *vmx = to_vmx(vcpu);
6575		− union vmx_exit_reason exit_reason = vmx->exit_reason;
	6575	+ union vmx_exit_reason exit_reason = vmx->vt.exit_reason;
6576	6576	unsigned long exit_qual;
6577	6577	u32 exit_intr_info;
6578	6578

arch/x86/kvm/vmx/pmu_intel.c+51−1

@@ -19,6 +19,7 @@ · 18 unmodified lines
19	19	#include "lapic.h"
20	20	#include "nested.h"
21	21	#include "pmu.h"
	22	+#include "tdx.h"
22	23
23	24	/*
24	25	* Perf's "BASE" is wildly misleading, architectural PMUs use bits 31:16 of ECX
@@ -34,6 +35,24 @@ · 9 unmodified lines
34	35
35	36	#define MSR_PMC_FULL_WIDTH_BIT (MSR_IA32_PMC0 - MSR_IA32_PERFCTR0)
36	37
	38	+static struct lbr_desc vcpu_to_lbr_desc(struct kvm_vcpu vcpu)
	39	+{
	40	+ if (is_td_vcpu(vcpu))
	41	+ return NULL;
	42	+
	43	+ return &to_vmx(vcpu)->lbr_desc;
	44	+}
	45	+
	46	+static struct x86_pmu_lbr vcpu_to_lbr_records(struct kvm_vcpu vcpu)
	47	+{
	48	+ if (is_td_vcpu(vcpu))
	49	+ return NULL;
	50	+
	51	+ return &to_vmx(vcpu)->lbr_desc.records;
	52	+}
	53	+
	54	+#pragma GCC poison to_vmx
	55	+
37	56	static void reprogram_fixed_counters(struct kvm_pmu *pmu, u64 data)
38	57	{
39	58	struct kvm_pmc *pmc;
@@ -129,6 +148,22 @@ static inline struct kvm_pmc get_fw_gp_pmc(struct kvm_pmu pmu, u32 msr) · 89 unmodified lines
129	148	return get_gp_pmc(pmu, msr, MSR_IA32_PMC0);
130	149	}
131	150
	151	+static bool intel_pmu_lbr_is_compatible(struct kvm_vcpu *vcpu)
	152	+{
	153	+ if (is_td_vcpu(vcpu))
	154	+ return false;
	155	+
	156	+ return cpuid_model_is_consistent(vcpu);
	157	+}
	158	+
	159	+bool intel_pmu_lbr_is_enabled(struct kvm_vcpu *vcpu)
	160	+{
	161	+ if (is_td_vcpu(vcpu))
	162	+ return false;
	163	+
	164	+ return !!vcpu_to_lbr_records(vcpu)->nr;
	165	+}
	166	+
132	167	static bool intel_pmu_is_valid_lbr_msr(struct kvm_vcpu *vcpu, u32 index)
133	168	{
134	169	struct x86_pmu_lbr *records = vcpu_to_lbr_records(vcpu);
@@ -194,6 +229,9 @@ static inline void intel_pmu_release_guest_lbr_event(struct kvm_vcpu *vcpu) · 59 unmodified lines
194	229	{
195	230	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
196	231
	232	+ if (!lbr_desc)
	233	+ return;
	234	+
197	235	if (lbr_desc->event) {
198	236	perf_event_release_kernel(lbr_desc->event);
199	237	lbr_desc->event = NULL;
@@ -235,6 +273,9 @@ int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu) · 35 unmodified lines
235	273	PERF_SAMPLE_BRANCH_USER,
236	274	};
237	275
	276	+ if (WARN_ON_ONCE(!lbr_desc))
	277	+ return 0;
	278	+
238	279	if (unlikely(lbr_desc->event)) {
239	280	__set_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use);
240	281	return 0;
@@ -466,6 +507,9 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) · 225 unmodified lines
466	507	u64 perf_capabilities;
467	508	u64 counter_rsvd;
468	509
	510	+ if (!lbr_desc)
	511	+ return;
	512	+
469	513	memset(&lbr_desc->records, 0, sizeof(lbr_desc->records));
470	514
471	515	/*
@@ -542,7 +586,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) · 70 unmodified lines
542	586	INTEL_PMC_MAX_GENERIC, pmu->nr_arch_fixed_counters);
543	587
544	588	perf_capabilities = vcpu_get_perf_capabilities(vcpu);
545		− if (cpuid_model_is_consistent(vcpu) &&
	589	+ if (intel_pmu_lbr_is_compatible(vcpu) &&
546	590	(perf_capabilities & PMU_CAP_LBR_FMT))
547	591	memcpy(&lbr_desc->records, &vmx_lbr_caps, sizeof(vmx_lbr_caps));
548	592	else
@@ -570,6 +614,9 @@ static void intel_pmu_init(struct kvm_vcpu *vcpu) · 21 unmodified lines
570	614	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
571	615	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
572	616
	617	+ if (!lbr_desc)
	618	+ return;
	619	+
573	620	for (i = 0; i < KVM_MAX_NR_INTEL_GP_COUNTERS; i++) {
574	621	pmu->gp_counters[i].type = KVM_PMC_GP;
575	622	pmu->gp_counters[i].vcpu = vcpu;
@@ -677,6 +724,9 @@ void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu) · 101 unmodified lines
677	724	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
678	725	struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
679	726
	727	+ if (WARN_ON_ONCE(!lbr_desc))
	728	+ return;
	729	+
680	730	if (!lbr_desc->event) {
681	731	vmx_disable_lbr_msrs_passthrough(vcpu);
682	732	if (vmcs_read64(GUEST_IA32_DEBUGCTL) & DEBUGCTLMSR_LBR)

arch/x86/kvm/vmx/pmu_intel.h+28−0

@@ -0,0 +1,28 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+#ifndef __KVM_X86_VMX_PMU_INTEL_H
	3	+#define __KVM_X86_VMX_PMU_INTEL_H
	4	+
	5	+#include <linux/kvm_host.h>
	6	+
	7	+bool intel_pmu_lbr_is_enabled(struct kvm_vcpu *vcpu);
	8	+int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu);
	9	+
	10	+struct lbr_desc {
	11	+ /* Basic info about guest LBR records. */
	12	+ struct x86_pmu_lbr records;
	13	+
	14	+ /*
	15	+ * Emulate LBR feature via passthrough LBR registers when the
	16	+ * per-vcpu guest LBR event is scheduled on the current pcpu.
	17	+ *
	18	+ * The records may be inaccurate if the host reclaims the LBR.
	19	+ */
	20	+ struct perf_event *event;
	21	+
	22	+ /* True if LBRs are marked as not intercepted in the MSR bitmap */
	23	+ bool msr_passthrough;
	24	+};
	25	+
	26	+extern struct x86_pmu_lbr vmx_lbr_caps;
	27	+
	28	+#endif /* __KVM_X86_VMX_PMU_INTEL_H */

arch/x86/kvm/vmx/posted_intr.c+16−12

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	11	#include "posted_intr.h"
12	12	#include "trace.h"
13	13	#include "vmx.h"
	14	+#include "tdx.h"
14	15
15	16	/*
16	17	* Maintain a per-CPU list of vCPUs that need to be awakened by wakeup_handler()
@@ -33,9 +34,9 @@ static DEFINE_PER_CPU(raw_spinlock_t, wakeup_vcpus_on_cpu_lock); · 16 unmodified lines
33	34
34	35	#define PI_LOCK_SCHED_OUT SINGLE_DEPTH_NESTING
35	36
36		−static inline struct pi_desc vcpu_to_pi_desc(struct kvm_vcpu vcpu)
	37	+struct pi_desc vcpu_to_pi_desc(struct kvm_vcpu vcpu)
37	38	{
38		− return &(to_vmx(vcpu)->pi_desc);
	39	+ return &(to_vt(vcpu)->pi_desc);
39	40	}
40	41
41	42	static int pi_try_set_control(struct pi_desc pi_desc, u64 pold, u64 new)
@@ -55,7 +56,7 @@ static int pi_try_set_control(struct pi_desc pi_desc, u64 pold, u64 new) · 13 unmodified lines
55	56	void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
56	57	{
57	58	struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
58		− struct vcpu_vmx *vmx = to_vmx(vcpu);
	59	+ struct vcpu_vt *vt = to_vt(vcpu);
59	60	struct pi_desc old, new;
60	61	unsigned long flags;
61	62	unsigned int dest;
@@ -102,7 +103,7 @@ void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) · 40 unmodified lines
102	103	*/
103	104	raw_spin_lock(spinlock);
104	105	spin_acquire(&spinlock->dep_map, PI_LOCK_SCHED_OUT, 0, _RET_IP_);
105		− list_del(&vmx->pi_wakeup_list);
	106	+ list_del(&vt->pi_wakeup_list);
106	107	spin_release(&spinlock->dep_map, _RET_IP_);
107	108	raw_spin_unlock(spinlock);
108	109	}
@@ -159,7 +160,7 @@ static bool vmx_can_use_vtd_pi(struct kvm *kvm) · 50 unmodified lines
159	160	static void pi_enable_wakeup_handler(struct kvm_vcpu *vcpu)
160	161	{
161	162	struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
162		− struct vcpu_vmx *vmx = to_vmx(vcpu);
	163	+ struct vcpu_vt *vt = to_vt(vcpu);
163	164	struct pi_desc old, new;
164	165
165	166	lockdep_assert_irqs_disabled();
@@ -178,7 +179,7 @@ static void pi_enable_wakeup_handler(struct kvm_vcpu *vcpu) · 12 unmodified lines
178	179	*/
179	180	raw_spin_lock_nested(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu),
180	181	PI_LOCK_SCHED_OUT);
181		− list_add_tail(&vmx->pi_wakeup_list,
	182	+ list_add_tail(&vt->pi_wakeup_list,
182	183	&per_cpu(wakeup_vcpus_on_cpu, vcpu->cpu));
183	184	raw_spin_unlock(&per_cpu(wakeup_vcpus_on_cpu_lock, vcpu->cpu));
184	185
@@ -213,7 +214,8 @@ static bool vmx_needs_pi_wakeup(struct kvm_vcpu *vcpu) · 28 unmodified lines
213	214	* notification vector is switched to the one that calls
214	215	* back to the pi_wakeup_handler() function.
215	216	*/
216		− return vmx_can_use_ipiv(vcpu) \|\| vmx_can_use_vtd_pi(vcpu->kvm);
	217	+ return (vmx_can_use_ipiv(vcpu) && !is_td_vcpu(vcpu)) \|\|
	218	+ vmx_can_use_vtd_pi(vcpu->kvm);
217	219	}
218	220
219	221	void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu)
@@ -223,7 +225,9 @@ void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu) · 3 unmodified lines
223	225	if (!vmx_needs_pi_wakeup(vcpu))
224	226	return;
225	227
226		− if (kvm_vcpu_is_blocking(vcpu) && !vmx_interrupt_blocked(vcpu))
	228	+ if (kvm_vcpu_is_blocking(vcpu) &&
	229	+ ((is_td_vcpu(vcpu) && tdx_interrupt_allowed(vcpu)) \|\|
	230	+ (!is_td_vcpu(vcpu) && !vmx_interrupt_blocked(vcpu))))
227	231	pi_enable_wakeup_handler(vcpu);
228	232
229	233	/*
@@ -243,13 +247,13 @@ void pi_wakeup_handler(void) · 13 unmodified lines
243	247	int cpu = smp_processor_id();
244	248	struct list_head *wakeup_list = &per_cpu(wakeup_vcpus_on_cpu, cpu);
245	249	raw_spinlock_t *spinlock = &per_cpu(wakeup_vcpus_on_cpu_lock, cpu);
246		− struct vcpu_vmx *vmx;
	250	+ struct vcpu_vt *vt;
247	251
248	252	raw_spin_lock(spinlock);
249		− list_for_each_entry(vmx, wakeup_list, pi_wakeup_list) {
	253	+ list_for_each_entry(vt, wakeup_list, pi_wakeup_list) {
250	254
251		− if (pi_test_on(&vmx->pi_desc))
252		− kvm_vcpu_wake_up(&vmx->vcpu);
	255	+ if (pi_test_on(&vt->pi_desc))
	256	+ kvm_vcpu_wake_up(vt_to_vcpu(vt));
253	257	}
254	258	raw_spin_unlock(spinlock);
255	259	}

arch/x86/kvm/vmx/posted_intr.h+2−0

@@ -5,6 +5,8 @@ · 4 unmodified lines
5	5	#include <linux/bitmap.h>
6	6	#include <asm/posted_intr.h>
7	7
	8	+struct pi_desc vcpu_to_pi_desc(struct kvm_vcpu vcpu);
	9	+
8	10	void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu);
9	11	void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu);
10	12	void pi_wakeup_handler(void);

arch/x86/kvm/vmx/tdx.c+3526−0

@@ -0,0 +1,3526 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+#include <linux/cleanup.h>
	3	+#include <linux/cpu.h>
	4	+#include <asm/cpufeature.h>
	5	+#include <asm/fpu/xcr.h>
	6	+#include <linux/misc_cgroup.h>
	7	+#include <linux/mmu_context.h>
	8	+#include <asm/tdx.h>
	9	+#include "capabilities.h"
	10	+#include "mmu.h"
	11	+#include "x86_ops.h"
	12	+#include "lapic.h"
	13	+#include "tdx.h"
	14	+#include "vmx.h"
	15	+#include "mmu/spte.h"
	16	+#include "common.h"
	17	+#include "posted_intr.h"
	18	+#include "irq.h"
	19	+#include <trace/events/kvm.h>
	20	+#include "trace.h"
	21	+
	22	+#pragma GCC poison to_vmx
	23	+
	24	+#undef pr_fmt
	25	+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	26	+
	27	+#define pr_tdx_error(__fn, __err) \
	28	+ pr_err_ratelimited("SEAMCALL %s failed: 0x%llx\n", #__fn, __err)
	29	+
	30	+#define __pr_tdx_error_N(__fn_str, __err, __fmt, ...) \
	31	+ pr_err_ratelimited("SEAMCALL " __fn_str " failed: 0x%llx, " __fmt, __err, __VA_ARGS__)
	32	+
	33	+#define pr_tdx_error_1(__fn, __err, __rcx) \
	34	+ __pr_tdx_error_N(#__fn, __err, "rcx 0x%llx\n", __rcx)
	35	+
	36	+#define pr_tdx_error_2(__fn, __err, __rcx, __rdx) \
	37	+ __pr_tdx_error_N(#__fn, __err, "rcx 0x%llx, rdx 0x%llx\n", __rcx, __rdx)
	38	+
	39	+#define pr_tdx_error_3(__fn, __err, __rcx, __rdx, __r8) \
	40	+ __pr_tdx_error_N(#__fn, __err, "rcx 0x%llx, rdx 0x%llx, r8 0x%llx\n", __rcx, __rdx, __r8)
	41	+
	42	+bool enable_tdx __ro_after_init;
	43	+module_param_named(tdx, enable_tdx, bool, 0444);
	44	+
	45	+#define TDX_SHARED_BIT_PWL_5 gpa_to_gfn(BIT_ULL(51))
	46	+#define TDX_SHARED_BIT_PWL_4 gpa_to_gfn(BIT_ULL(47))
	47	+
	48	+static enum cpuhp_state tdx_cpuhp_state;
	49	+
	50	+static const struct tdx_sys_info *tdx_sysinfo;
	51	+
	52	+void tdh_vp_rd_failed(struct vcpu_tdx tdx, char uclass, u32 field, u64 err)
	53	+{
	54	+ KVM_BUG_ON(1, tdx->vcpu.kvm);
	55	+ pr_err("TDH_VP_RD[%s.0x%x] failed 0x%llx\n", uclass, field, err);
	56	+}
	57	+
	58	+void tdh_vp_wr_failed(struct vcpu_tdx tdx, char uclass, char *op, u32 field,
	59	+ u64 val, u64 err)
	60	+{
	61	+ KVM_BUG_ON(1, tdx->vcpu.kvm);
	62	+ pr_err("TDH_VP_WR[%s.0x%x]%s0x%llx failed: 0x%llx\n", uclass, field, op, val, err);
	63	+}
	64	+
	65	+#define KVM_SUPPORTED_TD_ATTRS (TDX_TD_ATTR_SEPT_VE_DISABLE)
	66	+
	67	+static __always_inline struct kvm_tdx to_kvm_tdx(struct kvm kvm)
	68	+{
	69	+ return container_of(kvm, struct kvm_tdx, kvm);
	70	+}
	71	+
	72	+static __always_inline struct vcpu_tdx to_tdx(struct kvm_vcpu vcpu)
	73	+{
	74	+ return container_of(vcpu, struct vcpu_tdx, vcpu);
	75	+}
	76	+
	77	+static u64 tdx_get_supported_attrs(const struct tdx_sys_info_td_conf *td_conf)
	78	+{
	79	+ u64 val = KVM_SUPPORTED_TD_ATTRS;
	80	+
	81	+ if ((val & td_conf->attributes_fixed1) != td_conf->attributes_fixed1)
	82	+ return 0;
	83	+
	84	+ val &= td_conf->attributes_fixed0;
	85	+
	86	+ return val;
	87	+}
	88	+
	89	+static u64 tdx_get_supported_xfam(const struct tdx_sys_info_td_conf *td_conf)
	90	+{
	91	+ u64 val = kvm_caps.supported_xcr0 \| kvm_caps.supported_xss;
	92	+
	93	+ if ((val & td_conf->xfam_fixed1) != td_conf->xfam_fixed1)
	94	+ return 0;
	95	+
	96	+ val &= td_conf->xfam_fixed0;
	97	+
	98	+ return val;
	99	+}
	100	+
	101	+static int tdx_get_guest_phys_addr_bits(const u32 eax)
	102	+{
	103	+ return (eax & GENMASK(23, 16)) >> 16;
	104	+}
	105	+
	106	+static u32 tdx_set_guest_phys_addr_bits(const u32 eax, int addr_bits)
	107	+{
	108	+ return (eax & ~GENMASK(23, 16)) \| (addr_bits & 0xff) << 16;
	109	+}
	110	+
	111	+#define TDX_FEATURE_TSX (__feature_bit(X86_FEATURE_HLE) \| __feature_bit(X86_FEATURE_RTM))
	112	+
	113	+static bool has_tsx(const struct kvm_cpuid_entry2 *entry)
	114	+{
	115	+ return entry->function == 7 && entry->index == 0 &&
	116	+ (entry->ebx & TDX_FEATURE_TSX);
	117	+}
	118	+
	119	+static void clear_tsx(struct kvm_cpuid_entry2 *entry)
	120	+{
	121	+ entry->ebx &= ~TDX_FEATURE_TSX;
	122	+}
	123	+
	124	+static bool has_waitpkg(const struct kvm_cpuid_entry2 *entry)
	125	+{
	126	+ return entry->function == 7 && entry->index == 0 &&
	127	+ (entry->ecx & __feature_bit(X86_FEATURE_WAITPKG));
	128	+}
	129	+
	130	+static void clear_waitpkg(struct kvm_cpuid_entry2 *entry)
	131	+{
	132	+ entry->ecx &= ~__feature_bit(X86_FEATURE_WAITPKG);
	133	+}
	134	+
	135	+static void tdx_clear_unsupported_cpuid(struct kvm_cpuid_entry2 *entry)
	136	+{
	137	+ if (has_tsx(entry))
	138	+ clear_tsx(entry);
	139	+
	140	+ if (has_waitpkg(entry))
	141	+ clear_waitpkg(entry);
	142	+}
	143	+
	144	+static bool tdx_unsupported_cpuid(const struct kvm_cpuid_entry2 *entry)
	145	+{
	146	+ return has_tsx(entry) \|\| has_waitpkg(entry);
	147	+}
	148	+
	149	+#define KVM_TDX_CPUID_NO_SUBLEAF ((__u32)-1)
	150	+
	151	+static void td_init_cpuid_entry2(struct kvm_cpuid_entry2 *entry, unsigned char idx)
	152	+{
	153	+ const struct tdx_sys_info_td_conf *td_conf = &tdx_sysinfo->td_conf;
	154	+
	155	+ entry->function = (u32)td_conf->cpuid_config_leaves[idx];
	156	+ entry->index = td_conf->cpuid_config_leaves[idx] >> 32;
	157	+ entry->eax = (u32)td_conf->cpuid_config_values[idx][0];
	158	+ entry->ebx = td_conf->cpuid_config_values[idx][0] >> 32;
	159	+ entry->ecx = (u32)td_conf->cpuid_config_values[idx][1];
	160	+ entry->edx = td_conf->cpuid_config_values[idx][1] >> 32;
	161	+
	162	+ if (entry->index == KVM_TDX_CPUID_NO_SUBLEAF)
	163	+ entry->index = 0;
	164	+
	165	+ /*
	166	+ * The TDX module doesn't allow configuring the guest phys addr bits
	167	+ * (EAX[23:16]). However, KVM uses it as an interface to the userspace
	168	+ * to configure the GPAW. Report these bits as configurable.
	169	+ */
	170	+ if (entry->function == 0x80000008)
	171	+ entry->eax = tdx_set_guest_phys_addr_bits(entry->eax, 0xff);
	172	+
	173	+ tdx_clear_unsupported_cpuid(entry);
	174	+}
	175	+
	176	+static int init_kvm_tdx_caps(const struct tdx_sys_info_td_conf *td_conf,
	177	+ struct kvm_tdx_capabilities *caps)
	178	+{
	179	+ int i;
	180	+
	181	+ caps->supported_attrs = tdx_get_supported_attrs(td_conf);
	182	+ if (!caps->supported_attrs)
	183	+ return -EIO;
	184	+
	185	+ caps->supported_xfam = tdx_get_supported_xfam(td_conf);
	186	+ if (!caps->supported_xfam)
	187	+ return -EIO;
	188	+
	189	+ caps->cpuid.nent = td_conf->num_cpuid_config;
	190	+
	191	+ for (i = 0; i < td_conf->num_cpuid_config; i++)
	192	+ td_init_cpuid_entry2(&caps->cpuid.entries[i], i);
	193	+
	194	+ return 0;
	195	+}
	196	+
	197	+/*
	198	+ * Some SEAMCALLs acquire the TDX module globally, and can fail with
	199	+ * TDX_OPERAND_BUSY. Use a global mutex to serialize these SEAMCALLs.
	200	+ */
	201	+static DEFINE_MUTEX(tdx_lock);
	202	+
	203	+static atomic_t nr_configured_hkid;
	204	+
	205	+static bool tdx_operand_busy(u64 err)
	206	+{
	207	+ return (err & TDX_SEAMCALL_STATUS_MASK) == TDX_OPERAND_BUSY;
	208	+}
	209	+
	210	+
	211	+/*
	212	+ * A per-CPU list of TD vCPUs associated with a given CPU.
	213	+ * Protected by interrupt mask. Only manipulated by the CPU owning this per-CPU
	214	+ * list.
	215	+ * - When a vCPU is loaded onto a CPU, it is removed from the per-CPU list of
	216	+ * the old CPU during the IPI callback running on the old CPU, and then added
	217	+ * to the per-CPU list of the new CPU.
	218	+ * - When a TD is tearing down, all vCPUs are disassociated from their current
	219	+ * running CPUs and removed from the per-CPU list during the IPI callback
	220	+ * running on those CPUs.
	221	+ * - When a CPU is brought down, traverse the per-CPU list to disassociate all
	222	+ * associated TD vCPUs and remove them from the per-CPU list.
	223	+ */
	224	+static DEFINE_PER_CPU(struct list_head, associated_tdvcpus);
	225	+
	226	+static __always_inline unsigned long tdvmcall_exit_type(struct kvm_vcpu *vcpu)
	227	+{
	228	+ return to_tdx(vcpu)->vp_enter_args.r10;
	229	+}
	230	+
	231	+static __always_inline unsigned long tdvmcall_leaf(struct kvm_vcpu *vcpu)
	232	+{
	233	+ return to_tdx(vcpu)->vp_enter_args.r11;
	234	+}
	235	+
	236	+static __always_inline void tdvmcall_set_return_code(struct kvm_vcpu *vcpu,
	237	+ long val)
	238	+{
	239	+ to_tdx(vcpu)->vp_enter_args.r10 = val;
	240	+}
	241	+
	242	+static __always_inline void tdvmcall_set_return_val(struct kvm_vcpu *vcpu,
	243	+ unsigned long val)
	244	+{
	245	+ to_tdx(vcpu)->vp_enter_args.r11 = val;
	246	+}
	247	+
	248	+static inline void tdx_hkid_free(struct kvm_tdx *kvm_tdx)
	249	+{
	250	+ tdx_guest_keyid_free(kvm_tdx->hkid);
	251	+ kvm_tdx->hkid = -1;
	252	+ atomic_dec(&nr_configured_hkid);
	253	+ misc_cg_uncharge(MISC_CG_RES_TDX, kvm_tdx->misc_cg, 1);
	254	+ put_misc_cg(kvm_tdx->misc_cg);
	255	+ kvm_tdx->misc_cg = NULL;
	256	+}
	257	+
	258	+static inline bool is_hkid_assigned(struct kvm_tdx *kvm_tdx)
	259	+{
	260	+ return kvm_tdx->hkid > 0;
	261	+}
	262	+
	263	+static inline void tdx_disassociate_vp(struct kvm_vcpu *vcpu)
	264	+{
	265	+ lockdep_assert_irqs_disabled();
	266	+
	267	+ list_del(&to_tdx(vcpu)->cpu_list);
	268	+
	269	+ /*
	270	+ * Ensure tdx->cpu_list is updated before setting vcpu->cpu to -1,
	271	+ * otherwise, a different CPU can see vcpu->cpu = -1 and add the vCPU
	272	+ * to its list before it's deleted from this CPU's list.
	273	+ */
	274	+ smp_wmb();
	275	+
	276	+ vcpu->cpu = -1;
	277	+}
	278	+
	279	+static void tdx_clear_page(struct page *page)
	280	+{
	281	+ const void zero_page = (const void ) page_to_virt(ZERO_PAGE(0));
	282	+ void *dest = page_to_virt(page);
	283	+ unsigned long i;
	284	+
	285	+ /*
	286	+ * The page could have been poisoned. MOVDIR64B also clears
	287	+ * the poison bit so the kernel can safely use the page again.
	288	+ */
	289	+ for (i = 0; i < PAGE_SIZE; i += 64)
	290	+ movdir64b(dest + i, zero_page);
	291	+ /*
	292	+ * MOVDIR64B store uses WC buffer. Prevent following memory reads
	293	+ * from seeing potentially poisoned cache.
	294	+ */
	295	+ __mb();
	296	+}
	297	+
	298	+static void tdx_no_vcpus_enter_start(struct kvm *kvm)
	299	+{
	300	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	301	+
	302	+ lockdep_assert_held_write(&kvm->mmu_lock);
	303	+
	304	+ WRITE_ONCE(kvm_tdx->wait_for_sept_zap, true);
	305	+
	306	+ kvm_make_all_cpus_request(kvm, KVM_REQ_OUTSIDE_GUEST_MODE);
	307	+}
	308	+
	309	+static void tdx_no_vcpus_enter_stop(struct kvm *kvm)
	310	+{
	311	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	312	+
	313	+ lockdep_assert_held_write(&kvm->mmu_lock);
	314	+
	315	+ WRITE_ONCE(kvm_tdx->wait_for_sept_zap, false);
	316	+}
	317	+
	318	+/* TDH.PHYMEM.PAGE.RECLAIM is allowed only when destroying the TD. */
	319	+static int __tdx_reclaim_page(struct page *page)
	320	+{
	321	+ u64 err, rcx, rdx, r8;
	322	+
	323	+ err = tdh_phymem_page_reclaim(page, &rcx, &rdx, &r8);
	324	+
	325	+ /*
	326	+ * No need to check for TDX_OPERAND_BUSY; all TD pages are freed
	327	+ * before the HKID is released and control pages have also been
	328	+ * released at this point, so there is no possibility of contention.
	329	+ */
	330	+ if (WARN_ON_ONCE(err)) {
	331	+ pr_tdx_error_3(TDH_PHYMEM_PAGE_RECLAIM, err, rcx, rdx, r8);
	332	+ return -EIO;
	333	+ }
	334	+ return 0;
	335	+}
	336	+
	337	+static int tdx_reclaim_page(struct page *page)
	338	+{
	339	+ int r;
	340	+
	341	+ r = __tdx_reclaim_page(page);
	342	+ if (!r)
	343	+ tdx_clear_page(page);
	344	+ return r;
	345	+}
	346	+
	347	+
	348	+/*
	349	+ * Reclaim the TD control page(s) which are crypto-protected by TDX guest's
	350	+ * private KeyID. Assume the cache associated with the TDX private KeyID has
	351	+ * been flushed.
	352	+ */
	353	+static void tdx_reclaim_control_page(struct page *ctrl_page)
	354	+{
	355	+ /*
	356	+ * Leak the page if the kernel failed to reclaim the page.
	357	+ * The kernel cannot use it safely anymore.
	358	+ */
	359	+ if (tdx_reclaim_page(ctrl_page))
	360	+ return;
	361	+
	362	+ __free_page(ctrl_page);
	363	+}
	364	+
	365	+struct tdx_flush_vp_arg {
	366	+ struct kvm_vcpu *vcpu;
	367	+ u64 err;
	368	+};
	369	+
	370	+static void tdx_flush_vp(void *_arg)
	371	+{
	372	+ struct tdx_flush_vp_arg *arg = _arg;
	373	+ struct kvm_vcpu *vcpu = arg->vcpu;
	374	+ u64 err;
	375	+
	376	+ arg->err = 0;
	377	+ lockdep_assert_irqs_disabled();
	378	+
	379	+ /* Task migration can race with CPU offlining. */
	380	+ if (unlikely(vcpu->cpu != raw_smp_processor_id()))
	381	+ return;
	382	+
	383	+ /*
	384	+ * No need to do TDH_VP_FLUSH if the vCPU hasn't been initialized. The
	385	+ * list tracking still needs to be updated so that it's correct if/when
	386	+ * the vCPU does get initialized.
	387	+ */
	388	+ if (to_tdx(vcpu)->state != VCPU_TD_STATE_UNINITIALIZED) {
	389	+ /*
	390	+ * No need to retry. TDX Resources needed for TDH.VP.FLUSH are:
	391	+ * TDVPR as exclusive, TDR as shared, and TDCS as shared. This
	392	+ * vp flush function is called when destructing vCPU/TD or vCPU
	393	+ * migration. No other thread uses TDVPR in those cases.
	394	+ */
	395	+ err = tdh_vp_flush(&to_tdx(vcpu)->vp);
	396	+ if (unlikely(err && err != TDX_VCPU_NOT_ASSOCIATED)) {
	397	+ /*
	398	+ * This function is called in IPI context. Do not use
	399	+ * printk to avoid console semaphore.
	400	+ * The caller prints out the error message, instead.
	401	+ */
	402	+ if (err)
	403	+ arg->err = err;
	404	+ }
	405	+ }
	406	+
	407	+ tdx_disassociate_vp(vcpu);
	408	+}
	409	+
	410	+static void tdx_flush_vp_on_cpu(struct kvm_vcpu *vcpu)
	411	+{
	412	+ struct tdx_flush_vp_arg arg = {
	413	+ .vcpu = vcpu,
	414	+ };
	415	+ int cpu = vcpu->cpu;
	416	+
	417	+ if (unlikely(cpu == -1))
	418	+ return;
	419	+
	420	+ smp_call_function_single(cpu, tdx_flush_vp, &arg, 1);
	421	+ if (KVM_BUG_ON(arg.err, vcpu->kvm))
	422	+ pr_tdx_error(TDH_VP_FLUSH, arg.err);
	423	+}
	424	+
	425	+void tdx_disable_virtualization_cpu(void)
	426	+{
	427	+ int cpu = raw_smp_processor_id();
	428	+ struct list_head *tdvcpus = &per_cpu(associated_tdvcpus, cpu);
	429	+ struct tdx_flush_vp_arg arg;
	430	+ struct vcpu_tdx tdx, tmp;
	431	+ unsigned long flags;
	432	+
	433	+ local_irq_save(flags);
	434	+ /* Safe variant needed as tdx_disassociate_vp() deletes the entry. */
	435	+ list_for_each_entry_safe(tdx, tmp, tdvcpus, cpu_list) {
	436	+ arg.vcpu = &tdx->vcpu;
	437	+ tdx_flush_vp(&arg);
	438	+ }
	439	+ local_irq_restore(flags);
	440	+}
	441	+
	442	+#define TDX_SEAMCALL_RETRIES 10000
	443	+
	444	+static void smp_func_do_phymem_cache_wb(void *unused)
	445	+{
	446	+ u64 err = 0;
	447	+ bool resume;
	448	+ int i;
	449	+
	450	+ /*
	451	+ * TDH.PHYMEM.CACHE.WB flushes caches associated with any TDX private
	452	+ * KeyID on the package or core. The TDX module may not finish the
	453	+ * cache flush but return TDX_INTERRUPTED_RESUMEABLE instead. The
	454	+ * kernel should retry it until it returns success w/o rescheduling.
	455	+ */
	456	+ for (i = TDX_SEAMCALL_RETRIES; i > 0; i--) {
	457	+ resume = !!err;
	458	+ err = tdh_phymem_cache_wb(resume);
	459	+ switch (err) {
	460	+ case TDX_INTERRUPTED_RESUMABLE:
	461	+ continue;
	462	+ case TDX_NO_HKID_READY_TO_WBCACHE:
	463	+ err = TDX_SUCCESS; /* Already done by other thread */
	464	+ fallthrough;
	465	+ default:
	466	+ goto out;
	467	+ }
	468	+ }
	469	+
	470	+out:
	471	+ if (WARN_ON_ONCE(err))
	472	+ pr_tdx_error(TDH_PHYMEM_CACHE_WB, err);
	473	+}
	474	+
	475	+void tdx_mmu_release_hkid(struct kvm *kvm)
	476	+{
	477	+ bool packages_allocated, targets_allocated;
	478	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	479	+ cpumask_var_t packages, targets;
	480	+ struct kvm_vcpu *vcpu;
	481	+ unsigned long j;
	482	+ int i;
	483	+ u64 err;
	484	+
	485	+ if (!is_hkid_assigned(kvm_tdx))
	486	+ return;
	487	+
	488	+ packages_allocated = zalloc_cpumask_var(&packages, GFP_KERNEL);
	489	+ targets_allocated = zalloc_cpumask_var(&targets, GFP_KERNEL);
	490	+ cpus_read_lock();
	491	+
	492	+ kvm_for_each_vcpu(j, vcpu, kvm)
	493	+ tdx_flush_vp_on_cpu(vcpu);
	494	+
	495	+ /*
	496	+ * TDH.PHYMEM.CACHE.WB tries to acquire the TDX module global lock
	497	+ * and can fail with TDX_OPERAND_BUSY when it fails to get the lock.
	498	+ * Multiple TDX guests can be destroyed simultaneously. Take the
	499	+ * mutex to prevent it from getting error.
	500	+ */
	501	+ mutex_lock(&tdx_lock);
	502	+
	503	+ /*
	504	+ * Releasing HKID is in vm_destroy().
	505	+ * After the above flushing vps, there should be no more vCPU
	506	+ * associations, as all vCPU fds have been released at this stage.
	507	+ */
	508	+ err = tdh_mng_vpflushdone(&kvm_tdx->td);
	509	+ if (err == TDX_FLUSHVP_NOT_DONE)
	510	+ goto out;
	511	+ if (KVM_BUG_ON(err, kvm)) {
	512	+ pr_tdx_error(TDH_MNG_VPFLUSHDONE, err);
	513	+ pr_err("tdh_mng_vpflushdone() failed. HKID %d is leaked.\n",
	514	+ kvm_tdx->hkid);
	515	+ goto out;
	516	+ }
	517	+
	518	+ for_each_online_cpu(i) {
	519	+ if (packages_allocated &&
	520	+ cpumask_test_and_set_cpu(topology_physical_package_id(i),
	521	+ packages))
	522	+ continue;
	523	+ if (targets_allocated)
	524	+ cpumask_set_cpu(i, targets);
	525	+ }
	526	+ if (targets_allocated)
	527	+ on_each_cpu_mask(targets, smp_func_do_phymem_cache_wb, NULL, true);
	528	+ else
	529	+ on_each_cpu(smp_func_do_phymem_cache_wb, NULL, true);
	530	+ /*
	531	+ * In the case of error in smp_func_do_phymem_cache_wb(), the following
	532	+ * tdh_mng_key_freeid() will fail.
	533	+ */
	534	+ err = tdh_mng_key_freeid(&kvm_tdx->td);
	535	+ if (KVM_BUG_ON(err, kvm)) {
	536	+ pr_tdx_error(TDH_MNG_KEY_FREEID, err);
	537	+ pr_err("tdh_mng_key_freeid() failed. HKID %d is leaked.\n",
	538	+ kvm_tdx->hkid);
	539	+ } else {
	540	+ tdx_hkid_free(kvm_tdx);
	541	+ }
	542	+
	543	+out:
	544	+ mutex_unlock(&tdx_lock);
	545	+ cpus_read_unlock();
	546	+ free_cpumask_var(targets);
	547	+ free_cpumask_var(packages);
	548	+}
	549	+
	550	+static void tdx_reclaim_td_control_pages(struct kvm *kvm)
	551	+{
	552	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	553	+ u64 err;
	554	+ int i;
	555	+
	556	+ /*
	557	+ * tdx_mmu_release_hkid() failed to reclaim HKID. Something went wrong
	558	+ * heavily with TDX module. Give up freeing TD pages. As the function
	559	+ * already warned, don't warn it again.
	560	+ */
	561	+ if (is_hkid_assigned(kvm_tdx))
	562	+ return;
	563	+
	564	+ if (kvm_tdx->td.tdcs_pages) {
	565	+ for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) {
	566	+ if (!kvm_tdx->td.tdcs_pages[i])
	567	+ continue;
	568	+
	569	+ tdx_reclaim_control_page(kvm_tdx->td.tdcs_pages[i]);
	570	+ }
	571	+ kfree(kvm_tdx->td.tdcs_pages);
	572	+ kvm_tdx->td.tdcs_pages = NULL;
	573	+ }
	574	+
	575	+ if (!kvm_tdx->td.tdr_page)
	576	+ return;
	577	+
	578	+ if (__tdx_reclaim_page(kvm_tdx->td.tdr_page))
	579	+ return;
	580	+
	581	+ /*
	582	+ * Use a SEAMCALL to ask the TDX module to flush the cache based on the
	583	+ * KeyID. TDX module may access TDR while operating on TD (Especially
	584	+ * when it is reclaiming TDCS).
	585	+ */
	586	+ err = tdh_phymem_page_wbinvd_tdr(&kvm_tdx->td);
	587	+ if (KVM_BUG_ON(err, kvm)) {
	588	+ pr_tdx_error(TDH_PHYMEM_PAGE_WBINVD, err);
	589	+ return;
	590	+ }
	591	+ tdx_clear_page(kvm_tdx->td.tdr_page);
	592	+
	593	+ __free_page(kvm_tdx->td.tdr_page);
	594	+ kvm_tdx->td.tdr_page = NULL;
	595	+}
	596	+
	597	+void tdx_vm_destroy(struct kvm *kvm)
	598	+{
	599	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	600	+
	601	+ tdx_reclaim_td_control_pages(kvm);
	602	+
	603	+ kvm_tdx->state = TD_STATE_UNINITIALIZED;
	604	+}
	605	+
	606	+static int tdx_do_tdh_mng_key_config(void *param)
	607	+{
	608	+ struct kvm_tdx *kvm_tdx = param;
	609	+ u64 err;
	610	+
	611	+ /* TDX_RND_NO_ENTROPY related retries are handled by sc_retry() */
	612	+ err = tdh_mng_key_config(&kvm_tdx->td);
	613	+
	614	+ if (KVM_BUG_ON(err, &kvm_tdx->kvm)) {
	615	+ pr_tdx_error(TDH_MNG_KEY_CONFIG, err);
	616	+ return -EIO;
	617	+ }
	618	+
	619	+ return 0;
	620	+}
	621	+
	622	+int tdx_vm_init(struct kvm *kvm)
	623	+{
	624	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	625	+
	626	+ kvm->arch.has_protected_state = true;
	627	+ kvm->arch.has_private_mem = true;
	628	+ kvm->arch.disabled_quirks \|= KVM_X86_QUIRK_IGNORE_GUEST_PAT;
	629	+
	630	+ /*
	631	+ * Because guest TD is protected, VMM can't parse the instruction in TD.
	632	+ * Instead, guest uses MMIO hypercall. For unmodified device driver,
	633	+ * #VE needs to be injected for MMIO and #VE handler in TD converts MMIO
	634	+ * instruction into MMIO hypercall.
	635	+ *
	636	+ * SPTE value for MMIO needs to be setup so that #VE is injected into
	637	+ * TD instead of triggering EPT MISCONFIG.
	638	+ * - RWX=0 so that EPT violation is triggered.
	639	+ * - suppress #VE bit is cleared to inject #VE.
	640	+ */
	641	+ kvm_mmu_set_mmio_spte_value(kvm, 0);
	642	+
	643	+ /*
	644	+ * TDX has its own limit of maximum vCPUs it can support for all
	645	+ * TDX guests in addition to KVM_MAX_VCPUS. TDX module reports
	646	+ * such limit via the MAX_VCPU_PER_TD global metadata. In
	647	+ * practice, it reflects the number of logical CPUs that ALL
	648	+ * platforms that the TDX module supports can possibly have.
	649	+ *
	650	+ * Limit TDX guest's maximum vCPUs to the number of logical CPUs
	651	+ * the platform has. Simply forwarding the MAX_VCPU_PER_TD to
	652	+ * userspace would result in an unpredictable ABI.
	653	+ */
	654	+ kvm->max_vcpus = min_t(int, kvm->max_vcpus, num_present_cpus());
	655	+
	656	+ kvm_tdx->state = TD_STATE_UNINITIALIZED;
	657	+
	658	+ return 0;
	659	+}
	660	+
	661	+int tdx_vcpu_create(struct kvm_vcpu *vcpu)
	662	+{
	663	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);
	664	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	665	+
	666	+ if (kvm_tdx->state != TD_STATE_INITIALIZED)
	667	+ return -EIO;
	668	+
	669	+ /*
	670	+ * TDX module mandates APICv, which requires an in-kernel local APIC.
	671	+ * Disallow an in-kernel I/O APIC, because level-triggered interrupts
	672	+ * and thus the I/O APIC as a whole can't be faithfully emulated in KVM.
	673	+ */
	674	+ if (!irqchip_split(vcpu->kvm))
	675	+ return -EINVAL;
	676	+
	677	+ fpstate_set_confidential(&vcpu->arch.guest_fpu);
	678	+ vcpu->arch.apic->guest_apic_protected = true;
	679	+ INIT_LIST_HEAD(&tdx->vt.pi_wakeup_list);
	680	+
	681	+ vcpu->arch.efer = EFER_SCE \| EFER_LME \| EFER_LMA \| EFER_NX;
	682	+
	683	+ vcpu->arch.switch_db_regs = KVM_DEBUGREG_AUTO_SWITCH;
	684	+ vcpu->arch.cr0_guest_owned_bits = -1ul;
	685	+ vcpu->arch.cr4_guest_owned_bits = -1ul;
	686	+
	687	+ /* KVM can't change TSC offset/multiplier as TDX module manages them. */
	688	+ vcpu->arch.guest_tsc_protected = true;
	689	+ vcpu->arch.tsc_offset = kvm_tdx->tsc_offset;
	690	+ vcpu->arch.l1_tsc_offset = vcpu->arch.tsc_offset;
	691	+ vcpu->arch.tsc_scaling_ratio = kvm_tdx->tsc_multiplier;
	692	+ vcpu->arch.l1_tsc_scaling_ratio = kvm_tdx->tsc_multiplier;
	693	+
	694	+ vcpu->arch.guest_state_protected =
	695	+ !(to_kvm_tdx(vcpu->kvm)->attributes & TDX_TD_ATTR_DEBUG);
	696	+
	697	+ if ((kvm_tdx->xfam & XFEATURE_MASK_XTILE) == XFEATURE_MASK_XTILE)
	698	+ vcpu->arch.xfd_no_write_intercept = true;
	699	+
	700	+ tdx->vt.pi_desc.nv = POSTED_INTR_VECTOR;
	701	+ __pi_set_sn(&tdx->vt.pi_desc);
	702	+
	703	+ tdx->state = VCPU_TD_STATE_UNINITIALIZED;
	704	+
	705	+ return 0;
	706	+}
	707	+
	708	+void tdx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
	709	+{
	710	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	711	+
	712	+ vmx_vcpu_pi_load(vcpu, cpu);
	713	+ if (vcpu->cpu == cpu \|\| !is_hkid_assigned(to_kvm_tdx(vcpu->kvm)))
	714	+ return;
	715	+
	716	+ tdx_flush_vp_on_cpu(vcpu);
	717	+
	718	+ KVM_BUG_ON(cpu != raw_smp_processor_id(), vcpu->kvm);
	719	+ local_irq_disable();
	720	+ /*
	721	+ * Pairs with the smp_wmb() in tdx_disassociate_vp() to ensure
	722	+ * vcpu->cpu is read before tdx->cpu_list.
	723	+ */
	724	+ smp_rmb();
	725	+
	726	+ list_add(&tdx->cpu_list, &per_cpu(associated_tdvcpus, cpu));
	727	+ local_irq_enable();
	728	+}
	729	+
	730	+bool tdx_interrupt_allowed(struct kvm_vcpu *vcpu)
	731	+{
	732	+ /*
	733	+ * KVM can't get the interrupt status of TDX guest and it assumes
	734	+ * interrupt is always allowed unless TDX guest calls TDVMCALL with HLT,
	735	+ * which passes the interrupt blocked flag.
	736	+ */
	737	+ return vmx_get_exit_reason(vcpu).basic != EXIT_REASON_HLT \|\|
	738	+ !to_tdx(vcpu)->vp_enter_args.r12;
	739	+}
	740	+
	741	+bool tdx_protected_apic_has_interrupt(struct kvm_vcpu *vcpu)
	742	+{
	743	+ u64 vcpu_state_details;
	744	+
	745	+ if (pi_has_pending_interrupt(vcpu))
	746	+ return true;
	747	+
	748	+ /*
	749	+ * Only check RVI pending for HALTED case with IRQ enabled.
	750	+ * For non-HLT cases, KVM doesn't care about STI/SS shadows. And if the
	751	+ * interrupt was pending before TD exit, then it _must_ be blocked,
	752	+ * otherwise the interrupt would have been serviced at the instruction
	753	+ * boundary.
	754	+ */
	755	+ if (vmx_get_exit_reason(vcpu).basic != EXIT_REASON_HLT \|\|
	756	+ to_tdx(vcpu)->vp_enter_args.r12)
	757	+ return false;
	758	+
	759	+ vcpu_state_details =
	760	+ td_state_non_arch_read64(to_tdx(vcpu), TD_VCPU_STATE_DETAILS_NON_ARCH);
	761	+
	762	+ return tdx_vcpu_state_details_intr_pending(vcpu_state_details);
	763	+}
	764	+
	765	+/*
	766	+ * Compared to vmx_prepare_switch_to_guest(), there is not much to do
	767	+ * as SEAMCALL/SEAMRET calls take care of most of save and restore.
	768	+ */
	769	+void tdx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
	770	+{
	771	+ struct vcpu_vt *vt = to_vt(vcpu);
	772	+
	773	+ if (vt->guest_state_loaded)
	774	+ return;
	775	+
	776	+ if (likely(is_64bit_mm(current->mm)))
	777	+ vt->msr_host_kernel_gs_base = current->thread.gsbase;
	778	+ else
	779	+ vt->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE);
	780	+
	781	+ vt->host_debugctlmsr = get_debugctlmsr();
	782	+
	783	+ vt->guest_state_loaded = true;
	784	+}
	785	+
	786	+struct tdx_uret_msr {
	787	+ u32 msr;
	788	+ unsigned int slot;
	789	+ u64 defval;
	790	+};
	791	+
	792	+static struct tdx_uret_msr tdx_uret_msrs[] = {
	793	+ {.msr = MSR_SYSCALL_MASK, .defval = 0x20200 },
	794	+ {.msr = MSR_STAR,},
	795	+ {.msr = MSR_LSTAR,},
	796	+ {.msr = MSR_TSC_AUX,},
	797	+};
	798	+
	799	+static void tdx_user_return_msr_update_cache(void)
	800	+{
	801	+ int i;
	802	+
	803	+ for (i = 0; i < ARRAY_SIZE(tdx_uret_msrs); i++)
	804	+ kvm_user_return_msr_update_cache(tdx_uret_msrs[i].slot,
	805	+ tdx_uret_msrs[i].defval);
	806	+}
	807	+
	808	+static void tdx_prepare_switch_to_host(struct kvm_vcpu *vcpu)
	809	+{
	810	+ struct vcpu_vt *vt = to_vt(vcpu);
	811	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	812	+
	813	+ if (!vt->guest_state_loaded)
	814	+ return;
	815	+
	816	+ ++vcpu->stat.host_state_reload;
	817	+ wrmsrl(MSR_KERNEL_GS_BASE, vt->msr_host_kernel_gs_base);
	818	+
	819	+ if (tdx->guest_entered) {
	820	+ tdx_user_return_msr_update_cache();
	821	+ tdx->guest_entered = false;
	822	+ }
	823	+
	824	+ vt->guest_state_loaded = false;
	825	+}
	826	+
	827	+void tdx_vcpu_put(struct kvm_vcpu *vcpu)
	828	+{
	829	+ vmx_vcpu_pi_put(vcpu);
	830	+ tdx_prepare_switch_to_host(vcpu);
	831	+}
	832	+
	833	+void tdx_vcpu_free(struct kvm_vcpu *vcpu)
	834	+{
	835	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);
	836	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	837	+ int i;
	838	+
	839	+ /*
	840	+ * It is not possible to reclaim pages while hkid is assigned. It might
	841	+ * be assigned if:
	842	+ * 1. the TD VM is being destroyed but freeing hkid failed, in which
	843	+ * case the pages are leaked
	844	+ * 2. TD VCPU creation failed and this on the error path, in which case
	845	+ * there is nothing to do anyway
	846	+ */
	847	+ if (is_hkid_assigned(kvm_tdx))
	848	+ return;
	849	+
	850	+ if (tdx->vp.tdcx_pages) {
	851	+ for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) {
	852	+ if (tdx->vp.tdcx_pages[i])
	853	+ tdx_reclaim_control_page(tdx->vp.tdcx_pages[i]);
	854	+ }
	855	+ kfree(tdx->vp.tdcx_pages);
	856	+ tdx->vp.tdcx_pages = NULL;
	857	+ }
	858	+ if (tdx->vp.tdvpr_page) {
	859	+ tdx_reclaim_control_page(tdx->vp.tdvpr_page);
	860	+ tdx->vp.tdvpr_page = 0;
	861	+ }
	862	+
	863	+ tdx->state = VCPU_TD_STATE_UNINITIALIZED;
	864	+}
	865	+
	866	+int tdx_vcpu_pre_run(struct kvm_vcpu *vcpu)
	867	+{
	868	+ if (unlikely(to_tdx(vcpu)->state != VCPU_TD_STATE_INITIALIZED \|\|
	869	+ to_kvm_tdx(vcpu->kvm)->state != TD_STATE_RUNNABLE))
	870	+ return -EINVAL;
	871	+
	872	+ return 1;
	873	+}
	874	+
	875	+static __always_inline u32 tdcall_to_vmx_exit_reason(struct kvm_vcpu *vcpu)
	876	+{
	877	+ switch (tdvmcall_leaf(vcpu)) {
	878	+ case EXIT_REASON_CPUID:
	879	+ case EXIT_REASON_HLT:
	880	+ case EXIT_REASON_IO_INSTRUCTION:
	881	+ case EXIT_REASON_MSR_READ:
	882	+ case EXIT_REASON_MSR_WRITE:
	883	+ return tdvmcall_leaf(vcpu);
	884	+ case EXIT_REASON_EPT_VIOLATION:
	885	+ return EXIT_REASON_EPT_MISCONFIG;
	886	+ default:
	887	+ break;
	888	+ }
	889	+
	890	+ return EXIT_REASON_TDCALL;
	891	+}
	892	+
	893	+static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu)
	894	+{
	895	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	896	+ u32 exit_reason;
	897	+
	898	+ switch (tdx->vp_enter_ret & TDX_SEAMCALL_STATUS_MASK) {
	899	+ case TDX_SUCCESS:
	900	+ case TDX_NON_RECOVERABLE_VCPU:
	901	+ case TDX_NON_RECOVERABLE_TD:
	902	+ case TDX_NON_RECOVERABLE_TD_NON_ACCESSIBLE:
	903	+ case TDX_NON_RECOVERABLE_TD_WRONG_APIC_MODE:
	904	+ break;
	905	+ default:
	906	+ return -1u;
	907	+ }
	908	+
	909	+ exit_reason = tdx->vp_enter_ret;
	910	+
	911	+ switch (exit_reason) {
	912	+ case EXIT_REASON_TDCALL:
	913	+ if (tdvmcall_exit_type(vcpu))
	914	+ return EXIT_REASON_VMCALL;
	915	+
	916	+ return tdcall_to_vmx_exit_reason(vcpu);
	917	+ case EXIT_REASON_EPT_MISCONFIG:
	918	+ /*
	919	+ * Defer KVM_BUG_ON() until tdx_handle_exit() because this is in
	920	+ * non-instrumentable code with interrupts disabled.
	921	+ */
	922	+ return -1u;
	923	+ default:
	924	+ break;
	925	+ }
	926	+
	927	+ return exit_reason;
	928	+}
	929	+
	930	+static noinstr void tdx_vcpu_enter_exit(struct kvm_vcpu *vcpu)
	931	+{
	932	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	933	+ struct vcpu_vt *vt = to_vt(vcpu);
	934	+
	935	+ guest_state_enter_irqoff();
	936	+
	937	+ tdx->vp_enter_ret = tdh_vp_enter(&tdx->vp, &tdx->vp_enter_args);
	938	+
	939	+ vt->exit_reason.full = tdx_to_vmx_exit_reason(vcpu);
	940	+
	941	+ vt->exit_qualification = tdx->vp_enter_args.rcx;
	942	+ tdx->ext_exit_qualification = tdx->vp_enter_args.rdx;
	943	+ tdx->exit_gpa = tdx->vp_enter_args.r8;
	944	+ vt->exit_intr_info = tdx->vp_enter_args.r9;
	945	+
	946	+ vmx_handle_nmi(vcpu);
	947	+
	948	+ guest_state_exit_irqoff();
	949	+}
	950	+
	951	+static bool tdx_failed_vmentry(struct kvm_vcpu *vcpu)
	952	+{
	953	+ return vmx_get_exit_reason(vcpu).failed_vmentry &&
	954	+ vmx_get_exit_reason(vcpu).full != -1u;
	955	+}
	956	+
	957	+static fastpath_t tdx_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
	958	+{
	959	+ u64 vp_enter_ret = to_tdx(vcpu)->vp_enter_ret;
	960	+
	961	+ /*
	962	+ * TDX_OPERAND_BUSY could be returned for SEPT due to 0-step mitigation
	963	+ * or for TD EPOCH due to contention with TDH.MEM.TRACK on TDH.VP.ENTER.
	964	+ *
	965	+ * When KVM requests KVM_REQ_OUTSIDE_GUEST_MODE, which has both
	966	+ * KVM_REQUEST_WAIT and KVM_REQUEST_NO_ACTION set, it requires target
	967	+ * vCPUs leaving fastpath so that interrupt can be enabled to ensure the
	968	+ * IPIs can be delivered. Return EXIT_FASTPATH_EXIT_HANDLED instead of
	969	+ * EXIT_FASTPATH_REENTER_GUEST to exit fastpath, otherwise, the
	970	+ * requester may be blocked endlessly.
	971	+ */
	972	+ if (unlikely(tdx_operand_busy(vp_enter_ret)))
	973	+ return EXIT_FASTPATH_EXIT_HANDLED;
	974	+
	975	+ return EXIT_FASTPATH_NONE;
	976	+}
	977	+
	978	+#define TDX_REGS_AVAIL_SET (BIT_ULL(VCPU_EXREG_EXIT_INFO_1) \| \
	979	+ BIT_ULL(VCPU_EXREG_EXIT_INFO_2) \| \
	980	+ BIT_ULL(VCPU_REGS_RAX) \| \
	981	+ BIT_ULL(VCPU_REGS_RBX) \| \
	982	+ BIT_ULL(VCPU_REGS_RCX) \| \
	983	+ BIT_ULL(VCPU_REGS_RDX) \| \
	984	+ BIT_ULL(VCPU_REGS_RBP) \| \
	985	+ BIT_ULL(VCPU_REGS_RSI) \| \
	986	+ BIT_ULL(VCPU_REGS_RDI) \| \
	987	+ BIT_ULL(VCPU_REGS_R8) \| \
	988	+ BIT_ULL(VCPU_REGS_R9) \| \
	989	+ BIT_ULL(VCPU_REGS_R10) \| \
	990	+ BIT_ULL(VCPU_REGS_R11) \| \
	991	+ BIT_ULL(VCPU_REGS_R12) \| \
	992	+ BIT_ULL(VCPU_REGS_R13) \| \
	993	+ BIT_ULL(VCPU_REGS_R14) \| \
	994	+ BIT_ULL(VCPU_REGS_R15))
	995	+
	996	+static void tdx_load_host_xsave_state(struct kvm_vcpu *vcpu)
	997	+{
	998	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);
	999	+
	1000	+ /*
	1001	+ * All TDX hosts support PKRU; but even if they didn't,
	1002	+ * vcpu->arch.host_pkru would be 0 and the wrpkru would be
	1003	+ * skipped.
	1004	+ */
	1005	+ if (vcpu->arch.host_pkru != 0)
	1006	+ wrpkru(vcpu->arch.host_pkru);
	1007	+
	1008	+ if (kvm_host.xcr0 != (kvm_tdx->xfam & kvm_caps.supported_xcr0))
	1009	+ xsetbv(XCR_XFEATURE_ENABLED_MASK, kvm_host.xcr0);
	1010	+
	1011	+ /*
	1012	+ * Likewise, even if a TDX hosts didn't support XSS both arms of
	1013	+ * the comparison would be 0 and the wrmsrl would be skipped.
	1014	+ */
	1015	+ if (kvm_host.xss != (kvm_tdx->xfam & kvm_caps.supported_xss))
	1016	+ wrmsrl(MSR_IA32_XSS, kvm_host.xss);
	1017	+}
	1018	+
	1019	+#define TDX_DEBUGCTL_PRESERVED (DEBUGCTLMSR_BTF \| \
	1020	+ DEBUGCTLMSR_FREEZE_PERFMON_ON_PMI \| \
	1021	+ DEBUGCTLMSR_FREEZE_IN_SMM)
	1022	+
	1023	+fastpath_t tdx_vcpu_run(struct kvm_vcpu *vcpu, bool force_immediate_exit)
	1024	+{
	1025	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	1026	+ struct vcpu_vt *vt = to_vt(vcpu);
	1027	+
	1028	+ /*
	1029	+ * force_immediate_exit requires vCPU entering for events injection with
	1030	+ * an immediately exit followed. But The TDX module doesn't guarantee
	1031	+ * entry, it's already possible for KVM to _think_ it completely entry
	1032	+ * to the guest without actually having done so.
	1033	+ * Since KVM never needs to force an immediate exit for TDX, and can't
	1034	+ * do direct injection, just warn on force_immediate_exit.
	1035	+ */
	1036	+ WARN_ON_ONCE(force_immediate_exit);
	1037	+
	1038	+ /*
	1039	+ * Wait until retry of SEPT-zap-related SEAMCALL completes before
	1040	+ * allowing vCPU entry to avoid contention with tdh_vp_enter() and
	1041	+ * TDCALLs.
	1042	+ */
	1043	+ if (unlikely(READ_ONCE(to_kvm_tdx(vcpu->kvm)->wait_for_sept_zap)))
	1044	+ return EXIT_FASTPATH_EXIT_HANDLED;
	1045	+
	1046	+ trace_kvm_entry(vcpu, force_immediate_exit);
	1047	+
	1048	+ if (pi_test_on(&vt->pi_desc)) {
	1049	+ apic->send_IPI_self(POSTED_INTR_VECTOR);
	1050	+
	1051	+ if (pi_test_pir(kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVTT) &
	1052	+ APIC_VECTOR_MASK, &vt->pi_desc))
	1053	+ kvm_wait_lapic_expire(vcpu);
	1054	+ }
	1055	+
	1056	+ tdx_vcpu_enter_exit(vcpu);
	1057	+
	1058	+ if (vt->host_debugctlmsr & ~TDX_DEBUGCTL_PRESERVED)
	1059	+ update_debugctlmsr(vt->host_debugctlmsr);
	1060	+
	1061	+ tdx_load_host_xsave_state(vcpu);
	1062	+ tdx->guest_entered = true;
	1063	+
	1064	+ vcpu->arch.regs_avail &= TDX_REGS_AVAIL_SET;
	1065	+
	1066	+ if (unlikely(tdx->vp_enter_ret == EXIT_REASON_EPT_MISCONFIG))
	1067	+ return EXIT_FASTPATH_NONE;
	1068	+
	1069	+ if (unlikely((tdx->vp_enter_ret & TDX_SW_ERROR) == TDX_SW_ERROR))
	1070	+ return EXIT_FASTPATH_NONE;
	1071	+
	1072	+ if (unlikely(vmx_get_exit_reason(vcpu).basic == EXIT_REASON_MCE_DURING_VMENTRY))
	1073	+ kvm_machine_check();
	1074	+
	1075	+ trace_kvm_exit(vcpu, KVM_ISA_VMX);
	1076	+
	1077	+ if (unlikely(tdx_failed_vmentry(vcpu)))
	1078	+ return EXIT_FASTPATH_NONE;
	1079	+
	1080	+ return tdx_exit_handlers_fastpath(vcpu);
	1081	+}
	1082	+
	1083	+void tdx_inject_nmi(struct kvm_vcpu *vcpu)
	1084	+{
	1085	+ ++vcpu->stat.nmi_injections;
	1086	+ td_management_write8(to_tdx(vcpu), TD_VCPU_PEND_NMI, 1);
	1087	+ /*
	1088	+ * From KVM's perspective, NMI injection is completed right after
	1089	+ * writing to PEND_NMI. KVM doesn't care whether an NMI is injected by
	1090	+ * the TDX module or not.
	1091	+ */
	1092	+ vcpu->arch.nmi_injected = false;
	1093	+ /*
	1094	+ * TDX doesn't support KVM to request NMI window exit. If there is
	1095	+ * still a pending vNMI, KVM is not able to inject it along with the
	1096	+ * one pending in TDX module in a back-to-back way. Since the previous
	1097	+ * vNMI is still pending in TDX module, i.e. it has not been delivered
	1098	+ * to TDX guest yet, it's OK to collapse the pending vNMI into the
	1099	+ * previous one. The guest is expected to handle all the NMI sources
	1100	+ * when handling the first vNMI.
	1101	+ */
	1102	+ vcpu->arch.nmi_pending = 0;
	1103	+}
	1104	+
	1105	+static int tdx_handle_exception_nmi(struct kvm_vcpu *vcpu)
	1106	+{
	1107	+ u32 intr_info = vmx_get_intr_info(vcpu);
	1108	+
	1109	+ /*
	1110	+ * Machine checks are handled by handle_exception_irqoff(), or by
	1111	+ * tdx_handle_exit() with TDX_NON_RECOVERABLE set if a #MC occurs on
	1112	+ * VM-Entry. NMIs are handled by tdx_vcpu_enter_exit().
	1113	+ */
	1114	+ if (is_nmi(intr_info) \|\| is_machine_check(intr_info))
	1115	+ return 1;
	1116	+
	1117	+ vcpu->run->exit_reason = KVM_EXIT_EXCEPTION;
	1118	+ vcpu->run->ex.exception = intr_info & INTR_INFO_VECTOR_MASK;
	1119	+ vcpu->run->ex.error_code = 0;
	1120	+
	1121	+ return 0;
	1122	+}
	1123	+
	1124	+static int complete_hypercall_exit(struct kvm_vcpu *vcpu)
	1125	+{
	1126	+ tdvmcall_set_return_code(vcpu, vcpu->run->hypercall.ret);
	1127	+ return 1;
	1128	+}
	1129	+
	1130	+static int tdx_emulate_vmcall(struct kvm_vcpu *vcpu)
	1131	+{
	1132	+ kvm_rax_write(vcpu, to_tdx(vcpu)->vp_enter_args.r10);
	1133	+ kvm_rbx_write(vcpu, to_tdx(vcpu)->vp_enter_args.r11);
	1134	+ kvm_rcx_write(vcpu, to_tdx(vcpu)->vp_enter_args.r12);
	1135	+ kvm_rdx_write(vcpu, to_tdx(vcpu)->vp_enter_args.r13);
	1136	+ kvm_rsi_write(vcpu, to_tdx(vcpu)->vp_enter_args.r14);
	1137	+
	1138	+ return __kvm_emulate_hypercall(vcpu, 0, complete_hypercall_exit);
	1139	+}
	1140	+
	1141	+/*
	1142	+ * Split into chunks and check interrupt pending between chunks. This allows
	1143	+ * for timely injection of interrupts to prevent issues with guest lockup
	1144	+ * detection.
	1145	+ */
	1146	+#define TDX_MAP_GPA_MAX_LEN (2 * 1024 * 1024)
	1147	+static void __tdx_map_gpa(struct vcpu_tdx *tdx);
	1148	+
	1149	+static int tdx_complete_vmcall_map_gpa(struct kvm_vcpu *vcpu)
	1150	+{
	1151	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	1152	+
	1153	+ if (vcpu->run->hypercall.ret) {
	1154	+ tdvmcall_set_return_code(vcpu, TDVMCALL_STATUS_INVALID_OPERAND);
	1155	+ tdx->vp_enter_args.r11 = tdx->map_gpa_next;
	1156	+ return 1;
	1157	+ }
	1158	+
	1159	+ tdx->map_gpa_next += TDX_MAP_GPA_MAX_LEN;
	1160	+ if (tdx->map_gpa_next >= tdx->map_gpa_end)
	1161	+ return 1;
	1162	+
	1163	+ /*
	1164	+ * Stop processing the remaining part if there is a pending interrupt,
	1165	+ * which could be qualified to deliver. Skip checking pending RVI for
	1166	+ * TDVMCALL_MAP_GPA, see comments in tdx_protected_apic_has_interrupt().
	1167	+ */
	1168	+ if (kvm_vcpu_has_events(vcpu)) {
	1169	+ tdvmcall_set_return_code(vcpu, TDVMCALL_STATUS_RETRY);
	1170	+ tdx->vp_enter_args.r11 = tdx->map_gpa_next;
	1171	+ return 1;
	1172	+ }
	1173	+
	1174	+ __tdx_map_gpa(tdx);
	1175	+ return 0;
	1176	+}
	1177	+
	1178	+static void __tdx_map_gpa(struct vcpu_tdx *tdx)
	1179	+{
	1180	+ u64 gpa = tdx->map_gpa_next;
	1181	+ u64 size = tdx->map_gpa_end - tdx->map_gpa_next;
	1182	+
	1183	+ if (size > TDX_MAP_GPA_MAX_LEN)
	1184	+ size = TDX_MAP_GPA_MAX_LEN;
	1185	+
	1186	+ tdx->vcpu.run->exit_reason = KVM_EXIT_HYPERCALL;
	1187	+ tdx->vcpu.run->hypercall.nr = KVM_HC_MAP_GPA_RANGE;
	1188	+ /*
	1189	+ * In principle this should have been -KVM_ENOSYS, but userspace (QEMU <=9.2)
	1190	+ * assumed that vcpu->run->hypercall.ret is never changed by KVM and thus that
	1191	+ * it was always zero on KVM_EXIT_HYPERCALL. Since KVM is now overwriting
	1192	+ * vcpu->run->hypercall.ret, ensuring that it is zero to not break QEMU.
	1193	+ */
	1194	+ tdx->vcpu.run->hypercall.ret = 0;
	1195	+ tdx->vcpu.run->hypercall.args[0] = gpa & ~gfn_to_gpa(kvm_gfn_direct_bits(tdx->vcpu.kvm));
	1196	+ tdx->vcpu.run->hypercall.args[1] = size / PAGE_SIZE;
	1197	+ tdx->vcpu.run->hypercall.args[2] = vt_is_tdx_private_gpa(tdx->vcpu.kvm, gpa) ?
	1198	+ KVM_MAP_GPA_RANGE_ENCRYPTED :
	1199	+ KVM_MAP_GPA_RANGE_DECRYPTED;
	1200	+ tdx->vcpu.run->hypercall.flags = KVM_EXIT_HYPERCALL_LONG_MODE;
	1201	+
	1202	+ tdx->vcpu.arch.complete_userspace_io = tdx_complete_vmcall_map_gpa;
	1203	+}
	1204	+
	1205	+static int tdx_map_gpa(struct kvm_vcpu *vcpu)
	1206	+{
	1207	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	1208	+ u64 gpa = tdx->vp_enter_args.r12;
	1209	+ u64 size = tdx->vp_enter_args.r13;
	1210	+ u64 ret;
	1211	+
	1212	+ /*
	1213	+ * Converting TDVMCALL_MAP_GPA to KVM_HC_MAP_GPA_RANGE requires
	1214	+ * userspace to enable KVM_CAP_EXIT_HYPERCALL with KVM_HC_MAP_GPA_RANGE
	1215	+ * bit set. If not, the error code is not defined in GHCI for TDX, use
	1216	+ * TDVMCALL_STATUS_INVALID_OPERAND for this case.
	1217	+ */
	1218	+ if (!user_exit_on_hypercall(vcpu->kvm, KVM_HC_MAP_GPA_RANGE)) {
	1219	+ ret = TDVMCALL_STATUS_INVALID_OPERAND;
	1220	+ goto error;
	1221	+ }
	1222	+
	1223	+ if (gpa + size <= gpa \|\| !kvm_vcpu_is_legal_gpa(vcpu, gpa) \|\|
	1224	+ !kvm_vcpu_is_legal_gpa(vcpu, gpa + size - 1) \|\|
	1225	+ (vt_is_tdx_private_gpa(vcpu->kvm, gpa) !=
	1226	+ vt_is_tdx_private_gpa(vcpu->kvm, gpa + size - 1))) {
	1227	+ ret = TDVMCALL_STATUS_INVALID_OPERAND;
	1228	+ goto error;
	1229	+ }
	1230	+
	1231	+ if (!PAGE_ALIGNED(gpa) \|\| !PAGE_ALIGNED(size)) {
	1232	+ ret = TDVMCALL_STATUS_ALIGN_ERROR;
	1233	+ goto error;
	1234	+ }
	1235	+
	1236	+ tdx->map_gpa_end = gpa + size;
	1237	+ tdx->map_gpa_next = gpa;
	1238	+
	1239	+ __tdx_map_gpa(tdx);
	1240	+ return 0;
	1241	+
	1242	+error:
	1243	+ tdvmcall_set_return_code(vcpu, ret);
	1244	+ tdx->vp_enter_args.r11 = gpa;
	1245	+ return 1;
	1246	+}
	1247	+
	1248	+static int tdx_report_fatal_error(struct kvm_vcpu *vcpu)
	1249	+{
	1250	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	1251	+ u64 *regs = vcpu->run->system_event.data;
	1252	+ u64 *module_regs = &tdx->vp_enter_args.r8;
	1253	+ int index = VCPU_REGS_RAX;
	1254	+
	1255	+ vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
	1256	+ vcpu->run->system_event.type = KVM_SYSTEM_EVENT_TDX_FATAL;
	1257	+ vcpu->run->system_event.ndata = 16;
	1258	+
	1259	+ /* Dump 16 general-purpose registers to userspace in ascending order. */
	1260	+ regs[index++] = tdx->vp_enter_ret;
	1261	+ regs[index++] = tdx->vp_enter_args.rcx;
	1262	+ regs[index++] = tdx->vp_enter_args.rdx;
	1263	+ regs[index++] = tdx->vp_enter_args.rbx;
	1264	+ regs[index++] = 0;
	1265	+ regs[index++] = 0;
	1266	+ regs[index++] = tdx->vp_enter_args.rsi;
	1267	+ regs[index] = tdx->vp_enter_args.rdi;
	1268	+ for (index = 0; index < 8; index++)
	1269	+ regs[VCPU_REGS_R8 + index] = module_regs[index];
	1270	+
	1271	+ return 0;
	1272	+}
	1273	+
	1274	+static int tdx_emulate_cpuid(struct kvm_vcpu *vcpu)
	1275	+{
	1276	+ u32 eax, ebx, ecx, edx;
	1277	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	1278	+
	1279	+ /* EAX and ECX for cpuid is stored in R12 and R13. */
	1280	+ eax = tdx->vp_enter_args.r12;
	1281	+ ecx = tdx->vp_enter_args.r13;
	1282	+
	1283	+ kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx, false);
	1284	+
	1285	+ tdx->vp_enter_args.r12 = eax;
	1286	+ tdx->vp_enter_args.r13 = ebx;
	1287	+ tdx->vp_enter_args.r14 = ecx;
	1288	+ tdx->vp_enter_args.r15 = edx;
	1289	+
	1290	+ return 1;
	1291	+}
	1292	+
	1293	+static int tdx_complete_pio_out(struct kvm_vcpu *vcpu)
	1294	+{
	1295	+ vcpu->arch.pio.count = 0;
	1296	+ return 1;
	1297	+}
	1298	+
	1299	+static int tdx_complete_pio_in(struct kvm_vcpu *vcpu)
	1300	+{
	1301	+ struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
	1302	+ unsigned long val = 0;
	1303	+ int ret;
	1304	+
	1305	+ ret = ctxt->ops->pio_in_emulated(ctxt, vcpu->arch.pio.size,
	1306	+ vcpu->arch.pio.port, &val, 1);
	1307	+
	1308	+ WARN_ON_ONCE(!ret);
	1309	+
	1310	+ tdvmcall_set_return_val(vcpu, val);
	1311	+
	1312	+ return 1;
	1313	+}
	1314	+
	1315	+static int tdx_emulate_io(struct kvm_vcpu *vcpu)
	1316	+{
	1317	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	1318	+ struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
	1319	+ unsigned long val = 0;
	1320	+ unsigned int port;
	1321	+ u64 size, write;
	1322	+ int ret;
	1323	+
	1324	+ ++vcpu->stat.io_exits;
	1325	+
	1326	+ size = tdx->vp_enter_args.r12;
	1327	+ write = tdx->vp_enter_args.r13;
	1328	+ port = tdx->vp_enter_args.r14;
	1329	+
	1330	+ if ((write != 0 && write != 1) \|\| (size != 1 && size != 2 && size != 4)) {
	1331	+ tdvmcall_set_return_code(vcpu, TDVMCALL_STATUS_INVALID_OPERAND);
	1332	+ return 1;
	1333	+ }
	1334	+
	1335	+ if (write) {
	1336	+ val = tdx->vp_enter_args.r15;
	1337	+ ret = ctxt->ops->pio_out_emulated(ctxt, size, port, &val, 1);
	1338	+ } else {
	1339	+ ret = ctxt->ops->pio_in_emulated(ctxt, size, port, &val, 1);
	1340	+ }
	1341	+
	1342	+ if (!ret)
	1343	+ vcpu->arch.complete_userspace_io = write ? tdx_complete_pio_out :
	1344	+ tdx_complete_pio_in;
	1345	+ else if (!write)
	1346	+ tdvmcall_set_return_val(vcpu, val);
	1347	+
	1348	+ return ret;
	1349	+}
	1350	+
	1351	+static int tdx_complete_mmio_read(struct kvm_vcpu *vcpu)
	1352	+{
	1353	+ unsigned long val = 0;
	1354	+ gpa_t gpa;
	1355	+ int size;
	1356	+
	1357	+ gpa = vcpu->mmio_fragments[0].gpa;
	1358	+ size = vcpu->mmio_fragments[0].len;
	1359	+
	1360	+ memcpy(&val, vcpu->run->mmio.data, size);
	1361	+ tdvmcall_set_return_val(vcpu, val);
	1362	+ trace_kvm_mmio(KVM_TRACE_MMIO_READ, size, gpa, &val);
	1363	+ return 1;
	1364	+}
	1365	+
	1366	+static inline int tdx_mmio_write(struct kvm_vcpu *vcpu, gpa_t gpa, int size,
	1367	+ unsigned long val)
	1368	+{
	1369	+ if (!kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) {
	1370	+ trace_kvm_fast_mmio(gpa);
	1371	+ return 0;
	1372	+ }
	1373	+
	1374	+ trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, size, gpa, &val);
	1375	+ if (kvm_io_bus_write(vcpu, KVM_MMIO_BUS, gpa, size, &val))
	1376	+ return -EOPNOTSUPP;
	1377	+
	1378	+ return 0;
	1379	+}
	1380	+
	1381	+static inline int tdx_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, int size)
	1382	+{
	1383	+ unsigned long val;
	1384	+
	1385	+ if (kvm_io_bus_read(vcpu, KVM_MMIO_BUS, gpa, size, &val))
	1386	+ return -EOPNOTSUPP;
	1387	+
	1388	+ tdvmcall_set_return_val(vcpu, val);
	1389	+ trace_kvm_mmio(KVM_TRACE_MMIO_READ, size, gpa, &val);
	1390	+ return 0;
	1391	+}
	1392	+
	1393	+static int tdx_emulate_mmio(struct kvm_vcpu *vcpu)
	1394	+{
	1395	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	1396	+ int size, write, r;
	1397	+ unsigned long val;
	1398	+ gpa_t gpa;
	1399	+
	1400	+ size = tdx->vp_enter_args.r12;
	1401	+ write = tdx->vp_enter_args.r13;
	1402	+ gpa = tdx->vp_enter_args.r14;
	1403	+ val = write ? tdx->vp_enter_args.r15 : 0;
	1404	+
	1405	+ if (size != 1 && size != 2 && size != 4 && size != 8)
	1406	+ goto error;
	1407	+ if (write != 0 && write != 1)
	1408	+ goto error;
	1409	+
	1410	+ /*
	1411	+ * TDG.VP.VMCALL<MMIO> allows only shared GPA, it makes no sense to
	1412	+ * do MMIO emulation for private GPA.
	1413	+ */
	1414	+ if (vt_is_tdx_private_gpa(vcpu->kvm, gpa) \|\|
	1415	+ vt_is_tdx_private_gpa(vcpu->kvm, gpa + size - 1))
	1416	+ goto error;
	1417	+
	1418	+ gpa = gpa & ~gfn_to_gpa(kvm_gfn_direct_bits(vcpu->kvm));
	1419	+
	1420	+ if (write)
	1421	+ r = tdx_mmio_write(vcpu, gpa, size, val);
	1422	+ else
	1423	+ r = tdx_mmio_read(vcpu, gpa, size);
	1424	+ if (!r)
	1425	+ /* Kernel completed device emulation. */
	1426	+ return 1;
	1427	+
	1428	+ /* Request the device emulation to userspace device model. */
	1429	+ vcpu->mmio_is_write = write;
	1430	+ if (!write)
	1431	+ vcpu->arch.complete_userspace_io = tdx_complete_mmio_read;
	1432	+
	1433	+ vcpu->run->mmio.phys_addr = gpa;
	1434	+ vcpu->run->mmio.len = size;
	1435	+ vcpu->run->mmio.is_write = write;
	1436	+ vcpu->run->exit_reason = KVM_EXIT_MMIO;
	1437	+
	1438	+ if (write) {
	1439	+ memcpy(vcpu->run->mmio.data, &val, size);
	1440	+ } else {
	1441	+ vcpu->mmio_fragments[0].gpa = gpa;
	1442	+ vcpu->mmio_fragments[0].len = size;
	1443	+ trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, size, gpa, NULL);
	1444	+ }
	1445	+ return 0;
	1446	+
	1447	+error:
	1448	+ tdvmcall_set_return_code(vcpu, TDVMCALL_STATUS_INVALID_OPERAND);
	1449	+ return 1;
	1450	+}
	1451	+
	1452	+static int tdx_get_td_vm_call_info(struct kvm_vcpu *vcpu)
	1453	+{
	1454	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	1455	+
	1456	+ if (tdx->vp_enter_args.r12)
	1457	+ tdvmcall_set_return_code(vcpu, TDVMCALL_STATUS_INVALID_OPERAND);
	1458	+ else {
	1459	+ tdx->vp_enter_args.r11 = 0;
	1460	+ tdx->vp_enter_args.r13 = 0;
	1461	+ tdx->vp_enter_args.r14 = 0;
	1462	+ }
	1463	+ return 1;
	1464	+}
	1465	+
	1466	+static int handle_tdvmcall(struct kvm_vcpu *vcpu)
	1467	+{
	1468	+ switch (tdvmcall_leaf(vcpu)) {
	1469	+ case TDVMCALL_MAP_GPA:
	1470	+ return tdx_map_gpa(vcpu);
	1471	+ case TDVMCALL_REPORT_FATAL_ERROR:
	1472	+ return tdx_report_fatal_error(vcpu);
	1473	+ case TDVMCALL_GET_TD_VM_CALL_INFO:
	1474	+ return tdx_get_td_vm_call_info(vcpu);
	1475	+ default:
	1476	+ break;
	1477	+ }
	1478	+
	1479	+ tdvmcall_set_return_code(vcpu, TDVMCALL_STATUS_INVALID_OPERAND);
	1480	+ return 1;
	1481	+}
	1482	+
	1483	+void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int pgd_level)
	1484	+{
	1485	+ u64 shared_bit = (pgd_level == 5) ? TDX_SHARED_BIT_PWL_5 :
	1486	+ TDX_SHARED_BIT_PWL_4;
	1487	+
	1488	+ if (KVM_BUG_ON(shared_bit != kvm_gfn_direct_bits(vcpu->kvm), vcpu->kvm))
	1489	+ return;
	1490	+
	1491	+ td_vmcs_write64(to_tdx(vcpu), SHARED_EPT_POINTER, root_hpa);
	1492	+}
	1493	+
	1494	+static void tdx_unpin(struct kvm kvm, struct page page)
	1495	+{
	1496	+ put_page(page);
	1497	+}
	1498	+
	1499	+static int tdx_mem_page_aug(struct kvm *kvm, gfn_t gfn,
	1500	+ enum pg_level level, struct page *page)
	1501	+{
	1502	+ int tdx_level = pg_level_to_tdx_sept_level(level);
	1503	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	1504	+ gpa_t gpa = gfn_to_gpa(gfn);
	1505	+ u64 entry, level_state;
	1506	+ u64 err;
	1507	+
	1508	+ err = tdh_mem_page_aug(&kvm_tdx->td, gpa, tdx_level, page, &entry, &level_state);
	1509	+ if (unlikely(tdx_operand_busy(err))) {
	1510	+ tdx_unpin(kvm, page);
	1511	+ return -EBUSY;
	1512	+ }
	1513	+
	1514	+ if (KVM_BUG_ON(err, kvm)) {
	1515	+ pr_tdx_error_2(TDH_MEM_PAGE_AUG, err, entry, level_state);
	1516	+ tdx_unpin(kvm, page);
	1517	+ return -EIO;
	1518	+ }
	1519	+
	1520	+ return 0;
	1521	+}
	1522	+
	1523	+/*
	1524	+ * KVM_TDX_INIT_MEM_REGION calls kvm_gmem_populate() to map guest pages; the
	1525	+ * callback tdx_gmem_post_populate() then maps pages into private memory.
	1526	+ * through the a seamcall TDH.MEM.PAGE.ADD(). The SEAMCALL also requires the
	1527	+ * private EPT structures for the page to have been built before, which is
	1528	+ * done via kvm_tdp_map_page(). nr_premapped counts the number of pages that
	1529	+ * were added to the EPT structures but not added with TDH.MEM.PAGE.ADD().
	1530	+ * The counter has to be zero on KVM_TDX_FINALIZE_VM, to ensure that there
	1531	+ * are no half-initialized shared EPT pages.
	1532	+ */
	1533	+static int tdx_mem_page_record_premap_cnt(struct kvm *kvm, gfn_t gfn,
	1534	+ enum pg_level level, kvm_pfn_t pfn)
	1535	+{
	1536	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	1537	+
	1538	+ if (KVM_BUG_ON(kvm->arch.pre_fault_allowed, kvm))
	1539	+ return -EINVAL;
	1540	+
	1541	+ /* nr_premapped will be decreased when tdh_mem_page_add() is called. */
	1542	+ atomic64_inc(&kvm_tdx->nr_premapped);
	1543	+ return 0;
	1544	+}
	1545	+
	1546	+int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn,
	1547	+ enum pg_level level, kvm_pfn_t pfn)
	1548	+{
	1549	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	1550	+ struct page *page = pfn_to_page(pfn);
	1551	+
	1552	+ /* TODO: handle large pages. */
	1553	+ if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))
	1554	+ return -EINVAL;
	1555	+
	1556	+ /*
	1557	+ * Because guest_memfd doesn't support page migration with
	1558	+ * a_ops->migrate_folio (yet), no callback is triggered for KVM on page
	1559	+ * migration. Until guest_memfd supports page migration, prevent page
	1560	+ * migration.
	1561	+ * TODO: Once guest_memfd introduces callback on page migration,
	1562	+ * implement it and remove get_page/put_page().
	1563	+ */
	1564	+ get_page(page);
	1565	+
	1566	+ /*
	1567	+ * Read 'pre_fault_allowed' before 'kvm_tdx->state'; see matching
	1568	+ * barrier in tdx_td_finalize().
	1569	+ */
	1570	+ smp_rmb();
	1571	+ if (likely(kvm_tdx->state == TD_STATE_RUNNABLE))
	1572	+ return tdx_mem_page_aug(kvm, gfn, level, page);
	1573	+
	1574	+ return tdx_mem_page_record_premap_cnt(kvm, gfn, level, pfn);
	1575	+}
	1576	+
	1577	+static int tdx_sept_drop_private_spte(struct kvm *kvm, gfn_t gfn,
	1578	+ enum pg_level level, struct page *page)
	1579	+{
	1580	+ int tdx_level = pg_level_to_tdx_sept_level(level);
	1581	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	1582	+ gpa_t gpa = gfn_to_gpa(gfn);
	1583	+ u64 err, entry, level_state;
	1584	+
	1585	+ /* TODO: handle large pages. */
	1586	+ if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))
	1587	+ return -EINVAL;
	1588	+
	1589	+ if (KVM_BUG_ON(!is_hkid_assigned(kvm_tdx), kvm))
	1590	+ return -EINVAL;
	1591	+
	1592	+ /*
	1593	+ * When zapping private page, write lock is held. So no race condition
	1594	+ * with other vcpu sept operation.
	1595	+ * Race with TDH.VP.ENTER due to (0-step mitigation) and Guest TDCALLs.
	1596	+ */
	1597	+ err = tdh_mem_page_remove(&kvm_tdx->td, gpa, tdx_level, &entry,
	1598	+ &level_state);
	1599	+
	1600	+ if (unlikely(tdx_operand_busy(err))) {
	1601	+ /*
	1602	+ * The second retry is expected to succeed after kicking off all
	1603	+ * other vCPUs and prevent them from invoking TDH.VP.ENTER.
	1604	+ */
	1605	+ tdx_no_vcpus_enter_start(kvm);
	1606	+ err = tdh_mem_page_remove(&kvm_tdx->td, gpa, tdx_level, &entry,
	1607	+ &level_state);
	1608	+ tdx_no_vcpus_enter_stop(kvm);
	1609	+ }
	1610	+
	1611	+ if (KVM_BUG_ON(err, kvm)) {
	1612	+ pr_tdx_error_2(TDH_MEM_PAGE_REMOVE, err, entry, level_state);
	1613	+ return -EIO;
	1614	+ }
	1615	+
	1616	+ err = tdh_phymem_page_wbinvd_hkid((u16)kvm_tdx->hkid, page);
	1617	+
	1618	+ if (KVM_BUG_ON(err, kvm)) {
	1619	+ pr_tdx_error(TDH_PHYMEM_PAGE_WBINVD, err);
	1620	+ return -EIO;
	1621	+ }
	1622	+ tdx_clear_page(page);
	1623	+ tdx_unpin(kvm, page);
	1624	+ return 0;
	1625	+}
	1626	+
	1627	+int tdx_sept_link_private_spt(struct kvm *kvm, gfn_t gfn,
	1628	+ enum pg_level level, void *private_spt)
	1629	+{
	1630	+ int tdx_level = pg_level_to_tdx_sept_level(level);
	1631	+ gpa_t gpa = gfn_to_gpa(gfn);
	1632	+ struct page *page = virt_to_page(private_spt);
	1633	+ u64 err, entry, level_state;
	1634	+
	1635	+ err = tdh_mem_sept_add(&to_kvm_tdx(kvm)->td, gpa, tdx_level, page, &entry,
	1636	+ &level_state);
	1637	+ if (unlikely(tdx_operand_busy(err)))
	1638	+ return -EBUSY;
	1639	+
	1640	+ if (KVM_BUG_ON(err, kvm)) {
	1641	+ pr_tdx_error_2(TDH_MEM_SEPT_ADD, err, entry, level_state);
	1642	+ return -EIO;
	1643	+ }
	1644	+
	1645	+ return 0;
	1646	+}
	1647	+
	1648	+/*
	1649	+ * Check if the error returned from a SEPT zap SEAMCALL is due to that a page is
	1650	+ * mapped by KVM_TDX_INIT_MEM_REGION without tdh_mem_page_add() being called
	1651	+ * successfully.
	1652	+ *
	1653	+ * Since tdh_mem_sept_add() must have been invoked successfully before a
	1654	+ * non-leaf entry present in the mirrored page table, the SEPT ZAP related
	1655	+ * SEAMCALLs should not encounter err TDX_EPT_WALK_FAILED. They should instead
	1656	+ * find TDX_EPT_ENTRY_STATE_INCORRECT due to an empty leaf entry found in the
	1657	+ * SEPT.
	1658	+ *
	1659	+ * Further check if the returned entry from SEPT walking is with RWX permissions
	1660	+ * to filter out anything unexpected.
	1661	+ *
	1662	+ * Note: @level is pg_level, not the tdx_level. The tdx_level extracted from
	1663	+ * level_state returned from a SEAMCALL error is the same as that passed into
	1664	+ * the SEAMCALL.
	1665	+ */
	1666	+static int tdx_is_sept_zap_err_due_to_premap(struct kvm_tdx *kvm_tdx, u64 err,
	1667	+ u64 entry, int level)
	1668	+{
	1669	+ if (!err \|\| kvm_tdx->state == TD_STATE_RUNNABLE)
	1670	+ return false;
	1671	+
	1672	+ if (err != (TDX_EPT_ENTRY_STATE_INCORRECT \| TDX_OPERAND_ID_RCX))
	1673	+ return false;
	1674	+
	1675	+ if ((is_last_spte(entry, level) && (entry & VMX_EPT_RWX_MASK)))
	1676	+ return false;
	1677	+
	1678	+ return true;
	1679	+}
	1680	+
	1681	+static int tdx_sept_zap_private_spte(struct kvm *kvm, gfn_t gfn,
	1682	+ enum pg_level level, struct page *page)
	1683	+{
	1684	+ int tdx_level = pg_level_to_tdx_sept_level(level);
	1685	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	1686	+ gpa_t gpa = gfn_to_gpa(gfn) & KVM_HPAGE_MASK(level);
	1687	+ u64 err, entry, level_state;
	1688	+
	1689	+ /* For now large page isn't supported yet. */
	1690	+ WARN_ON_ONCE(level != PG_LEVEL_4K);
	1691	+
	1692	+ err = tdh_mem_range_block(&kvm_tdx->td, gpa, tdx_level, &entry, &level_state);
	1693	+
	1694	+ if (unlikely(tdx_operand_busy(err))) {
	1695	+ /* After no vCPUs enter, the second retry is expected to succeed */
	1696	+ tdx_no_vcpus_enter_start(kvm);
	1697	+ err = tdh_mem_range_block(&kvm_tdx->td, gpa, tdx_level, &entry, &level_state);
	1698	+ tdx_no_vcpus_enter_stop(kvm);
	1699	+ }
	1700	+ if (tdx_is_sept_zap_err_due_to_premap(kvm_tdx, err, entry, level) &&
	1701	+ !KVM_BUG_ON(!atomic64_read(&kvm_tdx->nr_premapped), kvm)) {
	1702	+ atomic64_dec(&kvm_tdx->nr_premapped);
	1703	+ tdx_unpin(kvm, page);
	1704	+ return 0;
	1705	+ }
	1706	+
	1707	+ if (KVM_BUG_ON(err, kvm)) {
	1708	+ pr_tdx_error_2(TDH_MEM_RANGE_BLOCK, err, entry, level_state);
	1709	+ return -EIO;
	1710	+ }
	1711	+ return 1;
	1712	+}
	1713	+
	1714	+/*
	1715	+ * Ensure shared and private EPTs to be flushed on all vCPUs.
	1716	+ * tdh_mem_track() is the only caller that increases TD epoch. An increase in
	1717	+ * the TD epoch (e.g., to value "N + 1") is successful only if no vCPUs are
	1718	+ * running in guest mode with the value "N - 1".
	1719	+ *
	1720	+ * A successful execution of tdh_mem_track() ensures that vCPUs can only run in
	1721	+ * guest mode with TD epoch value "N" if no TD exit occurs after the TD epoch
	1722	+ * being increased to "N + 1".
	1723	+ *
	1724	+ * Kicking off all vCPUs after that further results in no vCPUs can run in guest
	1725	+ * mode with TD epoch value "N", which unblocks the next tdh_mem_track() (e.g.
	1726	+ * to increase TD epoch to "N + 2").
	1727	+ *
	1728	+ * TDX module will flush EPT on the next TD enter and make vCPUs to run in
	1729	+ * guest mode with TD epoch value "N + 1".
	1730	+ *
	1731	+ * kvm_make_all_cpus_request() guarantees all vCPUs are out of guest mode by
	1732	+ * waiting empty IPI handler ack_kick().
	1733	+ *
	1734	+ * No action is required to the vCPUs being kicked off since the kicking off
	1735	+ * occurs certainly after TD epoch increment and before the next
	1736	+ * tdh_mem_track().
	1737	+ */
	1738	+static void tdx_track(struct kvm *kvm)
	1739	+{
	1740	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	1741	+ u64 err;
	1742	+
	1743	+ /* If TD isn't finalized, it's before any vcpu running. */
	1744	+ if (unlikely(kvm_tdx->state != TD_STATE_RUNNABLE))
	1745	+ return;
	1746	+
	1747	+ lockdep_assert_held_write(&kvm->mmu_lock);
	1748	+
	1749	+ err = tdh_mem_track(&kvm_tdx->td);
	1750	+ if (unlikely(tdx_operand_busy(err))) {
	1751	+ /* After no vCPUs enter, the second retry is expected to succeed */
	1752	+ tdx_no_vcpus_enter_start(kvm);
	1753	+ err = tdh_mem_track(&kvm_tdx->td);
	1754	+ tdx_no_vcpus_enter_stop(kvm);
	1755	+ }
	1756	+
	1757	+ if (KVM_BUG_ON(err, kvm))
	1758	+ pr_tdx_error(TDH_MEM_TRACK, err);
	1759	+
	1760	+ kvm_make_all_cpus_request(kvm, KVM_REQ_OUTSIDE_GUEST_MODE);
	1761	+}
	1762	+
	1763	+int tdx_sept_free_private_spt(struct kvm *kvm, gfn_t gfn,
	1764	+ enum pg_level level, void *private_spt)
	1765	+{
	1766	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	1767	+
	1768	+ /*
	1769	+ * free_external_spt() is only called after hkid is freed when TD is
	1770	+ * tearing down.
	1771	+ * KVM doesn't (yet) zap page table pages in mirror page table while
	1772	+ * TD is active, though guest pages mapped in mirror page table could be
	1773	+ * zapped during TD is active, e.g. for shared <-> private conversion
	1774	+ * and slot move/deletion.
	1775	+ */
	1776	+ if (KVM_BUG_ON(is_hkid_assigned(kvm_tdx), kvm))
	1777	+ return -EINVAL;
	1778	+
	1779	+ /*
	1780	+ * The HKID assigned to this TD was already freed and cache was
	1781	+ * already flushed. We don't have to flush again.
	1782	+ */
	1783	+ return tdx_reclaim_page(virt_to_page(private_spt));
	1784	+}
	1785	+
	1786	+int tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn,
	1787	+ enum pg_level level, kvm_pfn_t pfn)
	1788	+{
	1789	+ struct page *page = pfn_to_page(pfn);
	1790	+ int ret;
	1791	+
	1792	+ /*
	1793	+ * HKID is released after all private pages have been removed, and set
	1794	+ * before any might be populated. Warn if zapping is attempted when
	1795	+ * there can't be anything populated in the private EPT.
	1796	+ */
	1797	+ if (KVM_BUG_ON(!is_hkid_assigned(to_kvm_tdx(kvm)), kvm))
	1798	+ return -EINVAL;
	1799	+
	1800	+ ret = tdx_sept_zap_private_spte(kvm, gfn, level, page);
	1801	+ if (ret <= 0)
	1802	+ return ret;
	1803	+
	1804	+ /*
	1805	+ * TDX requires TLB tracking before dropping private page. Do
	1806	+ * it here, although it is also done later.
	1807	+ */
	1808	+ tdx_track(kvm);
	1809	+
	1810	+ return tdx_sept_drop_private_spte(kvm, gfn, level, page);
	1811	+}
	1812	+
	1813	+void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode,
	1814	+ int trig_mode, int vector)
	1815	+{
	1816	+ struct kvm_vcpu *vcpu = apic->vcpu;
	1817	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	1818	+
	1819	+ /* TDX supports only posted interrupt. No lapic emulation. */
	1820	+ __vmx_deliver_posted_interrupt(vcpu, &tdx->vt.pi_desc, vector);
	1821	+
	1822	+ trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector);
	1823	+}
	1824	+
	1825	+static inline bool tdx_is_sept_violation_unexpected_pending(struct kvm_vcpu *vcpu)
	1826	+{
	1827	+ u64 eeq_type = to_tdx(vcpu)->ext_exit_qualification & TDX_EXT_EXIT_QUAL_TYPE_MASK;
	1828	+ u64 eq = vmx_get_exit_qual(vcpu);
	1829	+
	1830	+ if (eeq_type != TDX_EXT_EXIT_QUAL_TYPE_PENDING_EPT_VIOLATION)
	1831	+ return false;
	1832	+
	1833	+ return !(eq & EPT_VIOLATION_PROT_MASK) && !(eq & EPT_VIOLATION_EXEC_FOR_RING3_LIN);
	1834	+}
	1835	+
	1836	+static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu)
	1837	+{
	1838	+ unsigned long exit_qual;
	1839	+ gpa_t gpa = to_tdx(vcpu)->exit_gpa;
	1840	+ bool local_retry = false;
	1841	+ int ret;
	1842	+
	1843	+ if (vt_is_tdx_private_gpa(vcpu->kvm, gpa)) {
	1844	+ if (tdx_is_sept_violation_unexpected_pending(vcpu)) {
	1845	+ pr_warn("Guest access before accepting 0x%llx on vCPU %d\n",
	1846	+ gpa, vcpu->vcpu_id);
	1847	+ kvm_vm_dead(vcpu->kvm);
	1848	+ return -EIO;
	1849	+ }
	1850	+ /*
	1851	+ * Always treat SEPT violations as write faults. Ignore the
	1852	+ * EXIT_QUALIFICATION reported by TDX-SEAM for SEPT violations.
	1853	+ * TD private pages are always RWX in the SEPT tables,
	1854	+ * i.e. they're always mapped writable. Just as importantly,
	1855	+ * treating SEPT violations as write faults is necessary to
	1856	+ * avoid COW allocations, which will cause TDAUGPAGE failures
	1857	+ * due to aliasing a single HPA to multiple GPAs.
	1858	+ */
	1859	+ exit_qual = EPT_VIOLATION_ACC_WRITE;
	1860	+
	1861	+ /* Only private GPA triggers zero-step mitigation */
	1862	+ local_retry = true;
	1863	+ } else {
	1864	+ exit_qual = vmx_get_exit_qual(vcpu);
	1865	+ /*
	1866	+ * EPT violation due to instruction fetch should never be
	1867	+ * triggered from shared memory in TDX guest. If such EPT
	1868	+ * violation occurs, treat it as broken hardware.
	1869	+ */
	1870	+ if (KVM_BUG_ON(exit_qual & EPT_VIOLATION_ACC_INSTR, vcpu->kvm))
	1871	+ return -EIO;
	1872	+ }
	1873	+
	1874	+ trace_kvm_page_fault(vcpu, gpa, exit_qual);
	1875	+
	1876	+ /*
	1877	+ * To minimize TDH.VP.ENTER invocations, retry locally for private GPA
	1878	+ * mapping in TDX.
	1879	+ *
	1880	+ * KVM may return RET_PF_RETRY for private GPA due to
	1881	+ * - contentions when atomically updating SPTEs of the mirror page table
	1882	+ * - in-progress GFN invalidation or memslot removal.
	1883	+ * - TDX_OPERAND_BUSY error from TDH.MEM.PAGE.AUG or TDH.MEM.SEPT.ADD,
	1884	+ * caused by contentions with TDH.VP.ENTER (with zero-step mitigation)
	1885	+ * or certain TDCALLs.
	1886	+ *
	1887	+ * If TDH.VP.ENTER is invoked more times than the threshold set by the
	1888	+ * TDX module before KVM resolves the private GPA mapping, the TDX
	1889	+ * module will activate zero-step mitigation during TDH.VP.ENTER. This
	1890	+ * process acquires an SEPT tree lock in the TDX module, leading to
	1891	+ * further contentions with TDH.MEM.PAGE.AUG or TDH.MEM.SEPT.ADD
	1892	+ * operations on other vCPUs.
	1893	+ *
	1894	+ * Breaking out of local retries for kvm_vcpu_has_events() is for
	1895	+ * interrupt injection. kvm_vcpu_has_events() should not see pending
	1896	+ * events for TDX. Since KVM can't determine if IRQs (or NMIs) are
	1897	+ * blocked by TDs, false positives are inevitable i.e., KVM may re-enter
	1898	+ * the guest even if the IRQ/NMI can't be delivered.
	1899	+ *
	1900	+ * Note: even without breaking out of local retries, zero-step
	1901	+ * mitigation may still occur due to
	1902	+ * - invoking of TDH.VP.ENTER after KVM_EXIT_MEMORY_FAULT,
	1903	+ * - a single RIP causing EPT violations for more GFNs than the
	1904	+ * threshold count.
	1905	+ * This is safe, as triggering zero-step mitigation only introduces
	1906	+ * contentions to page installation SEAMCALLs on other vCPUs, which will
	1907	+ * handle retries locally in their EPT violation handlers.
	1908	+ */
	1909	+ while (1) {
	1910	+ ret = __vmx_handle_ept_violation(vcpu, gpa, exit_qual);
	1911	+
	1912	+ if (ret != RET_PF_RETRY \|\| !local_retry)
	1913	+ break;
	1914	+
	1915	+ if (kvm_vcpu_has_events(vcpu) \|\| signal_pending(current))
	1916	+ break;
	1917	+
	1918	+ if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) {
	1919	+ ret = -EIO;
	1920	+ break;
	1921	+ }
	1922	+
	1923	+ cond_resched();
	1924	+ }
	1925	+ return ret;
	1926	+}
	1927	+
	1928	+int tdx_complete_emulated_msr(struct kvm_vcpu *vcpu, int err)
	1929	+{
	1930	+ if (err) {
	1931	+ tdvmcall_set_return_code(vcpu, TDVMCALL_STATUS_INVALID_OPERAND);
	1932	+ return 1;
	1933	+ }
	1934	+
	1935	+ if (vmx_get_exit_reason(vcpu).basic == EXIT_REASON_MSR_READ)
	1936	+ tdvmcall_set_return_val(vcpu, kvm_read_edx_eax(vcpu));
	1937	+
	1938	+ return 1;
	1939	+}
	1940	+
	1941	+
	1942	+int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath)
	1943	+{
	1944	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	1945	+ u64 vp_enter_ret = tdx->vp_enter_ret;
	1946	+ union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu);
	1947	+
	1948	+ if (fastpath != EXIT_FASTPATH_NONE)
	1949	+ return 1;
	1950	+
	1951	+ if (unlikely(vp_enter_ret == EXIT_REASON_EPT_MISCONFIG)) {
	1952	+ KVM_BUG_ON(1, vcpu->kvm);
	1953	+ return -EIO;
	1954	+ }
	1955	+
	1956	+ /*
	1957	+ * Handle TDX SW errors, including TDX_SEAMCALL_UD, TDX_SEAMCALL_GP and
	1958	+ * TDX_SEAMCALL_VMFAILINVALID.
	1959	+ */
	1960	+ if (unlikely((vp_enter_ret & TDX_SW_ERROR) == TDX_SW_ERROR)) {
	1961	+ KVM_BUG_ON(!kvm_rebooting, vcpu->kvm);
	1962	+ goto unhandled_exit;
	1963	+ }
	1964	+
	1965	+ if (unlikely(tdx_failed_vmentry(vcpu))) {
	1966	+ /*
	1967	+ * If the guest state is protected, that means off-TD debug is
	1968	+ * not enabled, TDX_NON_RECOVERABLE must be set.
	1969	+ */
	1970	+ WARN_ON_ONCE(vcpu->arch.guest_state_protected &&
	1971	+ !(vp_enter_ret & TDX_NON_RECOVERABLE));
	1972	+ vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
	1973	+ vcpu->run->fail_entry.hardware_entry_failure_reason = exit_reason.full;
	1974	+ vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;
	1975	+ return 0;
	1976	+ }
	1977	+
	1978	+ if (unlikely(vp_enter_ret & (TDX_ERROR \| TDX_NON_RECOVERABLE)) &&
	1979	+ exit_reason.basic != EXIT_REASON_TRIPLE_FAULT) {
	1980	+ kvm_pr_unimpl("TD vp_enter_ret 0x%llx\n", vp_enter_ret);
	1981	+ goto unhandled_exit;
	1982	+ }
	1983	+
	1984	+ WARN_ON_ONCE(exit_reason.basic != EXIT_REASON_TRIPLE_FAULT &&
	1985	+ (vp_enter_ret & TDX_SEAMCALL_STATUS_MASK) != TDX_SUCCESS);
	1986	+
	1987	+ switch (exit_reason.basic) {
	1988	+ case EXIT_REASON_TRIPLE_FAULT:
	1989	+ vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
	1990	+ vcpu->mmio_needed = 0;
	1991	+ return 0;
	1992	+ case EXIT_REASON_EXCEPTION_NMI:
	1993	+ return tdx_handle_exception_nmi(vcpu);
	1994	+ case EXIT_REASON_EXTERNAL_INTERRUPT:
	1995	+ ++vcpu->stat.irq_exits;
	1996	+ return 1;
	1997	+ case EXIT_REASON_CPUID:
	1998	+ return tdx_emulate_cpuid(vcpu);
	1999	+ case EXIT_REASON_HLT:
	2000	+ return kvm_emulate_halt_noskip(vcpu);
	2001	+ case EXIT_REASON_TDCALL:
	2002	+ return handle_tdvmcall(vcpu);
	2003	+ case EXIT_REASON_VMCALL:
	2004	+ return tdx_emulate_vmcall(vcpu);
	2005	+ case EXIT_REASON_IO_INSTRUCTION:
	2006	+ return tdx_emulate_io(vcpu);
	2007	+ case EXIT_REASON_MSR_READ:
	2008	+ kvm_rcx_write(vcpu, tdx->vp_enter_args.r12);
	2009	+ return kvm_emulate_rdmsr(vcpu);
	2010	+ case EXIT_REASON_MSR_WRITE:
	2011	+ kvm_rcx_write(vcpu, tdx->vp_enter_args.r12);
	2012	+ kvm_rax_write(vcpu, tdx->vp_enter_args.r13 & -1u);
	2013	+ kvm_rdx_write(vcpu, tdx->vp_enter_args.r13 >> 32);
	2014	+ return kvm_emulate_wrmsr(vcpu);
	2015	+ case EXIT_REASON_EPT_MISCONFIG:
	2016	+ return tdx_emulate_mmio(vcpu);
	2017	+ case EXIT_REASON_EPT_VIOLATION:
	2018	+ return tdx_handle_ept_violation(vcpu);
	2019	+ case EXIT_REASON_OTHER_SMI:
	2020	+ /*
	2021	+ * Unlike VMX, SMI in SEAM non-root mode (i.e. when
	2022	+ * TD guest vCPU is running) will cause VM exit to TDX module,
	2023	+ * then SEAMRET to KVM. Once it exits to KVM, SMI is delivered
	2024	+ * and handled by kernel handler right away.
	2025	+ *
	2026	+ * The Other SMI exit can also be caused by the SEAM non-root
	2027	+ * machine check delivered via Machine Check System Management
	2028	+ * Interrupt (MSMI), but it has already been handled by the
	2029	+ * kernel machine check handler, i.e., the memory page has been
	2030	+ * marked as poisoned and it won't be freed to the free list
	2031	+ * when the TDX guest is terminated (the TDX module marks the
	2032	+ * guest as dead and prevent it from further running when
	2033	+ * machine check happens in SEAM non-root).
	2034	+ *
	2035	+ * - A MSMI will not reach here, it's handled as non_recoverable
	2036	+ * case above.
	2037	+ * - If it's not an MSMI, no need to do anything here.
	2038	+ */
	2039	+ return 1;
	2040	+ default:
	2041	+ break;
	2042	+ }
	2043	+
	2044	+unhandled_exit:
	2045	+ vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
	2046	+ vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
	2047	+ vcpu->run->internal.ndata = 2;
	2048	+ vcpu->run->internal.data[0] = vp_enter_ret;
	2049	+ vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
	2050	+ return 0;
	2051	+}
	2052	+
	2053	+void tdx_get_exit_info(struct kvm_vcpu vcpu, u32 reason,
	2054	+ u64 info1, u64 info2, u32 intr_info, u32 error_code)
	2055	+{
	2056	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	2057	+
	2058	+ *reason = tdx->vt.exit_reason.full;
	2059	+ if (*reason != -1u) {
	2060	+ *info1 = vmx_get_exit_qual(vcpu);
	2061	+ *info2 = tdx->ext_exit_qualification;
	2062	+ *intr_info = vmx_get_intr_info(vcpu);
	2063	+ } else {
	2064	+ *info1 = 0;
	2065	+ *info2 = 0;
	2066	+ *intr_info = 0;
	2067	+ }
	2068	+
	2069	+ *error_code = 0;
	2070	+}
	2071	+
	2072	+bool tdx_has_emulated_msr(u32 index)
	2073	+{
	2074	+ switch (index) {
	2075	+ case MSR_IA32_UCODE_REV:
	2076	+ case MSR_IA32_ARCH_CAPABILITIES:
	2077	+ case MSR_IA32_POWER_CTL:
	2078	+ case MSR_IA32_CR_PAT:
	2079	+ case MSR_MTRRcap:
	2080	+ case MTRRphysBase_MSR(0) ... MSR_MTRRfix4K_F8000:
	2081	+ case MSR_MTRRdefType:
	2082	+ case MSR_IA32_TSC_DEADLINE:
	2083	+ case MSR_IA32_MISC_ENABLE:
	2084	+ case MSR_PLATFORM_INFO:
	2085	+ case MSR_MISC_FEATURES_ENABLES:
	2086	+ case MSR_IA32_APICBASE:
	2087	+ case MSR_EFER:
	2088	+ case MSR_IA32_FEAT_CTL:
	2089	+ case MSR_IA32_MCG_CAP:
	2090	+ case MSR_IA32_MCG_STATUS:
	2091	+ case MSR_IA32_MCG_CTL:
	2092	+ case MSR_IA32_MCG_EXT_CTL:
	2093	+ case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
	2094	+ case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
	2095	+ /* MSR_IA32_MCx_{CTL, STATUS, ADDR, MISC, CTL2} */
	2096	+ case MSR_KVM_POLL_CONTROL:
	2097	+ return true;
	2098	+ case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff:
	2099	+ /*
	2100	+ * x2APIC registers that are virtualized by the CPU can't be
	2101	+ * emulated, KVM doesn't have access to the virtual APIC page.
	2102	+ */
	2103	+ switch (index) {
	2104	+ case X2APIC_MSR(APIC_TASKPRI):
	2105	+ case X2APIC_MSR(APIC_PROCPRI):
	2106	+ case X2APIC_MSR(APIC_EOI):
	2107	+ case X2APIC_MSR(APIC_ISR) ... X2APIC_MSR(APIC_ISR + APIC_ISR_NR):
	2108	+ case X2APIC_MSR(APIC_TMR) ... X2APIC_MSR(APIC_TMR + APIC_ISR_NR):
	2109	+ case X2APIC_MSR(APIC_IRR) ... X2APIC_MSR(APIC_IRR + APIC_ISR_NR):
	2110	+ return false;
	2111	+ default:
	2112	+ return true;
	2113	+ }
	2114	+ default:
	2115	+ return false;
	2116	+ }
	2117	+}
	2118	+
	2119	+static bool tdx_is_read_only_msr(u32 index)
	2120	+{
	2121	+ return index == MSR_IA32_APICBASE \|\| index == MSR_EFER \|\|
	2122	+ index == MSR_IA32_FEAT_CTL;
	2123	+}
	2124	+
	2125	+int tdx_get_msr(struct kvm_vcpu vcpu, struct msr_data msr)
	2126	+{
	2127	+ switch (msr->index) {
	2128	+ case MSR_IA32_FEAT_CTL:
	2129	+ /*
	2130	+ * MCE and MCA are advertised via cpuid. Guest kernel could
	2131	+ * check if LMCE is enabled or not.
	2132	+ */
	2133	+ msr->data = FEAT_CTL_LOCKED;
	2134	+ if (vcpu->arch.mcg_cap & MCG_LMCE_P)
	2135	+ msr->data \|= FEAT_CTL_LMCE_ENABLED;
	2136	+ return 0;
	2137	+ case MSR_IA32_MCG_EXT_CTL:
	2138	+ if (!msr->host_initiated && !(vcpu->arch.mcg_cap & MCG_LMCE_P))
	2139	+ return 1;
	2140	+ msr->data = vcpu->arch.mcg_ext_ctl;
	2141	+ return 0;
	2142	+ default:
	2143	+ if (!tdx_has_emulated_msr(msr->index))
	2144	+ return 1;
	2145	+
	2146	+ return kvm_get_msr_common(vcpu, msr);
	2147	+ }
	2148	+}
	2149	+
	2150	+int tdx_set_msr(struct kvm_vcpu vcpu, struct msr_data msr)
	2151	+{
	2152	+ switch (msr->index) {
	2153	+ case MSR_IA32_MCG_EXT_CTL:
	2154	+ if ((!msr->host_initiated && !(vcpu->arch.mcg_cap & MCG_LMCE_P)) \|\|
	2155	+ (msr->data & ~MCG_EXT_CTL_LMCE_EN))
	2156	+ return 1;
	2157	+ vcpu->arch.mcg_ext_ctl = msr->data;
	2158	+ return 0;
	2159	+ default:
	2160	+ if (tdx_is_read_only_msr(msr->index))
	2161	+ return 1;
	2162	+
	2163	+ if (!tdx_has_emulated_msr(msr->index))
	2164	+ return 1;
	2165	+
	2166	+ return kvm_set_msr_common(vcpu, msr);
	2167	+ }
	2168	+}
	2169	+
	2170	+static int tdx_get_capabilities(struct kvm_tdx_cmd *cmd)
	2171	+{
	2172	+ const struct tdx_sys_info_td_conf *td_conf = &tdx_sysinfo->td_conf;
	2173	+ struct kvm_tdx_capabilities __user *user_caps;
	2174	+ struct kvm_tdx_capabilities *caps = NULL;
	2175	+ int ret = 0;
	2176	+
	2177	+ /* flags is reserved for future use */
	2178	+ if (cmd->flags)
	2179	+ return -EINVAL;
	2180	+
	2181	+ caps = kmalloc(sizeof(*caps) +
	2182	+ sizeof(struct kvm_cpuid_entry2) * td_conf->num_cpuid_config,
	2183	+ GFP_KERNEL);
	2184	+ if (!caps)
	2185	+ return -ENOMEM;
	2186	+
	2187	+ user_caps = u64_to_user_ptr(cmd->data);
	2188	+ if (copy_from_user(caps, user_caps, sizeof(*caps))) {
	2189	+ ret = -EFAULT;
	2190	+ goto out;
	2191	+ }
	2192	+
	2193	+ if (caps->cpuid.nent < td_conf->num_cpuid_config) {
	2194	+ ret = -E2BIG;
	2195	+ goto out;
	2196	+ }
	2197	+
	2198	+ ret = init_kvm_tdx_caps(td_conf, caps);
	2199	+ if (ret)
	2200	+ goto out;
	2201	+
	2202	+ if (copy_to_user(user_caps, caps, sizeof(*caps))) {
	2203	+ ret = -EFAULT;
	2204	+ goto out;
	2205	+ }
	2206	+
	2207	+ if (copy_to_user(user_caps->cpuid.entries, caps->cpuid.entries,
	2208	+ caps->cpuid.nent *
	2209	+ sizeof(caps->cpuid.entries[0])))
	2210	+ ret = -EFAULT;
	2211	+
	2212	+out:
	2213	+ /* kfree() accepts NULL. */
	2214	+ kfree(caps);
	2215	+ return ret;
	2216	+}
	2217	+
	2218	+/*
	2219	+ * KVM reports guest physical address in CPUID.0x800000008.EAX[23:16], which is
	2220	+ * similar to TDX's GPAW. Use this field as the interface for userspace to
	2221	+ * configure the GPAW and EPT level for TDs.
	2222	+ *
	2223	+ * Only values 48 and 52 are supported. Value 52 means GPAW-52 and EPT level
	2224	+ * 5, Value 48 means GPAW-48 and EPT level 4. For value 48, GPAW-48 is always
	2225	+ * supported. Value 52 is only supported when the platform supports 5 level
	2226	+ * EPT.
	2227	+ */
	2228	+static int setup_tdparams_eptp_controls(struct kvm_cpuid2 *cpuid,
	2229	+ struct td_params *td_params)
	2230	+{
	2231	+ const struct kvm_cpuid_entry2 *entry;
	2232	+ int guest_pa;
	2233	+
	2234	+ entry = kvm_find_cpuid_entry2(cpuid->entries, cpuid->nent, 0x80000008, 0);
	2235	+ if (!entry)
	2236	+ return -EINVAL;
	2237	+
	2238	+ guest_pa = tdx_get_guest_phys_addr_bits(entry->eax);
	2239	+
	2240	+ if (guest_pa != 48 && guest_pa != 52)
	2241	+ return -EINVAL;
	2242	+
	2243	+ if (guest_pa == 52 && !cpu_has_vmx_ept_5levels())
	2244	+ return -EINVAL;
	2245	+
	2246	+ td_params->eptp_controls = VMX_EPTP_MT_WB;
	2247	+ if (guest_pa == 52) {
	2248	+ td_params->eptp_controls \|= VMX_EPTP_PWL_5;
	2249	+ td_params->config_flags \|= TDX_CONFIG_FLAGS_MAX_GPAW;
	2250	+ } else {
	2251	+ td_params->eptp_controls \|= VMX_EPTP_PWL_4;
	2252	+ }
	2253	+
	2254	+ return 0;
	2255	+}
	2256	+
	2257	+static int setup_tdparams_cpuids(struct kvm_cpuid2 *cpuid,
	2258	+ struct td_params *td_params)
	2259	+{
	2260	+ const struct tdx_sys_info_td_conf *td_conf = &tdx_sysinfo->td_conf;
	2261	+ const struct kvm_cpuid_entry2 *entry;
	2262	+ struct tdx_cpuid_value *value;
	2263	+ int i, copy_cnt = 0;
	2264	+
	2265	+ /*
	2266	+ * td_params.cpuid_values: The number and the order of cpuid_value must
	2267	+ * be same to the one of struct tdsysinfo.{num_cpuid_config, cpuid_configs}
	2268	+ * It's assumed that td_params was zeroed.
	2269	+ */
	2270	+ for (i = 0; i < td_conf->num_cpuid_config; i++) {
	2271	+ struct kvm_cpuid_entry2 tmp;
	2272	+
	2273	+ td_init_cpuid_entry2(&tmp, i);
	2274	+
	2275	+ entry = kvm_find_cpuid_entry2(cpuid->entries, cpuid->nent,
	2276	+ tmp.function, tmp.index);
	2277	+ if (!entry)
	2278	+ continue;
	2279	+
	2280	+ if (tdx_unsupported_cpuid(entry))
	2281	+ return -EINVAL;
	2282	+
	2283	+ copy_cnt++;
	2284	+
	2285	+ value = &td_params->cpuid_values[i];
	2286	+ value->eax = entry->eax;
	2287	+ value->ebx = entry->ebx;
	2288	+ value->ecx = entry->ecx;
	2289	+ value->edx = entry->edx;
	2290	+
	2291	+ /*
	2292	+ * TDX module does not accept nonzero bits 16..23 for the
	2293	+ * CPUID[0x80000008].EAX, see setup_tdparams_eptp_controls().
	2294	+ */
	2295	+ if (tmp.function == 0x80000008)
	2296	+ value->eax = tdx_set_guest_phys_addr_bits(value->eax, 0);
	2297	+ }
	2298	+
	2299	+ /*
	2300	+ * Rely on the TDX module to reject invalid configuration, but it can't
	2301	+ * check of leafs that don't have a proper slot in td_params->cpuid_values
	2302	+ * to stick then. So fail if there were entries that didn't get copied to
	2303	+ * td_params.
	2304	+ */
	2305	+ if (copy_cnt != cpuid->nent)
	2306	+ return -EINVAL;
	2307	+
	2308	+ return 0;
	2309	+}
	2310	+
	2311	+static int setup_tdparams(struct kvm kvm, struct td_params td_params,
	2312	+ struct kvm_tdx_init_vm *init_vm)
	2313	+{
	2314	+ const struct tdx_sys_info_td_conf *td_conf = &tdx_sysinfo->td_conf;
	2315	+ struct kvm_cpuid2 *cpuid = &init_vm->cpuid;
	2316	+ int ret;
	2317	+
	2318	+ if (kvm->created_vcpus)
	2319	+ return -EBUSY;
	2320	+
	2321	+ if (init_vm->attributes & ~tdx_get_supported_attrs(td_conf))
	2322	+ return -EINVAL;
	2323	+
	2324	+ if (init_vm->xfam & ~tdx_get_supported_xfam(td_conf))
	2325	+ return -EINVAL;
	2326	+
	2327	+ td_params->max_vcpus = kvm->max_vcpus;
	2328	+ td_params->attributes = init_vm->attributes \| td_conf->attributes_fixed1;
	2329	+ td_params->xfam = init_vm->xfam \| td_conf->xfam_fixed1;
	2330	+
	2331	+ td_params->config_flags = TDX_CONFIG_FLAGS_NO_RBP_MOD;
	2332	+ td_params->tsc_frequency = TDX_TSC_KHZ_TO_25MHZ(kvm->arch.default_tsc_khz);
	2333	+
	2334	+ ret = setup_tdparams_eptp_controls(cpuid, td_params);
	2335	+ if (ret)
	2336	+ return ret;
	2337	+
	2338	+ ret = setup_tdparams_cpuids(cpuid, td_params);
	2339	+ if (ret)
	2340	+ return ret;
	2341	+
	2342	+#define MEMCPY_SAME_SIZE(dst, src) \
	2343	+ do { \
	2344	+ BUILD_BUG_ON(sizeof(dst) != sizeof(src)); \
	2345	+ memcpy((dst), (src), sizeof(dst)); \
	2346	+ } while (0)
	2347	+
	2348	+ MEMCPY_SAME_SIZE(td_params->mrconfigid, init_vm->mrconfigid);
	2349	+ MEMCPY_SAME_SIZE(td_params->mrowner, init_vm->mrowner);
	2350	+ MEMCPY_SAME_SIZE(td_params->mrownerconfig, init_vm->mrownerconfig);
	2351	+
	2352	+ return 0;
	2353	+}
	2354	+
	2355	+static int __tdx_td_init(struct kvm kvm, struct td_params td_params,
	2356	+ u64 *seamcall_err)
	2357	+{
	2358	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	2359	+ cpumask_var_t packages;
	2360	+ struct page **tdcs_pages = NULL;
	2361	+ struct page *tdr_page;
	2362	+ int ret, i;
	2363	+ u64 err, rcx;
	2364	+
	2365	+ *seamcall_err = 0;
	2366	+ ret = tdx_guest_keyid_alloc();
	2367	+ if (ret < 0)
	2368	+ return ret;
	2369	+ kvm_tdx->hkid = ret;
	2370	+ kvm_tdx->misc_cg = get_current_misc_cg();
	2371	+ ret = misc_cg_try_charge(MISC_CG_RES_TDX, kvm_tdx->misc_cg, 1);
	2372	+ if (ret)
	2373	+ goto free_hkid;
	2374	+
	2375	+ ret = -ENOMEM;
	2376	+
	2377	+ atomic_inc(&nr_configured_hkid);
	2378	+
	2379	+ tdr_page = alloc_page(GFP_KERNEL);
	2380	+ if (!tdr_page)
	2381	+ goto free_hkid;
	2382	+
	2383	+ kvm_tdx->td.tdcs_nr_pages = tdx_sysinfo->td_ctrl.tdcs_base_size / PAGE_SIZE;
	2384	+ /* TDVPS = TDVPR(4K page) + TDCX(multiple 4K pages), -1 for TDVPR. */
	2385	+ kvm_tdx->td.tdcx_nr_pages = tdx_sysinfo->td_ctrl.tdvps_base_size / PAGE_SIZE - 1;
	2386	+ tdcs_pages = kcalloc(kvm_tdx->td.tdcs_nr_pages, sizeof(*kvm_tdx->td.tdcs_pages),
	2387	+ GFP_KERNEL \| __GFP_ZERO);
	2388	+ if (!tdcs_pages)
	2389	+ goto free_tdr;
	2390	+
	2391	+ for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) {
	2392	+ tdcs_pages[i] = alloc_page(GFP_KERNEL);
	2393	+ if (!tdcs_pages[i])
	2394	+ goto free_tdcs;
	2395	+ }
	2396	+
	2397	+ if (!zalloc_cpumask_var(&packages, GFP_KERNEL))
	2398	+ goto free_tdcs;
	2399	+
	2400	+ cpus_read_lock();
	2401	+
	2402	+ /*
	2403	+ * Need at least one CPU of the package to be online in order to
	2404	+ * program all packages for host key id. Check it.
	2405	+ */
	2406	+ for_each_present_cpu(i)
	2407	+ cpumask_set_cpu(topology_physical_package_id(i), packages);
	2408	+ for_each_online_cpu(i)
	2409	+ cpumask_clear_cpu(topology_physical_package_id(i), packages);
	2410	+ if (!cpumask_empty(packages)) {
	2411	+ ret = -EIO;
	2412	+ /*
	2413	+ * Because it's hard for human operator to figure out the
	2414	+ * reason, warn it.
	2415	+ */
	2416	+#define MSG_ALLPKG "All packages need to have online CPU to create TD. Online CPU and retry.\n"
	2417	+ pr_warn_ratelimited(MSG_ALLPKG);
	2418	+ goto free_packages;
	2419	+ }
	2420	+
	2421	+ /*
	2422	+ * TDH.MNG.CREATE tries to grab the global TDX module and fails
	2423	+ * with TDX_OPERAND_BUSY when it fails to grab. Take the global
	2424	+ * lock to prevent it from failure.
	2425	+ */
	2426	+ mutex_lock(&tdx_lock);
	2427	+ kvm_tdx->td.tdr_page = tdr_page;
	2428	+ err = tdh_mng_create(&kvm_tdx->td, kvm_tdx->hkid);
	2429	+ mutex_unlock(&tdx_lock);
	2430	+
	2431	+ if (err == TDX_RND_NO_ENTROPY) {
	2432	+ ret = -EAGAIN;
	2433	+ goto free_packages;
	2434	+ }
	2435	+
	2436	+ if (WARN_ON_ONCE(err)) {
	2437	+ pr_tdx_error(TDH_MNG_CREATE, err);
	2438	+ ret = -EIO;
	2439	+ goto free_packages;
	2440	+ }
	2441	+
	2442	+ for_each_online_cpu(i) {
	2443	+ int pkg = topology_physical_package_id(i);
	2444	+
	2445	+ if (cpumask_test_and_set_cpu(pkg, packages))
	2446	+ continue;
	2447	+
	2448	+ /*
	2449	+ * Program the memory controller in the package with an
	2450	+ * encryption key associated to a TDX private host key id
	2451	+ * assigned to this TDR. Concurrent operations on same memory
	2452	+ * controller results in TDX_OPERAND_BUSY. No locking needed
	2453	+ * beyond the cpus_read_lock() above as it serializes against
	2454	+ * hotplug and the first online CPU of the package is always
	2455	+ * used. We never have two CPUs in the same socket trying to
	2456	+ * program the key.
	2457	+ */
	2458	+ ret = smp_call_on_cpu(i, tdx_do_tdh_mng_key_config,
	2459	+ kvm_tdx, true);
	2460	+ if (ret)
	2461	+ break;
	2462	+ }
	2463	+ cpus_read_unlock();
	2464	+ free_cpumask_var(packages);
	2465	+ if (ret) {
	2466	+ i = 0;
	2467	+ goto teardown;
	2468	+ }
	2469	+
	2470	+ kvm_tdx->td.tdcs_pages = tdcs_pages;
	2471	+ for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) {
	2472	+ err = tdh_mng_addcx(&kvm_tdx->td, tdcs_pages[i]);
	2473	+ if (err == TDX_RND_NO_ENTROPY) {
	2474	+ /* Here it's hard to allow userspace to retry. */
	2475	+ ret = -EAGAIN;
	2476	+ goto teardown;
	2477	+ }
	2478	+ if (WARN_ON_ONCE(err)) {
	2479	+ pr_tdx_error(TDH_MNG_ADDCX, err);
	2480	+ ret = -EIO;
	2481	+ goto teardown;
	2482	+ }
	2483	+ }
	2484	+
	2485	+ err = tdh_mng_init(&kvm_tdx->td, __pa(td_params), &rcx);
	2486	+ if ((err & TDX_SEAMCALL_STATUS_MASK) == TDX_OPERAND_INVALID) {
	2487	+ /*
	2488	+ * Because a user gives operands, don't warn.
	2489	+ * Return a hint to the user because it's sometimes hard for the
	2490	+ * user to figure out which operand is invalid. SEAMCALL status
	2491	+ * code includes which operand caused invalid operand error.
	2492	+ */
	2493	+ *seamcall_err = err;
	2494	+ ret = -EINVAL;
	2495	+ goto teardown;
	2496	+ } else if (WARN_ON_ONCE(err)) {
	2497	+ pr_tdx_error_1(TDH_MNG_INIT, err, rcx);
	2498	+ ret = -EIO;
	2499	+ goto teardown;
	2500	+ }
	2501	+
	2502	+ return 0;
	2503	+
	2504	+ /*
	2505	+ * The sequence for freeing resources from a partially initialized TD
	2506	+ * varies based on where in the initialization flow failure occurred.
	2507	+ * Simply use the full teardown and destroy, which naturally play nice
	2508	+ * with partial initialization.
	2509	+ */
	2510	+teardown:
	2511	+ /* Only free pages not yet added, so start at 'i' */
	2512	+ for (; i < kvm_tdx->td.tdcs_nr_pages; i++) {
	2513	+ if (tdcs_pages[i]) {
	2514	+ __free_page(tdcs_pages[i]);
	2515	+ tdcs_pages[i] = NULL;
	2516	+ }
	2517	+ }
	2518	+ if (!kvm_tdx->td.tdcs_pages)
	2519	+ kfree(tdcs_pages);
	2520	+
	2521	+ tdx_mmu_release_hkid(kvm);
	2522	+ tdx_reclaim_td_control_pages(kvm);
	2523	+
	2524	+ return ret;
	2525	+
	2526	+free_packages:
	2527	+ cpus_read_unlock();
	2528	+ free_cpumask_var(packages);
	2529	+
	2530	+free_tdcs:
	2531	+ for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) {
	2532	+ if (tdcs_pages[i])
	2533	+ __free_page(tdcs_pages[i]);
	2534	+ }
	2535	+ kfree(tdcs_pages);
	2536	+ kvm_tdx->td.tdcs_pages = NULL;
	2537	+
	2538	+free_tdr:
	2539	+ if (tdr_page)
	2540	+ __free_page(tdr_page);
	2541	+ kvm_tdx->td.tdr_page = 0;
	2542	+
	2543	+free_hkid:
	2544	+ tdx_hkid_free(kvm_tdx);
	2545	+
	2546	+ return ret;
	2547	+}
	2548	+
	2549	+static u64 tdx_td_metadata_field_read(struct kvm_tdx *tdx, u64 field_id,
	2550	+ u64 *data)
	2551	+{
	2552	+ u64 err;
	2553	+
	2554	+ err = tdh_mng_rd(&tdx->td, field_id, data);
	2555	+
	2556	+ return err;
	2557	+}
	2558	+
	2559	+#define TDX_MD_UNREADABLE_LEAF_MASK GENMASK(30, 7)
	2560	+#define TDX_MD_UNREADABLE_SUBLEAF_MASK GENMASK(31, 7)
	2561	+
	2562	+static int tdx_read_cpuid(struct kvm_vcpu *vcpu, u32 leaf, u32 sub_leaf,
	2563	+ bool sub_leaf_set, int *entry_index,
	2564	+ struct kvm_cpuid_entry2 *out)
	2565	+{
	2566	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);
	2567	+ u64 field_id = TD_MD_FIELD_ID_CPUID_VALUES;
	2568	+ u64 ebx_eax, edx_ecx;
	2569	+ u64 err = 0;
	2570	+
	2571	+ if (sub_leaf > 0b1111111)
	2572	+ return -EINVAL;
	2573	+
	2574	+ if (*entry_index >= KVM_MAX_CPUID_ENTRIES)
	2575	+ return -EINVAL;
	2576	+
	2577	+ if (leaf & TDX_MD_UNREADABLE_LEAF_MASK \|\|
	2578	+ sub_leaf & TDX_MD_UNREADABLE_SUBLEAF_MASK)
	2579	+ return -EINVAL;
	2580	+
	2581	+ /*
	2582	+ * bit 23:17, REVSERVED: reserved, must be 0;
	2583	+ * bit 16, LEAF_31: leaf number bit 31;
	2584	+ * bit 15:9, LEAF_6_0: leaf number bits 6:0, leaf bits 30:7 are
	2585	+ * implicitly 0;
	2586	+ * bit 8, SUBLEAF_NA: sub-leaf not applicable flag;
	2587	+ * bit 7:1, SUBLEAF_6_0: sub-leaf number bits 6:0. If SUBLEAF_NA is 1,
	2588	+ * the SUBLEAF_6_0 is all-1.
	2589	+ * sub-leaf bits 31:7 are implicitly 0;
	2590	+ * bit 0, ELEMENT_I: Element index within field;
	2591	+ */
	2592	+ field_id \|= ((leaf & 0x80000000) ? 1 : 0) << 16;
	2593	+ field_id \|= (leaf & 0x7f) << 9;
	2594	+ if (sub_leaf_set)
	2595	+ field_id \|= (sub_leaf & 0x7f) << 1;
	2596	+ else
	2597	+ field_id \|= 0x1fe;
	2598	+
	2599	+ err = tdx_td_metadata_field_read(kvm_tdx, field_id, &ebx_eax);
	2600	+ if (err) //TODO check for specific errors
	2601	+ goto err_out;
	2602	+
	2603	+ out->eax = (u32) ebx_eax;
	2604	+ out->ebx = (u32) (ebx_eax >> 32);
	2605	+
	2606	+ field_id++;
	2607	+ err = tdx_td_metadata_field_read(kvm_tdx, field_id, &edx_ecx);
	2608	+ /*
	2609	+ * It's weird that reading edx_ecx fails while reading ebx_eax
	2610	+ * succeeded.
	2611	+ */
	2612	+ if (WARN_ON_ONCE(err))
	2613	+ goto err_out;
	2614	+
	2615	+ out->ecx = (u32) edx_ecx;
	2616	+ out->edx = (u32) (edx_ecx >> 32);
	2617	+
	2618	+ out->function = leaf;
	2619	+ out->index = sub_leaf;
	2620	+ out->flags \|= sub_leaf_set ? KVM_CPUID_FLAG_SIGNIFCANT_INDEX : 0;
	2621	+
	2622	+ /*
	2623	+ * Work around missing support on old TDX modules, fetch
	2624	+ * guest maxpa from gfn_direct_bits.
	2625	+ */
	2626	+ if (leaf == 0x80000008) {
	2627	+ gpa_t gpa_bits = gfn_to_gpa(kvm_gfn_direct_bits(vcpu->kvm));
	2628	+ unsigned int g_maxpa = __ffs(gpa_bits) + 1;
	2629	+
	2630	+ out->eax = tdx_set_guest_phys_addr_bits(out->eax, g_maxpa);
	2631	+ }
	2632	+
	2633	+ (*entry_index)++;
	2634	+
	2635	+ return 0;
	2636	+
	2637	+err_out:
	2638	+ out->eax = 0;
	2639	+ out->ebx = 0;
	2640	+ out->ecx = 0;
	2641	+ out->edx = 0;
	2642	+
	2643	+ return -EIO;
	2644	+}
	2645	+
	2646	+static int tdx_td_init(struct kvm kvm, struct kvm_tdx_cmd cmd)
	2647	+{
	2648	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	2649	+ struct kvm_tdx_init_vm *init_vm;
	2650	+ struct td_params *td_params = NULL;
	2651	+ int ret;
	2652	+
	2653	+ BUILD_BUG_ON(sizeof(*init_vm) != 256 + sizeof_field(struct kvm_tdx_init_vm, cpuid));
	2654	+ BUILD_BUG_ON(sizeof(struct td_params) != 1024);
	2655	+
	2656	+ if (kvm_tdx->state != TD_STATE_UNINITIALIZED)
	2657	+ return -EINVAL;
	2658	+
	2659	+ if (cmd->flags)
	2660	+ return -EINVAL;
	2661	+
	2662	+ init_vm = kmalloc(sizeof(*init_vm) +
	2663	+ sizeof(init_vm->cpuid.entries[0]) * KVM_MAX_CPUID_ENTRIES,
	2664	+ GFP_KERNEL);
	2665	+ if (!init_vm)
	2666	+ return -ENOMEM;
	2667	+
	2668	+ if (copy_from_user(init_vm, u64_to_user_ptr(cmd->data), sizeof(*init_vm))) {
	2669	+ ret = -EFAULT;
	2670	+ goto out;
	2671	+ }
	2672	+
	2673	+ if (init_vm->cpuid.nent > KVM_MAX_CPUID_ENTRIES) {
	2674	+ ret = -E2BIG;
	2675	+ goto out;
	2676	+ }
	2677	+
	2678	+ if (copy_from_user(init_vm->cpuid.entries,
	2679	+ u64_to_user_ptr(cmd->data) + sizeof(*init_vm),
	2680	+ flex_array_size(init_vm, cpuid.entries, init_vm->cpuid.nent))) {
	2681	+ ret = -EFAULT;
	2682	+ goto out;
	2683	+ }
	2684	+
	2685	+ if (memchr_inv(init_vm->reserved, 0, sizeof(init_vm->reserved))) {
	2686	+ ret = -EINVAL;
	2687	+ goto out;
	2688	+ }
	2689	+
	2690	+ if (init_vm->cpuid.padding) {
	2691	+ ret = -EINVAL;
	2692	+ goto out;
	2693	+ }
	2694	+
	2695	+ td_params = kzalloc(sizeof(struct td_params), GFP_KERNEL);
	2696	+ if (!td_params) {
	2697	+ ret = -ENOMEM;
	2698	+ goto out;
	2699	+ }
	2700	+
	2701	+ ret = setup_tdparams(kvm, td_params, init_vm);
	2702	+ if (ret)
	2703	+ goto out;
	2704	+
	2705	+ ret = __tdx_td_init(kvm, td_params, &cmd->hw_error);
	2706	+ if (ret)
	2707	+ goto out;
	2708	+
	2709	+ kvm_tdx->tsc_offset = td_tdcs_exec_read64(kvm_tdx, TD_TDCS_EXEC_TSC_OFFSET);
	2710	+ kvm_tdx->tsc_multiplier = td_tdcs_exec_read64(kvm_tdx, TD_TDCS_EXEC_TSC_MULTIPLIER);
	2711	+ kvm_tdx->attributes = td_params->attributes;
	2712	+ kvm_tdx->xfam = td_params->xfam;
	2713	+
	2714	+ if (td_params->config_flags & TDX_CONFIG_FLAGS_MAX_GPAW)
	2715	+ kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_5;
	2716	+ else
	2717	+ kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_4;
	2718	+
	2719	+ kvm_tdx->state = TD_STATE_INITIALIZED;
	2720	+out:
	2721	+ /* kfree() accepts NULL. */
	2722	+ kfree(init_vm);
	2723	+ kfree(td_params);
	2724	+
	2725	+ return ret;
	2726	+}
	2727	+
	2728	+void tdx_flush_tlb_current(struct kvm_vcpu *vcpu)
	2729	+{
	2730	+ /*
	2731	+ * flush_tlb_current() is invoked when the first time for the vcpu to
	2732	+ * run or when root of shared EPT is invalidated.
	2733	+ * KVM only needs to flush shared EPT because the TDX module handles TLB
	2734	+ * invalidation for private EPT in tdh_vp_enter();
	2735	+ *
	2736	+ * A single context invalidation for shared EPT can be performed here.
	2737	+ * However, this single context invalidation requires the private EPTP
	2738	+ * rather than the shared EPTP to flush shared EPT, as shared EPT uses
	2739	+ * private EPTP as its ASID for TLB invalidation.
	2740	+ *
	2741	+ * To avoid reading back private EPTP, perform a global invalidation for
	2742	+ * shared EPT instead to keep this function simple.
	2743	+ */
	2744	+ ept_sync_global();
	2745	+}
	2746	+
	2747	+void tdx_flush_tlb_all(struct kvm_vcpu *vcpu)
	2748	+{
	2749	+ /*
	2750	+ * TDX has called tdx_track() in tdx_sept_remove_private_spte() to
	2751	+ * ensure that private EPT will be flushed on the next TD enter. No need
	2752	+ * to call tdx_track() here again even when this callback is a result of
	2753	+ * zapping private EPT.
	2754	+ *
	2755	+ * Due to the lack of the context to determine which EPT has been
	2756	+ * affected by zapping, invoke invept() directly here for both shared
	2757	+ * EPT and private EPT for simplicity, though it's not necessary for
	2758	+ * private EPT.
	2759	+ */
	2760	+ ept_sync_global();
	2761	+}
	2762	+
	2763	+static int tdx_td_finalize(struct kvm kvm, struct kvm_tdx_cmd cmd)
	2764	+{
	2765	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	2766	+
	2767	+ guard(mutex)(&kvm->slots_lock);
	2768	+
	2769	+ if (!is_hkid_assigned(kvm_tdx) \|\| kvm_tdx->state == TD_STATE_RUNNABLE)
	2770	+ return -EINVAL;
	2771	+ /*
	2772	+ * Pages are pending for KVM_TDX_INIT_MEM_REGION to issue
	2773	+ * TDH.MEM.PAGE.ADD().
	2774	+ */
	2775	+ if (atomic64_read(&kvm_tdx->nr_premapped))
	2776	+ return -EINVAL;
	2777	+
	2778	+ cmd->hw_error = tdh_mr_finalize(&kvm_tdx->td);
	2779	+ if (tdx_operand_busy(cmd->hw_error))
	2780	+ return -EBUSY;
	2781	+ if (KVM_BUG_ON(cmd->hw_error, kvm)) {
	2782	+ pr_tdx_error(TDH_MR_FINALIZE, cmd->hw_error);
	2783	+ return -EIO;
	2784	+ }
	2785	+
	2786	+ kvm_tdx->state = TD_STATE_RUNNABLE;
	2787	+ /* TD_STATE_RUNNABLE must be set before 'pre_fault_allowed' */
	2788	+ smp_wmb();
	2789	+ kvm->arch.pre_fault_allowed = true;
	2790	+ return 0;
	2791	+}
	2792	+
	2793	+int tdx_vm_ioctl(struct kvm kvm, void __user argp)
	2794	+{
	2795	+ struct kvm_tdx_cmd tdx_cmd;
	2796	+ int r;
	2797	+
	2798	+ if (copy_from_user(&tdx_cmd, argp, sizeof(struct kvm_tdx_cmd)))
	2799	+ return -EFAULT;
	2800	+
	2801	+ /*
	2802	+ * Userspace should never set hw_error. It is used to fill
	2803	+ * hardware-defined error by the kernel.
	2804	+ */
	2805	+ if (tdx_cmd.hw_error)
	2806	+ return -EINVAL;
	2807	+
	2808	+ mutex_lock(&kvm->lock);
	2809	+
	2810	+ switch (tdx_cmd.id) {
	2811	+ case KVM_TDX_CAPABILITIES:
	2812	+ r = tdx_get_capabilities(&tdx_cmd);
	2813	+ break;
	2814	+ case KVM_TDX_INIT_VM:
	2815	+ r = tdx_td_init(kvm, &tdx_cmd);
	2816	+ break;
	2817	+ case KVM_TDX_FINALIZE_VM:
	2818	+ r = tdx_td_finalize(kvm, &tdx_cmd);
	2819	+ break;
	2820	+ default:
	2821	+ r = -EINVAL;
	2822	+ goto out;
	2823	+ }
	2824	+
	2825	+ if (copy_to_user(argp, &tdx_cmd, sizeof(struct kvm_tdx_cmd)))
	2826	+ r = -EFAULT;
	2827	+
	2828	+out:
	2829	+ mutex_unlock(&kvm->lock);
	2830	+ return r;
	2831	+}
	2832	+
	2833	+/* VMM can pass one 64bit auxiliary data to vcpu via RCX for guest BIOS. */
	2834	+static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)
	2835	+{
	2836	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);
	2837	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	2838	+ struct page *page;
	2839	+ int ret, i;
	2840	+ u64 err;
	2841	+
	2842	+ page = alloc_page(GFP_KERNEL);
	2843	+ if (!page)
	2844	+ return -ENOMEM;
	2845	+ tdx->vp.tdvpr_page = page;
	2846	+
	2847	+ tdx->vp.tdcx_pages = kcalloc(kvm_tdx->td.tdcx_nr_pages, sizeof(*tdx->vp.tdcx_pages),
	2848	+ GFP_KERNEL);
	2849	+ if (!tdx->vp.tdcx_pages) {
	2850	+ ret = -ENOMEM;
	2851	+ goto free_tdvpr;
	2852	+ }
	2853	+
	2854	+ for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) {
	2855	+ page = alloc_page(GFP_KERNEL);
	2856	+ if (!page) {
	2857	+ ret = -ENOMEM;
	2858	+ goto free_tdcx;
	2859	+ }
	2860	+ tdx->vp.tdcx_pages[i] = page;
	2861	+ }
	2862	+
	2863	+ err = tdh_vp_create(&kvm_tdx->td, &tdx->vp);
	2864	+ if (KVM_BUG_ON(err, vcpu->kvm)) {
	2865	+ ret = -EIO;
	2866	+ pr_tdx_error(TDH_VP_CREATE, err);
	2867	+ goto free_tdcx;
	2868	+ }
	2869	+
	2870	+ for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) {
	2871	+ err = tdh_vp_addcx(&tdx->vp, tdx->vp.tdcx_pages[i]);
	2872	+ if (KVM_BUG_ON(err, vcpu->kvm)) {
	2873	+ pr_tdx_error(TDH_VP_ADDCX, err);
	2874	+ /*
	2875	+ * Pages already added are reclaimed by the vcpu_free
	2876	+ * method, but the rest are freed here.
	2877	+ */
	2878	+ for (; i < kvm_tdx->td.tdcx_nr_pages; i++) {
	2879	+ __free_page(tdx->vp.tdcx_pages[i]);
	2880	+ tdx->vp.tdcx_pages[i] = NULL;
	2881	+ }
	2882	+ return -EIO;
	2883	+ }
	2884	+ }
	2885	+
	2886	+ err = tdh_vp_init(&tdx->vp, vcpu_rcx, vcpu->vcpu_id);
	2887	+ if (KVM_BUG_ON(err, vcpu->kvm)) {
	2888	+ pr_tdx_error(TDH_VP_INIT, err);
	2889	+ return -EIO;
	2890	+ }
	2891	+
	2892	+ vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
	2893	+
	2894	+ return 0;
	2895	+
	2896	+free_tdcx:
	2897	+ for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) {
	2898	+ if (tdx->vp.tdcx_pages[i])
	2899	+ __free_page(tdx->vp.tdcx_pages[i]);
	2900	+ tdx->vp.tdcx_pages[i] = NULL;
	2901	+ }
	2902	+ kfree(tdx->vp.tdcx_pages);
	2903	+ tdx->vp.tdcx_pages = NULL;
	2904	+
	2905	+free_tdvpr:
	2906	+ if (tdx->vp.tdvpr_page)
	2907	+ __free_page(tdx->vp.tdvpr_page);
	2908	+ tdx->vp.tdvpr_page = 0;
	2909	+
	2910	+ return ret;
	2911	+}
	2912	+
	2913	+/* Sometimes reads multipple subleafs. Return how many enties were written. */
	2914	+static int tdx_vcpu_get_cpuid_leaf(struct kvm_vcpu vcpu, u32 leaf, int entry_index,
	2915	+ struct kvm_cpuid_entry2 *output_e)
	2916	+{
	2917	+ int sub_leaf = 0;
	2918	+ int ret;
	2919	+
	2920	+ /* First try without a subleaf */
	2921	+ ret = tdx_read_cpuid(vcpu, leaf, 0, false, entry_index, output_e);
	2922	+
	2923	+ /* If success, or invalid leaf, just give up */
	2924	+ if (ret != -EIO)
	2925	+ return ret;
	2926	+
	2927	+ /*
	2928	+ * If the try without a subleaf failed, try reading subleafs until
	2929	+ * failure. The TDX module only supports 6 bits of subleaf index.
	2930	+ */
	2931	+ while (1) {
	2932	+ /* Keep reading subleafs until there is a failure. */
	2933	+ if (tdx_read_cpuid(vcpu, leaf, sub_leaf, true, entry_index, output_e))
	2934	+ return !sub_leaf;
	2935	+
	2936	+ sub_leaf++;
	2937	+ output_e++;
	2938	+ }
	2939	+
	2940	+ return 0;
	2941	+}
	2942	+
	2943	+static int tdx_vcpu_get_cpuid(struct kvm_vcpu vcpu, struct kvm_tdx_cmd cmd)
	2944	+{
	2945	+ struct kvm_cpuid2 __user output, td_cpuid;
	2946	+ int r = 0, i = 0, leaf;
	2947	+ u32 level;
	2948	+
	2949	+ output = u64_to_user_ptr(cmd->data);
	2950	+ td_cpuid = kzalloc(sizeof(*td_cpuid) +
	2951	+ sizeof(output->entries[0]) * KVM_MAX_CPUID_ENTRIES,
	2952	+ GFP_KERNEL);
	2953	+ if (!td_cpuid)
	2954	+ return -ENOMEM;
	2955	+
	2956	+ if (copy_from_user(td_cpuid, output, sizeof(*output))) {
	2957	+ r = -EFAULT;
	2958	+ goto out;
	2959	+ }
	2960	+
	2961	+ /* Read max CPUID for normal range */
	2962	+ if (tdx_vcpu_get_cpuid_leaf(vcpu, 0, &i, &td_cpuid->entries[i])) {
	2963	+ r = -EIO;
	2964	+ goto out;
	2965	+ }
	2966	+ level = td_cpuid->entries[0].eax;
	2967	+
	2968	+ for (leaf = 1; leaf <= level; leaf++)
	2969	+ tdx_vcpu_get_cpuid_leaf(vcpu, leaf, &i, &td_cpuid->entries[i]);
	2970	+
	2971	+ /* Read max CPUID for extended range */
	2972	+ if (tdx_vcpu_get_cpuid_leaf(vcpu, 0x80000000, &i, &td_cpuid->entries[i])) {
	2973	+ r = -EIO;
	2974	+ goto out;
	2975	+ }
	2976	+ level = td_cpuid->entries[i - 1].eax;
	2977	+
	2978	+ for (leaf = 0x80000001; leaf <= level; leaf++)
	2979	+ tdx_vcpu_get_cpuid_leaf(vcpu, leaf, &i, &td_cpuid->entries[i]);
	2980	+
	2981	+ if (td_cpuid->nent < i)
	2982	+ r = -E2BIG;
	2983	+ td_cpuid->nent = i;
	2984	+
	2985	+ if (copy_to_user(output, td_cpuid, sizeof(*output))) {
	2986	+ r = -EFAULT;
	2987	+ goto out;
	2988	+ }
	2989	+
	2990	+ if (r == -E2BIG)
	2991	+ goto out;
	2992	+
	2993	+ if (copy_to_user(output->entries, td_cpuid->entries,
	2994	+ td_cpuid->nent * sizeof(struct kvm_cpuid_entry2)))
	2995	+ r = -EFAULT;
	2996	+
	2997	+out:
	2998	+ kfree(td_cpuid);
	2999	+
	3000	+ return r;
	3001	+}
	3002	+
	3003	+static int tdx_vcpu_init(struct kvm_vcpu vcpu, struct kvm_tdx_cmd cmd)
	3004	+{
	3005	+ u64 apic_base;
	3006	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	3007	+ int ret;
	3008	+
	3009	+ if (cmd->flags)
	3010	+ return -EINVAL;
	3011	+
	3012	+ if (tdx->state != VCPU_TD_STATE_UNINITIALIZED)
	3013	+ return -EINVAL;
	3014	+
	3015	+ /*
	3016	+ * TDX requires X2APIC, userspace is responsible for configuring guest
	3017	+ * CPUID accordingly.
	3018	+ */
	3019	+ apic_base = APIC_DEFAULT_PHYS_BASE \| LAPIC_MODE_X2APIC \|
	3020	+ (kvm_vcpu_is_reset_bsp(vcpu) ? MSR_IA32_APICBASE_BSP : 0);
	3021	+ if (kvm_apic_set_base(vcpu, apic_base, true))
	3022	+ return -EINVAL;
	3023	+
	3024	+ ret = tdx_td_vcpu_init(vcpu, (u64)cmd->data);
	3025	+ if (ret)
	3026	+ return ret;
	3027	+
	3028	+ td_vmcs_write16(tdx, POSTED_INTR_NV, POSTED_INTR_VECTOR);
	3029	+ td_vmcs_write64(tdx, POSTED_INTR_DESC_ADDR, __pa(&tdx->vt.pi_desc));
	3030	+ td_vmcs_setbit32(tdx, PIN_BASED_VM_EXEC_CONTROL, PIN_BASED_POSTED_INTR);
	3031	+
	3032	+ tdx->state = VCPU_TD_STATE_INITIALIZED;
	3033	+
	3034	+ return 0;
	3035	+}
	3036	+
	3037	+void tdx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
	3038	+{
	3039	+ /*
	3040	+ * Yell on INIT, as TDX doesn't support INIT, i.e. KVM should drop all
	3041	+ * INIT events.
	3042	+ *
	3043	+ * Defer initializing vCPU for RESET state until KVM_TDX_INIT_VCPU, as
	3044	+ * userspace needs to define the vCPU model before KVM can initialize
	3045	+ * vCPU state, e.g. to enable x2APIC.
	3046	+ */
	3047	+ WARN_ON_ONCE(init_event);
	3048	+}
	3049	+
	3050	+struct tdx_gmem_post_populate_arg {
	3051	+ struct kvm_vcpu *vcpu;
	3052	+ __u32 flags;
	3053	+};
	3054	+
	3055	+static int tdx_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
	3056	+ void __user src, int order, void _arg)
	3057	+{
	3058	+ u64 error_code = PFERR_GUEST_FINAL_MASK \| PFERR_PRIVATE_ACCESS;
	3059	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	3060	+ struct tdx_gmem_post_populate_arg *arg = _arg;
	3061	+ struct kvm_vcpu *vcpu = arg->vcpu;
	3062	+ gpa_t gpa = gfn_to_gpa(gfn);
	3063	+ u8 level = PG_LEVEL_4K;
	3064	+ struct page *src_page;
	3065	+ int ret, i;
	3066	+ u64 err, entry, level_state;
	3067	+
	3068	+ /*
	3069	+ * Get the source page if it has been faulted in. Return failure if the
	3070	+ * source page has been swapped out or unmapped in primary memory.
	3071	+ */
	3072	+ ret = get_user_pages_fast((unsigned long)src, 1, 0, &src_page);
	3073	+ if (ret < 0)
	3074	+ return ret;
	3075	+ if (ret != 1)
	3076	+ return -ENOMEM;
	3077	+
	3078	+ ret = kvm_tdp_map_page(vcpu, gpa, error_code, &level);
	3079	+ if (ret < 0)
	3080	+ goto out;
	3081	+
	3082	+ /*
	3083	+ * The private mem cannot be zapped after kvm_tdp_map_page()
	3084	+ * because all paths are covered by slots_lock and the
	3085	+ * filemap invalidate lock. Check that they are indeed enough.
	3086	+ */
	3087	+ if (IS_ENABLED(CONFIG_KVM_PROVE_MMU)) {
	3088	+ scoped_guard(read_lock, &kvm->mmu_lock) {
	3089	+ if (KVM_BUG_ON(!kvm_tdp_mmu_gpa_is_mapped(vcpu, gpa), kvm)) {
	3090	+ ret = -EIO;
	3091	+ goto out;
	3092	+ }
	3093	+ }
	3094	+ }
	3095	+
	3096	+ ret = 0;
	3097	+ err = tdh_mem_page_add(&kvm_tdx->td, gpa, pfn_to_page(pfn),
	3098	+ src_page, &entry, &level_state);
	3099	+ if (err) {
	3100	+ ret = unlikely(tdx_operand_busy(err)) ? -EBUSY : -EIO;
	3101	+ goto out;
	3102	+ }
	3103	+
	3104	+ if (!KVM_BUG_ON(!atomic64_read(&kvm_tdx->nr_premapped), kvm))
	3105	+ atomic64_dec(&kvm_tdx->nr_premapped);
	3106	+
	3107	+ if (arg->flags & KVM_TDX_MEASURE_MEMORY_REGION) {
	3108	+ for (i = 0; i < PAGE_SIZE; i += TDX_EXTENDMR_CHUNKSIZE) {
	3109	+ err = tdh_mr_extend(&kvm_tdx->td, gpa + i, &entry,
	3110	+ &level_state);
	3111	+ if (err) {
	3112	+ ret = -EIO;
	3113	+ break;
	3114	+ }
	3115	+ }
	3116	+ }
	3117	+
	3118	+out:
	3119	+ put_page(src_page);
	3120	+ return ret;
	3121	+}
	3122	+
	3123	+static int tdx_vcpu_init_mem_region(struct kvm_vcpu vcpu, struct kvm_tdx_cmd cmd)
	3124	+{
	3125	+ struct vcpu_tdx *tdx = to_tdx(vcpu);
	3126	+ struct kvm *kvm = vcpu->kvm;
	3127	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
	3128	+ struct kvm_tdx_init_mem_region region;
	3129	+ struct tdx_gmem_post_populate_arg arg;
	3130	+ long gmem_ret;
	3131	+ int ret;
	3132	+
	3133	+ if (tdx->state != VCPU_TD_STATE_INITIALIZED)
	3134	+ return -EINVAL;
	3135	+
	3136	+ guard(mutex)(&kvm->slots_lock);
	3137	+
	3138	+ /* Once TD is finalized, the initial guest memory is fixed. */
	3139	+ if (kvm_tdx->state == TD_STATE_RUNNABLE)
	3140	+ return -EINVAL;
	3141	+
	3142	+ if (cmd->flags & ~KVM_TDX_MEASURE_MEMORY_REGION)
	3143	+ return -EINVAL;
	3144	+
	3145	+ if (copy_from_user(&region, u64_to_user_ptr(cmd->data), sizeof(region)))
	3146	+ return -EFAULT;
	3147	+
	3148	+ if (!PAGE_ALIGNED(region.source_addr) \|\| !PAGE_ALIGNED(region.gpa) \|\|
	3149	+ !region.nr_pages \|\|
	3150	+ region.gpa + (region.nr_pages << PAGE_SHIFT) <= region.gpa \|\|
	3151	+ !vt_is_tdx_private_gpa(kvm, region.gpa) \|\|
	3152	+ !vt_is_tdx_private_gpa(kvm, region.gpa + (region.nr_pages << PAGE_SHIFT) - 1))
	3153	+ return -EINVAL;
	3154	+
	3155	+ kvm_mmu_reload(vcpu);
	3156	+ ret = 0;
	3157	+ while (region.nr_pages) {
	3158	+ if (signal_pending(current)) {
	3159	+ ret = -EINTR;
	3160	+ break;
	3161	+ }
	3162	+
	3163	+ arg = (struct tdx_gmem_post_populate_arg) {
	3164	+ .vcpu = vcpu,
	3165	+ .flags = cmd->flags,
	3166	+ };
	3167	+ gmem_ret = kvm_gmem_populate(kvm, gpa_to_gfn(region.gpa),
	3168	+ u64_to_user_ptr(region.source_addr),
	3169	+ 1, tdx_gmem_post_populate, &arg);
	3170	+ if (gmem_ret < 0) {
	3171	+ ret = gmem_ret;
	3172	+ break;
	3173	+ }
	3174	+
	3175	+ if (gmem_ret != 1) {
	3176	+ ret = -EIO;
	3177	+ break;
	3178	+ }
	3179	+
	3180	+ region.source_addr += PAGE_SIZE;
	3181	+ region.gpa += PAGE_SIZE;
	3182	+ region.nr_pages--;
	3183	+
	3184	+ cond_resched();
	3185	+ }
	3186	+
	3187	+ if (copy_to_user(u64_to_user_ptr(cmd->data), &region, sizeof(region)))
	3188	+ ret = -EFAULT;
	3189	+ return ret;
	3190	+}
	3191	+
	3192	+int tdx_vcpu_ioctl(struct kvm_vcpu vcpu, void __user argp)
	3193	+{
	3194	+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);
	3195	+ struct kvm_tdx_cmd cmd;
	3196	+ int ret;
	3197	+
	3198	+ if (!is_hkid_assigned(kvm_tdx) \|\| kvm_tdx->state == TD_STATE_RUNNABLE)
	3199	+ return -EINVAL;
	3200	+
	3201	+ if (copy_from_user(&cmd, argp, sizeof(cmd)))
	3202	+ return -EFAULT;
	3203	+
	3204	+ if (cmd.hw_error)
	3205	+ return -EINVAL;
	3206	+
	3207	+ switch (cmd.id) {
	3208	+ case KVM_TDX_INIT_VCPU:
	3209	+ ret = tdx_vcpu_init(vcpu, &cmd);
	3210	+ break;
	3211	+ case KVM_TDX_INIT_MEM_REGION:
	3212	+ ret = tdx_vcpu_init_mem_region(vcpu, &cmd);
	3213	+ break;
	3214	+ case KVM_TDX_GET_CPUID:
	3215	+ ret = tdx_vcpu_get_cpuid(vcpu, &cmd);
	3216	+ break;
	3217	+ default:
	3218	+ ret = -EINVAL;
	3219	+ break;
	3220	+ }
	3221	+
	3222	+ return ret;
	3223	+}
	3224	+
	3225	+int tdx_gmem_private_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn)
	3226	+{
	3227	+ return PG_LEVEL_4K;
	3228	+}
	3229	+
	3230	+static int tdx_online_cpu(unsigned int cpu)
	3231	+{
	3232	+ unsigned long flags;
	3233	+ int r;
	3234	+
	3235	+ /* Sanity check CPU is already in post-VMXON */
	3236	+ WARN_ON_ONCE(!(cr4_read_shadow() & X86_CR4_VMXE));
	3237	+
	3238	+ local_irq_save(flags);
	3239	+ r = tdx_cpu_enable();
	3240	+ local_irq_restore(flags);
	3241	+
	3242	+ return r;
	3243	+}
	3244	+
	3245	+static int tdx_offline_cpu(unsigned int cpu)
	3246	+{
	3247	+ int i;
	3248	+
	3249	+ /* No TD is running. Allow any cpu to be offline. */
	3250	+ if (!atomic_read(&nr_configured_hkid))
	3251	+ return 0;
	3252	+
	3253	+ /*
	3254	+ * In order to reclaim TDX HKID, (i.e. when deleting guest TD), need to
	3255	+ * call TDH.PHYMEM.PAGE.WBINVD on all packages to program all memory
	3256	+ * controller with pconfig. If we have active TDX HKID, refuse to
	3257	+ * offline the last online cpu.
	3258	+ */
	3259	+ for_each_online_cpu(i) {
	3260	+ /*
	3261	+ * Found another online cpu on the same package.
	3262	+ * Allow to offline.
	3263	+ */
	3264	+ if (i != cpu && topology_physical_package_id(i) ==
	3265	+ topology_physical_package_id(cpu))
	3266	+ return 0;
	3267	+ }
	3268	+
	3269	+ /*
	3270	+ * This is the last cpu of this package. Don't offline it.
	3271	+ *
	3272	+ * Because it's hard for human operator to understand the
	3273	+ * reason, warn it.
	3274	+ */
	3275	+#define MSG_ALLPKG_ONLINE \
	3276	+ "TDX requires all packages to have an online CPU. Delete all TDs in order to offline all CPUs of a package.\n"
	3277	+ pr_warn_ratelimited(MSG_ALLPKG_ONLINE);
	3278	+ return -EBUSY;
	3279	+}
	3280	+
	3281	+static void __do_tdx_cleanup(void)
	3282	+{
	3283	+ /*
	3284	+ * Once TDX module is initialized, it cannot be disabled and
	3285	+ * re-initialized again w/o runtime update (which isn't
	3286	+ * supported by kernel). Only need to remove the cpuhp here.
	3287	+ * The TDX host core code tracks TDX status and can handle
	3288	+ * 'multiple enabling' scenario.
	3289	+ */
	3290	+ WARN_ON_ONCE(!tdx_cpuhp_state);
	3291	+ cpuhp_remove_state_nocalls_cpuslocked(tdx_cpuhp_state);
	3292	+ tdx_cpuhp_state = 0;
	3293	+}
	3294	+
	3295	+static void __tdx_cleanup(void)
	3296	+{
	3297	+ cpus_read_lock();
	3298	+ __do_tdx_cleanup();
	3299	+ cpus_read_unlock();
	3300	+}
	3301	+
	3302	+static int __init __do_tdx_bringup(void)
	3303	+{
	3304	+ int r;
	3305	+
	3306	+ /*
	3307	+ * TDX-specific cpuhp callback to call tdx_cpu_enable() on all
	3308	+ * online CPUs before calling tdx_enable(), and on any new
	3309	+ * going-online CPU to make sure it is ready for TDX guest.
	3310	+ */
	3311	+ r = cpuhp_setup_state_cpuslocked(CPUHP_AP_ONLINE_DYN,
	3312	+ "kvm/cpu/tdx:online",
	3313	+ tdx_online_cpu, tdx_offline_cpu);
	3314	+ if (r < 0)
	3315	+ return r;
	3316	+
	3317	+ tdx_cpuhp_state = r;
	3318	+
	3319	+ r = tdx_enable();
	3320	+ if (r)
	3321	+ __do_tdx_cleanup();
	3322	+
	3323	+ return r;
	3324	+}
	3325	+
	3326	+static int __init __tdx_bringup(void)
	3327	+{
	3328	+ const struct tdx_sys_info_td_conf *td_conf;
	3329	+ int r, i;
	3330	+
	3331	+ for (i = 0; i < ARRAY_SIZE(tdx_uret_msrs); i++) {
	3332	+ /*
	3333	+ * Check if MSRs (tdx_uret_msrs) can be saved/restored
	3334	+ * before returning to user space.
	3335	+ *
	3336	+ * this_cpu_ptr(user_return_msrs)->registered isn't checked
	3337	+ * because the registration is done at vcpu runtime by
	3338	+ * tdx_user_return_msr_update_cache().
	3339	+ */
	3340	+ tdx_uret_msrs[i].slot = kvm_find_user_return_msr(tdx_uret_msrs[i].msr);
	3341	+ if (tdx_uret_msrs[i].slot == -1) {
	3342	+ /* If any MSR isn't supported, it is a KVM bug */
	3343	+ pr_err("MSR %x isn't included by kvm_find_user_return_msr\n",
	3344	+ tdx_uret_msrs[i].msr);
	3345	+ return -EIO;
	3346	+ }
	3347	+ }
	3348	+
	3349	+ /*
	3350	+ * Enabling TDX requires enabling hardware virtualization first,
	3351	+ * as making SEAMCALLs requires CPU being in post-VMXON state.
	3352	+ */
	3353	+ r = kvm_enable_virtualization();
	3354	+ if (r)
	3355	+ return r;
	3356	+
	3357	+ cpus_read_lock();
	3358	+ r = __do_tdx_bringup();
	3359	+ cpus_read_unlock();
	3360	+
	3361	+ if (r)
	3362	+ goto tdx_bringup_err;
	3363	+
	3364	+ /* Get TDX global information for later use */
	3365	+ tdx_sysinfo = tdx_get_sysinfo();
	3366	+ if (WARN_ON_ONCE(!tdx_sysinfo)) {
	3367	+ r = -EINVAL;
	3368	+ goto get_sysinfo_err;
	3369	+ }
	3370	+
	3371	+ /* Check TDX module and KVM capabilities */
	3372	+ if (!tdx_get_supported_attrs(&tdx_sysinfo->td_conf) \|\|
	3373	+ !tdx_get_supported_xfam(&tdx_sysinfo->td_conf))
	3374	+ goto get_sysinfo_err;
	3375	+
	3376	+ if (!(tdx_sysinfo->features.tdx_features0 & MD_FIELD_ID_FEATURES0_TOPOLOGY_ENUM))
	3377	+ goto get_sysinfo_err;
	3378	+
	3379	+ /*
	3380	+ * TDX has its own limit of maximum vCPUs it can support for all
	3381	+ * TDX guests in addition to KVM_MAX_VCPUS. Userspace needs to
	3382	+ * query TDX guest's maximum vCPUs by checking KVM_CAP_MAX_VCPU
	3383	+ * extension on per-VM basis.
	3384	+ *
	3385	+ * TDX module reports such limit via the MAX_VCPU_PER_TD global
	3386	+ * metadata. Different modules may report different values.
	3387	+ * Some old module may also not support this metadata (in which
	3388	+ * case this limit is U16_MAX).
	3389	+ *
	3390	+ * In practice, the reported value reflects the maximum logical
	3391	+ * CPUs that ALL the platforms that the module supports can
	3392	+ * possibly have.
	3393	+ *
	3394	+ * Simply forwarding the MAX_VCPU_PER_TD to userspace could
	3395	+ * result in an unpredictable ABI. KVM instead always advertise
	3396	+ * the number of logical CPUs the platform has as the maximum
	3397	+ * vCPUs for TDX guests.
	3398	+ *
	3399	+ * Make sure MAX_VCPU_PER_TD reported by TDX module is not
	3400	+ * smaller than the number of logical CPUs, otherwise KVM will
	3401	+ * report an unsupported value to userspace.
	3402	+ *
	3403	+ * Note, a platform with TDX enabled in the BIOS cannot support
	3404	+ * physical CPU hotplug, and TDX requires the BIOS has marked
	3405	+ * all logical CPUs in MADT table as enabled. Just use
	3406	+ * num_present_cpus() for the number of logical CPUs.
	3407	+ */
	3408	+ td_conf = &tdx_sysinfo->td_conf;
	3409	+ if (td_conf->max_vcpus_per_td < num_present_cpus()) {
	3410	+ pr_err("Disable TDX: MAX_VCPU_PER_TD (%u) smaller than number of logical CPUs (%u).\n",
	3411	+ td_conf->max_vcpus_per_td, num_present_cpus());
	3412	+ r = -EINVAL;
	3413	+ goto get_sysinfo_err;
	3414	+ }
	3415	+
	3416	+ if (misc_cg_set_capacity(MISC_CG_RES_TDX, tdx_get_nr_guest_keyids())) {
	3417	+ r = -EINVAL;
	3418	+ goto get_sysinfo_err;
	3419	+ }
	3420	+
	3421	+ /*
	3422	+ * Leave hardware virtualization enabled after TDX is enabled
	3423	+ * successfully. TDX CPU hotplug depends on this.
	3424	+ */
	3425	+ return 0;
	3426	+
	3427	+get_sysinfo_err:
	3428	+ __tdx_cleanup();
	3429	+tdx_bringup_err:
	3430	+ kvm_disable_virtualization();
	3431	+ return r;
	3432	+}
	3433	+
	3434	+void tdx_cleanup(void)
	3435	+{
	3436	+ if (enable_tdx) {
	3437	+ misc_cg_set_capacity(MISC_CG_RES_TDX, 0);
	3438	+ __tdx_cleanup();
	3439	+ kvm_disable_virtualization();
	3440	+ }
	3441	+}
	3442	+
	3443	+int __init tdx_bringup(void)
	3444	+{
	3445	+ int r, i;
	3446	+
	3447	+ /* tdx_disable_virtualization_cpu() uses associated_tdvcpus. */
	3448	+ for_each_possible_cpu(i)
	3449	+ INIT_LIST_HEAD(&per_cpu(associated_tdvcpus, i));
	3450	+
	3451	+ if (!enable_tdx)
	3452	+ return 0;
	3453	+
	3454	+ if (!enable_ept) {
	3455	+ pr_err("EPT is required for TDX\n");
	3456	+ goto success_disable_tdx;
	3457	+ }
	3458	+
	3459	+ if (!tdp_mmu_enabled \|\| !enable_mmio_caching \|\| !enable_ept_ad_bits) {
	3460	+ pr_err("TDP MMU and MMIO caching and EPT A/D bit is required for TDX\n");
	3461	+ goto success_disable_tdx;
	3462	+ }
	3463	+
	3464	+ if (!enable_apicv) {
	3465	+ pr_err("APICv is required for TDX\n");
	3466	+ goto success_disable_tdx;
	3467	+ }
	3468	+
	3469	+ if (!cpu_feature_enabled(X86_FEATURE_OSXSAVE)) {
	3470	+ pr_err("tdx: OSXSAVE is required for TDX\n");
	3471	+ goto success_disable_tdx;
	3472	+ }
	3473	+
	3474	+ if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) {
	3475	+ pr_err("tdx: MOVDIR64B is required for TDX\n");
	3476	+ goto success_disable_tdx;
	3477	+ }
	3478	+
	3479	+ if (!cpu_feature_enabled(X86_FEATURE_SELFSNOOP)) {
	3480	+ pr_err("Self-snoop is required for TDX\n");
	3481	+ goto success_disable_tdx;
	3482	+ }
	3483	+
	3484	+ if (!cpu_feature_enabled(X86_FEATURE_TDX_HOST_PLATFORM)) {
	3485	+ pr_err("tdx: no TDX private KeyIDs available\n");
	3486	+ goto success_disable_tdx;
	3487	+ }
	3488	+
	3489	+ if (!enable_virt_at_load) {
	3490	+ pr_err("tdx: tdx requires kvm.enable_virt_at_load=1\n");
	3491	+ goto success_disable_tdx;
	3492	+ }
	3493	+
	3494	+ /*
	3495	+ * Ideally KVM should probe whether TDX module has been loaded
	3496	+ * first and then try to bring it up. But TDX needs to use SEAMCALL
	3497	+ * to probe whether the module is loaded (there is no CPUID or MSR
	3498	+ * for that), and making SEAMCALL requires enabling virtualization
	3499	+ * first, just like the rest steps of bringing up TDX module.
	3500	+ *
	3501	+ * So, for simplicity do everything in __tdx_bringup(); the first
	3502	+ * SEAMCALL will return -ENODEV when the module is not loaded. The
	3503	+ * only complication is having to make sure that initialization
	3504	+ * SEAMCALLs don't return TDX_SEAMCALL_VMFAILINVALID in other
	3505	+ * cases.
	3506	+ */
	3507	+ r = __tdx_bringup();
	3508	+ if (r) {
	3509	+ /*
	3510	+ * Disable TDX only but don't fail to load module if
	3511	+ * the TDX module could not be loaded. No need to print
	3512	+ * message saying "module is not loaded" because it was
	3513	+ * printed when the first SEAMCALL failed.
	3514	+ */
	3515	+ if (r == -ENODEV)
	3516	+ goto success_disable_tdx;
	3517	+
	3518	+ enable_tdx = 0;
	3519	+ }
	3520	+
	3521	+ return r;
	3522	+
	3523	+success_disable_tdx:
	3524	+ enable_tdx = 0;
	3525	+ return 0;
	3526	+}

arch/x86/kvm/vmx/tdx.h+204−0

@@ -0,0 +1,204 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+#ifndef __KVM_X86_VMX_TDX_H
	3	+#define __KVM_X86_VMX_TDX_H
	4	+
	5	+#include "tdx_arch.h"
	6	+#include "tdx_errno.h"
	7	+
	8	+#ifdef CONFIG_KVM_INTEL_TDX
	9	+#include "common.h"
	10	+
	11	+int tdx_bringup(void);
	12	+void tdx_cleanup(void);
	13	+
	14	+extern bool enable_tdx;
	15	+
	16	+/* TDX module hardware states. These follow the TDX module OP_STATEs. */
	17	+enum kvm_tdx_state {
	18	+ TD_STATE_UNINITIALIZED = 0,
	19	+ TD_STATE_INITIALIZED,
	20	+ TD_STATE_RUNNABLE,
	21	+};
	22	+
	23	+struct kvm_tdx {
	24	+ struct kvm kvm;
	25	+
	26	+ struct misc_cg *misc_cg;
	27	+ int hkid;
	28	+ enum kvm_tdx_state state;
	29	+
	30	+ u64 attributes;
	31	+ u64 xfam;
	32	+
	33	+ u64 tsc_offset;
	34	+ u64 tsc_multiplier;
	35	+
	36	+ struct tdx_td td;
	37	+
	38	+ /* For KVM_TDX_INIT_MEM_REGION. */
	39	+ atomic64_t nr_premapped;
	40	+
	41	+ /*
	42	+ * Prevent vCPUs from TD entry to ensure SEPT zap related SEAMCALLs do
	43	+ * not contend with tdh_vp_enter() and TDCALLs.
	44	+ * Set/unset is protected with kvm->mmu_lock.
	45	+ */
	46	+ bool wait_for_sept_zap;
	47	+};
	48	+
	49	+/* TDX module vCPU states */
	50	+enum vcpu_tdx_state {
	51	+ VCPU_TD_STATE_UNINITIALIZED = 0,
	52	+ VCPU_TD_STATE_INITIALIZED,
	53	+};
	54	+
	55	+struct vcpu_tdx {
	56	+ struct kvm_vcpu vcpu;
	57	+ struct vcpu_vt vt;
	58	+ u64 ext_exit_qualification;
	59	+ gpa_t exit_gpa;
	60	+ struct tdx_module_args vp_enter_args;
	61	+
	62	+ struct tdx_vp vp;
	63	+
	64	+ struct list_head cpu_list;
	65	+
	66	+ u64 vp_enter_ret;
	67	+
	68	+ enum vcpu_tdx_state state;
	69	+ bool guest_entered;
	70	+
	71	+ u64 map_gpa_next;
	72	+ u64 map_gpa_end;
	73	+};
	74	+
	75	+void tdh_vp_rd_failed(struct vcpu_tdx tdx, char uclass, u32 field, u64 err);
	76	+void tdh_vp_wr_failed(struct vcpu_tdx tdx, char uclass, char *op, u32 field,
	77	+ u64 val, u64 err);
	78	+
	79	+static __always_inline u64 td_tdcs_exec_read64(struct kvm_tdx *kvm_tdx, u32 field)
	80	+{
	81	+ u64 err, data;
	82	+
	83	+ err = tdh_mng_rd(&kvm_tdx->td, TDCS_EXEC(field), &data);
	84	+ if (unlikely(err)) {
	85	+ pr_err("TDH_MNG_RD[EXEC.0x%x] failed: 0x%llx\n", field, err);
	86	+ return 0;
	87	+ }
	88	+ return data;
	89	+}
	90	+
	91	+static __always_inline void tdvps_vmcs_check(u32 field, u8 bits)
	92	+{
	93	+#define VMCS_ENC_ACCESS_TYPE_MASK 0x1UL
	94	+#define VMCS_ENC_ACCESS_TYPE_FULL 0x0UL
	95	+#define VMCS_ENC_ACCESS_TYPE_HIGH 0x1UL
	96	+#define VMCS_ENC_ACCESS_TYPE(field) ((field) & VMCS_ENC_ACCESS_TYPE_MASK)
	97	+
	98	+ /* TDX is 64bit only. HIGH field isn't supported. */
	99	+ BUILD_BUG_ON_MSG(__builtin_constant_p(field) &&
	100	+ VMCS_ENC_ACCESS_TYPE(field) == VMCS_ENC_ACCESS_TYPE_HIGH,
	101	+ "Read/Write to TD VMCS *_HIGH fields not supported");
	102	+
	103	+ BUILD_BUG_ON(bits != 16 && bits != 32 && bits != 64);
	104	+
	105	+#define VMCS_ENC_WIDTH_MASK GENMASK(14, 13)
	106	+#define VMCS_ENC_WIDTH_16BIT (0UL << 13)
	107	+#define VMCS_ENC_WIDTH_64BIT (1UL << 13)
	108	+#define VMCS_ENC_WIDTH_32BIT (2UL << 13)
	109	+#define VMCS_ENC_WIDTH_NATURAL (3UL << 13)
	110	+#define VMCS_ENC_WIDTH(field) ((field) & VMCS_ENC_WIDTH_MASK)
	111	+
	112	+ /* TDX is 64bit only. i.e. natural width = 64bit. */
	113	+ BUILD_BUG_ON_MSG(bits != 64 && __builtin_constant_p(field) &&
	114	+ (VMCS_ENC_WIDTH(field) == VMCS_ENC_WIDTH_64BIT \|\|
	115	+ VMCS_ENC_WIDTH(field) == VMCS_ENC_WIDTH_NATURAL),
	116	+ "Invalid TD VMCS access for 64-bit field");
	117	+ BUILD_BUG_ON_MSG(bits != 32 && __builtin_constant_p(field) &&
	118	+ VMCS_ENC_WIDTH(field) == VMCS_ENC_WIDTH_32BIT,
	119	+ "Invalid TD VMCS access for 32-bit field");
	120	+ BUILD_BUG_ON_MSG(bits != 16 && __builtin_constant_p(field) &&
	121	+ VMCS_ENC_WIDTH(field) == VMCS_ENC_WIDTH_16BIT,
	122	+ "Invalid TD VMCS access for 16-bit field");
	123	+}
	124	+
	125	+static __always_inline void tdvps_management_check(u64 field, u8 bits) {}
	126	+static __always_inline void tdvps_state_non_arch_check(u64 field, u8 bits) {}
	127	+
	128	+#define TDX_BUILD_TDVPS_ACCESSORS(bits, uclass, lclass) \
	129	+static __always_inline u##bits td_##lclass##_read##bits(struct vcpu_tdx *tdx, \
	130	+ u32 field) \
	131	+{ \
	132	+ u64 err, data; \
	133	+ \
	134	+ tdvps_##lclass##_check(field, bits); \
	135	+ err = tdh_vp_rd(&tdx->vp, TDVPS_##uclass(field), &data); \
	136	+ if (unlikely(err)) { \
	137	+ tdh_vp_rd_failed(tdx, #uclass, field, err); \
	138	+ return 0; \
	139	+ } \
	140	+ return (u##bits)data; \
	141	+} \
	142	+static __always_inline void td_##lclass##_write##bits(struct vcpu_tdx *tdx, \
	143	+ u32 field, u##bits val) \
	144	+{ \
	145	+ u64 err; \
	146	+ \
	147	+ tdvps_##lclass##_check(field, bits); \
	148	+ err = tdh_vp_wr(&tdx->vp, TDVPS_##uclass(field), val, \
	149	+ GENMASK_ULL(bits - 1, 0)); \
	150	+ if (unlikely(err)) \
	151	+ tdh_vp_wr_failed(tdx, #uclass, " = ", field, (u64)val, err); \
	152	+} \
	153	+static __always_inline void td_##lclass##_setbit##bits(struct vcpu_tdx *tdx, \
	154	+ u32 field, u64 bit) \
	155	+{ \
	156	+ u64 err; \
	157	+ \
	158	+ tdvps_##lclass##_check(field, bits); \
	159	+ err = tdh_vp_wr(&tdx->vp, TDVPS_##uclass(field), bit, bit); \
	160	+ if (unlikely(err)) \
	161	+ tdh_vp_wr_failed(tdx, #uclass, " \|= ", field, bit, err); \
	162	+} \
	163	+static __always_inline void td_##lclass##_clearbit##bits(struct vcpu_tdx *tdx, \
	164	+ u32 field, u64 bit) \
	165	+{ \
	166	+ u64 err; \
	167	+ \
	168	+ tdvps_##lclass##_check(field, bits); \
	169	+ err = tdh_vp_wr(&tdx->vp, TDVPS_##uclass(field), 0, bit); \
	170	+ if (unlikely(err)) \
	171	+ tdh_vp_wr_failed(tdx, #uclass, " &= ~", field, bit, err);\
	172	+}
	173	+
	174	+
	175	+bool tdx_interrupt_allowed(struct kvm_vcpu *vcpu);
	176	+int tdx_complete_emulated_msr(struct kvm_vcpu *vcpu, int err);
	177	+
	178	+TDX_BUILD_TDVPS_ACCESSORS(16, VMCS, vmcs);
	179	+TDX_BUILD_TDVPS_ACCESSORS(32, VMCS, vmcs);
	180	+TDX_BUILD_TDVPS_ACCESSORS(64, VMCS, vmcs);
	181	+
	182	+TDX_BUILD_TDVPS_ACCESSORS(8, MANAGEMENT, management);
	183	+TDX_BUILD_TDVPS_ACCESSORS(64, STATE_NON_ARCH, state_non_arch);
	184	+
	185	+#else
	186	+static inline int tdx_bringup(void) { return 0; }
	187	+static inline void tdx_cleanup(void) {}
	188	+
	189	+#define enable_tdx 0
	190	+
	191	+struct kvm_tdx {
	192	+ struct kvm kvm;
	193	+};
	194	+
	195	+struct vcpu_tdx {
	196	+ struct kvm_vcpu vcpu;
	197	+};
	198	+
	199	+static inline bool tdx_interrupt_allowed(struct kvm_vcpu *vcpu) { return false; }
	200	+static inline int tdx_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) { return 0; }
	201	+
	202	+#endif
	203	+
	204	+#endif

arch/x86/kvm/vmx/tdx_arch.h+167−0

@@ -0,0 +1,167 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+/* architectural constants/data definitions for TDX SEAMCALLs */
	3	+
	4	+#ifndef __KVM_X86_TDX_ARCH_H
	5	+#define __KVM_X86_TDX_ARCH_H
	6	+
	7	+#include <linux/types.h>
	8	+
	9	+/* TDX control structure (TDR/TDCS/TDVPS) field access codes */
	10	+#define TDX_NON_ARCH BIT_ULL(63)
	11	+#define TDX_CLASS_SHIFT 56
	12	+#define TDX_FIELD_MASK GENMASK_ULL(31, 0)
	13	+
	14	+#define __BUILD_TDX_FIELD(non_arch, class, field) \
	15	+ (((non_arch) ? TDX_NON_ARCH : 0) \| \
	16	+ ((u64)(class) << TDX_CLASS_SHIFT) \| \
	17	+ ((u64)(field) & TDX_FIELD_MASK))
	18	+
	19	+#define BUILD_TDX_FIELD(class, field) \
	20	+ __BUILD_TDX_FIELD(false, (class), (field))
	21	+
	22	+#define BUILD_TDX_FIELD_NON_ARCH(class, field) \
	23	+ __BUILD_TDX_FIELD(true, (class), (field))
	24	+
	25	+
	26	+/* Class code for TD */
	27	+#define TD_CLASS_EXECUTION_CONTROLS 17ULL
	28	+
	29	+/* Class code for TDVPS */
	30	+#define TDVPS_CLASS_VMCS 0ULL
	31	+#define TDVPS_CLASS_GUEST_GPR 16ULL
	32	+#define TDVPS_CLASS_OTHER_GUEST 17ULL
	33	+#define TDVPS_CLASS_MANAGEMENT 32ULL
	34	+
	35	+enum tdx_tdcs_execution_control {
	36	+ TD_TDCS_EXEC_TSC_OFFSET = 10,
	37	+ TD_TDCS_EXEC_TSC_MULTIPLIER = 11,
	38	+};
	39	+
	40	+enum tdx_vcpu_guest_other_state {
	41	+ TD_VCPU_STATE_DETAILS_NON_ARCH = 0x100,
	42	+};
	43	+
	44	+#define TDX_VCPU_STATE_DETAILS_INTR_PENDING BIT_ULL(0)
	45	+
	46	+static inline bool tdx_vcpu_state_details_intr_pending(u64 vcpu_state_details)
	47	+{
	48	+ return !!(vcpu_state_details & TDX_VCPU_STATE_DETAILS_INTR_PENDING);
	49	+}
	50	+
	51	+/* @field is any of enum tdx_tdcs_execution_control */
	52	+#define TDCS_EXEC(field) BUILD_TDX_FIELD(TD_CLASS_EXECUTION_CONTROLS, (field))
	53	+
	54	+/* @field is the VMCS field encoding */
	55	+#define TDVPS_VMCS(field) BUILD_TDX_FIELD(TDVPS_CLASS_VMCS, (field))
	56	+
	57	+/* @field is any of enum tdx_guest_other_state */
	58	+#define TDVPS_STATE(field) BUILD_TDX_FIELD(TDVPS_CLASS_OTHER_GUEST, (field))
	59	+#define TDVPS_STATE_NON_ARCH(field) BUILD_TDX_FIELD_NON_ARCH(TDVPS_CLASS_OTHER_GUEST, (field))
	60	+
	61	+/* Management class fields */
	62	+enum tdx_vcpu_guest_management {
	63	+ TD_VCPU_PEND_NMI = 11,
	64	+};
	65	+
	66	+/* @field is any of enum tdx_vcpu_guest_management */
	67	+#define TDVPS_MANAGEMENT(field) BUILD_TDX_FIELD(TDVPS_CLASS_MANAGEMENT, (field))
	68	+
	69	+#define TDX_EXTENDMR_CHUNKSIZE 256
	70	+
	71	+struct tdx_cpuid_value {
	72	+ u32 eax;
	73	+ u32 ebx;
	74	+ u32 ecx;
	75	+ u32 edx;
	76	+} __packed;
	77	+
	78	+#define TDX_TD_ATTR_DEBUG BIT_ULL(0)
	79	+#define TDX_TD_ATTR_SEPT_VE_DISABLE BIT_ULL(28)
	80	+#define TDX_TD_ATTR_PKS BIT_ULL(30)
	81	+#define TDX_TD_ATTR_KL BIT_ULL(31)
	82	+#define TDX_TD_ATTR_PERFMON BIT_ULL(63)
	83	+
	84	+#define TDX_EXT_EXIT_QUAL_TYPE_MASK GENMASK(3, 0)
	85	+#define TDX_EXT_EXIT_QUAL_TYPE_PENDING_EPT_VIOLATION 6
	86	+/*
	87	+ * TD_PARAMS is provided as an input to TDH_MNG_INIT, the size of which is 1024B.
	88	+ */
	89	+struct td_params {
	90	+ u64 attributes;
	91	+ u64 xfam;
	92	+ u16 max_vcpus;
	93	+ u8 reserved0[6];
	94	+
	95	+ u64 eptp_controls;
	96	+ u64 config_flags;
	97	+ u16 tsc_frequency;
	98	+ u8 reserved1[38];
	99	+
	100	+ u64 mrconfigid[6];
	101	+ u64 mrowner[6];
	102	+ u64 mrownerconfig[6];
	103	+ u64 reserved2[4];
	104	+
	105	+ union {
	106	+ DECLARE_FLEX_ARRAY(struct tdx_cpuid_value, cpuid_values);
	107	+ u8 reserved3[768];
	108	+ };
	109	+} __packed __aligned(1024);
	110	+
	111	+/*
	112	+ * Guest uses MAX_PA for GPAW when set.
	113	+ * 0: GPA.SHARED bit is GPA[47]
	114	+ * 1: GPA.SHARED bit is GPA[51]
	115	+ */
	116	+#define TDX_CONFIG_FLAGS_MAX_GPAW BIT_ULL(0)
	117	+
	118	+/*
	119	+ * TDH.VP.ENTER, TDG.VP.VMCALL preserves RBP
	120	+ * 0: RBP can be used for TDG.VP.VMCALL input. RBP is clobbered.
	121	+ * 1: RBP can't be used for TDG.VP.VMCALL input. RBP is preserved.
	122	+ */
	123	+#define TDX_CONFIG_FLAGS_NO_RBP_MOD BIT_ULL(2)
	124	+
	125	+
	126	+/*
	127	+ * TDX requires the frequency to be defined in units of 25MHz, which is the
	128	+ * frequency of the core crystal clock on TDX-capable platforms, i.e. the TDX
	129	+ * module can only program frequencies that are multiples of 25MHz. The
	130	+ * frequency must be between 100mhz and 10ghz (inclusive).
	131	+ */
	132	+#define TDX_TSC_KHZ_TO_25MHZ(tsc_in_khz) ((tsc_in_khz) / (25 * 1000))
	133	+#define TDX_TSC_25MHZ_TO_KHZ(tsc_in_25mhz) ((tsc_in_25mhz) * (25 * 1000))
	134	+#define TDX_MIN_TSC_FREQUENCY_KHZ (100 * 1000)
	135	+#define TDX_MAX_TSC_FREQUENCY_KHZ (10 * 1000 * 1000)
	136	+
	137	+/* Additional Secure EPT entry information */
	138	+#define TDX_SEPT_LEVEL_MASK GENMASK_ULL(2, 0)
	139	+#define TDX_SEPT_STATE_MASK GENMASK_ULL(15, 8)
	140	+#define TDX_SEPT_STATE_SHIFT 8
	141	+
	142	+enum tdx_sept_entry_state {
	143	+ TDX_SEPT_FREE = 0,
	144	+ TDX_SEPT_BLOCKED = 1,
	145	+ TDX_SEPT_PENDING = 2,
	146	+ TDX_SEPT_PENDING_BLOCKED = 3,
	147	+ TDX_SEPT_PRESENT = 4,
	148	+};
	149	+
	150	+static inline u8 tdx_get_sept_level(u64 sept_entry_info)
	151	+{
	152	+ return sept_entry_info & TDX_SEPT_LEVEL_MASK;
	153	+}
	154	+
	155	+static inline u8 tdx_get_sept_state(u64 sept_entry_info)
	156	+{
	157	+ return (sept_entry_info & TDX_SEPT_STATE_MASK) >> TDX_SEPT_STATE_SHIFT;
	158	+}
	159	+
	160	+#define MD_FIELD_ID_FEATURES0_TOPOLOGY_ENUM BIT_ULL(20)
	161	+
	162	+/*
	163	+ * TD scope metadata field ID.
	164	+ */
	165	+#define TD_MD_FIELD_ID_CPUID_VALUES 0x9410000300000000ULL
	166	+
	167	+#endif /* __KVM_X86_TDX_ARCH_H */

arch/x86/kvm/vmx/tdx_errno.h+40−0

@@ -0,0 +1,40 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+/* architectural status code for SEAMCALL */
	3	+
	4	+#ifndef __KVM_X86_TDX_ERRNO_H
	5	+#define __KVM_X86_TDX_ERRNO_H
	6	+
	7	+#define TDX_SEAMCALL_STATUS_MASK 0xFFFFFFFF00000000ULL
	8	+
	9	+/*
	10	+ * TDX SEAMCALL Status Codes (returned in RAX)
	11	+ */
	12	+#define TDX_NON_RECOVERABLE_VCPU 0x4000000100000000ULL
	13	+#define TDX_NON_RECOVERABLE_TD 0x4000000200000000ULL
	14	+#define TDX_NON_RECOVERABLE_TD_NON_ACCESSIBLE 0x6000000500000000ULL
	15	+#define TDX_NON_RECOVERABLE_TD_WRONG_APIC_MODE 0x6000000700000000ULL
	16	+#define TDX_INTERRUPTED_RESUMABLE 0x8000000300000000ULL
	17	+#define TDX_OPERAND_INVALID 0xC000010000000000ULL
	18	+#define TDX_OPERAND_BUSY 0x8000020000000000ULL
	19	+#define TDX_PREVIOUS_TLB_EPOCH_BUSY 0x8000020100000000ULL
	20	+#define TDX_PAGE_METADATA_INCORRECT 0xC000030000000000ULL
	21	+#define TDX_VCPU_NOT_ASSOCIATED 0x8000070200000000ULL
	22	+#define TDX_KEY_GENERATION_FAILED 0x8000080000000000ULL
	23	+#define TDX_KEY_STATE_INCORRECT 0xC000081100000000ULL
	24	+#define TDX_KEY_CONFIGURED 0x0000081500000000ULL
	25	+#define TDX_NO_HKID_READY_TO_WBCACHE 0x0000082100000000ULL
	26	+#define TDX_FLUSHVP_NOT_DONE 0x8000082400000000ULL
	27	+#define TDX_EPT_WALK_FAILED 0xC0000B0000000000ULL
	28	+#define TDX_EPT_ENTRY_STATE_INCORRECT 0xC0000B0D00000000ULL
	29	+#define TDX_METADATA_FIELD_NOT_READABLE 0xC0000C0200000000ULL
	30	+
	31	+/*
	32	+ * TDX module operand ID, appears in 31:0 part of error code as
	33	+ * detail information
	34	+ */
	35	+#define TDX_OPERAND_ID_RCX 0x01
	36	+#define TDX_OPERAND_ID_TDR 0x80
	37	+#define TDX_OPERAND_ID_SEPT 0x92
	38	+#define TDX_OPERAND_ID_TD_EPOCH 0xa9
	39	+
	40	+#endif /* __KVM_X86_TDX_ERRNO_H */

arch/x86/kvm/vmx/vmx.c+111−180

@@ -53,6 +53,7 @@ · 52 unmodified lines
53	53	#include <trace/events/ipi.h>
54	54
55	55	#include "capabilities.h"
	56	+#include "common.h"
56	57	#include "cpuid.h"
57	58	#include "hyperv.h"
58	59	#include "kvm_onhyperv.h"
@@ -1281,6 +1282,7 @@ void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel, · 1222 unmodified lines
1281	1282	void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
1282	1283	{
1283	1284	struct vcpu_vmx *vmx = to_vmx(vcpu);
	1285	+ struct vcpu_vt *vt = to_vt(vcpu);
1284	1286	struct vmcs_host_state *host_state;
1285	1287	#ifdef CONFIG_X86_64
1286	1288	int cpu = raw_smp_processor_id();
@@ -1309,7 +1311,7 @@ void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) · 22 unmodified lines
1309	1311	if (vmx->nested.need_vmcs12_to_shadow_sync)
1310	1312	nested_sync_vmcs12_to_shadow(vcpu);
1311	1313
1312		− if (vmx->guest_state_loaded)
	1314	+ if (vt->guest_state_loaded)
1313	1315	return;
1314	1316
1315	1317	host_state = &vmx->loaded_vmcs->host_state;
@@ -1330,12 +1332,12 @@ void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) · 14 unmodified lines
1330	1332	fs_sel = current->thread.fsindex;
1331	1333	gs_sel = current->thread.gsindex;
1332	1334	fs_base = current->thread.fsbase;
1333		− vmx->msr_host_kernel_gs_base = current->thread.gsbase;
	1335	+ vt->msr_host_kernel_gs_base = current->thread.gsbase;
1334	1336	} else {
1335	1337	savesegment(fs, fs_sel);
1336	1338	savesegment(gs, gs_sel);
1337	1339	fs_base = read_msr(MSR_FS_BASE);
1338		− vmx->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE);
	1340	+ vt->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE);
1339	1341	}
1340	1342
1341	1343	wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
@@ -1347,14 +1349,14 @@ void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) · 5 unmodified lines
1347	1349	#endif
1348	1350
1349	1351	vmx_set_host_fs_gs(host_state, fs_sel, gs_sel, fs_base, gs_base);
1350		− vmx->guest_state_loaded = true;
	1352	+ vt->guest_state_loaded = true;
1351	1353	}
1352	1354
1353	1355	static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx)
1354	1356	{
1355	1357	struct vmcs_host_state *host_state;
1356	1358
1357		− if (!vmx->guest_state_loaded)
	1359	+ if (!vmx->vt.guest_state_loaded)
1358	1360	return;
1359	1361
1360	1362	host_state = &vmx->loaded_vmcs->host_state;
@@ -1382,10 +1384,10 @@ static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx) · 21 unmodified lines
1382	1384	#endif
1383	1385	invalidate_tss_limit();
1384	1386	#ifdef CONFIG_X86_64
1385		− wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
	1387	+ wrmsrl(MSR_KERNEL_GS_BASE, vmx->vt.msr_host_kernel_gs_base);
1386	1388	#endif
1387	1389	load_fixmap_gdt(raw_smp_processor_id());
1388		− vmx->guest_state_loaded = false;
	1390	+ vmx->vt.guest_state_loaded = false;
1389	1391	vmx->guest_uret_msrs_loaded = false;
1390	1392	}
1391	1393
@@ -1393,7 +1395,7 @@ static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx) · 1 unmodified line
1393	1395	static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx)
1394	1396	{
1395	1397	preempt_disable();
1396		− if (vmx->guest_state_loaded)
	1398	+ if (vmx->vt.guest_state_loaded)
1397	1399	rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
1398	1400	preempt_enable();
1399	1401	return vmx->msr_guest_kernel_gs_base;
@@ -1402,7 +1404,7 @@ static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx) · 2 unmodified lines
1402	1404	static void vmx_write_guest_kernel_gs_base(struct vcpu_vmx *vmx, u64 data)
1403	1405	{
1404	1406	preempt_disable();
1405		− if (vmx->guest_state_loaded)
	1407	+ if (vmx->vt.guest_state_loaded)
1406	1408	wrmsrl(MSR_KERNEL_GS_BASE, data);
1407	1409	preempt_enable();
1408	1410	vmx->msr_guest_kernel_gs_base = data;
@@ -1579,7 +1581,7 @@ void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) · 170 unmodified lines
1579	1581	vmcs_writel(GUEST_RFLAGS, rflags);
1580	1582
1581	1583	if ((old_rflags ^ vmx->rflags) & X86_EFLAGS_VM)
1582		− vmx->emulation_required = vmx_emulation_required(vcpu);
	1584	+ vmx->vt.emulation_required = vmx_emulation_required(vcpu);
1583	1585	}
1584	1586
1585	1587	bool vmx_get_if_flag(struct kvm_vcpu *vcpu)
@@ -1699,7 +1701,7 @@ int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, · 113 unmodified lines
1699	1701	* so that guest userspace can't DoS the guest simply by triggering
1700	1702	* emulation (enclaves are CPL3 only).
1701	1703	*/
1702		− if (to_vmx(vcpu)->exit_reason.enclave_mode) {
	1704	+ if (vmx_get_exit_reason(vcpu).enclave_mode) {
1703	1705	kvm_queue_exception(vcpu, UD_VECTOR);
1704	1706	return X86EMUL_PROPAGATE_FAULT;
1705	1707	}
@@ -1714,7 +1716,7 @@ int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, · 8 unmodified lines
1714	1716
1715	1717	static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
1716	1718	{
1717		− union vmx_exit_reason exit_reason = to_vmx(vcpu)->exit_reason;
	1719	+ union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu);
1718	1720	unsigned long rip, orig_rip;
1719	1721	u32 instr_len;
1720	1722
@@ -1861,7 +1863,7 @@ void vmx_inject_exception(struct kvm_vcpu *vcpu) · 140 unmodified lines
1861	1863	return;
1862	1864	}
1863	1865
1864		− WARN_ON_ONCE(vmx->emulation_required);
	1866	+ WARN_ON_ONCE(vmx->vt.emulation_required);
1865	1867
1866	1868	if (kvm_exception_is_soft(ex->vector)) {
1867	1869	vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
@@ -3404,7 +3406,7 @@ void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) · 1536 unmodified lines
3404	3406	}
3405	3407
3406	3408	/* depends on vcpu->arch.cr0 to be set to a new value */
3407		− vmx->emulation_required = vmx_emulation_required(vcpu);
	3409	+ vmx->vt.emulation_required = vmx_emulation_required(vcpu);
3408	3410	}
3409	3411
3410	3412	static int vmx_get_max_ept_level(void)
@@ -3667,7 +3669,7 @@ void vmx_set_segment(struct kvm_vcpu vcpu, struct kvm_segment var, int seg) · 256 unmodified lines
3667	3669	{
3668	3670	__vmx_set_segment(vcpu, var, seg);
3669	3671
3670		− to_vmx(vcpu)->emulation_required = vmx_emulation_required(vcpu);
	3672	+ to_vmx(vcpu)->vt.emulation_required = vmx_emulation_required(vcpu);
3671	3673	}
3672	3674
3673	3675	void vmx_get_cs_db_l_bits(struct kvm_vcpu vcpu, int db, int *l)
@@ -4195,50 +4197,6 @@ void vmx_msr_filter_changed(struct kvm_vcpu *vcpu) · 521 unmodified lines
4195	4197	pt_update_intercept_for_msr(vcpu);
4196	4198	}
4197	4199
4198		−static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu,
4199		− int pi_vec)
4200		−{
4201		−#ifdef CONFIG_SMP
4202		− if (vcpu->mode == IN_GUEST_MODE) {
4203		− /*
4204		− * The vector of the virtual has already been set in the PIR.
4205		− * Send a notification event to deliver the virtual interrupt
4206		− * unless the vCPU is the currently running vCPU, i.e. the
4207		− * event is being sent from a fastpath VM-Exit handler, in
4208		− * which case the PIR will be synced to the vIRR before
4209		− * re-entering the guest.
4210		− *
4211		− * When the target is not the running vCPU, the following
4212		− * possibilities emerge:
4213		− *
4214		− * Case 1: vCPU stays in non-root mode. Sending a notification
4215		− * event posts the interrupt to the vCPU.
4216		− *
4217		− * Case 2: vCPU exits to root mode and is still runnable. The
4218		− * PIR will be synced to the vIRR before re-entering the guest.
4219		− * Sending a notification event is ok as the host IRQ handler
4220		− * will ignore the spurious event.
4221		− *
4222		− * Case 3: vCPU exits to root mode and is blocked. vcpu_block()
4223		− * has already synced PIR to vIRR and never blocks the vCPU if
4224		− * the vIRR is not empty. Therefore, a blocked vCPU here does
4225		− * not wait for any requested interrupts in PIR, and sending a
4226		− * notification event also results in a benign, spurious event.
4227		− */
4228		−
4229		− if (vcpu != kvm_get_running_vcpu())
4230		− __apic_send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec);
4231		− return;
4232		− }
4233		−#endif
4234		− /*
4235		− * The vCPU isn't in the guest; wake the vCPU in case it is blocking,
4236		− * otherwise do nothing as KVM will grab the highest priority pending
4237		− * IRQ via ->sync_pir_to_irr() in vcpu_enter_guest().
4238		− */
4239		− kvm_vcpu_wake_up(vcpu);
4240		−}
4241		−
4242	4200	static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu,
4243	4201	int vector)
4244	4202	{
@@ -4287,7 +4245,7 @@ static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu, · 42 unmodified lines
4287	4245	*/
4288	4246	static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector)
4289	4247	{
4290		− struct vcpu_vmx *vmx = to_vmx(vcpu);
	4248	+ struct vcpu_vt *vt = to_vt(vcpu);
4291	4249	int r;
4292	4250
4293	4251	r = vmx_deliver_nested_posted_interrupt(vcpu, vector);
@@ -4298,20 +4256,7 @@ static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector) · 4 unmodified lines
4298	4256	if (!vcpu->arch.apic->apicv_active)
4299	4257	return -1;
4300	4258
4301		− if (pi_test_and_set_pir(vector, &vmx->pi_desc))
4302		− return 0;
4303		−
4304		− /* If a previous notification has sent the IPI, nothing to do. */
4305		− if (pi_test_and_set_on(&vmx->pi_desc))
4306		− return 0;
4307		−
4308		− /*
4309		− * The implied barrier in pi_test_and_set_on() pairs with the smp_mb_*()
4310		− * after setting vcpu->mode in vcpu_enter_guest(), thus the vCPU is
4311		− * guaranteed to see PID.ON=1 and sync the PIR to IRR if triggering a
4312		− * posted interrupt "fails" because vcpu->mode != IN_GUEST_MODE.
4313		− */
4314		− kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_VECTOR);
	4259	+ __vmx_deliver_posted_interrupt(vcpu, &vt->pi_desc, vector);
4315	4260	return 0;
4316	4261	}
4317	4262
@@ -4778,7 +4723,7 @@ static void init_vmcs(struct vcpu_vmx *vmx) · 460 unmodified lines
4778	4723	vmcs_write16(GUEST_INTR_STATUS, 0);
4779	4724
4780	4725	vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
4781		− vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc)));
	4726	+ vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->vt.pi_desc)));
4782	4727	}
4783	4728
4784	4729	if (vmx_can_use_ipiv(&vmx->vcpu)) {
@@ -4891,8 +4836,8 @@ static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu) · 106 unmodified lines
4891	4836	* Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR
4892	4837	* or POSTED_INTR_WAKEUP_VECTOR.
4893	4838	*/
4894		− vmx->pi_desc.nv = POSTED_INTR_VECTOR;
4895		− __pi_set_sn(&vmx->pi_desc);
	4839	+ vmx->vt.pi_desc.nv = POSTED_INTR_VECTOR;
	4840	+ __pi_set_sn(&vmx->vt.pi_desc);
4896	4841	}
4897	4842
4898	4843	void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
@@ -5809,11 +5754,8 @@ static int handle_task_switch(struct kvm_vcpu *vcpu) · 910 unmodified lines
5809	5754
5810	5755	static int handle_ept_violation(struct kvm_vcpu *vcpu)
5811	5756	{
5812		− unsigned long exit_qualification;
	5757	+ unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
5813	5758	gpa_t gpa;
5814		− u64 error_code;
5815		−
5816		− exit_qualification = vmx_get_exit_qual(vcpu);
5817	5759
5818	5760	/*
5819	5761	* EPT violation happened while executing iret from NMI,
@@ -5829,23 +5771,6 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu) · 9 unmodified lines
5829	5771	gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
5830	5772	trace_kvm_page_fault(vcpu, gpa, exit_qualification);
5831	5773
5832		− /* Is it a read fault? */
5833		− error_code = (exit_qualification & EPT_VIOLATION_ACC_READ)
5834		− ? PFERR_USER_MASK : 0;
5835		− /* Is it a write fault? */
5836		− error_code \|= (exit_qualification & EPT_VIOLATION_ACC_WRITE)
5837		− ? PFERR_WRITE_MASK : 0;
5838		− /* Is it a fetch fault? */
5839		− error_code \|= (exit_qualification & EPT_VIOLATION_ACC_INSTR)
5840		− ? PFERR_FETCH_MASK : 0;
5841		− /* ept page table entry is present? */
5842		− error_code \|= (exit_qualification & EPT_VIOLATION_PROT_MASK)
5843		− ? PFERR_PRESENT_MASK : 0;
5844		−
5845		− if (error_code & EPT_VIOLATION_GVA_IS_VALID)
5846		− error_code \|= (exit_qualification & EPT_VIOLATION_GVA_TRANSLATED) ?
5847		− PFERR_GUEST_FINAL_MASK : PFERR_GUEST_PAGE_MASK;
5848		−
5849	5774	/*
5850	5775	* Check that the GPA doesn't exceed physical memory limits, as that is
5851	5776	* a guest page fault. We have to emulate the instruction here, because
@@ -5857,7 +5782,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu) · 5 unmodified lines
5857	5782	if (unlikely(allow_smaller_maxphyaddr && !kvm_vcpu_is_legal_gpa(vcpu, gpa)))
5858	5783	return kvm_emulate_instruction(vcpu, 0);
5859	5784
5860		− return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
	5785	+ return __vmx_handle_ept_violation(vcpu, gpa, exit_qualification);
5861	5786	}
5862	5787
5863	5788	static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
@@ -5902,7 +5827,7 @@ static bool vmx_unhandleable_emulation_required(struct kvm_vcpu *vcpu) · 38 unmodified lines
5902	5827	{
5903	5828	struct vcpu_vmx *vmx = to_vmx(vcpu);
5904	5829
5905		− if (!vmx->emulation_required)
	5830	+ if (!vmx->vt.emulation_required)
5906	5831	return false;
5907	5832
5908	5833	/*
@@ -5934,7 +5859,7 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu) · 25 unmodified lines
5934	5859	intr_window_requested = exec_controls_get(vmx) &
5935	5860	CPU_BASED_INTR_WINDOW_EXITING;
5936	5861
5937		− while (vmx->emulation_required && count-- != 0) {
	5862	+ while (vmx->vt.emulation_required && count-- != 0) {
5938	5863	if (intr_window_requested && !vmx_interrupt_blocked(vcpu))
5939	5864	return handle_interrupt_window(&vmx->vcpu);
5940	5865
@@ -6129,7 +6054,7 @@ static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) · 188 unmodified lines
6129	6054	* VM-Exits. Unconditionally set the flag here and leave the handling to
6130	6055	* vmx_handle_exit().
6131	6056	*/
6132		− to_vmx(vcpu)->exit_reason.bus_lock_detected = true;
	6057	+ to_vt(vcpu)->exit_reason.bus_lock_detected = true;
6133	6058	return 1;
6134	6059	}
6135	6060
@@ -6227,9 +6152,9 @@ void vmx_get_exit_info(struct kvm_vcpu vcpu, u32 reason, · 91 unmodified lines
6227	6152	{
6228	6153	struct vcpu_vmx *vmx = to_vmx(vcpu);
6229	6154
6230		− *reason = vmx->exit_reason.full;
	6155	+ *reason = vmx->vt.exit_reason.full;
6231	6156	*info1 = vmx_get_exit_qual(vcpu);
6232		− if (!(vmx->exit_reason.failed_vmentry)) {
	6157	+ if (!(vmx->vt.exit_reason.failed_vmentry)) {
6233	6158	*info2 = vmx->idt_vectoring_info;
6234	6159	*intr_info = vmx_get_intr_info(vcpu);
6235	6160	if (is_exception_with_error_code(*intr_info))
@@ -6525,7 +6450,7 @@ void dump_vmcs(struct kvm_vcpu *vcpu) · 289 unmodified lines
6525	6450	static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
6526	6451	{
6527	6452	struct vcpu_vmx *vmx = to_vmx(vcpu);
6528		− union vmx_exit_reason exit_reason = vmx->exit_reason;
	6453	+ union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu);
6529	6454	u32 vectoring_info = vmx->idt_vectoring_info;
6530	6455	u16 exit_handler_index;
6531	6456
@@ -6581,7 +6506,7 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) · 49 unmodified lines
6581	6506	* the least awful solution for the userspace case without
6582	6507	* risking false positives.
6583	6508	*/
6584		− if (vmx->emulation_required) {
	6509	+ if (vmx->vt.emulation_required) {
6585	6510	nested_vmx_vmexit(vcpu, EXIT_REASON_TRIPLE_FAULT, 0, 0);
6586	6511	return 1;
6587	6512	}
@@ -6591,7 +6516,7 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) · 3 unmodified lines
6591	6516	}
6592	6517
6593	6518	/* If guest state is invalid, start emulating. L2 is handled above. */
6594		− if (vmx->emulation_required)
	6519	+ if (vmx->vt.emulation_required)
6595	6520	return handle_invalid_guest_state(vcpu);
6596	6521
6597	6522	if (exit_reason.failed_vmentry) {
@@ -6691,7 +6616,7 @@ int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) · 93 unmodified lines
6691	6616	* Exit to user space when bus lock detected to inform that there is
6692	6617	* a bus lock in guest.
6693	6618	*/
6694		− if (to_vmx(vcpu)->exit_reason.bus_lock_detected) {
	6619	+ if (vmx_get_exit_reason(vcpu).bus_lock_detected) {
6695	6620	if (ret > 0)
6696	6621	vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK;
6697	6622
@@ -6970,22 +6895,22 @@ static void vmx_set_rvi(int vector) · 272 unmodified lines
6970	6895
6971	6896	int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
6972	6897	{
6973		− struct vcpu_vmx *vmx = to_vmx(vcpu);
	6898	+ struct vcpu_vt *vt = to_vt(vcpu);
6974	6899	int max_irr;
6975	6900	bool got_posted_interrupt;
6976	6901
6977	6902	if (KVM_BUG_ON(!enable_apicv, vcpu->kvm))
6978	6903	return -EIO;
6979	6904
6980		− if (pi_test_on(&vmx->pi_desc)) {
6981		− pi_clear_on(&vmx->pi_desc);
	6905	+ if (pi_test_on(&vt->pi_desc)) {
	6906	+ pi_clear_on(&vt->pi_desc);
6982	6907	/*
6983	6908	* IOMMU can write to PID.ON, so the barrier matters even on UP.
6984	6909	* But on x86 this is just a compiler barrier anyway.
6985	6910	*/
6986	6911	smp_mb__after_atomic();
6987	6912	got_posted_interrupt =
6988		− kvm_apic_update_irr(vcpu, vmx->pi_desc.pir, &max_irr);
	6913	+ kvm_apic_update_irr(vcpu, vt->pi_desc.pir, &max_irr);
6989	6914	} else {
6990	6915	max_irr = kvm_lapic_find_highest_irr(vcpu);
6991	6916	got_posted_interrupt = false;
@@ -7025,14 +6950,6 @@ void vmx_load_eoi_exitmap(struct kvm_vcpu vcpu, u64 eoi_exit_bitmap) · 33 unmodified lines
7025	6950	vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
7026	6951	}
7027	6952
7028		−void vmx_apicv_pre_state_restore(struct kvm_vcpu *vcpu)
7029		−{
7030		− struct vcpu_vmx *vmx = to_vmx(vcpu);
7031		−
7032		− pi_clear_on(&vmx->pi_desc);
7033		− memset(vmx->pi_desc.pir, 0, sizeof(vmx->pi_desc.pir));
7034		−}
7035		−
7036	6953	void vmx_do_interrupt_irqoff(unsigned long entry);
7037	6954	void vmx_do_nmi_irqoff(void);
7038	6955
@@ -7089,14 +7006,12 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, · 50 unmodified lines
7089	7006
7090	7007	void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu)
7091	7008	{
7092		− struct vcpu_vmx *vmx = to_vmx(vcpu);
7093		−
7094		− if (vmx->emulation_required)
	7009	+ if (to_vt(vcpu)->emulation_required)
7095	7010	return;
7096	7011
7097		− if (vmx->exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT)
	7012	+ if (vmx_get_exit_reason(vcpu).basic == EXIT_REASON_EXTERNAL_INTERRUPT)
7098	7013	handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu));
7099		− else if (vmx->exit_reason.basic == EXIT_REASON_EXCEPTION_NMI)
	7014	+ else if (vmx_get_exit_reason(vcpu).basic == EXIT_REASON_EXCEPTION_NMI)
7100	7015	handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu));
7101	7016	}
7102	7017
@@ -7331,10 +7246,10 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu, · 228 unmodified lines
7331	7246	* the fastpath even, all other exits must use the slow path.
7332	7247	*/
7333	7248	if (is_guest_mode(vcpu) &&
7334		− to_vmx(vcpu)->exit_reason.basic != EXIT_REASON_PREEMPTION_TIMER)
	7249	+ vmx_get_exit_reason(vcpu).basic != EXIT_REASON_PREEMPTION_TIMER)
7335	7250	return EXIT_FASTPATH_NONE;
7336	7251
7337		− switch (to_vmx(vcpu)->exit_reason.basic) {
	7252	+ switch (vmx_get_exit_reason(vcpu).basic) {
7338	7253	case EXIT_REASON_MSR_WRITE:
7339	7254	return handle_fastpath_set_msr_irqoff(vcpu);
7340	7255	case EXIT_REASON_PREEMPTION_TIMER:
@@ -7346,6 +7261,20 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu, · 5 unmodified lines
7346	7261	}
7347	7262	}
7348	7263
	7264	+noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu)
	7265	+{
	7266	+ if ((u16)vmx_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI \|\|
	7267	+ !is_nmi(vmx_get_intr_info(vcpu)))
	7268	+ return;
	7269	+
	7270	+ kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
	7271	+ if (cpu_feature_enabled(X86_FEATURE_FRED))
	7272	+ fred_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR);
	7273	+ else
	7274	+ vmx_do_nmi_irqoff();
	7275	+ kvm_after_interrupt(vcpu);
	7276	+}
	7277	+
7349	7278	static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,
7350	7279	unsigned int flags)
7351	7280	{
@@ -7381,23 +7310,15 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, · 29 unmodified lines
7381	7310	vmx_enable_fb_clear(vmx);
7382	7311
7383	7312	if (unlikely(vmx->fail)) {
7384		− vmx->exit_reason.full = 0xdead;
	7313	+ vmx->vt.exit_reason.full = 0xdead;
7385	7314	goto out;
7386	7315	}
7387	7316
7388		− vmx->exit_reason.full = vmcs_read32(VM_EXIT_REASON);
7389		− if (likely(!vmx->exit_reason.failed_vmentry))
	7317	+ vmx->vt.exit_reason.full = vmcs_read32(VM_EXIT_REASON);
	7318	+ if (likely(!vmx_get_exit_reason(vcpu).failed_vmentry))
7390	7319	vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
7391	7320
7392		− if ((u16)vmx->exit_reason.basic == EXIT_REASON_EXCEPTION_NMI &&
7393		− is_nmi(vmx_get_intr_info(vcpu))) {
7394		− kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
7395		− if (cpu_feature_enabled(X86_FEATURE_FRED))
7396		− fred_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR);
7397		− else
7398		− vmx_do_nmi_irqoff();
7399		− kvm_after_interrupt(vcpu);
7400		− }
	7321	+ vmx_handle_nmi(vcpu);
7401	7322
7402	7323	out:
7403	7324	guest_state_exit_irqoff();
@@ -7418,15 +7339,15 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, bool force_immediate_exit) · 14 unmodified lines
7418	7339	* start emulation until we arrive back to a valid state. Synthesize a
7419	7340	* consistency check VM-Exit due to invalid guest state and bail.
7420	7341	*/
7421		− if (unlikely(vmx->emulation_required)) {
	7342	+ if (unlikely(vmx->vt.emulation_required)) {
7422	7343	vmx->fail = 0;
7423	7344
7424		− vmx->exit_reason.full = EXIT_REASON_INVALID_STATE;
7425		− vmx->exit_reason.failed_vmentry = 1;
	7345	+ vmx->vt.exit_reason.full = EXIT_REASON_INVALID_STATE;
	7346	+ vmx->vt.exit_reason.failed_vmentry = 1;
7426	7347	kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_1);
7427		− vmx->exit_qualification = ENTRY_FAIL_DEFAULT;
	7348	+ vmx->vt.exit_qualification = ENTRY_FAIL_DEFAULT;
7428	7349	kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_2);
7429		− vmx->exit_intr_info = 0;
	7350	+ vmx->vt.exit_intr_info = 0;
7430	7351	return EXIT_FASTPATH_NONE;
7431	7352	}
7432	7353
@@ -7529,7 +7450,7 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, bool force_immediate_exit) · 96 unmodified lines
7529	7450	* checking.
7530	7451	*/
7531	7452	if (vmx->nested.nested_run_pending &&
7532		− !vmx->exit_reason.failed_vmentry)
	7453	+ !vmx_get_exit_reason(vcpu).failed_vmentry)
7533	7454	++vcpu->stat.nested_run;
7534	7455
7535	7456	vmx->nested.nested_run_pending = 0;
@@ -7538,12 +7459,12 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, bool force_immediate_exit) · 2 unmodified lines
7538	7459	if (unlikely(vmx->fail))
7539	7460	return EXIT_FASTPATH_NONE;
7540	7461
7541		− if (unlikely((u16)vmx->exit_reason.basic == EXIT_REASON_MCE_DURING_VMENTRY))
	7462	+ if (unlikely((u16)vmx_get_exit_reason(vcpu).basic == EXIT_REASON_MCE_DURING_VMENTRY))
7542	7463	kvm_machine_check();
7543	7464
7544	7465	trace_kvm_exit(vcpu, KVM_ISA_VMX);
7545	7466
7546		− if (unlikely(vmx->exit_reason.failed_vmentry))
	7467	+ if (unlikely(vmx_get_exit_reason(vcpu).failed_vmentry))
7547	7468	return EXIT_FASTPATH_NONE;
7548	7469
7549	7470	vmx->loaded_vmcs->launched = 1;
@@ -7575,7 +7496,7 @@ int vmx_vcpu_create(struct kvm_vcpu *vcpu) · 25 unmodified lines
7575	7496	BUILD_BUG_ON(offsetof(struct vcpu_vmx, vcpu) != 0);
7576	7497	vmx = to_vmx(vcpu);
7577	7498
7578		− INIT_LIST_HEAD(&vmx->pi_wakeup_list);
	7499	+ INIT_LIST_HEAD(&vmx->vt.pi_wakeup_list);
7579	7500
7580	7501	err = -ENOMEM;
7581	7502
@@ -7673,7 +7594,7 @@ int vmx_vcpu_create(struct kvm_vcpu *vcpu) · 91 unmodified lines
7673	7594
7674	7595	if (vmx_can_use_ipiv(vcpu))
7675	7596	WRITE_ONCE(to_kvm_vmx(vcpu->kvm)->pid_table[vcpu->vcpu_id],
7676		− __pa(&vmx->pi_desc) \| PID_TABLE_ENTRY_VALID);
	7597	+ __pa(&vmx->vt.pi_desc) \| PID_TABLE_ENTRY_VALID);
7677	7598
7678	7599	return 0;
7679	7600
@@ -7717,9 +7638,23 @@ int vmx_vm_init(struct kvm *kvm) · 37 unmodified lines
7717	7638	break;
7718	7639	}
7719	7640	}
	7641	+
	7642	+ if (enable_pml)
	7643	+ kvm->arch.cpu_dirty_log_size = PML_LOG_NR_ENTRIES;
7720	7644	return 0;
7721	7645	}
7722	7646
	7647	+static inline bool vmx_ignore_guest_pat(struct kvm *kvm)
	7648	+{
	7649	+ /*
	7650	+ * Non-coherent DMA devices need the guest to flush CPU properly.
	7651	+ * In that case it is not possible to map all guest RAM as WB, so
	7652	+ * always trust guest PAT.
	7653	+ */
	7654	+ return !kvm_arch_has_noncoherent_dma(kvm) &&
	7655	+ kvm_check_has_quirk(kvm, KVM_X86_QUIRK_IGNORE_GUEST_PAT);
	7656	+}
	7657	+
7723	7658	u8 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
7724	7659	{
7725	7660	/*
@@ -7729,13 +7664,8 @@ u8 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio) · 3 unmodified lines
7729	7664	if (is_mmio)
7730	7665	return MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT;
7731	7666
7732		− /*
7733		− * Force WB and ignore guest PAT if the VM does NOT have a non-coherent
7734		− * device attached. Letting the guest control memory types on Intel
7735		− * CPUs may result in unexpected behavior, and so KVM's ABI is to trust
7736		− * the guest to behave only as a last resort.
7737		− */
7738		− if (!kvm_arch_has_noncoherent_dma(vcpu->kvm))
	7667	+ /* Force WB if ignoring guest PAT */
	7668	+ if (vmx_ignore_guest_pat(vcpu->kvm))
7739	7669	return (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT) \| VMX_EPT_IPAT_BIT;
7740	7670
7741	7671	return (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT);
@@ -8597,6 +8527,8 @@ __init int vmx_hardware_setup(void) · 855 unmodified lines
8597	8527	if (enable_ept)
8598	8528	kvm_mmu_set_ept_masks(enable_ept_ad_bits,
8599	8529	cpu_has_vmx_ept_execute_only());
	8530	+ else
	8531	+ vt_x86_ops.get_mt_mask = NULL;
8600	8532
8601	8533	/*
8602	8534	* Setup shadow_me_value/shadow_me_mask to include MKTME KeyID
@@ -8614,9 +8546,6 @@ __init int vmx_hardware_setup(void) · 11 unmodified lines
8614	8546	if (!enable_ept \|\| !enable_ept_ad_bits \|\| !cpu_has_vmx_pml())
8615	8547	enable_pml = 0;
8616	8548
8617		− if (!enable_pml)
8618		− vt_x86_ops.cpu_dirty_log_size = 0;
8619		−
8620	8549	if (!cpu_has_vmx_preemption_timer())
8621	8550	enable_preemption_timer = false;
8622	8551
@@ -8674,6 +8603,27 @@ __init int vmx_hardware_setup(void) · 51 unmodified lines
8674	8603
8675	8604	kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler);
8676	8605
	8606	+ /*
	8607	+ * On Intel CPUs that lack self-snoop feature, letting the guest control
	8608	+ * memory types may result in unexpected behavior. So always ignore guest
	8609	+ * PAT on those CPUs and map VM as writeback, not allowing userspace to
	8610	+ * disable the quirk.
	8611	+ *
	8612	+ * On certain Intel CPUs (e.g. SPR, ICX), though self-snoop feature is
	8613	+ * supported, UC is slow enough to cause issues with some older guests (e.g.
	8614	+ * an old version of bochs driver uses ioremap() instead of ioremap_wc() to
	8615	+ * map the video RAM, causing wayland desktop to fail to get started
	8616	+ * correctly). To avoid breaking those older guests that rely on KVM to force
	8617	+ * memory type to WB, provide KVM_X86_QUIRK_IGNORE_GUEST_PAT to preserve the
	8618	+ * safer (for performance) default behavior.
	8619	+ *
	8620	+ * On top of this, non-coherent DMA devices need the guest to flush CPU
	8621	+ * caches properly. This also requires honoring guest PAT, and is forced
	8622	+ * independent of the quirk in vmx_ignore_guest_pat().
	8623	+ */
	8624	+ if (!static_cpu_has(X86_FEATURE_SELFSNOOP))
	8625	+ kvm_caps.supported_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT;
	8626	+ kvm_caps.inapplicable_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT;
8677	8627	return r;
8678	8628	}
8679	8629
@@ -8687,23 +8637,16 @@ static void vmx_cleanup_l1d_flush(void) · 7 unmodified lines
8687	8637	l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
8688	8638	}
8689	8639
8690		−static void __vmx_exit(void)
	8640	+void vmx_exit(void)
8691	8641	{
8692	8642	allow_smaller_maxphyaddr = false;
8693	8643
8694	8644	vmx_cleanup_l1d_flush();
8695		−}
8696	8645
8697		−static void __exit vmx_exit(void)
8698		−{
8699		− kvm_exit();
8700		− __vmx_exit();
8701	8646	kvm_x86_vendor_exit();
8702		−
8703	8647	}
8704		−module_exit(vmx_exit);
8705	8648
8706		−static int __init vmx_init(void)
	8649	+int __init vmx_init(void)
8707	8650	{
8708	8651	int r, cpu;
8709	8652
@@ -8747,21 +8690,9 @@ static int __init vmx_init(void) · 37 unmodified lines
8747	8690	if (!enable_ept)
8748	8691	allow_smaller_maxphyaddr = true;
8749	8692
8750		− /*
8751		− * Common KVM initialization _must_ come last, after this, /dev/kvm is
8752		− * exposed to userspace!
8753		− */
8754		− r = kvm_init(sizeof(struct vcpu_vmx), __alignof__(struct vcpu_vmx),
8755		− THIS_MODULE);
8756		− if (r)
8757		− goto err_kvm_init;
8758		−
8759	8693	return 0;
8760	8694
8761		−err_kvm_init:
8762		− __vmx_exit();
8763	8695	err_l1d_flush:
8764	8696	kvm_x86_vendor_exit();
8765	8697	return r;
8766	8698	}
8767		−module_init(vmx_init);

arch/x86/kvm/vmx/vmx.h+43−97

@@ -11,11 +11,13 @@ · 10 unmodified lines
11	11
12	12	#include "capabilities.h"
13	13	#include "../kvm_cache_regs.h"
	14	+#include "pmu_intel.h"
14	15	#include "vmcs.h"
15	16	#include "vmx_ops.h"
16	17	#include "../cpuid.h"
17	18	#include "run_flags.h"
18	19	#include "../mmu.h"
	20	+#include "common.h"
19	21
20	22	#define X2APIC_MSR(r) (APIC_BASE_MSR + ((r) >> 4))
21	23
@@ -67,47 +69,6 @@ struct pt_desc { · 45 unmodified lines
67	69	struct pt_ctx guest;
68	70	};
69	71
70		−union vmx_exit_reason {
71		− struct {
72		− u32 basic : 16;
73		− u32 reserved16 : 1;
74		− u32 reserved17 : 1;
75		− u32 reserved18 : 1;
76		− u32 reserved19 : 1;
77		− u32 reserved20 : 1;
78		− u32 reserved21 : 1;
79		− u32 reserved22 : 1;
80		− u32 reserved23 : 1;
81		− u32 reserved24 : 1;
82		− u32 reserved25 : 1;
83		− u32 bus_lock_detected : 1;
84		− u32 enclave_mode : 1;
85		− u32 smi_pending_mtf : 1;
86		− u32 smi_from_vmx_root : 1;
87		− u32 reserved30 : 1;
88		− u32 failed_vmentry : 1;
89		− };
90		− u32 full;
91		−};
92		−
93		−struct lbr_desc {
94		− /* Basic info about guest LBR records. */
95		− struct x86_pmu_lbr records;
96		−
97		− /*
98		− * Emulate LBR feature via passthrough LBR registers when the
99		− * per-vcpu guest LBR event is scheduled on the current pcpu.
100		− *
101		− * The records may be inaccurate if the host reclaims the LBR.
102		− */
103		− struct perf_event *event;
104		−
105		− /* True if LBRs are marked as not intercepted in the MSR bitmap */
106		− bool msr_passthrough;
107		−};
108		−
109		−extern struct x86_pmu_lbr vmx_lbr_caps;
110		−
111	72	/*
112	73	* The nested_vmx structure is part of vcpu_vmx, and holds information we need
113	74	* for correct emulation of VMX (i.e., nested VMX) on this vcpu.
@@ -248,20 +209,10 @@ struct nested_vmx { · 134 unmodified lines
248	209
249	210	struct vcpu_vmx {
250	211	struct kvm_vcpu vcpu;
	212	+ struct vcpu_vt vt;
251	213	u8 fail;
252	214	u8 x2apic_msr_bitmap_mode;
253	215
254		− /*
255		− * If true, host state has been stored in vmx->loaded_vmcs for
256		− * the CPU registers that only need to be switched when transitioning
257		− * to/from the kernel, and the registers have been loaded with guest
258		− * values. If false, host state is loaded in the CPU registers
259		− * and vmx->loaded_vmcs->host_state is invalid.
260		− */
261		− bool guest_state_loaded;
262		−
263		− unsigned long exit_qualification;
264		− u32 exit_intr_info;
265	216	u32 idt_vectoring_info;
266	217	ulong rflags;
267	218
@@ -274,7 +225,6 @@ struct vcpu_vmx { · 6 unmodified lines
274	225	struct vmx_uret_msr guest_uret_msrs[MAX_NR_USER_RETURN_MSRS];
275	226	bool guest_uret_msrs_loaded;
276	227	#ifdef CONFIG_X86_64
277		− u64 msr_host_kernel_gs_base;
278	228	u64 msr_guest_kernel_gs_base;
279	229	#endif
280	230
@@ -313,15 +263,6 @@ struct vcpu_vmx { · 32 unmodified lines
313	263	} seg[8];
314	264	} segment_cache;
315	265	int vpid;
316		− bool emulation_required;
317		−
318		− union vmx_exit_reason exit_reason;
319		−
320		− /* Posted interrupt descriptor */
321		− struct pi_desc pi_desc;
322		−
323		− /* Used if this vCPU is waiting for PI notification wakeup. */
324		− struct list_head pi_wakeup_list;
325	266
326	267	/* Support for a guest hypervisor (nested VMX) */
327	268	struct nested_vmx nested;
@@ -376,6 +317,43 @@ struct kvm_vmx { · 48 unmodified lines
376	317	u64 *pid_table;
377	318	};
378	319
	320	+static __always_inline struct vcpu_vt to_vt(struct kvm_vcpu vcpu)
	321	+{
	322	+ return &(container_of(vcpu, struct vcpu_vmx, vcpu)->vt);
	323	+}
	324	+
	325	+static __always_inline struct kvm_vcpu vt_to_vcpu(struct vcpu_vt vt)
	326	+{
	327	+ return &(container_of(vt, struct vcpu_vmx, vt)->vcpu);
	328	+}
	329	+
	330	+static __always_inline union vmx_exit_reason vmx_get_exit_reason(struct kvm_vcpu *vcpu)
	331	+{
	332	+ return to_vt(vcpu)->exit_reason;
	333	+}
	334	+
	335	+static __always_inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu)
	336	+{
	337	+ struct vcpu_vt *vt = to_vt(vcpu);
	338	+
	339	+ if (!kvm_register_test_and_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_1) &&
	340	+ !WARN_ON_ONCE(is_td_vcpu(vcpu)))
	341	+ vt->exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
	342	+
	343	+ return vt->exit_qualification;
	344	+}
	345	+
	346	+static __always_inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu)
	347	+{
	348	+ struct vcpu_vt *vt = to_vt(vcpu);
	349	+
	350	+ if (!kvm_register_test_and_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_2) &&
	351	+ !WARN_ON_ONCE(is_td_vcpu(vcpu)))
	352	+ vt->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
	353	+
	354	+ return vt->exit_intr_info;
	355	+}
	356	+
379	357	void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu,
380	358	struct loaded_vmcs *buddy);
381	359	int allocate_vpid(void);
@@ -662,45 +640,10 @@ static __always_inline struct vcpu_vmx to_vmx(struct kvm_vcpu vcpu) · 280 unmodified lines
662	640	return container_of(vcpu, struct vcpu_vmx, vcpu);
663	641	}
664	642
665		−static inline struct lbr_desc vcpu_to_lbr_desc(struct kvm_vcpu vcpu)
666		−{
667		− return &to_vmx(vcpu)->lbr_desc;
668		−}
669		−
670		−static inline struct x86_pmu_lbr vcpu_to_lbr_records(struct kvm_vcpu vcpu)
671		−{
672		− return &vcpu_to_lbr_desc(vcpu)->records;
673		−}
674		−
675		−static inline bool intel_pmu_lbr_is_enabled(struct kvm_vcpu *vcpu)
676		−{
677		− return !!vcpu_to_lbr_records(vcpu)->nr;
678		−}
679		−
680	643	void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu);
681	644	int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu);
682	645	void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu);
683	646
684		−static __always_inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu)
685		−{
686		− struct vcpu_vmx *vmx = to_vmx(vcpu);
687		−
688		− if (!kvm_register_test_and_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_1))
689		− vmx->exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
690		−
691		− return vmx->exit_qualification;
692		−}
693		−
694		−static __always_inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu)
695		−{
696		− struct vcpu_vmx *vmx = to_vmx(vcpu);
697		−
698		− if (!kvm_register_test_and_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_2))
699		− vmx->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
700		−
701		− return vmx->exit_intr_info;
702		−}
703		−
704	647	struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags);
705	648	void free_vmcs(struct vmcs *vmcs);
706	649	int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs);
@@ -758,4 +701,7 @@ static inline void vmx_segment_cache_clear(struct vcpu_vmx *vmx) · 51 unmodified lines
758	701	vmx->segment_cache.bitmask = 0;
759	702	}
760	703
	704	+int vmx_init(void);
	705	+void vmx_exit(void);
	706	+
761	707	#endif /* __KVM_X86_VMX_H */

arch/x86/kvm/vmx/x86_ops.h+110−1

@@ -46,7 +46,6 @@ int vmx_check_intercept(struct kvm_vcpu *vcpu, · 45 unmodified lines
46	46	bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu);
47	47	void vmx_migrate_timers(struct kvm_vcpu *vcpu);
48	48	void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu);
49		−void vmx_apicv_pre_state_restore(struct kvm_vcpu *vcpu);
50	49	void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr);
51	50	int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu);
52	51	void vmx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode,
@@ -121,4 +120,114 @@ void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu); · 68 unmodified lines
121	120	#endif
122	121	void vmx_setup_mce(struct kvm_vcpu *vcpu);
123	122
	123	+#ifdef CONFIG_KVM_INTEL_TDX
	124	+void tdx_disable_virtualization_cpu(void);
	125	+int tdx_vm_init(struct kvm *kvm);
	126	+void tdx_mmu_release_hkid(struct kvm *kvm);
	127	+void tdx_vm_destroy(struct kvm *kvm);
	128	+int tdx_vm_ioctl(struct kvm kvm, void __user argp);
	129	+
	130	+int tdx_vcpu_create(struct kvm_vcpu *vcpu);
	131	+void tdx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event);
	132	+void tdx_vcpu_free(struct kvm_vcpu *vcpu);
	133	+void tdx_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
	134	+int tdx_vcpu_pre_run(struct kvm_vcpu *vcpu);
	135	+fastpath_t tdx_vcpu_run(struct kvm_vcpu *vcpu, bool force_immediate_exit);
	136	+void tdx_prepare_switch_to_guest(struct kvm_vcpu *vcpu);
	137	+void tdx_vcpu_put(struct kvm_vcpu *vcpu);
	138	+bool tdx_protected_apic_has_interrupt(struct kvm_vcpu *vcpu);
	139	+int tdx_handle_exit(struct kvm_vcpu *vcpu,
	140	+ enum exit_fastpath_completion fastpath);
	141	+
	142	+void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode,
	143	+ int trig_mode, int vector);
	144	+void tdx_inject_nmi(struct kvm_vcpu *vcpu);
	145	+void tdx_get_exit_info(struct kvm_vcpu vcpu, u32 reason,
	146	+ u64 info1, u64 info2, u32 intr_info, u32 error_code);
	147	+bool tdx_has_emulated_msr(u32 index);
	148	+int tdx_get_msr(struct kvm_vcpu vcpu, struct msr_data msr);
	149	+int tdx_set_msr(struct kvm_vcpu vcpu, struct msr_data msr);
	150	+
	151	+int tdx_vcpu_ioctl(struct kvm_vcpu vcpu, void __user argp);
	152	+
	153	+int tdx_sept_link_private_spt(struct kvm *kvm, gfn_t gfn,
	154	+ enum pg_level level, void *private_spt);
	155	+int tdx_sept_free_private_spt(struct kvm *kvm, gfn_t gfn,
	156	+ enum pg_level level, void *private_spt);
	157	+int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn,
	158	+ enum pg_level level, kvm_pfn_t pfn);
	159	+int tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn,
	160	+ enum pg_level level, kvm_pfn_t pfn);
	161	+
	162	+void tdx_flush_tlb_current(struct kvm_vcpu *vcpu);
	163	+void tdx_flush_tlb_all(struct kvm_vcpu *vcpu);
	164	+void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level);
	165	+int tdx_gmem_private_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn);
	166	+#else
	167	+static inline void tdx_disable_virtualization_cpu(void) {}
	168	+static inline int tdx_vm_init(struct kvm *kvm) { return -EOPNOTSUPP; }
	169	+static inline void tdx_mmu_release_hkid(struct kvm *kvm) {}
	170	+static inline void tdx_vm_destroy(struct kvm *kvm) {}
	171	+static inline int tdx_vm_ioctl(struct kvm kvm, void __user argp) { return -EOPNOTSUPP; }
	172	+
	173	+static inline int tdx_vcpu_create(struct kvm_vcpu *vcpu) { return -EOPNOTSUPP; }
	174	+static inline void tdx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) {}
	175	+static inline void tdx_vcpu_free(struct kvm_vcpu *vcpu) {}
	176	+static inline void tdx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) {}
	177	+static inline int tdx_vcpu_pre_run(struct kvm_vcpu *vcpu) { return -EOPNOTSUPP; }
	178	+static inline fastpath_t tdx_vcpu_run(struct kvm_vcpu *vcpu, bool force_immediate_exit)
	179	+{
	180	+ return EXIT_FASTPATH_NONE;
	181	+}
	182	+static inline void tdx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) {}
	183	+static inline void tdx_vcpu_put(struct kvm_vcpu *vcpu) {}
	184	+static inline bool tdx_protected_apic_has_interrupt(struct kvm_vcpu *vcpu) { return false; }
	185	+static inline int tdx_handle_exit(struct kvm_vcpu *vcpu,
	186	+ enum exit_fastpath_completion fastpath) { return 0; }
	187	+
	188	+static inline void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode,
	189	+ int trig_mode, int vector) {}
	190	+static inline void tdx_inject_nmi(struct kvm_vcpu *vcpu) {}
	191	+static inline void tdx_get_exit_info(struct kvm_vcpu vcpu, u32 reason, u64 *info1,
	192	+ u64 info2, u32 intr_info, u32 *error_code) {}
	193	+static inline bool tdx_has_emulated_msr(u32 index) { return false; }
	194	+static inline int tdx_get_msr(struct kvm_vcpu vcpu, struct msr_data msr) { return 1; }
	195	+static inline int tdx_set_msr(struct kvm_vcpu vcpu, struct msr_data msr) { return 1; }
	196	+
	197	+static inline int tdx_vcpu_ioctl(struct kvm_vcpu vcpu, void __user argp) { return -EOPNOTSUPP; }
	198	+
	199	+static inline int tdx_sept_link_private_spt(struct kvm *kvm, gfn_t gfn,
	200	+ enum pg_level level,
	201	+ void *private_spt)
	202	+{
	203	+ return -EOPNOTSUPP;
	204	+}
	205	+
	206	+static inline int tdx_sept_free_private_spt(struct kvm *kvm, gfn_t gfn,
	207	+ enum pg_level level,
	208	+ void *private_spt)
	209	+{
	210	+ return -EOPNOTSUPP;
	211	+}
	212	+
	213	+static inline int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn,
	214	+ enum pg_level level,
	215	+ kvm_pfn_t pfn)
	216	+{
	217	+ return -EOPNOTSUPP;
	218	+}
	219	+
	220	+static inline int tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn,
	221	+ enum pg_level level,
	222	+ kvm_pfn_t pfn)
	223	+{
	224	+ return -EOPNOTSUPP;
	225	+}
	226	+
	227	+static inline void tdx_flush_tlb_current(struct kvm_vcpu *vcpu) {}
	228	+static inline void tdx_flush_tlb_all(struct kvm_vcpu *vcpu) {}
	229	+static inline void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level) {}
	230	+static inline int tdx_gmem_private_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn) { return 0; }
	231	+#endif
	232	+
124	233	#endif /* __KVM_X86_VMX_X86_OPS_H */

arch/x86/kvm/x86.c+68−31

@@ -90,7 +90,6 @@ · 89 unmodified lines
90	90	#include "trace.h"
91	91
92	92	#define MAX_IO_MSRS 256
93		−#define KVM_MAX_MCE_BANKS 32
94	93
95	94	/*
96	95	* Note, kvm_caps fields should never have default values, all fields must be
@@ -636,6 +635,15 @@ static void kvm_user_return_msr_cpu_online(void) · 539 unmodified lines
636	635	}
637	636	}
638	637
	638	+static void kvm_user_return_register_notifier(struct kvm_user_return_msrs *msrs)
	639	+{
	640	+ if (!msrs->registered) {
	641	+ msrs->urn.on_user_return = kvm_on_user_return;
	642	+ user_return_notifier_register(&msrs->urn);
	643	+ msrs->registered = true;
	644	+ }
	645	+}
	646	+
639	647	int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask)
640	648	{
641	649	struct kvm_user_return_msrs *msrs = this_cpu_ptr(user_return_msrs);
@@ -649,15 +657,20 @@ int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask) · 7 unmodified lines
649	657	return 1;
650	658
651	659	msrs->values[slot].curr = value;
652		− if (!msrs->registered) {
653		− msrs->urn.on_user_return = kvm_on_user_return;
654		− user_return_notifier_register(&msrs->urn);
655		− msrs->registered = true;
656		− }
	660	+ kvm_user_return_register_notifier(msrs);
657	661	return 0;
658	662	}
659	663	EXPORT_SYMBOL_GPL(kvm_set_user_return_msr);
660	664
	665	+void kvm_user_return_msr_update_cache(unsigned int slot, u64 value)
	666	+{
	667	+ struct kvm_user_return_msrs *msrs = this_cpu_ptr(user_return_msrs);
	668	+
	669	+ msrs->values[slot].curr = value;
	670	+ kvm_user_return_register_notifier(msrs);
	671	+}
	672	+EXPORT_SYMBOL_GPL(kvm_user_return_msr_update_cache);
	673	+
661	674	static void drop_user_return_notifiers(void)
662	675	{
663	676	struct kvm_user_return_msrs *msrs = this_cpu_ptr(user_return_msrs);
@@ -4737,6 +4750,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 4073 unmodified lines
4737	4750	break;
4738	4751	case KVM_CAP_MAX_VCPUS:
4739	4752	r = KVM_MAX_VCPUS;
	4753	+ if (kvm)
	4754	+ r = kvm->max_vcpus;
4740	4755	break;
4741	4756	case KVM_CAP_MAX_VCPU_ID:
4742	4757	r = KVM_MAX_VCPU_IDS;
@@ -4792,7 +4807,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 49 unmodified lines
4792	4807	r = enable_pmu ? KVM_CAP_PMU_VALID_MASK : 0;
4793	4808	break;
4794	4809	case KVM_CAP_DISABLE_QUIRKS2:
4795		− r = KVM_X86_VALID_QUIRKS;
	4810	+ r = kvm_caps.supported_quirks;
4796	4811	break;
4797	4812	case KVM_CAP_X86_NOTIFY_VMEXIT:
4798	4813	r = kvm_caps.has_notify_vmexit;
@@ -5115,6 +5130,9 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) · 316 unmodified lines
5115	5130	static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
5116	5131	struct kvm_lapic_state *s)
5117	5132	{
	5133	+ if (vcpu->arch.apic->guest_apic_protected)
	5134	+ return -EINVAL;
	5135	+
5118	5136	kvm_x86_call(sync_pir_to_irr)(vcpu);
5119	5137
5120	5138	return kvm_apic_get_state(vcpu, s);
@@ -5125,6 +5143,9 @@ static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, · 4 unmodified lines
5125	5143	{
5126	5144	int r;
5127	5145
	5146	+ if (vcpu->arch.apic->guest_apic_protected)
	5147	+ return -EINVAL;
	5148	+
5128	5149	r = kvm_apic_set_state(vcpu, s);
5129	5150	if (r)
5130	5151	return r;
@@ -6302,6 +6323,12 @@ long kvm_arch_vcpu_ioctl(struct file *filp, · 1171 unmodified lines
6302	6323	case KVM_SET_DEVICE_ATTR:
6303	6324	r = kvm_vcpu_ioctl_device_attr(vcpu, ioctl, argp);
6304	6325	break;
	6326	+ case KVM_MEMORY_ENCRYPT_OP:
	6327	+ r = -ENOTTY;
	6328	+ if (!kvm_x86_ops.vcpu_mem_enc_ioctl)
	6329	+ goto out;
	6330	+ r = kvm_x86_ops.vcpu_mem_enc_ioctl(vcpu, argp);
	6331	+ break;
6305	6332	default:
6306	6333	r = -EINVAL;
6307	6334	}
@@ -6489,7 +6516,7 @@ void kvm_arch_sync_dirty_log(struct kvm kvm, struct kvm_memory_slot memslot) · 181 unmodified lines
6489	6516	struct kvm_vcpu *vcpu;
6490	6517	unsigned long i;
6491	6518
6492		− if (!kvm_x86_ops.cpu_dirty_log_size)
	6519	+ if (!kvm->arch.cpu_dirty_log_size)
6493	6520	return;
6494	6521
6495	6522	kvm_for_each_vcpu(i, vcpu, kvm)
@@ -6519,11 +6546,11 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, · 23 unmodified lines
6519	6546	switch (cap->cap) {
6520	6547	case KVM_CAP_DISABLE_QUIRKS2:
6521	6548	r = -EINVAL;
6522		− if (cap->args[0] & ~KVM_X86_VALID_QUIRKS)
	6549	+ if (cap->args[0] & ~kvm_caps.supported_quirks)
6523	6550	break;
6524	6551	fallthrough;
6525	6552	case KVM_CAP_DISABLE_QUIRKS:
6526		− kvm->arch.disabled_quirks = cap->args[0];
	6553	+ kvm->arch.disabled_quirks \|= cap->args[0] & kvm_caps.supported_quirks;
6527	6554	r = 0;
6528	6555	break;
6529	6556	case KVM_CAP_SPLIT_IRQCHIP: {
@@ -7298,10 +7325,6 @@ set_pit2_out: · 768 unmodified lines
7298	7325	goto out;
7299	7326	}
7300	7327	case KVM_MEMORY_ENCRYPT_OP: {
7301		− r = -ENOTTY;
7302		− if (!kvm_x86_ops.mem_enc_ioctl)
7303		− goto out;
7304		−
7305	7328	r = kvm_x86_call(mem_enc_ioctl)(kvm, argp);
7306	7329	break;
7307	7330	}
@@ -9769,6 +9792,8 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops) · 2461 unmodified lines
9769	9792	kvm_host.xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
9770	9793	kvm_caps.supported_xcr0 = kvm_host.xcr0 & KVM_SUPPORTED_XCR0;
9771	9794	}
	9795	+ kvm_caps.supported_quirks = KVM_X86_VALID_QUIRKS;
	9796	+ kvm_caps.inapplicable_quirks = KVM_X86_CONDITIONAL_QUIRKS;
9772	9797
9773	9798	rdmsrl_safe(MSR_EFER, &kvm_host.efer);
9774	9799
@@ -9813,6 +9838,10 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops) · 38 unmodified lines
9813	9838	if (IS_ENABLED(CONFIG_KVM_SW_PROTECTED_VM) && tdp_mmu_enabled)
9814	9839	kvm_caps.supported_vm_types \|= BIT(KVM_X86_SW_PROTECTED_VM);
9815	9840
	9841	+ /* KVM always ignores guest PAT for shadow paging. */
	9842	+ if (!tdp_enabled)
	9843	+ kvm_caps.supported_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT;
	9844	+
9816	9845	if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
9817	9846	kvm_caps.supported_xss = 0;
9818	9847
@@ -10021,13 +10050,16 @@ static int complete_hypercall_exit(struct kvm_vcpu *vcpu) · 202 unmodified lines
10021	10050	return kvm_skip_emulated_instruction(vcpu);
10022	10051	}
10023	10052
10024		−int ____kvm_emulate_hypercall(struct kvm_vcpu *vcpu, unsigned long nr,
10025		− unsigned long a0, unsigned long a1,
10026		− unsigned long a2, unsigned long a3,
10027		− int op_64_bit, int cpl,
	10053	+int ____kvm_emulate_hypercall(struct kvm_vcpu *vcpu, int cpl,
10028	10054	int (complete_hypercall)(struct kvm_vcpu ))
10029	10055	{
10030	10056	unsigned long ret;
	10057	+ unsigned long nr = kvm_rax_read(vcpu);
	10058	+ unsigned long a0 = kvm_rbx_read(vcpu);
	10059	+ unsigned long a1 = kvm_rcx_read(vcpu);
	10060	+ unsigned long a2 = kvm_rdx_read(vcpu);
	10061	+ unsigned long a3 = kvm_rsi_read(vcpu);
	10062	+ int op_64_bit = is_64_bit_hypercall(vcpu);
10031	10063
10032	10064	++vcpu->stat.hypercalls;
10033	10065
@@ -10130,9 +10162,7 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) · 96 unmodified lines
10130	10162	if (kvm_hv_hypercall_enabled(vcpu))
10131	10163	return kvm_hv_hypercall(vcpu);
10132	10164
10133		− return __kvm_emulate_hypercall(vcpu, rax, rbx, rcx, rdx, rsi,
10134		− is_64_bit_hypercall(vcpu),
10135		− kvm_x86_call(get_cpl)(vcpu),
	10165	+ return __kvm_emulate_hypercall(vcpu, kvm_x86_call(get_cpl)(vcpu),
10136	10166	complete_hypercall_exit);
10137	10167	}
10138	10168	EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);
@@ -10976,7 +11006,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 837 unmodified lines
10976	11006	if (vcpu->arch.guest_fpu.xfd_err)
10977	11007	wrmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);
10978	11008
10979		− if (unlikely(vcpu->arch.switch_db_regs)) {
	11009	+ if (unlikely(vcpu->arch.switch_db_regs &&
	11010	+ !(vcpu->arch.switch_db_regs & KVM_DEBUGREG_AUTO_SWITCH))) {
10980	11011	set_debugreg(0, 7);
10981	11012	set_debugreg(vcpu->arch.eff_db[0], 0);
10982	11013	set_debugreg(vcpu->arch.eff_db[1], 1);
@@ -11028,6 +11059,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 45 unmodified lines
11028	11059	*/
11029	11060	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
11030	11061	WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
	11062	+ WARN_ON(vcpu->arch.switch_db_regs & KVM_DEBUGREG_AUTO_SWITCH);
11031	11063	kvm_x86_call(sync_dirty_debug_regs)(vcpu);
11032	11064	kvm_update_dr0123(vcpu);
11033	11065	kvm_update_dr7(vcpu);
@@ -11131,7 +11163,7 @@ static bool kvm_vcpu_running(struct kvm_vcpu *vcpu) · 97 unmodified lines
11131	11163	!vcpu->arch.apf.halted);
11132	11164	}
11133	11165
11134		−static bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
	11166	+bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
11135	11167	{
11136	11168	if (!list_empty_careful(&vcpu->async_pf.done))
11137	11169	return true;
@@ -11140,9 +11172,6 @@ static bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) · 2 unmodified lines
11140	11172	kvm_apic_init_sipi_allowed(vcpu))
11141	11173	return true;
11142	11174
11143		− if (vcpu->arch.pv.pv_unhalted)
11144		− return true;
11145		−
11146	11175	if (kvm_is_exception_pending(vcpu))
11147	11176	return true;
11148	11177
@@ -11180,10 +11209,12 @@ static bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) · 31 unmodified lines
11180	11209
11181	11210	return false;
11182	11211	}
	11212	+EXPORT_SYMBOL_GPL(kvm_vcpu_has_events);
11183	11213
11184	11214	int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
11185	11215	{
11186		− return kvm_vcpu_running(vcpu) \|\| kvm_vcpu_has_events(vcpu);
	11216	+ return kvm_vcpu_running(vcpu) \|\| vcpu->arch.pv.pv_unhalted \|\|
	11217	+ kvm_vcpu_has_events(vcpu);
11187	11218	}
11188	11219
11189	11220	/* Called within kvm->srcu read side. */
@@ -11317,7 +11348,7 @@ static int __kvm_emulate_halt(struct kvm_vcpu *vcpu, int state, int reason) · 127 unmodified lines
11317	11348	*/
11318	11349	++vcpu->stat.halt_exits;
11319	11350	if (lapic_in_kernel(vcpu)) {
11320		− if (kvm_vcpu_has_events(vcpu))
	11351	+ if (kvm_vcpu_has_events(vcpu) \|\| vcpu->arch.pv.pv_unhalted)
11321	11352	state = KVM_MP_STATE_RUNNABLE;
11322	11353	kvm_set_mp_state(vcpu, state);
11323	11354	return 1;
@@ -12691,6 +12722,7 @@ bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu) · 1367 unmodified lines
12691	12722	{
12692	12723	return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;
12693	12724	}
	12725	+EXPORT_SYMBOL_GPL(kvm_vcpu_is_reset_bsp);
12694	12726
12695	12727	bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
12696	12728	{
@@ -12720,6 +12752,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 23 unmodified lines
12720	12752	/* Decided by the vendor code for other VM types. */
12721	12753	kvm->arch.pre_fault_allowed =
12722	12754	type == KVM_X86_DEFAULT_VM \|\| type == KVM_X86_SW_PROTECTED_VM;
	12755	+ kvm->arch.disabled_quirks = kvm_caps.inapplicable_quirks & kvm_caps.supported_quirks;
12723	12756
12724	12757	ret = kvm_page_track_init(kvm);
12725	12758	if (ret)
@@ -12873,6 +12906,7 @@ void kvm_arch_pre_destroy_vm(struct kvm *kvm) · 147 unmodified lines
12873	12906	kvm_free_pit(kvm);
12874	12907
12875	12908	kvm_mmu_pre_destroy_vm(kvm);
	12909	+ static_call_cond(kvm_x86_vm_pre_destroy)(kvm);
12876	12910	}
12877	12911
12878	12912	void kvm_arch_destroy_vm(struct kvm *kvm)
@@ -13070,7 +13104,7 @@ static void kvm_mmu_update_cpu_dirty_logging(struct kvm *kvm, bool enable) · 191 unmodified lines
13070	13104	{
13071	13105	int nr_slots;
13072	13106
13073		− if (!kvm_x86_ops.cpu_dirty_log_size)
	13107	+ if (!kvm->arch.cpu_dirty_log_size)
13074	13108	return;
13075	13109
13076	13110	nr_slots = atomic_read(&kvm->nr_memslots_dirty_logging);
@@ -13142,7 +13176,7 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, · 65 unmodified lines
13142	13176	if (READ_ONCE(eager_page_split))
13143	13177	kvm_mmu_slot_try_split_huge_pages(kvm, new, PG_LEVEL_4K);
13144	13178
13145		− if (kvm_x86_ops.cpu_dirty_log_size) {
	13179	+ if (kvm->arch.cpu_dirty_log_size) {
13146	13180	kvm_mmu_slot_leaf_clear_dirty(kvm, new);
13147	13181	kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_2M);
13148	13182	} else {
@@ -13531,8 +13565,10 @@ static void kvm_noncoherent_dma_assignment_start_or_stop(struct kvm *kvm) · 382 unmodified lines
13531	13565	* due to toggling the "ignore PAT" bit. Zap all SPTEs when the first
13532	13566	* (or last) non-coherent device is (un)registered to so that new SPTEs
13533	13567	* with the correct "ignore guest PAT" setting are created.
	13568	+ *
	13569	+ * If KVM always honors guest PAT, however, there is nothing to do.
13534	13570	*/
13535		− if (kvm_mmu_may_ignore_guest_pat())
	13571	+ if (kvm_check_has_quirk(kvm, KVM_X86_QUIRK_IGNORE_GUEST_PAT))
13536	13572	kvm_zap_gfn_range(kvm, gpa_to_gfn(0), gpa_to_gfn(~0ULL));
13537	13573	}
13538	13574
@@ -14000,6 +14036,7 @@ EXPORT_SYMBOL_GPL(kvm_sev_es_string_io); · 461 unmodified lines
14000	14036
14001	14037	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_entry);
14002	14038	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
	14039	+EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_mmio);
14003	14040	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_fast_mmio);
14004	14041	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
14005	14042	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);

arch/x86/kvm/x86.h+14−17

@@ -10,6 +10,8 @@ · 9 unmodified lines
10	10	#include "kvm_emulate.h"
11	11	#include "cpuid.h"
12	12
	13	+#define KVM_MAX_MCE_BANKS 32
	14	+
13	15	struct kvm_caps {
14	16	/* control of guest tsc rate supported? */
15	17	bool has_tsc_control;
@@ -32,6 +34,9 @@ struct kvm_caps { · 16 unmodified lines
32	34	u64 supported_xcr0;
33	35	u64 supported_xss;
34	36	u64 supported_perf_cap;
	37	+
	38	+ u64 supported_quirks;
	39	+ u64 inapplicable_quirks;
35	40	};
36	41
37	42	struct kvm_host_values {
@@ -629,25 +634,17 @@ static inline bool user_exit_on_hypercall(struct kvm *kvm, unsigned long hc_nr) · 591 unmodified lines
629	634	return kvm->arch.hypercall_exit_enabled & BIT(hc_nr);
630	635	}
631	636
632		−int ____kvm_emulate_hypercall(struct kvm_vcpu *vcpu, unsigned long nr,
633		− unsigned long a0, unsigned long a1,
634		− unsigned long a2, unsigned long a3,
635		− int op_64_bit, int cpl,
	637	+int ____kvm_emulate_hypercall(struct kvm_vcpu *vcpu, int cpl,
636	638	int (complete_hypercall)(struct kvm_vcpu ));
637	639
638		−#define __kvm_emulate_hypercall(_vcpu, nr, a0, a1, a2, a3, op_64_bit, cpl, complete_hypercall) \
639		−({ \
640		− int __ret; \
641		− \
642		− __ret = ____kvm_emulate_hypercall(_vcpu, \
643		− kvm_##nr##_read(_vcpu), kvm_##a0##_read(_vcpu), \
644		− kvm_##a1##_read(_vcpu), kvm_##a2##_read(_vcpu), \
645		− kvm_##a3##_read(_vcpu), op_64_bit, cpl, \
646		− complete_hypercall); \
647		− \
648		− if (__ret > 0) \
649		− __ret = complete_hypercall(_vcpu); \
650		− __ret; \
	640	+#define __kvm_emulate_hypercall(_vcpu, cpl, complete_hypercall) \
	641	+({ \
	642	+ int __ret; \
	643	+ __ret = ____kvm_emulate_hypercall(_vcpu, cpl, complete_hypercall); \
	644	+ \
	645	+ if (__ret > 0) \
	646	+ __ret = complete_hypercall(_vcpu); \
	647	+ __ret; \
651	648	})
652	649
653	650	int kvm_emulate_hypercall(struct kvm_vcpu *vcpu);

arch/x86/virt/vmx/tdx/seamcall.S+3−0

@@ -41,6 +41,9 @@ SYM_FUNC_START(__seamcall_ret) · 40 unmodified lines
41	41	TDX_MODULE_CALL host=1 ret=1
42	42	SYM_FUNC_END(__seamcall_ret)
43	43
	44	+/* KVM requires non-instrumentable __seamcall_saved_ret() for TDH.VP.ENTER */
	45	+.section .noinstr.text, "ax"
	46	+
44	47	/*
45	48	* __seamcall_saved_ret() - Host-side interface functions to SEAM software
46	49	* (the P-SEAMLDR or the TDX module), with saving output registers to the

arch/x86/virt/vmx/tdx/tdx.c+418−5

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	5	* Intel Trusted Domain Extensions (TDX) support
6	6	*/
7	7
	8	+#include "asm/page_types.h"
8	9	#define pr_fmt(fmt) "virt/tdx: " fmt
9	10
10	11	#include <linux/types.h>
@@ -27,6 +28,7 @@ · 16 unmodified lines
27	28	#include <linux/log2.h>
28	29	#include <linux/acpi.h>
29	30	#include <linux/suspend.h>
	31	+#include <linux/idr.h>
30	32	#include <asm/page.h>
31	33	#include <asm/special_insns.h>
32	34	#include <asm/msr-index.h>
@@ -42,6 +44,8 @@ static u32 tdx_global_keyid __ro_after_init; · 9 unmodified lines
42	44	static u32 tdx_guest_keyid_start __ro_after_init;
43	45	static u32 tdx_nr_guest_keyids __ro_after_init;
44	46
	47	+static DEFINE_IDA(tdx_guest_keyid_pool);
	48	+
45	49	static DEFINE_PER_CPU(bool, tdx_lp_initialized);
46	50
47	51	static struct tdmr_info_list tdx_tdmr_list;
@@ -52,6 +56,8 @@ static DEFINE_MUTEX(tdx_module_lock); · 4 unmodified lines
52	56	/* All TDX-usable memory regions. Protected by mem_hotplug_lock. */
53	57	static LIST_HEAD(tdx_memlist);
54	58
	59	+static struct tdx_sys_info tdx_sysinfo;
	60	+
55	61	typedef void (sc_err_func_t)(u64 fn, u64 err, struct tdx_module_args args);
56	62
57	63	static inline void seamcall_err(u64 fn, u64 err, struct tdx_module_args *args)
@@ -1060,15 +1066,14 @@ static int init_tdmrs(struct tdmr_info_list *tdmr_list) · 1002 unmodified lines
1060	1066
1061	1067	static int init_tdx_module(void)
1062	1068	{
1063		− struct tdx_sys_info sysinfo;
1064	1069	int ret;
1065	1070
1066		− ret = get_tdx_sys_info(&sysinfo);
	1071	+ ret = get_tdx_sys_info(&tdx_sysinfo);
1067	1072	if (ret)
1068	1073	return ret;
1069	1074
1070	1075	/* Check whether the kernel can support this module */
1071		− ret = check_features(&sysinfo);
	1076	+ ret = check_features(&tdx_sysinfo);
1072	1077	if (ret)
1073	1078	return ret;
1074	1079
@@ -1089,12 +1094,12 @@ static int init_tdx_module(void) · 14 unmodified lines
1089	1094	goto out_put_tdxmem;
1090	1095
1091	1096	/* Allocate enough space for constructing TDMRs */
1092		− ret = alloc_tdmr_list(&tdx_tdmr_list, &sysinfo.tdmr);
	1097	+ ret = alloc_tdmr_list(&tdx_tdmr_list, &tdx_sysinfo.tdmr);
1093	1098	if (ret)
1094	1099	goto err_free_tdxmem;
1095	1100
1096	1101	/* Cover all TDX-usable memory regions in TDMRs */
1097		− ret = construct_tdmrs(&tdx_memlist, &tdx_tdmr_list, &sysinfo.tdmr);
	1102	+ ret = construct_tdmrs(&tdx_memlist, &tdx_tdmr_list, &tdx_sysinfo.tdmr);
1098	1103	if (ret)
1099	1104	goto err_free_tdmrs;
1100	1105
@@ -1456,3 +1461,411 @@ void __init tdx_init(void) · 355 unmodified lines
1456	1461
1457	1462	check_tdx_erratum();
1458	1463	}
	1464	+
	1465	+const struct tdx_sys_info *tdx_get_sysinfo(void)
	1466	+{
	1467	+ const struct tdx_sys_info *p = NULL;
	1468	+
	1469	+ /* Make sure all fields in @tdx_sysinfo have been populated */
	1470	+ mutex_lock(&tdx_module_lock);
	1471	+ if (tdx_module_status == TDX_MODULE_INITIALIZED)
	1472	+ p = (const struct tdx_sys_info *)&tdx_sysinfo;
	1473	+ mutex_unlock(&tdx_module_lock);
	1474	+
	1475	+ return p;
	1476	+}
	1477	+EXPORT_SYMBOL_GPL(tdx_get_sysinfo);
	1478	+
	1479	+u32 tdx_get_nr_guest_keyids(void)
	1480	+{
	1481	+ return tdx_nr_guest_keyids;
	1482	+}
	1483	+EXPORT_SYMBOL_GPL(tdx_get_nr_guest_keyids);
	1484	+
	1485	+int tdx_guest_keyid_alloc(void)
	1486	+{
	1487	+ return ida_alloc_range(&tdx_guest_keyid_pool, tdx_guest_keyid_start,
	1488	+ tdx_guest_keyid_start + tdx_nr_guest_keyids - 1,
	1489	+ GFP_KERNEL);
	1490	+}
	1491	+EXPORT_SYMBOL_GPL(tdx_guest_keyid_alloc);
	1492	+
	1493	+void tdx_guest_keyid_free(unsigned int keyid)
	1494	+{
	1495	+ ida_free(&tdx_guest_keyid_pool, keyid);
	1496	+}
	1497	+EXPORT_SYMBOL_GPL(tdx_guest_keyid_free);
	1498	+
	1499	+static inline u64 tdx_tdr_pa(struct tdx_td *td)
	1500	+{
	1501	+ return page_to_phys(td->tdr_page);
	1502	+}
	1503	+
	1504	+static inline u64 tdx_tdvpr_pa(struct tdx_vp *td)
	1505	+{
	1506	+ return page_to_phys(td->tdvpr_page);
	1507	+}
	1508	+
	1509	+/*
	1510	+ * The TDX module exposes a CLFLUSH_BEFORE_ALLOC bit to specify whether
	1511	+ * a CLFLUSH of pages is required before handing them to the TDX module.
	1512	+ * Be conservative and make the code simpler by doing the CLFLUSH
	1513	+ * unconditionally.
	1514	+ */
	1515	+static void tdx_clflush_page(struct page *page)
	1516	+{
	1517	+ clflush_cache_range(page_to_virt(page), PAGE_SIZE);
	1518	+}
	1519	+
	1520	+noinstr u64 tdh_vp_enter(struct tdx_vp td, struct tdx_module_args args)
	1521	+{
	1522	+ args->rcx = tdx_tdvpr_pa(td);
	1523	+
	1524	+ return __seamcall_saved_ret(TDH_VP_ENTER, args);
	1525	+}
	1526	+EXPORT_SYMBOL_GPL(tdh_vp_enter);
	1527	+
	1528	+u64 tdh_mng_addcx(struct tdx_td td, struct page tdcs_page)
	1529	+{
	1530	+ struct tdx_module_args args = {
	1531	+ .rcx = page_to_phys(tdcs_page),
	1532	+ .rdx = tdx_tdr_pa(td),
	1533	+ };
	1534	+
	1535	+ tdx_clflush_page(tdcs_page);
	1536	+ return seamcall(TDH_MNG_ADDCX, &args);
	1537	+}
	1538	+EXPORT_SYMBOL_GPL(tdh_mng_addcx);
	1539	+
	1540	+u64 tdh_mem_page_add(struct tdx_td td, u64 gpa, struct page page, struct page source, u64 ext_err1, u64 *ext_err2)
	1541	+{
	1542	+ struct tdx_module_args args = {
	1543	+ .rcx = gpa,
	1544	+ .rdx = tdx_tdr_pa(td),
	1545	+ .r8 = page_to_phys(page),
	1546	+ .r9 = page_to_phys(source),
	1547	+ };
	1548	+ u64 ret;
	1549	+
	1550	+ tdx_clflush_page(page);
	1551	+ ret = seamcall_ret(TDH_MEM_PAGE_ADD, &args);
	1552	+
	1553	+ *ext_err1 = args.rcx;
	1554	+ *ext_err2 = args.rdx;
	1555	+
	1556	+ return ret;
	1557	+}
	1558	+EXPORT_SYMBOL_GPL(tdh_mem_page_add);
	1559	+
	1560	+u64 tdh_mem_sept_add(struct tdx_td td, u64 gpa, int level, struct page page, u64 ext_err1, u64 ext_err2)
	1561	+{
	1562	+ struct tdx_module_args args = {
	1563	+ .rcx = gpa \| level,
	1564	+ .rdx = tdx_tdr_pa(td),
	1565	+ .r8 = page_to_phys(page),
	1566	+ };
	1567	+ u64 ret;
	1568	+
	1569	+ tdx_clflush_page(page);
	1570	+ ret = seamcall_ret(TDH_MEM_SEPT_ADD, &args);
	1571	+
	1572	+ *ext_err1 = args.rcx;
	1573	+ *ext_err2 = args.rdx;
	1574	+
	1575	+ return ret;
	1576	+}
	1577	+EXPORT_SYMBOL_GPL(tdh_mem_sept_add);
	1578	+
	1579	+u64 tdh_vp_addcx(struct tdx_vp vp, struct page tdcx_page)
	1580	+{
	1581	+ struct tdx_module_args args = {
	1582	+ .rcx = page_to_phys(tdcx_page),
	1583	+ .rdx = tdx_tdvpr_pa(vp),
	1584	+ };
	1585	+
	1586	+ tdx_clflush_page(tdcx_page);
	1587	+ return seamcall(TDH_VP_ADDCX, &args);
	1588	+}
	1589	+EXPORT_SYMBOL_GPL(tdh_vp_addcx);
	1590	+
	1591	+u64 tdh_mem_page_aug(struct tdx_td td, u64 gpa, int level, struct page page, u64 ext_err1, u64 ext_err2)
	1592	+{
	1593	+ struct tdx_module_args args = {
	1594	+ .rcx = gpa \| level,
	1595	+ .rdx = tdx_tdr_pa(td),
	1596	+ .r8 = page_to_phys(page),
	1597	+ };
	1598	+ u64 ret;
	1599	+
	1600	+ tdx_clflush_page(page);
	1601	+ ret = seamcall_ret(TDH_MEM_PAGE_AUG, &args);
	1602	+
	1603	+ *ext_err1 = args.rcx;
	1604	+ *ext_err2 = args.rdx;
	1605	+
	1606	+ return ret;
	1607	+}
	1608	+EXPORT_SYMBOL_GPL(tdh_mem_page_aug);
	1609	+
	1610	+u64 tdh_mem_range_block(struct tdx_td td, u64 gpa, int level, u64 ext_err1, u64 *ext_err2)
	1611	+{
	1612	+ struct tdx_module_args args = {
	1613	+ .rcx = gpa \| level,
	1614	+ .rdx = tdx_tdr_pa(td),
	1615	+ };
	1616	+ u64 ret;
	1617	+
	1618	+ ret = seamcall_ret(TDH_MEM_RANGE_BLOCK, &args);
	1619	+
	1620	+ *ext_err1 = args.rcx;
	1621	+ *ext_err2 = args.rdx;
	1622	+
	1623	+ return ret;
	1624	+}
	1625	+EXPORT_SYMBOL_GPL(tdh_mem_range_block);
	1626	+
	1627	+u64 tdh_mng_key_config(struct tdx_td *td)
	1628	+{
	1629	+ struct tdx_module_args args = {
	1630	+ .rcx = tdx_tdr_pa(td),
	1631	+ };
	1632	+
	1633	+ return seamcall(TDH_MNG_KEY_CONFIG, &args);
	1634	+}
	1635	+EXPORT_SYMBOL_GPL(tdh_mng_key_config);
	1636	+
	1637	+u64 tdh_mng_create(struct tdx_td *td, u16 hkid)
	1638	+{
	1639	+ struct tdx_module_args args = {
	1640	+ .rcx = tdx_tdr_pa(td),
	1641	+ .rdx = hkid,
	1642	+ };
	1643	+
	1644	+ tdx_clflush_page(td->tdr_page);
	1645	+ return seamcall(TDH_MNG_CREATE, &args);
	1646	+}
	1647	+EXPORT_SYMBOL_GPL(tdh_mng_create);
	1648	+
	1649	+u64 tdh_vp_create(struct tdx_td td, struct tdx_vp vp)
	1650	+{
	1651	+ struct tdx_module_args args = {
	1652	+ .rcx = tdx_tdvpr_pa(vp),
	1653	+ .rdx = tdx_tdr_pa(td),
	1654	+ };
	1655	+
	1656	+ tdx_clflush_page(vp->tdvpr_page);
	1657	+ return seamcall(TDH_VP_CREATE, &args);
	1658	+}
	1659	+EXPORT_SYMBOL_GPL(tdh_vp_create);
	1660	+
	1661	+u64 tdh_mng_rd(struct tdx_td td, u64 field, u64 data)
	1662	+{
	1663	+ struct tdx_module_args args = {
	1664	+ .rcx = tdx_tdr_pa(td),
	1665	+ .rdx = field,
	1666	+ };
	1667	+ u64 ret;
	1668	+
	1669	+ ret = seamcall_ret(TDH_MNG_RD, &args);
	1670	+
	1671	+ /* R8: Content of the field, or 0 in case of error. */
	1672	+ *data = args.r8;
	1673	+
	1674	+ return ret;
	1675	+}
	1676	+EXPORT_SYMBOL_GPL(tdh_mng_rd);
	1677	+
	1678	+u64 tdh_mr_extend(struct tdx_td td, u64 gpa, u64 ext_err1, u64 *ext_err2)
	1679	+{
	1680	+ struct tdx_module_args args = {
	1681	+ .rcx = gpa,
	1682	+ .rdx = tdx_tdr_pa(td),
	1683	+ };
	1684	+ u64 ret;
	1685	+
	1686	+ ret = seamcall_ret(TDH_MR_EXTEND, &args);
	1687	+
	1688	+ *ext_err1 = args.rcx;
	1689	+ *ext_err2 = args.rdx;
	1690	+
	1691	+ return ret;
	1692	+}
	1693	+EXPORT_SYMBOL_GPL(tdh_mr_extend);
	1694	+
	1695	+u64 tdh_mr_finalize(struct tdx_td *td)
	1696	+{
	1697	+ struct tdx_module_args args = {
	1698	+ .rcx = tdx_tdr_pa(td),
	1699	+ };
	1700	+
	1701	+ return seamcall(TDH_MR_FINALIZE, &args);
	1702	+}
	1703	+EXPORT_SYMBOL_GPL(tdh_mr_finalize);
	1704	+
	1705	+u64 tdh_vp_flush(struct tdx_vp *vp)
	1706	+{
	1707	+ struct tdx_module_args args = {
	1708	+ .rcx = tdx_tdvpr_pa(vp),
	1709	+ };
	1710	+
	1711	+ return seamcall(TDH_VP_FLUSH, &args);
	1712	+}
	1713	+EXPORT_SYMBOL_GPL(tdh_vp_flush);
	1714	+
	1715	+u64 tdh_mng_vpflushdone(struct tdx_td *td)
	1716	+{
	1717	+ struct tdx_module_args args = {
	1718	+ .rcx = tdx_tdr_pa(td),
	1719	+ };
	1720	+
	1721	+ return seamcall(TDH_MNG_VPFLUSHDONE, &args);
	1722	+}
	1723	+EXPORT_SYMBOL_GPL(tdh_mng_vpflushdone);
	1724	+
	1725	+u64 tdh_mng_key_freeid(struct tdx_td *td)
	1726	+{
	1727	+ struct tdx_module_args args = {
	1728	+ .rcx = tdx_tdr_pa(td),
	1729	+ };
	1730	+
	1731	+ return seamcall(TDH_MNG_KEY_FREEID, &args);
	1732	+}
	1733	+EXPORT_SYMBOL_GPL(tdh_mng_key_freeid);
	1734	+
	1735	+u64 tdh_mng_init(struct tdx_td td, u64 td_params, u64 extended_err)
	1736	+{
	1737	+ struct tdx_module_args args = {
	1738	+ .rcx = tdx_tdr_pa(td),
	1739	+ .rdx = td_params,
	1740	+ };
	1741	+ u64 ret;
	1742	+
	1743	+ ret = seamcall_ret(TDH_MNG_INIT, &args);
	1744	+
	1745	+ *extended_err = args.rcx;
	1746	+
	1747	+ return ret;
	1748	+}
	1749	+EXPORT_SYMBOL_GPL(tdh_mng_init);
	1750	+
	1751	+u64 tdh_vp_rd(struct tdx_vp vp, u64 field, u64 data)
	1752	+{
	1753	+ struct tdx_module_args args = {
	1754	+ .rcx = tdx_tdvpr_pa(vp),
	1755	+ .rdx = field,
	1756	+ };
	1757	+ u64 ret;
	1758	+
	1759	+ ret = seamcall_ret(TDH_VP_RD, &args);
	1760	+
	1761	+ /* R8: Content of the field, or 0 in case of error. */
	1762	+ *data = args.r8;
	1763	+
	1764	+ return ret;
	1765	+}
	1766	+EXPORT_SYMBOL_GPL(tdh_vp_rd);
	1767	+
	1768	+u64 tdh_vp_wr(struct tdx_vp *vp, u64 field, u64 data, u64 mask)
	1769	+{
	1770	+ struct tdx_module_args args = {
	1771	+ .rcx = tdx_tdvpr_pa(vp),
	1772	+ .rdx = field,
	1773	+ .r8 = data,
	1774	+ .r9 = mask,
	1775	+ };
	1776	+
	1777	+ return seamcall(TDH_VP_WR, &args);
	1778	+}
	1779	+EXPORT_SYMBOL_GPL(tdh_vp_wr);
	1780	+
	1781	+u64 tdh_vp_init(struct tdx_vp *vp, u64 initial_rcx, u32 x2apicid)
	1782	+{
	1783	+ struct tdx_module_args args = {
	1784	+ .rcx = tdx_tdvpr_pa(vp),
	1785	+ .rdx = initial_rcx,
	1786	+ .r8 = x2apicid,
	1787	+ };
	1788	+
	1789	+ /* apicid requires version == 1. */
	1790	+ return seamcall(TDH_VP_INIT \| (1ULL << TDX_VERSION_SHIFT), &args);
	1791	+}
	1792	+EXPORT_SYMBOL_GPL(tdh_vp_init);
	1793	+
	1794	+/*
	1795	+ * TDX ABI defines output operands as PT, OWNER and SIZE. These are TDX defined fomats.
	1796	+ * So despite the names, they must be interpted specially as described by the spec. Return
	1797	+ * them only for error reporting purposes.
	1798	+ */
	1799	+u64 tdh_phymem_page_reclaim(struct page page, u64 tdx_pt, u64 tdx_owner, u64 tdx_size)
	1800	+{
	1801	+ struct tdx_module_args args = {
	1802	+ .rcx = page_to_phys(page),
	1803	+ };
	1804	+ u64 ret;
	1805	+
	1806	+ ret = seamcall_ret(TDH_PHYMEM_PAGE_RECLAIM, &args);
	1807	+
	1808	+ *tdx_pt = args.rcx;
	1809	+ *tdx_owner = args.rdx;
	1810	+ *tdx_size = args.r8;
	1811	+
	1812	+ return ret;
	1813	+}
	1814	+EXPORT_SYMBOL_GPL(tdh_phymem_page_reclaim);
	1815	+
	1816	+u64 tdh_mem_track(struct tdx_td *td)
	1817	+{
	1818	+ struct tdx_module_args args = {
	1819	+ .rcx = tdx_tdr_pa(td),
	1820	+ };
	1821	+
	1822	+ return seamcall(TDH_MEM_TRACK, &args);
	1823	+}
	1824	+EXPORT_SYMBOL_GPL(tdh_mem_track);
	1825	+
	1826	+u64 tdh_mem_page_remove(struct tdx_td td, u64 gpa, u64 level, u64 ext_err1, u64 *ext_err2)
	1827	+{
	1828	+ struct tdx_module_args args = {
	1829	+ .rcx = gpa \| level,
	1830	+ .rdx = tdx_tdr_pa(td),
	1831	+ };
	1832	+ u64 ret;
	1833	+
	1834	+ ret = seamcall_ret(TDH_MEM_PAGE_REMOVE, &args);
	1835	+
	1836	+ *ext_err1 = args.rcx;
	1837	+ *ext_err2 = args.rdx;
	1838	+
	1839	+ return ret;
	1840	+}
	1841	+EXPORT_SYMBOL_GPL(tdh_mem_page_remove);
	1842	+
	1843	+u64 tdh_phymem_cache_wb(bool resume)
	1844	+{
	1845	+ struct tdx_module_args args = {
	1846	+ .rcx = resume ? 1 : 0,
	1847	+ };
	1848	+
	1849	+ return seamcall(TDH_PHYMEM_CACHE_WB, &args);
	1850	+}
	1851	+EXPORT_SYMBOL_GPL(tdh_phymem_cache_wb);
	1852	+
	1853	+u64 tdh_phymem_page_wbinvd_tdr(struct tdx_td *td)
	1854	+{
	1855	+ struct tdx_module_args args = {};
	1856	+
	1857	+ args.rcx = mk_keyed_paddr(tdx_global_keyid, td->tdr_page);
	1858	+
	1859	+ return seamcall(TDH_PHYMEM_PAGE_WBINVD, &args);
	1860	+}
	1861	+EXPORT_SYMBOL_GPL(tdh_phymem_page_wbinvd_tdr);
	1862	+
	1863	+u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, struct page *page)
	1864	+{
	1865	+ struct tdx_module_args args = {};
	1866	+
	1867	+ args.rcx = mk_keyed_paddr(hkid, page);
	1868	+
	1869	+ return seamcall(TDH_PHYMEM_PAGE_WBINVD, &args);
	1870	+}
	1871	+EXPORT_SYMBOL_GPL(tdh_phymem_page_wbinvd_hkid);

arch/x86/virt/vmx/tdx/tdx.h+40−8

@@ -3,7 +3,6 @@ · 2 unmodified lines
3	3	#define _X86_VIRT_TDX_H
4	4
5	5	#include <linux/bits.h>
6		−#include "tdx_global_metadata.h"
7	6
8	7	/*
9	8	* This file contains both macros and data structures defined by the TDX
@@ -15,13 +14,46 @@ · 5 unmodified lines
15	14	/*
16	15	* TDX module SEAMCALL leaf functions
17	16	*/
18		−#define TDH_PHYMEM_PAGE_RDMD 24
19		−#define TDH_SYS_KEY_CONFIG 31
20		−#define TDH_SYS_INIT 33
21		−#define TDH_SYS_RD 34
22		−#define TDH_SYS_LP_INIT 35
23		−#define TDH_SYS_TDMR_INIT 36
24		−#define TDH_SYS_CONFIG 45
	17	+#define TDH_VP_ENTER 0
	18	+#define TDH_MNG_ADDCX 1
	19	+#define TDH_MEM_PAGE_ADD 2
	20	+#define TDH_MEM_SEPT_ADD 3
	21	+#define TDH_VP_ADDCX 4
	22	+#define TDH_MEM_PAGE_AUG 6
	23	+#define TDH_MEM_RANGE_BLOCK 7
	24	+#define TDH_MNG_KEY_CONFIG 8
	25	+#define TDH_MNG_CREATE 9
	26	+#define TDH_MNG_RD 11
	27	+#define TDH_MR_EXTEND 16
	28	+#define TDH_MR_FINALIZE 17
	29	+#define TDH_VP_FLUSH 18
	30	+#define TDH_MNG_VPFLUSHDONE 19
	31	+#define TDH_VP_CREATE 10
	32	+#define TDH_MNG_KEY_FREEID 20
	33	+#define TDH_MNG_INIT 21
	34	+#define TDH_VP_INIT 22
	35	+#define TDH_PHYMEM_PAGE_RDMD 24
	36	+#define TDH_VP_RD 26
	37	+#define TDH_PHYMEM_PAGE_RECLAIM 28
	38	+#define TDH_MEM_PAGE_REMOVE 29
	39	+#define TDH_SYS_KEY_CONFIG 31
	40	+#define TDH_SYS_INIT 33
	41	+#define TDH_SYS_RD 34
	42	+#define TDH_SYS_LP_INIT 35
	43	+#define TDH_SYS_TDMR_INIT 36
	44	+#define TDH_MEM_TRACK 38
	45	+#define TDH_PHYMEM_CACHE_WB 40
	46	+#define TDH_PHYMEM_PAGE_WBINVD 41
	47	+#define TDH_VP_WR 43
	48	+#define TDH_SYS_CONFIG 45
	49	+
	50	+/*
	51	+ * SEAMCALL leaf:
	52	+ *
	53	+ * Bit 15:0 Leaf number
	54	+ * Bit 23:16 Version number
	55	+ */
	56	+#define TDX_VERSION_SHIFT 16
25	57
26	58	/* TDX page types */
27	59	#define PT_NDA 0x0

arch/x86/virt/vmx/tdx/tdx_global_metadata.c+50−0

@@ -37,12 +37,62 @@ static int get_tdx_sys_info_tdmr(struct tdx_sys_info_tdmr *sysinfo_tdmr) · 36 unmodified lines
37	37	return ret;
38	38	}
39	39
	40	+static int get_tdx_sys_info_td_ctrl(struct tdx_sys_info_td_ctrl *sysinfo_td_ctrl)
	41	+{
	42	+ int ret = 0;
	43	+ u64 val;
	44	+
	45	+ if (!ret && !(ret = read_sys_metadata_field(0x9800000100000000, &val)))
	46	+ sysinfo_td_ctrl->tdr_base_size = val;
	47	+ if (!ret && !(ret = read_sys_metadata_field(0x9800000100000100, &val)))
	48	+ sysinfo_td_ctrl->tdcs_base_size = val;
	49	+ if (!ret && !(ret = read_sys_metadata_field(0x9800000100000200, &val)))
	50	+ sysinfo_td_ctrl->tdvps_base_size = val;
	51	+
	52	+ return ret;
	53	+}
	54	+
	55	+static int get_tdx_sys_info_td_conf(struct tdx_sys_info_td_conf *sysinfo_td_conf)
	56	+{
	57	+ int ret = 0;
	58	+ u64 val;
	59	+ int i, j;
	60	+
	61	+ if (!ret && !(ret = read_sys_metadata_field(0x1900000300000000, &val)))
	62	+ sysinfo_td_conf->attributes_fixed0 = val;
	63	+ if (!ret && !(ret = read_sys_metadata_field(0x1900000300000001, &val)))
	64	+ sysinfo_td_conf->attributes_fixed1 = val;
	65	+ if (!ret && !(ret = read_sys_metadata_field(0x1900000300000002, &val)))
	66	+ sysinfo_td_conf->xfam_fixed0 = val;
	67	+ if (!ret && !(ret = read_sys_metadata_field(0x1900000300000003, &val)))
	68	+ sysinfo_td_conf->xfam_fixed1 = val;
	69	+ if (!ret && !(ret = read_sys_metadata_field(0x9900000100000004, &val)))
	70	+ sysinfo_td_conf->num_cpuid_config = val;
	71	+ if (!ret && !(ret = read_sys_metadata_field(0x9900000100000008, &val)))
	72	+ sysinfo_td_conf->max_vcpus_per_td = val;
	73	+ if (sysinfo_td_conf->num_cpuid_config > ARRAY_SIZE(sysinfo_td_conf->cpuid_config_leaves))
	74	+ return -EINVAL;
	75	+ for (i = 0; i < sysinfo_td_conf->num_cpuid_config; i++)
	76	+ if (!ret && !(ret = read_sys_metadata_field(0x9900000300000400 + i, &val)))
	77	+ sysinfo_td_conf->cpuid_config_leaves[i] = val;
	78	+ if (sysinfo_td_conf->num_cpuid_config > ARRAY_SIZE(sysinfo_td_conf->cpuid_config_values))
	79	+ return -EINVAL;
	80	+ for (i = 0; i < sysinfo_td_conf->num_cpuid_config; i++)
	81	+ for (j = 0; j < 2; j++)
	82	+ if (!ret && !(ret = read_sys_metadata_field(0x9900000300000500 + i * 2 + j, &val)))
	83	+ sysinfo_td_conf->cpuid_config_values[i][j] = val;
	84	+
	85	+ return ret;
	86	+}
	87	+
40	88	static int get_tdx_sys_info(struct tdx_sys_info *sysinfo)
41	89	{
42	90	int ret = 0;
43	91
44	92	ret = ret ?: get_tdx_sys_info_features(&sysinfo->features);
45	93	ret = ret ?: get_tdx_sys_info_tdmr(&sysinfo->tdmr);
	94	+ ret = ret ?: get_tdx_sys_info_td_ctrl(&sysinfo->td_ctrl);
	95	+ ret = ret ?: get_tdx_sys_info_td_conf(&sysinfo->td_conf);
46	96
47	97	return ret;
48	98	}

include/linux/kvm_dirty_ring.h+6−5

@@ -32,7 +32,7 @@ struct kvm_dirty_ring { · 31 unmodified lines
32	32	* If CONFIG_HAVE_HVM_DIRTY_RING not defined, kvm_dirty_ring.o should
33	33	* not be included as well, so define these nop functions for the arch.
34	34	*/
35		−static inline u32 kvm_dirty_ring_get_rsvd_entries(void)
	35	+static inline u32 kvm_dirty_ring_get_rsvd_entries(struct kvm *kvm)
36	36	{
37	37	return 0;
38	38	}
@@ -42,7 +42,7 @@ static inline bool kvm_use_dirty_bitmap(struct kvm *kvm) · 3 unmodified lines
42	42	return true;
43	43	}
44	44
45		−static inline int kvm_dirty_ring_alloc(struct kvm_dirty_ring *ring,
	45	+static inline int kvm_dirty_ring_alloc(struct kvm kvm, struct kvm_dirty_ring ring,
46	46	int index, u32 size)
47	47	{
48	48	return 0;
@@ -71,11 +71,12 @@ static inline void kvm_dirty_ring_free(struct kvm_dirty_ring *ring) · 22 unmodified lines
71	71
72	72	#else /* CONFIG_HAVE_KVM_DIRTY_RING */
73	73
74		−int kvm_cpu_dirty_log_size(void);
	74	+int kvm_cpu_dirty_log_size(struct kvm *kvm);
75	75	bool kvm_use_dirty_bitmap(struct kvm *kvm);
76	76	bool kvm_arch_allow_write_without_running_vcpu(struct kvm *kvm);
77		−u32 kvm_dirty_ring_get_rsvd_entries(void);
78		−int kvm_dirty_ring_alloc(struct kvm_dirty_ring *ring, int index, u32 size);
	77	+u32 kvm_dirty_ring_get_rsvd_entries(struct kvm *kvm);
	78	+int kvm_dirty_ring_alloc(struct kvm kvm, struct kvm_dirty_ring ring,
	79	+ int index, u32 size);
79	80
80	81	/*
81	82	* called with kvm->slots_lock held, returns the number of

include/linux/kvm_host.h+10−0

@@ -1610,6 +1610,7 @@ void kvm_arch_disable_virtualization(void); · 1609 unmodified lines
1610	1610	int kvm_arch_enable_virtualization_cpu(void);
1611	1611	void kvm_arch_disable_virtualization_cpu(void);
1612	1612	#endif
	1613	+bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu);
1613	1614	int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu);
1614	1615	bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu);
1615	1616	int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu);
@@ -2284,6 +2285,7 @@ static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu) · 668 unmodified lines
2284	2285	}
2285	2286
2286	2287	#ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING
	2288	+extern bool enable_virt_at_load;
2287	2289	extern bool kvm_rebooting;
2288	2290	#endif
2289	2291
@@ -2571,4 +2573,12 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, · 281 unmodified lines
2571	2573	struct kvm_pre_fault_memory *range);
2572	2574	#endif
2573	2575
	2576	+#ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING
	2577	+int kvm_enable_virtualization(void);
	2578	+void kvm_disable_virtualization(void);
	2579	+#else
	2580	+static inline int kvm_enable_virtualization(void) { return 0; }
	2581	+static inline void kvm_disable_virtualization(void) { }
	2582	+#endif
	2583	+
2574	2584	#endif

include/linux/misc_cgroup.h+4−0

@@ -17,6 +17,10 @@ enum misc_res_type { · 16 unmodified lines
17	17	MISC_CG_RES_SEV,
18	18	/** @MISC_CG_RES_SEV_ES: AMD SEV-ES ASIDs resource */
19	19	MISC_CG_RES_SEV_ES,
	20	+#endif
	21	+#ifdef CONFIG_INTEL_TDX_HOST
	22	+ /* Intel TDX HKIDs resource */
	23	+ MISC_CG_RES_TDX,
20	24	#endif
21	25	/** @MISC_CG_RES_TYPES: count of enum misc_res_type constants */
22	26	MISC_CG_RES_TYPES

include/uapi/linux/kvm.h+1−0

@@ -375,6 +375,7 @@ struct kvm_run { · 374 unmodified lines
375	375	#define KVM_SYSTEM_EVENT_WAKEUP 4
376	376	#define KVM_SYSTEM_EVENT_SUSPEND 5
377	377	#define KVM_SYSTEM_EVENT_SEV_TERM 6
	378	+#define KVM_SYSTEM_EVENT_TDX_FATAL 7
378	379	__u32 type;
379	380	__u32 ndata;
380	381	union {

kernel/cgroup/misc.c+4−0

@@ -24,6 +24,10 @@ static const char *const misc_res_name[] = { · 23 unmodified lines
24	24	/* AMD SEV-ES ASIDs resource */
25	25	"sev_es",
26	26	#endif
	27	+#ifdef CONFIG_INTEL_TDX_HOST
	28	+ /* Intel TDX HKIDs resource */
	29	+ "tdx",
	30	+#endif
27	31	};
28	32
29	33	/* Root misc cgroup */

virt/kvm/dirty_ring.c+6−5

@@ -11,14 +11,14 @@ · 10 unmodified lines
11	11	#include <trace/events/kvm.h>
12	12	#include "kvm_mm.h"
13	13
14		−int __weak kvm_cpu_dirty_log_size(void)
	14	+int __weak kvm_cpu_dirty_log_size(struct kvm *kvm)
15	15	{
16	16	return 0;
17	17	}
18	18
19		−u32 kvm_dirty_ring_get_rsvd_entries(void)
	19	+u32 kvm_dirty_ring_get_rsvd_entries(struct kvm *kvm)
20	20	{
21		− return KVM_DIRTY_RING_RSVD_ENTRIES + kvm_cpu_dirty_log_size();
	21	+ return KVM_DIRTY_RING_RSVD_ENTRIES + kvm_cpu_dirty_log_size(kvm);
22	22	}
23	23
24	24	bool kvm_use_dirty_bitmap(struct kvm *kvm)
@@ -74,14 +74,15 @@ static void kvm_reset_dirty_gfn(struct kvm *kvm, u32 slot, u64 offset, u64 mask) · 49 unmodified lines
74	74	KVM_MMU_UNLOCK(kvm);
75	75	}
76	76
77		−int kvm_dirty_ring_alloc(struct kvm_dirty_ring *ring, int index, u32 size)
	77	+int kvm_dirty_ring_alloc(struct kvm kvm, struct kvm_dirty_ring ring,
	78	+ int index, u32 size)
78	79	{
79	80	ring->dirty_gfns = vzalloc(size);
80	81	if (!ring->dirty_gfns)
81	82	return -ENOMEM;
82	83
83	84	ring->size = size / sizeof(struct kvm_dirty_gfn);
84		− ring->soft_limit = ring->size - kvm_dirty_ring_get_rsvd_entries();
	85	+ ring->soft_limit = ring->size - kvm_dirty_ring_get_rsvd_entries(kvm);
85	86	ring->dirty_index = 0;
86	87	ring->reset_index = 0;
87	88	ring->index = index;

virt/kvm/kvm_main.c+9−17

@@ -143,8 +143,6 @@ static int kvm_no_compat_open(struct inode inode, struct file file) · 142 unmodified lines
143	143	#define KVM_COMPAT(c) .compat_ioctl = kvm_no_compat_ioctl, \
144	144	.open = kvm_no_compat_open
145	145	#endif
146		−static int kvm_enable_virtualization(void);
147		−static void kvm_disable_virtualization(void);
148	146
149	147	static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
150	148
@@ -4126,7 +4124,7 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) · 3975 unmodified lines
4126	4124	goto vcpu_free_run_page;
4127	4125
4128	4126	if (kvm->dirty_ring_size) {
4129		− r = kvm_dirty_ring_alloc(&vcpu->dirty_ring,
	4127	+ r = kvm_dirty_ring_alloc(kvm, &vcpu->dirty_ring,
4130	4128	id, kvm->dirty_ring_size);
4131	4129	if (r)
4132	4130	goto arch_vcpu_destroy;
@@ -4864,7 +4862,7 @@ static int kvm_vm_ioctl_enable_dirty_log_ring(struct kvm *kvm, u32 size) · 731 unmodified lines
4864	4862	return -EINVAL;
4865	4863
4866	4864	/* Should be bigger to keep the reserved entries, or a page */
4867		− if (size < kvm_dirty_ring_get_rsvd_entries() *
	4865	+ if (size < kvm_dirty_ring_get_rsvd_entries(kvm) *
4868	4866	sizeof(struct kvm_dirty_gfn) \|\| size < PAGE_SIZE)
4869	4867	return -EINVAL;
4870	4868
@@ -5479,8 +5477,9 @@ static struct miscdevice kvm_dev = { · 608 unmodified lines
5479	5477	};
5480	5478
5481	5479	#ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING
5482		−static bool enable_virt_at_load = true;
	5480	+bool enable_virt_at_load = true;
5483	5481	module_param(enable_virt_at_load, bool, 0444);
	5482	+EXPORT_SYMBOL_GPL(enable_virt_at_load);
5484	5483
5485	5484	__visible bool kvm_rebooting;
5486	5485	EXPORT_SYMBOL_GPL(kvm_rebooting);
@@ -5589,7 +5588,7 @@ static struct syscore_ops kvm_syscore_ops = { · 102 unmodified lines
5589	5588	.shutdown = kvm_shutdown,
5590	5589	};
5591	5590
5592		−static int kvm_enable_virtualization(void)
	5591	+int kvm_enable_virtualization(void)
5593	5592	{
5594	5593	int r;
5595	5594
@@ -5634,8 +5633,9 @@ err_cpuhp: · 38 unmodified lines
5634	5633	--kvm_usage_count;
5635	5634	return r;
5636	5635	}
	5636	+EXPORT_SYMBOL_GPL(kvm_enable_virtualization);
5637	5637
5638		−static void kvm_disable_virtualization(void)
	5638	+void kvm_disable_virtualization(void)
5639	5639	{
5640	5640	guard(mutex)(&kvm_usage_lock);
5641	5641
@@ -5646,6 +5646,7 @@ static void kvm_disable_virtualization(void) · 4 unmodified lines
5646	5646	cpuhp_remove_state(CPUHP_AP_KVM_ONLINE);
5647	5647	kvm_arch_disable_virtualization();
5648	5648	}
	5649	+EXPORT_SYMBOL_GPL(kvm_disable_virtualization);
5649	5650
5650	5651	static int kvm_init_virtualization(void)
5651	5652	{
@@ -5661,21 +5662,11 @@ static void kvm_uninit_virtualization(void) · 9 unmodified lines
5661	5662	kvm_disable_virtualization();
5662	5663	}
5663	5664	#else /* CONFIG_KVM_GENERIC_HARDWARE_ENABLING */
5664		−static int kvm_enable_virtualization(void)
5665		−{
5666		− return 0;
5667		−}
5668		−
5669	5665	static int kvm_init_virtualization(void)
5670	5666	{
5671	5667	return 0;
5672	5668	}
5673	5669
5674		−static void kvm_disable_virtualization(void)
5675		−{
5676		−
5677		−}
5678		−
5679	5670	static void kvm_uninit_virtualization(void)
5680	5671	{
5681	5672
@@ -5864,6 +5855,7 @@ int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, · 182 unmodified lines
5864	5855	r = __kvm_io_bus_read(vcpu, bus, &range, val);
5865	5856	return r < 0 ? r : 0;
5866	5857	}
	5858	+EXPORT_SYMBOL_GPL(kvm_io_bus_read);
5867	5859
5868	5860	int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
5869	5861	int len, struct kvm_io_device *dev)