Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

Pull kvm fixes from Paolo Bonzini: "Two larger x86 series: - Redo incorrect fix for SEV/SMAP erratum - Windows 11 Hyper-V workaround Other x86 changes: - Various x86 cleanups - Re-enable access_tracking_perf_test - Fix for #GP handling on SVM - Fix for CPUID leaf 0Dh in KVM_GET_SUPPORTED_CPUID - Fix for ICEBP in interrupt shadow - Avoid false-positive RCU splat - Enable Enlightened MSR-Bitmap support for real ARM: - Correctly update the shadow register on exception injection when running in nVHE mode - Correctly use the mm_ops indirection when performing cache invalidation from the page-table walker - Restrict the vgic-v3 workaround for SEIS to the two known broken implementations Generic code changes: - Dead code cleanup" * tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (43 commits) KVM: eventfd: Fix false positive RCU usage warning KVM: nVMX: Allow VMREAD when Enlightened VMCS is in use KVM: nVMX: Implement evmcs_field_offset() suitable for handle_vmread() KVM: nVMX: Rename vmcs_to_field_offset{,_table} KVM: nVMX: eVMCS: Filter out VM_EXIT_SAVE_VMX_PREEMPTION_TIMER KVM: nVMX: Also filter MSR_IA32_VMX_TRUE_PINBASED_CTLS when eVMCS selftests: kvm: check dynamic bits against KVM_X86_XCOMP_GUEST_SUPP KVM: x86: add system attribute to retrieve full set of supported xsave states KVM: x86: Add a helper to retrieve userspace address from kvm_device_attr selftests: kvm: move vm_xsave_req_perm call to amx_test KVM: x86: Sync the states size with the XCR0/IA32_XSS at, any time KVM: x86: Update vCPU's runtime CPUID on write to MSR_IA32_XSS KVM: x86: Keep MSR_IA32_XSS unchanged for INIT KVM: x86: Free kvm_cpuid_entry2 array on post-KVM_RUN KVM_SET_CPUID{,2} KVM: nVMX: WARN on any attempt to allocate shadow VMCS for vmcs02 KVM: selftests: Don't skip L2's VMCALL in SMM test for SVM guest KVM: x86: Check .flags in kvm_cpuid_check_equal() too KVM: x86: Forcibly leave nested virt when SMM state is toggled KVM: SVM: drop unnecessary code in svm_hv_vmcb_dirty_nested_enlightenments() KVM: SVM: hyper-v: Enable Enlightened MSR-Bitmap support for real ...

Linus Torvalds committed 4 years ago

Commit 3cd7cd82 parents e015270 17179d035 files · +490 −228Browse files

Unified · Split

35 files

Documentation/virt/kvm/api.rst +3 −1
arch/arm64/kvm/hyp/exception.c +4 −1
arch/arm64/kvm/hyp/pgtable.c +6 −12
arch/arm64/kvm/hyp/vgic-v3-sr.c +3 −0
arch/arm64/kvm/vgic/vgic-v3.c +15 −2
arch/x86/include/asm/kvm_host.h +3 −2
arch/x86/include/uapi/asm/kvm.h +3 −0
arch/x86/kvm/cpuid.c +55 −35
arch/x86/kvm/lapic.c +1 −1
arch/x86/kvm/svm/nested.c +5 −4
arch/x86/kvm/svm/sev.c +7 −2
arch/x86/kvm/svm/svm.c +122 −55
arch/x86/kvm/svm/svm.h +1 −6
arch/x86/kvm/svm/svm_onhyperv.h +4 −8
arch/x86/kvm/vmx/capabilities.h +0 −1
arch/x86/kvm/vmx/evmcs.c +2 −2
arch/x86/kvm/vmx/evmcs.h +39 −9
arch/x86/kvm/vmx/nested.c +54 −28
arch/x86/kvm/vmx/vmcs12.c +2 −2
arch/x86/kvm/vmx/vmcs12.h +3 −3
arch/x86/kvm/vmx/vmx.c +38 −9
arch/x86/kvm/x86.c +81 −13
arch/x86/kvm/xen.c +2 −8
include/uapi/linux/kvm.h +1 −0
tools/arch/x86/include/uapi/asm/kvm.h +3 −0
tools/include/uapi/linux/kvm.h +1 −0
tools/testing/selftests/kvm/Makefile +1 −0
tools/testing/selftests/kvm/include/kvm_util_base.h +0 −1
tools/testing/selftests/kvm/include/x86_64/processor.h +1 −0
tools/testing/selftests/kvm/lib/kvm_util.c +0 −7
tools/testing/selftests/kvm/lib/x86_64/processor.c +21 −6
tools/testing/selftests/kvm/x86_64/amx_test.c +2 −0
tools/testing/selftests/kvm/x86_64/smm_test.c +0 −1
virt/kvm/eventfd.c +4 −4
virt/kvm/kvm_main.c +3 −5

Documentation/virt/kvm/api.rst+3−1

@@ -3268,6 +3268,7 @@ number. · 3267 unmodified lines
3268	3268
3269	3269	:Capability: KVM_CAP_DEVICE_CTRL, KVM_CAP_VM_ATTRIBUTES for vm device,
3270	3270	KVM_CAP_VCPU_ATTRIBUTES for vcpu device
	3271	+ KVM_CAP_SYS_ATTRIBUTES for system (/dev/kvm) device (no set)
3271	3272	:Type: device ioctl, vm ioctl, vcpu ioctl
3272	3273	:Parameters: struct kvm_device_attr
3273	3274	:Returns: 0 on success, -1 on error
@@ -3302,7 +3303,8 @@ transferred is defined by the particular attribute. · 28 unmodified lines
3302	3303	------------------------
3303	3304
3304	3305	:Capability: KVM_CAP_DEVICE_CTRL, KVM_CAP_VM_ATTRIBUTES for vm device,
3305		− KVM_CAP_VCPU_ATTRIBUTES for vcpu device
	3306	+ KVM_CAP_VCPU_ATTRIBUTES for vcpu device
	3307	+ KVM_CAP_SYS_ATTRIBUTES for system (/dev/kvm) device
3306	3308	:Type: device ioctl, vm ioctl, vcpu ioctl
3307	3309	:Parameters: struct kvm_device_attr
3308	3310	:Returns: 0 on success, -1 on error

arch/arm64/kvm/hyp/exception.c+4−1

@@ -38,7 +38,10 @@ static inline void __vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg) · 37 unmodified lines
38	38
39	39	static void __vcpu_write_spsr(struct kvm_vcpu *vcpu, u64 val)
40	40	{
41		− write_sysreg_el1(val, SYS_SPSR);
	41	+ if (has_vhe())
	42	+ write_sysreg_el1(val, SYS_SPSR);
	43	+ else
	44	+ __vcpu_sys_reg(vcpu, SPSR_EL1) = val;
42	45	}
43	46
44	47	static void __vcpu_write_spsr_abt(struct kvm_vcpu *vcpu, u64 val)

arch/arm64/kvm/hyp/pgtable.c+6−12

@@ -983,13 +983,9 @@ static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 982 unmodified lines
983	983	*/
984	984	stage2_put_pte(ptep, mmu, addr, level, mm_ops);
985	985
986		− if (need_flush) {
987		− kvm_pte_t *pte_follow = kvm_pte_follow(pte, mm_ops);
988		−
989		− dcache_clean_inval_poc((unsigned long)pte_follow,
990		− (unsigned long)pte_follow +
991		− kvm_granule_size(level));
992		− }
	986	+ if (need_flush && mm_ops->dcache_clean_inval_poc)
	987	+ mm_ops->dcache_clean_inval_poc(kvm_pte_follow(pte, mm_ops),
	988	+ kvm_granule_size(level));
993	989
994	990	if (childp)
995	991	mm_ops->put_page(childp);
@@ -1151,15 +1147,13 @@ static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 155 unmodified lines
1151	1147	struct kvm_pgtable *pgt = arg;
1152	1148	struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;
1153	1149	kvm_pte_t pte = *ptep;
1154		− kvm_pte_t *pte_follow;
1155	1150
1156	1151	if (!kvm_pte_valid(pte) \|\| !stage2_pte_cacheable(pgt, pte))
1157	1152	return 0;
1158	1153
1159		− pte_follow = kvm_pte_follow(pte, mm_ops);
1160		− dcache_clean_inval_poc((unsigned long)pte_follow,
1161		− (unsigned long)pte_follow +
1162		− kvm_granule_size(level));
	1154	+ if (mm_ops->dcache_clean_inval_poc)
	1155	+ mm_ops->dcache_clean_inval_poc(kvm_pte_follow(pte, mm_ops),
	1156	+ kvm_granule_size(level));
1163	1157	return 0;
1164	1158	}
1165	1159

arch/arm64/kvm/hyp/vgic-v3-sr.c+3−0

@@ -983,6 +983,9 @@ static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) · 982 unmodified lines
983	983	val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT;
984	984	/* IDbits */
985	985	val \|= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT;
	986	+ /* SEIS */
	987	+ if (kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_SEIS_MASK)
	988	+ val \|= BIT(ICC_CTLR_EL1_SEIS_SHIFT);
986	989	/* A3V */
987	990	val \|= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT;
988	991	/* EOImode */

arch/arm64/kvm/vgic/vgic-v3.c+15−2

@@ -609,6 +609,18 @@ static int __init early_gicv4_enable(char *buf) · 608 unmodified lines
609	609	}
610	610	early_param("kvm-arm.vgic_v4_enable", early_gicv4_enable);
611	611
	612	+static const struct midr_range broken_seis[] = {
	613	+ MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM),
	614	+ MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM),
	615	+ {},
	616	+};
	617	+
	618	+static bool vgic_v3_broken_seis(void)
	619	+{
	620	+ return ((kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_SEIS_MASK) &&
	621	+ is_midr_in_range_list(read_cpuid_id(), broken_seis));
	622	+}
	623	+
612	624	/**
613	625	* vgic_v3_probe - probe for a VGICv3 compatible interrupt controller
614	626	* @info: pointer to the GIC description
@@ -676,9 +688,10 @@ int vgic_v3_probe(const struct gic_kvm_info *info) · 61 unmodified lines
676	688	group1_trap = true;
677	689	}
678	690
679		− if (kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_SEIS_MASK) {
680		− kvm_info("GICv3 with locally generated SEI\n");
	691	+ if (vgic_v3_broken_seis()) {
	692	+ kvm_info("GICv3 with broken locally generated SEI\n");
681	693
	694	+ kvm_vgic_global_state.ich_vtr_el2 &= ~ICH_VTR_SEIS_MASK;
682	695	group0_trap = true;
683	696	group1_trap = true;
684	697	if (ich_vtr_el2 & ICH_VTR_TDS_MASK)

arch/x86/include/asm/kvm_host.h+3−2

@@ -1483,7 +1483,8 @@ struct kvm_x86_ops { · 1482 unmodified lines
1483	1483
1484	1484	int (get_msr_feature)(struct kvm_msr_entry entry);
1485	1485
1486		− bool (can_emulate_instruction)(struct kvm_vcpu vcpu, void *insn, int insn_len);
	1486	+ bool (can_emulate_instruction)(struct kvm_vcpu vcpu, int emul_type,
	1487	+ void *insn, int insn_len);
1487	1488
1488	1489	bool (apic_init_signal_blocked)(struct kvm_vcpu vcpu);
1489	1490	int (enable_direct_tlbflush)(struct kvm_vcpu vcpu);
@@ -1496,6 +1497,7 @@ struct kvm_x86_ops { · 6 unmodified lines
1496	1497	};
1497	1498
1498	1499	struct kvm_x86_nested_ops {
	1500	+ void (leave_nested)(struct kvm_vcpu vcpu);
1499	1501	int (check_events)(struct kvm_vcpu vcpu);
1500	1502	bool (hv_timer_pending)(struct kvm_vcpu vcpu);
1501	1503	void (triple_fault)(struct kvm_vcpu vcpu);
@@ -1861,7 +1863,6 @@ int kvm_cpu_has_extint(struct kvm_vcpu *v); · 359 unmodified lines
1861	1863	int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu);
1862	1864	int kvm_cpu_get_interrupt(struct kvm_vcpu *v);
1863	1865	void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event);
1864		−void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu);
1865	1866
1866	1867	int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
1867	1868	unsigned long ipi_bitmap_high, u32 min,

arch/x86/include/uapi/asm/kvm.h+3−0

@@ -452,6 +452,9 @@ struct kvm_sync_regs { · 451 unmodified lines
452	452
453	453	#define KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE 0x00000001
454	454
	455	+/* attributes for system fd (group 0) */
	456	+#define KVM_X86_XCOMP_GUEST_SUPP 0
	457	+
455	458	struct kvm_vmx_nested_state_data {
456	459	__u8 vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE];
457	460	__u8 shadow_vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE];

arch/x86/kvm/cpuid.c+55−35

@@ -133,6 +133,7 @@ static int kvm_cpuid_check_equal(struct kvm_vcpu *vcpu, struct kvm_cpuid_entry2 · 132 unmodified lines
133	133	orig = &vcpu->arch.cpuid_entries[i];
134	134	if (e2[i].function != orig->function \|\|
135	135	e2[i].index != orig->index \|\|
	136	+ e2[i].flags != orig->flags \|\|
136	137	e2[i].eax != orig->eax \|\| e2[i].ebx != orig->ebx \|\|
137	138	e2[i].ecx != orig->ecx \|\| e2[i].edx != orig->edx)
138	139	return -EINVAL;
@@ -196,10 +197,26 @@ void kvm_update_pv_runtime(struct kvm_vcpu *vcpu) · 57 unmodified lines
196	197	vcpu->arch.pv_cpuid.features = best->eax;
197	198	}
198	199
	200	+/*
	201	+ * Calculate guest's supported XCR0 taking into account guest CPUID data and
	202	+ * supported_xcr0 (comprised of host configuration and KVM_SUPPORTED_XCR0).
	203	+ */
	204	+static u64 cpuid_get_supported_xcr0(struct kvm_cpuid_entry2 *entries, int nent)
	205	+{
	206	+ struct kvm_cpuid_entry2 *best;
	207	+
	208	+ best = cpuid_entry2_find(entries, nent, 0xd, 0);
	209	+ if (!best)
	210	+ return 0;
	211	+
	212	+ return (best->eax \| ((u64)best->edx << 32)) & supported_xcr0;
	213	+}
	214	+
199	215	static void __kvm_update_cpuid_runtime(struct kvm_vcpu vcpu, struct kvm_cpuid_entry2 entries,
200	216	int nent)
201	217	{
202	218	struct kvm_cpuid_entry2 *best;
	219	+ u64 guest_supported_xcr0 = cpuid_get_supported_xcr0(entries, nent);
203	220
204	221	best = cpuid_entry2_find(entries, nent, 1, 0);
205	222	if (best) {
@@ -238,6 +255,21 @@ static void __kvm_update_cpuid_runtime(struct kvm_vcpu *vcpu, struct kvm_cpuid_e · 32 unmodified lines
238	255	vcpu->arch.ia32_misc_enable_msr &
239	256	MSR_IA32_MISC_ENABLE_MWAIT);
240	257	}
	258	+
	259	+ /*
	260	+ * Bits 127:0 of the allowed SECS.ATTRIBUTES (CPUID.0x12.0x1) enumerate
	261	+ * the supported XSAVE Feature Request Mask (XFRM), i.e. the enclave's
	262	+ * requested XCR0 value. The enclave's XFRM must be a subset of XCRO
	263	+ * at the time of EENTER, thus adjust the allowed XFRM by the guest's
	264	+ * supported XCR0. Similar to XCR0 handling, FP and SSE are forced to
	265	+ * '1' even on CPUs that don't support XSAVE.
	266	+ */
	267	+ best = cpuid_entry2_find(entries, nent, 0x12, 0x1);
	268	+ if (best) {
	269	+ best->ecx &= guest_supported_xcr0 & 0xffffffff;
	270	+ best->edx &= guest_supported_xcr0 >> 32;
	271	+ best->ecx \|= XFEATURE_MASK_FPSSE;
	272	+ }
241	273	}
242	274
243	275	void kvm_update_cpuid_runtime(struct kvm_vcpu *vcpu)
@@ -261,27 +293,8 @@ static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) · 17 unmodified lines
261	293	kvm_apic_set_version(vcpu);
262	294	}
263	295
264		− best = kvm_find_cpuid_entry(vcpu, 0xD, 0);
265		− if (!best)
266		− vcpu->arch.guest_supported_xcr0 = 0;
267		− else
268		− vcpu->arch.guest_supported_xcr0 =
269		− (best->eax \| ((u64)best->edx << 32)) & supported_xcr0;
270		−
271		− /*
272		− * Bits 127:0 of the allowed SECS.ATTRIBUTES (CPUID.0x12.0x1) enumerate
273		− * the supported XSAVE Feature Request Mask (XFRM), i.e. the enclave's
274		− * requested XCR0 value. The enclave's XFRM must be a subset of XCRO
275		− * at the time of EENTER, thus adjust the allowed XFRM by the guest's
276		− * supported XCR0. Similar to XCR0 handling, FP and SSE are forced to
277		− * '1' even on CPUs that don't support XSAVE.
278		− */
279		− best = kvm_find_cpuid_entry(vcpu, 0x12, 0x1);
280		− if (best) {
281		− best->ecx &= vcpu->arch.guest_supported_xcr0 & 0xffffffff;
282		− best->edx &= vcpu->arch.guest_supported_xcr0 >> 32;
283		− best->ecx \|= XFEATURE_MASK_FPSSE;
284		− }
	296	+ vcpu->arch.guest_supported_xcr0 =
	297	+ cpuid_get_supported_xcr0(vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent);
285	298
286	299	kvm_update_pv_runtime(vcpu);
287	300
@@ -346,8 +359,14 @@ static int kvm_set_cpuid(struct kvm_vcpu vcpu, struct kvm_cpuid_entry2 e2, · 58 unmodified lines
346	359	* KVM_SET_CPUID{,2} again. To support this legacy behavior, check
347	360	* whether the supplied CPUID data is equal to what's already set.
348	361	*/
349		− if (vcpu->arch.last_vmentry_cpu != -1)
350		− return kvm_cpuid_check_equal(vcpu, e2, nent);
	362	+ if (vcpu->arch.last_vmentry_cpu != -1) {
	363	+ r = kvm_cpuid_check_equal(vcpu, e2, nent);
	364	+ if (r)
	365	+ return r;
	366	+
	367	+ kvfree(e2);
	368	+ return 0;
	369	+ }
351	370
352	371	r = kvm_check_cpuid(vcpu, e2, nent);
353	372	if (r)
@@ -887,13 +906,14 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) · 533 unmodified lines
887	906	}
888	907	break;
889	908	case 0xd: {
890		− u64 guest_perm = xstate_get_guest_group_perm();
	909	+ u64 permitted_xcr0 = supported_xcr0 & xstate_get_guest_group_perm();
	910	+ u64 permitted_xss = supported_xss;
891	911
892		− entry->eax &= supported_xcr0 & guest_perm;
893		− entry->ebx = xstate_required_size(supported_xcr0, false);
	912	+ entry->eax &= permitted_xcr0;
	913	+ entry->ebx = xstate_required_size(permitted_xcr0, false);
894	914	entry->ecx = entry->ebx;
895		− entry->edx &= (supported_xcr0 & guest_perm) >> 32;
896		− if (!supported_xcr0)
	915	+ entry->edx &= permitted_xcr0 >> 32;
	916	+ if (!permitted_xcr0)
897	917	break;
898	918
899	919	entry = do_host_cpuid(array, function, 1);
@@ -902,20 +922,20 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) · 2 unmodified lines
902	922
903	923	cpuid_entry_override(entry, CPUID_D_1_EAX);
904	924	if (entry->eax & (F(XSAVES)\|F(XSAVEC)))
905		− entry->ebx = xstate_required_size(supported_xcr0 \| supported_xss,
	925	+ entry->ebx = xstate_required_size(permitted_xcr0 \| permitted_xss,
906	926	true);
907	927	else {
908		− WARN_ON_ONCE(supported_xss != 0);
	928	+ WARN_ON_ONCE(permitted_xss != 0);
909	929	entry->ebx = 0;
910	930	}
911		− entry->ecx &= supported_xss;
912		− entry->edx &= supported_xss >> 32;
	931	+ entry->ecx &= permitted_xss;
	932	+ entry->edx &= permitted_xss >> 32;
913	933
914	934	for (i = 2; i < 64; ++i) {
915	935	bool s_state;
916		− if (supported_xcr0 & BIT_ULL(i))
	936	+ if (permitted_xcr0 & BIT_ULL(i))
917	937	s_state = false;
918		− else if (supported_xss & BIT_ULL(i))
	938	+ else if (permitted_xss & BIT_ULL(i))
919	939	s_state = true;
920	940	else
921	941	continue;
@@ -929,7 +949,7 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) · 7 unmodified lines
929	949	* invalid sub-leafs. Only valid sub-leafs should
930	950	* reach this point, and they should have a non-zero
931	951	* save state size. Furthermore, check whether the
932		− * processor agrees with supported_xcr0/supported_xss
	952	+ * processor agrees with permitted_xcr0/permitted_xss
933	953	* on whether this is an XCR0- or IA32_XSS-managed area.
934	954	*/
935	955	if (WARN_ON_ONCE(!entry->eax \|\| (entry->ecx & 0x1) != s_state)) {

arch/x86/kvm/lapic.c+1−1

@@ -2629,7 +2629,7 @@ int kvm_apic_set_state(struct kvm_vcpu vcpu, struct kvm_lapic_state s) · 2628 unmodified lines
2629	2629	kvm_apic_set_version(vcpu);
2630	2630
2631	2631	apic_update_ppr(apic);
2632		− hrtimer_cancel(&apic->lapic_timer.timer);
	2632	+ cancel_apic_timer(apic);
2633	2633	apic->lapic_timer.expired_tscdeadline = 0;
2634	2634	apic_update_lvtt(apic);
2635	2635	apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));

arch/x86/kvm/svm/nested.c+5−4

@@ -983,9 +983,9 @@ void svm_free_nested(struct vcpu_svm *svm) · 982 unmodified lines
983	983	/*
984	984	* Forcibly leave nested mode in order to be able to reset the VCPU later on.
985	985	*/
986		−void svm_leave_nested(struct vcpu_svm *svm)
	986	+void svm_leave_nested(struct kvm_vcpu *vcpu)
987	987	{
988		− struct kvm_vcpu *vcpu = &svm->vcpu;
	988	+ struct vcpu_svm *svm = to_svm(vcpu);
989	989
990	990	if (is_guest_mode(vcpu)) {
991	991	svm->nested.nested_run_pending = 0;
@@ -1411,7 +1411,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, · 419 unmodified lines
1411	1411	return -EINVAL;
1412	1412
1413	1413	if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) {
1414		− svm_leave_nested(svm);
	1414	+ svm_leave_nested(vcpu);
1415	1415	svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET));
1416	1416	return 0;
1417	1417	}
@@ -1478,7 +1478,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, · 60 unmodified lines
1478	1478	*/
1479	1479
1480	1480	if (is_guest_mode(vcpu))
1481		− svm_leave_nested(svm);
	1481	+ svm_leave_nested(vcpu);
1482	1482	else
1483	1483	svm->nested.vmcb02.ptr->save = svm->vmcb01.ptr->save;
1484	1484
@@ -1532,6 +1532,7 @@ static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu) · 47 unmodified lines
1532	1532	}
1533	1533
1534	1534	struct kvm_x86_nested_ops svm_nested_ops = {
	1535	+ .leave_nested = svm_leave_nested,
1535	1536	.check_events = svm_check_nested_events,
1536	1537	.triple_fault = nested_svm_triple_fault,
1537	1538	.get_nested_state_pages = svm_get_nested_state_pages,

arch/x86/kvm/svm/sev.c+7−2

@@ -2100,8 +2100,13 @@ void __init sev_hardware_setup(void) · 2099 unmodified lines
2100	2100	if (!sev_enabled \|\| !npt_enabled)
2101	2101	goto out;
2102	2102
2103		− /* Does the CPU support SEV? */
2104		− if (!boot_cpu_has(X86_FEATURE_SEV))
	2103	+ /*
	2104	+ * SEV must obviously be supported in hardware. Sanity check that the
	2105	+ * CPU supports decode assists, which is mandatory for SEV guests to
	2106	+ * support instruction emulation.
	2107	+ */
	2108	+ if (!boot_cpu_has(X86_FEATURE_SEV) \|\|
	2109	+ WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_DECODEASSISTS)))
2105	2110	goto out;
2106	2111
2107	2112	/* Retrieve SEV CPUID information */

arch/x86/kvm/svm/svm.c+122−55

@@ -290,7 +290,7 @@ int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) · 289 unmodified lines
290	290
291	291	if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) {
292	292	if (!(efer & EFER_SVME)) {
293		− svm_leave_nested(svm);
	293	+ svm_leave_nested(vcpu);
294	294	svm_set_gif(svm, true);
295	295	/* #GP intercept is still needed for vmware backdoor */
296	296	if (!enable_vmware_backdoor)
@@ -312,7 +312,11 @@ int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) · 15 unmodified lines
312	312	return ret;
313	313	}
314	314
315		− if (svm_gp_erratum_intercept)
	315	+ /*
	316	+ * Never intercept #GP for SEV guests, KVM can't
	317	+ * decrypt guest memory to workaround the erratum.
	318	+ */
	319	+ if (svm_gp_erratum_intercept && !sev_guest(vcpu->kvm))
316	320	set_exception_intercept(svm, GP_VECTOR);
317	321	}
318	322	}
@@ -1010,9 +1014,10 @@ static void init_vmcb(struct kvm_vcpu *vcpu) · 691 unmodified lines
1010	1014	* Guest access to VMware backdoor ports could legitimately
1011	1015	* trigger #GP because of TSS I/O permission bitmap.
1012	1016	* We intercept those #GP and allow access to them anyway
1013		− * as VMware does.
	1017	+ * as VMware does. Don't intercept #GP for SEV guests as KVM can't
	1018	+ * decrypt guest memory to decode the faulting instruction.
1014	1019	*/
1015		− if (enable_vmware_backdoor)
	1020	+ if (enable_vmware_backdoor && !sev_guest(vcpu->kvm))
1016	1021	set_exception_intercept(svm, GP_VECTOR);
1017	1022
1018	1023	svm_set_intercept(svm, INTERCEPT_INTR);
@@ -2091,10 +2096,6 @@ static int gp_interception(struct kvm_vcpu *vcpu) · 1072 unmodified lines
2091	2096	if (error_code)
2092	2097	goto reinject;
2093	2098
2094		− /* All SVM instructions expect page aligned RAX */
2095		− if (svm->vmcb->save.rax & ~PAGE_MASK)
2096		− goto reinject;
2097		−
2098	2099	/* Decode the instruction for usage later */
2099	2100	if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK)
2100	2101	goto reinject;
@@ -2112,8 +2113,13 @@ static int gp_interception(struct kvm_vcpu *vcpu) · 11 unmodified lines
2112	2113	if (!is_guest_mode(vcpu))
2113	2114	return kvm_emulate_instruction(vcpu,
2114	2115	EMULTYPE_VMWARE_GP \| EMULTYPE_NO_DECODE);
2115		− } else
	2116	+ } else {
	2117	+ /* All SVM instructions expect page aligned RAX */
	2118	+ if (svm->vmcb->save.rax & ~PAGE_MASK)
	2119	+ goto reinject;
	2120	+
2116	2121	return emulate_svm_instr(vcpu, opcode);
	2122	+ }
2117	2123
2118	2124	reinject:
2119	2125	kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
@@ -4252,79 +4258,140 @@ static void svm_enable_smi_window(struct kvm_vcpu *vcpu) · 2132 unmodified lines
4252	4258	}
4253	4259	}
4254	4260
4255		−static bool svm_can_emulate_instruction(struct kvm_vcpu vcpu, void insn, int insn_len)
	4261	+static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
	4262	+ void *insn, int insn_len)
4256	4263	{
4257	4264	bool smep, smap, is_user;
4258	4265	unsigned long cr4;
	4266	+ u64 error_code;
	4267	+
	4268	+ /* Emulation is always possible when KVM has access to all guest state. */
	4269	+ if (!sev_guest(vcpu->kvm))
	4270	+ return true;
	4271	+
	4272	+ /* #UD and #GP should never be intercepted for SEV guests. */
	4273	+ WARN_ON_ONCE(emul_type & (EMULTYPE_TRAP_UD \|
	4274	+ EMULTYPE_TRAP_UD_FORCED \|
	4275	+ EMULTYPE_VMWARE_GP));
4259	4276
4260	4277	/*
4261		− * When the guest is an SEV-ES guest, emulation is not possible.
	4278	+ * Emulation is impossible for SEV-ES guests as KVM doesn't have access
	4279	+ * to guest register state.
4262	4280	*/
4263	4281	if (sev_es_guest(vcpu->kvm))
4264	4282	return false;
4265	4283
	4284	+ /*
	4285	+ * Emulation is possible if the instruction is already decoded, e.g.
	4286	+ * when completing I/O after returning from userspace.
	4287	+ */
	4288	+ if (emul_type & EMULTYPE_NO_DECODE)
	4289	+ return true;
	4290	+
	4291	+ /*
	4292	+ * Emulation is possible for SEV guests if and only if a prefilled
	4293	+ * buffer containing the bytes of the intercepted instruction is
	4294	+ * available. SEV guest memory is encrypted with a guest specific key
	4295	+ * and cannot be decrypted by KVM, i.e. KVM would read cyphertext and
	4296	+ * decode garbage.
	4297	+ *
	4298	+ * Inject #UD if KVM reached this point without an instruction buffer.
	4299	+ * In practice, this path should never be hit by a well-behaved guest,
	4300	+ * e.g. KVM doesn't intercept #UD or #GP for SEV guests, but this path
	4301	+ * is still theoretically reachable, e.g. via unaccelerated fault-like
	4302	+ * AVIC access, and needs to be handled by KVM to avoid putting the
	4303	+ * guest into an infinite loop. Injecting #UD is somewhat arbitrary,
	4304	+ * but its the least awful option given lack of insight into the guest.
	4305	+ */
	4306	+ if (unlikely(!insn)) {
	4307	+ kvm_queue_exception(vcpu, UD_VECTOR);
	4308	+ return false;
	4309	+ }
	4310	+
	4311	+ /*
	4312	+ * Emulate for SEV guests if the insn buffer is not empty. The buffer
	4313	+ * will be empty if the DecodeAssist microcode cannot fetch bytes for
	4314	+ * the faulting instruction because the code fetch itself faulted, e.g.
	4315	+ * the guest attempted to fetch from emulated MMIO or a guest page
	4316	+ * table used to translate CS:RIP resides in emulated MMIO.
	4317	+ */
	4318	+ if (likely(insn_len))
	4319	+ return true;
	4320	+
4266	4321	/*
4267	4322	* Detect and workaround Errata 1096 Fam_17h_00_0Fh.
4268	4323	*
4269	4324	* Errata:
4270		− * When CPU raise #NPF on guest data access and vCPU CR4.SMAP=1, it is
4271		− * possible that CPU microcode implementing DecodeAssist will fail
4272		− * to read bytes of instruction which caused #NPF. In this case,
4273		− * GuestIntrBytes field of the VMCB on a VMEXIT will incorrectly
4274		− * return 0 instead of the correct guest instruction bytes.
4275		− *
4276		− * This happens because CPU microcode reading instruction bytes
4277		− * uses a special opcode which attempts to read data using CPL=0
4278		− * privileges. The microcode reads CS:RIP and if it hits a SMAP
4279		− * fault, it gives up and returns no instruction bytes.
	4325	+ * When CPU raises #NPF on guest data access and vCPU CR4.SMAP=1, it is
	4326	+ * possible that CPU microcode implementing DecodeAssist will fail to
	4327	+ * read guest memory at CS:RIP and vmcb.GuestIntrBytes will incorrectly
	4328	+ * be '0'. This happens because microcode reads CS:RIP using a _data_
	4329	+ * loap uop with CPL=0 privileges. If the load hits a SMAP #PF, ucode
	4330	+ * gives up and does not fill the instruction bytes buffer.
4280	4331	*
4281		− * Detection:
4282		− * We reach here in case CPU supports DecodeAssist, raised #NPF and
4283		− * returned 0 in GuestIntrBytes field of the VMCB.
4284		− * First, errata can only be triggered in case vCPU CR4.SMAP=1.
4285		− * Second, if vCPU CR4.SMEP=1, errata could only be triggered
4286		− * in case vCPU CPL==3 (Because otherwise guest would have triggered
4287		− * a SMEP fault instead of #NPF).
4288		− * Otherwise, vCPU CR4.SMEP=0, errata could be triggered by any vCPU CPL.
4289		− * As most guests enable SMAP if they have also enabled SMEP, use above
4290		− * logic in order to attempt minimize false-positive of detecting errata
4291		− * while still preserving all cases semantic correctness.
	4332	+ * As above, KVM reaches this point iff the VM is an SEV guest, the CPU
	4333	+ * supports DecodeAssist, a #NPF was raised, KVM's page fault handler
	4334	+ * triggered emulation (e.g. for MMIO), and the CPU returned 0 in the
	4335	+ * GuestIntrBytes field of the VMCB.
4292	4336	*
4293		− * Workaround:
4294		− * To determine what instruction the guest was executing, the hypervisor
4295		− * will have to decode the instruction at the instruction pointer.
	4337	+ * This does _not_ mean that the erratum has been encountered, as the
	4338	+ * DecodeAssist will also fail if the load for CS:RIP hits a legitimate
	4339	+ * #PF, e.g. if the guest attempt to execute from emulated MMIO and
	4340	+ * encountered a reserved/not-present #PF.
4296	4341	*
4297		− * In non SEV guest, hypervisor will be able to read the guest
4298		− * memory to decode the instruction pointer when insn_len is zero
4299		− * so we return true to indicate that decoding is possible.
	4342	+ * To hit the erratum, the following conditions must be true:
	4343	+ * 1. CR4.SMAP=1 (obviously).
	4344	+ * 2. CR4.SMEP=0 \|\| CPL=3. If SMEP=1 and CPL<3, the erratum cannot
	4345	+ * have been hit as the guest would have encountered a SMEP
	4346	+ * violation #PF, not a #NPF.
	4347	+ * 3. The #NPF is not due to a code fetch, in which case failure to
	4348	+ * retrieve the instruction bytes is legitimate (see abvoe).
4300	4349	*
4301		− * But in the SEV guest, the guest memory is encrypted with the
4302		− * guest specific key and hypervisor will not be able to decode the
4303		− * instruction pointer so we will not able to workaround it. Lets
4304		− * print the error and request to kill the guest.
	4350	+ * In addition, don't apply the erratum workaround if the #NPF occurred
	4351	+ * while translating guest page tables (see below).
4305	4352	*/
4306		− if (likely(!insn \|\| insn_len))
4307		− return true;
4308		−
4309		− /*
4310		− * If RIP is invalid, go ahead with emulation which will cause an
4311		− * internal error exit.
4312		− */
4313		− if (!kvm_vcpu_gfn_to_memslot(vcpu, kvm_rip_read(vcpu) >> PAGE_SHIFT))
4314		− return true;
	4353	+ error_code = to_svm(vcpu)->vmcb->control.exit_info_1;
	4354	+ if (error_code & (PFERR_GUEST_PAGE_MASK \| PFERR_FETCH_MASK))
	4355	+ goto resume_guest;
4315	4356
4316	4357	cr4 = kvm_read_cr4(vcpu);
4317	4358	smep = cr4 & X86_CR4_SMEP;
4318	4359	smap = cr4 & X86_CR4_SMAP;
4319	4360	is_user = svm_get_cpl(vcpu) == 3;
4320	4361	if (smap && (!smep \|\| is_user)) {
4321		− if (!sev_guest(vcpu->kvm))
4322		− return true;
4323		−
4324	4362	pr_err_ratelimited("KVM: SEV Guest triggered AMD Erratum 1096\n");
4325		− kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
	4363	+
	4364	+ /*
	4365	+ * If the fault occurred in userspace, arbitrarily inject #GP
	4366	+ * to avoid killing the guest and to hopefully avoid confusing
	4367	+ * the guest kernel too much, e.g. injecting #PF would not be
	4368	+ * coherent with respect to the guest's page tables. Request
	4369	+ * triple fault if the fault occurred in the kernel as there's
	4370	+ * no fault that KVM can inject without confusing the guest.
	4371	+ * In practice, the triple fault is moot as no sane SEV kernel
	4372	+ * will execute from user memory while also running with SMAP=1.
	4373	+ */
	4374	+ if (is_user)
	4375	+ kvm_inject_gp(vcpu, 0);
	4376	+ else
	4377	+ kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4326	4378	}
4327	4379
	4380	+resume_guest:
	4381	+ /*
	4382	+ * If the erratum was not hit, simply resume the guest and let it fault
	4383	+ * again. While awful, e.g. the vCPU may get stuck in an infinite loop
	4384	+ * if the fault is at CPL=0, it's the lesser of all evils. Exiting to
	4385	+ * userspace will kill the guest, and letting the emulator read garbage
	4386	+ * will yield random behavior and potentially corrupt the guest.
	4387	+ *
	4388	+ * Simply resuming the guest is technically not a violation of the SEV
	4389	+ * architecture. AMD's APM states that all code fetches and page table
	4390	+ * accesses for SEV guest are encrypted, regardless of the C-Bit. The
	4391	+ * APM also states that encrypted accesses to MMIO are "ignored", but
	4392	+ * doesn't explicitly define "ignored", i.e. doing nothing and letting
	4393	+ * the guest spin is technically "ignoring" the access.
	4394	+ */
4328	4395	return false;
4329	4396	}
4330	4397

arch/x86/kvm/svm/svm.h+1−6

@@ -304,11 +304,6 @@ static inline void vmcb_mark_all_clean(struct vmcb *vmcb) · 303 unmodified lines
304	304	& ~VMCB_ALWAYS_DIRTY_MASK;
305	305	}
306	306
307		−static inline bool vmcb_is_clean(struct vmcb *vmcb, int bit)
308		−{
309		− return (vmcb->control.clean & (1 << bit));
310		−}
311		−
312	307	static inline void vmcb_mark_dirty(struct vmcb *vmcb, int bit)
313	308	{
314	309	vmcb->control.clean &= ~(1 << bit);
@@ -525,7 +520,7 @@ static inline bool nested_exit_on_nmi(struct vcpu_svm *svm) · 210 unmodified lines
525	520
526	521	int enter_svm_guest_mode(struct kvm_vcpu *vcpu,
527	522	u64 vmcb_gpa, struct vmcb *vmcb12, bool from_vmrun);
528		−void svm_leave_nested(struct vcpu_svm *svm);
	523	+void svm_leave_nested(struct kvm_vcpu *vcpu);
529	524	void svm_free_nested(struct vcpu_svm *svm);
530	525	int svm_allocate_nested(struct vcpu_svm *svm);
531	526	int nested_svm_vmrun(struct kvm_vcpu *vcpu);

arch/x86/kvm/svm/svm_onhyperv.h+4−8

@@ -46,6 +46,9 @@ static inline void svm_hv_init_vmcb(struct vmcb *vmcb) · 45 unmodified lines
46	46	if (npt_enabled &&
47	47	ms_hyperv.nested_features & HV_X64_NESTED_ENLIGHTENED_TLB)
48	48	hve->hv_enlightenments_control.enlightened_npt_tlb = 1;
	49	+
	50	+ if (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)
	51	+ hve->hv_enlightenments_control.msr_bitmap = 1;
49	52	}
50	53
51	54	static inline void svm_hv_hardware_setup(void)
@@ -83,14 +86,7 @@ static inline void svm_hv_vmcb_dirty_nested_enlightenments( · 31 unmodified lines
83	86	struct hv_enlightenments *hve =
84	87	(struct hv_enlightenments *)vmcb->control.reserved_sw;
85	88
86		− /*
87		− * vmcb can be NULL if called during early vcpu init.
88		− * And its okay not to mark vmcb dirty during vcpu init
89		− * as we mark it dirty unconditionally towards end of vcpu
90		− * init phase.
91		− */
92		− if (vmcb_is_clean(vmcb, VMCB_HV_NESTED_ENLIGHTENMENTS) &&
93		− hve->hv_enlightenments_control.msr_bitmap)
	89	+ if (hve->hv_enlightenments_control.msr_bitmap)
94	90	vmcb_mark_dirty(vmcb, VMCB_HV_NESTED_ENLIGHTENMENTS);
95	91	}
96	92

arch/x86/kvm/vmx/capabilities.h+0−1

@@ -54,7 +54,6 @@ struct nested_vmx_msrs { · 53 unmodified lines
54	54
55	55	struct vmcs_config {
56	56	int size;
57		− int order;
58	57	u32 basic_cap;
59	58	u32 revision_id;
60	59	u32 pin_based_exec_ctrl;

arch/x86/kvm/vmx/evmcs.c+2−2

@@ -12,8 +12,6 @@ · 11 unmodified lines
12	12
13	13	DEFINE_STATIC_KEY_FALSE(enable_evmcs);
14	14
15		−#if IS_ENABLED(CONFIG_HYPERV)
16		−
17	15	#define EVMCS1_OFFSET(x) offsetof(struct hv_enlightened_vmcs, x)
18	16	#define EVMCS1_FIELD(number, name, clean_field)[ROL16(number, 6)] = \
19	17	{EVMCS1_OFFSET(name), clean_field}
@@ -296,6 +294,7 @@ const struct evmcs_field vmcs_field_to_evmcs_1[] = { · 276 unmodified lines
296	294	};
297	295	const unsigned int nr_evmcs_1_fields = ARRAY_SIZE(vmcs_field_to_evmcs_1);
298	296
	297	+#if IS_ENABLED(CONFIG_HYPERV)
299	298	__init void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf)
300	299	{
301	300	vmcs_conf->pin_based_exec_ctrl &= ~EVMCS1_UNSUPPORTED_PINCTRL;
@@ -362,6 +361,7 @@ void nested_evmcs_filter_control_msr(u32 msr_index, u64 *pdata) · 60 unmodified lines
362	361	case MSR_IA32_VMX_PROCBASED_CTLS2:
363	362	ctl_high &= ~EVMCS1_UNSUPPORTED_2NDEXEC;
364	363	break;
	364	+ case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
365	365	case MSR_IA32_VMX_PINBASED_CTLS:
366	366	ctl_high &= ~EVMCS1_UNSUPPORTED_PINCTRL;
367	367	break;

arch/x86/kvm/vmx/evmcs.h+39−9

@@ -59,12 +59,12 @@ DECLARE_STATIC_KEY_FALSE(enable_evmcs); · 58 unmodified lines
59	59	SECONDARY_EXEC_SHADOW_VMCS \| \
60	60	SECONDARY_EXEC_TSC_SCALING \| \
61	61	SECONDARY_EXEC_PAUSE_LOOP_EXITING)
62		−#define EVMCS1_UNSUPPORTED_VMEXIT_CTRL (VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
	62	+#define EVMCS1_UNSUPPORTED_VMEXIT_CTRL \
	63	+ (VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL \| \
	64	+ VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)
63	65	#define EVMCS1_UNSUPPORTED_VMENTRY_CTRL (VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
64	66	#define EVMCS1_UNSUPPORTED_VMFUNC (VMX_VMFUNC_EPTP_SWITCHING)
65	67
66		−#if IS_ENABLED(CONFIG_HYPERV)
67		−
68	68	struct evmcs_field {
69	69	u16 offset;
70	70	u16 clean_field;
@@ -73,26 +73,56 @@ struct evmcs_field { · 2 unmodified lines
73	73	extern const struct evmcs_field vmcs_field_to_evmcs_1[];
74	74	extern const unsigned int nr_evmcs_1_fields;
75	75
76		−static __always_inline int get_evmcs_offset(unsigned long field,
77		− u16 *clean_field)
	76	+static __always_inline int evmcs_field_offset(unsigned long field,
	77	+ u16 *clean_field)
78	78	{
79	79	unsigned int index = ROL16(field, 6);
80	80	const struct evmcs_field *evmcs_field;
81	81
82		− if (unlikely(index >= nr_evmcs_1_fields)) {
83		− WARN_ONCE(1, "KVM: accessing unsupported EVMCS field %lx\n",
84		− field);
	82	+ if (unlikely(index >= nr_evmcs_1_fields))
85	83	return -ENOENT;
86		− }
87	84
88	85	evmcs_field = &vmcs_field_to_evmcs_1[index];
89	86
	87	+ /*
	88	+ * Use offset=0 to detect holes in eVMCS. This offset belongs to
	89	+ * 'revision_id' but this field has no encoding and is supposed to
	90	+ * be accessed directly.
	91	+ */
	92	+ if (unlikely(!evmcs_field->offset))
	93	+ return -ENOENT;
	94	+
90	95	if (clean_field)
91	96	*clean_field = evmcs_field->clean_field;
92	97
93	98	return evmcs_field->offset;
94	99	}
95	100
	101	+static inline u64 evmcs_read_any(struct hv_enlightened_vmcs *evmcs,
	102	+ unsigned long field, u16 offset)
	103	+{
	104	+ /*
	105	+ * vmcs12_read_any() doesn't care whether the supplied structure
	106	+ * is 'struct vmcs12' or 'struct hv_enlightened_vmcs' as it takes
	107	+ * the exact offset of the required field, use it for convenience
	108	+ * here.
	109	+ */
	110	+ return vmcs12_read_any((void *)evmcs, field, offset);
	111	+}
	112	+
	113	+#if IS_ENABLED(CONFIG_HYPERV)
	114	+
	115	+static __always_inline int get_evmcs_offset(unsigned long field,
	116	+ u16 *clean_field)
	117	+{
	118	+ int offset = evmcs_field_offset(field, clean_field);
	119	+
	120	+ WARN_ONCE(offset < 0, "KVM: accessing unsupported EVMCS field %lx\n",
	121	+ field);
	122	+
	123	+ return offset;
	124	+}
	125	+
96	126	static __always_inline void evmcs_write64(unsigned long field, u64 value)
97	127	{
98	128	u16 clean_field;

arch/x86/kvm/vmx/nested.c+54−28

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	#include <asm/mmu_context.h>
8	8
9	9	#include "cpuid.h"
	10	+#include "evmcs.h"
10	11	#include "hyperv.h"
11	12	#include "mmu.h"
12	13	#include "nested.h"
@@ -4851,18 +4852,20 @@ static struct vmcs alloc_shadow_vmcs(struct kvm_vcpu vcpu) · 4838 unmodified lines
4851	4852	struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs;
4852	4853
4853	4854	/*
4854		− * We should allocate a shadow vmcs for vmcs01 only when L1
4855		− * executes VMXON and free it when L1 executes VMXOFF.
4856		− * As it is invalid to execute VMXON twice, we shouldn't reach
4857		− * here when vmcs01 already have an allocated shadow vmcs.
	4855	+ * KVM allocates a shadow VMCS only when L1 executes VMXON and frees it
	4856	+ * when L1 executes VMXOFF or the vCPU is forced out of nested
	4857	+ * operation. VMXON faults if the CPU is already post-VMXON, so it
	4858	+ * should be impossible to already have an allocated shadow VMCS. KVM
	4859	+ * doesn't support virtualization of VMCS shadowing, so vmcs01 should
	4860	+ * always be the loaded VMCS.
4858	4861	*/
4859		− WARN_ON(loaded_vmcs == &vmx->vmcs01 && loaded_vmcs->shadow_vmcs);
	4862	+ if (WARN_ON(loaded_vmcs != &vmx->vmcs01 \|\| loaded_vmcs->shadow_vmcs))
	4863	+ return loaded_vmcs->shadow_vmcs;
	4864	+
	4865	+ loaded_vmcs->shadow_vmcs = alloc_vmcs(true);
	4866	+ if (loaded_vmcs->shadow_vmcs)
	4867	+ vmcs_clear(loaded_vmcs->shadow_vmcs);
4860	4868
4861		− if (!loaded_vmcs->shadow_vmcs) {
4862		− loaded_vmcs->shadow_vmcs = alloc_vmcs(true);
4863		− if (loaded_vmcs->shadow_vmcs)
4864		− vmcs_clear(loaded_vmcs->shadow_vmcs);
4865		− }
4866	4869	return loaded_vmcs->shadow_vmcs;
4867	4870	}
4868	4871
@@ -5099,27 +5102,49 @@ static int handle_vmread(struct kvm_vcpu *vcpu) · 230 unmodified lines
5099	5102	if (!nested_vmx_check_permission(vcpu))
5100	5103	return 1;
5101	5104
5102		− /*
5103		− * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA,
5104		− * any VMREAD sets the ALU flags for VMfailInvalid.
5105		− */
5106		− if (vmx->nested.current_vmptr == INVALID_GPA \|\|
5107		− (is_guest_mode(vcpu) &&
5108		− get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA))
5109		− return nested_vmx_failInvalid(vcpu);
5110		−
5111	5105	/* Decode instruction info and find the field to read */
5112	5106	field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf));
5113	5107
5114		− offset = vmcs_field_to_offset(field);
5115		− if (offset < 0)
5116		− return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
	5108	+ if (!evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) {
	5109	+ /*
	5110	+ * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA,
	5111	+ * any VMREAD sets the ALU flags for VMfailInvalid.
	5112	+ */
	5113	+ if (vmx->nested.current_vmptr == INVALID_GPA \|\|
	5114	+ (is_guest_mode(vcpu) &&
	5115	+ get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA))
	5116	+ return nested_vmx_failInvalid(vcpu);
5117	5117
5118		− if (!is_guest_mode(vcpu) && is_vmcs12_ext_field(field))
5119		− copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
	5118	+ offset = get_vmcs12_field_offset(field);
	5119	+ if (offset < 0)
	5120	+ return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
	5121	+
	5122	+ if (!is_guest_mode(vcpu) && is_vmcs12_ext_field(field))
	5123	+ copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
5120	5124
5121		− /* Read the field, zero-extended to a u64 value */
5122		− value = vmcs12_read_any(vmcs12, field, offset);
	5125	+ /* Read the field, zero-extended to a u64 value */
	5126	+ value = vmcs12_read_any(vmcs12, field, offset);
	5127	+ } else {
	5128	+ /*
	5129	+ * Hyper-V TLFS (as of 6.0b) explicitly states, that while an
	5130	+ * enlightened VMCS is active VMREAD/VMWRITE instructions are
	5131	+ * unsupported. Unfortunately, certain versions of Windows 11
	5132	+ * don't comply with this requirement which is not enforced in
	5133	+ * genuine Hyper-V. Allow VMREAD from an enlightened VMCS as a
	5134	+ * workaround, as misbehaving guests will panic on VM-Fail.
	5135	+ * Note, enlightened VMCS is incompatible with shadow VMCS so
	5136	+ * all VMREADs from L2 should go to L1.
	5137	+ */
	5138	+ if (WARN_ON_ONCE(is_guest_mode(vcpu)))
	5139	+ return nested_vmx_failInvalid(vcpu);
	5140	+
	5141	+ offset = evmcs_field_offset(field, NULL);
	5142	+ if (offset < 0)
	5143	+ return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
	5144	+
	5145	+ /* Read the field, zero-extended to a u64 value */
	5146	+ value = evmcs_read_any(vmx->nested.hv_evmcs, field, offset);
	5147	+ }
5123	5148
5124	5149	/*
5125	5150	* Now copy part of this value to register or memory, as requested.
@@ -5214,7 +5239,7 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu) · 88 unmodified lines
5214	5239
5215	5240	field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf));
5216	5241
5217		− offset = vmcs_field_to_offset(field);
	5242	+ offset = get_vmcs12_field_offset(field);
5218	5243	if (offset < 0)
5219	5244	return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
5220	5245
@@ -6462,7 +6487,7 @@ static u64 nested_vmx_calc_vmcs_enum_msr(void) · 1241 unmodified lines
6462	6487	max_idx = 0;
6463	6488	for (i = 0; i < nr_vmcs12_fields; i++) {
6464	6489	/* The vmcs12 table is very, very sparsely populated. */
6465		− if (!vmcs_field_to_offset_table[i])
	6490	+ if (!vmcs12_field_offsets[i])
6466	6491	continue;
6467	6492
6468	6493	idx = vmcs_field_index(VMCS12_IDX_TO_ENC(i));
@@ -6771,6 +6796,7 @@ __init int nested_vmx_hardware_setup(int (exit_handlers[])(struct kvm_vcpu )) · 302 unmodified lines
6771	6796	}
6772	6797
6773	6798	struct kvm_x86_nested_ops vmx_nested_ops = {
	6799	+ .leave_nested = vmx_leave_nested,
6774	6800	.check_events = vmx_check_nested_events,
6775	6801	.hv_timer_pending = nested_vmx_preemption_timer_pending,
6776	6802	.triple_fault = nested_vmx_triple_fault,

arch/x86/kvm/vmx/vmcs12.c+2−2

@@ -8,7 +8,7 @@ · 7 unmodified lines
8	8	FIELD(number, name), \
9	9	[ROL16(number##_HIGH, 6)] = VMCS12_OFFSET(name) + sizeof(u32)
10	10
11		−const unsigned short vmcs_field_to_offset_table[] = {
	11	+const unsigned short vmcs12_field_offsets[] = {
12	12	FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id),
13	13	FIELD(POSTED_INTR_NV, posted_intr_nv),
14	14	FIELD(GUEST_ES_SELECTOR, guest_es_selector),
@@ -151,4 +151,4 @@ const unsigned short vmcs_field_to_offset_table[] = { · 136 unmodified lines
151	151	FIELD(HOST_RSP, host_rsp),
152	152	FIELD(HOST_RIP, host_rip),
153	153	};
154		−const unsigned int nr_vmcs12_fields = ARRAY_SIZE(vmcs_field_to_offset_table);
	154	+const unsigned int nr_vmcs12_fields = ARRAY_SIZE(vmcs12_field_offsets);

arch/x86/kvm/vmx/vmcs12.h+3−3

@@ -361,10 +361,10 @@ static inline void vmx_check_vmcs12_offsets(void) · 360 unmodified lines
361	361	CHECK_OFFSET(guest_pml_index, 996);
362	362	}
363	363
364		−extern const unsigned short vmcs_field_to_offset_table[];
	364	+extern const unsigned short vmcs12_field_offsets[];
365	365	extern const unsigned int nr_vmcs12_fields;
366	366
367		−static inline short vmcs_field_to_offset(unsigned long field)
	367	+static inline short get_vmcs12_field_offset(unsigned long field)
368	368	{
369	369	unsigned short offset;
370	370	unsigned int index;
@@ -377,7 +377,7 @@ static inline short vmcs_field_to_offset(unsigned long field) · 6 unmodified lines
377	377	return -ENOENT;
378	378
379	379	index = array_index_nospec(index, nr_vmcs12_fields);
380		− offset = vmcs_field_to_offset_table[index];
	380	+ offset = vmcs12_field_offsets[index];
381	381	if (offset == 0)
382	382	return -ENOENT;
383	383	return offset;

arch/x86/kvm/vmx/vmx.c+38−9

@@ -1487,11 +1487,12 @@ static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data) · 1486 unmodified lines
1487	1487	return 0;
1488	1488	}
1489	1489
1490		−static bool vmx_can_emulate_instruction(struct kvm_vcpu vcpu, void insn, int insn_len)
	1490	+static bool vmx_can_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
	1491	+ void *insn, int insn_len)
1491	1492	{
1492	1493	/*
1493	1494	* Emulation of instructions in SGX enclaves is impossible as RIP does
1494		− * not point tthe failing instruction, and even if it did, the code
	1495	+ * not point at the failing instruction, and even if it did, the code
1495	1496	* stream is inaccessible. Inject #UD instead of exiting to userspace
1496	1497	* so that guest userspace can't DoS the guest simply by triggering
1497	1498	* emulation (enclaves are CPL3 only).
@@ -2603,7 +2604,6 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf, · 1105 unmodified lines
2603	2604	return -EIO;
2604	2605
2605	2606	vmcs_conf->size = vmx_msr_high & 0x1fff;
2606		− vmcs_conf->order = get_order(vmcs_conf->size);
2607	2607	vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff;
2608	2608
2609	2609	vmcs_conf->revision_id = vmx_msr_low;
@@ -2628,7 +2628,7 @@ struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags) · 18 unmodified lines
2628	2628	struct page *pages;
2629	2629	struct vmcs *vmcs;
2630	2630
2631		− pages = __alloc_pages_node(node, flags, vmcs_config.order);
	2631	+ pages = __alloc_pages_node(node, flags, 0);
2632	2632	if (!pages)
2633	2633	return NULL;
2634	2634	vmcs = page_address(pages);
@@ -2647,7 +2647,7 @@ struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags) · 12 unmodified lines
2647	2647
2648	2648	void free_vmcs(struct vmcs *vmcs)
2649	2649	{
2650		− free_pages((unsigned long)vmcs, vmcs_config.order);
	2650	+ free_page((unsigned long)vmcs);
2651	2651	}
2652	2652
2653	2653	/*
@@ -4094,10 +4094,14 @@ void vmx_set_constant_host_state(struct vcpu_vmx *vmx) · 1440 unmodified lines
4094	4094	vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
4095	4095
4096	4096	/*
4097		− * If 32-bit syscall is enabled, vmx_vcpu_load_vcms rewrites
4098		− * HOST_IA32_SYSENTER_ESP.
	4097	+ * SYSENTER is used for 32-bit system calls on either 32-bit or
	4098	+ * 64-bit kernels. It is always zero If neither is allowed, otherwise
	4099	+ * vmx_vcpu_load_vmcs loads it with the per-CPU entry stack (and may
	4100	+ * have already done so!).
4099	4101	*/
4100		− vmcs_writel(HOST_IA32_SYSENTER_ESP, 0);
	4102	+ if (!IS_ENABLED(CONFIG_IA32_EMULATION) && !IS_ENABLED(CONFIG_X86_32))
	4103	+ vmcs_writel(HOST_IA32_SYSENTER_ESP, 0);
	4104	+
4101	4105	rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl);
4102	4106	vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl); /* 22.2.3 */
4103	4107
@@ -4901,8 +4905,33 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) · 797 unmodified lines
4901	4905	dr6 = vmx_get_exit_qual(vcpu);
4902	4906	if (!(vcpu->guest_debug &
4903	4907	(KVM_GUESTDBG_SINGLESTEP \| KVM_GUESTDBG_USE_HW_BP))) {
	4908	+ /*
	4909	+ * If the #DB was due to ICEBP, a.k.a. INT1, skip the
	4910	+ * instruction. ICEBP generates a trap-like #DB, but
	4911	+ * despite its interception control being tied to #DB,
	4912	+ * is an instruction intercept, i.e. the VM-Exit occurs
	4913	+ * on the ICEBP itself. Note, skipping ICEBP also
	4914	+ * clears STI and MOVSS blocking.
	4915	+ *
	4916	+ * For all other #DBs, set vmcs.PENDING_DBG_EXCEPTIONS.BS
	4917	+ * if single-step is enabled in RFLAGS and STI or MOVSS
	4918	+ * blocking is active, as the CPU doesn't set the bit
	4919	+ * on VM-Exit due to #DB interception. VM-Entry has a
	4920	+ * consistency check that a single-step #DB is pending
	4921	+ * in this scenario as the previous instruction cannot
	4922	+ * have toggled RFLAGS.TF 0=>1 (because STI and POP/MOV
	4923	+ * don't modify RFLAGS), therefore the one instruction
	4924	+ * delay when activating single-step breakpoints must
	4925	+ * have already expired. Note, the CPU sets/clears BS
	4926	+ * as appropriate for all other VM-Exits types.
	4927	+ */
4904	4928	if (is_icebp(intr_info))
4905	4929	WARN_ON(!skip_emulated_instruction(vcpu));
	4930	+ else if ((vmx_get_rflags(vcpu) & X86_EFLAGS_TF) &&
	4931	+ (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
	4932	+ (GUEST_INTR_STATE_STI \| GUEST_INTR_STATE_MOV_SS)))
	4933	+ vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
	4934	+ vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS) \| DR6_BS);
4906	4935
4907	4936	kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
4908	4937	return 1;
@@ -5397,7 +5426,7 @@ static int handle_ept_misconfig(struct kvm_vcpu *vcpu) · 488 unmodified lines
5397	5426	{
5398	5427	gpa_t gpa;
5399	5428
5400		− if (!vmx_can_emulate_instruction(vcpu, NULL, 0))
	5429	+ if (!vmx_can_emulate_instruction(vcpu, EMULTYPE_PF, NULL, 0))
5401	5430	return 1;
5402	5431
5403	5432	/*

arch/x86/kvm/x86.c+81−13

@@ -3535,6 +3535,7 @@ int kvm_set_msr_common(struct kvm_vcpu vcpu, struct msr_data msr_info) · 3534 unmodified lines
3535	3535	if (data & ~supported_xss)
3536	3536	return 1;
3537	3537	vcpu->arch.ia32_xss = data;
	3538	+ kvm_update_cpuid_runtime(vcpu);
3538	3539	break;
3539	3540	case MSR_SMI_COUNT:
3540	3541	if (!msr_info->host_initiated)
@@ -4229,6 +4230,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 688 unmodified lines
4229	4230	case KVM_CAP_SREGS2:
4230	4231	case KVM_CAP_EXIT_ON_EMULATION_FAILURE:
4231	4232	case KVM_CAP_VCPU_ATTRIBUTES:
	4233	+ case KVM_CAP_SYS_ATTRIBUTES:
4232	4234	r = 1;
4233	4235	break;
4234	4236	case KVM_CAP_EXIT_HYPERCALL:
@@ -4331,7 +4333,49 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 96 unmodified lines
4331	4333	break;
4332	4334	}
4333	4335	return r;
	4336	+}
	4337	+
	4338	+static inline void __user kvm_get_attr_addr(struct kvm_device_attr attr)
	4339	+{
	4340	+ void __user uaddr = (void __user)(unsigned long)attr->addr;
4334	4341
	4342	+ if ((u64)(unsigned long)uaddr != attr->addr)
	4343	+ return ERR_PTR(-EFAULT);
	4344	+ return uaddr;
	4345	+}
	4346	+
	4347	+static int kvm_x86_dev_get_attr(struct kvm_device_attr *attr)
	4348	+{
	4349	+ u64 __user *uaddr = kvm_get_attr_addr(attr);
	4350	+
	4351	+ if (attr->group)
	4352	+ return -ENXIO;
	4353	+
	4354	+ if (IS_ERR(uaddr))
	4355	+ return PTR_ERR(uaddr);
	4356	+
	4357	+ switch (attr->attr) {
	4358	+ case KVM_X86_XCOMP_GUEST_SUPP:
	4359	+ if (put_user(supported_xcr0, uaddr))
	4360	+ return -EFAULT;
	4361	+ return 0;
	4362	+ default:
	4363	+ return -ENXIO;
	4364	+ break;
	4365	+ }
	4366	+}
	4367	+
	4368	+static int kvm_x86_dev_has_attr(struct kvm_device_attr *attr)
	4369	+{
	4370	+ if (attr->group)
	4371	+ return -ENXIO;
	4372	+
	4373	+ switch (attr->attr) {
	4374	+ case KVM_X86_XCOMP_GUEST_SUPP:
	4375	+ return 0;
	4376	+ default:
	4377	+ return -ENXIO;
	4378	+ }
4335	4379	}
4336	4380
4337	4381	long kvm_arch_dev_ioctl(struct file *filp,
@@ -4422,6 +4466,22 @@ long kvm_arch_dev_ioctl(struct file *filp, · 84 unmodified lines
4422	4466	case KVM_GET_SUPPORTED_HV_CPUID:
4423	4467	r = kvm_ioctl_get_supported_hv_cpuid(NULL, argp);
4424	4468	break;
	4469	+ case KVM_GET_DEVICE_ATTR: {
	4470	+ struct kvm_device_attr attr;
	4471	+ r = -EFAULT;
	4472	+ if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
	4473	+ break;
	4474	+ r = kvm_x86_dev_get_attr(&attr);
	4475	+ break;
	4476	+ }
	4477	+ case KVM_HAS_DEVICE_ATTR: {
	4478	+ struct kvm_device_attr attr;
	4479	+ r = -EFAULT;
	4480	+ if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
	4481	+ break;
	4482	+ r = kvm_x86_dev_has_attr(&attr);
	4483	+ break;
	4484	+ }
4425	4485	default:
4426	4486	r = -EINVAL;
4427	4487	break;
@@ -4860,8 +4920,10 @@ static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, · 432 unmodified lines
4860	4920	vcpu->arch.apic->sipi_vector = events->sipi_vector;
4861	4921
4862	4922	if (events->flags & KVM_VCPUEVENT_VALID_SMM) {
4863		− if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm)
	4923	+ if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {
	4924	+ kvm_x86_ops.nested_ops->leave_nested(vcpu);
4864	4925	kvm_smm_changed(vcpu, events->smi.smm);
	4926	+ }
4865	4927
4866	4928	vcpu->arch.smi_pending = events->smi.pending;
4867	4929
@@ -5022,11 +5084,11 @@ static int kvm_arch_tsc_has_attr(struct kvm_vcpu *vcpu, · 154 unmodified lines
5022	5084	static int kvm_arch_tsc_get_attr(struct kvm_vcpu *vcpu,
5023	5085	struct kvm_device_attr *attr)
5024	5086	{
5025		− u64 __user uaddr = (u64 __user )(unsigned long)attr->addr;
	5087	+ u64 __user *uaddr = kvm_get_attr_addr(attr);
5026	5088	int r;
5027	5089
5028		− if ((u64)(unsigned long)uaddr != attr->addr)
5029		− return -EFAULT;
	5090	+ if (IS_ERR(uaddr))
	5091	+ return PTR_ERR(uaddr);
5030	5092
5031	5093	switch (attr->attr) {
5032	5094	case KVM_VCPU_TSC_OFFSET:
@@ -5045,12 +5107,12 @@ static int kvm_arch_tsc_get_attr(struct kvm_vcpu *vcpu, · 12 unmodified lines
5045	5107	static int kvm_arch_tsc_set_attr(struct kvm_vcpu *vcpu,
5046	5108	struct kvm_device_attr *attr)
5047	5109	{
5048		− u64 __user uaddr = (u64 __user )(unsigned long)attr->addr;
	5110	+ u64 __user *uaddr = kvm_get_attr_addr(attr);
5049	5111	struct kvm *kvm = vcpu->kvm;
5050	5112	int r;
5051	5113
5052		− if ((u64)(unsigned long)uaddr != attr->addr)
5053		− return -EFAULT;
	5114	+ if (IS_ERR(uaddr))
	5115	+ return PTR_ERR(uaddr);
5054	5116
5055	5117	switch (attr->attr) {
5056	5118	case KVM_VCPU_TSC_OFFSET: {
@@ -6810,6 +6872,13 @@ int kvm_write_guest_virt_system(struct kvm_vcpu vcpu, gva_t addr, void val, · 1753 unmodified lines
6810	6872	}
6811	6873	EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
6812	6874
	6875	+static int kvm_can_emulate_insn(struct kvm_vcpu *vcpu, int emul_type,
	6876	+ void *insn, int insn_len)
	6877	+{
	6878	+ return static_call(kvm_x86_can_emulate_instruction)(vcpu, emul_type,
	6879	+ insn, insn_len);
	6880	+}
	6881	+
6813	6882	int handle_ud(struct kvm_vcpu *vcpu)
6814	6883	{
6815	6884	static const char kvm_emulate_prefix[] = { __KVM_EMULATE_PREFIX };
@@ -6817,7 +6886,7 @@ int handle_ud(struct kvm_vcpu *vcpu) · 1 unmodified line
6817	6886	char sig[5]; /* ud2; .ascii "kvm" */
6818	6887	struct x86_exception e;
6819	6888
6820		− if (unlikely(!static_call(kvm_x86_can_emulate_instruction)(vcpu, NULL, 0)))
	6889	+ if (unlikely(!kvm_can_emulate_insn(vcpu, emul_type, NULL, 0)))
6821	6890	return 1;
6822	6891
6823	6892	if (force_emulation_prefix &&
@@ -8193,7 +8262,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, · 1369 unmodified lines
8193	8262	bool writeback = true;
8194	8263	bool write_fault_to_spt;
8195	8264
8196		− if (unlikely(!static_call(kvm_x86_can_emulate_instruction)(vcpu, insn, insn_len)))
	8265	+ if (unlikely(!kvm_can_emulate_insn(vcpu, emulation_type, insn, insn_len)))
8197	8266	return 1;
8198	8267
8199	8268	vcpu->arch.l1tf_flush_l1d = true;
@@ -9706,7 +9775,7 @@ void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, · 1506 unmodified lines
9706	9775	kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD);
9707	9776	}
9708	9777
9709		−void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
	9778	+static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
9710	9779	{
9711	9780	if (!lapic_in_kernel(vcpu))
9712	9781	return;
@@ -11209,7 +11278,8 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) · 1496 unmodified lines
11209	11278
11210	11279	vcpu->arch.msr_misc_features_enables = 0;
11211	11280
11212		− vcpu->arch.xcr0 = XFEATURE_MASK_FP;
	11281	+ __kvm_set_xcr(vcpu, 0, XFEATURE_MASK_FP);
	11282	+ __kvm_set_msr(vcpu, MSR_IA32_XSS, 0, true);
11213	11283	}
11214	11284
11215	11285	/* All GPRs except RDX (handled below) are zeroed on RESET/INIT. */
@@ -11226,8 +11296,6 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) · 10 unmodified lines
11226	11296	cpuid_0x1 = kvm_find_cpuid_entry(vcpu, 1, 0);
11227	11297	kvm_rdx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600);
11228	11298
11229		− vcpu->arch.ia32_xss = 0;
11230		−
11231	11299	static_call(kvm_x86_vcpu_reset)(vcpu, init_event);
11232	11300
11233	11301	kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);

arch/x86/kvm/xen.c+2−8

@@ -316,10 +316,7 @@ int __kvm_xen_has_interrupt(struct kvm_vcpu *v) · 315 unmodified lines
316	316	"\tnotq %0\n"
317	317	"\t" LOCK_PREFIX "andq %0, %2\n"
318	318	"2:\n"
319		− "\t.section .fixup,\"ax\"\n"
320		− "3:\tjmp\t2b\n"
321		− "\t.previous\n"
322		− _ASM_EXTABLE_UA(1b, 3b)
	319	+ _ASM_EXTABLE_UA(1b, 2b)
323	320	: "=r" (evtchn_pending_sel),
324	321	"+m" (vi->evtchn_pending_sel),
325	322	"+m" (v->arch.xen.evtchn_pending_sel)
@@ -335,10 +332,7 @@ int __kvm_xen_has_interrupt(struct kvm_vcpu *v) · 9 unmodified lines
335	332	"\tnotl %0\n"
336	333	"\t" LOCK_PREFIX "andl %0, %2\n"
337	334	"2:\n"
338		− "\t.section .fixup,\"ax\"\n"
339		− "3:\tjmp\t2b\n"
340		− "\t.previous\n"
341		− _ASM_EXTABLE_UA(1b, 3b)
	335	+ _ASM_EXTABLE_UA(1b, 2b)
342	336	: "=r" (evtchn_pending_sel32),
343	337	"+m" (vi->evtchn_pending_sel),
344	338	"+m" (v->arch.xen.evtchn_pending_sel)

include/uapi/linux/kvm.h+1−0

@@ -1133,6 +1133,7 @@ struct kvm_ppc_resize_hpt { · 1132 unmodified lines
1133	1133	#define KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM 206
1134	1134	#define KVM_CAP_VM_GPA_BITS 207
1135	1135	#define KVM_CAP_XSAVE2 208
	1136	+#define KVM_CAP_SYS_ATTRIBUTES 209
1136	1137
1137	1138	#ifdef KVM_CAP_IRQ_ROUTING
1138	1139

tools/arch/x86/include/uapi/asm/kvm.h+3−0

@@ -452,6 +452,9 @@ struct kvm_sync_regs { · 451 unmodified lines
452	452
453	453	#define KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE 0x00000001
454	454
	455	+/* attributes for system fd (group 0) */
	456	+#define KVM_X86_XCOMP_GUEST_SUPP 0
	457	+
455	458	struct kvm_vmx_nested_state_data {
456	459	__u8 vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE];
457	460	__u8 shadow_vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE];

tools/include/uapi/linux/kvm.h+1−0

@@ -1133,6 +1133,7 @@ struct kvm_ppc_resize_hpt { · 1132 unmodified lines
1133	1133	#define KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM 206
1134	1134	#define KVM_CAP_VM_GPA_BITS 207
1135	1135	#define KVM_CAP_XSAVE2 208
	1136	+#define KVM_CAP_SYS_ATTRIBUTES 209
1136	1137
1137	1138	#ifdef KVM_CAP_IRQ_ROUTING
1138	1139

tools/testing/selftests/kvm/Makefile+1−0

@@ -85,6 +85,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/xen_vmcall_test · 84 unmodified lines
85	85	TEST_GEN_PROGS_x86_64 += x86_64/vmx_pi_mmio_test
86	86	TEST_GEN_PROGS_x86_64 += x86_64/sev_migrate_tests
87	87	TEST_GEN_PROGS_x86_64 += x86_64/amx_test
	88	+TEST_GEN_PROGS_x86_64 += access_tracking_perf_test
88	89	TEST_GEN_PROGS_x86_64 += demand_paging_test
89	90	TEST_GEN_PROGS_x86_64 += dirty_log_test
90	91	TEST_GEN_PROGS_x86_64 += dirty_log_perf_test

tools/testing/selftests/kvm/include/kvm_util_base.h+0−1

@@ -345,7 +345,6 @@ struct kvm_vm *vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus, · 344 unmodified lines
345	345	* guest_code - The vCPU's entry point
346	346	*/
347	347	void vm_vcpu_add_default(struct kvm_vm vm, uint32_t vcpuid, void guest_code);
348		−void vm_xsave_req_perm(void);
349	348
350	349	bool vm_is_unrestricted_guest(struct kvm_vm *vm);
351	350

tools/testing/selftests/kvm/include/x86_64/processor.h+1−0

@@ -458,6 +458,7 @@ uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2, · 457 unmodified lines
458	458	struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(void);
459	459	void vcpu_set_hv_cpuid(struct kvm_vm *vm, uint32_t vcpuid);
460	460	struct kvm_cpuid2 vcpu_get_supported_hv_cpuid(struct kvm_vm vm, uint32_t vcpuid);
	461	+void vm_xsave_req_perm(int bit);
461	462
462	463	enum x86_page_size {
463	464	X86_PAGE_SIZE_4K = 0,

tools/testing/selftests/kvm/lib/kvm_util.c+0−7

@@ -393,13 +393,6 @@ struct kvm_vm *vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus, · 392 unmodified lines
393	393	struct kvm_vm *vm;
394	394	int i;
395	395
396		−#ifdef __x86_64__
397		− /*
398		− * Permission needs to be requested before KVM_SET_CPUID2.
399		− */
400		− vm_xsave_req_perm();
401		−#endif
402		−
403	396	/* Force slot0 memory size not small than DEFAULT_GUEST_PHY_PAGES */
404	397	if (slot0_mem_pages < DEFAULT_GUEST_PHY_PAGES)
405	398	slot0_mem_pages = DEFAULT_GUEST_PHY_PAGES;

tools/testing/selftests/kvm/lib/x86_64/processor.c+21−6

@@ -665,16 +665,31 @@ static bool is_xfd_supported(void) · 664 unmodified lines
665	665	return !!(eax & CPUID_XFD_BIT);
666	666	}
667	667
668		−void vm_xsave_req_perm(void)
	668	+void vm_xsave_req_perm(int bit)
669	669	{
670		− unsigned long bitmask;
	670	+ int kvm_fd;
	671	+ u64 bitmask;
671	672	long rc;
	673	+ struct kvm_device_attr attr = {
	674	+ .group = 0,
	675	+ .attr = KVM_X86_XCOMP_GUEST_SUPP,
	676	+ .addr = (unsigned long) &bitmask
	677	+ };
	678	+
	679	+ kvm_fd = open_kvm_dev_path_or_exit();
	680	+ rc = ioctl(kvm_fd, KVM_GET_DEVICE_ATTR, &attr);
	681	+ close(kvm_fd);
	682	+ if (rc == -1 && (errno == ENXIO \|\| errno == EINVAL))
	683	+ exit(KSFT_SKIP);
	684	+ TEST_ASSERT(rc == 0, "KVM_GET_DEVICE_ATTR(0, KVM_X86_XCOMP_GUEST_SUPP) error: %ld", rc);
	685	+ if (!(bitmask & (1ULL << bit)))
	686	+ exit(KSFT_SKIP);
672	687
673	688	if (!is_xfd_supported())
674		− return;
	689	+ exit(KSFT_SKIP);
	690	+
	691	+ rc = syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_GUEST_PERM, bit);
675	692
676		− rc = syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_GUEST_PERM,
677		− XSTATE_XTILE_DATA_BIT);
678	693	/*
679	694	* The older kernel version(<5.15) can't support
680	695	* ARCH_REQ_XCOMP_GUEST_PERM and directly return.
@@ -684,7 +699,7 @@ void vm_xsave_req_perm(void) · 3 unmodified lines
684	699
685	700	rc = syscall(SYS_arch_prctl, ARCH_GET_XCOMP_GUEST_PERM, &bitmask);
686	701	TEST_ASSERT(rc == 0, "prctl(ARCH_GET_XCOMP_GUEST_PERM) error: %ld", rc);
687		− TEST_ASSERT(bitmask & XFEATURE_XTILE_MASK,
	702	+ TEST_ASSERT(bitmask & (1ULL << bit),
688	703	"prctl(ARCH_REQ_XCOMP_GUEST_PERM) failure bitmask=0x%lx",
689	704	bitmask);
690	705	}

tools/testing/selftests/kvm/x86_64/amx_test.c+2−0

@@ -329,6 +329,8 @@ int main(int argc, char *argv[]) · 328 unmodified lines
329	329	u32 amx_offset;
330	330	int stage, ret;
331	331
	332	+ vm_xsave_req_perm(XSTATE_XTILE_DATA_BIT);
	333	+
332	334	/* Create VM */
333	335	vm = vm_create_default(VCPU_ID, 0, guest_code);
334	336

tools/testing/selftests/kvm/x86_64/smm_test.c+0−1

@@ -105,7 +105,6 @@ static void guest_code(void *arg) · 104 unmodified lines
105	105
106	106	if (cpu_has_svm()) {
107	107	run_guest(svm->vmcb, svm->vmcb_gpa);
108		− svm->vmcb->save.rip += 3;
109	108	run_guest(svm->vmcb, svm->vmcb_gpa);
110	109	} else {
111	110	vmlaunch();

virt/kvm/eventfd.c+4−4

@@ -463,8 +463,8 @@ bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin) · 462 unmodified lines
463	463	idx = srcu_read_lock(&kvm->irq_srcu);
464	464	gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);
465	465	if (gsi != -1)
466		− hlist_for_each_entry_rcu(kian, &kvm->irq_ack_notifier_list,
467		− link)
	466	+ hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list,
	467	+ link, srcu_read_lock_held(&kvm->irq_srcu))
468	468	if (kian->gsi == gsi) {
469	469	srcu_read_unlock(&kvm->irq_srcu, idx);
470	470	return true;
@@ -480,8 +480,8 @@ void kvm_notify_acked_gsi(struct kvm *kvm, int gsi) · 9 unmodified lines
480	480	{
481	481	struct kvm_irq_ack_notifier *kian;
482	482
483		− hlist_for_each_entry_rcu(kian, &kvm->irq_ack_notifier_list,
484		− link)
	483	+ hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list,
	484	+ link, srcu_read_lock_held(&kvm->irq_srcu))
485	485	if (kian->gsi == gsi)
486	486	kian->irq_acked(kian);
487	487	}

virt/kvm/kvm_main.c+3−5

@@ -2248,7 +2248,6 @@ struct kvm_memory_slot kvm_vcpu_gfn_to_memslot(struct kvm_vcpu vcpu, gfn_t gfn · 2247 unmodified lines
2248	2248
2249	2249	return NULL;
2250	2250	}
2251		−EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_memslot);
2252	2251
2253	2252	bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
2254	2253	{
@@ -2463,9 +2462,8 @@ static int kvm_try_get_pfn(kvm_pfn_t pfn) · 208 unmodified lines
2463	2462	}
2464	2463
2465	2464	static int hva_to_pfn_remapped(struct vm_area_struct *vma,
2466		− unsigned long addr, bool *async,
2467		− bool write_fault, bool *writable,
2468		− kvm_pfn_t *p_pfn)
	2465	+ unsigned long addr, bool write_fault,
	2466	+ bool writable, kvm_pfn_t p_pfn)
2469	2467	{
2470	2468	kvm_pfn_t pfn;
2471	2469	pte_t *ptep;
@@ -2575,7 +2573,7 @@ retry: · 103 unmodified lines
2575	2573	if (vma == NULL)
2576	2574	pfn = KVM_PFN_ERR_FAULT;
2577	2575	else if (vma->vm_flags & (VM_IO \| VM_PFNMAP)) {
2578		− r = hva_to_pfn_remapped(vma, addr, async, write_fault, writable, &pfn);
	2576	+ r = hva_to_pfn_remapped(vma, addr, write_fault, writable, &pfn);
2579	2577	if (r == -EAGAIN)
2580	2578	goto retry;
2581	2579	if (r < 0)

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code